confluence_web/tests/integration/test_crash_reconcile.py

"""Backend crash while an agent runs; restart reconciles only application-owned containers."""

from __future__ import annotations

import json
import os
import signal
import socket
import subprocess
import sys
import threading
import time
import urllib.error
import urllib.request
import uuid
from pathlib import Path

import pytest

from tests.integration.conftest import APPROVED_ORIGIN, CANARY_PAT, REPO_ROOT, requires_runtime_image

pytestmark = [pytest.mark.integration, requires_runtime_image()]

LABEL_KEY = "com.confluence_web.app"


def docker(*args: str) -> str:
    return subprocess.run(["docker", *args], capture_output=True, text=True, check=True, timeout=60).stdout.strip()


def app_containers(label_value: str) -> list[str]:
    out = docker("ps", "-aq", "--filter", f"label={LABEL_KEY}={label_value}")
    return [line for line in out.splitlines() if line]


def free_port() -> int:
    with socket.socket() as s:
        s.bind(("127.0.0.1", 0))
        return s.getsockname()[1]


def start_server(env: dict) -> subprocess.Popen:
    proc = subprocess.Popen([sys.executable, str(REPO_ROOT / "tests/integration/crash_server.py")], env=env, cwd=REPO_ROOT)
    deadline = time.time() + 60
    while time.time() < deadline:
        if proc.poll() is not None:
            raise AssertionError(f"server exited early with {proc.returncode}")
        try:
            with urllib.request.urlopen(f"http://127.0.0.1:{env['CW_DEV_PORT']}/", timeout=1) as r:
                if r.status == 200:
                    return proc
        except Exception:
            time.sleep(0.3)
    proc.kill()
    raise AssertionError("server did not become ready")


def test_backend_crash_then_restart_reconciles_only_owned_containers(tmp_path: Path):
    label_value = f"integration-crash-{uuid.uuid4().hex[:8]}"
    port = free_port()
    artifact_dir = tmp_path / "artifacts"
    (artifact_dir / "committed").mkdir(parents=True)
    (artifact_dir / "staging").mkdir()
    (artifact_dir / "committed" / "stale-artifact.bin").write_bytes(b"old retained bytes")
    (artifact_dir / "staging" / "stale-staging.bin").write_bytes(b"old staged bytes")
    env = {"PATH": os.environ["PATH"], "PYTHONPATH": str(REPO_ROOT), "CW_LABEL_VALUE": label_value, "CW_ARTIFACT_DIR": str(artifact_dir), "CW_DEV_PORT": str(port)}
    if "DOCKER_HOST" in os.environ:
        env["DOCKER_HOST"] = os.environ["DOCKER_HOST"]

    unrelated = docker("run", "-d", "--rm", "--network", "none", "--name", f"cw-unrelated-{label_value[-8:]}", "alpine:3.20", "sleep", "600")
    stale = docker("run", "-d", "--network", "none", "--label", f"{LABEL_KEY}={label_value}", "--label", f"{LABEL_KEY}.query_id=stale", "alpine:3.20", "sleep", "600")
    server = None
    try:
        # 1. Startup reconciliation removes the stale owned container and purges old artifacts.
        server = start_server(env)
        assert stale not in docker("ps", "-aq", "--no-trunc")
        assert unrelated in docker("ps", "-aq", "--no-trunc"), "reconciliation must not touch unrelated containers"
        assert not any(p.name in ("stale-artifact.bin", "stale-staging.bin") for p in artifact_dir.rglob("*"))

        # 2. Start a query whose model stalls, so a real runtime container is alive.
        body = json.dumps({"prompt": "stall", "credentials": {"url": APPROVED_ORIGIN, "pat": CANARY_PAT}}).encode()
        req = urllib.request.Request(f"http://127.0.0.1:{port}/api/v1/query", data=body, method="POST", headers={"Content-Type": "application/json", "Origin": f"http://127.0.0.1:{port}"})
        outcome: dict = {}

        def post():
            try:
                with urllib.request.urlopen(req, timeout=300) as r:
                    outcome["status"] = r.status
            except urllib.error.HTTPError as e:
                outcome["status"] = e.code
            except Exception as e:  # connection dropped by the crash
                outcome["error"] = type(e).__name__

        t = threading.Thread(target=post, daemon=True)
        t.start()
        deadline = time.time() + 60
        while time.time() < deadline and not app_containers(label_value):
            time.sleep(0.5)
        running = app_containers(label_value)
        assert running, "runtime container must be running during the stalled query"

        # 3. Crash the backend (SIGKILL: no cleanup code runs).
        server.send_signal(signal.SIGKILL)
        server.wait(timeout=10)
        server = None
        t.join(timeout=30)
        assert "error" in outcome or outcome.get("status", 0) >= 500

        # 4. Restart: reconciliation removes owned leftovers, keeps the unrelated container.
        server = start_server(env)
        deadline = time.time() + 30
        while time.time() < deadline and app_containers(label_value):
            time.sleep(0.5)
        assert app_containers(label_value) == []
        assert unrelated in docker("ps", "-aq", "--no-trunc")

        # 5. The restarted backend serves requests again; a missing Origin is refused with the contract code.
        probe = json.dumps({"url": APPROVED_ORIGIN, "pat": CANARY_PAT}).encode()
        with pytest.raises(urllib.error.HTTPError) as excinfo:
            urllib.request.urlopen(urllib.request.Request(f"http://127.0.0.1:{port}/api/v1/auth/verify", data=probe, method="POST", headers={"Content-Type": "application/json"}), timeout=5)
        assert excinfo.value.code == 403
        assert json.loads(excinfo.value.read())["error"]["code"] == "origin_denied"
    finally:
        if server is not None:
            server.terminate()
            try:
                server.wait(timeout=10)
            except subprocess.TimeoutExpired:
                server.kill()
        subprocess.run(["docker", "rm", "-f", unrelated, stale], capture_output=True)
        for cid in app_containers(label_value):
            subprocess.run(["docker", "rm", "-f", cid], capture_output=True)