"""Backend crash while an agent runs; restart reconciles only application-owned containers.""" from __future__ import annotations import json import os import signal import socket import subprocess import sys import threading import time import urllib.error import urllib.request import uuid from pathlib import Path import pytest from tests.integration.conftest import APPROVED_ORIGIN, CANARY_PAT, REPO_ROOT, requires_runtime_image pytestmark = [pytest.mark.integration, requires_runtime_image()] LABEL_KEY = "com.confluence_web.app" def docker(*args: str) -> str: return subprocess.run(["docker", *args], capture_output=True, text=True, check=True, timeout=60).stdout.strip() def app_containers(label_value: str) -> list[str]: out = docker("ps", "-aq", "--filter", f"label={LABEL_KEY}={label_value}") return [line for line in out.splitlines() if line] def free_port() -> int: with socket.socket() as s: s.bind(("127.0.0.1", 0)) return s.getsockname()[1] def start_server(env: dict) -> subprocess.Popen: proc = subprocess.Popen([sys.executable, str(REPO_ROOT / "tests/integration/crash_server.py")], env=env, cwd=REPO_ROOT) deadline = time.time() + 60 while time.time() < deadline: if proc.poll() is not None: raise AssertionError(f"server exited early with {proc.returncode}") try: with urllib.request.urlopen(f"http://127.0.0.1:{env['CW_DEV_PORT']}/", timeout=1) as r: if r.status == 200: return proc except Exception: time.sleep(0.3) proc.kill() raise AssertionError("server did not become ready") def test_backend_crash_then_restart_reconciles_only_owned_containers(tmp_path: Path): label_value = f"integration-crash-{uuid.uuid4().hex[:8]}" port = free_port() artifact_dir = tmp_path / "artifacts" (artifact_dir / "committed").mkdir(parents=True) (artifact_dir / "staging").mkdir() (artifact_dir / "committed" / "stale-artifact.bin").write_bytes(b"old retained bytes") (artifact_dir / "staging" / "stale-staging.bin").write_bytes(b"old staged bytes") env = {"PATH": os.environ["PATH"], "PYTHONPATH": str(REPO_ROOT), "CW_LABEL_VALUE": label_value, "CW_ARTIFACT_DIR": str(artifact_dir), "CW_DEV_PORT": str(port)} if "DOCKER_HOST" in os.environ: env["DOCKER_HOST"] = os.environ["DOCKER_HOST"] unrelated = docker("run", "-d", "--rm", "--network", "none", "--name", f"cw-unrelated-{label_value[-8:]}", "alpine:3.20", "sleep", "600") stale = docker("run", "-d", "--network", "none", "--label", f"{LABEL_KEY}={label_value}", "--label", f"{LABEL_KEY}.query_id=stale", "alpine:3.20", "sleep", "600") server = None try: # 1. Startup reconciliation removes the stale owned container and purges old artifacts. server = start_server(env) assert stale not in docker("ps", "-aq", "--no-trunc") assert unrelated in docker("ps", "-aq", "--no-trunc"), "reconciliation must not touch unrelated containers" assert not any(p.name in ("stale-artifact.bin", "stale-staging.bin") for p in artifact_dir.rglob("*")) # 2. Start a query whose model stalls, so a real runtime container is alive. body = json.dumps({"prompt": "stall", "credentials": {"url": APPROVED_ORIGIN, "pat": CANARY_PAT}}).encode() req = urllib.request.Request(f"http://127.0.0.1:{port}/api/v1/query", data=body, method="POST", headers={"Content-Type": "application/json", "Origin": f"http://127.0.0.1:{port}"}) outcome: dict = {} def post(): try: with urllib.request.urlopen(req, timeout=300) as r: outcome["status"] = r.status except urllib.error.HTTPError as e: outcome["status"] = e.code except Exception as e: # connection dropped by the crash outcome["error"] = type(e).__name__ t = threading.Thread(target=post, daemon=True) t.start() deadline = time.time() + 60 while time.time() < deadline and not app_containers(label_value): time.sleep(0.5) running = app_containers(label_value) assert running, "runtime container must be running during the stalled query" # 3. Crash the backend (SIGKILL: no cleanup code runs). server.send_signal(signal.SIGKILL) server.wait(timeout=10) server = None t.join(timeout=30) assert "error" in outcome or outcome.get("status", 0) >= 500 # 4. Restart: reconciliation removes owned leftovers, keeps the unrelated container. server = start_server(env) deadline = time.time() + 30 while time.time() < deadline and app_containers(label_value): time.sleep(0.5) assert app_containers(label_value) == [] assert unrelated in docker("ps", "-aq", "--no-trunc") # 5. The restarted backend serves requests again; a missing Origin is refused with the contract code. probe = json.dumps({"url": APPROVED_ORIGIN, "pat": CANARY_PAT}).encode() with pytest.raises(urllib.error.HTTPError) as excinfo: urllib.request.urlopen(urllib.request.Request(f"http://127.0.0.1:{port}/api/v1/auth/verify", data=probe, method="POST", headers={"Content-Type": "application/json"}), timeout=5) assert excinfo.value.code == 403 assert json.loads(excinfo.value.read())["error"]["code"] == "origin_denied" finally: if server is not None: server.terminate() try: server.wait(timeout=10) except subprocess.TimeoutExpired: server.kill() subprocess.run(["docker", "rm", "-f", unrelated, stale], capture_output=True) for cid in app_containers(label_value): subprocess.run(["docker", "rm", "-f", cid], capture_output=True)