"""Rootless Docker container lifecycle, isolation, cleanup, and reconciliation.""" from __future__ import annotations import abc import asyncio import json import logging import os import shutil import time from typing import Any, Dict, List, Optional, Tuple from backend.errors import CleanupFailedError, ExecutionFailedError from backend.settings import Settings from backend.transport import GLOBAL_MAX_FRAME_BYTES logger = logging.getLogger(__name__) class ContainerHandle: """Represents an active attached container process.""" def __init__( self, container_id: str, stdin: asyncio.StreamWriter, stdout: asyncio.StreamReader, stderr: Optional[asyncio.StreamReader] = None, process: Optional[asyncio.subprocess.Process] = None, ): self.container_id = container_id self.stdin = stdin self.stdout = stdout self.stderr = stderr self.process = process async def wait(self) -> int: if self.process: return await self.process.wait() return 0 class ContainerManager(abc.ABC): """Abstract container lifecycle manager.""" @abc.abstractmethod async def verify_rootless(self) -> None: """Verify daemon is rootless and required isolation is supported.""" raise NotImplementedError @abc.abstractmethod async def create_and_run(self, query_id: str) -> ContainerHandle: """Create, configure, and launch isolated container, returning attached streams.""" raise NotImplementedError @abc.abstractmethod async def kill_and_remove(self, container_id: str, timeout: float = 10.0) -> None: """Kill and remove container, confirming its deletion within timeout.""" raise NotImplementedError @abc.abstractmethod async def reconcile_orphans(self, max_age_seconds: float = 200.0) -> int: """Find and remove residual application containers older than max_age_seconds.""" raise NotImplementedError class DockerContainerManager(ContainerManager): """Rootless Docker container lifecycle implementation.""" def __init__(self, settings: Settings): self.settings = settings self.docker_bin = shutil.which("docker") or "docker" # Clean minimal environment for Docker CLI subprocesses, avoiding secret leaks self._env = { "PATH": os.environ.get("PATH", "/usr/local/bin:/usr/bin:/bin"), "HOME": os.environ.get("HOME", "/root"), } self._processes: Dict[str, asyncio.subprocess.Process] = {} for key in ("XDG_RUNTIME_DIR", "DOCKER_CONTEXT", "DOCKER_CONFIG"): if key in os.environ: self._env[key] = os.environ[key] if settings.docker_host: self._env["DOCKER_HOST"] = settings.docker_host elif "DOCKER_HOST" in os.environ: self._env["DOCKER_HOST"] = os.environ["DOCKER_HOST"] async def _exec_docker(self, args: List[str], timeout: float = 15.0) -> Tuple[int, str, str]: cmd = [self.docker_bin] + args proc = await asyncio.create_subprocess_exec( *cmd, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE, env=self._env, ) try: stdout_bytes, stderr_bytes = await asyncio.wait_for( proc.communicate(), timeout=timeout ) return ( proc.returncode or 0, stdout_bytes.decode("utf-8", errors="replace"), stderr_bytes.decode("utf-8", errors="replace"), ) except BaseException: if proc.returncode is None: proc.kill() await proc.communicate() raise async def verify_rootless(self) -> None: """Verify Docker daemon is running in rootless mode and enforces cgroups.""" code, out, err = await self._exec_docker(["info", "--format", "{{json .}}"]) if code != 0: raise ExecutionFailedError(f"Failed to query Docker daemon: {err.strip()}") try: info = json.loads(out) except Exception as e: raise ExecutionFailedError("Failed to parse Docker info JSON") from e sec_opts = info.get("SecurityOptions", []) is_rootless = any(opt in ("name=rootless", "rootless") for opt in sec_opts) if not is_rootless: raise ExecutionFailedError( "Docker daemon is not running in rootless mode! Rootless mode is required." ) required = ("MemoryLimit", "PidsLimit", "CpuCfsQuota", "CpuCfsPeriod") if info.get("CgroupVersion") != "2" or info.get("CgroupDriver") in (None, "none", "") or any(info.get(k) is not True for k in required): raise ExecutionFailedError("Docker must enforce memory, CPU and PID limits with cgroup v2") async def create_and_run(self, query_id: str) -> ContainerHandle: """Launch isolated container attached via pipes.""" created_at = str(int(time.time())) container_name = f"cw-{query_id}" label_app = f"{self.settings.container_label_key}={self.settings.container_label_value}" label_qid = f"{self.settings.container_label_key}.query_id={query_id}" label_time = f"{self.settings.container_label_key}.created_at={created_at}" # Build isolated docker run command with structured args (no shell) cmd = [ self.docker_bin, "run", "--name", container_name, "-i", # interactive attached stdin # No --init: the image's Python supervisor must remain namespace PID 1 # (subreaper, signal immunity, independent deadline: 900 s maximum, 180 s without a start frame). "--rm", # automatically remove on exit if clean "--read-only", "--network", "none", "--user", "10001:10001", "-w", "/work", "-e", "HOME=/home/agent", "-e", "LANG=C.UTF-8", "-e", f"CONFLUENCE_WEB_MAX_DEADLINE_SECONDS={int(self.settings.max_deadline_seconds)}", "--tmpfs", "/work:rw,nosuid,nodev,size=256m,uid=10001,gid=10001", "--tmpfs", "/tmp:rw,nosuid,nodev,size=64m,uid=10001,gid=10001", "--tmpfs", "/home/agent:rw,nosuid,nodev,size=32m,uid=10001,gid=10001", "--memory", "1g", "--memory-swap", "1g", # no additional swap beyond the memory limit "--cpus", "1.0", "--pids-limit", "128", "--cap-drop", "ALL", "--security-opt", "no-new-privileges", "--log-driver", "none", "--label", label_app, "--label", label_qid, "--label", label_time, "--label", f"{self.settings.container_label_key}.instance_id={self.settings.container_instance_id}", self.settings.runtime_image, ] try: # Configure high limit so multi-MiB bridge frames do not raise LimitOverrunError proc = await asyncio.create_subprocess_exec( *cmd, stdin=asyncio.subprocess.PIPE, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE, limit=GLOBAL_MAX_FRAME_BYTES, env=self._env, ) except Exception as e: raise ExecutionFailedError(f"Failed to spawn container process: {e}") from e self._processes[query_id] = proc return ContainerHandle( container_id=query_id, stdin=proc.stdin, # type: ignore stdout=proc.stdout, # type: ignore stderr=proc.stderr, # type: ignore process=proc, ) async def _find_container_id(self, query_id: str, timeout: float = 3.0) -> Optional[str]: deadline = time.monotonic() + timeout cname = query_id if query_id.startswith("cw-") else f"cw-{query_id}" filters = [f"name=^/{cname}$"] if len(query_id) >= 12 and all(c in "0123456789abcdef" for c in query_id): filters.append(f"id={query_id}") for selection in filters: remaining = deadline - time.monotonic() if remaining <= 0: raise CleanupFailedError("Docker discovery deadline exceeded") code, out, _ = await self._exec_docker(["ps", "-a", "-q", "--filter", selection], timeout=remaining) if code != 0: raise CleanupFailedError("Docker daemon could not confirm container state") if out.strip(): return out.strip().splitlines()[0] return None async def kill_and_remove(self, container_id: str, timeout: float = 10.0) -> None: deadline = time.monotonic() + timeout def remaining(): budget = deadline - time.monotonic() if budget <= 0: raise CleanupFailedError("Container removal deadline exceeded") return budget try: # Stop the attached CLI first so a delayed launch cannot outlive cleanup. proc = self._processes.get(container_id) if proc is not None and proc.returncode is None: proc.kill() await asyncio.wait_for(proc.wait(), timeout=remaining()) target = await self._find_container_id(container_id, timeout=remaining()) if target is not None: code, _, err = await self._exec_docker(["rm", "-f", target], timeout=remaining()) if code != 0 and "already in progress" not in err.lower(): # --rm may have raced us; only a successful empty listing is acceptable. if await self._find_container_id(container_id, timeout=remaining()) is not None: raise CleanupFailedError("Docker refused container removal") while await self._find_container_id(container_id, timeout=remaining()) is not None: await asyncio.sleep(min(0.1, remaining())) self._processes.pop(container_id, None) except CleanupFailedError: raise except Exception as exc: raise CleanupFailedError("Container removal could not be confirmed") from exc async def reconcile_orphans(self, max_age_seconds: float = 200.0) -> int: label_filter = f"label={self.settings.container_label_key}={self.settings.container_label_value}" time_key = self.settings.container_label_key + ".created_at" code, out, _ = await self._exec_docker([ "ps", "-a", "--filter", label_filter, "--format", '{{.ID}}\t{{.Label "' + time_key + '"}}\t{{.Label "com.confluence_web.created_at"}}', ]) if code != 0: raise CleanupFailedError("Docker orphan discovery failed") removed = 0 for line in out.splitlines(): parts = line.split("\t") cid = parts[0].strip() timestamp = next((v.strip() for v in parts[1:] if v.strip()), "") try: age = time.time() - float(timestamp) except ValueError: age = float("inf") if max_age_seconds == 0 or age >= max_age_seconds: await self.kill_and_remove(cid, timeout=self.settings.cleanup_timeout_seconds) removed += 1 return removed class FakeContainerProcess: """In-memory fake container process with streams.""" def __init__(self, container_id: str, runner_coro): self.container_id = container_id # Streams between host and fake container with high limit for large frames self.to_container_reader = asyncio.StreamReader(limit=GLOBAL_MAX_FRAME_BYTES) self.to_container_writer = asyncio.StreamWriter( transport=_QueueTransport(self.to_container_reader), protocol=asyncio.StreamReaderProtocol(self.to_container_reader), reader=self.to_container_reader, loop=asyncio.get_running_loop(), ) self.from_container_reader = asyncio.StreamReader(limit=GLOBAL_MAX_FRAME_BYTES) self.from_container_writer = asyncio.StreamWriter( transport=_QueueTransport(self.from_container_reader), protocol=asyncio.StreamReaderProtocol(self.from_container_reader), reader=self.from_container_reader, loop=asyncio.get_running_loop(), ) # Host sees: stdin writes to to_container, stdout reads from from_container self.handle = ContainerHandle( container_id=container_id, stdin=self.to_container_writer, stdout=self.from_container_reader, stderr=asyncio.StreamReader(limit=GLOBAL_MAX_FRAME_BYTES), ) # Launch fake container task self.task = asyncio.create_task( self._run_peer(runner_coro) ) async def _run_peer(self, runner_coro): try: await runner_coro(self.to_container_reader, self.from_container_writer) finally: self.from_container_writer.close() self.handle.stderr.feed_eof() class _QueueTransport(asyncio.Transport): def __init__(self, reader: asyncio.StreamReader): super().__init__() self.reader = reader self._closing = False def write(self, data: bytes) -> None: self.reader.feed_data(data) def is_closing(self) -> bool: return self._closing def close(self) -> None: self._closing = True self.reader.feed_eof() class FakeContainerManager(ContainerManager): """In-process fake container manager for deterministic tests.""" def __init__(self, peer_factory: Optional[Any] = None): if peer_factory is None: from backend.dev.fake_peer import ScriptedContainerPeer peer_factory = ScriptedContainerPeer self.peer_factory = peer_factory self.active_containers: Dict[str, FakeContainerProcess] = {} self.removed_containers: List[str] = [] async def verify_rootless(self) -> None: pass async def create_and_run(self, query_id: str) -> ContainerHandle: fake_peer = self.peer_factory() proc = FakeContainerProcess(query_id, fake_peer.run) self.active_containers[query_id] = proc return proc.handle async def kill_and_remove(self, container_id: str, timeout: float = 10.0) -> None: proc = self.active_containers.pop(container_id, None) if proc: proc.task.cancel() try: await proc.task except asyncio.CancelledError: pass except Exception: pass self.removed_containers.append(container_id) async def reconcile_orphans(self, max_age_seconds: float = 200.0) -> int: count = len(self.active_containers) for cid in list(self.active_containers): await self.kill_and_remove(cid) return count