Artur Mukhamadiev
3751ab26b5
The 180 s query cap was enforced independently by the backend clamp, the agent limits, and the container supervisor. All three now follow CONFLUENCE_WEB_MAX_DEADLINE_SECONDS (default 900, allowed 60-3600): the backend passes it into the container at launch, the supervisor reads it and forwards it to the bridge, and both fall back to 900 s on invalid input. The query timeout must not exceed it (startup fails otherwise). The supervisor keeps a separate 180 s guard for a container that never receives a start frame. Add assets/book.svg as the tab icon: the backend serves assets/ and the CSP allows same-origin images (the sanitizer still never emits <img>).
365 lines
15 KiB
Python
365 lines
15 KiB
Python
"""Rootless Docker container lifecycle, isolation, cleanup, and reconciliation."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import abc
|
|
import asyncio
|
|
import json
|
|
import logging
|
|
import os
|
|
import shutil
|
|
import time
|
|
from typing import Any, Dict, List, Optional, Tuple
|
|
|
|
from backend.errors import CleanupFailedError, ExecutionFailedError
|
|
from backend.settings import Settings
|
|
from backend.transport import GLOBAL_MAX_FRAME_BYTES
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
class ContainerHandle:
|
|
"""Represents an active attached container process."""
|
|
|
|
def __init__(
|
|
self,
|
|
container_id: str,
|
|
stdin: asyncio.StreamWriter,
|
|
stdout: asyncio.StreamReader,
|
|
stderr: Optional[asyncio.StreamReader] = None,
|
|
process: Optional[asyncio.subprocess.Process] = None,
|
|
):
|
|
self.container_id = container_id
|
|
self.stdin = stdin
|
|
self.stdout = stdout
|
|
self.stderr = stderr
|
|
self.process = process
|
|
|
|
async def wait(self) -> int:
|
|
if self.process:
|
|
return await self.process.wait()
|
|
return 0
|
|
|
|
|
|
class ContainerManager(abc.ABC):
|
|
"""Abstract container lifecycle manager."""
|
|
|
|
@abc.abstractmethod
|
|
async def verify_rootless(self) -> None:
|
|
"""Verify daemon is rootless and required isolation is supported."""
|
|
raise NotImplementedError
|
|
|
|
@abc.abstractmethod
|
|
async def create_and_run(self, query_id: str) -> ContainerHandle:
|
|
"""Create, configure, and launch isolated container, returning attached streams."""
|
|
raise NotImplementedError
|
|
|
|
@abc.abstractmethod
|
|
async def kill_and_remove(self, container_id: str, timeout: float = 10.0) -> None:
|
|
"""Kill and remove container, confirming its deletion within timeout."""
|
|
raise NotImplementedError
|
|
|
|
@abc.abstractmethod
|
|
async def reconcile_orphans(self, max_age_seconds: float = 200.0) -> int:
|
|
"""Find and remove residual application containers older than max_age_seconds."""
|
|
raise NotImplementedError
|
|
|
|
|
|
class DockerContainerManager(ContainerManager):
|
|
"""Rootless Docker container lifecycle implementation."""
|
|
|
|
def __init__(self, settings: Settings):
|
|
self.settings = settings
|
|
self.docker_bin = shutil.which("docker") or "docker"
|
|
# Clean minimal environment for Docker CLI subprocesses, avoiding secret leaks
|
|
self._env = {
|
|
"PATH": os.environ.get("PATH", "/usr/local/bin:/usr/bin:/bin"),
|
|
"HOME": os.environ.get("HOME", "/root"),
|
|
}
|
|
self._processes: Dict[str, asyncio.subprocess.Process] = {}
|
|
for key in ("XDG_RUNTIME_DIR", "DOCKER_CONTEXT", "DOCKER_CONFIG"):
|
|
if key in os.environ:
|
|
self._env[key] = os.environ[key]
|
|
if settings.docker_host:
|
|
self._env["DOCKER_HOST"] = settings.docker_host
|
|
elif "DOCKER_HOST" in os.environ:
|
|
self._env["DOCKER_HOST"] = os.environ["DOCKER_HOST"]
|
|
|
|
async def _exec_docker(self, args: List[str], timeout: float = 15.0) -> Tuple[int, str, str]:
|
|
cmd = [self.docker_bin] + args
|
|
proc = await asyncio.create_subprocess_exec(
|
|
*cmd,
|
|
stdout=asyncio.subprocess.PIPE,
|
|
stderr=asyncio.subprocess.PIPE,
|
|
env=self._env,
|
|
)
|
|
try:
|
|
stdout_bytes, stderr_bytes = await asyncio.wait_for(
|
|
proc.communicate(), timeout=timeout
|
|
)
|
|
return (
|
|
proc.returncode or 0,
|
|
stdout_bytes.decode("utf-8", errors="replace"),
|
|
stderr_bytes.decode("utf-8", errors="replace"),
|
|
)
|
|
except BaseException:
|
|
if proc.returncode is None:
|
|
proc.kill()
|
|
await proc.communicate()
|
|
raise
|
|
|
|
async def verify_rootless(self) -> None:
|
|
"""Verify Docker daemon is running in rootless mode and enforces cgroups."""
|
|
code, out, err = await self._exec_docker(["info", "--format", "{{json .}}"])
|
|
if code != 0:
|
|
raise ExecutionFailedError(f"Failed to query Docker daemon: {err.strip()}")
|
|
|
|
try:
|
|
info = json.loads(out)
|
|
except Exception as e:
|
|
raise ExecutionFailedError("Failed to parse Docker info JSON") from e
|
|
|
|
sec_opts = info.get("SecurityOptions", [])
|
|
is_rootless = any(opt in ("name=rootless", "rootless") for opt in sec_opts)
|
|
|
|
if not is_rootless:
|
|
raise ExecutionFailedError(
|
|
"Docker daemon is not running in rootless mode! Rootless mode is required."
|
|
)
|
|
|
|
required = ("MemoryLimit", "PidsLimit", "CpuCfsQuota", "CpuCfsPeriod")
|
|
if info.get("CgroupVersion") != "2" or info.get("CgroupDriver") in (None, "none", "") or any(info.get(k) is not True for k in required):
|
|
raise ExecutionFailedError("Docker must enforce memory, CPU and PID limits with cgroup v2")
|
|
|
|
async def create_and_run(self, query_id: str) -> ContainerHandle:
|
|
"""Launch isolated container attached via pipes."""
|
|
created_at = str(int(time.time()))
|
|
container_name = f"cw-{query_id}"
|
|
label_app = f"{self.settings.container_label_key}={self.settings.container_label_value}"
|
|
label_qid = f"{self.settings.container_label_key}.query_id={query_id}"
|
|
label_time = f"{self.settings.container_label_key}.created_at={created_at}"
|
|
|
|
# Build isolated docker run command with structured args (no shell)
|
|
cmd = [
|
|
self.docker_bin,
|
|
"run",
|
|
"--name", container_name,
|
|
"-i", # interactive attached stdin
|
|
# No --init: the image's Python supervisor must remain namespace PID 1
|
|
# (subreaper, signal immunity, independent deadline: 900 s maximum, 180 s without a start frame).
|
|
"--rm", # automatically remove on exit if clean
|
|
"--read-only",
|
|
"--network", "none",
|
|
"--user", "10001:10001",
|
|
"-w", "/work",
|
|
"-e", "HOME=/home/agent",
|
|
"-e", "LANG=C.UTF-8",
|
|
"-e", f"CONFLUENCE_WEB_MAX_DEADLINE_SECONDS={int(self.settings.max_deadline_seconds)}",
|
|
"--tmpfs", "/work:rw,nosuid,nodev,size=256m,uid=10001,gid=10001",
|
|
"--tmpfs", "/tmp:rw,nosuid,nodev,size=64m,uid=10001,gid=10001",
|
|
"--tmpfs", "/home/agent:rw,nosuid,nodev,size=32m,uid=10001,gid=10001",
|
|
"--memory", "1g",
|
|
"--memory-swap", "1g", # no additional swap beyond the memory limit
|
|
"--cpus", "1.0",
|
|
"--pids-limit", "128",
|
|
"--cap-drop", "ALL",
|
|
"--security-opt", "no-new-privileges",
|
|
"--log-driver", "none",
|
|
"--label", label_app,
|
|
"--label", label_qid,
|
|
"--label", label_time,
|
|
"--label", f"{self.settings.container_label_key}.instance_id={self.settings.container_instance_id}",
|
|
self.settings.runtime_image,
|
|
]
|
|
|
|
try:
|
|
# Configure high limit so multi-MiB bridge frames do not raise LimitOverrunError
|
|
proc = await asyncio.create_subprocess_exec(
|
|
*cmd,
|
|
stdin=asyncio.subprocess.PIPE,
|
|
stdout=asyncio.subprocess.PIPE,
|
|
stderr=asyncio.subprocess.PIPE,
|
|
limit=GLOBAL_MAX_FRAME_BYTES,
|
|
env=self._env,
|
|
)
|
|
except Exception as e:
|
|
raise ExecutionFailedError(f"Failed to spawn container process: {e}") from e
|
|
|
|
self._processes[query_id] = proc
|
|
return ContainerHandle(
|
|
container_id=query_id,
|
|
stdin=proc.stdin, # type: ignore
|
|
stdout=proc.stdout, # type: ignore
|
|
stderr=proc.stderr, # type: ignore
|
|
process=proc,
|
|
)
|
|
|
|
async def _find_container_id(self, query_id: str, timeout: float = 3.0) -> Optional[str]:
|
|
deadline = time.monotonic() + timeout
|
|
cname = query_id if query_id.startswith("cw-") else f"cw-{query_id}"
|
|
filters = [f"name=^/{cname}$"]
|
|
if len(query_id) >= 12 and all(c in "0123456789abcdef" for c in query_id):
|
|
filters.append(f"id={query_id}")
|
|
for selection in filters:
|
|
remaining = deadline - time.monotonic()
|
|
if remaining <= 0:
|
|
raise CleanupFailedError("Docker discovery deadline exceeded")
|
|
code, out, _ = await self._exec_docker(["ps", "-a", "-q", "--filter", selection], timeout=remaining)
|
|
if code != 0:
|
|
raise CleanupFailedError("Docker daemon could not confirm container state")
|
|
if out.strip():
|
|
return out.strip().splitlines()[0]
|
|
return None
|
|
|
|
async def kill_and_remove(self, container_id: str, timeout: float = 10.0) -> None:
|
|
deadline = time.monotonic() + timeout
|
|
|
|
def remaining():
|
|
budget = deadline - time.monotonic()
|
|
if budget <= 0:
|
|
raise CleanupFailedError("Container removal deadline exceeded")
|
|
return budget
|
|
|
|
try:
|
|
# Stop the attached CLI first so a delayed launch cannot outlive cleanup.
|
|
proc = self._processes.get(container_id)
|
|
if proc is not None and proc.returncode is None:
|
|
proc.kill()
|
|
await asyncio.wait_for(proc.wait(), timeout=remaining())
|
|
target = await self._find_container_id(container_id, timeout=remaining())
|
|
if target is not None:
|
|
code, _, err = await self._exec_docker(["rm", "-f", target], timeout=remaining())
|
|
if code != 0 and "already in progress" not in err.lower():
|
|
# --rm may have raced us; only a successful empty listing is acceptable.
|
|
if await self._find_container_id(container_id, timeout=remaining()) is not None:
|
|
raise CleanupFailedError("Docker refused container removal")
|
|
while await self._find_container_id(container_id, timeout=remaining()) is not None:
|
|
await asyncio.sleep(min(0.1, remaining()))
|
|
self._processes.pop(container_id, None)
|
|
except CleanupFailedError:
|
|
raise
|
|
except Exception as exc:
|
|
raise CleanupFailedError("Container removal could not be confirmed") from exc
|
|
|
|
async def reconcile_orphans(self, max_age_seconds: float = 200.0) -> int:
|
|
label_filter = f"label={self.settings.container_label_key}={self.settings.container_label_value}"
|
|
time_key = self.settings.container_label_key + ".created_at"
|
|
code, out, _ = await self._exec_docker([
|
|
"ps", "-a", "--filter", label_filter,
|
|
"--format", '{{.ID}}\t{{.Label "' + time_key + '"}}\t{{.Label "com.confluence_web.created_at"}}',
|
|
])
|
|
if code != 0:
|
|
raise CleanupFailedError("Docker orphan discovery failed")
|
|
removed = 0
|
|
for line in out.splitlines():
|
|
parts = line.split("\t")
|
|
cid = parts[0].strip()
|
|
timestamp = next((v.strip() for v in parts[1:] if v.strip()), "")
|
|
try:
|
|
age = time.time() - float(timestamp)
|
|
except ValueError:
|
|
age = float("inf")
|
|
if max_age_seconds == 0 or age >= max_age_seconds:
|
|
await self.kill_and_remove(cid, timeout=self.settings.cleanup_timeout_seconds)
|
|
removed += 1
|
|
return removed
|
|
|
|
|
|
class FakeContainerProcess:
|
|
"""In-memory fake container process with streams."""
|
|
|
|
def __init__(self, container_id: str, runner_coro):
|
|
self.container_id = container_id
|
|
# Streams between host and fake container with high limit for large frames
|
|
self.to_container_reader = asyncio.StreamReader(limit=GLOBAL_MAX_FRAME_BYTES)
|
|
self.to_container_writer = asyncio.StreamWriter(
|
|
transport=_QueueTransport(self.to_container_reader),
|
|
protocol=asyncio.StreamReaderProtocol(self.to_container_reader),
|
|
reader=self.to_container_reader,
|
|
loop=asyncio.get_running_loop(),
|
|
)
|
|
|
|
self.from_container_reader = asyncio.StreamReader(limit=GLOBAL_MAX_FRAME_BYTES)
|
|
self.from_container_writer = asyncio.StreamWriter(
|
|
transport=_QueueTransport(self.from_container_reader),
|
|
protocol=asyncio.StreamReaderProtocol(self.from_container_reader),
|
|
reader=self.from_container_reader,
|
|
loop=asyncio.get_running_loop(),
|
|
)
|
|
|
|
# Host sees: stdin writes to to_container, stdout reads from from_container
|
|
self.handle = ContainerHandle(
|
|
container_id=container_id,
|
|
stdin=self.to_container_writer,
|
|
stdout=self.from_container_reader,
|
|
stderr=asyncio.StreamReader(limit=GLOBAL_MAX_FRAME_BYTES),
|
|
)
|
|
|
|
# Launch fake container task
|
|
self.task = asyncio.create_task(
|
|
self._run_peer(runner_coro)
|
|
)
|
|
|
|
async def _run_peer(self, runner_coro):
|
|
try:
|
|
await runner_coro(self.to_container_reader, self.from_container_writer)
|
|
finally:
|
|
self.from_container_writer.close()
|
|
self.handle.stderr.feed_eof()
|
|
|
|
|
|
class _QueueTransport(asyncio.Transport):
|
|
def __init__(self, reader: asyncio.StreamReader):
|
|
super().__init__()
|
|
self.reader = reader
|
|
self._closing = False
|
|
|
|
def write(self, data: bytes) -> None:
|
|
self.reader.feed_data(data)
|
|
|
|
def is_closing(self) -> bool:
|
|
return self._closing
|
|
|
|
def close(self) -> None:
|
|
self._closing = True
|
|
self.reader.feed_eof()
|
|
|
|
|
|
class FakeContainerManager(ContainerManager):
|
|
"""In-process fake container manager for deterministic tests."""
|
|
|
|
def __init__(self, peer_factory: Optional[Any] = None):
|
|
if peer_factory is None:
|
|
from backend.dev.fake_peer import ScriptedContainerPeer
|
|
peer_factory = ScriptedContainerPeer
|
|
self.peer_factory = peer_factory
|
|
self.active_containers: Dict[str, FakeContainerProcess] = {}
|
|
self.removed_containers: List[str] = []
|
|
|
|
async def verify_rootless(self) -> None:
|
|
pass
|
|
|
|
async def create_and_run(self, query_id: str) -> ContainerHandle:
|
|
fake_peer = self.peer_factory()
|
|
proc = FakeContainerProcess(query_id, fake_peer.run)
|
|
self.active_containers[query_id] = proc
|
|
return proc.handle
|
|
|
|
async def kill_and_remove(self, container_id: str, timeout: float = 10.0) -> None:
|
|
proc = self.active_containers.pop(container_id, None)
|
|
if proc:
|
|
proc.task.cancel()
|
|
try:
|
|
await proc.task
|
|
except asyncio.CancelledError:
|
|
pass
|
|
except Exception:
|
|
pass
|
|
self.removed_containers.append(container_id)
|
|
|
|
async def reconcile_orphans(self, max_age_seconds: float = 200.0) -> int:
|
|
count = len(self.active_containers)
|
|
for cid in list(self.active_containers):
|
|
await self.kill_and_remove(cid)
|
|
return count
|