Artur Mukhamadiev
1d7b867986
The bridge counted tool and model calls against fixed 100 and 50 caps, so CONFLUENCE_WEB_MAX_CONFLUENCE_CALLS / CONFLUENCE_WEB_MAX_MODEL_CALLS above the defaults were cut short inside the container, and the model cap failed the run outright. The backend now passes both values into the container at start, the supervisor forwards them to the bridge, and LIMITS reads them with the same 1-1000 validation as the backend, falling back to 100/50 when absent or invalid. Add a test that loads LIMITS in fresh processes with good and bad values for both budgets and the deadline.
367 lines
15 KiB
Python
367 lines
15 KiB
Python
"""Rootless Docker container lifecycle, isolation, cleanup, and reconciliation."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import abc
|
|
import asyncio
|
|
import json
|
|
import logging
|
|
import os
|
|
import shutil
|
|
import time
|
|
from typing import Any, Dict, List, Optional, Tuple
|
|
|
|
from backend.errors import CleanupFailedError, ExecutionFailedError
|
|
from backend.settings import Settings
|
|
from backend.transport import GLOBAL_MAX_FRAME_BYTES
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
class ContainerHandle:
|
|
"""Represents an active attached container process."""
|
|
|
|
def __init__(
|
|
self,
|
|
container_id: str,
|
|
stdin: asyncio.StreamWriter,
|
|
stdout: asyncio.StreamReader,
|
|
stderr: Optional[asyncio.StreamReader] = None,
|
|
process: Optional[asyncio.subprocess.Process] = None,
|
|
):
|
|
self.container_id = container_id
|
|
self.stdin = stdin
|
|
self.stdout = stdout
|
|
self.stderr = stderr
|
|
self.process = process
|
|
|
|
async def wait(self) -> int:
|
|
if self.process:
|
|
return await self.process.wait()
|
|
return 0
|
|
|
|
|
|
class ContainerManager(abc.ABC):
|
|
"""Abstract container lifecycle manager."""
|
|
|
|
@abc.abstractmethod
|
|
async def verify_rootless(self) -> None:
|
|
"""Verify daemon is rootless and required isolation is supported."""
|
|
raise NotImplementedError
|
|
|
|
@abc.abstractmethod
|
|
async def create_and_run(self, query_id: str) -> ContainerHandle:
|
|
"""Create, configure, and launch isolated container, returning attached streams."""
|
|
raise NotImplementedError
|
|
|
|
@abc.abstractmethod
|
|
async def kill_and_remove(self, container_id: str, timeout: float = 10.0) -> None:
|
|
"""Kill and remove container, confirming its deletion within timeout."""
|
|
raise NotImplementedError
|
|
|
|
@abc.abstractmethod
|
|
async def reconcile_orphans(self, max_age_seconds: float = 200.0) -> int:
|
|
"""Find and remove residual application containers older than max_age_seconds."""
|
|
raise NotImplementedError
|
|
|
|
|
|
class DockerContainerManager(ContainerManager):
|
|
"""Rootless Docker container lifecycle implementation."""
|
|
|
|
def __init__(self, settings: Settings):
|
|
self.settings = settings
|
|
self.docker_bin = shutil.which("docker") or "docker"
|
|
# Clean minimal environment for Docker CLI subprocesses, avoiding secret leaks
|
|
self._env = {
|
|
"PATH": os.environ.get("PATH", "/usr/local/bin:/usr/bin:/bin"),
|
|
"HOME": os.environ.get("HOME", "/root"),
|
|
}
|
|
self._processes: Dict[str, asyncio.subprocess.Process] = {}
|
|
for key in ("XDG_RUNTIME_DIR", "DOCKER_CONTEXT", "DOCKER_CONFIG"):
|
|
if key in os.environ:
|
|
self._env[key] = os.environ[key]
|
|
if settings.docker_host:
|
|
self._env["DOCKER_HOST"] = settings.docker_host
|
|
elif "DOCKER_HOST" in os.environ:
|
|
self._env["DOCKER_HOST"] = os.environ["DOCKER_HOST"]
|
|
|
|
async def _exec_docker(self, args: List[str], timeout: float = 15.0) -> Tuple[int, str, str]:
|
|
cmd = [self.docker_bin] + args
|
|
proc = await asyncio.create_subprocess_exec(
|
|
*cmd,
|
|
stdout=asyncio.subprocess.PIPE,
|
|
stderr=asyncio.subprocess.PIPE,
|
|
env=self._env,
|
|
)
|
|
try:
|
|
stdout_bytes, stderr_bytes = await asyncio.wait_for(
|
|
proc.communicate(), timeout=timeout
|
|
)
|
|
return (
|
|
proc.returncode or 0,
|
|
stdout_bytes.decode("utf-8", errors="replace"),
|
|
stderr_bytes.decode("utf-8", errors="replace"),
|
|
)
|
|
except BaseException:
|
|
if proc.returncode is None:
|
|
proc.kill()
|
|
await proc.communicate()
|
|
raise
|
|
|
|
async def verify_rootless(self) -> None:
|
|
"""Verify Docker daemon is running in rootless mode and enforces cgroups."""
|
|
code, out, err = await self._exec_docker(["info", "--format", "{{json .}}"])
|
|
if code != 0:
|
|
raise ExecutionFailedError(f"Failed to query Docker daemon: {err.strip()}")
|
|
|
|
try:
|
|
info = json.loads(out)
|
|
except Exception as e:
|
|
raise ExecutionFailedError("Failed to parse Docker info JSON") from e
|
|
|
|
sec_opts = info.get("SecurityOptions", [])
|
|
is_rootless = any(opt in ("name=rootless", "rootless") for opt in sec_opts)
|
|
|
|
if not is_rootless:
|
|
raise ExecutionFailedError(
|
|
"Docker daemon is not running in rootless mode! Rootless mode is required."
|
|
)
|
|
|
|
required = ("MemoryLimit", "PidsLimit", "CpuCfsQuota", "CpuCfsPeriod")
|
|
if info.get("CgroupVersion") != "2" or info.get("CgroupDriver") in (None, "none", "") or any(info.get(k) is not True for k in required):
|
|
raise ExecutionFailedError("Docker must enforce memory, CPU and PID limits with cgroup v2")
|
|
|
|
async def create_and_run(self, query_id: str) -> ContainerHandle:
|
|
"""Launch isolated container attached via pipes."""
|
|
created_at = str(int(time.time()))
|
|
container_name = f"cw-{query_id}"
|
|
label_app = f"{self.settings.container_label_key}={self.settings.container_label_value}"
|
|
label_qid = f"{self.settings.container_label_key}.query_id={query_id}"
|
|
label_time = f"{self.settings.container_label_key}.created_at={created_at}"
|
|
|
|
# Build isolated docker run command with structured args (no shell)
|
|
cmd = [
|
|
self.docker_bin,
|
|
"run",
|
|
"--name", container_name,
|
|
"-i", # interactive attached stdin
|
|
# No --init: the image's Python supervisor must remain namespace PID 1
|
|
# (subreaper, signal immunity, independent deadline: 900 s maximum, 180 s without a start frame).
|
|
"--rm", # automatically remove on exit if clean
|
|
"--read-only",
|
|
"--network", "none",
|
|
"--user", "10001:10001",
|
|
"-w", "/work",
|
|
"-e", "HOME=/home/agent",
|
|
"-e", "LANG=C.UTF-8",
|
|
"-e", f"CONFLUENCE_WEB_MAX_DEADLINE_SECONDS={int(self.settings.max_deadline_seconds)}",
|
|
"-e", f"CONFLUENCE_WEB_MAX_CONFLUENCE_CALLS={self.settings.max_confluence_calls}",
|
|
"-e", f"CONFLUENCE_WEB_MAX_MODEL_CALLS={self.settings.max_model_calls}",
|
|
"--tmpfs", "/work:rw,nosuid,nodev,size=256m,uid=10001,gid=10001",
|
|
"--tmpfs", "/tmp:rw,nosuid,nodev,size=64m,uid=10001,gid=10001",
|
|
"--tmpfs", "/home/agent:rw,nosuid,nodev,size=32m,uid=10001,gid=10001",
|
|
"--memory", "1g",
|
|
"--memory-swap", "1g", # no additional swap beyond the memory limit
|
|
"--cpus", "1.0",
|
|
"--pids-limit", "128",
|
|
"--cap-drop", "ALL",
|
|
"--security-opt", "no-new-privileges",
|
|
"--log-driver", "none",
|
|
"--label", label_app,
|
|
"--label", label_qid,
|
|
"--label", label_time,
|
|
"--label", f"{self.settings.container_label_key}.instance_id={self.settings.container_instance_id}",
|
|
self.settings.runtime_image,
|
|
]
|
|
|
|
try:
|
|
# Configure high limit so multi-MiB bridge frames do not raise LimitOverrunError
|
|
proc = await asyncio.create_subprocess_exec(
|
|
*cmd,
|
|
stdin=asyncio.subprocess.PIPE,
|
|
stdout=asyncio.subprocess.PIPE,
|
|
stderr=asyncio.subprocess.PIPE,
|
|
limit=GLOBAL_MAX_FRAME_BYTES,
|
|
env=self._env,
|
|
)
|
|
except Exception as e:
|
|
raise ExecutionFailedError(f"Failed to spawn container process: {e}") from e
|
|
|
|
self._processes[query_id] = proc
|
|
return ContainerHandle(
|
|
container_id=query_id,
|
|
stdin=proc.stdin, # type: ignore
|
|
stdout=proc.stdout, # type: ignore
|
|
stderr=proc.stderr, # type: ignore
|
|
process=proc,
|
|
)
|
|
|
|
async def _find_container_id(self, query_id: str, timeout: float = 3.0) -> Optional[str]:
|
|
deadline = time.monotonic() + timeout
|
|
cname = query_id if query_id.startswith("cw-") else f"cw-{query_id}"
|
|
filters = [f"name=^/{cname}$"]
|
|
if len(query_id) >= 12 and all(c in "0123456789abcdef" for c in query_id):
|
|
filters.append(f"id={query_id}")
|
|
for selection in filters:
|
|
remaining = deadline - time.monotonic()
|
|
if remaining <= 0:
|
|
raise CleanupFailedError("Docker discovery deadline exceeded")
|
|
code, out, _ = await self._exec_docker(["ps", "-a", "-q", "--filter", selection], timeout=remaining)
|
|
if code != 0:
|
|
raise CleanupFailedError("Docker daemon could not confirm container state")
|
|
if out.strip():
|
|
return out.strip().splitlines()[0]
|
|
return None
|
|
|
|
async def kill_and_remove(self, container_id: str, timeout: float = 10.0) -> None:
|
|
deadline = time.monotonic() + timeout
|
|
|
|
def remaining():
|
|
budget = deadline - time.monotonic()
|
|
if budget <= 0:
|
|
raise CleanupFailedError("Container removal deadline exceeded")
|
|
return budget
|
|
|
|
try:
|
|
# Stop the attached CLI first so a delayed launch cannot outlive cleanup.
|
|
proc = self._processes.get(container_id)
|
|
if proc is not None and proc.returncode is None:
|
|
proc.kill()
|
|
await asyncio.wait_for(proc.wait(), timeout=remaining())
|
|
target = await self._find_container_id(container_id, timeout=remaining())
|
|
if target is not None:
|
|
code, _, err = await self._exec_docker(["rm", "-f", target], timeout=remaining())
|
|
if code != 0 and "already in progress" not in err.lower():
|
|
# --rm may have raced us; only a successful empty listing is acceptable.
|
|
if await self._find_container_id(container_id, timeout=remaining()) is not None:
|
|
raise CleanupFailedError("Docker refused container removal")
|
|
while await self._find_container_id(container_id, timeout=remaining()) is not None:
|
|
await asyncio.sleep(min(0.1, remaining()))
|
|
self._processes.pop(container_id, None)
|
|
except CleanupFailedError:
|
|
raise
|
|
except Exception as exc:
|
|
raise CleanupFailedError("Container removal could not be confirmed") from exc
|
|
|
|
async def reconcile_orphans(self, max_age_seconds: float = 200.0) -> int:
|
|
label_filter = f"label={self.settings.container_label_key}={self.settings.container_label_value}"
|
|
time_key = self.settings.container_label_key + ".created_at"
|
|
code, out, _ = await self._exec_docker([
|
|
"ps", "-a", "--filter", label_filter,
|
|
"--format", '{{.ID}}\t{{.Label "' + time_key + '"}}\t{{.Label "com.confluence_web.created_at"}}',
|
|
])
|
|
if code != 0:
|
|
raise CleanupFailedError("Docker orphan discovery failed")
|
|
removed = 0
|
|
for line in out.splitlines():
|
|
parts = line.split("\t")
|
|
cid = parts[0].strip()
|
|
timestamp = next((v.strip() for v in parts[1:] if v.strip()), "")
|
|
try:
|
|
age = time.time() - float(timestamp)
|
|
except ValueError:
|
|
age = float("inf")
|
|
if max_age_seconds == 0 or age >= max_age_seconds:
|
|
await self.kill_and_remove(cid, timeout=self.settings.cleanup_timeout_seconds)
|
|
removed += 1
|
|
return removed
|
|
|
|
|
|
class FakeContainerProcess:
|
|
"""In-memory fake container process with streams."""
|
|
|
|
def __init__(self, container_id: str, runner_coro):
|
|
self.container_id = container_id
|
|
# Streams between host and fake container with high limit for large frames
|
|
self.to_container_reader = asyncio.StreamReader(limit=GLOBAL_MAX_FRAME_BYTES)
|
|
self.to_container_writer = asyncio.StreamWriter(
|
|
transport=_QueueTransport(self.to_container_reader),
|
|
protocol=asyncio.StreamReaderProtocol(self.to_container_reader),
|
|
reader=self.to_container_reader,
|
|
loop=asyncio.get_running_loop(),
|
|
)
|
|
|
|
self.from_container_reader = asyncio.StreamReader(limit=GLOBAL_MAX_FRAME_BYTES)
|
|
self.from_container_writer = asyncio.StreamWriter(
|
|
transport=_QueueTransport(self.from_container_reader),
|
|
protocol=asyncio.StreamReaderProtocol(self.from_container_reader),
|
|
reader=self.from_container_reader,
|
|
loop=asyncio.get_running_loop(),
|
|
)
|
|
|
|
# Host sees: stdin writes to to_container, stdout reads from from_container
|
|
self.handle = ContainerHandle(
|
|
container_id=container_id,
|
|
stdin=self.to_container_writer,
|
|
stdout=self.from_container_reader,
|
|
stderr=asyncio.StreamReader(limit=GLOBAL_MAX_FRAME_BYTES),
|
|
)
|
|
|
|
# Launch fake container task
|
|
self.task = asyncio.create_task(
|
|
self._run_peer(runner_coro)
|
|
)
|
|
|
|
async def _run_peer(self, runner_coro):
|
|
try:
|
|
await runner_coro(self.to_container_reader, self.from_container_writer)
|
|
finally:
|
|
self.from_container_writer.close()
|
|
self.handle.stderr.feed_eof()
|
|
|
|
|
|
class _QueueTransport(asyncio.Transport):
|
|
def __init__(self, reader: asyncio.StreamReader):
|
|
super().__init__()
|
|
self.reader = reader
|
|
self._closing = False
|
|
|
|
def write(self, data: bytes) -> None:
|
|
self.reader.feed_data(data)
|
|
|
|
def is_closing(self) -> bool:
|
|
return self._closing
|
|
|
|
def close(self) -> None:
|
|
self._closing = True
|
|
self.reader.feed_eof()
|
|
|
|
|
|
class FakeContainerManager(ContainerManager):
|
|
"""In-process fake container manager for deterministic tests."""
|
|
|
|
def __init__(self, peer_factory: Optional[Any] = None):
|
|
if peer_factory is None:
|
|
from backend.dev.fake_peer import ScriptedContainerPeer
|
|
peer_factory = ScriptedContainerPeer
|
|
self.peer_factory = peer_factory
|
|
self.active_containers: Dict[str, FakeContainerProcess] = {}
|
|
self.removed_containers: List[str] = []
|
|
|
|
async def verify_rootless(self) -> None:
|
|
pass
|
|
|
|
async def create_and_run(self, query_id: str) -> ContainerHandle:
|
|
fake_peer = self.peer_factory()
|
|
proc = FakeContainerProcess(query_id, fake_peer.run)
|
|
self.active_containers[query_id] = proc
|
|
return proc.handle
|
|
|
|
async def kill_and_remove(self, container_id: str, timeout: float = 10.0) -> None:
|
|
proc = self.active_containers.pop(container_id, None)
|
|
if proc:
|
|
proc.task.cancel()
|
|
try:
|
|
await proc.task
|
|
except asyncio.CancelledError:
|
|
pass
|
|
except Exception:
|
|
pass
|
|
self.removed_containers.append(container_id)
|
|
|
|
async def reconcile_orphans(self, max_age_seconds: float = 200.0) -> int:
|
|
count = len(self.active_containers)
|
|
for cid in list(self.active_containers):
|
|
await self.kill_and_remove(cid)
|
|
return count
|