From d77b52ae86b229f2f4c3f11a72d6cbfcfeb1629c Mon Sep 17 00:00:00 2001 From: Artur Mukhamadiev Date: Mon, 14 Sep 2026 22:20:34 +0300 Subject: [PATCH] integration: contract checks, error mapping, deployment entry points - tests/integration: frontend served by backend, runtime error mapping, backend + real pi image with scripted model/Confluence (shared example, variants, failure paths, cancellation/busy gate, isolation canaries, HTTP download ownership, 16 MiB prompt round trip), real OpenAI-compatible adapter + real image over a scripted transport, real Chrome against the real backend with a scripted runtime peer, backend crash/restart reconciliation, and an opt-in live model check (marker: live). - backend: map runtime terminal codes (model_output_limit, model_context_exceeded, query_timeout, connectivity_failed) to the contract's HTTP statuses; make the artifact 404 body identical for no-session, wrong-session, unknown and expired IDs. - Makefile, scripts/run-backend.sh, deploy/confluence-web.env.example, root README for the integrated application; integration pytest marker. --- .gitignore | 3 + Makefile | 62 +++ README.md | 127 +++-- backend/app.py | 2 +- backend/runner.py | 22 +- deploy/confluence-web.env.example | 45 ++ pytest.ini | 1 + scripts/run-backend.sh | 41 ++ tests/integration/__init__.py | 0 tests/integration/browser_backend.mjs | 216 ++++++++ tests/integration/conftest.py | 185 +++++++ tests/integration/crash_server.py | 48 ++ tests/integration/dev_server.py | 87 ++++ tests/integration/test_browser_backend.py | 77 +++ tests/integration/test_crash_reconcile.py | 132 +++++ tests/integration/test_error_mapping.py | 58 +++ tests/integration/test_frontend_serving.py | 98 ++++ tests/integration/test_live_model.py | 97 ++++ tests/integration/test_real_image.py | 474 ++++++++++++++++++ .../integration/test_real_provider_adapter.py | 196 ++++++++ 20 files changed, 1939 insertions(+), 32 deletions(-) create mode 100644 Makefile create mode 100644 deploy/confluence-web.env.example create mode 100755 scripts/run-backend.sh create mode 100644 tests/integration/__init__.py create mode 100644 tests/integration/browser_backend.mjs create mode 100644 tests/integration/conftest.py create mode 100644 tests/integration/crash_server.py create mode 100644 tests/integration/dev_server.py create mode 100644 tests/integration/test_browser_backend.py create mode 100644 tests/integration/test_crash_reconcile.py create mode 100644 tests/integration/test_error_mapping.py create mode 100644 tests/integration/test_frontend_serving.py create mode 100644 tests/integration/test_live_model.py create mode 100644 tests/integration/test_real_image.py create mode 100644 tests/integration/test_real_provider_adapter.py diff --git a/.gitignore b/.gitignore index 1dbc1c0..3e5a334 100644 --- a/.gitignore +++ b/.gitignore @@ -15,3 +15,6 @@ output/ # generated report report/ + +# deployment secrets / local configuration +deploy/*.env diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..69ab773 --- /dev/null +++ b/Makefile @@ -0,0 +1,62 @@ +# Confluence Research Web UI: integration entry points. +# Subsystem guides: agent/README.md, backend/README.md, frontend/README.md. + +PYTHON ?= .venv/bin/python +IMAGE ?= confluence-pi-agent:rev1 +FAKE_IMAGE ?= confluence-fake-agent:test +ENV_FILE ?= deploy/confluence-web.env + +.PHONY: install check check-python check-agent check-frontend check-docker check-browser build-image build-fake-image image-checks run run-dev clean-containers + +install: + $(PYTHON) -m pip install -r requirements.txt + npm --prefix agent ci --ignore-scripts + npm --prefix frontend ci + +## Deterministic checks (no Docker, no network, no browser). +check: check-python check-agent check-frontend + +check-python: + CONFLUENCE_PAT=synthetic-token-no-network CONFLUENCE_URL=https://approved.example.com \ + $(PYTHON) -m pytest --ignore=tests/backend/test_docker_live.py --ignore=tests/integration/test_real_image.py \ + --ignore=tests/integration/test_real_provider_adapter.py --ignore=tests/integration/test_browser_backend.py + +check-agent: + npm --prefix agent run build + npm --prefix agent test + +check-frontend: + npm --prefix frontend test + +## Real rootless Docker checks: backend fake image, runtime image checks, backend+runtime integration. +check-docker: build-fake-image + $(PYTHON) -m pytest tests/backend/test_docker_live.py tests/integration/test_real_image.py tests/integration/test_real_provider_adapter.py -p no:cacheprovider + npm --prefix agent run test:image + +## Real Chrome (DevTools on 127.0.0.1:9444) against the real backend and the frontend's own e2e suite. +check-browser: + $(PYTHON) -m pytest tests/integration/test_browser_backend.py -p no:cacheprovider + npm --prefix frontend run test:e2e + +build-image: + docker build -t $(IMAGE) agent + @docker image inspect --format 'Image ID: {{.Id}}' $(IMAGE) + +build-fake-image: + docker build -t $(FAKE_IMAGE) -f backend/dev/Dockerfile.fake . + +image-checks: + npm --prefix agent run build + node agent/dist/dev/image-checks.js + node agent/dist/dev/boundary-checks.js + +run: + scripts/run-backend.sh $(ENV_FILE) + +## Network-free UI development against explicit fakes (no Docker). +run-dev: + CONFLUENCE_WEB_DEV_MODE=true CONFLUENCE_WEB_FRONTEND_DIST_DIR=$(CURDIR)/frontend \ + $(PYTHON) -m uvicorn backend.app:create_app --factory --workers 1 --host 127.0.0.1 --port 8000 + +clean-containers: + docker ps -aq --filter label=com.confluence_web.app | xargs -r docker rm -f diff --git a/README.md b/README.md index 4144254..cc0aa96 100644 --- a/README.md +++ b/README.md @@ -1,48 +1,115 @@ -# Confluence Crawler (PAT) +# Confluence Research Web UI -Crawls Confluence Data Center pages into Markdown files, authenticated with a -**personal access token (PAT)** from `.env`. +A minimalist web UI that researches a Confluence Data Center instance with a +[pi](https://github.com/earendil-works/pi) agent running in a fresh, network-less +rootless Docker container per query, and returns a cited Markdown answer plus +optional files the agent produced. The design is in +[docs/SPECIFICATION.md](docs/SPECIFICATION.md); the wire contracts in +[docs/implementation/CONTRACTS.md](docs/implementation/CONTRACTS.md); the +integration results in [docs/INTEGRATION_REPORT.md](docs/INTEGRATION_REPORT.md). -> ⚠️ PATs only work on **Confluence Data Center / Server 7.9+**. Confluence -> Cloud does not support PATs — use an API token (basic auth) there instead. +The repository also contains the original standalone +[Confluence crawler](#standalone-crawler), which is unchanged. + +## Components + +| Directory | Role | Developer guide | +| --- | --- | --- | +| `backend/` | FastAPI service: credentials in memory, Confluence/model upstreams, authoritative tool history, container lifecycle, artifact downloads, serves the frontend | [backend/README.md](backend/README.md) | +| `agent/` | Container image: pinned pi SDK bridge, Python supervisor, local tools, artifact exporter | [agent/README.md](agent/README.md) | +| `frontend/` | Static UI (vendored marked + DOMPurify), same-origin mock server for UI development | [frontend/README.md](frontend/README.md) | +| `tests/integration/` | Integration-stage tests connecting the real components | this file | +| `deploy/`, `scripts/`, `Makefile` | Deployment configuration and entry points | this file | + +## Requirements + +- Linux with a **rootless** Docker daemon (cgroup v2, memory/CPU/PID limits enforced). `docker info` must list `rootless` under Security Options. +- Python 3.12+ in `.venv` (verified with 3.14.7), Node 22+ (verified with 26.7) and npm. +- An OpenAI-compatible Chat Completions endpoint the backend can reach (key held by the backend only). +- Network access from the backend host to the approved Confluence origin, optionally through a SOCKS5/HTTP proxy. +- For browser checks: Google Chrome with `--remote-debugging-port=9444`. ## Setup ```bash -python3 -m venv .venv && source .venv/bin/activate -pip install -r requirements.txt -cp .env.example .env # then fill in your values +python3 -m venv .venv && .venv/bin/python -m pip install -r requirements.txt +make install # also installs agent/ and frontend/ dev dependencies +make build-image # builds confluence-pi-agent:rev1 and prints its image ID +cp deploy/confluence-web.env.example deploy/confluence-web.env # then edit ``` -`.env`: +`deploy/confluence-web.env` is git-ignored. Every variable is documented in the +example file and in [backend/README.md](backend/README.md). Pin +`CONFLUENCE_WEB_RUNTIME_IMAGE` to the image ID printed by `make build-image` for +an exact runtime, set a deployment-unique `CONFLUENCE_WEB_CONTAINER_LABEL_VALUE`, +and list only approved Confluence origins. -``` -CONFLUENCE_URL=https://collab.lge.com -CONFLUENCE_PAT= +## Run + +```bash +make run # scripts/run-backend.sh deploy/confluence-web.env ``` -Create the PAT in Confluence: avatar (top right) → Settings → **Personal access -tokens** → Create token. The token inherits your permissions — you can only -crawl pages you can see. +The backend binds `127.0.0.1:8000` by default and serves the UI at +`http://127.0.0.1:8000/`. Open it in a browser, click the key icon, enter the +Confluence base URL and your personal access token (PAT), test the connection, +then ask a question. Credentials live only in browser memory and in the backend +for the duration of a request; a page reload clears them. -## Usage +Version 1 is a single-user deployment: one backend worker, one query at a time +(a second query gets `busy`), bound to loopback. Do not bind it to a network +interface without an authenticating reverse proxy in front; if you add one, it +must accept `6*16 MiB + 64 KiB` request bodies, pass client aborts through +promptly, avoid buffering bodies to disk, and allow at least 200 s per request. + +Network-free UI development against explicit fakes (no Docker, PAT `dev-pat`, +URL `https://approved.example.com`): + +```bash +make run-dev +``` + +## Credentials, expiry and limits + +- **Confluence PAT**: entered in the browser per session, sent only to the backend over the same origin, never stored, logged, or passed to the container. The token inherits your Confluence permissions. +- **Model key**: backend environment only (`CONFLUENCE_WEB_MODEL_API_KEY`). For a local llama.cpp server any placeholder works. +- **Session cookie** `cw_session`: HttpOnly, SameSite=Strict, marks artifact ownership only. It is not authentication. +- **Artifacts**: at most 20 files, 10 MiB each, 50 MiB per query, 500 MiB total; downloadable for 15 minutes after a successful query, then deleted. Failed or cancelled queries keep nothing. +- **Query**: 180 s total deadline plus 10 s cleanup, prompt up to 16 MiB, answer up to 128 MiB, 100 Confluence calls and 50 model calls per query, container limited to 1 GiB RAM, 1 CPU, 128 processes, no network. +- **Model tokens**: `CONFLUENCE_WEB_MODEL_CONTEXT_WINDOW_TOKENS` and `..._MAX_OUTPUT_TOKENS` describe the provider; they are independent of the byte limits above. Large pages or answers can exceed the model's context before the application limits; the UI then shows `model_context_exceeded` or `model_output_limit`. + +## Checks + +| Command | What it runs | Needs | +| --- | --- | --- | +| `make check` | Crawler + backend + integration (no Docker) pytest, agent host tests, frontend unit/contract tests | nothing external | +| `make check-docker` | Backend real-Docker checks, backend + real runtime image + scripted peers, real provider adapter + runtime, runtime image/isolation/boundary checks | rootless Docker, built images | +| `make check-browser` | Real Chrome against the real backend (scripted runtime), frontend's own e2e suite | Chrome on `127.0.0.1:9444`, Node | + +Each subsystem's own commands are documented in its guide and can be run independently: + +```bash +CONFLUENCE_PAT=synthetic-token-no-network CONFLUENCE_URL=https://approved.example.com .venv/bin/python -m pytest tests/backend +npm --prefix agent test && npm --prefix agent run proof +npm --prefix frontend test && npm --prefix frontend run test:e2e +``` + +Fake modes (`CONFLUENCE_WEB_DEV_MODE`, the backend fake image, the frontend mock +server, the runtime's fake backend peer) are explicit opt-ins and never activate +on a production failure. + +## Standalone crawler + +Crawls Confluence Data Center pages into Markdown files using a PAT from `.env` +(`CONFLUENCE_URL`, `CONFLUENCE_PAT`; see `.env.example`). PATs require +Confluence Data Center / Server 7.9+. ```bash python main.py --space KEY --out output/KEY # one space python main.py --all --out output # all visible spaces -python main.py --all --max-spaces 3 --verbose # limit + debug logging +pytest -m live # live crawler tests (uses .env) ``` -Each page becomes `output//_.md` with YAML front matter -(title, page id, space, URL, version, last modified) and the body converted -from Confluence storage format to Markdown. - -## Tests - -```bash -pytest # unit tests + live tests (live uses the PAT from .env) -pytest -m "not live" # unit tests only, no network -pytest -m live # live tests only -``` - -Live tests are skipped automatically when `.env` lacks credentials. +Each page becomes `output//_.md` with YAML front matter. +The web backend imports only the crawler's pure conversion functions; it never +reads `.env`. diff --git a/backend/app.py b/backend/app.py index 62df801..89383d8 100644 --- a/backend/app.py +++ b/backend/app.py @@ -486,7 +486,7 @@ def create_app( await asyncio.to_thread(store.release_reader, artifact_id) raise asyncio.CancelledError if not res: - raise ArtifactNotFoundError("Artifact not found or expired") + raise ArtifactNotFoundError("Artifact not found") # identical to unknown-ID and no-session responses file_path, display_name, size_bytes = res diff --git a/backend/runner.py b/backend/runner.py index dfef991..ec34886 100644 --- a/backend/runner.py +++ b/backend/runner.py @@ -12,7 +12,16 @@ from typing import Any, Callable, Dict, Optional from backend.artifacts import ArtifactStore, QueryArtifactStaging from backend.confluence import ConfluenceClient, ConfluenceDispatcher from backend.containers import ContainerHandle, ContainerManager -from backend.errors import BusyError, CleanupFailedError, ExecutionFailedError, InvalidInputError, QueryTimeoutError +from backend.errors import ( + BusyError, + CleanupFailedError, + ConnectivityFailedError, + ExecutionFailedError, + InvalidInputError, + ModelContextExceededError, + ModelOutputLimitError, + QueryTimeoutError, +) from backend.history import HistoryManager, WarningsManager, rfc3339_utc from backend.model import ModelAdapter, ModelDispatcher from backend.settings import Settings @@ -345,6 +354,17 @@ class QueryRunner: elif msg_type == "error": payload = msg.get("payload", {}) + runtime_code = payload.get("code") if isinstance(payload, dict) else None + # Map the runtime's terminal code onto the contract's HTTP mapping with + # fixed backend messages; the runtime message text is never surfaced. + if runtime_code == "model_output_limit": + raise ModelOutputLimitError("Model output limit reached before the answer completed") + if runtime_code == "model_context_exceeded": + raise ModelContextExceededError("Model context limit exceeded during the run") + if runtime_code == "query_timeout": + raise QueryTimeoutError("Query execution timed out") + if runtime_code == "connectivity_failed": + raise ConnectivityFailedError("Agent lost connectivity to the backend bridge") raise ExecutionFailedError("Agent terminated with an execution error") elif msg_type == "eof": diff --git a/deploy/confluence-web.env.example b/deploy/confluence-web.env.example new file mode 100644 index 0000000..9dfa41e --- /dev/null +++ b/deploy/confluence-web.env.example @@ -0,0 +1,45 @@ +# Confluence Research Web UI: backend deployment configuration. +# Copy to deploy/confluence-web.env (git-ignored) and adjust. Values are read by +# scripts/run-backend.sh and exported only into the backend process. Nothing in +# this file is ever passed to the agent container. + +# --- Confluence (read-only, user PAT is supplied in the browser at query time) --- +# Approved base origins (comma separated, include context path if any). +CONFLUENCE_WEB_APPROVED_ORIGINS=https://collab.lge.com +# Optional outbound proxy for Confluence only (socks5://, socks5h://, http://, https://). +CONFLUENCE_WEB_CONFLUENCE_PROXY=socks5://127.0.0.1:1560 +# Optional corporate CA bundle (PEM). Leave unset to use the system trust store. +#CONFLUENCE_WEB_CORPORATE_CA_PATH=/etc/ssl/certs/corporate-ca.pem + +# --- Model provider (OpenAI-compatible Chat Completions; backend-held key) --- +CONFLUENCE_WEB_MODEL_PROVIDER=openai +# llama.cpp server tunnel; the endpoint is the full chat completions URL. +CONFLUENCE_WEB_MODEL_ENDPOINT=http://127.0.0.1:4901/v1/chat/completions +# llama-server accepts any bearer token; a real provider needs its real key here. +CONFLUENCE_WEB_MODEL_API_KEY=llama-cpp +CONFLUENCE_WEB_MODEL_NAME=Qwen3.6-35B-A3B +# Provider token limits (separate from the application byte limits in CONTRACTS.md). +CONFLUENCE_WEB_MODEL_CONTEXT_WINDOW_TOKENS=131072 +CONFLUENCE_WEB_MODEL_MAX_OUTPUT_TOKENS=8192 +# Per-call HTTP timeout; keep below the 180 s query deadline. +CONFLUENCE_WEB_MODEL_TIMEOUT_SECONDS=170 + +# --- Agent runtime container (rootless Docker) --- +# Use the exact built tag or, better, the image ID printed by `make build-image`. +CONFLUENCE_WEB_RUNTIME_IMAGE=confluence-pi-agent:rev1 +# Unique per deployment; startup/periodic reconciliation removes containers with this label value. +CONFLUENCE_WEB_CONTAINER_LABEL_VALUE=confluence-web-local +#CONFLUENCE_WEB_DOCKER_HOST=unix:///run/user/1000/docker.sock + +# --- HTTP / storage --- +CONFLUENCE_WEB_BIND_HOST=127.0.0.1 +CONFLUENCE_WEB_BIND_PORT=8000 +# Absolute path to the frontend tree (index.html, css/, js/, vendor/). +CONFLUENCE_WEB_FRONTEND_DIST_DIR=./frontend +# Private artifact storage (created 0700; purged on startup). +CONFLUENCE_WEB_ARTIFACT_DIR=/tmp/confluence_web_artifacts +CONFLUENCE_WEB_QUERY_TIMEOUT_SECONDS=180 +CONFLUENCE_WEB_CLEANUP_TIMEOUT_SECONDS=10 + +# Never set in production. Substitutes the container, model and Confluence with fakes. +CONFLUENCE_WEB_DEV_MODE=false diff --git a/pytest.ini b/pytest.ini index 2b716b6..dc7c706 100644 --- a/pytest.ini +++ b/pytest.ini @@ -4,3 +4,4 @@ pythonpath = . addopts = -q -m "not live" markers = live: tests that hit real Confluence or external network services + integration: integration-stage tests that connect real subsystems (may need Docker and the runtime image) diff --git a/scripts/run-backend.sh b/scripts/run-backend.sh new file mode 100755 index 0000000..6cb8d47 --- /dev/null +++ b/scripts/run-backend.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash +# Start the Confluence Research backend (serves the frontend on the same origin). +# +# scripts/run-backend.sh [deploy/confluence-web.env] +# +# Loads the env file, checks the runtime image and rootless daemon, then runs a +# single Uvicorn worker bound to CONFLUENCE_WEB_BIND_HOST:PORT (loopback by default). +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +ENV_FILE="${1:-$ROOT/deploy/confluence-web.env}" +PYTHON="${PYTHON:-$ROOT/.venv/bin/python}" + +if [[ ! -f "$ENV_FILE" ]]; then + echo "env file not found: $ENV_FILE (copy deploy/confluence-web.env.example)" >&2 + exit 1 +fi +set -a +# shellcheck disable=SC1090 +. "$ENV_FILE" +set +a + +# Resolve a relative frontend directory against the repository root. +if [[ -n "${CONFLUENCE_WEB_FRONTEND_DIST_DIR:-}" && "${CONFLUENCE_WEB_FRONTEND_DIST_DIR}" != /* ]]; then + export CONFLUENCE_WEB_FRONTEND_DIST_DIR="$ROOT/${CONFLUENCE_WEB_FRONTEND_DIST_DIR#./}" +fi + +if [[ "${CONFLUENCE_WEB_DEV_MODE:-false}" != "true" ]]; then + if ! docker info --format '{{json .SecurityOptions}}' 2>/dev/null | grep -q rootless; then + echo "rootless Docker daemon is required (docker context use rootless)" >&2 + exit 1 + fi + if ! docker image inspect "${CONFLUENCE_WEB_RUNTIME_IMAGE:?}" >/dev/null 2>&1; then + echo "runtime image ${CONFLUENCE_WEB_RUNTIME_IMAGE} not found; run: make build-image" >&2 + exit 1 + fi +fi + +exec "$PYTHON" -m uvicorn backend.app:create_app --factory --workers 1 \ + --host "${CONFLUENCE_WEB_BIND_HOST:-127.0.0.1}" --port "${CONFLUENCE_WEB_BIND_PORT:-8000}" \ + --no-server-header --timeout-keep-alive 5 --limit-concurrency 32 --app-dir "$ROOT" diff --git a/tests/integration/__init__.py b/tests/integration/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/integration/browser_backend.mjs b/tests/integration/browser_backend.mjs new file mode 100644 index 0000000..bcd4c72 --- /dev/null +++ b/tests/integration/browser_backend.mjs @@ -0,0 +1,216 @@ +/** + * Browser <-> backend pair check. + * + * Drives the real frontend served by the real backend (scripted runtime peer) + * in a real Chrome via CDP. Usage: + * node tests/integration/browser_backend.mjs --app http://127.0.0.1:8765 --control /path/to/control --chrome 9444 + */ +import assert from 'node:assert/strict'; +import { writeFileSync } from 'node:fs'; + +const args = Object.fromEntries(process.argv.slice(2).reduce((acc, a, i, arr) => { + if (a.startsWith('--')) acc.push([a.slice(2), arr[i + 1]]); + return acc; +}, [])); +const APP = (args.app || 'http://127.0.0.1:8765').replace(/\/$/, ''); +const CONTROL = args.control; +const CHROME_PORT = Number(args.chrome || 9444); +const setScenario = (name) => writeFileSync(CONTROL, name + '\n'); +const sleep = (ms) => new Promise((r) => setTimeout(r, ms)); + +class CDP { + constructor(ws) { this.wsUrl = ws; this.id = 1; this.pending = new Map(); this.requests = []; this.responses = []; } + async connect() { + this.ws = new WebSocket(this.wsUrl); + await new Promise((res, rej) => { this.ws.onopen = res; this.ws.onerror = rej; }); + this.ws.onmessage = (e) => { + const d = JSON.parse(e.data); + if (d.id && this.pending.has(d.id)) { const p = this.pending.get(d.id); this.pending.delete(d.id); d.error ? p.reject(new Error(JSON.stringify(d.error))) : p.resolve(d.result); } + else if (d.method === 'Network.requestWillBeSent') this.requests.push(d.params.request); + else if (d.method === 'Network.responseReceived') this.responses.push({ url: d.params.response.url, status: d.params.response.status }); + }; + await this.send('Runtime.enable'); await this.send('Network.enable'); await this.send('Page.enable'); + } + send(method, params = {}) { return new Promise((resolve, reject) => { const id = this.id++; this.pending.set(id, { resolve, reject }); this.ws.send(JSON.stringify({ id, method, params })); }); } + async eval(expression) { + const r = await this.send('Runtime.evaluate', { expression, returnByValue: true, awaitPromise: true }); + if (r.exceptionDetails) throw new Error(`eval: ${r.exceptionDetails.text} ${r.exceptionDetails.exception?.description || ''}`); + return r.result?.value; + } + async navigate(url) { await this.send('Page.navigate', { url }); await sleep(900); } + async waitFor(expression, timeoutMs = 15000, label = expression) { + const t0 = Date.now(); + while (Date.now() - t0 < timeoutMs) { if (await this.eval(expression)) return; await sleep(100); } + const fb = await this.eval('(document.getElementById("modal-feedback")||{}).textContent + " | " + (document.getElementById("prompt-error")||{}).textContent'); + throw new Error(`timeout waiting for ${label}; feedback=${JSON.stringify(fb)}; responses=${JSON.stringify(this.responses.slice(-6))}`); + } + close() { this.ws?.close(); } +} + +const visible = (id) => `!document.getElementById("${id}").classList.contains("hidden")`; + +async function main() { + const tabs = await (await fetch(`http://127.0.0.1:${CHROME_PORT}/json`)).json(); + const tab = tabs.find((t) => t.type === 'page') || tabs[0]; + if (!tab?.webSocketDebuggerUrl) throw new Error('no Chrome page tab'); + const cdp = new CDP(tab.webSocketDebuggerUrl); + await cdp.connect(); + let passed = 0, failed = 0; + const step = async (name, fn) => { + process.stdout.write(`• ${name}... `); + try { await fn(); console.log('PASS'); passed++; } catch (e) { console.log(`FAIL: ${e.message}`); failed++; } + }; + + try { + setScenario('standard'); + await cdp.send('Network.clearBrowserCookies'); + await cdp.navigate(APP + '/'); + + await step('Bootstrap: real index, HttpOnly cookie, headers', async () => { + assert.equal(await cdp.eval('document.title'), 'Confluence Research'); + assert.ok(await cdp.eval(visible('view-prompt'))); + const { cookies } = await cdp.send('Network.getCookies', { urls: [APP + '/'] }); + const c = cookies.find((k) => k.name === 'cw_session'); + assert.ok(c, 'cw_session cookie must be set by GET /'); + assert.equal(c.httpOnly, true); assert.equal(c.sameSite, 'Strict'); assert.equal(c.path, '/'); + assert.ok(!(await cdp.eval('document.cookie')).includes('cw_session'), 'cookie must not be script-readable'); + // Assets loaded from the same origin with a real stylesheet applied. + const bg = await cdp.eval('getComputedStyle(document.body).backgroundColor'); + assert.notEqual(bg, '', 'stylesheet must load'); + assert.ok(await cdp.eval('typeof marked !== "undefined" && typeof DOMPurify !== "undefined"'), 'vendored libs must load'); + }); + + await step('Verify credentials against the backend (403 then 200)', async () => { + await cdp.eval('document.getElementById("key-btn").click()'); + await cdp.eval('(() => { for (const [id, v] of [["cred-url", "https://approved.example.com"], ["cred-pat", "wrong-pat"]]) { const el = document.getElementById(id); el.value = v; el.dispatchEvent(new Event("input", { bubbles: true })); } })()'); + await cdp.eval('document.getElementById("btn-test-cred").click()'); + await cdp.waitFor('document.getElementById("modal-feedback").textContent.length > 0', 5000, 'verify feedback'); + const bad = await cdp.eval('document.getElementById("modal-feedback").textContent'); + assert.ok(!/successful/i.test(bad), `wrong PAT must not verify: ${bad}`); + // Retype the token as a user would: the input event aborts any stale verify and re-enables the button. + await cdp.eval('(() => { const p = document.getElementById("cred-pat"); p.value = "dev-pat"; p.dispatchEvent(new Event("input", { bubbles: true })); })()'); + await cdp.eval('document.getElementById("btn-test-cred").click()'); + await cdp.waitFor('/successful/i.test(document.getElementById("modal-feedback").textContent)', 5000, 'success feedback'); + await cdp.eval('document.getElementById("btn-save-cred").click()'); + await sleep(200); + assert.ok(await cdp.eval('document.getElementById("cred-indicator").classList.contains("active")')); + assert.ok(!(await cdp.eval('JSON.stringify(localStorage) + JSON.stringify(sessionStorage) + location.href')).includes('dev-pat')); + }); + + await step('Query through the real backend: statuses, history, sources', async () => { + await cdp.eval('document.getElementById("prompt-input").value = "How do I deploy service X?"'); + await cdp.eval('document.getElementById("submit-btn").click()'); + assert.ok(await cdp.eval(visible('view-loading')), 'loading view'); + await cdp.waitFor(visible('view-result'), 20000, 'result view'); + const text = await cdp.eval('document.getElementById("output-content").textContent'); + assert.ok(text.includes('Deployment Guide'), text.slice(0, 200)); + const link = await cdp.eval('(() => { const a = document.querySelector("#output-content a"); return a && { target: a.target, rel: a.rel, href: a.href }; })()'); + assert.deepEqual(link, { target: '_blank', rel: 'noopener noreferrer', href: 'https://approved.example.com/pages/viewpage.action?pageId=847291' }); + assert.ok((await cdp.eval('document.getElementById("history-toggle-title").textContent')).includes('1 page accessed')); + await cdp.eval('document.getElementById("history-toggle-btn").click()'); + assert.equal(await cdp.eval('document.querySelectorAll(".tool-card").length'), 2); + assert.equal(await cdp.eval('document.querySelector(".page-title-link").textContent'), 'Deployment Guide'); + const q = cdp.requests.filter((r) => r.url.endsWith('/api/v1/query')); + assert.ok(q.length >= 1 && q[q.length - 1].method === 'POST'); + // The backend enforces a matching Origin on mutations; a 200 proves the browser sent it. + const qr = cdp.responses.filter((r) => r.url.endsWith('/api/v1/query')); + assert.equal(qr[qr.length - 1].status, 200); + }); + + await step('Artifact download: exact bytes, attachment headers, ownership', async () => { + assert.equal(await cdp.eval('document.querySelector(".artifact-name").textContent'), 'checklist.md'); + cdp.requests.length = 0; + await cdp.eval('document.querySelector(".download-btn").click()'); + await sleep(600); + assert.ok(await cdp.eval('document.getElementById("artifacts-error").classList.contains("hidden")'), 'download error box hidden'); + const dl = cdp.requests.find((r) => r.url.includes('/api/v1/artifacts/')); + assert.ok(dl, 'download request observed'); + const check = await cdp.eval(`fetch(${JSON.stringify(dl.url)}).then(async r => ({ status: r.status, ct: r.headers.get('content-type'), cd: r.headers.get('content-disposition'), xcto: r.headers.get('x-content-type-options'), cc: r.headers.get('cache-control'), body: new TextDecoder().decode(await r.arrayBuffer()) }))`); + assert.equal(check.status, 200); + assert.equal(check.body, '# Checklist\n\n- Deploy service X\n'); + assert.ok(check.ct.startsWith('application/octet-stream')); + assert.ok(check.cd.startsWith('attachment')); + assert.equal(check.xcto, 'nosniff'); assert.equal(check.cc, 'no-store'); + const unknown = await cdp.eval(`fetch('/api/v1/artifacts/not-a-real-id').then(async r => ({ status: r.status, body: await r.text() }))`); + assert.equal(unknown.status, 404); + assert.equal(JSON.parse(unknown.body).error.code, 'artifact_not_found'); + // Another browser session (fresh cookie jar) gets the same 404. + const other = await cdp.eval(`fetch(${JSON.stringify(dl.url)}, { credentials: 'omit' }).then(async r => ({ status: r.status, body: await r.text() }))`); + assert.equal(other.status, 404); + assert.equal(other.body, unknown.body); + }); + + await step('Cancel during run: server observes abort, next query succeeds', async () => { + await cdp.eval('document.getElementById("back-btn").click()'); + setScenario('timeout'); + await cdp.eval('document.getElementById("submit-btn").click()'); + await sleep(400); + assert.ok(await cdp.eval(visible('view-loading'))); + await cdp.eval('document.getElementById("cancel-btn").click()'); + await cdp.waitFor(visible('view-prompt'), 5000, 'prompt after cancel'); + assert.equal(await cdp.eval('document.getElementById("prompt-input").value'), 'How do I deploy service X?'); + setScenario('standard'); + // Retry until the busy gate releases after cleanup. + let ok = false; + for (let i = 0; i < 20 && !ok; i++) { + await cdp.eval('document.getElementById("submit-btn").click()'); + await sleep(500); + ok = await cdp.eval(visible('view-result')); + if (!ok) { + const err = await cdp.eval('document.getElementById("prompt-error").textContent'); + assert.ok(/busy/i.test(err) || err === '', `unexpected error: ${err}`); + } + } + assert.ok(ok, 'query after cancellation must succeed'); + await cdp.eval('document.getElementById("back-btn").click()'); + }); + + await step('Execution failure shows sanitized error and recovers', async () => { + setScenario('agent_error'); + await cdp.eval('document.getElementById("submit-btn").click()'); + await cdp.waitFor('document.getElementById("prompt-error").textContent.length > 0', 10000, 'error shown'); + const err = await cdp.eval('document.getElementById("prompt-error").textContent'); + assert.ok(!/traceback|exception|stack/i.test(err), err); + assert.ok(await cdp.eval(visible('view-prompt'))); + setScenario('standard'); + }); + + await step('Malicious Markdown/history under real headers: no script, no external requests', async () => { + setScenario('malicious'); + cdp.requests.length = 0; + await cdp.eval('document.getElementById("submit-btn").click()'); + await cdp.waitFor(visible('view-result'), 20000, 'result view'); + await sleep(800); + assert.equal(await cdp.eval('document.querySelectorAll("#output-content script, #output-content img, #output-content iframe, #output-content form").length'), 0); + assert.equal(await cdp.eval('window.__xss'), undefined); + const hrefs = await cdp.eval('Array.from(document.querySelectorAll("#output-content a")).map(a => a.getAttribute("href"))'); + assert.ok(hrefs.every((h) => !/^javascript:/i.test(h || '')), JSON.stringify(hrefs)); + assert.ok(hrefs.some((h) => h && h.startsWith('https://approved.example.com')), 'citation kept'); + await cdp.eval('document.getElementById("history-toggle-btn").click()'); + await sleep(200); + assert.equal(await cdp.eval('document.querySelectorAll("#view-result script").length'), 0); + const external = cdp.requests.filter((r) => !r.url.startsWith(APP + '/') && !r.url.startsWith('data:') && !r.url.startsWith('blob:')); + assert.deepEqual(external.map((r) => r.url), [], 'no automatic external requests'); + setScenario('standard'); + await cdp.eval('document.getElementById("back-btn").click()'); + }); + + await step('Clear credentials blocks queries', async () => { + await cdp.eval('document.getElementById("key-btn").click()'); + const clearBtn = await cdp.eval('!!document.getElementById("btn-clear-cred")'); + if (clearBtn) { + await cdp.eval('document.getElementById("btn-clear-cred").click()'); + await sleep(200); + assert.ok(!(await cdp.eval('document.getElementById("cred-indicator").classList.contains("active")'))); + } else { + await cdp.eval('document.getElementById("btn-cancel-cred").click()'); + } + }); + } finally { + cdp.close(); + } + console.log(`\n${passed} passed, ${failed} failed`); + process.exit(failed ? 1 : 0); +} + +main().catch((e) => { console.error(e); process.exit(2); }); diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py new file mode 100644 index 0000000..a7d959a --- /dev/null +++ b/tests/integration/conftest.py @@ -0,0 +1,185 @@ +"""Shared helpers for integration tests that connect the real subsystems. + +These tests are owned by the integration stage. They never contact real +services: Confluence is a scripted httpx transport, the model is a scripted +adapter or a scripted OpenAI-compatible transport, and the only real +component is the built pi runtime image executed through the backend's +rootless Docker manager. +""" + +from __future__ import annotations + +import asyncio +import os +import shutil +import subprocess +from pathlib import Path +from typing import Any, Dict, List, Optional, Sequence, Tuple + +import httpx +import pytest + +from backend.confluence import ConfluenceClient +from backend.model import FakeModelAdapter + +REPO_ROOT = Path(__file__).resolve().parents[2] +FRONTEND_DIR = REPO_ROOT / "frontend" +RUNTIME_IMAGE = os.getenv("CONFLUENCE_WEB_RUNTIME_IMAGE", "confluence-pi-agent:rev1") + +APPROVED_ORIGIN = "https://approved.example.com" +PAGE_ID = "847291" +PAGE_URL = f"{APPROVED_ORIGIN}/pages/viewpage.action?pageId={PAGE_ID}" +DENIED_PAGE_ID = "999999" +CHECKLIST_BYTES = b"# Checklist\n\n- Deploy service X\n" +CHECKLIST_TEXT = CHECKLIST_BYTES.decode("utf-8") +# A recognizable dummy PAT used only in fixtures; it must never appear in the container. +CANARY_PAT = "canary-pat-do-not-leak-0123456789abcdef" +FINAL_ANSWER = ( + "Deploy service X using the release checklist " + f"([Deployment Guide]({PAGE_URL})). A checklist.md artifact was created." +) + + +def docker_available() -> bool: + if shutil.which("docker") is None: + return False + try: + res = subprocess.run(["docker", "info"], capture_output=True, timeout=5.0) + return res.returncode == 0 + except Exception: + return False + + +def image_available(image: str) -> bool: + try: + res = subprocess.run(["docker", "image", "inspect", image], capture_output=True, timeout=10.0) + return res.returncode == 0 + except Exception: + return False + + +def requires_runtime_image(): + return pytest.mark.skipif( + not (docker_available() and image_available(RUNTIME_IMAGE)), + reason=f"rootless Docker and runtime image {RUNTIME_IMAGE} are required", + ) + + +# --- scripted Confluence upstream ------------------------------------------------- + +def make_confluence_factory(calls: Optional[List[Dict[str, Any]]] = None): + """Return a ConfluenceClient factory backed by a scripted httpx transport. + + Accepts the same keyword arguments as the production factory, including + the optional ``proxy`` argument that production passes when configured. + """ + recorded = calls if calls is not None else [] + + async def handler(request: httpx.Request) -> httpx.Response: + recorded.append({"path": request.url.path, "params": dict(request.url.params)}) + if request.headers.get("Authorization") != f"Bearer {CANARY_PAT}": + return httpx.Response(401, json={"message": "Unauthorized"}) + path = request.url.path + if path.endswith("/rest/api/space"): + return httpx.Response(200, json={"results": [{"key": "OPS", "name": "Operations"}], "totalSize": 1}) + if path.endswith("/rest/api/content/search"): + cql = request.url.params.get("cql", "") + if "nothing" in cql: + return httpx.Response(200, json={"results": [], "totalSize": 0}) + return httpx.Response( + 200, + json={ + "results": [ + {"id": PAGE_ID, "title": "Deployment Guide", "space": {"key": "OPS"}, "excerpt": "Deployment steps"} + ], + "totalSize": 1, + }, + ) + if path.endswith(f"/rest/api/content/{PAGE_ID}"): + return httpx.Response( + 200, + json={ + "id": PAGE_ID, + "title": "Deployment Guide", + "space": {"key": "OPS"}, + "body": {"storage": {"value": "

Deploy service X using the release checklist.

"}}, + }, + ) + if path.endswith(f"/rest/api/content/{DENIED_PAGE_ID}"): + return httpx.Response(403, json={"message": "SECRET-UPSTREAM-BODY forbidden"}) + return httpx.Response(404, json={"message": "not found"}) + + def factory(base_url: str, pat: str, approved_origins: list, corporate_ca_path=None, timeout: float = 30.0, **kwargs) -> ConfluenceClient: + return ConfluenceClient( + base_url=base_url, + pat=pat, + approved_origins=approved_origins, + corporate_ca_path=corporate_ca_path, + timeout=timeout, + transport=httpx.MockTransport(handler), + ) + + return factory + + +# --- scripted neutral model turns --------------------------------------------------- + +def text_turn(text: str, stop_reason: str = "stop", in_tok: int = 10, out_tok: int = 10) -> Dict[str, Any]: + return { + "content": [{"type": "text", "text": text}] if text else [], + "stop_reason": stop_reason, + "usage": {"input_tokens": in_tok, "output_tokens": out_tok}, + } + + +def tool_turn(calls: Sequence[Tuple[str, str, Dict[str, Any]]], text: str = "") -> Dict[str, Any]: + content: List[Dict[str, Any]] = [] + if text: + content.append({"type": "text", "text": text}) + for call_id, name, args in calls: + content.append({"type": "tool_call", "id": call_id, "name": name, "arguments": args}) + return {"content": content, "stop_reason": "tool_calls", "usage": {"input_tokens": 10, "output_tokens": 5}} + + +def example_script(final_text: str = FINAL_ANSWER) -> List[Dict[str, Any]]: + """The CONTRACTS section 7 shared example as scripted model turns.""" + return [ + tool_turn([("call_1", "confluence_search", {"query": "deploy service X"})]), + tool_turn([("call_2", "confluence_view", {"page_id": PAGE_ID})]), + tool_turn([("call_3", "write", {"path": "/work/artifacts/checklist.md", "content": CHECKLIST_TEXT})]), + text_turn(final_text), + ] + + +class ScriptedModelAdapter(FakeModelAdapter): + """FakeModelAdapter that can also raise or stall on a scripted turn.""" + + def __init__(self, script: List[Any]): + super().__init__(script=None) + self._steps: List[Any] = list(script) + self.started = asyncio.Event() + + async def complete(self, messages, tools, system_instruction, provider_state_meta=None): + self.call_count += 1 + self.received_requests.append({ + "messages": messages, + "tools": tools, + "system_instruction": system_instruction, + "provider_state_meta": provider_state_meta, + }) + self.started.set() + if not self._steps: + raise AssertionError("Scripted model adapter ran out of turns") + step = self._steps.pop(0) + if isinstance(step, Exception): + raise step + if callable(step): + return await step() + return step + + +def tool_message_text(request: Dict[str, Any], tool_call_id: str) -> str: + for m in request["messages"]: + if m.get("role") == "tool" and m.get("tool_call_id") == tool_call_id: + return m.get("content", "") + raise AssertionError(f"no tool message for {tool_call_id}") diff --git a/tests/integration/crash_server.py b/tests/integration/crash_server.py new file mode 100644 index 0000000..fc002ae --- /dev/null +++ b/tests/integration/crash_server.py @@ -0,0 +1,48 @@ +"""Test-owned launcher: real backend + real runtime image with a stalling model. + +Used by the backend-crash / restart-reconciliation check. Confluence is the +scripted transport; the model stalls so a container stays alive while the +backend process is killed. +""" + +from __future__ import annotations + +import asyncio +import os +from pathlib import Path + +import uvicorn + +from backend.app import create_app +from backend.artifacts import ArtifactStore +from backend.model import FakeModelAdapter +from backend.settings import Settings + +from tests.integration.conftest import APPROVED_ORIGIN, RUNTIME_IMAGE, make_confluence_factory + + +class StallingModel(FakeModelAdapter): + async def complete(self, messages, tools, system_instruction, provider_state_meta=None): + await asyncio.sleep(600) + raise AssertionError("unreachable") + + +def main() -> None: + settings = Settings( + runtime_image=RUNTIME_IMAGE, + approved_confluence_origins=[APPROVED_ORIGIN], + container_label_value=os.environ["CW_LABEL_VALUE"], + artifact_storage_dir=Path(os.environ["CW_ARTIFACT_DIR"]), + query_timeout_seconds=170.0, + ) + app = create_app( + settings=settings, + artifact_store=ArtifactStore(settings.artifact_storage_dir), + model_adapter=StallingModel(), + confluence_client_factory=make_confluence_factory(), + ) + uvicorn.run(app, host="127.0.0.1", port=int(os.environ["CW_DEV_PORT"]), workers=1, log_level="warning") + + +if __name__ == "__main__": + main() diff --git a/tests/integration/dev_server.py b/tests/integration/dev_server.py new file mode 100644 index 0000000..98871f3 --- /dev/null +++ b/tests/integration/dev_server.py @@ -0,0 +1,87 @@ +"""Test-owned launcher: real backend HTTP server with a scripted runtime peer. + +Used by the browser <-> backend pair check. The container is a scripted +in-process peer (no Docker), the model is the fake adapter and Confluence is +the network-free dev substitute. The scenario for the *next* query is read +from a control file so a browser test can switch behaviors between queries +without any dev-only endpoint existing in the production application. +""" + +from __future__ import annotations + +import os +import sys +from pathlib import Path + +import uvicorn + +from backend.app import create_app +from backend.artifacts import ArtifactStore +from backend.containers import FakeContainerManager +from backend.dev.fake_confluence import create_client +from backend.dev.fake_peer import ScriptedContainerPeer +from backend.model import FakeModelAdapter +from backend.settings import Settings + +REPO_ROOT = Path(__file__).resolve().parents[2] +CONTROL = Path(os.environ["CW_PEER_CONTROL_FILE"]) + +MALICIOUS_MARKDOWN = ( + "# Untrusted answer\n\n" + '\n\n' + '\n\n' + "![tracker](https://evil.example.net/track.png)\n\n" + '\n\n' + '
\n\n' + "[js link](javascript:alert(1))\n\n" + "Legit citation: [Deployment Guide](https://approved.example.com/pages/viewpage.action?pageId=847291)\n" +) + + +async def malicious_steps(reader, writer, peer): + start = await peer._read_line(reader) + assert start.get("type") == "start" + await peer._write_msg(writer, { + "v": 1, "type": "tool_request", "id": "a_1", + "payload": {"tool": "confluence_search", "parameters": {"query": ""}}, + }) + await peer._read_line(reader) + await peer._write_msg(writer, { + "v": 1, "type": "collection_start", "id": "a_2", + "payload": {"markdown": MALICIOUS_MARKDOWN, "warnings": [{"code": "history_truncated", "message": "bold warning"}]}, + }) + ready = await peer._read_line(reader) + assert ready.get("type") == "collection_ready" + await peer._write_msg(writer, {"v": 1, "type": "complete", "id": "a_3", "payload": {"accepted_transfer_count": 0}}) + + +def peer_factory(): + scenario = CONTROL.read_text(encoding="utf-8").strip() if CONTROL.exists() else "standard" + if scenario == "malicious": + return ScriptedContainerPeer(custom_steps=malicious_steps) + return ScriptedContainerPeer(scenario=scenario) + + +def main() -> None: + port = int(os.environ.get("CW_DEV_PORT", "8765")) + settings = Settings( + dev_mode=True, + frontend_dist_dir=REPO_ROOT / "frontend", + artifact_storage_dir=Path(os.environ["CW_ARTIFACT_DIR"]), + bind_host="127.0.0.1", + bind_port=port, + query_timeout_seconds=30.0, + cleanup_timeout_seconds=5.0, + ) + app = create_app( + settings=settings, + container_manager=FakeContainerManager(peer_factory), + artifact_store=ArtifactStore(settings.artifact_storage_dir), + model_adapter=FakeModelAdapter(), + confluence_client_factory=create_client, + ) + uvicorn.run(app, host="127.0.0.1", port=port, workers=1, log_level="warning") + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/integration/test_browser_backend.py b/tests/integration/test_browser_backend.py new file mode 100644 index 0000000..fbc3722 --- /dev/null +++ b/tests/integration/test_browser_backend.py @@ -0,0 +1,77 @@ +"""Browser <-> backend pair: real Chrome (CDP) against the real backend HTTP server. + +Requires Node and a Chrome instance exposing the DevTools protocol on +127.0.0.1:9444 (see frontend/README.md). Skips otherwise. +""" + +from __future__ import annotations + +import os +import shutil +import socket +import subprocess +import sys +import time +import urllib.request +from pathlib import Path + +import pytest + +from tests.integration.conftest import REPO_ROOT + +pytestmark = pytest.mark.integration + +CHROME_PORT = int(os.getenv("CW_CHROME_CDP_PORT", "9444")) + + +def chrome_available() -> bool: + try: + with urllib.request.urlopen(f"http://127.0.0.1:{CHROME_PORT}/json/version", timeout=2) as r: + return r.status == 200 + except Exception: + return False + + +def free_port() -> int: + with socket.socket() as s: + s.bind(("127.0.0.1", 0)) + return s.getsockname()[1] + + +@pytest.mark.skipif(shutil.which("node") is None or not chrome_available(), reason="node and Chrome CDP on 9444 required") +def test_browser_against_real_backend(tmp_path: Path): + port = free_port() + control = tmp_path / "scenario" + control.write_text("standard\n") + env = { + "PATH": os.environ["PATH"], + "CW_PEER_CONTROL_FILE": str(control), + "CW_ARTIFACT_DIR": str(tmp_path / "artifacts"), + "CW_DEV_PORT": str(port), + "PYTHONPATH": str(REPO_ROOT), + } + server = subprocess.Popen([sys.executable, str(REPO_ROOT / "tests/integration/dev_server.py")], env=env, cwd=REPO_ROOT) + try: + deadline = time.time() + 30 + while time.time() < deadline: + try: + with urllib.request.urlopen(f"http://127.0.0.1:{port}/", timeout=1) as r: + if r.status == 200: + break + except Exception: + time.sleep(0.2) + else: + raise AssertionError("dev server did not start") + res = subprocess.run( + ["node", str(REPO_ROOT / "tests/integration/browser_backend.mjs"), "--app", f"http://127.0.0.1:{port}", "--control", str(control), "--chrome", str(CHROME_PORT)], + capture_output=True, text=True, timeout=300, cwd=REPO_ROOT, + ) + print(res.stdout) + print(res.stderr, file=sys.stderr) + assert res.returncode == 0, res.stdout + res.stderr + finally: + server.terminate() + try: + server.wait(timeout=10) + except subprocess.TimeoutExpired: + server.kill() diff --git a/tests/integration/test_crash_reconcile.py b/tests/integration/test_crash_reconcile.py new file mode 100644 index 0000000..1eb6674 --- /dev/null +++ b/tests/integration/test_crash_reconcile.py @@ -0,0 +1,132 @@ +"""Backend crash while an agent runs; restart reconciles only application-owned containers.""" + +from __future__ import annotations + +import json +import os +import signal +import socket +import subprocess +import sys +import threading +import time +import urllib.error +import urllib.request +import uuid +from pathlib import Path + +import pytest + +from tests.integration.conftest import APPROVED_ORIGIN, CANARY_PAT, REPO_ROOT, requires_runtime_image + +pytestmark = [pytest.mark.integration, requires_runtime_image()] + +LABEL_KEY = "com.confluence_web.app" + + +def docker(*args: str) -> str: + return subprocess.run(["docker", *args], capture_output=True, text=True, check=True, timeout=60).stdout.strip() + + +def app_containers(label_value: str) -> list[str]: + out = docker("ps", "-aq", "--filter", f"label={LABEL_KEY}={label_value}") + return [line for line in out.splitlines() if line] + + +def free_port() -> int: + with socket.socket() as s: + s.bind(("127.0.0.1", 0)) + return s.getsockname()[1] + + +def start_server(env: dict) -> subprocess.Popen: + proc = subprocess.Popen([sys.executable, str(REPO_ROOT / "tests/integration/crash_server.py")], env=env, cwd=REPO_ROOT) + deadline = time.time() + 60 + while time.time() < deadline: + if proc.poll() is not None: + raise AssertionError(f"server exited early with {proc.returncode}") + try: + with urllib.request.urlopen(f"http://127.0.0.1:{env['CW_DEV_PORT']}/", timeout=1) as r: + if r.status == 200: + return proc + except Exception: + time.sleep(0.3) + proc.kill() + raise AssertionError("server did not become ready") + + +def test_backend_crash_then_restart_reconciles_only_owned_containers(tmp_path: Path): + label_value = f"integration-crash-{uuid.uuid4().hex[:8]}" + port = free_port() + artifact_dir = tmp_path / "artifacts" + (artifact_dir / "committed").mkdir(parents=True) + (artifact_dir / "staging").mkdir() + (artifact_dir / "committed" / "stale-artifact.bin").write_bytes(b"old retained bytes") + (artifact_dir / "staging" / "stale-staging.bin").write_bytes(b"old staged bytes") + env = {"PATH": os.environ["PATH"], "PYTHONPATH": str(REPO_ROOT), "CW_LABEL_VALUE": label_value, "CW_ARTIFACT_DIR": str(artifact_dir), "CW_DEV_PORT": str(port)} + if "DOCKER_HOST" in os.environ: + env["DOCKER_HOST"] = os.environ["DOCKER_HOST"] + + unrelated = docker("run", "-d", "--rm", "--network", "none", "--name", f"cw-unrelated-{label_value[-8:]}", "alpine:3.20", "sleep", "600") + stale = docker("run", "-d", "--network", "none", "--label", f"{LABEL_KEY}={label_value}", "--label", f"{LABEL_KEY}.query_id=stale", "alpine:3.20", "sleep", "600") + server = None + try: + # 1. Startup reconciliation removes the stale owned container and purges old artifacts. + server = start_server(env) + assert stale not in docker("ps", "-aq", "--no-trunc") + assert unrelated in docker("ps", "-aq", "--no-trunc"), "reconciliation must not touch unrelated containers" + assert not any(p.name in ("stale-artifact.bin", "stale-staging.bin") for p in artifact_dir.rglob("*")) + + # 2. Start a query whose model stalls, so a real runtime container is alive. + body = json.dumps({"prompt": "stall", "credentials": {"url": APPROVED_ORIGIN, "pat": CANARY_PAT}}).encode() + req = urllib.request.Request(f"http://127.0.0.1:{port}/api/v1/query", data=body, method="POST", headers={"Content-Type": "application/json", "Origin": f"http://127.0.0.1:{port}"}) + outcome: dict = {} + + def post(): + try: + with urllib.request.urlopen(req, timeout=300) as r: + outcome["status"] = r.status + except urllib.error.HTTPError as e: + outcome["status"] = e.code + except Exception as e: # connection dropped by the crash + outcome["error"] = type(e).__name__ + + t = threading.Thread(target=post, daemon=True) + t.start() + deadline = time.time() + 60 + while time.time() < deadline and not app_containers(label_value): + time.sleep(0.5) + running = app_containers(label_value) + assert running, "runtime container must be running during the stalled query" + + # 3. Crash the backend (SIGKILL: no cleanup code runs). + server.send_signal(signal.SIGKILL) + server.wait(timeout=10) + server = None + t.join(timeout=30) + assert "error" in outcome or outcome.get("status", 0) >= 500 + + # 4. Restart: reconciliation removes owned leftovers, keeps the unrelated container. + server = start_server(env) + deadline = time.time() + 30 + while time.time() < deadline and app_containers(label_value): + time.sleep(0.5) + assert app_containers(label_value) == [] + assert unrelated in docker("ps", "-aq", "--no-trunc") + + # 5. The restarted backend serves requests again; a missing Origin is refused with the contract code. + probe = json.dumps({"url": APPROVED_ORIGIN, "pat": CANARY_PAT}).encode() + with pytest.raises(urllib.error.HTTPError) as excinfo: + urllib.request.urlopen(urllib.request.Request(f"http://127.0.0.1:{port}/api/v1/auth/verify", data=probe, method="POST", headers={"Content-Type": "application/json"}), timeout=5) + assert excinfo.value.code == 403 + assert json.loads(excinfo.value.read())["error"]["code"] == "origin_denied" + finally: + if server is not None: + server.terminate() + try: + server.wait(timeout=10) + except subprocess.TimeoutExpired: + server.kill() + subprocess.run(["docker", "rm", "-f", unrelated, stale], capture_output=True) + for cid in app_containers(label_value): + subprocess.run(["docker", "rm", "-f", cid], capture_output=True) diff --git a/tests/integration/test_error_mapping.py b/tests/integration/test_error_mapping.py new file mode 100644 index 0000000..9c668f7 --- /dev/null +++ b/tests/integration/test_error_mapping.py @@ -0,0 +1,58 @@ +"""Runtime terminal error codes map to the contract's HTTP statuses (no Docker).""" + +from __future__ import annotations + +from pathlib import Path + +import httpx +import pytest +from httpx import ASGITransport + +from backend.app import create_app +from backend.artifacts import ArtifactStore +from backend.containers import FakeContainerManager +from backend.dev.fake_peer import ScriptedContainerPeer +from backend.model import FakeModelAdapter +from backend.settings import Settings + +from tests.backend.conftest import make_test_confluence_client_factory + +pytestmark = pytest.mark.integration + +ORIGIN = "http://testserver" + + +def error_peer(code: str): + async def steps(reader, writer, peer): + start = await peer._read_line(reader) + assert start["type"] == "start" + await peer._write_msg(writer, {"v": 1, "type": "error", "id": "a_err", "payload": {"code": code, "message": "RUNTIME-DETAIL-MUST-NOT-LEAK"}}) + return lambda: ScriptedContainerPeer(custom_steps=steps) + + +@pytest.mark.parametrize("code,status,expected", [ + ("model_output_limit", 502, "model_output_limit"), + ("model_context_exceeded", 400, "model_context_exceeded"), + ("query_timeout", 504, "query_timeout"), + ("connectivity_failed", 502, "connectivity_failed"), + ("execution_failed", 500, "execution_failed"), + ("invalid_input", 500, "execution_failed"), + ("something_unknown", 500, "execution_failed"), +]) +@pytest.mark.asyncio +async def test_runtime_error_code_mapping(tmp_path: Path, code, status, expected): + settings = Settings(query_timeout_seconds=20.0, artifact_storage_dir=tmp_path / "a") + app = create_app( + settings=settings, + container_manager=FakeContainerManager(error_peer(code)), + artifact_store=ArtifactStore(settings.artifact_storage_dir), + model_adapter=FakeModelAdapter(), + confluence_client_factory=make_test_confluence_client_factory(), + ) + async with httpx.AsyncClient(transport=ASGITransport(app=app), base_url=ORIGIN) as c: + r = await c.post("/api/v1/query", json={"prompt": "x", "credentials": {"url": "https://approved.example.com", "pat": "valid-pat"}}, headers={"Origin": ORIGIN}) + assert r.status_code == status + body = r.json() + assert body["error"]["code"] == expected + assert "RUNTIME-DETAIL" not in r.text + assert r.headers["cache-control"] == "no-store" diff --git a/tests/integration/test_frontend_serving.py b/tests/integration/test_frontend_serving.py new file mode 100644 index 0000000..49f2513 --- /dev/null +++ b/tests/integration/test_frontend_serving.py @@ -0,0 +1,98 @@ +"""Browser-facing surface: the backend serves the real frontend tree on one origin. + +Runs without Docker (explicit dev-mode fakes) and checks the exact requests +the shipped frontend JavaScript makes, plus that only production asset +directories are exposed. +""" + +from __future__ import annotations + +from pathlib import Path + +import httpx +import pytest +from httpx import ASGITransport + +from backend.app import CSP_POLICY, SESSION_COOKIE_NAME, create_app +from backend.artifacts import ArtifactStore +from backend.settings import Settings + +from tests.integration.conftest import CHECKLIST_BYTES, FRONTEND_DIR + +pytestmark = pytest.mark.integration + +ORIGIN = "http://127.0.0.1:8000" + + +@pytest.fixture +def app(tmp_path: Path): + settings = Settings( + dev_mode=True, + frontend_dist_dir=FRONTEND_DIR, + artifact_storage_dir=tmp_path / "artifacts", + query_timeout_seconds=30.0, + ) + return create_app(settings=settings, artifact_store=ArtifactStore(settings.artifact_storage_dir)) + + +@pytest.mark.asyncio +async def test_index_and_assets_served_with_headers(app): + async with httpx.AsyncClient(transport=ASGITransport(app=app), base_url=ORIGIN) as c: + index = await c.get("/") + assert index.status_code == 200 + assert index.headers["content-type"].startswith("text/html") + assert "Confluence Research" in index.text + assert index.headers["content-security-policy"] == CSP_POLICY + assert index.headers["referrer-policy"] == "no-referrer" + assert index.headers["x-content-type-options"] == "nosniff" + cookie = index.headers["set-cookie"] + assert cookie.startswith(f"{SESSION_COOKIE_NAME}=") and "HttpOnly" in cookie and "SameSite=strict" in cookie.replace("Strict", "strict") and "Path=/" in cookie + assert "Secure" not in cookie # plain HTTP loopback deployment + + for path, ctype in [ + ("/css/style.css", "text/css"), + ("/js/app.js", "javascript"), + ("/js/api.js", "javascript"), + ("/js/render.js", "javascript"), + ("/js/history.js", "javascript"), + ("/vendor/marked.min.js", "javascript"), + ("/vendor/purify.min.js", "javascript"), + ]: + r = await c.get(path) + assert r.status_code == 200, path + assert ctype in r.headers["content-type"], (path, r.headers["content-type"]) + assert r.headers["content-security-policy"] == CSP_POLICY + assert r.headers["x-content-type-options"] == "nosniff" + assert len(r.content) == (FRONTEND_DIR / path.lstrip("/")).stat().st_size + + for hidden in ["/dev/mock-server.js", "/tests/api.test.js", "/HANDOFF.md", "/README.md", "/package.json", "/static/js/app.js", "/js/../HANDOFF.md", "/index.html/../package.json"]: + r = await c.get(hidden) + assert r.status_code == 404, hidden + assert r.json()["error"]["code"] in {"invalid_input", "artifact_not_found"} + + +@pytest.mark.asyncio +async def test_frontend_request_shapes_against_backend(app): + """Replays the exact bodies/headers frontend/js/api.js sends, in dev mode.""" + async with httpx.AsyncClient(transport=ASGITransport(app=app), base_url=ORIGIN) as c: + await c.get("/") + headers = {"Origin": ORIGIN, "Content-Type": "application/json"} + bad = await c.post("/api/v1/auth/verify", json={"url": "https://approved.example.com", "pat": "wrong"}, headers=headers) + assert bad.status_code == 403 and bad.json()["error"]["code"] == "confluence_auth_failed" + ok = await c.post("/api/v1/auth/verify", json={"url": "https://approved.example.com", "pat": "dev-pat"}, headers=headers) + assert ok.status_code == 200 and ok.json() == {"valid": True} + denied = await c.post("/api/v1/auth/verify", json={"url": "https://elsewhere.example.org", "pat": "dev-pat"}, headers=headers) + assert denied.status_code == 403 and denied.json()["error"]["code"] == "destination_denied" + no_origin = await c.post("/api/v1/auth/verify", json={"url": "https://approved.example.com", "pat": "dev-pat"}) + assert no_origin.status_code == 403 and no_origin.json()["error"]["code"] == "origin_denied" + extra = await c.post("/api/v1/query", json={"prompt": "x", "credentials": {"url": "https://approved.example.com", "pat": "dev-pat"}, "extra": 1}, headers=headers) + assert extra.status_code == 400 and extra.json()["error"]["code"] == "invalid_input" + + q = await c.post("/api/v1/query", json={"prompt": "How do I deploy service X?", "credentials": {"url": "https://approved.example.com", "pat": "dev-pat"}}, headers=headers) + assert q.status_code == 200, q.text + body = q.json() + assert len(body["tool_history"]) == 2 and len(body["pages_accessed"]) == 1 + art = body["artifacts"][0] + dl = await c.get(f"/api/v1/artifacts/{art['id']}") + assert dl.status_code == 200 and dl.content == CHECKLIST_BYTES + assert dl.headers["content-disposition"].startswith("attachment") diff --git a/tests/integration/test_live_model.py b/tests/integration/test_live_model.py new file mode 100644 index 0000000..7d63a78 --- /dev/null +++ b/tests/integration/test_live_model.py @@ -0,0 +1,97 @@ +"""Live provider check: the configured OpenAI-compatible endpoint drives the real runtime. + +Deselected by default (marker ``live``). Confluence stays scripted (no PAT +needed); the model and the container are real. Run with: + + set -a; . deploy/confluence-web.env; set +a + .venv/bin/python -m pytest -m live tests/integration/test_live_model.py -s +""" + +from __future__ import annotations + +import json +import os +import socket +import urllib.parse +from pathlib import Path + +import pytest + +from backend.artifacts import ArtifactStore +from backend.containers import DockerContainerManager +from backend.model import OpenAIModelAdapter +from backend.runner import QueryRunner +from backend.settings import Settings + +from tests.integration.conftest import APPROVED_ORIGIN, CANARY_PAT, CHECKLIST_BYTES, PAGE_ID, PAGE_URL, RUNTIME_IMAGE, make_confluence_factory, requires_runtime_image + +pytestmark = [pytest.mark.live, pytest.mark.integration, requires_runtime_image()] + + +def endpoint_reachable(url: str) -> bool: + try: + parsed = urllib.parse.urlsplit(url) + with socket.create_connection((parsed.hostname, parsed.port or (443 if parsed.scheme == "https" else 80)), timeout=2): + return True + except Exception: + return False + + +@pytest.mark.skipif(os.getenv("CONFLUENCE_WEB_MODEL_PROVIDER") != "openai" or not os.getenv("CONFLUENCE_WEB_MODEL_ENDPOINT"), reason="deployment model configuration not exported") +@pytest.mark.asyncio +async def test_live_model_completes_search_read_write_cite(tmp_path: Path): + env = Settings.from_env() + if not endpoint_reachable(env.model_endpoint): + pytest.skip(f"model endpoint not reachable: {env.model_endpoint}") + settings = Settings( + runtime_image=RUNTIME_IMAGE, + approved_confluence_origins=[APPROVED_ORIGIN], + container_label_value=f"integration-live-{tmp_path.name[-6:]}", + artifact_storage_dir=tmp_path / "artifacts", + model_provider="openai", + model_name=env.model_name, + model_api_key=env.model_api_key, + model_endpoint=env.model_endpoint, + model_context_window_tokens=env.model_context_window_tokens, + model_max_output_tokens=env.model_max_output_tokens, + model_timeout_seconds=env.model_timeout_seconds, + ) + adapter = OpenAIModelAdapter( + api_key=settings.model_api_key, model_name=settings.model_name, endpoint=settings.model_endpoint, + context_window_tokens=settings.model_context_window_tokens, max_output_tokens=settings.model_max_output_tokens, + timeout=settings.model_timeout_seconds, + ) + store = ArtifactStore(settings.artifact_storage_dir) + mgr = DockerContainerManager(settings) + await mgr.verify_rootless() + runner = QueryRunner(settings=settings, container_manager=mgr, artifact_store=store, model_adapter=adapter, confluence_client_factory=make_confluence_factory()) + try: + result = await runner.run( + prompt=( + "How do I deploy service X? Search Confluence for it and read the most relevant page. " + "Then create the file /work/artifacts/checklist.md containing a short Markdown deployment checklist. " + "Answer briefly in Markdown and cite the Confluence page URL you used." + ), + confluence_url=APPROVED_ORIGIN, + confluence_pat=CANARY_PAT, + session_id="cw_live", + ) + finally: + await adapter.close() + + print(json.dumps({k: v for k, v in result.items() if k != "markdown"}, indent=1)[:3000]) + print("MARKDOWN:\n" + result["markdown"][:2000]) + tools = [h["tool"] for h in result["tool_history"]] + assert "confluence_search" in tools and "confluence_view" in tools + assert all(h["status"] == "success" for h in result["tool_history"] if h["tool"] == "confluence_view" and h["parameters"].get("page_id") == PAGE_ID) + assert [p["page_id"] for p in result["pages_accessed"]] == [PAGE_ID] + assert PAGE_URL in result["markdown"] or "pageId=847291" in result["markdown"] + names = [a["name"] for a in result["artifacts"]] + assert "checklist.md" in names, names + art = next(a for a in result["artifacts"] if a["name"] == "checklist.md") + dl = store.get_artifact_for_download(art["id"], session_id="cw_live") + assert dl is not None and dl[2] == art["size_bytes"] > 0 + store.release_reader(art["id"]) + code, out, _ = await mgr._exec_docker(["ps", "-aq", "--filter", f"label={settings.container_label_key}={settings.container_label_value}"]) + assert code == 0 and out.strip() == "" + assert CANARY_PAT not in json.dumps(result) diff --git a/tests/integration/test_real_image.py b/tests/integration/test_real_image.py new file mode 100644 index 0000000..4cea974 --- /dev/null +++ b/tests/integration/test_real_image.py @@ -0,0 +1,474 @@ +"""Backend + real pi runtime image with scripted model and Confluence peers. + +Proves the actual bridge peers connect: real SDK turns, tool arguments and +results, collection ordering, exact artifact bytes, cleanup, cancellation, +timeouts, busy gating, and container isolation canaries. No network, no +credentials: Confluence is a scripted transport and the model is scripted. +""" + +from __future__ import annotations + +import asyncio +import json +import time +from pathlib import Path + +import httpx +import pytest +from httpx import ASGITransport + +from backend.app import create_app, SESSION_COOKIE_NAME +from backend.artifacts import ArtifactStore +from backend.containers import DockerContainerManager +from backend.errors import AppError, BusyError, ModelOutputLimitError, QueryTimeoutError, UpstreamFailedError +from backend.runner import QueryRunner +from backend.settings import Settings + +from tests.integration.conftest import ( + APPROVED_ORIGIN, + CANARY_PAT, + CHECKLIST_BYTES, + DENIED_PAGE_ID, + FINAL_ANSWER, + PAGE_ID, + PAGE_URL, + RUNTIME_IMAGE, + ScriptedModelAdapter, + example_script, + make_confluence_factory, + requires_runtime_image, + text_turn, + tool_message_text, + tool_turn, +) + +pytestmark = [pytest.mark.integration, requires_runtime_image()] + +EXPECTED_TOOLS = {"confluence_search", "confluence_view", "confluence_list_spaces", "bash", "read", "write", "edit"} + + +def make_settings(tmp_path: Path, **overrides) -> Settings: + base = dict( + runtime_image=RUNTIME_IMAGE, + approved_confluence_origins=[APPROVED_ORIGIN], + query_timeout_seconds=120.0, + cleanup_timeout_seconds=10.0, + container_label_value=f"integration-{tmp_path.name[-8:]}", + artifact_storage_dir=tmp_path / "artifacts", + ) + base.update(overrides) + return Settings(**base) + + +def make_runner(tmp_path: Path, script, **overrides): + settings = make_settings(tmp_path, **overrides) + store = ArtifactStore(settings.artifact_storage_dir) + mgr = DockerContainerManager(settings) + adapter = ScriptedModelAdapter(script) + calls = [] + runner = QueryRunner( + settings=settings, + container_manager=mgr, + artifact_store=store, + model_adapter=adapter, + confluence_client_factory=make_confluence_factory(calls), + ) + return runner, settings, store, mgr, adapter, calls + + +async def assert_no_containers(mgr: DockerContainerManager, settings: Settings) -> None: + code, out, _ = await mgr._exec_docker( + ["ps", "-a", "-q", "--filter", f"label={settings.container_label_key}={settings.container_label_value}"] + ) + assert code == 0 + assert out.strip() == "", "no application container may remain after the run" + + +async def run_query(runner: QueryRunner, prompt: str = "How do I deploy service X? Create a checklist file too.", **kwargs): + return await runner.run( + prompt=prompt, + confluence_url=APPROVED_ORIGIN, + confluence_pat=CANARY_PAT, + session_id=kwargs.pop("session_id", "cw_sess_integration"), + **kwargs, + ) + + +# --- shared example --------------------------------------------------------------- + +@pytest.mark.asyncio +async def test_shared_example_end_to_end(tmp_path: Path): + runner, settings, store, mgr, adapter, calls = make_runner(tmp_path, example_script()) + started = time.monotonic() + result = await run_query(runner) + elapsed = time.monotonic() - started + + # Container removed before the result is returned. + await assert_no_containers(mgr, settings) + + # Authoritative history: exactly the two Confluence calls, in start order. + hist = result["tool_history"] + assert [h["tool"] for h in hist] == ["confluence_search", "confluence_view"] + assert all(h["status"] == "success" and h["error"] is None and h["cache_hit"] is False for h in hist) + assert hist[0]["parameters"]["query"] == "deploy service X" + assert hist[0]["parameters"]["limit"] == 10 and hist[0]["parameters"]["offset"] == 0 + assert hist[0]["result"]["pages"][0]["page_id"] == PAGE_ID + assert hist[0]["result"]["pages"][0]["url"] == PAGE_URL + assert hist[0]["result"]["pagination"] == {"offset": 0, "limit": 10, "has_more": False} + assert hist[1]["parameters"] == {"page_id": PAGE_ID} + assert hist[1]["result"]["markdown"].strip() == "Deploy service X using the release checklist." + assert hist[1]["result"]["truncated"] is False + assert hist[0]["tool_call_id"] != hist[1]["tool_call_id"] + assert all(h["tool_call_id"].startswith("a_") for h in hist) + assert hist[0]["started_at"] <= hist[1]["started_at"] + + # Search matches never count; the read does. + assert [p["page_id"] for p in result["pages_accessed"]] == [PAGE_ID] + assert result["pages_accessed"][0]["url"] == PAGE_URL + assert result["pages_accessed"][0]["accessed_at"] + + # Final answer and artifact. + assert result["markdown"] == FINAL_ANSWER + assert PAGE_URL in result["markdown"] + assert len(result["artifacts"]) == 1 + art = result["artifacts"][0] + assert art["name"] == "checklist.md" and art["size_bytes"] == 32 + dl = store.get_artifact_for_download(art["id"], session_id="cw_sess_integration") + assert dl is not None + path, name, size = dl + assert (path.read_bytes(), name, size) == (CHECKLIST_BYTES, "checklist.md", 32) + store.release_reader(art["id"]) + assert store.get_artifact_for_download(art["id"], session_id="cw_other") is None + assert result["duration_seconds"] >= 0 and elapsed < 90 + assert result["warnings"] == [] + + # Actual SDK turns reached the scripted model with the neutral contract shape. + reqs = adapter.received_requests + assert len(reqs) == 4 + assert all(r["system_instruction"] for r in reqs) + assert all(m["role"] != "system" for r in reqs for m in r["messages"]) + assert {t["name"] for t in reqs[0]["tools"]} == EXPECTED_TOOLS + assert all(isinstance(t["input_schema"], dict) for t in reqs[0]["tools"]) + assert reqs[0]["messages"][0]["role"] == "user" + assert "deploy service X" in reqs[0]["messages"][0]["content"][0]["text"] + + # Turn 2 carries the assistant tool call and the search result as a tool message. + roles = [m["role"] for m in reqs[1]["messages"]] + assert roles == ["user", "assistant", "tool"] + assistant = reqs[1]["messages"][1] + call = next(c for c in assistant["content"] if c["type"] == "tool_call") + assert call["name"] == "confluence_search" and call["arguments"] == {"query": "deploy service X"} + assert isinstance(call["arguments"], dict) + search_text = tool_message_text(reqs[1], call["id"]) + assert "Deployment Guide" in search_text and PAGE_URL in search_text + assert reqs[1]["messages"][2]["is_error"] is False + + # Turn 4 carries the local write result; local tools stay out of history. + write_call = next(c for m in reqs[3]["messages"] if m["role"] == "assistant" for c in m["content"] if c["type"] == "tool_call" and c["name"] == "write") + assert write_call["arguments"]["path"] == "/work/artifacts/checklist.md" + assert reqs[3]["messages"][-1]["role"] == "tool" and reqs[3]["messages"][-1]["is_error"] is False + assert "write" not in [h["tool"] for h in hist] + + # Backend called Confluence exactly twice, with backend-constructed CQL. + assert [c["path"] for c in calls] == ["/rest/api/content/search", f"/rest/api/content/{PAGE_ID}"] + assert 'text ~ "deploy service X"' in calls[0]["params"]["cql"] + + # The PAT never reaches the model or the result. + assert CANARY_PAT not in json.dumps(reqs) and CANARY_PAT not in json.dumps(result) + + +@pytest.mark.asyncio +async def test_no_artifacts_and_empty_search(tmp_path: Path): + script = [ + tool_turn([("call_1", "confluence_search", {"query": "nothing here", "limit": 5})]), + text_turn("No matching pages were found."), + ] + runner, settings, store, mgr, adapter, calls = make_runner(tmp_path, script) + result = await run_query(runner, prompt="Find nothing here") + await assert_no_containers(mgr, settings) + assert result["artifacts"] == [] and result["pages_accessed"] == [] + assert len(result["tool_history"]) == 1 + entry = result["tool_history"][0] + assert entry["status"] == "success" + assert entry["result"] == {"pages": [], "pagination": {"offset": 0, "limit": 5, "has_more": False}} + assert "[]" in tool_message_text(adapter.received_requests[1], "call_1") + assert result["markdown"] == "No matching pages were found." + + +@pytest.mark.asyncio +async def test_cached_repeat_view_and_denied_page(tmp_path: Path): + script = [ + tool_turn([("call_1", "confluence_view", {"page_id": PAGE_ID})]), + tool_turn([("call_2", "confluence_view", {"page_id": PAGE_ID})]), + tool_turn([("call_3", "confluence_view", {"page_id": DENIED_PAGE_ID})]), + text_turn(f"Only [Deployment Guide]({PAGE_URL}) was readable."), + ] + runner, settings, store, mgr, adapter, calls = make_runner(tmp_path, script) + result = await run_query(runner) + await assert_no_containers(mgr, settings) + + hist = result["tool_history"] + assert [h["tool"] for h in hist] == ["confluence_view"] * 3 + assert [h["cache_hit"] for h in hist] == [False, True, False] + assert hist[1]["result"] == hist[0]["result"] + assert len({h["tool_call_id"] for h in hist}) == 3 + assert hist[2]["status"] == "error" and hist[2]["result"] is None + err = hist[2]["error"] + assert set(err) == {"code", "message"} and err["code"] and len(err["message"].encode()) <= 1024 + assert "SECRET-UPSTREAM-BODY" not in json.dumps(result) + assert "SECRET-UPSTREAM-BODY" not in json.dumps(adapter.received_requests) + + # Cache hit did not call upstream again; the denied read did. + assert [c["path"] for c in calls] == [f"/rest/api/content/{PAGE_ID}", f"/rest/api/content/{DENIED_PAGE_ID}"] + # One unique page, the denied one never counted. + assert [p["page_id"] for p in result["pages_accessed"]] == [PAGE_ID] + # The agent saw the failure as a recoverable tool error and continued. + denied_msg = next(m for m in adapter.received_requests[3]["messages"] if m["role"] == "tool" and m["tool_call_id"] == "call_3") + assert denied_msg["is_error"] is True + assert result["markdown"].startswith("Only") + + +@pytest.mark.asyncio +async def test_empty_artifact_and_html_artifact(tmp_path: Path): + script = [ + tool_turn([("call_1", "bash", {"command": "touch /work/artifacts/empty.txt; printf '' > /work/artifacts/page.html"})]), + text_turn("Created two files."), + ] + runner, settings, store, mgr, adapter, calls = make_runner(tmp_path, script) + result = await run_query(runner) + await assert_no_containers(mgr, settings) + by_name = {a["name"]: a for a in result["artifacts"]} + assert set(by_name) == {"empty.txt", "page.html"} + assert by_name["empty.txt"]["size_bytes"] == 0 + dl = store.get_artifact_for_download(by_name["empty.txt"]["id"], session_id="cw_sess_integration") + assert dl is not None and dl[0].read_bytes() == b"" and dl[2] == 0 + store.release_reader(by_name["empty.txt"]["id"]) + assert result["pages_accessed"] == [] and result["tool_history"] == [] + + +# --- failure paths --------------------------------------------------------------------- + +@pytest.mark.asyncio +async def test_model_upstream_failure_discards_exports(tmp_path: Path): + script = [ + tool_turn([("call_1", "bash", {"command": "printf 'partial' > /work/artifacts/partial.txt"})]), + UpstreamFailedError("scripted provider outage"), + ] + runner, settings, store, mgr, adapter, calls = make_runner(tmp_path, script) + with pytest.raises(AppError) as excinfo: + await run_query(runner) + assert excinfo.value.code in {"upstream_failed", "execution_failed"} + await assert_no_containers(mgr, settings) + assert store.global_reserved_bytes == 0 + assert not any(p.is_file() for p in settings.artifact_storage_dir.rglob("*")) + + +@pytest.mark.asyncio +async def test_model_length_stop_is_output_limit(tmp_path: Path): + script = [text_turn("Truncated answer that never finished", stop_reason="length")] + runner, settings, store, mgr, adapter, calls = make_runner(tmp_path, script) + with pytest.raises(AppError) as excinfo: + await run_query(runner) + assert isinstance(excinfo.value, ModelOutputLimitError), excinfo.value.code + assert excinfo.value.status_code == 502 + await assert_no_containers(mgr, settings) + + +@pytest.mark.asyncio +async def test_empty_terminal_answer_fails(tmp_path: Path): + runner, settings, store, mgr, adapter, calls = make_runner(tmp_path, [text_turn("")]) + with pytest.raises(AppError) as excinfo: + await run_query(runner) + assert excinfo.value.code == "execution_failed" + await assert_no_containers(mgr, settings) + + +@pytest.mark.asyncio +async def test_timeout_during_model_work(tmp_path: Path): + async def stall(): + await asyncio.sleep(120) + runner, settings, store, mgr, adapter, calls = make_runner(tmp_path, [stall], query_timeout_seconds=8.0) + started = time.monotonic() + with pytest.raises(QueryTimeoutError): + await run_query(runner) + assert time.monotonic() - started < 40 + await assert_no_containers(mgr, settings) + assert store.global_reserved_bytes == 0 + + +@pytest.mark.asyncio +async def test_disconnect_cancels_and_busy_gate_holds(tmp_path: Path): + async def stall(): + await asyncio.sleep(120) + runner, settings, store, mgr, adapter, calls = make_runner( + tmp_path, [stall] + example_script(), query_timeout_seconds=90.0 + ) + disconnected = False + first = asyncio.create_task(run_query(runner, is_disconnected=lambda: disconnected)) + await asyncio.wait_for(adapter.started.wait(), 60) + + # Busy while the first run is active. + with pytest.raises(BusyError): + await run_query(runner, session_id="cw_second") + + disconnected = True + with pytest.raises(BaseException) as excinfo: + await first + assert isinstance(excinfo.value, (asyncio.CancelledError, AppError)), type(excinfo.value) + await assert_no_containers(mgr, settings) + assert store.global_reserved_bytes == 0 + + # After cleanup the gate admits a new run, which completes normally. + result = await run_query(runner, session_id="cw_third") + assert result["artifacts"][0]["size_bytes"] == 32 + await assert_no_containers(mgr, settings) + + +# --- isolation canaries under the actual image and launch flags ------------------------ + +PROBE = r""" +set +e +echo "UID=$(id -u) GID=$(id -g)" +echo "WORK=$(ls -A /work | tr '\n' ' ')" +echo "HOME_LS=$(ls -A "$HOME" | tr '\n' ' ')" +echo "TMP_LS=$(ls -A /tmp | tr '\n' ' ')" +echo "NET=$(ls /sys/class/net | tr '\n' ' ')" +echo "CAPEFF=$(grep CapEff /proc/self/status)" +echo "NNP=$(grep NoNewPrivs /proc/self/status)" +echo "ENV_BEGIN"; env; echo "ENV_END" +echo "PID1_ENVIRON=$(cat /proc/1/environ 2>&1 | tr '\0' ' ' | head -c 300)" +echo "PROCS=$(ps -eo pid,comm --no-headers | tr '\n' ';')" +echo "OPT_WRITABLE=$(touch /opt/agent/x 2>&1 || echo readonly)" +echo "ROOT_WRITABLE=$(touch /x 2>&1 || echo readonly)" +echo "DOCKER_SOCK=$(ls /var/run/docker.sock /run/docker.sock 2>&1)" +echo "HOST_FILES=$(ls /etc/shadow ~/.pi /root 2>&1 | tr '\n' ' ')" +echo "MOUNTS=$(awk '{print $2":"$3}' /proc/mounts | tr '\n' ' ')" +""" + + +@pytest.mark.asyncio +async def test_fresh_workspace_and_canaries_across_runs(tmp_path: Path): + seed = [ + tool_turn([("call_1", "bash", {"command": "echo scratch > /work/scratch.txt; echo m > $HOME/marker; echo t > /tmp/tmarker; echo ok"})]), + text_turn("seeded"), + ] + runner, settings, store, mgr, adapter, calls = make_runner(tmp_path, seed + [ + tool_turn([("call_2", "bash", {"command": PROBE})]), + text_turn("probed"), + ]) + first = await run_query(runner) + assert first["markdown"] == "seeded" + second = await run_query(runner) + assert second["markdown"] == "probed" + await assert_no_containers(mgr, settings) + + probe_out = tool_message_text(adapter.received_requests[3], "call_2") + assert "UID=10001 GID=10001" in probe_out + assert "scratch.txt" not in probe_out and "marker" not in probe_out + assert "WORK=artifacts" in probe_out + assert "NET=lo" in probe_out + assert "CapEff:\t0000000000000000" in probe_out + assert "NoNewPrivs:\t1" in probe_out + env_block = probe_out.split("ENV_BEGIN")[1].split("ENV_END")[0] + env_names = {line.split("=", 1)[0] for line in env_block.strip().splitlines() if "=" in line} + assert env_names <= {"HOME", "LANG", "PATH", "PWD", "SHLVL", "_", "OLDPWD", "NODE_OPTIONS", "AGENT_WORK_DIR"}, env_names + assert CANARY_PAT not in probe_out + assert "MODEL_API_KEY" not in probe_out and "CONFLUENCE" not in env_block + assert "PID1_ENVIRON=cat: /proc/1/environ: Permission denied" in probe_out or "PID1_ENVIRON=" in probe_out + assert "Read-only file system" in probe_out.split("OPT_WRITABLE=")[1].split("ROOT_WRITABLE=")[0] + assert "Read-only file system" in probe_out.split("ROOT_WRITABLE=")[1].split("DOCKER_SOCK=")[0] + assert "No such file" in probe_out.split("DOCKER_SOCK=")[1].split("\n")[0] + host_files = probe_out.split("HOST_FILES=")[1].split("MOUNTS=")[0] + assert "cannot access '/home/agent/.pi'" in host_files and "'/root': Permission denied" in host_files + assert "/work:tmpfs" in probe_out and "/tmp:tmpfs" in probe_out and "/home/agent:tmpfs" in probe_out + assert CANARY_PAT not in json.dumps(adapter.received_requests) + + +# --- HTTP surface with the real image -------------------------------------------------- + +@pytest.mark.asyncio +async def test_http_query_and_download_ownership(tmp_path: Path): + settings = make_settings(tmp_path) + store = ArtifactStore(settings.artifact_storage_dir) + mgr = DockerContainerManager(settings) + adapter = ScriptedModelAdapter(example_script()) + app = create_app( + settings=settings, + container_manager=mgr, + artifact_store=store, + model_adapter=adapter, + confluence_client_factory=make_confluence_factory(), + ) + transport = ASGITransport(app=app) + origin = "http://testserver" + async with httpx.AsyncClient(transport=transport, base_url=origin) as a, httpx.AsyncClient(transport=transport, base_url=origin) as b: + boot = await a.get("/") + assert boot.status_code == 200 and SESSION_COOKIE_NAME in boot.cookies + verify = await a.post("/api/v1/auth/verify", json={"url": APPROVED_ORIGIN, "pat": CANARY_PAT}, headers={"Origin": origin}) + assert verify.status_code == 200 and verify.json() == {"valid": True} + assert verify.headers["cache-control"] == "no-store" + + resp = await a.post( + "/api/v1/query", + json={"prompt": "How do I deploy service X?", "credentials": {"url": APPROVED_ORIGIN, "pat": CANARY_PAT}}, + headers={"Origin": origin}, + timeout=120, + ) + assert resp.status_code == 200, resp.text + assert resp.headers["cache-control"] == "no-store" + body = resp.json() + assert set(body) == {"session_id", "markdown", "pages_accessed", "tool_history", "artifacts", "warnings", "duration_seconds"} + await assert_no_containers(mgr, settings) + art = body["artifacts"][0] + assert art["id"] != body["session_id"] + + dl = await a.get(f"/api/v1/artifacts/{art['id']}") + assert dl.status_code == 200 and dl.content == CHECKLIST_BYTES + assert dl.headers["content-type"].startswith("application/octet-stream") + assert dl.headers["content-disposition"].startswith("attachment") + assert dl.headers["x-content-type-options"] == "nosniff" + assert dl.headers["cache-control"] == "no-store" + + # Session B, unknown ID and expired ID are indistinguishable 404s. + await b.get("/") + wrong = await b.get(f"/api/v1/artifacts/{art['id']}") + unknown = await a.get("/api/v1/artifacts/does-not-exist") + store._committed[art["id"]].expires_at_ts = 0 # force expiry + expired = await a.get(f"/api/v1/artifacts/{art['id']}") + for r in (wrong, unknown, expired): + assert r.status_code == 404 + assert r.json() == {"error": {"code": "artifact_not_found", "message": unknown.json()["error"]["message"]}} + assert r.headers["cache-control"] == "no-store" + assert wrong.content == unknown.content == expired.content + + +# --- generated boundaries through the whole stack --------------------------------------- + +@pytest.mark.asyncio +async def test_large_prompt_and_answer_round_trip(tmp_path: Path): + """Exactly 16 MiB decoded prompt with heavy JSON escaping, and a multi-MiB answer.""" + unit = 'line "quoted" \\ back\ttab é中\U0001f600 \x01\n' # quotes, backslash, control chars, multibyte + prompt = unit * (16 * 1024 * 1024 // len(unit.encode("utf-8"))) + prompt += "x" * (16 * 1024 * 1024 - len(prompt.encode("utf-8"))) + assert len(prompt.encode("utf-8")) == 16 * 1024 * 1024 + section = "## Section\n\n" + ("Deploy service X carefully. " * 40) + "\n\n```\ncode \"block\" \\ \n```\n\n" + answer = "# Big answer\n\n" + section * (4 * 1024 * 1024 // len(section.encode("utf-8"))) + f"\nSee [Deployment Guide]({PAGE_URL}).\n" + script = [ + tool_turn([("call_1", "bash", {"command": "wc -c /work/artifacts 2>/dev/null; echo ok"})]), + text_turn(answer, in_tok=1000, out_tok=1000), + ] + runner, settings, store, mgr, adapter, calls = make_runner(tmp_path, script) + started = time.monotonic() + result = await run_query(runner, prompt=prompt) + elapsed = time.monotonic() - started + await assert_no_containers(mgr, settings) + assert result["markdown"] == answer + first_user = adapter.received_requests[0]["messages"][0]["content"][0]["text"] + assert first_user == prompt, "prompt must reach the model byte-for-byte through the bridge" + assert elapsed < 150, elapsed + + # One byte over the decoded limit is rejected before any container starts. + with pytest.raises(AppError) as excinfo: + await run_query(runner, prompt=prompt + "y") + assert excinfo.value.code == "invalid_input" + await assert_no_containers(mgr, settings) diff --git a/tests/integration/test_real_provider_adapter.py b/tests/integration/test_real_provider_adapter.py new file mode 100644 index 0000000..ebcd801 --- /dev/null +++ b/tests/integration/test_real_provider_adapter.py @@ -0,0 +1,196 @@ +"""Real backend provider adapter + real runtime, without credentials or network. + +The OpenAI-compatible adapter talks to a scripted httpx transport that +records the provider-native requests and returns provider-native responses. +The runtime image runs for real; only the provider HTTP layer is scripted. +""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any, Dict, List + +import httpx +import pytest + +from backend.artifacts import ArtifactStore +from backend.containers import DockerContainerManager +from backend.errors import AppError +from backend.model import OpenAIModelAdapter +from backend.runner import QueryRunner +from backend.settings import Settings + +from tests.integration.conftest import ( + APPROVED_ORIGIN, + CANARY_PAT, + CHECKLIST_BYTES, + CHECKLIST_TEXT, + FINAL_ANSWER, + PAGE_ID, + RUNTIME_IMAGE, + make_confluence_factory, + requires_runtime_image, +) + +pytestmark = [pytest.mark.integration, requires_runtime_image()] + +ENDPOINT = "http://scripted-provider.invalid/v1/chat/completions" +DUMMY_KEY = "scripted-provider-key-not-real" + + +def oa_tool_turn(call_id: str, name: str, args: Dict[str, Any]) -> Dict[str, Any]: + return { + "id": "chatcmpl-scripted", + "object": "chat.completion", + "choices": [{ + "index": 0, + "message": { + "role": "assistant", + "content": None, + "tool_calls": [{"id": call_id, "type": "function", "function": {"name": name, "arguments": json.dumps(args)}}], + }, + "finish_reason": "tool_calls", + }], + "usage": {"prompt_tokens": 42, "completion_tokens": 7}, + } + + +def oa_text_turn(text: str, finish_reason: str = "stop") -> Dict[str, Any]: + return { + "id": "chatcmpl-scripted", + "object": "chat.completion", + "choices": [{"index": 0, "message": {"role": "assistant", "content": text}, "finish_reason": finish_reason}], + "usage": {"prompt_tokens": 50, "completion_tokens": 20}, + } + + +def scripted_provider(responses: List[Any]): + requests: List[Dict[str, Any]] = [] + pending = list(responses) + + async def handler(request: httpx.Request) -> httpx.Response: + body = json.loads(request.content) + requests.append({"headers": dict(request.headers), "body": body}) + assert pending, "provider script exhausted" + nxt = pending.pop(0) + if isinstance(nxt, httpx.Response): + return nxt + return httpx.Response(200, json=nxt) + + return httpx.MockTransport(handler), requests + + +def build(tmp_path: Path, responses: List[Any]): + settings = Settings( + runtime_image=RUNTIME_IMAGE, + approved_confluence_origins=[APPROVED_ORIGIN], + query_timeout_seconds=120.0, + container_label_value=f"integration-{tmp_path.name[-8:]}", + artifact_storage_dir=tmp_path / "artifacts", + model_provider="openai", + model_name="scripted-model", + model_api_key=DUMMY_KEY, + model_endpoint=ENDPOINT, + model_context_window_tokens=131072, + model_max_output_tokens=8192, + ) + transport, requests = scripted_provider(responses) + adapter = OpenAIModelAdapter( + api_key=settings.model_api_key, + model_name=settings.model_name, + endpoint=settings.model_endpoint, + context_window_tokens=settings.model_context_window_tokens, + max_output_tokens=settings.model_max_output_tokens, + transport=transport, + ) + store = ArtifactStore(settings.artifact_storage_dir) + mgr = DockerContainerManager(settings) + runner = QueryRunner( + settings=settings, + container_manager=mgr, + artifact_store=store, + model_adapter=adapter, + confluence_client_factory=make_confluence_factory(), + ) + return runner, settings, store, mgr, adapter, requests + + +async def run(runner: QueryRunner): + return await runner.run( + prompt="How do I deploy service X? Produce a deployment checklist file too.", + confluence_url=APPROVED_ORIGIN, + confluence_pat=CANARY_PAT, + session_id="cw_sess_provider", + ) + + +@pytest.mark.asyncio +async def test_openai_adapter_text_and_tool_turns_with_real_runtime(tmp_path: Path): + responses = [ + oa_tool_turn("call_s", "confluence_search", {"query": "deploy service X"}), + oa_tool_turn("call_v", "confluence_view", {"page_id": PAGE_ID}), + oa_tool_turn("call_w", "write", {"path": "/work/artifacts/checklist.md", "content": CHECKLIST_TEXT}), + oa_text_turn(FINAL_ANSWER), + ] + runner, settings, store, mgr, adapter, requests = build(tmp_path, responses) + try: + result = await run(runner) + finally: + await adapter.close() + + assert result["markdown"] == FINAL_ANSWER + assert [h["tool"] for h in result["tool_history"]] == ["confluence_search", "confluence_view"] + assert [p["page_id"] for p in result["pages_accessed"]] == [PAGE_ID] + art = result["artifacts"][0] + dl = store.get_artifact_for_download(art["id"], session_id="cw_sess_provider") + assert dl is not None and dl[0].read_bytes() == CHECKLIST_BYTES + store.release_reader(art["id"]) + + assert len(requests) == 4 + for r in requests: + assert r["headers"]["authorization"] == f"Bearer {DUMMY_KEY}" + body = r["body"] + assert body["model"] == "scripted-model" and body["max_tokens"] == 8192 + assert body["messages"][0]["role"] == "system" and body["messages"][0]["content"] + assert sum(1 for m in body["messages"] if m["role"] == "system") == 1 + assert {t["function"]["name"] for t in body["tools"]} >= {"confluence_search", "confluence_view", "confluence_list_spaces", "bash", "read", "write", "edit"} + assert all(t["type"] == "function" and isinstance(t["function"]["parameters"], dict) for t in body["tools"]) + + second = requests[1]["body"]["messages"] + assert [m["role"] for m in second] == ["system", "user", "assistant", "tool"] + assert second[2]["tool_calls"][0]["id"] == "call_s" + assert json.loads(second[2]["tool_calls"][0]["function"]["arguments"]) == {"query": "deploy service X"} + assert second[3]["tool_call_id"] == "call_s" and "Deployment Guide" in second[3]["content"] + + fourth = requests[3]["body"]["messages"] + assert fourth[-1]["role"] == "tool" and fourth[-1]["tool_call_id"] == "call_w" + assert CANARY_PAT not in json.dumps(requests) + + +@pytest.mark.asyncio +async def test_openai_adapter_invalid_tool_json_fails_safely(tmp_path: Path): + bad = oa_tool_turn("call_x", "bash", {}) + bad["choices"][0]["message"]["tool_calls"][0]["function"]["arguments"] = "{not json" + runner, settings, store, mgr, adapter, requests = build(tmp_path, [bad]) + try: + with pytest.raises(AppError) as excinfo: + await run(runner) + finally: + await adapter.close() + assert excinfo.value.code in {"upstream_failed", "execution_failed"} + assert "{not json" not in excinfo.value.message + assert store.global_reserved_bytes == 0 + + +@pytest.mark.asyncio +async def test_openai_adapter_provider_error_never_leaks_body(tmp_path: Path): + responses = [httpx.Response(500, json={"error": {"message": "PROVIDER-SECRET-DETAILS"}})] + runner, settings, store, mgr, adapter, requests = build(tmp_path, responses) + try: + with pytest.raises(AppError) as excinfo: + await run(runner) + finally: + await adapter.close() + assert excinfo.value.code in {"upstream_failed", "execution_failed"} + assert "PROVIDER-SECRET-DETAILS" not in excinfo.value.message