- tests/integration: frontend served by backend, runtime error mapping, backend + real pi image with scripted model/Confluence (shared example, variants, failure paths, cancellation/busy gate, isolation canaries, HTTP download ownership, 16 MiB prompt round trip), real OpenAI-compatible adapter + real image over a scripted transport, real Chrome against the real backend with a scripted runtime peer, backend crash/restart reconciliation, and an opt-in live model check (marker: live). - backend: map runtime terminal codes (model_output_limit, model_context_exceeded, query_timeout, connectivity_failed) to the contract's HTTP statuses; make the artifact 404 body identical for no-session, wrong-session, unknown and expired IDs. - Makefile, scripts/run-backend.sh, deploy/confluence-web.env.example, root README for the integrated application; integration pytest marker.
186 lines
7.1 KiB
Python
186 lines
7.1 KiB
Python
"""Shared helpers for integration tests that connect the real subsystems.
|
|
|
|
These tests are owned by the integration stage. They never contact real
|
|
services: Confluence is a scripted httpx transport, the model is a scripted
|
|
adapter or a scripted OpenAI-compatible transport, and the only real
|
|
component is the built pi runtime image executed through the backend's
|
|
rootless Docker manager.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import os
|
|
import shutil
|
|
import subprocess
|
|
from pathlib import Path
|
|
from typing import Any, Dict, List, Optional, Sequence, Tuple
|
|
|
|
import httpx
|
|
import pytest
|
|
|
|
from backend.confluence import ConfluenceClient
|
|
from backend.model import FakeModelAdapter
|
|
|
|
REPO_ROOT = Path(__file__).resolve().parents[2]
|
|
FRONTEND_DIR = REPO_ROOT / "frontend"
|
|
RUNTIME_IMAGE = os.getenv("CONFLUENCE_WEB_RUNTIME_IMAGE", "confluence-pi-agent:rev1")
|
|
|
|
APPROVED_ORIGIN = "https://approved.example.com"
|
|
PAGE_ID = "847291"
|
|
PAGE_URL = f"{APPROVED_ORIGIN}/pages/viewpage.action?pageId={PAGE_ID}"
|
|
DENIED_PAGE_ID = "999999"
|
|
CHECKLIST_BYTES = b"# Checklist\n\n- Deploy service X\n"
|
|
CHECKLIST_TEXT = CHECKLIST_BYTES.decode("utf-8")
|
|
# A recognizable dummy PAT used only in fixtures; it must never appear in the container.
|
|
CANARY_PAT = "canary-pat-do-not-leak-0123456789abcdef"
|
|
FINAL_ANSWER = (
|
|
"Deploy service X using the release checklist "
|
|
f"([Deployment Guide]({PAGE_URL})). A checklist.md artifact was created."
|
|
)
|
|
|
|
|
|
def docker_available() -> bool:
|
|
if shutil.which("docker") is None:
|
|
return False
|
|
try:
|
|
res = subprocess.run(["docker", "info"], capture_output=True, timeout=5.0)
|
|
return res.returncode == 0
|
|
except Exception:
|
|
return False
|
|
|
|
|
|
def image_available(image: str) -> bool:
|
|
try:
|
|
res = subprocess.run(["docker", "image", "inspect", image], capture_output=True, timeout=10.0)
|
|
return res.returncode == 0
|
|
except Exception:
|
|
return False
|
|
|
|
|
|
def requires_runtime_image():
|
|
return pytest.mark.skipif(
|
|
not (docker_available() and image_available(RUNTIME_IMAGE)),
|
|
reason=f"rootless Docker and runtime image {RUNTIME_IMAGE} are required",
|
|
)
|
|
|
|
|
|
# --- scripted Confluence upstream -------------------------------------------------
|
|
|
|
def make_confluence_factory(calls: Optional[List[Dict[str, Any]]] = None):
|
|
"""Return a ConfluenceClient factory backed by a scripted httpx transport.
|
|
|
|
Accepts the same keyword arguments as the production factory, including
|
|
the optional ``proxy`` argument that production passes when configured.
|
|
"""
|
|
recorded = calls if calls is not None else []
|
|
|
|
async def handler(request: httpx.Request) -> httpx.Response:
|
|
recorded.append({"path": request.url.path, "params": dict(request.url.params)})
|
|
if request.headers.get("Authorization") != f"Bearer {CANARY_PAT}":
|
|
return httpx.Response(401, json={"message": "Unauthorized"})
|
|
path = request.url.path
|
|
if path.endswith("/rest/api/space"):
|
|
return httpx.Response(200, json={"results": [{"key": "OPS", "name": "Operations"}], "totalSize": 1})
|
|
if path.endswith("/rest/api/content/search"):
|
|
cql = request.url.params.get("cql", "")
|
|
if "nothing" in cql:
|
|
return httpx.Response(200, json={"results": [], "totalSize": 0})
|
|
return httpx.Response(
|
|
200,
|
|
json={
|
|
"results": [
|
|
{"id": PAGE_ID, "title": "Deployment Guide", "space": {"key": "OPS"}, "excerpt": "Deployment steps"}
|
|
],
|
|
"totalSize": 1,
|
|
},
|
|
)
|
|
if path.endswith(f"/rest/api/content/{PAGE_ID}"):
|
|
return httpx.Response(
|
|
200,
|
|
json={
|
|
"id": PAGE_ID,
|
|
"title": "Deployment Guide",
|
|
"space": {"key": "OPS"},
|
|
"body": {"storage": {"value": "<p>Deploy service X using the release checklist.</p>"}},
|
|
},
|
|
)
|
|
if path.endswith(f"/rest/api/content/{DENIED_PAGE_ID}"):
|
|
return httpx.Response(403, json={"message": "SECRET-UPSTREAM-BODY forbidden"})
|
|
return httpx.Response(404, json={"message": "not found"})
|
|
|
|
def factory(base_url: str, pat: str, approved_origins: list, corporate_ca_path=None, timeout: float = 30.0, **kwargs) -> ConfluenceClient:
|
|
return ConfluenceClient(
|
|
base_url=base_url,
|
|
pat=pat,
|
|
approved_origins=approved_origins,
|
|
corporate_ca_path=corporate_ca_path,
|
|
timeout=timeout,
|
|
transport=httpx.MockTransport(handler),
|
|
)
|
|
|
|
return factory
|
|
|
|
|
|
# --- scripted neutral model turns ---------------------------------------------------
|
|
|
|
def text_turn(text: str, stop_reason: str = "stop", in_tok: int = 10, out_tok: int = 10) -> Dict[str, Any]:
|
|
return {
|
|
"content": [{"type": "text", "text": text}] if text else [],
|
|
"stop_reason": stop_reason,
|
|
"usage": {"input_tokens": in_tok, "output_tokens": out_tok},
|
|
}
|
|
|
|
|
|
def tool_turn(calls: Sequence[Tuple[str, str, Dict[str, Any]]], text: str = "") -> Dict[str, Any]:
|
|
content: List[Dict[str, Any]] = []
|
|
if text:
|
|
content.append({"type": "text", "text": text})
|
|
for call_id, name, args in calls:
|
|
content.append({"type": "tool_call", "id": call_id, "name": name, "arguments": args})
|
|
return {"content": content, "stop_reason": "tool_calls", "usage": {"input_tokens": 10, "output_tokens": 5}}
|
|
|
|
|
|
def example_script(final_text: str = FINAL_ANSWER) -> List[Dict[str, Any]]:
|
|
"""The CONTRACTS section 7 shared example as scripted model turns."""
|
|
return [
|
|
tool_turn([("call_1", "confluence_search", {"query": "deploy service X"})]),
|
|
tool_turn([("call_2", "confluence_view", {"page_id": PAGE_ID})]),
|
|
tool_turn([("call_3", "write", {"path": "/work/artifacts/checklist.md", "content": CHECKLIST_TEXT})]),
|
|
text_turn(final_text),
|
|
]
|
|
|
|
|
|
class ScriptedModelAdapter(FakeModelAdapter):
|
|
"""FakeModelAdapter that can also raise or stall on a scripted turn."""
|
|
|
|
def __init__(self, script: List[Any]):
|
|
super().__init__(script=None)
|
|
self._steps: List[Any] = list(script)
|
|
self.started = asyncio.Event()
|
|
|
|
async def complete(self, messages, tools, system_instruction, provider_state_meta=None):
|
|
self.call_count += 1
|
|
self.received_requests.append({
|
|
"messages": messages,
|
|
"tools": tools,
|
|
"system_instruction": system_instruction,
|
|
"provider_state_meta": provider_state_meta,
|
|
})
|
|
self.started.set()
|
|
if not self._steps:
|
|
raise AssertionError("Scripted model adapter ran out of turns")
|
|
step = self._steps.pop(0)
|
|
if isinstance(step, Exception):
|
|
raise step
|
|
if callable(step):
|
|
return await step()
|
|
return step
|
|
|
|
|
|
def tool_message_text(request: Dict[str, Any], tool_call_id: str) -> str:
|
|
for m in request["messages"]:
|
|
if m.get("role") == "tool" and m.get("tool_call_id") == tool_call_id:
|
|
return m.get("content", "")
|
|
raise AssertionError(f"no tool message for {tool_call_id}")
|