Artur Mukhamadiev
d77b52ae86
- tests/integration: frontend served by backend, runtime error mapping, backend + real pi image with scripted model/Confluence (shared example, variants, failure paths, cancellation/busy gate, isolation canaries, HTTP download ownership, 16 MiB prompt round trip), real OpenAI-compatible adapter + real image over a scripted transport, real Chrome against the real backend with a scripted runtime peer, backend crash/restart reconciliation, and an opt-in live model check (marker: live). - backend: map runtime terminal codes (model_output_limit, model_context_exceeded, query_timeout, connectivity_failed) to the contract's HTTP statuses; make the artifact 404 body identical for no-session, wrong-session, unknown and expired IDs. - Makefile, scripts/run-backend.sh, deploy/confluence-web.env.example, root README for the integrated application; integration pytest marker.
98 lines
4.5 KiB
Python
98 lines
4.5 KiB
Python
"""Live provider check: the configured OpenAI-compatible endpoint drives the real runtime.
|
|
|
|
Deselected by default (marker ``live``). Confluence stays scripted (no PAT
|
|
needed); the model and the container are real. Run with:
|
|
|
|
set -a; . deploy/confluence-web.env; set +a
|
|
.venv/bin/python -m pytest -m live tests/integration/test_live_model.py -s
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import os
|
|
import socket
|
|
import urllib.parse
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
from backend.artifacts import ArtifactStore
|
|
from backend.containers import DockerContainerManager
|
|
from backend.model import OpenAIModelAdapter
|
|
from backend.runner import QueryRunner
|
|
from backend.settings import Settings
|
|
|
|
from tests.integration.conftest import APPROVED_ORIGIN, CANARY_PAT, CHECKLIST_BYTES, PAGE_ID, PAGE_URL, RUNTIME_IMAGE, make_confluence_factory, requires_runtime_image
|
|
|
|
pytestmark = [pytest.mark.live, pytest.mark.integration, requires_runtime_image()]
|
|
|
|
|
|
def endpoint_reachable(url: str) -> bool:
|
|
try:
|
|
parsed = urllib.parse.urlsplit(url)
|
|
with socket.create_connection((parsed.hostname, parsed.port or (443 if parsed.scheme == "https" else 80)), timeout=2):
|
|
return True
|
|
except Exception:
|
|
return False
|
|
|
|
|
|
@pytest.mark.skipif(os.getenv("CONFLUENCE_WEB_MODEL_PROVIDER") != "openai" or not os.getenv("CONFLUENCE_WEB_MODEL_ENDPOINT"), reason="deployment model configuration not exported")
|
|
@pytest.mark.asyncio
|
|
async def test_live_model_completes_search_read_write_cite(tmp_path: Path):
|
|
env = Settings.from_env()
|
|
if not endpoint_reachable(env.model_endpoint):
|
|
pytest.skip(f"model endpoint not reachable: {env.model_endpoint}")
|
|
settings = Settings(
|
|
runtime_image=RUNTIME_IMAGE,
|
|
approved_confluence_origins=[APPROVED_ORIGIN],
|
|
container_label_value=f"integration-live-{tmp_path.name[-6:]}",
|
|
artifact_storage_dir=tmp_path / "artifacts",
|
|
model_provider="openai",
|
|
model_name=env.model_name,
|
|
model_api_key=env.model_api_key,
|
|
model_endpoint=env.model_endpoint,
|
|
model_context_window_tokens=env.model_context_window_tokens,
|
|
model_max_output_tokens=env.model_max_output_tokens,
|
|
model_timeout_seconds=env.model_timeout_seconds,
|
|
)
|
|
adapter = OpenAIModelAdapter(
|
|
api_key=settings.model_api_key, model_name=settings.model_name, endpoint=settings.model_endpoint,
|
|
context_window_tokens=settings.model_context_window_tokens, max_output_tokens=settings.model_max_output_tokens,
|
|
timeout=settings.model_timeout_seconds,
|
|
)
|
|
store = ArtifactStore(settings.artifact_storage_dir)
|
|
mgr = DockerContainerManager(settings)
|
|
await mgr.verify_rootless()
|
|
runner = QueryRunner(settings=settings, container_manager=mgr, artifact_store=store, model_adapter=adapter, confluence_client_factory=make_confluence_factory())
|
|
try:
|
|
result = await runner.run(
|
|
prompt=(
|
|
"How do I deploy service X? Search Confluence for it and read the most relevant page. "
|
|
"Then create the file /work/artifacts/checklist.md containing a short Markdown deployment checklist. "
|
|
"Answer briefly in Markdown and cite the Confluence page URL you used."
|
|
),
|
|
confluence_url=APPROVED_ORIGIN,
|
|
confluence_pat=CANARY_PAT,
|
|
session_id="cw_live",
|
|
)
|
|
finally:
|
|
await adapter.close()
|
|
|
|
print(json.dumps({k: v for k, v in result.items() if k != "markdown"}, indent=1)[:3000])
|
|
print("MARKDOWN:\n" + result["markdown"][:2000])
|
|
tools = [h["tool"] for h in result["tool_history"]]
|
|
assert "confluence_search" in tools and "confluence_view" in tools
|
|
assert all(h["status"] == "success" for h in result["tool_history"] if h["tool"] == "confluence_view" and h["parameters"].get("page_id") == PAGE_ID)
|
|
assert [p["page_id"] for p in result["pages_accessed"]] == [PAGE_ID]
|
|
assert PAGE_URL in result["markdown"] or "pageId=847291" in result["markdown"]
|
|
names = [a["name"] for a in result["artifacts"]]
|
|
assert "checklist.md" in names, names
|
|
art = next(a for a in result["artifacts"] if a["name"] == "checklist.md")
|
|
dl = store.get_artifact_for_download(art["id"], session_id="cw_live")
|
|
assert dl is not None and dl[2] == art["size_bytes"] > 0
|
|
store.release_reader(art["id"])
|
|
code, out, _ = await mgr._exec_docker(["ps", "-aq", "--filter", f"label={settings.container_label_key}={settings.container_label_value}"])
|
|
assert code == 0 and out.strip() == ""
|
|
assert CANARY_PAT not in json.dumps(result)
|