"""Real backend provider adapter + real runtime, without credentials or network. The OpenAI-compatible adapter talks to a scripted httpx transport that records the provider-native requests and returns provider-native responses. The runtime image runs for real; only the provider HTTP layer is scripted. """ from __future__ import annotations import json from pathlib import Path from typing import Any, Dict, List import httpx import pytest from backend.artifacts import ArtifactStore from backend.containers import DockerContainerManager from backend.errors import AppError from backend.model import OpenAIModelAdapter from backend.runner import QueryRunner from backend.settings import Settings from tests.integration.conftest import ( APPROVED_ORIGIN, CANARY_PAT, CHECKLIST_BYTES, CHECKLIST_TEXT, FINAL_ANSWER, PAGE_ID, RUNTIME_IMAGE, make_confluence_factory, requires_runtime_image, ) pytestmark = [pytest.mark.integration, requires_runtime_image()] ENDPOINT = "http://scripted-provider.invalid/v1/chat/completions" DUMMY_KEY = "scripted-provider-key-not-real" def oa_tool_turn(call_id: str, name: str, args: Dict[str, Any]) -> Dict[str, Any]: return { "id": "chatcmpl-scripted", "object": "chat.completion", "choices": [{ "index": 0, "message": { "role": "assistant", "content": None, "tool_calls": [{"id": call_id, "type": "function", "function": {"name": name, "arguments": json.dumps(args)}}], }, "finish_reason": "tool_calls", }], "usage": {"prompt_tokens": 42, "completion_tokens": 7}, } def oa_text_turn(text: str, finish_reason: str = "stop") -> Dict[str, Any]: return { "id": "chatcmpl-scripted", "object": "chat.completion", "choices": [{"index": 0, "message": {"role": "assistant", "content": text}, "finish_reason": finish_reason}], "usage": {"prompt_tokens": 50, "completion_tokens": 20}, } def scripted_provider(responses: List[Any]): requests: List[Dict[str, Any]] = [] pending = list(responses) async def handler(request: httpx.Request) -> httpx.Response: body = json.loads(request.content) requests.append({"headers": dict(request.headers), "body": body}) assert pending, "provider script exhausted" nxt = pending.pop(0) if isinstance(nxt, httpx.Response): return nxt return httpx.Response(200, json=nxt) return httpx.MockTransport(handler), requests def build(tmp_path: Path, responses: List[Any]): settings = Settings( runtime_image=RUNTIME_IMAGE, approved_confluence_origins=[APPROVED_ORIGIN], query_timeout_seconds=120.0, container_label_value=f"integration-{tmp_path.name[-8:]}", artifact_storage_dir=tmp_path / "artifacts", model_provider="openai", model_name="scripted-model", model_api_key=DUMMY_KEY, model_endpoint=ENDPOINT, model_context_window_tokens=131072, model_max_output_tokens=8192, ) transport, requests = scripted_provider(responses) adapter = OpenAIModelAdapter( api_key=settings.model_api_key, model_name=settings.model_name, endpoint=settings.model_endpoint, context_window_tokens=settings.model_context_window_tokens, max_output_tokens=settings.model_max_output_tokens, transport=transport, ) store = ArtifactStore(settings.artifact_storage_dir) mgr = DockerContainerManager(settings) runner = QueryRunner( settings=settings, container_manager=mgr, artifact_store=store, model_adapter=adapter, confluence_client_factory=make_confluence_factory(), ) return runner, settings, store, mgr, adapter, requests async def run(runner: QueryRunner): return await runner.run( prompt="How do I deploy service X? Produce a deployment checklist file too.", confluence_url=APPROVED_ORIGIN, confluence_pat=CANARY_PAT, session_id="cw_sess_provider", ) @pytest.mark.asyncio async def test_openai_adapter_text_and_tool_turns_with_real_runtime(tmp_path: Path): responses = [ oa_tool_turn("call_s", "confluence_search", {"query": "deploy service X"}), oa_tool_turn("call_v", "confluence_view", {"page_id": PAGE_ID}), oa_tool_turn("call_w", "write", {"path": "/work/artifacts/checklist.md", "content": CHECKLIST_TEXT}), oa_text_turn(FINAL_ANSWER), ] runner, settings, store, mgr, adapter, requests = build(tmp_path, responses) try: result = await run(runner) finally: await adapter.close() assert result["markdown"] == FINAL_ANSWER assert [h["tool"] for h in result["tool_history"]] == ["confluence_search", "confluence_view"] assert [p["page_id"] for p in result["pages_accessed"]] == [PAGE_ID] art = result["artifacts"][0] dl = store.get_artifact_for_download(art["id"], session_id="cw_sess_provider") assert dl is not None and dl[0].read_bytes() == CHECKLIST_BYTES store.release_reader(art["id"]) assert len(requests) == 4 for r in requests: assert r["headers"]["authorization"] == f"Bearer {DUMMY_KEY}" body = r["body"] assert body["model"] == "scripted-model" and body["max_tokens"] == 8192 assert body["messages"][0]["role"] == "system" and body["messages"][0]["content"] assert sum(1 for m in body["messages"] if m["role"] == "system") == 1 assert {t["function"]["name"] for t in body["tools"]} >= {"confluence_search", "confluence_view", "confluence_list_spaces", "bash", "read", "write", "edit"} assert all(t["type"] == "function" and isinstance(t["function"]["parameters"], dict) for t in body["tools"]) second = requests[1]["body"]["messages"] assert [m["role"] for m in second] == ["system", "user", "assistant", "tool"] assert second[2]["tool_calls"][0]["id"] == "call_s" assert json.loads(second[2]["tool_calls"][0]["function"]["arguments"]) == {"query": "deploy service X"} assert second[3]["tool_call_id"] == "call_s" and "Deployment Guide" in second[3]["content"] fourth = requests[3]["body"]["messages"] assert fourth[-1]["role"] == "tool" and fourth[-1]["tool_call_id"] == "call_w" assert CANARY_PAT not in json.dumps(requests) @pytest.mark.asyncio async def test_openai_adapter_invalid_tool_json_fails_safely(tmp_path: Path): bad = oa_tool_turn("call_x", "bash", {}) bad["choices"][0]["message"]["tool_calls"][0]["function"]["arguments"] = "{not json" runner, settings, store, mgr, adapter, requests = build(tmp_path, [bad]) try: with pytest.raises(AppError) as excinfo: await run(runner) finally: await adapter.close() assert excinfo.value.code in {"upstream_failed", "execution_failed"} assert "{not json" not in excinfo.value.message assert store.global_reserved_bytes == 0 @pytest.mark.asyncio async def test_openai_adapter_provider_error_never_leaks_body(tmp_path: Path): responses = [httpx.Response(500, json={"error": {"message": "PROVIDER-SECRET-DETAILS"}})] runner, settings, store, mgr, adapter, requests = build(tmp_path, responses) try: with pytest.raises(AppError) as excinfo: await run(runner) finally: await adapter.close() assert excinfo.value.code in {"upstream_failed", "execution_failed"} assert "PROVIDER-SECRET-DETAILS" not in excinfo.value.message