confluence_web/tests/integration/test_real_provider_adapter.py

"""Real backend provider adapter + real runtime, without credentials or network.

The OpenAI-compatible adapter talks to a scripted httpx transport that
records the provider-native requests and returns provider-native responses.
The runtime image runs for real; only the provider HTTP layer is scripted.
"""

from __future__ import annotations

import json
from pathlib import Path
from typing import Any, Dict, List

import httpx
import pytest

from backend.artifacts import ArtifactStore
from backend.containers import DockerContainerManager
from backend.errors import AppError
from backend.model import OpenAIModelAdapter
from backend.runner import QueryRunner
from backend.settings import Settings

from tests.integration.conftest import (
    APPROVED_ORIGIN,
    CANARY_PAT,
    CHECKLIST_BYTES,
    CHECKLIST_TEXT,
    FINAL_ANSWER,
    PAGE_ID,
    RUNTIME_IMAGE,
    make_confluence_factory,
    requires_runtime_image,
)

pytestmark = [pytest.mark.integration, requires_runtime_image()]

ENDPOINT = "http://scripted-provider.invalid/v1/chat/completions"
DUMMY_KEY = "scripted-provider-key-not-real"


def oa_tool_turn(call_id: str, name: str, args: Dict[str, Any]) -> Dict[str, Any]:
    return {
        "id": "chatcmpl-scripted",
        "object": "chat.completion",
        "choices": [{
            "index": 0,
            "message": {
                "role": "assistant",
                "content": None,
                "tool_calls": [{"id": call_id, "type": "function", "function": {"name": name, "arguments": json.dumps(args)}}],
            },
            "finish_reason": "tool_calls",
        }],
        "usage": {"prompt_tokens": 42, "completion_tokens": 7},
    }


def oa_text_turn(text: str, finish_reason: str = "stop") -> Dict[str, Any]:
    return {
        "id": "chatcmpl-scripted",
        "object": "chat.completion",
        "choices": [{"index": 0, "message": {"role": "assistant", "content": text}, "finish_reason": finish_reason}],
        "usage": {"prompt_tokens": 50, "completion_tokens": 20},
    }


def scripted_provider(responses: List[Any]):
    requests: List[Dict[str, Any]] = []
    pending = list(responses)

    async def handler(request: httpx.Request) -> httpx.Response:
        body = json.loads(request.content)
        requests.append({"headers": dict(request.headers), "body": body})
        assert pending, "provider script exhausted"
        nxt = pending.pop(0)
        if isinstance(nxt, httpx.Response):
            return nxt
        return httpx.Response(200, json=nxt)

    return httpx.MockTransport(handler), requests


def build(tmp_path: Path, responses: List[Any]):
    settings = Settings(
        runtime_image=RUNTIME_IMAGE,
        approved_confluence_origins=[APPROVED_ORIGIN],
        query_timeout_seconds=120.0,
        container_label_value=f"integration-{tmp_path.name[-8:]}",
        artifact_storage_dir=tmp_path / "artifacts",
        model_provider="openai",
        model_name="scripted-model",
        model_api_key=DUMMY_KEY,
        model_endpoint=ENDPOINT,
        model_context_window_tokens=131072,
        model_max_output_tokens=8192,
    )
    transport, requests = scripted_provider(responses)
    adapter = OpenAIModelAdapter(
        api_key=settings.model_api_key,
        model_name=settings.model_name,
        endpoint=settings.model_endpoint,
        context_window_tokens=settings.model_context_window_tokens,
        max_output_tokens=settings.model_max_output_tokens,
        transport=transport,
    )
    store = ArtifactStore(settings.artifact_storage_dir)
    mgr = DockerContainerManager(settings)
    runner = QueryRunner(
        settings=settings,
        container_manager=mgr,
        artifact_store=store,
        model_adapter=adapter,
        confluence_client_factory=make_confluence_factory(),
    )
    return runner, settings, store, mgr, adapter, requests


async def run(runner: QueryRunner):
    return await runner.run(
        prompt="How do I deploy service X? Produce a deployment checklist file too.",
        confluence_url=APPROVED_ORIGIN,
        confluence_pat=CANARY_PAT,
        session_id="cw_sess_provider",
    )


@pytest.mark.asyncio
async def test_openai_adapter_text_and_tool_turns_with_real_runtime(tmp_path: Path):
    responses = [
        oa_tool_turn("call_s", "confluence_search", {"query": "deploy service X"}),
        oa_tool_turn("call_v", "confluence_view", {"page_id": PAGE_ID}),
        oa_tool_turn("call_w", "write", {"path": "/work/artifacts/checklist.md", "content": CHECKLIST_TEXT}),
        oa_text_turn(FINAL_ANSWER),
    ]
    runner, settings, store, mgr, adapter, requests = build(tmp_path, responses)
    try:
        result = await run(runner)
    finally:
        await adapter.close()

    assert result["markdown"] == FINAL_ANSWER
    assert [h["tool"] for h in result["tool_history"]] == ["confluence_search", "confluence_view"]
    assert [p["page_id"] for p in result["pages_accessed"]] == [PAGE_ID]
    art = result["artifacts"][0]
    dl = store.get_artifact_for_download(art["id"], session_id="cw_sess_provider")
    assert dl is not None and dl[0].read_bytes() == CHECKLIST_BYTES
    store.release_reader(art["id"])

    assert len(requests) == 4
    for r in requests:
        assert r["headers"]["authorization"] == f"Bearer {DUMMY_KEY}"
        body = r["body"]
        assert body["model"] == "scripted-model" and body["max_tokens"] == 8192
        assert body["messages"][0]["role"] == "system" and body["messages"][0]["content"]
        assert sum(1 for m in body["messages"] if m["role"] == "system") == 1
        assert {t["function"]["name"] for t in body["tools"]} >= {"confluence_search", "confluence_view", "confluence_list_spaces", "bash", "read", "write", "edit"}
        assert all(t["type"] == "function" and isinstance(t["function"]["parameters"], dict) for t in body["tools"])

    second = requests[1]["body"]["messages"]
    assert [m["role"] for m in second] == ["system", "user", "assistant", "tool"]
    assert second[2]["tool_calls"][0]["id"] == "call_s"
    assert json.loads(second[2]["tool_calls"][0]["function"]["arguments"]) == {"query": "deploy service X"}
    assert second[3]["tool_call_id"] == "call_s" and "Deployment Guide" in second[3]["content"]

    fourth = requests[3]["body"]["messages"]
    assert fourth[-1]["role"] == "tool" and fourth[-1]["tool_call_id"] == "call_w"
    assert CANARY_PAT not in json.dumps(requests)


@pytest.mark.asyncio
async def test_openai_adapter_invalid_tool_json_fails_safely(tmp_path: Path):
    bad = oa_tool_turn("call_x", "bash", {})
    bad["choices"][0]["message"]["tool_calls"][0]["function"]["arguments"] = "{not json"
    runner, settings, store, mgr, adapter, requests = build(tmp_path, [bad])
    try:
        with pytest.raises(AppError) as excinfo:
            await run(runner)
    finally:
        await adapter.close()
    assert excinfo.value.code in {"upstream_failed", "execution_failed"}
    assert "{not json" not in excinfo.value.message
    assert store.global_reserved_bytes == 0


@pytest.mark.asyncio
async def test_openai_adapter_provider_error_never_leaks_body(tmp_path: Path):
    responses = [httpx.Response(500, json={"error": {"message": "PROVIDER-SECRET-DETAILS"}})]
    runner, settings, store, mgr, adapter, requests = build(tmp_path, responses)
    try:
        with pytest.raises(AppError) as excinfo:
            await run(runner)
    finally:
        await adapter.close()
    assert excinfo.value.code in {"upstream_failed", "execution_failed"}
    assert "PROVIDER-SECRET-DETAILS" not in excinfo.value.message