Artur Mukhamadiev
e65fbf4b67
FastAPI app, upstream Confluence/model adapters, authoritative history, rootless container lifecycle, artifact storage and downloads, fake peers under backend/dev, tests under tests/backend. Root pytest.ini deselects the live marker by default; requirements gain the backend dependencies.
367 lines
15 KiB
Python
367 lines
15 KiB
Python
"""Confluence client and tool dispatcher."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import json
|
|
import re
|
|
import ssl
|
|
import sys
|
|
import os
|
|
from typing import Any, Dict, List, Optional, Tuple
|
|
import httpx
|
|
|
|
from backend.errors import AppError, ConfluenceAuthFailedError, DestinationDeniedError, InvalidInputError, TlsFailedError, ConnectivityFailedError, UpstreamFailedError
|
|
from backend.errors import UpstreamResponseTooLargeError
|
|
from backend.settings import validate_confluence_url
|
|
from backend.upstream import read_json
|
|
|
|
MAX_RESPONSE_BYTES = 128 * 1024 * 1024 # 128 MiB wire and decompressed limit
|
|
MAX_TOOL_RESULT_BYTES = 120 * 1024 * 1024 # leave headroom for 128 MiB JSON payload
|
|
_PAGE_ID_RE = re.compile(r"^[0-9]+$")
|
|
|
|
|
|
def escape_cql_literal(s: str) -> str:
|
|
"""Escape a literal string for use in Confluence CQL queries."""
|
|
# Escape backslashes, quotes, and control characters
|
|
return s.replace("\\", "\\\\").replace('"', '\\"').replace("\n", " ").replace("\r", " ")
|
|
|
|
|
|
class ConfluenceClient:
|
|
"""Request-scoped Confluence client with streaming byte bounds and no redirects."""
|
|
|
|
def __init__(
|
|
self,
|
|
base_url: str,
|
|
pat: str,
|
|
approved_origins: List[str],
|
|
corporate_ca_path: Optional[str] = None,
|
|
timeout: float = 30.0,
|
|
transport: Optional[httpx.AsyncBaseTransport] = None,
|
|
):
|
|
self.canonical_url = validate_confluence_url(base_url, approved_origins)
|
|
self.pat = pat.strip()
|
|
if not self.pat:
|
|
raise InvalidInputError("Confluence PAT cannot be empty")
|
|
if len(self.pat.encode("utf-8")) > 8192:
|
|
raise InvalidInputError("Confluence PAT exceeds 8 KiB limit")
|
|
|
|
verify_param: Any = corporate_ca_path if corporate_ca_path else True
|
|
self._client = httpx.AsyncClient(
|
|
base_url=self.canonical_url,
|
|
verify=verify_param,
|
|
follow_redirects=False,
|
|
timeout=httpx.Timeout(timeout),
|
|
headers={
|
|
"Authorization": f"Bearer {self.pat}",
|
|
"Accept": "application/json",
|
|
"Accept-Encoding": "gzip, deflate",
|
|
},
|
|
transport=transport,
|
|
)
|
|
self._closed = False
|
|
|
|
async def close(self) -> None:
|
|
if not self._closed:
|
|
self._closed = True
|
|
await self._client.aclose()
|
|
|
|
async def __aenter__(self) -> ConfluenceClient:
|
|
return self
|
|
|
|
async def __aexit__(self, exc_type, exc_val, exc_tb) -> None:
|
|
await self.close()
|
|
|
|
async def _request_json(self, method: str, path: str, params: Optional[Dict[str, Any]] = None) -> Any:
|
|
"""Perform request with streaming size limits and sanitized errors."""
|
|
try:
|
|
req = self._client.build_request(method, path, params=params)
|
|
response = await self._client.send(req, stream=True)
|
|
except ssl.SSLError as e:
|
|
raise TlsFailedError(f"TLS certificate verification failed: {type(e).__name__}") from e
|
|
except httpx.ConnectError as e:
|
|
if isinstance(e.__cause__, ssl.SSLError) or "CERTIFICATE_VERIFY_FAILED" in str(e):
|
|
raise TlsFailedError() from e
|
|
raise ConnectivityFailedError(f"Failed to connect to Confluence: {type(e).__name__}") from e
|
|
except httpx.TimeoutException as e:
|
|
raise ConnectivityFailedError("Confluence request timed out") from e
|
|
except Exception as e:
|
|
raise UpstreamFailedError(f"Upstream request failed: {type(e).__name__}") from e
|
|
|
|
# Check for redirects
|
|
if 300 <= response.status_code < 400:
|
|
await response.aclose()
|
|
raise DestinationDeniedError("Confluence server returned an unexpected redirect")
|
|
|
|
# Check for auth failure
|
|
if response.status_code in (401, 403):
|
|
await response.aclose()
|
|
raise ConfluenceAuthFailedError("Confluence authentication failed")
|
|
|
|
if response.status_code != 200:
|
|
await response.aclose()
|
|
raise UpstreamFailedError(f"Confluence returned HTTP {response.status_code}")
|
|
|
|
# Check content type
|
|
content_type = response.headers.get("content-type", "")
|
|
if "application/json" not in content_type:
|
|
await response.aclose()
|
|
raise UpstreamFailedError("Confluence returned non-JSON response")
|
|
|
|
return await read_json(response, MAX_RESPONSE_BYTES)
|
|
|
|
async def verify_auth(self) -> bool:
|
|
"""Verify authentication with a bounded read without pi."""
|
|
data = await self._request_json("GET", "/rest/api/space", params={"limit": 1})
|
|
if not isinstance(data, dict) or "results" not in data or not isinstance(data.get("results"), list):
|
|
raise UpstreamFailedError("Unexpected Confluence response structure")
|
|
return True
|
|
|
|
@staticmethod
|
|
def _results(data):
|
|
if not isinstance(data, dict) or not isinstance(data.get("results"), list) or any(not isinstance(item, dict) for item in data["results"]):
|
|
raise UpstreamFailedError("Invalid Confluence result structure")
|
|
return data["results"]
|
|
|
|
async def search(
|
|
self,
|
|
query: str,
|
|
space: Optional[str] = None,
|
|
limit: int = 10,
|
|
offset: int = 0,
|
|
) -> Dict[str, Any]:
|
|
"""Search Confluence using escaped CQL."""
|
|
if not isinstance(query, str):
|
|
raise InvalidInputError("Query must be a string")
|
|
if type(limit) is not int or not (1 <= limit <= 50):
|
|
raise InvalidInputError("Limit must be between 1 and 50")
|
|
if type(offset) is not int or not (0 <= offset <= 10000):
|
|
raise InvalidInputError("Offset must be between 0 and 10000")
|
|
|
|
cql_parts = []
|
|
if space is not None and not isinstance(space, str):
|
|
raise InvalidInputError("Space key must be a string")
|
|
if space:
|
|
if len(space.encode("utf-8")) > 256:
|
|
raise InvalidInputError("Space key exceeds 256 bytes")
|
|
escaped_space = escape_cql_literal(space.strip())
|
|
cql_parts.append(f'space = "{escaped_space}"')
|
|
|
|
escaped_query = escape_cql_literal(query.strip())
|
|
if escaped_query:
|
|
cql_parts.append(f'text ~ "{escaped_query}"')
|
|
else:
|
|
cql_parts.append('type = "page"')
|
|
|
|
cql = " AND ".join(cql_parts)
|
|
params = {"cql": cql, "start": offset, "limit": limit}
|
|
|
|
data = await self._request_json("GET", "/rest/api/content/search", params=params)
|
|
results = self._results(data)
|
|
total_size = data.get("totalSize", offset + len(results))
|
|
if type(total_size) is not int or total_size < 0:
|
|
raise UpstreamFailedError("Invalid Confluence pagination")
|
|
|
|
pages = []
|
|
for item in results:
|
|
page_id = str(item.get("id", ""))
|
|
title = str(item.get("title", ""))
|
|
if not isinstance(item.get("space") or {}, dict):
|
|
raise UpstreamFailedError("Invalid Confluence space structure")
|
|
space_key = (item.get("space") or {}).get("key", "")
|
|
# Validate page_id format before interpolating into URL
|
|
if _PAGE_ID_RE.match(page_id):
|
|
url = f"{self.canonical_url}/pages/viewpage.action?pageId={page_id}"
|
|
else:
|
|
url = ""
|
|
snippet = str(item.get("excerpt", "") or "")
|
|
pages.append({
|
|
"page_id": page_id,
|
|
"title": title,
|
|
"space": space_key,
|
|
"url": url,
|
|
"snippet": snippet,
|
|
})
|
|
|
|
has_more = (offset + len(pages)) < total_size and len(pages) > 0
|
|
return {
|
|
"pages": pages,
|
|
"pagination": {
|
|
"offset": offset,
|
|
"limit": limit,
|
|
"has_more": has_more,
|
|
},
|
|
}
|
|
|
|
async def view(self, page_id: str) -> Dict[str, Any]:
|
|
"""View page content and convert to Markdown."""
|
|
if not isinstance(page_id, str) or not _PAGE_ID_RE.match(page_id):
|
|
raise InvalidInputError("page_id must be a decimal string")
|
|
|
|
path = f"/rest/api/content/{page_id}"
|
|
params = {"expand": "body.storage,version,space"}
|
|
|
|
data = await self._request_json("GET", path, params=params)
|
|
|
|
if not isinstance(data, dict) or not isinstance(data.get("body", {}), dict) or not isinstance(data.get("space", {}), dict):
|
|
raise UpstreamFailedError("Invalid Confluence page structure")
|
|
title = str(data.get("title", ""))
|
|
space_key = (data.get("space") or {}).get("key", "")
|
|
url = f"{self.canonical_url}/pages/viewpage.action?pageId={page_id}"
|
|
storage = (data.get("body") or {}).get("storage", {})
|
|
if not isinstance(storage, dict):
|
|
raise UpstreamFailedError("Invalid Confluence storage structure")
|
|
storage_html = storage.get("value", "")
|
|
|
|
if not isinstance(storage_html, str):
|
|
raise UpstreamFailedError("Invalid Confluence storage content")
|
|
# A disposable process can actually be terminated on timeout/disconnect.
|
|
proc = await asyncio.create_subprocess_exec(sys.executable, "-m", "backend.conversion", stdin=asyncio.subprocess.PIPE, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.DEVNULL, env={k: os.environ[k] for k in ("PATH", "LANG", "PYTHONPATH") if k in os.environ})
|
|
try:
|
|
output, _ = await asyncio.wait_for(proc.communicate(storage_html.encode("utf-8")), timeout=15)
|
|
if proc.returncode != 0:
|
|
raise UpstreamFailedError("Content conversion failed")
|
|
markdown = output.decode("utf-8")
|
|
except asyncio.TimeoutError as exc:
|
|
raise UpstreamFailedError("Content conversion timed out") from exc
|
|
finally:
|
|
if proc.returncode is None:
|
|
proc.kill()
|
|
await proc.communicate()
|
|
|
|
truncated = False
|
|
md_bytes = markdown.encode("utf-8")
|
|
if len(md_bytes) > MAX_TOOL_RESULT_BYTES:
|
|
truncated = True
|
|
markdown = md_bytes[:MAX_TOOL_RESULT_BYTES].decode("utf-8", errors="ignore") + "\n\n...[truncated]"
|
|
|
|
return {
|
|
"page_id": page_id,
|
|
"title": title,
|
|
"space": space_key,
|
|
"url": url,
|
|
"markdown": markdown,
|
|
"truncated": truncated,
|
|
}
|
|
|
|
async def list_spaces(self, limit: int = 25, offset: int = 0) -> Dict[str, Any]:
|
|
"""List spaces with pagination."""
|
|
if type(limit) is not int or not (1 <= limit <= 50):
|
|
raise InvalidInputError("Limit must be between 1 and 50")
|
|
if type(offset) is not int or not (0 <= offset <= 10000):
|
|
raise InvalidInputError("Offset must be between 0 and 10000")
|
|
|
|
params = {"start": offset, "limit": limit}
|
|
data = await self._request_json("GET", "/rest/api/space", params=params)
|
|
|
|
results = self._results(data)
|
|
total_size = data.get("totalSize", offset + len(results))
|
|
if type(total_size) is not int or total_size < 0:
|
|
raise UpstreamFailedError("Invalid Confluence pagination")
|
|
|
|
spaces = [
|
|
{"key": str(s.get("key", "")), "name": str(s.get("name", ""))}
|
|
for s in results
|
|
]
|
|
has_more = (offset + len(spaces)) < total_size and len(spaces) > 0
|
|
|
|
return {
|
|
"spaces": spaces,
|
|
"pagination": {
|
|
"offset": offset,
|
|
"limit": limit,
|
|
"has_more": has_more,
|
|
},
|
|
}
|
|
|
|
|
|
class ConfluenceDispatcher:
|
|
"""Tool dispatcher with caching, call limits, and error sanitization."""
|
|
|
|
def __init__(self, client: ConfluenceClient, max_calls: int = 100):
|
|
self.client = client
|
|
self.max_calls = max_calls
|
|
self.call_count = 0
|
|
self._cache: Dict[str, Any] = {}
|
|
|
|
async def _bounded_result(self, result):
|
|
size = await asyncio.to_thread(lambda: len(json.dumps(result).encode("utf-8")))
|
|
if size > 128 * 1024 * 1024:
|
|
raise UpstreamResponseTooLargeError("Serialized tool result exceeds 128 MiB")
|
|
return result
|
|
|
|
async def dispatch(self, tool_name: str, parameters: Dict[str, Any]) -> Tuple[Any, Optional[dict], bool]:
|
|
"""Dispatch a tool request. Returns (result, error_dict, cache_hit)."""
|
|
allowed = {
|
|
"confluence_search": {"query", "space", "limit", "offset"},
|
|
"confluence_view": {"page_id"},
|
|
"confluence_list_spaces": {"limit", "offset"},
|
|
}
|
|
if tool_name not in allowed or not isinstance(parameters, dict) or set(parameters) - allowed[tool_name]:
|
|
return None, InvalidInputError("Unsupported Confluence tool or parameter fields").to_error_dict(), False
|
|
parameters = dict(parameters)
|
|
if tool_name == "confluence_search":
|
|
parameters.setdefault("limit", 10)
|
|
parameters.setdefault("offset", 0)
|
|
elif tool_name == "confluence_list_spaces":
|
|
parameters.setdefault("limit", 25)
|
|
parameters.setdefault("offset", 0)
|
|
import hashlib
|
|
# Hash cache key and check cache BEFORE consuming call budget
|
|
cache_key = f"{tool_name}:" + hashlib.sha256(json.dumps(parameters, sort_keys=True).encode("utf-8")).hexdigest()
|
|
if cache_key in self._cache:
|
|
return self._cache[cache_key], None, True
|
|
|
|
if self.call_count >= self.max_calls:
|
|
err = UpstreamFailedError(f"Confluence call limit ({self.max_calls}) reached")
|
|
return None, err.to_error_dict(), False
|
|
|
|
self.call_count += 1
|
|
|
|
if tool_name == "confluence_search":
|
|
query = parameters.get("query", "")
|
|
space = parameters.get("space")
|
|
limit = parameters.get("limit", 10)
|
|
offset = parameters.get("offset", 0)
|
|
|
|
try:
|
|
res = await self._bounded_result(await self.client.search(query=query, space=space, limit=limit, offset=offset))
|
|
self._cache[cache_key] = res
|
|
return res, None, False
|
|
except AppError as e:
|
|
return None, e.to_error_dict(), False
|
|
except Exception as e:
|
|
err = UpstreamFailedError(f"Search failed: {type(e).__name__}")
|
|
return None, err.to_error_dict(), False
|
|
|
|
elif tool_name == "confluence_view":
|
|
page_id = parameters.get("page_id", "")
|
|
|
|
try:
|
|
res = await self._bounded_result(await self.client.view(page_id=page_id))
|
|
self._cache[cache_key] = res
|
|
return res, None, False
|
|
except AppError as e:
|
|
return None, e.to_error_dict(), False
|
|
except Exception as e:
|
|
err = UpstreamFailedError(f"View page failed: {type(e).__name__}")
|
|
return None, err.to_error_dict(), False
|
|
|
|
elif tool_name == "confluence_list_spaces":
|
|
limit = parameters.get("limit", 25)
|
|
offset = parameters.get("offset", 0)
|
|
|
|
try:
|
|
res = await self._bounded_result(await self.client.list_spaces(limit=limit, offset=offset))
|
|
self._cache[cache_key] = res
|
|
return res, None, False
|
|
except AppError as e:
|
|
return None, e.to_error_dict(), False
|
|
except Exception as e:
|
|
err = UpstreamFailedError(f"List spaces failed: {type(e).__name__}")
|
|
return None, err.to_error_dict(), False
|
|
|
|
else:
|
|
err = InvalidInputError(f"Unknown tool: '{tool_name}'")
|
|
return None, err.to_error_dict(), False
|