183 lines
7.1 KiB
Python
183 lines
7.1 KiB
Python
"""Crawl Confluence spaces and pages into Markdown files (PAT auth)."""
|
|
import json
|
|
import logging
|
|
import time
|
|
from pathlib import Path
|
|
|
|
from atlassian import Confluence
|
|
from requests.exceptions import HTTPError, RequestException
|
|
|
|
from .markdown import page_filename, render_markdown
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
class ConfluenceCrawler:
|
|
"""Crawls a Confluence Data Center instance using a personal access token.
|
|
|
|
Note: pagination is done manually with start/limit offsets because the
|
|
library's automatic next-link following resolves ``_links.next`` against
|
|
the site root, which breaks on instances deployed under a context path
|
|
(e.g. https://host/main).
|
|
"""
|
|
|
|
def __init__(self, url: str, pat: str, timeout: int = 30):
|
|
self.url = url
|
|
# token= sets "Authorization: Bearer <pat>" (Data Center PAT auth)
|
|
self.confluence = Confluence(url=url, token=pat, timeout=timeout)
|
|
|
|
# ------------------------------------------------------------------ #
|
|
# low-level helpers
|
|
# ------------------------------------------------------------------ #
|
|
|
|
def _get_with_retry(self, path: str, params: dict, retries: int = 3, backoff: float = 2.0) -> dict:
|
|
"""GET with retry on transient errors (5xx, 429, connection issues).
|
|
|
|
Client errors (4xx except 429) are raised immediately.
|
|
"""
|
|
last_exc: Exception | None = None
|
|
for attempt in range(retries + 1):
|
|
try:
|
|
return self.confluence.get(path, params=params)
|
|
except HTTPError as exc:
|
|
code = exc.response.status_code if exc.response is not None else None
|
|
if code is not None and 400 <= code < 500 and code != 429:
|
|
raise # non-retryable client error
|
|
last_exc = exc
|
|
except RequestException as exc:
|
|
last_exc = exc
|
|
if attempt < retries:
|
|
wait = backoff * (2**attempt)
|
|
logger.warning(
|
|
"GET %s failed (%s); retry %d/%d in %.0fs",
|
|
path, last_exc, attempt + 1, retries, wait,
|
|
)
|
|
time.sleep(wait)
|
|
assert last_exc is not None
|
|
raise last_exc
|
|
|
|
# ------------------------------------------------------------------ #
|
|
# listing
|
|
# ------------------------------------------------------------------ #
|
|
|
|
def list_spaces(self, limit: int = 50, max_spaces: int | None = None) -> list[dict]:
|
|
"""Return visible spaces (manual offset pagination).
|
|
|
|
max_spaces caps how many are fetched (recommended for tests —
|
|
enumerating the full directory on large instances is slow).
|
|
"""
|
|
spaces: list[dict] = []
|
|
start = 0
|
|
while True:
|
|
resp = self.confluence.get_all_spaces(start=start, limit=limit)
|
|
results = resp.get("results", [])
|
|
spaces.extend(results)
|
|
if max_spaces is not None and len(spaces) >= max_spaces:
|
|
return spaces[:max_spaces]
|
|
total = resp.get("totalSize")
|
|
if not results:
|
|
break
|
|
if total is not None and start + len(results) >= total:
|
|
break
|
|
if total is None and len(results) < limit:
|
|
break
|
|
start += len(results)
|
|
return spaces
|
|
|
|
def get_pages(
|
|
self,
|
|
space_key: str,
|
|
expand: str = "body.storage,version,space",
|
|
limit: int = 50,
|
|
max_pages: int | None = None,
|
|
) -> list[dict]:
|
|
"""Return pages in a space (manual offset pagination).
|
|
|
|
max_pages caps the number fetched (useful for tests / dry runs).
|
|
"""
|
|
pages: list[dict] = []
|
|
start = 0
|
|
while True:
|
|
resp = self._get_with_retry(
|
|
"rest/api/content",
|
|
params={
|
|
"spaceKey": space_key,
|
|
"type": "page",
|
|
"start": start,
|
|
"limit": limit,
|
|
"expand": expand,
|
|
},
|
|
)
|
|
results = resp.get("results", [])
|
|
pages.extend(results)
|
|
if max_pages is not None and len(pages) >= max_pages:
|
|
logger.info("space %s: reached max_pages=%d", space_key, max_pages)
|
|
return pages[:max_pages]
|
|
if len(results) < limit:
|
|
return pages
|
|
start += len(results)
|
|
logger.info("space %s: fetched %d pages so far...", space_key, len(pages))
|
|
|
|
# ------------------------------------------------------------------ #
|
|
# crawling
|
|
# ------------------------------------------------------------------ #
|
|
|
|
def crawl_space(self, space_key: str, out_dir: Path, max_pages: int | None = None) -> list[Path]:
|
|
"""Crawl one space; write one .md file per page + manifest.json."""
|
|
out_dir = Path(out_dir)
|
|
out_dir.mkdir(parents=True, exist_ok=True)
|
|
pages = self.get_pages(space_key, max_pages=max_pages)
|
|
|
|
written: list[Path] = []
|
|
manifest: list[dict] = []
|
|
failed: list[str] = []
|
|
for i, page in enumerate(pages, 1):
|
|
try:
|
|
md = render_markdown(page, self.url)
|
|
except Exception as exc: # never let one page kill the crawl
|
|
logger.error("render failed for page %s: %s", page.get("id"), exc)
|
|
failed.append(str(page.get("id")))
|
|
md = (
|
|
render_markdown(
|
|
{**page, "body": {"storage": {"value": ""}}}, self.url
|
|
)
|
|
+ f"\n> [!] conversion failed: {exc}\n"
|
|
)
|
|
path = out_dir / page_filename(page)
|
|
path.write_text(md, encoding="utf-8")
|
|
written.append(path)
|
|
manifest.append(
|
|
{
|
|
"id": page.get("id"),
|
|
"title": page.get("title"),
|
|
"file": path.name,
|
|
"url": f"{self.url}/pages/viewpage.action?pageId={page.get('id')}",
|
|
}
|
|
)
|
|
if i % 100 == 0:
|
|
logger.info("space %s: wrote %d/%d pages", space_key, i, len(pages))
|
|
|
|
(out_dir / "manifest.json").write_text(
|
|
json.dumps(manifest, indent=2, ensure_ascii=False),
|
|
encoding="utf-8",
|
|
)
|
|
logger.info("space %s: done, %d pages -> %s (failed: %d)",
|
|
space_key, len(pages), out_dir, len(failed))
|
|
return written
|
|
|
|
def crawl_all(self, out_dir: Path, max_spaces: int | None = None) -> dict[str, list[Path]]:
|
|
"""Crawl all visible spaces into out_dir/<space_key>/."""
|
|
out_dir = Path(out_dir)
|
|
spaces = self.list_spaces()
|
|
if max_spaces:
|
|
spaces = spaces[:max_spaces]
|
|
result: dict[str, list[Path]] = {}
|
|
for space in spaces:
|
|
key = space.get("key")
|
|
if not key:
|
|
continue
|
|
try:
|
|
result[key] = self.crawl_space(key, out_dir / key)
|
|
except Exception as exc: # keep crawling other spaces
|
|
logger.error("space %s failed: %s", key, exc)
|
|
return result |