56 lines
1.9 KiB
Python
56 lines
1.9 KiB
Python
#!/usr/bin/env python3
|
|
"""CLI: crawl Confluence into Markdown files.
|
|
|
|
Reads CONFLUENCE_PAT and CONFLUENCE_URL from .env.
|
|
|
|
Examples:
|
|
python main.py --space KEY --out output/KEY
|
|
python main.py --all --out output
|
|
python main.py --all --max-spaces 3 --verbose
|
|
"""
|
|
import argparse
|
|
import json
|
|
import logging
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
from confluence_crawler.config import get_config
|
|
from confluence_crawler.crawler import ConfluenceCrawler
|
|
|
|
|
|
def main(argv: list[str] | None = None) -> int:
|
|
parser = argparse.ArgumentParser(description="Crawl Confluence pages to Markdown.")
|
|
parser.add_argument("--space", help="crawl only this space key")
|
|
parser.add_argument("--all", action="store_true", help="crawl all visible spaces")
|
|
parser.add_argument("--out", default="output", help="output directory (default: output)")
|
|
parser.add_argument("--max-spaces", type=int, help="limit number of spaces with --all")
|
|
parser.add_argument("--max-pages", type=int, help="limit pages per space (dry run)")
|
|
parser.add_argument("--verbose", action="store_true", help="debug logging")
|
|
args = parser.parse_args(argv)
|
|
|
|
logging.basicConfig(level=logging.DEBUG if args.verbose else logging.INFO)
|
|
|
|
if not args.space and not args.all:
|
|
parser.error("specify --space KEY or --all")
|
|
|
|
try:
|
|
cfg = get_config()
|
|
except RuntimeError as exc:
|
|
print(f"error: {exc}", file=sys.stderr)
|
|
return 2
|
|
|
|
crawler = ConfluenceCrawler(cfg["url"], cfg["pat"])
|
|
out = Path(args.out)
|
|
if args.space:
|
|
paths = crawler.crawl_space(args.space, out, max_pages=args.max_pages)
|
|
print(f"wrote {len(paths)} pages to {out}")
|
|
else:
|
|
result = crawler.crawl_all(out, max_spaces=args.max_spaces)
|
|
total = sum(len(p) for p in result.values())
|
|
print(f"wrote {total} pages across {len(result)} spaces to {out}")
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|