from __future__ import annotations import argparse import hashlib import json from pathlib import Path from typing import Any from fetch_1c_its_docs import LinkParser, charset_from_content_type, is_content_src_url, normalize_url, safe_slug ROOT = Path(__file__).resolve().parents[1] DEFAULT_RAW_DIR = ROOT / "plugins" / "1c" / "rag" / "official-docs" / "raw" DEFAULT_MANIFEST = DEFAULT_RAW_DIR / "manifest.json" def load_json(path: Path) -> dict[str, Any]: if not path.exists(): return {} try: data = json.loads(path.read_text(encoding="utf-8-sig")) except json.JSONDecodeError: return {} return data if isinstance(data, dict) else {} def inspect_plan(manifest_path: Path, raw_dir: Path) -> dict[str, Any]: manifest = load_json(manifest_path) records = manifest.get("pages") or [] existing_urls = {str(record.get("url") or "") for record in records} candidates_by_url: dict[str, dict[str, Any]] = {} shell_pages = 0 for record in records: filename = str(record.get("file") or "") url = str(record.get("url") or "") if is_content_src_url(url): continue path = raw_dir / filename if not path.exists(): continue try: body = path.read_bytes() text = body.decode(charset_from_content_type(str(record.get("content_type") or "")), errors="replace") except OSError: continue parser = LinkParser() parser.feed(text) src_links = [] for href in parser.links: next_url = normalize_url(url, href) if next_url and is_content_src_url(next_url): src_links.append(next_url) if not src_links: continue shell_pages += 1 for src_url in src_links: if src_url.endswith("#_print"): continue candidates_by_url[src_url] = { "url": src_url, "expected_file": safe_slug(src_url), "already_in_manifest": src_url in existing_urls, "already_on_disk": (raw_dir / safe_slug(src_url)).exists(), "parent_url": url, "parent_title": record.get("title"), "source_id": record.get("source_id"), "source_type": record.get("source_type"), } candidates = sorted(candidates_by_url.values(), key=lambda item: item["url"]) missing = [item for item in candidates if not item["already_in_manifest"] and not item["already_on_disk"]] return { "schema": "onec_its_fetch_plan_inspection.v1", "manifest": str(manifest_path), "raw_dir": str(raw_dir), "manifest_sha256": hashlib.sha256(manifest_path.read_bytes()).hexdigest() if manifest_path.exists() else None, "counts": { "raw_pages": len(records), "shell_pages_with_src": shell_pages, "src_candidates": len(candidates), "missing_src_candidates": len(missing), }, "missing_src_candidates": missing, "src_candidates": candidates, } def main() -> int: parser = argparse.ArgumentParser(description="Inspect already fetched 1C:ITS hdoc shells for missing /src/ body pages.") parser.add_argument("--manifest", type=Path, default=DEFAULT_MANIFEST) parser.add_argument("--raw-dir", type=Path, default=DEFAULT_RAW_DIR) parser.add_argument("--output", type=Path) parser.add_argument("--print", action="store_true", dest="print_full") args = parser.parse_args() result = inspect_plan(args.manifest, args.raw_dir) if args.output: args.output.parent.mkdir(parents=True, exist_ok=True) args.output.write_text(json.dumps(result, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") payload = result if args.print_full else {"counts": result["counts"], "output": str(args.output) if args.output else None} print(json.dumps(payload, ensure_ascii=False, indent=2)) return 0 if __name__ == "__main__": raise SystemExit(main())