Initial SQL-only 1C adapter baseline

2026-07-22 03:03:47 +03:00
commit e2503b77e7
545 changed files with 184711 additions and 0 deletions
@@ -0,0 +1,107 @@
+from __future__ import annotations
+
+import argparse
+import hashlib
+import json
+from pathlib import Path
+from typing import Any
+
+from fetch_1c_its_docs import LinkParser, charset_from_content_type, is_content_src_url, normalize_url, safe_slug
+
+
+ROOT = Path(__file__).resolve().parents[1]
+DEFAULT_RAW_DIR = ROOT / "plugins" / "1c" / "rag" / "official-docs" / "raw"
+DEFAULT_MANIFEST = DEFAULT_RAW_DIR / "manifest.json"
+
+
+def load_json(path: Path) -> dict[str, Any]:
+    if not path.exists():
+        return {}
+    try:
+        data = json.loads(path.read_text(encoding="utf-8-sig"))
+    except json.JSONDecodeError:
+        return {}
+    return data if isinstance(data, dict) else {}
+
+
+def inspect_plan(manifest_path: Path, raw_dir: Path) -> dict[str, Any]:
+    manifest = load_json(manifest_path)
+    records = manifest.get("pages") or []
+    existing_urls = {str(record.get("url") or "") for record in records}
+    candidates_by_url: dict[str, dict[str, Any]] = {}
+    shell_pages = 0
+
+    for record in records:
+        filename = str(record.get("file") or "")
+        url = str(record.get("url") or "")
+        if is_content_src_url(url):
+            continue
+        path = raw_dir / filename
+        if not path.exists():
+            continue
+        try:
+            body = path.read_bytes()
+            text = body.decode(charset_from_content_type(str(record.get("content_type") or "")), errors="replace")
+        except OSError:
+            continue
+        parser = LinkParser()
+        parser.feed(text)
+        src_links = []
+        for href in parser.links:
+            next_url = normalize_url(url, href)
+            if next_url and is_content_src_url(next_url):
+                src_links.append(next_url)
+        if not src_links:
+            continue
+        shell_pages += 1
+        for src_url in src_links:
+            if src_url.endswith("#_print"):
+                continue
+            candidates_by_url[src_url] = {
+                "url": src_url,
+                "expected_file": safe_slug(src_url),
+                "already_in_manifest": src_url in existing_urls,
+                "already_on_disk": (raw_dir / safe_slug(src_url)).exists(),
+                "parent_url": url,
+                "parent_title": record.get("title"),
+                "source_id": record.get("source_id"),
+                "source_type": record.get("source_type"),
+            }
+
+    candidates = sorted(candidates_by_url.values(), key=lambda item: item["url"])
+    missing = [item for item in candidates if not item["already_in_manifest"] and not item["already_on_disk"]]
+    return {
+        "schema": "onec_its_fetch_plan_inspection.v1",
+        "manifest": str(manifest_path),
+        "raw_dir": str(raw_dir),
+        "manifest_sha256": hashlib.sha256(manifest_path.read_bytes()).hexdigest() if manifest_path.exists() else None,
+        "counts": {
+            "raw_pages": len(records),
+            "shell_pages_with_src": shell_pages,
+            "src_candidates": len(candidates),
+            "missing_src_candidates": len(missing),
+        },
+        "missing_src_candidates": missing,
+        "src_candidates": candidates,
+    }
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description="Inspect already fetched 1C:ITS hdoc shells for missing /src/ body pages.")
+    parser.add_argument("--manifest", type=Path, default=DEFAULT_MANIFEST)
+    parser.add_argument("--raw-dir", type=Path, default=DEFAULT_RAW_DIR)
+    parser.add_argument("--output", type=Path)
+    parser.add_argument("--print", action="store_true", dest="print_full")
+    args = parser.parse_args()
+
+    result = inspect_plan(args.manifest, args.raw_dir)
+    if args.output:
+        args.output.parent.mkdir(parents=True, exist_ok=True)
+        args.output.write_text(json.dumps(result, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
+    payload = result if args.print_full else {"counts": result["counts"], "output": str(args.output) if args.output else None}
+    print(json.dumps(payload, ensure_ascii=False, indent=2))
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())