108 lines
4.0 KiB
Python
108 lines
4.0 KiB
Python
from __future__ import annotations
|
|
|
|
import argparse
|
|
import hashlib
|
|
import json
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
from fetch_1c_its_docs import LinkParser, charset_from_content_type, is_content_src_url, normalize_url, safe_slug
|
|
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
DEFAULT_RAW_DIR = ROOT / "plugins" / "1c" / "rag" / "official-docs" / "raw"
|
|
DEFAULT_MANIFEST = DEFAULT_RAW_DIR / "manifest.json"
|
|
|
|
|
|
def load_json(path: Path) -> dict[str, Any]:
|
|
if not path.exists():
|
|
return {}
|
|
try:
|
|
data = json.loads(path.read_text(encoding="utf-8-sig"))
|
|
except json.JSONDecodeError:
|
|
return {}
|
|
return data if isinstance(data, dict) else {}
|
|
|
|
|
|
def inspect_plan(manifest_path: Path, raw_dir: Path) -> dict[str, Any]:
|
|
manifest = load_json(manifest_path)
|
|
records = manifest.get("pages") or []
|
|
existing_urls = {str(record.get("url") or "") for record in records}
|
|
candidates_by_url: dict[str, dict[str, Any]] = {}
|
|
shell_pages = 0
|
|
|
|
for record in records:
|
|
filename = str(record.get("file") or "")
|
|
url = str(record.get("url") or "")
|
|
if is_content_src_url(url):
|
|
continue
|
|
path = raw_dir / filename
|
|
if not path.exists():
|
|
continue
|
|
try:
|
|
body = path.read_bytes()
|
|
text = body.decode(charset_from_content_type(str(record.get("content_type") or "")), errors="replace")
|
|
except OSError:
|
|
continue
|
|
parser = LinkParser()
|
|
parser.feed(text)
|
|
src_links = []
|
|
for href in parser.links:
|
|
next_url = normalize_url(url, href)
|
|
if next_url and is_content_src_url(next_url):
|
|
src_links.append(next_url)
|
|
if not src_links:
|
|
continue
|
|
shell_pages += 1
|
|
for src_url in src_links:
|
|
if src_url.endswith("#_print"):
|
|
continue
|
|
candidates_by_url[src_url] = {
|
|
"url": src_url,
|
|
"expected_file": safe_slug(src_url),
|
|
"already_in_manifest": src_url in existing_urls,
|
|
"already_on_disk": (raw_dir / safe_slug(src_url)).exists(),
|
|
"parent_url": url,
|
|
"parent_title": record.get("title"),
|
|
"source_id": record.get("source_id"),
|
|
"source_type": record.get("source_type"),
|
|
}
|
|
|
|
candidates = sorted(candidates_by_url.values(), key=lambda item: item["url"])
|
|
missing = [item for item in candidates if not item["already_in_manifest"] and not item["already_on_disk"]]
|
|
return {
|
|
"schema": "onec_its_fetch_plan_inspection.v1",
|
|
"manifest": str(manifest_path),
|
|
"raw_dir": str(raw_dir),
|
|
"manifest_sha256": hashlib.sha256(manifest_path.read_bytes()).hexdigest() if manifest_path.exists() else None,
|
|
"counts": {
|
|
"raw_pages": len(records),
|
|
"shell_pages_with_src": shell_pages,
|
|
"src_candidates": len(candidates),
|
|
"missing_src_candidates": len(missing),
|
|
},
|
|
"missing_src_candidates": missing,
|
|
"src_candidates": candidates,
|
|
}
|
|
|
|
|
|
def main() -> int:
|
|
parser = argparse.ArgumentParser(description="Inspect already fetched 1C:ITS hdoc shells for missing /src/ body pages.")
|
|
parser.add_argument("--manifest", type=Path, default=DEFAULT_MANIFEST)
|
|
parser.add_argument("--raw-dir", type=Path, default=DEFAULT_RAW_DIR)
|
|
parser.add_argument("--output", type=Path)
|
|
parser.add_argument("--print", action="store_true", dest="print_full")
|
|
args = parser.parse_args()
|
|
|
|
result = inspect_plan(args.manifest, args.raw_dir)
|
|
if args.output:
|
|
args.output.parent.mkdir(parents=True, exist_ok=True)
|
|
args.output.write_text(json.dumps(result, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
|
payload = result if args.print_full else {"counts": result["counts"], "output": str(args.output) if args.output else None}
|
|
print(json.dumps(payload, ensure_ascii=False, indent=2))
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|