from __future__ import annotations import argparse import json from pathlib import Path from typing import Any ROOT = Path(__file__).resolve().parents[1] DEFAULT_NORMALIZED_MANIFEST = ROOT / "plugins" / "1c" / "rag" / "official-docs" / "normalized" / "manifest.json" DEFAULT_RAW_MANIFEST = ROOT / "plugins" / "1c" / "rag" / "official-docs" / "raw" / "manifest.json" DEFAULT_RAG_SOURCE_DIR = ROOT / "plugins" / "1c" / "rag" / "sources" / "official" / "its" DEFAULT_CORPUS = ROOT / "plugins" / "1c" / "datasets" / "prepared" / "rag_corpus.jsonl" FORBIDDEN_TEXT = ( "Мы используем файлы cookie", "Продолжая находиться на сайте", "Результаты поиска", "Купить кассу", "Календарь бухгалтера", "Последние результаты поиска", ) NAVIGATION_CLUES = ( "Руководство разработчика - Руководство администратора", "Глоссарий разработчика - 1 - 1CEClientSetupMake.exe", ) def load_json(path: Path) -> dict[str, Any]: if not path.exists(): return {} try: data = json.loads(path.read_text(encoding="utf-8-sig")) except json.JSONDecodeError: return {} return data if isinstance(data, dict) else {} def scan_text_file(path: Path, needles: tuple[str, ...]) -> list[dict[str, Any]]: try: text = path.read_text(encoding="utf-8-sig", errors="ignore") except OSError as exc: return [{"severity": "error", "code": "read_failed", "path": str(path), "message": str(exc)}] findings = [] for needle in needles: if needle in text: findings.append({"severity": "error", "code": "forbidden_text", "path": str(path), "text": needle}) return findings def scan_jsonl(path: Path, needles: tuple[str, ...]) -> list[dict[str, Any]]: findings: list[dict[str, Any]] = [] if not path.exists(): findings.append({"severity": "warning", "code": "missing_corpus", "path": str(path)}) return findings for line_no, line in enumerate(path.read_text(encoding="utf-8-sig", errors="ignore").splitlines(), start=1): if not line.strip(): continue try: item = json.loads(line) except json.JSONDecodeError: findings.append({"severity": "error", "code": "bad_jsonl", "path": str(path), "line": line_no}) continue content = str(item.get("content") or "") for needle in needles: if needle in content: findings.append( { "severity": "error", "code": "forbidden_text_in_corpus", "path": str(path), "line": line_no, "chunk_id": item.get("id"), "text": needle, } ) return findings def raw_url_counts(raw_manifest_path: Path) -> dict[str, int]: manifest = load_json(raw_manifest_path) counts = {"raw_pages": 0, "content_src_pages": 0, "hdoc_pages": 0, "root_pages": 0} for page in manifest.get("pages") or []: counts["raw_pages"] += 1 url = str(page.get("url") or "") if "/db/content/" in url and "/src/" in url: counts["content_src_pages"] += 1 elif "/content/" in url and url.endswith("/hdoc"): counts["hdoc_pages"] += 1 else: counts["root_pages"] += 1 return counts def check_quality(manifest_path: Path, raw_manifest_path: Path, rag_source_dir: Path, corpus_path: Path) -> dict[str, Any]: findings: list[dict[str, Any]] = [] manifest = load_json(manifest_path) raw_counts = raw_url_counts(raw_manifest_path) page_count = int(manifest.get("page_count") or 0) skipped_count = int(manifest.get("skipped_count") or 0) discovered_src_count = int(manifest.get("discovered_src_record_count") or 0) media_page_count = 0 media_image_count = 0 table_count = 0 for page in manifest.get("pages") or []: media = page.get("media") or {} images = media.get("images") or [] if images: media_page_count += 1 media_image_count += len(images) table_count += int(media.get("table_count") or 0) if not manifest: findings.append({"severity": "warning", "code": "missing_normalized_manifest", "path": str(manifest_path)}) elif page_count == 0: findings.append( { "severity": "warning", "code": "no_official_content_pages", "message": "No official 1C:ITS pages passed normalization quality gates. Refresh cookie and fetch with --no-resume.", "skipped_count": skipped_count, } ) if raw_counts["raw_pages"] and not raw_counts["content_src_pages"] and not discovered_src_count: findings.append( { "severity": "warning", "code": "no_raw_content_src_pages", "message": "Raw fetch has pages, but no /db/content/.../src/... pages. Fetch probably stopped at hdoc shells/navigation.", "raw_counts": raw_counts, } ) if rag_source_dir.exists(): for path in sorted(rag_source_dir.glob("*.md")): findings.extend(scan_text_file(path, FORBIDDEN_TEXT + NAVIGATION_CLUES)) else: findings.append({"severity": "warning", "code": "missing_rag_source_dir", "path": str(rag_source_dir)}) findings.extend(scan_jsonl(corpus_path, FORBIDDEN_TEXT + NAVIGATION_CLUES)) errors = [item for item in findings if item.get("severity") == "error"] warnings = [item for item in findings if item.get("severity") == "warning"] return { "schema": "onec_official_docs_quality_check.v1", "passed": not errors, "counts": { **raw_counts, "discovered_src_record_count": discovered_src_count, "media_pages": media_page_count, "media_images": media_image_count, "tables": table_count, "normalized_pages": page_count, "skipped_pages": skipped_count, "errors": len(errors), "warnings": len(warnings), "findings": len(findings), }, "manifest": str(manifest_path), "raw_manifest": str(raw_manifest_path), "rag_source_dir": str(rag_source_dir), "corpus": str(corpus_path), "findings": findings, } def main() -> int: parser = argparse.ArgumentParser(description="Check official 1C:ITS normalized docs and RAG corpus quality.") parser.add_argument("--manifest", type=Path, default=DEFAULT_NORMALIZED_MANIFEST) parser.add_argument("--raw-manifest", type=Path, default=DEFAULT_RAW_MANIFEST) parser.add_argument("--rag-source-dir", type=Path, default=DEFAULT_RAG_SOURCE_DIR) parser.add_argument("--corpus", type=Path, default=DEFAULT_CORPUS) parser.add_argument("--output", type=Path) parser.add_argument("--print", action="store_true", dest="print_full") args = parser.parse_args() result = check_quality(args.manifest, args.raw_manifest, args.rag_source_dir, args.corpus) if args.output: args.output.parent.mkdir(parents=True, exist_ok=True) args.output.write_text(json.dumps(result, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") payload = result if args.print_full else {"passed": result["passed"], "counts": result["counts"], "output": str(args.output) if args.output else None} print(json.dumps(payload, ensure_ascii=False, indent=2)) return 0 if result["passed"] else 2 if __name__ == "__main__": raise SystemExit(main())