Initial SQL-only 1C adapter baseline

2026-07-22 03:03:47 +03:00
commit e2503b77e7
545 changed files with 184711 additions and 0 deletions
@@ -0,0 +1,113 @@
+from __future__ import annotations
+
+import argparse
+import hashlib
+import json
+from pathlib import Path
+
+from common import read_json
+from prepare_1c_rag_corpus import SUPPORTED_EXTENSIONS, classify_source, normalize_text, parse_front_matter
+
+
+ROOT = Path(__file__).resolve().parents[1]
+DEFAULT_SOURCE_DIR = ROOT / "plugins" / "1c" / "rag" / "sources"
+DEFAULT_MANIFEST = ROOT / "plugins" / "1c" / "datasets" / "prepared" / "rag_manifest.json"
+
+
+def iter_source_files(source_dir: Path) -> list[Path]:
+    if not source_dir.exists():
+        return []
+    return sorted(
+        path
+        for path in source_dir.rglob("*")
+        if path.is_file() and path.suffix.lower() in SUPPORTED_EXTENSIONS
+    )
+
+
+def source_state(source_dir: Path) -> dict[str, dict]:
+    state = {}
+    for path in iter_source_files(source_dir):
+        relative_path = path.relative_to(source_dir).as_posix()
+        text = normalize_text(path.read_text(encoding="utf-8"))
+        front_matter, body = parse_front_matter(text)
+        chunk_source_text = body or text
+        state[relative_path] = {
+            "source_path": relative_path,
+            "source_type": classify_source(path, chunk_source_text, front_matter),
+            "file_type": path.suffix.lower().lstrip("."),
+            "content_hash": hashlib.sha256(text.encode("utf-8")).hexdigest(),
+        }
+    return state
+
+
+def compare_manifest(source_dir: Path, manifest_path: Path) -> dict:
+    current = source_state(source_dir)
+    if not manifest_path.exists():
+        return {
+            "status": "stale" if current else "missing",
+            "reason": "manifest is missing",
+            "new": sorted(current),
+            "changed": [],
+            "deleted": [],
+            "type_changed": [],
+        }
+
+    manifest = read_json(manifest_path)
+    recorded = {
+        str(source.get("source_path")): source
+        for source in manifest.get("sources") or []
+        if isinstance(source, dict) and source.get("source_path")
+    }
+
+    current_paths = set(current)
+    recorded_paths = set(recorded)
+    new = sorted(current_paths - recorded_paths)
+    deleted = sorted(recorded_paths - current_paths)
+    changed = []
+    type_changed = []
+
+    for source_path in sorted(current_paths & recorded_paths):
+        current_source = current[source_path]
+        recorded_source = recorded[source_path]
+        if current_source["content_hash"] != recorded_source.get("content_hash"):
+            changed.append(source_path)
+        if current_source["source_type"] != recorded_source.get("source_type"):
+            type_changed.append(
+                {
+                    "source_path": source_path,
+                    "current": current_source["source_type"],
+                    "manifest": recorded_source.get("source_type"),
+                }
+            )
+
+    stale = bool(new or deleted or changed or type_changed)
+    return {
+        "status": "stale" if stale else "fresh",
+        "source_dir": str(source_dir),
+        "manifest": str(manifest_path),
+        "source_count": len(current),
+        "manifest_source_count": len(recorded),
+        "new": new,
+        "changed": changed,
+        "deleted": deleted,
+        "type_changed": type_changed,
+    }
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description="Check whether the 1C RAG manifest is fresh.")
+    parser.add_argument("--source-dir", type=Path, default=DEFAULT_SOURCE_DIR)
+    parser.add_argument("--manifest", type=Path, default=DEFAULT_MANIFEST)
+    parser.add_argument("--print", action="store_true")
+    args = parser.parse_args()
+
+    report = compare_manifest(args.source_dir, args.manifest)
+    if args.print:
+        print(json.dumps(report, ensure_ascii=False, indent=2))
+    else:
+        print(f"1C RAG freshness: {report['status']}")
+    return 0 if report["status"] in {"fresh", "missing"} else 1
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())