from __future__ import annotations import argparse import hashlib import json from pathlib import Path from common import read_json from prepare_1c_rag_corpus import SUPPORTED_EXTENSIONS, classify_source, normalize_text, parse_front_matter ROOT = Path(__file__).resolve().parents[1] DEFAULT_SOURCE_DIR = ROOT / "plugins" / "1c" / "rag" / "sources" DEFAULT_MANIFEST = ROOT / "plugins" / "1c" / "datasets" / "prepared" / "rag_manifest.json" def iter_source_files(source_dir: Path) -> list[Path]: if not source_dir.exists(): return [] return sorted( path for path in source_dir.rglob("*") if path.is_file() and path.suffix.lower() in SUPPORTED_EXTENSIONS ) def source_state(source_dir: Path) -> dict[str, dict]: state = {} for path in iter_source_files(source_dir): relative_path = path.relative_to(source_dir).as_posix() text = normalize_text(path.read_text(encoding="utf-8")) front_matter, body = parse_front_matter(text) chunk_source_text = body or text state[relative_path] = { "source_path": relative_path, "source_type": classify_source(path, chunk_source_text, front_matter), "file_type": path.suffix.lower().lstrip("."), "content_hash": hashlib.sha256(text.encode("utf-8")).hexdigest(), } return state def compare_manifest(source_dir: Path, manifest_path: Path) -> dict: current = source_state(source_dir) if not manifest_path.exists(): return { "status": "stale" if current else "missing", "reason": "manifest is missing", "new": sorted(current), "changed": [], "deleted": [], "type_changed": [], } manifest = read_json(manifest_path) recorded = { str(source.get("source_path")): source for source in manifest.get("sources") or [] if isinstance(source, dict) and source.get("source_path") } current_paths = set(current) recorded_paths = set(recorded) new = sorted(current_paths - recorded_paths) deleted = sorted(recorded_paths - current_paths) changed = [] type_changed = [] for source_path in sorted(current_paths & recorded_paths): current_source = current[source_path] recorded_source = recorded[source_path] if current_source["content_hash"] != recorded_source.get("content_hash"): changed.append(source_path) if current_source["source_type"] != recorded_source.get("source_type"): type_changed.append( { "source_path": source_path, "current": current_source["source_type"], "manifest": recorded_source.get("source_type"), } ) stale = bool(new or deleted or changed or type_changed) return { "status": "stale" if stale else "fresh", "source_dir": str(source_dir), "manifest": str(manifest_path), "source_count": len(current), "manifest_source_count": len(recorded), "new": new, "changed": changed, "deleted": deleted, "type_changed": type_changed, } def main() -> int: parser = argparse.ArgumentParser(description="Check whether the 1C RAG manifest is fresh.") parser.add_argument("--source-dir", type=Path, default=DEFAULT_SOURCE_DIR) parser.add_argument("--manifest", type=Path, default=DEFAULT_MANIFEST) parser.add_argument("--print", action="store_true") args = parser.parse_args() report = compare_manifest(args.source_dir, args.manifest) if args.print: print(json.dumps(report, ensure_ascii=False, indent=2)) else: print(f"1C RAG freshness: {report['status']}") return 0 if report["status"] in {"fresh", "missing"} else 1 if __name__ == "__main__": raise SystemExit(main())