114 lines
3.8 KiB
Python
114 lines
3.8 KiB
Python
from __future__ import annotations
|
|
|
|
import argparse
|
|
import hashlib
|
|
import json
|
|
from pathlib import Path
|
|
|
|
from common import read_json
|
|
from prepare_1c_rag_corpus import SUPPORTED_EXTENSIONS, classify_source, normalize_text, parse_front_matter
|
|
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
DEFAULT_SOURCE_DIR = ROOT / "plugins" / "1c" / "rag" / "sources"
|
|
DEFAULT_MANIFEST = ROOT / "plugins" / "1c" / "datasets" / "prepared" / "rag_manifest.json"
|
|
|
|
|
|
def iter_source_files(source_dir: Path) -> list[Path]:
|
|
if not source_dir.exists():
|
|
return []
|
|
return sorted(
|
|
path
|
|
for path in source_dir.rglob("*")
|
|
if path.is_file() and path.suffix.lower() in SUPPORTED_EXTENSIONS
|
|
)
|
|
|
|
|
|
def source_state(source_dir: Path) -> dict[str, dict]:
|
|
state = {}
|
|
for path in iter_source_files(source_dir):
|
|
relative_path = path.relative_to(source_dir).as_posix()
|
|
text = normalize_text(path.read_text(encoding="utf-8"))
|
|
front_matter, body = parse_front_matter(text)
|
|
chunk_source_text = body or text
|
|
state[relative_path] = {
|
|
"source_path": relative_path,
|
|
"source_type": classify_source(path, chunk_source_text, front_matter),
|
|
"file_type": path.suffix.lower().lstrip("."),
|
|
"content_hash": hashlib.sha256(text.encode("utf-8")).hexdigest(),
|
|
}
|
|
return state
|
|
|
|
|
|
def compare_manifest(source_dir: Path, manifest_path: Path) -> dict:
|
|
current = source_state(source_dir)
|
|
if not manifest_path.exists():
|
|
return {
|
|
"status": "stale" if current else "missing",
|
|
"reason": "manifest is missing",
|
|
"new": sorted(current),
|
|
"changed": [],
|
|
"deleted": [],
|
|
"type_changed": [],
|
|
}
|
|
|
|
manifest = read_json(manifest_path)
|
|
recorded = {
|
|
str(source.get("source_path")): source
|
|
for source in manifest.get("sources") or []
|
|
if isinstance(source, dict) and source.get("source_path")
|
|
}
|
|
|
|
current_paths = set(current)
|
|
recorded_paths = set(recorded)
|
|
new = sorted(current_paths - recorded_paths)
|
|
deleted = sorted(recorded_paths - current_paths)
|
|
changed = []
|
|
type_changed = []
|
|
|
|
for source_path in sorted(current_paths & recorded_paths):
|
|
current_source = current[source_path]
|
|
recorded_source = recorded[source_path]
|
|
if current_source["content_hash"] != recorded_source.get("content_hash"):
|
|
changed.append(source_path)
|
|
if current_source["source_type"] != recorded_source.get("source_type"):
|
|
type_changed.append(
|
|
{
|
|
"source_path": source_path,
|
|
"current": current_source["source_type"],
|
|
"manifest": recorded_source.get("source_type"),
|
|
}
|
|
)
|
|
|
|
stale = bool(new or deleted or changed or type_changed)
|
|
return {
|
|
"status": "stale" if stale else "fresh",
|
|
"source_dir": str(source_dir),
|
|
"manifest": str(manifest_path),
|
|
"source_count": len(current),
|
|
"manifest_source_count": len(recorded),
|
|
"new": new,
|
|
"changed": changed,
|
|
"deleted": deleted,
|
|
"type_changed": type_changed,
|
|
}
|
|
|
|
|
|
def main() -> int:
|
|
parser = argparse.ArgumentParser(description="Check whether the 1C RAG manifest is fresh.")
|
|
parser.add_argument("--source-dir", type=Path, default=DEFAULT_SOURCE_DIR)
|
|
parser.add_argument("--manifest", type=Path, default=DEFAULT_MANIFEST)
|
|
parser.add_argument("--print", action="store_true")
|
|
args = parser.parse_args()
|
|
|
|
report = compare_manifest(args.source_dir, args.manifest)
|
|
if args.print:
|
|
print(json.dumps(report, ensure_ascii=False, indent=2))
|
|
else:
|
|
print(f"1C RAG freshness: {report['status']}")
|
|
return 0 if report["status"] in {"fresh", "missing"} else 1
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|