from __future__ import annotations import argparse import datetime as dt import hashlib import json from pathlib import Path from typing import Any ROOT = Path(__file__).resolve().parents[1] DEFAULT_OUTPUT = ROOT / "reports" / "llm-artifact-manifest.json" DEFAULT_ARTIFACTS = { "models": ROOT / "models", "datasets_raw": ROOT / "datasets" / "raw", "datasets_prepared": ROOT / "datasets" / "prepared", "1c_rag_sources": ROOT / "plugins" / "1c" / "rag" / "sources", "1c_rag_official_raw": ROOT / "plugins" / "1c" / "rag" / "official-docs" / "raw", "1c_rag_official_normalized": ROOT / "plugins" / "1c" / "rag" / "official-docs" / "normalized", "1c_rag_official_media": ROOT / "plugins" / "1c" / "rag" / "official-docs" / "media", "1c_rag_official_static": ROOT / "plugins" / "1c" / "rag" / "official-docs" / "static", "1c_rag_official_start_links": ROOT / "plugins" / "1c" / "rag" / "official-docs" / "start-links.json", "1c_rag_official_platform_versions": ROOT / "plugins" / "1c" / "rag" / "official-docs" / "platform-versions.json", "1c_datasets_raw": ROOT / "plugins" / "1c" / "datasets" / "raw", "1c_datasets_prepared": ROOT / "plugins" / "1c" / "datasets" / "prepared", "1c_metadata_snapshots": ROOT / "plugins" / "1c" / "metadata" / "snapshots", "1c_training_raw": ROOT / "plugins" / "1c" / "training" / "raw", "1c_training_prepared": ROOT / "plugins" / "1c" / "training" / "prepared", } SECRET_NAME_MARKERS = ("cookie", "secret", "password", "passwd", "credential", ".env", ".pem", ".pfx", ".p12", ".key") KNOWN_SAFE_TOKEN_FILES = ("tokenizer", "special_tokens", "added_tokens") def sha256_file(path: Path) -> str: digest = hashlib.sha256() with path.open("rb") as handle: for chunk in iter(lambda: handle.read(1024 * 1024), b""): digest.update(chunk) return digest.hexdigest() def is_secret_like(path: Path) -> bool: lowered = path.name.lower() if any(marker in lowered for marker in KNOWN_SAFE_TOKEN_FILES): return False return any(marker in lowered for marker in SECRET_NAME_MARKERS) def iter_files(path: Path) -> list[Path]: if not path.exists(): return [] return sorted(item for item in path.rglob("*") if item.is_file()) def summarize_artifact(name: str, path: Path, *, hash_files: bool, max_files: int) -> dict[str, Any]: files = iter_files(path) total_size = sum(item.stat().st_size for item in files) suffix_counts: dict[str, int] = {} secret_like = [] file_records = [] for item in files: suffix = item.suffix.lower() or "" suffix_counts[suffix] = suffix_counts.get(suffix, 0) + 1 if is_secret_like(item): secret_like.append(str(item.relative_to(path))) if len(file_records) < max_files: stat = item.stat() record = { "relative_path": str(item.relative_to(path)).replace("\\", "/"), "size_bytes": stat.st_size, "mtime_unix": int(stat.st_mtime), } if hash_files: record["sha256"] = sha256_file(item) file_records.append(record) return { "name": name, "path": str(path), "exists": path.exists(), "is_dir": path.is_dir(), "file_count": len(files), "total_size_bytes": total_size, "suffix_counts": dict(sorted(suffix_counts.items())), "secret_like_files": secret_like[:50], "secret_like_truncated": len(secret_like) > 50, "files_sampled": len(file_records), "files_truncated": len(files) > max_files, "files": file_records, } def build_manifest(*, hash_files: bool, max_files: int, artifacts: dict[str, Path]) -> dict[str, Any]: records = [summarize_artifact(name, path, hash_files=hash_files, max_files=max_files) for name, path in artifacts.items()] return { "schema": "llm_artifact_manifest.v1", "created_at": dt.datetime.now(dt.UTC).isoformat(), "workspace_root": str(ROOT), "hash_files": hash_files, "max_files_per_artifact": max_files, "portable_policy": { "store_large_artifacts_outside_git": True, "mount_artifacts_as_docker_volumes": True, "do_not_store_secrets_in_artifact_dirs": True, }, "docker_volume_recommendations": [ {"host": "models", "container": "/app/models", "required_for": ["model-inference", "training"]}, {"host": "plugins/1c/datasets", "container": "/app/plugins/1c/datasets", "required_for": ["1c-rag-api", "1c-adapter-api"]}, {"host": "plugins/1c/rag/sources", "container": "/app/plugins/1c/rag/sources", "required_for": ["1c-rag-api"]}, {"host": "plugins/1c/rag/official-docs", "container": "/app/plugins/1c/rag/official-docs", "required_for": ["1c-official-docs-ingest"]}, {"host": "plugins/1c/metadata/snapshots", "container": "/app/plugins/1c/metadata/snapshots", "required_for": ["1c-adapter-api"]}, ], "counts": { "artifacts": len(records), "existing": sum(1 for item in records if item["exists"]), "files": sum(int(item["file_count"]) for item in records), "total_size_bytes": sum(int(item["total_size_bytes"]) for item in records), "secret_like_files": sum(len(item["secret_like_files"]) for item in records), }, "artifacts": records, } def main() -> int: parser = argparse.ArgumentParser(description="Build an inventory manifest for local LLM/RAG artifacts that are intentionally outside git.") parser.add_argument("--output", type=Path, default=DEFAULT_OUTPUT) parser.add_argument("--hash-files", action="store_true", help="Hash sampled files. This can be slow for large model files.") parser.add_argument("--max-files", type=int, default=500, help="Maximum file records stored per artifact.") args = parser.parse_args() manifest = build_manifest(hash_files=args.hash_files, max_files=args.max_files, artifacts=DEFAULT_ARTIFACTS) args.output.parent.mkdir(parents=True, exist_ok=True) args.output.write_text(json.dumps(manifest, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") print( json.dumps( { "output": str(args.output), "artifacts": manifest["counts"]["artifacts"], "files": manifest["counts"]["files"], "total_size_bytes": manifest["counts"]["total_size_bytes"], "secret_like_files": manifest["counts"]["secret_like_files"], }, ensure_ascii=False, ) ) return 0 if __name__ == "__main__": raise SystemExit(main())