152 lines
6.6 KiB
Python
152 lines
6.6 KiB
Python
from __future__ import annotations
|
|
|
|
import argparse
|
|
import datetime as dt
|
|
import hashlib
|
|
import json
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
DEFAULT_OUTPUT = ROOT / "reports" / "llm-artifact-manifest.json"
|
|
DEFAULT_ARTIFACTS = {
|
|
"models": ROOT / "models",
|
|
"datasets_raw": ROOT / "datasets" / "raw",
|
|
"datasets_prepared": ROOT / "datasets" / "prepared",
|
|
"1c_rag_sources": ROOT / "plugins" / "1c" / "rag" / "sources",
|
|
"1c_rag_official_raw": ROOT / "plugins" / "1c" / "rag" / "official-docs" / "raw",
|
|
"1c_rag_official_normalized": ROOT / "plugins" / "1c" / "rag" / "official-docs" / "normalized",
|
|
"1c_rag_official_media": ROOT / "plugins" / "1c" / "rag" / "official-docs" / "media",
|
|
"1c_rag_official_static": ROOT / "plugins" / "1c" / "rag" / "official-docs" / "static",
|
|
"1c_rag_official_start_links": ROOT / "plugins" / "1c" / "rag" / "official-docs" / "start-links.json",
|
|
"1c_rag_official_platform_versions": ROOT / "plugins" / "1c" / "rag" / "official-docs" / "platform-versions.json",
|
|
"1c_datasets_raw": ROOT / "plugins" / "1c" / "datasets" / "raw",
|
|
"1c_datasets_prepared": ROOT / "plugins" / "1c" / "datasets" / "prepared",
|
|
"1c_metadata_snapshots": ROOT / "plugins" / "1c" / "metadata" / "snapshots",
|
|
"1c_training_raw": ROOT / "plugins" / "1c" / "training" / "raw",
|
|
"1c_training_prepared": ROOT / "plugins" / "1c" / "training" / "prepared",
|
|
}
|
|
SECRET_NAME_MARKERS = ("cookie", "secret", "password", "passwd", "credential", ".env", ".pem", ".pfx", ".p12", ".key")
|
|
KNOWN_SAFE_TOKEN_FILES = ("tokenizer", "special_tokens", "added_tokens")
|
|
|
|
|
|
def sha256_file(path: Path) -> str:
|
|
digest = hashlib.sha256()
|
|
with path.open("rb") as handle:
|
|
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
|
|
digest.update(chunk)
|
|
return digest.hexdigest()
|
|
|
|
|
|
def is_secret_like(path: Path) -> bool:
|
|
lowered = path.name.lower()
|
|
if any(marker in lowered for marker in KNOWN_SAFE_TOKEN_FILES):
|
|
return False
|
|
return any(marker in lowered for marker in SECRET_NAME_MARKERS)
|
|
|
|
|
|
def iter_files(path: Path) -> list[Path]:
|
|
if not path.exists():
|
|
return []
|
|
return sorted(item for item in path.rglob("*") if item.is_file())
|
|
|
|
|
|
def summarize_artifact(name: str, path: Path, *, hash_files: bool, max_files: int) -> dict[str, Any]:
|
|
files = iter_files(path)
|
|
total_size = sum(item.stat().st_size for item in files)
|
|
suffix_counts: dict[str, int] = {}
|
|
secret_like = []
|
|
file_records = []
|
|
|
|
for item in files:
|
|
suffix = item.suffix.lower() or "<no_ext>"
|
|
suffix_counts[suffix] = suffix_counts.get(suffix, 0) + 1
|
|
if is_secret_like(item):
|
|
secret_like.append(str(item.relative_to(path)))
|
|
if len(file_records) < max_files:
|
|
stat = item.stat()
|
|
record = {
|
|
"relative_path": str(item.relative_to(path)).replace("\\", "/"),
|
|
"size_bytes": stat.st_size,
|
|
"mtime_unix": int(stat.st_mtime),
|
|
}
|
|
if hash_files:
|
|
record["sha256"] = sha256_file(item)
|
|
file_records.append(record)
|
|
|
|
return {
|
|
"name": name,
|
|
"path": str(path),
|
|
"exists": path.exists(),
|
|
"is_dir": path.is_dir(),
|
|
"file_count": len(files),
|
|
"total_size_bytes": total_size,
|
|
"suffix_counts": dict(sorted(suffix_counts.items())),
|
|
"secret_like_files": secret_like[:50],
|
|
"secret_like_truncated": len(secret_like) > 50,
|
|
"files_sampled": len(file_records),
|
|
"files_truncated": len(files) > max_files,
|
|
"files": file_records,
|
|
}
|
|
|
|
|
|
def build_manifest(*, hash_files: bool, max_files: int, artifacts: dict[str, Path]) -> dict[str, Any]:
|
|
records = [summarize_artifact(name, path, hash_files=hash_files, max_files=max_files) for name, path in artifacts.items()]
|
|
return {
|
|
"schema": "llm_artifact_manifest.v1",
|
|
"created_at": dt.datetime.now(dt.UTC).isoformat(),
|
|
"workspace_root": str(ROOT),
|
|
"hash_files": hash_files,
|
|
"max_files_per_artifact": max_files,
|
|
"portable_policy": {
|
|
"store_large_artifacts_outside_git": True,
|
|
"mount_artifacts_as_docker_volumes": True,
|
|
"do_not_store_secrets_in_artifact_dirs": True,
|
|
},
|
|
"docker_volume_recommendations": [
|
|
{"host": "models", "container": "/app/models", "required_for": ["model-inference", "training"]},
|
|
{"host": "plugins/1c/datasets", "container": "/app/plugins/1c/datasets", "required_for": ["1c-rag-api", "1c-adapter-api"]},
|
|
{"host": "plugins/1c/rag/sources", "container": "/app/plugins/1c/rag/sources", "required_for": ["1c-rag-api"]},
|
|
{"host": "plugins/1c/rag/official-docs", "container": "/app/plugins/1c/rag/official-docs", "required_for": ["1c-official-docs-ingest"]},
|
|
{"host": "plugins/1c/metadata/snapshots", "container": "/app/plugins/1c/metadata/snapshots", "required_for": ["1c-adapter-api"]},
|
|
],
|
|
"counts": {
|
|
"artifacts": len(records),
|
|
"existing": sum(1 for item in records if item["exists"]),
|
|
"files": sum(int(item["file_count"]) for item in records),
|
|
"total_size_bytes": sum(int(item["total_size_bytes"]) for item in records),
|
|
"secret_like_files": sum(len(item["secret_like_files"]) for item in records),
|
|
},
|
|
"artifacts": records,
|
|
}
|
|
|
|
|
|
def main() -> int:
|
|
parser = argparse.ArgumentParser(description="Build an inventory manifest for local LLM/RAG artifacts that are intentionally outside git.")
|
|
parser.add_argument("--output", type=Path, default=DEFAULT_OUTPUT)
|
|
parser.add_argument("--hash-files", action="store_true", help="Hash sampled files. This can be slow for large model files.")
|
|
parser.add_argument("--max-files", type=int, default=500, help="Maximum file records stored per artifact.")
|
|
args = parser.parse_args()
|
|
|
|
manifest = build_manifest(hash_files=args.hash_files, max_files=args.max_files, artifacts=DEFAULT_ARTIFACTS)
|
|
args.output.parent.mkdir(parents=True, exist_ok=True)
|
|
args.output.write_text(json.dumps(manifest, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
|
print(
|
|
json.dumps(
|
|
{
|
|
"output": str(args.output),
|
|
"artifacts": manifest["counts"]["artifacts"],
|
|
"files": manifest["counts"]["files"],
|
|
"total_size_bytes": manifest["counts"]["total_size_bytes"],
|
|
"secret_like_files": manifest["counts"]["secret_like_files"],
|
|
},
|
|
ensure_ascii=False,
|
|
)
|
|
)
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|