Initial SQL-only 1C adapter baseline

2026-07-22 03:03:47 +03:00
commit e2503b77e7
545 changed files with 184711 additions and 0 deletions
@@ -0,0 +1,108 @@
+from __future__ import annotations
+
+import argparse
+import json
+import re
+import sys
+from pathlib import Path
+
+from prepare_1c_rag_corpus import SUPPORTED_EXTENSIONS, classify_source, normalize_text
+
+
+ROOT = Path(__file__).resolve().parents[1]
+DEFAULT_SOURCE_DIR = ROOT / "plugins" / "1c" / "rag" / "sources"
+MAX_FILE_BYTES = 5 * 1024 * 1024
+
+SECRET_PATTERNS = {
+    "password assignment": re.compile(r"(?i)(password|passwd|pwd|пароль)\s*[:=]\s*[^;\s]+"),
+    "api token": re.compile(r"(?i)(api[_-]?key|token|secret|ключ)\s*[:=]\s*[^;\s]+"),
+    "connection string": re.compile(r"(?i)(server|host|database|uid|user id|pwd)\s*="),
+    "private key": re.compile(r"-----BEGIN [A-Z ]*PRIVATE KEY-----"),
+    "basic auth url": re.compile(r"(?i)https?://[^/\s:@]+:[^/\s:@]+@"),
+}
+
+
+def iter_files(source_dir: Path) -> list[Path]:
+    if not source_dir.exists():
+        return []
+    return sorted(path for path in source_dir.rglob("*") if path.is_file() and path.name != ".gitkeep")
+
+
+def scan_secrets(text: str) -> list[str]:
+    return [name for name, pattern in SECRET_PATTERNS.items() if pattern.search(text)]
+
+
+def validate_file(path: Path, source_dir: Path) -> tuple[dict, list[str]]:
+    errors: list[str] = []
+    relative_path = path.relative_to(source_dir).as_posix()
+    suffix = path.suffix.lower()
+    size = path.stat().st_size
+    if suffix not in SUPPORTED_EXTENSIONS:
+        errors.append(f"{relative_path}: unsupported extension `{suffix}`")
+    if size > MAX_FILE_BYTES:
+        errors.append(f"{relative_path}: file is too large for source control review: {size} bytes")
+
+    try:
+        text = normalize_text(path.read_text(encoding="utf-8"))
+    except UnicodeDecodeError as exc:
+        return (
+            {
+                "source_path": relative_path,
+                "status": "failed",
+                "source_type": None,
+                "size_bytes": size,
+            },
+            [f"{relative_path}: must be valid UTF-8: {exc}"],
+        )
+
+    if not text:
+        errors.append(f"{relative_path}: file is empty")
+    for match in scan_secrets(text):
+        errors.append(f"{relative_path}: possible secret detected: {match}")
+
+    source_type = classify_source(path, text)
+    return (
+        {
+            "source_path": relative_path,
+            "status": "failed" if errors else "ok",
+            "source_type": source_type,
+            "file_type": suffix.lstrip("."),
+            "size_bytes": size,
+        },
+        errors,
+    )
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description="Validate 1C RAG source files before indexing.")
+    parser.add_argument("--source-dir", type=Path, default=DEFAULT_SOURCE_DIR)
+    parser.add_argument("--print", action="store_true")
+    args = parser.parse_args()
+
+    files = iter_files(args.source_dir)
+    sources = []
+    errors: list[str] = []
+    for path in files:
+        source, file_errors = validate_file(path, args.source_dir)
+        sources.append(source)
+        errors.extend(file_errors)
+
+    report = {
+        "status": "failed" if errors else "ok",
+        "source_dir": str(args.source_dir),
+        "source_count": len(sources),
+        "sources": sources,
+        "errors": errors,
+    }
+    if args.print:
+        print(json.dumps(report, ensure_ascii=False, indent=2))
+    else:
+        print(f"1C RAG source validation: {report['status']}, sources: {len(sources)}")
+    if errors:
+        for error in errors:
+            print(f"- {error}", file=sys.stderr)
+    return 1 if errors else 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())