189 lines
7.6 KiB
Python
189 lines
7.6 KiB
Python
from __future__ import annotations
|
|
|
|
import argparse
|
|
import json
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
DEFAULT_NORMALIZED_MANIFEST = ROOT / "plugins" / "1c" / "rag" / "official-docs" / "normalized" / "manifest.json"
|
|
DEFAULT_RAW_MANIFEST = ROOT / "plugins" / "1c" / "rag" / "official-docs" / "raw" / "manifest.json"
|
|
DEFAULT_RAG_SOURCE_DIR = ROOT / "plugins" / "1c" / "rag" / "sources" / "official" / "its"
|
|
DEFAULT_CORPUS = ROOT / "plugins" / "1c" / "datasets" / "prepared" / "rag_corpus.jsonl"
|
|
|
|
FORBIDDEN_TEXT = (
|
|
"Мы используем файлы cookie",
|
|
"Продолжая находиться на сайте",
|
|
"Результаты поиска",
|
|
"Купить кассу",
|
|
"Календарь бухгалтера",
|
|
"Последние результаты поиска",
|
|
)
|
|
|
|
NAVIGATION_CLUES = (
|
|
"Руководство разработчика - Руководство администратора",
|
|
"Глоссарий разработчика - 1 - 1CEClientSetupMake.exe",
|
|
)
|
|
|
|
|
|
def load_json(path: Path) -> dict[str, Any]:
|
|
if not path.exists():
|
|
return {}
|
|
try:
|
|
data = json.loads(path.read_text(encoding="utf-8-sig"))
|
|
except json.JSONDecodeError:
|
|
return {}
|
|
return data if isinstance(data, dict) else {}
|
|
|
|
|
|
def scan_text_file(path: Path, needles: tuple[str, ...]) -> list[dict[str, Any]]:
|
|
try:
|
|
text = path.read_text(encoding="utf-8-sig", errors="ignore")
|
|
except OSError as exc:
|
|
return [{"severity": "error", "code": "read_failed", "path": str(path), "message": str(exc)}]
|
|
findings = []
|
|
for needle in needles:
|
|
if needle in text:
|
|
findings.append({"severity": "error", "code": "forbidden_text", "path": str(path), "text": needle})
|
|
return findings
|
|
|
|
|
|
def scan_jsonl(path: Path, needles: tuple[str, ...]) -> list[dict[str, Any]]:
|
|
findings: list[dict[str, Any]] = []
|
|
if not path.exists():
|
|
findings.append({"severity": "warning", "code": "missing_corpus", "path": str(path)})
|
|
return findings
|
|
for line_no, line in enumerate(path.read_text(encoding="utf-8-sig", errors="ignore").splitlines(), start=1):
|
|
if not line.strip():
|
|
continue
|
|
try:
|
|
item = json.loads(line)
|
|
except json.JSONDecodeError:
|
|
findings.append({"severity": "error", "code": "bad_jsonl", "path": str(path), "line": line_no})
|
|
continue
|
|
content = str(item.get("content") or "")
|
|
for needle in needles:
|
|
if needle in content:
|
|
findings.append(
|
|
{
|
|
"severity": "error",
|
|
"code": "forbidden_text_in_corpus",
|
|
"path": str(path),
|
|
"line": line_no,
|
|
"chunk_id": item.get("id"),
|
|
"text": needle,
|
|
}
|
|
)
|
|
return findings
|
|
|
|
|
|
def raw_url_counts(raw_manifest_path: Path) -> dict[str, int]:
|
|
manifest = load_json(raw_manifest_path)
|
|
counts = {"raw_pages": 0, "content_src_pages": 0, "hdoc_pages": 0, "root_pages": 0}
|
|
for page in manifest.get("pages") or []:
|
|
counts["raw_pages"] += 1
|
|
url = str(page.get("url") or "")
|
|
if "/db/content/" in url and "/src/" in url:
|
|
counts["content_src_pages"] += 1
|
|
elif "/content/" in url and url.endswith("/hdoc"):
|
|
counts["hdoc_pages"] += 1
|
|
else:
|
|
counts["root_pages"] += 1
|
|
return counts
|
|
|
|
|
|
def check_quality(manifest_path: Path, raw_manifest_path: Path, rag_source_dir: Path, corpus_path: Path) -> dict[str, Any]:
|
|
findings: list[dict[str, Any]] = []
|
|
manifest = load_json(manifest_path)
|
|
raw_counts = raw_url_counts(raw_manifest_path)
|
|
page_count = int(manifest.get("page_count") or 0)
|
|
skipped_count = int(manifest.get("skipped_count") or 0)
|
|
discovered_src_count = int(manifest.get("discovered_src_record_count") or 0)
|
|
media_page_count = 0
|
|
media_image_count = 0
|
|
table_count = 0
|
|
for page in manifest.get("pages") or []:
|
|
media = page.get("media") or {}
|
|
images = media.get("images") or []
|
|
if images:
|
|
media_page_count += 1
|
|
media_image_count += len(images)
|
|
table_count += int(media.get("table_count") or 0)
|
|
|
|
if not manifest:
|
|
findings.append({"severity": "warning", "code": "missing_normalized_manifest", "path": str(manifest_path)})
|
|
elif page_count == 0:
|
|
findings.append(
|
|
{
|
|
"severity": "warning",
|
|
"code": "no_official_content_pages",
|
|
"message": "No official 1C:ITS pages passed normalization quality gates. Refresh cookie and fetch with --no-resume.",
|
|
"skipped_count": skipped_count,
|
|
}
|
|
)
|
|
if raw_counts["raw_pages"] and not raw_counts["content_src_pages"] and not discovered_src_count:
|
|
findings.append(
|
|
{
|
|
"severity": "warning",
|
|
"code": "no_raw_content_src_pages",
|
|
"message": "Raw fetch has pages, but no /db/content/.../src/... pages. Fetch probably stopped at hdoc shells/navigation.",
|
|
"raw_counts": raw_counts,
|
|
}
|
|
)
|
|
|
|
if rag_source_dir.exists():
|
|
for path in sorted(rag_source_dir.glob("*.md")):
|
|
findings.extend(scan_text_file(path, FORBIDDEN_TEXT + NAVIGATION_CLUES))
|
|
else:
|
|
findings.append({"severity": "warning", "code": "missing_rag_source_dir", "path": str(rag_source_dir)})
|
|
|
|
findings.extend(scan_jsonl(corpus_path, FORBIDDEN_TEXT + NAVIGATION_CLUES))
|
|
|
|
errors = [item for item in findings if item.get("severity") == "error"]
|
|
warnings = [item for item in findings if item.get("severity") == "warning"]
|
|
return {
|
|
"schema": "onec_official_docs_quality_check.v1",
|
|
"passed": not errors,
|
|
"counts": {
|
|
**raw_counts,
|
|
"discovered_src_record_count": discovered_src_count,
|
|
"media_pages": media_page_count,
|
|
"media_images": media_image_count,
|
|
"tables": table_count,
|
|
"normalized_pages": page_count,
|
|
"skipped_pages": skipped_count,
|
|
"errors": len(errors),
|
|
"warnings": len(warnings),
|
|
"findings": len(findings),
|
|
},
|
|
"manifest": str(manifest_path),
|
|
"raw_manifest": str(raw_manifest_path),
|
|
"rag_source_dir": str(rag_source_dir),
|
|
"corpus": str(corpus_path),
|
|
"findings": findings,
|
|
}
|
|
|
|
|
|
def main() -> int:
|
|
parser = argparse.ArgumentParser(description="Check official 1C:ITS normalized docs and RAG corpus quality.")
|
|
parser.add_argument("--manifest", type=Path, default=DEFAULT_NORMALIZED_MANIFEST)
|
|
parser.add_argument("--raw-manifest", type=Path, default=DEFAULT_RAW_MANIFEST)
|
|
parser.add_argument("--rag-source-dir", type=Path, default=DEFAULT_RAG_SOURCE_DIR)
|
|
parser.add_argument("--corpus", type=Path, default=DEFAULT_CORPUS)
|
|
parser.add_argument("--output", type=Path)
|
|
parser.add_argument("--print", action="store_true", dest="print_full")
|
|
args = parser.parse_args()
|
|
|
|
result = check_quality(args.manifest, args.raw_manifest, args.rag_source_dir, args.corpus)
|
|
if args.output:
|
|
args.output.parent.mkdir(parents=True, exist_ok=True)
|
|
args.output.write_text(json.dumps(result, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
|
payload = result if args.print_full else {"passed": result["passed"], "counts": result["counts"], "output": str(args.output) if args.output else None}
|
|
print(json.dumps(payload, ensure_ascii=False, indent=2))
|
|
return 0 if result["passed"] else 2
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|