from __future__ import annotations import argparse import json import sys from pathlib import Path ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT / "scripts")) from build_1c_agent_intake import build_intake # noqa: E402 DEFAULT_INDEX = ROOT / "reports" / "1c-sql" / "upo" / "unified-object-route-index.json" CASES = [ { "id": "docs_intake_uses_official_scope", "question": "Как работает событие ПриОткрытии формы?", "expected_route": "docs_rag", "expected_code_allowed": True, "expected_safe_scope": "official_1c_docs", }, { "id": "example_fact_requires_confirmation", "question": "В примере RAG есть реквизит Артикул у справочника Номенклатура. Напиши код для текущей базы.", "expected_route": "mixed_docs_and_current_config", "expected_examples_are_facts": False, "expected_confirmed_path": "Справочник.Номенклатура.Артикул", "expected_code_allowed": True, }, { "id": "missing_fact_blocks_code", "question": "Напиши код для текущей базы: заполни Справочник.Номенклатура.ВыдуманныйРеквизит.", "expected_route": "current_config_fact", "expected_unresolved_path": "Справочник.Номенклатура.ВыдуманныйРеквизит", "expected_code_allowed": False, }, ] def run_case(case: dict, *, index: Path, view: str) -> dict: intake = build_intake(case["question"], index=index, view=view) failures = [] route = (intake.get("route") or {}).get("decision") or {} policy = intake.get("answer_policy") or {} source_policy = intake.get("source_policy") or {} facts = intake.get("facts") or {} if route.get("route") != case.get("expected_route"): failures.append({"code": "route_mismatch", "expected": case.get("expected_route"), "actual": route.get("route")}) if policy.get("code_generation_allowed") is not case.get("expected_code_allowed"): failures.append({"code": "code_policy_mismatch", "expected": case.get("expected_code_allowed"), "actual": policy.get("code_generation_allowed")}) if case.get("expected_safe_scope") and policy.get("safe_rag_scope") != case.get("expected_safe_scope"): failures.append({"code": "safe_scope_mismatch", "expected": case.get("expected_safe_scope"), "actual": policy.get("safe_rag_scope")}) if "expected_examples_are_facts" in case and source_policy.get("examples_are_current_facts") is not case["expected_examples_are_facts"]: failures.append({"code": "examples_policy_mismatch", "expected": case["expected_examples_are_facts"], "actual": source_policy.get("examples_are_current_facts")}) confirmed_paths = {row.get("path") for row in facts.get("confirmed") or []} unresolved_paths = {row.get("path") for row in facts.get("unresolved") or []} if case.get("expected_confirmed_path") and case["expected_confirmed_path"] not in confirmed_paths: failures.append({"code": "confirmed_path_missing", "expected": case["expected_confirmed_path"], "actual": sorted(confirmed_paths)}) if case.get("expected_unresolved_path") and case["expected_unresolved_path"] not in unresolved_paths: failures.append({"code": "unresolved_path_missing", "expected": case["expected_unresolved_path"], "actual": sorted(unresolved_paths)}) return { "id": case["id"], "status": "passed" if not failures else "failed", "question": case["question"], "failures": failures, "summary": { "route": route.get("route"), "code_generation_allowed": policy.get("code_generation_allowed"), "safe_rag_scope": policy.get("safe_rag_scope"), "confirmed_paths": sorted(confirmed_paths), "unresolved_paths": sorted(unresolved_paths), }, } def run_check(index: Path, *, view: str) -> dict: if not index.exists(): return { "schema": "onec_agent_intake_check.v1", "status": "failed", "error": f"route index not found: {index}", "cases": [], } cases = [run_case(case, index=index, view=view) for case in CASES] return { "schema": "onec_agent_intake_check.v1", "status": "ok" if all(case["status"] == "passed" for case in cases) else "failed", "index": str(index), "view": view, "case_count": len(cases), "cases": cases, } def main() -> int: parser = argparse.ArgumentParser(description="Check 1C agent intake behavior.") parser.add_argument("--index", type=Path, default=DEFAULT_INDEX) parser.add_argument("--view", choices=["effective", "base"], default="effective") parser.add_argument("--output", type=Path) parser.add_argument("--print", action="store_true") args = parser.parse_args() report = run_check(args.index, view=args.view) if args.output: args.output.parent.mkdir(parents=True, exist_ok=True) args.output.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") if args.print or not args.output: print(json.dumps(report, ensure_ascii=False, indent=2)) return 0 if report["status"] == "ok" else 1 if __name__ == "__main__": raise SystemExit(main())