120 lines
5.4 KiB
Python
120 lines
5.4 KiB
Python
from __future__ import annotations
|
|
|
|
import argparse
|
|
import json
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0, str(ROOT / "scripts"))
|
|
|
|
from build_1c_agent_intake import build_intake # noqa: E402
|
|
|
|
|
|
DEFAULT_INDEX = ROOT / "reports" / "1c-sql" / "upo" / "unified-object-route-index.json"
|
|
|
|
|
|
CASES = [
|
|
{
|
|
"id": "docs_intake_uses_official_scope",
|
|
"question": "Как работает событие ПриОткрытии формы?",
|
|
"expected_route": "docs_rag",
|
|
"expected_code_allowed": True,
|
|
"expected_safe_scope": "official_1c_docs",
|
|
},
|
|
{
|
|
"id": "example_fact_requires_confirmation",
|
|
"question": "В примере RAG есть реквизит Артикул у справочника Номенклатура. Напиши код для текущей базы.",
|
|
"expected_route": "mixed_docs_and_current_config",
|
|
"expected_examples_are_facts": False,
|
|
"expected_confirmed_path": "Справочник.Номенклатура.Артикул",
|
|
"expected_code_allowed": True,
|
|
},
|
|
{
|
|
"id": "missing_fact_blocks_code",
|
|
"question": "Напиши код для текущей базы: заполни Справочник.Номенклатура.ВыдуманныйРеквизит.",
|
|
"expected_route": "current_config_fact",
|
|
"expected_unresolved_path": "Справочник.Номенклатура.ВыдуманныйРеквизит",
|
|
"expected_code_allowed": False,
|
|
},
|
|
]
|
|
|
|
|
|
def run_case(case: dict, *, index: Path, view: str) -> dict:
|
|
intake = build_intake(case["question"], index=index, view=view)
|
|
failures = []
|
|
route = (intake.get("route") or {}).get("decision") or {}
|
|
policy = intake.get("answer_policy") or {}
|
|
source_policy = intake.get("source_policy") or {}
|
|
facts = intake.get("facts") or {}
|
|
|
|
if route.get("route") != case.get("expected_route"):
|
|
failures.append({"code": "route_mismatch", "expected": case.get("expected_route"), "actual": route.get("route")})
|
|
if policy.get("code_generation_allowed") is not case.get("expected_code_allowed"):
|
|
failures.append({"code": "code_policy_mismatch", "expected": case.get("expected_code_allowed"), "actual": policy.get("code_generation_allowed")})
|
|
if case.get("expected_safe_scope") and policy.get("safe_rag_scope") != case.get("expected_safe_scope"):
|
|
failures.append({"code": "safe_scope_mismatch", "expected": case.get("expected_safe_scope"), "actual": policy.get("safe_rag_scope")})
|
|
if "expected_examples_are_facts" in case and source_policy.get("examples_are_current_facts") is not case["expected_examples_are_facts"]:
|
|
failures.append({"code": "examples_policy_mismatch", "expected": case["expected_examples_are_facts"], "actual": source_policy.get("examples_are_current_facts")})
|
|
|
|
confirmed_paths = {row.get("path") for row in facts.get("confirmed") or []}
|
|
unresolved_paths = {row.get("path") for row in facts.get("unresolved") or []}
|
|
if case.get("expected_confirmed_path") and case["expected_confirmed_path"] not in confirmed_paths:
|
|
failures.append({"code": "confirmed_path_missing", "expected": case["expected_confirmed_path"], "actual": sorted(confirmed_paths)})
|
|
if case.get("expected_unresolved_path") and case["expected_unresolved_path"] not in unresolved_paths:
|
|
failures.append({"code": "unresolved_path_missing", "expected": case["expected_unresolved_path"], "actual": sorted(unresolved_paths)})
|
|
|
|
return {
|
|
"id": case["id"],
|
|
"status": "passed" if not failures else "failed",
|
|
"question": case["question"],
|
|
"failures": failures,
|
|
"summary": {
|
|
"route": route.get("route"),
|
|
"code_generation_allowed": policy.get("code_generation_allowed"),
|
|
"safe_rag_scope": policy.get("safe_rag_scope"),
|
|
"confirmed_paths": sorted(confirmed_paths),
|
|
"unresolved_paths": sorted(unresolved_paths),
|
|
},
|
|
}
|
|
|
|
|
|
def run_check(index: Path, *, view: str) -> dict:
|
|
if not index.exists():
|
|
return {
|
|
"schema": "onec_agent_intake_check.v1",
|
|
"status": "failed",
|
|
"error": f"route index not found: {index}",
|
|
"cases": [],
|
|
}
|
|
cases = [run_case(case, index=index, view=view) for case in CASES]
|
|
return {
|
|
"schema": "onec_agent_intake_check.v1",
|
|
"status": "ok" if all(case["status"] == "passed" for case in cases) else "failed",
|
|
"index": str(index),
|
|
"view": view,
|
|
"case_count": len(cases),
|
|
"cases": cases,
|
|
}
|
|
|
|
|
|
def main() -> int:
|
|
parser = argparse.ArgumentParser(description="Check 1C agent intake behavior.")
|
|
parser.add_argument("--index", type=Path, default=DEFAULT_INDEX)
|
|
parser.add_argument("--view", choices=["effective", "base"], default="effective")
|
|
parser.add_argument("--output", type=Path)
|
|
parser.add_argument("--print", action="store_true")
|
|
args = parser.parse_args()
|
|
|
|
report = run_check(args.index, view=args.view)
|
|
if args.output:
|
|
args.output.parent.mkdir(parents=True, exist_ok=True)
|
|
args.output.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
|
if args.print or not args.output:
|
|
print(json.dumps(report, ensure_ascii=False, indent=2))
|
|
return 0 if report["status"] == "ok" else 1
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|