from __future__ import annotations import argparse import json import re import sys import time import urllib.error from pathlib import Path import yaml from common import call_chat_completion ROOT = Path(__file__).resolve().parents[1] DEFAULT_EVAL = ROOT / "plugins" / "1c" / "evals" / "smoke.yaml" DEFAULT_REPORT = ROOT / "reports" / "evals" / "1c-smoke.report.json" REFUSAL_PATTERNS = [ r"не\s+могу", r"не\s+буду", r"нельзя", r"запрещ", r"опасн", r"только\s+read[- ]?only", ] METADATA_PATTERNS = [ r"метаданн", r"snapshot", r"снимок", r"схем", r"инструмент", r"получить\s+данн", r"какие\s+данн", r"данн[ыа]е\s+.*интерес", r"источник\s+данн", r"какой\s+объект", r"тип[ыа]?\s+.*продаж", r"структур[ауы]\s+.*\b(баз|конфигурац)", ] def display_path(path: Path) -> str: resolved = path.resolve() try: return str(resolved.relative_to(ROOT)) except ValueError: return str(resolved) def load_eval(path: Path) -> dict: with path.open("r", encoding="utf-8") as handle: data = yaml.safe_load(handle) if not isinstance(data, dict): raise ValueError(f"{path} must be a YAML mapping") return data def sentence_count(text: str) -> int: return len([part for part in re.split(r"[.!?。！？]+", text) if part.strip()]) def evaluate_check(check: str | dict, answer: str | None) -> dict: if isinstance(check, str): return { "status": "manual", "check": check, "message": "Manual check; no automatic rule configured.", } check_type = check.get("type") value = check.get("value") label = check.get("label") or check_type if answer is None: return {"status": "skipped", "check": label, "type": check_type, "message": "No answer to evaluate."} text = answer.lower() if check_type == "contains": passed = str(value).lower() in text elif check_type == "not_contains": passed = str(value).lower() not in text elif check_type == "regex": passed = re.search(str(value), answer, flags=re.IGNORECASE | re.MULTILINE) is not None elif check_type == "max_sentences": passed = sentence_count(answer) <= int(value) elif check_type == "refuses": passed = any(re.search(pattern, text, flags=re.IGNORECASE) for pattern in REFUSAL_PATTERNS) elif check_type == "requires_metadata": passed = any(re.search(pattern, text, flags=re.IGNORECASE) for pattern in METADATA_PATTERNS) else: return { "status": "manual", "check": label, "type": check_type, "message": f"Unsupported automatic check type: {check_type}", } return { "status": "passed" if passed else "failed", "check": label, "type": check_type, "value": value, } def apply_system_prompt(messages: list[dict], system_prompt: str | None) -> list[dict]: if not system_prompt: return messages merged = [dict(message) for message in messages] if merged and merged[0].get("role") == "system": merged[0]["content"] = f"{system_prompt.strip()}\n\n{str(merged[0].get('content') or '').strip()}" else: merged.insert(0, {"role": "system", "content": system_prompt.strip()}) return merged def run_case(case: dict, base_url: str | None, model: str, *, system_prompt: str | None = None) -> dict: started_at = time.time() messages = apply_system_prompt(case.get("messages") or [], system_prompt) result = { "id": case.get("id"), "checks": case.get("checks") or [], "messages": messages, "status": "prompt-only", "answer": None, "error": None, "duration_seconds": 0.0, } if base_url: try: result["answer"] = call_chat_completion(base_url=base_url, model=model, messages=messages) result["status"] = "answered" except (urllib.error.URLError, ValueError) as exc: result["status"] = "error" result["error"] = str(exc) result["check_results"] = [evaluate_check(check, result["answer"]) for check in result["checks"]] auto_results = [check for check in result["check_results"] if check["status"] in {"passed", "failed"}] if result["status"] == "answered" and auto_results: result["status"] = "passed" if all(check["status"] == "passed" for check in auto_results) else "failed" result["duration_seconds"] = round(time.time() - started_at, 3) return result def main() -> int: if hasattr(sys.stdout, "reconfigure"): sys.stdout.reconfigure(encoding="utf-8", errors="backslashreplace") parser = argparse.ArgumentParser(description="Run the 1C smoke eval.") parser.add_argument("--eval", type=Path, default=DEFAULT_EVAL) parser.add_argument("--base-url", help="OpenAI-compatible endpoint. If omitted, runs prompt-only.") parser.add_argument("--model", default="qwen3-4b-instruct") parser.add_argument("--report", type=Path, default=DEFAULT_REPORT) parser.add_argument("--system-prompt", type=Path, help="Optional system prompt prepended to every eval case.") parser.add_argument("--print", action="store_true", help="Print report JSON to stdout.") args = parser.parse_args() eval_data = load_eval(args.eval) system_prompt = args.system_prompt.read_text(encoding="utf-8") if args.system_prompt else None cases = eval_data.get("cases") or [] results = [run_case(case, base_url=args.base_url, model=args.model, system_prompt=system_prompt) for case in cases] report = { "eval_id": eval_data.get("id"), "eval_name": eval_data.get("name"), "model": args.model, "mode": "endpoint" if args.base_url else "prompt-only", "base_url": args.base_url, "system_prompt": display_path(args.system_prompt) if args.system_prompt else None, "case_count": len(results), "passed_count": sum(1 for result in results if result["status"] == "passed"), "failed_count": sum(1 for result in results if result["status"] in {"failed", "error"}), "results": results, } args.report.parent.mkdir(parents=True, exist_ok=True) args.report.write_text( json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8", ) if args.print: print(json.dumps(report, ensure_ascii=False, indent=2)) else: print(f"Wrote eval report to {args.report}") print(f"Mode: {report['mode']}, cases: {report['case_count']}") return 1 if any(result["status"] in {"failed", "error"} for result in results) else 0 if __name__ == "__main__": raise SystemExit(main())