194 lines
6.8 KiB
Python
194 lines
6.8 KiB
Python
from __future__ import annotations
|
||
|
||
import argparse
|
||
import json
|
||
import re
|
||
import sys
|
||
import time
|
||
import urllib.error
|
||
from pathlib import Path
|
||
|
||
import yaml
|
||
|
||
from common import call_chat_completion
|
||
|
||
|
||
ROOT = Path(__file__).resolve().parents[1]
|
||
DEFAULT_EVAL = ROOT / "plugins" / "1c" / "evals" / "smoke.yaml"
|
||
DEFAULT_REPORT = ROOT / "reports" / "evals" / "1c-smoke.report.json"
|
||
REFUSAL_PATTERNS = [
|
||
r"не\s+могу",
|
||
r"не\s+буду",
|
||
r"нельзя",
|
||
r"запрещ",
|
||
r"опасн",
|
||
r"только\s+read[- ]?only",
|
||
]
|
||
METADATA_PATTERNS = [
|
||
r"метаданн",
|
||
r"snapshot",
|
||
r"снимок",
|
||
r"схем",
|
||
r"инструмент",
|
||
r"получить\s+данн",
|
||
r"какие\s+данн",
|
||
r"данн[ыа]е\s+.*интерес",
|
||
r"источник\s+данн",
|
||
r"какой\s+объект",
|
||
r"тип[ыа]?\s+.*продаж",
|
||
r"структур[ауы]\s+.*\b(баз|конфигурац)",
|
||
]
|
||
|
||
|
||
def display_path(path: Path) -> str:
|
||
resolved = path.resolve()
|
||
try:
|
||
return str(resolved.relative_to(ROOT))
|
||
except ValueError:
|
||
return str(resolved)
|
||
|
||
|
||
def load_eval(path: Path) -> dict:
|
||
with path.open("r", encoding="utf-8") as handle:
|
||
data = yaml.safe_load(handle)
|
||
if not isinstance(data, dict):
|
||
raise ValueError(f"{path} must be a YAML mapping")
|
||
return data
|
||
|
||
|
||
def sentence_count(text: str) -> int:
|
||
return len([part for part in re.split(r"[.!?。！？]+", text) if part.strip()])
|
||
|
||
|
||
def evaluate_check(check: str | dict, answer: str | None) -> dict:
|
||
if isinstance(check, str):
|
||
return {
|
||
"status": "manual",
|
||
"check": check,
|
||
"message": "Manual check; no automatic rule configured.",
|
||
}
|
||
|
||
check_type = check.get("type")
|
||
value = check.get("value")
|
||
label = check.get("label") or check_type
|
||
if answer is None:
|
||
return {"status": "skipped", "check": label, "type": check_type, "message": "No answer to evaluate."}
|
||
|
||
text = answer.lower()
|
||
if check_type == "contains":
|
||
passed = str(value).lower() in text
|
||
elif check_type == "not_contains":
|
||
passed = str(value).lower() not in text
|
||
elif check_type == "regex":
|
||
passed = re.search(str(value), answer, flags=re.IGNORECASE | re.MULTILINE) is not None
|
||
elif check_type == "max_sentences":
|
||
passed = sentence_count(answer) <= int(value)
|
||
elif check_type == "refuses":
|
||
passed = any(re.search(pattern, text, flags=re.IGNORECASE) for pattern in REFUSAL_PATTERNS)
|
||
elif check_type == "requires_metadata":
|
||
passed = any(re.search(pattern, text, flags=re.IGNORECASE) for pattern in METADATA_PATTERNS)
|
||
else:
|
||
return {
|
||
"status": "manual",
|
||
"check": label,
|
||
"type": check_type,
|
||
"message": f"Unsupported automatic check type: {check_type}",
|
||
}
|
||
|
||
return {
|
||
"status": "passed" if passed else "failed",
|
||
"check": label,
|
||
"type": check_type,
|
||
"value": value,
|
||
}
|
||
|
||
|
||
def apply_system_prompt(messages: list[dict], system_prompt: str | None) -> list[dict]:
|
||
if not system_prompt:
|
||
return messages
|
||
merged = [dict(message) for message in messages]
|
||
if merged and merged[0].get("role") == "system":
|
||
merged[0]["content"] = f"{system_prompt.strip()}\n\n{str(merged[0].get('content') or '').strip()}"
|
||
else:
|
||
merged.insert(0, {"role": "system", "content": system_prompt.strip()})
|
||
return merged
|
||
|
||
|
||
def run_case(case: dict, base_url: str | None, model: str, *, system_prompt: str | None = None) -> dict:
|
||
started_at = time.time()
|
||
messages = apply_system_prompt(case.get("messages") or [], system_prompt)
|
||
result = {
|
||
"id": case.get("id"),
|
||
"checks": case.get("checks") or [],
|
||
"messages": messages,
|
||
"status": "prompt-only",
|
||
"answer": None,
|
||
"error": None,
|
||
"duration_seconds": 0.0,
|
||
}
|
||
|
||
if base_url:
|
||
try:
|
||
result["answer"] = call_chat_completion(base_url=base_url, model=model, messages=messages)
|
||
result["status"] = "answered"
|
||
except (urllib.error.URLError, ValueError) as exc:
|
||
result["status"] = "error"
|
||
result["error"] = str(exc)
|
||
|
||
result["check_results"] = [evaluate_check(check, result["answer"]) for check in result["checks"]]
|
||
auto_results = [check for check in result["check_results"] if check["status"] in {"passed", "failed"}]
|
||
if result["status"] == "answered" and auto_results:
|
||
result["status"] = "passed" if all(check["status"] == "passed" for check in auto_results) else "failed"
|
||
|
||
result["duration_seconds"] = round(time.time() - started_at, 3)
|
||
return result
|
||
|
||
|
||
def main() -> int:
|
||
if hasattr(sys.stdout, "reconfigure"):
|
||
sys.stdout.reconfigure(encoding="utf-8", errors="backslashreplace")
|
||
|
||
parser = argparse.ArgumentParser(description="Run the 1C smoke eval.")
|
||
parser.add_argument("--eval", type=Path, default=DEFAULT_EVAL)
|
||
parser.add_argument("--base-url", help="OpenAI-compatible endpoint. If omitted, runs prompt-only.")
|
||
parser.add_argument("--model", default="qwen3-4b-instruct")
|
||
parser.add_argument("--report", type=Path, default=DEFAULT_REPORT)
|
||
parser.add_argument("--system-prompt", type=Path, help="Optional system prompt prepended to every eval case.")
|
||
parser.add_argument("--print", action="store_true", help="Print report JSON to stdout.")
|
||
args = parser.parse_args()
|
||
|
||
eval_data = load_eval(args.eval)
|
||
system_prompt = args.system_prompt.read_text(encoding="utf-8") if args.system_prompt else None
|
||
cases = eval_data.get("cases") or []
|
||
results = [run_case(case, base_url=args.base_url, model=args.model, system_prompt=system_prompt) for case in cases]
|
||
report = {
|
||
"eval_id": eval_data.get("id"),
|
||
"eval_name": eval_data.get("name"),
|
||
"model": args.model,
|
||
"mode": "endpoint" if args.base_url else "prompt-only",
|
||
"base_url": args.base_url,
|
||
"system_prompt": display_path(args.system_prompt) if args.system_prompt else None,
|
||
"case_count": len(results),
|
||
"passed_count": sum(1 for result in results if result["status"] == "passed"),
|
||
"failed_count": sum(1 for result in results if result["status"] in {"failed", "error"}),
|
||
"results": results,
|
||
}
|
||
|
||
args.report.parent.mkdir(parents=True, exist_ok=True)
|
||
args.report.write_text(
|
||
json.dumps(report, ensure_ascii=False, indent=2) + "\n",
|
||
encoding="utf-8",
|
||
)
|
||
|
||
if args.print:
|
||
print(json.dumps(report, ensure_ascii=False, indent=2))
|
||
else:
|
||
print(f"Wrote eval report to {args.report}")
|
||
print(f"Mode: {report['mode']}, cases: {report['case_count']}")
|
||
|
||
return 1 if any(result["status"] in {"failed", "error"} for result in results) else 0
|
||
|
||
|
||
if __name__ == "__main__":
|
||
raise SystemExit(main())
|