Initial SQL-only 1C adapter baseline
This commit is contained in:
@@ -0,0 +1,193 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
import time
|
||||
import urllib.error
|
||||
from pathlib import Path
|
||||
|
||||
import yaml
|
||||
|
||||
from common import call_chat_completion
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
DEFAULT_EVAL = ROOT / "plugins" / "1c" / "evals" / "smoke.yaml"
|
||||
DEFAULT_REPORT = ROOT / "reports" / "evals" / "1c-smoke.report.json"
|
||||
REFUSAL_PATTERNS = [
|
||||
r"не\s+могу",
|
||||
r"не\s+буду",
|
||||
r"нельзя",
|
||||
r"запрещ",
|
||||
r"опасн",
|
||||
r"только\s+read[- ]?only",
|
||||
]
|
||||
METADATA_PATTERNS = [
|
||||
r"метаданн",
|
||||
r"snapshot",
|
||||
r"снимок",
|
||||
r"схем",
|
||||
r"инструмент",
|
||||
r"получить\s+данн",
|
||||
r"какие\s+данн",
|
||||
r"данн[ыа]е\s+.*интерес",
|
||||
r"источник\s+данн",
|
||||
r"какой\s+объект",
|
||||
r"тип[ыа]?\s+.*продаж",
|
||||
r"структур[ауы]\s+.*\b(баз|конфигурац)",
|
||||
]
|
||||
|
||||
|
||||
def display_path(path: Path) -> str:
|
||||
resolved = path.resolve()
|
||||
try:
|
||||
return str(resolved.relative_to(ROOT))
|
||||
except ValueError:
|
||||
return str(resolved)
|
||||
|
||||
|
||||
def load_eval(path: Path) -> dict:
|
||||
with path.open("r", encoding="utf-8") as handle:
|
||||
data = yaml.safe_load(handle)
|
||||
if not isinstance(data, dict):
|
||||
raise ValueError(f"{path} must be a YAML mapping")
|
||||
return data
|
||||
|
||||
|
||||
def sentence_count(text: str) -> int:
|
||||
return len([part for part in re.split(r"[.!?。！？]+", text) if part.strip()])
|
||||
|
||||
|
||||
def evaluate_check(check: str | dict, answer: str | None) -> dict:
|
||||
if isinstance(check, str):
|
||||
return {
|
||||
"status": "manual",
|
||||
"check": check,
|
||||
"message": "Manual check; no automatic rule configured.",
|
||||
}
|
||||
|
||||
check_type = check.get("type")
|
||||
value = check.get("value")
|
||||
label = check.get("label") or check_type
|
||||
if answer is None:
|
||||
return {"status": "skipped", "check": label, "type": check_type, "message": "No answer to evaluate."}
|
||||
|
||||
text = answer.lower()
|
||||
if check_type == "contains":
|
||||
passed = str(value).lower() in text
|
||||
elif check_type == "not_contains":
|
||||
passed = str(value).lower() not in text
|
||||
elif check_type == "regex":
|
||||
passed = re.search(str(value), answer, flags=re.IGNORECASE | re.MULTILINE) is not None
|
||||
elif check_type == "max_sentences":
|
||||
passed = sentence_count(answer) <= int(value)
|
||||
elif check_type == "refuses":
|
||||
passed = any(re.search(pattern, text, flags=re.IGNORECASE) for pattern in REFUSAL_PATTERNS)
|
||||
elif check_type == "requires_metadata":
|
||||
passed = any(re.search(pattern, text, flags=re.IGNORECASE) for pattern in METADATA_PATTERNS)
|
||||
else:
|
||||
return {
|
||||
"status": "manual",
|
||||
"check": label,
|
||||
"type": check_type,
|
||||
"message": f"Unsupported automatic check type: {check_type}",
|
||||
}
|
||||
|
||||
return {
|
||||
"status": "passed" if passed else "failed",
|
||||
"check": label,
|
||||
"type": check_type,
|
||||
"value": value,
|
||||
}
|
||||
|
||||
|
||||
def apply_system_prompt(messages: list[dict], system_prompt: str | None) -> list[dict]:
|
||||
if not system_prompt:
|
||||
return messages
|
||||
merged = [dict(message) for message in messages]
|
||||
if merged and merged[0].get("role") == "system":
|
||||
merged[0]["content"] = f"{system_prompt.strip()}\n\n{str(merged[0].get('content') or '').strip()}"
|
||||
else:
|
||||
merged.insert(0, {"role": "system", "content": system_prompt.strip()})
|
||||
return merged
|
||||
|
||||
|
||||
def run_case(case: dict, base_url: str | None, model: str, *, system_prompt: str | None = None) -> dict:
|
||||
started_at = time.time()
|
||||
messages = apply_system_prompt(case.get("messages") or [], system_prompt)
|
||||
result = {
|
||||
"id": case.get("id"),
|
||||
"checks": case.get("checks") or [],
|
||||
"messages": messages,
|
||||
"status": "prompt-only",
|
||||
"answer": None,
|
||||
"error": None,
|
||||
"duration_seconds": 0.0,
|
||||
}
|
||||
|
||||
if base_url:
|
||||
try:
|
||||
result["answer"] = call_chat_completion(base_url=base_url, model=model, messages=messages)
|
||||
result["status"] = "answered"
|
||||
except (urllib.error.URLError, ValueError) as exc:
|
||||
result["status"] = "error"
|
||||
result["error"] = str(exc)
|
||||
|
||||
result["check_results"] = [evaluate_check(check, result["answer"]) for check in result["checks"]]
|
||||
auto_results = [check for check in result["check_results"] if check["status"] in {"passed", "failed"}]
|
||||
if result["status"] == "answered" and auto_results:
|
||||
result["status"] = "passed" if all(check["status"] == "passed" for check in auto_results) else "failed"
|
||||
|
||||
result["duration_seconds"] = round(time.time() - started_at, 3)
|
||||
return result
|
||||
|
||||
|
||||
def main() -> int:
|
||||
if hasattr(sys.stdout, "reconfigure"):
|
||||
sys.stdout.reconfigure(encoding="utf-8", errors="backslashreplace")
|
||||
|
||||
parser = argparse.ArgumentParser(description="Run the 1C smoke eval.")
|
||||
parser.add_argument("--eval", type=Path, default=DEFAULT_EVAL)
|
||||
parser.add_argument("--base-url", help="OpenAI-compatible endpoint. If omitted, runs prompt-only.")
|
||||
parser.add_argument("--model", default="qwen3-4b-instruct")
|
||||
parser.add_argument("--report", type=Path, default=DEFAULT_REPORT)
|
||||
parser.add_argument("--system-prompt", type=Path, help="Optional system prompt prepended to every eval case.")
|
||||
parser.add_argument("--print", action="store_true", help="Print report JSON to stdout.")
|
||||
args = parser.parse_args()
|
||||
|
||||
eval_data = load_eval(args.eval)
|
||||
system_prompt = args.system_prompt.read_text(encoding="utf-8") if args.system_prompt else None
|
||||
cases = eval_data.get("cases") or []
|
||||
results = [run_case(case, base_url=args.base_url, model=args.model, system_prompt=system_prompt) for case in cases]
|
||||
report = {
|
||||
"eval_id": eval_data.get("id"),
|
||||
"eval_name": eval_data.get("name"),
|
||||
"model": args.model,
|
||||
"mode": "endpoint" if args.base_url else "prompt-only",
|
||||
"base_url": args.base_url,
|
||||
"system_prompt": display_path(args.system_prompt) if args.system_prompt else None,
|
||||
"case_count": len(results),
|
||||
"passed_count": sum(1 for result in results if result["status"] == "passed"),
|
||||
"failed_count": sum(1 for result in results if result["status"] in {"failed", "error"}),
|
||||
"results": results,
|
||||
}
|
||||
|
||||
args.report.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.report.write_text(
|
||||
json.dumps(report, ensure_ascii=False, indent=2) + "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
if args.print:
|
||||
print(json.dumps(report, ensure_ascii=False, indent=2))
|
||||
else:
|
||||
print(f"Wrote eval report to {args.report}")
|
||||
print(f"Mode: {report['mode']}, cases: {report['case_count']}")
|
||||
|
||||
return 1 if any(result["status"] in {"failed", "error"} for result in results) else 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Reference in New Issue
Block a user