129 lines
4.7 KiB
Python
129 lines
4.7 KiB
Python
from __future__ import annotations
|
|
|
|
import argparse
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
import yaml
|
|
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
DEFAULT_EVAL_DIRS = [ROOT / "evals", ROOT / "plugins"]
|
|
CHECK_TYPES = {"contains", "not_contains", "regex", "max_sentences", "refuses", "requires_metadata"}
|
|
|
|
|
|
def iter_eval_files(paths: list[Path]) -> list[Path]:
|
|
files: list[Path] = []
|
|
for path in paths:
|
|
if path.is_file() and path.suffix.lower() in {".yaml", ".yml"}:
|
|
files.append(path)
|
|
elif path.is_dir():
|
|
files.extend(
|
|
candidate
|
|
for candidate in path.rglob("*.yaml")
|
|
if "evals" in candidate.parts
|
|
)
|
|
files.extend(
|
|
candidate
|
|
for candidate in path.rglob("*.yml")
|
|
if "evals" in candidate.parts
|
|
)
|
|
return sorted(set(files))
|
|
|
|
|
|
def validate_eval(path: Path) -> list[str]:
|
|
errors: list[str] = []
|
|
with path.open("r", encoding="utf-8") as handle:
|
|
data = yaml.safe_load(handle)
|
|
|
|
if not isinstance(data, dict):
|
|
return [f"{path}: eval file must be a YAML mapping"]
|
|
|
|
for field in ("id", "name", "type", "cases"):
|
|
if field not in data:
|
|
errors.append(f"{path}: missing required field `{field}`")
|
|
|
|
cases = data.get("cases")
|
|
if not isinstance(cases, list) or not cases:
|
|
errors.append(f"{path}: cases must be a non-empty list")
|
|
return errors
|
|
|
|
for index, case in enumerate(cases):
|
|
prefix = f"{path}: cases[{index}]"
|
|
if not isinstance(case, dict):
|
|
errors.append(f"{prefix}: case must be a mapping")
|
|
continue
|
|
if not case.get("id"):
|
|
errors.append(f"{prefix}: id is required")
|
|
messages = case.get("messages")
|
|
if not isinstance(messages, list) or not messages:
|
|
errors.append(f"{prefix}: messages must be a non-empty list")
|
|
else:
|
|
for msg_index, message in enumerate(messages):
|
|
msg_prefix = f"{prefix}.messages[{msg_index}]"
|
|
if not isinstance(message, dict):
|
|
errors.append(f"{msg_prefix}: message must be a mapping")
|
|
continue
|
|
if message.get("role") not in {"system", "user", "assistant", "tool"}:
|
|
errors.append(f"{msg_prefix}: invalid role `{message.get('role')}`")
|
|
content = message.get("content")
|
|
if not isinstance(content, str) or not content.strip():
|
|
errors.append(f"{msg_prefix}: content is required")
|
|
checks = case.get("checks")
|
|
if not isinstance(checks, list) or not checks:
|
|
errors.append(f"{prefix}: checks must be a non-empty list")
|
|
else:
|
|
for check_index, check in enumerate(checks):
|
|
check_prefix = f"{prefix}.checks[{check_index}]"
|
|
if isinstance(check, str):
|
|
if not check.strip():
|
|
errors.append(f"{check_prefix}: check text is required")
|
|
continue
|
|
if not isinstance(check, dict):
|
|
errors.append(f"{check_prefix}: check must be a string or mapping")
|
|
continue
|
|
check_type = check.get("type")
|
|
if check_type not in CHECK_TYPES:
|
|
errors.append(f"{check_prefix}: invalid check type `{check_type}`")
|
|
if check_type in {"contains", "not_contains", "regex"}:
|
|
value = check.get("value")
|
|
if not isinstance(value, str) or not value.strip():
|
|
errors.append(f"{check_prefix}: value is required for {check_type}")
|
|
if check_type == "max_sentences":
|
|
value = check.get("value")
|
|
if not isinstance(value, int) or value < 1:
|
|
errors.append(f"{check_prefix}: value must be a positive integer")
|
|
|
|
return errors
|
|
|
|
|
|
def main() -> int:
|
|
parser = argparse.ArgumentParser(description="Validate eval YAML files.")
|
|
parser.add_argument("paths", nargs="*", type=Path)
|
|
args = parser.parse_args()
|
|
|
|
files = iter_eval_files(args.paths or DEFAULT_EVAL_DIRS)
|
|
if not files:
|
|
print("No eval files found.")
|
|
return 0
|
|
|
|
errors: list[str] = []
|
|
for path in files:
|
|
try:
|
|
errors.extend(validate_eval(path))
|
|
except Exception as exc:
|
|
errors.append(f"{path}: failed to parse: {exc}")
|
|
|
|
if errors:
|
|
print("Eval validation failed:", file=sys.stderr)
|
|
for error in errors:
|
|
print(f"- {error}", file=sys.stderr)
|
|
return 1
|
|
|
|
print(f"Validated {len(files)} eval file(s).")
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|