from __future__ import annotations

import argparse
import sys
from pathlib import Path

import yaml


ROOT = Path(__file__).resolve().parents[1]
DEFAULT_EVAL_DIRS = [ROOT / "evals", ROOT / "plugins"]
CHECK_TYPES = {"contains", "not_contains", "regex", "max_sentences", "refuses", "requires_metadata"}


def iter_eval_files(paths: list[Path]) -> list[Path]:
    files: list[Path] = []
    for path in paths:
        if path.is_file() and path.suffix.lower() in {".yaml", ".yml"}:
            files.append(path)
        elif path.is_dir():
            files.extend(
                candidate
                for candidate in path.rglob("*.yaml")
                if "evals" in candidate.parts
            )
            files.extend(
                candidate
                for candidate in path.rglob("*.yml")
                if "evals" in candidate.parts
            )
    return sorted(set(files))


def validate_eval(path: Path) -> list[str]:
    errors: list[str] = []
    with path.open("r", encoding="utf-8") as handle:
        data = yaml.safe_load(handle)

    if not isinstance(data, dict):
        return [f"{path}: eval file must be a YAML mapping"]

    for field in ("id", "name", "type", "cases"):
        if field not in data:
            errors.append(f"{path}: missing required field `{field}`")

    cases = data.get("cases")
    if not isinstance(cases, list) or not cases:
        errors.append(f"{path}: cases must be a non-empty list")
        return errors

    for index, case in enumerate(cases):
        prefix = f"{path}: cases[{index}]"
        if not isinstance(case, dict):
            errors.append(f"{prefix}: case must be a mapping")
            continue
        if not case.get("id"):
            errors.append(f"{prefix}: id is required")
        messages = case.get("messages")
        if not isinstance(messages, list) or not messages:
            errors.append(f"{prefix}: messages must be a non-empty list")
        else:
            for msg_index, message in enumerate(messages):
                msg_prefix = f"{prefix}.messages[{msg_index}]"
                if not isinstance(message, dict):
                    errors.append(f"{msg_prefix}: message must be a mapping")
                    continue
                if message.get("role") not in {"system", "user", "assistant", "tool"}:
                    errors.append(f"{msg_prefix}: invalid role `{message.get('role')}`")
                content = message.get("content")
                if not isinstance(content, str) or not content.strip():
                    errors.append(f"{msg_prefix}: content is required")
        checks = case.get("checks")
        if not isinstance(checks, list) or not checks:
            errors.append(f"{prefix}: checks must be a non-empty list")
        else:
            for check_index, check in enumerate(checks):
                check_prefix = f"{prefix}.checks[{check_index}]"
                if isinstance(check, str):
                    if not check.strip():
                        errors.append(f"{check_prefix}: check text is required")
                    continue
                if not isinstance(check, dict):
                    errors.append(f"{check_prefix}: check must be a string or mapping")
                    continue
                check_type = check.get("type")
                if check_type not in CHECK_TYPES:
                    errors.append(f"{check_prefix}: invalid check type `{check_type}`")
                if check_type in {"contains", "not_contains", "regex"}:
                    value = check.get("value")
                    if not isinstance(value, str) or not value.strip():
                        errors.append(f"{check_prefix}: value is required for {check_type}")
                if check_type == "max_sentences":
                    value = check.get("value")
                    if not isinstance(value, int) or value < 1:
                        errors.append(f"{check_prefix}: value must be a positive integer")

    return errors


def main() -> int:
    parser = argparse.ArgumentParser(description="Validate eval YAML files.")
    parser.add_argument("paths", nargs="*", type=Path)
    args = parser.parse_args()

    files = iter_eval_files(args.paths or DEFAULT_EVAL_DIRS)
    if not files:
        print("No eval files found.")
        return 0

    errors: list[str] = []
    for path in files:
        try:
            errors.extend(validate_eval(path))
        except Exception as exc:
            errors.append(f"{path}: failed to parse: {exc}")

    if errors:
        print("Eval validation failed:", file=sys.stderr)
        for error in errors:
            print(f"- {error}", file=sys.stderr)
        return 1

    print(f"Validated {len(files)} eval file(s).")
    return 0


if __name__ == "__main__":
    raise SystemExit(main())