from __future__ import annotations import argparse import json from collections import defaultdict from pathlib import Path from typing import Any ROOT = Path(__file__).resolve().parents[1] DEFAULT_REPORT_DIR = ROOT / "reports" / "observability" def iter_turn_audit(report_dir: Path) -> list[dict[str, Any]]: records: list[dict[str, Any]] = [] if not report_dir.exists(): return records for path in sorted(report_dir.rglob("turn_audit/*.jsonl")): with path.open("r", encoding="utf-8") as handle: for line_number, line in enumerate(handle, start=1): line = line.strip() if not line: continue try: record = json.loads(line) except json.JSONDecodeError as exc: raise ValueError(f"{path}:{line_number}: invalid JSONL: {exc}") from exc if isinstance(record, dict): records.append(record) return records def normalize_prompt(text: str) -> str: return " ".join(str(text or "").split()) def group_repeated_failures(records: list[dict[str, Any]], *, min_count: int = 2, limit: int = 50) -> list[dict[str, Any]]: groups: dict[tuple[str, str], dict[str, Any]] = defaultdict( lambda: { "count": 0, "failure_type": "", "normalized_user_text": "", "examples": [], "_timestamps": [], } ) for record in records: if str(record.get("outcome") or "") == "success": continue failure_type = str(record.get("failure_type") or "unknown") normalized_user_text = normalize_prompt(str(record.get("user_text") or "")) if not normalized_user_text: continue key = (failure_type, normalized_user_text) item = groups[key] item["count"] += 1 item["failure_type"] = failure_type item["normalized_user_text"] = normalized_user_text item["_timestamps"].append(str(record.get("timestamp") or "")) if len(item["examples"]) < 3: item["examples"].append( { "timestamp": record.get("timestamp"), "turn_id": record.get("turn_id"), "error_code": record.get("error_code"), "error_message": record.get("error_message"), "request_id": record.get("request_id"), } ) rows = [] for item in groups.values(): if item["count"] < min_count: continue timestamps = sorted(ts for ts in item.pop("_timestamps") if ts) item["first_seen"] = timestamps[0] if timestamps else "" item["last_seen"] = timestamps[-1] if timestamps else "" rows.append(item) rows.sort(key=lambda row: (-int(row["count"]), str(row["failure_type"]), str(row["normalized_user_text"]))) return rows[:limit] def write_report(path: Path, payload: dict[str, Any]) -> None: path.parent.mkdir(parents=True, exist_ok=True) path.write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") def main() -> int: parser = argparse.ArgumentParser(description="Report repeated failed turns grouped by failure type and normalized prompt.") parser.add_argument("--report-dir", type=Path, default=DEFAULT_REPORT_DIR) parser.add_argument("--min-count", type=int, default=2) parser.add_argument("--limit", type=int, default=50) parser.add_argument("--output", type=Path, help="Optional JSON output path.") args = parser.parse_args() records = iter_turn_audit(args.report_dir) groups = group_repeated_failures(records, min_count=args.min_count, limit=args.limit) payload = {"records": len(records), "repeated_failures": groups} print(json.dumps(payload, ensure_ascii=False, indent=2)) if args.output: write_report(args.output, payload) return 0 if __name__ == "__main__": raise SystemExit(main())