109 lines
3.9 KiB
Python
109 lines
3.9 KiB
Python
from __future__ import annotations
|
|
|
|
import argparse
|
|
import json
|
|
from collections import defaultdict
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
DEFAULT_REPORT_DIR = ROOT / "reports" / "observability"
|
|
|
|
|
|
def iter_turn_audit(report_dir: Path) -> list[dict[str, Any]]:
|
|
records: list[dict[str, Any]] = []
|
|
if not report_dir.exists():
|
|
return records
|
|
for path in sorted(report_dir.rglob("turn_audit/*.jsonl")):
|
|
with path.open("r", encoding="utf-8") as handle:
|
|
for line_number, line in enumerate(handle, start=1):
|
|
line = line.strip()
|
|
if not line:
|
|
continue
|
|
try:
|
|
record = json.loads(line)
|
|
except json.JSONDecodeError as exc:
|
|
raise ValueError(f"{path}:{line_number}: invalid JSONL: {exc}") from exc
|
|
if isinstance(record, dict):
|
|
records.append(record)
|
|
return records
|
|
|
|
|
|
def normalize_prompt(text: str) -> str:
|
|
return " ".join(str(text or "").split())
|
|
|
|
|
|
def group_repeated_failures(records: list[dict[str, Any]], *, min_count: int = 2, limit: int = 50) -> list[dict[str, Any]]:
|
|
groups: dict[tuple[str, str], dict[str, Any]] = defaultdict(
|
|
lambda: {
|
|
"count": 0,
|
|
"failure_type": "",
|
|
"normalized_user_text": "",
|
|
"examples": [],
|
|
"_timestamps": [],
|
|
}
|
|
)
|
|
|
|
for record in records:
|
|
if str(record.get("outcome") or "") == "success":
|
|
continue
|
|
failure_type = str(record.get("failure_type") or "unknown")
|
|
normalized_user_text = normalize_prompt(str(record.get("user_text") or ""))
|
|
if not normalized_user_text:
|
|
continue
|
|
key = (failure_type, normalized_user_text)
|
|
item = groups[key]
|
|
item["count"] += 1
|
|
item["failure_type"] = failure_type
|
|
item["normalized_user_text"] = normalized_user_text
|
|
item["_timestamps"].append(str(record.get("timestamp") or ""))
|
|
if len(item["examples"]) < 3:
|
|
item["examples"].append(
|
|
{
|
|
"timestamp": record.get("timestamp"),
|
|
"turn_id": record.get("turn_id"),
|
|
"error_code": record.get("error_code"),
|
|
"error_message": record.get("error_message"),
|
|
"request_id": record.get("request_id"),
|
|
}
|
|
)
|
|
|
|
rows = []
|
|
for item in groups.values():
|
|
if item["count"] < min_count:
|
|
continue
|
|
timestamps = sorted(ts for ts in item.pop("_timestamps") if ts)
|
|
item["first_seen"] = timestamps[0] if timestamps else ""
|
|
item["last_seen"] = timestamps[-1] if timestamps else ""
|
|
rows.append(item)
|
|
|
|
rows.sort(key=lambda row: (-int(row["count"]), str(row["failure_type"]), str(row["normalized_user_text"])))
|
|
return rows[:limit]
|
|
|
|
|
|
def write_report(path: Path, payload: dict[str, Any]) -> None:
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
path.write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
|
|
|
|
|
def main() -> int:
|
|
parser = argparse.ArgumentParser(description="Report repeated failed turns grouped by failure type and normalized prompt.")
|
|
parser.add_argument("--report-dir", type=Path, default=DEFAULT_REPORT_DIR)
|
|
parser.add_argument("--min-count", type=int, default=2)
|
|
parser.add_argument("--limit", type=int, default=50)
|
|
parser.add_argument("--output", type=Path, help="Optional JSON output path.")
|
|
args = parser.parse_args()
|
|
|
|
records = iter_turn_audit(args.report_dir)
|
|
groups = group_repeated_failures(records, min_count=args.min_count, limit=args.limit)
|
|
payload = {"records": len(records), "repeated_failures": groups}
|
|
print(json.dumps(payload, ensure_ascii=False, indent=2))
|
|
if args.output:
|
|
write_report(args.output, payload)
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|