Initial SQL-only 1C adapter baseline

2026-07-22 03:03:47 +03:00
commit e2503b77e7
545 changed files with 184711 additions and 0 deletions
@@ -0,0 +1,108 @@
+from __future__ import annotations
+
+import argparse
+import json
+from collections import defaultdict
+from pathlib import Path
+from typing import Any
+
+
+ROOT = Path(__file__).resolve().parents[1]
+DEFAULT_REPORT_DIR = ROOT / "reports" / "observability"
+
+
+def iter_turn_audit(report_dir: Path) -> list[dict[str, Any]]:
+    records: list[dict[str, Any]] = []
+    if not report_dir.exists():
+        return records
+    for path in sorted(report_dir.rglob("turn_audit/*.jsonl")):
+        with path.open("r", encoding="utf-8") as handle:
+            for line_number, line in enumerate(handle, start=1):
+                line = line.strip()
+                if not line:
+                    continue
+                try:
+                    record = json.loads(line)
+                except json.JSONDecodeError as exc:
+                    raise ValueError(f"{path}:{line_number}: invalid JSONL: {exc}") from exc
+                if isinstance(record, dict):
+                    records.append(record)
+    return records
+
+
+def normalize_prompt(text: str) -> str:
+    return " ".join(str(text or "").split())
+
+
+def group_repeated_failures(records: list[dict[str, Any]], *, min_count: int = 2, limit: int = 50) -> list[dict[str, Any]]:
+    groups: dict[tuple[str, str], dict[str, Any]] = defaultdict(
+        lambda: {
+            "count": 0,
+            "failure_type": "",
+            "normalized_user_text": "",
+            "examples": [],
+            "_timestamps": [],
+        }
+    )
+
+    for record in records:
+        if str(record.get("outcome") or "") == "success":
+            continue
+        failure_type = str(record.get("failure_type") or "unknown")
+        normalized_user_text = normalize_prompt(str(record.get("user_text") or ""))
+        if not normalized_user_text:
+            continue
+        key = (failure_type, normalized_user_text)
+        item = groups[key]
+        item["count"] += 1
+        item["failure_type"] = failure_type
+        item["normalized_user_text"] = normalized_user_text
+        item["_timestamps"].append(str(record.get("timestamp") or ""))
+        if len(item["examples"]) < 3:
+            item["examples"].append(
+                {
+                    "timestamp": record.get("timestamp"),
+                    "turn_id": record.get("turn_id"),
+                    "error_code": record.get("error_code"),
+                    "error_message": record.get("error_message"),
+                    "request_id": record.get("request_id"),
+                }
+            )
+
+    rows = []
+    for item in groups.values():
+        if item["count"] < min_count:
+            continue
+        timestamps = sorted(ts for ts in item.pop("_timestamps") if ts)
+        item["first_seen"] = timestamps[0] if timestamps else ""
+        item["last_seen"] = timestamps[-1] if timestamps else ""
+        rows.append(item)
+
+    rows.sort(key=lambda row: (-int(row["count"]), str(row["failure_type"]), str(row["normalized_user_text"])))
+    return rows[:limit]
+
+
+def write_report(path: Path, payload: dict[str, Any]) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description="Report repeated failed turns grouped by failure type and normalized prompt.")
+    parser.add_argument("--report-dir", type=Path, default=DEFAULT_REPORT_DIR)
+    parser.add_argument("--min-count", type=int, default=2)
+    parser.add_argument("--limit", type=int, default=50)
+    parser.add_argument("--output", type=Path, help="Optional JSON output path.")
+    args = parser.parse_args()
+
+    records = iter_turn_audit(args.report_dir)
+    groups = group_repeated_failures(records, min_count=args.min_count, limit=args.limit)
+    payload = {"records": len(records), "repeated_failures": groups}
+    print(json.dumps(payload, ensure_ascii=False, indent=2))
+    if args.output:
+        write_report(args.output, payload)
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())