Initial SQL-only 1C adapter baseline

2026-07-22 03:03:47 +03:00
commit e2503b77e7
545 changed files with 184711 additions and 0 deletions
@@ -0,0 +1,119 @@
+from __future__ import annotations
+
+import argparse
+import csv
+import json
+from collections import defaultdict
+from pathlib import Path
+from typing import Any
+
+
+ROOT = Path(__file__).resolve().parents[1]
+DEFAULT_REPORT_DIR = ROOT / "reports" / "model-chat"
+
+
+def iter_records(report_dir: Path) -> list[dict[str, Any]]:
+    records: list[dict[str, Any]] = []
+    if not report_dir.exists():
+        return records
+    for path in sorted(report_dir.glob("*.jsonl")):
+        with path.open("r", encoding="utf-8") as handle:
+            for line_number, line in enumerate(handle, start=1):
+                line = line.strip()
+                if not line:
+                    continue
+                try:
+                    record = json.loads(line)
+                except json.JSONDecodeError as exc:
+                    raise ValueError(f"{path}:{line_number}: invalid JSONL: {exc}") from exc
+                if isinstance(record, dict):
+                    records.append(record)
+    return records
+
+
+def collect_rows(records: list[dict[str, Any]]) -> list[dict[str, Any]]:
+    rows = []
+    for record in records:
+        record_type = record.get("type")
+        if record_type == "chat":
+            rows.append(
+                {
+                    "type": "chat",
+                    "plugin": record.get("plugin"),
+                    "registry_model_id": record.get("registry_model_id"),
+                    "served_model_name": record.get("served_model_name"),
+                    "status": record.get("status"),
+                    "latency_ms": record.get("latency_ms"),
+                    "error": record.get("error"),
+                }
+            )
+        elif record_type == "compare":
+            for result in record.get("results") or []:
+                if not isinstance(result, dict):
+                    continue
+                rows.append(
+                    {
+                        "type": "compare",
+                        "plugin": record.get("plugin"),
+                        "registry_model_id": result.get("registry_model_id"),
+                        "served_model_name": result.get("served_model_name"),
+                        "status": result.get("status"),
+                        "latency_ms": result.get("latency_ms"),
+                        "error": result.get("error"),
+                    }
+                )
+    return rows
+
+
+def summarize(rows: list[dict[str, Any]]) -> list[dict[str, Any]]:
+    groups: dict[tuple[str, str], dict[str, Any]] = defaultdict(
+        lambda: {"requests": 0, "answered": 0, "errors": 0, "latencies": []}
+    )
+    for row in rows:
+        key = (str(row.get("plugin") or ""), str(row.get("served_model_name") or ""))
+        group = groups[key]
+        group["plugin"], group["served_model_name"] = key
+        group["registry_model_id"] = row.get("registry_model_id")
+        group["requests"] += 1
+        if row.get("status") == "answered":
+            group["answered"] += 1
+        if row.get("status") == "error":
+            group["errors"] += 1
+        latency = row.get("latency_ms")
+        if isinstance(latency, int | float):
+            group["latencies"].append(float(latency))
+
+    summary = []
+    for group in groups.values():
+        latencies = group.pop("latencies")
+        group["avg_latency_ms"] = round(sum(latencies) / len(latencies)) if latencies else None
+        summary.append(group)
+    return sorted(summary, key=lambda item: (item["plugin"], item["served_model_name"]))
+
+
+def write_csv(path: Path, rows: list[dict[str, Any]]) -> None:
+    fields = ["plugin", "served_model_name", "registry_model_id", "requests", "answered", "errors", "avg_latency_ms"]
+    path.parent.mkdir(parents=True, exist_ok=True)
+    with path.open("w", encoding="utf-8", newline="") as handle:
+        writer = csv.DictWriter(handle, fieldnames=fields)
+        writer.writeheader()
+        writer.writerows(rows)
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description="Summarize model-chat JSONL reports.")
+    parser.add_argument("--report-dir", type=Path, default=DEFAULT_REPORT_DIR)
+    parser.add_argument("--csv", type=Path, help="Optional CSV output path.")
+    args = parser.parse_args()
+
+    records = iter_records(args.report_dir)
+    rows = collect_rows(records)
+    summary = summarize(rows)
+    print(json.dumps({"records": len(records), "rows": len(rows), "models": summary}, ensure_ascii=False, indent=2))
+    if args.csv:
+        write_csv(args.csv, summary)
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())