109 lines
4.1 KiB
Python
109 lines
4.1 KiB
Python
from __future__ import annotations
|
|
|
|
import argparse
|
|
import json
|
|
from collections import Counter, defaultdict
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
DEFAULT_REPORT_DIR = ROOT / "reports" / "observability"
|
|
|
|
|
|
def iter_records(report_dir: Path) -> list[dict[str, Any]]:
|
|
records: list[dict[str, Any]] = []
|
|
if not report_dir.exists():
|
|
return records
|
|
for path in sorted(report_dir.rglob("*.jsonl")):
|
|
with path.open("r", encoding="utf-8") as handle:
|
|
for line_number, line in enumerate(handle, start=1):
|
|
line = line.strip()
|
|
if not line:
|
|
continue
|
|
try:
|
|
record = json.loads(line)
|
|
except json.JSONDecodeError as exc:
|
|
raise ValueError(f"{path}:{line_number}: invalid JSONL: {exc}") from exc
|
|
if isinstance(record, dict):
|
|
records.append(record)
|
|
return records
|
|
|
|
|
|
def summarize(records: list[dict[str, Any]]) -> dict[str, Any]:
|
|
by_event = Counter()
|
|
by_status = Counter()
|
|
failure_types = Counter()
|
|
plugin_turns = defaultdict(lambda: {"turns": 0, "success": 0, "partial": 0, "failure": 0})
|
|
model_routes = defaultdict(lambda: {"calls": 0, "avg_latency_ms": None, "_latencies": []})
|
|
tool_summary = defaultdict(lambda: {"calls": 0, "avg_duration_ms": None, "_durations": []})
|
|
|
|
for record in records:
|
|
event_type = str(record.get("event_type") or "")
|
|
by_event[event_type] += 1
|
|
|
|
if event_type == "access_events":
|
|
by_status[str(record.get("status_code") or "")] += 1
|
|
elif event_type == "turn_audit":
|
|
plugin = str(record.get("plugin") or "")
|
|
outcome = str(record.get("outcome") or "")
|
|
failure_type = str(record.get("failure_type") or "")
|
|
item = plugin_turns[plugin]
|
|
item["turns"] += 1
|
|
if outcome in {"success", "partial", "failure"}:
|
|
item[outcome] += 1
|
|
if failure_type and failure_type != "none":
|
|
failure_types[failure_type] += 1
|
|
elif event_type == "model_calls":
|
|
route = str(record.get("route_name") or "")
|
|
item = model_routes[route]
|
|
item["calls"] += 1
|
|
latency = record.get("latency_ms")
|
|
if isinstance(latency, int | float):
|
|
item["_latencies"].append(float(latency))
|
|
elif event_type == "tool_calls":
|
|
tool_name = str(record.get("tool_name") or "")
|
|
item = tool_summary[tool_name]
|
|
item["calls"] += 1
|
|
duration = record.get("duration_ms")
|
|
if isinstance(duration, int | float):
|
|
item["_durations"].append(float(duration))
|
|
|
|
model_route_rows = []
|
|
for route_name, item in sorted(model_routes.items()):
|
|
latencies = item.pop("_latencies")
|
|
item["route_name"] = route_name
|
|
item["avg_latency_ms"] = round(sum(latencies) / len(latencies)) if latencies else None
|
|
model_route_rows.append(item)
|
|
|
|
tool_rows = []
|
|
for tool_name, item in sorted(tool_summary.items()):
|
|
durations = item.pop("_durations")
|
|
item["tool_name"] = tool_name
|
|
item["avg_duration_ms"] = round(sum(durations) / len(durations)) if durations else None
|
|
tool_rows.append(item)
|
|
|
|
return {
|
|
"records": len(records),
|
|
"by_event_type": dict(sorted(by_event.items())),
|
|
"http_status_counts": dict(sorted(by_status.items())),
|
|
"failure_type_counts": dict(sorted(failure_types.items())),
|
|
"turns_by_plugin": {key: value for key, value in sorted(plugin_turns.items())},
|
|
"model_routes": model_route_rows,
|
|
"tool_calls": tool_rows,
|
|
}
|
|
|
|
|
|
def main() -> int:
|
|
parser = argparse.ArgumentParser(description="Summarize observability JSONL reports.")
|
|
parser.add_argument("--report-dir", type=Path, default=DEFAULT_REPORT_DIR)
|
|
args = parser.parse_args()
|
|
|
|
records = iter_records(args.report_dir)
|
|
print(json.dumps(summarize(records), ensure_ascii=False, indent=2))
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|