192 lines
7.2 KiB
Python
192 lines
7.2 KiB
Python
from __future__ import annotations
|
|
|
|
import argparse
|
|
import datetime as dt
|
|
import json
|
|
import time
|
|
import urllib.error
|
|
import urllib.request
|
|
from pathlib import Path
|
|
|
|
from common import ROOT, read_json, write_json
|
|
|
|
|
|
DEFAULT_MODEL_ID = "qwen3-coder-30b-a3b-instruct-q6_k"
|
|
DEFAULT_PROFILES = ["gpu-fast", "cpu-test"]
|
|
DEFAULT_REPORT = ROOT / "reports" / "benchmarks" / "runtime-profiles-qwen3-coder-q6.json"
|
|
RUNTIME_PROFILES = ROOT / "config" / "runtime_profiles.json"
|
|
DEFAULT_PROMPT = (
|
|
"Ты эксперт 1С. Кратко, но предметно опиши безопасный план анализа ошибки "
|
|
"проведения документа РеализацияТоваровУслуг, если нет metadata snapshot. Дай 8 пунктов."
|
|
)
|
|
|
|
|
|
def chat_completion(
|
|
*,
|
|
base_url: str,
|
|
model: str,
|
|
prompt: str,
|
|
temperature: float,
|
|
max_tokens: int,
|
|
timeout: int,
|
|
) -> dict:
|
|
payload = {
|
|
"model": model,
|
|
"messages": [
|
|
{"role": "system", "content": "Отвечай по-русски, кратко и по делу."},
|
|
{"role": "user", "content": prompt},
|
|
],
|
|
"temperature": temperature,
|
|
"max_tokens": max_tokens,
|
|
"stream": False,
|
|
}
|
|
request = urllib.request.Request(
|
|
f"{base_url.rstrip('/')}/v1/chat/completions",
|
|
data=json.dumps(payload, ensure_ascii=False).encode("utf-8"),
|
|
headers={"Content-Type": "application/json"},
|
|
method="POST",
|
|
)
|
|
started_at = time.perf_counter()
|
|
with urllib.request.urlopen(request, timeout=timeout) as response:
|
|
data = json.loads(response.read().decode("utf-8"))
|
|
elapsed_sec = time.perf_counter() - started_at
|
|
choices = data.get("choices") or []
|
|
answer = ""
|
|
if choices and isinstance(choices[0], dict):
|
|
answer = str((choices[0].get("message") or {}).get("content") or "")
|
|
usage = data.get("usage") or {}
|
|
completion_tokens = int(usage.get("completion_tokens") or 0)
|
|
total_tokens = int(usage.get("total_tokens") or 0)
|
|
prompt_tokens = int(usage.get("prompt_tokens") or 0)
|
|
return {
|
|
"status": "ok",
|
|
"elapsed_sec": round(elapsed_sec, 3),
|
|
"prompt_tokens": prompt_tokens,
|
|
"completion_tokens": completion_tokens,
|
|
"total_tokens": total_tokens,
|
|
"output_tokens_per_sec": round(completion_tokens / elapsed_sec, 3) if completion_tokens else None,
|
|
"total_tokens_per_sec": round(total_tokens / elapsed_sec, 3) if total_tokens else None,
|
|
"answer_preview": answer[:800],
|
|
}
|
|
|
|
|
|
def profile_target(profile: dict, model_id: str, plugin: str) -> dict:
|
|
overrides = profile.get("model_overrides") or {}
|
|
override = overrides.get(model_id) or {}
|
|
base_url = override.get("base_url") or (profile.get("endpoints") or {}).get(plugin)
|
|
served_model_name = override.get("served_model_name")
|
|
if not base_url:
|
|
raise ValueError(f"profile `{profile.get('id')}` has no endpoint for model `{model_id}`")
|
|
if not served_model_name:
|
|
raise ValueError(f"profile `{profile.get('id')}` has no served_model_name override for `{model_id}`")
|
|
return {
|
|
"base_url": str(base_url),
|
|
"served_model_name": str(served_model_name),
|
|
"container_name": override.get("container_name"),
|
|
"host": profile.get("host"),
|
|
"docker_endpoint": profile.get("docker_endpoint"),
|
|
"role": profile.get("role"),
|
|
}
|
|
|
|
|
|
def benchmark_profile(profile_id: str, profile: dict, *, model_id: str, plugin: str, args: argparse.Namespace) -> dict:
|
|
try:
|
|
target = profile_target(profile, model_id, plugin)
|
|
result = chat_completion(
|
|
base_url=target["base_url"],
|
|
model=target["served_model_name"],
|
|
prompt=args.prompt,
|
|
temperature=args.temperature,
|
|
max_tokens=args.max_tokens,
|
|
timeout=args.timeout,
|
|
)
|
|
return {
|
|
"profile_id": profile_id,
|
|
"label": profile.get("label") or profile_id,
|
|
"model_id": model_id,
|
|
"target": target,
|
|
**result,
|
|
}
|
|
except (ValueError, TimeoutError, urllib.error.URLError, json.JSONDecodeError) as exc:
|
|
return {
|
|
"profile_id": profile_id,
|
|
"label": profile.get("label") or profile_id,
|
|
"model_id": model_id,
|
|
"status": "error",
|
|
"error": str(exc),
|
|
}
|
|
|
|
|
|
def speedup_summary(results: list[dict]) -> dict:
|
|
speeds = {
|
|
str(item.get("profile_id")): float(item.get("output_tokens_per_sec") or 0)
|
|
for item in results
|
|
if item.get("status") == "ok" and item.get("output_tokens_per_sec")
|
|
}
|
|
gpu_speed = speeds.get("gpu-fast")
|
|
cpu_speed = speeds.get("cpu-test")
|
|
summary = {"output_tokens_per_sec": speeds}
|
|
if gpu_speed and cpu_speed:
|
|
summary["gpu_vs_cpu_ratio"] = round(gpu_speed / cpu_speed, 3)
|
|
return summary
|
|
|
|
|
|
def main() -> int:
|
|
parser = argparse.ArgumentParser(description="Benchmark the same model across runtime profiles.")
|
|
parser.add_argument("--model-id", default=DEFAULT_MODEL_ID)
|
|
parser.add_argument("--plugin", default="1c")
|
|
parser.add_argument("--profiles", nargs="+", default=DEFAULT_PROFILES)
|
|
parser.add_argument("--prompt", default=DEFAULT_PROMPT)
|
|
parser.add_argument("--temperature", type=float, default=0.1)
|
|
parser.add_argument("--max-tokens", type=int, default=384)
|
|
parser.add_argument("--timeout", type=int, default=600)
|
|
parser.add_argument("--report", type=Path, default=DEFAULT_REPORT)
|
|
parser.add_argument("--print", action="store_true")
|
|
args = parser.parse_args()
|
|
|
|
config = read_json(RUNTIME_PROFILES)
|
|
profiles = config.get("profiles") or {}
|
|
results = []
|
|
for profile_id in args.profiles:
|
|
profile = profiles.get(profile_id)
|
|
if not isinstance(profile, dict):
|
|
results.append(
|
|
{
|
|
"profile_id": profile_id,
|
|
"model_id": args.model_id,
|
|
"status": "error",
|
|
"error": f"unknown runtime profile: {profile_id}",
|
|
}
|
|
)
|
|
continue
|
|
profile = {**profile, "id": profile_id}
|
|
results.append(benchmark_profile(profile_id, profile, model_id=args.model_id, plugin=args.plugin, args=args))
|
|
|
|
ok_results = [item for item in results if item.get("status") == "ok"]
|
|
fastest = None
|
|
if ok_results:
|
|
fastest = max(ok_results, key=lambda item: float(item.get("output_tokens_per_sec") or 0)).get("profile_id")
|
|
report = {
|
|
"created_at": dt.datetime.now(dt.UTC).isoformat(),
|
|
"model_id": args.model_id,
|
|
"plugin": args.plugin,
|
|
"prompt": args.prompt,
|
|
"temperature": args.temperature,
|
|
"max_tokens": args.max_tokens,
|
|
"status": "ok" if len(ok_results) == len(results) else "partial" if ok_results else "failed",
|
|
"fastest_profile_id": fastest,
|
|
"speedup": speedup_summary(results),
|
|
"results": results,
|
|
}
|
|
write_json(args.report, report)
|
|
if args.print:
|
|
print(json.dumps(report, ensure_ascii=False, indent=2))
|
|
else:
|
|
print(f"Benchmark status: {report['status']}")
|
|
print(f"Wrote report to {args.report}")
|
|
return 0 if ok_results else 1
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|