2954 lines
126 KiB
Python
2954 lines
126 KiB
Python
from __future__ import annotations
|
||
|
||
import argparse
|
||
import datetime as dt
|
||
import json
|
||
import mimetypes
|
||
import os
|
||
import re
|
||
import shutil
|
||
import socket
|
||
import subprocess
|
||
import sys
|
||
import threading
|
||
import time
|
||
import urllib.error
|
||
import urllib.parse
|
||
import urllib.request
|
||
import uuid
|
||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||
from pathlib import Path
|
||
from urllib.parse import urlparse, urlunparse
|
||
|
||
from ask_1c_rag import DEFAULT_INDEX as DEFAULT_1C_RAG_INDEX
|
||
from ask_1c_rag import DEFAULT_RAG_PROMPT as DEFAULT_1C_RAG_PROMPT
|
||
from ask_1c_rag import format_context, render_prompt
|
||
from common import ROOT, call_chat_completion, localize_workspace_path, read_json, read_yaml_mapping, search_lexical_index
|
||
from rag_profiles import list_rag_profiles, resolve_rag_profile
|
||
|
||
|
||
DEFAULT_HOST = "127.0.0.1"
|
||
DEFAULT_PORT = 8765
|
||
DEFAULT_BASE_URL = "http://docker-gpu.cin.su:8000"
|
||
DEFAULT_STATIC_DIR = ROOT / "tools" / "model-chat"
|
||
REGISTRY_INDEX = ROOT / "registry" / "index.json"
|
||
REPORT_ROOT = Path(os.environ.get("MODEL_CHAT_REPORT_ROOT") or (ROOT / "reports"))
|
||
REPORT_DIR = REPORT_ROOT / "model-chat"
|
||
IMAGE_GALLERY_DIR = REPORT_DIR / "images"
|
||
IMAGE_JOBS = REPORT_DIR / "image-jobs.jsonl"
|
||
MODEL_INGEST_DIR = ROOT / "models" / "incoming"
|
||
MODEL_INGEST_JOBS = REPORT_ROOT / "model-ingest" / "jobs.jsonl"
|
||
GPU_PROFILES_PATH = ROOT / "config" / "gpu_profiles.json"
|
||
RUNTIME_PROFILES_PATH = ROOT / "config" / "runtime_profiles.json"
|
||
MODEL_CHAT_PREFLIGHT_REPORT = REPORT_ROOT / "model-chat" / "preflight.json"
|
||
IMAGE_PROXY_JOBS: dict[str, dict] = {}
|
||
IMAGE_PROXY_JOB_LOCK = threading.Lock()
|
||
PLUGIN_ORDER = ["text", "translation", "audio", "video", "image", "1c"]
|
||
PLUGIN_LABELS = {
|
||
"text": "Текст",
|
||
"translation": "Перевод",
|
||
"audio": "Звук",
|
||
"video": "Видео",
|
||
"image": "Фото",
|
||
"1c": "1С",
|
||
}
|
||
TASK_PLUGIN_MAP = {
|
||
"text": "text",
|
||
"chat": "text",
|
||
"summarization": "text",
|
||
"code": "text",
|
||
"tool-use": "text",
|
||
"translation": "translation",
|
||
"speech-to-text": "audio",
|
||
"speech-translation": "audio",
|
||
"video": "video",
|
||
"image-understanding": "video",
|
||
"document-understanding": "video",
|
||
"visual-question-answering": "video",
|
||
"image-generation": "image",
|
||
"image-editing": "image",
|
||
"inpainting": "image",
|
||
"1c": "1c",
|
||
"1c-rag": "1c",
|
||
"bsl-code": "1c",
|
||
"metadata-safety": "1c",
|
||
"1c-query": "1c",
|
||
}
|
||
DEFAULT_PROMPTS = {
|
||
"text": "Кратко объясни, зачем нужен реестр локальных моделей.",
|
||
"translation": "Переведи на английский: Нужно проверить качество локальной модели перевода.",
|
||
"audio": "Составь короткий план проверки модели распознавания речи для русского языка.",
|
||
"video": "Опиши, какие вопросы стоит задать vision-language модели при проверке видео.",
|
||
"image": "Фотореалистичный рабочий стол инженера 1С, два монитора, заметки, спокойный дневной свет.",
|
||
"1c": "Какие метаданные 1С нужно получить перед изменением формы документа?",
|
||
}
|
||
ENDPOINT_PRESETS = [
|
||
{"id": "vllm-text", "label": "vLLM text", "base_url": "http://docker-gpu.cin.su:8000"},
|
||
{"id": "llama-gguf", "label": "llama.cpp GGUF", "base_url": "http://docker-gpu.cin.su:8080"},
|
||
{"id": "llama-gguf-q6-test", "label": "llama.cpp Q6 test", "base_url": "http://docker-gpu.cin.su:8081"},
|
||
{"id": "docker-test-q6-cpu", "label": "docker-test Q6 CPU", "base_url": "http://docker-test.cin.su:18086"},
|
||
{"id": "translation-api", "label": "Translation API", "base_url": "http://docker-gpu.cin.su:8010"},
|
||
{"id": "audio-api", "label": "Audio API", "base_url": "http://docker-gpu.cin.su:8020"},
|
||
{"id": "video-api", "label": "Video API", "base_url": "http://docker-gpu.cin.su:8030"},
|
||
{"id": "image-api", "label": "Image API", "base_url": "http://docker-gpu.cin.su:8040"},
|
||
{"id": "local-vllm", "label": "local", "base_url": "http://127.0.0.1:8000"},
|
||
]
|
||
ALLOWED_ENDPOINT_HOSTS = {
|
||
("http", "docker-gpu.cin.su", 8000),
|
||
("http", "docker-gpu.cin.su", 8080),
|
||
("http", "docker-gpu.cin.su", 8081),
|
||
("http", "docker-gpu.cin.su", 8010),
|
||
("http", "docker-gpu.cin.su", 8020),
|
||
("http", "docker-gpu.cin.su", 8030),
|
||
("http", "docker-gpu.cin.su", 8040),
|
||
("http", "docker-test.cin.su", 18081),
|
||
("http", "docker-test.cin.su", 18086),
|
||
("http", "192.168.200.61", 18081),
|
||
("http", "192.168.200.61", 18086),
|
||
("http", "127.0.0.1", 8000),
|
||
("http", "127.0.0.1", 8080),
|
||
("http", "127.0.0.1", 8081),
|
||
("http", "127.0.0.1", 8010),
|
||
("http", "127.0.0.1", 8020),
|
||
("http", "127.0.0.1", 8030),
|
||
("http", "127.0.0.1", 8040),
|
||
("http", "127.0.0.1", 8765),
|
||
("http", "localhost", 8000),
|
||
("http", "localhost", 8080),
|
||
("http", "localhost", 8081),
|
||
("http", "localhost", 8010),
|
||
("http", "localhost", 8020),
|
||
("http", "localhost", 8030),
|
||
("http", "localhost", 8040),
|
||
}
|
||
SERVICE_PLANS = {
|
||
"vllm": {
|
||
"service_id": "vllm-text",
|
||
"label": "vLLM text",
|
||
"base_url": "http://docker-gpu.cin.su:8000",
|
||
"container_name": "llm-vllm-text",
|
||
"compose": "core/deploy/docker-gpu/vllm/compose.yaml",
|
||
"deploy_script": "scripts/deploy_vllm.ps1",
|
||
"health_path": "/v1/models",
|
||
},
|
||
"llama.cpp": {
|
||
"service_id": "llama-gguf",
|
||
"label": "llama.cpp GGUF",
|
||
"base_url": "http://docker-gpu.cin.su:8080",
|
||
"container_name": "llm-llama-devstral-1c",
|
||
"compose": "core/deploy/docker-gpu/llama-cpp/compose.yaml",
|
||
"deploy_script": "scripts/deploy_llama_cpp.ps1",
|
||
"health_path": "/v1/models",
|
||
},
|
||
"transformers:translation": {
|
||
"service_id": "translation-api",
|
||
"label": "Transformers translation",
|
||
"base_url": "http://docker-gpu.cin.su:8010",
|
||
"container_name": "llm-transformers-translation",
|
||
"compose": "core/deploy/docker-gpu/transformers/translation.compose.yaml",
|
||
"deploy_script": "scripts/deploy_transformers_service.ps1 -Plugin translation",
|
||
"health_path": "/health",
|
||
},
|
||
"transformers:audio": {
|
||
"service_id": "audio-api",
|
||
"label": "Transformers audio",
|
||
"base_url": "http://docker-gpu.cin.su:8020",
|
||
"container_name": "llm-transformers-audio",
|
||
"compose": "core/deploy/docker-gpu/transformers/audio.compose.yaml",
|
||
"deploy_script": "scripts/deploy_transformers_service.ps1 -Plugin audio",
|
||
"health_path": "/health",
|
||
},
|
||
"transformers:video": {
|
||
"service_id": "video-api",
|
||
"label": "Transformers video",
|
||
"base_url": "http://docker-gpu.cin.su:8030",
|
||
"container_name": "llm-transformers-video",
|
||
"compose": "core/deploy/docker-gpu/transformers/video.compose.yaml",
|
||
"deploy_script": "scripts/deploy_transformers_service.ps1 -Plugin video",
|
||
"health_path": "/health",
|
||
},
|
||
"transformers:image": {
|
||
"service_id": "image-api",
|
||
"label": "Transformers image",
|
||
"base_url": "http://docker-gpu.cin.su:8040",
|
||
"container_name": "llm-transformers-image",
|
||
"compose": "core/deploy/docker-gpu/transformers/image.compose.yaml",
|
||
"deploy_script": "scripts/deploy_transformers_service.ps1 -Plugin image",
|
||
"health_path": "/health",
|
||
},
|
||
}
|
||
SERVICE_CONTROL_ACTIONS = {"start", "stop", "restart", "status"}
|
||
DEFAULT_DOCKER_HOST = os.environ.get("DOCKER_HOST", "ssh://docker-gpu")
|
||
TEST_PACKS = {
|
||
"text": [
|
||
{"id": "short", "label": "Краткий ответ", "prompt": DEFAULT_PROMPTS["text"]},
|
||
{"id": "json", "label": "JSON", "prompt": "Верни JSON с полями status, summary, next_steps. Без markdown."},
|
||
{"id": "format", "label": "Формат", "prompt": "Сделай список из 5 пунктов: критерии приемки локальной LLM."},
|
||
{"id": "critique", "label": "Критика", "prompt": "Назови 3 риска при проверке локальной модели и как их снизить."},
|
||
],
|
||
"translation": [
|
||
{"id": "ru-en", "label": "RU -> EN", "prompt": "Переведи на английский: Нужно проверить качество локальной модели перевода."},
|
||
{"id": "en-ru", "label": "EN -> RU", "prompt": "Translate into Russian: The model must preserve technical terms and JSON structure."},
|
||
{"id": "terms", "label": "Термины", "prompt": "Переведи на английский, сохрани термины: регистр сведений, справочник, документ реализации."},
|
||
{"id": "format", "label": "Сохранить JSON", "prompt": "Переведи значения JSON на английский, ключи не меняй: {\"status\":\"готово\",\"risk\":\"нет доступа к GPU\"}"},
|
||
],
|
||
"audio": [
|
||
{"id": "plan", "label": "План", "prompt": DEFAULT_PROMPTS["audio"]},
|
||
{"id": "metrics", "label": "Метрики", "prompt": "Какие метрики использовать для оценки speech-to-text на русском языке?"},
|
||
{"id": "noise", "label": "Шум", "prompt": "Составь сценарии проверки распознавания речи при шуме и разных микрофонах."},
|
||
{"id": "diarization", "label": "Диалоги", "prompt": "Как проверить качество распознавания диалога двух пользователей?"},
|
||
],
|
||
"video": [
|
||
{"id": "plan", "label": "План", "prompt": DEFAULT_PROMPTS["video"]},
|
||
{"id": "scene", "label": "Сцена", "prompt": "Какие вопросы задать модели, чтобы проверить понимание сцены на видео?"},
|
||
{"id": "events", "label": "События", "prompt": "Составь тесты для поиска событий во временной шкале видео."},
|
||
{"id": "docs", "label": "Документы", "prompt": "Как проверить vision-language модель на чтении документов и экранных форм?"},
|
||
],
|
||
"image": [
|
||
{"id": "photo", "label": "Фото", "prompt": DEFAULT_PROMPTS["image"]},
|
||
{"id": "product", "label": "Предмет", "prompt": "Предметное фото локального GPU-сервера на чистом столе, реалистичный свет, 1024x1024."},
|
||
{"id": "edit", "label": "Редактирование", "prompt": "Заменить фон на светлую офисную стену, сохранить предмет и естественные тени."},
|
||
{"id": "ui", "label": "Интерфейс", "prompt": "Скриншот современного локального кабинета моделей, аккуратная панель, реалистичная фотография монитора."},
|
||
],
|
||
"1c": [
|
||
{"id": "metadata", "label": "Метаданные", "prompt": DEFAULT_PROMPTS["1c"]},
|
||
{"id": "bsl", "label": "BSL", "prompt": "Найди возможную ошибку в BSL-коде и объясни безопасное исправление: Если Объект.Сумма = 0 Тогда Возврат; КонецЕсли;"},
|
||
{"id": "query", "label": "Запрос 1С", "prompt": "Составь read-only запрос 1С для получения 10 последних документов реализации."},
|
||
{"id": "safety", "label": "Безопасность", "prompt": "Почему модель не должна напрямую менять живую базу 1С? Дай workflow согласования."},
|
||
],
|
||
}
|
||
STATUS_RANK = {
|
||
"production": 0,
|
||
"staging": 1,
|
||
"candidate": 2,
|
||
"draft": 3,
|
||
"archived": 9,
|
||
}
|
||
ONE_C_DANGEROUS_REQUEST_PATTERNS = [
|
||
r"(?i)\b(delete|drop|update|insert|truncate|alter)\b",
|
||
r"(?i)\b(удалить|удали|удаление|изменить|измени|обновить|обнови)\b",
|
||
r"(?i)\b(записать|запиши|провести|проведи)\b",
|
||
r"(?i)\b(объект\.записать|провести\s*\(|удалить\s+из|изменить\s+.*\s+установить)\b",
|
||
]
|
||
ONE_C_UNSAFE_ANSWER_PATTERNS = [
|
||
r"(?i)checklist\s+для\s+удален",
|
||
r"(?i)план\s+.*удален",
|
||
r"(?i)безопасн\w*\s+удален",
|
||
r"(?i)уточните\s+услов",
|
||
r"(?i)ограничьте\s+удален",
|
||
r"(?i)если\s+вы\s+хотите\s+удал",
|
||
r"(?i)помог[уа]\s+.*удал",
|
||
r"(?i)удалитьизсправочника",
|
||
]
|
||
ONE_C_SAFE_MUTATION_REFUSAL = (
|
||
"Я не могу выполнить, составить или уточнять операцию изменения данных в 1С. "
|
||
"Могу помочь только read-only анализом: показать запрос `ВЫБРАТЬ` для инвентаризации "
|
||
"затрагиваемых записей, проверить зависимости по metadata snapshot и подготовить checklist "
|
||
"согласования для ответственного администратора."
|
||
)
|
||
|
||
|
||
def sanitize_filename(value: str, *, fallback: str = "model.bin") -> str:
|
||
name = Path(str(value or fallback).replace("\\", "/")).name.strip()
|
||
name = re.sub(r"[^A-Za-z0-9._+@=-]+", "_", name)
|
||
name = name.strip("._")
|
||
return name or fallback
|
||
|
||
|
||
def sanitize_id(value: str, *, fallback: str = "model") -> str:
|
||
text = str(value or fallback).strip().lower()
|
||
text = re.sub(r"[^a-z0-9._-]+", "-", text)
|
||
text = text.strip(".-_")
|
||
return text or fallback
|
||
|
||
|
||
def now_iso() -> str:
|
||
return dt.datetime.now(dt.UTC).isoformat()
|
||
|
||
|
||
def display_path(path: Path) -> str:
|
||
try:
|
||
return str(path.relative_to(ROOT))
|
||
except ValueError:
|
||
return str(path)
|
||
|
||
|
||
def model_ingest_job_dir(job_id: str) -> Path:
|
||
return MODEL_INGEST_DIR / job_id
|
||
|
||
|
||
def latest_user_message(messages: list[dict]) -> str:
|
||
for message in reversed(messages):
|
||
if isinstance(message, dict) and message.get("role") == "user":
|
||
return str(message.get("content") or "")
|
||
return ""
|
||
|
||
|
||
def has_pattern(text: str, patterns: list[str]) -> bool:
|
||
return any(re.search(pattern, text or "") for pattern in patterns)
|
||
|
||
|
||
def guard_1c_answer(question: str, answer: str | None) -> tuple[str | None, dict | None]:
|
||
if not answer:
|
||
return answer, None
|
||
dangerous_request = has_pattern(question, ONE_C_DANGEROUS_REQUEST_PATTERNS)
|
||
unsafe_answer = has_pattern(answer, ONE_C_UNSAFE_ANSWER_PATTERNS)
|
||
if not dangerous_request or not unsafe_answer:
|
||
return answer, None
|
||
return ONE_C_SAFE_MUTATION_REFUSAL, {
|
||
"applied": True,
|
||
"reason": "dangerous_1c_mutation_answer",
|
||
}
|
||
|
||
|
||
def append_model_ingest_job(job: dict) -> None:
|
||
MODEL_INGEST_JOBS.parent.mkdir(parents=True, exist_ok=True)
|
||
with MODEL_INGEST_JOBS.open("a", encoding="utf-8") as handle:
|
||
handle.write(json.dumps(job, ensure_ascii=False) + "\n")
|
||
|
||
|
||
def append_image_job(job: dict) -> None:
|
||
IMAGE_JOBS.parent.mkdir(parents=True, exist_ok=True)
|
||
with IMAGE_JOBS.open("a", encoding="utf-8") as handle:
|
||
handle.write(json.dumps(job, ensure_ascii=False) + "\n")
|
||
|
||
|
||
def latest_image_jobs(limit: int = 30) -> list[dict]:
|
||
if not IMAGE_JOBS.exists():
|
||
return []
|
||
rows = []
|
||
with IMAGE_JOBS.open("r", encoding="utf-8") as handle:
|
||
for line in handle:
|
||
line = line.strip()
|
||
if not line:
|
||
continue
|
||
try:
|
||
rows.append(json.loads(line))
|
||
except json.JSONDecodeError:
|
||
continue
|
||
latest: dict[str, dict] = {}
|
||
for row in rows:
|
||
job_id = str(row.get("id") or "")
|
||
if job_id:
|
||
latest[job_id] = row
|
||
collapsed = list(latest.values())
|
||
collapsed.sort(key=lambda item: str(item.get("updated_at") or item.get("created_at") or ""), reverse=True)
|
||
return collapsed[:limit]
|
||
|
||
|
||
def read_model_ingest_jobs(limit: int = 30) -> list[dict]:
|
||
if not MODEL_INGEST_JOBS.exists():
|
||
return []
|
||
rows = []
|
||
with MODEL_INGEST_JOBS.open("r", encoding="utf-8") as handle:
|
||
for line in handle:
|
||
line = line.strip()
|
||
if not line:
|
||
continue
|
||
try:
|
||
rows.append(json.loads(line))
|
||
except json.JSONDecodeError:
|
||
continue
|
||
return rows[-limit:][::-1]
|
||
|
||
|
||
def latest_model_ingest_jobs(limit: int = 100) -> list[dict]:
|
||
latest: dict[str, dict] = {}
|
||
for job in read_model_ingest_jobs(limit=limit * 5):
|
||
job_id = str(job.get("id") or "")
|
||
if job_id and job_id not in latest:
|
||
latest[job_id] = job
|
||
return list(latest.values())[:limit]
|
||
|
||
|
||
def decode_data_url_bytes(value: str, *, field_name: str = "data") -> bytes:
|
||
data = str(value or "")
|
||
if "," in data and data.split(",", 1)[0].startswith("data:"):
|
||
data = data.split(",", 1)[1]
|
||
if not data:
|
||
raise ValueError(f"{field_name} is required")
|
||
import base64
|
||
|
||
return base64.b64decode(data)
|
||
|
||
|
||
def save_image_artifact(*, report_id: str, image_base64: str, kind: str, metadata: dict) -> dict:
|
||
created = dt.datetime.now(dt.UTC)
|
||
day_dir = IMAGE_GALLERY_DIR / created.strftime("%Y%m%d")
|
||
day_dir.mkdir(parents=True, exist_ok=True)
|
||
safe_kind = sanitize_id(kind, fallback="image")
|
||
filename = f"{created.strftime('%H%M%S')}-{report_id}-{safe_kind}.png"
|
||
image_path = day_dir / filename
|
||
image_path.write_bytes(decode_data_url_bytes(image_base64, field_name="image_base64"))
|
||
sidecar_path = image_path.with_suffix(".json")
|
||
record = {
|
||
"id": report_id,
|
||
"created_at": created.isoformat(),
|
||
"kind": kind,
|
||
"path": display_path(image_path),
|
||
"url": f"/generated-images/{created.strftime('%Y%m%d')}/{filename}",
|
||
"size_bytes": image_path.stat().st_size,
|
||
**metadata,
|
||
}
|
||
sidecar_path.write_text(json.dumps(record, ensure_ascii=False, indent=2), encoding="utf-8")
|
||
return record
|
||
|
||
|
||
def latest_image_gallery(limit: int = 40) -> list[dict]:
|
||
if not IMAGE_GALLERY_DIR.exists():
|
||
return []
|
||
records = []
|
||
for sidecar in IMAGE_GALLERY_DIR.glob("*/*.json"):
|
||
try:
|
||
data = json.loads(sidecar.read_text(encoding="utf-8"))
|
||
except (OSError, json.JSONDecodeError):
|
||
continue
|
||
if isinstance(data, dict) and data.get("url"):
|
||
records.append(data)
|
||
records.sort(key=lambda item: str(item.get("created_at") or ""), reverse=True)
|
||
return records[:limit]
|
||
|
||
|
||
def update_model_ingest_job(job: dict, *, status: str, error: str | None = None, **extra: object) -> dict:
|
||
updated = dict(job)
|
||
updated["status"] = status
|
||
updated["updated_at"] = now_iso()
|
||
if error is None:
|
||
updated.pop("error", None)
|
||
else:
|
||
updated["error"] = error
|
||
updated.update(extra)
|
||
write_model_ingest_metadata(model_ingest_job_dir(str(updated["id"])), updated)
|
||
append_model_ingest_job(updated)
|
||
return updated
|
||
|
||
|
||
def verify_model_ingest_job(job_id: str) -> dict:
|
||
job = next((item for item in latest_model_ingest_jobs() if str(item.get("id")) == job_id), None)
|
||
if not job:
|
||
raise ValueError("model ingest job not found")
|
||
|
||
files = job.get("files") or []
|
||
if not files:
|
||
return update_model_ingest_job(job, status="needs_file", error="job has no imported files")
|
||
|
||
checked_files = []
|
||
missing_files = []
|
||
total_size = 0
|
||
for file_info in files:
|
||
relative_path = str(file_info.get("path") or "")
|
||
path = (ROOT / relative_path).resolve()
|
||
if not path.exists() or not path.is_file():
|
||
missing_files.append(relative_path)
|
||
continue
|
||
size = path.stat().st_size
|
||
total_size += size
|
||
checked_files.append({"path": relative_path, "size_bytes": size})
|
||
|
||
if missing_files:
|
||
return update_model_ingest_job(
|
||
job,
|
||
status="missing_files",
|
||
error=f"missing files: {', '.join(missing_files)}",
|
||
checked_files=checked_files,
|
||
)
|
||
|
||
return update_model_ingest_job(
|
||
job,
|
||
status="verified",
|
||
checked_files=checked_files,
|
||
total_size_bytes=total_size,
|
||
)
|
||
|
||
|
||
def task_for_plugin(plugin: str) -> list[str]:
|
||
mapping = {
|
||
"text": ["text", "chat"],
|
||
"translation": ["translation"],
|
||
"audio": ["speech-to-text"],
|
||
"video": ["image-understanding", "visual-question-answering"],
|
||
"image": ["image-generation"],
|
||
"1c": ["1c", "1c-rag", "bsl-code"],
|
||
}
|
||
return mapping.get(plugin, [plugin])
|
||
|
||
|
||
def runtime_for_plugin_and_format(plugin: str, model_format: str) -> str:
|
||
if model_format.lower() == "gguf":
|
||
return "llama.cpp"
|
||
if plugin in {"translation", "audio", "video", "image"}:
|
||
return "transformers"
|
||
return "vllm"
|
||
|
||
|
||
def register_model_ingest_job(job_id: str) -> dict:
|
||
job = next((item for item in latest_model_ingest_jobs(limit=500) if str(item.get("id")) == job_id), None)
|
||
if not job:
|
||
raise ValueError("model ingest job not found")
|
||
if str(job.get("status")) != "verified":
|
||
job = verify_model_ingest_job(job_id)
|
||
if str(job.get("status")) != "verified":
|
||
raise ValueError(f"model ingest job is not verified: {job.get('status')}")
|
||
|
||
model_id = sanitize_id(job.get("model_id") or job_id)
|
||
plugin = sanitize_id(job.get("plugin") or "text")
|
||
model_format = str(job.get("format") or "").lower() or "custom"
|
||
card_path = ROOT / "registry" / "model-cards" / f"{model_id}.yaml"
|
||
if card_path.exists():
|
||
raise ValueError(f"model card already exists: {card_path.relative_to(ROOT)}")
|
||
files = job.get("checked_files") or job.get("files") or []
|
||
first_file = files[0] if files else {}
|
||
storage_path = str(job.get("target_dir") or "")
|
||
card = {
|
||
"id": model_id,
|
||
"name": str(job.get("name") or model_id),
|
||
"status": "draft",
|
||
"type": "imported-model",
|
||
"task": task_for_plugin(plugin),
|
||
"runtime": runtime_for_plugin_and_format(plugin, model_format),
|
||
"format": model_format,
|
||
"storage_path": storage_path,
|
||
"served_model_name": model_id,
|
||
"license": "unknown",
|
||
"source": {
|
||
"kind": job.get("source_kind"),
|
||
"source": job.get("source") or "",
|
||
"ingest_job_id": job_id,
|
||
},
|
||
"deployment": {
|
||
"runtime": runtime_for_plugin_and_format(plugin, model_format),
|
||
"notes": "Registered from model ingest. Review runtime and served_model_name before production use.",
|
||
},
|
||
}
|
||
if first_file.get("path"):
|
||
card["filename"] = Path(str(first_file["path"])).name
|
||
if job.get("quantization"):
|
||
card["quantization"] = job.get("quantization")
|
||
|
||
def yaml_scalar(value: object) -> str:
|
||
text = str(value)
|
||
if not text or any(char in text for char in ":#[]{}&,*!|>'\"%@`"):
|
||
return json.dumps(text, ensure_ascii=False)
|
||
return text
|
||
|
||
def write_yaml_lines(value: object, indent: int = 0) -> list[str]:
|
||
prefix = " " * indent
|
||
lines: list[str] = []
|
||
if isinstance(value, dict):
|
||
for key, item in value.items():
|
||
if isinstance(item, (dict, list)):
|
||
lines.append(f"{prefix}{key}:")
|
||
lines.extend(write_yaml_lines(item, indent + 2))
|
||
else:
|
||
lines.append(f"{prefix}{key}: {yaml_scalar(item)}")
|
||
elif isinstance(value, list):
|
||
for item in value:
|
||
if isinstance(item, (dict, list)):
|
||
lines.append(f"{prefix}-")
|
||
lines.extend(write_yaml_lines(item, indent + 2))
|
||
else:
|
||
lines.append(f"{prefix}- {yaml_scalar(item)}")
|
||
return lines
|
||
|
||
card_path.write_text("\n".join(write_yaml_lines(card)) + "\n", encoding="utf-8")
|
||
build = run_command([sys.executable, "scripts/build_model_index.py"], timeout=60)
|
||
status = "registered" if build["returncode"] == 0 else "registered_index_failed"
|
||
return update_model_ingest_job(
|
||
job,
|
||
status=status,
|
||
card_path=str(card_path.relative_to(ROOT)),
|
||
index_result=build,
|
||
)
|
||
|
||
|
||
def write_model_ingest_metadata(job_dir: Path, job: dict) -> None:
|
||
job_dir.mkdir(parents=True, exist_ok=True)
|
||
(job_dir / "metadata.json").write_text(json.dumps(job, ensure_ascii=False, indent=2), encoding="utf-8")
|
||
|
||
|
||
def create_model_ingest_job(payload: dict, *, source_kind: str) -> tuple[dict, Path]:
|
||
job_id = str(uuid.uuid4())
|
||
model_id = sanitize_id(payload.get("model_id") or payload.get("name") or "model")
|
||
plugin = sanitize_id(payload.get("plugin") or "text")
|
||
job_dir = model_ingest_job_dir(job_id)
|
||
job = {
|
||
"id": job_id,
|
||
"created_at": now_iso(),
|
||
"updated_at": now_iso(),
|
||
"status": "created",
|
||
"source_kind": source_kind,
|
||
"model_id": model_id,
|
||
"name": str(payload.get("name") or model_id),
|
||
"plugin": plugin,
|
||
"task": payload.get("task") or [plugin],
|
||
"format": payload.get("format") or "",
|
||
"quantization": payload.get("quantization") or "",
|
||
"notes": payload.get("notes") or "",
|
||
"target_dir": str(job_dir.relative_to(ROOT)),
|
||
"files": [],
|
||
}
|
||
return job, job_dir
|
||
|
||
|
||
def stream_request_to_file(handler: BaseHTTPRequestHandler, destination: Path, content_length: int) -> int:
|
||
destination.parent.mkdir(parents=True, exist_ok=True)
|
||
remaining = content_length
|
||
written = 0
|
||
with destination.open("wb") as handle:
|
||
while remaining > 0:
|
||
chunk = handler.rfile.read(min(1024 * 1024, remaining))
|
||
if not chunk:
|
||
break
|
||
handle.write(chunk)
|
||
written += len(chunk)
|
||
remaining -= len(chunk)
|
||
return written
|
||
|
||
|
||
def download_url_to_file(url: str, destination: Path) -> int:
|
||
destination.parent.mkdir(parents=True, exist_ok=True)
|
||
request = urllib.request.Request(url, headers={"User-Agent": "LLM-model-ingest/1.0"})
|
||
written = 0
|
||
with urllib.request.urlopen(request, timeout=60) as response, destination.open("wb") as handle:
|
||
while True:
|
||
chunk = response.read(1024 * 1024)
|
||
if not chunk:
|
||
break
|
||
handle.write(chunk)
|
||
written += len(chunk)
|
||
return written
|
||
|
||
|
||
def import_model_from_source(payload: dict) -> dict:
|
||
source = str(payload.get("source") or "").strip()
|
||
if not source:
|
||
raise ValueError("source is required")
|
||
|
||
job, job_dir = create_model_ingest_job(payload, source_kind="source")
|
||
parsed = urlparse(source)
|
||
source_name = sanitize_filename(payload.get("filename") or Path(parsed.path or source).name or job["model_id"])
|
||
destination = job_dir / source_name
|
||
job["source"] = source
|
||
|
||
try:
|
||
if parsed.scheme in {"http", "https"}:
|
||
bytes_written = download_url_to_file(source, destination)
|
||
job["status"] = "downloaded"
|
||
else:
|
||
source_path = Path(source)
|
||
if not source_path.exists():
|
||
job["status"] = "queued_manual_required"
|
||
job["error"] = "source path is not accessible from this server"
|
||
bytes_written = 0
|
||
else:
|
||
job_dir.mkdir(parents=True, exist_ok=True)
|
||
shutil.copy2(source_path, destination)
|
||
bytes_written = destination.stat().st_size
|
||
job["status"] = "imported"
|
||
if bytes_written:
|
||
job["files"].append({"path": str(destination.relative_to(ROOT)), "size_bytes": bytes_written})
|
||
except (OSError, urllib.error.URLError) as exc:
|
||
job["status"] = "failed"
|
||
job["error"] = str(exc)
|
||
|
||
job["updated_at"] = now_iso()
|
||
write_model_ingest_metadata(job_dir, job)
|
||
append_model_ingest_job(job)
|
||
return job
|
||
|
||
|
||
def model_plugins(model: dict) -> list[str]:
|
||
plugins = []
|
||
for task in model.get("task") or []:
|
||
plugin = TASK_PLUGIN_MAP.get(task)
|
||
if plugin and plugin not in plugins:
|
||
plugins.append(plugin)
|
||
return plugins or ["text"]
|
||
|
||
|
||
def plugin_model_score(model: dict, plugin: str) -> tuple[int, int, str]:
|
||
tasks = set(model.get("task") or model.get("tasks") or [])
|
||
status = str(model.get("status") or "")
|
||
model_id = str(model.get("id") or "")
|
||
runtime = str(model.get("runtime") or "")
|
||
model_type = str(model.get("type") or "")
|
||
quantization = str(model.get("quantization") or "")
|
||
score = 0
|
||
if plugin == "text":
|
||
if tasks & {"text", "chat"}:
|
||
score -= 5
|
||
if tasks & {"1c", "1c-rag", "bsl-code"}:
|
||
score += 6
|
||
elif plugin == "1c":
|
||
if tasks & {"1c", "1c-rag", "bsl-code", "1c-query"}:
|
||
score -= 6
|
||
if "qwen3-coder" in model_id:
|
||
score -= 8
|
||
if quantization == "Q6_K" or "q6" in model_id:
|
||
score -= 3
|
||
if runtime == "llama.cpp":
|
||
score -= 2
|
||
if model_type == "lora-adapter" or model_id.endswith("-lora-v1"):
|
||
score += 8
|
||
elif plugin in TASK_PLUGIN_MAP.values():
|
||
mapped_tasks = {task for task, mapped_plugin in TASK_PLUGIN_MAP.items() if mapped_plugin == plugin}
|
||
if tasks & mapped_tasks:
|
||
score -= 5
|
||
return (score, STATUS_RANK.get(status, 8), str(model.get("id") or ""))
|
||
|
||
|
||
def load_gpu_profiles() -> dict[str, dict]:
|
||
profiles = read_json(GPU_PROFILES_PATH)
|
||
if not isinstance(profiles, dict):
|
||
raise ValueError(f"{GPU_PROFILES_PATH} must contain a profile mapping")
|
||
normalized = {}
|
||
for profile_id, profile in profiles.items():
|
||
if not isinstance(profile, dict):
|
||
continue
|
||
item = dict(profile)
|
||
item["id"] = str(item.get("id") or profile_id)
|
||
item.setdefault("label", item["id"])
|
||
item.setdefault("command", f"powershell -ExecutionPolicy Bypass -File scripts\\switch_gpu_profile.ps1 -Profile {item['id']}")
|
||
item.setdefault("starts", [])
|
||
item.setdefault("stops", [])
|
||
item.setdefault("wait", [])
|
||
item.setdefault("notes", "")
|
||
normalized[str(profile_id)] = item
|
||
return normalized
|
||
|
||
|
||
def public_gpu_profiles() -> dict[str, dict]:
|
||
return {
|
||
profile_id: {
|
||
key: value
|
||
for key, value in profile.items()
|
||
if key in {"id", "label", "command", "starts", "stops", "notes"}
|
||
}
|
||
for profile_id, profile in load_gpu_profiles().items()
|
||
}
|
||
|
||
|
||
def load_runtime_profiles() -> dict:
|
||
if not RUNTIME_PROFILES_PATH.exists():
|
||
return {"default": "gpu-fast", "profiles": {}}
|
||
data = read_json(RUNTIME_PROFILES_PATH)
|
||
if not isinstance(data, dict):
|
||
raise ValueError(f"{RUNTIME_PROFILES_PATH} must contain an object")
|
||
profiles = data.get("profiles") or {}
|
||
if not isinstance(profiles, dict):
|
||
raise ValueError(f"{RUNTIME_PROFILES_PATH} profiles must contain a mapping")
|
||
normalized = {}
|
||
for profile_id, profile in profiles.items():
|
||
if not isinstance(profile, dict):
|
||
continue
|
||
item = dict(profile)
|
||
item["id"] = str(item.get("id") or profile_id)
|
||
item.setdefault("label", item["id"])
|
||
item.setdefault("host", "")
|
||
item.setdefault("docker_endpoint", "")
|
||
item.setdefault("role", "")
|
||
item.setdefault("notes", "")
|
||
item.setdefault("endpoints", {})
|
||
item.setdefault("model_overrides", {})
|
||
normalized[str(profile_id)] = item
|
||
default_id = str(data.get("default") or next(iter(normalized), ""))
|
||
return {"default": default_id, "profiles": normalized}
|
||
|
||
|
||
def public_runtime_profiles() -> dict:
|
||
data = load_runtime_profiles()
|
||
return {
|
||
"default": data["default"],
|
||
"profiles": {
|
||
profile_id: {
|
||
key: value
|
||
for key, value in profile.items()
|
||
if key in {"id", "label", "host", "docker_endpoint", "role", "notes", "endpoints", "model_overrides"}
|
||
}
|
||
for profile_id, profile in data["profiles"].items()
|
||
},
|
||
}
|
||
|
||
|
||
def load_catalog() -> dict:
|
||
registry = read_json(REGISTRY_INDEX)
|
||
models = []
|
||
plugins = {plugin: {"id": plugin, "label": PLUGIN_LABELS[plugin], "models": []} for plugin in PLUGIN_ORDER}
|
||
|
||
for model in registry.get("models") or []:
|
||
deployment_model = model.get("served_model_name") or model.get("id")
|
||
item = {
|
||
"id": model.get("id"),
|
||
"name": model.get("name"),
|
||
"type": model.get("type"),
|
||
"status": model.get("status"),
|
||
"runtime": model.get("runtime"),
|
||
"storage_path": model.get("storage_path"),
|
||
"format": model.get("format"),
|
||
"quantization": model.get("quantization"),
|
||
"served_model_name": deployment_model,
|
||
"tasks": model.get("task") or [],
|
||
"plugins": model_plugins(model),
|
||
"card_path": model.get("card_path"),
|
||
}
|
||
models.append(item)
|
||
for plugin in item["plugins"]:
|
||
plugins.setdefault(plugin, {"id": plugin, "label": plugin, "models": []})
|
||
plugins[plugin]["models"].append(item["id"])
|
||
|
||
by_id = {model["id"]: model for model in models}
|
||
for plugin_id, plugin in plugins.items():
|
||
plugin["models"].sort(key=lambda model_id: plugin_model_score(by_id[model_id], plugin_id))
|
||
|
||
ordered_plugins = [plugins[plugin] for plugin in PLUGIN_ORDER if plugin in plugins]
|
||
extras = [value for key, value in plugins.items() if key not in PLUGIN_ORDER]
|
||
return {
|
||
"default_base_url": DEFAULT_BASE_URL,
|
||
"endpoint_presets": ENDPOINT_PRESETS,
|
||
"plugins": ordered_plugins + extras,
|
||
"models": models,
|
||
"default_prompts": DEFAULT_PROMPTS,
|
||
"test_packs": TEST_PACKS,
|
||
"gpu_profiles": public_gpu_profiles(),
|
||
"runtime_profiles": public_runtime_profiles(),
|
||
"rag_profiles": {
|
||
"1c": list_rag_profiles(),
|
||
},
|
||
}
|
||
|
||
|
||
def local_model_card(model: dict) -> dict:
|
||
card_path = model.get("card_path")
|
||
if not card_path:
|
||
return {}
|
||
path = (ROOT / str(card_path)).resolve()
|
||
if not path.exists():
|
||
return {}
|
||
return read_yaml_mapping(path)
|
||
|
||
|
||
def has_any_file(path: Path, patterns: list[str]) -> bool:
|
||
return any(path.glob(pattern) for pattern in patterns)
|
||
|
||
|
||
def has_weight_file(path: Path) -> bool:
|
||
for file_path in path.iterdir() if path.exists() else []:
|
||
if file_path.is_file() and file_path.name.lower().endswith((".safetensors", ".bin", ".gguf")):
|
||
return True
|
||
return False
|
||
|
||
|
||
def storage_status_for_model(model: dict) -> dict:
|
||
card = local_model_card(model)
|
||
raw_storage_path = str(model.get("storage_path") or card.get("storage_path") or "")
|
||
storage_candidates = [localize_workspace_path(raw_storage_path)]
|
||
if raw_storage_path.startswith("/models/"):
|
||
raw_path = Path(raw_storage_path)
|
||
if Path("/models").exists():
|
||
storage_candidates.insert(0, raw_path)
|
||
else:
|
||
storage_candidates.append(raw_path)
|
||
storage_path = next((path for path in storage_candidates if path.exists()), storage_candidates[0])
|
||
result = {
|
||
"path": str(storage_path),
|
||
"exists": storage_path.exists(),
|
||
"status": "missing",
|
||
"reason": "storage path is missing",
|
||
}
|
||
if not storage_path.exists():
|
||
return result
|
||
|
||
model_format = str(card.get("format") or "").lower()
|
||
model_type = str(card.get("type") or "").lower()
|
||
if model_format == "diffusers" or model_type == "image-diffusion-model":
|
||
missing = [name for name in ["model_index.json"] if not (storage_path / name).exists()]
|
||
if missing:
|
||
result.update({"status": "missing", "reason": f"missing required file(s): {', '.join(missing)}"})
|
||
elif not any(storage_path.rglob("*.safetensors")) and not any(storage_path.rglob("*.bin")):
|
||
result.update({"status": "metadata-only", "reason": "diffusers weights are missing"})
|
||
else:
|
||
result.update({"status": "ok", "reason": "ready"})
|
||
return result
|
||
|
||
if model_format == "gguf":
|
||
filename = str(card.get("filename") or "")
|
||
if not filename:
|
||
result.update({"status": "failed", "reason": "GGUF filename is missing in model card"})
|
||
return result
|
||
file_path = storage_path / filename
|
||
if not file_path.exists():
|
||
result.update({"status": "missing", "reason": f"GGUF file is missing: {filename}"})
|
||
return result
|
||
size = file_path.stat().st_size
|
||
expected = card.get("file_size_bytes")
|
||
status = "ok" if not expected or size == int(expected) else "partial"
|
||
reason = "ready" if status == "ok" else f"size mismatch: {size} != {expected}"
|
||
result.update({"status": status, "reason": reason, "size_bytes": size, "expected_size_bytes": expected})
|
||
return result
|
||
|
||
if model_type == "lora-adapter":
|
||
ready = has_any_file(storage_path, ["adapter_config.json", "*.safetensors", "*.bin"])
|
||
result.update({"status": "ok" if ready else "metadata-only", "reason": "ready" if ready else "adapter artifact files are missing"})
|
||
return result
|
||
|
||
missing = [name for name in ["config.json"] if not (storage_path / name).exists()]
|
||
index_path = storage_path / "model.safetensors.index.json"
|
||
missing_shards: list[str] = []
|
||
if index_path.exists():
|
||
try:
|
||
index = json.loads(index_path.read_text(encoding="utf-8"))
|
||
shard_names = sorted(set((index.get("weight_map") or {}).values()))
|
||
missing_shards = [name for name in shard_names if not (storage_path / name).exists()]
|
||
except json.JSONDecodeError:
|
||
result.update({"status": "failed", "reason": "model.safetensors.index.json is invalid"})
|
||
return result
|
||
has_weights = has_weight_file(storage_path)
|
||
has_tokenizer = has_any_file(storage_path, ["tokenizer.json", "tokenizer.model", "vocab.json"])
|
||
if missing:
|
||
result.update({"status": "missing", "reason": f"missing required file(s): {', '.join(missing)}"})
|
||
elif missing_shards:
|
||
preview = ", ".join(missing_shards[:4])
|
||
suffix = f" and {len(missing_shards) - 4} more" if len(missing_shards) > 4 else ""
|
||
result.update({"status": "partial", "reason": f"missing shard file(s): {preview}{suffix}"})
|
||
elif not has_weights:
|
||
result.update({"status": "metadata-only", "reason": "model weights are missing"})
|
||
elif not has_tokenizer:
|
||
result.update({"status": "partial", "reason": "tokenizer files are missing"})
|
||
else:
|
||
result.update({"status": "ok", "reason": "ready"})
|
||
return result
|
||
|
||
|
||
def service_plan_for_model(model: dict) -> dict:
|
||
runtime = str(model.get("runtime") or "")
|
||
plugins = model.get("plugins") or []
|
||
key = runtime
|
||
if runtime == "transformers":
|
||
for plugin in ("translation", "audio", "video", "image"):
|
||
if plugin in plugins:
|
||
key = f"transformers:{plugin}"
|
||
break
|
||
plan = dict(SERVICE_PLANS.get(key) or {})
|
||
if not plan:
|
||
plan = {
|
||
"service_id": runtime or "unknown",
|
||
"label": runtime or "unknown runtime",
|
||
"base_url": DEFAULT_BASE_URL,
|
||
"container_name": None,
|
||
"compose": None,
|
||
"deploy_script": None,
|
||
"health_path": "/health",
|
||
}
|
||
return plan
|
||
|
||
|
||
def service_status_for_model(model: dict, endpoint_models: dict[str, list[str]]) -> dict:
|
||
plan = service_plan_for_model(model)
|
||
storage = storage_status_for_model(model)
|
||
served_name = str(model.get("served_model_name") or model.get("id") or "")
|
||
base_url = str(plan.get("base_url") or DEFAULT_BASE_URL)
|
||
models = endpoint_models.get(base_url, [])
|
||
online = served_name in models
|
||
status = "online" if online else "ready_to_start" if storage["status"] == "ok" else "blocked"
|
||
return {
|
||
"status": status,
|
||
"online": online,
|
||
"storage": storage,
|
||
"service": plan,
|
||
"served_model_name": served_name,
|
||
"endpoint_models": models,
|
||
}
|
||
|
||
|
||
def service_health_for_model(model: dict, endpoint_health: dict[str, dict]) -> dict:
|
||
plan = service_plan_for_model(model)
|
||
base_url = str(plan.get("base_url") or DEFAULT_BASE_URL)
|
||
return endpoint_health.get(base_url, {})
|
||
|
||
|
||
def model_readiness(model: dict, endpoint_models: dict[str, list[str]]) -> dict:
|
||
served_name = str(model.get("served_model_name") or model.get("id") or "")
|
||
plan = service_plan_for_model(model)
|
||
base_url = str(plan.get("base_url") or DEFAULT_BASE_URL)
|
||
available = served_name in endpoint_models.get(base_url, [])
|
||
return {"base_url": base_url, "served_model_name": served_name, "available": available, "service": plan}
|
||
|
||
|
||
def route_for_plugin(plugin: str, task: str | None = None) -> dict:
|
||
catalog = load_catalog()
|
||
models = [model for model in catalog["models"] if plugin in model.get("plugins", [])]
|
||
if task:
|
||
task_matches = [model for model in models if task in model.get("tasks", [])]
|
||
if task_matches:
|
||
models = task_matches
|
||
models.sort(key=lambda model: plugin_model_score(model, plugin))
|
||
selected = models[0] if models else None
|
||
if not selected:
|
||
raise ValueError(f"no model route for plugin: {plugin}")
|
||
|
||
endpoint = model_readiness(selected, {})
|
||
try:
|
||
runtime_data = load_runtime_profiles()
|
||
default_profile = runtime_data["profiles"].get(runtime_data["default"])
|
||
if default_profile:
|
||
target = runtime_profile_target(default_profile, model=selected, model_id=str(selected["id"]), plugin=plugin)
|
||
endpoint = {
|
||
**endpoint,
|
||
"base_url": target["base_url"],
|
||
"served_model_name": target["served_model_name"],
|
||
}
|
||
except Exception:
|
||
pass
|
||
return {
|
||
"plugin": plugin,
|
||
"task": task,
|
||
"model": selected,
|
||
"base_url": endpoint["base_url"],
|
||
"served_model_name": endpoint["served_model_name"],
|
||
"service": endpoint["service"],
|
||
"reason": f"Selected by plugin={plugin}, status={selected.get('status')}, runtime={selected.get('runtime')}",
|
||
}
|
||
|
||
|
||
def route_for_model(plugin: str, model_id: str, task: str | None = None) -> dict:
|
||
if not model_id:
|
||
return route_for_plugin(plugin, task)
|
||
catalog = load_catalog()
|
||
selected = next(
|
||
(
|
||
model
|
||
for model in catalog["models"]
|
||
if model.get("id") == model_id and plugin in model.get("plugins", [])
|
||
),
|
||
None,
|
||
)
|
||
if not selected:
|
||
return route_for_plugin(plugin, task)
|
||
|
||
endpoint = model_readiness(selected, {})
|
||
try:
|
||
runtime_data = load_runtime_profiles()
|
||
default_profile = runtime_data["profiles"].get(runtime_data["default"])
|
||
if default_profile:
|
||
target = runtime_profile_target(default_profile, model=selected, model_id=str(selected["id"]), plugin=plugin)
|
||
endpoint = {
|
||
**endpoint,
|
||
"base_url": target["base_url"],
|
||
"served_model_name": target["served_model_name"],
|
||
}
|
||
except Exception:
|
||
pass
|
||
return {
|
||
"plugin": plugin,
|
||
"task": task,
|
||
"model": selected,
|
||
"base_url": endpoint["base_url"],
|
||
"served_model_name": endpoint["served_model_name"],
|
||
"service": endpoint["service"],
|
||
"reason": f"Selected by model_id={model_id}, status={selected.get('status')}, runtime={selected.get('runtime')}",
|
||
}
|
||
|
||
|
||
def run_command(command: list[str], *, timeout: int = 60) -> dict:
|
||
started_at = time.perf_counter()
|
||
try:
|
||
result = subprocess.run(
|
||
command,
|
||
cwd=ROOT,
|
||
text=True,
|
||
capture_output=True,
|
||
timeout=timeout,
|
||
check=False,
|
||
)
|
||
return {
|
||
"command": command,
|
||
"returncode": result.returncode,
|
||
"stdout": result.stdout.strip(),
|
||
"stderr": result.stderr.strip(),
|
||
"latency_ms": round((time.perf_counter() - started_at) * 1000),
|
||
}
|
||
except FileNotFoundError as exc:
|
||
return {
|
||
"command": command,
|
||
"returncode": 127,
|
||
"stdout": "",
|
||
"stderr": str(exc),
|
||
"latency_ms": round((time.perf_counter() - started_at) * 1000),
|
||
}
|
||
except subprocess.TimeoutExpired as exc:
|
||
return {
|
||
"command": command,
|
||
"returncode": 124,
|
||
"stdout": (exc.stdout or "").strip() if isinstance(exc.stdout, str) else "",
|
||
"stderr": (exc.stderr or "").strip() if isinstance(exc.stderr, str) else f"timeout after {timeout}s",
|
||
"latency_ms": round((time.perf_counter() - started_at) * 1000),
|
||
}
|
||
|
||
|
||
def docker_compose_base_args(plan: dict, *, docker_host: str | None = None) -> list[str]:
|
||
compose = plan.get("compose")
|
||
if not compose:
|
||
raise ValueError("service has no compose file")
|
||
compose_path = ROOT / str(compose)
|
||
if not compose_path.exists():
|
||
raise ValueError(f"compose file is missing: {compose}")
|
||
args = ["docker"]
|
||
host = docker_host or DEFAULT_DOCKER_HOST
|
||
if host:
|
||
args.extend(["--host", host])
|
||
return [*args, "compose", "-f", str(compose_path)]
|
||
|
||
|
||
def control_model_service(payload: dict) -> dict:
|
||
action = str(payload.get("action") or "").strip().lower()
|
||
model_id = str(payload.get("model_id") or "").strip()
|
||
if action not in SERVICE_CONTROL_ACTIONS:
|
||
raise ValueError(f"action must be one of: {', '.join(sorted(SERVICE_CONTROL_ACTIONS))}")
|
||
if not model_id:
|
||
raise ValueError("model_id is required")
|
||
|
||
catalog = load_catalog()
|
||
model = next((item for item in catalog["models"] if item.get("id") == model_id), None)
|
||
if not model:
|
||
raise ValueError(f"unknown model_id: {model_id}")
|
||
|
||
plan = service_plan_for_model(model)
|
||
docker_args = ["docker"]
|
||
docker_host = payload.get("docker_host") or DEFAULT_DOCKER_HOST
|
||
if docker_host:
|
||
docker_args.extend(["--host", str(docker_host)])
|
||
if action == "start":
|
||
container_name = str(plan.get("container_name") or "")
|
||
if container_name:
|
||
existing = run_command(
|
||
[*docker_args, "ps", "-a", "--filter", f"name=^{container_name}$", "--format", "{{.Names}}"],
|
||
timeout=30,
|
||
)
|
||
if existing["returncode"] == 0 and container_name in existing["stdout"].splitlines():
|
||
command = [*docker_args, "start", container_name]
|
||
else:
|
||
base_args = docker_compose_base_args(plan, docker_host=str(docker_host) if docker_host else None)
|
||
command = [*base_args, "up", "-d"]
|
||
else:
|
||
base_args = docker_compose_base_args(plan, docker_host=str(docker_host) if docker_host else None)
|
||
command = [*base_args, "up", "-d"]
|
||
timeout = 120
|
||
elif action == "stop":
|
||
container_name = str(plan.get("container_name") or "")
|
||
if not container_name:
|
||
raise ValueError("service has no container_name")
|
||
command = [*docker_args, "stop", container_name]
|
||
timeout = 90
|
||
elif action == "restart":
|
||
container_name = str(plan.get("container_name") or "")
|
||
if not container_name:
|
||
raise ValueError("service has no container_name")
|
||
command = [*docker_args, "restart", container_name]
|
||
timeout = 120
|
||
else:
|
||
container_name = str(plan.get("container_name") or "")
|
||
if container_name:
|
||
command = [*docker_args, "ps", "-a", "--filter", f"name=^{container_name}$"]
|
||
else:
|
||
base_args = docker_compose_base_args(plan, docker_host=str(docker_host) if docker_host else None)
|
||
command = [*base_args, "ps"]
|
||
timeout = 60
|
||
|
||
result = run_command(command, timeout=timeout)
|
||
report_id = append_report(
|
||
{
|
||
"type": "service_control",
|
||
"model_id": model_id,
|
||
"model": model.get("name") or model_id,
|
||
"action": action,
|
||
"service": plan,
|
||
"result": result,
|
||
}
|
||
)
|
||
return {
|
||
"ok": result["returncode"] == 0,
|
||
"report_id": report_id,
|
||
"model_id": model_id,
|
||
"action": action,
|
||
"service": plan,
|
||
"result": result,
|
||
}
|
||
|
||
|
||
def run_runtime_benchmark(payload: dict) -> dict:
|
||
model_id = sanitize_id(str(payload.get("model_id") or "qwen3-coder-30b-a3b-instruct-q6_k"))
|
||
plugin = sanitize_id(str(payload.get("plugin") or "1c"))
|
||
profiles = payload.get("profiles") or ["gpu-fast", "cpu-test"]
|
||
if not isinstance(profiles, list) or not profiles:
|
||
raise ValueError("profiles must be a non-empty list")
|
||
safe_profiles = [sanitize_id(str(profile), fallback="profile") for profile in profiles]
|
||
max_tokens = int(payload.get("max_tokens") or 384)
|
||
temperature = float(payload.get("temperature") or 0.1)
|
||
prompt = str(payload.get("prompt") or DEFAULT_PROMPTS.get(plugin) or DEFAULT_PROMPTS["1c"]).strip()
|
||
if not prompt:
|
||
raise ValueError("prompt is required")
|
||
|
||
timestamp = dt.datetime.now(dt.UTC).strftime("%Y%m%dT%H%M%SZ")
|
||
report_path = REPORT_ROOT / "benchmarks" / f"runtime-profiles-{model_id}-{timestamp}.json"
|
||
command = [
|
||
sys.executable,
|
||
str(ROOT / "scripts" / "benchmark_runtime_profiles.py"),
|
||
"--model-id",
|
||
model_id,
|
||
"--plugin",
|
||
plugin,
|
||
"--max-tokens",
|
||
str(max_tokens),
|
||
"--temperature",
|
||
str(temperature),
|
||
"--timeout",
|
||
str(int(payload.get("timeout") or 600)),
|
||
"--report",
|
||
str(report_path),
|
||
"--profiles",
|
||
*safe_profiles,
|
||
]
|
||
if prompt:
|
||
command.extend(["--prompt", prompt])
|
||
|
||
result = run_command(command, timeout=int(payload.get("command_timeout") or 900))
|
||
report = {}
|
||
if report_path.exists():
|
||
try:
|
||
report = read_json(report_path)
|
||
except (OSError, json.JSONDecodeError, ValueError) as exc:
|
||
report = {"status": "error", "error": f"failed to read report: {exc}"}
|
||
report_id = append_report(
|
||
{
|
||
"type": "runtime_benchmark",
|
||
"model_id": model_id,
|
||
"plugin": plugin,
|
||
"profiles": safe_profiles,
|
||
"report_path": display_path(report_path),
|
||
"result": result,
|
||
"report": report,
|
||
}
|
||
)
|
||
return {
|
||
"ok": result["returncode"] == 0 and bool(report),
|
||
"report_id": report_id,
|
||
"report_path": display_path(report_path),
|
||
"result": result,
|
||
"benchmark": report,
|
||
}
|
||
|
||
|
||
def runtime_profile_target(profile: dict, *, model: dict, model_id: str, plugin: str) -> dict:
|
||
override = (profile.get("model_overrides") or {}).get(model_id) or {}
|
||
base_url = override.get("base_url") or (profile.get("endpoints") or {}).get(plugin)
|
||
served_model_name = override.get("served_model_name") or model.get("served_model_name") or model_id
|
||
if not base_url:
|
||
raise ValueError(f"profile `{profile.get('id')}` has no endpoint for plugin `{plugin}`")
|
||
return {
|
||
"base_url": normalize_base_url(str(base_url)),
|
||
"served_model_name": str(served_model_name),
|
||
"host": profile.get("host"),
|
||
"role": profile.get("role"),
|
||
"container_name": override.get("container_name"),
|
||
}
|
||
|
||
|
||
def runtime_benchmark_preflight(payload: dict) -> dict:
|
||
model_id = sanitize_id(str(payload.get("model_id") or "qwen3-coder-30b-a3b-instruct-q6_k"))
|
||
plugin = sanitize_id(str(payload.get("plugin") or "1c"))
|
||
profiles = payload.get("profiles") or ["gpu-fast", "cpu-test"]
|
||
if not isinstance(profiles, list) or not profiles:
|
||
raise ValueError("profiles must be a non-empty list")
|
||
catalog = load_catalog()
|
||
model = next((item for item in catalog["models"] if item.get("id") == model_id), None)
|
||
if not model:
|
||
raise ValueError(f"unknown model_id: {model_id}")
|
||
runtime_profiles = load_runtime_profiles()["profiles"]
|
||
checks = []
|
||
started_at = time.perf_counter()
|
||
for profile_id_raw in profiles:
|
||
profile_id = sanitize_id(str(profile_id_raw), fallback="profile")
|
||
profile = runtime_profiles.get(profile_id)
|
||
if not profile:
|
||
checks.append({"profile_id": profile_id, "status": "error", "error": "unknown runtime profile"})
|
||
continue
|
||
try:
|
||
target = runtime_profile_target(profile, model=model, model_id=model_id, plugin=plugin)
|
||
checked_at = time.perf_counter()
|
||
models = fetch_endpoint_models(target["base_url"], timeout=int(payload.get("timeout") or 20))
|
||
latency_ms = round((time.perf_counter() - checked_at) * 1000)
|
||
available = target["served_model_name"] in models
|
||
checks.append(
|
||
{
|
||
"profile_id": profile_id,
|
||
"label": profile.get("label") or profile_id,
|
||
"status": "ok" if available else "missing_model",
|
||
"available": available,
|
||
"latency_ms": latency_ms,
|
||
"target": target,
|
||
"models": models,
|
||
}
|
||
)
|
||
except (ValueError, urllib.error.URLError, json.JSONDecodeError) as exc:
|
||
checks.append(
|
||
{
|
||
"profile_id": profile_id,
|
||
"label": profile.get("label") or profile_id,
|
||
"status": "error",
|
||
"available": False,
|
||
"error": str(exc),
|
||
}
|
||
)
|
||
ready = bool(checks) and all(item.get("available") for item in checks)
|
||
return {
|
||
"status": "ready" if ready else "blocked",
|
||
"ready": ready,
|
||
"model_id": model_id,
|
||
"plugin": plugin,
|
||
"latency_ms": round((time.perf_counter() - started_at) * 1000),
|
||
"checks": checks,
|
||
}
|
||
|
||
|
||
def latest_runtime_benchmarks(limit: int = 10, *, model_id: str | None = None, plugin: str | None = None) -> list[dict]:
|
||
benchmark_dir = REPORT_ROOT / "benchmarks"
|
||
if not benchmark_dir.exists():
|
||
return []
|
||
records = []
|
||
for path in sorted(benchmark_dir.glob("runtime-profiles-*.json"), key=lambda item: item.stat().st_mtime, reverse=True):
|
||
try:
|
||
report = read_json(path)
|
||
except (OSError, json.JSONDecodeError, ValueError):
|
||
continue
|
||
if not isinstance(report, dict):
|
||
continue
|
||
if model_id and report.get("model_id") != model_id:
|
||
continue
|
||
if plugin and report.get("plugin") != plugin:
|
||
continue
|
||
summary_results = []
|
||
for row in report.get("results") or []:
|
||
if not isinstance(row, dict):
|
||
continue
|
||
target = row.get("target") or {}
|
||
summary_results.append(
|
||
{
|
||
"profile_id": row.get("profile_id"),
|
||
"status": row.get("status"),
|
||
"elapsed_sec": row.get("elapsed_sec"),
|
||
"output_tokens_per_sec": row.get("output_tokens_per_sec"),
|
||
"base_url": target.get("base_url"),
|
||
"served_model_name": target.get("served_model_name"),
|
||
"error": row.get("error"),
|
||
}
|
||
)
|
||
records.append(
|
||
{
|
||
"path": display_path(path),
|
||
"updated_at": dt.datetime.fromtimestamp(path.stat().st_mtime, dt.UTC).isoformat(),
|
||
"created_at": report.get("created_at"),
|
||
"status": report.get("status"),
|
||
"model_id": report.get("model_id"),
|
||
"plugin": report.get("plugin"),
|
||
"fastest_profile_id": report.get("fastest_profile_id"),
|
||
"speedup": report.get("speedup") or {},
|
||
"results": summary_results,
|
||
}
|
||
)
|
||
if len(records) >= limit:
|
||
break
|
||
return records
|
||
|
||
|
||
def benchmark_report_path(name: str) -> Path:
|
||
filename = Path(urllib.parse.unquote(name or "")).name
|
||
if not re.fullmatch(r"runtime-profiles-[A-Za-z0-9_.-]+\.json", filename):
|
||
raise ValueError("invalid benchmark report name")
|
||
path = (REPORT_ROOT / "benchmarks" / filename).resolve()
|
||
benchmark_root = (REPORT_ROOT / "benchmarks").resolve()
|
||
if benchmark_root not in path.parents or not path.exists():
|
||
raise FileNotFoundError(filename)
|
||
return path
|
||
|
||
|
||
def render_benchmark_markdown(report: dict, source_name: str) -> str:
|
||
title = report.get("model_id") or source_name
|
||
created = report.get("created_at") or "-"
|
||
plugin = report.get("plugin") or "-"
|
||
max_tokens = report.get("max_tokens") or "-"
|
||
fastest = report.get("fastest_profile_id") or "-"
|
||
ratio = (report.get("speedup") or {}).get("gpu_vs_cpu_ratio")
|
||
lines = [
|
||
f"# Runtime Benchmark: {title}",
|
||
"",
|
||
f"- Created: `{created}`",
|
||
f"- Plugin: `{plugin}`",
|
||
f"- Max tokens: `{max_tokens}`",
|
||
f"- Fastest profile: `{fastest}`",
|
||
]
|
||
if ratio:
|
||
lines.append(f"- GPU/CPU ratio: `x{ratio}`")
|
||
lines.extend(
|
||
[
|
||
"",
|
||
"| Profile | Host | Endpoint | Served model | Elapsed | Output speed | Result |",
|
||
"|---|---|---|---|---:|---:|---|",
|
||
]
|
||
)
|
||
for row in report.get("results") or []:
|
||
target = row.get("target") or {}
|
||
status = row.get("status") or "-"
|
||
elapsed = f"{row.get('elapsed_sec')}s" if row.get("elapsed_sec") is not None else "-"
|
||
speed = f"{row.get('output_tokens_per_sec')} tok/s" if row.get("output_tokens_per_sec") is not None else "-"
|
||
result = "fastest" if row.get("profile_id") == fastest else status
|
||
if status != "ok" and row.get("error"):
|
||
result = f"error: {row.get('error')}"
|
||
lines.append(
|
||
"| "
|
||
+ " | ".join(
|
||
[
|
||
f"`{row.get('profile_id') or '-'}`",
|
||
f"`{target.get('host') or '-'}`",
|
||
f"`{target.get('base_url') or '-'}`",
|
||
f"`{target.get('served_model_name') or '-'}`",
|
||
elapsed,
|
||
speed,
|
||
result,
|
||
]
|
||
)
|
||
+ " |"
|
||
)
|
||
lines.extend(["", "Raw JSON:", "", f"`{source_name}`", ""])
|
||
return "\n".join(lines)
|
||
|
||
|
||
def gpu_status() -> dict:
|
||
query = "index,name,memory.used,memory.total,utilization.gpu"
|
||
result = run_command(
|
||
["nvidia-smi", f"--query-gpu={query}", "--format=csv,noheader,nounits"],
|
||
timeout=5,
|
||
)
|
||
gpus = []
|
||
if result["returncode"] == 0:
|
||
for line in result["stdout"].splitlines():
|
||
parts = [part.strip() for part in line.split(",")]
|
||
if len(parts) != 5:
|
||
continue
|
||
try:
|
||
used = int(parts[2])
|
||
total = int(parts[3])
|
||
utilization = int(parts[4])
|
||
except ValueError:
|
||
continue
|
||
gpus.append(
|
||
{
|
||
"index": parts[0],
|
||
"name": parts[1],
|
||
"memory_used_mib": used,
|
||
"memory_total_mib": total,
|
||
"memory_free_mib": total - used,
|
||
"memory_used_percent": round((used / total) * 100, 1) if total else None,
|
||
"utilization_gpu_percent": utilization,
|
||
}
|
||
)
|
||
summary = "unavailable"
|
||
if gpus:
|
||
summary = " · ".join(
|
||
f"GPU{gpu['index']} {gpu['memory_free_mib']}/{gpu['memory_total_mib']} MiB free, {gpu['utilization_gpu_percent']}%"
|
||
for gpu in gpus
|
||
)
|
||
return {"available": True, "gpus": gpus, "summary": summary, "probe": result}
|
||
|
||
for base_url in ("http://docker-gpu.cin.su:8040", "http://docker-gpu.cin.su:8020", "http://docker-gpu.cin.su:8010"):
|
||
try:
|
||
request = urllib.request.Request(f"{base_url}/health", method="GET")
|
||
with urllib.request.urlopen(request, timeout=3) as response:
|
||
data = json.loads(response.read().decode("utf-8"))
|
||
gpu = data.get("gpu") if isinstance(data, dict) else None
|
||
if isinstance(gpu, dict) and gpu.get("available"):
|
||
return {
|
||
"available": True,
|
||
"gpus": gpu.get("devices") or [],
|
||
"summary": gpu.get("summary") or "cuda available",
|
||
"source": f"{base_url}/health",
|
||
"probe": result,
|
||
}
|
||
except Exception:
|
||
continue
|
||
return {"available": False, "gpus": [], "summary": summary, "probe": result}
|
||
|
||
|
||
def service_control_status() -> dict:
|
||
docker_path = shutil.which("docker")
|
||
return {
|
||
"available": bool(docker_path),
|
||
"docker_path": docker_path,
|
||
"docker_host": DEFAULT_DOCKER_HOST,
|
||
"actions": sorted(SERVICE_CONTROL_ACTIONS),
|
||
"fallback": "scripts/manage_model_service.ps1",
|
||
}
|
||
|
||
|
||
def gpu_profile_status(endpoint_statuses: list[dict]) -> dict[str, dict]:
|
||
endpoint_by_id = {str(item.get("id")): item for item in endpoint_statuses}
|
||
endpoint_by_id["model-chat-ui"] = {"id": "model-chat-ui", "status": "ok"}
|
||
profiles = {}
|
||
for profile_id, profile in load_gpu_profiles().items():
|
||
starts = list(profile.get("starts") or [])
|
||
stops = list(profile.get("stops") or [])
|
||
missing = [service for service in starts if endpoint_by_id.get(service, {}).get("status") != "ok"]
|
||
conflicts = [service for service in stops if endpoint_by_id.get(service, {}).get("status") == "ok"]
|
||
profiles[profile_id] = {
|
||
"id": profile_id,
|
||
"ready": not missing and not conflicts,
|
||
"missing": missing,
|
||
"conflicts": conflicts,
|
||
"starts": starts,
|
||
"stops": stops,
|
||
}
|
||
return profiles
|
||
|
||
|
||
def latest_model_chat_preflight() -> dict:
|
||
if not MODEL_CHAT_PREFLIGHT_REPORT.exists():
|
||
return {"status": "missing", "report_path": display_path(MODEL_CHAT_PREFLIGHT_REPORT)}
|
||
try:
|
||
report = read_json(MODEL_CHAT_PREFLIGHT_REPORT)
|
||
except (OSError, ValueError, json.JSONDecodeError) as exc:
|
||
return {
|
||
"status": "failed",
|
||
"report_path": display_path(MODEL_CHAT_PREFLIGHT_REPORT),
|
||
"error": str(exc),
|
||
}
|
||
|
||
steps = report.get("steps") if isinstance(report.get("steps"), list) else []
|
||
failed_steps = [
|
||
str(step.get("name") or "unknown")
|
||
for step in steps
|
||
if isinstance(step, dict) and step.get("status") != "ok"
|
||
]
|
||
return {
|
||
"status": str(report.get("status") or "unknown"),
|
||
"created_at": report.get("created_at"),
|
||
"report_path": display_path(MODEL_CHAT_PREFLIGHT_REPORT),
|
||
"step_count": len(steps),
|
||
"failed_steps": failed_steps,
|
||
}
|
||
|
||
|
||
def health_status() -> dict:
|
||
endpoint_statuses = []
|
||
endpoint_models: dict[str, list[str]] = {}
|
||
endpoint_health: dict[str, dict] = {}
|
||
health_detail_urls = {
|
||
str(plan.get("base_url") or "")
|
||
for plan in SERVICE_PLANS.values()
|
||
if plan.get("health_path") == "/health"
|
||
}
|
||
for preset in ENDPOINT_PRESETS:
|
||
base_url = preset["base_url"]
|
||
started_at = time.perf_counter()
|
||
try:
|
||
models = fetch_endpoint_models(base_url, timeout=3)
|
||
endpoint_models[base_url] = models
|
||
detail = fetch_endpoint_health(base_url, timeout=2) if base_url in health_detail_urls else {}
|
||
if detail:
|
||
endpoint_health[base_url] = detail
|
||
endpoint_statuses.append(
|
||
{
|
||
"id": preset["id"],
|
||
"label": preset["label"],
|
||
"base_url": base_url,
|
||
"status": "ok",
|
||
"models": models,
|
||
"health": detail,
|
||
"latency_ms": round((time.perf_counter() - started_at) * 1000),
|
||
}
|
||
)
|
||
except Exception as exc: # noqa: BLE001 - health endpoint must report all failures.
|
||
endpoint_statuses.append(
|
||
{
|
||
"id": preset["id"],
|
||
"label": preset["label"],
|
||
"base_url": base_url,
|
||
"status": "error",
|
||
"error": str(exc),
|
||
"latency_ms": round((time.perf_counter() - started_at) * 1000),
|
||
}
|
||
)
|
||
|
||
catalog = load_catalog()
|
||
routes = []
|
||
for plugin in PLUGIN_ORDER:
|
||
try:
|
||
route = route_for_plugin(plugin)
|
||
route["readiness"] = service_status_for_model(route["model"], endpoint_models)
|
||
route["service_health"] = service_health_for_model(route["model"], endpoint_health)
|
||
route["available"] = route["readiness"]["online"]
|
||
routes.append(route)
|
||
except ValueError as exc:
|
||
routes.append({"plugin": plugin, "status": "error", "error": str(exc)})
|
||
|
||
model_services = []
|
||
for model in catalog["models"]:
|
||
model_services.append(
|
||
{
|
||
"id": model["id"],
|
||
"name": model["name"],
|
||
"plugins": model.get("plugins") or [],
|
||
"runtime": model.get("runtime"),
|
||
"served_model_name": model.get("served_model_name"),
|
||
"service_health": service_health_for_model(model, endpoint_health),
|
||
**service_status_for_model(model, endpoint_models),
|
||
}
|
||
)
|
||
|
||
return {
|
||
"status": "ok" if any(item["status"] == "ok" for item in endpoint_statuses) else "degraded",
|
||
"host": socket.gethostname(),
|
||
"time": now_iso(),
|
||
"registry_models": len(catalog["models"]),
|
||
"plugins": len(catalog["plugins"]),
|
||
"rag_1c_index": {
|
||
"path": str(DEFAULT_1C_RAG_INDEX.relative_to(ROOT)),
|
||
"exists": DEFAULT_1C_RAG_INDEX.exists(),
|
||
},
|
||
"model_ingest_jobs": latest_model_ingest_jobs(limit=10),
|
||
"gpu": gpu_status(),
|
||
"service_control": service_control_status(),
|
||
"model_chat_preflight": latest_model_chat_preflight(),
|
||
"endpoints": endpoint_statuses,
|
||
"gpu_profile_status": gpu_profile_status(endpoint_statuses),
|
||
"routes": routes,
|
||
"model_services": model_services,
|
||
}
|
||
|
||
|
||
def append_report(record: dict) -> str:
|
||
report_id = record.get("id") or str(uuid.uuid4())
|
||
record["id"] = report_id
|
||
record.setdefault("created_at", dt.datetime.now(dt.UTC).isoformat())
|
||
REPORT_DIR.mkdir(parents=True, exist_ok=True)
|
||
path = REPORT_DIR / f"{dt.datetime.now().strftime('%Y%m%d')}.jsonl"
|
||
with path.open("a", encoding="utf-8") as handle:
|
||
handle.write(json.dumps(record, ensure_ascii=False) + "\n")
|
||
return report_id
|
||
|
||
|
||
def fetch_endpoint_models(base_url: str, timeout: int = 20) -> list[str]:
|
||
request = urllib.request.Request(f"{base_url.rstrip('/')}/v1/models", method="GET")
|
||
with urllib.request.urlopen(request, timeout=timeout) as response:
|
||
data = json.loads(response.read().decode("utf-8"))
|
||
model_rows = data.get("data") if isinstance(data, dict) else []
|
||
if not isinstance(model_rows, list):
|
||
return []
|
||
return [str(row.get("id")) for row in model_rows if isinstance(row, dict) and row.get("id")]
|
||
|
||
|
||
def fetch_endpoint_health(base_url: str, timeout: int = 3) -> dict:
|
||
request = urllib.request.Request(f"{base_url.rstrip('/')}/health", method="GET")
|
||
try:
|
||
with urllib.request.urlopen(request, timeout=timeout) as response:
|
||
data = json.loads(response.read().decode("utf-8"))
|
||
except Exception as exc: # noqa: BLE001 - health details are optional.
|
||
return {"status": "error", "error": str(exc)}
|
||
return data if isinstance(data, dict) else {}
|
||
|
||
|
||
def allowed_endpoint_hosts() -> set[tuple[str, str, int]]:
|
||
allowed = set(ALLOWED_ENDPOINT_HOSTS)
|
||
try:
|
||
runtime_profiles = load_runtime_profiles()["profiles"]
|
||
except Exception:
|
||
runtime_profiles = {}
|
||
for profile in runtime_profiles.values():
|
||
urls = list((profile.get("endpoints") or {}).values())
|
||
for override in (profile.get("model_overrides") or {}).values():
|
||
if isinstance(override, dict) and override.get("base_url"):
|
||
urls.append(str(override["base_url"]))
|
||
for url in urls:
|
||
parsed = urlparse(str(url).strip())
|
||
if parsed.scheme in {"http", "https"} and parsed.hostname:
|
||
port = parsed.port or (443 if parsed.scheme == "https" else 80)
|
||
allowed.add((parsed.scheme, parsed.hostname.lower(), port))
|
||
return allowed
|
||
|
||
|
||
def normalize_base_url(base_url: str) -> str:
|
||
parsed = urlparse(str(base_url).strip())
|
||
if parsed.scheme not in {"http", "https"} or not parsed.hostname:
|
||
raise ValueError("base_url must be an http(s) URL")
|
||
port = parsed.port or (443 if parsed.scheme == "https" else 80)
|
||
endpoint = (parsed.scheme, parsed.hostname.lower(), port)
|
||
allowed_hosts = allowed_endpoint_hosts()
|
||
if endpoint not in allowed_hosts:
|
||
allowed = ", ".join(f"{scheme}://{host}:{port}" for scheme, host, port in sorted(allowed_hosts))
|
||
raise ValueError(f"base_url is not allowed. Allowed endpoints: {allowed}")
|
||
netloc = parsed.hostname
|
||
if parsed.port:
|
||
netloc = f"{netloc}:{parsed.port}"
|
||
return urlunparse((parsed.scheme, netloc, "", "", "", ""))
|
||
|
||
|
||
def build_1c_rag_prompt(question: str, *, profile_name: str = "general", limit: int | None = None) -> dict:
|
||
if not DEFAULT_1C_RAG_INDEX.exists():
|
||
raise FileNotFoundError(
|
||
f"1C RAG index is missing: {DEFAULT_1C_RAG_INDEX}. "
|
||
"Run scripts/prepare_1c_rag_corpus.py and scripts/build_1c_rag_index.py first."
|
||
)
|
||
index = read_json(DEFAULT_1C_RAG_INDEX)
|
||
profile = resolve_rag_profile(profile_name, question)
|
||
results = search_lexical_index(
|
||
index,
|
||
question,
|
||
limit=int(limit or profile["limit"]),
|
||
candidate_limit=int(profile["candidate_limit"]),
|
||
dedupe_by_document=bool(profile["dedupe_by_document"]),
|
||
min_score=float(profile["min_score"]),
|
||
source_types=profile["source_types"],
|
||
)
|
||
context = format_context(results, max_chars=int(profile["max_context_chars"]))
|
||
prompt = render_prompt(DEFAULT_1C_RAG_PROMPT, context=context, question=question)
|
||
return {
|
||
"prompt": prompt,
|
||
"profile": profile["id"],
|
||
"context_count": len(results),
|
||
"sources": [
|
||
{
|
||
"source_path": result["document"].get("source_path"),
|
||
"title": result["document"].get("title"),
|
||
"chunk_index": result["document"].get("chunk_index"),
|
||
"score": round(float(result.get("score") or 0), 4),
|
||
}
|
||
for result in results
|
||
],
|
||
}
|
||
|
||
|
||
def run_chat_request(base_url: str, model: str, messages: list[dict], temperature: float, max_tokens: int) -> dict:
|
||
started_at = time.perf_counter()
|
||
try:
|
||
answer = call_chat_completion(
|
||
base_url=base_url,
|
||
model=model,
|
||
messages=messages,
|
||
temperature=temperature,
|
||
max_tokens=max_tokens,
|
||
timeout=180,
|
||
)
|
||
return {
|
||
"status": "answered",
|
||
"answer": answer,
|
||
"error": None,
|
||
"latency_ms": round((time.perf_counter() - started_at) * 1000),
|
||
}
|
||
except (urllib.error.URLError, ValueError) as exc:
|
||
return {
|
||
"status": "error",
|
||
"answer": None,
|
||
"error": str(exc),
|
||
"latency_ms": round((time.perf_counter() - started_at) * 1000),
|
||
}
|
||
|
||
|
||
def call_audio_transcription(
|
||
*,
|
||
base_url: str,
|
||
audio_base64: str,
|
||
filename: str,
|
||
language: str | None = None,
|
||
task: str | None = None,
|
||
timeout: int = 300,
|
||
) -> dict:
|
||
url = f"{base_url.rstrip('/')}/v1/audio/transcriptions"
|
||
payload = {
|
||
"audio_base64": audio_base64,
|
||
"filename": filename,
|
||
}
|
||
if language:
|
||
payload["language"] = language
|
||
if task:
|
||
payload["task"] = task
|
||
request = urllib.request.Request(
|
||
url,
|
||
data=json.dumps(payload).encode("utf-8"),
|
||
headers={"Content-Type": "application/json"},
|
||
method="POST",
|
||
)
|
||
with urllib.request.urlopen(request, timeout=timeout) as response:
|
||
return json.loads(response.read().decode("utf-8"))
|
||
|
||
|
||
def call_vision_analysis(
|
||
*,
|
||
base_url: str,
|
||
image_base64: str,
|
||
filename: str,
|
||
prompt: str,
|
||
model: str | None = None,
|
||
max_tokens: int = 512,
|
||
timeout: int = 300,
|
||
) -> dict:
|
||
url = f"{base_url.rstrip('/')}/v1/vision/analyze"
|
||
payload = {
|
||
"image_base64": image_base64,
|
||
"filename": filename,
|
||
"prompt": prompt,
|
||
"model": model,
|
||
"max_tokens": max_tokens,
|
||
}
|
||
request = urllib.request.Request(
|
||
url,
|
||
data=json.dumps(payload, ensure_ascii=False).encode("utf-8"),
|
||
headers={"Content-Type": "application/json"},
|
||
method="POST",
|
||
)
|
||
with urllib.request.urlopen(request, timeout=timeout) as response:
|
||
data = json.loads(response.read().decode("utf-8"))
|
||
if not isinstance(data, dict):
|
||
raise ValueError("vision response must be a JSON object")
|
||
if data.get("error"):
|
||
error = data["error"]
|
||
if isinstance(error, dict):
|
||
raise ValueError(str(error.get("message") or error))
|
||
raise ValueError(str(error))
|
||
return data
|
||
|
||
|
||
def call_image_generation(
|
||
*,
|
||
base_url: str,
|
||
prompt: str,
|
||
negative_prompt: str | None = None,
|
||
width: int = 1024,
|
||
height: int = 1024,
|
||
steps: int = 28,
|
||
guidance_scale: float = 6.0,
|
||
seed: int | None = None,
|
||
timeout: int = 900,
|
||
) -> dict:
|
||
url = f"{base_url.rstrip('/')}/v1/images/generations"
|
||
payload: dict[str, object] = {
|
||
"prompt": prompt,
|
||
"negative_prompt": negative_prompt or "",
|
||
"width": width,
|
||
"height": height,
|
||
"steps": steps,
|
||
"guidance_scale": guidance_scale,
|
||
}
|
||
if seed is not None:
|
||
payload["seed"] = seed
|
||
request = urllib.request.Request(
|
||
url,
|
||
data=json.dumps(payload, ensure_ascii=False).encode("utf-8"),
|
||
headers={"Content-Type": "application/json"},
|
||
method="POST",
|
||
)
|
||
with urllib.request.urlopen(request, timeout=timeout) as response:
|
||
data = json.loads(response.read().decode("utf-8"))
|
||
if not isinstance(data, dict):
|
||
raise ValueError("image generation response must be a JSON object")
|
||
if data.get("error"):
|
||
error = data["error"]
|
||
if isinstance(error, dict):
|
||
raise ValueError(str(error.get("message") or error))
|
||
raise ValueError(str(error))
|
||
return data
|
||
|
||
|
||
def call_image_edit(
|
||
*,
|
||
base_url: str,
|
||
prompt: str,
|
||
image_base64: str,
|
||
mask_base64: str,
|
||
negative_prompt: str | None = None,
|
||
width: int = 1024,
|
||
height: int = 1024,
|
||
steps: int = 28,
|
||
guidance_scale: float = 6.0,
|
||
strength: float = 0.95,
|
||
seed: int | None = None,
|
||
timeout: int = 900,
|
||
) -> dict:
|
||
url = f"{base_url.rstrip('/')}/v1/images/edits"
|
||
payload: dict[str, object] = {
|
||
"prompt": prompt,
|
||
"negative_prompt": negative_prompt or "",
|
||
"image_base64": image_base64,
|
||
"mask_base64": mask_base64,
|
||
"width": width,
|
||
"height": height,
|
||
"steps": steps,
|
||
"guidance_scale": guidance_scale,
|
||
"strength": strength,
|
||
}
|
||
if seed is not None:
|
||
payload["seed"] = seed
|
||
request = urllib.request.Request(
|
||
url,
|
||
data=json.dumps(payload, ensure_ascii=False).encode("utf-8"),
|
||
headers={"Content-Type": "application/json"},
|
||
method="POST",
|
||
)
|
||
with urllib.request.urlopen(request, timeout=timeout) as response:
|
||
data = json.loads(response.read().decode("utf-8"))
|
||
if not isinstance(data, dict):
|
||
raise ValueError("image edit response must be a JSON object")
|
||
if data.get("error"):
|
||
error = data["error"]
|
||
if isinstance(error, dict):
|
||
raise ValueError(str(error.get("message") or error))
|
||
raise ValueError(str(error))
|
||
return data
|
||
|
||
|
||
def submit_image_job(*, base_url: str, operation: str, payload: dict, job_id: str, timeout: int = 30) -> dict:
|
||
url = f"{base_url.rstrip('/')}/v1/images/jobs"
|
||
request_payload = {"operation": operation, "payload": payload, "job_id": job_id}
|
||
request = urllib.request.Request(
|
||
url,
|
||
data=json.dumps(request_payload, ensure_ascii=False).encode("utf-8"),
|
||
headers={"Content-Type": "application/json"},
|
||
method="POST",
|
||
)
|
||
with urllib.request.urlopen(request, timeout=timeout) as response:
|
||
data = json.loads(response.read().decode("utf-8"))
|
||
if not isinstance(data, dict):
|
||
raise ValueError("image job response must be a JSON object")
|
||
if data.get("error"):
|
||
error = data["error"]
|
||
if isinstance(error, dict):
|
||
raise ValueError(str(error.get("message") or error))
|
||
raise ValueError(str(error))
|
||
return data
|
||
|
||
|
||
def fetch_image_job(*, base_url: str, job_id: str, timeout: int = 30) -> dict:
|
||
url = f"{base_url.rstrip('/')}/v1/images/jobs/{urllib.parse.quote(job_id)}"
|
||
with urllib.request.urlopen(url, timeout=timeout) as response:
|
||
data = json.loads(response.read().decode("utf-8"))
|
||
if not isinstance(data, dict):
|
||
raise ValueError("image job status must be a JSON object")
|
||
if data.get("error"):
|
||
error = data["error"]
|
||
if isinstance(error, dict):
|
||
raise ValueError(str(error.get("message") or error))
|
||
raise ValueError(str(error))
|
||
return data
|
||
|
||
|
||
def cancel_image_job(*, base_url: str, job_id: str, timeout: int = 30) -> dict:
|
||
url = f"{base_url.rstrip('/')}/v1/images/jobs/{urllib.parse.quote(job_id)}/cancel"
|
||
request = urllib.request.Request(
|
||
url,
|
||
data=b"{}",
|
||
headers={"Content-Type": "application/json"},
|
||
method="POST",
|
||
)
|
||
with urllib.request.urlopen(request, timeout=timeout) as response:
|
||
data = json.loads(response.read().decode("utf-8"))
|
||
if not isinstance(data, dict):
|
||
raise ValueError("image cancel response must be a JSON object")
|
||
return data
|
||
|
||
|
||
def finalize_image_job(job: dict) -> dict:
|
||
job_id = str(job.get("id") or "")
|
||
if not job_id or job.get("status") != "completed":
|
||
return job
|
||
with IMAGE_PROXY_JOB_LOCK:
|
||
proxy = IMAGE_PROXY_JOBS.get(job_id)
|
||
if not proxy or proxy.get("artifact"):
|
||
if proxy and proxy.get("artifact"):
|
||
job["artifact"] = proxy["artifact"]
|
||
return job
|
||
result = job.get("result")
|
||
if not isinstance(result, dict):
|
||
return job
|
||
operation = str(proxy.get("operation") or job.get("operation") or "generate")
|
||
payload = dict(proxy.get("payload") or {})
|
||
artifact = save_image_artifact(
|
||
report_id=job_id,
|
||
image_base64=str(result.get("image_base64") or ""),
|
||
kind=operation,
|
||
metadata={
|
||
"prompt": payload.get("prompt") or "",
|
||
"negative_prompt": payload.get("negative_prompt") or "",
|
||
"width": result.get("width"),
|
||
"height": result.get("height"),
|
||
"seed": result.get("seed"),
|
||
"steps": payload.get("steps"),
|
||
"guidance_scale": payload.get("guidance_scale"),
|
||
"strength": payload.get("strength"),
|
||
"base_url": proxy.get("base_url"),
|
||
"model": result.get("model") or payload.get("model"),
|
||
},
|
||
)
|
||
latency_ms = job.get("latency_ms") or result.get("latency_ms")
|
||
append_report(
|
||
{
|
||
"id": job_id,
|
||
"type": "image_edit" if operation == "edit" else "image_generation",
|
||
"plugin": "image",
|
||
"prompt": payload.get("prompt") or "",
|
||
"negative_prompt": payload.get("negative_prompt") or "",
|
||
"width": result.get("width"),
|
||
"height": result.get("height"),
|
||
"seed": result.get("seed"),
|
||
"base_url": proxy.get("base_url"),
|
||
"model": result.get("model") or payload.get("model"),
|
||
"latency_ms": latency_ms,
|
||
"artifact": artifact,
|
||
}
|
||
)
|
||
append_image_job({**artifact, "type": operation, "status": "completed", "latency_ms": latency_ms, "updated_at": now_iso()})
|
||
proxy["artifact"] = artifact
|
||
job["artifact"] = artifact
|
||
if isinstance(job.get("result"), dict):
|
||
job["result"]["artifact"] = artifact
|
||
return job
|
||
|
||
|
||
def quality_check_answer(answer: str | None) -> dict:
|
||
text = (answer or "").strip()
|
||
if not text:
|
||
return {"passed": False, "reason": "empty answer"}
|
||
if len(text) < 12:
|
||
return {"passed": False, "reason": "answer is too short"}
|
||
return {"passed": True, "reason": "non-empty answer"}
|
||
|
||
|
||
def run_quality_suite(payload: dict) -> dict:
|
||
catalog = load_catalog()
|
||
plugin = str(payload.get("plugin") or "text")
|
||
model_name = str(payload.get("model") or "")
|
||
base_url = normalize_base_url(payload.get("base_url") or DEFAULT_BASE_URL)
|
||
tests = payload.get("tests") or TEST_PACKS.get(plugin) or []
|
||
if not model_name:
|
||
route = route_for_plugin(plugin)
|
||
model_name = route["served_model_name"]
|
||
if not isinstance(tests, list) or not tests:
|
||
raise ValueError("quality tests are empty")
|
||
|
||
results = []
|
||
for test in tests:
|
||
if not isinstance(test, dict):
|
||
raise ValueError("each quality test must be an object")
|
||
prompt = str(test.get("prompt") or "").strip()
|
||
if not prompt:
|
||
continue
|
||
messages = [
|
||
{"role": "system", "content": "Отвечай по-русски, кратко и проверяемо."},
|
||
{"role": "user", "content": prompt},
|
||
]
|
||
result = run_chat_request(base_url, model_name, messages, temperature=0.2, max_tokens=500)
|
||
guardrail = None
|
||
if plugin == "1c":
|
||
result["answer"], guardrail = guard_1c_answer(prompt, result.get("answer"))
|
||
check = quality_check_answer(result.get("answer"))
|
||
result.update(
|
||
{
|
||
"test_id": test.get("id"),
|
||
"label": test.get("label"),
|
||
"prompt": prompt,
|
||
"passed": check["passed"],
|
||
"check_reason": check["reason"],
|
||
"safety_guardrail": guardrail,
|
||
}
|
||
)
|
||
results.append(result)
|
||
|
||
passed = sum(1 for result in results if result.get("passed"))
|
||
report_id = append_report(
|
||
{
|
||
"type": "quality",
|
||
"plugin": plugin,
|
||
"served_model_name": model_name,
|
||
"base_url": base_url,
|
||
"passed": passed,
|
||
"total": len(results),
|
||
"results": results,
|
||
}
|
||
)
|
||
return {"report_id": report_id, "plugin": plugin, "model": model_name, "passed": passed, "total": len(results), "results": results}
|
||
|
||
|
||
def read_body(handler: BaseHTTPRequestHandler) -> dict:
|
||
length = int(handler.headers.get("Content-Length") or 0)
|
||
raw = handler.rfile.read(length) if length else b"{}"
|
||
data = json.loads(raw.decode("utf-8"))
|
||
if not isinstance(data, dict):
|
||
raise ValueError("request body must be a JSON object")
|
||
return data
|
||
|
||
|
||
def write_json_response(handler: BaseHTTPRequestHandler, status: int, payload: dict) -> None:
|
||
encoded = json.dumps(payload, ensure_ascii=False, indent=2).encode("utf-8")
|
||
handler.send_response(status)
|
||
handler.send_header("Content-Type", "application/json; charset=utf-8")
|
||
handler.send_header("Content-Length", str(len(encoded)))
|
||
handler.end_headers()
|
||
handler.wfile.write(encoded)
|
||
|
||
|
||
def safe_static_path(static_dir: Path, request_path: str) -> Path:
|
||
parsed = urlparse(request_path)
|
||
relative = parsed.path.lstrip("/") or "index.html"
|
||
candidate = (static_dir / relative).resolve()
|
||
static_root = static_dir.resolve()
|
||
if static_root not in candidate.parents and candidate != static_root:
|
||
raise ValueError("invalid static path")
|
||
if candidate.is_dir():
|
||
candidate = candidate / "index.html"
|
||
return candidate
|
||
|
||
|
||
class ChatHandler(BaseHTTPRequestHandler):
|
||
static_dir = DEFAULT_STATIC_DIR
|
||
|
||
def log_message(self, format: str, *args: object) -> None:
|
||
print(f"{self.address_string()} - {format % args}", file=sys.stderr)
|
||
|
||
def do_GET(self) -> None:
|
||
if self.path.startswith("/health") or self.path.startswith("/api/health"):
|
||
write_json_response(self, 200, health_status())
|
||
return
|
||
|
||
if self.path.startswith("/api/catalog"):
|
||
write_json_response(self, 200, load_catalog())
|
||
return
|
||
|
||
if self.path.startswith("/api/routes"):
|
||
try:
|
||
parsed = urlparse(self.path)
|
||
params = urllib.parse.parse_qs(parsed.query)
|
||
plugin = (params.get("plugin") or ["text"])[0]
|
||
task = (params.get("task") or [None])[0]
|
||
write_json_response(self, 200, route_for_plugin(plugin, task))
|
||
except ValueError as exc:
|
||
write_json_response(self, 400, {"error": str(exc)})
|
||
return
|
||
|
||
if self.path.startswith("/api/model-services"):
|
||
status = health_status()
|
||
write_json_response(self, 200, {"services": status["model_services"], "routes": status["routes"]})
|
||
return
|
||
|
||
if self.path.startswith("/api/benchmark/history"):
|
||
parsed = urlparse(self.path)
|
||
params = urllib.parse.parse_qs(parsed.query)
|
||
limit = int((params.get("limit") or ["10"])[0])
|
||
model_id = (params.get("model_id") or [None])[0]
|
||
plugin = (params.get("plugin") or [None])[0]
|
||
write_json_response(
|
||
self,
|
||
200,
|
||
{
|
||
"benchmarks": latest_runtime_benchmarks(
|
||
limit=max(1, min(limit, 50)),
|
||
model_id=model_id,
|
||
plugin=plugin,
|
||
)
|
||
},
|
||
)
|
||
return
|
||
|
||
if self.path.startswith("/api/benchmark/report.md"):
|
||
try:
|
||
parsed = urlparse(self.path)
|
||
params = urllib.parse.parse_qs(parsed.query)
|
||
name = (params.get("name") or [""])[0]
|
||
path = benchmark_report_path(name)
|
||
report = read_json(path)
|
||
content = render_benchmark_markdown(report, path.name).encode("utf-8")
|
||
except FileNotFoundError:
|
||
write_json_response(self, 404, {"error": "not found"})
|
||
return
|
||
except (ValueError, json.JSONDecodeError) as exc:
|
||
write_json_response(self, 400, {"error": str(exc)})
|
||
return
|
||
self.send_response(200)
|
||
self.send_header("Content-Type", "text/markdown; charset=utf-8")
|
||
self.send_header("Content-Length", str(len(content)))
|
||
self.end_headers()
|
||
self.wfile.write(content)
|
||
return
|
||
|
||
if self.path.startswith("/api/benchmark/report"):
|
||
try:
|
||
parsed = urlparse(self.path)
|
||
params = urllib.parse.parse_qs(parsed.query)
|
||
path = benchmark_report_path((params.get("name") or [""])[0])
|
||
except FileNotFoundError:
|
||
write_json_response(self, 404, {"error": "not found"})
|
||
return
|
||
except ValueError as exc:
|
||
write_json_response(self, 400, {"error": str(exc)})
|
||
return
|
||
content = path.read_bytes()
|
||
self.send_response(200)
|
||
self.send_header("Content-Type", "application/json; charset=utf-8")
|
||
self.send_header("Content-Length", str(len(content)))
|
||
self.end_headers()
|
||
self.wfile.write(content)
|
||
return
|
||
|
||
if self.path.startswith("/api/model-ingest/jobs"):
|
||
write_json_response(self, 200, {"jobs": latest_model_ingest_jobs()})
|
||
return
|
||
|
||
if self.path.startswith("/api/image/gallery"):
|
||
write_json_response(self, 200, {"images": latest_image_gallery()})
|
||
return
|
||
|
||
if urlparse(self.path).path == "/api/image/job":
|
||
try:
|
||
parsed = urlparse(self.path)
|
||
params = urllib.parse.parse_qs(parsed.query)
|
||
job_id = str((params.get("job_id") or [""])[0]).strip()
|
||
if not job_id:
|
||
raise ValueError("job_id is required")
|
||
with IMAGE_PROXY_JOB_LOCK:
|
||
proxy = dict(IMAGE_PROXY_JOBS.get(job_id) or {})
|
||
base_url = normalize_base_url((params.get("base_url") or [proxy.get("base_url") or route_for_plugin("image")["base_url"]])[0])
|
||
job = fetch_image_job(base_url=base_url, job_id=job_id)
|
||
if job.get("status") == "completed":
|
||
job = finalize_image_job(job)
|
||
if job.get("status") in {"cancelled", "error"}:
|
||
append_image_job(
|
||
{
|
||
"id": job_id,
|
||
"type": job.get("operation") or proxy.get("operation") or "generate",
|
||
"status": job.get("status"),
|
||
"error": job.get("error"),
|
||
"latency_ms": job.get("latency_ms"),
|
||
"updated_at": now_iso(),
|
||
}
|
||
)
|
||
write_json_response(self, 200, job)
|
||
except (TypeError, ValueError, urllib.error.URLError, json.JSONDecodeError) as exc:
|
||
write_json_response(self, 400, {"error": str(exc)})
|
||
return
|
||
|
||
if self.path.startswith("/api/image/jobs"):
|
||
write_json_response(self, 200, {"jobs": latest_image_jobs()})
|
||
return
|
||
|
||
if self.path.startswith("/generated-images/"):
|
||
relative = self.path.removeprefix("/generated-images/").split("?", 1)[0].lstrip("/")
|
||
image_path = (IMAGE_GALLERY_DIR / relative).resolve()
|
||
gallery_root = IMAGE_GALLERY_DIR.resolve()
|
||
if gallery_root not in image_path.parents or image_path.suffix.lower() != ".png" or not image_path.exists():
|
||
write_json_response(self, 404, {"error": "not found"})
|
||
return
|
||
content = image_path.read_bytes()
|
||
self.send_response(200)
|
||
self.send_header("Content-Type", "image/png")
|
||
self.send_header("Content-Length", str(len(content)))
|
||
self.end_headers()
|
||
self.wfile.write(content)
|
||
return
|
||
|
||
try:
|
||
path = safe_static_path(self.static_dir, self.path)
|
||
except ValueError as exc:
|
||
write_json_response(self, 400, {"error": str(exc)})
|
||
return
|
||
|
||
if not path.exists():
|
||
write_json_response(self, 404, {"error": "not found"})
|
||
return
|
||
|
||
content = path.read_bytes()
|
||
content_type = mimetypes.guess_type(path.name)[0] or "application/octet-stream"
|
||
if content_type.startswith("text/") or path.suffix in {".js", ".css"}:
|
||
content_type += "; charset=utf-8"
|
||
self.send_response(200)
|
||
self.send_header("Content-Type", content_type)
|
||
self.send_header("Content-Length", str(len(content)))
|
||
self.end_headers()
|
||
self.wfile.write(content)
|
||
|
||
def do_POST(self) -> None:
|
||
if self.path.startswith("/api/model-ingest/upload"):
|
||
try:
|
||
parsed = urlparse(self.path)
|
||
params = {}
|
||
for chunk in parsed.query.split("&"):
|
||
if not chunk or "=" not in chunk:
|
||
continue
|
||
key, value = chunk.split("=", 1)
|
||
params[urllib.parse.unquote_plus(key)] = urllib.parse.unquote_plus(value)
|
||
|
||
content_length = int(self.headers.get("Content-Length") or 0)
|
||
if content_length <= 0:
|
||
raise ValueError("upload body is required")
|
||
filename = sanitize_filename(
|
||
self.headers.get("X-Model-Filename")
|
||
or params.get("filename")
|
||
or "model.bin"
|
||
)
|
||
payload = {
|
||
"model_id": self.headers.get("X-Model-Id") or params.get("model_id") or Path(filename).stem,
|
||
"name": self.headers.get("X-Model-Name") or params.get("name") or Path(filename).stem,
|
||
"plugin": self.headers.get("X-Model-Plugin") or params.get("plugin") or "text",
|
||
"format": self.headers.get("X-Model-Format") or params.get("format") or Path(filename).suffix.lstrip("."),
|
||
"quantization": self.headers.get("X-Model-Quantization") or params.get("quantization") or "",
|
||
"notes": self.headers.get("X-Model-Notes") or params.get("notes") or "",
|
||
}
|
||
job, job_dir = create_model_ingest_job(payload, source_kind="upload")
|
||
destination = job_dir / filename
|
||
bytes_written = stream_request_to_file(self, destination, content_length)
|
||
job["status"] = "uploaded" if bytes_written == content_length else "partial"
|
||
job["updated_at"] = now_iso()
|
||
job["files"].append({"path": str(destination.relative_to(ROOT)), "size_bytes": bytes_written})
|
||
if bytes_written != content_length:
|
||
job["error"] = f"expected {content_length} bytes, received {bytes_written}"
|
||
write_model_ingest_metadata(job_dir, job)
|
||
append_model_ingest_job(job)
|
||
write_json_response(self, 200, job)
|
||
except (OSError, ValueError) as exc:
|
||
write_json_response(self, 400, {"error": str(exc)})
|
||
return
|
||
|
||
if self.path.startswith("/api/model-ingest/source"):
|
||
try:
|
||
payload = read_body(self)
|
||
job = import_model_from_source(payload)
|
||
status = 200 if job.get("status") not in {"failed"} else 400
|
||
write_json_response(self, status, job)
|
||
except (TypeError, ValueError) as exc:
|
||
write_json_response(self, 400, {"error": str(exc)})
|
||
return
|
||
|
||
if self.path.startswith("/api/model-ingest/verify"):
|
||
try:
|
||
payload = read_body(self)
|
||
job = verify_model_ingest_job(str(payload["job_id"]))
|
||
status = 200 if job.get("status") == "verified" else 400
|
||
write_json_response(self, status, job)
|
||
except (KeyError, TypeError, ValueError) as exc:
|
||
write_json_response(self, 400, {"error": str(exc)})
|
||
return
|
||
|
||
if self.path.startswith("/api/model-ingest/register"):
|
||
try:
|
||
payload = read_body(self)
|
||
job = register_model_ingest_job(str(payload["job_id"]))
|
||
status = 200 if str(job.get("status")).startswith("registered") else 400
|
||
write_json_response(self, status, job)
|
||
except (KeyError, TypeError, ValueError) as exc:
|
||
write_json_response(self, 400, {"error": str(exc)})
|
||
return
|
||
|
||
if self.path.startswith("/api/models"):
|
||
try:
|
||
payload = read_body(self)
|
||
base_url = normalize_base_url(payload.get("base_url") or DEFAULT_BASE_URL)
|
||
expected_model = payload.get("model")
|
||
started_at = time.perf_counter()
|
||
models = fetch_endpoint_models(base_url)
|
||
latency_ms = round((time.perf_counter() - started_at) * 1000)
|
||
write_json_response(
|
||
self,
|
||
200,
|
||
{
|
||
"base_url": base_url,
|
||
"models": models,
|
||
"latency_ms": latency_ms,
|
||
"expected_model": expected_model,
|
||
"available": expected_model in models if expected_model else None,
|
||
},
|
||
)
|
||
except (TypeError, ValueError, urllib.error.URLError) as exc:
|
||
write_json_response(self, 400, {"error": str(exc)})
|
||
return
|
||
|
||
if self.path.startswith("/api/route"):
|
||
try:
|
||
payload = read_body(self)
|
||
write_json_response(self, 200, route_for_plugin(str(payload.get("plugin") or "text"), payload.get("task")))
|
||
except (TypeError, ValueError) as exc:
|
||
write_json_response(self, 400, {"error": str(exc)})
|
||
return
|
||
|
||
if self.path.startswith("/api/quality/run"):
|
||
try:
|
||
write_json_response(self, 200, run_quality_suite(read_body(self)))
|
||
except (TypeError, ValueError) as exc:
|
||
write_json_response(self, 400, {"error": str(exc)})
|
||
return
|
||
|
||
if self.path.startswith("/api/service-control"):
|
||
try:
|
||
result = control_model_service(read_body(self))
|
||
status = 200 if result["ok"] else 500
|
||
write_json_response(self, status, result)
|
||
except (TypeError, ValueError) as exc:
|
||
write_json_response(self, 400, {"error": str(exc)})
|
||
return
|
||
|
||
if self.path.startswith("/api/benchmark/preflight"):
|
||
try:
|
||
result = runtime_benchmark_preflight(read_body(self))
|
||
status = 200 if result["ready"] else 409
|
||
write_json_response(self, status, result)
|
||
except (TypeError, ValueError) as exc:
|
||
write_json_response(self, 400, {"error": str(exc)})
|
||
return
|
||
|
||
if self.path.startswith("/api/benchmark/runtime"):
|
||
try:
|
||
result = run_runtime_benchmark(read_body(self))
|
||
status = 200 if result["ok"] else 500
|
||
write_json_response(self, status, result)
|
||
except (TypeError, ValueError) as exc:
|
||
write_json_response(self, 400, {"error": str(exc)})
|
||
return
|
||
|
||
if self.path.startswith("/api/feedback"):
|
||
try:
|
||
payload = read_body(self)
|
||
rating = str(payload["rating"])
|
||
if rating not in {"ok", "bad", "needs_review"}:
|
||
raise ValueError("rating must be ok, bad, or needs_review")
|
||
report_id = append_report(
|
||
{
|
||
"type": "feedback",
|
||
"parent_id": payload.get("report_id"),
|
||
"rating": rating,
|
||
"note": payload.get("note") or "",
|
||
}
|
||
)
|
||
write_json_response(self, 200, {"report_id": report_id})
|
||
except (KeyError, TypeError, ValueError) as exc:
|
||
write_json_response(self, 400, {"error": str(exc)})
|
||
return
|
||
|
||
if self.path.startswith("/api/1c/rag-prompt"):
|
||
try:
|
||
payload = read_body(self)
|
||
question = str(payload["question"]).strip()
|
||
if not question:
|
||
raise ValueError("question is required")
|
||
result = build_1c_rag_prompt(
|
||
question,
|
||
profile_name=str(payload.get("profile") or "auto"),
|
||
limit=int(payload["limit"]) if payload.get("limit") is not None else None,
|
||
)
|
||
report_id = append_report(
|
||
{
|
||
"type": "rag_prompt",
|
||
"plugin": "1c",
|
||
"profile": result["profile"],
|
||
"question": question,
|
||
"context_count": result["context_count"],
|
||
"sources": result["sources"],
|
||
}
|
||
)
|
||
result["report_id"] = report_id
|
||
write_json_response(self, 200, result)
|
||
except (KeyError, TypeError, ValueError, FileNotFoundError) as exc:
|
||
write_json_response(self, 400, {"error": str(exc)})
|
||
return
|
||
|
||
if self.path.startswith("/api/1c/rag-chat"):
|
||
report_id = str(uuid.uuid4())
|
||
started_at = time.perf_counter()
|
||
try:
|
||
payload = read_body(self)
|
||
question = str(payload["question"]).strip()
|
||
if not question:
|
||
raise ValueError("question is required")
|
||
base_url = normalize_base_url(payload.get("base_url") or DEFAULT_BASE_URL)
|
||
model = str(payload.get("model") or route_for_plugin("1c")["served_model_name"])
|
||
rag = build_1c_rag_prompt(
|
||
question,
|
||
profile_name=str(payload.get("profile") or "auto"),
|
||
limit=int(payload["limit"]) if payload.get("limit") is not None else None,
|
||
)
|
||
messages = [
|
||
{"role": "system", "content": "Ты помощник по 1С. Не предлагай прямые изменения живой базы без проверки, бэкапа и согласования."},
|
||
{"role": "user", "content": rag["prompt"]},
|
||
]
|
||
answer = call_chat_completion(
|
||
base_url=base_url,
|
||
model=model,
|
||
messages=messages,
|
||
temperature=float(payload.get("temperature", 0.2)),
|
||
max_tokens=int(payload.get("max_tokens", 1000)),
|
||
timeout=180,
|
||
)
|
||
answer, guardrail = guard_1c_answer(question, answer)
|
||
latency_ms = round((time.perf_counter() - started_at) * 1000)
|
||
append_report(
|
||
{
|
||
"id": report_id,
|
||
"type": "1c_rag_chat",
|
||
"plugin": "1c",
|
||
"profile": rag["profile"],
|
||
"question": question,
|
||
"answer": answer,
|
||
"sources": rag["sources"],
|
||
"context_count": rag["context_count"],
|
||
"served_model_name": model,
|
||
"base_url": base_url,
|
||
"latency_ms": latency_ms,
|
||
"safety_guardrail": guardrail,
|
||
}
|
||
)
|
||
write_json_response(self, 200, {**rag, "answer": answer, "report_id": report_id, "latency_ms": latency_ms, "safety_guardrail": guardrail})
|
||
except (KeyError, TypeError, ValueError, FileNotFoundError, urllib.error.URLError) as exc:
|
||
latency_ms = round((time.perf_counter() - started_at) * 1000)
|
||
append_report({"id": report_id, "type": "1c_rag_chat", "status": "error", "error": str(exc), "latency_ms": latency_ms})
|
||
write_json_response(self, 400, {"error": str(exc), "report_id": report_id, "latency_ms": latency_ms})
|
||
return
|
||
|
||
if self.path.startswith("/api/audio/transcribe"):
|
||
report_id = str(uuid.uuid4())
|
||
started_at = time.perf_counter()
|
||
try:
|
||
payload = read_body(self)
|
||
base_url = normalize_base_url(payload.get("base_url") or route_for_plugin("audio")["base_url"])
|
||
audio_base64 = str(payload.get("audio_base64") or "")
|
||
filename = sanitize_filename(payload.get("filename") or "audio.wav", fallback="audio.wav")
|
||
if not audio_base64:
|
||
raise ValueError("audio_base64 is required")
|
||
result = call_audio_transcription(
|
||
base_url=base_url,
|
||
audio_base64=audio_base64,
|
||
filename=filename,
|
||
language=str(payload.get("language") or "") or None,
|
||
task=str(payload.get("task") or "") or None,
|
||
)
|
||
latency_ms = round((time.perf_counter() - started_at) * 1000)
|
||
append_report(
|
||
{
|
||
"id": report_id,
|
||
"type": "audio_transcription",
|
||
"plugin": "audio",
|
||
"filename": filename,
|
||
"bytes": result.get("bytes"),
|
||
"text": result.get("text"),
|
||
"base_url": base_url,
|
||
"model": result.get("model") or payload.get("model"),
|
||
"latency_ms": latency_ms,
|
||
}
|
||
)
|
||
write_json_response(self, 200, {**result, "report_id": report_id, "latency_ms": latency_ms})
|
||
except (TypeError, ValueError, urllib.error.URLError, json.JSONDecodeError) as exc:
|
||
latency_ms = round((time.perf_counter() - started_at) * 1000)
|
||
append_report({"id": report_id, "type": "audio_transcription", "status": "error", "error": str(exc), "latency_ms": latency_ms})
|
||
write_json_response(self, 400, {"error": str(exc), "report_id": report_id, "latency_ms": latency_ms})
|
||
return
|
||
|
||
if self.path.startswith("/api/video/analyze"):
|
||
report_id = str(uuid.uuid4())
|
||
started_at = time.perf_counter()
|
||
try:
|
||
payload = read_body(self)
|
||
base_url = normalize_base_url(payload.get("base_url") or route_for_plugin("video")["base_url"])
|
||
image_base64 = str(payload.get("image_base64") or "")
|
||
filename = sanitize_filename(payload.get("filename") or "image.png", fallback="image.png")
|
||
prompt = str(payload.get("prompt") or "").strip() or "Опиши изображение и перечисли важные детали."
|
||
if not image_base64:
|
||
raise ValueError("image_base64 is required")
|
||
result = call_vision_analysis(
|
||
base_url=base_url,
|
||
image_base64=image_base64,
|
||
filename=filename,
|
||
prompt=prompt,
|
||
model=str(payload.get("model") or "") or None,
|
||
max_tokens=int(payload.get("max_tokens") or 512),
|
||
)
|
||
latency_ms = round((time.perf_counter() - started_at) * 1000)
|
||
append_report(
|
||
{
|
||
"id": report_id,
|
||
"type": "vision_analysis",
|
||
"plugin": "video",
|
||
"filename": filename,
|
||
"bytes": result.get("bytes"),
|
||
"width": result.get("width"),
|
||
"height": result.get("height"),
|
||
"prompt": prompt,
|
||
"text": result.get("text"),
|
||
"base_url": base_url,
|
||
"model": result.get("model") or payload.get("model"),
|
||
"latency_ms": latency_ms,
|
||
}
|
||
)
|
||
write_json_response(self, 200, {**result, "report_id": report_id, "latency_ms": latency_ms})
|
||
except (TypeError, ValueError, urllib.error.URLError, json.JSONDecodeError) as exc:
|
||
latency_ms = round((time.perf_counter() - started_at) * 1000)
|
||
append_report({"id": report_id, "type": "vision_analysis", "status": "error", "error": str(exc), "latency_ms": latency_ms})
|
||
write_json_response(self, 400, {"error": str(exc), "report_id": report_id, "latency_ms": latency_ms})
|
||
return
|
||
|
||
if self.path.startswith("/api/image/submit"):
|
||
report_id = str(uuid.uuid4())
|
||
started_at = time.perf_counter()
|
||
try:
|
||
payload = read_body(self)
|
||
operation = str(payload.get("operation") or payload.get("mode") or "generate")
|
||
if operation not in {"generate", "edit"}:
|
||
raise ValueError("operation must be generate or edit")
|
||
model_id = str(payload.get("model_id") or "").strip()
|
||
model_mode = str(payload.get("model_mode") or "").strip()
|
||
image_route = route_for_model("image", model_id, "image-editing" if operation == "edit" else None)
|
||
base_url = normalize_base_url(payload.get("base_url") or image_route["base_url"])
|
||
prompt = str(payload.get("prompt") or "").strip()
|
||
if not prompt:
|
||
raise ValueError("prompt is required")
|
||
seed_value = payload.get("seed")
|
||
seed = int(seed_value) if seed_value not in {None, ""} else None
|
||
job_payload: dict[str, object] = {
|
||
"prompt": prompt,
|
||
"negative_prompt": str(payload.get("negative_prompt") or "") or "",
|
||
"width": int(payload.get("width") or 1024),
|
||
"height": int(payload.get("height") or 1024),
|
||
"steps": int(payload.get("steps") or 28),
|
||
"guidance_scale": float(payload.get("guidance_scale") or 6.0),
|
||
"model": payload.get("model") or image_route["served_model_name"],
|
||
"model_id": model_id or image_route["model"].get("id"),
|
||
"model_mode": model_mode or image_route["model"].get("id"),
|
||
}
|
||
if seed is not None:
|
||
job_payload["seed"] = seed
|
||
if operation == "edit":
|
||
image_base64 = str(payload.get("image_base64") or "")
|
||
mask_base64 = str(payload.get("mask_base64") or "")
|
||
if not image_base64:
|
||
raise ValueError("image_base64 is required")
|
||
if not mask_base64:
|
||
raise ValueError("mask_base64 is required")
|
||
job_payload["image_base64"] = image_base64
|
||
job_payload["mask_base64"] = mask_base64
|
||
job_payload["strength"] = float(payload.get("strength") or 0.95)
|
||
expected_model = str(job_payload.get("model") or image_route["served_model_name"] or "")
|
||
endpoint_models = fetch_endpoint_models(base_url, timeout=5)
|
||
if expected_model and expected_model not in endpoint_models:
|
||
raise ValueError(
|
||
f"image endpoint {base_url} serves {endpoint_models or 'no models'}, "
|
||
f"not selected model {expected_model}"
|
||
)
|
||
append_image_job({"id": report_id, "type": operation, "status": "submitted", "created_at": now_iso()})
|
||
submitted = submit_image_job(base_url=base_url, operation=operation, payload=job_payload, job_id=report_id)
|
||
with IMAGE_PROXY_JOB_LOCK:
|
||
IMAGE_PROXY_JOBS[report_id] = {
|
||
"id": report_id,
|
||
"operation": operation,
|
||
"base_url": base_url,
|
||
"model_id": job_payload.get("model_id"),
|
||
"model_mode": job_payload.get("model_mode"),
|
||
"payload": job_payload,
|
||
"submitted_at": now_iso(),
|
||
"started_perf": started_at,
|
||
}
|
||
write_json_response(self, 202, {**submitted, "report_id": report_id, "base_url": base_url})
|
||
except (TypeError, ValueError, urllib.error.URLError, json.JSONDecodeError) as exc:
|
||
latency_ms = round((time.perf_counter() - started_at) * 1000)
|
||
append_report({"id": report_id, "type": "image_job_submit", "status": "error", "error": str(exc), "latency_ms": latency_ms})
|
||
append_image_job({"id": report_id, "type": "image", "status": "error", "error": str(exc), "latency_ms": latency_ms, "updated_at": now_iso()})
|
||
write_json_response(self, 400, {"error": str(exc), "report_id": report_id, "latency_ms": latency_ms})
|
||
return
|
||
|
||
if self.path.startswith("/api/image/cancel"):
|
||
try:
|
||
payload = read_body(self)
|
||
job_id = str(payload.get("job_id") or payload.get("report_id") or "").strip()
|
||
if not job_id:
|
||
raise ValueError("job_id is required")
|
||
with IMAGE_PROXY_JOB_LOCK:
|
||
proxy = dict(IMAGE_PROXY_JOBS.get(job_id) or {})
|
||
base_url = normalize_base_url(payload.get("base_url") or proxy.get("base_url") or route_for_plugin("image")["base_url"])
|
||
result = cancel_image_job(base_url=base_url, job_id=job_id)
|
||
append_image_job(
|
||
{
|
||
"id": job_id,
|
||
"type": result.get("operation") or proxy.get("operation") or "generate",
|
||
"status": result.get("status") or "cancel_requested",
|
||
"updated_at": now_iso(),
|
||
}
|
||
)
|
||
write_json_response(self, 200, result)
|
||
except (TypeError, ValueError, urllib.error.URLError, json.JSONDecodeError) as exc:
|
||
write_json_response(self, 400, {"error": str(exc)})
|
||
return
|
||
|
||
if self.path.startswith("/api/image/generate"):
|
||
report_id = str(uuid.uuid4())
|
||
started_at = time.perf_counter()
|
||
append_image_job({"id": report_id, "type": "generate", "status": "running", "created_at": now_iso()})
|
||
try:
|
||
payload = read_body(self)
|
||
base_url = normalize_base_url(payload.get("base_url") or route_for_plugin("image")["base_url"])
|
||
prompt = str(payload.get("prompt") or "").strip()
|
||
if not prompt:
|
||
raise ValueError("prompt is required")
|
||
seed_value = payload.get("seed")
|
||
seed = int(seed_value) if seed_value not in {None, ""} else None
|
||
result = call_image_generation(
|
||
base_url=base_url,
|
||
prompt=prompt,
|
||
negative_prompt=str(payload.get("negative_prompt") or "") or None,
|
||
width=int(payload.get("width") or 1024),
|
||
height=int(payload.get("height") or 1024),
|
||
steps=int(payload.get("steps") or 28),
|
||
guidance_scale=float(payload.get("guidance_scale") or 6.0),
|
||
seed=seed,
|
||
)
|
||
latency_ms = round((time.perf_counter() - started_at) * 1000)
|
||
artifact = save_image_artifact(
|
||
report_id=report_id,
|
||
image_base64=str(result.get("image_base64") or ""),
|
||
kind="generate",
|
||
metadata={
|
||
"prompt": prompt,
|
||
"negative_prompt": payload.get("negative_prompt") or "",
|
||
"width": result.get("width"),
|
||
"height": result.get("height"),
|
||
"seed": result.get("seed"),
|
||
"steps": int(payload.get("steps") or 28),
|
||
"guidance_scale": float(payload.get("guidance_scale") or 6.0),
|
||
"base_url": base_url,
|
||
"model": result.get("model") or payload.get("model"),
|
||
},
|
||
)
|
||
append_report(
|
||
{
|
||
"id": report_id,
|
||
"type": "image_generation",
|
||
"plugin": "image",
|
||
"prompt": prompt,
|
||
"negative_prompt": payload.get("negative_prompt") or "",
|
||
"width": result.get("width"),
|
||
"height": result.get("height"),
|
||
"seed": result.get("seed"),
|
||
"base_url": base_url,
|
||
"model": result.get("model") or payload.get("model"),
|
||
"latency_ms": latency_ms,
|
||
"artifact": artifact,
|
||
}
|
||
)
|
||
append_image_job({**artifact, "status": "completed", "latency_ms": latency_ms})
|
||
write_json_response(self, 200, {**result, "report_id": report_id, "latency_ms": latency_ms, "artifact": artifact})
|
||
except (TypeError, ValueError, urllib.error.URLError, json.JSONDecodeError) as exc:
|
||
latency_ms = round((time.perf_counter() - started_at) * 1000)
|
||
append_report({"id": report_id, "type": "image_generation", "status": "error", "error": str(exc), "latency_ms": latency_ms})
|
||
append_image_job({"id": report_id, "type": "generate", "status": "error", "error": str(exc), "latency_ms": latency_ms, "updated_at": now_iso()})
|
||
write_json_response(self, 400, {"error": str(exc), "report_id": report_id, "latency_ms": latency_ms})
|
||
return
|
||
|
||
if self.path.startswith("/api/image/edit"):
|
||
report_id = str(uuid.uuid4())
|
||
started_at = time.perf_counter()
|
||
append_image_job({"id": report_id, "type": "edit", "status": "running", "created_at": now_iso()})
|
||
try:
|
||
payload = read_body(self)
|
||
base_url = normalize_base_url(payload.get("base_url") or route_for_plugin("image", "image-editing")["base_url"])
|
||
prompt = str(payload.get("prompt") or "").strip()
|
||
image_base64 = str(payload.get("image_base64") or "")
|
||
mask_base64 = str(payload.get("mask_base64") or "")
|
||
if not prompt:
|
||
raise ValueError("prompt is required")
|
||
if not image_base64:
|
||
raise ValueError("image_base64 is required")
|
||
if not mask_base64:
|
||
raise ValueError("mask_base64 is required")
|
||
seed_value = payload.get("seed")
|
||
seed = int(seed_value) if seed_value not in {None, ""} else None
|
||
result = call_image_edit(
|
||
base_url=base_url,
|
||
prompt=prompt,
|
||
image_base64=image_base64,
|
||
mask_base64=mask_base64,
|
||
negative_prompt=str(payload.get("negative_prompt") or "") or None,
|
||
width=int(payload.get("width") or 1024),
|
||
height=int(payload.get("height") or 1024),
|
||
steps=int(payload.get("steps") or 28),
|
||
guidance_scale=float(payload.get("guidance_scale") or 6.0),
|
||
strength=float(payload.get("strength") or 0.95),
|
||
seed=seed,
|
||
)
|
||
latency_ms = round((time.perf_counter() - started_at) * 1000)
|
||
artifact = save_image_artifact(
|
||
report_id=report_id,
|
||
image_base64=str(result.get("image_base64") or ""),
|
||
kind="edit",
|
||
metadata={
|
||
"prompt": prompt,
|
||
"negative_prompt": payload.get("negative_prompt") or "",
|
||
"width": result.get("width"),
|
||
"height": result.get("height"),
|
||
"seed": result.get("seed"),
|
||
"steps": int(payload.get("steps") or 28),
|
||
"guidance_scale": float(payload.get("guidance_scale") or 6.0),
|
||
"strength": float(payload.get("strength") or 0.95),
|
||
"base_url": base_url,
|
||
"model": result.get("model") or payload.get("model"),
|
||
},
|
||
)
|
||
append_report(
|
||
{
|
||
"id": report_id,
|
||
"type": "image_edit",
|
||
"plugin": "image",
|
||
"prompt": prompt,
|
||
"negative_prompt": payload.get("negative_prompt") or "",
|
||
"width": result.get("width"),
|
||
"height": result.get("height"),
|
||
"seed": result.get("seed"),
|
||
"base_url": base_url,
|
||
"model": result.get("model") or payload.get("model"),
|
||
"latency_ms": latency_ms,
|
||
"artifact": artifact,
|
||
}
|
||
)
|
||
append_image_job({**artifact, "status": "completed", "latency_ms": latency_ms})
|
||
write_json_response(self, 200, {**result, "report_id": report_id, "latency_ms": latency_ms, "artifact": artifact})
|
||
except (TypeError, ValueError, urllib.error.URLError, json.JSONDecodeError) as exc:
|
||
latency_ms = round((time.perf_counter() - started_at) * 1000)
|
||
append_report({"id": report_id, "type": "image_edit", "status": "error", "error": str(exc), "latency_ms": latency_ms})
|
||
append_image_job({"id": report_id, "type": "edit", "status": "error", "error": str(exc), "latency_ms": latency_ms, "updated_at": now_iso()})
|
||
write_json_response(self, 400, {"error": str(exc), "report_id": report_id, "latency_ms": latency_ms})
|
||
return
|
||
|
||
if self.path.startswith("/api/compare"):
|
||
try:
|
||
payload = read_body(self)
|
||
base_url = normalize_base_url(payload.get("base_url") or DEFAULT_BASE_URL)
|
||
models = payload.get("models") or []
|
||
messages = payload.get("messages") or []
|
||
temperature = float(payload.get("temperature", 0.2))
|
||
max_tokens = int(payload.get("max_tokens", 1000))
|
||
if not isinstance(models, list) or not models:
|
||
raise ValueError("models must be a non-empty list")
|
||
if not isinstance(messages, list) or not messages:
|
||
raise ValueError("messages must be a non-empty list")
|
||
|
||
results = []
|
||
for model in models:
|
||
if not isinstance(model, dict):
|
||
raise ValueError("each model must be an object")
|
||
served_model = str(model["served_model_name"])
|
||
result = run_chat_request(base_url, served_model, messages, temperature, max_tokens)
|
||
guardrail = None
|
||
if payload.get("plugin") == "1c":
|
||
result["answer"], guardrail = guard_1c_answer(latest_user_message(messages), result.get("answer"))
|
||
result.update(
|
||
{
|
||
"registry_model_id": model.get("id"),
|
||
"name": model.get("name"),
|
||
"served_model_name": served_model,
|
||
"safety_guardrail": guardrail,
|
||
}
|
||
)
|
||
results.append(result)
|
||
|
||
report_id = append_report(
|
||
{
|
||
"type": "compare",
|
||
"plugin": payload.get("plugin"),
|
||
"test_id": payload.get("test_id"),
|
||
"base_url": base_url,
|
||
"messages": messages,
|
||
"temperature": temperature,
|
||
"max_tokens": max_tokens,
|
||
"results": results,
|
||
}
|
||
)
|
||
write_json_response(self, 200, {"report_id": report_id, "results": results})
|
||
except (KeyError, TypeError, ValueError) as exc:
|
||
write_json_response(self, 400, {"error": str(exc)})
|
||
return
|
||
|
||
if not self.path.startswith("/api/chat"):
|
||
write_json_response(self, 404, {"error": "not found"})
|
||
return
|
||
|
||
report_id = str(uuid.uuid4())
|
||
started_at = time.perf_counter()
|
||
try:
|
||
payload = read_body(self)
|
||
base_url = normalize_base_url(payload.get("base_url") or DEFAULT_BASE_URL)
|
||
model = str(payload["model"])
|
||
registry_model_id = payload.get("registry_model_id")
|
||
plugin = payload.get("plugin")
|
||
test_id = payload.get("test_id")
|
||
messages = payload.get("messages") or []
|
||
temperature = float(payload.get("temperature", 0.2))
|
||
max_tokens = int(payload.get("max_tokens", 1000))
|
||
if not isinstance(messages, list) or not messages:
|
||
raise ValueError("messages must be a non-empty list")
|
||
answer = call_chat_completion(
|
||
base_url=base_url,
|
||
model=model,
|
||
messages=messages,
|
||
temperature=temperature,
|
||
max_tokens=max_tokens,
|
||
timeout=180,
|
||
)
|
||
guardrail = None
|
||
if plugin == "1c":
|
||
answer, guardrail = guard_1c_answer(latest_user_message(messages), answer)
|
||
except (KeyError, TypeError, ValueError, urllib.error.URLError) as exc:
|
||
latency_ms = round((time.perf_counter() - started_at) * 1000)
|
||
append_report(
|
||
{
|
||
"id": report_id,
|
||
"type": "chat",
|
||
"status": "error",
|
||
"error": str(exc),
|
||
"payload": locals().get("payload", {}),
|
||
"latency_ms": latency_ms,
|
||
}
|
||
)
|
||
write_json_response(self, 400, {"error": str(exc), "report_id": report_id, "latency_ms": latency_ms})
|
||
return
|
||
|
||
latency_ms = round((time.perf_counter() - started_at) * 1000)
|
||
append_report(
|
||
{
|
||
"id": report_id,
|
||
"type": "chat",
|
||
"status": "answered",
|
||
"plugin": plugin,
|
||
"registry_model_id": registry_model_id,
|
||
"served_model_name": model,
|
||
"base_url": base_url,
|
||
"test_id": test_id,
|
||
"messages": messages,
|
||
"answer": answer,
|
||
"temperature": temperature,
|
||
"max_tokens": max_tokens,
|
||
"latency_ms": latency_ms,
|
||
"rating": None,
|
||
"safety_guardrail": guardrail,
|
||
}
|
||
)
|
||
write_json_response(self, 200, {"answer": answer, "report_id": report_id, "latency_ms": latency_ms, "safety_guardrail": guardrail})
|
||
|
||
|
||
def main() -> int:
|
||
parser = argparse.ArgumentParser(description="Run a local model chat test UI.")
|
||
parser.add_argument("--host", default=DEFAULT_HOST)
|
||
parser.add_argument("--port", type=int, default=DEFAULT_PORT)
|
||
parser.add_argument("--static-dir", type=Path, default=DEFAULT_STATIC_DIR)
|
||
args = parser.parse_args()
|
||
|
||
ChatHandler.static_dir = args.static_dir
|
||
server = ThreadingHTTPServer((args.host, args.port), ChatHandler)
|
||
print(f"Model chat UI: http://{args.host}:{args.port}")
|
||
print(f"Static dir: {args.static_dir}")
|
||
try:
|
||
server.serve_forever()
|
||
except KeyboardInterrupt:
|
||
print("\nStopping model chat UI.")
|
||
return 0
|
||
|
||
|
||
if __name__ == "__main__":
|
||
raise SystemExit(main())
|