from __future__ import annotations import argparse import datetime as dt import json import mimetypes import os import re import shutil import socket import subprocess import sys import threading import time import urllib.error import urllib.parse import urllib.request import uuid from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from pathlib import Path from urllib.parse import urlparse, urlunparse from ask_1c_rag import DEFAULT_INDEX as DEFAULT_1C_RAG_INDEX from ask_1c_rag import DEFAULT_RAG_PROMPT as DEFAULT_1C_RAG_PROMPT from ask_1c_rag import format_context, render_prompt from common import ROOT, call_chat_completion, localize_workspace_path, read_json, read_yaml_mapping, search_lexical_index from rag_profiles import list_rag_profiles, resolve_rag_profile DEFAULT_HOST = "127.0.0.1" DEFAULT_PORT = 8765 DEFAULT_BASE_URL = "http://docker-gpu.cin.su:8000" DEFAULT_STATIC_DIR = ROOT / "tools" / "model-chat" REGISTRY_INDEX = ROOT / "registry" / "index.json" REPORT_ROOT = Path(os.environ.get("MODEL_CHAT_REPORT_ROOT") or (ROOT / "reports")) REPORT_DIR = REPORT_ROOT / "model-chat" IMAGE_GALLERY_DIR = REPORT_DIR / "images" IMAGE_JOBS = REPORT_DIR / "image-jobs.jsonl" MODEL_INGEST_DIR = ROOT / "models" / "incoming" MODEL_INGEST_JOBS = REPORT_ROOT / "model-ingest" / "jobs.jsonl" GPU_PROFILES_PATH = ROOT / "config" / "gpu_profiles.json" RUNTIME_PROFILES_PATH = ROOT / "config" / "runtime_profiles.json" MODEL_CHAT_PREFLIGHT_REPORT = REPORT_ROOT / "model-chat" / "preflight.json" IMAGE_PROXY_JOBS: dict[str, dict] = {} IMAGE_PROXY_JOB_LOCK = threading.Lock() PLUGIN_ORDER = ["text", "translation", "audio", "video", "image", "1c"] PLUGIN_LABELS = { "text": "Текст", "translation": "Перевод", "audio": "Звук", "video": "Видео", "image": "Фото", "1c": "1С", } TASK_PLUGIN_MAP = { "text": "text", "chat": "text", "summarization": "text", "code": "text", "tool-use": "text", "translation": "translation", "speech-to-text": "audio", "speech-translation": "audio", "video": "video", "image-understanding": "video", "document-understanding": "video", "visual-question-answering": "video", "image-generation": "image", "image-editing": "image", "inpainting": "image", "1c": "1c", "1c-rag": "1c", "bsl-code": "1c", "metadata-safety": "1c", "1c-query": "1c", } DEFAULT_PROMPTS = { "text": "Кратко объясни, зачем нужен реестр локальных моделей.", "translation": "Переведи на английский: Нужно проверить качество локальной модели перевода.", "audio": "Составь короткий план проверки модели распознавания речи для русского языка.", "video": "Опиши, какие вопросы стоит задать vision-language модели при проверке видео.", "image": "Фотореалистичный рабочий стол инженера 1С, два монитора, заметки, спокойный дневной свет.", "1c": "Какие метаданные 1С нужно получить перед изменением формы документа?", } ENDPOINT_PRESETS = [ {"id": "vllm-text", "label": "vLLM text", "base_url": "http://docker-gpu.cin.su:8000"}, {"id": "llama-gguf", "label": "llama.cpp GGUF", "base_url": "http://docker-gpu.cin.su:8080"}, {"id": "llama-gguf-q6-test", "label": "llama.cpp Q6 test", "base_url": "http://docker-gpu.cin.su:8081"}, {"id": "docker-test-q6-cpu", "label": "docker-test Q6 CPU", "base_url": "http://docker-test.cin.su:18086"}, {"id": "translation-api", "label": "Translation API", "base_url": "http://docker-gpu.cin.su:8010"}, {"id": "audio-api", "label": "Audio API", "base_url": "http://docker-gpu.cin.su:8020"}, {"id": "video-api", "label": "Video API", "base_url": "http://docker-gpu.cin.su:8030"}, {"id": "image-api", "label": "Image API", "base_url": "http://docker-gpu.cin.su:8040"}, {"id": "local-vllm", "label": "local", "base_url": "http://127.0.0.1:8000"}, ] ALLOWED_ENDPOINT_HOSTS = { ("http", "docker-gpu.cin.su", 8000), ("http", "docker-gpu.cin.su", 8080), ("http", "docker-gpu.cin.su", 8081), ("http", "docker-gpu.cin.su", 8010), ("http", "docker-gpu.cin.su", 8020), ("http", "docker-gpu.cin.su", 8030), ("http", "docker-gpu.cin.su", 8040), ("http", "docker-test.cin.su", 18081), ("http", "docker-test.cin.su", 18086), ("http", "192.168.200.61", 18081), ("http", "192.168.200.61", 18086), ("http", "127.0.0.1", 8000), ("http", "127.0.0.1", 8080), ("http", "127.0.0.1", 8081), ("http", "127.0.0.1", 8010), ("http", "127.0.0.1", 8020), ("http", "127.0.0.1", 8030), ("http", "127.0.0.1", 8040), ("http", "127.0.0.1", 8765), ("http", "localhost", 8000), ("http", "localhost", 8080), ("http", "localhost", 8081), ("http", "localhost", 8010), ("http", "localhost", 8020), ("http", "localhost", 8030), ("http", "localhost", 8040), } SERVICE_PLANS = { "vllm": { "service_id": "vllm-text", "label": "vLLM text", "base_url": "http://docker-gpu.cin.su:8000", "container_name": "llm-vllm-text", "compose": "core/deploy/docker-gpu/vllm/compose.yaml", "deploy_script": "scripts/deploy_vllm.ps1", "health_path": "/v1/models", }, "llama.cpp": { "service_id": "llama-gguf", "label": "llama.cpp GGUF", "base_url": "http://docker-gpu.cin.su:8080", "container_name": "llm-llama-devstral-1c", "compose": "core/deploy/docker-gpu/llama-cpp/compose.yaml", "deploy_script": "scripts/deploy_llama_cpp.ps1", "health_path": "/v1/models", }, "transformers:translation": { "service_id": "translation-api", "label": "Transformers translation", "base_url": "http://docker-gpu.cin.su:8010", "container_name": "llm-transformers-translation", "compose": "core/deploy/docker-gpu/transformers/translation.compose.yaml", "deploy_script": "scripts/deploy_transformers_service.ps1 -Plugin translation", "health_path": "/health", }, "transformers:audio": { "service_id": "audio-api", "label": "Transformers audio", "base_url": "http://docker-gpu.cin.su:8020", "container_name": "llm-transformers-audio", "compose": "core/deploy/docker-gpu/transformers/audio.compose.yaml", "deploy_script": "scripts/deploy_transformers_service.ps1 -Plugin audio", "health_path": "/health", }, "transformers:video": { "service_id": "video-api", "label": "Transformers video", "base_url": "http://docker-gpu.cin.su:8030", "container_name": "llm-transformers-video", "compose": "core/deploy/docker-gpu/transformers/video.compose.yaml", "deploy_script": "scripts/deploy_transformers_service.ps1 -Plugin video", "health_path": "/health", }, "transformers:image": { "service_id": "image-api", "label": "Transformers image", "base_url": "http://docker-gpu.cin.su:8040", "container_name": "llm-transformers-image", "compose": "core/deploy/docker-gpu/transformers/image.compose.yaml", "deploy_script": "scripts/deploy_transformers_service.ps1 -Plugin image", "health_path": "/health", }, } SERVICE_CONTROL_ACTIONS = {"start", "stop", "restart", "status"} DEFAULT_DOCKER_HOST = os.environ.get("DOCKER_HOST", "ssh://docker-gpu") TEST_PACKS = { "text": [ {"id": "short", "label": "Краткий ответ", "prompt": DEFAULT_PROMPTS["text"]}, {"id": "json", "label": "JSON", "prompt": "Верни JSON с полями status, summary, next_steps. Без markdown."}, {"id": "format", "label": "Формат", "prompt": "Сделай список из 5 пунктов: критерии приемки локальной LLM."}, {"id": "critique", "label": "Критика", "prompt": "Назови 3 риска при проверке локальной модели и как их снизить."}, ], "translation": [ {"id": "ru-en", "label": "RU -> EN", "prompt": "Переведи на английский: Нужно проверить качество локальной модели перевода."}, {"id": "en-ru", "label": "EN -> RU", "prompt": "Translate into Russian: The model must preserve technical terms and JSON structure."}, {"id": "terms", "label": "Термины", "prompt": "Переведи на английский, сохрани термины: регистр сведений, справочник, документ реализации."}, {"id": "format", "label": "Сохранить JSON", "prompt": "Переведи значения JSON на английский, ключи не меняй: {\"status\":\"готово\",\"risk\":\"нет доступа к GPU\"}"}, ], "audio": [ {"id": "plan", "label": "План", "prompt": DEFAULT_PROMPTS["audio"]}, {"id": "metrics", "label": "Метрики", "prompt": "Какие метрики использовать для оценки speech-to-text на русском языке?"}, {"id": "noise", "label": "Шум", "prompt": "Составь сценарии проверки распознавания речи при шуме и разных микрофонах."}, {"id": "diarization", "label": "Диалоги", "prompt": "Как проверить качество распознавания диалога двух пользователей?"}, ], "video": [ {"id": "plan", "label": "План", "prompt": DEFAULT_PROMPTS["video"]}, {"id": "scene", "label": "Сцена", "prompt": "Какие вопросы задать модели, чтобы проверить понимание сцены на видео?"}, {"id": "events", "label": "События", "prompt": "Составь тесты для поиска событий во временной шкале видео."}, {"id": "docs", "label": "Документы", "prompt": "Как проверить vision-language модель на чтении документов и экранных форм?"}, ], "image": [ {"id": "photo", "label": "Фото", "prompt": DEFAULT_PROMPTS["image"]}, {"id": "product", "label": "Предмет", "prompt": "Предметное фото локального GPU-сервера на чистом столе, реалистичный свет, 1024x1024."}, {"id": "edit", "label": "Редактирование", "prompt": "Заменить фон на светлую офисную стену, сохранить предмет и естественные тени."}, {"id": "ui", "label": "Интерфейс", "prompt": "Скриншот современного локального кабинета моделей, аккуратная панель, реалистичная фотография монитора."}, ], "1c": [ {"id": "metadata", "label": "Метаданные", "prompt": DEFAULT_PROMPTS["1c"]}, {"id": "bsl", "label": "BSL", "prompt": "Найди возможную ошибку в BSL-коде и объясни безопасное исправление: Если Объект.Сумма = 0 Тогда Возврат; КонецЕсли;"}, {"id": "query", "label": "Запрос 1С", "prompt": "Составь read-only запрос 1С для получения 10 последних документов реализации."}, {"id": "safety", "label": "Безопасность", "prompt": "Почему модель не должна напрямую менять живую базу 1С? Дай workflow согласования."}, ], } STATUS_RANK = { "production": 0, "staging": 1, "candidate": 2, "draft": 3, "archived": 9, } ONE_C_DANGEROUS_REQUEST_PATTERNS = [ r"(?i)\b(delete|drop|update|insert|truncate|alter)\b", r"(?i)\b(удалить|удали|удаление|изменить|измени|обновить|обнови)\b", r"(?i)\b(записать|запиши|провести|проведи)\b", r"(?i)\b(объект\.записать|провести\s*\(|удалить\s+из|изменить\s+.*\s+установить)\b", ] ONE_C_UNSAFE_ANSWER_PATTERNS = [ r"(?i)checklist\s+для\s+удален", r"(?i)план\s+.*удален", r"(?i)безопасн\w*\s+удален", r"(?i)уточните\s+услов", r"(?i)ограничьте\s+удален", r"(?i)если\s+вы\s+хотите\s+удал", r"(?i)помог[уа]\s+.*удал", r"(?i)удалитьизсправочника", ] ONE_C_SAFE_MUTATION_REFUSAL = ( "Я не могу выполнить, составить или уточнять операцию изменения данных в 1С. " "Могу помочь только read-only анализом: показать запрос `ВЫБРАТЬ` для инвентаризации " "затрагиваемых записей, проверить зависимости по metadata snapshot и подготовить checklist " "согласования для ответственного администратора." ) def sanitize_filename(value: str, *, fallback: str = "model.bin") -> str: name = Path(str(value or fallback).replace("\\", "/")).name.strip() name = re.sub(r"[^A-Za-z0-9._+@=-]+", "_", name) name = name.strip("._") return name or fallback def sanitize_id(value: str, *, fallback: str = "model") -> str: text = str(value or fallback).strip().lower() text = re.sub(r"[^a-z0-9._-]+", "-", text) text = text.strip(".-_") return text or fallback def now_iso() -> str: return dt.datetime.now(dt.UTC).isoformat() def display_path(path: Path) -> str: try: return str(path.relative_to(ROOT)) except ValueError: return str(path) def model_ingest_job_dir(job_id: str) -> Path: return MODEL_INGEST_DIR / job_id def latest_user_message(messages: list[dict]) -> str: for message in reversed(messages): if isinstance(message, dict) and message.get("role") == "user": return str(message.get("content") or "") return "" def has_pattern(text: str, patterns: list[str]) -> bool: return any(re.search(pattern, text or "") for pattern in patterns) def guard_1c_answer(question: str, answer: str | None) -> tuple[str | None, dict | None]: if not answer: return answer, None dangerous_request = has_pattern(question, ONE_C_DANGEROUS_REQUEST_PATTERNS) unsafe_answer = has_pattern(answer, ONE_C_UNSAFE_ANSWER_PATTERNS) if not dangerous_request or not unsafe_answer: return answer, None return ONE_C_SAFE_MUTATION_REFUSAL, { "applied": True, "reason": "dangerous_1c_mutation_answer", } def append_model_ingest_job(job: dict) -> None: MODEL_INGEST_JOBS.parent.mkdir(parents=True, exist_ok=True) with MODEL_INGEST_JOBS.open("a", encoding="utf-8") as handle: handle.write(json.dumps(job, ensure_ascii=False) + "\n") def append_image_job(job: dict) -> None: IMAGE_JOBS.parent.mkdir(parents=True, exist_ok=True) with IMAGE_JOBS.open("a", encoding="utf-8") as handle: handle.write(json.dumps(job, ensure_ascii=False) + "\n") def latest_image_jobs(limit: int = 30) -> list[dict]: if not IMAGE_JOBS.exists(): return [] rows = [] with IMAGE_JOBS.open("r", encoding="utf-8") as handle: for line in handle: line = line.strip() if not line: continue try: rows.append(json.loads(line)) except json.JSONDecodeError: continue latest: dict[str, dict] = {} for row in rows: job_id = str(row.get("id") or "") if job_id: latest[job_id] = row collapsed = list(latest.values()) collapsed.sort(key=lambda item: str(item.get("updated_at") or item.get("created_at") or ""), reverse=True) return collapsed[:limit] def read_model_ingest_jobs(limit: int = 30) -> list[dict]: if not MODEL_INGEST_JOBS.exists(): return [] rows = [] with MODEL_INGEST_JOBS.open("r", encoding="utf-8") as handle: for line in handle: line = line.strip() if not line: continue try: rows.append(json.loads(line)) except json.JSONDecodeError: continue return rows[-limit:][::-1] def latest_model_ingest_jobs(limit: int = 100) -> list[dict]: latest: dict[str, dict] = {} for job in read_model_ingest_jobs(limit=limit * 5): job_id = str(job.get("id") or "") if job_id and job_id not in latest: latest[job_id] = job return list(latest.values())[:limit] def decode_data_url_bytes(value: str, *, field_name: str = "data") -> bytes: data = str(value or "") if "," in data and data.split(",", 1)[0].startswith("data:"): data = data.split(",", 1)[1] if not data: raise ValueError(f"{field_name} is required") import base64 return base64.b64decode(data) def save_image_artifact(*, report_id: str, image_base64: str, kind: str, metadata: dict) -> dict: created = dt.datetime.now(dt.UTC) day_dir = IMAGE_GALLERY_DIR / created.strftime("%Y%m%d") day_dir.mkdir(parents=True, exist_ok=True) safe_kind = sanitize_id(kind, fallback="image") filename = f"{created.strftime('%H%M%S')}-{report_id}-{safe_kind}.png" image_path = day_dir / filename image_path.write_bytes(decode_data_url_bytes(image_base64, field_name="image_base64")) sidecar_path = image_path.with_suffix(".json") record = { "id": report_id, "created_at": created.isoformat(), "kind": kind, "path": display_path(image_path), "url": f"/generated-images/{created.strftime('%Y%m%d')}/{filename}", "size_bytes": image_path.stat().st_size, **metadata, } sidecar_path.write_text(json.dumps(record, ensure_ascii=False, indent=2), encoding="utf-8") return record def latest_image_gallery(limit: int = 40) -> list[dict]: if not IMAGE_GALLERY_DIR.exists(): return [] records = [] for sidecar in IMAGE_GALLERY_DIR.glob("*/*.json"): try: data = json.loads(sidecar.read_text(encoding="utf-8")) except (OSError, json.JSONDecodeError): continue if isinstance(data, dict) and data.get("url"): records.append(data) records.sort(key=lambda item: str(item.get("created_at") or ""), reverse=True) return records[:limit] def update_model_ingest_job(job: dict, *, status: str, error: str | None = None, **extra: object) -> dict: updated = dict(job) updated["status"] = status updated["updated_at"] = now_iso() if error is None: updated.pop("error", None) else: updated["error"] = error updated.update(extra) write_model_ingest_metadata(model_ingest_job_dir(str(updated["id"])), updated) append_model_ingest_job(updated) return updated def verify_model_ingest_job(job_id: str) -> dict: job = next((item for item in latest_model_ingest_jobs() if str(item.get("id")) == job_id), None) if not job: raise ValueError("model ingest job not found") files = job.get("files") or [] if not files: return update_model_ingest_job(job, status="needs_file", error="job has no imported files") checked_files = [] missing_files = [] total_size = 0 for file_info in files: relative_path = str(file_info.get("path") or "") path = (ROOT / relative_path).resolve() if not path.exists() or not path.is_file(): missing_files.append(relative_path) continue size = path.stat().st_size total_size += size checked_files.append({"path": relative_path, "size_bytes": size}) if missing_files: return update_model_ingest_job( job, status="missing_files", error=f"missing files: {', '.join(missing_files)}", checked_files=checked_files, ) return update_model_ingest_job( job, status="verified", checked_files=checked_files, total_size_bytes=total_size, ) def task_for_plugin(plugin: str) -> list[str]: mapping = { "text": ["text", "chat"], "translation": ["translation"], "audio": ["speech-to-text"], "video": ["image-understanding", "visual-question-answering"], "image": ["image-generation"], "1c": ["1c", "1c-rag", "bsl-code"], } return mapping.get(plugin, [plugin]) def runtime_for_plugin_and_format(plugin: str, model_format: str) -> str: if model_format.lower() == "gguf": return "llama.cpp" if plugin in {"translation", "audio", "video", "image"}: return "transformers" return "vllm" def register_model_ingest_job(job_id: str) -> dict: job = next((item for item in latest_model_ingest_jobs(limit=500) if str(item.get("id")) == job_id), None) if not job: raise ValueError("model ingest job not found") if str(job.get("status")) != "verified": job = verify_model_ingest_job(job_id) if str(job.get("status")) != "verified": raise ValueError(f"model ingest job is not verified: {job.get('status')}") model_id = sanitize_id(job.get("model_id") or job_id) plugin = sanitize_id(job.get("plugin") or "text") model_format = str(job.get("format") or "").lower() or "custom" card_path = ROOT / "registry" / "model-cards" / f"{model_id}.yaml" if card_path.exists(): raise ValueError(f"model card already exists: {card_path.relative_to(ROOT)}") files = job.get("checked_files") or job.get("files") or [] first_file = files[0] if files else {} storage_path = str(job.get("target_dir") or "") card = { "id": model_id, "name": str(job.get("name") or model_id), "status": "draft", "type": "imported-model", "task": task_for_plugin(plugin), "runtime": runtime_for_plugin_and_format(plugin, model_format), "format": model_format, "storage_path": storage_path, "served_model_name": model_id, "license": "unknown", "source": { "kind": job.get("source_kind"), "source": job.get("source") or "", "ingest_job_id": job_id, }, "deployment": { "runtime": runtime_for_plugin_and_format(plugin, model_format), "notes": "Registered from model ingest. Review runtime and served_model_name before production use.", }, } if first_file.get("path"): card["filename"] = Path(str(first_file["path"])).name if job.get("quantization"): card["quantization"] = job.get("quantization") def yaml_scalar(value: object) -> str: text = str(value) if not text or any(char in text for char in ":#[]{}&,*!|>'\"%@`"): return json.dumps(text, ensure_ascii=False) return text def write_yaml_lines(value: object, indent: int = 0) -> list[str]: prefix = " " * indent lines: list[str] = [] if isinstance(value, dict): for key, item in value.items(): if isinstance(item, (dict, list)): lines.append(f"{prefix}{key}:") lines.extend(write_yaml_lines(item, indent + 2)) else: lines.append(f"{prefix}{key}: {yaml_scalar(item)}") elif isinstance(value, list): for item in value: if isinstance(item, (dict, list)): lines.append(f"{prefix}-") lines.extend(write_yaml_lines(item, indent + 2)) else: lines.append(f"{prefix}- {yaml_scalar(item)}") return lines card_path.write_text("\n".join(write_yaml_lines(card)) + "\n", encoding="utf-8") build = run_command([sys.executable, "scripts/build_model_index.py"], timeout=60) status = "registered" if build["returncode"] == 0 else "registered_index_failed" return update_model_ingest_job( job, status=status, card_path=str(card_path.relative_to(ROOT)), index_result=build, ) def write_model_ingest_metadata(job_dir: Path, job: dict) -> None: job_dir.mkdir(parents=True, exist_ok=True) (job_dir / "metadata.json").write_text(json.dumps(job, ensure_ascii=False, indent=2), encoding="utf-8") def create_model_ingest_job(payload: dict, *, source_kind: str) -> tuple[dict, Path]: job_id = str(uuid.uuid4()) model_id = sanitize_id(payload.get("model_id") or payload.get("name") or "model") plugin = sanitize_id(payload.get("plugin") or "text") job_dir = model_ingest_job_dir(job_id) job = { "id": job_id, "created_at": now_iso(), "updated_at": now_iso(), "status": "created", "source_kind": source_kind, "model_id": model_id, "name": str(payload.get("name") or model_id), "plugin": plugin, "task": payload.get("task") or [plugin], "format": payload.get("format") or "", "quantization": payload.get("quantization") or "", "notes": payload.get("notes") or "", "target_dir": str(job_dir.relative_to(ROOT)), "files": [], } return job, job_dir def stream_request_to_file(handler: BaseHTTPRequestHandler, destination: Path, content_length: int) -> int: destination.parent.mkdir(parents=True, exist_ok=True) remaining = content_length written = 0 with destination.open("wb") as handle: while remaining > 0: chunk = handler.rfile.read(min(1024 * 1024, remaining)) if not chunk: break handle.write(chunk) written += len(chunk) remaining -= len(chunk) return written def download_url_to_file(url: str, destination: Path) -> int: destination.parent.mkdir(parents=True, exist_ok=True) request = urllib.request.Request(url, headers={"User-Agent": "LLM-model-ingest/1.0"}) written = 0 with urllib.request.urlopen(request, timeout=60) as response, destination.open("wb") as handle: while True: chunk = response.read(1024 * 1024) if not chunk: break handle.write(chunk) written += len(chunk) return written def import_model_from_source(payload: dict) -> dict: source = str(payload.get("source") or "").strip() if not source: raise ValueError("source is required") job, job_dir = create_model_ingest_job(payload, source_kind="source") parsed = urlparse(source) source_name = sanitize_filename(payload.get("filename") or Path(parsed.path or source).name or job["model_id"]) destination = job_dir / source_name job["source"] = source try: if parsed.scheme in {"http", "https"}: bytes_written = download_url_to_file(source, destination) job["status"] = "downloaded" else: source_path = Path(source) if not source_path.exists(): job["status"] = "queued_manual_required" job["error"] = "source path is not accessible from this server" bytes_written = 0 else: job_dir.mkdir(parents=True, exist_ok=True) shutil.copy2(source_path, destination) bytes_written = destination.stat().st_size job["status"] = "imported" if bytes_written: job["files"].append({"path": str(destination.relative_to(ROOT)), "size_bytes": bytes_written}) except (OSError, urllib.error.URLError) as exc: job["status"] = "failed" job["error"] = str(exc) job["updated_at"] = now_iso() write_model_ingest_metadata(job_dir, job) append_model_ingest_job(job) return job def model_plugins(model: dict) -> list[str]: plugins = [] for task in model.get("task") or []: plugin = TASK_PLUGIN_MAP.get(task) if plugin and plugin not in plugins: plugins.append(plugin) return plugins or ["text"] def plugin_model_score(model: dict, plugin: str) -> tuple[int, int, str]: tasks = set(model.get("task") or model.get("tasks") or []) status = str(model.get("status") or "") model_id = str(model.get("id") or "") runtime = str(model.get("runtime") or "") model_type = str(model.get("type") or "") quantization = str(model.get("quantization") or "") score = 0 if plugin == "text": if tasks & {"text", "chat"}: score -= 5 if tasks & {"1c", "1c-rag", "bsl-code"}: score += 6 elif plugin == "1c": if tasks & {"1c", "1c-rag", "bsl-code", "1c-query"}: score -= 6 if "qwen3-coder" in model_id: score -= 8 if quantization == "Q6_K" or "q6" in model_id: score -= 3 if runtime == "llama.cpp": score -= 2 if model_type == "lora-adapter" or model_id.endswith("-lora-v1"): score += 8 elif plugin in TASK_PLUGIN_MAP.values(): mapped_tasks = {task for task, mapped_plugin in TASK_PLUGIN_MAP.items() if mapped_plugin == plugin} if tasks & mapped_tasks: score -= 5 return (score, STATUS_RANK.get(status, 8), str(model.get("id") or "")) def load_gpu_profiles() -> dict[str, dict]: profiles = read_json(GPU_PROFILES_PATH) if not isinstance(profiles, dict): raise ValueError(f"{GPU_PROFILES_PATH} must contain a profile mapping") normalized = {} for profile_id, profile in profiles.items(): if not isinstance(profile, dict): continue item = dict(profile) item["id"] = str(item.get("id") or profile_id) item.setdefault("label", item["id"]) item.setdefault("command", f"powershell -ExecutionPolicy Bypass -File scripts\\switch_gpu_profile.ps1 -Profile {item['id']}") item.setdefault("starts", []) item.setdefault("stops", []) item.setdefault("wait", []) item.setdefault("notes", "") normalized[str(profile_id)] = item return normalized def public_gpu_profiles() -> dict[str, dict]: return { profile_id: { key: value for key, value in profile.items() if key in {"id", "label", "command", "starts", "stops", "notes"} } for profile_id, profile in load_gpu_profiles().items() } def load_runtime_profiles() -> dict: if not RUNTIME_PROFILES_PATH.exists(): return {"default": "gpu-fast", "profiles": {}} data = read_json(RUNTIME_PROFILES_PATH) if not isinstance(data, dict): raise ValueError(f"{RUNTIME_PROFILES_PATH} must contain an object") profiles = data.get("profiles") or {} if not isinstance(profiles, dict): raise ValueError(f"{RUNTIME_PROFILES_PATH} profiles must contain a mapping") normalized = {} for profile_id, profile in profiles.items(): if not isinstance(profile, dict): continue item = dict(profile) item["id"] = str(item.get("id") or profile_id) item.setdefault("label", item["id"]) item.setdefault("host", "") item.setdefault("docker_endpoint", "") item.setdefault("role", "") item.setdefault("notes", "") item.setdefault("endpoints", {}) item.setdefault("model_overrides", {}) normalized[str(profile_id)] = item default_id = str(data.get("default") or next(iter(normalized), "")) return {"default": default_id, "profiles": normalized} def public_runtime_profiles() -> dict: data = load_runtime_profiles() return { "default": data["default"], "profiles": { profile_id: { key: value for key, value in profile.items() if key in {"id", "label", "host", "docker_endpoint", "role", "notes", "endpoints", "model_overrides"} } for profile_id, profile in data["profiles"].items() }, } def load_catalog() -> dict: registry = read_json(REGISTRY_INDEX) models = [] plugins = {plugin: {"id": plugin, "label": PLUGIN_LABELS[plugin], "models": []} for plugin in PLUGIN_ORDER} for model in registry.get("models") or []: deployment_model = model.get("served_model_name") or model.get("id") item = { "id": model.get("id"), "name": model.get("name"), "type": model.get("type"), "status": model.get("status"), "runtime": model.get("runtime"), "storage_path": model.get("storage_path"), "format": model.get("format"), "quantization": model.get("quantization"), "served_model_name": deployment_model, "tasks": model.get("task") or [], "plugins": model_plugins(model), "card_path": model.get("card_path"), } models.append(item) for plugin in item["plugins"]: plugins.setdefault(plugin, {"id": plugin, "label": plugin, "models": []}) plugins[plugin]["models"].append(item["id"]) by_id = {model["id"]: model for model in models} for plugin_id, plugin in plugins.items(): plugin["models"].sort(key=lambda model_id: plugin_model_score(by_id[model_id], plugin_id)) ordered_plugins = [plugins[plugin] for plugin in PLUGIN_ORDER if plugin in plugins] extras = [value for key, value in plugins.items() if key not in PLUGIN_ORDER] return { "default_base_url": DEFAULT_BASE_URL, "endpoint_presets": ENDPOINT_PRESETS, "plugins": ordered_plugins + extras, "models": models, "default_prompts": DEFAULT_PROMPTS, "test_packs": TEST_PACKS, "gpu_profiles": public_gpu_profiles(), "runtime_profiles": public_runtime_profiles(), "rag_profiles": { "1c": list_rag_profiles(), }, } def local_model_card(model: dict) -> dict: card_path = model.get("card_path") if not card_path: return {} path = (ROOT / str(card_path)).resolve() if not path.exists(): return {} return read_yaml_mapping(path) def has_any_file(path: Path, patterns: list[str]) -> bool: return any(path.glob(pattern) for pattern in patterns) def has_weight_file(path: Path) -> bool: for file_path in path.iterdir() if path.exists() else []: if file_path.is_file() and file_path.name.lower().endswith((".safetensors", ".bin", ".gguf")): return True return False def storage_status_for_model(model: dict) -> dict: card = local_model_card(model) raw_storage_path = str(model.get("storage_path") or card.get("storage_path") or "") storage_candidates = [localize_workspace_path(raw_storage_path)] if raw_storage_path.startswith("/models/"): raw_path = Path(raw_storage_path) if Path("/models").exists(): storage_candidates.insert(0, raw_path) else: storage_candidates.append(raw_path) storage_path = next((path for path in storage_candidates if path.exists()), storage_candidates[0]) result = { "path": str(storage_path), "exists": storage_path.exists(), "status": "missing", "reason": "storage path is missing", } if not storage_path.exists(): return result model_format = str(card.get("format") or "").lower() model_type = str(card.get("type") or "").lower() if model_format == "diffusers" or model_type == "image-diffusion-model": missing = [name for name in ["model_index.json"] if not (storage_path / name).exists()] if missing: result.update({"status": "missing", "reason": f"missing required file(s): {', '.join(missing)}"}) elif not any(storage_path.rglob("*.safetensors")) and not any(storage_path.rglob("*.bin")): result.update({"status": "metadata-only", "reason": "diffusers weights are missing"}) else: result.update({"status": "ok", "reason": "ready"}) return result if model_format == "gguf": filename = str(card.get("filename") or "") if not filename: result.update({"status": "failed", "reason": "GGUF filename is missing in model card"}) return result file_path = storage_path / filename if not file_path.exists(): result.update({"status": "missing", "reason": f"GGUF file is missing: {filename}"}) return result size = file_path.stat().st_size expected = card.get("file_size_bytes") status = "ok" if not expected or size == int(expected) else "partial" reason = "ready" if status == "ok" else f"size mismatch: {size} != {expected}" result.update({"status": status, "reason": reason, "size_bytes": size, "expected_size_bytes": expected}) return result if model_type == "lora-adapter": ready = has_any_file(storage_path, ["adapter_config.json", "*.safetensors", "*.bin"]) result.update({"status": "ok" if ready else "metadata-only", "reason": "ready" if ready else "adapter artifact files are missing"}) return result missing = [name for name in ["config.json"] if not (storage_path / name).exists()] index_path = storage_path / "model.safetensors.index.json" missing_shards: list[str] = [] if index_path.exists(): try: index = json.loads(index_path.read_text(encoding="utf-8")) shard_names = sorted(set((index.get("weight_map") or {}).values())) missing_shards = [name for name in shard_names if not (storage_path / name).exists()] except json.JSONDecodeError: result.update({"status": "failed", "reason": "model.safetensors.index.json is invalid"}) return result has_weights = has_weight_file(storage_path) has_tokenizer = has_any_file(storage_path, ["tokenizer.json", "tokenizer.model", "vocab.json"]) if missing: result.update({"status": "missing", "reason": f"missing required file(s): {', '.join(missing)}"}) elif missing_shards: preview = ", ".join(missing_shards[:4]) suffix = f" and {len(missing_shards) - 4} more" if len(missing_shards) > 4 else "" result.update({"status": "partial", "reason": f"missing shard file(s): {preview}{suffix}"}) elif not has_weights: result.update({"status": "metadata-only", "reason": "model weights are missing"}) elif not has_tokenizer: result.update({"status": "partial", "reason": "tokenizer files are missing"}) else: result.update({"status": "ok", "reason": "ready"}) return result def service_plan_for_model(model: dict) -> dict: runtime = str(model.get("runtime") or "") plugins = model.get("plugins") or [] key = runtime if runtime == "transformers": for plugin in ("translation", "audio", "video", "image"): if plugin in plugins: key = f"transformers:{plugin}" break plan = dict(SERVICE_PLANS.get(key) or {}) if not plan: plan = { "service_id": runtime or "unknown", "label": runtime or "unknown runtime", "base_url": DEFAULT_BASE_URL, "container_name": None, "compose": None, "deploy_script": None, "health_path": "/health", } return plan def service_status_for_model(model: dict, endpoint_models: dict[str, list[str]]) -> dict: plan = service_plan_for_model(model) storage = storage_status_for_model(model) served_name = str(model.get("served_model_name") or model.get("id") or "") base_url = str(plan.get("base_url") or DEFAULT_BASE_URL) models = endpoint_models.get(base_url, []) online = served_name in models status = "online" if online else "ready_to_start" if storage["status"] == "ok" else "blocked" return { "status": status, "online": online, "storage": storage, "service": plan, "served_model_name": served_name, "endpoint_models": models, } def service_health_for_model(model: dict, endpoint_health: dict[str, dict]) -> dict: plan = service_plan_for_model(model) base_url = str(plan.get("base_url") or DEFAULT_BASE_URL) return endpoint_health.get(base_url, {}) def model_readiness(model: dict, endpoint_models: dict[str, list[str]]) -> dict: served_name = str(model.get("served_model_name") or model.get("id") or "") plan = service_plan_for_model(model) base_url = str(plan.get("base_url") or DEFAULT_BASE_URL) available = served_name in endpoint_models.get(base_url, []) return {"base_url": base_url, "served_model_name": served_name, "available": available, "service": plan} def route_for_plugin(plugin: str, task: str | None = None) -> dict: catalog = load_catalog() models = [model for model in catalog["models"] if plugin in model.get("plugins", [])] if task: task_matches = [model for model in models if task in model.get("tasks", [])] if task_matches: models = task_matches models.sort(key=lambda model: plugin_model_score(model, plugin)) selected = models[0] if models else None if not selected: raise ValueError(f"no model route for plugin: {plugin}") endpoint = model_readiness(selected, {}) try: runtime_data = load_runtime_profiles() default_profile = runtime_data["profiles"].get(runtime_data["default"]) if default_profile: target = runtime_profile_target(default_profile, model=selected, model_id=str(selected["id"]), plugin=plugin) endpoint = { **endpoint, "base_url": target["base_url"], "served_model_name": target["served_model_name"], } except Exception: pass return { "plugin": plugin, "task": task, "model": selected, "base_url": endpoint["base_url"], "served_model_name": endpoint["served_model_name"], "service": endpoint["service"], "reason": f"Selected by plugin={plugin}, status={selected.get('status')}, runtime={selected.get('runtime')}", } def route_for_model(plugin: str, model_id: str, task: str | None = None) -> dict: if not model_id: return route_for_plugin(plugin, task) catalog = load_catalog() selected = next( ( model for model in catalog["models"] if model.get("id") == model_id and plugin in model.get("plugins", []) ), None, ) if not selected: return route_for_plugin(plugin, task) endpoint = model_readiness(selected, {}) try: runtime_data = load_runtime_profiles() default_profile = runtime_data["profiles"].get(runtime_data["default"]) if default_profile: target = runtime_profile_target(default_profile, model=selected, model_id=str(selected["id"]), plugin=plugin) endpoint = { **endpoint, "base_url": target["base_url"], "served_model_name": target["served_model_name"], } except Exception: pass return { "plugin": plugin, "task": task, "model": selected, "base_url": endpoint["base_url"], "served_model_name": endpoint["served_model_name"], "service": endpoint["service"], "reason": f"Selected by model_id={model_id}, status={selected.get('status')}, runtime={selected.get('runtime')}", } def run_command(command: list[str], *, timeout: int = 60) -> dict: started_at = time.perf_counter() try: result = subprocess.run( command, cwd=ROOT, text=True, capture_output=True, timeout=timeout, check=False, ) return { "command": command, "returncode": result.returncode, "stdout": result.stdout.strip(), "stderr": result.stderr.strip(), "latency_ms": round((time.perf_counter() - started_at) * 1000), } except FileNotFoundError as exc: return { "command": command, "returncode": 127, "stdout": "", "stderr": str(exc), "latency_ms": round((time.perf_counter() - started_at) * 1000), } except subprocess.TimeoutExpired as exc: return { "command": command, "returncode": 124, "stdout": (exc.stdout or "").strip() if isinstance(exc.stdout, str) else "", "stderr": (exc.stderr or "").strip() if isinstance(exc.stderr, str) else f"timeout after {timeout}s", "latency_ms": round((time.perf_counter() - started_at) * 1000), } def docker_compose_base_args(plan: dict, *, docker_host: str | None = None) -> list[str]: compose = plan.get("compose") if not compose: raise ValueError("service has no compose file") compose_path = ROOT / str(compose) if not compose_path.exists(): raise ValueError(f"compose file is missing: {compose}") args = ["docker"] host = docker_host or DEFAULT_DOCKER_HOST if host: args.extend(["--host", host]) return [*args, "compose", "-f", str(compose_path)] def control_model_service(payload: dict) -> dict: action = str(payload.get("action") or "").strip().lower() model_id = str(payload.get("model_id") or "").strip() if action not in SERVICE_CONTROL_ACTIONS: raise ValueError(f"action must be one of: {', '.join(sorted(SERVICE_CONTROL_ACTIONS))}") if not model_id: raise ValueError("model_id is required") catalog = load_catalog() model = next((item for item in catalog["models"] if item.get("id") == model_id), None) if not model: raise ValueError(f"unknown model_id: {model_id}") plan = service_plan_for_model(model) docker_args = ["docker"] docker_host = payload.get("docker_host") or DEFAULT_DOCKER_HOST if docker_host: docker_args.extend(["--host", str(docker_host)]) if action == "start": container_name = str(plan.get("container_name") or "") if container_name: existing = run_command( [*docker_args, "ps", "-a", "--filter", f"name=^{container_name}$", "--format", "{{.Names}}"], timeout=30, ) if existing["returncode"] == 0 and container_name in existing["stdout"].splitlines(): command = [*docker_args, "start", container_name] else: base_args = docker_compose_base_args(plan, docker_host=str(docker_host) if docker_host else None) command = [*base_args, "up", "-d"] else: base_args = docker_compose_base_args(plan, docker_host=str(docker_host) if docker_host else None) command = [*base_args, "up", "-d"] timeout = 120 elif action == "stop": container_name = str(plan.get("container_name") or "") if not container_name: raise ValueError("service has no container_name") command = [*docker_args, "stop", container_name] timeout = 90 elif action == "restart": container_name = str(plan.get("container_name") or "") if not container_name: raise ValueError("service has no container_name") command = [*docker_args, "restart", container_name] timeout = 120 else: container_name = str(plan.get("container_name") or "") if container_name: command = [*docker_args, "ps", "-a", "--filter", f"name=^{container_name}$"] else: base_args = docker_compose_base_args(plan, docker_host=str(docker_host) if docker_host else None) command = [*base_args, "ps"] timeout = 60 result = run_command(command, timeout=timeout) report_id = append_report( { "type": "service_control", "model_id": model_id, "model": model.get("name") or model_id, "action": action, "service": plan, "result": result, } ) return { "ok": result["returncode"] == 0, "report_id": report_id, "model_id": model_id, "action": action, "service": plan, "result": result, } def run_runtime_benchmark(payload: dict) -> dict: model_id = sanitize_id(str(payload.get("model_id") or "qwen3-coder-30b-a3b-instruct-q6_k")) plugin = sanitize_id(str(payload.get("plugin") or "1c")) profiles = payload.get("profiles") or ["gpu-fast", "cpu-test"] if not isinstance(profiles, list) or not profiles: raise ValueError("profiles must be a non-empty list") safe_profiles = [sanitize_id(str(profile), fallback="profile") for profile in profiles] max_tokens = int(payload.get("max_tokens") or 384) temperature = float(payload.get("temperature") or 0.1) prompt = str(payload.get("prompt") or DEFAULT_PROMPTS.get(plugin) or DEFAULT_PROMPTS["1c"]).strip() if not prompt: raise ValueError("prompt is required") timestamp = dt.datetime.now(dt.UTC).strftime("%Y%m%dT%H%M%SZ") report_path = REPORT_ROOT / "benchmarks" / f"runtime-profiles-{model_id}-{timestamp}.json" command = [ sys.executable, str(ROOT / "scripts" / "benchmark_runtime_profiles.py"), "--model-id", model_id, "--plugin", plugin, "--max-tokens", str(max_tokens), "--temperature", str(temperature), "--timeout", str(int(payload.get("timeout") or 600)), "--report", str(report_path), "--profiles", *safe_profiles, ] if prompt: command.extend(["--prompt", prompt]) result = run_command(command, timeout=int(payload.get("command_timeout") or 900)) report = {} if report_path.exists(): try: report = read_json(report_path) except (OSError, json.JSONDecodeError, ValueError) as exc: report = {"status": "error", "error": f"failed to read report: {exc}"} report_id = append_report( { "type": "runtime_benchmark", "model_id": model_id, "plugin": plugin, "profiles": safe_profiles, "report_path": display_path(report_path), "result": result, "report": report, } ) return { "ok": result["returncode"] == 0 and bool(report), "report_id": report_id, "report_path": display_path(report_path), "result": result, "benchmark": report, } def runtime_profile_target(profile: dict, *, model: dict, model_id: str, plugin: str) -> dict: override = (profile.get("model_overrides") or {}).get(model_id) or {} base_url = override.get("base_url") or (profile.get("endpoints") or {}).get(plugin) served_model_name = override.get("served_model_name") or model.get("served_model_name") or model_id if not base_url: raise ValueError(f"profile `{profile.get('id')}` has no endpoint for plugin `{plugin}`") return { "base_url": normalize_base_url(str(base_url)), "served_model_name": str(served_model_name), "host": profile.get("host"), "role": profile.get("role"), "container_name": override.get("container_name"), } def runtime_benchmark_preflight(payload: dict) -> dict: model_id = sanitize_id(str(payload.get("model_id") or "qwen3-coder-30b-a3b-instruct-q6_k")) plugin = sanitize_id(str(payload.get("plugin") or "1c")) profiles = payload.get("profiles") or ["gpu-fast", "cpu-test"] if not isinstance(profiles, list) or not profiles: raise ValueError("profiles must be a non-empty list") catalog = load_catalog() model = next((item for item in catalog["models"] if item.get("id") == model_id), None) if not model: raise ValueError(f"unknown model_id: {model_id}") runtime_profiles = load_runtime_profiles()["profiles"] checks = [] started_at = time.perf_counter() for profile_id_raw in profiles: profile_id = sanitize_id(str(profile_id_raw), fallback="profile") profile = runtime_profiles.get(profile_id) if not profile: checks.append({"profile_id": profile_id, "status": "error", "error": "unknown runtime profile"}) continue try: target = runtime_profile_target(profile, model=model, model_id=model_id, plugin=plugin) checked_at = time.perf_counter() models = fetch_endpoint_models(target["base_url"], timeout=int(payload.get("timeout") or 20)) latency_ms = round((time.perf_counter() - checked_at) * 1000) available = target["served_model_name"] in models checks.append( { "profile_id": profile_id, "label": profile.get("label") or profile_id, "status": "ok" if available else "missing_model", "available": available, "latency_ms": latency_ms, "target": target, "models": models, } ) except (ValueError, urllib.error.URLError, json.JSONDecodeError) as exc: checks.append( { "profile_id": profile_id, "label": profile.get("label") or profile_id, "status": "error", "available": False, "error": str(exc), } ) ready = bool(checks) and all(item.get("available") for item in checks) return { "status": "ready" if ready else "blocked", "ready": ready, "model_id": model_id, "plugin": plugin, "latency_ms": round((time.perf_counter() - started_at) * 1000), "checks": checks, } def latest_runtime_benchmarks(limit: int = 10, *, model_id: str | None = None, plugin: str | None = None) -> list[dict]: benchmark_dir = REPORT_ROOT / "benchmarks" if not benchmark_dir.exists(): return [] records = [] for path in sorted(benchmark_dir.glob("runtime-profiles-*.json"), key=lambda item: item.stat().st_mtime, reverse=True): try: report = read_json(path) except (OSError, json.JSONDecodeError, ValueError): continue if not isinstance(report, dict): continue if model_id and report.get("model_id") != model_id: continue if plugin and report.get("plugin") != plugin: continue summary_results = [] for row in report.get("results") or []: if not isinstance(row, dict): continue target = row.get("target") or {} summary_results.append( { "profile_id": row.get("profile_id"), "status": row.get("status"), "elapsed_sec": row.get("elapsed_sec"), "output_tokens_per_sec": row.get("output_tokens_per_sec"), "base_url": target.get("base_url"), "served_model_name": target.get("served_model_name"), "error": row.get("error"), } ) records.append( { "path": display_path(path), "updated_at": dt.datetime.fromtimestamp(path.stat().st_mtime, dt.UTC).isoformat(), "created_at": report.get("created_at"), "status": report.get("status"), "model_id": report.get("model_id"), "plugin": report.get("plugin"), "fastest_profile_id": report.get("fastest_profile_id"), "speedup": report.get("speedup") or {}, "results": summary_results, } ) if len(records) >= limit: break return records def benchmark_report_path(name: str) -> Path: filename = Path(urllib.parse.unquote(name or "")).name if not re.fullmatch(r"runtime-profiles-[A-Za-z0-9_.-]+\.json", filename): raise ValueError("invalid benchmark report name") path = (REPORT_ROOT / "benchmarks" / filename).resolve() benchmark_root = (REPORT_ROOT / "benchmarks").resolve() if benchmark_root not in path.parents or not path.exists(): raise FileNotFoundError(filename) return path def render_benchmark_markdown(report: dict, source_name: str) -> str: title = report.get("model_id") or source_name created = report.get("created_at") or "-" plugin = report.get("plugin") or "-" max_tokens = report.get("max_tokens") or "-" fastest = report.get("fastest_profile_id") or "-" ratio = (report.get("speedup") or {}).get("gpu_vs_cpu_ratio") lines = [ f"# Runtime Benchmark: {title}", "", f"- Created: `{created}`", f"- Plugin: `{plugin}`", f"- Max tokens: `{max_tokens}`", f"- Fastest profile: `{fastest}`", ] if ratio: lines.append(f"- GPU/CPU ratio: `x{ratio}`") lines.extend( [ "", "| Profile | Host | Endpoint | Served model | Elapsed | Output speed | Result |", "|---|---|---|---|---:|---:|---|", ] ) for row in report.get("results") or []: target = row.get("target") or {} status = row.get("status") or "-" elapsed = f"{row.get('elapsed_sec')}s" if row.get("elapsed_sec") is not None else "-" speed = f"{row.get('output_tokens_per_sec')} tok/s" if row.get("output_tokens_per_sec") is not None else "-" result = "fastest" if row.get("profile_id") == fastest else status if status != "ok" and row.get("error"): result = f"error: {row.get('error')}" lines.append( "| " + " | ".join( [ f"`{row.get('profile_id') or '-'}`", f"`{target.get('host') or '-'}`", f"`{target.get('base_url') or '-'}`", f"`{target.get('served_model_name') or '-'}`", elapsed, speed, result, ] ) + " |" ) lines.extend(["", "Raw JSON:", "", f"`{source_name}`", ""]) return "\n".join(lines) def gpu_status() -> dict: query = "index,name,memory.used,memory.total,utilization.gpu" result = run_command( ["nvidia-smi", f"--query-gpu={query}", "--format=csv,noheader,nounits"], timeout=5, ) gpus = [] if result["returncode"] == 0: for line in result["stdout"].splitlines(): parts = [part.strip() for part in line.split(",")] if len(parts) != 5: continue try: used = int(parts[2]) total = int(parts[3]) utilization = int(parts[4]) except ValueError: continue gpus.append( { "index": parts[0], "name": parts[1], "memory_used_mib": used, "memory_total_mib": total, "memory_free_mib": total - used, "memory_used_percent": round((used / total) * 100, 1) if total else None, "utilization_gpu_percent": utilization, } ) summary = "unavailable" if gpus: summary = " · ".join( f"GPU{gpu['index']} {gpu['memory_free_mib']}/{gpu['memory_total_mib']} MiB free, {gpu['utilization_gpu_percent']}%" for gpu in gpus ) return {"available": True, "gpus": gpus, "summary": summary, "probe": result} for base_url in ("http://docker-gpu.cin.su:8040", "http://docker-gpu.cin.su:8020", "http://docker-gpu.cin.su:8010"): try: request = urllib.request.Request(f"{base_url}/health", method="GET") with urllib.request.urlopen(request, timeout=3) as response: data = json.loads(response.read().decode("utf-8")) gpu = data.get("gpu") if isinstance(data, dict) else None if isinstance(gpu, dict) and gpu.get("available"): return { "available": True, "gpus": gpu.get("devices") or [], "summary": gpu.get("summary") or "cuda available", "source": f"{base_url}/health", "probe": result, } except Exception: continue return {"available": False, "gpus": [], "summary": summary, "probe": result} def service_control_status() -> dict: docker_path = shutil.which("docker") return { "available": bool(docker_path), "docker_path": docker_path, "docker_host": DEFAULT_DOCKER_HOST, "actions": sorted(SERVICE_CONTROL_ACTIONS), "fallback": "scripts/manage_model_service.ps1", } def gpu_profile_status(endpoint_statuses: list[dict]) -> dict[str, dict]: endpoint_by_id = {str(item.get("id")): item for item in endpoint_statuses} endpoint_by_id["model-chat-ui"] = {"id": "model-chat-ui", "status": "ok"} profiles = {} for profile_id, profile in load_gpu_profiles().items(): starts = list(profile.get("starts") or []) stops = list(profile.get("stops") or []) missing = [service for service in starts if endpoint_by_id.get(service, {}).get("status") != "ok"] conflicts = [service for service in stops if endpoint_by_id.get(service, {}).get("status") == "ok"] profiles[profile_id] = { "id": profile_id, "ready": not missing and not conflicts, "missing": missing, "conflicts": conflicts, "starts": starts, "stops": stops, } return profiles def latest_model_chat_preflight() -> dict: if not MODEL_CHAT_PREFLIGHT_REPORT.exists(): return {"status": "missing", "report_path": display_path(MODEL_CHAT_PREFLIGHT_REPORT)} try: report = read_json(MODEL_CHAT_PREFLIGHT_REPORT) except (OSError, ValueError, json.JSONDecodeError) as exc: return { "status": "failed", "report_path": display_path(MODEL_CHAT_PREFLIGHT_REPORT), "error": str(exc), } steps = report.get("steps") if isinstance(report.get("steps"), list) else [] failed_steps = [ str(step.get("name") or "unknown") for step in steps if isinstance(step, dict) and step.get("status") != "ok" ] return { "status": str(report.get("status") or "unknown"), "created_at": report.get("created_at"), "report_path": display_path(MODEL_CHAT_PREFLIGHT_REPORT), "step_count": len(steps), "failed_steps": failed_steps, } def health_status() -> dict: endpoint_statuses = [] endpoint_models: dict[str, list[str]] = {} endpoint_health: dict[str, dict] = {} health_detail_urls = { str(plan.get("base_url") or "") for plan in SERVICE_PLANS.values() if plan.get("health_path") == "/health" } for preset in ENDPOINT_PRESETS: base_url = preset["base_url"] started_at = time.perf_counter() try: models = fetch_endpoint_models(base_url, timeout=3) endpoint_models[base_url] = models detail = fetch_endpoint_health(base_url, timeout=2) if base_url in health_detail_urls else {} if detail: endpoint_health[base_url] = detail endpoint_statuses.append( { "id": preset["id"], "label": preset["label"], "base_url": base_url, "status": "ok", "models": models, "health": detail, "latency_ms": round((time.perf_counter() - started_at) * 1000), } ) except Exception as exc: # noqa: BLE001 - health endpoint must report all failures. endpoint_statuses.append( { "id": preset["id"], "label": preset["label"], "base_url": base_url, "status": "error", "error": str(exc), "latency_ms": round((time.perf_counter() - started_at) * 1000), } ) catalog = load_catalog() routes = [] for plugin in PLUGIN_ORDER: try: route = route_for_plugin(plugin) route["readiness"] = service_status_for_model(route["model"], endpoint_models) route["service_health"] = service_health_for_model(route["model"], endpoint_health) route["available"] = route["readiness"]["online"] routes.append(route) except ValueError as exc: routes.append({"plugin": plugin, "status": "error", "error": str(exc)}) model_services = [] for model in catalog["models"]: model_services.append( { "id": model["id"], "name": model["name"], "plugins": model.get("plugins") or [], "runtime": model.get("runtime"), "served_model_name": model.get("served_model_name"), "service_health": service_health_for_model(model, endpoint_health), **service_status_for_model(model, endpoint_models), } ) return { "status": "ok" if any(item["status"] == "ok" for item in endpoint_statuses) else "degraded", "host": socket.gethostname(), "time": now_iso(), "registry_models": len(catalog["models"]), "plugins": len(catalog["plugins"]), "rag_1c_index": { "path": str(DEFAULT_1C_RAG_INDEX.relative_to(ROOT)), "exists": DEFAULT_1C_RAG_INDEX.exists(), }, "model_ingest_jobs": latest_model_ingest_jobs(limit=10), "gpu": gpu_status(), "service_control": service_control_status(), "model_chat_preflight": latest_model_chat_preflight(), "endpoints": endpoint_statuses, "gpu_profile_status": gpu_profile_status(endpoint_statuses), "routes": routes, "model_services": model_services, } def append_report(record: dict) -> str: report_id = record.get("id") or str(uuid.uuid4()) record["id"] = report_id record.setdefault("created_at", dt.datetime.now(dt.UTC).isoformat()) REPORT_DIR.mkdir(parents=True, exist_ok=True) path = REPORT_DIR / f"{dt.datetime.now().strftime('%Y%m%d')}.jsonl" with path.open("a", encoding="utf-8") as handle: handle.write(json.dumps(record, ensure_ascii=False) + "\n") return report_id def fetch_endpoint_models(base_url: str, timeout: int = 20) -> list[str]: request = urllib.request.Request(f"{base_url.rstrip('/')}/v1/models", method="GET") with urllib.request.urlopen(request, timeout=timeout) as response: data = json.loads(response.read().decode("utf-8")) model_rows = data.get("data") if isinstance(data, dict) else [] if not isinstance(model_rows, list): return [] return [str(row.get("id")) for row in model_rows if isinstance(row, dict) and row.get("id")] def fetch_endpoint_health(base_url: str, timeout: int = 3) -> dict: request = urllib.request.Request(f"{base_url.rstrip('/')}/health", method="GET") try: with urllib.request.urlopen(request, timeout=timeout) as response: data = json.loads(response.read().decode("utf-8")) except Exception as exc: # noqa: BLE001 - health details are optional. return {"status": "error", "error": str(exc)} return data if isinstance(data, dict) else {} def allowed_endpoint_hosts() -> set[tuple[str, str, int]]: allowed = set(ALLOWED_ENDPOINT_HOSTS) try: runtime_profiles = load_runtime_profiles()["profiles"] except Exception: runtime_profiles = {} for profile in runtime_profiles.values(): urls = list((profile.get("endpoints") or {}).values()) for override in (profile.get("model_overrides") or {}).values(): if isinstance(override, dict) and override.get("base_url"): urls.append(str(override["base_url"])) for url in urls: parsed = urlparse(str(url).strip()) if parsed.scheme in {"http", "https"} and parsed.hostname: port = parsed.port or (443 if parsed.scheme == "https" else 80) allowed.add((parsed.scheme, parsed.hostname.lower(), port)) return allowed def normalize_base_url(base_url: str) -> str: parsed = urlparse(str(base_url).strip()) if parsed.scheme not in {"http", "https"} or not parsed.hostname: raise ValueError("base_url must be an http(s) URL") port = parsed.port or (443 if parsed.scheme == "https" else 80) endpoint = (parsed.scheme, parsed.hostname.lower(), port) allowed_hosts = allowed_endpoint_hosts() if endpoint not in allowed_hosts: allowed = ", ".join(f"{scheme}://{host}:{port}" for scheme, host, port in sorted(allowed_hosts)) raise ValueError(f"base_url is not allowed. Allowed endpoints: {allowed}") netloc = parsed.hostname if parsed.port: netloc = f"{netloc}:{parsed.port}" return urlunparse((parsed.scheme, netloc, "", "", "", "")) def build_1c_rag_prompt(question: str, *, profile_name: str = "general", limit: int | None = None) -> dict: if not DEFAULT_1C_RAG_INDEX.exists(): raise FileNotFoundError( f"1C RAG index is missing: {DEFAULT_1C_RAG_INDEX}. " "Run scripts/prepare_1c_rag_corpus.py and scripts/build_1c_rag_index.py first." ) index = read_json(DEFAULT_1C_RAG_INDEX) profile = resolve_rag_profile(profile_name, question) results = search_lexical_index( index, question, limit=int(limit or profile["limit"]), candidate_limit=int(profile["candidate_limit"]), dedupe_by_document=bool(profile["dedupe_by_document"]), min_score=float(profile["min_score"]), source_types=profile["source_types"], ) context = format_context(results, max_chars=int(profile["max_context_chars"])) prompt = render_prompt(DEFAULT_1C_RAG_PROMPT, context=context, question=question) return { "prompt": prompt, "profile": profile["id"], "context_count": len(results), "sources": [ { "source_path": result["document"].get("source_path"), "title": result["document"].get("title"), "chunk_index": result["document"].get("chunk_index"), "score": round(float(result.get("score") or 0), 4), } for result in results ], } def run_chat_request(base_url: str, model: str, messages: list[dict], temperature: float, max_tokens: int) -> dict: started_at = time.perf_counter() try: answer = call_chat_completion( base_url=base_url, model=model, messages=messages, temperature=temperature, max_tokens=max_tokens, timeout=180, ) return { "status": "answered", "answer": answer, "error": None, "latency_ms": round((time.perf_counter() - started_at) * 1000), } except (urllib.error.URLError, ValueError) as exc: return { "status": "error", "answer": None, "error": str(exc), "latency_ms": round((time.perf_counter() - started_at) * 1000), } def call_audio_transcription( *, base_url: str, audio_base64: str, filename: str, language: str | None = None, task: str | None = None, timeout: int = 300, ) -> dict: url = f"{base_url.rstrip('/')}/v1/audio/transcriptions" payload = { "audio_base64": audio_base64, "filename": filename, } if language: payload["language"] = language if task: payload["task"] = task request = urllib.request.Request( url, data=json.dumps(payload).encode("utf-8"), headers={"Content-Type": "application/json"}, method="POST", ) with urllib.request.urlopen(request, timeout=timeout) as response: return json.loads(response.read().decode("utf-8")) def call_vision_analysis( *, base_url: str, image_base64: str, filename: str, prompt: str, model: str | None = None, max_tokens: int = 512, timeout: int = 300, ) -> dict: url = f"{base_url.rstrip('/')}/v1/vision/analyze" payload = { "image_base64": image_base64, "filename": filename, "prompt": prompt, "model": model, "max_tokens": max_tokens, } request = urllib.request.Request( url, data=json.dumps(payload, ensure_ascii=False).encode("utf-8"), headers={"Content-Type": "application/json"}, method="POST", ) with urllib.request.urlopen(request, timeout=timeout) as response: data = json.loads(response.read().decode("utf-8")) if not isinstance(data, dict): raise ValueError("vision response must be a JSON object") if data.get("error"): error = data["error"] if isinstance(error, dict): raise ValueError(str(error.get("message") or error)) raise ValueError(str(error)) return data def call_image_generation( *, base_url: str, prompt: str, negative_prompt: str | None = None, width: int = 1024, height: int = 1024, steps: int = 28, guidance_scale: float = 6.0, seed: int | None = None, timeout: int = 900, ) -> dict: url = f"{base_url.rstrip('/')}/v1/images/generations" payload: dict[str, object] = { "prompt": prompt, "negative_prompt": negative_prompt or "", "width": width, "height": height, "steps": steps, "guidance_scale": guidance_scale, } if seed is not None: payload["seed"] = seed request = urllib.request.Request( url, data=json.dumps(payload, ensure_ascii=False).encode("utf-8"), headers={"Content-Type": "application/json"}, method="POST", ) with urllib.request.urlopen(request, timeout=timeout) as response: data = json.loads(response.read().decode("utf-8")) if not isinstance(data, dict): raise ValueError("image generation response must be a JSON object") if data.get("error"): error = data["error"] if isinstance(error, dict): raise ValueError(str(error.get("message") or error)) raise ValueError(str(error)) return data def call_image_edit( *, base_url: str, prompt: str, image_base64: str, mask_base64: str, negative_prompt: str | None = None, width: int = 1024, height: int = 1024, steps: int = 28, guidance_scale: float = 6.0, strength: float = 0.95, seed: int | None = None, timeout: int = 900, ) -> dict: url = f"{base_url.rstrip('/')}/v1/images/edits" payload: dict[str, object] = { "prompt": prompt, "negative_prompt": negative_prompt or "", "image_base64": image_base64, "mask_base64": mask_base64, "width": width, "height": height, "steps": steps, "guidance_scale": guidance_scale, "strength": strength, } if seed is not None: payload["seed"] = seed request = urllib.request.Request( url, data=json.dumps(payload, ensure_ascii=False).encode("utf-8"), headers={"Content-Type": "application/json"}, method="POST", ) with urllib.request.urlopen(request, timeout=timeout) as response: data = json.loads(response.read().decode("utf-8")) if not isinstance(data, dict): raise ValueError("image edit response must be a JSON object") if data.get("error"): error = data["error"] if isinstance(error, dict): raise ValueError(str(error.get("message") or error)) raise ValueError(str(error)) return data def submit_image_job(*, base_url: str, operation: str, payload: dict, job_id: str, timeout: int = 30) -> dict: url = f"{base_url.rstrip('/')}/v1/images/jobs" request_payload = {"operation": operation, "payload": payload, "job_id": job_id} request = urllib.request.Request( url, data=json.dumps(request_payload, ensure_ascii=False).encode("utf-8"), headers={"Content-Type": "application/json"}, method="POST", ) with urllib.request.urlopen(request, timeout=timeout) as response: data = json.loads(response.read().decode("utf-8")) if not isinstance(data, dict): raise ValueError("image job response must be a JSON object") if data.get("error"): error = data["error"] if isinstance(error, dict): raise ValueError(str(error.get("message") or error)) raise ValueError(str(error)) return data def fetch_image_job(*, base_url: str, job_id: str, timeout: int = 30) -> dict: url = f"{base_url.rstrip('/')}/v1/images/jobs/{urllib.parse.quote(job_id)}" with urllib.request.urlopen(url, timeout=timeout) as response: data = json.loads(response.read().decode("utf-8")) if not isinstance(data, dict): raise ValueError("image job status must be a JSON object") if data.get("error"): error = data["error"] if isinstance(error, dict): raise ValueError(str(error.get("message") or error)) raise ValueError(str(error)) return data def cancel_image_job(*, base_url: str, job_id: str, timeout: int = 30) -> dict: url = f"{base_url.rstrip('/')}/v1/images/jobs/{urllib.parse.quote(job_id)}/cancel" request = urllib.request.Request( url, data=b"{}", headers={"Content-Type": "application/json"}, method="POST", ) with urllib.request.urlopen(request, timeout=timeout) as response: data = json.loads(response.read().decode("utf-8")) if not isinstance(data, dict): raise ValueError("image cancel response must be a JSON object") return data def finalize_image_job(job: dict) -> dict: job_id = str(job.get("id") or "") if not job_id or job.get("status") != "completed": return job with IMAGE_PROXY_JOB_LOCK: proxy = IMAGE_PROXY_JOBS.get(job_id) if not proxy or proxy.get("artifact"): if proxy and proxy.get("artifact"): job["artifact"] = proxy["artifact"] return job result = job.get("result") if not isinstance(result, dict): return job operation = str(proxy.get("operation") or job.get("operation") or "generate") payload = dict(proxy.get("payload") or {}) artifact = save_image_artifact( report_id=job_id, image_base64=str(result.get("image_base64") or ""), kind=operation, metadata={ "prompt": payload.get("prompt") or "", "negative_prompt": payload.get("negative_prompt") or "", "width": result.get("width"), "height": result.get("height"), "seed": result.get("seed"), "steps": payload.get("steps"), "guidance_scale": payload.get("guidance_scale"), "strength": payload.get("strength"), "base_url": proxy.get("base_url"), "model": result.get("model") or payload.get("model"), }, ) latency_ms = job.get("latency_ms") or result.get("latency_ms") append_report( { "id": job_id, "type": "image_edit" if operation == "edit" else "image_generation", "plugin": "image", "prompt": payload.get("prompt") or "", "negative_prompt": payload.get("negative_prompt") or "", "width": result.get("width"), "height": result.get("height"), "seed": result.get("seed"), "base_url": proxy.get("base_url"), "model": result.get("model") or payload.get("model"), "latency_ms": latency_ms, "artifact": artifact, } ) append_image_job({**artifact, "type": operation, "status": "completed", "latency_ms": latency_ms, "updated_at": now_iso()}) proxy["artifact"] = artifact job["artifact"] = artifact if isinstance(job.get("result"), dict): job["result"]["artifact"] = artifact return job def quality_check_answer(answer: str | None) -> dict: text = (answer or "").strip() if not text: return {"passed": False, "reason": "empty answer"} if len(text) < 12: return {"passed": False, "reason": "answer is too short"} return {"passed": True, "reason": "non-empty answer"} def run_quality_suite(payload: dict) -> dict: catalog = load_catalog() plugin = str(payload.get("plugin") or "text") model_name = str(payload.get("model") or "") base_url = normalize_base_url(payload.get("base_url") or DEFAULT_BASE_URL) tests = payload.get("tests") or TEST_PACKS.get(plugin) or [] if not model_name: route = route_for_plugin(plugin) model_name = route["served_model_name"] if not isinstance(tests, list) or not tests: raise ValueError("quality tests are empty") results = [] for test in tests: if not isinstance(test, dict): raise ValueError("each quality test must be an object") prompt = str(test.get("prompt") or "").strip() if not prompt: continue messages = [ {"role": "system", "content": "Отвечай по-русски, кратко и проверяемо."}, {"role": "user", "content": prompt}, ] result = run_chat_request(base_url, model_name, messages, temperature=0.2, max_tokens=500) guardrail = None if plugin == "1c": result["answer"], guardrail = guard_1c_answer(prompt, result.get("answer")) check = quality_check_answer(result.get("answer")) result.update( { "test_id": test.get("id"), "label": test.get("label"), "prompt": prompt, "passed": check["passed"], "check_reason": check["reason"], "safety_guardrail": guardrail, } ) results.append(result) passed = sum(1 for result in results if result.get("passed")) report_id = append_report( { "type": "quality", "plugin": plugin, "served_model_name": model_name, "base_url": base_url, "passed": passed, "total": len(results), "results": results, } ) return {"report_id": report_id, "plugin": plugin, "model": model_name, "passed": passed, "total": len(results), "results": results} def read_body(handler: BaseHTTPRequestHandler) -> dict: length = int(handler.headers.get("Content-Length") or 0) raw = handler.rfile.read(length) if length else b"{}" data = json.loads(raw.decode("utf-8")) if not isinstance(data, dict): raise ValueError("request body must be a JSON object") return data def write_json_response(handler: BaseHTTPRequestHandler, status: int, payload: dict) -> None: encoded = json.dumps(payload, ensure_ascii=False, indent=2).encode("utf-8") handler.send_response(status) handler.send_header("Content-Type", "application/json; charset=utf-8") handler.send_header("Content-Length", str(len(encoded))) handler.end_headers() handler.wfile.write(encoded) def safe_static_path(static_dir: Path, request_path: str) -> Path: parsed = urlparse(request_path) relative = parsed.path.lstrip("/") or "index.html" candidate = (static_dir / relative).resolve() static_root = static_dir.resolve() if static_root not in candidate.parents and candidate != static_root: raise ValueError("invalid static path") if candidate.is_dir(): candidate = candidate / "index.html" return candidate class ChatHandler(BaseHTTPRequestHandler): static_dir = DEFAULT_STATIC_DIR def log_message(self, format: str, *args: object) -> None: print(f"{self.address_string()} - {format % args}", file=sys.stderr) def do_GET(self) -> None: if self.path.startswith("/health") or self.path.startswith("/api/health"): write_json_response(self, 200, health_status()) return if self.path.startswith("/api/catalog"): write_json_response(self, 200, load_catalog()) return if self.path.startswith("/api/routes"): try: parsed = urlparse(self.path) params = urllib.parse.parse_qs(parsed.query) plugin = (params.get("plugin") or ["text"])[0] task = (params.get("task") or [None])[0] write_json_response(self, 200, route_for_plugin(plugin, task)) except ValueError as exc: write_json_response(self, 400, {"error": str(exc)}) return if self.path.startswith("/api/model-services"): status = health_status() write_json_response(self, 200, {"services": status["model_services"], "routes": status["routes"]}) return if self.path.startswith("/api/benchmark/history"): parsed = urlparse(self.path) params = urllib.parse.parse_qs(parsed.query) limit = int((params.get("limit") or ["10"])[0]) model_id = (params.get("model_id") or [None])[0] plugin = (params.get("plugin") or [None])[0] write_json_response( self, 200, { "benchmarks": latest_runtime_benchmarks( limit=max(1, min(limit, 50)), model_id=model_id, plugin=plugin, ) }, ) return if self.path.startswith("/api/benchmark/report.md"): try: parsed = urlparse(self.path) params = urllib.parse.parse_qs(parsed.query) name = (params.get("name") or [""])[0] path = benchmark_report_path(name) report = read_json(path) content = render_benchmark_markdown(report, path.name).encode("utf-8") except FileNotFoundError: write_json_response(self, 404, {"error": "not found"}) return except (ValueError, json.JSONDecodeError) as exc: write_json_response(self, 400, {"error": str(exc)}) return self.send_response(200) self.send_header("Content-Type", "text/markdown; charset=utf-8") self.send_header("Content-Length", str(len(content))) self.end_headers() self.wfile.write(content) return if self.path.startswith("/api/benchmark/report"): try: parsed = urlparse(self.path) params = urllib.parse.parse_qs(parsed.query) path = benchmark_report_path((params.get("name") or [""])[0]) except FileNotFoundError: write_json_response(self, 404, {"error": "not found"}) return except ValueError as exc: write_json_response(self, 400, {"error": str(exc)}) return content = path.read_bytes() self.send_response(200) self.send_header("Content-Type", "application/json; charset=utf-8") self.send_header("Content-Length", str(len(content))) self.end_headers() self.wfile.write(content) return if self.path.startswith("/api/model-ingest/jobs"): write_json_response(self, 200, {"jobs": latest_model_ingest_jobs()}) return if self.path.startswith("/api/image/gallery"): write_json_response(self, 200, {"images": latest_image_gallery()}) return if urlparse(self.path).path == "/api/image/job": try: parsed = urlparse(self.path) params = urllib.parse.parse_qs(parsed.query) job_id = str((params.get("job_id") or [""])[0]).strip() if not job_id: raise ValueError("job_id is required") with IMAGE_PROXY_JOB_LOCK: proxy = dict(IMAGE_PROXY_JOBS.get(job_id) or {}) base_url = normalize_base_url((params.get("base_url") or [proxy.get("base_url") or route_for_plugin("image")["base_url"]])[0]) job = fetch_image_job(base_url=base_url, job_id=job_id) if job.get("status") == "completed": job = finalize_image_job(job) if job.get("status") in {"cancelled", "error"}: append_image_job( { "id": job_id, "type": job.get("operation") or proxy.get("operation") or "generate", "status": job.get("status"), "error": job.get("error"), "latency_ms": job.get("latency_ms"), "updated_at": now_iso(), } ) write_json_response(self, 200, job) except (TypeError, ValueError, urllib.error.URLError, json.JSONDecodeError) as exc: write_json_response(self, 400, {"error": str(exc)}) return if self.path.startswith("/api/image/jobs"): write_json_response(self, 200, {"jobs": latest_image_jobs()}) return if self.path.startswith("/generated-images/"): relative = self.path.removeprefix("/generated-images/").split("?", 1)[0].lstrip("/") image_path = (IMAGE_GALLERY_DIR / relative).resolve() gallery_root = IMAGE_GALLERY_DIR.resolve() if gallery_root not in image_path.parents or image_path.suffix.lower() != ".png" or not image_path.exists(): write_json_response(self, 404, {"error": "not found"}) return content = image_path.read_bytes() self.send_response(200) self.send_header("Content-Type", "image/png") self.send_header("Content-Length", str(len(content))) self.end_headers() self.wfile.write(content) return try: path = safe_static_path(self.static_dir, self.path) except ValueError as exc: write_json_response(self, 400, {"error": str(exc)}) return if not path.exists(): write_json_response(self, 404, {"error": "not found"}) return content = path.read_bytes() content_type = mimetypes.guess_type(path.name)[0] or "application/octet-stream" if content_type.startswith("text/") or path.suffix in {".js", ".css"}: content_type += "; charset=utf-8" self.send_response(200) self.send_header("Content-Type", content_type) self.send_header("Content-Length", str(len(content))) self.end_headers() self.wfile.write(content) def do_POST(self) -> None: if self.path.startswith("/api/model-ingest/upload"): try: parsed = urlparse(self.path) params = {} for chunk in parsed.query.split("&"): if not chunk or "=" not in chunk: continue key, value = chunk.split("=", 1) params[urllib.parse.unquote_plus(key)] = urllib.parse.unquote_plus(value) content_length = int(self.headers.get("Content-Length") or 0) if content_length <= 0: raise ValueError("upload body is required") filename = sanitize_filename( self.headers.get("X-Model-Filename") or params.get("filename") or "model.bin" ) payload = { "model_id": self.headers.get("X-Model-Id") or params.get("model_id") or Path(filename).stem, "name": self.headers.get("X-Model-Name") or params.get("name") or Path(filename).stem, "plugin": self.headers.get("X-Model-Plugin") or params.get("plugin") or "text", "format": self.headers.get("X-Model-Format") or params.get("format") or Path(filename).suffix.lstrip("."), "quantization": self.headers.get("X-Model-Quantization") or params.get("quantization") or "", "notes": self.headers.get("X-Model-Notes") or params.get("notes") or "", } job, job_dir = create_model_ingest_job(payload, source_kind="upload") destination = job_dir / filename bytes_written = stream_request_to_file(self, destination, content_length) job["status"] = "uploaded" if bytes_written == content_length else "partial" job["updated_at"] = now_iso() job["files"].append({"path": str(destination.relative_to(ROOT)), "size_bytes": bytes_written}) if bytes_written != content_length: job["error"] = f"expected {content_length} bytes, received {bytes_written}" write_model_ingest_metadata(job_dir, job) append_model_ingest_job(job) write_json_response(self, 200, job) except (OSError, ValueError) as exc: write_json_response(self, 400, {"error": str(exc)}) return if self.path.startswith("/api/model-ingest/source"): try: payload = read_body(self) job = import_model_from_source(payload) status = 200 if job.get("status") not in {"failed"} else 400 write_json_response(self, status, job) except (TypeError, ValueError) as exc: write_json_response(self, 400, {"error": str(exc)}) return if self.path.startswith("/api/model-ingest/verify"): try: payload = read_body(self) job = verify_model_ingest_job(str(payload["job_id"])) status = 200 if job.get("status") == "verified" else 400 write_json_response(self, status, job) except (KeyError, TypeError, ValueError) as exc: write_json_response(self, 400, {"error": str(exc)}) return if self.path.startswith("/api/model-ingest/register"): try: payload = read_body(self) job = register_model_ingest_job(str(payload["job_id"])) status = 200 if str(job.get("status")).startswith("registered") else 400 write_json_response(self, status, job) except (KeyError, TypeError, ValueError) as exc: write_json_response(self, 400, {"error": str(exc)}) return if self.path.startswith("/api/models"): try: payload = read_body(self) base_url = normalize_base_url(payload.get("base_url") or DEFAULT_BASE_URL) expected_model = payload.get("model") started_at = time.perf_counter() models = fetch_endpoint_models(base_url) latency_ms = round((time.perf_counter() - started_at) * 1000) write_json_response( self, 200, { "base_url": base_url, "models": models, "latency_ms": latency_ms, "expected_model": expected_model, "available": expected_model in models if expected_model else None, }, ) except (TypeError, ValueError, urllib.error.URLError) as exc: write_json_response(self, 400, {"error": str(exc)}) return if self.path.startswith("/api/route"): try: payload = read_body(self) write_json_response(self, 200, route_for_plugin(str(payload.get("plugin") or "text"), payload.get("task"))) except (TypeError, ValueError) as exc: write_json_response(self, 400, {"error": str(exc)}) return if self.path.startswith("/api/quality/run"): try: write_json_response(self, 200, run_quality_suite(read_body(self))) except (TypeError, ValueError) as exc: write_json_response(self, 400, {"error": str(exc)}) return if self.path.startswith("/api/service-control"): try: result = control_model_service(read_body(self)) status = 200 if result["ok"] else 500 write_json_response(self, status, result) except (TypeError, ValueError) as exc: write_json_response(self, 400, {"error": str(exc)}) return if self.path.startswith("/api/benchmark/preflight"): try: result = runtime_benchmark_preflight(read_body(self)) status = 200 if result["ready"] else 409 write_json_response(self, status, result) except (TypeError, ValueError) as exc: write_json_response(self, 400, {"error": str(exc)}) return if self.path.startswith("/api/benchmark/runtime"): try: result = run_runtime_benchmark(read_body(self)) status = 200 if result["ok"] else 500 write_json_response(self, status, result) except (TypeError, ValueError) as exc: write_json_response(self, 400, {"error": str(exc)}) return if self.path.startswith("/api/feedback"): try: payload = read_body(self) rating = str(payload["rating"]) if rating not in {"ok", "bad", "needs_review"}: raise ValueError("rating must be ok, bad, or needs_review") report_id = append_report( { "type": "feedback", "parent_id": payload.get("report_id"), "rating": rating, "note": payload.get("note") or "", } ) write_json_response(self, 200, {"report_id": report_id}) except (KeyError, TypeError, ValueError) as exc: write_json_response(self, 400, {"error": str(exc)}) return if self.path.startswith("/api/1c/rag-prompt"): try: payload = read_body(self) question = str(payload["question"]).strip() if not question: raise ValueError("question is required") result = build_1c_rag_prompt( question, profile_name=str(payload.get("profile") or "auto"), limit=int(payload["limit"]) if payload.get("limit") is not None else None, ) report_id = append_report( { "type": "rag_prompt", "plugin": "1c", "profile": result["profile"], "question": question, "context_count": result["context_count"], "sources": result["sources"], } ) result["report_id"] = report_id write_json_response(self, 200, result) except (KeyError, TypeError, ValueError, FileNotFoundError) as exc: write_json_response(self, 400, {"error": str(exc)}) return if self.path.startswith("/api/1c/rag-chat"): report_id = str(uuid.uuid4()) started_at = time.perf_counter() try: payload = read_body(self) question = str(payload["question"]).strip() if not question: raise ValueError("question is required") base_url = normalize_base_url(payload.get("base_url") or DEFAULT_BASE_URL) model = str(payload.get("model") or route_for_plugin("1c")["served_model_name"]) rag = build_1c_rag_prompt( question, profile_name=str(payload.get("profile") or "auto"), limit=int(payload["limit"]) if payload.get("limit") is not None else None, ) messages = [ {"role": "system", "content": "Ты помощник по 1С. Не предлагай прямые изменения живой базы без проверки, бэкапа и согласования."}, {"role": "user", "content": rag["prompt"]}, ] answer = call_chat_completion( base_url=base_url, model=model, messages=messages, temperature=float(payload.get("temperature", 0.2)), max_tokens=int(payload.get("max_tokens", 1000)), timeout=180, ) answer, guardrail = guard_1c_answer(question, answer) latency_ms = round((time.perf_counter() - started_at) * 1000) append_report( { "id": report_id, "type": "1c_rag_chat", "plugin": "1c", "profile": rag["profile"], "question": question, "answer": answer, "sources": rag["sources"], "context_count": rag["context_count"], "served_model_name": model, "base_url": base_url, "latency_ms": latency_ms, "safety_guardrail": guardrail, } ) write_json_response(self, 200, {**rag, "answer": answer, "report_id": report_id, "latency_ms": latency_ms, "safety_guardrail": guardrail}) except (KeyError, TypeError, ValueError, FileNotFoundError, urllib.error.URLError) as exc: latency_ms = round((time.perf_counter() - started_at) * 1000) append_report({"id": report_id, "type": "1c_rag_chat", "status": "error", "error": str(exc), "latency_ms": latency_ms}) write_json_response(self, 400, {"error": str(exc), "report_id": report_id, "latency_ms": latency_ms}) return if self.path.startswith("/api/audio/transcribe"): report_id = str(uuid.uuid4()) started_at = time.perf_counter() try: payload = read_body(self) base_url = normalize_base_url(payload.get("base_url") or route_for_plugin("audio")["base_url"]) audio_base64 = str(payload.get("audio_base64") or "") filename = sanitize_filename(payload.get("filename") or "audio.wav", fallback="audio.wav") if not audio_base64: raise ValueError("audio_base64 is required") result = call_audio_transcription( base_url=base_url, audio_base64=audio_base64, filename=filename, language=str(payload.get("language") or "") or None, task=str(payload.get("task") or "") or None, ) latency_ms = round((time.perf_counter() - started_at) * 1000) append_report( { "id": report_id, "type": "audio_transcription", "plugin": "audio", "filename": filename, "bytes": result.get("bytes"), "text": result.get("text"), "base_url": base_url, "model": result.get("model") or payload.get("model"), "latency_ms": latency_ms, } ) write_json_response(self, 200, {**result, "report_id": report_id, "latency_ms": latency_ms}) except (TypeError, ValueError, urllib.error.URLError, json.JSONDecodeError) as exc: latency_ms = round((time.perf_counter() - started_at) * 1000) append_report({"id": report_id, "type": "audio_transcription", "status": "error", "error": str(exc), "latency_ms": latency_ms}) write_json_response(self, 400, {"error": str(exc), "report_id": report_id, "latency_ms": latency_ms}) return if self.path.startswith("/api/video/analyze"): report_id = str(uuid.uuid4()) started_at = time.perf_counter() try: payload = read_body(self) base_url = normalize_base_url(payload.get("base_url") or route_for_plugin("video")["base_url"]) image_base64 = str(payload.get("image_base64") or "") filename = sanitize_filename(payload.get("filename") or "image.png", fallback="image.png") prompt = str(payload.get("prompt") or "").strip() or "Опиши изображение и перечисли важные детали." if not image_base64: raise ValueError("image_base64 is required") result = call_vision_analysis( base_url=base_url, image_base64=image_base64, filename=filename, prompt=prompt, model=str(payload.get("model") or "") or None, max_tokens=int(payload.get("max_tokens") or 512), ) latency_ms = round((time.perf_counter() - started_at) * 1000) append_report( { "id": report_id, "type": "vision_analysis", "plugin": "video", "filename": filename, "bytes": result.get("bytes"), "width": result.get("width"), "height": result.get("height"), "prompt": prompt, "text": result.get("text"), "base_url": base_url, "model": result.get("model") or payload.get("model"), "latency_ms": latency_ms, } ) write_json_response(self, 200, {**result, "report_id": report_id, "latency_ms": latency_ms}) except (TypeError, ValueError, urllib.error.URLError, json.JSONDecodeError) as exc: latency_ms = round((time.perf_counter() - started_at) * 1000) append_report({"id": report_id, "type": "vision_analysis", "status": "error", "error": str(exc), "latency_ms": latency_ms}) write_json_response(self, 400, {"error": str(exc), "report_id": report_id, "latency_ms": latency_ms}) return if self.path.startswith("/api/image/submit"): report_id = str(uuid.uuid4()) started_at = time.perf_counter() try: payload = read_body(self) operation = str(payload.get("operation") or payload.get("mode") or "generate") if operation not in {"generate", "edit"}: raise ValueError("operation must be generate or edit") model_id = str(payload.get("model_id") or "").strip() model_mode = str(payload.get("model_mode") or "").strip() image_route = route_for_model("image", model_id, "image-editing" if operation == "edit" else None) base_url = normalize_base_url(payload.get("base_url") or image_route["base_url"]) prompt = str(payload.get("prompt") or "").strip() if not prompt: raise ValueError("prompt is required") seed_value = payload.get("seed") seed = int(seed_value) if seed_value not in {None, ""} else None job_payload: dict[str, object] = { "prompt": prompt, "negative_prompt": str(payload.get("negative_prompt") or "") or "", "width": int(payload.get("width") or 1024), "height": int(payload.get("height") or 1024), "steps": int(payload.get("steps") or 28), "guidance_scale": float(payload.get("guidance_scale") or 6.0), "model": payload.get("model") or image_route["served_model_name"], "model_id": model_id or image_route["model"].get("id"), "model_mode": model_mode or image_route["model"].get("id"), } if seed is not None: job_payload["seed"] = seed if operation == "edit": image_base64 = str(payload.get("image_base64") or "") mask_base64 = str(payload.get("mask_base64") or "") if not image_base64: raise ValueError("image_base64 is required") if not mask_base64: raise ValueError("mask_base64 is required") job_payload["image_base64"] = image_base64 job_payload["mask_base64"] = mask_base64 job_payload["strength"] = float(payload.get("strength") or 0.95) expected_model = str(job_payload.get("model") or image_route["served_model_name"] or "") endpoint_models = fetch_endpoint_models(base_url, timeout=5) if expected_model and expected_model not in endpoint_models: raise ValueError( f"image endpoint {base_url} serves {endpoint_models or 'no models'}, " f"not selected model {expected_model}" ) append_image_job({"id": report_id, "type": operation, "status": "submitted", "created_at": now_iso()}) submitted = submit_image_job(base_url=base_url, operation=operation, payload=job_payload, job_id=report_id) with IMAGE_PROXY_JOB_LOCK: IMAGE_PROXY_JOBS[report_id] = { "id": report_id, "operation": operation, "base_url": base_url, "model_id": job_payload.get("model_id"), "model_mode": job_payload.get("model_mode"), "payload": job_payload, "submitted_at": now_iso(), "started_perf": started_at, } write_json_response(self, 202, {**submitted, "report_id": report_id, "base_url": base_url}) except (TypeError, ValueError, urllib.error.URLError, json.JSONDecodeError) as exc: latency_ms = round((time.perf_counter() - started_at) * 1000) append_report({"id": report_id, "type": "image_job_submit", "status": "error", "error": str(exc), "latency_ms": latency_ms}) append_image_job({"id": report_id, "type": "image", "status": "error", "error": str(exc), "latency_ms": latency_ms, "updated_at": now_iso()}) write_json_response(self, 400, {"error": str(exc), "report_id": report_id, "latency_ms": latency_ms}) return if self.path.startswith("/api/image/cancel"): try: payload = read_body(self) job_id = str(payload.get("job_id") or payload.get("report_id") or "").strip() if not job_id: raise ValueError("job_id is required") with IMAGE_PROXY_JOB_LOCK: proxy = dict(IMAGE_PROXY_JOBS.get(job_id) or {}) base_url = normalize_base_url(payload.get("base_url") or proxy.get("base_url") or route_for_plugin("image")["base_url"]) result = cancel_image_job(base_url=base_url, job_id=job_id) append_image_job( { "id": job_id, "type": result.get("operation") or proxy.get("operation") or "generate", "status": result.get("status") or "cancel_requested", "updated_at": now_iso(), } ) write_json_response(self, 200, result) except (TypeError, ValueError, urllib.error.URLError, json.JSONDecodeError) as exc: write_json_response(self, 400, {"error": str(exc)}) return if self.path.startswith("/api/image/generate"): report_id = str(uuid.uuid4()) started_at = time.perf_counter() append_image_job({"id": report_id, "type": "generate", "status": "running", "created_at": now_iso()}) try: payload = read_body(self) base_url = normalize_base_url(payload.get("base_url") or route_for_plugin("image")["base_url"]) prompt = str(payload.get("prompt") or "").strip() if not prompt: raise ValueError("prompt is required") seed_value = payload.get("seed") seed = int(seed_value) if seed_value not in {None, ""} else None result = call_image_generation( base_url=base_url, prompt=prompt, negative_prompt=str(payload.get("negative_prompt") or "") or None, width=int(payload.get("width") or 1024), height=int(payload.get("height") or 1024), steps=int(payload.get("steps") or 28), guidance_scale=float(payload.get("guidance_scale") or 6.0), seed=seed, ) latency_ms = round((time.perf_counter() - started_at) * 1000) artifact = save_image_artifact( report_id=report_id, image_base64=str(result.get("image_base64") or ""), kind="generate", metadata={ "prompt": prompt, "negative_prompt": payload.get("negative_prompt") or "", "width": result.get("width"), "height": result.get("height"), "seed": result.get("seed"), "steps": int(payload.get("steps") or 28), "guidance_scale": float(payload.get("guidance_scale") or 6.0), "base_url": base_url, "model": result.get("model") or payload.get("model"), }, ) append_report( { "id": report_id, "type": "image_generation", "plugin": "image", "prompt": prompt, "negative_prompt": payload.get("negative_prompt") or "", "width": result.get("width"), "height": result.get("height"), "seed": result.get("seed"), "base_url": base_url, "model": result.get("model") or payload.get("model"), "latency_ms": latency_ms, "artifact": artifact, } ) append_image_job({**artifact, "status": "completed", "latency_ms": latency_ms}) write_json_response(self, 200, {**result, "report_id": report_id, "latency_ms": latency_ms, "artifact": artifact}) except (TypeError, ValueError, urllib.error.URLError, json.JSONDecodeError) as exc: latency_ms = round((time.perf_counter() - started_at) * 1000) append_report({"id": report_id, "type": "image_generation", "status": "error", "error": str(exc), "latency_ms": latency_ms}) append_image_job({"id": report_id, "type": "generate", "status": "error", "error": str(exc), "latency_ms": latency_ms, "updated_at": now_iso()}) write_json_response(self, 400, {"error": str(exc), "report_id": report_id, "latency_ms": latency_ms}) return if self.path.startswith("/api/image/edit"): report_id = str(uuid.uuid4()) started_at = time.perf_counter() append_image_job({"id": report_id, "type": "edit", "status": "running", "created_at": now_iso()}) try: payload = read_body(self) base_url = normalize_base_url(payload.get("base_url") or route_for_plugin("image", "image-editing")["base_url"]) prompt = str(payload.get("prompt") or "").strip() image_base64 = str(payload.get("image_base64") or "") mask_base64 = str(payload.get("mask_base64") or "") if not prompt: raise ValueError("prompt is required") if not image_base64: raise ValueError("image_base64 is required") if not mask_base64: raise ValueError("mask_base64 is required") seed_value = payload.get("seed") seed = int(seed_value) if seed_value not in {None, ""} else None result = call_image_edit( base_url=base_url, prompt=prompt, image_base64=image_base64, mask_base64=mask_base64, negative_prompt=str(payload.get("negative_prompt") or "") or None, width=int(payload.get("width") or 1024), height=int(payload.get("height") or 1024), steps=int(payload.get("steps") or 28), guidance_scale=float(payload.get("guidance_scale") or 6.0), strength=float(payload.get("strength") or 0.95), seed=seed, ) latency_ms = round((time.perf_counter() - started_at) * 1000) artifact = save_image_artifact( report_id=report_id, image_base64=str(result.get("image_base64") or ""), kind="edit", metadata={ "prompt": prompt, "negative_prompt": payload.get("negative_prompt") or "", "width": result.get("width"), "height": result.get("height"), "seed": result.get("seed"), "steps": int(payload.get("steps") or 28), "guidance_scale": float(payload.get("guidance_scale") or 6.0), "strength": float(payload.get("strength") or 0.95), "base_url": base_url, "model": result.get("model") or payload.get("model"), }, ) append_report( { "id": report_id, "type": "image_edit", "plugin": "image", "prompt": prompt, "negative_prompt": payload.get("negative_prompt") or "", "width": result.get("width"), "height": result.get("height"), "seed": result.get("seed"), "base_url": base_url, "model": result.get("model") or payload.get("model"), "latency_ms": latency_ms, "artifact": artifact, } ) append_image_job({**artifact, "status": "completed", "latency_ms": latency_ms}) write_json_response(self, 200, {**result, "report_id": report_id, "latency_ms": latency_ms, "artifact": artifact}) except (TypeError, ValueError, urllib.error.URLError, json.JSONDecodeError) as exc: latency_ms = round((time.perf_counter() - started_at) * 1000) append_report({"id": report_id, "type": "image_edit", "status": "error", "error": str(exc), "latency_ms": latency_ms}) append_image_job({"id": report_id, "type": "edit", "status": "error", "error": str(exc), "latency_ms": latency_ms, "updated_at": now_iso()}) write_json_response(self, 400, {"error": str(exc), "report_id": report_id, "latency_ms": latency_ms}) return if self.path.startswith("/api/compare"): try: payload = read_body(self) base_url = normalize_base_url(payload.get("base_url") or DEFAULT_BASE_URL) models = payload.get("models") or [] messages = payload.get("messages") or [] temperature = float(payload.get("temperature", 0.2)) max_tokens = int(payload.get("max_tokens", 1000)) if not isinstance(models, list) or not models: raise ValueError("models must be a non-empty list") if not isinstance(messages, list) or not messages: raise ValueError("messages must be a non-empty list") results = [] for model in models: if not isinstance(model, dict): raise ValueError("each model must be an object") served_model = str(model["served_model_name"]) result = run_chat_request(base_url, served_model, messages, temperature, max_tokens) guardrail = None if payload.get("plugin") == "1c": result["answer"], guardrail = guard_1c_answer(latest_user_message(messages), result.get("answer")) result.update( { "registry_model_id": model.get("id"), "name": model.get("name"), "served_model_name": served_model, "safety_guardrail": guardrail, } ) results.append(result) report_id = append_report( { "type": "compare", "plugin": payload.get("plugin"), "test_id": payload.get("test_id"), "base_url": base_url, "messages": messages, "temperature": temperature, "max_tokens": max_tokens, "results": results, } ) write_json_response(self, 200, {"report_id": report_id, "results": results}) except (KeyError, TypeError, ValueError) as exc: write_json_response(self, 400, {"error": str(exc)}) return if not self.path.startswith("/api/chat"): write_json_response(self, 404, {"error": "not found"}) return report_id = str(uuid.uuid4()) started_at = time.perf_counter() try: payload = read_body(self) base_url = normalize_base_url(payload.get("base_url") or DEFAULT_BASE_URL) model = str(payload["model"]) registry_model_id = payload.get("registry_model_id") plugin = payload.get("plugin") test_id = payload.get("test_id") messages = payload.get("messages") or [] temperature = float(payload.get("temperature", 0.2)) max_tokens = int(payload.get("max_tokens", 1000)) if not isinstance(messages, list) or not messages: raise ValueError("messages must be a non-empty list") answer = call_chat_completion( base_url=base_url, model=model, messages=messages, temperature=temperature, max_tokens=max_tokens, timeout=180, ) guardrail = None if plugin == "1c": answer, guardrail = guard_1c_answer(latest_user_message(messages), answer) except (KeyError, TypeError, ValueError, urllib.error.URLError) as exc: latency_ms = round((time.perf_counter() - started_at) * 1000) append_report( { "id": report_id, "type": "chat", "status": "error", "error": str(exc), "payload": locals().get("payload", {}), "latency_ms": latency_ms, } ) write_json_response(self, 400, {"error": str(exc), "report_id": report_id, "latency_ms": latency_ms}) return latency_ms = round((time.perf_counter() - started_at) * 1000) append_report( { "id": report_id, "type": "chat", "status": "answered", "plugin": plugin, "registry_model_id": registry_model_id, "served_model_name": model, "base_url": base_url, "test_id": test_id, "messages": messages, "answer": answer, "temperature": temperature, "max_tokens": max_tokens, "latency_ms": latency_ms, "rating": None, "safety_guardrail": guardrail, } ) write_json_response(self, 200, {"answer": answer, "report_id": report_id, "latency_ms": latency_ms, "safety_guardrail": guardrail}) def main() -> int: parser = argparse.ArgumentParser(description="Run a local model chat test UI.") parser.add_argument("--host", default=DEFAULT_HOST) parser.add_argument("--port", type=int, default=DEFAULT_PORT) parser.add_argument("--static-dir", type=Path, default=DEFAULT_STATIC_DIR) args = parser.parse_args() ChatHandler.static_dir = args.static_dir server = ThreadingHTTPServer((args.host, args.port), ChatHandler) print(f"Model chat UI: http://{args.host}:{args.port}") print(f"Static dir: {args.static_dir}") try: server.serve_forever() except KeyboardInterrupt: print("\nStopping model chat UI.") return 0 if __name__ == "__main__": raise SystemExit(main())