from __future__ import annotations import argparse import datetime as dt import json import mimetypes import re import subprocess import sys import threading import uuid from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from pathlib import Path from urllib.parse import parse_qs, unquote, urlparse from urllib.request import Request, urlopen from build_1c_agent_intake import build_intake from common import search_lexical_index from normalize_1c_its_docs import TextExtractor, clean_its_text, decode_html from route_1c_question import route_question from resolve_1c_fact import member_area, resolve_from_route_index, resolve_from_snapshot, split_fact_path ROOT = Path(__file__).resolve().parents[1] DEFAULT_HOST = "127.0.0.1" DEFAULT_PORT = 8770 STATIC_DIR = ROOT / "tools" / "management-console" REPORTS = ROOT / "reports" JOB_LOCK = threading.Lock() JOBS: dict[str, dict] = {} RAG_INDEX = ROOT / "plugins" / "1c" / "datasets" / "prepared" / "rag_index.json" DEFAULT_1C_ROUTE_INDEX = ROOT / "reports" / "1c-sql" / "upo" / "unified-object-route-index.json" OFFICIAL_DOCS = ROOT / "plugins" / "1c" / "rag" / "official-docs" OFFICIAL_NORMALIZED_MANIFEST = OFFICIAL_DOCS / "normalized" / "manifest.json" OFFICIAL_STATIC_DIR = OFFICIAL_DOCS / "static" OFFICIAL_1C_SOURCE_TYPES = [ "official_1c_its_admin_guide", "official_1c_its_admin_methodical_support", "official_1c_its_bsp_doc", "official_1c_its_client_server_admin_guide", "official_1c_its_dev_section", "official_1c_its_developer_book", "official_1c_its_developer_guide", "official_1c_its_developer_methodical_support", "official_1c_its_development_standards", "official_1c_its_fresh_configuration_guide", "official_1c_its_fresh_developer_guide", "official_1c_its_glossary", "official_1c_its_integration_library_doc", "official_1c_its_methodical_support", "official_1c_its_platform_doc", "official_1c_its_taxi_user_guide", "official_1c_its_user_guide", ] COMMANDS: dict[str, dict] = { "artifact_manifest": { "label": "Build artifact manifest", "command": [sys.executable, "scripts/build_llm_artifact_manifest.py", "--output", "reports/llm-artifact-manifest.json"], "timeout": 120, "category": "artifacts", }, "artifact_check": { "label": "Check artifact manifest", "command": [sys.executable, "scripts/check_llm_artifact_manifest.py", "--manifest", "reports/llm-artifact-manifest.json", "--output", "reports/llm-artifact-check.json"], "timeout": 120, "category": "artifacts", }, "model_storage": { "label": "Check model storage", "command": [sys.executable, "scripts/check_model_storage.py", "--report", "reports/model-storage.json"], "timeout": 120, "category": "models", }, "q6_download_start": { "label": "Start Q6 1C model download", "command": ["powershell", "-ExecutionPolicy", "Bypass", "-File", "scripts/download_1c_q6_model.ps1", "-Action", "start"], "timeout": 30, "category": "models", }, "q6_download_status": { "label": "Check Q6 1C model download", "command": ["powershell", "-ExecutionPolicy", "Bypass", "-File", "scripts/download_1c_q6_model.ps1", "-Action", "status"], "timeout": 30, "category": "models", }, "q6_download_stop": { "label": "Stop Q6 1C model download", "command": ["powershell", "-ExecutionPolicy", "Bypass", "-File", "scripts/download_1c_q6_model.ps1", "-Action", "stop"], "timeout": 30, "category": "models", }, "q6_service_start": { "label": "Start Q6 1C model service", "command": ["powershell", "-ExecutionPolicy", "Bypass", "-File", "scripts/manage_gpu_q6_service.ps1", "-Action", "start"], "timeout": 60, "category": "models", }, "q6_service_stop": { "label": "Stop Q6 1C model service", "command": ["powershell", "-ExecutionPolicy", "Bypass", "-File", "scripts/manage_gpu_q6_service.ps1", "-Action", "stop"], "timeout": 60, "category": "models", }, "q6_service_restart": { "label": "Restart Q6 1C model service", "command": ["powershell", "-ExecutionPolicy", "Bypass", "-File", "scripts/manage_gpu_q6_service.ps1", "-Action", "restart"], "timeout": 90, "category": "models", }, "q6_service_status": { "label": "Check Q6 1C model service", "command": ["powershell", "-ExecutionPolicy", "Bypass", "-File", "scripts/manage_gpu_q6_service.ps1", "-Action", "status"], "timeout": 60, "category": "models", }, "1c_plugin": { "label": "Check 1C plugin", "command": [sys.executable, "scripts/check_1c_plugin.py", "--report", "reports/1c-plugin-health.json"], "timeout": 180, "category": "1c", }, "1c_question_router": { "label": "Check 1C question router", "command": [sys.executable, "scripts/check_1c_question_router.py", "--output", "reports/1c-question-router.json", "--print"], "timeout": 120, "category": "1c", }, "1c_agent_intake": { "label": "Check 1C agent intake", "command": [sys.executable, "scripts/check_1c_agent_intake.py", "--output", "reports/1c-agent-intake.json", "--print"], "timeout": 120, "category": "1c", }, "1c_rag_freshness": { "label": "Check 1C RAG freshness", "command": [sys.executable, "scripts/check_1c_rag_freshness.py", "--print"], "timeout": 60, "category": "rag", }, "1c_rag_source_governance": { "label": "Check 1C RAG source governance", "command": [sys.executable, "scripts/check_1c_rag_source_governance.py", "--output", "reports/1c-rag-source-governance.json", "--print"], "timeout": 60, "category": "rag", }, "official_docs_private": { "label": "Check official docs private artifacts", "command": [sys.executable, "scripts/check_1c_official_docs_private_artifacts.py", "--output", "reports/1c-official-docs-private-check.json"], "timeout": 60, "category": "rag", }, "official_docs_access": { "label": "Check 1C:ITS protected access", "command": ["powershell", "-NoProfile", "-ExecutionPolicy", "Bypass", "-Command", "$ErrorActionPreference='Stop'; Add-Type -AssemblyName System.Security; $p='plugins/1c/rag/official-docs/.local/its-cookie.dpapi.txt'; $b=[Convert]::FromBase64String((Get-Content -LiteralPath $p -Raw -Encoding UTF8).Trim()); $plain=[System.Security.Cryptography.ProtectedData]::Unprotect($b,$null,[System.Security.Cryptography.DataProtectionScope]::CurrentUser); $env:ONEC_ITS_COOKIE=[System.Text.Encoding]::UTF8.GetString($plain); python -X utf8 scripts/check_1c_its_access.py --print"], "timeout": 120, "category": "rag", }, "official_docs_cookie_normalizer": { "label": "Check 1C:ITS cookie normalizer", "command": [sys.executable, "scripts/check_1c_its_cookie_normalizer.py", "--print"], "timeout": 60, "category": "rag", }, "official_docs_quality": { "label": "Check official docs quality", "command": [sys.executable, "scripts/check_1c_official_docs_quality.py", "--output", "reports/1c-official-docs-quality.json", "--print"], "timeout": 60, "category": "rag", }, "official_docs_ingestion_logic": { "label": "Check official docs ingestion logic", "command": [sys.executable, "scripts/check_1c_its_ingestion_logic.py"], "timeout": 60, "category": "rag", }, "official_docs_fetch_plan": { "label": "Inspect official docs fetch plan", "command": [sys.executable, "scripts/inspect_1c_its_fetch_plan.py", "--output", "reports/1c-official-docs-fetch-plan.json", "--print"], "timeout": 60, "category": "rag", }, "official_docs_discover_starts": { "label": "Discover official docs start links", "command": ["powershell", "-NoProfile", "-ExecutionPolicy", "Bypass", "-Command", "$ErrorActionPreference='Stop'; Add-Type -AssemblyName System.Security; $p='plugins/1c/rag/official-docs/.local/its-cookie.dpapi.txt'; $b=[Convert]::FromBase64String((Get-Content -LiteralPath $p -Raw -Encoding UTF8).Trim()); $plain=[System.Security.Cryptography.ProtectedData]::Unprotect($b,$null,[System.Security.Cryptography.DataProtectionScope]::CurrentUser); $env:ONEC_ITS_COOKIE=[System.Text.Encoding]::UTF8.GetString($plain); python -X utf8 scripts/discover_1c_its_start_links.py --output plugins/1c/rag/official-docs/start-links.json --print"], "timeout": 120, "category": "rag", }, "official_docs_start_coverage": { "label": "Check official docs start coverage", "command": [sys.executable, "scripts/check_1c_its_start_link_coverage.py", "--output", "reports/1c-official-docs-start-coverage.json", "--print"], "timeout": 60, "category": "rag", }, "official_docs_platform_versions": { "label": "Build official docs platform version catalog", "command": [sys.executable, "scripts/build_1c_its_platform_version_catalog.py", "--print"], "timeout": 60, "category": "rag", }, "official_docs_download_media": { "label": "Download official docs media", "command": [sys.executable, "scripts/download_1c_its_media.py", "--output-manifest", "plugins/1c/rag/official-docs/media/manifest.json"], "timeout": 600, "category": "rag", }, "official_docs_static_site": { "label": "Build official docs static site", "command": [sys.executable, "scripts/build_1c_its_static_site.py"], "timeout": 180, "category": "rag", }, "official_docs_static_quality": { "label": "Check official docs static site", "command": [sys.executable, "scripts/check_1c_its_static_site.py", "--output", "reports/1c-official-docs-static-check.json", "--print"], "timeout": 60, "category": "rag", }, "official_docs_reload_fresh": { "label": "Reload official docs without resume", "command": ["powershell", "-NoProfile", "-ExecutionPolicy", "Bypass", "-File", "scripts/run_1c_its_docs_pipeline.ps1", "-MaxPages", "2000", "-NoResume"], "timeout": 7200, "category": "rag", }, "powershell_scripts": { "label": "Validate PowerShell scripts", "command": [sys.executable, "scripts/check_powershell_scripts.py"], "timeout": 120, "category": "repo", }, "its_cookie_prompt": { "label": "Open 1C:ITS cookie dialog", "command": ["powershell", "-NoProfile", "-ExecutionPolicy", "Bypass", "-File", "scripts/set_1c_its_cookie.ps1"], "timeout": 600, "category": "settings", }, } def now_iso() -> str: return dt.datetime.now(dt.UTC).isoformat() def load_json(path: Path) -> dict: if not path.exists(): return {} try: return json.loads(path.read_text(encoding="utf-8-sig")) except (json.JSONDecodeError, OSError): return {} def file_info(path: Path) -> dict: if not path.exists(): return {"path": str(path), "exists": False} stat = path.stat() return { "path": str(path), "exists": True, "size_bytes": stat.st_size, "mtime": dt.datetime.fromtimestamp(stat.st_mtime, dt.UTC).isoformat(), } def compact_artifact_manifest() -> dict: manifest = load_json(REPORTS / "llm-artifact-manifest.json") artifacts = [] for item in manifest.get("artifacts") or []: artifacts.append( { "name": item.get("name"), "exists": item.get("exists"), "files": item.get("file_count"), "size_bytes": item.get("total_size_bytes"), "secret_like_files": len(item.get("secret_like_files") or []), "path": item.get("path"), } ) return { "created_at": manifest.get("created_at"), "counts": manifest.get("counts") or {}, "artifacts": artifacts, "file": file_info(REPORTS / "llm-artifact-manifest.json"), } def compact_model_storage() -> dict: report = load_json(REPORTS / "model-storage.json") q6_download = load_json(REPORTS / "qwen3-coder-q6-download-status.json") models = [] for item in report.get("models") or []: models.append( { "id": item.get("id"), "type": item.get("type"), "runtime": item.get("runtime"), "quantization": item.get("quantization"), "served_model_name": item.get("served_model_name"), "filename": item.get("filename"), "status": item.get("status"), "reason": item.get("reason"), "required": item.get("required"), "storage_path": item.get("storage_path"), } ) return { "status": report.get("status"), "counts": report.get("counts") or {}, "models": models, "q6_download": q6_download, "q6_service": q6_service_status(), "file": file_info(REPORTS / "model-storage.json"), } def q6_service_status() -> dict: base_url = "http://docker-gpu.cin.su:8081" expected_model = "qwen3-coder-1c-q6" status = { "state": "unknown", "base_url": base_url, "openai_base_url": f"{base_url}/v1", "openai_api_key_hint": "dummy", "expected_model": expected_model, "models": [], "available": False, "container": "llm-llama-qwen3-coder-q6-test", "container_status": "", "error": "", } try: result = subprocess.run( [ "docker", "--host", "ssh://docker-gpu", "ps", "-a", "--filter", "name=^llm-llama-qwen3-coder-q6-test$", "--format", "{{.Status}}", ], cwd=ROOT, capture_output=True, text=True, timeout=8, check=False, ) status["container_status"] = result.stdout.strip() except (OSError, subprocess.TimeoutExpired) as exc: status["container_status"] = f"docker status error: {exc}" try: request = Request(f"{base_url}/v1/models", method="GET") with urlopen(request, timeout=5) as response: payload = json.loads(response.read().decode("utf-8")) models = [] for item in payload.get("data") or payload.get("models") or []: if isinstance(item, str): models.append(item) elif isinstance(item, dict): models.append(str(item.get("id") or item.get("model") or item.get("name") or "")) status["models"] = [model for model in models if model] status["available"] = expected_model in status["models"] status["state"] = "ok" if status["available"] else "warning" except Exception as exc: # pragma: no cover - depends on live GPU host status["state"] = "error" status["error"] = str(exc) return status def compact_1c_health() -> dict: report = load_json(REPORTS / "1c-plugin-health.json") return { "status": report.get("status"), "failed_checks": report.get("failed_checks") or [], "manifest": ((report.get("checks") or {}).get("manifest") or {}), "file": file_info(REPORTS / "1c-plugin-health.json"), } def compact_rag() -> dict: prepared = ROOT / "plugins" / "1c" / "datasets" / "prepared" official = ROOT / "plugins" / "1c" / "rag" / "official-docs" private_check = load_json(REPORTS / "1c-official-docs-private-check.json") quality_check = load_json(REPORTS / "1c-official-docs-quality.json") access_check = load_json(REPORTS / "1c-its-access-check.json") static_check = load_json(REPORTS / "1c-official-docs-static-check.json") start_coverage = load_json(REPORTS / "1c-official-docs-start-coverage.json") platform_versions = load_json(official / "platform-versions.json") fetch_progress = load_json(official / "raw" / "progress.json") fetch_manifest = load_json(official / "raw" / "manifest.json") return { "corpus": file_info(prepared / "rag_corpus.jsonl"), "index": file_info(prepared / "rag_index.json"), "manifest": file_info(prepared / "rag_manifest.json"), "official_raw": file_info(official / "raw" / "manifest.json"), "official_start_links": file_info(official / "start-links.json"), "official_platform_versions": { "counts": platform_versions.get("counts") or {}, "defaults": platform_versions.get("defaults") or {}, "file": file_info(official / "platform-versions.json"), }, "official_start_coverage": { "passed": start_coverage.get("passed"), "counts": start_coverage.get("counts") or {}, "by_category": start_coverage.get("by_category") or {}, "file": file_info(REPORTS / "1c-official-docs-start-coverage.json"), }, "official_progress": { "status": fetch_progress.get("status"), "page_count": fetch_progress.get("page_count") or fetch_manifest.get("page_count"), "error_count": fetch_progress.get("error_count") or fetch_manifest.get("error_count"), "seen_count": fetch_progress.get("seen_count"), "queue_size": fetch_progress.get("queue_size"), "current_url": fetch_progress.get("current_url"), "resumed_pages": fetch_progress.get("resumed_pages") or ((fetch_manifest.get("resume") or {}).get("resumable_pages")), "skipped_existing": fetch_progress.get("skipped_existing") or ((fetch_manifest.get("resume") or {}).get("skipped_existing")), "updated_at_unix": fetch_progress.get("updated_at_unix"), "file": file_info(official / "raw" / "progress.json"), }, "official_normalized": file_info(official / "normalized" / "manifest.json"), "official_media": file_info(official / "media" / "manifest.json"), "official_static": file_info(official / "static" / "manifest.json"), "official_static_url": "/official-docs-static/index.html", "official_static_check": { "passed": static_check.get("passed"), "counts": static_check.get("counts") or {}, "file": file_info(REPORTS / "1c-official-docs-static-check.json"), }, "official_private_check": { "passed": private_check.get("passed"), "counts": private_check.get("counts") or {}, "file": file_info(REPORTS / "1c-official-docs-private-check.json"), }, "official_access_check": { "status": access_check.get("status"), "findings": access_check.get("findings") or [], "page": access_check.get("page") or {}, "src": access_check.get("src") or {}, "file": file_info(REPORTS / "1c-its-access-check.json"), }, "official_quality": { "passed": quality_check.get("passed"), "counts": quality_check.get("counts") or {}, "file": file_info(REPORTS / "1c-official-docs-quality.json"), }, } def overview() -> dict: with JOB_LOCK: jobs = sorted(JOBS.values(), key=lambda item: item.get("created_at") or "", reverse=True)[:20] return { "schema": "management_console_overview.v1", "generated_at": now_iso(), "workspace": str(ROOT), "commands": [ {"id": key, "label": value["label"], "category": value["category"]} for key, value in COMMANDS.items() ], "artifacts": compact_artifact_manifest(), "models": compact_model_storage(), "rag": compact_rag(), "one_c": compact_1c_health(), "reports": { "artifact_manifest": file_info(REPORTS / "llm-artifact-manifest.json"), "artifact_check": file_info(REPORTS / "llm-artifact-check.json"), "model_chat_status": file_info(REPORTS / "model-chat" / "status.md"), }, "jobs": jobs, } def primary_doc_text(doc: dict) -> str: title = str(doc.get("title") or "").strip() content = str(doc.get("content") or "").strip() if not content: return "" metadata = doc.get("metadata") or {} source_type = str(metadata.get("source_type") or "") chunk_index = int(doc.get("chunk_index") or 0) if source_type == "official_1c_its_glossary" and chunk_index > 0: return "" lines = content.splitlines() heading_candidates = [title] heading_candidates.extend(str(item) for item in (metadata.get("headings") or []) if item) if source_type == "official_1c_its_glossary": heading_candidates.sort(key=lambda value: (":: Глоссарий" not in value, value == title)) for candidate in heading_candidates: if not candidate: continue exact_heading = f"# {candidate}" heading_indexes = [index for index, line in enumerate(lines) if line.strip() == exact_heading] if heading_indexes: lines = lines[heading_indexes[0] + 1 :] break trimmed_lines = [] for line in lines: if line.strip() == "Назад": break trimmed_lines.append(line) clean_lines = [] for line in trimmed_lines: normalized = line.strip() if not normalized or normalized in {"-", title}: continue clean_lines.append(normalized) result = "\n".join(clean_lines).strip() or title glossary_noise = ( "Глоссарий разработчика", "Информационная система 1С:ИТС", "Мы используем файлы cookie", "Результаты поиска", ) if source_type == "official_1c_its_glossary" and any(marker in result for marker in glossary_noise): return title return result def rag_snippet(doc: dict, max_chars: int = 1200) -> str: primary = primary_doc_text(doc) if primary: return primary[:max_chars] metadata = doc.get("metadata") or {} if str(metadata.get("source_type") or "") == "official_1c_its_glossary": return "" return str(doc.get("content") or "").strip()[:max_chars] def is_toc_like_text(text: str) -> bool: lines = [line.strip() for line in text.splitlines() if line.strip()] if not lines: return False list_like = 0 prose_like = 0 for line in lines: plain = line.lstrip("-*# ").strip() if not plain or plain in {"Документ", "Назад"}: continue if plain.startswith("Платформа 1С:Предприятие"): continue if re.match(r"^\d+(?:\.\d+)+\.\s+\S+", plain) or re.match(r"^Глава\s+\d+\.", plain): list_like += 1 continue if len(plain) >= 70 and not plain.endswith(":"): prose_like += 1 return list_like >= 6 and prose_like <= max(2, list_like // 5) def doc_content_kind(doc: dict) -> str: metadata = doc.get("metadata") or {} if str(metadata.get("access_blocked") or "").casefold() == "true": return "access_blocked" text = primary_doc_text(doc) meaningful = [ line.strip().lstrip("-*# ").strip() for line in text.splitlines() if line.strip() ] meaningful = [ line for line in meaningful if line and line not in {"Документ", "Назад"} and not line.startswith("Платформа 1С:Предприятие") ] if not meaningful: return "metadata_only" if is_toc_like_text(text): return "toc_like" return "content" def display_doc_title(doc: dict) -> str: metadata = doc.get("metadata") or {} source_type = str(metadata.get("source_type") or "") headings = [str(item) for item in (metadata.get("headings") or []) if item] if source_type == "official_1c_its_glossary": for heading in headings: if ":: Глоссарий" in heading: return heading return str(doc.get("title") or (headings[0] if headings else "")) def rag_answer_from_results(question: str, results: list[dict]) -> str: if not results: return "Контекст по вопросу в RAG не найден." content_kinds = [doc_content_kind(item["document"]) for item in results[:5]] if content_kinds and all(kind in {"access_blocked", "toc_like", "metadata_only"} for kind in content_kinds): if "access_blocked" in content_kinds: return ( "Найдена страница официальной документации, но тело документа закрыто текущим cookie/access. " "Обновите cookie ИТС и проверьте доступ; пока можно видеть только оболочку или оглавление." ) return ( "Найдена страница-оглавление официальной документации, а не полный текст раздела. " "Для уверенного ответа нужно загрузить конкретный подраздел из оглавления или получить тело документа через доступный hdoc/src-канал." ) question_lower = question.casefold() exact_titles = { "форма": {"форма", "общая форма", "элемент формы"}, } for keyword, titles in exact_titles.items(): if keyword in question_lower: exact_matches = [ item for item in results if str(item["document"].get("title") or "").strip().casefold() in titles ] if not exact_matches: return ( "Найденный контекст не содержит точной страницы термина. " "Ниже показаны ближайшие источники, но их недостаточно для уверенного ответа." ) exact_text = "\n\n".join(primary_doc_text(item["document"])[:1400] for item in exact_matches[:2]).casefold() definition_markers = ("предназнач", "объект", "содержит", "используется", "является", "отображ") if any(marker in exact_text for marker in definition_markers): return ( "Найдена точная страница термина в официальной документации. " "Проверьте фрагменты источников ниже: краткий ответ пока строится извлечением." ) return ( "Найдена точная страница термина в официальной документации, " "но в загруженном фрагменте нет развернутого определения. Ниже показан источник." ) top_text = "\n\n".join(primary_doc_text(item["document"])[:1400] for item in results[:3]) lowered = top_text.casefold() if any(marker in lowered for marker in ("предназнач", "объект", "содержит", "используется", "является", "отображ")): return "Контекст найден. Проверьте фрагменты источников ниже: краткий ответ пока строится извлечением." return "Найденный контекст не содержит достаточного определения для уверенного ответа. Ниже показаны источники, которые были найдены." def query_rag(payload: dict) -> dict: question = str(payload.get("question") or "").strip() if not question: raise ValueError("question is required") if "source_type" in payload: source_type = str(payload.get("source_type") or "").strip() else: source_type = "official_1c_its_glossary" if source_type in {"", "official_1c_docs"}: source_types = OFFICIAL_1C_SOURCE_TYPES source_scope = "official_1c_docs" elif source_type == "all_with_examples": source_types = None source_scope = "all_with_examples" else: source_types = [source_type] source_scope = source_type limit = max(1, min(int(payload.get("limit") or 5), 12)) metadata_filters = { "platform_version": str(payload.get("platform_version") or "").strip(), "platform_doc_id": str(payload.get("platform_doc_id") or "").strip(), } index = load_json(RAG_INDEX) if not index: return { "question": question, "answer": "RAG index is missing or empty. Rebuild the corpus and index first.", "results": [], } results = search_lexical_index( index, question, limit=limit, candidate_limit=max(limit * 5, 20), dedupe_by_document=False, min_score=0.0, source_types=source_types, metadata_filters=metadata_filters, ) compact = [] for item in results: doc = item["document"] metadata = doc.get("metadata") or {} snippet = rag_snippet(doc) if not snippet: continue content_kind = doc_content_kind(doc) compact.append( { "score": round(float(item.get("score") or 0), 4), "source_path": doc.get("source_path"), "title": display_doc_title(doc), "chunk_index": doc.get("chunk_index"), "source_type": metadata.get("source_type"), "url": metadata.get("url"), "platform_version": metadata.get("platform_version"), "platform_doc_id": metadata.get("platform_doc_id"), "doc_book": metadata.get("doc_book"), "doc_coordinate": metadata.get("doc_coordinate"), "access_blocked": metadata.get("access_blocked"), "access_findings": metadata.get("access_findings"), "content_kind": content_kind, "snippet": snippet, } ) answer = rag_answer_from_results(question, [item for item in results if rag_snippet(item["document"])]) if source_scope in {"metadata", "all_with_examples"}: answer = ( "Внимание: в выбранный контур входят примеры или снимки, они не подтверждают текущую базу. " + answer ) return { "question": question, "source_type": source_type, "source_scope": source_scope, "metadata_filters": {key: value for key, value in metadata_filters.items() if value}, "answer": answer, "result_count": len(compact), "results": compact, } def resolve_workspace_path(value: str) -> Path: if not value: raise ValueError("source_path is required") path = Path(value) if not path.is_absolute(): path = ROOT / path return path.resolve() def query_1c_fact(payload: dict) -> dict: source_kind = str(payload.get("source_kind") or "route_index").strip() source_path = str(payload.get("source_path") or "").strip() if not source_path and DEFAULT_1C_ROUTE_INDEX.exists() and source_kind == "route_index": source_path = str(DEFAULT_1C_ROUTE_INDEX) path = resolve_workspace_path(source_path) if not path.is_file(): raise ValueError(f"source_path does not exist: {path}") fact_path = str(payload.get("path") or "").strip() if fact_path: path_kind, path_name, path_section, path_member = split_fact_path(fact_path) kind = str(payload.get("kind") or path_kind or "").strip() or None name = str(payload.get("name") or path_name or "").strip() table_section = str(payload.get("table_section") or path_section or "").strip() or None member = str(payload.get("member") or path_member or "").strip() or None else: kind = str(payload.get("kind") or "").strip() or None name = str(payload.get("name") or "").strip() table_section = str(payload.get("table_section") or "").strip() or None member = str(payload.get("member") or "").strip() or None if not name: raise ValueError("Use path or name") common = { "kind": kind, "object_name": name, "member": member, "area": member_area(str(payload.get("area") or "any").strip() or "any"), "table_section": table_section, "view": str(payload.get("view") or "effective").strip() or "effective", "extension": str(payload.get("extension") or "").strip() or None, } if source_kind == "route_index": return resolve_from_route_index(load_json(path), index_path=path, **common) if source_kind == "metadata_snapshot": return resolve_from_snapshot(load_json(path), snapshot_path=path, **common) raise ValueError(f"Unsupported source_kind: {source_kind}") def query_1c_route(payload: dict) -> dict: question = str(payload.get("question") or "").strip() if not question: raise ValueError("question is required") source_path = str(payload.get("source_path") or "").strip() index_path = resolve_workspace_path(source_path) if source_path else DEFAULT_1C_ROUTE_INDEX view = str(payload.get("view") or "effective").strip() or "effective" return route_question(question, index_path=index_path, view=view) def query_1c_intake(payload: dict) -> dict: question = str(payload.get("question") or "").strip() if not question: raise ValueError("question is required") source_path = str(payload.get("source_path") or "").strip() index_path = resolve_workspace_path(source_path) if source_path else DEFAULT_1C_ROUTE_INDEX view = str(payload.get("view") or "effective").strip() or "effective" return build_intake(question, index=index_path, view=view) def official_doc_row(item: dict, status_name: str, index: int) -> dict: quality = item.get("quality") or {} return { "index": index, "status": status_name, "source_id": item.get("source_id"), "source_type": item.get("source_type"), "title": item.get("title"), "url": item.get("url"), "raw_file": item.get("raw_file"), "normalized_file": item.get("normalized_file"), "skip_reason": item.get("skip_reason"), "chars": item.get("chars") or quality.get("chars"), "quality": { "content_lines": quality.get("content_lines"), "prose_lines": quality.get("prose_lines"), "word_count": quality.get("word_count"), "is_content": quality.get("is_content"), }, } def official_docs_pages(params: dict[str, list[str]]) -> dict: manifest = load_json(OFFICIAL_NORMALIZED_MANIFEST) rows = [] for index, item in enumerate(manifest.get("pages") or []): rows.append(official_doc_row(item, "passed", index)) for index, item in enumerate(manifest.get("skipped_pages") or []): rows.append(official_doc_row(item, "skipped", index)) status_filter = (params.get("status") or ["all"])[0] source_filter = (params.get("source_type") or ["all"])[0] query = (params.get("q") or [""])[0].casefold().strip() if status_filter != "all": rows = [row for row in rows if row["status"] == status_filter] if source_filter != "all": rows = [row for row in rows if row["source_type"] == source_filter] if query: rows = [ row for row in rows if query in str(row.get("title") or "").casefold() or query in str(row.get("url") or "").casefold() or query in str(row.get("skip_reason") or "").casefold() ] limit = max(1, min(int((params.get("limit") or ["200"])[0]), 1000)) source_types = sorted({str(row.get("source_type") or "") for row in rows if row.get("source_type")}) return { "schema": "onec_official_docs_pages.v1", "manifest": str(OFFICIAL_NORMALIZED_MANIFEST), "counts": { "passed": len(manifest.get("pages") or []), "skipped": len(manifest.get("skipped_pages") or []), "filtered": len(rows), }, "source_types": source_types, "rows": rows[:limit], } def safe_child(parent: Path, filename: str) -> Path: if not filename or "/" in filename or "\\" in filename: raise ValueError("Invalid file name") path = (parent / filename).resolve() parent_resolved = parent.resolve() if not str(path).casefold().startswith(str(parent_resolved).casefold()): raise ValueError("Path is outside allowed directory") return path def official_docs_page_text(params: dict[str, list[str]]) -> dict: normalized_file = (params.get("normalized_file") or [""])[0] raw_file = (params.get("raw_file") or [""])[0] title = (params.get("title") or [""])[0] source_type = (params.get("source_type") or [""])[0] if normalized_file: path = safe_child(OFFICIAL_DOCS / "normalized", normalized_file) if not path.exists(): raise ValueError(f"Normalized file does not exist: {normalized_file}") text = path.read_text(encoding="utf-8-sig", errors="ignore") return {"schema": "onec_official_docs_page_text.v1", "kind": "normalized", "path": str(path), "text": text[:20000]} if not raw_file: raise ValueError("raw_file or normalized_file is required") path = safe_child(OFFICIAL_DOCS / "raw", raw_file) if not path.exists(): raise ValueError(f"Raw file does not exist: {raw_file}") raw_bytes = path.read_bytes() record = {"content_type": "text/html; charset=Windows-1251"} raw_text = decode_html(raw_bytes, record) extractor = TextExtractor() extractor.feed(raw_text) extracted = extractor.text() cleaned = clean_its_text(extracted, title or path.stem, source_type) return { "schema": "onec_official_docs_page_text.v1", "kind": "raw_cleaned_preview", "path": str(path), "text": (cleaned or extracted)[:20000], } def run_job(job_id: str, command_id: str) -> None: spec = COMMANDS[command_id] started = now_iso() with JOB_LOCK: JOBS[job_id].update({"status": "running", "started_at": started}) try: result = subprocess.run( spec["command"], cwd=ROOT, text=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=int(spec.get("timeout") or 120), check=False, ) status = "ok" if result.returncode == 0 else "failed" payload = { "status": status, "finished_at": now_iso(), "returncode": result.returncode, "stdout": result.stdout[-12000:], "stderr": result.stderr[-12000:], } except subprocess.TimeoutExpired as exc: payload = { "status": "timeout", "finished_at": now_iso(), "returncode": None, "stdout": (exc.stdout or "")[-12000:] if isinstance(exc.stdout, str) else "", "stderr": (exc.stderr or "")[-12000:] if isinstance(exc.stderr, str) else "", } except Exception as exc: # noqa: BLE001 - local console must report all failures. payload = { "status": "failed", "finished_at": now_iso(), "returncode": None, "stdout": "", "stderr": str(exc), } with JOB_LOCK: JOBS[job_id].update(payload) def run_custom_job(job_id: str, command: list[str]) -> None: started = now_iso() with JOB_LOCK: JOBS[job_id].update({"status": "running", "started_at": started}) try: result = subprocess.run( command, cwd=ROOT, text=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=1200, check=False, ) status = "ok" if result.returncode == 0 else "failed" payload = { "status": status, "finished_at": now_iso(), "returncode": result.returncode, "stdout": result.stdout[-12000:], "stderr": result.stderr[-12000:], } except subprocess.TimeoutExpired as exc: payload = { "status": "timeout", "finished_at": now_iso(), "returncode": None, "stdout": (exc.stdout or "")[-12000:] if isinstance(exc.stdout, str) else "", "stderr": (exc.stderr or "")[-12000:] if isinstance(exc.stderr, str) else "", } except Exception as exc: # noqa: BLE001 payload = {"status": "failed", "finished_at": now_iso(), "returncode": None, "stdout": "", "stderr": str(exc)} with JOB_LOCK: JOBS[job_id].update(payload) def start_job(command_id: str) -> dict: if command_id not in COMMANDS: raise ValueError(f"Unknown command: {command_id}") job_id = str(uuid.uuid4()) spec = COMMANDS[command_id] job = { "id": job_id, "command_id": command_id, "label": spec["label"], "category": spec["category"], "command": spec["command"], "status": "queued", "created_at": now_iso(), } with JOB_LOCK: JOBS[job_id] = job thread = threading.Thread(target=run_job, args=(job_id, command_id), daemon=True) thread.start() return job def start_target_its_job(url: str, *, max_pages: int = 12, max_depth: int = 2) -> dict: parsed = urlparse(url) if parsed.scheme not in {"http", "https"} or parsed.netloc.lower() != "its.1c.ru": raise ValueError("Only https://its.1c.ru/... URLs are allowed") if "/db/" not in parsed.path: raise ValueError("Only 1C:ITS /db/... documentation URLs are allowed") max_pages = max(1, min(max_pages, 50)) max_depth = max(0, min(max_depth, 4)) command = [ "powershell", "-NoProfile", "-ExecutionPolicy", "Bypass", "-File", "scripts/run_1c_its_target_doc_pipeline.ps1", "-Url", url, "-MaxPages", str(max_pages), "-MaxDepth", str(max_depth), ] job_id = str(uuid.uuid4()) job = { "id": job_id, "command_id": "official_docs_fetch_target", "label": "Fetch target 1C:ITS document", "category": "rag", "command": command, "status": "queued", "created_at": now_iso(), } with JOB_LOCK: JOBS[job_id] = job thread = threading.Thread(target=run_custom_job, args=(job_id, command), daemon=True) thread.start() return job def write_json(handler: BaseHTTPRequestHandler, status: int, data: dict) -> None: encoded = json.dumps(data, ensure_ascii=False, indent=2).encode("utf-8") handler.send_response(status) handler.send_header("Content-Type", "application/json; charset=utf-8") handler.send_header("Content-Length", str(len(encoded))) handler.end_headers() handler.wfile.write(encoded) def read_body(handler: BaseHTTPRequestHandler) -> dict: length = int(handler.headers.get("Content-Length") or 0) if not length: return {} raw = handler.rfile.read(length).decode("utf-8") return json.loads(raw) if raw.strip() else {} def static_path(raw_path: str) -> Path | None: path = unquote(urlparse(raw_path).path) if path == "/": path = "/index.html" candidate = (STATIC_DIR / path.lstrip("/")).resolve() static_root = STATIC_DIR.resolve() if not str(candidate).casefold().startswith(str(static_root).casefold()): return None if candidate.is_file(): return candidate return None def official_docs_static_path(raw_path: str) -> Path | None: path = unquote(urlparse(raw_path).path) prefix = "/official-docs-static" if not path.startswith(prefix): return None relative = path[len(prefix) :].lstrip("/") or "index.html" candidate = (OFFICIAL_STATIC_DIR / relative).resolve() static_root = OFFICIAL_STATIC_DIR.resolve() if not str(candidate).casefold().startswith(str(static_root).casefold()): return None if candidate.is_file(): return candidate return None class ConsoleHandler(BaseHTTPRequestHandler): def do_GET(self) -> None: parsed = urlparse(self.path) if parsed.path == "/api/overview": write_json(self, 200, overview()) return if parsed.path == "/api/official-docs/pages": write_json(self, 200, official_docs_pages(parse_qs(parsed.query))) return if parsed.path == "/api/official-docs/page-text": write_json(self, 200, official_docs_page_text(parse_qs(parsed.query))) return if parsed.path.startswith("/api/jobs/"): job_id = parsed.path.rsplit("/", 1)[-1] with JOB_LOCK: job = JOBS.get(job_id) write_json(self, 200 if job else 404, {"job": job}) return official_path = official_docs_static_path(self.path) if official_path: data = official_path.read_bytes() content_type = mimetypes.guess_type(str(official_path))[0] or "application/octet-stream" self.send_response(200) self.send_header("Content-Type", content_type) self.send_header("Content-Length", str(len(data))) self.end_headers() self.wfile.write(data) return path = static_path(self.path) if path: data = path.read_bytes() content_type = mimetypes.guess_type(str(path))[0] or "application/octet-stream" self.send_response(200) self.send_header("Content-Type", content_type) self.send_header("Content-Length", str(len(data))) self.end_headers() self.wfile.write(data) return write_json(self, 404, {"error": "not found"}) def do_POST(self) -> None: try: if urlparse(self.path).path == "/api/rag/query": write_json(self, 200, query_rag(read_body(self))) return if urlparse(self.path).path == "/api/1c/fact": write_json(self, 200, query_1c_fact(read_body(self))) return if urlparse(self.path).path == "/api/1c/route": write_json(self, 200, query_1c_route(read_body(self))) return if urlparse(self.path).path == "/api/1c/intake": write_json(self, 200, query_1c_intake(read_body(self))) return if urlparse(self.path).path == "/api/run": payload = read_body(self) job = start_job(str(payload.get("command_id") or "")) write_json(self, 202, {"job": job}) return if urlparse(self.path).path == "/api/official-docs/fetch-target": payload = read_body(self) job = start_target_its_job( str(payload.get("url") or ""), max_pages=int(payload.get("max_pages") or 12), max_depth=int(payload.get("max_depth") or 2), ) write_json(self, 202, {"job": job}) return write_json(self, 404, {"error": "not found"}) except Exception as exc: # noqa: BLE001 write_json(self, 400, {"error": str(exc)}) def log_message(self, format: str, *args) -> None: # noqa: A003 print(f"[management-console] {self.address_string()} - {format % args}") def main() -> int: parser = argparse.ArgumentParser(description="Run local LLM/1C management console.") parser.add_argument("--host", default=DEFAULT_HOST) parser.add_argument("--port", type=int, default=DEFAULT_PORT) args = parser.parse_args() server = ThreadingHTTPServer((args.host, args.port), ConsoleHandler) print(f"Management console: http://{args.host}:{args.port}/") try: server.serve_forever() except KeyboardInterrupt: return 130 return 0 if __name__ == "__main__": raise SystemExit(main())