Initial SQL-only 1C adapter baseline
This commit is contained in:
@@ -0,0 +1,228 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Atomically add a BSL handler, form command, and visible form button."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import base64
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from diff_1c_patch_workspace import build_diff
|
||||
from edit_1c_bsl_routine import edit_workspace as edit_bsl_routine
|
||||
from edit_1c_form_button import edit_workspace as edit_form_button
|
||||
from edit_1c_form_command import edit_workspace as edit_form_command
|
||||
from validate_1c_patch_workspace_semantics import validate_workspace
|
||||
|
||||
|
||||
def load_json(path: Path) -> dict[str, Any]:
|
||||
return json.loads(path.read_text(encoding="utf-8-sig"))
|
||||
|
||||
|
||||
def write_json(path: Path, data: dict[str, Any]) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(json.dumps(data, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
|
||||
|
||||
def read_bytes(path: Path) -> bytes:
|
||||
return path.read_bytes()
|
||||
|
||||
|
||||
def workspace_working_files(workspace: Path) -> list[Path]:
|
||||
manifest = load_json(workspace / "manifest.json")
|
||||
paths = []
|
||||
for record in manifest.get("files") or []:
|
||||
relative = Path(str(record.get("relative_path") or ""))
|
||||
if relative.is_absolute() or ".." in relative.parts:
|
||||
raise SystemExit(f"Unsafe manifest relative path: {relative}")
|
||||
path = workspace / "working" / relative
|
||||
if path.exists():
|
||||
paths.append(path)
|
||||
return paths
|
||||
|
||||
|
||||
def snapshot_working_files(workspace: Path) -> dict[Path, bytes]:
|
||||
return {path: read_bytes(path) for path in workspace_working_files(workspace)}
|
||||
|
||||
|
||||
def restore_snapshot(snapshot: dict[Path, bytes]) -> None:
|
||||
for path, content in snapshot.items():
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_bytes(content)
|
||||
|
||||
|
||||
def decode_routine_text(args: argparse.Namespace) -> str:
|
||||
sources = [bool(args.routine_text), bool(args.routine_text_b64), bool(args.routine_file)]
|
||||
if sum(sources) != 1:
|
||||
raise SystemExit("Use exactly one of --routine-text, --routine-text-b64, or --routine-file.")
|
||||
if args.routine_text is not None:
|
||||
return args.routine_text
|
||||
if args.routine_text_b64:
|
||||
return base64.b64decode(args.routine_text_b64).decode("utf-8")
|
||||
return Path(args.routine_file).read_text(encoding="utf-8-sig")
|
||||
|
||||
|
||||
def build_result(
|
||||
*,
|
||||
workspace: Path,
|
||||
form_relative_path: str,
|
||||
bsl_relative_path: str,
|
||||
operation: str,
|
||||
routine_text: str,
|
||||
command_name: str,
|
||||
command_title: str,
|
||||
command_action: str,
|
||||
button_parent_name: str,
|
||||
button_name: str,
|
||||
button_title: str,
|
||||
keep_on_failure: bool,
|
||||
max_patch_chars: int,
|
||||
) -> dict[str, Any]:
|
||||
snapshot = snapshot_working_files(workspace)
|
||||
steps: list[dict[str, Any]] = []
|
||||
rolled_back = False
|
||||
error: dict[str, Any] | None = None
|
||||
|
||||
try:
|
||||
bsl = edit_bsl_routine(
|
||||
workspace,
|
||||
bsl_relative_path,
|
||||
routine_text,
|
||||
operation=operation,
|
||||
keep_on_failure=True,
|
||||
)
|
||||
steps.append({"name": "bsl_routine", "result": bsl})
|
||||
if not bsl.get("semantic_validation", {}).get("passed"):
|
||||
raise RuntimeError("BSL routine edit failed semantic validation.")
|
||||
|
||||
command = edit_form_command(
|
||||
workspace,
|
||||
form_relative_path,
|
||||
name=command_name,
|
||||
title=command_title,
|
||||
action=command_action,
|
||||
tooltip=None,
|
||||
command_id=None,
|
||||
call_type="Override",
|
||||
operation=operation,
|
||||
keep_on_failure=True,
|
||||
)
|
||||
steps.append({"name": "form_command", "result": command})
|
||||
if not command.get("semantic_validation", {}).get("passed"):
|
||||
raise RuntimeError("Form command edit failed semantic validation.")
|
||||
|
||||
button = edit_form_button(
|
||||
workspace,
|
||||
form_relative_path,
|
||||
parent_name=button_parent_name,
|
||||
name=button_name,
|
||||
title=button_title,
|
||||
command_name=command_name,
|
||||
button_id=None,
|
||||
button_type="CommandBarButton",
|
||||
operation=operation,
|
||||
keep_on_failure=True,
|
||||
)
|
||||
steps.append({"name": "form_button", "result": button})
|
||||
if not button.get("semantic_validation", {}).get("passed"):
|
||||
raise RuntimeError("Form button edit failed semantic validation.")
|
||||
|
||||
semantic = validate_workspace(workspace)
|
||||
if not semantic.get("passed"):
|
||||
raise RuntimeError("Final semantic validation failed.")
|
||||
diff = build_diff(workspace, max_patch_chars=max_patch_chars)
|
||||
except (Exception, SystemExit) as exc:
|
||||
error = {"type": type(exc).__name__, "message": str(exc)}
|
||||
semantic = validate_workspace(workspace)
|
||||
diff = build_diff(workspace, max_patch_chars=max_patch_chars)
|
||||
if not keep_on_failure:
|
||||
restore_snapshot(snapshot)
|
||||
rolled_back = True
|
||||
semantic = validate_workspace(workspace)
|
||||
diff = build_diff(workspace, max_patch_chars=max_patch_chars)
|
||||
|
||||
passed = error is None and bool(semantic.get("passed")) and bool(diff.get("passed"))
|
||||
return {
|
||||
"schema": "onec_form_button_workflow.v1",
|
||||
"workspace": str(workspace),
|
||||
"operation": operation,
|
||||
"inputs": {
|
||||
"form_relative_path": form_relative_path,
|
||||
"bsl_relative_path": bsl_relative_path,
|
||||
"command_name": command_name,
|
||||
"command_title": command_title,
|
||||
"command_action": command_action,
|
||||
"button_parent_name": button_parent_name,
|
||||
"button_name": button_name,
|
||||
"button_title": button_title,
|
||||
},
|
||||
"passed": passed,
|
||||
"rolled_back": rolled_back,
|
||||
"error": error,
|
||||
"steps": steps,
|
||||
"semantic_validation": {
|
||||
"schema": semantic.get("schema"),
|
||||
"passed": semantic.get("passed"),
|
||||
"counts": semantic.get("counts"),
|
||||
"findings": semantic.get("findings"),
|
||||
},
|
||||
"diff_summary": diff.get("counts"),
|
||||
}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Atomically add BSL handler, form command, and visible button in a 1C patch workspace.")
|
||||
parser.add_argument("--workspace", type=Path, required=True)
|
||||
parser.add_argument("--form-relative-path", required=True)
|
||||
parser.add_argument("--bsl-relative-path", required=True)
|
||||
parser.add_argument("--operation", choices=["append", "replace", "upsert"], default="upsert")
|
||||
parser.add_argument("--routine-text")
|
||||
parser.add_argument("--routine-text-b64")
|
||||
parser.add_argument("--routine-file", type=Path)
|
||||
parser.add_argument("--command-name", required=True)
|
||||
parser.add_argument("--command-title", required=True)
|
||||
parser.add_argument("--command-action", required=True)
|
||||
parser.add_argument("--button-parent-name", required=True)
|
||||
parser.add_argument("--button-name", required=True)
|
||||
parser.add_argument("--button-title", required=True)
|
||||
parser.add_argument("--keep-on-failure", action="store_true", help="Keep partial edits when any step fails.")
|
||||
parser.add_argument("--max-patch-chars", type=int, default=200000)
|
||||
parser.add_argument("--output", type=Path)
|
||||
args = parser.parse_args()
|
||||
|
||||
result = build_result(
|
||||
workspace=args.workspace,
|
||||
form_relative_path=args.form_relative_path,
|
||||
bsl_relative_path=args.bsl_relative_path,
|
||||
operation=args.operation,
|
||||
routine_text=decode_routine_text(args),
|
||||
command_name=args.command_name,
|
||||
command_title=args.command_title,
|
||||
command_action=args.command_action,
|
||||
button_parent_name=args.button_parent_name,
|
||||
button_name=args.button_name,
|
||||
button_title=args.button_title,
|
||||
keep_on_failure=args.keep_on_failure,
|
||||
max_patch_chars=args.max_patch_chars,
|
||||
)
|
||||
if args.output:
|
||||
write_json(args.output, result)
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"output": str(args.output) if args.output else None,
|
||||
"passed": result["passed"],
|
||||
"rolled_back": result["rolled_back"],
|
||||
"error": result["error"],
|
||||
"semantic": result["semantic_validation"]["counts"],
|
||||
"diff": result["diff_summary"],
|
||||
},
|
||||
ensure_ascii=False,
|
||||
)
|
||||
)
|
||||
return 0 if result["passed"] else 2
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,42 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Compare one base Config SQL payload with XML files.
|
||||
|
||||
This is the base-configuration counterpart to extension manifest part analysis.
|
||||
It uses the same mechanical payload parser and XML matching logic.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
from analyze_1c_manifest_object_parts import load_xml_files, parse_cas_payload, public_payload_report
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Compare one Config object payload with XML files.")
|
||||
parser.add_argument("--config-file", type=Path, required=True)
|
||||
parser.add_argument("--object-guid", required=True)
|
||||
parser.add_argument("--xml-path", type=Path, action="append", default=[])
|
||||
parser.add_argument("--output", type=Path, required=True)
|
||||
args = parser.parse_args()
|
||||
|
||||
xml_files = load_xml_files(args.xml_path)
|
||||
payload = parse_cas_payload(args.config_file)
|
||||
report = {
|
||||
"schema": "onec_config_object_xml_compare.v1",
|
||||
"object_guid": args.object_guid.lower(),
|
||||
"config_file": str(args.config_file),
|
||||
"xml_paths": [str(path) for path in args.xml_path],
|
||||
"xml_file_count": len(xml_files),
|
||||
"payload": public_payload_report(payload, xml_files),
|
||||
}
|
||||
args.output.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.output.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
print(json.dumps({"output": str(args.output), "xml_files": len(xml_files)}, ensure_ascii=False))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,440 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Compare one extension manifest object parts with an XML export tree.
|
||||
|
||||
The script is intentionally evidence-first. It reports mechanical facts:
|
||||
manifest object_id parts, ConfigCAS keys, decoded payload structure, embedded
|
||||
base64 blobs, and exact/contains matches against files from an XML export.
|
||||
It does not assign semantic names to suffixes.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import base64
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from inspect_1c_sql_files import GUID_RE, Lexer, Parser, collect_strings, tree_shape, try_decode, try_decompress
|
||||
|
||||
|
||||
BASE64_RE = re.compile(r"[A-Za-z0-9+/]{40,}={0,2}")
|
||||
BSL_MARKERS = ("&На", "Процедура ", "Функция ", "#Область", "#КонецОбласти")
|
||||
HTML_MARKERS = ("<!DOCTYPE", "<html", "<HTML", "<body", "<BODY")
|
||||
STREAM_HEADER_RE = re.compile(rb"\r\n([0-9a-f]{8}) ([0-9a-f]{8}) 7fffffff \r\n")
|
||||
|
||||
|
||||
def scalar(node: Any) -> str:
|
||||
if isinstance(node, dict) and node.get("type") in {"atom", "string"}:
|
||||
return str(node.get("value") or "")
|
||||
return ""
|
||||
|
||||
|
||||
def suffix_of(object_id: str) -> str:
|
||||
parts = object_id.split(".", 1)
|
||||
return "" if len(parts) == 1 else "." + parts[1]
|
||||
|
||||
|
||||
def sha1_hex(data: bytes) -> str:
|
||||
return hashlib.sha1(data).hexdigest()
|
||||
|
||||
|
||||
def decode_text(data: bytes) -> tuple[str | None, str | None]:
|
||||
if data.startswith(b"\xef\xbb\xbf"):
|
||||
try:
|
||||
return data.decode("utf-8-sig"), "utf-8-sig"
|
||||
except UnicodeDecodeError:
|
||||
pass
|
||||
return try_decode(data)
|
||||
|
||||
|
||||
def decode_payload_text(data: bytes) -> tuple[str | None, str | None, int]:
|
||||
if data.startswith(b"\xef\xbb\xbf"):
|
||||
text, encoding = decode_text(data)
|
||||
return text, encoding, 0
|
||||
marker = data.find(b"\xef\xbb\xbf")
|
||||
if marker >= 0:
|
||||
try:
|
||||
return data[marker:].decode("utf-8-sig"), "utf-8-sig", marker
|
||||
except UnicodeDecodeError:
|
||||
pass
|
||||
text, encoding = try_decode(data)
|
||||
return text, encoding, 0
|
||||
|
||||
|
||||
def payload_markers(data: bytes) -> list[str]:
|
||||
markers = []
|
||||
if data.startswith(b"MOXCEL"):
|
||||
markers.append("MOXCEL")
|
||||
if data.startswith(b"\xef\xbb\xbf") or b"\xef\xbb\xbf" in data[:256]:
|
||||
markers.append("utf8_bom")
|
||||
if STREAM_HEADER_RE.search(data):
|
||||
markers.append("stream_headers")
|
||||
return markers
|
||||
|
||||
|
||||
def load_xml_files(paths: list[Path]) -> list[dict[str, Any]]:
|
||||
files: list[dict[str, Any]] = []
|
||||
seen: set[Path] = set()
|
||||
for root in paths:
|
||||
if not root.exists():
|
||||
continue
|
||||
candidates = [root] if root.is_file() else [item for item in root.rglob("*") if item.is_file()]
|
||||
for path in candidates:
|
||||
resolved = path.resolve()
|
||||
if resolved in seen:
|
||||
continue
|
||||
seen.add(resolved)
|
||||
data = path.read_bytes()
|
||||
text, encoding = decode_text(data)
|
||||
files.append(
|
||||
{
|
||||
"path": str(path),
|
||||
"name": path.name,
|
||||
"relative_hint": str(path),
|
||||
"bytes": len(data),
|
||||
"sha1": sha1_hex(data),
|
||||
"text": text,
|
||||
"encoding": encoding,
|
||||
}
|
||||
)
|
||||
return files
|
||||
|
||||
|
||||
def collect_atoms(value: Any) -> list[str]:
|
||||
atoms: list[str] = []
|
||||
|
||||
def walk(node: Any) -> None:
|
||||
if isinstance(node, dict) and node.get("type") == "atom":
|
||||
atoms.append(str(node.get("value") or ""))
|
||||
if isinstance(node, dict):
|
||||
for child in node.get("items") or []:
|
||||
walk(child)
|
||||
|
||||
walk(value)
|
||||
return atoms
|
||||
|
||||
|
||||
def collect_base64_blocks(value: Any) -> list[str]:
|
||||
blocks: list[str] = []
|
||||
|
||||
def walk(node: Any) -> None:
|
||||
if isinstance(node, dict) and node.get("type") == "list":
|
||||
items = node.get("items") or []
|
||||
if items and scalar(items[0]) == "#base64":
|
||||
chunks = [scalar(item) for item in items[1:] if BASE64_RE.fullmatch(scalar(item))]
|
||||
if chunks:
|
||||
blocks.append("".join(chunks))
|
||||
for child in items:
|
||||
walk(child)
|
||||
|
||||
walk(value)
|
||||
return blocks
|
||||
|
||||
|
||||
def decode_base64_atoms(atoms: list[str]) -> list[dict[str, Any]]:
|
||||
decoded: list[dict[str, Any]] = []
|
||||
for value in atoms:
|
||||
if not BASE64_RE.fullmatch(value):
|
||||
continue
|
||||
try:
|
||||
data = base64.b64decode(value, validate=True)
|
||||
except Exception:
|
||||
continue
|
||||
if not data:
|
||||
continue
|
||||
text, encoding = decode_text(data)
|
||||
text_preview = ""
|
||||
if text:
|
||||
text_preview = text.replace("\x00", "")[:300]
|
||||
decoded.append(
|
||||
{
|
||||
"atom_length": len(value),
|
||||
"bytes": len(data),
|
||||
"sha1": sha1_hex(data),
|
||||
"encoding": encoding,
|
||||
"text_preview": text_preview,
|
||||
"has_bsl_marker": bool(text and any(marker in text for marker in BSL_MARKERS)),
|
||||
"has_html_marker": bool(text and any(marker in text for marker in HTML_MARKERS)),
|
||||
"bytes_base64": value[:120],
|
||||
"data": data,
|
||||
"text": text,
|
||||
}
|
||||
)
|
||||
return decoded
|
||||
|
||||
|
||||
def decode_base64_blocks(blocks: list[str]) -> list[dict[str, Any]]:
|
||||
decoded: list[dict[str, Any]] = []
|
||||
for value in blocks:
|
||||
try:
|
||||
data = base64.b64decode(value, validate=True)
|
||||
except Exception:
|
||||
continue
|
||||
text, encoding = decode_text(data)
|
||||
decoded.append(
|
||||
{
|
||||
"block_length": len(value),
|
||||
"bytes": len(data),
|
||||
"sha1": sha1_hex(data),
|
||||
"encoding": encoding,
|
||||
"text_preview": (text or "").replace("\x00", "")[:500],
|
||||
"has_bsl_marker": bool(text and any(marker in text for marker in BSL_MARKERS)),
|
||||
"has_html_marker": bool(text and any(marker in text for marker in HTML_MARKERS)),
|
||||
"data": data,
|
||||
"text": text,
|
||||
}
|
||||
)
|
||||
return decoded
|
||||
|
||||
|
||||
def match_blob(blob: bytes, text: str | None, xml_files: list[dict[str, Any]]) -> dict[str, Any]:
|
||||
exact = []
|
||||
normalized_equal = []
|
||||
contains = []
|
||||
blob_sha1 = sha1_hex(blob)
|
||||
normalized_text = text.replace("\r\n", "\n").strip() if text else None
|
||||
for item in xml_files:
|
||||
file_text = item.get("text")
|
||||
data = Path(item["path"]).read_bytes()
|
||||
if item["sha1"] == blob_sha1:
|
||||
exact.append({"path": item["path"], "match": "sha1"})
|
||||
elif normalized_text and file_text and normalized_text == file_text.replace("\r\n", "\n").strip():
|
||||
normalized_equal.append({"path": item["path"], "match": "normalized_text_equal"})
|
||||
elif len(blob) >= 24 and blob in data:
|
||||
contains.append({"path": item["path"], "match": "bytes_contains"})
|
||||
elif text and file_text and len(text.strip()) >= 24 and text.strip() in file_text:
|
||||
contains.append({"path": item["path"], "match": "text_contains"})
|
||||
elif text and file_text and len(file_text.strip()) >= 24 and file_text.strip() in text:
|
||||
contains.append({"path": item["path"], "match": "payload_contains_file_text"})
|
||||
return {"exact": exact, "normalized_equal": normalized_equal, "contains": contains}
|
||||
|
||||
|
||||
def extract_stream_blocks(payload: bytes) -> list[dict[str, Any]]:
|
||||
blocks: list[dict[str, Any]] = []
|
||||
for match in STREAM_HEADER_RE.finditer(payload):
|
||||
declared_1 = int(match.group(1), 16)
|
||||
declared_2 = int(match.group(2), 16)
|
||||
start = match.end()
|
||||
size = declared_2
|
||||
if size <= 0 or start + size > len(payload):
|
||||
continue
|
||||
data = payload[start : start + size]
|
||||
text, encoding = decode_text(data)
|
||||
blocks.append(
|
||||
{
|
||||
"header_offset": match.start(),
|
||||
"data_offset": start,
|
||||
"declared_1": declared_1,
|
||||
"declared_2": declared_2,
|
||||
"bytes": len(data),
|
||||
"sha1": sha1_hex(data),
|
||||
"encoding": encoding,
|
||||
"text_preview": (text or "").replace("\x00", "")[:500],
|
||||
"has_bsl_marker": bool(text and any(marker in text for marker in BSL_MARKERS)),
|
||||
"has_html_marker": bool(text and any(marker in text for marker in HTML_MARKERS)),
|
||||
"data": data,
|
||||
"text": text,
|
||||
}
|
||||
)
|
||||
return blocks
|
||||
|
||||
|
||||
def match_strings(strings: list[str], xml_files: list[dict[str, Any]], *, limit: int = 80) -> list[dict[str, Any]]:
|
||||
hits: list[dict[str, Any]] = []
|
||||
for string in strings:
|
||||
if len(string.strip()) < 4:
|
||||
continue
|
||||
paths = []
|
||||
for item in xml_files:
|
||||
text = item.get("text") or ""
|
||||
if string in text:
|
||||
paths.append(item["path"])
|
||||
if len(paths) >= 8:
|
||||
break
|
||||
if paths:
|
||||
hits.append({"string": string[:200], "paths": paths})
|
||||
if len(hits) >= limit:
|
||||
break
|
||||
return hits
|
||||
|
||||
|
||||
def match_guids(text: str, xml_files: list[dict[str, Any]], *, limit: int = 200) -> list[dict[str, Any]]:
|
||||
hits: list[dict[str, Any]] = []
|
||||
for guid in sorted(set(match.lower() for match in GUID_RE.findall(text))):
|
||||
paths = []
|
||||
for item in xml_files:
|
||||
file_text = (item.get("text") or "").lower()
|
||||
if guid in file_text:
|
||||
paths.append(item["path"])
|
||||
if len(paths) >= 8:
|
||||
break
|
||||
if paths:
|
||||
hits.append({"guid": guid, "paths": paths})
|
||||
if len(hits) >= limit:
|
||||
break
|
||||
return hits
|
||||
|
||||
|
||||
def parse_cas_payload(path: Path) -> dict[str, Any]:
|
||||
raw = path.read_bytes()
|
||||
payload, compression = try_decompress(raw)
|
||||
markers = payload_markers(payload)
|
||||
if "stream_headers" in markers and not payload.startswith(b"\xef\xbb\xbf"):
|
||||
text, encoding, text_offset = None, None, 0
|
||||
else:
|
||||
text, encoding, text_offset = decode_payload_text(payload)
|
||||
report: dict[str, Any] = {
|
||||
"cas_file": path.name,
|
||||
"bytes": len(raw),
|
||||
"payload_bytes": len(payload),
|
||||
"payload_sha1": sha1_hex(payload),
|
||||
"compression": compression,
|
||||
"encoding": encoding,
|
||||
"text_offset": text_offset,
|
||||
"payload_markers": markers,
|
||||
"parse_status": "not_text",
|
||||
"text_preview": "",
|
||||
"strings": [],
|
||||
"base64_blobs": [],
|
||||
"base64_blocks": [],
|
||||
"stream_blocks": [],
|
||||
}
|
||||
report["_raw_payload"] = payload
|
||||
report["_stream_blocks"] = extract_stream_blocks(payload)
|
||||
if text is None:
|
||||
return report
|
||||
clean = text.replace("\x00", "").replace("\ufeff", "").lstrip("ï»¿п»ї")
|
||||
report["text_preview"] = clean[:500]
|
||||
report["has_bsl_marker"] = any(marker in clean for marker in BSL_MARKERS)
|
||||
report["has_html_marker"] = any(marker in clean for marker in HTML_MARKERS)
|
||||
if "{" not in clean:
|
||||
report["parse_status"] = "text_no_braces"
|
||||
return report
|
||||
try:
|
||||
parsed = Parser(Lexer(clean[:2_000_000]).tokens()).parse()
|
||||
except Exception as exc:
|
||||
report["parse_status"] = "parse_error"
|
||||
report["parse_error"] = str(exc)
|
||||
return report
|
||||
report["parse_status"] = "parsed"
|
||||
report["shape"] = tree_shape(parsed, max_depth=4)
|
||||
strings = collect_strings(parsed, limit=200)
|
||||
atoms = collect_atoms(parsed)
|
||||
blocks = collect_base64_blocks(parsed)
|
||||
report["strings"] = strings
|
||||
if isinstance(parsed, dict):
|
||||
items = parsed.get("items") or []
|
||||
report["root_type"] = parsed.get("type")
|
||||
report["root_len"] = len(items)
|
||||
report["root_marker"] = scalar(items[0]) if items else ""
|
||||
report["_raw_payload"] = payload
|
||||
report["_clean_text"] = clean
|
||||
report["_base64_decoded"] = decode_base64_atoms(atoms)
|
||||
report["_base64_blocks_decoded"] = decode_base64_blocks(blocks)
|
||||
return report
|
||||
|
||||
|
||||
def load_manifest_entries(manifest_path: Path, object_guid: str) -> list[dict[str, Any]]:
|
||||
manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
|
||||
target = object_guid.lower()
|
||||
entries = []
|
||||
for entry in manifest.get("entries") or []:
|
||||
object_id = str(entry.get("object_id") or "").lower()
|
||||
if object_id == target or object_id.startswith(target + "."):
|
||||
entries.append(entry)
|
||||
entries.sort(key=lambda item: (suffix_of(str(item["object_id"])), str(item["object_id"])))
|
||||
return entries
|
||||
|
||||
|
||||
def public_payload_report(payload_report: dict[str, Any], xml_files: list[dict[str, Any]]) -> dict[str, Any]:
|
||||
raw_payload = payload_report.pop("_raw_payload", b"")
|
||||
clean_text = payload_report.pop("_clean_text", "")
|
||||
decoded = payload_report.pop("_base64_decoded", [])
|
||||
decoded_blocks = payload_report.pop("_base64_blocks_decoded", [])
|
||||
stream_blocks = payload_report.pop("_stream_blocks", [])
|
||||
payload_report["payload_matches"] = match_blob(raw_payload, clean_text, xml_files)
|
||||
payload_report["string_matches"] = match_strings(payload_report.get("strings") or [], xml_files)
|
||||
payload_report["guid_matches"] = match_guids(clean_text, xml_files)
|
||||
public_decoded = []
|
||||
for blob in decoded:
|
||||
data = blob.pop("data")
|
||||
text = blob.pop("text")
|
||||
blob["matches"] = match_blob(data, text, xml_files)
|
||||
public_decoded.append(blob)
|
||||
payload_report["base64_blobs"] = public_decoded
|
||||
public_blocks = []
|
||||
for blob in decoded_blocks:
|
||||
data = blob.pop("data")
|
||||
text = blob.pop("text")
|
||||
blob["matches"] = match_blob(data, text, xml_files)
|
||||
public_blocks.append(blob)
|
||||
payload_report["base64_blocks"] = public_blocks
|
||||
public_streams = []
|
||||
for block in stream_blocks:
|
||||
data = block.pop("data")
|
||||
text = block.pop("text")
|
||||
block["matches"] = match_blob(data, text, xml_files)
|
||||
public_streams.append(block)
|
||||
payload_report["stream_blocks"] = public_streams
|
||||
payload_report["strings"] = (payload_report.get("strings") or [])[:80]
|
||||
return payload_report
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Compare one manifest object's CAS parts with XML files.")
|
||||
parser.add_argument("--manifest", type=Path, required=True)
|
||||
parser.add_argument("--cas-dir", type=Path, required=True)
|
||||
parser.add_argument("--object-guid", required=True)
|
||||
parser.add_argument("--xml-path", type=Path, action="append", default=[])
|
||||
parser.add_argument("--output", type=Path, required=True)
|
||||
args = parser.parse_args()
|
||||
|
||||
xml_files = load_xml_files(args.xml_path)
|
||||
entries = load_manifest_entries(args.manifest, args.object_guid)
|
||||
parts = []
|
||||
for entry in entries:
|
||||
cas_key = entry["cas_key"]
|
||||
cas_path = Path(entry.get("cas_path") or args.cas_dir / cas_key)
|
||||
if not cas_path.is_file():
|
||||
cas_path = args.cas_dir / cas_key
|
||||
payload = parse_cas_payload(cas_path) if cas_path.is_file() else {"parse_status": "missing_cas"}
|
||||
parts.append(
|
||||
{
|
||||
"object_id": entry["object_id"],
|
||||
"suffix": suffix_of(entry["object_id"]),
|
||||
"cas_key": cas_key,
|
||||
"cas_path": str(cas_path),
|
||||
"payload": public_payload_report(payload, xml_files) if cas_path.is_file() else payload,
|
||||
}
|
||||
)
|
||||
|
||||
report = {
|
||||
"schema": "onec_manifest_object_part_compare.v1",
|
||||
"object_guid": args.object_guid.lower(),
|
||||
"manifest": str(args.manifest),
|
||||
"cas_dir": str(args.cas_dir),
|
||||
"xml_paths": [str(path) for path in args.xml_path],
|
||||
"xml_file_count": len(xml_files),
|
||||
"xml_files": [
|
||||
{
|
||||
"path": item["path"],
|
||||
"bytes": item["bytes"],
|
||||
"sha1": item["sha1"],
|
||||
"encoding": item["encoding"],
|
||||
}
|
||||
for item in xml_files
|
||||
],
|
||||
"part_count": len(parts),
|
||||
"parts": parts,
|
||||
}
|
||||
args.output.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.output.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
print(json.dumps({"output": str(args.output), "parts": len(parts), "xml_files": len(xml_files)}, ensure_ascii=False))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,241 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
import xml.etree.ElementTree as ET
|
||||
|
||||
from analyze_1c_template_xml_profiles import merge_ranges
|
||||
|
||||
|
||||
def rpc(adapter_url: str, method: str, payload: dict[str, Any]) -> dict[str, Any]:
|
||||
body = json.dumps({"method": method, "payload": payload}, ensure_ascii=False).encode("utf-8")
|
||||
req = urllib.request.Request(
|
||||
f"{adapter_url.rstrip('/')}/rpc",
|
||||
data=body,
|
||||
headers={"Content-Type": "application/json; charset=utf-8"},
|
||||
method="POST",
|
||||
)
|
||||
with urllib.request.urlopen(req, timeout=180) as resp:
|
||||
return json.loads(resp.read().decode("utf-8", errors="replace"))
|
||||
|
||||
|
||||
def runs(values: list[int]) -> list[dict[str, int]]:
|
||||
if not values:
|
||||
return []
|
||||
result: list[dict[str, int]] = []
|
||||
start = previous = values[0]
|
||||
for value in values[1:]:
|
||||
if value == previous + 1:
|
||||
previous = value
|
||||
continue
|
||||
result.append({"start": start, "end": previous, "length": previous - start + 1})
|
||||
start = previous = value
|
||||
result.append({"start": start, "end": previous, "length": previous - start + 1})
|
||||
return result
|
||||
|
||||
|
||||
def div32_values(numbers: list[Any]) -> list[int]:
|
||||
return [
|
||||
int(value) // 32
|
||||
for value in numbers
|
||||
if isinstance(value, int) and value > 0 and value <= 4096 and value % 32 == 0
|
||||
]
|
||||
|
||||
|
||||
def small_values(numbers: list[Any]) -> list[int]:
|
||||
return [int(value) for value in numbers if isinstance(value, int) and 2 <= value <= 128]
|
||||
|
||||
|
||||
def template_xml_path(root: Path, template: str) -> Path:
|
||||
return root / template / "Ext" / "Template.xml"
|
||||
|
||||
|
||||
def merge_block_candidate(adapter_url: str, base_id: str, owner_kind: str, owner_name: str, template: str) -> dict[str, Any]:
|
||||
data = rpc(
|
||||
adapter_url,
|
||||
"templates.read",
|
||||
{
|
||||
"base_id": base_id,
|
||||
"kind": owner_kind,
|
||||
"name": owner_name,
|
||||
"template": template,
|
||||
"sections": "merges",
|
||||
"refresh_cache": False,
|
||||
},
|
||||
)
|
||||
return (((data.get("templates") or [{}])[0].get("structure") or {}).get("merge_record_block_candidates") or [{}])[0]
|
||||
|
||||
|
||||
def merge_block_records(
|
||||
adapter_url: str,
|
||||
base_id: str,
|
||||
owner_kind: str,
|
||||
owner_name: str,
|
||||
template: str,
|
||||
candidate: dict[str, Any],
|
||||
) -> list[dict[str, Any]]:
|
||||
position = str(candidate.get("tree_position") or "$.0")
|
||||
try:
|
||||
start = int(position.split(".")[1]) + 1
|
||||
except (IndexError, ValueError):
|
||||
start = 0
|
||||
count = int(candidate.get("count") or 0)
|
||||
data = rpc(
|
||||
adapter_url,
|
||||
"templates.read",
|
||||
{
|
||||
"base_id": base_id,
|
||||
"kind": owner_kind,
|
||||
"name": owner_name,
|
||||
"template": template,
|
||||
"sections": "moxel_records",
|
||||
"max_moxel_records": count + 40,
|
||||
"moxel_record_start": start,
|
||||
"moxel_record_end": start + count + 35,
|
||||
"refresh_cache": False,
|
||||
},
|
||||
)
|
||||
diagnostics = ((data.get("templates") or [{}])[0].get("structure") or {}).get("moxel_record_diagnostics") or [{}]
|
||||
if isinstance(diagnostics, list):
|
||||
diagnostics = diagnostics[0] if diagnostics else {}
|
||||
return [record for record in diagnostics.get("top_level_records") or [] if isinstance(record, dict)][:count]
|
||||
|
||||
|
||||
def analyze_template(
|
||||
*,
|
||||
adapter_url: str,
|
||||
base_id: str,
|
||||
owner_kind: str,
|
||||
owner_name: str,
|
||||
template: str,
|
||||
xml_root: Path,
|
||||
) -> dict[str, Any]:
|
||||
xml_path = template_xml_path(xml_root, template)
|
||||
merges = merge_ranges(ET.parse(xml_path).getroot(), limit=500)
|
||||
xml_rows = sorted(set(int(item["row"]) for item in merges))
|
||||
xml_columns = sorted(set(int(item["column"]) for item in merges) | set(int(item["column"]) + int(item["width"]) - 1 for item in merges))
|
||||
candidate = merge_block_candidate(adapter_url, base_id, owner_kind, owner_name, template)
|
||||
records = merge_block_records(adapter_url, base_id, owner_kind, owner_name, template, candidate)
|
||||
by_value: dict[int, list[int]] = {}
|
||||
coordinate_records: list[dict[str, Any]] = []
|
||||
for index, record in enumerate(records, 1):
|
||||
numbers = record.get("numeric_items") or []
|
||||
for value in set(small_values(numbers)):
|
||||
by_value.setdefault(value, []).append(index)
|
||||
packed_columns = div32_values(numbers)
|
||||
if packed_columns:
|
||||
coordinate_records.append(
|
||||
{
|
||||
"index": index,
|
||||
"tree_position": record.get("tree_position"),
|
||||
"numeric_items": numbers,
|
||||
"div32": packed_columns,
|
||||
"small": small_values(numbers),
|
||||
}
|
||||
)
|
||||
value_summaries = [
|
||||
{
|
||||
"value": value,
|
||||
"count": len(indexes),
|
||||
"record_indexes": indexes[:30],
|
||||
"runs": runs(indexes),
|
||||
"matches_xml_row": value in xml_rows,
|
||||
"matches_xml_column_or_edge": value in xml_columns,
|
||||
}
|
||||
for value, indexes in sorted(by_value.items())
|
||||
]
|
||||
xml_row_hits = [
|
||||
{
|
||||
"row": row,
|
||||
"count": len(by_value.get(row) or []),
|
||||
"record_indexes": (by_value.get(row) or [])[:20],
|
||||
"runs": runs(by_value.get(row) or [])[:8],
|
||||
}
|
||||
for row in xml_rows
|
||||
if by_value.get(row)
|
||||
]
|
||||
return {
|
||||
"template": template,
|
||||
"xml_merge_count": len(merges),
|
||||
"sql_block_count": int(candidate.get("count") or 0),
|
||||
"tree_position": candidate.get("tree_position"),
|
||||
"xml_rows": xml_rows,
|
||||
"xml_row_runs": runs(xml_rows),
|
||||
"xml_columns_and_right_edges": xml_columns,
|
||||
"value_summaries": value_summaries,
|
||||
"xml_row_hits": xml_row_hits,
|
||||
"coordinate_records": coordinate_records[:120],
|
||||
"coordinate_record_runs": runs([item["index"] for item in coordinate_records]),
|
||||
}
|
||||
|
||||
|
||||
def render_markdown(payload: dict[str, Any]) -> str:
|
||||
lines = ["# MOXCEL merge row-band analysis", ""]
|
||||
for item in payload.get("items") or []:
|
||||
lines.append(f"## {item.get('template')}")
|
||||
lines.append("")
|
||||
lines.append(f"- XML merges: `{item.get('xml_merge_count')}`")
|
||||
lines.append(f"- SQL block count: `{item.get('sql_block_count')}` at `{item.get('tree_position')}`")
|
||||
lines.append(f"- XML row runs: `{item.get('xml_row_runs')}`")
|
||||
lines.append(f"- SQL coordinate-record runs: `{(item.get('coordinate_record_runs') or [])[:20]}`")
|
||||
lines.append("")
|
||||
lines.append("### XML Row Hits In SQL Small Scalars")
|
||||
lines.append("")
|
||||
lines.append("| Row | Count | Runs | First indexes |")
|
||||
lines.append("| ---: | ---: | --- | --- |")
|
||||
for hit in (item.get("xml_row_hits") or [])[:60]:
|
||||
lines.append(f"| {hit.get('row')} | {hit.get('count')} | `{hit.get('runs')}` | `{(hit.get('record_indexes') or [])[:12]}` |")
|
||||
lines.append("")
|
||||
lines.append("### Top Small Scalar Values")
|
||||
lines.append("")
|
||||
lines.append("| Value | Count | XML row | XML col/edge | Runs |")
|
||||
lines.append("| ---: | ---: | --- | --- | --- |")
|
||||
for value in sorted(item.get("value_summaries") or [], key=lambda row: (-int(row.get("count") or 0), int(row.get("value") or 0)))[:30]:
|
||||
lines.append(
|
||||
f"| {value.get('value')} | {value.get('count')} | `{value.get('matches_xml_row')}` | "
|
||||
f"`{value.get('matches_xml_column_or_edge')}` | `{(value.get('runs') or [])[:8]}` |"
|
||||
)
|
||||
lines.append("")
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Analyze SQL MOXCEL merge-block row/size scalar bands against XML merge rows.")
|
||||
parser.add_argument("--adapter-url", default="http://docker-gpu.cin.su:8011")
|
||||
parser.add_argument("--base-id", default="upo_test")
|
||||
parser.add_argument("--owner-kind", default="Document")
|
||||
parser.add_argument("--owner-name", default="АвансовыйОтчет")
|
||||
parser.add_argument(
|
||||
"--xml-root",
|
||||
default=r"Z:\codex\1C\XML\UPO\Структура базы 1с\Конфигурация\Documents\АвансовыйОтчет\Templates",
|
||||
)
|
||||
parser.add_argument("--template", action="append", required=True)
|
||||
parser.add_argument("--output-json", default="reports/1c-template-baselines/moxel-merge-row-band-analysis.json")
|
||||
parser.add_argument("--output-markdown", default="reports/1c-template-baselines/moxel-merge-row-band-analysis.md")
|
||||
args = parser.parse_args()
|
||||
|
||||
payload = {
|
||||
"schema": "codex_1c_moxel_merge_row_band_analysis.v1",
|
||||
"items": [
|
||||
analyze_template(
|
||||
adapter_url=args.adapter_url,
|
||||
base_id=args.base_id,
|
||||
owner_kind=args.owner_kind,
|
||||
owner_name=args.owner_name,
|
||||
template=template,
|
||||
xml_root=Path(args.xml_root),
|
||||
)
|
||||
for template in args.template
|
||||
],
|
||||
}
|
||||
Path(args.output_json).write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
Path(args.output_markdown).write_text(render_markdown(payload), encoding="utf-8")
|
||||
print(json.dumps({"status": "ok", "json": args.output_json, "markdown": args.output_markdown, "items": len(payload["items"])}, ensure_ascii=False, indent=2))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,360 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
import xml.etree.ElementTree as ET
|
||||
|
||||
from analyze_1c_template_xml_profiles import merge_ranges
|
||||
|
||||
|
||||
def rpc(adapter_url: str, method: str, payload: dict[str, Any]) -> dict[str, Any]:
|
||||
body = json.dumps({"method": method, "payload": payload}, ensure_ascii=False).encode("utf-8")
|
||||
req = urllib.request.Request(
|
||||
f"{adapter_url.rstrip('/')}/rpc",
|
||||
data=body,
|
||||
headers={"Content-Type": "application/json; charset=utf-8"},
|
||||
method="POST",
|
||||
)
|
||||
with urllib.request.urlopen(req, timeout=180) as resp:
|
||||
return json.loads(resp.read().decode("utf-8", errors="replace"))
|
||||
|
||||
|
||||
def template_xml_path(root: Path, template: str) -> Path:
|
||||
return root / template / "Ext" / "Template.xml"
|
||||
|
||||
|
||||
def range_fields(merges: list[dict[str, Any]]) -> dict[str, set[int]]:
|
||||
fields: dict[str, set[int]] = {
|
||||
"top": set(),
|
||||
"left": set(),
|
||||
"bottom": set(),
|
||||
"right": set(),
|
||||
"width": set(),
|
||||
"height": set(),
|
||||
"top_zero": set(),
|
||||
"left_zero": set(),
|
||||
"bottom_zero": set(),
|
||||
"right_zero": set(),
|
||||
}
|
||||
for item in merges:
|
||||
one = (item.get("range") or {}).get("one_based") or {}
|
||||
zero = (item.get("range") or {}).get("zero_based") or {}
|
||||
for name in ("top", "left", "bottom", "right"):
|
||||
if isinstance(one.get(name), int):
|
||||
fields[name].add(int(one[name]))
|
||||
if isinstance(zero.get(name), int):
|
||||
fields[f"{name}_zero"].add(int(zero[name]))
|
||||
for name in ("width", "height"):
|
||||
if isinstance(item.get(name), int):
|
||||
fields[name].add(int(item[name]))
|
||||
return fields
|
||||
|
||||
|
||||
def fetch_merge_candidate(adapter_url: str, base_id: str, owner_kind: str, owner_name: str, template: str) -> dict[str, Any]:
|
||||
data = rpc(
|
||||
adapter_url,
|
||||
"templates.read",
|
||||
{
|
||||
"base_id": base_id,
|
||||
"kind": owner_kind,
|
||||
"name": owner_name,
|
||||
"template": template,
|
||||
"sections": "merges",
|
||||
"max_merged": 1,
|
||||
"refresh_cache": False,
|
||||
},
|
||||
)
|
||||
return (((data.get("templates") or [{}])[0].get("structure") or {}).get("merge_record_block_candidates") or [{}])[0]
|
||||
|
||||
|
||||
def fetch_merge_records(
|
||||
adapter_url: str,
|
||||
base_id: str,
|
||||
owner_kind: str,
|
||||
owner_name: str,
|
||||
template: str,
|
||||
candidate: dict[str, Any],
|
||||
) -> list[dict[str, Any]]:
|
||||
position = str(candidate.get("tree_position") or "$.0")
|
||||
try:
|
||||
start = int(position.split(".")[1]) + 1
|
||||
except (IndexError, ValueError):
|
||||
start = 0
|
||||
count = int(candidate.get("count") or 0)
|
||||
data = rpc(
|
||||
adapter_url,
|
||||
"templates.read",
|
||||
{
|
||||
"base_id": base_id,
|
||||
"kind": owner_kind,
|
||||
"name": owner_name,
|
||||
"template": template,
|
||||
"sections": "moxel_records",
|
||||
"max_moxel_records": count + 40,
|
||||
"moxel_record_start": start,
|
||||
"moxel_record_end": start + count + 35,
|
||||
"refresh_cache": False,
|
||||
},
|
||||
)
|
||||
diagnostics = ((data.get("templates") or [{}])[0].get("structure") or {}).get("moxel_record_diagnostics") or [{}]
|
||||
if isinstance(diagnostics, list):
|
||||
diagnostics = diagnostics[0] if diagnostics else {}
|
||||
return [record for record in diagnostics.get("top_level_records") or [] if isinstance(record, dict)][:count]
|
||||
|
||||
|
||||
def values_by_slot(records: list[dict[str, Any]]) -> dict[int, list[int]]:
|
||||
result: dict[int, list[int]] = {}
|
||||
for record in records:
|
||||
numbers = record.get("numeric_items") if isinstance(record.get("numeric_items"), list) else []
|
||||
for slot, value in enumerate(numbers):
|
||||
if isinstance(value, int):
|
||||
result.setdefault(slot, []).append(value)
|
||||
return result
|
||||
|
||||
|
||||
def score_values(values: list[int], expected: set[int]) -> dict[str, Any]:
|
||||
if not values or not expected:
|
||||
return {"hits": 0, "coverage": 0.0, "precision": 0.0, "score": 0.0}
|
||||
distinct = set(values)
|
||||
hits = distinct & expected
|
||||
coverage = len(hits) / len(expected)
|
||||
precision = len(hits) / len(distinct)
|
||||
return {
|
||||
"hits": len(hits),
|
||||
"coverage": round(coverage, 4),
|
||||
"precision": round(precision, 4),
|
||||
"score": round((coverage * 0.7) + (precision * 0.3), 4),
|
||||
"hit_values": sorted(hits)[:80],
|
||||
"distinct_values": len(distinct),
|
||||
}
|
||||
|
||||
|
||||
def slot_candidates(records: list[dict[str, Any]], fields: dict[str, set[int]]) -> list[dict[str, Any]]:
|
||||
candidates: list[dict[str, Any]] = []
|
||||
by_slot = values_by_slot(records)
|
||||
for slot, values in sorted(by_slot.items()):
|
||||
transforms = {
|
||||
"raw": values,
|
||||
"raw_plus_1": [value + 1 for value in values],
|
||||
"raw_div32": [value // 32 for value in values if value > 0 and value <= 4096 and value % 32 == 0],
|
||||
"raw_div32_plus_1": [(value // 32) + 1 for value in values if value > 0 and value <= 4096 and value % 32 == 0],
|
||||
}
|
||||
for transform, transformed_values in transforms.items():
|
||||
for field, expected in fields.items():
|
||||
score = score_values(transformed_values, expected)
|
||||
if score["hits"] <= 0:
|
||||
continue
|
||||
candidates.append(
|
||||
{
|
||||
"slot": slot,
|
||||
"transform": transform,
|
||||
"field": field,
|
||||
**score,
|
||||
"sample_values": sorted(set(transformed_values))[:30],
|
||||
}
|
||||
)
|
||||
candidates.sort(key=lambda item: (-float(item.get("score") or 0), -float(item.get("coverage") or 0), -float(item.get("precision") or 0), int(item.get("slot") or 0), str(item.get("field") or "")))
|
||||
return candidates
|
||||
|
||||
|
||||
def xml_ordered_fields(merges: list[dict[str, Any]]) -> list[dict[str, int]]:
|
||||
result: list[dict[str, int]] = []
|
||||
for item in merges:
|
||||
one = (item.get("range") or {}).get("one_based") or {}
|
||||
zero = (item.get("range") or {}).get("zero_based") or {}
|
||||
row: dict[str, int] = {}
|
||||
for name in ("top", "left", "bottom", "right"):
|
||||
if isinstance(one.get(name), int):
|
||||
row[name] = int(one[name])
|
||||
if isinstance(zero.get(name), int):
|
||||
row[f"{name}_zero"] = int(zero[name])
|
||||
for name in ("width", "height"):
|
||||
if isinstance(item.get(name), int):
|
||||
row[name] = int(item[name])
|
||||
result.append(row)
|
||||
return result
|
||||
|
||||
|
||||
def transformed_record_value(numbers: list[Any], slot: int, transform: str) -> int | None:
|
||||
if slot >= len(numbers) or not isinstance(numbers[slot], int):
|
||||
return None
|
||||
value = int(numbers[slot])
|
||||
if transform == "raw":
|
||||
return value
|
||||
if transform == "raw_plus_1":
|
||||
return value + 1
|
||||
if transform == "raw_div32":
|
||||
if value <= 0 or value > 4096 or value % 32 != 0:
|
||||
return None
|
||||
return value // 32
|
||||
if transform == "raw_div32_plus_1":
|
||||
if value <= 0 or value > 4096 or value % 32 != 0:
|
||||
return None
|
||||
return (value // 32) + 1
|
||||
return None
|
||||
|
||||
|
||||
def ordered_slot_candidates(records: list[dict[str, Any]], merges: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
||||
ordered = xml_ordered_fields(merges)
|
||||
transforms = ("raw", "raw_plus_1", "raw_div32", "raw_div32_plus_1")
|
||||
fields = ("top", "left", "bottom", "right", "width", "height", "top_zero", "left_zero", "bottom_zero", "right_zero")
|
||||
candidates: list[dict[str, Any]] = []
|
||||
max_slots = max((len(record.get("numeric_items") or []) for record in records), default=0)
|
||||
for offset in range(0, min(25, len(records))):
|
||||
pair_count = min(len(ordered), max(0, len(records) - offset))
|
||||
if pair_count < max(10, min(len(ordered), 20)):
|
||||
continue
|
||||
for slot in range(max_slots):
|
||||
for transform in transforms:
|
||||
values = [
|
||||
transformed_record_value(records[offset + index].get("numeric_items") or [], slot, transform)
|
||||
for index in range(pair_count)
|
||||
]
|
||||
available = sum(1 for value in values if value is not None)
|
||||
if available < max(5, pair_count // 3):
|
||||
continue
|
||||
for field in fields:
|
||||
matches = [
|
||||
index + 1
|
||||
for index, value in enumerate(values)
|
||||
if value is not None and ordered[index].get(field) == value
|
||||
]
|
||||
if not matches:
|
||||
continue
|
||||
exact_ratio = len(matches) / pair_count
|
||||
available_ratio = len(matches) / available
|
||||
if exact_ratio < 0.1 and len(matches) < 8:
|
||||
continue
|
||||
candidates.append(
|
||||
{
|
||||
"offset": offset,
|
||||
"slot": slot,
|
||||
"transform": transform,
|
||||
"field": field,
|
||||
"pairs": pair_count,
|
||||
"available": available,
|
||||
"matches": len(matches),
|
||||
"exact_ratio": round(exact_ratio, 4),
|
||||
"available_ratio": round(available_ratio, 4),
|
||||
"score": round((exact_ratio * 0.75) + (available_ratio * 0.25), 4),
|
||||
"first_match_indexes": matches[:30],
|
||||
}
|
||||
)
|
||||
candidates.sort(
|
||||
key=lambda item: (
|
||||
-float(item.get("score") or 0),
|
||||
-float(item.get("exact_ratio") or 0),
|
||||
-int(item.get("matches") or 0),
|
||||
int(item.get("offset") or 0),
|
||||
int(item.get("slot") or 0),
|
||||
)
|
||||
)
|
||||
return candidates
|
||||
|
||||
|
||||
def analyze_template(
|
||||
*,
|
||||
adapter_url: str,
|
||||
base_id: str,
|
||||
owner_kind: str,
|
||||
owner_name: str,
|
||||
template: str,
|
||||
xml_root: Path,
|
||||
) -> dict[str, Any]:
|
||||
merges = merge_ranges(ET.parse(template_xml_path(xml_root, template)).getroot(), limit=1000)
|
||||
candidate = fetch_merge_candidate(adapter_url, base_id, owner_kind, owner_name, template)
|
||||
analysis = ((candidate.get("evidence") or {}).get("record_analysis") or {})
|
||||
records = fetch_merge_records(adapter_url, base_id, owner_kind, owner_name, template, candidate)
|
||||
fields = range_fields(merges)
|
||||
return {
|
||||
"template": template,
|
||||
"xml_merge_count": len(merges),
|
||||
"sql_block_count": int(candidate.get("count") or 0),
|
||||
"tree_position": candidate.get("tree_position"),
|
||||
"xml_field_values": {name: sorted(values) for name, values in fields.items()},
|
||||
"record_analysis": {
|
||||
"schema": analysis.get("schema"),
|
||||
"records_analyzed": analysis.get("records_analyzed"),
|
||||
"records_available": len(records),
|
||||
"raw_records_source": "templates.read.sections=moxel_records",
|
||||
},
|
||||
"slot_candidates": slot_candidates(records, fields)[:120],
|
||||
"ordered_slot_candidates": ordered_slot_candidates(records, merges)[:120],
|
||||
}
|
||||
|
||||
|
||||
def render_markdown(payload: dict[str, Any]) -> str:
|
||||
lines = ["# MOXCEL merge slot candidate analysis", ""]
|
||||
lines.append("XML is used only as an analysis fixture; candidates are SQL decoder hypotheses.")
|
||||
lines.append("")
|
||||
for item in payload.get("items") or []:
|
||||
lines.append(f"## {item.get('template')}")
|
||||
lines.append("")
|
||||
lines.append(f"- XML merges: `{item.get('xml_merge_count')}`")
|
||||
lines.append(f"- SQL block count: `{item.get('sql_block_count')}` at `{item.get('tree_position')}`")
|
||||
ra = item.get("record_analysis") or {}
|
||||
lines.append(f"- Record analysis: `{ra.get('schema')}`, records `{ra.get('records_available')}/{ra.get('records_analyzed')}`")
|
||||
lines.append("")
|
||||
lines.append("| Slot | Transform | Field | Score | Coverage | Precision | Hit values | Sample values |")
|
||||
lines.append("| ---: | --- | --- | ---: | ---: | ---: | --- | --- |")
|
||||
for row in (item.get("slot_candidates") or [])[:40]:
|
||||
lines.append(
|
||||
f"| {row.get('slot')} | `{row.get('transform')}` | `{row.get('field')}` | "
|
||||
f"{row.get('score')} | {row.get('coverage')} | {row.get('precision')} | "
|
||||
f"`{row.get('hit_values')}` | `{row.get('sample_values')}` |"
|
||||
)
|
||||
lines.append("")
|
||||
lines.append("### Ordered Slot Candidates")
|
||||
lines.append("")
|
||||
lines.append("| Offset | Slot | Transform | Field | Score | Exact ratio | Available ratio | Matches | First match indexes |")
|
||||
lines.append("| ---: | ---: | --- | --- | ---: | ---: | ---: | ---: | --- |")
|
||||
for row in (item.get("ordered_slot_candidates") or [])[:40]:
|
||||
lines.append(
|
||||
f"| {row.get('offset')} | {row.get('slot')} | `{row.get('transform')}` | `{row.get('field')}` | "
|
||||
f"{row.get('score')} | {row.get('exact_ratio')} | {row.get('available_ratio')} | "
|
||||
f"{row.get('matches')}/{row.get('pairs')} | `{row.get('first_match_indexes')}` |"
|
||||
)
|
||||
lines.append("")
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Score SQL MOXCEL merge-block numeric slots against XML merge range fields.")
|
||||
parser.add_argument("--adapter-url", default="http://docker-gpu.cin.su:8011")
|
||||
parser.add_argument("--base-id", default="upo_test")
|
||||
parser.add_argument("--owner-kind", default="Document")
|
||||
parser.add_argument("--owner-name", default="АвансовыйОтчет")
|
||||
parser.add_argument(
|
||||
"--xml-root",
|
||||
default=r"Z:\codex\1C\XML\UPO\Структура базы 1с\Конфигурация\Documents\АвансовыйОтчет\Templates",
|
||||
)
|
||||
parser.add_argument("--template", action="append", required=True)
|
||||
parser.add_argument("--output-json", default="reports/1c-template-baselines/moxel-merge-slot-candidates.json")
|
||||
parser.add_argument("--output-markdown", default="reports/1c-template-baselines/moxel-merge-slot-candidates.md")
|
||||
args = parser.parse_args()
|
||||
payload = {
|
||||
"schema": "codex_1c_moxel_merge_slot_candidates.v1",
|
||||
"source": "analysis_only_xml_fixture",
|
||||
"items": [
|
||||
analyze_template(
|
||||
adapter_url=args.adapter_url,
|
||||
base_id=args.base_id,
|
||||
owner_kind=args.owner_kind,
|
||||
owner_name=args.owner_name,
|
||||
template=template,
|
||||
xml_root=Path(args.xml_root),
|
||||
)
|
||||
for template in args.template
|
||||
],
|
||||
}
|
||||
Path(args.output_json).write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
Path(args.output_markdown).write_text(render_markdown(payload), encoding="utf-8")
|
||||
print(json.dumps({"status": "ok", "json": args.output_json, "markdown": args.output_markdown, "items": len(payload["items"])}, ensure_ascii=False, indent=2))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,148 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
FIELDS = ("left", "right", "top", "bottom")
|
||||
|
||||
|
||||
def read_json(path: Path) -> dict[str, Any]:
|
||||
return json.loads(path.read_text(encoding="utf-8"))
|
||||
|
||||
|
||||
def as_int(value: Any) -> int | None:
|
||||
try:
|
||||
return int(value)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
def probe_ranges(payload: dict[str, Any]) -> list[dict[str, Any]]:
|
||||
probe = payload.get("probe") if isinstance(payload.get("probe"), dict) else payload
|
||||
ranges = probe.get("named_ranges") or probe.get("named_range_candidates") or []
|
||||
return [item for item in ranges if isinstance(item, dict)]
|
||||
|
||||
|
||||
def candidate_indexes(raw_scalars: list[Any], expected_one_based: int) -> list[int]:
|
||||
result = []
|
||||
for index, value in enumerate(raw_scalars):
|
||||
if index < 2 or index > 5:
|
||||
continue
|
||||
parsed = as_int(value)
|
||||
if parsed is not None and parsed + 1 == expected_one_based:
|
||||
result.append(index)
|
||||
return result
|
||||
|
||||
|
||||
def analyze_range(item: dict[str, Any]) -> dict[str, Any] | None:
|
||||
range_info = item.get("range") if isinstance(item.get("range"), dict) else {}
|
||||
one_based = range_info.get("one_based") if isinstance(range_info.get("one_based"), dict) else {}
|
||||
raw_scalars = item.get("raw_scalars") if isinstance(item.get("raw_scalars"), list) else []
|
||||
if not one_based or not raw_scalars:
|
||||
return None
|
||||
field_candidates: dict[str, list[int]] = {}
|
||||
for field in FIELDS:
|
||||
expected = as_int(one_based.get(field))
|
||||
if expected is None:
|
||||
continue
|
||||
field_candidates[field] = candidate_indexes(raw_scalars, expected)
|
||||
unique_values = len({one_based.get(field) for field in FIELDS if one_based.get(field) is not None})
|
||||
return {
|
||||
"name": item.get("name"),
|
||||
"kind": item.get("kind"),
|
||||
"one_based": {field: one_based.get(field) for field in FIELDS if field in one_based},
|
||||
"raw_scalars": raw_scalars,
|
||||
"field_candidates": field_candidates,
|
||||
"distinct_coordinate_values": unique_values,
|
||||
}
|
||||
|
||||
|
||||
def aggregate_rules(samples: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
||||
rules = []
|
||||
for field in FIELDS:
|
||||
sample_candidates = [set(sample.get("field_candidates", {}).get(field) or []) for sample in samples if sample.get("field_candidates", {}).get(field)]
|
||||
if not sample_candidates:
|
||||
continue
|
||||
intersection = set.intersection(*sample_candidates) if sample_candidates else set()
|
||||
all_distinct = all(int(sample.get("distinct_coordinate_values") or 0) >= 4 for sample in samples)
|
||||
confidence = "high" if len(intersection) == 1 and all_distinct else "medium" if intersection else "low"
|
||||
rules.append(
|
||||
{
|
||||
"target": f"moxel.named_range.{field}",
|
||||
"expression": "one_based = int(raw_scalar) + 1",
|
||||
"raw_scalar_indexes": sorted(intersection) if intersection else sorted(set.union(*sample_candidates)),
|
||||
"confidence": confidence,
|
||||
"evidence": {
|
||||
"samples": len(sample_candidates),
|
||||
"distinct_rectangular_samples": sum(1 for sample in samples if int(sample.get("distinct_coordinate_values") or 0) >= 4),
|
||||
},
|
||||
}
|
||||
)
|
||||
return rules
|
||||
|
||||
|
||||
def analyze(probes: list[dict[str, Any]], target_name: str | None = None) -> dict[str, Any]:
|
||||
samples = []
|
||||
for payload in probes:
|
||||
for item in probe_ranges(payload):
|
||||
if target_name and str(item.get("name") or "") != target_name:
|
||||
continue
|
||||
sample = analyze_range(item)
|
||||
if sample:
|
||||
samples.append(sample)
|
||||
return {
|
||||
"schema": "codex_1c_moxel_named_range_rule_analysis.v1",
|
||||
"target_name": target_name,
|
||||
"status": "ok",
|
||||
"samples": samples,
|
||||
"rules": aggregate_rules(samples),
|
||||
"counts": {
|
||||
"samples": len(samples),
|
||||
"rules": 0,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def render_markdown(payload: dict[str, Any]) -> str:
|
||||
payload["counts"]["rules"] = len(payload.get("rules") or [])
|
||||
lines = ["# 1C MOXCEL Named Range Rule Analysis", ""]
|
||||
lines.append(f"- Samples: `{payload.get('counts', {}).get('samples')}`")
|
||||
lines.append(f"- Rules: `{payload.get('counts', {}).get('rules')}`")
|
||||
lines.append("")
|
||||
lines.append("| Target | Confidence | Raw indexes | Samples |")
|
||||
lines.append("| --- | --- | --- | --- |")
|
||||
for rule in payload.get("rules") or []:
|
||||
evidence = rule.get("evidence") or {}
|
||||
lines.append(
|
||||
f"| `{rule.get('target')}` | `{rule.get('confidence')}` | "
|
||||
f"`{', '.join(map(str, rule.get('raw_scalar_indexes') or []))}` | `{evidence.get('samples')}` |"
|
||||
)
|
||||
lines.append("")
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Infer MOXCEL named range coordinate scalar indexes from probe snapshots.")
|
||||
parser.add_argument("--probe", action="append", required=True, help="Probe snapshot JSON. Repeatable.")
|
||||
parser.add_argument("--target-name", help="Optional named range to analyze.")
|
||||
parser.add_argument("--output-json", default="reports/1c-template-baselines/moxel-named-range-rules.json")
|
||||
parser.add_argument("--output-markdown", default="reports/1c-template-baselines/moxel-named-range-rules.md")
|
||||
args = parser.parse_args()
|
||||
|
||||
payload = analyze([read_json(Path(path)) for path in args.probe], args.target_name)
|
||||
payload["counts"]["rules"] = len(payload.get("rules") or [])
|
||||
json_path = Path(args.output_json)
|
||||
md_path = Path(args.output_markdown)
|
||||
json_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
md_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
json_path.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
md_path.write_text(render_markdown(payload), encoding="utf-8")
|
||||
print(json.dumps({"status": "ok", "json": str(json_path), "markdown": str(md_path), "counts": payload["counts"]}, ensure_ascii=False, indent=2))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,306 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
VOLATILE_KEYS = {
|
||||
"captured_at",
|
||||
"adapter_url",
|
||||
"modified",
|
||||
"bytes",
|
||||
"file_name",
|
||||
"template_file",
|
||||
"label",
|
||||
"diff",
|
||||
"cell_id",
|
||||
}
|
||||
|
||||
|
||||
def read_json(path: Path) -> dict[str, Any]:
|
||||
return json.loads(path.read_text(encoding="utf-8"))
|
||||
|
||||
|
||||
def resolve_manifest_path(value: str, base_dir: Path) -> Path:
|
||||
path = Path(value)
|
||||
if path.is_absolute():
|
||||
return path
|
||||
candidates = [
|
||||
base_dir / path,
|
||||
ROOT / path,
|
||||
Path.cwd() / path,
|
||||
path,
|
||||
]
|
||||
for candidate in candidates:
|
||||
resolved = candidate.resolve()
|
||||
if resolved.exists():
|
||||
return resolved
|
||||
return (base_dir / path).resolve()
|
||||
|
||||
|
||||
def unwrap_structure(payload: dict[str, Any]) -> dict[str, Any]:
|
||||
if isinstance(payload.get("probe"), dict):
|
||||
return payload["probe"]
|
||||
if isinstance(payload.get("structure"), dict):
|
||||
return payload["structure"]
|
||||
return payload
|
||||
|
||||
|
||||
def compact_next_record(value: Any) -> Any:
|
||||
if not isinstance(value, dict):
|
||||
return value
|
||||
result = {}
|
||||
for key in ("type", "value", "head", "scalar_prefix", "list_length", "tree_position"):
|
||||
if key in value:
|
||||
result[key] = value[key]
|
||||
return result
|
||||
|
||||
|
||||
def normalize_item(item: Any) -> Any:
|
||||
if not isinstance(item, dict):
|
||||
return item
|
||||
result: dict[str, Any] = {}
|
||||
for key, value in item.items():
|
||||
if key in VOLATILE_KEYS:
|
||||
continue
|
||||
if key == "next_moxel_record":
|
||||
result[key] = compact_next_record(value)
|
||||
elif key == "style_evidence" and isinstance(value, dict):
|
||||
result[key] = {
|
||||
style_key: style_value
|
||||
for style_key, style_value in value.items()
|
||||
if style_key in {"immediate_preceding_values", "last_7_preceding_values"}
|
||||
}
|
||||
elif isinstance(value, dict):
|
||||
result[key] = normalize_item(value)
|
||||
elif isinstance(value, list):
|
||||
result[key] = [normalize_item(child) for child in value]
|
||||
else:
|
||||
result[key] = value
|
||||
return result
|
||||
|
||||
|
||||
def stable_key(item: dict[str, Any], fallback_index: int) -> str:
|
||||
for key in ("text", "name"):
|
||||
if item.get(key) not in {None, ""}:
|
||||
return f"{key}:{item.get(key)}"
|
||||
if item.get("tree_position"):
|
||||
return f"tree:{item.get('tree_position')}"
|
||||
if item.get("one_based"):
|
||||
return f"cell:{json.dumps(item.get('one_based'), ensure_ascii=False, sort_keys=True)}"
|
||||
return f"index:{fallback_index}"
|
||||
|
||||
|
||||
def normalize_section_list(items: Any) -> dict[str, Any]:
|
||||
if not isinstance(items, list):
|
||||
return {}
|
||||
result: dict[str, Any] = {}
|
||||
for index, item in enumerate(items):
|
||||
if not isinstance(item, dict):
|
||||
result[f"index:{index}"] = normalize_item(item)
|
||||
continue
|
||||
key = stable_key(item, index)
|
||||
if key in result:
|
||||
key = f"{key}#{index}"
|
||||
result[key] = normalize_item(item)
|
||||
return result
|
||||
|
||||
|
||||
def normalized_structure(payload: dict[str, Any], *, target_text: str | None = None, target_name: str | None = None) -> dict[str, Any]:
|
||||
structure = unwrap_structure(payload)
|
||||
result: dict[str, Any] = {
|
||||
"counts": normalize_item(structure.get("counts") or {}),
|
||||
"dimensions": normalize_item(structure.get("dimensions") or {}),
|
||||
"cells": normalize_section_list(structure.get("cells") or []),
|
||||
"cell_style_candidates": normalize_section_list(structure.get("cell_style_candidates") or structure.get("cell_styles") or []),
|
||||
"named_range_candidates": normalize_section_list(structure.get("named_range_candidates") or structure.get("named_ranges") or []),
|
||||
"named_areas": normalize_section_list(structure.get("named_areas") or []),
|
||||
"column_widths": normalize_section_list(structure.get("column_widths") or []),
|
||||
"row_heights": normalize_section_list(structure.get("row_heights") or []),
|
||||
"merged_ranges": normalize_section_list(structure.get("merged_ranges") or []),
|
||||
"merged_range_candidates": normalize_section_list(structure.get("merged_range_candidates") or []),
|
||||
}
|
||||
if target_text:
|
||||
result["target_cell_styles"] = {
|
||||
key: value
|
||||
for key, value in result["cell_style_candidates"].items()
|
||||
if isinstance(value, dict) and str(value.get("text") or "") == target_text
|
||||
}
|
||||
result["target_cells"] = {
|
||||
key: value
|
||||
for key, value in result["cells"].items()
|
||||
if isinstance(value, dict) and str(value.get("text") or "") == target_text
|
||||
}
|
||||
if target_name:
|
||||
result["target_named_ranges"] = {
|
||||
key: value
|
||||
for key, value in result["named_range_candidates"].items()
|
||||
if isinstance(value, dict) and str(value.get("name") or "") == target_name
|
||||
}
|
||||
return result
|
||||
|
||||
|
||||
def diff_values(before: Any, after: Any, path: str = "$") -> list[dict[str, Any]]:
|
||||
if before == after:
|
||||
return []
|
||||
if isinstance(before, dict) and isinstance(after, dict):
|
||||
changes: list[dict[str, Any]] = []
|
||||
for key in sorted(set(before) | set(after)):
|
||||
changes.extend(diff_values(before.get(key), after.get(key), f"{path}.{key}"))
|
||||
return changes
|
||||
if isinstance(before, list) and isinstance(after, list):
|
||||
changes = []
|
||||
for index in range(max(len(before), len(after))):
|
||||
old = before[index] if index < len(before) else None
|
||||
new = after[index] if index < len(after) else None
|
||||
changes.extend(diff_values(old, new, f"{path}[{index}]"))
|
||||
return changes
|
||||
return [{"path": path, "before": before, "after": after}]
|
||||
|
||||
|
||||
def score_change(change: dict[str, Any], target_text: str | None, target_name: str | None) -> int:
|
||||
path = str(change.get("path") or "")
|
||||
score = 0
|
||||
if "target_" in path:
|
||||
score += 40
|
||||
if target_text and target_text in path:
|
||||
score += 30
|
||||
if target_name and target_name in path:
|
||||
score += 30
|
||||
if any(part in path for part in ("next_moxel_record", "style_evidence", "raw_scalars", "column_widths", "row_heights", "merged")):
|
||||
score += 15
|
||||
if ".counts." in path:
|
||||
score -= 20
|
||||
if ".tree_position" in path:
|
||||
score -= 10
|
||||
if path.endswith(".cell_id"):
|
||||
score -= 20
|
||||
if change.get("before") is None or change.get("after") is None:
|
||||
score -= 5
|
||||
return score
|
||||
|
||||
|
||||
def analyze_experiment(experiment: dict[str, Any], base_dir: Path) -> dict[str, Any]:
|
||||
before_path = resolve_manifest_path(str(experiment["before"]), base_dir)
|
||||
after_path = resolve_manifest_path(str(experiment["after"]), base_dir)
|
||||
target_text = experiment.get("target_text")
|
||||
target_name = experiment.get("target_name")
|
||||
before = normalized_structure(read_json(before_path), target_text=target_text, target_name=target_name)
|
||||
after = normalized_structure(read_json(after_path), target_text=target_text, target_name=target_name)
|
||||
changes = diff_values(before, after)
|
||||
scored = sorted(
|
||||
(
|
||||
{
|
||||
**change,
|
||||
"score": score_change(change, str(target_text) if target_text else None, str(target_name) if target_name else None),
|
||||
}
|
||||
for change in changes
|
||||
),
|
||||
key=lambda item: (-int(item.get("score") or 0), str(item.get("path") or "")),
|
||||
)
|
||||
min_positive = [item for item in scored if int(item.get("score") or 0) > 0]
|
||||
candidates = min_positive[: int(experiment.get("max_candidates") or 20)]
|
||||
confidence = "none"
|
||||
if len(candidates) == 1 and candidates[0]["score"] >= 40:
|
||||
confidence = "high"
|
||||
elif candidates and candidates[0]["score"] >= 40:
|
||||
confidence = "medium"
|
||||
elif candidates:
|
||||
confidence = "low"
|
||||
return {
|
||||
"property": experiment.get("property"),
|
||||
"operation": experiment.get("operation"),
|
||||
"target_text": target_text,
|
||||
"target_name": target_name,
|
||||
"before": str(before_path),
|
||||
"after": str(after_path),
|
||||
"confidence": confidence,
|
||||
"candidate_paths": candidates,
|
||||
"counts": {"changes": len(changes), "candidate_paths": len(candidates)},
|
||||
}
|
||||
|
||||
|
||||
def default_probe_plan() -> list[dict[str, Any]]:
|
||||
return [
|
||||
{"property": "ГоризонтальноеПоложение", "values": ["Лево", "Центр", "Право"], "target": "cell"},
|
||||
{"property": "ВертикальноеПоложение", "values": ["Верх", "Центр", "Низ"], "target": "cell"},
|
||||
{"property": "ЦветТекста", "values": ["Черный", "Красный", "Синий"], "target": "cell"},
|
||||
{"property": "ЦветФона", "values": ["Нет", "Желтый", "Серый"], "target": "cell"},
|
||||
{"property": "Шрифт.Имя", "values": ["Arial", "Courier New"], "target": "cell"},
|
||||
{"property": "Шрифт.Размер", "values": [8, 10, 14], "target": "cell"},
|
||||
{"property": "ГраницаЛево", "values": ["Нет", "Тонкая", "Толстая"], "target": "cell"},
|
||||
{"property": "ГраницаВерх", "values": ["Нет", "Тонкая", "Толстая"], "target": "cell"},
|
||||
{"property": "Защита", "values": [True, False], "target": "cell"},
|
||||
{"property": "Гиперссылка", "values": ["", "https://example.invalid/1c-moxel-probe"], "target": "cell"},
|
||||
{"property": "Переносить", "values": [True, False], "target": "cell"},
|
||||
{"property": "ШиринаКолонки", "values": [8, 12, 20], "target": "column"},
|
||||
{"property": "ВысотаСтроки", "values": [12, 18, 24], "target": "row"},
|
||||
{"property": "Объединение", "values": ["none", "R8C4:R8C5"], "target": "range"},
|
||||
]
|
||||
|
||||
|
||||
def render_markdown(payload: dict[str, Any]) -> str:
|
||||
lines: list[str] = ["# 1C MOXCEL property experiments", ""]
|
||||
lines.append(f"- Experiments: `{len(payload.get('experiments') or [])}`")
|
||||
lines.append("")
|
||||
if payload.get("probe_plan"):
|
||||
lines.append("## Probe Plan")
|
||||
lines.append("")
|
||||
lines.append("| Property | Target | Values |")
|
||||
lines.append("| --- | --- | --- |")
|
||||
for item in payload["probe_plan"]:
|
||||
lines.append(f"| `{item.get('property')}` | `{item.get('target')}` | `{json.dumps(item.get('values'), ensure_ascii=False)}` |")
|
||||
lines.append("")
|
||||
if payload.get("experiments"):
|
||||
lines.append("## Results")
|
||||
lines.append("")
|
||||
lines.append("| Property | Confidence | Changes | Top path |")
|
||||
lines.append("| --- | --- | --- | --- |")
|
||||
for item in payload["experiments"]:
|
||||
top = (item.get("candidate_paths") or [{}])[0]
|
||||
lines.append(
|
||||
f"| `{item.get('property')}` | `{item.get('confidence')}` | "
|
||||
f"`{(item.get('counts') or {}).get('changes')}` | `{top.get('path') or ''}` |"
|
||||
)
|
||||
lines.append("")
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Analyze controlled 1C MOXCEL one-property experiments.")
|
||||
parser.add_argument("--manifest", help="Experiment manifest JSON.")
|
||||
parser.add_argument("--output-json", default="reports/1c-template-baselines/Primer3_moxel_property_experiments.json")
|
||||
parser.add_argument("--output-markdown", default="reports/1c-template-baselines/Primer3_moxel_property_experiments.md")
|
||||
parser.add_argument("--emit-default-plan", action="store_true", help="Include the default next probe plan.")
|
||||
args = parser.parse_args()
|
||||
|
||||
manifest_path = Path(args.manifest).resolve() if args.manifest else None
|
||||
manifest = read_json(manifest_path) if manifest_path else {"experiments": []}
|
||||
base_dir = manifest_path.parent if manifest_path else Path.cwd()
|
||||
experiments = [
|
||||
analyze_experiment(experiment, base_dir)
|
||||
for experiment in manifest.get("experiments") or []
|
||||
if isinstance(experiment, dict) and experiment.get("before") and experiment.get("after")
|
||||
]
|
||||
payload = {
|
||||
"schema": "codex_1c_moxel_property_experiments.v1",
|
||||
"manifest": str(manifest_path) if manifest_path else None,
|
||||
"experiments": experiments,
|
||||
"probe_plan": default_probe_plan() if args.emit_default_plan or not experiments else [],
|
||||
}
|
||||
json_path = Path(args.output_json)
|
||||
md_path = Path(args.output_markdown)
|
||||
json_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
md_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
json_path.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
md_path.write_text(render_markdown(payload), encoding="utf-8")
|
||||
print(json.dumps({"status": "ok", "json": str(json_path), "markdown": str(md_path), "experiments": len(experiments)}, ensure_ascii=False, indent=2))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,376 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Analyze saved-state object changes beyond storage bytes."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import difflib
|
||||
import importlib.util
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
WORD_RE = re.compile(r"[\wА-Яа-яЁё]{3,}", re.UNICODE)
|
||||
BASE64ISH_RE = re.compile(r"^[A-Za-z0-9+/=_-]{24,}$")
|
||||
REPO_ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
|
||||
def load_payload_to_text():
|
||||
module_path = REPO_ROOT / "plugins" / "1c" / "parser" / "payload.py"
|
||||
spec = importlib.util.spec_from_file_location("onec_payload", module_path)
|
||||
if spec is None or spec.loader is None:
|
||||
raise RuntimeError(f"Cannot load payload parser: {module_path}")
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
sys.modules[spec.name] = module
|
||||
spec.loader.exec_module(module)
|
||||
return module.payload_to_text
|
||||
|
||||
|
||||
payload_to_text = load_payload_to_text()
|
||||
|
||||
|
||||
def load_json(path: Path) -> dict[str, Any]:
|
||||
return json.loads(path.read_text(encoding="utf-8-sig"))
|
||||
|
||||
|
||||
def write_json(path: Path, data: dict[str, Any]) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(json.dumps(data, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
|
||||
|
||||
def safe_join(root: Path, file_name: str) -> Path:
|
||||
relative = Path(file_name.replace("\\", "/"))
|
||||
if relative.is_absolute() or ".." in relative.parts or not str(relative):
|
||||
raise ValueError(file_name)
|
||||
return root / relative
|
||||
|
||||
|
||||
def words(text: str) -> list[str]:
|
||||
result: list[str] = []
|
||||
seen: set[str] = set()
|
||||
for match in WORD_RE.finditer(text):
|
||||
value = match.group(0)
|
||||
key = value.casefold()
|
||||
if key not in seen:
|
||||
result.append(value)
|
||||
seen.add(key)
|
||||
return result
|
||||
|
||||
|
||||
def semantic_words(values: list[str], *, limit: int = 40) -> list[str]:
|
||||
result: list[str] = []
|
||||
seen: set[str] = set()
|
||||
for value in values:
|
||||
if not value:
|
||||
continue
|
||||
if BASE64ISH_RE.match(value):
|
||||
continue
|
||||
if value.isdigit():
|
||||
continue
|
||||
if len(value) > 80:
|
||||
continue
|
||||
has_cyrillic = any("А" <= char <= "я" or char in "Ёё" for char in value)
|
||||
has_1c_shape = any(marker in value for marker in ("Форма", "Команда", "Реквизит", "Модуль", "Область", "Процедура", "Функция"))
|
||||
if not has_cyrillic and not has_1c_shape:
|
||||
continue
|
||||
key = value.casefold()
|
||||
if key in seen:
|
||||
continue
|
||||
result.append(value)
|
||||
seen.add(key)
|
||||
if len(result) >= limit:
|
||||
break
|
||||
return result
|
||||
|
||||
|
||||
def classify_payload_part(item: dict[str, Any], file_name: str) -> str:
|
||||
kind = str(item.get("kind") or "")
|
||||
suffix = ""
|
||||
if "." in file_name:
|
||||
suffix = file_name.rsplit(".", 1)[1]
|
||||
if kind in {"CommonModule", "ObjectModule", "ManagerModule"} and suffix == "0":
|
||||
return "bsl_module_text"
|
||||
if kind == "Form" and not suffix:
|
||||
return "form_descriptor"
|
||||
if kind == "Form" and suffix == "0":
|
||||
return "form_body"
|
||||
if suffix == "0":
|
||||
return "primary_payload"
|
||||
return "metadata_payload"
|
||||
|
||||
|
||||
def textish_from_payload(decoded: dict[str, Any]) -> tuple[str | None, str | None]:
|
||||
text = decoded.get("text")
|
||||
if text:
|
||||
return text, str(decoded.get("encoding") or "")
|
||||
payload = decoded.get("payload")
|
||||
if not isinstance(payload, bytes):
|
||||
return None, None
|
||||
candidates: list[tuple[str, str, int]] = []
|
||||
for encoding in ("utf-8-sig", "utf-8", "cp1251", "utf-16-le"):
|
||||
try:
|
||||
candidate = payload.decode(encoding, errors="ignore").replace("\x00", "").replace("\ufeff", "")
|
||||
except Exception:
|
||||
continue
|
||||
word_count = len(words(candidate))
|
||||
cyrillic_count = sum(1 for char in candidate if "А" <= char <= "я" or char in "Ёё")
|
||||
known_1c_terms = sum(
|
||||
candidate.count(term)
|
||||
for term in ("Процедура", "Функция", "Конец", "Если", "Тогда", "Область", "Перем", "Экспорт", "пример")
|
||||
)
|
||||
mojibake_penalty = candidate.count("Р") * 8 + candidate.count("С") * 4
|
||||
score = word_count * 5 + cyrillic_count + known_1c_terms * 500 - mojibake_penalty
|
||||
if word_count:
|
||||
candidates.append((candidate, f"{encoding}:lossy", score))
|
||||
if not candidates:
|
||||
return None, None
|
||||
candidates.sort(key=lambda item: item[2], reverse=True)
|
||||
return candidates[0][0], candidates[0][1]
|
||||
|
||||
|
||||
def common_edges(left: str, right: str) -> tuple[int, int]:
|
||||
prefix = 0
|
||||
for a, b in zip(left, right):
|
||||
if a != b:
|
||||
break
|
||||
prefix += 1
|
||||
suffix = 0
|
||||
left_tail = left[prefix:]
|
||||
right_tail = right[prefix:]
|
||||
for a, b in zip(reversed(left_tail), reversed(right_tail)):
|
||||
if a != b:
|
||||
break
|
||||
suffix += 1
|
||||
return prefix, suffix
|
||||
|
||||
|
||||
def text_window(text: str, center: int, size: int = 500) -> str:
|
||||
start = max(center - size // 2, 0)
|
||||
end = min(center + size // 2, len(text))
|
||||
return text[start:end].replace("\x00", "")
|
||||
|
||||
|
||||
def line_diff(left: str, right: str, *, limit: int) -> list[str]:
|
||||
left_lines = left.splitlines()
|
||||
right_lines = right.splitlines()
|
||||
diff = list(difflib.unified_diff(left_lines, right_lines, fromfile="active", tofile="saved", lineterm=""))
|
||||
if len(diff) > limit:
|
||||
return [*diff[:limit], f"... truncated {len(diff) - limit} lines ..."]
|
||||
return diff
|
||||
|
||||
|
||||
def analyze_payload(active_path: Path | None, saved_path: Path) -> dict[str, Any]:
|
||||
saved_raw = saved_path.read_bytes()
|
||||
saved_decoded = payload_to_text(saved_raw)
|
||||
active_decoded: dict[str, Any] | None = None
|
||||
if active_path and active_path.exists():
|
||||
active_decoded = payload_to_text(active_path.read_bytes())
|
||||
|
||||
result: dict[str, Any] = {
|
||||
"saved_path": str(saved_path),
|
||||
"active_path": str(active_path) if active_path else None,
|
||||
"saved": {
|
||||
"raw_bytes": saved_decoded.get("raw_bytes"),
|
||||
"payload_bytes": saved_decoded.get("payload_bytes"),
|
||||
"compression": saved_decoded.get("compression"),
|
||||
"encoding": saved_decoded.get("encoding"),
|
||||
},
|
||||
"active": None,
|
||||
"text_comparable": False,
|
||||
}
|
||||
if active_decoded:
|
||||
result["active"] = {
|
||||
"raw_bytes": active_decoded.get("raw_bytes"),
|
||||
"payload_bytes": active_decoded.get("payload_bytes"),
|
||||
"compression": active_decoded.get("compression"),
|
||||
"encoding": active_decoded.get("encoding"),
|
||||
}
|
||||
|
||||
saved_text, saved_text_mode = textish_from_payload(saved_decoded)
|
||||
active_text, active_text_mode = textish_from_payload(active_decoded) if active_decoded else (None, None)
|
||||
result["saved"]["text_mode"] = saved_text_mode
|
||||
if result["active"] is not None:
|
||||
result["active"]["text_mode"] = active_text_mode
|
||||
if saved_text is None:
|
||||
result["summary"] = "Saved payload is not text-decodable."
|
||||
return result
|
||||
result["saved_strings_sample"] = words(saved_text)[:80]
|
||||
if active_text is None:
|
||||
result["summary"] = "Saved text payload has no active counterpart."
|
||||
result["text_comparable"] = False
|
||||
result["saved_text_sample"] = text_window(saved_text, 0)
|
||||
return result
|
||||
|
||||
result["text_comparable"] = True
|
||||
prefix, suffix = common_edges(active_text, saved_text)
|
||||
active_words = {value.casefold(): value for value in words(active_text)}
|
||||
saved_words = {value.casefold(): value for value in words(saved_text)}
|
||||
added_keys = [key for key in saved_words if key not in active_words]
|
||||
removed_keys = [key for key in active_words if key not in saved_words]
|
||||
result["text_diff"] = {
|
||||
"active_chars": len(active_text),
|
||||
"saved_chars": len(saved_text),
|
||||
"delta_chars": len(saved_text) - len(active_text),
|
||||
"common_prefix_chars": prefix,
|
||||
"common_suffix_chars": suffix,
|
||||
"added_words": [saved_words[key] for key in added_keys[:80]],
|
||||
"removed_words": [active_words[key] for key in removed_keys[:80]],
|
||||
"active_window": text_window(active_text, prefix),
|
||||
"saved_window": text_window(saved_text, prefix),
|
||||
"unified_diff": line_diff(active_text, saved_text, limit=120),
|
||||
}
|
||||
result["semantic_hints"] = {
|
||||
"added_terms": semantic_words(result["text_diff"]["added_words"]),
|
||||
"removed_terms": semantic_words(result["text_diff"]["removed_words"]),
|
||||
}
|
||||
result["summary"] = "Text payload differs." if active_text != saved_text else "Text payload matches."
|
||||
return result
|
||||
|
||||
|
||||
def storage_root(saved_table: str, active_table: str, roots: dict[str, Path]) -> tuple[Path | None, Path | None]:
|
||||
saved_root = roots.get(saved_table)
|
||||
active_root = roots.get(active_table)
|
||||
return saved_root, active_root
|
||||
|
||||
|
||||
def build_extension_cas_map(summary_path: Path | None) -> dict[tuple[str, str], str]:
|
||||
if not summary_path or not summary_path.exists():
|
||||
return {}
|
||||
data = load_json(summary_path)
|
||||
result: dict[tuple[str, str], str] = {}
|
||||
for extension in data.get("extensions") or []:
|
||||
extension_name = str(extension.get("extension_name") or "")
|
||||
for obj in extension.get("sample_objects") or []:
|
||||
for part in obj.get("parts") or []:
|
||||
object_id = str(part.get("object_id") or "").casefold()
|
||||
cas_key = str(part.get("cas_key") or "")
|
||||
if extension_name and object_id and cas_key:
|
||||
result[(extension_name.casefold(), object_id)] = cas_key
|
||||
return result
|
||||
|
||||
|
||||
def extension_object_id_from_saved_file(file_name: str) -> str | None:
|
||||
if "__" not in file_name:
|
||||
return None
|
||||
object_id = file_name.split("__", 1)[1]
|
||||
if object_id == "configinfo":
|
||||
return None
|
||||
return object_id.casefold()
|
||||
|
||||
|
||||
def active_extension_path(item: dict[str, Any], file_name: str, extension_cas_map: dict[tuple[str, str], str], config_cas_all_dir: Path | None) -> tuple[Path | None, str | None]:
|
||||
if not config_cas_all_dir:
|
||||
return None, None
|
||||
extension = str(item.get("extension") or "").casefold()
|
||||
object_id = extension_object_id_from_saved_file(file_name)
|
||||
if not extension or not object_id:
|
||||
return None, None
|
||||
cas_key = extension_cas_map.get((extension, object_id))
|
||||
if not cas_key:
|
||||
return None, None
|
||||
path = config_cas_all_dir / cas_key
|
||||
return (path if path.exists() else None), cas_key
|
||||
|
||||
|
||||
def analyze(comparison: dict[str, Any], roots: dict[str, Path], *, extension_manifest_summary: Path | None = None, config_cas_all_dir: Path | None = None) -> dict[str, Any]:
|
||||
extension_cas_map = build_extension_cas_map(extension_manifest_summary)
|
||||
objects = []
|
||||
for item in comparison.get("object_changes") or []:
|
||||
details = []
|
||||
for storage in item.get("storage") or []:
|
||||
saved_root, active_root = storage_root(str(storage.get("saved_table")), str(storage.get("active_table")), roots)
|
||||
file_name = str(storage.get("file_name") or "")
|
||||
if not saved_root:
|
||||
details.append({"file_name": file_name, "error": f"Missing saved root for {storage.get('saved_table')}"})
|
||||
continue
|
||||
try:
|
||||
saved_path = safe_join(saved_root, file_name)
|
||||
active_path = safe_join(active_root, file_name) if active_root else None
|
||||
except ValueError:
|
||||
details.append({"file_name": file_name, "error": "Unsafe storage file name."})
|
||||
continue
|
||||
if not saved_path.exists():
|
||||
details.append({"file_name": file_name, "error": f"Saved payload file is missing: {saved_path}"})
|
||||
continue
|
||||
active_cas_key = None
|
||||
if not (active_path and active_path.exists()) and storage.get("saved_table") == "ConfigCASSave":
|
||||
active_path, active_cas_key = active_extension_path(item, file_name, extension_cas_map, config_cas_all_dir)
|
||||
payload_detail = analyze_payload(active_path if active_path and active_path.exists() else None, saved_path)
|
||||
details.append({
|
||||
"file_name": file_name,
|
||||
"payload_role": classify_payload_part(item, file_name),
|
||||
"saved_table": storage.get("saved_table"),
|
||||
"active_table": storage.get("active_table"),
|
||||
"active_exists": storage.get("active_exists"),
|
||||
"active_cas_key": active_cas_key,
|
||||
"payload": payload_detail,
|
||||
})
|
||||
objects.append({
|
||||
"full_name": item.get("full_name"),
|
||||
"layer": item.get("layer"),
|
||||
"extension": item.get("extension"),
|
||||
"kind": item.get("kind"),
|
||||
"kind_ru": item.get("kind_ru"),
|
||||
"name": item.get("name"),
|
||||
"synonym": item.get("synonym"),
|
||||
"change_state": item.get("change_state"),
|
||||
"details": details,
|
||||
})
|
||||
return {
|
||||
"schema": "onec_saved_state_object_detail.v1",
|
||||
"source_schema": comparison.get("schema"),
|
||||
"database": comparison.get("database"),
|
||||
"view": comparison.get("view"),
|
||||
"object_details": objects,
|
||||
"counts": {
|
||||
"objects": len(objects),
|
||||
"details": sum(len(item.get("details") or []) for item in objects),
|
||||
},
|
||||
"safety": {
|
||||
"read_only": True,
|
||||
"sql_write_performed": False,
|
||||
"public_terms_are_1c_objects": True,
|
||||
},
|
||||
"active_extension_resolution": {
|
||||
"extension_manifest_summary": str(extension_manifest_summary) if extension_manifest_summary else None,
|
||||
"config_cas_all_dir": str(config_cas_all_dir) if config_cas_all_dir else None,
|
||||
"mapped_parts": len(extension_cas_map),
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Analyze saved-state object changes beyond storage bytes.")
|
||||
parser.add_argument("--comparison", type=Path, required=True)
|
||||
parser.add_argument("--config-save-dir", type=Path, required=True)
|
||||
parser.add_argument("--config-dir", type=Path, required=True)
|
||||
parser.add_argument("--config-cas-save-dir", type=Path, required=True)
|
||||
parser.add_argument("--config-cas-dir", type=Path, required=True)
|
||||
parser.add_argument("--extension-manifest-summary", type=Path)
|
||||
parser.add_argument("--config-cas-all-dir", type=Path)
|
||||
parser.add_argument("--output", type=Path)
|
||||
args = parser.parse_args()
|
||||
|
||||
roots = {
|
||||
"ConfigSave": args.config_save_dir,
|
||||
"Config": args.config_dir,
|
||||
"ConfigCASSave": args.config_cas_save_dir,
|
||||
"ConfigCAS": args.config_cas_dir,
|
||||
}
|
||||
result = analyze(
|
||||
load_json(args.comparison),
|
||||
roots,
|
||||
extension_manifest_summary=args.extension_manifest_summary,
|
||||
config_cas_all_dir=args.config_cas_all_dir,
|
||||
)
|
||||
if args.output:
|
||||
write_json(args.output, result)
|
||||
print(json.dumps({"output": str(args.output) if args.output else None, "schema": result["schema"], "counts": result["counts"]}, ensure_ascii=False))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,189 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
def read_json(path: Path) -> dict[str, Any]:
|
||||
return json.loads(path.read_text(encoding="utf-8"))
|
||||
|
||||
|
||||
def dig(mapping: dict[str, Any] | None, *keys: str) -> Any:
|
||||
current: Any = mapping or {}
|
||||
for key in keys:
|
||||
if not isinstance(current, dict):
|
||||
return None
|
||||
current = current.get(key)
|
||||
return current
|
||||
|
||||
|
||||
def short_next_record(match: dict[str, Any] | None) -> dict[str, Any] | None:
|
||||
node = dig(match, "next_moxel_record")
|
||||
if not isinstance(node, dict):
|
||||
return None
|
||||
result: dict[str, Any] = {"type": node.get("type")}
|
||||
for key in ("head", "value", "scalar_prefix", "tree_position"):
|
||||
if key in node:
|
||||
result[key] = node.get(key)
|
||||
return result
|
||||
|
||||
|
||||
def compact_item(item: dict[str, Any]) -> dict[str, Any]:
|
||||
named = ((item.get("named_range_matches") or [{}])[0]) if item.get("named_range_matches") else {}
|
||||
text = ((item.get("text_matches") or [{}])[0]) if item.get("text_matches") else {}
|
||||
return {
|
||||
"file_name": item.get("file_name"),
|
||||
"bytes": item.get("bytes"),
|
||||
"modified": item.get("modified"),
|
||||
"counts": item.get("counts") or {},
|
||||
"named_range": {
|
||||
"name": named.get("name"),
|
||||
"tree_position": named.get("tree_position"),
|
||||
"one_based": dig(named, "range", "one_based"),
|
||||
"raw_scalars": dig(named, "range_candidate", "raw_scalars"),
|
||||
},
|
||||
"text_match": {
|
||||
"text": text.get("text"),
|
||||
"cell_id": text.get("cell_id"),
|
||||
"tree_position": text.get("tree_position"),
|
||||
"next_moxel_record": short_next_record(text),
|
||||
"immediate_preceding_values": dig(text, "style_evidence", "immediate_preceding_values"),
|
||||
"last_7_preceding_values": dig(text, "style_evidence", "last_7_preceding_values"),
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def diff_dict(before: dict[str, Any], after: dict[str, Any]) -> dict[str, dict[str, Any]]:
|
||||
changed: dict[str, dict[str, Any]] = {}
|
||||
for key in sorted(set(before) | set(after)):
|
||||
if before.get(key) != after.get(key):
|
||||
changed[key] = {"before": before.get(key), "after": after.get(key)}
|
||||
return changed
|
||||
|
||||
|
||||
def build_transitions(items: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
||||
compact = [compact_item(item) for item in reversed(items)]
|
||||
transitions: list[dict[str, Any]] = []
|
||||
for before, after in zip(compact, compact[1:]):
|
||||
named_before = before.get("named_range") if isinstance(before.get("named_range"), dict) else {}
|
||||
named_after = after.get("named_range") if isinstance(after.get("named_range"), dict) else {}
|
||||
text_before = before.get("text_match") if isinstance(before.get("text_match"), dict) else {}
|
||||
text_after = after.get("text_match") if isinstance(after.get("text_match"), dict) else {}
|
||||
transition = {
|
||||
"from_file": before.get("file_name"),
|
||||
"to_file": after.get("file_name"),
|
||||
"from_modified": before.get("modified"),
|
||||
"to_modified": after.get("modified"),
|
||||
"from_bytes": before.get("bytes"),
|
||||
"to_bytes": after.get("bytes"),
|
||||
"count_changes": diff_dict(before.get("counts") or {}, after.get("counts") or {}),
|
||||
"named_range_changes": diff_dict(named_before, named_after),
|
||||
"text_match_changes": diff_dict(text_before, text_after),
|
||||
}
|
||||
transitions.append(transition)
|
||||
return transitions
|
||||
|
||||
|
||||
def build_signature_groups(items: list[dict[str, Any]]) -> dict[str, Any]:
|
||||
signatures: dict[str, list[dict[str, Any]]] = {}
|
||||
for item in items:
|
||||
for match in item.get("text_matches") or []:
|
||||
if not isinstance(match, dict):
|
||||
continue
|
||||
signature = json.dumps(short_next_record(match), ensure_ascii=False, sort_keys=True)
|
||||
signatures.setdefault(signature, []).append(
|
||||
{
|
||||
"file_name": item.get("file_name"),
|
||||
"modified": item.get("modified"),
|
||||
"text": match.get("text"),
|
||||
"cell_id": match.get("cell_id"),
|
||||
"tree_position": match.get("tree_position"),
|
||||
"immediate_preceding_values": dig(match, "style_evidence", "immediate_preceding_values"),
|
||||
"last_7_preceding_values": dig(match, "style_evidence", "last_7_preceding_values"),
|
||||
}
|
||||
)
|
||||
result: list[dict[str, Any]] = []
|
||||
for signature, occurrences in signatures.items():
|
||||
result.append(
|
||||
{
|
||||
"signature": json.loads(signature),
|
||||
"occurrences": occurrences,
|
||||
"count": len(occurrences),
|
||||
}
|
||||
)
|
||||
result.sort(key=lambda item: (-int(item.get("count") or 0), json.dumps(item.get("signature"), ensure_ascii=False)))
|
||||
return {"next_moxel_record_signatures": result}
|
||||
|
||||
|
||||
def build_markdown(history: dict[str, Any], transitions: list[dict[str, Any]], signature_groups: dict[str, Any]) -> str:
|
||||
lines: list[str] = []
|
||||
lines.append("# 1C template history matrix")
|
||||
lines.append("")
|
||||
lines.append(f"- Base: `{history.get('base_id')}`")
|
||||
lines.append(f"- Track name: `{history.get('track_name')}`")
|
||||
lines.append(f"- Track text: `{history.get('track_text')}`")
|
||||
lines.append(f"- Snapshots: `{len(history.get('items') or [])}`")
|
||||
lines.append("")
|
||||
lines.append("## Current state")
|
||||
current = compact_item((history.get("items") or [{}])[0] if history.get("items") else {})
|
||||
lines.append("")
|
||||
lines.append("```json")
|
||||
lines.append(json.dumps(current, ensure_ascii=False, indent=2))
|
||||
lines.append("```")
|
||||
lines.append("")
|
||||
lines.append("## Transitions")
|
||||
for transition in transitions:
|
||||
lines.append("")
|
||||
lines.append(
|
||||
f"- `{transition['from_file']}` -> `{transition['to_file']}` "
|
||||
f"({transition['from_modified']} -> {transition['to_modified']})"
|
||||
)
|
||||
lines.append("")
|
||||
lines.append("```json")
|
||||
lines.append(json.dumps(transition, ensure_ascii=False, indent=2))
|
||||
lines.append("```")
|
||||
lines.append("")
|
||||
lines.append("## Signatures")
|
||||
lines.append("")
|
||||
lines.append("```json")
|
||||
lines.append(json.dumps(signature_groups, ensure_ascii=False, indent=2))
|
||||
lines.append("```")
|
||||
lines.append("")
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Build a transition matrix from tracked 1C template history.")
|
||||
parser.add_argument("history_json", help="Path to JSON generated by track_1c_template_history.py")
|
||||
parser.add_argument("--json-output", help="Optional JSON output path.")
|
||||
parser.add_argument("--markdown-output", help="Optional Markdown output path.")
|
||||
args = parser.parse_args()
|
||||
|
||||
history_path = Path(args.history_json)
|
||||
history = read_json(history_path)
|
||||
items = history.get("items") or []
|
||||
transitions = build_transitions(items)
|
||||
signature_groups = build_signature_groups(items)
|
||||
payload = {
|
||||
"schema": "codex_1c_template_history_matrix.v1",
|
||||
"source": str(history_path),
|
||||
"track_name": history.get("track_name"),
|
||||
"track_text": history.get("track_text"),
|
||||
"current": compact_item(items[0] if items else {}),
|
||||
"transitions": transitions,
|
||||
"signatures": signature_groups,
|
||||
}
|
||||
rendered = json.dumps(payload, ensure_ascii=False, indent=2)
|
||||
if args.json_output:
|
||||
Path(args.json_output).write_text(rendered, encoding="utf-8")
|
||||
markdown = build_markdown(history, transitions, signature_groups)
|
||||
if args.markdown_output:
|
||||
Path(args.markdown_output).write_text(markdown, encoding="utf-8")
|
||||
print(rendered)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,367 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import itertools
|
||||
import json
|
||||
import re
|
||||
import xml.etree.ElementTree as ET
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
DEFAULT_XML_ROOT = Path(r"Z:\codex\1C\XML\UPO\Структура базы 1с\Конфигурация")
|
||||
PLACEHOLDER_RE = re.compile(r"\[([A-Za-zА-Яа-яЁё_][A-Za-zА-Яа-яЁё0-9_.]*)\]")
|
||||
STYLE_TAGS = {
|
||||
"format",
|
||||
"formatIndex",
|
||||
"f",
|
||||
"width",
|
||||
"height",
|
||||
"horizontalAlignment",
|
||||
"verticalAlignment",
|
||||
"border",
|
||||
"font",
|
||||
"textColor",
|
||||
"backgroundColor",
|
||||
}
|
||||
|
||||
|
||||
def namespace_uri(value: str) -> str | None:
|
||||
if value.startswith("{") and "}" in value:
|
||||
return value[1:].split("}", 1)[0]
|
||||
return None
|
||||
|
||||
|
||||
def local_name(value: str) -> str:
|
||||
return value.rsplit("}", 1)[-1] if "}" in value else value
|
||||
|
||||
|
||||
def xml_kind(root: ET.Element) -> str:
|
||||
name = local_name(root.tag)
|
||||
namespace = namespace_uri(root.tag) or ""
|
||||
if name == "document" and "data/spreadsheet" in namespace:
|
||||
return "tabular_document"
|
||||
if name == "DataCompositionSchema":
|
||||
return "data_composition_schema"
|
||||
return name
|
||||
|
||||
|
||||
def text_value(node: ET.Element | None) -> str | None:
|
||||
if node is None or node.text is None:
|
||||
return None
|
||||
value = node.text.strip()
|
||||
return value or None
|
||||
|
||||
|
||||
def as_int(value: Any) -> int | None:
|
||||
try:
|
||||
return int(str(value).strip())
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
def children(node: ET.Element, name: str | None = None) -> list[ET.Element]:
|
||||
items = list(node)
|
||||
if name is None:
|
||||
return items
|
||||
return [item for item in items if local_name(item.tag) == name]
|
||||
|
||||
|
||||
def descendants(node: ET.Element, name: str | None = None) -> list[ET.Element]:
|
||||
result = []
|
||||
for item in node.iter():
|
||||
if item is node:
|
||||
continue
|
||||
if name is None or local_name(item.tag) == name:
|
||||
result.append(item)
|
||||
return result
|
||||
|
||||
|
||||
def first_child_text(node: ET.Element, name: str) -> str | None:
|
||||
for item in children(node, name):
|
||||
value = text_value(item)
|
||||
if value is not None:
|
||||
return value
|
||||
return None
|
||||
|
||||
|
||||
def direct_columns_size(root: ET.Element) -> int | None:
|
||||
for columns in children(root, "columns"):
|
||||
size = as_int(first_child_text(columns, "size"))
|
||||
if size is not None:
|
||||
return size
|
||||
return None
|
||||
|
||||
|
||||
def direct_height(root: ET.Element) -> int | None:
|
||||
for name in ("height", "vgRows"):
|
||||
value = as_int(first_child_text(root, name))
|
||||
if value is not None:
|
||||
return value
|
||||
return None
|
||||
|
||||
|
||||
def cell_texts(cell: ET.Element) -> list[str]:
|
||||
values = []
|
||||
for item in descendants(cell):
|
||||
if local_name(item.tag) in {"content", "parameter"}:
|
||||
value = text_value(item)
|
||||
if value:
|
||||
values.append(value)
|
||||
return values
|
||||
|
||||
|
||||
def row_cells(row: ET.Element) -> list[dict[str, Any]]:
|
||||
result = []
|
||||
current_column = 0
|
||||
for wrapper in children(row, "c"):
|
||||
explicit_index = as_int(first_child_text(wrapper, "i"))
|
||||
if explicit_index is not None:
|
||||
current_column = explicit_index
|
||||
payload = next((item for item in children(wrapper, "c")), wrapper)
|
||||
texts = cell_texts(payload)
|
||||
parameter = text_value(next((item for item in descendants(payload, "parameter")), None))
|
||||
format_index = as_int(first_child_text(payload, "f")) or as_int(first_child_text(payload, "formatIndex"))
|
||||
if texts or parameter or format_index is not None:
|
||||
result.append(
|
||||
{
|
||||
"column": current_column + 1,
|
||||
"zero_based": {"column": current_column},
|
||||
"formatIndex": format_index,
|
||||
"texts": texts,
|
||||
**({"parameter": parameter} if parameter else {}),
|
||||
}
|
||||
)
|
||||
current_column += 1
|
||||
return result
|
||||
|
||||
|
||||
def merge_ranges(root: ET.Element, *, limit: int = 200) -> list[dict[str, Any]]:
|
||||
result = []
|
||||
|
||||
def append_range(row: int | None, column: int | None, width: int | None, height: int | None) -> bool:
|
||||
if row is None or column is None:
|
||||
return False
|
||||
width = width or 1
|
||||
height = height or 1
|
||||
result.append(
|
||||
{
|
||||
"row": row + 1,
|
||||
"column": column + 1,
|
||||
"width": width,
|
||||
"height": height,
|
||||
"range": {
|
||||
"one_based": {
|
||||
"top": row + 1,
|
||||
"left": column + 1,
|
||||
"bottom": row + height,
|
||||
"right": column + width,
|
||||
},
|
||||
"zero_based": {
|
||||
"top": row,
|
||||
"left": column,
|
||||
"bottom": row + height - 1,
|
||||
"right": column + width - 1,
|
||||
},
|
||||
},
|
||||
"source": "xml_template_merge",
|
||||
}
|
||||
)
|
||||
return len(result) >= limit
|
||||
|
||||
for merge in descendants(root, "merge"):
|
||||
scalar_values = [(local_name(item.tag), as_int(text_value(item))) for item in children(merge)]
|
||||
index = 0
|
||||
while index < len(scalar_values):
|
||||
if scalar_values[index][0] != "r":
|
||||
index += 1
|
||||
continue
|
||||
row = scalar_values[index][1]
|
||||
column = None
|
||||
width = None
|
||||
height = None
|
||||
cursor = index + 1
|
||||
while cursor < len(scalar_values) and scalar_values[cursor][0] != "r":
|
||||
name, value = scalar_values[cursor]
|
||||
if name == "c":
|
||||
column = value
|
||||
elif name == "w":
|
||||
width = value
|
||||
elif name == "h":
|
||||
height = value
|
||||
cursor += 1
|
||||
if append_range(row, column, width, height):
|
||||
return result
|
||||
index = cursor
|
||||
for item in descendants(merge, "r"):
|
||||
row = as_int(first_child_text(item, "r"))
|
||||
column = as_int(first_child_text(item, "c"))
|
||||
width = as_int(first_child_text(item, "w")) or 1
|
||||
height = as_int(first_child_text(item, "h")) or 1
|
||||
if append_range(row, column, width, height):
|
||||
return result
|
||||
return result
|
||||
|
||||
|
||||
def profile_template(path: Path, root_dir: Path) -> dict[str, Any]:
|
||||
xml_root = ET.parse(path).getroot()
|
||||
rows = []
|
||||
max_row = 0
|
||||
max_column = 0
|
||||
parameter_names: list[str] = []
|
||||
text_values: list[str] = []
|
||||
placeholder_names: list[str] = []
|
||||
for rows_item in descendants(xml_root, "rowsItem"):
|
||||
row_index = as_int(first_child_text(rows_item, "index"))
|
||||
row = next((item for item in children(rows_item, "row")), None)
|
||||
if row_index is None or row is None:
|
||||
continue
|
||||
cells = row_cells(row)
|
||||
if cells:
|
||||
max_row = max(max_row, row_index + 1)
|
||||
for cell in cells:
|
||||
max_column = max(max_column, int(cell.get("column") or 0))
|
||||
for value in cell.get("texts") or []:
|
||||
text_values.append(value)
|
||||
for match in PLACEHOLDER_RE.finditer(value):
|
||||
placeholder_names.append(match.group(1))
|
||||
if cell.get("parameter"):
|
||||
parameter_names.append(str(cell["parameter"]))
|
||||
rows.append(
|
||||
{
|
||||
"index": row_index,
|
||||
"row": row_index + 1,
|
||||
"formatIndex": as_int(first_child_text(row, "formatIndex")),
|
||||
"cells": cells[:50],
|
||||
"cell_count": len(cells),
|
||||
}
|
||||
)
|
||||
merges = merge_ranges(xml_root)
|
||||
for item in merges:
|
||||
one_based = (item.get("range") or {}).get("one_based") or {}
|
||||
max_row = max(max_row, int(one_based.get("bottom") or 0))
|
||||
max_column = max(max_column, int(one_based.get("right") or 0))
|
||||
style_counts = {
|
||||
name: sum(1 for item in descendants(xml_root, name) if text_value(item) is not None)
|
||||
for name in sorted(STYLE_TAGS)
|
||||
}
|
||||
format_indexes = [
|
||||
as_int(text_value(item))
|
||||
for item in descendants(xml_root)
|
||||
if local_name(item.tag) in {"formatIndex", "f"} and as_int(text_value(item)) is not None
|
||||
]
|
||||
capacity_rows = direct_height(xml_root)
|
||||
capacity_columns = direct_columns_size(xml_root)
|
||||
return {
|
||||
"path": str(path),
|
||||
"relative_path": str(path.relative_to(root_dir)) if path.is_relative_to(root_dir) else str(path),
|
||||
"xml_kind": xml_kind(xml_root),
|
||||
"xml_root": {"name": local_name(xml_root.tag), "namespace": namespace_uri(xml_root.tag)},
|
||||
"capacity_dimensions": {"rows": capacity_rows, "columns": capacity_columns},
|
||||
"used_dimensions": {"rows": max_row or None, "columns": max_column or None, "evidence": ["rowsItem", "cells"] + (["merge"] if merges else [])},
|
||||
"counts": {
|
||||
"rows": len(rows),
|
||||
"cells": sum(int(row.get("cell_count") or 0) for row in rows),
|
||||
"texts": len(text_values),
|
||||
"parameters": len(set(parameter_names)),
|
||||
"placeholders": len(set(placeholder_names)),
|
||||
"merges": len(merges),
|
||||
"format_indexes": len(format_indexes),
|
||||
"distinct_format_indexes": len(set(format_indexes)),
|
||||
},
|
||||
"style_counts": style_counts,
|
||||
"samples": {
|
||||
"rows": rows[:20],
|
||||
"texts": text_values[:50],
|
||||
"parameters": sorted(set(parameter_names))[:50],
|
||||
"placeholders": sorted(set(placeholder_names))[:50],
|
||||
"merges": merges[:50],
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def analyze(root: Path, *, limit: int | None = None) -> dict[str, Any]:
|
||||
file_iter = root.rglob("Template.xml")
|
||||
files = list(itertools.islice(file_iter, limit)) if limit is not None else list(file_iter)
|
||||
templates = []
|
||||
errors = []
|
||||
for path in files:
|
||||
try:
|
||||
templates.append(profile_template(path, root))
|
||||
except Exception as exc:
|
||||
errors.append({"path": str(path), "error": str(exc)})
|
||||
return {
|
||||
"schema": "codex_1c_template_xml_profiles.v1",
|
||||
"source": "xml_analysis_fixture_only",
|
||||
"root": str(root),
|
||||
"templates": templates,
|
||||
"counts": {
|
||||
"files": len(files),
|
||||
"templates": len(templates),
|
||||
"errors": len(errors),
|
||||
"with_merges": sum(1 for item in templates if int((item.get("counts") or {}).get("merges") or 0) > 0),
|
||||
"with_parameters": sum(1 for item in templates if int((item.get("counts") or {}).get("parameters") or 0) > 0),
|
||||
"with_placeholders": sum(1 for item in templates if int((item.get("counts") or {}).get("placeholders") or 0) > 0),
|
||||
"by_xml_kind": {
|
||||
kind: sum(1 for item in templates if item.get("xml_kind") == kind)
|
||||
for kind in sorted({str(item.get("xml_kind") or "unknown") for item in templates})
|
||||
},
|
||||
},
|
||||
**({"errors": errors[:100]} if errors else {}),
|
||||
}
|
||||
|
||||
|
||||
def render_markdown(payload: dict[str, Any]) -> str:
|
||||
def dimension_text(value: dict[str, Any]) -> str:
|
||||
rows = value.get("rows") if value.get("rows") is not None else "-"
|
||||
columns = value.get("columns") if value.get("columns") is not None else "-"
|
||||
return f"{rows}x{columns}"
|
||||
|
||||
lines = ["# 1C Template XML Profiles", ""]
|
||||
counts = payload.get("counts") or {}
|
||||
lines.append(f"- Source: `{payload.get('source')}`")
|
||||
lines.append(f"- Templates: `{counts.get('templates')}`")
|
||||
lines.append(f"- With merges: `{counts.get('with_merges')}`")
|
||||
lines.append(f"- With parameters: `{counts.get('with_parameters')}`")
|
||||
lines.append(f"- Errors: `{counts.get('errors')}`")
|
||||
lines.append("")
|
||||
lines.append("| Template | XML kind | Capacity | Used | Cells | Texts | Params | Merges | Formats |")
|
||||
lines.append("| --- | --- | --- | --- | ---: | ---: | ---: | ---: | ---: |")
|
||||
for item in payload.get("templates") or []:
|
||||
item_counts = item.get("counts") or {}
|
||||
capacity = item.get("capacity_dimensions") or {}
|
||||
used = item.get("used_dimensions") or {}
|
||||
lines.append(
|
||||
f"| `{item.get('relative_path')}` | "
|
||||
f"`{item.get('xml_kind')}` | "
|
||||
f"`{dimension_text(capacity)}` | "
|
||||
f"`{dimension_text(used)}` | "
|
||||
f"{item_counts.get('cells')} | {item_counts.get('texts')} | {item_counts.get('parameters')} | "
|
||||
f"{item_counts.get('merges')} | {item_counts.get('distinct_format_indexes')} |"
|
||||
)
|
||||
lines.append("")
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Profile 1C Template.xml spreadsheet exports as analysis fixtures for SQL MOXCEL decoding.")
|
||||
parser.add_argument("--root", default=str(DEFAULT_XML_ROOT), help="XML export root. Use Конфигурация by default, not extensions.")
|
||||
parser.add_argument("--limit", type=int, help="Optional max Template.xml files to scan.")
|
||||
parser.add_argument("--output-json", default="reports/1c-template-baselines/xml-template-profiles.json")
|
||||
parser.add_argument("--output-markdown", default="reports/1c-template-baselines/xml-template-profiles.md")
|
||||
args = parser.parse_args()
|
||||
|
||||
root = Path(args.root).resolve()
|
||||
payload = analyze(root, limit=args.limit)
|
||||
json_path = Path(args.output_json)
|
||||
md_path = Path(args.output_markdown)
|
||||
json_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
md_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
json_path.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
md_path.write_text(render_markdown(payload), encoding="utf-8")
|
||||
print(json.dumps({"status": "ok", "json": str(json_path), "markdown": str(md_path), "counts": payload["counts"]}, ensure_ascii=False, indent=2))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,162 @@
|
||||
#!/usr/bin/env python3
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
from collections import Counter, defaultdict
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
def load_entries(report: dict[str, Any]) -> list[dict[str, Any]]:
|
||||
matrix = report.get("matrix") if isinstance(report.get("matrix"), dict) else {}
|
||||
entries = matrix.get("entries") if isinstance(matrix.get("entries"), list) else []
|
||||
return [entry for entry in entries if isinstance(entry, dict)]
|
||||
|
||||
|
||||
def gap_row(entry: dict[str, Any]) -> dict[str, Any]:
|
||||
requested = entry.get("requested_target") if isinstance(entry.get("requested_target"), dict) else {}
|
||||
effective = entry.get("effective_target") if isinstance(entry.get("effective_target"), dict) else {}
|
||||
source = entry.get("effective_source") if isinstance(entry.get("effective_source"), dict) else {}
|
||||
prop = entry.get("property") if isinstance(entry.get("property"), dict) else {}
|
||||
probe = entry.get("codec_probe") if isinstance(entry.get("codec_probe"), dict) else {}
|
||||
probe_node = probe.get("node") if isinstance(probe.get("node"), dict) else {}
|
||||
return {
|
||||
"reason": entry.get("reason") or "unknown",
|
||||
"requested_section": requested.get("section"),
|
||||
"requested_name": requested.get("name"),
|
||||
"requested_path": requested.get("path"),
|
||||
"effective_section": effective.get("section"),
|
||||
"effective_name": effective.get("name"),
|
||||
"effective_path": effective.get("path"),
|
||||
"source_kind": source.get("kind") or "local",
|
||||
"property": prop.get("semantic_name") or prop.get("canonical_property") or prop.get("property"),
|
||||
"canonical_property": prop.get("canonical_property") or prop.get("property"),
|
||||
"presentation": prop.get("presentation"),
|
||||
"semantic_name": prop.get("semantic_name"),
|
||||
"semantic_group": prop.get("semantic_group"),
|
||||
"semantic_source": prop.get("semantic_source"),
|
||||
"parameter_index": prop.get("parameter_index"),
|
||||
"value_type": prop.get("value_type"),
|
||||
"old": prop.get("old"),
|
||||
"read_path": prop.get("read_path"),
|
||||
"write_path": prop.get("write_path"),
|
||||
"verification": prop.get("verification"),
|
||||
"codec_probe": probe or None,
|
||||
"codec_probe_node_type": probe_node.get("type") or probe.get("error") if probe else None,
|
||||
}
|
||||
|
||||
|
||||
def classify_action(reason: str, prop: str | None, value_type: str | None) -> str:
|
||||
if reason == "identity_or_binding_property":
|
||||
return "manual_only_identity_or_binding"
|
||||
if reason == "empty_local_string_requires_codec_probe":
|
||||
return "add_empty_composite_string_codec_probe"
|
||||
if reason == "composite_node_requires_semantic_rule":
|
||||
return "learn_composite_node_semantics"
|
||||
if reason == "value_type_not_smoke_safe":
|
||||
if value_type in {"enum_atom", "bool_or_enum_atom", "color_or_enum_atom"}:
|
||||
return "learn_allowed_enum_values"
|
||||
if prop in {"group"}:
|
||||
return "learn_reference_or_container_write_rule"
|
||||
return "classify_scalar_semantics"
|
||||
return "inspect"
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Analyze not-smoked entries from a 1C saved-state write matrix report.")
|
||||
parser.add_argument("--matrix-report", type=Path, required=True, help="Report produced by scripts/smoke_1c_write_matrix.py.")
|
||||
parser.add_argument("--output", type=Path, required=True, help="Output JSON gap report.")
|
||||
parser.add_argument("--sample-limit", type=int, default=12, help="Samples per reason/action.")
|
||||
args = parser.parse_args()
|
||||
|
||||
report = json.loads(args.matrix_report.read_text(encoding="utf-8"))
|
||||
gaps = []
|
||||
for entry in load_entries(report):
|
||||
if entry.get("can_smoke"):
|
||||
continue
|
||||
row = gap_row(entry)
|
||||
row["next_action"] = classify_action(str(row.get("reason") or ""), row.get("property"), row.get("value_type"))
|
||||
gaps.append(row)
|
||||
|
||||
by_reason = Counter(row["reason"] for row in gaps)
|
||||
by_action = Counter(row["next_action"] for row in gaps)
|
||||
by_section = Counter(row["effective_section"] for row in gaps)
|
||||
by_property = Counter(row["property"] for row in gaps)
|
||||
by_probe_node_type = Counter(row.get("codec_probe_node_type") for row in gaps if row.get("codec_probe_node_type"))
|
||||
samples_by_reason: dict[str, list[dict[str, Any]]] = defaultdict(list)
|
||||
samples_by_action: dict[str, list[dict[str, Any]]] = defaultdict(list)
|
||||
shape_summary: dict[str, dict[str, Any]] = defaultdict(lambda: {"count": 0, "properties": Counter(), "sections": Counter(), "samples": []})
|
||||
for row in gaps:
|
||||
reason = str(row["reason"])
|
||||
action = str(row["next_action"])
|
||||
probe = row.get("codec_probe") if isinstance(row.get("codec_probe"), dict) else {}
|
||||
node = probe.get("node") if isinstance(probe.get("node"), dict) else {}
|
||||
children = node.get("children") if isinstance(node.get("children"), list) else []
|
||||
if node:
|
||||
shape = str(node.get("type") or "unknown") + "|" + ",".join(str((child or {}).get("type")) for child in children[:12])
|
||||
shape_row = shape_summary[shape]
|
||||
shape_row["count"] = int(shape_row["count"]) + 1
|
||||
shape_row["properties"][row.get("property")] += 1
|
||||
shape_row["sections"][row.get("effective_section")] += 1
|
||||
if len(shape_row["samples"]) < args.sample_limit:
|
||||
shape_row["samples"].append(
|
||||
{
|
||||
"target": row.get("requested_name") or row.get("requested_path"),
|
||||
"section": row.get("effective_section"),
|
||||
"property": row.get("presentation") or row.get("property"),
|
||||
"semantic_name": row.get("semantic_name"),
|
||||
"semantic_group": row.get("semantic_group"),
|
||||
"old": row.get("old"),
|
||||
"write_path": row.get("write_path"),
|
||||
}
|
||||
)
|
||||
sample = {
|
||||
"target": row.get("requested_name") or row.get("requested_path"),
|
||||
"section": row.get("effective_section"),
|
||||
"property": row.get("presentation") or row.get("property"),
|
||||
"semantic_name": row.get("semantic_name"),
|
||||
"semantic_group": row.get("semantic_group"),
|
||||
"value_type": row.get("value_type"),
|
||||
"old": row.get("old"),
|
||||
"write_path": row.get("write_path"),
|
||||
}
|
||||
if len(samples_by_reason[reason]) < args.sample_limit:
|
||||
samples_by_reason[reason].append(sample)
|
||||
if len(samples_by_action[action]) < args.sample_limit:
|
||||
samples_by_action[action].append(sample)
|
||||
|
||||
result = {
|
||||
"schema": "onec_form_write_matrix_gap_analysis.v1",
|
||||
"status": "ok",
|
||||
"source_report": str(args.matrix_report),
|
||||
"counts": {
|
||||
"gaps": len(gaps),
|
||||
"by_reason": dict(sorted(by_reason.items())),
|
||||
"by_next_action": dict(sorted(by_action.items())),
|
||||
"by_effective_section": dict(sorted(by_section.items())),
|
||||
"by_codec_probe_node_type": dict(sorted(by_probe_node_type.items())),
|
||||
"top_properties": by_property.most_common(40),
|
||||
},
|
||||
"samples_by_reason": dict(samples_by_reason),
|
||||
"samples_by_next_action": dict(samples_by_action),
|
||||
"codec_probe_shapes": [
|
||||
{
|
||||
"shape": shape,
|
||||
"count": row["count"],
|
||||
"properties": row["properties"].most_common(20),
|
||||
"sections": dict(row["sections"]),
|
||||
"samples": row["samples"],
|
||||
}
|
||||
for shape, row in sorted(shape_summary.items(), key=lambda item: int(item[1]["count"]), reverse=True)
|
||||
],
|
||||
"gaps": gaps,
|
||||
}
|
||||
args.output.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.output.write_text(json.dumps(result, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
print(json.dumps({"schema": result["schema"], "status": "ok", "counts": result["counts"], "path": str(args.output)}, ensure_ascii=False, indent=2))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,77 @@
|
||||
#!/usr/bin/env python3
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
def load_entries(path: Path) -> list[dict[str, Any]]:
|
||||
report = json.loads(path.read_text(encoding="utf-8"))
|
||||
matrix = report.get("matrix") if isinstance(report.get("matrix"), dict) else report
|
||||
entries = matrix.get("entries") if isinstance(matrix.get("entries"), list) else []
|
||||
return [entry for entry in entries if isinstance(entry, dict)]
|
||||
|
||||
|
||||
def target_map(entries: list[dict[str, Any]]) -> dict[tuple[str, str, str], dict[str, Any]]:
|
||||
result: dict[tuple[str, str, str], dict[str, Any]] = {}
|
||||
for entry in entries:
|
||||
target = entry.get("requested_target") if isinstance(entry.get("requested_target"), dict) else {}
|
||||
key = (str(target.get("section") or ""), str(target.get("name") or ""), str(target.get("id") or ""))
|
||||
if key == ("", "", "") or key in result:
|
||||
continue
|
||||
result[key] = {field: target.get(field) for field in ("section", "name", "id", "path", "marker", "type_name", "title")}
|
||||
return result
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Compare 1C write matrix reports for structural target moves.")
|
||||
parser.add_argument("--before", type=Path, required=True, help="Before write matrix report.")
|
||||
parser.add_argument("--after", type=Path, required=True, help="After write matrix report.")
|
||||
parser.add_argument("--output", type=Path, required=True, help="Output structural diff JSON path.")
|
||||
args = parser.parse_args()
|
||||
|
||||
before_targets = target_map(load_entries(args.before))
|
||||
after_targets = target_map(load_entries(args.after))
|
||||
moves = []
|
||||
for key, after in after_targets.items():
|
||||
before = before_targets.get(key)
|
||||
if not before:
|
||||
continue
|
||||
if str(before.get("path") or "") == str(after.get("path") or ""):
|
||||
continue
|
||||
moves.append(
|
||||
{
|
||||
"target": {
|
||||
"section": after.get("section"),
|
||||
"name": after.get("name"),
|
||||
"id": after.get("id"),
|
||||
"marker": after.get("marker"),
|
||||
"type_name": after.get("type_name"),
|
||||
},
|
||||
"old_path": before.get("path"),
|
||||
"new_path": after.get("path"),
|
||||
"presentation": f"{after.get('name') or after.get('path')}: {before.get('path')} -> {after.get('path')}",
|
||||
}
|
||||
)
|
||||
result = {
|
||||
"schema": "onec_form_write_matrix_structural_diff.v1",
|
||||
"status": "changed" if moves else "no_changes",
|
||||
"before": str(args.before),
|
||||
"after": str(args.after),
|
||||
"target_moves": moves,
|
||||
"counts": {
|
||||
"target_moves": len(moves),
|
||||
"before_targets": len(before_targets),
|
||||
"after_targets": len(after_targets),
|
||||
},
|
||||
}
|
||||
args.output.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.output.write_text(json.dumps(result, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
print(json.dumps({"schema": result["schema"], "status": result["status"], "counts": result["counts"], "path": str(args.output)}, ensure_ascii=False, indent=2))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,257 @@
|
||||
#!/usr/bin/env python3
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import sys
|
||||
import xml.etree.ElementTree as ET
|
||||
from collections import Counter, defaultdict
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any, Iterable
|
||||
|
||||
|
||||
DEFAULT_ROOT = Path(r"Z:\codex\1C\XML\UPO\Структура базы 1с")
|
||||
|
||||
|
||||
def local_name(tag: str) -> str:
|
||||
return tag.rsplit("}", 1)[-1]
|
||||
|
||||
|
||||
def direct_child(element: ET.Element, name: str) -> ET.Element | None:
|
||||
return next((child for child in element if local_name(child.tag) == name), None)
|
||||
|
||||
|
||||
def property_text(properties: ET.Element | None, name: str) -> str:
|
||||
if properties is None:
|
||||
return ""
|
||||
node = next((child for child in properties if local_name(child.tag) == name), None)
|
||||
return str(node.text or "").strip() if node is not None else ""
|
||||
|
||||
|
||||
def parse_metadata_file(path: Path, layer: str) -> dict[str, Any]:
|
||||
try:
|
||||
root = ET.parse(path).getroot()
|
||||
metadata = next(iter(root), None) if local_name(root.tag) == "MetaDataObject" else root
|
||||
if metadata is None:
|
||||
raise ValueError("metadata object element is missing")
|
||||
kind = local_name(metadata.tag)
|
||||
properties = direct_child(metadata, "Properties")
|
||||
children = direct_child(metadata, "ChildObjects")
|
||||
child_schemas: dict[str, dict[str, Any]] = {}
|
||||
if children is not None:
|
||||
grouped: dict[str, list[ET.Element]] = defaultdict(list)
|
||||
for child in children:
|
||||
grouped[local_name(child.tag)].append(child)
|
||||
for child_kind, values in grouped.items():
|
||||
child_properties: set[str] = set()
|
||||
for value in values:
|
||||
value_properties = direct_child(value, "Properties")
|
||||
if value_properties is not None:
|
||||
child_properties.update(local_name(item.tag) for item in value_properties)
|
||||
child_schemas[child_kind] = {"count": len(values), "properties": sorted(child_properties)}
|
||||
return {
|
||||
"status": "ok",
|
||||
"layer": layer,
|
||||
"path": str(path),
|
||||
"kind": kind,
|
||||
"name": property_text(properties, "Name"),
|
||||
"uuid": str(metadata.attrib.get("uuid") or "").lower(),
|
||||
"properties": sorted(local_name(child.tag) for child in properties) if properties is not None else [],
|
||||
"child_schemas": child_schemas,
|
||||
}
|
||||
except Exception as exc:
|
||||
return {"status": "error", "layer": layer, "path": str(path), "message": str(exc)}
|
||||
|
||||
|
||||
def layer_files(root: Path) -> list[Path]:
|
||||
files = [root / "Configuration.xml"] if (root / "Configuration.xml").is_file() else []
|
||||
for folder in root.iterdir():
|
||||
if folder.is_dir() and folder.name != "Ext":
|
||||
files.extend(sorted(folder.glob("*.xml")))
|
||||
return files
|
||||
|
||||
|
||||
def artifact_files(root: Path, max_depth: int = 5) -> list[Path]:
|
||||
result: set[Path] = set()
|
||||
for folder in (path for path in root.iterdir() if path.is_dir()):
|
||||
for depth in range(1, max_depth + 1):
|
||||
pattern = "/".join(["*"] * depth + ["Ext", "*.xml"])
|
||||
result.update(path for path in folder.glob(pattern) if path.is_file())
|
||||
return sorted(result)
|
||||
|
||||
|
||||
def parse_artifact(path: Path) -> dict[str, Any]:
|
||||
tags: set[str] = set()
|
||||
attributes: dict[str, set[str]] = defaultdict(set)
|
||||
root_tag = ""
|
||||
try:
|
||||
for _event, element in ET.iterparse(path, events=("start",)):
|
||||
tag = local_name(element.tag)
|
||||
if not root_tag:
|
||||
root_tag = tag
|
||||
tags.add(tag)
|
||||
attributes[tag].update(local_name(name) for name in element.attrib)
|
||||
return {
|
||||
"status": "ok",
|
||||
"path": str(path),
|
||||
"artifact": path.name,
|
||||
"root_tag": root_tag,
|
||||
"tags": sorted(tags),
|
||||
"attributes": {key: sorted(value) for key, value in sorted(attributes.items()) if value},
|
||||
}
|
||||
except Exception as exc:
|
||||
return {"status": "error", "path": str(path), "artifact": path.name, "message": str(exc)}
|
||||
|
||||
|
||||
def scan_artifacts(root: Path, workers: int) -> tuple[dict[str, Any], list[dict[str, Any]]]:
|
||||
files = artifact_files(root)
|
||||
with ThreadPoolExecutor(max_workers=max(1, workers)) as executor:
|
||||
parsed = list(executor.map(parse_artifact, files))
|
||||
errors = [item for item in parsed if item["status"] != "ok"]
|
||||
grouped: dict[tuple[str, str], list[dict[str, Any]]] = defaultdict(list)
|
||||
for item in parsed:
|
||||
if item["status"] == "ok":
|
||||
grouped[(str(item["artifact"]), str(item["root_tag"]))].append(item)
|
||||
schemas: dict[str, Any] = {}
|
||||
for (artifact, root_tag), values in sorted(grouped.items()):
|
||||
tags: set[str] = set()
|
||||
attributes: dict[str, set[str]] = defaultdict(set)
|
||||
for value in values:
|
||||
tags.update(value["tags"])
|
||||
for tag, names in value["attributes"].items():
|
||||
attributes[tag].update(names)
|
||||
key = f"{artifact}:{root_tag}"
|
||||
schemas[key] = {
|
||||
"files": len(values),
|
||||
"root_tag": root_tag,
|
||||
"tags": sorted(tags),
|
||||
"attributes": {tag: sorted(names) for tag, names in sorted(attributes.items())},
|
||||
"samples": [value["path"] for value in values[:3]],
|
||||
}
|
||||
return {"files": len(files), "schemas": schemas}, errors
|
||||
|
||||
|
||||
def scan_layer(root: Path, layer: str, workers: int) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]:
|
||||
files = layer_files(root)
|
||||
with ThreadPoolExecutor(max_workers=max(1, workers)) as executor:
|
||||
parsed = list(executor.map(lambda path: parse_metadata_file(path, layer), files))
|
||||
return [item for item in parsed if item["status"] == "ok"], [item for item in parsed if item["status"] != "ok"]
|
||||
|
||||
|
||||
def merge_kind_schemas(objects: Iterable[dict[str, Any]]) -> dict[str, Any]:
|
||||
kinds: dict[str, dict[str, Any]] = {}
|
||||
grouped: dict[str, list[dict[str, Any]]] = defaultdict(list)
|
||||
for item in objects:
|
||||
grouped[str(item["kind"])].append(item)
|
||||
for kind, values in sorted(grouped.items()):
|
||||
properties: set[str] = set()
|
||||
child_counts: Counter[str] = Counter()
|
||||
child_properties: dict[str, set[str]] = defaultdict(set)
|
||||
for value in values:
|
||||
properties.update(value.get("properties") or [])
|
||||
for child_kind, schema in (value.get("child_schemas") or {}).items():
|
||||
child_counts[child_kind] += int(schema.get("count") or 0)
|
||||
child_properties[child_kind].update(schema.get("properties") or [])
|
||||
kinds[kind] = {
|
||||
"objects": len(values),
|
||||
"properties": sorted(properties),
|
||||
"children": {
|
||||
child_kind: {"objects": child_counts[child_kind], "properties": sorted(child_properties[child_kind])}
|
||||
for child_kind in sorted(child_counts)
|
||||
},
|
||||
"samples": [
|
||||
{"ref": f"{kind}.{value['name']}" if value.get("name") else kind, "uuid": value.get("uuid")}
|
||||
for value in values[:3]
|
||||
],
|
||||
}
|
||||
return kinds
|
||||
|
||||
|
||||
def object_ref(item: dict[str, Any]) -> str:
|
||||
return f"{item.get('kind')}.{item.get('name')}" if item.get("name") else f"{item.get('kind')}#{item.get('uuid')}"
|
||||
|
||||
|
||||
def extension_summary(name: str, objects: list[dict[str, Any]], base_refs: set[str]) -> dict[str, Any]:
|
||||
refs = {object_ref(item) for item in objects}
|
||||
return {
|
||||
"name": name,
|
||||
"objects": len(objects),
|
||||
"kinds": merge_kind_schemas(objects),
|
||||
"overrides": sorted(refs & base_refs),
|
||||
"extension_only": sorted(refs - base_refs),
|
||||
"counts": {"overrides": len(refs & base_refs), "extension_only": len(refs - base_refs)},
|
||||
}
|
||||
|
||||
|
||||
def build_report(root: Path, workers: int, include_artifacts: bool) -> dict[str, Any]:
|
||||
configuration_root = root / "Конфигурация"
|
||||
extensions_root = root / "Расширения"
|
||||
base_objects, errors = scan_layer(configuration_root, "configuration", workers)
|
||||
base_refs = {object_ref(item) for item in base_objects}
|
||||
extensions: list[dict[str, Any]] = []
|
||||
configuration_artifacts: dict[str, Any] = {"status": "not_requested", "files": 0, "schemas": {}}
|
||||
if include_artifacts:
|
||||
configuration_artifacts, artifact_errors = scan_artifacts(configuration_root, workers)
|
||||
configuration_artifacts["status"] = "ok" if not artifact_errors else "partial"
|
||||
errors.extend(artifact_errors)
|
||||
if extensions_root.is_dir():
|
||||
for extension_root in sorted(path for path in extensions_root.iterdir() if path.is_dir()):
|
||||
objects, extension_errors = scan_layer(extension_root, f"extension:{extension_root.name}", workers)
|
||||
errors.extend(extension_errors)
|
||||
summary = extension_summary(extension_root.name, objects, base_refs)
|
||||
if include_artifacts:
|
||||
artifacts, artifact_errors = scan_artifacts(extension_root, workers)
|
||||
artifacts["status"] = "ok" if not artifact_errors else "partial"
|
||||
summary["artifacts"] = artifacts
|
||||
errors.extend(artifact_errors)
|
||||
extensions.append(summary)
|
||||
kinds = merge_kind_schemas(base_objects)
|
||||
return {
|
||||
"schema": "onec_xml_metadata_analysis.v1",
|
||||
"status": "ok" if not errors else "partial",
|
||||
"generated_at": datetime.now(timezone.utc).isoformat(),
|
||||
"source_root": str(root),
|
||||
"assumption": "The exported base configuration is equivalent to the SQL base; extensions are independent overlays and may differ.",
|
||||
"configuration": {
|
||||
"objects": len(base_objects),
|
||||
"kinds": kinds,
|
||||
"counts": {"kinds": len(kinds), "objects": len(base_objects)},
|
||||
"artifacts": configuration_artifacts,
|
||||
},
|
||||
"extensions": extensions,
|
||||
"counts": {
|
||||
"configuration_kinds": len(kinds),
|
||||
"configuration_objects": len(base_objects),
|
||||
"extensions": len(extensions),
|
||||
"extension_objects": sum(item["objects"] for item in extensions),
|
||||
"artifact_files": int(configuration_artifacts.get("files") or 0) + sum(int((item.get("artifacts") or {}).get("files") or 0) for item in extensions),
|
||||
"parse_errors": len(errors),
|
||||
},
|
||||
"errors": errors[:100],
|
||||
}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Extract complete declared metadata property schemas from a 1C XML configuration export.")
|
||||
parser.add_argument("--root", type=Path, default=DEFAULT_ROOT)
|
||||
parser.add_argument("--workers", type=int, default=8)
|
||||
parser.add_argument("--include-artifacts", action="store_true", help="Also scan nested Ext XML files such as forms, rights, and templates.")
|
||||
parser.add_argument("--output", type=Path)
|
||||
parser.add_argument("--json", action="store_true", help="Print the full JSON report instead of a compact summary.")
|
||||
args = parser.parse_args()
|
||||
report = build_report(args.root, args.workers, args.include_artifacts)
|
||||
rendered = json.dumps(report, ensure_ascii=False, indent=2)
|
||||
if args.output:
|
||||
args.output.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.output.write_text(rendered + "\n", encoding="utf-8")
|
||||
if args.json:
|
||||
print(rendered)
|
||||
else:
|
||||
print(json.dumps({"status": report["status"], **report["counts"]}, ensure_ascii=False))
|
||||
return 0 if report["status"] == "ok" else 2
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,121 @@
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
function Invoke-RemotePowerShell {
|
||||
param(
|
||||
[Parameter(Mandatory = $true)][string]$SshTarget,
|
||||
[Parameter(Mandatory = $true)][string]$Script
|
||||
)
|
||||
$encoded = [Convert]::ToBase64String([Text.Encoding]::Unicode.GetBytes($Script))
|
||||
ssh $SshTarget "powershell -NoProfile -EncodedCommand $encoded"
|
||||
}
|
||||
|
||||
function New-AppArchive {
|
||||
$archive = Join-Path $env:TEMP "llm-model-chat-app.zip"
|
||||
Remove-Item -Force $archive -ErrorAction SilentlyContinue
|
||||
Get-ChildItem -Path "scripts", "tools", "plugins" -Recurse -Directory -Filter "__pycache__" -ErrorAction SilentlyContinue |
|
||||
Sort-Object FullName -Descending |
|
||||
ForEach-Object {
|
||||
Remove-Item -LiteralPath $_.FullName -Recurse -Force -ErrorAction SilentlyContinue
|
||||
}
|
||||
$items = @(
|
||||
"config",
|
||||
"scripts",
|
||||
"tools",
|
||||
"registry",
|
||||
"plugins",
|
||||
"docs",
|
||||
"evals",
|
||||
"datasets",
|
||||
"requirements.txt",
|
||||
"requirements-training.txt",
|
||||
"README.md"
|
||||
)
|
||||
Compress-Archive -Path $items -DestinationPath $archive -Force
|
||||
|
||||
Add-Type -AssemblyName System.IO.Compression.FileSystem
|
||||
$zip = [System.IO.Compression.ZipFile]::Open($archive, [System.IO.Compression.ZipArchiveMode]::Update)
|
||||
try {
|
||||
@($zip.Entries | Where-Object { $_.FullName.Replace("\", "/") -match "/__pycache__/" }) |
|
||||
ForEach-Object { $_.Delete() }
|
||||
} finally {
|
||||
$zip.Dispose()
|
||||
}
|
||||
|
||||
$preflightReport = "reports/model-chat/preflight.json"
|
||||
if (Test-Path -LiteralPath $preflightReport) {
|
||||
$zip = [System.IO.Compression.ZipFile]::Open($archive, [System.IO.Compression.ZipArchiveMode]::Update)
|
||||
try {
|
||||
[System.IO.Compression.ZipFileExtensions]::CreateEntryFromFile(
|
||||
$zip,
|
||||
(Get-Item -LiteralPath $preflightReport).FullName,
|
||||
"reports/model-chat/preflight.json"
|
||||
) | Out-Null
|
||||
} finally {
|
||||
$zip.Dispose()
|
||||
}
|
||||
}
|
||||
return $archive
|
||||
}
|
||||
|
||||
function Test-AppArchive {
|
||||
param([Parameter(Mandatory = $true)][string]$Archive)
|
||||
|
||||
Add-Type -AssemblyName System.IO.Compression.FileSystem
|
||||
$requiredEntries = @(
|
||||
"config/gpu_profiles.json",
|
||||
"scripts/model_chat_server.py",
|
||||
"scripts/transformers_plugin_server.py",
|
||||
"scripts/common.py",
|
||||
"tools/model-chat/index.html",
|
||||
"registry/index.json",
|
||||
"plugins/1c/rag/profiles.yaml",
|
||||
"requirements.txt",
|
||||
"requirements-training.txt",
|
||||
"README.md"
|
||||
)
|
||||
if (Test-Path -LiteralPath "reports/model-chat/preflight.json") {
|
||||
$requiredEntries += "reports/model-chat/preflight.json"
|
||||
}
|
||||
|
||||
$zip = [System.IO.Compression.ZipFile]::OpenRead($Archive)
|
||||
try {
|
||||
$entryNames = [System.Collections.Generic.HashSet[string]]::new([StringComparer]::OrdinalIgnoreCase)
|
||||
foreach ($entry in $zip.Entries) {
|
||||
[void]$entryNames.Add($entry.FullName.Replace("\", "/"))
|
||||
}
|
||||
$missing = @($requiredEntries | Where-Object { -not $entryNames.Contains($_) })
|
||||
if ($missing.Count -gt 0) {
|
||||
throw "Archive is missing required entries: $($missing -join ', ')"
|
||||
}
|
||||
$pycacheEntries = @($entryNames | Where-Object { $_ -match "/__pycache__/" })
|
||||
if ($pycacheEntries.Count -gt 0) {
|
||||
throw "Archive contains __pycache__ entries: $($pycacheEntries[0])"
|
||||
}
|
||||
Write-Host "Archive validation passed: $Archive"
|
||||
} finally {
|
||||
$zip.Dispose()
|
||||
}
|
||||
}
|
||||
|
||||
function Sync-AppDirectory {
|
||||
param(
|
||||
[Parameter(Mandatory = $true)][string]$SshTarget,
|
||||
[Parameter(Mandatory = $true)][string]$RemoteArchive,
|
||||
[Parameter(Mandatory = $true)][string]$RemoteAppDir,
|
||||
[string]$Archive
|
||||
)
|
||||
if (-not $Archive) {
|
||||
$Archive = New-AppArchive
|
||||
Test-AppArchive -Archive $Archive
|
||||
}
|
||||
ssh $SshTarget "cmd /c if not exist C:\ProgramData\LLM mkdir C:\ProgramData\LLM"
|
||||
$remoteArchiveForScp = $RemoteArchive.Replace("\", "/")
|
||||
scp $Archive "${SshTarget}:$remoteArchiveForScp"
|
||||
$unpackScript = @"
|
||||
New-Item -ItemType Directory -Force '$RemoteAppDir' | Out-Null
|
||||
Remove-Item -Recurse -Force '$RemoteAppDir\*' -ErrorAction SilentlyContinue
|
||||
Expand-Archive -Path '$RemoteArchive' -DestinationPath '$RemoteAppDir' -Force
|
||||
New-Item -ItemType Directory -Force '$RemoteAppDir\reports','$RemoteAppDir\models\incoming' | Out-Null
|
||||
"@
|
||||
Invoke-RemotePowerShell -SshTarget $SshTarget -Script $unpackScript
|
||||
}
|
||||
@@ -0,0 +1,114 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import sys
|
||||
import urllib.error
|
||||
from pathlib import Path
|
||||
|
||||
from common import call_chat_completion, read_json, search_lexical_index
|
||||
from rag_profiles import resolve_rag_profile
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
DEFAULT_INDEX = ROOT / "plugins" / "1c" / "datasets" / "prepared" / "rag_index.json"
|
||||
DEFAULT_SYSTEM_PROMPT = ROOT / "plugins" / "1c" / "prompts" / "system.md"
|
||||
DEFAULT_RAG_PROMPT = ROOT / "plugins" / "1c" / "prompts" / "rag-answer.md"
|
||||
|
||||
|
||||
def format_context(results: list[dict], *, max_chars: int = 12000) -> str:
|
||||
if not results:
|
||||
return "Контекст не найден."
|
||||
|
||||
blocks = []
|
||||
used_chars = 0
|
||||
for position, result in enumerate(results, start=1):
|
||||
document = result["document"]
|
||||
source = document.get("source_path") or "unknown"
|
||||
chunk = document.get("chunk_index")
|
||||
title = document.get("title") or "unknown"
|
||||
content = (document.get("content") or "").strip()
|
||||
header = f"[{position}] source={source} title={title} chunk={chunk} score={result['score']:.4f}"
|
||||
remaining = max_chars - used_chars - len(header) - 2
|
||||
if remaining <= 0:
|
||||
break
|
||||
if len(content) > remaining:
|
||||
content = content[: max(0, remaining - 3)].rstrip() + "..."
|
||||
block = "\n".join([header, content])
|
||||
blocks.append(block)
|
||||
used_chars += len(block) + 2
|
||||
return "\n\n".join(blocks)
|
||||
|
||||
|
||||
def render_prompt(template_path: Path, context: str, question: str) -> str:
|
||||
template = template_path.read_text(encoding="utf-8")
|
||||
return template.replace("{{context}}", context).replace("{{question}}", question)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Ask the 1C RAG assistant.")
|
||||
parser.add_argument("question")
|
||||
parser.add_argument("--index", type=Path, default=DEFAULT_INDEX)
|
||||
parser.add_argument("--profile", default="auto")
|
||||
parser.add_argument("--limit", type=int)
|
||||
parser.add_argument("--candidate-limit", type=int)
|
||||
parser.add_argument("--dedupe-by-document", action="store_true")
|
||||
parser.add_argument("--min-score", type=float)
|
||||
parser.add_argument("--source-type", action="append", dest="source_types")
|
||||
parser.add_argument("--platform-version")
|
||||
parser.add_argument("--platform-doc-id")
|
||||
parser.add_argument("--max-context-chars", type=int)
|
||||
parser.add_argument("--base-url", help="OpenAI-compatible endpoint, for example http://docker-gpu.cin.su:8000")
|
||||
parser.add_argument("--model", default="qwen3-4b-instruct")
|
||||
parser.add_argument("--print-prompt", action="store_true", help="Print assembled prompt instead of calling a model.")
|
||||
parser.add_argument("--system-prompt", type=Path, default=DEFAULT_SYSTEM_PROMPT)
|
||||
parser.add_argument("--rag-prompt", type=Path, default=DEFAULT_RAG_PROMPT)
|
||||
args = parser.parse_args()
|
||||
|
||||
index = read_json(args.index)
|
||||
profile = resolve_rag_profile(args.profile, args.question)
|
||||
source_types = args.source_types if args.source_types is not None else profile["source_types"]
|
||||
results = search_lexical_index(
|
||||
index,
|
||||
args.question,
|
||||
limit=int(args.limit or profile["limit"]),
|
||||
candidate_limit=int(args.candidate_limit or profile["candidate_limit"]),
|
||||
dedupe_by_document=args.dedupe_by_document or bool(profile["dedupe_by_document"]),
|
||||
min_score=float(args.min_score if args.min_score is not None else profile["min_score"]),
|
||||
source_types=source_types,
|
||||
metadata_filters={
|
||||
"platform_version": args.platform_version or "",
|
||||
"platform_doc_id": args.platform_doc_id or "",
|
||||
},
|
||||
)
|
||||
context = format_context(results, max_chars=int(args.max_context_chars or profile["max_context_chars"]))
|
||||
rag_prompt = render_prompt(args.rag_prompt, context=context, question=args.question)
|
||||
|
||||
if args.print_prompt or not args.base_url:
|
||||
print(rag_prompt)
|
||||
if not args.base_url and not args.print_prompt:
|
||||
print(
|
||||
"\nNo --base-url provided, so only the prompt was rendered.",
|
||||
file=sys.stderr,
|
||||
)
|
||||
return 0
|
||||
|
||||
system_prompt = args.system_prompt.read_text(encoding="utf-8")
|
||||
try:
|
||||
answer = call_chat_completion(
|
||||
base_url=args.base_url,
|
||||
model=args.model,
|
||||
messages=[
|
||||
{"role": "system", "content": system_prompt},
|
||||
{"role": "user", "content": rag_prompt},
|
||||
],
|
||||
max_tokens=1200,
|
||||
)
|
||||
except (urllib.error.URLError, ValueError) as exc:
|
||||
print(f"Chat request failed: {exc}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
print(answer)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,156 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
if str(ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(ROOT))
|
||||
|
||||
import scripts.compare_1c_access_role_audit as compare_script
|
||||
import scripts.export_1c_access_role_audit as export_script
|
||||
|
||||
|
||||
def compare_stem(old_path: Path, new_path: Path) -> str:
|
||||
return f"compare-{old_path.stem.replace('.summary', '')}-{new_path.stem.replace('.summary', '')}"
|
||||
|
||||
|
||||
def verdict_for(export_summary: dict[str, Any], compare_result: dict[str, Any] | None) -> str:
|
||||
counts = compare_result.get("counts") if isinstance(compare_result, dict) and isinstance(compare_result.get("counts"), dict) else {}
|
||||
if any(int(counts.get(key) or 0) > 0 for key in ("added_users", "removed_users", "changed_access_paths")):
|
||||
return "changed"
|
||||
if str(export_summary.get("risk_level") or "").lower() in {"medium", "high"}:
|
||||
return "risk"
|
||||
return "ok"
|
||||
|
||||
|
||||
def audit_role(
|
||||
*,
|
||||
adapter_url: str,
|
||||
base_id: str,
|
||||
role: str,
|
||||
report_root: Path,
|
||||
timeout: int,
|
||||
user_threshold: int,
|
||||
limit: int,
|
||||
include_html: bool = True,
|
||||
) -> dict[str, Any]:
|
||||
export_summary = export_script.export_role_audit(
|
||||
adapter_url=adapter_url,
|
||||
base_id=base_id,
|
||||
role=role,
|
||||
report_root=report_root,
|
||||
timeout=timeout,
|
||||
user_threshold=user_threshold,
|
||||
include_analysis=True,
|
||||
limit=limit,
|
||||
include_html=include_html,
|
||||
)
|
||||
latest = compare_script.find_latest_summaries(report_root, base_id, role=role, count=2)
|
||||
compare_result: dict[str, Any] | None = None
|
||||
if len(latest) >= 2:
|
||||
new_path, old_path = latest[0], latest[1]
|
||||
folder = report_root / compare_script.slugify(base_id, max_length=60)
|
||||
stem = compare_stem(old_path, new_path)
|
||||
compare_result = compare_script.compare_files(
|
||||
old_path,
|
||||
new_path,
|
||||
output=folder / f"{stem}.json",
|
||||
html_output=folder / f"{stem}.html",
|
||||
)
|
||||
export_script.update_index(report_root, base_id)
|
||||
return {
|
||||
"schema": "onec_access_role_audit_run.v1",
|
||||
"status": export_summary.get("status"),
|
||||
"base_id": base_id,
|
||||
"role": role,
|
||||
"verdict": verdict_for(export_summary, compare_result),
|
||||
"export": export_summary,
|
||||
"compare": compare_result,
|
||||
}
|
||||
|
||||
|
||||
def audit_config(
|
||||
*,
|
||||
config_path: Path,
|
||||
adapter_url: str,
|
||||
report_root: Path,
|
||||
timeout: int,
|
||||
limit: int,
|
||||
include_html: bool = True,
|
||||
) -> dict[str, Any]:
|
||||
config = json.loads(config_path.read_text(encoding="utf-8"))
|
||||
roles = config.get("roles") if isinstance(config.get("roles"), list) else []
|
||||
default_base_id = str(config.get("base_id") or "upo_test")
|
||||
results: list[dict[str, Any]] = []
|
||||
for item in roles:
|
||||
if not isinstance(item, dict) or not item.get("role"):
|
||||
continue
|
||||
results.append(
|
||||
audit_role(
|
||||
adapter_url=adapter_url,
|
||||
base_id=str(item.get("base_id") or default_base_id),
|
||||
role=str(item.get("role")),
|
||||
report_root=report_root,
|
||||
timeout=timeout,
|
||||
user_threshold=int(item.get("user_threshold") or 50),
|
||||
limit=limit,
|
||||
include_html=include_html,
|
||||
)
|
||||
)
|
||||
verdicts = [str(item.get("verdict") or "") for item in results]
|
||||
overall = "changed" if "changed" in verdicts else "risk" if "risk" in verdicts else "ok"
|
||||
return {
|
||||
"schema": "onec_access_role_audit_batch.v1",
|
||||
"status": "ok" if all(item.get("status") == "ok" for item in results) else "error",
|
||||
"config": str(config_path),
|
||||
"overall_verdict": overall,
|
||||
"counts": {"roles": len(results), "changed": verdicts.count("changed"), "risk": verdicts.count("risk"), "ok": verdicts.count("ok")},
|
||||
"results": results,
|
||||
}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Run 1C role access audit: export, analyze, update index, compare with previous.")
|
||||
parser.add_argument("--adapter-url", default=export_script.DEFAULT_BASE_URL)
|
||||
parser.add_argument("--base-id", default="upo_test")
|
||||
parser.add_argument("--role")
|
||||
parser.add_argument("--config", type=Path, help="Run all roles from an access critical roles JSON config.")
|
||||
parser.add_argument("--report-root", type=Path, default=export_script.DEFAULT_REPORT_ROOT)
|
||||
parser.add_argument("--timeout", type=int, default=120)
|
||||
parser.add_argument("--limit", type=int, default=20000)
|
||||
parser.add_argument("--user-threshold", type=int, default=50)
|
||||
parser.add_argument("--no-html", action="store_true")
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.config:
|
||||
result = audit_config(
|
||||
config_path=args.config,
|
||||
adapter_url=args.adapter_url,
|
||||
report_root=args.report_root,
|
||||
timeout=args.timeout,
|
||||
limit=args.limit,
|
||||
include_html=not args.no_html,
|
||||
)
|
||||
else:
|
||||
if not args.role:
|
||||
parser.error("--role is required unless --config is used.")
|
||||
result = audit_role(
|
||||
adapter_url=args.adapter_url,
|
||||
base_id=args.base_id,
|
||||
role=args.role,
|
||||
report_root=args.report_root,
|
||||
timeout=args.timeout,
|
||||
user_threshold=args.user_threshold,
|
||||
limit=args.limit,
|
||||
include_html=not args.no_html,
|
||||
)
|
||||
print(json.dumps(result, ensure_ascii=False, indent=2))
|
||||
return 0 if result.get("status") == "ok" else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,562 @@
|
||||
#!/usr/bin/env python3
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import concurrent.futures
|
||||
import copy
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import time
|
||||
import threading
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any, Callable
|
||||
|
||||
|
||||
DEFAULT_BASE_URL = "http://docker-gpu.cin.su:8011"
|
||||
DEFAULT_BASE_ID = "upo_test"
|
||||
|
||||
# Metadata kinds that either own application data or expose values through the
|
||||
# public data facade. Kinds absent from a concrete base remain in the audit so
|
||||
# that coverage cannot be declared only from a convenient test configuration.
|
||||
DATA_KINDS = {
|
||||
"AccountingRegister",
|
||||
"AccumulationRegister",
|
||||
"BusinessProcess",
|
||||
"CalculationRegister",
|
||||
"Catalog",
|
||||
"ChartOfAccounts",
|
||||
"ChartOfCalculationTypes",
|
||||
"ChartOfCharacteristicTypes",
|
||||
"Constant",
|
||||
"Document",
|
||||
"Enum",
|
||||
"ExchangePlan",
|
||||
"InformationRegister",
|
||||
"Sequence",
|
||||
"Task",
|
||||
}
|
||||
|
||||
Rpc = Callable[[str, str, str, dict[str, Any], float], dict[str, Any]]
|
||||
|
||||
|
||||
def utc_now() -> str:
|
||||
return datetime.now(timezone.utc).isoformat()
|
||||
|
||||
|
||||
def write_json_atomic(path: Path, value: dict[str, Any]) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
temporary = path.with_name(path.name + ".tmp")
|
||||
temporary.write_text(json.dumps(value, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
temporary.replace(path)
|
||||
|
||||
|
||||
def rpc(base_url: str, token: str, method: str, payload: dict[str, Any], timeout: float) -> dict[str, Any]:
|
||||
request = urllib.request.Request(
|
||||
base_url.rstrip("/") + "/rpc",
|
||||
data=json.dumps({"method": method, "payload": payload}, ensure_ascii=False).encode("utf-8"),
|
||||
headers={"Authorization": f"Bearer {token}", "Content-Type": "application/json; charset=utf-8"},
|
||||
method="POST",
|
||||
)
|
||||
with urllib.request.urlopen(request, timeout=timeout) as response:
|
||||
value = json.loads(response.read().decode("utf-8"))
|
||||
return value if isinstance(value, dict) else {"status": "error", "error": "response_not_object"}
|
||||
|
||||
|
||||
def safe_rpc(
|
||||
rpc_call: Rpc,
|
||||
base_url: str,
|
||||
token: str,
|
||||
method: str,
|
||||
payload: dict[str, Any],
|
||||
timeout: float,
|
||||
) -> tuple[dict[str, Any], int]:
|
||||
started = time.monotonic()
|
||||
try:
|
||||
result = rpc_call(base_url, token, method, payload, timeout)
|
||||
except (TimeoutError, urllib.error.URLError, OSError, ValueError) as exc:
|
||||
result = {
|
||||
"status": "transport_error",
|
||||
"error": type(exc).__name__,
|
||||
"diagnostics": {"message": str(exc)[:500]},
|
||||
}
|
||||
return result, round((time.monotonic() - started) * 1000)
|
||||
|
||||
|
||||
def first_object(response: dict[str, Any]) -> dict[str, Any] | None:
|
||||
for key in ("objects", "items"):
|
||||
values = response.get(key)
|
||||
if isinstance(values, list) and values and isinstance(values[0], dict):
|
||||
return values[0]
|
||||
return None
|
||||
|
||||
|
||||
def public_selector(sample: dict[str, Any]) -> dict[str, Any]:
|
||||
return {
|
||||
key: sample[key]
|
||||
for key in ("ref", "kind", "name", "guid")
|
||||
if sample.get(key) not in {None, ""}
|
||||
}
|
||||
|
||||
|
||||
def operation_summary(result: dict[str, Any], duration_ms: int, **values: Any) -> dict[str, Any]:
|
||||
summary = {"status": result.get("status") or "unknown", "duration_ms": duration_ms, **values}
|
||||
diagnostics = result.get("diagnostics")
|
||||
if summary["status"] != "ok" and isinstance(diagnostics, dict) and diagnostics.get("message"):
|
||||
summary["message"] = str(diagnostics["message"])[:500]
|
||||
if result.get("error"):
|
||||
summary["error"] = str(result["error"])[:200]
|
||||
return summary
|
||||
|
||||
|
||||
def record_ref_from_row(row: Any) -> str | None:
|
||||
if not isinstance(row, dict):
|
||||
return None
|
||||
value = row.get("ref")
|
||||
if isinstance(value, dict):
|
||||
value = value.get("hex")
|
||||
compact = str(value or "").replace("-", "").strip()
|
||||
return compact if re.fullmatch(r"[0-9a-fA-F]{32}", compact) else None
|
||||
|
||||
|
||||
def audit_data_kind(
|
||||
base_url: str,
|
||||
base_id: str,
|
||||
token: str,
|
||||
kind: str,
|
||||
timeout: float,
|
||||
include_reads: bool,
|
||||
rpc_call: Rpc = rpc,
|
||||
existing: dict[str, Any] | None = None,
|
||||
progress: Callable[[dict[str, Any]], None] | None = None,
|
||||
) -> dict[str, Any]:
|
||||
common = {"base_id": base_id, "timeout_seconds": max(1, int(timeout))}
|
||||
result = copy.deepcopy(existing) if isinstance(existing, dict) else {}
|
||||
result.update({"kind": kind, "status": "degraded"})
|
||||
result.setdefault("operations", {})
|
||||
|
||||
def save_progress() -> None:
|
||||
if progress is not None:
|
||||
progress(copy.deepcopy(result))
|
||||
|
||||
prior_list = result["operations"].get("metadata.objects.list") or {}
|
||||
if prior_list.get("status") == "ok" and isinstance(result.get("sample"), dict):
|
||||
listed = {"status": "ok"}
|
||||
sample = {**(result["sample"].get("selector") or {}), "name": result["sample"].get("name")}
|
||||
else:
|
||||
listed, list_ms = safe_rpc(
|
||||
rpc_call,
|
||||
base_url,
|
||||
token,
|
||||
"metadata.objects.list",
|
||||
{**common, "kind": kind, "limit": 1},
|
||||
timeout,
|
||||
)
|
||||
sample = first_object(listed)
|
||||
result["operations"]["metadata.objects.list"] = operation_summary(
|
||||
listed,
|
||||
list_ms,
|
||||
objects=len(listed.get("objects") or listed.get("items") or []),
|
||||
)
|
||||
if sample:
|
||||
result["sample"] = {"selector": public_selector(sample), "name": sample.get("name")}
|
||||
save_progress()
|
||||
if not sample:
|
||||
result["status"] = "absent" if listed.get("status") == "ok" else "degraded"
|
||||
result["reason"] = "no_sample_object"
|
||||
save_progress()
|
||||
return result
|
||||
|
||||
selector = public_selector(sample)
|
||||
result["sample"] = {"selector": selector, "name": sample.get("name")}
|
||||
prior_schema = result["operations"].get("data.schema") or {}
|
||||
if prior_schema.get("status") == "ok":
|
||||
schema_status = "ok"
|
||||
else:
|
||||
schema, schema_ms = safe_rpc(
|
||||
rpc_call,
|
||||
base_url,
|
||||
token,
|
||||
"data.schema",
|
||||
{**common, **selector},
|
||||
timeout,
|
||||
)
|
||||
table = schema.get("table") if isinstance(schema.get("table"), dict) else {}
|
||||
result["operations"]["data.schema"] = operation_summary(
|
||||
schema,
|
||||
schema_ms,
|
||||
fields=len(schema.get("fields") or []),
|
||||
table=table.get("name"),
|
||||
cache=(schema.get("cache") or {}).get("status") if isinstance(schema.get("cache"), dict) else None,
|
||||
)
|
||||
schema_status = str(schema.get("status") or "unknown")
|
||||
save_progress()
|
||||
if schema_status != "ok" or not include_reads:
|
||||
result["status"] = "ok" if schema_status == "ok" else "degraded"
|
||||
save_progress()
|
||||
return result
|
||||
|
||||
prior_data_list = result["operations"].get("data.list") or {}
|
||||
must_repeat_list = prior_data_list.get("status") != "ok" or (
|
||||
"sample_record_ref" not in result and "sample_record_ref_status" not in result
|
||||
)
|
||||
if must_repeat_list:
|
||||
data_list, data_list_ms = safe_rpc(
|
||||
rpc_call,
|
||||
base_url,
|
||||
token,
|
||||
"data.list",
|
||||
{**common, **selector, "limit": 1},
|
||||
timeout,
|
||||
)
|
||||
rows = data_list.get("rows") if isinstance(data_list.get("rows"), list) else []
|
||||
list_summary = operation_summary(data_list, data_list_ms, rows=len(rows))
|
||||
if data_list.get("status") == "ok":
|
||||
result["operations"]["data.list"] = list_summary
|
||||
result["operations"].pop("data.list_retry", None)
|
||||
ref = record_ref_from_row(rows[0]) if rows else None
|
||||
if ref:
|
||||
result["sample_record_ref"] = ref
|
||||
result.pop("sample_record_ref_status", None)
|
||||
else:
|
||||
result.pop("sample_record_ref", None)
|
||||
result["sample_record_ref_status"] = "empty_object" if not rows else "object_has_no_reference_key"
|
||||
elif prior_data_list.get("status") == "ok":
|
||||
# A successful operation is evidence. Do not downgrade it only
|
||||
# because a later attempt to recover the sample ref timed out.
|
||||
result["operations"]["data.list_retry"] = list_summary
|
||||
ref = result.get("sample_record_ref")
|
||||
else:
|
||||
result["operations"]["data.list"] = list_summary
|
||||
result.pop("sample_record_ref", None)
|
||||
result["sample_record_ref_status"] = "data_list_failed"
|
||||
ref = None
|
||||
save_progress()
|
||||
else:
|
||||
ref = result.get("sample_record_ref")
|
||||
|
||||
if (result["operations"].get("data.count") or {}).get("status") != "ok":
|
||||
counted, count_ms = safe_rpc(
|
||||
rpc_call,
|
||||
base_url,
|
||||
token,
|
||||
"data.count",
|
||||
{**common, **selector},
|
||||
timeout,
|
||||
)
|
||||
result["operations"]["data.count"] = operation_summary(counted, count_ms, count=counted.get("count"))
|
||||
save_progress()
|
||||
|
||||
prior_get_status = (result["operations"].get("data.get") or {}).get("status")
|
||||
if prior_get_status not in {"ok", "not_applicable"}:
|
||||
if ref:
|
||||
fetched, get_ms = safe_rpc(
|
||||
rpc_call,
|
||||
base_url,
|
||||
token,
|
||||
"data.get",
|
||||
{**common, **selector, "record_ref": ref},
|
||||
timeout,
|
||||
)
|
||||
fetched_rows = fetched.get("rows") if isinstance(fetched.get("rows"), list) else []
|
||||
result["operations"]["data.get"] = operation_summary(fetched, get_ms, rows=len(fetched_rows))
|
||||
elif (result["operations"].get("data.list") or {}).get("status") == "ok":
|
||||
reason = str(result.get("sample_record_ref_status") or "object_has_no_reference_key")
|
||||
result["operations"]["data.get"] = {"status": "not_applicable", "reason": reason, "duration_ms": 0}
|
||||
else:
|
||||
result["operations"]["data.get"] = {"status": "blocked", "reason": "data_list_failed", "duration_ms": 0}
|
||||
save_progress()
|
||||
|
||||
required = ("data.schema", "data.list", "data.count")
|
||||
failures = [name for name in required if result["operations"].get(name, {}).get("status") != "ok"]
|
||||
get_status = result["operations"]["data.get"]["status"]
|
||||
if get_status not in {"ok", "not_applicable"}:
|
||||
failures.append("data.get")
|
||||
result["status"] = "ok" if not failures else "degraded"
|
||||
if failures:
|
||||
result["failed_operations"] = failures
|
||||
else:
|
||||
result.pop("failed_operations", None)
|
||||
save_progress()
|
||||
return result
|
||||
|
||||
|
||||
def load_checkpoint(
|
||||
path: Path | None,
|
||||
base_url: str,
|
||||
base_id: str,
|
||||
resume: bool,
|
||||
include_reads: bool,
|
||||
) -> dict[str, Any]:
|
||||
fresh = {
|
||||
"schema": "onec_adapter_data_audit_checkpoint.v1",
|
||||
"base_url": base_url,
|
||||
"base_id": base_id,
|
||||
"include_reads": include_reads,
|
||||
"started_at": utc_now(),
|
||||
"updated_at": utc_now(),
|
||||
"checks": {},
|
||||
}
|
||||
if not resume or path is None or not path.exists():
|
||||
return fresh
|
||||
value = json.loads(path.read_text(encoding="utf-8-sig"))
|
||||
if value.get("schema") != fresh["schema"]:
|
||||
raise ValueError(f"unsupported checkpoint schema in {path}")
|
||||
if value.get("base_url") != base_url or value.get("base_id") != base_id:
|
||||
raise ValueError(f"checkpoint {path} belongs to another adapter or base")
|
||||
if bool(value.get("include_reads")) != include_reads:
|
||||
raise ValueError(f"checkpoint {path} was created for another data audit mode")
|
||||
if not isinstance(value.get("checks"), dict):
|
||||
raise ValueError(f"checkpoint {path} has no checks object")
|
||||
return value
|
||||
|
||||
|
||||
def run_data_checks(
|
||||
base_url: str,
|
||||
base_id: str,
|
||||
token: str,
|
||||
kinds: list[str],
|
||||
timeout: float,
|
||||
include_reads: bool,
|
||||
workers: int,
|
||||
checkpoint_path: Path | None,
|
||||
resume: bool,
|
||||
retry_degraded: bool = False,
|
||||
rpc_call: Rpc = rpc,
|
||||
) -> tuple[dict[str, dict[str, Any]], int]:
|
||||
checkpoint = load_checkpoint(checkpoint_path, base_url, base_id, resume, include_reads)
|
||||
checks = checkpoint["checks"]
|
||||
checkpoint_lock = threading.Lock()
|
||||
reusable = {
|
||||
kind
|
||||
for kind in kinds
|
||||
if kind in checks and (not retry_degraded or checks[kind].get("status") == "ok")
|
||||
}
|
||||
resumed = len(reusable)
|
||||
pending = [kind for kind in kinds if kind not in reusable]
|
||||
|
||||
def execute(kind: str) -> dict[str, Any]:
|
||||
def save_partial(value: dict[str, Any]) -> None:
|
||||
with checkpoint_lock:
|
||||
checks[kind] = value
|
||||
checkpoint["updated_at"] = utc_now()
|
||||
if checkpoint_path is not None:
|
||||
write_json_atomic(checkpoint_path, checkpoint)
|
||||
|
||||
return audit_data_kind(
|
||||
base_url,
|
||||
base_id,
|
||||
token,
|
||||
kind,
|
||||
timeout,
|
||||
include_reads,
|
||||
rpc_call,
|
||||
existing=checks.get(kind),
|
||||
progress=save_partial,
|
||||
)
|
||||
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=max(1, workers)) as executor:
|
||||
futures = {executor.submit(execute, kind): kind for kind in pending}
|
||||
for future in concurrent.futures.as_completed(futures):
|
||||
kind = futures[future]
|
||||
try:
|
||||
checks[kind] = future.result()
|
||||
except Exception as exc: # a single kind must not discard completed evidence
|
||||
checks[kind] = {
|
||||
"kind": kind,
|
||||
"status": "degraded",
|
||||
"error": type(exc).__name__,
|
||||
"message": str(exc)[:500],
|
||||
}
|
||||
with checkpoint_lock:
|
||||
checkpoint["updated_at"] = utc_now()
|
||||
if checkpoint_path is not None:
|
||||
write_json_atomic(checkpoint_path, checkpoint)
|
||||
return {kind: checks[kind] for kind in kinds if kind in checks}, resumed
|
||||
|
||||
|
||||
def build_report(
|
||||
base_url: str,
|
||||
base_id: str,
|
||||
token: str,
|
||||
timeout: float,
|
||||
sample_objects: bool,
|
||||
sample_schemas: bool,
|
||||
*,
|
||||
sample_reads: bool = False,
|
||||
workers: int = 1,
|
||||
checkpoint_path: Path | None = None,
|
||||
resume: bool = False,
|
||||
retry_degraded: bool = False,
|
||||
rpc_call: Rpc = rpc,
|
||||
) -> dict[str, Any]:
|
||||
audit, _ = safe_rpc(
|
||||
rpc_call,
|
||||
base_url,
|
||||
token,
|
||||
"metadata.adapter.audit",
|
||||
{"base_id": base_id, "include_missing": True, "include_unmapped": True, "timeout_seconds": int(timeout)},
|
||||
timeout,
|
||||
)
|
||||
if audit.get("status") != "ok":
|
||||
return {"schema": "onec_adapter_coverage_audit.v1", "status": "error", "audit": audit}
|
||||
|
||||
matrix: list[dict[str, Any]] = []
|
||||
for support in audit.get("metadata_kinds") or []:
|
||||
if not isinstance(support, dict):
|
||||
continue
|
||||
kind = str(support.get("kind") or "")
|
||||
count = int(support.get("count") or 0)
|
||||
matrix.append({
|
||||
"kind": kind,
|
||||
"kind_ru": support.get("kind_ru"),
|
||||
"objects": count,
|
||||
"capabilities": support.get("capabilities") or [],
|
||||
"discovery": "present" if count else "absent_in_base",
|
||||
})
|
||||
|
||||
data_checks: dict[str, dict[str, Any]] = {}
|
||||
resumed_checks = 0
|
||||
if sample_schemas or sample_reads:
|
||||
present_data_kinds = sorted(row["kind"] for row in matrix if row["objects"] and row["kind"] in DATA_KINDS)
|
||||
data_checks, resumed_checks = run_data_checks(
|
||||
base_url,
|
||||
base_id,
|
||||
token,
|
||||
present_data_kinds,
|
||||
timeout,
|
||||
sample_reads,
|
||||
workers,
|
||||
checkpoint_path,
|
||||
resume,
|
||||
retry_degraded,
|
||||
rpc_call,
|
||||
)
|
||||
for row in matrix:
|
||||
check = data_checks.get(row["kind"])
|
||||
if check:
|
||||
row["data_check"] = check
|
||||
row["list_status"] = check.get("operations", {}).get("metadata.objects.list", {}).get("status")
|
||||
schema = check.get("operations", {}).get("data.schema")
|
||||
if schema:
|
||||
row["data_schema"] = schema
|
||||
elif sample_objects:
|
||||
for row in matrix:
|
||||
if not row["objects"]:
|
||||
continue
|
||||
listed, _ = safe_rpc(
|
||||
rpc_call,
|
||||
base_url,
|
||||
token,
|
||||
"metadata.objects.list",
|
||||
{"base_id": base_id, "kind": row["kind"], "limit": 1, "timeout_seconds": int(timeout)},
|
||||
timeout,
|
||||
)
|
||||
row["list_status"] = listed.get("status")
|
||||
sample = first_object(listed)
|
||||
if sample:
|
||||
row["sample_selector"] = public_selector(sample)
|
||||
|
||||
missing = [row["kind"] for row in matrix if row["discovery"] == "absent_in_base"]
|
||||
failures = [
|
||||
row["kind"]
|
||||
for row in matrix
|
||||
if sample_objects and row.get("objects") and "list_status" in row and row.get("list_status") != "ok"
|
||||
]
|
||||
data_failures = sorted(kind for kind, check in data_checks.items() if check.get("status") != "ok")
|
||||
status = "ok" if not failures and not data_failures else "degraded"
|
||||
return {
|
||||
"schema": "onec_adapter_coverage_audit.v1",
|
||||
"status": status,
|
||||
"generated_at": utc_now(),
|
||||
"base_url": base_url,
|
||||
"base_id": base_id,
|
||||
"sampling": {
|
||||
"objects": sample_objects,
|
||||
"data_schemas": sample_schemas or sample_reads,
|
||||
"data_reads": sample_reads,
|
||||
"workers": workers,
|
||||
"resumed_checks": resumed_checks,
|
||||
},
|
||||
"policy": {
|
||||
"application_data": "read_only",
|
||||
"metadata_structure": "read_only",
|
||||
"sql_identity": "configured_base_credentials_only",
|
||||
"writes": ["ConfigSave", "ConfigCASSave"],
|
||||
},
|
||||
"counts": {
|
||||
"kinds": len(matrix),
|
||||
"present_kinds": sum(1 for row in matrix if row["objects"]),
|
||||
"absent_kinds": len(missing),
|
||||
"list_failures": len(failures),
|
||||
"data_kinds_declared": len(DATA_KINDS),
|
||||
"data_kinds_checked": len(data_checks),
|
||||
"data_check_failures": len(data_failures),
|
||||
},
|
||||
"absent_in_base": missing,
|
||||
"list_failures": failures,
|
||||
"data_check_failures": data_failures,
|
||||
"matrix": matrix,
|
||||
"data_checks": data_checks,
|
||||
"child_objects": audit.get("child_objects") or {},
|
||||
"not_yet_decoded": audit.get("not_yet_decoded") or [],
|
||||
"optional_deep_reads": audit.get("optional_deep_reads") or [],
|
||||
"unmapped_source_roles": audit.get("unmapped_source_roles") or audit.get("unknown_source_roles") or {},
|
||||
"write_capabilities": audit.get("write_capabilities") or {},
|
||||
}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Audit live 1C adapter coverage without exposing SQL credentials.")
|
||||
parser.add_argument("--base-url", default=DEFAULT_BASE_URL)
|
||||
parser.add_argument("--base-id", default=DEFAULT_BASE_ID)
|
||||
parser.add_argument("--token-env", default="ONEC_ADAPTER_TOKEN")
|
||||
parser.add_argument("--timeout", type=float, default=120.0, help="Timeout for each adapter call, in seconds.")
|
||||
parser.add_argument("--workers", type=int, default=1, help="Concurrent data-kind checks (default: 1).")
|
||||
parser.add_argument("--sample-objects", action="store_true", help="List one object for each present metadata kind.")
|
||||
parser.add_argument("--sample-data-schemas", action="store_true", help="Decode one logical data schema for every present data kind.")
|
||||
parser.add_argument("--sample-data-reads", action="store_true", help="Run schema, list, get (when applicable), and count for every present data kind.")
|
||||
parser.add_argument("--checkpoint", type=Path, help="Atomically save progress after every completed data kind.")
|
||||
parser.add_argument("--resume", action="store_true", help="Reuse completed kinds from --checkpoint.")
|
||||
parser.add_argument("--retry-degraded", action="store_true", help="With --resume, rerun checkpoint entries whose status is not ok.")
|
||||
parser.add_argument("--output", type=Path)
|
||||
args = parser.parse_args()
|
||||
if args.workers < 1:
|
||||
parser.error("--workers must be >= 1")
|
||||
if args.resume and args.checkpoint is None:
|
||||
parser.error("--resume requires --checkpoint")
|
||||
if args.retry_degraded and not args.resume:
|
||||
parser.error("--retry-degraded requires --resume")
|
||||
token = os.environ.get(args.token_env, "").strip()
|
||||
if not token:
|
||||
parser.error(f"adapter token is required in environment variable {args.token_env}")
|
||||
sample_objects = args.sample_objects or args.sample_data_schemas or args.sample_data_reads
|
||||
try:
|
||||
report = build_report(
|
||||
args.base_url,
|
||||
args.base_id,
|
||||
token,
|
||||
args.timeout,
|
||||
sample_objects,
|
||||
args.sample_data_schemas,
|
||||
sample_reads=args.sample_data_reads,
|
||||
workers=args.workers,
|
||||
checkpoint_path=args.checkpoint,
|
||||
resume=args.resume,
|
||||
retry_degraded=args.retry_degraded,
|
||||
)
|
||||
except ValueError as exc:
|
||||
parser.error(str(exc))
|
||||
rendered = json.dumps(report, ensure_ascii=False, indent=2)
|
||||
if args.output:
|
||||
write_json_atomic(args.output, report)
|
||||
print(rendered)
|
||||
return 0 if report.get("status") == "ok" else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,191 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import datetime as dt
|
||||
import json
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
|
||||
from common import ROOT, read_json, write_json
|
||||
|
||||
|
||||
DEFAULT_MODEL_ID = "qwen3-coder-30b-a3b-instruct-q6_k"
|
||||
DEFAULT_PROFILES = ["gpu-fast", "cpu-test"]
|
||||
DEFAULT_REPORT = ROOT / "reports" / "benchmarks" / "runtime-profiles-qwen3-coder-q6.json"
|
||||
RUNTIME_PROFILES = ROOT / "config" / "runtime_profiles.json"
|
||||
DEFAULT_PROMPT = (
|
||||
"Ты эксперт 1С. Кратко, но предметно опиши безопасный план анализа ошибки "
|
||||
"проведения документа РеализацияТоваровУслуг, если нет metadata snapshot. Дай 8 пунктов."
|
||||
)
|
||||
|
||||
|
||||
def chat_completion(
|
||||
*,
|
||||
base_url: str,
|
||||
model: str,
|
||||
prompt: str,
|
||||
temperature: float,
|
||||
max_tokens: int,
|
||||
timeout: int,
|
||||
) -> dict:
|
||||
payload = {
|
||||
"model": model,
|
||||
"messages": [
|
||||
{"role": "system", "content": "Отвечай по-русски, кратко и по делу."},
|
||||
{"role": "user", "content": prompt},
|
||||
],
|
||||
"temperature": temperature,
|
||||
"max_tokens": max_tokens,
|
||||
"stream": False,
|
||||
}
|
||||
request = urllib.request.Request(
|
||||
f"{base_url.rstrip('/')}/v1/chat/completions",
|
||||
data=json.dumps(payload, ensure_ascii=False).encode("utf-8"),
|
||||
headers={"Content-Type": "application/json"},
|
||||
method="POST",
|
||||
)
|
||||
started_at = time.perf_counter()
|
||||
with urllib.request.urlopen(request, timeout=timeout) as response:
|
||||
data = json.loads(response.read().decode("utf-8"))
|
||||
elapsed_sec = time.perf_counter() - started_at
|
||||
choices = data.get("choices") or []
|
||||
answer = ""
|
||||
if choices and isinstance(choices[0], dict):
|
||||
answer = str((choices[0].get("message") or {}).get("content") or "")
|
||||
usage = data.get("usage") or {}
|
||||
completion_tokens = int(usage.get("completion_tokens") or 0)
|
||||
total_tokens = int(usage.get("total_tokens") or 0)
|
||||
prompt_tokens = int(usage.get("prompt_tokens") or 0)
|
||||
return {
|
||||
"status": "ok",
|
||||
"elapsed_sec": round(elapsed_sec, 3),
|
||||
"prompt_tokens": prompt_tokens,
|
||||
"completion_tokens": completion_tokens,
|
||||
"total_tokens": total_tokens,
|
||||
"output_tokens_per_sec": round(completion_tokens / elapsed_sec, 3) if completion_tokens else None,
|
||||
"total_tokens_per_sec": round(total_tokens / elapsed_sec, 3) if total_tokens else None,
|
||||
"answer_preview": answer[:800],
|
||||
}
|
||||
|
||||
|
||||
def profile_target(profile: dict, model_id: str, plugin: str) -> dict:
|
||||
overrides = profile.get("model_overrides") or {}
|
||||
override = overrides.get(model_id) or {}
|
||||
base_url = override.get("base_url") or (profile.get("endpoints") or {}).get(plugin)
|
||||
served_model_name = override.get("served_model_name")
|
||||
if not base_url:
|
||||
raise ValueError(f"profile `{profile.get('id')}` has no endpoint for model `{model_id}`")
|
||||
if not served_model_name:
|
||||
raise ValueError(f"profile `{profile.get('id')}` has no served_model_name override for `{model_id}`")
|
||||
return {
|
||||
"base_url": str(base_url),
|
||||
"served_model_name": str(served_model_name),
|
||||
"container_name": override.get("container_name"),
|
||||
"host": profile.get("host"),
|
||||
"docker_endpoint": profile.get("docker_endpoint"),
|
||||
"role": profile.get("role"),
|
||||
}
|
||||
|
||||
|
||||
def benchmark_profile(profile_id: str, profile: dict, *, model_id: str, plugin: str, args: argparse.Namespace) -> dict:
|
||||
try:
|
||||
target = profile_target(profile, model_id, plugin)
|
||||
result = chat_completion(
|
||||
base_url=target["base_url"],
|
||||
model=target["served_model_name"],
|
||||
prompt=args.prompt,
|
||||
temperature=args.temperature,
|
||||
max_tokens=args.max_tokens,
|
||||
timeout=args.timeout,
|
||||
)
|
||||
return {
|
||||
"profile_id": profile_id,
|
||||
"label": profile.get("label") or profile_id,
|
||||
"model_id": model_id,
|
||||
"target": target,
|
||||
**result,
|
||||
}
|
||||
except (ValueError, TimeoutError, urllib.error.URLError, json.JSONDecodeError) as exc:
|
||||
return {
|
||||
"profile_id": profile_id,
|
||||
"label": profile.get("label") or profile_id,
|
||||
"model_id": model_id,
|
||||
"status": "error",
|
||||
"error": str(exc),
|
||||
}
|
||||
|
||||
|
||||
def speedup_summary(results: list[dict]) -> dict:
|
||||
speeds = {
|
||||
str(item.get("profile_id")): float(item.get("output_tokens_per_sec") or 0)
|
||||
for item in results
|
||||
if item.get("status") == "ok" and item.get("output_tokens_per_sec")
|
||||
}
|
||||
gpu_speed = speeds.get("gpu-fast")
|
||||
cpu_speed = speeds.get("cpu-test")
|
||||
summary = {"output_tokens_per_sec": speeds}
|
||||
if gpu_speed and cpu_speed:
|
||||
summary["gpu_vs_cpu_ratio"] = round(gpu_speed / cpu_speed, 3)
|
||||
return summary
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Benchmark the same model across runtime profiles.")
|
||||
parser.add_argument("--model-id", default=DEFAULT_MODEL_ID)
|
||||
parser.add_argument("--plugin", default="1c")
|
||||
parser.add_argument("--profiles", nargs="+", default=DEFAULT_PROFILES)
|
||||
parser.add_argument("--prompt", default=DEFAULT_PROMPT)
|
||||
parser.add_argument("--temperature", type=float, default=0.1)
|
||||
parser.add_argument("--max-tokens", type=int, default=384)
|
||||
parser.add_argument("--timeout", type=int, default=600)
|
||||
parser.add_argument("--report", type=Path, default=DEFAULT_REPORT)
|
||||
parser.add_argument("--print", action="store_true")
|
||||
args = parser.parse_args()
|
||||
|
||||
config = read_json(RUNTIME_PROFILES)
|
||||
profiles = config.get("profiles") or {}
|
||||
results = []
|
||||
for profile_id in args.profiles:
|
||||
profile = profiles.get(profile_id)
|
||||
if not isinstance(profile, dict):
|
||||
results.append(
|
||||
{
|
||||
"profile_id": profile_id,
|
||||
"model_id": args.model_id,
|
||||
"status": "error",
|
||||
"error": f"unknown runtime profile: {profile_id}",
|
||||
}
|
||||
)
|
||||
continue
|
||||
profile = {**profile, "id": profile_id}
|
||||
results.append(benchmark_profile(profile_id, profile, model_id=args.model_id, plugin=args.plugin, args=args))
|
||||
|
||||
ok_results = [item for item in results if item.get("status") == "ok"]
|
||||
fastest = None
|
||||
if ok_results:
|
||||
fastest = max(ok_results, key=lambda item: float(item.get("output_tokens_per_sec") or 0)).get("profile_id")
|
||||
report = {
|
||||
"created_at": dt.datetime.now(dt.UTC).isoformat(),
|
||||
"model_id": args.model_id,
|
||||
"plugin": args.plugin,
|
||||
"prompt": args.prompt,
|
||||
"temperature": args.temperature,
|
||||
"max_tokens": args.max_tokens,
|
||||
"status": "ok" if len(ok_results) == len(results) else "partial" if ok_results else "failed",
|
||||
"fastest_profile_id": fastest,
|
||||
"speedup": speedup_summary(results),
|
||||
"results": results,
|
||||
}
|
||||
write_json(args.report, report)
|
||||
if args.print:
|
||||
print(json.dumps(report, ensure_ascii=False, indent=2))
|
||||
else:
|
||||
print(f"Benchmark status: {report['status']}")
|
||||
print(f"Wrote report to {args.report}")
|
||||
return 0 if ok_results else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,150 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Build the first agent-facing intake packet for a 1C user task."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import base64
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
||||
from route_1c_question import route_question # noqa: E402
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
DEFAULT_INDEX = ROOT / "reports" / "1c-sql" / "upo" / "unified-object-route-index.json"
|
||||
|
||||
|
||||
def decode_arg(value: str | None, encoded: str | None) -> str | None:
|
||||
if encoded:
|
||||
return base64.b64decode(encoded).decode("utf-8")
|
||||
return value
|
||||
|
||||
|
||||
def fact_summary(fact_checks: list[dict[str, Any]]) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]:
|
||||
confirmed = []
|
||||
unresolved = []
|
||||
for check in fact_checks:
|
||||
result = check.get("result") or {}
|
||||
row = {
|
||||
"path": check.get("path"),
|
||||
"status": check.get("status"),
|
||||
"exists": result.get("exists") if result else None,
|
||||
"confidence": result.get("confidence"),
|
||||
"reason": result.get("reason"),
|
||||
"object": result.get("object"),
|
||||
"match": result.get("match"),
|
||||
}
|
||||
if result.get("exists") is True:
|
||||
confirmed.append(row)
|
||||
else:
|
||||
unresolved.append(row)
|
||||
return confirmed, unresolved
|
||||
|
||||
|
||||
def next_commands(route: dict[str, Any], *, index: Path, view: str) -> list[dict[str, Any]]:
|
||||
commands = []
|
||||
decision = route.get("decision") or {}
|
||||
if decision.get("needs_docs_rag"):
|
||||
commands.append(
|
||||
{
|
||||
"tool": "docs_rag",
|
||||
"purpose": "official_documentation_context",
|
||||
"api": "/api/rag/query",
|
||||
"payload": {
|
||||
"question": route.get("question"),
|
||||
"source_type": decision.get("safe_rag_scope") or "official_1c_docs",
|
||||
"limit": 5,
|
||||
},
|
||||
}
|
||||
)
|
||||
for path in route.get("fact_paths") or []:
|
||||
commands.append(
|
||||
{
|
||||
"tool": "fact_resolver",
|
||||
"purpose": "current_configuration_fact",
|
||||
"command": f"python scripts/resolve_1c_fact.py --index {index} --path <utf8-base64:{path}> --view {view}",
|
||||
"api": "/api/1c/fact",
|
||||
"payload": {
|
||||
"source_kind": "route_index",
|
||||
"source_path": str(index),
|
||||
"path": path,
|
||||
"view": view,
|
||||
},
|
||||
}
|
||||
)
|
||||
if decision.get("needs_current_config") and not route.get("fact_paths"):
|
||||
commands.append(
|
||||
{
|
||||
"tool": "task_evidence",
|
||||
"purpose": "discover_objects_and_relevant_metadata",
|
||||
"command": f"python scripts/build_1c_task_evidence.py --index {index} --text <utf8-base64 task> --view {view}",
|
||||
}
|
||||
)
|
||||
return commands
|
||||
|
||||
|
||||
def build_intake(text: str, *, index: Path, view: str) -> dict[str, Any]:
|
||||
route = route_question(text, index_path=index, view=view)
|
||||
confirmed, unresolved = fact_summary(route.get("fact_checks") or [])
|
||||
decision = route.get("decision") or {}
|
||||
code_allowed = not unresolved and bool(confirmed or not decision.get("needs_current_config"))
|
||||
if decision.get("needs_current_config") and not confirmed and not route.get("fact_paths"):
|
||||
code_allowed = False
|
||||
|
||||
return {
|
||||
"schema": "onec_agent_intake.v1",
|
||||
"task": {"text": text},
|
||||
"index": str(index),
|
||||
"view": view,
|
||||
"route": route,
|
||||
"source_policy": {
|
||||
"allowed_for_current_facts": ["route_index", "metadata_snapshot_explicit_current", "1c_agent_current"],
|
||||
"allowed_for_documentation": ["official_1c_docs"],
|
||||
"examples_are_current_facts": False,
|
||||
"blocked_as_current_fact_sources": ["metadata.example", "synthetic-example", "rag_examples", "old_exports"],
|
||||
},
|
||||
"facts": {
|
||||
"confirmed": confirmed,
|
||||
"unresolved": unresolved,
|
||||
"confirmed_count": len(confirmed),
|
||||
"unresolved_count": len(unresolved),
|
||||
},
|
||||
"answer_policy": {
|
||||
"code_generation_allowed": code_allowed,
|
||||
"must_check_current_config_before_code": bool(decision.get("current_config_required_before_code")),
|
||||
"must_not_use_examples_as_facts": True,
|
||||
"safe_rag_scope": decision.get("safe_rag_scope"),
|
||||
},
|
||||
"next_commands": next_commands(route, index=index, view=view),
|
||||
}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Build first agent intake packet for a 1C task.")
|
||||
parser.add_argument("--text")
|
||||
parser.add_argument("--text-b64")
|
||||
parser.add_argument("--index", type=Path, default=DEFAULT_INDEX)
|
||||
parser.add_argument("--view", choices=["effective", "base"], default="effective")
|
||||
parser.add_argument("--output", type=Path)
|
||||
args = parser.parse_args()
|
||||
|
||||
text = decode_arg(args.text, args.text_b64)
|
||||
if not text:
|
||||
raise SystemExit("Use --text or --text-b64.")
|
||||
result = build_intake(text, index=args.index, view=args.view)
|
||||
output = json.dumps(result, ensure_ascii=False, indent=2) + "\n"
|
||||
if args.output:
|
||||
args.output.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.output.write_text(output, encoding="utf-8")
|
||||
print(json.dumps({"output": str(args.output), "route": result["route"]["decision"]["route"], "code_allowed": result["answer_policy"]["code_generation_allowed"]}, ensure_ascii=False))
|
||||
else:
|
||||
print(output)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,153 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Build enum order -> presentation map from 1C XML routes."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import xml.etree.ElementTree as ET
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
NS = {
|
||||
"md": "http://v8.3/MDClasses",
|
||||
"v8": "http://v8.1c.ru/8.1/data/core",
|
||||
}
|
||||
|
||||
|
||||
def load_json(path: Path) -> dict[str, Any]:
|
||||
return json.loads(path.read_text(encoding="utf-8-sig"))
|
||||
|
||||
|
||||
def local_name(tag: str) -> str:
|
||||
return tag.rsplit("}", 1)[-1]
|
||||
|
||||
|
||||
def child_text(parent: ET.Element, name: str) -> str | None:
|
||||
for child in list(parent):
|
||||
if local_name(child.tag) == name:
|
||||
return child.text or ""
|
||||
return None
|
||||
|
||||
|
||||
def properties(element: ET.Element) -> ET.Element | None:
|
||||
for child in list(element):
|
||||
if local_name(child.tag) == "Properties":
|
||||
return child
|
||||
return None
|
||||
|
||||
|
||||
def synonym(props: ET.Element | None) -> str | None:
|
||||
if props is None:
|
||||
return None
|
||||
for syn in list(props):
|
||||
if local_name(syn.tag) != "Synonym":
|
||||
continue
|
||||
for item in list(syn):
|
||||
lang = None
|
||||
content = None
|
||||
for child in list(item):
|
||||
if local_name(child.tag) == "lang":
|
||||
lang = child.text
|
||||
elif local_name(child.tag) == "content":
|
||||
content = child.text
|
||||
if lang == "ru" and content:
|
||||
return content
|
||||
return None
|
||||
|
||||
|
||||
def enum_values(path: Path) -> list[dict[str, Any]]:
|
||||
root = ET.parse(path).getroot()
|
||||
enum = next((node for node in root.iter() if local_name(node.tag) == "Enum"), None)
|
||||
if enum is None:
|
||||
return []
|
||||
child_objects = next((node for node in list(enum) if local_name(node.tag) == "ChildObjects"), None)
|
||||
if child_objects is None:
|
||||
return []
|
||||
values = []
|
||||
order = 0
|
||||
for node in list(child_objects):
|
||||
if local_name(node.tag) != "EnumValue":
|
||||
continue
|
||||
props = properties(node)
|
||||
name = child_text(props, "Name") if props is not None else None
|
||||
values.append(
|
||||
{
|
||||
"order": order,
|
||||
"uuid": (node.attrib.get("uuid") or "").lower() or None,
|
||||
"name": name,
|
||||
"synonym": synonym(props),
|
||||
}
|
||||
)
|
||||
order += 1
|
||||
return values
|
||||
|
||||
|
||||
def is_base_config(top: dict[str, Any]) -> bool:
|
||||
relative = str(top.get("relative_path") or "")
|
||||
return relative.startswith("Enums\\")
|
||||
|
||||
|
||||
def route_score(row: dict[str, Any]) -> tuple[int, str]:
|
||||
return (1 if str(row.get("relative_path") or "").startswith("Enums\\") else 0, str(row.get("relative_path") or ""))
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Build 1C enum presentation map.")
|
||||
parser.add_argument("--index", type=Path, required=True)
|
||||
parser.add_argument("--output", type=Path, required=True)
|
||||
args = parser.parse_args()
|
||||
|
||||
index = load_json(args.index)
|
||||
enums: dict[str, dict[str, Any]] = {}
|
||||
for guid, item in (index.get("objects") or {}).items():
|
||||
tops = [
|
||||
top
|
||||
for top in item.get("xml_top_objects") or []
|
||||
if top.get("xml_kind") == "Enum" and top.get("name") and top.get("path")
|
||||
]
|
||||
if not tops:
|
||||
continue
|
||||
tops.sort(key=lambda top: (not is_base_config(top), top.get("relative_path") or ""))
|
||||
top = tops[0]
|
||||
path = Path(top["path"])
|
||||
if not path.is_file():
|
||||
continue
|
||||
try:
|
||||
values = enum_values(path)
|
||||
except ET.ParseError as error:
|
||||
values = []
|
||||
parse_error = str(error)
|
||||
else:
|
||||
parse_error = None
|
||||
key = str(top["name"])
|
||||
candidate = {
|
||||
"guid": guid,
|
||||
"name": top.get("name"),
|
||||
"synonym": top.get("synonym"),
|
||||
"relative_path": top.get("relative_path"),
|
||||
"path": top.get("path"),
|
||||
"parse_error": parse_error,
|
||||
"values": values,
|
||||
"by_order": {str(value["order"]): value for value in values},
|
||||
"by_uuid": {value["uuid"]: value for value in values if value.get("uuid")},
|
||||
}
|
||||
existing = enums.get(key)
|
||||
if existing is None or route_score(candidate) > route_score(existing):
|
||||
enums[key] = candidate
|
||||
|
||||
result = {
|
||||
"schema": "onec_enum_presentation_map.v1",
|
||||
"index": str(args.index),
|
||||
"enum_count": len(enums),
|
||||
"enums": dict(sorted(enums.items())),
|
||||
}
|
||||
args.output.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.output.write_text(json.dumps(result, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
print(json.dumps({"output": str(args.output), "enum_count": len(enums)}, ensure_ascii=False))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,121 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Build an inventory that maps _ExtensionsInfo rows to DBNames-Ext files.
|
||||
|
||||
The script keeps the byte conversion explicit. SQL _ExtensionsInfo._IDRRef is
|
||||
stored as 16 bytes. Observed DBNames-Ext suffixes match this rearrangement:
|
||||
|
||||
b[12:16] b[10:12] b[8:10] b[0:2] b[2:8]
|
||||
|
||||
This is recorded as an observed conversion and validated against exported
|
||||
DBNames-Ext file names.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
def load_json(path: Path) -> dict[str, Any]:
|
||||
return json.loads(path.read_text(encoding="utf-8-sig"))
|
||||
|
||||
|
||||
def dbnames_ext_guid_from_idrref(data: bytes) -> str:
|
||||
if len(data) != 16:
|
||||
raise ValueError(f"_IDRRef must contain 16 bytes, got {len(data)}")
|
||||
reordered = data[12:16] + data[10:12] + data[8:10] + data[0:2] + data[2:8]
|
||||
return str(uuid.UUID(bytes=reordered))
|
||||
|
||||
|
||||
def binary_info(value: Any) -> dict[str, Any] | None:
|
||||
if isinstance(value, dict) and value.get("type") == "binary":
|
||||
return value
|
||||
return None
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Map _ExtensionsInfo rows to DBNames-Ext files.")
|
||||
parser.add_argument("--extensions-info", type=Path, required=True)
|
||||
parser.add_argument("--params-dir", type=Path, required=True)
|
||||
parser.add_argument("--output", type=Path, required=True)
|
||||
args = parser.parse_args()
|
||||
|
||||
report = load_json(args.extensions_info)
|
||||
dbnames_ext_files = {
|
||||
path.name.removeprefix("DBNames-Ext-").lower(): path
|
||||
for path in args.params_dir.iterdir()
|
||||
if path.is_file() and path.name.startswith("DBNames-Ext-")
|
||||
}
|
||||
|
||||
rows = []
|
||||
matched = 0
|
||||
for row in report.get("rows") or []:
|
||||
columns = row.get("columns") or {}
|
||||
idrref = binary_info(columns.get("_IDRRef"))
|
||||
if not idrref:
|
||||
continue
|
||||
idrref_bytes = Path(idrref["path"]).read_bytes()
|
||||
dbnames_guid = dbnames_ext_guid_from_idrref(idrref_bytes)
|
||||
dbnames_file = dbnames_ext_files.get(dbnames_guid)
|
||||
if dbnames_file:
|
||||
matched += 1
|
||||
rows.append(
|
||||
{
|
||||
"row_index": row.get("row_index"),
|
||||
"extension_name": columns.get("_ExtName"),
|
||||
"extension_order": columns.get("_ExtensionOrder"),
|
||||
"update_time": columns.get("_UpdateTime"),
|
||||
"use_purpose": columns.get("_ExtensionUsePurpose"),
|
||||
"scope": columns.get("_ExtensionScope"),
|
||||
"idrref_hex": idrref_bytes.hex(),
|
||||
"dbnames_ext_guid": dbnames_guid,
|
||||
"dbnames_ext_file": str(dbnames_file) if dbnames_file else None,
|
||||
"dbnames_ext_file_name": dbnames_file.name if dbnames_file else None,
|
||||
"dbnames_ext_file_bytes": dbnames_file.stat().st_size if dbnames_file else None,
|
||||
"extension_zipped_info": binary_info(columns.get("_ExtensionZippedInfo")),
|
||||
}
|
||||
)
|
||||
|
||||
known_from_rows = {row["dbnames_ext_guid"] for row in rows}
|
||||
orphan_dbnames_files = [
|
||||
{
|
||||
"file_name": path.name,
|
||||
"guid_or_marker": guid,
|
||||
"bytes": path.stat().st_size,
|
||||
}
|
||||
for guid, path in sorted(dbnames_ext_files.items())
|
||||
if guid not in known_from_rows
|
||||
]
|
||||
|
||||
result = {
|
||||
"schema": "onec_extension_inventory.v1",
|
||||
"extensions_info": str(args.extensions_info),
|
||||
"params_dir": str(args.params_dir),
|
||||
"observed_idrref_to_dbnames_ext_guid": "b[12:16] + b[10:12] + b[8:10] + b[0:2] + b[2:8]",
|
||||
"extension_row_count": len(rows),
|
||||
"matched_dbnames_ext_count": matched,
|
||||
"orphan_dbnames_ext_count": len(orphan_dbnames_files),
|
||||
"extensions": rows,
|
||||
"orphan_dbnames_ext_files": orphan_dbnames_files,
|
||||
}
|
||||
args.output.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.output.write_text(json.dumps(result, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"output": str(args.output),
|
||||
"extensions": len(rows),
|
||||
"matched_dbnames_ext": matched,
|
||||
"orphan_dbnames_ext": len(orphan_dbnames_files),
|
||||
},
|
||||
ensure_ascii=False,
|
||||
)
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,93 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Build root CAS manifests for all parsed 1C extensions."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
from extract_1c_extension_cas_manifest import extract_manifest
|
||||
|
||||
|
||||
def safe_name(value: str) -> str:
|
||||
result = "".join(char if char.isalnum() or char in "-_." else "_" for char in value)
|
||||
return result[:120] or "extension"
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Build CAS manifests for all extensions.")
|
||||
parser.add_argument("--zipped-info", type=Path, required=True)
|
||||
parser.add_argument("--cas-dir", type=Path, required=True)
|
||||
parser.add_argument("--output-dir", type=Path, required=True)
|
||||
parser.add_argument("--summary", type=Path, required=True)
|
||||
args = parser.parse_args()
|
||||
|
||||
zipped = json.loads(args.zipped_info.read_text(encoding="utf-8"))
|
||||
args.output_dir.mkdir(parents=True, exist_ok=True)
|
||||
items = []
|
||||
for item in zipped.get("items") or []:
|
||||
root_key = item.get("root_cas_key")
|
||||
if not root_key:
|
||||
continue
|
||||
root_path = args.cas_dir / root_key
|
||||
if not root_path.is_file():
|
||||
items.append({**item, "manifest_status": "root_cas_missing"})
|
||||
continue
|
||||
try:
|
||||
manifest = extract_manifest(root_path, args.cas_dir)
|
||||
except Exception as exc:
|
||||
items.append(
|
||||
{
|
||||
"extension_zipped_info_file": item.get("file_name"),
|
||||
"root_cas_key": root_key,
|
||||
"manifest_status": "parse_error",
|
||||
"error": str(exc),
|
||||
}
|
||||
)
|
||||
continue
|
||||
stem = safe_name(item.get("file_name", "").replace("__ExtensionZippedInfo.bin", ""))
|
||||
output_path = args.output_dir / f"{stem}-{root_key[:8]}.json"
|
||||
report = {
|
||||
"schema": "onec_extension_cas_manifest.v1",
|
||||
"extension_zipped_info": item,
|
||||
**manifest,
|
||||
}
|
||||
output_path.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
items.append(
|
||||
{
|
||||
"extension_zipped_info_file": item.get("file_name"),
|
||||
"root_cas_key": root_key,
|
||||
"manifest_path": str(output_path),
|
||||
"extension_configuration_guid": manifest.get("extension_configuration_guid"),
|
||||
"declared_count": manifest.get("declared_count"),
|
||||
"entry_count": manifest.get("entry_count"),
|
||||
"missing_cas_entries": sum(1 for entry in manifest.get("entries") or [] if not entry.get("cas_exists")),
|
||||
"manifest_status": "ok",
|
||||
}
|
||||
)
|
||||
summary = {
|
||||
"schema": "onec_extension_manifests_summary.v1",
|
||||
"zipped_info": str(args.zipped_info),
|
||||
"cas_dir": str(args.cas_dir),
|
||||
"output_dir": str(args.output_dir),
|
||||
"extension_count": len(items),
|
||||
"items": items,
|
||||
}
|
||||
args.summary.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.summary.write_text(json.dumps(summary, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"summary": str(args.summary),
|
||||
"extensions": len(items),
|
||||
"ok": sum(1 for item in items if item.get("manifest_status") == "ok"),
|
||||
},
|
||||
ensure_ascii=False,
|
||||
)
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,162 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import yaml
|
||||
|
||||
from one_c_its_platform import parse_doc_coordinate, platform_doc_id_from_url
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
DEFAULT_START_LINKS = ROOT / "plugins" / "1c" / "rag" / "official-docs" / "start-links.json"
|
||||
DEFAULT_SOURCES = ROOT / "plugins" / "1c" / "rag" / "official-docs" / "sources.yaml"
|
||||
DEFAULT_OUTPUT = ROOT / "plugins" / "1c" / "rag" / "official-docs" / "platform-versions.json"
|
||||
|
||||
|
||||
def load_json(path: Path) -> dict[str, Any]:
|
||||
if not path.exists():
|
||||
return {}
|
||||
data = json.loads(path.read_text(encoding="utf-8-sig"))
|
||||
return data if isinstance(data, dict) else {}
|
||||
|
||||
|
||||
def load_yaml(path: Path) -> dict[str, Any]:
|
||||
if not path.exists():
|
||||
return {}
|
||||
data = yaml.safe_load(path.read_text(encoding="utf-8"))
|
||||
return data if isinstance(data, dict) else {}
|
||||
|
||||
|
||||
def version_key(value: str | None) -> tuple[int, int, int, str]:
|
||||
if not value or value == "8.x":
|
||||
return (8, -1, -1, value or "")
|
||||
parts = value.split(".")
|
||||
numeric = []
|
||||
for part in parts[:3]:
|
||||
try:
|
||||
numeric.append(int(part))
|
||||
except ValueError:
|
||||
numeric.append(-1)
|
||||
while len(numeric) < 3:
|
||||
numeric.append(-1)
|
||||
return (numeric[0], numeric[1], numeric[2], value)
|
||||
|
||||
|
||||
def version_from_title(title: str) -> str | None:
|
||||
import re
|
||||
|
||||
match = re.search(r"\b(\d+\.\d+(?:\.\d+)?)\b", title)
|
||||
return match.group(1) if match else None
|
||||
|
||||
|
||||
def best_platform_version(coord: dict[str, str | None], title: str) -> str | None:
|
||||
version = coord.get("platform_version")
|
||||
if version == "8.x":
|
||||
return version_from_title(title) or version
|
||||
return version or version_from_title(title)
|
||||
|
||||
|
||||
def build_catalog(start_links_path: Path, sources_path: Path) -> dict[str, Any]:
|
||||
start_links = load_json(start_links_path)
|
||||
sources_config = load_yaml(sources_path)
|
||||
active_by_doc_id: dict[str, list[dict[str, Any]]] = {}
|
||||
for source in sources_config.get("sources") or []:
|
||||
url = str(source.get("url") or "")
|
||||
doc_id = platform_doc_id_from_url(url)
|
||||
if not doc_id:
|
||||
continue
|
||||
active_by_doc_id.setdefault(doc_id, []).append(
|
||||
{
|
||||
"source_id": source.get("id"),
|
||||
"title": source.get("title"),
|
||||
"url": url,
|
||||
"source_type": source.get("source_type"),
|
||||
}
|
||||
)
|
||||
|
||||
versions_by_doc_id: dict[str, dict[str, Any]] = {}
|
||||
for item in start_links.get("start_links") or []:
|
||||
url = str(item.get("url") or "")
|
||||
coord = parse_doc_coordinate(url)
|
||||
doc_id = coord.get("platform_doc_id")
|
||||
if not doc_id:
|
||||
continue
|
||||
title = str(item.get("text") or "")
|
||||
record = versions_by_doc_id.setdefault(
|
||||
doc_id,
|
||||
{
|
||||
"platform_doc_id": doc_id,
|
||||
"platform_version": best_platform_version(coord, title),
|
||||
"url": url,
|
||||
"title": title,
|
||||
"category": item.get("category"),
|
||||
"active": False,
|
||||
"active_sources": [],
|
||||
},
|
||||
)
|
||||
record["title"] = record.get("title") or item.get("text") or ""
|
||||
record["category"] = record.get("category") or item.get("category")
|
||||
|
||||
for doc_id, sources in active_by_doc_id.items():
|
||||
coord = parse_doc_coordinate(sources[0]["url"])
|
||||
title = str(sources[0].get("title") or "")
|
||||
record = versions_by_doc_id.setdefault(
|
||||
doc_id,
|
||||
{
|
||||
"platform_doc_id": doc_id,
|
||||
"platform_version": best_platform_version(coord, title),
|
||||
"url": sources[0]["url"],
|
||||
"title": title,
|
||||
"category": "platform_doc",
|
||||
"active": False,
|
||||
"active_sources": [],
|
||||
},
|
||||
)
|
||||
record["active"] = True
|
||||
record["active_sources"] = sources
|
||||
|
||||
versions = sorted(versions_by_doc_id.values(), key=lambda item: version_key(item.get("platform_version")), reverse=True)
|
||||
active_versions = [item for item in versions if item.get("active")]
|
||||
latest_8_3 = next((item for item in versions if str(item.get("platform_version") or "").startswith("8.3.")), None)
|
||||
latest_active_8_3 = next((item for item in active_versions if str(item.get("platform_version") or "").startswith("8.3.")), None)
|
||||
return {
|
||||
"schema": "onec_its_platform_versions.v1",
|
||||
"created_at_unix": int(time.time()),
|
||||
"sources": {
|
||||
"start_links": str(start_links_path),
|
||||
"sources_yaml": str(sources_path),
|
||||
},
|
||||
"counts": {
|
||||
"versions": len(versions),
|
||||
"active_versions": len(active_versions),
|
||||
},
|
||||
"defaults": {
|
||||
"latest_discovered_8_3": latest_8_3,
|
||||
"latest_active_8_3": latest_active_8_3,
|
||||
},
|
||||
"versions": versions,
|
||||
}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Build a version catalog for 1C:ITS platform documentation.")
|
||||
parser.add_argument("--start-links", type=Path, default=DEFAULT_START_LINKS)
|
||||
parser.add_argument("--sources", type=Path, default=DEFAULT_SOURCES)
|
||||
parser.add_argument("--output", type=Path, default=DEFAULT_OUTPUT)
|
||||
parser.add_argument("--print", action="store_true", dest="print_report")
|
||||
args = parser.parse_args()
|
||||
|
||||
report = build_catalog(args.start_links, args.sources)
|
||||
args.output.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.output.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
payload = report if args.print_report else {"output": str(args.output), "counts": report["counts"], "defaults": report["defaults"]}
|
||||
print(json.dumps(payload, ensure_ascii=False, indent=2))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,432 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import html
|
||||
import json
|
||||
import re
|
||||
import shutil
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
from html.parser import HTMLParser
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from normalize_1c_its_docs import decode_html
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
DEFAULT_MANIFEST = ROOT / "plugins" / "1c" / "rag" / "official-docs" / "normalized" / "manifest.json"
|
||||
DEFAULT_NORMALIZED_DIR = ROOT / "plugins" / "1c" / "rag" / "official-docs" / "normalized"
|
||||
DEFAULT_RAW_DIR = ROOT / "plugins" / "1c" / "rag" / "official-docs" / "raw"
|
||||
DEFAULT_MEDIA_MANIFEST = ROOT / "plugins" / "1c" / "rag" / "official-docs" / "media" / "manifest.json"
|
||||
DEFAULT_MEDIA_DIR = ROOT / "plugins" / "1c" / "rag" / "official-docs" / "media"
|
||||
DEFAULT_OUTPUT_DIR = ROOT / "plugins" / "1c" / "rag" / "official-docs" / "static"
|
||||
|
||||
|
||||
STYLE = """
|
||||
:root {
|
||||
--bg: #f2f3ef;
|
||||
--paper: #fffef9;
|
||||
--ink: #202522;
|
||||
--muted: #66706a;
|
||||
--line: #cdd5cd;
|
||||
--accent: #0f6b5f;
|
||||
--code: #17201c;
|
||||
}
|
||||
* { box-sizing: border-box; }
|
||||
body { margin: 0; background: var(--bg); color: var(--ink); font: 16px/1.55 "Aptos", "Segoe UI", Tahoma, sans-serif; }
|
||||
a { color: var(--accent); }
|
||||
.wrap { max-width: 1120px; margin: 0 auto; padding: 24px; }
|
||||
.doc { background: var(--paper); border: 1px solid var(--line); border-radius: 8px; padding: 24px; }
|
||||
.meta { color: var(--muted); font-size: 13px; overflow-wrap: anywhere; margin-bottom: 18px; }
|
||||
h1 { font-size: 28px; line-height: 1.2; margin: 0 0 12px; }
|
||||
h2 { font-size: 20px; margin-top: 28px; border-top: 1px solid var(--line); padding-top: 18px; }
|
||||
img { max-width: 100%; height: auto; border: 1px solid var(--line); background: #fff; }
|
||||
figure { margin: 18px 0; }
|
||||
figcaption { color: var(--muted); font-size: 13px; margin-top: 6px; }
|
||||
pre { background: var(--code); color: #e4ece6; padding: 12px; border-radius: 6px; overflow: auto; }
|
||||
code { font-family: "Cascadia Mono", Consolas, monospace; }
|
||||
table { width: 100%; border-collapse: collapse; }
|
||||
th, td { border-bottom: 1px solid var(--line); padding: 8px 10px; text-align: left; vertical-align: top; }
|
||||
.top { display: flex; gap: 8px; flex-wrap: wrap; margin-bottom: 14px; }
|
||||
.btn { border: 1px solid var(--line); border-radius: 6px; padding: 6px 10px; background: #fff; text-decoration: none; }
|
||||
.badge { display: inline-block; border: 1px solid var(--line); border-radius: 999px; padding: 2px 8px; color: var(--muted); font-size: 12px; }
|
||||
""".strip()
|
||||
|
||||
|
||||
class AssetCollector(HTMLParser):
|
||||
def __init__(self, *, page_url: str) -> None:
|
||||
super().__init__(convert_charrefs=True)
|
||||
self.page_url = page_url
|
||||
self.urls: set[str] = set()
|
||||
|
||||
def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
|
||||
tag_lower = tag.lower()
|
||||
attr_map = {name.lower(): value or "" for name, value in attrs}
|
||||
for name, value in attrs:
|
||||
if not value:
|
||||
continue
|
||||
name_lower = name.lower()
|
||||
if tag_lower == "link" and name_lower == "href" and is_static_link_asset(attr_map):
|
||||
self.urls.add(normalize_url(urllib.parse.urljoin(self.page_url, value)))
|
||||
elif tag_lower == "script" and name_lower == "src":
|
||||
self.urls.add(normalize_url(urllib.parse.urljoin(self.page_url, value)))
|
||||
|
||||
|
||||
class LinkRewriter(HTMLParser):
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
page_url: str,
|
||||
url_to_page: dict[str, str],
|
||||
media_url_to_file: dict[str, str],
|
||||
asset_url_to_file: dict[str, str],
|
||||
) -> None:
|
||||
super().__init__(convert_charrefs=False)
|
||||
self.page_url = page_url
|
||||
self.url_to_page = url_to_page
|
||||
self.media_url_to_file = media_url_to_file
|
||||
self.asset_url_to_file = asset_url_to_file
|
||||
self.parts: list[str] = []
|
||||
|
||||
def rewrite_url(self, value: str, *, is_media: bool = False) -> str:
|
||||
absolute = normalize_url(urllib.parse.urljoin(self.page_url, value))
|
||||
if is_media and absolute in self.media_url_to_file:
|
||||
return f"../media/{self.media_url_to_file[absolute]}"
|
||||
if absolute in self.asset_url_to_file:
|
||||
return f"../assets/{self.asset_url_to_file[absolute]}"
|
||||
if absolute in self.url_to_page:
|
||||
return f"../pages/{self.url_to_page[absolute]}"
|
||||
if absolute.startswith(("http://", "https://")):
|
||||
return absolute
|
||||
return value
|
||||
|
||||
def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
|
||||
rewritten = []
|
||||
for name, value in attrs:
|
||||
if value is None:
|
||||
rewritten.append((name, None))
|
||||
continue
|
||||
lowered = name.lower()
|
||||
if tag.lower() == "img" and lowered == "src":
|
||||
value = self.rewrite_url(value, is_media=True)
|
||||
elif tag.lower() == "a" and lowered == "href":
|
||||
value = self.rewrite_url(value)
|
||||
elif tag.lower() in {"link", "script"} and lowered in {"href", "src"}:
|
||||
value = self.rewrite_url(value)
|
||||
rewritten.append((name, value))
|
||||
attr_text = "".join(f" {name}" if value is None else f' {name}="{html.escape(value, quote=True)}"' for name, value in rewritten)
|
||||
self.parts.append(f"<{tag}{attr_text}>")
|
||||
|
||||
def handle_endtag(self, tag: str) -> None:
|
||||
self.parts.append(f"</{tag}>")
|
||||
|
||||
def handle_startendtag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
|
||||
self.handle_starttag(tag, attrs)
|
||||
|
||||
def handle_data(self, data: str) -> None:
|
||||
self.parts.append(data)
|
||||
|
||||
def handle_entityref(self, name: str) -> None:
|
||||
self.parts.append(f"&{name};")
|
||||
|
||||
def handle_charref(self, name: str) -> None:
|
||||
self.parts.append(f"&#{name};")
|
||||
|
||||
def handle_comment(self, data: str) -> None:
|
||||
self.parts.append(f"<!--{data}-->")
|
||||
|
||||
def html(self) -> str:
|
||||
return "".join(self.parts)
|
||||
|
||||
|
||||
def load_json(path: Path) -> dict[str, Any]:
|
||||
if not path.exists():
|
||||
return {}
|
||||
try:
|
||||
data = json.loads(path.read_text(encoding="utf-8-sig"))
|
||||
except json.JSONDecodeError:
|
||||
return {}
|
||||
return data if isinstance(data, dict) else {}
|
||||
|
||||
|
||||
def normalize_url(value: str) -> str:
|
||||
return urllib.parse.urlunparse(urllib.parse.urlparse(value)._replace(fragment=""))
|
||||
|
||||
|
||||
def is_static_link_asset(attrs: dict[str, str]) -> bool:
|
||||
rel = {part.casefold() for part in re.split(r"\s+", attrs.get("rel", "")) if part}
|
||||
href = attrs.get("href", "").casefold()
|
||||
as_type = attrs.get("as", "").casefold()
|
||||
if "stylesheet" in rel or href.endswith(".css"):
|
||||
return True
|
||||
if "icon" in rel or "shortcut" in rel:
|
||||
return True
|
||||
return "preload" in rel and as_type in {"style", "script", "font"}
|
||||
|
||||
|
||||
def safe_html_name(value: str, fallback: str) -> str:
|
||||
name = re.sub(r"[^A-Za-zА-Яа-яЁё0-9_.-]+", "_", value, flags=re.UNICODE).strip("_")
|
||||
return f"{(name or fallback)[:100]}.html"
|
||||
|
||||
|
||||
def safe_asset_name(url: str) -> str:
|
||||
parsed = urllib.parse.urlparse(url)
|
||||
name = Path(parsed.path).name or "asset"
|
||||
stem = re.sub(r"[^A-Za-z0-9_.-]+", "_", Path(name).stem).strip("_") or "asset"
|
||||
suffix = re.sub(r"[^A-Za-z0-9.]+", "", Path(name).suffix) or ".bin"
|
||||
digest = hashlib.sha256(url.encode("utf-8")).hexdigest()[:12]
|
||||
return f"{stem}__{digest}{suffix}"
|
||||
|
||||
|
||||
def media_map(media_manifest: dict[str, Any]) -> dict[str, str]:
|
||||
return {str(item.get("url")): str(item.get("file")) for item in media_manifest.get("items") or [] if item.get("url") and item.get("file")}
|
||||
|
||||
|
||||
def page_name_map(pages: list[dict[str, Any]]) -> dict[str, str]:
|
||||
result = {}
|
||||
used = set()
|
||||
for index, page in enumerate(pages, start=1):
|
||||
name = safe_html_name(str(page.get("title") or page.get("url") or ""), f"page_{index}")
|
||||
if name in used:
|
||||
stem = Path(name).stem
|
||||
name = f"{stem}_{index}.html"
|
||||
used.add(name)
|
||||
url = str(page.get("url") or "")
|
||||
result[url] = name
|
||||
parsed = urllib.parse.urlparse(url)
|
||||
if parsed.path.endswith("/hdoc"):
|
||||
result[normalize_url(urllib.parse.urlunparse(parsed._replace(path=f"{parsed.path}/01")))] = name
|
||||
return result
|
||||
|
||||
|
||||
def collect_asset_urls(pages: list[dict[str, Any]], raw_dir: Path) -> list[str]:
|
||||
urls: set[str] = set()
|
||||
for page in pages:
|
||||
raw_file = str(page.get("raw_file") or "")
|
||||
raw_path = raw_dir / raw_file
|
||||
if not raw_file or not raw_path.exists():
|
||||
continue
|
||||
page_url = str(page.get("url") or "")
|
||||
raw_text = decode_html(raw_path.read_bytes(), page)
|
||||
collector = AssetCollector(page_url=page_url)
|
||||
collector.feed(raw_text)
|
||||
urls.update(url for url in collector.urls if url.startswith(("http://", "https://")))
|
||||
return sorted(urls)
|
||||
|
||||
|
||||
def download_assets(asset_urls: list[str], assets_dir: Path) -> tuple[dict[str, str], list[dict[str, str]]]:
|
||||
assets_dir.mkdir(parents=True, exist_ok=True)
|
||||
url_to_file: dict[str, str] = {}
|
||||
errors: list[dict[str, str]] = []
|
||||
for url in asset_urls:
|
||||
filename = safe_asset_name(url)
|
||||
target = assets_dir / filename
|
||||
try:
|
||||
request = urllib.request.Request(url, headers={"User-Agent": "Codex 1C local static archive"})
|
||||
with urllib.request.urlopen(request, timeout=30) as response:
|
||||
target.write_bytes(response.read())
|
||||
except Exception as exc: # noqa: BLE001
|
||||
errors.append({"url": url, "error": str(exc)})
|
||||
continue
|
||||
url_to_file[url] = filename
|
||||
return url_to_file, errors
|
||||
|
||||
|
||||
def read_normalized_body(path: Path) -> str:
|
||||
text = path.read_text(encoding="utf-8-sig")
|
||||
if text.startswith("---"):
|
||||
parts = text.split("---", 2)
|
||||
if len(parts) == 3:
|
||||
text = parts[2]
|
||||
return text.strip()
|
||||
|
||||
|
||||
def markdown_to_html(markdown: str, media_url_to_file: dict[str, str]) -> str:
|
||||
lines = markdown.splitlines()
|
||||
out: list[str] = []
|
||||
in_list = False
|
||||
for line in lines:
|
||||
stripped = line.strip()
|
||||
if not stripped:
|
||||
if in_list:
|
||||
out.append("</ul>")
|
||||
in_list = False
|
||||
continue
|
||||
if stripped.startswith("# "):
|
||||
if in_list:
|
||||
out.append("</ul>")
|
||||
in_list = False
|
||||
out.append(f"<h1>{html.escape(stripped[2:].strip())}</h1>")
|
||||
continue
|
||||
if stripped.startswith("## "):
|
||||
if in_list:
|
||||
out.append("</ul>")
|
||||
in_list = False
|
||||
out.append(f"<h2>{html.escape(stripped[3:].strip())}</h2>")
|
||||
continue
|
||||
image_match = re.match(r"-\s*!\[(.*?)\]\((.*?)\)(.*)", stripped)
|
||||
if image_match:
|
||||
if in_list:
|
||||
out.append("</ul>")
|
||||
in_list = False
|
||||
alt, url, suffix = image_match.groups()
|
||||
image_src = f"../media/{media_url_to_file[url]}" if url in media_url_to_file else url
|
||||
out.append(f"<figure><img src=\"{html.escape(image_src, quote=True)}\" alt=\"{html.escape(alt)}\"><figcaption>{html.escape((alt + suffix).strip())}</figcaption></figure>")
|
||||
continue
|
||||
if stripped.startswith("- "):
|
||||
if not in_list:
|
||||
out.append("<ul>")
|
||||
in_list = True
|
||||
out.append(f"<li>{html.escape(stripped[2:].strip())}</li>")
|
||||
continue
|
||||
if in_list:
|
||||
out.append("</ul>")
|
||||
in_list = False
|
||||
out.append(f"<p>{html.escape(stripped)}</p>")
|
||||
if in_list:
|
||||
out.append("</ul>")
|
||||
return "\n".join(out)
|
||||
|
||||
|
||||
def write_static_page(
|
||||
page: dict[str, Any],
|
||||
*,
|
||||
normalized_dir: Path,
|
||||
pages_dir: Path,
|
||||
raw_dir: Path,
|
||||
raw_dir_out: Path,
|
||||
page_names: dict[str, str],
|
||||
media_url_to_file: dict[str, str],
|
||||
asset_url_to_file: dict[str, str],
|
||||
) -> dict[str, Any]:
|
||||
page_url = str(page.get("url") or "")
|
||||
page_file = page_names[page_url]
|
||||
normalized_file = str(page.get("normalized_file") or "")
|
||||
normalized_path = normalized_dir / normalized_file
|
||||
body = markdown_to_html(read_normalized_body(normalized_path), media_url_to_file)
|
||||
raw_file = str(page.get("raw_file") or "")
|
||||
raw_output_name = None
|
||||
if raw_file:
|
||||
raw_path = raw_dir / raw_file
|
||||
if raw_path.exists():
|
||||
raw_output_name = f"raw_{page_file}"
|
||||
raw_text = decode_html(raw_path.read_bytes(), page)
|
||||
rewriter = LinkRewriter(page_url=page_url, url_to_page=page_names, media_url_to_file=media_url_to_file, asset_url_to_file=asset_url_to_file)
|
||||
rewriter.feed(raw_text)
|
||||
raw_dir_out.mkdir(parents=True, exist_ok=True)
|
||||
(raw_dir_out / raw_output_name).write_text(rewriter.html(), encoding="utf-8")
|
||||
|
||||
raw_link = f'<a class="btn" href="../raw/{raw_output_name}">Raw HTML</a>' if raw_output_name else ""
|
||||
content = f"""<!doctype html>
|
||||
<html lang="ru">
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<title>{html.escape(str(page.get("title") or ""))}</title>
|
||||
<style>{STYLE}</style>
|
||||
</head>
|
||||
<body>
|
||||
<main class="wrap">
|
||||
<div class="top"><a class="btn" href="../index.html">Индекс</a>{raw_link}<span class="badge">{html.escape(str(page.get("source_type") or ""))}</span></div>
|
||||
<article class="doc">
|
||||
<div class="meta">{html.escape(page_url)}</div>
|
||||
{body}
|
||||
</article>
|
||||
</main>
|
||||
</body>
|
||||
</html>
|
||||
"""
|
||||
pages_dir.mkdir(parents=True, exist_ok=True)
|
||||
(pages_dir / page_file).write_text(content, encoding="utf-8")
|
||||
return {"title": page.get("title"), "url": page_url, "file": f"pages/{page_file}", "raw_file": f"raw/{raw_output_name}" if raw_output_name else None}
|
||||
|
||||
|
||||
def build_static_site(manifest_path: Path, normalized_dir: Path, raw_dir: Path, media_manifest_path: Path, media_dir: Path, output_dir: Path, *, download_external_assets: bool) -> dict[str, Any]:
|
||||
manifest = load_json(manifest_path)
|
||||
media_manifest = load_json(media_manifest_path)
|
||||
pages = manifest.get("pages") or []
|
||||
page_names = page_name_map(pages)
|
||||
media_url_to_file = media_map(media_manifest)
|
||||
pages_dir = output_dir / "pages"
|
||||
raw_dir_out = output_dir / "raw"
|
||||
static_media_dir = output_dir / "media"
|
||||
static_assets_dir = output_dir / "assets"
|
||||
if output_dir.exists():
|
||||
shutil.rmtree(output_dir)
|
||||
static_media_dir.mkdir(parents=True, exist_ok=True)
|
||||
for filename in media_url_to_file.values():
|
||||
source = media_dir / filename
|
||||
if source.exists():
|
||||
shutil.copy2(source, static_media_dir / filename)
|
||||
asset_urls = collect_asset_urls(pages, raw_dir) if download_external_assets else []
|
||||
asset_url_to_file, asset_errors = download_assets(asset_urls, static_assets_dir) if asset_urls else ({}, [])
|
||||
page_records = [
|
||||
write_static_page(
|
||||
page,
|
||||
normalized_dir=normalized_dir,
|
||||
pages_dir=pages_dir,
|
||||
raw_dir=raw_dir,
|
||||
raw_dir_out=raw_dir_out,
|
||||
page_names=page_names,
|
||||
media_url_to_file=media_url_to_file,
|
||||
asset_url_to_file=asset_url_to_file,
|
||||
)
|
||||
for page in pages
|
||||
if page.get("normalized_file")
|
||||
]
|
||||
rows = "\n".join(
|
||||
f'<tr><td><a href="{html.escape(record["file"], quote=True)}">{html.escape(str(record["title"] or ""))}</a></td><td>{html.escape(str(record["url"] or ""))}</td></tr>'
|
||||
for record in page_records
|
||||
)
|
||||
index = f"""<!doctype html>
|
||||
<html lang="ru">
|
||||
<head><meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1"><title>1C:ITS Static Archive</title><style>{STYLE}</style></head>
|
||||
<body><main class="wrap"><article class="doc"><h1>1C:ITS Static Archive</h1><p class="meta">Локальный статический просмотр нормализованных страниц.</p><table><thead><tr><th>Страница</th><th>URL</th></tr></thead><tbody>{rows}</tbody></table></article></main></body>
|
||||
</html>
|
||||
"""
|
||||
(output_dir / "index.html").write_text(index, encoding="utf-8")
|
||||
result = {
|
||||
"schema": "onec_its_static_site_manifest.v1",
|
||||
"output_dir": str(output_dir),
|
||||
"index": str(output_dir / "index.html"),
|
||||
"counts": {
|
||||
"pages": len(page_records),
|
||||
"media_files": len(list(static_media_dir.glob("*"))),
|
||||
"asset_files": len(list(static_assets_dir.glob("*"))) if static_assets_dir.exists() else 0,
|
||||
"asset_errors": len(asset_errors),
|
||||
},
|
||||
"pages": page_records,
|
||||
"assets": [{"url": url, "file": f"assets/{filename}"} for url, filename in sorted(asset_url_to_file.items())],
|
||||
"asset_errors": asset_errors,
|
||||
}
|
||||
(output_dir / "manifest.json").write_text(json.dumps(result, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
return result
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Build local static HTML viewer for normalized private 1C:ITS docs.")
|
||||
parser.add_argument("--manifest", type=Path, default=DEFAULT_MANIFEST)
|
||||
parser.add_argument("--normalized-dir", type=Path, default=DEFAULT_NORMALIZED_DIR)
|
||||
parser.add_argument("--raw-dir", type=Path, default=DEFAULT_RAW_DIR)
|
||||
parser.add_argument("--media-manifest", type=Path, default=DEFAULT_MEDIA_MANIFEST)
|
||||
parser.add_argument("--media-dir", type=Path, default=DEFAULT_MEDIA_DIR)
|
||||
parser.add_argument("--output-dir", type=Path, default=DEFAULT_OUTPUT_DIR)
|
||||
parser.add_argument("--skip-assets", action="store_true", help="Do not download and localize CSS/JS assets from raw HTML.")
|
||||
args = parser.parse_args()
|
||||
result = build_static_site(
|
||||
args.manifest,
|
||||
args.normalized_dir,
|
||||
args.raw_dir,
|
||||
args.media_manifest,
|
||||
args.media_dir,
|
||||
args.output_dir,
|
||||
download_external_assets=not args.skip_assets,
|
||||
)
|
||||
print(json.dumps({"counts": result["counts"], "index": result["index"]}, ensure_ascii=False, indent=2))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,114 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
DEFAULT_METADATA = ROOT / "plugins" / "1c" / "metadata" / "examples" / "metadata.example.json"
|
||||
DEFAULT_BSL_MODULES = ROOT / "plugins" / "1c" / "metadata" / "examples" / "bsl-modules.example.json"
|
||||
DEFAULT_SOURCE_DIR = ROOT / "plugins" / "1c" / "rag" / "sources"
|
||||
DEFAULT_CORPUS = ROOT / "plugins" / "1c" / "datasets" / "prepared" / "rag_corpus.jsonl"
|
||||
DEFAULT_MANIFEST = ROOT / "plugins" / "1c" / "datasets" / "prepared" / "rag_manifest.json"
|
||||
DEFAULT_INDEX = ROOT / "plugins" / "1c" / "datasets" / "prepared" / "rag_index.json"
|
||||
DEFAULT_VECTOR_INDEX = ROOT / "plugins" / "1c" / "datasets" / "prepared" / "rag_vector_index.sqlite"
|
||||
|
||||
|
||||
def run(command: list[str]) -> None:
|
||||
print(" ".join(command))
|
||||
result = subprocess.run(command, cwd=ROOT, text=True, check=False)
|
||||
if result.returncode != 0:
|
||||
raise SystemExit(result.returncode)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Build the 1C RAG knowledge base.")
|
||||
parser.add_argument("--metadata", type=Path, default=DEFAULT_METADATA)
|
||||
parser.add_argument("--bsl-modules", type=Path, help="Optional BSL module snapshot JSON.")
|
||||
parser.add_argument("--source-dir", type=Path, default=DEFAULT_SOURCE_DIR)
|
||||
parser.add_argument("--corpus", type=Path, default=DEFAULT_CORPUS)
|
||||
parser.add_argument("--manifest", type=Path, default=DEFAULT_MANIFEST)
|
||||
parser.add_argument("--index", type=Path, default=DEFAULT_INDEX)
|
||||
parser.add_argument("--vector-index", type=Path, default=DEFAULT_VECTOR_INDEX)
|
||||
parser.add_argument("--skip-vector-index", action="store_true")
|
||||
parser.add_argument(
|
||||
"--include-example-bsl",
|
||||
action="store_true",
|
||||
help="Use the bundled BSL example when --bsl-modules is not provided.",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
metadata_output = args.source_dir / f"{args.metadata.stem}.metadata.md"
|
||||
run([sys.executable, "scripts/validate_1c_metadata_snapshot.py", str(args.metadata)])
|
||||
run(
|
||||
[
|
||||
sys.executable,
|
||||
"scripts/convert_1c_metadata_to_rag.py",
|
||||
"--input",
|
||||
str(args.metadata),
|
||||
"--output",
|
||||
str(metadata_output),
|
||||
]
|
||||
)
|
||||
|
||||
bsl_modules = args.bsl_modules
|
||||
if bsl_modules is None and args.include_example_bsl:
|
||||
bsl_modules = DEFAULT_BSL_MODULES
|
||||
if bsl_modules:
|
||||
bsl_output = args.source_dir / f"{bsl_modules.stem}.bsl.md"
|
||||
run([sys.executable, "scripts/validate_1c_bsl_modules.py", str(bsl_modules)])
|
||||
run(
|
||||
[
|
||||
sys.executable,
|
||||
"scripts/convert_1c_bsl_modules_to_rag.py",
|
||||
"--input",
|
||||
str(bsl_modules),
|
||||
"--output",
|
||||
str(bsl_output),
|
||||
]
|
||||
)
|
||||
|
||||
run([sys.executable, "scripts/validate_1c_rag_sources.py", "--source-dir", str(args.source_dir)])
|
||||
run(
|
||||
[
|
||||
sys.executable,
|
||||
"scripts/prepare_1c_rag_corpus.py",
|
||||
"--source-dir",
|
||||
str(args.source_dir),
|
||||
"--output",
|
||||
str(args.corpus),
|
||||
"--manifest",
|
||||
str(args.manifest),
|
||||
]
|
||||
)
|
||||
run(
|
||||
[
|
||||
sys.executable,
|
||||
"scripts/build_1c_rag_index.py",
|
||||
"--corpus",
|
||||
str(args.corpus),
|
||||
"--output",
|
||||
str(args.index),
|
||||
]
|
||||
)
|
||||
if not args.skip_vector_index:
|
||||
run(
|
||||
[
|
||||
sys.executable,
|
||||
"scripts/build_1c_rag_vector_index.py",
|
||||
"--corpus",
|
||||
str(args.corpus),
|
||||
"--output",
|
||||
str(args.vector_index),
|
||||
]
|
||||
)
|
||||
print(f"Built 1C knowledge base: {args.index}")
|
||||
if not args.skip_vector_index:
|
||||
print(f"Built 1C vector index: {args.vector_index}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,465 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Build a read-projection metadata card from resolved XML object evidence."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import sys
|
||||
import xml.etree.ElementTree as ET
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
||||
from resolve_1c_object import load_json, physical_name, resolve_object # noqa: E402
|
||||
|
||||
|
||||
REFERENCE_MARKERS = ("Ref.",)
|
||||
|
||||
TYPE_PRESENTATION_RU = {
|
||||
"xs:string": "Строка",
|
||||
"xs:decimal": "Число",
|
||||
"xs:boolean": "Булево",
|
||||
"xs:dateTime": "Дата",
|
||||
"v8:UUID": "УникальныйИдентификатор",
|
||||
"cfg:AnyRef": "ЛюбаяСсылка",
|
||||
"cfg:AnyIBRef": "ЛюбаяСсылка",
|
||||
}
|
||||
|
||||
REFERENCE_PRESENTATION_RU = {
|
||||
"CatalogRef": "СправочникСсылка",
|
||||
"DocumentRef": "ДокументСсылка",
|
||||
"EnumRef": "ПеречислениеСсылка",
|
||||
"ChartOfAccountsRef": "ПланСчетовСсылка",
|
||||
"ChartOfCalculationTypesRef": "ПланВидовРасчетаСсылка",
|
||||
"ChartOfCharacteristicTypesRef": "ПланВидовХарактеристикСсылка",
|
||||
"BusinessProcessRef": "БизнесПроцессСсылка",
|
||||
"TaskRef": "ЗадачаСсылка",
|
||||
}
|
||||
|
||||
|
||||
def local_name(tag: str) -> str:
|
||||
return tag.rsplit("}", 1)[-1] if "}" in tag else tag
|
||||
|
||||
|
||||
def direct_child(node: ET.Element, name: str) -> ET.Element | None:
|
||||
return next((child for child in list(node) if local_name(child.tag) == name), None)
|
||||
|
||||
|
||||
def text_child(node: ET.Element | None, name: str) -> str | None:
|
||||
if node is None:
|
||||
return None
|
||||
child = direct_child(node, name)
|
||||
if child is None or child.text is None:
|
||||
return None
|
||||
return child.text.strip()
|
||||
|
||||
|
||||
def synonym(properties: ET.Element | None) -> str | None:
|
||||
if properties is None:
|
||||
return None
|
||||
syn = direct_child(properties, "Synonym")
|
||||
if syn is None:
|
||||
return None
|
||||
for node in syn.iter():
|
||||
if local_name(node.tag) == "content" and node.text:
|
||||
return node.text.strip()
|
||||
return None
|
||||
|
||||
|
||||
def value_types(properties: ET.Element | None) -> list[str]:
|
||||
if properties is None:
|
||||
return []
|
||||
type_node = direct_child(properties, "Type")
|
||||
if type_node is None:
|
||||
return []
|
||||
result = []
|
||||
for node in type_node.iter():
|
||||
if local_name(node.tag) == "Type" and node.text and ":" in node.text:
|
||||
result.append(node.text.strip())
|
||||
elif local_name(node.tag) == "TypeSet" and node.text and ":" in node.text:
|
||||
result.append(node.text.strip())
|
||||
return result
|
||||
|
||||
|
||||
def type_presentation_ru(value: str) -> str:
|
||||
if value in TYPE_PRESENTATION_RU:
|
||||
return TYPE_PRESENTATION_RU[value]
|
||||
if value.startswith("cfg:"):
|
||||
payload = value[4:]
|
||||
if "." in payload:
|
||||
family, name = payload.split(".", 1)
|
||||
prefix = REFERENCE_PRESENTATION_RU.get(family)
|
||||
if prefix:
|
||||
return f"{prefix}.{name}"
|
||||
return f"{family}.{name}"
|
||||
return value
|
||||
|
||||
|
||||
def route_kind(role: str) -> str:
|
||||
if role == "Fld":
|
||||
return "field"
|
||||
if role in {"VT", "LineNo"}:
|
||||
return "structural"
|
||||
return "table"
|
||||
|
||||
|
||||
def storage_routes(index: dict[str, Any], guid: str | None) -> list[dict[str, Any]]:
|
||||
if not guid:
|
||||
return []
|
||||
item = (index.get("objects") or {}).get(guid.lower()) or {}
|
||||
result = []
|
||||
for route in item.get("dbnames") or []:
|
||||
copy = {
|
||||
"guid": guid,
|
||||
"storage_role": route.get("storage_role"),
|
||||
"sql_number": route.get("sql_number"),
|
||||
"source": route.get("source_file") or "DBNames",
|
||||
"route_kind": route_kind(str(route.get("storage_role") or "")),
|
||||
"physical_name_candidate": physical_name(route),
|
||||
}
|
||||
result.append(copy)
|
||||
return result
|
||||
|
||||
|
||||
def value_type_payload(types: list[str]) -> dict[str, Any] | None:
|
||||
if not types:
|
||||
return None
|
||||
return {
|
||||
"types": types,
|
||||
"presentation": {
|
||||
"default_language": "ru",
|
||||
"ru": [type_presentation_ru(item) for item in types],
|
||||
},
|
||||
"qualifiers": {},
|
||||
"is_composite": len(types) > 1,
|
||||
}
|
||||
|
||||
|
||||
def field_columns(routes: list[dict[str, Any]], types: list[str]) -> list[dict[str, Any]]:
|
||||
fld = next((route for route in routes if route.get("storage_role") == "Fld" and route.get("sql_number") is not None), None)
|
||||
if not fld:
|
||||
return [{"status": "no_storage_route"}]
|
||||
base = f"_Fld{fld['sql_number']}"
|
||||
if len(types) > 1:
|
||||
return [
|
||||
{"column": f"{base}_TYPE", "reason": "composite value discriminator", "value_types": types},
|
||||
{"column": f"{base}_S", "reason": "composite string value", "value_types": types},
|
||||
{"column": f"{base}_N", "reason": "composite numeric value", "value_types": types},
|
||||
{"column": f"{base}_L", "reason": "composite boolean value", "value_types": types},
|
||||
{"column": f"{base}_T", "reason": "composite datetime value", "value_types": types},
|
||||
{"column": f"{base}_RTRef", "reason": "composite reference type id", "value_types": types},
|
||||
{"column": f"{base}_RRRef", "reason": "composite reference value", "value_types": types},
|
||||
]
|
||||
value_type = types[0] if types else None
|
||||
if value_type in {"cfg:AnyRef", "cfg:AnyIBRef"}:
|
||||
return [
|
||||
{"column": f"{base}_TYPE", "reason": "any reference discriminator", "value_types": types},
|
||||
{"column": f"{base}_RTRef", "reason": "any reference type id", "value_types": types},
|
||||
{"column": f"{base}_RRRef", "reason": "any reference value", "value_types": types},
|
||||
]
|
||||
if value_type and any(marker in value_type for marker in REFERENCE_MARKERS):
|
||||
return [{"column": f"{base}RRef", "reason": "single 1C reference type", "value_type": value_type}]
|
||||
return [{"column": base, "reason": "single primitive value", "value_type": value_type}]
|
||||
|
||||
|
||||
def extension_name_from_path(path: str | None) -> str | None:
|
||||
parts = str(path or "").replace("/", "\\").split("\\")
|
||||
lowered = [part.casefold() for part in parts]
|
||||
if "расширения" in lowered:
|
||||
index = lowered.index("расширения")
|
||||
if index + 1 < len(parts):
|
||||
return parts[index + 1]
|
||||
if "extensions" in lowered:
|
||||
index = lowered.index("extensions")
|
||||
if index + 1 < len(parts):
|
||||
return parts[index + 1]
|
||||
return None
|
||||
|
||||
|
||||
def metadata_item(
|
||||
node: ET.Element,
|
||||
*,
|
||||
index: dict[str, Any],
|
||||
category: str,
|
||||
parent_category: str,
|
||||
parent_name: str,
|
||||
parent_uuid: str,
|
||||
tabular_section_name: str | None = None,
|
||||
tabular_section_uuid: str | None = None,
|
||||
record_index: int,
|
||||
) -> dict[str, Any]:
|
||||
properties = direct_child(node, "Properties")
|
||||
guid = (node.get("uuid") or "").lower()
|
||||
name = text_child(properties, "Name")
|
||||
types = value_types(properties)
|
||||
routes = storage_routes(index, guid)
|
||||
item = {
|
||||
"category": category,
|
||||
"name": name,
|
||||
"synonym": synonym(properties),
|
||||
"uuid": guid,
|
||||
"value_type": value_type_payload(types),
|
||||
"parent_category": parent_category,
|
||||
"parent_name": parent_name,
|
||||
"parent_uuid": parent_uuid,
|
||||
"record_index": record_index,
|
||||
"evidence": {"name": bool(name), "synonym": bool(synonym(properties)), "uuid": bool(guid)},
|
||||
"storage_routes": routes,
|
||||
"storage_route_count": len(routes),
|
||||
"physical_columns": field_columns(routes, types) if category == "Attribute" else [{"status": "no_value_type"}],
|
||||
}
|
||||
object_belonging = text_child(properties, "ObjectBelonging")
|
||||
extended_object = text_child(properties, "ExtendedConfigurationObject")
|
||||
if object_belonging:
|
||||
item["object_belonging"] = object_belonging
|
||||
if extended_object:
|
||||
item["extended_configuration_object"] = extended_object.lower()
|
||||
if tabular_section_name:
|
||||
item["tabular_section_name"] = tabular_section_name
|
||||
item["tabular_section_uuid"] = tabular_section_uuid
|
||||
return item
|
||||
|
||||
|
||||
def object_node(root: ET.Element, kind: str) -> ET.Element:
|
||||
for node in root.iter():
|
||||
if local_name(node.tag) == kind:
|
||||
return node
|
||||
raise SystemExit(f"XML object node not found: {kind}")
|
||||
|
||||
|
||||
def merge_attribute_overlay(base: list[dict[str, Any]], overlay: dict[str, Any]) -> None:
|
||||
extended_uuid = overlay.get("extended_configuration_object")
|
||||
target = None
|
||||
if extended_uuid:
|
||||
target = next((item for item in base if item.get("uuid") == extended_uuid), None)
|
||||
if target is None and overlay.get("name"):
|
||||
target = next((item for item in base if item.get("name") == overlay.get("name")), None)
|
||||
if target is None:
|
||||
base.append(overlay)
|
||||
return
|
||||
|
||||
record = {
|
||||
"source": overlay.get("source"),
|
||||
"extension_name": overlay.get("extension_name"),
|
||||
"path": overlay.get("source_path"),
|
||||
"uuid": overlay.get("uuid"),
|
||||
"object_belonging": overlay.get("object_belonging"),
|
||||
"extended_configuration_object": overlay.get("extended_configuration_object"),
|
||||
"value_type": overlay.get("value_type"),
|
||||
"synonym": overlay.get("synonym"),
|
||||
}
|
||||
target.setdefault("extension_overrides", []).append(record)
|
||||
target["effective_source"] = "base+extension"
|
||||
if overlay.get("value_type"):
|
||||
target["base_value_type"] = target.get("base_value_type") or target.get("value_type")
|
||||
target["value_type"] = overlay["value_type"]
|
||||
target["physical_columns"] = field_columns(target.get("storage_routes") or [], overlay["value_type"].get("types") or [])
|
||||
if overlay.get("synonym"):
|
||||
target["synonym"] = overlay["synonym"]
|
||||
|
||||
|
||||
def apply_object_overlay(
|
||||
*,
|
||||
index: dict[str, Any],
|
||||
overlay: dict[str, Any],
|
||||
kind: str,
|
||||
attributes: list[dict[str, Any]],
|
||||
tabular_sections: list[dict[str, Any]],
|
||||
tabular_section_attributes: list[dict[str, Any]],
|
||||
) -> dict[str, int]:
|
||||
path = Path(str(overlay.get("path") or ""))
|
||||
if not path.is_file():
|
||||
return {"missing": 1, "attributes_added": 0, "attributes_changed": 0}
|
||||
root = ET.parse(path).getroot()
|
||||
node = object_node(root, kind)
|
||||
children = direct_child(node, "ChildObjects")
|
||||
if children is None:
|
||||
return {"missing": 0, "attributes_added": 0, "attributes_changed": 0}
|
||||
|
||||
added = 0
|
||||
changed = 0
|
||||
extension_name = extension_name_from_path(str(path))
|
||||
for child in list(children):
|
||||
if local_name(child.tag) != "Attribute":
|
||||
continue
|
||||
before = len(attributes)
|
||||
item = metadata_item(
|
||||
child,
|
||||
index=index,
|
||||
category="Attribute",
|
||||
parent_category=kind,
|
||||
parent_name=str(overlay.get("name") or ""),
|
||||
parent_uuid=str(overlay.get("guid") or ""),
|
||||
record_index=len(attributes),
|
||||
)
|
||||
item["source"] = "extension"
|
||||
item["extension_name"] = extension_name
|
||||
item["source_path"] = str(path)
|
||||
merge_attribute_overlay(attributes, item)
|
||||
if len(attributes) > before:
|
||||
added += 1
|
||||
else:
|
||||
changed += 1
|
||||
return {"missing": 0, "attributes_added": added, "attributes_changed": changed}
|
||||
|
||||
|
||||
def build_metadata(index: dict[str, Any], *, kind: str, name: str) -> dict[str, Any]:
|
||||
resolution = resolve_object(index, kind=kind, name=name, limit=50)
|
||||
canonical = resolution.get("canonical")
|
||||
if not canonical:
|
||||
raise SystemExit(f"Object not found: {kind}.{name}")
|
||||
xml_path = Path(str(canonical.get("path") or ""))
|
||||
if not xml_path.is_file():
|
||||
raise SystemExit(f"Object XML file not found: {xml_path}")
|
||||
|
||||
root = ET.parse(xml_path).getroot()
|
||||
node = object_node(root, str(canonical["kind"]))
|
||||
properties = direct_child(node, "Properties")
|
||||
object_name = text_child(properties, "Name") or str(canonical["name"])
|
||||
object_uuid = str(canonical["guid"]).lower()
|
||||
main_routes = storage_routes(index, object_uuid)
|
||||
main_table = next((physical_name(route) for route in main_routes if route.get("storage_role") == canonical["kind"]), None)
|
||||
if not main_table:
|
||||
main_table = next((route.get("physical_name_candidate") for route in main_routes if route.get("route_kind") == "table"), None)
|
||||
|
||||
attributes = []
|
||||
tabular_sections = []
|
||||
tabular_section_attributes = []
|
||||
children = direct_child(node, "ChildObjects")
|
||||
if children is not None:
|
||||
attr_index = 0
|
||||
ts_index = 0
|
||||
for child in list(children):
|
||||
child_kind = local_name(child.tag)
|
||||
if child_kind == "Attribute":
|
||||
attributes.append(
|
||||
{
|
||||
**metadata_item(
|
||||
child,
|
||||
index=index,
|
||||
category="Attribute",
|
||||
parent_category=str(canonical["kind"]),
|
||||
parent_name=object_name,
|
||||
parent_uuid=object_uuid,
|
||||
record_index=attr_index,
|
||||
),
|
||||
"source": "base",
|
||||
}
|
||||
)
|
||||
attr_index += 1
|
||||
elif child_kind == "TabularSection":
|
||||
section = metadata_item(
|
||||
child,
|
||||
index=index,
|
||||
category="TabularSection",
|
||||
parent_category=str(canonical["kind"]),
|
||||
parent_name=object_name,
|
||||
parent_uuid=object_uuid,
|
||||
record_index=ts_index,
|
||||
)
|
||||
vt_route = next((route for route in section["storage_routes"] if route.get("storage_role") == "VT"), None)
|
||||
line_numbers = [route.get("sql_number") for route in section["storage_routes"] if route.get("storage_role") == "LineNo"]
|
||||
if main_table and vt_route and vt_route.get("sql_number") is not None:
|
||||
section["physical_tables"] = [
|
||||
{
|
||||
"table": f"{main_table}_VT{vt_route['sql_number']}",
|
||||
"reason": "tabular section VT route under object table",
|
||||
"vt_sql_number": vt_route["sql_number"],
|
||||
"line_no_sql_numbers": line_numbers,
|
||||
}
|
||||
]
|
||||
tabular_sections.append(section)
|
||||
section_children = direct_child(child, "ChildObjects")
|
||||
if section_children is not None:
|
||||
for record_index, section_child in enumerate([item for item in list(section_children) if local_name(item.tag) == "Attribute"]):
|
||||
attr = metadata_item(
|
||||
section_child,
|
||||
index=index,
|
||||
category="Attribute",
|
||||
parent_category="TabularSection",
|
||||
parent_name=section["name"],
|
||||
parent_uuid=section["uuid"],
|
||||
tabular_section_name=section["name"],
|
||||
tabular_section_uuid=section["uuid"],
|
||||
record_index=record_index,
|
||||
)
|
||||
attr["parent_physical_tables"] = section.get("physical_tables") or []
|
||||
attr["source"] = "base"
|
||||
tabular_section_attributes.append(attr)
|
||||
ts_index += 1
|
||||
|
||||
overlay_stats = []
|
||||
for overlay in resolution.get("extension_overlays") or []:
|
||||
stats = apply_object_overlay(
|
||||
index=index,
|
||||
overlay=overlay,
|
||||
kind=str(canonical["kind"]),
|
||||
attributes=attributes,
|
||||
tabular_sections=tabular_sections,
|
||||
tabular_section_attributes=tabular_section_attributes,
|
||||
)
|
||||
overlay_stats.append(
|
||||
{
|
||||
"extension_name": extension_name_from_path(overlay.get("path")),
|
||||
"path": overlay.get("path"),
|
||||
**stats,
|
||||
}
|
||||
)
|
||||
|
||||
return {
|
||||
"schema": "onec_structured_metadata_from_resolved_xml.v1",
|
||||
"kind": canonical["kind"],
|
||||
"xml_file": str(xml_path),
|
||||
"identity": {
|
||||
"guid": object_uuid,
|
||||
"name": object_name,
|
||||
"synonyms": {"ru": synonym(properties)} if synonym(properties) else {},
|
||||
},
|
||||
"resolution": {"schema": resolution.get("schema"), "canonical": canonical, "summary": resolution.get("summary")},
|
||||
"effective_metadata": {
|
||||
"base_path": str(xml_path),
|
||||
"extension_overlays_applied": overlay_stats,
|
||||
},
|
||||
"attributes": attributes,
|
||||
"tabular_sections": tabular_sections,
|
||||
"tabular_section_attributes": tabular_section_attributes,
|
||||
"dimensions": [],
|
||||
"resources": [],
|
||||
"forms": [],
|
||||
"templates": [],
|
||||
"commands": [],
|
||||
"addressing_attributes": [],
|
||||
"accounting_flags": [],
|
||||
"columns": [],
|
||||
"enum_values": [],
|
||||
"object_storage_routes": main_routes,
|
||||
"storage_route_summary": {
|
||||
"metadata_items_with_routes": sum(1 for item in attributes + tabular_sections + tabular_section_attributes if item.get("storage_routes")),
|
||||
"object_routes": len(main_routes),
|
||||
},
|
||||
"counts": {
|
||||
"attributes": len(attributes),
|
||||
"tabular_sections": len(tabular_sections),
|
||||
"tabular_section_attributes": len(tabular_section_attributes),
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Build metadata card from resolved 1C object XML.")
|
||||
parser.add_argument("--index", type=Path, required=True)
|
||||
parser.add_argument("--kind", required=True)
|
||||
parser.add_argument("--name", required=True)
|
||||
parser.add_argument("--output", type=Path, required=True)
|
||||
args = parser.parse_args()
|
||||
|
||||
result = build_metadata(load_json(args.index), kind=args.kind, name=args.name)
|
||||
args.output.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.output.write_text(json.dumps(result, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
print(json.dumps({"output": str(args.output), "counts": result["counts"]}, ensure_ascii=False))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,156 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
def read_json(path: Path) -> dict[str, Any]:
|
||||
return json.loads(path.read_text(encoding="utf-8"))
|
||||
|
||||
|
||||
def rule_status(rule: dict[str, Any]) -> str:
|
||||
confidence = str(rule.get("confidence") or "none")
|
||||
evidence = rule.get("evidence") if isinstance(rule.get("evidence"), dict) else {}
|
||||
ok = evidence.get("ok")
|
||||
total = evidence.get("total")
|
||||
if confidence == "high" and isinstance(ok, int) and isinstance(total, int) and total > 0 and ok == total:
|
||||
return "verified_read"
|
||||
if confidence == "high" and int(evidence.get("distinct_rectangular_samples") or 0) > 0 and int(evidence.get("samples") or 0) > 0:
|
||||
return "verified_read"
|
||||
if confidence in {"medium", "high"}:
|
||||
return "candidate_read"
|
||||
return "needs_more_evidence"
|
||||
|
||||
|
||||
def write_status(status: str) -> str:
|
||||
if status == "verified_read":
|
||||
return "blocked_until_roundtrip"
|
||||
return "blocked_until_verified_read"
|
||||
|
||||
|
||||
def merge_named_range_analysis_rules(discovery: dict[str, Any], named_range_analysis: dict[str, Any] | None) -> dict[str, Any]:
|
||||
if not named_range_analysis:
|
||||
return discovery
|
||||
merged_rules = [rule for rule in discovery.get("rules") or [] if isinstance(rule, dict)]
|
||||
by_target = {str(rule.get("target") or ""): index for index, rule in enumerate(merged_rules)}
|
||||
for rule in named_range_analysis.get("rules") or []:
|
||||
if not isinstance(rule, dict) or not str(rule.get("target") or "").startswith("moxel.named_range."):
|
||||
continue
|
||||
target = str(rule.get("target") or "")
|
||||
promoted = {
|
||||
"id": rule.get("id") or target,
|
||||
"target": target,
|
||||
"expression": rule.get("expression"),
|
||||
"raw_scalar_indexes": rule.get("raw_scalar_indexes"),
|
||||
"confidence": rule.get("confidence") or "none",
|
||||
"evidence": rule.get("evidence") or {},
|
||||
"source": "named_range_analysis",
|
||||
}
|
||||
if target in by_target:
|
||||
existing = merged_rules[by_target[target]]
|
||||
confidence_order = {"none": 0, "low": 1, "medium": 2, "high": 3}
|
||||
if confidence_order.get(str(promoted.get("confidence")), 0) >= confidence_order.get(str(existing.get("confidence")), 0):
|
||||
merged_rules[by_target[target]] = {**existing, **promoted}
|
||||
else:
|
||||
by_target[target] = len(merged_rules)
|
||||
merged_rules.append(promoted)
|
||||
return {**discovery, "rules": merged_rules}
|
||||
|
||||
|
||||
def build_registry(discovery: dict[str, Any], sources: list[str], named_range_analysis: dict[str, Any] | None = None) -> dict[str, Any]:
|
||||
discovery = merge_named_range_analysis_rules(discovery, named_range_analysis)
|
||||
registry_rules = []
|
||||
for index, rule in enumerate(discovery.get("rules") or [], start=1):
|
||||
if not isinstance(rule, dict):
|
||||
continue
|
||||
status = rule_status(rule)
|
||||
registry_rules.append(
|
||||
{
|
||||
"id": rule.get("id") or f"moxel_rule_{index}",
|
||||
"target": rule.get("target"),
|
||||
"expression": rule.get("expression"),
|
||||
"raw_scalar_indexes": rule.get("raw_scalar_indexes"),
|
||||
"confidence": rule.get("confidence") or "none",
|
||||
"read_status": status,
|
||||
"write_status": write_status(status),
|
||||
"evidence": rule.get("evidence") or {},
|
||||
"source_rule": rule,
|
||||
}
|
||||
)
|
||||
return {
|
||||
"schema": "codex_1c_moxel_schema_registry.v1",
|
||||
"generated_at": datetime.now(UTC).isoformat(timespec="seconds").replace("+00:00", "Z"),
|
||||
"sources": sources,
|
||||
"policy": {
|
||||
"read_use": "Only verified_read rules may be used as decoder behavior without additional diagnostics.",
|
||||
"write_use": "All MOXCEL write rules are blocked until a disposable-base round-trip proves exact behavior.",
|
||||
},
|
||||
"rules": registry_rules,
|
||||
"counts": {
|
||||
"rules": len(registry_rules),
|
||||
"verified_read": sum(1 for rule in registry_rules if rule.get("read_status") == "verified_read"),
|
||||
"candidate_read": sum(1 for rule in registry_rules if rule.get("read_status") == "candidate_read"),
|
||||
"write_enabled": sum(1 for rule in registry_rules if rule.get("write_status") == "verified_roundtrip"),
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def render_markdown(registry: dict[str, Any]) -> str:
|
||||
lines = ["# 1C MOXCEL Schema Registry", ""]
|
||||
counts = registry.get("counts") or {}
|
||||
lines.append(f"- Rules: `{counts.get('rules')}`")
|
||||
lines.append(f"- Verified read: `{counts.get('verified_read')}`")
|
||||
lines.append(f"- Candidate read: `{counts.get('candidate_read')}`")
|
||||
lines.append(f"- Write enabled: `{counts.get('write_enabled')}`")
|
||||
lines.append("")
|
||||
lines.append("| Rule | Target | Read | Write | Confidence |")
|
||||
lines.append("| --- | --- | --- | --- | --- |")
|
||||
for rule in registry.get("rules") or []:
|
||||
lines.append(
|
||||
f"| `{rule.get('id')}` | `{rule.get('target')}` | `{rule.get('read_status')}` | "
|
||||
f"`{rule.get('write_status')}` | `{rule.get('confidence')}` |"
|
||||
)
|
||||
lines.append("")
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Build the stable 1C MOXCEL schema registry from discovery reports.")
|
||||
parser.add_argument("--discovery", action="append", required=True, help="Discovery JSON. Repeatable; rules are merged in order.")
|
||||
parser.add_argument("--named-range-analysis", help="Optional named range rule analysis JSON.")
|
||||
parser.add_argument("--output-json", default="plugins/1c/metadata/moxel-schema-registry.json")
|
||||
parser.add_argument("--output-markdown", default="reports/1c-template-baselines/moxel-schema-registry.md")
|
||||
args = parser.parse_args()
|
||||
|
||||
discoveries = [read_json(Path(path)) for path in args.discovery]
|
||||
merged = {"rules": []}
|
||||
seen: set[tuple[str, str]] = set()
|
||||
for discovery in discoveries:
|
||||
for rule in discovery.get("rules") or []:
|
||||
if not isinstance(rule, dict):
|
||||
continue
|
||||
key = (str(rule.get("id") or ""), str(rule.get("target") or ""))
|
||||
if key in seen:
|
||||
continue
|
||||
seen.add(key)
|
||||
merged["rules"].append(rule)
|
||||
named_range_analysis = read_json(Path(args.named_range_analysis)) if args.named_range_analysis else None
|
||||
sources = list(args.discovery)
|
||||
if args.named_range_analysis:
|
||||
sources.append(args.named_range_analysis)
|
||||
registry = build_registry(merged, sources, named_range_analysis)
|
||||
json_path = Path(args.output_json)
|
||||
md_path = Path(args.output_markdown)
|
||||
json_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
md_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
json_path.write_text(json.dumps(registry, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
md_path.write_text(render_markdown(registry), encoding="utf-8")
|
||||
print(json.dumps({"status": "ok", "json": str(json_path), "markdown": str(md_path), "counts": registry["counts"]}, ensure_ascii=False, indent=2))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,27 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
from pathlib import Path
|
||||
|
||||
from common import build_lexical_index, read_jsonl, write_json
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
DEFAULT_CORPUS = ROOT / "plugins" / "1c" / "datasets" / "prepared" / "rag_corpus.jsonl"
|
||||
DEFAULT_INDEX = ROOT / "plugins" / "1c" / "datasets" / "prepared" / "rag_index.json"
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Build a simple lexical RAG index for the 1C corpus.")
|
||||
parser.add_argument("--corpus", type=Path, default=DEFAULT_CORPUS)
|
||||
parser.add_argument("--output", type=Path, default=DEFAULT_INDEX)
|
||||
args = parser.parse_args()
|
||||
|
||||
records = read_jsonl(args.corpus)
|
||||
index = build_lexical_index(records)
|
||||
write_json(args.output, index)
|
||||
print(f"Wrote index with {index['doc_count']} document chunk(s) to {args.output}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,207 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import sqlite3
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
from common import corpus_content_hash, pack_float_vector, read_jsonl
|
||||
from rag_embedding_providers import LOCAL_HASHING_MODEL, LOCAL_HASHING_PROVIDER, embed_texts, provider_metadata
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
DEFAULT_CORPUS = ROOT / "plugins" / "1c" / "datasets" / "prepared" / "rag_corpus.jsonl"
|
||||
DEFAULT_OUTPUT = ROOT / "plugins" / "1c" / "datasets" / "prepared" / "rag_vector_index.sqlite"
|
||||
SCHEMA_VERSION = 1
|
||||
DEFAULT_EMBEDDING_MODEL = LOCAL_HASHING_MODEL
|
||||
|
||||
|
||||
def connect_index(path: Path) -> sqlite3.Connection:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
conn = sqlite3.connect(path)
|
||||
conn.execute("PRAGMA journal_mode=WAL")
|
||||
conn.execute("PRAGMA synchronous=NORMAL")
|
||||
return conn
|
||||
|
||||
|
||||
def reset_schema(conn: sqlite3.Connection) -> None:
|
||||
conn.executescript(
|
||||
"""
|
||||
DROP TABLE IF EXISTS vector_documents;
|
||||
DROP TABLE IF EXISTS vector_meta;
|
||||
|
||||
CREATE TABLE vector_meta (
|
||||
key TEXT PRIMARY KEY,
|
||||
value TEXT NOT NULL
|
||||
);
|
||||
|
||||
CREATE TABLE vector_documents (
|
||||
id TEXT PRIMARY KEY,
|
||||
document_id TEXT,
|
||||
source_path TEXT,
|
||||
source_type TEXT,
|
||||
title TEXT,
|
||||
chunk_index INTEGER,
|
||||
content TEXT NOT NULL,
|
||||
metadata_json TEXT NOT NULL,
|
||||
vector BLOB NOT NULL
|
||||
);
|
||||
|
||||
CREATE INDEX idx_vector_documents_source_type ON vector_documents(source_type);
|
||||
CREATE INDEX idx_vector_documents_source_path ON vector_documents(source_path);
|
||||
"""
|
||||
)
|
||||
|
||||
|
||||
def write_meta(conn: sqlite3.Connection, metadata: dict[str, object]) -> None:
|
||||
conn.executemany(
|
||||
"INSERT INTO vector_meta(key, value) VALUES(?, ?)",
|
||||
[(key, json.dumps(value, ensure_ascii=False, sort_keys=True)) for key, value in metadata.items()],
|
||||
)
|
||||
|
||||
|
||||
def document_embedding_text(record: dict) -> str:
|
||||
title = str(record.get("title") or "").strip()
|
||||
content = str(record.get("content") or "").strip()
|
||||
metadata = record.get("metadata") if isinstance(record.get("metadata"), dict) else {}
|
||||
headings = metadata.get("headings") if isinstance(metadata.get("headings"), list) else []
|
||||
heading_text = "\n".join(str(item) for item in headings if str(item).strip())
|
||||
return "\n\n".join(part for part in (title, heading_text, content) if part)
|
||||
|
||||
|
||||
def batched(items: list[dict], size: int) -> list[list[dict]]:
|
||||
return [items[index : index + size] for index in range(0, len(items), size)]
|
||||
|
||||
|
||||
def build_vector_index(
|
||||
corpus_path: Path,
|
||||
output_path: Path,
|
||||
*,
|
||||
dimensions: int,
|
||||
embedding_model: str,
|
||||
embedding_provider: str = LOCAL_HASHING_PROVIDER,
|
||||
embedding_base_url: str = "",
|
||||
embedding_api_key_env: str = "OPENAI_API_KEY",
|
||||
batch_size: int = 16,
|
||||
) -> dict:
|
||||
records = read_jsonl(corpus_path)
|
||||
corpus_hash = corpus_content_hash(records)
|
||||
if not records:
|
||||
inferred_dimensions = dimensions
|
||||
else:
|
||||
sample_vector = embed_texts(
|
||||
[document_embedding_text(records[0])],
|
||||
provider=embedding_provider,
|
||||
model=embedding_model,
|
||||
dimensions=dimensions,
|
||||
base_url=embedding_base_url,
|
||||
api_key_env=embedding_api_key_env,
|
||||
)[0]
|
||||
inferred_dimensions = len(sample_vector)
|
||||
conn = connect_index(output_path)
|
||||
try:
|
||||
with conn:
|
||||
reset_schema(conn)
|
||||
embedding_meta = provider_metadata(
|
||||
provider=embedding_provider,
|
||||
model=embedding_model,
|
||||
dimensions=inferred_dimensions,
|
||||
base_url=embedding_base_url,
|
||||
)
|
||||
write_meta(
|
||||
conn,
|
||||
{
|
||||
"schema": "onec_rag_vector_index.v1",
|
||||
"schema_version": SCHEMA_VERSION,
|
||||
"type": "sqlite-vector-scan",
|
||||
**embedding_meta,
|
||||
"corpus_path": str(corpus_path),
|
||||
"corpus_hash": corpus_hash,
|
||||
"doc_count": len(records),
|
||||
"built_at": datetime.now(timezone.utc).isoformat(),
|
||||
},
|
||||
)
|
||||
rows = []
|
||||
for batch in batched(records, max(int(batch_size or 1), 1)):
|
||||
texts = [document_embedding_text(record) for record in batch]
|
||||
vectors = embed_texts(
|
||||
texts,
|
||||
provider=embedding_provider,
|
||||
model=embedding_model,
|
||||
dimensions=inferred_dimensions,
|
||||
base_url=embedding_base_url,
|
||||
api_key_env=embedding_api_key_env,
|
||||
)
|
||||
for record, vector in zip(batch, vectors):
|
||||
if len(vector) != inferred_dimensions:
|
||||
raise ValueError(f"Embedding dimensions changed within the build: {len(vector)} != {inferred_dimensions}")
|
||||
metadata = record.get("metadata") if isinstance(record.get("metadata"), dict) else {}
|
||||
rows.append(
|
||||
(
|
||||
str(record.get("id") or ""),
|
||||
str(record.get("document_id") or ""),
|
||||
str(record.get("source_path") or ""),
|
||||
str(metadata.get("source_type") or ""),
|
||||
str(record.get("title") or ""),
|
||||
int(record.get("chunk_index") or 0),
|
||||
str(record.get("content") or ""),
|
||||
json.dumps(metadata, ensure_ascii=False, sort_keys=True),
|
||||
pack_float_vector(vector),
|
||||
)
|
||||
)
|
||||
conn.executemany(
|
||||
"""
|
||||
INSERT INTO vector_documents(
|
||||
id, document_id, source_path, source_type, title, chunk_index,
|
||||
content, metadata_json, vector
|
||||
)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
""",
|
||||
rows,
|
||||
)
|
||||
finally:
|
||||
conn.close()
|
||||
return {
|
||||
"status": "ok",
|
||||
"index": str(output_path),
|
||||
"doc_count": len(records),
|
||||
"corpus_hash": corpus_hash,
|
||||
"embedding_provider": provider_metadata(provider=embedding_provider, model=embedding_model, dimensions=inferred_dimensions, base_url=embedding_base_url)["embedding_provider"],
|
||||
"embedding_model": embedding_model,
|
||||
"embedding_dimensions": inferred_dimensions,
|
||||
}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Build a local SQLite vector index for the 1C RAG corpus.")
|
||||
parser.add_argument("--corpus", type=Path, default=DEFAULT_CORPUS)
|
||||
parser.add_argument("--output", type=Path, default=DEFAULT_OUTPUT)
|
||||
parser.add_argument("--dimensions", type=int, default=384)
|
||||
parser.add_argument("--embedding-provider", default=LOCAL_HASHING_PROVIDER, choices=[LOCAL_HASHING_PROVIDER, "openai-compatible"])
|
||||
parser.add_argument("--embedding-model", default=DEFAULT_EMBEDDING_MODEL)
|
||||
parser.add_argument("--embedding-base-url", default="")
|
||||
parser.add_argument("--embedding-api-key-env", default="OPENAI_API_KEY")
|
||||
parser.add_argument("--batch-size", type=int, default=16)
|
||||
parser.add_argument("--json", action="store_true")
|
||||
args = parser.parse_args()
|
||||
|
||||
result = build_vector_index(
|
||||
args.corpus,
|
||||
args.output,
|
||||
dimensions=args.dimensions,
|
||||
embedding_model=args.embedding_model,
|
||||
embedding_provider=args.embedding_provider,
|
||||
embedding_base_url=args.embedding_base_url,
|
||||
embedding_api_key_env=args.embedding_api_key_env,
|
||||
batch_size=args.batch_size,
|
||||
)
|
||||
if args.json:
|
||||
print(json.dumps(result, ensure_ascii=False, indent=2))
|
||||
else:
|
||||
print(f"Wrote vector index with {result['doc_count']} document chunk(s) to {args.output}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,325 @@
|
||||
param(
|
||||
[string]$Server = $env:ONEC_SQL_SERVER,
|
||||
[string]$Database = $env:ONEC_SQL_DATABASE,
|
||||
[string]$User = $env:ONEC_SQL_USER,
|
||||
[string]$Password = $env:ONEC_SQL_PASSWORD,
|
||||
[string]$OutputDir = "reports\1c-sql\saved-state-object-report",
|
||||
[string]$BaseMetadataDir = "reports\1c-sql\upo\structured-metadata-all-kinds",
|
||||
[string]$ExtensionGuidIndex = "reports\1c-sql\upo\xml-guid-index-extensions.json",
|
||||
[string]$ExtensionManifestSummary = "reports\1c-sql\upo\extension-manifest-xml-part-summary.json",
|
||||
[string]$ConfigCASAllDir = "reports\1c-sql\upo\ConfigCAS-all",
|
||||
[string]$Python = "python",
|
||||
[string]$MarkdownOutput = "",
|
||||
[int]$MarkdownMaxDiffLines = 80,
|
||||
[switch]$SkipMarkdown
|
||||
)
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
if (-not $Server) { throw "Server is required. Use -Server or ONEC_SQL_SERVER." }
|
||||
if (-not $Database) { throw "Database is required. Use -Database or ONEC_SQL_DATABASE." }
|
||||
if (-not $User) { throw "User is required. Use -User or ONEC_SQL_USER." }
|
||||
if (-not $Password) { throw "Password is required. Use -Password or ONEC_SQL_PASSWORD." }
|
||||
|
||||
$repoRoot = (Resolve-Path ".").ProviderPath
|
||||
$resolvedOutput = [System.IO.Path]::GetFullPath($OutputDir)
|
||||
New-Item -ItemType Directory -Force -Path $resolvedOutput | Out-Null
|
||||
|
||||
$comparisonPath = Join-Path $resolvedOutput "saved-state-object-comparison.json"
|
||||
$detailPath = Join-Path $resolvedOutput "saved-state-object-details.json"
|
||||
$checkPath = Join-Path $resolvedOutput "saved-state-object-report-check.json"
|
||||
$configSaveDir = Join-Path $resolvedOutput "ConfigSave"
|
||||
$configCASSaveDir = Join-Path $resolvedOutput "ConfigCASSave"
|
||||
$activeConfigDir = Join-Path $resolvedOutput "ActiveConfig"
|
||||
$activeConfigCASDir = Join-Path $resolvedOutput "ActiveConfigCAS"
|
||||
|
||||
foreach ($dir in @($configSaveDir, $configCASSaveDir, $activeConfigDir, $activeConfigCASDir)) {
|
||||
New-Item -ItemType Directory -Force -Path $dir | Out-Null
|
||||
}
|
||||
|
||||
function Invoke-Step {
|
||||
param(
|
||||
[string]$Name,
|
||||
[scriptblock]$Script
|
||||
)
|
||||
$started = Get-Date
|
||||
& $Script
|
||||
[pscustomobject]@{
|
||||
name = $Name
|
||||
started_at = $started.ToString("o")
|
||||
finished_at = (Get-Date).ToString("o")
|
||||
passed = $true
|
||||
}
|
||||
}
|
||||
|
||||
function Read-JsonFile {
|
||||
param([string]$Path)
|
||||
return Get-Content -LiteralPath $Path -Raw -Encoding UTF8 | ConvertFrom-Json
|
||||
}
|
||||
|
||||
function Export-TableFiles {
|
||||
param(
|
||||
[string]$Table,
|
||||
[string]$TargetDir,
|
||||
[string[]]$FileName
|
||||
)
|
||||
$args = @(
|
||||
"-NoProfile", "-ExecutionPolicy", "Bypass",
|
||||
"-File", "scripts\export_1c_sql_files.ps1",
|
||||
"-Server", $Server,
|
||||
"-Database", $Database,
|
||||
"-User", $User,
|
||||
"-Password", $Password,
|
||||
"-Table", $Table,
|
||||
"-OutputPath", $TargetDir
|
||||
)
|
||||
[object[]]$normalizedFileName = @($FileName)
|
||||
if (($normalizedFileName | Measure-Object).Count -gt 0) {
|
||||
$args += "-FileName"
|
||||
$args += $normalizedFileName
|
||||
}
|
||||
$json = & powershell @args
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "Export failed for $Table."
|
||||
}
|
||||
return ($json | Out-String | ConvertFrom-Json)
|
||||
}
|
||||
|
||||
function Get-JsonProperty {
|
||||
param(
|
||||
[object]$Object,
|
||||
[string]$Name,
|
||||
[object]$Default = $null
|
||||
)
|
||||
if ($null -eq $Object) { return $Default }
|
||||
if ($Object.PSObject.Properties.Name -contains $Name) {
|
||||
return $Object.$Name
|
||||
}
|
||||
return $Default
|
||||
}
|
||||
|
||||
function Join-LimitedValues {
|
||||
param(
|
||||
[object[]]$Values,
|
||||
[int]$Limit = 12
|
||||
)
|
||||
$items = @($Values | Where-Object { $null -ne $_ -and [string]$_ -ne "" } | Select-Object -First $Limit)
|
||||
return ,$items
|
||||
}
|
||||
|
||||
function New-AgentSummary {
|
||||
param(
|
||||
[object]$Comparison,
|
||||
[object]$Detail
|
||||
)
|
||||
$detailByName = @{}
|
||||
foreach ($item in @($Detail.object_details)) {
|
||||
$fullName = [string](Get-JsonProperty -Object $item -Name "full_name" -Default "")
|
||||
if ($fullName) { $detailByName[$fullName] = $item }
|
||||
}
|
||||
|
||||
$objects = @()
|
||||
foreach ($change in @($Comparison.object_changes)) {
|
||||
$fullName = [string](Get-JsonProperty -Object $change -Name "full_name" -Default (Get-JsonProperty -Object $change -Name "name" -Default ""))
|
||||
$item = $null
|
||||
if ($fullName -and $detailByName.ContainsKey($fullName)) {
|
||||
$item = $detailByName[$fullName]
|
||||
}
|
||||
|
||||
$parts = @()
|
||||
$addedTerms = @()
|
||||
$removedTerms = @()
|
||||
$textDiffParts = 0
|
||||
$activeMissingParts = 0
|
||||
foreach ($part in @((Get-JsonProperty -Object $item -Name "details" -Default @()))) {
|
||||
$payload = Get-JsonProperty -Object $part -Name "payload" -Default $null
|
||||
$diff = Get-JsonProperty -Object $payload -Name "text_diff" -Default $null
|
||||
$semanticHints = Get-JsonProperty -Object $payload -Name "semantic_hints" -Default $null
|
||||
$activeExists = Get-JsonProperty -Object $part -Name "active_exists" -Default $null
|
||||
if ($activeExists -eq $false) { $activeMissingParts += 1 }
|
||||
if ($null -ne $diff) {
|
||||
$textDiffParts += 1
|
||||
$addedTerms += @(Get-JsonProperty -Object $semanticHints -Name "added_terms" -Default @())
|
||||
$removedTerms += @(Get-JsonProperty -Object $semanticHints -Name "removed_terms" -Default @())
|
||||
}
|
||||
$parts += [pscustomobject]@{
|
||||
file_name = Get-JsonProperty -Object $part -Name "file_name" -Default $null
|
||||
payload_role = Get-JsonProperty -Object $part -Name "payload_role" -Default $null
|
||||
saved_table = Get-JsonProperty -Object $part -Name "saved_table" -Default $null
|
||||
active_table = Get-JsonProperty -Object $part -Name "active_table" -Default $null
|
||||
active_exists = $activeExists
|
||||
text_comparable = Get-JsonProperty -Object $payload -Name "text_comparable" -Default $null
|
||||
summary = Get-JsonProperty -Object $payload -Name "summary" -Default $null
|
||||
delta_chars = Get-JsonProperty -Object $diff -Name "delta_chars" -Default $null
|
||||
}
|
||||
}
|
||||
|
||||
$objects += [pscustomobject]@{
|
||||
full_name = $fullName
|
||||
layer = Get-JsonProperty -Object $change -Name "layer" -Default $null
|
||||
extension = Get-JsonProperty -Object $change -Name "extension" -Default $null
|
||||
kind = Get-JsonProperty -Object $change -Name "kind" -Default $null
|
||||
kind_ru = Get-JsonProperty -Object $change -Name "kind_ru" -Default $null
|
||||
name = Get-JsonProperty -Object $change -Name "name" -Default $null
|
||||
synonym = Get-JsonProperty -Object $change -Name "synonym" -Default $null
|
||||
change_state = Get-JsonProperty -Object $change -Name "change_state" -Default $null
|
||||
parts_count = ($parts | Measure-Object).Count
|
||||
text_diff_parts = $textDiffParts
|
||||
active_missing_parts = $activeMissingParts
|
||||
added_terms = Join-LimitedValues -Values ($addedTerms | Sort-Object -Unique) -Limit 12
|
||||
removed_terms = Join-LimitedValues -Values ($removedTerms | Sort-Object -Unique) -Limit 12
|
||||
parts = $parts
|
||||
}
|
||||
}
|
||||
|
||||
$systemChanges = @()
|
||||
foreach ($change in @($Comparison.system_changes)) {
|
||||
$systemChanges += [pscustomobject]@{
|
||||
name = Get-JsonProperty -Object $change -Name "name" -Default $null
|
||||
layer = Get-JsonProperty -Object $change -Name "layer" -Default $null
|
||||
extension = Get-JsonProperty -Object $change -Name "extension" -Default $null
|
||||
}
|
||||
}
|
||||
|
||||
return [pscustomobject]@{
|
||||
purpose = "Compact agent-facing summary of saved-but-not-applied changes in 1C terms."
|
||||
default_next_action = "Inspect object details before generating or applying any code changes."
|
||||
object_changes = $objects
|
||||
system_changes = $systemChanges
|
||||
object_names = @($objects | ForEach-Object { $_.full_name })
|
||||
system_change_names = @($systemChanges | ForEach-Object { $_.name })
|
||||
}
|
||||
}
|
||||
|
||||
$steps = @()
|
||||
|
||||
$steps += Invoke-Step "compare_saved_state_objects" {
|
||||
& powershell -NoProfile -ExecutionPolicy Bypass -File scripts\compare_1c_saved_state_objects.ps1 `
|
||||
-Server $Server `
|
||||
-Database $Database `
|
||||
-User $User `
|
||||
-Password $Password `
|
||||
-BaseMetadataDir $BaseMetadataDir `
|
||||
-ExtensionGuidIndex $ExtensionGuidIndex `
|
||||
-Output $comparisonPath | Out-Null
|
||||
if ($LASTEXITCODE -ne 0) { throw "Saved-state object comparison failed." }
|
||||
}
|
||||
|
||||
$comparison = Read-JsonFile $comparisonPath
|
||||
|
||||
$steps += Invoke-Step "export_config_save" {
|
||||
$null = Export-TableFiles -Table "ConfigSave" -TargetDir $configSaveDir
|
||||
}
|
||||
|
||||
$steps += Invoke-Step "export_config_cas_save" {
|
||||
$null = Export-TableFiles -Table "ConfigCASSave" -TargetDir $configCASSaveDir
|
||||
}
|
||||
|
||||
$activeConfigFiles = @(
|
||||
$comparison.object_changes |
|
||||
ForEach-Object { $_.storage } |
|
||||
Where-Object { $_.active_table -eq "Config" } |
|
||||
ForEach-Object { [string]$_.file_name } |
|
||||
Sort-Object -Unique
|
||||
)
|
||||
$activeConfigCASFiles = @(
|
||||
$comparison.object_changes |
|
||||
ForEach-Object { $_.storage } |
|
||||
Where-Object { $_.active_table -eq "ConfigCAS" -and $_.active_exists } |
|
||||
ForEach-Object { [string]$_.file_name } |
|
||||
Sort-Object -Unique
|
||||
)
|
||||
|
||||
$steps += Invoke-Step "export_active_config_payloads" {
|
||||
if (($activeConfigFiles | Measure-Object).Count -gt 0) {
|
||||
$null = Export-TableFiles -Table "Config" -TargetDir $activeConfigDir -FileName $activeConfigFiles
|
||||
}
|
||||
}
|
||||
|
||||
$steps += Invoke-Step "export_active_config_cas_payloads" {
|
||||
if (($activeConfigCASFiles | Measure-Object).Count -gt 0) {
|
||||
$null = Export-TableFiles -Table "ConfigCAS" -TargetDir $activeConfigCASDir -FileName $activeConfigCASFiles
|
||||
}
|
||||
}
|
||||
|
||||
$steps += Invoke-Step "analyze_saved_state_object_details" {
|
||||
& $Python scripts\analyze_1c_saved_state_object_details.py `
|
||||
--comparison $comparisonPath `
|
||||
--config-save-dir $configSaveDir `
|
||||
--config-dir $activeConfigDir `
|
||||
--config-cas-save-dir $configCASSaveDir `
|
||||
--config-cas-dir $activeConfigCASDir `
|
||||
--extension-manifest-summary $ExtensionManifestSummary `
|
||||
--config-cas-all-dir $ConfigCASAllDir `
|
||||
--output $detailPath | Out-Null
|
||||
if ($LASTEXITCODE -ne 0) { throw "Saved-state detail analysis failed." }
|
||||
}
|
||||
|
||||
$detail = Read-JsonFile $detailPath
|
||||
$agentSummary = New-AgentSummary -Comparison $comparison -Detail $detail
|
||||
|
||||
$reportPath = Join-Path $resolvedOutput "saved-state-object-report.json"
|
||||
if (-not $MarkdownOutput) {
|
||||
$MarkdownOutput = Join-Path $resolvedOutput "saved-state-object-report.md"
|
||||
} else {
|
||||
$MarkdownOutput = [System.IO.Path]::GetFullPath($MarkdownOutput)
|
||||
}
|
||||
|
||||
$result = [pscustomobject]@{
|
||||
schema = "onec_saved_state_object_report.v1"
|
||||
server = $Server
|
||||
database = $Database
|
||||
report = $reportPath
|
||||
markdown = if ($SkipMarkdown) { $null } else { $MarkdownOutput }
|
||||
check = $checkPath
|
||||
output_dir = $resolvedOutput
|
||||
comparison = $comparisonPath
|
||||
detail = $detailPath
|
||||
agent_summary = $agentSummary
|
||||
payload_dirs = [pscustomobject]@{
|
||||
config_save = $configSaveDir
|
||||
config_cas_save = $configCASSaveDir
|
||||
active_config = $activeConfigDir
|
||||
active_config_cas = $activeConfigCASDir
|
||||
active_config_cas_all = $ConfigCASAllDir
|
||||
}
|
||||
counts = [pscustomobject]@{
|
||||
object_changes = $comparison.counts.object_changes
|
||||
system_changes = $comparison.counts.system_changes
|
||||
detail_objects = $detail.counts.objects
|
||||
detail_parts = $detail.counts.details
|
||||
}
|
||||
steps = $steps
|
||||
safety = [pscustomobject]@{
|
||||
read_only = $true
|
||||
sql_write_performed = $false
|
||||
public_terms_are_1c_objects = $true
|
||||
secrets_in_report = $false
|
||||
}
|
||||
}
|
||||
|
||||
$result | ConvertTo-Json -Depth 12 | Set-Content -LiteralPath $reportPath -Encoding UTF8
|
||||
|
||||
if (-not $SkipMarkdown) {
|
||||
$steps += Invoke-Step "render_saved_state_markdown" {
|
||||
& $Python scripts\render_1c_saved_state_object_report_markdown.py `
|
||||
--report $reportPath `
|
||||
--output $MarkdownOutput `
|
||||
--max-diff-lines $MarkdownMaxDiffLines | Out-Null
|
||||
if ($LASTEXITCODE -ne 0) { throw "Saved-state Markdown rendering failed." }
|
||||
}
|
||||
$result.steps = $steps
|
||||
$result | ConvertTo-Json -Depth 12 | Set-Content -LiteralPath $reportPath -Encoding UTF8
|
||||
}
|
||||
|
||||
$steps += Invoke-Step "check_saved_state_report" {
|
||||
& $Python scripts\check_1c_saved_state_object_report.py `
|
||||
--report $reportPath `
|
||||
--output $checkPath | Out-Null
|
||||
if ($LASTEXITCODE -ne 0) { throw "Saved-state report check failed." }
|
||||
}
|
||||
$result.steps = $steps
|
||||
$result | ConvertTo-Json -Depth 12 | Set-Content -LiteralPath $reportPath -Encoding UTF8
|
||||
|
||||
$result | ConvertTo-Json -Depth 12
|
||||
@@ -0,0 +1,391 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Build an agent-friendly 1C SQL read view from raw and resolved reports."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import re
|
||||
from collections import defaultdict
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
def load_json(path: Path) -> dict[str, Any]:
|
||||
return json.loads(path.read_text(encoding="utf-8-sig"))
|
||||
|
||||
|
||||
def write_json(path: Path, data: dict[str, Any]) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(json.dumps(data, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
|
||||
|
||||
def scope_key(scope: str, row_index: int, cell_key: str) -> tuple[str, int, str]:
|
||||
return (scope, row_index, cell_key)
|
||||
|
||||
|
||||
def part_scope(part_name: str | None) -> str:
|
||||
return f"table_part:{part_name or ''}"
|
||||
|
||||
|
||||
def best_alternate(alternates: list[dict[str, Any]]) -> dict[str, Any] | None:
|
||||
if not alternates:
|
||||
return None
|
||||
for alternate in alternates:
|
||||
route = alternate.get("route") or {}
|
||||
if alternate.get("found") and route.get("base_table"):
|
||||
return alternate
|
||||
for alternate in alternates:
|
||||
if alternate.get("found") and str(alternate.get("table") or "").endswith("X1"):
|
||||
return alternate
|
||||
for alternate in alternates:
|
||||
if alternate.get("found"):
|
||||
return alternate
|
||||
return alternates[0]
|
||||
|
||||
|
||||
def presentation(values: dict[str, Any] | None) -> str | None:
|
||||
if not values:
|
||||
return None
|
||||
for key in ("_Description", "_Code", "_Number", "_EnumOrder"):
|
||||
value = values.get(key)
|
||||
if value is not None and value != "":
|
||||
return str(value)
|
||||
return None
|
||||
|
||||
|
||||
BOOLEAN_STANDARD_PATHS = {"standard._Marked", "standard._Posted", "standard._Active"}
|
||||
|
||||
|
||||
def normalize_1c_sql_date(value: Any) -> Any:
|
||||
if not isinstance(value, str):
|
||||
return value
|
||||
match = re.match(r"^(\d{4})(-\d{2}-\d{2}T.*)$", value)
|
||||
if not match:
|
||||
return value
|
||||
year = int(match.group(1))
|
||||
if year < 3000:
|
||||
return value
|
||||
return f"{year - 2000:04d}{match.group(2)}"
|
||||
|
||||
|
||||
def field_has_boolean_type(field: dict[str, Any]) -> bool:
|
||||
if field.get("metadata_path") in BOOLEAN_STANDARD_PATHS:
|
||||
return True
|
||||
for column in (field.get("columns") or {}).values():
|
||||
value_type = column.get("value_type")
|
||||
values = value_type if isinstance(value_type, list) else [value_type]
|
||||
if any(item == "xs:boolean" for item in values):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def normalize_display_value(value: Any, field: dict[str, Any]) -> Any:
|
||||
if isinstance(value, dict) and value.get("kind") == "binary" and value.get("length") == 1 and field_has_boolean_type(field):
|
||||
hex_value = str(value.get("hex") or "").lower()
|
||||
if hex_value == "00":
|
||||
return False
|
||||
if hex_value == "01":
|
||||
return True
|
||||
return normalize_1c_sql_date(value)
|
||||
|
||||
|
||||
def enum_order(values: dict[str, Any] | None) -> int | None:
|
||||
if not values:
|
||||
return None
|
||||
value = values.get("_EnumOrder")
|
||||
if value is None:
|
||||
return None
|
||||
try:
|
||||
return int(value)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
def enum_presentation(item: dict[str, Any], enum_map: dict[str, Any] | None) -> dict[str, Any] | None:
|
||||
if not enum_map:
|
||||
return None
|
||||
target = item.get("target") or {}
|
||||
if target.get("kind") != "Enum":
|
||||
return None
|
||||
name = target.get("name")
|
||||
order = enum_order(item.get("values"))
|
||||
if name is None or order is None:
|
||||
return None
|
||||
enum = (enum_map.get("enums") or {}).get(name)
|
||||
if not enum:
|
||||
return None
|
||||
value = (enum.get("by_order") or {}).get(str(order))
|
||||
if not value:
|
||||
return None
|
||||
return value
|
||||
|
||||
|
||||
def resolved_payload(item: dict[str, Any], *, mode: str, enum_map: dict[str, Any] | None = None) -> dict[str, Any]:
|
||||
enum_value = enum_presentation(item, enum_map)
|
||||
payload = {
|
||||
"mode": mode,
|
||||
"found": item.get("found"),
|
||||
"target": item.get("target"),
|
||||
"values": item.get("values"),
|
||||
"presentation": (enum_value or {}).get("synonym") or (enum_value or {}).get("name") or presentation(item.get("values")),
|
||||
"reason": item.get("reason"),
|
||||
}
|
||||
if enum_value:
|
||||
payload["enum_value"] = enum_value
|
||||
alternates = item.get("alternate_hits") or []
|
||||
if alternates:
|
||||
payload["alternate_hits"] = alternates
|
||||
selected = best_alternate(alternates)
|
||||
if selected:
|
||||
payload["selected_alternate"] = selected
|
||||
payload["selected_presentation"] = presentation(selected.get("values"))
|
||||
return payload
|
||||
|
||||
|
||||
def simple_reference_index(reference_report: dict[str, Any] | None, enum_map: dict[str, Any] | None = None) -> dict[tuple[str, int, str], dict[str, Any]]:
|
||||
if not reference_report:
|
||||
return {}
|
||||
result = {}
|
||||
for item in reference_report.get("references") or []:
|
||||
ref = item.get("reference") or {}
|
||||
scope = "main" if ref.get("scope") == "main" else part_scope(ref.get("table_part_name"))
|
||||
row_index = int(ref.get("row_index") or 0)
|
||||
cell_key = ref.get("cell_key")
|
||||
if not cell_key:
|
||||
continue
|
||||
result[scope_key(scope, row_index, cell_key)] = resolved_payload(item, mode="single_reference", enum_map=enum_map)
|
||||
return result
|
||||
|
||||
|
||||
def composite_reference_index(composite_report: dict[str, Any] | None, enum_map: dict[str, Any] | None = None) -> dict[tuple[str, int, str], dict[str, Any]]:
|
||||
if not composite_report:
|
||||
return {}
|
||||
result = {}
|
||||
for item in composite_report.get("composites") or []:
|
||||
comp = item.get("composite") or {}
|
||||
scope = "main" if comp.get("scope") == "main" else part_scope(comp.get("table_part_name"))
|
||||
row_index = int(comp.get("row_index") or 0)
|
||||
metadata_path = comp.get("metadata_path")
|
||||
if not metadata_path:
|
||||
continue
|
||||
payload = resolved_payload(item, mode="composite_reference", enum_map=enum_map)
|
||||
payload["type_hex"] = comp.get("type_hex")
|
||||
payload["rtref_hex"] = comp.get("rtref_hex")
|
||||
payload["rtref_sql_number"] = comp.get("rtref_sql_number")
|
||||
payload["rrref_hex"] = comp.get("rrref_hex")
|
||||
payload["columns"] = comp.get("columns")
|
||||
result[(scope, row_index, metadata_path)] = payload
|
||||
return result
|
||||
|
||||
|
||||
def composite_value_index(composite_value_report: dict[str, Any] | None) -> dict[tuple[str, int, str], dict[str, Any]]:
|
||||
if not composite_value_report:
|
||||
return {}
|
||||
result = {}
|
||||
for item in composite_value_report.get("composites") or []:
|
||||
scope = "main" if item.get("scope") == "main" else part_scope(item.get("table_part_name"))
|
||||
row_index = int(item.get("row_index") or 0)
|
||||
metadata_path = item.get("metadata_path")
|
||||
if metadata_path:
|
||||
result[(scope, row_index, metadata_path)] = item
|
||||
return result
|
||||
|
||||
|
||||
def display_value(field: dict[str, Any]) -> Any:
|
||||
resolved = field.get("resolved") or {}
|
||||
for key in ("presentation", "selected_presentation"):
|
||||
value = resolved.get(key)
|
||||
if value is not None:
|
||||
return value
|
||||
composite = field.get("composite_value") or {}
|
||||
selected = composite.get("selected") or {}
|
||||
if selected.get("branch") == "primitive":
|
||||
values = selected.get("primitive_values") or []
|
||||
if values:
|
||||
return normalize_display_value(values[0].get("value"), field)
|
||||
if field.get("value") is not None:
|
||||
return normalize_display_value(field.get("value"), field)
|
||||
columns = field.get("columns") or {}
|
||||
if len(columns) == 1:
|
||||
only = next(iter(columns.values()))
|
||||
return normalize_display_value(only.get("value"), field)
|
||||
return None
|
||||
|
||||
|
||||
def enrich_row(
|
||||
row: dict[str, Any],
|
||||
*,
|
||||
scope: str,
|
||||
row_index: int,
|
||||
simple_refs: dict[tuple[str, int, str], dict[str, Any]],
|
||||
composite_refs: dict[tuple[str, int, str], dict[str, Any]],
|
||||
composite_values: dict[tuple[str, int, str], dict[str, Any]],
|
||||
) -> dict[str, Any]:
|
||||
cells: dict[str, Any] = {}
|
||||
fields: dict[str, Any] = {}
|
||||
grouped: dict[str, list[tuple[str, dict[str, Any]]]] = defaultdict(list)
|
||||
|
||||
for cell_key, cell in row.items():
|
||||
enriched = dict(cell)
|
||||
simple = simple_refs.get(scope_key(scope, row_index, cell_key))
|
||||
if simple:
|
||||
enriched["resolved"] = simple
|
||||
cells[cell_key] = enriched
|
||||
grouped[str(cell.get("metadata_path") or cell_key)].append((cell_key, enriched))
|
||||
|
||||
for metadata_path, members in grouped.items():
|
||||
first = members[0][1]
|
||||
columns = {member["column"]: member for _, member in members if member.get("column")}
|
||||
field = {
|
||||
"metadata_path": metadata_path,
|
||||
"metadata_name": first.get("metadata_name"),
|
||||
"metadata_uuid": first.get("metadata_uuid"),
|
||||
"metadata_field": first.get("metadata_field"),
|
||||
"columns": columns,
|
||||
}
|
||||
composite = composite_refs.get((scope, row_index, metadata_path))
|
||||
composite_value = composite_values.get((scope, row_index, metadata_path))
|
||||
if composite:
|
||||
field["resolved"] = composite
|
||||
if composite_value:
|
||||
field["composite_value"] = composite_value
|
||||
if "resolved" not in field and len(members) == 1 and members[0][1].get("resolved"):
|
||||
field["resolved"] = members[0][1]["resolved"]
|
||||
elif "resolved" not in field and len(members) == 1:
|
||||
field["value"] = members[0][1].get("value")
|
||||
field["display_value"] = display_value(field)
|
||||
fields[metadata_path] = field
|
||||
|
||||
return {"cells": cells, "fields": fields}
|
||||
|
||||
|
||||
def build_view(
|
||||
read_result: dict[str, Any],
|
||||
reference_report: dict[str, Any] | None,
|
||||
composite_report: dict[str, Any] | None,
|
||||
composite_value_report: dict[str, Any] | None,
|
||||
enum_map: dict[str, Any] | None,
|
||||
) -> dict[str, Any]:
|
||||
simple_refs = simple_reference_index(reference_report, enum_map)
|
||||
composite_refs = composite_reference_index(composite_report, enum_map)
|
||||
composite_values = composite_value_index(composite_value_report)
|
||||
|
||||
main_rows = []
|
||||
for index, row in enumerate(read_result.get("main", {}).get("rows") or []):
|
||||
main_rows.append(
|
||||
{
|
||||
"row_index": index,
|
||||
**enrich_row(
|
||||
row,
|
||||
scope="main",
|
||||
row_index=index,
|
||||
simple_refs=simple_refs,
|
||||
composite_refs=composite_refs,
|
||||
composite_values=composite_values,
|
||||
),
|
||||
}
|
||||
)
|
||||
|
||||
table_parts = []
|
||||
for part in read_result.get("table_parts") or []:
|
||||
scope = part_scope(part.get("name"))
|
||||
rows = []
|
||||
for index, row in enumerate(part.get("rows") or []):
|
||||
rows.append(
|
||||
{
|
||||
"row_index": index,
|
||||
**enrich_row(
|
||||
row,
|
||||
scope=scope,
|
||||
row_index=index,
|
||||
simple_refs=simple_refs,
|
||||
composite_refs=composite_refs,
|
||||
composite_values=composite_values,
|
||||
),
|
||||
}
|
||||
)
|
||||
table_parts.append(
|
||||
{
|
||||
"name": part.get("name"),
|
||||
"uuid": part.get("uuid"),
|
||||
"table": part.get("table"),
|
||||
"row_count": part.get("row_count"),
|
||||
"rows": rows,
|
||||
}
|
||||
)
|
||||
|
||||
return {
|
||||
"schema": "onec_sql_read_view.v1",
|
||||
"source_schema": read_result.get("schema"),
|
||||
"server": read_result.get("server"),
|
||||
"database": read_result.get("database"),
|
||||
"kind": read_result.get("kind"),
|
||||
"identity": read_result.get("identity"),
|
||||
"inputs": {
|
||||
"read_result": read_result.get("projection_path"),
|
||||
"reference_resolution_schema": (reference_report or {}).get("schema"),
|
||||
"composite_reference_resolution_schema": (composite_report or {}).get("schema"),
|
||||
"composite_value_resolution_schema": (composite_value_report or {}).get("schema"),
|
||||
"enum_presentation_map_schema": (enum_map or {}).get("schema"),
|
||||
},
|
||||
"summary": {
|
||||
"main_rows": len(main_rows),
|
||||
"table_parts": len(table_parts),
|
||||
"simple_reference_cells": len(simple_refs),
|
||||
"composite_reference_groups": len(composite_refs),
|
||||
"composite_value_groups": len(composite_values),
|
||||
},
|
||||
"main": {
|
||||
"table": read_result.get("main", {}).get("table"),
|
||||
"row_count": read_result.get("main", {}).get("row_count"),
|
||||
"rows": main_rows,
|
||||
},
|
||||
"table_parts": table_parts,
|
||||
}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Build an enriched 1C SQL read view.")
|
||||
parser.add_argument("--read-result", type=Path, required=True)
|
||||
parser.add_argument("--reference-resolution", type=Path)
|
||||
parser.add_argument("--composite-reference-resolution", type=Path)
|
||||
parser.add_argument("--composite-value-resolution", type=Path)
|
||||
parser.add_argument("--enum-presentation-map", type=Path)
|
||||
parser.add_argument("--output", type=Path, required=True)
|
||||
args = parser.parse_args()
|
||||
|
||||
read_result = load_json(args.read_result)
|
||||
reference_report = load_json(args.reference_resolution) if args.reference_resolution else None
|
||||
composite_report = load_json(args.composite_reference_resolution) if args.composite_reference_resolution else None
|
||||
composite_value_report = load_json(args.composite_value_resolution) if args.composite_value_resolution else None
|
||||
enum_map = load_json(args.enum_presentation_map) if args.enum_presentation_map else None
|
||||
view = build_view(read_result, reference_report, composite_report, composite_value_report, enum_map)
|
||||
view["inputs"]["read_result_path"] = str(args.read_result)
|
||||
if args.reference_resolution:
|
||||
view["inputs"]["reference_resolution_path"] = str(args.reference_resolution)
|
||||
if args.composite_reference_resolution:
|
||||
view["inputs"]["composite_reference_resolution_path"] = str(args.composite_reference_resolution)
|
||||
if args.composite_value_resolution:
|
||||
view["inputs"]["composite_value_resolution_path"] = str(args.composite_value_resolution)
|
||||
if args.enum_presentation_map:
|
||||
view["inputs"]["enum_presentation_map_path"] = str(args.enum_presentation_map)
|
||||
write_json(args.output, view)
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"output": str(args.output),
|
||||
"main_rows": view["summary"]["main_rows"],
|
||||
"table_parts": view["summary"]["table_parts"],
|
||||
"simple_reference_cells": view["summary"]["simple_reference_cells"],
|
||||
"composite_reference_groups": view["summary"]["composite_reference_groups"],
|
||||
"composite_value_groups": view["summary"]["composite_value_groups"],
|
||||
},
|
||||
ensure_ascii=False,
|
||||
)
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,383 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Build a compact read-only evidence bundle for a 1C development task."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import base64
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
||||
from get_1c_form_context import build_context as build_form_context # noqa: E402
|
||||
from get_1c_module import build_module_result # noqa: E402
|
||||
from get_1c_object_metadata import build_object_metadata # noqa: E402
|
||||
from plan_1c_task_context import build_plan # noqa: E402
|
||||
from resolve_1c_object import load_json # noqa: E402
|
||||
|
||||
|
||||
def decode_arg(value: str | None, encoded: str | None) -> str | None:
|
||||
if encoded:
|
||||
return base64.b64decode(encoded).decode("utf-8")
|
||||
return value
|
||||
|
||||
|
||||
def collect_target_reads(investigation: dict[str, Any]) -> tuple[list[str], list[str]]:
|
||||
forms = []
|
||||
modules = []
|
||||
for read in investigation.get("recommended_reads") or []:
|
||||
form = read.get("form")
|
||||
module = read.get("module")
|
||||
if form and form not in forms:
|
||||
forms.append(form)
|
||||
if module and module not in modules:
|
||||
modules.append(module)
|
||||
for search in investigation.get("searches") or []:
|
||||
for match in search.get("matches") or []:
|
||||
form = match.get("form")
|
||||
if form and form not in forms:
|
||||
forms.append(form)
|
||||
area = match.get("area")
|
||||
module = match.get("name")
|
||||
if area in {"module", "module.code"} and module and module not in modules:
|
||||
modules.append(module)
|
||||
return forms, modules
|
||||
|
||||
|
||||
def compact_metadata(metadata: dict[str, Any], *, max_attributes: int) -> dict[str, Any]:
|
||||
attrs = metadata.get("attributes") or []
|
||||
sections = metadata.get("tabular_sections") or []
|
||||
return {
|
||||
"schema": metadata.get("schema"),
|
||||
"view": metadata.get("view"),
|
||||
"object": metadata.get("object"),
|
||||
"attributes": attrs[:max_attributes],
|
||||
"attributes_total": len(attrs),
|
||||
"attributes_truncated": len(attrs) > max_attributes,
|
||||
"tabular_sections": sections,
|
||||
"counts": metadata.get("counts"),
|
||||
}
|
||||
|
||||
|
||||
def compact_form_context(context: dict[str, Any], *, max_items: int, max_attributes: int, max_commands: int) -> dict[str, Any]:
|
||||
forms = []
|
||||
for form in context.get("forms") or []:
|
||||
copy = {key: form.get(key) for key in ("name", "synonym", "uuid", "origin", "effective_action", "meta_xml_path", "form_xml_path", "module_path", "extension_overlays") if form.get(key) not in (None, [], "")}
|
||||
structure = form.get("structure") or {}
|
||||
if structure:
|
||||
copy["structure"] = {
|
||||
"origin": structure.get("origin"),
|
||||
"form_xml_path": structure.get("form_xml_path"),
|
||||
"events": structure.get("events") or [],
|
||||
"items": (structure.get("items") or [])[:max_items],
|
||||
"attributes": (structure.get("attributes") or [])[:max_attributes],
|
||||
"commands": (structure.get("commands") or [])[:max_commands],
|
||||
"counts": structure.get("counts"),
|
||||
}
|
||||
overlays = []
|
||||
for overlay in form.get("extension_overlays") or []:
|
||||
overlay_copy = {key: overlay.get(key) for key in ("name", "synonym", "uuid", "origin", "effective_action", "meta_xml_path", "form_xml_path", "module_path") if overlay.get(key) not in (None, [], "")}
|
||||
structure = overlay.get("structure") or {}
|
||||
if structure:
|
||||
overlay_copy["structure"] = {
|
||||
"origin": structure.get("origin"),
|
||||
"form_xml_path": structure.get("form_xml_path"),
|
||||
"events": structure.get("events") or [],
|
||||
"items": (structure.get("items") or [])[:max_items],
|
||||
"attributes": (structure.get("attributes") or [])[:max_attributes],
|
||||
"commands": (structure.get("commands") or [])[:max_commands],
|
||||
"counts": structure.get("counts"),
|
||||
}
|
||||
overlays.append(overlay_copy)
|
||||
if overlays:
|
||||
copy["extension_overlays"] = overlays
|
||||
forms.append(copy)
|
||||
return {
|
||||
"schema": context.get("schema"),
|
||||
"view": context.get("view"),
|
||||
"object": context.get("object"),
|
||||
"query": context.get("query"),
|
||||
"forms": forms,
|
||||
"counts": context.get("counts"),
|
||||
}
|
||||
|
||||
|
||||
def compact_module_result(result: dict[str, Any]) -> dict[str, Any]:
|
||||
return {
|
||||
"schema": result.get("schema"),
|
||||
"view": result.get("view"),
|
||||
"object": result.get("object"),
|
||||
"query": result.get("query"),
|
||||
"modules": result.get("modules") or [],
|
||||
"counts": result.get("counts"),
|
||||
}
|
||||
|
||||
|
||||
def read_lines(path: str) -> list[str]:
|
||||
file_path = Path(path)
|
||||
try:
|
||||
return file_path.read_text(encoding="utf-8-sig").splitlines()
|
||||
except UnicodeDecodeError:
|
||||
return file_path.read_text(encoding="cp1251", errors="replace").splitlines()
|
||||
|
||||
|
||||
def code_snippet(path: str, line: int, *, radius: int, max_chars: int) -> dict[str, Any] | None:
|
||||
file_path = Path(path)
|
||||
if not file_path.is_file() or line <= 0:
|
||||
return None
|
||||
lines = read_lines(path)
|
||||
start = max(1, line - radius)
|
||||
end = min(len(lines), line + radius)
|
||||
text = "\n".join(lines[start - 1 : end])
|
||||
truncated = len(text) > max_chars
|
||||
return {
|
||||
"path": path,
|
||||
"line_start": start,
|
||||
"line_end": end,
|
||||
"focus_line": line,
|
||||
"text": text[:max_chars],
|
||||
"truncated": truncated,
|
||||
"char_count": len(text),
|
||||
}
|
||||
|
||||
|
||||
def collect_code_snippets(searches: list[dict[str, Any]], *, radius: int, max_chars: int, limit: int) -> list[dict[str, Any]]:
|
||||
snippets = []
|
||||
seen = set()
|
||||
for search in searches:
|
||||
for match in search.get("matches") or []:
|
||||
if match.get("area") != "module.code":
|
||||
continue
|
||||
evidence = match.get("evidence") or {}
|
||||
path = evidence.get("path")
|
||||
line = evidence.get("line")
|
||||
if not path or not line:
|
||||
continue
|
||||
key = (path, line)
|
||||
if key in seen:
|
||||
continue
|
||||
seen.add(key)
|
||||
snippet = code_snippet(str(path), int(line), radius=radius, max_chars=max_chars)
|
||||
if not snippet:
|
||||
continue
|
||||
snippet.update(
|
||||
{
|
||||
"search_text": search.get("text"),
|
||||
"module": match.get("name"),
|
||||
"origin": match.get("origin"),
|
||||
"effective_action": match.get("effective_action"),
|
||||
}
|
||||
)
|
||||
snippets.append(snippet)
|
||||
if len(snippets) >= limit:
|
||||
return snippets
|
||||
return snippets
|
||||
|
||||
|
||||
def evidence_for_investigation(
|
||||
index: dict[str, Any],
|
||||
investigation: dict[str, Any],
|
||||
*,
|
||||
view: str,
|
||||
max_attributes: int,
|
||||
max_form_items: int,
|
||||
max_form_attributes: int,
|
||||
max_form_commands: int,
|
||||
max_module_chars: int,
|
||||
code_snippet_radius: int,
|
||||
max_code_snippet_chars: int,
|
||||
max_code_snippets: int,
|
||||
max_forms: int,
|
||||
max_modules: int,
|
||||
) -> dict[str, Any]:
|
||||
candidate = investigation.get("candidate") or {}
|
||||
kind = candidate.get("kind")
|
||||
name = candidate.get("name")
|
||||
metadata = build_object_metadata(index, kind=kind, name=name, view=view, extension=None, include_storage=False)
|
||||
forms, modules = collect_target_reads(investigation)
|
||||
form_contexts = []
|
||||
for form_name in forms[:max_forms]:
|
||||
form_contexts.append(
|
||||
compact_form_context(
|
||||
build_form_context(index, kind=kind, name=name, form=form_name, view=view, extension=None, max_items=max_form_items),
|
||||
max_items=max_form_items,
|
||||
max_attributes=max_form_attributes,
|
||||
max_commands=max_form_commands,
|
||||
)
|
||||
)
|
||||
module_contexts = []
|
||||
for module_name in modules[:max_modules]:
|
||||
module_contexts.append(
|
||||
compact_module_result(
|
||||
build_module_result(
|
||||
index,
|
||||
kind=kind,
|
||||
name=name,
|
||||
module_name=module_name,
|
||||
view=view,
|
||||
extension=None,
|
||||
max_chars=max_module_chars,
|
||||
routine=None,
|
||||
)
|
||||
)
|
||||
)
|
||||
snippets = collect_code_snippets(
|
||||
investigation.get("searches") or [],
|
||||
radius=code_snippet_radius,
|
||||
max_chars=max_code_snippet_chars,
|
||||
limit=max_code_snippets,
|
||||
)
|
||||
return {
|
||||
"candidate": candidate,
|
||||
"brief": investigation.get("brief"),
|
||||
"searches": investigation.get("searches") or [],
|
||||
"metadata": compact_metadata(metadata, max_attributes=max_attributes),
|
||||
"forms": form_contexts,
|
||||
"modules": module_contexts,
|
||||
"code_snippets": snippets,
|
||||
"recommended_reads": investigation.get("recommended_reads") or [],
|
||||
"counts": {
|
||||
"forms_materialized": len(form_contexts),
|
||||
"modules_materialized": len(module_contexts),
|
||||
"code_snippets": len(snippets),
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def build_evidence(
|
||||
index: dict[str, Any],
|
||||
*,
|
||||
text: str,
|
||||
view: str,
|
||||
max_objects: int,
|
||||
max_terms: int,
|
||||
max_matches: int,
|
||||
max_attributes: int,
|
||||
max_form_items: int,
|
||||
max_form_attributes: int,
|
||||
max_form_commands: int,
|
||||
max_module_chars: int,
|
||||
code_snippet_radius: int,
|
||||
max_code_snippet_chars: int,
|
||||
max_code_snippets: int,
|
||||
max_forms: int,
|
||||
max_modules: int,
|
||||
) -> dict[str, Any]:
|
||||
plan = build_plan(index, text=text, view=view, max_objects=max_objects, max_terms=max_terms, max_matches=max_matches)
|
||||
investigations = []
|
||||
for investigation in (plan.get("investigations") or [])[:max_objects]:
|
||||
investigations.append(
|
||||
evidence_for_investigation(
|
||||
index,
|
||||
investigation,
|
||||
view=view,
|
||||
max_attributes=max_attributes,
|
||||
max_form_items=max_form_items,
|
||||
max_form_attributes=max_form_attributes,
|
||||
max_form_commands=max_form_commands,
|
||||
max_module_chars=max_module_chars,
|
||||
code_snippet_radius=code_snippet_radius,
|
||||
max_code_snippet_chars=max_code_snippet_chars,
|
||||
max_code_snippets=max_code_snippets,
|
||||
max_forms=max_forms,
|
||||
max_modules=max_modules,
|
||||
)
|
||||
)
|
||||
return {
|
||||
"schema": "onec_task_evidence_bundle.v1",
|
||||
"view": view,
|
||||
"task": {"text": text},
|
||||
"plan": {
|
||||
"schema": plan.get("schema"),
|
||||
"object_candidates": plan.get("object_candidates") or [],
|
||||
"search_terms": plan.get("search_terms") or [],
|
||||
"safety": plan.get("safety"),
|
||||
"counts": plan.get("counts"),
|
||||
},
|
||||
"investigations": investigations,
|
||||
"limits": {
|
||||
"max_objects": max_objects,
|
||||
"max_terms": max_terms,
|
||||
"max_matches": max_matches,
|
||||
"max_attributes": max_attributes,
|
||||
"max_form_items": max_form_items,
|
||||
"max_form_attributes": max_form_attributes,
|
||||
"max_form_commands": max_form_commands,
|
||||
"max_module_chars": max_module_chars,
|
||||
"code_snippet_radius": code_snippet_radius,
|
||||
"max_code_snippet_chars": max_code_snippet_chars,
|
||||
"max_code_snippets": max_code_snippets,
|
||||
"max_forms": max_forms,
|
||||
"max_modules": max_modules,
|
||||
},
|
||||
"safety": {
|
||||
"mode": "read_only",
|
||||
"write_status": "blocked_until_write_gates",
|
||||
"write_contract": "docs/1c-write-path-safety.md",
|
||||
},
|
||||
"counts": {
|
||||
"investigations": len(investigations),
|
||||
"forms_materialized": sum(item.get("counts", {}).get("forms_materialized", 0) for item in investigations),
|
||||
"modules_materialized": sum(item.get("counts", {}).get("modules_materialized", 0) for item in investigations),
|
||||
"code_snippets": sum(item.get("counts", {}).get("code_snippets", 0) for item in investigations),
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Build read-only 1C task evidence bundle.")
|
||||
parser.add_argument("--index", type=Path, required=True)
|
||||
parser.add_argument("--text")
|
||||
parser.add_argument("--text-b64")
|
||||
parser.add_argument("--view", choices=["effective", "base"], default="effective")
|
||||
parser.add_argument("--max-objects", type=int, default=2)
|
||||
parser.add_argument("--max-terms", type=int, default=8)
|
||||
parser.add_argument("--max-matches", type=int, default=8)
|
||||
parser.add_argument("--max-attributes", type=int, default=120)
|
||||
parser.add_argument("--max-form-items", type=int, default=250)
|
||||
parser.add_argument("--max-form-attributes", type=int, default=120)
|
||||
parser.add_argument("--max-form-commands", type=int, default=80)
|
||||
parser.add_argument("--max-module-chars", type=int, default=12000)
|
||||
parser.add_argument("--code-snippet-radius", type=int, default=8)
|
||||
parser.add_argument("--max-code-snippet-chars", type=int, default=8000)
|
||||
parser.add_argument("--max-code-snippets", type=int, default=20)
|
||||
parser.add_argument("--max-forms", type=int, default=3)
|
||||
parser.add_argument("--max-modules", type=int, default=4)
|
||||
parser.add_argument("--output", type=Path)
|
||||
args = parser.parse_args()
|
||||
|
||||
text = decode_arg(args.text, args.text_b64)
|
||||
if not text:
|
||||
raise SystemExit("Use --text or --text-b64.")
|
||||
result = build_evidence(
|
||||
load_json(args.index),
|
||||
text=text,
|
||||
view=args.view,
|
||||
max_objects=args.max_objects,
|
||||
max_terms=args.max_terms,
|
||||
max_matches=args.max_matches,
|
||||
max_attributes=args.max_attributes,
|
||||
max_form_items=args.max_form_items,
|
||||
max_form_attributes=args.max_form_attributes,
|
||||
max_form_commands=args.max_form_commands,
|
||||
max_module_chars=args.max_module_chars,
|
||||
code_snippet_radius=args.code_snippet_radius,
|
||||
max_code_snippet_chars=args.max_code_snippet_chars,
|
||||
max_code_snippets=args.max_code_snippets,
|
||||
max_forms=args.max_forms,
|
||||
max_modules=args.max_modules,
|
||||
)
|
||||
output = json.dumps(result, ensure_ascii=False, indent=2) + "\n"
|
||||
if args.output:
|
||||
args.output.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.output.write_text(output, encoding="utf-8")
|
||||
print(json.dumps({"output": str(args.output), "counts": result["counts"], "view": result["view"]}, ensure_ascii=False))
|
||||
else:
|
||||
print(output)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,234 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Build a deterministic route index for 1C metadata objects.
|
||||
|
||||
The index answers where an object or object part can be retrieved from:
|
||||
DBNames storage roles, base Config direct files, XML names, and extension
|
||||
manifest/CAS entries. It stores routes and observed payload signatures, not
|
||||
semantic guesses.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import re
|
||||
from collections import Counter, defaultdict
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from analyze_1c_manifest_object_parts import parse_cas_payload, suffix_of
|
||||
|
||||
|
||||
def load_json(path: Path) -> dict[str, Any]:
|
||||
return json.loads(path.read_text(encoding="utf-8-sig"))
|
||||
|
||||
|
||||
def extension_name_from_manifest(path: Path) -> str:
|
||||
stem = path.stem
|
||||
return re.sub(r"^\d+_", "", stem).rsplit("-", 1)[0]
|
||||
|
||||
|
||||
def ensure_object(objects: dict[str, dict[str, Any]], guid: str) -> dict[str, Any]:
|
||||
guid = guid.lower()
|
||||
item = objects.get(guid)
|
||||
if item is None:
|
||||
item = {
|
||||
"guid": guid,
|
||||
"dbnames": [],
|
||||
"xml_top_objects": [],
|
||||
"xml_occurrence_count": 0,
|
||||
"config_routes": [],
|
||||
"extension_routes": [],
|
||||
}
|
||||
objects[guid] = item
|
||||
return item
|
||||
|
||||
|
||||
def add_dbnames(objects: dict[str, dict[str, Any]], dbnames: dict[str, Any]) -> Counter[str]:
|
||||
role_counts: Counter[str] = Counter()
|
||||
for source in dbnames.get("dbnames") or []:
|
||||
file_name = source.get("file_name")
|
||||
for record in source.get("records") or []:
|
||||
if record.get("status") != "parsed":
|
||||
continue
|
||||
guid = str(record.get("guid") or "").lower()
|
||||
role = record.get("storage_role")
|
||||
role_counts[str(role)] += 1
|
||||
ensure_object(objects, guid)["dbnames"].append(
|
||||
{
|
||||
"source_file": file_name,
|
||||
"storage_role": role,
|
||||
"sql_number": record.get("sql_number"),
|
||||
"index": record.get("index"),
|
||||
}
|
||||
)
|
||||
return role_counts
|
||||
|
||||
|
||||
def add_xml(objects: dict[str, dict[str, Any]], xml_index: dict[str, Any]) -> None:
|
||||
for guid, item in (xml_index.get("guid_map") or {}).items():
|
||||
obj = ensure_object(objects, guid)
|
||||
obj["xml_top_objects"] = item.get("top_objects") or []
|
||||
obj["xml_occurrence_count"] = item.get("total_occurrences", 0)
|
||||
|
||||
|
||||
def payload_signature(payload: dict[str, Any]) -> dict[str, Any]:
|
||||
return {
|
||||
"parse_status": payload.get("parse_status"),
|
||||
"encoding": payload.get("encoding"),
|
||||
"text_offset": payload.get("text_offset"),
|
||||
"payload_bytes": payload.get("payload_bytes"),
|
||||
"compression": payload.get("compression"),
|
||||
"root_marker": payload.get("root_marker"),
|
||||
"root_len": payload.get("root_len"),
|
||||
"payload_markers": payload.get("payload_markers") or [],
|
||||
"base64_block_count": len(payload.get("_base64_blocks_decoded") or []),
|
||||
"stream_block_count": len(payload.get("_stream_blocks") or []),
|
||||
}
|
||||
|
||||
|
||||
def add_config_routes(objects: dict[str, dict[str, Any]], config_dirs: list[Path]) -> None:
|
||||
for directory in config_dirs:
|
||||
if not directory.exists():
|
||||
continue
|
||||
for path in sorted(item for item in directory.iterdir() if item.is_file()):
|
||||
guid = path.name.lower()
|
||||
if not re.fullmatch(r"[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}", guid):
|
||||
continue
|
||||
payload = parse_cas_payload(path)
|
||||
ensure_object(objects, guid)["config_routes"].append(
|
||||
{
|
||||
"route_type": "base_config_direct",
|
||||
"table": directory.name,
|
||||
"file_name": guid,
|
||||
"path": str(path),
|
||||
"payload_signature": payload_signature(payload),
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def load_extension_inventory(path: Path | None) -> dict[str, dict[str, Any]]:
|
||||
if not path or not path.is_file():
|
||||
return {}
|
||||
inventory = load_json(path)
|
||||
result = {}
|
||||
for ext in inventory.get("extensions") or []:
|
||||
name = ext.get("extension_name")
|
||||
if name:
|
||||
result[name] = ext
|
||||
return result
|
||||
|
||||
|
||||
def add_extension_routes(
|
||||
objects: dict[str, dict[str, Any]],
|
||||
manifest_dir: Path,
|
||||
cas_dir: Path,
|
||||
extension_inventory: dict[str, dict[str, Any]],
|
||||
) -> Counter[str]:
|
||||
signature_counts: Counter[str] = Counter()
|
||||
payload_cache: dict[str, dict[str, Any]] = {}
|
||||
for manifest_path in sorted(manifest_dir.glob("*.json")):
|
||||
manifest = load_json(manifest_path)
|
||||
extension_name = extension_name_from_manifest(manifest_path)
|
||||
ext = extension_inventory.get(extension_name, {})
|
||||
for entry in manifest.get("entries") or []:
|
||||
object_id = str(entry.get("object_id") or "").lower()
|
||||
if not object_id:
|
||||
continue
|
||||
guid = object_id.split(".", 1)[0]
|
||||
cas_key = entry.get("cas_key")
|
||||
cas_path = Path(entry.get("cas_path") or cas_dir / str(cas_key))
|
||||
if not cas_path.is_file():
|
||||
cas_path = cas_dir / str(cas_key)
|
||||
if cas_path.is_file():
|
||||
if str(cas_path) not in payload_cache:
|
||||
payload_cache[str(cas_path)] = payload_signature(parse_cas_payload(cas_path))
|
||||
signature = payload_cache[str(cas_path)]
|
||||
else:
|
||||
signature = {"parse_status": "missing_cas"}
|
||||
signature_key = (
|
||||
f"{signature.get('parse_status')}|root={signature.get('root_marker')}|"
|
||||
f"len={signature.get('root_len')}|markers={','.join(signature.get('payload_markers') or [])}"
|
||||
)
|
||||
signature_counts[signature_key] += 1
|
||||
ensure_object(objects, guid)["extension_routes"].append(
|
||||
{
|
||||
"route_type": "extension_manifest_cas",
|
||||
"extension_name": extension_name,
|
||||
"extension_order": ext.get("extension_order"),
|
||||
"dbnames_ext_guid": ext.get("dbnames_ext_guid"),
|
||||
"manifest_path": str(manifest_path),
|
||||
"root_cas_file": manifest.get("root_cas_file"),
|
||||
"extension_configuration_guid": manifest.get("extension_configuration_guid"),
|
||||
"object_id": object_id,
|
||||
"suffix": suffix_of(object_id),
|
||||
"cas_key": cas_key,
|
||||
"cas_path": str(cas_path) if cas_path.is_file() else None,
|
||||
"payload_signature": signature,
|
||||
}
|
||||
)
|
||||
return signature_counts
|
||||
|
||||
|
||||
def compact_object(item: dict[str, Any]) -> dict[str, Any]:
|
||||
route_kind = []
|
||||
if item["config_routes"]:
|
||||
route_kind.append("base_config_direct")
|
||||
if item["extension_routes"]:
|
||||
route_kind.append("extension_manifest_cas")
|
||||
if item["dbnames"]:
|
||||
route_kind.append("dbnames_storage")
|
||||
if item["xml_top_objects"]:
|
||||
route_kind.append("xml_top_object")
|
||||
item["route_kind"] = route_kind
|
||||
return item
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Build a unified 1C object route index.")
|
||||
parser.add_argument("--dbnames", type=Path, required=True)
|
||||
parser.add_argument("--xml-index", type=Path, required=True)
|
||||
parser.add_argument("--manifest-dir", type=Path, required=True)
|
||||
parser.add_argument("--cas-dir", type=Path, required=True)
|
||||
parser.add_argument("--extension-inventory", type=Path)
|
||||
parser.add_argument("--config-dir", type=Path, action="append", default=[])
|
||||
parser.add_argument("--output", type=Path, required=True)
|
||||
args = parser.parse_args()
|
||||
|
||||
objects: dict[str, dict[str, Any]] = {}
|
||||
role_counts = add_dbnames(objects, load_json(args.dbnames))
|
||||
add_xml(objects, load_json(args.xml_index))
|
||||
add_config_routes(objects, args.config_dir)
|
||||
signature_counts = add_extension_routes(
|
||||
objects,
|
||||
args.manifest_dir,
|
||||
args.cas_dir,
|
||||
load_extension_inventory(args.extension_inventory),
|
||||
)
|
||||
compacted = {guid: compact_object(item) for guid, item in sorted(objects.items())}
|
||||
route_counts = Counter()
|
||||
for item in compacted.values():
|
||||
for kind in item["route_kind"]:
|
||||
route_counts[kind] += 1
|
||||
|
||||
report = {
|
||||
"schema": "onec_unified_object_route_index.v1",
|
||||
"dbnames": str(args.dbnames),
|
||||
"xml_index": str(args.xml_index),
|
||||
"manifest_dir": str(args.manifest_dir),
|
||||
"cas_dir": str(args.cas_dir),
|
||||
"config_dirs": [str(path) for path in args.config_dir],
|
||||
"object_count": len(compacted),
|
||||
"route_kind_counts": dict(route_counts.most_common()),
|
||||
"dbnames_role_counts": dict(role_counts.most_common()),
|
||||
"extension_payload_signature_counts": dict(signature_counts.most_common()),
|
||||
"objects": compacted,
|
||||
}
|
||||
args.output.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.output.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
print(json.dumps({"output": str(args.output), "objects": len(compacted)}, ensure_ascii=False))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,114 @@
|
||||
#!/usr/bin/env python3
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
ACTION_PRIORITY = {
|
||||
"run_before_after_learning_for_parameter": 10,
|
||||
"collect_allowed_values_and_smoke": 20,
|
||||
"run_before_after_learning_for_named_scalar": 30,
|
||||
"learn_reference_write_rule": 80,
|
||||
"do_not_generic_write": 100,
|
||||
}
|
||||
|
||||
|
||||
def load_rows(path: Path) -> tuple[str, list[dict[str, Any]]]:
|
||||
data = json.loads(path.read_text(encoding="utf-8"))
|
||||
schema = str(data.get("schema") or "")
|
||||
if schema == "onec_form_write_scalar_registry.v1":
|
||||
return schema, [row for row in data.get("scalars", []) if isinstance(row, dict)]
|
||||
if schema == "onec_form_write_enum_registry.v1":
|
||||
return schema, [row for row in data.get("properties", []) if isinstance(row, dict)]
|
||||
raise SystemExit(f"Unsupported registry schema: {schema or '<missing>'}")
|
||||
|
||||
|
||||
def example_selector(example: dict[str, Any]) -> dict[str, Any]:
|
||||
target = example.get("target")
|
||||
section = example.get("effective_section") or example.get("requested_section")
|
||||
selector: dict[str, Any] = {}
|
||||
if section == "commands":
|
||||
selector["command"] = target
|
||||
elif section == "attributes":
|
||||
selector["attribute"] = target
|
||||
else:
|
||||
selector["element"] = target
|
||||
return selector
|
||||
|
||||
|
||||
def learning_case(row: dict[str, Any], index: int) -> dict[str, Any]:
|
||||
examples = row.get("examples") if isinstance(row.get("examples"), list) else []
|
||||
example = examples[0] if examples and isinstance(examples[0], dict) else {}
|
||||
counts = row.get("counts") if isinstance(row.get("counts"), dict) else {}
|
||||
action = str(row.get("recommended_action") or row.get("risk") or "")
|
||||
return {
|
||||
"id": f"learn-{index:03d}",
|
||||
"action": action,
|
||||
"property": row.get("property"),
|
||||
"marker": row.get("marker"),
|
||||
"parameter_index": row.get("parameter_index"),
|
||||
"value_type": row.get("value_type"),
|
||||
"entries": counts.get("entries"),
|
||||
"observed_values": row.get("observed_values"),
|
||||
"selector": example_selector(example),
|
||||
"write_path": example.get("write_path"),
|
||||
"current_value": example.get("old"),
|
||||
"manual_step": {
|
||||
"target": example.get("target"),
|
||||
"section": example.get("effective_section") or example.get("requested_section"),
|
||||
"presentation": example.get("presentation"),
|
||||
"instruction": "Измени это свойство в конфигураторе на другое допустимое значение, сохрани форму, затем запусти capture_after/diff/infer.",
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Build ordered before/after learning plan from 1C write registries.")
|
||||
parser.add_argument("--registry", type=Path, required=True, help="Scalar or enum registry JSON path.")
|
||||
parser.add_argument("--output", type=Path, required=True, help="Output learning plan JSON path.")
|
||||
parser.add_argument("--limit", type=int, default=50, help="Maximum cases to include.")
|
||||
parser.add_argument(
|
||||
"--include-actions",
|
||||
nargs="*",
|
||||
default=["run_before_after_learning_for_parameter", "collect_allowed_values_and_smoke", "run_before_after_learning_for_named_scalar"],
|
||||
help="Recommended actions/risks to include.",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
schema, rows = load_rows(args.registry)
|
||||
include = set(args.include_actions)
|
||||
filtered = [row for row in rows if str(row.get("recommended_action") or row.get("risk") or "") in include]
|
||||
filtered.sort(
|
||||
key=lambda row: (
|
||||
ACTION_PRIORITY.get(str(row.get("recommended_action") or row.get("risk") or ""), 50),
|
||||
-int((row.get("counts") if isinstance(row.get("counts"), dict) else {}).get("entries") or 0),
|
||||
str(row.get("marker")),
|
||||
str(row.get("parameter_index")),
|
||||
str(row.get("property")),
|
||||
)
|
||||
)
|
||||
cases = [learning_case(row, index + 1) for index, row in enumerate(filtered[: args.limit])]
|
||||
result = {
|
||||
"schema": "onec_form_write_learning_plan.v1",
|
||||
"status": "ok",
|
||||
"source_registry": str(args.registry),
|
||||
"source_schema": schema,
|
||||
"counts": {
|
||||
"cases": len(cases),
|
||||
"available": len(filtered),
|
||||
"included_actions": sorted(include),
|
||||
},
|
||||
"workflow": ["capture_before", "manual_configurator_change", "capture_after", "diff", "infer_rule", "smoke_rule"],
|
||||
"cases": cases,
|
||||
}
|
||||
args.output.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.output.write_text(json.dumps(result, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
print(json.dumps({"schema": result["schema"], "status": "ok", "counts": result["counts"], "path": str(args.output)}, ensure_ascii=False, indent=2))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,161 @@
|
||||
#!/usr/bin/env python3
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
from collections import Counter, defaultdict
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
KNOWN_ENUM_VALUES: dict[str, dict[str, str]] = {
|
||||
"Вид": {
|
||||
"1": "Поле надписи",
|
||||
"2": "Поле ввода",
|
||||
"4": "Страница",
|
||||
"5": "Группа",
|
||||
"9": "Командная панель",
|
||||
"12": "Расширенная подсказка",
|
||||
"31": "Кнопка командной панели",
|
||||
"48": "Поле формы",
|
||||
"55": "Динамический список",
|
||||
"73": "Таблица формы",
|
||||
},
|
||||
"ПоложениеЗаголовка": {"0": "Авто", "1": "Верх", "2": "Нет"},
|
||||
"ПоложениеВКоманднойПанели": {"0": "Авто", "1": "В командной панели", "2": "В дополнительном подменю"},
|
||||
"Отображение": {"3": "Авто"},
|
||||
"ЦветФона": {"3": "Авто"},
|
||||
"ЦветТекста": {"3": "Авто"},
|
||||
"ЦветРамки": {"3": "Авто"},
|
||||
}
|
||||
|
||||
PROPERTY_ALIASES = {
|
||||
"group": "Группа",
|
||||
"id": "Идентификатор",
|
||||
"name": "Имя",
|
||||
"view": "Вид",
|
||||
"title": "Заголовок",
|
||||
"command_bar_location": "ПоложениеВКоманднойПанели",
|
||||
}
|
||||
|
||||
|
||||
def load_entries(report: dict[str, Any]) -> list[dict[str, Any]]:
|
||||
matrix = report.get("matrix") if isinstance(report.get("matrix"), dict) else {}
|
||||
entries = matrix.get("entries") if isinstance(matrix.get("entries"), list) else []
|
||||
return [entry for entry in entries if isinstance(entry, dict)]
|
||||
|
||||
|
||||
def property_key(entry: dict[str, Any]) -> tuple[str, str, str, str]:
|
||||
prop = entry.get("property") if isinstance(entry.get("property"), dict) else {}
|
||||
target = entry.get("requested_target") if isinstance(entry.get("requested_target"), dict) else {}
|
||||
raw_name = str(prop.get("semantic_name") or prop.get("canonical_property") or prop.get("property") or "")
|
||||
name = PROPERTY_ALIASES.get(raw_name, raw_name)
|
||||
marker = str(target.get("marker") or "")
|
||||
index = str(prop.get("parameter_index") if prop.get("parameter_index") is not None else "")
|
||||
value_type = str(prop.get("value_type") or "")
|
||||
return name, marker, index, value_type
|
||||
|
||||
|
||||
def risk_class(name: str, value_type: str) -> str:
|
||||
normalized = PROPERTY_ALIASES.get(name, name)
|
||||
if normalized in {"Идентификатор", "Имя"} or "маркер" in normalized.casefold():
|
||||
return "manual_only_identity_or_marker"
|
||||
if name == "Вид":
|
||||
return "structural_type_no_generic_write"
|
||||
if normalized == "Группа":
|
||||
return "reference_or_container_rule_required"
|
||||
if normalized in KNOWN_ENUM_VALUES:
|
||||
return "allowed_values_known_needs_smoke_rule"
|
||||
if value_type in {"enum_atom", "bool_or_enum_atom", "color_or_enum_atom"}:
|
||||
return "allowed_values_unknown"
|
||||
return "scalar_semantics_unknown"
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Build enum/scalar learning registry from 1C write matrix gaps.")
|
||||
parser.add_argument("--matrix-report", type=Path, required=True, help="Report produced by scripts/smoke_1c_write_matrix.py.")
|
||||
parser.add_argument("--output", type=Path, required=True, help="Output enum registry JSON path.")
|
||||
parser.add_argument("--sample-limit", type=int, default=8, help="Examples per enum/scalar group.")
|
||||
args = parser.parse_args()
|
||||
|
||||
report = json.loads(args.matrix_report.read_text(encoding="utf-8"))
|
||||
groups: dict[tuple[str, str, str, str], dict[str, Any]] = {}
|
||||
for entry in load_entries(report):
|
||||
prop = entry.get("property") if isinstance(entry.get("property"), dict) else {}
|
||||
if entry.get("can_smoke"):
|
||||
continue
|
||||
if entry.get("reason") not in {"value_type_not_smoke_safe", "identity_or_binding_property"}:
|
||||
continue
|
||||
value_type = str(prop.get("value_type") or "")
|
||||
if value_type not in {"enum_atom", "bool_or_enum_atom", "color_or_enum_atom", "integer_atom", "scalar"}:
|
||||
continue
|
||||
key = property_key(entry)
|
||||
name, marker, index, _ = key
|
||||
row = groups.setdefault(
|
||||
key,
|
||||
{
|
||||
"property": name,
|
||||
"marker": marker or None,
|
||||
"parameter_index": index or None,
|
||||
"value_type": value_type,
|
||||
"risk": risk_class(name, value_type),
|
||||
"observed_values": Counter(),
|
||||
"reasons": Counter(),
|
||||
"sections": Counter(),
|
||||
"examples": [],
|
||||
},
|
||||
)
|
||||
target = entry.get("requested_target") if isinstance(entry.get("requested_target"), dict) else {}
|
||||
effective = entry.get("effective_target") if isinstance(entry.get("effective_target"), dict) else {}
|
||||
old = "" if prop.get("old") is None else str(prop.get("old"))
|
||||
row["observed_values"][old] += 1
|
||||
row["reasons"][str(entry.get("reason") or "")] += 1
|
||||
row["sections"][str(effective.get("section") or "")] += 1
|
||||
if len(row["examples"]) < args.sample_limit:
|
||||
row["examples"].append(
|
||||
{
|
||||
"target": target.get("name") or target.get("path"),
|
||||
"requested_section": target.get("section"),
|
||||
"effective_section": effective.get("section"),
|
||||
"presentation": prop.get("presentation"),
|
||||
"semantic_name": prop.get("semantic_name"),
|
||||
"old": old,
|
||||
"write_path": prop.get("write_path"),
|
||||
"reason": entry.get("reason"),
|
||||
}
|
||||
)
|
||||
|
||||
properties = []
|
||||
for row in groups.values():
|
||||
known = KNOWN_ENUM_VALUES.get(str(row.get("property") or ""))
|
||||
properties.append(
|
||||
{
|
||||
**{key: value for key, value in row.items() if key not in {"observed_values", "reasons", "sections"}},
|
||||
"observed_values": dict(row["observed_values"].most_common()),
|
||||
"known_values": known,
|
||||
"reasons": dict(row["reasons"]),
|
||||
"sections": dict(row["sections"]),
|
||||
"counts": {"entries": sum(row["observed_values"].values()), "observed_values": len(row["observed_values"])},
|
||||
}
|
||||
)
|
||||
properties.sort(key=lambda item: (-int(item["counts"]["entries"]), str(item.get("property")), str(item.get("marker")), str(item.get("parameter_index"))))
|
||||
|
||||
result = {
|
||||
"schema": "onec_form_write_enum_registry.v1",
|
||||
"status": "ok",
|
||||
"source_report": str(args.matrix_report),
|
||||
"counts": {
|
||||
"groups": len(properties),
|
||||
"entries": sum(int(item["counts"]["entries"]) for item in properties),
|
||||
"by_risk": dict(Counter(str(item.get("risk")) for item in properties)),
|
||||
},
|
||||
"properties": properties,
|
||||
}
|
||||
args.output.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.output.write_text(json.dumps(result, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
print(json.dumps({"schema": result["schema"], "status": "ok", "counts": result["counts"], "path": str(args.output)}, ensure_ascii=False, indent=2))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,194 @@
|
||||
#!/usr/bin/env python3
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
from collections import Counter
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
SCALAR_VALUE_TYPES = {"integer_atom", "scalar"}
|
||||
SCALAR_REASONS = {"value_type_not_smoke_safe", "identity_or_binding_property"}
|
||||
|
||||
PROPERTY_ALIASES = {
|
||||
"group": "Группа",
|
||||
"id": "Идентификатор",
|
||||
"name": "Имя",
|
||||
"view": "Вид",
|
||||
"title": "Заголовок",
|
||||
"command_bar_location": "ПоложениеВКоманднойПанели",
|
||||
}
|
||||
|
||||
MANUAL_ONLY_PROPERTIES = {"Идентификатор", "Имя", "ПутьКДанным", "Данные", "Вид"}
|
||||
REFERENCE_PROPERTIES = {"Группа", "group"}
|
||||
|
||||
|
||||
def load_entries(report: dict[str, Any]) -> list[dict[str, Any]]:
|
||||
matrix = report.get("matrix") if isinstance(report.get("matrix"), dict) else {}
|
||||
entries = matrix.get("entries") if isinstance(matrix.get("entries"), list) else []
|
||||
return [entry for entry in entries if isinstance(entry, dict)]
|
||||
|
||||
|
||||
def normalize_property_name(prop: dict[str, Any]) -> str:
|
||||
raw = str(prop.get("semantic_name") or prop.get("canonical_property") or prop.get("property") or "")
|
||||
return PROPERTY_ALIASES.get(raw, raw)
|
||||
|
||||
|
||||
def scalar_bucket(entry: dict[str, Any]) -> str:
|
||||
prop = entry.get("property") if isinstance(entry.get("property"), dict) else {}
|
||||
target = entry.get("requested_target") if isinstance(entry.get("requested_target"), dict) else {}
|
||||
name = normalize_property_name(prop)
|
||||
marker = str(target.get("marker") or "")
|
||||
parameter_index = prop.get("parameter_index")
|
||||
semantic_status = str(prop.get("semantic_status") or "")
|
||||
|
||||
if name in MANUAL_ONLY_PROPERTIES or "маркер" in name.casefold():
|
||||
return "manual_only_identity_or_structural"
|
||||
if name in REFERENCE_PROPERTIES:
|
||||
return "reference_or_container_rule_required"
|
||||
if semantic_status and semantic_status != "unknown":
|
||||
return "semantic_scalar_needs_allowed_values"
|
||||
if parameter_index is not None:
|
||||
return f"learn_marker_{marker}_parameter_{parameter_index}"
|
||||
return "learn_named_scalar_semantics"
|
||||
|
||||
|
||||
def recommended_action(bucket: str) -> str:
|
||||
if bucket == "manual_only_identity_or_structural":
|
||||
return "do_not_generic_write"
|
||||
if bucket == "reference_or_container_rule_required":
|
||||
return "learn_reference_write_rule"
|
||||
if bucket == "semantic_scalar_needs_allowed_values":
|
||||
return "collect_allowed_values_and_smoke"
|
||||
if bucket.startswith("learn_marker_"):
|
||||
return "run_before_after_learning_for_parameter"
|
||||
return "run_before_after_learning_for_named_scalar"
|
||||
|
||||
|
||||
def make_key(entry: dict[str, Any]) -> tuple[str, str, str, str, str]:
|
||||
prop = entry.get("property") if isinstance(entry.get("property"), dict) else {}
|
||||
target = entry.get("requested_target") if isinstance(entry.get("requested_target"), dict) else {}
|
||||
effective = entry.get("effective_target") if isinstance(entry.get("effective_target"), dict) else {}
|
||||
return (
|
||||
scalar_bucket(entry),
|
||||
normalize_property_name(prop),
|
||||
str(target.get("marker") or effective.get("marker") or ""),
|
||||
str(prop.get("parameter_index") if prop.get("parameter_index") is not None else ""),
|
||||
str(prop.get("value_type") or ""),
|
||||
)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Build scalar learning registry from 1C write matrix gaps.")
|
||||
parser.add_argument("--matrix-report", type=Path, required=True, help="Report produced by scripts/smoke_1c_write_matrix.py.")
|
||||
parser.add_argument("--output", type=Path, required=True, help="Output scalar registry JSON path.")
|
||||
parser.add_argument("--sample-limit", type=int, default=8, help="Examples per scalar group.")
|
||||
args = parser.parse_args()
|
||||
|
||||
report = json.loads(args.matrix_report.read_text(encoding="utf-8"))
|
||||
groups: dict[tuple[str, str, str, str, str], dict[str, Any]] = {}
|
||||
skipped = Counter()
|
||||
|
||||
for entry in load_entries(report):
|
||||
prop = entry.get("property") if isinstance(entry.get("property"), dict) else {}
|
||||
if entry.get("can_smoke"):
|
||||
skipped["can_smoke"] += 1
|
||||
continue
|
||||
reason = str(entry.get("reason") or "")
|
||||
value_type = str(prop.get("value_type") or "")
|
||||
if reason not in SCALAR_REASONS:
|
||||
skipped[f"reason:{reason}"] += 1
|
||||
continue
|
||||
if value_type not in SCALAR_VALUE_TYPES:
|
||||
skipped[f"value_type:{value_type}"] += 1
|
||||
continue
|
||||
|
||||
key = make_key(entry)
|
||||
bucket, name, marker, parameter_index, _ = key
|
||||
row = groups.setdefault(
|
||||
key,
|
||||
{
|
||||
"bucket": bucket,
|
||||
"property": name,
|
||||
"marker": marker or None,
|
||||
"parameter_index": parameter_index or None,
|
||||
"value_type": value_type,
|
||||
"recommended_action": recommended_action(bucket),
|
||||
"observed_values": Counter(),
|
||||
"reasons": Counter(),
|
||||
"sections": Counter(),
|
||||
"type_names": Counter(),
|
||||
"examples": [],
|
||||
},
|
||||
)
|
||||
target = entry.get("requested_target") if isinstance(entry.get("requested_target"), dict) else {}
|
||||
effective = entry.get("effective_target") if isinstance(entry.get("effective_target"), dict) else {}
|
||||
old = "" if prop.get("old") is None else str(prop.get("old"))
|
||||
row["observed_values"][old] += 1
|
||||
row["reasons"][reason] += 1
|
||||
row["sections"][str(effective.get("section") or target.get("section") or "")] += 1
|
||||
row["type_names"][str(effective.get("type_name") or target.get("type_name") or "")] += 1
|
||||
if len(row["examples"]) < args.sample_limit:
|
||||
row["examples"].append(
|
||||
{
|
||||
"target": target.get("name") or target.get("path"),
|
||||
"requested_section": target.get("section"),
|
||||
"effective_section": effective.get("section"),
|
||||
"type_name": effective.get("type_name") or target.get("type_name"),
|
||||
"presentation": prop.get("presentation"),
|
||||
"semantic_name": prop.get("semantic_name"),
|
||||
"semantic_group": prop.get("semantic_group"),
|
||||
"old": old,
|
||||
"write_path": prop.get("write_path"),
|
||||
"reason": reason,
|
||||
}
|
||||
)
|
||||
|
||||
scalars = []
|
||||
for row in groups.values():
|
||||
scalars.append(
|
||||
{
|
||||
**{key: value for key, value in row.items() if key not in {"observed_values", "reasons", "sections", "type_names"}},
|
||||
"observed_values": dict(row["observed_values"].most_common()),
|
||||
"reasons": dict(row["reasons"]),
|
||||
"sections": dict(row["sections"]),
|
||||
"type_names": dict(row["type_names"].most_common()),
|
||||
"counts": {
|
||||
"entries": sum(row["observed_values"].values()),
|
||||
"observed_values": len(row["observed_values"]),
|
||||
"examples": len(row["examples"]),
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
scalars.sort(
|
||||
key=lambda item: (
|
||||
str(item.get("recommended_action")),
|
||||
-int(item["counts"]["entries"]),
|
||||
str(item.get("marker")),
|
||||
str(item.get("parameter_index")),
|
||||
str(item.get("property")),
|
||||
)
|
||||
)
|
||||
result = {
|
||||
"schema": "onec_form_write_scalar_registry.v1",
|
||||
"status": "ok",
|
||||
"source_report": str(args.matrix_report),
|
||||
"counts": {
|
||||
"groups": len(scalars),
|
||||
"entries": sum(int(item["counts"]["entries"]) for item in scalars),
|
||||
"by_action": dict(Counter(str(item.get("recommended_action")) for item in scalars)),
|
||||
"by_bucket": dict(Counter(str(item.get("bucket")) for item in scalars)),
|
||||
"skipped": dict(skipped),
|
||||
},
|
||||
"scalars": scalars,
|
||||
}
|
||||
args.output.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.output.write_text(json.dumps(result, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
print(json.dumps({"schema": result["schema"], "status": "ok", "counts": result["counts"], "path": str(args.output)}, ensure_ascii=False, indent=2))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,188 @@
|
||||
#!/usr/bin/env python3
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
from collections import Counter
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
SOURCE_AWARE_ROUTE_EVIDENCE: list[dict[str, Any]] = [
|
||||
{
|
||||
"key": "attributes|title|string|3.3.4.2.1|data_path_form_attribute_title",
|
||||
"requested_section": "items",
|
||||
"requested_name": "А",
|
||||
"requested_path": "1.25.24.24.24",
|
||||
"effective_section": "attributes",
|
||||
"effective_name": "А",
|
||||
"effective_path": "3.3",
|
||||
"source_kind": "data_path_form_attribute_title",
|
||||
"property": "title",
|
||||
"presentation": "Заголовок",
|
||||
"value_type": "string",
|
||||
"read_path": "1.25.24.24.24.4.2.1",
|
||||
"write_path": "3.3.4.2.1",
|
||||
"verification": "source_aware_readback",
|
||||
"status": "verified",
|
||||
"evidence": "metadata.write apply_and_rollback verified on upo_test; route smoke verified",
|
||||
},
|
||||
{
|
||||
"key": "attribute_fields|title|string|3.6.14.4.2.1|data_path_form_attribute_field_title",
|
||||
"requested_section": "items",
|
||||
"requested_name": "ТЗК1",
|
||||
"requested_path": "1.25.24.26.68",
|
||||
"effective_section": "attribute_fields",
|
||||
"effective_name": "К1",
|
||||
"effective_path": "3.6.14",
|
||||
"source_kind": "data_path_form_attribute_field_title",
|
||||
"property": "title",
|
||||
"presentation": "Заголовок",
|
||||
"value_type": "string",
|
||||
"read_path": "1.25.24.26.68.4.2.1",
|
||||
"write_path": "3.6.14.4.2.1",
|
||||
"verification": "source_aware_readback",
|
||||
"status": "verified",
|
||||
"evidence": "metadata.write apply_and_rollback verified on upo_test; route smoke verified",
|
||||
},
|
||||
{
|
||||
"key": "commands|title|string|5.3.3.2.1|local",
|
||||
"requested_section": "commands",
|
||||
"requested_name": "КомандаПример1",
|
||||
"requested_path": "5.3",
|
||||
"effective_section": "commands",
|
||||
"effective_name": "КомандаПример1",
|
||||
"effective_path": "5.3",
|
||||
"source_kind": "local",
|
||||
"property": "title",
|
||||
"presentation": "Заголовок",
|
||||
"value_type": "string",
|
||||
"read_path": "5.3.3.2.1",
|
||||
"write_path": "5.3.3.2.1",
|
||||
"verification": "source_aware_readback",
|
||||
"status": "verified",
|
||||
"evidence": "metadata.write apply_and_rollback verified on upo_test; route smoke verified",
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
def entry_key(item: dict[str, Any]) -> str:
|
||||
return "|".join(
|
||||
str(item.get(key) or "")
|
||||
for key in ("effective_section", "property", "value_type", "write_path", "source_kind")
|
||||
)
|
||||
|
||||
|
||||
def registry_entry(row: dict[str, Any]) -> dict[str, Any] | None:
|
||||
if row.get("status") != "verified":
|
||||
return None
|
||||
entry = row.get("entry") if isinstance(row.get("entry"), dict) else {}
|
||||
prop = entry.get("property") if isinstance(entry.get("property"), dict) else {}
|
||||
requested = entry.get("requested_target") if isinstance(entry.get("requested_target"), dict) else {}
|
||||
effective = entry.get("effective_target") if isinstance(entry.get("effective_target"), dict) else {}
|
||||
source = entry.get("effective_source") if isinstance(entry.get("effective_source"), dict) else {}
|
||||
item = {
|
||||
"requested_section": requested.get("section"),
|
||||
"requested_name": requested.get("name"),
|
||||
"requested_path": requested.get("path"),
|
||||
"effective_section": effective.get("section"),
|
||||
"effective_name": effective.get("name"),
|
||||
"effective_path": effective.get("path"),
|
||||
"source_kind": source.get("kind") or "local",
|
||||
"property": prop.get("semantic_name") or prop.get("canonical_property") or prop.get("property"),
|
||||
"canonical_property": prop.get("canonical_property") or prop.get("property"),
|
||||
"presentation": prop.get("presentation"),
|
||||
"semantic_name": prop.get("semantic_name"),
|
||||
"semantic_group": prop.get("semantic_group"),
|
||||
"semantic_source": prop.get("semantic_source"),
|
||||
"parameter_index": prop.get("parameter_index"),
|
||||
"value_type": prop.get("value_type"),
|
||||
"read_path": prop.get("read_path"),
|
||||
"write_path": prop.get("write_path"),
|
||||
"verification": prop.get("verification"),
|
||||
"status": "verified",
|
||||
}
|
||||
item["key"] = entry_key(item)
|
||||
return item
|
||||
|
||||
|
||||
def pattern_from_entry(item: dict[str, Any]) -> dict[str, Any]:
|
||||
pattern = {
|
||||
key: value
|
||||
for key, value in item.items()
|
||||
if key not in {"requested_name", "requested_path", "effective_name", "effective_path", "read_path", "evidence"}
|
||||
}
|
||||
pattern["verified_count"] = 0
|
||||
pattern["examples"] = []
|
||||
return pattern
|
||||
|
||||
|
||||
def add_pattern_example(patterns: dict[str, dict[str, Any]], item: dict[str, Any]) -> None:
|
||||
pattern = patterns.setdefault(item["key"], pattern_from_entry(item))
|
||||
pattern["verified_count"] = int(pattern.get("verified_count") or 0) + 1
|
||||
example = {
|
||||
"requested_name": item.get("requested_name"),
|
||||
"requested_path": item.get("requested_path"),
|
||||
"effective_name": item.get("effective_name"),
|
||||
"effective_path": item.get("effective_path"),
|
||||
"read_path": item.get("read_path"),
|
||||
}
|
||||
examples = pattern.setdefault("examples", [])
|
||||
if example not in examples and len(examples) < 5:
|
||||
examples.append(example)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Build compact verified write registry from a write-matrix smoke report.")
|
||||
parser.add_argument("--smoke-report", type=Path, required=True, help="Full report produced by scripts/smoke_1c_write_matrix.py.")
|
||||
parser.add_argument("--output", type=Path, required=True, help="Output registry JSON path.")
|
||||
parser.add_argument("--include-route-evidence", action="store_true", help="Append known source-aware route evidence from the learning case.")
|
||||
args = parser.parse_args()
|
||||
|
||||
data = json.loads(args.smoke_report.read_text(encoding="utf-8"))
|
||||
smoke = data.get("smoke") if isinstance(data.get("smoke"), dict) else {}
|
||||
entries: list[dict[str, Any]] = []
|
||||
patterns: dict[str, dict[str, Any]] = {}
|
||||
for row in smoke.get("results") or []:
|
||||
if not isinstance(row, dict):
|
||||
continue
|
||||
item = registry_entry(row)
|
||||
if not item:
|
||||
continue
|
||||
entries.append(item)
|
||||
add_pattern_example(patterns, item)
|
||||
|
||||
if args.include_route_evidence:
|
||||
existing = {item.get("key") for item in entries}
|
||||
for item in SOURCE_AWARE_ROUTE_EVIDENCE:
|
||||
if item["key"] not in existing:
|
||||
entries.append(dict(item))
|
||||
existing.add(item["key"])
|
||||
add_pattern_example(patterns, item)
|
||||
|
||||
registry = {
|
||||
"schema": "onec_form_write_verified_registry.v1",
|
||||
"status": "ok",
|
||||
"source_report": str(args.smoke_report),
|
||||
"adapter_report_path": smoke.get("path"),
|
||||
"base_id": data.get("base_id"),
|
||||
"table": data.get("table"),
|
||||
"file_name": data.get("file_name"),
|
||||
"counts": {
|
||||
"verified_entries": len(entries),
|
||||
"verified_patterns": len(patterns),
|
||||
"by_effective_section": dict(sorted(Counter(item.get("effective_section") for item in entries).items())),
|
||||
"by_property": dict(sorted(Counter(item.get("property") for item in entries).items())),
|
||||
"by_source_kind": dict(sorted(Counter(item.get("source_kind") for item in entries).items())),
|
||||
},
|
||||
"patterns": sorted(patterns.values(), key=lambda item: (str(item.get("effective_section")), str(item.get("property")), str(item.get("write_path")), str(item.get("source_kind")))),
|
||||
"entries": entries,
|
||||
}
|
||||
args.output.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.output.write_text(json.dumps(registry, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
print(json.dumps({"schema": registry["schema"], "status": "ok", "counts": registry["counts"], "path": str(args.output)}, ensure_ascii=False, indent=2))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,224 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Build a GUID index for a 1C XML configuration dump.
|
||||
|
||||
The index is intentionally mechanical: it records GUID occurrences in XML
|
||||
attributes and element text, plus the top metadata object declared by each
|
||||
file. It does not infer SQL table names or storage roles.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
GUID_RE = re.compile(
|
||||
r"[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}"
|
||||
)
|
||||
TOP_OBJECT_RE = re.compile(
|
||||
r"<(?P<kind>[A-Za-z][A-Za-z0-9]*)\s+[^>]*uuid=\"(?P<guid>"
|
||||
+ GUID_RE.pattern
|
||||
+ r")\"",
|
||||
re.S,
|
||||
)
|
||||
NAME_RE = re.compile(r"<Name>(?P<name>.*?)</Name>", re.S)
|
||||
SYNONYM_RE = re.compile(r"<(?:[A-Za-z0-9]+:)?content>(?P<content>.*?)</(?:[A-Za-z0-9]+:)?content>", re.S)
|
||||
|
||||
|
||||
def clean_xml_text(value: str) -> str:
|
||||
return repair_mojibake(
|
||||
value.replace(""", '"')
|
||||
.replace("'", "'")
|
||||
.replace("<", "<")
|
||||
.replace(">", ">")
|
||||
.replace("&", "&")
|
||||
.strip()
|
||||
)
|
||||
|
||||
|
||||
def repair_mojibake(value: str) -> str:
|
||||
if not value:
|
||||
return value
|
||||
for source_encoding in ("gbk", "cp1255", "cp1252", "latin1"):
|
||||
try:
|
||||
candidate = value.encode(source_encoding).decode("cp1251")
|
||||
except UnicodeError:
|
||||
continue
|
||||
candidate_cyrillic = sum(1 for char in candidate if "А" <= char <= "я" or char == "ё" or char == "Ё")
|
||||
value_cyrillic = sum(1 for char in value if "А" <= char <= "я" or char == "ё" or char == "Ё")
|
||||
if candidate_cyrillic > value_cyrillic:
|
||||
return repair_mojibake(candidate)
|
||||
cjk_count = sum(1 for char in value if "\u4e00" <= char <= "\u9fff")
|
||||
if cjk_count:
|
||||
try:
|
||||
candidate = value.encode("gbk").decode("cp1251")
|
||||
return repair_mojibake(candidate)
|
||||
except UnicodeError:
|
||||
pass
|
||||
cyrillic_count = sum(1 for char in value if "А" <= char <= "я" or char == "ё" or char == "Ё")
|
||||
suspicious_count = sum(1 for char in value if char in "ÐÑÂÃÄÅÆÇÈÉÊËÌÍÎÏÒÓÔÕÖרÙÚÛÜÝÞßàáâãäåæçèéêëìíîïòóôõö÷øùúûüýþÿ")
|
||||
if suspicious_count <= cyrillic_count:
|
||||
return value
|
||||
try:
|
||||
repaired = value.encode("latin1").decode("cp1251")
|
||||
except UnicodeError:
|
||||
return value
|
||||
repaired_cyrillic = sum(1 for char in repaired if "А" <= char <= "я" or char == "ё" or char == "Ё")
|
||||
return repaired if repaired_cyrillic > cyrillic_count else value
|
||||
|
||||
|
||||
def inspect_xml_file(path: Path, root_dir: Path, *, max_occurrences_per_file: int) -> dict[str, Any]:
|
||||
item: dict[str, Any] = {
|
||||
"path": repair_mojibake(str(path)),
|
||||
"relative_path": repair_mojibake(str(path.relative_to(root_dir))),
|
||||
"status": "ok",
|
||||
"root_tag": "",
|
||||
"top_object": None,
|
||||
"occurrences": [],
|
||||
}
|
||||
try:
|
||||
text = path.read_text(encoding="utf-8", errors="replace")
|
||||
except OSError as exc:
|
||||
item["status"] = "read_error"
|
||||
item["error"] = str(exc)
|
||||
return item
|
||||
|
||||
root_match = re.search(r"<(?P<tag>[A-Za-z][A-Za-z0-9]*)[\s>]", text)
|
||||
item["root_tag"] = root_match.group("tag") if root_match else ""
|
||||
|
||||
object_match = TOP_OBJECT_RE.search(text)
|
||||
if object_match:
|
||||
object_start = object_match.start()
|
||||
object_end = TOP_OBJECT_RE.search(text, object_match.end())
|
||||
object_fragment = text[object_start : object_end.start() if object_end else min(len(text), object_start + 500_000)]
|
||||
name_match = NAME_RE.search(object_fragment)
|
||||
synonym_match = SYNONYM_RE.search(object_fragment)
|
||||
top_object = {
|
||||
"guid": object_match.group("guid").lower(),
|
||||
"xml_kind": object_match.group("kind"),
|
||||
"name": clean_xml_text(name_match.group("name")) if name_match else "",
|
||||
"synonym": clean_xml_text(synonym_match.group("content")) if synonym_match else "",
|
||||
}
|
||||
else:
|
||||
top_object = None
|
||||
item["top_object"] = top_object
|
||||
|
||||
occurrences = []
|
||||
for match in GUID_RE.finditer(text):
|
||||
occurrences.append({"guid": match.group(0).lower(), "offset": match.start()})
|
||||
if len(occurrences) >= max_occurrences_per_file:
|
||||
item["occurrences"] = occurrences
|
||||
item["truncated"] = True
|
||||
return item
|
||||
item["occurrences"] = occurrences
|
||||
item["truncated"] = False
|
||||
return item
|
||||
|
||||
|
||||
def build_guid_map(files: list[dict[str, Any]], *, max_occurrences_per_guid: int) -> dict[str, Any]:
|
||||
guid_map: dict[str, Any] = {}
|
||||
for file_item in files:
|
||||
if file_item.get("status") != "ok":
|
||||
continue
|
||||
top_object = file_item.get("top_object")
|
||||
if top_object:
|
||||
guid = top_object["guid"]
|
||||
entry = guid_map.setdefault(guid, {"total_occurrences": 0, "top_objects": [], "occurrences": []})
|
||||
entry["top_objects"].append(
|
||||
{
|
||||
"path": file_item["path"],
|
||||
"relative_path": file_item["relative_path"],
|
||||
"xml_kind": top_object["xml_kind"],
|
||||
"name": top_object["name"],
|
||||
"synonym": top_object["synonym"],
|
||||
}
|
||||
)
|
||||
for occurrence in file_item.get("occurrences") or []:
|
||||
guid = occurrence["guid"]
|
||||
entry = guid_map.setdefault(guid, {"total_occurrences": 0, "top_objects": [], "occurrences": []})
|
||||
entry["total_occurrences"] += 1
|
||||
if len(entry["occurrences"]) < max_occurrences_per_guid:
|
||||
entry["occurrences"].append(
|
||||
{
|
||||
"path": file_item["path"],
|
||||
"relative_path": file_item["relative_path"],
|
||||
"offset": occurrence["offset"],
|
||||
}
|
||||
)
|
||||
return dict(sorted(guid_map.items()))
|
||||
|
||||
|
||||
def list_xml_paths(root: Path, max_relative_depth: int) -> list[Path]:
|
||||
if max_relative_depth <= 0:
|
||||
return sorted(root.rglob("*.xml"))
|
||||
result: list[Path] = []
|
||||
root_parts = len(root.parts)
|
||||
for current, dirs, files in os.walk(root):
|
||||
current_path = Path(current)
|
||||
relative_depth = len(current_path.parts) - root_parts
|
||||
if relative_depth >= max_relative_depth - 1:
|
||||
dirs[:] = []
|
||||
for file_name in files:
|
||||
if file_name.lower().endswith(".xml"):
|
||||
path = current_path / file_name
|
||||
if len(path.relative_to(root).parts) <= max_relative_depth:
|
||||
result.append(path)
|
||||
return sorted(result)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Build a GUID index for a 1C XML dump.")
|
||||
parser.add_argument("xml_root", type=Path)
|
||||
parser.add_argument("--output", type=Path, required=True)
|
||||
parser.add_argument("--max-occurrences-per-file", type=int, default=5000)
|
||||
parser.add_argument("--max-occurrences-per-guid", type=int, default=20)
|
||||
parser.add_argument(
|
||||
"--max-relative-depth",
|
||||
type=int,
|
||||
default=0,
|
||||
help="Only scan XML files whose relative path has at most this many parts; 0 scans all files.",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
xml_root = args.xml_root.resolve()
|
||||
paths = list_xml_paths(xml_root, args.max_relative_depth)
|
||||
files = [
|
||||
inspect_xml_file(path, xml_root, max_occurrences_per_file=args.max_occurrences_per_file)
|
||||
for path in paths
|
||||
]
|
||||
guid_map = build_guid_map(files, max_occurrences_per_guid=args.max_occurrences_per_guid)
|
||||
top_object_count = sum(1 for item in files if item.get("top_object"))
|
||||
parse_errors = sum(1 for item in files if item.get("status") != "ok")
|
||||
report = {
|
||||
"schema": "onec_xml_guid_index.v1",
|
||||
"xml_root": repair_mojibake(str(xml_root)),
|
||||
"file_count": len(files),
|
||||
"top_object_count": top_object_count,
|
||||
"parse_error_count": parse_errors,
|
||||
"guid_count": len(guid_map),
|
||||
"files": files,
|
||||
"guid_map": guid_map,
|
||||
}
|
||||
args.output.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.output.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"output": str(args.output),
|
||||
"files": len(files),
|
||||
"top_objects": top_object_count,
|
||||
"guids": len(guid_map),
|
||||
"parse_errors": parse_errors,
|
||||
},
|
||||
ensure_ascii=False,
|
||||
)
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,151 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import datetime as dt
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
DEFAULT_OUTPUT = ROOT / "reports" / "llm-artifact-manifest.json"
|
||||
DEFAULT_ARTIFACTS = {
|
||||
"models": ROOT / "models",
|
||||
"datasets_raw": ROOT / "datasets" / "raw",
|
||||
"datasets_prepared": ROOT / "datasets" / "prepared",
|
||||
"1c_rag_sources": ROOT / "plugins" / "1c" / "rag" / "sources",
|
||||
"1c_rag_official_raw": ROOT / "plugins" / "1c" / "rag" / "official-docs" / "raw",
|
||||
"1c_rag_official_normalized": ROOT / "plugins" / "1c" / "rag" / "official-docs" / "normalized",
|
||||
"1c_rag_official_media": ROOT / "plugins" / "1c" / "rag" / "official-docs" / "media",
|
||||
"1c_rag_official_static": ROOT / "plugins" / "1c" / "rag" / "official-docs" / "static",
|
||||
"1c_rag_official_start_links": ROOT / "plugins" / "1c" / "rag" / "official-docs" / "start-links.json",
|
||||
"1c_rag_official_platform_versions": ROOT / "plugins" / "1c" / "rag" / "official-docs" / "platform-versions.json",
|
||||
"1c_datasets_raw": ROOT / "plugins" / "1c" / "datasets" / "raw",
|
||||
"1c_datasets_prepared": ROOT / "plugins" / "1c" / "datasets" / "prepared",
|
||||
"1c_metadata_snapshots": ROOT / "plugins" / "1c" / "metadata" / "snapshots",
|
||||
"1c_training_raw": ROOT / "plugins" / "1c" / "training" / "raw",
|
||||
"1c_training_prepared": ROOT / "plugins" / "1c" / "training" / "prepared",
|
||||
}
|
||||
SECRET_NAME_MARKERS = ("cookie", "secret", "password", "passwd", "credential", ".env", ".pem", ".pfx", ".p12", ".key")
|
||||
KNOWN_SAFE_TOKEN_FILES = ("tokenizer", "special_tokens", "added_tokens")
|
||||
|
||||
|
||||
def sha256_file(path: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as handle:
|
||||
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def is_secret_like(path: Path) -> bool:
|
||||
lowered = path.name.lower()
|
||||
if any(marker in lowered for marker in KNOWN_SAFE_TOKEN_FILES):
|
||||
return False
|
||||
return any(marker in lowered for marker in SECRET_NAME_MARKERS)
|
||||
|
||||
|
||||
def iter_files(path: Path) -> list[Path]:
|
||||
if not path.exists():
|
||||
return []
|
||||
return sorted(item for item in path.rglob("*") if item.is_file())
|
||||
|
||||
|
||||
def summarize_artifact(name: str, path: Path, *, hash_files: bool, max_files: int) -> dict[str, Any]:
|
||||
files = iter_files(path)
|
||||
total_size = sum(item.stat().st_size for item in files)
|
||||
suffix_counts: dict[str, int] = {}
|
||||
secret_like = []
|
||||
file_records = []
|
||||
|
||||
for item in files:
|
||||
suffix = item.suffix.lower() or "<no_ext>"
|
||||
suffix_counts[suffix] = suffix_counts.get(suffix, 0) + 1
|
||||
if is_secret_like(item):
|
||||
secret_like.append(str(item.relative_to(path)))
|
||||
if len(file_records) < max_files:
|
||||
stat = item.stat()
|
||||
record = {
|
||||
"relative_path": str(item.relative_to(path)).replace("\\", "/"),
|
||||
"size_bytes": stat.st_size,
|
||||
"mtime_unix": int(stat.st_mtime),
|
||||
}
|
||||
if hash_files:
|
||||
record["sha256"] = sha256_file(item)
|
||||
file_records.append(record)
|
||||
|
||||
return {
|
||||
"name": name,
|
||||
"path": str(path),
|
||||
"exists": path.exists(),
|
||||
"is_dir": path.is_dir(),
|
||||
"file_count": len(files),
|
||||
"total_size_bytes": total_size,
|
||||
"suffix_counts": dict(sorted(suffix_counts.items())),
|
||||
"secret_like_files": secret_like[:50],
|
||||
"secret_like_truncated": len(secret_like) > 50,
|
||||
"files_sampled": len(file_records),
|
||||
"files_truncated": len(files) > max_files,
|
||||
"files": file_records,
|
||||
}
|
||||
|
||||
|
||||
def build_manifest(*, hash_files: bool, max_files: int, artifacts: dict[str, Path]) -> dict[str, Any]:
|
||||
records = [summarize_artifact(name, path, hash_files=hash_files, max_files=max_files) for name, path in artifacts.items()]
|
||||
return {
|
||||
"schema": "llm_artifact_manifest.v1",
|
||||
"created_at": dt.datetime.now(dt.UTC).isoformat(),
|
||||
"workspace_root": str(ROOT),
|
||||
"hash_files": hash_files,
|
||||
"max_files_per_artifact": max_files,
|
||||
"portable_policy": {
|
||||
"store_large_artifacts_outside_git": True,
|
||||
"mount_artifacts_as_docker_volumes": True,
|
||||
"do_not_store_secrets_in_artifact_dirs": True,
|
||||
},
|
||||
"docker_volume_recommendations": [
|
||||
{"host": "models", "container": "/app/models", "required_for": ["model-inference", "training"]},
|
||||
{"host": "plugins/1c/datasets", "container": "/app/plugins/1c/datasets", "required_for": ["1c-rag-api", "1c-adapter-api"]},
|
||||
{"host": "plugins/1c/rag/sources", "container": "/app/plugins/1c/rag/sources", "required_for": ["1c-rag-api"]},
|
||||
{"host": "plugins/1c/rag/official-docs", "container": "/app/plugins/1c/rag/official-docs", "required_for": ["1c-official-docs-ingest"]},
|
||||
{"host": "plugins/1c/metadata/snapshots", "container": "/app/plugins/1c/metadata/snapshots", "required_for": ["1c-adapter-api"]},
|
||||
],
|
||||
"counts": {
|
||||
"artifacts": len(records),
|
||||
"existing": sum(1 for item in records if item["exists"]),
|
||||
"files": sum(int(item["file_count"]) for item in records),
|
||||
"total_size_bytes": sum(int(item["total_size_bytes"]) for item in records),
|
||||
"secret_like_files": sum(len(item["secret_like_files"]) for item in records),
|
||||
},
|
||||
"artifacts": records,
|
||||
}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Build an inventory manifest for local LLM/RAG artifacts that are intentionally outside git.")
|
||||
parser.add_argument("--output", type=Path, default=DEFAULT_OUTPUT)
|
||||
parser.add_argument("--hash-files", action="store_true", help="Hash sampled files. This can be slow for large model files.")
|
||||
parser.add_argument("--max-files", type=int, default=500, help="Maximum file records stored per artifact.")
|
||||
args = parser.parse_args()
|
||||
|
||||
manifest = build_manifest(hash_files=args.hash_files, max_files=args.max_files, artifacts=DEFAULT_ARTIFACTS)
|
||||
args.output.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.output.write_text(json.dumps(manifest, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"output": str(args.output),
|
||||
"artifacts": manifest["counts"]["artifacts"],
|
||||
"files": manifest["counts"]["files"],
|
||||
"total_size_bytes": manifest["counts"]["total_size_bytes"],
|
||||
"secret_like_files": manifest["counts"]["secret_like_files"],
|
||||
},
|
||||
ensure_ascii=False,
|
||||
)
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,51 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
|
||||
from common import ROOT, iter_model_card_paths, read_yaml_mapping
|
||||
|
||||
|
||||
INDEX_PATH = ROOT / "registry" / "index.json"
|
||||
|
||||
|
||||
def simplify_card(path: Path, data: dict) -> dict:
|
||||
deployment = data.get("deployment") or {}
|
||||
return {
|
||||
"id": data.get("id"),
|
||||
"name": data.get("name"),
|
||||
"type": data.get("type"),
|
||||
"status": data.get("status"),
|
||||
"task": data.get("task") or [],
|
||||
"language": data.get("language") or [],
|
||||
"source": data.get("source"),
|
||||
"upstream_id": data.get("upstream_id"),
|
||||
"license": data.get("license"),
|
||||
"storage_path": data.get("storage_path"),
|
||||
"format": data.get("format"),
|
||||
"quantization": data.get("quantization"),
|
||||
"runtime": deployment.get("runtime"),
|
||||
"served_model_name": deployment.get("served_model_name"),
|
||||
"card_path": str(path.relative_to(ROOT)).replace("\\", "/"),
|
||||
}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
models = []
|
||||
for path in iter_model_card_paths():
|
||||
models.append(simplify_card(path, read_yaml_mapping(path)))
|
||||
|
||||
index = {
|
||||
"schema_version": 1,
|
||||
"models": models,
|
||||
}
|
||||
|
||||
INDEX_PATH.write_text(
|
||||
json.dumps(index, ensure_ascii=False, indent=2) + "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
print(f"Wrote {INDEX_PATH.relative_to(ROOT)} with {len(models)} model(s).")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,353 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import re
|
||||
from collections import defaultdict
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from urllib import request
|
||||
|
||||
|
||||
MARKER_RE = re.compile(r"^\d+-\d+$")
|
||||
|
||||
|
||||
def rpc(adapter_url: str, method: str, payload: dict[str, Any]) -> dict[str, Any]:
|
||||
body = json.dumps({"method": method, "payload": payload}, ensure_ascii=False).encode("utf-8")
|
||||
req = request.Request(
|
||||
f"{adapter_url.rstrip('/')}/rpc",
|
||||
data=body,
|
||||
headers={"Content-Type": "application/json; charset=utf-8"},
|
||||
method="POST",
|
||||
)
|
||||
with request.urlopen(req, timeout=240) as resp:
|
||||
return json.loads(resp.read().decode("utf-8", errors="replace"))
|
||||
|
||||
|
||||
def latest_configcas_rows(adapter_url: str, base_id: str, limit: int) -> list[dict[str, Any]]:
|
||||
result = rpc(
|
||||
adapter_url,
|
||||
"query.run",
|
||||
{
|
||||
"base_id": base_id,
|
||||
"diagnostic": True,
|
||||
"query": (
|
||||
f"SELECT TOP {int(limit)} FileName, DATALENGTH(BinaryData) AS Bytes, PartNo, Creation, Modified "
|
||||
"FROM ConfigCAS ORDER BY Modified DESC"
|
||||
),
|
||||
"timeout_seconds": 120,
|
||||
},
|
||||
)
|
||||
return result.get("rows") or []
|
||||
|
||||
|
||||
def template_summary(adapter_url: str, base_id: str, file_name: str, max_cells: int) -> dict[str, Any]:
|
||||
result = rpc(
|
||||
adapter_url,
|
||||
"templates.map",
|
||||
{
|
||||
"base_id": base_id,
|
||||
"table": "ConfigCAS",
|
||||
"file_name": file_name,
|
||||
"view": "summary",
|
||||
"sections": "cells,styles,named_areas,named_ranges,diagnostics",
|
||||
"max_cells": max_cells,
|
||||
"max_areas": 200,
|
||||
"timeout_seconds": 120,
|
||||
},
|
||||
)
|
||||
templates = result.get("templates") or []
|
||||
if not templates:
|
||||
return {}
|
||||
return (templates[0] or {}).get("structure") or {}
|
||||
|
||||
|
||||
def choose_latest_moxel(adapter_url: str, base_id: str, limit: int, max_cells: int, explicit_file_name: str | None) -> tuple[str, dict[str, Any], dict[str, Any]]:
|
||||
if explicit_file_name:
|
||||
structure = template_summary(adapter_url, base_id, explicit_file_name, max_cells)
|
||||
return explicit_file_name, {"FileName": explicit_file_name}, structure
|
||||
for row in latest_configcas_rows(adapter_url, base_id, limit):
|
||||
file_name = str(row.get("FileName") or "")
|
||||
byte_count = int(row.get("Bytes") or 0)
|
||||
if not file_name or byte_count <= 0 or byte_count > 20000:
|
||||
continue
|
||||
structure = template_summary(adapter_url, base_id, file_name, max_cells)
|
||||
if str(structure.get("format") or "") == "MOXCEL":
|
||||
return file_name, row, structure
|
||||
raise RuntimeError("Could not find a recent MOXCEL payload in ConfigCAS.")
|
||||
|
||||
|
||||
def normalize_cell(cell: dict[str, Any]) -> dict[str, Any]:
|
||||
return {
|
||||
"text": cell.get("text"),
|
||||
"cell_id": cell.get("cell_id"),
|
||||
"type_code": cell.get("type_code"),
|
||||
"one_based": cell.get("one_based"),
|
||||
"zero_based": cell.get("zero_based"),
|
||||
"parameter": cell.get("parameter"),
|
||||
"reference": cell.get("reference"),
|
||||
"source": cell.get("source"),
|
||||
}
|
||||
|
||||
|
||||
def normalize_style(item: dict[str, Any]) -> dict[str, Any]:
|
||||
next_record = item.get("next_moxel_record") if isinstance(item.get("next_moxel_record"), dict) else None
|
||||
style = item.get("style_evidence") if isinstance(item.get("style_evidence"), dict) else {}
|
||||
return {
|
||||
"text": item.get("text"),
|
||||
"cell_id": item.get("cell_id"),
|
||||
"type_code": item.get("type_code"),
|
||||
"tree_position": item.get("tree_position"),
|
||||
"next_moxel_record": next_record,
|
||||
"immediate_preceding_values": style.get("immediate_preceding_values"),
|
||||
"last_7_preceding_values": style.get("last_7_preceding_values"),
|
||||
}
|
||||
|
||||
|
||||
def build_snapshot(structure: dict[str, Any]) -> dict[str, Any]:
|
||||
cells = [normalize_cell(item) for item in (structure.get("cells") or []) if isinstance(item, dict) and item.get("text")]
|
||||
styles = [normalize_style(item) for item in (structure.get("cell_style_candidates") or []) if isinstance(item, dict) and item.get("text")]
|
||||
named_ranges = []
|
||||
for item in structure.get("named_range_candidates") or []:
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
named_ranges.append(
|
||||
{
|
||||
"name": item.get("name"),
|
||||
"kind": item.get("kind"),
|
||||
"tree_position": item.get("tree_position"),
|
||||
"range": item.get("range"),
|
||||
"raw_scalars": ((item.get("range_candidate") or {}).get("raw_scalars") if isinstance(item.get("range_candidate"), dict) else None),
|
||||
}
|
||||
)
|
||||
|
||||
cells_by_text: dict[str, list[dict[str, Any]]] = defaultdict(list)
|
||||
for cell in cells:
|
||||
cells_by_text[str(cell.get("text"))].append(cell)
|
||||
|
||||
styles_by_text: dict[str, list[dict[str, Any]]] = defaultdict(list)
|
||||
for style in styles:
|
||||
styles_by_text[str(style.get("text"))].append(style)
|
||||
|
||||
marker_matrix = []
|
||||
for text, entries in sorted(cells_by_text.items(), key=lambda pair: tuple(map(int, pair[0].split("-"))) if MARKER_RE.fullmatch(pair[0]) else (10**9, 10**9)):
|
||||
if not MARKER_RE.fullmatch(text):
|
||||
continue
|
||||
expected_row, expected_col = map(int, text.split("-"))
|
||||
style_entries = styles_by_text.get(text) or []
|
||||
row: dict[str, Any] = {
|
||||
"text": text,
|
||||
"expected_row": expected_row,
|
||||
"expected_col": expected_col,
|
||||
"cells": entries,
|
||||
"styles": style_entries,
|
||||
}
|
||||
if entries:
|
||||
first = entries[0]
|
||||
one_based = first.get("one_based") or {}
|
||||
decoded_row = one_based.get("row")
|
||||
decoded_col = one_based.get("column")
|
||||
row["decoded_row"] = decoded_row
|
||||
row["decoded_col"] = decoded_col
|
||||
row["row_ok"] = decoded_row == expected_row
|
||||
row["col_ok"] = decoded_col == expected_col
|
||||
if isinstance(decoded_col, int):
|
||||
row["col_delta"] = decoded_col - expected_col
|
||||
marker_matrix.append(row)
|
||||
|
||||
return {
|
||||
"counts": structure.get("counts") or {},
|
||||
"dimensions": structure.get("dimensions"),
|
||||
"named_areas": structure.get("named_areas") or [],
|
||||
"named_ranges": named_ranges,
|
||||
"cells": cells,
|
||||
"cell_styles": styles,
|
||||
"cells_by_text": dict(cells_by_text),
|
||||
"styles_by_text": dict(styles_by_text),
|
||||
"marker_matrix": marker_matrix,
|
||||
}
|
||||
|
||||
|
||||
def read_json(path: Path) -> dict[str, Any]:
|
||||
return json.loads(path.read_text(encoding="utf-8"))
|
||||
|
||||
|
||||
def diff_simple(before: Any, after: Any) -> dict[str, Any] | None:
|
||||
if before == after:
|
||||
return None
|
||||
return {"before": before, "after": after}
|
||||
|
||||
|
||||
def index_text_entries(entries: dict[str, list[dict[str, Any]]]) -> dict[str, list[dict[str, Any]]]:
|
||||
indexed: dict[str, list[dict[str, Any]]] = {}
|
||||
for text, items in entries.items():
|
||||
indexed[text] = sorted(items, key=lambda item: json.dumps(item, ensure_ascii=False, sort_keys=True))
|
||||
return indexed
|
||||
|
||||
|
||||
def build_diff(before: dict[str, Any], after: dict[str, Any]) -> dict[str, Any]:
|
||||
result: dict[str, Any] = {
|
||||
"counts": diff_simple(before.get("counts"), after.get("counts")),
|
||||
"dimensions": diff_simple(before.get("dimensions"), after.get("dimensions")),
|
||||
}
|
||||
|
||||
before_cells = index_text_entries(before.get("cells_by_text") or {})
|
||||
after_cells = index_text_entries(after.get("cells_by_text") or {})
|
||||
before_styles = index_text_entries(before.get("styles_by_text") or {})
|
||||
after_styles = index_text_entries(after.get("styles_by_text") or {})
|
||||
|
||||
changed_cells: dict[str, Any] = {}
|
||||
for text in sorted(set(before_cells) | set(after_cells)):
|
||||
if before_cells.get(text) != after_cells.get(text):
|
||||
changed_cells[text] = {"before": before_cells.get(text), "after": after_cells.get(text)}
|
||||
|
||||
changed_styles: dict[str, Any] = {}
|
||||
for text in sorted(set(before_styles) | set(after_styles)):
|
||||
if before_styles.get(text) != after_styles.get(text):
|
||||
changed_styles[text] = {"before": before_styles.get(text), "after": after_styles.get(text)}
|
||||
|
||||
before_markers = {item["text"]: item for item in before.get("marker_matrix") or [] if isinstance(item, dict) and item.get("text")}
|
||||
after_markers = {item["text"]: item for item in after.get("marker_matrix") or [] if isinstance(item, dict) and item.get("text")}
|
||||
changed_markers: dict[str, Any] = {}
|
||||
for text in sorted(set(before_markers) | set(after_markers), key=lambda value: tuple(map(int, value.split("-"))) if MARKER_RE.fullmatch(value) else (10**9, 10**9)):
|
||||
if before_markers.get(text) != after_markers.get(text):
|
||||
changed_markers[text] = {"before": before_markers.get(text), "after": after_markers.get(text)}
|
||||
|
||||
before_named = {f"{item.get('kind')}::{item.get('name')}": item for item in before.get("named_ranges") or [] if isinstance(item, dict)}
|
||||
after_named = {f"{item.get('kind')}::{item.get('name')}": item for item in after.get("named_ranges") or [] if isinstance(item, dict)}
|
||||
changed_named: dict[str, Any] = {}
|
||||
for key in sorted(set(before_named) | set(after_named)):
|
||||
if before_named.get(key) != after_named.get(key):
|
||||
changed_named[key] = {"before": before_named.get(key), "after": after_named.get(key)}
|
||||
|
||||
result["changed_cells_by_text"] = changed_cells
|
||||
result["changed_styles_by_text"] = changed_styles
|
||||
result["changed_marker_matrix"] = changed_markers
|
||||
result["changed_named_ranges"] = changed_named
|
||||
result["summary"] = {
|
||||
"changed_cell_texts": len(changed_cells),
|
||||
"changed_style_texts": len(changed_styles),
|
||||
"changed_markers": len(changed_markers),
|
||||
"changed_named_ranges": len(changed_named),
|
||||
}
|
||||
return result
|
||||
|
||||
|
||||
def latest_previous_snapshot(output_dir: Path, current_path: Path) -> Path | None:
|
||||
candidates = sorted(output_dir.glob("*.json"))
|
||||
filtered = [path for path in candidates if path.resolve() != current_path.resolve()]
|
||||
return filtered[-1] if filtered else None
|
||||
|
||||
|
||||
def render_markdown(snapshot: dict[str, Any], diff: dict[str, Any] | None, previous_path: Path | None) -> str:
|
||||
lines: list[str] = []
|
||||
lines.append("# 1C template probe snapshot")
|
||||
lines.append("")
|
||||
lines.append(f"- Base: `{snapshot['base_id']}`")
|
||||
lines.append(f"- File: `{snapshot['file_name']}`")
|
||||
lines.append(f"- Modified: `{snapshot.get('modified')}`")
|
||||
lines.append(f"- Bytes: `{snapshot.get('bytes')}`")
|
||||
lines.append(f"- Previous snapshot: `{previous_path.name}`" if previous_path else "- Previous snapshot: none")
|
||||
lines.append("")
|
||||
lines.append("## Marker matrix")
|
||||
lines.append("")
|
||||
lines.append("| Marker | Decoded | Result | Style tree |")
|
||||
lines.append("| --- | --- | --- | --- |")
|
||||
for item in snapshot["probe"]["marker_matrix"]:
|
||||
decoded = f"R{item.get('decoded_row')}C{item.get('decoded_col')}" if item.get("decoded_row") else "n/a"
|
||||
if item.get("row_ok") is True and item.get("col_ok") is True:
|
||||
result = "ok"
|
||||
elif item.get("decoded_row") is None:
|
||||
result = "style-only"
|
||||
else:
|
||||
result = f"row_ok={item.get('row_ok')} col_ok={item.get('col_ok')} delta={item.get('col_delta')}"
|
||||
style_tree = ""
|
||||
styles = item.get("styles") or []
|
||||
if styles:
|
||||
style_tree = ", ".join(str(style.get("tree_position")) for style in styles if style.get("tree_position"))
|
||||
lines.append(f"| `{item['text']}` | `{decoded}` | `{result}` | `{style_tree}` |")
|
||||
if diff:
|
||||
lines.append("")
|
||||
lines.append("## Diff summary")
|
||||
lines.append("")
|
||||
lines.append("```json")
|
||||
lines.append(json.dumps(diff.get("summary") or {}, ensure_ascii=False, indent=2))
|
||||
lines.append("```")
|
||||
return "\n".join(lines) + "\n"
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Capture and diff a live 1C MOXCEL template probe snapshot.")
|
||||
parser.add_argument("--adapter-url", default="http://docker-gpu.cin.su:8011")
|
||||
parser.add_argument("--base-id", default="upo_test")
|
||||
parser.add_argument("--file-name", help="Explicit ConfigCAS file name. If omitted, use the newest MOXCEL payload.")
|
||||
parser.add_argument("--scan-limit", type=int, default=30)
|
||||
parser.add_argument("--max-cells", type=int, default=500)
|
||||
parser.add_argument(
|
||||
"--output-dir",
|
||||
default=str(Path("Z:/codex/LLM/reports/1c-template-probes")),
|
||||
help="Directory for snapshot JSON/Markdown files.",
|
||||
)
|
||||
parser.add_argument("--output-json", help="Optional exact JSON output path. Overrides generated timestamped name.")
|
||||
parser.add_argument("--output-markdown", help="Optional exact Markdown output path. Overrides generated timestamped name.")
|
||||
parser.add_argument("--compare-to", help="Optional previous snapshot JSON path.")
|
||||
parser.add_argument("--label", default="latest", help="Short label appended to the output file name.")
|
||||
args = parser.parse_args()
|
||||
|
||||
output_dir = Path(args.output_dir)
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
file_name, row, structure = choose_latest_moxel(args.adapter_url, args.base_id, args.scan_limit, args.max_cells, args.file_name)
|
||||
snapshot = {
|
||||
"schema": "codex_1c_template_probe_snapshot.v1",
|
||||
"captured_at": datetime.utcnow().isoformat(timespec="seconds") + "Z",
|
||||
"adapter_url": args.adapter_url,
|
||||
"base_id": args.base_id,
|
||||
"file_name": file_name,
|
||||
"modified": row.get("Modified"),
|
||||
"bytes": row.get("Bytes"),
|
||||
"label": args.label,
|
||||
"probe": build_snapshot(structure),
|
||||
}
|
||||
|
||||
timestamp = datetime.utcnow().strftime("%Y%m%dT%H%M%SZ")
|
||||
stem = f"{args.base_id}_{args.label}_{timestamp}_{file_name[:8]}"
|
||||
json_path = Path(args.output_json) if args.output_json else output_dir / f"{stem}.json"
|
||||
md_path = Path(args.output_markdown) if args.output_markdown else output_dir / f"{stem}.md"
|
||||
json_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
md_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
previous_path = Path(args.compare_to) if args.compare_to else latest_previous_snapshot(output_dir, json_path)
|
||||
diff: dict[str, Any] | None = None
|
||||
if previous_path and previous_path.exists():
|
||||
previous = read_json(previous_path)
|
||||
diff = build_diff(previous.get("probe") or {}, snapshot.get("probe") or {})
|
||||
snapshot["diff"] = {
|
||||
"compare_to": str(previous_path),
|
||||
"summary": diff.get("summary") or {},
|
||||
}
|
||||
|
||||
json_path.write_text(json.dumps(snapshot, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
md_path.write_text(render_markdown(snapshot, diff, previous_path if previous_path and previous_path.exists() else None), encoding="utf-8")
|
||||
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"status": "ok",
|
||||
"json": str(json_path),
|
||||
"markdown": str(md_path),
|
||||
"file_name": file_name,
|
||||
"modified": row.get("Modified"),
|
||||
"bytes": row.get("Bytes"),
|
||||
"diff_summary": (diff.get("summary") if diff else None),
|
||||
},
|
||||
ensure_ascii=False,
|
||||
indent=2,
|
||||
)
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,403 @@
|
||||
#!/usr/bin/env python3
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
DEFAULT_REST_ADAPTER_URL = "http://docker-gpu.cin.su:8011"
|
||||
DEFAULT_MCP_URL = "http://docker.cin.su:8021"
|
||||
SAVED_STATE_TABLES = ("ConfigSave", "ConfigCASSave")
|
||||
|
||||
|
||||
def duplicate_values(values: list[str]) -> list[str]:
|
||||
seen: set[str] = set()
|
||||
duplicates: set[str] = set()
|
||||
for value in values:
|
||||
if value in seen:
|
||||
duplicates.add(value)
|
||||
seen.add(value)
|
||||
return sorted(duplicates)
|
||||
|
||||
|
||||
def trim_output(value: str, max_chars: int) -> tuple[str, bool]:
|
||||
if max_chars <= 0 or len(value) <= max_chars:
|
||||
return value, False
|
||||
return value[-max_chars:], True
|
||||
|
||||
|
||||
def parse_json_output(value: str) -> dict[str, Any] | None:
|
||||
text = value.strip()
|
||||
if not text or not text.startswith("{"):
|
||||
return None
|
||||
try:
|
||||
parsed = json.loads(text)
|
||||
except json.JSONDecodeError:
|
||||
return None
|
||||
return parsed if isinstance(parsed, dict) else None
|
||||
|
||||
|
||||
def parsed_summary(parsed: dict[str, Any] | None) -> dict[str, Any] | None:
|
||||
if not isinstance(parsed, dict):
|
||||
return None
|
||||
result: dict[str, Any] = {"schema": parsed.get("schema")}
|
||||
for key in ("passed", "status", "failures", "issues"):
|
||||
if key in parsed:
|
||||
value = parsed.get(key)
|
||||
if isinstance(value, list):
|
||||
result[f"{key}_count"] = len(value)
|
||||
else:
|
||||
result[key] = value
|
||||
checks = parsed.get("checks")
|
||||
if isinstance(checks, dict):
|
||||
result["checks_count"] = len(checks)
|
||||
if all(isinstance(value, bool) for value in checks.values()):
|
||||
result["checks_failed"] = sorted(key for key, value in checks.items() if value is not True)
|
||||
counts = parsed.get("counts")
|
||||
if isinstance(counts, dict):
|
||||
result["counts"] = counts
|
||||
bases = parsed.get("bases")
|
||||
if isinstance(bases, dict):
|
||||
result["bases_count"] = len(bases)
|
||||
result["bases"] = {
|
||||
str(base_id): compact_base_summary(base)
|
||||
for base_id, base in bases.items()
|
||||
if isinstance(base, dict)
|
||||
}
|
||||
strict_codes = parsed.get("strict_skip_failure_codes")
|
||||
if isinstance(strict_codes, list):
|
||||
result["strict_skip_failure_codes"] = strict_codes
|
||||
coverage_codes = parsed.get("coverage_failure_codes")
|
||||
if isinstance(coverage_codes, list):
|
||||
result["coverage_failure_codes"] = coverage_codes
|
||||
consistency_codes = parsed.get("consistency_failure_codes")
|
||||
if isinstance(consistency_codes, list):
|
||||
result["consistency_failure_codes"] = consistency_codes
|
||||
safety_codes = parsed.get("safety_failure_codes")
|
||||
if isinstance(safety_codes, list):
|
||||
result["safety_failure_codes"] = safety_codes
|
||||
rollback_safety_codes = parsed.get("rollback_safety_failure_codes")
|
||||
if isinstance(rollback_safety_codes, list):
|
||||
result["rollback_safety_failure_codes"] = rollback_safety_codes
|
||||
saved_state_diff_codes = parsed.get("saved_state_diff_failure_codes")
|
||||
if isinstance(saved_state_diff_codes, list):
|
||||
result["saved_state_diff_failure_codes"] = saved_state_diff_codes
|
||||
schema_codes = parsed.get("schema_failure_codes")
|
||||
if isinstance(schema_codes, list):
|
||||
result["schema_failure_codes"] = schema_codes
|
||||
identity_codes = parsed.get("identity_failure_codes")
|
||||
if isinstance(identity_codes, list):
|
||||
result["identity_failure_codes"] = identity_codes
|
||||
endpoint_codes = parsed.get("endpoint_failure_codes")
|
||||
if isinstance(endpoint_codes, list):
|
||||
result["endpoint_failure_codes"] = endpoint_codes
|
||||
duplicate_codes = parsed.get("duplicate_failure_codes")
|
||||
if isinstance(duplicate_codes, list):
|
||||
result["duplicate_failure_codes"] = duplicate_codes
|
||||
saved_state_codes = parsed.get("saved_state_failure_codes")
|
||||
if isinstance(saved_state_codes, list):
|
||||
result["saved_state_failure_codes"] = saved_state_codes
|
||||
saved_state_strict_readiness_codes = parsed.get("saved_state_strict_readiness_failure_codes")
|
||||
if isinstance(saved_state_strict_readiness_codes, list):
|
||||
result["saved_state_strict_readiness_failure_codes"] = saved_state_strict_readiness_codes
|
||||
saved_state_copy_plan_codes = parsed.get("saved_state_copy_plan_failure_codes")
|
||||
if isinstance(saved_state_copy_plan_codes, list):
|
||||
result["saved_state_copy_plan_failure_codes"] = saved_state_copy_plan_codes
|
||||
saved_state_table_codes = parsed.get("saved_state_table_failure_codes")
|
||||
if isinstance(saved_state_table_codes, list):
|
||||
result["saved_state_table_failure_codes"] = saved_state_table_codes
|
||||
staleness_codes = parsed.get("staleness_failure_codes")
|
||||
if isinstance(staleness_codes, list):
|
||||
result["staleness_failure_codes"] = staleness_codes
|
||||
return result
|
||||
|
||||
|
||||
def compact_base_summary(base: dict[str, Any]) -> dict[str, Any]:
|
||||
reports = base.get("reports") if isinstance(base.get("reports"), dict) else {}
|
||||
summary: dict[str, Any] = {"passed": base.get("passed")}
|
||||
selector_chain: dict[str, Any] = {}
|
||||
write_plan_safety: dict[str, Any] = {}
|
||||
write_rollback_safety: dict[str, Any] = {}
|
||||
saved_state_diff: dict[str, Any] = {}
|
||||
saved_state: dict[str, Any] = {}
|
||||
|
||||
for name, report in reports.items():
|
||||
if not isinstance(report, dict):
|
||||
continue
|
||||
if name.startswith("selector_chain_"):
|
||||
transport = name.removeprefix("selector_chain_")
|
||||
selector_chain[transport] = {
|
||||
"passed": report.get("passed"),
|
||||
"base_id": report.get("base_id"),
|
||||
"transport": report.get("transport"),
|
||||
"endpoint_url": report.get("endpoint_url"),
|
||||
"resolve_overrides_status": report.get("resolve_overrides_status"),
|
||||
"write_plan_evidence": report.get("write_plan_evidence"),
|
||||
"next_method": report.get("next_method"),
|
||||
"saved_state_status": report.get("saved_state_status"),
|
||||
"saved_state_modules": report.get("saved_state_modules"),
|
||||
"write_plan_target": report.get("write_plan_target"),
|
||||
"composition_status": report.get("composition_status"),
|
||||
"composed": report.get("composed"),
|
||||
}
|
||||
elif name.startswith("write_plan_safety_"):
|
||||
transport = name.removeprefix("write_plan_safety_")
|
||||
write_plan_safety[transport] = {
|
||||
"status": report.get("status"),
|
||||
"checks": report.get("checks"),
|
||||
"base_id": report.get("base_id"),
|
||||
"transport": report.get("transport"),
|
||||
"endpoint_url": report.get("endpoint_url"),
|
||||
}
|
||||
elif name.startswith("write_rollback_safety_"):
|
||||
transport = name.removeprefix("write_rollback_safety_")
|
||||
write_rollback_safety[transport] = {
|
||||
"status": report.get("status"),
|
||||
"checks": report.get("checks"),
|
||||
"base_id": report.get("base_id"),
|
||||
"transport": report.get("transport"),
|
||||
"endpoint_url": report.get("endpoint_url"),
|
||||
}
|
||||
elif name.startswith("saved_state_diff_"):
|
||||
transport = name.removeprefix("saved_state_diff_")
|
||||
saved_state_diff[transport] = {
|
||||
"status": report.get("status"),
|
||||
"checks": report.get("checks"),
|
||||
"base_id": report.get("base_id"),
|
||||
"transport": report.get("transport"),
|
||||
"endpoint_url": report.get("endpoint_url"),
|
||||
"saved_state_table": report.get("saved_state_table"),
|
||||
"diff_status": report.get("diff_status"),
|
||||
"needs_prepare": report.get("needs_prepare"),
|
||||
}
|
||||
elif name == "saved_state_form_write":
|
||||
saved_state["form"] = {
|
||||
"passed": report.get("passed"),
|
||||
"status": report.get("status"),
|
||||
"base_id": report.get("base_id"),
|
||||
"table": report.get("table"),
|
||||
"routes": report.get("routes"),
|
||||
"preflight_status": report.get("preflight_status"),
|
||||
"preflight_counts": report.get("preflight_counts"),
|
||||
}
|
||||
elif name == "saved_state_module_write":
|
||||
saved_state["module"] = {
|
||||
"status": report.get("status"),
|
||||
"base_id": report.get("base_id"),
|
||||
"table": report.get("table"),
|
||||
"module_ref": report.get("module_ref"),
|
||||
"write_plan_allowed": report.get("write_plan_allowed"),
|
||||
"preflight_status": report.get("preflight_status"),
|
||||
"preflight_counts": report.get("preflight_counts"),
|
||||
}
|
||||
elif name == "saved_state_strict_readiness":
|
||||
saved_state["strict_readiness"] = {
|
||||
"status": report.get("status"),
|
||||
"ready": report.get("ready"),
|
||||
"base_id": report.get("base_id"),
|
||||
"table": report.get("table"),
|
||||
"tables": report.get("tables"),
|
||||
"saved_state_rows": report.get("saved_state_rows"),
|
||||
"forms": report.get("forms"),
|
||||
"modules": report.get("modules"),
|
||||
}
|
||||
elif name == "saved_state_copy_plan":
|
||||
source_family = report.get("source_family") if isinstance(report.get("source_family"), dict) else {}
|
||||
saved_state["copy_plan"] = {
|
||||
"status": report.get("status"),
|
||||
"ready_to_copy": report.get("ready_to_copy"),
|
||||
"base_id": report.get("base_id"),
|
||||
"target_table": report.get("target_table"),
|
||||
"source_family": {
|
||||
"expected_source_table": source_family.get("expected_source_table"),
|
||||
"source_tables": source_family.get("source_tables"),
|
||||
"valid": source_family.get("valid"),
|
||||
},
|
||||
"source_rows": report.get("source_rows"),
|
||||
"found_source_storage_rows": report.get("found_source_storage_rows"),
|
||||
"target_collision_status": report.get("target_collision_status"),
|
||||
"target_collision_rows": report.get("target_collision_rows"),
|
||||
}
|
||||
elif name in {"saved_state_prepare_sql", "saved_state_cleanup_sql"}:
|
||||
key = "prepare_sql" if name == "saved_state_prepare_sql" else "cleanup_sql"
|
||||
saved_state[key] = {
|
||||
"status": report.get("status"),
|
||||
"base_id": report.get("base_id"),
|
||||
"table": report.get("table"),
|
||||
"source_table": report.get("source_table"),
|
||||
"read_only": report.get("read_only"),
|
||||
"sql_write_performed": report.get("sql_write_performed"),
|
||||
}
|
||||
|
||||
if selector_chain:
|
||||
summary["selector_chain"] = selector_chain
|
||||
if write_plan_safety:
|
||||
summary["write_plan_safety"] = write_plan_safety
|
||||
if write_rollback_safety:
|
||||
summary["write_rollback_safety"] = write_rollback_safety
|
||||
if saved_state_diff:
|
||||
summary["saved_state_diff"] = saved_state_diff
|
||||
if saved_state:
|
||||
summary["saved_state"] = saved_state
|
||||
return summary
|
||||
|
||||
|
||||
def run(command: list[str], *, stream: bool, max_output_chars: int) -> dict[str, Any]:
|
||||
label = " ".join(command)
|
||||
if stream:
|
||||
print(f"\n== {label}", flush=True)
|
||||
result = subprocess.run(command, cwd=ROOT, text=True, check=False)
|
||||
return {"label": label, "command": command, "returncode": result.returncode}
|
||||
result = subprocess.run(command, cwd=ROOT, text=True, capture_output=True, check=False)
|
||||
parsed = parse_json_output(result.stdout)
|
||||
stdout, stdout_truncated = trim_output(result.stdout, max_output_chars)
|
||||
stderr, stderr_truncated = trim_output(result.stderr, max_output_chars)
|
||||
report = {
|
||||
"label": label,
|
||||
"command": command,
|
||||
"returncode": result.returncode,
|
||||
"stdout": stdout,
|
||||
"stderr": stderr,
|
||||
"stdout_truncated": stdout_truncated,
|
||||
"stderr_truncated": stderr_truncated,
|
||||
}
|
||||
summary = parsed_summary(parsed)
|
||||
if summary is not None:
|
||||
report["parsed"] = summary
|
||||
return report
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Run offline/static checks for the 1C adapter verification stack.")
|
||||
parser.add_argument("--base-id", nargs="+", default=["upo_test"], help="Base id(s) whose persisted verify reports should be checked.")
|
||||
parser.add_argument("--skip-persisted-reports", action="store_true", help="Skip checking reports/1c-sql/<base-id> artifacts.")
|
||||
parser.add_argument("--require-saved-state-write-smoke", action="store_true", help="Require persisted saved-state smoke reports to contain real write checks.")
|
||||
parser.add_argument("--require-selector-chain-write-plan-composition", action="store_true", help="Require persisted selector-chain reports to contain composed metadata.write.plan coverage.")
|
||||
parser.add_argument("--rest-adapter-url", default=DEFAULT_REST_ADAPTER_URL, help="Expected REST adapter endpoint_url in persisted reports.")
|
||||
parser.add_argument("--mcp-url", default=DEFAULT_MCP_URL, help="Expected MCP proxy endpoint_url in persisted reports.")
|
||||
parser.add_argument("--saved-state-table", choices=SAVED_STATE_TABLES, default="ConfigSave", help="Expected saved-state target table in persisted copy-plan/form/module reports.")
|
||||
parser.add_argument("--max-report-age-seconds", type=int, help="Fail if any checked persisted report file is older than this many seconds.")
|
||||
parser.add_argument("--report", type=Path, help="Optional JSON report path.")
|
||||
parser.add_argument("--json", action="store_true", help="Print JSON summary without streaming child command output.")
|
||||
parser.add_argument("--max-output-chars", type=int, default=4000, help="Maximum stdout/stderr characters retained per child command in JSON mode.")
|
||||
args = parser.parse_args()
|
||||
|
||||
duplicate_base_ids = duplicate_values(args.base_id)
|
||||
if duplicate_base_ids:
|
||||
report = {
|
||||
"schema": "onec_adapter_verification_stack_check.v1",
|
||||
"passed": False,
|
||||
"base_id": args.base_id[0] if len(args.base_id) == 1 else None,
|
||||
"base_ids": args.base_id,
|
||||
"rest_adapter_url": args.rest_adapter_url,
|
||||
"mcp_url": args.mcp_url,
|
||||
"saved_state_table": args.saved_state_table,
|
||||
"checks": [],
|
||||
"failures": [{"code": "duplicate_base_id", "base_ids": duplicate_base_ids}],
|
||||
}
|
||||
if args.report:
|
||||
report_path = args.report if args.report.is_absolute() else ROOT / args.report
|
||||
report_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
report_path.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
if args.json:
|
||||
print(json.dumps(report, ensure_ascii=False, indent=2))
|
||||
else:
|
||||
print("Duplicate --base-id value(s): " + ", ".join(duplicate_base_ids), file=sys.stderr)
|
||||
return 1
|
||||
|
||||
commands = [
|
||||
[
|
||||
sys.executable,
|
||||
"-m",
|
||||
"py_compile",
|
||||
"scripts/check_1c_adapter_verification_stack.py",
|
||||
"scripts/check_1c_verify_reports.py",
|
||||
"scripts/check_powershell_scripts.py",
|
||||
"scripts/smoke_1c_mcp_selector_chain.py",
|
||||
"scripts/smoke_1c_write_plan_safety.py",
|
||||
"scripts/smoke_1c_write_preflight.py",
|
||||
"scripts/smoke_1c_write_rollback_safety.py",
|
||||
"scripts/smoke_1c_saved_state_diff.py",
|
||||
"scripts/smoke_1c_saved_state_changes.py",
|
||||
"scripts/smoke_1c_saved_state_write_routes.py",
|
||||
"scripts/smoke_1c_saved_state_module_write.py",
|
||||
"scripts/check_1c_saved_state_strict_readiness.py",
|
||||
"scripts/plan_1c_saved_state_copy.py",
|
||||
"scripts/prepare_1c_saved_state_copy_sql.py",
|
||||
"scripts/prepare_1c_saved_state_cleanup_sql.py",
|
||||
"scripts/verify_1c_saved_state_copy.py",
|
||||
],
|
||||
[sys.executable, "scripts/check_powershell_scripts.py"],
|
||||
[sys.executable, "scripts/check_1c_mcp_adapter_contract.py", "--json"],
|
||||
[sys.executable, "scripts/check_1c_extension_action_contract.py", "--print"],
|
||||
[sys.executable, "scripts/check_1c_write_plan_contract.py", "--print"],
|
||||
[sys.executable, "scripts/smoke_1c_mcp_selector_chain.py", "--json", "--no-report"],
|
||||
[sys.executable, "scripts/check_1c_verify_reports.py", "--self-test", "--json"],
|
||||
]
|
||||
if not args.skip_persisted_reports:
|
||||
report_command = [
|
||||
sys.executable,
|
||||
"scripts/check_1c_verify_reports.py",
|
||||
"--base-id",
|
||||
*args.base_id,
|
||||
"--rest-adapter-url",
|
||||
args.rest_adapter_url,
|
||||
"--mcp-url",
|
||||
args.mcp_url,
|
||||
"--saved-state-table",
|
||||
args.saved_state_table,
|
||||
"--json",
|
||||
]
|
||||
if args.require_saved_state_write_smoke:
|
||||
report_command.append("--require-saved-state-write-smoke")
|
||||
if args.require_selector_chain_write_plan_composition:
|
||||
report_command.append("--require-selector-chain-write-plan-composition")
|
||||
if args.max_report_age_seconds is not None:
|
||||
report_command.extend(["--max-report-age-seconds", str(args.max_report_age_seconds)])
|
||||
commands.append(report_command)
|
||||
|
||||
results: list[dict[str, Any]] = []
|
||||
failures: list[str] = []
|
||||
for command in commands:
|
||||
result = run(command, stream=not args.json, max_output_chars=args.max_output_chars)
|
||||
results.append(result)
|
||||
if result["returncode"] != 0:
|
||||
failures.append(str(result["label"]))
|
||||
|
||||
report = {
|
||||
"schema": "onec_adapter_verification_stack_check.v1",
|
||||
"passed": not failures,
|
||||
"base_id": args.base_id[0] if len(args.base_id) == 1 else None,
|
||||
"base_ids": args.base_id,
|
||||
"rest_adapter_url": args.rest_adapter_url,
|
||||
"mcp_url": args.mcp_url,
|
||||
"saved_state_table": args.saved_state_table,
|
||||
"checks": results,
|
||||
"failures": failures,
|
||||
}
|
||||
if args.report:
|
||||
report_path = args.report if args.report.is_absolute() else ROOT / args.report
|
||||
report_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
report_path.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
|
||||
if args.json:
|
||||
print(json.dumps(report, ensure_ascii=False, indent=2))
|
||||
return 0 if report["passed"] else 1
|
||||
|
||||
if failures:
|
||||
print("\n1C adapter verification stack check failed:", file=sys.stderr)
|
||||
for failure in failures:
|
||||
print(f"- {failure}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
print("\n1C adapter verification stack checks passed.")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,119 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "scripts"))
|
||||
|
||||
from build_1c_agent_intake import build_intake # noqa: E402
|
||||
|
||||
|
||||
DEFAULT_INDEX = ROOT / "reports" / "1c-sql" / "upo" / "unified-object-route-index.json"
|
||||
|
||||
|
||||
CASES = [
|
||||
{
|
||||
"id": "docs_intake_uses_official_scope",
|
||||
"question": "Как работает событие ПриОткрытии формы?",
|
||||
"expected_route": "docs_rag",
|
||||
"expected_code_allowed": True,
|
||||
"expected_safe_scope": "official_1c_docs",
|
||||
},
|
||||
{
|
||||
"id": "example_fact_requires_confirmation",
|
||||
"question": "В примере RAG есть реквизит Артикул у справочника Номенклатура. Напиши код для текущей базы.",
|
||||
"expected_route": "mixed_docs_and_current_config",
|
||||
"expected_examples_are_facts": False,
|
||||
"expected_confirmed_path": "Справочник.Номенклатура.Артикул",
|
||||
"expected_code_allowed": True,
|
||||
},
|
||||
{
|
||||
"id": "missing_fact_blocks_code",
|
||||
"question": "Напиши код для текущей базы: заполни Справочник.Номенклатура.ВыдуманныйРеквизит.",
|
||||
"expected_route": "current_config_fact",
|
||||
"expected_unresolved_path": "Справочник.Номенклатура.ВыдуманныйРеквизит",
|
||||
"expected_code_allowed": False,
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
def run_case(case: dict, *, index: Path, view: str) -> dict:
|
||||
intake = build_intake(case["question"], index=index, view=view)
|
||||
failures = []
|
||||
route = (intake.get("route") or {}).get("decision") or {}
|
||||
policy = intake.get("answer_policy") or {}
|
||||
source_policy = intake.get("source_policy") or {}
|
||||
facts = intake.get("facts") or {}
|
||||
|
||||
if route.get("route") != case.get("expected_route"):
|
||||
failures.append({"code": "route_mismatch", "expected": case.get("expected_route"), "actual": route.get("route")})
|
||||
if policy.get("code_generation_allowed") is not case.get("expected_code_allowed"):
|
||||
failures.append({"code": "code_policy_mismatch", "expected": case.get("expected_code_allowed"), "actual": policy.get("code_generation_allowed")})
|
||||
if case.get("expected_safe_scope") and policy.get("safe_rag_scope") != case.get("expected_safe_scope"):
|
||||
failures.append({"code": "safe_scope_mismatch", "expected": case.get("expected_safe_scope"), "actual": policy.get("safe_rag_scope")})
|
||||
if "expected_examples_are_facts" in case and source_policy.get("examples_are_current_facts") is not case["expected_examples_are_facts"]:
|
||||
failures.append({"code": "examples_policy_mismatch", "expected": case["expected_examples_are_facts"], "actual": source_policy.get("examples_are_current_facts")})
|
||||
|
||||
confirmed_paths = {row.get("path") for row in facts.get("confirmed") or []}
|
||||
unresolved_paths = {row.get("path") for row in facts.get("unresolved") or []}
|
||||
if case.get("expected_confirmed_path") and case["expected_confirmed_path"] not in confirmed_paths:
|
||||
failures.append({"code": "confirmed_path_missing", "expected": case["expected_confirmed_path"], "actual": sorted(confirmed_paths)})
|
||||
if case.get("expected_unresolved_path") and case["expected_unresolved_path"] not in unresolved_paths:
|
||||
failures.append({"code": "unresolved_path_missing", "expected": case["expected_unresolved_path"], "actual": sorted(unresolved_paths)})
|
||||
|
||||
return {
|
||||
"id": case["id"],
|
||||
"status": "passed" if not failures else "failed",
|
||||
"question": case["question"],
|
||||
"failures": failures,
|
||||
"summary": {
|
||||
"route": route.get("route"),
|
||||
"code_generation_allowed": policy.get("code_generation_allowed"),
|
||||
"safe_rag_scope": policy.get("safe_rag_scope"),
|
||||
"confirmed_paths": sorted(confirmed_paths),
|
||||
"unresolved_paths": sorted(unresolved_paths),
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def run_check(index: Path, *, view: str) -> dict:
|
||||
if not index.exists():
|
||||
return {
|
||||
"schema": "onec_agent_intake_check.v1",
|
||||
"status": "failed",
|
||||
"error": f"route index not found: {index}",
|
||||
"cases": [],
|
||||
}
|
||||
cases = [run_case(case, index=index, view=view) for case in CASES]
|
||||
return {
|
||||
"schema": "onec_agent_intake_check.v1",
|
||||
"status": "ok" if all(case["status"] == "passed" for case in cases) else "failed",
|
||||
"index": str(index),
|
||||
"view": view,
|
||||
"case_count": len(cases),
|
||||
"cases": cases,
|
||||
}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Check 1C agent intake behavior.")
|
||||
parser.add_argument("--index", type=Path, default=DEFAULT_INDEX)
|
||||
parser.add_argument("--view", choices=["effective", "base"], default="effective")
|
||||
parser.add_argument("--output", type=Path)
|
||||
parser.add_argument("--print", action="store_true")
|
||||
args = parser.parse_args()
|
||||
|
||||
report = run_check(args.index, view=args.view)
|
||||
if args.output:
|
||||
args.output.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.output.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
if args.print or not args.output:
|
||||
print(json.dumps(report, ensure_ascii=False, indent=2))
|
||||
return 0 if report["status"] == "ok" else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,72 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Offline contract check for conservative BSL symbol resolution."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "scripts"))
|
||||
from resolve_1c_bsl_symbol import load_json, resolve_symbol # noqa: E402
|
||||
|
||||
|
||||
METADATA = ROOT / "plugins" / "1c" / "metadata" / "examples" / "metadata-v2.example.json"
|
||||
MODULES = ROOT / "plugins" / "1c" / "metadata" / "examples" / "bsl-modules.example.json"
|
||||
|
||||
|
||||
def example_resolve(expression: str) -> dict[str, Any]:
|
||||
return resolve_symbol(
|
||||
load_json(METADATA),
|
||||
load_json(MODULES),
|
||||
expression=expression,
|
||||
module_id="catalog.Номенклатура.object",
|
||||
object_kind="catalog",
|
||||
object_name="Номенклатура",
|
||||
routine_name="ПередЗаписью",
|
||||
)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Check BSL symbol resolver safety contract.")
|
||||
parser.add_argument("--print", action="store_true")
|
||||
args = parser.parse_args()
|
||||
|
||||
full_path = example_resolve("Справочник.Номенклатура.Артикул")
|
||||
context_member = example_resolve("Наименование")
|
||||
parameter = example_resolve("Отказ.Код")
|
||||
short_name = example_resolve("Номенклатура.ЕдИзмерение.Код")
|
||||
|
||||
checks = {
|
||||
"full_metadata_path": full_path.get("resolution_kind") == "metadata_path"
|
||||
and full_path.get("canonical_path") == "Справочник.Номенклатура.Артикул"
|
||||
and full_path.get("safe_as_metadata_path") is True,
|
||||
"context_standard_attribute": context_member.get("resolution_kind") == "context_metadata_member"
|
||||
and context_member.get("canonical_path") == "Справочник.Номенклатура.Наименование",
|
||||
"parameter_not_metadata": parameter.get("resolution_kind") == "parameter"
|
||||
and parameter.get("safe_as_metadata_path") is False,
|
||||
"short_name_not_metadata": short_name.get("status") == "unresolved"
|
||||
and short_name.get("safe_as_metadata_path") is False
|
||||
and bool(short_name.get("candidates")),
|
||||
}
|
||||
failures = [name for name, ok in checks.items() if not ok]
|
||||
report = {
|
||||
"schema": "onec_bsl_symbol_resolver_check.v1",
|
||||
"status": "ok" if not failures else "failed",
|
||||
"failures": failures,
|
||||
"checks": checks,
|
||||
}
|
||||
if args.print:
|
||||
print(json.dumps(report, ensure_ascii=False, indent=2))
|
||||
elif failures:
|
||||
print("1C BSL symbol resolver status: failed", file=sys.stderr)
|
||||
else:
|
||||
print("1C BSL symbol resolver status: ok")
|
||||
return 0 if not failures else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,201 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Validate that a 1C change proposal stays within the read-only/extension-first safety contract."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
FORBIDDEN_PATH_PARTS = {
|
||||
"config",
|
||||
"configsave",
|
||||
"configcas",
|
||||
}
|
||||
|
||||
|
||||
def load_json(path: Path) -> dict[str, Any]:
|
||||
return json.loads(path.read_text(encoding="utf-8-sig"))
|
||||
|
||||
|
||||
def issue(severity: str, code: str, message: str, *, target: dict[str, Any] | None = None) -> dict[str, Any]:
|
||||
result = {"severity": severity, "code": code, "message": message}
|
||||
if target:
|
||||
result["target"] = target
|
||||
return result
|
||||
|
||||
|
||||
def path_text(target: dict[str, Any]) -> str:
|
||||
return str(target.get("path") or target.get("module_path") or "")
|
||||
|
||||
|
||||
def normalized_path_parts(path: str) -> list[str]:
|
||||
return [part.casefold() for part in path.replace("/", "\\").split("\\") if part]
|
||||
|
||||
|
||||
def is_extension_origin(target: dict[str, Any], preferred_extension: str | None) -> bool:
|
||||
origin = target.get("origin") or {}
|
||||
if origin.get("layer") != "extension":
|
||||
return False
|
||||
if preferred_extension and origin.get("extension") != preferred_extension:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def is_extension_path(path: str, preferred_extension: str | None) -> bool:
|
||||
parts = normalized_path_parts(path)
|
||||
if "расширения" not in parts and "extensions" not in parts:
|
||||
return False
|
||||
if preferred_extension:
|
||||
lowered = preferred_extension.casefold()
|
||||
return lowered in parts
|
||||
return True
|
||||
|
||||
|
||||
def forbidden_path_reason(path: str) -> str | None:
|
||||
parts = normalized_path_parts(path)
|
||||
for part in parts:
|
||||
if part in FORBIDDEN_PATH_PARTS:
|
||||
return part
|
||||
if path.startswith("_") or "\\_" in path:
|
||||
return "sql_physical_name_like_path"
|
||||
return None
|
||||
|
||||
|
||||
def check_target_exists(target: dict[str, Any]) -> list[dict[str, Any]]:
|
||||
findings = []
|
||||
path = path_text(target)
|
||||
if not path:
|
||||
findings.append(issue("error", "missing_path", "Target has no path.", target=target))
|
||||
return findings
|
||||
if not Path(path).exists():
|
||||
findings.append(issue("error", "path_not_found", f"Target path does not exist: {path}", target=target))
|
||||
line = target.get("line")
|
||||
if line:
|
||||
try:
|
||||
line_int = int(line)
|
||||
if line_int < 1:
|
||||
findings.append(issue("error", "invalid_line", f"Invalid target line: {line}", target=target))
|
||||
except (TypeError, ValueError):
|
||||
findings.append(issue("error", "invalid_line", f"Invalid target line: {line}", target=target))
|
||||
return findings
|
||||
|
||||
|
||||
def check_write_candidate(target: dict[str, Any], preferred_extension: str | None) -> list[dict[str, Any]]:
|
||||
findings = []
|
||||
path = path_text(target)
|
||||
findings.extend(check_target_exists(target))
|
||||
if not is_extension_origin(target, preferred_extension):
|
||||
findings.append(issue("error", "write_candidate_not_preferred_extension_origin", "Write candidate is not in the preferred extension origin.", target=target))
|
||||
if not is_extension_path(path, preferred_extension):
|
||||
findings.append(issue("error", "write_candidate_not_preferred_extension_path", "Write candidate path is not inside the preferred extension directory.", target=target))
|
||||
forbidden = forbidden_path_reason(path)
|
||||
if forbidden:
|
||||
findings.append(issue("error", "forbidden_write_path", f"Write candidate path is forbidden: {forbidden}", target=target))
|
||||
if target.get("kind") not in {"bsl_module", "form_xml"}:
|
||||
findings.append(issue("warning", "unusual_write_candidate_kind", f"Unexpected write candidate kind: {target.get('kind')}", target=target))
|
||||
return findings
|
||||
|
||||
|
||||
def check_reference_target(target: dict[str, Any]) -> list[dict[str, Any]]:
|
||||
findings = check_target_exists(target)
|
||||
forbidden = forbidden_path_reason(path_text(target))
|
||||
if forbidden:
|
||||
findings.append(issue("warning", "forbidden_reference_path", f"Reference path is forbidden for writes and must remain read-only: {forbidden}", target=target))
|
||||
return findings
|
||||
|
||||
|
||||
def check_proposal(proposal: dict[str, Any]) -> dict[str, Any]:
|
||||
findings = []
|
||||
strategy = proposal.get("write_strategy") or {}
|
||||
preferred_extension = strategy.get("preferred_extension")
|
||||
if strategy.get("mode") != "extension_first_proposal":
|
||||
findings.append(issue("error", "unsupported_write_strategy", f"Unsupported write strategy: {strategy.get('mode')}"))
|
||||
if not preferred_extension:
|
||||
findings.append(issue("warning", "missing_preferred_extension", "No preferred extension selected; patch generation should create/choose an extension explicitly."))
|
||||
|
||||
policy = proposal.get("target_policy") or {}
|
||||
write_candidates = policy.get("write_candidates") or []
|
||||
references = policy.get("read_only_reference_files") or []
|
||||
if not write_candidates:
|
||||
findings.append(issue("warning", "no_write_candidates", "No write candidates were selected."))
|
||||
for target in write_candidates:
|
||||
findings.extend(check_write_candidate(target, preferred_extension))
|
||||
for target in references:
|
||||
findings.extend(check_reference_target(target))
|
||||
|
||||
forbidden = set(strategy.get("forbidden") or [])
|
||||
required_forbidden = {
|
||||
"direct SQL metadata/data updates",
|
||||
"direct Config/ConfigSave/ConfigCAS writes",
|
||||
"automatic production Designer update/apply",
|
||||
}
|
||||
missing = sorted(required_forbidden - forbidden)
|
||||
if missing:
|
||||
findings.append(issue("error", "missing_forbidden_strategy_items", "Write strategy is missing forbidden items: " + ", ".join(missing)))
|
||||
|
||||
errors = [row for row in findings if row.get("severity") == "error"]
|
||||
warnings = [row for row in findings if row.get("severity") == "warning"]
|
||||
return {
|
||||
"object": proposal.get("object"),
|
||||
"preferred_extension": preferred_extension,
|
||||
"passed": not errors,
|
||||
"findings": findings,
|
||||
"counts": {
|
||||
"errors": len(errors),
|
||||
"warnings": len(warnings),
|
||||
"write_candidates": len(write_candidates),
|
||||
"read_only_references": len(references),
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def check(data: dict[str, Any]) -> dict[str, Any]:
|
||||
proposal_checks = [check_proposal(proposal) for proposal in data.get("proposals") or []]
|
||||
errors = sum(item.get("counts", {}).get("errors", 0) for item in proposal_checks)
|
||||
warnings = sum(item.get("counts", {}).get("warnings", 0) for item in proposal_checks)
|
||||
return {
|
||||
"schema": "onec_change_proposal_safety_check.v1",
|
||||
"source_schema": data.get("schema"),
|
||||
"task": data.get("task"),
|
||||
"passed": errors == 0,
|
||||
"proposal_checks": proposal_checks,
|
||||
"required_gates_before_real_write": [
|
||||
"backup_gate",
|
||||
"round_trip_parser_gate",
|
||||
"designer_validation_gate",
|
||||
"saved_state_gate",
|
||||
"extension_packaging_gate",
|
||||
"diff_gate",
|
||||
"minimal_write_scope_gate",
|
||||
"recovery_test_gate",
|
||||
],
|
||||
"counts": {
|
||||
"proposals": len(proposal_checks),
|
||||
"errors": errors,
|
||||
"warnings": warnings,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Check 1C proposal safety.")
|
||||
parser.add_argument("--proposal", type=Path, required=True)
|
||||
parser.add_argument("--output", type=Path)
|
||||
args = parser.parse_args()
|
||||
|
||||
result = check(load_json(args.proposal))
|
||||
output = json.dumps(result, ensure_ascii=False, indent=2) + "\n"
|
||||
if args.output:
|
||||
args.output.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.output.write_text(output, encoding="utf-8")
|
||||
print(json.dumps({"output": str(args.output), "passed": result["passed"], "counts": result["counts"]}, ensure_ascii=False))
|
||||
else:
|
||||
print(output)
|
||||
return 0 if result["passed"] else 2
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,151 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT))
|
||||
sys.path.insert(0, str(ROOT / "plugins" / "1c"))
|
||||
sys.path.insert(0, str(ROOT / "plugins" / "1c" / "connector"))
|
||||
|
||||
import adapter_1c_server as adapter_server # noqa: E402
|
||||
|
||||
|
||||
def require(condition: bool, message: str, failures: list[str]) -> None:
|
||||
if not condition:
|
||||
failures.append(message)
|
||||
|
||||
|
||||
def patch_adapter_for_symbol_checks() -> None:
|
||||
def fake_definition_find(payload: dict[str, Any]) -> dict[str, Any]:
|
||||
query = str(payload.get("query") or "")
|
||||
if query == "Артикул":
|
||||
return {
|
||||
"schema": "onec_definition_find.v1",
|
||||
"status": "ok",
|
||||
"matches": [
|
||||
{
|
||||
"canonical_path": "Справочник.Номенклатура.Артикул",
|
||||
"kind": "Catalog",
|
||||
"name": "Номенклатура",
|
||||
}
|
||||
],
|
||||
}
|
||||
if query == "Номенклатура":
|
||||
return {
|
||||
"schema": "onec_definition_find.v1",
|
||||
"status": "ok",
|
||||
"matches": [
|
||||
{
|
||||
"canonical_path": "Справочник.Номенклатура",
|
||||
"kind": "Catalog",
|
||||
"name": "Номенклатура",
|
||||
}
|
||||
],
|
||||
}
|
||||
return {"schema": "onec_definition_find.v1", "status": "ok", "matches": []}
|
||||
|
||||
def fake_read_module(payload: dict[str, Any]) -> dict[str, Any]:
|
||||
return {
|
||||
"schema": "onec_module_read.v1",
|
||||
"status": "ok",
|
||||
"text": "Процедура ПередЗаписью(Отказ) Экспорт\n Отказ = Истина;\nКонецПроцедуры",
|
||||
"owner": {"kind": "Catalog", "name": "Номенклатура"},
|
||||
"module": {"name": "Модуль объекта"},
|
||||
}
|
||||
|
||||
def fake_attributes(payload: dict[str, Any]) -> dict[str, Any]:
|
||||
return {"schema": "onec_metadata_object_attributes.v1", "status": "ok", "attributes": []}
|
||||
|
||||
adapter_server.metadata_definition_find = fake_definition_find
|
||||
adapter_server.read_module = fake_read_module
|
||||
adapter_server.metadata_object_attributes = fake_attributes
|
||||
|
||||
|
||||
def check_full_path(failures: list[str]) -> None:
|
||||
result = adapter_server.call_method(
|
||||
"code.symbol.resolve",
|
||||
{
|
||||
"base_id": "upo_test",
|
||||
"expression": "Справочник.Номенклатура.Артикул",
|
||||
"module_ref": "ConfigSave:file:0",
|
||||
},
|
||||
)
|
||||
require(result.get("schema") == "onec_bsl_symbol_resolution.v1", "full path must return BSL symbol schema", failures)
|
||||
require(result.get("status") == "resolved", "full path must resolve", failures)
|
||||
require(result.get("resolution_kind") == "metadata_path", "full path must be classified as metadata_path", failures)
|
||||
require(result.get("canonical_path") == "Справочник.Номенклатура.Артикул", "full path must expose canonical_path publicly", failures)
|
||||
require(result.get("safe_as_metadata_path") is True, "full path must expose safe_as_metadata_path=true", failures)
|
||||
|
||||
|
||||
def check_parameter(failures: list[str]) -> None:
|
||||
result = adapter_server.call_method(
|
||||
"code.symbol.resolve",
|
||||
{
|
||||
"base_id": "upo_test",
|
||||
"expression": "Отказ.Код",
|
||||
"routine_name": "ПередЗаписью",
|
||||
"module_ref": "ConfigSave:file:0",
|
||||
},
|
||||
)
|
||||
require(result.get("status") == "resolved", "routine parameter must resolve", failures)
|
||||
require(result.get("resolution_kind") == "parameter", "routine parameter must not be metadata", failures)
|
||||
require(result.get("context_path") == "Отказ.Код", "routine parameter must expose context_path publicly", failures)
|
||||
require(result.get("safe_as_metadata_path") is False, "routine parameter must expose safe_as_metadata_path=false", failures)
|
||||
|
||||
|
||||
def check_short_name(failures: list[str]) -> None:
|
||||
result = adapter_server.call_method(
|
||||
"code.symbol.resolve",
|
||||
{
|
||||
"base_id": "upo_test",
|
||||
"expression": "Номенклатура.ЕдИзмерение.Код",
|
||||
"routine_name": "ПередЗаписью",
|
||||
"module_ref": "ConfigSave:file:0",
|
||||
},
|
||||
)
|
||||
candidates = result.get("candidates") if isinstance(result.get("candidates"), list) else []
|
||||
require(result.get("status") == "unresolved", "short object name must stay unresolved", failures)
|
||||
require(result.get("safe_as_metadata_path") is False, "short object name must expose safe_as_metadata_path=false", failures)
|
||||
require(bool(candidates), "short object name must return ambiguity candidates", failures)
|
||||
if candidates:
|
||||
require(candidates[0].get("canonical_path") == "Справочник.Номенклатура", "candidate canonical_path must stay public", failures)
|
||||
require(candidates[0].get("reason") == "short_object_name_requires_kind", "candidate must explain short name risk", failures)
|
||||
|
||||
|
||||
def run_checks() -> dict[str, Any]:
|
||||
patch_adapter_for_symbol_checks()
|
||||
failures: list[str] = []
|
||||
check_full_path(failures)
|
||||
check_parameter(failures)
|
||||
check_short_name(failures)
|
||||
return {
|
||||
"schema": "onec_code_symbol_contract_check.v1",
|
||||
"status": "ok" if not failures else "failed",
|
||||
"failures": failures,
|
||||
"checks": {
|
||||
"full_path_metadata": "full path must resolve as metadata_path",
|
||||
"parameter_not_metadata": "routine parameter must not be metadata",
|
||||
"short_name_unsafe": "short object name must remain unsafe",
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Check adapter-level code.symbol.resolve contract invariants.")
|
||||
parser.add_argument("--print", action="store_true")
|
||||
args = parser.parse_args()
|
||||
report = run_checks()
|
||||
if args.print or report["status"] != "ok":
|
||||
print(json.dumps(report, ensure_ascii=False, indent=2))
|
||||
else:
|
||||
print("1C code symbol contract status: ok")
|
||||
return 0 if report["status"] == "ok" else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,128 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import sys
|
||||
import tomllib
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import yaml
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
CONNECTOR = ROOT / "plugins" / "1c" / "connector"
|
||||
PARSER = ROOT / "plugins" / "1c" / "parser"
|
||||
|
||||
|
||||
def require(condition: bool, message: str, failures: list[str]) -> None:
|
||||
if not condition:
|
||||
failures.append(message)
|
||||
|
||||
|
||||
def read_yaml(path: Path) -> dict[str, Any]:
|
||||
with path.open("r", encoding="utf-8") as handle:
|
||||
data = yaml.safe_load(handle)
|
||||
return data if isinstance(data, dict) else {}
|
||||
|
||||
|
||||
def read_toml(path: Path) -> dict[str, Any]:
|
||||
with path.open("rb") as handle:
|
||||
data = tomllib.load(handle)
|
||||
return data if isinstance(data, dict) else {}
|
||||
|
||||
|
||||
def run_checks() -> dict[str, Any]:
|
||||
failures: list[str] = []
|
||||
required = [
|
||||
CONNECTOR / "adapter_1c_server.py",
|
||||
CONNECTOR / "contracts" / "openapi.yaml",
|
||||
CONNECTOR / "policies" / "read-only-query.yaml",
|
||||
CONNECTOR / "policies" / "sql-base-access-policy.yaml",
|
||||
CONNECTOR / "policies" / "change-workflow.yaml",
|
||||
CONNECTOR / "policies" / "config-layer-write-policy.yaml",
|
||||
CONNECTOR / "Dockerfile",
|
||||
CONNECTOR / "docker-compose.yml",
|
||||
CONNECTOR / ".env.example",
|
||||
CONNECTOR / "pyproject.toml",
|
||||
CONNECTOR / "service.yaml",
|
||||
CONNECTOR / "README.md",
|
||||
PARSER / "__init__.py",
|
||||
PARSER / "payload.py",
|
||||
PARSER / "cas_payload.py",
|
||||
]
|
||||
missing = [str(path.relative_to(ROOT)) for path in required if not path.exists()]
|
||||
require(not missing, f"missing standalone connector files: {missing}", failures)
|
||||
|
||||
service = read_yaml(CONNECTOR / "service.yaml") if (CONNECTOR / "service.yaml").exists() else {}
|
||||
compose = read_yaml(CONNECTOR / "docker-compose.yml") if (CONNECTOR / "docker-compose.yml").exists() else {}
|
||||
pyproject = read_toml(CONNECTOR / "pyproject.toml") if (CONNECTOR / "pyproject.toml").exists() else {}
|
||||
openapi = read_yaml(CONNECTOR / "contracts" / "openapi.yaml") if (CONNECTOR / "contracts" / "openapi.yaml").exists() else {}
|
||||
access_policy = read_yaml(CONNECTOR / "policies" / "sql-base-access-policy.yaml") if (CONNECTOR / "policies" / "sql-base-access-policy.yaml").exists() else {}
|
||||
|
||||
require(service.get("id") == "onec-adapter-connector", "service.yaml must identify onec-adapter-connector", failures)
|
||||
require(service.get("status") == "standalone-ready", "service.yaml status must be standalone-ready", failures)
|
||||
require((service.get("runtime") or {}).get("entrypoint") == "adapter_1c_server.py", "service entrypoint must be adapter_1c_server.py", failures)
|
||||
require("contracts/openapi.yaml" == (service.get("contracts") or {}).get("openapi"), "service must point to connector OpenAPI contract", failures)
|
||||
registered_policies = (service.get("contracts") or {}).get("policies") or []
|
||||
require("policies/sql-base-access-policy.yaml" in registered_policies, "service must register SQL base access policy", failures)
|
||||
|
||||
base_settings = access_policy.get("base_settings") if isinstance(access_policy.get("base_settings"), dict) else {}
|
||||
read_scope = access_policy.get("read_scope") if isinstance(access_policy.get("read_scope"), dict) else {}
|
||||
write_scope = access_policy.get("write_scope") if isinstance(access_policy.get("write_scope"), dict) else {}
|
||||
identity = access_policy.get("sql_identity_management") if isinstance(access_policy.get("sql_identity_management"), dict) else {}
|
||||
require(access_policy.get("status") == "active", "SQL base access policy must be active", failures)
|
||||
require(base_settings.get("selector") == "base_id", "SQL settings must be selected by base_id", failures)
|
||||
require(set(base_settings.get("required_fields") or []) == {"server", "database", "user"}, "SQL base settings must require server, database, and user", failures)
|
||||
require(read_scope.get("application_data") == "read_only", "application data must be read-only", failures)
|
||||
require(read_scope.get("metadata_structure") == "read_only", "metadata structure must be readable without mutation", failures)
|
||||
require(set((write_scope.get("allowed") or {}).values()) == {"ConfigSave", "ConfigCASSave"}, "only ConfigSave and ConfigCASSave may be write targets", failures)
|
||||
require(identity.get("mode") == "forbidden", "SQL identity management must be forbidden", failures)
|
||||
|
||||
project = pyproject.get("project") if isinstance(pyproject.get("project"), dict) else {}
|
||||
require(project.get("name") == "onec-adapter-connector", "pyproject project.name must be onec-adapter-connector", failures)
|
||||
scripts = project.get("scripts") if isinstance(project.get("scripts"), dict) else {}
|
||||
require(scripts.get("onec-adapter") == "adapter_1c_server:main", "pyproject must expose onec-adapter script", failures)
|
||||
dependencies = project.get("dependencies") if isinstance(project.get("dependencies"), list) else []
|
||||
require(any(str(dep).startswith("pymssql") for dep in dependencies), "pyproject must include pymssql dependency", failures)
|
||||
|
||||
services = compose.get("services") if isinstance(compose.get("services"), dict) else {}
|
||||
adapter_service = services.get("onec-adapter") if isinstance(services.get("onec-adapter"), dict) else {}
|
||||
build = adapter_service.get("build") if isinstance(adapter_service.get("build"), dict) else {}
|
||||
require(build.get("context") == "..", "docker-compose build context must include parser sibling", failures)
|
||||
require(build.get("dockerfile") == "connector/Dockerfile", "docker-compose must use connector/Dockerfile", failures)
|
||||
require(bool(adapter_service.get("healthcheck")), "docker-compose must define a healthcheck", failures)
|
||||
|
||||
require(openapi.get("openapi") == "3.1.0", "connector OpenAPI must parse as 3.1.0", failures)
|
||||
paths = openapi.get("paths") if isinstance(openapi.get("paths"), dict) else {}
|
||||
require("/health" in paths, "connector OpenAPI must expose /health", failures)
|
||||
require("/metadata/write-plan" in paths, "connector OpenAPI must expose /metadata/write-plan", failures)
|
||||
|
||||
return {
|
||||
"schema": "onec_connector_standalone_check.v1",
|
||||
"status": "ok" if not failures else "failed",
|
||||
"failures": failures,
|
||||
"checks": {
|
||||
"required_files": not missing,
|
||||
"service_manifest": service.get("id"),
|
||||
"pyproject": project.get("name"),
|
||||
"compose_service": "onec-adapter" in services,
|
||||
"openapi": openapi.get("openapi"),
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Check 1C connector standalone-ready service packaging.")
|
||||
parser.add_argument("--print", action="store_true")
|
||||
args = parser.parse_args()
|
||||
report = run_checks()
|
||||
if args.print or report["status"] != "ok":
|
||||
print(json.dumps(report, ensure_ascii=False, indent=2))
|
||||
else:
|
||||
print("1C connector standalone status: ok")
|
||||
return 0 if report["status"] == "ok" else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,227 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT))
|
||||
sys.path.insert(0, str(ROOT / "plugins" / "1c"))
|
||||
sys.path.insert(0, str(ROOT / "plugins" / "1c" / "connector"))
|
||||
|
||||
import adapter_1c_server as adapter_server # noqa: E402
|
||||
from parser.payload import compress_payload # noqa: E402
|
||||
|
||||
|
||||
def require(condition: bool, message: str, failures: list[str]) -> None:
|
||||
if not condition:
|
||||
failures.append(message)
|
||||
|
||||
|
||||
def run_override(action_evidence: dict[str, Any] | None) -> dict[str, Any]:
|
||||
adapter_server.metadata_object_modules = lambda payload: {
|
||||
"status": "ok",
|
||||
"object": {"kind": "Catalog", "name": "Номенклатура"},
|
||||
"modules": [{"module_id": "ConfigCAS:ext-guid__module-guid.0", "name": "object"}],
|
||||
}
|
||||
selection = {
|
||||
"routine_name": "ПередЗаписью",
|
||||
"line_start": 1,
|
||||
"line_end": 3,
|
||||
"match_by": "routine_exact",
|
||||
}
|
||||
if action_evidence:
|
||||
selection.update(action_evidence)
|
||||
adapter_server.read_module = lambda payload: {"status": "ok", "selection": selection}
|
||||
return adapter_server.call_method(
|
||||
"metadata.resolve_overrides",
|
||||
{
|
||||
"base_id": "upo_test",
|
||||
"object_type": "Catalog",
|
||||
"object_name": "Номенклатура",
|
||||
"method_name": "ПередЗаписью",
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def run_base_override() -> dict[str, Any]:
|
||||
adapter_server.metadata_object_modules = lambda payload: {
|
||||
"status": "ok",
|
||||
"object": {"kind": "Catalog", "name": "Номенклатура"},
|
||||
"modules": [{"module_id": "Config:object-module.0", "name": "object"}],
|
||||
}
|
||||
adapter_server.read_module = lambda payload: {
|
||||
"status": "ok",
|
||||
"selection": {
|
||||
"routine_name": "ПередЗаписью",
|
||||
"line_start": 1,
|
||||
"line_end": 3,
|
||||
"match_by": "routine_exact",
|
||||
},
|
||||
}
|
||||
return adapter_server.call_method(
|
||||
"metadata.resolve_overrides",
|
||||
{
|
||||
"base_id": "upo_test",
|
||||
"object_type": "Catalog",
|
||||
"object_name": "Номенклатура",
|
||||
"method_name": "ПередЗаписью",
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def run_saved_state_search_with_object_name(params: dict[str, Any] | None = None) -> dict[str, Any]:
|
||||
module_text = "Процедура ПередЗаписью(Отказ)\nКонецПроцедуры".encode("utf-8")
|
||||
header = f"\r\n{len(module_text):08x} {len(module_text):08x} 7fffffff \r\n".encode("ascii")
|
||||
stored = compress_payload(b"prefix" + header + module_text, "raw_deflate")
|
||||
adapter_server.metadata_cache_lookup_row = lambda base_id, kind, name: {
|
||||
"guid": "owner-guid",
|
||||
"kind": kind,
|
||||
"kind_ru": "Справочник",
|
||||
"public_kind": "catalog",
|
||||
"name": name,
|
||||
"source": "base",
|
||||
}
|
||||
adapter_server.storage_files_list = lambda payload: {
|
||||
"status": "ok",
|
||||
"files": [{"FileName": "owner-guid__module-guid.0", "PartCount": 1, "Bytes": len(stored)}],
|
||||
}
|
||||
adapter_server.read_storage_file_bytes = lambda base_id, table, file_name, timeout_seconds=30: (stored, {"database": base_id}, None)
|
||||
request_payload = {
|
||||
"base_id": "upo_test",
|
||||
"tables": ["ConfigCASSave"],
|
||||
"object_type": "Catalog",
|
||||
"object_name": "Номенклатура",
|
||||
"query": "ПередЗаписью",
|
||||
"limit": 10,
|
||||
}
|
||||
request_payload.update(params or {})
|
||||
return adapter_server.call_method(
|
||||
adapter_server.SAVED_STATE_MODULES_SEARCH_METHOD,
|
||||
request_payload,
|
||||
)
|
||||
|
||||
|
||||
def first_action(result: dict[str, Any]) -> dict[str, Any]:
|
||||
chain = result.get("chain") if isinstance(result.get("chain"), list) else []
|
||||
first = chain[0] if chain and isinstance(chain[0], dict) else {}
|
||||
action = first.get("extension_action") if isinstance(first.get("extension_action"), dict) else {}
|
||||
return action
|
||||
|
||||
|
||||
def run_checks() -> dict[str, Any]:
|
||||
failures: list[str] = []
|
||||
|
||||
unknown = run_override(None)
|
||||
unknown_action = first_action(unknown)
|
||||
unknown_evidence = unknown.get("write_plan_evidence") if isinstance(unknown.get("write_plan_evidence"), dict) else {}
|
||||
unknown_next = unknown_evidence.get("next_resolution") if isinstance(unknown_evidence.get("next_resolution"), dict) else {}
|
||||
unknown_next_params = unknown_next.get("params") if isinstance(unknown_next.get("params"), dict) else {}
|
||||
require(unknown.get("status") == "ok", "override chain with extension routine must resolve", failures)
|
||||
require(unknown_action.get("status") == "unknown", "extension routine without action evidence must be unknown", failures)
|
||||
require(unknown_action.get("operation_class") == "unknown_extension_action", "unknown action must not become replace", failures)
|
||||
require((unknown_evidence.get("target") or {}).get("extension_action") == unknown_action, "unknown action must be carried into write_plan_evidence target", failures)
|
||||
require("intent" not in unknown_evidence, "unknown action must not infer write intent", failures)
|
||||
require(unknown_next.get("method") == adapter_server.SAVED_STATE_MODULES_SEARCH_METHOD, "write_plan_evidence must expose saved-state module resolver", failures)
|
||||
require(unknown_next_params.get("tables") == ["ConfigCASSave"], "extension write_plan_evidence must search ConfigCASSave", failures)
|
||||
|
||||
controlled = run_override({"operation_class": "replace_with_control"})
|
||||
controlled_action = first_action(controlled)
|
||||
controlled_evidence = controlled.get("write_plan_evidence") if isinstance(controlled.get("write_plan_evidence"), dict) else {}
|
||||
controlled_next = controlled_evidence.get("next_resolution") if isinstance(controlled_evidence.get("next_resolution"), dict) else {}
|
||||
controlled_next_params = controlled_next.get("params") if isinstance(controlled_next.get("params"), dict) else {}
|
||||
require(controlled_action.get("status") == "ok", "known extension action must be ok", failures)
|
||||
require(controlled_action.get("operation_class") == "replace_with_control", "replace_with_control must be preserved", failures)
|
||||
require(controlled_action.get("requires_control_fragment") is True, "replace_with_control must require control fragment", failures)
|
||||
require((controlled_evidence.get("target") or {}).get("extension_action") == controlled_action, "known action must be carried into write_plan_evidence target", failures)
|
||||
require((controlled_evidence.get("intent") or {}).get("operation") == "replace_with_control", "known action must infer write_plan_evidence intent", failures)
|
||||
require(controlled_next.get("method") == adapter_server.SAVED_STATE_MODULES_SEARCH_METHOD, "known action evidence must expose saved-state module resolver", failures)
|
||||
require(controlled_next_params.get("query") == "ПередЗаписью", "known action resolver params must carry routine query", failures)
|
||||
|
||||
before = run_override({"action": "вставить до"})
|
||||
before_action = first_action(before)
|
||||
require(before_action.get("operation_class") == "insert_before", "Russian insert-before action must normalize", failures)
|
||||
|
||||
base = run_base_override()
|
||||
base_action = first_action(base)
|
||||
base_evidence = base.get("write_plan_evidence") if isinstance(base.get("write_plan_evidence"), dict) else {}
|
||||
base_next = base_evidence.get("next_resolution") if isinstance(base_evidence.get("next_resolution"), dict) else {}
|
||||
base_next_params = base_next.get("params") if isinstance(base_next.get("params"), dict) else {}
|
||||
require(base_action.get("operation_class") == "base_definition", "base routine must be marked as base_definition", failures)
|
||||
require(base_action.get("requires_control_fragment") is False, "base routine must not require control fragment", failures)
|
||||
require(base_next_params.get("tables") == ["ConfigSave"], "base write_plan_evidence must search ConfigSave", failures)
|
||||
|
||||
saved_state = run_saved_state_search_with_object_name(controlled_next_params)
|
||||
saved_state_owner = saved_state.get("owner_resolution") if isinstance(saved_state.get("owner_resolution"), dict) else {}
|
||||
saved_state_modules = saved_state.get("modules") if isinstance(saved_state.get("modules"), list) else []
|
||||
saved_state_streams = saved_state_modules[0].get("streams") if saved_state_modules and isinstance(saved_state_modules[0], dict) and isinstance(saved_state_modules[0].get("streams"), list) else []
|
||||
saved_state_target = saved_state_streams[0].get("write_plan_target") if saved_state_streams and isinstance(saved_state_streams[0], dict) and isinstance(saved_state_streams[0].get("write_plan_target"), dict) else {}
|
||||
require(saved_state.get("status") == "ok", "saved-state module search with object name must be accepted", failures)
|
||||
require(saved_state_owner.get("owner_guid") == "owner-guid", "saved-state module search must resolve object name to owner_guid", failures)
|
||||
require(bool(saved_state_modules), "saved-state module search with resolved owner must find module", failures)
|
||||
require(saved_state_target.get("module_ref") == "ConfigCASSave:owner-guid__module-guid.0#stream:0", "saved-state stream must expose write_plan_target.module_ref", failures)
|
||||
require(saved_state_target.get("expected_sha1"), "saved-state stream must expose write_plan_target.expected_sha1", failures)
|
||||
|
||||
concrete_plan_target = {
|
||||
**(controlled_evidence.get("target") if isinstance(controlled_evidence.get("target"), dict) else {}),
|
||||
**saved_state_target,
|
||||
}
|
||||
concrete_plan_intent = {
|
||||
**(controlled_evidence.get("intent") if isinstance(controlled_evidence.get("intent"), dict) else {}),
|
||||
"control_fragment": "Процедура ПередЗаписью",
|
||||
"new": "Процедура ПередЗаписью(Отказ)\n\t// smoke\nКонецПроцедуры",
|
||||
}
|
||||
concrete_plan = adapter_server.call_method(
|
||||
adapter_server.METADATA_WRITE_PLAN_METHOD,
|
||||
{
|
||||
"base_id": "upo_test",
|
||||
"target": concrete_plan_target,
|
||||
"intent": concrete_plan_intent,
|
||||
"resolve_origin": False,
|
||||
},
|
||||
)
|
||||
concrete_route = concrete_plan.get("route") if isinstance(concrete_plan.get("route"), dict) else {}
|
||||
concrete_hint = concrete_route.get("apply_payload_hint") if isinstance(concrete_route.get("apply_payload_hint"), dict) else {}
|
||||
concrete_hint_payload = concrete_hint.get("payload") if isinstance(concrete_hint.get("payload"), dict) else {}
|
||||
require(concrete_plan.get("allowed") is True, "write_plan_evidence plus saved-state write_plan_target must produce an allowed concrete plan", failures)
|
||||
require(concrete_route.get("apply_method") == adapter_server.MODULE_WRITE_APPLY_METHOD, "concrete override write plan must route to module write apply", failures)
|
||||
require(concrete_route.get("operation_class") == "replace_with_control", "concrete override write plan must preserve extension operation class", failures)
|
||||
require(concrete_hint.get("ready_for_apply_method") is True, "concrete override write plan hint must be ready for apply method", failures)
|
||||
require(concrete_hint_payload.get("module_ref") == saved_state_target.get("module_ref"), "concrete override write plan hint must carry module_ref", failures)
|
||||
require(concrete_hint_payload.get("expected_sha1") == saved_state_target.get("expected_sha1"), "concrete override write plan hint must carry expected_sha1", failures)
|
||||
|
||||
return {
|
||||
"schema": "onec_extension_action_contract_check.v1",
|
||||
"status": "ok" if not failures else "failed",
|
||||
"failures": failures,
|
||||
"checks": {
|
||||
"unknown_extension_action": "extension routine without action evidence stays unknown",
|
||||
"replace_with_control": "controlled replacement action requires control fragment",
|
||||
"russian_insert_before": "Russian action names normalize to stable classes",
|
||||
"base_definition": "base routines are not treated as extension actions",
|
||||
"write_plan_evidence": "override results include a ready metadata.write.plan evidence fragment",
|
||||
"write_plan_next_resolution": "write_plan_evidence points to the saved-state module resolver",
|
||||
"saved_state_module_name_selector": "saved-state module search resolves object_type/object_name to owner_guid",
|
||||
"saved_state_stream_write_plan_target": "saved-state module streams expose concrete metadata.write.plan target",
|
||||
"override_to_concrete_write_plan": "write_plan_evidence and write_plan_target compose into an allowed concrete module plan",
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Check extension routine action contract invariants.")
|
||||
parser.add_argument("--print", action="store_true")
|
||||
args = parser.parse_args()
|
||||
report = run_checks()
|
||||
if args.print or report["status"] != "ok":
|
||||
print(json.dumps(report, ensure_ascii=False, indent=2))
|
||||
else:
|
||||
print("1C extension action contract status: ok")
|
||||
return 0 if report["status"] == "ok" else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,165 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Validate safe runner configuration for disposable 1C extension validation."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import re
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
ALLOWED_RUNNER_KINDS = {"manual", "designer_cli", "onescript", "custom"}
|
||||
ALLOWED_VALIDATION_MODES = {"manual", "load_and_syntax_check", "load_syntax_and_smoke"}
|
||||
FORBIDDEN_BASE_MARKERS = {"prod", "production", "рабоч", "боев", "real", "main"}
|
||||
SECRET_KEY_RE = re.compile(r"(password|passwd|pwd|secret|token|ключ|парол)", re.IGNORECASE)
|
||||
CONNECTION_SECRET_RE = re.compile(r"(pwd|password|usr|user)\s*=", re.IGNORECASE)
|
||||
|
||||
|
||||
def load_json(path: Path) -> dict[str, Any]:
|
||||
return json.loads(path.read_text(encoding="utf-8-sig"))
|
||||
|
||||
|
||||
def write_json(path: Path, data: dict[str, Any]) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(json.dumps(data, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
|
||||
|
||||
def issue(severity: str, code: str, message: str, *, path: Path | str | None = None, detail: dict[str, Any] | None = None) -> dict[str, Any]:
|
||||
result: dict[str, Any] = {"severity": severity, "code": code, "message": message}
|
||||
if path is not None:
|
||||
result["path"] = str(path)
|
||||
if detail:
|
||||
result["detail"] = detail
|
||||
return result
|
||||
|
||||
|
||||
def find_secret_keys(value: Any, prefix: str = "") -> list[str]:
|
||||
hits: list[str] = []
|
||||
if isinstance(value, dict):
|
||||
for key, nested in value.items():
|
||||
current = f"{prefix}.{key}" if prefix else str(key)
|
||||
if SECRET_KEY_RE.search(str(key)):
|
||||
hits.append(current)
|
||||
hits.extend(find_secret_keys(nested, current))
|
||||
elif isinstance(value, list):
|
||||
for index, nested in enumerate(value):
|
||||
hits.extend(find_secret_keys(nested, f"{prefix}[{index}]"))
|
||||
return hits
|
||||
|
||||
|
||||
def string_contains_forbidden_marker(value: str) -> str | None:
|
||||
lowered = value.casefold()
|
||||
for marker in FORBIDDEN_BASE_MARKERS:
|
||||
if marker in lowered:
|
||||
return marker
|
||||
return None
|
||||
|
||||
|
||||
def sanitized_config(config: dict[str, Any]) -> dict[str, Any]:
|
||||
allowed = {
|
||||
"schema",
|
||||
"runner_id",
|
||||
"runner_kind",
|
||||
"platform_version",
|
||||
"platform_bin",
|
||||
"disposable_base_ref",
|
||||
"disposable_base_kind",
|
||||
"disposable_base_confirmed",
|
||||
"validation_mode",
|
||||
"evidence_root",
|
||||
"notes",
|
||||
}
|
||||
return {key: value for key, value in config.items() if key in allowed}
|
||||
|
||||
|
||||
def check_config(config_path: Path) -> dict[str, Any]:
|
||||
findings: list[dict[str, Any]] = []
|
||||
if not config_path.exists() or not config_path.is_file():
|
||||
findings.append(issue("error", "missing_runner_config", "Runner config file is missing.", path=config_path))
|
||||
return build_result(config_path, None, findings)
|
||||
|
||||
config = load_json(config_path)
|
||||
if config.get("schema") != "onec_extension_runner_config.v1":
|
||||
findings.append(issue("error", "invalid_runner_config_schema", "Runner config schema must be onec_extension_runner_config.v1.", path=config_path, detail={"schema": config.get("schema")}))
|
||||
|
||||
runner_id = config.get("runner_id")
|
||||
if not isinstance(runner_id, str) or not runner_id.strip():
|
||||
findings.append(issue("error", "missing_runner_id", "runner_id is required.", path=config_path))
|
||||
|
||||
runner_kind = config.get("runner_kind")
|
||||
if runner_kind not in ALLOWED_RUNNER_KINDS:
|
||||
findings.append(issue("error", "invalid_runner_kind", "runner_kind is not supported.", path=config_path, detail={"allowed": sorted(ALLOWED_RUNNER_KINDS), "actual": runner_kind}))
|
||||
|
||||
validation_mode = config.get("validation_mode")
|
||||
if validation_mode not in ALLOWED_VALIDATION_MODES:
|
||||
findings.append(issue("error", "invalid_validation_mode", "validation_mode is not supported.", path=config_path, detail={"allowed": sorted(ALLOWED_VALIDATION_MODES), "actual": validation_mode}))
|
||||
|
||||
disposable_base_ref = config.get("disposable_base_ref")
|
||||
if not isinstance(disposable_base_ref, str) or not disposable_base_ref.strip():
|
||||
findings.append(issue("error", "missing_disposable_base_ref", "disposable_base_ref is required.", path=config_path))
|
||||
else:
|
||||
marker = string_contains_forbidden_marker(disposable_base_ref)
|
||||
if marker:
|
||||
findings.append(issue("error", "production_like_base_ref", "disposable_base_ref contains a production-like marker.", path=config_path, detail={"marker": marker}))
|
||||
if CONNECTION_SECRET_RE.search(disposable_base_ref):
|
||||
findings.append(issue("error", "secret_in_disposable_base_ref", "disposable_base_ref must not contain user/password connection data.", path=config_path))
|
||||
|
||||
if config.get("disposable_base_confirmed") is not True:
|
||||
findings.append(issue("error", "disposable_base_not_confirmed", "disposable_base_confirmed must be true.", path=config_path))
|
||||
|
||||
platform_bin = config.get("platform_bin")
|
||||
if platform_bin is not None:
|
||||
if not isinstance(platform_bin, str) or not platform_bin.strip():
|
||||
findings.append(issue("error", "invalid_platform_bin", "platform_bin must be a non-empty string when provided.", path=config_path))
|
||||
elif runner_kind in {"designer_cli", "custom"} and not Path(platform_bin).exists():
|
||||
findings.append(issue("warning", "platform_bin_not_found", "platform_bin does not exist on this machine; runner may be remote or not installed here.", path=platform_bin))
|
||||
|
||||
evidence_root = config.get("evidence_root")
|
||||
if evidence_root is not None and (not isinstance(evidence_root, str) or not evidence_root.strip()):
|
||||
findings.append(issue("error", "invalid_evidence_root", "evidence_root must be a non-empty string when provided.", path=config_path))
|
||||
|
||||
secret_keys = find_secret_keys(config)
|
||||
for key in secret_keys:
|
||||
findings.append(issue("error", "secret_key_in_runner_config", "Runner config must not contain secrets or credentials.", path=config_path, detail={"key": key}))
|
||||
|
||||
unknown = sorted(set(config) - set(sanitized_config(config)))
|
||||
for key in unknown:
|
||||
findings.append(issue("warning", "unknown_runner_config_key", "Unknown runner config key will be ignored by the adapter.", path=config_path, detail={"key": key}))
|
||||
|
||||
return build_result(config_path, sanitized_config(config), findings)
|
||||
|
||||
|
||||
def build_result(config_path: Path, config: dict[str, Any] | None, findings: list[dict[str, Any]]) -> dict[str, Any]:
|
||||
errors = [row for row in findings if row.get("severity") == "error"]
|
||||
warnings = [row for row in findings if row.get("severity") == "warning"]
|
||||
return {
|
||||
"schema": "onec_extension_runner_config_check.v1",
|
||||
"config_path": str(config_path),
|
||||
"config_schema": (config or {}).get("schema"),
|
||||
"passed": not errors,
|
||||
"sanitized_config": config,
|
||||
"findings": findings,
|
||||
"counts": {
|
||||
"errors": len(errors),
|
||||
"warnings": len(warnings),
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Validate safe runner configuration for disposable 1C extension validation.")
|
||||
parser.add_argument("--config", type=Path, required=True)
|
||||
parser.add_argument("--output", type=Path)
|
||||
args = parser.parse_args()
|
||||
|
||||
result = check_config(args.config)
|
||||
if args.output:
|
||||
write_json(args.output, result)
|
||||
print(json.dumps({"output": str(args.output) if args.output else None, "passed": result["passed"], "counts": result["counts"]}, ensure_ascii=False))
|
||||
return 0 if result["passed"] else 2
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,195 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Validate a disposable 1C extension XML staging copy."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from check_1c_patch_bundle import check_bundle
|
||||
|
||||
|
||||
def load_json(path: Path) -> dict[str, Any]:
|
||||
return json.loads(path.read_text(encoding="utf-8-sig"))
|
||||
|
||||
|
||||
def write_json(path: Path, data: dict[str, Any]) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(json.dumps(data, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
|
||||
|
||||
def sha256_file(path: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as stream:
|
||||
for chunk in iter(lambda: stream.read(1024 * 1024), b""):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def issue(severity: str, code: str, message: str, *, path: Path | str | None = None, detail: dict[str, Any] | None = None) -> dict[str, Any]:
|
||||
result: dict[str, Any] = {"severity": severity, "code": code, "message": message}
|
||||
if path is not None:
|
||||
result["path"] = str(path)
|
||||
if detail:
|
||||
result["detail"] = detail
|
||||
return result
|
||||
|
||||
|
||||
def safe_relative(relative_path: str) -> Path:
|
||||
path = Path(relative_path.replace("\\", "/"))
|
||||
if path.is_absolute() or ".." in path.parts or not str(path):
|
||||
raise ValueError(relative_path)
|
||||
return path
|
||||
|
||||
|
||||
def is_within(path: Path, root: Path) -> bool:
|
||||
try:
|
||||
path.resolve().relative_to(root.resolve())
|
||||
return True
|
||||
except ValueError:
|
||||
return False
|
||||
|
||||
|
||||
def check_staging(staging_dir: Path) -> dict[str, Any]:
|
||||
findings: list[dict[str, Any]] = []
|
||||
file_checks: list[dict[str, Any]] = []
|
||||
manifest: dict[str, Any] | None = None
|
||||
|
||||
if not staging_dir.exists() or not staging_dir.is_dir():
|
||||
findings.append(issue("error", "missing_staging_dir", "Staging directory is missing.", path=staging_dir))
|
||||
return build_result(staging_dir, manifest, findings, file_checks, None)
|
||||
|
||||
manifest_path = staging_dir / "_codex_staging_manifest.json"
|
||||
if not manifest_path.exists():
|
||||
findings.append(issue("error", "missing_staging_manifest", "Staging manifest is missing.", path=manifest_path))
|
||||
return build_result(staging_dir, manifest, findings, file_checks, None)
|
||||
|
||||
manifest = load_json(manifest_path)
|
||||
if manifest.get("schema") != "onec_extension_staging.v1":
|
||||
findings.append(issue("error", "invalid_staging_schema", "Staging manifest schema is not onec_extension_staging.v1.", path=manifest_path, detail={"schema": manifest.get("schema")}))
|
||||
|
||||
declared_staging_dir = Path(str(manifest.get("staging_dir") or ""))
|
||||
if declared_staging_dir and declared_staging_dir.resolve() != staging_dir.resolve():
|
||||
findings.append(issue("error", "staging_dir_mismatch", "Manifest staging_dir does not match checked directory.", path=manifest_path, detail={"manifest_staging_dir": str(declared_staging_dir), "checked_staging_dir": str(staging_dir)}))
|
||||
|
||||
safety = manifest.get("safety") if isinstance(manifest.get("safety"), dict) else {}
|
||||
expected_safety = {
|
||||
"source_extension_modified": False,
|
||||
"sql_modified": False,
|
||||
"requires_disposable_1c_validation": True,
|
||||
}
|
||||
for key, expected in expected_safety.items():
|
||||
actual = safety.get(key)
|
||||
if actual is not expected:
|
||||
findings.append(issue("error", "invalid_staging_safety_flag", "Staging safety flag has an unexpected value.", path=manifest_path, detail={"flag": key, "expected": expected, "actual": actual}))
|
||||
|
||||
extension_root = Path(str(manifest.get("extension_root") or ""))
|
||||
if not extension_root.exists() or not extension_root.is_dir():
|
||||
findings.append(issue("error", "missing_extension_root", "Source extension root is missing.", path=extension_root))
|
||||
|
||||
bundle_dir = Path(str(manifest.get("bundle_dir") or ""))
|
||||
bundle_check: dict[str, Any] | None = None
|
||||
if not bundle_dir.exists() or not bundle_dir.is_dir():
|
||||
findings.append(issue("error", "missing_bundle_dir", "Bundle directory recorded in staging manifest is missing.", path=bundle_dir))
|
||||
else:
|
||||
bundle_check = check_bundle(bundle_dir)
|
||||
if not bundle_check.get("passed"):
|
||||
findings.append(issue("error", "bundle_check_failed", "Bundle recorded in staging manifest does not pass validation.", path=bundle_dir, detail={"counts": bundle_check.get("counts")}))
|
||||
|
||||
for record in manifest.get("files") or []:
|
||||
relative_raw = str(record.get("relative_path") or "")
|
||||
staged_path_raw = str(record.get("staged_path") or "")
|
||||
check: dict[str, Any] = {
|
||||
"relative_path": relative_raw,
|
||||
"staged_path": staged_path_raw,
|
||||
"exists": False,
|
||||
"expected_staged_sha256": record.get("staged_sha256"),
|
||||
"expected_working_sha256": record.get("expected_working_sha256"),
|
||||
"expected_source_original_sha256": record.get("source_original_sha256"),
|
||||
}
|
||||
try:
|
||||
relative = safe_relative(relative_raw)
|
||||
except ValueError:
|
||||
findings.append(issue("error", "unsafe_relative_path", "Unsafe relative_path in staging manifest.", path=manifest_path, detail={"relative_path": relative_raw}))
|
||||
file_checks.append(check)
|
||||
continue
|
||||
|
||||
staged_path = Path(staged_path_raw) if staged_path_raw else staging_dir / relative
|
||||
expected_staged_path = staging_dir / relative
|
||||
if staged_path.resolve() != expected_staged_path.resolve():
|
||||
findings.append(issue("error", "staged_path_mismatch", "Manifest staged_path does not match staging_dir/relative_path.", path=manifest_path, detail={"staged_path": str(staged_path), "expected": str(expected_staged_path)}))
|
||||
if not is_within(staged_path, staging_dir):
|
||||
findings.append(issue("error", "staged_path_escape", "Manifest staged_path escapes staging directory.", path=staged_path))
|
||||
file_checks.append(check)
|
||||
continue
|
||||
|
||||
check["exists"] = staged_path.exists()
|
||||
if not staged_path.exists():
|
||||
findings.append(issue("error", "missing_staged_file", "Staged file is missing.", path=staged_path))
|
||||
else:
|
||||
staged_hash = sha256_file(staged_path)
|
||||
check["staged_sha256"] = staged_hash
|
||||
expected_hashes = [record.get("staged_sha256"), record.get("expected_working_sha256")]
|
||||
for expected in [value for value in expected_hashes if value]:
|
||||
if staged_hash != expected:
|
||||
findings.append(issue("error", "staged_file_hash_mismatch", "Staged file hash does not match manifest.", path=staged_path, detail={"expected": expected, "actual": staged_hash}))
|
||||
|
||||
source_path = extension_root / relative
|
||||
check["source_path"] = str(source_path)
|
||||
check["source_exists"] = source_path.exists()
|
||||
expected_source_hash = record.get("source_original_sha256")
|
||||
if expected_source_hash is not None:
|
||||
if not source_path.exists():
|
||||
findings.append(issue("error", "missing_source_file", "Source extension file recorded during staging is now missing.", path=source_path))
|
||||
else:
|
||||
source_hash = sha256_file(source_path)
|
||||
check["source_sha256"] = source_hash
|
||||
if source_hash != expected_source_hash:
|
||||
findings.append(issue("error", "source_file_changed", "Source extension file changed after staging was created.", path=source_path, detail={"expected": expected_source_hash, "actual": source_hash}))
|
||||
file_checks.append(check)
|
||||
|
||||
return build_result(staging_dir, manifest, findings, file_checks, bundle_check)
|
||||
|
||||
|
||||
def build_result(staging_dir: Path, manifest: dict[str, Any] | None, findings: list[dict[str, Any]], file_checks: list[dict[str, Any]], bundle_check: dict[str, Any] | None) -> dict[str, Any]:
|
||||
errors = [row for row in findings if row.get("severity") == "error"]
|
||||
warnings = [row for row in findings if row.get("severity") == "warning"]
|
||||
return {
|
||||
"schema": "onec_extension_staging_check.v1",
|
||||
"staging_dir": str(staging_dir),
|
||||
"staging_schema": (manifest or {}).get("schema"),
|
||||
"bundle_dir": (manifest or {}).get("bundle_dir"),
|
||||
"bundle_check": {
|
||||
"schema": (bundle_check or {}).get("schema"),
|
||||
"passed": (bundle_check or {}).get("passed"),
|
||||
"counts": (bundle_check or {}).get("counts"),
|
||||
} if bundle_check else None,
|
||||
"passed": not errors,
|
||||
"findings": findings,
|
||||
"file_checks": file_checks,
|
||||
"counts": {
|
||||
"files": len(file_checks),
|
||||
"errors": len(errors),
|
||||
"warnings": len(warnings),
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Validate a disposable 1C extension XML staging copy.")
|
||||
parser.add_argument("--staging-dir", type=Path, required=True)
|
||||
parser.add_argument("--output", type=Path)
|
||||
args = parser.parse_args()
|
||||
|
||||
result = check_staging(args.staging_dir)
|
||||
if args.output:
|
||||
write_json(args.output, result)
|
||||
print(json.dumps({"output": str(args.output) if args.output else None, "passed": result["passed"], "counts": result["counts"]}, ensure_ascii=False))
|
||||
return 0 if result["passed"] else 2
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,172 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Check manual evidence files for a 1C extension validation plan."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import re
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
PENDING_MARKERS = ("status: pending", '"status": "pending"')
|
||||
PASSED_STATUSES = {"passed", "success", "ok"}
|
||||
STATUS_LINE_RE = re.compile(r"^\s*status\s*:\s*([A-Za-zА-Яа-я0-9_-]+)\s*$", re.IGNORECASE | re.MULTILINE)
|
||||
SECRET_TEXT_RE = re.compile(r"(password|passwd|pwd|secret|token|парол|секрет|usr|user)\s*[:=]", re.IGNORECASE)
|
||||
|
||||
|
||||
def load_json(path: Path) -> dict[str, Any]:
|
||||
return json.loads(path.read_text(encoding="utf-8-sig"))
|
||||
|
||||
|
||||
def write_json(path: Path, data: dict[str, Any]) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(json.dumps(data, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
|
||||
|
||||
def issue(severity: str, code: str, message: str, *, path: Path | str | None = None, detail: dict[str, Any] | None = None) -> dict[str, Any]:
|
||||
result: dict[str, Any] = {"severity": severity, "code": code, "message": message}
|
||||
if path is not None:
|
||||
result["path"] = str(path)
|
||||
if detail:
|
||||
result["detail"] = detail
|
||||
return result
|
||||
|
||||
|
||||
def evidence_root(plan: dict[str, Any], override: Path | None = None) -> Path:
|
||||
if override:
|
||||
return override
|
||||
configured = ((plan.get("runner_config") or {}).get("evidence_root")) or ((plan.get("evidence") or {}).get("root"))
|
||||
if not configured:
|
||||
raise SystemExit("Validation plan has no evidence root.")
|
||||
return Path(str(configured))
|
||||
|
||||
|
||||
def text_status(text: str) -> str | None:
|
||||
match = STATUS_LINE_RE.search(text)
|
||||
return match.group(1).casefold() if match else None
|
||||
|
||||
|
||||
def check_json_evidence(path: Path, text: str, findings: list[dict[str, Any]]) -> tuple[bool, str | None]:
|
||||
try:
|
||||
payload = json.loads(text)
|
||||
except json.JSONDecodeError as exc:
|
||||
findings.append(issue("error", "invalid_json_evidence", f"Evidence JSON is invalid: {exc}", path=path))
|
||||
return False, None
|
||||
status = str(payload.get("status") or "").casefold()
|
||||
if status not in PASSED_STATUSES:
|
||||
findings.append(issue("error", "evidence_status_not_passed", "Evidence JSON status must be passed/success/ok.", path=path, detail={"status": status or None}))
|
||||
return False, status or None
|
||||
if path.name == "changed-objects-smoke.json":
|
||||
objects = payload.get("objects")
|
||||
if not isinstance(objects, list) or not objects:
|
||||
findings.append(issue("error", "missing_smoke_objects", "changed-objects-smoke.json must contain a non-empty objects list.", path=path))
|
||||
return False, status
|
||||
failed = [
|
||||
{"object_name": item.get("object_name"), "status": item.get("status")}
|
||||
for item in objects
|
||||
if not isinstance(item, dict) or str(item.get("status") or "").casefold() not in PASSED_STATUSES
|
||||
]
|
||||
if failed:
|
||||
findings.append(issue("error", "smoke_object_status_not_passed", "Every changed object smoke record must have status passed/success/ok.", path=path, detail={"failed": failed}))
|
||||
return False, status
|
||||
return True, status
|
||||
|
||||
|
||||
def check_text_evidence(path: Path, text: str, findings: list[dict[str, Any]]) -> tuple[bool, str | None]:
|
||||
status = text_status(text)
|
||||
if status not in PASSED_STATUSES:
|
||||
findings.append(issue("error", "evidence_status_not_passed", "Evidence text must contain a Status: passed/success/ok line.", path=path, detail={"status": status}))
|
||||
return False, status
|
||||
return True, status
|
||||
|
||||
|
||||
def check_evidence(plan_path: Path, output_root: Path | None = None) -> dict[str, Any]:
|
||||
findings: list[dict[str, Any]] = []
|
||||
file_checks: list[dict[str, Any]] = []
|
||||
plan = load_json(plan_path)
|
||||
if plan.get("schema") != "onec_extension_validation_plan.v1":
|
||||
findings.append(issue("error", "invalid_plan_schema", "Validation plan schema is not onec_extension_validation_plan.v1.", path=plan_path, detail={"schema": plan.get("schema")}))
|
||||
return build_result(plan_path, Path("."), findings, file_checks)
|
||||
|
||||
root = evidence_root(plan, output_root)
|
||||
if not root.exists() or not root.is_dir():
|
||||
findings.append(issue("error", "missing_evidence_root", "Evidence root is missing.", path=root))
|
||||
return build_result(plan_path, root, findings, file_checks)
|
||||
|
||||
for name in (plan.get("evidence") or {}).get("required_files") or []:
|
||||
relative = Path(str(name).replace("\\", "/"))
|
||||
path = root / relative
|
||||
check: dict[str, Any] = {
|
||||
"relative_path": str(relative).replace("\\", "/"),
|
||||
"path": str(path),
|
||||
"exists": path.exists(),
|
||||
"filled": False,
|
||||
}
|
||||
if not path.exists():
|
||||
findings.append(issue("error", "missing_evidence_file", "Required evidence file is missing.", path=path))
|
||||
else:
|
||||
text = path.read_text(encoding="utf-8-sig", errors="replace")
|
||||
stripped = text.strip()
|
||||
check["size"] = len(text.encode("utf-8"))
|
||||
if SECRET_TEXT_RE.search(text):
|
||||
findings.append(issue("error", "secret_like_text_in_evidence", "Evidence file contains secret-like key/value text.", path=path))
|
||||
pending = any(marker in text.casefold() for marker in PENDING_MARKERS)
|
||||
if pending:
|
||||
findings.append(issue("error", "pending_evidence_file", "Evidence file still contains a pending template.", path=path))
|
||||
if not stripped:
|
||||
findings.append(issue("error", "empty_evidence_file", "Evidence file is empty.", path=path))
|
||||
elif path.suffix.casefold() == ".json":
|
||||
passed, status = check_json_evidence(path, text, findings)
|
||||
check["status"] = status
|
||||
check["filled"] = passed and not pending
|
||||
else:
|
||||
passed, status = check_text_evidence(path, text, findings)
|
||||
check["status"] = status
|
||||
check["filled"] = passed and not pending
|
||||
file_checks.append(check)
|
||||
|
||||
manifest_path = root / "_codex_validation_evidence_manifest.json"
|
||||
if not manifest_path.exists():
|
||||
findings.append(issue("warning", "missing_evidence_manifest", "Evidence manifest is missing.", path=manifest_path))
|
||||
|
||||
return build_result(plan_path, root, findings, file_checks)
|
||||
|
||||
|
||||
def build_result(plan_path: Path, root: Path, findings: list[dict[str, Any]], file_checks: list[dict[str, Any]]) -> dict[str, Any]:
|
||||
errors = [row for row in findings if row.get("severity") == "error"]
|
||||
warnings = [row for row in findings if row.get("severity") == "warning"]
|
||||
return {
|
||||
"schema": "onec_extension_validation_evidence_check.v1",
|
||||
"plan_path": str(plan_path),
|
||||
"evidence_root": str(root),
|
||||
"passed": not errors,
|
||||
"status": "validated" if not errors else "pending_or_blocked",
|
||||
"findings": findings,
|
||||
"file_checks": file_checks,
|
||||
"counts": {
|
||||
"files": len(file_checks),
|
||||
"filled": sum(1 for item in file_checks if item.get("filled")),
|
||||
"errors": len(errors),
|
||||
"warnings": len(warnings),
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Check manual evidence files for a 1C extension validation plan.")
|
||||
parser.add_argument("--plan", type=Path, required=True)
|
||||
parser.add_argument("--evidence-root", type=Path)
|
||||
parser.add_argument("--output", type=Path)
|
||||
args = parser.parse_args()
|
||||
|
||||
result = check_evidence(args.plan, args.evidence_root)
|
||||
if args.output:
|
||||
write_json(args.output, result)
|
||||
print(json.dumps({"output": str(args.output) if args.output else None, "passed": result["passed"], "status": result["status"], "counts": result["counts"]}, ensure_ascii=False))
|
||||
return 0 if result["passed"] else 2
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,142 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Aggregate final validation gates for a staged 1C extension."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from check_1c_extension_staging import check_staging
|
||||
from check_1c_extension_validation_evidence import check_evidence
|
||||
|
||||
|
||||
def load_json(path: Path) -> dict[str, Any]:
|
||||
return json.loads(path.read_text(encoding="utf-8-sig"))
|
||||
|
||||
|
||||
def write_json(path: Path, data: dict[str, Any]) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(json.dumps(data, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
|
||||
|
||||
def issue(severity: str, code: str, message: str, *, path: Path | str | None = None, detail: dict[str, Any] | None = None) -> dict[str, Any]:
|
||||
result: dict[str, Any] = {"severity": severity, "code": code, "message": message}
|
||||
if path is not None:
|
||||
result["path"] = str(path)
|
||||
if detail:
|
||||
result["detail"] = detail
|
||||
return result
|
||||
|
||||
|
||||
def collect_gate(name: str, data: dict[str, Any]) -> dict[str, Any]:
|
||||
return {
|
||||
"name": name,
|
||||
"schema": data.get("schema"),
|
||||
"passed": data.get("passed"),
|
||||
"status": data.get("status"),
|
||||
"counts": data.get("counts"),
|
||||
}
|
||||
|
||||
|
||||
def check_release(plan_path: Path, evidence_root: Path | None = None) -> dict[str, Any]:
|
||||
findings: list[dict[str, Any]] = []
|
||||
if not plan_path.exists():
|
||||
findings.append(issue("error", "missing_validation_plan", "Validation plan is missing.", path=plan_path))
|
||||
return build_result(plan_path, None, {}, {}, findings)
|
||||
|
||||
plan = load_json(plan_path)
|
||||
if plan.get("schema") != "onec_extension_validation_plan.v1":
|
||||
findings.append(issue("error", "invalid_validation_plan_schema", "Validation plan schema is not onec_extension_validation_plan.v1.", path=plan_path, detail={"schema": plan.get("schema")}))
|
||||
|
||||
if plan.get("status") != "ready_for_disposable_validation":
|
||||
findings.append(issue("error", "validation_plan_not_ready", "Validation plan must be ready_for_disposable_validation.", path=plan_path, detail={"status": plan.get("status")}))
|
||||
|
||||
staging_dir = Path(str(plan.get("staging_dir") or ""))
|
||||
staging_check = check_staging(staging_dir) if staging_dir else {"schema": "onec_extension_staging_check.v1", "passed": False, "counts": {"errors": 1}, "findings": []}
|
||||
if not staging_check.get("passed"):
|
||||
findings.append(issue("error", "staging_check_failed", "Staging check failed.", path=staging_dir, detail={"counts": staging_check.get("counts")}))
|
||||
|
||||
evidence_check = check_evidence(plan_path, evidence_root)
|
||||
if not evidence_check.get("passed"):
|
||||
findings.append(issue("error", "validation_evidence_check_failed", "Validation evidence check failed.", path=evidence_check.get("evidence_root"), detail={"counts": evidence_check.get("counts")}))
|
||||
|
||||
safety = plan.get("safety") if isinstance(plan.get("safety"), dict) else {}
|
||||
expected_safety = {
|
||||
"production_base_allowed": False,
|
||||
"sql_write_allowed": False,
|
||||
"source_extension_write_allowed": False,
|
||||
"disposable_base_required": True,
|
||||
}
|
||||
for key, expected in expected_safety.items():
|
||||
if safety.get(key) is not expected:
|
||||
findings.append(issue("error", "invalid_release_safety_flag", "Validation plan safety flag has an unexpected value.", path=plan_path, detail={"flag": key, "expected": expected, "actual": safety.get(key)}))
|
||||
|
||||
return build_result(plan_path, plan, staging_check, evidence_check, findings)
|
||||
|
||||
|
||||
def build_result(plan_path: Path, plan: dict[str, Any] | None, staging_check: dict[str, Any], evidence_check: dict[str, Any], findings: list[dict[str, Any]]) -> dict[str, Any]:
|
||||
errors = [row for row in findings if row.get("severity") == "error"]
|
||||
warnings = [row for row in findings if row.get("severity") == "warning"]
|
||||
return {
|
||||
"schema": "onec_extension_validation_release_check.v1",
|
||||
"created_at_utc": datetime.now(timezone.utc).isoformat(),
|
||||
"plan_path": str(plan_path),
|
||||
"staging_dir": (plan or {}).get("staging_dir"),
|
||||
"bundle_dir": (plan or {}).get("bundle_dir"),
|
||||
"preferred_extension": (plan or {}).get("preferred_extension"),
|
||||
"passed": not errors,
|
||||
"status": "validated_for_human_review" if not errors else "blocked",
|
||||
"safety": {
|
||||
"production_apply_allowed": False,
|
||||
"automatic_apply_allowed": False,
|
||||
"human_approval_required": True,
|
||||
},
|
||||
"gates": [
|
||||
{
|
||||
"name": "validation_plan",
|
||||
"schema": (plan or {}).get("schema"),
|
||||
"passed": (plan or {}).get("status") == "ready_for_disposable_validation",
|
||||
"status": (plan or {}).get("status"),
|
||||
},
|
||||
collect_gate("staging_check", staging_check),
|
||||
collect_gate("validation_evidence_check", evidence_check),
|
||||
],
|
||||
"findings": findings,
|
||||
"counts": {
|
||||
"errors": len(errors),
|
||||
"warnings": len(warnings),
|
||||
},
|
||||
"next_actions": [
|
||||
"Review validation evidence and changed files manually.",
|
||||
"Do not apply to production automatically.",
|
||||
"If approved, perform production action through the approved human-controlled 1C release process.",
|
||||
] if not errors else [
|
||||
"Fix failed gates before review.",
|
||||
"Do not package, apply, or release this extension from the current evidence.",
|
||||
],
|
||||
"details": {
|
||||
"staging_check": staging_check,
|
||||
"validation_evidence_check": evidence_check,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Aggregate final validation gates for a staged 1C extension.")
|
||||
parser.add_argument("--plan", type=Path, required=True)
|
||||
parser.add_argument("--evidence-root", type=Path)
|
||||
parser.add_argument("--output", type=Path)
|
||||
args = parser.parse_args()
|
||||
|
||||
result = check_release(args.plan, args.evidence_root)
|
||||
if args.output:
|
||||
write_json(args.output, result)
|
||||
print(json.dumps({"output": str(args.output) if args.output else None, "passed": result["passed"], "status": result["status"], "counts": result["counts"]}, ensure_ascii=False))
|
||||
return 0 if result["passed"] else 2
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,183 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
from html.parser import HTMLParser
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from fetch_1c_its_docs import charset_from_content_type, request_safe_url
|
||||
from one_c_its_platform import materialize_doc_url
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
DEFAULT_OUTPUT = ROOT / "reports" / "1c-its-access-check.json"
|
||||
DEFAULT_TEST_URL = "https://its.1c.ru/db/v8316doc#bookmark:dev:TI000000044"
|
||||
|
||||
|
||||
class AccessHtmlParser(HTMLParser):
|
||||
def __init__(self) -> None:
|
||||
super().__init__(convert_charrefs=True)
|
||||
self.title_parts: list[str] = []
|
||||
self._in_title = False
|
||||
self.login_links = 0
|
||||
self.user_profile_markers = 0
|
||||
self.paywall_markers = 0
|
||||
self.data_access_false = 0
|
||||
self.iframe_srcs: list[str] = []
|
||||
|
||||
def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
|
||||
tag = tag.lower()
|
||||
values = {name.lower(): value or "" for name, value in attrs}
|
||||
if tag == "title":
|
||||
self._in_title = True
|
||||
href = values.get("href", "")
|
||||
class_name = values.get("class", "")
|
||||
if "/user/auth" in href:
|
||||
self.login_links += 1
|
||||
if "paywall" in class_name:
|
||||
self.paywall_markers += 1
|
||||
if values.get("data-access") == "false":
|
||||
self.data_access_false += 1
|
||||
if tag == "iframe" and values.get("src"):
|
||||
self.iframe_srcs.append(values["src"])
|
||||
|
||||
def handle_endtag(self, tag: str) -> None:
|
||||
if tag.lower() == "title":
|
||||
self._in_title = False
|
||||
|
||||
def handle_data(self, data: str) -> None:
|
||||
if self._in_title:
|
||||
self.title_parts.append(data.strip())
|
||||
if "Общий профиль" in data or "Доступ до" in data:
|
||||
self.user_profile_markers += 1
|
||||
|
||||
@property
|
||||
def title(self) -> str:
|
||||
return " ".join(part for part in self.title_parts if part)
|
||||
|
||||
|
||||
def read_cookie(cookie_file: Path | None) -> str:
|
||||
if cookie_file:
|
||||
return cookie_file.read_text(encoding="utf-8").strip()
|
||||
return os.environ.get("ONEC_ITS_COOKIE", "").strip()
|
||||
|
||||
|
||||
def fetch_text(url: str, *, cookie: str, timeout: int, referer: str | None = None) -> tuple[int, str, str]:
|
||||
headers = {"User-Agent": "Codex 1C ITS access check"}
|
||||
if cookie:
|
||||
headers["Cookie"] = cookie
|
||||
if referer:
|
||||
headers["Referer"] = referer
|
||||
headers["X-Referer"] = referer
|
||||
request = urllib.request.Request(request_safe_url(url), headers=headers)
|
||||
with urllib.request.urlopen(request, timeout=timeout) as response:
|
||||
body = response.read()
|
||||
content_type = response.headers.get("Content-Type", "")
|
||||
charset = charset_from_content_type(content_type)
|
||||
return int(response.status), content_type, body.decode(charset, errors="replace")
|
||||
|
||||
|
||||
def first_src_url(page_url: str, iframe_srcs: list[str]) -> str | None:
|
||||
for src in iframe_srcs:
|
||||
if "/db/content/" in src and "/src/" in src:
|
||||
return urllib.parse.urljoin(page_url, src)
|
||||
return None
|
||||
|
||||
|
||||
def check_access(url: str, *, cookie: str, timeout: int) -> dict[str, Any]:
|
||||
materialized = materialize_doc_url(url)
|
||||
result: dict[str, Any] = {
|
||||
"schema": "onec_its_access_check.v1",
|
||||
"checked_at_unix": int(time.time()),
|
||||
"target_url": url,
|
||||
"materialized_url": materialized,
|
||||
"cookie_present": bool(cookie.strip()),
|
||||
"page": {},
|
||||
"src": {},
|
||||
"status": "unknown",
|
||||
"findings": [],
|
||||
}
|
||||
if not cookie.strip():
|
||||
result["status"] = "failed"
|
||||
result["findings"].append("cookie_missing")
|
||||
return result
|
||||
|
||||
try:
|
||||
status, content_type, text = fetch_text(materialized, cookie=cookie, timeout=timeout)
|
||||
parser = AccessHtmlParser()
|
||||
parser.feed(text)
|
||||
src_url = first_src_url(materialized, parser.iframe_srcs)
|
||||
result["page"] = {
|
||||
"http_status": status,
|
||||
"content_type": content_type,
|
||||
"title": parser.title,
|
||||
"login_links": parser.login_links,
|
||||
"user_profile_markers": parser.user_profile_markers,
|
||||
"paywall_markers": parser.paywall_markers,
|
||||
"data_access_false": parser.data_access_false,
|
||||
"iframe_src": src_url,
|
||||
}
|
||||
if parser.login_links:
|
||||
result["findings"].append("login_links_present")
|
||||
if parser.paywall_markers:
|
||||
result["findings"].append("paywall_marker_present")
|
||||
if parser.data_access_false:
|
||||
result["findings"].append("data_access_false")
|
||||
except Exception as exc: # noqa: BLE001
|
||||
result["page"] = {"error": f"{type(exc).__name__}: {exc}"}
|
||||
result["status"] = "failed"
|
||||
result["findings"].append("page_fetch_failed")
|
||||
return result
|
||||
|
||||
src_url = result["page"].get("iframe_src")
|
||||
if src_url:
|
||||
try:
|
||||
src_status, src_content_type, src_text = fetch_text(str(src_url), cookie=cookie, timeout=timeout, referer=materialized)
|
||||
visible_words = len(re.findall(r"[A-Za-zА-Яа-яЁё0-9_]+", src_text))
|
||||
result["src"] = {
|
||||
"http_status": src_status,
|
||||
"content_type": src_content_type,
|
||||
"chars": len(src_text),
|
||||
"word_count": visible_words,
|
||||
}
|
||||
if visible_words < 50:
|
||||
result["findings"].append("src_low_text")
|
||||
except urllib.error.HTTPError as exc:
|
||||
result["src"] = {"http_status": exc.code, "error": str(exc), "url": src_url}
|
||||
result["findings"].append(f"src_http_{exc.code}")
|
||||
except Exception as exc: # noqa: BLE001
|
||||
result["src"] = {"error": f"{type(exc).__name__}: {exc}", "url": src_url}
|
||||
result["findings"].append("src_fetch_failed")
|
||||
else:
|
||||
result["findings"].append("src_iframe_missing")
|
||||
|
||||
blocking = {"login_links_present", "paywall_marker_present", "data_access_false", "src_http_401", "src_fetch_failed"}
|
||||
result["status"] = "failed" if any(item in blocking for item in result["findings"]) else "ok"
|
||||
return result
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Check whether the stored 1C:ITS cookie can access protected documentation bodies.")
|
||||
parser.add_argument("--url", default=DEFAULT_TEST_URL)
|
||||
parser.add_argument("--cookie-file", type=Path)
|
||||
parser.add_argument("--output", type=Path, default=DEFAULT_OUTPUT)
|
||||
parser.add_argument("--timeout", type=int, default=30)
|
||||
parser.add_argument("--print", action="store_true", dest="print_report")
|
||||
args = parser.parse_args()
|
||||
|
||||
report = check_access(args.url, cookie=read_cookie(args.cookie_file), timeout=args.timeout)
|
||||
args.output.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.output.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
print(json.dumps(report if args.print_report else {"status": report["status"], "findings": report["findings"], "output": str(args.output)}, ensure_ascii=False, indent=2))
|
||||
return 0 if report["status"] == "ok" else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,95 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
from normalize_1c_its_cookie import normalize_cookie
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
DEFAULT_OUTPUT = ROOT / "reports" / "1c-its-cookie-normalizer-check.json"
|
||||
|
||||
|
||||
CASES = [
|
||||
{
|
||||
"name": "request_cookie_header",
|
||||
"input": "Cookie: sid=abc; theme=dark",
|
||||
"ok": True,
|
||||
"cookie": "sid=abc; theme=dark",
|
||||
},
|
||||
{
|
||||
"name": "reject_yandex_set_cookie",
|
||||
"input": "bh=abc; Domain=.yandex.com; Path=/",
|
||||
"ok": False,
|
||||
"error": "domain_attributes_do_not_match_target",
|
||||
},
|
||||
{
|
||||
"name": "accept_its_set_cookie",
|
||||
"input": "sid=abc; Domain=.its.1c.ru; Path=/",
|
||||
"ok": True,
|
||||
"cookie": "sid=abc",
|
||||
},
|
||||
{
|
||||
"name": "json_filters_target_domain",
|
||||
"input": '[{"domain":"its.1c.ru","name":"sid","value":"abc"},{"domain":"yandex.com","name":"bh","value":"no"}]',
|
||||
"ok": True,
|
||||
"cookie": "sid=abc",
|
||||
},
|
||||
{
|
||||
"name": "netscape_filters_target_domain",
|
||||
"input": ".its.1c.ru\tTRUE\t/\tTRUE\t0\tsid\tabc\n.yandex.com\tTRUE\t/\tTRUE\t0\tbh\tno",
|
||||
"ok": True,
|
||||
"cookie": "sid=abc",
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
def run_cases() -> dict:
|
||||
checks = []
|
||||
for case in CASES:
|
||||
result = normalize_cookie(case["input"], "its.1c.ru")
|
||||
passed = result["ok"] == case["ok"]
|
||||
if "cookie" in case:
|
||||
passed = passed and result["cookie"] == case["cookie"]
|
||||
if "error" in case:
|
||||
passed = passed and case["error"] in result["errors"]
|
||||
checks.append(
|
||||
{
|
||||
"name": case["name"],
|
||||
"status": "passed" if passed else "failed",
|
||||
"expected": {key: case[key] for key in ("ok", "cookie", "error") if key in case},
|
||||
"actual": {
|
||||
"ok": result["ok"],
|
||||
"cookie": result["cookie"],
|
||||
"warnings": result["warnings"],
|
||||
"errors": result["errors"],
|
||||
"format": result.get("format"),
|
||||
},
|
||||
}
|
||||
)
|
||||
return {
|
||||
"schema": "onec_its_cookie_normalizer_check.v1",
|
||||
"passed": all(check["status"] == "passed" for check in checks),
|
||||
"checks": checks,
|
||||
}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Check 1C:ITS cookie normalizer behavior.")
|
||||
parser.add_argument("--output", type=Path, default=DEFAULT_OUTPUT)
|
||||
parser.add_argument("--print", action="store_true", dest="print_report")
|
||||
args = parser.parse_args()
|
||||
|
||||
report = run_cases()
|
||||
args.output.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.output.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
if args.print_report:
|
||||
print(json.dumps(report, ensure_ascii=False, indent=2))
|
||||
else:
|
||||
print(json.dumps({"passed": report["passed"], "output": str(args.output)}, ensure_ascii=False))
|
||||
return 0 if report["passed"] else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,129 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from collections import deque
|
||||
from pathlib import Path
|
||||
|
||||
import yaml
|
||||
|
||||
from fetch_1c_its_docs import LinkParser, enqueue_links, merged_policy, page_record
|
||||
from normalize_1c_its_docs import TextExtractor, clean_its_text, content_quality
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
SOURCES = ROOT / "plugins" / "1c" / "rag" / "official-docs" / "sources.yaml"
|
||||
|
||||
|
||||
def assert_true(condition: bool, message: str) -> None:
|
||||
if not condition:
|
||||
raise AssertionError(message)
|
||||
|
||||
|
||||
def test_src_priority_and_title_hint() -> None:
|
||||
html = """
|
||||
<html><head><title>Форма :: Глоссарий разработчика</title></head>
|
||||
<body>
|
||||
<iframe src="/db/content/v8devgloss/src/term_000000052.htm"></iframe>
|
||||
<a href="/db/v8devgloss/content/900049/hdoc">1</a>
|
||||
<a href="/db/v8devgloss/content/99/hdoc">XDTO</a>
|
||||
</body></html>
|
||||
"""
|
||||
parser = LinkParser()
|
||||
parser.feed(html)
|
||||
config = yaml.safe_load(SOURCES.read_text(encoding="utf-8"))
|
||||
source = config["sources"][0]
|
||||
policy = merged_policy(config["default_policy"], source.get("policy"))
|
||||
queue: deque = deque()
|
||||
enqueue_links(
|
||||
queue,
|
||||
source=source,
|
||||
base_url="https://its.1c.ru/db/v8devgloss/content/52/hdoc",
|
||||
depth=1,
|
||||
policy=policy,
|
||||
links=parser.links,
|
||||
seen=set(),
|
||||
title_hint=parser.title,
|
||||
)
|
||||
first = queue[0]
|
||||
assert_true(first[1] == "https://its.1c.ru/db/content/v8devgloss/src/term_000000052.htm", "src iframe URL must be first")
|
||||
assert_true(first[4] == "Форма :: Глоссарий разработчика", "src iframe URL must inherit hdoc title")
|
||||
|
||||
|
||||
def test_src_only_at_max_depth() -> None:
|
||||
html = """
|
||||
<html><head><title>Форма :: Глоссарий разработчика</title></head>
|
||||
<body>
|
||||
<iframe src="/db/content/v8devgloss/src/term_000000052.htm"></iframe>
|
||||
<a href="/db/v8devgloss/content/99/hdoc">XDTO</a>
|
||||
<a href="/db/v8devgloss/content/14/hdoc">1С:Предприятие</a>
|
||||
</body></html>
|
||||
"""
|
||||
parser = LinkParser()
|
||||
parser.feed(html)
|
||||
config = yaml.safe_load(SOURCES.read_text(encoding="utf-8"))
|
||||
source = config["sources"][0]
|
||||
policy = merged_policy(config["default_policy"], source.get("policy"))
|
||||
queue: deque = deque()
|
||||
enqueue_links(
|
||||
queue,
|
||||
source=source,
|
||||
base_url="https://its.1c.ru/db/v8devgloss/content/52/hdoc",
|
||||
depth=2,
|
||||
policy=policy,
|
||||
links=parser.links,
|
||||
seen=set(),
|
||||
title_hint=parser.title,
|
||||
src_only=True,
|
||||
)
|
||||
urls = [item[1] for item in queue]
|
||||
assert_true(urls == ["https://its.1c.ru/db/content/v8devgloss/src/term_000000052.htm"], "max-depth src-only mode must keep only iframe src")
|
||||
|
||||
|
||||
def test_page_record_uses_title_hint() -> None:
|
||||
output_dir = ROOT / "reports" / ".tmp-its-ingestion-check"
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
record = page_record(
|
||||
source={"id": "v8devgloss", "title": "Глоссарий разработчика", "source_type": "official_1c_its_glossary"},
|
||||
url="https://its.1c.ru/db/content/v8devgloss/src/term_000000052.htm",
|
||||
depth=2,
|
||||
body="<html><body>Текст определения формы.</body></html>".encode("utf-8"),
|
||||
headers={"Content-Type": "text/html; charset=utf-8"},
|
||||
status=200,
|
||||
output_dir=output_dir,
|
||||
title_hint="Форма :: Глоссарий разработчика",
|
||||
)
|
||||
assert_true(record["title"] == "Форма :: Глоссарий разработчика", "title_hint must be used when src page has no title")
|
||||
|
||||
|
||||
def test_glossary_src_fallback_text() -> None:
|
||||
extractor = TextExtractor()
|
||||
extractor.feed(
|
||||
"<html><body><p>"
|
||||
"Форма предназначена для отображения и редактирования данных объекта, "
|
||||
"содержит элементы управления, команды, реквизиты формы и обработчики событий, "
|
||||
"которые используются прикладным решением при работе пользователя в интерфейсе приложения."
|
||||
"</p></body></html>"
|
||||
)
|
||||
text = clean_its_text(extractor.text(), "Форма :: Глоссарий разработчика", "official_1c_its_glossary")
|
||||
quality = content_quality(text, "Форма :: Глоссарий разработчика")
|
||||
assert_true("Форма предназначена" in text, "glossary src body must be preserved")
|
||||
assert_true(bool(quality["is_content"]), "glossary src body must pass quality gate")
|
||||
|
||||
|
||||
def main() -> int:
|
||||
checks = [
|
||||
test_src_priority_and_title_hint,
|
||||
test_src_only_at_max_depth,
|
||||
test_page_record_uses_title_hint,
|
||||
test_glossary_src_fallback_text,
|
||||
]
|
||||
results = []
|
||||
for check in checks:
|
||||
check()
|
||||
results.append({"id": check.__name__, "status": "passed"})
|
||||
print(json.dumps({"schema": "onec_its_ingestion_logic_check.v1", "status": "ok", "checks": results}, ensure_ascii=False, indent=2))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,89 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import yaml
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
DEFAULT_START_LINKS = ROOT / "plugins" / "1c" / "rag" / "official-docs" / "start-links.json"
|
||||
DEFAULT_SOURCES = ROOT / "plugins" / "1c" / "rag" / "official-docs" / "sources.yaml"
|
||||
DEFAULT_OUTPUT = ROOT / "reports" / "1c-official-docs-start-coverage.json"
|
||||
|
||||
|
||||
def load_json(path: Path) -> dict[str, Any]:
|
||||
return json.loads(path.read_text(encoding="utf-8-sig")) if path.exists() else {}
|
||||
|
||||
|
||||
def load_yaml(path: Path) -> dict[str, Any]:
|
||||
data = yaml.safe_load(path.read_text(encoding="utf-8"))
|
||||
return data if isinstance(data, dict) else {}
|
||||
|
||||
|
||||
def check_coverage(start_links_path: Path, sources_path: Path) -> dict[str, Any]:
|
||||
start_links = load_json(start_links_path)
|
||||
sources_config = load_yaml(sources_path)
|
||||
source_urls = {str(item.get("url") or "") for item in sources_config.get("sources") or []}
|
||||
candidates = start_links.get("start_links") or []
|
||||
rows = []
|
||||
by_category: dict[str, dict[str, int]] = {}
|
||||
for item in candidates:
|
||||
category = str(item.get("category") or "unknown")
|
||||
active = str(item.get("url") or "") in source_urls
|
||||
by_category.setdefault(category, {"candidates": 0, "active": 0, "inactive": 0})
|
||||
by_category[category]["candidates"] += 1
|
||||
by_category[category]["active" if active else "inactive"] += 1
|
||||
rows.append({**item, "active": active})
|
||||
inactive = [item for item in rows if not item["active"]]
|
||||
counts = {
|
||||
"sources": len(source_urls),
|
||||
"candidates": len(rows),
|
||||
"active_candidates": sum(1 for item in rows if item["active"]),
|
||||
"inactive_candidates": len(inactive),
|
||||
}
|
||||
findings = []
|
||||
required_categories = {
|
||||
"dev_section",
|
||||
"dev_section_index",
|
||||
"developer_glossary",
|
||||
"development_standards",
|
||||
"methodical_support",
|
||||
"platform_doc",
|
||||
}
|
||||
for category in sorted(required_categories):
|
||||
stats = by_category.get(category) or {}
|
||||
if stats.get("active", 0) == 0:
|
||||
findings.append({"severity": "error", "message": f"no active source for required category {category}"})
|
||||
return {
|
||||
"schema": "onec_its_start_link_coverage.v1",
|
||||
"passed": not any(item["severity"] == "error" for item in findings),
|
||||
"start_links": str(start_links_path),
|
||||
"sources": str(sources_path),
|
||||
"counts": counts,
|
||||
"by_category": dict(sorted(by_category.items())),
|
||||
"findings": findings,
|
||||
"inactive_samples": inactive[:50],
|
||||
}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Compare discovered 1C:ITS start links with active sources.yaml seeds.")
|
||||
parser.add_argument("--start-links", type=Path, default=DEFAULT_START_LINKS)
|
||||
parser.add_argument("--sources", type=Path, default=DEFAULT_SOURCES)
|
||||
parser.add_argument("--output", type=Path, default=DEFAULT_OUTPUT)
|
||||
parser.add_argument("--print", action="store_true", dest="print_report")
|
||||
args = parser.parse_args()
|
||||
|
||||
report = check_coverage(args.start_links, args.sources)
|
||||
args.output.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.output.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
if args.print_report:
|
||||
print(json.dumps(report, ensure_ascii=False, indent=2))
|
||||
return 0 if report["passed"] else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,124 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import re
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
DEFAULT_STATIC_DIR = ROOT / "plugins" / "1c" / "rag" / "official-docs" / "static"
|
||||
DEFAULT_OUTPUT = ROOT / "reports" / "1c-official-docs-static-check.json"
|
||||
|
||||
|
||||
LOCAL_REF_RE = re.compile(r"""(?:href|src)=["']([^"']+)["']""", re.IGNORECASE)
|
||||
EXTERNAL_IMG_RE = re.compile(r"""<img[^>]+src=["']https?://""", re.IGNORECASE)
|
||||
EXTERNAL_ASSET_RE = re.compile(r"""<(?:link|script)[^>]+(?:href|src)=["']https?://""", re.IGNORECASE)
|
||||
EXTERNAL_LINK_RE = re.compile(r"""<a[^>]+href=["']https?://""", re.IGNORECASE)
|
||||
|
||||
|
||||
def html_files(static_dir: Path) -> list[Path]:
|
||||
if not static_dir.exists():
|
||||
return []
|
||||
return sorted(static_dir.rglob("*.html"))
|
||||
|
||||
|
||||
def is_local_ref(value: str) -> bool:
|
||||
lowered = value.casefold()
|
||||
return not (
|
||||
lowered.startswith("http://")
|
||||
or lowered.startswith("https://")
|
||||
or lowered.startswith("mailto:")
|
||||
or lowered.startswith("javascript:")
|
||||
or lowered.startswith("#")
|
||||
)
|
||||
|
||||
|
||||
def check_local_refs(path: Path, text: str) -> list[dict[str, str]]:
|
||||
broken = []
|
||||
for match in LOCAL_REF_RE.finditer(text):
|
||||
ref = match.group(1)
|
||||
if not is_local_ref(ref):
|
||||
continue
|
||||
target = (path.parent / ref.split("#", 1)[0].split("?", 1)[0]).resolve()
|
||||
if not target.exists():
|
||||
broken.append({"file": str(path), "ref": ref})
|
||||
return broken
|
||||
|
||||
|
||||
def build_report(static_dir: Path) -> dict[str, Any]:
|
||||
files = html_files(static_dir)
|
||||
external_images = []
|
||||
external_assets = []
|
||||
external_links = []
|
||||
broken_refs = []
|
||||
for path in files:
|
||||
text = path.read_text(encoding="utf-8-sig", errors="replace")
|
||||
if EXTERNAL_IMG_RE.search(text):
|
||||
external_images.append(str(path))
|
||||
if EXTERNAL_ASSET_RE.search(text):
|
||||
external_assets.append(str(path))
|
||||
external_links.extend({"file": str(path), "count": len(EXTERNAL_LINK_RE.findall(text))} for _ in [0] if EXTERNAL_LINK_RE.search(text))
|
||||
broken_refs.extend(check_local_refs(path, text))
|
||||
|
||||
manifest_path = static_dir / "manifest.json"
|
||||
manifest = {}
|
||||
if manifest_path.exists():
|
||||
manifest = json.loads(manifest_path.read_text(encoding="utf-8-sig"))
|
||||
counts = {
|
||||
"html_files": len(files),
|
||||
"pages": len((manifest.get("pages") or [])) if isinstance(manifest, dict) else 0,
|
||||
"media_files": len(list((static_dir / "media").glob("*"))) if (static_dir / "media").exists() else 0,
|
||||
"asset_files": len(list((static_dir / "assets").glob("*"))) if (static_dir / "assets").exists() else 0,
|
||||
"external_image_pages": len(external_images),
|
||||
"external_asset_pages": len(external_assets),
|
||||
"external_link_pages": len(external_links),
|
||||
"broken_local_refs": len(broken_refs),
|
||||
"asset_errors": len((manifest.get("asset_errors") or [])) if isinstance(manifest, dict) else 0,
|
||||
}
|
||||
findings = []
|
||||
if not (static_dir / "index.html").exists():
|
||||
findings.append({"severity": "error", "message": "static index.html is missing"})
|
||||
if counts["external_image_pages"]:
|
||||
findings.append({"severity": "error", "message": "some static pages still reference remote images"})
|
||||
if counts["external_asset_pages"]:
|
||||
findings.append({"severity": "warning", "message": "some raw pages still reference remote CSS/JS assets"})
|
||||
if counts["broken_local_refs"]:
|
||||
findings.append({"severity": "error", "message": "some local href/src references are broken"})
|
||||
if counts["asset_errors"]:
|
||||
findings.append({"severity": "warning", "message": "some CSS/JS assets failed to download"})
|
||||
|
||||
return {
|
||||
"schema": "onec_its_static_site_check.v1",
|
||||
"passed": not any(item["severity"] == "error" for item in findings),
|
||||
"static_dir": str(static_dir),
|
||||
"counts": counts,
|
||||
"findings": findings,
|
||||
"samples": {
|
||||
"external_images": external_images[:20],
|
||||
"external_assets": external_assets[:20],
|
||||
"external_links": external_links[:20],
|
||||
"broken_local_refs": broken_refs[:20],
|
||||
"asset_errors": (manifest.get("asset_errors") or [])[:20] if isinstance(manifest, dict) else [],
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Check local static 1C:ITS archive self-containment and links.")
|
||||
parser.add_argument("--static-dir", type=Path, default=DEFAULT_STATIC_DIR)
|
||||
parser.add_argument("--output", type=Path, default=DEFAULT_OUTPUT)
|
||||
parser.add_argument("--print", action="store_true", dest="print_report")
|
||||
args = parser.parse_args()
|
||||
|
||||
report = build_report(args.static_dir)
|
||||
args.output.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.output.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
if args.print_report:
|
||||
print(json.dumps(report, ensure_ascii=False, indent=2))
|
||||
return 0 if report["passed"] else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,552 @@
|
||||
#!/usr/bin/env python3
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "scripts"))
|
||||
sys.path.insert(0, str(ROOT / "plugins" / "1c" / "connector"))
|
||||
sys.path.insert(0, str(ROOT / "plugins" / "1c" / "mcp"))
|
||||
|
||||
import adapter_1c_mcp as adapter_mcp # noqa: E402
|
||||
import adapter_1c_server as adapter_server # noqa: E402
|
||||
import smoke_1c_mcp_selector_chain as selector_chain_smoke # noqa: E402
|
||||
|
||||
|
||||
FORBIDDEN_CONCRETE_SELECTOR_VALUES = (
|
||||
"ПечатьЭтикеток",
|
||||
"УОП_Печать",
|
||||
"ЦенаСоСкидкой",
|
||||
"fs_Отчеты",
|
||||
)
|
||||
PRODUCTION_SELECTOR_CONTRACT_ROOTS = (
|
||||
ROOT / "plugins" / "1c",
|
||||
ROOT / "docs",
|
||||
)
|
||||
PRODUCTION_SELECTOR_CONTRACT_SUFFIXES = {
|
||||
".css",
|
||||
".html",
|
||||
".js",
|
||||
".json",
|
||||
".md",
|
||||
".ps1",
|
||||
".py",
|
||||
".toml",
|
||||
".txt",
|
||||
".yaml",
|
||||
".yml",
|
||||
}
|
||||
SYNTHETIC_SELECTOR_GUID = "00000000-0000-4000-8000-000000000001"
|
||||
PRODUCTION_SELECTOR_CONTRACT_EXCLUDED_PARTS = {
|
||||
"__pycache__",
|
||||
".pytest_cache",
|
||||
"reports",
|
||||
"node_modules",
|
||||
".git",
|
||||
}
|
||||
|
||||
|
||||
def find_tool(name: str) -> dict[str, Any] | None:
|
||||
for tool in adapter_mcp.TOOLS:
|
||||
if tool.get("name") == name:
|
||||
return tool
|
||||
return None
|
||||
|
||||
|
||||
def iter_production_selector_contract_files() -> list[Path]:
|
||||
files: list[Path] = []
|
||||
for root in PRODUCTION_SELECTOR_CONTRACT_ROOTS:
|
||||
if not root.exists():
|
||||
continue
|
||||
for path in root.rglob("*"):
|
||||
if not path.is_file():
|
||||
continue
|
||||
try:
|
||||
relative_parts = set(path.relative_to(ROOT).parts)
|
||||
except ValueError:
|
||||
relative_parts = set(path.parts)
|
||||
if relative_parts.intersection(PRODUCTION_SELECTOR_CONTRACT_EXCLUDED_PARTS):
|
||||
continue
|
||||
if path.suffix.lower() not in PRODUCTION_SELECTOR_CONTRACT_SUFFIXES:
|
||||
continue
|
||||
files.append(path)
|
||||
return sorted(files)
|
||||
|
||||
|
||||
def contract_checks(issues: list[dict[str, Any]], *, onec_request_present: bool) -> dict[str, bool]:
|
||||
return {
|
||||
"generic_onec_request_schema": onec_request_present
|
||||
and not any(
|
||||
issue["code"].startswith("mcp_onec_request")
|
||||
or issue["code"] in {"mcp_payload_not_open", "mcp_payload_selector_guidance_missing"}
|
||||
for issue in issues
|
||||
),
|
||||
"mcp_tool_selector_guidance": not any(issue["code"] in {"mcp_tool_selector_guidance_missing", "mcp_examples_public_ref_placeholder_missing"} for issue in issues),
|
||||
"all_adapter_methods_forward_over_rpc": not any(issue["code"].startswith("mcp_method") or issue["code"] == "mcp_rpc_body_method_mismatch" for issue in issues),
|
||||
"no_unified_shadowing_adapter_methods": not any(issue["code"] == "mcp_unified_shadows_adapter_methods" for issue in issues),
|
||||
"adapter_help_selector_guidance": not any(issue["code"] == "adapter_help_selector_guidance_missing" for issue in issues),
|
||||
"adapter_selector_argument_type_validation": not any(issue["code"] == "adapter_selector_argument_type_not_validated" for issue in issues),
|
||||
"adapter_selector_normalizer_type_validation": not any(issue["code"] == "adapter_selector_normalizer_type_not_validated" for issue in issues),
|
||||
"adapter_selector_required_message": not any(
|
||||
issue["code"] in {"adapter_selector_required_message_incomplete", "adapter_selector_message_constant_incomplete"}
|
||||
for issue in issues
|
||||
),
|
||||
"adapter_selector_presence_aliases": not any(issue["code"] == "adapter_selector_presence_alias_mismatch" for issue in issues),
|
||||
"adapter_selector_capability_descriptor": not any(
|
||||
issue["code"] in {"adapter_selector_capability_descriptor_mismatch", "adapter_help_selector_capabilities_missing"}
|
||||
for issue in issues
|
||||
),
|
||||
"adapter_definition_read_selector_public_ref": not any(issue["code"] == "adapter_definition_read_selector_public_ref_missing" for issue in issues),
|
||||
"adapter_module_read_selector_public_ref": not any(issue["code"] == "adapter_module_read_selector_public_ref_missing" for issue in issues),
|
||||
"adapter_parse_ordinal_unpacked": not any(issue["code"] == "adapter_parse_ordinal_not_unpacked" for issue in issues),
|
||||
"adapter_contract_version": not any(
|
||||
issue["code"] in {"adapter_contract_version_missing", "mcp_contract_version_mismatch", "adapter_help_contract_version_missing"}
|
||||
for issue in issues
|
||||
),
|
||||
"mcp_policy_blocks_missing_base_id": not any(issue["code"] == "mcp_policy_missing_base_id_not_blocked" for issue in issues),
|
||||
"mcp_policy_blocks_diagnostic_fallback": not any(issue["code"] == "mcp_policy_diagnostic_fallback_not_blocked" for issue in issues),
|
||||
"mcp_policy_allows_explicit_diagnostics": not any(issue["code"] == "mcp_policy_explicit_diagnostic_not_forwarded" for issue in issues),
|
||||
"mcp_selector_schema_template": not any(
|
||||
issue["code"] in {"mcp_selector_schema_properties_mismatch", "mcp_schema_requires_object_type_name_selector"}
|
||||
for issue in issues
|
||||
),
|
||||
"selector_chain_live_coverage_schema": not any(issue["code"].startswith("selector_chain_coverage_") for issue in issues),
|
||||
"selector_chain_strict_composition_gate": not any(issue["code"].startswith("selector_chain_strict_") for issue in issues),
|
||||
"selector_chain_working_state": not any(issue["code"].startswith("selector_chain_working_state_") for issue in issues),
|
||||
"no_concrete_selector_values_in_production": not any(issue["code"] == "production_concrete_selector_value_present" for issue in issues),
|
||||
}
|
||||
|
||||
|
||||
def check_contract() -> dict[str, Any]:
|
||||
issues: list[dict[str, Any]] = []
|
||||
methods = [str(row.get("name") or "") for row in adapter_server.METHODS if row.get("name")]
|
||||
method_set = set(methods)
|
||||
selector_guidance_terms = tuple(getattr(adapter_server, "OBJECT_SELECTOR_GUIDANCE_TERMS", ("ref", "kind/name/guid", "object_type/object_name/object_guid")))
|
||||
adapter_contract_version = str(getattr(adapter_server, "ADAPTER_CONTRACT_VERSION", ""))
|
||||
mcp_contract_version = str(getattr(adapter_mcp, "MCP_CONTRACT_VERSION", ""))
|
||||
if not adapter_contract_version:
|
||||
issues.append({"code": "adapter_contract_version_missing"})
|
||||
if adapter_contract_version != mcp_contract_version:
|
||||
issues.append({"code": "mcp_contract_version_mismatch", "adapter": adapter_contract_version, "mcp": mcp_contract_version})
|
||||
duplicate_methods = sorted({name for name in methods if methods.count(name) > 1})
|
||||
if duplicate_methods:
|
||||
issues.append({"code": "adapter_methods_not_unique", "methods": duplicate_methods})
|
||||
shadowed_methods = sorted(method_set.intersection(getattr(adapter_mcp, "UNIFIED_METHODS", set())))
|
||||
if shadowed_methods:
|
||||
issues.append({"code": "mcp_unified_shadows_adapter_methods", "methods": shadowed_methods})
|
||||
mcp_selector_schema_args = set(getattr(adapter_mcp, "OBJECT_SELECTOR_SCHEMA_PROPERTIES", {}))
|
||||
adapter_selector_args = set(getattr(adapter_server, "OBJECT_SELECTOR_ARGUMENTS", []))
|
||||
if mcp_selector_schema_args != adapter_selector_args:
|
||||
issues.append(
|
||||
{
|
||||
"code": "mcp_selector_schema_properties_mismatch",
|
||||
"mcp_arguments": sorted(mcp_selector_schema_args),
|
||||
"adapter_arguments": sorted(adapter_selector_args),
|
||||
}
|
||||
)
|
||||
selector_capability_methods = set(getattr(adapter_server, "OBJECT_SELECTOR_METHOD_CAPABILITIES", {}))
|
||||
selector_alias_methods = set(getattr(adapter_server, "OBJECT_SELECTOR_ALIAS_METHODS", set()))
|
||||
if selector_capability_methods != selector_alias_methods:
|
||||
issues.append(
|
||||
{
|
||||
"code": "adapter_selector_capability_descriptor_mismatch",
|
||||
"capability_methods": sorted(selector_capability_methods),
|
||||
"alias_methods": sorted(selector_alias_methods),
|
||||
}
|
||||
)
|
||||
mcp_source = Path(adapter_mcp.__file__).read_text(encoding="utf-8")
|
||||
adapter_source = Path(adapter_server.__file__).read_text(encoding="utf-8")
|
||||
for line_number, line in enumerate(adapter_source.splitlines(), start=1):
|
||||
if "parse_ordinal(" not in line or "=" not in line:
|
||||
continue
|
||||
left_side = line.split("=", 1)[0].strip()
|
||||
if "," not in left_side and not left_side.startswith(("return ", "raise ")):
|
||||
issues.append(
|
||||
{
|
||||
"code": "adapter_parse_ordinal_not_unpacked",
|
||||
"line": line_number,
|
||||
"source": line.strip(),
|
||||
}
|
||||
)
|
||||
if '"required": ["base_id", "object_type", "object_name"]' in mcp_source:
|
||||
issues.append({"code": "mcp_schema_requires_object_type_name_selector"})
|
||||
normalizer_check = adapter_server.normalize_object_selector_aliases({"kind": 123}, "contract.selector")
|
||||
if not isinstance(normalizer_check, dict) or normalizer_check.get("status") != "invalid_argument" or normalizer_check.get("argument") != "kind":
|
||||
issues.append(
|
||||
{
|
||||
"code": "adapter_selector_normalizer_type_not_validated",
|
||||
"result": normalizer_check,
|
||||
}
|
||||
)
|
||||
required_message = str(getattr(adapter_server, "OBJECT_SELECTOR_REQUIRED_MESSAGE", ""))
|
||||
for term in (*selector_guidance_terms, "ordinal"):
|
||||
if term not in required_message:
|
||||
issues.append(
|
||||
{
|
||||
"code": "adapter_selector_required_message_incomplete",
|
||||
"term": term,
|
||||
"message": required_message,
|
||||
}
|
||||
)
|
||||
selector_message_constants = {
|
||||
"OBJECT_SELECTOR_REQUIRED_MESSAGE_NO_ORDINAL": selector_guidance_terms,
|
||||
"OBJECT_SELECTOR_GLOBAL_REQUIRED_MESSAGE": (*selector_guidance_terms, "areas metadata/extensions"),
|
||||
"OBJECT_SELECTOR_OR_MODULE_REQUIRED_MESSAGE": (*selector_guidance_terms, "module_ref", "module_id"),
|
||||
"MODULE_READ_SELECTOR_OR_MODULE_ID_MESSAGE": (*selector_guidance_terms, "ordinal", "module_id"),
|
||||
}
|
||||
for constant_name, required_terms in selector_message_constants.items():
|
||||
message = str(getattr(adapter_server, constant_name, ""))
|
||||
for term in required_terms:
|
||||
if term not in message:
|
||||
issues.append(
|
||||
{
|
||||
"code": "adapter_selector_message_constant_incomplete",
|
||||
"constant": constant_name,
|
||||
"term": term,
|
||||
"message": message,
|
||||
}
|
||||
)
|
||||
selector_presence_cases = [
|
||||
({"ref": "Document.ObjectName"}, True),
|
||||
({"object_name": "ObjectName"}, True),
|
||||
({"object_guid": SYNTHETIC_SELECTOR_GUID}, True),
|
||||
({"kind": "Document"}, False),
|
||||
({"object_type": "Document"}, False),
|
||||
]
|
||||
for payload, expected in selector_presence_cases:
|
||||
actual = adapter_server.has_object_selector(payload)
|
||||
if actual is not expected:
|
||||
issues.append(
|
||||
{
|
||||
"code": "adapter_selector_presence_alias_mismatch",
|
||||
"payload": payload,
|
||||
"expected": expected,
|
||||
"actual": actual,
|
||||
}
|
||||
)
|
||||
read_selector = adapter_server.definition_read_selector(
|
||||
"contract_base",
|
||||
{
|
||||
"kind": "DataProcessor",
|
||||
"name": "ObjectName",
|
||||
"guid": SYNTHETIC_SELECTOR_GUID,
|
||||
},
|
||||
method="metadata.object.get",
|
||||
)
|
||||
if read_selector.get("ref") != "DataProcessor.ObjectName":
|
||||
issues.append(
|
||||
{
|
||||
"code": "adapter_definition_read_selector_public_ref_missing",
|
||||
"selector": read_selector,
|
||||
}
|
||||
)
|
||||
module_read_selector = adapter_server.enrich_selector_with_object_ref(
|
||||
{
|
||||
"base_id": "contract_base",
|
||||
"method": "modules.read",
|
||||
"kind": "DataProcessor",
|
||||
"guid": SYNTHETIC_SELECTOR_GUID,
|
||||
"module_ordinal": 1,
|
||||
},
|
||||
{
|
||||
"kind": "DataProcessor",
|
||||
"name": "ObjectName",
|
||||
"guid": SYNTHETIC_SELECTOR_GUID,
|
||||
},
|
||||
)
|
||||
if module_read_selector.get("ref") != "DataProcessor.ObjectName" or module_read_selector.get("name") != "ObjectName":
|
||||
issues.append(
|
||||
{
|
||||
"code": "adapter_module_read_selector_public_ref_missing",
|
||||
"selector": module_read_selector,
|
||||
}
|
||||
)
|
||||
synthetic_live_steps = [
|
||||
{
|
||||
"name": "metadata.resolve_overrides",
|
||||
"status": "not_found",
|
||||
"write_plan_evidence": True,
|
||||
"next_method": adapter_server.SAVED_STATE_MODULES_SEARCH_METHOD,
|
||||
},
|
||||
{
|
||||
"name": adapter_server.SAVED_STATE_MODULES_SEARCH_METHOD,
|
||||
"status": "ok",
|
||||
"modules": 0,
|
||||
"write_plan_target": False,
|
||||
},
|
||||
{
|
||||
"name": adapter_server.METADATA_WRITE_PLAN_METHOD,
|
||||
"status": "skipped_no_saved_state_target",
|
||||
"from_write_plan_target": False,
|
||||
},
|
||||
]
|
||||
live_coverage = selector_chain_smoke.live_coverage_from_steps(synthetic_live_steps)
|
||||
expected_coverage = {
|
||||
"resolve_overrides": {
|
||||
"attempted": True,
|
||||
"status": "not_found",
|
||||
"write_plan_evidence": True,
|
||||
"next_method": adapter_server.SAVED_STATE_MODULES_SEARCH_METHOD,
|
||||
},
|
||||
"saved_state_resolution": {
|
||||
"attempted": True,
|
||||
"status": "ok",
|
||||
"modules": 0,
|
||||
"write_plan_target": False,
|
||||
},
|
||||
"write_plan_composition": {
|
||||
"attempted": True,
|
||||
"status": "skipped_no_saved_state_target",
|
||||
"composed": False,
|
||||
"from_write_plan_target": False,
|
||||
},
|
||||
}
|
||||
for section, expected in expected_coverage.items():
|
||||
actual = live_coverage.get(section)
|
||||
if actual != expected:
|
||||
issues.append(
|
||||
{
|
||||
"code": "selector_chain_coverage_section_mismatch",
|
||||
"section": section,
|
||||
"expected": expected,
|
||||
"actual": actual,
|
||||
}
|
||||
)
|
||||
if live_coverage.get("skips") != [{"step": adapter_server.METADATA_WRITE_PLAN_METHOD, "status": "skipped_no_saved_state_target"}]:
|
||||
issues.append({"code": "selector_chain_coverage_skips_mismatch", "actual": live_coverage.get("skips")})
|
||||
strict_skip_issue = selector_chain_smoke.live_write_plan_composition_required_issue(live_coverage)
|
||||
if not isinstance(strict_skip_issue, dict) or strict_skip_issue.get("code") != "live_write_plan_composition_required":
|
||||
issues.append({"code": "selector_chain_strict_skip_not_blocked", "actual": strict_skip_issue})
|
||||
composed_coverage = selector_chain_smoke.live_coverage_from_steps(
|
||||
[
|
||||
{
|
||||
"name": "metadata.resolve_overrides",
|
||||
"status": "ok",
|
||||
"write_plan_evidence": True,
|
||||
"next_method": adapter_server.SAVED_STATE_MODULES_SEARCH_METHOD,
|
||||
},
|
||||
{
|
||||
"name": adapter_server.SAVED_STATE_MODULES_SEARCH_METHOD,
|
||||
"status": "ok",
|
||||
"modules": 1,
|
||||
"write_plan_target": True,
|
||||
},
|
||||
{
|
||||
"name": adapter_server.METADATA_WRITE_PLAN_METHOD,
|
||||
"status": "planned",
|
||||
"allowed": True,
|
||||
"from_write_plan_target": True,
|
||||
},
|
||||
]
|
||||
)
|
||||
strict_composed_issue = selector_chain_smoke.live_write_plan_composition_required_issue(composed_coverage)
|
||||
if strict_composed_issue is not None:
|
||||
issues.append({"code": "selector_chain_strict_composed_blocked", "actual": strict_composed_issue})
|
||||
selector_chain_report = selector_chain_smoke.build_report()
|
||||
if selector_chain_report.get("passed") is not True:
|
||||
issues.append({"code": "selector_chain_working_state_smoke_failed", "report": selector_chain_report})
|
||||
for chain in selector_chain_report.get("chains") or []:
|
||||
if not isinstance(chain, dict):
|
||||
continue
|
||||
for step in chain.get("steps") or []:
|
||||
if not isinstance(step, dict) or step.get("method") not in selector_chain_smoke.WORKING_STATE_METHODS:
|
||||
continue
|
||||
payload = step.get("payload") if isinstance(step.get("payload"), dict) else {}
|
||||
if payload.get("source_state") != "working" and payload.get("state") != "working":
|
||||
issues.append(
|
||||
{
|
||||
"code": "selector_chain_working_state_missing",
|
||||
"chain": chain.get("name"),
|
||||
"method": step.get("method"),
|
||||
}
|
||||
)
|
||||
production_selector_files = iter_production_selector_contract_files()
|
||||
for path in production_selector_files:
|
||||
if not path.exists():
|
||||
continue
|
||||
text = path.read_text(encoding="utf-8", errors="replace")
|
||||
for value in FORBIDDEN_CONCRETE_SELECTOR_VALUES:
|
||||
if value in text:
|
||||
issues.append(
|
||||
{
|
||||
"code": "production_concrete_selector_value_present",
|
||||
"path": str(path.relative_to(ROOT)),
|
||||
"value": value,
|
||||
}
|
||||
)
|
||||
public_methods = {
|
||||
str(row.get("name") or ""): row
|
||||
for row in adapter_server.call_method("help.methods", {}).get("methods", [])
|
||||
if isinstance(row, dict) and row.get("name")
|
||||
}
|
||||
help_result = adapter_server.call_method("help.methods", {})
|
||||
if help_result.get("contract_version") != adapter_contract_version:
|
||||
issues.append(
|
||||
{
|
||||
"code": "adapter_help_contract_version_missing",
|
||||
"expected": adapter_contract_version,
|
||||
"actual": help_result.get("contract_version"),
|
||||
}
|
||||
)
|
||||
for method in sorted(getattr(adapter_server, "OBJECT_SELECTOR_ALIAS_METHODS", set())):
|
||||
capabilities = (public_methods.get(method) or {}).get("selector_capabilities")
|
||||
if not isinstance(capabilities, dict) or not capabilities.get("accepts_ref") or not capabilities.get("accepts_object_aliases"):
|
||||
issues.append(
|
||||
{
|
||||
"code": "adapter_help_selector_capabilities_missing",
|
||||
"method": method,
|
||||
"selector_capabilities": capabilities,
|
||||
}
|
||||
)
|
||||
description = str((public_methods.get(method) or {}).get("description") or "")
|
||||
for term in selector_guidance_terms:
|
||||
if term not in description:
|
||||
issues.append(
|
||||
{
|
||||
"code": "adapter_help_selector_guidance_missing",
|
||||
"method": method,
|
||||
"term": term,
|
||||
"description": description,
|
||||
}
|
||||
)
|
||||
for argument in getattr(adapter_server, "OBJECT_SELECTOR_ARGUMENTS", ["ref"]):
|
||||
selector_check = adapter_server.validate_adapter_job_payload(method, {"base_id": "contract_base", argument: 123})
|
||||
if (
|
||||
not isinstance(selector_check, dict)
|
||||
or selector_check.get("status") != "invalid_argument"
|
||||
or selector_check.get("argument") != argument
|
||||
):
|
||||
issues.append(
|
||||
{
|
||||
"code": "adapter_selector_argument_type_not_validated",
|
||||
"method": method,
|
||||
"argument": argument,
|
||||
"result": selector_check,
|
||||
}
|
||||
)
|
||||
|
||||
onec_request = find_tool("onec_request")
|
||||
if not onec_request:
|
||||
issues.append({"code": "mcp_onec_request_missing"})
|
||||
else:
|
||||
schema = onec_request.get("inputSchema") or {}
|
||||
tool_description = str(onec_request.get("description") or "")
|
||||
for term in selector_guidance_terms:
|
||||
if term not in tool_description:
|
||||
issues.append(
|
||||
{
|
||||
"code": "mcp_tool_selector_guidance_missing",
|
||||
"term": term,
|
||||
"description": tool_description,
|
||||
}
|
||||
)
|
||||
examples_json = json.dumps(schema.get("examples") or [], ensure_ascii=False)
|
||||
if "<metadata-kind>.<metadata-object-name>" not in examples_json:
|
||||
issues.append({"code": "mcp_examples_public_ref_placeholder_missing"})
|
||||
if schema.get("required") != ["method"]:
|
||||
issues.append({"code": "mcp_onec_request_required_not_generic", "required": schema.get("required")})
|
||||
if "oneOf" in schema:
|
||||
issues.append({"code": "mcp_onec_request_enumerates_adapter_methods"})
|
||||
payload_schema = (schema.get("properties") or {}).get("payload") or {}
|
||||
if payload_schema.get("additionalProperties") is not True:
|
||||
issues.append({"code": "mcp_payload_not_open", "payload_schema": payload_schema})
|
||||
payload_description = str(payload_schema.get("description") or "")
|
||||
for term in selector_guidance_terms:
|
||||
if term not in payload_description:
|
||||
issues.append(
|
||||
{
|
||||
"code": "mcp_payload_selector_guidance_missing",
|
||||
"term": term,
|
||||
"description": payload_description,
|
||||
}
|
||||
)
|
||||
|
||||
calls: list[tuple[str, str, Any]] = []
|
||||
|
||||
def fake_http_json(method: str, path: str, body: Any | None = None, *, timeout: float | None = None) -> dict[str, Any]:
|
||||
calls.append((method, path, body))
|
||||
return {"status": "ok", "method": body.get("method") if isinstance(body, dict) else "health"}
|
||||
|
||||
original_http_json = adapter_mcp.http_json
|
||||
adapter_mcp.http_json = fake_http_json
|
||||
try:
|
||||
calls.clear()
|
||||
missing_base_id_result = adapter_mcp.run_or_enqueue_adapter_method("metadata.write.plan", {})
|
||||
if calls or not isinstance(missing_base_id_result, dict) or missing_base_id_result.get("schema") != "adapter_1c_mcp_policy.v1" or missing_base_id_result.get("reason") != "base_id_required":
|
||||
issues.append(
|
||||
{
|
||||
"code": "mcp_policy_missing_base_id_not_blocked",
|
||||
"result": missing_base_id_result,
|
||||
"calls": calls,
|
||||
}
|
||||
)
|
||||
|
||||
calls.clear()
|
||||
diagnostic_fallback_result = adapter_mcp.run_or_enqueue_adapter_method("storage.files.list", {"base_id": "contract_base"})
|
||||
if calls or not isinstance(diagnostic_fallback_result, dict) or diagnostic_fallback_result.get("schema") != "adapter_1c_mcp_policy.v1" or diagnostic_fallback_result.get("reason") != "diagnostic_method":
|
||||
issues.append(
|
||||
{
|
||||
"code": "mcp_policy_diagnostic_fallback_not_blocked",
|
||||
"result": diagnostic_fallback_result,
|
||||
"calls": calls,
|
||||
}
|
||||
)
|
||||
|
||||
calls.clear()
|
||||
adapter_mcp.run_or_enqueue_adapter_method("storage.files.list", {"base_id": "contract_base", "diagnostic": True})
|
||||
if not calls:
|
||||
issues.append({"code": "mcp_policy_explicit_diagnostic_not_forwarded"})
|
||||
|
||||
for method in methods:
|
||||
calls.clear()
|
||||
adapter_mcp.call_adapter_method(method, {"base_id": "contract_base"})
|
||||
if method == "health":
|
||||
expected = ("GET", "/health?base_id=contract_base")
|
||||
if not calls or calls[0][0:2] != expected:
|
||||
issues.append({"code": "mcp_health_not_get_health", "method": method, "calls": calls})
|
||||
continue
|
||||
if not calls:
|
||||
issues.append({"code": "mcp_method_not_forwarded", "method": method})
|
||||
continue
|
||||
http_method, path, body = calls[0]
|
||||
if http_method != "POST" or path != "/rpc":
|
||||
issues.append({"code": "mcp_method_not_rpc", "method": method, "calls": calls})
|
||||
continue
|
||||
if not isinstance(body, dict) or body.get("method") != method:
|
||||
issues.append({"code": "mcp_rpc_body_method_mismatch", "method": method, "body": body})
|
||||
finally:
|
||||
adapter_mcp.http_json = original_http_json
|
||||
|
||||
return {
|
||||
"schema": "onec_mcp_adapter_contract_check.v1",
|
||||
"passed": not issues,
|
||||
"adapter_methods": len(methods),
|
||||
"production_selector_files": len(production_selector_files),
|
||||
"mcp_tools": [tool.get("name") for tool in adapter_mcp.TOOLS],
|
||||
"checks": contract_checks(issues, onec_request_present=onec_request is not None),
|
||||
"issues": issues,
|
||||
}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Check that adapter-1c MCP stays a generic proxy for REST adapter methods.")
|
||||
parser.add_argument("--json", action="store_true", help="Print JSON report.")
|
||||
args = parser.parse_args()
|
||||
|
||||
report = check_contract()
|
||||
if args.json:
|
||||
print(json.dumps(report, ensure_ascii=False, indent=2))
|
||||
elif report["passed"]:
|
||||
print(f"OK: MCP generic proxy contract passed for {report['adapter_methods']} adapter methods.")
|
||||
else:
|
||||
print(json.dumps(report, ensure_ascii=False, indent=2), file=sys.stderr)
|
||||
return 0 if report["passed"] else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,129 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT))
|
||||
sys.path.insert(0, str(ROOT / "plugins" / "1c"))
|
||||
sys.path.insert(0, str(ROOT / "plugins" / "1c" / "connector"))
|
||||
|
||||
import adapter_1c_server as adapter_server # noqa: E402
|
||||
|
||||
|
||||
def require(condition: bool, message: str, failures: list[str]) -> None:
|
||||
if not condition:
|
||||
failures.append(message)
|
||||
|
||||
|
||||
def patch_adapter_reads() -> None:
|
||||
adapter_server.read_storage_file_bytes = lambda base_id, table, file_name, timeout_seconds=30: (b"fake", {"database": "fake"}, None)
|
||||
adapter_server.payload_text_from_bytes = lambda data: {
|
||||
"status": "ok",
|
||||
"text": "Процедура Проверка()\nКонецПроцедуры",
|
||||
}
|
||||
adapter_server.extension_module_owner_payload = lambda base_id, module_id, table, timeout_seconds=30: None
|
||||
adapter_server.cached_module_owner_payload = lambda base_id, module_id: None
|
||||
|
||||
|
||||
def read_origin(method: str, module_ref: str) -> dict[str, Any]:
|
||||
result = adapter_server.call_method(
|
||||
method,
|
||||
{
|
||||
"base_id": "upo_test",
|
||||
"module_ref": module_ref,
|
||||
"include_text": True,
|
||||
"max_chars": 200,
|
||||
},
|
||||
)
|
||||
origin = result.get("origin") if isinstance(result.get("origin"), dict) else {}
|
||||
return {"result": result, "origin": origin}
|
||||
|
||||
|
||||
def check_code_search_origin(failures: list[str]) -> None:
|
||||
adapter_server.search_modules = lambda payload: {
|
||||
"status": "ok",
|
||||
"source": {"kind": "live_metadata"},
|
||||
"matches": [
|
||||
{
|
||||
"snippet": {"text": "Процедура Проверка()", "offset": 0},
|
||||
"owner": {"status": "unresolved", "kind": None, "name": None},
|
||||
"origin": {
|
||||
"source": "cas_reference",
|
||||
"status": "owner_unresolved",
|
||||
"write_surface": "requires_owner_resolution",
|
||||
},
|
||||
"module": {"name": "Модуль БСЛ"},
|
||||
"read_selector": {"base_id": "upo_test", "module_ref": "ConfigCAS:object-module"},
|
||||
}
|
||||
],
|
||||
"counts": {"matches": 1, "complete": True, "scan_limit_hit": False},
|
||||
"diagnostics": {},
|
||||
}
|
||||
result = adapter_server.call_method("code.search", {"base_id": "upo_test", "query": "Проверка"})
|
||||
items = result.get("items") if isinstance(result.get("items"), list) else []
|
||||
origin = items[0].get("origin") if items and isinstance(items[0], dict) and isinstance(items[0].get("origin"), dict) else {}
|
||||
require(result.get("schema") == "onec_code_search.v1", "code.search must return code search schema", failures)
|
||||
require(bool(items), "code.search must return patched item", failures)
|
||||
require(origin.get("source") == "cas_reference", "code.search item must preserve modules.search origin", failures)
|
||||
require(origin.get("write_surface") == "requires_owner_resolution", "code.search item origin must keep write_surface", failures)
|
||||
|
||||
|
||||
def run_checks() -> dict[str, Any]:
|
||||
patch_adapter_reads()
|
||||
failures: list[str] = []
|
||||
|
||||
config = read_origin("modules.read", "Config:object-module")
|
||||
require(config["result"].get("status") == "ok", "Config module_ref must read in patched contract", failures)
|
||||
require(config["origin"].get("source") == "configuration", "Config module_ref must expose configuration origin", failures)
|
||||
require(config["origin"].get("write_surface") == "base_saved_state", "Config origin must point writes to base saved-state", failures)
|
||||
|
||||
save = read_origin("modules.read", "ConfigSave:object-module")
|
||||
require(save["result"].get("status") == "ok", "ConfigSave module_ref must read in patched contract", failures)
|
||||
require(save["origin"].get("source") == "saved_state", "ConfigSave module_ref must expose saved_state origin", failures)
|
||||
require(save["origin"].get("write_surface") == "base_saved_state", "ConfigSave origin must point writes to base saved-state", failures)
|
||||
|
||||
cas = read_origin("modules.read", "ConfigCAS:object-module")
|
||||
require(cas["result"].get("status") == "ok", "ConfigCAS module_ref must read in patched contract", failures)
|
||||
require(cas["origin"].get("source") == "cas_reference", "ConfigCAS fallback must not pretend extension/base owner", failures)
|
||||
require(cas["origin"].get("status") == "owner_unresolved", "ConfigCAS fallback must require owner resolution", failures)
|
||||
require(cas["origin"].get("write_surface") == "requires_owner_resolution", "ConfigCAS fallback must block direct write routing", failures)
|
||||
|
||||
code_save = read_origin("code.read", "ConfigSave:object-module")
|
||||
require(code_save["result"].get("schema") == "onec_code_read.v1", "code.read must wrap modules.read as onec_code_read", failures)
|
||||
require(code_save["origin"].get("source") == "saved_state", "code.read must preserve modules.read origin", failures)
|
||||
require(code_save["origin"].get("write_surface") == "base_saved_state", "code.read origin must keep write_surface", failures)
|
||||
check_code_search_origin(failures)
|
||||
|
||||
return {
|
||||
"schema": "onec_module_origin_contract_check.v1",
|
||||
"status": "ok" if not failures else "failed",
|
||||
"failures": failures,
|
||||
"checks": {
|
||||
"config_origin": "Config module_ref exposes configuration origin",
|
||||
"configsave_origin": "ConfigSave module_ref exposes saved_state origin",
|
||||
"configcas_owner_required": "ConfigCAS module_ref requires owner resolution",
|
||||
"code_read_preserves_origin": "code.read preserves origin evidence from modules.read",
|
||||
"code_search_preserves_origin": "code.search preserves origin evidence from modules.search",
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Check modules.read origin/provenance contract invariants.")
|
||||
parser.add_argument("--print", action="store_true")
|
||||
args = parser.parse_args()
|
||||
report = run_checks()
|
||||
if args.print or report["status"] != "ok":
|
||||
print(json.dumps(report, ensure_ascii=False, indent=2))
|
||||
else:
|
||||
print("1C module origin contract status: ok")
|
||||
return 0 if report["status"] == "ok" else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,57 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
def read_json(path: Path) -> dict[str, Any]:
|
||||
return json.loads(path.read_text(encoding="utf-8"))
|
||||
|
||||
|
||||
def check_next_action(plan: dict[str, Any], action_payload: dict[str, Any]) -> dict[str, Any]:
|
||||
failures: list[str] = []
|
||||
if action_payload.get("schema") != "codex_1c_moxel_next_action.v1":
|
||||
failures.append("unexpected next action schema")
|
||||
action = action_payload.get("action") if isinstance(action_payload.get("action"), dict) else None
|
||||
experiments = [item for item in plan.get("experiments") or [] if isinstance(item, dict)]
|
||||
if not action:
|
||||
if experiments:
|
||||
failures.append("next action is empty but experiments are available")
|
||||
else:
|
||||
action_id = action.get("id")
|
||||
matching = [item for item in experiments if item.get("id") == action_id]
|
||||
if not matching:
|
||||
failures.append(f"next action id is not present in plan: {action_id}")
|
||||
command = str(action.get("capture_command_with_pipeline") or "")
|
||||
if "--run-pipeline-after" not in command:
|
||||
failures.append("capture_command_with_pipeline must include --run-pipeline-after")
|
||||
for field in ("manual_action", "expected_signal", "target"):
|
||||
if not action.get(field):
|
||||
failures.append(f"next action must include {field}")
|
||||
return {
|
||||
"schema": "codex_1c_moxel_next_action_check.v1",
|
||||
"status": "ok" if not failures else "failed",
|
||||
"failures": failures,
|
||||
"counts": {"experiments": len(experiments), "has_action": action is not None},
|
||||
}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Validate the machine-readable 1C MOXCEL next action artifact.")
|
||||
parser.add_argument("--plan", default="reports/1c-template-baselines/moxel-next-experiments.json")
|
||||
parser.add_argument("--action", default="reports/1c-template-baselines/moxel-next-action.json")
|
||||
parser.add_argument("--output", default="reports/1c-template-baselines/moxel-next-action-check.json")
|
||||
args = parser.parse_args()
|
||||
|
||||
report = check_next_action(read_json(Path(args.plan)), read_json(Path(args.action)))
|
||||
output_path = Path(args.output)
|
||||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
output_path.write_text(json.dumps(report, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
print(json.dumps(report, ensure_ascii=False, indent=2))
|
||||
return 0 if report["status"] == "ok" else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,85 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
def read_json(path: Path) -> dict[str, Any]:
|
||||
return json.loads(path.read_text(encoding="utf-8"))
|
||||
|
||||
|
||||
def fail(message: str, failures: list[str]) -> None:
|
||||
failures.append(message)
|
||||
|
||||
|
||||
def check_registry(registry: dict[str, Any]) -> dict[str, Any]:
|
||||
failures: list[str] = []
|
||||
if registry.get("schema") != "codex_1c_moxel_schema_registry.v1":
|
||||
fail("unexpected schema", failures)
|
||||
rules = [rule for rule in registry.get("rules") or [] if isinstance(rule, dict)]
|
||||
if not rules:
|
||||
fail("registry must contain rules", failures)
|
||||
verified = [rule for rule in rules if rule.get("read_status") == "verified_read"]
|
||||
if not verified:
|
||||
fail("registry must contain at least one verified_read rule", failures)
|
||||
write_enabled = [rule for rule in rules if rule.get("write_status") == "verified_roundtrip"]
|
||||
if write_enabled:
|
||||
fail("write rules must stay disabled until explicit round-trip evidence is implemented", failures)
|
||||
counts = registry.get("counts") if isinstance(registry.get("counts"), dict) else {}
|
||||
expected_counts = {
|
||||
"rules": len(rules),
|
||||
"verified_read": len(verified),
|
||||
"candidate_read": sum(1 for rule in rules if rule.get("read_status") == "candidate_read"),
|
||||
"write_enabled": len(write_enabled),
|
||||
}
|
||||
for key, expected in expected_counts.items():
|
||||
if counts.get(key) != expected:
|
||||
fail(f"counts.{key} must be {expected}, got {counts.get(key)}", failures)
|
||||
|
||||
inline = [rule for rule in rules if rule.get("target") == "moxel.inline_text_cell.column"]
|
||||
if not inline:
|
||||
fail("missing moxel.inline_text_cell.column rule", failures)
|
||||
else:
|
||||
rule = inline[0]
|
||||
evidence = rule.get("evidence") if isinstance(rule.get("evidence"), dict) else {}
|
||||
if rule.get("read_status") != "verified_read":
|
||||
fail("inline column rule must be verified_read", failures)
|
||||
if evidence.get("ok") != evidence.get("total") or not isinstance(evidence.get("total"), int) or evidence.get("total") <= 0:
|
||||
fail("inline column rule evidence must be complete", failures)
|
||||
|
||||
for rule in rules:
|
||||
if not rule.get("target"):
|
||||
fail(f"rule {rule.get('id')} has no target", failures)
|
||||
if rule.get("read_status") not in {"verified_read", "candidate_read", "needs_more_evidence"}:
|
||||
fail(f"rule {rule.get('id')} has unsupported read_status", failures)
|
||||
if rule.get("write_status") not in {"blocked_until_roundtrip", "blocked_until_verified_read", "verified_roundtrip"}:
|
||||
fail(f"rule {rule.get('id')} has unsupported write_status", failures)
|
||||
|
||||
return {
|
||||
"schema": "codex_1c_moxel_schema_registry_check.v1",
|
||||
"status": "ok" if not failures else "failed",
|
||||
"failures": failures,
|
||||
"counts": {
|
||||
**expected_counts,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Validate the 1C MOXCEL schema registry safety contract.")
|
||||
parser.add_argument("--registry", default="plugins/1c/metadata/moxel-schema-registry.json")
|
||||
parser.add_argument("--output", default="reports/1c-template-baselines/moxel-schema-registry-check.json")
|
||||
args = parser.parse_args()
|
||||
|
||||
report = check_registry(read_json(Path(args.registry)))
|
||||
output_path = Path(args.output)
|
||||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
output_path.write_text(json.dumps(report, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
print(json.dumps(report, ensure_ascii=False, indent=2))
|
||||
return 0 if report["status"] == "ok" else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,60 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
DEFAULT_ROOT = ROOT / "plugins" / "1c" / "rag" / "official-docs"
|
||||
PRIVATE_DIRS = ("raw", "normalized")
|
||||
ALLOWED_PRIVATE_FILES = {".gitkeep"}
|
||||
|
||||
|
||||
def check_private_artifacts(root: Path) -> dict[str, Any]:
|
||||
findings: list[dict[str, Any]] = []
|
||||
counts = {"private_files": 0, "unexpected_private_files": 0}
|
||||
for dirname in PRIVATE_DIRS:
|
||||
directory = root / dirname
|
||||
if not directory.exists():
|
||||
findings.append({"severity": "warning", "code": "missing_private_dir", "path": str(directory)})
|
||||
continue
|
||||
for path in sorted(item for item in directory.rglob("*") if item.is_file()):
|
||||
if path.name in ALLOWED_PRIVATE_FILES:
|
||||
continue
|
||||
counts["private_files"] += 1
|
||||
counts["unexpected_private_files"] += 1
|
||||
findings.append(
|
||||
{
|
||||
"severity": "info",
|
||||
"code": "private_artifact_present",
|
||||
"message": "Private official documentation artifact exists locally; it must remain ignored and uncommitted.",
|
||||
"path": str(path),
|
||||
}
|
||||
)
|
||||
return {
|
||||
"schema": "onec_official_docs_private_artifact_check.v1",
|
||||
"root": str(root),
|
||||
"passed": not any(item["severity"] == "error" for item in findings),
|
||||
"counts": counts,
|
||||
"findings": findings,
|
||||
}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Check local private 1C official-doc artifacts.")
|
||||
parser.add_argument("--root", type=Path, default=DEFAULT_ROOT)
|
||||
parser.add_argument("--output", type=Path)
|
||||
args = parser.parse_args()
|
||||
|
||||
result = check_private_artifacts(args.root)
|
||||
if args.output:
|
||||
args.output.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.output.write_text(json.dumps(result, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
print(json.dumps({"passed": result["passed"], "counts": result["counts"], "output": str(args.output) if args.output else None}, ensure_ascii=False))
|
||||
return 0 if result["passed"] else 2
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,188 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
DEFAULT_NORMALIZED_MANIFEST = ROOT / "plugins" / "1c" / "rag" / "official-docs" / "normalized" / "manifest.json"
|
||||
DEFAULT_RAW_MANIFEST = ROOT / "plugins" / "1c" / "rag" / "official-docs" / "raw" / "manifest.json"
|
||||
DEFAULT_RAG_SOURCE_DIR = ROOT / "plugins" / "1c" / "rag" / "sources" / "official" / "its"
|
||||
DEFAULT_CORPUS = ROOT / "plugins" / "1c" / "datasets" / "prepared" / "rag_corpus.jsonl"
|
||||
|
||||
FORBIDDEN_TEXT = (
|
||||
"Мы используем файлы cookie",
|
||||
"Продолжая находиться на сайте",
|
||||
"Результаты поиска",
|
||||
"Купить кассу",
|
||||
"Календарь бухгалтера",
|
||||
"Последние результаты поиска",
|
||||
)
|
||||
|
||||
NAVIGATION_CLUES = (
|
||||
"Руководство разработчика - Руководство администратора",
|
||||
"Глоссарий разработчика - 1 - 1CEClientSetupMake.exe",
|
||||
)
|
||||
|
||||
|
||||
def load_json(path: Path) -> dict[str, Any]:
|
||||
if not path.exists():
|
||||
return {}
|
||||
try:
|
||||
data = json.loads(path.read_text(encoding="utf-8-sig"))
|
||||
except json.JSONDecodeError:
|
||||
return {}
|
||||
return data if isinstance(data, dict) else {}
|
||||
|
||||
|
||||
def scan_text_file(path: Path, needles: tuple[str, ...]) -> list[dict[str, Any]]:
|
||||
try:
|
||||
text = path.read_text(encoding="utf-8-sig", errors="ignore")
|
||||
except OSError as exc:
|
||||
return [{"severity": "error", "code": "read_failed", "path": str(path), "message": str(exc)}]
|
||||
findings = []
|
||||
for needle in needles:
|
||||
if needle in text:
|
||||
findings.append({"severity": "error", "code": "forbidden_text", "path": str(path), "text": needle})
|
||||
return findings
|
||||
|
||||
|
||||
def scan_jsonl(path: Path, needles: tuple[str, ...]) -> list[dict[str, Any]]:
|
||||
findings: list[dict[str, Any]] = []
|
||||
if not path.exists():
|
||||
findings.append({"severity": "warning", "code": "missing_corpus", "path": str(path)})
|
||||
return findings
|
||||
for line_no, line in enumerate(path.read_text(encoding="utf-8-sig", errors="ignore").splitlines(), start=1):
|
||||
if not line.strip():
|
||||
continue
|
||||
try:
|
||||
item = json.loads(line)
|
||||
except json.JSONDecodeError:
|
||||
findings.append({"severity": "error", "code": "bad_jsonl", "path": str(path), "line": line_no})
|
||||
continue
|
||||
content = str(item.get("content") or "")
|
||||
for needle in needles:
|
||||
if needle in content:
|
||||
findings.append(
|
||||
{
|
||||
"severity": "error",
|
||||
"code": "forbidden_text_in_corpus",
|
||||
"path": str(path),
|
||||
"line": line_no,
|
||||
"chunk_id": item.get("id"),
|
||||
"text": needle,
|
||||
}
|
||||
)
|
||||
return findings
|
||||
|
||||
|
||||
def raw_url_counts(raw_manifest_path: Path) -> dict[str, int]:
|
||||
manifest = load_json(raw_manifest_path)
|
||||
counts = {"raw_pages": 0, "content_src_pages": 0, "hdoc_pages": 0, "root_pages": 0}
|
||||
for page in manifest.get("pages") or []:
|
||||
counts["raw_pages"] += 1
|
||||
url = str(page.get("url") or "")
|
||||
if "/db/content/" in url and "/src/" in url:
|
||||
counts["content_src_pages"] += 1
|
||||
elif "/content/" in url and url.endswith("/hdoc"):
|
||||
counts["hdoc_pages"] += 1
|
||||
else:
|
||||
counts["root_pages"] += 1
|
||||
return counts
|
||||
|
||||
|
||||
def check_quality(manifest_path: Path, raw_manifest_path: Path, rag_source_dir: Path, corpus_path: Path) -> dict[str, Any]:
|
||||
findings: list[dict[str, Any]] = []
|
||||
manifest = load_json(manifest_path)
|
||||
raw_counts = raw_url_counts(raw_manifest_path)
|
||||
page_count = int(manifest.get("page_count") or 0)
|
||||
skipped_count = int(manifest.get("skipped_count") or 0)
|
||||
discovered_src_count = int(manifest.get("discovered_src_record_count") or 0)
|
||||
media_page_count = 0
|
||||
media_image_count = 0
|
||||
table_count = 0
|
||||
for page in manifest.get("pages") or []:
|
||||
media = page.get("media") or {}
|
||||
images = media.get("images") or []
|
||||
if images:
|
||||
media_page_count += 1
|
||||
media_image_count += len(images)
|
||||
table_count += int(media.get("table_count") or 0)
|
||||
|
||||
if not manifest:
|
||||
findings.append({"severity": "warning", "code": "missing_normalized_manifest", "path": str(manifest_path)})
|
||||
elif page_count == 0:
|
||||
findings.append(
|
||||
{
|
||||
"severity": "warning",
|
||||
"code": "no_official_content_pages",
|
||||
"message": "No official 1C:ITS pages passed normalization quality gates. Refresh cookie and fetch with --no-resume.",
|
||||
"skipped_count": skipped_count,
|
||||
}
|
||||
)
|
||||
if raw_counts["raw_pages"] and not raw_counts["content_src_pages"] and not discovered_src_count:
|
||||
findings.append(
|
||||
{
|
||||
"severity": "warning",
|
||||
"code": "no_raw_content_src_pages",
|
||||
"message": "Raw fetch has pages, but no /db/content/.../src/... pages. Fetch probably stopped at hdoc shells/navigation.",
|
||||
"raw_counts": raw_counts,
|
||||
}
|
||||
)
|
||||
|
||||
if rag_source_dir.exists():
|
||||
for path in sorted(rag_source_dir.glob("*.md")):
|
||||
findings.extend(scan_text_file(path, FORBIDDEN_TEXT + NAVIGATION_CLUES))
|
||||
else:
|
||||
findings.append({"severity": "warning", "code": "missing_rag_source_dir", "path": str(rag_source_dir)})
|
||||
|
||||
findings.extend(scan_jsonl(corpus_path, FORBIDDEN_TEXT + NAVIGATION_CLUES))
|
||||
|
||||
errors = [item for item in findings if item.get("severity") == "error"]
|
||||
warnings = [item for item in findings if item.get("severity") == "warning"]
|
||||
return {
|
||||
"schema": "onec_official_docs_quality_check.v1",
|
||||
"passed": not errors,
|
||||
"counts": {
|
||||
**raw_counts,
|
||||
"discovered_src_record_count": discovered_src_count,
|
||||
"media_pages": media_page_count,
|
||||
"media_images": media_image_count,
|
||||
"tables": table_count,
|
||||
"normalized_pages": page_count,
|
||||
"skipped_pages": skipped_count,
|
||||
"errors": len(errors),
|
||||
"warnings": len(warnings),
|
||||
"findings": len(findings),
|
||||
},
|
||||
"manifest": str(manifest_path),
|
||||
"raw_manifest": str(raw_manifest_path),
|
||||
"rag_source_dir": str(rag_source_dir),
|
||||
"corpus": str(corpus_path),
|
||||
"findings": findings,
|
||||
}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Check official 1C:ITS normalized docs and RAG corpus quality.")
|
||||
parser.add_argument("--manifest", type=Path, default=DEFAULT_NORMALIZED_MANIFEST)
|
||||
parser.add_argument("--raw-manifest", type=Path, default=DEFAULT_RAW_MANIFEST)
|
||||
parser.add_argument("--rag-source-dir", type=Path, default=DEFAULT_RAG_SOURCE_DIR)
|
||||
parser.add_argument("--corpus", type=Path, default=DEFAULT_CORPUS)
|
||||
parser.add_argument("--output", type=Path)
|
||||
parser.add_argument("--print", action="store_true", dest="print_full")
|
||||
args = parser.parse_args()
|
||||
|
||||
result = check_quality(args.manifest, args.raw_manifest, args.rag_source_dir, args.corpus)
|
||||
if args.output:
|
||||
args.output.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.output.write_text(json.dumps(result, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
payload = result if args.print_full else {"passed": result["passed"], "counts": result["counts"], "output": str(args.output) if args.output else None}
|
||||
print(json.dumps(payload, ensure_ascii=False, indent=2))
|
||||
return 0 if result["passed"] else 2
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,185 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Validate a 1C patch review bundle directory and optional zip archive."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import zipfile
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
REQUIRED_FILES = {"manifest.json", "preflight.json", "preflight.md", "diff.json", "README.md"}
|
||||
|
||||
|
||||
def load_json(path: Path) -> dict[str, Any]:
|
||||
return json.loads(path.read_text(encoding="utf-8-sig"))
|
||||
|
||||
|
||||
def write_json(path: Path, data: dict[str, Any]) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(json.dumps(data, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
|
||||
|
||||
def sha256_file(path: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as stream:
|
||||
for chunk in iter(lambda: stream.read(1024 * 1024), b""):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def issue(severity: str, code: str, message: str, *, path: Path | str | None = None, detail: dict[str, Any] | None = None) -> dict[str, Any]:
|
||||
result: dict[str, Any] = {"severity": severity, "code": code, "message": message}
|
||||
if path is not None:
|
||||
result["path"] = str(path)
|
||||
if detail:
|
||||
result["detail"] = detail
|
||||
return result
|
||||
|
||||
|
||||
def safe_relative(relative_path: str) -> Path:
|
||||
path = Path(relative_path.replace("\\", "/"))
|
||||
if path.is_absolute() or ".." in path.parts or not str(path):
|
||||
raise ValueError(relative_path)
|
||||
return path
|
||||
|
||||
|
||||
def all_files(root: Path) -> set[str]:
|
||||
if not root.exists():
|
||||
return set()
|
||||
return {str(path.relative_to(root)).replace("\\", "/") for path in root.rglob("*") if path.is_file()}
|
||||
|
||||
|
||||
def expected_working_hash(record: dict[str, Any]) -> str | None:
|
||||
sha = record.get("sha256") or {}
|
||||
if isinstance(sha, dict):
|
||||
return sha.get("working")
|
||||
return None
|
||||
|
||||
|
||||
def check_bundle(bundle_dir: Path, zip_path: Path | None = None) -> dict[str, Any]:
|
||||
findings: list[dict[str, Any]] = []
|
||||
file_checks: list[dict[str, Any]] = []
|
||||
|
||||
if not bundle_dir.exists() or not bundle_dir.is_dir():
|
||||
findings.append(issue("error", "missing_bundle_dir", "Bundle directory is missing.", path=bundle_dir))
|
||||
return build_result(bundle_dir, zip_path, findings, file_checks, None)
|
||||
|
||||
present = all_files(bundle_dir)
|
||||
for required in sorted(REQUIRED_FILES):
|
||||
if required not in present:
|
||||
findings.append(issue("error", "missing_bundle_file", "Required bundle file is missing.", path=bundle_dir / required))
|
||||
|
||||
manifest_path = bundle_dir / "manifest.json"
|
||||
if not manifest_path.exists():
|
||||
return build_result(bundle_dir, zip_path, findings, file_checks, None)
|
||||
manifest = load_json(manifest_path)
|
||||
if manifest.get("schema") != "onec_patch_bundle.v1":
|
||||
findings.append(issue("error", "invalid_bundle_schema", "Bundle manifest schema is not onec_patch_bundle.v1.", path=manifest_path, detail={"schema": manifest.get("schema")}))
|
||||
|
||||
preflight = load_json(bundle_dir / "preflight.json") if (bundle_dir / "preflight.json").exists() else {}
|
||||
if preflight.get("status") != "ready_for_review":
|
||||
findings.append(issue("error", "invalid_preflight_status", "Bundle preflight must be ready_for_review.", path=bundle_dir / "preflight.json", detail={"status": preflight.get("status")}))
|
||||
if not preflight.get("passed"):
|
||||
findings.append(issue("error", "preflight_not_passed", "Bundle preflight is not passed.", path=bundle_dir / "preflight.json"))
|
||||
|
||||
diff = load_json(bundle_dir / "diff.json") if (bundle_dir / "diff.json").exists() else {}
|
||||
modified = [
|
||||
str(item.get("relative_path") or "").replace("\\", "/")
|
||||
for item in diff.get("files") or []
|
||||
if item.get("status") == "modified"
|
||||
]
|
||||
manifest_relatives = [str(item.get("relative_path") or "").replace("\\", "/") for item in manifest.get("files") or []]
|
||||
if sorted(modified) != sorted(manifest_relatives):
|
||||
findings.append(issue("error", "bundle_diff_manifest_mismatch", "Modified diff files do not match bundle manifest files.", detail={"diff_modified": modified, "manifest_files": manifest_relatives}))
|
||||
|
||||
expected_bundle_files = set(REQUIRED_FILES)
|
||||
for record in manifest.get("files") or []:
|
||||
bundle_path_raw = str(record.get("bundle_path") or "")
|
||||
try:
|
||||
bundle_path = safe_relative(bundle_path_raw)
|
||||
except ValueError:
|
||||
findings.append(issue("error", "unsafe_bundle_path", "Unsafe bundle_path in manifest.", detail={"bundle_path": bundle_path_raw}))
|
||||
continue
|
||||
expected_bundle_files.add(str(bundle_path).replace("\\", "/"))
|
||||
path = bundle_dir / bundle_path
|
||||
check: dict[str, Any] = {
|
||||
"relative_path": record.get("relative_path"),
|
||||
"bundle_path": str(bundle_path).replace("\\", "/"),
|
||||
"exists": path.exists(),
|
||||
"expected_sha256": expected_working_hash(record),
|
||||
}
|
||||
if not path.exists():
|
||||
findings.append(issue("error", "missing_modified_file", "Modified bundle file is missing.", path=path))
|
||||
else:
|
||||
actual = sha256_file(path)
|
||||
check["sha256"] = actual
|
||||
expected = expected_working_hash(record)
|
||||
if expected and actual != expected:
|
||||
findings.append(issue("error", "modified_file_hash_mismatch", "Modified bundle file hash does not match working hash.", path=path, detail={"expected": expected, "actual": actual}))
|
||||
file_checks.append(check)
|
||||
|
||||
extra_files = sorted(present - expected_bundle_files)
|
||||
for relative in extra_files:
|
||||
findings.append(issue("warning", "extra_bundle_file", "Unexpected file in bundle directory.", path=bundle_dir / relative))
|
||||
|
||||
if zip_path is None:
|
||||
candidate = bundle_dir.with_suffix(".zip")
|
||||
zip_path = candidate if candidate.exists() else None
|
||||
if zip_path is None:
|
||||
findings.append(issue("warning", "missing_bundle_zip", "Bundle zip archive was not found."))
|
||||
elif not zip_path.exists():
|
||||
findings.append(issue("error", "missing_bundle_zip", "Bundle zip archive path does not exist.", path=zip_path))
|
||||
else:
|
||||
try:
|
||||
with zipfile.ZipFile(zip_path, "r") as archive:
|
||||
bad_member = archive.testzip()
|
||||
if bad_member:
|
||||
findings.append(issue("error", "invalid_bundle_zip_member", "Zip archive contains a corrupt member.", path=zip_path, detail={"member": bad_member}))
|
||||
zip_files = {name.replace("\\", "/") for name in archive.namelist() if not name.endswith("/")}
|
||||
if zip_files != present:
|
||||
findings.append(issue("error", "bundle_zip_mismatch", "Zip contents differ from bundle directory files.", path=zip_path, detail={"missing_in_zip": sorted(present - zip_files), "extra_in_zip": sorted(zip_files - present)}))
|
||||
except zipfile.BadZipFile as exc:
|
||||
findings.append(issue("error", "invalid_bundle_zip", f"Invalid zip archive: {exc}", path=zip_path))
|
||||
|
||||
return build_result(bundle_dir, zip_path, findings, file_checks, manifest)
|
||||
|
||||
|
||||
def build_result(bundle_dir: Path, zip_path: Path | None, findings: list[dict[str, Any]], file_checks: list[dict[str, Any]], manifest: dict[str, Any] | None) -> dict[str, Any]:
|
||||
errors = [row for row in findings if row.get("severity") == "error"]
|
||||
warnings = [row for row in findings if row.get("severity") == "warning"]
|
||||
return {
|
||||
"schema": "onec_patch_bundle_check.v1",
|
||||
"bundle_dir": str(bundle_dir),
|
||||
"zip_path": str(zip_path) if zip_path else None,
|
||||
"bundle_schema": (manifest or {}).get("schema"),
|
||||
"passed": not errors,
|
||||
"findings": findings,
|
||||
"file_checks": file_checks,
|
||||
"counts": {
|
||||
"files": len(file_checks),
|
||||
"errors": len(errors),
|
||||
"warnings": len(warnings),
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Validate a 1C patch review bundle.")
|
||||
parser.add_argument("--bundle-dir", type=Path, required=True)
|
||||
parser.add_argument("--zip", type=Path)
|
||||
parser.add_argument("--output", type=Path)
|
||||
args = parser.parse_args()
|
||||
|
||||
result = check_bundle(args.bundle_dir, args.zip)
|
||||
if args.output:
|
||||
write_json(args.output, result)
|
||||
print(json.dumps({"output": str(args.output) if args.output else None, "passed": result["passed"], "counts": result["counts"]}, ensure_ascii=False))
|
||||
return 0 if result["passed"] else 2
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,116 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Run the full read-only preflight for a 1C patch workspace."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from check_1c_patch_source_freshness import check_workspace_sources
|
||||
from check_1c_patch_workspace_integrity import check_workspace
|
||||
from diff_1c_patch_workspace import build_diff
|
||||
from validate_1c_patch_workspace_semantics import validate_workspace
|
||||
|
||||
|
||||
def load_json(path: Path) -> dict[str, Any]:
|
||||
return json.loads(path.read_text(encoding="utf-8-sig"))
|
||||
|
||||
|
||||
def safety_from_workspace(workspace: Path) -> dict[str, Any]:
|
||||
path = workspace / "safety.json"
|
||||
if not path.exists():
|
||||
return {
|
||||
"schema": "onec_change_proposal_safety_check.v1",
|
||||
"passed": False,
|
||||
"findings": [{"severity": "error", "code": "missing_safety_json", "message": f"Missing {path}"}],
|
||||
"counts": {"errors": 1, "warnings": 0},
|
||||
}
|
||||
return load_json(path)
|
||||
|
||||
|
||||
def status_from_checks(safety: dict[str, Any], integrity: dict[str, Any], freshness: dict[str, Any], semantic: dict[str, Any], diff: dict[str, Any]) -> str:
|
||||
if not safety.get("passed") or not integrity.get("passed") or not freshness.get("passed") or not semantic.get("passed") or not diff.get("passed"):
|
||||
return "blocked"
|
||||
modified = (diff.get("counts") or {}).get("modified", 0)
|
||||
if modified:
|
||||
return "ready_for_review"
|
||||
return "ready_for_editing"
|
||||
|
||||
|
||||
def collect_gate(name: str, data: dict[str, Any]) -> dict[str, Any]:
|
||||
return {
|
||||
"name": name,
|
||||
"schema": data.get("schema"),
|
||||
"passed": data.get("passed"),
|
||||
"counts": data.get("counts"),
|
||||
}
|
||||
|
||||
|
||||
def next_actions(status: str) -> list[str]:
|
||||
if status == "blocked":
|
||||
return [
|
||||
"Inspect failed gates and recreate the workspace if source files changed.",
|
||||
"Do not generate, package, or apply patches until all gates pass.",
|
||||
]
|
||||
if status == "ready_for_editing":
|
||||
return [
|
||||
"Edit only files under working/.",
|
||||
"Run preflight again after edits; BSL/Form.xml semantic validation is included.",
|
||||
]
|
||||
return [
|
||||
"Review the workspace diff.",
|
||||
"Run external BSL/1C validation in a disposable base before packaging or applying.",
|
||||
]
|
||||
|
||||
|
||||
def build_preflight(workspace: Path, *, max_patch_chars: int) -> dict[str, Any]:
|
||||
safety = safety_from_workspace(workspace)
|
||||
integrity = check_workspace(workspace)
|
||||
freshness = check_workspace_sources(workspace)
|
||||
semantic = validate_workspace(workspace)
|
||||
diff = build_diff(workspace, max_patch_chars=max_patch_chars)
|
||||
status = status_from_checks(safety, integrity, freshness, semantic, diff)
|
||||
return {
|
||||
"schema": "onec_patch_preflight.v1",
|
||||
"workspace": str(workspace),
|
||||
"status": status,
|
||||
"passed": status != "blocked",
|
||||
"gates": [
|
||||
collect_gate("proposal_safety", safety),
|
||||
collect_gate("workspace_integrity", integrity),
|
||||
collect_gate("source_freshness", freshness),
|
||||
collect_gate("workspace_semantic_validation", semantic),
|
||||
collect_gate("workspace_diff", diff),
|
||||
],
|
||||
"diff_summary": diff.get("counts"),
|
||||
"next_actions": next_actions(status),
|
||||
"details": {
|
||||
"safety": safety,
|
||||
"integrity": integrity,
|
||||
"freshness": freshness,
|
||||
"semantic": semantic,
|
||||
"diff": diff,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Run 1C patch workspace preflight.")
|
||||
parser.add_argument("--workspace", type=Path, required=True)
|
||||
parser.add_argument("--max-patch-chars", type=int, default=200000)
|
||||
parser.add_argument("--output", type=Path)
|
||||
args = parser.parse_args()
|
||||
|
||||
result = build_preflight(args.workspace, max_patch_chars=args.max_patch_chars)
|
||||
output = json.dumps(result, ensure_ascii=False, indent=2) + "\n"
|
||||
if args.output:
|
||||
args.output.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.output.write_text(output, encoding="utf-8")
|
||||
print(json.dumps({"output": str(args.output) if args.output else None, "passed": result["passed"], "status": result["status"], "diff": result["diff_summary"]}, ensure_ascii=False))
|
||||
return 0 if result["passed"] else 2
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,98 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Check whether source extension files still match a 1C patch workspace manifest."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
def load_json(path: Path) -> dict[str, Any]:
|
||||
return json.loads(path.read_text(encoding="utf-8-sig"))
|
||||
|
||||
|
||||
def sha256_file(path: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as stream:
|
||||
for chunk in iter(lambda: stream.read(1024 * 1024), b""):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def issue(severity: str, code: str, message: str, *, record: dict[str, Any] | None = None) -> dict[str, Any]:
|
||||
result = {"severity": severity, "code": code, "message": message}
|
||||
if record:
|
||||
result["record"] = record
|
||||
return result
|
||||
|
||||
|
||||
def check_workspace_sources(workspace: Path) -> dict[str, Any]:
|
||||
manifest_path = workspace / "manifest.json"
|
||||
findings = []
|
||||
file_checks = []
|
||||
if not manifest_path.exists():
|
||||
findings.append(issue("error", "missing_manifest", f"Workspace manifest.json is missing: {manifest_path}"))
|
||||
return result(workspace, findings, file_checks)
|
||||
manifest = load_json(manifest_path)
|
||||
for record in manifest.get("files") or []:
|
||||
source = Path(str(record.get("source_path") or ""))
|
||||
expected = record.get("sha256")
|
||||
check = {
|
||||
"relative_path": record.get("relative_path"),
|
||||
"source_path": str(source),
|
||||
"expected_sha256": expected,
|
||||
"source_exists": source.exists(),
|
||||
}
|
||||
if not source.exists():
|
||||
findings.append(issue("error", "source_missing", f"Source file is missing: {source}", record=record))
|
||||
file_checks.append(check)
|
||||
continue
|
||||
actual = sha256_file(source)
|
||||
check["source_sha256"] = actual
|
||||
check["fresh"] = actual == expected
|
||||
if expected and actual != expected:
|
||||
findings.append(issue("error", "source_hash_mismatch", "Source file changed since patch workspace creation.", record={**record, "current_sha256": actual}))
|
||||
file_checks.append(check)
|
||||
return result(workspace, findings, file_checks, manifest=manifest)
|
||||
|
||||
|
||||
def result(workspace: Path, findings: list[dict[str, Any]], file_checks: list[dict[str, Any]], manifest: dict[str, Any] | None = None) -> dict[str, Any]:
|
||||
errors = [row for row in findings if row.get("severity") == "error"]
|
||||
warnings = [row for row in findings if row.get("severity") == "warning"]
|
||||
return {
|
||||
"schema": "onec_patch_source_freshness.v1",
|
||||
"workspace": str(workspace),
|
||||
"manifest_schema": (manifest or {}).get("schema"),
|
||||
"passed": not errors,
|
||||
"findings": findings,
|
||||
"file_checks": file_checks,
|
||||
"counts": {
|
||||
"files": len(file_checks),
|
||||
"errors": len(errors),
|
||||
"warnings": len(warnings),
|
||||
"stale": len([row for row in file_checks if row.get("fresh") is False]),
|
||||
"missing": len([row for row in file_checks if not row.get("source_exists")]),
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Check 1C patch source freshness.")
|
||||
parser.add_argument("--workspace", type=Path, required=True)
|
||||
parser.add_argument("--output", type=Path)
|
||||
args = parser.parse_args()
|
||||
|
||||
check = check_workspace_sources(args.workspace)
|
||||
output = json.dumps(check, ensure_ascii=False, indent=2) + "\n"
|
||||
if args.output:
|
||||
args.output.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.output.write_text(output, encoding="utf-8")
|
||||
print(json.dumps({"output": str(args.output) if args.output else None, "passed": check["passed"], "counts": check["counts"]}, ensure_ascii=False))
|
||||
return 0 if check["passed"] else 2
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,145 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Check integrity of a 1C patch workspace before diff/package/apply steps."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
def load_json(path: Path) -> dict[str, Any]:
|
||||
return json.loads(path.read_text(encoding="utf-8-sig"))
|
||||
|
||||
|
||||
def sha256_file(path: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as stream:
|
||||
for chunk in iter(lambda: stream.read(1024 * 1024), b""):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def issue(severity: str, code: str, message: str, *, path: str | None = None, record: dict[str, Any] | None = None) -> dict[str, Any]:
|
||||
result = {"severity": severity, "code": code, "message": message}
|
||||
if path:
|
||||
result["path"] = path
|
||||
if record:
|
||||
result["record"] = record
|
||||
return result
|
||||
|
||||
|
||||
def is_relative_safe(relative_path: str) -> bool:
|
||||
path = Path(relative_path)
|
||||
return not path.is_absolute() and ".." not in path.parts
|
||||
|
||||
|
||||
def expected_paths(workspace: Path, relative_path: str) -> tuple[Path, Path]:
|
||||
rel = Path(relative_path)
|
||||
return workspace / "original" / rel, workspace / "working" / rel
|
||||
|
||||
|
||||
def all_files(root: Path) -> list[Path]:
|
||||
if not root.exists():
|
||||
return []
|
||||
return sorted(path for path in root.rglob("*") if path.is_file())
|
||||
|
||||
|
||||
def rel_set(root: Path) -> set[str]:
|
||||
result = set()
|
||||
for path in all_files(root):
|
||||
result.add(str(path.relative_to(root)).replace("\\", "/"))
|
||||
return result
|
||||
|
||||
|
||||
def check_workspace(workspace: Path) -> dict[str, Any]:
|
||||
manifest_path = workspace / "manifest.json"
|
||||
findings = []
|
||||
if not manifest_path.exists():
|
||||
findings.append(issue("error", "missing_manifest", "Workspace manifest.json is missing.", path=str(manifest_path)))
|
||||
return result(workspace, findings, [])
|
||||
|
||||
manifest = load_json(manifest_path)
|
||||
records = manifest.get("files") or []
|
||||
manifest_relatives = set()
|
||||
file_checks = []
|
||||
for record in records:
|
||||
relative = str(record.get("relative_path") or "")
|
||||
manifest_relatives.add(relative)
|
||||
if not relative or not is_relative_safe(relative):
|
||||
findings.append(issue("error", "unsafe_relative_path", f"Unsafe relative path in manifest: {relative}", record=record))
|
||||
continue
|
||||
original, working = expected_paths(workspace, relative)
|
||||
check = {
|
||||
"relative_path": relative,
|
||||
"original_path": str(original),
|
||||
"working_path": str(working),
|
||||
"expected_sha256": record.get("sha256"),
|
||||
"original_exists": original.exists(),
|
||||
"working_exists": working.exists(),
|
||||
}
|
||||
if not original.exists():
|
||||
findings.append(issue("error", "missing_original_file", "Original file is missing.", path=str(original), record=record))
|
||||
else:
|
||||
actual = sha256_file(original)
|
||||
check["original_sha256"] = actual
|
||||
if record.get("sha256") and actual != record.get("sha256"):
|
||||
findings.append(issue("error", "original_hash_mismatch", "Original file hash differs from manifest; original/ must stay immutable.", path=str(original), record=record))
|
||||
if not working.exists():
|
||||
findings.append(issue("error", "missing_working_file", "Working file is missing.", path=str(working), record=record))
|
||||
else:
|
||||
check["working_sha256"] = sha256_file(working)
|
||||
file_checks.append(check)
|
||||
|
||||
original_extra = sorted(rel_set(workspace / "original") - manifest_relatives)
|
||||
working_extra = sorted(rel_set(workspace / "working") - manifest_relatives)
|
||||
for relative in original_extra:
|
||||
findings.append(issue("error", "extra_original_file", "Unexpected file under original/.", path=str(workspace / "original" / Path(relative))))
|
||||
for relative in working_extra:
|
||||
findings.append(issue("warning", "extra_working_file", "Unexpected file under working/; future packaging must explicitly include or reject it.", path=str(workspace / "working" / Path(relative))))
|
||||
|
||||
if not (workspace / "proposal.json").exists():
|
||||
findings.append(issue("warning", "missing_proposal_copy", "proposal.json is missing from workspace.", path=str(workspace / "proposal.json")))
|
||||
if not (workspace / "safety.json").exists():
|
||||
findings.append(issue("warning", "missing_safety_copy", "safety.json is missing from workspace.", path=str(workspace / "safety.json")))
|
||||
|
||||
return result(workspace, findings, file_checks, manifest=manifest)
|
||||
|
||||
|
||||
def result(workspace: Path, findings: list[dict[str, Any]], file_checks: list[dict[str, Any]], manifest: dict[str, Any] | None = None) -> dict[str, Any]:
|
||||
errors = [row for row in findings if row.get("severity") == "error"]
|
||||
warnings = [row for row in findings if row.get("severity") == "warning"]
|
||||
return {
|
||||
"schema": "onec_patch_workspace_integrity.v1",
|
||||
"workspace": str(workspace),
|
||||
"manifest_schema": (manifest or {}).get("schema"),
|
||||
"passed": not errors,
|
||||
"findings": findings,
|
||||
"file_checks": file_checks,
|
||||
"counts": {
|
||||
"files": len(file_checks),
|
||||
"errors": len(errors),
|
||||
"warnings": len(warnings),
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Check 1C patch workspace integrity.")
|
||||
parser.add_argument("--workspace", type=Path, required=True)
|
||||
parser.add_argument("--output", type=Path)
|
||||
args = parser.parse_args()
|
||||
|
||||
check = check_workspace(args.workspace)
|
||||
output = json.dumps(check, ensure_ascii=False, indent=2) + "\n"
|
||||
if args.output:
|
||||
args.output.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.output.write_text(output, encoding="utf-8")
|
||||
print(json.dumps({"output": str(args.output) if args.output else None, "passed": check["passed"], "counts": check["counts"]}, ensure_ascii=False))
|
||||
return 0 if check["passed"] else 2
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,272 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
import yaml
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
PLUGIN = ROOT / "plugins" / "1c"
|
||||
DEFAULT_REPORT = ROOT / "reports" / "1c-plugin-health.json"
|
||||
|
||||
|
||||
def run(command: list[str], *, allow_fail: bool = False) -> dict:
|
||||
result = subprocess.run(
|
||||
command,
|
||||
cwd=ROOT,
|
||||
text=True,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
check=False,
|
||||
)
|
||||
status = "ok" if result.returncode == 0 else "failed"
|
||||
if allow_fail and result.returncode != 0:
|
||||
status = "blocked"
|
||||
return {
|
||||
"command": command,
|
||||
"status": status,
|
||||
"returncode": result.returncode,
|
||||
"stdout": result.stdout.strip(),
|
||||
"stderr": result.stderr.strip(),
|
||||
}
|
||||
|
||||
|
||||
def prepare_rag_smoke_index(temp_path: Path) -> list[dict]:
|
||||
rag_source = temp_path / "metadata.health.generated.md"
|
||||
corpus = temp_path / "rag_corpus.jsonl"
|
||||
manifest = temp_path / "rag_manifest.json"
|
||||
index = temp_path / "rag_index.json"
|
||||
steps = [
|
||||
[
|
||||
sys.executable,
|
||||
"scripts/convert_1c_metadata_to_rag.py",
|
||||
"--input",
|
||||
"plugins/1c/metadata/examples/metadata.example.json",
|
||||
"--output",
|
||||
str(rag_source),
|
||||
],
|
||||
[
|
||||
sys.executable,
|
||||
"scripts/convert_1c_bsl_modules_to_rag.py",
|
||||
"--input",
|
||||
"plugins/1c/metadata/examples/bsl-modules.example.json",
|
||||
"--output",
|
||||
str(temp_path / "bsl.health.generated.md"),
|
||||
],
|
||||
[
|
||||
sys.executable,
|
||||
"scripts/validate_1c_rag_sources.py",
|
||||
"--source-dir",
|
||||
str(temp_path),
|
||||
],
|
||||
[
|
||||
sys.executable,
|
||||
"scripts/prepare_1c_rag_corpus.py",
|
||||
"--source-dir",
|
||||
str(temp_path),
|
||||
"--output",
|
||||
str(corpus),
|
||||
"--manifest",
|
||||
str(manifest),
|
||||
],
|
||||
[
|
||||
sys.executable,
|
||||
"scripts/build_1c_rag_index.py",
|
||||
"--corpus",
|
||||
str(corpus),
|
||||
"--output",
|
||||
str(index),
|
||||
],
|
||||
]
|
||||
return [run(step, allow_fail=True) for step in steps]
|
||||
|
||||
|
||||
def read_yaml(path: Path) -> dict:
|
||||
with path.open("r", encoding="utf-8") as handle:
|
||||
data = yaml.safe_load(handle)
|
||||
if not isinstance(data, dict):
|
||||
raise ValueError(f"{path} must contain a YAML mapping")
|
||||
return data
|
||||
|
||||
|
||||
def check_required_files() -> dict:
|
||||
required = [
|
||||
PLUGIN / "plugin.yaml",
|
||||
PLUGIN / "prompts" / "system.md",
|
||||
PLUGIN / "prompts" / "rag-answer.md",
|
||||
PLUGIN / "rag" / "profiles.yaml",
|
||||
PLUGIN / "rag" / "quality-smoke.json",
|
||||
PLUGIN / "rag" / "profile-routing-smoke.json",
|
||||
PLUGIN / "tools" / "tool-contract.yaml",
|
||||
PLUGIN / "connector" / "contracts" / "openapi.yaml",
|
||||
PLUGIN / "connector" / "policies" / "read-only-query.yaml",
|
||||
PLUGIN / "connector" / "policies" / "change-workflow.yaml",
|
||||
PLUGIN / "connector" / "policies" / "config-layer-write-policy.yaml",
|
||||
PLUGIN / "connector" / "Dockerfile",
|
||||
PLUGIN / "connector" / "docker-compose.yml",
|
||||
PLUGIN / "connector" / ".env.example",
|
||||
PLUGIN / "connector" / "pyproject.toml",
|
||||
PLUGIN / "connector" / "service.yaml",
|
||||
PLUGIN / "metadata" / "schema.json",
|
||||
PLUGIN / "metadata" / "moxel-schema-registry.json",
|
||||
PLUGIN / "metadata" / "examples" / "metadata.example.json",
|
||||
PLUGIN / "metadata" / "examples" / "metadata-v2.example.json",
|
||||
PLUGIN / "metadata" / "examples" / "bsl-modules.example.json",
|
||||
PLUGIN / "schemas" / "metadata-snapshot-v2.schema.json",
|
||||
PLUGIN / "schemas" / "bsl-module-snapshot.schema.json",
|
||||
PLUGIN / "schemas" / "moxel-schema-registry.schema.json",
|
||||
PLUGIN / "training" / "examples" / "instruction.examples.jsonl",
|
||||
PLUGIN / "training" / "configs" / "qwen3-coder-30b-a3b-lora.yaml",
|
||||
PLUGIN / "evals" / "smoke.yaml",
|
||||
]
|
||||
missing = [str(path.relative_to(ROOT)) for path in required if not path.exists()]
|
||||
return {
|
||||
"status": "ok" if not missing else "failed",
|
||||
"missing": missing,
|
||||
}
|
||||
|
||||
|
||||
def summarize_manifest() -> dict:
|
||||
manifest = read_yaml(PLUGIN / "plugin.yaml")
|
||||
return {
|
||||
"id": manifest.get("id"),
|
||||
"version": manifest.get("version"),
|
||||
"status": manifest.get("status"),
|
||||
"tasks": manifest.get("tasks") or [],
|
||||
"entrypoints": sorted((manifest.get("entrypoints") or {}).keys()),
|
||||
}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Run 1C plugin health checks.")
|
||||
parser.add_argument("--report", type=Path, default=DEFAULT_REPORT)
|
||||
parser.add_argument("--print", action="store_true")
|
||||
parser.add_argument(
|
||||
"--no-report",
|
||||
action="store_true",
|
||||
help="Do not write a health report file; useful for clean local checks.",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
with tempfile.TemporaryDirectory(prefix="llm-1c-health-") as temp_dir:
|
||||
temp_index = Path(temp_dir) / "rag_index.json"
|
||||
commands = {
|
||||
"metadata_example": [
|
||||
sys.executable,
|
||||
"scripts/validate_1c_metadata_snapshot.py",
|
||||
"plugins/1c/metadata/examples/metadata.example.json",
|
||||
],
|
||||
"metadata_v2_example": [
|
||||
sys.executable,
|
||||
"scripts/validate_1c_metadata_snapshot.py",
|
||||
"plugins/1c/metadata/examples/metadata-v2.example.json",
|
||||
],
|
||||
"bsl_modules_example": [
|
||||
sys.executable,
|
||||
"scripts/validate_1c_bsl_modules.py",
|
||||
"plugins/1c/metadata/examples/bsl-modules.example.json",
|
||||
],
|
||||
"readonly_query_allowed": [
|
||||
sys.executable,
|
||||
"scripts/validate_1c_readonly_query.py",
|
||||
"--query",
|
||||
"ВЫБРАТЬ Первые 10 Ссылка ИЗ Справочник.Номенклатура",
|
||||
],
|
||||
"training_examples": [
|
||||
sys.executable,
|
||||
"scripts/validate_1c_training_data.py",
|
||||
"plugins/1c/training/examples/instruction.examples.jsonl",
|
||||
],
|
||||
"evals": [sys.executable, "scripts/validate_evals.py", "plugins/1c/evals/smoke.yaml"],
|
||||
"connector_standalone_check": [sys.executable, "scripts/check_1c_connector_standalone.py"],
|
||||
"write_plan_contract": [sys.executable, "scripts/check_1c_write_plan_contract.py"],
|
||||
"bsl_symbol_check": [sys.executable, "scripts/check_1c_bsl_symbol_resolver.py"],
|
||||
"code_symbol_contract": [sys.executable, "scripts/check_1c_code_symbol_contract.py"],
|
||||
"module_origin_contract": [sys.executable, "scripts/check_1c_module_origin_contract.py"],
|
||||
"extension_action_contract": [sys.executable, "scripts/check_1c_extension_action_contract.py"],
|
||||
"moxel_schema_registry_contract": [
|
||||
sys.executable,
|
||||
"scripts/check_1c_moxel_schema_registry.py",
|
||||
"--registry",
|
||||
"plugins/1c/metadata/moxel-schema-registry.json",
|
||||
"--output",
|
||||
str(Path(temp_dir) / "moxel-schema-registry-check.json"),
|
||||
],
|
||||
"moxel_status": [
|
||||
sys.executable,
|
||||
"scripts/status_1c_moxel.py",
|
||||
"--output-json",
|
||||
str(Path(temp_dir) / "moxel-status.json"),
|
||||
"--output-markdown",
|
||||
str(Path(temp_dir) / "moxel-status.md"),
|
||||
],
|
||||
"rag_prompt_guardrails": [
|
||||
sys.executable,
|
||||
"scripts/check_1c_rag_prompt.py",
|
||||
"--index",
|
||||
str(temp_index),
|
||||
],
|
||||
"rag_quality": [
|
||||
sys.executable,
|
||||
"scripts/check_1c_rag_quality.py",
|
||||
"--index",
|
||||
str(temp_index),
|
||||
],
|
||||
"rag_profile_routing": [
|
||||
sys.executable,
|
||||
"scripts/check_1c_rag_profiles.py",
|
||||
],
|
||||
"training_preflight": [sys.executable, "scripts/preflight_1c_training.py"],
|
||||
}
|
||||
|
||||
checks = {
|
||||
"manifest": summarize_manifest(),
|
||||
"required_files": check_required_files(),
|
||||
"rag_smoke_prepare": prepare_rag_smoke_index(Path(temp_dir)),
|
||||
"commands": {},
|
||||
}
|
||||
|
||||
for name, command in commands.items():
|
||||
checks["commands"][name] = run(
|
||||
command,
|
||||
allow_fail=name in {"training_preflight"},
|
||||
)
|
||||
|
||||
failed = []
|
||||
for name, result in checks["commands"].items():
|
||||
if result["status"] == "failed":
|
||||
failed.append(name)
|
||||
if checks["required_files"]["status"] == "failed":
|
||||
failed.append("required_files")
|
||||
|
||||
report = {
|
||||
"plugin": "1c",
|
||||
"status": "failed" if failed else "ok",
|
||||
"failed_checks": failed,
|
||||
"checks": checks,
|
||||
}
|
||||
|
||||
if not args.no_report:
|
||||
args.report.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.report.write_text(
|
||||
json.dumps(report, ensure_ascii=False, indent=2) + "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
if args.print:
|
||||
print(json.dumps(report, ensure_ascii=False, indent=2))
|
||||
else:
|
||||
if not args.no_report:
|
||||
print(f"Wrote 1C plugin health report to {args.report}")
|
||||
print(f"Status: {report['status']}")
|
||||
|
||||
return 1 if failed else 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,128 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "scripts"))
|
||||
|
||||
from route_1c_question import route_question # noqa: E402
|
||||
|
||||
|
||||
DEFAULT_INDEX = ROOT / "reports" / "1c-sql" / "upo" / "unified-object-route-index.json"
|
||||
|
||||
|
||||
CASES = [
|
||||
{
|
||||
"id": "docs_only_form_open",
|
||||
"question": "Как работает событие ПриОткрытии формы?",
|
||||
"expected_route": "docs_rag",
|
||||
"expected_fact_paths": [],
|
||||
},
|
||||
{
|
||||
"id": "current_fact_extension_attribute",
|
||||
"question": "Есть ли реквизит ДатаСоздания у документа ПриходнаяНакладная?",
|
||||
"expected_route": "current_config_fact",
|
||||
"expected_fact_paths": ["Документ.ПриходнаяНакладная.ДатаСоздания"],
|
||||
"expected_exists": {"Документ.ПриходнаяНакладная.ДатаСоздания": True},
|
||||
},
|
||||
{
|
||||
"id": "rag_example_requires_current_fact_check",
|
||||
"question": "В примере RAG есть реквизит Артикул у справочника Номенклатура. Напиши код для текущей базы.",
|
||||
"expected_route": "mixed_docs_and_current_config",
|
||||
"expected_fact_paths": ["Справочник.Номенклатура.Артикул"],
|
||||
"expected_risk": "example_is_not_current_fact",
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
def fact_exists_by_path(route: dict) -> dict[str, bool | None]:
|
||||
result = {}
|
||||
for row in route.get("fact_checks") or []:
|
||||
path = row.get("path")
|
||||
if not path:
|
||||
continue
|
||||
if row.get("status") != "checked":
|
||||
result[path] = None
|
||||
else:
|
||||
result[path] = bool((row.get("result") or {}).get("exists"))
|
||||
return result
|
||||
|
||||
|
||||
def run_case(case: dict, *, index: Path, view: str) -> dict:
|
||||
route = route_question(case["question"], index_path=index, view=view)
|
||||
failures = []
|
||||
decision = route.get("decision") or {}
|
||||
if decision.get("route") != case.get("expected_route"):
|
||||
failures.append({"code": "route_mismatch", "expected": case.get("expected_route"), "actual": decision.get("route")})
|
||||
|
||||
actual_paths = route.get("fact_paths") or []
|
||||
expected_paths = case.get("expected_fact_paths") or []
|
||||
if actual_paths != expected_paths:
|
||||
failures.append({"code": "fact_paths_mismatch", "expected": expected_paths, "actual": actual_paths})
|
||||
|
||||
expected_risk = case.get("expected_risk")
|
||||
if expected_risk:
|
||||
risks = {row.get("code") for row in route.get("source_risks") or []}
|
||||
if expected_risk not in risks:
|
||||
failures.append({"code": "risk_missing", "expected": expected_risk, "actual": sorted(risks)})
|
||||
|
||||
exists = fact_exists_by_path(route)
|
||||
for path, expected in (case.get("expected_exists") or {}).items():
|
||||
if exists.get(path) is not expected:
|
||||
failures.append({"code": "fact_exists_mismatch", "path": path, "expected": expected, "actual": exists.get(path)})
|
||||
|
||||
return {
|
||||
"id": case["id"],
|
||||
"status": "passed" if not failures else "failed",
|
||||
"question": case["question"],
|
||||
"failures": failures,
|
||||
"route": {
|
||||
"decision": route.get("decision"),
|
||||
"source_risks": route.get("source_risks"),
|
||||
"fact_paths": route.get("fact_paths"),
|
||||
"fact_exists": exists,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def run_check(index: Path, *, view: str) -> dict:
|
||||
if not index.exists():
|
||||
return {
|
||||
"schema": "onec_question_router_check.v1",
|
||||
"status": "failed",
|
||||
"error": f"route index not found: {index}",
|
||||
"cases": [],
|
||||
}
|
||||
results = [run_case(case, index=index, view=view) for case in CASES]
|
||||
return {
|
||||
"schema": "onec_question_router_check.v1",
|
||||
"status": "ok" if all(row["status"] == "passed" for row in results) else "failed",
|
||||
"index": str(index),
|
||||
"view": view,
|
||||
"case_count": len(results),
|
||||
"cases": results,
|
||||
}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Check 1C question router behavior.")
|
||||
parser.add_argument("--index", type=Path, default=DEFAULT_INDEX)
|
||||
parser.add_argument("--view", choices=["effective", "base"], default="effective")
|
||||
parser.add_argument("--output", type=Path)
|
||||
parser.add_argument("--print", action="store_true")
|
||||
args = parser.parse_args()
|
||||
|
||||
report = run_check(args.index, view=args.view)
|
||||
if args.output:
|
||||
args.output.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.output.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
if args.print or not args.output:
|
||||
print(json.dumps(report, ensure_ascii=False, indent=2))
|
||||
return 0 if report["status"] == "ok" else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,113 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
from common import read_json
|
||||
from prepare_1c_rag_corpus import SUPPORTED_EXTENSIONS, classify_source, normalize_text, parse_front_matter
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
DEFAULT_SOURCE_DIR = ROOT / "plugins" / "1c" / "rag" / "sources"
|
||||
DEFAULT_MANIFEST = ROOT / "plugins" / "1c" / "datasets" / "prepared" / "rag_manifest.json"
|
||||
|
||||
|
||||
def iter_source_files(source_dir: Path) -> list[Path]:
|
||||
if not source_dir.exists():
|
||||
return []
|
||||
return sorted(
|
||||
path
|
||||
for path in source_dir.rglob("*")
|
||||
if path.is_file() and path.suffix.lower() in SUPPORTED_EXTENSIONS
|
||||
)
|
||||
|
||||
|
||||
def source_state(source_dir: Path) -> dict[str, dict]:
|
||||
state = {}
|
||||
for path in iter_source_files(source_dir):
|
||||
relative_path = path.relative_to(source_dir).as_posix()
|
||||
text = normalize_text(path.read_text(encoding="utf-8"))
|
||||
front_matter, body = parse_front_matter(text)
|
||||
chunk_source_text = body or text
|
||||
state[relative_path] = {
|
||||
"source_path": relative_path,
|
||||
"source_type": classify_source(path, chunk_source_text, front_matter),
|
||||
"file_type": path.suffix.lower().lstrip("."),
|
||||
"content_hash": hashlib.sha256(text.encode("utf-8")).hexdigest(),
|
||||
}
|
||||
return state
|
||||
|
||||
|
||||
def compare_manifest(source_dir: Path, manifest_path: Path) -> dict:
|
||||
current = source_state(source_dir)
|
||||
if not manifest_path.exists():
|
||||
return {
|
||||
"status": "stale" if current else "missing",
|
||||
"reason": "manifest is missing",
|
||||
"new": sorted(current),
|
||||
"changed": [],
|
||||
"deleted": [],
|
||||
"type_changed": [],
|
||||
}
|
||||
|
||||
manifest = read_json(manifest_path)
|
||||
recorded = {
|
||||
str(source.get("source_path")): source
|
||||
for source in manifest.get("sources") or []
|
||||
if isinstance(source, dict) and source.get("source_path")
|
||||
}
|
||||
|
||||
current_paths = set(current)
|
||||
recorded_paths = set(recorded)
|
||||
new = sorted(current_paths - recorded_paths)
|
||||
deleted = sorted(recorded_paths - current_paths)
|
||||
changed = []
|
||||
type_changed = []
|
||||
|
||||
for source_path in sorted(current_paths & recorded_paths):
|
||||
current_source = current[source_path]
|
||||
recorded_source = recorded[source_path]
|
||||
if current_source["content_hash"] != recorded_source.get("content_hash"):
|
||||
changed.append(source_path)
|
||||
if current_source["source_type"] != recorded_source.get("source_type"):
|
||||
type_changed.append(
|
||||
{
|
||||
"source_path": source_path,
|
||||
"current": current_source["source_type"],
|
||||
"manifest": recorded_source.get("source_type"),
|
||||
}
|
||||
)
|
||||
|
||||
stale = bool(new or deleted or changed or type_changed)
|
||||
return {
|
||||
"status": "stale" if stale else "fresh",
|
||||
"source_dir": str(source_dir),
|
||||
"manifest": str(manifest_path),
|
||||
"source_count": len(current),
|
||||
"manifest_source_count": len(recorded),
|
||||
"new": new,
|
||||
"changed": changed,
|
||||
"deleted": deleted,
|
||||
"type_changed": type_changed,
|
||||
}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Check whether the 1C RAG manifest is fresh.")
|
||||
parser.add_argument("--source-dir", type=Path, default=DEFAULT_SOURCE_DIR)
|
||||
parser.add_argument("--manifest", type=Path, default=DEFAULT_MANIFEST)
|
||||
parser.add_argument("--print", action="store_true")
|
||||
args = parser.parse_args()
|
||||
|
||||
report = compare_manifest(args.source_dir, args.manifest)
|
||||
if args.print:
|
||||
print(json.dumps(report, ensure_ascii=False, indent=2))
|
||||
else:
|
||||
print(f"1C RAG freshness: {report['status']}")
|
||||
return 0 if report["status"] in {"fresh", "missing"} else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,63 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
from common import read_json
|
||||
from rag_profiles import detect_rag_profile
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
DEFAULT_CASES = ROOT / "plugins" / "1c" / "rag" / "profile-routing-smoke.json"
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Check 1C RAG auto profile routing.")
|
||||
parser.add_argument("--cases", type=Path, default=DEFAULT_CASES)
|
||||
parser.add_argument("--print", action="store_true")
|
||||
args = parser.parse_args()
|
||||
|
||||
data = read_json(args.cases)
|
||||
cases = data.get("cases")
|
||||
if not isinstance(cases, list):
|
||||
print(f"{args.cases}: cases must be a list", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
results = []
|
||||
for case in cases:
|
||||
query = str(case.get("query") or "")
|
||||
expected = str(case.get("expected_profile") or "")
|
||||
actual = detect_rag_profile(query)
|
||||
results.append(
|
||||
{
|
||||
"id": case.get("id"),
|
||||
"query": query,
|
||||
"expected_profile": expected,
|
||||
"actual_profile": actual,
|
||||
"status": "passed" if actual == expected else "failed",
|
||||
}
|
||||
)
|
||||
|
||||
report = {
|
||||
"status": "ok" if all(result["status"] == "passed" for result in results) else "failed",
|
||||
"case_count": len(results),
|
||||
"results": results,
|
||||
}
|
||||
if args.print:
|
||||
print(json.dumps(report, ensure_ascii=False, indent=2))
|
||||
else:
|
||||
print(f"1C RAG profile routing status: {report['status']}")
|
||||
if report["status"] != "ok":
|
||||
for result in results:
|
||||
if result["status"] != "passed":
|
||||
print(
|
||||
f"- {result['id']}: expected {result['expected_profile']}, got {result['actual_profile']}",
|
||||
file=sys.stderr,
|
||||
)
|
||||
return 0 if report["status"] == "ok" else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,57 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
DEFAULT_INDEX = ROOT / "plugins" / "1c" / "datasets" / "prepared" / "rag_index.json"
|
||||
|
||||
REQUIRED_PHRASES = [
|
||||
"Не выдумывай метаданные 1С",
|
||||
"запроси метаданные через инструмент",
|
||||
"источники",
|
||||
"Какие реквизиты есть у справочника Номенклатура?",
|
||||
]
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Check that the 1C RAG prompt contains safety-critical instructions.")
|
||||
parser.add_argument("--index", type=Path, default=DEFAULT_INDEX)
|
||||
args = parser.parse_args()
|
||||
|
||||
command = [
|
||||
sys.executable,
|
||||
"scripts/ask_1c_rag.py",
|
||||
"Какие реквизиты есть у справочника Номенклатура?",
|
||||
"--index",
|
||||
str(args.index),
|
||||
"--print-prompt",
|
||||
]
|
||||
result = subprocess.run(
|
||||
command,
|
||||
cwd=ROOT,
|
||||
check=False,
|
||||
text=True,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
)
|
||||
if result.returncode != 0:
|
||||
print(result.stderr, file=sys.stderr)
|
||||
return result.returncode
|
||||
|
||||
missing = [phrase for phrase in REQUIRED_PHRASES if phrase not in result.stdout]
|
||||
if missing:
|
||||
print("1C RAG prompt check failed. Missing phrases:", file=sys.stderr)
|
||||
for phrase in missing:
|
||||
print(f"- {phrase}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
print("1C RAG prompt check passed.")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,87 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
from common import read_json, search_lexical_index
|
||||
from rag_profiles import resolve_rag_profile
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
DEFAULT_INDEX = ROOT / "plugins" / "1c" / "datasets" / "prepared" / "rag_index.json"
|
||||
DEFAULT_CASES = ROOT / "plugins" / "1c" / "rag" / "quality-smoke.json"
|
||||
|
||||
|
||||
def load_cases(path: Path) -> list[dict]:
|
||||
data = read_json(path)
|
||||
cases = data.get("cases")
|
||||
if not isinstance(cases, list):
|
||||
raise ValueError(f"{path} must contain a cases list")
|
||||
return cases
|
||||
|
||||
|
||||
def case_text(results: list[dict]) -> str:
|
||||
return "\n".join((result["document"].get("content") or "") for result in results).lower()
|
||||
|
||||
|
||||
def run_case(index: dict, case: dict, limit: int) -> dict:
|
||||
profile = resolve_rag_profile(case.get("profile") or "auto", str(case["query"]))
|
||||
results = search_lexical_index(
|
||||
index,
|
||||
str(case["query"]),
|
||||
limit=limit or int(profile["limit"]),
|
||||
candidate_limit=int(profile["candidate_limit"]),
|
||||
dedupe_by_document=bool(profile["dedupe_by_document"]),
|
||||
min_score=float(profile["min_score"]),
|
||||
source_types=profile["source_types"],
|
||||
)
|
||||
text = case_text(results)
|
||||
missing = [word for word in case.get("must_contain") or [] if str(word).lower() not in text]
|
||||
return {
|
||||
"id": case.get("id"),
|
||||
"profile": profile["id"],
|
||||
"query": case.get("query"),
|
||||
"status": "passed" if results and not missing else "failed",
|
||||
"missing": missing,
|
||||
"top_sources": [
|
||||
{
|
||||
"source_path": result["document"].get("source_path"),
|
||||
"title": result["document"].get("title"),
|
||||
"chunk_index": result["document"].get("chunk_index"),
|
||||
"score": round(float(result.get("score") or 0), 4),
|
||||
}
|
||||
for result in results
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Run smoke quality checks for the 1C RAG index.")
|
||||
parser.add_argument("--index", type=Path, default=DEFAULT_INDEX)
|
||||
parser.add_argument("--cases", type=Path, default=DEFAULT_CASES)
|
||||
parser.add_argument("--limit", type=int, default=5)
|
||||
parser.add_argument("--print", action="store_true")
|
||||
args = parser.parse_args()
|
||||
|
||||
index = read_json(args.index)
|
||||
results = [run_case(index, case, args.limit) for case in load_cases(args.cases)]
|
||||
report = {
|
||||
"status": "ok" if all(result["status"] == "passed" for result in results) else "failed",
|
||||
"case_count": len(results),
|
||||
"results": results,
|
||||
}
|
||||
if args.print:
|
||||
print(json.dumps(report, ensure_ascii=False, indent=2))
|
||||
else:
|
||||
print(f"1C RAG quality status: {report['status']}")
|
||||
if report["status"] != "ok":
|
||||
for result in results:
|
||||
if result["status"] != "passed":
|
||||
print(f"- {result['id']}: missing {result['missing']}", file=sys.stderr)
|
||||
return 0 if report["status"] == "ok" else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,84 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "scripts"))
|
||||
|
||||
from management_console_server import query_rag # noqa: E402
|
||||
|
||||
|
||||
def run_checks() -> dict:
|
||||
official = query_rag(
|
||||
{
|
||||
"question": "форма при открытии пример артикул",
|
||||
"source_type": "official_1c_docs",
|
||||
"limit": 8,
|
||||
}
|
||||
)
|
||||
official_bad = [
|
||||
row
|
||||
for row in official.get("results") or []
|
||||
if row.get("source_type") in {"metadata", "bsl_modules", "examples"}
|
||||
or "example" in str(row.get("source_path") or "").casefold()
|
||||
]
|
||||
|
||||
metadata = query_rag(
|
||||
{
|
||||
"question": "какие реквизиты у справочника Номенклатура",
|
||||
"source_type": "metadata",
|
||||
"limit": 5,
|
||||
}
|
||||
)
|
||||
metadata_answer = str(metadata.get("answer") or "").casefold()
|
||||
|
||||
checks = [
|
||||
{
|
||||
"id": "official_docs_exclude_examples",
|
||||
"status": "passed" if not official_bad else "failed",
|
||||
"details": official_bad,
|
||||
},
|
||||
{
|
||||
"id": "metadata_scope_warns",
|
||||
"status": "passed" if "примеры" in metadata_answer and "не подтверждают текущую базу" in metadata_answer else "failed",
|
||||
"answer": metadata.get("answer"),
|
||||
},
|
||||
]
|
||||
return {
|
||||
"schema": "onec_rag_source_governance_check.v1",
|
||||
"status": "ok" if all(item["status"] == "passed" for item in checks) else "failed",
|
||||
"checks": checks,
|
||||
"samples": {
|
||||
"official_1c_docs": {
|
||||
"source_scope": official.get("source_scope"),
|
||||
"result_count": official.get("result_count"),
|
||||
"source_types": sorted({row.get("source_type") for row in official.get("results") or [] if row.get("source_type")}),
|
||||
},
|
||||
"metadata": {
|
||||
"source_scope": metadata.get("source_scope"),
|
||||
"result_count": metadata.get("result_count"),
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Check 1C RAG source governance.")
|
||||
parser.add_argument("--output", type=Path)
|
||||
parser.add_argument("--print", action="store_true")
|
||||
args = parser.parse_args()
|
||||
|
||||
report = run_checks()
|
||||
if args.output:
|
||||
args.output.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.output.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
if args.print or not args.output:
|
||||
print(json.dumps(report, ensure_ascii=False, indent=2))
|
||||
return 0 if report["status"] == "ok" else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,59 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import sqlite3
|
||||
from pathlib import Path
|
||||
|
||||
from search_1c_rag_vector import DEFAULT_CORPUS, DEFAULT_INDEX, freshness, load_meta
|
||||
|
||||
|
||||
def check_vector_freshness(index_path: Path, corpus_path: Path) -> dict:
|
||||
if not index_path.exists():
|
||||
return {
|
||||
"status": "missing",
|
||||
"reason": "vector index is missing",
|
||||
"index": str(index_path),
|
||||
"corpus": str(corpus_path),
|
||||
}
|
||||
try:
|
||||
conn = sqlite3.connect(index_path)
|
||||
try:
|
||||
meta = load_meta(conn)
|
||||
finally:
|
||||
conn.close()
|
||||
except sqlite3.Error as exc:
|
||||
return {
|
||||
"status": "invalid",
|
||||
"reason": str(exc),
|
||||
"index": str(index_path),
|
||||
"corpus": str(corpus_path),
|
||||
}
|
||||
report = freshness(meta, corpus_path)
|
||||
return {
|
||||
**report,
|
||||
"index": str(index_path),
|
||||
"embedding_model": meta.get("embedding_model"),
|
||||
"embedding_dimensions": meta.get("embedding_dimensions"),
|
||||
"doc_count": meta.get("doc_count"),
|
||||
"built_at": meta.get("built_at"),
|
||||
}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Check whether the 1C RAG vector index is fresh.")
|
||||
parser.add_argument("--index", type=Path, default=DEFAULT_INDEX)
|
||||
parser.add_argument("--corpus", type=Path, default=DEFAULT_CORPUS)
|
||||
parser.add_argument("--print", action="store_true")
|
||||
args = parser.parse_args()
|
||||
|
||||
report = check_vector_freshness(args.index, args.corpus)
|
||||
if args.print:
|
||||
print(json.dumps(report, ensure_ascii=False, indent=2))
|
||||
else:
|
||||
print(f"1C RAG vector freshness: {report['status']}")
|
||||
return 0 if report["status"] in {"fresh", "missing"} else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,138 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Validate a latest 1C saved-state watch run lookup."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
EXPECTED_SCHEMA = "onec_saved_state_latest_watch_run.v1"
|
||||
|
||||
|
||||
def load_json(path: Path) -> dict[str, Any]:
|
||||
return json.loads(path.read_text(encoding="utf-8-sig"))
|
||||
|
||||
|
||||
def write_json(path: Path, data: dict[str, Any]) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(json.dumps(data, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
|
||||
|
||||
def issue(severity: str, code: str, message: str, *, path: Path | str | None = None, detail: dict[str, Any] | None = None) -> dict[str, Any]:
|
||||
result: dict[str, Any] = {"severity": severity, "code": code, "message": message}
|
||||
if path is not None:
|
||||
result["path"] = str(path)
|
||||
if detail:
|
||||
result["detail"] = detail
|
||||
return result
|
||||
|
||||
|
||||
def linked_path(latest_path: Path, value: Any) -> Path | None:
|
||||
if not value:
|
||||
return None
|
||||
path = Path(str(value))
|
||||
if path.is_absolute():
|
||||
return path
|
||||
return (latest_path.parent / path).resolve()
|
||||
|
||||
|
||||
def check_linked_check(latest_path: Path, run: dict[str, Any], key: str, passed_key: str, findings: list[dict[str, Any]]) -> None:
|
||||
path = linked_path(latest_path, run.get(key))
|
||||
if path is None:
|
||||
return
|
||||
if not path.exists():
|
||||
findings.append(issue("error", f"missing_{key}", f"Linked {key} file is missing.", path=path))
|
||||
return
|
||||
linked = load_json(path)
|
||||
if run.get(passed_key) != linked.get("passed"):
|
||||
findings.append(issue("error", f"{passed_key}_mismatch", f"{passed_key} differs from linked check.", path=path))
|
||||
|
||||
|
||||
def has_delta_changes(run: dict[str, Any]) -> bool:
|
||||
counts = run.get("counts") or {}
|
||||
return any(int(counts.get(key) or 0) > 0 for key in ("delta_objects_added", "delta_objects_removed", "delta_objects_changed"))
|
||||
|
||||
|
||||
def check_latest(latest_path: Path) -> dict[str, Any]:
|
||||
findings: list[dict[str, Any]] = []
|
||||
if not latest_path.exists():
|
||||
findings.append(issue("error", "missing_latest", "Latest watch lookup file is missing.", path=latest_path))
|
||||
return build_result(latest_path, findings)
|
||||
|
||||
data = load_json(latest_path)
|
||||
if data.get("schema") != EXPECTED_SCHEMA:
|
||||
findings.append(issue("error", "invalid_schema", "Latest watch lookup schema is invalid.", detail={"schema": data.get("schema")}))
|
||||
|
||||
safety = data.get("safety") or {}
|
||||
if safety.get("read_only") is not True:
|
||||
findings.append(issue("error", "not_read_only", "Latest watch lookup safety.read_only must be true."))
|
||||
if safety.get("sql_write_performed") is not False:
|
||||
findings.append(issue("error", "sql_write_flag", "Latest watch lookup safety.sql_write_performed must be false."))
|
||||
|
||||
markdown = linked_path(latest_path, data.get("markdown"))
|
||||
if data.get("markdown") and (markdown is None or not markdown.exists()):
|
||||
findings.append(issue("error", "missing_markdown", "Linked latest Markdown file is missing.", path=markdown or "<null>"))
|
||||
|
||||
found = data.get("found")
|
||||
latest = data.get("latest")
|
||||
if found is True:
|
||||
if not isinstance(latest, dict):
|
||||
findings.append(issue("error", "missing_latest_run", "found=true requires latest object."))
|
||||
return build_result(latest_path, findings)
|
||||
run_dir = linked_path(latest_path, latest.get("run_dir"))
|
||||
if run_dir is None or not run_dir.exists():
|
||||
findings.append(issue("error", "missing_run_dir", "Latest run directory is missing.", path=run_dir or "<null>"))
|
||||
for key in ("report", "manifest_check"):
|
||||
path = linked_path(latest_path, latest.get(key))
|
||||
if path is not None and not path.exists():
|
||||
findings.append(issue("error", f"missing_{key}", f"Linked {key} is missing.", path=path))
|
||||
check_linked_check(latest_path, latest, "manifest_check", "manifest_check_passed", findings)
|
||||
check_linked_check(latest_path, latest, "delta_check", "delta_check_passed", findings)
|
||||
if data.get("require_delta") is True and not latest.get("delta"):
|
||||
findings.append(issue("error", "required_delta_missing", "require_delta=true but latest run has no delta."))
|
||||
if data.get("require_changed") is True and not has_delta_changes(latest):
|
||||
findings.append(issue("error", "required_changed_missing", "require_changed=true but latest run has no delta changes."))
|
||||
elif found is False:
|
||||
if latest is not None:
|
||||
findings.append(issue("error", "unexpected_latest", "found=false requires latest=null."))
|
||||
else:
|
||||
findings.append(issue("error", "invalid_found", "found must be boolean."))
|
||||
|
||||
counts = data.get("counts")
|
||||
if not isinstance(counts, dict):
|
||||
findings.append(issue("error", "missing_counts", "Latest lookup must include counts."))
|
||||
|
||||
return build_result(latest_path, findings)
|
||||
|
||||
|
||||
def build_result(latest_path: Path, findings: list[dict[str, Any]]) -> dict[str, Any]:
|
||||
errors = [row for row in findings if row.get("severity") == "error"]
|
||||
warnings = [row for row in findings if row.get("severity") == "warning"]
|
||||
return {
|
||||
"schema": "onec_saved_state_latest_watch_run_check.v1",
|
||||
"latest": str(latest_path),
|
||||
"passed": not errors,
|
||||
"findings": findings,
|
||||
"counts": {"errors": len(errors), "warnings": len(warnings)},
|
||||
"safety": {"read_only": True, "sql_write_performed": False},
|
||||
}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Validate latest 1C saved-state watch run lookup.")
|
||||
parser.add_argument("--latest", type=Path, required=True)
|
||||
parser.add_argument("--output", type=Path)
|
||||
args = parser.parse_args()
|
||||
|
||||
result = check_latest(args.latest)
|
||||
if args.output:
|
||||
write_json(args.output, result)
|
||||
print(json.dumps({"output": str(args.output) if args.output else None, "passed": result["passed"], "counts": result["counts"]}, ensure_ascii=False))
|
||||
return 0 if result["passed"] else 2
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,238 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Validate a 1C saved-state object report contract."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
EXPECTED_SCHEMA = "onec_saved_state_object_report.v1"
|
||||
EXPECTED_COMPARISON_SCHEMA = "onec_saved_state_object_comparison.v1"
|
||||
EXPECTED_DETAIL_SCHEMA = "onec_saved_state_object_detail.v1"
|
||||
KNOWN_PAYLOAD_ROLES = {
|
||||
"bsl_module_text",
|
||||
"form_descriptor",
|
||||
"form_body",
|
||||
"primary_payload",
|
||||
"metadata_payload",
|
||||
}
|
||||
|
||||
|
||||
def load_json(path: Path) -> dict[str, Any]:
|
||||
return json.loads(path.read_text(encoding="utf-8-sig"))
|
||||
|
||||
|
||||
def write_json(path: Path, data: dict[str, Any]) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(json.dumps(data, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
|
||||
|
||||
def issue(severity: str, code: str, message: str, *, path: Path | str | None = None, detail: dict[str, Any] | None = None) -> dict[str, Any]:
|
||||
result: dict[str, Any] = {"severity": severity, "code": code, "message": message}
|
||||
if path is not None:
|
||||
result["path"] = str(path)
|
||||
if detail:
|
||||
result["detail"] = detail
|
||||
return result
|
||||
|
||||
|
||||
def linked_path(report_path: Path, value: Any) -> Path | None:
|
||||
if not value:
|
||||
return None
|
||||
path = Path(str(value))
|
||||
if path.is_absolute():
|
||||
return path
|
||||
return (report_path.parent / path).resolve()
|
||||
|
||||
|
||||
def is_list(value: Any) -> bool:
|
||||
return isinstance(value, list)
|
||||
|
||||
|
||||
def check_agent_summary(report: dict[str, Any], findings: list[dict[str, Any]]) -> None:
|
||||
summary = report.get("agent_summary")
|
||||
if not isinstance(summary, dict):
|
||||
findings.append(issue("error", "missing_agent_summary", "Report must include agent_summary."))
|
||||
return
|
||||
|
||||
object_changes = summary.get("object_changes")
|
||||
if not isinstance(object_changes, list):
|
||||
findings.append(issue("error", "invalid_agent_summary_object_changes", "agent_summary.object_changes must be an array."))
|
||||
object_changes = []
|
||||
object_names = summary.get("object_names")
|
||||
if not isinstance(object_names, list):
|
||||
findings.append(issue("error", "invalid_agent_summary_object_names", "agent_summary.object_names must be an array."))
|
||||
object_names = []
|
||||
|
||||
names_from_objects = [item.get("full_name") for item in object_changes if isinstance(item, dict)]
|
||||
if names_from_objects != object_names:
|
||||
findings.append(issue(
|
||||
"error",
|
||||
"agent_summary_names_mismatch",
|
||||
"agent_summary.object_names must match object_changes full_name order.",
|
||||
detail={"object_names": object_names, "from_objects": names_from_objects},
|
||||
))
|
||||
|
||||
for index, item in enumerate(object_changes):
|
||||
if not isinstance(item, dict):
|
||||
findings.append(issue("error", "invalid_agent_summary_object", "agent_summary object item must be an object.", detail={"index": index}))
|
||||
continue
|
||||
full_name = item.get("full_name")
|
||||
if not full_name:
|
||||
findings.append(issue("error", "missing_agent_summary_full_name", "agent_summary object item is missing full_name.", detail={"index": index}))
|
||||
for key in ("added_terms", "removed_terms", "parts"):
|
||||
if not is_list(item.get(key)):
|
||||
findings.append(issue("error", f"invalid_{key}", f"agent_summary object field {key} must be an array.", detail={"object": full_name, "type": type(item.get(key)).__name__}))
|
||||
parts = item.get("parts") if isinstance(item.get("parts"), list) else []
|
||||
if item.get("parts_count") != len(parts):
|
||||
findings.append(issue("error", "agent_summary_parts_count_mismatch", "parts_count must match parts length.", detail={"object": full_name, "parts_count": item.get("parts_count"), "actual": len(parts)}))
|
||||
text_diff_parts = 0
|
||||
active_missing_parts = 0
|
||||
for part in parts:
|
||||
if not isinstance(part, dict):
|
||||
continue
|
||||
role = part.get("payload_role")
|
||||
if role not in KNOWN_PAYLOAD_ROLES:
|
||||
findings.append(issue("error", "unknown_payload_role", "Unknown payload_role in agent_summary part.", detail={"object": full_name, "role": role, "file_name": part.get("file_name")}))
|
||||
if part.get("summary") in {"Text payload differs.", "Text payload matches."}:
|
||||
text_diff_parts += 1
|
||||
if part.get("active_exists") is False:
|
||||
active_missing_parts += 1
|
||||
if item.get("text_diff_parts") != text_diff_parts:
|
||||
findings.append(issue("warning", "agent_summary_text_diff_count_mismatch", "text_diff_parts differs from counted comparable parts.", detail={"object": full_name, "reported": item.get("text_diff_parts"), "counted": text_diff_parts}))
|
||||
if item.get("active_missing_parts") != active_missing_parts:
|
||||
findings.append(issue("error", "agent_summary_active_missing_count_mismatch", "active_missing_parts must match parts with active_exists=false.", detail={"object": full_name, "reported": item.get("active_missing_parts"), "counted": active_missing_parts}))
|
||||
|
||||
system_changes = summary.get("system_changes")
|
||||
if not isinstance(system_changes, list):
|
||||
findings.append(issue("error", "invalid_agent_summary_system_changes", "agent_summary.system_changes must be an array."))
|
||||
system_changes = []
|
||||
system_names = summary.get("system_change_names")
|
||||
if not isinstance(system_names, list):
|
||||
findings.append(issue("error", "invalid_agent_summary_system_names", "agent_summary.system_change_names must be an array."))
|
||||
system_names = []
|
||||
names_from_system = [item.get("name") for item in system_changes if isinstance(item, dict)]
|
||||
if names_from_system != system_names:
|
||||
findings.append(issue("error", "agent_summary_system_names_mismatch", "system_change_names must match system_changes name order.", detail={"system_change_names": system_names, "from_system": names_from_system}))
|
||||
|
||||
|
||||
def check_detail(detail: dict[str, Any], findings: list[dict[str, Any]]) -> None:
|
||||
if detail.get("schema") != EXPECTED_DETAIL_SCHEMA:
|
||||
findings.append(issue("error", "invalid_detail_schema", "Detail schema is invalid.", detail={"schema": detail.get("schema")}))
|
||||
for obj in detail.get("object_details") or []:
|
||||
if not isinstance(obj, dict):
|
||||
continue
|
||||
full_name = obj.get("full_name")
|
||||
for part in obj.get("details") or []:
|
||||
if not isinstance(part, dict):
|
||||
continue
|
||||
role = part.get("payload_role")
|
||||
if role not in KNOWN_PAYLOAD_ROLES:
|
||||
findings.append(issue("error", "detail_unknown_payload_role", "Unknown payload_role in detail part.", detail={"object": full_name, "role": role, "file_name": part.get("file_name")}))
|
||||
payload = part.get("payload") or {}
|
||||
hints = payload.get("semantic_hints")
|
||||
if hints is not None:
|
||||
for key in ("added_terms", "removed_terms"):
|
||||
if not isinstance(hints.get(key), list):
|
||||
findings.append(issue("error", "invalid_semantic_hints", f"semantic_hints.{key} must be an array.", detail={"object": full_name, "file_name": part.get("file_name")}))
|
||||
|
||||
|
||||
def check_report(report_path: Path) -> dict[str, Any]:
|
||||
findings: list[dict[str, Any]] = []
|
||||
if not report_path.exists():
|
||||
findings.append(issue("error", "missing_report", "Saved-state report file is missing.", path=report_path))
|
||||
return build_result(report_path, findings, None, None)
|
||||
|
||||
report = load_json(report_path)
|
||||
if report.get("schema") != EXPECTED_SCHEMA:
|
||||
findings.append(issue("error", "invalid_report_schema", "Report schema is invalid.", path=report_path, detail={"schema": report.get("schema")}))
|
||||
|
||||
safety = report.get("safety") or {}
|
||||
if safety.get("read_only") is not True:
|
||||
findings.append(issue("error", "report_not_read_only", "Report safety.read_only must be true."))
|
||||
if safety.get("sql_write_performed") is not False:
|
||||
findings.append(issue("error", "report_sql_write_flag", "Report safety.sql_write_performed must be false."))
|
||||
if safety.get("public_terms_are_1c_objects") is not True:
|
||||
findings.append(issue("error", "report_public_terms_flag", "Report must expose public terms as 1C objects."))
|
||||
if safety.get("secrets_in_report") is not False:
|
||||
findings.append(issue("error", "report_secrets_flag", "Report safety.secrets_in_report must be false."))
|
||||
|
||||
comparison_path = linked_path(report_path, report.get("comparison"))
|
||||
detail_path = linked_path(report_path, report.get("detail"))
|
||||
markdown_path = linked_path(report_path, report.get("markdown"))
|
||||
comparison: dict[str, Any] | None = None
|
||||
detail: dict[str, Any] | None = None
|
||||
|
||||
if comparison_path is None or not comparison_path.exists():
|
||||
findings.append(issue("error", "missing_comparison", "Linked comparison JSON is missing.", path=comparison_path or "<null>"))
|
||||
else:
|
||||
comparison = load_json(comparison_path)
|
||||
if comparison.get("schema") != EXPECTED_COMPARISON_SCHEMA:
|
||||
findings.append(issue("error", "invalid_comparison_schema", "Comparison schema is invalid.", path=comparison_path, detail={"schema": comparison.get("schema")}))
|
||||
|
||||
if detail_path is None or not detail_path.exists():
|
||||
findings.append(issue("error", "missing_detail", "Linked detail JSON is missing.", path=detail_path or "<null>"))
|
||||
else:
|
||||
detail = load_json(detail_path)
|
||||
check_detail(detail, findings)
|
||||
|
||||
if report.get("markdown") is not None and (markdown_path is None or not markdown_path.exists()):
|
||||
findings.append(issue("error", "missing_markdown", "Linked Markdown report is missing.", path=markdown_path or "<null>"))
|
||||
|
||||
counts = report.get("counts") or {}
|
||||
if comparison:
|
||||
comparison_counts = comparison.get("counts") or {}
|
||||
if counts.get("object_changes") != comparison_counts.get("object_changes"):
|
||||
findings.append(issue("error", "object_change_count_mismatch", "Report object_changes count differs from comparison."))
|
||||
if counts.get("system_changes") != comparison_counts.get("system_changes"):
|
||||
findings.append(issue("error", "system_change_count_mismatch", "Report system_changes count differs from comparison."))
|
||||
if detail:
|
||||
detail_counts = detail.get("counts") or {}
|
||||
if counts.get("detail_objects") != detail_counts.get("objects"):
|
||||
findings.append(issue("error", "detail_object_count_mismatch", "Report detail_objects count differs from detail."))
|
||||
if counts.get("detail_parts") != detail_counts.get("details"):
|
||||
findings.append(issue("error", "detail_part_count_mismatch", "Report detail_parts count differs from detail."))
|
||||
|
||||
check_agent_summary(report, findings)
|
||||
return build_result(report_path, findings, comparison, detail)
|
||||
|
||||
|
||||
def build_result(report_path: Path, findings: list[dict[str, Any]], comparison: dict[str, Any] | None, detail: dict[str, Any] | None) -> dict[str, Any]:
|
||||
errors = [row for row in findings if row.get("severity") == "error"]
|
||||
warnings = [row for row in findings if row.get("severity") == "warning"]
|
||||
return {
|
||||
"schema": "onec_saved_state_object_report_check.v1",
|
||||
"report": str(report_path),
|
||||
"comparison_schema": (comparison or {}).get("schema"),
|
||||
"detail_schema": (detail or {}).get("schema"),
|
||||
"passed": not errors,
|
||||
"findings": findings,
|
||||
"counts": {
|
||||
"errors": len(errors),
|
||||
"warnings": len(warnings),
|
||||
},
|
||||
"safety": {
|
||||
"read_only": True,
|
||||
"sql_write_performed": False,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Validate a 1C saved-state object report.")
|
||||
parser.add_argument("--report", type=Path, required=True)
|
||||
parser.add_argument("--output", type=Path)
|
||||
args = parser.parse_args()
|
||||
|
||||
result = check_report(args.report)
|
||||
if args.output:
|
||||
write_json(args.output, result)
|
||||
print(json.dumps({"output": str(args.output) if args.output else None, "passed": result["passed"], "counts": result["counts"]}, ensure_ascii=False))
|
||||
return 0 if result["passed"] else 2
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,217 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Validate a 1C saved-state report delta contract."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
EXPECTED_SCHEMA = "onec_saved_state_object_report_delta.v1"
|
||||
KNOWN_PAYLOAD_ROLES = {
|
||||
"bsl_module_text",
|
||||
"form_descriptor",
|
||||
"form_body",
|
||||
"primary_payload",
|
||||
"metadata_payload",
|
||||
}
|
||||
|
||||
|
||||
def load_json(path: Path) -> dict[str, Any]:
|
||||
return json.loads(path.read_text(encoding="utf-8-sig"))
|
||||
|
||||
|
||||
def write_json(path: Path, data: dict[str, Any]) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(json.dumps(data, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
|
||||
|
||||
def issue(severity: str, code: str, message: str, *, path: Path | str | None = None, detail: dict[str, Any] | None = None) -> dict[str, Any]:
|
||||
result: dict[str, Any] = {"severity": severity, "code": code, "message": message}
|
||||
if path is not None:
|
||||
result["path"] = str(path)
|
||||
if detail:
|
||||
result["detail"] = detail
|
||||
return result
|
||||
|
||||
|
||||
def linked_path(delta_path: Path, value: Any) -> Path | None:
|
||||
if not value:
|
||||
return None
|
||||
path = Path(str(value))
|
||||
if path.is_absolute():
|
||||
return path
|
||||
return (delta_path.parent / path).resolve()
|
||||
|
||||
|
||||
def check_payload_roles(parts: list[Any], findings: list[dict[str, Any]], *, context: str) -> None:
|
||||
for part in parts:
|
||||
if not isinstance(part, dict):
|
||||
findings.append(issue("error", "invalid_part", "Payload part must be an object.", detail={"context": context}))
|
||||
continue
|
||||
role = part.get("payload_role")
|
||||
if role not in KNOWN_PAYLOAD_ROLES:
|
||||
findings.append(issue("error", "unknown_payload_role", "Unknown payload_role.", detail={"context": context, "role": role, "file_name": part.get("file_name")}))
|
||||
|
||||
|
||||
def check_compact_object(item: dict[str, Any], findings: list[dict[str, Any]], *, context: str) -> None:
|
||||
if not item.get("full_name"):
|
||||
findings.append(issue("error", "missing_full_name", "Object delta item is missing full_name.", detail={"context": context}))
|
||||
for key in ("added_terms", "removed_terms", "parts"):
|
||||
if not isinstance(item.get(key), list):
|
||||
findings.append(issue("error", f"invalid_{key}", f"Object field {key} must be an array.", detail={"context": context, "full_name": item.get("full_name")}))
|
||||
parts = item.get("parts") if isinstance(item.get("parts"), list) else []
|
||||
if item.get("parts_count") != len(parts):
|
||||
findings.append(issue("error", "parts_count_mismatch", "parts_count must match parts length.", detail={"context": context, "full_name": item.get("full_name"), "parts_count": item.get("parts_count"), "actual": len(parts)}))
|
||||
check_payload_roles(parts, findings, context=context)
|
||||
|
||||
|
||||
def check_changed_object(item: dict[str, Any], findings: list[dict[str, Any]]) -> None:
|
||||
full_name = item.get("full_name")
|
||||
before = item.get("before")
|
||||
after = item.get("after")
|
||||
if not isinstance(before, dict) or not isinstance(after, dict):
|
||||
findings.append(issue("error", "invalid_changed_object_shape", "Changed object must include before and after objects.", detail={"full_name": full_name}))
|
||||
return
|
||||
check_compact_object(before, findings, context=f"changed.before:{full_name}")
|
||||
check_compact_object(after, findings, context=f"changed.after:{full_name}")
|
||||
for key in ("before_fingerprint", "after_fingerprint"):
|
||||
if not item.get(key):
|
||||
findings.append(issue("error", f"missing_{key}", f"Changed object is missing {key}.", detail={"full_name": full_name}))
|
||||
if item.get("before_fingerprint") == item.get("after_fingerprint"):
|
||||
findings.append(issue("error", "unchanged_fingerprint_in_changed", "Changed object has equal before and after fingerprints.", detail={"full_name": full_name}))
|
||||
|
||||
term_delta = item.get("term_delta")
|
||||
if not isinstance(term_delta, dict):
|
||||
findings.append(issue("error", "invalid_term_delta", "Changed object term_delta must be an object.", detail={"full_name": full_name}))
|
||||
else:
|
||||
for list_name in ("added_terms", "removed_terms"):
|
||||
delta = term_delta.get(list_name)
|
||||
if not isinstance(delta, dict) or not isinstance(delta.get("added"), list) or not isinstance(delta.get("removed"), list):
|
||||
findings.append(issue("error", "invalid_term_delta_list", "Term delta must contain added and removed arrays.", detail={"full_name": full_name, "list": list_name}))
|
||||
|
||||
part_delta = item.get("part_delta")
|
||||
if not isinstance(part_delta, dict):
|
||||
findings.append(issue("error", "invalid_part_delta", "Changed object part_delta must be an object.", detail={"full_name": full_name}))
|
||||
else:
|
||||
for key in ("added", "removed", "changed"):
|
||||
if not isinstance(part_delta.get(key), list):
|
||||
findings.append(issue("error", f"invalid_part_delta_{key}", f"part_delta.{key} must be an array.", detail={"full_name": full_name}))
|
||||
check_payload_roles(part_delta.get("added") or [], findings, context=f"part_delta.added:{full_name}")
|
||||
check_payload_roles(part_delta.get("removed") or [], findings, context=f"part_delta.removed:{full_name}")
|
||||
for part in part_delta.get("changed") or []:
|
||||
if not isinstance(part, dict):
|
||||
findings.append(issue("error", "invalid_changed_part", "part_delta.changed item must be an object.", detail={"full_name": full_name}))
|
||||
continue
|
||||
before_part = part.get("before")
|
||||
after_part = part.get("after")
|
||||
if not isinstance(before_part, dict) or not isinstance(after_part, dict):
|
||||
findings.append(issue("error", "invalid_changed_part_shape", "Changed part must include before and after.", detail={"full_name": full_name, "file_name": part.get("file_name")}))
|
||||
continue
|
||||
check_payload_roles([before_part, after_part], findings, context=f"part_delta.changed:{full_name}")
|
||||
|
||||
|
||||
def check_delta(delta_path: Path) -> dict[str, Any]:
|
||||
findings: list[dict[str, Any]] = []
|
||||
if not delta_path.exists():
|
||||
findings.append(issue("error", "missing_delta", "Delta report file is missing.", path=delta_path))
|
||||
return build_result(delta_path, findings)
|
||||
|
||||
data = load_json(delta_path)
|
||||
if data.get("schema") != EXPECTED_SCHEMA:
|
||||
findings.append(issue("error", "invalid_delta_schema", "Delta schema is invalid.", path=delta_path, detail={"schema": data.get("schema")}))
|
||||
|
||||
safety = data.get("safety") or {}
|
||||
if safety.get("read_only") is not True:
|
||||
findings.append(issue("error", "delta_not_read_only", "Delta safety.read_only must be true."))
|
||||
if safety.get("sql_write_performed") is not False:
|
||||
findings.append(issue("error", "delta_sql_write_flag", "Delta safety.sql_write_performed must be false."))
|
||||
if safety.get("public_terms_are_1c_objects") is not True:
|
||||
findings.append(issue("error", "delta_public_terms_flag", "Delta must expose public terms as 1C objects."))
|
||||
|
||||
for key in ("before_report", "after_report"):
|
||||
path = linked_path(delta_path, data.get(key))
|
||||
if path is None or not path.exists():
|
||||
findings.append(issue("warning", f"missing_{key}", f"Linked {key} is missing.", path=path or "<null>"))
|
||||
markdown_path = linked_path(delta_path, data.get("markdown"))
|
||||
if data.get("markdown") is not None and (markdown_path is None or not markdown_path.exists()):
|
||||
findings.append(issue("error", "missing_markdown", "Linked Markdown delta report is missing.", path=markdown_path or "<null>"))
|
||||
|
||||
objects = data.get("objects")
|
||||
if not isinstance(objects, dict):
|
||||
findings.append(issue("error", "invalid_objects", "Delta objects must be an object."))
|
||||
objects = {}
|
||||
for key in ("added", "removed", "changed", "unchanged"):
|
||||
if not isinstance(objects.get(key), list):
|
||||
findings.append(issue("error", f"invalid_objects_{key}", f"objects.{key} must be an array."))
|
||||
for item in objects.get("added") or []:
|
||||
if isinstance(item, dict):
|
||||
check_compact_object(item, findings, context="objects.added")
|
||||
for item in objects.get("removed") or []:
|
||||
if isinstance(item, dict):
|
||||
check_compact_object(item, findings, context="objects.removed")
|
||||
for item in objects.get("changed") or []:
|
||||
if isinstance(item, dict):
|
||||
check_changed_object(item, findings)
|
||||
|
||||
system = data.get("system_changes")
|
||||
if not isinstance(system, dict):
|
||||
findings.append(issue("error", "invalid_system_changes", "Delta system_changes must be an object."))
|
||||
system = {}
|
||||
for key in ("added", "removed", "changed"):
|
||||
if not isinstance(system.get(key), list):
|
||||
findings.append(issue("error", f"invalid_system_changes_{key}", f"system_changes.{key} must be an array."))
|
||||
|
||||
counts = data.get("counts") or {}
|
||||
expected_counts = {
|
||||
"objects_added": len(objects.get("added") or []),
|
||||
"objects_removed": len(objects.get("removed") or []),
|
||||
"objects_changed": len(objects.get("changed") or []),
|
||||
"objects_unchanged": len(objects.get("unchanged") or []),
|
||||
"system_added": len(system.get("added") or []),
|
||||
"system_removed": len(system.get("removed") or []),
|
||||
"system_changed": len(system.get("changed") or []),
|
||||
}
|
||||
for key, expected in expected_counts.items():
|
||||
if counts.get(key) != expected:
|
||||
findings.append(issue("error", "delta_count_mismatch", "Delta count does not match payload.", detail={"key": key, "reported": counts.get(key), "expected": expected}))
|
||||
|
||||
return build_result(delta_path, findings)
|
||||
|
||||
|
||||
def build_result(delta_path: Path, findings: list[dict[str, Any]]) -> dict[str, Any]:
|
||||
errors = [row for row in findings if row.get("severity") == "error"]
|
||||
warnings = [row for row in findings if row.get("severity") == "warning"]
|
||||
return {
|
||||
"schema": "onec_saved_state_object_report_delta_check.v1",
|
||||
"delta": str(delta_path),
|
||||
"passed": not errors,
|
||||
"findings": findings,
|
||||
"counts": {
|
||||
"errors": len(errors),
|
||||
"warnings": len(warnings),
|
||||
},
|
||||
"safety": {
|
||||
"read_only": True,
|
||||
"sql_write_performed": False,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Validate a 1C saved-state report delta.")
|
||||
parser.add_argument("--delta", type=Path, required=True)
|
||||
parser.add_argument("--output", type=Path)
|
||||
args = parser.parse_args()
|
||||
|
||||
result = check_delta(args.delta)
|
||||
if args.output:
|
||||
write_json(args.output, result)
|
||||
print(json.dumps({"output": str(args.output) if args.output else None, "passed": result["passed"], "counts": result["counts"]}, ensure_ascii=False))
|
||||
return 0 if result["passed"] else 2
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,178 @@
|
||||
#!/usr/bin/env python3
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import sys
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
DEFAULT_BASE_URL = "http://docker-gpu.cin.su:8011"
|
||||
DEFAULT_TABLES = ("ConfigCASSave", "ConfigSave")
|
||||
ALLOWED_TABLES = {"ConfigCASSave", "ConfigSave"}
|
||||
|
||||
|
||||
def request_json(method: str, url: str, *, payload: dict[str, Any] | None, timeout: float) -> tuple[int, dict[str, Any] | str]:
|
||||
data = json.dumps(payload, ensure_ascii=False).encode("utf-8") if payload is not None else None
|
||||
request = urllib.request.Request(
|
||||
url,
|
||||
data=data,
|
||||
method=method,
|
||||
headers={"Content-Type": "application/json; charset=utf-8"},
|
||||
)
|
||||
with urllib.request.urlopen(request, timeout=timeout) as response:
|
||||
raw = response.read().decode("utf-8")
|
||||
return response.status, json.loads(raw) if raw else {}
|
||||
|
||||
|
||||
def rpc(base_url: str, method: str, payload: dict[str, Any], timeout: float) -> dict[str, Any]:
|
||||
status, response = request_json(
|
||||
"POST",
|
||||
base_url.rstrip("/") + "/rpc",
|
||||
payload={"method": method, "payload": payload},
|
||||
timeout=timeout,
|
||||
)
|
||||
if status != 200 or not isinstance(response, dict):
|
||||
raise RuntimeError(f"{method} failed: status={status}, response={response!r}")
|
||||
return response
|
||||
|
||||
|
||||
def health_summary(base_url: str, base_id: str, timeout: float) -> dict[str, Any]:
|
||||
status, response = request_json("GET", f"{base_url.rstrip('/')}/health?base_id={base_id}", payload=None, timeout=timeout)
|
||||
if status != 200 or not isinstance(response, dict):
|
||||
return {"status": "error", "http_status": status, "response": response}
|
||||
live_sql = response.get("live_sql") if isinstance(response.get("live_sql"), dict) else {}
|
||||
return {
|
||||
"status": response.get("status"),
|
||||
"contract_version": response.get("contract_version"),
|
||||
"live_sql": {
|
||||
"configured": live_sql.get("configured"),
|
||||
"server": live_sql.get("server"),
|
||||
"database": live_sql.get("database"),
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def saved_state_row_counts(base_url: str, base_id: str, tables: list[str], timeout: float) -> dict[str, Any]:
|
||||
selects = [f"SELECT '{table}' AS TableName, COUNT(*) AS RowsCount FROM {table}" for table in tables]
|
||||
result = rpc(
|
||||
base_url,
|
||||
"query.run",
|
||||
{
|
||||
"base_id": base_id,
|
||||
"diagnostic": True,
|
||||
"query": "\nUNION ALL\n".join(selects),
|
||||
"timeout_seconds": int(timeout),
|
||||
},
|
||||
timeout,
|
||||
)
|
||||
counts: dict[str, int] = {table: 0 for table in tables}
|
||||
for row in result.get("rows") or []:
|
||||
if not isinstance(row, dict):
|
||||
continue
|
||||
table = str(row.get("TableName") or "")
|
||||
if table in counts and isinstance(row.get("RowsCount"), int):
|
||||
counts[table] = int(row["RowsCount"])
|
||||
return {
|
||||
"status": result.get("status"),
|
||||
"validation": result.get("validation"),
|
||||
"counts": counts,
|
||||
"raw_counts": result.get("counts"),
|
||||
}
|
||||
|
||||
|
||||
def search_saved_state(base_url: str, base_id: str, tables: list[str], timeout: float) -> dict[str, Any]:
|
||||
common = {"base_id": base_id, "tables": tables, "limit": 3, "timeout_seconds": int(timeout)}
|
||||
forms = rpc(base_url, "metadata.saved_state.forms.search", common, timeout)
|
||||
modules = rpc(base_url, "metadata.saved_state.modules.search", {**common, "scan_limit": 100}, timeout)
|
||||
return {
|
||||
"forms": {
|
||||
"status": forms.get("status"),
|
||||
"counts": forms.get("counts"),
|
||||
"sample": forms.get("forms") or [],
|
||||
},
|
||||
"modules": {
|
||||
"status": modules.get("status"),
|
||||
"counts": modules.get("counts"),
|
||||
"sample": modules.get("modules") or [],
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def build_report(base_url: str, base_id: str, tables: list[str], timeout: float, saved_state_table: str | None) -> dict[str, Any]:
|
||||
report: dict[str, Any] = {
|
||||
"schema": "onec_saved_state_strict_readiness.v1",
|
||||
"base_url": base_url,
|
||||
"base_id": base_id,
|
||||
"saved_state_table": saved_state_table,
|
||||
"tables": tables,
|
||||
"ready": False,
|
||||
"status": "error",
|
||||
"checks": {},
|
||||
"recommendations": [],
|
||||
}
|
||||
report["checks"]["health"] = health_summary(base_url, base_id, timeout)
|
||||
report["checks"]["row_counts"] = saved_state_row_counts(base_url, base_id, tables, timeout)
|
||||
report["checks"]["saved_state_search"] = search_saved_state(base_url, base_id, tables, timeout)
|
||||
|
||||
row_counts = (report["checks"]["row_counts"] or {}).get("counts") or {}
|
||||
forms_counts = (((report["checks"]["saved_state_search"] or {}).get("forms") or {}).get("counts") or {})
|
||||
modules_counts = (((report["checks"]["saved_state_search"] or {}).get("modules") or {}).get("counts") or {})
|
||||
total_rows = sum(value for value in row_counts.values() if isinstance(value, int))
|
||||
forms = forms_counts.get("forms") if isinstance(forms_counts.get("forms"), int) else 0
|
||||
modules = modules_counts.get("modules") if isinstance(modules_counts.get("modules"), int) else 0
|
||||
|
||||
report["summary"] = {
|
||||
"saved_state_rows": total_rows,
|
||||
"forms": forms,
|
||||
"modules": modules,
|
||||
}
|
||||
if forms > 0 and modules > 0:
|
||||
report["ready"] = True
|
||||
report["status"] = "ready"
|
||||
report["recommendations"].append("Strict saved-state smoke can be attempted: form and module saved-state candidates are present in the selected save layer.")
|
||||
elif total_rows == 0:
|
||||
report["status"] = "blocked_no_saved_state_rows"
|
||||
report["recommendations"].append(
|
||||
f"No unactivated Configurator changes are present in {', '.join(tables)}. To prepare a strict write test, copy the target object from Config/ConfigCAS into the selected save layer using the approved saved-state workflow, then rerun readiness."
|
||||
)
|
||||
else:
|
||||
report["status"] = "blocked_no_strict_candidates"
|
||||
report["recommendations"].append(
|
||||
"Saved-state rows exist, but the adapter did not find both form and module candidates needed by strict write-and-rollback smoke."
|
||||
)
|
||||
return report
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Read-only readiness check for strict 1C saved-state smoke tests.")
|
||||
parser.add_argument("--base-url", default=DEFAULT_BASE_URL)
|
||||
parser.add_argument("--base-id", default="upo_test")
|
||||
parser.add_argument("--table", action="append", choices=sorted(ALLOWED_TABLES), help="Saved-state table to inspect. Repeatable.")
|
||||
parser.add_argument("--saved-state-table", choices=sorted(ALLOWED_TABLES), help="Expected saved-state table for the strict smoke target.")
|
||||
parser.add_argument("--timeout", type=float, default=30.0)
|
||||
parser.add_argument("--report", type=Path)
|
||||
parser.add_argument("--require-ready", action="store_true", help="Exit non-zero when strict saved-state smoke is not ready.")
|
||||
parser.add_argument("--json", action="store_true", help="Print full JSON report.")
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.saved_state_table and args.table and any(table != args.saved_state_table for table in args.table):
|
||||
parser.error("--saved-state-table must match --table when both are provided")
|
||||
tables = args.table or ([args.saved_state_table] if args.saved_state_table else list(DEFAULT_TABLES))
|
||||
report = build_report(args.base_url, args.base_id, tables, args.timeout, args.saved_state_table)
|
||||
if args.report:
|
||||
args.report.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.report.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
|
||||
if args.json or args.require_ready or not report["ready"]:
|
||||
print(json.dumps(report, ensure_ascii=True, indent=2), file=sys.stderr if args.require_ready and not report["ready"] else sys.stdout)
|
||||
else:
|
||||
print(f"OK: strict saved-state smoke readiness passed for {args.base_id}.")
|
||||
return 0 if report["ready"] or not args.require_ready else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
|
||||
@@ -0,0 +1,138 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Validate a one-shot 1C saved-state watch manifest."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
EXPECTED_SCHEMA = "onec_saved_state_watch_once.v1"
|
||||
|
||||
|
||||
def load_json(path: Path) -> dict[str, Any]:
|
||||
return json.loads(path.read_text(encoding="utf-8-sig"))
|
||||
|
||||
|
||||
def write_json(path: Path, data: dict[str, Any]) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(json.dumps(data, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
|
||||
|
||||
def issue(severity: str, code: str, message: str, *, path: Path | str | None = None) -> dict[str, Any]:
|
||||
result = {"severity": severity, "code": code, "message": message}
|
||||
if path is not None:
|
||||
result["path"] = str(path)
|
||||
return result
|
||||
|
||||
|
||||
def linked_path(manifest_path: Path, value: Any) -> Path | None:
|
||||
if not value:
|
||||
return None
|
||||
path = Path(str(value))
|
||||
if path.is_absolute():
|
||||
return path
|
||||
return (manifest_path.parent / path).resolve()
|
||||
|
||||
|
||||
def check_existing(manifest_path: Path, data: dict[str, Any], key: str, findings: list[dict[str, Any]], *, required: bool = True) -> Path | None:
|
||||
path = linked_path(manifest_path, data.get(key))
|
||||
if path is None:
|
||||
if required:
|
||||
findings.append(issue("error", f"missing_{key}", f"Manifest field {key} is missing."))
|
||||
return None
|
||||
if not path.exists():
|
||||
findings.append(issue("error" if required else "warning", f"missing_{key}_file", f"Linked {key} file is missing.", path=path))
|
||||
return path
|
||||
|
||||
|
||||
def check_manifest(manifest_path: Path) -> dict[str, Any]:
|
||||
findings: list[dict[str, Any]] = []
|
||||
if not manifest_path.exists():
|
||||
findings.append(issue("error", "missing_manifest", "Watch manifest is missing.", path=manifest_path))
|
||||
return build_result(manifest_path, findings)
|
||||
data = load_json(manifest_path)
|
||||
if data.get("schema") != EXPECTED_SCHEMA:
|
||||
findings.append(issue("error", "invalid_schema", "Watch manifest schema is invalid."))
|
||||
|
||||
safety = data.get("safety") or {}
|
||||
if safety.get("read_only") is not True:
|
||||
findings.append(issue("error", "not_read_only", "Watch manifest safety.read_only must be true."))
|
||||
if safety.get("sql_write_performed") is not False:
|
||||
findings.append(issue("error", "sql_write_flag", "Watch manifest safety.sql_write_performed must be false."))
|
||||
if safety.get("secrets_in_report") is not False:
|
||||
findings.append(issue("error", "secrets_flag", "Watch manifest safety.secrets_in_report must be false."))
|
||||
|
||||
report_path = check_existing(manifest_path, data, "report", findings)
|
||||
check_existing(manifest_path, data, "markdown", findings, required=False)
|
||||
report_check_path = check_existing(manifest_path, data, "check", findings)
|
||||
check_existing(manifest_path, data, "manifest_markdown", findings, required=False)
|
||||
delta_path = check_existing(manifest_path, data, "delta", findings, required=False)
|
||||
check_existing(manifest_path, data, "delta_markdown", findings, required=False)
|
||||
delta_check_path = check_existing(manifest_path, data, "delta_check", findings, required=False)
|
||||
|
||||
if report_path and report_path.exists():
|
||||
report = load_json(report_path)
|
||||
counts = data.get("counts") or {}
|
||||
report_counts = report.get("counts") or {}
|
||||
if counts.get("object_changes") != report_counts.get("object_changes"):
|
||||
findings.append(issue("error", "object_count_mismatch", "Watch object_changes count differs from report."))
|
||||
if counts.get("system_changes") != report_counts.get("system_changes"):
|
||||
findings.append(issue("error", "system_count_mismatch", "Watch system_changes count differs from report."))
|
||||
|
||||
if report_check_path and report_check_path.exists():
|
||||
report_check = load_json(report_check_path)
|
||||
if report_check.get("passed") is not True:
|
||||
findings.append(issue("error", "report_check_failed", "Linked saved-state report check did not pass.", path=report_check_path))
|
||||
|
||||
if delta_path and delta_path.exists():
|
||||
delta = load_json(delta_path)
|
||||
counts = data.get("counts") or {}
|
||||
delta_counts = delta.get("counts") or {}
|
||||
mapping = {
|
||||
"delta_objects_added": "objects_added",
|
||||
"delta_objects_removed": "objects_removed",
|
||||
"delta_objects_changed": "objects_changed",
|
||||
"delta_objects_unchanged": "objects_unchanged",
|
||||
}
|
||||
for watch_key, delta_key in mapping.items():
|
||||
if counts.get(watch_key) != delta_counts.get(delta_key):
|
||||
findings.append(issue("error", "delta_count_mismatch", f"Watch {watch_key} differs from delta {delta_key}."))
|
||||
if delta_check_path and delta_check_path.exists():
|
||||
delta_check = load_json(delta_check_path)
|
||||
if delta_check.get("passed") is not True:
|
||||
findings.append(issue("error", "delta_check_failed", "Linked delta check did not pass.", path=delta_check_path))
|
||||
|
||||
return build_result(manifest_path, findings)
|
||||
|
||||
|
||||
def build_result(manifest_path: Path, findings: list[dict[str, Any]]) -> dict[str, Any]:
|
||||
errors = [row for row in findings if row.get("severity") == "error"]
|
||||
warnings = [row for row in findings if row.get("severity") == "warning"]
|
||||
return {
|
||||
"schema": "onec_saved_state_watch_once_check.v1",
|
||||
"manifest": str(manifest_path),
|
||||
"passed": not errors,
|
||||
"findings": findings,
|
||||
"counts": {"errors": len(errors), "warnings": len(warnings)},
|
||||
"safety": {"read_only": True, "sql_write_performed": False},
|
||||
}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Validate a one-shot 1C saved-state watch manifest.")
|
||||
parser.add_argument("--manifest", type=Path, required=True)
|
||||
parser.add_argument("--output", type=Path)
|
||||
args = parser.parse_args()
|
||||
|
||||
result = check_manifest(args.manifest)
|
||||
if args.output:
|
||||
write_json(args.output, result)
|
||||
print(json.dumps({"output": str(args.output) if args.output else None, "passed": result["passed"], "counts": result["counts"]}, ensure_ascii=False))
|
||||
return 0 if result["passed"] else 2
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,154 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Validate a 1C saved-state watch run list contract."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
EXPECTED_SCHEMA = "onec_saved_state_watch_run_list.v1"
|
||||
|
||||
|
||||
def load_json(path: Path) -> dict[str, Any]:
|
||||
return json.loads(path.read_text(encoding="utf-8-sig"))
|
||||
|
||||
|
||||
def write_json(path: Path, data: dict[str, Any]) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(json.dumps(data, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
|
||||
|
||||
def issue(severity: str, code: str, message: str, *, path: Path | str | None = None, detail: dict[str, Any] | None = None) -> dict[str, Any]:
|
||||
result: dict[str, Any] = {"severity": severity, "code": code, "message": message}
|
||||
if path is not None:
|
||||
result["path"] = str(path)
|
||||
if detail:
|
||||
result["detail"] = detail
|
||||
return result
|
||||
|
||||
|
||||
def linked_path(list_path: Path, value: Any) -> Path | None:
|
||||
if not value:
|
||||
return None
|
||||
path = Path(str(value))
|
||||
if path.is_absolute():
|
||||
return path
|
||||
return (list_path.parent / path).resolve()
|
||||
|
||||
|
||||
def has_delta_changes(run: dict[str, Any]) -> bool:
|
||||
counts = run.get("counts") or {}
|
||||
return any(int(counts.get(key) or 0) > 0 for key in ("delta_objects_added", "delta_objects_removed", "delta_objects_changed"))
|
||||
|
||||
|
||||
def check_run(list_path: Path, run: dict[str, Any], findings: list[dict[str, Any]], *, index: int) -> None:
|
||||
if not run.get("run"):
|
||||
findings.append(issue("error", "missing_run_name", "Run item is missing run name.", detail={"index": index}))
|
||||
run_dir = linked_path(list_path, run.get("run_dir"))
|
||||
if run_dir is None or not run_dir.exists():
|
||||
findings.append(issue("error", "missing_run_dir", "Run directory is missing.", path=run_dir or "<null>", detail={"index": index}))
|
||||
|
||||
manifest_check = linked_path(list_path, run.get("manifest_check"))
|
||||
if manifest_check is not None:
|
||||
if not manifest_check.exists():
|
||||
findings.append(issue("error", "missing_manifest_check", "Run manifest_check file is missing.", path=manifest_check))
|
||||
elif run.get("manifest_check_passed") != load_json(manifest_check).get("passed"):
|
||||
findings.append(issue("error", "manifest_check_status_mismatch", "Run manifest_check_passed differs from linked check.", path=manifest_check))
|
||||
|
||||
delta = linked_path(list_path, run.get("delta"))
|
||||
if run.get("delta") and (delta is None or not delta.exists()):
|
||||
findings.append(issue("error", "missing_delta", "Run delta file is missing.", path=delta or "<null>"))
|
||||
delta_check = linked_path(list_path, run.get("delta_check"))
|
||||
if delta_check is not None:
|
||||
if not delta_check.exists():
|
||||
findings.append(issue("error", "missing_delta_check", "Run delta_check file is missing.", path=delta_check))
|
||||
elif run.get("delta_check_passed") != load_json(delta_check).get("passed"):
|
||||
findings.append(issue("error", "delta_check_status_mismatch", "Run delta_check_passed differs from linked check.", path=delta_check))
|
||||
|
||||
counts = run.get("counts")
|
||||
if not isinstance(counts, dict):
|
||||
findings.append(issue("error", "missing_run_counts", "Run item must include counts.", detail={"run": run.get("run")}))
|
||||
|
||||
|
||||
def check_list(list_path: Path) -> dict[str, Any]:
|
||||
findings: list[dict[str, Any]] = []
|
||||
if not list_path.exists():
|
||||
findings.append(issue("error", "missing_list", "Watch run list file is missing.", path=list_path))
|
||||
return build_result(list_path, findings)
|
||||
data = load_json(list_path)
|
||||
if data.get("schema") != EXPECTED_SCHEMA:
|
||||
findings.append(issue("error", "invalid_schema", "Watch run list schema is invalid.", detail={"schema": data.get("schema")}))
|
||||
|
||||
safety = data.get("safety") or {}
|
||||
if safety.get("read_only") is not True:
|
||||
findings.append(issue("error", "not_read_only", "Watch run list safety.read_only must be true."))
|
||||
if safety.get("sql_write_performed") is not False:
|
||||
findings.append(issue("error", "sql_write_flag", "Watch run list safety.sql_write_performed must be false."))
|
||||
|
||||
markdown = linked_path(list_path, data.get("markdown"))
|
||||
if data.get("markdown") and (markdown is None or not markdown.exists()):
|
||||
findings.append(issue("error", "missing_markdown", "Linked Markdown run-list report is missing.", path=markdown or "<null>"))
|
||||
|
||||
runs = data.get("runs")
|
||||
if not isinstance(runs, list):
|
||||
findings.append(issue("error", "invalid_runs", "runs must be an array."))
|
||||
runs = []
|
||||
for index, run in enumerate(runs):
|
||||
if isinstance(run, dict):
|
||||
check_run(list_path, run, findings, index=index)
|
||||
else:
|
||||
findings.append(issue("error", "invalid_run_item", "Run item must be an object.", detail={"index": index}))
|
||||
|
||||
latest = data.get("latest")
|
||||
if runs:
|
||||
if not isinstance(latest, dict):
|
||||
findings.append(issue("error", "missing_latest", "latest must be present when runs are present."))
|
||||
elif latest.get("run") != runs[0].get("run"):
|
||||
findings.append(issue("error", "latest_mismatch", "latest must match the first run item.", detail={"latest": latest.get("run"), "first": runs[0].get("run")}))
|
||||
elif latest is not None:
|
||||
findings.append(issue("error", "unexpected_latest", "latest must be null when no runs are present."))
|
||||
|
||||
counts = data.get("counts") or {}
|
||||
expected = {
|
||||
"runs": len(runs),
|
||||
"with_delta": sum(1 for run in runs if isinstance(run, dict) and run.get("delta")),
|
||||
"with_delta_changes": sum(1 for run in runs if isinstance(run, dict) and has_delta_changes(run)),
|
||||
}
|
||||
for key, value in expected.items():
|
||||
if counts.get(key) != value:
|
||||
findings.append(issue("error", "count_mismatch", "Watch run list count mismatch.", detail={"key": key, "reported": counts.get(key), "expected": value}))
|
||||
|
||||
return build_result(list_path, findings)
|
||||
|
||||
|
||||
def build_result(list_path: Path, findings: list[dict[str, Any]]) -> dict[str, Any]:
|
||||
errors = [row for row in findings if row.get("severity") == "error"]
|
||||
warnings = [row for row in findings if row.get("severity") == "warning"]
|
||||
return {
|
||||
"schema": "onec_saved_state_watch_run_list_check.v1",
|
||||
"list": str(list_path),
|
||||
"passed": not errors,
|
||||
"findings": findings,
|
||||
"counts": {"errors": len(errors), "warnings": len(warnings)},
|
||||
"safety": {"read_only": True, "sql_write_performed": False},
|
||||
}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Validate a 1C saved-state watch run list.")
|
||||
parser.add_argument("--list", type=Path, required=True)
|
||||
parser.add_argument("--output", type=Path)
|
||||
args = parser.parse_args()
|
||||
|
||||
result = check_list(args.list)
|
||||
if args.output:
|
||||
write_json(args.output, result)
|
||||
print(json.dumps({"output": str(args.output) if args.output else None, "passed": result["passed"], "counts": result["counts"]}, ensure_ascii=False))
|
||||
return 0 if result["passed"] else 2
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,379 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT))
|
||||
sys.path.insert(0, str(ROOT / "plugins" / "1c"))
|
||||
sys.path.insert(0, str(ROOT / "plugins" / "1c" / "connector"))
|
||||
|
||||
import adapter_1c_server as adapter_server # noqa: E402
|
||||
|
||||
|
||||
def problem_codes(result: dict[str, Any]) -> set[str]:
|
||||
return {str(problem.get("code") or "") for problem in result.get("problems") or [] if isinstance(problem, dict)}
|
||||
|
||||
|
||||
def require(condition: bool, message: str, failures: list[str]) -> None:
|
||||
if not condition:
|
||||
failures.append(message)
|
||||
|
||||
|
||||
def check_blocked_effective_form_path(failures: list[str]) -> None:
|
||||
result = adapter_server.metadata_write(
|
||||
{
|
||||
"base_id": "upo_test",
|
||||
"target": {"canonical_path": "Документ.Заказ.Форма.ФормаДокумента.КнопкаЗаписать"},
|
||||
"mode": "plan",
|
||||
"edits": [{"property": "Заголовок", "value": "Записать"}],
|
||||
}
|
||||
)
|
||||
require(result.get("status") == "blocked", "effective form path must be blocked without concrete route", failures)
|
||||
require(result.get("error") == "write_plan_required", "effective form path must return write_plan_required", failures)
|
||||
require((result.get("next_resolution") or {}).get("method") == adapter_server.FORM_WRITE_TARGET_RESOLVE_METHOD, "form path must expose form write target resolver", failures)
|
||||
hint = result.get("apply_payload_hint") if isinstance(result.get("apply_payload_hint"), dict) else {}
|
||||
require(hint.get("ready_for_apply_method") is False, "selector-only form hint must not be ready for apply", failures)
|
||||
|
||||
|
||||
def check_module_control_guard(failures: list[str]) -> None:
|
||||
result = adapter_server.metadata_write(
|
||||
{
|
||||
"base_id": "upo_test",
|
||||
"target": {
|
||||
"canonical_path": "ОбщийМодуль.Интеграция.Отправить",
|
||||
"module_ref": "ConfigCASSave:common_module.0#stream:0",
|
||||
},
|
||||
"intent": {"operation": "replace_with_control", "new": "Сообщить(\"new\");"},
|
||||
}
|
||||
)
|
||||
require(result.get("status") == "blocked", "replace_with_control without control fragment must be blocked", failures)
|
||||
require(result.get("error") == "write_plan_blocked", "blocked control guard must return write_plan_blocked", failures)
|
||||
require("missing_control_fragment" in problem_codes(result), "blocked control guard must expose missing_control_fragment", failures)
|
||||
|
||||
allowed = adapter_server.metadata_write_plan(
|
||||
{
|
||||
"base_id": "upo_test",
|
||||
"target": {
|
||||
"kind": "module",
|
||||
"module_ref": "ConfigCASSave:common_module.0#stream:0",
|
||||
},
|
||||
"intent": {"operation": "replace_with_control", "expected_old_contains": "old", "new": "new", "current_text": "prefix old suffix"},
|
||||
}
|
||||
)
|
||||
require(allowed.get("allowed") is True, "replace_with_control with expected_old_contains must be allowed on concrete route", failures)
|
||||
|
||||
drift = adapter_server.metadata_write_plan(
|
||||
{
|
||||
"base_id": "upo_test",
|
||||
"target": {
|
||||
"kind": "module",
|
||||
"module_ref": "ConfigCASSave:common_module.0#stream:0",
|
||||
},
|
||||
"intent": {"operation": "replace_with_control", "control_fragment": "old", "new": "new", "current_text": "prefix changed suffix"},
|
||||
}
|
||||
)
|
||||
require(drift.get("allowed") is False, "replace_with_control drift must be blocked when current_text is provided", failures)
|
||||
require("control_fragment_drift" in problem_codes(drift), "replace_with_control drift must expose control_fragment_drift", failures)
|
||||
|
||||
|
||||
def check_reference_identity(failures: list[str]) -> None:
|
||||
module_plan = adapter_server.metadata_write_plan(
|
||||
{
|
||||
"base_id": "upo_test",
|
||||
"target": {"kind": "module", "file_name": "object-guid__module-guid.0", "stream_index": 4},
|
||||
"intent": {"operation": "replace", "old": "a", "new": "b"},
|
||||
}
|
||||
)
|
||||
module_hint = ((module_plan.get("route") or {}).get("apply_payload_hint") or {}).get("payload") or {}
|
||||
require(module_plan.get("allowed") is True, "module file_name concrete route must be allowed", failures)
|
||||
require((module_plan.get("target") or {}).get("concrete_reference_field") == "file_name", "module file_name route must preserve concrete_reference_field", failures)
|
||||
require(module_hint.get("file_name") == "object-guid__module-guid.0", "module hint must preserve file_name", failures)
|
||||
require("module_ref" not in module_hint, "module file_name hint must not invent module_ref", failures)
|
||||
|
||||
form_plan = adapter_server.metadata_write_plan(
|
||||
{
|
||||
"base_id": "upo_test",
|
||||
"target": {"kind": "form", "form_guid": "form-guid"},
|
||||
"intent": {"operation": "property_change", "property": "Заголовок", "value": "Новый"},
|
||||
}
|
||||
)
|
||||
form_hint = ((form_plan.get("route") or {}).get("apply_payload_hint") or {}).get("payload") or {}
|
||||
require(form_plan.get("allowed") is True, "form_guid concrete route must be allowed", failures)
|
||||
require((form_plan.get("target") or {}).get("concrete_reference_field") == "form_guid", "form_guid route must preserve concrete_reference_field", failures)
|
||||
require(form_hint.get("form_guid") == "form-guid", "form hint must preserve form_guid", failures)
|
||||
require("file_name" not in form_hint, "form_guid hint must not invent file_name", failures)
|
||||
|
||||
|
||||
def check_reference_mismatch(failures: list[str]) -> None:
|
||||
plan = adapter_server.metadata_write_plan(
|
||||
{
|
||||
"base_id": "upo_test",
|
||||
"target": {"kind": "module", "form_guid": "form-guid"},
|
||||
"intent": {"operation": "replace", "old": "a", "new": "b"},
|
||||
}
|
||||
)
|
||||
require(plan.get("allowed") is False, "module target with form_guid must be blocked", failures)
|
||||
require("concrete_reference_kind_mismatch" in problem_codes(plan), "module target with form_guid must expose concrete_reference_kind_mismatch", failures)
|
||||
|
||||
|
||||
def check_provided_origin_evidence(failures: list[str]) -> None:
|
||||
plan = adapter_server.metadata_write_plan(
|
||||
{
|
||||
"base_id": "upo_test",
|
||||
"target": {
|
||||
"canonical_path": "ОбщийМодуль.Интеграция.Отправить",
|
||||
"origin": {"source": "configuration", "presentation": "Конфигурация", "status": "ok"},
|
||||
},
|
||||
"intent": {"operation": "replace", "old": "a", "new": "b"},
|
||||
}
|
||||
)
|
||||
recommended = (plan.get("route") or {}).get("recommended_write") or {}
|
||||
require((plan.get("origin_lookup") or {}).get("method") == "provided_origin_evidence", "planner must accept provided origin evidence", failures)
|
||||
require(recommended.get("write_surface") == "base_saved_state", "provided configuration origin must recommend base_saved_state", failures)
|
||||
require("write_route_required" in problem_codes(plan), "provided origin must still require concrete write route", failures)
|
||||
|
||||
cas_plan = adapter_server.metadata_write_plan(
|
||||
{
|
||||
"base_id": "upo_test",
|
||||
"target": {
|
||||
"canonical_path": "ОбщийМодуль.Интеграция.Отправить",
|
||||
"origin": {"source": "cas_reference", "status": "owner_unresolved", "write_surface": "requires_owner_resolution"},
|
||||
},
|
||||
"intent": {"operation": "replace", "old": "a", "new": "b"},
|
||||
}
|
||||
)
|
||||
cas_recommended = (cas_plan.get("route") or {}).get("recommended_write") or {}
|
||||
require(cas_recommended.get("write_surface") == "blocked_unknown", "unresolved CAS origin must be blocked_unknown", failures)
|
||||
require("blocked_unknown" in problem_codes(cas_plan), "unresolved CAS origin must expose blocked_unknown", failures)
|
||||
|
||||
|
||||
def check_origin_ambiguity(failures: list[str]) -> None:
|
||||
original_definition_find = adapter_server.metadata_definition_find
|
||||
|
||||
def fake_definition_find(payload: dict[str, Any]) -> dict[str, Any]:
|
||||
return {
|
||||
"schema": "onec_metadata_definition_find.v1",
|
||||
"status": "ok",
|
||||
"matches": [
|
||||
{
|
||||
"area": "object",
|
||||
"kind": "Document",
|
||||
"name": "Заказ",
|
||||
"match_by": "synthetic_contract",
|
||||
"location": {"presentation": "Документ.Заказ.Реквизит.КнопкаЗаписать"},
|
||||
"origin": {"source": "configuration", "status": "ok"},
|
||||
},
|
||||
{
|
||||
"area": "form",
|
||||
"kind": "Document",
|
||||
"name": "Заказ",
|
||||
"match_by": "synthetic_contract",
|
||||
"location": {"presentation": "Документ.Заказ.Форма.ФормаДокумента.КнопкаЗаписать"},
|
||||
"origin": {"source": "configuration", "status": "ok"},
|
||||
},
|
||||
],
|
||||
"counts": {"matches": 2},
|
||||
}
|
||||
|
||||
adapter_server.metadata_definition_find = fake_definition_find
|
||||
try:
|
||||
plan = adapter_server.metadata_write_plan(
|
||||
{
|
||||
"base_id": "upo_test",
|
||||
"target": {"canonical_path": "Документ.Заказ.Форма.ФормаДокумента.КнопкаЗаписать"},
|
||||
"intent": {"operation": "property_change", "property": "Заголовок", "value": "Записать"},
|
||||
}
|
||||
)
|
||||
finally:
|
||||
adapter_server.metadata_definition_find = original_definition_find
|
||||
|
||||
require(plan.get("allowed") is False, "ambiguous origin plan must not be allowed", failures)
|
||||
require("ambiguous_origin_matches" in problem_codes(plan), "ambiguous origin plan must expose ambiguous_origin_matches", failures)
|
||||
ambiguity = next((problem for problem in plan.get("problems") or [] if isinstance(problem, dict) and problem.get("code") == "ambiguous_origin_matches"), {})
|
||||
require(ambiguity.get("match_count") == 2, "ambiguous origin plan must expose match_count", failures)
|
||||
|
||||
|
||||
def check_extension_action_evidence(failures: list[str]) -> None:
|
||||
inferred = adapter_server.metadata_write_plan(
|
||||
{
|
||||
"base_id": "upo_test",
|
||||
"target": {
|
||||
"kind": "module",
|
||||
"module_ref": "ConfigCASSave:common_module.0#stream:0",
|
||||
"extension_action": {"status": "ok", "operation_class": "replace_with_control"},
|
||||
},
|
||||
"intent": {"control_fragment": "old", "new": "new"},
|
||||
}
|
||||
)
|
||||
require(inferred.get("allowed") is True, "known extension action with guards must be allowed on concrete route", failures)
|
||||
require((inferred.get("route") or {}).get("operation_class") == "replace_with_control", "planner must infer operation from extension_action", failures)
|
||||
require((inferred.get("route") or {}).get("operation_inferred_from") == "extension_action", "planner must mark operation_inferred_from", failures)
|
||||
|
||||
unknown = adapter_server.metadata_write_plan(
|
||||
{
|
||||
"base_id": "upo_test",
|
||||
"target": {
|
||||
"kind": "module",
|
||||
"module_ref": "ConfigCASSave:common_module.0#stream:0",
|
||||
"extension_action": {"status": "unknown", "operation_class": "unknown_extension_action"},
|
||||
},
|
||||
"intent": {"new": "new"},
|
||||
}
|
||||
)
|
||||
require(unknown.get("allowed") is False, "unknown extension action must block module write planning", failures)
|
||||
require("extension_action_unknown" in problem_codes(unknown), "unknown extension action must expose extension_action_unknown", failures)
|
||||
|
||||
mismatch = adapter_server.metadata_write_plan(
|
||||
{
|
||||
"base_id": "upo_test",
|
||||
"target": {
|
||||
"kind": "module",
|
||||
"module_ref": "ConfigCASSave:common_module.0#stream:0",
|
||||
"extension_action": {"status": "ok", "operation_class": "insert_after"},
|
||||
},
|
||||
"intent": {"operation": "replace", "old": "old", "new": "new"},
|
||||
}
|
||||
)
|
||||
require(mismatch.get("allowed") is False, "operation mismatch with extension action must be blocked", failures)
|
||||
require("extension_action_operation_mismatch" in problem_codes(mismatch), "operation mismatch must expose extension_action_operation_mismatch", failures)
|
||||
|
||||
ambiguous = adapter_server.metadata_write_plan(
|
||||
{
|
||||
"base_id": "upo_test",
|
||||
"target": {"kind": "module", "module_ref": "ConfigCASSave:common_module.0#stream:0"},
|
||||
"extension_actions": [
|
||||
{"status": "ok", "operation_class": "insert_before"},
|
||||
{"status": "ok", "operation_class": "replace"},
|
||||
],
|
||||
"intent": {"operation": "replace", "old": "old", "new": "new"},
|
||||
}
|
||||
)
|
||||
require(ambiguous.get("allowed") is False, "multiple extension actions must block module write planning", failures)
|
||||
require("extension_action_ambiguous" in problem_codes(ambiguous), "multiple extension actions must expose extension_action_ambiguous", failures)
|
||||
|
||||
|
||||
def check_apply_methods_use_write_plan_gate(failures: list[str]) -> None:
|
||||
original_changes_propose = adapter_server.changes_propose
|
||||
original_apply = adapter_server.storage_saved_state_apply_proposal
|
||||
apply_calls: list[dict[str, Any]] = []
|
||||
|
||||
def fake_changes_propose(payload: dict[str, Any]) -> dict[str, Any]:
|
||||
source = payload.get("source") if isinstance(payload.get("source"), dict) else {}
|
||||
return {
|
||||
"schema": "onec_change_proposal.v1",
|
||||
"status": "accepted_for_review",
|
||||
"source": {
|
||||
"table": source.get("table") or "ConfigCASSave",
|
||||
"file_name": source.get("file_name") or "common_module.0",
|
||||
},
|
||||
"original": {"sha1": "0" * 40, "bytes": 10},
|
||||
"encoded": {"sha1": "1" * 40, "bytes": 11, "payload_hex": "00"},
|
||||
"validation": {"status": "ok"},
|
||||
}
|
||||
|
||||
def fake_apply(payload: dict[str, Any]) -> dict[str, Any]:
|
||||
apply_calls.append(payload)
|
||||
return {"schema": "onec_storage_saved_state_apply.v1", "status": "applied", "applied": True}
|
||||
|
||||
adapter_server.changes_propose = fake_changes_propose
|
||||
adapter_server.storage_saved_state_apply_proposal = fake_apply
|
||||
try:
|
||||
blocked = adapter_server.metadata_module_write_apply(
|
||||
{
|
||||
"base_id": "upo_test",
|
||||
"module_ref": "ConfigCASSave:common_module.0#stream:0",
|
||||
"mode": "apply",
|
||||
"allow_saved_state_write": True,
|
||||
"allow_sql_saved_state_apply": True,
|
||||
"operation": "replace_with_control",
|
||||
"old": "old",
|
||||
"new": "Сообщить(\"new\");",
|
||||
}
|
||||
)
|
||||
require(blocked.get("status") == "blocked", "module apply must be blocked when write_plan rejects guards", failures)
|
||||
require(blocked.get("error") == "write_plan_blocked", "module apply block must report write_plan_blocked", failures)
|
||||
require("missing_control_fragment" in problem_codes(blocked), "module apply block must expose missing_control_fragment", failures)
|
||||
require(not apply_calls, "blocked module apply must not call storage_saved_state_apply_proposal", failures)
|
||||
|
||||
drift_blocked = adapter_server.metadata_module_write_apply(
|
||||
{
|
||||
"base_id": "upo_test",
|
||||
"module_ref": "ConfigCASSave:common_module.0#stream:0",
|
||||
"mode": "apply",
|
||||
"allow_saved_state_write": True,
|
||||
"allow_sql_saved_state_apply": True,
|
||||
"operation": "replace_with_control",
|
||||
"control_fragment": "old",
|
||||
"old": "old",
|
||||
"current_text": "changed",
|
||||
"new": "Сообщить(\"new\");",
|
||||
}
|
||||
)
|
||||
require(drift_blocked.get("status") == "blocked", "module apply must be blocked when write_plan detects control drift", failures)
|
||||
require(drift_blocked.get("error") == "write_plan_blocked", "module apply drift block must report write_plan_blocked", failures)
|
||||
require("control_fragment_drift" in problem_codes(drift_blocked), "module apply drift block must expose control_fragment_drift", failures)
|
||||
require(not apply_calls, "drift-blocked module apply must not call storage_saved_state_apply_proposal", failures)
|
||||
|
||||
planned = adapter_server.metadata_module_write_apply(
|
||||
{
|
||||
"base_id": "upo_test",
|
||||
"module_ref": "ConfigCASSave:common_module.0#stream:0",
|
||||
"mode": "plan",
|
||||
"allow_saved_state_write": True,
|
||||
"old": "old",
|
||||
"new": "new",
|
||||
}
|
||||
)
|
||||
require(planned.get("status") == "planned", "module plan with concrete route must remain planned", failures)
|
||||
require((planned.get("write_plan") or {}).get("allowed") is True, "module plan must include an allowed write_plan", failures)
|
||||
finally:
|
||||
adapter_server.changes_propose = original_changes_propose
|
||||
adapter_server.storage_saved_state_apply_proposal = original_apply
|
||||
|
||||
|
||||
def run_checks() -> dict[str, Any]:
|
||||
failures: list[str] = []
|
||||
check_blocked_effective_form_path(failures)
|
||||
check_module_control_guard(failures)
|
||||
check_reference_identity(failures)
|
||||
check_reference_mismatch(failures)
|
||||
check_provided_origin_evidence(failures)
|
||||
check_origin_ambiguity(failures)
|
||||
check_extension_action_evidence(failures)
|
||||
check_apply_methods_use_write_plan_gate(failures)
|
||||
return {
|
||||
"schema": "onec_write_plan_contract_check.v1",
|
||||
"status": "ok" if not failures else "failed",
|
||||
"failures": failures,
|
||||
"checks": {
|
||||
"blocked_effective_form_path": True,
|
||||
"module_control_guard": True,
|
||||
"reference_identity": True,
|
||||
"reference_mismatch": True,
|
||||
"provided_origin_evidence": True,
|
||||
"origin_ambiguity": True,
|
||||
"extension_action_evidence": True,
|
||||
"apply_methods_use_write_plan_gate": True,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Check local 1C metadata write-plan contract invariants.")
|
||||
parser.add_argument("--print", action="store_true")
|
||||
args = parser.parse_args()
|
||||
report = run_checks()
|
||||
if args.print or report["status"] != "ok":
|
||||
print(json.dumps(report, ensure_ascii=False, indent=2))
|
||||
else:
|
||||
print("1C write-plan contract status: ok")
|
||||
return 0 if report["status"] == "ok" else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,81 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
GENERATED_PATHS = [
|
||||
ROOT / "plugins" / "1c" / "rag" / "sources" / "metadata.health.generated.md",
|
||||
]
|
||||
|
||||
|
||||
def run(command: list[str]) -> tuple[str, int]:
|
||||
label = " ".join(command)
|
||||
print(f"\n== {label}")
|
||||
result = subprocess.run(command, cwd=ROOT, text=True, check=False)
|
||||
return label, result.returncode
|
||||
|
||||
|
||||
def check_no_generated_artifacts() -> tuple[str, int]:
|
||||
label = "generated artifact check"
|
||||
print(f"\n== {label}")
|
||||
leftovers = [path for path in GENERATED_PATHS if path.exists()]
|
||||
if leftovers:
|
||||
print("Unexpected generated artifact(s):", file=sys.stderr)
|
||||
for path in leftovers:
|
||||
print(f"- {path.relative_to(ROOT)}", file=sys.stderr)
|
||||
return label, 1
|
||||
print("No unexpected generated artifacts.")
|
||||
return label, 0
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Run local repository checks.")
|
||||
parser.add_argument(
|
||||
"--with-training-preflight",
|
||||
action="store_true",
|
||||
help="Run checks that may report blocked when local CUDA/training dependencies are unavailable.",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
py_files = [str(path.relative_to(ROOT)) for path in sorted((ROOT / "scripts").glob("*.py"))]
|
||||
commands = [
|
||||
[sys.executable, "scripts/validate_model_cards.py"],
|
||||
[sys.executable, "scripts/validate_evals.py"],
|
||||
[sys.executable, "scripts/validate_gpu_profiles.py"],
|
||||
[sys.executable, "scripts/check_powershell_scripts.py"],
|
||||
[sys.executable, "scripts/check_model_storage.py", "--no-report", "--warn-only"],
|
||||
[sys.executable, "scripts/check_1c_plugin.py", "--no-report"],
|
||||
[sys.executable, "scripts/check_1c_mcp_adapter_contract.py"],
|
||||
[sys.executable, "scripts/check_1c_adapter_verification_stack.py"],
|
||||
[sys.executable, "scripts/smoke_1c_mcp_selector_chain.py", "--no-report"],
|
||||
[sys.executable, "-m", "py_compile", *py_files],
|
||||
]
|
||||
if args.with_training_preflight:
|
||||
commands.append([sys.executable, "scripts/preflight_1c_training.py"])
|
||||
|
||||
failures = []
|
||||
for command in commands:
|
||||
label, returncode = run(command)
|
||||
if returncode != 0:
|
||||
failures.append(label)
|
||||
|
||||
label, returncode = check_no_generated_artifacts()
|
||||
if returncode != 0:
|
||||
failures.append(label)
|
||||
|
||||
if failures:
|
||||
print("\nCheck failed:", file=sys.stderr)
|
||||
for failure in failures:
|
||||
print(f"- {failure}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
print("\nAll checks passed.")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,44 @@
|
||||
param(
|
||||
[string]$SshTarget = "docker-gpu.cin.su",
|
||||
[int]$ConnectTimeoutSeconds = 5
|
||||
)
|
||||
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
function Invoke-Remote {
|
||||
param(
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$Command
|
||||
)
|
||||
|
||||
ssh `
|
||||
-o BatchMode=yes `
|
||||
-o ConnectTimeout=$ConnectTimeoutSeconds `
|
||||
$SshTarget `
|
||||
$Command
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "Remote command failed on ${SshTarget} with exit code ${LASTEXITCODE}: ${Command}"
|
||||
}
|
||||
}
|
||||
|
||||
Write-Host "Checking SSH access to $SshTarget..."
|
||||
Invoke-Remote "hostname"
|
||||
|
||||
Write-Host ""
|
||||
Write-Host "Checking NVIDIA GPU..."
|
||||
Invoke-Remote "nvidia-smi --query-gpu=name,memory.total,driver_version --format=csv,noheader"
|
||||
|
||||
Write-Host ""
|
||||
Write-Host "Checking Docker..."
|
||||
Invoke-Remote "docker version --format '{{.Server.Version}}'"
|
||||
|
||||
Write-Host ""
|
||||
Write-Host "Checking Docker Compose..."
|
||||
Invoke-Remote "docker compose version"
|
||||
|
||||
Write-Host ""
|
||||
Write-Host "Checking Docker GPU runtime..."
|
||||
Invoke-Remote "docker run --rm --gpus all nvidia/cuda:12.4.1-base-ubuntu22.04 nvidia-smi --query-gpu=name,memory.total --format=csv,noheader"
|
||||
|
||||
Write-Host ""
|
||||
Write-Host "GPU host preflight completed."
|
||||
@@ -0,0 +1,158 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import datetime as dt
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
|
||||
from check_inference_endpoint import check_endpoint
|
||||
from common import read_json
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
DEFAULT_REPORT = ROOT / "reports" / "gpu-readiness.json"
|
||||
GPU_PROFILES = ROOT / "config" / "gpu_profiles.json"
|
||||
PROFILE_MODEL_EXPECTATIONS = {
|
||||
"vLLM": "qwen3-4b-instruct",
|
||||
"llama.cpp": "devstral-1c-q4",
|
||||
}
|
||||
|
||||
|
||||
def run_command(command: list[str], timeout: int) -> dict:
|
||||
try:
|
||||
result = subprocess.run(
|
||||
command,
|
||||
cwd=ROOT,
|
||||
text=True,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
timeout=timeout,
|
||||
check=False,
|
||||
)
|
||||
except subprocess.TimeoutExpired as exc:
|
||||
return {
|
||||
"command": command,
|
||||
"status": "failed",
|
||||
"returncode": None,
|
||||
"stdout": exc.stdout or "",
|
||||
"stderr": f"Timed out after {timeout}s",
|
||||
}
|
||||
return {
|
||||
"command": command,
|
||||
"status": "ok" if result.returncode == 0 else "failed",
|
||||
"returncode": result.returncode,
|
||||
"stdout": result.stdout.strip(),
|
||||
"stderr": result.stderr.strip(),
|
||||
}
|
||||
|
||||
|
||||
def check_http_url(url: str, timeout: int, *, expected_model: str | None = None) -> dict:
|
||||
if url.rstrip("/").endswith("/v1/models"):
|
||||
base_url = url.rstrip("/")[: -len("/v1/models")]
|
||||
return check_endpoint(base_url, expected_model, timeout)
|
||||
|
||||
started_at = time.perf_counter()
|
||||
result = {
|
||||
"url": url,
|
||||
"status": "failed",
|
||||
"latency_ms": None,
|
||||
"http_status": None,
|
||||
"error": None,
|
||||
}
|
||||
try:
|
||||
request = urllib.request.Request(url, method="GET")
|
||||
with urllib.request.urlopen(request, timeout=timeout) as response:
|
||||
result["http_status"] = response.status
|
||||
except (TimeoutError, OSError, urllib.error.URLError) as exc:
|
||||
result["error"] = str(exc)
|
||||
return result
|
||||
|
||||
result["latency_ms"] = round((time.perf_counter() - started_at) * 1000)
|
||||
result["status"] = "ok" if 200 <= int(result["http_status"] or 0) < 500 else "failed"
|
||||
return result
|
||||
|
||||
|
||||
def profile_checks(profile_id: str, timeout: int) -> dict[str, dict]:
|
||||
profiles = read_json(GPU_PROFILES)
|
||||
profile = profiles.get(profile_id)
|
||||
if not isinstance(profile, dict):
|
||||
available = ", ".join(sorted(str(name) for name in profiles))
|
||||
raise ValueError(f"Unknown GPU profile `{profile_id}`. Available: {available}")
|
||||
|
||||
checks: dict[str, dict] = {}
|
||||
for index, item in enumerate(profile.get("wait") or [], start=1):
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
name = str(item.get("name") or f"wait_{index}")
|
||||
url = str(item.get("url") or "")
|
||||
if not url:
|
||||
continue
|
||||
key = f"profile_{profile_id}_{index}_{name.lower().replace('.', '').replace(' ', '_')}"
|
||||
checks[key] = check_http_url(url, timeout, expected_model=PROFILE_MODEL_EXPECTATIONS.get(name))
|
||||
return checks
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Check GPU host and inference endpoint readiness.")
|
||||
parser.add_argument("--profile", default="text", help="GPU profile from config/gpu_profiles.json to check.")
|
||||
parser.add_argument("--all-endpoints", action="store_true", help="Legacy mode: require vLLM and llama.cpp endpoints at the same time.")
|
||||
parser.add_argument("--vllm-url", default="http://docker-gpu.cin.su:8000")
|
||||
parser.add_argument("--vllm-model", default="qwen3-4b-instruct")
|
||||
parser.add_argument("--llama-url", default="http://docker-gpu.cin.su:8080")
|
||||
parser.add_argument("--llama-model", default="devstral-1c-q4")
|
||||
parser.add_argument("--ssh-target", default="docker-gpu.cin.su")
|
||||
parser.add_argument("--timeout", type=int, default=10)
|
||||
parser.add_argument("--report", type=Path, default=DEFAULT_REPORT)
|
||||
parser.add_argument("--print", action="store_true")
|
||||
args = parser.parse_args()
|
||||
|
||||
checks: dict[str, dict] = {
|
||||
"ssh_preflight": run_command(
|
||||
[
|
||||
"powershell",
|
||||
"-NoProfile",
|
||||
"-ExecutionPolicy",
|
||||
"Bypass",
|
||||
"-File",
|
||||
"scripts/check_gpu_host.ps1",
|
||||
"-SshTarget",
|
||||
args.ssh_target,
|
||||
"-ConnectTimeoutSeconds",
|
||||
str(args.timeout),
|
||||
],
|
||||
timeout=max(args.timeout * 6, 30),
|
||||
)
|
||||
}
|
||||
if args.all_endpoints:
|
||||
checks["vllm_endpoint"] = check_endpoint(args.vllm_url, args.vllm_model, args.timeout)
|
||||
checks["llama_endpoint"] = check_endpoint(args.llama_url, args.llama_model, args.timeout)
|
||||
else:
|
||||
checks.update(profile_checks(args.profile, args.timeout))
|
||||
|
||||
failed = [name for name, check in checks.items() if check.get("status") != "ok"]
|
||||
report = {
|
||||
"created_at": dt.datetime.now(dt.UTC).isoformat(),
|
||||
"profile": args.profile,
|
||||
"all_endpoints": args.all_endpoints,
|
||||
"status": "failed" if failed else "ok",
|
||||
"failed_checks": failed,
|
||||
"checks": checks,
|
||||
}
|
||||
|
||||
args.report.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.report.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
if args.print:
|
||||
print(json.dumps(report, ensure_ascii=False, indent=2))
|
||||
else:
|
||||
print(f"GPU readiness status: {report['status']}")
|
||||
print(f"Wrote report to {args.report}")
|
||||
return 0 if not failed else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,84 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import datetime as dt
|
||||
import json
|
||||
import sys
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
DEFAULT_REPORT = ROOT / "reports" / "inference-endpoint-check.json"
|
||||
|
||||
|
||||
def fetch_models(base_url: str, timeout: int) -> tuple[list[str], int]:
|
||||
started_at = time.perf_counter()
|
||||
request = urllib.request.Request(f"{base_url.rstrip('/')}/v1/models", method="GET")
|
||||
with urllib.request.urlopen(request, timeout=timeout) as response:
|
||||
payload = json.loads(response.read().decode("utf-8"))
|
||||
latency_ms = round((time.perf_counter() - started_at) * 1000)
|
||||
rows = payload.get("data") if isinstance(payload, dict) else []
|
||||
if not isinstance(rows, list):
|
||||
return [], latency_ms
|
||||
return [str(row.get("id")) for row in rows if isinstance(row, dict) and row.get("id")], latency_ms
|
||||
|
||||
|
||||
def check_endpoint(base_url: str, expected_model: str | None, timeout: int) -> dict:
|
||||
result = {
|
||||
"base_url": base_url,
|
||||
"expected_model": expected_model,
|
||||
"status": "failed",
|
||||
"models": [],
|
||||
"available": None,
|
||||
"latency_ms": None,
|
||||
"error": None,
|
||||
}
|
||||
try:
|
||||
models, latency_ms = fetch_models(base_url, timeout)
|
||||
except (TimeoutError, OSError, urllib.error.URLError, ValueError) as exc:
|
||||
result["error"] = str(exc)
|
||||
return result
|
||||
|
||||
result["models"] = models
|
||||
result["latency_ms"] = latency_ms
|
||||
result["available"] = expected_model in models if expected_model else None
|
||||
result["status"] = "ok" if models and (expected_model is None or result["available"]) else "failed"
|
||||
return result
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Check an OpenAI-compatible inference endpoint.")
|
||||
parser.add_argument("--base-url", default="http://docker-gpu.cin.su:8000")
|
||||
parser.add_argument("--expected-model")
|
||||
parser.add_argument("--timeout", type=int, default=10)
|
||||
parser.add_argument("--report", type=Path, default=DEFAULT_REPORT)
|
||||
parser.add_argument("--print", action="store_true")
|
||||
parser.add_argument("--no-report", action="store_true")
|
||||
args = parser.parse_args()
|
||||
|
||||
result = check_endpoint(args.base_url, args.expected_model, args.timeout)
|
||||
report = {
|
||||
"created_at": dt.datetime.now(dt.UTC).isoformat(),
|
||||
"status": result["status"],
|
||||
"checks": [result],
|
||||
}
|
||||
|
||||
if not args.no_report:
|
||||
args.report.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.report.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
|
||||
if args.print:
|
||||
print(json.dumps(report, ensure_ascii=False, indent=2))
|
||||
else:
|
||||
print(f"Endpoint status: {result['status']} ({args.base_url})")
|
||||
if result["error"]:
|
||||
print(result["error"], file=sys.stderr)
|
||||
|
||||
return 0 if result["status"] == "ok" else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,142 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
DEFAULT_MANIFEST = ROOT / "reports" / "llm-artifact-manifest.json"
|
||||
DEFAULT_OUTPUT = ROOT / "reports" / "llm-artifact-check.json"
|
||||
|
||||
|
||||
def load_json(path: Path) -> dict[str, Any]:
|
||||
data = json.loads(path.read_text(encoding="utf-8-sig"))
|
||||
if not isinstance(data, dict):
|
||||
raise ValueError(f"{path} must contain a JSON object")
|
||||
return data
|
||||
|
||||
|
||||
def iter_files(path: Path) -> list[Path]:
|
||||
if not path.exists():
|
||||
return []
|
||||
return sorted(item for item in path.rglob("*") if item.is_file())
|
||||
|
||||
|
||||
def resolve_record_path(record_path: str, *, source_root: Path, target_root: Path | None) -> Path:
|
||||
path = Path(record_path)
|
||||
if target_root is None:
|
||||
return path
|
||||
try:
|
||||
relative = path.relative_to(source_root)
|
||||
return target_root / relative
|
||||
except ValueError:
|
||||
return target_root / path.name
|
||||
|
||||
|
||||
def check_artifact(record: dict[str, Any], *, source_root: Path, target_root: Path | None, strict_counts: bool) -> list[dict[str, Any]]:
|
||||
findings: list[dict[str, Any]] = []
|
||||
path = resolve_record_path(str(record.get("path") or ""), source_root=source_root, target_root=target_root)
|
||||
expected_exists = bool(record.get("exists"))
|
||||
if expected_exists and not path.exists():
|
||||
return [{"severity": "error", "code": "artifact_missing", "artifact": record.get("name"), "path": str(path)}]
|
||||
if not path.exists():
|
||||
return []
|
||||
if not path.is_dir():
|
||||
findings.append({"severity": "error", "code": "artifact_not_directory", "artifact": record.get("name"), "path": str(path)})
|
||||
return findings
|
||||
|
||||
files = iter_files(path)
|
||||
size = sum(item.stat().st_size for item in files)
|
||||
expected_count = int(record.get("file_count") or 0)
|
||||
expected_size = int(record.get("total_size_bytes") or 0)
|
||||
if strict_counts and len(files) != expected_count:
|
||||
findings.append(
|
||||
{
|
||||
"severity": "error",
|
||||
"code": "file_count_mismatch",
|
||||
"artifact": record.get("name"),
|
||||
"path": str(path),
|
||||
"expected": expected_count,
|
||||
"actual": len(files),
|
||||
}
|
||||
)
|
||||
elif len(files) < expected_count:
|
||||
findings.append(
|
||||
{
|
||||
"severity": "warning",
|
||||
"code": "file_count_decreased",
|
||||
"artifact": record.get("name"),
|
||||
"path": str(path),
|
||||
"expected_at_least": expected_count,
|
||||
"actual": len(files),
|
||||
}
|
||||
)
|
||||
if strict_counts and size != expected_size:
|
||||
findings.append(
|
||||
{
|
||||
"severity": "error",
|
||||
"code": "total_size_mismatch",
|
||||
"artifact": record.get("name"),
|
||||
"path": str(path),
|
||||
"expected": expected_size,
|
||||
"actual": size,
|
||||
}
|
||||
)
|
||||
elif size < expected_size:
|
||||
findings.append(
|
||||
{
|
||||
"severity": "warning",
|
||||
"code": "total_size_decreased",
|
||||
"artifact": record.get("name"),
|
||||
"path": str(path),
|
||||
"expected_at_least": expected_size,
|
||||
"actual": size,
|
||||
}
|
||||
)
|
||||
return findings
|
||||
|
||||
|
||||
def check_manifest(manifest: dict[str, Any], *, target_root: Path | None, strict_counts: bool) -> dict[str, Any]:
|
||||
source_root = Path(str(manifest.get("workspace_root") or ROOT))
|
||||
findings: list[dict[str, Any]] = []
|
||||
for record in manifest.get("artifacts") or []:
|
||||
findings.extend(check_artifact(record, source_root=source_root, target_root=target_root, strict_counts=strict_counts))
|
||||
|
||||
errors = [item for item in findings if item.get("severity") == "error"]
|
||||
warnings = [item for item in findings if item.get("severity") == "warning"]
|
||||
return {
|
||||
"schema": "llm_artifact_manifest_check.v1",
|
||||
"manifest_schema": manifest.get("schema"),
|
||||
"manifest_created_at": manifest.get("created_at"),
|
||||
"target_root": str(target_root) if target_root else None,
|
||||
"strict_counts": strict_counts,
|
||||
"passed": not errors,
|
||||
"counts": {
|
||||
"artifacts": len(manifest.get("artifacts") or []),
|
||||
"errors": len(errors),
|
||||
"warnings": len(warnings),
|
||||
"findings": len(findings),
|
||||
},
|
||||
"findings": findings,
|
||||
}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Check that local LLM/RAG artifacts from a manifest are present after transfer or before Docker launch.")
|
||||
parser.add_argument("--manifest", type=Path, default=DEFAULT_MANIFEST)
|
||||
parser.add_argument("--output", type=Path, default=DEFAULT_OUTPUT)
|
||||
parser.add_argument("--target-root", type=Path, help="New workspace root after transfer. If omitted, paths are checked as recorded.")
|
||||
parser.add_argument("--strict-counts", action="store_true", help="Fail when file counts or total sizes differ exactly.")
|
||||
args = parser.parse_args()
|
||||
|
||||
result = check_manifest(load_json(args.manifest), target_root=args.target_root, strict_counts=args.strict_counts)
|
||||
args.output.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.output.write_text(json.dumps(result, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
print(json.dumps({"output": str(args.output), "passed": result["passed"], "counts": result["counts"]}, ensure_ascii=False))
|
||||
return 0 if result["passed"] else 2
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,195 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import datetime as dt
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from common import iter_model_card_paths, localize_workspace_path, read_yaml_mapping
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
DEFAULT_REPORT = ROOT / "reports" / "model-storage.json"
|
||||
REQUIRED_MODEL_STATUSES = {"staging", "production"}
|
||||
|
||||
|
||||
def has_any(path: Path, patterns: list[str]) -> bool:
|
||||
return any(path.glob(pattern) for pattern in patterns)
|
||||
|
||||
|
||||
def has_weight_file(path: Path) -> bool:
|
||||
for file_path in path.iterdir() if path.exists() else []:
|
||||
if not file_path.is_file():
|
||||
continue
|
||||
name = file_path.name.lower()
|
||||
if name.endswith((".safetensors", ".bin", ".gguf")):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def check_gguf(path: Path, card: dict[str, Any]) -> dict[str, Any]:
|
||||
filename = card.get("filename")
|
||||
if not filename:
|
||||
return {"status": "failed", "reason": "filename is missing in model card"}
|
||||
file_path = path / str(filename)
|
||||
if not file_path.exists():
|
||||
return {"status": "missing", "reason": f"file is missing: {file_path.relative_to(ROOT)}"}
|
||||
size = file_path.stat().st_size
|
||||
expected_size = card.get("file_size_bytes")
|
||||
if expected_size and size != int(expected_size):
|
||||
return {
|
||||
"status": "partial",
|
||||
"reason": f"size mismatch: {size} != {expected_size}",
|
||||
"size_bytes": size,
|
||||
"expected_size_bytes": expected_size,
|
||||
}
|
||||
return {"status": "ok", "size_bytes": size, "expected_size_bytes": expected_size}
|
||||
|
||||
|
||||
def check_hf_model(path: Path) -> dict[str, Any]:
|
||||
required = ["config.json"]
|
||||
missing = [name for name in required if not (path / name).exists()]
|
||||
index_path = path / "model.safetensors.index.json"
|
||||
missing_shards: list[str] = []
|
||||
if index_path.exists():
|
||||
try:
|
||||
index = json.loads(index_path.read_text(encoding="utf-8"))
|
||||
shard_names = sorted(set((index.get("weight_map") or {}).values()))
|
||||
missing_shards = [name for name in shard_names if not (path / name).exists()]
|
||||
except json.JSONDecodeError:
|
||||
return {"status": "failed", "reason": "model.safetensors.index.json is invalid"}
|
||||
has_weights = has_weight_file(path)
|
||||
has_tokenizer = has_any(path, ["tokenizer.json", "tokenizer.model", "vocab.json"])
|
||||
if missing:
|
||||
return {"status": "missing", "reason": f"missing required file(s): {', '.join(missing)}"}
|
||||
if missing_shards:
|
||||
preview = ", ".join(missing_shards[:4])
|
||||
suffix = f" and {len(missing_shards) - 4} more" if len(missing_shards) > 4 else ""
|
||||
return {"status": "partial", "reason": f"missing shard file(s): {preview}{suffix}"}
|
||||
if not has_weights:
|
||||
return {"status": "metadata-only", "reason": "model weights are missing"}
|
||||
if not has_tokenizer:
|
||||
return {"status": "partial", "reason": "tokenizer files are missing"}
|
||||
return {"status": "ok"}
|
||||
|
||||
|
||||
def check_diffusers_model(path: Path) -> dict[str, Any]:
|
||||
missing = [name for name in ["model_index.json"] if not (path / name).exists()]
|
||||
if missing:
|
||||
return {"status": "missing", "reason": f"missing required file(s): {', '.join(missing)}"}
|
||||
if not any(path.rglob("*.safetensors")) and not any(path.rglob("*.bin")):
|
||||
return {"status": "metadata-only", "reason": "diffusers weights are missing"}
|
||||
return {"status": "ok"}
|
||||
|
||||
|
||||
def check_adapter(path: Path) -> dict[str, Any]:
|
||||
if not path.exists():
|
||||
return {"status": "missing", "reason": f"adapter path is missing: {path}"}
|
||||
if has_any(path, ["adapter_config.json", "*.safetensors", "*.bin"]):
|
||||
return {"status": "ok"}
|
||||
return {"status": "metadata-only", "reason": "adapter artifact files are missing"}
|
||||
|
||||
|
||||
def resolve_storage_path(raw_path: str, models_root: Path | None = None) -> Path:
|
||||
if models_root and raw_path.startswith("/models/"):
|
||||
return models_root / raw_path.removeprefix("/models/")
|
||||
return localize_workspace_path(raw_path)
|
||||
|
||||
|
||||
def check_card(path: Path, *, models_root: Path | None = None) -> dict[str, Any]:
|
||||
card = read_yaml_mapping(path)
|
||||
storage_path = resolve_storage_path(str(card.get("storage_path") or ""), models_root=models_root)
|
||||
model_status = str(card.get("status") or "draft")
|
||||
item = {
|
||||
"id": card.get("id"),
|
||||
"name": card.get("name"),
|
||||
"type": card.get("type"),
|
||||
"model_status": model_status,
|
||||
"format": card.get("format"),
|
||||
"quantization": card.get("quantization"),
|
||||
"runtime": (card.get("deployment") or {}).get("runtime"),
|
||||
"served_model_name": (card.get("deployment") or {}).get("served_model_name"),
|
||||
"filename": card.get("filename"),
|
||||
"storage_path": str(storage_path),
|
||||
"card_path": str(path.relative_to(ROOT)),
|
||||
"status": "missing",
|
||||
"reason": None,
|
||||
"required": model_status in REQUIRED_MODEL_STATUSES,
|
||||
}
|
||||
if not storage_path.exists():
|
||||
item["reason"] = "storage path is missing"
|
||||
return item
|
||||
|
||||
model_format = str(card.get("format") or "").lower()
|
||||
model_type = str(card.get("type") or "").lower()
|
||||
if model_format == "gguf":
|
||||
result = check_gguf(storage_path, card)
|
||||
elif model_format == "diffusers" or model_type == "image-diffusion-model":
|
||||
result = check_diffusers_model(storage_path)
|
||||
elif model_type == "lora-adapter":
|
||||
result = check_adapter(storage_path)
|
||||
else:
|
||||
result = check_hf_model(storage_path)
|
||||
item.update(result)
|
||||
return item
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Check local model storage against model cards.")
|
||||
parser.add_argument("--report", type=Path, default=DEFAULT_REPORT)
|
||||
parser.add_argument("--print", action="store_true")
|
||||
parser.add_argument("--no-report", action="store_true")
|
||||
parser.add_argument("--warn-only", action="store_true", help="Return success even when models are partial or missing.")
|
||||
parser.add_argument("--strict", action="store_true", help="Fail on incomplete draft/candidate models too.")
|
||||
parser.add_argument(
|
||||
"--models-root",
|
||||
type=Path,
|
||||
default=None,
|
||||
help="Override /models paths, for example Z:/LLM/models or /models inside the GPU host container.",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
models_root = args.models_root.resolve() if args.models_root else None
|
||||
models = [check_card(path, models_root=models_root) for path in iter_model_card_paths()]
|
||||
incomplete_statuses = {"failed", "missing", "partial"}
|
||||
incomplete = [model["id"] for model in models if model["status"] in incomplete_statuses]
|
||||
required_failed = [
|
||||
model["id"]
|
||||
for model in models
|
||||
if model["required"] and model["status"] in incomplete_statuses
|
||||
]
|
||||
failed = incomplete if args.strict else required_failed
|
||||
report = {
|
||||
"created_at": dt.datetime.now(dt.UTC).isoformat(),
|
||||
"models_root": str(models_root) if models_root else None,
|
||||
"status": "failed" if failed else "ok",
|
||||
"failed": failed,
|
||||
"required_failed": required_failed,
|
||||
"planned_incomplete": [
|
||||
model["id"]
|
||||
for model in models
|
||||
if not model["required"] and model["status"] in incomplete_statuses
|
||||
],
|
||||
"strict": args.strict,
|
||||
"counts": {
|
||||
"ok": sum(1 for model in models if model["status"] == "ok"),
|
||||
"missing": sum(1 for model in models if model["status"] == "missing"),
|
||||
"partial": sum(1 for model in models if model["status"] == "partial"),
|
||||
"metadata_only": sum(1 for model in models if model["status"] == "metadata-only"),
|
||||
"failed": sum(1 for model in models if model["status"] == "failed"),
|
||||
},
|
||||
"models": models,
|
||||
}
|
||||
if not args.no_report:
|
||||
args.report.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.report.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
if args.print:
|
||||
print(json.dumps(report, ensure_ascii=False, indent=2))
|
||||
else:
|
||||
print(f"Model storage status: {report['status']}")
|
||||
return 0 if args.warn_only else 1 if failed else 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,420 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
DEFAULT_PATHS = [ROOT / "scripts", ROOT / "core" / "deploy"]
|
||||
|
||||
|
||||
def iter_powershell_scripts(paths: list[Path]) -> list[Path]:
|
||||
scripts: list[Path] = []
|
||||
for path in paths:
|
||||
if path.is_file() and path.suffix.lower() == ".ps1":
|
||||
scripts.append(path)
|
||||
elif path.is_dir():
|
||||
scripts.extend(path.rglob("*.ps1"))
|
||||
return sorted(set(scripts))
|
||||
|
||||
|
||||
def powershell_executable() -> str | None:
|
||||
return shutil.which("pwsh") or shutil.which("powershell")
|
||||
|
||||
|
||||
def ps_single_quoted(value: str) -> str:
|
||||
return "'" + value.replace("'", "''") + "'"
|
||||
|
||||
|
||||
def check_script(executable: str, path: Path) -> tuple[bool, str]:
|
||||
path_literal = ps_single_quoted(str(path))
|
||||
command = [
|
||||
executable,
|
||||
"-NoProfile",
|
||||
"-ExecutionPolicy",
|
||||
"Bypass",
|
||||
"-Command",
|
||||
(
|
||||
"$ErrorActionPreference = 'Stop'; "
|
||||
f"$path = {path_literal}; "
|
||||
"$errors = $null; "
|
||||
"[System.Management.Automation.PSParser]::Tokenize((Get-Content -Raw -LiteralPath $path), [ref]$errors) | Out-Null; "
|
||||
"if ($errors) { "
|
||||
" foreach ($errorItem in $errors) { "
|
||||
" Write-Error ('{0}: {1} at line {2}, column {3}' -f $path, $errorItem.Message, $errorItem.Token.StartLine, $errorItem.Token.StartColumn); "
|
||||
" }; "
|
||||
" exit 1 "
|
||||
"}"
|
||||
),
|
||||
]
|
||||
result = subprocess.run(command, cwd=ROOT, text=True, capture_output=True, check=False)
|
||||
output = "\n".join(part for part in [result.stdout.strip(), result.stderr.strip()] if part)
|
||||
return result.returncode == 0, output
|
||||
|
||||
|
||||
def run_powershell_contract_command(executable: str, args: list[str]) -> subprocess.CompletedProcess[str]:
|
||||
return subprocess.run(
|
||||
[executable, "-NoProfile", "-ExecutionPolicy", "Bypass", *args],
|
||||
cwd=ROOT,
|
||||
text=True,
|
||||
capture_output=True,
|
||||
check=False,
|
||||
)
|
||||
|
||||
|
||||
def combined_output(result: subprocess.CompletedProcess[str]) -> str:
|
||||
return "\n".join(part for part in [result.stdout.strip(), result.stderr.strip()] if part)
|
||||
|
||||
|
||||
def check_adapter_base_id_runtime(executable: str) -> list[str]:
|
||||
failures: list[str] = []
|
||||
duplicate_verify = run_powershell_contract_command(
|
||||
executable,
|
||||
["-File", "scripts/verify_1c_adapter_deployment.ps1", "-BaseId", "upo_test,upo_test", "-SkipRest", "-SkipMcp"],
|
||||
)
|
||||
duplicate_verify_output = combined_output(duplicate_verify)
|
||||
if duplicate_verify.returncode == 0 or "Duplicate BaseId value(s): upo_test" not in duplicate_verify_output:
|
||||
failures.append(
|
||||
"verify_1c_adapter_deployment.ps1 must fail duplicate comma-separated -BaseId values at runtime."
|
||||
)
|
||||
|
||||
multi_verify = run_powershell_contract_command(
|
||||
executable,
|
||||
["-File", "scripts/verify_1c_adapter_deployment.ps1", "-BaseId", "upo_test,another_test", "-SkipRest", "-SkipMcp"],
|
||||
)
|
||||
multi_verify_output = combined_output(multi_verify)
|
||||
if multi_verify.returncode != 0 or "--base-id upo_test another_test" not in multi_verify_output:
|
||||
failures.append(
|
||||
"verify_1c_adapter_deployment.ps1 must expand comma-separated -BaseId values before calling the persisted report validator."
|
||||
)
|
||||
|
||||
duplicate_deploy = run_powershell_contract_command(
|
||||
executable,
|
||||
["-File", "scripts/deploy_1c_adapter_stack.ps1", "-BaseId", "upo_test,upo_test", "-SkipRest", "-SkipMcp"],
|
||||
)
|
||||
duplicate_deploy_output = combined_output(duplicate_deploy)
|
||||
if duplicate_deploy.returncode == 0 or "Duplicate BaseId value(s): upo_test" not in duplicate_deploy_output:
|
||||
failures.append(
|
||||
"deploy_1c_adapter_stack.ps1 must fail duplicate comma-separated -BaseId values at runtime."
|
||||
)
|
||||
|
||||
multi_deploy = run_powershell_contract_command(
|
||||
executable,
|
||||
["-File", "scripts/deploy_1c_adapter_stack.ps1", "-BaseId", "upo_test,another_test", "-SkipRest", "-SkipMcp"],
|
||||
)
|
||||
multi_deploy_output = combined_output(multi_deploy)
|
||||
if multi_deploy.returncode != 0 or "--base-id upo_test another_test" not in multi_deploy_output:
|
||||
failures.append(
|
||||
"deploy_1c_adapter_stack.ps1 must preserve multiple normalized -BaseId values when invoking verification."
|
||||
)
|
||||
|
||||
return failures
|
||||
|
||||
|
||||
def check_adapter_verify_wiring(scripts: list[Path], executable: str) -> list[str]:
|
||||
script_set = set(scripts)
|
||||
verify_path = ROOT / "scripts" / "verify_1c_adapter_deployment.ps1"
|
||||
deploy_path = ROOT / "scripts" / "deploy_1c_adapter_stack.ps1"
|
||||
stack_path = ROOT / "scripts" / "check_1c_adapter_verification_stack.py"
|
||||
readiness_path = ROOT / "scripts" / "check_1c_saved_state_strict_readiness.py"
|
||||
prepare_copy_sql_path = ROOT / "scripts" / "prepare_1c_saved_state_copy_sql.py"
|
||||
prepare_cleanup_sql_path = ROOT / "scripts" / "prepare_1c_saved_state_cleanup_sql.py"
|
||||
verify_copy_path = ROOT / "scripts" / "verify_1c_saved_state_copy.py"
|
||||
execute_copy_sql_path = ROOT / "scripts" / "execute_1c_saved_state_copy_sql.ps1"
|
||||
if verify_path not in script_set and deploy_path not in script_set:
|
||||
return []
|
||||
|
||||
failures: list[str] = []
|
||||
if not verify_path.exists():
|
||||
failures.append("scripts/verify_1c_adapter_deployment.ps1 is missing.")
|
||||
return failures
|
||||
if not deploy_path.exists():
|
||||
failures.append("scripts/deploy_1c_adapter_stack.ps1 is missing.")
|
||||
return failures
|
||||
if not stack_path.exists():
|
||||
failures.append("scripts/check_1c_adapter_verification_stack.py is missing.")
|
||||
return failures
|
||||
if not readiness_path.exists():
|
||||
failures.append("scripts/check_1c_saved_state_strict_readiness.py is missing.")
|
||||
if not prepare_copy_sql_path.exists():
|
||||
failures.append("scripts/prepare_1c_saved_state_copy_sql.py is missing.")
|
||||
if not prepare_cleanup_sql_path.exists():
|
||||
failures.append("scripts/prepare_1c_saved_state_cleanup_sql.py is missing.")
|
||||
if not verify_copy_path.exists():
|
||||
failures.append("scripts/verify_1c_saved_state_copy.py is missing.")
|
||||
return failures
|
||||
if not execute_copy_sql_path.exists():
|
||||
failures.append("scripts/execute_1c_saved_state_copy_sql.ps1 is missing.")
|
||||
return failures
|
||||
|
||||
verify_text = verify_path.read_text(encoding="utf-8", errors="replace")
|
||||
deploy_text = deploy_path.read_text(encoding="utf-8", errors="replace")
|
||||
stack_text = stack_path.read_text(encoding="utf-8", errors="replace")
|
||||
readiness_text = readiness_path.read_text(encoding="utf-8", errors="replace")
|
||||
prepare_copy_sql_text = prepare_copy_sql_path.read_text(encoding="utf-8", errors="replace") if prepare_copy_sql_path.exists() else ""
|
||||
prepare_cleanup_sql_text = prepare_cleanup_sql_path.read_text(encoding="utf-8", errors="replace") if prepare_cleanup_sql_path.exists() else ""
|
||||
verify_copy_text = verify_copy_path.read_text(encoding="utf-8", errors="replace") if verify_copy_path.exists() else ""
|
||||
execute_copy_sql_text = execute_copy_sql_path.read_text(encoding="utf-8", errors="replace") if execute_copy_sql_path.exists() else ""
|
||||
if "[switch]$RequireSelectorChainWritePlanComposition" not in verify_text:
|
||||
failures.append("verify_1c_adapter_deployment.ps1 must declare -RequireSelectorChainWritePlanComposition.")
|
||||
if "[switch]$RequireSavedStateWriteSmoke" not in verify_text:
|
||||
failures.append("verify_1c_adapter_deployment.ps1 must declare -RequireSavedStateWriteSmoke.")
|
||||
if "[string]$SavedStateTable" not in verify_text or '[ValidateSet("ConfigSave", "ConfigCASSave")]' not in verify_text:
|
||||
failures.append("verify_1c_adapter_deployment.ps1 must declare -SavedStateTable with ConfigSave/ConfigCASSave validation.")
|
||||
if verify_text.count("$SavedStateTable") < 4:
|
||||
failures.append("verify_1c_adapter_deployment.ps1 must use -SavedStateTable for copy plan and saved-state smoke commands.")
|
||||
if verify_text.count("--require-write-plan-composition") < 2:
|
||||
failures.append("verify_1c_adapter_deployment.ps1 must pass --require-write-plan-composition to both REST and MCP selector-chain smoke commands.")
|
||||
if "--allow-empty-saved-state" not in verify_text or "if (-not $RequireSavedStateWriteSmoke)" not in verify_text:
|
||||
failures.append("verify_1c_adapter_deployment.ps1 must allow empty saved-state only when -RequireSavedStateWriteSmoke is not set.")
|
||||
if "function Get-DuplicateValues" not in verify_text or "Duplicate BaseId value(s)" not in verify_text:
|
||||
failures.append("verify_1c_adapter_deployment.ps1 must reject duplicate -BaseId values before writing reports.")
|
||||
if "function Normalize-BaseIds" not in verify_text or '-split ","' not in verify_text:
|
||||
failures.append("verify_1c_adapter_deployment.ps1 must split comma-separated -BaseId values before verification.")
|
||||
if "function Assert-SelectorChainReport" not in verify_text or verify_text.count("Assert-SelectorChainReport") < 3:
|
||||
failures.append("verify_1c_adapter_deployment.ps1 must validate both persisted selector-chain JSON reports after smoke commands.")
|
||||
if "working_state" not in verify_text or 'did not use working state' not in verify_text:
|
||||
failures.append("verify_1c_adapter_deployment.ps1 must validate selector-chain working_state=working in persisted reports.")
|
||||
for function_name in (
|
||||
"Assert-WritePlanSafetyReport",
|
||||
"Assert-WritePreflightReport",
|
||||
"Assert-WriteRollbackSafetyReport",
|
||||
"Assert-SavedStateDiffReport",
|
||||
"Assert-SavedStateChangesReport",
|
||||
"Assert-SavedStateFormWriteReport",
|
||||
"Assert-SavedStateModuleWriteReport",
|
||||
):
|
||||
if f"function {function_name}" not in verify_text or verify_text.count(function_name) < 2:
|
||||
failures.append(f"verify_1c_adapter_deployment.ps1 must validate reports with {function_name}.")
|
||||
if "scripts/check_1c_verify_reports.py" not in verify_text:
|
||||
failures.append("verify_1c_adapter_deployment.ps1 must run the offline persisted report validator.")
|
||||
if "scripts/plan_1c_saved_state_copy.py" not in verify_text or "saved-state-copy-plan.json" not in verify_text:
|
||||
failures.append("verify_1c_adapter_deployment.ps1 must generate saved-state-copy-plan.json before persisted report validation.")
|
||||
if "scripts/prepare_1c_saved_state_copy_sql.py" not in verify_text or "prepare-saved-state-copy-sql.json" not in verify_text:
|
||||
failures.append("verify_1c_adapter_deployment.ps1 must generate prepare-saved-state-copy-sql.json before persisted report validation.")
|
||||
if "scripts/prepare_1c_saved_state_cleanup_sql.py" not in verify_text or "cleanup-saved-state-copy-sql.json" not in verify_text:
|
||||
failures.append("verify_1c_adapter_deployment.ps1 must generate cleanup-saved-state-copy-sql.json before persisted report validation.")
|
||||
if "scripts/check_1c_saved_state_strict_readiness.py" not in verify_text or "saved-state-strict-readiness.json" not in verify_text:
|
||||
failures.append("verify_1c_adapter_deployment.ps1 must generate saved-state-strict-readiness.json before persisted report validation.")
|
||||
for flag in (
|
||||
"--skip-rest",
|
||||
"--skip-mcp",
|
||||
"--skip-write-plan-safety-smoke",
|
||||
"--skip-write-rollback-safety-smoke",
|
||||
"--skip-saved-state-diff-smoke",
|
||||
"--skip-saved-state-write-smoke",
|
||||
"--require-saved-state-write-smoke",
|
||||
"--require-selector-chain-write-plan-composition",
|
||||
"--rest-adapter-url",
|
||||
"--mcp-url",
|
||||
"--saved-state-table",
|
||||
):
|
||||
if flag not in verify_text:
|
||||
failures.append(f"verify_1c_adapter_deployment.ps1 must forward {flag} to scripts/check_1c_verify_reports.py.")
|
||||
verify_reports_self_test = subprocess.run(
|
||||
[sys.executable, "scripts/check_1c_verify_reports.py", "--self-test", "--json"],
|
||||
cwd=ROOT,
|
||||
text=True,
|
||||
capture_output=True,
|
||||
check=False,
|
||||
)
|
||||
if verify_reports_self_test.returncode != 0:
|
||||
output = "\n".join(part for part in [verify_reports_self_test.stdout.strip(), verify_reports_self_test.stderr.strip()] if part)
|
||||
failures.append(f"scripts/check_1c_verify_reports.py --self-test failed: {output}")
|
||||
else:
|
||||
try:
|
||||
self_test_report = json.loads(verify_reports_self_test.stdout)
|
||||
except json.JSONDecodeError as exc:
|
||||
failures.append(f"scripts/check_1c_verify_reports.py --self-test --json returned invalid JSON: {exc}")
|
||||
else:
|
||||
expected_self_test_codes = {
|
||||
"strict_skip_failure_codes": {
|
||||
"selector_chain_composition_required",
|
||||
"saved_state_form_write_required",
|
||||
"saved_state_module_write_required",
|
||||
},
|
||||
"coverage_failure_codes": {
|
||||
"selector_chain_write_plan_evidence_missing",
|
||||
"selector_chain_next_method_unexpected",
|
||||
"selector_chain_write_plan_target_not_boolean",
|
||||
"selector_chain_working_state_unexpected",
|
||||
},
|
||||
"consistency_failure_codes": {
|
||||
"selector_chain_write_plan_target_not_composed",
|
||||
"selector_chain_composed_status_unexpected",
|
||||
},
|
||||
"safety_failure_codes": {
|
||||
"write_plan_safety_check_field_unexpected",
|
||||
"write_plan_safety_check_missing",
|
||||
},
|
||||
"rollback_safety_failure_codes": {
|
||||
"write_rollback_safety_check_field_unexpected",
|
||||
"write_rollback_safety_check_missing",
|
||||
},
|
||||
"saved_state_diff_failure_codes": {
|
||||
"saved_state_diff_check_field_unexpected",
|
||||
"saved_state_diff_check_missing",
|
||||
},
|
||||
"schema_failure_codes": {
|
||||
"report_schema_unexpected",
|
||||
},
|
||||
"identity_failure_codes": {
|
||||
"report_base_id_unexpected",
|
||||
"report_transport_unexpected",
|
||||
},
|
||||
"endpoint_failure_codes": {
|
||||
"report_endpoint_url_unexpected",
|
||||
},
|
||||
"staleness_failure_codes": {
|
||||
"report_stale",
|
||||
},
|
||||
"duplicate_failure_codes": {
|
||||
"duplicate_base_id",
|
||||
},
|
||||
"saved_state_failure_codes": {
|
||||
"saved_state_form_route_write_plan_not_allowed",
|
||||
"saved_state_form_route_field_missing",
|
||||
"saved_state_module_rollback_missing",
|
||||
},
|
||||
"saved_state_strict_readiness_failure_codes": {
|
||||
"saved_state_strict_readiness_required",
|
||||
"saved_state_strict_readiness_table_unexpected",
|
||||
},
|
||||
"saved_state_copy_plan_failure_codes": {
|
||||
"saved_state_copy_plan_source_family_invalid",
|
||||
"saved_state_copy_plan_status_unexpected",
|
||||
"saved_state_copy_plan_source_row_table_unexpected",
|
||||
"saved_state_copy_plan_target_collisions_present",
|
||||
},
|
||||
"saved_state_table_failure_codes": {
|
||||
"saved_state_table_mismatch",
|
||||
"saved_state_table_unexpected",
|
||||
},
|
||||
}
|
||||
for field, expected_codes in expected_self_test_codes.items():
|
||||
actual_codes = set(self_test_report.get(field) or [])
|
||||
missing_codes = sorted(expected_codes - actual_codes)
|
||||
if missing_codes:
|
||||
failures.append(f"scripts/check_1c_verify_reports.py --self-test must cover {field}: missing {missing_codes}.")
|
||||
if "ConvertFrom-Json" not in verify_text:
|
||||
failures.append("verify_1c_adapter_deployment.ps1 must parse selector-chain JSON reports with ConvertFrom-Json.")
|
||||
if "--saved-state-table" not in readiness_text or "saved_state_table" not in readiness_text:
|
||||
failures.append("check_1c_saved_state_strict_readiness.py must support --saved-state-table and include it in reports.")
|
||||
for token in ("source_family.valid", "target_collisions.status", "sql_write_performed", "BEGIN TRANSACTION", "THROW 51001"):
|
||||
if token not in prepare_copy_sql_text:
|
||||
failures.append(f"prepare_1c_saved_state_copy_sql.py must include guarded SQL generation token: {token}.")
|
||||
for token in ("onec_saved_state_cleanup_sql_plan.v1", "DELETE t FROM", "BinarySHA1", "THROW 51102", "sql_write_performed"):
|
||||
if token not in prepare_cleanup_sql_text:
|
||||
failures.append(f"prepare_1c_saved_state_cleanup_sql.py must include guarded cleanup SQL token: {token}.")
|
||||
for token in ("onec_saved_state_copy_verify.v1", "blocked_missing_target_rows", "BinarySHA1", "sql_write_performed", "--require-ready"):
|
||||
if token not in verify_copy_text:
|
||||
failures.append(f"verify_1c_saved_state_copy.py must include read-only verification token: {token}.")
|
||||
for token in (
|
||||
"[switch]$IUnderstandThisWritesToSql",
|
||||
"Refusing to execute SQL without -IUnderstandThisWritesToSql",
|
||||
"onec_saved_state_copy_sql_execution.v1",
|
||||
"onec_saved_state_copy_sql_plan.v1",
|
||||
"sql_execution_attempted",
|
||||
"sql_write_performed",
|
||||
"Get-FileHash",
|
||||
"scripts/verify_1c_saved_state_copy.py",
|
||||
"--require-ready",
|
||||
):
|
||||
if token not in execute_copy_sql_text:
|
||||
failures.append(f"execute_1c_saved_state_copy_sql.ps1 must include guarded execution token: {token}.")
|
||||
if "[switch]$RequireSelectorChainWritePlanComposition" not in deploy_text:
|
||||
failures.append("deploy_1c_adapter_stack.ps1 must declare -RequireSelectorChainWritePlanComposition.")
|
||||
if "[switch]$RequireSavedStateWriteSmoke" not in deploy_text:
|
||||
failures.append("deploy_1c_adapter_stack.ps1 must declare -RequireSavedStateWriteSmoke.")
|
||||
if "[switch]$SkipWriteRollbackSafetySmoke" not in deploy_text:
|
||||
failures.append("deploy_1c_adapter_stack.ps1 must declare -SkipWriteRollbackSafetySmoke.")
|
||||
if "[switch]$SkipSavedStateDiffSmoke" not in deploy_text:
|
||||
failures.append("deploy_1c_adapter_stack.ps1 must declare -SkipSavedStateDiffSmoke.")
|
||||
if "[string]$SavedStateTable" not in deploy_text or '[ValidateSet("ConfigSave", "ConfigCASSave")]' not in deploy_text:
|
||||
failures.append("deploy_1c_adapter_stack.ps1 must declare -SavedStateTable with ConfigSave/ConfigCASSave validation.")
|
||||
if '"-RequireSelectorChainWritePlanComposition"' not in deploy_text:
|
||||
failures.append("deploy_1c_adapter_stack.ps1 must forward -RequireSelectorChainWritePlanComposition to verify_1c_adapter_deployment.ps1.")
|
||||
if '"-RequireSavedStateWriteSmoke"' not in deploy_text:
|
||||
failures.append("deploy_1c_adapter_stack.ps1 must forward -RequireSavedStateWriteSmoke to verify_1c_adapter_deployment.ps1.")
|
||||
if '"-SavedStateTable"' not in deploy_text:
|
||||
failures.append("deploy_1c_adapter_stack.ps1 must forward -SavedStateTable to verify_1c_adapter_deployment.ps1.")
|
||||
if '"-SkipWriteRollbackSafetySmoke"' not in deploy_text:
|
||||
failures.append("deploy_1c_adapter_stack.ps1 must forward -SkipWriteRollbackSafetySmoke to verify_1c_adapter_deployment.ps1.")
|
||||
if '"-SkipSavedStateDiffSmoke"' not in deploy_text:
|
||||
failures.append("deploy_1c_adapter_stack.ps1 must forward -SkipSavedStateDiffSmoke to verify_1c_adapter_deployment.ps1.")
|
||||
if "function Get-DuplicateValues" not in deploy_text or "Duplicate BaseId value(s)" not in deploy_text:
|
||||
failures.append("deploy_1c_adapter_stack.ps1 must reject duplicate -BaseId values before invoking verification.")
|
||||
if "function Normalize-BaseIds" not in deploy_text or '-split ","' not in deploy_text:
|
||||
failures.append("deploy_1c_adapter_stack.ps1 must split comma-separated -BaseId values before invoking verification.")
|
||||
if '$baseIds -join ","' not in deploy_text:
|
||||
failures.append("deploy_1c_adapter_stack.ps1 must pass normalized BaseId values to nested verification as a comma-separated argument.")
|
||||
for flag in ("--rest-adapter-url", "--mcp-url", "--saved-state-table", "--max-report-age-seconds"):
|
||||
if flag not in stack_text:
|
||||
failures.append(f"check_1c_adapter_verification_stack.py must pass {flag} to scripts/check_1c_verify_reports.py.")
|
||||
for script_name in (
|
||||
"scripts/smoke_1c_write_plan_safety.py",
|
||||
"scripts/smoke_1c_write_preflight.py",
|
||||
"scripts/smoke_1c_write_rollback_safety.py",
|
||||
"scripts/smoke_1c_saved_state_diff.py",
|
||||
"scripts/smoke_1c_saved_state_changes.py",
|
||||
"scripts/smoke_1c_saved_state_write_routes.py",
|
||||
"scripts/smoke_1c_saved_state_module_write.py",
|
||||
"scripts/check_1c_saved_state_strict_readiness.py",
|
||||
"scripts/plan_1c_saved_state_copy.py",
|
||||
"scripts/prepare_1c_saved_state_copy_sql.py",
|
||||
"scripts/prepare_1c_saved_state_cleanup_sql.py",
|
||||
"scripts/verify_1c_saved_state_copy.py",
|
||||
):
|
||||
if script_name not in stack_text:
|
||||
failures.append(f"check_1c_adapter_verification_stack.py must py_compile {script_name}.")
|
||||
if 'nargs="+"' not in stack_text or "*args.base_id" not in stack_text:
|
||||
failures.append("check_1c_adapter_verification_stack.py must support multiple --base-id values and forward them to scripts/check_1c_verify_reports.py.")
|
||||
if "duplicate_base_id" not in stack_text:
|
||||
failures.append("check_1c_adapter_verification_stack.py must reject duplicate --base-id values.")
|
||||
failures.extend(check_adapter_base_id_runtime(executable))
|
||||
return failures
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Check PowerShell script syntax with PSParser.")
|
||||
parser.add_argument("paths", nargs="*", type=Path, help="Files or directories to scan. Defaults to scripts/ and core/deploy/.")
|
||||
args = parser.parse_args()
|
||||
|
||||
executable = powershell_executable()
|
||||
if not executable:
|
||||
print("PowerShell executable not found.", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
paths = [path if path.is_absolute() else ROOT / path for path in args.paths] if args.paths else DEFAULT_PATHS
|
||||
scripts = iter_powershell_scripts(paths)
|
||||
if not scripts:
|
||||
print("No PowerShell scripts found.")
|
||||
return 0
|
||||
|
||||
failures = []
|
||||
for script in scripts:
|
||||
ok, output = check_script(executable, script)
|
||||
if not ok:
|
||||
failures.append((script, output))
|
||||
contract_failures = check_adapter_verify_wiring(scripts, executable)
|
||||
|
||||
if failures or contract_failures:
|
||||
print("PowerShell script check failed:", file=sys.stderr)
|
||||
for script, output in failures:
|
||||
print(f"- {script.relative_to(ROOT)}", file=sys.stderr)
|
||||
if output:
|
||||
print(output, file=sys.stderr)
|
||||
for failure in contract_failures:
|
||||
print(f"- adapter verify wiring: {failure}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
print(f"Validated {len(scripts)} PowerShell script(s).")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,71 @@
|
||||
param(
|
||||
[string]$UiBaseUrl = "http://192.168.220.91:8765",
|
||||
[string]$ModelId = "qwen3-coder-30b-a3b-instruct-q6_k",
|
||||
[string]$Plugin = "1c",
|
||||
[string[]]$Profiles = @("gpu-fast", "cpu-test"),
|
||||
[string]$Report = "reports/benchmarks/runtime-preflight-latest.json",
|
||||
[int]$TimeoutSec = 60
|
||||
)
|
||||
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
$root = Split-Path -Parent (Split-Path -Parent $MyInvocation.MyCommand.Path)
|
||||
$reportPath = Join-Path $root $Report
|
||||
$url = "$($UiBaseUrl.TrimEnd('/'))/api/benchmark/preflight"
|
||||
$payload = @{
|
||||
model_id = $ModelId
|
||||
plugin = $Plugin
|
||||
profiles = $Profiles
|
||||
} | ConvertTo-Json -Depth 8
|
||||
|
||||
Write-Host "Runtime benchmark preflight"
|
||||
Write-Host "URL: $url"
|
||||
Write-Host "Model: $ModelId"
|
||||
Write-Host "Plugin: $Plugin"
|
||||
Write-Host "Profiles: $($Profiles -join ', ')"
|
||||
Write-Host ""
|
||||
|
||||
try {
|
||||
$result = Invoke-RestMethod -Uri $url -Method Post -ContentType "application/json" -Body $payload -TimeoutSec $TimeoutSec
|
||||
} catch {
|
||||
$response = $_.Exception.Response
|
||||
if ($response) {
|
||||
try {
|
||||
$stream = $response.GetResponseStream()
|
||||
$reader = [System.IO.StreamReader]::new($stream)
|
||||
$body = $reader.ReadToEnd()
|
||||
if ($body) {
|
||||
$result = $body | ConvertFrom-Json
|
||||
} else {
|
||||
throw
|
||||
}
|
||||
} catch {
|
||||
throw $_.Exception
|
||||
}
|
||||
} else {
|
||||
throw
|
||||
}
|
||||
}
|
||||
|
||||
New-Item -ItemType Directory -Force (Split-Path -Parent $reportPath) | Out-Null
|
||||
$result | ConvertTo-Json -Depth 10 | Set-Content -LiteralPath $reportPath -Encoding UTF8
|
||||
|
||||
foreach ($check in @($result.checks)) {
|
||||
$target = $check.target
|
||||
$served = if ($target) { $target.served_model_name } else { "-" }
|
||||
$endpoint = if ($target) { $target.base_url } else { "-" }
|
||||
$latency = if ($check.latency_ms -ne $null) { "$($check.latency_ms)ms" } else { "-" }
|
||||
$status = if ($check.available) { "ready" } else { $check.status }
|
||||
Write-Host ("{0,-12} {1,-14} {2,6} {3} {4}" -f $check.profile_id, $status, $latency, $endpoint, $served)
|
||||
if (-not $check.available -and $check.error) {
|
||||
Write-Host " error: $($check.error)"
|
||||
}
|
||||
}
|
||||
|
||||
Write-Host ""
|
||||
Write-Host "Status: $($result.status)"
|
||||
Write-Host "Report: $reportPath"
|
||||
|
||||
if (-not $result.ready) {
|
||||
exit 1
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
param(
|
||||
[string]$BaseUrl = "http://docker-gpu.cin.su:8000",
|
||||
[string]$Model = "qwen3-4b-instruct"
|
||||
)
|
||||
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
$modelsUrl = "$($BaseUrl.TrimEnd('/'))/v1/models"
|
||||
Write-Host "Checking models endpoint: $modelsUrl"
|
||||
Invoke-RestMethod -Method Get -Uri $modelsUrl | ConvertTo-Json -Depth 10
|
||||
|
||||
Write-Host ""
|
||||
Write-Host "Checking chat endpoint..."
|
||||
python scripts/smoke_chat.py --base-url $BaseUrl --model $Model
|
||||
@@ -0,0 +1,85 @@
|
||||
param(
|
||||
[string]$SshTarget = "",
|
||||
[string]$ModelsRoot = "Z:\LLM\models",
|
||||
[int]$ConnectTimeoutSeconds = 5
|
||||
)
|
||||
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
function Invoke-LocalCheck {
|
||||
param(
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$Name,
|
||||
[Parameter(Mandatory = $true)]
|
||||
[scriptblock]$Block
|
||||
)
|
||||
|
||||
Write-Host ""
|
||||
Write-Host "== $Name"
|
||||
& $Block
|
||||
}
|
||||
|
||||
function Invoke-RemoteCheck {
|
||||
param(
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$Name,
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$Command
|
||||
)
|
||||
|
||||
Write-Host ""
|
||||
Write-Host "== $Name"
|
||||
ssh -o BatchMode=yes -o ConnectTimeout=$ConnectTimeoutSeconds $SshTarget "powershell -NoProfile -Command $Command"
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "Remote check failed on ${SshTarget}: ${Name}"
|
||||
}
|
||||
}
|
||||
|
||||
if ($SshTarget) {
|
||||
Invoke-RemoteCheck "Host identity" "whoami; hostname; `$PSVersionTable.PSVersion.ToString()"
|
||||
Invoke-RemoteCheck "GPU driver" "nvidia-smi --query-gpu=name,memory.total,driver_version --format=csv,noheader"
|
||||
Invoke-RemoteCheck "Docker version" "docker version"
|
||||
Invoke-RemoteCheck "Docker compose" "docker compose version"
|
||||
Invoke-RemoteCheck "Docker info" "docker info --format '{{json .}}'"
|
||||
Invoke-RemoteCheck "Models drive" "if (-not (Test-Path '$ModelsRoot')) { New-Item -ItemType Directory -Force '$ModelsRoot' | Out-Null }; Get-Item '$ModelsRoot' | Select-Object FullName,Exists"
|
||||
Invoke-RemoteCheck "CUDA container" "docker run --rm --gpus all nvidia/cuda:12.4.1-base-ubuntu22.04 nvidia-smi --query-gpu=name,memory.total --format=csv,noheader"
|
||||
Write-Host ""
|
||||
Write-Host "Windows GPU host preflight completed through SSH."
|
||||
exit 0
|
||||
}
|
||||
|
||||
Invoke-LocalCheck "Host identity" {
|
||||
whoami
|
||||
hostname
|
||||
$PSVersionTable.PSVersion.ToString()
|
||||
}
|
||||
|
||||
Invoke-LocalCheck "GPU driver" {
|
||||
nvidia-smi --query-gpu=name,memory.total,driver_version --format=csv,noheader
|
||||
}
|
||||
|
||||
Invoke-LocalCheck "Docker version" {
|
||||
docker version
|
||||
}
|
||||
|
||||
Invoke-LocalCheck "Docker compose" {
|
||||
docker compose version
|
||||
}
|
||||
|
||||
Invoke-LocalCheck "Docker info" {
|
||||
docker info --format '{{json .}}'
|
||||
}
|
||||
|
||||
Invoke-LocalCheck "Models drive" {
|
||||
if (-not (Test-Path $ModelsRoot)) {
|
||||
New-Item -ItemType Directory -Force $ModelsRoot | Out-Null
|
||||
}
|
||||
Get-Item $ModelsRoot | Select-Object FullName,Exists
|
||||
}
|
||||
|
||||
Invoke-LocalCheck "CUDA container" {
|
||||
docker run --rm --gpus all nvidia/cuda:12.4.1-base-ubuntu22.04 nvidia-smi --query-gpu=name,memory.total --format=csv,noheader
|
||||
}
|
||||
|
||||
Write-Host ""
|
||||
Write-Host "Windows GPU host preflight completed."
|
||||
@@ -0,0 +1,191 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Describe extension manifest payloads by observable structure.
|
||||
|
||||
This script intentionally reports structural facts first. Semantic labels are
|
||||
only added when the evidence is direct, for example a payload contains BSL text
|
||||
or embedded HTML help.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import base64
|
||||
import json
|
||||
import re
|
||||
from collections import Counter, defaultdict
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from inspect_1c_sql_files import Lexer, Parser, collect_strings, tree_shape, try_decode, try_decompress
|
||||
|
||||
|
||||
BSL_MARKERS = ("&На", "Процедура ", "Функция ", "#Область", "#КонецОбласти")
|
||||
HTML_MARKERS = ("<!DOCTYPE HTML", "<html", "<HTML", "<body", "<BODY")
|
||||
|
||||
|
||||
def scalar(node: Any) -> str:
|
||||
if isinstance(node, dict) and node.get("type") in {"atom", "string"}:
|
||||
return str(node.get("value") or "")
|
||||
return ""
|
||||
|
||||
|
||||
def suffix_of(object_id: str) -> str:
|
||||
parts = object_id.split(".", 1)
|
||||
return "" if len(parts) == 1 else "." + parts[1]
|
||||
|
||||
|
||||
def parse_payload(path: Path) -> dict[str, Any]:
|
||||
raw = path.read_bytes()
|
||||
payload, compression = try_decompress(raw)
|
||||
text, encoding = try_decode(payload)
|
||||
result: dict[str, Any] = {
|
||||
"bytes": len(raw),
|
||||
"payload_bytes": len(payload),
|
||||
"compression": compression,
|
||||
"encoding": encoding,
|
||||
"parse_status": "not_text",
|
||||
"root_kind": "",
|
||||
"root_len": None,
|
||||
"root_marker": "",
|
||||
"strings_sample": [],
|
||||
"bsl_marker_count": 0,
|
||||
"html_marker_count": 0,
|
||||
"base64_atom_count": 0,
|
||||
"embedded_base64_html_count": 0,
|
||||
"semantic_evidence": [],
|
||||
}
|
||||
if text is None:
|
||||
return result
|
||||
clean = text.replace("\x00", "").replace("\ufeff", "").lstrip("ï»¿п»ї")
|
||||
result["bsl_marker_count"] = sum(clean.count(marker) for marker in BSL_MARKERS)
|
||||
result["html_marker_count"] = sum(clean.count(marker) for marker in HTML_MARKERS)
|
||||
try:
|
||||
parsed = Parser(Lexer(clean[:2_000_000]).tokens()).parse()
|
||||
except Exception as exc:
|
||||
result["parse_status"] = "parse_error"
|
||||
result["parse_error"] = str(exc)
|
||||
return result
|
||||
result["parse_status"] = "parsed"
|
||||
result["shape"] = tree_shape(parsed, max_depth=3)
|
||||
strings = collect_strings(parsed, limit=80)
|
||||
result["strings_sample"] = strings[:40]
|
||||
if isinstance(parsed, dict):
|
||||
result["root_kind"] = parsed.get("type") or ""
|
||||
items = parsed.get("items") or []
|
||||
result["root_len"] = len(items)
|
||||
if items:
|
||||
result["root_marker"] = scalar(items[0])
|
||||
atoms = []
|
||||
|
||||
def walk(node: Any) -> None:
|
||||
if isinstance(node, dict) and node.get("type") == "atom":
|
||||
atoms.append(str(node.get("value") or ""))
|
||||
if isinstance(node, dict):
|
||||
for child in node.get("items") or []:
|
||||
walk(child)
|
||||
|
||||
walk(parsed)
|
||||
b64_atoms = [value for value in atoms if re.fullmatch(r"[A-Za-z0-9+/]{40,}={0,2}", value)]
|
||||
result["base64_atom_count"] = len(b64_atoms)
|
||||
html_count = 0
|
||||
for value in b64_atoms[:200]:
|
||||
try:
|
||||
decoded = base64.b64decode(value, validate=False)
|
||||
except Exception:
|
||||
continue
|
||||
if any(marker.encode("utf-8") in decoded or marker.encode("cp1251", errors="ignore") in decoded for marker in HTML_MARKERS):
|
||||
html_count += 1
|
||||
result["embedded_base64_html_count"] = html_count
|
||||
evidence = []
|
||||
if result["bsl_marker_count"]:
|
||||
evidence.append("contains_bsl_text")
|
||||
if result["html_marker_count"] or html_count:
|
||||
evidence.append("contains_html")
|
||||
if result["base64_atom_count"]:
|
||||
evidence.append("contains_base64_atoms")
|
||||
result["semantic_evidence"] = evidence
|
||||
return result
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Classify manifest payloads by observed structure.")
|
||||
parser.add_argument("--manifest-dir", type=Path, required=True)
|
||||
parser.add_argument("--cas-dir", type=Path, required=True)
|
||||
parser.add_argument("--xml-index", type=Path)
|
||||
parser.add_argument("--output", type=Path, required=True)
|
||||
parser.add_argument("--limit", type=int, default=0)
|
||||
args = parser.parse_args()
|
||||
|
||||
xml_map = {}
|
||||
if args.xml_index and args.xml_index.is_file():
|
||||
xml = json.loads(args.xml_index.read_text(encoding="utf-8"))
|
||||
xml_map = xml.get("guid_map") or {}
|
||||
|
||||
entries = []
|
||||
for manifest_path in sorted(args.manifest_dir.glob("*.json")):
|
||||
manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
|
||||
extension_file = (manifest.get("extension_zipped_info") or {}).get("file_name") or manifest_path.name
|
||||
for entry in manifest.get("entries") or []:
|
||||
cas_path = Path(entry.get("cas_path") or args.cas_dir / entry["cas_key"])
|
||||
if not cas_path.is_file():
|
||||
continue
|
||||
object_id = entry["object_id"]
|
||||
base_guid = object_id.split(".", 1)[0].lower()
|
||||
payload = parse_payload(cas_path)
|
||||
xml_item = xml_map.get(base_guid) or {}
|
||||
top_objects = xml_item.get("top_objects") or []
|
||||
entries.append(
|
||||
{
|
||||
"extension_file": extension_file,
|
||||
"manifest_path": str(manifest_path),
|
||||
"object_id": object_id,
|
||||
"base_guid": base_guid,
|
||||
"suffix": suffix_of(object_id),
|
||||
"cas_key": entry["cas_key"],
|
||||
"xml_top_objects": top_objects[:5],
|
||||
"payload": payload,
|
||||
}
|
||||
)
|
||||
if args.limit and len(entries) >= args.limit:
|
||||
break
|
||||
if args.limit and len(entries) >= args.limit:
|
||||
break
|
||||
|
||||
suffix_counts = Counter(item["suffix"] for item in entries)
|
||||
suffix_root_counts: dict[str, Counter[str]] = defaultdict(Counter)
|
||||
suffix_evidence_counts: dict[str, Counter[str]] = defaultdict(Counter)
|
||||
for item in entries:
|
||||
suffix = item["suffix"]
|
||||
payload = item["payload"]
|
||||
root_signature = f"{payload.get('root_kind')}:{payload.get('root_marker')}:{payload.get('root_len')}"
|
||||
suffix_root_counts[suffix][root_signature] += 1
|
||||
for evidence in payload.get("semantic_evidence") or ["<none>"]:
|
||||
suffix_evidence_counts[suffix][evidence] += 1
|
||||
|
||||
report = {
|
||||
"schema": "onec_manifest_payload_structure.v1",
|
||||
"manifest_dir": str(args.manifest_dir),
|
||||
"cas_dir": str(args.cas_dir),
|
||||
"entry_count": len(entries),
|
||||
"suffix_counts": dict(sorted(suffix_counts.items())),
|
||||
"suffix_root_counts": {key: dict(value.most_common()) for key, value in sorted(suffix_root_counts.items())},
|
||||
"suffix_evidence_counts": {key: dict(value.most_common()) for key, value in sorted(suffix_evidence_counts.items())},
|
||||
"entries": entries,
|
||||
}
|
||||
args.output.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.output.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"output": str(args.output),
|
||||
"entries": len(entries),
|
||||
"suffixes": len(suffix_counts),
|
||||
},
|
||||
ensure_ascii=False,
|
||||
)
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,156 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import datetime as dt
|
||||
import json
|
||||
import socket
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
DEFAULT_REPORT = ROOT / "reports" / "platform-status.json"
|
||||
DEFAULT_DOCKER_HOST = "ssh://docker-gpu"
|
||||
MODEL_CHAT_CONTAINER = "llm-model-chat-ui"
|
||||
|
||||
|
||||
def run_json(command: list[str], timeout: int = 60) -> dict[str, Any]:
|
||||
result = subprocess.run(
|
||||
command,
|
||||
cwd=ROOT,
|
||||
text=True,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
timeout=timeout,
|
||||
check=False,
|
||||
)
|
||||
payload = None
|
||||
if result.stdout.strip().startswith("{"):
|
||||
try:
|
||||
payload = json.loads(result.stdout)
|
||||
except json.JSONDecodeError:
|
||||
payload = None
|
||||
return {
|
||||
"command": command,
|
||||
"status": "ok" if result.returncode == 0 else "failed",
|
||||
"returncode": result.returncode,
|
||||
"stdout": result.stdout.strip(),
|
||||
"stderr": result.stderr.strip(),
|
||||
"json": payload,
|
||||
}
|
||||
|
||||
|
||||
def read_report(path: Path) -> dict[str, Any] | None:
|
||||
if not path.exists():
|
||||
return None
|
||||
try:
|
||||
with path.open("r", encoding="utf-8") as handle:
|
||||
data = json.load(handle)
|
||||
return data if isinstance(data, dict) else None
|
||||
except json.JSONDecodeError:
|
||||
return None
|
||||
|
||||
|
||||
def is_port_open(host: str, port: int, timeout: float = 1.0) -> bool:
|
||||
try:
|
||||
with socket.create_connection((host, port), timeout=timeout):
|
||||
return True
|
||||
except OSError:
|
||||
return False
|
||||
|
||||
|
||||
def deployed_model_storage(docker_host: str) -> dict[str, Any]:
|
||||
return run_json(
|
||||
[
|
||||
"docker",
|
||||
"-H",
|
||||
docker_host,
|
||||
"exec",
|
||||
MODEL_CHAT_CONTAINER,
|
||||
"python3",
|
||||
"scripts/check_model_storage.py",
|
||||
"--models-root",
|
||||
"/models",
|
||||
"--print",
|
||||
"--no-report",
|
||||
"--warn-only",
|
||||
],
|
||||
timeout=90,
|
||||
)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Collect local platform status into one report.")
|
||||
parser.add_argument("--report", type=Path, default=DEFAULT_REPORT)
|
||||
parser.add_argument("--print", action="store_true")
|
||||
parser.add_argument("--check-endpoints", action="store_true", help="Run GPU readiness check with network timeouts.")
|
||||
parser.add_argument("--docker-host", default=DEFAULT_DOCKER_HOST, help="Docker endpoint for deployed GPU checks.")
|
||||
parser.add_argument(
|
||||
"--skip-deployed-storage",
|
||||
action="store_true",
|
||||
help="Skip model storage check inside the deployed model-chat container.",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
model_storage = run_json(
|
||||
[sys.executable, "scripts/check_model_storage.py", "--print", "--no-report", "--warn-only"],
|
||||
timeout=60,
|
||||
)
|
||||
if model_storage.get("json", {}).get("status") == "failed":
|
||||
model_storage["status"] = "blocked"
|
||||
|
||||
checks = {
|
||||
"model_cards": run_json([sys.executable, "scripts/validate_model_cards.py"], timeout=30),
|
||||
"eval_files": run_json([sys.executable, "scripts/validate_evals.py"], timeout=30),
|
||||
"model_storage": model_storage,
|
||||
"deployed_model_storage": None if args.skip_deployed_storage else deployed_model_storage(args.docker_host),
|
||||
"1c_plugin": run_json([sys.executable, "scripts/check_1c_plugin.py", "--no-report"], timeout=60),
|
||||
"model_chat": {
|
||||
"status": "ok" if is_port_open("127.0.0.1", 8765) else "stopped",
|
||||
"url": "http://127.0.0.1:8765",
|
||||
},
|
||||
"gpu_readiness": read_report(ROOT / "reports" / "gpu-readiness.json"),
|
||||
"live_model_evals": read_report(ROOT / "reports" / "evals" / "live-model-evals.json"),
|
||||
}
|
||||
if args.check_endpoints:
|
||||
checks["gpu_readiness"] = run_json(
|
||||
[sys.executable, "scripts/check_gpu_readiness.py", "--print", "--timeout", "8"],
|
||||
timeout=45,
|
||||
).get("json")
|
||||
if isinstance(checks["gpu_readiness"], dict) and checks["gpu_readiness"].get("status") == "failed":
|
||||
checks["gpu_readiness"]["status"] = "blocked"
|
||||
|
||||
failed = []
|
||||
blocked = []
|
||||
for name, check in checks.items():
|
||||
if not check:
|
||||
if name != "deployed_model_storage":
|
||||
blocked.append(name)
|
||||
continue
|
||||
status = check.get("status")
|
||||
if status == "failed":
|
||||
failed.append(name)
|
||||
elif status in {"blocked", "stopped"}:
|
||||
blocked.append(name)
|
||||
|
||||
report = {
|
||||
"created_at": dt.datetime.now(dt.UTC).isoformat(),
|
||||
"status": "failed" if failed else "blocked" if blocked else "ok",
|
||||
"failed": failed,
|
||||
"blocked": blocked,
|
||||
"checks": checks,
|
||||
}
|
||||
args.report.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.report.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
if args.print:
|
||||
print(json.dumps(report, ensure_ascii=False, indent=2))
|
||||
else:
|
||||
print(f"Platform status: {report['status']}")
|
||||
print(f"Wrote report to {args.report}")
|
||||
return 1 if failed else 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,366 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import math
|
||||
import re
|
||||
import urllib.request
|
||||
import struct
|
||||
from collections import Counter
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import yaml
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
MODEL_CARDS_DIR = ROOT / "registry" / "model-cards"
|
||||
TOKEN_RE = re.compile(r"[A-Za-zА-Яа-яЁё0-9_]+", re.UNICODE)
|
||||
RUSSIAN_ENDINGS = (
|
||||
"иями",
|
||||
"ями",
|
||||
"ами",
|
||||
"ого",
|
||||
"ему",
|
||||
"ыми",
|
||||
"ими",
|
||||
"ой",
|
||||
"ей",
|
||||
"ых",
|
||||
"их",
|
||||
"ую",
|
||||
"юю",
|
||||
"ая",
|
||||
"яя",
|
||||
"ое",
|
||||
"ее",
|
||||
"ом",
|
||||
"ем",
|
||||
"ам",
|
||||
"ям",
|
||||
"ах",
|
||||
"ях",
|
||||
"ы",
|
||||
"и",
|
||||
"а",
|
||||
"я",
|
||||
"е",
|
||||
"у",
|
||||
"ю",
|
||||
)
|
||||
TOKEN_ALIASES = {
|
||||
"1с": ["1c", "bsl", "конфигурация"],
|
||||
"1c": ["1с", "bsl", "configuration"],
|
||||
"бсл": ["bsl", "1с"],
|
||||
"bsl": ["бсл", "1с"],
|
||||
"справочник": ["catalog", "справочники"],
|
||||
"справочники": ["справочник", "catalog"],
|
||||
"документ": ["documents", "документы"],
|
||||
"документы": ["документ", "documents"],
|
||||
"регистр": ["register", "регистры"],
|
||||
"регистры": ["регистр", "register"],
|
||||
"реквизит": ["attribute", "реквизиты"],
|
||||
"реквизиты": ["реквизит", "attribute"],
|
||||
"табличная": ["табличные", "часть"],
|
||||
"табличные": ["табличная", "часть"],
|
||||
"запрос": ["query", "read", "select", "выбрать"],
|
||||
"выбрать": ["запрос", "query", "select"],
|
||||
"форма": ["forms", "управляемая"],
|
||||
"модуль": ["module", "bsl"],
|
||||
"метаданные": ["metadata", "схема", "snapshot"],
|
||||
"схема": ["metadata", "метаданные"],
|
||||
"номенклатура": ["справочник", "catalog"],
|
||||
}
|
||||
|
||||
|
||||
def read_json(path: Path) -> dict:
|
||||
with path.open("r", encoding="utf-8-sig") as handle:
|
||||
data = json.load(handle)
|
||||
if not isinstance(data, dict):
|
||||
raise ValueError(f"{path} must contain a JSON object")
|
||||
return data
|
||||
|
||||
|
||||
def read_jsonl(path: Path) -> list[dict]:
|
||||
if not path.exists():
|
||||
return []
|
||||
|
||||
records: list[dict] = []
|
||||
with path.open("r", encoding="utf-8") as handle:
|
||||
for line_number, line in enumerate(handle, start=1):
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
try:
|
||||
record = json.loads(line)
|
||||
except json.JSONDecodeError as exc:
|
||||
raise ValueError(f"{path}:{line_number}: invalid JSONL: {exc}") from exc
|
||||
if not isinstance(record, dict):
|
||||
raise ValueError(f"{path}:{line_number}: record must be an object")
|
||||
records.append(record)
|
||||
return records
|
||||
|
||||
|
||||
def write_json(path: Path, data: dict) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(json.dumps(data, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
|
||||
|
||||
def read_yaml_mapping(path: Path) -> dict[str, Any]:
|
||||
with path.open("r", encoding="utf-8") as handle:
|
||||
data = yaml.safe_load(handle)
|
||||
if not isinstance(data, dict):
|
||||
raise ValueError(f"{path} must contain a YAML mapping")
|
||||
return data
|
||||
|
||||
|
||||
def iter_model_card_paths(*, include_examples: bool = False) -> list[Path]:
|
||||
paths = sorted(MODEL_CARDS_DIR.rglob("*.yaml")) + sorted(MODEL_CARDS_DIR.rglob("*.yml"))
|
||||
if include_examples:
|
||||
return paths
|
||||
return [path for path in paths if "examples" not in path.parts]
|
||||
|
||||
|
||||
def load_model_card(card_id: str) -> dict[str, Any]:
|
||||
for suffix in (".yaml", ".yml"):
|
||||
path = MODEL_CARDS_DIR / f"{card_id}{suffix}"
|
||||
if path.exists():
|
||||
return read_yaml_mapping(path)
|
||||
raise FileNotFoundError(f"Model card not found for id `{card_id}` in {MODEL_CARDS_DIR}")
|
||||
|
||||
|
||||
def localize_workspace_path(path: str) -> Path:
|
||||
if path.startswith("/workspace/"):
|
||||
if Path("/workspace").exists():
|
||||
return Path(path)
|
||||
return ROOT / path.removeprefix("/workspace/")
|
||||
if path.startswith("/models/"):
|
||||
if Path("/models").exists():
|
||||
return Path(path)
|
||||
return ROOT / "models" / path.removeprefix("/models/")
|
||||
return Path(path)
|
||||
|
||||
|
||||
def stem_russian_token(token: str) -> str:
|
||||
if not re.search(r"[а-яё]", token, flags=re.IGNORECASE) or len(token) < 6:
|
||||
return token
|
||||
for ending in RUSSIAN_ENDINGS:
|
||||
if token.endswith(ending) and len(token) - len(ending) >= 4:
|
||||
return token[: -len(ending)]
|
||||
return token
|
||||
|
||||
|
||||
def normalize_token(token: str) -> str:
|
||||
token = token.lower().replace("ё", "е")
|
||||
return stem_russian_token(token)
|
||||
|
||||
|
||||
def tokenize(text: str, *, expand_aliases: bool = False) -> list[str]:
|
||||
tokens: list[str] = []
|
||||
for raw_token in TOKEN_RE.findall(text):
|
||||
token = normalize_token(raw_token)
|
||||
tokens.append(token)
|
||||
if expand_aliases:
|
||||
tokens.extend(normalize_token(alias) for alias in TOKEN_ALIASES.get(token, []))
|
||||
return tokens
|
||||
|
||||
|
||||
def corpus_content_hash(records: list[dict]) -> str:
|
||||
import hashlib
|
||||
|
||||
hasher = hashlib.sha256()
|
||||
for record in records:
|
||||
stable = {
|
||||
"id": record.get("id"),
|
||||
"document_id": record.get("document_id"),
|
||||
"source_path": record.get("source_path"),
|
||||
"chunk_index": record.get("chunk_index"),
|
||||
"title": record.get("title"),
|
||||
"content": record.get("content"),
|
||||
"metadata": record.get("metadata") or {},
|
||||
}
|
||||
hasher.update(json.dumps(stable, ensure_ascii=False, sort_keys=True, separators=(",", ":")).encode("utf-8"))
|
||||
hasher.update(b"\n")
|
||||
return hasher.hexdigest()
|
||||
|
||||
|
||||
def hashing_embedding(text: str, *, dimensions: int = 384) -> list[float]:
|
||||
import hashlib
|
||||
|
||||
if dimensions < 8:
|
||||
raise ValueError("dimensions must be >= 8")
|
||||
vector = [0.0] * dimensions
|
||||
tokens = tokenize(text, expand_aliases=True)
|
||||
if not tokens:
|
||||
return vector
|
||||
counts = Counter(tokens)
|
||||
for token, count in counts.items():
|
||||
digest = hashlib.blake2b(token.encode("utf-8"), digest_size=8).digest()
|
||||
bucket = int.from_bytes(digest[:4], "little") % dimensions
|
||||
sign = 1.0 if digest[4] & 1 else -1.0
|
||||
vector[bucket] += sign * (1.0 + math.log(float(count)))
|
||||
norm = math.sqrt(sum(value * value for value in vector))
|
||||
if norm <= 0:
|
||||
return vector
|
||||
return [value / norm for value in vector]
|
||||
|
||||
|
||||
def pack_float_vector(vector: list[float]) -> bytes:
|
||||
return struct.pack(f"<{len(vector)}f", *vector)
|
||||
|
||||
|
||||
def unpack_float_vector(data: bytes, dimensions: int) -> list[float]:
|
||||
expected_size = dimensions * 4
|
||||
if len(data) != expected_size:
|
||||
raise ValueError(f"Vector blob has {len(data)} bytes, expected {expected_size}")
|
||||
return list(struct.unpack(f"<{dimensions}f", data))
|
||||
|
||||
|
||||
def cosine_similarity(left: list[float], right: list[float]) -> float:
|
||||
if not left or not right or len(left) != len(right):
|
||||
return 0.0
|
||||
return float(sum(a * b for a, b in zip(left, right)))
|
||||
|
||||
|
||||
def build_lexical_index(records: list[dict]) -> dict:
|
||||
documents = []
|
||||
document_frequency: Counter[str] = Counter()
|
||||
total_length = 0
|
||||
|
||||
for record in records:
|
||||
content = record.get("content") or ""
|
||||
tokens = tokenize(content)
|
||||
title_tokens = tokenize(str(record.get("title") or ""))
|
||||
term_frequency = Counter(tokens)
|
||||
document_frequency.update(term_frequency.keys())
|
||||
total_length += len(tokens)
|
||||
documents.append(
|
||||
{
|
||||
"id": record.get("id"),
|
||||
"document_id": record.get("document_id"),
|
||||
"source_path": record.get("source_path"),
|
||||
"title": record.get("title"),
|
||||
"chunk_index": record.get("chunk_index"),
|
||||
"content": content,
|
||||
"metadata": record.get("metadata") or {},
|
||||
"length": len(tokens),
|
||||
"title_tokens": title_tokens,
|
||||
"term_frequency": dict(term_frequency),
|
||||
}
|
||||
)
|
||||
|
||||
doc_count = len(documents)
|
||||
idf = {
|
||||
token: math.log((1 + doc_count) / (1 + frequency)) + 1
|
||||
for token, frequency in document_frequency.items()
|
||||
}
|
||||
|
||||
return {
|
||||
"schema_version": 2,
|
||||
"type": "lexical-bm25",
|
||||
"doc_count": doc_count,
|
||||
"avg_doc_length": round(total_length / doc_count, 4) if doc_count else 0,
|
||||
"idf": idf,
|
||||
"documents": documents,
|
||||
}
|
||||
|
||||
|
||||
def score_lexical_document(query_tf: Counter[str], document: dict, idf: dict, avg_doc_length: float) -> float:
|
||||
doc_tf = document.get("term_frequency") or {}
|
||||
doc_length = max(float(document.get("length") or 0), 1.0)
|
||||
avg_doc_length = max(float(avg_doc_length or doc_length), 1.0)
|
||||
title_tokens = set(document.get("title_tokens") or [])
|
||||
score = 0.0
|
||||
k1 = 1.4
|
||||
b = 0.72
|
||||
for token, query_count in query_tf.items():
|
||||
doc_count = doc_tf.get(token, 0)
|
||||
if doc_count:
|
||||
numerator = doc_count * (k1 + 1)
|
||||
denominator = doc_count + k1 * (1 - b + b * (doc_length / avg_doc_length))
|
||||
score += query_count * float(idf.get(token, 1.0)) * (numerator / denominator)
|
||||
if token in title_tokens:
|
||||
score += 0.35 * query_count
|
||||
return score
|
||||
|
||||
|
||||
def search_lexical_index(
|
||||
index: dict,
|
||||
query: str,
|
||||
limit: int,
|
||||
*,
|
||||
candidate_limit: int | None = None,
|
||||
dedupe_by_document: bool = False,
|
||||
min_score: float = 0.0,
|
||||
source_types: list[str] | None = None,
|
||||
metadata_filters: dict[str, str] | None = None,
|
||||
) -> list[dict]:
|
||||
query_tf = Counter(tokenize(query, expand_aliases=True))
|
||||
if not query_tf:
|
||||
return []
|
||||
|
||||
results = []
|
||||
idf = index.get("idf") or {}
|
||||
avg_doc_length = float(index.get("avg_doc_length") or 0)
|
||||
allowed_source_types = {source_type.lower() for source_type in source_types or []}
|
||||
exact_filters = {key: str(value).lower() for key, value in (metadata_filters or {}).items() if str(value).strip()}
|
||||
for document in index.get("documents") or []:
|
||||
metadata = document.get("metadata") or {}
|
||||
source_type = str(metadata.get("source_type") or "").lower()
|
||||
if allowed_source_types and source_type not in allowed_source_types:
|
||||
continue
|
||||
if exact_filters and any(str(metadata.get(key) or "").lower() != value for key, value in exact_filters.items()):
|
||||
continue
|
||||
score = score_lexical_document(query_tf, document, idf, avg_doc_length)
|
||||
if score > min_score:
|
||||
results.append({"score": score, "document": document})
|
||||
|
||||
results.sort(key=lambda item: item["score"], reverse=True)
|
||||
if candidate_limit:
|
||||
results = results[:candidate_limit]
|
||||
if dedupe_by_document:
|
||||
deduped = []
|
||||
seen = set()
|
||||
for result in results:
|
||||
document_id = result["document"].get("document_id") or result["document"].get("source_path")
|
||||
if document_id in seen:
|
||||
continue
|
||||
seen.add(document_id)
|
||||
deduped.append(result)
|
||||
results = deduped
|
||||
return results[:limit]
|
||||
|
||||
|
||||
def call_chat_completion(
|
||||
base_url: str,
|
||||
model: str,
|
||||
messages: list[dict],
|
||||
*,
|
||||
temperature: float = 0.2,
|
||||
max_tokens: int = 1000,
|
||||
timeout: int = 180,
|
||||
) -> str:
|
||||
url = f"{base_url.rstrip('/')}/v1/chat/completions"
|
||||
payload = {
|
||||
"model": model,
|
||||
"messages": messages,
|
||||
"temperature": temperature,
|
||||
"max_tokens": max_tokens,
|
||||
}
|
||||
request = urllib.request.Request(
|
||||
url,
|
||||
data=json.dumps(payload, ensure_ascii=False).encode("utf-8"),
|
||||
headers={"Content-Type": "application/json"},
|
||||
method="POST",
|
||||
)
|
||||
with urllib.request.urlopen(request, timeout=timeout) as response:
|
||||
data = json.loads(response.read().decode("utf-8"))
|
||||
|
||||
choices = data.get("choices") or []
|
||||
if not choices:
|
||||
raise ValueError("chat response has no choices")
|
||||
|
||||
message = choices[0].get("message") or {}
|
||||
content = (message.get("content") or "").strip()
|
||||
if not content:
|
||||
raise ValueError("chat response content is empty")
|
||||
return content
|
||||
@@ -0,0 +1,249 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import html
|
||||
import json
|
||||
import re
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
DEFAULT_REPORT_ROOT = ROOT / "reports" / "1c-access"
|
||||
|
||||
|
||||
def slugify(value: str, *, max_length: int = 80) -> str:
|
||||
slug = re.sub(r"[^0-9A-Za-zА-Яа-яЁё._-]+", "-", value.strip())
|
||||
slug = re.sub(r"-+", "-", slug).strip("-._")
|
||||
return (slug or "role-audit")[:max_length]
|
||||
|
||||
|
||||
def esc(value: Any) -> str:
|
||||
return html.escape("" if value is None else str(value), quote=True)
|
||||
|
||||
|
||||
def read_json(path: Path) -> dict[str, Any]:
|
||||
data = json.loads(path.read_text(encoding="utf-8-sig"))
|
||||
if not isinstance(data, dict):
|
||||
raise ValueError(f"JSON root is not an object: {path}")
|
||||
return data
|
||||
|
||||
|
||||
def load_export(path: Path) -> dict[str, Any]:
|
||||
data = read_json(path)
|
||||
artifacts = data.get("artifacts") if isinstance(data.get("artifacts"), dict) else {}
|
||||
export_path = artifacts.get("json")
|
||||
if export_path:
|
||||
return read_json(Path(export_path))
|
||||
return data
|
||||
|
||||
|
||||
def summary_role(summary: dict[str, Any]) -> str | None:
|
||||
query = summary.get("query") if isinstance(summary.get("query"), dict) else {}
|
||||
role = query.get("role") if isinstance(query, dict) else None
|
||||
return str(role) if role is not None else None
|
||||
|
||||
|
||||
def find_latest_summaries(report_root: Path, base_id: str, *, role: str | None = None, count: int = 2) -> list[Path]:
|
||||
folder = report_root / slugify(base_id, max_length=60)
|
||||
summaries: list[tuple[str, Path]] = []
|
||||
role_filter = role.casefold() if role else None
|
||||
for path in folder.glob("*.summary.json"):
|
||||
try:
|
||||
summary = read_json(path)
|
||||
except (OSError, json.JSONDecodeError, ValueError):
|
||||
continue
|
||||
summary_role_value = summary_role(summary)
|
||||
if role_filter is not None and (summary_role_value or "").casefold() != role_filter:
|
||||
continue
|
||||
generated_at = str(summary.get("generated_at") or "")
|
||||
summaries.append((generated_at, path))
|
||||
summaries.sort(key=lambda item: item[0], reverse=True)
|
||||
return [path for _, path in summaries[:count]]
|
||||
|
||||
|
||||
def user_key(row: dict[str, Any]) -> str:
|
||||
value = row.get("user_id") or row.get("user_name")
|
||||
return str(value or "").strip()
|
||||
|
||||
|
||||
def row_user(row: dict[str, Any]) -> dict[str, Any]:
|
||||
return {
|
||||
"user_id": row.get("user_id"),
|
||||
"user_name": row.get("user_name"),
|
||||
"user_type": row.get("user_type"),
|
||||
"user_active": row.get("user_active"),
|
||||
"user_marked": row.get("user_marked"),
|
||||
}
|
||||
|
||||
|
||||
def group_rows_by_user(export: dict[str, Any]) -> dict[str, dict[str, Any]]:
|
||||
users: dict[str, dict[str, Any]] = {}
|
||||
rows = export.get("rows") if isinstance(export.get("rows"), list) else []
|
||||
for row in rows:
|
||||
if not isinstance(row, dict):
|
||||
continue
|
||||
key = user_key(row)
|
||||
if not key:
|
||||
continue
|
||||
item = users.setdefault(key, {"user": row_user(row), "access_paths": set(), "rows": []})
|
||||
if row.get("access_path"):
|
||||
item["access_paths"].add(str(row.get("access_path")))
|
||||
item["rows"].append(row)
|
||||
for item in users.values():
|
||||
item["access_paths"] = sorted(item["access_paths"])
|
||||
return users
|
||||
|
||||
|
||||
def compare_exports(old_export: dict[str, Any], new_export: dict[str, Any]) -> dict[str, Any]:
|
||||
old_users = group_rows_by_user(old_export)
|
||||
new_users = group_rows_by_user(new_export)
|
||||
old_keys = set(old_users)
|
||||
new_keys = set(new_users)
|
||||
added_keys = sorted(new_keys - old_keys)
|
||||
removed_keys = sorted(old_keys - new_keys)
|
||||
common_keys = sorted(old_keys & new_keys)
|
||||
changed_paths = [
|
||||
{
|
||||
"user": new_users[key]["user"],
|
||||
"old_access_paths": old_users[key]["access_paths"],
|
||||
"new_access_paths": new_users[key]["access_paths"],
|
||||
}
|
||||
for key in common_keys
|
||||
if old_users[key]["access_paths"] != new_users[key]["access_paths"]
|
||||
]
|
||||
return {
|
||||
"schema": "onec_access_role_audit_compare.v1",
|
||||
"status": "ok",
|
||||
"generated_at": datetime.now(timezone.utc).isoformat(),
|
||||
"counts": {
|
||||
"old_users": len(old_keys),
|
||||
"new_users": len(new_keys),
|
||||
"added_users": len(added_keys),
|
||||
"removed_users": len(removed_keys),
|
||||
"unchanged_users": len(common_keys),
|
||||
"changed_access_paths": len(changed_paths),
|
||||
},
|
||||
"added_users": [new_users[key]["user"] for key in added_keys],
|
||||
"removed_users": [old_users[key]["user"] for key in removed_keys],
|
||||
"changed_access_paths": changed_paths,
|
||||
}
|
||||
|
||||
|
||||
def render_html_report(compare: dict[str, Any]) -> str:
|
||||
counts = compare.get("counts") if isinstance(compare.get("counts"), dict) else {}
|
||||
|
||||
def user_rows(name: str) -> str:
|
||||
users = compare.get(name) if isinstance(compare.get(name), list) else []
|
||||
return "\n".join(
|
||||
"<tr>"
|
||||
f"<td>{esc(item.get('user_name'))}</td>"
|
||||
f"<td>{esc(item.get('user_id'))}</td>"
|
||||
f"<td>{esc(item.get('user_type'))}</td>"
|
||||
f"<td>{esc(item.get('user_active'))}</td>"
|
||||
f"<td>{esc(item.get('user_marked'))}</td>"
|
||||
"</tr>"
|
||||
for item in users
|
||||
if isinstance(item, dict)
|
||||
) or "<tr><td colspan=\"5\">Absent.</td></tr>"
|
||||
|
||||
changed = compare.get("changed_access_paths") if isinstance(compare.get("changed_access_paths"), list) else []
|
||||
changed_rows = "\n".join(
|
||||
"<tr>"
|
||||
f"<td>{esc((item.get('user') or {}).get('user_name') if isinstance(item.get('user'), dict) else None)}</td>"
|
||||
f"<td><pre>{esc('\\n'.join(item.get('old_access_paths') or []))}</pre></td>"
|
||||
f"<td><pre>{esc('\\n'.join(item.get('new_access_paths') or []))}</pre></td>"
|
||||
"</tr>"
|
||||
for item in changed
|
||||
if isinstance(item, dict)
|
||||
) or "<tr><td colspan=\"3\">Absent.</td></tr>"
|
||||
|
||||
return f"""<!doctype html>
|
||||
<html lang="ru">
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<title>1C Access Audit Compare</title>
|
||||
<style>
|
||||
body {{ font-family: Segoe UI, Arial, sans-serif; margin: 24px; color: #1f2933; }}
|
||||
.cards {{ display: flex; flex-wrap: wrap; gap: 12px; margin: 18px 0; }}
|
||||
.card {{ border: 1px solid #d9e2ec; border-radius: 8px; padding: 12px 14px; min-width: 130px; }}
|
||||
.card b {{ display: block; font-size: 20px; }}
|
||||
table {{ border-collapse: collapse; width: 100%; margin: 12px 0 24px; }}
|
||||
th, td {{ border: 1px solid #d9e2ec; padding: 8px; vertical-align: top; }}
|
||||
th {{ background: #f0f4f8; text-align: left; }}
|
||||
pre {{ white-space: pre-wrap; margin: 0; font-family: Consolas, monospace; }}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<h1>1C Access Audit Compare</h1>
|
||||
<div>Generated: {esc(compare.get('generated_at'))}</div>
|
||||
<div class="cards">
|
||||
<div class="card"><b>{esc(counts.get('old_users'))}</b>old users</div>
|
||||
<div class="card"><b>{esc(counts.get('new_users'))}</b>new users</div>
|
||||
<div class="card"><b>{esc(counts.get('added_users'))}</b>added</div>
|
||||
<div class="card"><b>{esc(counts.get('removed_users'))}</b>removed</div>
|
||||
<div class="card"><b>{esc(counts.get('changed_access_paths'))}</b>path changes</div>
|
||||
</div>
|
||||
<h2>Added users</h2>
|
||||
<table><thead><tr><th>User</th><th>ID</th><th>Type</th><th>Active</th><th>Marked</th></tr></thead><tbody>{user_rows('added_users')}</tbody></table>
|
||||
<h2>Removed users</h2>
|
||||
<table><thead><tr><th>User</th><th>ID</th><th>Type</th><th>Active</th><th>Marked</th></tr></thead><tbody>{user_rows('removed_users')}</tbody></table>
|
||||
<h2>Changed access paths</h2>
|
||||
<table><thead><tr><th>User</th><th>Old paths</th><th>New paths</th></tr></thead><tbody>{changed_rows}</tbody></table>
|
||||
</body>
|
||||
</html>
|
||||
"""
|
||||
|
||||
|
||||
def compare_files(old_path: Path, new_path: Path, *, output: Path | None = None, html_output: Path | None = None) -> dict[str, Any]:
|
||||
result = compare_exports(load_export(old_path), load_export(new_path))
|
||||
result["sources"] = {"old": str(old_path), "new": str(new_path)}
|
||||
result["artifacts"] = {
|
||||
**({"json": str(output)} if output is not None else {}),
|
||||
**({"html": str(html_output)} if html_output is not None else {}),
|
||||
}
|
||||
if output is not None:
|
||||
output.parent.mkdir(parents=True, exist_ok=True)
|
||||
output.write_text(json.dumps(result, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
if html_output is not None:
|
||||
html_output.parent.mkdir(parents=True, exist_ok=True)
|
||||
html_output.write_text(render_html_report(result), encoding="utf-8")
|
||||
return result
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Compare two 1C access role audit exports or summaries.")
|
||||
parser.add_argument("old", type=Path, nargs="?")
|
||||
parser.add_argument("new", type=Path, nargs="?")
|
||||
parser.add_argument("--latest", action="store_true", help="Compare the latest two summaries for a base/role.")
|
||||
parser.add_argument("--base-id", default="upo_test")
|
||||
parser.add_argument("--role")
|
||||
parser.add_argument("--report-root", type=Path, default=DEFAULT_REPORT_ROOT)
|
||||
parser.add_argument("--output", type=Path)
|
||||
parser.add_argument("--html", type=Path)
|
||||
args = parser.parse_args()
|
||||
|
||||
old_path = args.old
|
||||
new_path = args.new
|
||||
output = args.output
|
||||
html_output = args.html
|
||||
if args.latest:
|
||||
latest = find_latest_summaries(args.report_root, args.base_id, role=args.role, count=2)
|
||||
if len(latest) < 2:
|
||||
parser.error("Not enough matching summaries for --latest; need at least two.")
|
||||
new_path, old_path = latest[0], latest[1]
|
||||
folder = args.report_root / slugify(args.base_id, max_length=60)
|
||||
stem = f"compare-{old_path.stem.replace('.summary', '')}-{new_path.stem.replace('.summary', '')}"
|
||||
output = output or folder / f"{stem}.json"
|
||||
html_output = html_output or folder / f"{stem}.html"
|
||||
if old_path is None or new_path is None:
|
||||
parser.error("Either provide OLD and NEW paths or use --latest.")
|
||||
|
||||
result = compare_files(old_path, new_path, output=output, html_output=html_output)
|
||||
print(json.dumps(result, ensure_ascii=False, indent=2))
|
||||
return 0 if result.get("status") == "ok" else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,633 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Compare decoded SQL form semantics with exported 1C Form.xml semantics."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import sys
|
||||
from collections import Counter
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "plugins" / "1c"))
|
||||
|
||||
from parser.form_payload import enrich_form_common_semantic, form_common_semantic # noqa: E402
|
||||
from parser.form_xml import decode_form_xml # noqa: E402
|
||||
|
||||
|
||||
PROPERTY_NAME_ALIASES = {
|
||||
"Action": "Действие",
|
||||
"MainAttribute": "ОсновнойРеквизит",
|
||||
"AutoEditMode": "АвтоРежимРедактирования",
|
||||
"HeightInTableRows": "ВысотаВСтрокахТаблицы",
|
||||
"RowSelectionMode": "РежимВыделенияСтроки",
|
||||
"HorizontalLinesBWA": "ГоризонтальныеЛинии",
|
||||
"VerticalLinesBWA": "ВертикальныеЛинии",
|
||||
"UseAlternationRowColorBWA": "ЧередованиеЦветовСтрок",
|
||||
"AutoInsertNewRow": "АвтоВставкаНовойСтроки",
|
||||
"EnableStartDrag": "РазрешитьНачалоПеретаскивания",
|
||||
"EnableDrag": "РазрешитьПеретаскивание",
|
||||
"FileDragMode": "РежимПеретаскиванияФайлов",
|
||||
"CommandBarLocation": "ПоложениеКоманднойПанели",
|
||||
"DefaultItem": "АктивизироватьПоУмолчанию",
|
||||
"Autofill": "Автозаполнение",
|
||||
"AutoMaxWidth": "АвтоМаксимальнаяШирина",
|
||||
"MaxWidth": "МаксимальнаяШирина",
|
||||
"MultiLine": "МногострочныйРежим",
|
||||
"AutoCommandBar": "АвтоКоманднаяПанель",
|
||||
"SearchStringAddition": "ДополнениеСтрокиПоиска",
|
||||
"ViewStatusAddition": "ДополнениеСостоянияПросмотра",
|
||||
"SearchControlAddition": "ДополнениеУправленияПоиском",
|
||||
"Width": "Ширина",
|
||||
"Height": "Высота",
|
||||
"HorizontalStretch": "РастягиватьПоГоризонтали",
|
||||
"VerticalStretch": "РастягиватьПоВертикали",
|
||||
"TextColor": "ЦветТекста",
|
||||
"BackColor": "ЦветФона",
|
||||
"HorizontalAlign": "ГоризонтальноеПоложениеВГруппе",
|
||||
"AutoMaxHeight": "АвтоМаксимальнаяВысота",
|
||||
"MaxHeight": "МаксимальнаяВысота",
|
||||
"AutoMarkIncomplete": "АвтоОтметкаНезаполненного",
|
||||
"ToolTipRepresentation": "ОтображениеПодсказки",
|
||||
"SpinButton": "КнопкаРегулирования",
|
||||
"Representation": "Отображение",
|
||||
"DefaultButton": "КнопкаПоУмолчанию",
|
||||
"OpenButton": "КнопкаОткрытия",
|
||||
"CreateButton": "КнопкаСоздания",
|
||||
"ChoiceHistoryOnInput": "ИсторияВыбораПриВводе",
|
||||
"BorderColor": "ЦветРамки",
|
||||
"ChangeRowSet": "ИзменятьСоставСтрок",
|
||||
"ShowInHeader": "ОтображатьВШапке",
|
||||
"AutoCellHeight": "АвтоВысотаЯчейки",
|
||||
"SearchStringLocation": "ПоложениеСтрокиПоиска",
|
||||
"ViewStatusLocation": "ПоложениеСостоянияПросмотра",
|
||||
"SearchControlLocation": "ПоложениеУправленияПоиском",
|
||||
"GroupHorizontalAlign": "ГоризонтальноеПоложениеВГруппе",
|
||||
"GroupVerticalAlign": "ВертикальноеПоложениеВГруппе",
|
||||
"ShapeRepresentation": "ОтображениеФигуры",
|
||||
}
|
||||
|
||||
|
||||
def read_json(path: Path) -> dict[str, Any]:
|
||||
data = json.loads(path.read_text(encoding="utf-8-sig"))
|
||||
if not isinstance(data, dict):
|
||||
raise ValueError(f"{path} must contain a JSON object")
|
||||
return data
|
||||
|
||||
|
||||
def write_json(path: Path, data: dict[str, Any]) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(json.dumps(data, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
|
||||
|
||||
def scalar_text(value: Any) -> str:
|
||||
if value is True:
|
||||
return "true"
|
||||
if value is False:
|
||||
return "false"
|
||||
if value is None:
|
||||
return ""
|
||||
return str(value).strip()
|
||||
|
||||
|
||||
def comparable(value: Any) -> str:
|
||||
text = scalar_text(value)
|
||||
lowered = text.casefold()
|
||||
if lowered in {"истина", "true", "1"}:
|
||||
return "true"
|
||||
if lowered in {"ложь", "false", "0"}:
|
||||
return "false"
|
||||
if lowered in {"таблица формы", "таблица", "динамический список"}:
|
||||
return "table"
|
||||
if lowered in {"кнопка", "кнопка командной панели", "commandbarbutton"}:
|
||||
return "button"
|
||||
if lowered in {"декорация надписи", "labeldecoration"}:
|
||||
return "label_decoration"
|
||||
if lowered in {"декорация картинки", "picturedecoration"}:
|
||||
return "picture_decoration"
|
||||
if lowered in {"поле переключателя", "radiobuttonfield"}:
|
||||
return "radio_button_field"
|
||||
if lowered in {"в дополнительном подменю", "inadditionalsubmenu"}:
|
||||
return "in_additional_submenu"
|
||||
if lowered in {"в командной панели", "incommandbar"}:
|
||||
return "in_command_bar"
|
||||
if lowered in {"поле", "checkboxfield", "поле флажка"}:
|
||||
return "field"
|
||||
if lowered in {"колонка динамического списка", "column", "колонка реквизита"}:
|
||||
return "column"
|
||||
if lowered in {"attribute", "реквизит формы"}:
|
||||
return "attribute"
|
||||
if lowered in {"event", "событие"}:
|
||||
return "event"
|
||||
if lowered in {"command", "команда формы"}:
|
||||
return "command"
|
||||
if lowered in {"searchstringaddition", "дополнение строки поиска"}:
|
||||
return "search_string_addition"
|
||||
if lowered in {"viewstatusaddition", "дополнение состояния просмотра"}:
|
||||
return "view_status_addition"
|
||||
if lowered in {"searchcontroladdition", "дополнение управления поиском"}:
|
||||
return "search_control_addition"
|
||||
return lowered
|
||||
|
||||
|
||||
def canonical_property_name(value: Any) -> str:
|
||||
text = str(value or "").strip()
|
||||
return PROPERTY_NAME_ALIASES.get(text, text)
|
||||
|
||||
|
||||
def semantic_properties(row: dict[str, Any]) -> dict[str, dict[str, Any]]:
|
||||
result: dict[str, dict[str, Any]] = {}
|
||||
for direct_name, key in (
|
||||
("Имя", "name"),
|
||||
("Идентификатор", "id"),
|
||||
("Заголовок", "title"),
|
||||
("ПутьКДанным", "path_to_data"),
|
||||
("Вид", "type_name"),
|
||||
("Обработчик", "handler"),
|
||||
):
|
||||
if key in row and row.get(key) not in {None, ""}:
|
||||
result[direct_name] = {"name": direct_name, "value": row.get(key), "source": "decoded_direct"}
|
||||
semantic = row.get("semantic") if isinstance(row.get("semantic"), dict) else {}
|
||||
for group, props in (semantic.get("groups") or {}).items():
|
||||
for prop in props or []:
|
||||
if not isinstance(prop, dict):
|
||||
continue
|
||||
name = canonical_property_name(prop.get("name"))
|
||||
if name and name not in result:
|
||||
result[name] = {**prop, "group": group}
|
||||
return result
|
||||
|
||||
|
||||
def iter_sql_rows(form: dict[str, Any]) -> list[dict[str, Any]]:
|
||||
profile = form.get("profile") if isinstance(form.get("profile"), dict) else {}
|
||||
rows = []
|
||||
for section in ("items", "attributes", "parameters", "commands", "tables", "command_bars", "events"):
|
||||
for row in profile.get(section) or []:
|
||||
if isinstance(row, dict):
|
||||
rows.append({**row, "_profile_section": section})
|
||||
for item in profile.get("items") or []:
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
for event in item.get("events") or []:
|
||||
if not isinstance(event, dict):
|
||||
continue
|
||||
event_name = event.get("event_name") or event.get("name")
|
||||
rows.append(
|
||||
{
|
||||
**event,
|
||||
"name": event_name,
|
||||
"type_name": "Event",
|
||||
"_profile_section": "events",
|
||||
"_event_owner": item.get("name"),
|
||||
}
|
||||
)
|
||||
return rows
|
||||
|
||||
|
||||
def row_key(row: dict[str, Any]) -> tuple[str, str]:
|
||||
if row.get("id") not in {None, ""}:
|
||||
return ("id", str(row.get("id")))
|
||||
return ("name", str(row.get("name") or ""))
|
||||
|
||||
|
||||
def xml_match_keys(xml_item: dict[str, Any]) -> list[tuple[str, str]]:
|
||||
name = str(xml_item.get("name") or "")
|
||||
item_id = str(xml_item.get("id") or "")
|
||||
kind = str(xml_item.get("kind") or "")
|
||||
keys: list[tuple[str, str]] = []
|
||||
if kind in {"ExtendedTooltip", "Event", "Column"}:
|
||||
if name:
|
||||
keys.append(("name", name.casefold()))
|
||||
return keys
|
||||
if kind == "Button":
|
||||
if name:
|
||||
keys.append(("name", name.casefold()))
|
||||
if item_id:
|
||||
keys.append(("id", item_id))
|
||||
return keys
|
||||
if item_id:
|
||||
keys.append(("id", item_id))
|
||||
if name:
|
||||
keys.append(("name", name.casefold()))
|
||||
return keys
|
||||
|
||||
|
||||
def index_rows(rows: list[dict[str, Any]], *, sections: set[str] | None = None) -> dict[tuple[str, str], dict[str, Any]]:
|
||||
by_key: dict[tuple[str, str], dict[str, Any]] = {}
|
||||
by_name: dict[str, dict[str, Any]] = {}
|
||||
for row in rows:
|
||||
if sections is not None and str(row.get("_profile_section") or "") not in sections:
|
||||
continue
|
||||
key = row_key(row)
|
||||
if key[1]:
|
||||
by_key.setdefault(key, row)
|
||||
name = str(row.get("name") or "")
|
||||
if name:
|
||||
by_name.setdefault(name.casefold(), row)
|
||||
for name, row in by_name.items():
|
||||
by_key.setdefault(("name", name), row)
|
||||
return by_key
|
||||
|
||||
|
||||
def xml_sql_sections(xml_item: dict[str, Any]) -> set[str]:
|
||||
kind = str(xml_item.get("kind") or "")
|
||||
if kind == "Attribute":
|
||||
return {"attributes"}
|
||||
if kind == "Parameter":
|
||||
return {"parameters"}
|
||||
if kind == "Column":
|
||||
return {"items", "tables"}
|
||||
if kind == "Command":
|
||||
return {"commands"}
|
||||
if kind == "Button":
|
||||
return {"items", "commands"}
|
||||
if kind in {"CommandBar", "AutoCommandBar"}:
|
||||
return {"items", "command_bars"}
|
||||
if kind == "Event":
|
||||
return {"events"}
|
||||
return {"items", "commands", "command_bars"}
|
||||
|
||||
|
||||
def xml_profile_from_context_form(form: dict[str, Any], *, max_items: int) -> dict[str, Any] | None:
|
||||
path_text = str(form.get("form_xml_path") or "")
|
||||
if not path_text:
|
||||
structure = form.get("structure") if isinstance(form.get("structure"), dict) else {}
|
||||
path_text = str(structure.get("form_xml_path") or "")
|
||||
if not path_text:
|
||||
return None
|
||||
path = Path(path_text)
|
||||
if not path.is_file():
|
||||
return {"status": "missing_xml_file", "source": {"path": path_text}, "items": []}
|
||||
profile = decode_form_xml(path, max_items=max_items)
|
||||
profile["form"]["name"] = form.get("name")
|
||||
profile["form"]["guid"] = form.get("uuid")
|
||||
return profile
|
||||
|
||||
|
||||
def matching_sql_form(xml_form: dict[str, Any], sql_forms: list[dict[str, Any]]) -> dict[str, Any] | None:
|
||||
guid = str((xml_form.get("form") or {}).get("guid") or "").casefold()
|
||||
name = str((xml_form.get("form") or {}).get("name") or "").casefold()
|
||||
for form in sql_forms:
|
||||
if guid and str(form.get("guid") or "").casefold() == guid:
|
||||
return form
|
||||
for form in sql_forms:
|
||||
if name and str(form.get("name") or "").casefold() == name:
|
||||
return form
|
||||
return None
|
||||
|
||||
|
||||
def compare_form(sql_form: dict[str, Any] | None, xml_profile: dict[str, Any], *, sample_limit: int) -> dict[str, Any]:
|
||||
xml_items = [item for item in xml_profile.get("items") or [] if isinstance(item, dict)]
|
||||
sql_rows = iter_sql_rows(sql_form or {})
|
||||
sql_any_index = index_rows(sql_rows)
|
||||
sql_indexes_by_sections: dict[tuple[str, ...], dict[tuple[str, str], dict[str, Any]]] = {}
|
||||
counts = Counter()
|
||||
property_route_counts: Counter[tuple[str, str, str, str, str, str]] = Counter()
|
||||
samples: dict[str, list[dict[str, Any]]] = {
|
||||
"matched": [],
|
||||
"missing_sql_item": [],
|
||||
"xml_only_property": [],
|
||||
"mismatch": [],
|
||||
"command_name_match": [],
|
||||
"command_name_mismatch": [],
|
||||
"matched_form_property": [],
|
||||
"xml_only_form_property": [],
|
||||
"form_mismatch": [],
|
||||
}
|
||||
|
||||
sql_profile = (sql_form or {}).get("profile") if isinstance((sql_form or {}).get("profile"), dict) else {}
|
||||
sql_form_semantic = sql_profile.get("form_semantic") if isinstance(sql_profile.get("form_semantic"), dict) else None
|
||||
if sql_form_semantic is None:
|
||||
sql_form_semantic = form_common_semantic(
|
||||
[item for item in sql_profile.get("form_parameters") or [] if isinstance(item, dict)],
|
||||
include_diagnostics=True,
|
||||
)
|
||||
enrich_form_common_semantic(
|
||||
sql_form_semantic,
|
||||
[item for item in sql_profile.get("items") or [] if isinstance(item, dict)],
|
||||
)
|
||||
xml_form = xml_profile.get("form") if isinstance(xml_profile.get("form"), dict) else {}
|
||||
xml_form_semantic = xml_form.get("semantic") if isinstance(xml_form.get("semantic"), dict) else {}
|
||||
sql_root_props = semantic_properties({"type_name": "Форма", "semantic": sql_form_semantic})
|
||||
xml_root_props = semantic_properties({"type_name": "Форма", "semantic": xml_form_semantic})
|
||||
form_property_routes = []
|
||||
for name, xml_prop in xml_root_props.items():
|
||||
sql_prop = sql_root_props.get(name)
|
||||
if sql_prop is None:
|
||||
counts["xml_only_form_properties"] += 1
|
||||
if len(samples["xml_only_form_property"]) < sample_limit:
|
||||
samples["xml_only_form_property"].append(
|
||||
{"property": name, "xml_name": xml_prop.get("xml_name"), "xml_value": xml_prop.get("value")}
|
||||
)
|
||||
continue
|
||||
if comparable(sql_prop.get("value")) != comparable(xml_prop.get("value")):
|
||||
counts["form_mismatches"] += 1
|
||||
if len(samples["form_mismatch"]) < sample_limit:
|
||||
samples["form_mismatch"].append(
|
||||
{
|
||||
"property": name,
|
||||
"xml_name": xml_prop.get("xml_name"),
|
||||
"sql_value": sql_prop.get("value"),
|
||||
"xml_value": xml_prop.get("value"),
|
||||
"sql_source": sql_prop.get("source"),
|
||||
}
|
||||
)
|
||||
continue
|
||||
counts["matched_form_properties"] += 1
|
||||
if len(samples["matched_form_property"]) < sample_limit:
|
||||
samples["matched_form_property"].append(
|
||||
{"property": name, "xml_name": xml_prop.get("xml_name"), "value": xml_prop.get("value")}
|
||||
)
|
||||
parameter_indices = sql_prop.get("parameter_indices")
|
||||
if isinstance(parameter_indices, list) and parameter_indices:
|
||||
form_property_routes.append(
|
||||
{
|
||||
"xml_name": xml_prop.get("xml_name") or name,
|
||||
"property": name,
|
||||
"parameter_indices": parameter_indices,
|
||||
"sql_source": sql_prop.get("source"),
|
||||
"write_shape": sql_prop.get("write_shape"),
|
||||
}
|
||||
)
|
||||
|
||||
for xml_item in xml_items:
|
||||
wanted_sections = xml_sql_sections(xml_item)
|
||||
section_key = tuple(sorted(wanted_sections))
|
||||
sql_item_index = sql_indexes_by_sections.setdefault(section_key, index_rows(sql_rows, sections=wanted_sections))
|
||||
keys = xml_match_keys(xml_item)
|
||||
sql_row = None
|
||||
if xml_item.get("kind") == "Column" and xml_item.get("additional_columns_table"):
|
||||
logical_path = f"{xml_item.get('additional_columns_table')}.{xml_item.get('name')}"
|
||||
sql_row = next((row for row in sql_rows if comparable(row.get("path_to_data")) == comparable(logical_path)), None)
|
||||
if sql_row is not None:
|
||||
counts["logical_column_sql_match"] += 1
|
||||
if len(samples.setdefault("logical_column_sql_match", [])) < sample_limit:
|
||||
samples["logical_column_sql_match"].append(
|
||||
{
|
||||
"name": xml_item.get("name"),
|
||||
"id": xml_item.get("id"),
|
||||
"logical_path": logical_path,
|
||||
"sql_element": sql_row.get("name"),
|
||||
"sql_path": sql_row.get("path"),
|
||||
}
|
||||
)
|
||||
if xml_item.get("kind") == "Event" and xml_item.get("owner"):
|
||||
sql_row = next(
|
||||
(
|
||||
row
|
||||
for row in sql_rows
|
||||
if str(row.get("_profile_section") or "") == "events"
|
||||
and str(row.get("_event_owner") or "").casefold() == str(xml_item.get("owner") or "").casefold()
|
||||
and str(row.get("name") or "").casefold() == str(xml_item.get("name") or "").casefold()
|
||||
),
|
||||
None,
|
||||
)
|
||||
if sql_row is None and not (xml_item.get("kind") == "Event" and xml_item.get("owner")):
|
||||
sql_row = next((sql_item_index[key] for key in keys if key in sql_item_index), None)
|
||||
if sql_row is not None and xml_item.get("kind") == "Column" and xml_item.get("additional_columns_table"):
|
||||
counts["matched_items"] += 1
|
||||
if len(samples["matched"]) < sample_limit:
|
||||
samples["matched"].append(
|
||||
{
|
||||
"name": xml_item.get("name"),
|
||||
"id": xml_item.get("id"),
|
||||
"sql_path": sql_row.get("path"),
|
||||
"sql_type": sql_row.get("type_name"),
|
||||
"xml_kind": xml_item.get("kind_ru"),
|
||||
"match_by": "additional_columns_data_path",
|
||||
}
|
||||
)
|
||||
continue
|
||||
if sql_row is None:
|
||||
fallback = next((sql_any_index[key] for key in keys if key in sql_any_index), None)
|
||||
if fallback is not None:
|
||||
counts["non_item_sql_match"] += 1
|
||||
if len(samples.setdefault("non_item_sql_match", [])) < sample_limit:
|
||||
samples["non_item_sql_match"].append({"name": xml_item.get("name"), "id": xml_item.get("id"), "xml_kind": xml_item.get("kind"), "sql_section": fallback.get("_profile_section"), "sql_type": fallback.get("type_name")})
|
||||
if sql_row is None:
|
||||
counts["missing_sql_item"] += 1
|
||||
if len(samples["missing_sql_item"]) < sample_limit:
|
||||
samples["missing_sql_item"].append({"name": xml_item.get("name"), "id": xml_item.get("id"), "kind": xml_item.get("kind")})
|
||||
continue
|
||||
counts["matched_items"] += 1
|
||||
if len(samples["matched"]) < sample_limit:
|
||||
samples["matched"].append({"name": xml_item.get("name"), "id": xml_item.get("id"), "sql_path": sql_row.get("path"), "sql_type": sql_row.get("type_name"), "xml_kind": xml_item.get("kind_ru")})
|
||||
sql_props = semantic_properties(sql_row)
|
||||
xml_props = semantic_properties(xml_item)
|
||||
sql_command = sql_props.get("ИмяКоманды")
|
||||
xml_command = xml_props.get("ИмяКоманды")
|
||||
if sql_command is not None and xml_command is not None:
|
||||
sql_command_value = sql_command.get("value")
|
||||
xml_command_value = xml_command.get("value")
|
||||
command_row = {
|
||||
"item": xml_item.get("name"),
|
||||
"id": xml_item.get("id"),
|
||||
"sql_value": sql_command_value,
|
||||
"xml_value": xml_command_value,
|
||||
"sql_path": sql_row.get("path"),
|
||||
"sql_source": sql_command.get("source"),
|
||||
}
|
||||
if comparable(sql_command_value) == comparable(xml_command_value):
|
||||
counts["command_name_matches"] += 1
|
||||
if len(samples["command_name_match"]) < sample_limit:
|
||||
samples["command_name_match"].append(command_row)
|
||||
else:
|
||||
counts["command_name_mismatches"] += 1
|
||||
if len(samples["command_name_mismatch"]) < sample_limit:
|
||||
samples["command_name_mismatch"].append(command_row)
|
||||
for name, xml_prop in xml_props.items():
|
||||
if name == "Идентификатор" and xml_item.get("kind") == "Button" and sql_row.get("_profile_section") == "commands":
|
||||
continue
|
||||
xml_value = xml_prop.get("value")
|
||||
sql_prop = sql_props.get(name)
|
||||
if sql_prop is None:
|
||||
counts["xml_only_properties"] += 1
|
||||
if len(samples["xml_only_property"]) < sample_limit:
|
||||
samples["xml_only_property"].append({"item": xml_item.get("name"), "id": xml_item.get("id"), "property": name, "xml_value": xml_value, "xml_name": xml_prop.get("xml_name")})
|
||||
continue
|
||||
sql_value = sql_prop.get("value")
|
||||
if comparable(sql_value) != comparable(xml_value):
|
||||
counts["mismatches"] += 1
|
||||
if len(samples["mismatch"]) < sample_limit:
|
||||
samples["mismatch"].append({"item": xml_item.get("name"), "id": xml_item.get("id"), "property": name, "sql_value": sql_value, "xml_value": xml_value, "sql_source": sql_prop.get("source"), "xml_name": xml_prop.get("xml_name")})
|
||||
else:
|
||||
counts["matched_properties"] += 1
|
||||
parameter_index = sql_prop.get("parameter_index")
|
||||
if parameter_index is not None:
|
||||
property_route_counts[
|
||||
(
|
||||
str(xml_item.get("kind") or ""),
|
||||
str(xml_prop.get("xml_name") or name),
|
||||
name,
|
||||
str(sql_row.get("marker") or ""),
|
||||
str(parameter_index),
|
||||
str(sql_prop.get("source") or ""),
|
||||
)
|
||||
] += 1
|
||||
|
||||
property_routes = [
|
||||
{
|
||||
"xml_kind": key[0],
|
||||
"xml_name": key[1],
|
||||
"property": key[2],
|
||||
"sql_marker": key[3] or None,
|
||||
"parameter_index": int(key[4]) if key[4].lstrip("-").isdigit() else key[4],
|
||||
"sql_source": key[5] or None,
|
||||
"matches": match_count,
|
||||
}
|
||||
for key, match_count in sorted(property_route_counts.items(), key=lambda item: (-item[1], item[0]))
|
||||
]
|
||||
|
||||
return {
|
||||
"sql_form": {key: (sql_form or {}).get(key) for key in ("name", "guid", "source") if (sql_form or {}).get(key) is not None},
|
||||
"xml_form": xml_profile.get("form"),
|
||||
"xml_source": xml_profile.get("source"),
|
||||
"counts": dict(counts),
|
||||
"form_property_routes": form_property_routes,
|
||||
"property_routes": property_routes,
|
||||
"samples": samples,
|
||||
}
|
||||
|
||||
|
||||
def build_report(sql_details: dict[str, Any], xml_context: dict[str, Any], *, sample_limit: int, max_items: int) -> dict[str, Any]:
|
||||
sql_forms = [form for form in sql_details.get("forms") or [] if isinstance(form, dict)]
|
||||
comparisons = []
|
||||
for form in xml_context.get("forms") or []:
|
||||
if not isinstance(form, dict):
|
||||
continue
|
||||
profiles = []
|
||||
seen_xml_paths: set[str] = set()
|
||||
xml_profile = xml_profile_from_context_form(form, max_items=max_items)
|
||||
if xml_profile is not None:
|
||||
source_path = str((xml_profile.get("source") or {}).get("path") or "").casefold()
|
||||
if source_path:
|
||||
seen_xml_paths.add(source_path)
|
||||
profiles.append(xml_profile)
|
||||
for overlay in form.get("extension_overlays") or []:
|
||||
if isinstance(overlay, dict):
|
||||
overlay_profile = xml_profile_from_context_form(overlay, max_items=max_items)
|
||||
if overlay_profile is not None:
|
||||
source_path = str((overlay_profile.get("source") or {}).get("path") or "").casefold()
|
||||
if source_path and source_path in seen_xml_paths:
|
||||
continue
|
||||
if source_path:
|
||||
seen_xml_paths.add(source_path)
|
||||
profiles.append(overlay_profile)
|
||||
for profile in profiles:
|
||||
comparisons.append(compare_form(matching_sql_form(profile, sql_forms), profile, sample_limit=sample_limit))
|
||||
totals = Counter()
|
||||
route_counts: Counter[tuple[str, str, str, str, str, str]] = Counter()
|
||||
for comparison in comparisons:
|
||||
totals.update(comparison.get("counts") or {})
|
||||
for route in comparison.get("property_routes") or []:
|
||||
route_counts[
|
||||
(
|
||||
str(route.get("xml_kind") or ""),
|
||||
str(route.get("xml_name") or ""),
|
||||
str(route.get("property") or ""),
|
||||
str(route.get("sql_marker") or ""),
|
||||
str(route.get("parameter_index") if route.get("parameter_index") is not None else ""),
|
||||
str(route.get("sql_source") or ""),
|
||||
)
|
||||
] += int(route.get("matches") or 0)
|
||||
grouped_routes: dict[tuple[str, str, str], list[tuple[tuple[str, str, str, str, str, str], int]]] = {}
|
||||
for key, match_count in route_counts.items():
|
||||
grouped_routes.setdefault(key[:3], []).append((key, match_count))
|
||||
stable_property_routes = []
|
||||
ambiguous_property_routes = []
|
||||
for identity, variants in sorted(grouped_routes.items()):
|
||||
rows = [
|
||||
{
|
||||
"xml_kind": key[0],
|
||||
"xml_name": key[1],
|
||||
"property": key[2],
|
||||
"sql_marker": key[3] or None,
|
||||
"parameter_index": int(key[4]) if key[4].lstrip("-").isdigit() else key[4],
|
||||
"sql_source": key[5] or None,
|
||||
"matches": match_count,
|
||||
}
|
||||
for key, match_count in sorted(variants, key=lambda item: (-item[1], item[0]))
|
||||
]
|
||||
if len(rows) == 1 and rows[0]["matches"] >= 2:
|
||||
stable_property_routes.append(rows[0])
|
||||
elif len(rows) > 1:
|
||||
ambiguous_property_routes.append(
|
||||
{"xml_kind": identity[0], "xml_name": identity[1], "property": identity[2], "routes": rows}
|
||||
)
|
||||
totals["property_routes"] = len(route_counts)
|
||||
totals["stable_property_routes"] = len(stable_property_routes)
|
||||
totals["ambiguous_property_routes"] = len(ambiguous_property_routes)
|
||||
return {
|
||||
"schema": "onec_form_sql_xml_comparison.v1",
|
||||
"status": "ok",
|
||||
"object": sql_details.get("object") or xml_context.get("object"),
|
||||
"counts": {"forms_compared": len(comparisons), **dict(totals)},
|
||||
"stable_property_routes": stable_property_routes,
|
||||
"ambiguous_property_routes": ambiguous_property_routes,
|
||||
"comparisons": comparisons,
|
||||
}
|
||||
|
||||
|
||||
def markdown_table_row(values: list[Any]) -> str:
|
||||
return "| " + " | ".join(str(value).replace("\n", " ") for value in values) + " |"
|
||||
|
||||
|
||||
def render_markdown(report: dict[str, Any]) -> str:
|
||||
lines = ["# 1C Form SQL/XML Comparison", ""]
|
||||
obj = report.get("object") or {}
|
||||
lines.append(f"- Object: `{obj.get('kind')}.{obj.get('name')}`")
|
||||
counts = report.get("counts") or {}
|
||||
lines.append(f"- Forms compared: `{counts.get('forms_compared')}`")
|
||||
lines.append(f"- Matched items: `{counts.get('matched_items', 0)}`")
|
||||
lines.append(f"- Matched properties: `{counts.get('matched_properties', 0)}`")
|
||||
lines.append(f"- XML-only properties: `{counts.get('xml_only_properties', 0)}`")
|
||||
lines.append(f"- Mismatches: `{counts.get('mismatches', 0)}`")
|
||||
lines.append(f"- CommandName matches: `{counts.get('command_name_matches', 0)}`")
|
||||
lines.append(f"- CommandName mismatches: `{counts.get('command_name_mismatches', 0)}`")
|
||||
lines.append("")
|
||||
for comparison in report.get("comparisons") or []:
|
||||
xml_form = comparison.get("xml_form") or {}
|
||||
sql_form = comparison.get("sql_form") or {}
|
||||
lines.append(f"## {xml_form.get('name') or sql_form.get('name')}")
|
||||
lines.append("")
|
||||
lines.append(markdown_table_row(["Metric", "Count"]))
|
||||
lines.append(markdown_table_row(["---", "---:"]))
|
||||
for key, value in sorted((comparison.get("counts") or {}).items()):
|
||||
lines.append(markdown_table_row([key, value]))
|
||||
for title, key in (("CommandName Matches", "command_name_match"), ("CommandName Mismatches", "command_name_mismatch"), ("XML-Only Properties", "xml_only_property"), ("Mismatches", "mismatch"), ("Missing SQL Items", "missing_sql_item")):
|
||||
sample = (comparison.get("samples") or {}).get(key) or []
|
||||
if not sample:
|
||||
continue
|
||||
lines.append("")
|
||||
lines.append(f"### {title}")
|
||||
lines.append("")
|
||||
lines.append("```json")
|
||||
lines.append(json.dumps(sample[:10], ensure_ascii=False, indent=2))
|
||||
lines.append("```")
|
||||
lines.append("")
|
||||
return "\n".join(lines).rstrip() + "\n"
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Compare decoded SQL form semantics with Form.xml semantics.")
|
||||
parser.add_argument("--sql-details", type=Path, required=True)
|
||||
parser.add_argument("--xml-context", type=Path, required=True)
|
||||
parser.add_argument("--output-json", type=Path, required=True)
|
||||
parser.add_argument("--output-markdown", type=Path)
|
||||
parser.add_argument("--sample-limit", type=int, default=20)
|
||||
parser.add_argument("--max-items", type=int, default=5000)
|
||||
args = parser.parse_args()
|
||||
|
||||
report = build_report(read_json(args.sql_details), read_json(args.xml_context), sample_limit=args.sample_limit, max_items=args.max_items)
|
||||
write_json(args.output_json, report)
|
||||
if args.output_markdown:
|
||||
args.output_markdown.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.output_markdown.write_text(render_markdown(report), encoding="utf-8")
|
||||
print(json.dumps({"schema": "onec_form_sql_xml_comparison_cli_summary.v1", "status": report["status"], "counts": report["counts"], "output_json": str(args.output_json), "output_markdown": str(args.output_markdown) if args.output_markdown else None}, ensure_ascii=False, indent=2))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,265 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Compare two 1C saved-state object reports in 1C object terms."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import importlib.util
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
|
||||
def load_json(path: Path) -> dict[str, Any]:
|
||||
return json.loads(path.read_text(encoding="utf-8-sig"))
|
||||
|
||||
|
||||
def write_json(path: Path, data: dict[str, Any]) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(json.dumps(data, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
|
||||
|
||||
def object_map(report: dict[str, Any]) -> dict[str, dict[str, Any]]:
|
||||
items = ((report.get("agent_summary") or {}).get("object_changes") or [])
|
||||
result: dict[str, dict[str, Any]] = {}
|
||||
for item in items:
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
full_name = str(item.get("full_name") or "")
|
||||
if full_name:
|
||||
result[full_name] = item
|
||||
return result
|
||||
|
||||
|
||||
def system_map(report: dict[str, Any]) -> dict[str, dict[str, Any]]:
|
||||
items = ((report.get("agent_summary") or {}).get("system_changes") or [])
|
||||
result: dict[str, dict[str, Any]] = {}
|
||||
for item in items:
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
key = f"{item.get('layer')}::{item.get('extension')}::{item.get('name')}"
|
||||
result[key] = item
|
||||
return result
|
||||
|
||||
|
||||
def stable_fingerprint(value: Any) -> str:
|
||||
encoded = json.dumps(value, ensure_ascii=False, sort_keys=True, separators=(",", ":")).encode("utf-8")
|
||||
return hashlib.sha256(encoded).hexdigest()
|
||||
|
||||
|
||||
def comparable_object(item: dict[str, Any]) -> dict[str, Any]:
|
||||
return {
|
||||
"full_name": item.get("full_name"),
|
||||
"layer": item.get("layer"),
|
||||
"extension": item.get("extension"),
|
||||
"kind": item.get("kind"),
|
||||
"kind_ru": item.get("kind_ru"),
|
||||
"name": item.get("name"),
|
||||
"synonym": item.get("synonym"),
|
||||
"parts_count": item.get("parts_count"),
|
||||
"text_diff_parts": item.get("text_diff_parts"),
|
||||
"active_missing_parts": item.get("active_missing_parts"),
|
||||
"added_terms": item.get("added_terms") or [],
|
||||
"removed_terms": item.get("removed_terms") or [],
|
||||
"parts": [
|
||||
{
|
||||
"file_name": part.get("file_name"),
|
||||
"payload_role": part.get("payload_role"),
|
||||
"active_exists": part.get("active_exists"),
|
||||
"summary": part.get("summary"),
|
||||
"delta_chars": part.get("delta_chars"),
|
||||
}
|
||||
for part in item.get("parts") or []
|
||||
if isinstance(part, dict)
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def terms_delta(before: list[Any], after: list[Any]) -> dict[str, list[Any]]:
|
||||
before_set = {str(value) for value in before}
|
||||
after_set = {str(value) for value in after}
|
||||
return {
|
||||
"added": [value for value in after if str(value) not in before_set],
|
||||
"removed": [value for value in before if str(value) not in after_set],
|
||||
}
|
||||
|
||||
|
||||
def parts_delta(before: list[dict[str, Any]], after: list[dict[str, Any]]) -> dict[str, Any]:
|
||||
before_by_name = {str(part.get("file_name")): part for part in before if part.get("file_name")}
|
||||
after_by_name = {str(part.get("file_name")): part for part in after if part.get("file_name")}
|
||||
before_names = set(before_by_name)
|
||||
after_names = set(after_by_name)
|
||||
changed = []
|
||||
for name in sorted(before_names & after_names):
|
||||
before_part = before_by_name[name]
|
||||
after_part = after_by_name[name]
|
||||
if stable_fingerprint(before_part) != stable_fingerprint(after_part):
|
||||
changed.append({
|
||||
"file_name": name,
|
||||
"before": before_part,
|
||||
"after": after_part,
|
||||
})
|
||||
return {
|
||||
"added": [after_by_name[name] for name in sorted(after_names - before_names)],
|
||||
"removed": [before_by_name[name] for name in sorted(before_names - after_names)],
|
||||
"changed": changed,
|
||||
}
|
||||
|
||||
|
||||
def object_delta(before: dict[str, Any], after: dict[str, Any]) -> dict[str, Any]:
|
||||
before_cmp = comparable_object(before)
|
||||
after_cmp = comparable_object(after)
|
||||
return {
|
||||
"full_name": after.get("full_name") or before.get("full_name"),
|
||||
"before_fingerprint": stable_fingerprint(before_cmp),
|
||||
"after_fingerprint": stable_fingerprint(after_cmp),
|
||||
"before": before_cmp,
|
||||
"after": after_cmp,
|
||||
"term_delta": {
|
||||
"added_terms": terms_delta(before_cmp.get("added_terms") or [], after_cmp.get("added_terms") or []),
|
||||
"removed_terms": terms_delta(before_cmp.get("removed_terms") or [], after_cmp.get("removed_terms") or []),
|
||||
},
|
||||
"part_delta": parts_delta(before_cmp.get("parts") or [], after_cmp.get("parts") or []),
|
||||
}
|
||||
|
||||
|
||||
def compact_object(item: dict[str, Any]) -> dict[str, Any]:
|
||||
comparable = comparable_object(item)
|
||||
comparable["fingerprint"] = stable_fingerprint(comparable)
|
||||
return comparable
|
||||
|
||||
|
||||
def compare_reports(before_path: Path, after_path: Path) -> dict[str, Any]:
|
||||
before_path = before_path.resolve()
|
||||
after_path = after_path.resolve()
|
||||
before = load_json(before_path)
|
||||
after = load_json(after_path)
|
||||
before_objects = object_map(before)
|
||||
after_objects = object_map(after)
|
||||
before_names = set(before_objects)
|
||||
after_names = set(after_objects)
|
||||
|
||||
changed = []
|
||||
unchanged = []
|
||||
for name in sorted(before_names & after_names):
|
||||
before_fp = stable_fingerprint(comparable_object(before_objects[name]))
|
||||
after_fp = stable_fingerprint(comparable_object(after_objects[name]))
|
||||
if before_fp == after_fp:
|
||||
unchanged.append(name)
|
||||
else:
|
||||
changed.append(object_delta(before_objects[name], after_objects[name]))
|
||||
|
||||
before_system = system_map(before)
|
||||
after_system = system_map(after)
|
||||
system_added = sorted(set(after_system) - set(before_system))
|
||||
system_removed = sorted(set(before_system) - set(after_system))
|
||||
system_changed = [
|
||||
key
|
||||
for key in sorted(set(before_system) & set(after_system))
|
||||
if stable_fingerprint(before_system[key]) != stable_fingerprint(after_system[key])
|
||||
]
|
||||
|
||||
return {
|
||||
"schema": "onec_saved_state_object_report_delta.v1",
|
||||
"before_report": str(before_path),
|
||||
"after_report": str(after_path),
|
||||
"database": after.get("database") or before.get("database"),
|
||||
"objects": {
|
||||
"added": [compact_object(after_objects[name]) for name in sorted(after_names - before_names)],
|
||||
"removed": [compact_object(before_objects[name]) for name in sorted(before_names - after_names)],
|
||||
"changed": changed,
|
||||
"unchanged": unchanged,
|
||||
},
|
||||
"system_changes": {
|
||||
"added": [after_system[key] for key in system_added],
|
||||
"removed": [before_system[key] for key in system_removed],
|
||||
"changed": [{"key": key, "before": before_system[key], "after": after_system[key]} for key in system_changed],
|
||||
},
|
||||
"counts": {
|
||||
"objects_added": len(after_names - before_names),
|
||||
"objects_removed": len(before_names - after_names),
|
||||
"objects_changed": len(changed),
|
||||
"objects_unchanged": len(unchanged),
|
||||
"system_added": len(system_added),
|
||||
"system_removed": len(system_removed),
|
||||
"system_changed": len(system_changed),
|
||||
},
|
||||
"safety": {
|
||||
"read_only": True,
|
||||
"sql_write_performed": False,
|
||||
"public_terms_are_1c_objects": True,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def render_markdown(delta: dict[str, Any]) -> str:
|
||||
module_path = REPO_ROOT / "scripts" / "render_1c_saved_state_object_report_delta_markdown.py"
|
||||
spec = importlib.util.spec_from_file_location("saved_state_delta_markdown", module_path)
|
||||
if spec is None or spec.loader is None:
|
||||
raise RuntimeError(f"Cannot load Markdown renderer: {module_path}")
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
sys.modules[spec.name] = module
|
||||
spec.loader.exec_module(module)
|
||||
return module.render(delta)
|
||||
|
||||
|
||||
def check_delta(delta_path: Path, check_output: Path) -> None:
|
||||
module_path = REPO_ROOT / "scripts" / "check_1c_saved_state_object_report_delta.py"
|
||||
spec = importlib.util.spec_from_file_location("saved_state_delta_check", module_path)
|
||||
if spec is None or spec.loader is None:
|
||||
raise RuntimeError(f"Cannot load delta checker: {module_path}")
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
sys.modules[spec.name] = module
|
||||
spec.loader.exec_module(module)
|
||||
result = module.check_delta(delta_path)
|
||||
module.write_json(check_output, result)
|
||||
if not result.get("passed"):
|
||||
raise RuntimeError(f"Saved-state delta check failed: {check_output}")
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Compare two 1C saved-state object reports.")
|
||||
parser.add_argument("--before", type=Path, required=True)
|
||||
parser.add_argument("--after", type=Path, required=True)
|
||||
parser.add_argument("--output", type=Path)
|
||||
parser.add_argument("--markdown-output", type=Path)
|
||||
parser.add_argument("--skip-markdown", action="store_true")
|
||||
parser.add_argument("--check-output", type=Path)
|
||||
parser.add_argument("--skip-check", action="store_true")
|
||||
args = parser.parse_args()
|
||||
|
||||
result = compare_reports(args.before, args.after)
|
||||
markdown_output = args.markdown_output
|
||||
if markdown_output is None and args.output and not args.skip_markdown:
|
||||
markdown_output = args.output.with_suffix(".md")
|
||||
if markdown_output and not args.skip_markdown:
|
||||
markdown_output = markdown_output.resolve()
|
||||
result["markdown"] = str(markdown_output)
|
||||
check_output = args.check_output
|
||||
if check_output is None and args.output and not args.skip_check:
|
||||
check_output = args.output.with_name(f"{args.output.stem}-check.json")
|
||||
if check_output and not args.skip_check:
|
||||
check_output = check_output.resolve()
|
||||
result["check"] = str(check_output)
|
||||
if args.output:
|
||||
write_json(args.output, result)
|
||||
if markdown_output and not args.skip_markdown:
|
||||
markdown_output.parent.mkdir(parents=True, exist_ok=True)
|
||||
markdown_output.write_text(render_markdown(result), encoding="utf-8")
|
||||
if args.output:
|
||||
write_json(args.output, result)
|
||||
if check_output and not args.skip_check:
|
||||
if not args.output:
|
||||
raise SystemExit("Use --output when delta check output is enabled.")
|
||||
check_delta(args.output, check_output)
|
||||
print(json.dumps({"output": str(args.output) if args.output else None, "schema": result["schema"], "counts": result["counts"]}, ensure_ascii=False))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,461 @@
|
||||
param(
|
||||
[string]$Server = $env:ONEC_SQL_SERVER,
|
||||
[string]$Database = $env:ONEC_SQL_DATABASE,
|
||||
[string]$User = $env:ONEC_SQL_USER,
|
||||
[string]$Password = $env:ONEC_SQL_PASSWORD,
|
||||
[string]$BaseMetadataDir = "reports\1c-sql\upo\structured-metadata-all-kinds",
|
||||
[string]$ExtensionGuidIndex = "reports\1c-sql\upo\xml-guid-index-extensions.json",
|
||||
[string]$Output
|
||||
)
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
if (-not $Server) { throw "Server is required. Use -Server or ONEC_SQL_SERVER." }
|
||||
if (-not $Database) { throw "Database is required. Use -Database or ONEC_SQL_DATABASE." }
|
||||
if (-not $User) { throw "User is required. Use -User or ONEC_SQL_USER." }
|
||||
if (-not $Password) { throw "Password is required. Use -Password or ONEC_SQL_PASSWORD." }
|
||||
|
||||
function U {
|
||||
param([string]$Base64)
|
||||
return [System.Text.Encoding]::UTF8.GetString([Convert]::FromBase64String($Base64))
|
||||
}
|
||||
|
||||
$ruExtension = U "0KDQsNGB0YjQuNGA0LXQvdC40LU="
|
||||
$ruConfigObject = U "0J7QsdGK0LXQutGC0JrQvtC90YTQuNCz0YPRgNCw0YbQuNC4"
|
||||
$ruForm = U "0KTQvtGA0LzQsA=="
|
||||
|
||||
$kindRu = @{
|
||||
Configuration = (U "0JrQvtC90YTQuNCz0YPRgNCw0YbQuNGP")
|
||||
CommonModule = (U "0J7QsdGJ0LjQudCc0L7QtNGD0LvRjA==")
|
||||
CommonForm = (U "0J7QsdGJ0LDRj9Ck0L7RgNC80LA=")
|
||||
CommonCommand = (U "0J7QsdGJ0LDRj9Ca0L7QvNCw0L3QtNCw")
|
||||
CommonAttribute = (U "0J7QsdGJ0LjQudCg0LXQutCy0LjQt9C40YI=")
|
||||
CommonTemplate = (U "0J7QsdGJ0LjQudCc0LDQutC10YI=")
|
||||
CommonPicture = (U "0J7QsdGJ0LDRj9Ca0LDRgNGC0LjQvdC60LA=")
|
||||
Catalog = (U "0KHQv9GA0LDQstC+0YfQvdC40Lo=")
|
||||
Document = (U "0JTQvtC60YPQvNC10L3Rgg==")
|
||||
DataProcessor = (U "0J7QsdGA0LDQsdC+0YLQutCw")
|
||||
Report = (U "0J7RgtGH0LXRgg==")
|
||||
InformationRegister = (U "0KDQtdCz0LjRgdGC0YDQodCy0LXQtNC10L3QuNC5")
|
||||
AccumulationRegister = (U "0KDQtdCz0LjRgdGC0YDQndCw0LrQvtC/0LvQtdC90LjRjw==")
|
||||
AccountingRegister = (U "0KDQtdCz0LjRgdGC0YDQkdGD0YXQs9Cw0LvRgtC10YDQuNC4")
|
||||
CalculationRegister = (U "0KDQtdCz0LjRgdGC0YDQoNCw0YHRh9C10YLQsA==")
|
||||
Enum = (U "0J/QtdGA0LXRh9C40YHQu9C10L3QuNC1")
|
||||
Form = $ruForm
|
||||
Template = (U "0JzQsNC60LXRgg==")
|
||||
Role = (U "0KDQvtC70Yw=")
|
||||
Subsystem = (U "0J/QvtC00YHQuNGB0YLQtdC80LA=")
|
||||
ExchangePlan = (U "0J/Qu9Cw0L3QntCx0LzQtdC90LA=")
|
||||
BusinessProcess = (U "0JHQuNC30L3QtdGB0J/RgNC+0YbQtdGB0YE=")
|
||||
Task = (U "0JfQsNC00LDRh9Cw")
|
||||
Constant = (U "0JrQvtC90YHRgtCw0L3RgtCw")
|
||||
ChartOfCharacteristicTypes = (U "0J/Qu9Cw0L3QktC40LTQvtCy0KXQsNGA0LDQutGC0LXRgNC40YHRgtC40Lo=")
|
||||
ChartOfAccounts = (U "0J/Qu9Cw0L3QodGH0LXRgtC+0LI=")
|
||||
ChartOfCalculationTypes = (U "0J/Qu9Cw0L3QktC40LTQvtCy0KDQsNGB0YfQtdGC0LA=")
|
||||
}
|
||||
|
||||
$folderKind = @{
|
||||
CommonModules = "CommonModule"
|
||||
CommonForms = "CommonForm"
|
||||
CommonCommands = "CommonCommand"
|
||||
CommonAttributes = "CommonAttribute"
|
||||
CommonTemplates = "CommonTemplate"
|
||||
CommonPictures = "CommonPicture"
|
||||
Catalogs = "Catalog"
|
||||
Documents = "Document"
|
||||
DataProcessors = "DataProcessor"
|
||||
Reports = "Report"
|
||||
InformationRegisters = "InformationRegister"
|
||||
AccumulationRegisters = "AccumulationRegister"
|
||||
AccountingRegisters = "AccountingRegister"
|
||||
CalculationRegisters = "CalculationRegister"
|
||||
Enums = "Enum"
|
||||
Forms = "Form"
|
||||
Templates = "Template"
|
||||
Roles = "Role"
|
||||
Subsystems = "Subsystem"
|
||||
ExchangePlans = "ExchangePlan"
|
||||
BusinessProcesses = "BusinessProcess"
|
||||
Tasks = "Task"
|
||||
Constants = "Constant"
|
||||
ChartsOfCharacteristicTypes = "ChartOfCharacteristicTypes"
|
||||
ChartsOfAccounts = "ChartOfAccounts"
|
||||
ChartsOfCalculationTypes = "ChartOfCalculationTypes"
|
||||
}
|
||||
|
||||
function Convert-RefBytesToGuid {
|
||||
param([byte[]]$Bytes)
|
||||
if ($Bytes.Length -ne 16) { return $null }
|
||||
$hex = ($Bytes | ForEach-Object { $_.ToString("x2") }) -join ""
|
||||
return "{0}-{1}-{2}-{3}-{4}" -f $hex.Substring(24, 8), $hex.Substring(20, 4), $hex.Substring(16, 4), $hex.Substring(0, 4), $hex.Substring(4, 12)
|
||||
}
|
||||
|
||||
function Convert-ToSha256Hex {
|
||||
param([byte[]]$Bytes)
|
||||
$sha = [System.Security.Cryptography.SHA256]::Create()
|
||||
try {
|
||||
return (($sha.ComputeHash($Bytes) | ForEach-Object { $_.ToString("x2") }) -join "")
|
||||
}
|
||||
finally {
|
||||
$sha.Dispose()
|
||||
}
|
||||
}
|
||||
|
||||
function Join-ByteArrays {
|
||||
param([object[]]$Rows)
|
||||
$ordered = @($Rows | Sort-Object part_no)
|
||||
$total = 0
|
||||
foreach ($row in $ordered) { $total += $row.bytes.Length }
|
||||
$buffer = [byte[]]::new($total)
|
||||
$offset = 0
|
||||
foreach ($row in $ordered) {
|
||||
[Array]::Copy($row.bytes, 0, $buffer, $offset, $row.bytes.Length)
|
||||
$offset += $row.bytes.Length
|
||||
}
|
||||
return $buffer
|
||||
}
|
||||
|
||||
function New-Connection {
|
||||
$connectionString = "Server=$Server;Database=$Database;User ID=$User;Password=$Password;Encrypt=False;TrustServerCertificate=True;MultipleActiveResultSets=True;Application Name=Codex 1C Saved State Object Compare;"
|
||||
$connection = [System.Data.SqlClient.SqlConnection]::new($connectionString)
|
||||
$connection.Open()
|
||||
return $connection
|
||||
}
|
||||
|
||||
function Read-TableFiles {
|
||||
param(
|
||||
[System.Data.SqlClient.SqlConnection]$Connection,
|
||||
[string]$Table,
|
||||
[string[]]$FileNames
|
||||
)
|
||||
$command = $Connection.CreateCommand()
|
||||
if ($FileNames -and $FileNames.Count -gt 0) {
|
||||
$placeholders = @()
|
||||
for ($i = 0; $i -lt $FileNames.Count; $i++) {
|
||||
$paramName = "@p$i"
|
||||
$placeholders += $paramName
|
||||
$null = $command.Parameters.Add($paramName, [System.Data.SqlDbType]::NVarChar, 512)
|
||||
$command.Parameters[$paramName].Value = $FileNames[$i]
|
||||
}
|
||||
$command.CommandText = "SELECT FileName, PartNo, BinaryData FROM dbo.[$Table] WHERE FileName IN ($($placeholders -join ', ')) ORDER BY FileName, PartNo"
|
||||
}
|
||||
else {
|
||||
$command.CommandText = "SELECT FileName, PartNo, BinaryData FROM dbo.[$Table] ORDER BY FileName, PartNo"
|
||||
}
|
||||
$reader = $null
|
||||
$groups = @{}
|
||||
try {
|
||||
$reader = $command.ExecuteReader()
|
||||
while ($reader.Read()) {
|
||||
$fileName = [string]$reader.GetValue(0)
|
||||
if (-not $groups.ContainsKey($fileName)) { $groups[$fileName] = @() }
|
||||
$groups[$fileName] += [pscustomobject]@{
|
||||
part_no = [int]$reader.GetValue(1)
|
||||
bytes = [byte[]]$reader.GetValue(2)
|
||||
}
|
||||
}
|
||||
}
|
||||
finally {
|
||||
if ($reader -ne $null) { $reader.Dispose() }
|
||||
if ($command -ne $null) { $command.Dispose() }
|
||||
}
|
||||
|
||||
$result = @{}
|
||||
foreach ($fileName in $groups.Keys) {
|
||||
$bytes = Join-ByteArrays @($groups[$fileName])
|
||||
$result[$fileName] = [pscustomobject]@{
|
||||
file_name = $fileName
|
||||
chunks = @($groups[$fileName]).Count
|
||||
bytes = $bytes.Length
|
||||
sha256 = Convert-ToSha256Hex $bytes
|
||||
}
|
||||
}
|
||||
return $result
|
||||
}
|
||||
|
||||
function Read-Extensions {
|
||||
param([System.Data.SqlClient.SqlConnection]$Connection)
|
||||
$command = $Connection.CreateCommand()
|
||||
$command.CommandText = "SELECT [_IDRRef], [_ExtName], [_ExtensionOrder], [_ExtensionUsePurpose], [_ExtensionScope] FROM dbo.[_ExtensionsInfo] ORDER BY [_ExtensionOrder], [_ExtName]"
|
||||
$reader = $null
|
||||
$map = @{}
|
||||
try {
|
||||
$reader = $command.ExecuteReader()
|
||||
while ($reader.Read()) {
|
||||
$bytes = [byte[]]$reader.GetValue(0)
|
||||
$guid = Convert-RefBytesToGuid $bytes
|
||||
$map[$guid] = [pscustomobject]@{
|
||||
guid = $guid
|
||||
name = [string]$reader.GetValue(1)
|
||||
order = [decimal]$reader.GetValue(2)
|
||||
use_purpose = [decimal]$reader.GetValue(3)
|
||||
scope = [decimal]$reader.GetValue(4)
|
||||
}
|
||||
}
|
||||
}
|
||||
finally {
|
||||
if ($reader -ne $null) { $reader.Dispose() }
|
||||
if ($command -ne $null) { $command.Dispose() }
|
||||
}
|
||||
return $map
|
||||
}
|
||||
|
||||
function Get-BaseObjectInfo {
|
||||
param([string]$Guid)
|
||||
$metadataRoot = [System.IO.Path]::GetFullPath($BaseMetadataDir)
|
||||
if (-not (Test-Path -LiteralPath $metadataRoot)) { return $null }
|
||||
$match = Get-ChildItem -LiteralPath $metadataRoot -Recurse -Filter *.json -File |
|
||||
Select-String -SimpleMatch $Guid -List |
|
||||
Select-Object -First 1
|
||||
if (-not $match) { return $null }
|
||||
$data = Get-Content -LiteralPath $match.Path -Raw -Encoding UTF8 | ConvertFrom-Json
|
||||
$kind = [string]$data.kind
|
||||
$name = [string]$data.identity.name
|
||||
$synonym = $null
|
||||
if ($data.identity.synonyms -and $data.identity.synonyms.ru) { $synonym = [string]$data.identity.synonyms.ru }
|
||||
return [pscustomobject]@{
|
||||
layer = "base"
|
||||
kind = $kind
|
||||
kind_ru = if ($kindRu.ContainsKey($kind)) { $kindRu[$kind] } else { $kind }
|
||||
name = $name
|
||||
synonym = $synonym
|
||||
full_name = "$(if ($kindRu.ContainsKey($kind)) { $kindRu[$kind] } else { $kind }).$name"
|
||||
guid = $Guid
|
||||
evidence = [pscustomobject]@{
|
||||
metadata_file = $match.Path
|
||||
xml_file = $data.xml_file
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function Get-ExtensionGuidMap {
|
||||
$path = [System.IO.Path]::GetFullPath($ExtensionGuidIndex)
|
||||
if (-not (Test-Path -LiteralPath $path)) { return @{} }
|
||||
$data = Get-Content -LiteralPath $path -Raw -Encoding UTF8 | ConvertFrom-Json
|
||||
return $data.guid_map
|
||||
}
|
||||
|
||||
function Get-ExtensionObjectInfo {
|
||||
param(
|
||||
[string]$ExtensionName,
|
||||
[string]$ObjectGuid,
|
||||
[object]$GuidMap
|
||||
)
|
||||
$entry = $GuidMap.$ObjectGuid
|
||||
$top = $null
|
||||
if ($entry -and $entry.top_objects -and @($entry.top_objects).Count -gt 0) {
|
||||
$top = @($entry.top_objects)[0]
|
||||
}
|
||||
$relativePath = if ($top) { [string]$top.relative_path } else { $null }
|
||||
$parts = if ($relativePath) { $relativePath -split "[\\/]" } else { @() }
|
||||
$parentKind = $null
|
||||
$parentName = $null
|
||||
$artifactKind = if ($top) { [string]$top.xml_kind } else { "ConfigCASObject" }
|
||||
$artifactName = if ($top) { [string]$top.name } else { $ObjectGuid }
|
||||
$synonym = if ($top -and $top.synonym) { [string]$top.synonym } else { $null }
|
||||
for ($i = 0; $i -lt $parts.Count - 1; $i++) {
|
||||
if ($folderKind.ContainsKey($parts[$i]) -and $parts[$i] -ne "Forms" -and $parts[$i] -ne "Templates") {
|
||||
$parentKind = $folderKind[$parts[$i]]
|
||||
if ($i + 1 -lt $parts.Count) { $parentName = $parts[$i + 1] }
|
||||
break
|
||||
}
|
||||
}
|
||||
$artifactKindRu = if ($kindRu.ContainsKey($artifactKind)) { $kindRu[$artifactKind] } else { $artifactKind }
|
||||
$parentKindRu = if ($parentKind -and $kindRu.ContainsKey($parentKind)) { $kindRu[$parentKind] } else { $parentKind }
|
||||
$fullName = if ($parentKind -and $artifactKind -eq "Form") {
|
||||
"$ruExtension.$ExtensionName.$parentKindRu.$parentName.$ruForm.$artifactName"
|
||||
}
|
||||
elseif ($parentKind) {
|
||||
"$ruExtension.$ExtensionName.$parentKindRu.$parentName.$artifactKindRu.$artifactName"
|
||||
}
|
||||
else {
|
||||
"$ruExtension.$ExtensionName.$artifactKindRu.$artifactName"
|
||||
}
|
||||
return [pscustomobject]@{
|
||||
layer = "extension"
|
||||
extension = $ExtensionName
|
||||
parent_kind = $parentKind
|
||||
parent_kind_ru = $parentKindRu
|
||||
parent_name = $parentName
|
||||
kind = $artifactKind
|
||||
kind_ru = $artifactKindRu
|
||||
name = $artifactName
|
||||
synonym = $synonym
|
||||
full_name = $fullName
|
||||
guid = $ObjectGuid
|
||||
relative_path = $relativePath
|
||||
evidence = [pscustomobject]@{
|
||||
xml_path = if ($top) { [string]$top.path } else { $null }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function Split-ObjectPart {
|
||||
param([string]$FileName)
|
||||
if ($FileName -match '^([0-9a-fA-F-]{36})(\..+)?$') {
|
||||
return [pscustomobject]@{ object_id = $Matches[1].ToLowerInvariant(); suffix = if ($Matches[2]) { $Matches[2] } else { "" } }
|
||||
}
|
||||
return $null
|
||||
}
|
||||
|
||||
function Split-ExtensionFileName {
|
||||
param([string]$FileName)
|
||||
if ($FileName -match '^([0-9a-fA-F-]{36})__(.+)$') {
|
||||
$rest = $Matches[2]
|
||||
if ($rest -eq "configinfo") {
|
||||
return [pscustomobject]@{ extension_id = $Matches[1].ToLowerInvariant(); object_id = $null; suffix = ""; system_name = "configinfo" }
|
||||
}
|
||||
$part = Split-ObjectPart $rest
|
||||
return [pscustomobject]@{ extension_id = $Matches[1].ToLowerInvariant(); object_id = $part.object_id; suffix = $part.suffix; system_name = $null }
|
||||
}
|
||||
return $null
|
||||
}
|
||||
|
||||
function New-StorageEvidence {
|
||||
param(
|
||||
[string]$SavedTable,
|
||||
[string]$ActiveTable,
|
||||
[string]$FileName,
|
||||
[object]$Saved,
|
||||
[object]$Active
|
||||
)
|
||||
return [pscustomobject]@{
|
||||
saved_table = $SavedTable
|
||||
active_table = $ActiveTable
|
||||
file_name = $FileName
|
||||
saved_bytes = $Saved.bytes
|
||||
active_bytes = if ($Active) { $Active.bytes } else { $null }
|
||||
saved_sha256 = $Saved.sha256
|
||||
active_sha256 = if ($Active) { $Active.sha256 } else { $null }
|
||||
active_exists = [bool]$Active
|
||||
changed = (-not $Active) -or ($Saved.sha256 -ne $Active.sha256) -or ($Saved.bytes -ne $Active.bytes)
|
||||
}
|
||||
}
|
||||
|
||||
$connection = New-Connection
|
||||
try {
|
||||
$configSave = Read-TableFiles $connection "ConfigSave" $null
|
||||
$configCassave = Read-TableFiles $connection "ConfigCASSave" $null
|
||||
$configActive = Read-TableFiles $connection "Config" @($configSave.Keys)
|
||||
$configCasActive = Read-TableFiles $connection "ConfigCAS" @($configCassave.Keys)
|
||||
$extensions = Read-Extensions $connection
|
||||
}
|
||||
finally {
|
||||
if ($connection -ne $null) { $connection.Dispose() }
|
||||
}
|
||||
|
||||
$objectChangesByKey = @{}
|
||||
$systemChanges = @()
|
||||
|
||||
foreach ($fileName in $configSave.Keys) {
|
||||
$saved = $configSave[$fileName]
|
||||
$active = if ($configActive.ContainsKey($fileName)) { $configActive[$fileName] } else { $null }
|
||||
$evidence = New-StorageEvidence "ConfigSave" "Config" $fileName $saved $active
|
||||
if (-not $evidence.changed) { continue }
|
||||
$part = Split-ObjectPart $fileName
|
||||
if (-not $part) {
|
||||
$systemChanges += [pscustomobject]@{ layer = "base"; name = $fileName; storage = $evidence }
|
||||
continue
|
||||
}
|
||||
$info = Get-BaseObjectInfo $part.object_id
|
||||
if (-not $info) {
|
||||
$info = [pscustomobject]@{
|
||||
layer = "base"
|
||||
kind = "ConfigObject"
|
||||
kind_ru = $ruConfigObject
|
||||
name = $part.object_id
|
||||
synonym = $null
|
||||
full_name = "$ruConfigObject.$($part.object_id)"
|
||||
guid = $part.object_id
|
||||
evidence = $null
|
||||
}
|
||||
}
|
||||
$key = "base::$($part.object_id)"
|
||||
if (-not $objectChangesByKey.ContainsKey($key)) {
|
||||
$objectChangesByKey[$key] = [ordered]@{
|
||||
layer = $info.layer
|
||||
kind = $info.kind
|
||||
kind_ru = $info.kind_ru
|
||||
name = $info.name
|
||||
synonym = $info.synonym
|
||||
full_name = $info.full_name
|
||||
guid = $info.guid
|
||||
change_state = "saved_not_applied"
|
||||
storage = @()
|
||||
evidence = $info.evidence
|
||||
}
|
||||
}
|
||||
$objectChangesByKey[$key].storage += $evidence
|
||||
}
|
||||
|
||||
$extensionGuidMap = Get-ExtensionGuidMap
|
||||
foreach ($fileName in $configCassave.Keys) {
|
||||
$saved = $configCassave[$fileName]
|
||||
$active = if ($configCasActive.ContainsKey($fileName)) { $configCasActive[$fileName] } else { $null }
|
||||
$evidence = New-StorageEvidence "ConfigCASSave" "ConfigCAS" $fileName $saved $active
|
||||
if (-not $evidence.changed) { continue }
|
||||
$part = Split-ExtensionFileName $fileName
|
||||
if (-not $part) {
|
||||
$systemChanges += [pscustomobject]@{ layer = "extension"; name = $fileName; storage = $evidence }
|
||||
continue
|
||||
}
|
||||
$extension = if ($extensions.ContainsKey($part.extension_id)) { $extensions[$part.extension_id] } else { $null }
|
||||
$extensionName = if ($extension) { $extension.name } else { $part.extension_id }
|
||||
if ($part.system_name) {
|
||||
$systemChanges += [pscustomobject]@{ layer = "extension"; extension = $extensionName; name = $part.system_name; storage = $evidence }
|
||||
continue
|
||||
}
|
||||
$info = Get-ExtensionObjectInfo $extensionName $part.object_id $extensionGuidMap
|
||||
$key = "extension::$($part.extension_id)::$($part.object_id)"
|
||||
if (-not $objectChangesByKey.ContainsKey($key)) {
|
||||
$objectChangesByKey[$key] = [ordered]@{
|
||||
layer = $info.layer
|
||||
extension = $info.extension
|
||||
parent_kind = $info.parent_kind
|
||||
parent_kind_ru = $info.parent_kind_ru
|
||||
parent_name = $info.parent_name
|
||||
kind = $info.kind
|
||||
kind_ru = $info.kind_ru
|
||||
name = $info.name
|
||||
synonym = $info.synonym
|
||||
full_name = $info.full_name
|
||||
guid = $info.guid
|
||||
relative_path = $info.relative_path
|
||||
change_state = "saved_not_applied"
|
||||
storage = @()
|
||||
evidence = $info.evidence
|
||||
}
|
||||
}
|
||||
$objectChangesByKey[$key].storage += $evidence
|
||||
}
|
||||
|
||||
$objectChanges = @($objectChangesByKey.Values | ForEach-Object { [pscustomobject]$_ } | Sort-Object layer, extension, full_name)
|
||||
$result = [pscustomobject]@{
|
||||
schema = "onec_saved_state_object_comparison.v1"
|
||||
server = $Server
|
||||
database = $Database
|
||||
view = "saved_not_applied_vs_active"
|
||||
object_changes = $objectChanges
|
||||
system_changes = $systemChanges
|
||||
counts = [pscustomobject]@{
|
||||
object_changes = $objectChanges.Count
|
||||
system_changes = $systemChanges.Count
|
||||
config_save_files = $configSave.Count
|
||||
config_cas_save_files = $configCassave.Count
|
||||
}
|
||||
safety = [pscustomobject]@{
|
||||
read_only = $true
|
||||
sql_write_performed = $false
|
||||
exposes_storage_evidence = $true
|
||||
public_terms_are_1c_objects = $true
|
||||
}
|
||||
}
|
||||
|
||||
$json = $result | ConvertTo-Json -Depth 20
|
||||
if ($Output) {
|
||||
$parent = Split-Path -Parent $Output
|
||||
if ($parent) { New-Item -ItemType Directory -Force -Path $parent | Out-Null }
|
||||
$json | Set-Content -LiteralPath $Output -Encoding UTF8
|
||||
}
|
||||
$json
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user