Initial SQL-only 1C adapter baseline

This commit is contained in:
2026-07-22 03:03:47 +03:00
commit e2503b77e7
545 changed files with 184711 additions and 0 deletions
+228
View File
@@ -0,0 +1,228 @@
#!/usr/bin/env python3
"""Atomically add a BSL handler, form command, and visible form button."""
from __future__ import annotations
import argparse
import base64
import json
from pathlib import Path
from typing import Any
from diff_1c_patch_workspace import build_diff
from edit_1c_bsl_routine import edit_workspace as edit_bsl_routine
from edit_1c_form_button import edit_workspace as edit_form_button
from edit_1c_form_command import edit_workspace as edit_form_command
from validate_1c_patch_workspace_semantics import validate_workspace
def load_json(path: Path) -> dict[str, Any]:
return json.loads(path.read_text(encoding="utf-8-sig"))
def write_json(path: Path, data: dict[str, Any]) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(json.dumps(data, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
def read_bytes(path: Path) -> bytes:
return path.read_bytes()
def workspace_working_files(workspace: Path) -> list[Path]:
manifest = load_json(workspace / "manifest.json")
paths = []
for record in manifest.get("files") or []:
relative = Path(str(record.get("relative_path") or ""))
if relative.is_absolute() or ".." in relative.parts:
raise SystemExit(f"Unsafe manifest relative path: {relative}")
path = workspace / "working" / relative
if path.exists():
paths.append(path)
return paths
def snapshot_working_files(workspace: Path) -> dict[Path, bytes]:
return {path: read_bytes(path) for path in workspace_working_files(workspace)}
def restore_snapshot(snapshot: dict[Path, bytes]) -> None:
for path, content in snapshot.items():
path.parent.mkdir(parents=True, exist_ok=True)
path.write_bytes(content)
def decode_routine_text(args: argparse.Namespace) -> str:
sources = [bool(args.routine_text), bool(args.routine_text_b64), bool(args.routine_file)]
if sum(sources) != 1:
raise SystemExit("Use exactly one of --routine-text, --routine-text-b64, or --routine-file.")
if args.routine_text is not None:
return args.routine_text
if args.routine_text_b64:
return base64.b64decode(args.routine_text_b64).decode("utf-8")
return Path(args.routine_file).read_text(encoding="utf-8-sig")
def build_result(
*,
workspace: Path,
form_relative_path: str,
bsl_relative_path: str,
operation: str,
routine_text: str,
command_name: str,
command_title: str,
command_action: str,
button_parent_name: str,
button_name: str,
button_title: str,
keep_on_failure: bool,
max_patch_chars: int,
) -> dict[str, Any]:
snapshot = snapshot_working_files(workspace)
steps: list[dict[str, Any]] = []
rolled_back = False
error: dict[str, Any] | None = None
try:
bsl = edit_bsl_routine(
workspace,
bsl_relative_path,
routine_text,
operation=operation,
keep_on_failure=True,
)
steps.append({"name": "bsl_routine", "result": bsl})
if not bsl.get("semantic_validation", {}).get("passed"):
raise RuntimeError("BSL routine edit failed semantic validation.")
command = edit_form_command(
workspace,
form_relative_path,
name=command_name,
title=command_title,
action=command_action,
tooltip=None,
command_id=None,
call_type="Override",
operation=operation,
keep_on_failure=True,
)
steps.append({"name": "form_command", "result": command})
if not command.get("semantic_validation", {}).get("passed"):
raise RuntimeError("Form command edit failed semantic validation.")
button = edit_form_button(
workspace,
form_relative_path,
parent_name=button_parent_name,
name=button_name,
title=button_title,
command_name=command_name,
button_id=None,
button_type="CommandBarButton",
operation=operation,
keep_on_failure=True,
)
steps.append({"name": "form_button", "result": button})
if not button.get("semantic_validation", {}).get("passed"):
raise RuntimeError("Form button edit failed semantic validation.")
semantic = validate_workspace(workspace)
if not semantic.get("passed"):
raise RuntimeError("Final semantic validation failed.")
diff = build_diff(workspace, max_patch_chars=max_patch_chars)
except (Exception, SystemExit) as exc:
error = {"type": type(exc).__name__, "message": str(exc)}
semantic = validate_workspace(workspace)
diff = build_diff(workspace, max_patch_chars=max_patch_chars)
if not keep_on_failure:
restore_snapshot(snapshot)
rolled_back = True
semantic = validate_workspace(workspace)
diff = build_diff(workspace, max_patch_chars=max_patch_chars)
passed = error is None and bool(semantic.get("passed")) and bool(diff.get("passed"))
return {
"schema": "onec_form_button_workflow.v1",
"workspace": str(workspace),
"operation": operation,
"inputs": {
"form_relative_path": form_relative_path,
"bsl_relative_path": bsl_relative_path,
"command_name": command_name,
"command_title": command_title,
"command_action": command_action,
"button_parent_name": button_parent_name,
"button_name": button_name,
"button_title": button_title,
},
"passed": passed,
"rolled_back": rolled_back,
"error": error,
"steps": steps,
"semantic_validation": {
"schema": semantic.get("schema"),
"passed": semantic.get("passed"),
"counts": semantic.get("counts"),
"findings": semantic.get("findings"),
},
"diff_summary": diff.get("counts"),
}
def main() -> int:
parser = argparse.ArgumentParser(description="Atomically add BSL handler, form command, and visible button in a 1C patch workspace.")
parser.add_argument("--workspace", type=Path, required=True)
parser.add_argument("--form-relative-path", required=True)
parser.add_argument("--bsl-relative-path", required=True)
parser.add_argument("--operation", choices=["append", "replace", "upsert"], default="upsert")
parser.add_argument("--routine-text")
parser.add_argument("--routine-text-b64")
parser.add_argument("--routine-file", type=Path)
parser.add_argument("--command-name", required=True)
parser.add_argument("--command-title", required=True)
parser.add_argument("--command-action", required=True)
parser.add_argument("--button-parent-name", required=True)
parser.add_argument("--button-name", required=True)
parser.add_argument("--button-title", required=True)
parser.add_argument("--keep-on-failure", action="store_true", help="Keep partial edits when any step fails.")
parser.add_argument("--max-patch-chars", type=int, default=200000)
parser.add_argument("--output", type=Path)
args = parser.parse_args()
result = build_result(
workspace=args.workspace,
form_relative_path=args.form_relative_path,
bsl_relative_path=args.bsl_relative_path,
operation=args.operation,
routine_text=decode_routine_text(args),
command_name=args.command_name,
command_title=args.command_title,
command_action=args.command_action,
button_parent_name=args.button_parent_name,
button_name=args.button_name,
button_title=args.button_title,
keep_on_failure=args.keep_on_failure,
max_patch_chars=args.max_patch_chars,
)
if args.output:
write_json(args.output, result)
print(
json.dumps(
{
"output": str(args.output) if args.output else None,
"passed": result["passed"],
"rolled_back": result["rolled_back"],
"error": result["error"],
"semantic": result["semantic_validation"]["counts"],
"diff": result["diff_summary"],
},
ensure_ascii=False,
)
)
return 0 if result["passed"] else 2
if __name__ == "__main__":
raise SystemExit(main())
+42
View File
@@ -0,0 +1,42 @@
#!/usr/bin/env python3
"""Compare one base Config SQL payload with XML files.
This is the base-configuration counterpart to extension manifest part analysis.
It uses the same mechanical payload parser and XML matching logic.
"""
from __future__ import annotations
import argparse
import json
from pathlib import Path
from analyze_1c_manifest_object_parts import load_xml_files, parse_cas_payload, public_payload_report
def main() -> int:
parser = argparse.ArgumentParser(description="Compare one Config object payload with XML files.")
parser.add_argument("--config-file", type=Path, required=True)
parser.add_argument("--object-guid", required=True)
parser.add_argument("--xml-path", type=Path, action="append", default=[])
parser.add_argument("--output", type=Path, required=True)
args = parser.parse_args()
xml_files = load_xml_files(args.xml_path)
payload = parse_cas_payload(args.config_file)
report = {
"schema": "onec_config_object_xml_compare.v1",
"object_guid": args.object_guid.lower(),
"config_file": str(args.config_file),
"xml_paths": [str(path) for path in args.xml_path],
"xml_file_count": len(xml_files),
"payload": public_payload_report(payload, xml_files),
}
args.output.parent.mkdir(parents=True, exist_ok=True)
args.output.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
print(json.dumps({"output": str(args.output), "xml_files": len(xml_files)}, ensure_ascii=False))
return 0
if __name__ == "__main__":
raise SystemExit(main())
+440
View File
@@ -0,0 +1,440 @@
#!/usr/bin/env python3
"""Compare one extension manifest object parts with an XML export tree.
The script is intentionally evidence-first. It reports mechanical facts:
manifest object_id parts, ConfigCAS keys, decoded payload structure, embedded
base64 blobs, and exact/contains matches against files from an XML export.
It does not assign semantic names to suffixes.
"""
from __future__ import annotations
import argparse
import base64
import hashlib
import json
import re
from pathlib import Path
from typing import Any
from inspect_1c_sql_files import GUID_RE, Lexer, Parser, collect_strings, tree_shape, try_decode, try_decompress
BASE64_RE = re.compile(r"[A-Za-z0-9+/]{40,}={0,2}")
BSL_MARKERS = ("&На", "Процедура ", "Функция ", "#Область", "#КонецОбласти")
HTML_MARKERS = ("<!DOCTYPE", "<html", "<HTML", "<body", "<BODY")
STREAM_HEADER_RE = re.compile(rb"\r\n([0-9a-f]{8}) ([0-9a-f]{8}) 7fffffff \r\n")
def scalar(node: Any) -> str:
if isinstance(node, dict) and node.get("type") in {"atom", "string"}:
return str(node.get("value") or "")
return ""
def suffix_of(object_id: str) -> str:
parts = object_id.split(".", 1)
return "" if len(parts) == 1 else "." + parts[1]
def sha1_hex(data: bytes) -> str:
return hashlib.sha1(data).hexdigest()
def decode_text(data: bytes) -> tuple[str | None, str | None]:
if data.startswith(b"\xef\xbb\xbf"):
try:
return data.decode("utf-8-sig"), "utf-8-sig"
except UnicodeDecodeError:
pass
return try_decode(data)
def decode_payload_text(data: bytes) -> tuple[str | None, str | None, int]:
if data.startswith(b"\xef\xbb\xbf"):
text, encoding = decode_text(data)
return text, encoding, 0
marker = data.find(b"\xef\xbb\xbf")
if marker >= 0:
try:
return data[marker:].decode("utf-8-sig"), "utf-8-sig", marker
except UnicodeDecodeError:
pass
text, encoding = try_decode(data)
return text, encoding, 0
def payload_markers(data: bytes) -> list[str]:
markers = []
if data.startswith(b"MOXCEL"):
markers.append("MOXCEL")
if data.startswith(b"\xef\xbb\xbf") or b"\xef\xbb\xbf" in data[:256]:
markers.append("utf8_bom")
if STREAM_HEADER_RE.search(data):
markers.append("stream_headers")
return markers
def load_xml_files(paths: list[Path]) -> list[dict[str, Any]]:
files: list[dict[str, Any]] = []
seen: set[Path] = set()
for root in paths:
if not root.exists():
continue
candidates = [root] if root.is_file() else [item for item in root.rglob("*") if item.is_file()]
for path in candidates:
resolved = path.resolve()
if resolved in seen:
continue
seen.add(resolved)
data = path.read_bytes()
text, encoding = decode_text(data)
files.append(
{
"path": str(path),
"name": path.name,
"relative_hint": str(path),
"bytes": len(data),
"sha1": sha1_hex(data),
"text": text,
"encoding": encoding,
}
)
return files
def collect_atoms(value: Any) -> list[str]:
atoms: list[str] = []
def walk(node: Any) -> None:
if isinstance(node, dict) and node.get("type") == "atom":
atoms.append(str(node.get("value") or ""))
if isinstance(node, dict):
for child in node.get("items") or []:
walk(child)
walk(value)
return atoms
def collect_base64_blocks(value: Any) -> list[str]:
blocks: list[str] = []
def walk(node: Any) -> None:
if isinstance(node, dict) and node.get("type") == "list":
items = node.get("items") or []
if items and scalar(items[0]) == "#base64":
chunks = [scalar(item) for item in items[1:] if BASE64_RE.fullmatch(scalar(item))]
if chunks:
blocks.append("".join(chunks))
for child in items:
walk(child)
walk(value)
return blocks
def decode_base64_atoms(atoms: list[str]) -> list[dict[str, Any]]:
decoded: list[dict[str, Any]] = []
for value in atoms:
if not BASE64_RE.fullmatch(value):
continue
try:
data = base64.b64decode(value, validate=True)
except Exception:
continue
if not data:
continue
text, encoding = decode_text(data)
text_preview = ""
if text:
text_preview = text.replace("\x00", "")[:300]
decoded.append(
{
"atom_length": len(value),
"bytes": len(data),
"sha1": sha1_hex(data),
"encoding": encoding,
"text_preview": text_preview,
"has_bsl_marker": bool(text and any(marker in text for marker in BSL_MARKERS)),
"has_html_marker": bool(text and any(marker in text for marker in HTML_MARKERS)),
"bytes_base64": value[:120],
"data": data,
"text": text,
}
)
return decoded
def decode_base64_blocks(blocks: list[str]) -> list[dict[str, Any]]:
decoded: list[dict[str, Any]] = []
for value in blocks:
try:
data = base64.b64decode(value, validate=True)
except Exception:
continue
text, encoding = decode_text(data)
decoded.append(
{
"block_length": len(value),
"bytes": len(data),
"sha1": sha1_hex(data),
"encoding": encoding,
"text_preview": (text or "").replace("\x00", "")[:500],
"has_bsl_marker": bool(text and any(marker in text for marker in BSL_MARKERS)),
"has_html_marker": bool(text and any(marker in text for marker in HTML_MARKERS)),
"data": data,
"text": text,
}
)
return decoded
def match_blob(blob: bytes, text: str | None, xml_files: list[dict[str, Any]]) -> dict[str, Any]:
exact = []
normalized_equal = []
contains = []
blob_sha1 = sha1_hex(blob)
normalized_text = text.replace("\r\n", "\n").strip() if text else None
for item in xml_files:
file_text = item.get("text")
data = Path(item["path"]).read_bytes()
if item["sha1"] == blob_sha1:
exact.append({"path": item["path"], "match": "sha1"})
elif normalized_text and file_text and normalized_text == file_text.replace("\r\n", "\n").strip():
normalized_equal.append({"path": item["path"], "match": "normalized_text_equal"})
elif len(blob) >= 24 and blob in data:
contains.append({"path": item["path"], "match": "bytes_contains"})
elif text and file_text and len(text.strip()) >= 24 and text.strip() in file_text:
contains.append({"path": item["path"], "match": "text_contains"})
elif text and file_text and len(file_text.strip()) >= 24 and file_text.strip() in text:
contains.append({"path": item["path"], "match": "payload_contains_file_text"})
return {"exact": exact, "normalized_equal": normalized_equal, "contains": contains}
def extract_stream_blocks(payload: bytes) -> list[dict[str, Any]]:
blocks: list[dict[str, Any]] = []
for match in STREAM_HEADER_RE.finditer(payload):
declared_1 = int(match.group(1), 16)
declared_2 = int(match.group(2), 16)
start = match.end()
size = declared_2
if size <= 0 or start + size > len(payload):
continue
data = payload[start : start + size]
text, encoding = decode_text(data)
blocks.append(
{
"header_offset": match.start(),
"data_offset": start,
"declared_1": declared_1,
"declared_2": declared_2,
"bytes": len(data),
"sha1": sha1_hex(data),
"encoding": encoding,
"text_preview": (text or "").replace("\x00", "")[:500],
"has_bsl_marker": bool(text and any(marker in text for marker in BSL_MARKERS)),
"has_html_marker": bool(text and any(marker in text for marker in HTML_MARKERS)),
"data": data,
"text": text,
}
)
return blocks
def match_strings(strings: list[str], xml_files: list[dict[str, Any]], *, limit: int = 80) -> list[dict[str, Any]]:
hits: list[dict[str, Any]] = []
for string in strings:
if len(string.strip()) < 4:
continue
paths = []
for item in xml_files:
text = item.get("text") or ""
if string in text:
paths.append(item["path"])
if len(paths) >= 8:
break
if paths:
hits.append({"string": string[:200], "paths": paths})
if len(hits) >= limit:
break
return hits
def match_guids(text: str, xml_files: list[dict[str, Any]], *, limit: int = 200) -> list[dict[str, Any]]:
hits: list[dict[str, Any]] = []
for guid in sorted(set(match.lower() for match in GUID_RE.findall(text))):
paths = []
for item in xml_files:
file_text = (item.get("text") or "").lower()
if guid in file_text:
paths.append(item["path"])
if len(paths) >= 8:
break
if paths:
hits.append({"guid": guid, "paths": paths})
if len(hits) >= limit:
break
return hits
def parse_cas_payload(path: Path) -> dict[str, Any]:
raw = path.read_bytes()
payload, compression = try_decompress(raw)
markers = payload_markers(payload)
if "stream_headers" in markers and not payload.startswith(b"\xef\xbb\xbf"):
text, encoding, text_offset = None, None, 0
else:
text, encoding, text_offset = decode_payload_text(payload)
report: dict[str, Any] = {
"cas_file": path.name,
"bytes": len(raw),
"payload_bytes": len(payload),
"payload_sha1": sha1_hex(payload),
"compression": compression,
"encoding": encoding,
"text_offset": text_offset,
"payload_markers": markers,
"parse_status": "not_text",
"text_preview": "",
"strings": [],
"base64_blobs": [],
"base64_blocks": [],
"stream_blocks": [],
}
report["_raw_payload"] = payload
report["_stream_blocks"] = extract_stream_blocks(payload)
if text is None:
return report
clean = text.replace("\x00", "").replace("\ufeff", "").lstrip("ï»¿п»ї")
report["text_preview"] = clean[:500]
report["has_bsl_marker"] = any(marker in clean for marker in BSL_MARKERS)
report["has_html_marker"] = any(marker in clean for marker in HTML_MARKERS)
if "{" not in clean:
report["parse_status"] = "text_no_braces"
return report
try:
parsed = Parser(Lexer(clean[:2_000_000]).tokens()).parse()
except Exception as exc:
report["parse_status"] = "parse_error"
report["parse_error"] = str(exc)
return report
report["parse_status"] = "parsed"
report["shape"] = tree_shape(parsed, max_depth=4)
strings = collect_strings(parsed, limit=200)
atoms = collect_atoms(parsed)
blocks = collect_base64_blocks(parsed)
report["strings"] = strings
if isinstance(parsed, dict):
items = parsed.get("items") or []
report["root_type"] = parsed.get("type")
report["root_len"] = len(items)
report["root_marker"] = scalar(items[0]) if items else ""
report["_raw_payload"] = payload
report["_clean_text"] = clean
report["_base64_decoded"] = decode_base64_atoms(atoms)
report["_base64_blocks_decoded"] = decode_base64_blocks(blocks)
return report
def load_manifest_entries(manifest_path: Path, object_guid: str) -> list[dict[str, Any]]:
manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
target = object_guid.lower()
entries = []
for entry in manifest.get("entries") or []:
object_id = str(entry.get("object_id") or "").lower()
if object_id == target or object_id.startswith(target + "."):
entries.append(entry)
entries.sort(key=lambda item: (suffix_of(str(item["object_id"])), str(item["object_id"])))
return entries
def public_payload_report(payload_report: dict[str, Any], xml_files: list[dict[str, Any]]) -> dict[str, Any]:
raw_payload = payload_report.pop("_raw_payload", b"")
clean_text = payload_report.pop("_clean_text", "")
decoded = payload_report.pop("_base64_decoded", [])
decoded_blocks = payload_report.pop("_base64_blocks_decoded", [])
stream_blocks = payload_report.pop("_stream_blocks", [])
payload_report["payload_matches"] = match_blob(raw_payload, clean_text, xml_files)
payload_report["string_matches"] = match_strings(payload_report.get("strings") or [], xml_files)
payload_report["guid_matches"] = match_guids(clean_text, xml_files)
public_decoded = []
for blob in decoded:
data = blob.pop("data")
text = blob.pop("text")
blob["matches"] = match_blob(data, text, xml_files)
public_decoded.append(blob)
payload_report["base64_blobs"] = public_decoded
public_blocks = []
for blob in decoded_blocks:
data = blob.pop("data")
text = blob.pop("text")
blob["matches"] = match_blob(data, text, xml_files)
public_blocks.append(blob)
payload_report["base64_blocks"] = public_blocks
public_streams = []
for block in stream_blocks:
data = block.pop("data")
text = block.pop("text")
block["matches"] = match_blob(data, text, xml_files)
public_streams.append(block)
payload_report["stream_blocks"] = public_streams
payload_report["strings"] = (payload_report.get("strings") or [])[:80]
return payload_report
def main() -> int:
parser = argparse.ArgumentParser(description="Compare one manifest object's CAS parts with XML files.")
parser.add_argument("--manifest", type=Path, required=True)
parser.add_argument("--cas-dir", type=Path, required=True)
parser.add_argument("--object-guid", required=True)
parser.add_argument("--xml-path", type=Path, action="append", default=[])
parser.add_argument("--output", type=Path, required=True)
args = parser.parse_args()
xml_files = load_xml_files(args.xml_path)
entries = load_manifest_entries(args.manifest, args.object_guid)
parts = []
for entry in entries:
cas_key = entry["cas_key"]
cas_path = Path(entry.get("cas_path") or args.cas_dir / cas_key)
if not cas_path.is_file():
cas_path = args.cas_dir / cas_key
payload = parse_cas_payload(cas_path) if cas_path.is_file() else {"parse_status": "missing_cas"}
parts.append(
{
"object_id": entry["object_id"],
"suffix": suffix_of(entry["object_id"]),
"cas_key": cas_key,
"cas_path": str(cas_path),
"payload": public_payload_report(payload, xml_files) if cas_path.is_file() else payload,
}
)
report = {
"schema": "onec_manifest_object_part_compare.v1",
"object_guid": args.object_guid.lower(),
"manifest": str(args.manifest),
"cas_dir": str(args.cas_dir),
"xml_paths": [str(path) for path in args.xml_path],
"xml_file_count": len(xml_files),
"xml_files": [
{
"path": item["path"],
"bytes": item["bytes"],
"sha1": item["sha1"],
"encoding": item["encoding"],
}
for item in xml_files
],
"part_count": len(parts),
"parts": parts,
}
args.output.parent.mkdir(parents=True, exist_ok=True)
args.output.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
print(json.dumps({"output": str(args.output), "parts": len(parts), "xml_files": len(xml_files)}, ensure_ascii=False))
return 0
if __name__ == "__main__":
raise SystemExit(main())
+241
View File
@@ -0,0 +1,241 @@
from __future__ import annotations
import argparse
import json
import urllib.request
from pathlib import Path
from typing import Any
import xml.etree.ElementTree as ET
from analyze_1c_template_xml_profiles import merge_ranges
def rpc(adapter_url: str, method: str, payload: dict[str, Any]) -> dict[str, Any]:
body = json.dumps({"method": method, "payload": payload}, ensure_ascii=False).encode("utf-8")
req = urllib.request.Request(
f"{adapter_url.rstrip('/')}/rpc",
data=body,
headers={"Content-Type": "application/json; charset=utf-8"},
method="POST",
)
with urllib.request.urlopen(req, timeout=180) as resp:
return json.loads(resp.read().decode("utf-8", errors="replace"))
def runs(values: list[int]) -> list[dict[str, int]]:
if not values:
return []
result: list[dict[str, int]] = []
start = previous = values[0]
for value in values[1:]:
if value == previous + 1:
previous = value
continue
result.append({"start": start, "end": previous, "length": previous - start + 1})
start = previous = value
result.append({"start": start, "end": previous, "length": previous - start + 1})
return result
def div32_values(numbers: list[Any]) -> list[int]:
return [
int(value) // 32
for value in numbers
if isinstance(value, int) and value > 0 and value <= 4096 and value % 32 == 0
]
def small_values(numbers: list[Any]) -> list[int]:
return [int(value) for value in numbers if isinstance(value, int) and 2 <= value <= 128]
def template_xml_path(root: Path, template: str) -> Path:
return root / template / "Ext" / "Template.xml"
def merge_block_candidate(adapter_url: str, base_id: str, owner_kind: str, owner_name: str, template: str) -> dict[str, Any]:
data = rpc(
adapter_url,
"templates.read",
{
"base_id": base_id,
"kind": owner_kind,
"name": owner_name,
"template": template,
"sections": "merges",
"refresh_cache": False,
},
)
return (((data.get("templates") or [{}])[0].get("structure") or {}).get("merge_record_block_candidates") or [{}])[0]
def merge_block_records(
adapter_url: str,
base_id: str,
owner_kind: str,
owner_name: str,
template: str,
candidate: dict[str, Any],
) -> list[dict[str, Any]]:
position = str(candidate.get("tree_position") or "$.0")
try:
start = int(position.split(".")[1]) + 1
except (IndexError, ValueError):
start = 0
count = int(candidate.get("count") or 0)
data = rpc(
adapter_url,
"templates.read",
{
"base_id": base_id,
"kind": owner_kind,
"name": owner_name,
"template": template,
"sections": "moxel_records",
"max_moxel_records": count + 40,
"moxel_record_start": start,
"moxel_record_end": start + count + 35,
"refresh_cache": False,
},
)
diagnostics = ((data.get("templates") or [{}])[0].get("structure") or {}).get("moxel_record_diagnostics") or [{}]
if isinstance(diagnostics, list):
diagnostics = diagnostics[0] if diagnostics else {}
return [record for record in diagnostics.get("top_level_records") or [] if isinstance(record, dict)][:count]
def analyze_template(
*,
adapter_url: str,
base_id: str,
owner_kind: str,
owner_name: str,
template: str,
xml_root: Path,
) -> dict[str, Any]:
xml_path = template_xml_path(xml_root, template)
merges = merge_ranges(ET.parse(xml_path).getroot(), limit=500)
xml_rows = sorted(set(int(item["row"]) for item in merges))
xml_columns = sorted(set(int(item["column"]) for item in merges) | set(int(item["column"]) + int(item["width"]) - 1 for item in merges))
candidate = merge_block_candidate(adapter_url, base_id, owner_kind, owner_name, template)
records = merge_block_records(adapter_url, base_id, owner_kind, owner_name, template, candidate)
by_value: dict[int, list[int]] = {}
coordinate_records: list[dict[str, Any]] = []
for index, record in enumerate(records, 1):
numbers = record.get("numeric_items") or []
for value in set(small_values(numbers)):
by_value.setdefault(value, []).append(index)
packed_columns = div32_values(numbers)
if packed_columns:
coordinate_records.append(
{
"index": index,
"tree_position": record.get("tree_position"),
"numeric_items": numbers,
"div32": packed_columns,
"small": small_values(numbers),
}
)
value_summaries = [
{
"value": value,
"count": len(indexes),
"record_indexes": indexes[:30],
"runs": runs(indexes),
"matches_xml_row": value in xml_rows,
"matches_xml_column_or_edge": value in xml_columns,
}
for value, indexes in sorted(by_value.items())
]
xml_row_hits = [
{
"row": row,
"count": len(by_value.get(row) or []),
"record_indexes": (by_value.get(row) or [])[:20],
"runs": runs(by_value.get(row) or [])[:8],
}
for row in xml_rows
if by_value.get(row)
]
return {
"template": template,
"xml_merge_count": len(merges),
"sql_block_count": int(candidate.get("count") or 0),
"tree_position": candidate.get("tree_position"),
"xml_rows": xml_rows,
"xml_row_runs": runs(xml_rows),
"xml_columns_and_right_edges": xml_columns,
"value_summaries": value_summaries,
"xml_row_hits": xml_row_hits,
"coordinate_records": coordinate_records[:120],
"coordinate_record_runs": runs([item["index"] for item in coordinate_records]),
}
def render_markdown(payload: dict[str, Any]) -> str:
lines = ["# MOXCEL merge row-band analysis", ""]
for item in payload.get("items") or []:
lines.append(f"## {item.get('template')}")
lines.append("")
lines.append(f"- XML merges: `{item.get('xml_merge_count')}`")
lines.append(f"- SQL block count: `{item.get('sql_block_count')}` at `{item.get('tree_position')}`")
lines.append(f"- XML row runs: `{item.get('xml_row_runs')}`")
lines.append(f"- SQL coordinate-record runs: `{(item.get('coordinate_record_runs') or [])[:20]}`")
lines.append("")
lines.append("### XML Row Hits In SQL Small Scalars")
lines.append("")
lines.append("| Row | Count | Runs | First indexes |")
lines.append("| ---: | ---: | --- | --- |")
for hit in (item.get("xml_row_hits") or [])[:60]:
lines.append(f"| {hit.get('row')} | {hit.get('count')} | `{hit.get('runs')}` | `{(hit.get('record_indexes') or [])[:12]}` |")
lines.append("")
lines.append("### Top Small Scalar Values")
lines.append("")
lines.append("| Value | Count | XML row | XML col/edge | Runs |")
lines.append("| ---: | ---: | --- | --- | --- |")
for value in sorted(item.get("value_summaries") or [], key=lambda row: (-int(row.get("count") or 0), int(row.get("value") or 0)))[:30]:
lines.append(
f"| {value.get('value')} | {value.get('count')} | `{value.get('matches_xml_row')}` | "
f"`{value.get('matches_xml_column_or_edge')}` | `{(value.get('runs') or [])[:8]}` |"
)
lines.append("")
return "\n".join(lines)
def main() -> int:
parser = argparse.ArgumentParser(description="Analyze SQL MOXCEL merge-block row/size scalar bands against XML merge rows.")
parser.add_argument("--adapter-url", default="http://docker-gpu.cin.su:8011")
parser.add_argument("--base-id", default="upo_test")
parser.add_argument("--owner-kind", default="Document")
parser.add_argument("--owner-name", default="АвансовыйОтчет")
parser.add_argument(
"--xml-root",
default=r"Z:\codex\1C\XML\UPO\Структура базы 1с\Конфигурация\Documents\АвансовыйОтчет\Templates",
)
parser.add_argument("--template", action="append", required=True)
parser.add_argument("--output-json", default="reports/1c-template-baselines/moxel-merge-row-band-analysis.json")
parser.add_argument("--output-markdown", default="reports/1c-template-baselines/moxel-merge-row-band-analysis.md")
args = parser.parse_args()
payload = {
"schema": "codex_1c_moxel_merge_row_band_analysis.v1",
"items": [
analyze_template(
adapter_url=args.adapter_url,
base_id=args.base_id,
owner_kind=args.owner_kind,
owner_name=args.owner_name,
template=template,
xml_root=Path(args.xml_root),
)
for template in args.template
],
}
Path(args.output_json).write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
Path(args.output_markdown).write_text(render_markdown(payload), encoding="utf-8")
print(json.dumps({"status": "ok", "json": args.output_json, "markdown": args.output_markdown, "items": len(payload["items"])}, ensure_ascii=False, indent=2))
return 0
if __name__ == "__main__":
raise SystemExit(main())
@@ -0,0 +1,360 @@
from __future__ import annotations
import argparse
import json
import urllib.request
from pathlib import Path
from typing import Any
import xml.etree.ElementTree as ET
from analyze_1c_template_xml_profiles import merge_ranges
def rpc(adapter_url: str, method: str, payload: dict[str, Any]) -> dict[str, Any]:
body = json.dumps({"method": method, "payload": payload}, ensure_ascii=False).encode("utf-8")
req = urllib.request.Request(
f"{adapter_url.rstrip('/')}/rpc",
data=body,
headers={"Content-Type": "application/json; charset=utf-8"},
method="POST",
)
with urllib.request.urlopen(req, timeout=180) as resp:
return json.loads(resp.read().decode("utf-8", errors="replace"))
def template_xml_path(root: Path, template: str) -> Path:
return root / template / "Ext" / "Template.xml"
def range_fields(merges: list[dict[str, Any]]) -> dict[str, set[int]]:
fields: dict[str, set[int]] = {
"top": set(),
"left": set(),
"bottom": set(),
"right": set(),
"width": set(),
"height": set(),
"top_zero": set(),
"left_zero": set(),
"bottom_zero": set(),
"right_zero": set(),
}
for item in merges:
one = (item.get("range") or {}).get("one_based") or {}
zero = (item.get("range") or {}).get("zero_based") or {}
for name in ("top", "left", "bottom", "right"):
if isinstance(one.get(name), int):
fields[name].add(int(one[name]))
if isinstance(zero.get(name), int):
fields[f"{name}_zero"].add(int(zero[name]))
for name in ("width", "height"):
if isinstance(item.get(name), int):
fields[name].add(int(item[name]))
return fields
def fetch_merge_candidate(adapter_url: str, base_id: str, owner_kind: str, owner_name: str, template: str) -> dict[str, Any]:
data = rpc(
adapter_url,
"templates.read",
{
"base_id": base_id,
"kind": owner_kind,
"name": owner_name,
"template": template,
"sections": "merges",
"max_merged": 1,
"refresh_cache": False,
},
)
return (((data.get("templates") or [{}])[0].get("structure") or {}).get("merge_record_block_candidates") or [{}])[0]
def fetch_merge_records(
adapter_url: str,
base_id: str,
owner_kind: str,
owner_name: str,
template: str,
candidate: dict[str, Any],
) -> list[dict[str, Any]]:
position = str(candidate.get("tree_position") or "$.0")
try:
start = int(position.split(".")[1]) + 1
except (IndexError, ValueError):
start = 0
count = int(candidate.get("count") or 0)
data = rpc(
adapter_url,
"templates.read",
{
"base_id": base_id,
"kind": owner_kind,
"name": owner_name,
"template": template,
"sections": "moxel_records",
"max_moxel_records": count + 40,
"moxel_record_start": start,
"moxel_record_end": start + count + 35,
"refresh_cache": False,
},
)
diagnostics = ((data.get("templates") or [{}])[0].get("structure") or {}).get("moxel_record_diagnostics") or [{}]
if isinstance(diagnostics, list):
diagnostics = diagnostics[0] if diagnostics else {}
return [record for record in diagnostics.get("top_level_records") or [] if isinstance(record, dict)][:count]
def values_by_slot(records: list[dict[str, Any]]) -> dict[int, list[int]]:
result: dict[int, list[int]] = {}
for record in records:
numbers = record.get("numeric_items") if isinstance(record.get("numeric_items"), list) else []
for slot, value in enumerate(numbers):
if isinstance(value, int):
result.setdefault(slot, []).append(value)
return result
def score_values(values: list[int], expected: set[int]) -> dict[str, Any]:
if not values or not expected:
return {"hits": 0, "coverage": 0.0, "precision": 0.0, "score": 0.0}
distinct = set(values)
hits = distinct & expected
coverage = len(hits) / len(expected)
precision = len(hits) / len(distinct)
return {
"hits": len(hits),
"coverage": round(coverage, 4),
"precision": round(precision, 4),
"score": round((coverage * 0.7) + (precision * 0.3), 4),
"hit_values": sorted(hits)[:80],
"distinct_values": len(distinct),
}
def slot_candidates(records: list[dict[str, Any]], fields: dict[str, set[int]]) -> list[dict[str, Any]]:
candidates: list[dict[str, Any]] = []
by_slot = values_by_slot(records)
for slot, values in sorted(by_slot.items()):
transforms = {
"raw": values,
"raw_plus_1": [value + 1 for value in values],
"raw_div32": [value // 32 for value in values if value > 0 and value <= 4096 and value % 32 == 0],
"raw_div32_plus_1": [(value // 32) + 1 for value in values if value > 0 and value <= 4096 and value % 32 == 0],
}
for transform, transformed_values in transforms.items():
for field, expected in fields.items():
score = score_values(transformed_values, expected)
if score["hits"] <= 0:
continue
candidates.append(
{
"slot": slot,
"transform": transform,
"field": field,
**score,
"sample_values": sorted(set(transformed_values))[:30],
}
)
candidates.sort(key=lambda item: (-float(item.get("score") or 0), -float(item.get("coverage") or 0), -float(item.get("precision") or 0), int(item.get("slot") or 0), str(item.get("field") or "")))
return candidates
def xml_ordered_fields(merges: list[dict[str, Any]]) -> list[dict[str, int]]:
result: list[dict[str, int]] = []
for item in merges:
one = (item.get("range") or {}).get("one_based") or {}
zero = (item.get("range") or {}).get("zero_based") or {}
row: dict[str, int] = {}
for name in ("top", "left", "bottom", "right"):
if isinstance(one.get(name), int):
row[name] = int(one[name])
if isinstance(zero.get(name), int):
row[f"{name}_zero"] = int(zero[name])
for name in ("width", "height"):
if isinstance(item.get(name), int):
row[name] = int(item[name])
result.append(row)
return result
def transformed_record_value(numbers: list[Any], slot: int, transform: str) -> int | None:
if slot >= len(numbers) or not isinstance(numbers[slot], int):
return None
value = int(numbers[slot])
if transform == "raw":
return value
if transform == "raw_plus_1":
return value + 1
if transform == "raw_div32":
if value <= 0 or value > 4096 or value % 32 != 0:
return None
return value // 32
if transform == "raw_div32_plus_1":
if value <= 0 or value > 4096 or value % 32 != 0:
return None
return (value // 32) + 1
return None
def ordered_slot_candidates(records: list[dict[str, Any]], merges: list[dict[str, Any]]) -> list[dict[str, Any]]:
ordered = xml_ordered_fields(merges)
transforms = ("raw", "raw_plus_1", "raw_div32", "raw_div32_plus_1")
fields = ("top", "left", "bottom", "right", "width", "height", "top_zero", "left_zero", "bottom_zero", "right_zero")
candidates: list[dict[str, Any]] = []
max_slots = max((len(record.get("numeric_items") or []) for record in records), default=0)
for offset in range(0, min(25, len(records))):
pair_count = min(len(ordered), max(0, len(records) - offset))
if pair_count < max(10, min(len(ordered), 20)):
continue
for slot in range(max_slots):
for transform in transforms:
values = [
transformed_record_value(records[offset + index].get("numeric_items") or [], slot, transform)
for index in range(pair_count)
]
available = sum(1 for value in values if value is not None)
if available < max(5, pair_count // 3):
continue
for field in fields:
matches = [
index + 1
for index, value in enumerate(values)
if value is not None and ordered[index].get(field) == value
]
if not matches:
continue
exact_ratio = len(matches) / pair_count
available_ratio = len(matches) / available
if exact_ratio < 0.1 and len(matches) < 8:
continue
candidates.append(
{
"offset": offset,
"slot": slot,
"transform": transform,
"field": field,
"pairs": pair_count,
"available": available,
"matches": len(matches),
"exact_ratio": round(exact_ratio, 4),
"available_ratio": round(available_ratio, 4),
"score": round((exact_ratio * 0.75) + (available_ratio * 0.25), 4),
"first_match_indexes": matches[:30],
}
)
candidates.sort(
key=lambda item: (
-float(item.get("score") or 0),
-float(item.get("exact_ratio") or 0),
-int(item.get("matches") or 0),
int(item.get("offset") or 0),
int(item.get("slot") or 0),
)
)
return candidates
def analyze_template(
*,
adapter_url: str,
base_id: str,
owner_kind: str,
owner_name: str,
template: str,
xml_root: Path,
) -> dict[str, Any]:
merges = merge_ranges(ET.parse(template_xml_path(xml_root, template)).getroot(), limit=1000)
candidate = fetch_merge_candidate(adapter_url, base_id, owner_kind, owner_name, template)
analysis = ((candidate.get("evidence") or {}).get("record_analysis") or {})
records = fetch_merge_records(adapter_url, base_id, owner_kind, owner_name, template, candidate)
fields = range_fields(merges)
return {
"template": template,
"xml_merge_count": len(merges),
"sql_block_count": int(candidate.get("count") or 0),
"tree_position": candidate.get("tree_position"),
"xml_field_values": {name: sorted(values) for name, values in fields.items()},
"record_analysis": {
"schema": analysis.get("schema"),
"records_analyzed": analysis.get("records_analyzed"),
"records_available": len(records),
"raw_records_source": "templates.read.sections=moxel_records",
},
"slot_candidates": slot_candidates(records, fields)[:120],
"ordered_slot_candidates": ordered_slot_candidates(records, merges)[:120],
}
def render_markdown(payload: dict[str, Any]) -> str:
lines = ["# MOXCEL merge slot candidate analysis", ""]
lines.append("XML is used only as an analysis fixture; candidates are SQL decoder hypotheses.")
lines.append("")
for item in payload.get("items") or []:
lines.append(f"## {item.get('template')}")
lines.append("")
lines.append(f"- XML merges: `{item.get('xml_merge_count')}`")
lines.append(f"- SQL block count: `{item.get('sql_block_count')}` at `{item.get('tree_position')}`")
ra = item.get("record_analysis") or {}
lines.append(f"- Record analysis: `{ra.get('schema')}`, records `{ra.get('records_available')}/{ra.get('records_analyzed')}`")
lines.append("")
lines.append("| Slot | Transform | Field | Score | Coverage | Precision | Hit values | Sample values |")
lines.append("| ---: | --- | --- | ---: | ---: | ---: | --- | --- |")
for row in (item.get("slot_candidates") or [])[:40]:
lines.append(
f"| {row.get('slot')} | `{row.get('transform')}` | `{row.get('field')}` | "
f"{row.get('score')} | {row.get('coverage')} | {row.get('precision')} | "
f"`{row.get('hit_values')}` | `{row.get('sample_values')}` |"
)
lines.append("")
lines.append("### Ordered Slot Candidates")
lines.append("")
lines.append("| Offset | Slot | Transform | Field | Score | Exact ratio | Available ratio | Matches | First match indexes |")
lines.append("| ---: | ---: | --- | --- | ---: | ---: | ---: | ---: | --- |")
for row in (item.get("ordered_slot_candidates") or [])[:40]:
lines.append(
f"| {row.get('offset')} | {row.get('slot')} | `{row.get('transform')}` | `{row.get('field')}` | "
f"{row.get('score')} | {row.get('exact_ratio')} | {row.get('available_ratio')} | "
f"{row.get('matches')}/{row.get('pairs')} | `{row.get('first_match_indexes')}` |"
)
lines.append("")
return "\n".join(lines)
def main() -> int:
parser = argparse.ArgumentParser(description="Score SQL MOXCEL merge-block numeric slots against XML merge range fields.")
parser.add_argument("--adapter-url", default="http://docker-gpu.cin.su:8011")
parser.add_argument("--base-id", default="upo_test")
parser.add_argument("--owner-kind", default="Document")
parser.add_argument("--owner-name", default="АвансовыйОтчет")
parser.add_argument(
"--xml-root",
default=r"Z:\codex\1C\XML\UPO\Структура базы 1с\Конфигурация\Documents\АвансовыйОтчет\Templates",
)
parser.add_argument("--template", action="append", required=True)
parser.add_argument("--output-json", default="reports/1c-template-baselines/moxel-merge-slot-candidates.json")
parser.add_argument("--output-markdown", default="reports/1c-template-baselines/moxel-merge-slot-candidates.md")
args = parser.parse_args()
payload = {
"schema": "codex_1c_moxel_merge_slot_candidates.v1",
"source": "analysis_only_xml_fixture",
"items": [
analyze_template(
adapter_url=args.adapter_url,
base_id=args.base_id,
owner_kind=args.owner_kind,
owner_name=args.owner_name,
template=template,
xml_root=Path(args.xml_root),
)
for template in args.template
],
}
Path(args.output_json).write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
Path(args.output_markdown).write_text(render_markdown(payload), encoding="utf-8")
print(json.dumps({"status": "ok", "json": args.output_json, "markdown": args.output_markdown, "items": len(payload["items"])}, ensure_ascii=False, indent=2))
return 0
if __name__ == "__main__":
raise SystemExit(main())
@@ -0,0 +1,148 @@
from __future__ import annotations
import argparse
import json
from pathlib import Path
from typing import Any
FIELDS = ("left", "right", "top", "bottom")
def read_json(path: Path) -> dict[str, Any]:
return json.loads(path.read_text(encoding="utf-8"))
def as_int(value: Any) -> int | None:
try:
return int(value)
except (TypeError, ValueError):
return None
def probe_ranges(payload: dict[str, Any]) -> list[dict[str, Any]]:
probe = payload.get("probe") if isinstance(payload.get("probe"), dict) else payload
ranges = probe.get("named_ranges") or probe.get("named_range_candidates") or []
return [item for item in ranges if isinstance(item, dict)]
def candidate_indexes(raw_scalars: list[Any], expected_one_based: int) -> list[int]:
result = []
for index, value in enumerate(raw_scalars):
if index < 2 or index > 5:
continue
parsed = as_int(value)
if parsed is not None and parsed + 1 == expected_one_based:
result.append(index)
return result
def analyze_range(item: dict[str, Any]) -> dict[str, Any] | None:
range_info = item.get("range") if isinstance(item.get("range"), dict) else {}
one_based = range_info.get("one_based") if isinstance(range_info.get("one_based"), dict) else {}
raw_scalars = item.get("raw_scalars") if isinstance(item.get("raw_scalars"), list) else []
if not one_based or not raw_scalars:
return None
field_candidates: dict[str, list[int]] = {}
for field in FIELDS:
expected = as_int(one_based.get(field))
if expected is None:
continue
field_candidates[field] = candidate_indexes(raw_scalars, expected)
unique_values = len({one_based.get(field) for field in FIELDS if one_based.get(field) is not None})
return {
"name": item.get("name"),
"kind": item.get("kind"),
"one_based": {field: one_based.get(field) for field in FIELDS if field in one_based},
"raw_scalars": raw_scalars,
"field_candidates": field_candidates,
"distinct_coordinate_values": unique_values,
}
def aggregate_rules(samples: list[dict[str, Any]]) -> list[dict[str, Any]]:
rules = []
for field in FIELDS:
sample_candidates = [set(sample.get("field_candidates", {}).get(field) or []) for sample in samples if sample.get("field_candidates", {}).get(field)]
if not sample_candidates:
continue
intersection = set.intersection(*sample_candidates) if sample_candidates else set()
all_distinct = all(int(sample.get("distinct_coordinate_values") or 0) >= 4 for sample in samples)
confidence = "high" if len(intersection) == 1 and all_distinct else "medium" if intersection else "low"
rules.append(
{
"target": f"moxel.named_range.{field}",
"expression": "one_based = int(raw_scalar) + 1",
"raw_scalar_indexes": sorted(intersection) if intersection else sorted(set.union(*sample_candidates)),
"confidence": confidence,
"evidence": {
"samples": len(sample_candidates),
"distinct_rectangular_samples": sum(1 for sample in samples if int(sample.get("distinct_coordinate_values") or 0) >= 4),
},
}
)
return rules
def analyze(probes: list[dict[str, Any]], target_name: str | None = None) -> dict[str, Any]:
samples = []
for payload in probes:
for item in probe_ranges(payload):
if target_name and str(item.get("name") or "") != target_name:
continue
sample = analyze_range(item)
if sample:
samples.append(sample)
return {
"schema": "codex_1c_moxel_named_range_rule_analysis.v1",
"target_name": target_name,
"status": "ok",
"samples": samples,
"rules": aggregate_rules(samples),
"counts": {
"samples": len(samples),
"rules": 0,
},
}
def render_markdown(payload: dict[str, Any]) -> str:
payload["counts"]["rules"] = len(payload.get("rules") or [])
lines = ["# 1C MOXCEL Named Range Rule Analysis", ""]
lines.append(f"- Samples: `{payload.get('counts', {}).get('samples')}`")
lines.append(f"- Rules: `{payload.get('counts', {}).get('rules')}`")
lines.append("")
lines.append("| Target | Confidence | Raw indexes | Samples |")
lines.append("| --- | --- | --- | --- |")
for rule in payload.get("rules") or []:
evidence = rule.get("evidence") or {}
lines.append(
f"| `{rule.get('target')}` | `{rule.get('confidence')}` | "
f"`{', '.join(map(str, rule.get('raw_scalar_indexes') or []))}` | `{evidence.get('samples')}` |"
)
lines.append("")
return "\n".join(lines)
def main() -> int:
parser = argparse.ArgumentParser(description="Infer MOXCEL named range coordinate scalar indexes from probe snapshots.")
parser.add_argument("--probe", action="append", required=True, help="Probe snapshot JSON. Repeatable.")
parser.add_argument("--target-name", help="Optional named range to analyze.")
parser.add_argument("--output-json", default="reports/1c-template-baselines/moxel-named-range-rules.json")
parser.add_argument("--output-markdown", default="reports/1c-template-baselines/moxel-named-range-rules.md")
args = parser.parse_args()
payload = analyze([read_json(Path(path)) for path in args.probe], args.target_name)
payload["counts"]["rules"] = len(payload.get("rules") or [])
json_path = Path(args.output_json)
md_path = Path(args.output_markdown)
json_path.parent.mkdir(parents=True, exist_ok=True)
md_path.parent.mkdir(parents=True, exist_ok=True)
json_path.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8")
md_path.write_text(render_markdown(payload), encoding="utf-8")
print(json.dumps({"status": "ok", "json": str(json_path), "markdown": str(md_path), "counts": payload["counts"]}, ensure_ascii=False, indent=2))
return 0
if __name__ == "__main__":
raise SystemExit(main())
@@ -0,0 +1,306 @@
from __future__ import annotations
import argparse
import json
from pathlib import Path
from typing import Any
ROOT = Path(__file__).resolve().parents[1]
VOLATILE_KEYS = {
"captured_at",
"adapter_url",
"modified",
"bytes",
"file_name",
"template_file",
"label",
"diff",
"cell_id",
}
def read_json(path: Path) -> dict[str, Any]:
return json.loads(path.read_text(encoding="utf-8"))
def resolve_manifest_path(value: str, base_dir: Path) -> Path:
path = Path(value)
if path.is_absolute():
return path
candidates = [
base_dir / path,
ROOT / path,
Path.cwd() / path,
path,
]
for candidate in candidates:
resolved = candidate.resolve()
if resolved.exists():
return resolved
return (base_dir / path).resolve()
def unwrap_structure(payload: dict[str, Any]) -> dict[str, Any]:
if isinstance(payload.get("probe"), dict):
return payload["probe"]
if isinstance(payload.get("structure"), dict):
return payload["structure"]
return payload
def compact_next_record(value: Any) -> Any:
if not isinstance(value, dict):
return value
result = {}
for key in ("type", "value", "head", "scalar_prefix", "list_length", "tree_position"):
if key in value:
result[key] = value[key]
return result
def normalize_item(item: Any) -> Any:
if not isinstance(item, dict):
return item
result: dict[str, Any] = {}
for key, value in item.items():
if key in VOLATILE_KEYS:
continue
if key == "next_moxel_record":
result[key] = compact_next_record(value)
elif key == "style_evidence" and isinstance(value, dict):
result[key] = {
style_key: style_value
for style_key, style_value in value.items()
if style_key in {"immediate_preceding_values", "last_7_preceding_values"}
}
elif isinstance(value, dict):
result[key] = normalize_item(value)
elif isinstance(value, list):
result[key] = [normalize_item(child) for child in value]
else:
result[key] = value
return result
def stable_key(item: dict[str, Any], fallback_index: int) -> str:
for key in ("text", "name"):
if item.get(key) not in {None, ""}:
return f"{key}:{item.get(key)}"
if item.get("tree_position"):
return f"tree:{item.get('tree_position')}"
if item.get("one_based"):
return f"cell:{json.dumps(item.get('one_based'), ensure_ascii=False, sort_keys=True)}"
return f"index:{fallback_index}"
def normalize_section_list(items: Any) -> dict[str, Any]:
if not isinstance(items, list):
return {}
result: dict[str, Any] = {}
for index, item in enumerate(items):
if not isinstance(item, dict):
result[f"index:{index}"] = normalize_item(item)
continue
key = stable_key(item, index)
if key in result:
key = f"{key}#{index}"
result[key] = normalize_item(item)
return result
def normalized_structure(payload: dict[str, Any], *, target_text: str | None = None, target_name: str | None = None) -> dict[str, Any]:
structure = unwrap_structure(payload)
result: dict[str, Any] = {
"counts": normalize_item(structure.get("counts") or {}),
"dimensions": normalize_item(structure.get("dimensions") or {}),
"cells": normalize_section_list(structure.get("cells") or []),
"cell_style_candidates": normalize_section_list(structure.get("cell_style_candidates") or structure.get("cell_styles") or []),
"named_range_candidates": normalize_section_list(structure.get("named_range_candidates") or structure.get("named_ranges") or []),
"named_areas": normalize_section_list(structure.get("named_areas") or []),
"column_widths": normalize_section_list(structure.get("column_widths") or []),
"row_heights": normalize_section_list(structure.get("row_heights") or []),
"merged_ranges": normalize_section_list(structure.get("merged_ranges") or []),
"merged_range_candidates": normalize_section_list(structure.get("merged_range_candidates") or []),
}
if target_text:
result["target_cell_styles"] = {
key: value
for key, value in result["cell_style_candidates"].items()
if isinstance(value, dict) and str(value.get("text") or "") == target_text
}
result["target_cells"] = {
key: value
for key, value in result["cells"].items()
if isinstance(value, dict) and str(value.get("text") or "") == target_text
}
if target_name:
result["target_named_ranges"] = {
key: value
for key, value in result["named_range_candidates"].items()
if isinstance(value, dict) and str(value.get("name") or "") == target_name
}
return result
def diff_values(before: Any, after: Any, path: str = "$") -> list[dict[str, Any]]:
if before == after:
return []
if isinstance(before, dict) and isinstance(after, dict):
changes: list[dict[str, Any]] = []
for key in sorted(set(before) | set(after)):
changes.extend(diff_values(before.get(key), after.get(key), f"{path}.{key}"))
return changes
if isinstance(before, list) and isinstance(after, list):
changes = []
for index in range(max(len(before), len(after))):
old = before[index] if index < len(before) else None
new = after[index] if index < len(after) else None
changes.extend(diff_values(old, new, f"{path}[{index}]"))
return changes
return [{"path": path, "before": before, "after": after}]
def score_change(change: dict[str, Any], target_text: str | None, target_name: str | None) -> int:
path = str(change.get("path") or "")
score = 0
if "target_" in path:
score += 40
if target_text and target_text in path:
score += 30
if target_name and target_name in path:
score += 30
if any(part in path for part in ("next_moxel_record", "style_evidence", "raw_scalars", "column_widths", "row_heights", "merged")):
score += 15
if ".counts." in path:
score -= 20
if ".tree_position" in path:
score -= 10
if path.endswith(".cell_id"):
score -= 20
if change.get("before") is None or change.get("after") is None:
score -= 5
return score
def analyze_experiment(experiment: dict[str, Any], base_dir: Path) -> dict[str, Any]:
before_path = resolve_manifest_path(str(experiment["before"]), base_dir)
after_path = resolve_manifest_path(str(experiment["after"]), base_dir)
target_text = experiment.get("target_text")
target_name = experiment.get("target_name")
before = normalized_structure(read_json(before_path), target_text=target_text, target_name=target_name)
after = normalized_structure(read_json(after_path), target_text=target_text, target_name=target_name)
changes = diff_values(before, after)
scored = sorted(
(
{
**change,
"score": score_change(change, str(target_text) if target_text else None, str(target_name) if target_name else None),
}
for change in changes
),
key=lambda item: (-int(item.get("score") or 0), str(item.get("path") or "")),
)
min_positive = [item for item in scored if int(item.get("score") or 0) > 0]
candidates = min_positive[: int(experiment.get("max_candidates") or 20)]
confidence = "none"
if len(candidates) == 1 and candidates[0]["score"] >= 40:
confidence = "high"
elif candidates and candidates[0]["score"] >= 40:
confidence = "medium"
elif candidates:
confidence = "low"
return {
"property": experiment.get("property"),
"operation": experiment.get("operation"),
"target_text": target_text,
"target_name": target_name,
"before": str(before_path),
"after": str(after_path),
"confidence": confidence,
"candidate_paths": candidates,
"counts": {"changes": len(changes), "candidate_paths": len(candidates)},
}
def default_probe_plan() -> list[dict[str, Any]]:
return [
{"property": "ГоризонтальноеПоложение", "values": ["Лево", "Центр", "Право"], "target": "cell"},
{"property": "ВертикальноеПоложение", "values": ["Верх", "Центр", "Низ"], "target": "cell"},
{"property": "ЦветТекста", "values": ["Черный", "Красный", "Синий"], "target": "cell"},
{"property": "ЦветФона", "values": ["Нет", "Желтый", "Серый"], "target": "cell"},
{"property": "Шрифт.Имя", "values": ["Arial", "Courier New"], "target": "cell"},
{"property": "Шрифт.Размер", "values": [8, 10, 14], "target": "cell"},
{"property": "ГраницаЛево", "values": ["Нет", "Тонкая", "Толстая"], "target": "cell"},
{"property": "ГраницаВерх", "values": ["Нет", "Тонкая", "Толстая"], "target": "cell"},
{"property": "Защита", "values": [True, False], "target": "cell"},
{"property": "Гиперссылка", "values": ["", "https://example.invalid/1c-moxel-probe"], "target": "cell"},
{"property": "Переносить", "values": [True, False], "target": "cell"},
{"property": "ШиринаКолонки", "values": [8, 12, 20], "target": "column"},
{"property": "ВысотаСтроки", "values": [12, 18, 24], "target": "row"},
{"property": "Объединение", "values": ["none", "R8C4:R8C5"], "target": "range"},
]
def render_markdown(payload: dict[str, Any]) -> str:
lines: list[str] = ["# 1C MOXCEL property experiments", ""]
lines.append(f"- Experiments: `{len(payload.get('experiments') or [])}`")
lines.append("")
if payload.get("probe_plan"):
lines.append("## Probe Plan")
lines.append("")
lines.append("| Property | Target | Values |")
lines.append("| --- | --- | --- |")
for item in payload["probe_plan"]:
lines.append(f"| `{item.get('property')}` | `{item.get('target')}` | `{json.dumps(item.get('values'), ensure_ascii=False)}` |")
lines.append("")
if payload.get("experiments"):
lines.append("## Results")
lines.append("")
lines.append("| Property | Confidence | Changes | Top path |")
lines.append("| --- | --- | --- | --- |")
for item in payload["experiments"]:
top = (item.get("candidate_paths") or [{}])[0]
lines.append(
f"| `{item.get('property')}` | `{item.get('confidence')}` | "
f"`{(item.get('counts') or {}).get('changes')}` | `{top.get('path') or ''}` |"
)
lines.append("")
return "\n".join(lines)
def main() -> int:
parser = argparse.ArgumentParser(description="Analyze controlled 1C MOXCEL one-property experiments.")
parser.add_argument("--manifest", help="Experiment manifest JSON.")
parser.add_argument("--output-json", default="reports/1c-template-baselines/Primer3_moxel_property_experiments.json")
parser.add_argument("--output-markdown", default="reports/1c-template-baselines/Primer3_moxel_property_experiments.md")
parser.add_argument("--emit-default-plan", action="store_true", help="Include the default next probe plan.")
args = parser.parse_args()
manifest_path = Path(args.manifest).resolve() if args.manifest else None
manifest = read_json(manifest_path) if manifest_path else {"experiments": []}
base_dir = manifest_path.parent if manifest_path else Path.cwd()
experiments = [
analyze_experiment(experiment, base_dir)
for experiment in manifest.get("experiments") or []
if isinstance(experiment, dict) and experiment.get("before") and experiment.get("after")
]
payload = {
"schema": "codex_1c_moxel_property_experiments.v1",
"manifest": str(manifest_path) if manifest_path else None,
"experiments": experiments,
"probe_plan": default_probe_plan() if args.emit_default_plan or not experiments else [],
}
json_path = Path(args.output_json)
md_path = Path(args.output_markdown)
json_path.parent.mkdir(parents=True, exist_ok=True)
md_path.parent.mkdir(parents=True, exist_ok=True)
json_path.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8")
md_path.write_text(render_markdown(payload), encoding="utf-8")
print(json.dumps({"status": "ok", "json": str(json_path), "markdown": str(md_path), "experiments": len(experiments)}, ensure_ascii=False, indent=2))
return 0
if __name__ == "__main__":
raise SystemExit(main())
@@ -0,0 +1,376 @@
#!/usr/bin/env python3
"""Analyze saved-state object changes beyond storage bytes."""
from __future__ import annotations
import argparse
import difflib
import importlib.util
import json
import re
import sys
from pathlib import Path
from typing import Any
WORD_RE = re.compile(r"[\wА-Яа-яЁё]{3,}", re.UNICODE)
BASE64ISH_RE = re.compile(r"^[A-Za-z0-9+/=_-]{24,}$")
REPO_ROOT = Path(__file__).resolve().parents[1]
def load_payload_to_text():
module_path = REPO_ROOT / "plugins" / "1c" / "parser" / "payload.py"
spec = importlib.util.spec_from_file_location("onec_payload", module_path)
if spec is None or spec.loader is None:
raise RuntimeError(f"Cannot load payload parser: {module_path}")
module = importlib.util.module_from_spec(spec)
sys.modules[spec.name] = module
spec.loader.exec_module(module)
return module.payload_to_text
payload_to_text = load_payload_to_text()
def load_json(path: Path) -> dict[str, Any]:
return json.loads(path.read_text(encoding="utf-8-sig"))
def write_json(path: Path, data: dict[str, Any]) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(json.dumps(data, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
def safe_join(root: Path, file_name: str) -> Path:
relative = Path(file_name.replace("\\", "/"))
if relative.is_absolute() or ".." in relative.parts or not str(relative):
raise ValueError(file_name)
return root / relative
def words(text: str) -> list[str]:
result: list[str] = []
seen: set[str] = set()
for match in WORD_RE.finditer(text):
value = match.group(0)
key = value.casefold()
if key not in seen:
result.append(value)
seen.add(key)
return result
def semantic_words(values: list[str], *, limit: int = 40) -> list[str]:
result: list[str] = []
seen: set[str] = set()
for value in values:
if not value:
continue
if BASE64ISH_RE.match(value):
continue
if value.isdigit():
continue
if len(value) > 80:
continue
has_cyrillic = any("А" <= char <= "я" or char in "Ёё" for char in value)
has_1c_shape = any(marker in value for marker in ("Форма", "Команда", "Реквизит", "Модуль", "Область", "Процедура", "Функция"))
if not has_cyrillic and not has_1c_shape:
continue
key = value.casefold()
if key in seen:
continue
result.append(value)
seen.add(key)
if len(result) >= limit:
break
return result
def classify_payload_part(item: dict[str, Any], file_name: str) -> str:
kind = str(item.get("kind") or "")
suffix = ""
if "." in file_name:
suffix = file_name.rsplit(".", 1)[1]
if kind in {"CommonModule", "ObjectModule", "ManagerModule"} and suffix == "0":
return "bsl_module_text"
if kind == "Form" and not suffix:
return "form_descriptor"
if kind == "Form" and suffix == "0":
return "form_body"
if suffix == "0":
return "primary_payload"
return "metadata_payload"
def textish_from_payload(decoded: dict[str, Any]) -> tuple[str | None, str | None]:
text = decoded.get("text")
if text:
return text, str(decoded.get("encoding") or "")
payload = decoded.get("payload")
if not isinstance(payload, bytes):
return None, None
candidates: list[tuple[str, str, int]] = []
for encoding in ("utf-8-sig", "utf-8", "cp1251", "utf-16-le"):
try:
candidate = payload.decode(encoding, errors="ignore").replace("\x00", "").replace("\ufeff", "")
except Exception:
continue
word_count = len(words(candidate))
cyrillic_count = sum(1 for char in candidate if "А" <= char <= "я" or char in "Ёё")
known_1c_terms = sum(
candidate.count(term)
for term in ("Процедура", "Функция", "Конец", "Если", "Тогда", "Область", "Перем", "Экспорт", "пример")
)
mojibake_penalty = candidate.count("Р") * 8 + candidate.count("С") * 4
score = word_count * 5 + cyrillic_count + known_1c_terms * 500 - mojibake_penalty
if word_count:
candidates.append((candidate, f"{encoding}:lossy", score))
if not candidates:
return None, None
candidates.sort(key=lambda item: item[2], reverse=True)
return candidates[0][0], candidates[0][1]
def common_edges(left: str, right: str) -> tuple[int, int]:
prefix = 0
for a, b in zip(left, right):
if a != b:
break
prefix += 1
suffix = 0
left_tail = left[prefix:]
right_tail = right[prefix:]
for a, b in zip(reversed(left_tail), reversed(right_tail)):
if a != b:
break
suffix += 1
return prefix, suffix
def text_window(text: str, center: int, size: int = 500) -> str:
start = max(center - size // 2, 0)
end = min(center + size // 2, len(text))
return text[start:end].replace("\x00", "")
def line_diff(left: str, right: str, *, limit: int) -> list[str]:
left_lines = left.splitlines()
right_lines = right.splitlines()
diff = list(difflib.unified_diff(left_lines, right_lines, fromfile="active", tofile="saved", lineterm=""))
if len(diff) > limit:
return [*diff[:limit], f"... truncated {len(diff) - limit} lines ..."]
return diff
def analyze_payload(active_path: Path | None, saved_path: Path) -> dict[str, Any]:
saved_raw = saved_path.read_bytes()
saved_decoded = payload_to_text(saved_raw)
active_decoded: dict[str, Any] | None = None
if active_path and active_path.exists():
active_decoded = payload_to_text(active_path.read_bytes())
result: dict[str, Any] = {
"saved_path": str(saved_path),
"active_path": str(active_path) if active_path else None,
"saved": {
"raw_bytes": saved_decoded.get("raw_bytes"),
"payload_bytes": saved_decoded.get("payload_bytes"),
"compression": saved_decoded.get("compression"),
"encoding": saved_decoded.get("encoding"),
},
"active": None,
"text_comparable": False,
}
if active_decoded:
result["active"] = {
"raw_bytes": active_decoded.get("raw_bytes"),
"payload_bytes": active_decoded.get("payload_bytes"),
"compression": active_decoded.get("compression"),
"encoding": active_decoded.get("encoding"),
}
saved_text, saved_text_mode = textish_from_payload(saved_decoded)
active_text, active_text_mode = textish_from_payload(active_decoded) if active_decoded else (None, None)
result["saved"]["text_mode"] = saved_text_mode
if result["active"] is not None:
result["active"]["text_mode"] = active_text_mode
if saved_text is None:
result["summary"] = "Saved payload is not text-decodable."
return result
result["saved_strings_sample"] = words(saved_text)[:80]
if active_text is None:
result["summary"] = "Saved text payload has no active counterpart."
result["text_comparable"] = False
result["saved_text_sample"] = text_window(saved_text, 0)
return result
result["text_comparable"] = True
prefix, suffix = common_edges(active_text, saved_text)
active_words = {value.casefold(): value for value in words(active_text)}
saved_words = {value.casefold(): value for value in words(saved_text)}
added_keys = [key for key in saved_words if key not in active_words]
removed_keys = [key for key in active_words if key not in saved_words]
result["text_diff"] = {
"active_chars": len(active_text),
"saved_chars": len(saved_text),
"delta_chars": len(saved_text) - len(active_text),
"common_prefix_chars": prefix,
"common_suffix_chars": suffix,
"added_words": [saved_words[key] for key in added_keys[:80]],
"removed_words": [active_words[key] for key in removed_keys[:80]],
"active_window": text_window(active_text, prefix),
"saved_window": text_window(saved_text, prefix),
"unified_diff": line_diff(active_text, saved_text, limit=120),
}
result["semantic_hints"] = {
"added_terms": semantic_words(result["text_diff"]["added_words"]),
"removed_terms": semantic_words(result["text_diff"]["removed_words"]),
}
result["summary"] = "Text payload differs." if active_text != saved_text else "Text payload matches."
return result
def storage_root(saved_table: str, active_table: str, roots: dict[str, Path]) -> tuple[Path | None, Path | None]:
saved_root = roots.get(saved_table)
active_root = roots.get(active_table)
return saved_root, active_root
def build_extension_cas_map(summary_path: Path | None) -> dict[tuple[str, str], str]:
if not summary_path or not summary_path.exists():
return {}
data = load_json(summary_path)
result: dict[tuple[str, str], str] = {}
for extension in data.get("extensions") or []:
extension_name = str(extension.get("extension_name") or "")
for obj in extension.get("sample_objects") or []:
for part in obj.get("parts") or []:
object_id = str(part.get("object_id") or "").casefold()
cas_key = str(part.get("cas_key") or "")
if extension_name and object_id and cas_key:
result[(extension_name.casefold(), object_id)] = cas_key
return result
def extension_object_id_from_saved_file(file_name: str) -> str | None:
if "__" not in file_name:
return None
object_id = file_name.split("__", 1)[1]
if object_id == "configinfo":
return None
return object_id.casefold()
def active_extension_path(item: dict[str, Any], file_name: str, extension_cas_map: dict[tuple[str, str], str], config_cas_all_dir: Path | None) -> tuple[Path | None, str | None]:
if not config_cas_all_dir:
return None, None
extension = str(item.get("extension") or "").casefold()
object_id = extension_object_id_from_saved_file(file_name)
if not extension or not object_id:
return None, None
cas_key = extension_cas_map.get((extension, object_id))
if not cas_key:
return None, None
path = config_cas_all_dir / cas_key
return (path if path.exists() else None), cas_key
def analyze(comparison: dict[str, Any], roots: dict[str, Path], *, extension_manifest_summary: Path | None = None, config_cas_all_dir: Path | None = None) -> dict[str, Any]:
extension_cas_map = build_extension_cas_map(extension_manifest_summary)
objects = []
for item in comparison.get("object_changes") or []:
details = []
for storage in item.get("storage") or []:
saved_root, active_root = storage_root(str(storage.get("saved_table")), str(storage.get("active_table")), roots)
file_name = str(storage.get("file_name") or "")
if not saved_root:
details.append({"file_name": file_name, "error": f"Missing saved root for {storage.get('saved_table')}"})
continue
try:
saved_path = safe_join(saved_root, file_name)
active_path = safe_join(active_root, file_name) if active_root else None
except ValueError:
details.append({"file_name": file_name, "error": "Unsafe storage file name."})
continue
if not saved_path.exists():
details.append({"file_name": file_name, "error": f"Saved payload file is missing: {saved_path}"})
continue
active_cas_key = None
if not (active_path and active_path.exists()) and storage.get("saved_table") == "ConfigCASSave":
active_path, active_cas_key = active_extension_path(item, file_name, extension_cas_map, config_cas_all_dir)
payload_detail = analyze_payload(active_path if active_path and active_path.exists() else None, saved_path)
details.append({
"file_name": file_name,
"payload_role": classify_payload_part(item, file_name),
"saved_table": storage.get("saved_table"),
"active_table": storage.get("active_table"),
"active_exists": storage.get("active_exists"),
"active_cas_key": active_cas_key,
"payload": payload_detail,
})
objects.append({
"full_name": item.get("full_name"),
"layer": item.get("layer"),
"extension": item.get("extension"),
"kind": item.get("kind"),
"kind_ru": item.get("kind_ru"),
"name": item.get("name"),
"synonym": item.get("synonym"),
"change_state": item.get("change_state"),
"details": details,
})
return {
"schema": "onec_saved_state_object_detail.v1",
"source_schema": comparison.get("schema"),
"database": comparison.get("database"),
"view": comparison.get("view"),
"object_details": objects,
"counts": {
"objects": len(objects),
"details": sum(len(item.get("details") or []) for item in objects),
},
"safety": {
"read_only": True,
"sql_write_performed": False,
"public_terms_are_1c_objects": True,
},
"active_extension_resolution": {
"extension_manifest_summary": str(extension_manifest_summary) if extension_manifest_summary else None,
"config_cas_all_dir": str(config_cas_all_dir) if config_cas_all_dir else None,
"mapped_parts": len(extension_cas_map),
},
}
def main() -> int:
parser = argparse.ArgumentParser(description="Analyze saved-state object changes beyond storage bytes.")
parser.add_argument("--comparison", type=Path, required=True)
parser.add_argument("--config-save-dir", type=Path, required=True)
parser.add_argument("--config-dir", type=Path, required=True)
parser.add_argument("--config-cas-save-dir", type=Path, required=True)
parser.add_argument("--config-cas-dir", type=Path, required=True)
parser.add_argument("--extension-manifest-summary", type=Path)
parser.add_argument("--config-cas-all-dir", type=Path)
parser.add_argument("--output", type=Path)
args = parser.parse_args()
roots = {
"ConfigSave": args.config_save_dir,
"Config": args.config_dir,
"ConfigCASSave": args.config_cas_save_dir,
"ConfigCAS": args.config_cas_dir,
}
result = analyze(
load_json(args.comparison),
roots,
extension_manifest_summary=args.extension_manifest_summary,
config_cas_all_dir=args.config_cas_all_dir,
)
if args.output:
write_json(args.output, result)
print(json.dumps({"output": str(args.output) if args.output else None, "schema": result["schema"], "counts": result["counts"]}, ensure_ascii=False))
return 0
if __name__ == "__main__":
raise SystemExit(main())
@@ -0,0 +1,189 @@
from __future__ import annotations
import argparse
import json
from pathlib import Path
from typing import Any
def read_json(path: Path) -> dict[str, Any]:
return json.loads(path.read_text(encoding="utf-8"))
def dig(mapping: dict[str, Any] | None, *keys: str) -> Any:
current: Any = mapping or {}
for key in keys:
if not isinstance(current, dict):
return None
current = current.get(key)
return current
def short_next_record(match: dict[str, Any] | None) -> dict[str, Any] | None:
node = dig(match, "next_moxel_record")
if not isinstance(node, dict):
return None
result: dict[str, Any] = {"type": node.get("type")}
for key in ("head", "value", "scalar_prefix", "tree_position"):
if key in node:
result[key] = node.get(key)
return result
def compact_item(item: dict[str, Any]) -> dict[str, Any]:
named = ((item.get("named_range_matches") or [{}])[0]) if item.get("named_range_matches") else {}
text = ((item.get("text_matches") or [{}])[0]) if item.get("text_matches") else {}
return {
"file_name": item.get("file_name"),
"bytes": item.get("bytes"),
"modified": item.get("modified"),
"counts": item.get("counts") or {},
"named_range": {
"name": named.get("name"),
"tree_position": named.get("tree_position"),
"one_based": dig(named, "range", "one_based"),
"raw_scalars": dig(named, "range_candidate", "raw_scalars"),
},
"text_match": {
"text": text.get("text"),
"cell_id": text.get("cell_id"),
"tree_position": text.get("tree_position"),
"next_moxel_record": short_next_record(text),
"immediate_preceding_values": dig(text, "style_evidence", "immediate_preceding_values"),
"last_7_preceding_values": dig(text, "style_evidence", "last_7_preceding_values"),
},
}
def diff_dict(before: dict[str, Any], after: dict[str, Any]) -> dict[str, dict[str, Any]]:
changed: dict[str, dict[str, Any]] = {}
for key in sorted(set(before) | set(after)):
if before.get(key) != after.get(key):
changed[key] = {"before": before.get(key), "after": after.get(key)}
return changed
def build_transitions(items: list[dict[str, Any]]) -> list[dict[str, Any]]:
compact = [compact_item(item) for item in reversed(items)]
transitions: list[dict[str, Any]] = []
for before, after in zip(compact, compact[1:]):
named_before = before.get("named_range") if isinstance(before.get("named_range"), dict) else {}
named_after = after.get("named_range") if isinstance(after.get("named_range"), dict) else {}
text_before = before.get("text_match") if isinstance(before.get("text_match"), dict) else {}
text_after = after.get("text_match") if isinstance(after.get("text_match"), dict) else {}
transition = {
"from_file": before.get("file_name"),
"to_file": after.get("file_name"),
"from_modified": before.get("modified"),
"to_modified": after.get("modified"),
"from_bytes": before.get("bytes"),
"to_bytes": after.get("bytes"),
"count_changes": diff_dict(before.get("counts") or {}, after.get("counts") or {}),
"named_range_changes": diff_dict(named_before, named_after),
"text_match_changes": diff_dict(text_before, text_after),
}
transitions.append(transition)
return transitions
def build_signature_groups(items: list[dict[str, Any]]) -> dict[str, Any]:
signatures: dict[str, list[dict[str, Any]]] = {}
for item in items:
for match in item.get("text_matches") or []:
if not isinstance(match, dict):
continue
signature = json.dumps(short_next_record(match), ensure_ascii=False, sort_keys=True)
signatures.setdefault(signature, []).append(
{
"file_name": item.get("file_name"),
"modified": item.get("modified"),
"text": match.get("text"),
"cell_id": match.get("cell_id"),
"tree_position": match.get("tree_position"),
"immediate_preceding_values": dig(match, "style_evidence", "immediate_preceding_values"),
"last_7_preceding_values": dig(match, "style_evidence", "last_7_preceding_values"),
}
)
result: list[dict[str, Any]] = []
for signature, occurrences in signatures.items():
result.append(
{
"signature": json.loads(signature),
"occurrences": occurrences,
"count": len(occurrences),
}
)
result.sort(key=lambda item: (-int(item.get("count") or 0), json.dumps(item.get("signature"), ensure_ascii=False)))
return {"next_moxel_record_signatures": result}
def build_markdown(history: dict[str, Any], transitions: list[dict[str, Any]], signature_groups: dict[str, Any]) -> str:
lines: list[str] = []
lines.append("# 1C template history matrix")
lines.append("")
lines.append(f"- Base: `{history.get('base_id')}`")
lines.append(f"- Track name: `{history.get('track_name')}`")
lines.append(f"- Track text: `{history.get('track_text')}`")
lines.append(f"- Snapshots: `{len(history.get('items') or [])}`")
lines.append("")
lines.append("## Current state")
current = compact_item((history.get("items") or [{}])[0] if history.get("items") else {})
lines.append("")
lines.append("```json")
lines.append(json.dumps(current, ensure_ascii=False, indent=2))
lines.append("```")
lines.append("")
lines.append("## Transitions")
for transition in transitions:
lines.append("")
lines.append(
f"- `{transition['from_file']}` -> `{transition['to_file']}` "
f"({transition['from_modified']} -> {transition['to_modified']})"
)
lines.append("")
lines.append("```json")
lines.append(json.dumps(transition, ensure_ascii=False, indent=2))
lines.append("```")
lines.append("")
lines.append("## Signatures")
lines.append("")
lines.append("```json")
lines.append(json.dumps(signature_groups, ensure_ascii=False, indent=2))
lines.append("```")
lines.append("")
return "\n".join(lines)
def main() -> int:
parser = argparse.ArgumentParser(description="Build a transition matrix from tracked 1C template history.")
parser.add_argument("history_json", help="Path to JSON generated by track_1c_template_history.py")
parser.add_argument("--json-output", help="Optional JSON output path.")
parser.add_argument("--markdown-output", help="Optional Markdown output path.")
args = parser.parse_args()
history_path = Path(args.history_json)
history = read_json(history_path)
items = history.get("items") or []
transitions = build_transitions(items)
signature_groups = build_signature_groups(items)
payload = {
"schema": "codex_1c_template_history_matrix.v1",
"source": str(history_path),
"track_name": history.get("track_name"),
"track_text": history.get("track_text"),
"current": compact_item(items[0] if items else {}),
"transitions": transitions,
"signatures": signature_groups,
}
rendered = json.dumps(payload, ensure_ascii=False, indent=2)
if args.json_output:
Path(args.json_output).write_text(rendered, encoding="utf-8")
markdown = build_markdown(history, transitions, signature_groups)
if args.markdown_output:
Path(args.markdown_output).write_text(markdown, encoding="utf-8")
print(rendered)
return 0
if __name__ == "__main__":
raise SystemExit(main())
+367
View File
@@ -0,0 +1,367 @@
from __future__ import annotations
import argparse
import itertools
import json
import re
import xml.etree.ElementTree as ET
from pathlib import Path
from typing import Any
DEFAULT_XML_ROOT = Path(r"Z:\codex\1C\XML\UPO\Структура базы 1с\Конфигурация")
PLACEHOLDER_RE = re.compile(r"\[([A-Za-zА-Яа-яЁё_][A-Za-zА-Яа-яЁё0-9_.]*)\]")
STYLE_TAGS = {
"format",
"formatIndex",
"f",
"width",
"height",
"horizontalAlignment",
"verticalAlignment",
"border",
"font",
"textColor",
"backgroundColor",
}
def namespace_uri(value: str) -> str | None:
if value.startswith("{") and "}" in value:
return value[1:].split("}", 1)[0]
return None
def local_name(value: str) -> str:
return value.rsplit("}", 1)[-1] if "}" in value else value
def xml_kind(root: ET.Element) -> str:
name = local_name(root.tag)
namespace = namespace_uri(root.tag) or ""
if name == "document" and "data/spreadsheet" in namespace:
return "tabular_document"
if name == "DataCompositionSchema":
return "data_composition_schema"
return name
def text_value(node: ET.Element | None) -> str | None:
if node is None or node.text is None:
return None
value = node.text.strip()
return value or None
def as_int(value: Any) -> int | None:
try:
return int(str(value).strip())
except (TypeError, ValueError):
return None
def children(node: ET.Element, name: str | None = None) -> list[ET.Element]:
items = list(node)
if name is None:
return items
return [item for item in items if local_name(item.tag) == name]
def descendants(node: ET.Element, name: str | None = None) -> list[ET.Element]:
result = []
for item in node.iter():
if item is node:
continue
if name is None or local_name(item.tag) == name:
result.append(item)
return result
def first_child_text(node: ET.Element, name: str) -> str | None:
for item in children(node, name):
value = text_value(item)
if value is not None:
return value
return None
def direct_columns_size(root: ET.Element) -> int | None:
for columns in children(root, "columns"):
size = as_int(first_child_text(columns, "size"))
if size is not None:
return size
return None
def direct_height(root: ET.Element) -> int | None:
for name in ("height", "vgRows"):
value = as_int(first_child_text(root, name))
if value is not None:
return value
return None
def cell_texts(cell: ET.Element) -> list[str]:
values = []
for item in descendants(cell):
if local_name(item.tag) in {"content", "parameter"}:
value = text_value(item)
if value:
values.append(value)
return values
def row_cells(row: ET.Element) -> list[dict[str, Any]]:
result = []
current_column = 0
for wrapper in children(row, "c"):
explicit_index = as_int(first_child_text(wrapper, "i"))
if explicit_index is not None:
current_column = explicit_index
payload = next((item for item in children(wrapper, "c")), wrapper)
texts = cell_texts(payload)
parameter = text_value(next((item for item in descendants(payload, "parameter")), None))
format_index = as_int(first_child_text(payload, "f")) or as_int(first_child_text(payload, "formatIndex"))
if texts or parameter or format_index is not None:
result.append(
{
"column": current_column + 1,
"zero_based": {"column": current_column},
"formatIndex": format_index,
"texts": texts,
**({"parameter": parameter} if parameter else {}),
}
)
current_column += 1
return result
def merge_ranges(root: ET.Element, *, limit: int = 200) -> list[dict[str, Any]]:
result = []
def append_range(row: int | None, column: int | None, width: int | None, height: int | None) -> bool:
if row is None or column is None:
return False
width = width or 1
height = height or 1
result.append(
{
"row": row + 1,
"column": column + 1,
"width": width,
"height": height,
"range": {
"one_based": {
"top": row + 1,
"left": column + 1,
"bottom": row + height,
"right": column + width,
},
"zero_based": {
"top": row,
"left": column,
"bottom": row + height - 1,
"right": column + width - 1,
},
},
"source": "xml_template_merge",
}
)
return len(result) >= limit
for merge in descendants(root, "merge"):
scalar_values = [(local_name(item.tag), as_int(text_value(item))) for item in children(merge)]
index = 0
while index < len(scalar_values):
if scalar_values[index][0] != "r":
index += 1
continue
row = scalar_values[index][1]
column = None
width = None
height = None
cursor = index + 1
while cursor < len(scalar_values) and scalar_values[cursor][0] != "r":
name, value = scalar_values[cursor]
if name == "c":
column = value
elif name == "w":
width = value
elif name == "h":
height = value
cursor += 1
if append_range(row, column, width, height):
return result
index = cursor
for item in descendants(merge, "r"):
row = as_int(first_child_text(item, "r"))
column = as_int(first_child_text(item, "c"))
width = as_int(first_child_text(item, "w")) or 1
height = as_int(first_child_text(item, "h")) or 1
if append_range(row, column, width, height):
return result
return result
def profile_template(path: Path, root_dir: Path) -> dict[str, Any]:
xml_root = ET.parse(path).getroot()
rows = []
max_row = 0
max_column = 0
parameter_names: list[str] = []
text_values: list[str] = []
placeholder_names: list[str] = []
for rows_item in descendants(xml_root, "rowsItem"):
row_index = as_int(first_child_text(rows_item, "index"))
row = next((item for item in children(rows_item, "row")), None)
if row_index is None or row is None:
continue
cells = row_cells(row)
if cells:
max_row = max(max_row, row_index + 1)
for cell in cells:
max_column = max(max_column, int(cell.get("column") or 0))
for value in cell.get("texts") or []:
text_values.append(value)
for match in PLACEHOLDER_RE.finditer(value):
placeholder_names.append(match.group(1))
if cell.get("parameter"):
parameter_names.append(str(cell["parameter"]))
rows.append(
{
"index": row_index,
"row": row_index + 1,
"formatIndex": as_int(first_child_text(row, "formatIndex")),
"cells": cells[:50],
"cell_count": len(cells),
}
)
merges = merge_ranges(xml_root)
for item in merges:
one_based = (item.get("range") or {}).get("one_based") or {}
max_row = max(max_row, int(one_based.get("bottom") or 0))
max_column = max(max_column, int(one_based.get("right") or 0))
style_counts = {
name: sum(1 for item in descendants(xml_root, name) if text_value(item) is not None)
for name in sorted(STYLE_TAGS)
}
format_indexes = [
as_int(text_value(item))
for item in descendants(xml_root)
if local_name(item.tag) in {"formatIndex", "f"} and as_int(text_value(item)) is not None
]
capacity_rows = direct_height(xml_root)
capacity_columns = direct_columns_size(xml_root)
return {
"path": str(path),
"relative_path": str(path.relative_to(root_dir)) if path.is_relative_to(root_dir) else str(path),
"xml_kind": xml_kind(xml_root),
"xml_root": {"name": local_name(xml_root.tag), "namespace": namespace_uri(xml_root.tag)},
"capacity_dimensions": {"rows": capacity_rows, "columns": capacity_columns},
"used_dimensions": {"rows": max_row or None, "columns": max_column or None, "evidence": ["rowsItem", "cells"] + (["merge"] if merges else [])},
"counts": {
"rows": len(rows),
"cells": sum(int(row.get("cell_count") or 0) for row in rows),
"texts": len(text_values),
"parameters": len(set(parameter_names)),
"placeholders": len(set(placeholder_names)),
"merges": len(merges),
"format_indexes": len(format_indexes),
"distinct_format_indexes": len(set(format_indexes)),
},
"style_counts": style_counts,
"samples": {
"rows": rows[:20],
"texts": text_values[:50],
"parameters": sorted(set(parameter_names))[:50],
"placeholders": sorted(set(placeholder_names))[:50],
"merges": merges[:50],
},
}
def analyze(root: Path, *, limit: int | None = None) -> dict[str, Any]:
file_iter = root.rglob("Template.xml")
files = list(itertools.islice(file_iter, limit)) if limit is not None else list(file_iter)
templates = []
errors = []
for path in files:
try:
templates.append(profile_template(path, root))
except Exception as exc:
errors.append({"path": str(path), "error": str(exc)})
return {
"schema": "codex_1c_template_xml_profiles.v1",
"source": "xml_analysis_fixture_only",
"root": str(root),
"templates": templates,
"counts": {
"files": len(files),
"templates": len(templates),
"errors": len(errors),
"with_merges": sum(1 for item in templates if int((item.get("counts") or {}).get("merges") or 0) > 0),
"with_parameters": sum(1 for item in templates if int((item.get("counts") or {}).get("parameters") or 0) > 0),
"with_placeholders": sum(1 for item in templates if int((item.get("counts") or {}).get("placeholders") or 0) > 0),
"by_xml_kind": {
kind: sum(1 for item in templates if item.get("xml_kind") == kind)
for kind in sorted({str(item.get("xml_kind") or "unknown") for item in templates})
},
},
**({"errors": errors[:100]} if errors else {}),
}
def render_markdown(payload: dict[str, Any]) -> str:
def dimension_text(value: dict[str, Any]) -> str:
rows = value.get("rows") if value.get("rows") is not None else "-"
columns = value.get("columns") if value.get("columns") is not None else "-"
return f"{rows}x{columns}"
lines = ["# 1C Template XML Profiles", ""]
counts = payload.get("counts") or {}
lines.append(f"- Source: `{payload.get('source')}`")
lines.append(f"- Templates: `{counts.get('templates')}`")
lines.append(f"- With merges: `{counts.get('with_merges')}`")
lines.append(f"- With parameters: `{counts.get('with_parameters')}`")
lines.append(f"- Errors: `{counts.get('errors')}`")
lines.append("")
lines.append("| Template | XML kind | Capacity | Used | Cells | Texts | Params | Merges | Formats |")
lines.append("| --- | --- | --- | --- | ---: | ---: | ---: | ---: | ---: |")
for item in payload.get("templates") or []:
item_counts = item.get("counts") or {}
capacity = item.get("capacity_dimensions") or {}
used = item.get("used_dimensions") or {}
lines.append(
f"| `{item.get('relative_path')}` | "
f"`{item.get('xml_kind')}` | "
f"`{dimension_text(capacity)}` | "
f"`{dimension_text(used)}` | "
f"{item_counts.get('cells')} | {item_counts.get('texts')} | {item_counts.get('parameters')} | "
f"{item_counts.get('merges')} | {item_counts.get('distinct_format_indexes')} |"
)
lines.append("")
return "\n".join(lines)
def main() -> int:
parser = argparse.ArgumentParser(description="Profile 1C Template.xml spreadsheet exports as analysis fixtures for SQL MOXCEL decoding.")
parser.add_argument("--root", default=str(DEFAULT_XML_ROOT), help="XML export root. Use Конфигурация by default, not extensions.")
parser.add_argument("--limit", type=int, help="Optional max Template.xml files to scan.")
parser.add_argument("--output-json", default="reports/1c-template-baselines/xml-template-profiles.json")
parser.add_argument("--output-markdown", default="reports/1c-template-baselines/xml-template-profiles.md")
args = parser.parse_args()
root = Path(args.root).resolve()
payload = analyze(root, limit=args.limit)
json_path = Path(args.output_json)
md_path = Path(args.output_markdown)
json_path.parent.mkdir(parents=True, exist_ok=True)
md_path.parent.mkdir(parents=True, exist_ok=True)
json_path.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8")
md_path.write_text(render_markdown(payload), encoding="utf-8")
print(json.dumps({"status": "ok", "json": str(json_path), "markdown": str(md_path), "counts": payload["counts"]}, ensure_ascii=False, indent=2))
return 0
if __name__ == "__main__":
raise SystemExit(main())
+162
View File
@@ -0,0 +1,162 @@
#!/usr/bin/env python3
from __future__ import annotations
import argparse
import json
from collections import Counter, defaultdict
from pathlib import Path
from typing import Any
def load_entries(report: dict[str, Any]) -> list[dict[str, Any]]:
matrix = report.get("matrix") if isinstance(report.get("matrix"), dict) else {}
entries = matrix.get("entries") if isinstance(matrix.get("entries"), list) else []
return [entry for entry in entries if isinstance(entry, dict)]
def gap_row(entry: dict[str, Any]) -> dict[str, Any]:
requested = entry.get("requested_target") if isinstance(entry.get("requested_target"), dict) else {}
effective = entry.get("effective_target") if isinstance(entry.get("effective_target"), dict) else {}
source = entry.get("effective_source") if isinstance(entry.get("effective_source"), dict) else {}
prop = entry.get("property") if isinstance(entry.get("property"), dict) else {}
probe = entry.get("codec_probe") if isinstance(entry.get("codec_probe"), dict) else {}
probe_node = probe.get("node") if isinstance(probe.get("node"), dict) else {}
return {
"reason": entry.get("reason") or "unknown",
"requested_section": requested.get("section"),
"requested_name": requested.get("name"),
"requested_path": requested.get("path"),
"effective_section": effective.get("section"),
"effective_name": effective.get("name"),
"effective_path": effective.get("path"),
"source_kind": source.get("kind") or "local",
"property": prop.get("semantic_name") or prop.get("canonical_property") or prop.get("property"),
"canonical_property": prop.get("canonical_property") or prop.get("property"),
"presentation": prop.get("presentation"),
"semantic_name": prop.get("semantic_name"),
"semantic_group": prop.get("semantic_group"),
"semantic_source": prop.get("semantic_source"),
"parameter_index": prop.get("parameter_index"),
"value_type": prop.get("value_type"),
"old": prop.get("old"),
"read_path": prop.get("read_path"),
"write_path": prop.get("write_path"),
"verification": prop.get("verification"),
"codec_probe": probe or None,
"codec_probe_node_type": probe_node.get("type") or probe.get("error") if probe else None,
}
def classify_action(reason: str, prop: str | None, value_type: str | None) -> str:
if reason == "identity_or_binding_property":
return "manual_only_identity_or_binding"
if reason == "empty_local_string_requires_codec_probe":
return "add_empty_composite_string_codec_probe"
if reason == "composite_node_requires_semantic_rule":
return "learn_composite_node_semantics"
if reason == "value_type_not_smoke_safe":
if value_type in {"enum_atom", "bool_or_enum_atom", "color_or_enum_atom"}:
return "learn_allowed_enum_values"
if prop in {"group"}:
return "learn_reference_or_container_write_rule"
return "classify_scalar_semantics"
return "inspect"
def main() -> int:
parser = argparse.ArgumentParser(description="Analyze not-smoked entries from a 1C saved-state write matrix report.")
parser.add_argument("--matrix-report", type=Path, required=True, help="Report produced by scripts/smoke_1c_write_matrix.py.")
parser.add_argument("--output", type=Path, required=True, help="Output JSON gap report.")
parser.add_argument("--sample-limit", type=int, default=12, help="Samples per reason/action.")
args = parser.parse_args()
report = json.loads(args.matrix_report.read_text(encoding="utf-8"))
gaps = []
for entry in load_entries(report):
if entry.get("can_smoke"):
continue
row = gap_row(entry)
row["next_action"] = classify_action(str(row.get("reason") or ""), row.get("property"), row.get("value_type"))
gaps.append(row)
by_reason = Counter(row["reason"] for row in gaps)
by_action = Counter(row["next_action"] for row in gaps)
by_section = Counter(row["effective_section"] for row in gaps)
by_property = Counter(row["property"] for row in gaps)
by_probe_node_type = Counter(row.get("codec_probe_node_type") for row in gaps if row.get("codec_probe_node_type"))
samples_by_reason: dict[str, list[dict[str, Any]]] = defaultdict(list)
samples_by_action: dict[str, list[dict[str, Any]]] = defaultdict(list)
shape_summary: dict[str, dict[str, Any]] = defaultdict(lambda: {"count": 0, "properties": Counter(), "sections": Counter(), "samples": []})
for row in gaps:
reason = str(row["reason"])
action = str(row["next_action"])
probe = row.get("codec_probe") if isinstance(row.get("codec_probe"), dict) else {}
node = probe.get("node") if isinstance(probe.get("node"), dict) else {}
children = node.get("children") if isinstance(node.get("children"), list) else []
if node:
shape = str(node.get("type") or "unknown") + "|" + ",".join(str((child or {}).get("type")) for child in children[:12])
shape_row = shape_summary[shape]
shape_row["count"] = int(shape_row["count"]) + 1
shape_row["properties"][row.get("property")] += 1
shape_row["sections"][row.get("effective_section")] += 1
if len(shape_row["samples"]) < args.sample_limit:
shape_row["samples"].append(
{
"target": row.get("requested_name") or row.get("requested_path"),
"section": row.get("effective_section"),
"property": row.get("presentation") or row.get("property"),
"semantic_name": row.get("semantic_name"),
"semantic_group": row.get("semantic_group"),
"old": row.get("old"),
"write_path": row.get("write_path"),
}
)
sample = {
"target": row.get("requested_name") or row.get("requested_path"),
"section": row.get("effective_section"),
"property": row.get("presentation") or row.get("property"),
"semantic_name": row.get("semantic_name"),
"semantic_group": row.get("semantic_group"),
"value_type": row.get("value_type"),
"old": row.get("old"),
"write_path": row.get("write_path"),
}
if len(samples_by_reason[reason]) < args.sample_limit:
samples_by_reason[reason].append(sample)
if len(samples_by_action[action]) < args.sample_limit:
samples_by_action[action].append(sample)
result = {
"schema": "onec_form_write_matrix_gap_analysis.v1",
"status": "ok",
"source_report": str(args.matrix_report),
"counts": {
"gaps": len(gaps),
"by_reason": dict(sorted(by_reason.items())),
"by_next_action": dict(sorted(by_action.items())),
"by_effective_section": dict(sorted(by_section.items())),
"by_codec_probe_node_type": dict(sorted(by_probe_node_type.items())),
"top_properties": by_property.most_common(40),
},
"samples_by_reason": dict(samples_by_reason),
"samples_by_next_action": dict(samples_by_action),
"codec_probe_shapes": [
{
"shape": shape,
"count": row["count"],
"properties": row["properties"].most_common(20),
"sections": dict(row["sections"]),
"samples": row["samples"],
}
for shape, row in sorted(shape_summary.items(), key=lambda item: int(item[1]["count"]), reverse=True)
],
"gaps": gaps,
}
args.output.parent.mkdir(parents=True, exist_ok=True)
args.output.write_text(json.dumps(result, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
print(json.dumps({"schema": result["schema"], "status": "ok", "counts": result["counts"], "path": str(args.output)}, ensure_ascii=False, indent=2))
return 0
if __name__ == "__main__":
raise SystemExit(main())
@@ -0,0 +1,77 @@
#!/usr/bin/env python3
from __future__ import annotations
import argparse
import json
from pathlib import Path
from typing import Any
def load_entries(path: Path) -> list[dict[str, Any]]:
report = json.loads(path.read_text(encoding="utf-8"))
matrix = report.get("matrix") if isinstance(report.get("matrix"), dict) else report
entries = matrix.get("entries") if isinstance(matrix.get("entries"), list) else []
return [entry for entry in entries if isinstance(entry, dict)]
def target_map(entries: list[dict[str, Any]]) -> dict[tuple[str, str, str], dict[str, Any]]:
result: dict[tuple[str, str, str], dict[str, Any]] = {}
for entry in entries:
target = entry.get("requested_target") if isinstance(entry.get("requested_target"), dict) else {}
key = (str(target.get("section") or ""), str(target.get("name") or ""), str(target.get("id") or ""))
if key == ("", "", "") or key in result:
continue
result[key] = {field: target.get(field) for field in ("section", "name", "id", "path", "marker", "type_name", "title")}
return result
def main() -> int:
parser = argparse.ArgumentParser(description="Compare 1C write matrix reports for structural target moves.")
parser.add_argument("--before", type=Path, required=True, help="Before write matrix report.")
parser.add_argument("--after", type=Path, required=True, help="After write matrix report.")
parser.add_argument("--output", type=Path, required=True, help="Output structural diff JSON path.")
args = parser.parse_args()
before_targets = target_map(load_entries(args.before))
after_targets = target_map(load_entries(args.after))
moves = []
for key, after in after_targets.items():
before = before_targets.get(key)
if not before:
continue
if str(before.get("path") or "") == str(after.get("path") or ""):
continue
moves.append(
{
"target": {
"section": after.get("section"),
"name": after.get("name"),
"id": after.get("id"),
"marker": after.get("marker"),
"type_name": after.get("type_name"),
},
"old_path": before.get("path"),
"new_path": after.get("path"),
"presentation": f"{after.get('name') or after.get('path')}: {before.get('path')} -> {after.get('path')}",
}
)
result = {
"schema": "onec_form_write_matrix_structural_diff.v1",
"status": "changed" if moves else "no_changes",
"before": str(args.before),
"after": str(args.after),
"target_moves": moves,
"counts": {
"target_moves": len(moves),
"before_targets": len(before_targets),
"after_targets": len(after_targets),
},
}
args.output.parent.mkdir(parents=True, exist_ok=True)
args.output.write_text(json.dumps(result, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
print(json.dumps({"schema": result["schema"], "status": result["status"], "counts": result["counts"], "path": str(args.output)}, ensure_ascii=False, indent=2))
return 0
if __name__ == "__main__":
raise SystemExit(main())
+257
View File
@@ -0,0 +1,257 @@
#!/usr/bin/env python3
from __future__ import annotations
import argparse
import json
import sys
import xml.etree.ElementTree as ET
from collections import Counter, defaultdict
from concurrent.futures import ThreadPoolExecutor
from datetime import datetime, timezone
from pathlib import Path
from typing import Any, Iterable
DEFAULT_ROOT = Path(r"Z:\codex\1C\XML\UPO\Структура базы 1с")
def local_name(tag: str) -> str:
return tag.rsplit("}", 1)[-1]
def direct_child(element: ET.Element, name: str) -> ET.Element | None:
return next((child for child in element if local_name(child.tag) == name), None)
def property_text(properties: ET.Element | None, name: str) -> str:
if properties is None:
return ""
node = next((child for child in properties if local_name(child.tag) == name), None)
return str(node.text or "").strip() if node is not None else ""
def parse_metadata_file(path: Path, layer: str) -> dict[str, Any]:
try:
root = ET.parse(path).getroot()
metadata = next(iter(root), None) if local_name(root.tag) == "MetaDataObject" else root
if metadata is None:
raise ValueError("metadata object element is missing")
kind = local_name(metadata.tag)
properties = direct_child(metadata, "Properties")
children = direct_child(metadata, "ChildObjects")
child_schemas: dict[str, dict[str, Any]] = {}
if children is not None:
grouped: dict[str, list[ET.Element]] = defaultdict(list)
for child in children:
grouped[local_name(child.tag)].append(child)
for child_kind, values in grouped.items():
child_properties: set[str] = set()
for value in values:
value_properties = direct_child(value, "Properties")
if value_properties is not None:
child_properties.update(local_name(item.tag) for item in value_properties)
child_schemas[child_kind] = {"count": len(values), "properties": sorted(child_properties)}
return {
"status": "ok",
"layer": layer,
"path": str(path),
"kind": kind,
"name": property_text(properties, "Name"),
"uuid": str(metadata.attrib.get("uuid") or "").lower(),
"properties": sorted(local_name(child.tag) for child in properties) if properties is not None else [],
"child_schemas": child_schemas,
}
except Exception as exc:
return {"status": "error", "layer": layer, "path": str(path), "message": str(exc)}
def layer_files(root: Path) -> list[Path]:
files = [root / "Configuration.xml"] if (root / "Configuration.xml").is_file() else []
for folder in root.iterdir():
if folder.is_dir() and folder.name != "Ext":
files.extend(sorted(folder.glob("*.xml")))
return files
def artifact_files(root: Path, max_depth: int = 5) -> list[Path]:
result: set[Path] = set()
for folder in (path for path in root.iterdir() if path.is_dir()):
for depth in range(1, max_depth + 1):
pattern = "/".join(["*"] * depth + ["Ext", "*.xml"])
result.update(path for path in folder.glob(pattern) if path.is_file())
return sorted(result)
def parse_artifact(path: Path) -> dict[str, Any]:
tags: set[str] = set()
attributes: dict[str, set[str]] = defaultdict(set)
root_tag = ""
try:
for _event, element in ET.iterparse(path, events=("start",)):
tag = local_name(element.tag)
if not root_tag:
root_tag = tag
tags.add(tag)
attributes[tag].update(local_name(name) for name in element.attrib)
return {
"status": "ok",
"path": str(path),
"artifact": path.name,
"root_tag": root_tag,
"tags": sorted(tags),
"attributes": {key: sorted(value) for key, value in sorted(attributes.items()) if value},
}
except Exception as exc:
return {"status": "error", "path": str(path), "artifact": path.name, "message": str(exc)}
def scan_artifacts(root: Path, workers: int) -> tuple[dict[str, Any], list[dict[str, Any]]]:
files = artifact_files(root)
with ThreadPoolExecutor(max_workers=max(1, workers)) as executor:
parsed = list(executor.map(parse_artifact, files))
errors = [item for item in parsed if item["status"] != "ok"]
grouped: dict[tuple[str, str], list[dict[str, Any]]] = defaultdict(list)
for item in parsed:
if item["status"] == "ok":
grouped[(str(item["artifact"]), str(item["root_tag"]))].append(item)
schemas: dict[str, Any] = {}
for (artifact, root_tag), values in sorted(grouped.items()):
tags: set[str] = set()
attributes: dict[str, set[str]] = defaultdict(set)
for value in values:
tags.update(value["tags"])
for tag, names in value["attributes"].items():
attributes[tag].update(names)
key = f"{artifact}:{root_tag}"
schemas[key] = {
"files": len(values),
"root_tag": root_tag,
"tags": sorted(tags),
"attributes": {tag: sorted(names) for tag, names in sorted(attributes.items())},
"samples": [value["path"] for value in values[:3]],
}
return {"files": len(files), "schemas": schemas}, errors
def scan_layer(root: Path, layer: str, workers: int) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]:
files = layer_files(root)
with ThreadPoolExecutor(max_workers=max(1, workers)) as executor:
parsed = list(executor.map(lambda path: parse_metadata_file(path, layer), files))
return [item for item in parsed if item["status"] == "ok"], [item for item in parsed if item["status"] != "ok"]
def merge_kind_schemas(objects: Iterable[dict[str, Any]]) -> dict[str, Any]:
kinds: dict[str, dict[str, Any]] = {}
grouped: dict[str, list[dict[str, Any]]] = defaultdict(list)
for item in objects:
grouped[str(item["kind"])].append(item)
for kind, values in sorted(grouped.items()):
properties: set[str] = set()
child_counts: Counter[str] = Counter()
child_properties: dict[str, set[str]] = defaultdict(set)
for value in values:
properties.update(value.get("properties") or [])
for child_kind, schema in (value.get("child_schemas") or {}).items():
child_counts[child_kind] += int(schema.get("count") or 0)
child_properties[child_kind].update(schema.get("properties") or [])
kinds[kind] = {
"objects": len(values),
"properties": sorted(properties),
"children": {
child_kind: {"objects": child_counts[child_kind], "properties": sorted(child_properties[child_kind])}
for child_kind in sorted(child_counts)
},
"samples": [
{"ref": f"{kind}.{value['name']}" if value.get("name") else kind, "uuid": value.get("uuid")}
for value in values[:3]
],
}
return kinds
def object_ref(item: dict[str, Any]) -> str:
return f"{item.get('kind')}.{item.get('name')}" if item.get("name") else f"{item.get('kind')}#{item.get('uuid')}"
def extension_summary(name: str, objects: list[dict[str, Any]], base_refs: set[str]) -> dict[str, Any]:
refs = {object_ref(item) for item in objects}
return {
"name": name,
"objects": len(objects),
"kinds": merge_kind_schemas(objects),
"overrides": sorted(refs & base_refs),
"extension_only": sorted(refs - base_refs),
"counts": {"overrides": len(refs & base_refs), "extension_only": len(refs - base_refs)},
}
def build_report(root: Path, workers: int, include_artifacts: bool) -> dict[str, Any]:
configuration_root = root / "Конфигурация"
extensions_root = root / "Расширения"
base_objects, errors = scan_layer(configuration_root, "configuration", workers)
base_refs = {object_ref(item) for item in base_objects}
extensions: list[dict[str, Any]] = []
configuration_artifacts: dict[str, Any] = {"status": "not_requested", "files": 0, "schemas": {}}
if include_artifacts:
configuration_artifacts, artifact_errors = scan_artifacts(configuration_root, workers)
configuration_artifacts["status"] = "ok" if not artifact_errors else "partial"
errors.extend(artifact_errors)
if extensions_root.is_dir():
for extension_root in sorted(path for path in extensions_root.iterdir() if path.is_dir()):
objects, extension_errors = scan_layer(extension_root, f"extension:{extension_root.name}", workers)
errors.extend(extension_errors)
summary = extension_summary(extension_root.name, objects, base_refs)
if include_artifacts:
artifacts, artifact_errors = scan_artifacts(extension_root, workers)
artifacts["status"] = "ok" if not artifact_errors else "partial"
summary["artifacts"] = artifacts
errors.extend(artifact_errors)
extensions.append(summary)
kinds = merge_kind_schemas(base_objects)
return {
"schema": "onec_xml_metadata_analysis.v1",
"status": "ok" if not errors else "partial",
"generated_at": datetime.now(timezone.utc).isoformat(),
"source_root": str(root),
"assumption": "The exported base configuration is equivalent to the SQL base; extensions are independent overlays and may differ.",
"configuration": {
"objects": len(base_objects),
"kinds": kinds,
"counts": {"kinds": len(kinds), "objects": len(base_objects)},
"artifacts": configuration_artifacts,
},
"extensions": extensions,
"counts": {
"configuration_kinds": len(kinds),
"configuration_objects": len(base_objects),
"extensions": len(extensions),
"extension_objects": sum(item["objects"] for item in extensions),
"artifact_files": int(configuration_artifacts.get("files") or 0) + sum(int((item.get("artifacts") or {}).get("files") or 0) for item in extensions),
"parse_errors": len(errors),
},
"errors": errors[:100],
}
def main() -> int:
parser = argparse.ArgumentParser(description="Extract complete declared metadata property schemas from a 1C XML configuration export.")
parser.add_argument("--root", type=Path, default=DEFAULT_ROOT)
parser.add_argument("--workers", type=int, default=8)
parser.add_argument("--include-artifacts", action="store_true", help="Also scan nested Ext XML files such as forms, rights, and templates.")
parser.add_argument("--output", type=Path)
parser.add_argument("--json", action="store_true", help="Print the full JSON report instead of a compact summary.")
args = parser.parse_args()
report = build_report(args.root, args.workers, args.include_artifacts)
rendered = json.dumps(report, ensure_ascii=False, indent=2)
if args.output:
args.output.parent.mkdir(parents=True, exist_ok=True)
args.output.write_text(rendered + "\n", encoding="utf-8")
if args.json:
print(rendered)
else:
print(json.dumps({"status": report["status"], **report["counts"]}, ensure_ascii=False))
return 0 if report["status"] == "ok" else 2
if __name__ == "__main__":
sys.exit(main())
+121
View File
@@ -0,0 +1,121 @@
$ErrorActionPreference = "Stop"
function Invoke-RemotePowerShell {
param(
[Parameter(Mandatory = $true)][string]$SshTarget,
[Parameter(Mandatory = $true)][string]$Script
)
$encoded = [Convert]::ToBase64String([Text.Encoding]::Unicode.GetBytes($Script))
ssh $SshTarget "powershell -NoProfile -EncodedCommand $encoded"
}
function New-AppArchive {
$archive = Join-Path $env:TEMP "llm-model-chat-app.zip"
Remove-Item -Force $archive -ErrorAction SilentlyContinue
Get-ChildItem -Path "scripts", "tools", "plugins" -Recurse -Directory -Filter "__pycache__" -ErrorAction SilentlyContinue |
Sort-Object FullName -Descending |
ForEach-Object {
Remove-Item -LiteralPath $_.FullName -Recurse -Force -ErrorAction SilentlyContinue
}
$items = @(
"config",
"scripts",
"tools",
"registry",
"plugins",
"docs",
"evals",
"datasets",
"requirements.txt",
"requirements-training.txt",
"README.md"
)
Compress-Archive -Path $items -DestinationPath $archive -Force
Add-Type -AssemblyName System.IO.Compression.FileSystem
$zip = [System.IO.Compression.ZipFile]::Open($archive, [System.IO.Compression.ZipArchiveMode]::Update)
try {
@($zip.Entries | Where-Object { $_.FullName.Replace("\", "/") -match "/__pycache__/" }) |
ForEach-Object { $_.Delete() }
} finally {
$zip.Dispose()
}
$preflightReport = "reports/model-chat/preflight.json"
if (Test-Path -LiteralPath $preflightReport) {
$zip = [System.IO.Compression.ZipFile]::Open($archive, [System.IO.Compression.ZipArchiveMode]::Update)
try {
[System.IO.Compression.ZipFileExtensions]::CreateEntryFromFile(
$zip,
(Get-Item -LiteralPath $preflightReport).FullName,
"reports/model-chat/preflight.json"
) | Out-Null
} finally {
$zip.Dispose()
}
}
return $archive
}
function Test-AppArchive {
param([Parameter(Mandatory = $true)][string]$Archive)
Add-Type -AssemblyName System.IO.Compression.FileSystem
$requiredEntries = @(
"config/gpu_profiles.json",
"scripts/model_chat_server.py",
"scripts/transformers_plugin_server.py",
"scripts/common.py",
"tools/model-chat/index.html",
"registry/index.json",
"plugins/1c/rag/profiles.yaml",
"requirements.txt",
"requirements-training.txt",
"README.md"
)
if (Test-Path -LiteralPath "reports/model-chat/preflight.json") {
$requiredEntries += "reports/model-chat/preflight.json"
}
$zip = [System.IO.Compression.ZipFile]::OpenRead($Archive)
try {
$entryNames = [System.Collections.Generic.HashSet[string]]::new([StringComparer]::OrdinalIgnoreCase)
foreach ($entry in $zip.Entries) {
[void]$entryNames.Add($entry.FullName.Replace("\", "/"))
}
$missing = @($requiredEntries | Where-Object { -not $entryNames.Contains($_) })
if ($missing.Count -gt 0) {
throw "Archive is missing required entries: $($missing -join ', ')"
}
$pycacheEntries = @($entryNames | Where-Object { $_ -match "/__pycache__/" })
if ($pycacheEntries.Count -gt 0) {
throw "Archive contains __pycache__ entries: $($pycacheEntries[0])"
}
Write-Host "Archive validation passed: $Archive"
} finally {
$zip.Dispose()
}
}
function Sync-AppDirectory {
param(
[Parameter(Mandatory = $true)][string]$SshTarget,
[Parameter(Mandatory = $true)][string]$RemoteArchive,
[Parameter(Mandatory = $true)][string]$RemoteAppDir,
[string]$Archive
)
if (-not $Archive) {
$Archive = New-AppArchive
Test-AppArchive -Archive $Archive
}
ssh $SshTarget "cmd /c if not exist C:\ProgramData\LLM mkdir C:\ProgramData\LLM"
$remoteArchiveForScp = $RemoteArchive.Replace("\", "/")
scp $Archive "${SshTarget}:$remoteArchiveForScp"
$unpackScript = @"
New-Item -ItemType Directory -Force '$RemoteAppDir' | Out-Null
Remove-Item -Recurse -Force '$RemoteAppDir\*' -ErrorAction SilentlyContinue
Expand-Archive -Path '$RemoteArchive' -DestinationPath '$RemoteAppDir' -Force
New-Item -ItemType Directory -Force '$RemoteAppDir\reports','$RemoteAppDir\models\incoming' | Out-Null
"@
Invoke-RemotePowerShell -SshTarget $SshTarget -Script $unpackScript
}
+114
View File
@@ -0,0 +1,114 @@
from __future__ import annotations
import argparse
import sys
import urllib.error
from pathlib import Path
from common import call_chat_completion, read_json, search_lexical_index
from rag_profiles import resolve_rag_profile
ROOT = Path(__file__).resolve().parents[1]
DEFAULT_INDEX = ROOT / "plugins" / "1c" / "datasets" / "prepared" / "rag_index.json"
DEFAULT_SYSTEM_PROMPT = ROOT / "plugins" / "1c" / "prompts" / "system.md"
DEFAULT_RAG_PROMPT = ROOT / "plugins" / "1c" / "prompts" / "rag-answer.md"
def format_context(results: list[dict], *, max_chars: int = 12000) -> str:
if not results:
return "Контекст не найден."
blocks = []
used_chars = 0
for position, result in enumerate(results, start=1):
document = result["document"]
source = document.get("source_path") or "unknown"
chunk = document.get("chunk_index")
title = document.get("title") or "unknown"
content = (document.get("content") or "").strip()
header = f"[{position}] source={source} title={title} chunk={chunk} score={result['score']:.4f}"
remaining = max_chars - used_chars - len(header) - 2
if remaining <= 0:
break
if len(content) > remaining:
content = content[: max(0, remaining - 3)].rstrip() + "..."
block = "\n".join([header, content])
blocks.append(block)
used_chars += len(block) + 2
return "\n\n".join(blocks)
def render_prompt(template_path: Path, context: str, question: str) -> str:
template = template_path.read_text(encoding="utf-8")
return template.replace("{{context}}", context).replace("{{question}}", question)
def main() -> int:
parser = argparse.ArgumentParser(description="Ask the 1C RAG assistant.")
parser.add_argument("question")
parser.add_argument("--index", type=Path, default=DEFAULT_INDEX)
parser.add_argument("--profile", default="auto")
parser.add_argument("--limit", type=int)
parser.add_argument("--candidate-limit", type=int)
parser.add_argument("--dedupe-by-document", action="store_true")
parser.add_argument("--min-score", type=float)
parser.add_argument("--source-type", action="append", dest="source_types")
parser.add_argument("--platform-version")
parser.add_argument("--platform-doc-id")
parser.add_argument("--max-context-chars", type=int)
parser.add_argument("--base-url", help="OpenAI-compatible endpoint, for example http://docker-gpu.cin.su:8000")
parser.add_argument("--model", default="qwen3-4b-instruct")
parser.add_argument("--print-prompt", action="store_true", help="Print assembled prompt instead of calling a model.")
parser.add_argument("--system-prompt", type=Path, default=DEFAULT_SYSTEM_PROMPT)
parser.add_argument("--rag-prompt", type=Path, default=DEFAULT_RAG_PROMPT)
args = parser.parse_args()
index = read_json(args.index)
profile = resolve_rag_profile(args.profile, args.question)
source_types = args.source_types if args.source_types is not None else profile["source_types"]
results = search_lexical_index(
index,
args.question,
limit=int(args.limit or profile["limit"]),
candidate_limit=int(args.candidate_limit or profile["candidate_limit"]),
dedupe_by_document=args.dedupe_by_document or bool(profile["dedupe_by_document"]),
min_score=float(args.min_score if args.min_score is not None else profile["min_score"]),
source_types=source_types,
metadata_filters={
"platform_version": args.platform_version or "",
"platform_doc_id": args.platform_doc_id or "",
},
)
context = format_context(results, max_chars=int(args.max_context_chars or profile["max_context_chars"]))
rag_prompt = render_prompt(args.rag_prompt, context=context, question=args.question)
if args.print_prompt or not args.base_url:
print(rag_prompt)
if not args.base_url and not args.print_prompt:
print(
"\nNo --base-url provided, so only the prompt was rendered.",
file=sys.stderr,
)
return 0
system_prompt = args.system_prompt.read_text(encoding="utf-8")
try:
answer = call_chat_completion(
base_url=args.base_url,
model=args.model,
messages=[
{"role": "system", "content": system_prompt},
{"role": "user", "content": rag_prompt},
],
max_tokens=1200,
)
except (urllib.error.URLError, ValueError) as exc:
print(f"Chat request failed: {exc}", file=sys.stderr)
return 1
print(answer)
return 0
if __name__ == "__main__":
raise SystemExit(main())
+156
View File
@@ -0,0 +1,156 @@
from __future__ import annotations
import argparse
import json
import sys
from pathlib import Path
from typing import Any
ROOT = Path(__file__).resolve().parents[1]
if str(ROOT) not in sys.path:
sys.path.insert(0, str(ROOT))
import scripts.compare_1c_access_role_audit as compare_script
import scripts.export_1c_access_role_audit as export_script
def compare_stem(old_path: Path, new_path: Path) -> str:
return f"compare-{old_path.stem.replace('.summary', '')}-{new_path.stem.replace('.summary', '')}"
def verdict_for(export_summary: dict[str, Any], compare_result: dict[str, Any] | None) -> str:
counts = compare_result.get("counts") if isinstance(compare_result, dict) and isinstance(compare_result.get("counts"), dict) else {}
if any(int(counts.get(key) or 0) > 0 for key in ("added_users", "removed_users", "changed_access_paths")):
return "changed"
if str(export_summary.get("risk_level") or "").lower() in {"medium", "high"}:
return "risk"
return "ok"
def audit_role(
*,
adapter_url: str,
base_id: str,
role: str,
report_root: Path,
timeout: int,
user_threshold: int,
limit: int,
include_html: bool = True,
) -> dict[str, Any]:
export_summary = export_script.export_role_audit(
adapter_url=adapter_url,
base_id=base_id,
role=role,
report_root=report_root,
timeout=timeout,
user_threshold=user_threshold,
include_analysis=True,
limit=limit,
include_html=include_html,
)
latest = compare_script.find_latest_summaries(report_root, base_id, role=role, count=2)
compare_result: dict[str, Any] | None = None
if len(latest) >= 2:
new_path, old_path = latest[0], latest[1]
folder = report_root / compare_script.slugify(base_id, max_length=60)
stem = compare_stem(old_path, new_path)
compare_result = compare_script.compare_files(
old_path,
new_path,
output=folder / f"{stem}.json",
html_output=folder / f"{stem}.html",
)
export_script.update_index(report_root, base_id)
return {
"schema": "onec_access_role_audit_run.v1",
"status": export_summary.get("status"),
"base_id": base_id,
"role": role,
"verdict": verdict_for(export_summary, compare_result),
"export": export_summary,
"compare": compare_result,
}
def audit_config(
*,
config_path: Path,
adapter_url: str,
report_root: Path,
timeout: int,
limit: int,
include_html: bool = True,
) -> dict[str, Any]:
config = json.loads(config_path.read_text(encoding="utf-8"))
roles = config.get("roles") if isinstance(config.get("roles"), list) else []
default_base_id = str(config.get("base_id") or "upo_test")
results: list[dict[str, Any]] = []
for item in roles:
if not isinstance(item, dict) or not item.get("role"):
continue
results.append(
audit_role(
adapter_url=adapter_url,
base_id=str(item.get("base_id") or default_base_id),
role=str(item.get("role")),
report_root=report_root,
timeout=timeout,
user_threshold=int(item.get("user_threshold") or 50),
limit=limit,
include_html=include_html,
)
)
verdicts = [str(item.get("verdict") or "") for item in results]
overall = "changed" if "changed" in verdicts else "risk" if "risk" in verdicts else "ok"
return {
"schema": "onec_access_role_audit_batch.v1",
"status": "ok" if all(item.get("status") == "ok" for item in results) else "error",
"config": str(config_path),
"overall_verdict": overall,
"counts": {"roles": len(results), "changed": verdicts.count("changed"), "risk": verdicts.count("risk"), "ok": verdicts.count("ok")},
"results": results,
}
def main() -> int:
parser = argparse.ArgumentParser(description="Run 1C role access audit: export, analyze, update index, compare with previous.")
parser.add_argument("--adapter-url", default=export_script.DEFAULT_BASE_URL)
parser.add_argument("--base-id", default="upo_test")
parser.add_argument("--role")
parser.add_argument("--config", type=Path, help="Run all roles from an access critical roles JSON config.")
parser.add_argument("--report-root", type=Path, default=export_script.DEFAULT_REPORT_ROOT)
parser.add_argument("--timeout", type=int, default=120)
parser.add_argument("--limit", type=int, default=20000)
parser.add_argument("--user-threshold", type=int, default=50)
parser.add_argument("--no-html", action="store_true")
args = parser.parse_args()
if args.config:
result = audit_config(
config_path=args.config,
adapter_url=args.adapter_url,
report_root=args.report_root,
timeout=args.timeout,
limit=args.limit,
include_html=not args.no_html,
)
else:
if not args.role:
parser.error("--role is required unless --config is used.")
result = audit_role(
adapter_url=args.adapter_url,
base_id=args.base_id,
role=args.role,
report_root=args.report_root,
timeout=args.timeout,
user_threshold=args.user_threshold,
limit=args.limit,
include_html=not args.no_html,
)
print(json.dumps(result, ensure_ascii=False, indent=2))
return 0 if result.get("status") == "ok" else 1
if __name__ == "__main__":
raise SystemExit(main())
+562
View File
@@ -0,0 +1,562 @@
#!/usr/bin/env python3
from __future__ import annotations
import argparse
import concurrent.futures
import copy
import json
import os
import re
import sys
import time
import threading
import urllib.error
import urllib.request
from datetime import datetime, timezone
from pathlib import Path
from typing import Any, Callable
DEFAULT_BASE_URL = "http://docker-gpu.cin.su:8011"
DEFAULT_BASE_ID = "upo_test"
# Metadata kinds that either own application data or expose values through the
# public data facade. Kinds absent from a concrete base remain in the audit so
# that coverage cannot be declared only from a convenient test configuration.
DATA_KINDS = {
"AccountingRegister",
"AccumulationRegister",
"BusinessProcess",
"CalculationRegister",
"Catalog",
"ChartOfAccounts",
"ChartOfCalculationTypes",
"ChartOfCharacteristicTypes",
"Constant",
"Document",
"Enum",
"ExchangePlan",
"InformationRegister",
"Sequence",
"Task",
}
Rpc = Callable[[str, str, str, dict[str, Any], float], dict[str, Any]]
def utc_now() -> str:
return datetime.now(timezone.utc).isoformat()
def write_json_atomic(path: Path, value: dict[str, Any]) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
temporary = path.with_name(path.name + ".tmp")
temporary.write_text(json.dumps(value, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
temporary.replace(path)
def rpc(base_url: str, token: str, method: str, payload: dict[str, Any], timeout: float) -> dict[str, Any]:
request = urllib.request.Request(
base_url.rstrip("/") + "/rpc",
data=json.dumps({"method": method, "payload": payload}, ensure_ascii=False).encode("utf-8"),
headers={"Authorization": f"Bearer {token}", "Content-Type": "application/json; charset=utf-8"},
method="POST",
)
with urllib.request.urlopen(request, timeout=timeout) as response:
value = json.loads(response.read().decode("utf-8"))
return value if isinstance(value, dict) else {"status": "error", "error": "response_not_object"}
def safe_rpc(
rpc_call: Rpc,
base_url: str,
token: str,
method: str,
payload: dict[str, Any],
timeout: float,
) -> tuple[dict[str, Any], int]:
started = time.monotonic()
try:
result = rpc_call(base_url, token, method, payload, timeout)
except (TimeoutError, urllib.error.URLError, OSError, ValueError) as exc:
result = {
"status": "transport_error",
"error": type(exc).__name__,
"diagnostics": {"message": str(exc)[:500]},
}
return result, round((time.monotonic() - started) * 1000)
def first_object(response: dict[str, Any]) -> dict[str, Any] | None:
for key in ("objects", "items"):
values = response.get(key)
if isinstance(values, list) and values and isinstance(values[0], dict):
return values[0]
return None
def public_selector(sample: dict[str, Any]) -> dict[str, Any]:
return {
key: sample[key]
for key in ("ref", "kind", "name", "guid")
if sample.get(key) not in {None, ""}
}
def operation_summary(result: dict[str, Any], duration_ms: int, **values: Any) -> dict[str, Any]:
summary = {"status": result.get("status") or "unknown", "duration_ms": duration_ms, **values}
diagnostics = result.get("diagnostics")
if summary["status"] != "ok" and isinstance(diagnostics, dict) and diagnostics.get("message"):
summary["message"] = str(diagnostics["message"])[:500]
if result.get("error"):
summary["error"] = str(result["error"])[:200]
return summary
def record_ref_from_row(row: Any) -> str | None:
if not isinstance(row, dict):
return None
value = row.get("ref")
if isinstance(value, dict):
value = value.get("hex")
compact = str(value or "").replace("-", "").strip()
return compact if re.fullmatch(r"[0-9a-fA-F]{32}", compact) else None
def audit_data_kind(
base_url: str,
base_id: str,
token: str,
kind: str,
timeout: float,
include_reads: bool,
rpc_call: Rpc = rpc,
existing: dict[str, Any] | None = None,
progress: Callable[[dict[str, Any]], None] | None = None,
) -> dict[str, Any]:
common = {"base_id": base_id, "timeout_seconds": max(1, int(timeout))}
result = copy.deepcopy(existing) if isinstance(existing, dict) else {}
result.update({"kind": kind, "status": "degraded"})
result.setdefault("operations", {})
def save_progress() -> None:
if progress is not None:
progress(copy.deepcopy(result))
prior_list = result["operations"].get("metadata.objects.list") or {}
if prior_list.get("status") == "ok" and isinstance(result.get("sample"), dict):
listed = {"status": "ok"}
sample = {**(result["sample"].get("selector") or {}), "name": result["sample"].get("name")}
else:
listed, list_ms = safe_rpc(
rpc_call,
base_url,
token,
"metadata.objects.list",
{**common, "kind": kind, "limit": 1},
timeout,
)
sample = first_object(listed)
result["operations"]["metadata.objects.list"] = operation_summary(
listed,
list_ms,
objects=len(listed.get("objects") or listed.get("items") or []),
)
if sample:
result["sample"] = {"selector": public_selector(sample), "name": sample.get("name")}
save_progress()
if not sample:
result["status"] = "absent" if listed.get("status") == "ok" else "degraded"
result["reason"] = "no_sample_object"
save_progress()
return result
selector = public_selector(sample)
result["sample"] = {"selector": selector, "name": sample.get("name")}
prior_schema = result["operations"].get("data.schema") or {}
if prior_schema.get("status") == "ok":
schema_status = "ok"
else:
schema, schema_ms = safe_rpc(
rpc_call,
base_url,
token,
"data.schema",
{**common, **selector},
timeout,
)
table = schema.get("table") if isinstance(schema.get("table"), dict) else {}
result["operations"]["data.schema"] = operation_summary(
schema,
schema_ms,
fields=len(schema.get("fields") or []),
table=table.get("name"),
cache=(schema.get("cache") or {}).get("status") if isinstance(schema.get("cache"), dict) else None,
)
schema_status = str(schema.get("status") or "unknown")
save_progress()
if schema_status != "ok" or not include_reads:
result["status"] = "ok" if schema_status == "ok" else "degraded"
save_progress()
return result
prior_data_list = result["operations"].get("data.list") or {}
must_repeat_list = prior_data_list.get("status") != "ok" or (
"sample_record_ref" not in result and "sample_record_ref_status" not in result
)
if must_repeat_list:
data_list, data_list_ms = safe_rpc(
rpc_call,
base_url,
token,
"data.list",
{**common, **selector, "limit": 1},
timeout,
)
rows = data_list.get("rows") if isinstance(data_list.get("rows"), list) else []
list_summary = operation_summary(data_list, data_list_ms, rows=len(rows))
if data_list.get("status") == "ok":
result["operations"]["data.list"] = list_summary
result["operations"].pop("data.list_retry", None)
ref = record_ref_from_row(rows[0]) if rows else None
if ref:
result["sample_record_ref"] = ref
result.pop("sample_record_ref_status", None)
else:
result.pop("sample_record_ref", None)
result["sample_record_ref_status"] = "empty_object" if not rows else "object_has_no_reference_key"
elif prior_data_list.get("status") == "ok":
# A successful operation is evidence. Do not downgrade it only
# because a later attempt to recover the sample ref timed out.
result["operations"]["data.list_retry"] = list_summary
ref = result.get("sample_record_ref")
else:
result["operations"]["data.list"] = list_summary
result.pop("sample_record_ref", None)
result["sample_record_ref_status"] = "data_list_failed"
ref = None
save_progress()
else:
ref = result.get("sample_record_ref")
if (result["operations"].get("data.count") or {}).get("status") != "ok":
counted, count_ms = safe_rpc(
rpc_call,
base_url,
token,
"data.count",
{**common, **selector},
timeout,
)
result["operations"]["data.count"] = operation_summary(counted, count_ms, count=counted.get("count"))
save_progress()
prior_get_status = (result["operations"].get("data.get") or {}).get("status")
if prior_get_status not in {"ok", "not_applicable"}:
if ref:
fetched, get_ms = safe_rpc(
rpc_call,
base_url,
token,
"data.get",
{**common, **selector, "record_ref": ref},
timeout,
)
fetched_rows = fetched.get("rows") if isinstance(fetched.get("rows"), list) else []
result["operations"]["data.get"] = operation_summary(fetched, get_ms, rows=len(fetched_rows))
elif (result["operations"].get("data.list") or {}).get("status") == "ok":
reason = str(result.get("sample_record_ref_status") or "object_has_no_reference_key")
result["operations"]["data.get"] = {"status": "not_applicable", "reason": reason, "duration_ms": 0}
else:
result["operations"]["data.get"] = {"status": "blocked", "reason": "data_list_failed", "duration_ms": 0}
save_progress()
required = ("data.schema", "data.list", "data.count")
failures = [name for name in required if result["operations"].get(name, {}).get("status") != "ok"]
get_status = result["operations"]["data.get"]["status"]
if get_status not in {"ok", "not_applicable"}:
failures.append("data.get")
result["status"] = "ok" if not failures else "degraded"
if failures:
result["failed_operations"] = failures
else:
result.pop("failed_operations", None)
save_progress()
return result
def load_checkpoint(
path: Path | None,
base_url: str,
base_id: str,
resume: bool,
include_reads: bool,
) -> dict[str, Any]:
fresh = {
"schema": "onec_adapter_data_audit_checkpoint.v1",
"base_url": base_url,
"base_id": base_id,
"include_reads": include_reads,
"started_at": utc_now(),
"updated_at": utc_now(),
"checks": {},
}
if not resume or path is None or not path.exists():
return fresh
value = json.loads(path.read_text(encoding="utf-8-sig"))
if value.get("schema") != fresh["schema"]:
raise ValueError(f"unsupported checkpoint schema in {path}")
if value.get("base_url") != base_url or value.get("base_id") != base_id:
raise ValueError(f"checkpoint {path} belongs to another adapter or base")
if bool(value.get("include_reads")) != include_reads:
raise ValueError(f"checkpoint {path} was created for another data audit mode")
if not isinstance(value.get("checks"), dict):
raise ValueError(f"checkpoint {path} has no checks object")
return value
def run_data_checks(
base_url: str,
base_id: str,
token: str,
kinds: list[str],
timeout: float,
include_reads: bool,
workers: int,
checkpoint_path: Path | None,
resume: bool,
retry_degraded: bool = False,
rpc_call: Rpc = rpc,
) -> tuple[dict[str, dict[str, Any]], int]:
checkpoint = load_checkpoint(checkpoint_path, base_url, base_id, resume, include_reads)
checks = checkpoint["checks"]
checkpoint_lock = threading.Lock()
reusable = {
kind
for kind in kinds
if kind in checks and (not retry_degraded or checks[kind].get("status") == "ok")
}
resumed = len(reusable)
pending = [kind for kind in kinds if kind not in reusable]
def execute(kind: str) -> dict[str, Any]:
def save_partial(value: dict[str, Any]) -> None:
with checkpoint_lock:
checks[kind] = value
checkpoint["updated_at"] = utc_now()
if checkpoint_path is not None:
write_json_atomic(checkpoint_path, checkpoint)
return audit_data_kind(
base_url,
base_id,
token,
kind,
timeout,
include_reads,
rpc_call,
existing=checks.get(kind),
progress=save_partial,
)
with concurrent.futures.ThreadPoolExecutor(max_workers=max(1, workers)) as executor:
futures = {executor.submit(execute, kind): kind for kind in pending}
for future in concurrent.futures.as_completed(futures):
kind = futures[future]
try:
checks[kind] = future.result()
except Exception as exc: # a single kind must not discard completed evidence
checks[kind] = {
"kind": kind,
"status": "degraded",
"error": type(exc).__name__,
"message": str(exc)[:500],
}
with checkpoint_lock:
checkpoint["updated_at"] = utc_now()
if checkpoint_path is not None:
write_json_atomic(checkpoint_path, checkpoint)
return {kind: checks[kind] for kind in kinds if kind in checks}, resumed
def build_report(
base_url: str,
base_id: str,
token: str,
timeout: float,
sample_objects: bool,
sample_schemas: bool,
*,
sample_reads: bool = False,
workers: int = 1,
checkpoint_path: Path | None = None,
resume: bool = False,
retry_degraded: bool = False,
rpc_call: Rpc = rpc,
) -> dict[str, Any]:
audit, _ = safe_rpc(
rpc_call,
base_url,
token,
"metadata.adapter.audit",
{"base_id": base_id, "include_missing": True, "include_unmapped": True, "timeout_seconds": int(timeout)},
timeout,
)
if audit.get("status") != "ok":
return {"schema": "onec_adapter_coverage_audit.v1", "status": "error", "audit": audit}
matrix: list[dict[str, Any]] = []
for support in audit.get("metadata_kinds") or []:
if not isinstance(support, dict):
continue
kind = str(support.get("kind") or "")
count = int(support.get("count") or 0)
matrix.append({
"kind": kind,
"kind_ru": support.get("kind_ru"),
"objects": count,
"capabilities": support.get("capabilities") or [],
"discovery": "present" if count else "absent_in_base",
})
data_checks: dict[str, dict[str, Any]] = {}
resumed_checks = 0
if sample_schemas or sample_reads:
present_data_kinds = sorted(row["kind"] for row in matrix if row["objects"] and row["kind"] in DATA_KINDS)
data_checks, resumed_checks = run_data_checks(
base_url,
base_id,
token,
present_data_kinds,
timeout,
sample_reads,
workers,
checkpoint_path,
resume,
retry_degraded,
rpc_call,
)
for row in matrix:
check = data_checks.get(row["kind"])
if check:
row["data_check"] = check
row["list_status"] = check.get("operations", {}).get("metadata.objects.list", {}).get("status")
schema = check.get("operations", {}).get("data.schema")
if schema:
row["data_schema"] = schema
elif sample_objects:
for row in matrix:
if not row["objects"]:
continue
listed, _ = safe_rpc(
rpc_call,
base_url,
token,
"metadata.objects.list",
{"base_id": base_id, "kind": row["kind"], "limit": 1, "timeout_seconds": int(timeout)},
timeout,
)
row["list_status"] = listed.get("status")
sample = first_object(listed)
if sample:
row["sample_selector"] = public_selector(sample)
missing = [row["kind"] for row in matrix if row["discovery"] == "absent_in_base"]
failures = [
row["kind"]
for row in matrix
if sample_objects and row.get("objects") and "list_status" in row and row.get("list_status") != "ok"
]
data_failures = sorted(kind for kind, check in data_checks.items() if check.get("status") != "ok")
status = "ok" if not failures and not data_failures else "degraded"
return {
"schema": "onec_adapter_coverage_audit.v1",
"status": status,
"generated_at": utc_now(),
"base_url": base_url,
"base_id": base_id,
"sampling": {
"objects": sample_objects,
"data_schemas": sample_schemas or sample_reads,
"data_reads": sample_reads,
"workers": workers,
"resumed_checks": resumed_checks,
},
"policy": {
"application_data": "read_only",
"metadata_structure": "read_only",
"sql_identity": "configured_base_credentials_only",
"writes": ["ConfigSave", "ConfigCASSave"],
},
"counts": {
"kinds": len(matrix),
"present_kinds": sum(1 for row in matrix if row["objects"]),
"absent_kinds": len(missing),
"list_failures": len(failures),
"data_kinds_declared": len(DATA_KINDS),
"data_kinds_checked": len(data_checks),
"data_check_failures": len(data_failures),
},
"absent_in_base": missing,
"list_failures": failures,
"data_check_failures": data_failures,
"matrix": matrix,
"data_checks": data_checks,
"child_objects": audit.get("child_objects") or {},
"not_yet_decoded": audit.get("not_yet_decoded") or [],
"optional_deep_reads": audit.get("optional_deep_reads") or [],
"unmapped_source_roles": audit.get("unmapped_source_roles") or audit.get("unknown_source_roles") or {},
"write_capabilities": audit.get("write_capabilities") or {},
}
def main() -> int:
parser = argparse.ArgumentParser(description="Audit live 1C adapter coverage without exposing SQL credentials.")
parser.add_argument("--base-url", default=DEFAULT_BASE_URL)
parser.add_argument("--base-id", default=DEFAULT_BASE_ID)
parser.add_argument("--token-env", default="ONEC_ADAPTER_TOKEN")
parser.add_argument("--timeout", type=float, default=120.0, help="Timeout for each adapter call, in seconds.")
parser.add_argument("--workers", type=int, default=1, help="Concurrent data-kind checks (default: 1).")
parser.add_argument("--sample-objects", action="store_true", help="List one object for each present metadata kind.")
parser.add_argument("--sample-data-schemas", action="store_true", help="Decode one logical data schema for every present data kind.")
parser.add_argument("--sample-data-reads", action="store_true", help="Run schema, list, get (when applicable), and count for every present data kind.")
parser.add_argument("--checkpoint", type=Path, help="Atomically save progress after every completed data kind.")
parser.add_argument("--resume", action="store_true", help="Reuse completed kinds from --checkpoint.")
parser.add_argument("--retry-degraded", action="store_true", help="With --resume, rerun checkpoint entries whose status is not ok.")
parser.add_argument("--output", type=Path)
args = parser.parse_args()
if args.workers < 1:
parser.error("--workers must be >= 1")
if args.resume and args.checkpoint is None:
parser.error("--resume requires --checkpoint")
if args.retry_degraded and not args.resume:
parser.error("--retry-degraded requires --resume")
token = os.environ.get(args.token_env, "").strip()
if not token:
parser.error(f"adapter token is required in environment variable {args.token_env}")
sample_objects = args.sample_objects or args.sample_data_schemas or args.sample_data_reads
try:
report = build_report(
args.base_url,
args.base_id,
token,
args.timeout,
sample_objects,
args.sample_data_schemas,
sample_reads=args.sample_data_reads,
workers=args.workers,
checkpoint_path=args.checkpoint,
resume=args.resume,
retry_degraded=args.retry_degraded,
)
except ValueError as exc:
parser.error(str(exc))
rendered = json.dumps(report, ensure_ascii=False, indent=2)
if args.output:
write_json_atomic(args.output, report)
print(rendered)
return 0 if report.get("status") == "ok" else 1
if __name__ == "__main__":
sys.exit(main())
+191
View File
@@ -0,0 +1,191 @@
from __future__ import annotations
import argparse
import datetime as dt
import json
import time
import urllib.error
import urllib.request
from pathlib import Path
from common import ROOT, read_json, write_json
DEFAULT_MODEL_ID = "qwen3-coder-30b-a3b-instruct-q6_k"
DEFAULT_PROFILES = ["gpu-fast", "cpu-test"]
DEFAULT_REPORT = ROOT / "reports" / "benchmarks" / "runtime-profiles-qwen3-coder-q6.json"
RUNTIME_PROFILES = ROOT / "config" / "runtime_profiles.json"
DEFAULT_PROMPT = (
"Ты эксперт 1С. Кратко, но предметно опиши безопасный план анализа ошибки "
"проведения документа РеализацияТоваровУслуг, если нет metadata snapshot. Дай 8 пунктов."
)
def chat_completion(
*,
base_url: str,
model: str,
prompt: str,
temperature: float,
max_tokens: int,
timeout: int,
) -> dict:
payload = {
"model": model,
"messages": [
{"role": "system", "content": "Отвечай по-русски, кратко и по делу."},
{"role": "user", "content": prompt},
],
"temperature": temperature,
"max_tokens": max_tokens,
"stream": False,
}
request = urllib.request.Request(
f"{base_url.rstrip('/')}/v1/chat/completions",
data=json.dumps(payload, ensure_ascii=False).encode("utf-8"),
headers={"Content-Type": "application/json"},
method="POST",
)
started_at = time.perf_counter()
with urllib.request.urlopen(request, timeout=timeout) as response:
data = json.loads(response.read().decode("utf-8"))
elapsed_sec = time.perf_counter() - started_at
choices = data.get("choices") or []
answer = ""
if choices and isinstance(choices[0], dict):
answer = str((choices[0].get("message") or {}).get("content") or "")
usage = data.get("usage") or {}
completion_tokens = int(usage.get("completion_tokens") or 0)
total_tokens = int(usage.get("total_tokens") or 0)
prompt_tokens = int(usage.get("prompt_tokens") or 0)
return {
"status": "ok",
"elapsed_sec": round(elapsed_sec, 3),
"prompt_tokens": prompt_tokens,
"completion_tokens": completion_tokens,
"total_tokens": total_tokens,
"output_tokens_per_sec": round(completion_tokens / elapsed_sec, 3) if completion_tokens else None,
"total_tokens_per_sec": round(total_tokens / elapsed_sec, 3) if total_tokens else None,
"answer_preview": answer[:800],
}
def profile_target(profile: dict, model_id: str, plugin: str) -> dict:
overrides = profile.get("model_overrides") or {}
override = overrides.get(model_id) or {}
base_url = override.get("base_url") or (profile.get("endpoints") or {}).get(plugin)
served_model_name = override.get("served_model_name")
if not base_url:
raise ValueError(f"profile `{profile.get('id')}` has no endpoint for model `{model_id}`")
if not served_model_name:
raise ValueError(f"profile `{profile.get('id')}` has no served_model_name override for `{model_id}`")
return {
"base_url": str(base_url),
"served_model_name": str(served_model_name),
"container_name": override.get("container_name"),
"host": profile.get("host"),
"docker_endpoint": profile.get("docker_endpoint"),
"role": profile.get("role"),
}
def benchmark_profile(profile_id: str, profile: dict, *, model_id: str, plugin: str, args: argparse.Namespace) -> dict:
try:
target = profile_target(profile, model_id, plugin)
result = chat_completion(
base_url=target["base_url"],
model=target["served_model_name"],
prompt=args.prompt,
temperature=args.temperature,
max_tokens=args.max_tokens,
timeout=args.timeout,
)
return {
"profile_id": profile_id,
"label": profile.get("label") or profile_id,
"model_id": model_id,
"target": target,
**result,
}
except (ValueError, TimeoutError, urllib.error.URLError, json.JSONDecodeError) as exc:
return {
"profile_id": profile_id,
"label": profile.get("label") or profile_id,
"model_id": model_id,
"status": "error",
"error": str(exc),
}
def speedup_summary(results: list[dict]) -> dict:
speeds = {
str(item.get("profile_id")): float(item.get("output_tokens_per_sec") or 0)
for item in results
if item.get("status") == "ok" and item.get("output_tokens_per_sec")
}
gpu_speed = speeds.get("gpu-fast")
cpu_speed = speeds.get("cpu-test")
summary = {"output_tokens_per_sec": speeds}
if gpu_speed and cpu_speed:
summary["gpu_vs_cpu_ratio"] = round(gpu_speed / cpu_speed, 3)
return summary
def main() -> int:
parser = argparse.ArgumentParser(description="Benchmark the same model across runtime profiles.")
parser.add_argument("--model-id", default=DEFAULT_MODEL_ID)
parser.add_argument("--plugin", default="1c")
parser.add_argument("--profiles", nargs="+", default=DEFAULT_PROFILES)
parser.add_argument("--prompt", default=DEFAULT_PROMPT)
parser.add_argument("--temperature", type=float, default=0.1)
parser.add_argument("--max-tokens", type=int, default=384)
parser.add_argument("--timeout", type=int, default=600)
parser.add_argument("--report", type=Path, default=DEFAULT_REPORT)
parser.add_argument("--print", action="store_true")
args = parser.parse_args()
config = read_json(RUNTIME_PROFILES)
profiles = config.get("profiles") or {}
results = []
for profile_id in args.profiles:
profile = profiles.get(profile_id)
if not isinstance(profile, dict):
results.append(
{
"profile_id": profile_id,
"model_id": args.model_id,
"status": "error",
"error": f"unknown runtime profile: {profile_id}",
}
)
continue
profile = {**profile, "id": profile_id}
results.append(benchmark_profile(profile_id, profile, model_id=args.model_id, plugin=args.plugin, args=args))
ok_results = [item for item in results if item.get("status") == "ok"]
fastest = None
if ok_results:
fastest = max(ok_results, key=lambda item: float(item.get("output_tokens_per_sec") or 0)).get("profile_id")
report = {
"created_at": dt.datetime.now(dt.UTC).isoformat(),
"model_id": args.model_id,
"plugin": args.plugin,
"prompt": args.prompt,
"temperature": args.temperature,
"max_tokens": args.max_tokens,
"status": "ok" if len(ok_results) == len(results) else "partial" if ok_results else "failed",
"fastest_profile_id": fastest,
"speedup": speedup_summary(results),
"results": results,
}
write_json(args.report, report)
if args.print:
print(json.dumps(report, ensure_ascii=False, indent=2))
else:
print(f"Benchmark status: {report['status']}")
print(f"Wrote report to {args.report}")
return 0 if ok_results else 1
if __name__ == "__main__":
raise SystemExit(main())
+150
View File
@@ -0,0 +1,150 @@
#!/usr/bin/env python3
"""Build the first agent-facing intake packet for a 1C user task."""
from __future__ import annotations
import argparse
import base64
import json
import sys
from pathlib import Path
from typing import Any
sys.path.insert(0, str(Path(__file__).resolve().parent))
from route_1c_question import route_question # noqa: E402
ROOT = Path(__file__).resolve().parents[1]
DEFAULT_INDEX = ROOT / "reports" / "1c-sql" / "upo" / "unified-object-route-index.json"
def decode_arg(value: str | None, encoded: str | None) -> str | None:
if encoded:
return base64.b64decode(encoded).decode("utf-8")
return value
def fact_summary(fact_checks: list[dict[str, Any]]) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]:
confirmed = []
unresolved = []
for check in fact_checks:
result = check.get("result") or {}
row = {
"path": check.get("path"),
"status": check.get("status"),
"exists": result.get("exists") if result else None,
"confidence": result.get("confidence"),
"reason": result.get("reason"),
"object": result.get("object"),
"match": result.get("match"),
}
if result.get("exists") is True:
confirmed.append(row)
else:
unresolved.append(row)
return confirmed, unresolved
def next_commands(route: dict[str, Any], *, index: Path, view: str) -> list[dict[str, Any]]:
commands = []
decision = route.get("decision") or {}
if decision.get("needs_docs_rag"):
commands.append(
{
"tool": "docs_rag",
"purpose": "official_documentation_context",
"api": "/api/rag/query",
"payload": {
"question": route.get("question"),
"source_type": decision.get("safe_rag_scope") or "official_1c_docs",
"limit": 5,
},
}
)
for path in route.get("fact_paths") or []:
commands.append(
{
"tool": "fact_resolver",
"purpose": "current_configuration_fact",
"command": f"python scripts/resolve_1c_fact.py --index {index} --path <utf8-base64:{path}> --view {view}",
"api": "/api/1c/fact",
"payload": {
"source_kind": "route_index",
"source_path": str(index),
"path": path,
"view": view,
},
}
)
if decision.get("needs_current_config") and not route.get("fact_paths"):
commands.append(
{
"tool": "task_evidence",
"purpose": "discover_objects_and_relevant_metadata",
"command": f"python scripts/build_1c_task_evidence.py --index {index} --text <utf8-base64 task> --view {view}",
}
)
return commands
def build_intake(text: str, *, index: Path, view: str) -> dict[str, Any]:
route = route_question(text, index_path=index, view=view)
confirmed, unresolved = fact_summary(route.get("fact_checks") or [])
decision = route.get("decision") or {}
code_allowed = not unresolved and bool(confirmed or not decision.get("needs_current_config"))
if decision.get("needs_current_config") and not confirmed and not route.get("fact_paths"):
code_allowed = False
return {
"schema": "onec_agent_intake.v1",
"task": {"text": text},
"index": str(index),
"view": view,
"route": route,
"source_policy": {
"allowed_for_current_facts": ["route_index", "metadata_snapshot_explicit_current", "1c_agent_current"],
"allowed_for_documentation": ["official_1c_docs"],
"examples_are_current_facts": False,
"blocked_as_current_fact_sources": ["metadata.example", "synthetic-example", "rag_examples", "old_exports"],
},
"facts": {
"confirmed": confirmed,
"unresolved": unresolved,
"confirmed_count": len(confirmed),
"unresolved_count": len(unresolved),
},
"answer_policy": {
"code_generation_allowed": code_allowed,
"must_check_current_config_before_code": bool(decision.get("current_config_required_before_code")),
"must_not_use_examples_as_facts": True,
"safe_rag_scope": decision.get("safe_rag_scope"),
},
"next_commands": next_commands(route, index=index, view=view),
}
def main() -> int:
parser = argparse.ArgumentParser(description="Build first agent intake packet for a 1C task.")
parser.add_argument("--text")
parser.add_argument("--text-b64")
parser.add_argument("--index", type=Path, default=DEFAULT_INDEX)
parser.add_argument("--view", choices=["effective", "base"], default="effective")
parser.add_argument("--output", type=Path)
args = parser.parse_args()
text = decode_arg(args.text, args.text_b64)
if not text:
raise SystemExit("Use --text or --text-b64.")
result = build_intake(text, index=args.index, view=args.view)
output = json.dumps(result, ensure_ascii=False, indent=2) + "\n"
if args.output:
args.output.parent.mkdir(parents=True, exist_ok=True)
args.output.write_text(output, encoding="utf-8")
print(json.dumps({"output": str(args.output), "route": result["route"]["decision"]["route"], "code_allowed": result["answer_policy"]["code_generation_allowed"]}, ensure_ascii=False))
else:
print(output)
return 0
if __name__ == "__main__":
raise SystemExit(main())
+153
View File
@@ -0,0 +1,153 @@
#!/usr/bin/env python3
"""Build enum order -> presentation map from 1C XML routes."""
from __future__ import annotations
import argparse
import json
import xml.etree.ElementTree as ET
from pathlib import Path
from typing import Any
NS = {
"md": "http://v8.3/MDClasses",
"v8": "http://v8.1c.ru/8.1/data/core",
}
def load_json(path: Path) -> dict[str, Any]:
return json.loads(path.read_text(encoding="utf-8-sig"))
def local_name(tag: str) -> str:
return tag.rsplit("}", 1)[-1]
def child_text(parent: ET.Element, name: str) -> str | None:
for child in list(parent):
if local_name(child.tag) == name:
return child.text or ""
return None
def properties(element: ET.Element) -> ET.Element | None:
for child in list(element):
if local_name(child.tag) == "Properties":
return child
return None
def synonym(props: ET.Element | None) -> str | None:
if props is None:
return None
for syn in list(props):
if local_name(syn.tag) != "Synonym":
continue
for item in list(syn):
lang = None
content = None
for child in list(item):
if local_name(child.tag) == "lang":
lang = child.text
elif local_name(child.tag) == "content":
content = child.text
if lang == "ru" and content:
return content
return None
def enum_values(path: Path) -> list[dict[str, Any]]:
root = ET.parse(path).getroot()
enum = next((node for node in root.iter() if local_name(node.tag) == "Enum"), None)
if enum is None:
return []
child_objects = next((node for node in list(enum) if local_name(node.tag) == "ChildObjects"), None)
if child_objects is None:
return []
values = []
order = 0
for node in list(child_objects):
if local_name(node.tag) != "EnumValue":
continue
props = properties(node)
name = child_text(props, "Name") if props is not None else None
values.append(
{
"order": order,
"uuid": (node.attrib.get("uuid") or "").lower() or None,
"name": name,
"synonym": synonym(props),
}
)
order += 1
return values
def is_base_config(top: dict[str, Any]) -> bool:
relative = str(top.get("relative_path") or "")
return relative.startswith("Enums\\")
def route_score(row: dict[str, Any]) -> tuple[int, str]:
return (1 if str(row.get("relative_path") or "").startswith("Enums\\") else 0, str(row.get("relative_path") or ""))
def main() -> int:
parser = argparse.ArgumentParser(description="Build 1C enum presentation map.")
parser.add_argument("--index", type=Path, required=True)
parser.add_argument("--output", type=Path, required=True)
args = parser.parse_args()
index = load_json(args.index)
enums: dict[str, dict[str, Any]] = {}
for guid, item in (index.get("objects") or {}).items():
tops = [
top
for top in item.get("xml_top_objects") or []
if top.get("xml_kind") == "Enum" and top.get("name") and top.get("path")
]
if not tops:
continue
tops.sort(key=lambda top: (not is_base_config(top), top.get("relative_path") or ""))
top = tops[0]
path = Path(top["path"])
if not path.is_file():
continue
try:
values = enum_values(path)
except ET.ParseError as error:
values = []
parse_error = str(error)
else:
parse_error = None
key = str(top["name"])
candidate = {
"guid": guid,
"name": top.get("name"),
"synonym": top.get("synonym"),
"relative_path": top.get("relative_path"),
"path": top.get("path"),
"parse_error": parse_error,
"values": values,
"by_order": {str(value["order"]): value for value in values},
"by_uuid": {value["uuid"]: value for value in values if value.get("uuid")},
}
existing = enums.get(key)
if existing is None or route_score(candidate) > route_score(existing):
enums[key] = candidate
result = {
"schema": "onec_enum_presentation_map.v1",
"index": str(args.index),
"enum_count": len(enums),
"enums": dict(sorted(enums.items())),
}
args.output.parent.mkdir(parents=True, exist_ok=True)
args.output.write_text(json.dumps(result, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
print(json.dumps({"output": str(args.output), "enum_count": len(enums)}, ensure_ascii=False))
return 0
if __name__ == "__main__":
raise SystemExit(main())
+121
View File
@@ -0,0 +1,121 @@
#!/usr/bin/env python3
"""Build an inventory that maps _ExtensionsInfo rows to DBNames-Ext files.
The script keeps the byte conversion explicit. SQL _ExtensionsInfo._IDRRef is
stored as 16 bytes. Observed DBNames-Ext suffixes match this rearrangement:
b[12:16] b[10:12] b[8:10] b[0:2] b[2:8]
This is recorded as an observed conversion and validated against exported
DBNames-Ext file names.
"""
from __future__ import annotations
import argparse
import json
import uuid
from pathlib import Path
from typing import Any
def load_json(path: Path) -> dict[str, Any]:
return json.loads(path.read_text(encoding="utf-8-sig"))
def dbnames_ext_guid_from_idrref(data: bytes) -> str:
if len(data) != 16:
raise ValueError(f"_IDRRef must contain 16 bytes, got {len(data)}")
reordered = data[12:16] + data[10:12] + data[8:10] + data[0:2] + data[2:8]
return str(uuid.UUID(bytes=reordered))
def binary_info(value: Any) -> dict[str, Any] | None:
if isinstance(value, dict) and value.get("type") == "binary":
return value
return None
def main() -> int:
parser = argparse.ArgumentParser(description="Map _ExtensionsInfo rows to DBNames-Ext files.")
parser.add_argument("--extensions-info", type=Path, required=True)
parser.add_argument("--params-dir", type=Path, required=True)
parser.add_argument("--output", type=Path, required=True)
args = parser.parse_args()
report = load_json(args.extensions_info)
dbnames_ext_files = {
path.name.removeprefix("DBNames-Ext-").lower(): path
for path in args.params_dir.iterdir()
if path.is_file() and path.name.startswith("DBNames-Ext-")
}
rows = []
matched = 0
for row in report.get("rows") or []:
columns = row.get("columns") or {}
idrref = binary_info(columns.get("_IDRRef"))
if not idrref:
continue
idrref_bytes = Path(idrref["path"]).read_bytes()
dbnames_guid = dbnames_ext_guid_from_idrref(idrref_bytes)
dbnames_file = dbnames_ext_files.get(dbnames_guid)
if dbnames_file:
matched += 1
rows.append(
{
"row_index": row.get("row_index"),
"extension_name": columns.get("_ExtName"),
"extension_order": columns.get("_ExtensionOrder"),
"update_time": columns.get("_UpdateTime"),
"use_purpose": columns.get("_ExtensionUsePurpose"),
"scope": columns.get("_ExtensionScope"),
"idrref_hex": idrref_bytes.hex(),
"dbnames_ext_guid": dbnames_guid,
"dbnames_ext_file": str(dbnames_file) if dbnames_file else None,
"dbnames_ext_file_name": dbnames_file.name if dbnames_file else None,
"dbnames_ext_file_bytes": dbnames_file.stat().st_size if dbnames_file else None,
"extension_zipped_info": binary_info(columns.get("_ExtensionZippedInfo")),
}
)
known_from_rows = {row["dbnames_ext_guid"] for row in rows}
orphan_dbnames_files = [
{
"file_name": path.name,
"guid_or_marker": guid,
"bytes": path.stat().st_size,
}
for guid, path in sorted(dbnames_ext_files.items())
if guid not in known_from_rows
]
result = {
"schema": "onec_extension_inventory.v1",
"extensions_info": str(args.extensions_info),
"params_dir": str(args.params_dir),
"observed_idrref_to_dbnames_ext_guid": "b[12:16] + b[10:12] + b[8:10] + b[0:2] + b[2:8]",
"extension_row_count": len(rows),
"matched_dbnames_ext_count": matched,
"orphan_dbnames_ext_count": len(orphan_dbnames_files),
"extensions": rows,
"orphan_dbnames_ext_files": orphan_dbnames_files,
}
args.output.parent.mkdir(parents=True, exist_ok=True)
args.output.write_text(json.dumps(result, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
print(
json.dumps(
{
"output": str(args.output),
"extensions": len(rows),
"matched_dbnames_ext": matched,
"orphan_dbnames_ext": len(orphan_dbnames_files),
},
ensure_ascii=False,
)
)
return 0
if __name__ == "__main__":
raise SystemExit(main())
+93
View File
@@ -0,0 +1,93 @@
#!/usr/bin/env python3
"""Build root CAS manifests for all parsed 1C extensions."""
from __future__ import annotations
import argparse
import json
from pathlib import Path
from extract_1c_extension_cas_manifest import extract_manifest
def safe_name(value: str) -> str:
result = "".join(char if char.isalnum() or char in "-_." else "_" for char in value)
return result[:120] or "extension"
def main() -> int:
parser = argparse.ArgumentParser(description="Build CAS manifests for all extensions.")
parser.add_argument("--zipped-info", type=Path, required=True)
parser.add_argument("--cas-dir", type=Path, required=True)
parser.add_argument("--output-dir", type=Path, required=True)
parser.add_argument("--summary", type=Path, required=True)
args = parser.parse_args()
zipped = json.loads(args.zipped_info.read_text(encoding="utf-8"))
args.output_dir.mkdir(parents=True, exist_ok=True)
items = []
for item in zipped.get("items") or []:
root_key = item.get("root_cas_key")
if not root_key:
continue
root_path = args.cas_dir / root_key
if not root_path.is_file():
items.append({**item, "manifest_status": "root_cas_missing"})
continue
try:
manifest = extract_manifest(root_path, args.cas_dir)
except Exception as exc:
items.append(
{
"extension_zipped_info_file": item.get("file_name"),
"root_cas_key": root_key,
"manifest_status": "parse_error",
"error": str(exc),
}
)
continue
stem = safe_name(item.get("file_name", "").replace("__ExtensionZippedInfo.bin", ""))
output_path = args.output_dir / f"{stem}-{root_key[:8]}.json"
report = {
"schema": "onec_extension_cas_manifest.v1",
"extension_zipped_info": item,
**manifest,
}
output_path.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
items.append(
{
"extension_zipped_info_file": item.get("file_name"),
"root_cas_key": root_key,
"manifest_path": str(output_path),
"extension_configuration_guid": manifest.get("extension_configuration_guid"),
"declared_count": manifest.get("declared_count"),
"entry_count": manifest.get("entry_count"),
"missing_cas_entries": sum(1 for entry in manifest.get("entries") or [] if not entry.get("cas_exists")),
"manifest_status": "ok",
}
)
summary = {
"schema": "onec_extension_manifests_summary.v1",
"zipped_info": str(args.zipped_info),
"cas_dir": str(args.cas_dir),
"output_dir": str(args.output_dir),
"extension_count": len(items),
"items": items,
}
args.summary.parent.mkdir(parents=True, exist_ok=True)
args.summary.write_text(json.dumps(summary, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
print(
json.dumps(
{
"summary": str(args.summary),
"extensions": len(items),
"ok": sum(1 for item in items if item.get("manifest_status") == "ok"),
},
ensure_ascii=False,
)
)
return 0
if __name__ == "__main__":
raise SystemExit(main())
@@ -0,0 +1,162 @@
from __future__ import annotations
import argparse
import json
import time
from pathlib import Path
from typing import Any
import yaml
from one_c_its_platform import parse_doc_coordinate, platform_doc_id_from_url
ROOT = Path(__file__).resolve().parents[1]
DEFAULT_START_LINKS = ROOT / "plugins" / "1c" / "rag" / "official-docs" / "start-links.json"
DEFAULT_SOURCES = ROOT / "plugins" / "1c" / "rag" / "official-docs" / "sources.yaml"
DEFAULT_OUTPUT = ROOT / "plugins" / "1c" / "rag" / "official-docs" / "platform-versions.json"
def load_json(path: Path) -> dict[str, Any]:
if not path.exists():
return {}
data = json.loads(path.read_text(encoding="utf-8-sig"))
return data if isinstance(data, dict) else {}
def load_yaml(path: Path) -> dict[str, Any]:
if not path.exists():
return {}
data = yaml.safe_load(path.read_text(encoding="utf-8"))
return data if isinstance(data, dict) else {}
def version_key(value: str | None) -> tuple[int, int, int, str]:
if not value or value == "8.x":
return (8, -1, -1, value or "")
parts = value.split(".")
numeric = []
for part in parts[:3]:
try:
numeric.append(int(part))
except ValueError:
numeric.append(-1)
while len(numeric) < 3:
numeric.append(-1)
return (numeric[0], numeric[1], numeric[2], value)
def version_from_title(title: str) -> str | None:
import re
match = re.search(r"\b(\d+\.\d+(?:\.\d+)?)\b", title)
return match.group(1) if match else None
def best_platform_version(coord: dict[str, str | None], title: str) -> str | None:
version = coord.get("platform_version")
if version == "8.x":
return version_from_title(title) or version
return version or version_from_title(title)
def build_catalog(start_links_path: Path, sources_path: Path) -> dict[str, Any]:
start_links = load_json(start_links_path)
sources_config = load_yaml(sources_path)
active_by_doc_id: dict[str, list[dict[str, Any]]] = {}
for source in sources_config.get("sources") or []:
url = str(source.get("url") or "")
doc_id = platform_doc_id_from_url(url)
if not doc_id:
continue
active_by_doc_id.setdefault(doc_id, []).append(
{
"source_id": source.get("id"),
"title": source.get("title"),
"url": url,
"source_type": source.get("source_type"),
}
)
versions_by_doc_id: dict[str, dict[str, Any]] = {}
for item in start_links.get("start_links") or []:
url = str(item.get("url") or "")
coord = parse_doc_coordinate(url)
doc_id = coord.get("platform_doc_id")
if not doc_id:
continue
title = str(item.get("text") or "")
record = versions_by_doc_id.setdefault(
doc_id,
{
"platform_doc_id": doc_id,
"platform_version": best_platform_version(coord, title),
"url": url,
"title": title,
"category": item.get("category"),
"active": False,
"active_sources": [],
},
)
record["title"] = record.get("title") or item.get("text") or ""
record["category"] = record.get("category") or item.get("category")
for doc_id, sources in active_by_doc_id.items():
coord = parse_doc_coordinate(sources[0]["url"])
title = str(sources[0].get("title") or "")
record = versions_by_doc_id.setdefault(
doc_id,
{
"platform_doc_id": doc_id,
"platform_version": best_platform_version(coord, title),
"url": sources[0]["url"],
"title": title,
"category": "platform_doc",
"active": False,
"active_sources": [],
},
)
record["active"] = True
record["active_sources"] = sources
versions = sorted(versions_by_doc_id.values(), key=lambda item: version_key(item.get("platform_version")), reverse=True)
active_versions = [item for item in versions if item.get("active")]
latest_8_3 = next((item for item in versions if str(item.get("platform_version") or "").startswith("8.3.")), None)
latest_active_8_3 = next((item for item in active_versions if str(item.get("platform_version") or "").startswith("8.3.")), None)
return {
"schema": "onec_its_platform_versions.v1",
"created_at_unix": int(time.time()),
"sources": {
"start_links": str(start_links_path),
"sources_yaml": str(sources_path),
},
"counts": {
"versions": len(versions),
"active_versions": len(active_versions),
},
"defaults": {
"latest_discovered_8_3": latest_8_3,
"latest_active_8_3": latest_active_8_3,
},
"versions": versions,
}
def main() -> int:
parser = argparse.ArgumentParser(description="Build a version catalog for 1C:ITS platform documentation.")
parser.add_argument("--start-links", type=Path, default=DEFAULT_START_LINKS)
parser.add_argument("--sources", type=Path, default=DEFAULT_SOURCES)
parser.add_argument("--output", type=Path, default=DEFAULT_OUTPUT)
parser.add_argument("--print", action="store_true", dest="print_report")
args = parser.parse_args()
report = build_catalog(args.start_links, args.sources)
args.output.parent.mkdir(parents=True, exist_ok=True)
args.output.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
payload = report if args.print_report else {"output": str(args.output), "counts": report["counts"], "defaults": report["defaults"]}
print(json.dumps(payload, ensure_ascii=False, indent=2))
return 0
if __name__ == "__main__":
raise SystemExit(main())
+432
View File
@@ -0,0 +1,432 @@
from __future__ import annotations
import argparse
import hashlib
import html
import json
import re
import shutil
import urllib.parse
import urllib.request
from html.parser import HTMLParser
from pathlib import Path
from typing import Any
from normalize_1c_its_docs import decode_html
ROOT = Path(__file__).resolve().parents[1]
DEFAULT_MANIFEST = ROOT / "plugins" / "1c" / "rag" / "official-docs" / "normalized" / "manifest.json"
DEFAULT_NORMALIZED_DIR = ROOT / "plugins" / "1c" / "rag" / "official-docs" / "normalized"
DEFAULT_RAW_DIR = ROOT / "plugins" / "1c" / "rag" / "official-docs" / "raw"
DEFAULT_MEDIA_MANIFEST = ROOT / "plugins" / "1c" / "rag" / "official-docs" / "media" / "manifest.json"
DEFAULT_MEDIA_DIR = ROOT / "plugins" / "1c" / "rag" / "official-docs" / "media"
DEFAULT_OUTPUT_DIR = ROOT / "plugins" / "1c" / "rag" / "official-docs" / "static"
STYLE = """
:root {
--bg: #f2f3ef;
--paper: #fffef9;
--ink: #202522;
--muted: #66706a;
--line: #cdd5cd;
--accent: #0f6b5f;
--code: #17201c;
}
* { box-sizing: border-box; }
body { margin: 0; background: var(--bg); color: var(--ink); font: 16px/1.55 "Aptos", "Segoe UI", Tahoma, sans-serif; }
a { color: var(--accent); }
.wrap { max-width: 1120px; margin: 0 auto; padding: 24px; }
.doc { background: var(--paper); border: 1px solid var(--line); border-radius: 8px; padding: 24px; }
.meta { color: var(--muted); font-size: 13px; overflow-wrap: anywhere; margin-bottom: 18px; }
h1 { font-size: 28px; line-height: 1.2; margin: 0 0 12px; }
h2 { font-size: 20px; margin-top: 28px; border-top: 1px solid var(--line); padding-top: 18px; }
img { max-width: 100%; height: auto; border: 1px solid var(--line); background: #fff; }
figure { margin: 18px 0; }
figcaption { color: var(--muted); font-size: 13px; margin-top: 6px; }
pre { background: var(--code); color: #e4ece6; padding: 12px; border-radius: 6px; overflow: auto; }
code { font-family: "Cascadia Mono", Consolas, monospace; }
table { width: 100%; border-collapse: collapse; }
th, td { border-bottom: 1px solid var(--line); padding: 8px 10px; text-align: left; vertical-align: top; }
.top { display: flex; gap: 8px; flex-wrap: wrap; margin-bottom: 14px; }
.btn { border: 1px solid var(--line); border-radius: 6px; padding: 6px 10px; background: #fff; text-decoration: none; }
.badge { display: inline-block; border: 1px solid var(--line); border-radius: 999px; padding: 2px 8px; color: var(--muted); font-size: 12px; }
""".strip()
class AssetCollector(HTMLParser):
def __init__(self, *, page_url: str) -> None:
super().__init__(convert_charrefs=True)
self.page_url = page_url
self.urls: set[str] = set()
def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
tag_lower = tag.lower()
attr_map = {name.lower(): value or "" for name, value in attrs}
for name, value in attrs:
if not value:
continue
name_lower = name.lower()
if tag_lower == "link" and name_lower == "href" and is_static_link_asset(attr_map):
self.urls.add(normalize_url(urllib.parse.urljoin(self.page_url, value)))
elif tag_lower == "script" and name_lower == "src":
self.urls.add(normalize_url(urllib.parse.urljoin(self.page_url, value)))
class LinkRewriter(HTMLParser):
def __init__(
self,
*,
page_url: str,
url_to_page: dict[str, str],
media_url_to_file: dict[str, str],
asset_url_to_file: dict[str, str],
) -> None:
super().__init__(convert_charrefs=False)
self.page_url = page_url
self.url_to_page = url_to_page
self.media_url_to_file = media_url_to_file
self.asset_url_to_file = asset_url_to_file
self.parts: list[str] = []
def rewrite_url(self, value: str, *, is_media: bool = False) -> str:
absolute = normalize_url(urllib.parse.urljoin(self.page_url, value))
if is_media and absolute in self.media_url_to_file:
return f"../media/{self.media_url_to_file[absolute]}"
if absolute in self.asset_url_to_file:
return f"../assets/{self.asset_url_to_file[absolute]}"
if absolute in self.url_to_page:
return f"../pages/{self.url_to_page[absolute]}"
if absolute.startswith(("http://", "https://")):
return absolute
return value
def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
rewritten = []
for name, value in attrs:
if value is None:
rewritten.append((name, None))
continue
lowered = name.lower()
if tag.lower() == "img" and lowered == "src":
value = self.rewrite_url(value, is_media=True)
elif tag.lower() == "a" and lowered == "href":
value = self.rewrite_url(value)
elif tag.lower() in {"link", "script"} and lowered in {"href", "src"}:
value = self.rewrite_url(value)
rewritten.append((name, value))
attr_text = "".join(f" {name}" if value is None else f' {name}="{html.escape(value, quote=True)}"' for name, value in rewritten)
self.parts.append(f"<{tag}{attr_text}>")
def handle_endtag(self, tag: str) -> None:
self.parts.append(f"</{tag}>")
def handle_startendtag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
self.handle_starttag(tag, attrs)
def handle_data(self, data: str) -> None:
self.parts.append(data)
def handle_entityref(self, name: str) -> None:
self.parts.append(f"&{name};")
def handle_charref(self, name: str) -> None:
self.parts.append(f"&#{name};")
def handle_comment(self, data: str) -> None:
self.parts.append(f"<!--{data}-->")
def html(self) -> str:
return "".join(self.parts)
def load_json(path: Path) -> dict[str, Any]:
if not path.exists():
return {}
try:
data = json.loads(path.read_text(encoding="utf-8-sig"))
except json.JSONDecodeError:
return {}
return data if isinstance(data, dict) else {}
def normalize_url(value: str) -> str:
return urllib.parse.urlunparse(urllib.parse.urlparse(value)._replace(fragment=""))
def is_static_link_asset(attrs: dict[str, str]) -> bool:
rel = {part.casefold() for part in re.split(r"\s+", attrs.get("rel", "")) if part}
href = attrs.get("href", "").casefold()
as_type = attrs.get("as", "").casefold()
if "stylesheet" in rel or href.endswith(".css"):
return True
if "icon" in rel or "shortcut" in rel:
return True
return "preload" in rel and as_type in {"style", "script", "font"}
def safe_html_name(value: str, fallback: str) -> str:
name = re.sub(r"[^A-Za-zА-Яа-яЁё0-9_.-]+", "_", value, flags=re.UNICODE).strip("_")
return f"{(name or fallback)[:100]}.html"
def safe_asset_name(url: str) -> str:
parsed = urllib.parse.urlparse(url)
name = Path(parsed.path).name or "asset"
stem = re.sub(r"[^A-Za-z0-9_.-]+", "_", Path(name).stem).strip("_") or "asset"
suffix = re.sub(r"[^A-Za-z0-9.]+", "", Path(name).suffix) or ".bin"
digest = hashlib.sha256(url.encode("utf-8")).hexdigest()[:12]
return f"{stem}__{digest}{suffix}"
def media_map(media_manifest: dict[str, Any]) -> dict[str, str]:
return {str(item.get("url")): str(item.get("file")) for item in media_manifest.get("items") or [] if item.get("url") and item.get("file")}
def page_name_map(pages: list[dict[str, Any]]) -> dict[str, str]:
result = {}
used = set()
for index, page in enumerate(pages, start=1):
name = safe_html_name(str(page.get("title") or page.get("url") or ""), f"page_{index}")
if name in used:
stem = Path(name).stem
name = f"{stem}_{index}.html"
used.add(name)
url = str(page.get("url") or "")
result[url] = name
parsed = urllib.parse.urlparse(url)
if parsed.path.endswith("/hdoc"):
result[normalize_url(urllib.parse.urlunparse(parsed._replace(path=f"{parsed.path}/01")))] = name
return result
def collect_asset_urls(pages: list[dict[str, Any]], raw_dir: Path) -> list[str]:
urls: set[str] = set()
for page in pages:
raw_file = str(page.get("raw_file") or "")
raw_path = raw_dir / raw_file
if not raw_file or not raw_path.exists():
continue
page_url = str(page.get("url") or "")
raw_text = decode_html(raw_path.read_bytes(), page)
collector = AssetCollector(page_url=page_url)
collector.feed(raw_text)
urls.update(url for url in collector.urls if url.startswith(("http://", "https://")))
return sorted(urls)
def download_assets(asset_urls: list[str], assets_dir: Path) -> tuple[dict[str, str], list[dict[str, str]]]:
assets_dir.mkdir(parents=True, exist_ok=True)
url_to_file: dict[str, str] = {}
errors: list[dict[str, str]] = []
for url in asset_urls:
filename = safe_asset_name(url)
target = assets_dir / filename
try:
request = urllib.request.Request(url, headers={"User-Agent": "Codex 1C local static archive"})
with urllib.request.urlopen(request, timeout=30) as response:
target.write_bytes(response.read())
except Exception as exc: # noqa: BLE001
errors.append({"url": url, "error": str(exc)})
continue
url_to_file[url] = filename
return url_to_file, errors
def read_normalized_body(path: Path) -> str:
text = path.read_text(encoding="utf-8-sig")
if text.startswith("---"):
parts = text.split("---", 2)
if len(parts) == 3:
text = parts[2]
return text.strip()
def markdown_to_html(markdown: str, media_url_to_file: dict[str, str]) -> str:
lines = markdown.splitlines()
out: list[str] = []
in_list = False
for line in lines:
stripped = line.strip()
if not stripped:
if in_list:
out.append("</ul>")
in_list = False
continue
if stripped.startswith("# "):
if in_list:
out.append("</ul>")
in_list = False
out.append(f"<h1>{html.escape(stripped[2:].strip())}</h1>")
continue
if stripped.startswith("## "):
if in_list:
out.append("</ul>")
in_list = False
out.append(f"<h2>{html.escape(stripped[3:].strip())}</h2>")
continue
image_match = re.match(r"-\s*!\[(.*?)\]\((.*?)\)(.*)", stripped)
if image_match:
if in_list:
out.append("</ul>")
in_list = False
alt, url, suffix = image_match.groups()
image_src = f"../media/{media_url_to_file[url]}" if url in media_url_to_file else url
out.append(f"<figure><img src=\"{html.escape(image_src, quote=True)}\" alt=\"{html.escape(alt)}\"><figcaption>{html.escape((alt + suffix).strip())}</figcaption></figure>")
continue
if stripped.startswith("- "):
if not in_list:
out.append("<ul>")
in_list = True
out.append(f"<li>{html.escape(stripped[2:].strip())}</li>")
continue
if in_list:
out.append("</ul>")
in_list = False
out.append(f"<p>{html.escape(stripped)}</p>")
if in_list:
out.append("</ul>")
return "\n".join(out)
def write_static_page(
page: dict[str, Any],
*,
normalized_dir: Path,
pages_dir: Path,
raw_dir: Path,
raw_dir_out: Path,
page_names: dict[str, str],
media_url_to_file: dict[str, str],
asset_url_to_file: dict[str, str],
) -> dict[str, Any]:
page_url = str(page.get("url") or "")
page_file = page_names[page_url]
normalized_file = str(page.get("normalized_file") or "")
normalized_path = normalized_dir / normalized_file
body = markdown_to_html(read_normalized_body(normalized_path), media_url_to_file)
raw_file = str(page.get("raw_file") or "")
raw_output_name = None
if raw_file:
raw_path = raw_dir / raw_file
if raw_path.exists():
raw_output_name = f"raw_{page_file}"
raw_text = decode_html(raw_path.read_bytes(), page)
rewriter = LinkRewriter(page_url=page_url, url_to_page=page_names, media_url_to_file=media_url_to_file, asset_url_to_file=asset_url_to_file)
rewriter.feed(raw_text)
raw_dir_out.mkdir(parents=True, exist_ok=True)
(raw_dir_out / raw_output_name).write_text(rewriter.html(), encoding="utf-8")
raw_link = f'<a class="btn" href="../raw/{raw_output_name}">Raw HTML</a>' if raw_output_name else ""
content = f"""<!doctype html>
<html lang="ru">
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>{html.escape(str(page.get("title") or ""))}</title>
<style>{STYLE}</style>
</head>
<body>
<main class="wrap">
<div class="top"><a class="btn" href="../index.html">Индекс</a>{raw_link}<span class="badge">{html.escape(str(page.get("source_type") or ""))}</span></div>
<article class="doc">
<div class="meta">{html.escape(page_url)}</div>
{body}
</article>
</main>
</body>
</html>
"""
pages_dir.mkdir(parents=True, exist_ok=True)
(pages_dir / page_file).write_text(content, encoding="utf-8")
return {"title": page.get("title"), "url": page_url, "file": f"pages/{page_file}", "raw_file": f"raw/{raw_output_name}" if raw_output_name else None}
def build_static_site(manifest_path: Path, normalized_dir: Path, raw_dir: Path, media_manifest_path: Path, media_dir: Path, output_dir: Path, *, download_external_assets: bool) -> dict[str, Any]:
manifest = load_json(manifest_path)
media_manifest = load_json(media_manifest_path)
pages = manifest.get("pages") or []
page_names = page_name_map(pages)
media_url_to_file = media_map(media_manifest)
pages_dir = output_dir / "pages"
raw_dir_out = output_dir / "raw"
static_media_dir = output_dir / "media"
static_assets_dir = output_dir / "assets"
if output_dir.exists():
shutil.rmtree(output_dir)
static_media_dir.mkdir(parents=True, exist_ok=True)
for filename in media_url_to_file.values():
source = media_dir / filename
if source.exists():
shutil.copy2(source, static_media_dir / filename)
asset_urls = collect_asset_urls(pages, raw_dir) if download_external_assets else []
asset_url_to_file, asset_errors = download_assets(asset_urls, static_assets_dir) if asset_urls else ({}, [])
page_records = [
write_static_page(
page,
normalized_dir=normalized_dir,
pages_dir=pages_dir,
raw_dir=raw_dir,
raw_dir_out=raw_dir_out,
page_names=page_names,
media_url_to_file=media_url_to_file,
asset_url_to_file=asset_url_to_file,
)
for page in pages
if page.get("normalized_file")
]
rows = "\n".join(
f'<tr><td><a href="{html.escape(record["file"], quote=True)}">{html.escape(str(record["title"] or ""))}</a></td><td>{html.escape(str(record["url"] or ""))}</td></tr>'
for record in page_records
)
index = f"""<!doctype html>
<html lang="ru">
<head><meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1"><title>1C:ITS Static Archive</title><style>{STYLE}</style></head>
<body><main class="wrap"><article class="doc"><h1>1C:ITS Static Archive</h1><p class="meta">Локальный статический просмотр нормализованных страниц.</p><table><thead><tr><th>Страница</th><th>URL</th></tr></thead><tbody>{rows}</tbody></table></article></main></body>
</html>
"""
(output_dir / "index.html").write_text(index, encoding="utf-8")
result = {
"schema": "onec_its_static_site_manifest.v1",
"output_dir": str(output_dir),
"index": str(output_dir / "index.html"),
"counts": {
"pages": len(page_records),
"media_files": len(list(static_media_dir.glob("*"))),
"asset_files": len(list(static_assets_dir.glob("*"))) if static_assets_dir.exists() else 0,
"asset_errors": len(asset_errors),
},
"pages": page_records,
"assets": [{"url": url, "file": f"assets/{filename}"} for url, filename in sorted(asset_url_to_file.items())],
"asset_errors": asset_errors,
}
(output_dir / "manifest.json").write_text(json.dumps(result, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
return result
def main() -> int:
parser = argparse.ArgumentParser(description="Build local static HTML viewer for normalized private 1C:ITS docs.")
parser.add_argument("--manifest", type=Path, default=DEFAULT_MANIFEST)
parser.add_argument("--normalized-dir", type=Path, default=DEFAULT_NORMALIZED_DIR)
parser.add_argument("--raw-dir", type=Path, default=DEFAULT_RAW_DIR)
parser.add_argument("--media-manifest", type=Path, default=DEFAULT_MEDIA_MANIFEST)
parser.add_argument("--media-dir", type=Path, default=DEFAULT_MEDIA_DIR)
parser.add_argument("--output-dir", type=Path, default=DEFAULT_OUTPUT_DIR)
parser.add_argument("--skip-assets", action="store_true", help="Do not download and localize CSS/JS assets from raw HTML.")
args = parser.parse_args()
result = build_static_site(
args.manifest,
args.normalized_dir,
args.raw_dir,
args.media_manifest,
args.media_dir,
args.output_dir,
download_external_assets=not args.skip_assets,
)
print(json.dumps({"counts": result["counts"], "index": result["index"]}, ensure_ascii=False, indent=2))
return 0
if __name__ == "__main__":
raise SystemExit(main())
+114
View File
@@ -0,0 +1,114 @@
from __future__ import annotations
import argparse
import subprocess
import sys
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
DEFAULT_METADATA = ROOT / "plugins" / "1c" / "metadata" / "examples" / "metadata.example.json"
DEFAULT_BSL_MODULES = ROOT / "plugins" / "1c" / "metadata" / "examples" / "bsl-modules.example.json"
DEFAULT_SOURCE_DIR = ROOT / "plugins" / "1c" / "rag" / "sources"
DEFAULT_CORPUS = ROOT / "plugins" / "1c" / "datasets" / "prepared" / "rag_corpus.jsonl"
DEFAULT_MANIFEST = ROOT / "plugins" / "1c" / "datasets" / "prepared" / "rag_manifest.json"
DEFAULT_INDEX = ROOT / "plugins" / "1c" / "datasets" / "prepared" / "rag_index.json"
DEFAULT_VECTOR_INDEX = ROOT / "plugins" / "1c" / "datasets" / "prepared" / "rag_vector_index.sqlite"
def run(command: list[str]) -> None:
print(" ".join(command))
result = subprocess.run(command, cwd=ROOT, text=True, check=False)
if result.returncode != 0:
raise SystemExit(result.returncode)
def main() -> int:
parser = argparse.ArgumentParser(description="Build the 1C RAG knowledge base.")
parser.add_argument("--metadata", type=Path, default=DEFAULT_METADATA)
parser.add_argument("--bsl-modules", type=Path, help="Optional BSL module snapshot JSON.")
parser.add_argument("--source-dir", type=Path, default=DEFAULT_SOURCE_DIR)
parser.add_argument("--corpus", type=Path, default=DEFAULT_CORPUS)
parser.add_argument("--manifest", type=Path, default=DEFAULT_MANIFEST)
parser.add_argument("--index", type=Path, default=DEFAULT_INDEX)
parser.add_argument("--vector-index", type=Path, default=DEFAULT_VECTOR_INDEX)
parser.add_argument("--skip-vector-index", action="store_true")
parser.add_argument(
"--include-example-bsl",
action="store_true",
help="Use the bundled BSL example when --bsl-modules is not provided.",
)
args = parser.parse_args()
metadata_output = args.source_dir / f"{args.metadata.stem}.metadata.md"
run([sys.executable, "scripts/validate_1c_metadata_snapshot.py", str(args.metadata)])
run(
[
sys.executable,
"scripts/convert_1c_metadata_to_rag.py",
"--input",
str(args.metadata),
"--output",
str(metadata_output),
]
)
bsl_modules = args.bsl_modules
if bsl_modules is None and args.include_example_bsl:
bsl_modules = DEFAULT_BSL_MODULES
if bsl_modules:
bsl_output = args.source_dir / f"{bsl_modules.stem}.bsl.md"
run([sys.executable, "scripts/validate_1c_bsl_modules.py", str(bsl_modules)])
run(
[
sys.executable,
"scripts/convert_1c_bsl_modules_to_rag.py",
"--input",
str(bsl_modules),
"--output",
str(bsl_output),
]
)
run([sys.executable, "scripts/validate_1c_rag_sources.py", "--source-dir", str(args.source_dir)])
run(
[
sys.executable,
"scripts/prepare_1c_rag_corpus.py",
"--source-dir",
str(args.source_dir),
"--output",
str(args.corpus),
"--manifest",
str(args.manifest),
]
)
run(
[
sys.executable,
"scripts/build_1c_rag_index.py",
"--corpus",
str(args.corpus),
"--output",
str(args.index),
]
)
if not args.skip_vector_index:
run(
[
sys.executable,
"scripts/build_1c_rag_vector_index.py",
"--corpus",
str(args.corpus),
"--output",
str(args.vector_index),
]
)
print(f"Built 1C knowledge base: {args.index}")
if not args.skip_vector_index:
print(f"Built 1C vector index: {args.vector_index}")
return 0
if __name__ == "__main__":
raise SystemExit(main())
@@ -0,0 +1,465 @@
#!/usr/bin/env python3
"""Build a read-projection metadata card from resolved XML object evidence."""
from __future__ import annotations
import argparse
import json
import sys
import xml.etree.ElementTree as ET
from pathlib import Path
from typing import Any
sys.path.insert(0, str(Path(__file__).resolve().parent))
from resolve_1c_object import load_json, physical_name, resolve_object # noqa: E402
REFERENCE_MARKERS = ("Ref.",)
TYPE_PRESENTATION_RU = {
"xs:string": "Строка",
"xs:decimal": "Число",
"xs:boolean": "Булево",
"xs:dateTime": "Дата",
"v8:UUID": "УникальныйИдентификатор",
"cfg:AnyRef": "ЛюбаяСсылка",
"cfg:AnyIBRef": "ЛюбаяСсылка",
}
REFERENCE_PRESENTATION_RU = {
"CatalogRef": "СправочникСсылка",
"DocumentRef": "ДокументСсылка",
"EnumRef": "ПеречислениеСсылка",
"ChartOfAccountsRef": "ПланСчетовСсылка",
"ChartOfCalculationTypesRef": "ПланВидовРасчетаСсылка",
"ChartOfCharacteristicTypesRef": "ПланВидовХарактеристикСсылка",
"BusinessProcessRef": "БизнесПроцессСсылка",
"TaskRef": "ЗадачаСсылка",
}
def local_name(tag: str) -> str:
return tag.rsplit("}", 1)[-1] if "}" in tag else tag
def direct_child(node: ET.Element, name: str) -> ET.Element | None:
return next((child for child in list(node) if local_name(child.tag) == name), None)
def text_child(node: ET.Element | None, name: str) -> str | None:
if node is None:
return None
child = direct_child(node, name)
if child is None or child.text is None:
return None
return child.text.strip()
def synonym(properties: ET.Element | None) -> str | None:
if properties is None:
return None
syn = direct_child(properties, "Synonym")
if syn is None:
return None
for node in syn.iter():
if local_name(node.tag) == "content" and node.text:
return node.text.strip()
return None
def value_types(properties: ET.Element | None) -> list[str]:
if properties is None:
return []
type_node = direct_child(properties, "Type")
if type_node is None:
return []
result = []
for node in type_node.iter():
if local_name(node.tag) == "Type" and node.text and ":" in node.text:
result.append(node.text.strip())
elif local_name(node.tag) == "TypeSet" and node.text and ":" in node.text:
result.append(node.text.strip())
return result
def type_presentation_ru(value: str) -> str:
if value in TYPE_PRESENTATION_RU:
return TYPE_PRESENTATION_RU[value]
if value.startswith("cfg:"):
payload = value[4:]
if "." in payload:
family, name = payload.split(".", 1)
prefix = REFERENCE_PRESENTATION_RU.get(family)
if prefix:
return f"{prefix}.{name}"
return f"{family}.{name}"
return value
def route_kind(role: str) -> str:
if role == "Fld":
return "field"
if role in {"VT", "LineNo"}:
return "structural"
return "table"
def storage_routes(index: dict[str, Any], guid: str | None) -> list[dict[str, Any]]:
if not guid:
return []
item = (index.get("objects") or {}).get(guid.lower()) or {}
result = []
for route in item.get("dbnames") or []:
copy = {
"guid": guid,
"storage_role": route.get("storage_role"),
"sql_number": route.get("sql_number"),
"source": route.get("source_file") or "DBNames",
"route_kind": route_kind(str(route.get("storage_role") or "")),
"physical_name_candidate": physical_name(route),
}
result.append(copy)
return result
def value_type_payload(types: list[str]) -> dict[str, Any] | None:
if not types:
return None
return {
"types": types,
"presentation": {
"default_language": "ru",
"ru": [type_presentation_ru(item) for item in types],
},
"qualifiers": {},
"is_composite": len(types) > 1,
}
def field_columns(routes: list[dict[str, Any]], types: list[str]) -> list[dict[str, Any]]:
fld = next((route for route in routes if route.get("storage_role") == "Fld" and route.get("sql_number") is not None), None)
if not fld:
return [{"status": "no_storage_route"}]
base = f"_Fld{fld['sql_number']}"
if len(types) > 1:
return [
{"column": f"{base}_TYPE", "reason": "composite value discriminator", "value_types": types},
{"column": f"{base}_S", "reason": "composite string value", "value_types": types},
{"column": f"{base}_N", "reason": "composite numeric value", "value_types": types},
{"column": f"{base}_L", "reason": "composite boolean value", "value_types": types},
{"column": f"{base}_T", "reason": "composite datetime value", "value_types": types},
{"column": f"{base}_RTRef", "reason": "composite reference type id", "value_types": types},
{"column": f"{base}_RRRef", "reason": "composite reference value", "value_types": types},
]
value_type = types[0] if types else None
if value_type in {"cfg:AnyRef", "cfg:AnyIBRef"}:
return [
{"column": f"{base}_TYPE", "reason": "any reference discriminator", "value_types": types},
{"column": f"{base}_RTRef", "reason": "any reference type id", "value_types": types},
{"column": f"{base}_RRRef", "reason": "any reference value", "value_types": types},
]
if value_type and any(marker in value_type for marker in REFERENCE_MARKERS):
return [{"column": f"{base}RRef", "reason": "single 1C reference type", "value_type": value_type}]
return [{"column": base, "reason": "single primitive value", "value_type": value_type}]
def extension_name_from_path(path: str | None) -> str | None:
parts = str(path or "").replace("/", "\\").split("\\")
lowered = [part.casefold() for part in parts]
if "расширения" in lowered:
index = lowered.index("расширения")
if index + 1 < len(parts):
return parts[index + 1]
if "extensions" in lowered:
index = lowered.index("extensions")
if index + 1 < len(parts):
return parts[index + 1]
return None
def metadata_item(
node: ET.Element,
*,
index: dict[str, Any],
category: str,
parent_category: str,
parent_name: str,
parent_uuid: str,
tabular_section_name: str | None = None,
tabular_section_uuid: str | None = None,
record_index: int,
) -> dict[str, Any]:
properties = direct_child(node, "Properties")
guid = (node.get("uuid") or "").lower()
name = text_child(properties, "Name")
types = value_types(properties)
routes = storage_routes(index, guid)
item = {
"category": category,
"name": name,
"synonym": synonym(properties),
"uuid": guid,
"value_type": value_type_payload(types),
"parent_category": parent_category,
"parent_name": parent_name,
"parent_uuid": parent_uuid,
"record_index": record_index,
"evidence": {"name": bool(name), "synonym": bool(synonym(properties)), "uuid": bool(guid)},
"storage_routes": routes,
"storage_route_count": len(routes),
"physical_columns": field_columns(routes, types) if category == "Attribute" else [{"status": "no_value_type"}],
}
object_belonging = text_child(properties, "ObjectBelonging")
extended_object = text_child(properties, "ExtendedConfigurationObject")
if object_belonging:
item["object_belonging"] = object_belonging
if extended_object:
item["extended_configuration_object"] = extended_object.lower()
if tabular_section_name:
item["tabular_section_name"] = tabular_section_name
item["tabular_section_uuid"] = tabular_section_uuid
return item
def object_node(root: ET.Element, kind: str) -> ET.Element:
for node in root.iter():
if local_name(node.tag) == kind:
return node
raise SystemExit(f"XML object node not found: {kind}")
def merge_attribute_overlay(base: list[dict[str, Any]], overlay: dict[str, Any]) -> None:
extended_uuid = overlay.get("extended_configuration_object")
target = None
if extended_uuid:
target = next((item for item in base if item.get("uuid") == extended_uuid), None)
if target is None and overlay.get("name"):
target = next((item for item in base if item.get("name") == overlay.get("name")), None)
if target is None:
base.append(overlay)
return
record = {
"source": overlay.get("source"),
"extension_name": overlay.get("extension_name"),
"path": overlay.get("source_path"),
"uuid": overlay.get("uuid"),
"object_belonging": overlay.get("object_belonging"),
"extended_configuration_object": overlay.get("extended_configuration_object"),
"value_type": overlay.get("value_type"),
"synonym": overlay.get("synonym"),
}
target.setdefault("extension_overrides", []).append(record)
target["effective_source"] = "base+extension"
if overlay.get("value_type"):
target["base_value_type"] = target.get("base_value_type") or target.get("value_type")
target["value_type"] = overlay["value_type"]
target["physical_columns"] = field_columns(target.get("storage_routes") or [], overlay["value_type"].get("types") or [])
if overlay.get("synonym"):
target["synonym"] = overlay["synonym"]
def apply_object_overlay(
*,
index: dict[str, Any],
overlay: dict[str, Any],
kind: str,
attributes: list[dict[str, Any]],
tabular_sections: list[dict[str, Any]],
tabular_section_attributes: list[dict[str, Any]],
) -> dict[str, int]:
path = Path(str(overlay.get("path") or ""))
if not path.is_file():
return {"missing": 1, "attributes_added": 0, "attributes_changed": 0}
root = ET.parse(path).getroot()
node = object_node(root, kind)
children = direct_child(node, "ChildObjects")
if children is None:
return {"missing": 0, "attributes_added": 0, "attributes_changed": 0}
added = 0
changed = 0
extension_name = extension_name_from_path(str(path))
for child in list(children):
if local_name(child.tag) != "Attribute":
continue
before = len(attributes)
item = metadata_item(
child,
index=index,
category="Attribute",
parent_category=kind,
parent_name=str(overlay.get("name") or ""),
parent_uuid=str(overlay.get("guid") or ""),
record_index=len(attributes),
)
item["source"] = "extension"
item["extension_name"] = extension_name
item["source_path"] = str(path)
merge_attribute_overlay(attributes, item)
if len(attributes) > before:
added += 1
else:
changed += 1
return {"missing": 0, "attributes_added": added, "attributes_changed": changed}
def build_metadata(index: dict[str, Any], *, kind: str, name: str) -> dict[str, Any]:
resolution = resolve_object(index, kind=kind, name=name, limit=50)
canonical = resolution.get("canonical")
if not canonical:
raise SystemExit(f"Object not found: {kind}.{name}")
xml_path = Path(str(canonical.get("path") or ""))
if not xml_path.is_file():
raise SystemExit(f"Object XML file not found: {xml_path}")
root = ET.parse(xml_path).getroot()
node = object_node(root, str(canonical["kind"]))
properties = direct_child(node, "Properties")
object_name = text_child(properties, "Name") or str(canonical["name"])
object_uuid = str(canonical["guid"]).lower()
main_routes = storage_routes(index, object_uuid)
main_table = next((physical_name(route) for route in main_routes if route.get("storage_role") == canonical["kind"]), None)
if not main_table:
main_table = next((route.get("physical_name_candidate") for route in main_routes if route.get("route_kind") == "table"), None)
attributes = []
tabular_sections = []
tabular_section_attributes = []
children = direct_child(node, "ChildObjects")
if children is not None:
attr_index = 0
ts_index = 0
for child in list(children):
child_kind = local_name(child.tag)
if child_kind == "Attribute":
attributes.append(
{
**metadata_item(
child,
index=index,
category="Attribute",
parent_category=str(canonical["kind"]),
parent_name=object_name,
parent_uuid=object_uuid,
record_index=attr_index,
),
"source": "base",
}
)
attr_index += 1
elif child_kind == "TabularSection":
section = metadata_item(
child,
index=index,
category="TabularSection",
parent_category=str(canonical["kind"]),
parent_name=object_name,
parent_uuid=object_uuid,
record_index=ts_index,
)
vt_route = next((route for route in section["storage_routes"] if route.get("storage_role") == "VT"), None)
line_numbers = [route.get("sql_number") for route in section["storage_routes"] if route.get("storage_role") == "LineNo"]
if main_table and vt_route and vt_route.get("sql_number") is not None:
section["physical_tables"] = [
{
"table": f"{main_table}_VT{vt_route['sql_number']}",
"reason": "tabular section VT route under object table",
"vt_sql_number": vt_route["sql_number"],
"line_no_sql_numbers": line_numbers,
}
]
tabular_sections.append(section)
section_children = direct_child(child, "ChildObjects")
if section_children is not None:
for record_index, section_child in enumerate([item for item in list(section_children) if local_name(item.tag) == "Attribute"]):
attr = metadata_item(
section_child,
index=index,
category="Attribute",
parent_category="TabularSection",
parent_name=section["name"],
parent_uuid=section["uuid"],
tabular_section_name=section["name"],
tabular_section_uuid=section["uuid"],
record_index=record_index,
)
attr["parent_physical_tables"] = section.get("physical_tables") or []
attr["source"] = "base"
tabular_section_attributes.append(attr)
ts_index += 1
overlay_stats = []
for overlay in resolution.get("extension_overlays") or []:
stats = apply_object_overlay(
index=index,
overlay=overlay,
kind=str(canonical["kind"]),
attributes=attributes,
tabular_sections=tabular_sections,
tabular_section_attributes=tabular_section_attributes,
)
overlay_stats.append(
{
"extension_name": extension_name_from_path(overlay.get("path")),
"path": overlay.get("path"),
**stats,
}
)
return {
"schema": "onec_structured_metadata_from_resolved_xml.v1",
"kind": canonical["kind"],
"xml_file": str(xml_path),
"identity": {
"guid": object_uuid,
"name": object_name,
"synonyms": {"ru": synonym(properties)} if synonym(properties) else {},
},
"resolution": {"schema": resolution.get("schema"), "canonical": canonical, "summary": resolution.get("summary")},
"effective_metadata": {
"base_path": str(xml_path),
"extension_overlays_applied": overlay_stats,
},
"attributes": attributes,
"tabular_sections": tabular_sections,
"tabular_section_attributes": tabular_section_attributes,
"dimensions": [],
"resources": [],
"forms": [],
"templates": [],
"commands": [],
"addressing_attributes": [],
"accounting_flags": [],
"columns": [],
"enum_values": [],
"object_storage_routes": main_routes,
"storage_route_summary": {
"metadata_items_with_routes": sum(1 for item in attributes + tabular_sections + tabular_section_attributes if item.get("storage_routes")),
"object_routes": len(main_routes),
},
"counts": {
"attributes": len(attributes),
"tabular_sections": len(tabular_sections),
"tabular_section_attributes": len(tabular_section_attributes),
},
}
def main() -> int:
parser = argparse.ArgumentParser(description="Build metadata card from resolved 1C object XML.")
parser.add_argument("--index", type=Path, required=True)
parser.add_argument("--kind", required=True)
parser.add_argument("--name", required=True)
parser.add_argument("--output", type=Path, required=True)
args = parser.parse_args()
result = build_metadata(load_json(args.index), kind=args.kind, name=args.name)
args.output.parent.mkdir(parents=True, exist_ok=True)
args.output.write_text(json.dumps(result, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
print(json.dumps({"output": str(args.output), "counts": result["counts"]}, ensure_ascii=False))
return 0
if __name__ == "__main__":
raise SystemExit(main())
+156
View File
@@ -0,0 +1,156 @@
from __future__ import annotations
import argparse
import json
from datetime import UTC, datetime
from pathlib import Path
from typing import Any
def read_json(path: Path) -> dict[str, Any]:
return json.loads(path.read_text(encoding="utf-8"))
def rule_status(rule: dict[str, Any]) -> str:
confidence = str(rule.get("confidence") or "none")
evidence = rule.get("evidence") if isinstance(rule.get("evidence"), dict) else {}
ok = evidence.get("ok")
total = evidence.get("total")
if confidence == "high" and isinstance(ok, int) and isinstance(total, int) and total > 0 and ok == total:
return "verified_read"
if confidence == "high" and int(evidence.get("distinct_rectangular_samples") or 0) > 0 and int(evidence.get("samples") or 0) > 0:
return "verified_read"
if confidence in {"medium", "high"}:
return "candidate_read"
return "needs_more_evidence"
def write_status(status: str) -> str:
if status == "verified_read":
return "blocked_until_roundtrip"
return "blocked_until_verified_read"
def merge_named_range_analysis_rules(discovery: dict[str, Any], named_range_analysis: dict[str, Any] | None) -> dict[str, Any]:
if not named_range_analysis:
return discovery
merged_rules = [rule for rule in discovery.get("rules") or [] if isinstance(rule, dict)]
by_target = {str(rule.get("target") or ""): index for index, rule in enumerate(merged_rules)}
for rule in named_range_analysis.get("rules") or []:
if not isinstance(rule, dict) or not str(rule.get("target") or "").startswith("moxel.named_range."):
continue
target = str(rule.get("target") or "")
promoted = {
"id": rule.get("id") or target,
"target": target,
"expression": rule.get("expression"),
"raw_scalar_indexes": rule.get("raw_scalar_indexes"),
"confidence": rule.get("confidence") or "none",
"evidence": rule.get("evidence") or {},
"source": "named_range_analysis",
}
if target in by_target:
existing = merged_rules[by_target[target]]
confidence_order = {"none": 0, "low": 1, "medium": 2, "high": 3}
if confidence_order.get(str(promoted.get("confidence")), 0) >= confidence_order.get(str(existing.get("confidence")), 0):
merged_rules[by_target[target]] = {**existing, **promoted}
else:
by_target[target] = len(merged_rules)
merged_rules.append(promoted)
return {**discovery, "rules": merged_rules}
def build_registry(discovery: dict[str, Any], sources: list[str], named_range_analysis: dict[str, Any] | None = None) -> dict[str, Any]:
discovery = merge_named_range_analysis_rules(discovery, named_range_analysis)
registry_rules = []
for index, rule in enumerate(discovery.get("rules") or [], start=1):
if not isinstance(rule, dict):
continue
status = rule_status(rule)
registry_rules.append(
{
"id": rule.get("id") or f"moxel_rule_{index}",
"target": rule.get("target"),
"expression": rule.get("expression"),
"raw_scalar_indexes": rule.get("raw_scalar_indexes"),
"confidence": rule.get("confidence") or "none",
"read_status": status,
"write_status": write_status(status),
"evidence": rule.get("evidence") or {},
"source_rule": rule,
}
)
return {
"schema": "codex_1c_moxel_schema_registry.v1",
"generated_at": datetime.now(UTC).isoformat(timespec="seconds").replace("+00:00", "Z"),
"sources": sources,
"policy": {
"read_use": "Only verified_read rules may be used as decoder behavior without additional diagnostics.",
"write_use": "All MOXCEL write rules are blocked until a disposable-base round-trip proves exact behavior.",
},
"rules": registry_rules,
"counts": {
"rules": len(registry_rules),
"verified_read": sum(1 for rule in registry_rules if rule.get("read_status") == "verified_read"),
"candidate_read": sum(1 for rule in registry_rules if rule.get("read_status") == "candidate_read"),
"write_enabled": sum(1 for rule in registry_rules if rule.get("write_status") == "verified_roundtrip"),
},
}
def render_markdown(registry: dict[str, Any]) -> str:
lines = ["# 1C MOXCEL Schema Registry", ""]
counts = registry.get("counts") or {}
lines.append(f"- Rules: `{counts.get('rules')}`")
lines.append(f"- Verified read: `{counts.get('verified_read')}`")
lines.append(f"- Candidate read: `{counts.get('candidate_read')}`")
lines.append(f"- Write enabled: `{counts.get('write_enabled')}`")
lines.append("")
lines.append("| Rule | Target | Read | Write | Confidence |")
lines.append("| --- | --- | --- | --- | --- |")
for rule in registry.get("rules") or []:
lines.append(
f"| `{rule.get('id')}` | `{rule.get('target')}` | `{rule.get('read_status')}` | "
f"`{rule.get('write_status')}` | `{rule.get('confidence')}` |"
)
lines.append("")
return "\n".join(lines)
def main() -> int:
parser = argparse.ArgumentParser(description="Build the stable 1C MOXCEL schema registry from discovery reports.")
parser.add_argument("--discovery", action="append", required=True, help="Discovery JSON. Repeatable; rules are merged in order.")
parser.add_argument("--named-range-analysis", help="Optional named range rule analysis JSON.")
parser.add_argument("--output-json", default="plugins/1c/metadata/moxel-schema-registry.json")
parser.add_argument("--output-markdown", default="reports/1c-template-baselines/moxel-schema-registry.md")
args = parser.parse_args()
discoveries = [read_json(Path(path)) for path in args.discovery]
merged = {"rules": []}
seen: set[tuple[str, str]] = set()
for discovery in discoveries:
for rule in discovery.get("rules") or []:
if not isinstance(rule, dict):
continue
key = (str(rule.get("id") or ""), str(rule.get("target") or ""))
if key in seen:
continue
seen.add(key)
merged["rules"].append(rule)
named_range_analysis = read_json(Path(args.named_range_analysis)) if args.named_range_analysis else None
sources = list(args.discovery)
if args.named_range_analysis:
sources.append(args.named_range_analysis)
registry = build_registry(merged, sources, named_range_analysis)
json_path = Path(args.output_json)
md_path = Path(args.output_markdown)
json_path.parent.mkdir(parents=True, exist_ok=True)
md_path.parent.mkdir(parents=True, exist_ok=True)
json_path.write_text(json.dumps(registry, ensure_ascii=False, indent=2), encoding="utf-8")
md_path.write_text(render_markdown(registry), encoding="utf-8")
print(json.dumps({"status": "ok", "json": str(json_path), "markdown": str(md_path), "counts": registry["counts"]}, ensure_ascii=False, indent=2))
return 0
if __name__ == "__main__":
raise SystemExit(main())
+27
View File
@@ -0,0 +1,27 @@
from __future__ import annotations
import argparse
from pathlib import Path
from common import build_lexical_index, read_jsonl, write_json
ROOT = Path(__file__).resolve().parents[1]
DEFAULT_CORPUS = ROOT / "plugins" / "1c" / "datasets" / "prepared" / "rag_corpus.jsonl"
DEFAULT_INDEX = ROOT / "plugins" / "1c" / "datasets" / "prepared" / "rag_index.json"
def main() -> int:
parser = argparse.ArgumentParser(description="Build a simple lexical RAG index for the 1C corpus.")
parser.add_argument("--corpus", type=Path, default=DEFAULT_CORPUS)
parser.add_argument("--output", type=Path, default=DEFAULT_INDEX)
args = parser.parse_args()
records = read_jsonl(args.corpus)
index = build_lexical_index(records)
write_json(args.output, index)
print(f"Wrote index with {index['doc_count']} document chunk(s) to {args.output}")
return 0
if __name__ == "__main__":
raise SystemExit(main())
+207
View File
@@ -0,0 +1,207 @@
from __future__ import annotations
import argparse
import json
import sqlite3
from datetime import datetime, timezone
from pathlib import Path
from common import corpus_content_hash, pack_float_vector, read_jsonl
from rag_embedding_providers import LOCAL_HASHING_MODEL, LOCAL_HASHING_PROVIDER, embed_texts, provider_metadata
ROOT = Path(__file__).resolve().parents[1]
DEFAULT_CORPUS = ROOT / "plugins" / "1c" / "datasets" / "prepared" / "rag_corpus.jsonl"
DEFAULT_OUTPUT = ROOT / "plugins" / "1c" / "datasets" / "prepared" / "rag_vector_index.sqlite"
SCHEMA_VERSION = 1
DEFAULT_EMBEDDING_MODEL = LOCAL_HASHING_MODEL
def connect_index(path: Path) -> sqlite3.Connection:
path.parent.mkdir(parents=True, exist_ok=True)
conn = sqlite3.connect(path)
conn.execute("PRAGMA journal_mode=WAL")
conn.execute("PRAGMA synchronous=NORMAL")
return conn
def reset_schema(conn: sqlite3.Connection) -> None:
conn.executescript(
"""
DROP TABLE IF EXISTS vector_documents;
DROP TABLE IF EXISTS vector_meta;
CREATE TABLE vector_meta (
key TEXT PRIMARY KEY,
value TEXT NOT NULL
);
CREATE TABLE vector_documents (
id TEXT PRIMARY KEY,
document_id TEXT,
source_path TEXT,
source_type TEXT,
title TEXT,
chunk_index INTEGER,
content TEXT NOT NULL,
metadata_json TEXT NOT NULL,
vector BLOB NOT NULL
);
CREATE INDEX idx_vector_documents_source_type ON vector_documents(source_type);
CREATE INDEX idx_vector_documents_source_path ON vector_documents(source_path);
"""
)
def write_meta(conn: sqlite3.Connection, metadata: dict[str, object]) -> None:
conn.executemany(
"INSERT INTO vector_meta(key, value) VALUES(?, ?)",
[(key, json.dumps(value, ensure_ascii=False, sort_keys=True)) for key, value in metadata.items()],
)
def document_embedding_text(record: dict) -> str:
title = str(record.get("title") or "").strip()
content = str(record.get("content") or "").strip()
metadata = record.get("metadata") if isinstance(record.get("metadata"), dict) else {}
headings = metadata.get("headings") if isinstance(metadata.get("headings"), list) else []
heading_text = "\n".join(str(item) for item in headings if str(item).strip())
return "\n\n".join(part for part in (title, heading_text, content) if part)
def batched(items: list[dict], size: int) -> list[list[dict]]:
return [items[index : index + size] for index in range(0, len(items), size)]
def build_vector_index(
corpus_path: Path,
output_path: Path,
*,
dimensions: int,
embedding_model: str,
embedding_provider: str = LOCAL_HASHING_PROVIDER,
embedding_base_url: str = "",
embedding_api_key_env: str = "OPENAI_API_KEY",
batch_size: int = 16,
) -> dict:
records = read_jsonl(corpus_path)
corpus_hash = corpus_content_hash(records)
if not records:
inferred_dimensions = dimensions
else:
sample_vector = embed_texts(
[document_embedding_text(records[0])],
provider=embedding_provider,
model=embedding_model,
dimensions=dimensions,
base_url=embedding_base_url,
api_key_env=embedding_api_key_env,
)[0]
inferred_dimensions = len(sample_vector)
conn = connect_index(output_path)
try:
with conn:
reset_schema(conn)
embedding_meta = provider_metadata(
provider=embedding_provider,
model=embedding_model,
dimensions=inferred_dimensions,
base_url=embedding_base_url,
)
write_meta(
conn,
{
"schema": "onec_rag_vector_index.v1",
"schema_version": SCHEMA_VERSION,
"type": "sqlite-vector-scan",
**embedding_meta,
"corpus_path": str(corpus_path),
"corpus_hash": corpus_hash,
"doc_count": len(records),
"built_at": datetime.now(timezone.utc).isoformat(),
},
)
rows = []
for batch in batched(records, max(int(batch_size or 1), 1)):
texts = [document_embedding_text(record) for record in batch]
vectors = embed_texts(
texts,
provider=embedding_provider,
model=embedding_model,
dimensions=inferred_dimensions,
base_url=embedding_base_url,
api_key_env=embedding_api_key_env,
)
for record, vector in zip(batch, vectors):
if len(vector) != inferred_dimensions:
raise ValueError(f"Embedding dimensions changed within the build: {len(vector)} != {inferred_dimensions}")
metadata = record.get("metadata") if isinstance(record.get("metadata"), dict) else {}
rows.append(
(
str(record.get("id") or ""),
str(record.get("document_id") or ""),
str(record.get("source_path") or ""),
str(metadata.get("source_type") or ""),
str(record.get("title") or ""),
int(record.get("chunk_index") or 0),
str(record.get("content") or ""),
json.dumps(metadata, ensure_ascii=False, sort_keys=True),
pack_float_vector(vector),
)
)
conn.executemany(
"""
INSERT INTO vector_documents(
id, document_id, source_path, source_type, title, chunk_index,
content, metadata_json, vector
)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
""",
rows,
)
finally:
conn.close()
return {
"status": "ok",
"index": str(output_path),
"doc_count": len(records),
"corpus_hash": corpus_hash,
"embedding_provider": provider_metadata(provider=embedding_provider, model=embedding_model, dimensions=inferred_dimensions, base_url=embedding_base_url)["embedding_provider"],
"embedding_model": embedding_model,
"embedding_dimensions": inferred_dimensions,
}
def main() -> int:
parser = argparse.ArgumentParser(description="Build a local SQLite vector index for the 1C RAG corpus.")
parser.add_argument("--corpus", type=Path, default=DEFAULT_CORPUS)
parser.add_argument("--output", type=Path, default=DEFAULT_OUTPUT)
parser.add_argument("--dimensions", type=int, default=384)
parser.add_argument("--embedding-provider", default=LOCAL_HASHING_PROVIDER, choices=[LOCAL_HASHING_PROVIDER, "openai-compatible"])
parser.add_argument("--embedding-model", default=DEFAULT_EMBEDDING_MODEL)
parser.add_argument("--embedding-base-url", default="")
parser.add_argument("--embedding-api-key-env", default="OPENAI_API_KEY")
parser.add_argument("--batch-size", type=int, default=16)
parser.add_argument("--json", action="store_true")
args = parser.parse_args()
result = build_vector_index(
args.corpus,
args.output,
dimensions=args.dimensions,
embedding_model=args.embedding_model,
embedding_provider=args.embedding_provider,
embedding_base_url=args.embedding_base_url,
embedding_api_key_env=args.embedding_api_key_env,
batch_size=args.batch_size,
)
if args.json:
print(json.dumps(result, ensure_ascii=False, indent=2))
else:
print(f"Wrote vector index with {result['doc_count']} document chunk(s) to {args.output}")
return 0
if __name__ == "__main__":
raise SystemExit(main())
@@ -0,0 +1,325 @@
param(
[string]$Server = $env:ONEC_SQL_SERVER,
[string]$Database = $env:ONEC_SQL_DATABASE,
[string]$User = $env:ONEC_SQL_USER,
[string]$Password = $env:ONEC_SQL_PASSWORD,
[string]$OutputDir = "reports\1c-sql\saved-state-object-report",
[string]$BaseMetadataDir = "reports\1c-sql\upo\structured-metadata-all-kinds",
[string]$ExtensionGuidIndex = "reports\1c-sql\upo\xml-guid-index-extensions.json",
[string]$ExtensionManifestSummary = "reports\1c-sql\upo\extension-manifest-xml-part-summary.json",
[string]$ConfigCASAllDir = "reports\1c-sql\upo\ConfigCAS-all",
[string]$Python = "python",
[string]$MarkdownOutput = "",
[int]$MarkdownMaxDiffLines = 80,
[switch]$SkipMarkdown
)
Set-StrictMode -Version Latest
$ErrorActionPreference = "Stop"
if (-not $Server) { throw "Server is required. Use -Server or ONEC_SQL_SERVER." }
if (-not $Database) { throw "Database is required. Use -Database or ONEC_SQL_DATABASE." }
if (-not $User) { throw "User is required. Use -User or ONEC_SQL_USER." }
if (-not $Password) { throw "Password is required. Use -Password or ONEC_SQL_PASSWORD." }
$repoRoot = (Resolve-Path ".").ProviderPath
$resolvedOutput = [System.IO.Path]::GetFullPath($OutputDir)
New-Item -ItemType Directory -Force -Path $resolvedOutput | Out-Null
$comparisonPath = Join-Path $resolvedOutput "saved-state-object-comparison.json"
$detailPath = Join-Path $resolvedOutput "saved-state-object-details.json"
$checkPath = Join-Path $resolvedOutput "saved-state-object-report-check.json"
$configSaveDir = Join-Path $resolvedOutput "ConfigSave"
$configCASSaveDir = Join-Path $resolvedOutput "ConfigCASSave"
$activeConfigDir = Join-Path $resolvedOutput "ActiveConfig"
$activeConfigCASDir = Join-Path $resolvedOutput "ActiveConfigCAS"
foreach ($dir in @($configSaveDir, $configCASSaveDir, $activeConfigDir, $activeConfigCASDir)) {
New-Item -ItemType Directory -Force -Path $dir | Out-Null
}
function Invoke-Step {
param(
[string]$Name,
[scriptblock]$Script
)
$started = Get-Date
& $Script
[pscustomobject]@{
name = $Name
started_at = $started.ToString("o")
finished_at = (Get-Date).ToString("o")
passed = $true
}
}
function Read-JsonFile {
param([string]$Path)
return Get-Content -LiteralPath $Path -Raw -Encoding UTF8 | ConvertFrom-Json
}
function Export-TableFiles {
param(
[string]$Table,
[string]$TargetDir,
[string[]]$FileName
)
$args = @(
"-NoProfile", "-ExecutionPolicy", "Bypass",
"-File", "scripts\export_1c_sql_files.ps1",
"-Server", $Server,
"-Database", $Database,
"-User", $User,
"-Password", $Password,
"-Table", $Table,
"-OutputPath", $TargetDir
)
[object[]]$normalizedFileName = @($FileName)
if (($normalizedFileName | Measure-Object).Count -gt 0) {
$args += "-FileName"
$args += $normalizedFileName
}
$json = & powershell @args
if ($LASTEXITCODE -ne 0) {
throw "Export failed for $Table."
}
return ($json | Out-String | ConvertFrom-Json)
}
function Get-JsonProperty {
param(
[object]$Object,
[string]$Name,
[object]$Default = $null
)
if ($null -eq $Object) { return $Default }
if ($Object.PSObject.Properties.Name -contains $Name) {
return $Object.$Name
}
return $Default
}
function Join-LimitedValues {
param(
[object[]]$Values,
[int]$Limit = 12
)
$items = @($Values | Where-Object { $null -ne $_ -and [string]$_ -ne "" } | Select-Object -First $Limit)
return ,$items
}
function New-AgentSummary {
param(
[object]$Comparison,
[object]$Detail
)
$detailByName = @{}
foreach ($item in @($Detail.object_details)) {
$fullName = [string](Get-JsonProperty -Object $item -Name "full_name" -Default "")
if ($fullName) { $detailByName[$fullName] = $item }
}
$objects = @()
foreach ($change in @($Comparison.object_changes)) {
$fullName = [string](Get-JsonProperty -Object $change -Name "full_name" -Default (Get-JsonProperty -Object $change -Name "name" -Default ""))
$item = $null
if ($fullName -and $detailByName.ContainsKey($fullName)) {
$item = $detailByName[$fullName]
}
$parts = @()
$addedTerms = @()
$removedTerms = @()
$textDiffParts = 0
$activeMissingParts = 0
foreach ($part in @((Get-JsonProperty -Object $item -Name "details" -Default @()))) {
$payload = Get-JsonProperty -Object $part -Name "payload" -Default $null
$diff = Get-JsonProperty -Object $payload -Name "text_diff" -Default $null
$semanticHints = Get-JsonProperty -Object $payload -Name "semantic_hints" -Default $null
$activeExists = Get-JsonProperty -Object $part -Name "active_exists" -Default $null
if ($activeExists -eq $false) { $activeMissingParts += 1 }
if ($null -ne $diff) {
$textDiffParts += 1
$addedTerms += @(Get-JsonProperty -Object $semanticHints -Name "added_terms" -Default @())
$removedTerms += @(Get-JsonProperty -Object $semanticHints -Name "removed_terms" -Default @())
}
$parts += [pscustomobject]@{
file_name = Get-JsonProperty -Object $part -Name "file_name" -Default $null
payload_role = Get-JsonProperty -Object $part -Name "payload_role" -Default $null
saved_table = Get-JsonProperty -Object $part -Name "saved_table" -Default $null
active_table = Get-JsonProperty -Object $part -Name "active_table" -Default $null
active_exists = $activeExists
text_comparable = Get-JsonProperty -Object $payload -Name "text_comparable" -Default $null
summary = Get-JsonProperty -Object $payload -Name "summary" -Default $null
delta_chars = Get-JsonProperty -Object $diff -Name "delta_chars" -Default $null
}
}
$objects += [pscustomobject]@{
full_name = $fullName
layer = Get-JsonProperty -Object $change -Name "layer" -Default $null
extension = Get-JsonProperty -Object $change -Name "extension" -Default $null
kind = Get-JsonProperty -Object $change -Name "kind" -Default $null
kind_ru = Get-JsonProperty -Object $change -Name "kind_ru" -Default $null
name = Get-JsonProperty -Object $change -Name "name" -Default $null
synonym = Get-JsonProperty -Object $change -Name "synonym" -Default $null
change_state = Get-JsonProperty -Object $change -Name "change_state" -Default $null
parts_count = ($parts | Measure-Object).Count
text_diff_parts = $textDiffParts
active_missing_parts = $activeMissingParts
added_terms = Join-LimitedValues -Values ($addedTerms | Sort-Object -Unique) -Limit 12
removed_terms = Join-LimitedValues -Values ($removedTerms | Sort-Object -Unique) -Limit 12
parts = $parts
}
}
$systemChanges = @()
foreach ($change in @($Comparison.system_changes)) {
$systemChanges += [pscustomobject]@{
name = Get-JsonProperty -Object $change -Name "name" -Default $null
layer = Get-JsonProperty -Object $change -Name "layer" -Default $null
extension = Get-JsonProperty -Object $change -Name "extension" -Default $null
}
}
return [pscustomobject]@{
purpose = "Compact agent-facing summary of saved-but-not-applied changes in 1C terms."
default_next_action = "Inspect object details before generating or applying any code changes."
object_changes = $objects
system_changes = $systemChanges
object_names = @($objects | ForEach-Object { $_.full_name })
system_change_names = @($systemChanges | ForEach-Object { $_.name })
}
}
$steps = @()
$steps += Invoke-Step "compare_saved_state_objects" {
& powershell -NoProfile -ExecutionPolicy Bypass -File scripts\compare_1c_saved_state_objects.ps1 `
-Server $Server `
-Database $Database `
-User $User `
-Password $Password `
-BaseMetadataDir $BaseMetadataDir `
-ExtensionGuidIndex $ExtensionGuidIndex `
-Output $comparisonPath | Out-Null
if ($LASTEXITCODE -ne 0) { throw "Saved-state object comparison failed." }
}
$comparison = Read-JsonFile $comparisonPath
$steps += Invoke-Step "export_config_save" {
$null = Export-TableFiles -Table "ConfigSave" -TargetDir $configSaveDir
}
$steps += Invoke-Step "export_config_cas_save" {
$null = Export-TableFiles -Table "ConfigCASSave" -TargetDir $configCASSaveDir
}
$activeConfigFiles = @(
$comparison.object_changes |
ForEach-Object { $_.storage } |
Where-Object { $_.active_table -eq "Config" } |
ForEach-Object { [string]$_.file_name } |
Sort-Object -Unique
)
$activeConfigCASFiles = @(
$comparison.object_changes |
ForEach-Object { $_.storage } |
Where-Object { $_.active_table -eq "ConfigCAS" -and $_.active_exists } |
ForEach-Object { [string]$_.file_name } |
Sort-Object -Unique
)
$steps += Invoke-Step "export_active_config_payloads" {
if (($activeConfigFiles | Measure-Object).Count -gt 0) {
$null = Export-TableFiles -Table "Config" -TargetDir $activeConfigDir -FileName $activeConfigFiles
}
}
$steps += Invoke-Step "export_active_config_cas_payloads" {
if (($activeConfigCASFiles | Measure-Object).Count -gt 0) {
$null = Export-TableFiles -Table "ConfigCAS" -TargetDir $activeConfigCASDir -FileName $activeConfigCASFiles
}
}
$steps += Invoke-Step "analyze_saved_state_object_details" {
& $Python scripts\analyze_1c_saved_state_object_details.py `
--comparison $comparisonPath `
--config-save-dir $configSaveDir `
--config-dir $activeConfigDir `
--config-cas-save-dir $configCASSaveDir `
--config-cas-dir $activeConfigCASDir `
--extension-manifest-summary $ExtensionManifestSummary `
--config-cas-all-dir $ConfigCASAllDir `
--output $detailPath | Out-Null
if ($LASTEXITCODE -ne 0) { throw "Saved-state detail analysis failed." }
}
$detail = Read-JsonFile $detailPath
$agentSummary = New-AgentSummary -Comparison $comparison -Detail $detail
$reportPath = Join-Path $resolvedOutput "saved-state-object-report.json"
if (-not $MarkdownOutput) {
$MarkdownOutput = Join-Path $resolvedOutput "saved-state-object-report.md"
} else {
$MarkdownOutput = [System.IO.Path]::GetFullPath($MarkdownOutput)
}
$result = [pscustomobject]@{
schema = "onec_saved_state_object_report.v1"
server = $Server
database = $Database
report = $reportPath
markdown = if ($SkipMarkdown) { $null } else { $MarkdownOutput }
check = $checkPath
output_dir = $resolvedOutput
comparison = $comparisonPath
detail = $detailPath
agent_summary = $agentSummary
payload_dirs = [pscustomobject]@{
config_save = $configSaveDir
config_cas_save = $configCASSaveDir
active_config = $activeConfigDir
active_config_cas = $activeConfigCASDir
active_config_cas_all = $ConfigCASAllDir
}
counts = [pscustomobject]@{
object_changes = $comparison.counts.object_changes
system_changes = $comparison.counts.system_changes
detail_objects = $detail.counts.objects
detail_parts = $detail.counts.details
}
steps = $steps
safety = [pscustomobject]@{
read_only = $true
sql_write_performed = $false
public_terms_are_1c_objects = $true
secrets_in_report = $false
}
}
$result | ConvertTo-Json -Depth 12 | Set-Content -LiteralPath $reportPath -Encoding UTF8
if (-not $SkipMarkdown) {
$steps += Invoke-Step "render_saved_state_markdown" {
& $Python scripts\render_1c_saved_state_object_report_markdown.py `
--report $reportPath `
--output $MarkdownOutput `
--max-diff-lines $MarkdownMaxDiffLines | Out-Null
if ($LASTEXITCODE -ne 0) { throw "Saved-state Markdown rendering failed." }
}
$result.steps = $steps
$result | ConvertTo-Json -Depth 12 | Set-Content -LiteralPath $reportPath -Encoding UTF8
}
$steps += Invoke-Step "check_saved_state_report" {
& $Python scripts\check_1c_saved_state_object_report.py `
--report $reportPath `
--output $checkPath | Out-Null
if ($LASTEXITCODE -ne 0) { throw "Saved-state report check failed." }
}
$result.steps = $steps
$result | ConvertTo-Json -Depth 12 | Set-Content -LiteralPath $reportPath -Encoding UTF8
$result | ConvertTo-Json -Depth 12
+391
View File
@@ -0,0 +1,391 @@
#!/usr/bin/env python3
"""Build an agent-friendly 1C SQL read view from raw and resolved reports."""
from __future__ import annotations
import argparse
import json
import re
from collections import defaultdict
from pathlib import Path
from typing import Any
def load_json(path: Path) -> dict[str, Any]:
return json.loads(path.read_text(encoding="utf-8-sig"))
def write_json(path: Path, data: dict[str, Any]) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(json.dumps(data, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
def scope_key(scope: str, row_index: int, cell_key: str) -> tuple[str, int, str]:
return (scope, row_index, cell_key)
def part_scope(part_name: str | None) -> str:
return f"table_part:{part_name or ''}"
def best_alternate(alternates: list[dict[str, Any]]) -> dict[str, Any] | None:
if not alternates:
return None
for alternate in alternates:
route = alternate.get("route") or {}
if alternate.get("found") and route.get("base_table"):
return alternate
for alternate in alternates:
if alternate.get("found") and str(alternate.get("table") or "").endswith("X1"):
return alternate
for alternate in alternates:
if alternate.get("found"):
return alternate
return alternates[0]
def presentation(values: dict[str, Any] | None) -> str | None:
if not values:
return None
for key in ("_Description", "_Code", "_Number", "_EnumOrder"):
value = values.get(key)
if value is not None and value != "":
return str(value)
return None
BOOLEAN_STANDARD_PATHS = {"standard._Marked", "standard._Posted", "standard._Active"}
def normalize_1c_sql_date(value: Any) -> Any:
if not isinstance(value, str):
return value
match = re.match(r"^(\d{4})(-\d{2}-\d{2}T.*)$", value)
if not match:
return value
year = int(match.group(1))
if year < 3000:
return value
return f"{year - 2000:04d}{match.group(2)}"
def field_has_boolean_type(field: dict[str, Any]) -> bool:
if field.get("metadata_path") in BOOLEAN_STANDARD_PATHS:
return True
for column in (field.get("columns") or {}).values():
value_type = column.get("value_type")
values = value_type if isinstance(value_type, list) else [value_type]
if any(item == "xs:boolean" for item in values):
return True
return False
def normalize_display_value(value: Any, field: dict[str, Any]) -> Any:
if isinstance(value, dict) and value.get("kind") == "binary" and value.get("length") == 1 and field_has_boolean_type(field):
hex_value = str(value.get("hex") or "").lower()
if hex_value == "00":
return False
if hex_value == "01":
return True
return normalize_1c_sql_date(value)
def enum_order(values: dict[str, Any] | None) -> int | None:
if not values:
return None
value = values.get("_EnumOrder")
if value is None:
return None
try:
return int(value)
except (TypeError, ValueError):
return None
def enum_presentation(item: dict[str, Any], enum_map: dict[str, Any] | None) -> dict[str, Any] | None:
if not enum_map:
return None
target = item.get("target") or {}
if target.get("kind") != "Enum":
return None
name = target.get("name")
order = enum_order(item.get("values"))
if name is None or order is None:
return None
enum = (enum_map.get("enums") or {}).get(name)
if not enum:
return None
value = (enum.get("by_order") or {}).get(str(order))
if not value:
return None
return value
def resolved_payload(item: dict[str, Any], *, mode: str, enum_map: dict[str, Any] | None = None) -> dict[str, Any]:
enum_value = enum_presentation(item, enum_map)
payload = {
"mode": mode,
"found": item.get("found"),
"target": item.get("target"),
"values": item.get("values"),
"presentation": (enum_value or {}).get("synonym") or (enum_value or {}).get("name") or presentation(item.get("values")),
"reason": item.get("reason"),
}
if enum_value:
payload["enum_value"] = enum_value
alternates = item.get("alternate_hits") or []
if alternates:
payload["alternate_hits"] = alternates
selected = best_alternate(alternates)
if selected:
payload["selected_alternate"] = selected
payload["selected_presentation"] = presentation(selected.get("values"))
return payload
def simple_reference_index(reference_report: dict[str, Any] | None, enum_map: dict[str, Any] | None = None) -> dict[tuple[str, int, str], dict[str, Any]]:
if not reference_report:
return {}
result = {}
for item in reference_report.get("references") or []:
ref = item.get("reference") or {}
scope = "main" if ref.get("scope") == "main" else part_scope(ref.get("table_part_name"))
row_index = int(ref.get("row_index") or 0)
cell_key = ref.get("cell_key")
if not cell_key:
continue
result[scope_key(scope, row_index, cell_key)] = resolved_payload(item, mode="single_reference", enum_map=enum_map)
return result
def composite_reference_index(composite_report: dict[str, Any] | None, enum_map: dict[str, Any] | None = None) -> dict[tuple[str, int, str], dict[str, Any]]:
if not composite_report:
return {}
result = {}
for item in composite_report.get("composites") or []:
comp = item.get("composite") or {}
scope = "main" if comp.get("scope") == "main" else part_scope(comp.get("table_part_name"))
row_index = int(comp.get("row_index") or 0)
metadata_path = comp.get("metadata_path")
if not metadata_path:
continue
payload = resolved_payload(item, mode="composite_reference", enum_map=enum_map)
payload["type_hex"] = comp.get("type_hex")
payload["rtref_hex"] = comp.get("rtref_hex")
payload["rtref_sql_number"] = comp.get("rtref_sql_number")
payload["rrref_hex"] = comp.get("rrref_hex")
payload["columns"] = comp.get("columns")
result[(scope, row_index, metadata_path)] = payload
return result
def composite_value_index(composite_value_report: dict[str, Any] | None) -> dict[tuple[str, int, str], dict[str, Any]]:
if not composite_value_report:
return {}
result = {}
for item in composite_value_report.get("composites") or []:
scope = "main" if item.get("scope") == "main" else part_scope(item.get("table_part_name"))
row_index = int(item.get("row_index") or 0)
metadata_path = item.get("metadata_path")
if metadata_path:
result[(scope, row_index, metadata_path)] = item
return result
def display_value(field: dict[str, Any]) -> Any:
resolved = field.get("resolved") or {}
for key in ("presentation", "selected_presentation"):
value = resolved.get(key)
if value is not None:
return value
composite = field.get("composite_value") or {}
selected = composite.get("selected") or {}
if selected.get("branch") == "primitive":
values = selected.get("primitive_values") or []
if values:
return normalize_display_value(values[0].get("value"), field)
if field.get("value") is not None:
return normalize_display_value(field.get("value"), field)
columns = field.get("columns") or {}
if len(columns) == 1:
only = next(iter(columns.values()))
return normalize_display_value(only.get("value"), field)
return None
def enrich_row(
row: dict[str, Any],
*,
scope: str,
row_index: int,
simple_refs: dict[tuple[str, int, str], dict[str, Any]],
composite_refs: dict[tuple[str, int, str], dict[str, Any]],
composite_values: dict[tuple[str, int, str], dict[str, Any]],
) -> dict[str, Any]:
cells: dict[str, Any] = {}
fields: dict[str, Any] = {}
grouped: dict[str, list[tuple[str, dict[str, Any]]]] = defaultdict(list)
for cell_key, cell in row.items():
enriched = dict(cell)
simple = simple_refs.get(scope_key(scope, row_index, cell_key))
if simple:
enriched["resolved"] = simple
cells[cell_key] = enriched
grouped[str(cell.get("metadata_path") or cell_key)].append((cell_key, enriched))
for metadata_path, members in grouped.items():
first = members[0][1]
columns = {member["column"]: member for _, member in members if member.get("column")}
field = {
"metadata_path": metadata_path,
"metadata_name": first.get("metadata_name"),
"metadata_uuid": first.get("metadata_uuid"),
"metadata_field": first.get("metadata_field"),
"columns": columns,
}
composite = composite_refs.get((scope, row_index, metadata_path))
composite_value = composite_values.get((scope, row_index, metadata_path))
if composite:
field["resolved"] = composite
if composite_value:
field["composite_value"] = composite_value
if "resolved" not in field and len(members) == 1 and members[0][1].get("resolved"):
field["resolved"] = members[0][1]["resolved"]
elif "resolved" not in field and len(members) == 1:
field["value"] = members[0][1].get("value")
field["display_value"] = display_value(field)
fields[metadata_path] = field
return {"cells": cells, "fields": fields}
def build_view(
read_result: dict[str, Any],
reference_report: dict[str, Any] | None,
composite_report: dict[str, Any] | None,
composite_value_report: dict[str, Any] | None,
enum_map: dict[str, Any] | None,
) -> dict[str, Any]:
simple_refs = simple_reference_index(reference_report, enum_map)
composite_refs = composite_reference_index(composite_report, enum_map)
composite_values = composite_value_index(composite_value_report)
main_rows = []
for index, row in enumerate(read_result.get("main", {}).get("rows") or []):
main_rows.append(
{
"row_index": index,
**enrich_row(
row,
scope="main",
row_index=index,
simple_refs=simple_refs,
composite_refs=composite_refs,
composite_values=composite_values,
),
}
)
table_parts = []
for part in read_result.get("table_parts") or []:
scope = part_scope(part.get("name"))
rows = []
for index, row in enumerate(part.get("rows") or []):
rows.append(
{
"row_index": index,
**enrich_row(
row,
scope=scope,
row_index=index,
simple_refs=simple_refs,
composite_refs=composite_refs,
composite_values=composite_values,
),
}
)
table_parts.append(
{
"name": part.get("name"),
"uuid": part.get("uuid"),
"table": part.get("table"),
"row_count": part.get("row_count"),
"rows": rows,
}
)
return {
"schema": "onec_sql_read_view.v1",
"source_schema": read_result.get("schema"),
"server": read_result.get("server"),
"database": read_result.get("database"),
"kind": read_result.get("kind"),
"identity": read_result.get("identity"),
"inputs": {
"read_result": read_result.get("projection_path"),
"reference_resolution_schema": (reference_report or {}).get("schema"),
"composite_reference_resolution_schema": (composite_report or {}).get("schema"),
"composite_value_resolution_schema": (composite_value_report or {}).get("schema"),
"enum_presentation_map_schema": (enum_map or {}).get("schema"),
},
"summary": {
"main_rows": len(main_rows),
"table_parts": len(table_parts),
"simple_reference_cells": len(simple_refs),
"composite_reference_groups": len(composite_refs),
"composite_value_groups": len(composite_values),
},
"main": {
"table": read_result.get("main", {}).get("table"),
"row_count": read_result.get("main", {}).get("row_count"),
"rows": main_rows,
},
"table_parts": table_parts,
}
def main() -> int:
parser = argparse.ArgumentParser(description="Build an enriched 1C SQL read view.")
parser.add_argument("--read-result", type=Path, required=True)
parser.add_argument("--reference-resolution", type=Path)
parser.add_argument("--composite-reference-resolution", type=Path)
parser.add_argument("--composite-value-resolution", type=Path)
parser.add_argument("--enum-presentation-map", type=Path)
parser.add_argument("--output", type=Path, required=True)
args = parser.parse_args()
read_result = load_json(args.read_result)
reference_report = load_json(args.reference_resolution) if args.reference_resolution else None
composite_report = load_json(args.composite_reference_resolution) if args.composite_reference_resolution else None
composite_value_report = load_json(args.composite_value_resolution) if args.composite_value_resolution else None
enum_map = load_json(args.enum_presentation_map) if args.enum_presentation_map else None
view = build_view(read_result, reference_report, composite_report, composite_value_report, enum_map)
view["inputs"]["read_result_path"] = str(args.read_result)
if args.reference_resolution:
view["inputs"]["reference_resolution_path"] = str(args.reference_resolution)
if args.composite_reference_resolution:
view["inputs"]["composite_reference_resolution_path"] = str(args.composite_reference_resolution)
if args.composite_value_resolution:
view["inputs"]["composite_value_resolution_path"] = str(args.composite_value_resolution)
if args.enum_presentation_map:
view["inputs"]["enum_presentation_map_path"] = str(args.enum_presentation_map)
write_json(args.output, view)
print(
json.dumps(
{
"output": str(args.output),
"main_rows": view["summary"]["main_rows"],
"table_parts": view["summary"]["table_parts"],
"simple_reference_cells": view["summary"]["simple_reference_cells"],
"composite_reference_groups": view["summary"]["composite_reference_groups"],
"composite_value_groups": view["summary"]["composite_value_groups"],
},
ensure_ascii=False,
)
)
return 0
if __name__ == "__main__":
raise SystemExit(main())
+383
View File
@@ -0,0 +1,383 @@
#!/usr/bin/env python3
"""Build a compact read-only evidence bundle for a 1C development task."""
from __future__ import annotations
import argparse
import base64
import json
import sys
from pathlib import Path
from typing import Any
sys.path.insert(0, str(Path(__file__).resolve().parent))
from get_1c_form_context import build_context as build_form_context # noqa: E402
from get_1c_module import build_module_result # noqa: E402
from get_1c_object_metadata import build_object_metadata # noqa: E402
from plan_1c_task_context import build_plan # noqa: E402
from resolve_1c_object import load_json # noqa: E402
def decode_arg(value: str | None, encoded: str | None) -> str | None:
if encoded:
return base64.b64decode(encoded).decode("utf-8")
return value
def collect_target_reads(investigation: dict[str, Any]) -> tuple[list[str], list[str]]:
forms = []
modules = []
for read in investigation.get("recommended_reads") or []:
form = read.get("form")
module = read.get("module")
if form and form not in forms:
forms.append(form)
if module and module not in modules:
modules.append(module)
for search in investigation.get("searches") or []:
for match in search.get("matches") or []:
form = match.get("form")
if form and form not in forms:
forms.append(form)
area = match.get("area")
module = match.get("name")
if area in {"module", "module.code"} and module and module not in modules:
modules.append(module)
return forms, modules
def compact_metadata(metadata: dict[str, Any], *, max_attributes: int) -> dict[str, Any]:
attrs = metadata.get("attributes") or []
sections = metadata.get("tabular_sections") or []
return {
"schema": metadata.get("schema"),
"view": metadata.get("view"),
"object": metadata.get("object"),
"attributes": attrs[:max_attributes],
"attributes_total": len(attrs),
"attributes_truncated": len(attrs) > max_attributes,
"tabular_sections": sections,
"counts": metadata.get("counts"),
}
def compact_form_context(context: dict[str, Any], *, max_items: int, max_attributes: int, max_commands: int) -> dict[str, Any]:
forms = []
for form in context.get("forms") or []:
copy = {key: form.get(key) for key in ("name", "synonym", "uuid", "origin", "effective_action", "meta_xml_path", "form_xml_path", "module_path", "extension_overlays") if form.get(key) not in (None, [], "")}
structure = form.get("structure") or {}
if structure:
copy["structure"] = {
"origin": structure.get("origin"),
"form_xml_path": structure.get("form_xml_path"),
"events": structure.get("events") or [],
"items": (structure.get("items") or [])[:max_items],
"attributes": (structure.get("attributes") or [])[:max_attributes],
"commands": (structure.get("commands") or [])[:max_commands],
"counts": structure.get("counts"),
}
overlays = []
for overlay in form.get("extension_overlays") or []:
overlay_copy = {key: overlay.get(key) for key in ("name", "synonym", "uuid", "origin", "effective_action", "meta_xml_path", "form_xml_path", "module_path") if overlay.get(key) not in (None, [], "")}
structure = overlay.get("structure") or {}
if structure:
overlay_copy["structure"] = {
"origin": structure.get("origin"),
"form_xml_path": structure.get("form_xml_path"),
"events": structure.get("events") or [],
"items": (structure.get("items") or [])[:max_items],
"attributes": (structure.get("attributes") or [])[:max_attributes],
"commands": (structure.get("commands") or [])[:max_commands],
"counts": structure.get("counts"),
}
overlays.append(overlay_copy)
if overlays:
copy["extension_overlays"] = overlays
forms.append(copy)
return {
"schema": context.get("schema"),
"view": context.get("view"),
"object": context.get("object"),
"query": context.get("query"),
"forms": forms,
"counts": context.get("counts"),
}
def compact_module_result(result: dict[str, Any]) -> dict[str, Any]:
return {
"schema": result.get("schema"),
"view": result.get("view"),
"object": result.get("object"),
"query": result.get("query"),
"modules": result.get("modules") or [],
"counts": result.get("counts"),
}
def read_lines(path: str) -> list[str]:
file_path = Path(path)
try:
return file_path.read_text(encoding="utf-8-sig").splitlines()
except UnicodeDecodeError:
return file_path.read_text(encoding="cp1251", errors="replace").splitlines()
def code_snippet(path: str, line: int, *, radius: int, max_chars: int) -> dict[str, Any] | None:
file_path = Path(path)
if not file_path.is_file() or line <= 0:
return None
lines = read_lines(path)
start = max(1, line - radius)
end = min(len(lines), line + radius)
text = "\n".join(lines[start - 1 : end])
truncated = len(text) > max_chars
return {
"path": path,
"line_start": start,
"line_end": end,
"focus_line": line,
"text": text[:max_chars],
"truncated": truncated,
"char_count": len(text),
}
def collect_code_snippets(searches: list[dict[str, Any]], *, radius: int, max_chars: int, limit: int) -> list[dict[str, Any]]:
snippets = []
seen = set()
for search in searches:
for match in search.get("matches") or []:
if match.get("area") != "module.code":
continue
evidence = match.get("evidence") or {}
path = evidence.get("path")
line = evidence.get("line")
if not path or not line:
continue
key = (path, line)
if key in seen:
continue
seen.add(key)
snippet = code_snippet(str(path), int(line), radius=radius, max_chars=max_chars)
if not snippet:
continue
snippet.update(
{
"search_text": search.get("text"),
"module": match.get("name"),
"origin": match.get("origin"),
"effective_action": match.get("effective_action"),
}
)
snippets.append(snippet)
if len(snippets) >= limit:
return snippets
return snippets
def evidence_for_investigation(
index: dict[str, Any],
investigation: dict[str, Any],
*,
view: str,
max_attributes: int,
max_form_items: int,
max_form_attributes: int,
max_form_commands: int,
max_module_chars: int,
code_snippet_radius: int,
max_code_snippet_chars: int,
max_code_snippets: int,
max_forms: int,
max_modules: int,
) -> dict[str, Any]:
candidate = investigation.get("candidate") or {}
kind = candidate.get("kind")
name = candidate.get("name")
metadata = build_object_metadata(index, kind=kind, name=name, view=view, extension=None, include_storage=False)
forms, modules = collect_target_reads(investigation)
form_contexts = []
for form_name in forms[:max_forms]:
form_contexts.append(
compact_form_context(
build_form_context(index, kind=kind, name=name, form=form_name, view=view, extension=None, max_items=max_form_items),
max_items=max_form_items,
max_attributes=max_form_attributes,
max_commands=max_form_commands,
)
)
module_contexts = []
for module_name in modules[:max_modules]:
module_contexts.append(
compact_module_result(
build_module_result(
index,
kind=kind,
name=name,
module_name=module_name,
view=view,
extension=None,
max_chars=max_module_chars,
routine=None,
)
)
)
snippets = collect_code_snippets(
investigation.get("searches") or [],
radius=code_snippet_radius,
max_chars=max_code_snippet_chars,
limit=max_code_snippets,
)
return {
"candidate": candidate,
"brief": investigation.get("brief"),
"searches": investigation.get("searches") or [],
"metadata": compact_metadata(metadata, max_attributes=max_attributes),
"forms": form_contexts,
"modules": module_contexts,
"code_snippets": snippets,
"recommended_reads": investigation.get("recommended_reads") or [],
"counts": {
"forms_materialized": len(form_contexts),
"modules_materialized": len(module_contexts),
"code_snippets": len(snippets),
},
}
def build_evidence(
index: dict[str, Any],
*,
text: str,
view: str,
max_objects: int,
max_terms: int,
max_matches: int,
max_attributes: int,
max_form_items: int,
max_form_attributes: int,
max_form_commands: int,
max_module_chars: int,
code_snippet_radius: int,
max_code_snippet_chars: int,
max_code_snippets: int,
max_forms: int,
max_modules: int,
) -> dict[str, Any]:
plan = build_plan(index, text=text, view=view, max_objects=max_objects, max_terms=max_terms, max_matches=max_matches)
investigations = []
for investigation in (plan.get("investigations") or [])[:max_objects]:
investigations.append(
evidence_for_investigation(
index,
investigation,
view=view,
max_attributes=max_attributes,
max_form_items=max_form_items,
max_form_attributes=max_form_attributes,
max_form_commands=max_form_commands,
max_module_chars=max_module_chars,
code_snippet_radius=code_snippet_radius,
max_code_snippet_chars=max_code_snippet_chars,
max_code_snippets=max_code_snippets,
max_forms=max_forms,
max_modules=max_modules,
)
)
return {
"schema": "onec_task_evidence_bundle.v1",
"view": view,
"task": {"text": text},
"plan": {
"schema": plan.get("schema"),
"object_candidates": plan.get("object_candidates") or [],
"search_terms": plan.get("search_terms") or [],
"safety": plan.get("safety"),
"counts": plan.get("counts"),
},
"investigations": investigations,
"limits": {
"max_objects": max_objects,
"max_terms": max_terms,
"max_matches": max_matches,
"max_attributes": max_attributes,
"max_form_items": max_form_items,
"max_form_attributes": max_form_attributes,
"max_form_commands": max_form_commands,
"max_module_chars": max_module_chars,
"code_snippet_radius": code_snippet_radius,
"max_code_snippet_chars": max_code_snippet_chars,
"max_code_snippets": max_code_snippets,
"max_forms": max_forms,
"max_modules": max_modules,
},
"safety": {
"mode": "read_only",
"write_status": "blocked_until_write_gates",
"write_contract": "docs/1c-write-path-safety.md",
},
"counts": {
"investigations": len(investigations),
"forms_materialized": sum(item.get("counts", {}).get("forms_materialized", 0) for item in investigations),
"modules_materialized": sum(item.get("counts", {}).get("modules_materialized", 0) for item in investigations),
"code_snippets": sum(item.get("counts", {}).get("code_snippets", 0) for item in investigations),
},
}
def main() -> int:
parser = argparse.ArgumentParser(description="Build read-only 1C task evidence bundle.")
parser.add_argument("--index", type=Path, required=True)
parser.add_argument("--text")
parser.add_argument("--text-b64")
parser.add_argument("--view", choices=["effective", "base"], default="effective")
parser.add_argument("--max-objects", type=int, default=2)
parser.add_argument("--max-terms", type=int, default=8)
parser.add_argument("--max-matches", type=int, default=8)
parser.add_argument("--max-attributes", type=int, default=120)
parser.add_argument("--max-form-items", type=int, default=250)
parser.add_argument("--max-form-attributes", type=int, default=120)
parser.add_argument("--max-form-commands", type=int, default=80)
parser.add_argument("--max-module-chars", type=int, default=12000)
parser.add_argument("--code-snippet-radius", type=int, default=8)
parser.add_argument("--max-code-snippet-chars", type=int, default=8000)
parser.add_argument("--max-code-snippets", type=int, default=20)
parser.add_argument("--max-forms", type=int, default=3)
parser.add_argument("--max-modules", type=int, default=4)
parser.add_argument("--output", type=Path)
args = parser.parse_args()
text = decode_arg(args.text, args.text_b64)
if not text:
raise SystemExit("Use --text or --text-b64.")
result = build_evidence(
load_json(args.index),
text=text,
view=args.view,
max_objects=args.max_objects,
max_terms=args.max_terms,
max_matches=args.max_matches,
max_attributes=args.max_attributes,
max_form_items=args.max_form_items,
max_form_attributes=args.max_form_attributes,
max_form_commands=args.max_form_commands,
max_module_chars=args.max_module_chars,
code_snippet_radius=args.code_snippet_radius,
max_code_snippet_chars=args.max_code_snippet_chars,
max_code_snippets=args.max_code_snippets,
max_forms=args.max_forms,
max_modules=args.max_modules,
)
output = json.dumps(result, ensure_ascii=False, indent=2) + "\n"
if args.output:
args.output.parent.mkdir(parents=True, exist_ok=True)
args.output.write_text(output, encoding="utf-8")
print(json.dumps({"output": str(args.output), "counts": result["counts"], "view": result["view"]}, ensure_ascii=False))
else:
print(output)
return 0
if __name__ == "__main__":
raise SystemExit(main())
@@ -0,0 +1,234 @@
#!/usr/bin/env python3
"""Build a deterministic route index for 1C metadata objects.
The index answers where an object or object part can be retrieved from:
DBNames storage roles, base Config direct files, XML names, and extension
manifest/CAS entries. It stores routes and observed payload signatures, not
semantic guesses.
"""
from __future__ import annotations
import argparse
import json
import re
from collections import Counter, defaultdict
from pathlib import Path
from typing import Any
from analyze_1c_manifest_object_parts import parse_cas_payload, suffix_of
def load_json(path: Path) -> dict[str, Any]:
return json.loads(path.read_text(encoding="utf-8-sig"))
def extension_name_from_manifest(path: Path) -> str:
stem = path.stem
return re.sub(r"^\d+_", "", stem).rsplit("-", 1)[0]
def ensure_object(objects: dict[str, dict[str, Any]], guid: str) -> dict[str, Any]:
guid = guid.lower()
item = objects.get(guid)
if item is None:
item = {
"guid": guid,
"dbnames": [],
"xml_top_objects": [],
"xml_occurrence_count": 0,
"config_routes": [],
"extension_routes": [],
}
objects[guid] = item
return item
def add_dbnames(objects: dict[str, dict[str, Any]], dbnames: dict[str, Any]) -> Counter[str]:
role_counts: Counter[str] = Counter()
for source in dbnames.get("dbnames") or []:
file_name = source.get("file_name")
for record in source.get("records") or []:
if record.get("status") != "parsed":
continue
guid = str(record.get("guid") or "").lower()
role = record.get("storage_role")
role_counts[str(role)] += 1
ensure_object(objects, guid)["dbnames"].append(
{
"source_file": file_name,
"storage_role": role,
"sql_number": record.get("sql_number"),
"index": record.get("index"),
}
)
return role_counts
def add_xml(objects: dict[str, dict[str, Any]], xml_index: dict[str, Any]) -> None:
for guid, item in (xml_index.get("guid_map") or {}).items():
obj = ensure_object(objects, guid)
obj["xml_top_objects"] = item.get("top_objects") or []
obj["xml_occurrence_count"] = item.get("total_occurrences", 0)
def payload_signature(payload: dict[str, Any]) -> dict[str, Any]:
return {
"parse_status": payload.get("parse_status"),
"encoding": payload.get("encoding"),
"text_offset": payload.get("text_offset"),
"payload_bytes": payload.get("payload_bytes"),
"compression": payload.get("compression"),
"root_marker": payload.get("root_marker"),
"root_len": payload.get("root_len"),
"payload_markers": payload.get("payload_markers") or [],
"base64_block_count": len(payload.get("_base64_blocks_decoded") or []),
"stream_block_count": len(payload.get("_stream_blocks") or []),
}
def add_config_routes(objects: dict[str, dict[str, Any]], config_dirs: list[Path]) -> None:
for directory in config_dirs:
if not directory.exists():
continue
for path in sorted(item for item in directory.iterdir() if item.is_file()):
guid = path.name.lower()
if not re.fullmatch(r"[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}", guid):
continue
payload = parse_cas_payload(path)
ensure_object(objects, guid)["config_routes"].append(
{
"route_type": "base_config_direct",
"table": directory.name,
"file_name": guid,
"path": str(path),
"payload_signature": payload_signature(payload),
}
)
def load_extension_inventory(path: Path | None) -> dict[str, dict[str, Any]]:
if not path or not path.is_file():
return {}
inventory = load_json(path)
result = {}
for ext in inventory.get("extensions") or []:
name = ext.get("extension_name")
if name:
result[name] = ext
return result
def add_extension_routes(
objects: dict[str, dict[str, Any]],
manifest_dir: Path,
cas_dir: Path,
extension_inventory: dict[str, dict[str, Any]],
) -> Counter[str]:
signature_counts: Counter[str] = Counter()
payload_cache: dict[str, dict[str, Any]] = {}
for manifest_path in sorted(manifest_dir.glob("*.json")):
manifest = load_json(manifest_path)
extension_name = extension_name_from_manifest(manifest_path)
ext = extension_inventory.get(extension_name, {})
for entry in manifest.get("entries") or []:
object_id = str(entry.get("object_id") or "").lower()
if not object_id:
continue
guid = object_id.split(".", 1)[0]
cas_key = entry.get("cas_key")
cas_path = Path(entry.get("cas_path") or cas_dir / str(cas_key))
if not cas_path.is_file():
cas_path = cas_dir / str(cas_key)
if cas_path.is_file():
if str(cas_path) not in payload_cache:
payload_cache[str(cas_path)] = payload_signature(parse_cas_payload(cas_path))
signature = payload_cache[str(cas_path)]
else:
signature = {"parse_status": "missing_cas"}
signature_key = (
f"{signature.get('parse_status')}|root={signature.get('root_marker')}|"
f"len={signature.get('root_len')}|markers={','.join(signature.get('payload_markers') or [])}"
)
signature_counts[signature_key] += 1
ensure_object(objects, guid)["extension_routes"].append(
{
"route_type": "extension_manifest_cas",
"extension_name": extension_name,
"extension_order": ext.get("extension_order"),
"dbnames_ext_guid": ext.get("dbnames_ext_guid"),
"manifest_path": str(manifest_path),
"root_cas_file": manifest.get("root_cas_file"),
"extension_configuration_guid": manifest.get("extension_configuration_guid"),
"object_id": object_id,
"suffix": suffix_of(object_id),
"cas_key": cas_key,
"cas_path": str(cas_path) if cas_path.is_file() else None,
"payload_signature": signature,
}
)
return signature_counts
def compact_object(item: dict[str, Any]) -> dict[str, Any]:
route_kind = []
if item["config_routes"]:
route_kind.append("base_config_direct")
if item["extension_routes"]:
route_kind.append("extension_manifest_cas")
if item["dbnames"]:
route_kind.append("dbnames_storage")
if item["xml_top_objects"]:
route_kind.append("xml_top_object")
item["route_kind"] = route_kind
return item
def main() -> int:
parser = argparse.ArgumentParser(description="Build a unified 1C object route index.")
parser.add_argument("--dbnames", type=Path, required=True)
parser.add_argument("--xml-index", type=Path, required=True)
parser.add_argument("--manifest-dir", type=Path, required=True)
parser.add_argument("--cas-dir", type=Path, required=True)
parser.add_argument("--extension-inventory", type=Path)
parser.add_argument("--config-dir", type=Path, action="append", default=[])
parser.add_argument("--output", type=Path, required=True)
args = parser.parse_args()
objects: dict[str, dict[str, Any]] = {}
role_counts = add_dbnames(objects, load_json(args.dbnames))
add_xml(objects, load_json(args.xml_index))
add_config_routes(objects, args.config_dir)
signature_counts = add_extension_routes(
objects,
args.manifest_dir,
args.cas_dir,
load_extension_inventory(args.extension_inventory),
)
compacted = {guid: compact_object(item) for guid, item in sorted(objects.items())}
route_counts = Counter()
for item in compacted.values():
for kind in item["route_kind"]:
route_counts[kind] += 1
report = {
"schema": "onec_unified_object_route_index.v1",
"dbnames": str(args.dbnames),
"xml_index": str(args.xml_index),
"manifest_dir": str(args.manifest_dir),
"cas_dir": str(args.cas_dir),
"config_dirs": [str(path) for path in args.config_dir],
"object_count": len(compacted),
"route_kind_counts": dict(route_counts.most_common()),
"dbnames_role_counts": dict(role_counts.most_common()),
"extension_payload_signature_counts": dict(signature_counts.most_common()),
"objects": compacted,
}
args.output.parent.mkdir(parents=True, exist_ok=True)
args.output.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
print(json.dumps({"output": str(args.output), "objects": len(compacted)}, ensure_ascii=False))
return 0
if __name__ == "__main__":
raise SystemExit(main())
+114
View File
@@ -0,0 +1,114 @@
#!/usr/bin/env python3
from __future__ import annotations
import argparse
import json
from pathlib import Path
from typing import Any
ACTION_PRIORITY = {
"run_before_after_learning_for_parameter": 10,
"collect_allowed_values_and_smoke": 20,
"run_before_after_learning_for_named_scalar": 30,
"learn_reference_write_rule": 80,
"do_not_generic_write": 100,
}
def load_rows(path: Path) -> tuple[str, list[dict[str, Any]]]:
data = json.loads(path.read_text(encoding="utf-8"))
schema = str(data.get("schema") or "")
if schema == "onec_form_write_scalar_registry.v1":
return schema, [row for row in data.get("scalars", []) if isinstance(row, dict)]
if schema == "onec_form_write_enum_registry.v1":
return schema, [row for row in data.get("properties", []) if isinstance(row, dict)]
raise SystemExit(f"Unsupported registry schema: {schema or '<missing>'}")
def example_selector(example: dict[str, Any]) -> dict[str, Any]:
target = example.get("target")
section = example.get("effective_section") or example.get("requested_section")
selector: dict[str, Any] = {}
if section == "commands":
selector["command"] = target
elif section == "attributes":
selector["attribute"] = target
else:
selector["element"] = target
return selector
def learning_case(row: dict[str, Any], index: int) -> dict[str, Any]:
examples = row.get("examples") if isinstance(row.get("examples"), list) else []
example = examples[0] if examples and isinstance(examples[0], dict) else {}
counts = row.get("counts") if isinstance(row.get("counts"), dict) else {}
action = str(row.get("recommended_action") or row.get("risk") or "")
return {
"id": f"learn-{index:03d}",
"action": action,
"property": row.get("property"),
"marker": row.get("marker"),
"parameter_index": row.get("parameter_index"),
"value_type": row.get("value_type"),
"entries": counts.get("entries"),
"observed_values": row.get("observed_values"),
"selector": example_selector(example),
"write_path": example.get("write_path"),
"current_value": example.get("old"),
"manual_step": {
"target": example.get("target"),
"section": example.get("effective_section") or example.get("requested_section"),
"presentation": example.get("presentation"),
"instruction": "Измени это свойство в конфигураторе на другое допустимое значение, сохрани форму, затем запусти capture_after/diff/infer.",
},
}
def main() -> int:
parser = argparse.ArgumentParser(description="Build ordered before/after learning plan from 1C write registries.")
parser.add_argument("--registry", type=Path, required=True, help="Scalar or enum registry JSON path.")
parser.add_argument("--output", type=Path, required=True, help="Output learning plan JSON path.")
parser.add_argument("--limit", type=int, default=50, help="Maximum cases to include.")
parser.add_argument(
"--include-actions",
nargs="*",
default=["run_before_after_learning_for_parameter", "collect_allowed_values_and_smoke", "run_before_after_learning_for_named_scalar"],
help="Recommended actions/risks to include.",
)
args = parser.parse_args()
schema, rows = load_rows(args.registry)
include = set(args.include_actions)
filtered = [row for row in rows if str(row.get("recommended_action") or row.get("risk") or "") in include]
filtered.sort(
key=lambda row: (
ACTION_PRIORITY.get(str(row.get("recommended_action") or row.get("risk") or ""), 50),
-int((row.get("counts") if isinstance(row.get("counts"), dict) else {}).get("entries") or 0),
str(row.get("marker")),
str(row.get("parameter_index")),
str(row.get("property")),
)
)
cases = [learning_case(row, index + 1) for index, row in enumerate(filtered[: args.limit])]
result = {
"schema": "onec_form_write_learning_plan.v1",
"status": "ok",
"source_registry": str(args.registry),
"source_schema": schema,
"counts": {
"cases": len(cases),
"available": len(filtered),
"included_actions": sorted(include),
},
"workflow": ["capture_before", "manual_configurator_change", "capture_after", "diff", "infer_rule", "smoke_rule"],
"cases": cases,
}
args.output.parent.mkdir(parents=True, exist_ok=True)
args.output.write_text(json.dumps(result, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
print(json.dumps({"schema": result["schema"], "status": "ok", "counts": result["counts"], "path": str(args.output)}, ensure_ascii=False, indent=2))
return 0
if __name__ == "__main__":
raise SystemExit(main())
@@ -0,0 +1,161 @@
#!/usr/bin/env python3
from __future__ import annotations
import argparse
import json
from collections import Counter, defaultdict
from pathlib import Path
from typing import Any
KNOWN_ENUM_VALUES: dict[str, dict[str, str]] = {
"Вид": {
"1": "Поле надписи",
"2": "Поле ввода",
"4": "Страница",
"5": "Группа",
"9": "Командная панель",
"12": "Расширенная подсказка",
"31": "Кнопка командной панели",
"48": "Поле формы",
"55": "Динамический список",
"73": "Таблица формы",
},
"ПоложениеЗаголовка": {"0": "Авто", "1": "Верх", "2": "Нет"},
"ПоложениеВКоманднойПанели": {"0": "Авто", "1": "В командной панели", "2": "В дополнительном подменю"},
"Отображение": {"3": "Авто"},
"ЦветФона": {"3": "Авто"},
"ЦветТекста": {"3": "Авто"},
"ЦветРамки": {"3": "Авто"},
}
PROPERTY_ALIASES = {
"group": "Группа",
"id": "Идентификатор",
"name": "Имя",
"view": "Вид",
"title": "Заголовок",
"command_bar_location": "ПоложениеВКоманднойПанели",
}
def load_entries(report: dict[str, Any]) -> list[dict[str, Any]]:
matrix = report.get("matrix") if isinstance(report.get("matrix"), dict) else {}
entries = matrix.get("entries") if isinstance(matrix.get("entries"), list) else []
return [entry for entry in entries if isinstance(entry, dict)]
def property_key(entry: dict[str, Any]) -> tuple[str, str, str, str]:
prop = entry.get("property") if isinstance(entry.get("property"), dict) else {}
target = entry.get("requested_target") if isinstance(entry.get("requested_target"), dict) else {}
raw_name = str(prop.get("semantic_name") or prop.get("canonical_property") or prop.get("property") or "")
name = PROPERTY_ALIASES.get(raw_name, raw_name)
marker = str(target.get("marker") or "")
index = str(prop.get("parameter_index") if prop.get("parameter_index") is not None else "")
value_type = str(prop.get("value_type") or "")
return name, marker, index, value_type
def risk_class(name: str, value_type: str) -> str:
normalized = PROPERTY_ALIASES.get(name, name)
if normalized in {"Идентификатор", "Имя"} or "маркер" in normalized.casefold():
return "manual_only_identity_or_marker"
if name == "Вид":
return "structural_type_no_generic_write"
if normalized == "Группа":
return "reference_or_container_rule_required"
if normalized in KNOWN_ENUM_VALUES:
return "allowed_values_known_needs_smoke_rule"
if value_type in {"enum_atom", "bool_or_enum_atom", "color_or_enum_atom"}:
return "allowed_values_unknown"
return "scalar_semantics_unknown"
def main() -> int:
parser = argparse.ArgumentParser(description="Build enum/scalar learning registry from 1C write matrix gaps.")
parser.add_argument("--matrix-report", type=Path, required=True, help="Report produced by scripts/smoke_1c_write_matrix.py.")
parser.add_argument("--output", type=Path, required=True, help="Output enum registry JSON path.")
parser.add_argument("--sample-limit", type=int, default=8, help="Examples per enum/scalar group.")
args = parser.parse_args()
report = json.loads(args.matrix_report.read_text(encoding="utf-8"))
groups: dict[tuple[str, str, str, str], dict[str, Any]] = {}
for entry in load_entries(report):
prop = entry.get("property") if isinstance(entry.get("property"), dict) else {}
if entry.get("can_smoke"):
continue
if entry.get("reason") not in {"value_type_not_smoke_safe", "identity_or_binding_property"}:
continue
value_type = str(prop.get("value_type") or "")
if value_type not in {"enum_atom", "bool_or_enum_atom", "color_or_enum_atom", "integer_atom", "scalar"}:
continue
key = property_key(entry)
name, marker, index, _ = key
row = groups.setdefault(
key,
{
"property": name,
"marker": marker or None,
"parameter_index": index or None,
"value_type": value_type,
"risk": risk_class(name, value_type),
"observed_values": Counter(),
"reasons": Counter(),
"sections": Counter(),
"examples": [],
},
)
target = entry.get("requested_target") if isinstance(entry.get("requested_target"), dict) else {}
effective = entry.get("effective_target") if isinstance(entry.get("effective_target"), dict) else {}
old = "" if prop.get("old") is None else str(prop.get("old"))
row["observed_values"][old] += 1
row["reasons"][str(entry.get("reason") or "")] += 1
row["sections"][str(effective.get("section") or "")] += 1
if len(row["examples"]) < args.sample_limit:
row["examples"].append(
{
"target": target.get("name") or target.get("path"),
"requested_section": target.get("section"),
"effective_section": effective.get("section"),
"presentation": prop.get("presentation"),
"semantic_name": prop.get("semantic_name"),
"old": old,
"write_path": prop.get("write_path"),
"reason": entry.get("reason"),
}
)
properties = []
for row in groups.values():
known = KNOWN_ENUM_VALUES.get(str(row.get("property") or ""))
properties.append(
{
**{key: value for key, value in row.items() if key not in {"observed_values", "reasons", "sections"}},
"observed_values": dict(row["observed_values"].most_common()),
"known_values": known,
"reasons": dict(row["reasons"]),
"sections": dict(row["sections"]),
"counts": {"entries": sum(row["observed_values"].values()), "observed_values": len(row["observed_values"])},
}
)
properties.sort(key=lambda item: (-int(item["counts"]["entries"]), str(item.get("property")), str(item.get("marker")), str(item.get("parameter_index"))))
result = {
"schema": "onec_form_write_enum_registry.v1",
"status": "ok",
"source_report": str(args.matrix_report),
"counts": {
"groups": len(properties),
"entries": sum(int(item["counts"]["entries"]) for item in properties),
"by_risk": dict(Counter(str(item.get("risk")) for item in properties)),
},
"properties": properties,
}
args.output.parent.mkdir(parents=True, exist_ok=True)
args.output.write_text(json.dumps(result, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
print(json.dumps({"schema": result["schema"], "status": "ok", "counts": result["counts"], "path": str(args.output)}, ensure_ascii=False, indent=2))
return 0
if __name__ == "__main__":
raise SystemExit(main())
@@ -0,0 +1,194 @@
#!/usr/bin/env python3
from __future__ import annotations
import argparse
import json
from collections import Counter
from pathlib import Path
from typing import Any
SCALAR_VALUE_TYPES = {"integer_atom", "scalar"}
SCALAR_REASONS = {"value_type_not_smoke_safe", "identity_or_binding_property"}
PROPERTY_ALIASES = {
"group": "Группа",
"id": "Идентификатор",
"name": "Имя",
"view": "Вид",
"title": "Заголовок",
"command_bar_location": "ПоложениеВКоманднойПанели",
}
MANUAL_ONLY_PROPERTIES = {"Идентификатор", "Имя", "ПутьКДанным", "Данные", "Вид"}
REFERENCE_PROPERTIES = {"Группа", "group"}
def load_entries(report: dict[str, Any]) -> list[dict[str, Any]]:
matrix = report.get("matrix") if isinstance(report.get("matrix"), dict) else {}
entries = matrix.get("entries") if isinstance(matrix.get("entries"), list) else []
return [entry for entry in entries if isinstance(entry, dict)]
def normalize_property_name(prop: dict[str, Any]) -> str:
raw = str(prop.get("semantic_name") or prop.get("canonical_property") or prop.get("property") or "")
return PROPERTY_ALIASES.get(raw, raw)
def scalar_bucket(entry: dict[str, Any]) -> str:
prop = entry.get("property") if isinstance(entry.get("property"), dict) else {}
target = entry.get("requested_target") if isinstance(entry.get("requested_target"), dict) else {}
name = normalize_property_name(prop)
marker = str(target.get("marker") or "")
parameter_index = prop.get("parameter_index")
semantic_status = str(prop.get("semantic_status") or "")
if name in MANUAL_ONLY_PROPERTIES or "маркер" in name.casefold():
return "manual_only_identity_or_structural"
if name in REFERENCE_PROPERTIES:
return "reference_or_container_rule_required"
if semantic_status and semantic_status != "unknown":
return "semantic_scalar_needs_allowed_values"
if parameter_index is not None:
return f"learn_marker_{marker}_parameter_{parameter_index}"
return "learn_named_scalar_semantics"
def recommended_action(bucket: str) -> str:
if bucket == "manual_only_identity_or_structural":
return "do_not_generic_write"
if bucket == "reference_or_container_rule_required":
return "learn_reference_write_rule"
if bucket == "semantic_scalar_needs_allowed_values":
return "collect_allowed_values_and_smoke"
if bucket.startswith("learn_marker_"):
return "run_before_after_learning_for_parameter"
return "run_before_after_learning_for_named_scalar"
def make_key(entry: dict[str, Any]) -> tuple[str, str, str, str, str]:
prop = entry.get("property") if isinstance(entry.get("property"), dict) else {}
target = entry.get("requested_target") if isinstance(entry.get("requested_target"), dict) else {}
effective = entry.get("effective_target") if isinstance(entry.get("effective_target"), dict) else {}
return (
scalar_bucket(entry),
normalize_property_name(prop),
str(target.get("marker") or effective.get("marker") or ""),
str(prop.get("parameter_index") if prop.get("parameter_index") is not None else ""),
str(prop.get("value_type") or ""),
)
def main() -> int:
parser = argparse.ArgumentParser(description="Build scalar learning registry from 1C write matrix gaps.")
parser.add_argument("--matrix-report", type=Path, required=True, help="Report produced by scripts/smoke_1c_write_matrix.py.")
parser.add_argument("--output", type=Path, required=True, help="Output scalar registry JSON path.")
parser.add_argument("--sample-limit", type=int, default=8, help="Examples per scalar group.")
args = parser.parse_args()
report = json.loads(args.matrix_report.read_text(encoding="utf-8"))
groups: dict[tuple[str, str, str, str, str], dict[str, Any]] = {}
skipped = Counter()
for entry in load_entries(report):
prop = entry.get("property") if isinstance(entry.get("property"), dict) else {}
if entry.get("can_smoke"):
skipped["can_smoke"] += 1
continue
reason = str(entry.get("reason") or "")
value_type = str(prop.get("value_type") or "")
if reason not in SCALAR_REASONS:
skipped[f"reason:{reason}"] += 1
continue
if value_type not in SCALAR_VALUE_TYPES:
skipped[f"value_type:{value_type}"] += 1
continue
key = make_key(entry)
bucket, name, marker, parameter_index, _ = key
row = groups.setdefault(
key,
{
"bucket": bucket,
"property": name,
"marker": marker or None,
"parameter_index": parameter_index or None,
"value_type": value_type,
"recommended_action": recommended_action(bucket),
"observed_values": Counter(),
"reasons": Counter(),
"sections": Counter(),
"type_names": Counter(),
"examples": [],
},
)
target = entry.get("requested_target") if isinstance(entry.get("requested_target"), dict) else {}
effective = entry.get("effective_target") if isinstance(entry.get("effective_target"), dict) else {}
old = "" if prop.get("old") is None else str(prop.get("old"))
row["observed_values"][old] += 1
row["reasons"][reason] += 1
row["sections"][str(effective.get("section") or target.get("section") or "")] += 1
row["type_names"][str(effective.get("type_name") or target.get("type_name") or "")] += 1
if len(row["examples"]) < args.sample_limit:
row["examples"].append(
{
"target": target.get("name") or target.get("path"),
"requested_section": target.get("section"),
"effective_section": effective.get("section"),
"type_name": effective.get("type_name") or target.get("type_name"),
"presentation": prop.get("presentation"),
"semantic_name": prop.get("semantic_name"),
"semantic_group": prop.get("semantic_group"),
"old": old,
"write_path": prop.get("write_path"),
"reason": reason,
}
)
scalars = []
for row in groups.values():
scalars.append(
{
**{key: value for key, value in row.items() if key not in {"observed_values", "reasons", "sections", "type_names"}},
"observed_values": dict(row["observed_values"].most_common()),
"reasons": dict(row["reasons"]),
"sections": dict(row["sections"]),
"type_names": dict(row["type_names"].most_common()),
"counts": {
"entries": sum(row["observed_values"].values()),
"observed_values": len(row["observed_values"]),
"examples": len(row["examples"]),
},
}
)
scalars.sort(
key=lambda item: (
str(item.get("recommended_action")),
-int(item["counts"]["entries"]),
str(item.get("marker")),
str(item.get("parameter_index")),
str(item.get("property")),
)
)
result = {
"schema": "onec_form_write_scalar_registry.v1",
"status": "ok",
"source_report": str(args.matrix_report),
"counts": {
"groups": len(scalars),
"entries": sum(int(item["counts"]["entries"]) for item in scalars),
"by_action": dict(Counter(str(item.get("recommended_action")) for item in scalars)),
"by_bucket": dict(Counter(str(item.get("bucket")) for item in scalars)),
"skipped": dict(skipped),
},
"scalars": scalars,
}
args.output.parent.mkdir(parents=True, exist_ok=True)
args.output.write_text(json.dumps(result, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
print(json.dumps({"schema": result["schema"], "status": "ok", "counts": result["counts"], "path": str(args.output)}, ensure_ascii=False, indent=2))
return 0
if __name__ == "__main__":
raise SystemExit(main())
@@ -0,0 +1,188 @@
#!/usr/bin/env python3
from __future__ import annotations
import argparse
import json
from collections import Counter
from pathlib import Path
from typing import Any
SOURCE_AWARE_ROUTE_EVIDENCE: list[dict[str, Any]] = [
{
"key": "attributes|title|string|3.3.4.2.1|data_path_form_attribute_title",
"requested_section": "items",
"requested_name": "А",
"requested_path": "1.25.24.24.24",
"effective_section": "attributes",
"effective_name": "А",
"effective_path": "3.3",
"source_kind": "data_path_form_attribute_title",
"property": "title",
"presentation": "Заголовок",
"value_type": "string",
"read_path": "1.25.24.24.24.4.2.1",
"write_path": "3.3.4.2.1",
"verification": "source_aware_readback",
"status": "verified",
"evidence": "metadata.write apply_and_rollback verified on upo_test; route smoke verified",
},
{
"key": "attribute_fields|title|string|3.6.14.4.2.1|data_path_form_attribute_field_title",
"requested_section": "items",
"requested_name": "ТЗК1",
"requested_path": "1.25.24.26.68",
"effective_section": "attribute_fields",
"effective_name": "К1",
"effective_path": "3.6.14",
"source_kind": "data_path_form_attribute_field_title",
"property": "title",
"presentation": "Заголовок",
"value_type": "string",
"read_path": "1.25.24.26.68.4.2.1",
"write_path": "3.6.14.4.2.1",
"verification": "source_aware_readback",
"status": "verified",
"evidence": "metadata.write apply_and_rollback verified on upo_test; route smoke verified",
},
{
"key": "commands|title|string|5.3.3.2.1|local",
"requested_section": "commands",
"requested_name": "КомандаПример1",
"requested_path": "5.3",
"effective_section": "commands",
"effective_name": "КомандаПример1",
"effective_path": "5.3",
"source_kind": "local",
"property": "title",
"presentation": "Заголовок",
"value_type": "string",
"read_path": "5.3.3.2.1",
"write_path": "5.3.3.2.1",
"verification": "source_aware_readback",
"status": "verified",
"evidence": "metadata.write apply_and_rollback verified on upo_test; route smoke verified",
},
]
def entry_key(item: dict[str, Any]) -> str:
return "|".join(
str(item.get(key) or "")
for key in ("effective_section", "property", "value_type", "write_path", "source_kind")
)
def registry_entry(row: dict[str, Any]) -> dict[str, Any] | None:
if row.get("status") != "verified":
return None
entry = row.get("entry") if isinstance(row.get("entry"), dict) else {}
prop = entry.get("property") if isinstance(entry.get("property"), dict) else {}
requested = entry.get("requested_target") if isinstance(entry.get("requested_target"), dict) else {}
effective = entry.get("effective_target") if isinstance(entry.get("effective_target"), dict) else {}
source = entry.get("effective_source") if isinstance(entry.get("effective_source"), dict) else {}
item = {
"requested_section": requested.get("section"),
"requested_name": requested.get("name"),
"requested_path": requested.get("path"),
"effective_section": effective.get("section"),
"effective_name": effective.get("name"),
"effective_path": effective.get("path"),
"source_kind": source.get("kind") or "local",
"property": prop.get("semantic_name") or prop.get("canonical_property") or prop.get("property"),
"canonical_property": prop.get("canonical_property") or prop.get("property"),
"presentation": prop.get("presentation"),
"semantic_name": prop.get("semantic_name"),
"semantic_group": prop.get("semantic_group"),
"semantic_source": prop.get("semantic_source"),
"parameter_index": prop.get("parameter_index"),
"value_type": prop.get("value_type"),
"read_path": prop.get("read_path"),
"write_path": prop.get("write_path"),
"verification": prop.get("verification"),
"status": "verified",
}
item["key"] = entry_key(item)
return item
def pattern_from_entry(item: dict[str, Any]) -> dict[str, Any]:
pattern = {
key: value
for key, value in item.items()
if key not in {"requested_name", "requested_path", "effective_name", "effective_path", "read_path", "evidence"}
}
pattern["verified_count"] = 0
pattern["examples"] = []
return pattern
def add_pattern_example(patterns: dict[str, dict[str, Any]], item: dict[str, Any]) -> None:
pattern = patterns.setdefault(item["key"], pattern_from_entry(item))
pattern["verified_count"] = int(pattern.get("verified_count") or 0) + 1
example = {
"requested_name": item.get("requested_name"),
"requested_path": item.get("requested_path"),
"effective_name": item.get("effective_name"),
"effective_path": item.get("effective_path"),
"read_path": item.get("read_path"),
}
examples = pattern.setdefault("examples", [])
if example not in examples and len(examples) < 5:
examples.append(example)
def main() -> int:
parser = argparse.ArgumentParser(description="Build compact verified write registry from a write-matrix smoke report.")
parser.add_argument("--smoke-report", type=Path, required=True, help="Full report produced by scripts/smoke_1c_write_matrix.py.")
parser.add_argument("--output", type=Path, required=True, help="Output registry JSON path.")
parser.add_argument("--include-route-evidence", action="store_true", help="Append known source-aware route evidence from the learning case.")
args = parser.parse_args()
data = json.loads(args.smoke_report.read_text(encoding="utf-8"))
smoke = data.get("smoke") if isinstance(data.get("smoke"), dict) else {}
entries: list[dict[str, Any]] = []
patterns: dict[str, dict[str, Any]] = {}
for row in smoke.get("results") or []:
if not isinstance(row, dict):
continue
item = registry_entry(row)
if not item:
continue
entries.append(item)
add_pattern_example(patterns, item)
if args.include_route_evidence:
existing = {item.get("key") for item in entries}
for item in SOURCE_AWARE_ROUTE_EVIDENCE:
if item["key"] not in existing:
entries.append(dict(item))
existing.add(item["key"])
add_pattern_example(patterns, item)
registry = {
"schema": "onec_form_write_verified_registry.v1",
"status": "ok",
"source_report": str(args.smoke_report),
"adapter_report_path": smoke.get("path"),
"base_id": data.get("base_id"),
"table": data.get("table"),
"file_name": data.get("file_name"),
"counts": {
"verified_entries": len(entries),
"verified_patterns": len(patterns),
"by_effective_section": dict(sorted(Counter(item.get("effective_section") for item in entries).items())),
"by_property": dict(sorted(Counter(item.get("property") for item in entries).items())),
"by_source_kind": dict(sorted(Counter(item.get("source_kind") for item in entries).items())),
},
"patterns": sorted(patterns.values(), key=lambda item: (str(item.get("effective_section")), str(item.get("property")), str(item.get("write_path")), str(item.get("source_kind")))),
"entries": entries,
}
args.output.parent.mkdir(parents=True, exist_ok=True)
args.output.write_text(json.dumps(registry, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
print(json.dumps({"schema": registry["schema"], "status": "ok", "counts": registry["counts"], "path": str(args.output)}, ensure_ascii=False, indent=2))
return 0
if __name__ == "__main__":
raise SystemExit(main())
+224
View File
@@ -0,0 +1,224 @@
#!/usr/bin/env python3
"""Build a GUID index for a 1C XML configuration dump.
The index is intentionally mechanical: it records GUID occurrences in XML
attributes and element text, plus the top metadata object declared by each
file. It does not infer SQL table names or storage roles.
"""
from __future__ import annotations
import argparse
import json
import os
import re
from pathlib import Path
from typing import Any
GUID_RE = re.compile(
r"[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}"
)
TOP_OBJECT_RE = re.compile(
r"<(?P<kind>[A-Za-z][A-Za-z0-9]*)\s+[^>]*uuid=\"(?P<guid>"
+ GUID_RE.pattern
+ r")\"",
re.S,
)
NAME_RE = re.compile(r"<Name>(?P<name>.*?)</Name>", re.S)
SYNONYM_RE = re.compile(r"<(?:[A-Za-z0-9]+:)?content>(?P<content>.*?)</(?:[A-Za-z0-9]+:)?content>", re.S)
def clean_xml_text(value: str) -> str:
return repair_mojibake(
value.replace("&quot;", '"')
.replace("&apos;", "'")
.replace("&lt;", "<")
.replace("&gt;", ">")
.replace("&amp;", "&")
.strip()
)
def repair_mojibake(value: str) -> str:
if not value:
return value
for source_encoding in ("gbk", "cp1255", "cp1252", "latin1"):
try:
candidate = value.encode(source_encoding).decode("cp1251")
except UnicodeError:
continue
candidate_cyrillic = sum(1 for char in candidate if "А" <= char <= "я" or char == "ё" or char == "Ё")
value_cyrillic = sum(1 for char in value if "А" <= char <= "я" or char == "ё" or char == "Ё")
if candidate_cyrillic > value_cyrillic:
return repair_mojibake(candidate)
cjk_count = sum(1 for char in value if "\u4e00" <= char <= "\u9fff")
if cjk_count:
try:
candidate = value.encode("gbk").decode("cp1251")
return repair_mojibake(candidate)
except UnicodeError:
pass
cyrillic_count = sum(1 for char in value if "А" <= char <= "я" or char == "ё" or char == "Ё")
suspicious_count = sum(1 for char in value if char in "ÐÑÂÃÄÅÆÇÈÉÊËÌÍÎÏÒÓÔÕÖרÙÚÛÜÝÞßàáâãäåæçèéêëìíîïòóôõö÷øùúûüýþÿ")
if suspicious_count <= cyrillic_count:
return value
try:
repaired = value.encode("latin1").decode("cp1251")
except UnicodeError:
return value
repaired_cyrillic = sum(1 for char in repaired if "А" <= char <= "я" or char == "ё" or char == "Ё")
return repaired if repaired_cyrillic > cyrillic_count else value
def inspect_xml_file(path: Path, root_dir: Path, *, max_occurrences_per_file: int) -> dict[str, Any]:
item: dict[str, Any] = {
"path": repair_mojibake(str(path)),
"relative_path": repair_mojibake(str(path.relative_to(root_dir))),
"status": "ok",
"root_tag": "",
"top_object": None,
"occurrences": [],
}
try:
text = path.read_text(encoding="utf-8", errors="replace")
except OSError as exc:
item["status"] = "read_error"
item["error"] = str(exc)
return item
root_match = re.search(r"<(?P<tag>[A-Za-z][A-Za-z0-9]*)[\s>]", text)
item["root_tag"] = root_match.group("tag") if root_match else ""
object_match = TOP_OBJECT_RE.search(text)
if object_match:
object_start = object_match.start()
object_end = TOP_OBJECT_RE.search(text, object_match.end())
object_fragment = text[object_start : object_end.start() if object_end else min(len(text), object_start + 500_000)]
name_match = NAME_RE.search(object_fragment)
synonym_match = SYNONYM_RE.search(object_fragment)
top_object = {
"guid": object_match.group("guid").lower(),
"xml_kind": object_match.group("kind"),
"name": clean_xml_text(name_match.group("name")) if name_match else "",
"synonym": clean_xml_text(synonym_match.group("content")) if synonym_match else "",
}
else:
top_object = None
item["top_object"] = top_object
occurrences = []
for match in GUID_RE.finditer(text):
occurrences.append({"guid": match.group(0).lower(), "offset": match.start()})
if len(occurrences) >= max_occurrences_per_file:
item["occurrences"] = occurrences
item["truncated"] = True
return item
item["occurrences"] = occurrences
item["truncated"] = False
return item
def build_guid_map(files: list[dict[str, Any]], *, max_occurrences_per_guid: int) -> dict[str, Any]:
guid_map: dict[str, Any] = {}
for file_item in files:
if file_item.get("status") != "ok":
continue
top_object = file_item.get("top_object")
if top_object:
guid = top_object["guid"]
entry = guid_map.setdefault(guid, {"total_occurrences": 0, "top_objects": [], "occurrences": []})
entry["top_objects"].append(
{
"path": file_item["path"],
"relative_path": file_item["relative_path"],
"xml_kind": top_object["xml_kind"],
"name": top_object["name"],
"synonym": top_object["synonym"],
}
)
for occurrence in file_item.get("occurrences") or []:
guid = occurrence["guid"]
entry = guid_map.setdefault(guid, {"total_occurrences": 0, "top_objects": [], "occurrences": []})
entry["total_occurrences"] += 1
if len(entry["occurrences"]) < max_occurrences_per_guid:
entry["occurrences"].append(
{
"path": file_item["path"],
"relative_path": file_item["relative_path"],
"offset": occurrence["offset"],
}
)
return dict(sorted(guid_map.items()))
def list_xml_paths(root: Path, max_relative_depth: int) -> list[Path]:
if max_relative_depth <= 0:
return sorted(root.rglob("*.xml"))
result: list[Path] = []
root_parts = len(root.parts)
for current, dirs, files in os.walk(root):
current_path = Path(current)
relative_depth = len(current_path.parts) - root_parts
if relative_depth >= max_relative_depth - 1:
dirs[:] = []
for file_name in files:
if file_name.lower().endswith(".xml"):
path = current_path / file_name
if len(path.relative_to(root).parts) <= max_relative_depth:
result.append(path)
return sorted(result)
def main() -> int:
parser = argparse.ArgumentParser(description="Build a GUID index for a 1C XML dump.")
parser.add_argument("xml_root", type=Path)
parser.add_argument("--output", type=Path, required=True)
parser.add_argument("--max-occurrences-per-file", type=int, default=5000)
parser.add_argument("--max-occurrences-per-guid", type=int, default=20)
parser.add_argument(
"--max-relative-depth",
type=int,
default=0,
help="Only scan XML files whose relative path has at most this many parts; 0 scans all files.",
)
args = parser.parse_args()
xml_root = args.xml_root.resolve()
paths = list_xml_paths(xml_root, args.max_relative_depth)
files = [
inspect_xml_file(path, xml_root, max_occurrences_per_file=args.max_occurrences_per_file)
for path in paths
]
guid_map = build_guid_map(files, max_occurrences_per_guid=args.max_occurrences_per_guid)
top_object_count = sum(1 for item in files if item.get("top_object"))
parse_errors = sum(1 for item in files if item.get("status") != "ok")
report = {
"schema": "onec_xml_guid_index.v1",
"xml_root": repair_mojibake(str(xml_root)),
"file_count": len(files),
"top_object_count": top_object_count,
"parse_error_count": parse_errors,
"guid_count": len(guid_map),
"files": files,
"guid_map": guid_map,
}
args.output.parent.mkdir(parents=True, exist_ok=True)
args.output.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
print(
json.dumps(
{
"output": str(args.output),
"files": len(files),
"top_objects": top_object_count,
"guids": len(guid_map),
"parse_errors": parse_errors,
},
ensure_ascii=False,
)
)
return 0
if __name__ == "__main__":
raise SystemExit(main())
+151
View File
@@ -0,0 +1,151 @@
from __future__ import annotations
import argparse
import datetime as dt
import hashlib
import json
from pathlib import Path
from typing import Any
ROOT = Path(__file__).resolve().parents[1]
DEFAULT_OUTPUT = ROOT / "reports" / "llm-artifact-manifest.json"
DEFAULT_ARTIFACTS = {
"models": ROOT / "models",
"datasets_raw": ROOT / "datasets" / "raw",
"datasets_prepared": ROOT / "datasets" / "prepared",
"1c_rag_sources": ROOT / "plugins" / "1c" / "rag" / "sources",
"1c_rag_official_raw": ROOT / "plugins" / "1c" / "rag" / "official-docs" / "raw",
"1c_rag_official_normalized": ROOT / "plugins" / "1c" / "rag" / "official-docs" / "normalized",
"1c_rag_official_media": ROOT / "plugins" / "1c" / "rag" / "official-docs" / "media",
"1c_rag_official_static": ROOT / "plugins" / "1c" / "rag" / "official-docs" / "static",
"1c_rag_official_start_links": ROOT / "plugins" / "1c" / "rag" / "official-docs" / "start-links.json",
"1c_rag_official_platform_versions": ROOT / "plugins" / "1c" / "rag" / "official-docs" / "platform-versions.json",
"1c_datasets_raw": ROOT / "plugins" / "1c" / "datasets" / "raw",
"1c_datasets_prepared": ROOT / "plugins" / "1c" / "datasets" / "prepared",
"1c_metadata_snapshots": ROOT / "plugins" / "1c" / "metadata" / "snapshots",
"1c_training_raw": ROOT / "plugins" / "1c" / "training" / "raw",
"1c_training_prepared": ROOT / "plugins" / "1c" / "training" / "prepared",
}
SECRET_NAME_MARKERS = ("cookie", "secret", "password", "passwd", "credential", ".env", ".pem", ".pfx", ".p12", ".key")
KNOWN_SAFE_TOKEN_FILES = ("tokenizer", "special_tokens", "added_tokens")
def sha256_file(path: Path) -> str:
digest = hashlib.sha256()
with path.open("rb") as handle:
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
digest.update(chunk)
return digest.hexdigest()
def is_secret_like(path: Path) -> bool:
lowered = path.name.lower()
if any(marker in lowered for marker in KNOWN_SAFE_TOKEN_FILES):
return False
return any(marker in lowered for marker in SECRET_NAME_MARKERS)
def iter_files(path: Path) -> list[Path]:
if not path.exists():
return []
return sorted(item for item in path.rglob("*") if item.is_file())
def summarize_artifact(name: str, path: Path, *, hash_files: bool, max_files: int) -> dict[str, Any]:
files = iter_files(path)
total_size = sum(item.stat().st_size for item in files)
suffix_counts: dict[str, int] = {}
secret_like = []
file_records = []
for item in files:
suffix = item.suffix.lower() or "<no_ext>"
suffix_counts[suffix] = suffix_counts.get(suffix, 0) + 1
if is_secret_like(item):
secret_like.append(str(item.relative_to(path)))
if len(file_records) < max_files:
stat = item.stat()
record = {
"relative_path": str(item.relative_to(path)).replace("\\", "/"),
"size_bytes": stat.st_size,
"mtime_unix": int(stat.st_mtime),
}
if hash_files:
record["sha256"] = sha256_file(item)
file_records.append(record)
return {
"name": name,
"path": str(path),
"exists": path.exists(),
"is_dir": path.is_dir(),
"file_count": len(files),
"total_size_bytes": total_size,
"suffix_counts": dict(sorted(suffix_counts.items())),
"secret_like_files": secret_like[:50],
"secret_like_truncated": len(secret_like) > 50,
"files_sampled": len(file_records),
"files_truncated": len(files) > max_files,
"files": file_records,
}
def build_manifest(*, hash_files: bool, max_files: int, artifacts: dict[str, Path]) -> dict[str, Any]:
records = [summarize_artifact(name, path, hash_files=hash_files, max_files=max_files) for name, path in artifacts.items()]
return {
"schema": "llm_artifact_manifest.v1",
"created_at": dt.datetime.now(dt.UTC).isoformat(),
"workspace_root": str(ROOT),
"hash_files": hash_files,
"max_files_per_artifact": max_files,
"portable_policy": {
"store_large_artifacts_outside_git": True,
"mount_artifacts_as_docker_volumes": True,
"do_not_store_secrets_in_artifact_dirs": True,
},
"docker_volume_recommendations": [
{"host": "models", "container": "/app/models", "required_for": ["model-inference", "training"]},
{"host": "plugins/1c/datasets", "container": "/app/plugins/1c/datasets", "required_for": ["1c-rag-api", "1c-adapter-api"]},
{"host": "plugins/1c/rag/sources", "container": "/app/plugins/1c/rag/sources", "required_for": ["1c-rag-api"]},
{"host": "plugins/1c/rag/official-docs", "container": "/app/plugins/1c/rag/official-docs", "required_for": ["1c-official-docs-ingest"]},
{"host": "plugins/1c/metadata/snapshots", "container": "/app/plugins/1c/metadata/snapshots", "required_for": ["1c-adapter-api"]},
],
"counts": {
"artifacts": len(records),
"existing": sum(1 for item in records if item["exists"]),
"files": sum(int(item["file_count"]) for item in records),
"total_size_bytes": sum(int(item["total_size_bytes"]) for item in records),
"secret_like_files": sum(len(item["secret_like_files"]) for item in records),
},
"artifacts": records,
}
def main() -> int:
parser = argparse.ArgumentParser(description="Build an inventory manifest for local LLM/RAG artifacts that are intentionally outside git.")
parser.add_argument("--output", type=Path, default=DEFAULT_OUTPUT)
parser.add_argument("--hash-files", action="store_true", help="Hash sampled files. This can be slow for large model files.")
parser.add_argument("--max-files", type=int, default=500, help="Maximum file records stored per artifact.")
args = parser.parse_args()
manifest = build_manifest(hash_files=args.hash_files, max_files=args.max_files, artifacts=DEFAULT_ARTIFACTS)
args.output.parent.mkdir(parents=True, exist_ok=True)
args.output.write_text(json.dumps(manifest, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
print(
json.dumps(
{
"output": str(args.output),
"artifacts": manifest["counts"]["artifacts"],
"files": manifest["counts"]["files"],
"total_size_bytes": manifest["counts"]["total_size_bytes"],
"secret_like_files": manifest["counts"]["secret_like_files"],
},
ensure_ascii=False,
)
)
return 0
if __name__ == "__main__":
raise SystemExit(main())
+51
View File
@@ -0,0 +1,51 @@
from __future__ import annotations
import json
from common import ROOT, iter_model_card_paths, read_yaml_mapping
INDEX_PATH = ROOT / "registry" / "index.json"
def simplify_card(path: Path, data: dict) -> dict:
deployment = data.get("deployment") or {}
return {
"id": data.get("id"),
"name": data.get("name"),
"type": data.get("type"),
"status": data.get("status"),
"task": data.get("task") or [],
"language": data.get("language") or [],
"source": data.get("source"),
"upstream_id": data.get("upstream_id"),
"license": data.get("license"),
"storage_path": data.get("storage_path"),
"format": data.get("format"),
"quantization": data.get("quantization"),
"runtime": deployment.get("runtime"),
"served_model_name": deployment.get("served_model_name"),
"card_path": str(path.relative_to(ROOT)).replace("\\", "/"),
}
def main() -> int:
models = []
for path in iter_model_card_paths():
models.append(simplify_card(path, read_yaml_mapping(path)))
index = {
"schema_version": 1,
"models": models,
}
INDEX_PATH.write_text(
json.dumps(index, ensure_ascii=False, indent=2) + "\n",
encoding="utf-8",
)
print(f"Wrote {INDEX_PATH.relative_to(ROOT)} with {len(models)} model(s).")
return 0
if __name__ == "__main__":
raise SystemExit(main())
+353
View File
@@ -0,0 +1,353 @@
from __future__ import annotations
import argparse
import json
import re
from collections import defaultdict
from datetime import datetime
from pathlib import Path
from typing import Any
from urllib import request
MARKER_RE = re.compile(r"^\d+-\d+$")
def rpc(adapter_url: str, method: str, payload: dict[str, Any]) -> dict[str, Any]:
body = json.dumps({"method": method, "payload": payload}, ensure_ascii=False).encode("utf-8")
req = request.Request(
f"{adapter_url.rstrip('/')}/rpc",
data=body,
headers={"Content-Type": "application/json; charset=utf-8"},
method="POST",
)
with request.urlopen(req, timeout=240) as resp:
return json.loads(resp.read().decode("utf-8", errors="replace"))
def latest_configcas_rows(adapter_url: str, base_id: str, limit: int) -> list[dict[str, Any]]:
result = rpc(
adapter_url,
"query.run",
{
"base_id": base_id,
"diagnostic": True,
"query": (
f"SELECT TOP {int(limit)} FileName, DATALENGTH(BinaryData) AS Bytes, PartNo, Creation, Modified "
"FROM ConfigCAS ORDER BY Modified DESC"
),
"timeout_seconds": 120,
},
)
return result.get("rows") or []
def template_summary(adapter_url: str, base_id: str, file_name: str, max_cells: int) -> dict[str, Any]:
result = rpc(
adapter_url,
"templates.map",
{
"base_id": base_id,
"table": "ConfigCAS",
"file_name": file_name,
"view": "summary",
"sections": "cells,styles,named_areas,named_ranges,diagnostics",
"max_cells": max_cells,
"max_areas": 200,
"timeout_seconds": 120,
},
)
templates = result.get("templates") or []
if not templates:
return {}
return (templates[0] or {}).get("structure") or {}
def choose_latest_moxel(adapter_url: str, base_id: str, limit: int, max_cells: int, explicit_file_name: str | None) -> tuple[str, dict[str, Any], dict[str, Any]]:
if explicit_file_name:
structure = template_summary(adapter_url, base_id, explicit_file_name, max_cells)
return explicit_file_name, {"FileName": explicit_file_name}, structure
for row in latest_configcas_rows(adapter_url, base_id, limit):
file_name = str(row.get("FileName") or "")
byte_count = int(row.get("Bytes") or 0)
if not file_name or byte_count <= 0 or byte_count > 20000:
continue
structure = template_summary(adapter_url, base_id, file_name, max_cells)
if str(structure.get("format") or "") == "MOXCEL":
return file_name, row, structure
raise RuntimeError("Could not find a recent MOXCEL payload in ConfigCAS.")
def normalize_cell(cell: dict[str, Any]) -> dict[str, Any]:
return {
"text": cell.get("text"),
"cell_id": cell.get("cell_id"),
"type_code": cell.get("type_code"),
"one_based": cell.get("one_based"),
"zero_based": cell.get("zero_based"),
"parameter": cell.get("parameter"),
"reference": cell.get("reference"),
"source": cell.get("source"),
}
def normalize_style(item: dict[str, Any]) -> dict[str, Any]:
next_record = item.get("next_moxel_record") if isinstance(item.get("next_moxel_record"), dict) else None
style = item.get("style_evidence") if isinstance(item.get("style_evidence"), dict) else {}
return {
"text": item.get("text"),
"cell_id": item.get("cell_id"),
"type_code": item.get("type_code"),
"tree_position": item.get("tree_position"),
"next_moxel_record": next_record,
"immediate_preceding_values": style.get("immediate_preceding_values"),
"last_7_preceding_values": style.get("last_7_preceding_values"),
}
def build_snapshot(structure: dict[str, Any]) -> dict[str, Any]:
cells = [normalize_cell(item) for item in (structure.get("cells") or []) if isinstance(item, dict) and item.get("text")]
styles = [normalize_style(item) for item in (structure.get("cell_style_candidates") or []) if isinstance(item, dict) and item.get("text")]
named_ranges = []
for item in structure.get("named_range_candidates") or []:
if not isinstance(item, dict):
continue
named_ranges.append(
{
"name": item.get("name"),
"kind": item.get("kind"),
"tree_position": item.get("tree_position"),
"range": item.get("range"),
"raw_scalars": ((item.get("range_candidate") or {}).get("raw_scalars") if isinstance(item.get("range_candidate"), dict) else None),
}
)
cells_by_text: dict[str, list[dict[str, Any]]] = defaultdict(list)
for cell in cells:
cells_by_text[str(cell.get("text"))].append(cell)
styles_by_text: dict[str, list[dict[str, Any]]] = defaultdict(list)
for style in styles:
styles_by_text[str(style.get("text"))].append(style)
marker_matrix = []
for text, entries in sorted(cells_by_text.items(), key=lambda pair: tuple(map(int, pair[0].split("-"))) if MARKER_RE.fullmatch(pair[0]) else (10**9, 10**9)):
if not MARKER_RE.fullmatch(text):
continue
expected_row, expected_col = map(int, text.split("-"))
style_entries = styles_by_text.get(text) or []
row: dict[str, Any] = {
"text": text,
"expected_row": expected_row,
"expected_col": expected_col,
"cells": entries,
"styles": style_entries,
}
if entries:
first = entries[0]
one_based = first.get("one_based") or {}
decoded_row = one_based.get("row")
decoded_col = one_based.get("column")
row["decoded_row"] = decoded_row
row["decoded_col"] = decoded_col
row["row_ok"] = decoded_row == expected_row
row["col_ok"] = decoded_col == expected_col
if isinstance(decoded_col, int):
row["col_delta"] = decoded_col - expected_col
marker_matrix.append(row)
return {
"counts": structure.get("counts") or {},
"dimensions": structure.get("dimensions"),
"named_areas": structure.get("named_areas") or [],
"named_ranges": named_ranges,
"cells": cells,
"cell_styles": styles,
"cells_by_text": dict(cells_by_text),
"styles_by_text": dict(styles_by_text),
"marker_matrix": marker_matrix,
}
def read_json(path: Path) -> dict[str, Any]:
return json.loads(path.read_text(encoding="utf-8"))
def diff_simple(before: Any, after: Any) -> dict[str, Any] | None:
if before == after:
return None
return {"before": before, "after": after}
def index_text_entries(entries: dict[str, list[dict[str, Any]]]) -> dict[str, list[dict[str, Any]]]:
indexed: dict[str, list[dict[str, Any]]] = {}
for text, items in entries.items():
indexed[text] = sorted(items, key=lambda item: json.dumps(item, ensure_ascii=False, sort_keys=True))
return indexed
def build_diff(before: dict[str, Any], after: dict[str, Any]) -> dict[str, Any]:
result: dict[str, Any] = {
"counts": diff_simple(before.get("counts"), after.get("counts")),
"dimensions": diff_simple(before.get("dimensions"), after.get("dimensions")),
}
before_cells = index_text_entries(before.get("cells_by_text") or {})
after_cells = index_text_entries(after.get("cells_by_text") or {})
before_styles = index_text_entries(before.get("styles_by_text") or {})
after_styles = index_text_entries(after.get("styles_by_text") or {})
changed_cells: dict[str, Any] = {}
for text in sorted(set(before_cells) | set(after_cells)):
if before_cells.get(text) != after_cells.get(text):
changed_cells[text] = {"before": before_cells.get(text), "after": after_cells.get(text)}
changed_styles: dict[str, Any] = {}
for text in sorted(set(before_styles) | set(after_styles)):
if before_styles.get(text) != after_styles.get(text):
changed_styles[text] = {"before": before_styles.get(text), "after": after_styles.get(text)}
before_markers = {item["text"]: item for item in before.get("marker_matrix") or [] if isinstance(item, dict) and item.get("text")}
after_markers = {item["text"]: item for item in after.get("marker_matrix") or [] if isinstance(item, dict) and item.get("text")}
changed_markers: dict[str, Any] = {}
for text in sorted(set(before_markers) | set(after_markers), key=lambda value: tuple(map(int, value.split("-"))) if MARKER_RE.fullmatch(value) else (10**9, 10**9)):
if before_markers.get(text) != after_markers.get(text):
changed_markers[text] = {"before": before_markers.get(text), "after": after_markers.get(text)}
before_named = {f"{item.get('kind')}::{item.get('name')}": item for item in before.get("named_ranges") or [] if isinstance(item, dict)}
after_named = {f"{item.get('kind')}::{item.get('name')}": item for item in after.get("named_ranges") or [] if isinstance(item, dict)}
changed_named: dict[str, Any] = {}
for key in sorted(set(before_named) | set(after_named)):
if before_named.get(key) != after_named.get(key):
changed_named[key] = {"before": before_named.get(key), "after": after_named.get(key)}
result["changed_cells_by_text"] = changed_cells
result["changed_styles_by_text"] = changed_styles
result["changed_marker_matrix"] = changed_markers
result["changed_named_ranges"] = changed_named
result["summary"] = {
"changed_cell_texts": len(changed_cells),
"changed_style_texts": len(changed_styles),
"changed_markers": len(changed_markers),
"changed_named_ranges": len(changed_named),
}
return result
def latest_previous_snapshot(output_dir: Path, current_path: Path) -> Path | None:
candidates = sorted(output_dir.glob("*.json"))
filtered = [path for path in candidates if path.resolve() != current_path.resolve()]
return filtered[-1] if filtered else None
def render_markdown(snapshot: dict[str, Any], diff: dict[str, Any] | None, previous_path: Path | None) -> str:
lines: list[str] = []
lines.append("# 1C template probe snapshot")
lines.append("")
lines.append(f"- Base: `{snapshot['base_id']}`")
lines.append(f"- File: `{snapshot['file_name']}`")
lines.append(f"- Modified: `{snapshot.get('modified')}`")
lines.append(f"- Bytes: `{snapshot.get('bytes')}`")
lines.append(f"- Previous snapshot: `{previous_path.name}`" if previous_path else "- Previous snapshot: none")
lines.append("")
lines.append("## Marker matrix")
lines.append("")
lines.append("| Marker | Decoded | Result | Style tree |")
lines.append("| --- | --- | --- | --- |")
for item in snapshot["probe"]["marker_matrix"]:
decoded = f"R{item.get('decoded_row')}C{item.get('decoded_col')}" if item.get("decoded_row") else "n/a"
if item.get("row_ok") is True and item.get("col_ok") is True:
result = "ok"
elif item.get("decoded_row") is None:
result = "style-only"
else:
result = f"row_ok={item.get('row_ok')} col_ok={item.get('col_ok')} delta={item.get('col_delta')}"
style_tree = ""
styles = item.get("styles") or []
if styles:
style_tree = ", ".join(str(style.get("tree_position")) for style in styles if style.get("tree_position"))
lines.append(f"| `{item['text']}` | `{decoded}` | `{result}` | `{style_tree}` |")
if diff:
lines.append("")
lines.append("## Diff summary")
lines.append("")
lines.append("```json")
lines.append(json.dumps(diff.get("summary") or {}, ensure_ascii=False, indent=2))
lines.append("```")
return "\n".join(lines) + "\n"
def main() -> int:
parser = argparse.ArgumentParser(description="Capture and diff a live 1C MOXCEL template probe snapshot.")
parser.add_argument("--adapter-url", default="http://docker-gpu.cin.su:8011")
parser.add_argument("--base-id", default="upo_test")
parser.add_argument("--file-name", help="Explicit ConfigCAS file name. If omitted, use the newest MOXCEL payload.")
parser.add_argument("--scan-limit", type=int, default=30)
parser.add_argument("--max-cells", type=int, default=500)
parser.add_argument(
"--output-dir",
default=str(Path("Z:/codex/LLM/reports/1c-template-probes")),
help="Directory for snapshot JSON/Markdown files.",
)
parser.add_argument("--output-json", help="Optional exact JSON output path. Overrides generated timestamped name.")
parser.add_argument("--output-markdown", help="Optional exact Markdown output path. Overrides generated timestamped name.")
parser.add_argument("--compare-to", help="Optional previous snapshot JSON path.")
parser.add_argument("--label", default="latest", help="Short label appended to the output file name.")
args = parser.parse_args()
output_dir = Path(args.output_dir)
output_dir.mkdir(parents=True, exist_ok=True)
file_name, row, structure = choose_latest_moxel(args.adapter_url, args.base_id, args.scan_limit, args.max_cells, args.file_name)
snapshot = {
"schema": "codex_1c_template_probe_snapshot.v1",
"captured_at": datetime.utcnow().isoformat(timespec="seconds") + "Z",
"adapter_url": args.adapter_url,
"base_id": args.base_id,
"file_name": file_name,
"modified": row.get("Modified"),
"bytes": row.get("Bytes"),
"label": args.label,
"probe": build_snapshot(structure),
}
timestamp = datetime.utcnow().strftime("%Y%m%dT%H%M%SZ")
stem = f"{args.base_id}_{args.label}_{timestamp}_{file_name[:8]}"
json_path = Path(args.output_json) if args.output_json else output_dir / f"{stem}.json"
md_path = Path(args.output_markdown) if args.output_markdown else output_dir / f"{stem}.md"
json_path.parent.mkdir(parents=True, exist_ok=True)
md_path.parent.mkdir(parents=True, exist_ok=True)
previous_path = Path(args.compare_to) if args.compare_to else latest_previous_snapshot(output_dir, json_path)
diff: dict[str, Any] | None = None
if previous_path and previous_path.exists():
previous = read_json(previous_path)
diff = build_diff(previous.get("probe") or {}, snapshot.get("probe") or {})
snapshot["diff"] = {
"compare_to": str(previous_path),
"summary": diff.get("summary") or {},
}
json_path.write_text(json.dumps(snapshot, ensure_ascii=False, indent=2), encoding="utf-8")
md_path.write_text(render_markdown(snapshot, diff, previous_path if previous_path and previous_path.exists() else None), encoding="utf-8")
print(
json.dumps(
{
"status": "ok",
"json": str(json_path),
"markdown": str(md_path),
"file_name": file_name,
"modified": row.get("Modified"),
"bytes": row.get("Bytes"),
"diff_summary": (diff.get("summary") if diff else None),
},
ensure_ascii=False,
indent=2,
)
)
return 0
if __name__ == "__main__":
raise SystemExit(main())
@@ -0,0 +1,403 @@
#!/usr/bin/env python3
from __future__ import annotations
import argparse
import json
import subprocess
import sys
from pathlib import Path
from typing import Any
ROOT = Path(__file__).resolve().parents[1]
DEFAULT_REST_ADAPTER_URL = "http://docker-gpu.cin.su:8011"
DEFAULT_MCP_URL = "http://docker.cin.su:8021"
SAVED_STATE_TABLES = ("ConfigSave", "ConfigCASSave")
def duplicate_values(values: list[str]) -> list[str]:
seen: set[str] = set()
duplicates: set[str] = set()
for value in values:
if value in seen:
duplicates.add(value)
seen.add(value)
return sorted(duplicates)
def trim_output(value: str, max_chars: int) -> tuple[str, bool]:
if max_chars <= 0 or len(value) <= max_chars:
return value, False
return value[-max_chars:], True
def parse_json_output(value: str) -> dict[str, Any] | None:
text = value.strip()
if not text or not text.startswith("{"):
return None
try:
parsed = json.loads(text)
except json.JSONDecodeError:
return None
return parsed if isinstance(parsed, dict) else None
def parsed_summary(parsed: dict[str, Any] | None) -> dict[str, Any] | None:
if not isinstance(parsed, dict):
return None
result: dict[str, Any] = {"schema": parsed.get("schema")}
for key in ("passed", "status", "failures", "issues"):
if key in parsed:
value = parsed.get(key)
if isinstance(value, list):
result[f"{key}_count"] = len(value)
else:
result[key] = value
checks = parsed.get("checks")
if isinstance(checks, dict):
result["checks_count"] = len(checks)
if all(isinstance(value, bool) for value in checks.values()):
result["checks_failed"] = sorted(key for key, value in checks.items() if value is not True)
counts = parsed.get("counts")
if isinstance(counts, dict):
result["counts"] = counts
bases = parsed.get("bases")
if isinstance(bases, dict):
result["bases_count"] = len(bases)
result["bases"] = {
str(base_id): compact_base_summary(base)
for base_id, base in bases.items()
if isinstance(base, dict)
}
strict_codes = parsed.get("strict_skip_failure_codes")
if isinstance(strict_codes, list):
result["strict_skip_failure_codes"] = strict_codes
coverage_codes = parsed.get("coverage_failure_codes")
if isinstance(coverage_codes, list):
result["coverage_failure_codes"] = coverage_codes
consistency_codes = parsed.get("consistency_failure_codes")
if isinstance(consistency_codes, list):
result["consistency_failure_codes"] = consistency_codes
safety_codes = parsed.get("safety_failure_codes")
if isinstance(safety_codes, list):
result["safety_failure_codes"] = safety_codes
rollback_safety_codes = parsed.get("rollback_safety_failure_codes")
if isinstance(rollback_safety_codes, list):
result["rollback_safety_failure_codes"] = rollback_safety_codes
saved_state_diff_codes = parsed.get("saved_state_diff_failure_codes")
if isinstance(saved_state_diff_codes, list):
result["saved_state_diff_failure_codes"] = saved_state_diff_codes
schema_codes = parsed.get("schema_failure_codes")
if isinstance(schema_codes, list):
result["schema_failure_codes"] = schema_codes
identity_codes = parsed.get("identity_failure_codes")
if isinstance(identity_codes, list):
result["identity_failure_codes"] = identity_codes
endpoint_codes = parsed.get("endpoint_failure_codes")
if isinstance(endpoint_codes, list):
result["endpoint_failure_codes"] = endpoint_codes
duplicate_codes = parsed.get("duplicate_failure_codes")
if isinstance(duplicate_codes, list):
result["duplicate_failure_codes"] = duplicate_codes
saved_state_codes = parsed.get("saved_state_failure_codes")
if isinstance(saved_state_codes, list):
result["saved_state_failure_codes"] = saved_state_codes
saved_state_strict_readiness_codes = parsed.get("saved_state_strict_readiness_failure_codes")
if isinstance(saved_state_strict_readiness_codes, list):
result["saved_state_strict_readiness_failure_codes"] = saved_state_strict_readiness_codes
saved_state_copy_plan_codes = parsed.get("saved_state_copy_plan_failure_codes")
if isinstance(saved_state_copy_plan_codes, list):
result["saved_state_copy_plan_failure_codes"] = saved_state_copy_plan_codes
saved_state_table_codes = parsed.get("saved_state_table_failure_codes")
if isinstance(saved_state_table_codes, list):
result["saved_state_table_failure_codes"] = saved_state_table_codes
staleness_codes = parsed.get("staleness_failure_codes")
if isinstance(staleness_codes, list):
result["staleness_failure_codes"] = staleness_codes
return result
def compact_base_summary(base: dict[str, Any]) -> dict[str, Any]:
reports = base.get("reports") if isinstance(base.get("reports"), dict) else {}
summary: dict[str, Any] = {"passed": base.get("passed")}
selector_chain: dict[str, Any] = {}
write_plan_safety: dict[str, Any] = {}
write_rollback_safety: dict[str, Any] = {}
saved_state_diff: dict[str, Any] = {}
saved_state: dict[str, Any] = {}
for name, report in reports.items():
if not isinstance(report, dict):
continue
if name.startswith("selector_chain_"):
transport = name.removeprefix("selector_chain_")
selector_chain[transport] = {
"passed": report.get("passed"),
"base_id": report.get("base_id"),
"transport": report.get("transport"),
"endpoint_url": report.get("endpoint_url"),
"resolve_overrides_status": report.get("resolve_overrides_status"),
"write_plan_evidence": report.get("write_plan_evidence"),
"next_method": report.get("next_method"),
"saved_state_status": report.get("saved_state_status"),
"saved_state_modules": report.get("saved_state_modules"),
"write_plan_target": report.get("write_plan_target"),
"composition_status": report.get("composition_status"),
"composed": report.get("composed"),
}
elif name.startswith("write_plan_safety_"):
transport = name.removeprefix("write_plan_safety_")
write_plan_safety[transport] = {
"status": report.get("status"),
"checks": report.get("checks"),
"base_id": report.get("base_id"),
"transport": report.get("transport"),
"endpoint_url": report.get("endpoint_url"),
}
elif name.startswith("write_rollback_safety_"):
transport = name.removeprefix("write_rollback_safety_")
write_rollback_safety[transport] = {
"status": report.get("status"),
"checks": report.get("checks"),
"base_id": report.get("base_id"),
"transport": report.get("transport"),
"endpoint_url": report.get("endpoint_url"),
}
elif name.startswith("saved_state_diff_"):
transport = name.removeprefix("saved_state_diff_")
saved_state_diff[transport] = {
"status": report.get("status"),
"checks": report.get("checks"),
"base_id": report.get("base_id"),
"transport": report.get("transport"),
"endpoint_url": report.get("endpoint_url"),
"saved_state_table": report.get("saved_state_table"),
"diff_status": report.get("diff_status"),
"needs_prepare": report.get("needs_prepare"),
}
elif name == "saved_state_form_write":
saved_state["form"] = {
"passed": report.get("passed"),
"status": report.get("status"),
"base_id": report.get("base_id"),
"table": report.get("table"),
"routes": report.get("routes"),
"preflight_status": report.get("preflight_status"),
"preflight_counts": report.get("preflight_counts"),
}
elif name == "saved_state_module_write":
saved_state["module"] = {
"status": report.get("status"),
"base_id": report.get("base_id"),
"table": report.get("table"),
"module_ref": report.get("module_ref"),
"write_plan_allowed": report.get("write_plan_allowed"),
"preflight_status": report.get("preflight_status"),
"preflight_counts": report.get("preflight_counts"),
}
elif name == "saved_state_strict_readiness":
saved_state["strict_readiness"] = {
"status": report.get("status"),
"ready": report.get("ready"),
"base_id": report.get("base_id"),
"table": report.get("table"),
"tables": report.get("tables"),
"saved_state_rows": report.get("saved_state_rows"),
"forms": report.get("forms"),
"modules": report.get("modules"),
}
elif name == "saved_state_copy_plan":
source_family = report.get("source_family") if isinstance(report.get("source_family"), dict) else {}
saved_state["copy_plan"] = {
"status": report.get("status"),
"ready_to_copy": report.get("ready_to_copy"),
"base_id": report.get("base_id"),
"target_table": report.get("target_table"),
"source_family": {
"expected_source_table": source_family.get("expected_source_table"),
"source_tables": source_family.get("source_tables"),
"valid": source_family.get("valid"),
},
"source_rows": report.get("source_rows"),
"found_source_storage_rows": report.get("found_source_storage_rows"),
"target_collision_status": report.get("target_collision_status"),
"target_collision_rows": report.get("target_collision_rows"),
}
elif name in {"saved_state_prepare_sql", "saved_state_cleanup_sql"}:
key = "prepare_sql" if name == "saved_state_prepare_sql" else "cleanup_sql"
saved_state[key] = {
"status": report.get("status"),
"base_id": report.get("base_id"),
"table": report.get("table"),
"source_table": report.get("source_table"),
"read_only": report.get("read_only"),
"sql_write_performed": report.get("sql_write_performed"),
}
if selector_chain:
summary["selector_chain"] = selector_chain
if write_plan_safety:
summary["write_plan_safety"] = write_plan_safety
if write_rollback_safety:
summary["write_rollback_safety"] = write_rollback_safety
if saved_state_diff:
summary["saved_state_diff"] = saved_state_diff
if saved_state:
summary["saved_state"] = saved_state
return summary
def run(command: list[str], *, stream: bool, max_output_chars: int) -> dict[str, Any]:
label = " ".join(command)
if stream:
print(f"\n== {label}", flush=True)
result = subprocess.run(command, cwd=ROOT, text=True, check=False)
return {"label": label, "command": command, "returncode": result.returncode}
result = subprocess.run(command, cwd=ROOT, text=True, capture_output=True, check=False)
parsed = parse_json_output(result.stdout)
stdout, stdout_truncated = trim_output(result.stdout, max_output_chars)
stderr, stderr_truncated = trim_output(result.stderr, max_output_chars)
report = {
"label": label,
"command": command,
"returncode": result.returncode,
"stdout": stdout,
"stderr": stderr,
"stdout_truncated": stdout_truncated,
"stderr_truncated": stderr_truncated,
}
summary = parsed_summary(parsed)
if summary is not None:
report["parsed"] = summary
return report
def main() -> int:
parser = argparse.ArgumentParser(description="Run offline/static checks for the 1C adapter verification stack.")
parser.add_argument("--base-id", nargs="+", default=["upo_test"], help="Base id(s) whose persisted verify reports should be checked.")
parser.add_argument("--skip-persisted-reports", action="store_true", help="Skip checking reports/1c-sql/<base-id> artifacts.")
parser.add_argument("--require-saved-state-write-smoke", action="store_true", help="Require persisted saved-state smoke reports to contain real write checks.")
parser.add_argument("--require-selector-chain-write-plan-composition", action="store_true", help="Require persisted selector-chain reports to contain composed metadata.write.plan coverage.")
parser.add_argument("--rest-adapter-url", default=DEFAULT_REST_ADAPTER_URL, help="Expected REST adapter endpoint_url in persisted reports.")
parser.add_argument("--mcp-url", default=DEFAULT_MCP_URL, help="Expected MCP proxy endpoint_url in persisted reports.")
parser.add_argument("--saved-state-table", choices=SAVED_STATE_TABLES, default="ConfigSave", help="Expected saved-state target table in persisted copy-plan/form/module reports.")
parser.add_argument("--max-report-age-seconds", type=int, help="Fail if any checked persisted report file is older than this many seconds.")
parser.add_argument("--report", type=Path, help="Optional JSON report path.")
parser.add_argument("--json", action="store_true", help="Print JSON summary without streaming child command output.")
parser.add_argument("--max-output-chars", type=int, default=4000, help="Maximum stdout/stderr characters retained per child command in JSON mode.")
args = parser.parse_args()
duplicate_base_ids = duplicate_values(args.base_id)
if duplicate_base_ids:
report = {
"schema": "onec_adapter_verification_stack_check.v1",
"passed": False,
"base_id": args.base_id[0] if len(args.base_id) == 1 else None,
"base_ids": args.base_id,
"rest_adapter_url": args.rest_adapter_url,
"mcp_url": args.mcp_url,
"saved_state_table": args.saved_state_table,
"checks": [],
"failures": [{"code": "duplicate_base_id", "base_ids": duplicate_base_ids}],
}
if args.report:
report_path = args.report if args.report.is_absolute() else ROOT / args.report
report_path.parent.mkdir(parents=True, exist_ok=True)
report_path.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
if args.json:
print(json.dumps(report, ensure_ascii=False, indent=2))
else:
print("Duplicate --base-id value(s): " + ", ".join(duplicate_base_ids), file=sys.stderr)
return 1
commands = [
[
sys.executable,
"-m",
"py_compile",
"scripts/check_1c_adapter_verification_stack.py",
"scripts/check_1c_verify_reports.py",
"scripts/check_powershell_scripts.py",
"scripts/smoke_1c_mcp_selector_chain.py",
"scripts/smoke_1c_write_plan_safety.py",
"scripts/smoke_1c_write_preflight.py",
"scripts/smoke_1c_write_rollback_safety.py",
"scripts/smoke_1c_saved_state_diff.py",
"scripts/smoke_1c_saved_state_changes.py",
"scripts/smoke_1c_saved_state_write_routes.py",
"scripts/smoke_1c_saved_state_module_write.py",
"scripts/check_1c_saved_state_strict_readiness.py",
"scripts/plan_1c_saved_state_copy.py",
"scripts/prepare_1c_saved_state_copy_sql.py",
"scripts/prepare_1c_saved_state_cleanup_sql.py",
"scripts/verify_1c_saved_state_copy.py",
],
[sys.executable, "scripts/check_powershell_scripts.py"],
[sys.executable, "scripts/check_1c_mcp_adapter_contract.py", "--json"],
[sys.executable, "scripts/check_1c_extension_action_contract.py", "--print"],
[sys.executable, "scripts/check_1c_write_plan_contract.py", "--print"],
[sys.executable, "scripts/smoke_1c_mcp_selector_chain.py", "--json", "--no-report"],
[sys.executable, "scripts/check_1c_verify_reports.py", "--self-test", "--json"],
]
if not args.skip_persisted_reports:
report_command = [
sys.executable,
"scripts/check_1c_verify_reports.py",
"--base-id",
*args.base_id,
"--rest-adapter-url",
args.rest_adapter_url,
"--mcp-url",
args.mcp_url,
"--saved-state-table",
args.saved_state_table,
"--json",
]
if args.require_saved_state_write_smoke:
report_command.append("--require-saved-state-write-smoke")
if args.require_selector_chain_write_plan_composition:
report_command.append("--require-selector-chain-write-plan-composition")
if args.max_report_age_seconds is not None:
report_command.extend(["--max-report-age-seconds", str(args.max_report_age_seconds)])
commands.append(report_command)
results: list[dict[str, Any]] = []
failures: list[str] = []
for command in commands:
result = run(command, stream=not args.json, max_output_chars=args.max_output_chars)
results.append(result)
if result["returncode"] != 0:
failures.append(str(result["label"]))
report = {
"schema": "onec_adapter_verification_stack_check.v1",
"passed": not failures,
"base_id": args.base_id[0] if len(args.base_id) == 1 else None,
"base_ids": args.base_id,
"rest_adapter_url": args.rest_adapter_url,
"mcp_url": args.mcp_url,
"saved_state_table": args.saved_state_table,
"checks": results,
"failures": failures,
}
if args.report:
report_path = args.report if args.report.is_absolute() else ROOT / args.report
report_path.parent.mkdir(parents=True, exist_ok=True)
report_path.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
if args.json:
print(json.dumps(report, ensure_ascii=False, indent=2))
return 0 if report["passed"] else 1
if failures:
print("\n1C adapter verification stack check failed:", file=sys.stderr)
for failure in failures:
print(f"- {failure}", file=sys.stderr)
return 1
print("\n1C adapter verification stack checks passed.")
return 0
if __name__ == "__main__":
raise SystemExit(main())
+119
View File
@@ -0,0 +1,119 @@
from __future__ import annotations
import argparse
import json
import sys
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "scripts"))
from build_1c_agent_intake import build_intake # noqa: E402
DEFAULT_INDEX = ROOT / "reports" / "1c-sql" / "upo" / "unified-object-route-index.json"
CASES = [
{
"id": "docs_intake_uses_official_scope",
"question": "Как работает событие ПриОткрытии формы?",
"expected_route": "docs_rag",
"expected_code_allowed": True,
"expected_safe_scope": "official_1c_docs",
},
{
"id": "example_fact_requires_confirmation",
"question": "В примере RAG есть реквизит Артикул у справочника Номенклатура. Напиши код для текущей базы.",
"expected_route": "mixed_docs_and_current_config",
"expected_examples_are_facts": False,
"expected_confirmed_path": "Справочник.Номенклатура.Артикул",
"expected_code_allowed": True,
},
{
"id": "missing_fact_blocks_code",
"question": "Напиши код для текущей базы: заполни Справочник.Номенклатура.ВыдуманныйРеквизит.",
"expected_route": "current_config_fact",
"expected_unresolved_path": "Справочник.Номенклатура.ВыдуманныйРеквизит",
"expected_code_allowed": False,
},
]
def run_case(case: dict, *, index: Path, view: str) -> dict:
intake = build_intake(case["question"], index=index, view=view)
failures = []
route = (intake.get("route") or {}).get("decision") or {}
policy = intake.get("answer_policy") or {}
source_policy = intake.get("source_policy") or {}
facts = intake.get("facts") or {}
if route.get("route") != case.get("expected_route"):
failures.append({"code": "route_mismatch", "expected": case.get("expected_route"), "actual": route.get("route")})
if policy.get("code_generation_allowed") is not case.get("expected_code_allowed"):
failures.append({"code": "code_policy_mismatch", "expected": case.get("expected_code_allowed"), "actual": policy.get("code_generation_allowed")})
if case.get("expected_safe_scope") and policy.get("safe_rag_scope") != case.get("expected_safe_scope"):
failures.append({"code": "safe_scope_mismatch", "expected": case.get("expected_safe_scope"), "actual": policy.get("safe_rag_scope")})
if "expected_examples_are_facts" in case and source_policy.get("examples_are_current_facts") is not case["expected_examples_are_facts"]:
failures.append({"code": "examples_policy_mismatch", "expected": case["expected_examples_are_facts"], "actual": source_policy.get("examples_are_current_facts")})
confirmed_paths = {row.get("path") for row in facts.get("confirmed") or []}
unresolved_paths = {row.get("path") for row in facts.get("unresolved") or []}
if case.get("expected_confirmed_path") and case["expected_confirmed_path"] not in confirmed_paths:
failures.append({"code": "confirmed_path_missing", "expected": case["expected_confirmed_path"], "actual": sorted(confirmed_paths)})
if case.get("expected_unresolved_path") and case["expected_unresolved_path"] not in unresolved_paths:
failures.append({"code": "unresolved_path_missing", "expected": case["expected_unresolved_path"], "actual": sorted(unresolved_paths)})
return {
"id": case["id"],
"status": "passed" if not failures else "failed",
"question": case["question"],
"failures": failures,
"summary": {
"route": route.get("route"),
"code_generation_allowed": policy.get("code_generation_allowed"),
"safe_rag_scope": policy.get("safe_rag_scope"),
"confirmed_paths": sorted(confirmed_paths),
"unresolved_paths": sorted(unresolved_paths),
},
}
def run_check(index: Path, *, view: str) -> dict:
if not index.exists():
return {
"schema": "onec_agent_intake_check.v1",
"status": "failed",
"error": f"route index not found: {index}",
"cases": [],
}
cases = [run_case(case, index=index, view=view) for case in CASES]
return {
"schema": "onec_agent_intake_check.v1",
"status": "ok" if all(case["status"] == "passed" for case in cases) else "failed",
"index": str(index),
"view": view,
"case_count": len(cases),
"cases": cases,
}
def main() -> int:
parser = argparse.ArgumentParser(description="Check 1C agent intake behavior.")
parser.add_argument("--index", type=Path, default=DEFAULT_INDEX)
parser.add_argument("--view", choices=["effective", "base"], default="effective")
parser.add_argument("--output", type=Path)
parser.add_argument("--print", action="store_true")
args = parser.parse_args()
report = run_check(args.index, view=args.view)
if args.output:
args.output.parent.mkdir(parents=True, exist_ok=True)
args.output.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
if args.print or not args.output:
print(json.dumps(report, ensure_ascii=False, indent=2))
return 0 if report["status"] == "ok" else 1
if __name__ == "__main__":
raise SystemExit(main())
+72
View File
@@ -0,0 +1,72 @@
#!/usr/bin/env python3
"""Offline contract check for conservative BSL symbol resolution."""
from __future__ import annotations
import argparse
import json
import sys
from pathlib import Path
from typing import Any
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "scripts"))
from resolve_1c_bsl_symbol import load_json, resolve_symbol # noqa: E402
METADATA = ROOT / "plugins" / "1c" / "metadata" / "examples" / "metadata-v2.example.json"
MODULES = ROOT / "plugins" / "1c" / "metadata" / "examples" / "bsl-modules.example.json"
def example_resolve(expression: str) -> dict[str, Any]:
return resolve_symbol(
load_json(METADATA),
load_json(MODULES),
expression=expression,
module_id="catalog.Номенклатура.object",
object_kind="catalog",
object_name="Номенклатура",
routine_name="ПередЗаписью",
)
def main() -> int:
parser = argparse.ArgumentParser(description="Check BSL symbol resolver safety contract.")
parser.add_argument("--print", action="store_true")
args = parser.parse_args()
full_path = example_resolve("Справочник.Номенклатура.Артикул")
context_member = example_resolve("Наименование")
parameter = example_resolve("Отказ.Код")
short_name = example_resolve("Номенклатура.ЕдИзмерение.Код")
checks = {
"full_metadata_path": full_path.get("resolution_kind") == "metadata_path"
and full_path.get("canonical_path") == "Справочник.Номенклатура.Артикул"
and full_path.get("safe_as_metadata_path") is True,
"context_standard_attribute": context_member.get("resolution_kind") == "context_metadata_member"
and context_member.get("canonical_path") == "Справочник.Номенклатура.Наименование",
"parameter_not_metadata": parameter.get("resolution_kind") == "parameter"
and parameter.get("safe_as_metadata_path") is False,
"short_name_not_metadata": short_name.get("status") == "unresolved"
and short_name.get("safe_as_metadata_path") is False
and bool(short_name.get("candidates")),
}
failures = [name for name, ok in checks.items() if not ok]
report = {
"schema": "onec_bsl_symbol_resolver_check.v1",
"status": "ok" if not failures else "failed",
"failures": failures,
"checks": checks,
}
if args.print:
print(json.dumps(report, ensure_ascii=False, indent=2))
elif failures:
print("1C BSL symbol resolver status: failed", file=sys.stderr)
else:
print("1C BSL symbol resolver status: ok")
return 0 if not failures else 1
if __name__ == "__main__":
raise SystemExit(main())
+201
View File
@@ -0,0 +1,201 @@
#!/usr/bin/env python3
"""Validate that a 1C change proposal stays within the read-only/extension-first safety contract."""
from __future__ import annotations
import argparse
import json
from pathlib import Path
from typing import Any
FORBIDDEN_PATH_PARTS = {
"config",
"configsave",
"configcas",
}
def load_json(path: Path) -> dict[str, Any]:
return json.loads(path.read_text(encoding="utf-8-sig"))
def issue(severity: str, code: str, message: str, *, target: dict[str, Any] | None = None) -> dict[str, Any]:
result = {"severity": severity, "code": code, "message": message}
if target:
result["target"] = target
return result
def path_text(target: dict[str, Any]) -> str:
return str(target.get("path") or target.get("module_path") or "")
def normalized_path_parts(path: str) -> list[str]:
return [part.casefold() for part in path.replace("/", "\\").split("\\") if part]
def is_extension_origin(target: dict[str, Any], preferred_extension: str | None) -> bool:
origin = target.get("origin") or {}
if origin.get("layer") != "extension":
return False
if preferred_extension and origin.get("extension") != preferred_extension:
return False
return True
def is_extension_path(path: str, preferred_extension: str | None) -> bool:
parts = normalized_path_parts(path)
if "расширения" not in parts and "extensions" not in parts:
return False
if preferred_extension:
lowered = preferred_extension.casefold()
return lowered in parts
return True
def forbidden_path_reason(path: str) -> str | None:
parts = normalized_path_parts(path)
for part in parts:
if part in FORBIDDEN_PATH_PARTS:
return part
if path.startswith("_") or "\\_" in path:
return "sql_physical_name_like_path"
return None
def check_target_exists(target: dict[str, Any]) -> list[dict[str, Any]]:
findings = []
path = path_text(target)
if not path:
findings.append(issue("error", "missing_path", "Target has no path.", target=target))
return findings
if not Path(path).exists():
findings.append(issue("error", "path_not_found", f"Target path does not exist: {path}", target=target))
line = target.get("line")
if line:
try:
line_int = int(line)
if line_int < 1:
findings.append(issue("error", "invalid_line", f"Invalid target line: {line}", target=target))
except (TypeError, ValueError):
findings.append(issue("error", "invalid_line", f"Invalid target line: {line}", target=target))
return findings
def check_write_candidate(target: dict[str, Any], preferred_extension: str | None) -> list[dict[str, Any]]:
findings = []
path = path_text(target)
findings.extend(check_target_exists(target))
if not is_extension_origin(target, preferred_extension):
findings.append(issue("error", "write_candidate_not_preferred_extension_origin", "Write candidate is not in the preferred extension origin.", target=target))
if not is_extension_path(path, preferred_extension):
findings.append(issue("error", "write_candidate_not_preferred_extension_path", "Write candidate path is not inside the preferred extension directory.", target=target))
forbidden = forbidden_path_reason(path)
if forbidden:
findings.append(issue("error", "forbidden_write_path", f"Write candidate path is forbidden: {forbidden}", target=target))
if target.get("kind") not in {"bsl_module", "form_xml"}:
findings.append(issue("warning", "unusual_write_candidate_kind", f"Unexpected write candidate kind: {target.get('kind')}", target=target))
return findings
def check_reference_target(target: dict[str, Any]) -> list[dict[str, Any]]:
findings = check_target_exists(target)
forbidden = forbidden_path_reason(path_text(target))
if forbidden:
findings.append(issue("warning", "forbidden_reference_path", f"Reference path is forbidden for writes and must remain read-only: {forbidden}", target=target))
return findings
def check_proposal(proposal: dict[str, Any]) -> dict[str, Any]:
findings = []
strategy = proposal.get("write_strategy") or {}
preferred_extension = strategy.get("preferred_extension")
if strategy.get("mode") != "extension_first_proposal":
findings.append(issue("error", "unsupported_write_strategy", f"Unsupported write strategy: {strategy.get('mode')}"))
if not preferred_extension:
findings.append(issue("warning", "missing_preferred_extension", "No preferred extension selected; patch generation should create/choose an extension explicitly."))
policy = proposal.get("target_policy") or {}
write_candidates = policy.get("write_candidates") or []
references = policy.get("read_only_reference_files") or []
if not write_candidates:
findings.append(issue("warning", "no_write_candidates", "No write candidates were selected."))
for target in write_candidates:
findings.extend(check_write_candidate(target, preferred_extension))
for target in references:
findings.extend(check_reference_target(target))
forbidden = set(strategy.get("forbidden") or [])
required_forbidden = {
"direct SQL metadata/data updates",
"direct Config/ConfigSave/ConfigCAS writes",
"automatic production Designer update/apply",
}
missing = sorted(required_forbidden - forbidden)
if missing:
findings.append(issue("error", "missing_forbidden_strategy_items", "Write strategy is missing forbidden items: " + ", ".join(missing)))
errors = [row for row in findings if row.get("severity") == "error"]
warnings = [row for row in findings if row.get("severity") == "warning"]
return {
"object": proposal.get("object"),
"preferred_extension": preferred_extension,
"passed": not errors,
"findings": findings,
"counts": {
"errors": len(errors),
"warnings": len(warnings),
"write_candidates": len(write_candidates),
"read_only_references": len(references),
},
}
def check(data: dict[str, Any]) -> dict[str, Any]:
proposal_checks = [check_proposal(proposal) for proposal in data.get("proposals") or []]
errors = sum(item.get("counts", {}).get("errors", 0) for item in proposal_checks)
warnings = sum(item.get("counts", {}).get("warnings", 0) for item in proposal_checks)
return {
"schema": "onec_change_proposal_safety_check.v1",
"source_schema": data.get("schema"),
"task": data.get("task"),
"passed": errors == 0,
"proposal_checks": proposal_checks,
"required_gates_before_real_write": [
"backup_gate",
"round_trip_parser_gate",
"designer_validation_gate",
"saved_state_gate",
"extension_packaging_gate",
"diff_gate",
"minimal_write_scope_gate",
"recovery_test_gate",
],
"counts": {
"proposals": len(proposal_checks),
"errors": errors,
"warnings": warnings,
},
}
def main() -> int:
parser = argparse.ArgumentParser(description="Check 1C proposal safety.")
parser.add_argument("--proposal", type=Path, required=True)
parser.add_argument("--output", type=Path)
args = parser.parse_args()
result = check(load_json(args.proposal))
output = json.dumps(result, ensure_ascii=False, indent=2) + "\n"
if args.output:
args.output.parent.mkdir(parents=True, exist_ok=True)
args.output.write_text(output, encoding="utf-8")
print(json.dumps({"output": str(args.output), "passed": result["passed"], "counts": result["counts"]}, ensure_ascii=False))
else:
print(output)
return 0 if result["passed"] else 2
if __name__ == "__main__":
raise SystemExit(main())
+151
View File
@@ -0,0 +1,151 @@
from __future__ import annotations
import argparse
import json
import sys
from pathlib import Path
from typing import Any
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT))
sys.path.insert(0, str(ROOT / "plugins" / "1c"))
sys.path.insert(0, str(ROOT / "plugins" / "1c" / "connector"))
import adapter_1c_server as adapter_server # noqa: E402
def require(condition: bool, message: str, failures: list[str]) -> None:
if not condition:
failures.append(message)
def patch_adapter_for_symbol_checks() -> None:
def fake_definition_find(payload: dict[str, Any]) -> dict[str, Any]:
query = str(payload.get("query") or "")
if query == "Артикул":
return {
"schema": "onec_definition_find.v1",
"status": "ok",
"matches": [
{
"canonical_path": "Справочник.Номенклатура.Артикул",
"kind": "Catalog",
"name": "Номенклатура",
}
],
}
if query == "Номенклатура":
return {
"schema": "onec_definition_find.v1",
"status": "ok",
"matches": [
{
"canonical_path": "Справочник.Номенклатура",
"kind": "Catalog",
"name": "Номенклатура",
}
],
}
return {"schema": "onec_definition_find.v1", "status": "ok", "matches": []}
def fake_read_module(payload: dict[str, Any]) -> dict[str, Any]:
return {
"schema": "onec_module_read.v1",
"status": "ok",
"text": "Процедура ПередЗаписью(Отказ) Экспорт\n Отказ = Истина;\nКонецПроцедуры",
"owner": {"kind": "Catalog", "name": "Номенклатура"},
"module": {"name": "Модуль объекта"},
}
def fake_attributes(payload: dict[str, Any]) -> dict[str, Any]:
return {"schema": "onec_metadata_object_attributes.v1", "status": "ok", "attributes": []}
adapter_server.metadata_definition_find = fake_definition_find
adapter_server.read_module = fake_read_module
adapter_server.metadata_object_attributes = fake_attributes
def check_full_path(failures: list[str]) -> None:
result = adapter_server.call_method(
"code.symbol.resolve",
{
"base_id": "upo_test",
"expression": "Справочник.Номенклатура.Артикул",
"module_ref": "ConfigSave:file:0",
},
)
require(result.get("schema") == "onec_bsl_symbol_resolution.v1", "full path must return BSL symbol schema", failures)
require(result.get("status") == "resolved", "full path must resolve", failures)
require(result.get("resolution_kind") == "metadata_path", "full path must be classified as metadata_path", failures)
require(result.get("canonical_path") == "Справочник.Номенклатура.Артикул", "full path must expose canonical_path publicly", failures)
require(result.get("safe_as_metadata_path") is True, "full path must expose safe_as_metadata_path=true", failures)
def check_parameter(failures: list[str]) -> None:
result = adapter_server.call_method(
"code.symbol.resolve",
{
"base_id": "upo_test",
"expression": "Отказ.Код",
"routine_name": "ПередЗаписью",
"module_ref": "ConfigSave:file:0",
},
)
require(result.get("status") == "resolved", "routine parameter must resolve", failures)
require(result.get("resolution_kind") == "parameter", "routine parameter must not be metadata", failures)
require(result.get("context_path") == "Отказ.Код", "routine parameter must expose context_path publicly", failures)
require(result.get("safe_as_metadata_path") is False, "routine parameter must expose safe_as_metadata_path=false", failures)
def check_short_name(failures: list[str]) -> None:
result = adapter_server.call_method(
"code.symbol.resolve",
{
"base_id": "upo_test",
"expression": "Номенклатура.ЕдИзмерение.Код",
"routine_name": "ПередЗаписью",
"module_ref": "ConfigSave:file:0",
},
)
candidates = result.get("candidates") if isinstance(result.get("candidates"), list) else []
require(result.get("status") == "unresolved", "short object name must stay unresolved", failures)
require(result.get("safe_as_metadata_path") is False, "short object name must expose safe_as_metadata_path=false", failures)
require(bool(candidates), "short object name must return ambiguity candidates", failures)
if candidates:
require(candidates[0].get("canonical_path") == "Справочник.Номенклатура", "candidate canonical_path must stay public", failures)
require(candidates[0].get("reason") == "short_object_name_requires_kind", "candidate must explain short name risk", failures)
def run_checks() -> dict[str, Any]:
patch_adapter_for_symbol_checks()
failures: list[str] = []
check_full_path(failures)
check_parameter(failures)
check_short_name(failures)
return {
"schema": "onec_code_symbol_contract_check.v1",
"status": "ok" if not failures else "failed",
"failures": failures,
"checks": {
"full_path_metadata": "full path must resolve as metadata_path",
"parameter_not_metadata": "routine parameter must not be metadata",
"short_name_unsafe": "short object name must remain unsafe",
},
}
def main() -> int:
parser = argparse.ArgumentParser(description="Check adapter-level code.symbol.resolve contract invariants.")
parser.add_argument("--print", action="store_true")
args = parser.parse_args()
report = run_checks()
if args.print or report["status"] != "ok":
print(json.dumps(report, ensure_ascii=False, indent=2))
else:
print("1C code symbol contract status: ok")
return 0 if report["status"] == "ok" else 1
if __name__ == "__main__":
raise SystemExit(main())
+128
View File
@@ -0,0 +1,128 @@
from __future__ import annotations
import argparse
import json
import sys
import tomllib
from pathlib import Path
from typing import Any
import yaml
ROOT = Path(__file__).resolve().parents[1]
CONNECTOR = ROOT / "plugins" / "1c" / "connector"
PARSER = ROOT / "plugins" / "1c" / "parser"
def require(condition: bool, message: str, failures: list[str]) -> None:
if not condition:
failures.append(message)
def read_yaml(path: Path) -> dict[str, Any]:
with path.open("r", encoding="utf-8") as handle:
data = yaml.safe_load(handle)
return data if isinstance(data, dict) else {}
def read_toml(path: Path) -> dict[str, Any]:
with path.open("rb") as handle:
data = tomllib.load(handle)
return data if isinstance(data, dict) else {}
def run_checks() -> dict[str, Any]:
failures: list[str] = []
required = [
CONNECTOR / "adapter_1c_server.py",
CONNECTOR / "contracts" / "openapi.yaml",
CONNECTOR / "policies" / "read-only-query.yaml",
CONNECTOR / "policies" / "sql-base-access-policy.yaml",
CONNECTOR / "policies" / "change-workflow.yaml",
CONNECTOR / "policies" / "config-layer-write-policy.yaml",
CONNECTOR / "Dockerfile",
CONNECTOR / "docker-compose.yml",
CONNECTOR / ".env.example",
CONNECTOR / "pyproject.toml",
CONNECTOR / "service.yaml",
CONNECTOR / "README.md",
PARSER / "__init__.py",
PARSER / "payload.py",
PARSER / "cas_payload.py",
]
missing = [str(path.relative_to(ROOT)) for path in required if not path.exists()]
require(not missing, f"missing standalone connector files: {missing}", failures)
service = read_yaml(CONNECTOR / "service.yaml") if (CONNECTOR / "service.yaml").exists() else {}
compose = read_yaml(CONNECTOR / "docker-compose.yml") if (CONNECTOR / "docker-compose.yml").exists() else {}
pyproject = read_toml(CONNECTOR / "pyproject.toml") if (CONNECTOR / "pyproject.toml").exists() else {}
openapi = read_yaml(CONNECTOR / "contracts" / "openapi.yaml") if (CONNECTOR / "contracts" / "openapi.yaml").exists() else {}
access_policy = read_yaml(CONNECTOR / "policies" / "sql-base-access-policy.yaml") if (CONNECTOR / "policies" / "sql-base-access-policy.yaml").exists() else {}
require(service.get("id") == "onec-adapter-connector", "service.yaml must identify onec-adapter-connector", failures)
require(service.get("status") == "standalone-ready", "service.yaml status must be standalone-ready", failures)
require((service.get("runtime") or {}).get("entrypoint") == "adapter_1c_server.py", "service entrypoint must be adapter_1c_server.py", failures)
require("contracts/openapi.yaml" == (service.get("contracts") or {}).get("openapi"), "service must point to connector OpenAPI contract", failures)
registered_policies = (service.get("contracts") or {}).get("policies") or []
require("policies/sql-base-access-policy.yaml" in registered_policies, "service must register SQL base access policy", failures)
base_settings = access_policy.get("base_settings") if isinstance(access_policy.get("base_settings"), dict) else {}
read_scope = access_policy.get("read_scope") if isinstance(access_policy.get("read_scope"), dict) else {}
write_scope = access_policy.get("write_scope") if isinstance(access_policy.get("write_scope"), dict) else {}
identity = access_policy.get("sql_identity_management") if isinstance(access_policy.get("sql_identity_management"), dict) else {}
require(access_policy.get("status") == "active", "SQL base access policy must be active", failures)
require(base_settings.get("selector") == "base_id", "SQL settings must be selected by base_id", failures)
require(set(base_settings.get("required_fields") or []) == {"server", "database", "user"}, "SQL base settings must require server, database, and user", failures)
require(read_scope.get("application_data") == "read_only", "application data must be read-only", failures)
require(read_scope.get("metadata_structure") == "read_only", "metadata structure must be readable without mutation", failures)
require(set((write_scope.get("allowed") or {}).values()) == {"ConfigSave", "ConfigCASSave"}, "only ConfigSave and ConfigCASSave may be write targets", failures)
require(identity.get("mode") == "forbidden", "SQL identity management must be forbidden", failures)
project = pyproject.get("project") if isinstance(pyproject.get("project"), dict) else {}
require(project.get("name") == "onec-adapter-connector", "pyproject project.name must be onec-adapter-connector", failures)
scripts = project.get("scripts") if isinstance(project.get("scripts"), dict) else {}
require(scripts.get("onec-adapter") == "adapter_1c_server:main", "pyproject must expose onec-adapter script", failures)
dependencies = project.get("dependencies") if isinstance(project.get("dependencies"), list) else []
require(any(str(dep).startswith("pymssql") for dep in dependencies), "pyproject must include pymssql dependency", failures)
services = compose.get("services") if isinstance(compose.get("services"), dict) else {}
adapter_service = services.get("onec-adapter") if isinstance(services.get("onec-adapter"), dict) else {}
build = adapter_service.get("build") if isinstance(adapter_service.get("build"), dict) else {}
require(build.get("context") == "..", "docker-compose build context must include parser sibling", failures)
require(build.get("dockerfile") == "connector/Dockerfile", "docker-compose must use connector/Dockerfile", failures)
require(bool(adapter_service.get("healthcheck")), "docker-compose must define a healthcheck", failures)
require(openapi.get("openapi") == "3.1.0", "connector OpenAPI must parse as 3.1.0", failures)
paths = openapi.get("paths") if isinstance(openapi.get("paths"), dict) else {}
require("/health" in paths, "connector OpenAPI must expose /health", failures)
require("/metadata/write-plan" in paths, "connector OpenAPI must expose /metadata/write-plan", failures)
return {
"schema": "onec_connector_standalone_check.v1",
"status": "ok" if not failures else "failed",
"failures": failures,
"checks": {
"required_files": not missing,
"service_manifest": service.get("id"),
"pyproject": project.get("name"),
"compose_service": "onec-adapter" in services,
"openapi": openapi.get("openapi"),
},
}
def main() -> int:
parser = argparse.ArgumentParser(description="Check 1C connector standalone-ready service packaging.")
parser.add_argument("--print", action="store_true")
args = parser.parse_args()
report = run_checks()
if args.print or report["status"] != "ok":
print(json.dumps(report, ensure_ascii=False, indent=2))
else:
print("1C connector standalone status: ok")
return 0 if report["status"] == "ok" else 1
if __name__ == "__main__":
raise SystemExit(main())
@@ -0,0 +1,227 @@
from __future__ import annotations
import argparse
import json
import sys
from pathlib import Path
from typing import Any
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT))
sys.path.insert(0, str(ROOT / "plugins" / "1c"))
sys.path.insert(0, str(ROOT / "plugins" / "1c" / "connector"))
import adapter_1c_server as adapter_server # noqa: E402
from parser.payload import compress_payload # noqa: E402
def require(condition: bool, message: str, failures: list[str]) -> None:
if not condition:
failures.append(message)
def run_override(action_evidence: dict[str, Any] | None) -> dict[str, Any]:
adapter_server.metadata_object_modules = lambda payload: {
"status": "ok",
"object": {"kind": "Catalog", "name": "Номенклатура"},
"modules": [{"module_id": "ConfigCAS:ext-guid__module-guid.0", "name": "object"}],
}
selection = {
"routine_name": "ПередЗаписью",
"line_start": 1,
"line_end": 3,
"match_by": "routine_exact",
}
if action_evidence:
selection.update(action_evidence)
adapter_server.read_module = lambda payload: {"status": "ok", "selection": selection}
return adapter_server.call_method(
"metadata.resolve_overrides",
{
"base_id": "upo_test",
"object_type": "Catalog",
"object_name": "Номенклатура",
"method_name": "ПередЗаписью",
},
)
def run_base_override() -> dict[str, Any]:
adapter_server.metadata_object_modules = lambda payload: {
"status": "ok",
"object": {"kind": "Catalog", "name": "Номенклатура"},
"modules": [{"module_id": "Config:object-module.0", "name": "object"}],
}
adapter_server.read_module = lambda payload: {
"status": "ok",
"selection": {
"routine_name": "ПередЗаписью",
"line_start": 1,
"line_end": 3,
"match_by": "routine_exact",
},
}
return adapter_server.call_method(
"metadata.resolve_overrides",
{
"base_id": "upo_test",
"object_type": "Catalog",
"object_name": "Номенклатура",
"method_name": "ПередЗаписью",
},
)
def run_saved_state_search_with_object_name(params: dict[str, Any] | None = None) -> dict[str, Any]:
module_text = "Процедура ПередЗаписью(Отказ)\nКонецПроцедуры".encode("utf-8")
header = f"\r\n{len(module_text):08x} {len(module_text):08x} 7fffffff \r\n".encode("ascii")
stored = compress_payload(b"prefix" + header + module_text, "raw_deflate")
adapter_server.metadata_cache_lookup_row = lambda base_id, kind, name: {
"guid": "owner-guid",
"kind": kind,
"kind_ru": "Справочник",
"public_kind": "catalog",
"name": name,
"source": "base",
}
adapter_server.storage_files_list = lambda payload: {
"status": "ok",
"files": [{"FileName": "owner-guid__module-guid.0", "PartCount": 1, "Bytes": len(stored)}],
}
adapter_server.read_storage_file_bytes = lambda base_id, table, file_name, timeout_seconds=30: (stored, {"database": base_id}, None)
request_payload = {
"base_id": "upo_test",
"tables": ["ConfigCASSave"],
"object_type": "Catalog",
"object_name": "Номенклатура",
"query": "ПередЗаписью",
"limit": 10,
}
request_payload.update(params or {})
return adapter_server.call_method(
adapter_server.SAVED_STATE_MODULES_SEARCH_METHOD,
request_payload,
)
def first_action(result: dict[str, Any]) -> dict[str, Any]:
chain = result.get("chain") if isinstance(result.get("chain"), list) else []
first = chain[0] if chain and isinstance(chain[0], dict) else {}
action = first.get("extension_action") if isinstance(first.get("extension_action"), dict) else {}
return action
def run_checks() -> dict[str, Any]:
failures: list[str] = []
unknown = run_override(None)
unknown_action = first_action(unknown)
unknown_evidence = unknown.get("write_plan_evidence") if isinstance(unknown.get("write_plan_evidence"), dict) else {}
unknown_next = unknown_evidence.get("next_resolution") if isinstance(unknown_evidence.get("next_resolution"), dict) else {}
unknown_next_params = unknown_next.get("params") if isinstance(unknown_next.get("params"), dict) else {}
require(unknown.get("status") == "ok", "override chain with extension routine must resolve", failures)
require(unknown_action.get("status") == "unknown", "extension routine without action evidence must be unknown", failures)
require(unknown_action.get("operation_class") == "unknown_extension_action", "unknown action must not become replace", failures)
require((unknown_evidence.get("target") or {}).get("extension_action") == unknown_action, "unknown action must be carried into write_plan_evidence target", failures)
require("intent" not in unknown_evidence, "unknown action must not infer write intent", failures)
require(unknown_next.get("method") == adapter_server.SAVED_STATE_MODULES_SEARCH_METHOD, "write_plan_evidence must expose saved-state module resolver", failures)
require(unknown_next_params.get("tables") == ["ConfigCASSave"], "extension write_plan_evidence must search ConfigCASSave", failures)
controlled = run_override({"operation_class": "replace_with_control"})
controlled_action = first_action(controlled)
controlled_evidence = controlled.get("write_plan_evidence") if isinstance(controlled.get("write_plan_evidence"), dict) else {}
controlled_next = controlled_evidence.get("next_resolution") if isinstance(controlled_evidence.get("next_resolution"), dict) else {}
controlled_next_params = controlled_next.get("params") if isinstance(controlled_next.get("params"), dict) else {}
require(controlled_action.get("status") == "ok", "known extension action must be ok", failures)
require(controlled_action.get("operation_class") == "replace_with_control", "replace_with_control must be preserved", failures)
require(controlled_action.get("requires_control_fragment") is True, "replace_with_control must require control fragment", failures)
require((controlled_evidence.get("target") or {}).get("extension_action") == controlled_action, "known action must be carried into write_plan_evidence target", failures)
require((controlled_evidence.get("intent") or {}).get("operation") == "replace_with_control", "known action must infer write_plan_evidence intent", failures)
require(controlled_next.get("method") == adapter_server.SAVED_STATE_MODULES_SEARCH_METHOD, "known action evidence must expose saved-state module resolver", failures)
require(controlled_next_params.get("query") == "ПередЗаписью", "known action resolver params must carry routine query", failures)
before = run_override({"action": "вставить до"})
before_action = first_action(before)
require(before_action.get("operation_class") == "insert_before", "Russian insert-before action must normalize", failures)
base = run_base_override()
base_action = first_action(base)
base_evidence = base.get("write_plan_evidence") if isinstance(base.get("write_plan_evidence"), dict) else {}
base_next = base_evidence.get("next_resolution") if isinstance(base_evidence.get("next_resolution"), dict) else {}
base_next_params = base_next.get("params") if isinstance(base_next.get("params"), dict) else {}
require(base_action.get("operation_class") == "base_definition", "base routine must be marked as base_definition", failures)
require(base_action.get("requires_control_fragment") is False, "base routine must not require control fragment", failures)
require(base_next_params.get("tables") == ["ConfigSave"], "base write_plan_evidence must search ConfigSave", failures)
saved_state = run_saved_state_search_with_object_name(controlled_next_params)
saved_state_owner = saved_state.get("owner_resolution") if isinstance(saved_state.get("owner_resolution"), dict) else {}
saved_state_modules = saved_state.get("modules") if isinstance(saved_state.get("modules"), list) else []
saved_state_streams = saved_state_modules[0].get("streams") if saved_state_modules and isinstance(saved_state_modules[0], dict) and isinstance(saved_state_modules[0].get("streams"), list) else []
saved_state_target = saved_state_streams[0].get("write_plan_target") if saved_state_streams and isinstance(saved_state_streams[0], dict) and isinstance(saved_state_streams[0].get("write_plan_target"), dict) else {}
require(saved_state.get("status") == "ok", "saved-state module search with object name must be accepted", failures)
require(saved_state_owner.get("owner_guid") == "owner-guid", "saved-state module search must resolve object name to owner_guid", failures)
require(bool(saved_state_modules), "saved-state module search with resolved owner must find module", failures)
require(saved_state_target.get("module_ref") == "ConfigCASSave:owner-guid__module-guid.0#stream:0", "saved-state stream must expose write_plan_target.module_ref", failures)
require(saved_state_target.get("expected_sha1"), "saved-state stream must expose write_plan_target.expected_sha1", failures)
concrete_plan_target = {
**(controlled_evidence.get("target") if isinstance(controlled_evidence.get("target"), dict) else {}),
**saved_state_target,
}
concrete_plan_intent = {
**(controlled_evidence.get("intent") if isinstance(controlled_evidence.get("intent"), dict) else {}),
"control_fragment": "Процедура ПередЗаписью",
"new": "Процедура ПередЗаписью(Отказ)\n\t// smoke\nКонецПроцедуры",
}
concrete_plan = adapter_server.call_method(
adapter_server.METADATA_WRITE_PLAN_METHOD,
{
"base_id": "upo_test",
"target": concrete_plan_target,
"intent": concrete_plan_intent,
"resolve_origin": False,
},
)
concrete_route = concrete_plan.get("route") if isinstance(concrete_plan.get("route"), dict) else {}
concrete_hint = concrete_route.get("apply_payload_hint") if isinstance(concrete_route.get("apply_payload_hint"), dict) else {}
concrete_hint_payload = concrete_hint.get("payload") if isinstance(concrete_hint.get("payload"), dict) else {}
require(concrete_plan.get("allowed") is True, "write_plan_evidence plus saved-state write_plan_target must produce an allowed concrete plan", failures)
require(concrete_route.get("apply_method") == adapter_server.MODULE_WRITE_APPLY_METHOD, "concrete override write plan must route to module write apply", failures)
require(concrete_route.get("operation_class") == "replace_with_control", "concrete override write plan must preserve extension operation class", failures)
require(concrete_hint.get("ready_for_apply_method") is True, "concrete override write plan hint must be ready for apply method", failures)
require(concrete_hint_payload.get("module_ref") == saved_state_target.get("module_ref"), "concrete override write plan hint must carry module_ref", failures)
require(concrete_hint_payload.get("expected_sha1") == saved_state_target.get("expected_sha1"), "concrete override write plan hint must carry expected_sha1", failures)
return {
"schema": "onec_extension_action_contract_check.v1",
"status": "ok" if not failures else "failed",
"failures": failures,
"checks": {
"unknown_extension_action": "extension routine without action evidence stays unknown",
"replace_with_control": "controlled replacement action requires control fragment",
"russian_insert_before": "Russian action names normalize to stable classes",
"base_definition": "base routines are not treated as extension actions",
"write_plan_evidence": "override results include a ready metadata.write.plan evidence fragment",
"write_plan_next_resolution": "write_plan_evidence points to the saved-state module resolver",
"saved_state_module_name_selector": "saved-state module search resolves object_type/object_name to owner_guid",
"saved_state_stream_write_plan_target": "saved-state module streams expose concrete metadata.write.plan target",
"override_to_concrete_write_plan": "write_plan_evidence and write_plan_target compose into an allowed concrete module plan",
},
}
def main() -> int:
parser = argparse.ArgumentParser(description="Check extension routine action contract invariants.")
parser.add_argument("--print", action="store_true")
args = parser.parse_args()
report = run_checks()
if args.print or report["status"] != "ok":
print(json.dumps(report, ensure_ascii=False, indent=2))
else:
print("1C extension action contract status: ok")
return 0 if report["status"] == "ok" else 1
if __name__ == "__main__":
raise SystemExit(main())
+165
View File
@@ -0,0 +1,165 @@
#!/usr/bin/env python3
"""Validate safe runner configuration for disposable 1C extension validation."""
from __future__ import annotations
import argparse
import json
import re
from pathlib import Path
from typing import Any
ALLOWED_RUNNER_KINDS = {"manual", "designer_cli", "onescript", "custom"}
ALLOWED_VALIDATION_MODES = {"manual", "load_and_syntax_check", "load_syntax_and_smoke"}
FORBIDDEN_BASE_MARKERS = {"prod", "production", "рабоч", "боев", "real", "main"}
SECRET_KEY_RE = re.compile(r"(password|passwd|pwd|secret|token|ключ|парол)", re.IGNORECASE)
CONNECTION_SECRET_RE = re.compile(r"(pwd|password|usr|user)\s*=", re.IGNORECASE)
def load_json(path: Path) -> dict[str, Any]:
return json.loads(path.read_text(encoding="utf-8-sig"))
def write_json(path: Path, data: dict[str, Any]) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(json.dumps(data, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
def issue(severity: str, code: str, message: str, *, path: Path | str | None = None, detail: dict[str, Any] | None = None) -> dict[str, Any]:
result: dict[str, Any] = {"severity": severity, "code": code, "message": message}
if path is not None:
result["path"] = str(path)
if detail:
result["detail"] = detail
return result
def find_secret_keys(value: Any, prefix: str = "") -> list[str]:
hits: list[str] = []
if isinstance(value, dict):
for key, nested in value.items():
current = f"{prefix}.{key}" if prefix else str(key)
if SECRET_KEY_RE.search(str(key)):
hits.append(current)
hits.extend(find_secret_keys(nested, current))
elif isinstance(value, list):
for index, nested in enumerate(value):
hits.extend(find_secret_keys(nested, f"{prefix}[{index}]"))
return hits
def string_contains_forbidden_marker(value: str) -> str | None:
lowered = value.casefold()
for marker in FORBIDDEN_BASE_MARKERS:
if marker in lowered:
return marker
return None
def sanitized_config(config: dict[str, Any]) -> dict[str, Any]:
allowed = {
"schema",
"runner_id",
"runner_kind",
"platform_version",
"platform_bin",
"disposable_base_ref",
"disposable_base_kind",
"disposable_base_confirmed",
"validation_mode",
"evidence_root",
"notes",
}
return {key: value for key, value in config.items() if key in allowed}
def check_config(config_path: Path) -> dict[str, Any]:
findings: list[dict[str, Any]] = []
if not config_path.exists() or not config_path.is_file():
findings.append(issue("error", "missing_runner_config", "Runner config file is missing.", path=config_path))
return build_result(config_path, None, findings)
config = load_json(config_path)
if config.get("schema") != "onec_extension_runner_config.v1":
findings.append(issue("error", "invalid_runner_config_schema", "Runner config schema must be onec_extension_runner_config.v1.", path=config_path, detail={"schema": config.get("schema")}))
runner_id = config.get("runner_id")
if not isinstance(runner_id, str) or not runner_id.strip():
findings.append(issue("error", "missing_runner_id", "runner_id is required.", path=config_path))
runner_kind = config.get("runner_kind")
if runner_kind not in ALLOWED_RUNNER_KINDS:
findings.append(issue("error", "invalid_runner_kind", "runner_kind is not supported.", path=config_path, detail={"allowed": sorted(ALLOWED_RUNNER_KINDS), "actual": runner_kind}))
validation_mode = config.get("validation_mode")
if validation_mode not in ALLOWED_VALIDATION_MODES:
findings.append(issue("error", "invalid_validation_mode", "validation_mode is not supported.", path=config_path, detail={"allowed": sorted(ALLOWED_VALIDATION_MODES), "actual": validation_mode}))
disposable_base_ref = config.get("disposable_base_ref")
if not isinstance(disposable_base_ref, str) or not disposable_base_ref.strip():
findings.append(issue("error", "missing_disposable_base_ref", "disposable_base_ref is required.", path=config_path))
else:
marker = string_contains_forbidden_marker(disposable_base_ref)
if marker:
findings.append(issue("error", "production_like_base_ref", "disposable_base_ref contains a production-like marker.", path=config_path, detail={"marker": marker}))
if CONNECTION_SECRET_RE.search(disposable_base_ref):
findings.append(issue("error", "secret_in_disposable_base_ref", "disposable_base_ref must not contain user/password connection data.", path=config_path))
if config.get("disposable_base_confirmed") is not True:
findings.append(issue("error", "disposable_base_not_confirmed", "disposable_base_confirmed must be true.", path=config_path))
platform_bin = config.get("platform_bin")
if platform_bin is not None:
if not isinstance(platform_bin, str) or not platform_bin.strip():
findings.append(issue("error", "invalid_platform_bin", "platform_bin must be a non-empty string when provided.", path=config_path))
elif runner_kind in {"designer_cli", "custom"} and not Path(platform_bin).exists():
findings.append(issue("warning", "platform_bin_not_found", "platform_bin does not exist on this machine; runner may be remote or not installed here.", path=platform_bin))
evidence_root = config.get("evidence_root")
if evidence_root is not None and (not isinstance(evidence_root, str) or not evidence_root.strip()):
findings.append(issue("error", "invalid_evidence_root", "evidence_root must be a non-empty string when provided.", path=config_path))
secret_keys = find_secret_keys(config)
for key in secret_keys:
findings.append(issue("error", "secret_key_in_runner_config", "Runner config must not contain secrets or credentials.", path=config_path, detail={"key": key}))
unknown = sorted(set(config) - set(sanitized_config(config)))
for key in unknown:
findings.append(issue("warning", "unknown_runner_config_key", "Unknown runner config key will be ignored by the adapter.", path=config_path, detail={"key": key}))
return build_result(config_path, sanitized_config(config), findings)
def build_result(config_path: Path, config: dict[str, Any] | None, findings: list[dict[str, Any]]) -> dict[str, Any]:
errors = [row for row in findings if row.get("severity") == "error"]
warnings = [row for row in findings if row.get("severity") == "warning"]
return {
"schema": "onec_extension_runner_config_check.v1",
"config_path": str(config_path),
"config_schema": (config or {}).get("schema"),
"passed": not errors,
"sanitized_config": config,
"findings": findings,
"counts": {
"errors": len(errors),
"warnings": len(warnings),
},
}
def main() -> int:
parser = argparse.ArgumentParser(description="Validate safe runner configuration for disposable 1C extension validation.")
parser.add_argument("--config", type=Path, required=True)
parser.add_argument("--output", type=Path)
args = parser.parse_args()
result = check_config(args.config)
if args.output:
write_json(args.output, result)
print(json.dumps({"output": str(args.output) if args.output else None, "passed": result["passed"], "counts": result["counts"]}, ensure_ascii=False))
return 0 if result["passed"] else 2
if __name__ == "__main__":
raise SystemExit(main())
+195
View File
@@ -0,0 +1,195 @@
#!/usr/bin/env python3
"""Validate a disposable 1C extension XML staging copy."""
from __future__ import annotations
import argparse
import hashlib
import json
from pathlib import Path
from typing import Any
from check_1c_patch_bundle import check_bundle
def load_json(path: Path) -> dict[str, Any]:
return json.loads(path.read_text(encoding="utf-8-sig"))
def write_json(path: Path, data: dict[str, Any]) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(json.dumps(data, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
def sha256_file(path: Path) -> str:
digest = hashlib.sha256()
with path.open("rb") as stream:
for chunk in iter(lambda: stream.read(1024 * 1024), b""):
digest.update(chunk)
return digest.hexdigest()
def issue(severity: str, code: str, message: str, *, path: Path | str | None = None, detail: dict[str, Any] | None = None) -> dict[str, Any]:
result: dict[str, Any] = {"severity": severity, "code": code, "message": message}
if path is not None:
result["path"] = str(path)
if detail:
result["detail"] = detail
return result
def safe_relative(relative_path: str) -> Path:
path = Path(relative_path.replace("\\", "/"))
if path.is_absolute() or ".." in path.parts or not str(path):
raise ValueError(relative_path)
return path
def is_within(path: Path, root: Path) -> bool:
try:
path.resolve().relative_to(root.resolve())
return True
except ValueError:
return False
def check_staging(staging_dir: Path) -> dict[str, Any]:
findings: list[dict[str, Any]] = []
file_checks: list[dict[str, Any]] = []
manifest: dict[str, Any] | None = None
if not staging_dir.exists() or not staging_dir.is_dir():
findings.append(issue("error", "missing_staging_dir", "Staging directory is missing.", path=staging_dir))
return build_result(staging_dir, manifest, findings, file_checks, None)
manifest_path = staging_dir / "_codex_staging_manifest.json"
if not manifest_path.exists():
findings.append(issue("error", "missing_staging_manifest", "Staging manifest is missing.", path=manifest_path))
return build_result(staging_dir, manifest, findings, file_checks, None)
manifest = load_json(manifest_path)
if manifest.get("schema") != "onec_extension_staging.v1":
findings.append(issue("error", "invalid_staging_schema", "Staging manifest schema is not onec_extension_staging.v1.", path=manifest_path, detail={"schema": manifest.get("schema")}))
declared_staging_dir = Path(str(manifest.get("staging_dir") or ""))
if declared_staging_dir and declared_staging_dir.resolve() != staging_dir.resolve():
findings.append(issue("error", "staging_dir_mismatch", "Manifest staging_dir does not match checked directory.", path=manifest_path, detail={"manifest_staging_dir": str(declared_staging_dir), "checked_staging_dir": str(staging_dir)}))
safety = manifest.get("safety") if isinstance(manifest.get("safety"), dict) else {}
expected_safety = {
"source_extension_modified": False,
"sql_modified": False,
"requires_disposable_1c_validation": True,
}
for key, expected in expected_safety.items():
actual = safety.get(key)
if actual is not expected:
findings.append(issue("error", "invalid_staging_safety_flag", "Staging safety flag has an unexpected value.", path=manifest_path, detail={"flag": key, "expected": expected, "actual": actual}))
extension_root = Path(str(manifest.get("extension_root") or ""))
if not extension_root.exists() or not extension_root.is_dir():
findings.append(issue("error", "missing_extension_root", "Source extension root is missing.", path=extension_root))
bundle_dir = Path(str(manifest.get("bundle_dir") or ""))
bundle_check: dict[str, Any] | None = None
if not bundle_dir.exists() or not bundle_dir.is_dir():
findings.append(issue("error", "missing_bundle_dir", "Bundle directory recorded in staging manifest is missing.", path=bundle_dir))
else:
bundle_check = check_bundle(bundle_dir)
if not bundle_check.get("passed"):
findings.append(issue("error", "bundle_check_failed", "Bundle recorded in staging manifest does not pass validation.", path=bundle_dir, detail={"counts": bundle_check.get("counts")}))
for record in manifest.get("files") or []:
relative_raw = str(record.get("relative_path") or "")
staged_path_raw = str(record.get("staged_path") or "")
check: dict[str, Any] = {
"relative_path": relative_raw,
"staged_path": staged_path_raw,
"exists": False,
"expected_staged_sha256": record.get("staged_sha256"),
"expected_working_sha256": record.get("expected_working_sha256"),
"expected_source_original_sha256": record.get("source_original_sha256"),
}
try:
relative = safe_relative(relative_raw)
except ValueError:
findings.append(issue("error", "unsafe_relative_path", "Unsafe relative_path in staging manifest.", path=manifest_path, detail={"relative_path": relative_raw}))
file_checks.append(check)
continue
staged_path = Path(staged_path_raw) if staged_path_raw else staging_dir / relative
expected_staged_path = staging_dir / relative
if staged_path.resolve() != expected_staged_path.resolve():
findings.append(issue("error", "staged_path_mismatch", "Manifest staged_path does not match staging_dir/relative_path.", path=manifest_path, detail={"staged_path": str(staged_path), "expected": str(expected_staged_path)}))
if not is_within(staged_path, staging_dir):
findings.append(issue("error", "staged_path_escape", "Manifest staged_path escapes staging directory.", path=staged_path))
file_checks.append(check)
continue
check["exists"] = staged_path.exists()
if not staged_path.exists():
findings.append(issue("error", "missing_staged_file", "Staged file is missing.", path=staged_path))
else:
staged_hash = sha256_file(staged_path)
check["staged_sha256"] = staged_hash
expected_hashes = [record.get("staged_sha256"), record.get("expected_working_sha256")]
for expected in [value for value in expected_hashes if value]:
if staged_hash != expected:
findings.append(issue("error", "staged_file_hash_mismatch", "Staged file hash does not match manifest.", path=staged_path, detail={"expected": expected, "actual": staged_hash}))
source_path = extension_root / relative
check["source_path"] = str(source_path)
check["source_exists"] = source_path.exists()
expected_source_hash = record.get("source_original_sha256")
if expected_source_hash is not None:
if not source_path.exists():
findings.append(issue("error", "missing_source_file", "Source extension file recorded during staging is now missing.", path=source_path))
else:
source_hash = sha256_file(source_path)
check["source_sha256"] = source_hash
if source_hash != expected_source_hash:
findings.append(issue("error", "source_file_changed", "Source extension file changed after staging was created.", path=source_path, detail={"expected": expected_source_hash, "actual": source_hash}))
file_checks.append(check)
return build_result(staging_dir, manifest, findings, file_checks, bundle_check)
def build_result(staging_dir: Path, manifest: dict[str, Any] | None, findings: list[dict[str, Any]], file_checks: list[dict[str, Any]], bundle_check: dict[str, Any] | None) -> dict[str, Any]:
errors = [row for row in findings if row.get("severity") == "error"]
warnings = [row for row in findings if row.get("severity") == "warning"]
return {
"schema": "onec_extension_staging_check.v1",
"staging_dir": str(staging_dir),
"staging_schema": (manifest or {}).get("schema"),
"bundle_dir": (manifest or {}).get("bundle_dir"),
"bundle_check": {
"schema": (bundle_check or {}).get("schema"),
"passed": (bundle_check or {}).get("passed"),
"counts": (bundle_check or {}).get("counts"),
} if bundle_check else None,
"passed": not errors,
"findings": findings,
"file_checks": file_checks,
"counts": {
"files": len(file_checks),
"errors": len(errors),
"warnings": len(warnings),
},
}
def main() -> int:
parser = argparse.ArgumentParser(description="Validate a disposable 1C extension XML staging copy.")
parser.add_argument("--staging-dir", type=Path, required=True)
parser.add_argument("--output", type=Path)
args = parser.parse_args()
result = check_staging(args.staging_dir)
if args.output:
write_json(args.output, result)
print(json.dumps({"output": str(args.output) if args.output else None, "passed": result["passed"], "counts": result["counts"]}, ensure_ascii=False))
return 0 if result["passed"] else 2
if __name__ == "__main__":
raise SystemExit(main())
@@ -0,0 +1,172 @@
#!/usr/bin/env python3
"""Check manual evidence files for a 1C extension validation plan."""
from __future__ import annotations
import argparse
import json
import re
from pathlib import Path
from typing import Any
PENDING_MARKERS = ("status: pending", '"status": "pending"')
PASSED_STATUSES = {"passed", "success", "ok"}
STATUS_LINE_RE = re.compile(r"^\s*status\s*:\s*([A-Za-zА-Яа-я0-9_-]+)\s*$", re.IGNORECASE | re.MULTILINE)
SECRET_TEXT_RE = re.compile(r"(password|passwd|pwd|secret|token|парол|секрет|usr|user)\s*[:=]", re.IGNORECASE)
def load_json(path: Path) -> dict[str, Any]:
return json.loads(path.read_text(encoding="utf-8-sig"))
def write_json(path: Path, data: dict[str, Any]) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(json.dumps(data, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
def issue(severity: str, code: str, message: str, *, path: Path | str | None = None, detail: dict[str, Any] | None = None) -> dict[str, Any]:
result: dict[str, Any] = {"severity": severity, "code": code, "message": message}
if path is not None:
result["path"] = str(path)
if detail:
result["detail"] = detail
return result
def evidence_root(plan: dict[str, Any], override: Path | None = None) -> Path:
if override:
return override
configured = ((plan.get("runner_config") or {}).get("evidence_root")) or ((plan.get("evidence") or {}).get("root"))
if not configured:
raise SystemExit("Validation plan has no evidence root.")
return Path(str(configured))
def text_status(text: str) -> str | None:
match = STATUS_LINE_RE.search(text)
return match.group(1).casefold() if match else None
def check_json_evidence(path: Path, text: str, findings: list[dict[str, Any]]) -> tuple[bool, str | None]:
try:
payload = json.loads(text)
except json.JSONDecodeError as exc:
findings.append(issue("error", "invalid_json_evidence", f"Evidence JSON is invalid: {exc}", path=path))
return False, None
status = str(payload.get("status") or "").casefold()
if status not in PASSED_STATUSES:
findings.append(issue("error", "evidence_status_not_passed", "Evidence JSON status must be passed/success/ok.", path=path, detail={"status": status or None}))
return False, status or None
if path.name == "changed-objects-smoke.json":
objects = payload.get("objects")
if not isinstance(objects, list) or not objects:
findings.append(issue("error", "missing_smoke_objects", "changed-objects-smoke.json must contain a non-empty objects list.", path=path))
return False, status
failed = [
{"object_name": item.get("object_name"), "status": item.get("status")}
for item in objects
if not isinstance(item, dict) or str(item.get("status") or "").casefold() not in PASSED_STATUSES
]
if failed:
findings.append(issue("error", "smoke_object_status_not_passed", "Every changed object smoke record must have status passed/success/ok.", path=path, detail={"failed": failed}))
return False, status
return True, status
def check_text_evidence(path: Path, text: str, findings: list[dict[str, Any]]) -> tuple[bool, str | None]:
status = text_status(text)
if status not in PASSED_STATUSES:
findings.append(issue("error", "evidence_status_not_passed", "Evidence text must contain a Status: passed/success/ok line.", path=path, detail={"status": status}))
return False, status
return True, status
def check_evidence(plan_path: Path, output_root: Path | None = None) -> dict[str, Any]:
findings: list[dict[str, Any]] = []
file_checks: list[dict[str, Any]] = []
plan = load_json(plan_path)
if plan.get("schema") != "onec_extension_validation_plan.v1":
findings.append(issue("error", "invalid_plan_schema", "Validation plan schema is not onec_extension_validation_plan.v1.", path=plan_path, detail={"schema": plan.get("schema")}))
return build_result(plan_path, Path("."), findings, file_checks)
root = evidence_root(plan, output_root)
if not root.exists() or not root.is_dir():
findings.append(issue("error", "missing_evidence_root", "Evidence root is missing.", path=root))
return build_result(plan_path, root, findings, file_checks)
for name in (plan.get("evidence") or {}).get("required_files") or []:
relative = Path(str(name).replace("\\", "/"))
path = root / relative
check: dict[str, Any] = {
"relative_path": str(relative).replace("\\", "/"),
"path": str(path),
"exists": path.exists(),
"filled": False,
}
if not path.exists():
findings.append(issue("error", "missing_evidence_file", "Required evidence file is missing.", path=path))
else:
text = path.read_text(encoding="utf-8-sig", errors="replace")
stripped = text.strip()
check["size"] = len(text.encode("utf-8"))
if SECRET_TEXT_RE.search(text):
findings.append(issue("error", "secret_like_text_in_evidence", "Evidence file contains secret-like key/value text.", path=path))
pending = any(marker in text.casefold() for marker in PENDING_MARKERS)
if pending:
findings.append(issue("error", "pending_evidence_file", "Evidence file still contains a pending template.", path=path))
if not stripped:
findings.append(issue("error", "empty_evidence_file", "Evidence file is empty.", path=path))
elif path.suffix.casefold() == ".json":
passed, status = check_json_evidence(path, text, findings)
check["status"] = status
check["filled"] = passed and not pending
else:
passed, status = check_text_evidence(path, text, findings)
check["status"] = status
check["filled"] = passed and not pending
file_checks.append(check)
manifest_path = root / "_codex_validation_evidence_manifest.json"
if not manifest_path.exists():
findings.append(issue("warning", "missing_evidence_manifest", "Evidence manifest is missing.", path=manifest_path))
return build_result(plan_path, root, findings, file_checks)
def build_result(plan_path: Path, root: Path, findings: list[dict[str, Any]], file_checks: list[dict[str, Any]]) -> dict[str, Any]:
errors = [row for row in findings if row.get("severity") == "error"]
warnings = [row for row in findings if row.get("severity") == "warning"]
return {
"schema": "onec_extension_validation_evidence_check.v1",
"plan_path": str(plan_path),
"evidence_root": str(root),
"passed": not errors,
"status": "validated" if not errors else "pending_or_blocked",
"findings": findings,
"file_checks": file_checks,
"counts": {
"files": len(file_checks),
"filled": sum(1 for item in file_checks if item.get("filled")),
"errors": len(errors),
"warnings": len(warnings),
},
}
def main() -> int:
parser = argparse.ArgumentParser(description="Check manual evidence files for a 1C extension validation plan.")
parser.add_argument("--plan", type=Path, required=True)
parser.add_argument("--evidence-root", type=Path)
parser.add_argument("--output", type=Path)
args = parser.parse_args()
result = check_evidence(args.plan, args.evidence_root)
if args.output:
write_json(args.output, result)
print(json.dumps({"output": str(args.output) if args.output else None, "passed": result["passed"], "status": result["status"], "counts": result["counts"]}, ensure_ascii=False))
return 0 if result["passed"] else 2
if __name__ == "__main__":
raise SystemExit(main())
@@ -0,0 +1,142 @@
#!/usr/bin/env python3
"""Aggregate final validation gates for a staged 1C extension."""
from __future__ import annotations
import argparse
import json
from datetime import datetime, timezone
from pathlib import Path
from typing import Any
from check_1c_extension_staging import check_staging
from check_1c_extension_validation_evidence import check_evidence
def load_json(path: Path) -> dict[str, Any]:
return json.loads(path.read_text(encoding="utf-8-sig"))
def write_json(path: Path, data: dict[str, Any]) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(json.dumps(data, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
def issue(severity: str, code: str, message: str, *, path: Path | str | None = None, detail: dict[str, Any] | None = None) -> dict[str, Any]:
result: dict[str, Any] = {"severity": severity, "code": code, "message": message}
if path is not None:
result["path"] = str(path)
if detail:
result["detail"] = detail
return result
def collect_gate(name: str, data: dict[str, Any]) -> dict[str, Any]:
return {
"name": name,
"schema": data.get("schema"),
"passed": data.get("passed"),
"status": data.get("status"),
"counts": data.get("counts"),
}
def check_release(plan_path: Path, evidence_root: Path | None = None) -> dict[str, Any]:
findings: list[dict[str, Any]] = []
if not plan_path.exists():
findings.append(issue("error", "missing_validation_plan", "Validation plan is missing.", path=plan_path))
return build_result(plan_path, None, {}, {}, findings)
plan = load_json(plan_path)
if plan.get("schema") != "onec_extension_validation_plan.v1":
findings.append(issue("error", "invalid_validation_plan_schema", "Validation plan schema is not onec_extension_validation_plan.v1.", path=plan_path, detail={"schema": plan.get("schema")}))
if plan.get("status") != "ready_for_disposable_validation":
findings.append(issue("error", "validation_plan_not_ready", "Validation plan must be ready_for_disposable_validation.", path=plan_path, detail={"status": plan.get("status")}))
staging_dir = Path(str(plan.get("staging_dir") or ""))
staging_check = check_staging(staging_dir) if staging_dir else {"schema": "onec_extension_staging_check.v1", "passed": False, "counts": {"errors": 1}, "findings": []}
if not staging_check.get("passed"):
findings.append(issue("error", "staging_check_failed", "Staging check failed.", path=staging_dir, detail={"counts": staging_check.get("counts")}))
evidence_check = check_evidence(plan_path, evidence_root)
if not evidence_check.get("passed"):
findings.append(issue("error", "validation_evidence_check_failed", "Validation evidence check failed.", path=evidence_check.get("evidence_root"), detail={"counts": evidence_check.get("counts")}))
safety = plan.get("safety") if isinstance(plan.get("safety"), dict) else {}
expected_safety = {
"production_base_allowed": False,
"sql_write_allowed": False,
"source_extension_write_allowed": False,
"disposable_base_required": True,
}
for key, expected in expected_safety.items():
if safety.get(key) is not expected:
findings.append(issue("error", "invalid_release_safety_flag", "Validation plan safety flag has an unexpected value.", path=plan_path, detail={"flag": key, "expected": expected, "actual": safety.get(key)}))
return build_result(plan_path, plan, staging_check, evidence_check, findings)
def build_result(plan_path: Path, plan: dict[str, Any] | None, staging_check: dict[str, Any], evidence_check: dict[str, Any], findings: list[dict[str, Any]]) -> dict[str, Any]:
errors = [row for row in findings if row.get("severity") == "error"]
warnings = [row for row in findings if row.get("severity") == "warning"]
return {
"schema": "onec_extension_validation_release_check.v1",
"created_at_utc": datetime.now(timezone.utc).isoformat(),
"plan_path": str(plan_path),
"staging_dir": (plan or {}).get("staging_dir"),
"bundle_dir": (plan or {}).get("bundle_dir"),
"preferred_extension": (plan or {}).get("preferred_extension"),
"passed": not errors,
"status": "validated_for_human_review" if not errors else "blocked",
"safety": {
"production_apply_allowed": False,
"automatic_apply_allowed": False,
"human_approval_required": True,
},
"gates": [
{
"name": "validation_plan",
"schema": (plan or {}).get("schema"),
"passed": (plan or {}).get("status") == "ready_for_disposable_validation",
"status": (plan or {}).get("status"),
},
collect_gate("staging_check", staging_check),
collect_gate("validation_evidence_check", evidence_check),
],
"findings": findings,
"counts": {
"errors": len(errors),
"warnings": len(warnings),
},
"next_actions": [
"Review validation evidence and changed files manually.",
"Do not apply to production automatically.",
"If approved, perform production action through the approved human-controlled 1C release process.",
] if not errors else [
"Fix failed gates before review.",
"Do not package, apply, or release this extension from the current evidence.",
],
"details": {
"staging_check": staging_check,
"validation_evidence_check": evidence_check,
},
}
def main() -> int:
parser = argparse.ArgumentParser(description="Aggregate final validation gates for a staged 1C extension.")
parser.add_argument("--plan", type=Path, required=True)
parser.add_argument("--evidence-root", type=Path)
parser.add_argument("--output", type=Path)
args = parser.parse_args()
result = check_release(args.plan, args.evidence_root)
if args.output:
write_json(args.output, result)
print(json.dumps({"output": str(args.output) if args.output else None, "passed": result["passed"], "status": result["status"], "counts": result["counts"]}, ensure_ascii=False))
return 0 if result["passed"] else 2
if __name__ == "__main__":
raise SystemExit(main())
+183
View File
@@ -0,0 +1,183 @@
from __future__ import annotations
import argparse
import json
import os
import re
import time
import urllib.error
import urllib.parse
import urllib.request
from html.parser import HTMLParser
from pathlib import Path
from typing import Any
from fetch_1c_its_docs import charset_from_content_type, request_safe_url
from one_c_its_platform import materialize_doc_url
ROOT = Path(__file__).resolve().parents[1]
DEFAULT_OUTPUT = ROOT / "reports" / "1c-its-access-check.json"
DEFAULT_TEST_URL = "https://its.1c.ru/db/v8316doc#bookmark:dev:TI000000044"
class AccessHtmlParser(HTMLParser):
def __init__(self) -> None:
super().__init__(convert_charrefs=True)
self.title_parts: list[str] = []
self._in_title = False
self.login_links = 0
self.user_profile_markers = 0
self.paywall_markers = 0
self.data_access_false = 0
self.iframe_srcs: list[str] = []
def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
tag = tag.lower()
values = {name.lower(): value or "" for name, value in attrs}
if tag == "title":
self._in_title = True
href = values.get("href", "")
class_name = values.get("class", "")
if "/user/auth" in href:
self.login_links += 1
if "paywall" in class_name:
self.paywall_markers += 1
if values.get("data-access") == "false":
self.data_access_false += 1
if tag == "iframe" and values.get("src"):
self.iframe_srcs.append(values["src"])
def handle_endtag(self, tag: str) -> None:
if tag.lower() == "title":
self._in_title = False
def handle_data(self, data: str) -> None:
if self._in_title:
self.title_parts.append(data.strip())
if "Общий профиль" in data or "Доступ до" in data:
self.user_profile_markers += 1
@property
def title(self) -> str:
return " ".join(part for part in self.title_parts if part)
def read_cookie(cookie_file: Path | None) -> str:
if cookie_file:
return cookie_file.read_text(encoding="utf-8").strip()
return os.environ.get("ONEC_ITS_COOKIE", "").strip()
def fetch_text(url: str, *, cookie: str, timeout: int, referer: str | None = None) -> tuple[int, str, str]:
headers = {"User-Agent": "Codex 1C ITS access check"}
if cookie:
headers["Cookie"] = cookie
if referer:
headers["Referer"] = referer
headers["X-Referer"] = referer
request = urllib.request.Request(request_safe_url(url), headers=headers)
with urllib.request.urlopen(request, timeout=timeout) as response:
body = response.read()
content_type = response.headers.get("Content-Type", "")
charset = charset_from_content_type(content_type)
return int(response.status), content_type, body.decode(charset, errors="replace")
def first_src_url(page_url: str, iframe_srcs: list[str]) -> str | None:
for src in iframe_srcs:
if "/db/content/" in src and "/src/" in src:
return urllib.parse.urljoin(page_url, src)
return None
def check_access(url: str, *, cookie: str, timeout: int) -> dict[str, Any]:
materialized = materialize_doc_url(url)
result: dict[str, Any] = {
"schema": "onec_its_access_check.v1",
"checked_at_unix": int(time.time()),
"target_url": url,
"materialized_url": materialized,
"cookie_present": bool(cookie.strip()),
"page": {},
"src": {},
"status": "unknown",
"findings": [],
}
if not cookie.strip():
result["status"] = "failed"
result["findings"].append("cookie_missing")
return result
try:
status, content_type, text = fetch_text(materialized, cookie=cookie, timeout=timeout)
parser = AccessHtmlParser()
parser.feed(text)
src_url = first_src_url(materialized, parser.iframe_srcs)
result["page"] = {
"http_status": status,
"content_type": content_type,
"title": parser.title,
"login_links": parser.login_links,
"user_profile_markers": parser.user_profile_markers,
"paywall_markers": parser.paywall_markers,
"data_access_false": parser.data_access_false,
"iframe_src": src_url,
}
if parser.login_links:
result["findings"].append("login_links_present")
if parser.paywall_markers:
result["findings"].append("paywall_marker_present")
if parser.data_access_false:
result["findings"].append("data_access_false")
except Exception as exc: # noqa: BLE001
result["page"] = {"error": f"{type(exc).__name__}: {exc}"}
result["status"] = "failed"
result["findings"].append("page_fetch_failed")
return result
src_url = result["page"].get("iframe_src")
if src_url:
try:
src_status, src_content_type, src_text = fetch_text(str(src_url), cookie=cookie, timeout=timeout, referer=materialized)
visible_words = len(re.findall(r"[A-Za-zА-Яа-яЁё0-9_]+", src_text))
result["src"] = {
"http_status": src_status,
"content_type": src_content_type,
"chars": len(src_text),
"word_count": visible_words,
}
if visible_words < 50:
result["findings"].append("src_low_text")
except urllib.error.HTTPError as exc:
result["src"] = {"http_status": exc.code, "error": str(exc), "url": src_url}
result["findings"].append(f"src_http_{exc.code}")
except Exception as exc: # noqa: BLE001
result["src"] = {"error": f"{type(exc).__name__}: {exc}", "url": src_url}
result["findings"].append("src_fetch_failed")
else:
result["findings"].append("src_iframe_missing")
blocking = {"login_links_present", "paywall_marker_present", "data_access_false", "src_http_401", "src_fetch_failed"}
result["status"] = "failed" if any(item in blocking for item in result["findings"]) else "ok"
return result
def main() -> int:
parser = argparse.ArgumentParser(description="Check whether the stored 1C:ITS cookie can access protected documentation bodies.")
parser.add_argument("--url", default=DEFAULT_TEST_URL)
parser.add_argument("--cookie-file", type=Path)
parser.add_argument("--output", type=Path, default=DEFAULT_OUTPUT)
parser.add_argument("--timeout", type=int, default=30)
parser.add_argument("--print", action="store_true", dest="print_report")
args = parser.parse_args()
report = check_access(args.url, cookie=read_cookie(args.cookie_file), timeout=args.timeout)
args.output.parent.mkdir(parents=True, exist_ok=True)
args.output.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
print(json.dumps(report if args.print_report else {"status": report["status"], "findings": report["findings"], "output": str(args.output)}, ensure_ascii=False, indent=2))
return 0 if report["status"] == "ok" else 1
if __name__ == "__main__":
raise SystemExit(main())
+95
View File
@@ -0,0 +1,95 @@
from __future__ import annotations
import argparse
import json
from pathlib import Path
from normalize_1c_its_cookie import normalize_cookie
ROOT = Path(__file__).resolve().parents[1]
DEFAULT_OUTPUT = ROOT / "reports" / "1c-its-cookie-normalizer-check.json"
CASES = [
{
"name": "request_cookie_header",
"input": "Cookie: sid=abc; theme=dark",
"ok": True,
"cookie": "sid=abc; theme=dark",
},
{
"name": "reject_yandex_set_cookie",
"input": "bh=abc; Domain=.yandex.com; Path=/",
"ok": False,
"error": "domain_attributes_do_not_match_target",
},
{
"name": "accept_its_set_cookie",
"input": "sid=abc; Domain=.its.1c.ru; Path=/",
"ok": True,
"cookie": "sid=abc",
},
{
"name": "json_filters_target_domain",
"input": '[{"domain":"its.1c.ru","name":"sid","value":"abc"},{"domain":"yandex.com","name":"bh","value":"no"}]',
"ok": True,
"cookie": "sid=abc",
},
{
"name": "netscape_filters_target_domain",
"input": ".its.1c.ru\tTRUE\t/\tTRUE\t0\tsid\tabc\n.yandex.com\tTRUE\t/\tTRUE\t0\tbh\tno",
"ok": True,
"cookie": "sid=abc",
},
]
def run_cases() -> dict:
checks = []
for case in CASES:
result = normalize_cookie(case["input"], "its.1c.ru")
passed = result["ok"] == case["ok"]
if "cookie" in case:
passed = passed and result["cookie"] == case["cookie"]
if "error" in case:
passed = passed and case["error"] in result["errors"]
checks.append(
{
"name": case["name"],
"status": "passed" if passed else "failed",
"expected": {key: case[key] for key in ("ok", "cookie", "error") if key in case},
"actual": {
"ok": result["ok"],
"cookie": result["cookie"],
"warnings": result["warnings"],
"errors": result["errors"],
"format": result.get("format"),
},
}
)
return {
"schema": "onec_its_cookie_normalizer_check.v1",
"passed": all(check["status"] == "passed" for check in checks),
"checks": checks,
}
def main() -> int:
parser = argparse.ArgumentParser(description="Check 1C:ITS cookie normalizer behavior.")
parser.add_argument("--output", type=Path, default=DEFAULT_OUTPUT)
parser.add_argument("--print", action="store_true", dest="print_report")
args = parser.parse_args()
report = run_cases()
args.output.parent.mkdir(parents=True, exist_ok=True)
args.output.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
if args.print_report:
print(json.dumps(report, ensure_ascii=False, indent=2))
else:
print(json.dumps({"passed": report["passed"], "output": str(args.output)}, ensure_ascii=False))
return 0 if report["passed"] else 1
if __name__ == "__main__":
raise SystemExit(main())
+129
View File
@@ -0,0 +1,129 @@
from __future__ import annotations
import json
from collections import deque
from pathlib import Path
import yaml
from fetch_1c_its_docs import LinkParser, enqueue_links, merged_policy, page_record
from normalize_1c_its_docs import TextExtractor, clean_its_text, content_quality
ROOT = Path(__file__).resolve().parents[1]
SOURCES = ROOT / "plugins" / "1c" / "rag" / "official-docs" / "sources.yaml"
def assert_true(condition: bool, message: str) -> None:
if not condition:
raise AssertionError(message)
def test_src_priority_and_title_hint() -> None:
html = """
<html><head><title>Форма :: Глоссарий разработчика</title></head>
<body>
<iframe src="/db/content/v8devgloss/src/term_000000052.htm"></iframe>
<a href="/db/v8devgloss/content/900049/hdoc">1</a>
<a href="/db/v8devgloss/content/99/hdoc">XDTO</a>
</body></html>
"""
parser = LinkParser()
parser.feed(html)
config = yaml.safe_load(SOURCES.read_text(encoding="utf-8"))
source = config["sources"][0]
policy = merged_policy(config["default_policy"], source.get("policy"))
queue: deque = deque()
enqueue_links(
queue,
source=source,
base_url="https://its.1c.ru/db/v8devgloss/content/52/hdoc",
depth=1,
policy=policy,
links=parser.links,
seen=set(),
title_hint=parser.title,
)
first = queue[0]
assert_true(first[1] == "https://its.1c.ru/db/content/v8devgloss/src/term_000000052.htm", "src iframe URL must be first")
assert_true(first[4] == "Форма :: Глоссарий разработчика", "src iframe URL must inherit hdoc title")
def test_src_only_at_max_depth() -> None:
html = """
<html><head><title>Форма :: Глоссарий разработчика</title></head>
<body>
<iframe src="/db/content/v8devgloss/src/term_000000052.htm"></iframe>
<a href="/db/v8devgloss/content/99/hdoc">XDTO</a>
<a href="/db/v8devgloss/content/14/hdoc">1С:Предприятие</a>
</body></html>
"""
parser = LinkParser()
parser.feed(html)
config = yaml.safe_load(SOURCES.read_text(encoding="utf-8"))
source = config["sources"][0]
policy = merged_policy(config["default_policy"], source.get("policy"))
queue: deque = deque()
enqueue_links(
queue,
source=source,
base_url="https://its.1c.ru/db/v8devgloss/content/52/hdoc",
depth=2,
policy=policy,
links=parser.links,
seen=set(),
title_hint=parser.title,
src_only=True,
)
urls = [item[1] for item in queue]
assert_true(urls == ["https://its.1c.ru/db/content/v8devgloss/src/term_000000052.htm"], "max-depth src-only mode must keep only iframe src")
def test_page_record_uses_title_hint() -> None:
output_dir = ROOT / "reports" / ".tmp-its-ingestion-check"
output_dir.mkdir(parents=True, exist_ok=True)
record = page_record(
source={"id": "v8devgloss", "title": "Глоссарий разработчика", "source_type": "official_1c_its_glossary"},
url="https://its.1c.ru/db/content/v8devgloss/src/term_000000052.htm",
depth=2,
body="<html><body>Текст определения формы.</body></html>".encode("utf-8"),
headers={"Content-Type": "text/html; charset=utf-8"},
status=200,
output_dir=output_dir,
title_hint="Форма :: Глоссарий разработчика",
)
assert_true(record["title"] == "Форма :: Глоссарий разработчика", "title_hint must be used when src page has no title")
def test_glossary_src_fallback_text() -> None:
extractor = TextExtractor()
extractor.feed(
"<html><body><p>"
"Форма предназначена для отображения и редактирования данных объекта, "
"содержит элементы управления, команды, реквизиты формы и обработчики событий, "
"которые используются прикладным решением при работе пользователя в интерфейсе приложения."
"</p></body></html>"
)
text = clean_its_text(extractor.text(), "Форма :: Глоссарий разработчика", "official_1c_its_glossary")
quality = content_quality(text, "Форма :: Глоссарий разработчика")
assert_true("Форма предназначена" in text, "glossary src body must be preserved")
assert_true(bool(quality["is_content"]), "glossary src body must pass quality gate")
def main() -> int:
checks = [
test_src_priority_and_title_hint,
test_src_only_at_max_depth,
test_page_record_uses_title_hint,
test_glossary_src_fallback_text,
]
results = []
for check in checks:
check()
results.append({"id": check.__name__, "status": "passed"})
print(json.dumps({"schema": "onec_its_ingestion_logic_check.v1", "status": "ok", "checks": results}, ensure_ascii=False, indent=2))
return 0
if __name__ == "__main__":
raise SystemExit(main())
@@ -0,0 +1,89 @@
from __future__ import annotations
import argparse
import json
from pathlib import Path
from typing import Any
import yaml
ROOT = Path(__file__).resolve().parents[1]
DEFAULT_START_LINKS = ROOT / "plugins" / "1c" / "rag" / "official-docs" / "start-links.json"
DEFAULT_SOURCES = ROOT / "plugins" / "1c" / "rag" / "official-docs" / "sources.yaml"
DEFAULT_OUTPUT = ROOT / "reports" / "1c-official-docs-start-coverage.json"
def load_json(path: Path) -> dict[str, Any]:
return json.loads(path.read_text(encoding="utf-8-sig")) if path.exists() else {}
def load_yaml(path: Path) -> dict[str, Any]:
data = yaml.safe_load(path.read_text(encoding="utf-8"))
return data if isinstance(data, dict) else {}
def check_coverage(start_links_path: Path, sources_path: Path) -> dict[str, Any]:
start_links = load_json(start_links_path)
sources_config = load_yaml(sources_path)
source_urls = {str(item.get("url") or "") for item in sources_config.get("sources") or []}
candidates = start_links.get("start_links") or []
rows = []
by_category: dict[str, dict[str, int]] = {}
for item in candidates:
category = str(item.get("category") or "unknown")
active = str(item.get("url") or "") in source_urls
by_category.setdefault(category, {"candidates": 0, "active": 0, "inactive": 0})
by_category[category]["candidates"] += 1
by_category[category]["active" if active else "inactive"] += 1
rows.append({**item, "active": active})
inactive = [item for item in rows if not item["active"]]
counts = {
"sources": len(source_urls),
"candidates": len(rows),
"active_candidates": sum(1 for item in rows if item["active"]),
"inactive_candidates": len(inactive),
}
findings = []
required_categories = {
"dev_section",
"dev_section_index",
"developer_glossary",
"development_standards",
"methodical_support",
"platform_doc",
}
for category in sorted(required_categories):
stats = by_category.get(category) or {}
if stats.get("active", 0) == 0:
findings.append({"severity": "error", "message": f"no active source for required category {category}"})
return {
"schema": "onec_its_start_link_coverage.v1",
"passed": not any(item["severity"] == "error" for item in findings),
"start_links": str(start_links_path),
"sources": str(sources_path),
"counts": counts,
"by_category": dict(sorted(by_category.items())),
"findings": findings,
"inactive_samples": inactive[:50],
}
def main() -> int:
parser = argparse.ArgumentParser(description="Compare discovered 1C:ITS start links with active sources.yaml seeds.")
parser.add_argument("--start-links", type=Path, default=DEFAULT_START_LINKS)
parser.add_argument("--sources", type=Path, default=DEFAULT_SOURCES)
parser.add_argument("--output", type=Path, default=DEFAULT_OUTPUT)
parser.add_argument("--print", action="store_true", dest="print_report")
args = parser.parse_args()
report = check_coverage(args.start_links, args.sources)
args.output.parent.mkdir(parents=True, exist_ok=True)
args.output.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
if args.print_report:
print(json.dumps(report, ensure_ascii=False, indent=2))
return 0 if report["passed"] else 1
if __name__ == "__main__":
raise SystemExit(main())
+124
View File
@@ -0,0 +1,124 @@
from __future__ import annotations
import argparse
import json
import re
from pathlib import Path
from typing import Any
ROOT = Path(__file__).resolve().parents[1]
DEFAULT_STATIC_DIR = ROOT / "plugins" / "1c" / "rag" / "official-docs" / "static"
DEFAULT_OUTPUT = ROOT / "reports" / "1c-official-docs-static-check.json"
LOCAL_REF_RE = re.compile(r"""(?:href|src)=["']([^"']+)["']""", re.IGNORECASE)
EXTERNAL_IMG_RE = re.compile(r"""<img[^>]+src=["']https?://""", re.IGNORECASE)
EXTERNAL_ASSET_RE = re.compile(r"""<(?:link|script)[^>]+(?:href|src)=["']https?://""", re.IGNORECASE)
EXTERNAL_LINK_RE = re.compile(r"""<a[^>]+href=["']https?://""", re.IGNORECASE)
def html_files(static_dir: Path) -> list[Path]:
if not static_dir.exists():
return []
return sorted(static_dir.rglob("*.html"))
def is_local_ref(value: str) -> bool:
lowered = value.casefold()
return not (
lowered.startswith("http://")
or lowered.startswith("https://")
or lowered.startswith("mailto:")
or lowered.startswith("javascript:")
or lowered.startswith("#")
)
def check_local_refs(path: Path, text: str) -> list[dict[str, str]]:
broken = []
for match in LOCAL_REF_RE.finditer(text):
ref = match.group(1)
if not is_local_ref(ref):
continue
target = (path.parent / ref.split("#", 1)[0].split("?", 1)[0]).resolve()
if not target.exists():
broken.append({"file": str(path), "ref": ref})
return broken
def build_report(static_dir: Path) -> dict[str, Any]:
files = html_files(static_dir)
external_images = []
external_assets = []
external_links = []
broken_refs = []
for path in files:
text = path.read_text(encoding="utf-8-sig", errors="replace")
if EXTERNAL_IMG_RE.search(text):
external_images.append(str(path))
if EXTERNAL_ASSET_RE.search(text):
external_assets.append(str(path))
external_links.extend({"file": str(path), "count": len(EXTERNAL_LINK_RE.findall(text))} for _ in [0] if EXTERNAL_LINK_RE.search(text))
broken_refs.extend(check_local_refs(path, text))
manifest_path = static_dir / "manifest.json"
manifest = {}
if manifest_path.exists():
manifest = json.loads(manifest_path.read_text(encoding="utf-8-sig"))
counts = {
"html_files": len(files),
"pages": len((manifest.get("pages") or [])) if isinstance(manifest, dict) else 0,
"media_files": len(list((static_dir / "media").glob("*"))) if (static_dir / "media").exists() else 0,
"asset_files": len(list((static_dir / "assets").glob("*"))) if (static_dir / "assets").exists() else 0,
"external_image_pages": len(external_images),
"external_asset_pages": len(external_assets),
"external_link_pages": len(external_links),
"broken_local_refs": len(broken_refs),
"asset_errors": len((manifest.get("asset_errors") or [])) if isinstance(manifest, dict) else 0,
}
findings = []
if not (static_dir / "index.html").exists():
findings.append({"severity": "error", "message": "static index.html is missing"})
if counts["external_image_pages"]:
findings.append({"severity": "error", "message": "some static pages still reference remote images"})
if counts["external_asset_pages"]:
findings.append({"severity": "warning", "message": "some raw pages still reference remote CSS/JS assets"})
if counts["broken_local_refs"]:
findings.append({"severity": "error", "message": "some local href/src references are broken"})
if counts["asset_errors"]:
findings.append({"severity": "warning", "message": "some CSS/JS assets failed to download"})
return {
"schema": "onec_its_static_site_check.v1",
"passed": not any(item["severity"] == "error" for item in findings),
"static_dir": str(static_dir),
"counts": counts,
"findings": findings,
"samples": {
"external_images": external_images[:20],
"external_assets": external_assets[:20],
"external_links": external_links[:20],
"broken_local_refs": broken_refs[:20],
"asset_errors": (manifest.get("asset_errors") or [])[:20] if isinstance(manifest, dict) else [],
},
}
def main() -> int:
parser = argparse.ArgumentParser(description="Check local static 1C:ITS archive self-containment and links.")
parser.add_argument("--static-dir", type=Path, default=DEFAULT_STATIC_DIR)
parser.add_argument("--output", type=Path, default=DEFAULT_OUTPUT)
parser.add_argument("--print", action="store_true", dest="print_report")
args = parser.parse_args()
report = build_report(args.static_dir)
args.output.parent.mkdir(parents=True, exist_ok=True)
args.output.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
if args.print_report:
print(json.dumps(report, ensure_ascii=False, indent=2))
return 0 if report["passed"] else 1
if __name__ == "__main__":
raise SystemExit(main())
+552
View File
@@ -0,0 +1,552 @@
#!/usr/bin/env python3
from __future__ import annotations
import argparse
import json
import sys
from pathlib import Path
from typing import Any
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "scripts"))
sys.path.insert(0, str(ROOT / "plugins" / "1c" / "connector"))
sys.path.insert(0, str(ROOT / "plugins" / "1c" / "mcp"))
import adapter_1c_mcp as adapter_mcp # noqa: E402
import adapter_1c_server as adapter_server # noqa: E402
import smoke_1c_mcp_selector_chain as selector_chain_smoke # noqa: E402
FORBIDDEN_CONCRETE_SELECTOR_VALUES = (
"ПечатьЭтикеток",
"УОП_Печать",
"ЦенаСоСкидкой",
"fs_Отчеты",
)
PRODUCTION_SELECTOR_CONTRACT_ROOTS = (
ROOT / "plugins" / "1c",
ROOT / "docs",
)
PRODUCTION_SELECTOR_CONTRACT_SUFFIXES = {
".css",
".html",
".js",
".json",
".md",
".ps1",
".py",
".toml",
".txt",
".yaml",
".yml",
}
SYNTHETIC_SELECTOR_GUID = "00000000-0000-4000-8000-000000000001"
PRODUCTION_SELECTOR_CONTRACT_EXCLUDED_PARTS = {
"__pycache__",
".pytest_cache",
"reports",
"node_modules",
".git",
}
def find_tool(name: str) -> dict[str, Any] | None:
for tool in adapter_mcp.TOOLS:
if tool.get("name") == name:
return tool
return None
def iter_production_selector_contract_files() -> list[Path]:
files: list[Path] = []
for root in PRODUCTION_SELECTOR_CONTRACT_ROOTS:
if not root.exists():
continue
for path in root.rglob("*"):
if not path.is_file():
continue
try:
relative_parts = set(path.relative_to(ROOT).parts)
except ValueError:
relative_parts = set(path.parts)
if relative_parts.intersection(PRODUCTION_SELECTOR_CONTRACT_EXCLUDED_PARTS):
continue
if path.suffix.lower() not in PRODUCTION_SELECTOR_CONTRACT_SUFFIXES:
continue
files.append(path)
return sorted(files)
def contract_checks(issues: list[dict[str, Any]], *, onec_request_present: bool) -> dict[str, bool]:
return {
"generic_onec_request_schema": onec_request_present
and not any(
issue["code"].startswith("mcp_onec_request")
or issue["code"] in {"mcp_payload_not_open", "mcp_payload_selector_guidance_missing"}
for issue in issues
),
"mcp_tool_selector_guidance": not any(issue["code"] in {"mcp_tool_selector_guidance_missing", "mcp_examples_public_ref_placeholder_missing"} for issue in issues),
"all_adapter_methods_forward_over_rpc": not any(issue["code"].startswith("mcp_method") or issue["code"] == "mcp_rpc_body_method_mismatch" for issue in issues),
"no_unified_shadowing_adapter_methods": not any(issue["code"] == "mcp_unified_shadows_adapter_methods" for issue in issues),
"adapter_help_selector_guidance": not any(issue["code"] == "adapter_help_selector_guidance_missing" for issue in issues),
"adapter_selector_argument_type_validation": not any(issue["code"] == "adapter_selector_argument_type_not_validated" for issue in issues),
"adapter_selector_normalizer_type_validation": not any(issue["code"] == "adapter_selector_normalizer_type_not_validated" for issue in issues),
"adapter_selector_required_message": not any(
issue["code"] in {"adapter_selector_required_message_incomplete", "adapter_selector_message_constant_incomplete"}
for issue in issues
),
"adapter_selector_presence_aliases": not any(issue["code"] == "adapter_selector_presence_alias_mismatch" for issue in issues),
"adapter_selector_capability_descriptor": not any(
issue["code"] in {"adapter_selector_capability_descriptor_mismatch", "adapter_help_selector_capabilities_missing"}
for issue in issues
),
"adapter_definition_read_selector_public_ref": not any(issue["code"] == "adapter_definition_read_selector_public_ref_missing" for issue in issues),
"adapter_module_read_selector_public_ref": not any(issue["code"] == "adapter_module_read_selector_public_ref_missing" for issue in issues),
"adapter_parse_ordinal_unpacked": not any(issue["code"] == "adapter_parse_ordinal_not_unpacked" for issue in issues),
"adapter_contract_version": not any(
issue["code"] in {"adapter_contract_version_missing", "mcp_contract_version_mismatch", "adapter_help_contract_version_missing"}
for issue in issues
),
"mcp_policy_blocks_missing_base_id": not any(issue["code"] == "mcp_policy_missing_base_id_not_blocked" for issue in issues),
"mcp_policy_blocks_diagnostic_fallback": not any(issue["code"] == "mcp_policy_diagnostic_fallback_not_blocked" for issue in issues),
"mcp_policy_allows_explicit_diagnostics": not any(issue["code"] == "mcp_policy_explicit_diagnostic_not_forwarded" for issue in issues),
"mcp_selector_schema_template": not any(
issue["code"] in {"mcp_selector_schema_properties_mismatch", "mcp_schema_requires_object_type_name_selector"}
for issue in issues
),
"selector_chain_live_coverage_schema": not any(issue["code"].startswith("selector_chain_coverage_") for issue in issues),
"selector_chain_strict_composition_gate": not any(issue["code"].startswith("selector_chain_strict_") for issue in issues),
"selector_chain_working_state": not any(issue["code"].startswith("selector_chain_working_state_") for issue in issues),
"no_concrete_selector_values_in_production": not any(issue["code"] == "production_concrete_selector_value_present" for issue in issues),
}
def check_contract() -> dict[str, Any]:
issues: list[dict[str, Any]] = []
methods = [str(row.get("name") or "") for row in adapter_server.METHODS if row.get("name")]
method_set = set(methods)
selector_guidance_terms = tuple(getattr(adapter_server, "OBJECT_SELECTOR_GUIDANCE_TERMS", ("ref", "kind/name/guid", "object_type/object_name/object_guid")))
adapter_contract_version = str(getattr(adapter_server, "ADAPTER_CONTRACT_VERSION", ""))
mcp_contract_version = str(getattr(adapter_mcp, "MCP_CONTRACT_VERSION", ""))
if not adapter_contract_version:
issues.append({"code": "adapter_contract_version_missing"})
if adapter_contract_version != mcp_contract_version:
issues.append({"code": "mcp_contract_version_mismatch", "adapter": adapter_contract_version, "mcp": mcp_contract_version})
duplicate_methods = sorted({name for name in methods if methods.count(name) > 1})
if duplicate_methods:
issues.append({"code": "adapter_methods_not_unique", "methods": duplicate_methods})
shadowed_methods = sorted(method_set.intersection(getattr(adapter_mcp, "UNIFIED_METHODS", set())))
if shadowed_methods:
issues.append({"code": "mcp_unified_shadows_adapter_methods", "methods": shadowed_methods})
mcp_selector_schema_args = set(getattr(adapter_mcp, "OBJECT_SELECTOR_SCHEMA_PROPERTIES", {}))
adapter_selector_args = set(getattr(adapter_server, "OBJECT_SELECTOR_ARGUMENTS", []))
if mcp_selector_schema_args != adapter_selector_args:
issues.append(
{
"code": "mcp_selector_schema_properties_mismatch",
"mcp_arguments": sorted(mcp_selector_schema_args),
"adapter_arguments": sorted(adapter_selector_args),
}
)
selector_capability_methods = set(getattr(adapter_server, "OBJECT_SELECTOR_METHOD_CAPABILITIES", {}))
selector_alias_methods = set(getattr(adapter_server, "OBJECT_SELECTOR_ALIAS_METHODS", set()))
if selector_capability_methods != selector_alias_methods:
issues.append(
{
"code": "adapter_selector_capability_descriptor_mismatch",
"capability_methods": sorted(selector_capability_methods),
"alias_methods": sorted(selector_alias_methods),
}
)
mcp_source = Path(adapter_mcp.__file__).read_text(encoding="utf-8")
adapter_source = Path(adapter_server.__file__).read_text(encoding="utf-8")
for line_number, line in enumerate(adapter_source.splitlines(), start=1):
if "parse_ordinal(" not in line or "=" not in line:
continue
left_side = line.split("=", 1)[0].strip()
if "," not in left_side and not left_side.startswith(("return ", "raise ")):
issues.append(
{
"code": "adapter_parse_ordinal_not_unpacked",
"line": line_number,
"source": line.strip(),
}
)
if '"required": ["base_id", "object_type", "object_name"]' in mcp_source:
issues.append({"code": "mcp_schema_requires_object_type_name_selector"})
normalizer_check = adapter_server.normalize_object_selector_aliases({"kind": 123}, "contract.selector")
if not isinstance(normalizer_check, dict) or normalizer_check.get("status") != "invalid_argument" or normalizer_check.get("argument") != "kind":
issues.append(
{
"code": "adapter_selector_normalizer_type_not_validated",
"result": normalizer_check,
}
)
required_message = str(getattr(adapter_server, "OBJECT_SELECTOR_REQUIRED_MESSAGE", ""))
for term in (*selector_guidance_terms, "ordinal"):
if term not in required_message:
issues.append(
{
"code": "adapter_selector_required_message_incomplete",
"term": term,
"message": required_message,
}
)
selector_message_constants = {
"OBJECT_SELECTOR_REQUIRED_MESSAGE_NO_ORDINAL": selector_guidance_terms,
"OBJECT_SELECTOR_GLOBAL_REQUIRED_MESSAGE": (*selector_guidance_terms, "areas metadata/extensions"),
"OBJECT_SELECTOR_OR_MODULE_REQUIRED_MESSAGE": (*selector_guidance_terms, "module_ref", "module_id"),
"MODULE_READ_SELECTOR_OR_MODULE_ID_MESSAGE": (*selector_guidance_terms, "ordinal", "module_id"),
}
for constant_name, required_terms in selector_message_constants.items():
message = str(getattr(adapter_server, constant_name, ""))
for term in required_terms:
if term not in message:
issues.append(
{
"code": "adapter_selector_message_constant_incomplete",
"constant": constant_name,
"term": term,
"message": message,
}
)
selector_presence_cases = [
({"ref": "Document.ObjectName"}, True),
({"object_name": "ObjectName"}, True),
({"object_guid": SYNTHETIC_SELECTOR_GUID}, True),
({"kind": "Document"}, False),
({"object_type": "Document"}, False),
]
for payload, expected in selector_presence_cases:
actual = adapter_server.has_object_selector(payload)
if actual is not expected:
issues.append(
{
"code": "adapter_selector_presence_alias_mismatch",
"payload": payload,
"expected": expected,
"actual": actual,
}
)
read_selector = adapter_server.definition_read_selector(
"contract_base",
{
"kind": "DataProcessor",
"name": "ObjectName",
"guid": SYNTHETIC_SELECTOR_GUID,
},
method="metadata.object.get",
)
if read_selector.get("ref") != "DataProcessor.ObjectName":
issues.append(
{
"code": "adapter_definition_read_selector_public_ref_missing",
"selector": read_selector,
}
)
module_read_selector = adapter_server.enrich_selector_with_object_ref(
{
"base_id": "contract_base",
"method": "modules.read",
"kind": "DataProcessor",
"guid": SYNTHETIC_SELECTOR_GUID,
"module_ordinal": 1,
},
{
"kind": "DataProcessor",
"name": "ObjectName",
"guid": SYNTHETIC_SELECTOR_GUID,
},
)
if module_read_selector.get("ref") != "DataProcessor.ObjectName" or module_read_selector.get("name") != "ObjectName":
issues.append(
{
"code": "adapter_module_read_selector_public_ref_missing",
"selector": module_read_selector,
}
)
synthetic_live_steps = [
{
"name": "metadata.resolve_overrides",
"status": "not_found",
"write_plan_evidence": True,
"next_method": adapter_server.SAVED_STATE_MODULES_SEARCH_METHOD,
},
{
"name": adapter_server.SAVED_STATE_MODULES_SEARCH_METHOD,
"status": "ok",
"modules": 0,
"write_plan_target": False,
},
{
"name": adapter_server.METADATA_WRITE_PLAN_METHOD,
"status": "skipped_no_saved_state_target",
"from_write_plan_target": False,
},
]
live_coverage = selector_chain_smoke.live_coverage_from_steps(synthetic_live_steps)
expected_coverage = {
"resolve_overrides": {
"attempted": True,
"status": "not_found",
"write_plan_evidence": True,
"next_method": adapter_server.SAVED_STATE_MODULES_SEARCH_METHOD,
},
"saved_state_resolution": {
"attempted": True,
"status": "ok",
"modules": 0,
"write_plan_target": False,
},
"write_plan_composition": {
"attempted": True,
"status": "skipped_no_saved_state_target",
"composed": False,
"from_write_plan_target": False,
},
}
for section, expected in expected_coverage.items():
actual = live_coverage.get(section)
if actual != expected:
issues.append(
{
"code": "selector_chain_coverage_section_mismatch",
"section": section,
"expected": expected,
"actual": actual,
}
)
if live_coverage.get("skips") != [{"step": adapter_server.METADATA_WRITE_PLAN_METHOD, "status": "skipped_no_saved_state_target"}]:
issues.append({"code": "selector_chain_coverage_skips_mismatch", "actual": live_coverage.get("skips")})
strict_skip_issue = selector_chain_smoke.live_write_plan_composition_required_issue(live_coverage)
if not isinstance(strict_skip_issue, dict) or strict_skip_issue.get("code") != "live_write_plan_composition_required":
issues.append({"code": "selector_chain_strict_skip_not_blocked", "actual": strict_skip_issue})
composed_coverage = selector_chain_smoke.live_coverage_from_steps(
[
{
"name": "metadata.resolve_overrides",
"status": "ok",
"write_plan_evidence": True,
"next_method": adapter_server.SAVED_STATE_MODULES_SEARCH_METHOD,
},
{
"name": adapter_server.SAVED_STATE_MODULES_SEARCH_METHOD,
"status": "ok",
"modules": 1,
"write_plan_target": True,
},
{
"name": adapter_server.METADATA_WRITE_PLAN_METHOD,
"status": "planned",
"allowed": True,
"from_write_plan_target": True,
},
]
)
strict_composed_issue = selector_chain_smoke.live_write_plan_composition_required_issue(composed_coverage)
if strict_composed_issue is not None:
issues.append({"code": "selector_chain_strict_composed_blocked", "actual": strict_composed_issue})
selector_chain_report = selector_chain_smoke.build_report()
if selector_chain_report.get("passed") is not True:
issues.append({"code": "selector_chain_working_state_smoke_failed", "report": selector_chain_report})
for chain in selector_chain_report.get("chains") or []:
if not isinstance(chain, dict):
continue
for step in chain.get("steps") or []:
if not isinstance(step, dict) or step.get("method") not in selector_chain_smoke.WORKING_STATE_METHODS:
continue
payload = step.get("payload") if isinstance(step.get("payload"), dict) else {}
if payload.get("source_state") != "working" and payload.get("state") != "working":
issues.append(
{
"code": "selector_chain_working_state_missing",
"chain": chain.get("name"),
"method": step.get("method"),
}
)
production_selector_files = iter_production_selector_contract_files()
for path in production_selector_files:
if not path.exists():
continue
text = path.read_text(encoding="utf-8", errors="replace")
for value in FORBIDDEN_CONCRETE_SELECTOR_VALUES:
if value in text:
issues.append(
{
"code": "production_concrete_selector_value_present",
"path": str(path.relative_to(ROOT)),
"value": value,
}
)
public_methods = {
str(row.get("name") or ""): row
for row in adapter_server.call_method("help.methods", {}).get("methods", [])
if isinstance(row, dict) and row.get("name")
}
help_result = adapter_server.call_method("help.methods", {})
if help_result.get("contract_version") != adapter_contract_version:
issues.append(
{
"code": "adapter_help_contract_version_missing",
"expected": adapter_contract_version,
"actual": help_result.get("contract_version"),
}
)
for method in sorted(getattr(adapter_server, "OBJECT_SELECTOR_ALIAS_METHODS", set())):
capabilities = (public_methods.get(method) or {}).get("selector_capabilities")
if not isinstance(capabilities, dict) or not capabilities.get("accepts_ref") or not capabilities.get("accepts_object_aliases"):
issues.append(
{
"code": "adapter_help_selector_capabilities_missing",
"method": method,
"selector_capabilities": capabilities,
}
)
description = str((public_methods.get(method) or {}).get("description") or "")
for term in selector_guidance_terms:
if term not in description:
issues.append(
{
"code": "adapter_help_selector_guidance_missing",
"method": method,
"term": term,
"description": description,
}
)
for argument in getattr(adapter_server, "OBJECT_SELECTOR_ARGUMENTS", ["ref"]):
selector_check = adapter_server.validate_adapter_job_payload(method, {"base_id": "contract_base", argument: 123})
if (
not isinstance(selector_check, dict)
or selector_check.get("status") != "invalid_argument"
or selector_check.get("argument") != argument
):
issues.append(
{
"code": "adapter_selector_argument_type_not_validated",
"method": method,
"argument": argument,
"result": selector_check,
}
)
onec_request = find_tool("onec_request")
if not onec_request:
issues.append({"code": "mcp_onec_request_missing"})
else:
schema = onec_request.get("inputSchema") or {}
tool_description = str(onec_request.get("description") or "")
for term in selector_guidance_terms:
if term not in tool_description:
issues.append(
{
"code": "mcp_tool_selector_guidance_missing",
"term": term,
"description": tool_description,
}
)
examples_json = json.dumps(schema.get("examples") or [], ensure_ascii=False)
if "<metadata-kind>.<metadata-object-name>" not in examples_json:
issues.append({"code": "mcp_examples_public_ref_placeholder_missing"})
if schema.get("required") != ["method"]:
issues.append({"code": "mcp_onec_request_required_not_generic", "required": schema.get("required")})
if "oneOf" in schema:
issues.append({"code": "mcp_onec_request_enumerates_adapter_methods"})
payload_schema = (schema.get("properties") or {}).get("payload") or {}
if payload_schema.get("additionalProperties") is not True:
issues.append({"code": "mcp_payload_not_open", "payload_schema": payload_schema})
payload_description = str(payload_schema.get("description") or "")
for term in selector_guidance_terms:
if term not in payload_description:
issues.append(
{
"code": "mcp_payload_selector_guidance_missing",
"term": term,
"description": payload_description,
}
)
calls: list[tuple[str, str, Any]] = []
def fake_http_json(method: str, path: str, body: Any | None = None, *, timeout: float | None = None) -> dict[str, Any]:
calls.append((method, path, body))
return {"status": "ok", "method": body.get("method") if isinstance(body, dict) else "health"}
original_http_json = adapter_mcp.http_json
adapter_mcp.http_json = fake_http_json
try:
calls.clear()
missing_base_id_result = adapter_mcp.run_or_enqueue_adapter_method("metadata.write.plan", {})
if calls or not isinstance(missing_base_id_result, dict) or missing_base_id_result.get("schema") != "adapter_1c_mcp_policy.v1" or missing_base_id_result.get("reason") != "base_id_required":
issues.append(
{
"code": "mcp_policy_missing_base_id_not_blocked",
"result": missing_base_id_result,
"calls": calls,
}
)
calls.clear()
diagnostic_fallback_result = adapter_mcp.run_or_enqueue_adapter_method("storage.files.list", {"base_id": "contract_base"})
if calls or not isinstance(diagnostic_fallback_result, dict) or diagnostic_fallback_result.get("schema") != "adapter_1c_mcp_policy.v1" or diagnostic_fallback_result.get("reason") != "diagnostic_method":
issues.append(
{
"code": "mcp_policy_diagnostic_fallback_not_blocked",
"result": diagnostic_fallback_result,
"calls": calls,
}
)
calls.clear()
adapter_mcp.run_or_enqueue_adapter_method("storage.files.list", {"base_id": "contract_base", "diagnostic": True})
if not calls:
issues.append({"code": "mcp_policy_explicit_diagnostic_not_forwarded"})
for method in methods:
calls.clear()
adapter_mcp.call_adapter_method(method, {"base_id": "contract_base"})
if method == "health":
expected = ("GET", "/health?base_id=contract_base")
if not calls or calls[0][0:2] != expected:
issues.append({"code": "mcp_health_not_get_health", "method": method, "calls": calls})
continue
if not calls:
issues.append({"code": "mcp_method_not_forwarded", "method": method})
continue
http_method, path, body = calls[0]
if http_method != "POST" or path != "/rpc":
issues.append({"code": "mcp_method_not_rpc", "method": method, "calls": calls})
continue
if not isinstance(body, dict) or body.get("method") != method:
issues.append({"code": "mcp_rpc_body_method_mismatch", "method": method, "body": body})
finally:
adapter_mcp.http_json = original_http_json
return {
"schema": "onec_mcp_adapter_contract_check.v1",
"passed": not issues,
"adapter_methods": len(methods),
"production_selector_files": len(production_selector_files),
"mcp_tools": [tool.get("name") for tool in adapter_mcp.TOOLS],
"checks": contract_checks(issues, onec_request_present=onec_request is not None),
"issues": issues,
}
def main() -> int:
parser = argparse.ArgumentParser(description="Check that adapter-1c MCP stays a generic proxy for REST adapter methods.")
parser.add_argument("--json", action="store_true", help="Print JSON report.")
args = parser.parse_args()
report = check_contract()
if args.json:
print(json.dumps(report, ensure_ascii=False, indent=2))
elif report["passed"]:
print(f"OK: MCP generic proxy contract passed for {report['adapter_methods']} adapter methods.")
else:
print(json.dumps(report, ensure_ascii=False, indent=2), file=sys.stderr)
return 0 if report["passed"] else 1
if __name__ == "__main__":
raise SystemExit(main())
+129
View File
@@ -0,0 +1,129 @@
from __future__ import annotations
import argparse
import json
import sys
from pathlib import Path
from typing import Any
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT))
sys.path.insert(0, str(ROOT / "plugins" / "1c"))
sys.path.insert(0, str(ROOT / "plugins" / "1c" / "connector"))
import adapter_1c_server as adapter_server # noqa: E402
def require(condition: bool, message: str, failures: list[str]) -> None:
if not condition:
failures.append(message)
def patch_adapter_reads() -> None:
adapter_server.read_storage_file_bytes = lambda base_id, table, file_name, timeout_seconds=30: (b"fake", {"database": "fake"}, None)
adapter_server.payload_text_from_bytes = lambda data: {
"status": "ok",
"text": "Процедура Проверка()\nКонецПроцедуры",
}
adapter_server.extension_module_owner_payload = lambda base_id, module_id, table, timeout_seconds=30: None
adapter_server.cached_module_owner_payload = lambda base_id, module_id: None
def read_origin(method: str, module_ref: str) -> dict[str, Any]:
result = adapter_server.call_method(
method,
{
"base_id": "upo_test",
"module_ref": module_ref,
"include_text": True,
"max_chars": 200,
},
)
origin = result.get("origin") if isinstance(result.get("origin"), dict) else {}
return {"result": result, "origin": origin}
def check_code_search_origin(failures: list[str]) -> None:
adapter_server.search_modules = lambda payload: {
"status": "ok",
"source": {"kind": "live_metadata"},
"matches": [
{
"snippet": {"text": "Процедура Проверка()", "offset": 0},
"owner": {"status": "unresolved", "kind": None, "name": None},
"origin": {
"source": "cas_reference",
"status": "owner_unresolved",
"write_surface": "requires_owner_resolution",
},
"module": {"name": "Модуль БСЛ"},
"read_selector": {"base_id": "upo_test", "module_ref": "ConfigCAS:object-module"},
}
],
"counts": {"matches": 1, "complete": True, "scan_limit_hit": False},
"diagnostics": {},
}
result = adapter_server.call_method("code.search", {"base_id": "upo_test", "query": "Проверка"})
items = result.get("items") if isinstance(result.get("items"), list) else []
origin = items[0].get("origin") if items and isinstance(items[0], dict) and isinstance(items[0].get("origin"), dict) else {}
require(result.get("schema") == "onec_code_search.v1", "code.search must return code search schema", failures)
require(bool(items), "code.search must return patched item", failures)
require(origin.get("source") == "cas_reference", "code.search item must preserve modules.search origin", failures)
require(origin.get("write_surface") == "requires_owner_resolution", "code.search item origin must keep write_surface", failures)
def run_checks() -> dict[str, Any]:
patch_adapter_reads()
failures: list[str] = []
config = read_origin("modules.read", "Config:object-module")
require(config["result"].get("status") == "ok", "Config module_ref must read in patched contract", failures)
require(config["origin"].get("source") == "configuration", "Config module_ref must expose configuration origin", failures)
require(config["origin"].get("write_surface") == "base_saved_state", "Config origin must point writes to base saved-state", failures)
save = read_origin("modules.read", "ConfigSave:object-module")
require(save["result"].get("status") == "ok", "ConfigSave module_ref must read in patched contract", failures)
require(save["origin"].get("source") == "saved_state", "ConfigSave module_ref must expose saved_state origin", failures)
require(save["origin"].get("write_surface") == "base_saved_state", "ConfigSave origin must point writes to base saved-state", failures)
cas = read_origin("modules.read", "ConfigCAS:object-module")
require(cas["result"].get("status") == "ok", "ConfigCAS module_ref must read in patched contract", failures)
require(cas["origin"].get("source") == "cas_reference", "ConfigCAS fallback must not pretend extension/base owner", failures)
require(cas["origin"].get("status") == "owner_unresolved", "ConfigCAS fallback must require owner resolution", failures)
require(cas["origin"].get("write_surface") == "requires_owner_resolution", "ConfigCAS fallback must block direct write routing", failures)
code_save = read_origin("code.read", "ConfigSave:object-module")
require(code_save["result"].get("schema") == "onec_code_read.v1", "code.read must wrap modules.read as onec_code_read", failures)
require(code_save["origin"].get("source") == "saved_state", "code.read must preserve modules.read origin", failures)
require(code_save["origin"].get("write_surface") == "base_saved_state", "code.read origin must keep write_surface", failures)
check_code_search_origin(failures)
return {
"schema": "onec_module_origin_contract_check.v1",
"status": "ok" if not failures else "failed",
"failures": failures,
"checks": {
"config_origin": "Config module_ref exposes configuration origin",
"configsave_origin": "ConfigSave module_ref exposes saved_state origin",
"configcas_owner_required": "ConfigCAS module_ref requires owner resolution",
"code_read_preserves_origin": "code.read preserves origin evidence from modules.read",
"code_search_preserves_origin": "code.search preserves origin evidence from modules.search",
},
}
def main() -> int:
parser = argparse.ArgumentParser(description="Check modules.read origin/provenance contract invariants.")
parser.add_argument("--print", action="store_true")
args = parser.parse_args()
report = run_checks()
if args.print or report["status"] != "ok":
print(json.dumps(report, ensure_ascii=False, indent=2))
else:
print("1C module origin contract status: ok")
return 0 if report["status"] == "ok" else 1
if __name__ == "__main__":
raise SystemExit(main())
+57
View File
@@ -0,0 +1,57 @@
from __future__ import annotations
import argparse
import json
from pathlib import Path
from typing import Any
def read_json(path: Path) -> dict[str, Any]:
return json.loads(path.read_text(encoding="utf-8"))
def check_next_action(plan: dict[str, Any], action_payload: dict[str, Any]) -> dict[str, Any]:
failures: list[str] = []
if action_payload.get("schema") != "codex_1c_moxel_next_action.v1":
failures.append("unexpected next action schema")
action = action_payload.get("action") if isinstance(action_payload.get("action"), dict) else None
experiments = [item for item in plan.get("experiments") or [] if isinstance(item, dict)]
if not action:
if experiments:
failures.append("next action is empty but experiments are available")
else:
action_id = action.get("id")
matching = [item for item in experiments if item.get("id") == action_id]
if not matching:
failures.append(f"next action id is not present in plan: {action_id}")
command = str(action.get("capture_command_with_pipeline") or "")
if "--run-pipeline-after" not in command:
failures.append("capture_command_with_pipeline must include --run-pipeline-after")
for field in ("manual_action", "expected_signal", "target"):
if not action.get(field):
failures.append(f"next action must include {field}")
return {
"schema": "codex_1c_moxel_next_action_check.v1",
"status": "ok" if not failures else "failed",
"failures": failures,
"counts": {"experiments": len(experiments), "has_action": action is not None},
}
def main() -> int:
parser = argparse.ArgumentParser(description="Validate the machine-readable 1C MOXCEL next action artifact.")
parser.add_argument("--plan", default="reports/1c-template-baselines/moxel-next-experiments.json")
parser.add_argument("--action", default="reports/1c-template-baselines/moxel-next-action.json")
parser.add_argument("--output", default="reports/1c-template-baselines/moxel-next-action-check.json")
args = parser.parse_args()
report = check_next_action(read_json(Path(args.plan)), read_json(Path(args.action)))
output_path = Path(args.output)
output_path.parent.mkdir(parents=True, exist_ok=True)
output_path.write_text(json.dumps(report, ensure_ascii=False, indent=2), encoding="utf-8")
print(json.dumps(report, ensure_ascii=False, indent=2))
return 0 if report["status"] == "ok" else 1
if __name__ == "__main__":
raise SystemExit(main())
+85
View File
@@ -0,0 +1,85 @@
from __future__ import annotations
import argparse
import json
from pathlib import Path
from typing import Any
def read_json(path: Path) -> dict[str, Any]:
return json.loads(path.read_text(encoding="utf-8"))
def fail(message: str, failures: list[str]) -> None:
failures.append(message)
def check_registry(registry: dict[str, Any]) -> dict[str, Any]:
failures: list[str] = []
if registry.get("schema") != "codex_1c_moxel_schema_registry.v1":
fail("unexpected schema", failures)
rules = [rule for rule in registry.get("rules") or [] if isinstance(rule, dict)]
if not rules:
fail("registry must contain rules", failures)
verified = [rule for rule in rules if rule.get("read_status") == "verified_read"]
if not verified:
fail("registry must contain at least one verified_read rule", failures)
write_enabled = [rule for rule in rules if rule.get("write_status") == "verified_roundtrip"]
if write_enabled:
fail("write rules must stay disabled until explicit round-trip evidence is implemented", failures)
counts = registry.get("counts") if isinstance(registry.get("counts"), dict) else {}
expected_counts = {
"rules": len(rules),
"verified_read": len(verified),
"candidate_read": sum(1 for rule in rules if rule.get("read_status") == "candidate_read"),
"write_enabled": len(write_enabled),
}
for key, expected in expected_counts.items():
if counts.get(key) != expected:
fail(f"counts.{key} must be {expected}, got {counts.get(key)}", failures)
inline = [rule for rule in rules if rule.get("target") == "moxel.inline_text_cell.column"]
if not inline:
fail("missing moxel.inline_text_cell.column rule", failures)
else:
rule = inline[0]
evidence = rule.get("evidence") if isinstance(rule.get("evidence"), dict) else {}
if rule.get("read_status") != "verified_read":
fail("inline column rule must be verified_read", failures)
if evidence.get("ok") != evidence.get("total") or not isinstance(evidence.get("total"), int) or evidence.get("total") <= 0:
fail("inline column rule evidence must be complete", failures)
for rule in rules:
if not rule.get("target"):
fail(f"rule {rule.get('id')} has no target", failures)
if rule.get("read_status") not in {"verified_read", "candidate_read", "needs_more_evidence"}:
fail(f"rule {rule.get('id')} has unsupported read_status", failures)
if rule.get("write_status") not in {"blocked_until_roundtrip", "blocked_until_verified_read", "verified_roundtrip"}:
fail(f"rule {rule.get('id')} has unsupported write_status", failures)
return {
"schema": "codex_1c_moxel_schema_registry_check.v1",
"status": "ok" if not failures else "failed",
"failures": failures,
"counts": {
**expected_counts,
},
}
def main() -> int:
parser = argparse.ArgumentParser(description="Validate the 1C MOXCEL schema registry safety contract.")
parser.add_argument("--registry", default="plugins/1c/metadata/moxel-schema-registry.json")
parser.add_argument("--output", default="reports/1c-template-baselines/moxel-schema-registry-check.json")
args = parser.parse_args()
report = check_registry(read_json(Path(args.registry)))
output_path = Path(args.output)
output_path.parent.mkdir(parents=True, exist_ok=True)
output_path.write_text(json.dumps(report, ensure_ascii=False, indent=2), encoding="utf-8")
print(json.dumps(report, ensure_ascii=False, indent=2))
return 0 if report["status"] == "ok" else 1
if __name__ == "__main__":
raise SystemExit(main())
@@ -0,0 +1,60 @@
from __future__ import annotations
import argparse
import json
from pathlib import Path
from typing import Any
ROOT = Path(__file__).resolve().parents[1]
DEFAULT_ROOT = ROOT / "plugins" / "1c" / "rag" / "official-docs"
PRIVATE_DIRS = ("raw", "normalized")
ALLOWED_PRIVATE_FILES = {".gitkeep"}
def check_private_artifacts(root: Path) -> dict[str, Any]:
findings: list[dict[str, Any]] = []
counts = {"private_files": 0, "unexpected_private_files": 0}
for dirname in PRIVATE_DIRS:
directory = root / dirname
if not directory.exists():
findings.append({"severity": "warning", "code": "missing_private_dir", "path": str(directory)})
continue
for path in sorted(item for item in directory.rglob("*") if item.is_file()):
if path.name in ALLOWED_PRIVATE_FILES:
continue
counts["private_files"] += 1
counts["unexpected_private_files"] += 1
findings.append(
{
"severity": "info",
"code": "private_artifact_present",
"message": "Private official documentation artifact exists locally; it must remain ignored and uncommitted.",
"path": str(path),
}
)
return {
"schema": "onec_official_docs_private_artifact_check.v1",
"root": str(root),
"passed": not any(item["severity"] == "error" for item in findings),
"counts": counts,
"findings": findings,
}
def main() -> int:
parser = argparse.ArgumentParser(description="Check local private 1C official-doc artifacts.")
parser.add_argument("--root", type=Path, default=DEFAULT_ROOT)
parser.add_argument("--output", type=Path)
args = parser.parse_args()
result = check_private_artifacts(args.root)
if args.output:
args.output.parent.mkdir(parents=True, exist_ok=True)
args.output.write_text(json.dumps(result, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
print(json.dumps({"passed": result["passed"], "counts": result["counts"], "output": str(args.output) if args.output else None}, ensure_ascii=False))
return 0 if result["passed"] else 2
if __name__ == "__main__":
raise SystemExit(main())
+188
View File
@@ -0,0 +1,188 @@
from __future__ import annotations
import argparse
import json
from pathlib import Path
from typing import Any
ROOT = Path(__file__).resolve().parents[1]
DEFAULT_NORMALIZED_MANIFEST = ROOT / "plugins" / "1c" / "rag" / "official-docs" / "normalized" / "manifest.json"
DEFAULT_RAW_MANIFEST = ROOT / "plugins" / "1c" / "rag" / "official-docs" / "raw" / "manifest.json"
DEFAULT_RAG_SOURCE_DIR = ROOT / "plugins" / "1c" / "rag" / "sources" / "official" / "its"
DEFAULT_CORPUS = ROOT / "plugins" / "1c" / "datasets" / "prepared" / "rag_corpus.jsonl"
FORBIDDEN_TEXT = (
"Мы используем файлы cookie",
"Продолжая находиться на сайте",
"Результаты поиска",
"Купить кассу",
"Календарь бухгалтера",
"Последние результаты поиска",
)
NAVIGATION_CLUES = (
"Руководство разработчика - Руководство администратора",
"Глоссарий разработчика - 1 - 1CEClientSetupMake.exe",
)
def load_json(path: Path) -> dict[str, Any]:
if not path.exists():
return {}
try:
data = json.loads(path.read_text(encoding="utf-8-sig"))
except json.JSONDecodeError:
return {}
return data if isinstance(data, dict) else {}
def scan_text_file(path: Path, needles: tuple[str, ...]) -> list[dict[str, Any]]:
try:
text = path.read_text(encoding="utf-8-sig", errors="ignore")
except OSError as exc:
return [{"severity": "error", "code": "read_failed", "path": str(path), "message": str(exc)}]
findings = []
for needle in needles:
if needle in text:
findings.append({"severity": "error", "code": "forbidden_text", "path": str(path), "text": needle})
return findings
def scan_jsonl(path: Path, needles: tuple[str, ...]) -> list[dict[str, Any]]:
findings: list[dict[str, Any]] = []
if not path.exists():
findings.append({"severity": "warning", "code": "missing_corpus", "path": str(path)})
return findings
for line_no, line in enumerate(path.read_text(encoding="utf-8-sig", errors="ignore").splitlines(), start=1):
if not line.strip():
continue
try:
item = json.loads(line)
except json.JSONDecodeError:
findings.append({"severity": "error", "code": "bad_jsonl", "path": str(path), "line": line_no})
continue
content = str(item.get("content") or "")
for needle in needles:
if needle in content:
findings.append(
{
"severity": "error",
"code": "forbidden_text_in_corpus",
"path": str(path),
"line": line_no,
"chunk_id": item.get("id"),
"text": needle,
}
)
return findings
def raw_url_counts(raw_manifest_path: Path) -> dict[str, int]:
manifest = load_json(raw_manifest_path)
counts = {"raw_pages": 0, "content_src_pages": 0, "hdoc_pages": 0, "root_pages": 0}
for page in manifest.get("pages") or []:
counts["raw_pages"] += 1
url = str(page.get("url") or "")
if "/db/content/" in url and "/src/" in url:
counts["content_src_pages"] += 1
elif "/content/" in url and url.endswith("/hdoc"):
counts["hdoc_pages"] += 1
else:
counts["root_pages"] += 1
return counts
def check_quality(manifest_path: Path, raw_manifest_path: Path, rag_source_dir: Path, corpus_path: Path) -> dict[str, Any]:
findings: list[dict[str, Any]] = []
manifest = load_json(manifest_path)
raw_counts = raw_url_counts(raw_manifest_path)
page_count = int(manifest.get("page_count") or 0)
skipped_count = int(manifest.get("skipped_count") or 0)
discovered_src_count = int(manifest.get("discovered_src_record_count") or 0)
media_page_count = 0
media_image_count = 0
table_count = 0
for page in manifest.get("pages") or []:
media = page.get("media") or {}
images = media.get("images") or []
if images:
media_page_count += 1
media_image_count += len(images)
table_count += int(media.get("table_count") or 0)
if not manifest:
findings.append({"severity": "warning", "code": "missing_normalized_manifest", "path": str(manifest_path)})
elif page_count == 0:
findings.append(
{
"severity": "warning",
"code": "no_official_content_pages",
"message": "No official 1C:ITS pages passed normalization quality gates. Refresh cookie and fetch with --no-resume.",
"skipped_count": skipped_count,
}
)
if raw_counts["raw_pages"] and not raw_counts["content_src_pages"] and not discovered_src_count:
findings.append(
{
"severity": "warning",
"code": "no_raw_content_src_pages",
"message": "Raw fetch has pages, but no /db/content/.../src/... pages. Fetch probably stopped at hdoc shells/navigation.",
"raw_counts": raw_counts,
}
)
if rag_source_dir.exists():
for path in sorted(rag_source_dir.glob("*.md")):
findings.extend(scan_text_file(path, FORBIDDEN_TEXT + NAVIGATION_CLUES))
else:
findings.append({"severity": "warning", "code": "missing_rag_source_dir", "path": str(rag_source_dir)})
findings.extend(scan_jsonl(corpus_path, FORBIDDEN_TEXT + NAVIGATION_CLUES))
errors = [item for item in findings if item.get("severity") == "error"]
warnings = [item for item in findings if item.get("severity") == "warning"]
return {
"schema": "onec_official_docs_quality_check.v1",
"passed": not errors,
"counts": {
**raw_counts,
"discovered_src_record_count": discovered_src_count,
"media_pages": media_page_count,
"media_images": media_image_count,
"tables": table_count,
"normalized_pages": page_count,
"skipped_pages": skipped_count,
"errors": len(errors),
"warnings": len(warnings),
"findings": len(findings),
},
"manifest": str(manifest_path),
"raw_manifest": str(raw_manifest_path),
"rag_source_dir": str(rag_source_dir),
"corpus": str(corpus_path),
"findings": findings,
}
def main() -> int:
parser = argparse.ArgumentParser(description="Check official 1C:ITS normalized docs and RAG corpus quality.")
parser.add_argument("--manifest", type=Path, default=DEFAULT_NORMALIZED_MANIFEST)
parser.add_argument("--raw-manifest", type=Path, default=DEFAULT_RAW_MANIFEST)
parser.add_argument("--rag-source-dir", type=Path, default=DEFAULT_RAG_SOURCE_DIR)
parser.add_argument("--corpus", type=Path, default=DEFAULT_CORPUS)
parser.add_argument("--output", type=Path)
parser.add_argument("--print", action="store_true", dest="print_full")
args = parser.parse_args()
result = check_quality(args.manifest, args.raw_manifest, args.rag_source_dir, args.corpus)
if args.output:
args.output.parent.mkdir(parents=True, exist_ok=True)
args.output.write_text(json.dumps(result, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
payload = result if args.print_full else {"passed": result["passed"], "counts": result["counts"], "output": str(args.output) if args.output else None}
print(json.dumps(payload, ensure_ascii=False, indent=2))
return 0 if result["passed"] else 2
if __name__ == "__main__":
raise SystemExit(main())
+185
View File
@@ -0,0 +1,185 @@
#!/usr/bin/env python3
"""Validate a 1C patch review bundle directory and optional zip archive."""
from __future__ import annotations
import argparse
import hashlib
import json
import zipfile
from pathlib import Path
from typing import Any
REQUIRED_FILES = {"manifest.json", "preflight.json", "preflight.md", "diff.json", "README.md"}
def load_json(path: Path) -> dict[str, Any]:
return json.loads(path.read_text(encoding="utf-8-sig"))
def write_json(path: Path, data: dict[str, Any]) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(json.dumps(data, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
def sha256_file(path: Path) -> str:
digest = hashlib.sha256()
with path.open("rb") as stream:
for chunk in iter(lambda: stream.read(1024 * 1024), b""):
digest.update(chunk)
return digest.hexdigest()
def issue(severity: str, code: str, message: str, *, path: Path | str | None = None, detail: dict[str, Any] | None = None) -> dict[str, Any]:
result: dict[str, Any] = {"severity": severity, "code": code, "message": message}
if path is not None:
result["path"] = str(path)
if detail:
result["detail"] = detail
return result
def safe_relative(relative_path: str) -> Path:
path = Path(relative_path.replace("\\", "/"))
if path.is_absolute() or ".." in path.parts or not str(path):
raise ValueError(relative_path)
return path
def all_files(root: Path) -> set[str]:
if not root.exists():
return set()
return {str(path.relative_to(root)).replace("\\", "/") for path in root.rglob("*") if path.is_file()}
def expected_working_hash(record: dict[str, Any]) -> str | None:
sha = record.get("sha256") or {}
if isinstance(sha, dict):
return sha.get("working")
return None
def check_bundle(bundle_dir: Path, zip_path: Path | None = None) -> dict[str, Any]:
findings: list[dict[str, Any]] = []
file_checks: list[dict[str, Any]] = []
if not bundle_dir.exists() or not bundle_dir.is_dir():
findings.append(issue("error", "missing_bundle_dir", "Bundle directory is missing.", path=bundle_dir))
return build_result(bundle_dir, zip_path, findings, file_checks, None)
present = all_files(bundle_dir)
for required in sorted(REQUIRED_FILES):
if required not in present:
findings.append(issue("error", "missing_bundle_file", "Required bundle file is missing.", path=bundle_dir / required))
manifest_path = bundle_dir / "manifest.json"
if not manifest_path.exists():
return build_result(bundle_dir, zip_path, findings, file_checks, None)
manifest = load_json(manifest_path)
if manifest.get("schema") != "onec_patch_bundle.v1":
findings.append(issue("error", "invalid_bundle_schema", "Bundle manifest schema is not onec_patch_bundle.v1.", path=manifest_path, detail={"schema": manifest.get("schema")}))
preflight = load_json(bundle_dir / "preflight.json") if (bundle_dir / "preflight.json").exists() else {}
if preflight.get("status") != "ready_for_review":
findings.append(issue("error", "invalid_preflight_status", "Bundle preflight must be ready_for_review.", path=bundle_dir / "preflight.json", detail={"status": preflight.get("status")}))
if not preflight.get("passed"):
findings.append(issue("error", "preflight_not_passed", "Bundle preflight is not passed.", path=bundle_dir / "preflight.json"))
diff = load_json(bundle_dir / "diff.json") if (bundle_dir / "diff.json").exists() else {}
modified = [
str(item.get("relative_path") or "").replace("\\", "/")
for item in diff.get("files") or []
if item.get("status") == "modified"
]
manifest_relatives = [str(item.get("relative_path") or "").replace("\\", "/") for item in manifest.get("files") or []]
if sorted(modified) != sorted(manifest_relatives):
findings.append(issue("error", "bundle_diff_manifest_mismatch", "Modified diff files do not match bundle manifest files.", detail={"diff_modified": modified, "manifest_files": manifest_relatives}))
expected_bundle_files = set(REQUIRED_FILES)
for record in manifest.get("files") or []:
bundle_path_raw = str(record.get("bundle_path") or "")
try:
bundle_path = safe_relative(bundle_path_raw)
except ValueError:
findings.append(issue("error", "unsafe_bundle_path", "Unsafe bundle_path in manifest.", detail={"bundle_path": bundle_path_raw}))
continue
expected_bundle_files.add(str(bundle_path).replace("\\", "/"))
path = bundle_dir / bundle_path
check: dict[str, Any] = {
"relative_path": record.get("relative_path"),
"bundle_path": str(bundle_path).replace("\\", "/"),
"exists": path.exists(),
"expected_sha256": expected_working_hash(record),
}
if not path.exists():
findings.append(issue("error", "missing_modified_file", "Modified bundle file is missing.", path=path))
else:
actual = sha256_file(path)
check["sha256"] = actual
expected = expected_working_hash(record)
if expected and actual != expected:
findings.append(issue("error", "modified_file_hash_mismatch", "Modified bundle file hash does not match working hash.", path=path, detail={"expected": expected, "actual": actual}))
file_checks.append(check)
extra_files = sorted(present - expected_bundle_files)
for relative in extra_files:
findings.append(issue("warning", "extra_bundle_file", "Unexpected file in bundle directory.", path=bundle_dir / relative))
if zip_path is None:
candidate = bundle_dir.with_suffix(".zip")
zip_path = candidate if candidate.exists() else None
if zip_path is None:
findings.append(issue("warning", "missing_bundle_zip", "Bundle zip archive was not found."))
elif not zip_path.exists():
findings.append(issue("error", "missing_bundle_zip", "Bundle zip archive path does not exist.", path=zip_path))
else:
try:
with zipfile.ZipFile(zip_path, "r") as archive:
bad_member = archive.testzip()
if bad_member:
findings.append(issue("error", "invalid_bundle_zip_member", "Zip archive contains a corrupt member.", path=zip_path, detail={"member": bad_member}))
zip_files = {name.replace("\\", "/") for name in archive.namelist() if not name.endswith("/")}
if zip_files != present:
findings.append(issue("error", "bundle_zip_mismatch", "Zip contents differ from bundle directory files.", path=zip_path, detail={"missing_in_zip": sorted(present - zip_files), "extra_in_zip": sorted(zip_files - present)}))
except zipfile.BadZipFile as exc:
findings.append(issue("error", "invalid_bundle_zip", f"Invalid zip archive: {exc}", path=zip_path))
return build_result(bundle_dir, zip_path, findings, file_checks, manifest)
def build_result(bundle_dir: Path, zip_path: Path | None, findings: list[dict[str, Any]], file_checks: list[dict[str, Any]], manifest: dict[str, Any] | None) -> dict[str, Any]:
errors = [row for row in findings if row.get("severity") == "error"]
warnings = [row for row in findings if row.get("severity") == "warning"]
return {
"schema": "onec_patch_bundle_check.v1",
"bundle_dir": str(bundle_dir),
"zip_path": str(zip_path) if zip_path else None,
"bundle_schema": (manifest or {}).get("schema"),
"passed": not errors,
"findings": findings,
"file_checks": file_checks,
"counts": {
"files": len(file_checks),
"errors": len(errors),
"warnings": len(warnings),
},
}
def main() -> int:
parser = argparse.ArgumentParser(description="Validate a 1C patch review bundle.")
parser.add_argument("--bundle-dir", type=Path, required=True)
parser.add_argument("--zip", type=Path)
parser.add_argument("--output", type=Path)
args = parser.parse_args()
result = check_bundle(args.bundle_dir, args.zip)
if args.output:
write_json(args.output, result)
print(json.dumps({"output": str(args.output) if args.output else None, "passed": result["passed"], "counts": result["counts"]}, ensure_ascii=False))
return 0 if result["passed"] else 2
if __name__ == "__main__":
raise SystemExit(main())
+116
View File
@@ -0,0 +1,116 @@
#!/usr/bin/env python3
"""Run the full read-only preflight for a 1C patch workspace."""
from __future__ import annotations
import argparse
import json
from pathlib import Path
from typing import Any
from check_1c_patch_source_freshness import check_workspace_sources
from check_1c_patch_workspace_integrity import check_workspace
from diff_1c_patch_workspace import build_diff
from validate_1c_patch_workspace_semantics import validate_workspace
def load_json(path: Path) -> dict[str, Any]:
return json.loads(path.read_text(encoding="utf-8-sig"))
def safety_from_workspace(workspace: Path) -> dict[str, Any]:
path = workspace / "safety.json"
if not path.exists():
return {
"schema": "onec_change_proposal_safety_check.v1",
"passed": False,
"findings": [{"severity": "error", "code": "missing_safety_json", "message": f"Missing {path}"}],
"counts": {"errors": 1, "warnings": 0},
}
return load_json(path)
def status_from_checks(safety: dict[str, Any], integrity: dict[str, Any], freshness: dict[str, Any], semantic: dict[str, Any], diff: dict[str, Any]) -> str:
if not safety.get("passed") or not integrity.get("passed") or not freshness.get("passed") or not semantic.get("passed") or not diff.get("passed"):
return "blocked"
modified = (diff.get("counts") or {}).get("modified", 0)
if modified:
return "ready_for_review"
return "ready_for_editing"
def collect_gate(name: str, data: dict[str, Any]) -> dict[str, Any]:
return {
"name": name,
"schema": data.get("schema"),
"passed": data.get("passed"),
"counts": data.get("counts"),
}
def next_actions(status: str) -> list[str]:
if status == "blocked":
return [
"Inspect failed gates and recreate the workspace if source files changed.",
"Do not generate, package, or apply patches until all gates pass.",
]
if status == "ready_for_editing":
return [
"Edit only files under working/.",
"Run preflight again after edits; BSL/Form.xml semantic validation is included.",
]
return [
"Review the workspace diff.",
"Run external BSL/1C validation in a disposable base before packaging or applying.",
]
def build_preflight(workspace: Path, *, max_patch_chars: int) -> dict[str, Any]:
safety = safety_from_workspace(workspace)
integrity = check_workspace(workspace)
freshness = check_workspace_sources(workspace)
semantic = validate_workspace(workspace)
diff = build_diff(workspace, max_patch_chars=max_patch_chars)
status = status_from_checks(safety, integrity, freshness, semantic, diff)
return {
"schema": "onec_patch_preflight.v1",
"workspace": str(workspace),
"status": status,
"passed": status != "blocked",
"gates": [
collect_gate("proposal_safety", safety),
collect_gate("workspace_integrity", integrity),
collect_gate("source_freshness", freshness),
collect_gate("workspace_semantic_validation", semantic),
collect_gate("workspace_diff", diff),
],
"diff_summary": diff.get("counts"),
"next_actions": next_actions(status),
"details": {
"safety": safety,
"integrity": integrity,
"freshness": freshness,
"semantic": semantic,
"diff": diff,
},
}
def main() -> int:
parser = argparse.ArgumentParser(description="Run 1C patch workspace preflight.")
parser.add_argument("--workspace", type=Path, required=True)
parser.add_argument("--max-patch-chars", type=int, default=200000)
parser.add_argument("--output", type=Path)
args = parser.parse_args()
result = build_preflight(args.workspace, max_patch_chars=args.max_patch_chars)
output = json.dumps(result, ensure_ascii=False, indent=2) + "\n"
if args.output:
args.output.parent.mkdir(parents=True, exist_ok=True)
args.output.write_text(output, encoding="utf-8")
print(json.dumps({"output": str(args.output) if args.output else None, "passed": result["passed"], "status": result["status"], "diff": result["diff_summary"]}, ensure_ascii=False))
return 0 if result["passed"] else 2
if __name__ == "__main__":
raise SystemExit(main())
@@ -0,0 +1,98 @@
#!/usr/bin/env python3
"""Check whether source extension files still match a 1C patch workspace manifest."""
from __future__ import annotations
import argparse
import hashlib
import json
from pathlib import Path
from typing import Any
def load_json(path: Path) -> dict[str, Any]:
return json.loads(path.read_text(encoding="utf-8-sig"))
def sha256_file(path: Path) -> str:
digest = hashlib.sha256()
with path.open("rb") as stream:
for chunk in iter(lambda: stream.read(1024 * 1024), b""):
digest.update(chunk)
return digest.hexdigest()
def issue(severity: str, code: str, message: str, *, record: dict[str, Any] | None = None) -> dict[str, Any]:
result = {"severity": severity, "code": code, "message": message}
if record:
result["record"] = record
return result
def check_workspace_sources(workspace: Path) -> dict[str, Any]:
manifest_path = workspace / "manifest.json"
findings = []
file_checks = []
if not manifest_path.exists():
findings.append(issue("error", "missing_manifest", f"Workspace manifest.json is missing: {manifest_path}"))
return result(workspace, findings, file_checks)
manifest = load_json(manifest_path)
for record in manifest.get("files") or []:
source = Path(str(record.get("source_path") or ""))
expected = record.get("sha256")
check = {
"relative_path": record.get("relative_path"),
"source_path": str(source),
"expected_sha256": expected,
"source_exists": source.exists(),
}
if not source.exists():
findings.append(issue("error", "source_missing", f"Source file is missing: {source}", record=record))
file_checks.append(check)
continue
actual = sha256_file(source)
check["source_sha256"] = actual
check["fresh"] = actual == expected
if expected and actual != expected:
findings.append(issue("error", "source_hash_mismatch", "Source file changed since patch workspace creation.", record={**record, "current_sha256": actual}))
file_checks.append(check)
return result(workspace, findings, file_checks, manifest=manifest)
def result(workspace: Path, findings: list[dict[str, Any]], file_checks: list[dict[str, Any]], manifest: dict[str, Any] | None = None) -> dict[str, Any]:
errors = [row for row in findings if row.get("severity") == "error"]
warnings = [row for row in findings if row.get("severity") == "warning"]
return {
"schema": "onec_patch_source_freshness.v1",
"workspace": str(workspace),
"manifest_schema": (manifest or {}).get("schema"),
"passed": not errors,
"findings": findings,
"file_checks": file_checks,
"counts": {
"files": len(file_checks),
"errors": len(errors),
"warnings": len(warnings),
"stale": len([row for row in file_checks if row.get("fresh") is False]),
"missing": len([row for row in file_checks if not row.get("source_exists")]),
},
}
def main() -> int:
parser = argparse.ArgumentParser(description="Check 1C patch source freshness.")
parser.add_argument("--workspace", type=Path, required=True)
parser.add_argument("--output", type=Path)
args = parser.parse_args()
check = check_workspace_sources(args.workspace)
output = json.dumps(check, ensure_ascii=False, indent=2) + "\n"
if args.output:
args.output.parent.mkdir(parents=True, exist_ok=True)
args.output.write_text(output, encoding="utf-8")
print(json.dumps({"output": str(args.output) if args.output else None, "passed": check["passed"], "counts": check["counts"]}, ensure_ascii=False))
return 0 if check["passed"] else 2
if __name__ == "__main__":
raise SystemExit(main())
@@ -0,0 +1,145 @@
#!/usr/bin/env python3
"""Check integrity of a 1C patch workspace before diff/package/apply steps."""
from __future__ import annotations
import argparse
import hashlib
import json
from pathlib import Path
from typing import Any
def load_json(path: Path) -> dict[str, Any]:
return json.loads(path.read_text(encoding="utf-8-sig"))
def sha256_file(path: Path) -> str:
digest = hashlib.sha256()
with path.open("rb") as stream:
for chunk in iter(lambda: stream.read(1024 * 1024), b""):
digest.update(chunk)
return digest.hexdigest()
def issue(severity: str, code: str, message: str, *, path: str | None = None, record: dict[str, Any] | None = None) -> dict[str, Any]:
result = {"severity": severity, "code": code, "message": message}
if path:
result["path"] = path
if record:
result["record"] = record
return result
def is_relative_safe(relative_path: str) -> bool:
path = Path(relative_path)
return not path.is_absolute() and ".." not in path.parts
def expected_paths(workspace: Path, relative_path: str) -> tuple[Path, Path]:
rel = Path(relative_path)
return workspace / "original" / rel, workspace / "working" / rel
def all_files(root: Path) -> list[Path]:
if not root.exists():
return []
return sorted(path for path in root.rglob("*") if path.is_file())
def rel_set(root: Path) -> set[str]:
result = set()
for path in all_files(root):
result.add(str(path.relative_to(root)).replace("\\", "/"))
return result
def check_workspace(workspace: Path) -> dict[str, Any]:
manifest_path = workspace / "manifest.json"
findings = []
if not manifest_path.exists():
findings.append(issue("error", "missing_manifest", "Workspace manifest.json is missing.", path=str(manifest_path)))
return result(workspace, findings, [])
manifest = load_json(manifest_path)
records = manifest.get("files") or []
manifest_relatives = set()
file_checks = []
for record in records:
relative = str(record.get("relative_path") or "")
manifest_relatives.add(relative)
if not relative or not is_relative_safe(relative):
findings.append(issue("error", "unsafe_relative_path", f"Unsafe relative path in manifest: {relative}", record=record))
continue
original, working = expected_paths(workspace, relative)
check = {
"relative_path": relative,
"original_path": str(original),
"working_path": str(working),
"expected_sha256": record.get("sha256"),
"original_exists": original.exists(),
"working_exists": working.exists(),
}
if not original.exists():
findings.append(issue("error", "missing_original_file", "Original file is missing.", path=str(original), record=record))
else:
actual = sha256_file(original)
check["original_sha256"] = actual
if record.get("sha256") and actual != record.get("sha256"):
findings.append(issue("error", "original_hash_mismatch", "Original file hash differs from manifest; original/ must stay immutable.", path=str(original), record=record))
if not working.exists():
findings.append(issue("error", "missing_working_file", "Working file is missing.", path=str(working), record=record))
else:
check["working_sha256"] = sha256_file(working)
file_checks.append(check)
original_extra = sorted(rel_set(workspace / "original") - manifest_relatives)
working_extra = sorted(rel_set(workspace / "working") - manifest_relatives)
for relative in original_extra:
findings.append(issue("error", "extra_original_file", "Unexpected file under original/.", path=str(workspace / "original" / Path(relative))))
for relative in working_extra:
findings.append(issue("warning", "extra_working_file", "Unexpected file under working/; future packaging must explicitly include or reject it.", path=str(workspace / "working" / Path(relative))))
if not (workspace / "proposal.json").exists():
findings.append(issue("warning", "missing_proposal_copy", "proposal.json is missing from workspace.", path=str(workspace / "proposal.json")))
if not (workspace / "safety.json").exists():
findings.append(issue("warning", "missing_safety_copy", "safety.json is missing from workspace.", path=str(workspace / "safety.json")))
return result(workspace, findings, file_checks, manifest=manifest)
def result(workspace: Path, findings: list[dict[str, Any]], file_checks: list[dict[str, Any]], manifest: dict[str, Any] | None = None) -> dict[str, Any]:
errors = [row for row in findings if row.get("severity") == "error"]
warnings = [row for row in findings if row.get("severity") == "warning"]
return {
"schema": "onec_patch_workspace_integrity.v1",
"workspace": str(workspace),
"manifest_schema": (manifest or {}).get("schema"),
"passed": not errors,
"findings": findings,
"file_checks": file_checks,
"counts": {
"files": len(file_checks),
"errors": len(errors),
"warnings": len(warnings),
},
}
def main() -> int:
parser = argparse.ArgumentParser(description="Check 1C patch workspace integrity.")
parser.add_argument("--workspace", type=Path, required=True)
parser.add_argument("--output", type=Path)
args = parser.parse_args()
check = check_workspace(args.workspace)
output = json.dumps(check, ensure_ascii=False, indent=2) + "\n"
if args.output:
args.output.parent.mkdir(parents=True, exist_ok=True)
args.output.write_text(output, encoding="utf-8")
print(json.dumps({"output": str(args.output) if args.output else None, "passed": check["passed"], "counts": check["counts"]}, ensure_ascii=False))
return 0 if check["passed"] else 2
if __name__ == "__main__":
raise SystemExit(main())
+272
View File
@@ -0,0 +1,272 @@
from __future__ import annotations
import argparse
import json
import subprocess
import sys
import tempfile
from pathlib import Path
import yaml
ROOT = Path(__file__).resolve().parents[1]
PLUGIN = ROOT / "plugins" / "1c"
DEFAULT_REPORT = ROOT / "reports" / "1c-plugin-health.json"
def run(command: list[str], *, allow_fail: bool = False) -> dict:
result = subprocess.run(
command,
cwd=ROOT,
text=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
check=False,
)
status = "ok" if result.returncode == 0 else "failed"
if allow_fail and result.returncode != 0:
status = "blocked"
return {
"command": command,
"status": status,
"returncode": result.returncode,
"stdout": result.stdout.strip(),
"stderr": result.stderr.strip(),
}
def prepare_rag_smoke_index(temp_path: Path) -> list[dict]:
rag_source = temp_path / "metadata.health.generated.md"
corpus = temp_path / "rag_corpus.jsonl"
manifest = temp_path / "rag_manifest.json"
index = temp_path / "rag_index.json"
steps = [
[
sys.executable,
"scripts/convert_1c_metadata_to_rag.py",
"--input",
"plugins/1c/metadata/examples/metadata.example.json",
"--output",
str(rag_source),
],
[
sys.executable,
"scripts/convert_1c_bsl_modules_to_rag.py",
"--input",
"plugins/1c/metadata/examples/bsl-modules.example.json",
"--output",
str(temp_path / "bsl.health.generated.md"),
],
[
sys.executable,
"scripts/validate_1c_rag_sources.py",
"--source-dir",
str(temp_path),
],
[
sys.executable,
"scripts/prepare_1c_rag_corpus.py",
"--source-dir",
str(temp_path),
"--output",
str(corpus),
"--manifest",
str(manifest),
],
[
sys.executable,
"scripts/build_1c_rag_index.py",
"--corpus",
str(corpus),
"--output",
str(index),
],
]
return [run(step, allow_fail=True) for step in steps]
def read_yaml(path: Path) -> dict:
with path.open("r", encoding="utf-8") as handle:
data = yaml.safe_load(handle)
if not isinstance(data, dict):
raise ValueError(f"{path} must contain a YAML mapping")
return data
def check_required_files() -> dict:
required = [
PLUGIN / "plugin.yaml",
PLUGIN / "prompts" / "system.md",
PLUGIN / "prompts" / "rag-answer.md",
PLUGIN / "rag" / "profiles.yaml",
PLUGIN / "rag" / "quality-smoke.json",
PLUGIN / "rag" / "profile-routing-smoke.json",
PLUGIN / "tools" / "tool-contract.yaml",
PLUGIN / "connector" / "contracts" / "openapi.yaml",
PLUGIN / "connector" / "policies" / "read-only-query.yaml",
PLUGIN / "connector" / "policies" / "change-workflow.yaml",
PLUGIN / "connector" / "policies" / "config-layer-write-policy.yaml",
PLUGIN / "connector" / "Dockerfile",
PLUGIN / "connector" / "docker-compose.yml",
PLUGIN / "connector" / ".env.example",
PLUGIN / "connector" / "pyproject.toml",
PLUGIN / "connector" / "service.yaml",
PLUGIN / "metadata" / "schema.json",
PLUGIN / "metadata" / "moxel-schema-registry.json",
PLUGIN / "metadata" / "examples" / "metadata.example.json",
PLUGIN / "metadata" / "examples" / "metadata-v2.example.json",
PLUGIN / "metadata" / "examples" / "bsl-modules.example.json",
PLUGIN / "schemas" / "metadata-snapshot-v2.schema.json",
PLUGIN / "schemas" / "bsl-module-snapshot.schema.json",
PLUGIN / "schemas" / "moxel-schema-registry.schema.json",
PLUGIN / "training" / "examples" / "instruction.examples.jsonl",
PLUGIN / "training" / "configs" / "qwen3-coder-30b-a3b-lora.yaml",
PLUGIN / "evals" / "smoke.yaml",
]
missing = [str(path.relative_to(ROOT)) for path in required if not path.exists()]
return {
"status": "ok" if not missing else "failed",
"missing": missing,
}
def summarize_manifest() -> dict:
manifest = read_yaml(PLUGIN / "plugin.yaml")
return {
"id": manifest.get("id"),
"version": manifest.get("version"),
"status": manifest.get("status"),
"tasks": manifest.get("tasks") or [],
"entrypoints": sorted((manifest.get("entrypoints") or {}).keys()),
}
def main() -> int:
parser = argparse.ArgumentParser(description="Run 1C plugin health checks.")
parser.add_argument("--report", type=Path, default=DEFAULT_REPORT)
parser.add_argument("--print", action="store_true")
parser.add_argument(
"--no-report",
action="store_true",
help="Do not write a health report file; useful for clean local checks.",
)
args = parser.parse_args()
with tempfile.TemporaryDirectory(prefix="llm-1c-health-") as temp_dir:
temp_index = Path(temp_dir) / "rag_index.json"
commands = {
"metadata_example": [
sys.executable,
"scripts/validate_1c_metadata_snapshot.py",
"plugins/1c/metadata/examples/metadata.example.json",
],
"metadata_v2_example": [
sys.executable,
"scripts/validate_1c_metadata_snapshot.py",
"plugins/1c/metadata/examples/metadata-v2.example.json",
],
"bsl_modules_example": [
sys.executable,
"scripts/validate_1c_bsl_modules.py",
"plugins/1c/metadata/examples/bsl-modules.example.json",
],
"readonly_query_allowed": [
sys.executable,
"scripts/validate_1c_readonly_query.py",
"--query",
"ВЫБРАТЬ Первые 10 Ссылка ИЗ Справочник.Номенклатура",
],
"training_examples": [
sys.executable,
"scripts/validate_1c_training_data.py",
"plugins/1c/training/examples/instruction.examples.jsonl",
],
"evals": [sys.executable, "scripts/validate_evals.py", "plugins/1c/evals/smoke.yaml"],
"connector_standalone_check": [sys.executable, "scripts/check_1c_connector_standalone.py"],
"write_plan_contract": [sys.executable, "scripts/check_1c_write_plan_contract.py"],
"bsl_symbol_check": [sys.executable, "scripts/check_1c_bsl_symbol_resolver.py"],
"code_symbol_contract": [sys.executable, "scripts/check_1c_code_symbol_contract.py"],
"module_origin_contract": [sys.executable, "scripts/check_1c_module_origin_contract.py"],
"extension_action_contract": [sys.executable, "scripts/check_1c_extension_action_contract.py"],
"moxel_schema_registry_contract": [
sys.executable,
"scripts/check_1c_moxel_schema_registry.py",
"--registry",
"plugins/1c/metadata/moxel-schema-registry.json",
"--output",
str(Path(temp_dir) / "moxel-schema-registry-check.json"),
],
"moxel_status": [
sys.executable,
"scripts/status_1c_moxel.py",
"--output-json",
str(Path(temp_dir) / "moxel-status.json"),
"--output-markdown",
str(Path(temp_dir) / "moxel-status.md"),
],
"rag_prompt_guardrails": [
sys.executable,
"scripts/check_1c_rag_prompt.py",
"--index",
str(temp_index),
],
"rag_quality": [
sys.executable,
"scripts/check_1c_rag_quality.py",
"--index",
str(temp_index),
],
"rag_profile_routing": [
sys.executable,
"scripts/check_1c_rag_profiles.py",
],
"training_preflight": [sys.executable, "scripts/preflight_1c_training.py"],
}
checks = {
"manifest": summarize_manifest(),
"required_files": check_required_files(),
"rag_smoke_prepare": prepare_rag_smoke_index(Path(temp_dir)),
"commands": {},
}
for name, command in commands.items():
checks["commands"][name] = run(
command,
allow_fail=name in {"training_preflight"},
)
failed = []
for name, result in checks["commands"].items():
if result["status"] == "failed":
failed.append(name)
if checks["required_files"]["status"] == "failed":
failed.append("required_files")
report = {
"plugin": "1c",
"status": "failed" if failed else "ok",
"failed_checks": failed,
"checks": checks,
}
if not args.no_report:
args.report.parent.mkdir(parents=True, exist_ok=True)
args.report.write_text(
json.dumps(report, ensure_ascii=False, indent=2) + "\n",
encoding="utf-8",
)
if args.print:
print(json.dumps(report, ensure_ascii=False, indent=2))
else:
if not args.no_report:
print(f"Wrote 1C plugin health report to {args.report}")
print(f"Status: {report['status']}")
return 1 if failed else 0
if __name__ == "__main__":
raise SystemExit(main())
+128
View File
@@ -0,0 +1,128 @@
from __future__ import annotations
import argparse
import json
import sys
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "scripts"))
from route_1c_question import route_question # noqa: E402
DEFAULT_INDEX = ROOT / "reports" / "1c-sql" / "upo" / "unified-object-route-index.json"
CASES = [
{
"id": "docs_only_form_open",
"question": "Как работает событие ПриОткрытии формы?",
"expected_route": "docs_rag",
"expected_fact_paths": [],
},
{
"id": "current_fact_extension_attribute",
"question": "Есть ли реквизит ДатаСоздания у документа ПриходнаяНакладная?",
"expected_route": "current_config_fact",
"expected_fact_paths": ["Документ.ПриходнаяНакладная.ДатаСоздания"],
"expected_exists": {"Документ.ПриходнаяНакладная.ДатаСоздания": True},
},
{
"id": "rag_example_requires_current_fact_check",
"question": "В примере RAG есть реквизит Артикул у справочника Номенклатура. Напиши код для текущей базы.",
"expected_route": "mixed_docs_and_current_config",
"expected_fact_paths": ["Справочник.Номенклатура.Артикул"],
"expected_risk": "example_is_not_current_fact",
},
]
def fact_exists_by_path(route: dict) -> dict[str, bool | None]:
result = {}
for row in route.get("fact_checks") or []:
path = row.get("path")
if not path:
continue
if row.get("status") != "checked":
result[path] = None
else:
result[path] = bool((row.get("result") or {}).get("exists"))
return result
def run_case(case: dict, *, index: Path, view: str) -> dict:
route = route_question(case["question"], index_path=index, view=view)
failures = []
decision = route.get("decision") or {}
if decision.get("route") != case.get("expected_route"):
failures.append({"code": "route_mismatch", "expected": case.get("expected_route"), "actual": decision.get("route")})
actual_paths = route.get("fact_paths") or []
expected_paths = case.get("expected_fact_paths") or []
if actual_paths != expected_paths:
failures.append({"code": "fact_paths_mismatch", "expected": expected_paths, "actual": actual_paths})
expected_risk = case.get("expected_risk")
if expected_risk:
risks = {row.get("code") for row in route.get("source_risks") or []}
if expected_risk not in risks:
failures.append({"code": "risk_missing", "expected": expected_risk, "actual": sorted(risks)})
exists = fact_exists_by_path(route)
for path, expected in (case.get("expected_exists") or {}).items():
if exists.get(path) is not expected:
failures.append({"code": "fact_exists_mismatch", "path": path, "expected": expected, "actual": exists.get(path)})
return {
"id": case["id"],
"status": "passed" if not failures else "failed",
"question": case["question"],
"failures": failures,
"route": {
"decision": route.get("decision"),
"source_risks": route.get("source_risks"),
"fact_paths": route.get("fact_paths"),
"fact_exists": exists,
},
}
def run_check(index: Path, *, view: str) -> dict:
if not index.exists():
return {
"schema": "onec_question_router_check.v1",
"status": "failed",
"error": f"route index not found: {index}",
"cases": [],
}
results = [run_case(case, index=index, view=view) for case in CASES]
return {
"schema": "onec_question_router_check.v1",
"status": "ok" if all(row["status"] == "passed" for row in results) else "failed",
"index": str(index),
"view": view,
"case_count": len(results),
"cases": results,
}
def main() -> int:
parser = argparse.ArgumentParser(description="Check 1C question router behavior.")
parser.add_argument("--index", type=Path, default=DEFAULT_INDEX)
parser.add_argument("--view", choices=["effective", "base"], default="effective")
parser.add_argument("--output", type=Path)
parser.add_argument("--print", action="store_true")
args = parser.parse_args()
report = run_check(args.index, view=args.view)
if args.output:
args.output.parent.mkdir(parents=True, exist_ok=True)
args.output.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
if args.print or not args.output:
print(json.dumps(report, ensure_ascii=False, indent=2))
return 0 if report["status"] == "ok" else 1
if __name__ == "__main__":
raise SystemExit(main())
+113
View File
@@ -0,0 +1,113 @@
from __future__ import annotations
import argparse
import hashlib
import json
from pathlib import Path
from common import read_json
from prepare_1c_rag_corpus import SUPPORTED_EXTENSIONS, classify_source, normalize_text, parse_front_matter
ROOT = Path(__file__).resolve().parents[1]
DEFAULT_SOURCE_DIR = ROOT / "plugins" / "1c" / "rag" / "sources"
DEFAULT_MANIFEST = ROOT / "plugins" / "1c" / "datasets" / "prepared" / "rag_manifest.json"
def iter_source_files(source_dir: Path) -> list[Path]:
if not source_dir.exists():
return []
return sorted(
path
for path in source_dir.rglob("*")
if path.is_file() and path.suffix.lower() in SUPPORTED_EXTENSIONS
)
def source_state(source_dir: Path) -> dict[str, dict]:
state = {}
for path in iter_source_files(source_dir):
relative_path = path.relative_to(source_dir).as_posix()
text = normalize_text(path.read_text(encoding="utf-8"))
front_matter, body = parse_front_matter(text)
chunk_source_text = body or text
state[relative_path] = {
"source_path": relative_path,
"source_type": classify_source(path, chunk_source_text, front_matter),
"file_type": path.suffix.lower().lstrip("."),
"content_hash": hashlib.sha256(text.encode("utf-8")).hexdigest(),
}
return state
def compare_manifest(source_dir: Path, manifest_path: Path) -> dict:
current = source_state(source_dir)
if not manifest_path.exists():
return {
"status": "stale" if current else "missing",
"reason": "manifest is missing",
"new": sorted(current),
"changed": [],
"deleted": [],
"type_changed": [],
}
manifest = read_json(manifest_path)
recorded = {
str(source.get("source_path")): source
for source in manifest.get("sources") or []
if isinstance(source, dict) and source.get("source_path")
}
current_paths = set(current)
recorded_paths = set(recorded)
new = sorted(current_paths - recorded_paths)
deleted = sorted(recorded_paths - current_paths)
changed = []
type_changed = []
for source_path in sorted(current_paths & recorded_paths):
current_source = current[source_path]
recorded_source = recorded[source_path]
if current_source["content_hash"] != recorded_source.get("content_hash"):
changed.append(source_path)
if current_source["source_type"] != recorded_source.get("source_type"):
type_changed.append(
{
"source_path": source_path,
"current": current_source["source_type"],
"manifest": recorded_source.get("source_type"),
}
)
stale = bool(new or deleted or changed or type_changed)
return {
"status": "stale" if stale else "fresh",
"source_dir": str(source_dir),
"manifest": str(manifest_path),
"source_count": len(current),
"manifest_source_count": len(recorded),
"new": new,
"changed": changed,
"deleted": deleted,
"type_changed": type_changed,
}
def main() -> int:
parser = argparse.ArgumentParser(description="Check whether the 1C RAG manifest is fresh.")
parser.add_argument("--source-dir", type=Path, default=DEFAULT_SOURCE_DIR)
parser.add_argument("--manifest", type=Path, default=DEFAULT_MANIFEST)
parser.add_argument("--print", action="store_true")
args = parser.parse_args()
report = compare_manifest(args.source_dir, args.manifest)
if args.print:
print(json.dumps(report, ensure_ascii=False, indent=2))
else:
print(f"1C RAG freshness: {report['status']}")
return 0 if report["status"] in {"fresh", "missing"} else 1
if __name__ == "__main__":
raise SystemExit(main())
+63
View File
@@ -0,0 +1,63 @@
from __future__ import annotations
import argparse
import json
import sys
from pathlib import Path
from common import read_json
from rag_profiles import detect_rag_profile
ROOT = Path(__file__).resolve().parents[1]
DEFAULT_CASES = ROOT / "plugins" / "1c" / "rag" / "profile-routing-smoke.json"
def main() -> int:
parser = argparse.ArgumentParser(description="Check 1C RAG auto profile routing.")
parser.add_argument("--cases", type=Path, default=DEFAULT_CASES)
parser.add_argument("--print", action="store_true")
args = parser.parse_args()
data = read_json(args.cases)
cases = data.get("cases")
if not isinstance(cases, list):
print(f"{args.cases}: cases must be a list", file=sys.stderr)
return 1
results = []
for case in cases:
query = str(case.get("query") or "")
expected = str(case.get("expected_profile") or "")
actual = detect_rag_profile(query)
results.append(
{
"id": case.get("id"),
"query": query,
"expected_profile": expected,
"actual_profile": actual,
"status": "passed" if actual == expected else "failed",
}
)
report = {
"status": "ok" if all(result["status"] == "passed" for result in results) else "failed",
"case_count": len(results),
"results": results,
}
if args.print:
print(json.dumps(report, ensure_ascii=False, indent=2))
else:
print(f"1C RAG profile routing status: {report['status']}")
if report["status"] != "ok":
for result in results:
if result["status"] != "passed":
print(
f"- {result['id']}: expected {result['expected_profile']}, got {result['actual_profile']}",
file=sys.stderr,
)
return 0 if report["status"] == "ok" else 1
if __name__ == "__main__":
raise SystemExit(main())
+57
View File
@@ -0,0 +1,57 @@
from __future__ import annotations
import argparse
import subprocess
import sys
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
DEFAULT_INDEX = ROOT / "plugins" / "1c" / "datasets" / "prepared" / "rag_index.json"
REQUIRED_PHRASES = [
"Не выдумывай метаданные 1С",
"запроси метаданные через инструмент",
"источники",
"Какие реквизиты есть у справочника Номенклатура?",
]
def main() -> int:
parser = argparse.ArgumentParser(description="Check that the 1C RAG prompt contains safety-critical instructions.")
parser.add_argument("--index", type=Path, default=DEFAULT_INDEX)
args = parser.parse_args()
command = [
sys.executable,
"scripts/ask_1c_rag.py",
"Какие реквизиты есть у справочника Номенклатура?",
"--index",
str(args.index),
"--print-prompt",
]
result = subprocess.run(
command,
cwd=ROOT,
check=False,
text=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
if result.returncode != 0:
print(result.stderr, file=sys.stderr)
return result.returncode
missing = [phrase for phrase in REQUIRED_PHRASES if phrase not in result.stdout]
if missing:
print("1C RAG prompt check failed. Missing phrases:", file=sys.stderr)
for phrase in missing:
print(f"- {phrase}", file=sys.stderr)
return 1
print("1C RAG prompt check passed.")
return 0
if __name__ == "__main__":
raise SystemExit(main())
+87
View File
@@ -0,0 +1,87 @@
from __future__ import annotations
import argparse
import json
import sys
from pathlib import Path
from common import read_json, search_lexical_index
from rag_profiles import resolve_rag_profile
ROOT = Path(__file__).resolve().parents[1]
DEFAULT_INDEX = ROOT / "plugins" / "1c" / "datasets" / "prepared" / "rag_index.json"
DEFAULT_CASES = ROOT / "plugins" / "1c" / "rag" / "quality-smoke.json"
def load_cases(path: Path) -> list[dict]:
data = read_json(path)
cases = data.get("cases")
if not isinstance(cases, list):
raise ValueError(f"{path} must contain a cases list")
return cases
def case_text(results: list[dict]) -> str:
return "\n".join((result["document"].get("content") or "") for result in results).lower()
def run_case(index: dict, case: dict, limit: int) -> dict:
profile = resolve_rag_profile(case.get("profile") or "auto", str(case["query"]))
results = search_lexical_index(
index,
str(case["query"]),
limit=limit or int(profile["limit"]),
candidate_limit=int(profile["candidate_limit"]),
dedupe_by_document=bool(profile["dedupe_by_document"]),
min_score=float(profile["min_score"]),
source_types=profile["source_types"],
)
text = case_text(results)
missing = [word for word in case.get("must_contain") or [] if str(word).lower() not in text]
return {
"id": case.get("id"),
"profile": profile["id"],
"query": case.get("query"),
"status": "passed" if results and not missing else "failed",
"missing": missing,
"top_sources": [
{
"source_path": result["document"].get("source_path"),
"title": result["document"].get("title"),
"chunk_index": result["document"].get("chunk_index"),
"score": round(float(result.get("score") or 0), 4),
}
for result in results
],
}
def main() -> int:
parser = argparse.ArgumentParser(description="Run smoke quality checks for the 1C RAG index.")
parser.add_argument("--index", type=Path, default=DEFAULT_INDEX)
parser.add_argument("--cases", type=Path, default=DEFAULT_CASES)
parser.add_argument("--limit", type=int, default=5)
parser.add_argument("--print", action="store_true")
args = parser.parse_args()
index = read_json(args.index)
results = [run_case(index, case, args.limit) for case in load_cases(args.cases)]
report = {
"status": "ok" if all(result["status"] == "passed" for result in results) else "failed",
"case_count": len(results),
"results": results,
}
if args.print:
print(json.dumps(report, ensure_ascii=False, indent=2))
else:
print(f"1C RAG quality status: {report['status']}")
if report["status"] != "ok":
for result in results:
if result["status"] != "passed":
print(f"- {result['id']}: missing {result['missing']}", file=sys.stderr)
return 0 if report["status"] == "ok" else 1
if __name__ == "__main__":
raise SystemExit(main())
+84
View File
@@ -0,0 +1,84 @@
from __future__ import annotations
import argparse
import json
import sys
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "scripts"))
from management_console_server import query_rag # noqa: E402
def run_checks() -> dict:
official = query_rag(
{
"question": "форма при открытии пример артикул",
"source_type": "official_1c_docs",
"limit": 8,
}
)
official_bad = [
row
for row in official.get("results") or []
if row.get("source_type") in {"metadata", "bsl_modules", "examples"}
or "example" in str(row.get("source_path") or "").casefold()
]
metadata = query_rag(
{
"question": "какие реквизиты у справочника Номенклатура",
"source_type": "metadata",
"limit": 5,
}
)
metadata_answer = str(metadata.get("answer") or "").casefold()
checks = [
{
"id": "official_docs_exclude_examples",
"status": "passed" if not official_bad else "failed",
"details": official_bad,
},
{
"id": "metadata_scope_warns",
"status": "passed" if "примеры" in metadata_answer and "не подтверждают текущую базу" in metadata_answer else "failed",
"answer": metadata.get("answer"),
},
]
return {
"schema": "onec_rag_source_governance_check.v1",
"status": "ok" if all(item["status"] == "passed" for item in checks) else "failed",
"checks": checks,
"samples": {
"official_1c_docs": {
"source_scope": official.get("source_scope"),
"result_count": official.get("result_count"),
"source_types": sorted({row.get("source_type") for row in official.get("results") or [] if row.get("source_type")}),
},
"metadata": {
"source_scope": metadata.get("source_scope"),
"result_count": metadata.get("result_count"),
},
},
}
def main() -> int:
parser = argparse.ArgumentParser(description="Check 1C RAG source governance.")
parser.add_argument("--output", type=Path)
parser.add_argument("--print", action="store_true")
args = parser.parse_args()
report = run_checks()
if args.output:
args.output.parent.mkdir(parents=True, exist_ok=True)
args.output.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
if args.print or not args.output:
print(json.dumps(report, ensure_ascii=False, indent=2))
return 0 if report["status"] == "ok" else 1
if __name__ == "__main__":
raise SystemExit(main())
+59
View File
@@ -0,0 +1,59 @@
from __future__ import annotations
import argparse
import json
import sqlite3
from pathlib import Path
from search_1c_rag_vector import DEFAULT_CORPUS, DEFAULT_INDEX, freshness, load_meta
def check_vector_freshness(index_path: Path, corpus_path: Path) -> dict:
if not index_path.exists():
return {
"status": "missing",
"reason": "vector index is missing",
"index": str(index_path),
"corpus": str(corpus_path),
}
try:
conn = sqlite3.connect(index_path)
try:
meta = load_meta(conn)
finally:
conn.close()
except sqlite3.Error as exc:
return {
"status": "invalid",
"reason": str(exc),
"index": str(index_path),
"corpus": str(corpus_path),
}
report = freshness(meta, corpus_path)
return {
**report,
"index": str(index_path),
"embedding_model": meta.get("embedding_model"),
"embedding_dimensions": meta.get("embedding_dimensions"),
"doc_count": meta.get("doc_count"),
"built_at": meta.get("built_at"),
}
def main() -> int:
parser = argparse.ArgumentParser(description="Check whether the 1C RAG vector index is fresh.")
parser.add_argument("--index", type=Path, default=DEFAULT_INDEX)
parser.add_argument("--corpus", type=Path, default=DEFAULT_CORPUS)
parser.add_argument("--print", action="store_true")
args = parser.parse_args()
report = check_vector_freshness(args.index, args.corpus)
if args.print:
print(json.dumps(report, ensure_ascii=False, indent=2))
else:
print(f"1C RAG vector freshness: {report['status']}")
return 0 if report["status"] in {"fresh", "missing"} else 1
if __name__ == "__main__":
raise SystemExit(main())
@@ -0,0 +1,138 @@
#!/usr/bin/env python3
"""Validate a latest 1C saved-state watch run lookup."""
from __future__ import annotations
import argparse
import json
from pathlib import Path
from typing import Any
EXPECTED_SCHEMA = "onec_saved_state_latest_watch_run.v1"
def load_json(path: Path) -> dict[str, Any]:
return json.loads(path.read_text(encoding="utf-8-sig"))
def write_json(path: Path, data: dict[str, Any]) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(json.dumps(data, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
def issue(severity: str, code: str, message: str, *, path: Path | str | None = None, detail: dict[str, Any] | None = None) -> dict[str, Any]:
result: dict[str, Any] = {"severity": severity, "code": code, "message": message}
if path is not None:
result["path"] = str(path)
if detail:
result["detail"] = detail
return result
def linked_path(latest_path: Path, value: Any) -> Path | None:
if not value:
return None
path = Path(str(value))
if path.is_absolute():
return path
return (latest_path.parent / path).resolve()
def check_linked_check(latest_path: Path, run: dict[str, Any], key: str, passed_key: str, findings: list[dict[str, Any]]) -> None:
path = linked_path(latest_path, run.get(key))
if path is None:
return
if not path.exists():
findings.append(issue("error", f"missing_{key}", f"Linked {key} file is missing.", path=path))
return
linked = load_json(path)
if run.get(passed_key) != linked.get("passed"):
findings.append(issue("error", f"{passed_key}_mismatch", f"{passed_key} differs from linked check.", path=path))
def has_delta_changes(run: dict[str, Any]) -> bool:
counts = run.get("counts") or {}
return any(int(counts.get(key) or 0) > 0 for key in ("delta_objects_added", "delta_objects_removed", "delta_objects_changed"))
def check_latest(latest_path: Path) -> dict[str, Any]:
findings: list[dict[str, Any]] = []
if not latest_path.exists():
findings.append(issue("error", "missing_latest", "Latest watch lookup file is missing.", path=latest_path))
return build_result(latest_path, findings)
data = load_json(latest_path)
if data.get("schema") != EXPECTED_SCHEMA:
findings.append(issue("error", "invalid_schema", "Latest watch lookup schema is invalid.", detail={"schema": data.get("schema")}))
safety = data.get("safety") or {}
if safety.get("read_only") is not True:
findings.append(issue("error", "not_read_only", "Latest watch lookup safety.read_only must be true."))
if safety.get("sql_write_performed") is not False:
findings.append(issue("error", "sql_write_flag", "Latest watch lookup safety.sql_write_performed must be false."))
markdown = linked_path(latest_path, data.get("markdown"))
if data.get("markdown") and (markdown is None or not markdown.exists()):
findings.append(issue("error", "missing_markdown", "Linked latest Markdown file is missing.", path=markdown or "<null>"))
found = data.get("found")
latest = data.get("latest")
if found is True:
if not isinstance(latest, dict):
findings.append(issue("error", "missing_latest_run", "found=true requires latest object."))
return build_result(latest_path, findings)
run_dir = linked_path(latest_path, latest.get("run_dir"))
if run_dir is None or not run_dir.exists():
findings.append(issue("error", "missing_run_dir", "Latest run directory is missing.", path=run_dir or "<null>"))
for key in ("report", "manifest_check"):
path = linked_path(latest_path, latest.get(key))
if path is not None and not path.exists():
findings.append(issue("error", f"missing_{key}", f"Linked {key} is missing.", path=path))
check_linked_check(latest_path, latest, "manifest_check", "manifest_check_passed", findings)
check_linked_check(latest_path, latest, "delta_check", "delta_check_passed", findings)
if data.get("require_delta") is True and not latest.get("delta"):
findings.append(issue("error", "required_delta_missing", "require_delta=true but latest run has no delta."))
if data.get("require_changed") is True and not has_delta_changes(latest):
findings.append(issue("error", "required_changed_missing", "require_changed=true but latest run has no delta changes."))
elif found is False:
if latest is not None:
findings.append(issue("error", "unexpected_latest", "found=false requires latest=null."))
else:
findings.append(issue("error", "invalid_found", "found must be boolean."))
counts = data.get("counts")
if not isinstance(counts, dict):
findings.append(issue("error", "missing_counts", "Latest lookup must include counts."))
return build_result(latest_path, findings)
def build_result(latest_path: Path, findings: list[dict[str, Any]]) -> dict[str, Any]:
errors = [row for row in findings if row.get("severity") == "error"]
warnings = [row for row in findings if row.get("severity") == "warning"]
return {
"schema": "onec_saved_state_latest_watch_run_check.v1",
"latest": str(latest_path),
"passed": not errors,
"findings": findings,
"counts": {"errors": len(errors), "warnings": len(warnings)},
"safety": {"read_only": True, "sql_write_performed": False},
}
def main() -> int:
parser = argparse.ArgumentParser(description="Validate latest 1C saved-state watch run lookup.")
parser.add_argument("--latest", type=Path, required=True)
parser.add_argument("--output", type=Path)
args = parser.parse_args()
result = check_latest(args.latest)
if args.output:
write_json(args.output, result)
print(json.dumps({"output": str(args.output) if args.output else None, "passed": result["passed"], "counts": result["counts"]}, ensure_ascii=False))
return 0 if result["passed"] else 2
if __name__ == "__main__":
raise SystemExit(main())
@@ -0,0 +1,238 @@
#!/usr/bin/env python3
"""Validate a 1C saved-state object report contract."""
from __future__ import annotations
import argparse
import json
from pathlib import Path
from typing import Any
EXPECTED_SCHEMA = "onec_saved_state_object_report.v1"
EXPECTED_COMPARISON_SCHEMA = "onec_saved_state_object_comparison.v1"
EXPECTED_DETAIL_SCHEMA = "onec_saved_state_object_detail.v1"
KNOWN_PAYLOAD_ROLES = {
"bsl_module_text",
"form_descriptor",
"form_body",
"primary_payload",
"metadata_payload",
}
def load_json(path: Path) -> dict[str, Any]:
return json.loads(path.read_text(encoding="utf-8-sig"))
def write_json(path: Path, data: dict[str, Any]) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(json.dumps(data, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
def issue(severity: str, code: str, message: str, *, path: Path | str | None = None, detail: dict[str, Any] | None = None) -> dict[str, Any]:
result: dict[str, Any] = {"severity": severity, "code": code, "message": message}
if path is not None:
result["path"] = str(path)
if detail:
result["detail"] = detail
return result
def linked_path(report_path: Path, value: Any) -> Path | None:
if not value:
return None
path = Path(str(value))
if path.is_absolute():
return path
return (report_path.parent / path).resolve()
def is_list(value: Any) -> bool:
return isinstance(value, list)
def check_agent_summary(report: dict[str, Any], findings: list[dict[str, Any]]) -> None:
summary = report.get("agent_summary")
if not isinstance(summary, dict):
findings.append(issue("error", "missing_agent_summary", "Report must include agent_summary."))
return
object_changes = summary.get("object_changes")
if not isinstance(object_changes, list):
findings.append(issue("error", "invalid_agent_summary_object_changes", "agent_summary.object_changes must be an array."))
object_changes = []
object_names = summary.get("object_names")
if not isinstance(object_names, list):
findings.append(issue("error", "invalid_agent_summary_object_names", "agent_summary.object_names must be an array."))
object_names = []
names_from_objects = [item.get("full_name") for item in object_changes if isinstance(item, dict)]
if names_from_objects != object_names:
findings.append(issue(
"error",
"agent_summary_names_mismatch",
"agent_summary.object_names must match object_changes full_name order.",
detail={"object_names": object_names, "from_objects": names_from_objects},
))
for index, item in enumerate(object_changes):
if not isinstance(item, dict):
findings.append(issue("error", "invalid_agent_summary_object", "agent_summary object item must be an object.", detail={"index": index}))
continue
full_name = item.get("full_name")
if not full_name:
findings.append(issue("error", "missing_agent_summary_full_name", "agent_summary object item is missing full_name.", detail={"index": index}))
for key in ("added_terms", "removed_terms", "parts"):
if not is_list(item.get(key)):
findings.append(issue("error", f"invalid_{key}", f"agent_summary object field {key} must be an array.", detail={"object": full_name, "type": type(item.get(key)).__name__}))
parts = item.get("parts") if isinstance(item.get("parts"), list) else []
if item.get("parts_count") != len(parts):
findings.append(issue("error", "agent_summary_parts_count_mismatch", "parts_count must match parts length.", detail={"object": full_name, "parts_count": item.get("parts_count"), "actual": len(parts)}))
text_diff_parts = 0
active_missing_parts = 0
for part in parts:
if not isinstance(part, dict):
continue
role = part.get("payload_role")
if role not in KNOWN_PAYLOAD_ROLES:
findings.append(issue("error", "unknown_payload_role", "Unknown payload_role in agent_summary part.", detail={"object": full_name, "role": role, "file_name": part.get("file_name")}))
if part.get("summary") in {"Text payload differs.", "Text payload matches."}:
text_diff_parts += 1
if part.get("active_exists") is False:
active_missing_parts += 1
if item.get("text_diff_parts") != text_diff_parts:
findings.append(issue("warning", "agent_summary_text_diff_count_mismatch", "text_diff_parts differs from counted comparable parts.", detail={"object": full_name, "reported": item.get("text_diff_parts"), "counted": text_diff_parts}))
if item.get("active_missing_parts") != active_missing_parts:
findings.append(issue("error", "agent_summary_active_missing_count_mismatch", "active_missing_parts must match parts with active_exists=false.", detail={"object": full_name, "reported": item.get("active_missing_parts"), "counted": active_missing_parts}))
system_changes = summary.get("system_changes")
if not isinstance(system_changes, list):
findings.append(issue("error", "invalid_agent_summary_system_changes", "agent_summary.system_changes must be an array."))
system_changes = []
system_names = summary.get("system_change_names")
if not isinstance(system_names, list):
findings.append(issue("error", "invalid_agent_summary_system_names", "agent_summary.system_change_names must be an array."))
system_names = []
names_from_system = [item.get("name") for item in system_changes if isinstance(item, dict)]
if names_from_system != system_names:
findings.append(issue("error", "agent_summary_system_names_mismatch", "system_change_names must match system_changes name order.", detail={"system_change_names": system_names, "from_system": names_from_system}))
def check_detail(detail: dict[str, Any], findings: list[dict[str, Any]]) -> None:
if detail.get("schema") != EXPECTED_DETAIL_SCHEMA:
findings.append(issue("error", "invalid_detail_schema", "Detail schema is invalid.", detail={"schema": detail.get("schema")}))
for obj in detail.get("object_details") or []:
if not isinstance(obj, dict):
continue
full_name = obj.get("full_name")
for part in obj.get("details") or []:
if not isinstance(part, dict):
continue
role = part.get("payload_role")
if role not in KNOWN_PAYLOAD_ROLES:
findings.append(issue("error", "detail_unknown_payload_role", "Unknown payload_role in detail part.", detail={"object": full_name, "role": role, "file_name": part.get("file_name")}))
payload = part.get("payload") or {}
hints = payload.get("semantic_hints")
if hints is not None:
for key in ("added_terms", "removed_terms"):
if not isinstance(hints.get(key), list):
findings.append(issue("error", "invalid_semantic_hints", f"semantic_hints.{key} must be an array.", detail={"object": full_name, "file_name": part.get("file_name")}))
def check_report(report_path: Path) -> dict[str, Any]:
findings: list[dict[str, Any]] = []
if not report_path.exists():
findings.append(issue("error", "missing_report", "Saved-state report file is missing.", path=report_path))
return build_result(report_path, findings, None, None)
report = load_json(report_path)
if report.get("schema") != EXPECTED_SCHEMA:
findings.append(issue("error", "invalid_report_schema", "Report schema is invalid.", path=report_path, detail={"schema": report.get("schema")}))
safety = report.get("safety") or {}
if safety.get("read_only") is not True:
findings.append(issue("error", "report_not_read_only", "Report safety.read_only must be true."))
if safety.get("sql_write_performed") is not False:
findings.append(issue("error", "report_sql_write_flag", "Report safety.sql_write_performed must be false."))
if safety.get("public_terms_are_1c_objects") is not True:
findings.append(issue("error", "report_public_terms_flag", "Report must expose public terms as 1C objects."))
if safety.get("secrets_in_report") is not False:
findings.append(issue("error", "report_secrets_flag", "Report safety.secrets_in_report must be false."))
comparison_path = linked_path(report_path, report.get("comparison"))
detail_path = linked_path(report_path, report.get("detail"))
markdown_path = linked_path(report_path, report.get("markdown"))
comparison: dict[str, Any] | None = None
detail: dict[str, Any] | None = None
if comparison_path is None or not comparison_path.exists():
findings.append(issue("error", "missing_comparison", "Linked comparison JSON is missing.", path=comparison_path or "<null>"))
else:
comparison = load_json(comparison_path)
if comparison.get("schema") != EXPECTED_COMPARISON_SCHEMA:
findings.append(issue("error", "invalid_comparison_schema", "Comparison schema is invalid.", path=comparison_path, detail={"schema": comparison.get("schema")}))
if detail_path is None or not detail_path.exists():
findings.append(issue("error", "missing_detail", "Linked detail JSON is missing.", path=detail_path or "<null>"))
else:
detail = load_json(detail_path)
check_detail(detail, findings)
if report.get("markdown") is not None and (markdown_path is None or not markdown_path.exists()):
findings.append(issue("error", "missing_markdown", "Linked Markdown report is missing.", path=markdown_path or "<null>"))
counts = report.get("counts") or {}
if comparison:
comparison_counts = comparison.get("counts") or {}
if counts.get("object_changes") != comparison_counts.get("object_changes"):
findings.append(issue("error", "object_change_count_mismatch", "Report object_changes count differs from comparison."))
if counts.get("system_changes") != comparison_counts.get("system_changes"):
findings.append(issue("error", "system_change_count_mismatch", "Report system_changes count differs from comparison."))
if detail:
detail_counts = detail.get("counts") or {}
if counts.get("detail_objects") != detail_counts.get("objects"):
findings.append(issue("error", "detail_object_count_mismatch", "Report detail_objects count differs from detail."))
if counts.get("detail_parts") != detail_counts.get("details"):
findings.append(issue("error", "detail_part_count_mismatch", "Report detail_parts count differs from detail."))
check_agent_summary(report, findings)
return build_result(report_path, findings, comparison, detail)
def build_result(report_path: Path, findings: list[dict[str, Any]], comparison: dict[str, Any] | None, detail: dict[str, Any] | None) -> dict[str, Any]:
errors = [row for row in findings if row.get("severity") == "error"]
warnings = [row for row in findings if row.get("severity") == "warning"]
return {
"schema": "onec_saved_state_object_report_check.v1",
"report": str(report_path),
"comparison_schema": (comparison or {}).get("schema"),
"detail_schema": (detail or {}).get("schema"),
"passed": not errors,
"findings": findings,
"counts": {
"errors": len(errors),
"warnings": len(warnings),
},
"safety": {
"read_only": True,
"sql_write_performed": False,
},
}
def main() -> int:
parser = argparse.ArgumentParser(description="Validate a 1C saved-state object report.")
parser.add_argument("--report", type=Path, required=True)
parser.add_argument("--output", type=Path)
args = parser.parse_args()
result = check_report(args.report)
if args.output:
write_json(args.output, result)
print(json.dumps({"output": str(args.output) if args.output else None, "passed": result["passed"], "counts": result["counts"]}, ensure_ascii=False))
return 0 if result["passed"] else 2
if __name__ == "__main__":
raise SystemExit(main())
@@ -0,0 +1,217 @@
#!/usr/bin/env python3
"""Validate a 1C saved-state report delta contract."""
from __future__ import annotations
import argparse
import json
from pathlib import Path
from typing import Any
EXPECTED_SCHEMA = "onec_saved_state_object_report_delta.v1"
KNOWN_PAYLOAD_ROLES = {
"bsl_module_text",
"form_descriptor",
"form_body",
"primary_payload",
"metadata_payload",
}
def load_json(path: Path) -> dict[str, Any]:
return json.loads(path.read_text(encoding="utf-8-sig"))
def write_json(path: Path, data: dict[str, Any]) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(json.dumps(data, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
def issue(severity: str, code: str, message: str, *, path: Path | str | None = None, detail: dict[str, Any] | None = None) -> dict[str, Any]:
result: dict[str, Any] = {"severity": severity, "code": code, "message": message}
if path is not None:
result["path"] = str(path)
if detail:
result["detail"] = detail
return result
def linked_path(delta_path: Path, value: Any) -> Path | None:
if not value:
return None
path = Path(str(value))
if path.is_absolute():
return path
return (delta_path.parent / path).resolve()
def check_payload_roles(parts: list[Any], findings: list[dict[str, Any]], *, context: str) -> None:
for part in parts:
if not isinstance(part, dict):
findings.append(issue("error", "invalid_part", "Payload part must be an object.", detail={"context": context}))
continue
role = part.get("payload_role")
if role not in KNOWN_PAYLOAD_ROLES:
findings.append(issue("error", "unknown_payload_role", "Unknown payload_role.", detail={"context": context, "role": role, "file_name": part.get("file_name")}))
def check_compact_object(item: dict[str, Any], findings: list[dict[str, Any]], *, context: str) -> None:
if not item.get("full_name"):
findings.append(issue("error", "missing_full_name", "Object delta item is missing full_name.", detail={"context": context}))
for key in ("added_terms", "removed_terms", "parts"):
if not isinstance(item.get(key), list):
findings.append(issue("error", f"invalid_{key}", f"Object field {key} must be an array.", detail={"context": context, "full_name": item.get("full_name")}))
parts = item.get("parts") if isinstance(item.get("parts"), list) else []
if item.get("parts_count") != len(parts):
findings.append(issue("error", "parts_count_mismatch", "parts_count must match parts length.", detail={"context": context, "full_name": item.get("full_name"), "parts_count": item.get("parts_count"), "actual": len(parts)}))
check_payload_roles(parts, findings, context=context)
def check_changed_object(item: dict[str, Any], findings: list[dict[str, Any]]) -> None:
full_name = item.get("full_name")
before = item.get("before")
after = item.get("after")
if not isinstance(before, dict) or not isinstance(after, dict):
findings.append(issue("error", "invalid_changed_object_shape", "Changed object must include before and after objects.", detail={"full_name": full_name}))
return
check_compact_object(before, findings, context=f"changed.before:{full_name}")
check_compact_object(after, findings, context=f"changed.after:{full_name}")
for key in ("before_fingerprint", "after_fingerprint"):
if not item.get(key):
findings.append(issue("error", f"missing_{key}", f"Changed object is missing {key}.", detail={"full_name": full_name}))
if item.get("before_fingerprint") == item.get("after_fingerprint"):
findings.append(issue("error", "unchanged_fingerprint_in_changed", "Changed object has equal before and after fingerprints.", detail={"full_name": full_name}))
term_delta = item.get("term_delta")
if not isinstance(term_delta, dict):
findings.append(issue("error", "invalid_term_delta", "Changed object term_delta must be an object.", detail={"full_name": full_name}))
else:
for list_name in ("added_terms", "removed_terms"):
delta = term_delta.get(list_name)
if not isinstance(delta, dict) or not isinstance(delta.get("added"), list) or not isinstance(delta.get("removed"), list):
findings.append(issue("error", "invalid_term_delta_list", "Term delta must contain added and removed arrays.", detail={"full_name": full_name, "list": list_name}))
part_delta = item.get("part_delta")
if not isinstance(part_delta, dict):
findings.append(issue("error", "invalid_part_delta", "Changed object part_delta must be an object.", detail={"full_name": full_name}))
else:
for key in ("added", "removed", "changed"):
if not isinstance(part_delta.get(key), list):
findings.append(issue("error", f"invalid_part_delta_{key}", f"part_delta.{key} must be an array.", detail={"full_name": full_name}))
check_payload_roles(part_delta.get("added") or [], findings, context=f"part_delta.added:{full_name}")
check_payload_roles(part_delta.get("removed") or [], findings, context=f"part_delta.removed:{full_name}")
for part in part_delta.get("changed") or []:
if not isinstance(part, dict):
findings.append(issue("error", "invalid_changed_part", "part_delta.changed item must be an object.", detail={"full_name": full_name}))
continue
before_part = part.get("before")
after_part = part.get("after")
if not isinstance(before_part, dict) or not isinstance(after_part, dict):
findings.append(issue("error", "invalid_changed_part_shape", "Changed part must include before and after.", detail={"full_name": full_name, "file_name": part.get("file_name")}))
continue
check_payload_roles([before_part, after_part], findings, context=f"part_delta.changed:{full_name}")
def check_delta(delta_path: Path) -> dict[str, Any]:
findings: list[dict[str, Any]] = []
if not delta_path.exists():
findings.append(issue("error", "missing_delta", "Delta report file is missing.", path=delta_path))
return build_result(delta_path, findings)
data = load_json(delta_path)
if data.get("schema") != EXPECTED_SCHEMA:
findings.append(issue("error", "invalid_delta_schema", "Delta schema is invalid.", path=delta_path, detail={"schema": data.get("schema")}))
safety = data.get("safety") or {}
if safety.get("read_only") is not True:
findings.append(issue("error", "delta_not_read_only", "Delta safety.read_only must be true."))
if safety.get("sql_write_performed") is not False:
findings.append(issue("error", "delta_sql_write_flag", "Delta safety.sql_write_performed must be false."))
if safety.get("public_terms_are_1c_objects") is not True:
findings.append(issue("error", "delta_public_terms_flag", "Delta must expose public terms as 1C objects."))
for key in ("before_report", "after_report"):
path = linked_path(delta_path, data.get(key))
if path is None or not path.exists():
findings.append(issue("warning", f"missing_{key}", f"Linked {key} is missing.", path=path or "<null>"))
markdown_path = linked_path(delta_path, data.get("markdown"))
if data.get("markdown") is not None and (markdown_path is None or not markdown_path.exists()):
findings.append(issue("error", "missing_markdown", "Linked Markdown delta report is missing.", path=markdown_path or "<null>"))
objects = data.get("objects")
if not isinstance(objects, dict):
findings.append(issue("error", "invalid_objects", "Delta objects must be an object."))
objects = {}
for key in ("added", "removed", "changed", "unchanged"):
if not isinstance(objects.get(key), list):
findings.append(issue("error", f"invalid_objects_{key}", f"objects.{key} must be an array."))
for item in objects.get("added") or []:
if isinstance(item, dict):
check_compact_object(item, findings, context="objects.added")
for item in objects.get("removed") or []:
if isinstance(item, dict):
check_compact_object(item, findings, context="objects.removed")
for item in objects.get("changed") or []:
if isinstance(item, dict):
check_changed_object(item, findings)
system = data.get("system_changes")
if not isinstance(system, dict):
findings.append(issue("error", "invalid_system_changes", "Delta system_changes must be an object."))
system = {}
for key in ("added", "removed", "changed"):
if not isinstance(system.get(key), list):
findings.append(issue("error", f"invalid_system_changes_{key}", f"system_changes.{key} must be an array."))
counts = data.get("counts") or {}
expected_counts = {
"objects_added": len(objects.get("added") or []),
"objects_removed": len(objects.get("removed") or []),
"objects_changed": len(objects.get("changed") or []),
"objects_unchanged": len(objects.get("unchanged") or []),
"system_added": len(system.get("added") or []),
"system_removed": len(system.get("removed") or []),
"system_changed": len(system.get("changed") or []),
}
for key, expected in expected_counts.items():
if counts.get(key) != expected:
findings.append(issue("error", "delta_count_mismatch", "Delta count does not match payload.", detail={"key": key, "reported": counts.get(key), "expected": expected}))
return build_result(delta_path, findings)
def build_result(delta_path: Path, findings: list[dict[str, Any]]) -> dict[str, Any]:
errors = [row for row in findings if row.get("severity") == "error"]
warnings = [row for row in findings if row.get("severity") == "warning"]
return {
"schema": "onec_saved_state_object_report_delta_check.v1",
"delta": str(delta_path),
"passed": not errors,
"findings": findings,
"counts": {
"errors": len(errors),
"warnings": len(warnings),
},
"safety": {
"read_only": True,
"sql_write_performed": False,
},
}
def main() -> int:
parser = argparse.ArgumentParser(description="Validate a 1C saved-state report delta.")
parser.add_argument("--delta", type=Path, required=True)
parser.add_argument("--output", type=Path)
args = parser.parse_args()
result = check_delta(args.delta)
if args.output:
write_json(args.output, result)
print(json.dumps({"output": str(args.output) if args.output else None, "passed": result["passed"], "counts": result["counts"]}, ensure_ascii=False))
return 0 if result["passed"] else 2
if __name__ == "__main__":
raise SystemExit(main())
@@ -0,0 +1,178 @@
#!/usr/bin/env python3
from __future__ import annotations
import argparse
import json
import sys
import urllib.request
from pathlib import Path
from typing import Any
DEFAULT_BASE_URL = "http://docker-gpu.cin.su:8011"
DEFAULT_TABLES = ("ConfigCASSave", "ConfigSave")
ALLOWED_TABLES = {"ConfigCASSave", "ConfigSave"}
def request_json(method: str, url: str, *, payload: dict[str, Any] | None, timeout: float) -> tuple[int, dict[str, Any] | str]:
data = json.dumps(payload, ensure_ascii=False).encode("utf-8") if payload is not None else None
request = urllib.request.Request(
url,
data=data,
method=method,
headers={"Content-Type": "application/json; charset=utf-8"},
)
with urllib.request.urlopen(request, timeout=timeout) as response:
raw = response.read().decode("utf-8")
return response.status, json.loads(raw) if raw else {}
def rpc(base_url: str, method: str, payload: dict[str, Any], timeout: float) -> dict[str, Any]:
status, response = request_json(
"POST",
base_url.rstrip("/") + "/rpc",
payload={"method": method, "payload": payload},
timeout=timeout,
)
if status != 200 or not isinstance(response, dict):
raise RuntimeError(f"{method} failed: status={status}, response={response!r}")
return response
def health_summary(base_url: str, base_id: str, timeout: float) -> dict[str, Any]:
status, response = request_json("GET", f"{base_url.rstrip('/')}/health?base_id={base_id}", payload=None, timeout=timeout)
if status != 200 or not isinstance(response, dict):
return {"status": "error", "http_status": status, "response": response}
live_sql = response.get("live_sql") if isinstance(response.get("live_sql"), dict) else {}
return {
"status": response.get("status"),
"contract_version": response.get("contract_version"),
"live_sql": {
"configured": live_sql.get("configured"),
"server": live_sql.get("server"),
"database": live_sql.get("database"),
},
}
def saved_state_row_counts(base_url: str, base_id: str, tables: list[str], timeout: float) -> dict[str, Any]:
selects = [f"SELECT '{table}' AS TableName, COUNT(*) AS RowsCount FROM {table}" for table in tables]
result = rpc(
base_url,
"query.run",
{
"base_id": base_id,
"diagnostic": True,
"query": "\nUNION ALL\n".join(selects),
"timeout_seconds": int(timeout),
},
timeout,
)
counts: dict[str, int] = {table: 0 for table in tables}
for row in result.get("rows") or []:
if not isinstance(row, dict):
continue
table = str(row.get("TableName") or "")
if table in counts and isinstance(row.get("RowsCount"), int):
counts[table] = int(row["RowsCount"])
return {
"status": result.get("status"),
"validation": result.get("validation"),
"counts": counts,
"raw_counts": result.get("counts"),
}
def search_saved_state(base_url: str, base_id: str, tables: list[str], timeout: float) -> dict[str, Any]:
common = {"base_id": base_id, "tables": tables, "limit": 3, "timeout_seconds": int(timeout)}
forms = rpc(base_url, "metadata.saved_state.forms.search", common, timeout)
modules = rpc(base_url, "metadata.saved_state.modules.search", {**common, "scan_limit": 100}, timeout)
return {
"forms": {
"status": forms.get("status"),
"counts": forms.get("counts"),
"sample": forms.get("forms") or [],
},
"modules": {
"status": modules.get("status"),
"counts": modules.get("counts"),
"sample": modules.get("modules") or [],
},
}
def build_report(base_url: str, base_id: str, tables: list[str], timeout: float, saved_state_table: str | None) -> dict[str, Any]:
report: dict[str, Any] = {
"schema": "onec_saved_state_strict_readiness.v1",
"base_url": base_url,
"base_id": base_id,
"saved_state_table": saved_state_table,
"tables": tables,
"ready": False,
"status": "error",
"checks": {},
"recommendations": [],
}
report["checks"]["health"] = health_summary(base_url, base_id, timeout)
report["checks"]["row_counts"] = saved_state_row_counts(base_url, base_id, tables, timeout)
report["checks"]["saved_state_search"] = search_saved_state(base_url, base_id, tables, timeout)
row_counts = (report["checks"]["row_counts"] or {}).get("counts") or {}
forms_counts = (((report["checks"]["saved_state_search"] or {}).get("forms") or {}).get("counts") or {})
modules_counts = (((report["checks"]["saved_state_search"] or {}).get("modules") or {}).get("counts") or {})
total_rows = sum(value for value in row_counts.values() if isinstance(value, int))
forms = forms_counts.get("forms") if isinstance(forms_counts.get("forms"), int) else 0
modules = modules_counts.get("modules") if isinstance(modules_counts.get("modules"), int) else 0
report["summary"] = {
"saved_state_rows": total_rows,
"forms": forms,
"modules": modules,
}
if forms > 0 and modules > 0:
report["ready"] = True
report["status"] = "ready"
report["recommendations"].append("Strict saved-state smoke can be attempted: form and module saved-state candidates are present in the selected save layer.")
elif total_rows == 0:
report["status"] = "blocked_no_saved_state_rows"
report["recommendations"].append(
f"No unactivated Configurator changes are present in {', '.join(tables)}. To prepare a strict write test, copy the target object from Config/ConfigCAS into the selected save layer using the approved saved-state workflow, then rerun readiness."
)
else:
report["status"] = "blocked_no_strict_candidates"
report["recommendations"].append(
"Saved-state rows exist, but the adapter did not find both form and module candidates needed by strict write-and-rollback smoke."
)
return report
def main() -> int:
parser = argparse.ArgumentParser(description="Read-only readiness check for strict 1C saved-state smoke tests.")
parser.add_argument("--base-url", default=DEFAULT_BASE_URL)
parser.add_argument("--base-id", default="upo_test")
parser.add_argument("--table", action="append", choices=sorted(ALLOWED_TABLES), help="Saved-state table to inspect. Repeatable.")
parser.add_argument("--saved-state-table", choices=sorted(ALLOWED_TABLES), help="Expected saved-state table for the strict smoke target.")
parser.add_argument("--timeout", type=float, default=30.0)
parser.add_argument("--report", type=Path)
parser.add_argument("--require-ready", action="store_true", help="Exit non-zero when strict saved-state smoke is not ready.")
parser.add_argument("--json", action="store_true", help="Print full JSON report.")
args = parser.parse_args()
if args.saved_state_table and args.table and any(table != args.saved_state_table for table in args.table):
parser.error("--saved-state-table must match --table when both are provided")
tables = args.table or ([args.saved_state_table] if args.saved_state_table else list(DEFAULT_TABLES))
report = build_report(args.base_url, args.base_id, tables, args.timeout, args.saved_state_table)
if args.report:
args.report.parent.mkdir(parents=True, exist_ok=True)
args.report.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
if args.json or args.require_ready or not report["ready"]:
print(json.dumps(report, ensure_ascii=True, indent=2), file=sys.stderr if args.require_ready and not report["ready"] else sys.stdout)
else:
print(f"OK: strict saved-state smoke readiness passed for {args.base_id}.")
return 0 if report["ready"] or not args.require_ready else 1
if __name__ == "__main__":
raise SystemExit(main())
+138
View File
@@ -0,0 +1,138 @@
#!/usr/bin/env python3
"""Validate a one-shot 1C saved-state watch manifest."""
from __future__ import annotations
import argparse
import json
from pathlib import Path
from typing import Any
EXPECTED_SCHEMA = "onec_saved_state_watch_once.v1"
def load_json(path: Path) -> dict[str, Any]:
return json.loads(path.read_text(encoding="utf-8-sig"))
def write_json(path: Path, data: dict[str, Any]) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(json.dumps(data, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
def issue(severity: str, code: str, message: str, *, path: Path | str | None = None) -> dict[str, Any]:
result = {"severity": severity, "code": code, "message": message}
if path is not None:
result["path"] = str(path)
return result
def linked_path(manifest_path: Path, value: Any) -> Path | None:
if not value:
return None
path = Path(str(value))
if path.is_absolute():
return path
return (manifest_path.parent / path).resolve()
def check_existing(manifest_path: Path, data: dict[str, Any], key: str, findings: list[dict[str, Any]], *, required: bool = True) -> Path | None:
path = linked_path(manifest_path, data.get(key))
if path is None:
if required:
findings.append(issue("error", f"missing_{key}", f"Manifest field {key} is missing."))
return None
if not path.exists():
findings.append(issue("error" if required else "warning", f"missing_{key}_file", f"Linked {key} file is missing.", path=path))
return path
def check_manifest(manifest_path: Path) -> dict[str, Any]:
findings: list[dict[str, Any]] = []
if not manifest_path.exists():
findings.append(issue("error", "missing_manifest", "Watch manifest is missing.", path=manifest_path))
return build_result(manifest_path, findings)
data = load_json(manifest_path)
if data.get("schema") != EXPECTED_SCHEMA:
findings.append(issue("error", "invalid_schema", "Watch manifest schema is invalid."))
safety = data.get("safety") or {}
if safety.get("read_only") is not True:
findings.append(issue("error", "not_read_only", "Watch manifest safety.read_only must be true."))
if safety.get("sql_write_performed") is not False:
findings.append(issue("error", "sql_write_flag", "Watch manifest safety.sql_write_performed must be false."))
if safety.get("secrets_in_report") is not False:
findings.append(issue("error", "secrets_flag", "Watch manifest safety.secrets_in_report must be false."))
report_path = check_existing(manifest_path, data, "report", findings)
check_existing(manifest_path, data, "markdown", findings, required=False)
report_check_path = check_existing(manifest_path, data, "check", findings)
check_existing(manifest_path, data, "manifest_markdown", findings, required=False)
delta_path = check_existing(manifest_path, data, "delta", findings, required=False)
check_existing(manifest_path, data, "delta_markdown", findings, required=False)
delta_check_path = check_existing(manifest_path, data, "delta_check", findings, required=False)
if report_path and report_path.exists():
report = load_json(report_path)
counts = data.get("counts") or {}
report_counts = report.get("counts") or {}
if counts.get("object_changes") != report_counts.get("object_changes"):
findings.append(issue("error", "object_count_mismatch", "Watch object_changes count differs from report."))
if counts.get("system_changes") != report_counts.get("system_changes"):
findings.append(issue("error", "system_count_mismatch", "Watch system_changes count differs from report."))
if report_check_path and report_check_path.exists():
report_check = load_json(report_check_path)
if report_check.get("passed") is not True:
findings.append(issue("error", "report_check_failed", "Linked saved-state report check did not pass.", path=report_check_path))
if delta_path and delta_path.exists():
delta = load_json(delta_path)
counts = data.get("counts") or {}
delta_counts = delta.get("counts") or {}
mapping = {
"delta_objects_added": "objects_added",
"delta_objects_removed": "objects_removed",
"delta_objects_changed": "objects_changed",
"delta_objects_unchanged": "objects_unchanged",
}
for watch_key, delta_key in mapping.items():
if counts.get(watch_key) != delta_counts.get(delta_key):
findings.append(issue("error", "delta_count_mismatch", f"Watch {watch_key} differs from delta {delta_key}."))
if delta_check_path and delta_check_path.exists():
delta_check = load_json(delta_check_path)
if delta_check.get("passed") is not True:
findings.append(issue("error", "delta_check_failed", "Linked delta check did not pass.", path=delta_check_path))
return build_result(manifest_path, findings)
def build_result(manifest_path: Path, findings: list[dict[str, Any]]) -> dict[str, Any]:
errors = [row for row in findings if row.get("severity") == "error"]
warnings = [row for row in findings if row.get("severity") == "warning"]
return {
"schema": "onec_saved_state_watch_once_check.v1",
"manifest": str(manifest_path),
"passed": not errors,
"findings": findings,
"counts": {"errors": len(errors), "warnings": len(warnings)},
"safety": {"read_only": True, "sql_write_performed": False},
}
def main() -> int:
parser = argparse.ArgumentParser(description="Validate a one-shot 1C saved-state watch manifest.")
parser.add_argument("--manifest", type=Path, required=True)
parser.add_argument("--output", type=Path)
args = parser.parse_args()
result = check_manifest(args.manifest)
if args.output:
write_json(args.output, result)
print(json.dumps({"output": str(args.output) if args.output else None, "passed": result["passed"], "counts": result["counts"]}, ensure_ascii=False))
return 0 if result["passed"] else 2
if __name__ == "__main__":
raise SystemExit(main())
@@ -0,0 +1,154 @@
#!/usr/bin/env python3
"""Validate a 1C saved-state watch run list contract."""
from __future__ import annotations
import argparse
import json
from pathlib import Path
from typing import Any
EXPECTED_SCHEMA = "onec_saved_state_watch_run_list.v1"
def load_json(path: Path) -> dict[str, Any]:
return json.loads(path.read_text(encoding="utf-8-sig"))
def write_json(path: Path, data: dict[str, Any]) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(json.dumps(data, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
def issue(severity: str, code: str, message: str, *, path: Path | str | None = None, detail: dict[str, Any] | None = None) -> dict[str, Any]:
result: dict[str, Any] = {"severity": severity, "code": code, "message": message}
if path is not None:
result["path"] = str(path)
if detail:
result["detail"] = detail
return result
def linked_path(list_path: Path, value: Any) -> Path | None:
if not value:
return None
path = Path(str(value))
if path.is_absolute():
return path
return (list_path.parent / path).resolve()
def has_delta_changes(run: dict[str, Any]) -> bool:
counts = run.get("counts") or {}
return any(int(counts.get(key) or 0) > 0 for key in ("delta_objects_added", "delta_objects_removed", "delta_objects_changed"))
def check_run(list_path: Path, run: dict[str, Any], findings: list[dict[str, Any]], *, index: int) -> None:
if not run.get("run"):
findings.append(issue("error", "missing_run_name", "Run item is missing run name.", detail={"index": index}))
run_dir = linked_path(list_path, run.get("run_dir"))
if run_dir is None or not run_dir.exists():
findings.append(issue("error", "missing_run_dir", "Run directory is missing.", path=run_dir or "<null>", detail={"index": index}))
manifest_check = linked_path(list_path, run.get("manifest_check"))
if manifest_check is not None:
if not manifest_check.exists():
findings.append(issue("error", "missing_manifest_check", "Run manifest_check file is missing.", path=manifest_check))
elif run.get("manifest_check_passed") != load_json(manifest_check).get("passed"):
findings.append(issue("error", "manifest_check_status_mismatch", "Run manifest_check_passed differs from linked check.", path=manifest_check))
delta = linked_path(list_path, run.get("delta"))
if run.get("delta") and (delta is None or not delta.exists()):
findings.append(issue("error", "missing_delta", "Run delta file is missing.", path=delta or "<null>"))
delta_check = linked_path(list_path, run.get("delta_check"))
if delta_check is not None:
if not delta_check.exists():
findings.append(issue("error", "missing_delta_check", "Run delta_check file is missing.", path=delta_check))
elif run.get("delta_check_passed") != load_json(delta_check).get("passed"):
findings.append(issue("error", "delta_check_status_mismatch", "Run delta_check_passed differs from linked check.", path=delta_check))
counts = run.get("counts")
if not isinstance(counts, dict):
findings.append(issue("error", "missing_run_counts", "Run item must include counts.", detail={"run": run.get("run")}))
def check_list(list_path: Path) -> dict[str, Any]:
findings: list[dict[str, Any]] = []
if not list_path.exists():
findings.append(issue("error", "missing_list", "Watch run list file is missing.", path=list_path))
return build_result(list_path, findings)
data = load_json(list_path)
if data.get("schema") != EXPECTED_SCHEMA:
findings.append(issue("error", "invalid_schema", "Watch run list schema is invalid.", detail={"schema": data.get("schema")}))
safety = data.get("safety") or {}
if safety.get("read_only") is not True:
findings.append(issue("error", "not_read_only", "Watch run list safety.read_only must be true."))
if safety.get("sql_write_performed") is not False:
findings.append(issue("error", "sql_write_flag", "Watch run list safety.sql_write_performed must be false."))
markdown = linked_path(list_path, data.get("markdown"))
if data.get("markdown") and (markdown is None or not markdown.exists()):
findings.append(issue("error", "missing_markdown", "Linked Markdown run-list report is missing.", path=markdown or "<null>"))
runs = data.get("runs")
if not isinstance(runs, list):
findings.append(issue("error", "invalid_runs", "runs must be an array."))
runs = []
for index, run in enumerate(runs):
if isinstance(run, dict):
check_run(list_path, run, findings, index=index)
else:
findings.append(issue("error", "invalid_run_item", "Run item must be an object.", detail={"index": index}))
latest = data.get("latest")
if runs:
if not isinstance(latest, dict):
findings.append(issue("error", "missing_latest", "latest must be present when runs are present."))
elif latest.get("run") != runs[0].get("run"):
findings.append(issue("error", "latest_mismatch", "latest must match the first run item.", detail={"latest": latest.get("run"), "first": runs[0].get("run")}))
elif latest is not None:
findings.append(issue("error", "unexpected_latest", "latest must be null when no runs are present."))
counts = data.get("counts") or {}
expected = {
"runs": len(runs),
"with_delta": sum(1 for run in runs if isinstance(run, dict) and run.get("delta")),
"with_delta_changes": sum(1 for run in runs if isinstance(run, dict) and has_delta_changes(run)),
}
for key, value in expected.items():
if counts.get(key) != value:
findings.append(issue("error", "count_mismatch", "Watch run list count mismatch.", detail={"key": key, "reported": counts.get(key), "expected": value}))
return build_result(list_path, findings)
def build_result(list_path: Path, findings: list[dict[str, Any]]) -> dict[str, Any]:
errors = [row for row in findings if row.get("severity") == "error"]
warnings = [row for row in findings if row.get("severity") == "warning"]
return {
"schema": "onec_saved_state_watch_run_list_check.v1",
"list": str(list_path),
"passed": not errors,
"findings": findings,
"counts": {"errors": len(errors), "warnings": len(warnings)},
"safety": {"read_only": True, "sql_write_performed": False},
}
def main() -> int:
parser = argparse.ArgumentParser(description="Validate a 1C saved-state watch run list.")
parser.add_argument("--list", type=Path, required=True)
parser.add_argument("--output", type=Path)
args = parser.parse_args()
result = check_list(args.list)
if args.output:
write_json(args.output, result)
print(json.dumps({"output": str(args.output) if args.output else None, "passed": result["passed"], "counts": result["counts"]}, ensure_ascii=False))
return 0 if result["passed"] else 2
if __name__ == "__main__":
raise SystemExit(main())
File diff suppressed because it is too large Load Diff
+379
View File
@@ -0,0 +1,379 @@
from __future__ import annotations
import argparse
import json
import sys
from pathlib import Path
from typing import Any
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT))
sys.path.insert(0, str(ROOT / "plugins" / "1c"))
sys.path.insert(0, str(ROOT / "plugins" / "1c" / "connector"))
import adapter_1c_server as adapter_server # noqa: E402
def problem_codes(result: dict[str, Any]) -> set[str]:
return {str(problem.get("code") or "") for problem in result.get("problems") or [] if isinstance(problem, dict)}
def require(condition: bool, message: str, failures: list[str]) -> None:
if not condition:
failures.append(message)
def check_blocked_effective_form_path(failures: list[str]) -> None:
result = adapter_server.metadata_write(
{
"base_id": "upo_test",
"target": {"canonical_path": "Документ.Заказ.Форма.ФормаДокумента.КнопкаЗаписать"},
"mode": "plan",
"edits": [{"property": "Заголовок", "value": "Записать"}],
}
)
require(result.get("status") == "blocked", "effective form path must be blocked without concrete route", failures)
require(result.get("error") == "write_plan_required", "effective form path must return write_plan_required", failures)
require((result.get("next_resolution") or {}).get("method") == adapter_server.FORM_WRITE_TARGET_RESOLVE_METHOD, "form path must expose form write target resolver", failures)
hint = result.get("apply_payload_hint") if isinstance(result.get("apply_payload_hint"), dict) else {}
require(hint.get("ready_for_apply_method") is False, "selector-only form hint must not be ready for apply", failures)
def check_module_control_guard(failures: list[str]) -> None:
result = adapter_server.metadata_write(
{
"base_id": "upo_test",
"target": {
"canonical_path": "ОбщийМодуль.Интеграция.Отправить",
"module_ref": "ConfigCASSave:common_module.0#stream:0",
},
"intent": {"operation": "replace_with_control", "new": "Сообщить(\"new\");"},
}
)
require(result.get("status") == "blocked", "replace_with_control without control fragment must be blocked", failures)
require(result.get("error") == "write_plan_blocked", "blocked control guard must return write_plan_blocked", failures)
require("missing_control_fragment" in problem_codes(result), "blocked control guard must expose missing_control_fragment", failures)
allowed = adapter_server.metadata_write_plan(
{
"base_id": "upo_test",
"target": {
"kind": "module",
"module_ref": "ConfigCASSave:common_module.0#stream:0",
},
"intent": {"operation": "replace_with_control", "expected_old_contains": "old", "new": "new", "current_text": "prefix old suffix"},
}
)
require(allowed.get("allowed") is True, "replace_with_control with expected_old_contains must be allowed on concrete route", failures)
drift = adapter_server.metadata_write_plan(
{
"base_id": "upo_test",
"target": {
"kind": "module",
"module_ref": "ConfigCASSave:common_module.0#stream:0",
},
"intent": {"operation": "replace_with_control", "control_fragment": "old", "new": "new", "current_text": "prefix changed suffix"},
}
)
require(drift.get("allowed") is False, "replace_with_control drift must be blocked when current_text is provided", failures)
require("control_fragment_drift" in problem_codes(drift), "replace_with_control drift must expose control_fragment_drift", failures)
def check_reference_identity(failures: list[str]) -> None:
module_plan = adapter_server.metadata_write_plan(
{
"base_id": "upo_test",
"target": {"kind": "module", "file_name": "object-guid__module-guid.0", "stream_index": 4},
"intent": {"operation": "replace", "old": "a", "new": "b"},
}
)
module_hint = ((module_plan.get("route") or {}).get("apply_payload_hint") or {}).get("payload") or {}
require(module_plan.get("allowed") is True, "module file_name concrete route must be allowed", failures)
require((module_plan.get("target") or {}).get("concrete_reference_field") == "file_name", "module file_name route must preserve concrete_reference_field", failures)
require(module_hint.get("file_name") == "object-guid__module-guid.0", "module hint must preserve file_name", failures)
require("module_ref" not in module_hint, "module file_name hint must not invent module_ref", failures)
form_plan = adapter_server.metadata_write_plan(
{
"base_id": "upo_test",
"target": {"kind": "form", "form_guid": "form-guid"},
"intent": {"operation": "property_change", "property": "Заголовок", "value": "Новый"},
}
)
form_hint = ((form_plan.get("route") or {}).get("apply_payload_hint") or {}).get("payload") or {}
require(form_plan.get("allowed") is True, "form_guid concrete route must be allowed", failures)
require((form_plan.get("target") or {}).get("concrete_reference_field") == "form_guid", "form_guid route must preserve concrete_reference_field", failures)
require(form_hint.get("form_guid") == "form-guid", "form hint must preserve form_guid", failures)
require("file_name" not in form_hint, "form_guid hint must not invent file_name", failures)
def check_reference_mismatch(failures: list[str]) -> None:
plan = adapter_server.metadata_write_plan(
{
"base_id": "upo_test",
"target": {"kind": "module", "form_guid": "form-guid"},
"intent": {"operation": "replace", "old": "a", "new": "b"},
}
)
require(plan.get("allowed") is False, "module target with form_guid must be blocked", failures)
require("concrete_reference_kind_mismatch" in problem_codes(plan), "module target with form_guid must expose concrete_reference_kind_mismatch", failures)
def check_provided_origin_evidence(failures: list[str]) -> None:
plan = adapter_server.metadata_write_plan(
{
"base_id": "upo_test",
"target": {
"canonical_path": "ОбщийМодуль.Интеграция.Отправить",
"origin": {"source": "configuration", "presentation": "Конфигурация", "status": "ok"},
},
"intent": {"operation": "replace", "old": "a", "new": "b"},
}
)
recommended = (plan.get("route") or {}).get("recommended_write") or {}
require((plan.get("origin_lookup") or {}).get("method") == "provided_origin_evidence", "planner must accept provided origin evidence", failures)
require(recommended.get("write_surface") == "base_saved_state", "provided configuration origin must recommend base_saved_state", failures)
require("write_route_required" in problem_codes(plan), "provided origin must still require concrete write route", failures)
cas_plan = adapter_server.metadata_write_plan(
{
"base_id": "upo_test",
"target": {
"canonical_path": "ОбщийМодуль.Интеграция.Отправить",
"origin": {"source": "cas_reference", "status": "owner_unresolved", "write_surface": "requires_owner_resolution"},
},
"intent": {"operation": "replace", "old": "a", "new": "b"},
}
)
cas_recommended = (cas_plan.get("route") or {}).get("recommended_write") or {}
require(cas_recommended.get("write_surface") == "blocked_unknown", "unresolved CAS origin must be blocked_unknown", failures)
require("blocked_unknown" in problem_codes(cas_plan), "unresolved CAS origin must expose blocked_unknown", failures)
def check_origin_ambiguity(failures: list[str]) -> None:
original_definition_find = adapter_server.metadata_definition_find
def fake_definition_find(payload: dict[str, Any]) -> dict[str, Any]:
return {
"schema": "onec_metadata_definition_find.v1",
"status": "ok",
"matches": [
{
"area": "object",
"kind": "Document",
"name": "Заказ",
"match_by": "synthetic_contract",
"location": {"presentation": "Документ.Заказ.Реквизит.КнопкаЗаписать"},
"origin": {"source": "configuration", "status": "ok"},
},
{
"area": "form",
"kind": "Document",
"name": "Заказ",
"match_by": "synthetic_contract",
"location": {"presentation": "Документ.Заказ.Форма.ФормаДокумента.КнопкаЗаписать"},
"origin": {"source": "configuration", "status": "ok"},
},
],
"counts": {"matches": 2},
}
adapter_server.metadata_definition_find = fake_definition_find
try:
plan = adapter_server.metadata_write_plan(
{
"base_id": "upo_test",
"target": {"canonical_path": "Документ.Заказ.Форма.ФормаДокумента.КнопкаЗаписать"},
"intent": {"operation": "property_change", "property": "Заголовок", "value": "Записать"},
}
)
finally:
adapter_server.metadata_definition_find = original_definition_find
require(plan.get("allowed") is False, "ambiguous origin plan must not be allowed", failures)
require("ambiguous_origin_matches" in problem_codes(plan), "ambiguous origin plan must expose ambiguous_origin_matches", failures)
ambiguity = next((problem for problem in plan.get("problems") or [] if isinstance(problem, dict) and problem.get("code") == "ambiguous_origin_matches"), {})
require(ambiguity.get("match_count") == 2, "ambiguous origin plan must expose match_count", failures)
def check_extension_action_evidence(failures: list[str]) -> None:
inferred = adapter_server.metadata_write_plan(
{
"base_id": "upo_test",
"target": {
"kind": "module",
"module_ref": "ConfigCASSave:common_module.0#stream:0",
"extension_action": {"status": "ok", "operation_class": "replace_with_control"},
},
"intent": {"control_fragment": "old", "new": "new"},
}
)
require(inferred.get("allowed") is True, "known extension action with guards must be allowed on concrete route", failures)
require((inferred.get("route") or {}).get("operation_class") == "replace_with_control", "planner must infer operation from extension_action", failures)
require((inferred.get("route") or {}).get("operation_inferred_from") == "extension_action", "planner must mark operation_inferred_from", failures)
unknown = adapter_server.metadata_write_plan(
{
"base_id": "upo_test",
"target": {
"kind": "module",
"module_ref": "ConfigCASSave:common_module.0#stream:0",
"extension_action": {"status": "unknown", "operation_class": "unknown_extension_action"},
},
"intent": {"new": "new"},
}
)
require(unknown.get("allowed") is False, "unknown extension action must block module write planning", failures)
require("extension_action_unknown" in problem_codes(unknown), "unknown extension action must expose extension_action_unknown", failures)
mismatch = adapter_server.metadata_write_plan(
{
"base_id": "upo_test",
"target": {
"kind": "module",
"module_ref": "ConfigCASSave:common_module.0#stream:0",
"extension_action": {"status": "ok", "operation_class": "insert_after"},
},
"intent": {"operation": "replace", "old": "old", "new": "new"},
}
)
require(mismatch.get("allowed") is False, "operation mismatch with extension action must be blocked", failures)
require("extension_action_operation_mismatch" in problem_codes(mismatch), "operation mismatch must expose extension_action_operation_mismatch", failures)
ambiguous = adapter_server.metadata_write_plan(
{
"base_id": "upo_test",
"target": {"kind": "module", "module_ref": "ConfigCASSave:common_module.0#stream:0"},
"extension_actions": [
{"status": "ok", "operation_class": "insert_before"},
{"status": "ok", "operation_class": "replace"},
],
"intent": {"operation": "replace", "old": "old", "new": "new"},
}
)
require(ambiguous.get("allowed") is False, "multiple extension actions must block module write planning", failures)
require("extension_action_ambiguous" in problem_codes(ambiguous), "multiple extension actions must expose extension_action_ambiguous", failures)
def check_apply_methods_use_write_plan_gate(failures: list[str]) -> None:
original_changes_propose = adapter_server.changes_propose
original_apply = adapter_server.storage_saved_state_apply_proposal
apply_calls: list[dict[str, Any]] = []
def fake_changes_propose(payload: dict[str, Any]) -> dict[str, Any]:
source = payload.get("source") if isinstance(payload.get("source"), dict) else {}
return {
"schema": "onec_change_proposal.v1",
"status": "accepted_for_review",
"source": {
"table": source.get("table") or "ConfigCASSave",
"file_name": source.get("file_name") or "common_module.0",
},
"original": {"sha1": "0" * 40, "bytes": 10},
"encoded": {"sha1": "1" * 40, "bytes": 11, "payload_hex": "00"},
"validation": {"status": "ok"},
}
def fake_apply(payload: dict[str, Any]) -> dict[str, Any]:
apply_calls.append(payload)
return {"schema": "onec_storage_saved_state_apply.v1", "status": "applied", "applied": True}
adapter_server.changes_propose = fake_changes_propose
adapter_server.storage_saved_state_apply_proposal = fake_apply
try:
blocked = adapter_server.metadata_module_write_apply(
{
"base_id": "upo_test",
"module_ref": "ConfigCASSave:common_module.0#stream:0",
"mode": "apply",
"allow_saved_state_write": True,
"allow_sql_saved_state_apply": True,
"operation": "replace_with_control",
"old": "old",
"new": "Сообщить(\"new\");",
}
)
require(blocked.get("status") == "blocked", "module apply must be blocked when write_plan rejects guards", failures)
require(blocked.get("error") == "write_plan_blocked", "module apply block must report write_plan_blocked", failures)
require("missing_control_fragment" in problem_codes(blocked), "module apply block must expose missing_control_fragment", failures)
require(not apply_calls, "blocked module apply must not call storage_saved_state_apply_proposal", failures)
drift_blocked = adapter_server.metadata_module_write_apply(
{
"base_id": "upo_test",
"module_ref": "ConfigCASSave:common_module.0#stream:0",
"mode": "apply",
"allow_saved_state_write": True,
"allow_sql_saved_state_apply": True,
"operation": "replace_with_control",
"control_fragment": "old",
"old": "old",
"current_text": "changed",
"new": "Сообщить(\"new\");",
}
)
require(drift_blocked.get("status") == "blocked", "module apply must be blocked when write_plan detects control drift", failures)
require(drift_blocked.get("error") == "write_plan_blocked", "module apply drift block must report write_plan_blocked", failures)
require("control_fragment_drift" in problem_codes(drift_blocked), "module apply drift block must expose control_fragment_drift", failures)
require(not apply_calls, "drift-blocked module apply must not call storage_saved_state_apply_proposal", failures)
planned = adapter_server.metadata_module_write_apply(
{
"base_id": "upo_test",
"module_ref": "ConfigCASSave:common_module.0#stream:0",
"mode": "plan",
"allow_saved_state_write": True,
"old": "old",
"new": "new",
}
)
require(planned.get("status") == "planned", "module plan with concrete route must remain planned", failures)
require((planned.get("write_plan") or {}).get("allowed") is True, "module plan must include an allowed write_plan", failures)
finally:
adapter_server.changes_propose = original_changes_propose
adapter_server.storage_saved_state_apply_proposal = original_apply
def run_checks() -> dict[str, Any]:
failures: list[str] = []
check_blocked_effective_form_path(failures)
check_module_control_guard(failures)
check_reference_identity(failures)
check_reference_mismatch(failures)
check_provided_origin_evidence(failures)
check_origin_ambiguity(failures)
check_extension_action_evidence(failures)
check_apply_methods_use_write_plan_gate(failures)
return {
"schema": "onec_write_plan_contract_check.v1",
"status": "ok" if not failures else "failed",
"failures": failures,
"checks": {
"blocked_effective_form_path": True,
"module_control_guard": True,
"reference_identity": True,
"reference_mismatch": True,
"provided_origin_evidence": True,
"origin_ambiguity": True,
"extension_action_evidence": True,
"apply_methods_use_write_plan_gate": True,
},
}
def main() -> int:
parser = argparse.ArgumentParser(description="Check local 1C metadata write-plan contract invariants.")
parser.add_argument("--print", action="store_true")
args = parser.parse_args()
report = run_checks()
if args.print or report["status"] != "ok":
print(json.dumps(report, ensure_ascii=False, indent=2))
else:
print("1C write-plan contract status: ok")
return 0 if report["status"] == "ok" else 1
if __name__ == "__main__":
raise SystemExit(main())
+81
View File
@@ -0,0 +1,81 @@
from __future__ import annotations
import argparse
import subprocess
import sys
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
GENERATED_PATHS = [
ROOT / "plugins" / "1c" / "rag" / "sources" / "metadata.health.generated.md",
]
def run(command: list[str]) -> tuple[str, int]:
label = " ".join(command)
print(f"\n== {label}")
result = subprocess.run(command, cwd=ROOT, text=True, check=False)
return label, result.returncode
def check_no_generated_artifacts() -> tuple[str, int]:
label = "generated artifact check"
print(f"\n== {label}")
leftovers = [path for path in GENERATED_PATHS if path.exists()]
if leftovers:
print("Unexpected generated artifact(s):", file=sys.stderr)
for path in leftovers:
print(f"- {path.relative_to(ROOT)}", file=sys.stderr)
return label, 1
print("No unexpected generated artifacts.")
return label, 0
def main() -> int:
parser = argparse.ArgumentParser(description="Run local repository checks.")
parser.add_argument(
"--with-training-preflight",
action="store_true",
help="Run checks that may report blocked when local CUDA/training dependencies are unavailable.",
)
args = parser.parse_args()
py_files = [str(path.relative_to(ROOT)) for path in sorted((ROOT / "scripts").glob("*.py"))]
commands = [
[sys.executable, "scripts/validate_model_cards.py"],
[sys.executable, "scripts/validate_evals.py"],
[sys.executable, "scripts/validate_gpu_profiles.py"],
[sys.executable, "scripts/check_powershell_scripts.py"],
[sys.executable, "scripts/check_model_storage.py", "--no-report", "--warn-only"],
[sys.executable, "scripts/check_1c_plugin.py", "--no-report"],
[sys.executable, "scripts/check_1c_mcp_adapter_contract.py"],
[sys.executable, "scripts/check_1c_adapter_verification_stack.py"],
[sys.executable, "scripts/smoke_1c_mcp_selector_chain.py", "--no-report"],
[sys.executable, "-m", "py_compile", *py_files],
]
if args.with_training_preflight:
commands.append([sys.executable, "scripts/preflight_1c_training.py"])
failures = []
for command in commands:
label, returncode = run(command)
if returncode != 0:
failures.append(label)
label, returncode = check_no_generated_artifacts()
if returncode != 0:
failures.append(label)
if failures:
print("\nCheck failed:", file=sys.stderr)
for failure in failures:
print(f"- {failure}", file=sys.stderr)
return 1
print("\nAll checks passed.")
return 0
if __name__ == "__main__":
raise SystemExit(main())
+44
View File
@@ -0,0 +1,44 @@
param(
[string]$SshTarget = "docker-gpu.cin.su",
[int]$ConnectTimeoutSeconds = 5
)
$ErrorActionPreference = "Stop"
function Invoke-Remote {
param(
[Parameter(Mandatory = $true)]
[string]$Command
)
ssh `
-o BatchMode=yes `
-o ConnectTimeout=$ConnectTimeoutSeconds `
$SshTarget `
$Command
if ($LASTEXITCODE -ne 0) {
throw "Remote command failed on ${SshTarget} with exit code ${LASTEXITCODE}: ${Command}"
}
}
Write-Host "Checking SSH access to $SshTarget..."
Invoke-Remote "hostname"
Write-Host ""
Write-Host "Checking NVIDIA GPU..."
Invoke-Remote "nvidia-smi --query-gpu=name,memory.total,driver_version --format=csv,noheader"
Write-Host ""
Write-Host "Checking Docker..."
Invoke-Remote "docker version --format '{{.Server.Version}}'"
Write-Host ""
Write-Host "Checking Docker Compose..."
Invoke-Remote "docker compose version"
Write-Host ""
Write-Host "Checking Docker GPU runtime..."
Invoke-Remote "docker run --rm --gpus all nvidia/cuda:12.4.1-base-ubuntu22.04 nvidia-smi --query-gpu=name,memory.total --format=csv,noheader"
Write-Host ""
Write-Host "GPU host preflight completed."
+158
View File
@@ -0,0 +1,158 @@
from __future__ import annotations
import argparse
import datetime as dt
import json
import subprocess
import sys
import time
import urllib.error
import urllib.request
from pathlib import Path
from check_inference_endpoint import check_endpoint
from common import read_json
ROOT = Path(__file__).resolve().parents[1]
DEFAULT_REPORT = ROOT / "reports" / "gpu-readiness.json"
GPU_PROFILES = ROOT / "config" / "gpu_profiles.json"
PROFILE_MODEL_EXPECTATIONS = {
"vLLM": "qwen3-4b-instruct",
"llama.cpp": "devstral-1c-q4",
}
def run_command(command: list[str], timeout: int) -> dict:
try:
result = subprocess.run(
command,
cwd=ROOT,
text=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
timeout=timeout,
check=False,
)
except subprocess.TimeoutExpired as exc:
return {
"command": command,
"status": "failed",
"returncode": None,
"stdout": exc.stdout or "",
"stderr": f"Timed out after {timeout}s",
}
return {
"command": command,
"status": "ok" if result.returncode == 0 else "failed",
"returncode": result.returncode,
"stdout": result.stdout.strip(),
"stderr": result.stderr.strip(),
}
def check_http_url(url: str, timeout: int, *, expected_model: str | None = None) -> dict:
if url.rstrip("/").endswith("/v1/models"):
base_url = url.rstrip("/")[: -len("/v1/models")]
return check_endpoint(base_url, expected_model, timeout)
started_at = time.perf_counter()
result = {
"url": url,
"status": "failed",
"latency_ms": None,
"http_status": None,
"error": None,
}
try:
request = urllib.request.Request(url, method="GET")
with urllib.request.urlopen(request, timeout=timeout) as response:
result["http_status"] = response.status
except (TimeoutError, OSError, urllib.error.URLError) as exc:
result["error"] = str(exc)
return result
result["latency_ms"] = round((time.perf_counter() - started_at) * 1000)
result["status"] = "ok" if 200 <= int(result["http_status"] or 0) < 500 else "failed"
return result
def profile_checks(profile_id: str, timeout: int) -> dict[str, dict]:
profiles = read_json(GPU_PROFILES)
profile = profiles.get(profile_id)
if not isinstance(profile, dict):
available = ", ".join(sorted(str(name) for name in profiles))
raise ValueError(f"Unknown GPU profile `{profile_id}`. Available: {available}")
checks: dict[str, dict] = {}
for index, item in enumerate(profile.get("wait") or [], start=1):
if not isinstance(item, dict):
continue
name = str(item.get("name") or f"wait_{index}")
url = str(item.get("url") or "")
if not url:
continue
key = f"profile_{profile_id}_{index}_{name.lower().replace('.', '').replace(' ', '_')}"
checks[key] = check_http_url(url, timeout, expected_model=PROFILE_MODEL_EXPECTATIONS.get(name))
return checks
def main() -> int:
parser = argparse.ArgumentParser(description="Check GPU host and inference endpoint readiness.")
parser.add_argument("--profile", default="text", help="GPU profile from config/gpu_profiles.json to check.")
parser.add_argument("--all-endpoints", action="store_true", help="Legacy mode: require vLLM and llama.cpp endpoints at the same time.")
parser.add_argument("--vllm-url", default="http://docker-gpu.cin.su:8000")
parser.add_argument("--vllm-model", default="qwen3-4b-instruct")
parser.add_argument("--llama-url", default="http://docker-gpu.cin.su:8080")
parser.add_argument("--llama-model", default="devstral-1c-q4")
parser.add_argument("--ssh-target", default="docker-gpu.cin.su")
parser.add_argument("--timeout", type=int, default=10)
parser.add_argument("--report", type=Path, default=DEFAULT_REPORT)
parser.add_argument("--print", action="store_true")
args = parser.parse_args()
checks: dict[str, dict] = {
"ssh_preflight": run_command(
[
"powershell",
"-NoProfile",
"-ExecutionPolicy",
"Bypass",
"-File",
"scripts/check_gpu_host.ps1",
"-SshTarget",
args.ssh_target,
"-ConnectTimeoutSeconds",
str(args.timeout),
],
timeout=max(args.timeout * 6, 30),
)
}
if args.all_endpoints:
checks["vllm_endpoint"] = check_endpoint(args.vllm_url, args.vllm_model, args.timeout)
checks["llama_endpoint"] = check_endpoint(args.llama_url, args.llama_model, args.timeout)
else:
checks.update(profile_checks(args.profile, args.timeout))
failed = [name for name, check in checks.items() if check.get("status") != "ok"]
report = {
"created_at": dt.datetime.now(dt.UTC).isoformat(),
"profile": args.profile,
"all_endpoints": args.all_endpoints,
"status": "failed" if failed else "ok",
"failed_checks": failed,
"checks": checks,
}
args.report.parent.mkdir(parents=True, exist_ok=True)
args.report.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
if args.print:
print(json.dumps(report, ensure_ascii=False, indent=2))
else:
print(f"GPU readiness status: {report['status']}")
print(f"Wrote report to {args.report}")
return 0 if not failed else 1
if __name__ == "__main__":
raise SystemExit(main())
+84
View File
@@ -0,0 +1,84 @@
from __future__ import annotations
import argparse
import datetime as dt
import json
import sys
import time
import urllib.error
import urllib.request
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
DEFAULT_REPORT = ROOT / "reports" / "inference-endpoint-check.json"
def fetch_models(base_url: str, timeout: int) -> tuple[list[str], int]:
started_at = time.perf_counter()
request = urllib.request.Request(f"{base_url.rstrip('/')}/v1/models", method="GET")
with urllib.request.urlopen(request, timeout=timeout) as response:
payload = json.loads(response.read().decode("utf-8"))
latency_ms = round((time.perf_counter() - started_at) * 1000)
rows = payload.get("data") if isinstance(payload, dict) else []
if not isinstance(rows, list):
return [], latency_ms
return [str(row.get("id")) for row in rows if isinstance(row, dict) and row.get("id")], latency_ms
def check_endpoint(base_url: str, expected_model: str | None, timeout: int) -> dict:
result = {
"base_url": base_url,
"expected_model": expected_model,
"status": "failed",
"models": [],
"available": None,
"latency_ms": None,
"error": None,
}
try:
models, latency_ms = fetch_models(base_url, timeout)
except (TimeoutError, OSError, urllib.error.URLError, ValueError) as exc:
result["error"] = str(exc)
return result
result["models"] = models
result["latency_ms"] = latency_ms
result["available"] = expected_model in models if expected_model else None
result["status"] = "ok" if models and (expected_model is None or result["available"]) else "failed"
return result
def main() -> int:
parser = argparse.ArgumentParser(description="Check an OpenAI-compatible inference endpoint.")
parser.add_argument("--base-url", default="http://docker-gpu.cin.su:8000")
parser.add_argument("--expected-model")
parser.add_argument("--timeout", type=int, default=10)
parser.add_argument("--report", type=Path, default=DEFAULT_REPORT)
parser.add_argument("--print", action="store_true")
parser.add_argument("--no-report", action="store_true")
args = parser.parse_args()
result = check_endpoint(args.base_url, args.expected_model, args.timeout)
report = {
"created_at": dt.datetime.now(dt.UTC).isoformat(),
"status": result["status"],
"checks": [result],
}
if not args.no_report:
args.report.parent.mkdir(parents=True, exist_ok=True)
args.report.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
if args.print:
print(json.dumps(report, ensure_ascii=False, indent=2))
else:
print(f"Endpoint status: {result['status']} ({args.base_url})")
if result["error"]:
print(result["error"], file=sys.stderr)
return 0 if result["status"] == "ok" else 1
if __name__ == "__main__":
raise SystemExit(main())
+142
View File
@@ -0,0 +1,142 @@
from __future__ import annotations
import argparse
import json
from pathlib import Path
from typing import Any
ROOT = Path(__file__).resolve().parents[1]
DEFAULT_MANIFEST = ROOT / "reports" / "llm-artifact-manifest.json"
DEFAULT_OUTPUT = ROOT / "reports" / "llm-artifact-check.json"
def load_json(path: Path) -> dict[str, Any]:
data = json.loads(path.read_text(encoding="utf-8-sig"))
if not isinstance(data, dict):
raise ValueError(f"{path} must contain a JSON object")
return data
def iter_files(path: Path) -> list[Path]:
if not path.exists():
return []
return sorted(item for item in path.rglob("*") if item.is_file())
def resolve_record_path(record_path: str, *, source_root: Path, target_root: Path | None) -> Path:
path = Path(record_path)
if target_root is None:
return path
try:
relative = path.relative_to(source_root)
return target_root / relative
except ValueError:
return target_root / path.name
def check_artifact(record: dict[str, Any], *, source_root: Path, target_root: Path | None, strict_counts: bool) -> list[dict[str, Any]]:
findings: list[dict[str, Any]] = []
path = resolve_record_path(str(record.get("path") or ""), source_root=source_root, target_root=target_root)
expected_exists = bool(record.get("exists"))
if expected_exists and not path.exists():
return [{"severity": "error", "code": "artifact_missing", "artifact": record.get("name"), "path": str(path)}]
if not path.exists():
return []
if not path.is_dir():
findings.append({"severity": "error", "code": "artifact_not_directory", "artifact": record.get("name"), "path": str(path)})
return findings
files = iter_files(path)
size = sum(item.stat().st_size for item in files)
expected_count = int(record.get("file_count") or 0)
expected_size = int(record.get("total_size_bytes") or 0)
if strict_counts and len(files) != expected_count:
findings.append(
{
"severity": "error",
"code": "file_count_mismatch",
"artifact": record.get("name"),
"path": str(path),
"expected": expected_count,
"actual": len(files),
}
)
elif len(files) < expected_count:
findings.append(
{
"severity": "warning",
"code": "file_count_decreased",
"artifact": record.get("name"),
"path": str(path),
"expected_at_least": expected_count,
"actual": len(files),
}
)
if strict_counts and size != expected_size:
findings.append(
{
"severity": "error",
"code": "total_size_mismatch",
"artifact": record.get("name"),
"path": str(path),
"expected": expected_size,
"actual": size,
}
)
elif size < expected_size:
findings.append(
{
"severity": "warning",
"code": "total_size_decreased",
"artifact": record.get("name"),
"path": str(path),
"expected_at_least": expected_size,
"actual": size,
}
)
return findings
def check_manifest(manifest: dict[str, Any], *, target_root: Path | None, strict_counts: bool) -> dict[str, Any]:
source_root = Path(str(manifest.get("workspace_root") or ROOT))
findings: list[dict[str, Any]] = []
for record in manifest.get("artifacts") or []:
findings.extend(check_artifact(record, source_root=source_root, target_root=target_root, strict_counts=strict_counts))
errors = [item for item in findings if item.get("severity") == "error"]
warnings = [item for item in findings if item.get("severity") == "warning"]
return {
"schema": "llm_artifact_manifest_check.v1",
"manifest_schema": manifest.get("schema"),
"manifest_created_at": manifest.get("created_at"),
"target_root": str(target_root) if target_root else None,
"strict_counts": strict_counts,
"passed": not errors,
"counts": {
"artifacts": len(manifest.get("artifacts") or []),
"errors": len(errors),
"warnings": len(warnings),
"findings": len(findings),
},
"findings": findings,
}
def main() -> int:
parser = argparse.ArgumentParser(description="Check that local LLM/RAG artifacts from a manifest are present after transfer or before Docker launch.")
parser.add_argument("--manifest", type=Path, default=DEFAULT_MANIFEST)
parser.add_argument("--output", type=Path, default=DEFAULT_OUTPUT)
parser.add_argument("--target-root", type=Path, help="New workspace root after transfer. If omitted, paths are checked as recorded.")
parser.add_argument("--strict-counts", action="store_true", help="Fail when file counts or total sizes differ exactly.")
args = parser.parse_args()
result = check_manifest(load_json(args.manifest), target_root=args.target_root, strict_counts=args.strict_counts)
args.output.parent.mkdir(parents=True, exist_ok=True)
args.output.write_text(json.dumps(result, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
print(json.dumps({"output": str(args.output), "passed": result["passed"], "counts": result["counts"]}, ensure_ascii=False))
return 0 if result["passed"] else 2
if __name__ == "__main__":
raise SystemExit(main())
+195
View File
@@ -0,0 +1,195 @@
from __future__ import annotations
import argparse
import datetime as dt
import json
from pathlib import Path
from typing import Any
from common import iter_model_card_paths, localize_workspace_path, read_yaml_mapping
ROOT = Path(__file__).resolve().parents[1]
DEFAULT_REPORT = ROOT / "reports" / "model-storage.json"
REQUIRED_MODEL_STATUSES = {"staging", "production"}
def has_any(path: Path, patterns: list[str]) -> bool:
return any(path.glob(pattern) for pattern in patterns)
def has_weight_file(path: Path) -> bool:
for file_path in path.iterdir() if path.exists() else []:
if not file_path.is_file():
continue
name = file_path.name.lower()
if name.endswith((".safetensors", ".bin", ".gguf")):
return True
return False
def check_gguf(path: Path, card: dict[str, Any]) -> dict[str, Any]:
filename = card.get("filename")
if not filename:
return {"status": "failed", "reason": "filename is missing in model card"}
file_path = path / str(filename)
if not file_path.exists():
return {"status": "missing", "reason": f"file is missing: {file_path.relative_to(ROOT)}"}
size = file_path.stat().st_size
expected_size = card.get("file_size_bytes")
if expected_size and size != int(expected_size):
return {
"status": "partial",
"reason": f"size mismatch: {size} != {expected_size}",
"size_bytes": size,
"expected_size_bytes": expected_size,
}
return {"status": "ok", "size_bytes": size, "expected_size_bytes": expected_size}
def check_hf_model(path: Path) -> dict[str, Any]:
required = ["config.json"]
missing = [name for name in required if not (path / name).exists()]
index_path = path / "model.safetensors.index.json"
missing_shards: list[str] = []
if index_path.exists():
try:
index = json.loads(index_path.read_text(encoding="utf-8"))
shard_names = sorted(set((index.get("weight_map") or {}).values()))
missing_shards = [name for name in shard_names if not (path / name).exists()]
except json.JSONDecodeError:
return {"status": "failed", "reason": "model.safetensors.index.json is invalid"}
has_weights = has_weight_file(path)
has_tokenizer = has_any(path, ["tokenizer.json", "tokenizer.model", "vocab.json"])
if missing:
return {"status": "missing", "reason": f"missing required file(s): {', '.join(missing)}"}
if missing_shards:
preview = ", ".join(missing_shards[:4])
suffix = f" and {len(missing_shards) - 4} more" if len(missing_shards) > 4 else ""
return {"status": "partial", "reason": f"missing shard file(s): {preview}{suffix}"}
if not has_weights:
return {"status": "metadata-only", "reason": "model weights are missing"}
if not has_tokenizer:
return {"status": "partial", "reason": "tokenizer files are missing"}
return {"status": "ok"}
def check_diffusers_model(path: Path) -> dict[str, Any]:
missing = [name for name in ["model_index.json"] if not (path / name).exists()]
if missing:
return {"status": "missing", "reason": f"missing required file(s): {', '.join(missing)}"}
if not any(path.rglob("*.safetensors")) and not any(path.rglob("*.bin")):
return {"status": "metadata-only", "reason": "diffusers weights are missing"}
return {"status": "ok"}
def check_adapter(path: Path) -> dict[str, Any]:
if not path.exists():
return {"status": "missing", "reason": f"adapter path is missing: {path}"}
if has_any(path, ["adapter_config.json", "*.safetensors", "*.bin"]):
return {"status": "ok"}
return {"status": "metadata-only", "reason": "adapter artifact files are missing"}
def resolve_storage_path(raw_path: str, models_root: Path | None = None) -> Path:
if models_root and raw_path.startswith("/models/"):
return models_root / raw_path.removeprefix("/models/")
return localize_workspace_path(raw_path)
def check_card(path: Path, *, models_root: Path | None = None) -> dict[str, Any]:
card = read_yaml_mapping(path)
storage_path = resolve_storage_path(str(card.get("storage_path") or ""), models_root=models_root)
model_status = str(card.get("status") or "draft")
item = {
"id": card.get("id"),
"name": card.get("name"),
"type": card.get("type"),
"model_status": model_status,
"format": card.get("format"),
"quantization": card.get("quantization"),
"runtime": (card.get("deployment") or {}).get("runtime"),
"served_model_name": (card.get("deployment") or {}).get("served_model_name"),
"filename": card.get("filename"),
"storage_path": str(storage_path),
"card_path": str(path.relative_to(ROOT)),
"status": "missing",
"reason": None,
"required": model_status in REQUIRED_MODEL_STATUSES,
}
if not storage_path.exists():
item["reason"] = "storage path is missing"
return item
model_format = str(card.get("format") or "").lower()
model_type = str(card.get("type") or "").lower()
if model_format == "gguf":
result = check_gguf(storage_path, card)
elif model_format == "diffusers" or model_type == "image-diffusion-model":
result = check_diffusers_model(storage_path)
elif model_type == "lora-adapter":
result = check_adapter(storage_path)
else:
result = check_hf_model(storage_path)
item.update(result)
return item
def main() -> int:
parser = argparse.ArgumentParser(description="Check local model storage against model cards.")
parser.add_argument("--report", type=Path, default=DEFAULT_REPORT)
parser.add_argument("--print", action="store_true")
parser.add_argument("--no-report", action="store_true")
parser.add_argument("--warn-only", action="store_true", help="Return success even when models are partial or missing.")
parser.add_argument("--strict", action="store_true", help="Fail on incomplete draft/candidate models too.")
parser.add_argument(
"--models-root",
type=Path,
default=None,
help="Override /models paths, for example Z:/LLM/models or /models inside the GPU host container.",
)
args = parser.parse_args()
models_root = args.models_root.resolve() if args.models_root else None
models = [check_card(path, models_root=models_root) for path in iter_model_card_paths()]
incomplete_statuses = {"failed", "missing", "partial"}
incomplete = [model["id"] for model in models if model["status"] in incomplete_statuses]
required_failed = [
model["id"]
for model in models
if model["required"] and model["status"] in incomplete_statuses
]
failed = incomplete if args.strict else required_failed
report = {
"created_at": dt.datetime.now(dt.UTC).isoformat(),
"models_root": str(models_root) if models_root else None,
"status": "failed" if failed else "ok",
"failed": failed,
"required_failed": required_failed,
"planned_incomplete": [
model["id"]
for model in models
if not model["required"] and model["status"] in incomplete_statuses
],
"strict": args.strict,
"counts": {
"ok": sum(1 for model in models if model["status"] == "ok"),
"missing": sum(1 for model in models if model["status"] == "missing"),
"partial": sum(1 for model in models if model["status"] == "partial"),
"metadata_only": sum(1 for model in models if model["status"] == "metadata-only"),
"failed": sum(1 for model in models if model["status"] == "failed"),
},
"models": models,
}
if not args.no_report:
args.report.parent.mkdir(parents=True, exist_ok=True)
args.report.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
if args.print:
print(json.dumps(report, ensure_ascii=False, indent=2))
else:
print(f"Model storage status: {report['status']}")
return 0 if args.warn_only else 1 if failed else 0
if __name__ == "__main__":
raise SystemExit(main())
+420
View File
@@ -0,0 +1,420 @@
from __future__ import annotations
import argparse
import json
import shutil
import subprocess
import sys
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
DEFAULT_PATHS = [ROOT / "scripts", ROOT / "core" / "deploy"]
def iter_powershell_scripts(paths: list[Path]) -> list[Path]:
scripts: list[Path] = []
for path in paths:
if path.is_file() and path.suffix.lower() == ".ps1":
scripts.append(path)
elif path.is_dir():
scripts.extend(path.rglob("*.ps1"))
return sorted(set(scripts))
def powershell_executable() -> str | None:
return shutil.which("pwsh") or shutil.which("powershell")
def ps_single_quoted(value: str) -> str:
return "'" + value.replace("'", "''") + "'"
def check_script(executable: str, path: Path) -> tuple[bool, str]:
path_literal = ps_single_quoted(str(path))
command = [
executable,
"-NoProfile",
"-ExecutionPolicy",
"Bypass",
"-Command",
(
"$ErrorActionPreference = 'Stop'; "
f"$path = {path_literal}; "
"$errors = $null; "
"[System.Management.Automation.PSParser]::Tokenize((Get-Content -Raw -LiteralPath $path), [ref]$errors) | Out-Null; "
"if ($errors) { "
" foreach ($errorItem in $errors) { "
" Write-Error ('{0}: {1} at line {2}, column {3}' -f $path, $errorItem.Message, $errorItem.Token.StartLine, $errorItem.Token.StartColumn); "
" }; "
" exit 1 "
"}"
),
]
result = subprocess.run(command, cwd=ROOT, text=True, capture_output=True, check=False)
output = "\n".join(part for part in [result.stdout.strip(), result.stderr.strip()] if part)
return result.returncode == 0, output
def run_powershell_contract_command(executable: str, args: list[str]) -> subprocess.CompletedProcess[str]:
return subprocess.run(
[executable, "-NoProfile", "-ExecutionPolicy", "Bypass", *args],
cwd=ROOT,
text=True,
capture_output=True,
check=False,
)
def combined_output(result: subprocess.CompletedProcess[str]) -> str:
return "\n".join(part for part in [result.stdout.strip(), result.stderr.strip()] if part)
def check_adapter_base_id_runtime(executable: str) -> list[str]:
failures: list[str] = []
duplicate_verify = run_powershell_contract_command(
executable,
["-File", "scripts/verify_1c_adapter_deployment.ps1", "-BaseId", "upo_test,upo_test", "-SkipRest", "-SkipMcp"],
)
duplicate_verify_output = combined_output(duplicate_verify)
if duplicate_verify.returncode == 0 or "Duplicate BaseId value(s): upo_test" not in duplicate_verify_output:
failures.append(
"verify_1c_adapter_deployment.ps1 must fail duplicate comma-separated -BaseId values at runtime."
)
multi_verify = run_powershell_contract_command(
executable,
["-File", "scripts/verify_1c_adapter_deployment.ps1", "-BaseId", "upo_test,another_test", "-SkipRest", "-SkipMcp"],
)
multi_verify_output = combined_output(multi_verify)
if multi_verify.returncode != 0 or "--base-id upo_test another_test" not in multi_verify_output:
failures.append(
"verify_1c_adapter_deployment.ps1 must expand comma-separated -BaseId values before calling the persisted report validator."
)
duplicate_deploy = run_powershell_contract_command(
executable,
["-File", "scripts/deploy_1c_adapter_stack.ps1", "-BaseId", "upo_test,upo_test", "-SkipRest", "-SkipMcp"],
)
duplicate_deploy_output = combined_output(duplicate_deploy)
if duplicate_deploy.returncode == 0 or "Duplicate BaseId value(s): upo_test" not in duplicate_deploy_output:
failures.append(
"deploy_1c_adapter_stack.ps1 must fail duplicate comma-separated -BaseId values at runtime."
)
multi_deploy = run_powershell_contract_command(
executable,
["-File", "scripts/deploy_1c_adapter_stack.ps1", "-BaseId", "upo_test,another_test", "-SkipRest", "-SkipMcp"],
)
multi_deploy_output = combined_output(multi_deploy)
if multi_deploy.returncode != 0 or "--base-id upo_test another_test" not in multi_deploy_output:
failures.append(
"deploy_1c_adapter_stack.ps1 must preserve multiple normalized -BaseId values when invoking verification."
)
return failures
def check_adapter_verify_wiring(scripts: list[Path], executable: str) -> list[str]:
script_set = set(scripts)
verify_path = ROOT / "scripts" / "verify_1c_adapter_deployment.ps1"
deploy_path = ROOT / "scripts" / "deploy_1c_adapter_stack.ps1"
stack_path = ROOT / "scripts" / "check_1c_adapter_verification_stack.py"
readiness_path = ROOT / "scripts" / "check_1c_saved_state_strict_readiness.py"
prepare_copy_sql_path = ROOT / "scripts" / "prepare_1c_saved_state_copy_sql.py"
prepare_cleanup_sql_path = ROOT / "scripts" / "prepare_1c_saved_state_cleanup_sql.py"
verify_copy_path = ROOT / "scripts" / "verify_1c_saved_state_copy.py"
execute_copy_sql_path = ROOT / "scripts" / "execute_1c_saved_state_copy_sql.ps1"
if verify_path not in script_set and deploy_path not in script_set:
return []
failures: list[str] = []
if not verify_path.exists():
failures.append("scripts/verify_1c_adapter_deployment.ps1 is missing.")
return failures
if not deploy_path.exists():
failures.append("scripts/deploy_1c_adapter_stack.ps1 is missing.")
return failures
if not stack_path.exists():
failures.append("scripts/check_1c_adapter_verification_stack.py is missing.")
return failures
if not readiness_path.exists():
failures.append("scripts/check_1c_saved_state_strict_readiness.py is missing.")
if not prepare_copy_sql_path.exists():
failures.append("scripts/prepare_1c_saved_state_copy_sql.py is missing.")
if not prepare_cleanup_sql_path.exists():
failures.append("scripts/prepare_1c_saved_state_cleanup_sql.py is missing.")
if not verify_copy_path.exists():
failures.append("scripts/verify_1c_saved_state_copy.py is missing.")
return failures
if not execute_copy_sql_path.exists():
failures.append("scripts/execute_1c_saved_state_copy_sql.ps1 is missing.")
return failures
verify_text = verify_path.read_text(encoding="utf-8", errors="replace")
deploy_text = deploy_path.read_text(encoding="utf-8", errors="replace")
stack_text = stack_path.read_text(encoding="utf-8", errors="replace")
readiness_text = readiness_path.read_text(encoding="utf-8", errors="replace")
prepare_copy_sql_text = prepare_copy_sql_path.read_text(encoding="utf-8", errors="replace") if prepare_copy_sql_path.exists() else ""
prepare_cleanup_sql_text = prepare_cleanup_sql_path.read_text(encoding="utf-8", errors="replace") if prepare_cleanup_sql_path.exists() else ""
verify_copy_text = verify_copy_path.read_text(encoding="utf-8", errors="replace") if verify_copy_path.exists() else ""
execute_copy_sql_text = execute_copy_sql_path.read_text(encoding="utf-8", errors="replace") if execute_copy_sql_path.exists() else ""
if "[switch]$RequireSelectorChainWritePlanComposition" not in verify_text:
failures.append("verify_1c_adapter_deployment.ps1 must declare -RequireSelectorChainWritePlanComposition.")
if "[switch]$RequireSavedStateWriteSmoke" not in verify_text:
failures.append("verify_1c_adapter_deployment.ps1 must declare -RequireSavedStateWriteSmoke.")
if "[string]$SavedStateTable" not in verify_text or '[ValidateSet("ConfigSave", "ConfigCASSave")]' not in verify_text:
failures.append("verify_1c_adapter_deployment.ps1 must declare -SavedStateTable with ConfigSave/ConfigCASSave validation.")
if verify_text.count("$SavedStateTable") < 4:
failures.append("verify_1c_adapter_deployment.ps1 must use -SavedStateTable for copy plan and saved-state smoke commands.")
if verify_text.count("--require-write-plan-composition") < 2:
failures.append("verify_1c_adapter_deployment.ps1 must pass --require-write-plan-composition to both REST and MCP selector-chain smoke commands.")
if "--allow-empty-saved-state" not in verify_text or "if (-not $RequireSavedStateWriteSmoke)" not in verify_text:
failures.append("verify_1c_adapter_deployment.ps1 must allow empty saved-state only when -RequireSavedStateWriteSmoke is not set.")
if "function Get-DuplicateValues" not in verify_text or "Duplicate BaseId value(s)" not in verify_text:
failures.append("verify_1c_adapter_deployment.ps1 must reject duplicate -BaseId values before writing reports.")
if "function Normalize-BaseIds" not in verify_text or '-split ","' not in verify_text:
failures.append("verify_1c_adapter_deployment.ps1 must split comma-separated -BaseId values before verification.")
if "function Assert-SelectorChainReport" not in verify_text or verify_text.count("Assert-SelectorChainReport") < 3:
failures.append("verify_1c_adapter_deployment.ps1 must validate both persisted selector-chain JSON reports after smoke commands.")
if "working_state" not in verify_text or 'did not use working state' not in verify_text:
failures.append("verify_1c_adapter_deployment.ps1 must validate selector-chain working_state=working in persisted reports.")
for function_name in (
"Assert-WritePlanSafetyReport",
"Assert-WritePreflightReport",
"Assert-WriteRollbackSafetyReport",
"Assert-SavedStateDiffReport",
"Assert-SavedStateChangesReport",
"Assert-SavedStateFormWriteReport",
"Assert-SavedStateModuleWriteReport",
):
if f"function {function_name}" not in verify_text or verify_text.count(function_name) < 2:
failures.append(f"verify_1c_adapter_deployment.ps1 must validate reports with {function_name}.")
if "scripts/check_1c_verify_reports.py" not in verify_text:
failures.append("verify_1c_adapter_deployment.ps1 must run the offline persisted report validator.")
if "scripts/plan_1c_saved_state_copy.py" not in verify_text or "saved-state-copy-plan.json" not in verify_text:
failures.append("verify_1c_adapter_deployment.ps1 must generate saved-state-copy-plan.json before persisted report validation.")
if "scripts/prepare_1c_saved_state_copy_sql.py" not in verify_text or "prepare-saved-state-copy-sql.json" not in verify_text:
failures.append("verify_1c_adapter_deployment.ps1 must generate prepare-saved-state-copy-sql.json before persisted report validation.")
if "scripts/prepare_1c_saved_state_cleanup_sql.py" not in verify_text or "cleanup-saved-state-copy-sql.json" not in verify_text:
failures.append("verify_1c_adapter_deployment.ps1 must generate cleanup-saved-state-copy-sql.json before persisted report validation.")
if "scripts/check_1c_saved_state_strict_readiness.py" not in verify_text or "saved-state-strict-readiness.json" not in verify_text:
failures.append("verify_1c_adapter_deployment.ps1 must generate saved-state-strict-readiness.json before persisted report validation.")
for flag in (
"--skip-rest",
"--skip-mcp",
"--skip-write-plan-safety-smoke",
"--skip-write-rollback-safety-smoke",
"--skip-saved-state-diff-smoke",
"--skip-saved-state-write-smoke",
"--require-saved-state-write-smoke",
"--require-selector-chain-write-plan-composition",
"--rest-adapter-url",
"--mcp-url",
"--saved-state-table",
):
if flag not in verify_text:
failures.append(f"verify_1c_adapter_deployment.ps1 must forward {flag} to scripts/check_1c_verify_reports.py.")
verify_reports_self_test = subprocess.run(
[sys.executable, "scripts/check_1c_verify_reports.py", "--self-test", "--json"],
cwd=ROOT,
text=True,
capture_output=True,
check=False,
)
if verify_reports_self_test.returncode != 0:
output = "\n".join(part for part in [verify_reports_self_test.stdout.strip(), verify_reports_self_test.stderr.strip()] if part)
failures.append(f"scripts/check_1c_verify_reports.py --self-test failed: {output}")
else:
try:
self_test_report = json.loads(verify_reports_self_test.stdout)
except json.JSONDecodeError as exc:
failures.append(f"scripts/check_1c_verify_reports.py --self-test --json returned invalid JSON: {exc}")
else:
expected_self_test_codes = {
"strict_skip_failure_codes": {
"selector_chain_composition_required",
"saved_state_form_write_required",
"saved_state_module_write_required",
},
"coverage_failure_codes": {
"selector_chain_write_plan_evidence_missing",
"selector_chain_next_method_unexpected",
"selector_chain_write_plan_target_not_boolean",
"selector_chain_working_state_unexpected",
},
"consistency_failure_codes": {
"selector_chain_write_plan_target_not_composed",
"selector_chain_composed_status_unexpected",
},
"safety_failure_codes": {
"write_plan_safety_check_field_unexpected",
"write_plan_safety_check_missing",
},
"rollback_safety_failure_codes": {
"write_rollback_safety_check_field_unexpected",
"write_rollback_safety_check_missing",
},
"saved_state_diff_failure_codes": {
"saved_state_diff_check_field_unexpected",
"saved_state_diff_check_missing",
},
"schema_failure_codes": {
"report_schema_unexpected",
},
"identity_failure_codes": {
"report_base_id_unexpected",
"report_transport_unexpected",
},
"endpoint_failure_codes": {
"report_endpoint_url_unexpected",
},
"staleness_failure_codes": {
"report_stale",
},
"duplicate_failure_codes": {
"duplicate_base_id",
},
"saved_state_failure_codes": {
"saved_state_form_route_write_plan_not_allowed",
"saved_state_form_route_field_missing",
"saved_state_module_rollback_missing",
},
"saved_state_strict_readiness_failure_codes": {
"saved_state_strict_readiness_required",
"saved_state_strict_readiness_table_unexpected",
},
"saved_state_copy_plan_failure_codes": {
"saved_state_copy_plan_source_family_invalid",
"saved_state_copy_plan_status_unexpected",
"saved_state_copy_plan_source_row_table_unexpected",
"saved_state_copy_plan_target_collisions_present",
},
"saved_state_table_failure_codes": {
"saved_state_table_mismatch",
"saved_state_table_unexpected",
},
}
for field, expected_codes in expected_self_test_codes.items():
actual_codes = set(self_test_report.get(field) or [])
missing_codes = sorted(expected_codes - actual_codes)
if missing_codes:
failures.append(f"scripts/check_1c_verify_reports.py --self-test must cover {field}: missing {missing_codes}.")
if "ConvertFrom-Json" not in verify_text:
failures.append("verify_1c_adapter_deployment.ps1 must parse selector-chain JSON reports with ConvertFrom-Json.")
if "--saved-state-table" not in readiness_text or "saved_state_table" not in readiness_text:
failures.append("check_1c_saved_state_strict_readiness.py must support --saved-state-table and include it in reports.")
for token in ("source_family.valid", "target_collisions.status", "sql_write_performed", "BEGIN TRANSACTION", "THROW 51001"):
if token not in prepare_copy_sql_text:
failures.append(f"prepare_1c_saved_state_copy_sql.py must include guarded SQL generation token: {token}.")
for token in ("onec_saved_state_cleanup_sql_plan.v1", "DELETE t FROM", "BinarySHA1", "THROW 51102", "sql_write_performed"):
if token not in prepare_cleanup_sql_text:
failures.append(f"prepare_1c_saved_state_cleanup_sql.py must include guarded cleanup SQL token: {token}.")
for token in ("onec_saved_state_copy_verify.v1", "blocked_missing_target_rows", "BinarySHA1", "sql_write_performed", "--require-ready"):
if token not in verify_copy_text:
failures.append(f"verify_1c_saved_state_copy.py must include read-only verification token: {token}.")
for token in (
"[switch]$IUnderstandThisWritesToSql",
"Refusing to execute SQL without -IUnderstandThisWritesToSql",
"onec_saved_state_copy_sql_execution.v1",
"onec_saved_state_copy_sql_plan.v1",
"sql_execution_attempted",
"sql_write_performed",
"Get-FileHash",
"scripts/verify_1c_saved_state_copy.py",
"--require-ready",
):
if token not in execute_copy_sql_text:
failures.append(f"execute_1c_saved_state_copy_sql.ps1 must include guarded execution token: {token}.")
if "[switch]$RequireSelectorChainWritePlanComposition" not in deploy_text:
failures.append("deploy_1c_adapter_stack.ps1 must declare -RequireSelectorChainWritePlanComposition.")
if "[switch]$RequireSavedStateWriteSmoke" not in deploy_text:
failures.append("deploy_1c_adapter_stack.ps1 must declare -RequireSavedStateWriteSmoke.")
if "[switch]$SkipWriteRollbackSafetySmoke" not in deploy_text:
failures.append("deploy_1c_adapter_stack.ps1 must declare -SkipWriteRollbackSafetySmoke.")
if "[switch]$SkipSavedStateDiffSmoke" not in deploy_text:
failures.append("deploy_1c_adapter_stack.ps1 must declare -SkipSavedStateDiffSmoke.")
if "[string]$SavedStateTable" not in deploy_text or '[ValidateSet("ConfigSave", "ConfigCASSave")]' not in deploy_text:
failures.append("deploy_1c_adapter_stack.ps1 must declare -SavedStateTable with ConfigSave/ConfigCASSave validation.")
if '"-RequireSelectorChainWritePlanComposition"' not in deploy_text:
failures.append("deploy_1c_adapter_stack.ps1 must forward -RequireSelectorChainWritePlanComposition to verify_1c_adapter_deployment.ps1.")
if '"-RequireSavedStateWriteSmoke"' not in deploy_text:
failures.append("deploy_1c_adapter_stack.ps1 must forward -RequireSavedStateWriteSmoke to verify_1c_adapter_deployment.ps1.")
if '"-SavedStateTable"' not in deploy_text:
failures.append("deploy_1c_adapter_stack.ps1 must forward -SavedStateTable to verify_1c_adapter_deployment.ps1.")
if '"-SkipWriteRollbackSafetySmoke"' not in deploy_text:
failures.append("deploy_1c_adapter_stack.ps1 must forward -SkipWriteRollbackSafetySmoke to verify_1c_adapter_deployment.ps1.")
if '"-SkipSavedStateDiffSmoke"' not in deploy_text:
failures.append("deploy_1c_adapter_stack.ps1 must forward -SkipSavedStateDiffSmoke to verify_1c_adapter_deployment.ps1.")
if "function Get-DuplicateValues" not in deploy_text or "Duplicate BaseId value(s)" not in deploy_text:
failures.append("deploy_1c_adapter_stack.ps1 must reject duplicate -BaseId values before invoking verification.")
if "function Normalize-BaseIds" not in deploy_text or '-split ","' not in deploy_text:
failures.append("deploy_1c_adapter_stack.ps1 must split comma-separated -BaseId values before invoking verification.")
if '$baseIds -join ","' not in deploy_text:
failures.append("deploy_1c_adapter_stack.ps1 must pass normalized BaseId values to nested verification as a comma-separated argument.")
for flag in ("--rest-adapter-url", "--mcp-url", "--saved-state-table", "--max-report-age-seconds"):
if flag not in stack_text:
failures.append(f"check_1c_adapter_verification_stack.py must pass {flag} to scripts/check_1c_verify_reports.py.")
for script_name in (
"scripts/smoke_1c_write_plan_safety.py",
"scripts/smoke_1c_write_preflight.py",
"scripts/smoke_1c_write_rollback_safety.py",
"scripts/smoke_1c_saved_state_diff.py",
"scripts/smoke_1c_saved_state_changes.py",
"scripts/smoke_1c_saved_state_write_routes.py",
"scripts/smoke_1c_saved_state_module_write.py",
"scripts/check_1c_saved_state_strict_readiness.py",
"scripts/plan_1c_saved_state_copy.py",
"scripts/prepare_1c_saved_state_copy_sql.py",
"scripts/prepare_1c_saved_state_cleanup_sql.py",
"scripts/verify_1c_saved_state_copy.py",
):
if script_name not in stack_text:
failures.append(f"check_1c_adapter_verification_stack.py must py_compile {script_name}.")
if 'nargs="+"' not in stack_text or "*args.base_id" not in stack_text:
failures.append("check_1c_adapter_verification_stack.py must support multiple --base-id values and forward them to scripts/check_1c_verify_reports.py.")
if "duplicate_base_id" not in stack_text:
failures.append("check_1c_adapter_verification_stack.py must reject duplicate --base-id values.")
failures.extend(check_adapter_base_id_runtime(executable))
return failures
def main() -> int:
parser = argparse.ArgumentParser(description="Check PowerShell script syntax with PSParser.")
parser.add_argument("paths", nargs="*", type=Path, help="Files or directories to scan. Defaults to scripts/ and core/deploy/.")
args = parser.parse_args()
executable = powershell_executable()
if not executable:
print("PowerShell executable not found.", file=sys.stderr)
return 1
paths = [path if path.is_absolute() else ROOT / path for path in args.paths] if args.paths else DEFAULT_PATHS
scripts = iter_powershell_scripts(paths)
if not scripts:
print("No PowerShell scripts found.")
return 0
failures = []
for script in scripts:
ok, output = check_script(executable, script)
if not ok:
failures.append((script, output))
contract_failures = check_adapter_verify_wiring(scripts, executable)
if failures or contract_failures:
print("PowerShell script check failed:", file=sys.stderr)
for script, output in failures:
print(f"- {script.relative_to(ROOT)}", file=sys.stderr)
if output:
print(output, file=sys.stderr)
for failure in contract_failures:
print(f"- adapter verify wiring: {failure}", file=sys.stderr)
return 1
print(f"Validated {len(scripts)} PowerShell script(s).")
return 0
if __name__ == "__main__":
raise SystemExit(main())
+71
View File
@@ -0,0 +1,71 @@
param(
[string]$UiBaseUrl = "http://192.168.220.91:8765",
[string]$ModelId = "qwen3-coder-30b-a3b-instruct-q6_k",
[string]$Plugin = "1c",
[string[]]$Profiles = @("gpu-fast", "cpu-test"),
[string]$Report = "reports/benchmarks/runtime-preflight-latest.json",
[int]$TimeoutSec = 60
)
$ErrorActionPreference = "Stop"
$root = Split-Path -Parent (Split-Path -Parent $MyInvocation.MyCommand.Path)
$reportPath = Join-Path $root $Report
$url = "$($UiBaseUrl.TrimEnd('/'))/api/benchmark/preflight"
$payload = @{
model_id = $ModelId
plugin = $Plugin
profiles = $Profiles
} | ConvertTo-Json -Depth 8
Write-Host "Runtime benchmark preflight"
Write-Host "URL: $url"
Write-Host "Model: $ModelId"
Write-Host "Plugin: $Plugin"
Write-Host "Profiles: $($Profiles -join ', ')"
Write-Host ""
try {
$result = Invoke-RestMethod -Uri $url -Method Post -ContentType "application/json" -Body $payload -TimeoutSec $TimeoutSec
} catch {
$response = $_.Exception.Response
if ($response) {
try {
$stream = $response.GetResponseStream()
$reader = [System.IO.StreamReader]::new($stream)
$body = $reader.ReadToEnd()
if ($body) {
$result = $body | ConvertFrom-Json
} else {
throw
}
} catch {
throw $_.Exception
}
} else {
throw
}
}
New-Item -ItemType Directory -Force (Split-Path -Parent $reportPath) | Out-Null
$result | ConvertTo-Json -Depth 10 | Set-Content -LiteralPath $reportPath -Encoding UTF8
foreach ($check in @($result.checks)) {
$target = $check.target
$served = if ($target) { $target.served_model_name } else { "-" }
$endpoint = if ($target) { $target.base_url } else { "-" }
$latency = if ($check.latency_ms -ne $null) { "$($check.latency_ms)ms" } else { "-" }
$status = if ($check.available) { "ready" } else { $check.status }
Write-Host ("{0,-12} {1,-14} {2,6} {3} {4}" -f $check.profile_id, $status, $latency, $endpoint, $served)
if (-not $check.available -and $check.error) {
Write-Host " error: $($check.error)"
}
}
Write-Host ""
Write-Host "Status: $($result.status)"
Write-Host "Report: $reportPath"
if (-not $result.ready) {
exit 1
}
+14
View File
@@ -0,0 +1,14 @@
param(
[string]$BaseUrl = "http://docker-gpu.cin.su:8000",
[string]$Model = "qwen3-4b-instruct"
)
$ErrorActionPreference = "Stop"
$modelsUrl = "$($BaseUrl.TrimEnd('/'))/v1/models"
Write-Host "Checking models endpoint: $modelsUrl"
Invoke-RestMethod -Method Get -Uri $modelsUrl | ConvertTo-Json -Depth 10
Write-Host ""
Write-Host "Checking chat endpoint..."
python scripts/smoke_chat.py --base-url $BaseUrl --model $Model
+85
View File
@@ -0,0 +1,85 @@
param(
[string]$SshTarget = "",
[string]$ModelsRoot = "Z:\LLM\models",
[int]$ConnectTimeoutSeconds = 5
)
$ErrorActionPreference = "Stop"
function Invoke-LocalCheck {
param(
[Parameter(Mandatory = $true)]
[string]$Name,
[Parameter(Mandatory = $true)]
[scriptblock]$Block
)
Write-Host ""
Write-Host "== $Name"
& $Block
}
function Invoke-RemoteCheck {
param(
[Parameter(Mandatory = $true)]
[string]$Name,
[Parameter(Mandatory = $true)]
[string]$Command
)
Write-Host ""
Write-Host "== $Name"
ssh -o BatchMode=yes -o ConnectTimeout=$ConnectTimeoutSeconds $SshTarget "powershell -NoProfile -Command $Command"
if ($LASTEXITCODE -ne 0) {
throw "Remote check failed on ${SshTarget}: ${Name}"
}
}
if ($SshTarget) {
Invoke-RemoteCheck "Host identity" "whoami; hostname; `$PSVersionTable.PSVersion.ToString()"
Invoke-RemoteCheck "GPU driver" "nvidia-smi --query-gpu=name,memory.total,driver_version --format=csv,noheader"
Invoke-RemoteCheck "Docker version" "docker version"
Invoke-RemoteCheck "Docker compose" "docker compose version"
Invoke-RemoteCheck "Docker info" "docker info --format '{{json .}}'"
Invoke-RemoteCheck "Models drive" "if (-not (Test-Path '$ModelsRoot')) { New-Item -ItemType Directory -Force '$ModelsRoot' | Out-Null }; Get-Item '$ModelsRoot' | Select-Object FullName,Exists"
Invoke-RemoteCheck "CUDA container" "docker run --rm --gpus all nvidia/cuda:12.4.1-base-ubuntu22.04 nvidia-smi --query-gpu=name,memory.total --format=csv,noheader"
Write-Host ""
Write-Host "Windows GPU host preflight completed through SSH."
exit 0
}
Invoke-LocalCheck "Host identity" {
whoami
hostname
$PSVersionTable.PSVersion.ToString()
}
Invoke-LocalCheck "GPU driver" {
nvidia-smi --query-gpu=name,memory.total,driver_version --format=csv,noheader
}
Invoke-LocalCheck "Docker version" {
docker version
}
Invoke-LocalCheck "Docker compose" {
docker compose version
}
Invoke-LocalCheck "Docker info" {
docker info --format '{{json .}}'
}
Invoke-LocalCheck "Models drive" {
if (-not (Test-Path $ModelsRoot)) {
New-Item -ItemType Directory -Force $ModelsRoot | Out-Null
}
Get-Item $ModelsRoot | Select-Object FullName,Exists
}
Invoke-LocalCheck "CUDA container" {
docker run --rm --gpus all nvidia/cuda:12.4.1-base-ubuntu22.04 nvidia-smi --query-gpu=name,memory.total --format=csv,noheader
}
Write-Host ""
Write-Host "Windows GPU host preflight completed."
+191
View File
@@ -0,0 +1,191 @@
#!/usr/bin/env python3
"""Describe extension manifest payloads by observable structure.
This script intentionally reports structural facts first. Semantic labels are
only added when the evidence is direct, for example a payload contains BSL text
or embedded HTML help.
"""
from __future__ import annotations
import argparse
import base64
import json
import re
from collections import Counter, defaultdict
from pathlib import Path
from typing import Any
from inspect_1c_sql_files import Lexer, Parser, collect_strings, tree_shape, try_decode, try_decompress
BSL_MARKERS = ("&На", "Процедура ", "Функция ", "#Область", "#КонецОбласти")
HTML_MARKERS = ("<!DOCTYPE HTML", "<html", "<HTML", "<body", "<BODY")
def scalar(node: Any) -> str:
if isinstance(node, dict) and node.get("type") in {"atom", "string"}:
return str(node.get("value") or "")
return ""
def suffix_of(object_id: str) -> str:
parts = object_id.split(".", 1)
return "" if len(parts) == 1 else "." + parts[1]
def parse_payload(path: Path) -> dict[str, Any]:
raw = path.read_bytes()
payload, compression = try_decompress(raw)
text, encoding = try_decode(payload)
result: dict[str, Any] = {
"bytes": len(raw),
"payload_bytes": len(payload),
"compression": compression,
"encoding": encoding,
"parse_status": "not_text",
"root_kind": "",
"root_len": None,
"root_marker": "",
"strings_sample": [],
"bsl_marker_count": 0,
"html_marker_count": 0,
"base64_atom_count": 0,
"embedded_base64_html_count": 0,
"semantic_evidence": [],
}
if text is None:
return result
clean = text.replace("\x00", "").replace("\ufeff", "").lstrip("ï»¿п»ї")
result["bsl_marker_count"] = sum(clean.count(marker) for marker in BSL_MARKERS)
result["html_marker_count"] = sum(clean.count(marker) for marker in HTML_MARKERS)
try:
parsed = Parser(Lexer(clean[:2_000_000]).tokens()).parse()
except Exception as exc:
result["parse_status"] = "parse_error"
result["parse_error"] = str(exc)
return result
result["parse_status"] = "parsed"
result["shape"] = tree_shape(parsed, max_depth=3)
strings = collect_strings(parsed, limit=80)
result["strings_sample"] = strings[:40]
if isinstance(parsed, dict):
result["root_kind"] = parsed.get("type") or ""
items = parsed.get("items") or []
result["root_len"] = len(items)
if items:
result["root_marker"] = scalar(items[0])
atoms = []
def walk(node: Any) -> None:
if isinstance(node, dict) and node.get("type") == "atom":
atoms.append(str(node.get("value") or ""))
if isinstance(node, dict):
for child in node.get("items") or []:
walk(child)
walk(parsed)
b64_atoms = [value for value in atoms if re.fullmatch(r"[A-Za-z0-9+/]{40,}={0,2}", value)]
result["base64_atom_count"] = len(b64_atoms)
html_count = 0
for value in b64_atoms[:200]:
try:
decoded = base64.b64decode(value, validate=False)
except Exception:
continue
if any(marker.encode("utf-8") in decoded or marker.encode("cp1251", errors="ignore") in decoded for marker in HTML_MARKERS):
html_count += 1
result["embedded_base64_html_count"] = html_count
evidence = []
if result["bsl_marker_count"]:
evidence.append("contains_bsl_text")
if result["html_marker_count"] or html_count:
evidence.append("contains_html")
if result["base64_atom_count"]:
evidence.append("contains_base64_atoms")
result["semantic_evidence"] = evidence
return result
def main() -> int:
parser = argparse.ArgumentParser(description="Classify manifest payloads by observed structure.")
parser.add_argument("--manifest-dir", type=Path, required=True)
parser.add_argument("--cas-dir", type=Path, required=True)
parser.add_argument("--xml-index", type=Path)
parser.add_argument("--output", type=Path, required=True)
parser.add_argument("--limit", type=int, default=0)
args = parser.parse_args()
xml_map = {}
if args.xml_index and args.xml_index.is_file():
xml = json.loads(args.xml_index.read_text(encoding="utf-8"))
xml_map = xml.get("guid_map") or {}
entries = []
for manifest_path in sorted(args.manifest_dir.glob("*.json")):
manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
extension_file = (manifest.get("extension_zipped_info") or {}).get("file_name") or manifest_path.name
for entry in manifest.get("entries") or []:
cas_path = Path(entry.get("cas_path") or args.cas_dir / entry["cas_key"])
if not cas_path.is_file():
continue
object_id = entry["object_id"]
base_guid = object_id.split(".", 1)[0].lower()
payload = parse_payload(cas_path)
xml_item = xml_map.get(base_guid) or {}
top_objects = xml_item.get("top_objects") or []
entries.append(
{
"extension_file": extension_file,
"manifest_path": str(manifest_path),
"object_id": object_id,
"base_guid": base_guid,
"suffix": suffix_of(object_id),
"cas_key": entry["cas_key"],
"xml_top_objects": top_objects[:5],
"payload": payload,
}
)
if args.limit and len(entries) >= args.limit:
break
if args.limit and len(entries) >= args.limit:
break
suffix_counts = Counter(item["suffix"] for item in entries)
suffix_root_counts: dict[str, Counter[str]] = defaultdict(Counter)
suffix_evidence_counts: dict[str, Counter[str]] = defaultdict(Counter)
for item in entries:
suffix = item["suffix"]
payload = item["payload"]
root_signature = f"{payload.get('root_kind')}:{payload.get('root_marker')}:{payload.get('root_len')}"
suffix_root_counts[suffix][root_signature] += 1
for evidence in payload.get("semantic_evidence") or ["<none>"]:
suffix_evidence_counts[suffix][evidence] += 1
report = {
"schema": "onec_manifest_payload_structure.v1",
"manifest_dir": str(args.manifest_dir),
"cas_dir": str(args.cas_dir),
"entry_count": len(entries),
"suffix_counts": dict(sorted(suffix_counts.items())),
"suffix_root_counts": {key: dict(value.most_common()) for key, value in sorted(suffix_root_counts.items())},
"suffix_evidence_counts": {key: dict(value.most_common()) for key, value in sorted(suffix_evidence_counts.items())},
"entries": entries,
}
args.output.parent.mkdir(parents=True, exist_ok=True)
args.output.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
print(
json.dumps(
{
"output": str(args.output),
"entries": len(entries),
"suffixes": len(suffix_counts),
},
ensure_ascii=False,
)
)
return 0
if __name__ == "__main__":
raise SystemExit(main())
+156
View File
@@ -0,0 +1,156 @@
from __future__ import annotations
import argparse
import datetime as dt
import json
import socket
import subprocess
import sys
from pathlib import Path
from typing import Any
ROOT = Path(__file__).resolve().parents[1]
DEFAULT_REPORT = ROOT / "reports" / "platform-status.json"
DEFAULT_DOCKER_HOST = "ssh://docker-gpu"
MODEL_CHAT_CONTAINER = "llm-model-chat-ui"
def run_json(command: list[str], timeout: int = 60) -> dict[str, Any]:
result = subprocess.run(
command,
cwd=ROOT,
text=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
timeout=timeout,
check=False,
)
payload = None
if result.stdout.strip().startswith("{"):
try:
payload = json.loads(result.stdout)
except json.JSONDecodeError:
payload = None
return {
"command": command,
"status": "ok" if result.returncode == 0 else "failed",
"returncode": result.returncode,
"stdout": result.stdout.strip(),
"stderr": result.stderr.strip(),
"json": payload,
}
def read_report(path: Path) -> dict[str, Any] | None:
if not path.exists():
return None
try:
with path.open("r", encoding="utf-8") as handle:
data = json.load(handle)
return data if isinstance(data, dict) else None
except json.JSONDecodeError:
return None
def is_port_open(host: str, port: int, timeout: float = 1.0) -> bool:
try:
with socket.create_connection((host, port), timeout=timeout):
return True
except OSError:
return False
def deployed_model_storage(docker_host: str) -> dict[str, Any]:
return run_json(
[
"docker",
"-H",
docker_host,
"exec",
MODEL_CHAT_CONTAINER,
"python3",
"scripts/check_model_storage.py",
"--models-root",
"/models",
"--print",
"--no-report",
"--warn-only",
],
timeout=90,
)
def main() -> int:
parser = argparse.ArgumentParser(description="Collect local platform status into one report.")
parser.add_argument("--report", type=Path, default=DEFAULT_REPORT)
parser.add_argument("--print", action="store_true")
parser.add_argument("--check-endpoints", action="store_true", help="Run GPU readiness check with network timeouts.")
parser.add_argument("--docker-host", default=DEFAULT_DOCKER_HOST, help="Docker endpoint for deployed GPU checks.")
parser.add_argument(
"--skip-deployed-storage",
action="store_true",
help="Skip model storage check inside the deployed model-chat container.",
)
args = parser.parse_args()
model_storage = run_json(
[sys.executable, "scripts/check_model_storage.py", "--print", "--no-report", "--warn-only"],
timeout=60,
)
if model_storage.get("json", {}).get("status") == "failed":
model_storage["status"] = "blocked"
checks = {
"model_cards": run_json([sys.executable, "scripts/validate_model_cards.py"], timeout=30),
"eval_files": run_json([sys.executable, "scripts/validate_evals.py"], timeout=30),
"model_storage": model_storage,
"deployed_model_storage": None if args.skip_deployed_storage else deployed_model_storage(args.docker_host),
"1c_plugin": run_json([sys.executable, "scripts/check_1c_plugin.py", "--no-report"], timeout=60),
"model_chat": {
"status": "ok" if is_port_open("127.0.0.1", 8765) else "stopped",
"url": "http://127.0.0.1:8765",
},
"gpu_readiness": read_report(ROOT / "reports" / "gpu-readiness.json"),
"live_model_evals": read_report(ROOT / "reports" / "evals" / "live-model-evals.json"),
}
if args.check_endpoints:
checks["gpu_readiness"] = run_json(
[sys.executable, "scripts/check_gpu_readiness.py", "--print", "--timeout", "8"],
timeout=45,
).get("json")
if isinstance(checks["gpu_readiness"], dict) and checks["gpu_readiness"].get("status") == "failed":
checks["gpu_readiness"]["status"] = "blocked"
failed = []
blocked = []
for name, check in checks.items():
if not check:
if name != "deployed_model_storage":
blocked.append(name)
continue
status = check.get("status")
if status == "failed":
failed.append(name)
elif status in {"blocked", "stopped"}:
blocked.append(name)
report = {
"created_at": dt.datetime.now(dt.UTC).isoformat(),
"status": "failed" if failed else "blocked" if blocked else "ok",
"failed": failed,
"blocked": blocked,
"checks": checks,
}
args.report.parent.mkdir(parents=True, exist_ok=True)
args.report.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
if args.print:
print(json.dumps(report, ensure_ascii=False, indent=2))
else:
print(f"Platform status: {report['status']}")
print(f"Wrote report to {args.report}")
return 1 if failed else 0
if __name__ == "__main__":
raise SystemExit(main())
+366
View File
@@ -0,0 +1,366 @@
from __future__ import annotations
import json
import math
import re
import urllib.request
import struct
from collections import Counter
from pathlib import Path
from typing import Any
import yaml
ROOT = Path(__file__).resolve().parents[1]
MODEL_CARDS_DIR = ROOT / "registry" / "model-cards"
TOKEN_RE = re.compile(r"[A-Za-zА-Яа-яЁё0-9_]+", re.UNICODE)
RUSSIAN_ENDINGS = (
"иями",
"ями",
"ами",
"ого",
"ему",
"ыми",
"ими",
"ой",
"ей",
"ых",
"их",
"ую",
"юю",
"ая",
"яя",
"ое",
"ее",
"ом",
"ем",
"ам",
"ям",
"ах",
"ях",
"ы",
"и",
"а",
"я",
"е",
"у",
"ю",
)
TOKEN_ALIASES = {
"1с": ["1c", "bsl", "конфигурация"],
"1c": ["1с", "bsl", "configuration"],
"бсл": ["bsl", "1с"],
"bsl": ["бсл", "1с"],
"справочник": ["catalog", "справочники"],
"справочники": ["справочник", "catalog"],
"документ": ["documents", "документы"],
"документы": ["документ", "documents"],
"регистр": ["register", "регистры"],
"регистры": ["регистр", "register"],
"реквизит": ["attribute", "реквизиты"],
"реквизиты": ["реквизит", "attribute"],
"табличная": ["табличные", "часть"],
"табличные": ["табличная", "часть"],
"запрос": ["query", "read", "select", "выбрать"],
"выбрать": ["запрос", "query", "select"],
"форма": ["forms", "управляемая"],
"модуль": ["module", "bsl"],
"метаданные": ["metadata", "схема", "snapshot"],
"схема": ["metadata", "метаданные"],
"номенклатура": ["справочник", "catalog"],
}
def read_json(path: Path) -> dict:
with path.open("r", encoding="utf-8-sig") as handle:
data = json.load(handle)
if not isinstance(data, dict):
raise ValueError(f"{path} must contain a JSON object")
return data
def read_jsonl(path: Path) -> list[dict]:
if not path.exists():
return []
records: list[dict] = []
with path.open("r", encoding="utf-8") as handle:
for line_number, line in enumerate(handle, start=1):
line = line.strip()
if not line:
continue
try:
record = json.loads(line)
except json.JSONDecodeError as exc:
raise ValueError(f"{path}:{line_number}: invalid JSONL: {exc}") from exc
if not isinstance(record, dict):
raise ValueError(f"{path}:{line_number}: record must be an object")
records.append(record)
return records
def write_json(path: Path, data: dict) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(json.dumps(data, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
def read_yaml_mapping(path: Path) -> dict[str, Any]:
with path.open("r", encoding="utf-8") as handle:
data = yaml.safe_load(handle)
if not isinstance(data, dict):
raise ValueError(f"{path} must contain a YAML mapping")
return data
def iter_model_card_paths(*, include_examples: bool = False) -> list[Path]:
paths = sorted(MODEL_CARDS_DIR.rglob("*.yaml")) + sorted(MODEL_CARDS_DIR.rglob("*.yml"))
if include_examples:
return paths
return [path for path in paths if "examples" not in path.parts]
def load_model_card(card_id: str) -> dict[str, Any]:
for suffix in (".yaml", ".yml"):
path = MODEL_CARDS_DIR / f"{card_id}{suffix}"
if path.exists():
return read_yaml_mapping(path)
raise FileNotFoundError(f"Model card not found for id `{card_id}` in {MODEL_CARDS_DIR}")
def localize_workspace_path(path: str) -> Path:
if path.startswith("/workspace/"):
if Path("/workspace").exists():
return Path(path)
return ROOT / path.removeprefix("/workspace/")
if path.startswith("/models/"):
if Path("/models").exists():
return Path(path)
return ROOT / "models" / path.removeprefix("/models/")
return Path(path)
def stem_russian_token(token: str) -> str:
if not re.search(r"[а-яё]", token, flags=re.IGNORECASE) or len(token) < 6:
return token
for ending in RUSSIAN_ENDINGS:
if token.endswith(ending) and len(token) - len(ending) >= 4:
return token[: -len(ending)]
return token
def normalize_token(token: str) -> str:
token = token.lower().replace("ё", "е")
return stem_russian_token(token)
def tokenize(text: str, *, expand_aliases: bool = False) -> list[str]:
tokens: list[str] = []
for raw_token in TOKEN_RE.findall(text):
token = normalize_token(raw_token)
tokens.append(token)
if expand_aliases:
tokens.extend(normalize_token(alias) for alias in TOKEN_ALIASES.get(token, []))
return tokens
def corpus_content_hash(records: list[dict]) -> str:
import hashlib
hasher = hashlib.sha256()
for record in records:
stable = {
"id": record.get("id"),
"document_id": record.get("document_id"),
"source_path": record.get("source_path"),
"chunk_index": record.get("chunk_index"),
"title": record.get("title"),
"content": record.get("content"),
"metadata": record.get("metadata") or {},
}
hasher.update(json.dumps(stable, ensure_ascii=False, sort_keys=True, separators=(",", ":")).encode("utf-8"))
hasher.update(b"\n")
return hasher.hexdigest()
def hashing_embedding(text: str, *, dimensions: int = 384) -> list[float]:
import hashlib
if dimensions < 8:
raise ValueError("dimensions must be >= 8")
vector = [0.0] * dimensions
tokens = tokenize(text, expand_aliases=True)
if not tokens:
return vector
counts = Counter(tokens)
for token, count in counts.items():
digest = hashlib.blake2b(token.encode("utf-8"), digest_size=8).digest()
bucket = int.from_bytes(digest[:4], "little") % dimensions
sign = 1.0 if digest[4] & 1 else -1.0
vector[bucket] += sign * (1.0 + math.log(float(count)))
norm = math.sqrt(sum(value * value for value in vector))
if norm <= 0:
return vector
return [value / norm for value in vector]
def pack_float_vector(vector: list[float]) -> bytes:
return struct.pack(f"<{len(vector)}f", *vector)
def unpack_float_vector(data: bytes, dimensions: int) -> list[float]:
expected_size = dimensions * 4
if len(data) != expected_size:
raise ValueError(f"Vector blob has {len(data)} bytes, expected {expected_size}")
return list(struct.unpack(f"<{dimensions}f", data))
def cosine_similarity(left: list[float], right: list[float]) -> float:
if not left or not right or len(left) != len(right):
return 0.0
return float(sum(a * b for a, b in zip(left, right)))
def build_lexical_index(records: list[dict]) -> dict:
documents = []
document_frequency: Counter[str] = Counter()
total_length = 0
for record in records:
content = record.get("content") or ""
tokens = tokenize(content)
title_tokens = tokenize(str(record.get("title") or ""))
term_frequency = Counter(tokens)
document_frequency.update(term_frequency.keys())
total_length += len(tokens)
documents.append(
{
"id": record.get("id"),
"document_id": record.get("document_id"),
"source_path": record.get("source_path"),
"title": record.get("title"),
"chunk_index": record.get("chunk_index"),
"content": content,
"metadata": record.get("metadata") or {},
"length": len(tokens),
"title_tokens": title_tokens,
"term_frequency": dict(term_frequency),
}
)
doc_count = len(documents)
idf = {
token: math.log((1 + doc_count) / (1 + frequency)) + 1
for token, frequency in document_frequency.items()
}
return {
"schema_version": 2,
"type": "lexical-bm25",
"doc_count": doc_count,
"avg_doc_length": round(total_length / doc_count, 4) if doc_count else 0,
"idf": idf,
"documents": documents,
}
def score_lexical_document(query_tf: Counter[str], document: dict, idf: dict, avg_doc_length: float) -> float:
doc_tf = document.get("term_frequency") or {}
doc_length = max(float(document.get("length") or 0), 1.0)
avg_doc_length = max(float(avg_doc_length or doc_length), 1.0)
title_tokens = set(document.get("title_tokens") or [])
score = 0.0
k1 = 1.4
b = 0.72
for token, query_count in query_tf.items():
doc_count = doc_tf.get(token, 0)
if doc_count:
numerator = doc_count * (k1 + 1)
denominator = doc_count + k1 * (1 - b + b * (doc_length / avg_doc_length))
score += query_count * float(idf.get(token, 1.0)) * (numerator / denominator)
if token in title_tokens:
score += 0.35 * query_count
return score
def search_lexical_index(
index: dict,
query: str,
limit: int,
*,
candidate_limit: int | None = None,
dedupe_by_document: bool = False,
min_score: float = 0.0,
source_types: list[str] | None = None,
metadata_filters: dict[str, str] | None = None,
) -> list[dict]:
query_tf = Counter(tokenize(query, expand_aliases=True))
if not query_tf:
return []
results = []
idf = index.get("idf") or {}
avg_doc_length = float(index.get("avg_doc_length") or 0)
allowed_source_types = {source_type.lower() for source_type in source_types or []}
exact_filters = {key: str(value).lower() for key, value in (metadata_filters or {}).items() if str(value).strip()}
for document in index.get("documents") or []:
metadata = document.get("metadata") or {}
source_type = str(metadata.get("source_type") or "").lower()
if allowed_source_types and source_type not in allowed_source_types:
continue
if exact_filters and any(str(metadata.get(key) or "").lower() != value for key, value in exact_filters.items()):
continue
score = score_lexical_document(query_tf, document, idf, avg_doc_length)
if score > min_score:
results.append({"score": score, "document": document})
results.sort(key=lambda item: item["score"], reverse=True)
if candidate_limit:
results = results[:candidate_limit]
if dedupe_by_document:
deduped = []
seen = set()
for result in results:
document_id = result["document"].get("document_id") or result["document"].get("source_path")
if document_id in seen:
continue
seen.add(document_id)
deduped.append(result)
results = deduped
return results[:limit]
def call_chat_completion(
base_url: str,
model: str,
messages: list[dict],
*,
temperature: float = 0.2,
max_tokens: int = 1000,
timeout: int = 180,
) -> str:
url = f"{base_url.rstrip('/')}/v1/chat/completions"
payload = {
"model": model,
"messages": messages,
"temperature": temperature,
"max_tokens": max_tokens,
}
request = urllib.request.Request(
url,
data=json.dumps(payload, ensure_ascii=False).encode("utf-8"),
headers={"Content-Type": "application/json"},
method="POST",
)
with urllib.request.urlopen(request, timeout=timeout) as response:
data = json.loads(response.read().decode("utf-8"))
choices = data.get("choices") or []
if not choices:
raise ValueError("chat response has no choices")
message = choices[0].get("message") or {}
content = (message.get("content") or "").strip()
if not content:
raise ValueError("chat response content is empty")
return content
+249
View File
@@ -0,0 +1,249 @@
from __future__ import annotations
import argparse
import html
import json
import re
from datetime import datetime, timezone
from pathlib import Path
from typing import Any
ROOT = Path(__file__).resolve().parents[1]
DEFAULT_REPORT_ROOT = ROOT / "reports" / "1c-access"
def slugify(value: str, *, max_length: int = 80) -> str:
slug = re.sub(r"[^0-9A-Za-zА-Яа-яЁё._-]+", "-", value.strip())
slug = re.sub(r"-+", "-", slug).strip("-._")
return (slug or "role-audit")[:max_length]
def esc(value: Any) -> str:
return html.escape("" if value is None else str(value), quote=True)
def read_json(path: Path) -> dict[str, Any]:
data = json.loads(path.read_text(encoding="utf-8-sig"))
if not isinstance(data, dict):
raise ValueError(f"JSON root is not an object: {path}")
return data
def load_export(path: Path) -> dict[str, Any]:
data = read_json(path)
artifacts = data.get("artifacts") if isinstance(data.get("artifacts"), dict) else {}
export_path = artifacts.get("json")
if export_path:
return read_json(Path(export_path))
return data
def summary_role(summary: dict[str, Any]) -> str | None:
query = summary.get("query") if isinstance(summary.get("query"), dict) else {}
role = query.get("role") if isinstance(query, dict) else None
return str(role) if role is not None else None
def find_latest_summaries(report_root: Path, base_id: str, *, role: str | None = None, count: int = 2) -> list[Path]:
folder = report_root / slugify(base_id, max_length=60)
summaries: list[tuple[str, Path]] = []
role_filter = role.casefold() if role else None
for path in folder.glob("*.summary.json"):
try:
summary = read_json(path)
except (OSError, json.JSONDecodeError, ValueError):
continue
summary_role_value = summary_role(summary)
if role_filter is not None and (summary_role_value or "").casefold() != role_filter:
continue
generated_at = str(summary.get("generated_at") or "")
summaries.append((generated_at, path))
summaries.sort(key=lambda item: item[0], reverse=True)
return [path for _, path in summaries[:count]]
def user_key(row: dict[str, Any]) -> str:
value = row.get("user_id") or row.get("user_name")
return str(value or "").strip()
def row_user(row: dict[str, Any]) -> dict[str, Any]:
return {
"user_id": row.get("user_id"),
"user_name": row.get("user_name"),
"user_type": row.get("user_type"),
"user_active": row.get("user_active"),
"user_marked": row.get("user_marked"),
}
def group_rows_by_user(export: dict[str, Any]) -> dict[str, dict[str, Any]]:
users: dict[str, dict[str, Any]] = {}
rows = export.get("rows") if isinstance(export.get("rows"), list) else []
for row in rows:
if not isinstance(row, dict):
continue
key = user_key(row)
if not key:
continue
item = users.setdefault(key, {"user": row_user(row), "access_paths": set(), "rows": []})
if row.get("access_path"):
item["access_paths"].add(str(row.get("access_path")))
item["rows"].append(row)
for item in users.values():
item["access_paths"] = sorted(item["access_paths"])
return users
def compare_exports(old_export: dict[str, Any], new_export: dict[str, Any]) -> dict[str, Any]:
old_users = group_rows_by_user(old_export)
new_users = group_rows_by_user(new_export)
old_keys = set(old_users)
new_keys = set(new_users)
added_keys = sorted(new_keys - old_keys)
removed_keys = sorted(old_keys - new_keys)
common_keys = sorted(old_keys & new_keys)
changed_paths = [
{
"user": new_users[key]["user"],
"old_access_paths": old_users[key]["access_paths"],
"new_access_paths": new_users[key]["access_paths"],
}
for key in common_keys
if old_users[key]["access_paths"] != new_users[key]["access_paths"]
]
return {
"schema": "onec_access_role_audit_compare.v1",
"status": "ok",
"generated_at": datetime.now(timezone.utc).isoformat(),
"counts": {
"old_users": len(old_keys),
"new_users": len(new_keys),
"added_users": len(added_keys),
"removed_users": len(removed_keys),
"unchanged_users": len(common_keys),
"changed_access_paths": len(changed_paths),
},
"added_users": [new_users[key]["user"] for key in added_keys],
"removed_users": [old_users[key]["user"] for key in removed_keys],
"changed_access_paths": changed_paths,
}
def render_html_report(compare: dict[str, Any]) -> str:
counts = compare.get("counts") if isinstance(compare.get("counts"), dict) else {}
def user_rows(name: str) -> str:
users = compare.get(name) if isinstance(compare.get(name), list) else []
return "\n".join(
"<tr>"
f"<td>{esc(item.get('user_name'))}</td>"
f"<td>{esc(item.get('user_id'))}</td>"
f"<td>{esc(item.get('user_type'))}</td>"
f"<td>{esc(item.get('user_active'))}</td>"
f"<td>{esc(item.get('user_marked'))}</td>"
"</tr>"
for item in users
if isinstance(item, dict)
) or "<tr><td colspan=\"5\">Absent.</td></tr>"
changed = compare.get("changed_access_paths") if isinstance(compare.get("changed_access_paths"), list) else []
changed_rows = "\n".join(
"<tr>"
f"<td>{esc((item.get('user') or {}).get('user_name') if isinstance(item.get('user'), dict) else None)}</td>"
f"<td><pre>{esc('\\n'.join(item.get('old_access_paths') or []))}</pre></td>"
f"<td><pre>{esc('\\n'.join(item.get('new_access_paths') or []))}</pre></td>"
"</tr>"
for item in changed
if isinstance(item, dict)
) or "<tr><td colspan=\"3\">Absent.</td></tr>"
return f"""<!doctype html>
<html lang="ru">
<head>
<meta charset="utf-8">
<title>1C Access Audit Compare</title>
<style>
body {{ font-family: Segoe UI, Arial, sans-serif; margin: 24px; color: #1f2933; }}
.cards {{ display: flex; flex-wrap: wrap; gap: 12px; margin: 18px 0; }}
.card {{ border: 1px solid #d9e2ec; border-radius: 8px; padding: 12px 14px; min-width: 130px; }}
.card b {{ display: block; font-size: 20px; }}
table {{ border-collapse: collapse; width: 100%; margin: 12px 0 24px; }}
th, td {{ border: 1px solid #d9e2ec; padding: 8px; vertical-align: top; }}
th {{ background: #f0f4f8; text-align: left; }}
pre {{ white-space: pre-wrap; margin: 0; font-family: Consolas, monospace; }}
</style>
</head>
<body>
<h1>1C Access Audit Compare</h1>
<div>Generated: {esc(compare.get('generated_at'))}</div>
<div class="cards">
<div class="card"><b>{esc(counts.get('old_users'))}</b>old users</div>
<div class="card"><b>{esc(counts.get('new_users'))}</b>new users</div>
<div class="card"><b>{esc(counts.get('added_users'))}</b>added</div>
<div class="card"><b>{esc(counts.get('removed_users'))}</b>removed</div>
<div class="card"><b>{esc(counts.get('changed_access_paths'))}</b>path changes</div>
</div>
<h2>Added users</h2>
<table><thead><tr><th>User</th><th>ID</th><th>Type</th><th>Active</th><th>Marked</th></tr></thead><tbody>{user_rows('added_users')}</tbody></table>
<h2>Removed users</h2>
<table><thead><tr><th>User</th><th>ID</th><th>Type</th><th>Active</th><th>Marked</th></tr></thead><tbody>{user_rows('removed_users')}</tbody></table>
<h2>Changed access paths</h2>
<table><thead><tr><th>User</th><th>Old paths</th><th>New paths</th></tr></thead><tbody>{changed_rows}</tbody></table>
</body>
</html>
"""
def compare_files(old_path: Path, new_path: Path, *, output: Path | None = None, html_output: Path | None = None) -> dict[str, Any]:
result = compare_exports(load_export(old_path), load_export(new_path))
result["sources"] = {"old": str(old_path), "new": str(new_path)}
result["artifacts"] = {
**({"json": str(output)} if output is not None else {}),
**({"html": str(html_output)} if html_output is not None else {}),
}
if output is not None:
output.parent.mkdir(parents=True, exist_ok=True)
output.write_text(json.dumps(result, ensure_ascii=False, indent=2), encoding="utf-8")
if html_output is not None:
html_output.parent.mkdir(parents=True, exist_ok=True)
html_output.write_text(render_html_report(result), encoding="utf-8")
return result
def main() -> int:
parser = argparse.ArgumentParser(description="Compare two 1C access role audit exports or summaries.")
parser.add_argument("old", type=Path, nargs="?")
parser.add_argument("new", type=Path, nargs="?")
parser.add_argument("--latest", action="store_true", help="Compare the latest two summaries for a base/role.")
parser.add_argument("--base-id", default="upo_test")
parser.add_argument("--role")
parser.add_argument("--report-root", type=Path, default=DEFAULT_REPORT_ROOT)
parser.add_argument("--output", type=Path)
parser.add_argument("--html", type=Path)
args = parser.parse_args()
old_path = args.old
new_path = args.new
output = args.output
html_output = args.html
if args.latest:
latest = find_latest_summaries(args.report_root, args.base_id, role=args.role, count=2)
if len(latest) < 2:
parser.error("Not enough matching summaries for --latest; need at least two.")
new_path, old_path = latest[0], latest[1]
folder = args.report_root / slugify(args.base_id, max_length=60)
stem = f"compare-{old_path.stem.replace('.summary', '')}-{new_path.stem.replace('.summary', '')}"
output = output or folder / f"{stem}.json"
html_output = html_output or folder / f"{stem}.html"
if old_path is None or new_path is None:
parser.error("Either provide OLD and NEW paths or use --latest.")
result = compare_files(old_path, new_path, output=output, html_output=html_output)
print(json.dumps(result, ensure_ascii=False, indent=2))
return 0 if result.get("status") == "ok" else 1
if __name__ == "__main__":
raise SystemExit(main())
+633
View File
@@ -0,0 +1,633 @@
#!/usr/bin/env python3
"""Compare decoded SQL form semantics with exported 1C Form.xml semantics."""
from __future__ import annotations
import argparse
import json
import sys
from collections import Counter
from pathlib import Path
from typing import Any
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "plugins" / "1c"))
from parser.form_payload import enrich_form_common_semantic, form_common_semantic # noqa: E402
from parser.form_xml import decode_form_xml # noqa: E402
PROPERTY_NAME_ALIASES = {
"Action": "Действие",
"MainAttribute": "ОсновнойРеквизит",
"AutoEditMode": "АвтоРежимРедактирования",
"HeightInTableRows": "ВысотаВСтрокахТаблицы",
"RowSelectionMode": "РежимВыделенияСтроки",
"HorizontalLinesBWA": "ГоризонтальныеЛинии",
"VerticalLinesBWA": "ВертикальныеЛинии",
"UseAlternationRowColorBWA": "ЧередованиеЦветовСтрок",
"AutoInsertNewRow": "АвтоВставкаНовойСтроки",
"EnableStartDrag": "РазрешитьНачалоПеретаскивания",
"EnableDrag": "РазрешитьПеретаскивание",
"FileDragMode": "РежимПеретаскиванияФайлов",
"CommandBarLocation": "ПоложениеКоманднойПанели",
"DefaultItem": "АктивизироватьПоУмолчанию",
"Autofill": "Автозаполнение",
"AutoMaxWidth": "АвтоМаксимальнаяШирина",
"MaxWidth": "МаксимальнаяШирина",
"MultiLine": "МногострочныйРежим",
"AutoCommandBar": "АвтоКоманднаяПанель",
"SearchStringAddition": "ДополнениеСтрокиПоиска",
"ViewStatusAddition": "ДополнениеСостоянияПросмотра",
"SearchControlAddition": "ДополнениеУправленияПоиском",
"Width": "Ширина",
"Height": "Высота",
"HorizontalStretch": "РастягиватьПоГоризонтали",
"VerticalStretch": "РастягиватьПоВертикали",
"TextColor": "ЦветТекста",
"BackColor": "ЦветФона",
"HorizontalAlign": "ГоризонтальноеПоложениеВГруппе",
"AutoMaxHeight": "АвтоМаксимальнаяВысота",
"MaxHeight": "МаксимальнаяВысота",
"AutoMarkIncomplete": "АвтоОтметкаНезаполненного",
"ToolTipRepresentation": "ОтображениеПодсказки",
"SpinButton": "КнопкаРегулирования",
"Representation": "Отображение",
"DefaultButton": "КнопкаПоУмолчанию",
"OpenButton": "КнопкаОткрытия",
"CreateButton": "КнопкаСоздания",
"ChoiceHistoryOnInput": "ИсторияВыбораПриВводе",
"BorderColor": "ЦветРамки",
"ChangeRowSet": "ИзменятьСоставСтрок",
"ShowInHeader": "ОтображатьВШапке",
"AutoCellHeight": "АвтоВысотаЯчейки",
"SearchStringLocation": "ПоложениеСтрокиПоиска",
"ViewStatusLocation": "ПоложениеСостоянияПросмотра",
"SearchControlLocation": "ПоложениеУправленияПоиском",
"GroupHorizontalAlign": "ГоризонтальноеПоложениеВГруппе",
"GroupVerticalAlign": "ВертикальноеПоложениеВГруппе",
"ShapeRepresentation": "ОтображениеФигуры",
}
def read_json(path: Path) -> dict[str, Any]:
data = json.loads(path.read_text(encoding="utf-8-sig"))
if not isinstance(data, dict):
raise ValueError(f"{path} must contain a JSON object")
return data
def write_json(path: Path, data: dict[str, Any]) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(json.dumps(data, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
def scalar_text(value: Any) -> str:
if value is True:
return "true"
if value is False:
return "false"
if value is None:
return ""
return str(value).strip()
def comparable(value: Any) -> str:
text = scalar_text(value)
lowered = text.casefold()
if lowered in {"истина", "true", "1"}:
return "true"
if lowered in {"ложь", "false", "0"}:
return "false"
if lowered in {"таблица формы", "таблица", "динамический список"}:
return "table"
if lowered in {"кнопка", "кнопка командной панели", "commandbarbutton"}:
return "button"
if lowered in {"декорация надписи", "labeldecoration"}:
return "label_decoration"
if lowered in {"декорация картинки", "picturedecoration"}:
return "picture_decoration"
if lowered in {"поле переключателя", "radiobuttonfield"}:
return "radio_button_field"
if lowered in {"в дополнительном подменю", "inadditionalsubmenu"}:
return "in_additional_submenu"
if lowered in {"в командной панели", "incommandbar"}:
return "in_command_bar"
if lowered in {"поле", "checkboxfield", "поле флажка"}:
return "field"
if lowered in {"колонка динамического списка", "column", "колонка реквизита"}:
return "column"
if lowered in {"attribute", "реквизит формы"}:
return "attribute"
if lowered in {"event", "событие"}:
return "event"
if lowered in {"command", "команда формы"}:
return "command"
if lowered in {"searchstringaddition", "дополнение строки поиска"}:
return "search_string_addition"
if lowered in {"viewstatusaddition", "дополнение состояния просмотра"}:
return "view_status_addition"
if lowered in {"searchcontroladdition", "дополнение управления поиском"}:
return "search_control_addition"
return lowered
def canonical_property_name(value: Any) -> str:
text = str(value or "").strip()
return PROPERTY_NAME_ALIASES.get(text, text)
def semantic_properties(row: dict[str, Any]) -> dict[str, dict[str, Any]]:
result: dict[str, dict[str, Any]] = {}
for direct_name, key in (
("Имя", "name"),
("Идентификатор", "id"),
("Заголовок", "title"),
("ПутьКДанным", "path_to_data"),
("Вид", "type_name"),
("Обработчик", "handler"),
):
if key in row and row.get(key) not in {None, ""}:
result[direct_name] = {"name": direct_name, "value": row.get(key), "source": "decoded_direct"}
semantic = row.get("semantic") if isinstance(row.get("semantic"), dict) else {}
for group, props in (semantic.get("groups") or {}).items():
for prop in props or []:
if not isinstance(prop, dict):
continue
name = canonical_property_name(prop.get("name"))
if name and name not in result:
result[name] = {**prop, "group": group}
return result
def iter_sql_rows(form: dict[str, Any]) -> list[dict[str, Any]]:
profile = form.get("profile") if isinstance(form.get("profile"), dict) else {}
rows = []
for section in ("items", "attributes", "parameters", "commands", "tables", "command_bars", "events"):
for row in profile.get(section) or []:
if isinstance(row, dict):
rows.append({**row, "_profile_section": section})
for item in profile.get("items") or []:
if not isinstance(item, dict):
continue
for event in item.get("events") or []:
if not isinstance(event, dict):
continue
event_name = event.get("event_name") or event.get("name")
rows.append(
{
**event,
"name": event_name,
"type_name": "Event",
"_profile_section": "events",
"_event_owner": item.get("name"),
}
)
return rows
def row_key(row: dict[str, Any]) -> tuple[str, str]:
if row.get("id") not in {None, ""}:
return ("id", str(row.get("id")))
return ("name", str(row.get("name") or ""))
def xml_match_keys(xml_item: dict[str, Any]) -> list[tuple[str, str]]:
name = str(xml_item.get("name") or "")
item_id = str(xml_item.get("id") or "")
kind = str(xml_item.get("kind") or "")
keys: list[tuple[str, str]] = []
if kind in {"ExtendedTooltip", "Event", "Column"}:
if name:
keys.append(("name", name.casefold()))
return keys
if kind == "Button":
if name:
keys.append(("name", name.casefold()))
if item_id:
keys.append(("id", item_id))
return keys
if item_id:
keys.append(("id", item_id))
if name:
keys.append(("name", name.casefold()))
return keys
def index_rows(rows: list[dict[str, Any]], *, sections: set[str] | None = None) -> dict[tuple[str, str], dict[str, Any]]:
by_key: dict[tuple[str, str], dict[str, Any]] = {}
by_name: dict[str, dict[str, Any]] = {}
for row in rows:
if sections is not None and str(row.get("_profile_section") or "") not in sections:
continue
key = row_key(row)
if key[1]:
by_key.setdefault(key, row)
name = str(row.get("name") or "")
if name:
by_name.setdefault(name.casefold(), row)
for name, row in by_name.items():
by_key.setdefault(("name", name), row)
return by_key
def xml_sql_sections(xml_item: dict[str, Any]) -> set[str]:
kind = str(xml_item.get("kind") or "")
if kind == "Attribute":
return {"attributes"}
if kind == "Parameter":
return {"parameters"}
if kind == "Column":
return {"items", "tables"}
if kind == "Command":
return {"commands"}
if kind == "Button":
return {"items", "commands"}
if kind in {"CommandBar", "AutoCommandBar"}:
return {"items", "command_bars"}
if kind == "Event":
return {"events"}
return {"items", "commands", "command_bars"}
def xml_profile_from_context_form(form: dict[str, Any], *, max_items: int) -> dict[str, Any] | None:
path_text = str(form.get("form_xml_path") or "")
if not path_text:
structure = form.get("structure") if isinstance(form.get("structure"), dict) else {}
path_text = str(structure.get("form_xml_path") or "")
if not path_text:
return None
path = Path(path_text)
if not path.is_file():
return {"status": "missing_xml_file", "source": {"path": path_text}, "items": []}
profile = decode_form_xml(path, max_items=max_items)
profile["form"]["name"] = form.get("name")
profile["form"]["guid"] = form.get("uuid")
return profile
def matching_sql_form(xml_form: dict[str, Any], sql_forms: list[dict[str, Any]]) -> dict[str, Any] | None:
guid = str((xml_form.get("form") or {}).get("guid") or "").casefold()
name = str((xml_form.get("form") or {}).get("name") or "").casefold()
for form in sql_forms:
if guid and str(form.get("guid") or "").casefold() == guid:
return form
for form in sql_forms:
if name and str(form.get("name") or "").casefold() == name:
return form
return None
def compare_form(sql_form: dict[str, Any] | None, xml_profile: dict[str, Any], *, sample_limit: int) -> dict[str, Any]:
xml_items = [item for item in xml_profile.get("items") or [] if isinstance(item, dict)]
sql_rows = iter_sql_rows(sql_form or {})
sql_any_index = index_rows(sql_rows)
sql_indexes_by_sections: dict[tuple[str, ...], dict[tuple[str, str], dict[str, Any]]] = {}
counts = Counter()
property_route_counts: Counter[tuple[str, str, str, str, str, str]] = Counter()
samples: dict[str, list[dict[str, Any]]] = {
"matched": [],
"missing_sql_item": [],
"xml_only_property": [],
"mismatch": [],
"command_name_match": [],
"command_name_mismatch": [],
"matched_form_property": [],
"xml_only_form_property": [],
"form_mismatch": [],
}
sql_profile = (sql_form or {}).get("profile") if isinstance((sql_form or {}).get("profile"), dict) else {}
sql_form_semantic = sql_profile.get("form_semantic") if isinstance(sql_profile.get("form_semantic"), dict) else None
if sql_form_semantic is None:
sql_form_semantic = form_common_semantic(
[item for item in sql_profile.get("form_parameters") or [] if isinstance(item, dict)],
include_diagnostics=True,
)
enrich_form_common_semantic(
sql_form_semantic,
[item for item in sql_profile.get("items") or [] if isinstance(item, dict)],
)
xml_form = xml_profile.get("form") if isinstance(xml_profile.get("form"), dict) else {}
xml_form_semantic = xml_form.get("semantic") if isinstance(xml_form.get("semantic"), dict) else {}
sql_root_props = semantic_properties({"type_name": "Форма", "semantic": sql_form_semantic})
xml_root_props = semantic_properties({"type_name": "Форма", "semantic": xml_form_semantic})
form_property_routes = []
for name, xml_prop in xml_root_props.items():
sql_prop = sql_root_props.get(name)
if sql_prop is None:
counts["xml_only_form_properties"] += 1
if len(samples["xml_only_form_property"]) < sample_limit:
samples["xml_only_form_property"].append(
{"property": name, "xml_name": xml_prop.get("xml_name"), "xml_value": xml_prop.get("value")}
)
continue
if comparable(sql_prop.get("value")) != comparable(xml_prop.get("value")):
counts["form_mismatches"] += 1
if len(samples["form_mismatch"]) < sample_limit:
samples["form_mismatch"].append(
{
"property": name,
"xml_name": xml_prop.get("xml_name"),
"sql_value": sql_prop.get("value"),
"xml_value": xml_prop.get("value"),
"sql_source": sql_prop.get("source"),
}
)
continue
counts["matched_form_properties"] += 1
if len(samples["matched_form_property"]) < sample_limit:
samples["matched_form_property"].append(
{"property": name, "xml_name": xml_prop.get("xml_name"), "value": xml_prop.get("value")}
)
parameter_indices = sql_prop.get("parameter_indices")
if isinstance(parameter_indices, list) and parameter_indices:
form_property_routes.append(
{
"xml_name": xml_prop.get("xml_name") or name,
"property": name,
"parameter_indices": parameter_indices,
"sql_source": sql_prop.get("source"),
"write_shape": sql_prop.get("write_shape"),
}
)
for xml_item in xml_items:
wanted_sections = xml_sql_sections(xml_item)
section_key = tuple(sorted(wanted_sections))
sql_item_index = sql_indexes_by_sections.setdefault(section_key, index_rows(sql_rows, sections=wanted_sections))
keys = xml_match_keys(xml_item)
sql_row = None
if xml_item.get("kind") == "Column" and xml_item.get("additional_columns_table"):
logical_path = f"{xml_item.get('additional_columns_table')}.{xml_item.get('name')}"
sql_row = next((row for row in sql_rows if comparable(row.get("path_to_data")) == comparable(logical_path)), None)
if sql_row is not None:
counts["logical_column_sql_match"] += 1
if len(samples.setdefault("logical_column_sql_match", [])) < sample_limit:
samples["logical_column_sql_match"].append(
{
"name": xml_item.get("name"),
"id": xml_item.get("id"),
"logical_path": logical_path,
"sql_element": sql_row.get("name"),
"sql_path": sql_row.get("path"),
}
)
if xml_item.get("kind") == "Event" and xml_item.get("owner"):
sql_row = next(
(
row
for row in sql_rows
if str(row.get("_profile_section") or "") == "events"
and str(row.get("_event_owner") or "").casefold() == str(xml_item.get("owner") or "").casefold()
and str(row.get("name") or "").casefold() == str(xml_item.get("name") or "").casefold()
),
None,
)
if sql_row is None and not (xml_item.get("kind") == "Event" and xml_item.get("owner")):
sql_row = next((sql_item_index[key] for key in keys if key in sql_item_index), None)
if sql_row is not None and xml_item.get("kind") == "Column" and xml_item.get("additional_columns_table"):
counts["matched_items"] += 1
if len(samples["matched"]) < sample_limit:
samples["matched"].append(
{
"name": xml_item.get("name"),
"id": xml_item.get("id"),
"sql_path": sql_row.get("path"),
"sql_type": sql_row.get("type_name"),
"xml_kind": xml_item.get("kind_ru"),
"match_by": "additional_columns_data_path",
}
)
continue
if sql_row is None:
fallback = next((sql_any_index[key] for key in keys if key in sql_any_index), None)
if fallback is not None:
counts["non_item_sql_match"] += 1
if len(samples.setdefault("non_item_sql_match", [])) < sample_limit:
samples["non_item_sql_match"].append({"name": xml_item.get("name"), "id": xml_item.get("id"), "xml_kind": xml_item.get("kind"), "sql_section": fallback.get("_profile_section"), "sql_type": fallback.get("type_name")})
if sql_row is None:
counts["missing_sql_item"] += 1
if len(samples["missing_sql_item"]) < sample_limit:
samples["missing_sql_item"].append({"name": xml_item.get("name"), "id": xml_item.get("id"), "kind": xml_item.get("kind")})
continue
counts["matched_items"] += 1
if len(samples["matched"]) < sample_limit:
samples["matched"].append({"name": xml_item.get("name"), "id": xml_item.get("id"), "sql_path": sql_row.get("path"), "sql_type": sql_row.get("type_name"), "xml_kind": xml_item.get("kind_ru")})
sql_props = semantic_properties(sql_row)
xml_props = semantic_properties(xml_item)
sql_command = sql_props.get("ИмяКоманды")
xml_command = xml_props.get("ИмяКоманды")
if sql_command is not None and xml_command is not None:
sql_command_value = sql_command.get("value")
xml_command_value = xml_command.get("value")
command_row = {
"item": xml_item.get("name"),
"id": xml_item.get("id"),
"sql_value": sql_command_value,
"xml_value": xml_command_value,
"sql_path": sql_row.get("path"),
"sql_source": sql_command.get("source"),
}
if comparable(sql_command_value) == comparable(xml_command_value):
counts["command_name_matches"] += 1
if len(samples["command_name_match"]) < sample_limit:
samples["command_name_match"].append(command_row)
else:
counts["command_name_mismatches"] += 1
if len(samples["command_name_mismatch"]) < sample_limit:
samples["command_name_mismatch"].append(command_row)
for name, xml_prop in xml_props.items():
if name == "Идентификатор" and xml_item.get("kind") == "Button" and sql_row.get("_profile_section") == "commands":
continue
xml_value = xml_prop.get("value")
sql_prop = sql_props.get(name)
if sql_prop is None:
counts["xml_only_properties"] += 1
if len(samples["xml_only_property"]) < sample_limit:
samples["xml_only_property"].append({"item": xml_item.get("name"), "id": xml_item.get("id"), "property": name, "xml_value": xml_value, "xml_name": xml_prop.get("xml_name")})
continue
sql_value = sql_prop.get("value")
if comparable(sql_value) != comparable(xml_value):
counts["mismatches"] += 1
if len(samples["mismatch"]) < sample_limit:
samples["mismatch"].append({"item": xml_item.get("name"), "id": xml_item.get("id"), "property": name, "sql_value": sql_value, "xml_value": xml_value, "sql_source": sql_prop.get("source"), "xml_name": xml_prop.get("xml_name")})
else:
counts["matched_properties"] += 1
parameter_index = sql_prop.get("parameter_index")
if parameter_index is not None:
property_route_counts[
(
str(xml_item.get("kind") or ""),
str(xml_prop.get("xml_name") or name),
name,
str(sql_row.get("marker") or ""),
str(parameter_index),
str(sql_prop.get("source") or ""),
)
] += 1
property_routes = [
{
"xml_kind": key[0],
"xml_name": key[1],
"property": key[2],
"sql_marker": key[3] or None,
"parameter_index": int(key[4]) if key[4].lstrip("-").isdigit() else key[4],
"sql_source": key[5] or None,
"matches": match_count,
}
for key, match_count in sorted(property_route_counts.items(), key=lambda item: (-item[1], item[0]))
]
return {
"sql_form": {key: (sql_form or {}).get(key) for key in ("name", "guid", "source") if (sql_form or {}).get(key) is not None},
"xml_form": xml_profile.get("form"),
"xml_source": xml_profile.get("source"),
"counts": dict(counts),
"form_property_routes": form_property_routes,
"property_routes": property_routes,
"samples": samples,
}
def build_report(sql_details: dict[str, Any], xml_context: dict[str, Any], *, sample_limit: int, max_items: int) -> dict[str, Any]:
sql_forms = [form for form in sql_details.get("forms") or [] if isinstance(form, dict)]
comparisons = []
for form in xml_context.get("forms") or []:
if not isinstance(form, dict):
continue
profiles = []
seen_xml_paths: set[str] = set()
xml_profile = xml_profile_from_context_form(form, max_items=max_items)
if xml_profile is not None:
source_path = str((xml_profile.get("source") or {}).get("path") or "").casefold()
if source_path:
seen_xml_paths.add(source_path)
profiles.append(xml_profile)
for overlay in form.get("extension_overlays") or []:
if isinstance(overlay, dict):
overlay_profile = xml_profile_from_context_form(overlay, max_items=max_items)
if overlay_profile is not None:
source_path = str((overlay_profile.get("source") or {}).get("path") or "").casefold()
if source_path and source_path in seen_xml_paths:
continue
if source_path:
seen_xml_paths.add(source_path)
profiles.append(overlay_profile)
for profile in profiles:
comparisons.append(compare_form(matching_sql_form(profile, sql_forms), profile, sample_limit=sample_limit))
totals = Counter()
route_counts: Counter[tuple[str, str, str, str, str, str]] = Counter()
for comparison in comparisons:
totals.update(comparison.get("counts") or {})
for route in comparison.get("property_routes") or []:
route_counts[
(
str(route.get("xml_kind") or ""),
str(route.get("xml_name") or ""),
str(route.get("property") or ""),
str(route.get("sql_marker") or ""),
str(route.get("parameter_index") if route.get("parameter_index") is not None else ""),
str(route.get("sql_source") or ""),
)
] += int(route.get("matches") or 0)
grouped_routes: dict[tuple[str, str, str], list[tuple[tuple[str, str, str, str, str, str], int]]] = {}
for key, match_count in route_counts.items():
grouped_routes.setdefault(key[:3], []).append((key, match_count))
stable_property_routes = []
ambiguous_property_routes = []
for identity, variants in sorted(grouped_routes.items()):
rows = [
{
"xml_kind": key[0],
"xml_name": key[1],
"property": key[2],
"sql_marker": key[3] or None,
"parameter_index": int(key[4]) if key[4].lstrip("-").isdigit() else key[4],
"sql_source": key[5] or None,
"matches": match_count,
}
for key, match_count in sorted(variants, key=lambda item: (-item[1], item[0]))
]
if len(rows) == 1 and rows[0]["matches"] >= 2:
stable_property_routes.append(rows[0])
elif len(rows) > 1:
ambiguous_property_routes.append(
{"xml_kind": identity[0], "xml_name": identity[1], "property": identity[2], "routes": rows}
)
totals["property_routes"] = len(route_counts)
totals["stable_property_routes"] = len(stable_property_routes)
totals["ambiguous_property_routes"] = len(ambiguous_property_routes)
return {
"schema": "onec_form_sql_xml_comparison.v1",
"status": "ok",
"object": sql_details.get("object") or xml_context.get("object"),
"counts": {"forms_compared": len(comparisons), **dict(totals)},
"stable_property_routes": stable_property_routes,
"ambiguous_property_routes": ambiguous_property_routes,
"comparisons": comparisons,
}
def markdown_table_row(values: list[Any]) -> str:
return "| " + " | ".join(str(value).replace("\n", " ") for value in values) + " |"
def render_markdown(report: dict[str, Any]) -> str:
lines = ["# 1C Form SQL/XML Comparison", ""]
obj = report.get("object") or {}
lines.append(f"- Object: `{obj.get('kind')}.{obj.get('name')}`")
counts = report.get("counts") or {}
lines.append(f"- Forms compared: `{counts.get('forms_compared')}`")
lines.append(f"- Matched items: `{counts.get('matched_items', 0)}`")
lines.append(f"- Matched properties: `{counts.get('matched_properties', 0)}`")
lines.append(f"- XML-only properties: `{counts.get('xml_only_properties', 0)}`")
lines.append(f"- Mismatches: `{counts.get('mismatches', 0)}`")
lines.append(f"- CommandName matches: `{counts.get('command_name_matches', 0)}`")
lines.append(f"- CommandName mismatches: `{counts.get('command_name_mismatches', 0)}`")
lines.append("")
for comparison in report.get("comparisons") or []:
xml_form = comparison.get("xml_form") or {}
sql_form = comparison.get("sql_form") or {}
lines.append(f"## {xml_form.get('name') or sql_form.get('name')}")
lines.append("")
lines.append(markdown_table_row(["Metric", "Count"]))
lines.append(markdown_table_row(["---", "---:"]))
for key, value in sorted((comparison.get("counts") or {}).items()):
lines.append(markdown_table_row([key, value]))
for title, key in (("CommandName Matches", "command_name_match"), ("CommandName Mismatches", "command_name_mismatch"), ("XML-Only Properties", "xml_only_property"), ("Mismatches", "mismatch"), ("Missing SQL Items", "missing_sql_item")):
sample = (comparison.get("samples") or {}).get(key) or []
if not sample:
continue
lines.append("")
lines.append(f"### {title}")
lines.append("")
lines.append("```json")
lines.append(json.dumps(sample[:10], ensure_ascii=False, indent=2))
lines.append("```")
lines.append("")
return "\n".join(lines).rstrip() + "\n"
def main() -> int:
parser = argparse.ArgumentParser(description="Compare decoded SQL form semantics with Form.xml semantics.")
parser.add_argument("--sql-details", type=Path, required=True)
parser.add_argument("--xml-context", type=Path, required=True)
parser.add_argument("--output-json", type=Path, required=True)
parser.add_argument("--output-markdown", type=Path)
parser.add_argument("--sample-limit", type=int, default=20)
parser.add_argument("--max-items", type=int, default=5000)
args = parser.parse_args()
report = build_report(read_json(args.sql_details), read_json(args.xml_context), sample_limit=args.sample_limit, max_items=args.max_items)
write_json(args.output_json, report)
if args.output_markdown:
args.output_markdown.parent.mkdir(parents=True, exist_ok=True)
args.output_markdown.write_text(render_markdown(report), encoding="utf-8")
print(json.dumps({"schema": "onec_form_sql_xml_comparison_cli_summary.v1", "status": report["status"], "counts": report["counts"], "output_json": str(args.output_json), "output_markdown": str(args.output_markdown) if args.output_markdown else None}, ensure_ascii=False, indent=2))
return 0
if __name__ == "__main__":
raise SystemExit(main())
@@ -0,0 +1,265 @@
#!/usr/bin/env python3
"""Compare two 1C saved-state object reports in 1C object terms."""
from __future__ import annotations
import argparse
import hashlib
import importlib.util
import json
import sys
from pathlib import Path
from typing import Any
REPO_ROOT = Path(__file__).resolve().parents[1]
def load_json(path: Path) -> dict[str, Any]:
return json.loads(path.read_text(encoding="utf-8-sig"))
def write_json(path: Path, data: dict[str, Any]) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(json.dumps(data, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
def object_map(report: dict[str, Any]) -> dict[str, dict[str, Any]]:
items = ((report.get("agent_summary") or {}).get("object_changes") or [])
result: dict[str, dict[str, Any]] = {}
for item in items:
if not isinstance(item, dict):
continue
full_name = str(item.get("full_name") or "")
if full_name:
result[full_name] = item
return result
def system_map(report: dict[str, Any]) -> dict[str, dict[str, Any]]:
items = ((report.get("agent_summary") or {}).get("system_changes") or [])
result: dict[str, dict[str, Any]] = {}
for item in items:
if not isinstance(item, dict):
continue
key = f"{item.get('layer')}::{item.get('extension')}::{item.get('name')}"
result[key] = item
return result
def stable_fingerprint(value: Any) -> str:
encoded = json.dumps(value, ensure_ascii=False, sort_keys=True, separators=(",", ":")).encode("utf-8")
return hashlib.sha256(encoded).hexdigest()
def comparable_object(item: dict[str, Any]) -> dict[str, Any]:
return {
"full_name": item.get("full_name"),
"layer": item.get("layer"),
"extension": item.get("extension"),
"kind": item.get("kind"),
"kind_ru": item.get("kind_ru"),
"name": item.get("name"),
"synonym": item.get("synonym"),
"parts_count": item.get("parts_count"),
"text_diff_parts": item.get("text_diff_parts"),
"active_missing_parts": item.get("active_missing_parts"),
"added_terms": item.get("added_terms") or [],
"removed_terms": item.get("removed_terms") or [],
"parts": [
{
"file_name": part.get("file_name"),
"payload_role": part.get("payload_role"),
"active_exists": part.get("active_exists"),
"summary": part.get("summary"),
"delta_chars": part.get("delta_chars"),
}
for part in item.get("parts") or []
if isinstance(part, dict)
],
}
def terms_delta(before: list[Any], after: list[Any]) -> dict[str, list[Any]]:
before_set = {str(value) for value in before}
after_set = {str(value) for value in after}
return {
"added": [value for value in after if str(value) not in before_set],
"removed": [value for value in before if str(value) not in after_set],
}
def parts_delta(before: list[dict[str, Any]], after: list[dict[str, Any]]) -> dict[str, Any]:
before_by_name = {str(part.get("file_name")): part for part in before if part.get("file_name")}
after_by_name = {str(part.get("file_name")): part for part in after if part.get("file_name")}
before_names = set(before_by_name)
after_names = set(after_by_name)
changed = []
for name in sorted(before_names & after_names):
before_part = before_by_name[name]
after_part = after_by_name[name]
if stable_fingerprint(before_part) != stable_fingerprint(after_part):
changed.append({
"file_name": name,
"before": before_part,
"after": after_part,
})
return {
"added": [after_by_name[name] for name in sorted(after_names - before_names)],
"removed": [before_by_name[name] for name in sorted(before_names - after_names)],
"changed": changed,
}
def object_delta(before: dict[str, Any], after: dict[str, Any]) -> dict[str, Any]:
before_cmp = comparable_object(before)
after_cmp = comparable_object(after)
return {
"full_name": after.get("full_name") or before.get("full_name"),
"before_fingerprint": stable_fingerprint(before_cmp),
"after_fingerprint": stable_fingerprint(after_cmp),
"before": before_cmp,
"after": after_cmp,
"term_delta": {
"added_terms": terms_delta(before_cmp.get("added_terms") or [], after_cmp.get("added_terms") or []),
"removed_terms": terms_delta(before_cmp.get("removed_terms") or [], after_cmp.get("removed_terms") or []),
},
"part_delta": parts_delta(before_cmp.get("parts") or [], after_cmp.get("parts") or []),
}
def compact_object(item: dict[str, Any]) -> dict[str, Any]:
comparable = comparable_object(item)
comparable["fingerprint"] = stable_fingerprint(comparable)
return comparable
def compare_reports(before_path: Path, after_path: Path) -> dict[str, Any]:
before_path = before_path.resolve()
after_path = after_path.resolve()
before = load_json(before_path)
after = load_json(after_path)
before_objects = object_map(before)
after_objects = object_map(after)
before_names = set(before_objects)
after_names = set(after_objects)
changed = []
unchanged = []
for name in sorted(before_names & after_names):
before_fp = stable_fingerprint(comparable_object(before_objects[name]))
after_fp = stable_fingerprint(comparable_object(after_objects[name]))
if before_fp == after_fp:
unchanged.append(name)
else:
changed.append(object_delta(before_objects[name], after_objects[name]))
before_system = system_map(before)
after_system = system_map(after)
system_added = sorted(set(after_system) - set(before_system))
system_removed = sorted(set(before_system) - set(after_system))
system_changed = [
key
for key in sorted(set(before_system) & set(after_system))
if stable_fingerprint(before_system[key]) != stable_fingerprint(after_system[key])
]
return {
"schema": "onec_saved_state_object_report_delta.v1",
"before_report": str(before_path),
"after_report": str(after_path),
"database": after.get("database") or before.get("database"),
"objects": {
"added": [compact_object(after_objects[name]) for name in sorted(after_names - before_names)],
"removed": [compact_object(before_objects[name]) for name in sorted(before_names - after_names)],
"changed": changed,
"unchanged": unchanged,
},
"system_changes": {
"added": [after_system[key] for key in system_added],
"removed": [before_system[key] for key in system_removed],
"changed": [{"key": key, "before": before_system[key], "after": after_system[key]} for key in system_changed],
},
"counts": {
"objects_added": len(after_names - before_names),
"objects_removed": len(before_names - after_names),
"objects_changed": len(changed),
"objects_unchanged": len(unchanged),
"system_added": len(system_added),
"system_removed": len(system_removed),
"system_changed": len(system_changed),
},
"safety": {
"read_only": True,
"sql_write_performed": False,
"public_terms_are_1c_objects": True,
},
}
def render_markdown(delta: dict[str, Any]) -> str:
module_path = REPO_ROOT / "scripts" / "render_1c_saved_state_object_report_delta_markdown.py"
spec = importlib.util.spec_from_file_location("saved_state_delta_markdown", module_path)
if spec is None or spec.loader is None:
raise RuntimeError(f"Cannot load Markdown renderer: {module_path}")
module = importlib.util.module_from_spec(spec)
sys.modules[spec.name] = module
spec.loader.exec_module(module)
return module.render(delta)
def check_delta(delta_path: Path, check_output: Path) -> None:
module_path = REPO_ROOT / "scripts" / "check_1c_saved_state_object_report_delta.py"
spec = importlib.util.spec_from_file_location("saved_state_delta_check", module_path)
if spec is None or spec.loader is None:
raise RuntimeError(f"Cannot load delta checker: {module_path}")
module = importlib.util.module_from_spec(spec)
sys.modules[spec.name] = module
spec.loader.exec_module(module)
result = module.check_delta(delta_path)
module.write_json(check_output, result)
if not result.get("passed"):
raise RuntimeError(f"Saved-state delta check failed: {check_output}")
def main() -> int:
parser = argparse.ArgumentParser(description="Compare two 1C saved-state object reports.")
parser.add_argument("--before", type=Path, required=True)
parser.add_argument("--after", type=Path, required=True)
parser.add_argument("--output", type=Path)
parser.add_argument("--markdown-output", type=Path)
parser.add_argument("--skip-markdown", action="store_true")
parser.add_argument("--check-output", type=Path)
parser.add_argument("--skip-check", action="store_true")
args = parser.parse_args()
result = compare_reports(args.before, args.after)
markdown_output = args.markdown_output
if markdown_output is None and args.output and not args.skip_markdown:
markdown_output = args.output.with_suffix(".md")
if markdown_output and not args.skip_markdown:
markdown_output = markdown_output.resolve()
result["markdown"] = str(markdown_output)
check_output = args.check_output
if check_output is None and args.output and not args.skip_check:
check_output = args.output.with_name(f"{args.output.stem}-check.json")
if check_output and not args.skip_check:
check_output = check_output.resolve()
result["check"] = str(check_output)
if args.output:
write_json(args.output, result)
if markdown_output and not args.skip_markdown:
markdown_output.parent.mkdir(parents=True, exist_ok=True)
markdown_output.write_text(render_markdown(result), encoding="utf-8")
if args.output:
write_json(args.output, result)
if check_output and not args.skip_check:
if not args.output:
raise SystemExit("Use --output when delta check output is enabled.")
check_delta(args.output, check_output)
print(json.dumps({"output": str(args.output) if args.output else None, "schema": result["schema"], "counts": result["counts"]}, ensure_ascii=False))
return 0
if __name__ == "__main__":
raise SystemExit(main())
+461
View File
@@ -0,0 +1,461 @@
param(
[string]$Server = $env:ONEC_SQL_SERVER,
[string]$Database = $env:ONEC_SQL_DATABASE,
[string]$User = $env:ONEC_SQL_USER,
[string]$Password = $env:ONEC_SQL_PASSWORD,
[string]$BaseMetadataDir = "reports\1c-sql\upo\structured-metadata-all-kinds",
[string]$ExtensionGuidIndex = "reports\1c-sql\upo\xml-guid-index-extensions.json",
[string]$Output
)
Set-StrictMode -Version Latest
$ErrorActionPreference = "Stop"
if (-not $Server) { throw "Server is required. Use -Server or ONEC_SQL_SERVER." }
if (-not $Database) { throw "Database is required. Use -Database or ONEC_SQL_DATABASE." }
if (-not $User) { throw "User is required. Use -User or ONEC_SQL_USER." }
if (-not $Password) { throw "Password is required. Use -Password or ONEC_SQL_PASSWORD." }
function U {
param([string]$Base64)
return [System.Text.Encoding]::UTF8.GetString([Convert]::FromBase64String($Base64))
}
$ruExtension = U "0KDQsNGB0YjQuNGA0LXQvdC40LU="
$ruConfigObject = U "0J7QsdGK0LXQutGC0JrQvtC90YTQuNCz0YPRgNCw0YbQuNC4"
$ruForm = U "0KTQvtGA0LzQsA=="
$kindRu = @{
Configuration = (U "0JrQvtC90YTQuNCz0YPRgNCw0YbQuNGP")
CommonModule = (U "0J7QsdGJ0LjQudCc0L7QtNGD0LvRjA==")
CommonForm = (U "0J7QsdGJ0LDRj9Ck0L7RgNC80LA=")
CommonCommand = (U "0J7QsdGJ0LDRj9Ca0L7QvNCw0L3QtNCw")
CommonAttribute = (U "0J7QsdGJ0LjQudCg0LXQutCy0LjQt9C40YI=")
CommonTemplate = (U "0J7QsdGJ0LjQudCc0LDQutC10YI=")
CommonPicture = (U "0J7QsdGJ0LDRj9Ca0LDRgNGC0LjQvdC60LA=")
Catalog = (U "0KHQv9GA0LDQstC+0YfQvdC40Lo=")
Document = (U "0JTQvtC60YPQvNC10L3Rgg==")
DataProcessor = (U "0J7QsdGA0LDQsdC+0YLQutCw")
Report = (U "0J7RgtGH0LXRgg==")
InformationRegister = (U "0KDQtdCz0LjRgdGC0YDQodCy0LXQtNC10L3QuNC5")
AccumulationRegister = (U "0KDQtdCz0LjRgdGC0YDQndCw0LrQvtC/0LvQtdC90LjRjw==")
AccountingRegister = (U "0KDQtdCz0LjRgdGC0YDQkdGD0YXQs9Cw0LvRgtC10YDQuNC4")
CalculationRegister = (U "0KDQtdCz0LjRgdGC0YDQoNCw0YHRh9C10YLQsA==")
Enum = (U "0J/QtdGA0LXRh9C40YHQu9C10L3QuNC1")
Form = $ruForm
Template = (U "0JzQsNC60LXRgg==")
Role = (U "0KDQvtC70Yw=")
Subsystem = (U "0J/QvtC00YHQuNGB0YLQtdC80LA=")
ExchangePlan = (U "0J/Qu9Cw0L3QntCx0LzQtdC90LA=")
BusinessProcess = (U "0JHQuNC30L3QtdGB0J/RgNC+0YbQtdGB0YE=")
Task = (U "0JfQsNC00LDRh9Cw")
Constant = (U "0JrQvtC90YHRgtCw0L3RgtCw")
ChartOfCharacteristicTypes = (U "0J/Qu9Cw0L3QktC40LTQvtCy0KXQsNGA0LDQutGC0LXRgNC40YHRgtC40Lo=")
ChartOfAccounts = (U "0J/Qu9Cw0L3QodGH0LXRgtC+0LI=")
ChartOfCalculationTypes = (U "0J/Qu9Cw0L3QktC40LTQvtCy0KDQsNGB0YfQtdGC0LA=")
}
$folderKind = @{
CommonModules = "CommonModule"
CommonForms = "CommonForm"
CommonCommands = "CommonCommand"
CommonAttributes = "CommonAttribute"
CommonTemplates = "CommonTemplate"
CommonPictures = "CommonPicture"
Catalogs = "Catalog"
Documents = "Document"
DataProcessors = "DataProcessor"
Reports = "Report"
InformationRegisters = "InformationRegister"
AccumulationRegisters = "AccumulationRegister"
AccountingRegisters = "AccountingRegister"
CalculationRegisters = "CalculationRegister"
Enums = "Enum"
Forms = "Form"
Templates = "Template"
Roles = "Role"
Subsystems = "Subsystem"
ExchangePlans = "ExchangePlan"
BusinessProcesses = "BusinessProcess"
Tasks = "Task"
Constants = "Constant"
ChartsOfCharacteristicTypes = "ChartOfCharacteristicTypes"
ChartsOfAccounts = "ChartOfAccounts"
ChartsOfCalculationTypes = "ChartOfCalculationTypes"
}
function Convert-RefBytesToGuid {
param([byte[]]$Bytes)
if ($Bytes.Length -ne 16) { return $null }
$hex = ($Bytes | ForEach-Object { $_.ToString("x2") }) -join ""
return "{0}-{1}-{2}-{3}-{4}" -f $hex.Substring(24, 8), $hex.Substring(20, 4), $hex.Substring(16, 4), $hex.Substring(0, 4), $hex.Substring(4, 12)
}
function Convert-ToSha256Hex {
param([byte[]]$Bytes)
$sha = [System.Security.Cryptography.SHA256]::Create()
try {
return (($sha.ComputeHash($Bytes) | ForEach-Object { $_.ToString("x2") }) -join "")
}
finally {
$sha.Dispose()
}
}
function Join-ByteArrays {
param([object[]]$Rows)
$ordered = @($Rows | Sort-Object part_no)
$total = 0
foreach ($row in $ordered) { $total += $row.bytes.Length }
$buffer = [byte[]]::new($total)
$offset = 0
foreach ($row in $ordered) {
[Array]::Copy($row.bytes, 0, $buffer, $offset, $row.bytes.Length)
$offset += $row.bytes.Length
}
return $buffer
}
function New-Connection {
$connectionString = "Server=$Server;Database=$Database;User ID=$User;Password=$Password;Encrypt=False;TrustServerCertificate=True;MultipleActiveResultSets=True;Application Name=Codex 1C Saved State Object Compare;"
$connection = [System.Data.SqlClient.SqlConnection]::new($connectionString)
$connection.Open()
return $connection
}
function Read-TableFiles {
param(
[System.Data.SqlClient.SqlConnection]$Connection,
[string]$Table,
[string[]]$FileNames
)
$command = $Connection.CreateCommand()
if ($FileNames -and $FileNames.Count -gt 0) {
$placeholders = @()
for ($i = 0; $i -lt $FileNames.Count; $i++) {
$paramName = "@p$i"
$placeholders += $paramName
$null = $command.Parameters.Add($paramName, [System.Data.SqlDbType]::NVarChar, 512)
$command.Parameters[$paramName].Value = $FileNames[$i]
}
$command.CommandText = "SELECT FileName, PartNo, BinaryData FROM dbo.[$Table] WHERE FileName IN ($($placeholders -join ', ')) ORDER BY FileName, PartNo"
}
else {
$command.CommandText = "SELECT FileName, PartNo, BinaryData FROM dbo.[$Table] ORDER BY FileName, PartNo"
}
$reader = $null
$groups = @{}
try {
$reader = $command.ExecuteReader()
while ($reader.Read()) {
$fileName = [string]$reader.GetValue(0)
if (-not $groups.ContainsKey($fileName)) { $groups[$fileName] = @() }
$groups[$fileName] += [pscustomobject]@{
part_no = [int]$reader.GetValue(1)
bytes = [byte[]]$reader.GetValue(2)
}
}
}
finally {
if ($reader -ne $null) { $reader.Dispose() }
if ($command -ne $null) { $command.Dispose() }
}
$result = @{}
foreach ($fileName in $groups.Keys) {
$bytes = Join-ByteArrays @($groups[$fileName])
$result[$fileName] = [pscustomobject]@{
file_name = $fileName
chunks = @($groups[$fileName]).Count
bytes = $bytes.Length
sha256 = Convert-ToSha256Hex $bytes
}
}
return $result
}
function Read-Extensions {
param([System.Data.SqlClient.SqlConnection]$Connection)
$command = $Connection.CreateCommand()
$command.CommandText = "SELECT [_IDRRef], [_ExtName], [_ExtensionOrder], [_ExtensionUsePurpose], [_ExtensionScope] FROM dbo.[_ExtensionsInfo] ORDER BY [_ExtensionOrder], [_ExtName]"
$reader = $null
$map = @{}
try {
$reader = $command.ExecuteReader()
while ($reader.Read()) {
$bytes = [byte[]]$reader.GetValue(0)
$guid = Convert-RefBytesToGuid $bytes
$map[$guid] = [pscustomobject]@{
guid = $guid
name = [string]$reader.GetValue(1)
order = [decimal]$reader.GetValue(2)
use_purpose = [decimal]$reader.GetValue(3)
scope = [decimal]$reader.GetValue(4)
}
}
}
finally {
if ($reader -ne $null) { $reader.Dispose() }
if ($command -ne $null) { $command.Dispose() }
}
return $map
}
function Get-BaseObjectInfo {
param([string]$Guid)
$metadataRoot = [System.IO.Path]::GetFullPath($BaseMetadataDir)
if (-not (Test-Path -LiteralPath $metadataRoot)) { return $null }
$match = Get-ChildItem -LiteralPath $metadataRoot -Recurse -Filter *.json -File |
Select-String -SimpleMatch $Guid -List |
Select-Object -First 1
if (-not $match) { return $null }
$data = Get-Content -LiteralPath $match.Path -Raw -Encoding UTF8 | ConvertFrom-Json
$kind = [string]$data.kind
$name = [string]$data.identity.name
$synonym = $null
if ($data.identity.synonyms -and $data.identity.synonyms.ru) { $synonym = [string]$data.identity.synonyms.ru }
return [pscustomobject]@{
layer = "base"
kind = $kind
kind_ru = if ($kindRu.ContainsKey($kind)) { $kindRu[$kind] } else { $kind }
name = $name
synonym = $synonym
full_name = "$(if ($kindRu.ContainsKey($kind)) { $kindRu[$kind] } else { $kind }).$name"
guid = $Guid
evidence = [pscustomobject]@{
metadata_file = $match.Path
xml_file = $data.xml_file
}
}
}
function Get-ExtensionGuidMap {
$path = [System.IO.Path]::GetFullPath($ExtensionGuidIndex)
if (-not (Test-Path -LiteralPath $path)) { return @{} }
$data = Get-Content -LiteralPath $path -Raw -Encoding UTF8 | ConvertFrom-Json
return $data.guid_map
}
function Get-ExtensionObjectInfo {
param(
[string]$ExtensionName,
[string]$ObjectGuid,
[object]$GuidMap
)
$entry = $GuidMap.$ObjectGuid
$top = $null
if ($entry -and $entry.top_objects -and @($entry.top_objects).Count -gt 0) {
$top = @($entry.top_objects)[0]
}
$relativePath = if ($top) { [string]$top.relative_path } else { $null }
$parts = if ($relativePath) { $relativePath -split "[\\/]" } else { @() }
$parentKind = $null
$parentName = $null
$artifactKind = if ($top) { [string]$top.xml_kind } else { "ConfigCASObject" }
$artifactName = if ($top) { [string]$top.name } else { $ObjectGuid }
$synonym = if ($top -and $top.synonym) { [string]$top.synonym } else { $null }
for ($i = 0; $i -lt $parts.Count - 1; $i++) {
if ($folderKind.ContainsKey($parts[$i]) -and $parts[$i] -ne "Forms" -and $parts[$i] -ne "Templates") {
$parentKind = $folderKind[$parts[$i]]
if ($i + 1 -lt $parts.Count) { $parentName = $parts[$i + 1] }
break
}
}
$artifactKindRu = if ($kindRu.ContainsKey($artifactKind)) { $kindRu[$artifactKind] } else { $artifactKind }
$parentKindRu = if ($parentKind -and $kindRu.ContainsKey($parentKind)) { $kindRu[$parentKind] } else { $parentKind }
$fullName = if ($parentKind -and $artifactKind -eq "Form") {
"$ruExtension.$ExtensionName.$parentKindRu.$parentName.$ruForm.$artifactName"
}
elseif ($parentKind) {
"$ruExtension.$ExtensionName.$parentKindRu.$parentName.$artifactKindRu.$artifactName"
}
else {
"$ruExtension.$ExtensionName.$artifactKindRu.$artifactName"
}
return [pscustomobject]@{
layer = "extension"
extension = $ExtensionName
parent_kind = $parentKind
parent_kind_ru = $parentKindRu
parent_name = $parentName
kind = $artifactKind
kind_ru = $artifactKindRu
name = $artifactName
synonym = $synonym
full_name = $fullName
guid = $ObjectGuid
relative_path = $relativePath
evidence = [pscustomobject]@{
xml_path = if ($top) { [string]$top.path } else { $null }
}
}
}
function Split-ObjectPart {
param([string]$FileName)
if ($FileName -match '^([0-9a-fA-F-]{36})(\..+)?$') {
return [pscustomobject]@{ object_id = $Matches[1].ToLowerInvariant(); suffix = if ($Matches[2]) { $Matches[2] } else { "" } }
}
return $null
}
function Split-ExtensionFileName {
param([string]$FileName)
if ($FileName -match '^([0-9a-fA-F-]{36})__(.+)$') {
$rest = $Matches[2]
if ($rest -eq "configinfo") {
return [pscustomobject]@{ extension_id = $Matches[1].ToLowerInvariant(); object_id = $null; suffix = ""; system_name = "configinfo" }
}
$part = Split-ObjectPart $rest
return [pscustomobject]@{ extension_id = $Matches[1].ToLowerInvariant(); object_id = $part.object_id; suffix = $part.suffix; system_name = $null }
}
return $null
}
function New-StorageEvidence {
param(
[string]$SavedTable,
[string]$ActiveTable,
[string]$FileName,
[object]$Saved,
[object]$Active
)
return [pscustomobject]@{
saved_table = $SavedTable
active_table = $ActiveTable
file_name = $FileName
saved_bytes = $Saved.bytes
active_bytes = if ($Active) { $Active.bytes } else { $null }
saved_sha256 = $Saved.sha256
active_sha256 = if ($Active) { $Active.sha256 } else { $null }
active_exists = [bool]$Active
changed = (-not $Active) -or ($Saved.sha256 -ne $Active.sha256) -or ($Saved.bytes -ne $Active.bytes)
}
}
$connection = New-Connection
try {
$configSave = Read-TableFiles $connection "ConfigSave" $null
$configCassave = Read-TableFiles $connection "ConfigCASSave" $null
$configActive = Read-TableFiles $connection "Config" @($configSave.Keys)
$configCasActive = Read-TableFiles $connection "ConfigCAS" @($configCassave.Keys)
$extensions = Read-Extensions $connection
}
finally {
if ($connection -ne $null) { $connection.Dispose() }
}
$objectChangesByKey = @{}
$systemChanges = @()
foreach ($fileName in $configSave.Keys) {
$saved = $configSave[$fileName]
$active = if ($configActive.ContainsKey($fileName)) { $configActive[$fileName] } else { $null }
$evidence = New-StorageEvidence "ConfigSave" "Config" $fileName $saved $active
if (-not $evidence.changed) { continue }
$part = Split-ObjectPart $fileName
if (-not $part) {
$systemChanges += [pscustomobject]@{ layer = "base"; name = $fileName; storage = $evidence }
continue
}
$info = Get-BaseObjectInfo $part.object_id
if (-not $info) {
$info = [pscustomobject]@{
layer = "base"
kind = "ConfigObject"
kind_ru = $ruConfigObject
name = $part.object_id
synonym = $null
full_name = "$ruConfigObject.$($part.object_id)"
guid = $part.object_id
evidence = $null
}
}
$key = "base::$($part.object_id)"
if (-not $objectChangesByKey.ContainsKey($key)) {
$objectChangesByKey[$key] = [ordered]@{
layer = $info.layer
kind = $info.kind
kind_ru = $info.kind_ru
name = $info.name
synonym = $info.synonym
full_name = $info.full_name
guid = $info.guid
change_state = "saved_not_applied"
storage = @()
evidence = $info.evidence
}
}
$objectChangesByKey[$key].storage += $evidence
}
$extensionGuidMap = Get-ExtensionGuidMap
foreach ($fileName in $configCassave.Keys) {
$saved = $configCassave[$fileName]
$active = if ($configCasActive.ContainsKey($fileName)) { $configCasActive[$fileName] } else { $null }
$evidence = New-StorageEvidence "ConfigCASSave" "ConfigCAS" $fileName $saved $active
if (-not $evidence.changed) { continue }
$part = Split-ExtensionFileName $fileName
if (-not $part) {
$systemChanges += [pscustomobject]@{ layer = "extension"; name = $fileName; storage = $evidence }
continue
}
$extension = if ($extensions.ContainsKey($part.extension_id)) { $extensions[$part.extension_id] } else { $null }
$extensionName = if ($extension) { $extension.name } else { $part.extension_id }
if ($part.system_name) {
$systemChanges += [pscustomobject]@{ layer = "extension"; extension = $extensionName; name = $part.system_name; storage = $evidence }
continue
}
$info = Get-ExtensionObjectInfo $extensionName $part.object_id $extensionGuidMap
$key = "extension::$($part.extension_id)::$($part.object_id)"
if (-not $objectChangesByKey.ContainsKey($key)) {
$objectChangesByKey[$key] = [ordered]@{
layer = $info.layer
extension = $info.extension
parent_kind = $info.parent_kind
parent_kind_ru = $info.parent_kind_ru
parent_name = $info.parent_name
kind = $info.kind
kind_ru = $info.kind_ru
name = $info.name
synonym = $info.synonym
full_name = $info.full_name
guid = $info.guid
relative_path = $info.relative_path
change_state = "saved_not_applied"
storage = @()
evidence = $info.evidence
}
}
$objectChangesByKey[$key].storage += $evidence
}
$objectChanges = @($objectChangesByKey.Values | ForEach-Object { [pscustomobject]$_ } | Sort-Object layer, extension, full_name)
$result = [pscustomobject]@{
schema = "onec_saved_state_object_comparison.v1"
server = $Server
database = $Database
view = "saved_not_applied_vs_active"
object_changes = $objectChanges
system_changes = $systemChanges
counts = [pscustomobject]@{
object_changes = $objectChanges.Count
system_changes = $systemChanges.Count
config_save_files = $configSave.Count
config_cas_save_files = $configCassave.Count
}
safety = [pscustomobject]@{
read_only = $true
sql_write_performed = $false
exposes_storage_evidence = $true
public_terms_are_1c_objects = $true
}
}
$json = $result | ConvertTo-Json -Depth 20
if ($Output) {
$parent = Split-Path -Parent $Output
if ($parent) { New-Item -ItemType Directory -Force -Path $parent | Out-Null }
$json | Set-Content -LiteralPath $Output -Encoding UTF8
}
$json

Some files were not shown because too many files have changed in this diff Show More