Initial SQL-only 1C adapter baseline
This commit is contained in:
@@ -0,0 +1,83 @@
|
||||
# 1C SQL Parser Core
|
||||
|
||||
This package contains universal parser primitives for 1C SQL metadata storage.
|
||||
It must not hardcode object names, GUIDs, or table numbers from a concrete
|
||||
infobase.
|
||||
|
||||
## Modules
|
||||
|
||||
- `payload.py`: compression decoding, text decoding, and generic brace-tree
|
||||
parser.
|
||||
- `dbnames.py`: parser for `Params/DBNames*` files.
|
||||
- `extensions.py`: parser for `_ExtensionZippedInfo` blobs and extension root
|
||||
CAS manifests.
|
||||
- `config_object.py`: conservative identity extraction for top-level metadata
|
||||
object payloads.
|
||||
- `storage.py`: mechanical DBNames role to physical SQL name route helpers.
|
||||
- `config_sections.py`: mechanical section summaries for Config brace trees.
|
||||
- `child_records.py`: mechanical child-record boundary detection for section
|
||||
containers.
|
||||
- `xml_metadata.py`: small XML metadata extractor used as validation oracle.
|
||||
- `structured_metadata.py`: evidence-based projection from Config payloads to
|
||||
normalized metadata records.
|
||||
|
||||
## Current Guarantees
|
||||
|
||||
The parser can currently:
|
||||
|
||||
- decode raw-deflate Config payloads;
|
||||
- parse brace trees without semantic guesses;
|
||||
- read DBNames records as `{guid, storage_role, sql_number}`;
|
||||
- read extension root CAS keys from `_ExtensionZippedInfo`;
|
||||
- read extension manifest `object_id -> cas_key` entries.
|
||||
- extract top-level metadata identity when the observed identity block is
|
||||
present: GUID, name, localized synonyms, and evidence path.
|
||||
- map DBNames table-like roles to physical table-name candidates and field roles
|
||||
to physical column-name candidates.
|
||||
- summarize Config tree sections by path, shape, strings, and GUIDs without
|
||||
semantic labels.
|
||||
- map repeated object-kind sections to XML metadata categories by exact
|
||||
name/synonym/UUID evidence.
|
||||
- project proven sections into normalized metadata records with per-item
|
||||
evidence paths.
|
||||
- attach child metadata items to concrete section record paths when a declared
|
||||
child-record container is present.
|
||||
|
||||
## Non-Goals At This Layer
|
||||
|
||||
This layer does not know concrete configuration objects. For example, it does
|
||||
not know that a particular database has `Document.АвансовыйОтчет`.
|
||||
|
||||
Concrete infobase snapshots are built by applying this parser to SQL files and
|
||||
then resolving routes.
|
||||
|
||||
## Smoke Test
|
||||
|
||||
From repository root:
|
||||
|
||||
```powershell
|
||||
$env:PYTHONIOENCODING='utf-8'
|
||||
@'
|
||||
from pathlib import Path
|
||||
import sys, json
|
||||
sys.path.insert(0, str(Path('plugins/1c').resolve()))
|
||||
from parser.dbnames import parse_dbnames_file
|
||||
from parser.payload import parse_payload_file, root_signature
|
||||
from parser.storage import storage_routes
|
||||
|
||||
db = parse_dbnames_file(Path('reports/1c-sql/upo/Params/DBNames'))
|
||||
config = parse_payload_file(Path('reports/1c-sql/upo/Config-samples/84e4c0c3-2a21-4aba-a7b0-f92b3f2878ec'))
|
||||
print(len(db['records']), root_signature(config['tree']))
|
||||
print(storage_routes(db['records'][:1])[0])
|
||||
'@ | python -
|
||||
```
|
||||
|
||||
## Current Use
|
||||
|
||||
This package is a library layer for current adapter rebuild scripts. Normal
|
||||
agent work should not call these primitives directly; use the tools listed in
|
||||
`plugins/1c/tools/README.md`.
|
||||
|
||||
The latest adapter flow resolves objects by 1C names, then reads metadata,
|
||||
forms, modules, data views, and patch workspaces through the public scripts in
|
||||
`scripts/`.
|
||||
@@ -0,0 +1,52 @@
|
||||
"""Universal 1C SQL metadata parser primitives."""
|
||||
|
||||
from .payload import (
|
||||
BraceNode,
|
||||
Lexer,
|
||||
Parser,
|
||||
collect_strings,
|
||||
parse_brace_text,
|
||||
payload_to_text,
|
||||
try_decompress,
|
||||
)
|
||||
from .dbnames import DBNamesRecord, parse_dbnames_bytes, parse_dbnames_file
|
||||
from .extensions import (
|
||||
ExtensionZippedInfo,
|
||||
ManifestEntry,
|
||||
parse_extension_manifest_bytes,
|
||||
parse_extension_zipped_info,
|
||||
)
|
||||
from .config_object import MetadataObjectIdentity, find_identity, parse_config_object_file
|
||||
from .storage import StorageRoute, group_records_by_guid, storage_route, storage_routes
|
||||
from .config_sections import SectionSummary, summarize_section, summarize_sections
|
||||
from .xml_metadata import XmlMetadataItem, extract_xml_metadata_items, group_xml_items
|
||||
|
||||
__all__ = [
|
||||
"BraceNode",
|
||||
"Lexer",
|
||||
"Parser",
|
||||
"collect_strings",
|
||||
"parse_brace_text",
|
||||
"payload_to_text",
|
||||
"try_decompress",
|
||||
"DBNamesRecord",
|
||||
"parse_dbnames_bytes",
|
||||
"parse_dbnames_file",
|
||||
"ExtensionZippedInfo",
|
||||
"ManifestEntry",
|
||||
"parse_extension_manifest_bytes",
|
||||
"parse_extension_zipped_info",
|
||||
"MetadataObjectIdentity",
|
||||
"find_identity",
|
||||
"parse_config_object_file",
|
||||
"StorageRoute",
|
||||
"group_records_by_guid",
|
||||
"storage_route",
|
||||
"storage_routes",
|
||||
"SectionSummary",
|
||||
"summarize_section",
|
||||
"summarize_sections",
|
||||
"XmlMetadataItem",
|
||||
"extract_xml_metadata_items",
|
||||
"group_xml_items",
|
||||
]
|
||||
@@ -0,0 +1,271 @@
|
||||
"""Lightweight structural checks for 1C BSL text."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
import hashlib
|
||||
from typing import Any
|
||||
|
||||
|
||||
WORD = r"А-Яа-яA-Za-z0-9_"
|
||||
ROUTINE_START_RE = re.compile(r"(?im)^\s*(?:Асинх\s+)?(Процедура|Функция)\s+([A-Za-zА-Яа-яЁё_][\wА-Яа-яЁё]*)\s*\(")
|
||||
ROUTINE_END_RE = re.compile(r"(?im)^\s*(КонецПроцедуры|КонецФункции)\b")
|
||||
ROUTINE_RE = re.compile(rf"(?im)^\s*(?:Асинх\s+)?(Процедура|Функция)\s+([А-Яа-яA-Za-z_][{WORD}]*)\s*\(")
|
||||
END_RE = {
|
||||
"процедура": re.compile(rf"(?<![{WORD}])КонецПроцедуры(?![{WORD}])", re.IGNORECASE),
|
||||
"функция": re.compile(rf"(?<![{WORD}])КонецФункции(?![{WORD}])", re.IGNORECASE),
|
||||
}
|
||||
REGION_START_RE = re.compile(r"(?im)^\s*#Область\b")
|
||||
REGION_END_RE = re.compile(r"(?im)^\s*#КонецОбласти\b")
|
||||
PREPROC_IF_RE = re.compile(r"(?im)^\s*#Если\b")
|
||||
PREPROC_ENDIF_RE = re.compile(r"(?im)^\s*#КонецЕсли\b")
|
||||
|
||||
|
||||
def normalize_name(value: str | None) -> str:
|
||||
return re.sub(r"[\s._-]+", "", str(value or "")).casefold()
|
||||
|
||||
|
||||
def line_starts(text: str) -> list[int]:
|
||||
starts = [0]
|
||||
for match in re.finditer(r"\n", text):
|
||||
starts.append(match.end())
|
||||
return starts
|
||||
|
||||
|
||||
def offset_to_line(starts: list[int], offset: int) -> int:
|
||||
line = 1
|
||||
for index, start in enumerate(starts, start=1):
|
||||
if start > offset:
|
||||
break
|
||||
line = index
|
||||
return line
|
||||
|
||||
|
||||
def routine_blocks(text: str) -> list[dict[str, Any]]:
|
||||
blocks = []
|
||||
lines = text.splitlines(keepends=True)
|
||||
line_offsets: list[int] = []
|
||||
offset = 0
|
||||
for line in lines:
|
||||
line_offsets.append(offset)
|
||||
offset += len(line)
|
||||
for line_index, line in enumerate(lines):
|
||||
code = strip_line_comment(line)
|
||||
match = ROUTINE_RE.match(code)
|
||||
if not match:
|
||||
continue
|
||||
kind = match.group(1)
|
||||
name = match.group(2)
|
||||
declaration_start = line_offsets[line_index] + match.start()
|
||||
end = len(text)
|
||||
line_end = len(lines) or 1
|
||||
end_re = END_RE[kind.casefold()]
|
||||
for end_line_index in range(line_index + 1, len(lines)):
|
||||
end_code = strip_line_comment(lines[end_line_index])
|
||||
end_match = end_re.search(end_code)
|
||||
if end_match:
|
||||
end = line_offsets[end_line_index] + end_match.end()
|
||||
line_end = end_line_index + 1
|
||||
break
|
||||
blocks.append(
|
||||
{
|
||||
"kind": kind,
|
||||
"name": name,
|
||||
"normalized_name": normalize_name(name),
|
||||
"start": declaration_start,
|
||||
"declaration_start": declaration_start,
|
||||
"end": end,
|
||||
"line_start": line_index + 1,
|
||||
"line_end": line_end,
|
||||
}
|
||||
)
|
||||
return blocks
|
||||
|
||||
|
||||
def directive_start(text: str, declaration_start: int) -> int:
|
||||
prefix = text[:declaration_start]
|
||||
lines = prefix.splitlines(keepends=True)
|
||||
start_offset = len(prefix)
|
||||
index = len(lines) - 1
|
||||
while index >= 0:
|
||||
line = lines[index]
|
||||
stripped = line.strip()
|
||||
if stripped.startswith("&"):
|
||||
start_offset -= len(line)
|
||||
index -= 1
|
||||
continue
|
||||
if stripped == "":
|
||||
candidate = index - 1
|
||||
while candidate >= 0 and lines[candidate].strip() == "":
|
||||
candidate -= 1
|
||||
if candidate >= 0 and lines[candidate].strip().startswith("&"):
|
||||
start_offset -= len(line)
|
||||
index -= 1
|
||||
continue
|
||||
break
|
||||
return start_offset
|
||||
|
||||
|
||||
def one_routine_from_text(routine_text: str) -> dict[str, Any]:
|
||||
blocks = routine_blocks(routine_text)
|
||||
if len(blocks) != 1:
|
||||
raise ValueError(f"routine_text must contain exactly one procedure/function, found {len(blocks)}")
|
||||
block = blocks[0]
|
||||
if block["end"] < len(routine_text.rstrip()):
|
||||
suffix = routine_text[block["end"] :].strip()
|
||||
if suffix:
|
||||
raise ValueError("routine_text must not contain extra code after the routine end")
|
||||
return block
|
||||
|
||||
|
||||
def text_sha1(value: str) -> str:
|
||||
return hashlib.sha1(value.encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def dominant_eol(text: str) -> str:
|
||||
crlf = text.count("\r\n")
|
||||
without_crlf = text.replace("\r\n", "")
|
||||
lf = without_crlf.count("\n")
|
||||
cr = without_crlf.count("\r")
|
||||
if crlf >= lf and crlf >= cr and crlf > 0:
|
||||
return "\r\n"
|
||||
if cr > lf and cr > 0:
|
||||
return "\r"
|
||||
return "\n"
|
||||
|
||||
|
||||
def normalize_eol(text: str, eol: str) -> str:
|
||||
normalized = text.replace("\r\n", "\n").replace("\r", "\n")
|
||||
return normalized.replace("\n", eol)
|
||||
|
||||
|
||||
def replace_routine_text(
|
||||
text: str,
|
||||
routine_text: str,
|
||||
*,
|
||||
operation: str = "replace",
|
||||
name: str | None = None,
|
||||
expected_old_sha1: str | None = None,
|
||||
expected_old_contains: str | None = None,
|
||||
) -> tuple[str, dict[str, Any]]:
|
||||
if operation not in {"replace", "append", "upsert"}:
|
||||
raise ValueError("routine operation must be replace, append, or upsert")
|
||||
new_block = one_routine_from_text(routine_text)
|
||||
wanted = normalize_name(name or new_block["name"])
|
||||
blocks = routine_blocks(text)
|
||||
matches = [block for block in blocks if block["normalized_name"] == wanted]
|
||||
if len(matches) > 1:
|
||||
raise ValueError(f"target module has duplicate routine: {name or new_block['name']}")
|
||||
exists = bool(matches)
|
||||
if operation == "append" and exists:
|
||||
raise ValueError(f"routine already exists: {new_block['name']}")
|
||||
if operation == "replace" and not exists:
|
||||
raise ValueError(f"routine does not exist: {name or new_block['name']}")
|
||||
eol = dominant_eol(text)
|
||||
replacement = normalize_eol(routine_text.strip(), eol)
|
||||
if exists:
|
||||
old = matches[0]
|
||||
start = directive_start(text, int(old["declaration_start"]))
|
||||
end = int(old["end"])
|
||||
old_text = text[start:end]
|
||||
old_sha1 = text_sha1(old_text)
|
||||
if expected_old_sha1 and expected_old_sha1.lower() != old_sha1:
|
||||
raise ValueError("routine expected_old_sha1 does not match current routine text")
|
||||
if expected_old_contains and expected_old_contains not in old_text:
|
||||
raise ValueError("routine expected_old_contains was not found in current routine text")
|
||||
updated = text[:start] + replacement + text[end:]
|
||||
status = "replaced"
|
||||
span = {
|
||||
"old_line_start": old["line_start"],
|
||||
"old_line_end": old["line_end"],
|
||||
"old_sha1": old_sha1,
|
||||
}
|
||||
else:
|
||||
if expected_old_sha1 or expected_old_contains:
|
||||
raise ValueError("routine old preconditions require an existing routine")
|
||||
separator = eol + eol if text.strip() else ""
|
||||
updated = text.rstrip("\r\n") + separator + replacement + eol
|
||||
status = "appended"
|
||||
span = {}
|
||||
return updated, {
|
||||
"status": status,
|
||||
"routine": {"kind": new_block["kind"], "name": new_block["name"]},
|
||||
**span,
|
||||
}
|
||||
|
||||
|
||||
def strip_line_comment(line: str) -> str:
|
||||
in_string = False
|
||||
index = 0
|
||||
while index < len(line):
|
||||
char = line[index]
|
||||
if char == '"':
|
||||
if in_string and index + 1 < len(line) and line[index + 1] == '"':
|
||||
index += 2
|
||||
continue
|
||||
in_string = not in_string
|
||||
if not in_string and line[index : index + 2] == "//":
|
||||
return line[:index]
|
||||
index += 1
|
||||
return line
|
||||
|
||||
|
||||
def code_lines(text: str) -> list[str]:
|
||||
return [strip_line_comment(line) for line in text.replace("\r\n", "\n").replace("\r", "\n").split("\n")]
|
||||
|
||||
|
||||
def validate_bsl_text(text: str) -> dict[str, Any]:
|
||||
lines = code_lines(text)
|
||||
clean = "\n".join(lines)
|
||||
starts = ROUTINE_START_RE.findall(clean)
|
||||
ends = ROUTINE_END_RE.findall(clean)
|
||||
region_starts = REGION_START_RE.findall(clean)
|
||||
region_ends = REGION_END_RE.findall(clean)
|
||||
preproc_ifs = PREPROC_IF_RE.findall(clean)
|
||||
preproc_ends = PREPROC_ENDIF_RE.findall(clean)
|
||||
issues = []
|
||||
if len(starts) != len(ends):
|
||||
issues.append(
|
||||
{
|
||||
"severity": "error",
|
||||
"code": "routine_balance",
|
||||
"message": "Routine start/end count mismatch.",
|
||||
"starts": len(starts),
|
||||
"ends": len(ends),
|
||||
}
|
||||
)
|
||||
if len(region_starts) != len(region_ends):
|
||||
issues.append(
|
||||
{
|
||||
"severity": "warning",
|
||||
"code": "region_balance",
|
||||
"message": "Region start/end count mismatch.",
|
||||
"starts": len(region_starts),
|
||||
"ends": len(region_ends),
|
||||
}
|
||||
)
|
||||
if len(preproc_ifs) != len(preproc_ends):
|
||||
issues.append(
|
||||
{
|
||||
"severity": "warning",
|
||||
"code": "preprocessor_if_balance",
|
||||
"message": "Preprocessor #Если/#КонецЕсли count mismatch.",
|
||||
"starts": len(preproc_ifs),
|
||||
"ends": len(preproc_ends),
|
||||
}
|
||||
)
|
||||
return {
|
||||
"schema": "onec_bsl_structural_validation.v1",
|
||||
"status": "ok" if not any(issue["severity"] == "error" for issue in issues) else "error",
|
||||
"counts": {
|
||||
"lines": len(lines),
|
||||
"routine_starts": len(starts),
|
||||
"routine_ends": len(ends),
|
||||
"regions": len(region_starts),
|
||||
"region_ends": len(region_ends),
|
||||
"preprocessor_ifs": len(preproc_ifs),
|
||||
"preprocessor_ends": len(preproc_ends),
|
||||
},
|
||||
"routines_sample": [{"kind": kind, "name": name} for kind, name in starts[:80]],
|
||||
"issues": issues,
|
||||
}
|
||||
@@ -0,0 +1,303 @@
|
||||
"""Classify 1C Config/ConfigCAS payload parts without infobase-specific names."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import hashlib
|
||||
import re
|
||||
from typing import Any
|
||||
|
||||
from .payload import collect_strings, decode_payload_lossless, encode_text, parse_brace_text, root_signature, scalar
|
||||
|
||||
|
||||
BASE64_RE = re.compile(r"[A-Za-z0-9+/]{40,}={0,2}")
|
||||
BSL_MARKERS = ("&На", "Процедура ", "Функция ", "#Область", "#КонецОбласти")
|
||||
HTML_MARKERS = ("<!DOCTYPE", "<html", "<HTML", "<body", "<BODY")
|
||||
STREAM_HEADER_RE = re.compile(rb"\r\n([0-9a-f]{8}) ([0-9a-f]{8}) 7fffffff \r\n")
|
||||
|
||||
|
||||
def sha1_hex(data: bytes) -> str:
|
||||
return hashlib.sha1(data).hexdigest()
|
||||
|
||||
|
||||
def normalized_text_sha1(text: str) -> str:
|
||||
normalized = str(text or "").replace("\r\n", "\n").replace("\r", "\n")
|
||||
return hashlib.sha1(normalized.encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def decode_text(data: bytes) -> tuple[str | None, str | None]:
|
||||
if data.startswith(b"\xef\xbb\xbf"):
|
||||
try:
|
||||
return data.decode("utf-8-sig"), "utf-8-sig"
|
||||
except UnicodeDecodeError:
|
||||
pass
|
||||
candidates = ("utf-8-sig", "utf-8", "utf-16-le", "utf-16-be", "cp1251")
|
||||
best: tuple[str | None, str | None, int] = (None, None, -1)
|
||||
for encoding in candidates:
|
||||
try:
|
||||
text = data.decode(encoding)
|
||||
except UnicodeDecodeError:
|
||||
continue
|
||||
sample = text[:20000]
|
||||
marker_score = sum(500 for marker in (*BSL_MARKERS, *HTML_MARKERS) if marker in sample)
|
||||
printable = sum(1 for char in sample if char.isprintable() or char in "\r\n\t")
|
||||
score = printable + marker_score - sample.count("\x00") * 10
|
||||
if score > best[2]:
|
||||
best = (text, encoding, score)
|
||||
return best[0], best[1]
|
||||
|
||||
|
||||
def payload_markers(payload: bytes) -> list[str]:
|
||||
markers = []
|
||||
if payload.startswith(b"MOXCEL"):
|
||||
markers.append("MOXCEL")
|
||||
if payload.startswith(b"\xef\xbb\xbf") or b"\xef\xbb\xbf" in payload[:256]:
|
||||
markers.append("utf8_bom")
|
||||
if STREAM_HEADER_RE.search(payload):
|
||||
markers.append("stream_headers")
|
||||
return markers
|
||||
|
||||
|
||||
def extract_stream_blocks(payload: bytes, *, include_text: bool = False, limit: int = 100) -> list[dict[str, Any]]:
|
||||
blocks: list[dict[str, Any]] = []
|
||||
for match in STREAM_HEADER_RE.finditer(payload):
|
||||
declared_1 = int(match.group(1), 16)
|
||||
declared_2 = int(match.group(2), 16)
|
||||
start = match.end()
|
||||
size = declared_2
|
||||
if size <= 0 or start + size > len(payload):
|
||||
continue
|
||||
data = payload[start : start + size]
|
||||
text, encoding = decode_text(data)
|
||||
clean = (text or "").replace("\x00", "")
|
||||
item: dict[str, Any] = {
|
||||
"header_offset": match.start(),
|
||||
"data_offset": start,
|
||||
"declared_1": declared_1,
|
||||
"declared_2": declared_2,
|
||||
"bytes": len(data),
|
||||
"sha1": sha1_hex(data),
|
||||
"encoding": encoding,
|
||||
"text_preview": clean[:500],
|
||||
"has_bsl_marker": bool(text and any(marker in clean for marker in BSL_MARKERS)),
|
||||
"has_html_marker": bool(text and any(marker in clean for marker in HTML_MARKERS)),
|
||||
}
|
||||
if include_text:
|
||||
item["text"] = text
|
||||
blocks.append(item)
|
||||
if len(blocks) >= limit:
|
||||
break
|
||||
return blocks
|
||||
|
||||
|
||||
def stream_blocks_with_data(payload: bytes, *, limit: int = 100) -> list[dict[str, Any]]:
|
||||
blocks: list[dict[str, Any]] = []
|
||||
for match in STREAM_HEADER_RE.finditer(payload):
|
||||
declared_1 = int(match.group(1), 16)
|
||||
declared_2 = int(match.group(2), 16)
|
||||
start = match.end()
|
||||
size = declared_2
|
||||
if size <= 0 or start + size > len(payload):
|
||||
continue
|
||||
data = payload[start : start + size]
|
||||
text, encoding = decode_text(data)
|
||||
blocks.append(
|
||||
{
|
||||
"header_offset": match.start(),
|
||||
"header_end": match.end(),
|
||||
"data_offset": start,
|
||||
"data_end": start + size,
|
||||
"declared_1": declared_1,
|
||||
"declared_2": declared_2,
|
||||
"bytes": len(data),
|
||||
"sha1": sha1_hex(data),
|
||||
"encoding": encoding,
|
||||
"text": text,
|
||||
"data": data,
|
||||
}
|
||||
)
|
||||
if len(blocks) >= limit:
|
||||
break
|
||||
return blocks
|
||||
|
||||
|
||||
def stream_header(size: int) -> bytes:
|
||||
if size < 0 or size > 0xFFFFFFFF:
|
||||
raise ValueError("stream size is outside 8-hex header range")
|
||||
encoded = f"{size:08x}"
|
||||
return f"\r\n{encoded} {encoded} 7fffffff \r\n".encode("ascii")
|
||||
|
||||
|
||||
def replace_stream_block(
|
||||
payload: bytes,
|
||||
stream_index: int,
|
||||
*,
|
||||
text: str | None = None,
|
||||
data: bytes | None = None,
|
||||
replace: dict[str, Any] | None = None,
|
||||
routine: dict[str, Any] | None = None,
|
||||
expected_contains: str | None = None,
|
||||
expected_text_sha1: str | None = None,
|
||||
) -> tuple[bytes, dict[str, Any]]:
|
||||
blocks = stream_blocks_with_data(payload)
|
||||
if stream_index < 0 or stream_index >= len(blocks):
|
||||
raise IndexError(f"stream_index {stream_index} is outside {len(blocks)} stream blocks")
|
||||
block = blocks[stream_index]
|
||||
old_data = bytes(block["data"])
|
||||
old_text = block.get("text")
|
||||
encoding = block.get("encoding")
|
||||
if expected_contains and (old_text is None or expected_contains not in old_text):
|
||||
raise ValueError("expected_contains was not found in stream text")
|
||||
old_text_sha1 = normalized_text_sha1(old_text or "") if old_text is not None else None
|
||||
if expected_text_sha1 and (old_text_sha1 is None or expected_text_sha1.lower() != old_text_sha1):
|
||||
raise ValueError("expected_text_sha1 does not match current stream text")
|
||||
if replace is not None:
|
||||
if old_text is None:
|
||||
raise ValueError("stream text is not decodable")
|
||||
old = str(replace.get("old") or "")
|
||||
new = str(replace.get("new") or "")
|
||||
if not old:
|
||||
raise ValueError("replace.old is required")
|
||||
count = int(replace.get("count") or 1)
|
||||
if old not in old_text:
|
||||
raise ValueError("replace.old was not found in stream text")
|
||||
text = old_text.replace(old, new, count)
|
||||
routine_edit = None
|
||||
if routine is not None:
|
||||
if old_text is None:
|
||||
raise ValueError("stream text is not decodable")
|
||||
from .bsl_validation import replace_routine_text
|
||||
|
||||
text, routine_edit = replace_routine_text(
|
||||
old_text,
|
||||
str(routine.get("text") or ""),
|
||||
operation=str(routine.get("operation") or "replace"),
|
||||
name=str(routine.get("name")) if routine.get("name") else None,
|
||||
expected_old_sha1=str(routine.get("expected_old_sha1")) if routine.get("expected_old_sha1") else None,
|
||||
expected_old_contains=str(routine.get("expected_old_contains")) if routine.get("expected_old_contains") else None,
|
||||
)
|
||||
if data is None:
|
||||
if text is None:
|
||||
raise ValueError("text, data, replace, or routine is required")
|
||||
data = encode_text(text, encoding)
|
||||
header = stream_header(len(data))
|
||||
new_payload = payload[: block["header_offset"]] + header + data + payload[block["data_end"] :]
|
||||
return new_payload, {
|
||||
"stream_index": stream_index,
|
||||
"encoding": encoding,
|
||||
"old_sha1": sha1_hex(old_data),
|
||||
"new_sha1": sha1_hex(data),
|
||||
"old_text_sha1": old_text_sha1,
|
||||
"new_text_sha1": normalized_text_sha1(text) if text is not None else None,
|
||||
"old_bytes": len(old_data),
|
||||
"new_bytes": len(data),
|
||||
"old_text_preview": (old_text or "")[:500],
|
||||
"new_text_preview": (text or "")[:500] if text is not None else None,
|
||||
**({"routine": routine_edit} if routine_edit else {}),
|
||||
}
|
||||
|
||||
|
||||
def collect_base64_blocks(value: Any) -> list[str]:
|
||||
blocks: list[str] = []
|
||||
|
||||
def walk(node: Any) -> None:
|
||||
if isinstance(node, dict) and node.get("type") == "list":
|
||||
items = node.get("items") or []
|
||||
if items and scalar(items[0]) == "#base64":
|
||||
chunks = [scalar(item) for item in items[1:] if BASE64_RE.fullmatch(scalar(item))]
|
||||
if chunks:
|
||||
blocks.append("".join(chunks))
|
||||
for child in items:
|
||||
walk(child)
|
||||
|
||||
walk(value)
|
||||
return blocks
|
||||
|
||||
|
||||
def decode_base64_blocks(blocks: list[str], *, include_text: bool = False, limit: int = 50) -> list[dict[str, Any]]:
|
||||
result: list[dict[str, Any]] = []
|
||||
for value in blocks[:limit]:
|
||||
try:
|
||||
data = base64.b64decode(value, validate=True)
|
||||
except Exception:
|
||||
continue
|
||||
text, encoding = decode_text(data)
|
||||
clean = (text or "").replace("\x00", "")
|
||||
item: dict[str, Any] = {
|
||||
"block_length": len(value),
|
||||
"bytes": len(data),
|
||||
"sha1": sha1_hex(data),
|
||||
"encoding": encoding,
|
||||
"text_preview": clean[:500],
|
||||
"has_bsl_marker": bool(text and any(marker in clean for marker in BSL_MARKERS)),
|
||||
"has_html_marker": bool(text and any(marker in clean for marker in HTML_MARKERS)),
|
||||
}
|
||||
if include_text:
|
||||
item["text"] = text
|
||||
result.append(item)
|
||||
return result
|
||||
|
||||
|
||||
def classify_role(root_marker: str | None, markers: list[str], text: str | None, stream_blocks: list[dict[str, Any]], base64_blocks: list[dict[str, Any]]) -> str:
|
||||
clean = (text or "").replace("\x00", "")
|
||||
if root_marker == "1":
|
||||
return "metadata_payload"
|
||||
if root_marker == "4":
|
||||
return "form_payload"
|
||||
if root_marker == "5" or any(block.get("has_html_marker") for block in base64_blocks):
|
||||
return "help_or_html_payload"
|
||||
if root_marker == "8" or "MOXCEL" in markers:
|
||||
return "template_payload"
|
||||
if any(block.get("has_bsl_marker") for block in stream_blocks) or any(marker in clean for marker in BSL_MARKERS):
|
||||
return "bsl_module_payload"
|
||||
if "stream_headers" in markers:
|
||||
return "stream_container"
|
||||
if root_marker:
|
||||
return "brace_payload"
|
||||
return "binary_or_unknown_payload"
|
||||
|
||||
|
||||
def classify_payload(data: bytes, *, include_text: bool = False, include_tree: bool = False) -> dict[str, Any]:
|
||||
decoded = decode_payload_lossless(data)
|
||||
payload = decoded.get("payload") if isinstance(decoded.get("payload"), (bytes, bytearray)) else b""
|
||||
markers = payload_markers(bytes(payload))
|
||||
stream_blocks = extract_stream_blocks(bytes(payload), include_text=include_text)
|
||||
text = decoded.get("text")
|
||||
tree = None
|
||||
root = None
|
||||
base64_decoded: list[dict[str, Any]] = []
|
||||
strings_sample: list[str] = []
|
||||
if text and "{" in text and "stream_headers" not in markers:
|
||||
try:
|
||||
tree = parse_brace_text(text)
|
||||
root = root_signature(tree)
|
||||
strings_sample = collect_strings(tree, limit=80)
|
||||
base64_decoded = decode_base64_blocks(collect_base64_blocks(tree), include_text=include_text)
|
||||
except Exception:
|
||||
tree = None
|
||||
root_marker = root.get("root_marker") if isinstance(root, dict) else None
|
||||
result: dict[str, Any] = {
|
||||
"status": "ok" if payload else "undecodable",
|
||||
"compression": decoded.get("compression"),
|
||||
"encoding": decoded.get("encoding"),
|
||||
"raw_bytes": decoded.get("raw_bytes"),
|
||||
"payload_bytes": decoded.get("payload_bytes"),
|
||||
"sha1": sha1_hex(data),
|
||||
"payload_sha1": sha1_hex(bytes(payload)),
|
||||
"markers": markers,
|
||||
"root": root,
|
||||
"role": classify_role(root_marker, markers, text, stream_blocks, base64_decoded),
|
||||
"strings_sample": strings_sample,
|
||||
"stream_blocks": stream_blocks,
|
||||
"base64_blocks": base64_decoded,
|
||||
"counts": {
|
||||
"stream_blocks": len(stream_blocks),
|
||||
"base64_blocks": len(base64_decoded),
|
||||
"strings_sample": len(strings_sample),
|
||||
},
|
||||
}
|
||||
if include_text:
|
||||
result["text"] = text
|
||||
if include_tree:
|
||||
result["tree"] = tree
|
||||
return result
|
||||
@@ -0,0 +1,79 @@
|
||||
"""Mechanical child-record detection for 1C Config section containers."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import asdict, dataclass
|
||||
from typing import Any
|
||||
|
||||
from .payload import GUID_RE, scalar
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ChildRecord:
|
||||
index: int
|
||||
path: str
|
||||
node: Any
|
||||
evidence: dict[str, set[str]]
|
||||
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
data = asdict(self)
|
||||
data.pop("node", None)
|
||||
data["evidence"] = {key: sorted(value) for key, value in self.evidence.items()}
|
||||
return data
|
||||
|
||||
|
||||
def children(node: Any) -> list[Any]:
|
||||
if isinstance(node, dict) and node.get("type") in {"list", "sequence"}:
|
||||
return node.get("items") or []
|
||||
return []
|
||||
|
||||
|
||||
def collect_evidence(node: Any) -> dict[str, set[str]]:
|
||||
strings: set[str] = set()
|
||||
guids: set[str] = set()
|
||||
|
||||
def walk(value: Any) -> None:
|
||||
if isinstance(value, dict) and value.get("type") in {"atom", "string"}:
|
||||
text = scalar(value)
|
||||
if not text:
|
||||
return
|
||||
if value.get("type") == "string":
|
||||
strings.add(text)
|
||||
if GUID_RE.fullmatch(text):
|
||||
guids.add(text.lower())
|
||||
return
|
||||
for child in children(value):
|
||||
walk(child)
|
||||
|
||||
walk(node)
|
||||
return {"strings": strings, "guids": guids}
|
||||
|
||||
|
||||
def declared_child_records(section: Any, section_path: str, *, include_evidence: bool = True) -> list[ChildRecord]:
|
||||
"""Return records for the common `{marker, count, record...}` container.
|
||||
|
||||
The function is deliberately structural. It does not assume that records are
|
||||
attributes, dimensions, enum values, or any other metadata category.
|
||||
"""
|
||||
|
||||
items = children(section)
|
||||
if len(items) < 2:
|
||||
return []
|
||||
try:
|
||||
declared_count = int(scalar(items[1]))
|
||||
except ValueError:
|
||||
return []
|
||||
if declared_count < 0:
|
||||
return []
|
||||
candidates = items[2 : 2 + declared_count]
|
||||
if len(candidates) != declared_count:
|
||||
return []
|
||||
return [
|
||||
ChildRecord(
|
||||
index=index,
|
||||
path=f"{section_path}.{index + 2}",
|
||||
node=record,
|
||||
evidence=collect_evidence(record) if include_evidence else {"strings": set(), "guids": set()},
|
||||
)
|
||||
for index, record in enumerate(candidates)
|
||||
]
|
||||
@@ -0,0 +1,108 @@
|
||||
"""Conservative parser for top-level Config metadata object identity."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import asdict, dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from .payload import GUID_RE, parse_payload_file, root_signature, scalar
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class MetadataObjectIdentity:
|
||||
guid: str
|
||||
name: str
|
||||
synonyms: dict[str, str]
|
||||
evidence_path: str
|
||||
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
return asdict(self)
|
||||
|
||||
|
||||
def _is_guid(value: str) -> bool:
|
||||
return bool(GUID_RE.fullmatch(value))
|
||||
|
||||
|
||||
def _identity_guid(node: Any) -> str:
|
||||
if not (isinstance(node, dict) and node.get("type") == "list"):
|
||||
return ""
|
||||
items = node.get("items") or []
|
||||
if len(items) != 3:
|
||||
return ""
|
||||
if scalar(items[0]) != "1" or scalar(items[1]) != "0":
|
||||
return ""
|
||||
guid = scalar(items[2]).lower()
|
||||
return guid if _is_guid(guid) else ""
|
||||
|
||||
|
||||
def _synonyms(node: Any) -> dict[str, str]:
|
||||
if not (isinstance(node, dict) and node.get("type") == "list"):
|
||||
return {}
|
||||
items = node.get("items") or []
|
||||
if not items:
|
||||
return {}
|
||||
try:
|
||||
declared_count = int(scalar(items[0]))
|
||||
except ValueError:
|
||||
return {}
|
||||
if declared_count < 0 or len(items) < 1 + declared_count * 2:
|
||||
return {}
|
||||
result = {}
|
||||
index = 1
|
||||
end = 1 + declared_count * 2
|
||||
while index + 1 < end:
|
||||
language = scalar(items[index])
|
||||
value = scalar(items[index + 1])
|
||||
if language and value:
|
||||
result[language] = value
|
||||
index += 2
|
||||
return result
|
||||
|
||||
|
||||
def find_identity(tree: Any) -> MetadataObjectIdentity | None:
|
||||
"""Find the observed object identity block in a generic brace tree."""
|
||||
|
||||
def walk(node: Any, path: list[int]) -> MetadataObjectIdentity | None:
|
||||
if isinstance(node, dict) and node.get("type") == "list":
|
||||
items = node.get("items") or []
|
||||
for index in range(0, max(len(items) - 2, 0)):
|
||||
guid = _identity_guid(items[index])
|
||||
name = scalar(items[index + 1])
|
||||
synonym_node = items[index + 2]
|
||||
synonym_items = synonym_node.get("items") if isinstance(synonym_node, dict) and synonym_node.get("type") == "list" else None
|
||||
try:
|
||||
synonym_count = int(scalar(synonym_items[0])) if synonym_items else -1
|
||||
except ValueError:
|
||||
synonym_count = -1
|
||||
synonyms = _synonyms(synonym_node)
|
||||
if guid and name and synonym_count >= 0 and len(synonym_items or []) >= 1 + synonym_count * 2:
|
||||
return MetadataObjectIdentity(
|
||||
guid=guid,
|
||||
name=name,
|
||||
synonyms=synonyms,
|
||||
evidence_path=".".join(str(part) for part in [*path, index]),
|
||||
)
|
||||
for child_index, child in enumerate(items):
|
||||
found = walk(child, [*path, child_index])
|
||||
if found:
|
||||
return found
|
||||
return None
|
||||
|
||||
return walk(tree, [])
|
||||
|
||||
|
||||
def parse_config_object_file(path: Path) -> dict[str, Any]:
|
||||
payload = parse_payload_file(path)
|
||||
tree = payload.get("tree")
|
||||
identity = find_identity(tree)
|
||||
return {
|
||||
"source_path": str(path),
|
||||
"compression": payload.get("compression"),
|
||||
"encoding": payload.get("encoding"),
|
||||
"raw_bytes": payload.get("raw_bytes"),
|
||||
"payload_bytes": payload.get("payload_bytes"),
|
||||
"root": root_signature(tree),
|
||||
"identity": identity.to_dict() if identity else None,
|
||||
"tree": tree,
|
||||
}
|
||||
@@ -0,0 +1,84 @@
|
||||
"""Mechanical section summaries for Config brace trees.
|
||||
|
||||
This module intentionally does not name sections as attributes, tabular
|
||||
sections, forms, etc. It only reports paths, shapes, strings, and GUIDs so a
|
||||
higher-level validator can attach semantics using XML or other evidence.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import asdict, dataclass
|
||||
from typing import Any
|
||||
|
||||
from .payload import GUID_RE, collect_strings, scalar
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class SectionSummary:
|
||||
path: str
|
||||
node_type: str
|
||||
list_len: int | None
|
||||
first_scalars: list[str]
|
||||
string_count_sampled: int
|
||||
guid_count_sampled: int
|
||||
strings_sample: list[str]
|
||||
guids_sample: list[str]
|
||||
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
return asdict(self)
|
||||
|
||||
|
||||
def _children(node: Any) -> list[Any]:
|
||||
if isinstance(node, dict) and node.get("type") in {"list", "sequence"}:
|
||||
return node.get("items") or []
|
||||
return []
|
||||
|
||||
|
||||
def _atoms(node: Any, limit: int) -> list[str]:
|
||||
values: list[str] = []
|
||||
|
||||
def walk(value: Any) -> None:
|
||||
if len(values) >= limit:
|
||||
return
|
||||
if isinstance(value, dict) and value.get("type") in {"atom", "string"}:
|
||||
text = scalar(value)
|
||||
if text:
|
||||
values.append(text)
|
||||
return
|
||||
for child in _children(value):
|
||||
walk(child)
|
||||
|
||||
walk(node)
|
||||
return values
|
||||
|
||||
|
||||
def summarize_section(node: Any, path: str, *, limit: int = 200) -> SectionSummary:
|
||||
children = _children(node)
|
||||
atoms = _atoms(node, limit)
|
||||
strings = collect_strings(node, limit=limit)
|
||||
guids = sorted(set(value.lower() for value in atoms if GUID_RE.fullmatch(value)))
|
||||
return SectionSummary(
|
||||
path=path,
|
||||
node_type=node.get("type") if isinstance(node, dict) else type(node).__name__,
|
||||
list_len=len(children) if children else None,
|
||||
first_scalars=[scalar(child) for child in children[:12]],
|
||||
string_count_sampled=len(strings),
|
||||
guid_count_sampled=len(guids),
|
||||
strings_sample=strings[:50],
|
||||
guids_sample=guids[:50],
|
||||
)
|
||||
|
||||
|
||||
def summarize_sections(tree: Any, *, max_depth: int = 2, limit: int = 200) -> list[SectionSummary]:
|
||||
summaries: list[SectionSummary] = []
|
||||
|
||||
def walk(node: Any, path: list[int], depth: int) -> None:
|
||||
if depth > max_depth:
|
||||
return
|
||||
if path:
|
||||
summaries.append(summarize_section(node, ".".join(str(part) for part in path), limit=limit))
|
||||
for index, child in enumerate(_children(node)):
|
||||
walk(child, [*path, index], depth + 1)
|
||||
|
||||
walk(tree, [], 0)
|
||||
return summaries
|
||||
@@ -0,0 +1,421 @@
|
||||
"""Universal semantic profile helpers for 1C Config brace trees.
|
||||
|
||||
The rules here name repeatedly observed section paths, but every returned item
|
||||
keeps structural evidence. Runtime data still comes from the decoded Config
|
||||
tree and DBNames records of the requested base.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import asdict, dataclass
|
||||
from typing import Any
|
||||
|
||||
from .child_records import ChildRecord, collect_evidence, declared_child_records
|
||||
from .config_object import find_identity
|
||||
from .config_sections import summarize_sections
|
||||
from .payload import GUID_RE, root_signature, scalar
|
||||
from .structured_metadata import get_by_path
|
||||
|
||||
|
||||
SECTION_RULES: dict[str, list[dict[str, Any]]] = {
|
||||
"AccountingRegister": [
|
||||
{"path": "3", "category": "Dimension"},
|
||||
{"path": "5", "category": "Resource"},
|
||||
{"path": "7", "category": "Attribute"},
|
||||
],
|
||||
"AccumulationRegister": [
|
||||
{"path": "5", "category": "Resource"},
|
||||
{"path": "6", "category": "Attribute"},
|
||||
{"path": "7", "category": "Dimension"},
|
||||
],
|
||||
"BusinessProcess": [{"path": "6", "category": "Attribute"}],
|
||||
"Catalog": [
|
||||
{"path": "5", "category": "TabularSection"},
|
||||
{"path": "6", "category": "Attribute"},
|
||||
],
|
||||
"ChartOfAccounts": [
|
||||
{"path": "5", "category": "TabularSection"},
|
||||
{"path": "7", "category": "Attribute"},
|
||||
{"path": "8", "category": "AccountingFlag"},
|
||||
],
|
||||
"ChartOfCalculationTypes": [
|
||||
{"path": "3", "category": "TabularSection"},
|
||||
{"path": "4", "category": "Attribute"},
|
||||
],
|
||||
"CalculationRegister": [
|
||||
{"path": "3", "category": "Attribute"},
|
||||
{"path": "4", "category": "Recalculation"},
|
||||
{"path": "6", "category": "Resource"},
|
||||
{"path": "9", "category": "Dimension"},
|
||||
],
|
||||
"Document": [
|
||||
{"path": "3", "category": "TabularSection"},
|
||||
{"path": "5", "category": "Attribute"},
|
||||
],
|
||||
"Enum": [{"path": "6", "category": "EnumValue"}],
|
||||
"InformationRegister": [
|
||||
{"path": "3", "category": "Resource"},
|
||||
{"path": "4", "category": "Dimension"},
|
||||
{"path": "5", "category": "Attribute"},
|
||||
],
|
||||
"Report": [{"path": "4", "category": "Attribute"}],
|
||||
"Task": [
|
||||
{"path": "5", "category": "Attribute"},
|
||||
{"path": "6", "category": "AddressingAttribute"},
|
||||
{"path": "8", "category": "Command"},
|
||||
],
|
||||
}
|
||||
|
||||
ROLE_ROUTE_KIND = {
|
||||
"Fld": "field",
|
||||
"TabularSection": "tabular_section",
|
||||
"VT": "tabular_section",
|
||||
"EnumValue": "enum_value",
|
||||
"Dimension": "dimension",
|
||||
"Resource": "resource",
|
||||
"Document": "object",
|
||||
"Reference": "object",
|
||||
"Enum": "object",
|
||||
"InfoRg": "object",
|
||||
"AccumRg": "object",
|
||||
"AccRg": "object",
|
||||
"BPr": "object",
|
||||
"Task": "object",
|
||||
}
|
||||
|
||||
CATEGORY_ROUTE_KINDS = {
|
||||
"Attribute": {"field"},
|
||||
"AddressingAttribute": {"field"},
|
||||
"AccountingFlag": {"field"},
|
||||
"Column": {"field"},
|
||||
"Dimension": {"dimension", "field"},
|
||||
"Resource": {"resource", "field"},
|
||||
"TabularSection": {"tabular_section"},
|
||||
"EnumValue": {"enum_value"},
|
||||
}
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class DBNamesRoute:
|
||||
guid: str
|
||||
storage_role: str
|
||||
sql_number: int
|
||||
source: str
|
||||
route_kind: str
|
||||
physical_name_candidate: str | None
|
||||
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
return asdict(self)
|
||||
|
||||
|
||||
def _children(node: Any) -> list[Any]:
|
||||
if isinstance(node, dict) and node.get("type") in {"list", "sequence"}:
|
||||
return node.get("items") or []
|
||||
return []
|
||||
|
||||
|
||||
def _ordered_scalars(node: Any, *, limit: int = 200) -> list[str]:
|
||||
values: list[str] = []
|
||||
|
||||
def walk(value: Any) -> None:
|
||||
if len(values) >= limit:
|
||||
return
|
||||
text = scalar(value)
|
||||
if text:
|
||||
values.append(text)
|
||||
return
|
||||
for child in _children(value):
|
||||
walk(child)
|
||||
|
||||
walk(node)
|
||||
return values
|
||||
|
||||
|
||||
def _record_type(node: Any) -> dict[str, Any] | None:
|
||||
values = _ordered_scalars(node)
|
||||
try:
|
||||
index = values.index("Pattern")
|
||||
except ValueError:
|
||||
return None
|
||||
if index + 1 >= len(values):
|
||||
return None
|
||||
code = values[index + 1]
|
||||
result: dict[str, Any] = {"code": code}
|
||||
if code == "D":
|
||||
result.update({"kind": "date", "presentation": "Дата"})
|
||||
elif code == "B":
|
||||
result.update({"kind": "boolean", "presentation": "Булево"})
|
||||
elif code == "S":
|
||||
result.update({"kind": "string", "presentation": "Строка"})
|
||||
if index + 2 < len(values):
|
||||
try:
|
||||
length = int(values[index + 2])
|
||||
result["length"] = length
|
||||
if length > 0:
|
||||
result["presentation"] = f"Строка({length})"
|
||||
except ValueError:
|
||||
pass
|
||||
elif code == "N":
|
||||
result.update({"kind": "number", "presentation": "Число"})
|
||||
if index + 2 < len(values):
|
||||
try:
|
||||
precision = int(values[index + 2])
|
||||
result["precision"] = precision
|
||||
except ValueError:
|
||||
pass
|
||||
if index + 3 < len(values):
|
||||
try:
|
||||
scale = int(values[index + 3])
|
||||
result["scale"] = scale
|
||||
except ValueError:
|
||||
pass
|
||||
if "precision" in result:
|
||||
scale = result.get("scale")
|
||||
result["presentation"] = f"Число({result['precision']}, {scale})" if scale is not None else f"Число({result['precision']})"
|
||||
elif code == "#":
|
||||
result.update({"kind": "reference", "presentation": "Ссылка"})
|
||||
if index + 2 < len(values) and GUID_RE.fullmatch(values[index + 2]):
|
||||
result["type_guid"] = values[index + 2].lower()
|
||||
else:
|
||||
result.update({"kind": "unknown"})
|
||||
return result
|
||||
|
||||
|
||||
def _record_title(record: ChildRecord, *, include_samples: bool = True) -> dict[str, Any]:
|
||||
identity = find_identity(record.node)
|
||||
strings = sorted(record.evidence.get("strings") or [])
|
||||
guids = sorted(record.evidence.get("guids") or [])
|
||||
likely_name = identity.name if identity else next((value for value in strings if value and not GUID_RE.fullmatch(value)), None)
|
||||
result = {
|
||||
"index": record.index,
|
||||
"path": record.path,
|
||||
"identity": identity.to_dict() if identity else None,
|
||||
"likely_name": likely_name,
|
||||
"type": _record_type(record.node),
|
||||
}
|
||||
if include_samples:
|
||||
result.update(
|
||||
{
|
||||
"strings_sample": strings[:12],
|
||||
"guids_sample": guids[:12],
|
||||
"string_count": len(strings),
|
||||
"guid_count": len(guids),
|
||||
}
|
||||
)
|
||||
return result
|
||||
|
||||
|
||||
def dbnames_routes(records: list[Any] | None) -> dict[str, list[DBNamesRoute]]:
|
||||
result: dict[str, list[DBNamesRoute]] = {}
|
||||
for record in records or []:
|
||||
guid = str(getattr(record, "guid", "") or "").lower()
|
||||
if not guid:
|
||||
continue
|
||||
role = str(getattr(record, "storage_role", "") or "")
|
||||
sql_number = int(getattr(record, "sql_number", 0) or 0)
|
||||
route_kind = ROLE_ROUTE_KIND.get(role, "storage")
|
||||
physical = f"_{role}{sql_number}" if role and sql_number and route_kind != "object" else None
|
||||
result.setdefault(guid, []).append(
|
||||
DBNamesRoute(
|
||||
guid=guid,
|
||||
storage_role=role,
|
||||
sql_number=sql_number,
|
||||
source=str(getattr(record, "source", "") or ""),
|
||||
route_kind=route_kind,
|
||||
physical_name_candidate=physical,
|
||||
)
|
||||
)
|
||||
return result
|
||||
|
||||
|
||||
def _routes_for_evidence(
|
||||
evidence: dict[str, set[str]],
|
||||
routes_by_guid: dict[str, list[DBNamesRoute]],
|
||||
*,
|
||||
category: str | None = None,
|
||||
) -> list[dict[str, Any]]:
|
||||
routes: list[dict[str, Any]] = []
|
||||
seen: set[tuple[str, str, int]] = set()
|
||||
allowed_route_kinds = CATEGORY_ROUTE_KINDS.get(str(category or ""))
|
||||
for guid in sorted(evidence.get("guids") or []):
|
||||
for route in routes_by_guid.get(guid.lower(), []):
|
||||
if allowed_route_kinds and route.route_kind not in allowed_route_kinds:
|
||||
continue
|
||||
key = (route.guid, route.storage_role, route.sql_number)
|
||||
if key in seen:
|
||||
continue
|
||||
seen.add(key)
|
||||
routes.append(route.to_dict())
|
||||
return routes
|
||||
|
||||
|
||||
def _routes_for_record(
|
||||
record: ChildRecord,
|
||||
routes_by_guid: dict[str, list[DBNamesRoute]],
|
||||
*,
|
||||
category: str | None = None,
|
||||
) -> list[dict[str, Any]]:
|
||||
identity = find_identity(record.node)
|
||||
if identity:
|
||||
direct = _routes_for_evidence({"guids": {identity.guid}, "strings": set()}, routes_by_guid, category=category)
|
||||
if direct:
|
||||
return direct
|
||||
return _routes_for_evidence(record.evidence, routes_by_guid, category=category)
|
||||
|
||||
|
||||
def _section_profile(
|
||||
tree: Any,
|
||||
rule: dict[str, Any],
|
||||
routes_by_guid: dict[str, list[DBNamesRoute]],
|
||||
*,
|
||||
lightweight: bool = False,
|
||||
) -> dict[str, Any]:
|
||||
path = str(rule["path"])
|
||||
category = str(rule["category"])
|
||||
node = get_by_path(tree, path)
|
||||
if node is None:
|
||||
return {
|
||||
"path": path,
|
||||
"category": category,
|
||||
"status": "missing",
|
||||
"declared_record_count": 0,
|
||||
"records": [],
|
||||
}
|
||||
records = declared_child_records(node, path, include_evidence=not lightweight)
|
||||
section_evidence = collect_evidence(node) if not lightweight else {"strings": set(), "guids": set()}
|
||||
def record_profile(record: ChildRecord) -> dict[str, Any]:
|
||||
item = {
|
||||
**_record_title(record, include_samples=not lightweight),
|
||||
"storage_routes": [] if lightweight else _routes_for_record(record, routes_by_guid, category=category),
|
||||
}
|
||||
if category == "TabularSection":
|
||||
item["columns"] = _tabular_section_columns(record, routes_by_guid)
|
||||
return item
|
||||
|
||||
return {
|
||||
"path": path,
|
||||
"category": category,
|
||||
"status": "ok",
|
||||
"list_len": len(_children(node)),
|
||||
"declared_record_count": len(records),
|
||||
"storage_routes": _routes_for_evidence(section_evidence, routes_by_guid, category=category),
|
||||
"records": [record_profile(record) for record in records],
|
||||
}
|
||||
|
||||
|
||||
def _nested_record_containers(node: Any, path: str, *, max_depth: int = 5, include_root: bool = False) -> list[list[ChildRecord]]:
|
||||
containers: list[list[ChildRecord]] = []
|
||||
|
||||
def walk(value: Any, current_path: str, depth: int) -> None:
|
||||
records = declared_child_records(value, current_path)
|
||||
if records and (include_root or depth > 0):
|
||||
containers.append(records)
|
||||
if depth >= max_depth:
|
||||
return
|
||||
for index, child in enumerate(_children(value)):
|
||||
walk(child, f"{current_path}.{index}", depth + 1)
|
||||
|
||||
walk(node, path, 0)
|
||||
return containers
|
||||
|
||||
|
||||
def _container_score(records: list[ChildRecord]) -> tuple[int, int, int]:
|
||||
titles = [_record_title(record) for record in records]
|
||||
identities = sum(1 for title in titles if title.get("identity"))
|
||||
names = sum(1 for title in titles if title.get("likely_name"))
|
||||
return identities, names, len(records)
|
||||
|
||||
|
||||
def _tabular_section_columns(record: ChildRecord, routes_by_guid: dict[str, list[DBNamesRoute]]) -> list[dict[str, Any]]:
|
||||
containers = _nested_record_containers(record.node, record.path, include_root=False)
|
||||
candidates = [records for records in containers if len(records) >= 1]
|
||||
if not candidates:
|
||||
return []
|
||||
best = max(candidates, key=_container_score)
|
||||
columns = []
|
||||
for column_record in best:
|
||||
title = _record_title(column_record)
|
||||
if not title.get("likely_name") and not title.get("identity"):
|
||||
continue
|
||||
columns.append(
|
||||
{
|
||||
**title,
|
||||
"storage_routes": _routes_for_record(column_record, routes_by_guid, category="Column"),
|
||||
}
|
||||
)
|
||||
return columns
|
||||
|
||||
|
||||
def _generic_record_containers(tree: Any, *, max_depth: int = 3, limit: int = 200) -> list[dict[str, Any]]:
|
||||
containers: list[dict[str, Any]] = []
|
||||
|
||||
def walk(node: Any, path: list[int], depth: int) -> None:
|
||||
if len(containers) >= limit:
|
||||
return
|
||||
current_path = ".".join(str(part) for part in path)
|
||||
records = declared_child_records(node, current_path)
|
||||
if records:
|
||||
containers.append(
|
||||
{
|
||||
"path": current_path,
|
||||
"declared_record_count": len(records),
|
||||
"record_paths_sample": [record.path for record in records[:20]],
|
||||
}
|
||||
)
|
||||
if depth >= max_depth:
|
||||
return
|
||||
for index, child in enumerate(_children(node)):
|
||||
walk(child, [*path, index], depth + 1)
|
||||
|
||||
walk(tree, [], 0)
|
||||
return containers
|
||||
|
||||
|
||||
def decode_config_semantic(
|
||||
tree: Any,
|
||||
*,
|
||||
kind: str | None = None,
|
||||
dbnames_records: list[Any] | None = None,
|
||||
max_depth: int = 3,
|
||||
section_sample_limit: int = 200,
|
||||
include_generic: bool = True,
|
||||
categories: set[str] | list[str] | tuple[str, ...] | None = None,
|
||||
lightweight: bool = False,
|
||||
) -> dict[str, Any]:
|
||||
"""Return a structured, evidence-first profile for a Config object tree."""
|
||||
|
||||
identity = find_identity(tree)
|
||||
routes_by_guid = dbnames_routes(dbnames_records)
|
||||
object_routes = routes_by_guid.get((identity.guid if identity else "").lower(), [])
|
||||
wanted_categories = {str(category) for category in (categories or [])}
|
||||
rules = [
|
||||
rule
|
||||
for rule in SECTION_RULES.get(str(kind or ""), [])
|
||||
if not wanted_categories or str(rule.get("category") or "") in wanted_categories
|
||||
]
|
||||
sections = [_section_profile(tree, rule, routes_by_guid, lightweight=lightweight) for rule in rules]
|
||||
generic_sections = [item.to_dict() for item in summarize_sections(tree, max_depth=max_depth, limit=section_sample_limit)] if include_generic else []
|
||||
generic_record_containers = _generic_record_containers(tree, max_depth=max_depth) if include_generic else []
|
||||
return {
|
||||
"schema": "onec_config_semantic_profile.v1",
|
||||
"kind": kind,
|
||||
"root": root_signature(tree),
|
||||
"identity": identity.to_dict() if identity else None,
|
||||
"object_storage_routes": [route.to_dict() for route in object_routes],
|
||||
"section_rules": [
|
||||
{
|
||||
**rule,
|
||||
"source": "built_in_observed_rules",
|
||||
"note": "Rule names an observed section path; returned records are decoded from the current live Config payload.",
|
||||
}
|
||||
for rule in rules
|
||||
],
|
||||
"sections": sections,
|
||||
"generic_sections": generic_sections,
|
||||
"generic_record_containers": generic_record_containers,
|
||||
"counts": {
|
||||
"sections": len(sections),
|
||||
"ok_sections": sum(1 for section in sections if section.get("status") == "ok"),
|
||||
"generic_record_containers": len(generic_record_containers),
|
||||
},
|
||||
}
|
||||
@@ -0,0 +1,79 @@
|
||||
"""Parser for Params/DBNames files."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import asdict, dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from .payload import parse_brace_text, payload_to_text, scalar
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class DBNamesRecord:
|
||||
guid: str
|
||||
storage_role: str
|
||||
sql_number: int
|
||||
index: int
|
||||
source: str
|
||||
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
return asdict(self)
|
||||
|
||||
|
||||
def _unwrap_bom_sequence(value: Any) -> Any:
|
||||
if (
|
||||
isinstance(value, dict)
|
||||
and value.get("type") == "sequence"
|
||||
and len(value.get("items") or []) == 2
|
||||
and isinstance(value["items"][0], dict)
|
||||
and value["items"][0].get("type") == "atom"
|
||||
and str(value["items"][0].get("value") or "").strip("\ufeff") == ""
|
||||
):
|
||||
return value["items"][1]
|
||||
return value
|
||||
|
||||
|
||||
def parse_dbnames_bytes(data: bytes, *, source: str = "DBNames") -> dict[str, Any]:
|
||||
decoded = payload_to_text(data)
|
||||
text = decoded.get("text")
|
||||
if text is None:
|
||||
raise ValueError(f"{source}: cannot decode DBNames text")
|
||||
parsed = _unwrap_bom_sequence(parse_brace_text(text))
|
||||
if not (isinstance(parsed, dict) and parsed.get("type") == "list"):
|
||||
raise ValueError(f"{source}: expected root list")
|
||||
items = parsed.get("items") or []
|
||||
if len(items) != 2:
|
||||
raise ValueError(f"{source}: expected 2 root items, got {len(items)}")
|
||||
records_node = items[1]
|
||||
if not (isinstance(records_node, dict) and records_node.get("type") == "list"):
|
||||
raise ValueError(f"{source}: expected records list")
|
||||
record_items = records_node.get("items") or []
|
||||
records: list[DBNamesRecord] = []
|
||||
for index, node in enumerate(record_items[1:], start=1):
|
||||
if not (isinstance(node, dict) and node.get("type") == "list"):
|
||||
continue
|
||||
fields = node.get("items") or []
|
||||
if len(fields) != 3:
|
||||
continue
|
||||
records.append(
|
||||
DBNamesRecord(
|
||||
guid=scalar(fields[0]).lower(),
|
||||
storage_role=scalar(fields[1]),
|
||||
sql_number=int(scalar(fields[2])),
|
||||
index=index,
|
||||
source=source,
|
||||
)
|
||||
)
|
||||
return {
|
||||
"source": source,
|
||||
"compression": decoded["compression"],
|
||||
"encoding": decoded["encoding"],
|
||||
"root_number": int(scalar(items[0])),
|
||||
"declared_count": int(scalar(record_items[0]) or "0") if record_items else 0,
|
||||
"records": records,
|
||||
}
|
||||
|
||||
|
||||
def parse_dbnames_file(path: Path) -> dict[str, Any]:
|
||||
return parse_dbnames_bytes(path.read_bytes(), source=path.name)
|
||||
@@ -0,0 +1,107 @@
|
||||
"""Parsers for extension root package pointers and CAS manifests."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import re
|
||||
from dataclasses import asdict, dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from .payload import GUID_RE, parse_brace_text, payload_to_text, scalar
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ExtensionZippedInfo:
|
||||
marker_hex: str
|
||||
root_cas_key: str
|
||||
text_fragment: str
|
||||
guids: list[str]
|
||||
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
return asdict(self)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ManifestEntry:
|
||||
object_id: str
|
||||
cas_key: str
|
||||
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
return asdict(self)
|
||||
|
||||
|
||||
def _extract_utf16le_fragment(data: bytes) -> str:
|
||||
starts = [pos for pos in (data.find(b"{\x00"), data.find(b'"\x00#\x00"\x00')) if pos >= 0]
|
||||
if not starts:
|
||||
return ""
|
||||
start = min(starts)
|
||||
fragment = data[start:]
|
||||
if len(fragment) % 2:
|
||||
fragment = fragment[:-1]
|
||||
return fragment.decode("utf-16-le", errors="ignore").strip("\x00")
|
||||
|
||||
|
||||
def parse_extension_zipped_info(data: bytes) -> ExtensionZippedInfo:
|
||||
text = _extract_utf16le_fragment(data)
|
||||
return ExtensionZippedInfo(
|
||||
marker_hex=data[:4].hex(),
|
||||
root_cas_key=data[4:24].hex() if len(data) >= 24 else "",
|
||||
text_fragment=text,
|
||||
guids=sorted(set(match.lower() for match in GUID_RE.findall(text))),
|
||||
)
|
||||
|
||||
|
||||
def _base64_to_sha1(value: str) -> str | None:
|
||||
if not re.fullmatch(r"[A-Za-z0-9+/]+={0,2}", value):
|
||||
return None
|
||||
try:
|
||||
data = base64.b64decode(value, validate=True)
|
||||
except Exception:
|
||||
return None
|
||||
return data.hex() if len(data) == 20 else None
|
||||
|
||||
|
||||
def parse_extension_manifest_bytes(data: bytes) -> dict[str, Any]:
|
||||
decoded = payload_to_text(data)
|
||||
text = decoded.get("text")
|
||||
if text is None:
|
||||
raise ValueError("cannot decode extension manifest")
|
||||
parsed = parse_brace_text(text.lstrip("ï»¿п»ї"))
|
||||
if not (isinstance(parsed, dict) and parsed.get("type") == "sequence"):
|
||||
raise ValueError("expected extension root manifest sequence")
|
||||
items = parsed.get("items") or []
|
||||
if len(items) == 4 and scalar(items[0]) in {"", "п»ї"}:
|
||||
items = items[1:]
|
||||
if len(items) < 3:
|
||||
raise ValueError("expected at least 3 sequence items")
|
||||
|
||||
payload_block = items[1]
|
||||
manifest_block = items[2]
|
||||
extension_guid = ""
|
||||
if isinstance(payload_block, dict) and payload_block.get("type") == "list":
|
||||
block_items = payload_block.get("items") or []
|
||||
if len(block_items) > 1:
|
||||
extension_guid = scalar(block_items[1]).lower()
|
||||
|
||||
manifest_items = manifest_block.get("items") if isinstance(manifest_block, dict) else []
|
||||
declared_count = int(scalar(manifest_items[0]) or "0") if manifest_items else 0
|
||||
entries: list[ManifestEntry] = []
|
||||
for index in range(1, len(manifest_items or []), 2):
|
||||
object_id = scalar(manifest_items[index])
|
||||
encoded_key = scalar(manifest_items[index + 1]) if index + 1 < len(manifest_items) else ""
|
||||
cas_key = _base64_to_sha1(encoded_key)
|
||||
if object_id and cas_key:
|
||||
entries.append(ManifestEntry(object_id=object_id.lower(), cas_key=cas_key))
|
||||
|
||||
return {
|
||||
"compression": decoded["compression"],
|
||||
"encoding": decoded["encoding"],
|
||||
"extension_configuration_guid": extension_guid,
|
||||
"declared_count": declared_count,
|
||||
"entries": entries,
|
||||
}
|
||||
|
||||
|
||||
def parse_extension_manifest_file(path: Path) -> dict[str, Any]:
|
||||
return parse_extension_manifest_bytes(path.read_bytes())
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,403 @@
|
||||
"""Semantic profiles for 1C managed form XML exports."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import xml.etree.ElementTree as ET
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
FORM_XML_KIND_RU = {
|
||||
"Form": "Форма",
|
||||
"Event": "Событие",
|
||||
"Attribute": "Реквизит формы",
|
||||
"Column": "Колонка реквизита",
|
||||
"Command": "Команда формы",
|
||||
"AutoCommandBar": "Командная панель",
|
||||
"Button": "Кнопка",
|
||||
"ButtonGroup": "Группа кнопок",
|
||||
"CommandBar": "Командная панель",
|
||||
"Popup": "Подменю",
|
||||
"Pages": "Страницы",
|
||||
"ColumnGroup": "Группа колонок",
|
||||
"LabelField": "Поле надписи",
|
||||
"InputField": "Поле ввода",
|
||||
"CheckBoxField": "Поле флажка",
|
||||
"Table": "Таблица",
|
||||
"TableColumn": "Колонка таблицы",
|
||||
"Page": "Страница",
|
||||
"UsualGroup": "Группа",
|
||||
"DecorativeLabel": "Декорация надпись",
|
||||
"ContextMenu": "Контекстное меню",
|
||||
"ExtendedTooltip": "Расширенная подсказка",
|
||||
"SearchStringAddition": "Дополнение строки поиска",
|
||||
"ViewStatusAddition": "Дополнение состояния просмотра",
|
||||
"SearchControlAddition": "Дополнение управления поиском",
|
||||
}
|
||||
|
||||
FORM_XML_PROPERTY_NAMES = {
|
||||
"name": "Имя",
|
||||
"id": "Идентификатор",
|
||||
"Title": "Заголовок",
|
||||
"Type": "Вид",
|
||||
"DataPath": "ПутьКДанным",
|
||||
"TitleLocation": "ПоложениеЗаголовка",
|
||||
"Visible": "Видимость",
|
||||
"UserVisible": "ПользовательскаяВидимость",
|
||||
"Enabled": "Доступность",
|
||||
"ReadOnly": "ТолькоПросмотр",
|
||||
"SkipOnInput": "ПропускатьПриВводе",
|
||||
"DefaultItem": "АктивизироватьПоУмолчанию",
|
||||
"Importance": "ВажностьПриОтображении",
|
||||
"ServerUnavailabilityBehavior": "ПоведениеПриНедоступностиОсновногоСервера",
|
||||
"PasswordMode": "РежимПароля",
|
||||
"Hyperlink": "Гиперссылка",
|
||||
"CommandName": "ИмяКоманды",
|
||||
"WindowOpeningMode": "РежимОткрытияОкна",
|
||||
"AutoSaveDataInSettings": "АвтоСохранениеДанныхВНастройках",
|
||||
"CommandBarLocation": "ПоложениеКоманднойПанели",
|
||||
"ShowCommandBar": "ОтображатьКоманднуюПанель",
|
||||
"AutoCommandBar": "АвтоКоманднаяПанель",
|
||||
"SearchStringAddition": "ДополнениеСтрокиПоиска",
|
||||
"ViewStatusAddition": "ДополнениеСостоянияПросмотра",
|
||||
"SearchControlAddition": "ДополнениеУправленияПоиском",
|
||||
"Representation": "Отображение",
|
||||
"ChangeRowSet": "ИзменятьСоставСтрок",
|
||||
"RowSelectionMode": "РежимВыделенияСтроки",
|
||||
"HorizontalLinesBWA": "ГоризонтальныеЛинии",
|
||||
"VerticalLinesBWA": "ВертикальныеЛинии",
|
||||
"UseAlternationRowColorBWA": "ЧередованиеЦветовСтрок",
|
||||
"AutoInsertNewRow": "АвтоВставкаНовойСтроки",
|
||||
"EnableStartDrag": "РазрешитьНачалоПеретаскивания",
|
||||
"EnableDrag": "РазрешитьПеретаскивание",
|
||||
"RowFilter": "ОтборСтрок",
|
||||
"HeightInTableRows": "ВысотаВСтрокахТаблицы",
|
||||
"Footer": "Подвал",
|
||||
"FileDragMode": "РежимПеретаскиванияФайлов",
|
||||
"SearchStringLocation": "ПоложениеСтрокиПоиска",
|
||||
"ViewStatusLocation": "ПоложениеСостоянияПросмотра",
|
||||
"SearchControlLocation": "ПоложениеУправленияПоиском",
|
||||
"ButtonGroup": "СоставКоманд",
|
||||
"Group": "Группировка",
|
||||
"Behavior": "Поведение",
|
||||
"ShowTitle": "ПоказыватьЗаголовок",
|
||||
"Autofill": "Автозаполнение",
|
||||
"EditWarningRepresentation": "ОтображениеПредупрежденияПриРедактировании",
|
||||
"EditWarning": "ПредупреждениеПриРедактировании",
|
||||
"EditMode": "РежимРедактирования",
|
||||
"AutoEditMode": "АвтоРежимРедактирования",
|
||||
"AutoCellHeight": "АвтоВысотаЯчейки",
|
||||
"FixInTable": "ФиксацияВТаблице",
|
||||
"Shortcut": "СочетаниеКлавиш",
|
||||
"CommandSet": "СоставКоманд",
|
||||
"UseCopy": "ИспользоватьКопирование",
|
||||
"ShowInHeader": "ОтображатьВШапке",
|
||||
"ShowInFooter": "ОтображатьВПодвале",
|
||||
"ToolTip": "Подсказка",
|
||||
"ToolTipRepresentation": "ОтображениеПодсказки",
|
||||
"DropListButton": "КнопкаВыпадающегоСписка",
|
||||
"ChoiceButton": "КнопкаВыбора",
|
||||
"ClearButton": "КнопкаОчистки",
|
||||
"SpinButton": "КнопкаРегулирования",
|
||||
"OpenButton": "КнопкаОткрытия",
|
||||
"CreateButton": "КнопкаСоздания",
|
||||
"QuickChoice": "БыстрыйВыбор",
|
||||
"ChooseType": "ВыбиратьТип",
|
||||
"ChoiceList": "СписокВыбора",
|
||||
"IncompleteChoiceMode": "РежимВыбораНезаполненного",
|
||||
"TextEdit": "РедактированиеТекста",
|
||||
"TextEditUpdate": "ОбновлениеТекстаРедактирования",
|
||||
"MultiLine": "МногострочныйРежим",
|
||||
"AutoLineBreak": "АвтоПереносСтрок",
|
||||
"AutoMarkIncomplete": "АвтоОтметкаНезаполненного",
|
||||
"AutoChoiceIncomplete": "АвтоВыборНезаполненного",
|
||||
"InputHint": "ПодсказкаВвода",
|
||||
"ChoiceHistoryOnInput": "ИсторияВыбораПриВводе",
|
||||
"Picture": "Картинка",
|
||||
"HeaderPicture": "КартинкаШапки",
|
||||
"FooterPicture": "КартинкаПодвала",
|
||||
"BackColor": "ЦветФона",
|
||||
"TextColor": "ЦветТекста",
|
||||
"BorderColor": "ЦветРамки",
|
||||
"Font": "Шрифт",
|
||||
"Shape": "Фигура",
|
||||
"ShapeRepresentation": "ОтображениеФигуры",
|
||||
"PictureLocation": "ПоложениеКартинки",
|
||||
"HeaderBackColor": "ЦветФонаЗаголовка",
|
||||
"Width": "Ширина",
|
||||
"Height": "Высота",
|
||||
"AutoMaxWidth": "АвтоМаксимальнаяШирина",
|
||||
"AutoMaxHeight": "АвтоМаксимальнаяВысота",
|
||||
"MaxWidth": "МаксимальнаяШирина",
|
||||
"MaxHeight": "МаксимальнаяВысота",
|
||||
"TitleHeight": "ВысотаЗаголовка",
|
||||
"HorizontalAlign": "ГоризонтальноеПоложениеВГруппе",
|
||||
"VerticalAlign": "ВертикальноеПоложениеВГруппе",
|
||||
"HorizontalStretch": "РастягиватьПоГоризонтали",
|
||||
"VerticalStretch": "РастягиватьПоВертикали",
|
||||
"LocationInCommandBar": "ПоложениеВКоманднойПанели",
|
||||
"UniqueCommands": "УникальностьКоманд",
|
||||
"DefaultButton": "КнопкаПоУмолчанию",
|
||||
"Check": "Пометка",
|
||||
"MainAttribute": "ОсновнойРеквизит",
|
||||
"Action": "Действие",
|
||||
"AdditionSource": "ИсточникДополнения",
|
||||
"Columns": "Колонки",
|
||||
"Save": "СохраняемыеДанные",
|
||||
"ContextMenu": "КонтекстноеМеню",
|
||||
"ExtendedTooltip": "РасширеннаяПодсказка",
|
||||
"Events": "События",
|
||||
"Handler": "Обработчик",
|
||||
"ChildItems": "ПодчиненныеЭлементы",
|
||||
}
|
||||
|
||||
FORM_XML_PROPERTY_GROUPS = {
|
||||
"Основные": {
|
||||
"name",
|
||||
"id",
|
||||
"Title",
|
||||
"Type",
|
||||
"DataPath",
|
||||
"CommandName",
|
||||
"WindowOpeningMode",
|
||||
"AutoSaveDataInSettings",
|
||||
"CommandBarLocation",
|
||||
"ShowCommandBar",
|
||||
"AutoCommandBar",
|
||||
"SearchStringAddition",
|
||||
"ViewStatusAddition",
|
||||
"SearchControlAddition",
|
||||
"TitleLocation",
|
||||
"Visible",
|
||||
"UserVisible",
|
||||
"Enabled",
|
||||
"ReadOnly",
|
||||
"SkipOnInput",
|
||||
"DefaultItem",
|
||||
"Importance",
|
||||
"ServerUnavailabilityBehavior",
|
||||
"PasswordMode",
|
||||
"Hyperlink",
|
||||
"Representation",
|
||||
"ButtonGroup",
|
||||
"Group",
|
||||
"Behavior",
|
||||
"ShowTitle",
|
||||
"Autofill",
|
||||
},
|
||||
"Использование": {
|
||||
"EditWarningRepresentation",
|
||||
"EditWarning",
|
||||
"EditMode",
|
||||
"AutoEditMode",
|
||||
"AutoCellHeight",
|
||||
"FixInTable",
|
||||
"Shortcut",
|
||||
"CommandSet",
|
||||
"UseCopy",
|
||||
"ChangeRowSet",
|
||||
"RowSelectionMode",
|
||||
"AutoInsertNewRow",
|
||||
"EnableStartDrag",
|
||||
"EnableDrag",
|
||||
"RowFilter",
|
||||
"FileDragMode",
|
||||
"ShowInHeader",
|
||||
"ShowInFooter",
|
||||
"ToolTip",
|
||||
"ToolTipRepresentation",
|
||||
"DropListButton",
|
||||
"ChoiceButton",
|
||||
"ClearButton",
|
||||
"SpinButton",
|
||||
"OpenButton",
|
||||
"CreateButton",
|
||||
"QuickChoice",
|
||||
"ChooseType",
|
||||
"ChoiceList",
|
||||
"IncompleteChoiceMode",
|
||||
"TextEdit",
|
||||
"TextEditUpdate",
|
||||
"MultiLine",
|
||||
"AutoLineBreak",
|
||||
"AutoMarkIncomplete",
|
||||
"AutoChoiceIncomplete",
|
||||
"InputHint",
|
||||
"ChoiceHistoryOnInput",
|
||||
"MainAttribute",
|
||||
"Action",
|
||||
"Save",
|
||||
},
|
||||
"Оформление": {
|
||||
"Picture",
|
||||
"HeaderPicture",
|
||||
"FooterPicture",
|
||||
"BackColor",
|
||||
"TextColor",
|
||||
"BorderColor",
|
||||
"Font",
|
||||
"Shape",
|
||||
"ShapeRepresentation",
|
||||
"PictureLocation",
|
||||
"HeaderBackColor",
|
||||
"Footer",
|
||||
"HorizontalLinesBWA",
|
||||
"VerticalLinesBWA",
|
||||
"UseAlternationRowColorBWA",
|
||||
},
|
||||
"Расположение": {
|
||||
"Width",
|
||||
"Height",
|
||||
"HeightInTableRows",
|
||||
"SearchStringLocation",
|
||||
"ViewStatusLocation",
|
||||
"SearchControlLocation",
|
||||
"AutoMaxWidth",
|
||||
"AutoMaxHeight",
|
||||
"MaxWidth",
|
||||
"MaxHeight",
|
||||
"TitleHeight",
|
||||
"HorizontalAlign",
|
||||
"VerticalAlign",
|
||||
"HorizontalStretch",
|
||||
"VerticalStretch",
|
||||
"LocationInCommandBar",
|
||||
"UniqueCommands",
|
||||
"DefaultButton",
|
||||
"Check",
|
||||
"AdditionSource",
|
||||
"Columns",
|
||||
},
|
||||
}
|
||||
|
||||
FORM_XML_CONTAINER_TAGS = {"ChildItems", "CommandSet", "Events"}
|
||||
|
||||
|
||||
def local_name(tag: str) -> str:
|
||||
return tag.rsplit("}", 1)[-1]
|
||||
|
||||
|
||||
def property_group(tag: str) -> str:
|
||||
for group, names in FORM_XML_PROPERTY_GROUPS.items():
|
||||
if tag in names:
|
||||
return group
|
||||
return "Прочее"
|
||||
|
||||
|
||||
def xml_scalar(element: ET.Element) -> Any:
|
||||
text = (element.text or "").strip()
|
||||
if text:
|
||||
if text == "true":
|
||||
return True
|
||||
if text == "false":
|
||||
return False
|
||||
return text
|
||||
if "name" in element.attrib:
|
||||
return element.attrib.get("name")
|
||||
if "id" in element.attrib:
|
||||
return element.attrib.get("id")
|
||||
return None
|
||||
|
||||
|
||||
def add_property(groups: dict[str, list[dict[str, Any]]], xml_name: str, value: Any, *, source: str = "form_xml") -> None:
|
||||
if value is None:
|
||||
return
|
||||
group = property_group(xml_name)
|
||||
groups.setdefault(group, []).append(
|
||||
{
|
||||
"name": FORM_XML_PROPERTY_NAMES.get(xml_name, xml_name),
|
||||
"xml_name": xml_name,
|
||||
"value": value,
|
||||
"source": source,
|
||||
"status": "ok" if value not in {"", None} else "empty",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def element_semantic(element: ET.Element) -> dict[str, Any]:
|
||||
tag = local_name(element.tag)
|
||||
groups: dict[str, list[dict[str, Any]]] = {}
|
||||
add_property(groups, "name", element.attrib.get("name"), source="form_xml_attribute")
|
||||
add_property(groups, "id", element.attrib.get("id"), source="form_xml_attribute")
|
||||
add_property(groups, "Type", FORM_XML_KIND_RU.get(tag, tag), source="form_xml_tag")
|
||||
if tag == "Event" and (element.text or "").strip():
|
||||
add_property(groups, "Handler", (element.text or "").strip(), source="form_xml_text")
|
||||
for child in list(element):
|
||||
child_tag = local_name(child.tag)
|
||||
if child_tag in FORM_XML_CONTAINER_TAGS:
|
||||
continue
|
||||
add_property(groups, child_tag, xml_scalar(child))
|
||||
mapped = sum(len(items) for items in groups.values())
|
||||
return {
|
||||
"groups": groups,
|
||||
"coverage": {
|
||||
"mapped": mapped,
|
||||
"unmapped": 0,
|
||||
"total": mapped,
|
||||
"status": "ok",
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def walk_form_elements(root: ET.Element, *, limit: int = 5000) -> tuple[list[dict[str, Any]], int, bool]:
|
||||
records: list[dict[str, Any]] = []
|
||||
total = 0
|
||||
|
||||
def walk(
|
||||
element: ET.Element,
|
||||
path: list[int],
|
||||
additional_columns_table: str | None = None,
|
||||
owner_name: str | None = None,
|
||||
) -> None:
|
||||
nonlocal total
|
||||
tag = local_name(element.tag)
|
||||
if tag == "AdditionalColumns" and element.attrib.get("table"):
|
||||
additional_columns_table = element.attrib.get("table")
|
||||
if "name" in element.attrib:
|
||||
total += 1
|
||||
if len(records) < limit:
|
||||
records.append(
|
||||
{
|
||||
"name": element.attrib.get("name"),
|
||||
"id": element.attrib.get("id"),
|
||||
"kind": tag,
|
||||
"kind_ru": FORM_XML_KIND_RU.get(tag, tag),
|
||||
"xml_path": ".".join(str(part) for part in path),
|
||||
"semantic": element_semantic(element),
|
||||
**({"additional_columns_table": additional_columns_table} if additional_columns_table else {}),
|
||||
**({"owner": owner_name} if tag == "Event" and owner_name else {}),
|
||||
}
|
||||
)
|
||||
child_owner = element.attrib.get("name") if "name" in element.attrib and tag != "Event" else owner_name
|
||||
for index, child in enumerate(list(element)):
|
||||
walk(child, [*path, index], additional_columns_table, child_owner)
|
||||
|
||||
walk(root, [])
|
||||
return records, total, total > len(records)
|
||||
|
||||
|
||||
def decode_form_xml(xml: str | bytes | Path, *, max_items: int = 5000) -> dict[str, Any]:
|
||||
if isinstance(xml, Path):
|
||||
root = ET.parse(xml).getroot()
|
||||
source = {"kind": "xml_file", "path": str(xml)}
|
||||
else:
|
||||
root = ET.fromstring(xml)
|
||||
source = {"kind": "xml_text"}
|
||||
items, total, truncated = walk_form_elements(root, limit=max_items)
|
||||
return {
|
||||
"schema": "onec_form_xml_profile.v1",
|
||||
"status": "ok",
|
||||
"source": source,
|
||||
"form": {
|
||||
"version": root.attrib.get("version"),
|
||||
"kind": local_name(root.tag),
|
||||
"kind_ru": FORM_XML_KIND_RU.get(local_name(root.tag), local_name(root.tag)),
|
||||
"semantic": element_semantic(root),
|
||||
},
|
||||
"items": items,
|
||||
"counts": {
|
||||
"items": len(items),
|
||||
"items_total": total,
|
||||
"items_truncated": truncated,
|
||||
},
|
||||
}
|
||||
@@ -0,0 +1,553 @@
|
||||
"""Low-level payload decoding and brace-tree parsing for 1C SQL metadata."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import gzip
|
||||
import copy
|
||||
import re
|
||||
import zlib
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
GUID_RE = re.compile(
|
||||
r"[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}"
|
||||
)
|
||||
|
||||
BraceNode = dict[str, Any]
|
||||
|
||||
|
||||
def try_decompress(data: bytes) -> tuple[bytes, str]:
|
||||
"""Try known 1C payload compression envelopes."""
|
||||
|
||||
attempts = (
|
||||
("raw_deflate", lambda value: zlib.decompress(value, -15)),
|
||||
("zlib", zlib.decompress),
|
||||
("gzip", gzip.decompress),
|
||||
)
|
||||
for name, func in attempts:
|
||||
try:
|
||||
return func(data), name
|
||||
except Exception:
|
||||
pass
|
||||
return data, "none"
|
||||
|
||||
|
||||
def compress_payload(data: bytes, compression: str) -> bytes:
|
||||
"""Compress payload bytes using a known 1C storage envelope."""
|
||||
|
||||
if compression == "raw_deflate":
|
||||
compressor = zlib.compressobj(level=6, wbits=-15)
|
||||
return compressor.compress(data) + compressor.flush()
|
||||
if compression == "zlib":
|
||||
return zlib.compress(data)
|
||||
if compression == "gzip":
|
||||
return gzip.compress(data)
|
||||
if compression == "none":
|
||||
return data
|
||||
raise ValueError(f"unsupported compression: {compression}")
|
||||
|
||||
|
||||
def try_decode(data: bytes) -> tuple[str | None, str | None]:
|
||||
"""Decode payload text, preferring explicit BOM over heuristic scoring."""
|
||||
|
||||
if data.startswith(b"\xef\xbb\xbf"):
|
||||
try:
|
||||
return data.decode("utf-8-sig"), "utf-8-sig"
|
||||
except UnicodeDecodeError:
|
||||
pass
|
||||
|
||||
candidates = ("utf-8-sig", "utf-8", "utf-16-le", "utf-16-be", "cp1251")
|
||||
best: tuple[str | None, str | None, int] = (None, None, -1)
|
||||
for encoding in candidates:
|
||||
try:
|
||||
text = data.decode(encoding)
|
||||
except UnicodeDecodeError:
|
||||
continue
|
||||
sample = text[:20000]
|
||||
printable = sum(1 for char in sample if char.isprintable() or char in "\r\n\t")
|
||||
cyrillic = sum(1 for char in sample if "\u0400" <= char <= "\u04ff")
|
||||
cjk = sum(1 for char in sample if "\u4e00" <= char <= "\u9fff")
|
||||
replacement = sample.count("\ufffd")
|
||||
controls = sum(1 for char in sample if ord(char) < 32 and char not in "\r\n\t\x00")
|
||||
score = printable + cyrillic * 4 - sample.count("\x00") * 10 - cjk * 12 - replacement * 20 - controls * 5
|
||||
if score > best[2]:
|
||||
best = (text, encoding, score)
|
||||
return best[0], best[1]
|
||||
|
||||
|
||||
def encode_text(text: str, encoding: str | None) -> bytes:
|
||||
if not encoding:
|
||||
raise ValueError("encoding is required")
|
||||
return text.encode(encoding)
|
||||
|
||||
|
||||
def decode_payload_lossless(data: bytes) -> dict[str, Any]:
|
||||
"""Decode payload without stripping text bytes so it can be encoded back."""
|
||||
|
||||
payload, compression = try_decompress(data)
|
||||
text, encoding = try_decode(payload)
|
||||
return {
|
||||
"payload": payload,
|
||||
"compression": compression,
|
||||
"text": text,
|
||||
"encoding": encoding,
|
||||
"raw_bytes": len(data),
|
||||
"payload_bytes": len(payload),
|
||||
}
|
||||
|
||||
|
||||
def encode_payload_lossless(decoded: dict[str, Any], *, text: str | None = None, payload: bytes | None = None) -> bytes:
|
||||
"""Encode a decoded payload using its original compression/encoding metadata."""
|
||||
|
||||
compression = str(decoded.get("compression") or "none")
|
||||
if payload is None:
|
||||
if text is None:
|
||||
text = decoded.get("text")
|
||||
payload = encode_text(str(text), decoded.get("encoding")) if text is not None else decoded.get("payload")
|
||||
if not isinstance(payload, (bytes, bytearray)):
|
||||
raise ValueError("payload bytes or text are required")
|
||||
return compress_payload(bytes(payload), compression)
|
||||
|
||||
|
||||
def payload_to_text(data: bytes) -> dict[str, Any]:
|
||||
"""Decode compressed SQL BinaryData to text and generic metadata."""
|
||||
|
||||
payload, compression = try_decompress(data)
|
||||
text, encoding = try_decode(payload)
|
||||
return {
|
||||
"payload": payload,
|
||||
"compression": compression,
|
||||
"text": text.replace("\x00", "").replace("\ufeff", "") if text is not None else None,
|
||||
"encoding": encoding,
|
||||
"raw_bytes": len(data),
|
||||
"payload_bytes": len(payload),
|
||||
}
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Token:
|
||||
kind: str
|
||||
value: str
|
||||
pos: int
|
||||
end: int
|
||||
|
||||
|
||||
class Lexer:
|
||||
def __init__(self, text: str) -> None:
|
||||
self.text = text
|
||||
self.pos = 0
|
||||
|
||||
def tokens(self) -> list[Token]:
|
||||
result: list[Token] = []
|
||||
while self.pos < len(self.text):
|
||||
char = self.text[self.pos]
|
||||
if char.isspace():
|
||||
self.pos += 1
|
||||
continue
|
||||
if char in "{},:":
|
||||
result.append(Token(char, char, self.pos, self.pos + 1))
|
||||
self.pos += 1
|
||||
continue
|
||||
if char == '"':
|
||||
result.append(self._string())
|
||||
continue
|
||||
result.append(self._atom())
|
||||
result.append(Token("EOF", "", self.pos, self.pos))
|
||||
return result
|
||||
|
||||
def _string(self) -> Token:
|
||||
start = self.pos
|
||||
self.pos += 1
|
||||
chars: list[str] = []
|
||||
while self.pos < len(self.text):
|
||||
char = self.text[self.pos]
|
||||
self.pos += 1
|
||||
if char == '"':
|
||||
if self.pos < len(self.text) and self.text[self.pos] == '"':
|
||||
chars.append('"')
|
||||
self.pos += 1
|
||||
continue
|
||||
break
|
||||
chars.append(char)
|
||||
return Token("string", "".join(chars), start, self.pos)
|
||||
|
||||
def _atom(self) -> Token:
|
||||
start = self.pos
|
||||
while self.pos < len(self.text):
|
||||
char = self.text[self.pos]
|
||||
if char.isspace() or char in "{},:":
|
||||
break
|
||||
self.pos += 1
|
||||
return Token("atom", self.text[start : self.pos], start, self.pos)
|
||||
|
||||
|
||||
class Parser:
|
||||
def __init__(self, tokens: list[Token]) -> None:
|
||||
self.tokens = tokens
|
||||
self.index = 0
|
||||
|
||||
def parse(self) -> Any:
|
||||
values = []
|
||||
while not self._peek("EOF"):
|
||||
if self._peek(","):
|
||||
self.index += 1
|
||||
continue
|
||||
values.append(self._value())
|
||||
if len(values) == 1:
|
||||
return values[0]
|
||||
return {"type": "sequence", "items": values}
|
||||
|
||||
def _value(self) -> Any:
|
||||
if self._peek("{"):
|
||||
return self._list()
|
||||
token = self._next()
|
||||
if token.kind == "string":
|
||||
return {"type": "string", "value": token.value, "pos": token.pos, "end": token.end}
|
||||
if token.kind == "atom":
|
||||
return {"type": "atom", "value": token.value, "pos": token.pos, "end": token.end}
|
||||
return {"type": "token", "kind": token.kind, "value": token.value, "pos": token.pos, "end": token.end}
|
||||
|
||||
def _list(self) -> Any:
|
||||
start = self._next()
|
||||
items = []
|
||||
while not self._peek("EOF") and not self._peek("}"):
|
||||
if self._peek(","):
|
||||
self.index += 1
|
||||
continue
|
||||
items.append(self._value())
|
||||
end = items[-1].get("end", items[-1].get("pos", start.end)) if items and isinstance(items[-1], dict) else start.end
|
||||
if self._peek("}"):
|
||||
end = self.tokens[self.index].end
|
||||
self.index += 1
|
||||
return {"type": "list", "pos": start.pos, "end": end, "items": items}
|
||||
|
||||
def _peek(self, kind: str) -> bool:
|
||||
return self.tokens[self.index].kind == kind
|
||||
|
||||
def _next(self) -> Token:
|
||||
token = self.tokens[self.index]
|
||||
self.index += 1
|
||||
return token
|
||||
|
||||
|
||||
def parse_brace_text(text: str) -> Any:
|
||||
"""Parse brace text into a generic tree without semantic interpretation."""
|
||||
|
||||
clean = text.replace("\x00", "").replace("\ufeff", "")
|
||||
first_brace = clean.find("{")
|
||||
if first_brace > 0:
|
||||
clean = clean[first_brace:]
|
||||
return Parser(Lexer(clean).tokens()).parse()
|
||||
|
||||
|
||||
def quote_string(value: str) -> str:
|
||||
return '"' + value.replace('"', '""') + '"'
|
||||
|
||||
|
||||
def patch_brace_text_path(text: str, path: str, value: Any, *, node_type: str = "auto") -> tuple[str, dict[str, Any]]:
|
||||
"""Patch one scalar token in-place without canonicalizing the whole brace tree."""
|
||||
|
||||
if "\x00" in text:
|
||||
raise ValueError("byte-preserving patch does not support NUL-stripped payload text")
|
||||
offset = text.find("{")
|
||||
if offset < 0:
|
||||
raise ValueError("brace text payload is required")
|
||||
source = text[offset:]
|
||||
tree = Parser(Lexer(source).tokens()).parse()
|
||||
old = get_tree_path(tree, path)
|
||||
if not isinstance(old, dict) or old.get("type") not in {"atom", "string"}:
|
||||
raise ValueError("byte-preserving patch supports scalar atom/string nodes only")
|
||||
replacement_type = node_type
|
||||
if replacement_type == "auto":
|
||||
replacement_type = str(old.get("type") or "string")
|
||||
replacement = scalar_node(value, replacement_type)
|
||||
if replacement.get("type") == "string":
|
||||
rendered = quote_string(str(replacement.get("value") or ""))
|
||||
elif replacement.get("type") == "atom":
|
||||
rendered = str(replacement.get("value") or "")
|
||||
else:
|
||||
raise ValueError("byte-preserving patch replacement must be atom or string")
|
||||
start = offset + int(old.get("pos"))
|
||||
end = offset + int(old.get("end"))
|
||||
if start < 0 or end < start or end > len(text):
|
||||
raise ValueError("invalid scalar token span")
|
||||
patched = text[:start] + rendered + text[end:]
|
||||
return patched, {
|
||||
"path": path,
|
||||
"old": scalar(old),
|
||||
"new": str(replacement.get("value") or ""),
|
||||
"old_node_type": old.get("type"),
|
||||
"new_node_type": replacement.get("type"),
|
||||
"span": [start, end],
|
||||
}
|
||||
|
||||
|
||||
def swap_brace_text_paths(text: str, path_a: str, path_b: str) -> tuple[str, dict[str, Any]]:
|
||||
"""Swap two parsed brace-tree node text spans without canonicalizing the tree."""
|
||||
|
||||
if "\x00" in text:
|
||||
raise ValueError("byte-preserving swap does not support NUL-stripped payload text")
|
||||
offset = text.find("{")
|
||||
if offset < 0:
|
||||
raise ValueError("brace text payload is required")
|
||||
if path_a == path_b:
|
||||
raise ValueError("swap paths must be different")
|
||||
parts_a = tree_path_parts(path_a)
|
||||
parts_b = tree_path_parts(path_b)
|
||||
if parts_a == parts_b[: len(parts_a)] or parts_b == parts_a[: len(parts_b)]:
|
||||
raise ValueError("swap paths must not be ancestor/descendant")
|
||||
source = text[offset:]
|
||||
tree = Parser(Lexer(source).tokens()).parse()
|
||||
node_a = get_tree_path(tree, path_a)
|
||||
node_b = get_tree_path(tree, path_b)
|
||||
if not isinstance(node_a, dict) or not isinstance(node_b, dict):
|
||||
raise ValueError("swap paths must resolve to parsed nodes")
|
||||
if "pos" not in node_a or "end" not in node_a or "pos" not in node_b or "end" not in node_b:
|
||||
raise ValueError("swap nodes must have text spans")
|
||||
span_a = [offset + int(node_a["pos"]), offset + int(node_a["end"])]
|
||||
span_b = [offset + int(node_b["pos"]), offset + int(node_b["end"])]
|
||||
if span_a[0] >= span_a[1] or span_b[0] >= span_b[1]:
|
||||
raise ValueError("swap node span is empty")
|
||||
if not (span_a[1] <= span_b[0] or span_b[1] <= span_a[0]):
|
||||
raise ValueError("swap node spans overlap")
|
||||
first_span, second_span = (span_a, span_b) if span_a[0] < span_b[0] else (span_b, span_a)
|
||||
first_text = text[first_span[0] : first_span[1]]
|
||||
second_text = text[second_span[0] : second_span[1]]
|
||||
swapped = text[: first_span[0]] + second_text + text[first_span[1] : second_span[0]] + first_text + text[second_span[1] :]
|
||||
return swapped, {
|
||||
"path_a": path_a,
|
||||
"path_b": path_b,
|
||||
"span_a": span_a,
|
||||
"span_b": span_b,
|
||||
"node_a_type": node_a.get("type"),
|
||||
"node_b_type": node_b.get("type"),
|
||||
}
|
||||
|
||||
|
||||
def append_brace_text_child(text: str, parent_path: str, child_node: Any) -> tuple[str, dict[str, Any]]:
|
||||
"""Append one child node to a parsed brace-tree container without reformatting the rest."""
|
||||
|
||||
if "\x00" in text:
|
||||
raise ValueError("byte-preserving append does not support NUL-stripped payload text")
|
||||
offset = text.find("{")
|
||||
if offset < 0:
|
||||
raise ValueError("brace text payload is required")
|
||||
source = text[offset:]
|
||||
tree = Parser(Lexer(source).tokens()).parse()
|
||||
parent = get_tree_path(tree, parent_path)
|
||||
if not isinstance(parent, dict) or parent.get("type") not in {"list", "sequence"}:
|
||||
raise ValueError("append parent must be a list or sequence node")
|
||||
if "end" not in parent:
|
||||
raise ValueError("append parent span is unavailable")
|
||||
items = parent.get("items") if isinstance(parent.get("items"), list) else []
|
||||
insertion = offset + int(parent["end"]) - 1
|
||||
if insertion < 0 or insertion > len(text) or text[insertion] != "}":
|
||||
raise ValueError("append insertion point is not a closing brace")
|
||||
rendered_child = serialize_brace_tree(child_node)
|
||||
inserted = ("," if items else "") + rendered_child
|
||||
count_updated = False
|
||||
count_before = None
|
||||
count_after = None
|
||||
count_span = None
|
||||
original_insertion = insertion
|
||||
prefix = text[:original_insertion]
|
||||
if len(items) >= 2:
|
||||
count_node = items[1]
|
||||
count_text = scalar(count_node)
|
||||
try:
|
||||
declared_count = int(count_text) if count_text is not None else None
|
||||
except ValueError:
|
||||
declared_count = None
|
||||
actual_record_count = len(items) - 2
|
||||
if declared_count is not None and declared_count == actual_record_count and isinstance(count_node, dict) and "pos" in count_node and "end" in count_node:
|
||||
count_start = offset + int(count_node["pos"])
|
||||
count_end = offset + int(count_node["end"])
|
||||
if 0 <= count_start < count_end <= len(prefix):
|
||||
count_before = declared_count
|
||||
count_after = declared_count + 1
|
||||
count_span = [count_start, count_end]
|
||||
prefix = prefix[:count_start] + str(count_after) + prefix[count_end:]
|
||||
count_updated = True
|
||||
insertion = len(prefix)
|
||||
patched = prefix + inserted + text[original_insertion:]
|
||||
return patched, {
|
||||
"parent_path": parent_path,
|
||||
"inserted_index": len(items),
|
||||
"span": [insertion, insertion],
|
||||
"inserted_bytes": len(inserted.encode("utf-8")),
|
||||
"count_updated": count_updated,
|
||||
**({"count_before": count_before, "count_after": count_after, "count_span": count_span} if count_updated else {}),
|
||||
}
|
||||
|
||||
|
||||
def serialize_brace_tree(node: Any) -> str:
|
||||
"""Serialize parsed brace tree to canonical 1C brace text."""
|
||||
|
||||
if isinstance(node, dict):
|
||||
node_type = node.get("type")
|
||||
if node_type == "list":
|
||||
return "{" + ",".join(serialize_brace_tree(item) for item in (node.get("items") or [])) + "}"
|
||||
if node_type == "sequence":
|
||||
return ",".join(serialize_brace_tree(item) for item in (node.get("items") or []))
|
||||
if node_type == "string":
|
||||
return quote_string(str(node.get("value") or ""))
|
||||
if node_type == "atom":
|
||||
return str(node.get("value") or "")
|
||||
if node_type == "token":
|
||||
return str(node.get("value") or "")
|
||||
if isinstance(node, str):
|
||||
return quote_string(node)
|
||||
if node is None:
|
||||
return ""
|
||||
return str(node)
|
||||
|
||||
|
||||
def tree_path_parts(path: str) -> list[int]:
|
||||
if not str(path or "").strip():
|
||||
raise ValueError("path is required")
|
||||
parts: list[int] = []
|
||||
for part in str(path).split("."):
|
||||
if not part.isdigit():
|
||||
raise ValueError(f"path segment is not a non-negative integer: {part}")
|
||||
parts.append(int(part))
|
||||
return parts
|
||||
|
||||
|
||||
def get_tree_path(tree: Any, path: str) -> Any:
|
||||
node = tree
|
||||
for index in tree_path_parts(path):
|
||||
if not isinstance(node, dict) or node.get("type") not in {"list", "sequence"}:
|
||||
raise ValueError(f"path enters a non-container node at segment {index}")
|
||||
items = node.get("items") or []
|
||||
if index >= len(items):
|
||||
raise IndexError(f"path segment {index} is outside node with {len(items)} items")
|
||||
node = items[index]
|
||||
return node
|
||||
|
||||
|
||||
def inspect_tree_node(node: Any, *, depth: int = 0, max_depth: int = 2, max_children: int = 8) -> dict[str, Any]:
|
||||
if not isinstance(node, dict):
|
||||
return {"type": type(node).__name__, "repr": repr(node)[:200]}
|
||||
node_type = str(node.get("type") or "")
|
||||
result: dict[str, Any] = {"type": node_type}
|
||||
for key in ("value", "pos", "end", "kind"):
|
||||
if key in node:
|
||||
result[key] = node.get(key)
|
||||
items = node.get("items") if isinstance(node.get("items"), list) else None
|
||||
if items is not None:
|
||||
result["items"] = len(items)
|
||||
if depth < max_depth:
|
||||
result["children"] = [
|
||||
inspect_tree_node(child, depth=depth + 1, max_depth=max_depth, max_children=max_children)
|
||||
for child in items[:max_children]
|
||||
]
|
||||
return result
|
||||
|
||||
|
||||
def inspect_brace_text_path(text: str, path: str, *, max_depth: int = 2, max_children: int = 8) -> dict[str, Any]:
|
||||
if "\x00" in text:
|
||||
raise ValueError("byte-preserving probe does not support NUL-stripped payload text")
|
||||
offset = text.find("{")
|
||||
if offset < 0:
|
||||
raise ValueError("brace text payload is required")
|
||||
source = text[offset:]
|
||||
tree = Parser(Lexer(source).tokens()).parse()
|
||||
node = get_tree_path(tree, path)
|
||||
result = {"path": path, "node": inspect_tree_node(node, max_depth=max_depth, max_children=max_children)}
|
||||
if isinstance(node, dict) and "pos" in node:
|
||||
result["span"] = [offset + int(node.get("pos")), offset + int(node.get("end", node.get("pos")))]
|
||||
return result
|
||||
|
||||
|
||||
def scalar_node(value: Any, node_type: str = "auto") -> dict[str, Any]:
|
||||
if isinstance(value, dict) and value.get("type") in {"atom", "string", "list", "sequence", "token"}:
|
||||
return value
|
||||
if node_type not in {"auto", "atom", "string"}:
|
||||
raise ValueError("node_type must be auto, atom, or string")
|
||||
if node_type == "atom":
|
||||
return {"type": "atom", "value": str(value)}
|
||||
if node_type == "string":
|
||||
return {"type": "string", "value": str(value)}
|
||||
if isinstance(value, bool):
|
||||
return {"type": "atom", "value": "true" if value else "false"}
|
||||
if isinstance(value, (int, float)) and not isinstance(value, bool):
|
||||
return {"type": "atom", "value": str(value)}
|
||||
return {"type": "string", "value": str(value)}
|
||||
|
||||
|
||||
def set_tree_path(tree: Any, path: str, value: Any, *, node_type: str = "auto") -> Any:
|
||||
"""Return a deep-copied tree with one node replaced by path."""
|
||||
|
||||
parts = tree_path_parts(path)
|
||||
result = copy.deepcopy(tree)
|
||||
parent = result
|
||||
for index in parts[:-1]:
|
||||
if not isinstance(parent, dict) or parent.get("type") not in {"list", "sequence"}:
|
||||
raise ValueError(f"path enters a non-container node at segment {index}")
|
||||
items = parent.get("items") or []
|
||||
if index >= len(items):
|
||||
raise IndexError(f"path segment {index} is outside node with {len(items)} items")
|
||||
parent = items[index]
|
||||
if not isinstance(parent, dict) or parent.get("type") not in {"list", "sequence"}:
|
||||
raise ValueError("path parent is not a container")
|
||||
items = parent.get("items") or []
|
||||
last = parts[-1]
|
||||
if last >= len(items):
|
||||
raise IndexError(f"path segment {last} is outside node with {len(items)} items")
|
||||
old = items[last]
|
||||
replacement_type = node_type
|
||||
if replacement_type == "auto" and isinstance(old, dict) and old.get("type") in {"atom", "string"}:
|
||||
replacement_type = str(old.get("type"))
|
||||
items[last] = scalar_node(value, replacement_type)
|
||||
return result
|
||||
|
||||
|
||||
def encode_brace_tree(tree: Any, decoded: dict[str, Any]) -> bytes:
|
||||
"""Encode a modified brace tree with canonical formatting."""
|
||||
|
||||
return encode_payload_lossless(decoded, text=serialize_brace_tree(tree))
|
||||
|
||||
|
||||
def parse_payload_file(path: Path) -> dict[str, Any]:
|
||||
decoded = payload_to_text(path.read_bytes())
|
||||
text = decoded.get("text")
|
||||
decoded["tree"] = parse_brace_text(text) if text and "{" in text else None
|
||||
return decoded
|
||||
|
||||
|
||||
def scalar(node: Any) -> str:
|
||||
if isinstance(node, dict) and node.get("type") in {"atom", "string"}:
|
||||
return str(node.get("value") or "")
|
||||
return ""
|
||||
|
||||
|
||||
def collect_strings(value: Any, limit: int = 200) -> list[str]:
|
||||
result: list[str] = []
|
||||
|
||||
def walk(node: Any) -> None:
|
||||
if len(result) >= limit:
|
||||
return
|
||||
if isinstance(node, dict) and node.get("type") == "string":
|
||||
text = str(node.get("value") or "")
|
||||
if text:
|
||||
result.append(text)
|
||||
return
|
||||
if isinstance(node, dict):
|
||||
for child in node.get("items") or []:
|
||||
walk(child)
|
||||
|
||||
walk(value)
|
||||
return result
|
||||
|
||||
|
||||
def root_signature(tree: Any) -> dict[str, Any]:
|
||||
if not (isinstance(tree, dict) and tree.get("type") == "list"):
|
||||
return {"root_type": tree.get("type") if isinstance(tree, dict) else None}
|
||||
items = tree.get("items") or []
|
||||
return {
|
||||
"root_type": "list",
|
||||
"root_len": len(items),
|
||||
"root_marker": scalar(items[0]) if items else "",
|
||||
}
|
||||
@@ -0,0 +1,140 @@
|
||||
"""Storage-route helpers based on DBNames records.
|
||||
|
||||
This module maps platform DBNames roles to physical SQL name candidates. It
|
||||
does not infer business semantics or concrete metadata object names.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import asdict, dataclass
|
||||
from typing import Iterable
|
||||
|
||||
from .dbnames import DBNamesRecord
|
||||
|
||||
|
||||
TABLE_ROLE_PREFIX = {
|
||||
"Reference": "_Reference",
|
||||
"ReferenceChngR": "_ReferenceChngR",
|
||||
"Document": "_Document",
|
||||
"DocumentChngR": "_DocumentChngR",
|
||||
"Enum": "_Enum",
|
||||
"InfoRg": "_InfoRg",
|
||||
"InfoRgChngR": "_InfoRgChngR",
|
||||
"AccumRg": "_AccumRg",
|
||||
"AccumRgChngR": "_AccumRgChngR",
|
||||
"AccumRgOpt": "_AccumRgOpt",
|
||||
"AccumRgT": "_AccumRgT",
|
||||
"AccRg": "_AccRg",
|
||||
"AccRgAT0": "_AccRgAT0",
|
||||
"AccRgCT": "_AccRgCT",
|
||||
"AccRgChngR": "_AccRgChngR",
|
||||
"AccRgOpt": "_AccRgOpt",
|
||||
"Const": "_Const",
|
||||
"ConstChngR": "_ConstChngR",
|
||||
"DocumentJournal": "_DocumentJournal",
|
||||
"Node": "_Node",
|
||||
"ScheduledJobs": "_ScheduledJobs",
|
||||
"BPr": "_BPr",
|
||||
"BPrPoints": "_BPrPoints",
|
||||
"BPrChngR": "_BPrChngR",
|
||||
"Task": "_Task",
|
||||
"TaskChngR": "_TaskChngR",
|
||||
"Acc": "_Acc",
|
||||
"AccSInf": "_AccSInf",
|
||||
"AccChngR": "_AccChngR",
|
||||
"CKinds": "_CKinds",
|
||||
"CKindsChngR": "_CKindsChngR",
|
||||
"IntegServiceSettings": "_IntegServiceSettings",
|
||||
"IntegServiceMsgBody": "_IntegServiceMsgBody",
|
||||
"IntegServiceExtMsgBody": "_IntegServiceExtMsgBody",
|
||||
"IntegChannelInQueue": "_IntegChannelInQueue",
|
||||
"IntegChannelOutQueue": "_IntegChannelOutQueue",
|
||||
}
|
||||
|
||||
|
||||
FIELD_ROLE_PREFIX = {
|
||||
"Fld": "_Fld",
|
||||
}
|
||||
|
||||
|
||||
STRUCTURAL_ROLES = {
|
||||
"VT",
|
||||
"LineNo",
|
||||
"ByDims",
|
||||
"ByField",
|
||||
"ByResource",
|
||||
"ByProperty",
|
||||
"TurnoverDt",
|
||||
"TurnoverCt",
|
||||
"Turnover",
|
||||
}
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class StorageRoute:
|
||||
guid: str
|
||||
storage_role: str
|
||||
sql_number: int
|
||||
source: str
|
||||
route_kind: str
|
||||
physical_name_candidate: str | None
|
||||
note: str
|
||||
|
||||
def to_dict(self) -> dict[str, object]:
|
||||
return asdict(self)
|
||||
|
||||
|
||||
def storage_route(record: DBNamesRecord) -> StorageRoute:
|
||||
"""Return the mechanical SQL route candidate for one DBNames record."""
|
||||
|
||||
role = record.storage_role
|
||||
if role in TABLE_ROLE_PREFIX:
|
||||
return StorageRoute(
|
||||
guid=record.guid,
|
||||
storage_role=role,
|
||||
sql_number=record.sql_number,
|
||||
source=record.source,
|
||||
route_kind="table",
|
||||
physical_name_candidate=f"{TABLE_ROLE_PREFIX[role]}{record.sql_number}",
|
||||
note="table-like DBNames storage role",
|
||||
)
|
||||
if role in FIELD_ROLE_PREFIX:
|
||||
return StorageRoute(
|
||||
guid=record.guid,
|
||||
storage_role=role,
|
||||
sql_number=record.sql_number,
|
||||
source=record.source,
|
||||
route_kind="field",
|
||||
physical_name_candidate=f"{FIELD_ROLE_PREFIX[role]}{record.sql_number}",
|
||||
note="field DBNames storage role; value suffixes depend on type evidence",
|
||||
)
|
||||
if role in STRUCTURAL_ROLES:
|
||||
return StorageRoute(
|
||||
guid=record.guid,
|
||||
storage_role=role,
|
||||
sql_number=record.sql_number,
|
||||
source=record.source,
|
||||
route_kind="structural",
|
||||
physical_name_candidate=None,
|
||||
note="structural DBNames role; requires parent object/section context",
|
||||
)
|
||||
return StorageRoute(
|
||||
guid=record.guid,
|
||||
storage_role=role,
|
||||
sql_number=record.sql_number,
|
||||
source=record.source,
|
||||
route_kind="unknown",
|
||||
physical_name_candidate=None,
|
||||
note="unclassified DBNames storage role",
|
||||
)
|
||||
|
||||
|
||||
def group_records_by_guid(records: Iterable[DBNamesRecord]) -> dict[str, list[DBNamesRecord]]:
|
||||
grouped: dict[str, list[DBNamesRecord]] = {}
|
||||
for record in records:
|
||||
grouped.setdefault(record.guid, []).append(record)
|
||||
return grouped
|
||||
|
||||
|
||||
def storage_routes(records: Iterable[DBNamesRecord]) -> list[StorageRoute]:
|
||||
return [storage_route(record) for record in records]
|
||||
@@ -0,0 +1,279 @@
|
||||
"""Evidence-based structured metadata projection for Config object payloads."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import asdict, dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from .child_records import collect_evidence, declared_child_records
|
||||
from .config_object import find_identity
|
||||
from .payload import GUID_RE, parse_payload_file, root_signature, scalar
|
||||
from .xml_metadata import XmlMetadataItem, extract_xml_metadata_items, group_xml_items
|
||||
|
||||
|
||||
CATEGORY_FIELDS = {
|
||||
"Attribute": "attributes",
|
||||
"TabularSection": "tabular_sections",
|
||||
"Dimension": "dimensions",
|
||||
"Resource": "resources",
|
||||
"Form": "forms",
|
||||
"Template": "templates",
|
||||
"Command": "commands",
|
||||
"AddressingAttribute": "addressing_attributes",
|
||||
"AccountingFlag": "accounting_flags",
|
||||
"Column": "columns",
|
||||
"EnumValue": "enum_values",
|
||||
"IntegrationServiceChannel": "integration_service_channels",
|
||||
"Operation": "operations",
|
||||
"URLTemplate": "url_templates",
|
||||
}
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class MetadataItemEvidence:
|
||||
category: str
|
||||
name: str
|
||||
synonym: str
|
||||
uuid: str | None
|
||||
value_type: dict[str, Any] | None
|
||||
parent_category: str | None
|
||||
parent_name: str | None
|
||||
parent_uuid: str | None
|
||||
section_path: str
|
||||
record_path: str | None
|
||||
record_index: int | None
|
||||
evidence: dict[str, bool]
|
||||
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
return asdict(self)
|
||||
|
||||
|
||||
def _children(node: Any) -> list[Any]:
|
||||
if isinstance(node, dict) and node.get("type") in {"list", "sequence"}:
|
||||
return node.get("items") or []
|
||||
return []
|
||||
|
||||
|
||||
def get_by_path(tree: Any, path: str) -> Any | None:
|
||||
node = tree
|
||||
if path == "":
|
||||
return node
|
||||
for part in path.split("."):
|
||||
if not isinstance(node, dict) or node.get("type") not in {"list", "sequence"}:
|
||||
return None
|
||||
items = node.get("items") or []
|
||||
index = int(part)
|
||||
if index < 0 or index >= len(items):
|
||||
return None
|
||||
node = items[index]
|
||||
return node
|
||||
|
||||
|
||||
def section_rules(summary: dict[str, Any], kind: str, min_support_ratio: float) -> list[dict[str, Any]]:
|
||||
rules = []
|
||||
for section in summary.get("sections") or []:
|
||||
if section.get("kind") != kind:
|
||||
continue
|
||||
category = section.get("candidate_semantic")
|
||||
if not category or category == kind:
|
||||
continue
|
||||
support_ratio = float(section.get("candidate_support_ratio") or 0)
|
||||
if support_ratio < min_support_ratio:
|
||||
continue
|
||||
rules.append(
|
||||
{
|
||||
"path": section["path"],
|
||||
"category": category,
|
||||
"support_ratio": support_ratio,
|
||||
"sample_count": section.get("sample_count"),
|
||||
}
|
||||
)
|
||||
return rules
|
||||
|
||||
|
||||
def item_evidence(
|
||||
item: XmlMetadataItem,
|
||||
evidence: dict[str, set[str]],
|
||||
section_path: str,
|
||||
*,
|
||||
record_path: str | None = None,
|
||||
record_index: int | None = None,
|
||||
) -> MetadataItemEvidence | None:
|
||||
strings = evidence["strings"]
|
||||
guids = evidence["guids"]
|
||||
hits = {
|
||||
"name": bool(item.name and item.name in strings),
|
||||
"synonym": bool(item.synonym and item.synonym in strings),
|
||||
"uuid": bool(item.uuid and item.uuid.lower() in guids),
|
||||
}
|
||||
if not any(hits.values()):
|
||||
return None
|
||||
return MetadataItemEvidence(
|
||||
category=item.category,
|
||||
name=item.name,
|
||||
synonym=item.synonym,
|
||||
uuid=item.uuid.lower() if item.uuid else None,
|
||||
value_type=item.value_type,
|
||||
parent_category=item.parent_category,
|
||||
parent_name=item.parent_name,
|
||||
parent_uuid=item.parent_uuid.lower() if item.parent_uuid else None,
|
||||
section_path=section_path,
|
||||
record_path=record_path,
|
||||
record_index=record_index,
|
||||
evidence=hits,
|
||||
)
|
||||
|
||||
|
||||
def best_item_record_match(item: XmlMetadataItem, records: list[Any], section_path: str) -> MetadataItemEvidence | None:
|
||||
best: MetadataItemEvidence | None = None
|
||||
best_score = -1
|
||||
for record in records:
|
||||
match = item_evidence(
|
||||
item,
|
||||
record.evidence,
|
||||
section_path,
|
||||
record_path=record.path,
|
||||
record_index=record.index,
|
||||
)
|
||||
if not match:
|
||||
continue
|
||||
score = int(match.evidence["uuid"]) * 4 + int(match.evidence["name"]) * 2 + int(match.evidence["synonym"])
|
||||
if score > best_score:
|
||||
best = match
|
||||
best_score = score
|
||||
return best
|
||||
|
||||
|
||||
def items_for_parent(grouped: dict[str, list[XmlMetadataItem]], category: str, parent_uuid: str | None) -> list[XmlMetadataItem]:
|
||||
return [
|
||||
item
|
||||
for item in grouped.get(category, [])
|
||||
if (item.parent_uuid or "").lower() == (parent_uuid or "").lower()
|
||||
]
|
||||
|
||||
|
||||
def declared_record_containers(node: Any, path: str, *, max_depth: int = 3, include_root: bool = True) -> list[list[Any]]:
|
||||
result = []
|
||||
|
||||
def walk(value: Any, current_path: str, depth: int) -> None:
|
||||
records = declared_child_records(value, current_path)
|
||||
if records and (include_root or depth > 0):
|
||||
result.append(records)
|
||||
if depth >= max_depth:
|
||||
return
|
||||
for index, child in enumerate(_children(value)):
|
||||
walk(child, f"{current_path}.{index}", depth + 1)
|
||||
|
||||
walk(node, path, 0)
|
||||
return result
|
||||
|
||||
|
||||
def nested_tabular_attributes(
|
||||
tree: Any,
|
||||
tabular_section_item: dict[str, Any],
|
||||
grouped: dict[str, list[XmlMetadataItem]],
|
||||
) -> list[dict[str, Any]]:
|
||||
parent_uuid = tabular_section_item.get("uuid")
|
||||
record_path = tabular_section_item.get("record_path")
|
||||
if not parent_uuid or not record_path:
|
||||
return []
|
||||
node = get_by_path(tree, record_path)
|
||||
if node is None:
|
||||
return []
|
||||
xml_items = items_for_parent(grouped, "Attribute", parent_uuid)
|
||||
if not xml_items:
|
||||
return []
|
||||
containers = declared_record_containers(node, record_path, include_root=False)
|
||||
all_records = [record for records in containers for record in records]
|
||||
result = []
|
||||
for item in xml_items:
|
||||
match = best_item_record_match(item, all_records, record_path)
|
||||
if match:
|
||||
data = match.to_dict()
|
||||
data["tabular_section_name"] = tabular_section_item.get("name")
|
||||
data["tabular_section_uuid"] = parent_uuid
|
||||
result.append(data)
|
||||
result.sort(key=lambda item: (item["tabular_section_name"] or "", item["name"], item.get("uuid") or ""))
|
||||
return result
|
||||
|
||||
|
||||
def parse_structured_metadata(
|
||||
config_file: Path,
|
||||
xml_file: Path,
|
||||
kind: str,
|
||||
category_summary: dict[str, Any],
|
||||
*,
|
||||
min_support_ratio: float = 1.0,
|
||||
) -> dict[str, Any]:
|
||||
parsed = parse_payload_file(config_file)
|
||||
tree = parsed.get("tree")
|
||||
identity = find_identity(tree)
|
||||
xml_items = extract_xml_metadata_items(xml_file)
|
||||
grouped = group_xml_items(xml_items)
|
||||
rules = section_rules(category_summary, kind, min_support_ratio)
|
||||
|
||||
result: dict[str, Any] = {
|
||||
"schema": "onec_structured_metadata_projection.v1",
|
||||
"kind": kind,
|
||||
"config_file": str(config_file),
|
||||
"xml_file": str(xml_file),
|
||||
"root": root_signature(tree),
|
||||
"identity": identity.to_dict() if identity else None,
|
||||
"min_support_ratio": min_support_ratio,
|
||||
"rules": rules,
|
||||
"attributes": [],
|
||||
"tabular_sections": [],
|
||||
"tabular_section_attributes": [],
|
||||
"dimensions": [],
|
||||
"resources": [],
|
||||
"forms": [],
|
||||
"templates": [],
|
||||
"commands": [],
|
||||
"addressing_attributes": [],
|
||||
"accounting_flags": [],
|
||||
"columns": [],
|
||||
"enum_values": [],
|
||||
"integration_service_channels": [],
|
||||
"operations": [],
|
||||
"url_templates": [],
|
||||
"unmapped_rules": [],
|
||||
"record_boundary_rules": [],
|
||||
}
|
||||
|
||||
for rule in rules:
|
||||
category = rule["category"]
|
||||
field = CATEGORY_FIELDS.get(category)
|
||||
node = get_by_path(tree, rule["path"]) if tree else None
|
||||
if not field or node is None:
|
||||
result["unmapped_rules"].append(rule)
|
||||
continue
|
||||
records = declared_child_records(node, rule["path"])
|
||||
if records:
|
||||
result["record_boundary_rules"].append(
|
||||
{
|
||||
"path": rule["path"],
|
||||
"category": category,
|
||||
"declared_record_count": len(records),
|
||||
"record_paths_sample": [record.path for record in records[:10]],
|
||||
}
|
||||
)
|
||||
section_evidence = collect_evidence(node)
|
||||
matched = []
|
||||
for item in items_for_parent(grouped, category, identity.guid if identity else None):
|
||||
item_match = best_item_record_match(item, records, rule["path"]) if records else None
|
||||
if not item_match:
|
||||
item_match = item_evidence(item, section_evidence, rule["path"])
|
||||
if item_match:
|
||||
matched.append(item_match.to_dict())
|
||||
matched.sort(key=lambda item: (item["name"], item.get("uuid") or ""))
|
||||
result[field].extend(matched)
|
||||
if category == "TabularSection":
|
||||
for tabular_section in matched:
|
||||
result["tabular_section_attributes"].extend(nested_tabular_attributes(tree, tabular_section, grouped))
|
||||
|
||||
result["counts"] = {
|
||||
field: len(result[field])
|
||||
for field in sorted({*set(CATEGORY_FIELDS.values()), "tabular_section_attributes"})
|
||||
}
|
||||
return result
|
||||
@@ -0,0 +1,162 @@
|
||||
"""Small XML metadata extractor used as validation oracle for SQL payloads."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import xml.etree.ElementTree as ET
|
||||
from dataclasses import asdict, dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
def local_name(tag: str) -> str:
|
||||
return tag.rsplit("}", 1)[-1] if "}" in tag else tag
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class XmlMetadataItem:
|
||||
category: str
|
||||
name: str
|
||||
synonym: str
|
||||
uuid: str | None
|
||||
value_type: dict[str, Any] | None = None
|
||||
parent_category: str | None = None
|
||||
parent_name: str | None = None
|
||||
parent_uuid: str | None = None
|
||||
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
return asdict(self)
|
||||
|
||||
|
||||
def direct_child(parent: ET.Element, name: str) -> ET.Element | None:
|
||||
for child in list(parent):
|
||||
if local_name(child.tag) == name:
|
||||
return child
|
||||
return None
|
||||
|
||||
|
||||
def child_text(parent: ET.Element, name: str) -> str:
|
||||
child = direct_child(parent, name)
|
||||
return (child.text or "").strip() if child is not None else ""
|
||||
|
||||
|
||||
def synonym_text(properties: ET.Element | None) -> str:
|
||||
if properties is None:
|
||||
return ""
|
||||
synonym = direct_child(properties, "Synonym")
|
||||
if synonym is None:
|
||||
return ""
|
||||
for node in synonym.iter():
|
||||
if local_name(node.tag) == "content" and node.text:
|
||||
return node.text.strip()
|
||||
return ""
|
||||
|
||||
|
||||
def value_type(properties: ET.Element | None) -> dict[str, Any] | None:
|
||||
if properties is None:
|
||||
return None
|
||||
type_node = direct_child(properties, "Type")
|
||||
if type_node is None:
|
||||
return None
|
||||
types = []
|
||||
qualifiers: dict[str, Any] = {}
|
||||
for node in type_node.iter():
|
||||
name = local_name(node.tag)
|
||||
text = (node.text or "").strip()
|
||||
if name == "Type" and node is not type_node and text:
|
||||
types.append(text)
|
||||
elif name in {"Length", "AllowedLength", "Digits", "FractionDigits", "AllowedSign", "DateFractions"} and text:
|
||||
qualifiers[name] = text
|
||||
if not types and not qualifiers:
|
||||
return None
|
||||
return {
|
||||
"types": types,
|
||||
"qualifiers": qualifiers,
|
||||
"is_composite": len(types) > 1,
|
||||
}
|
||||
|
||||
|
||||
def item_from_properties(
|
||||
category: str,
|
||||
node: ET.Element,
|
||||
properties: ET.Element | None,
|
||||
*,
|
||||
parent: XmlMetadataItem | None = None,
|
||||
) -> XmlMetadataItem:
|
||||
return XmlMetadataItem(
|
||||
category=category,
|
||||
name=child_text(properties, "Name") if properties is not None else node.attrib.get("name", ""),
|
||||
synonym=synonym_text(properties),
|
||||
uuid=node.attrib.get("uuid"),
|
||||
value_type=value_type(properties),
|
||||
parent_category=parent.category if parent else None,
|
||||
parent_name=parent.name if parent else None,
|
||||
parent_uuid=parent.uuid.lower() if parent and parent.uuid else None,
|
||||
)
|
||||
|
||||
|
||||
def extract_xml_metadata_items(path: Path) -> list[XmlMetadataItem]:
|
||||
root = ET.parse(path).getroot()
|
||||
metadata_node = next(iter(list(root)), None)
|
||||
if metadata_node is None:
|
||||
return []
|
||||
|
||||
result: list[XmlMetadataItem] = []
|
||||
properties = direct_child(metadata_node, "Properties")
|
||||
root_item = item_from_properties(local_name(metadata_node.tag), metadata_node, properties)
|
||||
result.append(root_item)
|
||||
|
||||
internal = direct_child(metadata_node, "InternalInfo")
|
||||
if internal is not None:
|
||||
for generated in internal.iter():
|
||||
if local_name(generated.tag) == "GeneratedType":
|
||||
result.append(
|
||||
XmlMetadataItem(
|
||||
category="GeneratedType",
|
||||
name=generated.attrib.get("name", ""),
|
||||
synonym=generated.attrib.get("category", ""),
|
||||
uuid=None,
|
||||
)
|
||||
)
|
||||
|
||||
if properties is not None:
|
||||
standard = direct_child(properties, "StandardAttributes")
|
||||
if standard is not None:
|
||||
for node in list(standard):
|
||||
if local_name(node.tag) == "StandardAttribute":
|
||||
result.append(
|
||||
XmlMetadataItem(
|
||||
category="StandardAttribute",
|
||||
name=node.attrib.get("name", ""),
|
||||
synonym=synonym_text(node),
|
||||
uuid=None,
|
||||
)
|
||||
)
|
||||
|
||||
child_objects = direct_child(metadata_node, "ChildObjects")
|
||||
if child_objects is not None:
|
||||
for node in list(child_objects):
|
||||
category = local_name(node.tag)
|
||||
props = direct_child(node, "Properties")
|
||||
child_item = item_from_properties(category, node, props, parent=root_item)
|
||||
result.append(child_item)
|
||||
append_nested_child_objects(result, node, child_item)
|
||||
return result
|
||||
|
||||
|
||||
def append_nested_child_objects(result: list[XmlMetadataItem], parent_node: ET.Element, parent_item: XmlMetadataItem) -> None:
|
||||
child_objects = direct_child(parent_node, "ChildObjects")
|
||||
if child_objects is None:
|
||||
return
|
||||
for node in list(child_objects):
|
||||
category = local_name(node.tag)
|
||||
props = direct_child(node, "Properties")
|
||||
child_item = item_from_properties(category, node, props, parent=parent_item)
|
||||
result.append(child_item)
|
||||
append_nested_child_objects(result, node, child_item)
|
||||
|
||||
|
||||
def group_xml_items(items: list[XmlMetadataItem]) -> dict[str, list[XmlMetadataItem]]:
|
||||
grouped: dict[str, list[XmlMetadataItem]] = {}
|
||||
for item in items:
|
||||
grouped.setdefault(item.category, []).append(item)
|
||||
return grouped
|
||||
Reference in New Issue
Block a user