464 lines
21 KiB
Python
464 lines
21 KiB
Python
"""Lossless read-only decoder for 1C Data Composition Schema SQL payloads.
|
|
|
|
The payload stored in ConfigCAS is commonly a compressed stream with a small
|
|
binary prefix followed by an XML ``SchemaFile`` document. This module does
|
|
not infer SCD semantics from names: every returned item is backed by an XML
|
|
node in that document.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import xml.etree.ElementTree as ET
|
|
import xml.parsers.expat as expat
|
|
import html
|
|
import hashlib
|
|
import re
|
|
from typing import Any
|
|
|
|
from .payload import decode_payload_lossless
|
|
|
|
|
|
QUERY_PARAMETER_RE = re.compile(r"&([A-Za-z_\u0400-\u04ff][A-Za-z0-9_\u0400-\u04ff]*)")
|
|
QUERY_SOURCE_RE = re.compile(r"(?:\bИЗ|\bFROM|\bJOIN|\bСОЕДИНЕНИЕ)\s+([A-Za-z_\u0400-\u04ff][A-Za-z0-9_\u0400-\u04ff]*(?:\.[A-Za-z_\u0400-\u04ff][A-Za-z0-9_\u0400-\u04ff]*)+)", re.IGNORECASE)
|
|
QUERY_SOURCE_BINDING_RE = re.compile(r"(?:\bИЗ|\bFROM|\bJOIN|\bСОЕДИНЕНИЕ)\s+([A-Za-z_\u0400-\u04ff][A-Za-z0-9_\u0400-\u04ff]*(?:\.[A-Za-z_\u0400-\u04ff][A-Za-z0-9_\u0400-\u04ff]*)+)(?:\s+(?:КАК|AS)\s+([A-Za-z_\u0400-\u04ff][A-Za-z0-9_\u0400-\u04ff]*))?", re.IGNORECASE)
|
|
QUERY_SELECT_RE = re.compile(r"\b(?:ВЫБРАТЬ|SELECT)\b(.*?)(?=\b(?:ИЗ|FROM)\b)", re.IGNORECASE | re.DOTALL)
|
|
QUERY_ALIAS_RE = re.compile(r"\b(?:КАК|AS)\s+([A-Za-z_\u0400-\u04ff][A-Za-z0-9_\u0400-\u04ff]*)(?=\s*(?:,|\r?\n|$))", re.IGNORECASE)
|
|
|
|
|
|
def local_name(tag: str) -> str:
|
|
return tag.rsplit("}", 1)[-1] if "}" in tag else tag
|
|
|
|
|
|
def direct_child(node: ET.Element, name: str) -> ET.Element | None:
|
|
return next((child for child in node if local_name(child.tag) == name), None)
|
|
|
|
|
|
def child_text(node: ET.Element, *names: str) -> str:
|
|
for name in names:
|
|
child = direct_child(node, name)
|
|
if child is not None:
|
|
value = "".join(child.itertext()).strip()
|
|
if value:
|
|
return value
|
|
return ""
|
|
|
|
|
|
def query_without_line_comments(text: str) -> str:
|
|
"""Remove 1C query ``//`` comments without touching quoted string literals."""
|
|
|
|
result: list[str] = []
|
|
index = 0
|
|
quoted = False
|
|
while index < len(text):
|
|
char = text[index]
|
|
if char == '"':
|
|
result.append(char)
|
|
if quoted and index + 1 < len(text) and text[index + 1] == '"':
|
|
result.append('"')
|
|
index += 2
|
|
continue
|
|
quoted = not quoted
|
|
index += 1
|
|
continue
|
|
if not quoted and char == "/" and index + 1 < len(text) and text[index + 1] == "/":
|
|
line_end = text.find("\n", index)
|
|
if line_end < 0:
|
|
break
|
|
result.append("\n")
|
|
index = line_end + 1
|
|
continue
|
|
result.append(char)
|
|
index += 1
|
|
return "".join(result)
|
|
|
|
|
|
def node_path(root: ET.Element, target: ET.Element) -> str:
|
|
"""Produce a stable, human-readable evidence path without XML prefixes."""
|
|
|
|
def visit(node: ET.Element, prefix: str) -> str | None:
|
|
name = local_name(node.tag)
|
|
current = f"{prefix}/{name}" if prefix else f"/{name}"
|
|
if node is target:
|
|
return current
|
|
positions: dict[str, int] = {}
|
|
for child in node:
|
|
child_name = local_name(child.tag)
|
|
positions[child_name] = positions.get(child_name, 0) + 1
|
|
child_prefix = f"{current}[{positions[child_name]}]"
|
|
found = visit(child, child_prefix)
|
|
if found:
|
|
return found
|
|
return None
|
|
|
|
return visit(root, "") or "/"
|
|
|
|
|
|
def xml_from_scd_payload(data: bytes) -> tuple[ET.Element | None, dict[str, Any]]:
|
|
decoded = decode_payload_lossless(data)
|
|
payload = decoded.get("payload")
|
|
if not isinstance(payload, (bytes, bytearray)):
|
|
return None, {"status": "undecodable", "code": "SCD_PAYLOAD_EMPTY"}
|
|
raw = bytes(payload)
|
|
start = raw.find(b"<?xml")
|
|
if start < 0:
|
|
start = raw.find(b"<SchemaFile")
|
|
if start < 0:
|
|
return None, {
|
|
"status": "undecodable",
|
|
"code": "SCD_XML_NOT_FOUND",
|
|
"compression": decoded.get("compression"),
|
|
"raw_bytes": decoded.get("raw_bytes"),
|
|
"payload_bytes": decoded.get("payload_bytes"),
|
|
}
|
|
# 1C appends a binary trailer after the XML document in some releases.
|
|
# ElementTree correctly rejects that trailer, so keep the exact XML range.
|
|
end_marker = b"</SchemaFile>"
|
|
end = raw.find(end_marker, start)
|
|
xml_bytes = raw[start : end + len(end_marker)] if end >= 0 else raw[start:]
|
|
try:
|
|
root = ET.fromstring(xml_bytes.decode("utf-8-sig"))
|
|
except (UnicodeDecodeError, ET.ParseError) as exc:
|
|
return None, {
|
|
"status": "undecodable",
|
|
"code": "SCD_XML_INVALID",
|
|
"message": str(exc),
|
|
"compression": decoded.get("compression"),
|
|
"raw_bytes": decoded.get("raw_bytes"),
|
|
"payload_bytes": decoded.get("payload_bytes"),
|
|
}
|
|
return root, {
|
|
"status": "ok",
|
|
"compression": decoded.get("compression"),
|
|
"raw_bytes": decoded.get("raw_bytes"),
|
|
"payload_bytes": decoded.get("payload_bytes"),
|
|
"xml_offset": start,
|
|
"xml_bytes": len(xml_bytes),
|
|
"xml_root": local_name(root.tag),
|
|
}
|
|
|
|
|
|
def scd_node_item(root: ET.Element, node: ET.Element, category: str) -> dict[str, Any]:
|
|
"""Return only direct, documented XML values for one SCD item."""
|
|
|
|
item_name = child_text(node, "name", "dataPath", "field")
|
|
if not item_name and not list(node):
|
|
item_name = (node.text or "").strip()
|
|
item: dict[str, Any] = {
|
|
"name": item_name,
|
|
"source": {"kind": "scd_xml", "path": node_path(root, node)},
|
|
}
|
|
expression = child_text(node, "expression")
|
|
if expression:
|
|
item["expression"] = expression
|
|
query = child_text(node, "query")
|
|
if query:
|
|
item["query"] = query
|
|
value_type_node = direct_child(node, "valueType")
|
|
if value_type_node is None:
|
|
value_type_node = direct_child(node, "type")
|
|
value_type = ""
|
|
if value_type_node is not None:
|
|
value_type = child_text(value_type_node, "type") or (value_type_node.text or "").strip()
|
|
if value_type:
|
|
item["value_type"] = value_type
|
|
if category == "datasets":
|
|
item["type"] = node.attrib.get("{http://www.w3.org/2001/XMLSchema-instance}type") or node.attrib.get("type") or ""
|
|
return item
|
|
|
|
|
|
def inspect_scd_payload(data: bytes, *, sections: list[str] | None = None) -> dict[str, Any]:
|
|
"""Decode a DataCompositionSchema XML stream from SQL storage.
|
|
|
|
Unknown or absent XML nodes become empty lists. They are deliberately not
|
|
synthesized from report code or form attributes.
|
|
"""
|
|
|
|
requested = sections or ["parameters", "datasets", "fields", "calculated_fields", "resources", "settings", "variants", "total_fields"]
|
|
root, container = xml_from_scd_payload(data)
|
|
if root is None:
|
|
return {"status": "partial", "container": container, "sections": {name: [] for name in requested}}
|
|
schema = next((node for node in root.iter() if local_name(node.tag) == "dataCompositionSchema"), None)
|
|
if schema is None:
|
|
return {
|
|
"status": "partial",
|
|
"container": {**container, "code": "SCD_SCHEMA_NODE_NOT_FOUND"},
|
|
"sections": {name: [] for name in requested},
|
|
}
|
|
node_names = {
|
|
"parameters": {"parameter"},
|
|
"datasets": {"dataSet"},
|
|
"fields": {"field"},
|
|
"calculated_fields": {"calculatedField"},
|
|
"resources": {"resource"},
|
|
"settings": {"settings", "Settings"},
|
|
"variants": {"settingsVariant", "variant"},
|
|
"total_fields": {"totalField"},
|
|
}
|
|
result: dict[str, list[dict[str, Any]]] = {}
|
|
skipped_unnamed: dict[str, int] = {}
|
|
for section in requested:
|
|
names = node_names.get(section)
|
|
if not names:
|
|
result[section] = []
|
|
continue
|
|
raw_items = [scd_node_item(schema, node, section) for node in schema.iter() if local_name(node.tag) in names]
|
|
result[section] = [item for item in raw_items if item.get("name")]
|
|
if len(raw_items) != len(result[section]):
|
|
skipped_unnamed[section] = len(raw_items) - len(result[section])
|
|
declared = [str(item.get("name")) for item in result.get("parameters") or [] if item.get("name")]
|
|
declared_by_normalized = {name.casefold(): name for name in declared}
|
|
query_references: list[dict[str, Any]] = []
|
|
referenced_normalized: set[str] = set()
|
|
for dataset in result.get("datasets") or []:
|
|
references: list[str] = []
|
|
for found in QUERY_PARAMETER_RE.finditer(query_without_line_comments(str(dataset.get("query") or ""))):
|
|
name = found.group(1)
|
|
if name.casefold() not in {value.casefold() for value in references}:
|
|
references.append(name)
|
|
referenced_normalized.add(name.casefold())
|
|
if references:
|
|
query_references.append({"dataset": dataset.get("name"), "parameters": references})
|
|
analysis = {
|
|
"kind": "raw_query_parameter_token_scan",
|
|
"declared_parameters": declared,
|
|
"query_parameter_references": query_references,
|
|
"referenced_not_declared_in_schema": sorted(
|
|
{name for item in query_references for name in item["parameters"] if name.casefold() not in declared_by_normalized},
|
|
key=str.casefold,
|
|
),
|
|
"declared_not_referenced_in_dataset_queries": [name for name in declared if name.casefold() not in referenced_normalized],
|
|
}
|
|
settings_tags = {
|
|
"groupings": {"groupItems", "grouping"},
|
|
"filters": {"selection", "filter"},
|
|
"orders": {"order", "sorting"},
|
|
"conditional_appearance": {"appearance", "conditionalAppearance"},
|
|
}
|
|
settings_context: dict[str, Any] = {"status": "not_present", "sections": {}}
|
|
for context_name, tags in settings_tags.items():
|
|
nodes = [node for node in schema.iter() if local_name(node.tag) in tags]
|
|
if not nodes:
|
|
continue
|
|
records: list[dict[str, Any]] = []
|
|
for node in nodes:
|
|
tokens = []
|
|
for child in node.iter():
|
|
if local_name(child.tag) not in {"field", "dataPath", "left", "right", "group"} or list(child):
|
|
continue
|
|
value = (child.text or "").strip()
|
|
if value and value.casefold() not in {item.casefold() for item in tokens}:
|
|
tokens.append(value)
|
|
if tokens:
|
|
records.append({"path": node_path(schema, node), "tokens": tokens})
|
|
if records:
|
|
settings_context["status"] = "found"
|
|
settings_context["sections"][context_name] = records
|
|
analysis["settings_context"] = settings_context
|
|
query_sources: list[dict[str, Any]] = []
|
|
query_output_aliases: list[dict[str, Any]] = []
|
|
for dataset in result.get("datasets") or []:
|
|
query = query_without_line_comments(str(dataset.get("query") or ""))
|
|
sources = list(dict.fromkeys(match.group(1) for match in QUERY_SOURCE_RE.finditer(query)))
|
|
if sources:
|
|
bindings = []
|
|
for match in QUERY_SOURCE_BINDING_RE.finditer(query):
|
|
source, alias = match.group(1), match.group(2)
|
|
item = {"source": source}
|
|
if alias:
|
|
item["alias"] = alias
|
|
if item not in bindings:
|
|
bindings.append(item)
|
|
query_sources.append({"dataset": dataset.get("name"), "sources": sources, "bindings": bindings})
|
|
select_match = QUERY_SELECT_RE.search(query)
|
|
if select_match:
|
|
aliases = list(dict.fromkeys(match.group(1) for match in QUERY_ALIAS_RE.finditer(select_match.group(1))))
|
|
if aliases:
|
|
query_output_aliases.append({"dataset": dataset.get("name"), "aliases": aliases})
|
|
if query_sources:
|
|
analysis["data_source_references"] = {"kind": "raw_query_source_token_scan", "datasets": query_sources}
|
|
direct_field_references: list[dict[str, Any]] = []
|
|
for dataset in query_sources:
|
|
query = query_without_line_comments(str(next((item.get("query") for item in result.get("datasets") or [] if item.get("name") == dataset.get("dataset")), "")))
|
|
references: list[dict[str, str]] = []
|
|
for binding in dataset.get("bindings") or []:
|
|
alias = str(binding.get("alias") or "")
|
|
if not alias:
|
|
continue
|
|
matcher = re.compile(r"\b" + re.escape(alias) + r"\.([A-Za-z_\u0400-\u04ff][A-Za-z0-9_\u0400-\u04ff]*)(?![A-Za-z0-9_\u0400-\u04ff.])", re.IGNORECASE)
|
|
for match in matcher.finditer(query):
|
|
item = {"alias": alias, "field": match.group(1)}
|
|
if item not in references:
|
|
references.append(item)
|
|
if references:
|
|
direct_field_references.append({"dataset": dataset.get("dataset"), "references": references})
|
|
if direct_field_references:
|
|
analysis["query_direct_field_references"] = {"kind": "direct_alias_field_token_scan", "datasets": direct_field_references}
|
|
field_names = {str(item.get("name")).casefold(): str(item.get("name")) for item in result.get("fields") or [] if item.get("name")}
|
|
calculated_field_names = {str(item.get("name")).casefold(): str(item.get("name")) for item in result.get("calculated_fields") or [] if item.get("name")}
|
|
declared_field_names = {**field_names, **calculated_field_names}
|
|
total_names = [str(item.get("name")) for item in result.get("total_fields") or [] if item.get("name")]
|
|
if total_names:
|
|
analysis["total_field_references"] = {
|
|
"fields": total_names,
|
|
"missing_from_declared_fields": [name for name in total_names if name.casefold() not in declared_field_names],
|
|
"status": "checked" if "fields" in result and "calculated_fields" in result else "field_sections_not_requested",
|
|
}
|
|
if query_output_aliases:
|
|
analysis["query_output_aliases"] = {
|
|
"kind": "select_clause_alias_scan",
|
|
"datasets": query_output_aliases,
|
|
"not_declared_as_scd_fields": sorted(
|
|
{
|
|
alias
|
|
for dataset in query_output_aliases
|
|
for alias in dataset["aliases"]
|
|
if alias.casefold() not in declared_field_names
|
|
},
|
|
key=str.casefold,
|
|
),
|
|
}
|
|
return {
|
|
"status": "ok",
|
|
"container": container,
|
|
"sections": result,
|
|
"analysis": analysis,
|
|
"diagnostics": {"skipped_unnamed_xml_nodes": skipped_unnamed} if skipped_unnamed else {},
|
|
}
|
|
|
|
|
|
def plan_scd_scalar_patch(
|
|
data: bytes,
|
|
*,
|
|
section: str,
|
|
name: str,
|
|
property_name: str,
|
|
value: str,
|
|
) -> dict[str, Any]:
|
|
"""Build a byte-preserving patch for one direct scalar SCD XML property.
|
|
|
|
Only query/expression properties are accepted in this first writer layer.
|
|
The XML element span is collected by Expat from the original byte stream;
|
|
all bytes outside the scalar content stay unchanged, including the 1C
|
|
binary prefix/trailer. No database operation is performed here.
|
|
"""
|
|
|
|
allowed = {
|
|
"datasets": ({"dataSet"}, {"query"}),
|
|
"calculated_fields": ({"calculatedField"}, {"expression"}),
|
|
"resources": ({"resource"}, {"expression"}),
|
|
}
|
|
tags_and_properties = allowed.get(section)
|
|
if not tags_and_properties or property_name not in tags_and_properties[1]:
|
|
return {
|
|
"status": "invalid_argument",
|
|
"code": "SCD_PATCH_PROPERTY_UNSUPPORTED",
|
|
"message": "Only datasets.query, calculated_fields.expression, and resources.expression are writable.",
|
|
}
|
|
root, container = xml_from_scd_payload(data)
|
|
if root is None:
|
|
return {"status": "undecodable", "container": container}
|
|
decoded = decode_payload_lossless(data)
|
|
payload = bytes(decoded["payload"])
|
|
xml_start = payload.find(b"<?xml")
|
|
if xml_start < 0:
|
|
xml_start = payload.find(b"<SchemaFile")
|
|
xml_end_marker = b"</SchemaFile>"
|
|
xml_end = payload.find(xml_end_marker, xml_start)
|
|
if xml_start < 0 or xml_end < 0:
|
|
return {"status": "undecodable", "container": container}
|
|
xml_end += len(xml_end_marker)
|
|
xml = payload[xml_start:xml_end]
|
|
target_tags = tags_and_properties[0]
|
|
stack: list[dict[str, Any]] = []
|
|
records: list[dict[str, Any]] = []
|
|
|
|
def start_element(tag: str, _attrs: dict[str, str]) -> None:
|
|
local = local_name(tag)
|
|
position = parser.CurrentByteIndex
|
|
end = xml.find(b">", position)
|
|
frame: dict[str, Any] = {"tag": local, "depth": len(stack) + 1, "content_start": end + 1}
|
|
if local in target_tags:
|
|
frame["record"] = {"tag": local, "depth": len(stack) + 1, "properties": {}}
|
|
if stack:
|
|
parent_record = next((item.get("record") for item in reversed(stack) if item.get("record")), None)
|
|
if parent_record and len(stack) + 1 == parent_record["depth"] + 1 and local in {"name", "dataPath", property_name}:
|
|
frame["property_record"] = parent_record
|
|
stack.append(frame)
|
|
|
|
def end_element(_tag: str) -> None:
|
|
frame = stack.pop()
|
|
end = parser.CurrentByteIndex
|
|
property_record = frame.get("property_record")
|
|
if property_record is not None:
|
|
raw_text = xml[int(frame["content_start"]):end]
|
|
if b"<" not in raw_text:
|
|
property_record["properties"][frame["tag"]] = {
|
|
"start": int(frame["content_start"]),
|
|
"end": end,
|
|
"text": html.unescape(raw_text.decode("utf-8")),
|
|
}
|
|
record = frame.get("record")
|
|
if record is not None:
|
|
identity = record["properties"].get("name") or record["properties"].get("dataPath")
|
|
record["name"] = identity.get("text") if identity else ""
|
|
records.append(record)
|
|
|
|
parser = expat.ParserCreate()
|
|
parser.StartElementHandler = start_element
|
|
parser.EndElementHandler = end_element
|
|
try:
|
|
parser.Parse(xml, True)
|
|
except expat.ExpatError as exc:
|
|
return {"status": "undecodable", "container": container, "code": "SCD_XML_INVALID", "message": str(exc)}
|
|
matches = [record for record in records if str(record.get("name") or "") == name]
|
|
if not matches:
|
|
return {"status": "not_found", "code": "SCD_PATCH_TARGET_NOT_FOUND", "container": container}
|
|
if len(matches) > 1:
|
|
return {"status": "ambiguous", "code": "SCD_PATCH_TARGET_AMBIGUOUS", "container": container, "matches": len(matches)}
|
|
property_record = (matches[0].get("properties") or {}).get(property_name)
|
|
if not property_record:
|
|
return {"status": "not_found", "code": "SCD_PATCH_PROPERTY_NOT_FOUND", "container": container}
|
|
old = str(property_record["text"])
|
|
if old == value:
|
|
return {"status": "unchanged", "container": container, "old": old, "new": value}
|
|
escaped = html.escape(value, quote=False).encode("utf-8")
|
|
patched_xml = xml[: property_record["start"]] + escaped + xml[property_record["end"] :]
|
|
patched_payload = payload[:xml_start] + patched_xml + payload[xml_end:]
|
|
from .payload import encode_payload_lossless
|
|
patched_data = encode_payload_lossless(decoded, payload=patched_payload)
|
|
return {
|
|
"status": "planned",
|
|
"container": container,
|
|
"old": old,
|
|
"new": value,
|
|
"payload": patched_data,
|
|
"expected_sha1": hashlib.sha1(data).hexdigest(),
|
|
"result_sha1": hashlib.sha1(patched_data).hexdigest(),
|
|
"changed_bytes": len(patched_data) - len(data),
|
|
}
|
|
|
|
|
|
def compare_scd_semantics(active: dict[str, Any], saved: dict[str, Any]) -> dict[str, Any]:
|
|
"""Compare decoded SCD sections by semantic content, never by storage id."""
|
|
|
|
section_names = sorted(set((active.get("sections") or {}).keys()) | set((saved.get("sections") or {}).keys()))
|
|
sections: dict[str, dict[str, Any]] = {}
|
|
counts = {"added": 0, "removed": 0, "changed": 0, "unchanged": 0}
|
|
for section in section_names:
|
|
def index(items: Any) -> dict[str, Any]:
|
|
result: dict[str, Any] = {}
|
|
for ordinal, item in enumerate(items or []):
|
|
if not isinstance(item, dict):
|
|
continue
|
|
key = str(item.get("name") or f"#{ordinal}")
|
|
result[key] = {key: value for key, value in item.items() if key != "source"}
|
|
return result
|
|
active_items, saved_items = index((active.get("sections") or {}).get(section)), index((saved.get("sections") or {}).get(section))
|
|
added = sorted(set(saved_items) - set(active_items), key=str.casefold)
|
|
removed = sorted(set(active_items) - set(saved_items), key=str.casefold)
|
|
changed = sorted([name for name in set(active_items) & set(saved_items) if active_items[name] != saved_items[name]], key=str.casefold)
|
|
unchanged = len(set(active_items) & set(saved_items)) - len(changed)
|
|
sections[section] = {"added": added, "removed": removed, "changed": changed, "unchanged": unchanged}
|
|
counts["added"] += len(added); counts["removed"] += len(removed); counts["changed"] += len(changed); counts["unchanged"] += unchanged
|
|
return {"status": "unchanged" if not any(counts[key] for key in ("added", "removed", "changed")) else "changed", "sections": sections, "counts": counts}
|