m/llm
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Initial project import

Browse Source
This commit is contained in:
Mikhail
2026-08-14 09:40:51 +03:00
parent 00040e5ce4
commit d7099bf80d
146 changed files with 30509 additions and 1055 deletions
Download Patch File Download Diff File Expand all files Collapse all files
plugins/1c/parser/cas_payload.py
+223 -3
View File
@@ -25,6 +25,11 @@ def normalized_text_sha1(text: str) -> str:
return hashlib.sha1(normalized.encode("utf-8")).hexdigest()
def _text_with_line_ending(text: str, line_ending: str) -> str:
"""Normalize caller text first, then render it in a stream's convention."""
return str(text or "").replace("\r\n", "\n").replace("\r", "\n").replace("\n", line_ending)
def decode_text(data: bytes) -> tuple[str | None, str | None]:
if data.startswith(b"\xef\xbb\xbf"):
try:
@@ -121,6 +126,188 @@ def stream_blocks_with_data(payload: bytes, *, limit: int = 100) -> list[dict[st
return blocks
def structural_stream_blocks_with_data(payload: bytes, *, limit: int = 100) -> list[dict[str, Any]]:
"""Read one contiguous stream chain without matching headers inside data.
Some 1C stream payloads legitimately contain the ASCII sequence used by a
stream header inside a binary member. ``stream_blocks_with_data`` remains
a discovery heuristic for legacy readers; this function follows only the
next header located exactly at the previous member's end and is suitable
for evidence-bearing module decoding.
"""
blocks: list[dict[str, Any]] = []
first = STREAM_HEADER_RE.search(payload)
if first is None:
return blocks
match = first
while match is not None and len(blocks) < limit:
declared_1 = int(match.group(1), 16)
declared_2 = int(match.group(2), 16)
data_offset = match.end()
data_end = data_offset + declared_2
if declared_2 <= 0 or data_end > len(payload):
break
data = payload[data_offset:data_end]
text, encoding = decode_text(data)
blocks.append(
{
"header_offset": match.start(),
"header_end": match.end(),
"data_offset": data_offset,
"data_end": data_end,
"declared_1": declared_1,
"declared_2": declared_2,
"bytes": len(data),
"sha1": sha1_hex(data),
"encoding": encoding,
"text": text,
"data": data,
"structural": True,
}
)
match = STREAM_HEADER_RE.match(payload, data_end)
return blocks
def extract_structural_stream_blocks(payload: bytes, *, include_text: bool = False, limit: int = 100) -> list[dict[str, Any]]:
"""Public structural stream view with the same shape as discovery blocks."""
result: list[dict[str, Any]] = []
for block in structural_stream_blocks_with_data(payload, limit=limit):
text = str(block.get("text") or "")
clean = text.replace("\x00", "")
item = {
**{key: value for key, value in block.items() if key not in {"text", "data"}},
"text_preview": clean[:500],
"has_bsl_marker": bool(text and any(marker in clean for marker in BSL_MARKERS)),
"has_html_marker": bool(text and any(marker in clean for marker in HTML_MARKERS)),
}
if include_text:
item["text"] = block.get("text")
result.append(item)
return result
def decode_declared_utf8_bsl_prefix(payload: bytes, stream_index: int) -> dict[str, Any]:
"""Decode a BSL prefix whose byte length is declared by a stream header.
Object-module containers observed in 1C keep the editable UTF-8 BSL bytes
in the first ``declared_1`` bytes of a fixed-size member. The remaining
member bytes are opaque platform metadata, not source text. This helper
is read-only evidence; it intentionally does not construct replacements.
"""
blocks = structural_stream_blocks_with_data(payload)
if stream_index < 0 or stream_index >= len(blocks):
return {"status": "not_found", "error": "stream_index_not_found"}
block = blocks[stream_index]
data = bytes(block["data"])
prefix_bytes = int(block["declared_1"])
if prefix_bytes <= 0 or prefix_bytes > len(data):
return {
"status": "unsupported",
"error": "invalid_declared_bsl_prefix_length",
"declared_1": prefix_bytes,
"member_bytes": len(data),
}
prefix = data[:prefix_bytes]
if not prefix.startswith(b"\xef\xbb\xbf"):
return {
"status": "unsupported",
"error": "declared_bsl_prefix_not_utf8_bom",
"declared_1": prefix_bytes,
"member_bytes": len(data),
}
try:
text = prefix.decode("utf-8-sig", errors="strict")
except UnicodeDecodeError as exc:
return {"status": "unsupported", "error": "declared_bsl_prefix_decode_error", "diagnostics": {"message": str(exc)}}
return {
"status": "ok",
"text": text,
"stream_index": stream_index,
"header_offset": block["header_offset"],
"data_offset": block["data_offset"],
"bsl_prefix_bytes": prefix_bytes,
"opaque_tail_bytes": len(data) - prefix_bytes,
"member_bytes": len(data),
"text_sha1": normalized_text_sha1(text),
"structural": True,
}
def replace_declared_utf8_bsl_prefix_same_width(
payload: bytes,
stream_index: int,
*,
text: str,
expected_text_sha1: str | None = None,
) -> tuple[bytes, dict[str, Any]]:
"""Replace a proven fixed-width BSL prefix without touching its tail.
This is deliberately narrower than ``replace_stream_block``. The report
object-module carrier has a fixed-size stream member whose first declared
bytes are UTF-8 source and whose remaining bytes are opaque. A shorter
source is right-padded with spaces *inside the declared source field*;
longer source is rejected. Consequently the member, every following
stream, and the opaque tail stay byte-for-byte identical.
It does not attempt to synthesize the platform's independent version
atoms. The caller remains responsible for the proven paired
``__configinfo`` SHA-1 update.
"""
decoded = decode_declared_utf8_bsl_prefix(payload, stream_index)
if decoded.get("status") != "ok":
raise ValueError(str(decoded.get("error") or "declared_bsl_prefix_unavailable"))
old_text = str(decoded["text"])
if not is_declared_utf8_bsl_source(old_text):
raise ValueError("declared_bsl_prefix_is_not_bsl_source")
old_sha1 = normalized_text_sha1(old_text)
if expected_text_sha1 and expected_text_sha1.lower() != old_sha1:
raise ValueError("expected_text_sha1 does not match declared BSL prefix")
line_ending = "\r\n" if "\r\n" in old_text else "\r" if "\r" in old_text else "\n"
rendered = _text_with_line_ending(text, line_ending)
encoded = b"\xef\xbb\xbf" + rendered.encode("utf-8")
prefix_bytes = int(decoded["bsl_prefix_bytes"])
if len(encoded) > prefix_bytes:
raise ValueError("replacement_declared_bsl_prefix_exceeds_fixed_width")
# BSL whitespace outside string literals is semantically inert. Padding
# is restricted to the fixed source field and is observable in readback.
padded = encoded + (b" " * (prefix_bytes - len(encoded)))
if len(padded) != prefix_bytes:
raise AssertionError("declared BSL prefix width changed")
data_offset = int(decoded["data_offset"])
new_payload = payload[:data_offset] + padded + payload[data_offset + prefix_bytes :]
if payload[data_offset + prefix_bytes :] != new_payload[data_offset + prefix_bytes :]:
raise AssertionError("opaque member tail changed")
return new_payload, {
"stream_index": stream_index,
"mode": "declared_utf8_bsl_prefix_same_width",
"old_text_sha1": old_sha1,
"new_text_sha1": normalized_text_sha1(rendered),
"old_bsl_prefix_bytes": prefix_bytes,
"new_bsl_source_bytes": len(encoded),
"padding_bytes": prefix_bytes - len(encoded),
"opaque_tail_bytes": int(decoded["opaque_tail_bytes"]),
"opaque_tail_preserved": True,
"old_text_preview": old_text[:500],
"new_text_preview": rendered[:500],
}
def is_declared_utf8_bsl_source(text: str) -> bool:
"""Recognize source evidence in a declared UTF-8 stream prefix.
A module may legitimately consist solely of comments, while other stream
members can also have a UTF-8 prefix (for example a brace descriptor).
The prefix is BSL evidence only when it has a normal BSL marker or every
nonblank source line is a BSL line comment.
"""
source = str(text or "").lstrip("\ufeff")
if any(marker in source for marker in BSL_MARKERS):
return True
lines = [line.strip() for line in source.replace("\r\n", "\n").replace("\r", "\n").split("\n") if line.strip()]
return bool(lines) and all(line.startswith("//") for line in lines)
def stream_header(size: int) -> bytes:
if size < 0 or size > 0xFFFFFFFF:
raise ValueError("stream size is outside 8-hex header range")
@@ -159,9 +346,20 @@ def replace_stream_block(
if not old:
raise ValueError("replace.old is required")
count = int(replace.get("count") or 1)
if old not in old_text:
# Public code.read normalizes BSL to LF while streams often retain
# CRLF. Treat that representation difference as irrelevant, but do
# not loosen matching of any other character (spaces/tabs remain
# exact). The replacement is rendered back in the stream's original
# line-ending convention to avoid unrelated formatting churn.
line_ending = "\r\n" if "\r\n" in old_text else "\r" if "\r" in old_text else "\n"
source_old = old
source_new = _text_with_line_ending(new, line_ending) if ("\n" in new or "\r" in new) else new
if source_old not in old_text:
source_old = _text_with_line_ending(old, line_ending)
source_new = _text_with_line_ending(new, line_ending)
if source_old not in old_text:
raise ValueError("replace.old was not found in stream text")
text = old_text.replace(old, new, count)
text = old_text.replace(source_old, source_new, count)
routine_edit = None
if routine is not None:
if old_text is None:
@@ -261,7 +459,29 @@ def classify_payload(data: bytes, *, include_text: bool = False, include_tree: b
decoded = decode_payload_lossless(data)
payload = decoded.get("payload") if isinstance(decoded.get("payload"), (bytes, bytearray)) else b""
markers = payload_markers(bytes(payload))
stream_blocks = extract_stream_blocks(bytes(payload), include_text=include_text)
# Prefer proven contiguous boundaries for normal container decoding. Keep
# the regex scan only as a discovery fallback for legacy irregular blobs.
stream_blocks = extract_structural_stream_blocks(bytes(payload), include_text=include_text)
if not stream_blocks and "stream_headers" in markers:
stream_blocks = extract_stream_blocks(bytes(payload), include_text=include_text)
# A report object module observed in ConfigCAS stores source in the
# declared UTF-8 prefix of a fixed-size stream member. The remainder is
# opaque platform state and must never be exposed as BSL. Keep this as
# read-only evidence: replacement still requires a separately proven
# reverse codec for that carrier.
if stream_blocks:
for stream_index, stream in enumerate(stream_blocks):
declared_prefix = decode_declared_utf8_bsl_prefix(bytes(payload), stream_index)
if declared_prefix.get("status") != "ok" or not is_declared_utf8_bsl_source(str(declared_prefix.get("text") or "")):
continue
stream["declared_utf8_bsl_prefix"] = {
key: declared_prefix[key]
for key in ("bsl_prefix_bytes", "opaque_tail_bytes", "member_bytes", "text_sha1", "structural")
if key in declared_prefix
}
if include_text:
stream["text"] = declared_prefix["text"]
stream["text_preview"] = str(declared_prefix["text"] or "").replace("\x00", "")[:500]
text = decoded.get("text")
tree = None
root = None