"""Classify 1C Config/ConfigCAS payload parts without infobase-specific names.""" from __future__ import annotations import base64 import hashlib import re from typing import Any from .payload import collect_strings, decode_payload_lossless, encode_text, parse_brace_text, root_signature, scalar BASE64_RE = re.compile(r"[A-Za-z0-9+/]{40,}={0,2}") BSL_MARKERS = ("&На", "Процедура ", "Функция ", "#Область", "#КонецОбласти") HTML_MARKERS = (" str: return hashlib.sha1(data).hexdigest() def normalized_text_sha1(text: str) -> str: normalized = str(text or "").replace("\r\n", "\n").replace("\r", "\n") return hashlib.sha1(normalized.encode("utf-8")).hexdigest() def _text_with_line_ending(text: str, line_ending: str) -> str: """Normalize caller text first, then render it in a stream's convention.""" return str(text or "").replace("\r\n", "\n").replace("\r", "\n").replace("\n", line_ending) def decode_text(data: bytes) -> tuple[str | None, str | None]: if data.startswith(b"\xef\xbb\xbf"): try: return data.decode("utf-8-sig"), "utf-8-sig" except UnicodeDecodeError: pass candidates = ("utf-8-sig", "utf-8", "utf-16-le", "utf-16-be", "cp1251") best: tuple[str | None, str | None, int] = (None, None, -1) for encoding in candidates: try: text = data.decode(encoding) except UnicodeDecodeError: continue sample = text[:20000] marker_score = sum(500 for marker in (*BSL_MARKERS, *HTML_MARKERS) if marker in sample) printable = sum(1 for char in sample if char.isprintable() or char in "\r\n\t") score = printable + marker_score - sample.count("\x00") * 10 if score > best[2]: best = (text, encoding, score) return best[0], best[1] def payload_markers(payload: bytes) -> list[str]: markers = [] if payload.startswith(b"MOXCEL"): markers.append("MOXCEL") if payload.startswith(b"\xef\xbb\xbf") or b"\xef\xbb\xbf" in payload[:256]: markers.append("utf8_bom") if STREAM_HEADER_RE.search(payload): markers.append("stream_headers") return markers def extract_stream_blocks(payload: bytes, *, include_text: bool = False, limit: int = 100) -> list[dict[str, Any]]: blocks: list[dict[str, Any]] = [] for match in STREAM_HEADER_RE.finditer(payload): declared_1 = int(match.group(1), 16) declared_2 = int(match.group(2), 16) start = match.end() size = declared_2 if size <= 0 or start + size > len(payload): continue data = payload[start : start + size] text, encoding = decode_text(data) clean = (text or "").replace("\x00", "") item: dict[str, Any] = { "header_offset": match.start(), "data_offset": start, "declared_1": declared_1, "declared_2": declared_2, "bytes": len(data), "sha1": sha1_hex(data), "encoding": encoding, "text_preview": clean[:500], "has_bsl_marker": bool(text and any(marker in clean for marker in BSL_MARKERS)), "has_html_marker": bool(text and any(marker in clean for marker in HTML_MARKERS)), } if include_text: item["text"] = text blocks.append(item) if len(blocks) >= limit: break return blocks def stream_blocks_with_data(payload: bytes, *, limit: int = 100) -> list[dict[str, Any]]: blocks: list[dict[str, Any]] = [] for match in STREAM_HEADER_RE.finditer(payload): declared_1 = int(match.group(1), 16) declared_2 = int(match.group(2), 16) start = match.end() size = declared_2 if size <= 0 or start + size > len(payload): continue data = payload[start : start + size] text, encoding = decode_text(data) blocks.append( { "header_offset": match.start(), "header_end": match.end(), "data_offset": start, "data_end": start + size, "declared_1": declared_1, "declared_2": declared_2, "bytes": len(data), "sha1": sha1_hex(data), "encoding": encoding, "text": text, "data": data, } ) if len(blocks) >= limit: break return blocks def structural_stream_blocks_with_data(payload: bytes, *, limit: int = 100) -> list[dict[str, Any]]: """Read one contiguous stream chain without matching headers inside data. Some 1C stream payloads legitimately contain the ASCII sequence used by a stream header inside a binary member. ``stream_blocks_with_data`` remains a discovery heuristic for legacy readers; this function follows only the next header located exactly at the previous member's end and is suitable for evidence-bearing module decoding. """ blocks: list[dict[str, Any]] = [] first = STREAM_HEADER_RE.search(payload) if first is None: return blocks match = first while match is not None and len(blocks) < limit: declared_1 = int(match.group(1), 16) declared_2 = int(match.group(2), 16) data_offset = match.end() data_end = data_offset + declared_2 if declared_2 <= 0 or data_end > len(payload): break data = payload[data_offset:data_end] text, encoding = decode_text(data) blocks.append( { "header_offset": match.start(), "header_end": match.end(), "data_offset": data_offset, "data_end": data_end, "declared_1": declared_1, "declared_2": declared_2, "bytes": len(data), "sha1": sha1_hex(data), "encoding": encoding, "text": text, "data": data, "structural": True, } ) match = STREAM_HEADER_RE.match(payload, data_end) return blocks def extract_structural_stream_blocks(payload: bytes, *, include_text: bool = False, limit: int = 100) -> list[dict[str, Any]]: """Public structural stream view with the same shape as discovery blocks.""" result: list[dict[str, Any]] = [] for block in structural_stream_blocks_with_data(payload, limit=limit): text = str(block.get("text") or "") clean = text.replace("\x00", "") item = { **{key: value for key, value in block.items() if key not in {"text", "data"}}, "text_preview": clean[:500], "has_bsl_marker": bool(text and any(marker in clean for marker in BSL_MARKERS)), "has_html_marker": bool(text and any(marker in clean for marker in HTML_MARKERS)), } if include_text: item["text"] = block.get("text") result.append(item) return result def decode_declared_utf8_bsl_prefix(payload: bytes, stream_index: int) -> dict[str, Any]: """Decode a BSL prefix whose byte length is declared by a stream header. Object-module containers observed in 1C keep the editable UTF-8 BSL bytes in the first ``declared_1`` bytes of a fixed-size member. The remaining member bytes are opaque platform metadata, not source text. This helper is read-only evidence; it intentionally does not construct replacements. """ blocks = structural_stream_blocks_with_data(payload) if stream_index < 0 or stream_index >= len(blocks): return {"status": "not_found", "error": "stream_index_not_found"} block = blocks[stream_index] data = bytes(block["data"]) prefix_bytes = int(block["declared_1"]) if prefix_bytes <= 0 or prefix_bytes > len(data): return { "status": "unsupported", "error": "invalid_declared_bsl_prefix_length", "declared_1": prefix_bytes, "member_bytes": len(data), } prefix = data[:prefix_bytes] if not prefix.startswith(b"\xef\xbb\xbf"): return { "status": "unsupported", "error": "declared_bsl_prefix_not_utf8_bom", "declared_1": prefix_bytes, "member_bytes": len(data), } try: text = prefix.decode("utf-8-sig", errors="strict") except UnicodeDecodeError as exc: return {"status": "unsupported", "error": "declared_bsl_prefix_decode_error", "diagnostics": {"message": str(exc)}} return { "status": "ok", "text": text, "stream_index": stream_index, "header_offset": block["header_offset"], "data_offset": block["data_offset"], "bsl_prefix_bytes": prefix_bytes, "opaque_tail_bytes": len(data) - prefix_bytes, "member_bytes": len(data), "text_sha1": normalized_text_sha1(text), "structural": True, } def replace_declared_utf8_bsl_prefix_same_width( payload: bytes, stream_index: int, *, text: str, expected_text_sha1: str | None = None, ) -> tuple[bytes, dict[str, Any]]: """Replace a proven fixed-width BSL prefix without touching its tail. This is deliberately narrower than ``replace_stream_block``. The report object-module carrier has a fixed-size stream member whose first declared bytes are UTF-8 source and whose remaining bytes are opaque. A shorter source is right-padded with spaces *inside the declared source field*; longer source is rejected. Consequently the member, every following stream, and the opaque tail stay byte-for-byte identical. It does not attempt to synthesize the platform's independent version atoms. The caller remains responsible for the proven paired ``__configinfo`` SHA-1 update. """ decoded = decode_declared_utf8_bsl_prefix(payload, stream_index) if decoded.get("status") != "ok": raise ValueError(str(decoded.get("error") or "declared_bsl_prefix_unavailable")) old_text = str(decoded["text"]) if not is_declared_utf8_bsl_source(old_text): raise ValueError("declared_bsl_prefix_is_not_bsl_source") old_sha1 = normalized_text_sha1(old_text) if expected_text_sha1 and expected_text_sha1.lower() != old_sha1: raise ValueError("expected_text_sha1 does not match declared BSL prefix") line_ending = "\r\n" if "\r\n" in old_text else "\r" if "\r" in old_text else "\n" rendered = _text_with_line_ending(text, line_ending) encoded = b"\xef\xbb\xbf" + rendered.encode("utf-8") prefix_bytes = int(decoded["bsl_prefix_bytes"]) if len(encoded) > prefix_bytes: raise ValueError("replacement_declared_bsl_prefix_exceeds_fixed_width") # BSL whitespace outside string literals is semantically inert. Padding # is restricted to the fixed source field and is observable in readback. padded = encoded + (b" " * (prefix_bytes - len(encoded))) if len(padded) != prefix_bytes: raise AssertionError("declared BSL prefix width changed") data_offset = int(decoded["data_offset"]) new_payload = payload[:data_offset] + padded + payload[data_offset + prefix_bytes :] if payload[data_offset + prefix_bytes :] != new_payload[data_offset + prefix_bytes :]: raise AssertionError("opaque member tail changed") return new_payload, { "stream_index": stream_index, "mode": "declared_utf8_bsl_prefix_same_width", "old_text_sha1": old_sha1, "new_text_sha1": normalized_text_sha1(rendered), "old_bsl_prefix_bytes": prefix_bytes, "new_bsl_source_bytes": len(encoded), "padding_bytes": prefix_bytes - len(encoded), "opaque_tail_bytes": int(decoded["opaque_tail_bytes"]), "opaque_tail_preserved": True, "old_text_preview": old_text[:500], "new_text_preview": rendered[:500], } def is_declared_utf8_bsl_source(text: str) -> bool: """Recognize source evidence in a declared UTF-8 stream prefix. A module may legitimately consist solely of comments, while other stream members can also have a UTF-8 prefix (for example a brace descriptor). The prefix is BSL evidence only when it has a normal BSL marker or every nonblank source line is a BSL line comment. """ source = str(text or "").lstrip("\ufeff") if any(marker in source for marker in BSL_MARKERS): return True lines = [line.strip() for line in source.replace("\r\n", "\n").replace("\r", "\n").split("\n") if line.strip()] return bool(lines) and all(line.startswith("//") for line in lines) def stream_header(size: int) -> bytes: if size < 0 or size > 0xFFFFFFFF: raise ValueError("stream size is outside 8-hex header range") encoded = f"{size:08x}" return f"\r\n{encoded} {encoded} 7fffffff \r\n".encode("ascii") def replace_stream_block( payload: bytes, stream_index: int, *, text: str | None = None, data: bytes | None = None, replace: dict[str, Any] | None = None, routine: dict[str, Any] | None = None, expected_contains: str | None = None, expected_text_sha1: str | None = None, ) -> tuple[bytes, dict[str, Any]]: blocks = stream_blocks_with_data(payload) if stream_index < 0 or stream_index >= len(blocks): raise IndexError(f"stream_index {stream_index} is outside {len(blocks)} stream blocks") block = blocks[stream_index] old_data = bytes(block["data"]) old_text = block.get("text") encoding = block.get("encoding") if expected_contains and (old_text is None or expected_contains not in old_text): raise ValueError("expected_contains was not found in stream text") old_text_sha1 = normalized_text_sha1(old_text or "") if old_text is not None else None if expected_text_sha1 and (old_text_sha1 is None or expected_text_sha1.lower() != old_text_sha1): raise ValueError("expected_text_sha1 does not match current stream text") if replace is not None: if old_text is None: raise ValueError("stream text is not decodable") old = str(replace.get("old") or "") new = str(replace.get("new") or "") if not old: raise ValueError("replace.old is required") count = int(replace.get("count") or 1) # Public code.read normalizes BSL to LF while streams often retain # CRLF. Treat that representation difference as irrelevant, but do # not loosen matching of any other character (spaces/tabs remain # exact). The replacement is rendered back in the stream's original # line-ending convention to avoid unrelated formatting churn. line_ending = "\r\n" if "\r\n" in old_text else "\r" if "\r" in old_text else "\n" source_old = old source_new = _text_with_line_ending(new, line_ending) if ("\n" in new or "\r" in new) else new if source_old not in old_text: source_old = _text_with_line_ending(old, line_ending) source_new = _text_with_line_ending(new, line_ending) if source_old not in old_text: raise ValueError("replace.old was not found in stream text") text = old_text.replace(source_old, source_new, count) routine_edit = None if routine is not None: if old_text is None: raise ValueError("stream text is not decodable") from .bsl_validation import replace_routine_text text, routine_edit = replace_routine_text( old_text, str(routine.get("text") or ""), operation=str(routine.get("operation") or "replace"), name=str(routine.get("name")) if routine.get("name") else None, expected_old_sha1=str(routine.get("expected_old_sha1")) if routine.get("expected_old_sha1") else None, expected_old_contains=str(routine.get("expected_old_contains")) if routine.get("expected_old_contains") else None, ) if data is None: if text is None: raise ValueError("text, data, replace, or routine is required") data = encode_text(text, encoding) header = stream_header(len(data)) new_payload = payload[: block["header_offset"]] + header + data + payload[block["data_end"] :] return new_payload, { "stream_index": stream_index, "encoding": encoding, "old_sha1": sha1_hex(old_data), "new_sha1": sha1_hex(data), "old_text_sha1": old_text_sha1, "new_text_sha1": normalized_text_sha1(text) if text is not None else None, "old_bytes": len(old_data), "new_bytes": len(data), "old_text_preview": (old_text or "")[:500], "new_text_preview": (text or "")[:500] if text is not None else None, **({"routine": routine_edit} if routine_edit else {}), } def collect_base64_blocks(value: Any) -> list[str]: blocks: list[str] = [] def walk(node: Any) -> None: if isinstance(node, dict) and node.get("type") == "list": items = node.get("items") or [] if items and scalar(items[0]) == "#base64": chunks = [scalar(item) for item in items[1:] if BASE64_RE.fullmatch(scalar(item))] if chunks: blocks.append("".join(chunks)) for child in items: walk(child) walk(value) return blocks def decode_base64_blocks(blocks: list[str], *, include_text: bool = False, limit: int = 50) -> list[dict[str, Any]]: result: list[dict[str, Any]] = [] for value in blocks[:limit]: try: data = base64.b64decode(value, validate=True) except Exception: continue text, encoding = decode_text(data) clean = (text or "").replace("\x00", "") item: dict[str, Any] = { "block_length": len(value), "bytes": len(data), "sha1": sha1_hex(data), "encoding": encoding, "text_preview": clean[:500], "has_bsl_marker": bool(text and any(marker in clean for marker in BSL_MARKERS)), "has_html_marker": bool(text and any(marker in clean for marker in HTML_MARKERS)), } if include_text: item["text"] = text result.append(item) return result def classify_role(root_marker: str | None, markers: list[str], text: str | None, stream_blocks: list[dict[str, Any]], base64_blocks: list[dict[str, Any]]) -> str: clean = (text or "").replace("\x00", "") if root_marker == "1": return "metadata_payload" if root_marker == "4": return "form_payload" if root_marker == "5" or any(block.get("has_html_marker") for block in base64_blocks): return "help_or_html_payload" if root_marker == "8" or "MOXCEL" in markers: return "template_payload" if any(block.get("has_bsl_marker") for block in stream_blocks) or any(marker in clean for marker in BSL_MARKERS): return "bsl_module_payload" if "stream_headers" in markers: return "stream_container" if root_marker: return "brace_payload" return "binary_or_unknown_payload" def classify_payload(data: bytes, *, include_text: bool = False, include_tree: bool = False) -> dict[str, Any]: decoded = decode_payload_lossless(data) payload = decoded.get("payload") if isinstance(decoded.get("payload"), (bytes, bytearray)) else b"" markers = payload_markers(bytes(payload)) # Prefer proven contiguous boundaries for normal container decoding. Keep # the regex scan only as a discovery fallback for legacy irregular blobs. stream_blocks = extract_structural_stream_blocks(bytes(payload), include_text=include_text) if not stream_blocks and "stream_headers" in markers: stream_blocks = extract_stream_blocks(bytes(payload), include_text=include_text) # A report object module observed in ConfigCAS stores source in the # declared UTF-8 prefix of a fixed-size stream member. The remainder is # opaque platform state and must never be exposed as BSL. Keep this as # read-only evidence: replacement still requires a separately proven # reverse codec for that carrier. if stream_blocks: for stream_index, stream in enumerate(stream_blocks): declared_prefix = decode_declared_utf8_bsl_prefix(bytes(payload), stream_index) if declared_prefix.get("status") != "ok" or not is_declared_utf8_bsl_source(str(declared_prefix.get("text") or "")): continue stream["declared_utf8_bsl_prefix"] = { key: declared_prefix[key] for key in ("bsl_prefix_bytes", "opaque_tail_bytes", "member_bytes", "text_sha1", "structural") if key in declared_prefix } if include_text: stream["text"] = declared_prefix["text"] stream["text_preview"] = str(declared_prefix["text"] or "").replace("\x00", "")[:500] text = decoded.get("text") tree = None root = None base64_decoded: list[dict[str, Any]] = [] strings_sample: list[str] = [] if text and "{" in text and "stream_headers" not in markers: try: tree = parse_brace_text(text) root = root_signature(tree) strings_sample = collect_strings(tree, limit=80) base64_decoded = decode_base64_blocks(collect_base64_blocks(tree), include_text=include_text) except Exception: tree = None root_marker = root.get("root_marker") if isinstance(root, dict) else None result: dict[str, Any] = { "status": "ok" if payload else "undecodable", "compression": decoded.get("compression"), "encoding": decoded.get("encoding"), "raw_bytes": decoded.get("raw_bytes"), "payload_bytes": decoded.get("payload_bytes"), "sha1": sha1_hex(data), "payload_sha1": sha1_hex(bytes(payload)), "markers": markers, "root": root, "role": classify_role(root_marker, markers, text, stream_blocks, base64_decoded), "strings_sample": strings_sample, "stream_blocks": stream_blocks, "base64_blocks": base64_decoded, "counts": { "stream_blocks": len(stream_blocks), "base64_blocks": len(base64_decoded), "strings_sample": len(strings_sample), }, } if include_text: result["text"] = text if include_tree: result["tree"] = tree return result