from __future__ import annotations import argparse import json import re from pathlib import Path from typing import Any MARKER_RE = re.compile(r"^(\d+)-(\d+)$") def read_json(path: Path) -> dict[str, Any]: return json.loads(path.read_text(encoding="utf-8")) def as_int(value: Any) -> int | None: try: return int(value) except (TypeError, ValueError): return None def tree_index(value: Any) -> int: text = str(value or "") if text.startswith("$."): text = text[2:] first = text.split(".", 1)[0] return as_int(first) if as_int(first) is not None else 10**9 def marker_rows_from_payload(payload: dict[str, Any]) -> list[dict[str, Any]]: if isinstance(payload.get("rows"), list): return [row for row in payload["rows"] if isinstance(row, dict)] probe = payload.get("probe") if isinstance(payload.get("probe"), dict) else {} rows = probe.get("marker_matrix") if isinstance(probe.get("marker_matrix"), list) else [] result: list[dict[str, Any]] = [] for row in rows: if not isinstance(row, dict): continue styles = row.get("styles") if isinstance(row.get("styles"), list) else [] style = styles[0] if styles and isinstance(styles[0], dict) else {} next_record = style.get("next_moxel_record") if isinstance(style.get("next_moxel_record"), dict) else {} result.append( { "text": row.get("text"), "expected_row": row.get("expected_row"), "expected_col": row.get("expected_col"), "decoded_row": row.get("decoded_row"), "decoded_col": row.get("decoded_col"), "style_tree_position": style.get("tree_position"), "next_value": next_record.get("value") or next_record.get("head"), "preceding": style.get("last_7_preceding_values") or style.get("immediate_preceding_values"), "row_ok": row.get("row_ok"), "col_ok": row.get("col_ok"), } ) return result def last_preceding_int(row: dict[str, Any]) -> int | None: preceding = row.get("preceding") if isinstance(row.get("preceding"), list) else [] for value in reversed(preceding): parsed = as_int(value) if parsed is not None: return parsed return None def analyze_marker_matrix(payloads: list[dict[str, Any]]) -> dict[str, Any]: rows: list[dict[str, Any]] = [] for payload in payloads: rows.extend(marker_rows_from_payload(payload)) marker_rows = [row for row in rows if MARKER_RE.match(str(row.get("text") or ""))] total = len(marker_rows) row_ok = sum(1 for row in marker_rows if row.get("row_ok") is True or row.get("decoded_row") == row.get("expected_row")) col_ok = sum(1 for row in marker_rows if row.get("col_ok") is True or row.get("decoded_col") == row.get("expected_col")) last_preceding_predictions = [] for row in marker_rows: expected_col = as_int(row.get("expected_col")) last = last_preceding_int(row) if expected_col is None or last is None: continue predicted = last + 1 last_preceding_predictions.append( { "text": row.get("text"), "style_tree_position": row.get("style_tree_position"), "expected_col": expected_col, "predicted_col": predicted, "ok": predicted == expected_col, "preceding_last": last, "current_decoded_col": row.get("decoded_col"), } ) prediction_total = len(last_preceding_predictions) prediction_ok = sum(1 for item in last_preceding_predictions if item.get("ok") is True) rows_by_expected_row: dict[int, list[dict[str, Any]]] = {} for row in marker_rows: expected_row = as_int(row.get("expected_row")) if expected_row is None: continue rows_by_expected_row.setdefault(expected_row, []).append(row) row_runs = [] for expected_row, run_rows in sorted(rows_by_expected_row.items()): sorted_rows = sorted(run_rows, key=lambda item: tree_index(item.get("style_tree_position"))) expected_cols = [as_int(row.get("expected_col")) for row in sorted_rows] predicted_cols = [(last_preceding_int(row) + 1) if last_preceding_int(row) is not None else None for row in sorted_rows] row_runs.append( { "expected_row": expected_row, "texts": [row.get("text") for row in sorted_rows], "style_positions": [row.get("style_tree_position") for row in sorted_rows], "expected_columns": expected_cols, "predicted_columns": predicted_cols, "sequence_ok": predicted_cols == expected_cols, } ) confidence = "none" if prediction_total and prediction_ok == prediction_total and prediction_total >= 3: confidence = "high" elif prediction_total and prediction_ok / prediction_total >= 0.8: confidence = "medium" elif prediction_total: confidence = "low" return { "marker_rows": total, "current_decoder": { "row_accuracy": {"ok": row_ok, "total": total}, "column_accuracy": {"ok": col_ok, "total": total}, }, "candidate_rules": [ { "id": "inline_text_column_from_last_preceding_scalar_plus_one", "target": "moxel.inline_text_cell.column", "expression": "one_based_column = int(last_numeric(preceding_scalars)) + 1", "confidence": confidence, "evidence": {"ok": prediction_ok, "total": prediction_total}, "samples": last_preceding_predictions[:20], } ], "row_runs": row_runs, } def raw_scalar_diffs(before: list[Any], after: list[Any]) -> list[int]: return [index for index, pair in enumerate(zip(before, after)) if pair[0] != pair[1]] def infer_named_range_rules(history_matrix: dict[str, Any], property_candidates: dict[str, Any]) -> dict[str, Any]: coordinate_fields: dict[str, set[int]] = {"left": set(), "right": set(), "top": set(), "bottom": set()} evidence: list[dict[str, Any]] = [] transitions: list[dict[str, Any]] = [] transitions.extend([item for item in history_matrix.get("transitions") or [] if isinstance(item, dict)]) transitions.extend([item for item in property_candidates.get("classified_transitions") or [] if isinstance(item, dict)]) for transition in transitions: named = transition.get("named_range_changes") if isinstance(transition.get("named_range_changes"), dict) else {} if "named_range_diff" in transition and isinstance(transition["named_range_diff"], dict): named = { "one_based": { "before": transition["named_range_diff"].get("before_coords") or {}, "after": transition["named_range_diff"].get("after_coords") or {}, }, "raw_scalars": { "before": transition["named_range_diff"].get("before_raw") or [], "after": transition["named_range_diff"].get("after_raw") or [], }, } coords = named.get("one_based") if isinstance(named.get("one_based"), dict) else {} raw = named.get("raw_scalars") if isinstance(named.get("raw_scalars"), dict) else {} before_coords = coords.get("before") if isinstance(coords.get("before"), dict) else {} after_coords = coords.get("after") if isinstance(coords.get("after"), dict) else {} before_raw = raw.get("before") if isinstance(raw.get("before"), list) else [] after_raw = raw.get("after") if isinstance(raw.get("after"), list) else [] changed = raw_scalar_diffs(before_raw, after_raw) if not changed: continue changed_coord_names = [ name for name in ("left", "right", "top", "bottom", "row_start", "row_end", "column_start", "column_end") if before_coords.get(name) != after_coords.get(name) ] for name in changed_coord_names: if name in {"left", "column_start"}: coordinate_fields["left"].update(changed) elif name in {"right", "column_end"}: coordinate_fields["right"].update(changed) elif name in {"top", "row_start"}: coordinate_fields["top"].update(changed) elif name in {"bottom", "row_end"}: coordinate_fields["bottom"].update(changed) evidence.append( { "from_file": transition.get("from_file"), "to_file": transition.get("to_file"), "changed_raw_indexes": changed, "changed_coordinates": changed_coord_names, } ) rules = [] for target, indexes in coordinate_fields.items(): if not indexes: continue rules.append( { "target": f"moxel.named_range.{target}", "raw_scalar_indexes": sorted(indexes), "expression": "one_based = int(raw_scalar) + 1", "confidence": "high" if len(indexes) == 1 else "medium", } ) return {"rules": rules, "evidence": evidence} def summarize_property_candidates(property_candidates: dict[str, Any]) -> dict[str, Any]: classified = [item for item in property_candidates.get("classified_transitions") or [] if isinstance(item, dict)] label_counts: dict[str, int] = {} for item in classified: for label in item.get("labels") or []: label_counts[str(label)] = label_counts.get(str(label), 0) + 1 unmapped = [ { "from_file": item.get("from_file"), "to_file": item.get("to_file"), "labels": item.get("labels") or [], "reason": "No stable property field yet; capture one-property-per-save snapshots on the same cell.", } for item in classified if "uncategorized" in (item.get("labels") or []) or "tracked_signature_unchanged" in (item.get("labels") or []) ] return { "label_counts": [{"label": key, "count": label_counts[key]} for key in sorted(label_counts)], "unmapped_transitions": unmapped, "recommended_next_steps": property_candidates.get("recommended_next_steps") or [], } def build_registry( marker_payloads: list[dict[str, Any]], history_matrix: dict[str, Any], property_candidates: dict[str, Any], sources: dict[str, list[str] | str | None], ) -> dict[str, Any]: marker_analysis = analyze_marker_matrix(marker_payloads) named_range = infer_named_range_rules(history_matrix, property_candidates) properties = summarize_property_candidates(property_candidates) rules = [] rules.extend(marker_analysis.get("candidate_rules") or []) rules.extend(named_range.get("rules") or []) return { "schema": "codex_1c_moxel_schema_discovery.v1", "sources": sources, "status": "ok", "rules": rules, "analysis": { "inline_text_cells": marker_analysis, "named_ranges": named_range, "properties": properties, }, "next_automation": [ "Capture one-property-per-save probes for alignment, font, border, fill, protection, wrapping, merge, row height, and column width.", "Promote high-confidence rules into the adapter decoder only after fixture tests and a disposable 1C validation run.", "For write support, require a separate round-trip proof for every scalar path before enabling mutation.", ], } def render_markdown(payload: dict[str, Any]) -> str: lines: list[str] = [] lines.append("# 1C MOXCEL schema discovery") lines.append("") lines.append(f"- Status: `{payload.get('status')}`") lines.append(f"- Rules: `{len(payload.get('rules') or [])}`") inline = ((payload.get("analysis") or {}).get("inline_text_cells") or {}) current = inline.get("current_decoder") or {} row_acc = current.get("row_accuracy") or {} col_acc = current.get("column_accuracy") or {} lines.append(f"- Inline row accuracy: `{row_acc.get('ok')}/{row_acc.get('total')}`") lines.append(f"- Inline column accuracy: `{col_acc.get('ok')}/{col_acc.get('total')}`") lines.append("") lines.append("## Rules") lines.append("") lines.append("| Rule | Target | Confidence | Evidence |") lines.append("| --- | --- | --- | --- |") for rule in payload.get("rules") or []: evidence = rule.get("evidence") or {} evidence_text = f"{evidence.get('ok')}/{evidence.get('total')}" if evidence else ", ".join(map(str, rule.get("raw_scalar_indexes") or [])) lines.append(f"| `{rule.get('id') or rule.get('target')}` | `{rule.get('target')}` | `{rule.get('confidence')}` | `{evidence_text}` |") lines.append("") lines.append("## Property Gaps") lines.append("") properties = ((payload.get("analysis") or {}).get("properties") or {}) for item in properties.get("unmapped_transitions") or []: lines.append(f"- `{item.get('from_file')}` -> `{item.get('to_file')}`: `{', '.join(item.get('labels') or [])}`") if not properties.get("unmapped_transitions"): lines.append("- none") lines.append("") lines.append("## Next Automation") lines.append("") for step in payload.get("next_automation") or []: lines.append(f"- {step}") lines.append("") return "\n".join(lines) def main() -> int: parser = argparse.ArgumentParser(description="Infer 1C MOXCEL decoder/property rules from captured template probes.") parser.add_argument("--marker-matrix", action="append", default=[], help="Marker matrix JSON. Repeatable.") parser.add_argument("--probe-snapshot", action="append", default=[], help="Probe snapshot JSON from capture_1c_template_probe.py. Repeatable.") parser.add_argument("--history-matrix", help="History matrix JSON from analyze_1c_template_history_matrix.py.") parser.add_argument("--property-candidates", help="Property candidate JSON from infer_1c_template_property_candidates.py.") parser.add_argument("--output-json", default="reports/1c-template-baselines/Primer3_moxel_schema_discovery.json") parser.add_argument("--output-markdown", default="reports/1c-template-baselines/Primer3_moxel_schema_discovery.md") args = parser.parse_args() marker_paths = [Path(path) for path in args.marker_matrix] probe_paths = [Path(path) for path in args.probe_snapshot] marker_payloads = [read_json(path) for path in [*marker_paths, *probe_paths]] history_matrix = read_json(Path(args.history_matrix)) if args.history_matrix else {} property_candidates = read_json(Path(args.property_candidates)) if args.property_candidates else {} payload = build_registry( marker_payloads, history_matrix, property_candidates, { "marker_matrix": [str(path) for path in marker_paths], "probe_snapshot": [str(path) for path in probe_paths], "history_matrix": args.history_matrix, "property_candidates": args.property_candidates, }, ) json_path = Path(args.output_json) md_path = Path(args.output_markdown) json_path.parent.mkdir(parents=True, exist_ok=True) md_path.parent.mkdir(parents=True, exist_ok=True) json_path.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8") md_path.write_text(render_markdown(payload), encoding="utf-8") print(json.dumps({"status": "ok", "json": str(json_path), "markdown": str(md_path), "rules": len(payload["rules"])}, ensure_ascii=False, indent=2)) return 0 if __name__ == "__main__": raise SystemExit(main())