Initial SQL-only 1C adapter baseline
This commit is contained in:
@@ -0,0 +1,350 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import re
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
MARKER_RE = re.compile(r"^(\d+)-(\d+)$")
|
||||
|
||||
|
||||
def read_json(path: Path) -> dict[str, Any]:
|
||||
return json.loads(path.read_text(encoding="utf-8"))
|
||||
|
||||
|
||||
def as_int(value: Any) -> int | None:
|
||||
try:
|
||||
return int(value)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
def tree_index(value: Any) -> int:
|
||||
text = str(value or "")
|
||||
if text.startswith("$."):
|
||||
text = text[2:]
|
||||
first = text.split(".", 1)[0]
|
||||
return as_int(first) if as_int(first) is not None else 10**9
|
||||
|
||||
|
||||
def marker_rows_from_payload(payload: dict[str, Any]) -> list[dict[str, Any]]:
|
||||
if isinstance(payload.get("rows"), list):
|
||||
return [row for row in payload["rows"] if isinstance(row, dict)]
|
||||
probe = payload.get("probe") if isinstance(payload.get("probe"), dict) else {}
|
||||
rows = probe.get("marker_matrix") if isinstance(probe.get("marker_matrix"), list) else []
|
||||
result: list[dict[str, Any]] = []
|
||||
for row in rows:
|
||||
if not isinstance(row, dict):
|
||||
continue
|
||||
styles = row.get("styles") if isinstance(row.get("styles"), list) else []
|
||||
style = styles[0] if styles and isinstance(styles[0], dict) else {}
|
||||
next_record = style.get("next_moxel_record") if isinstance(style.get("next_moxel_record"), dict) else {}
|
||||
result.append(
|
||||
{
|
||||
"text": row.get("text"),
|
||||
"expected_row": row.get("expected_row"),
|
||||
"expected_col": row.get("expected_col"),
|
||||
"decoded_row": row.get("decoded_row"),
|
||||
"decoded_col": row.get("decoded_col"),
|
||||
"style_tree_position": style.get("tree_position"),
|
||||
"next_value": next_record.get("value") or next_record.get("head"),
|
||||
"preceding": style.get("last_7_preceding_values") or style.get("immediate_preceding_values"),
|
||||
"row_ok": row.get("row_ok"),
|
||||
"col_ok": row.get("col_ok"),
|
||||
}
|
||||
)
|
||||
return result
|
||||
|
||||
|
||||
def last_preceding_int(row: dict[str, Any]) -> int | None:
|
||||
preceding = row.get("preceding") if isinstance(row.get("preceding"), list) else []
|
||||
for value in reversed(preceding):
|
||||
parsed = as_int(value)
|
||||
if parsed is not None:
|
||||
return parsed
|
||||
return None
|
||||
|
||||
|
||||
def analyze_marker_matrix(payloads: list[dict[str, Any]]) -> dict[str, Any]:
|
||||
rows: list[dict[str, Any]] = []
|
||||
for payload in payloads:
|
||||
rows.extend(marker_rows_from_payload(payload))
|
||||
marker_rows = [row for row in rows if MARKER_RE.match(str(row.get("text") or ""))]
|
||||
total = len(marker_rows)
|
||||
row_ok = sum(1 for row in marker_rows if row.get("row_ok") is True or row.get("decoded_row") == row.get("expected_row"))
|
||||
col_ok = sum(1 for row in marker_rows if row.get("col_ok") is True or row.get("decoded_col") == row.get("expected_col"))
|
||||
|
||||
last_preceding_predictions = []
|
||||
for row in marker_rows:
|
||||
expected_col = as_int(row.get("expected_col"))
|
||||
last = last_preceding_int(row)
|
||||
if expected_col is None or last is None:
|
||||
continue
|
||||
predicted = last + 1
|
||||
last_preceding_predictions.append(
|
||||
{
|
||||
"text": row.get("text"),
|
||||
"style_tree_position": row.get("style_tree_position"),
|
||||
"expected_col": expected_col,
|
||||
"predicted_col": predicted,
|
||||
"ok": predicted == expected_col,
|
||||
"preceding_last": last,
|
||||
"current_decoded_col": row.get("decoded_col"),
|
||||
}
|
||||
)
|
||||
|
||||
prediction_total = len(last_preceding_predictions)
|
||||
prediction_ok = sum(1 for item in last_preceding_predictions if item.get("ok") is True)
|
||||
rows_by_expected_row: dict[int, list[dict[str, Any]]] = {}
|
||||
for row in marker_rows:
|
||||
expected_row = as_int(row.get("expected_row"))
|
||||
if expected_row is None:
|
||||
continue
|
||||
rows_by_expected_row.setdefault(expected_row, []).append(row)
|
||||
row_runs = []
|
||||
for expected_row, run_rows in sorted(rows_by_expected_row.items()):
|
||||
sorted_rows = sorted(run_rows, key=lambda item: tree_index(item.get("style_tree_position")))
|
||||
expected_cols = [as_int(row.get("expected_col")) for row in sorted_rows]
|
||||
predicted_cols = [(last_preceding_int(row) + 1) if last_preceding_int(row) is not None else None for row in sorted_rows]
|
||||
row_runs.append(
|
||||
{
|
||||
"expected_row": expected_row,
|
||||
"texts": [row.get("text") for row in sorted_rows],
|
||||
"style_positions": [row.get("style_tree_position") for row in sorted_rows],
|
||||
"expected_columns": expected_cols,
|
||||
"predicted_columns": predicted_cols,
|
||||
"sequence_ok": predicted_cols == expected_cols,
|
||||
}
|
||||
)
|
||||
|
||||
confidence = "none"
|
||||
if prediction_total and prediction_ok == prediction_total and prediction_total >= 3:
|
||||
confidence = "high"
|
||||
elif prediction_total and prediction_ok / prediction_total >= 0.8:
|
||||
confidence = "medium"
|
||||
elif prediction_total:
|
||||
confidence = "low"
|
||||
|
||||
return {
|
||||
"marker_rows": total,
|
||||
"current_decoder": {
|
||||
"row_accuracy": {"ok": row_ok, "total": total},
|
||||
"column_accuracy": {"ok": col_ok, "total": total},
|
||||
},
|
||||
"candidate_rules": [
|
||||
{
|
||||
"id": "inline_text_column_from_last_preceding_scalar_plus_one",
|
||||
"target": "moxel.inline_text_cell.column",
|
||||
"expression": "one_based_column = int(last_numeric(preceding_scalars)) + 1",
|
||||
"confidence": confidence,
|
||||
"evidence": {"ok": prediction_ok, "total": prediction_total},
|
||||
"samples": last_preceding_predictions[:20],
|
||||
}
|
||||
],
|
||||
"row_runs": row_runs,
|
||||
}
|
||||
|
||||
|
||||
def raw_scalar_diffs(before: list[Any], after: list[Any]) -> list[int]:
|
||||
return [index for index, pair in enumerate(zip(before, after)) if pair[0] != pair[1]]
|
||||
|
||||
|
||||
def infer_named_range_rules(history_matrix: dict[str, Any], property_candidates: dict[str, Any]) -> dict[str, Any]:
|
||||
coordinate_fields: dict[str, set[int]] = {"left": set(), "right": set(), "top": set(), "bottom": set()}
|
||||
evidence: list[dict[str, Any]] = []
|
||||
transitions: list[dict[str, Any]] = []
|
||||
transitions.extend([item for item in history_matrix.get("transitions") or [] if isinstance(item, dict)])
|
||||
transitions.extend([item for item in property_candidates.get("classified_transitions") or [] if isinstance(item, dict)])
|
||||
|
||||
for transition in transitions:
|
||||
named = transition.get("named_range_changes") if isinstance(transition.get("named_range_changes"), dict) else {}
|
||||
if "named_range_diff" in transition and isinstance(transition["named_range_diff"], dict):
|
||||
named = {
|
||||
"one_based": {
|
||||
"before": transition["named_range_diff"].get("before_coords") or {},
|
||||
"after": transition["named_range_diff"].get("after_coords") or {},
|
||||
},
|
||||
"raw_scalars": {
|
||||
"before": transition["named_range_diff"].get("before_raw") or [],
|
||||
"after": transition["named_range_diff"].get("after_raw") or [],
|
||||
},
|
||||
}
|
||||
coords = named.get("one_based") if isinstance(named.get("one_based"), dict) else {}
|
||||
raw = named.get("raw_scalars") if isinstance(named.get("raw_scalars"), dict) else {}
|
||||
before_coords = coords.get("before") if isinstance(coords.get("before"), dict) else {}
|
||||
after_coords = coords.get("after") if isinstance(coords.get("after"), dict) else {}
|
||||
before_raw = raw.get("before") if isinstance(raw.get("before"), list) else []
|
||||
after_raw = raw.get("after") if isinstance(raw.get("after"), list) else []
|
||||
changed = raw_scalar_diffs(before_raw, after_raw)
|
||||
if not changed:
|
||||
continue
|
||||
changed_coord_names = [
|
||||
name
|
||||
for name in ("left", "right", "top", "bottom", "row_start", "row_end", "column_start", "column_end")
|
||||
if before_coords.get(name) != after_coords.get(name)
|
||||
]
|
||||
for name in changed_coord_names:
|
||||
if name in {"left", "column_start"}:
|
||||
coordinate_fields["left"].update(changed)
|
||||
elif name in {"right", "column_end"}:
|
||||
coordinate_fields["right"].update(changed)
|
||||
elif name in {"top", "row_start"}:
|
||||
coordinate_fields["top"].update(changed)
|
||||
elif name in {"bottom", "row_end"}:
|
||||
coordinate_fields["bottom"].update(changed)
|
||||
evidence.append(
|
||||
{
|
||||
"from_file": transition.get("from_file"),
|
||||
"to_file": transition.get("to_file"),
|
||||
"changed_raw_indexes": changed,
|
||||
"changed_coordinates": changed_coord_names,
|
||||
}
|
||||
)
|
||||
|
||||
rules = []
|
||||
for target, indexes in coordinate_fields.items():
|
||||
if not indexes:
|
||||
continue
|
||||
rules.append(
|
||||
{
|
||||
"target": f"moxel.named_range.{target}",
|
||||
"raw_scalar_indexes": sorted(indexes),
|
||||
"expression": "one_based = int(raw_scalar) + 1",
|
||||
"confidence": "high" if len(indexes) == 1 else "medium",
|
||||
}
|
||||
)
|
||||
return {"rules": rules, "evidence": evidence}
|
||||
|
||||
|
||||
def summarize_property_candidates(property_candidates: dict[str, Any]) -> dict[str, Any]:
|
||||
classified = [item for item in property_candidates.get("classified_transitions") or [] if isinstance(item, dict)]
|
||||
label_counts: dict[str, int] = {}
|
||||
for item in classified:
|
||||
for label in item.get("labels") or []:
|
||||
label_counts[str(label)] = label_counts.get(str(label), 0) + 1
|
||||
unmapped = [
|
||||
{
|
||||
"from_file": item.get("from_file"),
|
||||
"to_file": item.get("to_file"),
|
||||
"labels": item.get("labels") or [],
|
||||
"reason": "No stable property field yet; capture one-property-per-save snapshots on the same cell.",
|
||||
}
|
||||
for item in classified
|
||||
if "uncategorized" in (item.get("labels") or []) or "tracked_signature_unchanged" in (item.get("labels") or [])
|
||||
]
|
||||
return {
|
||||
"label_counts": [{"label": key, "count": label_counts[key]} for key in sorted(label_counts)],
|
||||
"unmapped_transitions": unmapped,
|
||||
"recommended_next_steps": property_candidates.get("recommended_next_steps") or [],
|
||||
}
|
||||
|
||||
|
||||
def build_registry(
|
||||
marker_payloads: list[dict[str, Any]],
|
||||
history_matrix: dict[str, Any],
|
||||
property_candidates: dict[str, Any],
|
||||
sources: dict[str, list[str] | str | None],
|
||||
) -> dict[str, Any]:
|
||||
marker_analysis = analyze_marker_matrix(marker_payloads)
|
||||
named_range = infer_named_range_rules(history_matrix, property_candidates)
|
||||
properties = summarize_property_candidates(property_candidates)
|
||||
rules = []
|
||||
rules.extend(marker_analysis.get("candidate_rules") or [])
|
||||
rules.extend(named_range.get("rules") or [])
|
||||
return {
|
||||
"schema": "codex_1c_moxel_schema_discovery.v1",
|
||||
"sources": sources,
|
||||
"status": "ok",
|
||||
"rules": rules,
|
||||
"analysis": {
|
||||
"inline_text_cells": marker_analysis,
|
||||
"named_ranges": named_range,
|
||||
"properties": properties,
|
||||
},
|
||||
"next_automation": [
|
||||
"Capture one-property-per-save probes for alignment, font, border, fill, protection, wrapping, merge, row height, and column width.",
|
||||
"Promote high-confidence rules into the adapter decoder only after fixture tests and a disposable 1C validation run.",
|
||||
"For write support, require a separate round-trip proof for every scalar path before enabling mutation.",
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def render_markdown(payload: dict[str, Any]) -> str:
|
||||
lines: list[str] = []
|
||||
lines.append("# 1C MOXCEL schema discovery")
|
||||
lines.append("")
|
||||
lines.append(f"- Status: `{payload.get('status')}`")
|
||||
lines.append(f"- Rules: `{len(payload.get('rules') or [])}`")
|
||||
inline = ((payload.get("analysis") or {}).get("inline_text_cells") or {})
|
||||
current = inline.get("current_decoder") or {}
|
||||
row_acc = current.get("row_accuracy") or {}
|
||||
col_acc = current.get("column_accuracy") or {}
|
||||
lines.append(f"- Inline row accuracy: `{row_acc.get('ok')}/{row_acc.get('total')}`")
|
||||
lines.append(f"- Inline column accuracy: `{col_acc.get('ok')}/{col_acc.get('total')}`")
|
||||
lines.append("")
|
||||
lines.append("## Rules")
|
||||
lines.append("")
|
||||
lines.append("| Rule | Target | Confidence | Evidence |")
|
||||
lines.append("| --- | --- | --- | --- |")
|
||||
for rule in payload.get("rules") or []:
|
||||
evidence = rule.get("evidence") or {}
|
||||
evidence_text = f"{evidence.get('ok')}/{evidence.get('total')}" if evidence else ", ".join(map(str, rule.get("raw_scalar_indexes") or []))
|
||||
lines.append(f"| `{rule.get('id') or rule.get('target')}` | `{rule.get('target')}` | `{rule.get('confidence')}` | `{evidence_text}` |")
|
||||
lines.append("")
|
||||
lines.append("## Property Gaps")
|
||||
lines.append("")
|
||||
properties = ((payload.get("analysis") or {}).get("properties") or {})
|
||||
for item in properties.get("unmapped_transitions") or []:
|
||||
lines.append(f"- `{item.get('from_file')}` -> `{item.get('to_file')}`: `{', '.join(item.get('labels') or [])}`")
|
||||
if not properties.get("unmapped_transitions"):
|
||||
lines.append("- none")
|
||||
lines.append("")
|
||||
lines.append("## Next Automation")
|
||||
lines.append("")
|
||||
for step in payload.get("next_automation") or []:
|
||||
lines.append(f"- {step}")
|
||||
lines.append("")
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Infer 1C MOXCEL decoder/property rules from captured template probes.")
|
||||
parser.add_argument("--marker-matrix", action="append", default=[], help="Marker matrix JSON. Repeatable.")
|
||||
parser.add_argument("--probe-snapshot", action="append", default=[], help="Probe snapshot JSON from capture_1c_template_probe.py. Repeatable.")
|
||||
parser.add_argument("--history-matrix", help="History matrix JSON from analyze_1c_template_history_matrix.py.")
|
||||
parser.add_argument("--property-candidates", help="Property candidate JSON from infer_1c_template_property_candidates.py.")
|
||||
parser.add_argument("--output-json", default="reports/1c-template-baselines/Primer3_moxel_schema_discovery.json")
|
||||
parser.add_argument("--output-markdown", default="reports/1c-template-baselines/Primer3_moxel_schema_discovery.md")
|
||||
args = parser.parse_args()
|
||||
|
||||
marker_paths = [Path(path) for path in args.marker_matrix]
|
||||
probe_paths = [Path(path) for path in args.probe_snapshot]
|
||||
marker_payloads = [read_json(path) for path in [*marker_paths, *probe_paths]]
|
||||
history_matrix = read_json(Path(args.history_matrix)) if args.history_matrix else {}
|
||||
property_candidates = read_json(Path(args.property_candidates)) if args.property_candidates else {}
|
||||
payload = build_registry(
|
||||
marker_payloads,
|
||||
history_matrix,
|
||||
property_candidates,
|
||||
{
|
||||
"marker_matrix": [str(path) for path in marker_paths],
|
||||
"probe_snapshot": [str(path) for path in probe_paths],
|
||||
"history_matrix": args.history_matrix,
|
||||
"property_candidates": args.property_candidates,
|
||||
},
|
||||
)
|
||||
|
||||
json_path = Path(args.output_json)
|
||||
md_path = Path(args.output_markdown)
|
||||
json_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
md_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
json_path.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
md_path.write_text(render_markdown(payload), encoding="utf-8")
|
||||
print(json.dumps({"status": "ok", "json": str(json_path), "markdown": str(md_path), "rules": len(payload["rules"])}, ensure_ascii=False, indent=2))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Reference in New Issue
Block a user