280 lines
9.2 KiB
Python
280 lines
9.2 KiB
Python
"""Evidence-based structured metadata projection for Config object payloads."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from dataclasses import asdict, dataclass
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
from .child_records import collect_evidence, declared_child_records
|
|
from .config_object import find_identity
|
|
from .payload import GUID_RE, parse_payload_file, root_signature, scalar
|
|
from .xml_metadata import XmlMetadataItem, extract_xml_metadata_items, group_xml_items
|
|
|
|
|
|
CATEGORY_FIELDS = {
|
|
"Attribute": "attributes",
|
|
"TabularSection": "tabular_sections",
|
|
"Dimension": "dimensions",
|
|
"Resource": "resources",
|
|
"Form": "forms",
|
|
"Template": "templates",
|
|
"Command": "commands",
|
|
"AddressingAttribute": "addressing_attributes",
|
|
"AccountingFlag": "accounting_flags",
|
|
"Column": "columns",
|
|
"EnumValue": "enum_values",
|
|
"IntegrationServiceChannel": "integration_service_channels",
|
|
"Operation": "operations",
|
|
"URLTemplate": "url_templates",
|
|
}
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class MetadataItemEvidence:
|
|
category: str
|
|
name: str
|
|
synonym: str
|
|
uuid: str | None
|
|
value_type: dict[str, Any] | None
|
|
parent_category: str | None
|
|
parent_name: str | None
|
|
parent_uuid: str | None
|
|
section_path: str
|
|
record_path: str | None
|
|
record_index: int | None
|
|
evidence: dict[str, bool]
|
|
|
|
def to_dict(self) -> dict[str, Any]:
|
|
return asdict(self)
|
|
|
|
|
|
def _children(node: Any) -> list[Any]:
|
|
if isinstance(node, dict) and node.get("type") in {"list", "sequence"}:
|
|
return node.get("items") or []
|
|
return []
|
|
|
|
|
|
def get_by_path(tree: Any, path: str) -> Any | None:
|
|
node = tree
|
|
if path == "":
|
|
return node
|
|
for part in path.split("."):
|
|
if not isinstance(node, dict) or node.get("type") not in {"list", "sequence"}:
|
|
return None
|
|
items = node.get("items") or []
|
|
index = int(part)
|
|
if index < 0 or index >= len(items):
|
|
return None
|
|
node = items[index]
|
|
return node
|
|
|
|
|
|
def section_rules(summary: dict[str, Any], kind: str, min_support_ratio: float) -> list[dict[str, Any]]:
|
|
rules = []
|
|
for section in summary.get("sections") or []:
|
|
if section.get("kind") != kind:
|
|
continue
|
|
category = section.get("candidate_semantic")
|
|
if not category or category == kind:
|
|
continue
|
|
support_ratio = float(section.get("candidate_support_ratio") or 0)
|
|
if support_ratio < min_support_ratio:
|
|
continue
|
|
rules.append(
|
|
{
|
|
"path": section["path"],
|
|
"category": category,
|
|
"support_ratio": support_ratio,
|
|
"sample_count": section.get("sample_count"),
|
|
}
|
|
)
|
|
return rules
|
|
|
|
|
|
def item_evidence(
|
|
item: XmlMetadataItem,
|
|
evidence: dict[str, set[str]],
|
|
section_path: str,
|
|
*,
|
|
record_path: str | None = None,
|
|
record_index: int | None = None,
|
|
) -> MetadataItemEvidence | None:
|
|
strings = evidence["strings"]
|
|
guids = evidence["guids"]
|
|
hits = {
|
|
"name": bool(item.name and item.name in strings),
|
|
"synonym": bool(item.synonym and item.synonym in strings),
|
|
"uuid": bool(item.uuid and item.uuid.lower() in guids),
|
|
}
|
|
if not any(hits.values()):
|
|
return None
|
|
return MetadataItemEvidence(
|
|
category=item.category,
|
|
name=item.name,
|
|
synonym=item.synonym,
|
|
uuid=item.uuid.lower() if item.uuid else None,
|
|
value_type=item.value_type,
|
|
parent_category=item.parent_category,
|
|
parent_name=item.parent_name,
|
|
parent_uuid=item.parent_uuid.lower() if item.parent_uuid else None,
|
|
section_path=section_path,
|
|
record_path=record_path,
|
|
record_index=record_index,
|
|
evidence=hits,
|
|
)
|
|
|
|
|
|
def best_item_record_match(item: XmlMetadataItem, records: list[Any], section_path: str) -> MetadataItemEvidence | None:
|
|
best: MetadataItemEvidence | None = None
|
|
best_score = -1
|
|
for record in records:
|
|
match = item_evidence(
|
|
item,
|
|
record.evidence,
|
|
section_path,
|
|
record_path=record.path,
|
|
record_index=record.index,
|
|
)
|
|
if not match:
|
|
continue
|
|
score = int(match.evidence["uuid"]) * 4 + int(match.evidence["name"]) * 2 + int(match.evidence["synonym"])
|
|
if score > best_score:
|
|
best = match
|
|
best_score = score
|
|
return best
|
|
|
|
|
|
def items_for_parent(grouped: dict[str, list[XmlMetadataItem]], category: str, parent_uuid: str | None) -> list[XmlMetadataItem]:
|
|
return [
|
|
item
|
|
for item in grouped.get(category, [])
|
|
if (item.parent_uuid or "").lower() == (parent_uuid or "").lower()
|
|
]
|
|
|
|
|
|
def declared_record_containers(node: Any, path: str, *, max_depth: int = 3, include_root: bool = True) -> list[list[Any]]:
|
|
result = []
|
|
|
|
def walk(value: Any, current_path: str, depth: int) -> None:
|
|
records = declared_child_records(value, current_path)
|
|
if records and (include_root or depth > 0):
|
|
result.append(records)
|
|
if depth >= max_depth:
|
|
return
|
|
for index, child in enumerate(_children(value)):
|
|
walk(child, f"{current_path}.{index}", depth + 1)
|
|
|
|
walk(node, path, 0)
|
|
return result
|
|
|
|
|
|
def nested_tabular_attributes(
|
|
tree: Any,
|
|
tabular_section_item: dict[str, Any],
|
|
grouped: dict[str, list[XmlMetadataItem]],
|
|
) -> list[dict[str, Any]]:
|
|
parent_uuid = tabular_section_item.get("uuid")
|
|
record_path = tabular_section_item.get("record_path")
|
|
if not parent_uuid or not record_path:
|
|
return []
|
|
node = get_by_path(tree, record_path)
|
|
if node is None:
|
|
return []
|
|
xml_items = items_for_parent(grouped, "Attribute", parent_uuid)
|
|
if not xml_items:
|
|
return []
|
|
containers = declared_record_containers(node, record_path, include_root=False)
|
|
all_records = [record for records in containers for record in records]
|
|
result = []
|
|
for item in xml_items:
|
|
match = best_item_record_match(item, all_records, record_path)
|
|
if match:
|
|
data = match.to_dict()
|
|
data["tabular_section_name"] = tabular_section_item.get("name")
|
|
data["tabular_section_uuid"] = parent_uuid
|
|
result.append(data)
|
|
result.sort(key=lambda item: (item["tabular_section_name"] or "", item["name"], item.get("uuid") or ""))
|
|
return result
|
|
|
|
|
|
def parse_structured_metadata(
|
|
config_file: Path,
|
|
xml_file: Path,
|
|
kind: str,
|
|
category_summary: dict[str, Any],
|
|
*,
|
|
min_support_ratio: float = 1.0,
|
|
) -> dict[str, Any]:
|
|
parsed = parse_payload_file(config_file)
|
|
tree = parsed.get("tree")
|
|
identity = find_identity(tree)
|
|
xml_items = extract_xml_metadata_items(xml_file)
|
|
grouped = group_xml_items(xml_items)
|
|
rules = section_rules(category_summary, kind, min_support_ratio)
|
|
|
|
result: dict[str, Any] = {
|
|
"schema": "onec_structured_metadata_projection.v1",
|
|
"kind": kind,
|
|
"config_file": str(config_file),
|
|
"xml_file": str(xml_file),
|
|
"root": root_signature(tree),
|
|
"identity": identity.to_dict() if identity else None,
|
|
"min_support_ratio": min_support_ratio,
|
|
"rules": rules,
|
|
"attributes": [],
|
|
"tabular_sections": [],
|
|
"tabular_section_attributes": [],
|
|
"dimensions": [],
|
|
"resources": [],
|
|
"forms": [],
|
|
"templates": [],
|
|
"commands": [],
|
|
"addressing_attributes": [],
|
|
"accounting_flags": [],
|
|
"columns": [],
|
|
"enum_values": [],
|
|
"integration_service_channels": [],
|
|
"operations": [],
|
|
"url_templates": [],
|
|
"unmapped_rules": [],
|
|
"record_boundary_rules": [],
|
|
}
|
|
|
|
for rule in rules:
|
|
category = rule["category"]
|
|
field = CATEGORY_FIELDS.get(category)
|
|
node = get_by_path(tree, rule["path"]) if tree else None
|
|
if not field or node is None:
|
|
result["unmapped_rules"].append(rule)
|
|
continue
|
|
records = declared_child_records(node, rule["path"])
|
|
if records:
|
|
result["record_boundary_rules"].append(
|
|
{
|
|
"path": rule["path"],
|
|
"category": category,
|
|
"declared_record_count": len(records),
|
|
"record_paths_sample": [record.path for record in records[:10]],
|
|
}
|
|
)
|
|
section_evidence = collect_evidence(node)
|
|
matched = []
|
|
for item in items_for_parent(grouped, category, identity.guid if identity else None):
|
|
item_match = best_item_record_match(item, records, rule["path"]) if records else None
|
|
if not item_match:
|
|
item_match = item_evidence(item, section_evidence, rule["path"])
|
|
if item_match:
|
|
matched.append(item_match.to_dict())
|
|
matched.sort(key=lambda item: (item["name"], item.get("uuid") or ""))
|
|
result[field].extend(matched)
|
|
if category == "TabularSection":
|
|
for tabular_section in matched:
|
|
result["tabular_section_attributes"].extend(nested_tabular_attributes(tree, tabular_section, grouped))
|
|
|
|
result["counts"] = {
|
|
field: len(result[field])
|
|
for field in sorted({*set(CATEGORY_FIELDS.values()), "tabular_section_attributes"})
|
|
}
|
|
return result
|