Initial SQL-only 1C adapter baseline

2026-07-22 03:03:47 +03:00
commit e2503b77e7
545 changed files with 184711 additions and 0 deletions
@@ -0,0 +1,129 @@
+from __future__ import annotations
+
+import json
+from collections import deque
+from pathlib import Path
+
+import yaml
+
+from fetch_1c_its_docs import LinkParser, enqueue_links, merged_policy, page_record
+from normalize_1c_its_docs import TextExtractor, clean_its_text, content_quality
+
+
+ROOT = Path(__file__).resolve().parents[1]
+SOURCES = ROOT / "plugins" / "1c" / "rag" / "official-docs" / "sources.yaml"
+
+
+def assert_true(condition: bool, message: str) -> None:
+    if not condition:
+        raise AssertionError(message)
+
+
+def test_src_priority_and_title_hint() -> None:
+    html = """
+    <html><head><title>Форма :: Глоссарий разработчика</title></head>
+    <body>
+      <iframe src="/db/content/v8devgloss/src/term_000000052.htm"></iframe>
+      <a href="/db/v8devgloss/content/900049/hdoc">1</a>
+      <a href="/db/v8devgloss/content/99/hdoc">XDTO</a>
+    </body></html>
+    """
+    parser = LinkParser()
+    parser.feed(html)
+    config = yaml.safe_load(SOURCES.read_text(encoding="utf-8"))
+    source = config["sources"][0]
+    policy = merged_policy(config["default_policy"], source.get("policy"))
+    queue: deque = deque()
+    enqueue_links(
+        queue,
+        source=source,
+        base_url="https://its.1c.ru/db/v8devgloss/content/52/hdoc",
+        depth=1,
+        policy=policy,
+        links=parser.links,
+        seen=set(),
+        title_hint=parser.title,
+    )
+    first = queue[0]
+    assert_true(first[1] == "https://its.1c.ru/db/content/v8devgloss/src/term_000000052.htm", "src iframe URL must be first")
+    assert_true(first[4] == "Форма :: Глоссарий разработчика", "src iframe URL must inherit hdoc title")
+
+
+def test_src_only_at_max_depth() -> None:
+    html = """
+    <html><head><title>Форма :: Глоссарий разработчика</title></head>
+    <body>
+      <iframe src="/db/content/v8devgloss/src/term_000000052.htm"></iframe>
+      <a href="/db/v8devgloss/content/99/hdoc">XDTO</a>
+      <a href="/db/v8devgloss/content/14/hdoc">1С:Предприятие</a>
+    </body></html>
+    """
+    parser = LinkParser()
+    parser.feed(html)
+    config = yaml.safe_load(SOURCES.read_text(encoding="utf-8"))
+    source = config["sources"][0]
+    policy = merged_policy(config["default_policy"], source.get("policy"))
+    queue: deque = deque()
+    enqueue_links(
+        queue,
+        source=source,
+        base_url="https://its.1c.ru/db/v8devgloss/content/52/hdoc",
+        depth=2,
+        policy=policy,
+        links=parser.links,
+        seen=set(),
+        title_hint=parser.title,
+        src_only=True,
+    )
+    urls = [item[1] for item in queue]
+    assert_true(urls == ["https://its.1c.ru/db/content/v8devgloss/src/term_000000052.htm"], "max-depth src-only mode must keep only iframe src")
+
+
+def test_page_record_uses_title_hint() -> None:
+    output_dir = ROOT / "reports" / ".tmp-its-ingestion-check"
+    output_dir.mkdir(parents=True, exist_ok=True)
+    record = page_record(
+        source={"id": "v8devgloss", "title": "Глоссарий разработчика", "source_type": "official_1c_its_glossary"},
+        url="https://its.1c.ru/db/content/v8devgloss/src/term_000000052.htm",
+        depth=2,
+        body="<html><body>Текст определения формы.</body></html>".encode("utf-8"),
+        headers={"Content-Type": "text/html; charset=utf-8"},
+        status=200,
+        output_dir=output_dir,
+        title_hint="Форма :: Глоссарий разработчика",
+    )
+    assert_true(record["title"] == "Форма :: Глоссарий разработчика", "title_hint must be used when src page has no title")
+
+
+def test_glossary_src_fallback_text() -> None:
+    extractor = TextExtractor()
+    extractor.feed(
+        "<html><body><p>"
+        "Форма предназначена для отображения и редактирования данных объекта, "
+        "содержит элементы управления, команды, реквизиты формы и обработчики событий, "
+        "которые используются прикладным решением при работе пользователя в интерфейсе приложения."
+        "</p></body></html>"
+    )
+    text = clean_its_text(extractor.text(), "Форма :: Глоссарий разработчика", "official_1c_its_glossary")
+    quality = content_quality(text, "Форма :: Глоссарий разработчика")
+    assert_true("Форма предназначена" in text, "glossary src body must be preserved")
+    assert_true(bool(quality["is_content"]), "glossary src body must pass quality gate")
+
+
+def main() -> int:
+    checks = [
+        test_src_priority_and_title_hint,
+        test_src_only_at_max_depth,
+        test_page_record_uses_title_hint,
+        test_glossary_src_fallback_text,
+    ]
+    results = []
+    for check in checks:
+        check()
+        results.append({"id": check.__name__, "status": "passed"})
+    print(json.dumps({"schema": "onec_its_ingestion_logic_check.v1", "status": "ok", "checks": results}, ensure_ascii=False, indent=2))
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())