130 lines
5.1 KiB
Python
130 lines
5.1 KiB
Python
from __future__ import annotations
|
||
|
||
import json
|
||
from collections import deque
|
||
from pathlib import Path
|
||
|
||
import yaml
|
||
|
||
from fetch_1c_its_docs import LinkParser, enqueue_links, merged_policy, page_record
|
||
from normalize_1c_its_docs import TextExtractor, clean_its_text, content_quality
|
||
|
||
|
||
ROOT = Path(__file__).resolve().parents[1]
|
||
SOURCES = ROOT / "plugins" / "1c" / "rag" / "official-docs" / "sources.yaml"
|
||
|
||
|
||
def assert_true(condition: bool, message: str) -> None:
|
||
if not condition:
|
||
raise AssertionError(message)
|
||
|
||
|
||
def test_src_priority_and_title_hint() -> None:
|
||
html = """
|
||
<html><head><title>Форма :: Глоссарий разработчика</title></head>
|
||
<body>
|
||
<iframe src="/db/content/v8devgloss/src/term_000000052.htm"></iframe>
|
||
<a href="/db/v8devgloss/content/900049/hdoc">1</a>
|
||
<a href="/db/v8devgloss/content/99/hdoc">XDTO</a>
|
||
</body></html>
|
||
"""
|
||
parser = LinkParser()
|
||
parser.feed(html)
|
||
config = yaml.safe_load(SOURCES.read_text(encoding="utf-8"))
|
||
source = config["sources"][0]
|
||
policy = merged_policy(config["default_policy"], source.get("policy"))
|
||
queue: deque = deque()
|
||
enqueue_links(
|
||
queue,
|
||
source=source,
|
||
base_url="https://its.1c.ru/db/v8devgloss/content/52/hdoc",
|
||
depth=1,
|
||
policy=policy,
|
||
links=parser.links,
|
||
seen=set(),
|
||
title_hint=parser.title,
|
||
)
|
||
first = queue[0]
|
||
assert_true(first[1] == "https://its.1c.ru/db/content/v8devgloss/src/term_000000052.htm", "src iframe URL must be first")
|
||
assert_true(first[4] == "Форма :: Глоссарий разработчика", "src iframe URL must inherit hdoc title")
|
||
|
||
|
||
def test_src_only_at_max_depth() -> None:
|
||
html = """
|
||
<html><head><title>Форма :: Глоссарий разработчика</title></head>
|
||
<body>
|
||
<iframe src="/db/content/v8devgloss/src/term_000000052.htm"></iframe>
|
||
<a href="/db/v8devgloss/content/99/hdoc">XDTO</a>
|
||
<a href="/db/v8devgloss/content/14/hdoc">1С:Предприятие</a>
|
||
</body></html>
|
||
"""
|
||
parser = LinkParser()
|
||
parser.feed(html)
|
||
config = yaml.safe_load(SOURCES.read_text(encoding="utf-8"))
|
||
source = config["sources"][0]
|
||
policy = merged_policy(config["default_policy"], source.get("policy"))
|
||
queue: deque = deque()
|
||
enqueue_links(
|
||
queue,
|
||
source=source,
|
||
base_url="https://its.1c.ru/db/v8devgloss/content/52/hdoc",
|
||
depth=2,
|
||
policy=policy,
|
||
links=parser.links,
|
||
seen=set(),
|
||
title_hint=parser.title,
|
||
src_only=True,
|
||
)
|
||
urls = [item[1] for item in queue]
|
||
assert_true(urls == ["https://its.1c.ru/db/content/v8devgloss/src/term_000000052.htm"], "max-depth src-only mode must keep only iframe src")
|
||
|
||
|
||
def test_page_record_uses_title_hint() -> None:
|
||
output_dir = ROOT / "reports" / ".tmp-its-ingestion-check"
|
||
output_dir.mkdir(parents=True, exist_ok=True)
|
||
record = page_record(
|
||
source={"id": "v8devgloss", "title": "Глоссарий разработчика", "source_type": "official_1c_its_glossary"},
|
||
url="https://its.1c.ru/db/content/v8devgloss/src/term_000000052.htm",
|
||
depth=2,
|
||
body="<html><body>Текст определения формы.</body></html>".encode("utf-8"),
|
||
headers={"Content-Type": "text/html; charset=utf-8"},
|
||
status=200,
|
||
output_dir=output_dir,
|
||
title_hint="Форма :: Глоссарий разработчика",
|
||
)
|
||
assert_true(record["title"] == "Форма :: Глоссарий разработчика", "title_hint must be used when src page has no title")
|
||
|
||
|
||
def test_glossary_src_fallback_text() -> None:
|
||
extractor = TextExtractor()
|
||
extractor.feed(
|
||
"<html><body><p>"
|
||
"Форма предназначена для отображения и редактирования данных объекта, "
|
||
"содержит элементы управления, команды, реквизиты формы и обработчики событий, "
|
||
"которые используются прикладным решением при работе пользователя в интерфейсе приложения."
|
||
"</p></body></html>"
|
||
)
|
||
text = clean_its_text(extractor.text(), "Форма :: Глоссарий разработчика", "official_1c_its_glossary")
|
||
quality = content_quality(text, "Форма :: Глоссарий разработчика")
|
||
assert_true("Форма предназначена" in text, "glossary src body must be preserved")
|
||
assert_true(bool(quality["is_content"]), "glossary src body must pass quality gate")
|
||
|
||
|
||
def main() -> int:
|
||
checks = [
|
||
test_src_priority_and_title_hint,
|
||
test_src_only_at_max_depth,
|
||
test_page_record_uses_title_hint,
|
||
test_glossary_src_fallback_text,
|
||
]
|
||
results = []
|
||
for check in checks:
|
||
check()
|
||
results.append({"id": check.__name__, "status": "passed"})
|
||
print(json.dumps({"schema": "onec_its_ingestion_logic_check.v1", "status": "ok", "checks": results}, ensure_ascii=False, indent=2))
|
||
return 0
|
||
|
||
|
||
if __name__ == "__main__":
|
||
raise SystemExit(main())
|