Initial SQL-only 1C adapter baseline
This commit is contained in:
@@ -0,0 +1,673 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import html
|
||||
import json
|
||||
import re
|
||||
import urllib.parse
|
||||
from html.parser import HTMLParser
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from one_c_its_platform import merge_platform_metadata
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
DEFAULT_RAW_DIR = ROOT / "plugins" / "1c" / "rag" / "official-docs" / "raw"
|
||||
DEFAULT_RAW_MANIFEST = DEFAULT_RAW_DIR / "manifest.json"
|
||||
DEFAULT_OUTPUT_DIR = ROOT / "plugins" / "1c" / "rag" / "official-docs" / "normalized"
|
||||
DEFAULT_RAG_SOURCE_DIR = ROOT / "plugins" / "1c" / "rag" / "sources" / "official" / "its"
|
||||
DEFAULT_MANIFEST_OUTPUT = DEFAULT_OUTPUT_DIR / "manifest.json"
|
||||
|
||||
|
||||
BLOCK_TAGS = {
|
||||
"address",
|
||||
"article",
|
||||
"aside",
|
||||
"blockquote",
|
||||
"br",
|
||||
"div",
|
||||
"dl",
|
||||
"fieldset",
|
||||
"figcaption",
|
||||
"figure",
|
||||
"footer",
|
||||
"form",
|
||||
"h1",
|
||||
"h2",
|
||||
"h3",
|
||||
"h4",
|
||||
"h5",
|
||||
"h6",
|
||||
"header",
|
||||
"hr",
|
||||
"li",
|
||||
"main",
|
||||
"nav",
|
||||
"ol",
|
||||
"p",
|
||||
"pre",
|
||||
"section",
|
||||
"table",
|
||||
"td",
|
||||
"th",
|
||||
"tr",
|
||||
"ul",
|
||||
}
|
||||
|
||||
|
||||
BOILERPLATE_LINES = {
|
||||
"Вход",
|
||||
"Об 1С:ИТС",
|
||||
"Тест-драйв",
|
||||
"Заказать ИТС",
|
||||
"Задать вопрос",
|
||||
"Обновить ПО",
|
||||
"Оценить 1С",
|
||||
"Купить кассу",
|
||||
"Тематические подборки",
|
||||
"Календарь бухгалтера",
|
||||
"Калькуляторы",
|
||||
"Подбор КБК",
|
||||
"Последние результаты поиска",
|
||||
"Подписаться на рассылку",
|
||||
"Главная",
|
||||
"Инструкции по разработке на 1С",
|
||||
"Платформа 1С:Предприятие. Документация",
|
||||
"Содержание",
|
||||
"Результаты поиска",
|
||||
"Вконтакте",
|
||||
"Принимаю",
|
||||
"Методические материалы для разработчиков и администраторов 1С",
|
||||
"Глоссарий разработчика",
|
||||
"Назад",
|
||||
}
|
||||
|
||||
BOILERPLATE_PREFIXES = (
|
||||
"© Фирма «1С»",
|
||||
"Информационная система 1С:ИТС",
|
||||
"Инструкции по учету в программах 1С",
|
||||
"Новости1С:Лекторий",
|
||||
"Мы используем файлы cookie",
|
||||
"Продолжая находиться на сайте",
|
||||
"на условиях, указанных по ссылке",
|
||||
)
|
||||
|
||||
NAVIGATION_ONLY_TERMS = {
|
||||
"Руководство разработчика",
|
||||
"Руководство администратора",
|
||||
"Клиент-серверный вариант. Руководство администратора",
|
||||
"Руководство разработчика. Обычный режим",
|
||||
"Руководство пользователя",
|
||||
}
|
||||
|
||||
VERSION_SELECTOR_LINE_RE = re.compile(r"^(?:\d+\.\d+(?:\.\d+)?){4,}$")
|
||||
|
||||
|
||||
def charset_from_content_type(content_type: str) -> str | None:
|
||||
match = re.search(r"charset=([^;\s]+)", content_type, flags=re.IGNORECASE)
|
||||
if match:
|
||||
return match.group(1).strip("\"'")
|
||||
return None
|
||||
|
||||
|
||||
def charset_from_html_head(raw_bytes: bytes) -> str | None:
|
||||
head = raw_bytes[:4096].decode("ascii", errors="ignore")
|
||||
match = re.search(r"charset\s*=\s*['\"]?([^'\"\s/>;]+)", head, flags=re.IGNORECASE)
|
||||
if match:
|
||||
return match.group(1).strip()
|
||||
return None
|
||||
|
||||
|
||||
def decode_html(raw_bytes: bytes, record: dict[str, Any]) -> str:
|
||||
candidates = [
|
||||
charset_from_content_type(str(record.get("content_type") or "")),
|
||||
charset_from_html_head(raw_bytes),
|
||||
"utf-8",
|
||||
"cp1251",
|
||||
]
|
||||
seen: set[str] = set()
|
||||
for charset in candidates:
|
||||
if not charset:
|
||||
continue
|
||||
key = charset.lower()
|
||||
if key in seen:
|
||||
continue
|
||||
seen.add(key)
|
||||
try:
|
||||
return raw_bytes.decode(charset)
|
||||
except (LookupError, UnicodeDecodeError):
|
||||
continue
|
||||
return raw_bytes.decode("utf-8", errors="replace")
|
||||
|
||||
|
||||
class TextExtractor(HTMLParser):
|
||||
def __init__(self) -> None:
|
||||
super().__init__(convert_charrefs=True)
|
||||
self.parts: list[str] = []
|
||||
self.skip_stack: list[str] = []
|
||||
self.heading_stack: list[str] = []
|
||||
|
||||
def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
|
||||
tag = tag.lower()
|
||||
if tag in {"head", "script", "style", "noscript", "svg"}:
|
||||
self.skip_stack.append(tag)
|
||||
return
|
||||
if self.skip_stack:
|
||||
return
|
||||
if tag in {"h1", "h2", "h3", "h4", "h5", "h6"}:
|
||||
self.heading_stack.append(tag)
|
||||
self.parts.append("\n\n" + "#" * int(tag[1]) + " ")
|
||||
elif tag == "li":
|
||||
self.parts.append("\n- ")
|
||||
elif tag == "br":
|
||||
self.parts.append("\n")
|
||||
elif tag in BLOCK_TAGS:
|
||||
self.parts.append("\n\n")
|
||||
|
||||
def handle_endtag(self, tag: str) -> None:
|
||||
tag = tag.lower()
|
||||
if self.skip_stack and self.skip_stack[-1] == tag:
|
||||
self.skip_stack.pop()
|
||||
return
|
||||
if self.skip_stack:
|
||||
return
|
||||
if tag in BLOCK_TAGS or tag in self.heading_stack:
|
||||
self.parts.append("\n\n")
|
||||
if self.heading_stack and self.heading_stack[-1] == tag:
|
||||
self.heading_stack.pop()
|
||||
|
||||
def handle_data(self, data: str) -> None:
|
||||
if self.skip_stack:
|
||||
return
|
||||
text = html.unescape(data)
|
||||
if text.strip():
|
||||
self.parts.append(text)
|
||||
|
||||
def text(self) -> str:
|
||||
raw = "".join(self.parts)
|
||||
raw = raw.replace("\r\n", "\n").replace("\r", "\n")
|
||||
raw = re.sub(r"[ \t]+", " ", raw)
|
||||
raw = re.sub(r"\n{3,}", "\n\n", raw)
|
||||
lines = [line.strip() for line in raw.split("\n")]
|
||||
return "\n".join(line for line in lines).strip()
|
||||
|
||||
|
||||
class LinkExtractor(HTMLParser):
|
||||
def __init__(self) -> None:
|
||||
super().__init__(convert_charrefs=True)
|
||||
self.links: list[str] = []
|
||||
|
||||
def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
|
||||
tag = tag.lower()
|
||||
attr_name = "href" if tag == "a" else "src" if tag in {"iframe", "frame"} else None
|
||||
if attr_name is None:
|
||||
return
|
||||
for name, value in attrs:
|
||||
if name.lower() == attr_name and value:
|
||||
self.links.append(value)
|
||||
|
||||
|
||||
class MediaExtractor(HTMLParser):
|
||||
def __init__(self, base_url: str) -> None:
|
||||
super().__init__(convert_charrefs=True)
|
||||
self.base_url = base_url
|
||||
self.images: list[dict[str, Any]] = []
|
||||
self.table_count = 0
|
||||
|
||||
def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
|
||||
tag = tag.lower()
|
||||
values = {name.lower(): value for name, value in attrs if value is not None}
|
||||
if tag == "table":
|
||||
self.table_count += 1
|
||||
if tag != "img":
|
||||
return
|
||||
src = values.get("src") or ""
|
||||
url = normalize_url(self.base_url, src)
|
||||
if not url or is_tracking_image_url(url):
|
||||
return
|
||||
self.images.append(
|
||||
{
|
||||
"url": url,
|
||||
"src": src,
|
||||
"alt": values.get("alt") or "",
|
||||
"title": values.get("title") or "",
|
||||
"width": values.get("width") or "",
|
||||
"height": values.get("height") or "",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def load_json(path: Path) -> dict[str, Any]:
|
||||
data = json.loads(path.read_text(encoding="utf-8-sig"))
|
||||
if not isinstance(data, dict):
|
||||
raise ValueError(f"{path} must contain a JSON object")
|
||||
return data
|
||||
|
||||
|
||||
def normalize_url(base_url: str, href: str) -> str | None:
|
||||
if href.startswith(("mailto:", "tel:", "javascript:")):
|
||||
return None
|
||||
absolute = urllib.parse.urljoin(base_url, href)
|
||||
parsed = urllib.parse.urlparse(absolute)
|
||||
if parsed.scheme not in {"http", "https"}:
|
||||
return None
|
||||
parsed = parsed._replace(fragment="")
|
||||
return urllib.parse.urlunparse(parsed)
|
||||
|
||||
|
||||
def is_content_src_url(url: str) -> bool:
|
||||
path = urllib.parse.urlparse(url).path
|
||||
return "/db/content/" in path and "/src/" in path
|
||||
|
||||
|
||||
def is_tracking_image_url(url: str) -> bool:
|
||||
lowered = url.lower()
|
||||
return any(
|
||||
marker in lowered
|
||||
for marker in (
|
||||
"mc.yandex",
|
||||
"counter.yadro",
|
||||
"logo_its",
|
||||
"favicon",
|
||||
"apple_touch",
|
||||
"/watch/",
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def raw_slug_for_url(value: str) -> str:
|
||||
parsed = urllib.parse.urlparse(value)
|
||||
raw = f"{parsed.netloc}{parsed.path}".strip("/") or "index"
|
||||
if parsed.query:
|
||||
raw += "_" + parsed.query
|
||||
slug = re.sub(r"[^A-Za-zА-Яа-яЁё0-9_.-]+", "_", raw, flags=re.UNICODE).strip("_")
|
||||
digest = hashlib.sha256(value.encode("utf-8")).hexdigest()[:12]
|
||||
return f"{slug[:120]}__{digest}.html"
|
||||
|
||||
|
||||
def strip_list_marker(line: str) -> str:
|
||||
return re.sub(r"^[-*]\s+", "", line.strip()).strip()
|
||||
|
||||
|
||||
def is_boilerplate_line(line: str) -> bool:
|
||||
normalized = strip_list_marker(line)
|
||||
if not normalized or normalized == "-":
|
||||
return True
|
||||
compact = normalized.replace(" ", "")
|
||||
if VERSION_SELECTOR_LINE_RE.match(compact):
|
||||
return True
|
||||
if normalized in BOILERPLATE_LINES:
|
||||
return True
|
||||
if normalized in NAVIGATION_ONLY_TERMS:
|
||||
return True
|
||||
return any(normalized.startswith(prefix) for prefix in BOILERPLATE_PREFIXES)
|
||||
|
||||
|
||||
def compact_lines(lines: list[str]) -> str:
|
||||
compact = []
|
||||
previous_blank = False
|
||||
for line in lines:
|
||||
blank = not line
|
||||
if blank and previous_blank:
|
||||
continue
|
||||
compact.append(line)
|
||||
previous_blank = blank
|
||||
cleaned = "\n".join(compact).strip()
|
||||
cleaned = re.sub(r"\n{3,}", "\n\n", cleaned)
|
||||
return cleaned
|
||||
|
||||
|
||||
def clean_glossary_text(text: str, title: str) -> str | None:
|
||||
term = title.split("::", 1)[0].strip()
|
||||
raw_lines = [line.strip() for line in text.splitlines()]
|
||||
start_index = None
|
||||
for index, line in enumerate(raw_lines):
|
||||
plain = strip_list_marker(line).lstrip("#").strip()
|
||||
if plain == term:
|
||||
start_index = index + 1
|
||||
break
|
||||
if start_index is None:
|
||||
return None
|
||||
|
||||
body = []
|
||||
for line in raw_lines[start_index:]:
|
||||
plain = strip_list_marker(line).lstrip("#").strip()
|
||||
if plain == "Назад":
|
||||
break
|
||||
if plain in {term, title}:
|
||||
continue
|
||||
if is_boilerplate_line(line):
|
||||
continue
|
||||
body.append(line)
|
||||
return compact_lines(body)
|
||||
|
||||
|
||||
def clean_its_text(text: str, title: str, source_type: str | None = None) -> str:
|
||||
if source_type == "official_1c_its_glossary":
|
||||
glossary_text = clean_glossary_text(text, title)
|
||||
if glossary_text is not None:
|
||||
return glossary_text
|
||||
term = title.split("::", 1)[0].strip()
|
||||
fallback_lines = []
|
||||
for raw_line in text.splitlines():
|
||||
line = raw_line.strip()
|
||||
plain = strip_list_marker(line).lstrip("#").strip()
|
||||
if plain in {term, title}:
|
||||
continue
|
||||
if is_boilerplate_line(line):
|
||||
continue
|
||||
fallback_lines.append(line)
|
||||
return compact_lines(fallback_lines)
|
||||
|
||||
lines = []
|
||||
for raw_line in text.splitlines():
|
||||
line = raw_line.strip()
|
||||
if is_boilerplate_line(line):
|
||||
continue
|
||||
# Keep real headings, but drop duplicate title-only body lines.
|
||||
if strip_list_marker(line) == title:
|
||||
continue
|
||||
lines.append(line)
|
||||
return compact_lines(lines)
|
||||
|
||||
|
||||
def content_quality(text: str, title: str) -> dict[str, Any]:
|
||||
lines = [line.strip() for line in text.splitlines() if line.strip()]
|
||||
content_lines = []
|
||||
for line in lines:
|
||||
plain = strip_list_marker(line).lstrip("#").strip()
|
||||
if not plain or plain == title:
|
||||
continue
|
||||
if is_boilerplate_line(plain):
|
||||
continue
|
||||
content_lines.append(plain)
|
||||
prose_lines = [
|
||||
line
|
||||
for line in content_lines
|
||||
if len(line) >= 45
|
||||
and not line.startswith("#")
|
||||
and not line.startswith("- ")
|
||||
and not line.endswith(":")
|
||||
]
|
||||
word_count = sum(len(re.findall(r"[A-Za-zА-Яа-яЁё0-9_]+", line)) for line in content_lines)
|
||||
return {
|
||||
"content_lines": len(content_lines),
|
||||
"prose_lines": len(prose_lines),
|
||||
"word_count": word_count,
|
||||
"chars": len(text),
|
||||
"is_content": len(prose_lines) >= 1 and word_count >= 25,
|
||||
}
|
||||
|
||||
|
||||
def safe_name(value: str, fallback: str) -> str:
|
||||
name = re.sub(r"[^A-Za-zА-Яа-яЁё0-9_.-]+", "_", value, flags=re.UNICODE).strip("_")
|
||||
return (name or fallback)[:120]
|
||||
|
||||
|
||||
def front_matter(record: dict[str, Any], source_hash: str) -> str:
|
||||
record = merge_platform_metadata(record, str(record.get("url") or ""))
|
||||
fields = {
|
||||
"source": "official_1c_its",
|
||||
"access": "licensed_private",
|
||||
"source_id": record.get("source_id"),
|
||||
"source_type": record.get("source_type"),
|
||||
"title": record.get("title"),
|
||||
"url": record.get("url"),
|
||||
"platform_family": record.get("platform_family"),
|
||||
"platform_version": record.get("platform_version"),
|
||||
"platform_doc_id": record.get("platform_doc_id"),
|
||||
"doc_book": record.get("doc_book"),
|
||||
"doc_bookmark": record.get("doc_bookmark"),
|
||||
"doc_content_id": record.get("doc_content_id"),
|
||||
"doc_coordinate": record.get("doc_coordinate"),
|
||||
"access_blocked": "true" if record.get("access_blocked") else "",
|
||||
"access_findings": ",".join(str(item) for item in (record.get("access_findings") or [])),
|
||||
"source_sha256": source_hash,
|
||||
}
|
||||
lines = ["---"]
|
||||
for key, value in fields.items():
|
||||
if value in (None, ""):
|
||||
continue
|
||||
escaped = str(value).replace('"', '\\"')
|
||||
lines.append(f'{key}: "{escaped}"')
|
||||
lines.append("---")
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def media_markdown(media: dict[str, Any]) -> str:
|
||||
images = media.get("images") or []
|
||||
if not images:
|
||||
return ""
|
||||
lines = ["## Иллюстрации", ""]
|
||||
for index, image in enumerate(images, start=1):
|
||||
alt = str(image.get("alt") or image.get("title") or f"Иллюстрация {index}").strip()
|
||||
url = str(image.get("url") or "").strip()
|
||||
size = "x".join(part for part in [str(image.get("width") or ""), str(image.get("height") or "")] if part)
|
||||
suffix = f" ({size})" if size else ""
|
||||
lines.append(f"- {suffix}")
|
||||
return "\n".join(lines).strip() + "\n"
|
||||
|
||||
|
||||
def normalize_page(record: dict[str, Any], *, raw_dir: Path, output_dir: Path, rag_source_dir: Path) -> dict[str, Any] | None:
|
||||
record = merge_platform_metadata(record, str(record.get("url") or ""))
|
||||
raw_path = raw_dir / str(record.get("file") or "")
|
||||
if not raw_path.exists():
|
||||
return None
|
||||
raw_bytes = raw_path.read_bytes()
|
||||
raw_text = decode_html(raw_bytes, record)
|
||||
extractor = TextExtractor()
|
||||
extractor.feed(raw_text)
|
||||
media_extractor = MediaExtractor(str(record.get("url") or ""))
|
||||
media_extractor.feed(raw_text)
|
||||
media = {"images": media_extractor.images, "table_count": media_extractor.table_count}
|
||||
title = str(record.get("title") or record.get("source_title") or record.get("source_id") or "its_doc")
|
||||
text = clean_its_text(extractor.text(), title, str(record.get("source_type") or ""))
|
||||
if not text:
|
||||
return {
|
||||
"source_id": record.get("source_id"),
|
||||
"source_type": record.get("source_type"),
|
||||
"access": "licensed_private",
|
||||
"title": title,
|
||||
"url": record.get("url"),
|
||||
"platform_family": record.get("platform_family"),
|
||||
"platform_version": record.get("platform_version"),
|
||||
"platform_doc_id": record.get("platform_doc_id"),
|
||||
"doc_book": record.get("doc_book"),
|
||||
"doc_bookmark": record.get("doc_bookmark"),
|
||||
"doc_content_id": record.get("doc_content_id"),
|
||||
"doc_coordinate": record.get("doc_coordinate"),
|
||||
"access_blocked": record.get("access_blocked"),
|
||||
"access_findings": record.get("access_findings") or [],
|
||||
"raw_file": record.get("file"),
|
||||
"skipped": True,
|
||||
"skip_reason": "empty_after_cleaning",
|
||||
"quality": content_quality("", title),
|
||||
"media": media,
|
||||
}
|
||||
quality = content_quality(text, title)
|
||||
if not quality["is_content"]:
|
||||
return {
|
||||
"source_id": record.get("source_id"),
|
||||
"source_type": record.get("source_type"),
|
||||
"access": "licensed_private",
|
||||
"title": title,
|
||||
"url": record.get("url"),
|
||||
"platform_family": record.get("platform_family"),
|
||||
"platform_version": record.get("platform_version"),
|
||||
"platform_doc_id": record.get("platform_doc_id"),
|
||||
"doc_book": record.get("doc_book"),
|
||||
"doc_bookmark": record.get("doc_bookmark"),
|
||||
"doc_content_id": record.get("doc_content_id"),
|
||||
"doc_coordinate": record.get("doc_coordinate"),
|
||||
"access_blocked": record.get("access_blocked"),
|
||||
"access_findings": record.get("access_findings") or [],
|
||||
"raw_file": record.get("file"),
|
||||
"skipped": True,
|
||||
"skip_reason": "navigation_or_low_content",
|
||||
"quality": quality,
|
||||
"media": media,
|
||||
}
|
||||
source_hash = hashlib.sha256(raw_bytes).hexdigest()
|
||||
stem = safe_name(f"{record.get('source_id')}_{title}", fallback=source_hash[:12])
|
||||
file_name = f"{stem}__{source_hash[:12]}.md"
|
||||
media_section = media_markdown(media)
|
||||
content = f"{front_matter(record, source_hash)}\n\n# {title}\n\n{text}\n"
|
||||
if media_section:
|
||||
content += f"\n{media_section}"
|
||||
for target_dir in (output_dir, rag_source_dir):
|
||||
target_dir.mkdir(parents=True, exist_ok=True)
|
||||
(target_dir / file_name).write_text(content, encoding="utf-8")
|
||||
return {
|
||||
"source_id": record.get("source_id"),
|
||||
"source_type": record.get("source_type"),
|
||||
"access": "licensed_private",
|
||||
"title": title,
|
||||
"url": record.get("url"),
|
||||
"platform_family": record.get("platform_family"),
|
||||
"platform_version": record.get("platform_version"),
|
||||
"platform_doc_id": record.get("platform_doc_id"),
|
||||
"doc_book": record.get("doc_book"),
|
||||
"doc_bookmark": record.get("doc_bookmark"),
|
||||
"doc_content_id": record.get("doc_content_id"),
|
||||
"doc_coordinate": record.get("doc_coordinate"),
|
||||
"access_blocked": record.get("access_blocked"),
|
||||
"access_findings": record.get("access_findings") or [],
|
||||
"raw_file": record.get("file"),
|
||||
"normalized_file": file_name,
|
||||
"raw_sha256": source_hash,
|
||||
"text_sha256": hashlib.sha256(text.encode("utf-8")).hexdigest(),
|
||||
"chars": len(text),
|
||||
"quality": quality,
|
||||
"media": media,
|
||||
}
|
||||
|
||||
|
||||
def discover_orphan_src_records(records: list[dict[str, Any]], *, raw_dir: Path) -> list[dict[str, Any]]:
|
||||
known_urls = {str(record.get("url") or "") for record in records}
|
||||
discovered_by_url: dict[str, dict[str, Any]] = {}
|
||||
for record in records:
|
||||
raw_path = raw_dir / str(record.get("file") or "")
|
||||
parent_url = str(record.get("url") or "")
|
||||
if not raw_path.exists() or not parent_url:
|
||||
continue
|
||||
try:
|
||||
raw_bytes = raw_path.read_bytes()
|
||||
raw_text = decode_html(raw_bytes, record)
|
||||
except OSError:
|
||||
continue
|
||||
extractor = LinkExtractor()
|
||||
extractor.feed(raw_text)
|
||||
for href in extractor.links:
|
||||
src_url = normalize_url(parent_url, href)
|
||||
if not src_url or not is_content_src_url(src_url) or src_url.endswith("#_print"):
|
||||
continue
|
||||
if src_url in known_urls or src_url in discovered_by_url:
|
||||
continue
|
||||
src_file = raw_slug_for_url(src_url)
|
||||
src_path = raw_dir / src_file
|
||||
if not src_path.exists():
|
||||
continue
|
||||
discovered = dict(record)
|
||||
discovered.update(
|
||||
{
|
||||
"url": src_url,
|
||||
"title": record.get("title") or record.get("source_title") or src_url,
|
||||
"depth": int(record.get("depth") or 0) + 1,
|
||||
"file": src_file,
|
||||
"bytes": src_path.stat().st_size,
|
||||
"sha256": hashlib.sha256(src_path.read_bytes()).hexdigest(),
|
||||
"discovered_from_parent_url": parent_url,
|
||||
"discovered_from_parent_file": record.get("file"),
|
||||
}
|
||||
)
|
||||
discovered_by_url[src_url] = discovered
|
||||
return sorted(discovered_by_url.values(), key=lambda item: str(item.get("url") or ""))
|
||||
|
||||
|
||||
def normalize_manifest(manifest: dict[str, Any], *, raw_dir: Path, output_dir: Path, rag_source_dir: Path) -> dict[str, Any]:
|
||||
pages = []
|
||||
skipped = []
|
||||
missing = 0
|
||||
seen_text_keys: dict[tuple[str, str, str], dict[str, Any]] = {}
|
||||
source_records = list(manifest.get("pages") or [])
|
||||
discovered_records = discover_orphan_src_records(source_records, raw_dir=raw_dir)
|
||||
all_records = [*source_records, *discovered_records]
|
||||
for record in all_records:
|
||||
normalized = normalize_page(record, raw_dir=raw_dir, output_dir=output_dir, rag_source_dir=rag_source_dir)
|
||||
if normalized and normalized.get("skipped"):
|
||||
skipped.append(normalized)
|
||||
elif normalized:
|
||||
text_key = (
|
||||
str(normalized.get("text_sha256") or ""),
|
||||
str(normalized.get("platform_version") or ""),
|
||||
str(normalized.get("title") or ""),
|
||||
)
|
||||
if text_key[0] and text_key in seen_text_keys:
|
||||
normalized["skipped"] = True
|
||||
normalized["skip_reason"] = "duplicate_normalized_text"
|
||||
normalized["duplicate_of"] = seen_text_keys[text_key].get("normalized_file")
|
||||
duplicate_file = str(normalized.get("normalized_file") or "")
|
||||
if duplicate_file:
|
||||
for directory in (output_dir, rag_source_dir):
|
||||
duplicate_path = directory / duplicate_file
|
||||
if duplicate_path.exists():
|
||||
duplicate_path.unlink()
|
||||
skipped.append(normalized)
|
||||
continue
|
||||
seen_text_keys[text_key] = normalized
|
||||
pages.append(normalized)
|
||||
else:
|
||||
missing += 1
|
||||
return {
|
||||
"schema": "onec_its_normalized_docs_manifest.v1",
|
||||
"access": "licensed_private",
|
||||
"raw_manifest": manifest.get("output_dir"),
|
||||
"output_dir": str(output_dir),
|
||||
"rag_source_dir": str(rag_source_dir),
|
||||
"page_count": len(pages),
|
||||
"skipped_count": len(skipped),
|
||||
"missing_count": missing,
|
||||
"source_record_count": len(source_records),
|
||||
"discovered_src_record_count": len(discovered_records),
|
||||
"pages": pages,
|
||||
"skipped_pages": skipped,
|
||||
}
|
||||
|
||||
|
||||
def clean_generated_markdown(*dirs: Path) -> None:
|
||||
for directory in dirs:
|
||||
if not directory.exists():
|
||||
continue
|
||||
for path in directory.glob("*.md"):
|
||||
try:
|
||||
text = path.read_text(encoding="utf-8", errors="ignore")
|
||||
except OSError:
|
||||
continue
|
||||
if 'source: "official_1c_its"' in text[:600]:
|
||||
path.unlink()
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Normalize private 1C:ITS raw HTML pages into Markdown RAG sources.")
|
||||
parser.add_argument("--manifest", type=Path, default=DEFAULT_RAW_MANIFEST)
|
||||
parser.add_argument("--raw-dir", type=Path, default=DEFAULT_RAW_DIR)
|
||||
parser.add_argument("--output-dir", type=Path, default=DEFAULT_OUTPUT_DIR)
|
||||
parser.add_argument("--rag-source-dir", type=Path, default=DEFAULT_RAG_SOURCE_DIR)
|
||||
parser.add_argument("--manifest-output", type=Path, default=DEFAULT_MANIFEST_OUTPUT)
|
||||
parser.add_argument("--no-clean", action="store_true", help="Do not remove previously generated official 1C:ITS Markdown files.")
|
||||
args = parser.parse_args()
|
||||
|
||||
raw_manifest = load_json(args.manifest)
|
||||
if not args.no_clean:
|
||||
clean_generated_markdown(args.output_dir, args.rag_source_dir)
|
||||
result = normalize_manifest(raw_manifest, raw_dir=args.raw_dir, output_dir=args.output_dir, rag_source_dir=args.rag_source_dir)
|
||||
args.manifest_output.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.manifest_output.write_text(json.dumps(result, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
print(json.dumps({"output": str(args.manifest_output), "pages": result["page_count"], "skipped": result["skipped_count"], "missing": result["missing_count"]}, ensure_ascii=False))
|
||||
return 0 if result["page_count"] or result["skipped_count"] else 2
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Reference in New Issue
Block a user