from __future__ import annotations import argparse import hashlib import html import json import re import urllib.parse from html.parser import HTMLParser from pathlib import Path from typing import Any from one_c_its_platform import merge_platform_metadata ROOT = Path(__file__).resolve().parents[1] DEFAULT_RAW_DIR = ROOT / "plugins" / "1c" / "rag" / "official-docs" / "raw" DEFAULT_RAW_MANIFEST = DEFAULT_RAW_DIR / "manifest.json" DEFAULT_OUTPUT_DIR = ROOT / "plugins" / "1c" / "rag" / "official-docs" / "normalized" DEFAULT_RAG_SOURCE_DIR = ROOT / "plugins" / "1c" / "rag" / "sources" / "official" / "its" DEFAULT_MANIFEST_OUTPUT = DEFAULT_OUTPUT_DIR / "manifest.json" BLOCK_TAGS = { "address", "article", "aside", "blockquote", "br", "div", "dl", "fieldset", "figcaption", "figure", "footer", "form", "h1", "h2", "h3", "h4", "h5", "h6", "header", "hr", "li", "main", "nav", "ol", "p", "pre", "section", "table", "td", "th", "tr", "ul", } BOILERPLATE_LINES = { "Вход", "Об 1С:ИТС", "Тест-драйв", "Заказать ИТС", "Задать вопрос", "Обновить ПО", "Оценить 1С", "Купить кассу", "Тематические подборки", "Календарь бухгалтера", "Калькуляторы", "Подбор КБК", "Последние результаты поиска", "Подписаться на рассылку", "Главная", "Инструкции по разработке на 1С", "Платформа 1С:Предприятие. Документация", "Содержание", "Результаты поиска", "Вконтакте", "Принимаю", "Методические материалы для разработчиков и администраторов 1С", "Глоссарий разработчика", "Назад", } BOILERPLATE_PREFIXES = ( "© Фирма «1С»", "Информационная система 1С:ИТС", "Инструкции по учету в программах 1С", "Новости1С:Лекторий", "Мы используем файлы cookie", "Продолжая находиться на сайте", "на условиях, указанных по ссылке", ) NAVIGATION_ONLY_TERMS = { "Руководство разработчика", "Руководство администратора", "Клиент-серверный вариант. Руководство администратора", "Руководство разработчика. Обычный режим", "Руководство пользователя", } VERSION_SELECTOR_LINE_RE = re.compile(r"^(?:\d+\.\d+(?:\.\d+)?){4,}$") def charset_from_content_type(content_type: str) -> str | None: match = re.search(r"charset=([^;\s]+)", content_type, flags=re.IGNORECASE) if match: return match.group(1).strip("\"'") return None def charset_from_html_head(raw_bytes: bytes) -> str | None: head = raw_bytes[:4096].decode("ascii", errors="ignore") match = re.search(r"charset\s*=\s*['\"]?([^'\"\s/>;]+)", head, flags=re.IGNORECASE) if match: return match.group(1).strip() return None def decode_html(raw_bytes: bytes, record: dict[str, Any]) -> str: candidates = [ charset_from_content_type(str(record.get("content_type") or "")), charset_from_html_head(raw_bytes), "utf-8", "cp1251", ] seen: set[str] = set() for charset in candidates: if not charset: continue key = charset.lower() if key in seen: continue seen.add(key) try: return raw_bytes.decode(charset) except (LookupError, UnicodeDecodeError): continue return raw_bytes.decode("utf-8", errors="replace") class TextExtractor(HTMLParser): def __init__(self) -> None: super().__init__(convert_charrefs=True) self.parts: list[str] = [] self.skip_stack: list[str] = [] self.heading_stack: list[str] = [] def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None: tag = tag.lower() if tag in {"head", "script", "style", "noscript", "svg"}: self.skip_stack.append(tag) return if self.skip_stack: return if tag in {"h1", "h2", "h3", "h4", "h5", "h6"}: self.heading_stack.append(tag) self.parts.append("\n\n" + "#" * int(tag[1]) + " ") elif tag == "li": self.parts.append("\n- ") elif tag == "br": self.parts.append("\n") elif tag in BLOCK_TAGS: self.parts.append("\n\n") def handle_endtag(self, tag: str) -> None: tag = tag.lower() if self.skip_stack and self.skip_stack[-1] == tag: self.skip_stack.pop() return if self.skip_stack: return if tag in BLOCK_TAGS or tag in self.heading_stack: self.parts.append("\n\n") if self.heading_stack and self.heading_stack[-1] == tag: self.heading_stack.pop() def handle_data(self, data: str) -> None: if self.skip_stack: return text = html.unescape(data) if text.strip(): self.parts.append(text) def text(self) -> str: raw = "".join(self.parts) raw = raw.replace("\r\n", "\n").replace("\r", "\n") raw = re.sub(r"[ \t]+", " ", raw) raw = re.sub(r"\n{3,}", "\n\n", raw) lines = [line.strip() for line in raw.split("\n")] return "\n".join(line for line in lines).strip() class LinkExtractor(HTMLParser): def __init__(self) -> None: super().__init__(convert_charrefs=True) self.links: list[str] = [] def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None: tag = tag.lower() attr_name = "href" if tag == "a" else "src" if tag in {"iframe", "frame"} else None if attr_name is None: return for name, value in attrs: if name.lower() == attr_name and value: self.links.append(value) class MediaExtractor(HTMLParser): def __init__(self, base_url: str) -> None: super().__init__(convert_charrefs=True) self.base_url = base_url self.images: list[dict[str, Any]] = [] self.table_count = 0 def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None: tag = tag.lower() values = {name.lower(): value for name, value in attrs if value is not None} if tag == "table": self.table_count += 1 if tag != "img": return src = values.get("src") or "" url = normalize_url(self.base_url, src) if not url or is_tracking_image_url(url): return self.images.append( { "url": url, "src": src, "alt": values.get("alt") or "", "title": values.get("title") or "", "width": values.get("width") or "", "height": values.get("height") or "", } ) def load_json(path: Path) -> dict[str, Any]: data = json.loads(path.read_text(encoding="utf-8-sig")) if not isinstance(data, dict): raise ValueError(f"{path} must contain a JSON object") return data def normalize_url(base_url: str, href: str) -> str | None: if href.startswith(("mailto:", "tel:", "javascript:")): return None absolute = urllib.parse.urljoin(base_url, href) parsed = urllib.parse.urlparse(absolute) if parsed.scheme not in {"http", "https"}: return None parsed = parsed._replace(fragment="") return urllib.parse.urlunparse(parsed) def is_content_src_url(url: str) -> bool: path = urllib.parse.urlparse(url).path return "/db/content/" in path and "/src/" in path def is_tracking_image_url(url: str) -> bool: lowered = url.lower() return any( marker in lowered for marker in ( "mc.yandex", "counter.yadro", "logo_its", "favicon", "apple_touch", "/watch/", ) ) def raw_slug_for_url(value: str) -> str: parsed = urllib.parse.urlparse(value) raw = f"{parsed.netloc}{parsed.path}".strip("/") or "index" if parsed.query: raw += "_" + parsed.query slug = re.sub(r"[^A-Za-zА-Яа-яЁё0-9_.-]+", "_", raw, flags=re.UNICODE).strip("_") digest = hashlib.sha256(value.encode("utf-8")).hexdigest()[:12] return f"{slug[:120]}__{digest}.html" def strip_list_marker(line: str) -> str: return re.sub(r"^[-*]\s+", "", line.strip()).strip() def is_boilerplate_line(line: str) -> bool: normalized = strip_list_marker(line) if not normalized or normalized == "-": return True compact = normalized.replace(" ", "") if VERSION_SELECTOR_LINE_RE.match(compact): return True if normalized in BOILERPLATE_LINES: return True if normalized in NAVIGATION_ONLY_TERMS: return True return any(normalized.startswith(prefix) for prefix in BOILERPLATE_PREFIXES) def compact_lines(lines: list[str]) -> str: compact = [] previous_blank = False for line in lines: blank = not line if blank and previous_blank: continue compact.append(line) previous_blank = blank cleaned = "\n".join(compact).strip() cleaned = re.sub(r"\n{3,}", "\n\n", cleaned) return cleaned def clean_glossary_text(text: str, title: str) -> str | None: term = title.split("::", 1)[0].strip() raw_lines = [line.strip() for line in text.splitlines()] start_index = None for index, line in enumerate(raw_lines): plain = strip_list_marker(line).lstrip("#").strip() if plain == term: start_index = index + 1 break if start_index is None: return None body = [] for line in raw_lines[start_index:]: plain = strip_list_marker(line).lstrip("#").strip() if plain == "Назад": break if plain in {term, title}: continue if is_boilerplate_line(line): continue body.append(line) return compact_lines(body) def clean_its_text(text: str, title: str, source_type: str | None = None) -> str: if source_type == "official_1c_its_glossary": glossary_text = clean_glossary_text(text, title) if glossary_text is not None: return glossary_text term = title.split("::", 1)[0].strip() fallback_lines = [] for raw_line in text.splitlines(): line = raw_line.strip() plain = strip_list_marker(line).lstrip("#").strip() if plain in {term, title}: continue if is_boilerplate_line(line): continue fallback_lines.append(line) return compact_lines(fallback_lines) lines = [] for raw_line in text.splitlines(): line = raw_line.strip() if is_boilerplate_line(line): continue # Keep real headings, but drop duplicate title-only body lines. if strip_list_marker(line) == title: continue lines.append(line) return compact_lines(lines) def content_quality(text: str, title: str) -> dict[str, Any]: lines = [line.strip() for line in text.splitlines() if line.strip()] content_lines = [] for line in lines: plain = strip_list_marker(line).lstrip("#").strip() if not plain or plain == title: continue if is_boilerplate_line(plain): continue content_lines.append(plain) prose_lines = [ line for line in content_lines if len(line) >= 45 and not line.startswith("#") and not line.startswith("- ") and not line.endswith(":") ] word_count = sum(len(re.findall(r"[A-Za-zА-Яа-яЁё0-9_]+", line)) for line in content_lines) return { "content_lines": len(content_lines), "prose_lines": len(prose_lines), "word_count": word_count, "chars": len(text), "is_content": len(prose_lines) >= 1 and word_count >= 25, } def safe_name(value: str, fallback: str) -> str: name = re.sub(r"[^A-Za-zА-Яа-яЁё0-9_.-]+", "_", value, flags=re.UNICODE).strip("_") return (name or fallback)[:120] def front_matter(record: dict[str, Any], source_hash: str) -> str: record = merge_platform_metadata(record, str(record.get("url") or "")) fields = { "source": "official_1c_its", "access": "licensed_private", "source_id": record.get("source_id"), "source_type": record.get("source_type"), "title": record.get("title"), "url": record.get("url"), "platform_family": record.get("platform_family"), "platform_version": record.get("platform_version"), "platform_doc_id": record.get("platform_doc_id"), "doc_book": record.get("doc_book"), "doc_bookmark": record.get("doc_bookmark"), "doc_content_id": record.get("doc_content_id"), "doc_coordinate": record.get("doc_coordinate"), "access_blocked": "true" if record.get("access_blocked") else "", "access_findings": ",".join(str(item) for item in (record.get("access_findings") or [])), "source_sha256": source_hash, } lines = ["---"] for key, value in fields.items(): if value in (None, ""): continue escaped = str(value).replace('"', '\\"') lines.append(f'{key}: "{escaped}"') lines.append("---") return "\n".join(lines) def media_markdown(media: dict[str, Any]) -> str: images = media.get("images") or [] if not images: return "" lines = ["## Иллюстрации", ""] for index, image in enumerate(images, start=1): alt = str(image.get("alt") or image.get("title") or f"Иллюстрация {index}").strip() url = str(image.get("url") or "").strip() size = "x".join(part for part in [str(image.get("width") or ""), str(image.get("height") or "")] if part) suffix = f" ({size})" if size else "" lines.append(f"- ![{alt}]({url}){suffix}") return "\n".join(lines).strip() + "\n" def normalize_page(record: dict[str, Any], *, raw_dir: Path, output_dir: Path, rag_source_dir: Path) -> dict[str, Any] | None: record = merge_platform_metadata(record, str(record.get("url") or "")) raw_path = raw_dir / str(record.get("file") or "") if not raw_path.exists(): return None raw_bytes = raw_path.read_bytes() raw_text = decode_html(raw_bytes, record) extractor = TextExtractor() extractor.feed(raw_text) media_extractor = MediaExtractor(str(record.get("url") or "")) media_extractor.feed(raw_text) media = {"images": media_extractor.images, "table_count": media_extractor.table_count} title = str(record.get("title") or record.get("source_title") or record.get("source_id") or "its_doc") text = clean_its_text(extractor.text(), title, str(record.get("source_type") or "")) if not text: return { "source_id": record.get("source_id"), "source_type": record.get("source_type"), "access": "licensed_private", "title": title, "url": record.get("url"), "platform_family": record.get("platform_family"), "platform_version": record.get("platform_version"), "platform_doc_id": record.get("platform_doc_id"), "doc_book": record.get("doc_book"), "doc_bookmark": record.get("doc_bookmark"), "doc_content_id": record.get("doc_content_id"), "doc_coordinate": record.get("doc_coordinate"), "access_blocked": record.get("access_blocked"), "access_findings": record.get("access_findings") or [], "raw_file": record.get("file"), "skipped": True, "skip_reason": "empty_after_cleaning", "quality": content_quality("", title), "media": media, } quality = content_quality(text, title) if not quality["is_content"]: return { "source_id": record.get("source_id"), "source_type": record.get("source_type"), "access": "licensed_private", "title": title, "url": record.get("url"), "platform_family": record.get("platform_family"), "platform_version": record.get("platform_version"), "platform_doc_id": record.get("platform_doc_id"), "doc_book": record.get("doc_book"), "doc_bookmark": record.get("doc_bookmark"), "doc_content_id": record.get("doc_content_id"), "doc_coordinate": record.get("doc_coordinate"), "access_blocked": record.get("access_blocked"), "access_findings": record.get("access_findings") or [], "raw_file": record.get("file"), "skipped": True, "skip_reason": "navigation_or_low_content", "quality": quality, "media": media, } source_hash = hashlib.sha256(raw_bytes).hexdigest() stem = safe_name(f"{record.get('source_id')}_{title}", fallback=source_hash[:12]) file_name = f"{stem}__{source_hash[:12]}.md" media_section = media_markdown(media) content = f"{front_matter(record, source_hash)}\n\n# {title}\n\n{text}\n" if media_section: content += f"\n{media_section}" for target_dir in (output_dir, rag_source_dir): target_dir.mkdir(parents=True, exist_ok=True) (target_dir / file_name).write_text(content, encoding="utf-8") return { "source_id": record.get("source_id"), "source_type": record.get("source_type"), "access": "licensed_private", "title": title, "url": record.get("url"), "platform_family": record.get("platform_family"), "platform_version": record.get("platform_version"), "platform_doc_id": record.get("platform_doc_id"), "doc_book": record.get("doc_book"), "doc_bookmark": record.get("doc_bookmark"), "doc_content_id": record.get("doc_content_id"), "doc_coordinate": record.get("doc_coordinate"), "access_blocked": record.get("access_blocked"), "access_findings": record.get("access_findings") or [], "raw_file": record.get("file"), "normalized_file": file_name, "raw_sha256": source_hash, "text_sha256": hashlib.sha256(text.encode("utf-8")).hexdigest(), "chars": len(text), "quality": quality, "media": media, } def discover_orphan_src_records(records: list[dict[str, Any]], *, raw_dir: Path) -> list[dict[str, Any]]: known_urls = {str(record.get("url") or "") for record in records} discovered_by_url: dict[str, dict[str, Any]] = {} for record in records: raw_path = raw_dir / str(record.get("file") or "") parent_url = str(record.get("url") or "") if not raw_path.exists() or not parent_url: continue try: raw_bytes = raw_path.read_bytes() raw_text = decode_html(raw_bytes, record) except OSError: continue extractor = LinkExtractor() extractor.feed(raw_text) for href in extractor.links: src_url = normalize_url(parent_url, href) if not src_url or not is_content_src_url(src_url) or src_url.endswith("#_print"): continue if src_url in known_urls or src_url in discovered_by_url: continue src_file = raw_slug_for_url(src_url) src_path = raw_dir / src_file if not src_path.exists(): continue discovered = dict(record) discovered.update( { "url": src_url, "title": record.get("title") or record.get("source_title") or src_url, "depth": int(record.get("depth") or 0) + 1, "file": src_file, "bytes": src_path.stat().st_size, "sha256": hashlib.sha256(src_path.read_bytes()).hexdigest(), "discovered_from_parent_url": parent_url, "discovered_from_parent_file": record.get("file"), } ) discovered_by_url[src_url] = discovered return sorted(discovered_by_url.values(), key=lambda item: str(item.get("url") or "")) def normalize_manifest(manifest: dict[str, Any], *, raw_dir: Path, output_dir: Path, rag_source_dir: Path) -> dict[str, Any]: pages = [] skipped = [] missing = 0 seen_text_keys: dict[tuple[str, str, str], dict[str, Any]] = {} source_records = list(manifest.get("pages") or []) discovered_records = discover_orphan_src_records(source_records, raw_dir=raw_dir) all_records = [*source_records, *discovered_records] for record in all_records: normalized = normalize_page(record, raw_dir=raw_dir, output_dir=output_dir, rag_source_dir=rag_source_dir) if normalized and normalized.get("skipped"): skipped.append(normalized) elif normalized: text_key = ( str(normalized.get("text_sha256") or ""), str(normalized.get("platform_version") or ""), str(normalized.get("title") or ""), ) if text_key[0] and text_key in seen_text_keys: normalized["skipped"] = True normalized["skip_reason"] = "duplicate_normalized_text" normalized["duplicate_of"] = seen_text_keys[text_key].get("normalized_file") duplicate_file = str(normalized.get("normalized_file") or "") if duplicate_file: for directory in (output_dir, rag_source_dir): duplicate_path = directory / duplicate_file if duplicate_path.exists(): duplicate_path.unlink() skipped.append(normalized) continue seen_text_keys[text_key] = normalized pages.append(normalized) else: missing += 1 return { "schema": "onec_its_normalized_docs_manifest.v1", "access": "licensed_private", "raw_manifest": manifest.get("output_dir"), "output_dir": str(output_dir), "rag_source_dir": str(rag_source_dir), "page_count": len(pages), "skipped_count": len(skipped), "missing_count": missing, "source_record_count": len(source_records), "discovered_src_record_count": len(discovered_records), "pages": pages, "skipped_pages": skipped, } def clean_generated_markdown(*dirs: Path) -> None: for directory in dirs: if not directory.exists(): continue for path in directory.glob("*.md"): try: text = path.read_text(encoding="utf-8", errors="ignore") except OSError: continue if 'source: "official_1c_its"' in text[:600]: path.unlink() def main() -> int: parser = argparse.ArgumentParser(description="Normalize private 1C:ITS raw HTML pages into Markdown RAG sources.") parser.add_argument("--manifest", type=Path, default=DEFAULT_RAW_MANIFEST) parser.add_argument("--raw-dir", type=Path, default=DEFAULT_RAW_DIR) parser.add_argument("--output-dir", type=Path, default=DEFAULT_OUTPUT_DIR) parser.add_argument("--rag-source-dir", type=Path, default=DEFAULT_RAG_SOURCE_DIR) parser.add_argument("--manifest-output", type=Path, default=DEFAULT_MANIFEST_OUTPUT) parser.add_argument("--no-clean", action="store_true", help="Do not remove previously generated official 1C:ITS Markdown files.") args = parser.parse_args() raw_manifest = load_json(args.manifest) if not args.no_clean: clean_generated_markdown(args.output_dir, args.rag_source_dir) result = normalize_manifest(raw_manifest, raw_dir=args.raw_dir, output_dir=args.output_dir, rag_source_dir=args.rag_source_dir) args.manifest_output.parent.mkdir(parents=True, exist_ok=True) args.manifest_output.write_text(json.dumps(result, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") print(json.dumps({"output": str(args.manifest_output), "pages": result["page_count"], "skipped": result["skipped_count"], "missing": result["missing_count"]}, ensure_ascii=False)) return 0 if result["page_count"] or result["skipped_count"] else 2 if __name__ == "__main__": raise SystemExit(main())