from __future__ import annotations import argparse import hashlib import html import json import re import shutil import urllib.parse import urllib.request from html.parser import HTMLParser from pathlib import Path from typing import Any from normalize_1c_its_docs import decode_html ROOT = Path(__file__).resolve().parents[1] DEFAULT_MANIFEST = ROOT / "plugins" / "1c" / "rag" / "official-docs" / "normalized" / "manifest.json" DEFAULT_NORMALIZED_DIR = ROOT / "plugins" / "1c" / "rag" / "official-docs" / "normalized" DEFAULT_RAW_DIR = ROOT / "plugins" / "1c" / "rag" / "official-docs" / "raw" DEFAULT_MEDIA_MANIFEST = ROOT / "plugins" / "1c" / "rag" / "official-docs" / "media" / "manifest.json" DEFAULT_MEDIA_DIR = ROOT / "plugins" / "1c" / "rag" / "official-docs" / "media" DEFAULT_OUTPUT_DIR = ROOT / "plugins" / "1c" / "rag" / "official-docs" / "static" STYLE = """ :root { --bg: #f2f3ef; --paper: #fffef9; --ink: #202522; --muted: #66706a; --line: #cdd5cd; --accent: #0f6b5f; --code: #17201c; } * { box-sizing: border-box; } body { margin: 0; background: var(--bg); color: var(--ink); font: 16px/1.55 "Aptos", "Segoe UI", Tahoma, sans-serif; } a { color: var(--accent); } .wrap { max-width: 1120px; margin: 0 auto; padding: 24px; } .doc { background: var(--paper); border: 1px solid var(--line); border-radius: 8px; padding: 24px; } .meta { color: var(--muted); font-size: 13px; overflow-wrap: anywhere; margin-bottom: 18px; } h1 { font-size: 28px; line-height: 1.2; margin: 0 0 12px; } h2 { font-size: 20px; margin-top: 28px; border-top: 1px solid var(--line); padding-top: 18px; } img { max-width: 100%; height: auto; border: 1px solid var(--line); background: #fff; } figure { margin: 18px 0; } figcaption { color: var(--muted); font-size: 13px; margin-top: 6px; } pre { background: var(--code); color: #e4ece6; padding: 12px; border-radius: 6px; overflow: auto; } code { font-family: "Cascadia Mono", Consolas, monospace; } table { width: 100%; border-collapse: collapse; } th, td { border-bottom: 1px solid var(--line); padding: 8px 10px; text-align: left; vertical-align: top; } .top { display: flex; gap: 8px; flex-wrap: wrap; margin-bottom: 14px; } .btn { border: 1px solid var(--line); border-radius: 6px; padding: 6px 10px; background: #fff; text-decoration: none; } .badge { display: inline-block; border: 1px solid var(--line); border-radius: 999px; padding: 2px 8px; color: var(--muted); font-size: 12px; } """.strip() class AssetCollector(HTMLParser): def __init__(self, *, page_url: str) -> None: super().__init__(convert_charrefs=True) self.page_url = page_url self.urls: set[str] = set() def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None: tag_lower = tag.lower() attr_map = {name.lower(): value or "" for name, value in attrs} for name, value in attrs: if not value: continue name_lower = name.lower() if tag_lower == "link" and name_lower == "href" and is_static_link_asset(attr_map): self.urls.add(normalize_url(urllib.parse.urljoin(self.page_url, value))) elif tag_lower == "script" and name_lower == "src": self.urls.add(normalize_url(urllib.parse.urljoin(self.page_url, value))) class LinkRewriter(HTMLParser): def __init__( self, *, page_url: str, url_to_page: dict[str, str], media_url_to_file: dict[str, str], asset_url_to_file: dict[str, str], ) -> None: super().__init__(convert_charrefs=False) self.page_url = page_url self.url_to_page = url_to_page self.media_url_to_file = media_url_to_file self.asset_url_to_file = asset_url_to_file self.parts: list[str] = [] def rewrite_url(self, value: str, *, is_media: bool = False) -> str: absolute = normalize_url(urllib.parse.urljoin(self.page_url, value)) if is_media and absolute in self.media_url_to_file: return f"../media/{self.media_url_to_file[absolute]}" if absolute in self.asset_url_to_file: return f"../assets/{self.asset_url_to_file[absolute]}" if absolute in self.url_to_page: return f"../pages/{self.url_to_page[absolute]}" if absolute.startswith(("http://", "https://")): return absolute return value def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None: rewritten = [] for name, value in attrs: if value is None: rewritten.append((name, None)) continue lowered = name.lower() if tag.lower() == "img" and lowered == "src": value = self.rewrite_url(value, is_media=True) elif tag.lower() == "a" and lowered == "href": value = self.rewrite_url(value) elif tag.lower() in {"link", "script"} and lowered in {"href", "src"}: value = self.rewrite_url(value) rewritten.append((name, value)) attr_text = "".join(f" {name}" if value is None else f' {name}="{html.escape(value, quote=True)}"' for name, value in rewritten) self.parts.append(f"<{tag}{attr_text}>") def handle_endtag(self, tag: str) -> None: self.parts.append(f"") def handle_startendtag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None: self.handle_starttag(tag, attrs) def handle_data(self, data: str) -> None: self.parts.append(data) def handle_entityref(self, name: str) -> None: self.parts.append(f"&{name};") def handle_charref(self, name: str) -> None: self.parts.append(f"&#{name};") def handle_comment(self, data: str) -> None: self.parts.append(f"") def html(self) -> str: return "".join(self.parts) def load_json(path: Path) -> dict[str, Any]: if not path.exists(): return {} try: data = json.loads(path.read_text(encoding="utf-8-sig")) except json.JSONDecodeError: return {} return data if isinstance(data, dict) else {} def normalize_url(value: str) -> str: return urllib.parse.urlunparse(urllib.parse.urlparse(value)._replace(fragment="")) def is_static_link_asset(attrs: dict[str, str]) -> bool: rel = {part.casefold() for part in re.split(r"\s+", attrs.get("rel", "")) if part} href = attrs.get("href", "").casefold() as_type = attrs.get("as", "").casefold() if "stylesheet" in rel or href.endswith(".css"): return True if "icon" in rel or "shortcut" in rel: return True return "preload" in rel and as_type in {"style", "script", "font"} def safe_html_name(value: str, fallback: str) -> str: name = re.sub(r"[^A-Za-zА-Яа-яЁё0-9_.-]+", "_", value, flags=re.UNICODE).strip("_") return f"{(name or fallback)[:100]}.html" def safe_asset_name(url: str) -> str: parsed = urllib.parse.urlparse(url) name = Path(parsed.path).name or "asset" stem = re.sub(r"[^A-Za-z0-9_.-]+", "_", Path(name).stem).strip("_") or "asset" suffix = re.sub(r"[^A-Za-z0-9.]+", "", Path(name).suffix) or ".bin" digest = hashlib.sha256(url.encode("utf-8")).hexdigest()[:12] return f"{stem}__{digest}{suffix}" def media_map(media_manifest: dict[str, Any]) -> dict[str, str]: return {str(item.get("url")): str(item.get("file")) for item in media_manifest.get("items") or [] if item.get("url") and item.get("file")} def page_name_map(pages: list[dict[str, Any]]) -> dict[str, str]: result = {} used = set() for index, page in enumerate(pages, start=1): name = safe_html_name(str(page.get("title") or page.get("url") or ""), f"page_{index}") if name in used: stem = Path(name).stem name = f"{stem}_{index}.html" used.add(name) url = str(page.get("url") or "") result[url] = name parsed = urllib.parse.urlparse(url) if parsed.path.endswith("/hdoc"): result[normalize_url(urllib.parse.urlunparse(parsed._replace(path=f"{parsed.path}/01")))] = name return result def collect_asset_urls(pages: list[dict[str, Any]], raw_dir: Path) -> list[str]: urls: set[str] = set() for page in pages: raw_file = str(page.get("raw_file") or "") raw_path = raw_dir / raw_file if not raw_file or not raw_path.exists(): continue page_url = str(page.get("url") or "") raw_text = decode_html(raw_path.read_bytes(), page) collector = AssetCollector(page_url=page_url) collector.feed(raw_text) urls.update(url for url in collector.urls if url.startswith(("http://", "https://"))) return sorted(urls) def download_assets(asset_urls: list[str], assets_dir: Path) -> tuple[dict[str, str], list[dict[str, str]]]: assets_dir.mkdir(parents=True, exist_ok=True) url_to_file: dict[str, str] = {} errors: list[dict[str, str]] = [] for url in asset_urls: filename = safe_asset_name(url) target = assets_dir / filename try: request = urllib.request.Request(url, headers={"User-Agent": "Codex 1C local static archive"}) with urllib.request.urlopen(request, timeout=30) as response: target.write_bytes(response.read()) except Exception as exc: # noqa: BLE001 errors.append({"url": url, "error": str(exc)}) continue url_to_file[url] = filename return url_to_file, errors def read_normalized_body(path: Path) -> str: text = path.read_text(encoding="utf-8-sig") if text.startswith("---"): parts = text.split("---", 2) if len(parts) == 3: text = parts[2] return text.strip() def markdown_to_html(markdown: str, media_url_to_file: dict[str, str]) -> str: lines = markdown.splitlines() out: list[str] = [] in_list = False for line in lines: stripped = line.strip() if not stripped: if in_list: out.append("") in_list = False continue if stripped.startswith("# "): if in_list: out.append("") in_list = False out.append(f"

{html.escape(stripped[2:].strip())}

") continue if stripped.startswith("## "): if in_list: out.append("") in_list = False out.append(f"

{html.escape(stripped[3:].strip())}

") continue image_match = re.match(r"-\s*!\[(.*?)\]\((.*?)\)(.*)", stripped) if image_match: if in_list: out.append("") in_list = False alt, url, suffix = image_match.groups() image_src = f"../media/{media_url_to_file[url]}" if url in media_url_to_file else url out.append(f"
\"{html.escape(alt)}\"
{html.escape((alt + suffix).strip())}
") continue if stripped.startswith("- "): if not in_list: out.append("") in_list = False out.append(f"

{html.escape(stripped)}

") if in_list: out.append("") return "\n".join(out) def write_static_page( page: dict[str, Any], *, normalized_dir: Path, pages_dir: Path, raw_dir: Path, raw_dir_out: Path, page_names: dict[str, str], media_url_to_file: dict[str, str], asset_url_to_file: dict[str, str], ) -> dict[str, Any]: page_url = str(page.get("url") or "") page_file = page_names[page_url] normalized_file = str(page.get("normalized_file") or "") normalized_path = normalized_dir / normalized_file body = markdown_to_html(read_normalized_body(normalized_path), media_url_to_file) raw_file = str(page.get("raw_file") or "") raw_output_name = None if raw_file: raw_path = raw_dir / raw_file if raw_path.exists(): raw_output_name = f"raw_{page_file}" raw_text = decode_html(raw_path.read_bytes(), page) rewriter = LinkRewriter(page_url=page_url, url_to_page=page_names, media_url_to_file=media_url_to_file, asset_url_to_file=asset_url_to_file) rewriter.feed(raw_text) raw_dir_out.mkdir(parents=True, exist_ok=True) (raw_dir_out / raw_output_name).write_text(rewriter.html(), encoding="utf-8") raw_link = f'Raw HTML' if raw_output_name else "" content = f""" {html.escape(str(page.get("title") or ""))}
Индекс{raw_link}{html.escape(str(page.get("source_type") or ""))}
{html.escape(page_url)}
{body}
""" pages_dir.mkdir(parents=True, exist_ok=True) (pages_dir / page_file).write_text(content, encoding="utf-8") return {"title": page.get("title"), "url": page_url, "file": f"pages/{page_file}", "raw_file": f"raw/{raw_output_name}" if raw_output_name else None} def build_static_site(manifest_path: Path, normalized_dir: Path, raw_dir: Path, media_manifest_path: Path, media_dir: Path, output_dir: Path, *, download_external_assets: bool) -> dict[str, Any]: manifest = load_json(manifest_path) media_manifest = load_json(media_manifest_path) pages = manifest.get("pages") or [] page_names = page_name_map(pages) media_url_to_file = media_map(media_manifest) pages_dir = output_dir / "pages" raw_dir_out = output_dir / "raw" static_media_dir = output_dir / "media" static_assets_dir = output_dir / "assets" if output_dir.exists(): shutil.rmtree(output_dir) static_media_dir.mkdir(parents=True, exist_ok=True) for filename in media_url_to_file.values(): source = media_dir / filename if source.exists(): shutil.copy2(source, static_media_dir / filename) asset_urls = collect_asset_urls(pages, raw_dir) if download_external_assets else [] asset_url_to_file, asset_errors = download_assets(asset_urls, static_assets_dir) if asset_urls else ({}, []) page_records = [ write_static_page( page, normalized_dir=normalized_dir, pages_dir=pages_dir, raw_dir=raw_dir, raw_dir_out=raw_dir_out, page_names=page_names, media_url_to_file=media_url_to_file, asset_url_to_file=asset_url_to_file, ) for page in pages if page.get("normalized_file") ] rows = "\n".join( f'{html.escape(str(record["title"] or ""))}{html.escape(str(record["url"] or ""))}' for record in page_records ) index = f""" 1C:ITS Static Archive

1C:ITS Static Archive

Локальный статический просмотр нормализованных страниц.

{rows}
СтраницаURL
""" (output_dir / "index.html").write_text(index, encoding="utf-8") result = { "schema": "onec_its_static_site_manifest.v1", "output_dir": str(output_dir), "index": str(output_dir / "index.html"), "counts": { "pages": len(page_records), "media_files": len(list(static_media_dir.glob("*"))), "asset_files": len(list(static_assets_dir.glob("*"))) if static_assets_dir.exists() else 0, "asset_errors": len(asset_errors), }, "pages": page_records, "assets": [{"url": url, "file": f"assets/{filename}"} for url, filename in sorted(asset_url_to_file.items())], "asset_errors": asset_errors, } (output_dir / "manifest.json").write_text(json.dumps(result, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") return result def main() -> int: parser = argparse.ArgumentParser(description="Build local static HTML viewer for normalized private 1C:ITS docs.") parser.add_argument("--manifest", type=Path, default=DEFAULT_MANIFEST) parser.add_argument("--normalized-dir", type=Path, default=DEFAULT_NORMALIZED_DIR) parser.add_argument("--raw-dir", type=Path, default=DEFAULT_RAW_DIR) parser.add_argument("--media-manifest", type=Path, default=DEFAULT_MEDIA_MANIFEST) parser.add_argument("--media-dir", type=Path, default=DEFAULT_MEDIA_DIR) parser.add_argument("--output-dir", type=Path, default=DEFAULT_OUTPUT_DIR) parser.add_argument("--skip-assets", action="store_true", help="Do not download and localize CSS/JS assets from raw HTML.") args = parser.parse_args() result = build_static_site( args.manifest, args.normalized_dir, args.raw_dir, args.media_manifest, args.media_dir, args.output_dir, download_external_assets=not args.skip_assets, ) print(json.dumps({"counts": result["counts"], "index": result["index"]}, ensure_ascii=False, indent=2)) return 0 if __name__ == "__main__": raise SystemExit(main())