Initial SQL-only 1C adapter baseline
This commit is contained in:
@@ -0,0 +1,432 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import html
|
||||
import json
|
||||
import re
|
||||
import shutil
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
from html.parser import HTMLParser
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from normalize_1c_its_docs import decode_html
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
DEFAULT_MANIFEST = ROOT / "plugins" / "1c" / "rag" / "official-docs" / "normalized" / "manifest.json"
|
||||
DEFAULT_NORMALIZED_DIR = ROOT / "plugins" / "1c" / "rag" / "official-docs" / "normalized"
|
||||
DEFAULT_RAW_DIR = ROOT / "plugins" / "1c" / "rag" / "official-docs" / "raw"
|
||||
DEFAULT_MEDIA_MANIFEST = ROOT / "plugins" / "1c" / "rag" / "official-docs" / "media" / "manifest.json"
|
||||
DEFAULT_MEDIA_DIR = ROOT / "plugins" / "1c" / "rag" / "official-docs" / "media"
|
||||
DEFAULT_OUTPUT_DIR = ROOT / "plugins" / "1c" / "rag" / "official-docs" / "static"
|
||||
|
||||
|
||||
STYLE = """
|
||||
:root {
|
||||
--bg: #f2f3ef;
|
||||
--paper: #fffef9;
|
||||
--ink: #202522;
|
||||
--muted: #66706a;
|
||||
--line: #cdd5cd;
|
||||
--accent: #0f6b5f;
|
||||
--code: #17201c;
|
||||
}
|
||||
* { box-sizing: border-box; }
|
||||
body { margin: 0; background: var(--bg); color: var(--ink); font: 16px/1.55 "Aptos", "Segoe UI", Tahoma, sans-serif; }
|
||||
a { color: var(--accent); }
|
||||
.wrap { max-width: 1120px; margin: 0 auto; padding: 24px; }
|
||||
.doc { background: var(--paper); border: 1px solid var(--line); border-radius: 8px; padding: 24px; }
|
||||
.meta { color: var(--muted); font-size: 13px; overflow-wrap: anywhere; margin-bottom: 18px; }
|
||||
h1 { font-size: 28px; line-height: 1.2; margin: 0 0 12px; }
|
||||
h2 { font-size: 20px; margin-top: 28px; border-top: 1px solid var(--line); padding-top: 18px; }
|
||||
img { max-width: 100%; height: auto; border: 1px solid var(--line); background: #fff; }
|
||||
figure { margin: 18px 0; }
|
||||
figcaption { color: var(--muted); font-size: 13px; margin-top: 6px; }
|
||||
pre { background: var(--code); color: #e4ece6; padding: 12px; border-radius: 6px; overflow: auto; }
|
||||
code { font-family: "Cascadia Mono", Consolas, monospace; }
|
||||
table { width: 100%; border-collapse: collapse; }
|
||||
th, td { border-bottom: 1px solid var(--line); padding: 8px 10px; text-align: left; vertical-align: top; }
|
||||
.top { display: flex; gap: 8px; flex-wrap: wrap; margin-bottom: 14px; }
|
||||
.btn { border: 1px solid var(--line); border-radius: 6px; padding: 6px 10px; background: #fff; text-decoration: none; }
|
||||
.badge { display: inline-block; border: 1px solid var(--line); border-radius: 999px; padding: 2px 8px; color: var(--muted); font-size: 12px; }
|
||||
""".strip()
|
||||
|
||||
|
||||
class AssetCollector(HTMLParser):
|
||||
def __init__(self, *, page_url: str) -> None:
|
||||
super().__init__(convert_charrefs=True)
|
||||
self.page_url = page_url
|
||||
self.urls: set[str] = set()
|
||||
|
||||
def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
|
||||
tag_lower = tag.lower()
|
||||
attr_map = {name.lower(): value or "" for name, value in attrs}
|
||||
for name, value in attrs:
|
||||
if not value:
|
||||
continue
|
||||
name_lower = name.lower()
|
||||
if tag_lower == "link" and name_lower == "href" and is_static_link_asset(attr_map):
|
||||
self.urls.add(normalize_url(urllib.parse.urljoin(self.page_url, value)))
|
||||
elif tag_lower == "script" and name_lower == "src":
|
||||
self.urls.add(normalize_url(urllib.parse.urljoin(self.page_url, value)))
|
||||
|
||||
|
||||
class LinkRewriter(HTMLParser):
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
page_url: str,
|
||||
url_to_page: dict[str, str],
|
||||
media_url_to_file: dict[str, str],
|
||||
asset_url_to_file: dict[str, str],
|
||||
) -> None:
|
||||
super().__init__(convert_charrefs=False)
|
||||
self.page_url = page_url
|
||||
self.url_to_page = url_to_page
|
||||
self.media_url_to_file = media_url_to_file
|
||||
self.asset_url_to_file = asset_url_to_file
|
||||
self.parts: list[str] = []
|
||||
|
||||
def rewrite_url(self, value: str, *, is_media: bool = False) -> str:
|
||||
absolute = normalize_url(urllib.parse.urljoin(self.page_url, value))
|
||||
if is_media and absolute in self.media_url_to_file:
|
||||
return f"../media/{self.media_url_to_file[absolute]}"
|
||||
if absolute in self.asset_url_to_file:
|
||||
return f"../assets/{self.asset_url_to_file[absolute]}"
|
||||
if absolute in self.url_to_page:
|
||||
return f"../pages/{self.url_to_page[absolute]}"
|
||||
if absolute.startswith(("http://", "https://")):
|
||||
return absolute
|
||||
return value
|
||||
|
||||
def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
|
||||
rewritten = []
|
||||
for name, value in attrs:
|
||||
if value is None:
|
||||
rewritten.append((name, None))
|
||||
continue
|
||||
lowered = name.lower()
|
||||
if tag.lower() == "img" and lowered == "src":
|
||||
value = self.rewrite_url(value, is_media=True)
|
||||
elif tag.lower() == "a" and lowered == "href":
|
||||
value = self.rewrite_url(value)
|
||||
elif tag.lower() in {"link", "script"} and lowered in {"href", "src"}:
|
||||
value = self.rewrite_url(value)
|
||||
rewritten.append((name, value))
|
||||
attr_text = "".join(f" {name}" if value is None else f' {name}="{html.escape(value, quote=True)}"' for name, value in rewritten)
|
||||
self.parts.append(f"<{tag}{attr_text}>")
|
||||
|
||||
def handle_endtag(self, tag: str) -> None:
|
||||
self.parts.append(f"</{tag}>")
|
||||
|
||||
def handle_startendtag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
|
||||
self.handle_starttag(tag, attrs)
|
||||
|
||||
def handle_data(self, data: str) -> None:
|
||||
self.parts.append(data)
|
||||
|
||||
def handle_entityref(self, name: str) -> None:
|
||||
self.parts.append(f"&{name};")
|
||||
|
||||
def handle_charref(self, name: str) -> None:
|
||||
self.parts.append(f"&#{name};")
|
||||
|
||||
def handle_comment(self, data: str) -> None:
|
||||
self.parts.append(f"<!--{data}-->")
|
||||
|
||||
def html(self) -> str:
|
||||
return "".join(self.parts)
|
||||
|
||||
|
||||
def load_json(path: Path) -> dict[str, Any]:
|
||||
if not path.exists():
|
||||
return {}
|
||||
try:
|
||||
data = json.loads(path.read_text(encoding="utf-8-sig"))
|
||||
except json.JSONDecodeError:
|
||||
return {}
|
||||
return data if isinstance(data, dict) else {}
|
||||
|
||||
|
||||
def normalize_url(value: str) -> str:
|
||||
return urllib.parse.urlunparse(urllib.parse.urlparse(value)._replace(fragment=""))
|
||||
|
||||
|
||||
def is_static_link_asset(attrs: dict[str, str]) -> bool:
|
||||
rel = {part.casefold() for part in re.split(r"\s+", attrs.get("rel", "")) if part}
|
||||
href = attrs.get("href", "").casefold()
|
||||
as_type = attrs.get("as", "").casefold()
|
||||
if "stylesheet" in rel or href.endswith(".css"):
|
||||
return True
|
||||
if "icon" in rel or "shortcut" in rel:
|
||||
return True
|
||||
return "preload" in rel and as_type in {"style", "script", "font"}
|
||||
|
||||
|
||||
def safe_html_name(value: str, fallback: str) -> str:
|
||||
name = re.sub(r"[^A-Za-zА-Яа-яЁё0-9_.-]+", "_", value, flags=re.UNICODE).strip("_")
|
||||
return f"{(name or fallback)[:100]}.html"
|
||||
|
||||
|
||||
def safe_asset_name(url: str) -> str:
|
||||
parsed = urllib.parse.urlparse(url)
|
||||
name = Path(parsed.path).name or "asset"
|
||||
stem = re.sub(r"[^A-Za-z0-9_.-]+", "_", Path(name).stem).strip("_") or "asset"
|
||||
suffix = re.sub(r"[^A-Za-z0-9.]+", "", Path(name).suffix) or ".bin"
|
||||
digest = hashlib.sha256(url.encode("utf-8")).hexdigest()[:12]
|
||||
return f"{stem}__{digest}{suffix}"
|
||||
|
||||
|
||||
def media_map(media_manifest: dict[str, Any]) -> dict[str, str]:
|
||||
return {str(item.get("url")): str(item.get("file")) for item in media_manifest.get("items") or [] if item.get("url") and item.get("file")}
|
||||
|
||||
|
||||
def page_name_map(pages: list[dict[str, Any]]) -> dict[str, str]:
|
||||
result = {}
|
||||
used = set()
|
||||
for index, page in enumerate(pages, start=1):
|
||||
name = safe_html_name(str(page.get("title") or page.get("url") or ""), f"page_{index}")
|
||||
if name in used:
|
||||
stem = Path(name).stem
|
||||
name = f"{stem}_{index}.html"
|
||||
used.add(name)
|
||||
url = str(page.get("url") or "")
|
||||
result[url] = name
|
||||
parsed = urllib.parse.urlparse(url)
|
||||
if parsed.path.endswith("/hdoc"):
|
||||
result[normalize_url(urllib.parse.urlunparse(parsed._replace(path=f"{parsed.path}/01")))] = name
|
||||
return result
|
||||
|
||||
|
||||
def collect_asset_urls(pages: list[dict[str, Any]], raw_dir: Path) -> list[str]:
|
||||
urls: set[str] = set()
|
||||
for page in pages:
|
||||
raw_file = str(page.get("raw_file") or "")
|
||||
raw_path = raw_dir / raw_file
|
||||
if not raw_file or not raw_path.exists():
|
||||
continue
|
||||
page_url = str(page.get("url") or "")
|
||||
raw_text = decode_html(raw_path.read_bytes(), page)
|
||||
collector = AssetCollector(page_url=page_url)
|
||||
collector.feed(raw_text)
|
||||
urls.update(url for url in collector.urls if url.startswith(("http://", "https://")))
|
||||
return sorted(urls)
|
||||
|
||||
|
||||
def download_assets(asset_urls: list[str], assets_dir: Path) -> tuple[dict[str, str], list[dict[str, str]]]:
|
||||
assets_dir.mkdir(parents=True, exist_ok=True)
|
||||
url_to_file: dict[str, str] = {}
|
||||
errors: list[dict[str, str]] = []
|
||||
for url in asset_urls:
|
||||
filename = safe_asset_name(url)
|
||||
target = assets_dir / filename
|
||||
try:
|
||||
request = urllib.request.Request(url, headers={"User-Agent": "Codex 1C local static archive"})
|
||||
with urllib.request.urlopen(request, timeout=30) as response:
|
||||
target.write_bytes(response.read())
|
||||
except Exception as exc: # noqa: BLE001
|
||||
errors.append({"url": url, "error": str(exc)})
|
||||
continue
|
||||
url_to_file[url] = filename
|
||||
return url_to_file, errors
|
||||
|
||||
|
||||
def read_normalized_body(path: Path) -> str:
|
||||
text = path.read_text(encoding="utf-8-sig")
|
||||
if text.startswith("---"):
|
||||
parts = text.split("---", 2)
|
||||
if len(parts) == 3:
|
||||
text = parts[2]
|
||||
return text.strip()
|
||||
|
||||
|
||||
def markdown_to_html(markdown: str, media_url_to_file: dict[str, str]) -> str:
|
||||
lines = markdown.splitlines()
|
||||
out: list[str] = []
|
||||
in_list = False
|
||||
for line in lines:
|
||||
stripped = line.strip()
|
||||
if not stripped:
|
||||
if in_list:
|
||||
out.append("</ul>")
|
||||
in_list = False
|
||||
continue
|
||||
if stripped.startswith("# "):
|
||||
if in_list:
|
||||
out.append("</ul>")
|
||||
in_list = False
|
||||
out.append(f"<h1>{html.escape(stripped[2:].strip())}</h1>")
|
||||
continue
|
||||
if stripped.startswith("## "):
|
||||
if in_list:
|
||||
out.append("</ul>")
|
||||
in_list = False
|
||||
out.append(f"<h2>{html.escape(stripped[3:].strip())}</h2>")
|
||||
continue
|
||||
image_match = re.match(r"-\s*!\[(.*?)\]\((.*?)\)(.*)", stripped)
|
||||
if image_match:
|
||||
if in_list:
|
||||
out.append("</ul>")
|
||||
in_list = False
|
||||
alt, url, suffix = image_match.groups()
|
||||
image_src = f"../media/{media_url_to_file[url]}" if url in media_url_to_file else url
|
||||
out.append(f"<figure><img src=\"{html.escape(image_src, quote=True)}\" alt=\"{html.escape(alt)}\"><figcaption>{html.escape((alt + suffix).strip())}</figcaption></figure>")
|
||||
continue
|
||||
if stripped.startswith("- "):
|
||||
if not in_list:
|
||||
out.append("<ul>")
|
||||
in_list = True
|
||||
out.append(f"<li>{html.escape(stripped[2:].strip())}</li>")
|
||||
continue
|
||||
if in_list:
|
||||
out.append("</ul>")
|
||||
in_list = False
|
||||
out.append(f"<p>{html.escape(stripped)}</p>")
|
||||
if in_list:
|
||||
out.append("</ul>")
|
||||
return "\n".join(out)
|
||||
|
||||
|
||||
def write_static_page(
|
||||
page: dict[str, Any],
|
||||
*,
|
||||
normalized_dir: Path,
|
||||
pages_dir: Path,
|
||||
raw_dir: Path,
|
||||
raw_dir_out: Path,
|
||||
page_names: dict[str, str],
|
||||
media_url_to_file: dict[str, str],
|
||||
asset_url_to_file: dict[str, str],
|
||||
) -> dict[str, Any]:
|
||||
page_url = str(page.get("url") or "")
|
||||
page_file = page_names[page_url]
|
||||
normalized_file = str(page.get("normalized_file") or "")
|
||||
normalized_path = normalized_dir / normalized_file
|
||||
body = markdown_to_html(read_normalized_body(normalized_path), media_url_to_file)
|
||||
raw_file = str(page.get("raw_file") or "")
|
||||
raw_output_name = None
|
||||
if raw_file:
|
||||
raw_path = raw_dir / raw_file
|
||||
if raw_path.exists():
|
||||
raw_output_name = f"raw_{page_file}"
|
||||
raw_text = decode_html(raw_path.read_bytes(), page)
|
||||
rewriter = LinkRewriter(page_url=page_url, url_to_page=page_names, media_url_to_file=media_url_to_file, asset_url_to_file=asset_url_to_file)
|
||||
rewriter.feed(raw_text)
|
||||
raw_dir_out.mkdir(parents=True, exist_ok=True)
|
||||
(raw_dir_out / raw_output_name).write_text(rewriter.html(), encoding="utf-8")
|
||||
|
||||
raw_link = f'<a class="btn" href="../raw/{raw_output_name}">Raw HTML</a>' if raw_output_name else ""
|
||||
content = f"""<!doctype html>
|
||||
<html lang="ru">
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<title>{html.escape(str(page.get("title") or ""))}</title>
|
||||
<style>{STYLE}</style>
|
||||
</head>
|
||||
<body>
|
||||
<main class="wrap">
|
||||
<div class="top"><a class="btn" href="../index.html">Индекс</a>{raw_link}<span class="badge">{html.escape(str(page.get("source_type") or ""))}</span></div>
|
||||
<article class="doc">
|
||||
<div class="meta">{html.escape(page_url)}</div>
|
||||
{body}
|
||||
</article>
|
||||
</main>
|
||||
</body>
|
||||
</html>
|
||||
"""
|
||||
pages_dir.mkdir(parents=True, exist_ok=True)
|
||||
(pages_dir / page_file).write_text(content, encoding="utf-8")
|
||||
return {"title": page.get("title"), "url": page_url, "file": f"pages/{page_file}", "raw_file": f"raw/{raw_output_name}" if raw_output_name else None}
|
||||
|
||||
|
||||
def build_static_site(manifest_path: Path, normalized_dir: Path, raw_dir: Path, media_manifest_path: Path, media_dir: Path, output_dir: Path, *, download_external_assets: bool) -> dict[str, Any]:
|
||||
manifest = load_json(manifest_path)
|
||||
media_manifest = load_json(media_manifest_path)
|
||||
pages = manifest.get("pages") or []
|
||||
page_names = page_name_map(pages)
|
||||
media_url_to_file = media_map(media_manifest)
|
||||
pages_dir = output_dir / "pages"
|
||||
raw_dir_out = output_dir / "raw"
|
||||
static_media_dir = output_dir / "media"
|
||||
static_assets_dir = output_dir / "assets"
|
||||
if output_dir.exists():
|
||||
shutil.rmtree(output_dir)
|
||||
static_media_dir.mkdir(parents=True, exist_ok=True)
|
||||
for filename in media_url_to_file.values():
|
||||
source = media_dir / filename
|
||||
if source.exists():
|
||||
shutil.copy2(source, static_media_dir / filename)
|
||||
asset_urls = collect_asset_urls(pages, raw_dir) if download_external_assets else []
|
||||
asset_url_to_file, asset_errors = download_assets(asset_urls, static_assets_dir) if asset_urls else ({}, [])
|
||||
page_records = [
|
||||
write_static_page(
|
||||
page,
|
||||
normalized_dir=normalized_dir,
|
||||
pages_dir=pages_dir,
|
||||
raw_dir=raw_dir,
|
||||
raw_dir_out=raw_dir_out,
|
||||
page_names=page_names,
|
||||
media_url_to_file=media_url_to_file,
|
||||
asset_url_to_file=asset_url_to_file,
|
||||
)
|
||||
for page in pages
|
||||
if page.get("normalized_file")
|
||||
]
|
||||
rows = "\n".join(
|
||||
f'<tr><td><a href="{html.escape(record["file"], quote=True)}">{html.escape(str(record["title"] or ""))}</a></td><td>{html.escape(str(record["url"] or ""))}</td></tr>'
|
||||
for record in page_records
|
||||
)
|
||||
index = f"""<!doctype html>
|
||||
<html lang="ru">
|
||||
<head><meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1"><title>1C:ITS Static Archive</title><style>{STYLE}</style></head>
|
||||
<body><main class="wrap"><article class="doc"><h1>1C:ITS Static Archive</h1><p class="meta">Локальный статический просмотр нормализованных страниц.</p><table><thead><tr><th>Страница</th><th>URL</th></tr></thead><tbody>{rows}</tbody></table></article></main></body>
|
||||
</html>
|
||||
"""
|
||||
(output_dir / "index.html").write_text(index, encoding="utf-8")
|
||||
result = {
|
||||
"schema": "onec_its_static_site_manifest.v1",
|
||||
"output_dir": str(output_dir),
|
||||
"index": str(output_dir / "index.html"),
|
||||
"counts": {
|
||||
"pages": len(page_records),
|
||||
"media_files": len(list(static_media_dir.glob("*"))),
|
||||
"asset_files": len(list(static_assets_dir.glob("*"))) if static_assets_dir.exists() else 0,
|
||||
"asset_errors": len(asset_errors),
|
||||
},
|
||||
"pages": page_records,
|
||||
"assets": [{"url": url, "file": f"assets/{filename}"} for url, filename in sorted(asset_url_to_file.items())],
|
||||
"asset_errors": asset_errors,
|
||||
}
|
||||
(output_dir / "manifest.json").write_text(json.dumps(result, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
return result
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Build local static HTML viewer for normalized private 1C:ITS docs.")
|
||||
parser.add_argument("--manifest", type=Path, default=DEFAULT_MANIFEST)
|
||||
parser.add_argument("--normalized-dir", type=Path, default=DEFAULT_NORMALIZED_DIR)
|
||||
parser.add_argument("--raw-dir", type=Path, default=DEFAULT_RAW_DIR)
|
||||
parser.add_argument("--media-manifest", type=Path, default=DEFAULT_MEDIA_MANIFEST)
|
||||
parser.add_argument("--media-dir", type=Path, default=DEFAULT_MEDIA_DIR)
|
||||
parser.add_argument("--output-dir", type=Path, default=DEFAULT_OUTPUT_DIR)
|
||||
parser.add_argument("--skip-assets", action="store_true", help="Do not download and localize CSS/JS assets from raw HTML.")
|
||||
args = parser.parse_args()
|
||||
result = build_static_site(
|
||||
args.manifest,
|
||||
args.normalized_dir,
|
||||
args.raw_dir,
|
||||
args.media_manifest,
|
||||
args.media_dir,
|
||||
args.output_dir,
|
||||
download_external_assets=not args.skip_assets,
|
||||
)
|
||||
print(json.dumps({"counts": result["counts"], "index": result["index"]}, ensure_ascii=False, indent=2))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Reference in New Issue
Block a user