from __future__ import annotations import argparse import json import os import re import time import urllib.parse import urllib.request from html.parser import HTMLParser from pathlib import Path from typing import Any ROOT = Path(__file__).resolve().parents[1] DEFAULT_OUTPUT = ROOT / "plugins" / "1c" / "rag" / "official-docs" / "start-links.json" DEFAULT_ROOT_URL = "https://its.1c.ru/" class AnchorParser(HTMLParser): def __init__(self) -> None: super().__init__(convert_charrefs=True) self.title_parts: list[str] = [] self.links: list[dict[str, str]] = [] self._in_title = False self._current: dict[str, Any] | None = None def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None: tag = tag.lower() if tag == "title": self._in_title = True if tag != "a": return href = "" for name, value in attrs: if name.lower() == "href" and value: href = value break if href: self._current = {"href": href, "text": []} def handle_endtag(self, tag: str) -> None: tag = tag.lower() if tag == "title": self._in_title = False if tag == "a" and self._current: self.links.append({"href": str(self._current["href"]), "text": " ".join(self._current["text"]).strip()}) self._current = None def handle_data(self, data: str) -> None: text = data.strip() if not text: return if self._in_title: self.title_parts.append(text) if self._current is not None: self._current["text"].append(text) @property def title(self) -> str: return " ".join(self.title_parts).strip() def read_cookie(cookie_file: Path | None) -> str: if cookie_file: return cookie_file.read_text(encoding="utf-8").strip() return os.environ.get("ONEC_ITS_COOKIE", "").strip() def charset_from_content_type(content_type: str) -> str: match = re.search(r"charset=([^;\s]+)", content_type, flags=re.IGNORECASE) return match.group(1).strip("\"'") if match else "utf-8" def fetch_html(url: str, *, cookie: str, timeout: int, user_agent: str) -> tuple[str, str, int]: headers = {"User-Agent": user_agent} if cookie: headers["Cookie"] = cookie request = urllib.request.Request(url, headers=headers) with urllib.request.urlopen(request, timeout=timeout) as response: body = response.read() encoding = charset_from_content_type(response.headers.get("Content-Type", "")) return body.decode(encoding, errors="replace"), response.geturl(), int(response.status) def normalize_url(base_url: str, href: str) -> str | None: if href.startswith(("mailto:", "tel:", "javascript:")): return None absolute = urllib.parse.urljoin(base_url, href) parsed = urllib.parse.urlparse(absolute) if parsed.scheme not in {"http", "https"} or not parsed.netloc.endswith("its.1c.ru"): return None return urllib.parse.urlunparse(parsed._replace(fragment="")) def parse_page(url: str, *, cookie: str, timeout: int, user_agent: str) -> dict[str, Any]: html, final_url, status = fetch_html(url, cookie=cookie, timeout=timeout, user_agent=user_agent) parser = AnchorParser() parser.feed(html) links = [] seen = set() for item in parser.links: next_url = normalize_url(final_url, item["href"]) if not next_url or next_url in seen: continue seen.add(next_url) links.append({"url": next_url, "text": item["text"]}) return {"url": url, "final_url": final_url, "status": status, "title": parser.title, "bytes": len(html.encode("utf-8")), "links": links} def classify_link(url: str, text: str) -> str | None: path = urllib.parse.urlparse(url).path lowered = text.casefold() if path == "/section/dev": return "dev_section" if path.startswith("/section/dev/"): if any(marker in path for marker in ("/doc_dev", "/method_dev", "/doc_edt", "/doc_bsp", "/doc_fresh")): return "dev_section_index" return None if re.fullmatch(r"/db/v8\d*doc", path) or re.fullmatch(r"/db/v83\d+doc", path) or re.fullmatch(r"/db/v85\d+doc", path): return "platform_doc" if path.startswith("/db/v8devgloss"): return "developer_glossary" if path.startswith("/db/metod8dev"): return "methodical_support" if path.startswith("/db/v8std"): return "development_standards" if path.startswith("/db/fresh") or path.startswith("/db/sdadmin"): return "platform_related_doc" if path.startswith("/db/bsp") or path.startswith("/db/bid") or path.startswith("/db/bia"): return "library_doc" if path.startswith("/db/pub") and any(word in lowered for word in ("разработ", "1с:предприятие", "мобильн", "расширен", "интеграц", "отчет")): return "developer_book" if path.startswith("/db/intgr83") or path.startswith("/db/coldev"): return "developer_book" return None def discover(root_url: str, *, cookie: str, timeout: int, user_agent: str) -> dict[str, Any]: root = parse_page(root_url, cookie=cookie, timeout=timeout, user_agent=user_agent) dev_url = next((item["url"] for item in root["links"] if item["url"].rstrip("/") == "https://its.1c.ru/section/dev"), "https://its.1c.ru/section/dev") dev = parse_page(dev_url, cookie=cookie, timeout=timeout, user_agent=user_agent) candidates_by_url: dict[str, dict[str, str]] = {} for page in (root, dev): for item in page["links"]: category = classify_link(item["url"], item["text"]) if not category: continue candidates_by_url[item["url"]] = {"url": item["url"], "text": item["text"], "category": category} candidates = sorted(candidates_by_url.values(), key=lambda item: (item["category"], item["url"])) categories = sorted({item["category"] for item in candidates}) return { "schema": "onec_its_start_links.v1", "created_at_unix": int(time.time()), "root": {"url": root["url"], "title": root["title"], "link_count": len(root["links"])}, "dev_section": {"url": dev["url"], "title": dev["title"], "link_count": len(dev["links"])}, "counts": { "candidates": len(candidates), "by_category": {category: sum(1 for item in candidates if item["category"] == category) for category in categories}, }, "start_links": candidates, } def main() -> int: parser = argparse.ArgumentParser(description="Discover useful 1C:ITS start links from https://its.1c.ru/ and /section/dev.") parser.add_argument("--root-url", default=DEFAULT_ROOT_URL) parser.add_argument("--cookie-file", type=Path) parser.add_argument("--output", type=Path, default=DEFAULT_OUTPUT) parser.add_argument("--timeout", type=int, default=30) parser.add_argument("--user-agent", default="Codex 1C ITS start-link discovery") parser.add_argument("--print", action="store_true", dest="print_report") args = parser.parse_args() report = discover(args.root_url, cookie=read_cookie(args.cookie_file), timeout=args.timeout, user_agent=args.user_agent) args.output.parent.mkdir(parents=True, exist_ok=True) args.output.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") payload = report if args.print_report else {"output": str(args.output), "counts": report["counts"]} print(json.dumps(payload, ensure_ascii=False, indent=2)) return 0 if __name__ == "__main__": raise SystemExit(main())