185 lines
7.6 KiB
Python
185 lines
7.6 KiB
Python
from __future__ import annotations
|
||
|
||
import argparse
|
||
import json
|
||
import os
|
||
import re
|
||
import time
|
||
import urllib.parse
|
||
import urllib.request
|
||
from html.parser import HTMLParser
|
||
from pathlib import Path
|
||
from typing import Any
|
||
|
||
|
||
ROOT = Path(__file__).resolve().parents[1]
|
||
DEFAULT_OUTPUT = ROOT / "plugins" / "1c" / "rag" / "official-docs" / "start-links.json"
|
||
DEFAULT_ROOT_URL = "https://its.1c.ru/"
|
||
|
||
|
||
class AnchorParser(HTMLParser):
|
||
def __init__(self) -> None:
|
||
super().__init__(convert_charrefs=True)
|
||
self.title_parts: list[str] = []
|
||
self.links: list[dict[str, str]] = []
|
||
self._in_title = False
|
||
self._current: dict[str, Any] | None = None
|
||
|
||
def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
|
||
tag = tag.lower()
|
||
if tag == "title":
|
||
self._in_title = True
|
||
if tag != "a":
|
||
return
|
||
href = ""
|
||
for name, value in attrs:
|
||
if name.lower() == "href" and value:
|
||
href = value
|
||
break
|
||
if href:
|
||
self._current = {"href": href, "text": []}
|
||
|
||
def handle_endtag(self, tag: str) -> None:
|
||
tag = tag.lower()
|
||
if tag == "title":
|
||
self._in_title = False
|
||
if tag == "a" and self._current:
|
||
self.links.append({"href": str(self._current["href"]), "text": " ".join(self._current["text"]).strip()})
|
||
self._current = None
|
||
|
||
def handle_data(self, data: str) -> None:
|
||
text = data.strip()
|
||
if not text:
|
||
return
|
||
if self._in_title:
|
||
self.title_parts.append(text)
|
||
if self._current is not None:
|
||
self._current["text"].append(text)
|
||
|
||
@property
|
||
def title(self) -> str:
|
||
return " ".join(self.title_parts).strip()
|
||
|
||
|
||
def read_cookie(cookie_file: Path | None) -> str:
|
||
if cookie_file:
|
||
return cookie_file.read_text(encoding="utf-8").strip()
|
||
return os.environ.get("ONEC_ITS_COOKIE", "").strip()
|
||
|
||
|
||
def charset_from_content_type(content_type: str) -> str:
|
||
match = re.search(r"charset=([^;\s]+)", content_type, flags=re.IGNORECASE)
|
||
return match.group(1).strip("\"'") if match else "utf-8"
|
||
|
||
|
||
def fetch_html(url: str, *, cookie: str, timeout: int, user_agent: str) -> tuple[str, str, int]:
|
||
headers = {"User-Agent": user_agent}
|
||
if cookie:
|
||
headers["Cookie"] = cookie
|
||
request = urllib.request.Request(url, headers=headers)
|
||
with urllib.request.urlopen(request, timeout=timeout) as response:
|
||
body = response.read()
|
||
encoding = charset_from_content_type(response.headers.get("Content-Type", ""))
|
||
return body.decode(encoding, errors="replace"), response.geturl(), int(response.status)
|
||
|
||
|
||
def normalize_url(base_url: str, href: str) -> str | None:
|
||
if href.startswith(("mailto:", "tel:", "javascript:")):
|
||
return None
|
||
absolute = urllib.parse.urljoin(base_url, href)
|
||
parsed = urllib.parse.urlparse(absolute)
|
||
if parsed.scheme not in {"http", "https"} or not parsed.netloc.endswith("its.1c.ru"):
|
||
return None
|
||
return urllib.parse.urlunparse(parsed._replace(fragment=""))
|
||
|
||
|
||
def parse_page(url: str, *, cookie: str, timeout: int, user_agent: str) -> dict[str, Any]:
|
||
html, final_url, status = fetch_html(url, cookie=cookie, timeout=timeout, user_agent=user_agent)
|
||
parser = AnchorParser()
|
||
parser.feed(html)
|
||
links = []
|
||
seen = set()
|
||
for item in parser.links:
|
||
next_url = normalize_url(final_url, item["href"])
|
||
if not next_url or next_url in seen:
|
||
continue
|
||
seen.add(next_url)
|
||
links.append({"url": next_url, "text": item["text"]})
|
||
return {"url": url, "final_url": final_url, "status": status, "title": parser.title, "bytes": len(html.encode("utf-8")), "links": links}
|
||
|
||
|
||
def classify_link(url: str, text: str) -> str | None:
|
||
path = urllib.parse.urlparse(url).path
|
||
lowered = text.casefold()
|
||
if path == "/section/dev":
|
||
return "dev_section"
|
||
if path.startswith("/section/dev/"):
|
||
if any(marker in path for marker in ("/doc_dev", "/method_dev", "/doc_edt", "/doc_bsp", "/doc_fresh")):
|
||
return "dev_section_index"
|
||
return None
|
||
if re.fullmatch(r"/db/v8\d*doc", path) or re.fullmatch(r"/db/v83\d+doc", path) or re.fullmatch(r"/db/v85\d+doc", path):
|
||
return "platform_doc"
|
||
if path.startswith("/db/v8devgloss"):
|
||
return "developer_glossary"
|
||
if path.startswith("/db/metod8dev"):
|
||
return "methodical_support"
|
||
if path.startswith("/db/v8std"):
|
||
return "development_standards"
|
||
if path.startswith("/db/fresh") or path.startswith("/db/sdadmin"):
|
||
return "platform_related_doc"
|
||
if path.startswith("/db/bsp") or path.startswith("/db/bid") or path.startswith("/db/bia"):
|
||
return "library_doc"
|
||
if path.startswith("/db/pub") and any(word in lowered for word in ("разработ", "1с:предприятие", "мобильн", "расширен", "интеграц", "отчет")):
|
||
return "developer_book"
|
||
if path.startswith("/db/intgr83") or path.startswith("/db/coldev"):
|
||
return "developer_book"
|
||
return None
|
||
|
||
|
||
def discover(root_url: str, *, cookie: str, timeout: int, user_agent: str) -> dict[str, Any]:
|
||
root = parse_page(root_url, cookie=cookie, timeout=timeout, user_agent=user_agent)
|
||
dev_url = next((item["url"] for item in root["links"] if item["url"].rstrip("/") == "https://its.1c.ru/section/dev"), "https://its.1c.ru/section/dev")
|
||
dev = parse_page(dev_url, cookie=cookie, timeout=timeout, user_agent=user_agent)
|
||
candidates_by_url: dict[str, dict[str, str]] = {}
|
||
for page in (root, dev):
|
||
for item in page["links"]:
|
||
category = classify_link(item["url"], item["text"])
|
||
if not category:
|
||
continue
|
||
candidates_by_url[item["url"]] = {"url": item["url"], "text": item["text"], "category": category}
|
||
candidates = sorted(candidates_by_url.values(), key=lambda item: (item["category"], item["url"]))
|
||
categories = sorted({item["category"] for item in candidates})
|
||
return {
|
||
"schema": "onec_its_start_links.v1",
|
||
"created_at_unix": int(time.time()),
|
||
"root": {"url": root["url"], "title": root["title"], "link_count": len(root["links"])},
|
||
"dev_section": {"url": dev["url"], "title": dev["title"], "link_count": len(dev["links"])},
|
||
"counts": {
|
||
"candidates": len(candidates),
|
||
"by_category": {category: sum(1 for item in candidates if item["category"] == category) for category in categories},
|
||
},
|
||
"start_links": candidates,
|
||
}
|
||
|
||
|
||
def main() -> int:
|
||
parser = argparse.ArgumentParser(description="Discover useful 1C:ITS start links from https://its.1c.ru/ and /section/dev.")
|
||
parser.add_argument("--root-url", default=DEFAULT_ROOT_URL)
|
||
parser.add_argument("--cookie-file", type=Path)
|
||
parser.add_argument("--output", type=Path, default=DEFAULT_OUTPUT)
|
||
parser.add_argument("--timeout", type=int, default=30)
|
||
parser.add_argument("--user-agent", default="Codex 1C ITS start-link discovery")
|
||
parser.add_argument("--print", action="store_true", dest="print_report")
|
||
args = parser.parse_args()
|
||
|
||
report = discover(args.root_url, cookie=read_cookie(args.cookie_file), timeout=args.timeout, user_agent=args.user_agent)
|
||
args.output.parent.mkdir(parents=True, exist_ok=True)
|
||
args.output.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||
payload = report if args.print_report else {"output": str(args.output), "counts": report["counts"]}
|
||
print(json.dumps(payload, ensure_ascii=False, indent=2))
|
||
return 0
|
||
|
||
|
||
if __name__ == "__main__":
|
||
raise SystemExit(main())
|