Initial SQL-only 1C adapter baseline
This commit is contained in:
@@ -0,0 +1,184 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import time
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
from html.parser import HTMLParser
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
DEFAULT_OUTPUT = ROOT / "plugins" / "1c" / "rag" / "official-docs" / "start-links.json"
|
||||
DEFAULT_ROOT_URL = "https://its.1c.ru/"
|
||||
|
||||
|
||||
class AnchorParser(HTMLParser):
|
||||
def __init__(self) -> None:
|
||||
super().__init__(convert_charrefs=True)
|
||||
self.title_parts: list[str] = []
|
||||
self.links: list[dict[str, str]] = []
|
||||
self._in_title = False
|
||||
self._current: dict[str, Any] | None = None
|
||||
|
||||
def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
|
||||
tag = tag.lower()
|
||||
if tag == "title":
|
||||
self._in_title = True
|
||||
if tag != "a":
|
||||
return
|
||||
href = ""
|
||||
for name, value in attrs:
|
||||
if name.lower() == "href" and value:
|
||||
href = value
|
||||
break
|
||||
if href:
|
||||
self._current = {"href": href, "text": []}
|
||||
|
||||
def handle_endtag(self, tag: str) -> None:
|
||||
tag = tag.lower()
|
||||
if tag == "title":
|
||||
self._in_title = False
|
||||
if tag == "a" and self._current:
|
||||
self.links.append({"href": str(self._current["href"]), "text": " ".join(self._current["text"]).strip()})
|
||||
self._current = None
|
||||
|
||||
def handle_data(self, data: str) -> None:
|
||||
text = data.strip()
|
||||
if not text:
|
||||
return
|
||||
if self._in_title:
|
||||
self.title_parts.append(text)
|
||||
if self._current is not None:
|
||||
self._current["text"].append(text)
|
||||
|
||||
@property
|
||||
def title(self) -> str:
|
||||
return " ".join(self.title_parts).strip()
|
||||
|
||||
|
||||
def read_cookie(cookie_file: Path | None) -> str:
|
||||
if cookie_file:
|
||||
return cookie_file.read_text(encoding="utf-8").strip()
|
||||
return os.environ.get("ONEC_ITS_COOKIE", "").strip()
|
||||
|
||||
|
||||
def charset_from_content_type(content_type: str) -> str:
|
||||
match = re.search(r"charset=([^;\s]+)", content_type, flags=re.IGNORECASE)
|
||||
return match.group(1).strip("\"'") if match else "utf-8"
|
||||
|
||||
|
||||
def fetch_html(url: str, *, cookie: str, timeout: int, user_agent: str) -> tuple[str, str, int]:
|
||||
headers = {"User-Agent": user_agent}
|
||||
if cookie:
|
||||
headers["Cookie"] = cookie
|
||||
request = urllib.request.Request(url, headers=headers)
|
||||
with urllib.request.urlopen(request, timeout=timeout) as response:
|
||||
body = response.read()
|
||||
encoding = charset_from_content_type(response.headers.get("Content-Type", ""))
|
||||
return body.decode(encoding, errors="replace"), response.geturl(), int(response.status)
|
||||
|
||||
|
||||
def normalize_url(base_url: str, href: str) -> str | None:
|
||||
if href.startswith(("mailto:", "tel:", "javascript:")):
|
||||
return None
|
||||
absolute = urllib.parse.urljoin(base_url, href)
|
||||
parsed = urllib.parse.urlparse(absolute)
|
||||
if parsed.scheme not in {"http", "https"} or not parsed.netloc.endswith("its.1c.ru"):
|
||||
return None
|
||||
return urllib.parse.urlunparse(parsed._replace(fragment=""))
|
||||
|
||||
|
||||
def parse_page(url: str, *, cookie: str, timeout: int, user_agent: str) -> dict[str, Any]:
|
||||
html, final_url, status = fetch_html(url, cookie=cookie, timeout=timeout, user_agent=user_agent)
|
||||
parser = AnchorParser()
|
||||
parser.feed(html)
|
||||
links = []
|
||||
seen = set()
|
||||
for item in parser.links:
|
||||
next_url = normalize_url(final_url, item["href"])
|
||||
if not next_url or next_url in seen:
|
||||
continue
|
||||
seen.add(next_url)
|
||||
links.append({"url": next_url, "text": item["text"]})
|
||||
return {"url": url, "final_url": final_url, "status": status, "title": parser.title, "bytes": len(html.encode("utf-8")), "links": links}
|
||||
|
||||
|
||||
def classify_link(url: str, text: str) -> str | None:
|
||||
path = urllib.parse.urlparse(url).path
|
||||
lowered = text.casefold()
|
||||
if path == "/section/dev":
|
||||
return "dev_section"
|
||||
if path.startswith("/section/dev/"):
|
||||
if any(marker in path for marker in ("/doc_dev", "/method_dev", "/doc_edt", "/doc_bsp", "/doc_fresh")):
|
||||
return "dev_section_index"
|
||||
return None
|
||||
if re.fullmatch(r"/db/v8\d*doc", path) or re.fullmatch(r"/db/v83\d+doc", path) or re.fullmatch(r"/db/v85\d+doc", path):
|
||||
return "platform_doc"
|
||||
if path.startswith("/db/v8devgloss"):
|
||||
return "developer_glossary"
|
||||
if path.startswith("/db/metod8dev"):
|
||||
return "methodical_support"
|
||||
if path.startswith("/db/v8std"):
|
||||
return "development_standards"
|
||||
if path.startswith("/db/fresh") or path.startswith("/db/sdadmin"):
|
||||
return "platform_related_doc"
|
||||
if path.startswith("/db/bsp") or path.startswith("/db/bid") or path.startswith("/db/bia"):
|
||||
return "library_doc"
|
||||
if path.startswith("/db/pub") and any(word in lowered for word in ("разработ", "1с:предприятие", "мобильн", "расширен", "интеграц", "отчет")):
|
||||
return "developer_book"
|
||||
if path.startswith("/db/intgr83") or path.startswith("/db/coldev"):
|
||||
return "developer_book"
|
||||
return None
|
||||
|
||||
|
||||
def discover(root_url: str, *, cookie: str, timeout: int, user_agent: str) -> dict[str, Any]:
|
||||
root = parse_page(root_url, cookie=cookie, timeout=timeout, user_agent=user_agent)
|
||||
dev_url = next((item["url"] for item in root["links"] if item["url"].rstrip("/") == "https://its.1c.ru/section/dev"), "https://its.1c.ru/section/dev")
|
||||
dev = parse_page(dev_url, cookie=cookie, timeout=timeout, user_agent=user_agent)
|
||||
candidates_by_url: dict[str, dict[str, str]] = {}
|
||||
for page in (root, dev):
|
||||
for item in page["links"]:
|
||||
category = classify_link(item["url"], item["text"])
|
||||
if not category:
|
||||
continue
|
||||
candidates_by_url[item["url"]] = {"url": item["url"], "text": item["text"], "category": category}
|
||||
candidates = sorted(candidates_by_url.values(), key=lambda item: (item["category"], item["url"]))
|
||||
categories = sorted({item["category"] for item in candidates})
|
||||
return {
|
||||
"schema": "onec_its_start_links.v1",
|
||||
"created_at_unix": int(time.time()),
|
||||
"root": {"url": root["url"], "title": root["title"], "link_count": len(root["links"])},
|
||||
"dev_section": {"url": dev["url"], "title": dev["title"], "link_count": len(dev["links"])},
|
||||
"counts": {
|
||||
"candidates": len(candidates),
|
||||
"by_category": {category: sum(1 for item in candidates if item["category"] == category) for category in categories},
|
||||
},
|
||||
"start_links": candidates,
|
||||
}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Discover useful 1C:ITS start links from https://its.1c.ru/ and /section/dev.")
|
||||
parser.add_argument("--root-url", default=DEFAULT_ROOT_URL)
|
||||
parser.add_argument("--cookie-file", type=Path)
|
||||
parser.add_argument("--output", type=Path, default=DEFAULT_OUTPUT)
|
||||
parser.add_argument("--timeout", type=int, default=30)
|
||||
parser.add_argument("--user-agent", default="Codex 1C ITS start-link discovery")
|
||||
parser.add_argument("--print", action="store_true", dest="print_report")
|
||||
args = parser.parse_args()
|
||||
|
||||
report = discover(args.root_url, cookie=read_cookie(args.cookie_file), timeout=args.timeout, user_agent=args.user_agent)
|
||||
args.output.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.output.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
payload = report if args.print_report else {"output": str(args.output), "counts": report["counts"]}
|
||||
print(json.dumps(payload, ensure_ascii=False, indent=2))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Reference in New Issue
Block a user