from __future__ import annotations import argparse import hashlib import json import os import re import time import urllib.error import urllib.parse import urllib.request from collections import deque from html.parser import HTMLParser from pathlib import Path from typing import Any import yaml from one_c_its_platform import merge_platform_metadata ROOT = Path(__file__).resolve().parents[1] DEFAULT_SOURCES = ROOT / "plugins" / "1c" / "rag" / "official-docs" / "sources.yaml" DEFAULT_OUTPUT_DIR = ROOT / "plugins" / "1c" / "rag" / "official-docs" / "raw" DEFAULT_MANIFEST = DEFAULT_OUTPUT_DIR / "manifest.json" DEFAULT_PROGRESS = DEFAULT_OUTPUT_DIR / "progress.json" class LinkParser(HTMLParser): def __init__(self) -> None: super().__init__() self.links: list[str] = [] self.title_parts: list[str] = [] self._in_title = False def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None: tag = tag.lower() if tag == "title": self._in_title = True attr_name = "href" if tag == "a" else "src" if tag in {"iframe", "frame"} else None if attr_name is None: return for name, value in attrs: if name.lower() == attr_name and value: self.links.append(value) def handle_endtag(self, tag: str) -> None: if tag.lower() == "title": self._in_title = False def handle_data(self, data: str) -> None: if self._in_title: self.title_parts.append(data.strip()) @property def title(self) -> str: return " ".join(part for part in self.title_parts if part).strip() def charset_from_content_type(content_type: str) -> str: match = re.search(r"charset=([^;\s]+)", content_type, flags=re.IGNORECASE) if match: return match.group(1).strip("\"'") return "utf-8" def load_yaml(path: Path) -> dict[str, Any]: data = yaml.safe_load(path.read_text(encoding="utf-8")) if not isinstance(data, dict): raise ValueError(f"{path} must contain a YAML mapping") return data def read_cookie(cookie_file: Path | None) -> str: if cookie_file: return cookie_file.read_text(encoding="utf-8").strip() return os.environ.get("ONEC_ITS_COOKIE", "").strip() def safe_slug(value: str) -> str: parsed = urllib.parse.urlparse(value) raw = f"{parsed.netloc}{parsed.path}".strip("/") or "index" if parsed.query: raw += "_" + parsed.query slug = re.sub(r"[^A-Za-zА-Яа-яЁё0-9_.-]+", "_", raw, flags=re.UNICODE).strip("_") digest = hashlib.sha256(value.encode("utf-8")).hexdigest()[:12] return f"{slug[:120]}__{digest}.html" def normalize_url(base_url: str, href: str) -> str | None: if href.startswith(("mailto:", "tel:", "javascript:")): return None absolute = urllib.parse.urljoin(base_url, href) parsed = urllib.parse.urlparse(absolute) if parsed.scheme not in {"http", "https"}: return None parsed = parsed._replace(fragment="") return urllib.parse.urlunparse(parsed) def request_safe_url(url: str) -> str: parsed = urllib.parse.urlparse(url) path = urllib.parse.quote(urllib.parse.unquote(parsed.path), safe="/") query = urllib.parse.quote(urllib.parse.unquote(parsed.query), safe="=&?/:;%[]@!$'()*+,") return urllib.parse.urlunparse(parsed._replace(path=path, query=query)) def compile_patterns(patterns: list[str] | None) -> list[re.Pattern[str]]: return [re.compile(pattern) for pattern in patterns or []] def allowed_url(url: str, *, seed_host: str, policy: dict[str, Any]) -> bool: parsed = urllib.parse.urlparse(url) if policy.get("same_host", True) and parsed.netloc.lower() != seed_host.lower(): return False include = compile_patterns(policy.get("include_patterns")) exclude = compile_patterns(policy.get("exclude_patterns")) if include and not any(pattern.search(url) for pattern in include): return False if any(pattern.search(url) for pattern in exclude): return False return True def url_priority(url: str) -> int: parsed = urllib.parse.urlparse(url) path = parsed.path if is_content_src_url(url): return 0 if re.search(r"/db/[^/]+/content/\d+/hdoc(?:/\d+)?$", path): return 1 if re.search(r"/db/[^/]+/content/\d+/\d+$", path): return 1 return 2 def is_content_src_url(url: str) -> bool: path = urllib.parse.urlparse(url).path return "/db/content/" in path and "/src/" in path def enqueue_links( queue: deque[tuple[dict[str, Any], str, int, dict[str, Any], str | None]], *, source: dict[str, Any], base_url: str, depth: int, policy: dict[str, Any], links: list[str], seen: set[str], title_hint: str | None = None, src_only: bool = False, front_priority: bool = False, ) -> int: seed_host = urllib.parse.urlparse(source["url"]).netloc candidates = [] queued_urls = {item[1] for item in queue} for href in links: next_url = normalize_url(base_url, href) if not next_url or next_url in seen or next_url in queued_urls: continue if src_only and not is_content_src_url(next_url): continue if not allowed_url(next_url, seed_host=seed_host, policy=policy): continue candidates.append(next_url) queued_urls.add(next_url) candidates.sort(key=url_priority) # Real article bodies live under /db/content/.../src/... and hdoc pages # usually contain the iframe pointing there, so put those before nav links. priority_items = [url for url in candidates if url_priority(url) <= 1] normal_items = [url for url in candidates if url_priority(url) > 1] if front_priority: for next_url in reversed(priority_items): hint = title_hint if is_content_src_url(next_url) else None queue.appendleft((source, next_url, depth + 1, policy, hint)) else: for next_url in priority_items: hint = title_hint if is_content_src_url(next_url) else None queue.append((source, next_url, depth + 1, policy, hint)) for next_url in normal_items: queue.append((source, next_url, depth + 1, policy, None)) return len(candidates) def merged_policy(default_policy: dict[str, Any], source_policy: dict[str, Any] | None) -> dict[str, Any]: policy = dict(default_policy) for key, value in (source_policy or {}).items(): if key in {"include_patterns", "exclude_patterns"}: policy[key] = [*policy.get(key, []), *value] else: policy[key] = value return policy def fetch_url(url: str, *, cookie: str, timeout: int, user_agent: str) -> tuple[bytes, dict[str, str], int]: headers = {"User-Agent": user_agent} if cookie: headers["Cookie"] = cookie request = urllib.request.Request(request_safe_url(url), headers=headers) with urllib.request.urlopen(request, timeout=timeout) as response: body = response.read() response_headers = {key: value for key, value in response.headers.items()} return body, response_headers, int(response.status) def load_json(path: Path) -> dict[str, Any]: if not path.exists(): return {} try: data = json.loads(path.read_text(encoding="utf-8-sig")) except json.JSONDecodeError: return {} return data if isinstance(data, dict) else {} def resumable_pages(*, output_dir: Path, manifest_path: Path, progress_path: Path) -> dict[str, dict[str, Any]]: pages_by_url: dict[str, dict[str, Any]] = {} for source in (load_json(manifest_path), load_json(progress_path)): for record in source.get("pages") or []: url = record.get("url") filename = record.get("file") sha256 = record.get("sha256") if not url or not filename or not sha256: continue path = output_dir / str(filename) if not path.exists(): continue if hashlib.sha256(path.read_bytes()).hexdigest() != sha256: continue pages_by_url[str(url)] = record return pages_by_url def write_progress( path: Path, *, config: dict[str, Any], output_dir: Path, max_pages: int, records: list[dict[str, Any]], errors: list[dict[str, Any]], seen: set[str], queue_size: int, current_url: str | None, resumed_pages: int, skipped_existing: int, status: str, stop_reason: str | None = None, ) -> None: progress = { "schema": "onec_its_fetch_progress.v1", "status": status, "updated_at_unix": int(time.time()), "access": config.get("access") or "licensed_private", "output_dir": str(output_dir), "max_pages": max_pages, "page_count": len(records), "error_count": len(errors), "seen_count": len(seen), "queue_size": queue_size, "current_url": current_url, "stop_reason": stop_reason, "resumed_pages": resumed_pages, "skipped_existing": skipped_existing, "pages": records, "errors": errors, } path.parent.mkdir(parents=True, exist_ok=True) path.write_text(json.dumps(progress, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") def page_record( *, source: dict[str, Any], url: str, depth: int, body: bytes, headers: dict[str, str], status: int, output_dir: Path, title_hint: str | None = None, ) -> dict[str, Any]: parser = LinkParser() text = body.decode(charset_from_content_type(headers.get("Content-Type", "")), errors="replace") parser.feed(text) filename = safe_slug(url) (output_dir / filename).write_bytes(body) return merge_platform_metadata({ "source_id": source.get("id"), "source_title": source.get("title"), "source_type": source.get("source_type"), "platform_family": source.get("platform_family"), "platform_version": source.get("platform_version"), "access": "licensed_private", "url": url, "title": parser.title or title_hint or source.get("title") or url, "depth": depth, "status": status, "content_type": headers.get("Content-Type", ""), "file": filename, "bytes": len(body), "sha256": hashlib.sha256(body).hexdigest(), "fetched_at_unix": int(time.time()), "links": parser.links, }, url) def fetch_sources( config: dict[str, Any], *, output_dir: Path, cookie: str, max_pages: int, delay_seconds: float, timeout: int, user_agent: str, manifest_path: Path, progress_path: Path, resume: bool, max_attempts: int, ) -> dict[str, Any]: output_dir.mkdir(parents=True, exist_ok=True) default_policy = config.get("default_policy") or {} records: list[dict[str, Any]] = [] errors: list[dict[str, Any]] = [] seen: set[str] = set() skipped_existing = 0 resume_records = resumable_pages(output_dir=output_dir, manifest_path=manifest_path, progress_path=progress_path) if resume else {} records_by_url: dict[str, dict[str, Any]] = {} records_by_source: dict[str, int] = {} errors_by_source: dict[str, int] = {} queue: deque[tuple[dict[str, Any], str, int, dict[str, Any], str | None]] = deque() seed_urls = set() priority_repacked = False for source in config.get("sources") or []: policy = merged_policy(default_policy, source.get("policy")) seed_urls.add(str(source["url"])) queue.append((source, source["url"], 0, policy, None)) while queue and len(records) < max_pages and (max_attempts <= 0 or len(seen) < max_attempts): if not priority_repacked and seed_urls and seed_urls.issubset(seen): queue = deque(sorted(queue, key=lambda item: (url_priority(item[1]), item[2], item[1]))) priority_repacked = True source, url, depth, policy, title_hint = queue.popleft() source_id = str(source.get("id") or "") max_pages_per_source = int(policy.get("max_pages_per_source") or 0) max_errors_per_source = int(policy.get("max_errors_per_source") or 0) if max_pages_per_source > 0 and records_by_source.get(source_id, 0) >= max_pages_per_source: continue if max_errors_per_source > 0 and errors_by_source.get(source_id, 0) >= max_errors_per_source: continue if url in seen: continue seen.add(url) if resume and url in resume_records: record = resume_records[url] records_by_url[url] = record records = list(records_by_url.values()) records_by_source[source_id] = records_by_source.get(source_id, 0) + 1 skipped_existing += 1 write_progress( progress_path, config=config, output_dir=output_dir, max_pages=max_pages, records=records, errors=errors, seen=seen, queue_size=len(queue), current_url=url, resumed_pages=len(resume_records), skipped_existing=skipped_existing, status="running", ) max_depth = int(policy.get("max_depth", 0)) try: body = (output_dir / str(record["file"])).read_bytes() text = body.decode(charset_from_content_type(str(record.get("content_type") or "")), errors="replace") parser = LinkParser() parser.feed(text) if depth < max_depth or any(is_content_src_url(normalize_url(url, href) or "") for href in parser.links): enqueue_links( queue, source=source, base_url=url, depth=depth, policy=policy, links=parser.links, seen=seen, title_hint=str(record.get("title") or title_hint or ""), src_only=depth >= max_depth, front_priority=seed_urls.issubset(seen), ) except OSError as exc: errors.append({"source_id": source.get("id"), "url": url, "depth": depth, "error": f"resume link parse failed: {exc}"}) continue try: write_progress( progress_path, config=config, output_dir=output_dir, max_pages=max_pages, records=records, errors=errors, seen=seen, queue_size=len(queue), current_url=url, resumed_pages=len(resume_records), skipped_existing=skipped_existing, status="running", ) body, headers, status = fetch_url(url, cookie=cookie, timeout=timeout, user_agent=user_agent) record = page_record(source=source, url=url, depth=depth, body=body, headers=headers, status=status, output_dir=output_dir, title_hint=title_hint) records_by_url[url] = {key: value for key, value in record.items() if key != "links"} records = list(records_by_url.values()) records_by_source[source_id] = records_by_source.get(source_id, 0) + 1 max_depth = int(policy.get("max_depth", 0)) if depth < max_depth or any(is_content_src_url(normalize_url(url, href) or "") for href in record["links"]): enqueue_links( queue, source=source, base_url=url, depth=depth, policy=policy, links=record["links"], seen=seen, title_hint=str(record.get("title") or title_hint or ""), src_only=depth >= max_depth, front_priority=seed_urls.issubset(seen), ) if delay_seconds > 0: time.sleep(delay_seconds) except (urllib.error.URLError, TimeoutError, OSError, UnicodeDecodeError) as exc: errors.append({"source_id": source.get("id"), "url": url, "depth": depth, "error": str(exc)}) errors_by_source[source_id] = errors_by_source.get(source_id, 0) + 1 write_progress( progress_path, config=config, output_dir=output_dir, max_pages=max_pages, records=records, errors=errors, seen=seen, queue_size=len(queue), current_url=url, resumed_pages=len(resume_records), skipped_existing=skipped_existing, status="running", ) if len(records) >= max_pages: final_status = "complete" stop_reason = "max_pages_reached" elif max_attempts > 0 and len(seen) >= max_attempts: final_status = "stopped_by_limit" stop_reason = "max_attempts_reached" elif not queue: final_status = "complete" stop_reason = "queue_empty" else: final_status = "empty" stop_reason = "no_pages" result = { "schema": "onec_its_raw_fetch_manifest.v1", "access": config.get("access") or "licensed_private", "output_dir": str(output_dir), "page_count": len(records), "error_count": len(errors), "resume": { "enabled": resume, "resumable_pages": len(resume_records), "skipped_existing": skipped_existing, }, "stop": { "status": final_status, "reason": stop_reason, "seen_count": len(seen), "queue_size": len(queue), "max_attempts": max_attempts, "max_pages": max_pages, }, "records_by_source": dict(sorted(records_by_source.items())), "errors_by_source": dict(sorted(errors_by_source.items())), "pages": records, "errors": errors, } write_progress( progress_path, config=config, output_dir=output_dir, max_pages=max_pages, records=records, errors=errors, seen=seen, queue_size=len(queue), current_url=None, resumed_pages=len(resume_records), skipped_existing=skipped_existing, status=final_status, stop_reason=stop_reason, ) return result def main() -> int: parser = argparse.ArgumentParser(description="Fetch official 1C:ITS documentation pages into a private raw cache.") parser.add_argument("--sources", type=Path, default=DEFAULT_SOURCES) parser.add_argument("--output-dir", type=Path, default=DEFAULT_OUTPUT_DIR) parser.add_argument("--manifest", type=Path, default=DEFAULT_MANIFEST) parser.add_argument("--progress", type=Path, default=DEFAULT_PROGRESS) parser.add_argument("--cookie-file", type=Path) parser.add_argument("--max-pages", type=int, default=50) parser.add_argument("--max-attempts", type=int, default=0, help="Maximum URL attempts, including failed pages. 0 means max(max_pages * 5, max_pages).") parser.add_argument("--no-resume", action="store_true", help="Do not reuse already downloaded pages from manifest/progress.") parser.add_argument("--delay-seconds", type=float, default=0.4) parser.add_argument("--timeout", type=int, default=30) parser.add_argument("--user-agent", default="Codex-1C-RAG/1.0 (+licensed private 1C:ITS access)") args = parser.parse_args() config = load_yaml(args.sources) cookie = read_cookie(args.cookie_file) max_attempts = args.max_attempts if args.max_attempts > 0 else max(args.max_pages * 5, args.max_pages) result = fetch_sources( config, output_dir=args.output_dir, cookie=cookie, max_pages=args.max_pages, delay_seconds=args.delay_seconds, timeout=args.timeout, user_agent=args.user_agent, manifest_path=args.manifest, progress_path=args.progress, resume=not args.no_resume, max_attempts=max_attempts, ) args.manifest.parent.mkdir(parents=True, exist_ok=True) args.manifest.write_text(json.dumps(result, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") print(json.dumps({"output": str(args.manifest), "progress": str(args.progress), "pages": result["page_count"], "errors": result["error_count"], "resume": result["resume"]}, ensure_ascii=False)) return 0 if result["page_count"] else 2 if __name__ == "__main__": raise SystemExit(main())