548 lines
20 KiB
Python
548 lines
20 KiB
Python
from __future__ import annotations
|
||
|
||
import argparse
|
||
import hashlib
|
||
import json
|
||
import os
|
||
import re
|
||
import time
|
||
import urllib.error
|
||
import urllib.parse
|
||
import urllib.request
|
||
from collections import deque
|
||
from html.parser import HTMLParser
|
||
from pathlib import Path
|
||
from typing import Any
|
||
|
||
import yaml
|
||
|
||
from one_c_its_platform import merge_platform_metadata
|
||
|
||
|
||
ROOT = Path(__file__).resolve().parents[1]
|
||
DEFAULT_SOURCES = ROOT / "plugins" / "1c" / "rag" / "official-docs" / "sources.yaml"
|
||
DEFAULT_OUTPUT_DIR = ROOT / "plugins" / "1c" / "rag" / "official-docs" / "raw"
|
||
DEFAULT_MANIFEST = DEFAULT_OUTPUT_DIR / "manifest.json"
|
||
DEFAULT_PROGRESS = DEFAULT_OUTPUT_DIR / "progress.json"
|
||
|
||
|
||
class LinkParser(HTMLParser):
|
||
def __init__(self) -> None:
|
||
super().__init__()
|
||
self.links: list[str] = []
|
||
self.title_parts: list[str] = []
|
||
self._in_title = False
|
||
|
||
def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
|
||
tag = tag.lower()
|
||
if tag == "title":
|
||
self._in_title = True
|
||
attr_name = "href" if tag == "a" else "src" if tag in {"iframe", "frame"} else None
|
||
if attr_name is None:
|
||
return
|
||
for name, value in attrs:
|
||
if name.lower() == attr_name and value:
|
||
self.links.append(value)
|
||
|
||
def handle_endtag(self, tag: str) -> None:
|
||
if tag.lower() == "title":
|
||
self._in_title = False
|
||
|
||
def handle_data(self, data: str) -> None:
|
||
if self._in_title:
|
||
self.title_parts.append(data.strip())
|
||
|
||
@property
|
||
def title(self) -> str:
|
||
return " ".join(part for part in self.title_parts if part).strip()
|
||
|
||
|
||
def charset_from_content_type(content_type: str) -> str:
|
||
match = re.search(r"charset=([^;\s]+)", content_type, flags=re.IGNORECASE)
|
||
if match:
|
||
return match.group(1).strip("\"'")
|
||
return "utf-8"
|
||
|
||
|
||
def load_yaml(path: Path) -> dict[str, Any]:
|
||
data = yaml.safe_load(path.read_text(encoding="utf-8"))
|
||
if not isinstance(data, dict):
|
||
raise ValueError(f"{path} must contain a YAML mapping")
|
||
return data
|
||
|
||
|
||
def read_cookie(cookie_file: Path | None) -> str:
|
||
if cookie_file:
|
||
return cookie_file.read_text(encoding="utf-8").strip()
|
||
return os.environ.get("ONEC_ITS_COOKIE", "").strip()
|
||
|
||
|
||
def safe_slug(value: str) -> str:
|
||
parsed = urllib.parse.urlparse(value)
|
||
raw = f"{parsed.netloc}{parsed.path}".strip("/") or "index"
|
||
if parsed.query:
|
||
raw += "_" + parsed.query
|
||
slug = re.sub(r"[^A-Za-zА-Яа-яЁё0-9_.-]+", "_", raw, flags=re.UNICODE).strip("_")
|
||
digest = hashlib.sha256(value.encode("utf-8")).hexdigest()[:12]
|
||
return f"{slug[:120]}__{digest}.html"
|
||
|
||
|
||
def normalize_url(base_url: str, href: str) -> str | None:
|
||
if href.startswith(("mailto:", "tel:", "javascript:")):
|
||
return None
|
||
absolute = urllib.parse.urljoin(base_url, href)
|
||
parsed = urllib.parse.urlparse(absolute)
|
||
if parsed.scheme not in {"http", "https"}:
|
||
return None
|
||
parsed = parsed._replace(fragment="")
|
||
return urllib.parse.urlunparse(parsed)
|
||
|
||
|
||
def request_safe_url(url: str) -> str:
|
||
parsed = urllib.parse.urlparse(url)
|
||
path = urllib.parse.quote(urllib.parse.unquote(parsed.path), safe="/")
|
||
query = urllib.parse.quote(urllib.parse.unquote(parsed.query), safe="=&?/:;%[]@!$'()*+,")
|
||
return urllib.parse.urlunparse(parsed._replace(path=path, query=query))
|
||
|
||
|
||
def compile_patterns(patterns: list[str] | None) -> list[re.Pattern[str]]:
|
||
return [re.compile(pattern) for pattern in patterns or []]
|
||
|
||
|
||
def allowed_url(url: str, *, seed_host: str, policy: dict[str, Any]) -> bool:
|
||
parsed = urllib.parse.urlparse(url)
|
||
if policy.get("same_host", True) and parsed.netloc.lower() != seed_host.lower():
|
||
return False
|
||
include = compile_patterns(policy.get("include_patterns"))
|
||
exclude = compile_patterns(policy.get("exclude_patterns"))
|
||
if include and not any(pattern.search(url) for pattern in include):
|
||
return False
|
||
if any(pattern.search(url) for pattern in exclude):
|
||
return False
|
||
return True
|
||
|
||
|
||
def url_priority(url: str) -> int:
|
||
parsed = urllib.parse.urlparse(url)
|
||
path = parsed.path
|
||
if is_content_src_url(url):
|
||
return 0
|
||
if re.search(r"/db/[^/]+/content/\d+/hdoc(?:/\d+)?$", path):
|
||
return 1
|
||
if re.search(r"/db/[^/]+/content/\d+/\d+$", path):
|
||
return 1
|
||
return 2
|
||
|
||
|
||
def is_content_src_url(url: str) -> bool:
|
||
path = urllib.parse.urlparse(url).path
|
||
return "/db/content/" in path and "/src/" in path
|
||
|
||
|
||
def enqueue_links(
|
||
queue: deque[tuple[dict[str, Any], str, int, dict[str, Any], str | None]],
|
||
*,
|
||
source: dict[str, Any],
|
||
base_url: str,
|
||
depth: int,
|
||
policy: dict[str, Any],
|
||
links: list[str],
|
||
seen: set[str],
|
||
title_hint: str | None = None,
|
||
src_only: bool = False,
|
||
front_priority: bool = False,
|
||
) -> int:
|
||
seed_host = urllib.parse.urlparse(source["url"]).netloc
|
||
candidates = []
|
||
queued_urls = {item[1] for item in queue}
|
||
for href in links:
|
||
next_url = normalize_url(base_url, href)
|
||
if not next_url or next_url in seen or next_url in queued_urls:
|
||
continue
|
||
if src_only and not is_content_src_url(next_url):
|
||
continue
|
||
if not allowed_url(next_url, seed_host=seed_host, policy=policy):
|
||
continue
|
||
candidates.append(next_url)
|
||
queued_urls.add(next_url)
|
||
candidates.sort(key=url_priority)
|
||
|
||
# Real article bodies live under /db/content/.../src/... and hdoc pages
|
||
# usually contain the iframe pointing there, so put those before nav links.
|
||
priority_items = [url for url in candidates if url_priority(url) <= 1]
|
||
normal_items = [url for url in candidates if url_priority(url) > 1]
|
||
if front_priority:
|
||
for next_url in reversed(priority_items):
|
||
hint = title_hint if is_content_src_url(next_url) else None
|
||
queue.appendleft((source, next_url, depth + 1, policy, hint))
|
||
else:
|
||
for next_url in priority_items:
|
||
hint = title_hint if is_content_src_url(next_url) else None
|
||
queue.append((source, next_url, depth + 1, policy, hint))
|
||
for next_url in normal_items:
|
||
queue.append((source, next_url, depth + 1, policy, None))
|
||
return len(candidates)
|
||
|
||
|
||
def merged_policy(default_policy: dict[str, Any], source_policy: dict[str, Any] | None) -> dict[str, Any]:
|
||
policy = dict(default_policy)
|
||
for key, value in (source_policy or {}).items():
|
||
if key in {"include_patterns", "exclude_patterns"}:
|
||
policy[key] = [*policy.get(key, []), *value]
|
||
else:
|
||
policy[key] = value
|
||
return policy
|
||
|
||
|
||
def fetch_url(url: str, *, cookie: str, timeout: int, user_agent: str) -> tuple[bytes, dict[str, str], int]:
|
||
headers = {"User-Agent": user_agent}
|
||
if cookie:
|
||
headers["Cookie"] = cookie
|
||
request = urllib.request.Request(request_safe_url(url), headers=headers)
|
||
with urllib.request.urlopen(request, timeout=timeout) as response:
|
||
body = response.read()
|
||
response_headers = {key: value for key, value in response.headers.items()}
|
||
return body, response_headers, int(response.status)
|
||
|
||
|
||
def load_json(path: Path) -> dict[str, Any]:
|
||
if not path.exists():
|
||
return {}
|
||
try:
|
||
data = json.loads(path.read_text(encoding="utf-8-sig"))
|
||
except json.JSONDecodeError:
|
||
return {}
|
||
return data if isinstance(data, dict) else {}
|
||
|
||
|
||
def resumable_pages(*, output_dir: Path, manifest_path: Path, progress_path: Path) -> dict[str, dict[str, Any]]:
|
||
pages_by_url: dict[str, dict[str, Any]] = {}
|
||
for source in (load_json(manifest_path), load_json(progress_path)):
|
||
for record in source.get("pages") or []:
|
||
url = record.get("url")
|
||
filename = record.get("file")
|
||
sha256 = record.get("sha256")
|
||
if not url or not filename or not sha256:
|
||
continue
|
||
path = output_dir / str(filename)
|
||
if not path.exists():
|
||
continue
|
||
if hashlib.sha256(path.read_bytes()).hexdigest() != sha256:
|
||
continue
|
||
pages_by_url[str(url)] = record
|
||
return pages_by_url
|
||
|
||
|
||
def write_progress(
|
||
path: Path,
|
||
*,
|
||
config: dict[str, Any],
|
||
output_dir: Path,
|
||
max_pages: int,
|
||
records: list[dict[str, Any]],
|
||
errors: list[dict[str, Any]],
|
||
seen: set[str],
|
||
queue_size: int,
|
||
current_url: str | None,
|
||
resumed_pages: int,
|
||
skipped_existing: int,
|
||
status: str,
|
||
stop_reason: str | None = None,
|
||
) -> None:
|
||
progress = {
|
||
"schema": "onec_its_fetch_progress.v1",
|
||
"status": status,
|
||
"updated_at_unix": int(time.time()),
|
||
"access": config.get("access") or "licensed_private",
|
||
"output_dir": str(output_dir),
|
||
"max_pages": max_pages,
|
||
"page_count": len(records),
|
||
"error_count": len(errors),
|
||
"seen_count": len(seen),
|
||
"queue_size": queue_size,
|
||
"current_url": current_url,
|
||
"stop_reason": stop_reason,
|
||
"resumed_pages": resumed_pages,
|
||
"skipped_existing": skipped_existing,
|
||
"pages": records,
|
||
"errors": errors,
|
||
}
|
||
path.parent.mkdir(parents=True, exist_ok=True)
|
||
path.write_text(json.dumps(progress, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||
|
||
|
||
def page_record(
|
||
*,
|
||
source: dict[str, Any],
|
||
url: str,
|
||
depth: int,
|
||
body: bytes,
|
||
headers: dict[str, str],
|
||
status: int,
|
||
output_dir: Path,
|
||
title_hint: str | None = None,
|
||
) -> dict[str, Any]:
|
||
parser = LinkParser()
|
||
text = body.decode(charset_from_content_type(headers.get("Content-Type", "")), errors="replace")
|
||
parser.feed(text)
|
||
filename = safe_slug(url)
|
||
(output_dir / filename).write_bytes(body)
|
||
return merge_platform_metadata({
|
||
"source_id": source.get("id"),
|
||
"source_title": source.get("title"),
|
||
"source_type": source.get("source_type"),
|
||
"platform_family": source.get("platform_family"),
|
||
"platform_version": source.get("platform_version"),
|
||
"access": "licensed_private",
|
||
"url": url,
|
||
"title": parser.title or title_hint or source.get("title") or url,
|
||
"depth": depth,
|
||
"status": status,
|
||
"content_type": headers.get("Content-Type", ""),
|
||
"file": filename,
|
||
"bytes": len(body),
|
||
"sha256": hashlib.sha256(body).hexdigest(),
|
||
"fetched_at_unix": int(time.time()),
|
||
"links": parser.links,
|
||
}, url)
|
||
|
||
|
||
def fetch_sources(
|
||
config: dict[str, Any],
|
||
*,
|
||
output_dir: Path,
|
||
cookie: str,
|
||
max_pages: int,
|
||
delay_seconds: float,
|
||
timeout: int,
|
||
user_agent: str,
|
||
manifest_path: Path,
|
||
progress_path: Path,
|
||
resume: bool,
|
||
max_attempts: int,
|
||
) -> dict[str, Any]:
|
||
output_dir.mkdir(parents=True, exist_ok=True)
|
||
default_policy = config.get("default_policy") or {}
|
||
records: list[dict[str, Any]] = []
|
||
errors: list[dict[str, Any]] = []
|
||
seen: set[str] = set()
|
||
skipped_existing = 0
|
||
resume_records = resumable_pages(output_dir=output_dir, manifest_path=manifest_path, progress_path=progress_path) if resume else {}
|
||
records_by_url: dict[str, dict[str, Any]] = {}
|
||
records_by_source: dict[str, int] = {}
|
||
errors_by_source: dict[str, int] = {}
|
||
|
||
queue: deque[tuple[dict[str, Any], str, int, dict[str, Any], str | None]] = deque()
|
||
seed_urls = set()
|
||
priority_repacked = False
|
||
for source in config.get("sources") or []:
|
||
policy = merged_policy(default_policy, source.get("policy"))
|
||
seed_urls.add(str(source["url"]))
|
||
queue.append((source, source["url"], 0, policy, None))
|
||
|
||
while queue and len(records) < max_pages and (max_attempts <= 0 or len(seen) < max_attempts):
|
||
if not priority_repacked and seed_urls and seed_urls.issubset(seen):
|
||
queue = deque(sorted(queue, key=lambda item: (url_priority(item[1]), item[2], item[1])))
|
||
priority_repacked = True
|
||
source, url, depth, policy, title_hint = queue.popleft()
|
||
source_id = str(source.get("id") or "")
|
||
max_pages_per_source = int(policy.get("max_pages_per_source") or 0)
|
||
max_errors_per_source = int(policy.get("max_errors_per_source") or 0)
|
||
if max_pages_per_source > 0 and records_by_source.get(source_id, 0) >= max_pages_per_source:
|
||
continue
|
||
if max_errors_per_source > 0 and errors_by_source.get(source_id, 0) >= max_errors_per_source:
|
||
continue
|
||
if url in seen:
|
||
continue
|
||
seen.add(url)
|
||
if resume and url in resume_records:
|
||
record = resume_records[url]
|
||
records_by_url[url] = record
|
||
records = list(records_by_url.values())
|
||
records_by_source[source_id] = records_by_source.get(source_id, 0) + 1
|
||
skipped_existing += 1
|
||
write_progress(
|
||
progress_path,
|
||
config=config,
|
||
output_dir=output_dir,
|
||
max_pages=max_pages,
|
||
records=records,
|
||
errors=errors,
|
||
seen=seen,
|
||
queue_size=len(queue),
|
||
current_url=url,
|
||
resumed_pages=len(resume_records),
|
||
skipped_existing=skipped_existing,
|
||
status="running",
|
||
)
|
||
max_depth = int(policy.get("max_depth", 0))
|
||
try:
|
||
body = (output_dir / str(record["file"])).read_bytes()
|
||
text = body.decode(charset_from_content_type(str(record.get("content_type") or "")), errors="replace")
|
||
parser = LinkParser()
|
||
parser.feed(text)
|
||
if depth < max_depth or any(is_content_src_url(normalize_url(url, href) or "") for href in parser.links):
|
||
enqueue_links(
|
||
queue,
|
||
source=source,
|
||
base_url=url,
|
||
depth=depth,
|
||
policy=policy,
|
||
links=parser.links,
|
||
seen=seen,
|
||
title_hint=str(record.get("title") or title_hint or ""),
|
||
src_only=depth >= max_depth,
|
||
front_priority=seed_urls.issubset(seen),
|
||
)
|
||
except OSError as exc:
|
||
errors.append({"source_id": source.get("id"), "url": url, "depth": depth, "error": f"resume link parse failed: {exc}"})
|
||
continue
|
||
try:
|
||
write_progress(
|
||
progress_path,
|
||
config=config,
|
||
output_dir=output_dir,
|
||
max_pages=max_pages,
|
||
records=records,
|
||
errors=errors,
|
||
seen=seen,
|
||
queue_size=len(queue),
|
||
current_url=url,
|
||
resumed_pages=len(resume_records),
|
||
skipped_existing=skipped_existing,
|
||
status="running",
|
||
)
|
||
body, headers, status = fetch_url(url, cookie=cookie, timeout=timeout, user_agent=user_agent)
|
||
record = page_record(source=source, url=url, depth=depth, body=body, headers=headers, status=status, output_dir=output_dir, title_hint=title_hint)
|
||
records_by_url[url] = {key: value for key, value in record.items() if key != "links"}
|
||
records = list(records_by_url.values())
|
||
records_by_source[source_id] = records_by_source.get(source_id, 0) + 1
|
||
max_depth = int(policy.get("max_depth", 0))
|
||
if depth < max_depth or any(is_content_src_url(normalize_url(url, href) or "") for href in record["links"]):
|
||
enqueue_links(
|
||
queue,
|
||
source=source,
|
||
base_url=url,
|
||
depth=depth,
|
||
policy=policy,
|
||
links=record["links"],
|
||
seen=seen,
|
||
title_hint=str(record.get("title") or title_hint or ""),
|
||
src_only=depth >= max_depth,
|
||
front_priority=seed_urls.issubset(seen),
|
||
)
|
||
if delay_seconds > 0:
|
||
time.sleep(delay_seconds)
|
||
except (urllib.error.URLError, TimeoutError, OSError, UnicodeDecodeError) as exc:
|
||
errors.append({"source_id": source.get("id"), "url": url, "depth": depth, "error": str(exc)})
|
||
errors_by_source[source_id] = errors_by_source.get(source_id, 0) + 1
|
||
write_progress(
|
||
progress_path,
|
||
config=config,
|
||
output_dir=output_dir,
|
||
max_pages=max_pages,
|
||
records=records,
|
||
errors=errors,
|
||
seen=seen,
|
||
queue_size=len(queue),
|
||
current_url=url,
|
||
resumed_pages=len(resume_records),
|
||
skipped_existing=skipped_existing,
|
||
status="running",
|
||
)
|
||
|
||
if len(records) >= max_pages:
|
||
final_status = "complete"
|
||
stop_reason = "max_pages_reached"
|
||
elif max_attempts > 0 and len(seen) >= max_attempts:
|
||
final_status = "stopped_by_limit"
|
||
stop_reason = "max_attempts_reached"
|
||
elif not queue:
|
||
final_status = "complete"
|
||
stop_reason = "queue_empty"
|
||
else:
|
||
final_status = "empty"
|
||
stop_reason = "no_pages"
|
||
|
||
result = {
|
||
"schema": "onec_its_raw_fetch_manifest.v1",
|
||
"access": config.get("access") or "licensed_private",
|
||
"output_dir": str(output_dir),
|
||
"page_count": len(records),
|
||
"error_count": len(errors),
|
||
"resume": {
|
||
"enabled": resume,
|
||
"resumable_pages": len(resume_records),
|
||
"skipped_existing": skipped_existing,
|
||
},
|
||
"stop": {
|
||
"status": final_status,
|
||
"reason": stop_reason,
|
||
"seen_count": len(seen),
|
||
"queue_size": len(queue),
|
||
"max_attempts": max_attempts,
|
||
"max_pages": max_pages,
|
||
},
|
||
"records_by_source": dict(sorted(records_by_source.items())),
|
||
"errors_by_source": dict(sorted(errors_by_source.items())),
|
||
"pages": records,
|
||
"errors": errors,
|
||
}
|
||
write_progress(
|
||
progress_path,
|
||
config=config,
|
||
output_dir=output_dir,
|
||
max_pages=max_pages,
|
||
records=records,
|
||
errors=errors,
|
||
seen=seen,
|
||
queue_size=len(queue),
|
||
current_url=None,
|
||
resumed_pages=len(resume_records),
|
||
skipped_existing=skipped_existing,
|
||
status=final_status,
|
||
stop_reason=stop_reason,
|
||
)
|
||
return result
|
||
|
||
|
||
def main() -> int:
|
||
parser = argparse.ArgumentParser(description="Fetch official 1C:ITS documentation pages into a private raw cache.")
|
||
parser.add_argument("--sources", type=Path, default=DEFAULT_SOURCES)
|
||
parser.add_argument("--output-dir", type=Path, default=DEFAULT_OUTPUT_DIR)
|
||
parser.add_argument("--manifest", type=Path, default=DEFAULT_MANIFEST)
|
||
parser.add_argument("--progress", type=Path, default=DEFAULT_PROGRESS)
|
||
parser.add_argument("--cookie-file", type=Path)
|
||
parser.add_argument("--max-pages", type=int, default=50)
|
||
parser.add_argument("--max-attempts", type=int, default=0, help="Maximum URL attempts, including failed pages. 0 means max(max_pages * 5, max_pages).")
|
||
parser.add_argument("--no-resume", action="store_true", help="Do not reuse already downloaded pages from manifest/progress.")
|
||
parser.add_argument("--delay-seconds", type=float, default=0.4)
|
||
parser.add_argument("--timeout", type=int, default=30)
|
||
parser.add_argument("--user-agent", default="Codex-1C-RAG/1.0 (+licensed private 1C:ITS access)")
|
||
args = parser.parse_args()
|
||
|
||
config = load_yaml(args.sources)
|
||
cookie = read_cookie(args.cookie_file)
|
||
max_attempts = args.max_attempts if args.max_attempts > 0 else max(args.max_pages * 5, args.max_pages)
|
||
result = fetch_sources(
|
||
config,
|
||
output_dir=args.output_dir,
|
||
cookie=cookie,
|
||
max_pages=args.max_pages,
|
||
delay_seconds=args.delay_seconds,
|
||
timeout=args.timeout,
|
||
user_agent=args.user_agent,
|
||
manifest_path=args.manifest,
|
||
progress_path=args.progress,
|
||
resume=not args.no_resume,
|
||
max_attempts=max_attempts,
|
||
)
|
||
args.manifest.parent.mkdir(parents=True, exist_ok=True)
|
||
args.manifest.write_text(json.dumps(result, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||
print(json.dumps({"output": str(args.manifest), "progress": str(args.progress), "pages": result["page_count"], "errors": result["error_count"], "resume": result["resume"]}, ensure_ascii=False))
|
||
return 0 if result["page_count"] else 2
|
||
|
||
|
||
if __name__ == "__main__":
|
||
raise SystemExit(main())
|