Initial SQL-only 1C adapter baseline
This commit is contained in:
@@ -0,0 +1,183 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
from html.parser import HTMLParser
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from fetch_1c_its_docs import charset_from_content_type, request_safe_url
|
||||
from one_c_its_platform import materialize_doc_url
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
DEFAULT_OUTPUT = ROOT / "reports" / "1c-its-access-check.json"
|
||||
DEFAULT_TEST_URL = "https://its.1c.ru/db/v8316doc#bookmark:dev:TI000000044"
|
||||
|
||||
|
||||
class AccessHtmlParser(HTMLParser):
|
||||
def __init__(self) -> None:
|
||||
super().__init__(convert_charrefs=True)
|
||||
self.title_parts: list[str] = []
|
||||
self._in_title = False
|
||||
self.login_links = 0
|
||||
self.user_profile_markers = 0
|
||||
self.paywall_markers = 0
|
||||
self.data_access_false = 0
|
||||
self.iframe_srcs: list[str] = []
|
||||
|
||||
def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
|
||||
tag = tag.lower()
|
||||
values = {name.lower(): value or "" for name, value in attrs}
|
||||
if tag == "title":
|
||||
self._in_title = True
|
||||
href = values.get("href", "")
|
||||
class_name = values.get("class", "")
|
||||
if "/user/auth" in href:
|
||||
self.login_links += 1
|
||||
if "paywall" in class_name:
|
||||
self.paywall_markers += 1
|
||||
if values.get("data-access") == "false":
|
||||
self.data_access_false += 1
|
||||
if tag == "iframe" and values.get("src"):
|
||||
self.iframe_srcs.append(values["src"])
|
||||
|
||||
def handle_endtag(self, tag: str) -> None:
|
||||
if tag.lower() == "title":
|
||||
self._in_title = False
|
||||
|
||||
def handle_data(self, data: str) -> None:
|
||||
if self._in_title:
|
||||
self.title_parts.append(data.strip())
|
||||
if "Общий профиль" in data or "Доступ до" in data:
|
||||
self.user_profile_markers += 1
|
||||
|
||||
@property
|
||||
def title(self) -> str:
|
||||
return " ".join(part for part in self.title_parts if part)
|
||||
|
||||
|
||||
def read_cookie(cookie_file: Path | None) -> str:
|
||||
if cookie_file:
|
||||
return cookie_file.read_text(encoding="utf-8").strip()
|
||||
return os.environ.get("ONEC_ITS_COOKIE", "").strip()
|
||||
|
||||
|
||||
def fetch_text(url: str, *, cookie: str, timeout: int, referer: str | None = None) -> tuple[int, str, str]:
|
||||
headers = {"User-Agent": "Codex 1C ITS access check"}
|
||||
if cookie:
|
||||
headers["Cookie"] = cookie
|
||||
if referer:
|
||||
headers["Referer"] = referer
|
||||
headers["X-Referer"] = referer
|
||||
request = urllib.request.Request(request_safe_url(url), headers=headers)
|
||||
with urllib.request.urlopen(request, timeout=timeout) as response:
|
||||
body = response.read()
|
||||
content_type = response.headers.get("Content-Type", "")
|
||||
charset = charset_from_content_type(content_type)
|
||||
return int(response.status), content_type, body.decode(charset, errors="replace")
|
||||
|
||||
|
||||
def first_src_url(page_url: str, iframe_srcs: list[str]) -> str | None:
|
||||
for src in iframe_srcs:
|
||||
if "/db/content/" in src and "/src/" in src:
|
||||
return urllib.parse.urljoin(page_url, src)
|
||||
return None
|
||||
|
||||
|
||||
def check_access(url: str, *, cookie: str, timeout: int) -> dict[str, Any]:
|
||||
materialized = materialize_doc_url(url)
|
||||
result: dict[str, Any] = {
|
||||
"schema": "onec_its_access_check.v1",
|
||||
"checked_at_unix": int(time.time()),
|
||||
"target_url": url,
|
||||
"materialized_url": materialized,
|
||||
"cookie_present": bool(cookie.strip()),
|
||||
"page": {},
|
||||
"src": {},
|
||||
"status": "unknown",
|
||||
"findings": [],
|
||||
}
|
||||
if not cookie.strip():
|
||||
result["status"] = "failed"
|
||||
result["findings"].append("cookie_missing")
|
||||
return result
|
||||
|
||||
try:
|
||||
status, content_type, text = fetch_text(materialized, cookie=cookie, timeout=timeout)
|
||||
parser = AccessHtmlParser()
|
||||
parser.feed(text)
|
||||
src_url = first_src_url(materialized, parser.iframe_srcs)
|
||||
result["page"] = {
|
||||
"http_status": status,
|
||||
"content_type": content_type,
|
||||
"title": parser.title,
|
||||
"login_links": parser.login_links,
|
||||
"user_profile_markers": parser.user_profile_markers,
|
||||
"paywall_markers": parser.paywall_markers,
|
||||
"data_access_false": parser.data_access_false,
|
||||
"iframe_src": src_url,
|
||||
}
|
||||
if parser.login_links:
|
||||
result["findings"].append("login_links_present")
|
||||
if parser.paywall_markers:
|
||||
result["findings"].append("paywall_marker_present")
|
||||
if parser.data_access_false:
|
||||
result["findings"].append("data_access_false")
|
||||
except Exception as exc: # noqa: BLE001
|
||||
result["page"] = {"error": f"{type(exc).__name__}: {exc}"}
|
||||
result["status"] = "failed"
|
||||
result["findings"].append("page_fetch_failed")
|
||||
return result
|
||||
|
||||
src_url = result["page"].get("iframe_src")
|
||||
if src_url:
|
||||
try:
|
||||
src_status, src_content_type, src_text = fetch_text(str(src_url), cookie=cookie, timeout=timeout, referer=materialized)
|
||||
visible_words = len(re.findall(r"[A-Za-zА-Яа-яЁё0-9_]+", src_text))
|
||||
result["src"] = {
|
||||
"http_status": src_status,
|
||||
"content_type": src_content_type,
|
||||
"chars": len(src_text),
|
||||
"word_count": visible_words,
|
||||
}
|
||||
if visible_words < 50:
|
||||
result["findings"].append("src_low_text")
|
||||
except urllib.error.HTTPError as exc:
|
||||
result["src"] = {"http_status": exc.code, "error": str(exc), "url": src_url}
|
||||
result["findings"].append(f"src_http_{exc.code}")
|
||||
except Exception as exc: # noqa: BLE001
|
||||
result["src"] = {"error": f"{type(exc).__name__}: {exc}", "url": src_url}
|
||||
result["findings"].append("src_fetch_failed")
|
||||
else:
|
||||
result["findings"].append("src_iframe_missing")
|
||||
|
||||
blocking = {"login_links_present", "paywall_marker_present", "data_access_false", "src_http_401", "src_fetch_failed"}
|
||||
result["status"] = "failed" if any(item in blocking for item in result["findings"]) else "ok"
|
||||
return result
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Check whether the stored 1C:ITS cookie can access protected documentation bodies.")
|
||||
parser.add_argument("--url", default=DEFAULT_TEST_URL)
|
||||
parser.add_argument("--cookie-file", type=Path)
|
||||
parser.add_argument("--output", type=Path, default=DEFAULT_OUTPUT)
|
||||
parser.add_argument("--timeout", type=int, default=30)
|
||||
parser.add_argument("--print", action="store_true", dest="print_report")
|
||||
args = parser.parse_args()
|
||||
|
||||
report = check_access(args.url, cookie=read_cookie(args.cookie_file), timeout=args.timeout)
|
||||
args.output.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.output.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
print(json.dumps(report if args.print_report else {"status": report["status"], "findings": report["findings"], "output": str(args.output)}, ensure_ascii=False, indent=2))
|
||||
return 0 if report["status"] == "ok" else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Reference in New Issue
Block a user