Initial SQL-only 1C adapter baseline

2026-07-22 03:03:47 +03:00
commit e2503b77e7
545 changed files with 184711 additions and 0 deletions
@@ -0,0 +1,195 @@
+from __future__ import annotations
+
+import argparse
+import os
+import sys
+import time
+import urllib.error
+import urllib.request
+from pathlib import Path
+
+from huggingface_hub import HfApi, hf_hub_url
+
+from common import load_model_card
+
+
+def selected_files(repo_id: str, allow_files: list[str] | None) -> list[tuple[str, int]]:
+    api = HfApi()
+    info = api.model_info(repo_id=repo_id, files_metadata=True)
+    selected = []
+    allowed = set(allow_files or [])
+    for sibling in info.siblings:
+        name = sibling.rfilename
+        if name == ".gitattributes":
+            continue
+        if allowed and name not in allowed:
+            continue
+        if sibling.size is None:
+            raise ValueError(f"Unknown remote size for {name}")
+        selected.append((name, int(sibling.size)))
+    return selected
+
+
+def request_range(url: str, start: int, end: int, token: str | None) -> urllib.request.Request:
+    headers = {
+        "Range": f"bytes={start}-{end}",
+        "User-Agent": "LLM-local-range-downloader/1.0",
+    }
+    if token:
+        headers["Authorization"] = f"Bearer {token}"
+    return urllib.request.Request(url, headers=headers, method="GET")
+
+
+def local_size(path: Path) -> int:
+    return path.stat().st_size if path.exists() else 0
+
+
+def download_file(
+    repo_id: str,
+    filename: str,
+    output: Path,
+    expected_size: int,
+    *,
+    chunk_size: int,
+    retries: int,
+    retry_delay: float,
+    timeout: int,
+    token: str | None,
+) -> bool:
+    output.parent.mkdir(parents=True, exist_ok=True)
+    url = hf_hub_url(repo_id, filename)
+
+    while True:
+        current_size = local_size(output)
+        if current_size == expected_size:
+            print(f"OK {filename}: {current_size}/{expected_size}")
+            return True
+        if current_size > expected_size:
+            print(
+                f"ERROR {filename}: local file is larger than expected: {current_size}/{expected_size}",
+                file=sys.stderr,
+            )
+            return False
+
+        start = current_size
+        end = min(start + chunk_size - 1, expected_size - 1)
+        print(f"GET {filename}: bytes {start}-{end} of {expected_size}")
+
+        recompute_range = False
+        for attempt in range(1, retries + 1):
+            try:
+                request = request_range(url, start, end, token)
+                with urllib.request.urlopen(request, timeout=timeout) as response:
+                    status = response.status
+                    if status == 200:
+                        raise RuntimeError("server ignored Range request")
+                    if status not in (200, 206):
+                        raise RuntimeError(f"unexpected HTTP status {status}")
+
+                    mode = "ab" if start else "wb"
+                    written = 0
+                    with output.open(mode + "") as handle:
+                        while True:
+                            block = response.read(1024 * 1024)
+                            if not block:
+                                break
+                            handle.write(block)
+                            written += len(block)
+
+                if written == 0:
+                    raise RuntimeError("downloaded zero bytes")
+
+                new_size = local_size(output)
+                if new_size <= current_size:
+                    raise RuntimeError(f"file did not grow: {new_size}")
+                if new_size > expected_size:
+                    raise RuntimeError(f"file grew past expected size: {new_size}/{expected_size}")
+                break
+            except (OSError, RuntimeError, urllib.error.URLError, urllib.error.HTTPError) as exc:
+                new_size = local_size(output)
+                if new_size > current_size:
+                    print(
+                        f"WARN {filename}: partial chunk saved, size is now {new_size}/{expected_size}",
+                        file=sys.stderr,
+                    )
+                    recompute_range = True
+                    break
+                print(f"WARN {filename}: attempt {attempt}/{retries} failed: {exc}", file=sys.stderr)
+                if attempt == retries:
+                    return False
+                time.sleep(retry_delay)
+        if recompute_range:
+            continue
+
+
+def parse_size(value: str) -> int:
+    text = value.strip().lower()
+    multipliers = {
+        "k": 1024,
+        "kb": 1024,
+        "m": 1024**2,
+        "mb": 1024**2,
+        "g": 1024**3,
+        "gb": 1024**3,
+    }
+    for suffix, multiplier in multipliers.items():
+        if text.endswith(suffix):
+            return int(float(text[: -len(suffix)]) * multiplier)
+    return int(text)
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description="Download Hugging Face files using explicit HTTP ranges.")
+    parser.add_argument("card_id")
+    parser.add_argument("--local-dir", required=True)
+    parser.add_argument("--allow-file", action="append", dest="allow_files")
+    parser.add_argument("--chunk-size", default="64mb")
+    parser.add_argument("--retries", type=int, default=20)
+    parser.add_argument("--retry-delay", type=float, default=2.0)
+    parser.add_argument("--timeout", type=int, default=120)
+    args = parser.parse_args()
+
+    card = load_model_card(args.card_id)
+    repo_id = card.get("upstream_id")
+    if not repo_id:
+        print(f"{args.card_id}: upstream_id is required", file=sys.stderr)
+        return 1
+
+    files = selected_files(repo_id, args.allow_files)
+    if not files:
+        print("No files selected.", file=sys.stderr)
+        return 1
+
+    token = os.environ.get("HF_TOKEN") or None
+    local_dir = Path(args.local_dir)
+    chunk_size = parse_size(args.chunk_size)
+    failed = []
+
+    for filename, expected_size in files:
+        ok = download_file(
+            repo_id,
+            filename,
+            local_dir / filename,
+            expected_size,
+            chunk_size=chunk_size,
+            retries=args.retries,
+            retry_delay=args.retry_delay,
+            timeout=args.timeout,
+            token=token,
+        )
+        if not ok:
+            current_size = local_size(local_dir / filename)
+            failed.append((filename, current_size, expected_size))
+
+    if failed:
+        print("Incomplete downloads:", file=sys.stderr)
+        for filename, current_size, expected_size in failed:
+            print(f"- {filename}: {current_size}/{expected_size}", file=sys.stderr)
+        return 1
+
+    print(f"Downloaded {len(files)} file(s) for {args.card_id}.")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())