Initial SQL-only 1C adapter baseline

2026-07-22 03:03:47 +03:00
commit e2503b77e7
545 changed files with 184711 additions and 0 deletions
@@ -0,0 +1,78 @@
+[CmdletBinding()]
+param(
+    [int]$MaxPages = 50,
+    [switch]$PromptCookie,
+    [switch]$NoResume,
+    [switch]$FetchOnly,
+    [string]$CookieStore = "plugins/1c/rag/official-docs/.local/its-cookie.dpapi.txt",
+    [string]$Sources = "plugins/1c/rag/official-docs/sources.yaml",
+    [string]$RawDir = "plugins/1c/rag/official-docs/raw",
+    [string]$NormalizedDir = "plugins/1c/rag/official-docs/normalized",
+    [string]$RagSourceDir = "plugins/1c/rag/sources/official/its"
+)
+
+$ErrorActionPreference = "Stop"
+
+Add-Type -AssemblyName System.Security
+
+function Unprotect-Text([string]$EncryptedText) {
+    $bytes = [Convert]::FromBase64String($EncryptedText.Trim())
+    $plain = [System.Security.Cryptography.ProtectedData]::Unprotect(
+        $bytes,
+        $null,
+        [System.Security.Cryptography.DataProtectionScope]::CurrentUser
+    )
+    return [System.Text.Encoding]::UTF8.GetString($plain)
+}
+
+if ($PromptCookie -or -not (Test-Path -LiteralPath $CookieStore)) {
+    & (Join-Path $PSScriptRoot "set_1c_its_cookie.ps1") -CookieStore $CookieStore
+}
+
+if (-not (Test-Path -LiteralPath $CookieStore)) {
+    throw "Cookie store was not created: $CookieStore"
+}
+
+$env:ONEC_ITS_COOKIE = Unprotect-Text (Get-Content -LiteralPath $CookieStore -Raw -Encoding UTF8)
+if (-not $env:ONEC_ITS_COOKIE) {
+    throw "Stored 1C:ITS cookie is empty. Run with -PromptCookie."
+}
+
+$rawManifest = Join-Path $RawDir "manifest.json"
+$rawProgress = Join-Path $RawDir "progress.json"
+$normalizedManifest = Join-Path $NormalizedDir "manifest.json"
+
+$fetchArgs = @(
+    "scripts/fetch_1c_its_docs.py",
+    "--sources", $Sources,
+    "--output-dir", $RawDir,
+    "--manifest", $rawManifest,
+    "--progress", $rawProgress,
+    "--max-pages", $MaxPages
+)
+
+if ($NoResume) {
+    $fetchArgs += "--no-resume"
+}
+
+python @fetchArgs
+
+if ($FetchOnly) {
+    Write-Host "Fetch completed. Raw manifest: $rawManifest"
+    exit 0
+}
+
+python scripts/normalize_1c_its_docs.py `
+    --manifest $rawManifest `
+    --raw-dir $RawDir `
+    --output-dir $NormalizedDir `
+    --rag-source-dir $RagSourceDir `
+    --manifest-output $normalizedManifest
+
+python scripts/prepare_1c_rag_corpus.py
+python scripts/build_1c_rag_index.py
+
+Write-Host "1C:ITS docs pipeline completed."
+Write-Host "Raw manifest: $rawManifest"
+Write-Host "Raw progress: $rawProgress"
+Write-Host "Normalized manifest: $normalizedManifest"