Promote encrypted off-host backup #2

Merged
m merged 1 commits from codex/foundation-v1 into main 2026-08-16 15:13:49 +00:00
7 changed files with 337 additions and 2 deletions
@@ -0,0 +1,52 @@
[CmdletBinding()]
param(
[string]$ScriptPath = '\\nas\MST\codex\assistent\deploy\windows\Sync-PostgresBackupToNas.ps1',
[string]$KeyRecordPath = "$env:LOCALAPPDATA\Domovoy\postgres-backup-key.dpapi.json",
[string]$NasKeyBackupPath = '\\nas\MST\codex\_backups\domovoy\keys\postgres-backup-key.dpapi.json',
[string]$TaskName = 'Domovoy PostgreSQL Off-host Backup'
)
$ErrorActionPreference = 'Stop'
Set-StrictMode -Version Latest
$keyDirectory = Split-Path -Parent $KeyRecordPath
New-Item -ItemType Directory -Force -Path $keyDirectory | Out-Null
$identity = [Security.Principal.WindowsIdentity]::GetCurrent().Name
if (-not (Test-Path -LiteralPath $KeyRecordPath) -and (Test-Path -LiteralPath $NasKeyBackupPath -PathType Leaf)) {
Copy-Item -LiteralPath $NasKeyBackupPath -Destination $KeyRecordPath
}
if (-not (Test-Path -LiteralPath $KeyRecordPath)) {
$random = [byte[]]::new(48)
[Security.Cryptography.RandomNumberGenerator]::Fill($random)
$plain = [Convert]::ToBase64String($random)
try {
$protected = ConvertFrom-SecureString (ConvertTo-SecureString $plain -AsPlainText -Force)
$record = [ordered]@{ version = 1; project = 'domovoy'; purpose = 'postgres off-host backup'; protected_value = $protected; created_at_utc = [DateTime]::UtcNow.ToString('o') }
[IO.File]::WriteAllText($KeyRecordPath, ($record | ConvertTo-Json) + "`n", [Text.UTF8Encoding]::new($false))
}
finally {
[Array]::Clear($random, 0, $random.Length)
$plain = $null
}
}
& icacls.exe $KeyRecordPath /inheritance:r /grant:r "${identity}:F" | Out-Null
if ($LASTEXITCODE -ne 0) { throw 'DPAPI key ACL hardening failed' }
$nasKeyDirectory = Split-Path -Parent $NasKeyBackupPath
New-Item -ItemType Directory -Force -Path $nasKeyDirectory | Out-Null
$nasKeyPartial = "$NasKeyBackupPath.$PID.partial"
Copy-Item -LiteralPath $KeyRecordPath -Destination $nasKeyPartial
Move-Item -LiteralPath $nasKeyPartial -Destination $NasKeyBackupPath -Force
if (-not (Test-Path -LiteralPath $ScriptPath -PathType Leaf)) { throw "Scheduled sync script not found" }
$pwsh = (Get-Command pwsh.exe -ErrorAction Stop).Source
$arguments = "-NoProfile -NonInteractive -ExecutionPolicy Bypass -File `"$ScriptPath`""
$action = New-ScheduledTaskAction -Execute $pwsh -Argument $arguments
$daily = New-ScheduledTaskTrigger -Daily -At '04:00'
$logon = New-ScheduledTaskTrigger -AtLogOn -User $identity
$principal = New-ScheduledTaskPrincipal -UserId $identity -LogonType Interactive -RunLevel Limited
$settings = New-ScheduledTaskSettingsSet -StartWhenAvailable -MultipleInstances IgnoreNew -ExecutionTimeLimit (New-TimeSpan -Hours 2)
Register-ScheduledTask -TaskName $TaskName -Action $action -Trigger @($daily, $logon) -Principal $principal -Settings $settings -Description 'Encrypt and replicate the latest Domovoy PostgreSQL backup to the private NAS.' -Force | Out-Null
Write-Output "Scheduled task registered: $TaskName"
+115
View File
@@ -0,0 +1,115 @@
[CmdletBinding()]
param(
[string]$SshHost = 'docker',
[string]$RemoteBackupRoot = '/home/m/assistant-backups/postgres',
[string]$NasBackupRoot = '\\nas\MST\codex\_backups\domovoy\postgres',
[string]$KeyRecordPath = "$env:LOCALAPPDATA\Domovoy\postgres-backup-key.dpapi.json",
[string]$GpgPath = 'gpg.exe'
)
$ErrorActionPreference = 'Stop'
Set-StrictMode -Version Latest
function Assert-SourceHashes([string]$Directory) {
$allowed = @('assistant.dump', 'globals.sql', 'database.name', 'owner.name')
$expected = @{}
foreach ($line in Get-Content -LiteralPath (Join-Path $Directory 'SHA256SUMS')) {
if ($line -notmatch '^([0-9a-f]{64}) ([A-Za-z0-9.]+)$' -or $matches[2] -notin $allowed) {
throw "Invalid SHA256SUMS entry"
}
$expected[$matches[2]] = $matches[1]
}
foreach ($name in $allowed) {
if (-not $expected.ContainsKey($name)) { throw "Missing SHA256SUMS entry: $name" }
$actual = (Get-FileHash -Algorithm SHA256 -LiteralPath (Join-Path $Directory $name)).Hash.ToLowerInvariant()
if ($actual -ne $expected[$name]) { throw "Source hash mismatch: $name" }
}
}
function Get-RecoveryPassphrase([string]$Path) {
if (-not (Test-Path -LiteralPath $Path -PathType Leaf)) { throw "DPAPI recovery key is missing" }
$record = Get-Content -LiteralPath $Path -Raw | ConvertFrom-Json
if ($record.version -ne 1 -or [string]::IsNullOrWhiteSpace($record.protected_value)) { throw "Invalid DPAPI recovery key record" }
$secure = ConvertTo-SecureString -String $record.protected_value
$pointer = [Runtime.InteropServices.Marshal]::SecureStringToBSTR($secure)
try { return [Runtime.InteropServices.Marshal]::PtrToStringBSTR($pointer) }
finally { [Runtime.InteropServices.Marshal]::ZeroFreeBSTR($pointer) }
}
$tempRoot = [IO.Path]::GetFullPath([IO.Path]::GetTempPath())
$work = Join-Path $tempRoot ("domovoy-backup-sync-" + [guid]::NewGuid().ToString('N'))
if (-not ([IO.Path]::GetFullPath($work)).StartsWith($tempRoot, [StringComparison]::OrdinalIgnoreCase)) { throw 'Unsafe temporary path' }
New-Item -ItemType Directory -Path $work | Out-Null
$passphrase = $null
$cipherPartial = $null
$manifestPartial = $null
try {
$latest = ((& ssh.exe $SshHost "find '$RemoteBackupRoot' -mindepth 1 -maxdepth 1 -type d ! -name '.*' -printf '%f\n' | sort | tail -n1") -join '').Trim()
if ($LASTEXITCODE -ne 0 -or $latest -notmatch '^\d{8}T\d{6}Z$') { throw 'No valid completed production backup found' }
New-Item -ItemType Directory -Force -Path $NasBackupRoot | Out-Null
$cipherPath = Join-Path $NasBackupRoot "$latest.tar.gz.gpg"
$manifestPath = Join-Path $NasBackupRoot "$latest.json"
if ((Test-Path -LiteralPath $cipherPath) -xor (Test-Path -LiteralPath $manifestPath)) {
# Neither lone file is a completed backup. Remove only this exact incomplete pair before retry.
Remove-Item -LiteralPath $cipherPath, $manifestPath -Force -ErrorAction SilentlyContinue
}
if ((Test-Path -LiteralPath $cipherPath) -or (Test-Path -LiteralPath $manifestPath)) {
if (-not ((Test-Path -LiteralPath $cipherPath -PathType Leaf) -and (Test-Path -LiteralPath $manifestPath -PathType Leaf))) { throw 'Incomplete existing NAS artifact' }
$existing = Get-Content -LiteralPath $manifestPath -Raw | ConvertFrom-Json
$actualCipherHash = (Get-FileHash -Algorithm SHA256 -LiteralPath $cipherPath).Hash.ToLowerInvariant()
if ($existing.source_backup -ne $latest -or $existing.ciphertext_sha256 -ne $actualCipherHash) { throw 'Existing NAS artifact does not match its manifest' }
Write-Output "Off-host backup already current: $latest"
return
}
& scp.exe -q -r "${SshHost}:${RemoteBackupRoot}/${latest}" $work
if ($LASTEXITCODE -ne 0) { throw 'SCP backup copy failed' }
$sourceDirectory = Join-Path $work $latest
Assert-SourceHashes $sourceDirectory
$archive = Join-Path $work "$latest.tar.gz"
& tar.exe -czf $archive -C $work $latest
if ($LASTEXITCODE -ne 0 -or -not (Test-Path -LiteralPath $archive -PathType Leaf)) { throw 'Archive creation failed' }
$archiveHash = (Get-FileHash -Algorithm SHA256 -LiteralPath $archive).Hash.ToLowerInvariant()
$passphrase = Get-RecoveryPassphrase $KeyRecordPath
$passphraseFile = Join-Path $work 'passphrase.txt'
[IO.File]::WriteAllText($passphraseFile, $passphrase, [Text.UTF8Encoding]::new($false))
& icacls.exe $work /inheritance:r /grant:r "$([Security.Principal.WindowsIdentity]::GetCurrent().Name):(OI)(CI)F" | Out-Null
if ($LASTEXITCODE -ne 0) { throw 'Temporary ACL hardening failed' }
$cipherPartial = Join-Path $NasBackupRoot (".$latest.$PID.gpg.partial")
$manifestPartial = Join-Path $NasBackupRoot (".$latest.$PID.json.partial")
& $GpgPath --batch --yes --pinentry-mode loopback --passphrase-file $passphraseFile --symmetric --cipher-algo AES256 --s2k-digest-algo SHA512 --s2k-cipher-algo AES256 --s2k-count 65011712 --compress-algo none --output $cipherPartial $archive
if ($LASTEXITCODE -ne 0 -or -not (Test-Path -LiteralPath $cipherPartial -PathType Leaf)) { throw 'GnuPG encryption failed' }
$roundTrip = Join-Path $work 'roundtrip.tar.gz'
& $GpgPath --batch --yes --pinentry-mode loopback --passphrase-file $passphraseFile --decrypt --output $roundTrip $cipherPartial
if ($LASTEXITCODE -ne 0) { throw 'GnuPG round-trip decryption failed' }
if ((Get-FileHash -Algorithm SHA256 -LiteralPath $roundTrip).Hash.ToLowerInvariant() -ne $archiveHash) { throw 'Encrypted round-trip archive hash mismatch' }
$manifest = [ordered]@{
version = 1
project = 'domovoy'
source_host = 'docker.cin.su'
source_backup = $latest
source_sha256sums_sha256 = (Get-FileHash -Algorithm SHA256 -LiteralPath (Join-Path $sourceDirectory 'SHA256SUMS')).Hash.ToLowerInvariant()
archive_sha256 = $archiveHash
ciphertext_sha256 = (Get-FileHash -Algorithm SHA256 -LiteralPath $cipherPartial).Hash.ToLowerInvariant()
encryption = 'GnuPG symmetric AES256, SHA512 S2K'
created_at_utc = [DateTime]::UtcNow.ToString('o')
}
[IO.File]::WriteAllText($manifestPartial, ($manifest | ConvertTo-Json) + "`n", [Text.UTF8Encoding]::new($false))
Move-Item -LiteralPath $cipherPartial -Destination $cipherPath
Move-Item -LiteralPath $manifestPartial -Destination $manifestPath
Write-Output "Off-host backup created: $latest"
}
finally {
$passphrase = $null
foreach ($partial in @($cipherPartial, $manifestPartial)) {
if ($partial -and (Test-Path -LiteralPath $partial)) { Remove-Item -LiteralPath $partial -Force }
}
if (Test-Path -LiteralPath $work) { Remove-Item -LiteralPath $work -Recurse -Force }
}
@@ -0,0 +1,75 @@
[CmdletBinding()]
param(
[string]$NasBackupRoot = '\\nas\MST\codex\_backups\domovoy\postgres',
[string]$KeyRecordPath = "$env:LOCALAPPDATA\Domovoy\postgres-backup-key.dpapi.json",
[string]$GpgPath = 'gpg.exe',
[string]$SshHost = 'docker',
[switch]$VerifyRemoteRestore
)
$ErrorActionPreference = 'Stop'
Set-StrictMode -Version Latest
function Get-RecoveryPassphrase([string]$Path) {
$record = Get-Content -LiteralPath $Path -Raw | ConvertFrom-Json
if ($record.version -ne 1 -or [string]::IsNullOrWhiteSpace($record.protected_value)) { throw 'Invalid DPAPI recovery key record' }
$secure = ConvertTo-SecureString -String $record.protected_value
$pointer = [Runtime.InteropServices.Marshal]::SecureStringToBSTR($secure)
try { return [Runtime.InteropServices.Marshal]::PtrToStringBSTR($pointer) }
finally { [Runtime.InteropServices.Marshal]::ZeroFreeBSTR($pointer) }
}
$manifestFile = Get-ChildItem -LiteralPath $NasBackupRoot -Filter '*.json' -File | Sort-Object BaseName | Select-Object -Last 1
if (-not $manifestFile -or $manifestFile.BaseName -notmatch '^\d{8}T\d{6}Z$') { throw 'No valid NAS backup manifest found' }
$manifest = Get-Content -LiteralPath $manifestFile.FullName -Raw | ConvertFrom-Json
$latest = $manifestFile.BaseName
$cipherPath = Join-Path $NasBackupRoot "$latest.tar.gz.gpg"
if ((Get-FileHash -Algorithm SHA256 -LiteralPath $cipherPath).Hash.ToLowerInvariant() -ne $manifest.ciphertext_sha256) { throw 'NAS ciphertext hash mismatch' }
$tempRoot = [IO.Path]::GetFullPath([IO.Path]::GetTempPath())
$work = Join-Path $tempRoot ("domovoy-backup-test-" + [guid]::NewGuid().ToString('N'))
New-Item -ItemType Directory -Path $work | Out-Null
$passphrase = $null
$remoteCheck = $null
try {
& icacls.exe $work /inheritance:r /grant:r "$([Security.Principal.WindowsIdentity]::GetCurrent().Name):(OI)(CI)F" | Out-Null
if ($LASTEXITCODE -ne 0) { throw 'Temporary ACL hardening failed' }
$passphrase = Get-RecoveryPassphrase $KeyRecordPath
$passphraseFile = Join-Path $work 'passphrase.txt'
[IO.File]::WriteAllText($passphraseFile, $passphrase, [Text.UTF8Encoding]::new($false))
$archive = Join-Path $work "$latest.tar.gz"
& $GpgPath --batch --yes --pinentry-mode loopback --passphrase-file $passphraseFile --decrypt --output $archive $cipherPath
if ($LASTEXITCODE -ne 0) { throw 'NAS backup decryption failed' }
if ((Get-FileHash -Algorithm SHA256 -LiteralPath $archive).Hash.ToLowerInvariant() -ne $manifest.archive_sha256) { throw 'Decrypted archive hash mismatch' }
& tar.exe -xzf $archive -C $work
if ($LASTEXITCODE -ne 0) { throw 'Decrypted archive extraction failed' }
$sourceDirectory = Join-Path $work $latest
Push-Location $sourceDirectory
try {
foreach ($line in Get-Content -LiteralPath 'SHA256SUMS') {
if ($line -notmatch '^([0-9a-f]{64}) ([A-Za-z0-9.]+)$') { throw 'Invalid decrypted SHA256SUMS' }
if ((Get-FileHash -Algorithm SHA256 -LiteralPath $matches[2]).Hash.ToLowerInvariant() -ne $matches[1]) { throw "Decrypted file hash mismatch: $($matches[2])" }
}
}
finally { Pop-Location }
if ($VerifyRemoteRestore) {
$remoteCheck = "/home/m/assistant-backups/offhost-restore-check/$latest-$([guid]::NewGuid().ToString('N'))"
if ($remoteCheck -notmatch '^/home/m/assistant-backups/offhost-restore-check/\d{8}T\d{6}Z-[0-9a-f]{32}$') { throw 'Unsafe remote restore path' }
& ssh.exe $SshHost "mkdir -p '$remoteCheck'"
if ($LASTEXITCODE -ne 0) { throw 'Remote restore-check directory creation failed' }
& scp.exe -q -r "$sourceDirectory\*" "${SshHost}:${remoteCheck}/"
if ($LASTEXITCODE -ne 0) { throw 'Remote restore-check copy failed' }
& ssh.exe $SshHost "/home/m/assistant-ops/verify-postgres-restore.sh '$remoteCheck'"
if ($LASTEXITCODE -ne 0) { throw 'Remote PostgreSQL restore check failed' }
}
Write-Output "Off-host backup verified: $latest"
}
finally {
if ($remoteCheck -and $remoteCheck -match '^/home/m/assistant-backups/offhost-restore-check/\d{8}T\d{6}Z-[0-9a-f]{32}$') {
& ssh.exe $SshHost "rm -rf -- '$remoteCheck'" | Out-Null
}
$passphrase = $null
if (Test-Path -LiteralPath $work) { Remove-Item -LiteralPath $work -Recurse -Force }
}
+2 -2
View File
@@ -1,5 +1,5 @@
{
"task_file": "dev/tasks/DEV-000036-creator-secret-files.json",
"task_id": "DEV-000036",
"task_file": "dev/tasks/DEV-000037-encrypted-offhost-backup.json",
"task_id": "DEV-000037",
"task_revision": 1
}
@@ -0,0 +1,85 @@
{
"task_id": "DEV-000037",
"task_revision": 1,
"architecture_revision": "ADR-0004",
"base_repo_revision": "03dde8c1961f214fbb5ebdefa131eb38fd0bef57",
"risk_tier": "E",
"intent": "Replicate verified production PostgreSQL backups to the private NAS as authenticated encrypted artifacts, with a DPAPI-protected recovery key and tested restore path, without using docker-test.",
"requirement_ids": ["CONSTITUTION-1.3.36", "USER-DOCKER-PROD-ONLY"],
"allowed_change_scope": {
"modules": [],
"paths": [
"deploy/windows/Sync-PostgresBackupToNas.ps1",
"deploy/windows/Test-PostgresBackupFromNas.ps1",
"deploy/windows/Register-PostgresBackupNasTask.ps1",
"docs/CREDENTIALS_AND_SECRETS.md",
"docs/DEPLOYMENT.md",
"dev/ACTIVE_TASK.json",
"dev/tasks/DEV-000037-encrypted-offhost-backup.json"
]
},
"forbidden_change_scope": {
"paths": [
"architecture/**",
"ci/**",
".gitea/**",
"cmd/**",
"contracts/**",
"modules/**",
"database/**",
"security/**",
"deploy/docker-compose*.yml"
]
},
"allowed_architecture_delta": {
"new_cross_module_dependencies": 0,
"new_canonical_owners": 0,
"new_privileges": 0,
"new_operator_dependencies": 1
},
"dependencies": {
"operator": ["GnuPG 2.4+ already installed on the authorized Windows backup workstation"]
},
"contracts": {
"consume": ["DEV-000035 verified PostgreSQL backup directory", "SSH key to docker.cin.su", "authenticated NAS session", "Windows DPAPI CurrentUser"],
"may_change": []
},
"semantic_owner": "none; disaster-recovery transport only",
"canonical_state": ["none; backup artifacts are exported recovery copies"],
"state_transitions": ["verified local backup -> encrypted off-host copy -> decrypt-and-restore verified copy"],
"side_effects_and_privileges": ["reads completed production backup artifacts over SSH", "writes encrypted artifacts to the private NAS", "stores a random recovery passphrase as CurrentUser DPAPI ciphertext outside Git", "registers a current-user scheduled task"],
"persistence_and_migrations": ["no database migration", "encrypted .gpg artifact plus non-secret JSON manifest on NAS", "no automatic deletion or retention policy"],
"failure_retry_restart": ["NAS completion is indicated only after ciphertext and manifest are atomically renamed", "existing matching artifacts are idempotently retained", "temporary plaintext is confined to a validated per-run Windows temp directory and removed on exit", "scheduled retries run at logon and daily", "failed remote restore checks remove only their generated temporary directory"],
"assumptions": [
"The authorized Windows user retains access to both the SSH alias docker and the private NAS share",
"GnuPG 2.4+ remains installed on that workstation",
"The DPAPI profile and NAS are independent from the production Docker host",
"docker-test remains prohibited"
],
"acceptance_scenarios": [
"the latest completed production backup is copied and source hashes are checked before encryption",
"NAS contains ciphertext and a non-secret manifest but no plaintext PostgreSQL dump",
"the recovery passphrase is random and stored only as DPAPI ciphertext outside the repository",
"ciphertext decrypts and reproduces the source archive hash",
"the decrypted off-host copy passes SHA-256 checks and an isolated PostgreSQL restore on docker.cin.su",
"daily and logon triggers are registered for the current user",
"no backup is automatically deleted",
"exact diff validation observes only declared paths"
],
"required_tests": [
"PowerShell parser",
"architecture preflight",
"exact diff scope",
"source SHA-256 validation",
"GnuPG encrypt/decrypt round trip",
"NAS ciphertext hash validation",
"isolated restore from decrypted NAS artifact",
"scheduled task registration and manual invocation",
"production health",
"CI"
],
"terminal_gap_states": [
"BLOCKED_BY_MISSING_CAPABILITY",
"BLOCKED_BY_CONFLICT"
]
}
+2
View File
@@ -22,3 +22,5 @@ Do not guess Portainer credentials and do not search other project source trees
Each privileged runtime gets only its required secret/capability bindings. `commit-runtime` is intentionally isolated from the general Core Runtime.
Creator authentication secrets are stored in the external Docker volume `assistant_core_secrets` and mounted read-only only at `/run/secrets` in `core-runtime`. The production container environment contains `_FILE` paths, not the secret values. Files are owned by runtime uid 100/gid 101 with mode 0400. `CREATOR_USERNAME` is not secret and remains an environment value. Development may continue to use direct environment values, but a direct value and its matching `_FILE` setting must never be configured together.
The PostgreSQL off-host backup passphrase is generated randomly and stored as CurrentUser DPAPI ciphertext at `%LOCALAPPDATA%\Domovoy\postgres-backup-key.dpapi.json`. An atomic copy of that DPAPI ciphertext, never the plaintext passphrase, is retained at `\\nas\MST\codex\_backups\domovoy\keys\postgres-backup-key.dpapi.json`. The passphrase is never stored in Git, the production Docker host, a scheduled-task argument, or NAS plaintext. The NAS backup directory receives only GnuPG AES-256 ciphertext and a non-secret integrity manifest. Recovery therefore requires both the NAS artifacts and the authorized Windows DPAPI profile.
+6
View File
@@ -27,3 +27,9 @@ Production backups run only on `docker.cin.su`; this project must not use the sh
`deploy/verify-postgres-restore.sh <backup-directory>` verifies all hashes and restores into an isolated PostgreSQL 18.6 container backed by tmpfs. It never mounts or writes `assistant_assistant_pg`, and removes its temporary container on exit. A backup is operationally accepted only after this restore check passes.
The production user crontab installs the single entry from `deploy/cron/assistant-postgres-backup.crontab`. It runs the backup and restore check daily at 00:15 UTC (03:15 Europe/Moscow), survives host reboot without a persistent login session, and writes a private operational log beside the backups.
## Encrypted off-host backup
`deploy/windows/Sync-PostgresBackupToNas.ps1` copies only a completed, hash-verified production backup over SSH, archives it locally, encrypts it with GnuPG AES-256, verifies an immediate decrypt round trip, and atomically publishes ciphertext plus a non-secret manifest under `\\nas\MST\codex\_backups\domovoy\postgres`. Temporary plaintext is removed on exit. The script is idempotent and never deletes existing backups.
`deploy/windows/Test-PostgresBackupFromNas.ps1 -VerifyRemoteRestore` verifies the NAS ciphertext hash, decrypts it with the DPAPI-protected recovery key, checks every source hash, and performs a full isolated PostgreSQL restore on `docker.cin.su`. `deploy/windows/Register-PostgresBackupNasTask.ps1` generates the key when absent, atomically backs up only its DPAPI ciphertext to the NAS, and registers current-user daily and logon triggers. The task intentionally does not use `docker-test`.