ops: add encrypted off-host PostgreSQL backups
This commit was merged in pull request #2.
This commit is contained in:
@@ -0,0 +1,52 @@
|
||||
[CmdletBinding()]
|
||||
param(
|
||||
[string]$ScriptPath = '\\nas\MST\codex\assistent\deploy\windows\Sync-PostgresBackupToNas.ps1',
|
||||
[string]$KeyRecordPath = "$env:LOCALAPPDATA\Domovoy\postgres-backup-key.dpapi.json",
|
||||
[string]$NasKeyBackupPath = '\\nas\MST\codex\_backups\domovoy\keys\postgres-backup-key.dpapi.json',
|
||||
[string]$TaskName = 'Domovoy PostgreSQL Off-host Backup'
|
||||
)
|
||||
|
||||
$ErrorActionPreference = 'Stop'
|
||||
Set-StrictMode -Version Latest
|
||||
|
||||
$keyDirectory = Split-Path -Parent $KeyRecordPath
|
||||
New-Item -ItemType Directory -Force -Path $keyDirectory | Out-Null
|
||||
$identity = [Security.Principal.WindowsIdentity]::GetCurrent().Name
|
||||
|
||||
if (-not (Test-Path -LiteralPath $KeyRecordPath) -and (Test-Path -LiteralPath $NasKeyBackupPath -PathType Leaf)) {
|
||||
Copy-Item -LiteralPath $NasKeyBackupPath -Destination $KeyRecordPath
|
||||
}
|
||||
|
||||
if (-not (Test-Path -LiteralPath $KeyRecordPath)) {
|
||||
$random = [byte[]]::new(48)
|
||||
[Security.Cryptography.RandomNumberGenerator]::Fill($random)
|
||||
$plain = [Convert]::ToBase64String($random)
|
||||
try {
|
||||
$protected = ConvertFrom-SecureString (ConvertTo-SecureString $plain -AsPlainText -Force)
|
||||
$record = [ordered]@{ version = 1; project = 'domovoy'; purpose = 'postgres off-host backup'; protected_value = $protected; created_at_utc = [DateTime]::UtcNow.ToString('o') }
|
||||
[IO.File]::WriteAllText($KeyRecordPath, ($record | ConvertTo-Json) + "`n", [Text.UTF8Encoding]::new($false))
|
||||
}
|
||||
finally {
|
||||
[Array]::Clear($random, 0, $random.Length)
|
||||
$plain = $null
|
||||
}
|
||||
}
|
||||
& icacls.exe $KeyRecordPath /inheritance:r /grant:r "${identity}:F" | Out-Null
|
||||
if ($LASTEXITCODE -ne 0) { throw 'DPAPI key ACL hardening failed' }
|
||||
|
||||
$nasKeyDirectory = Split-Path -Parent $NasKeyBackupPath
|
||||
New-Item -ItemType Directory -Force -Path $nasKeyDirectory | Out-Null
|
||||
$nasKeyPartial = "$NasKeyBackupPath.$PID.partial"
|
||||
Copy-Item -LiteralPath $KeyRecordPath -Destination $nasKeyPartial
|
||||
Move-Item -LiteralPath $nasKeyPartial -Destination $NasKeyBackupPath -Force
|
||||
|
||||
if (-not (Test-Path -LiteralPath $ScriptPath -PathType Leaf)) { throw "Scheduled sync script not found" }
|
||||
$pwsh = (Get-Command pwsh.exe -ErrorAction Stop).Source
|
||||
$arguments = "-NoProfile -NonInteractive -ExecutionPolicy Bypass -File `"$ScriptPath`""
|
||||
$action = New-ScheduledTaskAction -Execute $pwsh -Argument $arguments
|
||||
$daily = New-ScheduledTaskTrigger -Daily -At '04:00'
|
||||
$logon = New-ScheduledTaskTrigger -AtLogOn -User $identity
|
||||
$principal = New-ScheduledTaskPrincipal -UserId $identity -LogonType Interactive -RunLevel Limited
|
||||
$settings = New-ScheduledTaskSettingsSet -StartWhenAvailable -MultipleInstances IgnoreNew -ExecutionTimeLimit (New-TimeSpan -Hours 2)
|
||||
Register-ScheduledTask -TaskName $TaskName -Action $action -Trigger @($daily, $logon) -Principal $principal -Settings $settings -Description 'Encrypt and replicate the latest Domovoy PostgreSQL backup to the private NAS.' -Force | Out-Null
|
||||
Write-Output "Scheduled task registered: $TaskName"
|
||||
@@ -0,0 +1,115 @@
|
||||
[CmdletBinding()]
|
||||
param(
|
||||
[string]$SshHost = 'docker',
|
||||
[string]$RemoteBackupRoot = '/home/m/assistant-backups/postgres',
|
||||
[string]$NasBackupRoot = '\\nas\MST\codex\_backups\domovoy\postgres',
|
||||
[string]$KeyRecordPath = "$env:LOCALAPPDATA\Domovoy\postgres-backup-key.dpapi.json",
|
||||
[string]$GpgPath = 'gpg.exe'
|
||||
)
|
||||
|
||||
$ErrorActionPreference = 'Stop'
|
||||
Set-StrictMode -Version Latest
|
||||
|
||||
function Assert-SourceHashes([string]$Directory) {
|
||||
$allowed = @('assistant.dump', 'globals.sql', 'database.name', 'owner.name')
|
||||
$expected = @{}
|
||||
foreach ($line in Get-Content -LiteralPath (Join-Path $Directory 'SHA256SUMS')) {
|
||||
if ($line -notmatch '^([0-9a-f]{64}) ([A-Za-z0-9.]+)$' -or $matches[2] -notin $allowed) {
|
||||
throw "Invalid SHA256SUMS entry"
|
||||
}
|
||||
$expected[$matches[2]] = $matches[1]
|
||||
}
|
||||
foreach ($name in $allowed) {
|
||||
if (-not $expected.ContainsKey($name)) { throw "Missing SHA256SUMS entry: $name" }
|
||||
$actual = (Get-FileHash -Algorithm SHA256 -LiteralPath (Join-Path $Directory $name)).Hash.ToLowerInvariant()
|
||||
if ($actual -ne $expected[$name]) { throw "Source hash mismatch: $name" }
|
||||
}
|
||||
}
|
||||
|
||||
function Get-RecoveryPassphrase([string]$Path) {
|
||||
if (-not (Test-Path -LiteralPath $Path -PathType Leaf)) { throw "DPAPI recovery key is missing" }
|
||||
$record = Get-Content -LiteralPath $Path -Raw | ConvertFrom-Json
|
||||
if ($record.version -ne 1 -or [string]::IsNullOrWhiteSpace($record.protected_value)) { throw "Invalid DPAPI recovery key record" }
|
||||
$secure = ConvertTo-SecureString -String $record.protected_value
|
||||
$pointer = [Runtime.InteropServices.Marshal]::SecureStringToBSTR($secure)
|
||||
try { return [Runtime.InteropServices.Marshal]::PtrToStringBSTR($pointer) }
|
||||
finally { [Runtime.InteropServices.Marshal]::ZeroFreeBSTR($pointer) }
|
||||
}
|
||||
|
||||
$tempRoot = [IO.Path]::GetFullPath([IO.Path]::GetTempPath())
|
||||
$work = Join-Path $tempRoot ("domovoy-backup-sync-" + [guid]::NewGuid().ToString('N'))
|
||||
if (-not ([IO.Path]::GetFullPath($work)).StartsWith($tempRoot, [StringComparison]::OrdinalIgnoreCase)) { throw 'Unsafe temporary path' }
|
||||
New-Item -ItemType Directory -Path $work | Out-Null
|
||||
$passphrase = $null
|
||||
$cipherPartial = $null
|
||||
$manifestPartial = $null
|
||||
|
||||
try {
|
||||
$latest = ((& ssh.exe $SshHost "find '$RemoteBackupRoot' -mindepth 1 -maxdepth 1 -type d ! -name '.*' -printf '%f\n' | sort | tail -n1") -join '').Trim()
|
||||
if ($LASTEXITCODE -ne 0 -or $latest -notmatch '^\d{8}T\d{6}Z$') { throw 'No valid completed production backup found' }
|
||||
|
||||
New-Item -ItemType Directory -Force -Path $NasBackupRoot | Out-Null
|
||||
$cipherPath = Join-Path $NasBackupRoot "$latest.tar.gz.gpg"
|
||||
$manifestPath = Join-Path $NasBackupRoot "$latest.json"
|
||||
if ((Test-Path -LiteralPath $cipherPath) -xor (Test-Path -LiteralPath $manifestPath)) {
|
||||
# Neither lone file is a completed backup. Remove only this exact incomplete pair before retry.
|
||||
Remove-Item -LiteralPath $cipherPath, $manifestPath -Force -ErrorAction SilentlyContinue
|
||||
}
|
||||
if ((Test-Path -LiteralPath $cipherPath) -or (Test-Path -LiteralPath $manifestPath)) {
|
||||
if (-not ((Test-Path -LiteralPath $cipherPath -PathType Leaf) -and (Test-Path -LiteralPath $manifestPath -PathType Leaf))) { throw 'Incomplete existing NAS artifact' }
|
||||
$existing = Get-Content -LiteralPath $manifestPath -Raw | ConvertFrom-Json
|
||||
$actualCipherHash = (Get-FileHash -Algorithm SHA256 -LiteralPath $cipherPath).Hash.ToLowerInvariant()
|
||||
if ($existing.source_backup -ne $latest -or $existing.ciphertext_sha256 -ne $actualCipherHash) { throw 'Existing NAS artifact does not match its manifest' }
|
||||
Write-Output "Off-host backup already current: $latest"
|
||||
return
|
||||
}
|
||||
|
||||
& scp.exe -q -r "${SshHost}:${RemoteBackupRoot}/${latest}" $work
|
||||
if ($LASTEXITCODE -ne 0) { throw 'SCP backup copy failed' }
|
||||
$sourceDirectory = Join-Path $work $latest
|
||||
Assert-SourceHashes $sourceDirectory
|
||||
|
||||
$archive = Join-Path $work "$latest.tar.gz"
|
||||
& tar.exe -czf $archive -C $work $latest
|
||||
if ($LASTEXITCODE -ne 0 -or -not (Test-Path -LiteralPath $archive -PathType Leaf)) { throw 'Archive creation failed' }
|
||||
$archiveHash = (Get-FileHash -Algorithm SHA256 -LiteralPath $archive).Hash.ToLowerInvariant()
|
||||
|
||||
$passphrase = Get-RecoveryPassphrase $KeyRecordPath
|
||||
$passphraseFile = Join-Path $work 'passphrase.txt'
|
||||
[IO.File]::WriteAllText($passphraseFile, $passphrase, [Text.UTF8Encoding]::new($false))
|
||||
& icacls.exe $work /inheritance:r /grant:r "$([Security.Principal.WindowsIdentity]::GetCurrent().Name):(OI)(CI)F" | Out-Null
|
||||
if ($LASTEXITCODE -ne 0) { throw 'Temporary ACL hardening failed' }
|
||||
|
||||
$cipherPartial = Join-Path $NasBackupRoot (".$latest.$PID.gpg.partial")
|
||||
$manifestPartial = Join-Path $NasBackupRoot (".$latest.$PID.json.partial")
|
||||
& $GpgPath --batch --yes --pinentry-mode loopback --passphrase-file $passphraseFile --symmetric --cipher-algo AES256 --s2k-digest-algo SHA512 --s2k-cipher-algo AES256 --s2k-count 65011712 --compress-algo none --output $cipherPartial $archive
|
||||
if ($LASTEXITCODE -ne 0 -or -not (Test-Path -LiteralPath $cipherPartial -PathType Leaf)) { throw 'GnuPG encryption failed' }
|
||||
|
||||
$roundTrip = Join-Path $work 'roundtrip.tar.gz'
|
||||
& $GpgPath --batch --yes --pinentry-mode loopback --passphrase-file $passphraseFile --decrypt --output $roundTrip $cipherPartial
|
||||
if ($LASTEXITCODE -ne 0) { throw 'GnuPG round-trip decryption failed' }
|
||||
if ((Get-FileHash -Algorithm SHA256 -LiteralPath $roundTrip).Hash.ToLowerInvariant() -ne $archiveHash) { throw 'Encrypted round-trip archive hash mismatch' }
|
||||
|
||||
$manifest = [ordered]@{
|
||||
version = 1
|
||||
project = 'domovoy'
|
||||
source_host = 'docker.cin.su'
|
||||
source_backup = $latest
|
||||
source_sha256sums_sha256 = (Get-FileHash -Algorithm SHA256 -LiteralPath (Join-Path $sourceDirectory 'SHA256SUMS')).Hash.ToLowerInvariant()
|
||||
archive_sha256 = $archiveHash
|
||||
ciphertext_sha256 = (Get-FileHash -Algorithm SHA256 -LiteralPath $cipherPartial).Hash.ToLowerInvariant()
|
||||
encryption = 'GnuPG symmetric AES256, SHA512 S2K'
|
||||
created_at_utc = [DateTime]::UtcNow.ToString('o')
|
||||
}
|
||||
[IO.File]::WriteAllText($manifestPartial, ($manifest | ConvertTo-Json) + "`n", [Text.UTF8Encoding]::new($false))
|
||||
Move-Item -LiteralPath $cipherPartial -Destination $cipherPath
|
||||
Move-Item -LiteralPath $manifestPartial -Destination $manifestPath
|
||||
Write-Output "Off-host backup created: $latest"
|
||||
}
|
||||
finally {
|
||||
$passphrase = $null
|
||||
foreach ($partial in @($cipherPartial, $manifestPartial)) {
|
||||
if ($partial -and (Test-Path -LiteralPath $partial)) { Remove-Item -LiteralPath $partial -Force }
|
||||
}
|
||||
if (Test-Path -LiteralPath $work) { Remove-Item -LiteralPath $work -Recurse -Force }
|
||||
}
|
||||
@@ -0,0 +1,75 @@
|
||||
[CmdletBinding()]
|
||||
param(
|
||||
[string]$NasBackupRoot = '\\nas\MST\codex\_backups\domovoy\postgres',
|
||||
[string]$KeyRecordPath = "$env:LOCALAPPDATA\Domovoy\postgres-backup-key.dpapi.json",
|
||||
[string]$GpgPath = 'gpg.exe',
|
||||
[string]$SshHost = 'docker',
|
||||
[switch]$VerifyRemoteRestore
|
||||
)
|
||||
|
||||
$ErrorActionPreference = 'Stop'
|
||||
Set-StrictMode -Version Latest
|
||||
|
||||
function Get-RecoveryPassphrase([string]$Path) {
|
||||
$record = Get-Content -LiteralPath $Path -Raw | ConvertFrom-Json
|
||||
if ($record.version -ne 1 -or [string]::IsNullOrWhiteSpace($record.protected_value)) { throw 'Invalid DPAPI recovery key record' }
|
||||
$secure = ConvertTo-SecureString -String $record.protected_value
|
||||
$pointer = [Runtime.InteropServices.Marshal]::SecureStringToBSTR($secure)
|
||||
try { return [Runtime.InteropServices.Marshal]::PtrToStringBSTR($pointer) }
|
||||
finally { [Runtime.InteropServices.Marshal]::ZeroFreeBSTR($pointer) }
|
||||
}
|
||||
|
||||
$manifestFile = Get-ChildItem -LiteralPath $NasBackupRoot -Filter '*.json' -File | Sort-Object BaseName | Select-Object -Last 1
|
||||
if (-not $manifestFile -or $manifestFile.BaseName -notmatch '^\d{8}T\d{6}Z$') { throw 'No valid NAS backup manifest found' }
|
||||
$manifest = Get-Content -LiteralPath $manifestFile.FullName -Raw | ConvertFrom-Json
|
||||
$latest = $manifestFile.BaseName
|
||||
$cipherPath = Join-Path $NasBackupRoot "$latest.tar.gz.gpg"
|
||||
if ((Get-FileHash -Algorithm SHA256 -LiteralPath $cipherPath).Hash.ToLowerInvariant() -ne $manifest.ciphertext_sha256) { throw 'NAS ciphertext hash mismatch' }
|
||||
|
||||
$tempRoot = [IO.Path]::GetFullPath([IO.Path]::GetTempPath())
|
||||
$work = Join-Path $tempRoot ("domovoy-backup-test-" + [guid]::NewGuid().ToString('N'))
|
||||
New-Item -ItemType Directory -Path $work | Out-Null
|
||||
$passphrase = $null
|
||||
$remoteCheck = $null
|
||||
|
||||
try {
|
||||
& icacls.exe $work /inheritance:r /grant:r "$([Security.Principal.WindowsIdentity]::GetCurrent().Name):(OI)(CI)F" | Out-Null
|
||||
if ($LASTEXITCODE -ne 0) { throw 'Temporary ACL hardening failed' }
|
||||
$passphrase = Get-RecoveryPassphrase $KeyRecordPath
|
||||
$passphraseFile = Join-Path $work 'passphrase.txt'
|
||||
[IO.File]::WriteAllText($passphraseFile, $passphrase, [Text.UTF8Encoding]::new($false))
|
||||
$archive = Join-Path $work "$latest.tar.gz"
|
||||
& $GpgPath --batch --yes --pinentry-mode loopback --passphrase-file $passphraseFile --decrypt --output $archive $cipherPath
|
||||
if ($LASTEXITCODE -ne 0) { throw 'NAS backup decryption failed' }
|
||||
if ((Get-FileHash -Algorithm SHA256 -LiteralPath $archive).Hash.ToLowerInvariant() -ne $manifest.archive_sha256) { throw 'Decrypted archive hash mismatch' }
|
||||
& tar.exe -xzf $archive -C $work
|
||||
if ($LASTEXITCODE -ne 0) { throw 'Decrypted archive extraction failed' }
|
||||
$sourceDirectory = Join-Path $work $latest
|
||||
Push-Location $sourceDirectory
|
||||
try {
|
||||
foreach ($line in Get-Content -LiteralPath 'SHA256SUMS') {
|
||||
if ($line -notmatch '^([0-9a-f]{64}) ([A-Za-z0-9.]+)$') { throw 'Invalid decrypted SHA256SUMS' }
|
||||
if ((Get-FileHash -Algorithm SHA256 -LiteralPath $matches[2]).Hash.ToLowerInvariant() -ne $matches[1]) { throw "Decrypted file hash mismatch: $($matches[2])" }
|
||||
}
|
||||
}
|
||||
finally { Pop-Location }
|
||||
|
||||
if ($VerifyRemoteRestore) {
|
||||
$remoteCheck = "/home/m/assistant-backups/offhost-restore-check/$latest-$([guid]::NewGuid().ToString('N'))"
|
||||
if ($remoteCheck -notmatch '^/home/m/assistant-backups/offhost-restore-check/\d{8}T\d{6}Z-[0-9a-f]{32}$') { throw 'Unsafe remote restore path' }
|
||||
& ssh.exe $SshHost "mkdir -p '$remoteCheck'"
|
||||
if ($LASTEXITCODE -ne 0) { throw 'Remote restore-check directory creation failed' }
|
||||
& scp.exe -q -r "$sourceDirectory\*" "${SshHost}:${remoteCheck}/"
|
||||
if ($LASTEXITCODE -ne 0) { throw 'Remote restore-check copy failed' }
|
||||
& ssh.exe $SshHost "/home/m/assistant-ops/verify-postgres-restore.sh '$remoteCheck'"
|
||||
if ($LASTEXITCODE -ne 0) { throw 'Remote PostgreSQL restore check failed' }
|
||||
}
|
||||
Write-Output "Off-host backup verified: $latest"
|
||||
}
|
||||
finally {
|
||||
if ($remoteCheck -and $remoteCheck -match '^/home/m/assistant-backups/offhost-restore-check/\d{8}T\d{6}Z-[0-9a-f]{32}$') {
|
||||
& ssh.exe $SshHost "rm -rf -- '$remoteCheck'" | Out-Null
|
||||
}
|
||||
$passphrase = $null
|
||||
if (Test-Path -LiteralPath $work) { Remove-Item -LiteralPath $work -Recurse -Force }
|
||||
}
|
||||
Reference in New Issue
Block a user