[CmdletBinding()] param( [string]$DockerContext = 'docker-prod', [string]$SshHost = 'docker', [string]$Endpoint = 'https://assistant.cin.su/healthz', [string]$RemoteBackupRoot = '/home/m/assistant-backups/postgres', [string]$NasBackupRoot = '\\nas\MST\codex\_backups\domovoy\postgres', [string]$NasStatusPath = '\\nas\MST\codex\_monitoring\domovoy\status.json', [string]$LocalStatusPath = "$env:LOCALAPPDATA\Domovoy\monitoring\status.json", [string]$DeploymentManifestPath = "$env:LOCALAPPDATA\Domovoy\monitoring\deployment-manifest.json", [double]$MaxProductionBackupAgeHours = 30, [double]$MaxNasBackupAgeHours = 36, [int]$MinimumCertificateDays = 21, [string[]]$ExpectedContainers = @( 'assistant-core-runtime-1', 'assistant-creator-web-1', 'assistant-commit-runtime-1', 'assistant-notification-receiver-1', 'assistant-continuity-runtime-1', 'assistant-cognitive-worker-1', 'assistant-postgres-1', 'assistant-nats-1', 'assistant-gitea-runner' ) ) $ErrorActionPreference = 'Stop' Set-StrictMode -Version Latest $checks = [Collections.Generic.List[object]]::new() function Add-Check([string]$Name, [bool]$Ok, [string]$Detail) { $checks.Add([ordered]@{ name = $Name; ok = $Ok; detail = $Detail }) } function Write-AtomicJson([string]$Path, [string]$Json) { $directory = Split-Path -Parent $Path New-Item -ItemType Directory -Force -Path $directory | Out-Null $partial = Join-Path $directory ('.' + [IO.Path]::GetFileName($Path) + ".${PID}.partial") try { [IO.File]::WriteAllText($partial, $Json + "`n", [Text.UTF8Encoding]::new($false)) Move-Item -LiteralPath $partial -Destination $Path -Force } finally { if (Test-Path -LiteralPath $partial) { Remove-Item -LiteralPath $partial -Force } } } foreach ($container in $ExpectedContainers) { try { $raw = ((& docker.exe --context $DockerContext inspect $container --format '{{json .State}}') -join '').Trim() if ($LASTEXITCODE -ne 0 -or [string]::IsNullOrWhiteSpace($raw)) { throw 'inspect failed' } $state = $raw | ConvertFrom-Json $ok = $state.Status -eq 'running' $detail = "status=$($state.Status)" if ($state.PSObject.Properties.Name -contains 'Health' -and $null -ne $state.Health) { $ok = $ok -and $state.Health.Status -eq 'healthy' $detail += " health=$($state.Health.Status)" } Add-Check "container:$container" $ok $detail } catch { Add-Check "container:$container" $false $_.Exception.Message } } try { $manifest = Get-Content -LiteralPath $DeploymentManifestPath -Raw | ConvertFrom-Json if ($manifest.version -ne 1 -or $manifest.revision -notmatch '^[0-9a-f]{40}$') { throw 'invalid deployment manifest' } if (@($manifest.artifacts).Count -ne 7) { throw 'deployment manifest artifact count mismatch' } foreach ($artifact in $manifest.artifacts) { $raw = ((& docker.exe --context $DockerContext inspect $artifact.container --format '{{json .}}') -join '').Trim() if ($LASTEXITCODE -ne 0 -or [string]::IsNullOrWhiteSpace($raw)) { throw "cannot inspect $($artifact.container)" } $inspect = $raw | ConvertFrom-Json $label = [string]$inspect.Config.Labels.'org.opencontainers.image.revision' if ($label -ne $manifest.revision -or [string]$inspect.Image -ne [string]$artifact.image_id) { throw "artifact drift: $($artifact.container)" } } Add-Check 'deployment-artifact' $true "revision=$($manifest.revision) containers=$(@($manifest.artifacts).Count)" } catch { Add-Check 'deployment-artifact' $false $_.Exception.Message } try { $response = Invoke-RestMethod -Uri $Endpoint -TimeoutSec 15 $ok = $response.status -eq 'ok' Add-Check 'https-health' $ok "status=$($response.status) runtime=$($response.runtime)" } catch { Add-Check 'https-health' $false $_.Exception.Message } try { $uri = [Uri]$Endpoint $tcp = [Net.Sockets.TcpClient]::new() try { $tcp.Connect($uri.Host, 443) $ssl = [Net.Security.SslStream]::new($tcp.GetStream(), $false) try { $ssl.AuthenticateAsClient($uri.Host) $certificate = [Security.Cryptography.X509Certificates.X509Certificate2]::new($ssl.RemoteCertificate) $days = [math]::Floor(($certificate.NotAfter.ToUniversalTime() - [DateTime]::UtcNow).TotalDays) Add-Check 'tls-certificate' ($days -ge $MinimumCertificateDays) "expires=$($certificate.NotAfter.ToUniversalTime().ToString('o')) days=$days" } finally { $ssl.Dispose() } } finally { $tcp.Dispose() } } catch { Add-Check 'tls-certificate' $false $_.Exception.Message } try { $assistantAddresses = @(Resolve-DnsName assistant.cin.su -Server 8.8.8.8 -Type A -DnsOnly | Where-Object Type -eq A | Select-Object -ExpandProperty IPAddress | Sort-Object -Unique) $dockerAddresses = @(Resolve-DnsName docker.cin.su -Server 8.8.8.8 -Type A -DnsOnly | Where-Object Type -eq A | Select-Object -ExpandProperty IPAddress | Sort-Object -Unique) $same = $assistantAddresses.Count -gt 0 -and $dockerAddresses.Count -gt 0 -and @(Compare-Object $assistantAddresses $dockerAddresses).Count -eq 0 Add-Check 'public-dns' $same "assistant=$($assistantAddresses -join ',') docker=$($dockerAddresses -join ',')" } catch { Add-Check 'public-dns' $false $_.Exception.Message } try { $latestProduction = ((& ssh.exe $SshHost "find '$RemoteBackupRoot' -mindepth 1 -maxdepth 1 -type d ! -name '.*' -printf '%f\n' | grep -E '^[0-9]{8}T[0-9]{6}Z$' | sort | tail -n1") -join '').Trim() if ($LASTEXITCODE -ne 0 -or $latestProduction -notmatch '^\d{8}T\d{6}Z$') { throw 'invalid latest production backup' } $productionTime = [DateTime]::ParseExact($latestProduction, "yyyyMMdd'T'HHmmss'Z'", [Globalization.CultureInfo]::InvariantCulture, [Globalization.DateTimeStyles]::AssumeUniversal -bor [Globalization.DateTimeStyles]::AdjustToUniversal) $age = ([DateTime]::UtcNow - $productionTime).TotalHours $verify = (& ssh.exe $SshHost "cd '$RemoteBackupRoot/$latestProduction' && sha256sum -c SHA256SUMS") -join '; ' if ($LASTEXITCODE -ne 0) { throw 'production backup hash verification failed' } Add-Check 'production-backup' ($age -le $MaxProductionBackupAgeHours) ("backup=$latestProduction age_hours={0:N2} hashes=ok" -f $age) } catch { Add-Check 'production-backup' $false $_.Exception.Message } try { $manifestFile = Get-ChildItem -LiteralPath $NasBackupRoot -Filter '*.json' -File | Sort-Object BaseName | Select-Object -Last 1 if (-not $manifestFile -or $manifestFile.BaseName -notmatch '^\d{8}T\d{6}Z$') { throw 'invalid latest NAS backup manifest' } $manifest = Get-Content -LiteralPath $manifestFile.FullName -Raw | ConvertFrom-Json $nasTime = [DateTime]::ParseExact($manifestFile.BaseName, "yyyyMMdd'T'HHmmss'Z'", [Globalization.CultureInfo]::InvariantCulture, [Globalization.DateTimeStyles]::AssumeUniversal -bor [Globalization.DateTimeStyles]::AdjustToUniversal) $age = ([DateTime]::UtcNow - $nasTime).TotalHours $cipher = Join-Path $NasBackupRoot "$($manifestFile.BaseName).tar.gz.gpg" $hash = (Get-FileHash -Algorithm SHA256 -LiteralPath $cipher).Hash.ToLowerInvariant() $ok = $hash -eq $manifest.ciphertext_sha256 -and $age -le $MaxNasBackupAgeHours Add-Check 'nas-backup' $ok ("backup=$($manifestFile.BaseName) age_hours={0:N2} ciphertext_hash={1}" -f $age, $(if ($hash -eq $manifest.ciphertext_sha256) { 'ok' } else { 'mismatch' })) } catch { Add-Check 'nas-backup' $false $_.Exception.Message } try { $cron = ((& ssh.exe $SshHost "systemctl is-active cron && crontab -l | grep -F '/home/m/assistant-ops/backup-postgres.sh'") -join '; ').Trim() Add-Check 'production-backup-cron' ($LASTEXITCODE -eq 0 -and $cron -match '^active') $cron } catch { Add-Check 'production-backup-cron' $false $_.Exception.Message } try { $backupTask = Get-ScheduledTask -TaskName 'Domovoy PostgreSQL Off-host Backup' $backupInfo = Get-ScheduledTaskInfo -TaskName $backupTask.TaskName $ok = $backupTask.State -ne 'Disabled' -and $backupInfo.LastTaskResult -eq 0 Add-Check 'offhost-backup-task' $ok "state=$($backupTask.State) last_result=$($backupInfo.LastTaskResult) next=$($backupInfo.NextRunTime.ToString('o'))" } catch { Add-Check 'offhost-backup-task' $false $_.Exception.Message } try { $notificationTask = Get-ScheduledTask -TaskName 'Domovoy Notification Receiver' $notificationInfo = Get-ScheduledTaskInfo -TaskName $notificationTask.TaskName $ok = $notificationTask.State -ne 'Disabled' -and $notificationInfo.LastTaskResult -eq 0 Add-Check 'notification-receiver-task' $ok "state=$($notificationTask.State) last_result=$($notificationInfo.LastTaskResult) next=$($notificationInfo.NextRunTime.ToString('o'))" } catch { Add-Check 'notification-receiver-task' $false $_.Exception.Message } $overall = if (@($checks | Where-Object { -not $_.ok }).Count -eq 0) { 'healthy' } else { 'degraded' } $record = [ordered]@{ version = 1 project = 'domovoy' observed_at_utc = [DateTime]::UtcNow.ToString('o') overall = $overall checks = $checks } $json = $record | ConvertTo-Json -Depth 6 Write-AtomicJson $LocalStatusPath $json try { Write-AtomicJson $NasStatusPath $json } catch { Add-Check 'nas-status-publication' $false $_.Exception.Message $record.overall = 'degraded' $record.checks = $checks Write-AtomicJson $LocalStatusPath ($record | ConvertTo-Json -Depth 6) $overall = 'degraded' } Write-Output "Domovoy production health: $overall" if ($overall -ne 'healthy') { exit 1 }