179 lines
9.5 KiB
PowerShell
179 lines
9.5 KiB
PowerShell
[CmdletBinding()]
|
|
param(
|
|
[string]$DockerContext = 'docker-prod',
|
|
[string]$SshHost = 'docker',
|
|
[string]$Endpoint = 'https://assistant.cin.su/healthz',
|
|
[string]$RemoteBackupRoot = '/home/m/assistant-backups/postgres',
|
|
[string]$NasBackupRoot = '\\nas\MST\codex\_backups\domovoy\postgres',
|
|
[string]$NasStatusPath = '\\nas\MST\codex\_monitoring\domovoy\status.json',
|
|
[string]$LocalStatusPath = "$env:LOCALAPPDATA\Domovoy\monitoring\status.json",
|
|
[string]$DeploymentManifestPath = "$env:LOCALAPPDATA\Domovoy\monitoring\deployment-manifest.json",
|
|
[double]$MaxProductionBackupAgeHours = 30,
|
|
[double]$MaxNasBackupAgeHours = 36,
|
|
[int]$MinimumCertificateDays = 21,
|
|
[string[]]$ExpectedContainers = @(
|
|
'assistant-core-runtime-1',
|
|
'assistant-creator-web-1',
|
|
'assistant-commit-runtime-1',
|
|
'assistant-notification-receiver-1',
|
|
'assistant-continuity-runtime-1',
|
|
'assistant-cognitive-worker-1',
|
|
'assistant-postgres-1',
|
|
'assistant-nats-1',
|
|
'assistant-gitea-runner'
|
|
)
|
|
)
|
|
|
|
$ErrorActionPreference = 'Stop'
|
|
Set-StrictMode -Version Latest
|
|
$checks = [Collections.Generic.List[object]]::new()
|
|
|
|
function Add-Check([string]$Name, [bool]$Ok, [string]$Detail) {
|
|
$checks.Add([ordered]@{ name = $Name; ok = $Ok; detail = $Detail })
|
|
}
|
|
|
|
function Write-AtomicJson([string]$Path, [string]$Json) {
|
|
$directory = Split-Path -Parent $Path
|
|
New-Item -ItemType Directory -Force -Path $directory | Out-Null
|
|
$partial = Join-Path $directory ('.' + [IO.Path]::GetFileName($Path) + ".${PID}.partial")
|
|
try {
|
|
[IO.File]::WriteAllText($partial, $Json + "`n", [Text.UTF8Encoding]::new($false))
|
|
Move-Item -LiteralPath $partial -Destination $Path -Force
|
|
}
|
|
finally {
|
|
if (Test-Path -LiteralPath $partial) { Remove-Item -LiteralPath $partial -Force }
|
|
}
|
|
}
|
|
|
|
foreach ($container in $ExpectedContainers) {
|
|
try {
|
|
$raw = ((& docker.exe --context $DockerContext inspect $container --format '{{json .State}}') -join '').Trim()
|
|
if ($LASTEXITCODE -ne 0 -or [string]::IsNullOrWhiteSpace($raw)) { throw 'inspect failed' }
|
|
$state = $raw | ConvertFrom-Json
|
|
$ok = $state.Status -eq 'running'
|
|
$detail = "status=$($state.Status)"
|
|
if ($state.PSObject.Properties.Name -contains 'Health' -and $null -ne $state.Health) {
|
|
$ok = $ok -and $state.Health.Status -eq 'healthy'
|
|
$detail += " health=$($state.Health.Status)"
|
|
}
|
|
Add-Check "container:$container" $ok $detail
|
|
}
|
|
catch { Add-Check "container:$container" $false $_.Exception.Message }
|
|
}
|
|
|
|
try {
|
|
$manifest = Get-Content -LiteralPath $DeploymentManifestPath -Raw | ConvertFrom-Json
|
|
if ($manifest.version -ne 1 -or $manifest.revision -notmatch '^[0-9a-f]{40}$') { throw 'invalid deployment manifest' }
|
|
if (@($manifest.artifacts).Count -ne 7) { throw 'deployment manifest artifact count mismatch' }
|
|
foreach ($artifact in $manifest.artifacts) {
|
|
$raw = ((& docker.exe --context $DockerContext inspect $artifact.container --format '{{json .}}') -join '').Trim()
|
|
if ($LASTEXITCODE -ne 0 -or [string]::IsNullOrWhiteSpace($raw)) { throw "cannot inspect $($artifact.container)" }
|
|
$inspect = $raw | ConvertFrom-Json
|
|
$label = [string]$inspect.Config.Labels.'org.opencontainers.image.revision'
|
|
if ($label -ne $manifest.revision -or [string]$inspect.Image -ne [string]$artifact.image_id) { throw "artifact drift: $($artifact.container)" }
|
|
}
|
|
Add-Check 'deployment-artifact' $true "revision=$($manifest.revision) containers=$(@($manifest.artifacts).Count)"
|
|
}
|
|
catch { Add-Check 'deployment-artifact' $false $_.Exception.Message }
|
|
|
|
try {
|
|
$response = Invoke-RestMethod -Uri $Endpoint -TimeoutSec 15
|
|
$ok = $response.status -eq 'ok'
|
|
Add-Check 'https-health' $ok "status=$($response.status) runtime=$($response.runtime)"
|
|
}
|
|
catch { Add-Check 'https-health' $false $_.Exception.Message }
|
|
|
|
try {
|
|
$uri = [Uri]$Endpoint
|
|
$tcp = [Net.Sockets.TcpClient]::new()
|
|
try {
|
|
$tcp.Connect($uri.Host, 443)
|
|
$ssl = [Net.Security.SslStream]::new($tcp.GetStream(), $false)
|
|
try {
|
|
$ssl.AuthenticateAsClient($uri.Host)
|
|
$certificate = [Security.Cryptography.X509Certificates.X509Certificate2]::new($ssl.RemoteCertificate)
|
|
$days = [math]::Floor(($certificate.NotAfter.ToUniversalTime() - [DateTime]::UtcNow).TotalDays)
|
|
Add-Check 'tls-certificate' ($days -ge $MinimumCertificateDays) "expires=$($certificate.NotAfter.ToUniversalTime().ToString('o')) days=$days"
|
|
}
|
|
finally { $ssl.Dispose() }
|
|
}
|
|
finally { $tcp.Dispose() }
|
|
}
|
|
catch { Add-Check 'tls-certificate' $false $_.Exception.Message }
|
|
|
|
try {
|
|
$assistantAddresses = @(Resolve-DnsName assistant.cin.su -Server 8.8.8.8 -Type A -DnsOnly | Where-Object Type -eq A | Select-Object -ExpandProperty IPAddress | Sort-Object -Unique)
|
|
$dockerAddresses = @(Resolve-DnsName docker.cin.su -Server 8.8.8.8 -Type A -DnsOnly | Where-Object Type -eq A | Select-Object -ExpandProperty IPAddress | Sort-Object -Unique)
|
|
$same = $assistantAddresses.Count -gt 0 -and $dockerAddresses.Count -gt 0 -and @(Compare-Object $assistantAddresses $dockerAddresses).Count -eq 0
|
|
Add-Check 'public-dns' $same "assistant=$($assistantAddresses -join ',') docker=$($dockerAddresses -join ',')"
|
|
}
|
|
catch { Add-Check 'public-dns' $false $_.Exception.Message }
|
|
|
|
try {
|
|
$latestProduction = ((& ssh.exe $SshHost "find '$RemoteBackupRoot' -mindepth 1 -maxdepth 1 -type d ! -name '.*' -printf '%f\n' | grep -E '^[0-9]{8}T[0-9]{6}Z$' | sort | tail -n1") -join '').Trim()
|
|
if ($LASTEXITCODE -ne 0 -or $latestProduction -notmatch '^\d{8}T\d{6}Z$') { throw 'invalid latest production backup' }
|
|
$productionTime = [DateTime]::ParseExact($latestProduction, "yyyyMMdd'T'HHmmss'Z'", [Globalization.CultureInfo]::InvariantCulture, [Globalization.DateTimeStyles]::AssumeUniversal -bor [Globalization.DateTimeStyles]::AdjustToUniversal)
|
|
$age = ([DateTime]::UtcNow - $productionTime).TotalHours
|
|
$verify = (& ssh.exe $SshHost "cd '$RemoteBackupRoot/$latestProduction' && sha256sum -c SHA256SUMS") -join '; '
|
|
if ($LASTEXITCODE -ne 0) { throw 'production backup hash verification failed' }
|
|
Add-Check 'production-backup' ($age -le $MaxProductionBackupAgeHours) ("backup=$latestProduction age_hours={0:N2} hashes=ok" -f $age)
|
|
}
|
|
catch { Add-Check 'production-backup' $false $_.Exception.Message }
|
|
|
|
try {
|
|
$manifestFile = Get-ChildItem -LiteralPath $NasBackupRoot -Filter '*.json' -File | Sort-Object BaseName | Select-Object -Last 1
|
|
if (-not $manifestFile -or $manifestFile.BaseName -notmatch '^\d{8}T\d{6}Z$') { throw 'invalid latest NAS backup manifest' }
|
|
$manifest = Get-Content -LiteralPath $manifestFile.FullName -Raw | ConvertFrom-Json
|
|
$nasTime = [DateTime]::ParseExact($manifestFile.BaseName, "yyyyMMdd'T'HHmmss'Z'", [Globalization.CultureInfo]::InvariantCulture, [Globalization.DateTimeStyles]::AssumeUniversal -bor [Globalization.DateTimeStyles]::AdjustToUniversal)
|
|
$age = ([DateTime]::UtcNow - $nasTime).TotalHours
|
|
$cipher = Join-Path $NasBackupRoot "$($manifestFile.BaseName).tar.gz.gpg"
|
|
$hash = (Get-FileHash -Algorithm SHA256 -LiteralPath $cipher).Hash.ToLowerInvariant()
|
|
$ok = $hash -eq $manifest.ciphertext_sha256 -and $age -le $MaxNasBackupAgeHours
|
|
Add-Check 'nas-backup' $ok ("backup=$($manifestFile.BaseName) age_hours={0:N2} ciphertext_hash={1}" -f $age, $(if ($hash -eq $manifest.ciphertext_sha256) { 'ok' } else { 'mismatch' }))
|
|
}
|
|
catch { Add-Check 'nas-backup' $false $_.Exception.Message }
|
|
|
|
try {
|
|
$cron = ((& ssh.exe $SshHost "systemctl is-active cron && crontab -l | grep -F '/home/m/assistant-ops/backup-postgres.sh'") -join '; ').Trim()
|
|
Add-Check 'production-backup-cron' ($LASTEXITCODE -eq 0 -and $cron -match '^active') $cron
|
|
}
|
|
catch { Add-Check 'production-backup-cron' $false $_.Exception.Message }
|
|
|
|
try {
|
|
$backupTask = Get-ScheduledTask -TaskName 'Domovoy PostgreSQL Off-host Backup'
|
|
$backupInfo = Get-ScheduledTaskInfo -TaskName $backupTask.TaskName
|
|
$ok = $backupTask.State -ne 'Disabled' -and $backupInfo.LastTaskResult -eq 0
|
|
Add-Check 'offhost-backup-task' $ok "state=$($backupTask.State) last_result=$($backupInfo.LastTaskResult) next=$($backupInfo.NextRunTime.ToString('o'))"
|
|
}
|
|
catch { Add-Check 'offhost-backup-task' $false $_.Exception.Message }
|
|
|
|
try {
|
|
$notificationTask = Get-ScheduledTask -TaskName 'Domovoy Notification Receiver'
|
|
$notificationInfo = Get-ScheduledTaskInfo -TaskName $notificationTask.TaskName
|
|
$ok = $notificationTask.State -ne 'Disabled' -and $notificationInfo.LastTaskResult -eq 0
|
|
Add-Check 'notification-receiver-task' $ok "state=$($notificationTask.State) last_result=$($notificationInfo.LastTaskResult) next=$($notificationInfo.NextRunTime.ToString('o'))"
|
|
}
|
|
catch { Add-Check 'notification-receiver-task' $false $_.Exception.Message }
|
|
|
|
$overall = if (@($checks | Where-Object { -not $_.ok }).Count -eq 0) { 'healthy' } else { 'degraded' }
|
|
$record = [ordered]@{
|
|
version = 1
|
|
project = 'domovoy'
|
|
observed_at_utc = [DateTime]::UtcNow.ToString('o')
|
|
overall = $overall
|
|
checks = $checks
|
|
}
|
|
$json = $record | ConvertTo-Json -Depth 6
|
|
Write-AtomicJson $LocalStatusPath $json
|
|
try { Write-AtomicJson $NasStatusPath $json }
|
|
catch {
|
|
Add-Check 'nas-status-publication' $false $_.Exception.Message
|
|
$record.overall = 'degraded'
|
|
$record.checks = $checks
|
|
Write-AtomicJson $LocalStatusPath ($record | ConvertTo-Json -Depth 6)
|
|
$overall = 'degraded'
|
|
}
|
|
|
|
Write-Output "Domovoy production health: $overall"
|
|
if ($overall -ne 'healthy') { exit 1 }
|