ops: add Domovoy production health monitor
This commit was merged in pull request #3.
This commit is contained in:
@@ -0,0 +1,20 @@
|
||||
[CmdletBinding()]
|
||||
param(
|
||||
[string]$ScriptPath = '\\nas\MST\codex\assistent\deploy\windows\Test-DomovoyProductionHealth.ps1',
|
||||
[string]$TaskName = 'Domovoy Production Health Monitor'
|
||||
)
|
||||
|
||||
$ErrorActionPreference = 'Stop'
|
||||
Set-StrictMode -Version Latest
|
||||
|
||||
if (-not (Test-Path -LiteralPath $ScriptPath -PathType Leaf)) { throw 'Monitoring script not found' }
|
||||
$identity = [Security.Principal.WindowsIdentity]::GetCurrent().Name
|
||||
$pwsh = (Get-Command pwsh.exe -ErrorAction Stop).Source
|
||||
$arguments = "-NoProfile -NonInteractive -ExecutionPolicy Bypass -File `"$ScriptPath`""
|
||||
$action = New-ScheduledTaskAction -Execute $pwsh -Argument $arguments
|
||||
$repeat = New-ScheduledTaskTrigger -Once -At (Get-Date).AddMinutes(1) -RepetitionInterval (New-TimeSpan -Minutes 15) -RepetitionDuration (New-TimeSpan -Days 3650)
|
||||
$logon = New-ScheduledTaskTrigger -AtLogOn -User $identity
|
||||
$principal = New-ScheduledTaskPrincipal -UserId $identity -LogonType Interactive -RunLevel Limited
|
||||
$settings = New-ScheduledTaskSettingsSet -StartWhenAvailable -MultipleInstances IgnoreNew -ExecutionTimeLimit (New-TimeSpan -Minutes 10)
|
||||
Register-ScheduledTask -TaskName $TaskName -Action $action -Trigger @($repeat, $logon) -Principal $principal -Settings $settings -Description 'Monitor Domovoy production health, TLS, DNS, backups, and schedules.' -Force | Out-Null
|
||||
Write-Output "Scheduled task registered: $TaskName"
|
||||
@@ -0,0 +1,153 @@
|
||||
[CmdletBinding()]
|
||||
param(
|
||||
[string]$DockerContext = 'docker-prod',
|
||||
[string]$SshHost = 'docker',
|
||||
[string]$Endpoint = 'https://assistent.cin.su/healthz',
|
||||
[string]$RemoteBackupRoot = '/home/m/assistant-backups/postgres',
|
||||
[string]$NasBackupRoot = '\\nas\MST\codex\_backups\domovoy\postgres',
|
||||
[string]$NasStatusPath = '\\nas\MST\codex\_monitoring\domovoy\status.json',
|
||||
[string]$LocalStatusPath = "$env:LOCALAPPDATA\Domovoy\monitoring\status.json",
|
||||
[double]$MaxProductionBackupAgeHours = 30,
|
||||
[double]$MaxNasBackupAgeHours = 36,
|
||||
[int]$MinimumCertificateDays = 21,
|
||||
[string[]]$ExpectedContainers = @(
|
||||
'assistant-core-runtime-1',
|
||||
'assistant-creator-web-1',
|
||||
'assistant-commit-runtime-1',
|
||||
'assistant-continuity-runtime-1',
|
||||
'assistant-cognitive-worker-1',
|
||||
'assistant-postgres-1',
|
||||
'assistant-nats-1',
|
||||
'assistant-gitea-runner'
|
||||
)
|
||||
)
|
||||
|
||||
$ErrorActionPreference = 'Stop'
|
||||
Set-StrictMode -Version Latest
|
||||
$checks = [Collections.Generic.List[object]]::new()
|
||||
|
||||
function Add-Check([string]$Name, [bool]$Ok, [string]$Detail) {
|
||||
$checks.Add([ordered]@{ name = $Name; ok = $Ok; detail = $Detail })
|
||||
}
|
||||
|
||||
function Write-AtomicJson([string]$Path, [string]$Json) {
|
||||
$directory = Split-Path -Parent $Path
|
||||
New-Item -ItemType Directory -Force -Path $directory | Out-Null
|
||||
$partial = Join-Path $directory ('.' + [IO.Path]::GetFileName($Path) + ".${PID}.partial")
|
||||
try {
|
||||
[IO.File]::WriteAllText($partial, $Json + "`n", [Text.UTF8Encoding]::new($false))
|
||||
Move-Item -LiteralPath $partial -Destination $Path -Force
|
||||
}
|
||||
finally {
|
||||
if (Test-Path -LiteralPath $partial) { Remove-Item -LiteralPath $partial -Force }
|
||||
}
|
||||
}
|
||||
|
||||
foreach ($container in $ExpectedContainers) {
|
||||
try {
|
||||
$raw = ((& docker.exe --context $DockerContext inspect $container --format '{{json .State}}') -join '').Trim()
|
||||
if ($LASTEXITCODE -ne 0 -or [string]::IsNullOrWhiteSpace($raw)) { throw 'inspect failed' }
|
||||
$state = $raw | ConvertFrom-Json
|
||||
$ok = $state.Status -eq 'running'
|
||||
$detail = "status=$($state.Status)"
|
||||
if ($state.PSObject.Properties.Name -contains 'Health' -and $null -ne $state.Health) {
|
||||
$ok = $ok -and $state.Health.Status -eq 'healthy'
|
||||
$detail += " health=$($state.Health.Status)"
|
||||
}
|
||||
Add-Check "container:$container" $ok $detail
|
||||
}
|
||||
catch { Add-Check "container:$container" $false $_.Exception.Message }
|
||||
}
|
||||
|
||||
try {
|
||||
$response = Invoke-RestMethod -Uri $Endpoint -TimeoutSec 15
|
||||
$ok = $response.status -eq 'ok'
|
||||
Add-Check 'https-health' $ok "status=$($response.status) runtime=$($response.runtime)"
|
||||
}
|
||||
catch { Add-Check 'https-health' $false $_.Exception.Message }
|
||||
|
||||
try {
|
||||
$uri = [Uri]$Endpoint
|
||||
$tcp = [Net.Sockets.TcpClient]::new()
|
||||
try {
|
||||
$tcp.Connect($uri.Host, 443)
|
||||
$ssl = [Net.Security.SslStream]::new($tcp.GetStream(), $false)
|
||||
try {
|
||||
$ssl.AuthenticateAsClient($uri.Host)
|
||||
$certificate = [Security.Cryptography.X509Certificates.X509Certificate2]::new($ssl.RemoteCertificate)
|
||||
$days = [math]::Floor(($certificate.NotAfter.ToUniversalTime() - [DateTime]::UtcNow).TotalDays)
|
||||
Add-Check 'tls-certificate' ($days -ge $MinimumCertificateDays) "expires=$($certificate.NotAfter.ToUniversalTime().ToString('o')) days=$days"
|
||||
}
|
||||
finally { $ssl.Dispose() }
|
||||
}
|
||||
finally { $tcp.Dispose() }
|
||||
}
|
||||
catch { Add-Check 'tls-certificate' $false $_.Exception.Message }
|
||||
|
||||
try {
|
||||
$assistantAddresses = @(Resolve-DnsName assistent.cin.su -Server 8.8.8.8 -Type A -DnsOnly | Where-Object Type -eq A | Select-Object -ExpandProperty IPAddress | Sort-Object -Unique)
|
||||
$dockerAddresses = @(Resolve-DnsName docker.cin.su -Server 8.8.8.8 -Type A -DnsOnly | Where-Object Type -eq A | Select-Object -ExpandProperty IPAddress | Sort-Object -Unique)
|
||||
$same = $assistantAddresses.Count -gt 0 -and $dockerAddresses.Count -gt 0 -and @(Compare-Object $assistantAddresses $dockerAddresses).Count -eq 0
|
||||
Add-Check 'public-dns' $same "assistent=$($assistantAddresses -join ',') docker=$($dockerAddresses -join ',')"
|
||||
}
|
||||
catch { Add-Check 'public-dns' $false $_.Exception.Message }
|
||||
|
||||
try {
|
||||
$latestProduction = ((& ssh.exe $SshHost "find '$RemoteBackupRoot' -mindepth 1 -maxdepth 1 -type d ! -name '.*' -printf '%f\n' | sort | tail -n1") -join '').Trim()
|
||||
if ($LASTEXITCODE -ne 0 -or $latestProduction -notmatch '^\d{8}T\d{6}Z$') { throw 'invalid latest production backup' }
|
||||
$productionTime = [DateTime]::ParseExact($latestProduction, "yyyyMMdd'T'HHmmss'Z'", [Globalization.CultureInfo]::InvariantCulture, [Globalization.DateTimeStyles]::AssumeUniversal -bor [Globalization.DateTimeStyles]::AdjustToUniversal)
|
||||
$age = ([DateTime]::UtcNow - $productionTime).TotalHours
|
||||
$verify = (& ssh.exe $SshHost "cd '$RemoteBackupRoot/$latestProduction' && sha256sum -c SHA256SUMS") -join '; '
|
||||
if ($LASTEXITCODE -ne 0) { throw 'production backup hash verification failed' }
|
||||
Add-Check 'production-backup' ($age -le $MaxProductionBackupAgeHours) ("backup=$latestProduction age_hours={0:N2} hashes=ok" -f $age)
|
||||
}
|
||||
catch { Add-Check 'production-backup' $false $_.Exception.Message }
|
||||
|
||||
try {
|
||||
$manifestFile = Get-ChildItem -LiteralPath $NasBackupRoot -Filter '*.json' -File | Sort-Object BaseName | Select-Object -Last 1
|
||||
if (-not $manifestFile -or $manifestFile.BaseName -notmatch '^\d{8}T\d{6}Z$') { throw 'invalid latest NAS backup manifest' }
|
||||
$manifest = Get-Content -LiteralPath $manifestFile.FullName -Raw | ConvertFrom-Json
|
||||
$nasTime = [DateTime]::ParseExact($manifestFile.BaseName, "yyyyMMdd'T'HHmmss'Z'", [Globalization.CultureInfo]::InvariantCulture, [Globalization.DateTimeStyles]::AssumeUniversal -bor [Globalization.DateTimeStyles]::AdjustToUniversal)
|
||||
$age = ([DateTime]::UtcNow - $nasTime).TotalHours
|
||||
$cipher = Join-Path $NasBackupRoot "$($manifestFile.BaseName).tar.gz.gpg"
|
||||
$hash = (Get-FileHash -Algorithm SHA256 -LiteralPath $cipher).Hash.ToLowerInvariant()
|
||||
$ok = $hash -eq $manifest.ciphertext_sha256 -and $age -le $MaxNasBackupAgeHours
|
||||
Add-Check 'nas-backup' $ok ("backup=$($manifestFile.BaseName) age_hours={0:N2} ciphertext_hash={1}" -f $age, $(if ($hash -eq $manifest.ciphertext_sha256) { 'ok' } else { 'mismatch' }))
|
||||
}
|
||||
catch { Add-Check 'nas-backup' $false $_.Exception.Message }
|
||||
|
||||
try {
|
||||
$cron = ((& ssh.exe $SshHost "systemctl is-active cron && crontab -l | grep -F '/home/m/assistant-ops/backup-postgres.sh'") -join '; ').Trim()
|
||||
Add-Check 'production-backup-cron' ($LASTEXITCODE -eq 0 -and $cron -match '^active') $cron
|
||||
}
|
||||
catch { Add-Check 'production-backup-cron' $false $_.Exception.Message }
|
||||
|
||||
try {
|
||||
$backupTask = Get-ScheduledTask -TaskName 'Domovoy PostgreSQL Off-host Backup'
|
||||
$backupInfo = Get-ScheduledTaskInfo -TaskName $backupTask.TaskName
|
||||
$ok = $backupTask.State -ne 'Disabled' -and $backupInfo.LastTaskResult -eq 0
|
||||
Add-Check 'offhost-backup-task' $ok "state=$($backupTask.State) last_result=$($backupInfo.LastTaskResult) next=$($backupInfo.NextRunTime.ToString('o'))"
|
||||
}
|
||||
catch { Add-Check 'offhost-backup-task' $false $_.Exception.Message }
|
||||
|
||||
$overall = if (@($checks | Where-Object { -not $_.ok }).Count -eq 0) { 'healthy' } else { 'degraded' }
|
||||
$record = [ordered]@{
|
||||
version = 1
|
||||
project = 'domovoy'
|
||||
observed_at_utc = [DateTime]::UtcNow.ToString('o')
|
||||
overall = $overall
|
||||
checks = $checks
|
||||
}
|
||||
$json = $record | ConvertTo-Json -Depth 6
|
||||
Write-AtomicJson $LocalStatusPath $json
|
||||
try { Write-AtomicJson $NasStatusPath $json }
|
||||
catch {
|
||||
Add-Check 'nas-status-publication' $false $_.Exception.Message
|
||||
$record.overall = 'degraded'
|
||||
$record.checks = $checks
|
||||
Write-AtomicJson $LocalStatusPath ($record | ConvertTo-Json -Depth 6)
|
||||
$overall = 'degraded'
|
||||
}
|
||||
|
||||
Write-Output "Domovoy production health: $overall"
|
||||
if ($overall -ne 'healthy') { exit 1 }
|
||||
Reference in New Issue
Block a user