Promote production health monitoring #3
@@ -0,0 +1,20 @@
|
||||
[CmdletBinding()]
|
||||
param(
|
||||
[string]$ScriptPath = '\\nas\MST\codex\assistent\deploy\windows\Test-DomovoyProductionHealth.ps1',
|
||||
[string]$TaskName = 'Domovoy Production Health Monitor'
|
||||
)
|
||||
|
||||
$ErrorActionPreference = 'Stop'
|
||||
Set-StrictMode -Version Latest
|
||||
|
||||
if (-not (Test-Path -LiteralPath $ScriptPath -PathType Leaf)) { throw 'Monitoring script not found' }
|
||||
$identity = [Security.Principal.WindowsIdentity]::GetCurrent().Name
|
||||
$pwsh = (Get-Command pwsh.exe -ErrorAction Stop).Source
|
||||
$arguments = "-NoProfile -NonInteractive -ExecutionPolicy Bypass -File `"$ScriptPath`""
|
||||
$action = New-ScheduledTaskAction -Execute $pwsh -Argument $arguments
|
||||
$repeat = New-ScheduledTaskTrigger -Once -At (Get-Date).AddMinutes(1) -RepetitionInterval (New-TimeSpan -Minutes 15) -RepetitionDuration (New-TimeSpan -Days 3650)
|
||||
$logon = New-ScheduledTaskTrigger -AtLogOn -User $identity
|
||||
$principal = New-ScheduledTaskPrincipal -UserId $identity -LogonType Interactive -RunLevel Limited
|
||||
$settings = New-ScheduledTaskSettingsSet -StartWhenAvailable -MultipleInstances IgnoreNew -ExecutionTimeLimit (New-TimeSpan -Minutes 10)
|
||||
Register-ScheduledTask -TaskName $TaskName -Action $action -Trigger @($repeat, $logon) -Principal $principal -Settings $settings -Description 'Monitor Domovoy production health, TLS, DNS, backups, and schedules.' -Force | Out-Null
|
||||
Write-Output "Scheduled task registered: $TaskName"
|
||||
@@ -0,0 +1,153 @@
|
||||
[CmdletBinding()]
|
||||
param(
|
||||
[string]$DockerContext = 'docker-prod',
|
||||
[string]$SshHost = 'docker',
|
||||
[string]$Endpoint = 'https://assistent.cin.su/healthz',
|
||||
[string]$RemoteBackupRoot = '/home/m/assistant-backups/postgres',
|
||||
[string]$NasBackupRoot = '\\nas\MST\codex\_backups\domovoy\postgres',
|
||||
[string]$NasStatusPath = '\\nas\MST\codex\_monitoring\domovoy\status.json',
|
||||
[string]$LocalStatusPath = "$env:LOCALAPPDATA\Domovoy\monitoring\status.json",
|
||||
[double]$MaxProductionBackupAgeHours = 30,
|
||||
[double]$MaxNasBackupAgeHours = 36,
|
||||
[int]$MinimumCertificateDays = 21,
|
||||
[string[]]$ExpectedContainers = @(
|
||||
'assistant-core-runtime-1',
|
||||
'assistant-creator-web-1',
|
||||
'assistant-commit-runtime-1',
|
||||
'assistant-continuity-runtime-1',
|
||||
'assistant-cognitive-worker-1',
|
||||
'assistant-postgres-1',
|
||||
'assistant-nats-1',
|
||||
'assistant-gitea-runner'
|
||||
)
|
||||
)
|
||||
|
||||
$ErrorActionPreference = 'Stop'
|
||||
Set-StrictMode -Version Latest
|
||||
$checks = [Collections.Generic.List[object]]::new()
|
||||
|
||||
function Add-Check([string]$Name, [bool]$Ok, [string]$Detail) {
|
||||
$checks.Add([ordered]@{ name = $Name; ok = $Ok; detail = $Detail })
|
||||
}
|
||||
|
||||
function Write-AtomicJson([string]$Path, [string]$Json) {
|
||||
$directory = Split-Path -Parent $Path
|
||||
New-Item -ItemType Directory -Force -Path $directory | Out-Null
|
||||
$partial = Join-Path $directory ('.' + [IO.Path]::GetFileName($Path) + ".${PID}.partial")
|
||||
try {
|
||||
[IO.File]::WriteAllText($partial, $Json + "`n", [Text.UTF8Encoding]::new($false))
|
||||
Move-Item -LiteralPath $partial -Destination $Path -Force
|
||||
}
|
||||
finally {
|
||||
if (Test-Path -LiteralPath $partial) { Remove-Item -LiteralPath $partial -Force }
|
||||
}
|
||||
}
|
||||
|
||||
foreach ($container in $ExpectedContainers) {
|
||||
try {
|
||||
$raw = ((& docker.exe --context $DockerContext inspect $container --format '{{json .State}}') -join '').Trim()
|
||||
if ($LASTEXITCODE -ne 0 -or [string]::IsNullOrWhiteSpace($raw)) { throw 'inspect failed' }
|
||||
$state = $raw | ConvertFrom-Json
|
||||
$ok = $state.Status -eq 'running'
|
||||
$detail = "status=$($state.Status)"
|
||||
if ($state.PSObject.Properties.Name -contains 'Health' -and $null -ne $state.Health) {
|
||||
$ok = $ok -and $state.Health.Status -eq 'healthy'
|
||||
$detail += " health=$($state.Health.Status)"
|
||||
}
|
||||
Add-Check "container:$container" $ok $detail
|
||||
}
|
||||
catch { Add-Check "container:$container" $false $_.Exception.Message }
|
||||
}
|
||||
|
||||
try {
|
||||
$response = Invoke-RestMethod -Uri $Endpoint -TimeoutSec 15
|
||||
$ok = $response.status -eq 'ok'
|
||||
Add-Check 'https-health' $ok "status=$($response.status) runtime=$($response.runtime)"
|
||||
}
|
||||
catch { Add-Check 'https-health' $false $_.Exception.Message }
|
||||
|
||||
try {
|
||||
$uri = [Uri]$Endpoint
|
||||
$tcp = [Net.Sockets.TcpClient]::new()
|
||||
try {
|
||||
$tcp.Connect($uri.Host, 443)
|
||||
$ssl = [Net.Security.SslStream]::new($tcp.GetStream(), $false)
|
||||
try {
|
||||
$ssl.AuthenticateAsClient($uri.Host)
|
||||
$certificate = [Security.Cryptography.X509Certificates.X509Certificate2]::new($ssl.RemoteCertificate)
|
||||
$days = [math]::Floor(($certificate.NotAfter.ToUniversalTime() - [DateTime]::UtcNow).TotalDays)
|
||||
Add-Check 'tls-certificate' ($days -ge $MinimumCertificateDays) "expires=$($certificate.NotAfter.ToUniversalTime().ToString('o')) days=$days"
|
||||
}
|
||||
finally { $ssl.Dispose() }
|
||||
}
|
||||
finally { $tcp.Dispose() }
|
||||
}
|
||||
catch { Add-Check 'tls-certificate' $false $_.Exception.Message }
|
||||
|
||||
try {
|
||||
$assistantAddresses = @(Resolve-DnsName assistent.cin.su -Server 8.8.8.8 -Type A -DnsOnly | Where-Object Type -eq A | Select-Object -ExpandProperty IPAddress | Sort-Object -Unique)
|
||||
$dockerAddresses = @(Resolve-DnsName docker.cin.su -Server 8.8.8.8 -Type A -DnsOnly | Where-Object Type -eq A | Select-Object -ExpandProperty IPAddress | Sort-Object -Unique)
|
||||
$same = $assistantAddresses.Count -gt 0 -and $dockerAddresses.Count -gt 0 -and @(Compare-Object $assistantAddresses $dockerAddresses).Count -eq 0
|
||||
Add-Check 'public-dns' $same "assistent=$($assistantAddresses -join ',') docker=$($dockerAddresses -join ',')"
|
||||
}
|
||||
catch { Add-Check 'public-dns' $false $_.Exception.Message }
|
||||
|
||||
try {
|
||||
$latestProduction = ((& ssh.exe $SshHost "find '$RemoteBackupRoot' -mindepth 1 -maxdepth 1 -type d ! -name '.*' -printf '%f\n' | sort | tail -n1") -join '').Trim()
|
||||
if ($LASTEXITCODE -ne 0 -or $latestProduction -notmatch '^\d{8}T\d{6}Z$') { throw 'invalid latest production backup' }
|
||||
$productionTime = [DateTime]::ParseExact($latestProduction, "yyyyMMdd'T'HHmmss'Z'", [Globalization.CultureInfo]::InvariantCulture, [Globalization.DateTimeStyles]::AssumeUniversal -bor [Globalization.DateTimeStyles]::AdjustToUniversal)
|
||||
$age = ([DateTime]::UtcNow - $productionTime).TotalHours
|
||||
$verify = (& ssh.exe $SshHost "cd '$RemoteBackupRoot/$latestProduction' && sha256sum -c SHA256SUMS") -join '; '
|
||||
if ($LASTEXITCODE -ne 0) { throw 'production backup hash verification failed' }
|
||||
Add-Check 'production-backup' ($age -le $MaxProductionBackupAgeHours) ("backup=$latestProduction age_hours={0:N2} hashes=ok" -f $age)
|
||||
}
|
||||
catch { Add-Check 'production-backup' $false $_.Exception.Message }
|
||||
|
||||
try {
|
||||
$manifestFile = Get-ChildItem -LiteralPath $NasBackupRoot -Filter '*.json' -File | Sort-Object BaseName | Select-Object -Last 1
|
||||
if (-not $manifestFile -or $manifestFile.BaseName -notmatch '^\d{8}T\d{6}Z$') { throw 'invalid latest NAS backup manifest' }
|
||||
$manifest = Get-Content -LiteralPath $manifestFile.FullName -Raw | ConvertFrom-Json
|
||||
$nasTime = [DateTime]::ParseExact($manifestFile.BaseName, "yyyyMMdd'T'HHmmss'Z'", [Globalization.CultureInfo]::InvariantCulture, [Globalization.DateTimeStyles]::AssumeUniversal -bor [Globalization.DateTimeStyles]::AdjustToUniversal)
|
||||
$age = ([DateTime]::UtcNow - $nasTime).TotalHours
|
||||
$cipher = Join-Path $NasBackupRoot "$($manifestFile.BaseName).tar.gz.gpg"
|
||||
$hash = (Get-FileHash -Algorithm SHA256 -LiteralPath $cipher).Hash.ToLowerInvariant()
|
||||
$ok = $hash -eq $manifest.ciphertext_sha256 -and $age -le $MaxNasBackupAgeHours
|
||||
Add-Check 'nas-backup' $ok ("backup=$($manifestFile.BaseName) age_hours={0:N2} ciphertext_hash={1}" -f $age, $(if ($hash -eq $manifest.ciphertext_sha256) { 'ok' } else { 'mismatch' }))
|
||||
}
|
||||
catch { Add-Check 'nas-backup' $false $_.Exception.Message }
|
||||
|
||||
try {
|
||||
$cron = ((& ssh.exe $SshHost "systemctl is-active cron && crontab -l | grep -F '/home/m/assistant-ops/backup-postgres.sh'") -join '; ').Trim()
|
||||
Add-Check 'production-backup-cron' ($LASTEXITCODE -eq 0 -and $cron -match '^active') $cron
|
||||
}
|
||||
catch { Add-Check 'production-backup-cron' $false $_.Exception.Message }
|
||||
|
||||
try {
|
||||
$backupTask = Get-ScheduledTask -TaskName 'Domovoy PostgreSQL Off-host Backup'
|
||||
$backupInfo = Get-ScheduledTaskInfo -TaskName $backupTask.TaskName
|
||||
$ok = $backupTask.State -ne 'Disabled' -and $backupInfo.LastTaskResult -eq 0
|
||||
Add-Check 'offhost-backup-task' $ok "state=$($backupTask.State) last_result=$($backupInfo.LastTaskResult) next=$($backupInfo.NextRunTime.ToString('o'))"
|
||||
}
|
||||
catch { Add-Check 'offhost-backup-task' $false $_.Exception.Message }
|
||||
|
||||
$overall = if (@($checks | Where-Object { -not $_.ok }).Count -eq 0) { 'healthy' } else { 'degraded' }
|
||||
$record = [ordered]@{
|
||||
version = 1
|
||||
project = 'domovoy'
|
||||
observed_at_utc = [DateTime]::UtcNow.ToString('o')
|
||||
overall = $overall
|
||||
checks = $checks
|
||||
}
|
||||
$json = $record | ConvertTo-Json -Depth 6
|
||||
Write-AtomicJson $LocalStatusPath $json
|
||||
try { Write-AtomicJson $NasStatusPath $json }
|
||||
catch {
|
||||
Add-Check 'nas-status-publication' $false $_.Exception.Message
|
||||
$record.overall = 'degraded'
|
||||
$record.checks = $checks
|
||||
Write-AtomicJson $LocalStatusPath ($record | ConvertTo-Json -Depth 6)
|
||||
$overall = 'degraded'
|
||||
}
|
||||
|
||||
Write-Output "Domovoy production health: $overall"
|
||||
if ($overall -ne 'healthy') { exit 1 }
|
||||
@@ -1,5 +1,5 @@
|
||||
{
|
||||
"task_file": "dev/tasks/DEV-000037-encrypted-offhost-backup.json",
|
||||
"task_id": "DEV-000037",
|
||||
"task_file": "dev/tasks/DEV-000038-production-health-monitor.json",
|
||||
"task_id": "DEV-000038",
|
||||
"task_revision": 1
|
||||
}
|
||||
|
||||
@@ -0,0 +1,81 @@
|
||||
{
|
||||
"task_id": "DEV-000038",
|
||||
"task_revision": 1,
|
||||
"architecture_revision": "ADR-0004",
|
||||
"base_repo_revision": "0a188ddafe24dc2b31271f0d47ac31baed1d237f",
|
||||
"risk_tier": "E",
|
||||
"intent": "Add an external production health monitor for Domovoy containers, HTTPS certificate, public DNS, verified backups, and scheduled jobs without creating canonical Monitor semantics or using docker-test.",
|
||||
"requirement_ids": ["CONSTITUTION-1.3.11", "CONSTITUTION-1.3.31", "USER-DOCKER-PROD-ONLY"],
|
||||
"allowed_change_scope": {
|
||||
"modules": [],
|
||||
"paths": [
|
||||
"deploy/windows/Test-DomovoyProductionHealth.ps1",
|
||||
"deploy/windows/Register-DomovoyProductionHealthTask.ps1",
|
||||
"docs/DEPLOYMENT.md",
|
||||
"dev/ACTIVE_TASK.json",
|
||||
"dev/tasks/DEV-000038-production-health-monitor.json"
|
||||
]
|
||||
},
|
||||
"forbidden_change_scope": {
|
||||
"paths": [
|
||||
"architecture/**",
|
||||
"ci/**",
|
||||
".gitea/**",
|
||||
"cmd/**",
|
||||
"contracts/**",
|
||||
"modules/**",
|
||||
"database/**",
|
||||
"security/**",
|
||||
"deploy/docker-compose*.yml"
|
||||
]
|
||||
},
|
||||
"allowed_architecture_delta": {
|
||||
"new_cross_module_dependencies": 0,
|
||||
"new_canonical_owners": 0,
|
||||
"new_privileges": 0,
|
||||
"new_third_party_dependencies": 0
|
||||
},
|
||||
"contracts": {
|
||||
"consume": ["Docker read-only inspection", "HTTPS/TLS endpoint", "public DNS", "DEV-000035 backup manifest", "DEV-000037 NAS manifest", "Windows Scheduled Tasks"],
|
||||
"may_change": []
|
||||
},
|
||||
"semantic_owner": "none; external rebuildable operational projection",
|
||||
"canonical_state": ["none; status JSON is non-canonical and may be regenerated"],
|
||||
"state_transitions": ["observed checks -> healthy or degraded operational snapshot"],
|
||||
"side_effects_and_privileges": ["read-only inspection of production Docker and backup state", "atomic write of local and NAS status JSON", "registration of a current-user scheduled task"],
|
||||
"persistence_and_migrations": ["no database migration", "latest status JSON only; no hidden retention deletion"],
|
||||
"failure_retry_restart": ["checks are independent and record explicit failure details", "the monitor exits nonzero when any check fails", "local status remains available if NAS publication fails", "scheduled execution retries every 15 minutes and at logon"],
|
||||
"assumptions": [
|
||||
"The authorized Windows user retains Docker context docker-prod, SSH alias docker, public DNS, and NAS access",
|
||||
"Production backup should be younger than 30 hours and off-host backup younger than 36 hours",
|
||||
"TLS certificate should retain at least 21 days",
|
||||
"docker-test remains prohibited"
|
||||
],
|
||||
"acceptance_scenarios": [
|
||||
"all expected Assistant containers are running and health-enabled containers are healthy",
|
||||
"HTTPS returns healthy and the certificate has at least 21 remaining days",
|
||||
"public assistent.cin.su and docker.cin.su A records agree",
|
||||
"production and NAS backups are recent and their hashes validate",
|
||||
"backup scheduled tasks and production cron report success",
|
||||
"status JSON is atomically published locally and to NAS",
|
||||
"a failed check produces overall degraded and a nonzero exit code",
|
||||
"monitor runs every 15 minutes and at logon",
|
||||
"exact diff validation observes only declared paths"
|
||||
],
|
||||
"required_tests": [
|
||||
"PowerShell parser",
|
||||
"architecture preflight",
|
||||
"exact diff scope",
|
||||
"healthy live monitoring run",
|
||||
"synthetic degraded run",
|
||||
"atomic local and NAS status validation",
|
||||
"scheduled task registration and manual invocation",
|
||||
"go test",
|
||||
"go vet",
|
||||
"CI"
|
||||
],
|
||||
"terminal_gap_states": [
|
||||
"BLOCKED_BY_MISSING_CAPABILITY",
|
||||
"BLOCKED_BY_CONFLICT"
|
||||
]
|
||||
}
|
||||
@@ -33,3 +33,9 @@ The production user crontab installs the single entry from `deploy/cron/assistan
|
||||
`deploy/windows/Sync-PostgresBackupToNas.ps1` copies only a completed, hash-verified production backup over SSH, archives it locally, encrypts it with GnuPG AES-256, verifies an immediate decrypt round trip, and atomically publishes ciphertext plus a non-secret manifest under `\\nas\MST\codex\_backups\domovoy\postgres`. Temporary plaintext is removed on exit. The script is idempotent and never deletes existing backups.
|
||||
|
||||
`deploy/windows/Test-PostgresBackupFromNas.ps1 -VerifyRemoteRestore` verifies the NAS ciphertext hash, decrypts it with the DPAPI-protected recovery key, checks every source hash, and performs a full isolated PostgreSQL restore on `docker.cin.su`. `deploy/windows/Register-PostgresBackupNasTask.ps1` generates the key when absent, atomically backs up only its DPAPI ciphertext to the NAS, and registers current-user daily and logon triggers. The task intentionally does not use `docker-test`.
|
||||
|
||||
## Production health monitoring
|
||||
|
||||
`deploy/windows/Test-DomovoyProductionHealth.ps1` is an external, rebuildable operational projection; it is not a canonical Core `Monitor`. It checks all expected production containers, HTTPS health, TLS expiry, public DNS agreement, production backup age and hashes, NAS backup age and ciphertext hash, cron, and the off-host backup task. Thresholds are 30 hours for the production backup, 36 hours for the NAS copy, and 21 remaining certificate days.
|
||||
|
||||
The script atomically writes the latest JSON snapshot to `%LOCALAPPDATA%\Domovoy\monitoring\status.json` and `\\nas\MST\codex\_monitoring\domovoy\status.json`, then exits nonzero when any check is degraded. `deploy/windows/Register-DomovoyProductionHealthTask.ps1` registers a current-user execution every 15 minutes and at logon. Monitoring never uses `docker-test` and never mutates product state.
|
||||
|
||||
Reference in New Issue
Block a user