DEV-000039: production health notifications #4
@@ -0,0 +1,20 @@
|
||||
[CmdletBinding()]
|
||||
param(
|
||||
[string]$HealthScriptPath = (Join-Path $PSScriptRoot 'Test-DomovoyProductionHealth.ps1'),
|
||||
[string]$NotificationScriptPath = (Join-Path $PSScriptRoot 'Send-DomovoyHealthNotification.ps1')
|
||||
)
|
||||
|
||||
$ErrorActionPreference = 'Stop'
|
||||
Set-StrictMode -Version Latest
|
||||
|
||||
if (-not (Test-Path -LiteralPath $HealthScriptPath -PathType Leaf)) { throw 'Health monitoring script not found' }
|
||||
if (-not (Test-Path -LiteralPath $NotificationScriptPath -PathType Leaf)) { throw 'Health notification script not found' }
|
||||
|
||||
$pwsh = (Get-Command pwsh.exe -ErrorAction Stop).Source
|
||||
& $pwsh -NoProfile -NonInteractive -ExecutionPolicy Bypass -File $HealthScriptPath
|
||||
$healthExitCode = $LASTEXITCODE
|
||||
|
||||
& $pwsh -NoProfile -NonInteractive -ExecutionPolicy Bypass -File $NotificationScriptPath
|
||||
$notificationExitCode = $LASTEXITCODE
|
||||
if ($notificationExitCode -ne 0) { exit 2 }
|
||||
exit $healthExitCode
|
||||
@@ -1,6 +1,6 @@
|
||||
[CmdletBinding()]
|
||||
param(
|
||||
[string]$ScriptPath = '\\nas\MST\codex\assistent\deploy\windows\Test-DomovoyProductionHealth.ps1',
|
||||
[string]$ScriptPath = '\\nas\MST\codex\assistent\deploy\windows\Invoke-DomovoyProductionMonitoring.ps1',
|
||||
[string]$TaskName = 'Domovoy Production Health Monitor'
|
||||
)
|
||||
|
||||
@@ -16,5 +16,5 @@ $repeat = New-ScheduledTaskTrigger -Once -At (Get-Date).AddMinutes(1) -Repetitio
|
||||
$logon = New-ScheduledTaskTrigger -AtLogOn -User $identity
|
||||
$principal = New-ScheduledTaskPrincipal -UserId $identity -LogonType Interactive -RunLevel Limited
|
||||
$settings = New-ScheduledTaskSettingsSet -StartWhenAvailable -MultipleInstances IgnoreNew -ExecutionTimeLimit (New-TimeSpan -Minutes 10)
|
||||
Register-ScheduledTask -TaskName $TaskName -Action $action -Trigger @($repeat, $logon) -Principal $principal -Settings $settings -Description 'Monitor Domovoy production health, TLS, DNS, backups, and schedules.' -Force | Out-Null
|
||||
Register-ScheduledTask -TaskName $TaskName -Action $action -Trigger @($repeat, $logon) -Principal $principal -Settings $settings -Description 'Monitor Domovoy production health and notify the interactive user about transitions.' -Force | Out-Null
|
||||
Write-Output "Scheduled task registered: $TaskName"
|
||||
|
||||
@@ -0,0 +1,125 @@
|
||||
[CmdletBinding()]
|
||||
param(
|
||||
[string]$StatusPath = (Join-Path $env:LOCALAPPDATA 'Domovoy\monitoring\status.json'),
|
||||
[string]$StatePath = (Join-Path $env:LOCALAPPDATA 'Domovoy\monitoring\notification-state.json'),
|
||||
[ValidateRange(1, 168)]
|
||||
[int]$ReminderHours = 6,
|
||||
[switch]$Force,
|
||||
[switch]$NoDisplay
|
||||
)
|
||||
|
||||
$ErrorActionPreference = 'Stop'
|
||||
Set-StrictMode -Version Latest
|
||||
|
||||
function Write-AtomicJson {
|
||||
param([Parameter(Mandatory)]$Value, [Parameter(Mandatory)][string]$Path)
|
||||
|
||||
$directory = Split-Path -Parent $Path
|
||||
if (-not (Test-Path -LiteralPath $directory -PathType Container)) {
|
||||
New-Item -ItemType Directory -Path $directory -Force | Out-Null
|
||||
}
|
||||
$temporary = Join-Path $directory ('.{0}.{1}.tmp' -f (Split-Path -Leaf $Path), [guid]::NewGuid().ToString('N'))
|
||||
try {
|
||||
$Value | ConvertTo-Json -Depth 8 | Set-Content -LiteralPath $temporary -Encoding utf8NoBOM
|
||||
Move-Item -LiteralPath $temporary -Destination $Path -Force
|
||||
}
|
||||
finally {
|
||||
if (Test-Path -LiteralPath $temporary) { Remove-Item -LiteralPath $temporary -Force }
|
||||
}
|
||||
}
|
||||
|
||||
if (-not (Test-Path -LiteralPath $StatusPath -PathType Leaf)) { throw "Health status not found: $StatusPath" }
|
||||
$status = Get-Content -LiteralPath $StatusPath -Raw | ConvertFrom-Json
|
||||
if ($status.version -ne 1 -or $status.overall -notin @('healthy', 'degraded')) { throw 'Unsupported health status document' }
|
||||
|
||||
$now = [DateTimeOffset]::UtcNow
|
||||
$previous = $null
|
||||
if (Test-Path -LiteralPath $StatePath -PathType Leaf) {
|
||||
$previous = Get-Content -LiteralPath $StatePath -Raw | ConvertFrom-Json
|
||||
}
|
||||
|
||||
$kind = $null
|
||||
if ($Force) {
|
||||
$kind = if ($status.overall -eq 'healthy') { 'recovered' } else { 'degraded' }
|
||||
}
|
||||
elseif ($status.overall -eq 'degraded') {
|
||||
if ($null -eq $previous -or $previous.last_status -ne 'degraded') {
|
||||
$kind = 'degraded'
|
||||
}
|
||||
elseif ($null -ne $previous.last_notification_at_utc) {
|
||||
$lastNotification = if ($previous.last_notification_at_utc -is [DateTime]) {
|
||||
[DateTimeOffset]$previous.last_notification_at_utc
|
||||
}
|
||||
else {
|
||||
[DateTimeOffset]::Parse([string]$previous.last_notification_at_utc, [Globalization.CultureInfo]::InvariantCulture)
|
||||
}
|
||||
if (($now - $lastNotification).TotalHours -ge $ReminderHours) { $kind = 'reminder' }
|
||||
}
|
||||
}
|
||||
elseif ($null -ne $previous -and $previous.last_status -eq 'degraded') {
|
||||
$kind = 'recovered'
|
||||
}
|
||||
|
||||
$lastNotificationAt = if ($null -ne $previous -and $null -ne $previous.last_notification_at_utc) {
|
||||
([DateTimeOffset]$previous.last_notification_at_utc).ToString('o')
|
||||
}
|
||||
else { $null }
|
||||
$lastNotificationKind = if ($null -ne $previous) { $previous.last_notification_kind } else { $null }
|
||||
|
||||
if ($null -ne $kind) {
|
||||
$failedNames = @($status.checks | Where-Object { -not $_.ok } | ForEach-Object { [string]$_.name })
|
||||
switch ($kind) {
|
||||
'degraded' {
|
||||
$title = 'Домовой: требуется внимание'
|
||||
$body = if ($failedNames.Count -gt 0) { 'Сбой: ' + ($failedNames -join ', ') } else { 'Мониторинг обнаружил сбой.' }
|
||||
$icon = 'Error'
|
||||
}
|
||||
'reminder' {
|
||||
$title = 'Домовой: сбой продолжается'
|
||||
$body = if ($failedNames.Count -gt 0) { 'Не исправлено: ' + ($failedNames -join ', ') } else { 'Состояние остается ухудшенным.' }
|
||||
$icon = 'Warning'
|
||||
}
|
||||
default {
|
||||
$title = 'Домовой: работа восстановлена'
|
||||
$body = 'Все проверки снова проходят успешно.'
|
||||
$icon = 'Info'
|
||||
}
|
||||
}
|
||||
|
||||
if (-not $NoDisplay) {
|
||||
Add-Type -AssemblyName System.Windows.Forms
|
||||
Add-Type -AssemblyName System.Drawing
|
||||
$notification = [System.Windows.Forms.NotifyIcon]::new()
|
||||
try {
|
||||
$notification.Icon = switch ($icon) {
|
||||
'Error' { [System.Drawing.SystemIcons]::Error }
|
||||
'Warning' { [System.Drawing.SystemIcons]::Warning }
|
||||
default { [System.Drawing.SystemIcons]::Information }
|
||||
}
|
||||
$notification.Visible = $true
|
||||
$notification.BalloonTipIcon = [System.Enum]::Parse([System.Windows.Forms.ToolTipIcon], $icon)
|
||||
$notification.BalloonTipTitle = $title
|
||||
$notification.BalloonTipText = $body
|
||||
$notification.ShowBalloonTip(8000)
|
||||
Start-Sleep -Seconds 9
|
||||
}
|
||||
finally {
|
||||
$notification.Dispose()
|
||||
}
|
||||
}
|
||||
|
||||
$lastNotificationAt = $now.ToString('o')
|
||||
$lastNotificationKind = $kind
|
||||
}
|
||||
|
||||
$state = [ordered]@{
|
||||
version = 1
|
||||
last_status = [string]$status.overall
|
||||
last_observed_at_utc = $now.ToString('o')
|
||||
last_notification_at_utc = $lastNotificationAt
|
||||
last_notification_kind = $lastNotificationKind
|
||||
}
|
||||
Write-AtomicJson -Value $state -Path $StatePath
|
||||
|
||||
$result = if ($null -eq $kind) { 'suppressed' } elseif ($NoDisplay) { 'simulated' } else { 'sent' }
|
||||
Write-Output "notification=$result status=$($status.overall) kind=$kind"
|
||||
@@ -1,5 +1,5 @@
|
||||
{
|
||||
"task_file": "dev/tasks/DEV-000038-production-health-monitor.json",
|
||||
"task_id": "DEV-000038",
|
||||
"task_file": "dev/tasks/DEV-000039-production-health-notifications.json",
|
||||
"task_id": "DEV-000039",
|
||||
"task_revision": 1
|
||||
}
|
||||
|
||||
@@ -0,0 +1,79 @@
|
||||
{
|
||||
"task_id": "DEV-000039",
|
||||
"task_revision": 1,
|
||||
"architecture_revision": "ADR-0004",
|
||||
"base_repo_revision": "4016820541229544aae81240cb2e12513fb2ce12",
|
||||
"risk_tier": "E",
|
||||
"intent": "Add transition-aware local Windows notifications to the external Domovoy production health monitor without external services, canonical Monitor semantics, or docker-test.",
|
||||
"requirement_ids": ["CONSTITUTION-1.3.11", "CONSTITUTION-1.3.31", "USER-DOCKER-PROD-ONLY"],
|
||||
"allowed_change_scope": {
|
||||
"modules": [],
|
||||
"paths": [
|
||||
"deploy/windows/Send-DomovoyHealthNotification.ps1",
|
||||
"deploy/windows/Invoke-DomovoyProductionMonitoring.ps1",
|
||||
"deploy/windows/Register-DomovoyProductionHealthTask.ps1",
|
||||
"docs/DEPLOYMENT.md",
|
||||
"dev/ACTIVE_TASK.json",
|
||||
"dev/tasks/DEV-000039-production-health-notifications.json"
|
||||
]
|
||||
},
|
||||
"forbidden_change_scope": {
|
||||
"paths": [
|
||||
"architecture/**",
|
||||
"ci/**",
|
||||
".gitea/**",
|
||||
"cmd/**",
|
||||
"contracts/**",
|
||||
"modules/**",
|
||||
"database/**",
|
||||
"security/**",
|
||||
"deploy/docker-compose*.yml"
|
||||
]
|
||||
},
|
||||
"allowed_architecture_delta": {
|
||||
"new_cross_module_dependencies": 0,
|
||||
"new_canonical_owners": 0,
|
||||
"new_privileges": 0,
|
||||
"new_third_party_dependencies": 0
|
||||
},
|
||||
"contracts": {
|
||||
"consume": ["DEV-000038 local production health status JSON", "Windows interactive notification area", "Windows Scheduled Tasks"],
|
||||
"may_change": []
|
||||
},
|
||||
"semantic_owner": "none; external rebuildable operational projection",
|
||||
"canonical_state": ["none; notification suppression state is non-canonical and may be regenerated"],
|
||||
"state_transitions": ["first degraded observation -> degraded notification", "healthy to degraded -> degraded notification", "degraded to healthy -> recovery notification", "continued degraded after reminder interval -> reminder notification"],
|
||||
"side_effects_and_privileges": ["display a local Windows notification for the current interactive user", "atomically write local non-canonical notification state", "update the current-user scheduled task action"],
|
||||
"persistence_and_migrations": ["no database migration", "single local notification state JSON; no history or retention deletion"],
|
||||
"failure_retry_restart": ["health collection runs in a child process and publishes status before notification evaluation", "duplicate degraded observations are suppressed for six hours", "notification failure is distinct from health degradation", "the scheduled task returns the health exit code unless notification delivery itself fails"],
|
||||
"assumptions": [
|
||||
"The scheduled task runs for an interactive Windows user so notification balloons are visible",
|
||||
"The DEV-000038 status schema remains version 1 with overall and checks fields",
|
||||
"Local notification state is not authoritative and can be deleted safely",
|
||||
"docker-test remains prohibited"
|
||||
],
|
||||
"acceptance_scenarios": [
|
||||
"an initial healthy observation records state without notifying",
|
||||
"a transition to degraded sends one notification naming failed checks",
|
||||
"a repeated degraded observation within six hours is suppressed",
|
||||
"a transition back to healthy sends one recovery notification",
|
||||
"the wrapper evaluates notifications even when the health child exits nonzero",
|
||||
"the registered task invokes the wrapper every 15 minutes and at logon",
|
||||
"exact diff validation observes only declared paths"
|
||||
],
|
||||
"required_tests": [
|
||||
"PowerShell parser",
|
||||
"architecture preflight",
|
||||
"exact diff scope",
|
||||
"synthetic healthy/degraded/repeated/recovered notification transitions",
|
||||
"healthy live wrapper run",
|
||||
"scheduled task registration and manual invocation",
|
||||
"go test",
|
||||
"go vet",
|
||||
"CI"
|
||||
],
|
||||
"terminal_gap_states": [
|
||||
"BLOCKED_BY_MISSING_CAPABILITY",
|
||||
"BLOCKED_BY_CONFLICT"
|
||||
]
|
||||
}
|
||||
@@ -39,3 +39,5 @@ The production user crontab installs the single entry from `deploy/cron/assistan
|
||||
`deploy/windows/Test-DomovoyProductionHealth.ps1` is an external, rebuildable operational projection; it is not a canonical Core `Monitor`. It checks all expected production containers, HTTPS health, TLS expiry, public DNS agreement, production backup age and hashes, NAS backup age and ciphertext hash, cron, and the off-host backup task. Thresholds are 30 hours for the production backup, 36 hours for the NAS copy, and 21 remaining certificate days.
|
||||
|
||||
The script atomically writes the latest JSON snapshot to `%LOCALAPPDATA%\Domovoy\monitoring\status.json` and `\\nas\MST\codex\_monitoring\domovoy\status.json`, then exits nonzero when any check is degraded. `deploy/windows/Register-DomovoyProductionHealthTask.ps1` registers a current-user execution every 15 minutes and at logon. Monitoring never uses `docker-test` and never mutates product state.
|
||||
|
||||
`deploy/windows/Invoke-DomovoyProductionMonitoring.ps1` runs health collection in a child process and then evaluates `deploy/windows/Send-DomovoyHealthNotification.ps1`, including when health is degraded. The notifier shows a local Windows notification on the first degraded observation, recovery, and no more than one continuing-failure reminder per six hours. Its suppression state is an external, non-canonical JSON file at `%LOCALAPPDATA%\Domovoy\monitoring\notification-state.json` and may be regenerated. Notifications require the registered user to have an interactive Windows session; no external notification service or credential is used.
|
||||
|
||||
Reference in New Issue
Block a user