param( [string]$DockerSshAlias = "test-docker", [string]$BackendImageTag = "rap-backend:c17z19-route-health-feedback-smoke", [string]$AdminEmail = "fabric-owner-c17z19@example.local", [string]$AdminPassword = "SmokePass!123", [int]$ApiPort = 18122, [int]$PostgresPort = 15444, [int]$RedisPort = 16444, [string]$ResultPath = "artifacts\c17z19-route-health-feedback-smoke-result.json", [switch]$KeepRunning ) Set-StrictMode -Version Latest $ErrorActionPreference = "Stop" $repoRoot = (Resolve-Path (Join-Path $PSScriptRoot "..\..")).ProviderPath $backendPublicBaseUrl = "http://192.168.200.61:$ApiPort/api/v1" $runId = "c17z19-" + (Get-Date -Format "yyyyMMdd-HHmmss") $remoteBuildDir = "/tmp/rap-c17z19-build-$runId" $postgresName = "rap_c17z19_postgres" $redisName = "rap_c17z19_redis" $backendName = "rap_c17z19_backend" function Invoke-RemoteDocker { param([string[]]$Arguments) & ssh $DockerSshAlias docker @Arguments if ($LASTEXITCODE -ne 0) { throw "ssh $DockerSshAlias docker $($Arguments -join ' ') failed with exit code $LASTEXITCODE" } } function Invoke-RemoteDockerText { param([string[]]$Arguments) $previousErrorActionPreference = $ErrorActionPreference $ErrorActionPreference = "Continue" try { $output = & ssh $DockerSshAlias docker @Arguments 2>&1 | ForEach-Object { $_.ToString() } } finally { $ErrorActionPreference = $previousErrorActionPreference } if ($LASTEXITCODE -ne 0) { throw "ssh $DockerSshAlias docker $($Arguments -join ' ') failed with exit code $LASTEXITCODE" } return $output } function Invoke-RemoteShell { param([string]$Command) & ssh $DockerSshAlias $Command if ($LASTEXITCODE -ne 0) { throw "ssh $DockerSshAlias $Command failed with exit code $LASTEXITCODE" } } function Send-RemoteBuildContext { Write-Host "Uploading backend build context to $DockerSshAlias..." Invoke-RemoteShell -Command "rm -rf '$remoteBuildDir' && mkdir -p '$remoteBuildDir'" & tar -czf - -C $repoRoot "backend" | & ssh $DockerSshAlias "tar -xzf - -C '$remoteBuildDir'" if ($LASTEXITCODE -ne 0) { throw "upload build context failed" } } function Invoke-Api { param( [string]$Method, [string]$Path, [object]$Body = $null ) $uri = "$backendPublicBaseUrl$Path" try { if ($null -eq $Body) { return Invoke-RestMethod -Method $Method -Uri $uri -TimeoutSec 30 } return Invoke-RestMethod -Method $Method -Uri $uri -ContentType "application/json" -Body ($Body | ConvertTo-Json -Depth 80) -TimeoutSec 30 } catch { $statusCode = $null if ($_.Exception.Response) { $statusCode = [int]$_.Exception.Response.StatusCode } $details = $_.ErrorDetails.Message if (-not $details) { $details = $_.Exception.Message } throw "$Method $Path failed with HTTP $statusCode`: $details" } } function Wait-HttpReady { param([string]$Url) for ($i = 0; $i -lt 90; $i++) { try { $response = Invoke-WebRequest -UseBasicParsing -Uri $Url -TimeoutSec 2 if ($response.StatusCode -ge 200 -and $response.StatusCode -lt 300) { return } } catch { Start-Sleep -Seconds 1 } } throw "Timed out waiting for $Url" } function Remove-SmokeContainers { foreach ($name in @($backendName, $postgresName, $redisName)) { & ssh $DockerSshAlias docker rm -f $name 2>$null | Out-Null } } function New-EndpointCandidate { param( [string]$EndpointID, [string]$NodeID, [string]$Address, [string]$Transport, [string]$Reachability, [string]$ConnectivityMode, [int]$Priority, [string[]]$PolicyTags = @() ) return @{ endpoint_id = $EndpointID node_id = $NodeID transport = $Transport address = $Address address_family = "ipv4" reachability = $Reachability nat_type = "none" connectivity_mode = $ConnectivityMode region = "docker-test" priority = $Priority policy_tags = $PolicyTags last_verified_at = (Get-Date).ToUniversalTime().ToString("o") metadata = @{ stage = "c17z19" run_id = $runId production_payload_forwarding = $false } } } function New-Node { param( [string]$Key, [string[]]$Roles ) $fingerprint = "c17z19-fp-$Key-$([guid]::NewGuid().ToString('N'))" $publicKey = "c17z19-pub-$Key-$([guid]::NewGuid().ToString('N'))" $joinRequest = Invoke-Api -Method Post -Path "/clusters/$clusterID/join-requests" -Body @{ join_token = $joinToken.join_token.token node_name = "c17z19-node-$Key" node_fingerprint = $fingerprint public_key = $publicKey reported_capabilities = @{ can_route_mesh = $true testing_node = $true mesh_route_health_feedback = $true } reported_facts = @{ stage = "c17z19" run_id = $runId } requested_roles = @() } $approved = Invoke-Api -Method Post -Path "/clusters/$clusterID/join-requests/$($joinRequest.join_request.id)/approve" -Body @{ actor_user_id = $actorUserID node_key = $fingerprint ownership_type = "platform_managed" owner_organization_id = $null } $nodeID = $approved.node_bootstrap.node_id foreach ($role in $Roles) { Invoke-Api -Method Post -Path "/clusters/$clusterID/nodes/$nodeID/roles" -Body @{ actor_user_id = $actorUserID role = $role status = "active" policy = @{ stage = "c17z19" run_id = $runId synthetic_only = $true production_payload_forwarding = $false } } | Out-Null } return [pscustomobject]@{ id = $nodeID fingerprint = $fingerprint public_key = $publicKey } } function Get-OptionalProperty { param( [object]$Object, [string]$PropertyName ) if ($null -eq $Object) { return $null } $property = $Object.PSObject.Properties[$PropertyName] if ($null -eq $property) { return $null } return $property.Value } Write-Host "C17Z19 route-health feedback smoke run: $runId" Write-Host "Using SSH Docker host: $DockerSshAlias" Remove-SmokeContainers Send-RemoteBuildContext Write-Host "Building backend image on docker-test..." Invoke-RemoteDocker -Arguments @("build", "-f", "$remoteBuildDir/backend/Dockerfile", "-t", $BackendImageTag, "$remoteBuildDir/backend") Write-Host "Starting isolated PostgreSQL and Redis..." Invoke-RemoteDocker -Arguments @( "run", "-d", "--name", $postgresName, "-e", "POSTGRES_DB=remote_access_platform", "-e", "POSTGRES_USER=rap_user", "-e", "POSTGRES_PASSWORD=rap_password", "-p", "$PostgresPort`:5432", "postgres:16" ) Invoke-RemoteDocker -Arguments @( "run", "-d", "--name", $redisName, "-p", "$RedisPort`:6379", "redis:7" ) Invoke-RemoteShell -Command "for i in `$(seq 1 60); do docker exec $postgresName pg_isready -U rap_user -d remote_access_platform >/dev/null 2>&1 && exit 0; sleep 1; done; exit 1" Write-Host "Applying migrations..." Invoke-RemoteShell -Command "for f in `$(find '$remoteBuildDir/backend/migrations' -name '*.up.sql' | sort); do docker exec -i $postgresName psql -U rap_user -d remote_access_platform -v ON_ERROR_STOP=1 -f - < `$f; done" $secretBytes = New-Object byte[] 32 [Security.Cryptography.RandomNumberGenerator]::Fill($secretBytes) $secretKeyB64 = [Convert]::ToBase64String($secretBytes) Write-Host "Starting backend..." Invoke-RemoteDocker -Arguments @( "run", "-d", "--name", $backendName, "--network", "host", "-e", "APP_NAME=rap-api", "-e", "APP_ENV=c17z19-smoke", "-e", "HTTP_HOST=0.0.0.0", "-e", "HTTP_PORT=$ApiPort", "-e", "POSTGRES_DSN=postgres://rap_user:rap_password@127.0.0.1:$PostgresPort/remote_access_platform?sslmode=disable", "-e", "REDIS_ADDR=127.0.0.1:$RedisPort", "-e", "AUTH_ACCESS_TOKEN_SECRET=c17z19-access-secret", "-e", "AUTH_REFRESH_HASH_SECRET=c17z19-refresh-secret", "-e", "INSTALLATION_AUTHORITY_MODE=compat", "-e", "INSTALLATION_INSECURE_BOOTSTRAP_ENABLED=true", "-e", "SECRET_ENCRYPTION_KEY_B64=$secretKeyB64", "-e", "SECRET_ENCRYPTION_KEY_ID=$runId", $BackendImageTag ) Wait-HttpReady -Url "http://192.168.200.61:$ApiPort/readyz" Write-Host "Bootstrapping owner, cluster, and synthetic route..." Invoke-Api -Method Post -Path "/installation/bootstrap-owner" -Body @{ email = $AdminEmail password = $AdminPassword } | Out-Null $login = Invoke-Api -Method Post -Path "/auth/login" -Body @{ email = $AdminEmail password = $AdminPassword device_fingerprint = "c17z19-smoke-device" device_label = "C17Z19 route health feedback smoke" trust_device = $true } $actorUserID = $login.user.id $cluster = Invoke-Api -Method Post -Path "/clusters/" -Body @{ actor_user_id = $actorUserID slug = "c17z19-$((New-Guid).Guid.Substring(0, 8))" name = "C17Z19 Route Health Feedback Smoke" region = "docker-test" metadata = @{ stage = "c17z19" run_id = $runId production_forwarding = $false service_payload_forwarding = $false } } $clusterID = $cluster.cluster.id Invoke-Api -Method Put -Path "/fabric/testing-flags" -Body @{ actor_user_id = $actorUserID scope_type = "platform" scope_id = $null cluster_id = $null enabled = $true telemetry_enabled = $true synthetic_links_enabled = $true history_retention_hours = 24 metadata = @{ stage = "c17z19" run_id = $runId production_forwarding = $false service_payload_forwarding = $false } } | Out-Null $joinToken = Invoke-Api -Method Post -Path "/clusters/$clusterID/join-tokens" -Body @{ actor_user_id = $actorUserID scope = @{ purpose = "c17z19-route-health-feedback-smoke"; roles = @("core-mesh", "relay-node") } expires_at = (Get-Date).ToUniversalTime().AddHours(2).ToString("o") max_uses = 4 } $nodeA = New-Node -Key "a" -Roles @("core-mesh") $nodeB = New-Node -Key "b" -Roles @("core-mesh") $nodeS = New-Node -Key "s" -Roles @("core-mesh", "relay-node") $nodeT = New-Node -Key "t" -Roles @("core-mesh", "relay-node") $nodeBCandidate = New-EndpointCandidate -EndpointID "node-b-outbound" -NodeID $nodeB.id -Address "node-b.reverse.local" -Transport "outbound_reverse" -Reachability "outbound_only" -ConnectivityMode "outbound_only" -Priority 5 $nodeSCandidate = New-EndpointCandidate -EndpointID "node-s-public" -NodeID $nodeS.id -Address "http://node-s:19000" -Transport "direct_tcp_tls" -Reachability "public" -ConnectivityMode "direct" -Priority 1 -PolicyTags @("fast-path") $nodeTCandidate = New-EndpointCandidate -EndpointID "node-t-public" -NodeID $nodeT.id -Address "http://node-t:19000" -Transport "direct_tcp_tls" -Reachability "public" -ConnectivityMode "direct" -Priority 50 $routeIntent = Invoke-Api -Method Post -Path "/clusters/$clusterID/mesh/route-intents" -Body @{ actor_user_id = $actorUserID source_selector = @{ node_id = $nodeA.id } destination_selector = @{ node_id = $nodeB.id } service_class = "synthetic" priority = 100 policy = @{ synthetic_enabled = $true hops = @($nodeA.id, $nodeS.id, $nodeT.id, $nodeB.id) allowed_channels = @("fabric_control", "route_control") peer_endpoint_candidates = @{ "$($nodeB.id)" = @($nodeBCandidate) "$($nodeS.id)" = @($nodeSCandidate) "$($nodeT.id)" = @($nodeTCandidate) } } } $routeID = $routeIntent.route_intent.id $initialConfig = Invoke-Api -Method Get -Path "/clusters/$clusterID/nodes/$($nodeA.id)/mesh/synthetic-config" $initialLease = @($initialConfig.synthetic_mesh_config.rendezvous_leases | Where-Object { $_.peer_node_id -eq $nodeB.id }) | Select-Object -First 1 Write-Host "Injecting drift route-health for initially selected relay..." Invoke-Api -Method Post -Path "/clusters/$clusterID/mesh/links" -Body @{ source_node_id = $nodeA.id target_node_id = $nodeB.id link_status = "reachable" metadata = @{ stage = "c17z19" observation_type = "synthetic_route_health" route_id = $routeID route_path_decision_applied = $true route_path_decision_selected_relay_id = $nodeS.id route_path_decision_rendezvous_peer_node_id = $nodeB.id route_path_decision_rendezvous_lease_id = "$routeID-rv-$($nodeB.id)-via-$($nodeS.id)" route_path_decision_rendezvous_lease_reason = "auto_rendezvous_required" expected_effective_hops = @($nodeA.id, $nodeS.id, $nodeB.id) observed_ack_path = @($nodeA.id, $nodeT.id, $nodeB.id) route_path_drift_detected = $true control_plane_only = $true production_forwarding = $false production_payload_forwarding = $false route_health_production_payload_forwarding = $false route_health_service_payload_forwarding = $false } } | Out-Null $driftConfig = Invoke-Api -Method Get -Path "/clusters/$clusterID/nodes/$($nodeA.id)/mesh/synthetic-config" $driftLease = @($driftConfig.synthetic_mesh_config.rendezvous_leases | Where-Object { $_.peer_node_id -eq $nodeB.id }) | Select-Object -First 1 $driftDecision = @($driftConfig.synthetic_mesh_config.route_path_decisions.decisions | Where-Object { $_.route_id -eq $routeID }) | Select-Object -First 1 Write-Host "Injecting healthy low-latency route-health for alternate relay..." $latency = 5 $quality = 99 Invoke-Api -Method Post -Path "/clusters/$clusterID/mesh/links" -Body @{ source_node_id = $nodeA.id target_node_id = $nodeB.id link_status = "reachable" latency_ms = $latency quality_score = $quality metadata = @{ stage = "c17z19" observation_type = "synthetic_route_health" route_id = $routeID route_path_decision_applied = $true route_path_decision_selected_relay_id = $nodeT.id route_path_decision_rendezvous_peer_node_id = $nodeB.id expected_effective_hops = @($nodeA.id, $nodeT.id, $nodeB.id) observed_ack_path = @($nodeA.id, $nodeT.id, $nodeB.id) route_path_drift_detected = $false control_plane_only = $true production_forwarding = $false production_payload_forwarding = $false route_health_production_payload_forwarding = $false route_health_service_payload_forwarding = $false } } | Out-Null $healthyConfig = Invoke-Api -Method Get -Path "/clusters/$clusterID/nodes/$($nodeA.id)/mesh/synthetic-config" $healthyLease = @($healthyConfig.synthetic_mesh_config.rendezvous_leases | Where-Object { $_.peer_node_id -eq $nodeB.id }) | Select-Object -First 1 $healthyLeaseReasons = @(Get-OptionalProperty -Object $healthyLease.metadata -PropertyName "relay_selection_score_reasons") $meshLinks = Invoke-Api -Method Get -Path "/clusters/$clusterID/mesh/links?actor_user_id=$actorUserID" $backendLogs = Invoke-RemoteDockerText -Arguments @("logs", "--tail", "80", $backendName) $passMatrix = [ordered]@{ backend_ready = $true owner_login = [bool]$actorUserID cluster_created = [bool]$clusterID route_intent_created = [bool]$routeID initial_fast_path_prefers_node_s = ($null -ne $initialLease -and $initialLease.relay_node_id -eq $nodeS.id) drift_feedback_reselects_node_t = ($null -ne $driftLease -and $driftLease.relay_node_id -eq $nodeT.id -and $driftLease.reason -eq "stale_relay_replacement") drift_route_decision_uses_node_t = ($null -ne $driftDecision -and $driftDecision.selected_relay_id -eq $nodeT.id -and $driftDecision.stale_relay_node_id -eq $nodeS.id) relay_policy_scoring_mode_c17z19 = ($driftConfig.synthetic_mesh_config.rendezvous_relay_policy.scoring_mode -eq "route_adjacency_endpoint_priority_mesh_link_health_synthetic_route_health_feedback") healthy_latency_keeps_node_t_selected = ($null -ne $healthyLease -and $healthyLease.relay_node_id -eq $nodeT.id) healthy_latency_score_reason_present = ($healthyLeaseReasons -contains "route_health_latency" -and $healthyLeaseReasons -contains "route_health_reachable") synthetic_config_signed = [bool]($healthyConfig.synthetic_mesh_config.authority_required -and $healthyConfig.synthetic_mesh_config.authority_payload -and $healthyConfig.synthetic_mesh_config.authority_signature) production_forwarding_disabled = ($healthyConfig.synthetic_mesh_config.production_forwarding -eq $false) } $result = [ordered]@{ run_id = $runId stage = "C17Z19 route-health feedback scoring smoke" docker_host = $DockerSshAlias backend_base_url = $backendPublicBaseUrl containers = @{ backend = $backendName postgres = $postgresName redis = $redisName } cluster_id = $clusterID route_id = $routeID nodes = @{ a = $nodeA.id b = $nodeB.id s = $nodeS.id t = $nodeT.id } initial_lease = $initialLease drift_lease = $driftLease drift_route_decision = $driftDecision healthy_lease = $healthyLease mesh_links = $meshLinks.mesh_links pass_matrix = $passMatrix backend_log_tail = $backendLogs containers_left_running = [bool]$KeepRunning } $failed = @($passMatrix.GetEnumerator() | Where-Object { -not $_.Value }) $resultJson = $result | ConvertTo-Json -Depth 80 if ($ResultPath -ne "") { if ([System.IO.Path]::IsPathRooted($ResultPath)) { $resultFullPath = $ResultPath } else { $resultFullPath = Join-Path $repoRoot $ResultPath } New-Item -ItemType Directory -Force -Path (Split-Path $resultFullPath) | Out-Null Set-Content -Path $resultFullPath -Value $resultJson -Encoding UTF8 Write-Host "Result written to $resultFullPath" } $resultJson if ($failed.Count -gt 0) { throw "C17Z19 route-health feedback smoke failed: $($failed.Name -join ', ')" } Invoke-RemoteShell -Command "rm -rf '$remoteBuildDir'" if (-not $KeepRunning) { Write-Host "Cleaning up C17Z19 containers..." Remove-SmokeContainers }