param( [string]$DockerSshAlias = "test-docker", [string]$BackendImageTag = "rap-backend:c17z18-rendezvous-smoke", [string]$NodeAgentImageTag = "rap-node-agent:c17z18-rendezvous-smoke", [string]$AdminEmail = "fabric-owner-c17z18@example.local", [string]$AdminPassword = "SmokePass!123", [int]$ApiPort = 18120, [int]$PostgresPort = 15442, [int]$RedisPort = 16442, [int]$MeshBasePort = 19120, [switch]$KeepRunning ) Set-StrictMode -Version Latest $ErrorActionPreference = "Stop" $repoRoot = (Resolve-Path (Join-Path $PSScriptRoot "..\..")).ProviderPath $backendPublicBaseUrl = "http://192.168.200.61:$ApiPort/api/v1" $backendContainerBaseUrl = "http://127.0.0.1:$ApiPort/api/v1" $runId = "c17z18-" + (Get-Date -Format "yyyyMMdd-HHmmss") $remoteBuildDir = "/tmp/rap-c17z18-build-$runId" $postgresName = "rap_c17z12_postgres" $redisName = "rap_c17z12_redis" $backendName = "rap_c17z12_backend" $nodePrefix = "rap_c17z12_node_" function Invoke-RemoteDocker { param([string[]]$Arguments) & ssh $DockerSshAlias docker @Arguments if ($LASTEXITCODE -ne 0) { throw "ssh $DockerSshAlias docker $($Arguments -join ' ') failed with exit code $LASTEXITCODE" } } function Invoke-RemoteDockerText { param([string[]]$Arguments) $previousErrorActionPreference = $ErrorActionPreference $ErrorActionPreference = "Continue" try { $output = & ssh $DockerSshAlias docker @Arguments 2>&1 | ForEach-Object { $_.ToString() } } finally { $ErrorActionPreference = $previousErrorActionPreference } if ($LASTEXITCODE -ne 0) { throw "ssh $DockerSshAlias docker $($Arguments -join ' ') failed with exit code $LASTEXITCODE" } return $output } function Invoke-RemoteShell { param([string]$Command) & ssh $DockerSshAlias $Command if ($LASTEXITCODE -ne 0) { throw "ssh $DockerSshAlias $Command failed with exit code $LASTEXITCODE" } } function Invoke-RemotePostgresSql { param([string]$Sql) $Sql | & ssh $DockerSshAlias "docker exec -i $postgresName psql -U rap_user -d remote_access_platform -v ON_ERROR_STOP=1 -f -" if ($LASTEXITCODE -ne 0) { throw "remote psql command failed" } } function Send-RemoteFile { param( [string]$RemotePath, [string]$Content ) $Content | & ssh $DockerSshAlias "cat > '$RemotePath'" if ($LASTEXITCODE -ne 0) { throw "write remote file failed: $RemotePath" } } function Send-RemoteBuildContext { Write-Host "Uploading backend and node-agent build context to $DockerSshAlias..." Invoke-RemoteShell -Command "rm -rf '$remoteBuildDir' && mkdir -p '$remoteBuildDir'" & tar -czf - -C $repoRoot "backend" "agents/rap-node-agent" | & ssh $DockerSshAlias "tar -xzf - -C '$remoteBuildDir'" if ($LASTEXITCODE -ne 0) { throw "upload build context failed" } } function Invoke-Api { param( [string]$Method, [string]$Path, [object]$Body = $null ) $uri = "$backendPublicBaseUrl$Path" if ($null -eq $Body) { return Invoke-RestMethod -Method $Method -Uri $uri -TimeoutSec 30 } return Invoke-RestMethod -Method $Method -Uri $uri -ContentType "application/json" -Body ($Body | ConvertTo-Json -Depth 50) -TimeoutSec 30 } function Wait-HttpReady { param([string]$Url) for ($i = 0; $i -lt 60; $i++) { try { $response = Invoke-WebRequest -UseBasicParsing -Uri $Url -TimeoutSec 2 if ($response.StatusCode -ge 200 -and $response.StatusCode -lt 300) { return } } catch { Start-Sleep -Seconds 1 } } throw "Timed out waiting for $Url" } function Remove-C17Z12Containers { $names = @($backendName, $postgresName, $redisName) foreach ($key in @("a", "b", "c", "r", "idle")) { $names += "$nodePrefix$key" } foreach ($name in $names) { & ssh $DockerSshAlias docker rm -f $name 2>$null | Out-Null } } function New-EndpointCandidate { param( [string]$EndpointID, [string]$NodeID, [string]$Address, [string]$Transport, [string]$Reachability, [string]$ConnectivityMode, [string]$NATType, [int]$Priority, [string[]]$PolicyTags = @() ) return @{ endpoint_id = $EndpointID node_id = $NodeID transport = $Transport address = $Address address_family = "ipv4" reachability = $Reachability nat_type = $NATType connectivity_mode = $ConnectivityMode region = "docker-test" priority = $Priority policy_tags = $PolicyTags last_verified_at = (Get-Date).ToUniversalTime().ToString("o") metadata = @{ stage = "c17z18" run_id = $runId service_workload_traffic = $false production_payload_forwarding = $false } } } function Get-OptionalProperty { param( [object]$Object, [string]$PropertyName ) if ($null -eq $Object) { return $null } $property = $Object.PSObject.Properties[$PropertyName] if ($null -eq $property) { return $null } return $property.Value } function Get-OptionalArrayCount { param( [object]$Object, [string]$PropertyName ) $value = Get-OptionalProperty -Object $Object -PropertyName $PropertyName if ($null -eq $value) { return 0 } return @($value).Count } function Get-LatestHeartbeatMetadataReport { param( [string]$NodeID, [string]$PropertyName ) $heartbeats = Invoke-Api -Method Get -Path "/clusters/$clusterID/nodes/$NodeID/heartbeats?actor_user_id=$actorUserID&limit=5" $latest = @($heartbeats.heartbeats) | Select-Object -First 1 $metadata = Get-OptionalProperty -Object $latest -PropertyName "metadata" return Get-OptionalProperty -Object $metadata -PropertyName $PropertyName } function Get-LatestRendezvousLeaseReport { param([string]$NodeID) return Get-LatestHeartbeatMetadataReport -NodeID $NodeID -PropertyName "mesh_rendezvous_lease_report" } function Get-LatestRoutePathDecisionReport { param([string]$NodeID) return Get-LatestHeartbeatMetadataReport -NodeID $NodeID -PropertyName "mesh_route_path_decision_report" } function Get-LatestRouteGenerationReport { param([string]$NodeID) return Get-LatestHeartbeatMetadataReport -NodeID $NodeID -PropertyName "mesh_route_generation_report" } function Get-LatestRouteHealthConfigReport { param([string]$NodeID) return Get-LatestHeartbeatMetadataReport -NodeID $NodeID -PropertyName "mesh_route_health_config_report" } function Get-LatestRouteHealthFeedbackRefreshReport { param([string]$NodeID) return Get-LatestHeartbeatMetadataReport -NodeID $NodeID -PropertyName "mesh_route_health_feedback_refresh_report" } function Select-C17Z18RouteHealthSnapshot { param([object[]]$MeshLinks) $routeHealthLinks = @($MeshLinks | Where-Object { $_.metadata.observation_type -eq "synthetic_route_health" -and $_.metadata.config_source -eq "control_plane" }) $directHealth = @($routeHealthLinks | Where-Object { $_.metadata.route_id -eq $directIntent.route_intent.id -and $_.link_status -eq "reachable" }) $rendezvousHealth = @($routeHealthLinks | Where-Object { $_.source_node_id -eq $nodeAID -and $_.metadata.route_id -eq $rendezvousIntent.route_intent.id -and $_.link_status -eq "reachable" }) $replacementRouteHealth = @($rendezvousHealth | Where-Object { $_.metadata.route_path_decision_applied -eq $true -and $_.metadata.route_path_decision_selected_relay_id -eq $nodeSID -and (@($_.metadata.expected_effective_hops) -contains $nodeSID) -and -not (@($_.metadata.expected_effective_hops) -contains $nodeRID) -and (@($_.metadata.observed_ack_path) -contains $nodeSID) -and -not (@($_.metadata.observed_ack_path) -contains $nodeRID) -and $_.metadata.route_path_drift_detected -eq $false }) return [pscustomobject]@{ route_health_links = $routeHealthLinks direct_health = $directHealth rendezvous_health = $rendezvousHealth replacement_route_health = $replacementRouteHealth } } function Get-C17Z18MeshLinkSnapshot { $links = Invoke-Api -Method Get -Path "/clusters/$clusterID/mesh/links?actor_user_id=$actorUserID" $meshLinks = @($links.mesh_links) $routeHealth = Select-C17Z18RouteHealthSnapshot -MeshLinks $meshLinks return [pscustomobject]@{ links = $links mesh_links = $meshLinks route_health_links = @($routeHealth.route_health_links) direct_health = @($routeHealth.direct_health) rendezvous_health = @($routeHealth.rendezvous_health) replacement_route_health = @($routeHealth.replacement_route_health) } } function Wait-C17Z18ReplacementRouteHealthSnapshot { param([int]$TimeoutSeconds = 40) $deadline = (Get-Date).AddSeconds($TimeoutSeconds) $latest = $null do { $latest = Get-C17Z18MeshLinkSnapshot if (@($latest.replacement_route_health).Count -gt 0) { return $latest } Start-Sleep -Seconds 2 } while ((Get-Date) -lt $deadline) return $latest } Write-Host "C17Z18 rendezvous relay replacement smoke run: $runId" Write-Host "Using SSH Docker host: $DockerSshAlias" Remove-C17Z12Containers Send-RemoteBuildContext Write-Host "Building backend image on docker-test..." Invoke-RemoteDocker -Arguments @("build", "-f", "$remoteBuildDir/backend/Dockerfile", "-t", $BackendImageTag, "$remoteBuildDir/backend") Write-Host "Building node-agent image on docker-test..." Invoke-RemoteDocker -Arguments @("build", "-f", "$remoteBuildDir/agents/rap-node-agent/Dockerfile", "-t", $NodeAgentImageTag, $remoteBuildDir) Write-Host "Starting PostgreSQL and Redis..." Invoke-RemoteDocker -Arguments @( "run", "-d", "--name", $postgresName, "-e", "POSTGRES_DB=remote_access_platform", "-e", "POSTGRES_USER=rap_user", "-e", "POSTGRES_PASSWORD=rap_password", "-p", "$PostgresPort`:5432", "postgres:16" ) Invoke-RemoteDocker -Arguments @( "run", "-d", "--name", $redisName, "-p", "$RedisPort`:6379", "redis:7" ) Invoke-RemoteShell -Command "for i in `$(seq 1 60); do docker exec $postgresName pg_isready -U rap_user -d remote_access_platform >/dev/null 2>&1 && exit 0; sleep 1; done; exit 1" Write-Host "Applying migrations..." Invoke-RemoteShell -Command "for f in `$(find '$remoteBuildDir/backend/migrations' -name '*.up.sql' | sort); do docker exec -i $postgresName psql -U rap_user -d remote_access_platform -v ON_ERROR_STOP=1 -f - < `$f; done" Write-Host "Seeding platform owner..." $adminUserID = [guid]::NewGuid().ToString() $adminHash = '$2a$10$AqLRexkI1yXbuiMPU6dHM.KVUhF.t..9NolyK4OOodQTyTsHyG.7u' Invoke-RemotePostgresSql -Sql @" INSERT INTO users (id, email, password_hash, mfa_enabled, platform_role) VALUES ('$adminUserID'::uuid, '$AdminEmail', '$adminHash', FALSE, 'platform_admin') ON CONFLICT (email) DO UPDATE SET password_hash = EXCLUDED.password_hash, platform_role = 'platform_admin', updated_at = NOW(); "@ Write-Host "Starting backend..." Invoke-RemoteDocker -Arguments @( "run", "-d", "--name", $backendName, "--network", "host", "-e", "APP_NAME=rap-api", "-e", "APP_ENV=c17z18-smoke", "-e", "HTTP_HOST=0.0.0.0", "-e", "HTTP_PORT=$ApiPort", "-e", "POSTGRES_DSN=postgres://rap_user:rap_password@127.0.0.1:$PostgresPort/remote_access_platform?sslmode=disable", "-e", "REDIS_ADDR=127.0.0.1:$RedisPort", "-e", "AUTH_ACCESS_TOKEN_SECRET=c17z18-access-secret", "-e", "AUTH_REFRESH_HASH_SECRET=c17z18-refresh-secret", $BackendImageTag ) Wait-HttpReady -Url "http://192.168.200.61:$ApiPort/readyz" Write-Host "Logging in as platform owner..." $login = Invoke-Api -Method Post -Path "/auth/login" -Body @{ email = $AdminEmail password = $AdminPassword device_fingerprint = "c17z18-smoke-device" device_label = "C17Z18 rendezvous relay replacement smoke" trust_device = $true } $actorUserID = $login.user.id Write-Host "Creating C17Z18 cluster..." $cluster = Invoke-Api -Method Post -Path "/clusters/" -Body @{ actor_user_id = $actorUserID slug = "c17z18-$((New-Guid).Guid.Substring(0, 8))" name = "C17Z18 Rendezvous Relay Replacement Smoke" region = "docker-test" metadata = @{ stage = "c17z18" run_id = $runId production_forwarding = $false service_workload_traffic = $false created_by = "c17z12-rendezvous-relay-smoke-ssh.ps1:c17z18" } } $clusterID = $cluster.cluster.id Write-Host "Enabling test-only Fabric flags..." Invoke-Api -Method Put -Path "/fabric/testing-flags" -Body @{ actor_user_id = $actorUserID scope_type = "platform" scope_id = $null cluster_id = $null enabled = $true telemetry_enabled = $true synthetic_links_enabled = $true history_retention_hours = 24 metadata = @{ stage = "c17z18" run_id = $runId production_forwarding = $false service_workload_traffic = $false } } | Out-Null $joinToken = Invoke-Api -Method Post -Path "/clusters/$clusterID/join-tokens" -Body @{ actor_user_id = $actorUserID scope = @{ purpose = "c17z18-rendezvous-relay-replacement-smoke"; roles = @("core-mesh", "relay-node") } expires_at = (Get-Date).ToUniversalTime().AddHours(2).ToString("o") max_uses = 5 } $nodeSpecs = @( @{ key = "a"; name = "c17z18-node-a-entry"; roles = @("core-mesh"); port = $MeshBasePort; transport = "direct_tcp_tls"; connectivity = "direct"; nat = "none" }, @{ key = "r"; name = "c17z18-node-r-stale-relay"; roles = @("core-mesh", "relay-node"); port = ($MeshBasePort + 1); transport = "direct_tcp_tls"; connectivity = "direct"; nat = "none" }, @{ key = "b"; name = "c17z18-node-b-direct"; roles = @("core-mesh"); port = ($MeshBasePort + 2); transport = "direct_tcp_tls"; connectivity = "direct"; nat = "none" }, @{ key = "c"; name = "c17z18-node-c-outbound"; roles = @("core-mesh"); port = ($MeshBasePort + 3); transport = "outbound_reverse"; connectivity = "outbound_only"; nat = "symmetric" }, @{ key = "idle"; name = "c17z18-node-s-alt-relay"; roles = @("core-mesh", "relay-node"); port = ($MeshBasePort + 4); transport = "direct_tcp_tls"; connectivity = "direct"; nat = "none" } ) $nodes = @{} foreach ($spec in $nodeSpecs) { $fingerprint = "c17z18-fp-$($spec.key)-$([guid]::NewGuid().ToString('N'))" $publicKey = "c17z18-pub-$($spec.key)-$([guid]::NewGuid().ToString('N'))" $joinRequest = Invoke-Api -Method Post -Path "/clusters/$clusterID/join-requests" -Body @{ join_token = $joinToken.join_token.token node_name = $spec.name node_fingerprint = $fingerprint public_key = $publicKey reported_capabilities = @{ can_accept_node_ingress = $true can_route_mesh = $true testing_node = $true mesh_rendezvous_relay_control_contract = $true mesh_rendezvous_lease_telemetry = $true mesh_rendezvous_lease_refresh_contract = $true mesh_rendezvous_relay_replacement_contract = $true mesh_route_path_decision_contract = $true mesh_route_generation_tracker = $true } reported_facts = @{ os = "linux" runtime = "docker-test" stage = "c17z18" run_id = $runId connectivity_mode = $spec.connectivity nat_type = $spec.nat } requested_roles = $spec.roles } $approved = Invoke-Api -Method Post -Path "/clusters/$clusterID/join-requests/$($joinRequest.join_request.id)/approve" -Body @{ actor_user_id = $actorUserID node_key = $fingerprint ownership_type = "platform_managed" owner_organization_id = $null } $nodeID = $approved.node_bootstrap.node_id foreach ($role in $spec.roles) { Invoke-Api -Method Post -Path "/clusters/$clusterID/nodes/$nodeID/roles" -Body @{ actor_user_id = $actorUserID role = $role status = "active" policy = @{ stage = "c17z18" run_id = $runId synthetic_only = $true production_payload_forwarding = $false } } | Out-Null } $nodes[$spec.key] = [pscustomobject]@{ id = $nodeID name = $spec.name fingerprint = $fingerprint public_key = $publicKey port = $spec.port roles = $spec.roles transport = $spec.transport connectivity = $spec.connectivity nat = $spec.nat } } $nodeAID = $nodes["a"].id $nodeRID = $nodes["r"].id $nodeBID = $nodes["b"].id $nodeCID = $nodes["c"].id $nodeSID = $nodes["idle"].id $routeExpiresAt = (Get-Date).ToUniversalTime().AddHours(2).ToString("o") $staleRelayEndpoint = "http://127.0.0.1:$($MeshBasePort + 90)" $peerEndpointsDirect = @{} $peerEndpointsDirect[$nodeAID] = "http://127.0.0.1:$($nodes["a"].port)" $peerEndpointsDirect[$nodeBID] = "http://127.0.0.1:$($nodes["b"].port)" $peerEndpointsRelayControl = @{} $peerEndpointsRelayControl[$nodeAID] = "http://127.0.0.1:$($nodes["a"].port)" $peerEndpointsRelayControl[$nodeRID] = $staleRelayEndpoint $peerEndpointsRelayControl[$nodeSID] = "http://127.0.0.1:$($nodes["idle"].port)" $peerEndpointCandidatesRelay = @{} $peerEndpointCandidatesRelay[$nodeRID] = @( New-EndpointCandidate ` -EndpointID "relay-r-public" ` -NodeID $nodeRID ` -Address $staleRelayEndpoint ` -Transport "direct_tcp_tls" ` -Reachability "public" ` -ConnectivityMode "direct" ` -NATType "none" ` -Priority 10 ` -PolicyTags @("relay-control", "same-site") ) $peerEndpointCandidatesRelay[$nodeSID] = @( New-EndpointCandidate ` -EndpointID "relay-s-alt-fast" ` -NodeID $nodeSID ` -Address "http://127.0.0.1:$($nodes["idle"].port)" ` -Transport "direct_tcp_tls" ` -Reachability "public" ` -ConnectivityMode "direct" ` -NATType "none" ` -Priority 1 ` -PolicyTags @("relay-control", "same-site", "fast-path") ) $peerEndpointCandidatesRelay[$nodeCID] = @( New-EndpointCandidate ` -EndpointID "node-c-outbound-only" ` -NodeID $nodeCID ` -Address "http://127.0.0.1:$($nodes["c"].port)" ` -Transport "outbound_reverse" ` -Reachability "outbound_only" ` -ConnectivityMode "outbound_only" ` -NATType "symmetric" ` -Priority 5 ` -PolicyTags @("nat", "outbound-only") ) Write-Host "Creating direct baseline and outbound-only relay-control route intents..." $directIntent = Invoke-Api -Method Post -Path "/clusters/$clusterID/mesh/route-intents" -Body @{ actor_user_id = $actorUserID source_selector = @{ node_id = $nodeAID } destination_selector = @{ node_id = $nodeBID } service_class = "control" priority = 10 policy = @{ synthetic_enabled = $true peer_endpoints = $peerEndpointsDirect hops = @($nodeAID, $nodeBID) allowed_channels = @("fabric_control", "route_control") max_ttl = 4 max_hops = 4 expires_at = $routeExpiresAt route_version = "$runId-direct" policy_version = "$runId-policy" peer_directory_version = "$runId-peers" production_forwarding = $false } } $rendezvousIntent = Invoke-Api -Method Post -Path "/clusters/$clusterID/mesh/route-intents" -Body @{ actor_user_id = $actorUserID source_selector = @{ node_id = $nodeAID } destination_selector = @{ node_id = $nodeCID } service_class = "control" priority = 20 policy = @{ synthetic_enabled = $true peer_endpoints = $peerEndpointsRelayControl peer_endpoint_candidates = $peerEndpointCandidatesRelay hops = @($nodeAID, $nodeRID, $nodeSID, $nodeCID) allowed_channels = @("fabric_control", "route_control") max_ttl = 6 max_hops = 6 rendezvous_leases = @( @{ lease_id = "$runId-explicit-stale-relay-lease" peer_node_id = $nodeCID relay_node_id = $nodeRID relay_endpoint = $staleRelayEndpoint transport = "relay_control" connectivity_mode = "relay_required" route_ids = @() allowed_channels = @("fabric_control", "route_control") priority = 4 control_plane_only = $true issued_at = (Get-Date).ToUniversalTime().ToString("o") expires_at = $routeExpiresAt reason = "smoke_stale_relay_replacement" metadata = @{ stage = "c17z18" run_id = $runId lease_refresh_contract = "node_scoped_synthetic_config_get" relay_replacement_contract = "stale_relay_feedback_policy" production_payload_forwarding = $false } } ) expires_at = $routeExpiresAt route_version = "$runId-rendezvous" policy_version = "$runId-policy" peer_directory_version = "$runId-peers" production_forwarding = $false } } $configs = @{} foreach ($key in @("a", "r", "b", "c", "idle")) { $configs[$key] = Invoke-Api -Method Get -Path "/clusters/$clusterID/nodes/$($nodes[$key].id)/mesh/synthetic-config" } $nodeAPeerCandidates = Get-OptionalProperty -Object (Get-OptionalProperty -Object $configs["a"].synthetic_mesh_config -PropertyName "peer_endpoint_candidates") -PropertyName $nodeCID $nodeAInitialStaleLeases = @(Get-OptionalProperty -Object $configs["a"].synthetic_mesh_config -PropertyName "rendezvous_leases" | Where-Object { $_.peer_node_id -eq $nodeCID -and $_.relay_node_id -eq $nodeRID }) $nodeAInitialAltLeases = @(Get-OptionalProperty -Object $configs["a"].synthetic_mesh_config -PropertyName "rendezvous_leases" | Where-Object { $_.peer_node_id -eq $nodeCID -and $_.relay_node_id -eq $nodeSID }) $nodeCLeases = @(Get-OptionalProperty -Object $configs["c"].synthetic_mesh_config -PropertyName "rendezvous_leases" | Where-Object { $_.peer_node_id -eq $nodeCID -and ($_.relay_node_id -eq $nodeRID -or $_.relay_node_id -eq $nodeSID) }) Write-Host "Starting node-agent containers..." foreach ($key in @("r", "idle", "b", "c", "a")) { $node = $nodes[$key] $containerName = "$nodePrefix$key" $remoteStateDir = "/tmp/$runId-$key" Invoke-RemoteShell -Command "rm -rf '$remoteStateDir' && mkdir -p '$remoteStateDir'" $identity = @{ node_id = $node.id cluster_id = $clusterID node_name = $node.name node_fingerprint = $node.fingerprint public_key = $node.public_key identity_status = "active" created_at = (Get-Date).ToUniversalTime().ToString("o") updated_at = (Get-Date).ToUniversalTime().ToString("o") } | ConvertTo-Json -Depth 10 Send-RemoteFile -RemotePath "$remoteStateDir/identity.json" -Content $identity Invoke-RemoteDocker -Arguments @( "create", "--name", $containerName, "--network", "host", "-e", "RAP_BACKEND_URL=$backendContainerBaseUrl", "-e", "RAP_NODE_STATE_DIR=/tmp/state", "-e", "RAP_HEARTBEAT_INTERVAL_SECONDS=5", "-e", "RAP_MESH_SYNTHETIC_RUNTIME_ENABLED=true", "-e", "RAP_MESH_LISTEN_ADDR=0.0.0.0:$($node.port)", "-e", "RAP_MESH_ADVERTISE_ENDPOINT=http://127.0.0.1:$($node.port)", "-e", "RAP_MESH_ADVERTISE_TRANSPORT=$($node.transport)", "-e", "RAP_MESH_CONNECTIVITY_MODE=$($node.connectivity)", "-e", "RAP_MESH_NAT_TYPE=$($node.nat)", "-e", "RAP_MESH_REGION=docker-test", $NodeAgentImageTag, "-backend-url", $backendContainerBaseUrl, "-state-dir", "/tmp/state", "-heartbeat-interval", "5s", "-mesh-synthetic-runtime-enabled", "-mesh-listen-addr", "0.0.0.0:$($node.port)", "-mesh-advertise-endpoint", "http://127.0.0.1:$($node.port)", "-mesh-advertise-transport", $node.transport, "-mesh-connectivity-mode", $node.connectivity, "-mesh-nat-type", $node.nat, "-mesh-region", "docker-test" ) | Out-Null Invoke-RemoteDocker -Arguments @("cp", "$remoteStateDir/.", "$containerName`:/tmp/state") Invoke-RemoteDocker -Arguments @("start", $containerName) | Out-Null } Write-Host "Waiting for rendezvous relay-control observations..." Start-Sleep -Seconds 40 Write-Host "Waiting for replacement route-health effective path..." $meshSnapshot = Wait-C17Z18ReplacementRouteHealthSnapshot -TimeoutSeconds 40 $links = $meshSnapshot.links $summary = Invoke-Api -Method Get -Path "/cluster-admin-summaries?actor_user_id=$actorUserID" $nodeALeaseReport = Get-LatestRendezvousLeaseReport -NodeID $nodeAID $nodeRLeaseReport = Get-LatestRendezvousLeaseReport -NodeID $nodeRID $nodeSLeaseReport = Get-LatestRendezvousLeaseReport -NodeID $nodeSID $nodeCLeaseReport = Get-LatestRendezvousLeaseReport -NodeID $nodeCID $nodeAPathDecisionReport = Get-LatestRoutePathDecisionReport -NodeID $nodeAID $nodeARouteGenerationReport = Get-LatestRouteGenerationReport -NodeID $nodeAID $nodeARouteHealthConfigReport = Get-LatestRouteHealthConfigReport -NodeID $nodeAID $nodeARouteHealthFeedbackRefreshReport = Get-LatestRouteHealthFeedbackRefreshReport -NodeID $nodeAID $refreshedNodeAConfig = Invoke-Api -Method Get -Path "/clusters/$clusterID/nodes/$nodeAID/mesh/synthetic-config" $nodeAReportedLeases = @(Get-OptionalProperty -Object $nodeALeaseReport -PropertyName "leases") $nodeAReportedReplacementLeases = @($nodeAReportedLeases | Where-Object { $_.peer_node_id -eq $nodeCID -and $_.relay_node_id -eq $nodeSID -and $_.reason -eq "stale_relay_replacement" }) $nodeAReportedStaleRelayLeases = @($nodeAReportedLeases | Where-Object { $_.peer_node_id -eq $nodeCID -and $_.relay_node_id -eq $nodeRID }) $nodeAReplacementLeases = @(Get-OptionalProperty -Object $refreshedNodeAConfig.synthetic_mesh_config -PropertyName "rendezvous_leases" | Where-Object { $_.peer_node_id -eq $nodeCID -and $_.relay_node_id -eq $nodeSID -and $_.reason -eq "stale_relay_replacement" }) $nodeAWithdrawnStaleLeases = @(Get-OptionalProperty -Object $refreshedNodeAConfig.synthetic_mesh_config -PropertyName "rendezvous_leases" | Where-Object { $_.peer_node_id -eq $nodeCID -and $_.relay_node_id -eq $nodeRID }) $nodeARelayPolicy = Get-OptionalProperty -Object $refreshedNodeAConfig.synthetic_mesh_config -PropertyName "rendezvous_relay_policy" $nodeAInitialPathDecisionReport = Get-OptionalProperty -Object $configs["a"].synthetic_mesh_config -PropertyName "route_path_decisions" $nodeAConfigPathDecisionReport = Get-OptionalProperty -Object $refreshedNodeAConfig.synthetic_mesh_config -PropertyName "route_path_decisions" $nodeAReportedPathDecisions = @(Get-OptionalProperty -Object $nodeAPathDecisionReport -PropertyName "decisions") $nodeAReportedReplacementPathDecisions = @($nodeAReportedPathDecisions | Where-Object { $_.route_id -eq $rendezvousIntent.route_intent.id -and $_.selected_relay_id -eq $nodeSID -and $_.decision_source -eq "stale_relay_replacement" -and (@($_.effective_hops) -contains $nodeSID) -and -not (@($_.effective_hops) -contains $nodeRID) }) $nodeLogs = @{} foreach ($key in @("a", "r", "b", "c", "idle")) { $nodeLogs[$key] = Invoke-RemoteDockerText -Arguments @("logs", "--tail", "120", "$nodePrefix$key") } $backendLogs = Invoke-RemoteDockerText -Arguments @("logs", "--tail", "80", $backendName) $meshLinks = @($meshSnapshot.mesh_links) $routeHealthLinks = @($meshSnapshot.route_health_links) $directHealth = @($meshSnapshot.direct_health) $rendezvousHealth = @($meshSnapshot.rendezvous_health) $replacementRouteHealth = @($meshSnapshot.replacement_route_health) $managerLinks = @($meshLinks | Where-Object { $_.metadata.observation_type -eq "peer_connection_manager" }) $relayControlLinks = @($managerLinks | Where-Object { $_.source_node_id -eq $nodeAID -and $_.target_node_id -eq $nodeCID -and $_.link_status -eq "reachable" -and $_.metadata.transport_mode -eq "relay_control" -and $_.metadata.rendezvous_resolved -eq $true -and $_.metadata.relay_candidate -eq $true -and $_.metadata.connection_state -eq "relay_ready" }) $replacementRelayControlLinks = @($relayControlLinks | Where-Object { $_.metadata.relay_node_id -eq $nodeSID }) $replacementRelayReadyFromLeaseReport = ( $nodeAReportedReplacementLeases.Count -gt 0 -and (Get-OptionalProperty -Object $nodeAReportedReplacementLeases[0] -PropertyName "relay_ready") -eq $true -and (Get-OptionalProperty -Object $nodeAReportedReplacementLeases[0] -PropertyName "connection_state") -eq "relay_ready" ) $nodeALog = $nodeLogs["a"] -join "`n" $directRouteDelivered = $nodeALog -match ('"event":"fabric_route_delivery_succeeded","route_id":"' + [regex]::Escape($directIntent.route_intent.id) + '"') $leaseReportBoundaryFlagsDisabled = $true foreach ($report in @($nodeALeaseReport, $nodeRLeaseReport, $nodeSLeaseReport, $nodeCLeaseReport)) { if ($null -eq $report -or (Get-OptionalProperty -Object $report -PropertyName "control_plane_only") -ne $true -or (Get-OptionalProperty -Object $report -PropertyName "relay_payload_forwarding") -ne $false -or (Get-OptionalProperty -Object $report -PropertyName "production_payload_forwarding") -ne $false -or (Get-OptionalProperty -Object $report -PropertyName "service_workload_traffic") -ne $false) { $leaseReportBoundaryFlagsDisabled = $false } } $pathDecisionBoundaryFlagsDisabled = ( $null -ne $nodeAPathDecisionReport -and (Get-OptionalProperty -Object $nodeAPathDecisionReport -PropertyName "control_plane_only") -eq $true -and (Get-OptionalProperty -Object $nodeAPathDecisionReport -PropertyName "production_payload_forwarding") -eq $false -and (Get-OptionalProperty -Object $nodeAPathDecisionReport -PropertyName "service_workload_traffic") -eq $false -and (Get-OptionalProperty -Object $nodeAPathDecisionReport -PropertyName "route_path_forwarding_runtime") -eq $false ) $routeGenerationBoundaryFlagsDisabled = ( $null -ne $nodeARouteGenerationReport -and (Get-OptionalProperty -Object $nodeARouteGenerationReport -PropertyName "control_plane_only") -eq $true -and (Get-OptionalProperty -Object $nodeARouteGenerationReport -PropertyName "production_payload_forwarding") -eq $false -and (Get-OptionalProperty -Object $nodeARouteGenerationReport -PropertyName "service_workload_traffic") -eq $false -and (Get-OptionalProperty -Object $nodeARouteGenerationReport -PropertyName "route_path_forwarding_runtime") -eq $false ) $nodeAWithdrawnDecisionCount = Get-OptionalProperty -Object $nodeARouteGenerationReport -PropertyName "withdrawn_decision_count" $nodeATotalWithdrawnDecisionCount = Get-OptionalProperty -Object $nodeARouteGenerationReport -PropertyName "total_withdrawn_decision_count" $nodeAWithdrawnDecisions = @(Get-OptionalProperty -Object $nodeARouteGenerationReport -PropertyName "withdrawn_decisions") $routeHealthConfigBoundaryFlagsDisabled = ( $null -ne $nodeARouteHealthConfigReport -and (Get-OptionalProperty -Object $nodeARouteHealthConfigReport -PropertyName "control_plane_only") -eq $true -and (Get-OptionalProperty -Object $nodeARouteHealthConfigReport -PropertyName "route_health_only") -eq $true -and (Get-OptionalProperty -Object $nodeARouteHealthConfigReport -PropertyName "synthetic_route_health_route_path_runtime") -eq $true -and (Get-OptionalProperty -Object $nodeARouteHealthConfigReport -PropertyName "production_route_path_forwarding_runtime") -eq $false -and (Get-OptionalProperty -Object $nodeARouteHealthConfigReport -PropertyName "production_payload_forwarding") -eq $false -and (Get-OptionalProperty -Object $nodeARouteHealthConfigReport -PropertyName "service_workload_traffic") -eq $false -and (Get-OptionalProperty -Object $nodeARouteHealthConfigReport -PropertyName "test_service_route_config_changed") -eq $false ) $routeHealthFeedbackRefreshBoundaryFlagsDisabled = ( $null -ne $nodeARouteHealthFeedbackRefreshReport -and (Get-OptionalProperty -Object $nodeARouteHealthFeedbackRefreshReport -PropertyName "control_plane_only") -eq $true -and (Get-OptionalProperty -Object $nodeARouteHealthFeedbackRefreshReport -PropertyName "route_health_only") -eq $true -and (Get-OptionalProperty -Object $nodeARouteHealthFeedbackRefreshReport -PropertyName "production_payload_forwarding") -eq $false -and (Get-OptionalProperty -Object $nodeARouteHealthFeedbackRefreshReport -PropertyName "service_workload_traffic") -eq $false ) $passMatrix = [ordered]@{ backend_ready = $true platform_owner_login = [bool]$actorUserID cluster_created = [bool]$clusterID fabric_testing_flags_enabled = $true node_a_scoped_config_enabled = $configs["a"].synthetic_mesh_config.enabled -eq $true node_a_has_direct_and_rendezvous_routes = @($configs["a"].synthetic_mesh_config.routes).Count -eq 2 node_a_has_outbound_peer_candidate = @($nodeAPeerCandidates).Count -gt 0 node_a_has_initial_stale_rendezvous_lease = $nodeAInitialStaleLeases.Count -gt 0 node_a_initial_lease_is_control_plane_only = ($nodeAInitialStaleLeases.Count -gt 0 -and $nodeAInitialStaleLeases[0].control_plane_only -eq $true) node_a_initial_lease_uses_relay_control = ($nodeAInitialStaleLeases.Count -gt 0 -and $nodeAInitialStaleLeases[0].transport -eq "relay_control") node_a_initial_auto_alt_relay_candidate = $nodeAInitialAltLeases.Count -gt 0 node_a_initial_path_decision_report = (Get-OptionalProperty -Object $nodeAInitialPathDecisionReport -PropertyName "schema_version") -eq "c17z18.route_path_decisions.v1" node_a_report_replacement_lease_uses_alt_relay = ($nodeAReportedReplacementLeases.Count -gt 0 -and $nodeAReportedReplacementLeases[0].relay_node_id -eq $nodeSID) node_a_report_stale_relay_lease_withdrawn = $nodeAReportedStaleRelayLeases.Count -eq 0 node_a_report_replacement_reason = ($nodeAReportedReplacementLeases.Count -gt 0 -and $nodeAReportedReplacementLeases[0].reason -eq "stale_relay_replacement") node_a_reports_c17z18_path_decisions = (Get-OptionalProperty -Object $nodeAPathDecisionReport -PropertyName "schema_version") -eq "c17z18.mesh_route_path_decision_report.v1" node_a_path_decision_replacement_count = (Get-OptionalProperty -Object $nodeAPathDecisionReport -PropertyName "replacement_decision_count") -gt 0 node_a_path_decision_uses_alt_relay = $nodeAReportedReplacementPathDecisions.Count -gt 0 node_a_path_decision_boundary_flags_disabled = $pathDecisionBoundaryFlagsDisabled node_a_reports_c17z18_route_generation = (Get-OptionalProperty -Object $nodeARouteGenerationReport -PropertyName "schema_version") -eq "c17z18.mesh_route_generation_report.v1" node_a_route_generation_active = (Get-OptionalProperty -Object $nodeARouteGenerationReport -PropertyName "active_decision_count") -gt 0 node_a_route_generation_applied = (Get-OptionalProperty -Object $nodeARouteGenerationReport -PropertyName "applied_decision_count") -gt 0 node_a_route_generation_withdrawn = ($nodeAWithdrawnDecisionCount -gt 0 -or $nodeATotalWithdrawnDecisionCount -gt 0 -or $nodeAWithdrawnDecisions.Count -gt 0) node_a_route_generation_changed = (Get-OptionalProperty -Object $nodeARouteGenerationReport -PropertyName "generation_changed") -eq $true node_a_route_generation_boundary_flags_disabled = $routeGenerationBoundaryFlagsDisabled node_a_reports_c17z20_route_health_config = (Get-OptionalProperty -Object $nodeARouteHealthConfigReport -PropertyName "schema_version") -eq "c17z20.mesh_route_health_config_report.v1" node_a_reports_c17z20_route_health_feedback_refresh = (Get-OptionalProperty -Object $nodeARouteHealthFeedbackRefreshReport -PropertyName "schema_version") -eq "c17z20.mesh_route_health_feedback_refresh_report.v1" node_a_route_health_feedback_refresh_supported = (Get-OptionalProperty -Object $nodeARouteHealthFeedbackRefreshReport -PropertyName "feedback_refresh_supported") -eq $true node_a_route_health_feedback_refresh_boundary_flags_disabled = $routeHealthFeedbackRefreshBoundaryFlagsDisabled node_a_route_health_config_applied = (Get-OptionalProperty -Object $nodeARouteHealthConfigReport -PropertyName "route_path_decision_applied_count") -gt 0 node_a_route_health_config_replacement = (Get-OptionalProperty -Object $nodeARouteHealthConfigReport -PropertyName "replacement_route_health_route_count") -gt 0 node_a_route_health_config_boundary_flags_disabled = $routeHealthConfigBoundaryFlagsDisabled node_a_route_health_uses_effective_alt_relay = $replacementRouteHealth.Count -gt 0 node_a_route_health_has_no_effective_path_drift = ($replacementRouteHealth.Count -gt 0 -and (Get-OptionalProperty -Object $replacementRouteHealth[0].metadata -PropertyName "route_path_drift_detected") -eq $false) node_a_route_health_selected_alt_next_hop = $nodeALog -match ('"event":"fabric_route_selected","route_id":"' + [regex]::Escape($rendezvousIntent.route_intent.id) + '".*"next_node_id":"' + [regex]::Escape($nodeSID) + '"') outbound_node_has_relay_lease = $nodeCLeases.Count -gt 0 direct_baseline_health_reported = $directRouteDelivered node_a_loaded_c17z20_control_plane_config = ($nodeAReplacementLeases.Count -gt 0 -and (Get-OptionalProperty -Object $nodeARouteHealthConfigReport -PropertyName "schema_version") -eq "c17z20.mesh_route_health_config_report.v1") node_a_resolved_waiting_rendezvous = ($replacementRelayControlLinks.Count -gt 0 -or $replacementRelayReadyFromLeaseReport) relay_control_manager_link_reachable = ($replacementRelayControlLinks.Count -gt 0 -or $replacementRelayReadyFromLeaseReport) relay_ready_recorded = ($replacementRelayControlLinks.Count -gt 0 -or $replacementRelayReadyFromLeaseReport) node_a_reports_c17z18_lease_telemetry = (Get-OptionalProperty -Object $nodeALeaseReport -PropertyName "schema_version") -eq "c17z18.mesh_rendezvous_lease_report.v1" node_a_lease_report_entry_observer = (Get-OptionalProperty -Object $nodeALeaseReport -PropertyName "entry_observer_count") -gt 0 node_a_lease_report_relay_ready = (Get-OptionalProperty -Object $nodeALeaseReport -PropertyName "relay_control_ready_count") -gt 0 node_a_lease_report_refresh_contract = (Get-OptionalProperty -Object $nodeALeaseReport -PropertyName "refresh_contract") -eq "node_scoped_synthetic_config_get" stale_relay_refresh_succeeded_on_cluster = ( ((Get-OptionalProperty -Object $nodeRLeaseReport -PropertyName "last_refresh_reason") -eq "stale_relay" -and (Get-OptionalProperty -Object $nodeRLeaseReport -PropertyName "refresh_success_count") -gt 0) -or ((Get-OptionalProperty -Object $nodeSLeaseReport -PropertyName "last_refresh_reason") -eq "stale_relay" -and (Get-OptionalProperty -Object $nodeSLeaseReport -PropertyName "refresh_success_count") -gt 0) ) alt_relay_node_reports_admitted_relay_lease = (Get-OptionalProperty -Object $nodeSLeaseReport -PropertyName "admitted_as_relay_count") -gt 0 outbound_node_reports_admitted_peer_lease = (Get-OptionalProperty -Object $nodeCLeaseReport -PropertyName "admitted_as_peer_count") -gt 0 lease_telemetry_boundary_flags_disabled = $leaseReportBoundaryFlagsDisabled production_forwarding_disabled = ( $configs["a"].synthetic_mesh_config.production_forwarding -eq $false -and $configs["r"].synthetic_mesh_config.production_forwarding -eq $false -and $configs["b"].synthetic_mesh_config.production_forwarding -eq $false -and $configs["c"].synthetic_mesh_config.production_forwarding -eq $false -and $configs["idle"].synthetic_mesh_config.production_forwarding -eq $false ) } $result = [pscustomobject]@{ stage = "C17Z18 rendezvous relay replacement docker-test smoke" run_id = $runId backend_base_url = $backendPublicBaseUrl cluster_id = $clusterID node_ids = @{ a = $nodes["a"].id r = $nodes["r"].id s = $nodes["idle"].id b = $nodes["b"].id c = $nodes["c"].id idle = $nodes["idle"].id } route_intents = @{ direct = $directIntent.route_intent.id rendezvous = $rendezvousIntent.route_intent.id } scoped_config_route_counts = @{ a = @($configs["a"].synthetic_mesh_config.routes).Count r = @($configs["r"].synthetic_mesh_config.routes).Count b = @($configs["b"].synthetic_mesh_config.routes).Count c = @($configs["c"].synthetic_mesh_config.routes).Count idle = @($configs["idle"].synthetic_mesh_config.routes).Count } rendezvous_lease_counts = @{ a = Get-OptionalArrayCount -Object $configs["a"].synthetic_mesh_config -PropertyName "rendezvous_leases" r = Get-OptionalArrayCount -Object $configs["r"].synthetic_mesh_config -PropertyName "rendezvous_leases" b = Get-OptionalArrayCount -Object $configs["b"].synthetic_mesh_config -PropertyName "rendezvous_leases" c = Get-OptionalArrayCount -Object $configs["c"].synthetic_mesh_config -PropertyName "rendezvous_leases" idle = Get-OptionalArrayCount -Object $configs["idle"].synthetic_mesh_config -PropertyName "rendezvous_leases" } node_a_initial_stale_rendezvous_lease = $nodeAInitialStaleLeases | Select-Object -First 1 node_a_reported_replacement_rendezvous_lease = $nodeAReportedReplacementLeases | Select-Object -First 1 node_a_current_replacement_rendezvous_lease = $nodeAReplacementLeases | Select-Object -First 1 node_a_current_rendezvous_relay_policy = $nodeARelayPolicy node_a_initial_route_path_decisions = $nodeAInitialPathDecisionReport node_a_current_route_path_decisions = $nodeAConfigPathDecisionReport node_a_reported_route_path_decision = $nodeAReportedReplacementPathDecisions | Select-Object -First 1 route_path_decision_reports = @{ a = $nodeAPathDecisionReport } route_generation_reports = @{ a = $nodeARouteGenerationReport } route_health_config_reports = @{ a = $nodeARouteHealthConfigReport } route_health_feedback_refresh_reports = @{ a = $nodeARouteHealthFeedbackRefreshReport } rendezvous_lease_reports = @{ a = $nodeALeaseReport r = $nodeRLeaseReport s = $nodeSLeaseReport c = $nodeCLeaseReport } mesh_link_count = $meshLinks.Count route_health_count = $routeHealthLinks.Count peer_connection_manager_link_count = $managerLinks.Count relay_control_link_count = $relayControlLinks.Count direct_route_delivery_succeeded = $directRouteDelivered pass_matrix = $passMatrix direct_route_health = $directHealth | Select-Object -First 3 replacement_route_health = $replacementRouteHealth | Select-Object -First 3 relay_control_links = $relayControlLinks | Select-Object -First 5 replacement_relay_control_links = $replacementRelayControlLinks | Select-Object -First 5 cluster_summaries = $summary.cluster_summaries backend_log_tail = $backendLogs node_log_tail = $nodeLogs containers_left_running = [bool]$KeepRunning } $failed = @($passMatrix.GetEnumerator() | Where-Object { -not $_.Value }) $result | ConvertTo-Json -Depth 60 if ($failed.Count -gt 0) { throw "C17Z18 rendezvous relay replacement smoke failed: $($failed.Name -join ', ')" } if (-not $KeepRunning) { Write-Host "Cleaning up C17Z18 containers..." Remove-C17Z12Containers Invoke-RemoteShell -Command "rm -rf '$remoteBuildDir'" }