param( [string]$DockerSshAlias = "test-docker", [string]$NodeAgentImageTag = "rap-node-agent:c17h-synthetic-smoke", [string]$AdminEmail = "fabric-owner-c17h@example.local", [string]$AdminPassword = "SmokePass!123", [int]$ApiPort = 18080, [int]$MeshBasePort = 19081, [switch]$KeepRunning ) Set-StrictMode -Version Latest $ErrorActionPreference = "Stop" $backendPublicBaseUrl = "http://192.168.200.61:$ApiPort/api/v1" $backendContainerBaseUrl = "http://127.0.0.1:$ApiPort/api/v1" $runId = "c17h-ssh-" + (Get-Date -Format "yyyyMMdd-HHmmss") $nodePrefix = "rap_c17h_node_" function Invoke-RemoteDocker { param([string[]]$Arguments) & ssh $DockerSshAlias docker @Arguments if ($LASTEXITCODE -ne 0) { throw "ssh $DockerSshAlias docker $($Arguments -join ' ') failed with exit code $LASTEXITCODE" } } function Invoke-RemoteDockerText { param([string[]]$Arguments) $previousErrorActionPreference = $ErrorActionPreference $ErrorActionPreference = "Continue" try { $output = & ssh $DockerSshAlias docker @Arguments 2>&1 | ForEach-Object { $_.ToString() } } finally { $ErrorActionPreference = $previousErrorActionPreference } if ($LASTEXITCODE -ne 0) { throw "ssh $DockerSshAlias docker $($Arguments -join ' ') failed with exit code $LASTEXITCODE" } return $output } function Invoke-RemoteShell { param([string]$Command) & ssh $DockerSshAlias $Command if ($LASTEXITCODE -ne 0) { throw "ssh $DockerSshAlias $Command failed with exit code $LASTEXITCODE" } } function Send-RemoteFile { param( [string]$RemotePath, [string]$Content ) $Content | & ssh $DockerSshAlias "cat > '$RemotePath'" if ($LASTEXITCODE -ne 0) { throw "write remote file failed: $RemotePath" } } function Invoke-Api { param( [string]$Method, [string]$Path, [object]$Body = $null ) $uri = "$backendPublicBaseUrl$Path" if ($null -eq $Body) { return Invoke-RestMethod -Method $Method -Uri $uri -TimeoutSec 30 } return Invoke-RestMethod -Method $Method -Uri $uri -ContentType "application/json" -Body ($Body | ConvertTo-Json -Depth 40) -TimeoutSec 30 } function Wait-HttpReady { param([string]$Url) for ($i = 0; $i -lt 60; $i++) { try { $response = Invoke-WebRequest -UseBasicParsing -Uri $Url -TimeoutSec 2 if ($response.StatusCode -ge 200 -and $response.StatusCode -lt 300) { return } } catch { Start-Sleep -Seconds 1 } } throw "Timed out waiting for $Url" } function Remove-NodeContainers { foreach ($key in @("a", "b", "c", "r", "idle")) { & ssh $DockerSshAlias docker rm -f "$nodePrefix$key" 2>$null | Out-Null } } Wait-HttpReady -Url "http://192.168.200.61:$ApiPort/readyz" $login = Invoke-Api -Method Post -Path "/auth/login" -Body @{ email = $AdminEmail password = $AdminPassword device_fingerprint = "c17h-ssh-smoke-device" device_label = "C17H SSH synthetic smoke" trust_device = $true } $actorUserID = $login.user.id $cluster = Invoke-Api -Method Post -Path "/clusters/" -Body @{ actor_user_id = $actorUserID slug = "c17h-ssh-$((New-Guid).Guid.Substring(0, 8))" name = "C17H SSH Synthetic Mesh Smoke" region = "docker-test" metadata = @{ stage = "c17h" run_id = $runId production_forwarding = $false created_by = "c17h-multi-agent-synthetic-smoke-ssh.ps1" } } $clusterID = $cluster.cluster.id Invoke-Api -Method Put -Path "/fabric/testing-flags" -Body @{ actor_user_id = $actorUserID scope_type = "platform" scope_id = $null cluster_id = $null enabled = $true telemetry_enabled = $true synthetic_links_enabled = $true history_retention_hours = 24 metadata = @{ stage = "c17h" run_id = $runId production_forwarding = $false service_workload_traffic = $false } } | Out-Null $joinToken = Invoke-Api -Method Post -Path "/clusters/$clusterID/join-tokens" -Body @{ actor_user_id = $actorUserID scope = @{ purpose = "c17h-synthetic-smoke"; roles = @("core-mesh", "relay-node") } expires_at = (Get-Date).ToUniversalTime().AddHours(2).ToString("o") max_uses = 5 } $nodeSpecs = @( @{ key = "a"; name = "c17h-node-a"; roles = @("core-mesh"); port = $MeshBasePort }, @{ key = "r"; name = "c17h-node-r"; roles = @("core-mesh", "relay-node"); port = ($MeshBasePort + 1) }, @{ key = "b"; name = "c17h-node-b"; roles = @("core-mesh"); port = ($MeshBasePort + 2) }, @{ key = "c"; name = "c17h-node-c"; roles = @("core-mesh"); port = ($MeshBasePort + 3) }, @{ key = "idle"; name = "c17h-node-idle"; roles = @("core-mesh"); port = ($MeshBasePort + 4) } ) $nodes = @{} foreach ($spec in $nodeSpecs) { $fingerprint = "c17h-fp-$($spec.key)-$([guid]::NewGuid().ToString('N'))" $publicKey = "c17h-pub-$($spec.key)-$([guid]::NewGuid().ToString('N'))" $joinRequest = Invoke-Api -Method Post -Path "/clusters/$clusterID/join-requests" -Body @{ join_token = $joinToken.join_token.token node_name = $spec.name node_fingerprint = $fingerprint public_key = $publicKey reported_capabilities = @{ can_accept_node_ingress = $true can_route_mesh = $true testing_node = $true } reported_facts = @{ os = "linux" runtime = "docker-test" stage = "c17h" run_id = $runId } requested_roles = $spec.roles } $approved = Invoke-Api -Method Post -Path "/clusters/$clusterID/join-requests/$($joinRequest.join_request.id)/approve" -Body @{ actor_user_id = $actorUserID node_key = $fingerprint ownership_type = "platform_managed" owner_organization_id = $null } $nodeID = $approved.node_bootstrap.node_id foreach ($role in $spec.roles) { Invoke-Api -Method Post -Path "/clusters/$clusterID/nodes/$nodeID/roles" -Body @{ actor_user_id = $actorUserID role = $role status = "active" policy = @{ stage = "c17h" run_id = $runId synthetic_only = $true } } | Out-Null } $nodes[$spec.key] = [pscustomobject]@{ id = $nodeID name = $spec.name fingerprint = $fingerprint public_key = $publicKey port = $spec.port roles = $spec.roles } } $nodeAID = $nodes["a"].id $nodeRID = $nodes["r"].id $nodeBID = $nodes["b"].id $nodeCID = $nodes["c"].id $routeExpiresAt = (Get-Date).ToUniversalTime().AddHours(2).ToString("o") $peerEndpointsDirect = @{} $peerEndpointsDirect[$nodeAID] = "http://127.0.0.1:$($nodes["a"].port)" $peerEndpointsDirect[$nodeBID] = "http://127.0.0.1:$($nodes["b"].port)" $peerEndpointsRelay = @{} $peerEndpointsRelay[$nodeAID] = "http://127.0.0.1:$($nodes["a"].port)" $peerEndpointsRelay[$nodeRID] = "http://127.0.0.1:$($nodes["r"].port)" $peerEndpointsRelay[$nodeCID] = "http://127.0.0.1:$($nodes["c"].port)" $directIntent = Invoke-Api -Method Post -Path "/clusters/$clusterID/mesh/route-intents" -Body @{ actor_user_id = $actorUserID source_selector = @{ node_id = $nodeAID } destination_selector = @{ node_id = $nodeBID } service_class = "control" priority = 10 policy = @{ synthetic_enabled = $true peer_endpoints = $peerEndpointsDirect hops = @($nodeAID, $nodeBID) allowed_channels = @("fabric_control", "route_control") max_ttl = 4 max_hops = 4 expires_at = $routeExpiresAt route_version = "$runId-direct" policy_version = "$runId-policy" peer_directory_version = "$runId-peers" production_forwarding = $false } } $relayIntent = Invoke-Api -Method Post -Path "/clusters/$clusterID/mesh/route-intents" -Body @{ actor_user_id = $actorUserID source_selector = @{ node_id = $nodeAID } destination_selector = @{ node_id = $nodeCID } service_class = "control" priority = 20 policy = @{ synthetic_enabled = $true peer_endpoints = $peerEndpointsRelay hops = @($nodeAID, $nodeRID, $nodeCID) allowed_channels = @("fabric_control", "route_control") max_ttl = 6 max_hops = 6 expires_at = $routeExpiresAt route_version = "$runId-relay" policy_version = "$runId-policy" peer_directory_version = "$runId-peers" production_forwarding = $false } } $configs = @{} foreach ($key in @("a", "r", "b", "c", "idle")) { $configs[$key] = Invoke-Api -Method Get -Path "/clusters/$clusterID/nodes/$($nodes[$key].id)/mesh/synthetic-config" } Remove-NodeContainers foreach ($key in @("a", "r", "b", "c", "idle")) { $node = $nodes[$key] $containerName = "$nodePrefix$key" $remoteStateDir = "/tmp/$runId-$key" Invoke-RemoteShell -Command "rm -rf '$remoteStateDir' && mkdir -p '$remoteStateDir'" $identity = @{ node_id = $node.id cluster_id = $clusterID node_name = $node.name node_fingerprint = $node.fingerprint public_key = $node.public_key identity_status = "active" created_at = (Get-Date).ToUniversalTime().ToString("o") updated_at = (Get-Date).ToUniversalTime().ToString("o") } | ConvertTo-Json -Depth 10 Send-RemoteFile -RemotePath "$remoteStateDir/identity.json" -Content $identity Invoke-RemoteDocker -Arguments @( "create", "--name", $containerName, "--network", "host", "-e", "RAP_BACKEND_URL=$backendContainerBaseUrl", "-e", "RAP_NODE_STATE_DIR=/tmp/state", "-e", "RAP_HEARTBEAT_INTERVAL_SECONDS=5", "-e", "RAP_MESH_SYNTHETIC_RUNTIME_ENABLED=true", "-e", "RAP_MESH_LISTEN_ADDR=0.0.0.0:$($node.port)", $NodeAgentImageTag, "-backend-url", $backendContainerBaseUrl, "-state-dir", "/tmp/state", "-heartbeat-interval", "5s", "-mesh-synthetic-runtime-enabled", "-mesh-listen-addr", "0.0.0.0:$($node.port)" ) | Out-Null Invoke-RemoteDocker -Arguments @("cp", "$remoteStateDir/.", "$containerName`:/tmp/state") Invoke-RemoteDocker -Arguments @("start", $containerName) | Out-Null } Start-Sleep -Seconds 25 $links = Invoke-Api -Method Get -Path "/clusters/$clusterID/mesh/links?actor_user_id=$actorUserID" $summary = Invoke-Api -Method Get -Path "/cluster-admin-summaries?actor_user_id=$actorUserID" $nodeLogs = @{} foreach ($key in @("a", "r", "b", "c", "idle")) { $nodeLogs[$key] = Invoke-RemoteDockerText -Arguments @("logs", "$nodePrefix$key") } $backendLogs = Invoke-RemoteDockerText -Arguments @("logs", "--tail", "50", "rap_c17h_backend") $meshLinks = @($links.mesh_links) $routeHealthLinks = @($meshLinks | Where-Object { $_.metadata.observation_type -eq "synthetic_route_health" -and $_.metadata.config_source -eq "control_plane" }) $directHealth = @($routeHealthLinks | Where-Object { $_.metadata.route_id -eq $directIntent.route_intent.id -and $_.link_status -eq "reachable" }) $relayHealth = @($routeHealthLinks | Where-Object { $_.metadata.route_id -eq $relayIntent.route_intent.id -and $_.link_status -eq "reachable" }) $passMatrix = [ordered]@{ backend_ready = $true platform_owner_login = [bool]$actorUserID cluster_created = [bool]$clusterID fabric_testing_flags_enabled = $true node_a_scoped_config_enabled = $configs["a"].synthetic_mesh_config.enabled -eq $true node_a_has_direct_and_relay_routes = @($configs["a"].synthetic_mesh_config.routes).Count -eq 2 relay_node_has_only_relay_route = @($configs["r"].synthetic_mesh_config.routes).Count -eq 1 direct_destination_node_has_only_direct_route = @($configs["b"].synthetic_mesh_config.routes).Count -eq 1 relay_destination_node_has_only_relay_route = @($configs["c"].synthetic_mesh_config.routes).Count -eq 1 idle_node_has_no_routes = @($configs["idle"].synthetic_mesh_config.routes).Count -eq 0 control_plane_config_used = (@("a", "r", "b", "c", "idle") | Where-Object { ($nodeLogs[$_] -join "`n") -match "source=control_plane" }).Count -eq 5 direct_route_health_reported = $directHealth.Count -gt 0 relay_route_health_reported = $relayHealth.Count -gt 0 production_forwarding_disabled = ( $configs["a"].synthetic_mesh_config.production_forwarding -eq $false -and $configs["r"].synthetic_mesh_config.production_forwarding -eq $false -and $configs["b"].synthetic_mesh_config.production_forwarding -eq $false -and $configs["c"].synthetic_mesh_config.production_forwarding -eq $false -and $configs["idle"].synthetic_mesh_config.production_forwarding -eq $false ) } $result = [pscustomobject]@{ stage = "C17H deployed multi-agent synthetic config smoke" run_id = $runId backend_base_url = $backendPublicBaseUrl cluster_id = $clusterID node_ids = @{ a = $nodes["a"].id r = $nodes["r"].id b = $nodes["b"].id c = $nodes["c"].id idle = $nodes["idle"].id } route_intents = @{ direct = $directIntent.route_intent.id relay = $relayIntent.route_intent.id } scoped_config_route_counts = @{ a = @($configs["a"].synthetic_mesh_config.routes).Count r = @($configs["r"].synthetic_mesh_config.routes).Count b = @($configs["b"].synthetic_mesh_config.routes).Count c = @($configs["c"].synthetic_mesh_config.routes).Count idle = @($configs["idle"].synthetic_mesh_config.routes).Count } mesh_link_count = $meshLinks.Count route_health_count = $routeHealthLinks.Count pass_matrix = $passMatrix direct_route_health = $directHealth | Select-Object -First 3 relay_route_health = $relayHealth | Select-Object -First 3 cluster_summaries = $summary.cluster_summaries backend_log_tail = $backendLogs node_log_tail = $nodeLogs containers_left_running = [bool]$KeepRunning } $failed = @($passMatrix.GetEnumerator() | Where-Object { -not $_.Value }) $result | ConvertTo-Json -Depth 50 if ($failed.Count -gt 0) { throw "C17H smoke failed: $($failed.Name -join ', ')" } if (-not $KeepRunning) { Remove-NodeContainers }