param( [string]$DockerContext = "test-ubuntu", [string]$BackendImageTag = "rap-backend:c17h-synthetic-smoke", [string]$NodeAgentImageTag = "rap-node-agent:c17h-synthetic-smoke", [string]$AdminEmail = "fabric-owner-c17h@example.local", [string]$AdminPassword = "SmokePass!123", [int]$ApiPort = 18080, [int]$PostgresPort = 15432, [int]$RedisPort = 16379, [int]$MeshBasePort = 19081, [switch]$KeepRunning ) Set-StrictMode -Version Latest $ErrorActionPreference = "Stop" $repoRoot = (Resolve-Path (Join-Path $PSScriptRoot "..\..")).ProviderPath $backendBaseUrl = "http://127.0.0.1:$ApiPort/api/v1" $backendPublicBaseUrl = "http://192.168.200.61:$ApiPort/api/v1" $runId = "c17h-" + (Get-Date -Format "yyyyMMdd-HHmmss") $postgresName = "rap_c17h_postgres" $redisName = "rap_c17h_redis" $backendName = "rap_c17h_backend" $nodePrefix = "rap_c17h_node_" function Invoke-Docker { param([string[]]$Arguments) docker --context $DockerContext @Arguments if ($LASTEXITCODE -ne 0) { throw "docker --context $DockerContext $($Arguments -join ' ') failed with exit code $LASTEXITCODE" } } function Invoke-DockerText { param([string[]]$Arguments) $output = docker --context $DockerContext @Arguments 2>&1 | ForEach-Object { $_.ToString() } if ($LASTEXITCODE -ne 0) { throw "docker --context $DockerContext $($Arguments -join ' ') failed with exit code $LASTEXITCODE" } return $output } function Invoke-Api { param( [string]$Method, [string]$Path, [object]$Body = $null ) $uri = "$backendPublicBaseUrl$Path" if ($null -eq $Body) { return Invoke-RestMethod -Method $Method -Uri $uri -TimeoutSec 30 } return Invoke-RestMethod -Method $Method -Uri $uri -ContentType "application/json" -Body ($Body | ConvertTo-Json -Depth 30) -TimeoutSec 30 } function Wait-HttpReady { param([string]$Url) for ($i = 0; $i -lt 60; $i++) { try { $response = Invoke-WebRequest -UseBasicParsing -Uri $Url -TimeoutSec 2 if ($response.StatusCode -ge 200 -and $response.StatusCode -lt 300) { return } } catch { Start-Sleep -Seconds 1 } } throw "Timed out waiting for $Url" } function Invoke-PostgresSql { param([string]$Sql) $Sql | docker --context $DockerContext exec -i $postgresName psql -U rap_user -d remote_access_platform -v ON_ERROR_STOP=1 -f - if ($LASTEXITCODE -ne 0) { throw "psql command failed" } } function Remove-C17HContainers { $names = @($backendName, $postgresName, $redisName) foreach ($letter in @("a", "b", "c", "r", "idle")) { $names += "$nodePrefix$letter" } foreach ($name in $names) { docker --context $DockerContext rm -f $name 2>$null | Out-Null } } Write-Host "C17H smoke run: $runId" Write-Host "Using Docker context: $DockerContext" Remove-C17HContainers Write-Host "Building backend image..." Invoke-Docker -Arguments @("build", "-f", "$repoRoot\backend\Dockerfile", "-t", $BackendImageTag, "$repoRoot\backend") Write-Host "Building node-agent image..." Invoke-Docker -Arguments @("build", "-f", "$repoRoot\agents\rap-node-agent\Dockerfile", "-t", $NodeAgentImageTag, "$repoRoot") Write-Host "Starting PostgreSQL and Redis..." Invoke-Docker -Arguments @( "run", "-d", "--name", $postgresName, "-e", "POSTGRES_DB=remote_access_platform", "-e", "POSTGRES_USER=rap_user", "-e", "POSTGRES_PASSWORD=rap_password", "-p", "$PostgresPort`:5432", "postgres:16" ) Invoke-Docker -Arguments @( "run", "-d", "--name", $redisName, "-p", "$RedisPort`:6379", "redis:7" ) for ($i = 0; $i -lt 60; $i++) { docker --context $DockerContext exec $postgresName pg_isready -U rap_user -d remote_access_platform | Out-Null if ($LASTEXITCODE -eq 0) { break } Start-Sleep -Seconds 1 } docker --context $DockerContext exec $postgresName pg_isready -U rap_user -d remote_access_platform | Out-Null if ($LASTEXITCODE -ne 0) { throw "PostgreSQL did not become ready" } Write-Host "Applying migrations..." $migrationFiles = Get-ChildItem -Path (Join-Path $repoRoot "backend\migrations") -Filter "*.up.sql" | Sort-Object Name foreach ($migration in $migrationFiles) { Write-Host "Applying $($migration.Name)" Get-Content -Raw $migration.FullName | docker --context $DockerContext exec -i $postgresName psql -U rap_user -d remote_access_platform -v ON_ERROR_STOP=1 -f - if ($LASTEXITCODE -ne 0) { throw "Migration failed: $($migration.Name)" } } Write-Host "Seeding platform owner..." $adminUserID = [guid]::NewGuid().ToString() $adminHash = '$2a$10$AqLRexkI1yXbuiMPU6dHM.KVUhF.t..9NolyK4OOodQTyTsHyG.7u' Invoke-PostgresSql -Sql @" INSERT INTO users (id, email, password_hash, mfa_enabled, platform_role) VALUES ('$adminUserID'::uuid, '$AdminEmail', '$adminHash', FALSE, 'platform_admin') ON CONFLICT (email) DO UPDATE SET password_hash = EXCLUDED.password_hash, platform_role = 'platform_admin', updated_at = NOW(); "@ Write-Host "Starting backend..." Invoke-Docker -Arguments @( "run", "-d", "--name", $backendName, "--network", "host", "-e", "APP_NAME=rap-api", "-e", "APP_ENV=c17h-smoke", "-e", "HTTP_HOST=0.0.0.0", "-e", "HTTP_PORT=$ApiPort", "-e", "POSTGRES_DSN=postgres://rap_user:rap_password@127.0.0.1:$PostgresPort/remote_access_platform?sslmode=disable", "-e", "REDIS_ADDR=127.0.0.1:$RedisPort", "-e", "AUTH_ACCESS_TOKEN_SECRET=c17h-access-secret", "-e", "AUTH_REFRESH_HASH_SECRET=c17h-refresh-secret", $BackendImageTag ) Wait-HttpReady -Url "http://192.168.200.61:$ApiPort/readyz" Write-Host "Logging in as platform owner..." $login = Invoke-Api -Method Post -Path "/auth/login" -Body @{ email = $AdminEmail password = $AdminPassword device_fingerprint = "c17h-smoke-device" device_label = "C17H synthetic smoke" trust_device = $true } $actorUserID = $login.user.id Write-Host "Creating C17H cluster..." $cluster = Invoke-Api -Method Post -Path "/clusters/" -Body @{ actor_user_id = $actorUserID slug = "c17h-$((New-Guid).Guid.Substring(0, 8))" name = "C17H Synthetic Mesh Smoke" region = "test-docker" metadata = @{ stage = "c17h" production_forwarding = $false created_by = "c17h-multi-agent-synthetic-smoke.ps1" } } $clusterID = $cluster.cluster.id Write-Host "Enabling test-only Fabric flags..." Invoke-Api -Method Put -Path "/fabric/testing-flags" -Body @{ actor_user_id = $actorUserID scope_type = "platform" scope_id = $null cluster_id = $null enabled = $true telemetry_enabled = $true synthetic_links_enabled = $true history_retention_hours = 24 metadata = @{ stage = "c17h" production_forwarding = $false service_workload_traffic = $false } } | Out-Null $joinToken = Invoke-Api -Method Post -Path "/clusters/$clusterID/join-tokens" -Body @{ actor_user_id = $actorUserID scope = @{ purpose = "c17h-synthetic-smoke"; roles = @("core-mesh", "relay-node") } expires_at = (Get-Date).ToUniversalTime().AddHours(2).ToString("o") max_uses = 5 } $nodeSpecs = @( @{ key = "a"; name = "c17h-node-a"; roles = @("core-mesh"); port = $MeshBasePort }, @{ key = "r"; name = "c17h-node-r"; roles = @("core-mesh", "relay-node"); port = ($MeshBasePort + 1) }, @{ key = "b"; name = "c17h-node-b"; roles = @("core-mesh"); port = ($MeshBasePort + 2) }, @{ key = "c"; name = "c17h-node-c"; roles = @("core-mesh"); port = ($MeshBasePort + 3) }, @{ key = "idle"; name = "c17h-node-idle"; roles = @("core-mesh"); port = ($MeshBasePort + 4) } ) $nodes = @{} foreach ($spec in $nodeSpecs) { $fingerprint = "c17h-fp-$($spec.key)-$([guid]::NewGuid().ToString('N'))" $publicKey = "c17h-pub-$($spec.key)-$([guid]::NewGuid().ToString('N'))" $joinRequest = Invoke-Api -Method Post -Path "/clusters/$clusterID/join-requests" -Body @{ join_token = $joinToken.join_token.token node_name = $spec.name node_fingerprint = $fingerprint public_key = $publicKey reported_capabilities = @{ can_accept_node_ingress = $true can_route_mesh = $true testing_node = $true } reported_facts = @{ os = "linux" runtime = "docker-test" stage = "c17h" } requested_roles = $spec.roles } $approved = Invoke-Api -Method Post -Path "/clusters/$clusterID/join-requests/$($joinRequest.join_request.id)/approve" -Body @{ actor_user_id = $actorUserID node_key = $fingerprint ownership_type = "platform_managed" owner_organization_id = $null } $nodeID = $approved.node_bootstrap.node_id foreach ($role in $spec.roles) { Invoke-Api -Method Post -Path "/clusters/$clusterID/nodes/$nodeID/roles" -Body @{ actor_user_id = $actorUserID role = $role status = "active" policy = @{ stage = "c17h" synthetic_only = $true } } | Out-Null } $nodes[$spec.key] = [pscustomobject]@{ id = $nodeID name = $spec.name fingerprint = $fingerprint public_key = $publicKey port = $spec.port roles = $spec.roles } } $nodeAID = $nodes["a"].id $nodeRID = $nodes["r"].id $nodeBID = $nodes["b"].id $nodeCID = $nodes["c"].id $peerEndpointsDirect = @{} $peerEndpointsDirect[$nodeAID] = "http://127.0.0.1:$($nodes["a"].port)" $peerEndpointsDirect[$nodeBID] = "http://127.0.0.1:$($nodes["b"].port)" $peerEndpointsRelay = @{} $peerEndpointsRelay[$nodeAID] = "http://127.0.0.1:$($nodes["a"].port)" $peerEndpointsRelay[$nodeRID] = "http://127.0.0.1:$($nodes["r"].port)" $peerEndpointsRelay[$nodeCID] = "http://127.0.0.1:$($nodes["c"].port)" Write-Host "Creating synthetic route intents..." $directIntent = Invoke-Api -Method Post -Path "/clusters/$clusterID/mesh/route-intents" -Body @{ actor_user_id = $actorUserID source_selector = @{ node_id = $nodeAID } destination_selector = @{ node_id = $nodeBID } service_class = "control" priority = 10 policy = @{ synthetic_enabled = $true peer_endpoints = $peerEndpointsDirect hops = @($nodeAID, $nodeBID) allowed_channels = @("fabric_control", "route_control") max_ttl = 4 max_hops = 4 route_version = "$runId-direct" policy_version = "$runId-policy" peer_directory_version = "$runId-peers" production_forwarding = $false } } $relayIntent = Invoke-Api -Method Post -Path "/clusters/$clusterID/mesh/route-intents" -Body @{ actor_user_id = $actorUserID source_selector = @{ node_id = $nodeAID } destination_selector = @{ node_id = $nodeCID } service_class = "control" priority = 20 policy = @{ synthetic_enabled = $true peer_endpoints = $peerEndpointsRelay hops = @($nodeAID, $nodeRID, $nodeCID) allowed_channels = @("fabric_control", "route_control") max_ttl = 6 max_hops = 6 route_version = "$runId-relay" policy_version = "$runId-policy" peer_directory_version = "$runId-peers" production_forwarding = $false } } $configs = @{} foreach ($key in @("a", "r", "b", "c", "idle")) { $configs[$key] = Invoke-Api -Method Get -Path "/clusters/$clusterID/nodes/$($nodes[$key].id)/mesh/synthetic-config" } Write-Host "Starting node-agent containers..." foreach ($key in @("a", "r", "b", "c", "idle")) { $node = $nodes[$key] $containerName = "$nodePrefix$key" docker --context $DockerContext rm -f $containerName 2>$null | Out-Null $stateDir = Join-Path $env:TEMP "$runId-$key" New-Item -ItemType Directory -Force -Path $stateDir | Out-Null @{ node_id = $node.id cluster_id = $clusterID node_name = $node.name node_fingerprint = $node.fingerprint public_key = $node.public_key identity_status = "active" created_at = (Get-Date).ToUniversalTime().ToString("o") updated_at = (Get-Date).ToUniversalTime().ToString("o") } | ConvertTo-Json -Depth 10 | Set-Content -Encoding UTF8 -Path (Join-Path $stateDir "identity.json") Invoke-Docker -Arguments @( "create", "--name", $containerName, "--network", "host", "-e", "RAP_BACKEND_URL=$backendBaseUrl", "-e", "RAP_NODE_STATE_DIR=/tmp/state", "-e", "RAP_HEARTBEAT_INTERVAL_SECONDS=5", "-e", "RAP_MESH_SYNTHETIC_RUNTIME_ENABLED=true", "-e", "RAP_MESH_LISTEN_ADDR=0.0.0.0:$($node.port)", $NodeAgentImageTag, "-backend-url", $backendBaseUrl, "-state-dir", "/tmp/state", "-heartbeat-interval", "5s", "-mesh-synthetic-runtime-enabled", "-mesh-listen-addr", "0.0.0.0:$($node.port)" ) | Out-Null Invoke-Docker -Arguments @("cp", "$stateDir\.", "$containerName`:/tmp/state") Invoke-Docker -Arguments @("start", $containerName) | Out-Null } Write-Host "Waiting for synthetic observations..." Start-Sleep -Seconds 25 $links = Invoke-Api -Method Get -Path "/clusters/$clusterID/mesh/links?actor_user_id=$actorUserID" $summary = Invoke-Api -Method Get -Path "/cluster-admin-summaries?actor_user_id=$actorUserID" $nodeLogs = @{} foreach ($key in @("a", "r", "b", "c", "idle")) { $nodeLogs[$key] = Invoke-DockerText -Arguments @("logs", "--tail", "30", "$nodePrefix$key") } $backendLogs = Invoke-DockerText -Arguments @("logs", "--tail", "30", $backendName) $meshLinks = @($links.mesh_links) $routeHealthLinks = @($meshLinks | Where-Object { $_.metadata.observation_type -eq "synthetic_route_health" -and $_.metadata.config_source -eq "control_plane" }) $directHealth = @($routeHealthLinks | Where-Object { $_.metadata.route_id -eq $directIntent.route_intent.id -and $_.link_status -eq "reachable" }) $relayHealth = @($routeHealthLinks | Where-Object { $_.metadata.route_id -eq $relayIntent.route_intent.id -and $_.link_status -eq "reachable" }) $passMatrix = [ordered]@{ backend_ready = $true platform_owner_login = [bool]$actorUserID cluster_created = [bool]$clusterID fabric_testing_flags_enabled = $true node_a_scoped_config_enabled = $configs["a"].synthetic_mesh_config.enabled -eq $true node_a_has_direct_and_relay_routes = @($configs["a"].synthetic_mesh_config.routes).Count -eq 2 relay_node_has_only_relay_route = @($configs["r"].synthetic_mesh_config.routes).Count -eq 1 direct_destination_node_has_only_direct_route = @($configs["b"].synthetic_mesh_config.routes).Count -eq 1 relay_destination_node_has_only_relay_route = @($configs["c"].synthetic_mesh_config.routes).Count -eq 1 idle_node_has_no_routes = @($configs["idle"].synthetic_mesh_config.routes).Count -eq 0 node_agents_started = @("a", "r", "b", "c", "idle").Count -eq 5 control_plane_config_used = ($nodeLogs["a"] -join "`n") -match "control_plane" direct_route_health_reported = $directHealth.Count -gt 0 relay_route_health_reported = $relayHealth.Count -gt 0 production_forwarding_disabled = ( $configs["a"].synthetic_mesh_config.production_forwarding -eq $false -and $configs["r"].synthetic_mesh_config.production_forwarding -eq $false -and $configs["b"].synthetic_mesh_config.production_forwarding -eq $false -and $configs["c"].synthetic_mesh_config.production_forwarding -eq $false -and $configs["idle"].synthetic_mesh_config.production_forwarding -eq $false ) } $failed = @($passMatrix.GetEnumerator() | Where-Object { -not $_.Value }) $result = [pscustomobject]@{ stage = "C17H deployed multi-agent synthetic config smoke" run_id = $runId backend_base_url = $backendPublicBaseUrl cluster_id = $clusterID node_ids = @{ a = $nodes["a"].id r = $nodes["r"].id b = $nodes["b"].id c = $nodes["c"].id idle = $nodes["idle"].id } route_intents = @{ direct = $directIntent.route_intent.id relay = $relayIntent.route_intent.id } scoped_config_route_counts = @{ a = @($configs["a"].synthetic_mesh_config.routes).Count r = @($configs["r"].synthetic_mesh_config.routes).Count b = @($configs["b"].synthetic_mesh_config.routes).Count c = @($configs["c"].synthetic_mesh_config.routes).Count idle = @($configs["idle"].synthetic_mesh_config.routes).Count } mesh_link_count = $meshLinks.Count route_health_count = $routeHealthLinks.Count pass_matrix = $passMatrix direct_route_health = $directHealth | Select-Object -First 3 relay_route_health = $relayHealth | Select-Object -First 3 cluster_summaries = $summary.cluster_summaries backend_log_tail = $backendLogs node_log_tail = $nodeLogs containers_left_running = [bool]$KeepRunning } $result | ConvertTo-Json -Depth 40 if ($failed.Count -gt 0) { throw "C17H smoke failed: $($failed.Name -join ', ')" } if (-not $KeepRunning) { Write-Host "Cleaning up C17H containers..." Remove-C17HContainers }