param( [string]$DockerSshAlias = "test-docker", [string]$BackendImageTag = "rap-backend:dev-enrollment-bootstrap-smoke", [string]$NodeAgentImageTag = "rap-node-agent:dev-enrollment-bootstrap-smoke", [string]$AdminEmail = "fabric-owner-dev-bootstrap@example.local", [string]$AdminPassword = "SmokePass!123", [int]$ApiPort = 18121, [int]$PostgresPort = 15443, [int]$RedisPort = 16443, [int]$MeshPort = 19131, [string]$ResultPath = "artifacts\dev-cluster-enrollment-bootstrap-smoke-result.json", [switch]$KeepRunning ) Set-StrictMode -Version Latest $ErrorActionPreference = "Stop" $repoRoot = (Resolve-Path (Join-Path $PSScriptRoot "..\..")).ProviderPath $backendPublicBaseUrl = "http://192.168.200.61:$ApiPort/api/v1" $backendContainerBaseUrl = "http://127.0.0.1:$ApiPort/api/v1" $runId = "dev-bootstrap-" + (Get-Date -Format "yyyyMMdd-HHmmss") $remoteBuildDir = "/tmp/rap-dev-bootstrap-build-$runId" $postgresName = "rap_dev_bootstrap_postgres" $redisName = "rap_dev_bootstrap_redis" $backendName = "rap_dev_bootstrap_backend" $nodeName = "rap-dev-bootstrap-node-core" $nodeContainerName = "rap_dev_bootstrap_node_core" function Invoke-RemoteDocker { param([string[]]$Arguments) & ssh $DockerSshAlias docker @Arguments if ($LASTEXITCODE -ne 0) { throw "ssh $DockerSshAlias docker $($Arguments -join ' ') failed with exit code $LASTEXITCODE" } } function Invoke-RemoteDockerText { param([string[]]$Arguments) $previousErrorActionPreference = $ErrorActionPreference $ErrorActionPreference = "Continue" try { $output = & ssh $DockerSshAlias docker @Arguments 2>&1 | ForEach-Object { $_.ToString() } } finally { $ErrorActionPreference = $previousErrorActionPreference } if ($LASTEXITCODE -ne 0) { throw "ssh $DockerSshAlias docker $($Arguments -join ' ') failed with exit code $LASTEXITCODE" } return $output } function Invoke-RemoteShell { param([string]$Command) & ssh $DockerSshAlias $Command if ($LASTEXITCODE -ne 0) { throw "ssh $DockerSshAlias $Command failed with exit code $LASTEXITCODE" } } function Invoke-RemoteShellOptionalText { param([string]$Command) $previousErrorActionPreference = $ErrorActionPreference $ErrorActionPreference = "Continue" try { $output = & ssh $DockerSshAlias $Command 2>&1 | ForEach-Object { $_.ToString() } $exitCode = $LASTEXITCODE } finally { $ErrorActionPreference = $previousErrorActionPreference } return [pscustomobject]@{ exit_code = $exitCode output = $output } } function Send-RemoteBuildContext { Write-Host "Uploading backend and node-agent build context to $DockerSshAlias..." Invoke-RemoteShell -Command "rm -rf '$remoteBuildDir' && mkdir -p '$remoteBuildDir'" & tar -czf - -C $repoRoot "backend" "agents/rap-node-agent" | & ssh $DockerSshAlias "tar -xzf - -C '$remoteBuildDir'" if ($LASTEXITCODE -ne 0) { throw "upload build context failed" } } function Invoke-Api { param( [string]$Method, [string]$Path, [object]$Body = $null ) $uri = "$backendPublicBaseUrl$Path" if ($null -eq $Body) { return Invoke-RestMethod -Method $Method -Uri $uri -TimeoutSec 30 } return Invoke-RestMethod -Method $Method -Uri $uri -ContentType "application/json" -Body ($Body | ConvertTo-Json -Depth 60) -TimeoutSec 30 } function Wait-HttpReady { param([string]$Url) for ($i = 0; $i -lt 90; $i++) { try { $response = Invoke-WebRequest -UseBasicParsing -Uri $Url -TimeoutSec 2 if ($response.StatusCode -ge 200 -and $response.StatusCode -lt 300) { return } } catch { Start-Sleep -Seconds 1 } } throw "Timed out waiting for $Url" } function Remove-SmokeContainers { foreach ($name in @($nodeContainerName, $backendName, $postgresName, $redisName)) { & ssh $DockerSshAlias docker rm -f $name 2>$null | Out-Null } } function Wait-JoinRequest { param( [string]$ClusterID, [string]$ActorUserID, [string]$ExpectedNodeName, [int]$TimeoutSeconds = 60 ) $deadline = (Get-Date).AddSeconds($TimeoutSeconds) do { $response = Invoke-Api -Method Get -Path "/clusters/$ClusterID/join-requests?actor_user_id=$ActorUserID" $match = @($response.join_requests | Where-Object { $_.node_name -eq $ExpectedNodeName -and $_.status -eq "pending" } | Sort-Object created_at -Descending | Select-Object -First 1) if ($match.Count -gt 0) { return $match[0] } Start-Sleep -Seconds 1 } while ((Get-Date) -lt $deadline) throw "Timed out waiting for node-agent join request" } function Get-RemoteContainerIdentity { $remoteIdentity = "/tmp/$runId-identity.json" $command = "rm -f '$remoteIdentity'; docker cp '${nodeContainerName}:/tmp/state/identity.json' '$remoteIdentity' >/dev/null 2>&1 && cat '$remoteIdentity'" $result = Invoke-RemoteShellOptionalText -Command $command if ($result.exit_code -ne 0) { return $null } $raw = ($result.output -join "`n").Trim() if ($raw -eq "") { return $null } return $raw | ConvertFrom-Json } function Wait-AgentApprovedIdentity { param([int]$TimeoutSeconds = 90) $deadline = (Get-Date).AddSeconds($TimeoutSeconds) do { $identity = Get-RemoteContainerIdentity if ($null -ne $identity -and $identity.node_id -and $identity.identity_status -eq "active" -and $identity.cluster_authority_public_key -and $identity.cluster_authority_fingerprint) { return $identity } Start-Sleep -Seconds 1 } while ((Get-Date) -lt $deadline) throw "Timed out waiting for node-agent approved identity state" } function Wait-NodeHeartbeat { param( [string]$ClusterID, [string]$NodeID, [string]$ActorUserID, [int]$TimeoutSeconds = 60 ) $deadline = (Get-Date).AddSeconds($TimeoutSeconds) do { $response = Invoke-Api -Method Get -Path "/clusters/$ClusterID/nodes/$NodeID/heartbeats?actor_user_id=$ActorUserID&limit=5" $heartbeats = @($response.heartbeats) if ($heartbeats.Count -gt 0) { return $heartbeats[0] } Start-Sleep -Seconds 1 } while ((Get-Date) -lt $deadline) throw "Timed out waiting for node-agent heartbeat" } function Wait-AgentLogContains { param( [string]$Pattern, [int]$TimeoutSeconds = 60 ) $deadline = (Get-Date).AddSeconds($TimeoutSeconds) do { $logs = Invoke-RemoteDockerText -Arguments @("logs", "--tail", "200", $nodeContainerName) $joined = $logs -join "`n" if ($joined -match $Pattern) { return $logs } Start-Sleep -Seconds 1 } while ((Get-Date) -lt $deadline) return Invoke-RemoteDockerText -Arguments @("logs", "--tail", "200", $nodeContainerName) } Write-Host "Dev cluster enrollment/bootstrap smoke run: $runId" Write-Host "Using SSH Docker host: $DockerSshAlias" Remove-SmokeContainers Send-RemoteBuildContext Write-Host "Building backend image on docker-test..." Invoke-RemoteDocker -Arguments @("build", "-f", "$remoteBuildDir/backend/Dockerfile", "-t", $BackendImageTag, "$remoteBuildDir/backend") Write-Host "Building node-agent image on docker-test..." Invoke-RemoteDocker -Arguments @("build", "-f", "$remoteBuildDir/agents/rap-node-agent/Dockerfile", "-t", $NodeAgentImageTag, $remoteBuildDir) Write-Host "Starting isolated PostgreSQL and Redis..." Invoke-RemoteDocker -Arguments @( "run", "-d", "--name", $postgresName, "-e", "POSTGRES_DB=remote_access_platform", "-e", "POSTGRES_USER=rap_user", "-e", "POSTGRES_PASSWORD=rap_password", "-p", "$PostgresPort`:5432", "postgres:16" ) Invoke-RemoteDocker -Arguments @( "run", "-d", "--name", $redisName, "-p", "$RedisPort`:6379", "redis:7" ) Invoke-RemoteShell -Command "for i in `$(seq 1 60); do docker exec $postgresName pg_isready -U rap_user -d remote_access_platform >/dev/null 2>&1 && exit 0; sleep 1; done; exit 1" Write-Host "Applying migrations..." Invoke-RemoteShell -Command "for f in `$(find '$remoteBuildDir/backend/migrations' -name '*.up.sql' | sort); do docker exec -i $postgresName psql -U rap_user -d remote_access_platform -v ON_ERROR_STOP=1 -f - < `$f; done" $secretBytes = New-Object byte[] 32 [Security.Cryptography.RandomNumberGenerator]::Fill($secretBytes) $secretKeyB64 = [Convert]::ToBase64String($secretBytes) Write-Host "Starting backend..." Invoke-RemoteDocker -Arguments @( "run", "-d", "--name", $backendName, "--network", "host", "-e", "APP_NAME=rap-api", "-e", "APP_ENV=dev-bootstrap-smoke", "-e", "HTTP_HOST=0.0.0.0", "-e", "HTTP_PORT=$ApiPort", "-e", "POSTGRES_DSN=postgres://rap_user:rap_password@127.0.0.1:$PostgresPort/remote_access_platform?sslmode=disable", "-e", "REDIS_ADDR=127.0.0.1:$RedisPort", "-e", "AUTH_ACCESS_TOKEN_SECRET=dev-bootstrap-access-secret", "-e", "AUTH_REFRESH_HASH_SECRET=dev-bootstrap-refresh-secret", "-e", "INSTALLATION_AUTHORITY_MODE=legacy", "-e", "INSTALLATION_INSECURE_BOOTSTRAP_ENABLED=true", "-e", "SECRET_ENCRYPTION_KEY_B64=$secretKeyB64", "-e", "SECRET_ENCRYPTION_KEY_ID=$runId", $BackendImageTag ) Wait-HttpReady -Url "http://192.168.200.61:$ApiPort/readyz" Write-Host "Bootstrapping dev platform owner through installation API..." $bootstrap = Invoke-Api -Method Post -Path "/installation/bootstrap-owner" -Body @{ email = $AdminEmail password = $AdminPassword } $installationStatus = Invoke-Api -Method Get -Path "/installation/status" Write-Host "Logging in as platform owner..." $login = Invoke-Api -Method Post -Path "/auth/login" -Body @{ email = $AdminEmail password = $AdminPassword device_fingerprint = "dev-bootstrap-smoke-device" device_label = "Dev bootstrap lifecycle smoke" trust_device = $true } $actorUserID = $login.user.id Write-Host "Creating dev cluster and join token..." $cluster = Invoke-Api -Method Post -Path "/clusters/" -Body @{ actor_user_id = $actorUserID slug = "dev-bootstrap-$((New-Guid).Guid.Substring(0, 8))" name = "Dev Enrollment Bootstrap Smoke" region = "docker-test" metadata = @{ stage = "dev-enrollment-bootstrap-smoke" run_id = $runId mandatory_roles_only = $true production_forwarding = $false service_payload_forwarding = $false } } $clusterID = $cluster.cluster.id Invoke-Api -Method Put -Path "/fabric/testing-flags" -Body @{ actor_user_id = $actorUserID scope_type = "platform" scope_id = $null cluster_id = $null enabled = $true telemetry_enabled = $true synthetic_links_enabled = $true history_retention_hours = 24 metadata = @{ stage = "dev-enrollment-bootstrap-smoke" run_id = $runId production_forwarding = $false service_payload_forwarding = $false } } | Out-Null $joinToken = Invoke-Api -Method Post -Path "/clusters/$clusterID/join-tokens" -Body @{ actor_user_id = $actorUserID scope = @{ purpose = "dev-enrollment-bootstrap-smoke" roles = @("core-mesh") mandatory_only = $true production_forwarding = $false } expires_at = (Get-Date).ToUniversalTime().AddHours(2).ToString("o") max_uses = 1 } Write-Host "Starting real node-agent enrollment container..." Invoke-RemoteDocker -Arguments @( "run", "-d", "--name", $nodeContainerName, "--network", "host", "-e", "RAP_BACKEND_URL=$backendContainerBaseUrl", "-e", "RAP_CLUSTER_ID=$clusterID", "-e", "RAP_JOIN_TOKEN=$($joinToken.join_token.token)", "-e", "RAP_NODE_NAME=$nodeName", "-e", "RAP_NODE_STATE_DIR=/tmp/state", "-e", "RAP_HEARTBEAT_INTERVAL_SECONDS=2", "-e", "RAP_ENROLLMENT_POLL_INTERVAL_SECONDS=1", "-e", "RAP_ENROLLMENT_POLL_TIMEOUT_SECONDS=90", "-e", "RAP_MESH_SYNTHETIC_RUNTIME_ENABLED=true", "-e", "RAP_MESH_LISTEN_ADDR=0.0.0.0:$MeshPort", "-e", "RAP_MESH_ADVERTISE_ENDPOINT=http://127.0.0.1:$MeshPort", "-e", "RAP_MESH_ADVERTISE_TRANSPORT=direct_tcp_tls", "-e", "RAP_MESH_CONNECTIVITY_MODE=direct", "-e", "RAP_MESH_NAT_TYPE=none", "-e", "RAP_MESH_REGION=docker-test", $NodeAgentImageTag ) Write-Host "Waiting for pending join request from node-agent..." $joinRequest = Wait-JoinRequest -ClusterID $clusterID -ActorUserID $actorUserID -ExpectedNodeName $nodeName Write-Host "Approving join request..." $approved = Invoke-Api -Method Post -Path "/clusters/$clusterID/join-requests/$($joinRequest.id)/approve" -Body @{ actor_user_id = $actorUserID node_key = $joinRequest.node_fingerprint ownership_type = "platform_managed" owner_organization_id = $null } Write-Host "Waiting for node-agent to persist signed bootstrap authority pin..." $identity = Wait-AgentApprovedIdentity $nodeID = $identity.node_id Write-Host "Assigning mandatory core-mesh role after approved identity..." Invoke-Api -Method Post -Path "/clusters/$clusterID/nodes/$nodeID/roles" -Body @{ actor_user_id = $actorUserID role = "core-mesh" status = "active" policy = @{ stage = "dev-enrollment-bootstrap-smoke" run_id = $runId mandatory = $true production_forwarding = $false service_payload_forwarding = $false } } | Out-Null Write-Host "Waiting for heartbeat and signed synthetic config verification..." $heartbeat = Wait-NodeHeartbeat -ClusterID $clusterID -NodeID $nodeID -ActorUserID $actorUserID $configResponse = Invoke-Api -Method Get -Path "/clusters/$clusterID/nodes/$nodeID/mesh/synthetic-config" $syntheticConfig = $configResponse.synthetic_mesh_config $nodeLogs = Wait-AgentLogContains -Pattern "synthetic mesh config loaded: source=control_plane" -TimeoutSeconds 60 $backendLogs = Invoke-RemoteDockerText -Arguments @("logs", "--tail", "120", $backendName) $summaries = Invoke-Api -Method Get -Path "/cluster-admin-summaries?actor_user_id=$actorUserID" $agentLogText = $nodeLogs -join "`n" $passMatrix = [ordered]@{ installation_bootstrapped = [bool]$installationStatus.installation.bootstrapped cluster_created = [bool]$clusterID join_token_has_cluster_authority_signature = [bool]($joinToken.join_token.authority_payload -and $joinToken.join_token.authority_signature) agent_submitted_pending_join_request = [bool]($joinRequest.id -and $joinRequest.status -eq "pending") approval_returned_signed_bootstrap = [bool]($approved.node_bootstrap.cluster_authority -and $approved.node_bootstrap.authority_payload -and $approved.node_bootstrap.authority_signature) agent_persisted_approved_identity = [bool]($identity.node_id -eq $approved.node_bootstrap.node_id -and $identity.identity_status -eq "active") agent_persisted_cluster_authority_pin = [bool]($identity.cluster_authority_public_key -and $identity.cluster_authority_fingerprint) heartbeat_after_auto_bootstrap = [bool]$heartbeat.id synthetic_config_has_cluster_authority_signature = [bool]($syntheticConfig.authority_required -and $syntheticConfig.cluster_authority -and $syntheticConfig.authority_payload -and $syntheticConfig.authority_signature) agent_verified_control_plane_synthetic_config = [bool]($agentLogText -match "synthetic mesh config loaded: source=control_plane") production_forwarding_disabled = [bool](-not $syntheticConfig.production_forwarding) } $result = [ordered]@{ run_id = $runId stage = "dev cluster enrollment/bootstrap lifecycle smoke" docker_host = $DockerSshAlias backend_base_url = $backendPublicBaseUrl api_port = $ApiPort postgres_port = $PostgresPort redis_port = $RedisPort mesh_port = $MeshPort backend_image = $BackendImageTag node_agent_image = $NodeAgentImageTag containers = @{ backend = $backendName postgres = $postgresName redis = $redisName node_agent = $nodeContainerName } installation = $installationStatus.installation admin_user_id = $actorUserID cluster_id = $clusterID node_id = $nodeID join_request_id = $joinRequest.id join_token_signature = $joinToken.join_token.authority_signature approval_authority = $approved.node_bootstrap.cluster_authority identity_authority_fingerprint = $identity.cluster_authority_fingerprint synthetic_config_schema = $syntheticConfig.schema_version synthetic_config_version = $syntheticConfig.config_version synthetic_config_authority = $syntheticConfig.cluster_authority latest_heartbeat = $heartbeat cluster_summaries = $summaries.cluster_summaries pass_matrix = $passMatrix backend_log_tail = $backendLogs node_log_tail = $nodeLogs containers_left_running = [bool]$KeepRunning } $failed = @($passMatrix.GetEnumerator() | Where-Object { -not $_.Value }) $resultJson = $result | ConvertTo-Json -Depth 70 if ($ResultPath -ne "") { if ([System.IO.Path]::IsPathRooted($ResultPath)) { $resultFullPath = $ResultPath } else { $resultFullPath = Join-Path $repoRoot $ResultPath } New-Item -ItemType Directory -Force -Path (Split-Path $resultFullPath) | Out-Null Set-Content -Path $resultFullPath -Value $resultJson -Encoding UTF8 Write-Host "Result written to $resultFullPath" } $resultJson if ($failed.Count -gt 0) { throw "Dev enrollment/bootstrap smoke failed: $($failed.Name -join ', ')" } Invoke-RemoteShell -Command "rm -rf '$remoteBuildDir'" if (-not $KeepRunning) { Write-Host "Cleaning up dev enrollment/bootstrap containers..." Remove-SmokeContainers }