469 lines
17 KiB
PowerShell
469 lines
17 KiB
PowerShell
param(
|
|
[string]$DockerSshAlias = "test-docker",
|
|
[string]$BackendImageTag = "rap-backend:dev-enrollment-bootstrap-smoke",
|
|
[string]$NodeAgentImageTag = "rap-node-agent:dev-enrollment-bootstrap-smoke",
|
|
[string]$AdminEmail = "fabric-owner-dev-bootstrap@example.local",
|
|
[string]$AdminPassword = "SmokePass!123",
|
|
[int]$ApiPort = 18121,
|
|
[int]$PostgresPort = 15443,
|
|
[int]$RedisPort = 16443,
|
|
[int]$MeshPort = 19131,
|
|
[string]$ResultPath = "artifacts\dev-cluster-enrollment-bootstrap-smoke-result.json",
|
|
[switch]$KeepRunning
|
|
)
|
|
|
|
Set-StrictMode -Version Latest
|
|
$ErrorActionPreference = "Stop"
|
|
|
|
$repoRoot = (Resolve-Path (Join-Path $PSScriptRoot "..\..")).ProviderPath
|
|
$backendPublicBaseUrl = "http://192.168.200.61:$ApiPort/api/v1"
|
|
$backendContainerBaseUrl = "http://127.0.0.1:$ApiPort/api/v1"
|
|
$runId = "dev-bootstrap-" + (Get-Date -Format "yyyyMMdd-HHmmss")
|
|
$remoteBuildDir = "/tmp/rap-dev-bootstrap-build-$runId"
|
|
|
|
$postgresName = "rap_dev_bootstrap_postgres"
|
|
$redisName = "rap_dev_bootstrap_redis"
|
|
$backendName = "rap_dev_bootstrap_backend"
|
|
$nodeName = "rap-dev-bootstrap-node-core"
|
|
$nodeContainerName = "rap_dev_bootstrap_node_core"
|
|
|
|
function Invoke-RemoteDocker {
|
|
param([string[]]$Arguments)
|
|
& ssh $DockerSshAlias docker @Arguments
|
|
if ($LASTEXITCODE -ne 0) {
|
|
throw "ssh $DockerSshAlias docker $($Arguments -join ' ') failed with exit code $LASTEXITCODE"
|
|
}
|
|
}
|
|
|
|
function Invoke-RemoteDockerText {
|
|
param([string[]]$Arguments)
|
|
$previousErrorActionPreference = $ErrorActionPreference
|
|
$ErrorActionPreference = "Continue"
|
|
try {
|
|
$output = & ssh $DockerSshAlias docker @Arguments 2>&1 | ForEach-Object { $_.ToString() }
|
|
}
|
|
finally {
|
|
$ErrorActionPreference = $previousErrorActionPreference
|
|
}
|
|
if ($LASTEXITCODE -ne 0) {
|
|
throw "ssh $DockerSshAlias docker $($Arguments -join ' ') failed with exit code $LASTEXITCODE"
|
|
}
|
|
return $output
|
|
}
|
|
|
|
function Invoke-RemoteShell {
|
|
param([string]$Command)
|
|
& ssh $DockerSshAlias $Command
|
|
if ($LASTEXITCODE -ne 0) {
|
|
throw "ssh $DockerSshAlias $Command failed with exit code $LASTEXITCODE"
|
|
}
|
|
}
|
|
|
|
function Invoke-RemoteShellOptionalText {
|
|
param([string]$Command)
|
|
$previousErrorActionPreference = $ErrorActionPreference
|
|
$ErrorActionPreference = "Continue"
|
|
try {
|
|
$output = & ssh $DockerSshAlias $Command 2>&1 | ForEach-Object { $_.ToString() }
|
|
$exitCode = $LASTEXITCODE
|
|
}
|
|
finally {
|
|
$ErrorActionPreference = $previousErrorActionPreference
|
|
}
|
|
return [pscustomobject]@{
|
|
exit_code = $exitCode
|
|
output = $output
|
|
}
|
|
}
|
|
|
|
function Send-RemoteBuildContext {
|
|
Write-Host "Uploading backend and node-agent build context to $DockerSshAlias..."
|
|
Invoke-RemoteShell -Command "rm -rf '$remoteBuildDir' && mkdir -p '$remoteBuildDir'"
|
|
& tar -czf - -C $repoRoot "backend" "agents/rap-node-agent" | & ssh $DockerSshAlias "tar -xzf - -C '$remoteBuildDir'"
|
|
if ($LASTEXITCODE -ne 0) {
|
|
throw "upload build context failed"
|
|
}
|
|
}
|
|
|
|
function Invoke-Api {
|
|
param(
|
|
[string]$Method,
|
|
[string]$Path,
|
|
[object]$Body = $null
|
|
)
|
|
$uri = "$backendPublicBaseUrl$Path"
|
|
if ($null -eq $Body) {
|
|
return Invoke-RestMethod -Method $Method -Uri $uri -TimeoutSec 30
|
|
}
|
|
return Invoke-RestMethod -Method $Method -Uri $uri -ContentType "application/json" -Body ($Body | ConvertTo-Json -Depth 60) -TimeoutSec 30
|
|
}
|
|
|
|
function Wait-HttpReady {
|
|
param([string]$Url)
|
|
for ($i = 0; $i -lt 90; $i++) {
|
|
try {
|
|
$response = Invoke-WebRequest -UseBasicParsing -Uri $Url -TimeoutSec 2
|
|
if ($response.StatusCode -ge 200 -and $response.StatusCode -lt 300) {
|
|
return
|
|
}
|
|
}
|
|
catch {
|
|
Start-Sleep -Seconds 1
|
|
}
|
|
}
|
|
throw "Timed out waiting for $Url"
|
|
}
|
|
|
|
function Remove-SmokeContainers {
|
|
foreach ($name in @($nodeContainerName, $backendName, $postgresName, $redisName)) {
|
|
& ssh $DockerSshAlias docker rm -f $name 2>$null | Out-Null
|
|
}
|
|
}
|
|
|
|
function Wait-JoinRequest {
|
|
param(
|
|
[string]$ClusterID,
|
|
[string]$ActorUserID,
|
|
[string]$ExpectedNodeName,
|
|
[int]$TimeoutSeconds = 60
|
|
)
|
|
$deadline = (Get-Date).AddSeconds($TimeoutSeconds)
|
|
do {
|
|
$response = Invoke-Api -Method Get -Path "/clusters/$ClusterID/join-requests?actor_user_id=$ActorUserID"
|
|
$match = @($response.join_requests | Where-Object {
|
|
$_.node_name -eq $ExpectedNodeName -and $_.status -eq "pending"
|
|
} | Sort-Object created_at -Descending | Select-Object -First 1)
|
|
if ($match.Count -gt 0) {
|
|
return $match[0]
|
|
}
|
|
Start-Sleep -Seconds 1
|
|
} while ((Get-Date) -lt $deadline)
|
|
throw "Timed out waiting for node-agent join request"
|
|
}
|
|
|
|
function Get-RemoteContainerIdentity {
|
|
$remoteIdentity = "/tmp/$runId-identity.json"
|
|
$command = "rm -f '$remoteIdentity'; docker cp '${nodeContainerName}:/tmp/state/identity.json' '$remoteIdentity' >/dev/null 2>&1 && cat '$remoteIdentity'"
|
|
$result = Invoke-RemoteShellOptionalText -Command $command
|
|
if ($result.exit_code -ne 0) {
|
|
return $null
|
|
}
|
|
$raw = ($result.output -join "`n").Trim()
|
|
if ($raw -eq "") {
|
|
return $null
|
|
}
|
|
return $raw | ConvertFrom-Json
|
|
}
|
|
|
|
function Wait-AgentApprovedIdentity {
|
|
param([int]$TimeoutSeconds = 90)
|
|
$deadline = (Get-Date).AddSeconds($TimeoutSeconds)
|
|
do {
|
|
$identity = Get-RemoteContainerIdentity
|
|
if ($null -ne $identity -and
|
|
$identity.node_id -and
|
|
$identity.identity_status -eq "active" -and
|
|
$identity.cluster_authority_public_key -and
|
|
$identity.cluster_authority_fingerprint) {
|
|
return $identity
|
|
}
|
|
Start-Sleep -Seconds 1
|
|
} while ((Get-Date) -lt $deadline)
|
|
throw "Timed out waiting for node-agent approved identity state"
|
|
}
|
|
|
|
function Wait-NodeHeartbeat {
|
|
param(
|
|
[string]$ClusterID,
|
|
[string]$NodeID,
|
|
[string]$ActorUserID,
|
|
[int]$TimeoutSeconds = 60
|
|
)
|
|
$deadline = (Get-Date).AddSeconds($TimeoutSeconds)
|
|
do {
|
|
$response = Invoke-Api -Method Get -Path "/clusters/$ClusterID/nodes/$NodeID/heartbeats?actor_user_id=$ActorUserID&limit=5"
|
|
$heartbeats = @($response.heartbeats)
|
|
if ($heartbeats.Count -gt 0) {
|
|
return $heartbeats[0]
|
|
}
|
|
Start-Sleep -Seconds 1
|
|
} while ((Get-Date) -lt $deadline)
|
|
throw "Timed out waiting for node-agent heartbeat"
|
|
}
|
|
|
|
function Wait-AgentLogContains {
|
|
param(
|
|
[string]$Pattern,
|
|
[int]$TimeoutSeconds = 60
|
|
)
|
|
$deadline = (Get-Date).AddSeconds($TimeoutSeconds)
|
|
do {
|
|
$logs = Invoke-RemoteDockerText -Arguments @("logs", "--tail", "200", $nodeContainerName)
|
|
$joined = $logs -join "`n"
|
|
if ($joined -match $Pattern) {
|
|
return $logs
|
|
}
|
|
Start-Sleep -Seconds 1
|
|
} while ((Get-Date) -lt $deadline)
|
|
return Invoke-RemoteDockerText -Arguments @("logs", "--tail", "200", $nodeContainerName)
|
|
}
|
|
|
|
Write-Host "Dev cluster enrollment/bootstrap smoke run: $runId"
|
|
Write-Host "Using SSH Docker host: $DockerSshAlias"
|
|
|
|
Remove-SmokeContainers
|
|
Send-RemoteBuildContext
|
|
|
|
Write-Host "Building backend image on docker-test..."
|
|
Invoke-RemoteDocker -Arguments @("build", "-f", "$remoteBuildDir/backend/Dockerfile", "-t", $BackendImageTag, "$remoteBuildDir/backend")
|
|
|
|
Write-Host "Building node-agent image on docker-test..."
|
|
Invoke-RemoteDocker -Arguments @("build", "-f", "$remoteBuildDir/agents/rap-node-agent/Dockerfile", "-t", $NodeAgentImageTag, $remoteBuildDir)
|
|
|
|
Write-Host "Starting isolated PostgreSQL and Redis..."
|
|
Invoke-RemoteDocker -Arguments @(
|
|
"run", "-d",
|
|
"--name", $postgresName,
|
|
"-e", "POSTGRES_DB=remote_access_platform",
|
|
"-e", "POSTGRES_USER=rap_user",
|
|
"-e", "POSTGRES_PASSWORD=rap_password",
|
|
"-p", "$PostgresPort`:5432",
|
|
"postgres:16"
|
|
)
|
|
Invoke-RemoteDocker -Arguments @(
|
|
"run", "-d",
|
|
"--name", $redisName,
|
|
"-p", "$RedisPort`:6379",
|
|
"redis:7"
|
|
)
|
|
|
|
Invoke-RemoteShell -Command "for i in `$(seq 1 60); do docker exec $postgresName pg_isready -U rap_user -d remote_access_platform >/dev/null 2>&1 && exit 0; sleep 1; done; exit 1"
|
|
|
|
Write-Host "Applying migrations..."
|
|
Invoke-RemoteShell -Command "for f in `$(find '$remoteBuildDir/backend/migrations' -name '*.up.sql' | sort); do docker exec -i $postgresName psql -U rap_user -d remote_access_platform -v ON_ERROR_STOP=1 -f - < `$f; done"
|
|
|
|
$secretBytes = New-Object byte[] 32
|
|
[Security.Cryptography.RandomNumberGenerator]::Fill($secretBytes)
|
|
$secretKeyB64 = [Convert]::ToBase64String($secretBytes)
|
|
|
|
Write-Host "Starting backend..."
|
|
Invoke-RemoteDocker -Arguments @(
|
|
"run", "-d",
|
|
"--name", $backendName,
|
|
"--network", "host",
|
|
"-e", "APP_NAME=rap-api",
|
|
"-e", "APP_ENV=dev-bootstrap-smoke",
|
|
"-e", "HTTP_HOST=0.0.0.0",
|
|
"-e", "HTTP_PORT=$ApiPort",
|
|
"-e", "POSTGRES_DSN=postgres://rap_user:rap_password@127.0.0.1:$PostgresPort/remote_access_platform?sslmode=disable",
|
|
"-e", "REDIS_ADDR=127.0.0.1:$RedisPort",
|
|
"-e", "AUTH_ACCESS_TOKEN_SECRET=dev-bootstrap-access-secret",
|
|
"-e", "AUTH_REFRESH_HASH_SECRET=dev-bootstrap-refresh-secret",
|
|
"-e", "INSTALLATION_AUTHORITY_MODE=legacy",
|
|
"-e", "INSTALLATION_INSECURE_BOOTSTRAP_ENABLED=true",
|
|
"-e", "SECRET_ENCRYPTION_KEY_B64=$secretKeyB64",
|
|
"-e", "SECRET_ENCRYPTION_KEY_ID=$runId",
|
|
$BackendImageTag
|
|
)
|
|
Wait-HttpReady -Url "http://192.168.200.61:$ApiPort/readyz"
|
|
|
|
Write-Host "Bootstrapping dev platform owner through installation API..."
|
|
$bootstrap = Invoke-Api -Method Post -Path "/installation/bootstrap-owner" -Body @{
|
|
email = $AdminEmail
|
|
password = $AdminPassword
|
|
}
|
|
$installationStatus = Invoke-Api -Method Get -Path "/installation/status"
|
|
|
|
Write-Host "Logging in as platform owner..."
|
|
$login = Invoke-Api -Method Post -Path "/auth/login" -Body @{
|
|
email = $AdminEmail
|
|
password = $AdminPassword
|
|
device_fingerprint = "dev-bootstrap-smoke-device"
|
|
device_label = "Dev bootstrap lifecycle smoke"
|
|
trust_device = $true
|
|
}
|
|
$actorUserID = $login.user.id
|
|
|
|
Write-Host "Creating dev cluster and join token..."
|
|
$cluster = Invoke-Api -Method Post -Path "/clusters/" -Body @{
|
|
actor_user_id = $actorUserID
|
|
slug = "dev-bootstrap-$((New-Guid).Guid.Substring(0, 8))"
|
|
name = "Dev Enrollment Bootstrap Smoke"
|
|
region = "docker-test"
|
|
metadata = @{
|
|
stage = "dev-enrollment-bootstrap-smoke"
|
|
run_id = $runId
|
|
mandatory_roles_only = $true
|
|
production_forwarding = $false
|
|
service_payload_forwarding = $false
|
|
}
|
|
}
|
|
$clusterID = $cluster.cluster.id
|
|
|
|
Invoke-Api -Method Put -Path "/fabric/testing-flags" -Body @{
|
|
actor_user_id = $actorUserID
|
|
scope_type = "platform"
|
|
scope_id = $null
|
|
cluster_id = $null
|
|
enabled = $true
|
|
telemetry_enabled = $true
|
|
synthetic_links_enabled = $true
|
|
history_retention_hours = 24
|
|
metadata = @{
|
|
stage = "dev-enrollment-bootstrap-smoke"
|
|
run_id = $runId
|
|
production_forwarding = $false
|
|
service_payload_forwarding = $false
|
|
}
|
|
} | Out-Null
|
|
|
|
$joinToken = Invoke-Api -Method Post -Path "/clusters/$clusterID/join-tokens" -Body @{
|
|
actor_user_id = $actorUserID
|
|
scope = @{
|
|
purpose = "dev-enrollment-bootstrap-smoke"
|
|
roles = @("core-mesh")
|
|
mandatory_only = $true
|
|
production_forwarding = $false
|
|
}
|
|
expires_at = (Get-Date).ToUniversalTime().AddHours(2).ToString("o")
|
|
max_uses = 1
|
|
}
|
|
|
|
Write-Host "Starting real node-agent enrollment container..."
|
|
Invoke-RemoteDocker -Arguments @(
|
|
"run", "-d",
|
|
"--name", $nodeContainerName,
|
|
"--network", "host",
|
|
"-e", "RAP_BACKEND_URL=$backendContainerBaseUrl",
|
|
"-e", "RAP_CLUSTER_ID=$clusterID",
|
|
"-e", "RAP_JOIN_TOKEN=$($joinToken.join_token.token)",
|
|
"-e", "RAP_NODE_NAME=$nodeName",
|
|
"-e", "RAP_NODE_STATE_DIR=/tmp/state",
|
|
"-e", "RAP_HEARTBEAT_INTERVAL_SECONDS=2",
|
|
"-e", "RAP_ENROLLMENT_POLL_INTERVAL_SECONDS=1",
|
|
"-e", "RAP_ENROLLMENT_POLL_TIMEOUT_SECONDS=90",
|
|
"-e", "RAP_MESH_SYNTHETIC_RUNTIME_ENABLED=true",
|
|
"-e", "RAP_MESH_LISTEN_ADDR=0.0.0.0:$MeshPort",
|
|
"-e", "RAP_MESH_ADVERTISE_ENDPOINT=http://127.0.0.1:$MeshPort",
|
|
"-e", "RAP_MESH_ADVERTISE_TRANSPORT=direct_tcp_tls",
|
|
"-e", "RAP_MESH_CONNECTIVITY_MODE=direct",
|
|
"-e", "RAP_MESH_NAT_TYPE=none",
|
|
"-e", "RAP_MESH_REGION=docker-test",
|
|
$NodeAgentImageTag
|
|
)
|
|
|
|
Write-Host "Waiting for pending join request from node-agent..."
|
|
$joinRequest = Wait-JoinRequest -ClusterID $clusterID -ActorUserID $actorUserID -ExpectedNodeName $nodeName
|
|
|
|
Write-Host "Approving join request..."
|
|
$approved = Invoke-Api -Method Post -Path "/clusters/$clusterID/join-requests/$($joinRequest.id)/approve" -Body @{
|
|
actor_user_id = $actorUserID
|
|
node_key = $joinRequest.node_fingerprint
|
|
ownership_type = "platform_managed"
|
|
owner_organization_id = $null
|
|
}
|
|
|
|
Write-Host "Waiting for node-agent to persist signed bootstrap authority pin..."
|
|
$identity = Wait-AgentApprovedIdentity
|
|
$nodeID = $identity.node_id
|
|
|
|
Write-Host "Assigning mandatory core-mesh role after approved identity..."
|
|
Invoke-Api -Method Post -Path "/clusters/$clusterID/nodes/$nodeID/roles" -Body @{
|
|
actor_user_id = $actorUserID
|
|
role = "core-mesh"
|
|
status = "active"
|
|
policy = @{
|
|
stage = "dev-enrollment-bootstrap-smoke"
|
|
run_id = $runId
|
|
mandatory = $true
|
|
production_forwarding = $false
|
|
service_payload_forwarding = $false
|
|
}
|
|
} | Out-Null
|
|
|
|
Write-Host "Waiting for heartbeat and signed synthetic config verification..."
|
|
$heartbeat = Wait-NodeHeartbeat -ClusterID $clusterID -NodeID $nodeID -ActorUserID $actorUserID
|
|
$configResponse = Invoke-Api -Method Get -Path "/clusters/$clusterID/nodes/$nodeID/mesh/synthetic-config"
|
|
$syntheticConfig = $configResponse.synthetic_mesh_config
|
|
$nodeLogs = Wait-AgentLogContains -Pattern "synthetic mesh config loaded: source=control_plane" -TimeoutSeconds 60
|
|
$backendLogs = Invoke-RemoteDockerText -Arguments @("logs", "--tail", "120", $backendName)
|
|
$summaries = Invoke-Api -Method Get -Path "/cluster-admin-summaries?actor_user_id=$actorUserID"
|
|
|
|
$agentLogText = $nodeLogs -join "`n"
|
|
$passMatrix = [ordered]@{
|
|
installation_bootstrapped = [bool]$installationStatus.installation.bootstrapped
|
|
cluster_created = [bool]$clusterID
|
|
join_token_has_cluster_authority_signature = [bool]($joinToken.join_token.authority_payload -and $joinToken.join_token.authority_signature)
|
|
agent_submitted_pending_join_request = [bool]($joinRequest.id -and $joinRequest.status -eq "pending")
|
|
approval_returned_signed_bootstrap = [bool]($approved.node_bootstrap.cluster_authority -and $approved.node_bootstrap.authority_payload -and $approved.node_bootstrap.authority_signature)
|
|
agent_persisted_approved_identity = [bool]($identity.node_id -eq $approved.node_bootstrap.node_id -and $identity.identity_status -eq "active")
|
|
agent_persisted_cluster_authority_pin = [bool]($identity.cluster_authority_public_key -and $identity.cluster_authority_fingerprint)
|
|
heartbeat_after_auto_bootstrap = [bool]$heartbeat.id
|
|
synthetic_config_has_cluster_authority_signature = [bool]($syntheticConfig.authority_required -and $syntheticConfig.cluster_authority -and $syntheticConfig.authority_payload -and $syntheticConfig.authority_signature)
|
|
agent_verified_control_plane_synthetic_config = [bool]($agentLogText -match "synthetic mesh config loaded: source=control_plane")
|
|
production_forwarding_disabled = [bool](-not $syntheticConfig.production_forwarding)
|
|
}
|
|
|
|
$result = [ordered]@{
|
|
run_id = $runId
|
|
stage = "dev cluster enrollment/bootstrap lifecycle smoke"
|
|
docker_host = $DockerSshAlias
|
|
backend_base_url = $backendPublicBaseUrl
|
|
api_port = $ApiPort
|
|
postgres_port = $PostgresPort
|
|
redis_port = $RedisPort
|
|
mesh_port = $MeshPort
|
|
backend_image = $BackendImageTag
|
|
node_agent_image = $NodeAgentImageTag
|
|
containers = @{
|
|
backend = $backendName
|
|
postgres = $postgresName
|
|
redis = $redisName
|
|
node_agent = $nodeContainerName
|
|
}
|
|
installation = $installationStatus.installation
|
|
admin_user_id = $actorUserID
|
|
cluster_id = $clusterID
|
|
node_id = $nodeID
|
|
join_request_id = $joinRequest.id
|
|
join_token_signature = $joinToken.join_token.authority_signature
|
|
approval_authority = $approved.node_bootstrap.cluster_authority
|
|
identity_authority_fingerprint = $identity.cluster_authority_fingerprint
|
|
synthetic_config_schema = $syntheticConfig.schema_version
|
|
synthetic_config_version = $syntheticConfig.config_version
|
|
synthetic_config_authority = $syntheticConfig.cluster_authority
|
|
latest_heartbeat = $heartbeat
|
|
cluster_summaries = $summaries.cluster_summaries
|
|
pass_matrix = $passMatrix
|
|
backend_log_tail = $backendLogs
|
|
node_log_tail = $nodeLogs
|
|
containers_left_running = [bool]$KeepRunning
|
|
}
|
|
|
|
$failed = @($passMatrix.GetEnumerator() | Where-Object { -not $_.Value })
|
|
$resultJson = $result | ConvertTo-Json -Depth 70
|
|
|
|
if ($ResultPath -ne "") {
|
|
if ([System.IO.Path]::IsPathRooted($ResultPath)) {
|
|
$resultFullPath = $ResultPath
|
|
} else {
|
|
$resultFullPath = Join-Path $repoRoot $ResultPath
|
|
}
|
|
New-Item -ItemType Directory -Force -Path (Split-Path $resultFullPath) | Out-Null
|
|
Set-Content -Path $resultFullPath -Value $resultJson -Encoding UTF8
|
|
Write-Host "Result written to $resultFullPath"
|
|
}
|
|
|
|
$resultJson
|
|
|
|
if ($failed.Count -gt 0) {
|
|
throw "Dev enrollment/bootstrap smoke failed: $($failed.Name -join ', ')"
|
|
}
|
|
|
|
Invoke-RemoteShell -Command "rm -rf '$remoteBuildDir'"
|
|
|
|
if (-not $KeepRunning) {
|
|
Write-Host "Cleaning up dev enrollment/bootstrap containers..."
|
|
Remove-SmokeContainers
|
|
}
|