461 lines
17 KiB
PowerShell
461 lines
17 KiB
PowerShell
param(
|
|
[string]$DockerContext = "test-ubuntu",
|
|
[string]$BackendImageTag = "rap-backend:c17h-synthetic-smoke",
|
|
[string]$NodeAgentImageTag = "rap-node-agent:c17h-synthetic-smoke",
|
|
[string]$AdminEmail = "fabric-owner-c17h@example.local",
|
|
[string]$AdminPassword = "SmokePass!123",
|
|
[int]$ApiPort = 18080,
|
|
[int]$PostgresPort = 15432,
|
|
[int]$RedisPort = 16379,
|
|
[int]$MeshBasePort = 19081,
|
|
[switch]$KeepRunning
|
|
)
|
|
|
|
Set-StrictMode -Version Latest
|
|
$ErrorActionPreference = "Stop"
|
|
|
|
$repoRoot = (Resolve-Path (Join-Path $PSScriptRoot "..\..")).ProviderPath
|
|
$backendBaseUrl = "http://127.0.0.1:$ApiPort/api/v1"
|
|
$backendPublicBaseUrl = "http://192.168.200.61:$ApiPort/api/v1"
|
|
$runId = "c17h-" + (Get-Date -Format "yyyyMMdd-HHmmss")
|
|
|
|
$postgresName = "rap_c17h_postgres"
|
|
$redisName = "rap_c17h_redis"
|
|
$backendName = "rap_c17h_backend"
|
|
$nodePrefix = "rap_c17h_node_"
|
|
|
|
function Invoke-Docker {
|
|
param([string[]]$Arguments)
|
|
docker --context $DockerContext @Arguments
|
|
if ($LASTEXITCODE -ne 0) {
|
|
throw "docker --context $DockerContext $($Arguments -join ' ') failed with exit code $LASTEXITCODE"
|
|
}
|
|
}
|
|
|
|
function Invoke-DockerText {
|
|
param([string[]]$Arguments)
|
|
$output = docker --context $DockerContext @Arguments 2>&1 | ForEach-Object { $_.ToString() }
|
|
if ($LASTEXITCODE -ne 0) {
|
|
throw "docker --context $DockerContext $($Arguments -join ' ') failed with exit code $LASTEXITCODE"
|
|
}
|
|
return $output
|
|
}
|
|
|
|
function Invoke-Api {
|
|
param(
|
|
[string]$Method,
|
|
[string]$Path,
|
|
[object]$Body = $null
|
|
)
|
|
$uri = "$backendPublicBaseUrl$Path"
|
|
if ($null -eq $Body) {
|
|
return Invoke-RestMethod -Method $Method -Uri $uri -TimeoutSec 30
|
|
}
|
|
return Invoke-RestMethod -Method $Method -Uri $uri -ContentType "application/json" -Body ($Body | ConvertTo-Json -Depth 30) -TimeoutSec 30
|
|
}
|
|
|
|
function Wait-HttpReady {
|
|
param([string]$Url)
|
|
for ($i = 0; $i -lt 60; $i++) {
|
|
try {
|
|
$response = Invoke-WebRequest -UseBasicParsing -Uri $Url -TimeoutSec 2
|
|
if ($response.StatusCode -ge 200 -and $response.StatusCode -lt 300) {
|
|
return
|
|
}
|
|
}
|
|
catch {
|
|
Start-Sleep -Seconds 1
|
|
}
|
|
}
|
|
throw "Timed out waiting for $Url"
|
|
}
|
|
|
|
function Invoke-PostgresSql {
|
|
param([string]$Sql)
|
|
$Sql | docker --context $DockerContext exec -i $postgresName psql -U rap_user -d remote_access_platform -v ON_ERROR_STOP=1 -f -
|
|
if ($LASTEXITCODE -ne 0) {
|
|
throw "psql command failed"
|
|
}
|
|
}
|
|
|
|
function Remove-C17HContainers {
|
|
$names = @($backendName, $postgresName, $redisName)
|
|
foreach ($letter in @("a", "b", "c", "r", "idle")) {
|
|
$names += "$nodePrefix$letter"
|
|
}
|
|
foreach ($name in $names) {
|
|
docker --context $DockerContext rm -f $name 2>$null | Out-Null
|
|
}
|
|
}
|
|
|
|
Write-Host "C17H smoke run: $runId"
|
|
Write-Host "Using Docker context: $DockerContext"
|
|
|
|
Remove-C17HContainers
|
|
|
|
Write-Host "Building backend image..."
|
|
Invoke-Docker -Arguments @("build", "-f", "$repoRoot\backend\Dockerfile", "-t", $BackendImageTag, "$repoRoot\backend")
|
|
|
|
Write-Host "Building node-agent image..."
|
|
Invoke-Docker -Arguments @("build", "-f", "$repoRoot\agents\rap-node-agent\Dockerfile", "-t", $NodeAgentImageTag, "$repoRoot")
|
|
|
|
Write-Host "Starting PostgreSQL and Redis..."
|
|
Invoke-Docker -Arguments @(
|
|
"run", "-d",
|
|
"--name", $postgresName,
|
|
"-e", "POSTGRES_DB=remote_access_platform",
|
|
"-e", "POSTGRES_USER=rap_user",
|
|
"-e", "POSTGRES_PASSWORD=rap_password",
|
|
"-p", "$PostgresPort`:5432",
|
|
"postgres:16"
|
|
)
|
|
Invoke-Docker -Arguments @(
|
|
"run", "-d",
|
|
"--name", $redisName,
|
|
"-p", "$RedisPort`:6379",
|
|
"redis:7"
|
|
)
|
|
|
|
for ($i = 0; $i -lt 60; $i++) {
|
|
docker --context $DockerContext exec $postgresName pg_isready -U rap_user -d remote_access_platform | Out-Null
|
|
if ($LASTEXITCODE -eq 0) {
|
|
break
|
|
}
|
|
Start-Sleep -Seconds 1
|
|
}
|
|
docker --context $DockerContext exec $postgresName pg_isready -U rap_user -d remote_access_platform | Out-Null
|
|
if ($LASTEXITCODE -ne 0) {
|
|
throw "PostgreSQL did not become ready"
|
|
}
|
|
|
|
Write-Host "Applying migrations..."
|
|
$migrationFiles = Get-ChildItem -Path (Join-Path $repoRoot "backend\migrations") -Filter "*.up.sql" | Sort-Object Name
|
|
foreach ($migration in $migrationFiles) {
|
|
Write-Host "Applying $($migration.Name)"
|
|
Get-Content -Raw $migration.FullName | docker --context $DockerContext exec -i $postgresName psql -U rap_user -d remote_access_platform -v ON_ERROR_STOP=1 -f -
|
|
if ($LASTEXITCODE -ne 0) {
|
|
throw "Migration failed: $($migration.Name)"
|
|
}
|
|
}
|
|
|
|
Write-Host "Seeding platform owner..."
|
|
$adminUserID = [guid]::NewGuid().ToString()
|
|
$adminHash = '$2a$10$AqLRexkI1yXbuiMPU6dHM.KVUhF.t..9NolyK4OOodQTyTsHyG.7u'
|
|
Invoke-PostgresSql -Sql @"
|
|
INSERT INTO users (id, email, password_hash, mfa_enabled, platform_role)
|
|
VALUES ('$adminUserID'::uuid, '$AdminEmail', '$adminHash', FALSE, 'platform_admin')
|
|
ON CONFLICT (email) DO UPDATE SET
|
|
password_hash = EXCLUDED.password_hash,
|
|
platform_role = 'platform_admin',
|
|
updated_at = NOW();
|
|
"@
|
|
|
|
Write-Host "Starting backend..."
|
|
Invoke-Docker -Arguments @(
|
|
"run", "-d",
|
|
"--name", $backendName,
|
|
"--network", "host",
|
|
"-e", "APP_NAME=rap-api",
|
|
"-e", "APP_ENV=c17h-smoke",
|
|
"-e", "HTTP_HOST=0.0.0.0",
|
|
"-e", "HTTP_PORT=$ApiPort",
|
|
"-e", "POSTGRES_DSN=postgres://rap_user:rap_password@127.0.0.1:$PostgresPort/remote_access_platform?sslmode=disable",
|
|
"-e", "REDIS_ADDR=127.0.0.1:$RedisPort",
|
|
"-e", "AUTH_ACCESS_TOKEN_SECRET=c17h-access-secret",
|
|
"-e", "AUTH_REFRESH_HASH_SECRET=c17h-refresh-secret",
|
|
$BackendImageTag
|
|
)
|
|
Wait-HttpReady -Url "http://192.168.200.61:$ApiPort/readyz"
|
|
|
|
Write-Host "Logging in as platform owner..."
|
|
$login = Invoke-Api -Method Post -Path "/auth/login" -Body @{
|
|
email = $AdminEmail
|
|
password = $AdminPassword
|
|
device_fingerprint = "c17h-smoke-device"
|
|
device_label = "C17H synthetic smoke"
|
|
trust_device = $true
|
|
}
|
|
$actorUserID = $login.user.id
|
|
|
|
Write-Host "Creating C17H cluster..."
|
|
$cluster = Invoke-Api -Method Post -Path "/clusters/" -Body @{
|
|
actor_user_id = $actorUserID
|
|
slug = "c17h-$((New-Guid).Guid.Substring(0, 8))"
|
|
name = "C17H Synthetic Mesh Smoke"
|
|
region = "test-docker"
|
|
metadata = @{
|
|
stage = "c17h"
|
|
production_forwarding = $false
|
|
created_by = "c17h-multi-agent-synthetic-smoke.ps1"
|
|
}
|
|
}
|
|
$clusterID = $cluster.cluster.id
|
|
|
|
Write-Host "Enabling test-only Fabric flags..."
|
|
Invoke-Api -Method Put -Path "/fabric/testing-flags" -Body @{
|
|
actor_user_id = $actorUserID
|
|
scope_type = "platform"
|
|
scope_id = $null
|
|
cluster_id = $null
|
|
enabled = $true
|
|
telemetry_enabled = $true
|
|
synthetic_links_enabled = $true
|
|
history_retention_hours = 24
|
|
metadata = @{
|
|
stage = "c17h"
|
|
production_forwarding = $false
|
|
service_workload_traffic = $false
|
|
}
|
|
} | Out-Null
|
|
|
|
$joinToken = Invoke-Api -Method Post -Path "/clusters/$clusterID/join-tokens" -Body @{
|
|
actor_user_id = $actorUserID
|
|
scope = @{ purpose = "c17h-synthetic-smoke"; roles = @("core-mesh", "relay-node") }
|
|
expires_at = (Get-Date).ToUniversalTime().AddHours(2).ToString("o")
|
|
max_uses = 5
|
|
}
|
|
|
|
$nodeSpecs = @(
|
|
@{ key = "a"; name = "c17h-node-a"; roles = @("core-mesh"); port = $MeshBasePort },
|
|
@{ key = "r"; name = "c17h-node-r"; roles = @("core-mesh", "relay-node"); port = ($MeshBasePort + 1) },
|
|
@{ key = "b"; name = "c17h-node-b"; roles = @("core-mesh"); port = ($MeshBasePort + 2) },
|
|
@{ key = "c"; name = "c17h-node-c"; roles = @("core-mesh"); port = ($MeshBasePort + 3) },
|
|
@{ key = "idle"; name = "c17h-node-idle"; roles = @("core-mesh"); port = ($MeshBasePort + 4) }
|
|
)
|
|
|
|
$nodes = @{}
|
|
foreach ($spec in $nodeSpecs) {
|
|
$fingerprint = "c17h-fp-$($spec.key)-$([guid]::NewGuid().ToString('N'))"
|
|
$publicKey = "c17h-pub-$($spec.key)-$([guid]::NewGuid().ToString('N'))"
|
|
$joinRequest = Invoke-Api -Method Post -Path "/clusters/$clusterID/join-requests" -Body @{
|
|
join_token = $joinToken.join_token.token
|
|
node_name = $spec.name
|
|
node_fingerprint = $fingerprint
|
|
public_key = $publicKey
|
|
reported_capabilities = @{
|
|
can_accept_node_ingress = $true
|
|
can_route_mesh = $true
|
|
testing_node = $true
|
|
}
|
|
reported_facts = @{
|
|
os = "linux"
|
|
runtime = "docker-test"
|
|
stage = "c17h"
|
|
}
|
|
requested_roles = $spec.roles
|
|
}
|
|
$approved = Invoke-Api -Method Post -Path "/clusters/$clusterID/join-requests/$($joinRequest.join_request.id)/approve" -Body @{
|
|
actor_user_id = $actorUserID
|
|
node_key = $fingerprint
|
|
ownership_type = "platform_managed"
|
|
owner_organization_id = $null
|
|
}
|
|
$nodeID = $approved.node_bootstrap.node_id
|
|
foreach ($role in $spec.roles) {
|
|
Invoke-Api -Method Post -Path "/clusters/$clusterID/nodes/$nodeID/roles" -Body @{
|
|
actor_user_id = $actorUserID
|
|
role = $role
|
|
status = "active"
|
|
policy = @{
|
|
stage = "c17h"
|
|
synthetic_only = $true
|
|
}
|
|
} | Out-Null
|
|
}
|
|
$nodes[$spec.key] = [pscustomobject]@{
|
|
id = $nodeID
|
|
name = $spec.name
|
|
fingerprint = $fingerprint
|
|
public_key = $publicKey
|
|
port = $spec.port
|
|
roles = $spec.roles
|
|
}
|
|
}
|
|
|
|
$nodeAID = $nodes["a"].id
|
|
$nodeRID = $nodes["r"].id
|
|
$nodeBID = $nodes["b"].id
|
|
$nodeCID = $nodes["c"].id
|
|
|
|
$peerEndpointsDirect = @{}
|
|
$peerEndpointsDirect[$nodeAID] = "http://127.0.0.1:$($nodes["a"].port)"
|
|
$peerEndpointsDirect[$nodeBID] = "http://127.0.0.1:$($nodes["b"].port)"
|
|
|
|
$peerEndpointsRelay = @{}
|
|
$peerEndpointsRelay[$nodeAID] = "http://127.0.0.1:$($nodes["a"].port)"
|
|
$peerEndpointsRelay[$nodeRID] = "http://127.0.0.1:$($nodes["r"].port)"
|
|
$peerEndpointsRelay[$nodeCID] = "http://127.0.0.1:$($nodes["c"].port)"
|
|
|
|
Write-Host "Creating synthetic route intents..."
|
|
$directIntent = Invoke-Api -Method Post -Path "/clusters/$clusterID/mesh/route-intents" -Body @{
|
|
actor_user_id = $actorUserID
|
|
source_selector = @{ node_id = $nodeAID }
|
|
destination_selector = @{ node_id = $nodeBID }
|
|
service_class = "control"
|
|
priority = 10
|
|
policy = @{
|
|
synthetic_enabled = $true
|
|
peer_endpoints = $peerEndpointsDirect
|
|
hops = @($nodeAID, $nodeBID)
|
|
allowed_channels = @("fabric_control", "route_control")
|
|
max_ttl = 4
|
|
max_hops = 4
|
|
route_version = "$runId-direct"
|
|
policy_version = "$runId-policy"
|
|
peer_directory_version = "$runId-peers"
|
|
production_forwarding = $false
|
|
}
|
|
}
|
|
$relayIntent = Invoke-Api -Method Post -Path "/clusters/$clusterID/mesh/route-intents" -Body @{
|
|
actor_user_id = $actorUserID
|
|
source_selector = @{ node_id = $nodeAID }
|
|
destination_selector = @{ node_id = $nodeCID }
|
|
service_class = "control"
|
|
priority = 20
|
|
policy = @{
|
|
synthetic_enabled = $true
|
|
peer_endpoints = $peerEndpointsRelay
|
|
hops = @($nodeAID, $nodeRID, $nodeCID)
|
|
allowed_channels = @("fabric_control", "route_control")
|
|
max_ttl = 6
|
|
max_hops = 6
|
|
route_version = "$runId-relay"
|
|
policy_version = "$runId-policy"
|
|
peer_directory_version = "$runId-peers"
|
|
production_forwarding = $false
|
|
}
|
|
}
|
|
|
|
$configs = @{}
|
|
foreach ($key in @("a", "r", "b", "c", "idle")) {
|
|
$configs[$key] = Invoke-Api -Method Get -Path "/clusters/$clusterID/nodes/$($nodes[$key].id)/mesh/synthetic-config"
|
|
}
|
|
|
|
Write-Host "Starting node-agent containers..."
|
|
foreach ($key in @("a", "r", "b", "c", "idle")) {
|
|
$node = $nodes[$key]
|
|
$containerName = "$nodePrefix$key"
|
|
docker --context $DockerContext rm -f $containerName 2>$null | Out-Null
|
|
$stateDir = Join-Path $env:TEMP "$runId-$key"
|
|
New-Item -ItemType Directory -Force -Path $stateDir | Out-Null
|
|
@{
|
|
node_id = $node.id
|
|
cluster_id = $clusterID
|
|
node_name = $node.name
|
|
node_fingerprint = $node.fingerprint
|
|
public_key = $node.public_key
|
|
identity_status = "active"
|
|
created_at = (Get-Date).ToUniversalTime().ToString("o")
|
|
updated_at = (Get-Date).ToUniversalTime().ToString("o")
|
|
} | ConvertTo-Json -Depth 10 | Set-Content -Encoding UTF8 -Path (Join-Path $stateDir "identity.json")
|
|
|
|
Invoke-Docker -Arguments @(
|
|
"create",
|
|
"--name", $containerName,
|
|
"--network", "host",
|
|
"-e", "RAP_BACKEND_URL=$backendBaseUrl",
|
|
"-e", "RAP_NODE_STATE_DIR=/tmp/state",
|
|
"-e", "RAP_HEARTBEAT_INTERVAL_SECONDS=5",
|
|
"-e", "RAP_MESH_SYNTHETIC_RUNTIME_ENABLED=true",
|
|
"-e", "RAP_MESH_LISTEN_ADDR=0.0.0.0:$($node.port)",
|
|
$NodeAgentImageTag,
|
|
"-backend-url", $backendBaseUrl,
|
|
"-state-dir", "/tmp/state",
|
|
"-heartbeat-interval", "5s",
|
|
"-mesh-synthetic-runtime-enabled",
|
|
"-mesh-listen-addr", "0.0.0.0:$($node.port)"
|
|
) | Out-Null
|
|
Invoke-Docker -Arguments @("cp", "$stateDir\.", "$containerName`:/tmp/state")
|
|
Invoke-Docker -Arguments @("start", $containerName) | Out-Null
|
|
}
|
|
|
|
Write-Host "Waiting for synthetic observations..."
|
|
Start-Sleep -Seconds 25
|
|
|
|
$links = Invoke-Api -Method Get -Path "/clusters/$clusterID/mesh/links?actor_user_id=$actorUserID"
|
|
$summary = Invoke-Api -Method Get -Path "/cluster-admin-summaries?actor_user_id=$actorUserID"
|
|
|
|
$nodeLogs = @{}
|
|
foreach ($key in @("a", "r", "b", "c", "idle")) {
|
|
$nodeLogs[$key] = Invoke-DockerText -Arguments @("logs", "--tail", "30", "$nodePrefix$key")
|
|
}
|
|
$backendLogs = Invoke-DockerText -Arguments @("logs", "--tail", "30", $backendName)
|
|
|
|
$meshLinks = @($links.mesh_links)
|
|
$routeHealthLinks = @($meshLinks | Where-Object {
|
|
$_.metadata.observation_type -eq "synthetic_route_health" -and
|
|
$_.metadata.config_source -eq "control_plane"
|
|
})
|
|
$directHealth = @($routeHealthLinks | Where-Object { $_.metadata.route_id -eq $directIntent.route_intent.id -and $_.link_status -eq "reachable" })
|
|
$relayHealth = @($routeHealthLinks | Where-Object { $_.metadata.route_id -eq $relayIntent.route_intent.id -and $_.link_status -eq "reachable" })
|
|
|
|
$passMatrix = [ordered]@{
|
|
backend_ready = $true
|
|
platform_owner_login = [bool]$actorUserID
|
|
cluster_created = [bool]$clusterID
|
|
fabric_testing_flags_enabled = $true
|
|
node_a_scoped_config_enabled = $configs["a"].synthetic_mesh_config.enabled -eq $true
|
|
node_a_has_direct_and_relay_routes = @($configs["a"].synthetic_mesh_config.routes).Count -eq 2
|
|
relay_node_has_only_relay_route = @($configs["r"].synthetic_mesh_config.routes).Count -eq 1
|
|
direct_destination_node_has_only_direct_route = @($configs["b"].synthetic_mesh_config.routes).Count -eq 1
|
|
relay_destination_node_has_only_relay_route = @($configs["c"].synthetic_mesh_config.routes).Count -eq 1
|
|
idle_node_has_no_routes = @($configs["idle"].synthetic_mesh_config.routes).Count -eq 0
|
|
node_agents_started = @("a", "r", "b", "c", "idle").Count -eq 5
|
|
control_plane_config_used = ($nodeLogs["a"] -join "`n") -match "control_plane"
|
|
direct_route_health_reported = $directHealth.Count -gt 0
|
|
relay_route_health_reported = $relayHealth.Count -gt 0
|
|
production_forwarding_disabled = (
|
|
$configs["a"].synthetic_mesh_config.production_forwarding -eq $false -and
|
|
$configs["r"].synthetic_mesh_config.production_forwarding -eq $false -and
|
|
$configs["b"].synthetic_mesh_config.production_forwarding -eq $false -and
|
|
$configs["c"].synthetic_mesh_config.production_forwarding -eq $false -and
|
|
$configs["idle"].synthetic_mesh_config.production_forwarding -eq $false
|
|
)
|
|
}
|
|
|
|
$failed = @($passMatrix.GetEnumerator() | Where-Object { -not $_.Value })
|
|
$result = [pscustomobject]@{
|
|
stage = "C17H deployed multi-agent synthetic config smoke"
|
|
run_id = $runId
|
|
backend_base_url = $backendPublicBaseUrl
|
|
cluster_id = $clusterID
|
|
node_ids = @{
|
|
a = $nodes["a"].id
|
|
r = $nodes["r"].id
|
|
b = $nodes["b"].id
|
|
c = $nodes["c"].id
|
|
idle = $nodes["idle"].id
|
|
}
|
|
route_intents = @{
|
|
direct = $directIntent.route_intent.id
|
|
relay = $relayIntent.route_intent.id
|
|
}
|
|
scoped_config_route_counts = @{
|
|
a = @($configs["a"].synthetic_mesh_config.routes).Count
|
|
r = @($configs["r"].synthetic_mesh_config.routes).Count
|
|
b = @($configs["b"].synthetic_mesh_config.routes).Count
|
|
c = @($configs["c"].synthetic_mesh_config.routes).Count
|
|
idle = @($configs["idle"].synthetic_mesh_config.routes).Count
|
|
}
|
|
mesh_link_count = $meshLinks.Count
|
|
route_health_count = $routeHealthLinks.Count
|
|
pass_matrix = $passMatrix
|
|
direct_route_health = $directHealth | Select-Object -First 3
|
|
relay_route_health = $relayHealth | Select-Object -First 3
|
|
cluster_summaries = $summary.cluster_summaries
|
|
backend_log_tail = $backendLogs
|
|
node_log_tail = $nodeLogs
|
|
containers_left_running = [bool]$KeepRunning
|
|
}
|
|
|
|
$result | ConvertTo-Json -Depth 40
|
|
|
|
if ($failed.Count -gt 0) {
|
|
throw "C17H smoke failed: $($failed.Name -join ', ')"
|
|
}
|
|
|
|
if (-not $KeepRunning) {
|
|
Write-Host "Cleaning up C17H containers..."
|
|
Remove-C17HContainers
|
|
}
|