Initial project snapshot
This commit is contained in:
@@ -0,0 +1,390 @@
|
||||
param(
|
||||
[string]$DockerSshAlias = "test-docker",
|
||||
[string]$NodeAgentImageTag = "rap-node-agent:c17h-synthetic-smoke",
|
||||
[string]$AdminEmail = "fabric-owner-c17h@example.local",
|
||||
[string]$AdminPassword = "SmokePass!123",
|
||||
[int]$ApiPort = 18080,
|
||||
[int]$MeshBasePort = 19081,
|
||||
[switch]$KeepRunning
|
||||
)
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
$backendPublicBaseUrl = "http://192.168.200.61:$ApiPort/api/v1"
|
||||
$backendContainerBaseUrl = "http://127.0.0.1:$ApiPort/api/v1"
|
||||
$runId = "c17h-ssh-" + (Get-Date -Format "yyyyMMdd-HHmmss")
|
||||
$nodePrefix = "rap_c17h_node_"
|
||||
|
||||
function Invoke-RemoteDocker {
|
||||
param([string[]]$Arguments)
|
||||
& ssh $DockerSshAlias docker @Arguments
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "ssh $DockerSshAlias docker $($Arguments -join ' ') failed with exit code $LASTEXITCODE"
|
||||
}
|
||||
}
|
||||
|
||||
function Invoke-RemoteDockerText {
|
||||
param([string[]]$Arguments)
|
||||
$previousErrorActionPreference = $ErrorActionPreference
|
||||
$ErrorActionPreference = "Continue"
|
||||
try {
|
||||
$output = & ssh $DockerSshAlias docker @Arguments 2>&1 | ForEach-Object { $_.ToString() }
|
||||
}
|
||||
finally {
|
||||
$ErrorActionPreference = $previousErrorActionPreference
|
||||
}
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "ssh $DockerSshAlias docker $($Arguments -join ' ') failed with exit code $LASTEXITCODE"
|
||||
}
|
||||
return $output
|
||||
}
|
||||
|
||||
function Invoke-RemoteShell {
|
||||
param([string]$Command)
|
||||
& ssh $DockerSshAlias $Command
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "ssh $DockerSshAlias $Command failed with exit code $LASTEXITCODE"
|
||||
}
|
||||
}
|
||||
|
||||
function Send-RemoteFile {
|
||||
param(
|
||||
[string]$RemotePath,
|
||||
[string]$Content
|
||||
)
|
||||
$Content | & ssh $DockerSshAlias "cat > '$RemotePath'"
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "write remote file failed: $RemotePath"
|
||||
}
|
||||
}
|
||||
|
||||
function Invoke-Api {
|
||||
param(
|
||||
[string]$Method,
|
||||
[string]$Path,
|
||||
[object]$Body = $null
|
||||
)
|
||||
$uri = "$backendPublicBaseUrl$Path"
|
||||
if ($null -eq $Body) {
|
||||
return Invoke-RestMethod -Method $Method -Uri $uri -TimeoutSec 30
|
||||
}
|
||||
return Invoke-RestMethod -Method $Method -Uri $uri -ContentType "application/json" -Body ($Body | ConvertTo-Json -Depth 40) -TimeoutSec 30
|
||||
}
|
||||
|
||||
function Wait-HttpReady {
|
||||
param([string]$Url)
|
||||
for ($i = 0; $i -lt 60; $i++) {
|
||||
try {
|
||||
$response = Invoke-WebRequest -UseBasicParsing -Uri $Url -TimeoutSec 2
|
||||
if ($response.StatusCode -ge 200 -and $response.StatusCode -lt 300) {
|
||||
return
|
||||
}
|
||||
}
|
||||
catch {
|
||||
Start-Sleep -Seconds 1
|
||||
}
|
||||
}
|
||||
throw "Timed out waiting for $Url"
|
||||
}
|
||||
|
||||
function Remove-NodeContainers {
|
||||
foreach ($key in @("a", "b", "c", "r", "idle")) {
|
||||
& ssh $DockerSshAlias docker rm -f "$nodePrefix$key" 2>$null | Out-Null
|
||||
}
|
||||
}
|
||||
|
||||
Wait-HttpReady -Url "http://192.168.200.61:$ApiPort/readyz"
|
||||
|
||||
$login = Invoke-Api -Method Post -Path "/auth/login" -Body @{
|
||||
email = $AdminEmail
|
||||
password = $AdminPassword
|
||||
device_fingerprint = "c17h-ssh-smoke-device"
|
||||
device_label = "C17H SSH synthetic smoke"
|
||||
trust_device = $true
|
||||
}
|
||||
$actorUserID = $login.user.id
|
||||
|
||||
$cluster = Invoke-Api -Method Post -Path "/clusters/" -Body @{
|
||||
actor_user_id = $actorUserID
|
||||
slug = "c17h-ssh-$((New-Guid).Guid.Substring(0, 8))"
|
||||
name = "C17H SSH Synthetic Mesh Smoke"
|
||||
region = "docker-test"
|
||||
metadata = @{
|
||||
stage = "c17h"
|
||||
run_id = $runId
|
||||
production_forwarding = $false
|
||||
created_by = "c17h-multi-agent-synthetic-smoke-ssh.ps1"
|
||||
}
|
||||
}
|
||||
$clusterID = $cluster.cluster.id
|
||||
|
||||
Invoke-Api -Method Put -Path "/fabric/testing-flags" -Body @{
|
||||
actor_user_id = $actorUserID
|
||||
scope_type = "platform"
|
||||
scope_id = $null
|
||||
cluster_id = $null
|
||||
enabled = $true
|
||||
telemetry_enabled = $true
|
||||
synthetic_links_enabled = $true
|
||||
history_retention_hours = 24
|
||||
metadata = @{
|
||||
stage = "c17h"
|
||||
run_id = $runId
|
||||
production_forwarding = $false
|
||||
service_workload_traffic = $false
|
||||
}
|
||||
} | Out-Null
|
||||
|
||||
$joinToken = Invoke-Api -Method Post -Path "/clusters/$clusterID/join-tokens" -Body @{
|
||||
actor_user_id = $actorUserID
|
||||
scope = @{ purpose = "c17h-synthetic-smoke"; roles = @("core-mesh", "relay-node") }
|
||||
expires_at = (Get-Date).ToUniversalTime().AddHours(2).ToString("o")
|
||||
max_uses = 5
|
||||
}
|
||||
|
||||
$nodeSpecs = @(
|
||||
@{ key = "a"; name = "c17h-node-a"; roles = @("core-mesh"); port = $MeshBasePort },
|
||||
@{ key = "r"; name = "c17h-node-r"; roles = @("core-mesh", "relay-node"); port = ($MeshBasePort + 1) },
|
||||
@{ key = "b"; name = "c17h-node-b"; roles = @("core-mesh"); port = ($MeshBasePort + 2) },
|
||||
@{ key = "c"; name = "c17h-node-c"; roles = @("core-mesh"); port = ($MeshBasePort + 3) },
|
||||
@{ key = "idle"; name = "c17h-node-idle"; roles = @("core-mesh"); port = ($MeshBasePort + 4) }
|
||||
)
|
||||
|
||||
$nodes = @{}
|
||||
foreach ($spec in $nodeSpecs) {
|
||||
$fingerprint = "c17h-fp-$($spec.key)-$([guid]::NewGuid().ToString('N'))"
|
||||
$publicKey = "c17h-pub-$($spec.key)-$([guid]::NewGuid().ToString('N'))"
|
||||
$joinRequest = Invoke-Api -Method Post -Path "/clusters/$clusterID/join-requests" -Body @{
|
||||
join_token = $joinToken.join_token.token
|
||||
node_name = $spec.name
|
||||
node_fingerprint = $fingerprint
|
||||
public_key = $publicKey
|
||||
reported_capabilities = @{
|
||||
can_accept_node_ingress = $true
|
||||
can_route_mesh = $true
|
||||
testing_node = $true
|
||||
}
|
||||
reported_facts = @{
|
||||
os = "linux"
|
||||
runtime = "docker-test"
|
||||
stage = "c17h"
|
||||
run_id = $runId
|
||||
}
|
||||
requested_roles = $spec.roles
|
||||
}
|
||||
$approved = Invoke-Api -Method Post -Path "/clusters/$clusterID/join-requests/$($joinRequest.join_request.id)/approve" -Body @{
|
||||
actor_user_id = $actorUserID
|
||||
node_key = $fingerprint
|
||||
ownership_type = "platform_managed"
|
||||
owner_organization_id = $null
|
||||
}
|
||||
$nodeID = $approved.node_bootstrap.node_id
|
||||
foreach ($role in $spec.roles) {
|
||||
Invoke-Api -Method Post -Path "/clusters/$clusterID/nodes/$nodeID/roles" -Body @{
|
||||
actor_user_id = $actorUserID
|
||||
role = $role
|
||||
status = "active"
|
||||
policy = @{
|
||||
stage = "c17h"
|
||||
run_id = $runId
|
||||
synthetic_only = $true
|
||||
}
|
||||
} | Out-Null
|
||||
}
|
||||
$nodes[$spec.key] = [pscustomobject]@{
|
||||
id = $nodeID
|
||||
name = $spec.name
|
||||
fingerprint = $fingerprint
|
||||
public_key = $publicKey
|
||||
port = $spec.port
|
||||
roles = $spec.roles
|
||||
}
|
||||
}
|
||||
|
||||
$nodeAID = $nodes["a"].id
|
||||
$nodeRID = $nodes["r"].id
|
||||
$nodeBID = $nodes["b"].id
|
||||
$nodeCID = $nodes["c"].id
|
||||
$routeExpiresAt = (Get-Date).ToUniversalTime().AddHours(2).ToString("o")
|
||||
|
||||
$peerEndpointsDirect = @{}
|
||||
$peerEndpointsDirect[$nodeAID] = "http://127.0.0.1:$($nodes["a"].port)"
|
||||
$peerEndpointsDirect[$nodeBID] = "http://127.0.0.1:$($nodes["b"].port)"
|
||||
|
||||
$peerEndpointsRelay = @{}
|
||||
$peerEndpointsRelay[$nodeAID] = "http://127.0.0.1:$($nodes["a"].port)"
|
||||
$peerEndpointsRelay[$nodeRID] = "http://127.0.0.1:$($nodes["r"].port)"
|
||||
$peerEndpointsRelay[$nodeCID] = "http://127.0.0.1:$($nodes["c"].port)"
|
||||
|
||||
$directIntent = Invoke-Api -Method Post -Path "/clusters/$clusterID/mesh/route-intents" -Body @{
|
||||
actor_user_id = $actorUserID
|
||||
source_selector = @{ node_id = $nodeAID }
|
||||
destination_selector = @{ node_id = $nodeBID }
|
||||
service_class = "control"
|
||||
priority = 10
|
||||
policy = @{
|
||||
synthetic_enabled = $true
|
||||
peer_endpoints = $peerEndpointsDirect
|
||||
hops = @($nodeAID, $nodeBID)
|
||||
allowed_channels = @("fabric_control", "route_control")
|
||||
max_ttl = 4
|
||||
max_hops = 4
|
||||
expires_at = $routeExpiresAt
|
||||
route_version = "$runId-direct"
|
||||
policy_version = "$runId-policy"
|
||||
peer_directory_version = "$runId-peers"
|
||||
production_forwarding = $false
|
||||
}
|
||||
}
|
||||
|
||||
$relayIntent = Invoke-Api -Method Post -Path "/clusters/$clusterID/mesh/route-intents" -Body @{
|
||||
actor_user_id = $actorUserID
|
||||
source_selector = @{ node_id = $nodeAID }
|
||||
destination_selector = @{ node_id = $nodeCID }
|
||||
service_class = "control"
|
||||
priority = 20
|
||||
policy = @{
|
||||
synthetic_enabled = $true
|
||||
peer_endpoints = $peerEndpointsRelay
|
||||
hops = @($nodeAID, $nodeRID, $nodeCID)
|
||||
allowed_channels = @("fabric_control", "route_control")
|
||||
max_ttl = 6
|
||||
max_hops = 6
|
||||
expires_at = $routeExpiresAt
|
||||
route_version = "$runId-relay"
|
||||
policy_version = "$runId-policy"
|
||||
peer_directory_version = "$runId-peers"
|
||||
production_forwarding = $false
|
||||
}
|
||||
}
|
||||
|
||||
$configs = @{}
|
||||
foreach ($key in @("a", "r", "b", "c", "idle")) {
|
||||
$configs[$key] = Invoke-Api -Method Get -Path "/clusters/$clusterID/nodes/$($nodes[$key].id)/mesh/synthetic-config"
|
||||
}
|
||||
|
||||
Remove-NodeContainers
|
||||
|
||||
foreach ($key in @("a", "r", "b", "c", "idle")) {
|
||||
$node = $nodes[$key]
|
||||
$containerName = "$nodePrefix$key"
|
||||
$remoteStateDir = "/tmp/$runId-$key"
|
||||
Invoke-RemoteShell -Command "rm -rf '$remoteStateDir' && mkdir -p '$remoteStateDir'"
|
||||
$identity = @{
|
||||
node_id = $node.id
|
||||
cluster_id = $clusterID
|
||||
node_name = $node.name
|
||||
node_fingerprint = $node.fingerprint
|
||||
public_key = $node.public_key
|
||||
identity_status = "active"
|
||||
created_at = (Get-Date).ToUniversalTime().ToString("o")
|
||||
updated_at = (Get-Date).ToUniversalTime().ToString("o")
|
||||
} | ConvertTo-Json -Depth 10
|
||||
Send-RemoteFile -RemotePath "$remoteStateDir/identity.json" -Content $identity
|
||||
Invoke-RemoteDocker -Arguments @(
|
||||
"create",
|
||||
"--name", $containerName,
|
||||
"--network", "host",
|
||||
"-e", "RAP_BACKEND_URL=$backendContainerBaseUrl",
|
||||
"-e", "RAP_NODE_STATE_DIR=/tmp/state",
|
||||
"-e", "RAP_HEARTBEAT_INTERVAL_SECONDS=5",
|
||||
"-e", "RAP_MESH_SYNTHETIC_RUNTIME_ENABLED=true",
|
||||
"-e", "RAP_MESH_LISTEN_ADDR=0.0.0.0:$($node.port)",
|
||||
$NodeAgentImageTag,
|
||||
"-backend-url", $backendContainerBaseUrl,
|
||||
"-state-dir", "/tmp/state",
|
||||
"-heartbeat-interval", "5s",
|
||||
"-mesh-synthetic-runtime-enabled",
|
||||
"-mesh-listen-addr", "0.0.0.0:$($node.port)"
|
||||
) | Out-Null
|
||||
Invoke-RemoteDocker -Arguments @("cp", "$remoteStateDir/.", "$containerName`:/tmp/state")
|
||||
Invoke-RemoteDocker -Arguments @("start", $containerName) | Out-Null
|
||||
}
|
||||
|
||||
Start-Sleep -Seconds 25
|
||||
|
||||
$links = Invoke-Api -Method Get -Path "/clusters/$clusterID/mesh/links?actor_user_id=$actorUserID"
|
||||
$summary = Invoke-Api -Method Get -Path "/cluster-admin-summaries?actor_user_id=$actorUserID"
|
||||
|
||||
$nodeLogs = @{}
|
||||
foreach ($key in @("a", "r", "b", "c", "idle")) {
|
||||
$nodeLogs[$key] = Invoke-RemoteDockerText -Arguments @("logs", "$nodePrefix$key")
|
||||
}
|
||||
$backendLogs = Invoke-RemoteDockerText -Arguments @("logs", "--tail", "50", "rap_c17h_backend")
|
||||
|
||||
$meshLinks = @($links.mesh_links)
|
||||
$routeHealthLinks = @($meshLinks | Where-Object {
|
||||
$_.metadata.observation_type -eq "synthetic_route_health" -and
|
||||
$_.metadata.config_source -eq "control_plane"
|
||||
})
|
||||
$directHealth = @($routeHealthLinks | Where-Object { $_.metadata.route_id -eq $directIntent.route_intent.id -and $_.link_status -eq "reachable" })
|
||||
$relayHealth = @($routeHealthLinks | Where-Object { $_.metadata.route_id -eq $relayIntent.route_intent.id -and $_.link_status -eq "reachable" })
|
||||
|
||||
$passMatrix = [ordered]@{
|
||||
backend_ready = $true
|
||||
platform_owner_login = [bool]$actorUserID
|
||||
cluster_created = [bool]$clusterID
|
||||
fabric_testing_flags_enabled = $true
|
||||
node_a_scoped_config_enabled = $configs["a"].synthetic_mesh_config.enabled -eq $true
|
||||
node_a_has_direct_and_relay_routes = @($configs["a"].synthetic_mesh_config.routes).Count -eq 2
|
||||
relay_node_has_only_relay_route = @($configs["r"].synthetic_mesh_config.routes).Count -eq 1
|
||||
direct_destination_node_has_only_direct_route = @($configs["b"].synthetic_mesh_config.routes).Count -eq 1
|
||||
relay_destination_node_has_only_relay_route = @($configs["c"].synthetic_mesh_config.routes).Count -eq 1
|
||||
idle_node_has_no_routes = @($configs["idle"].synthetic_mesh_config.routes).Count -eq 0
|
||||
control_plane_config_used = (@("a", "r", "b", "c", "idle") | Where-Object { ($nodeLogs[$_] -join "`n") -match "source=control_plane" }).Count -eq 5
|
||||
direct_route_health_reported = $directHealth.Count -gt 0
|
||||
relay_route_health_reported = $relayHealth.Count -gt 0
|
||||
production_forwarding_disabled = (
|
||||
$configs["a"].synthetic_mesh_config.production_forwarding -eq $false -and
|
||||
$configs["r"].synthetic_mesh_config.production_forwarding -eq $false -and
|
||||
$configs["b"].synthetic_mesh_config.production_forwarding -eq $false -and
|
||||
$configs["c"].synthetic_mesh_config.production_forwarding -eq $false -and
|
||||
$configs["idle"].synthetic_mesh_config.production_forwarding -eq $false
|
||||
)
|
||||
}
|
||||
|
||||
$result = [pscustomobject]@{
|
||||
stage = "C17H deployed multi-agent synthetic config smoke"
|
||||
run_id = $runId
|
||||
backend_base_url = $backendPublicBaseUrl
|
||||
cluster_id = $clusterID
|
||||
node_ids = @{
|
||||
a = $nodes["a"].id
|
||||
r = $nodes["r"].id
|
||||
b = $nodes["b"].id
|
||||
c = $nodes["c"].id
|
||||
idle = $nodes["idle"].id
|
||||
}
|
||||
route_intents = @{
|
||||
direct = $directIntent.route_intent.id
|
||||
relay = $relayIntent.route_intent.id
|
||||
}
|
||||
scoped_config_route_counts = @{
|
||||
a = @($configs["a"].synthetic_mesh_config.routes).Count
|
||||
r = @($configs["r"].synthetic_mesh_config.routes).Count
|
||||
b = @($configs["b"].synthetic_mesh_config.routes).Count
|
||||
c = @($configs["c"].synthetic_mesh_config.routes).Count
|
||||
idle = @($configs["idle"].synthetic_mesh_config.routes).Count
|
||||
}
|
||||
mesh_link_count = $meshLinks.Count
|
||||
route_health_count = $routeHealthLinks.Count
|
||||
pass_matrix = $passMatrix
|
||||
direct_route_health = $directHealth | Select-Object -First 3
|
||||
relay_route_health = $relayHealth | Select-Object -First 3
|
||||
cluster_summaries = $summary.cluster_summaries
|
||||
backend_log_tail = $backendLogs
|
||||
node_log_tail = $nodeLogs
|
||||
containers_left_running = [bool]$KeepRunning
|
||||
}
|
||||
|
||||
$failed = @($passMatrix.GetEnumerator() | Where-Object { -not $_.Value })
|
||||
$result | ConvertTo-Json -Depth 50
|
||||
|
||||
if ($failed.Count -gt 0) {
|
||||
throw "C17H smoke failed: $($failed.Name -join ', ')"
|
||||
}
|
||||
|
||||
if (-not $KeepRunning) {
|
||||
Remove-NodeContainers
|
||||
}
|
||||
@@ -0,0 +1,460 @@
|
||||
param(
|
||||
[string]$DockerContext = "test-ubuntu",
|
||||
[string]$BackendImageTag = "rap-backend:c17h-synthetic-smoke",
|
||||
[string]$NodeAgentImageTag = "rap-node-agent:c17h-synthetic-smoke",
|
||||
[string]$AdminEmail = "fabric-owner-c17h@example.local",
|
||||
[string]$AdminPassword = "SmokePass!123",
|
||||
[int]$ApiPort = 18080,
|
||||
[int]$PostgresPort = 15432,
|
||||
[int]$RedisPort = 16379,
|
||||
[int]$MeshBasePort = 19081,
|
||||
[switch]$KeepRunning
|
||||
)
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
$repoRoot = (Resolve-Path (Join-Path $PSScriptRoot "..\..")).ProviderPath
|
||||
$backendBaseUrl = "http://127.0.0.1:$ApiPort/api/v1"
|
||||
$backendPublicBaseUrl = "http://192.168.200.61:$ApiPort/api/v1"
|
||||
$runId = "c17h-" + (Get-Date -Format "yyyyMMdd-HHmmss")
|
||||
|
||||
$postgresName = "rap_c17h_postgres"
|
||||
$redisName = "rap_c17h_redis"
|
||||
$backendName = "rap_c17h_backend"
|
||||
$nodePrefix = "rap_c17h_node_"
|
||||
|
||||
function Invoke-Docker {
|
||||
param([string[]]$Arguments)
|
||||
docker --context $DockerContext @Arguments
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "docker --context $DockerContext $($Arguments -join ' ') failed with exit code $LASTEXITCODE"
|
||||
}
|
||||
}
|
||||
|
||||
function Invoke-DockerText {
|
||||
param([string[]]$Arguments)
|
||||
$output = docker --context $DockerContext @Arguments 2>&1 | ForEach-Object { $_.ToString() }
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "docker --context $DockerContext $($Arguments -join ' ') failed with exit code $LASTEXITCODE"
|
||||
}
|
||||
return $output
|
||||
}
|
||||
|
||||
function Invoke-Api {
|
||||
param(
|
||||
[string]$Method,
|
||||
[string]$Path,
|
||||
[object]$Body = $null
|
||||
)
|
||||
$uri = "$backendPublicBaseUrl$Path"
|
||||
if ($null -eq $Body) {
|
||||
return Invoke-RestMethod -Method $Method -Uri $uri -TimeoutSec 30
|
||||
}
|
||||
return Invoke-RestMethod -Method $Method -Uri $uri -ContentType "application/json" -Body ($Body | ConvertTo-Json -Depth 30) -TimeoutSec 30
|
||||
}
|
||||
|
||||
function Wait-HttpReady {
|
||||
param([string]$Url)
|
||||
for ($i = 0; $i -lt 60; $i++) {
|
||||
try {
|
||||
$response = Invoke-WebRequest -UseBasicParsing -Uri $Url -TimeoutSec 2
|
||||
if ($response.StatusCode -ge 200 -and $response.StatusCode -lt 300) {
|
||||
return
|
||||
}
|
||||
}
|
||||
catch {
|
||||
Start-Sleep -Seconds 1
|
||||
}
|
||||
}
|
||||
throw "Timed out waiting for $Url"
|
||||
}
|
||||
|
||||
function Invoke-PostgresSql {
|
||||
param([string]$Sql)
|
||||
$Sql | docker --context $DockerContext exec -i $postgresName psql -U rap_user -d remote_access_platform -v ON_ERROR_STOP=1 -f -
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "psql command failed"
|
||||
}
|
||||
}
|
||||
|
||||
function Remove-C17HContainers {
|
||||
$names = @($backendName, $postgresName, $redisName)
|
||||
foreach ($letter in @("a", "b", "c", "r", "idle")) {
|
||||
$names += "$nodePrefix$letter"
|
||||
}
|
||||
foreach ($name in $names) {
|
||||
docker --context $DockerContext rm -f $name 2>$null | Out-Null
|
||||
}
|
||||
}
|
||||
|
||||
Write-Host "C17H smoke run: $runId"
|
||||
Write-Host "Using Docker context: $DockerContext"
|
||||
|
||||
Remove-C17HContainers
|
||||
|
||||
Write-Host "Building backend image..."
|
||||
Invoke-Docker -Arguments @("build", "-f", "$repoRoot\backend\Dockerfile", "-t", $BackendImageTag, "$repoRoot\backend")
|
||||
|
||||
Write-Host "Building node-agent image..."
|
||||
Invoke-Docker -Arguments @("build", "-f", "$repoRoot\agents\rap-node-agent\Dockerfile", "-t", $NodeAgentImageTag, "$repoRoot")
|
||||
|
||||
Write-Host "Starting PostgreSQL and Redis..."
|
||||
Invoke-Docker -Arguments @(
|
||||
"run", "-d",
|
||||
"--name", $postgresName,
|
||||
"-e", "POSTGRES_DB=remote_access_platform",
|
||||
"-e", "POSTGRES_USER=rap_user",
|
||||
"-e", "POSTGRES_PASSWORD=rap_password",
|
||||
"-p", "$PostgresPort`:5432",
|
||||
"postgres:16"
|
||||
)
|
||||
Invoke-Docker -Arguments @(
|
||||
"run", "-d",
|
||||
"--name", $redisName,
|
||||
"-p", "$RedisPort`:6379",
|
||||
"redis:7"
|
||||
)
|
||||
|
||||
for ($i = 0; $i -lt 60; $i++) {
|
||||
docker --context $DockerContext exec $postgresName pg_isready -U rap_user -d remote_access_platform | Out-Null
|
||||
if ($LASTEXITCODE -eq 0) {
|
||||
break
|
||||
}
|
||||
Start-Sleep -Seconds 1
|
||||
}
|
||||
docker --context $DockerContext exec $postgresName pg_isready -U rap_user -d remote_access_platform | Out-Null
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "PostgreSQL did not become ready"
|
||||
}
|
||||
|
||||
Write-Host "Applying migrations..."
|
||||
$migrationFiles = Get-ChildItem -Path (Join-Path $repoRoot "backend\migrations") -Filter "*.up.sql" | Sort-Object Name
|
||||
foreach ($migration in $migrationFiles) {
|
||||
Write-Host "Applying $($migration.Name)"
|
||||
Get-Content -Raw $migration.FullName | docker --context $DockerContext exec -i $postgresName psql -U rap_user -d remote_access_platform -v ON_ERROR_STOP=1 -f -
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "Migration failed: $($migration.Name)"
|
||||
}
|
||||
}
|
||||
|
||||
Write-Host "Seeding platform owner..."
|
||||
$adminUserID = [guid]::NewGuid().ToString()
|
||||
$adminHash = '$2a$10$AqLRexkI1yXbuiMPU6dHM.KVUhF.t..9NolyK4OOodQTyTsHyG.7u'
|
||||
Invoke-PostgresSql -Sql @"
|
||||
INSERT INTO users (id, email, password_hash, mfa_enabled, platform_role)
|
||||
VALUES ('$adminUserID'::uuid, '$AdminEmail', '$adminHash', FALSE, 'platform_admin')
|
||||
ON CONFLICT (email) DO UPDATE SET
|
||||
password_hash = EXCLUDED.password_hash,
|
||||
platform_role = 'platform_admin',
|
||||
updated_at = NOW();
|
||||
"@
|
||||
|
||||
Write-Host "Starting backend..."
|
||||
Invoke-Docker -Arguments @(
|
||||
"run", "-d",
|
||||
"--name", $backendName,
|
||||
"--network", "host",
|
||||
"-e", "APP_NAME=rap-api",
|
||||
"-e", "APP_ENV=c17h-smoke",
|
||||
"-e", "HTTP_HOST=0.0.0.0",
|
||||
"-e", "HTTP_PORT=$ApiPort",
|
||||
"-e", "POSTGRES_DSN=postgres://rap_user:rap_password@127.0.0.1:$PostgresPort/remote_access_platform?sslmode=disable",
|
||||
"-e", "REDIS_ADDR=127.0.0.1:$RedisPort",
|
||||
"-e", "AUTH_ACCESS_TOKEN_SECRET=c17h-access-secret",
|
||||
"-e", "AUTH_REFRESH_HASH_SECRET=c17h-refresh-secret",
|
||||
$BackendImageTag
|
||||
)
|
||||
Wait-HttpReady -Url "http://192.168.200.61:$ApiPort/readyz"
|
||||
|
||||
Write-Host "Logging in as platform owner..."
|
||||
$login = Invoke-Api -Method Post -Path "/auth/login" -Body @{
|
||||
email = $AdminEmail
|
||||
password = $AdminPassword
|
||||
device_fingerprint = "c17h-smoke-device"
|
||||
device_label = "C17H synthetic smoke"
|
||||
trust_device = $true
|
||||
}
|
||||
$actorUserID = $login.user.id
|
||||
|
||||
Write-Host "Creating C17H cluster..."
|
||||
$cluster = Invoke-Api -Method Post -Path "/clusters/" -Body @{
|
||||
actor_user_id = $actorUserID
|
||||
slug = "c17h-$((New-Guid).Guid.Substring(0, 8))"
|
||||
name = "C17H Synthetic Mesh Smoke"
|
||||
region = "test-docker"
|
||||
metadata = @{
|
||||
stage = "c17h"
|
||||
production_forwarding = $false
|
||||
created_by = "c17h-multi-agent-synthetic-smoke.ps1"
|
||||
}
|
||||
}
|
||||
$clusterID = $cluster.cluster.id
|
||||
|
||||
Write-Host "Enabling test-only Fabric flags..."
|
||||
Invoke-Api -Method Put -Path "/fabric/testing-flags" -Body @{
|
||||
actor_user_id = $actorUserID
|
||||
scope_type = "platform"
|
||||
scope_id = $null
|
||||
cluster_id = $null
|
||||
enabled = $true
|
||||
telemetry_enabled = $true
|
||||
synthetic_links_enabled = $true
|
||||
history_retention_hours = 24
|
||||
metadata = @{
|
||||
stage = "c17h"
|
||||
production_forwarding = $false
|
||||
service_workload_traffic = $false
|
||||
}
|
||||
} | Out-Null
|
||||
|
||||
$joinToken = Invoke-Api -Method Post -Path "/clusters/$clusterID/join-tokens" -Body @{
|
||||
actor_user_id = $actorUserID
|
||||
scope = @{ purpose = "c17h-synthetic-smoke"; roles = @("core-mesh", "relay-node") }
|
||||
expires_at = (Get-Date).ToUniversalTime().AddHours(2).ToString("o")
|
||||
max_uses = 5
|
||||
}
|
||||
|
||||
$nodeSpecs = @(
|
||||
@{ key = "a"; name = "c17h-node-a"; roles = @("core-mesh"); port = $MeshBasePort },
|
||||
@{ key = "r"; name = "c17h-node-r"; roles = @("core-mesh", "relay-node"); port = ($MeshBasePort + 1) },
|
||||
@{ key = "b"; name = "c17h-node-b"; roles = @("core-mesh"); port = ($MeshBasePort + 2) },
|
||||
@{ key = "c"; name = "c17h-node-c"; roles = @("core-mesh"); port = ($MeshBasePort + 3) },
|
||||
@{ key = "idle"; name = "c17h-node-idle"; roles = @("core-mesh"); port = ($MeshBasePort + 4) }
|
||||
)
|
||||
|
||||
$nodes = @{}
|
||||
foreach ($spec in $nodeSpecs) {
|
||||
$fingerprint = "c17h-fp-$($spec.key)-$([guid]::NewGuid().ToString('N'))"
|
||||
$publicKey = "c17h-pub-$($spec.key)-$([guid]::NewGuid().ToString('N'))"
|
||||
$joinRequest = Invoke-Api -Method Post -Path "/clusters/$clusterID/join-requests" -Body @{
|
||||
join_token = $joinToken.join_token.token
|
||||
node_name = $spec.name
|
||||
node_fingerprint = $fingerprint
|
||||
public_key = $publicKey
|
||||
reported_capabilities = @{
|
||||
can_accept_node_ingress = $true
|
||||
can_route_mesh = $true
|
||||
testing_node = $true
|
||||
}
|
||||
reported_facts = @{
|
||||
os = "linux"
|
||||
runtime = "docker-test"
|
||||
stage = "c17h"
|
||||
}
|
||||
requested_roles = $spec.roles
|
||||
}
|
||||
$approved = Invoke-Api -Method Post -Path "/clusters/$clusterID/join-requests/$($joinRequest.join_request.id)/approve" -Body @{
|
||||
actor_user_id = $actorUserID
|
||||
node_key = $fingerprint
|
||||
ownership_type = "platform_managed"
|
||||
owner_organization_id = $null
|
||||
}
|
||||
$nodeID = $approved.node_bootstrap.node_id
|
||||
foreach ($role in $spec.roles) {
|
||||
Invoke-Api -Method Post -Path "/clusters/$clusterID/nodes/$nodeID/roles" -Body @{
|
||||
actor_user_id = $actorUserID
|
||||
role = $role
|
||||
status = "active"
|
||||
policy = @{
|
||||
stage = "c17h"
|
||||
synthetic_only = $true
|
||||
}
|
||||
} | Out-Null
|
||||
}
|
||||
$nodes[$spec.key] = [pscustomobject]@{
|
||||
id = $nodeID
|
||||
name = $spec.name
|
||||
fingerprint = $fingerprint
|
||||
public_key = $publicKey
|
||||
port = $spec.port
|
||||
roles = $spec.roles
|
||||
}
|
||||
}
|
||||
|
||||
$nodeAID = $nodes["a"].id
|
||||
$nodeRID = $nodes["r"].id
|
||||
$nodeBID = $nodes["b"].id
|
||||
$nodeCID = $nodes["c"].id
|
||||
|
||||
$peerEndpointsDirect = @{}
|
||||
$peerEndpointsDirect[$nodeAID] = "http://127.0.0.1:$($nodes["a"].port)"
|
||||
$peerEndpointsDirect[$nodeBID] = "http://127.0.0.1:$($nodes["b"].port)"
|
||||
|
||||
$peerEndpointsRelay = @{}
|
||||
$peerEndpointsRelay[$nodeAID] = "http://127.0.0.1:$($nodes["a"].port)"
|
||||
$peerEndpointsRelay[$nodeRID] = "http://127.0.0.1:$($nodes["r"].port)"
|
||||
$peerEndpointsRelay[$nodeCID] = "http://127.0.0.1:$($nodes["c"].port)"
|
||||
|
||||
Write-Host "Creating synthetic route intents..."
|
||||
$directIntent = Invoke-Api -Method Post -Path "/clusters/$clusterID/mesh/route-intents" -Body @{
|
||||
actor_user_id = $actorUserID
|
||||
source_selector = @{ node_id = $nodeAID }
|
||||
destination_selector = @{ node_id = $nodeBID }
|
||||
service_class = "control"
|
||||
priority = 10
|
||||
policy = @{
|
||||
synthetic_enabled = $true
|
||||
peer_endpoints = $peerEndpointsDirect
|
||||
hops = @($nodeAID, $nodeBID)
|
||||
allowed_channels = @("fabric_control", "route_control")
|
||||
max_ttl = 4
|
||||
max_hops = 4
|
||||
route_version = "$runId-direct"
|
||||
policy_version = "$runId-policy"
|
||||
peer_directory_version = "$runId-peers"
|
||||
production_forwarding = $false
|
||||
}
|
||||
}
|
||||
$relayIntent = Invoke-Api -Method Post -Path "/clusters/$clusterID/mesh/route-intents" -Body @{
|
||||
actor_user_id = $actorUserID
|
||||
source_selector = @{ node_id = $nodeAID }
|
||||
destination_selector = @{ node_id = $nodeCID }
|
||||
service_class = "control"
|
||||
priority = 20
|
||||
policy = @{
|
||||
synthetic_enabled = $true
|
||||
peer_endpoints = $peerEndpointsRelay
|
||||
hops = @($nodeAID, $nodeRID, $nodeCID)
|
||||
allowed_channels = @("fabric_control", "route_control")
|
||||
max_ttl = 6
|
||||
max_hops = 6
|
||||
route_version = "$runId-relay"
|
||||
policy_version = "$runId-policy"
|
||||
peer_directory_version = "$runId-peers"
|
||||
production_forwarding = $false
|
||||
}
|
||||
}
|
||||
|
||||
$configs = @{}
|
||||
foreach ($key in @("a", "r", "b", "c", "idle")) {
|
||||
$configs[$key] = Invoke-Api -Method Get -Path "/clusters/$clusterID/nodes/$($nodes[$key].id)/mesh/synthetic-config"
|
||||
}
|
||||
|
||||
Write-Host "Starting node-agent containers..."
|
||||
foreach ($key in @("a", "r", "b", "c", "idle")) {
|
||||
$node = $nodes[$key]
|
||||
$containerName = "$nodePrefix$key"
|
||||
docker --context $DockerContext rm -f $containerName 2>$null | Out-Null
|
||||
$stateDir = Join-Path $env:TEMP "$runId-$key"
|
||||
New-Item -ItemType Directory -Force -Path $stateDir | Out-Null
|
||||
@{
|
||||
node_id = $node.id
|
||||
cluster_id = $clusterID
|
||||
node_name = $node.name
|
||||
node_fingerprint = $node.fingerprint
|
||||
public_key = $node.public_key
|
||||
identity_status = "active"
|
||||
created_at = (Get-Date).ToUniversalTime().ToString("o")
|
||||
updated_at = (Get-Date).ToUniversalTime().ToString("o")
|
||||
} | ConvertTo-Json -Depth 10 | Set-Content -Encoding UTF8 -Path (Join-Path $stateDir "identity.json")
|
||||
|
||||
Invoke-Docker -Arguments @(
|
||||
"create",
|
||||
"--name", $containerName,
|
||||
"--network", "host",
|
||||
"-e", "RAP_BACKEND_URL=$backendBaseUrl",
|
||||
"-e", "RAP_NODE_STATE_DIR=/tmp/state",
|
||||
"-e", "RAP_HEARTBEAT_INTERVAL_SECONDS=5",
|
||||
"-e", "RAP_MESH_SYNTHETIC_RUNTIME_ENABLED=true",
|
||||
"-e", "RAP_MESH_LISTEN_ADDR=0.0.0.0:$($node.port)",
|
||||
$NodeAgentImageTag,
|
||||
"-backend-url", $backendBaseUrl,
|
||||
"-state-dir", "/tmp/state",
|
||||
"-heartbeat-interval", "5s",
|
||||
"-mesh-synthetic-runtime-enabled",
|
||||
"-mesh-listen-addr", "0.0.0.0:$($node.port)"
|
||||
) | Out-Null
|
||||
Invoke-Docker -Arguments @("cp", "$stateDir\.", "$containerName`:/tmp/state")
|
||||
Invoke-Docker -Arguments @("start", $containerName) | Out-Null
|
||||
}
|
||||
|
||||
Write-Host "Waiting for synthetic observations..."
|
||||
Start-Sleep -Seconds 25
|
||||
|
||||
$links = Invoke-Api -Method Get -Path "/clusters/$clusterID/mesh/links?actor_user_id=$actorUserID"
|
||||
$summary = Invoke-Api -Method Get -Path "/cluster-admin-summaries?actor_user_id=$actorUserID"
|
||||
|
||||
$nodeLogs = @{}
|
||||
foreach ($key in @("a", "r", "b", "c", "idle")) {
|
||||
$nodeLogs[$key] = Invoke-DockerText -Arguments @("logs", "--tail", "30", "$nodePrefix$key")
|
||||
}
|
||||
$backendLogs = Invoke-DockerText -Arguments @("logs", "--tail", "30", $backendName)
|
||||
|
||||
$meshLinks = @($links.mesh_links)
|
||||
$routeHealthLinks = @($meshLinks | Where-Object {
|
||||
$_.metadata.observation_type -eq "synthetic_route_health" -and
|
||||
$_.metadata.config_source -eq "control_plane"
|
||||
})
|
||||
$directHealth = @($routeHealthLinks | Where-Object { $_.metadata.route_id -eq $directIntent.route_intent.id -and $_.link_status -eq "reachable" })
|
||||
$relayHealth = @($routeHealthLinks | Where-Object { $_.metadata.route_id -eq $relayIntent.route_intent.id -and $_.link_status -eq "reachable" })
|
||||
|
||||
$passMatrix = [ordered]@{
|
||||
backend_ready = $true
|
||||
platform_owner_login = [bool]$actorUserID
|
||||
cluster_created = [bool]$clusterID
|
||||
fabric_testing_flags_enabled = $true
|
||||
node_a_scoped_config_enabled = $configs["a"].synthetic_mesh_config.enabled -eq $true
|
||||
node_a_has_direct_and_relay_routes = @($configs["a"].synthetic_mesh_config.routes).Count -eq 2
|
||||
relay_node_has_only_relay_route = @($configs["r"].synthetic_mesh_config.routes).Count -eq 1
|
||||
direct_destination_node_has_only_direct_route = @($configs["b"].synthetic_mesh_config.routes).Count -eq 1
|
||||
relay_destination_node_has_only_relay_route = @($configs["c"].synthetic_mesh_config.routes).Count -eq 1
|
||||
idle_node_has_no_routes = @($configs["idle"].synthetic_mesh_config.routes).Count -eq 0
|
||||
node_agents_started = @("a", "r", "b", "c", "idle").Count -eq 5
|
||||
control_plane_config_used = ($nodeLogs["a"] -join "`n") -match "control_plane"
|
||||
direct_route_health_reported = $directHealth.Count -gt 0
|
||||
relay_route_health_reported = $relayHealth.Count -gt 0
|
||||
production_forwarding_disabled = (
|
||||
$configs["a"].synthetic_mesh_config.production_forwarding -eq $false -and
|
||||
$configs["r"].synthetic_mesh_config.production_forwarding -eq $false -and
|
||||
$configs["b"].synthetic_mesh_config.production_forwarding -eq $false -and
|
||||
$configs["c"].synthetic_mesh_config.production_forwarding -eq $false -and
|
||||
$configs["idle"].synthetic_mesh_config.production_forwarding -eq $false
|
||||
)
|
||||
}
|
||||
|
||||
$failed = @($passMatrix.GetEnumerator() | Where-Object { -not $_.Value })
|
||||
$result = [pscustomobject]@{
|
||||
stage = "C17H deployed multi-agent synthetic config smoke"
|
||||
run_id = $runId
|
||||
backend_base_url = $backendPublicBaseUrl
|
||||
cluster_id = $clusterID
|
||||
node_ids = @{
|
||||
a = $nodes["a"].id
|
||||
r = $nodes["r"].id
|
||||
b = $nodes["b"].id
|
||||
c = $nodes["c"].id
|
||||
idle = $nodes["idle"].id
|
||||
}
|
||||
route_intents = @{
|
||||
direct = $directIntent.route_intent.id
|
||||
relay = $relayIntent.route_intent.id
|
||||
}
|
||||
scoped_config_route_counts = @{
|
||||
a = @($configs["a"].synthetic_mesh_config.routes).Count
|
||||
r = @($configs["r"].synthetic_mesh_config.routes).Count
|
||||
b = @($configs["b"].synthetic_mesh_config.routes).Count
|
||||
c = @($configs["c"].synthetic_mesh_config.routes).Count
|
||||
idle = @($configs["idle"].synthetic_mesh_config.routes).Count
|
||||
}
|
||||
mesh_link_count = $meshLinks.Count
|
||||
route_health_count = $routeHealthLinks.Count
|
||||
pass_matrix = $passMatrix
|
||||
direct_route_health = $directHealth | Select-Object -First 3
|
||||
relay_route_health = $relayHealth | Select-Object -First 3
|
||||
cluster_summaries = $summary.cluster_summaries
|
||||
backend_log_tail = $backendLogs
|
||||
node_log_tail = $nodeLogs
|
||||
containers_left_running = [bool]$KeepRunning
|
||||
}
|
||||
|
||||
$result | ConvertTo-Json -Depth 40
|
||||
|
||||
if ($failed.Count -gt 0) {
|
||||
throw "C17H smoke failed: $($failed.Name -join ', ')"
|
||||
}
|
||||
|
||||
if (-not $KeepRunning) {
|
||||
Write-Host "Cleaning up C17H containers..."
|
||||
Remove-C17HContainers
|
||||
}
|
||||
@@ -0,0 +1,917 @@
|
||||
param(
|
||||
[string]$DockerSshAlias = "test-docker",
|
||||
[string]$BackendImageTag = "rap-backend:c17z18-rendezvous-smoke",
|
||||
[string]$NodeAgentImageTag = "rap-node-agent:c17z18-rendezvous-smoke",
|
||||
[string]$AdminEmail = "fabric-owner-c17z18@example.local",
|
||||
[string]$AdminPassword = "SmokePass!123",
|
||||
[int]$ApiPort = 18120,
|
||||
[int]$PostgresPort = 15442,
|
||||
[int]$RedisPort = 16442,
|
||||
[int]$MeshBasePort = 19120,
|
||||
[switch]$KeepRunning
|
||||
)
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
$repoRoot = (Resolve-Path (Join-Path $PSScriptRoot "..\..")).ProviderPath
|
||||
$backendPublicBaseUrl = "http://192.168.200.61:$ApiPort/api/v1"
|
||||
$backendContainerBaseUrl = "http://127.0.0.1:$ApiPort/api/v1"
|
||||
$runId = "c17z18-" + (Get-Date -Format "yyyyMMdd-HHmmss")
|
||||
$remoteBuildDir = "/tmp/rap-c17z18-build-$runId"
|
||||
|
||||
$postgresName = "rap_c17z12_postgres"
|
||||
$redisName = "rap_c17z12_redis"
|
||||
$backendName = "rap_c17z12_backend"
|
||||
$nodePrefix = "rap_c17z12_node_"
|
||||
|
||||
function Invoke-RemoteDocker {
|
||||
param([string[]]$Arguments)
|
||||
& ssh $DockerSshAlias docker @Arguments
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "ssh $DockerSshAlias docker $($Arguments -join ' ') failed with exit code $LASTEXITCODE"
|
||||
}
|
||||
}
|
||||
|
||||
function Invoke-RemoteDockerText {
|
||||
param([string[]]$Arguments)
|
||||
$previousErrorActionPreference = $ErrorActionPreference
|
||||
$ErrorActionPreference = "Continue"
|
||||
try {
|
||||
$output = & ssh $DockerSshAlias docker @Arguments 2>&1 | ForEach-Object { $_.ToString() }
|
||||
}
|
||||
finally {
|
||||
$ErrorActionPreference = $previousErrorActionPreference
|
||||
}
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "ssh $DockerSshAlias docker $($Arguments -join ' ') failed with exit code $LASTEXITCODE"
|
||||
}
|
||||
return $output
|
||||
}
|
||||
|
||||
function Invoke-RemoteShell {
|
||||
param([string]$Command)
|
||||
& ssh $DockerSshAlias $Command
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "ssh $DockerSshAlias $Command failed with exit code $LASTEXITCODE"
|
||||
}
|
||||
}
|
||||
|
||||
function Invoke-RemotePostgresSql {
|
||||
param([string]$Sql)
|
||||
$Sql | & ssh $DockerSshAlias "docker exec -i $postgresName psql -U rap_user -d remote_access_platform -v ON_ERROR_STOP=1 -f -"
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "remote psql command failed"
|
||||
}
|
||||
}
|
||||
|
||||
function Send-RemoteFile {
|
||||
param(
|
||||
[string]$RemotePath,
|
||||
[string]$Content
|
||||
)
|
||||
$Content | & ssh $DockerSshAlias "cat > '$RemotePath'"
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "write remote file failed: $RemotePath"
|
||||
}
|
||||
}
|
||||
|
||||
function Send-RemoteBuildContext {
|
||||
Write-Host "Uploading backend and node-agent build context to $DockerSshAlias..."
|
||||
Invoke-RemoteShell -Command "rm -rf '$remoteBuildDir' && mkdir -p '$remoteBuildDir'"
|
||||
& tar -czf - -C $repoRoot "backend" "agents/rap-node-agent" | & ssh $DockerSshAlias "tar -xzf - -C '$remoteBuildDir'"
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "upload build context failed"
|
||||
}
|
||||
}
|
||||
|
||||
function Invoke-Api {
|
||||
param(
|
||||
[string]$Method,
|
||||
[string]$Path,
|
||||
[object]$Body = $null
|
||||
)
|
||||
$uri = "$backendPublicBaseUrl$Path"
|
||||
if ($null -eq $Body) {
|
||||
return Invoke-RestMethod -Method $Method -Uri $uri -TimeoutSec 30
|
||||
}
|
||||
return Invoke-RestMethod -Method $Method -Uri $uri -ContentType "application/json" -Body ($Body | ConvertTo-Json -Depth 50) -TimeoutSec 30
|
||||
}
|
||||
|
||||
function Wait-HttpReady {
|
||||
param([string]$Url)
|
||||
for ($i = 0; $i -lt 60; $i++) {
|
||||
try {
|
||||
$response = Invoke-WebRequest -UseBasicParsing -Uri $Url -TimeoutSec 2
|
||||
if ($response.StatusCode -ge 200 -and $response.StatusCode -lt 300) {
|
||||
return
|
||||
}
|
||||
}
|
||||
catch {
|
||||
Start-Sleep -Seconds 1
|
||||
}
|
||||
}
|
||||
throw "Timed out waiting for $Url"
|
||||
}
|
||||
|
||||
function Remove-C17Z12Containers {
|
||||
$names = @($backendName, $postgresName, $redisName)
|
||||
foreach ($key in @("a", "b", "c", "r", "idle")) {
|
||||
$names += "$nodePrefix$key"
|
||||
}
|
||||
foreach ($name in $names) {
|
||||
& ssh $DockerSshAlias docker rm -f $name 2>$null | Out-Null
|
||||
}
|
||||
}
|
||||
|
||||
function New-EndpointCandidate {
|
||||
param(
|
||||
[string]$EndpointID,
|
||||
[string]$NodeID,
|
||||
[string]$Address,
|
||||
[string]$Transport,
|
||||
[string]$Reachability,
|
||||
[string]$ConnectivityMode,
|
||||
[string]$NATType,
|
||||
[int]$Priority,
|
||||
[string[]]$PolicyTags = @()
|
||||
)
|
||||
return @{
|
||||
endpoint_id = $EndpointID
|
||||
node_id = $NodeID
|
||||
transport = $Transport
|
||||
address = $Address
|
||||
address_family = "ipv4"
|
||||
reachability = $Reachability
|
||||
nat_type = $NATType
|
||||
connectivity_mode = $ConnectivityMode
|
||||
region = "docker-test"
|
||||
priority = $Priority
|
||||
policy_tags = $PolicyTags
|
||||
last_verified_at = (Get-Date).ToUniversalTime().ToString("o")
|
||||
metadata = @{
|
||||
stage = "c17z18"
|
||||
run_id = $runId
|
||||
service_workload_traffic = $false
|
||||
production_payload_forwarding = $false
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function Get-OptionalProperty {
|
||||
param(
|
||||
[object]$Object,
|
||||
[string]$PropertyName
|
||||
)
|
||||
if ($null -eq $Object) {
|
||||
return $null
|
||||
}
|
||||
$property = $Object.PSObject.Properties[$PropertyName]
|
||||
if ($null -eq $property) {
|
||||
return $null
|
||||
}
|
||||
return $property.Value
|
||||
}
|
||||
|
||||
function Get-OptionalArrayCount {
|
||||
param(
|
||||
[object]$Object,
|
||||
[string]$PropertyName
|
||||
)
|
||||
$value = Get-OptionalProperty -Object $Object -PropertyName $PropertyName
|
||||
if ($null -eq $value) {
|
||||
return 0
|
||||
}
|
||||
return @($value).Count
|
||||
}
|
||||
|
||||
function Get-LatestHeartbeatMetadataReport {
|
||||
param(
|
||||
[string]$NodeID,
|
||||
[string]$PropertyName
|
||||
)
|
||||
$heartbeats = Invoke-Api -Method Get -Path "/clusters/$clusterID/nodes/$NodeID/heartbeats?actor_user_id=$actorUserID&limit=5"
|
||||
$latest = @($heartbeats.heartbeats) | Select-Object -First 1
|
||||
$metadata = Get-OptionalProperty -Object $latest -PropertyName "metadata"
|
||||
return Get-OptionalProperty -Object $metadata -PropertyName $PropertyName
|
||||
}
|
||||
|
||||
function Get-LatestRendezvousLeaseReport {
|
||||
param([string]$NodeID)
|
||||
return Get-LatestHeartbeatMetadataReport -NodeID $NodeID -PropertyName "mesh_rendezvous_lease_report"
|
||||
}
|
||||
|
||||
function Get-LatestRoutePathDecisionReport {
|
||||
param([string]$NodeID)
|
||||
return Get-LatestHeartbeatMetadataReport -NodeID $NodeID -PropertyName "mesh_route_path_decision_report"
|
||||
}
|
||||
|
||||
function Get-LatestRouteGenerationReport {
|
||||
param([string]$NodeID)
|
||||
return Get-LatestHeartbeatMetadataReport -NodeID $NodeID -PropertyName "mesh_route_generation_report"
|
||||
}
|
||||
|
||||
function Get-LatestRouteHealthConfigReport {
|
||||
param([string]$NodeID)
|
||||
return Get-LatestHeartbeatMetadataReport -NodeID $NodeID -PropertyName "mesh_route_health_config_report"
|
||||
}
|
||||
|
||||
function Get-LatestRouteHealthFeedbackRefreshReport {
|
||||
param([string]$NodeID)
|
||||
return Get-LatestHeartbeatMetadataReport -NodeID $NodeID -PropertyName "mesh_route_health_feedback_refresh_report"
|
||||
}
|
||||
|
||||
function Select-C17Z18RouteHealthSnapshot {
|
||||
param([object[]]$MeshLinks)
|
||||
$routeHealthLinks = @($MeshLinks | Where-Object {
|
||||
$_.metadata.observation_type -eq "synthetic_route_health" -and
|
||||
$_.metadata.config_source -eq "control_plane"
|
||||
})
|
||||
$directHealth = @($routeHealthLinks | Where-Object {
|
||||
$_.metadata.route_id -eq $directIntent.route_intent.id -and
|
||||
$_.link_status -eq "reachable"
|
||||
})
|
||||
$rendezvousHealth = @($routeHealthLinks | Where-Object {
|
||||
$_.source_node_id -eq $nodeAID -and
|
||||
$_.metadata.route_id -eq $rendezvousIntent.route_intent.id -and
|
||||
$_.link_status -eq "reachable"
|
||||
})
|
||||
$replacementRouteHealth = @($rendezvousHealth | Where-Object {
|
||||
$_.metadata.route_path_decision_applied -eq $true -and
|
||||
$_.metadata.route_path_decision_selected_relay_id -eq $nodeSID -and
|
||||
(@($_.metadata.expected_effective_hops) -contains $nodeSID) -and
|
||||
-not (@($_.metadata.expected_effective_hops) -contains $nodeRID) -and
|
||||
(@($_.metadata.observed_ack_path) -contains $nodeSID) -and
|
||||
-not (@($_.metadata.observed_ack_path) -contains $nodeRID) -and
|
||||
$_.metadata.route_path_drift_detected -eq $false
|
||||
})
|
||||
return [pscustomobject]@{
|
||||
route_health_links = $routeHealthLinks
|
||||
direct_health = $directHealth
|
||||
rendezvous_health = $rendezvousHealth
|
||||
replacement_route_health = $replacementRouteHealth
|
||||
}
|
||||
}
|
||||
|
||||
function Get-C17Z18MeshLinkSnapshot {
|
||||
$links = Invoke-Api -Method Get -Path "/clusters/$clusterID/mesh/links?actor_user_id=$actorUserID"
|
||||
$meshLinks = @($links.mesh_links)
|
||||
$routeHealth = Select-C17Z18RouteHealthSnapshot -MeshLinks $meshLinks
|
||||
return [pscustomobject]@{
|
||||
links = $links
|
||||
mesh_links = $meshLinks
|
||||
route_health_links = @($routeHealth.route_health_links)
|
||||
direct_health = @($routeHealth.direct_health)
|
||||
rendezvous_health = @($routeHealth.rendezvous_health)
|
||||
replacement_route_health = @($routeHealth.replacement_route_health)
|
||||
}
|
||||
}
|
||||
|
||||
function Wait-C17Z18ReplacementRouteHealthSnapshot {
|
||||
param([int]$TimeoutSeconds = 40)
|
||||
$deadline = (Get-Date).AddSeconds($TimeoutSeconds)
|
||||
$latest = $null
|
||||
do {
|
||||
$latest = Get-C17Z18MeshLinkSnapshot
|
||||
if (@($latest.replacement_route_health).Count -gt 0) {
|
||||
return $latest
|
||||
}
|
||||
Start-Sleep -Seconds 2
|
||||
} while ((Get-Date) -lt $deadline)
|
||||
return $latest
|
||||
}
|
||||
|
||||
Write-Host "C17Z18 rendezvous relay replacement smoke run: $runId"
|
||||
Write-Host "Using SSH Docker host: $DockerSshAlias"
|
||||
|
||||
Remove-C17Z12Containers
|
||||
Send-RemoteBuildContext
|
||||
|
||||
Write-Host "Building backend image on docker-test..."
|
||||
Invoke-RemoteDocker -Arguments @("build", "-f", "$remoteBuildDir/backend/Dockerfile", "-t", $BackendImageTag, "$remoteBuildDir/backend")
|
||||
|
||||
Write-Host "Building node-agent image on docker-test..."
|
||||
Invoke-RemoteDocker -Arguments @("build", "-f", "$remoteBuildDir/agents/rap-node-agent/Dockerfile", "-t", $NodeAgentImageTag, $remoteBuildDir)
|
||||
|
||||
Write-Host "Starting PostgreSQL and Redis..."
|
||||
Invoke-RemoteDocker -Arguments @(
|
||||
"run", "-d",
|
||||
"--name", $postgresName,
|
||||
"-e", "POSTGRES_DB=remote_access_platform",
|
||||
"-e", "POSTGRES_USER=rap_user",
|
||||
"-e", "POSTGRES_PASSWORD=rap_password",
|
||||
"-p", "$PostgresPort`:5432",
|
||||
"postgres:16"
|
||||
)
|
||||
Invoke-RemoteDocker -Arguments @(
|
||||
"run", "-d",
|
||||
"--name", $redisName,
|
||||
"-p", "$RedisPort`:6379",
|
||||
"redis:7"
|
||||
)
|
||||
|
||||
Invoke-RemoteShell -Command "for i in `$(seq 1 60); do docker exec $postgresName pg_isready -U rap_user -d remote_access_platform >/dev/null 2>&1 && exit 0; sleep 1; done; exit 1"
|
||||
|
||||
Write-Host "Applying migrations..."
|
||||
Invoke-RemoteShell -Command "for f in `$(find '$remoteBuildDir/backend/migrations' -name '*.up.sql' | sort); do docker exec -i $postgresName psql -U rap_user -d remote_access_platform -v ON_ERROR_STOP=1 -f - < `$f; done"
|
||||
|
||||
Write-Host "Seeding platform owner..."
|
||||
$adminUserID = [guid]::NewGuid().ToString()
|
||||
$adminHash = '$2a$10$AqLRexkI1yXbuiMPU6dHM.KVUhF.t..9NolyK4OOodQTyTsHyG.7u'
|
||||
Invoke-RemotePostgresSql -Sql @"
|
||||
INSERT INTO users (id, email, password_hash, mfa_enabled, platform_role)
|
||||
VALUES ('$adminUserID'::uuid, '$AdminEmail', '$adminHash', FALSE, 'platform_admin')
|
||||
ON CONFLICT (email) DO UPDATE SET
|
||||
password_hash = EXCLUDED.password_hash,
|
||||
platform_role = 'platform_admin',
|
||||
updated_at = NOW();
|
||||
"@
|
||||
|
||||
Write-Host "Starting backend..."
|
||||
Invoke-RemoteDocker -Arguments @(
|
||||
"run", "-d",
|
||||
"--name", $backendName,
|
||||
"--network", "host",
|
||||
"-e", "APP_NAME=rap-api",
|
||||
"-e", "APP_ENV=c17z18-smoke",
|
||||
"-e", "HTTP_HOST=0.0.0.0",
|
||||
"-e", "HTTP_PORT=$ApiPort",
|
||||
"-e", "POSTGRES_DSN=postgres://rap_user:rap_password@127.0.0.1:$PostgresPort/remote_access_platform?sslmode=disable",
|
||||
"-e", "REDIS_ADDR=127.0.0.1:$RedisPort",
|
||||
"-e", "AUTH_ACCESS_TOKEN_SECRET=c17z18-access-secret",
|
||||
"-e", "AUTH_REFRESH_HASH_SECRET=c17z18-refresh-secret",
|
||||
$BackendImageTag
|
||||
)
|
||||
Wait-HttpReady -Url "http://192.168.200.61:$ApiPort/readyz"
|
||||
|
||||
Write-Host "Logging in as platform owner..."
|
||||
$login = Invoke-Api -Method Post -Path "/auth/login" -Body @{
|
||||
email = $AdminEmail
|
||||
password = $AdminPassword
|
||||
device_fingerprint = "c17z18-smoke-device"
|
||||
device_label = "C17Z18 rendezvous relay replacement smoke"
|
||||
trust_device = $true
|
||||
}
|
||||
$actorUserID = $login.user.id
|
||||
|
||||
Write-Host "Creating C17Z18 cluster..."
|
||||
$cluster = Invoke-Api -Method Post -Path "/clusters/" -Body @{
|
||||
actor_user_id = $actorUserID
|
||||
slug = "c17z18-$((New-Guid).Guid.Substring(0, 8))"
|
||||
name = "C17Z18 Rendezvous Relay Replacement Smoke"
|
||||
region = "docker-test"
|
||||
metadata = @{
|
||||
stage = "c17z18"
|
||||
run_id = $runId
|
||||
production_forwarding = $false
|
||||
service_workload_traffic = $false
|
||||
created_by = "c17z12-rendezvous-relay-smoke-ssh.ps1:c17z18"
|
||||
}
|
||||
}
|
||||
$clusterID = $cluster.cluster.id
|
||||
|
||||
Write-Host "Enabling test-only Fabric flags..."
|
||||
Invoke-Api -Method Put -Path "/fabric/testing-flags" -Body @{
|
||||
actor_user_id = $actorUserID
|
||||
scope_type = "platform"
|
||||
scope_id = $null
|
||||
cluster_id = $null
|
||||
enabled = $true
|
||||
telemetry_enabled = $true
|
||||
synthetic_links_enabled = $true
|
||||
history_retention_hours = 24
|
||||
metadata = @{
|
||||
stage = "c17z18"
|
||||
run_id = $runId
|
||||
production_forwarding = $false
|
||||
service_workload_traffic = $false
|
||||
}
|
||||
} | Out-Null
|
||||
|
||||
$joinToken = Invoke-Api -Method Post -Path "/clusters/$clusterID/join-tokens" -Body @{
|
||||
actor_user_id = $actorUserID
|
||||
scope = @{ purpose = "c17z18-rendezvous-relay-replacement-smoke"; roles = @("core-mesh", "relay-node") }
|
||||
expires_at = (Get-Date).ToUniversalTime().AddHours(2).ToString("o")
|
||||
max_uses = 5
|
||||
}
|
||||
|
||||
$nodeSpecs = @(
|
||||
@{ key = "a"; name = "c17z18-node-a-entry"; roles = @("core-mesh"); port = $MeshBasePort; transport = "direct_tcp_tls"; connectivity = "direct"; nat = "none" },
|
||||
@{ key = "r"; name = "c17z18-node-r-stale-relay"; roles = @("core-mesh", "relay-node"); port = ($MeshBasePort + 1); transport = "direct_tcp_tls"; connectivity = "direct"; nat = "none" },
|
||||
@{ key = "b"; name = "c17z18-node-b-direct"; roles = @("core-mesh"); port = ($MeshBasePort + 2); transport = "direct_tcp_tls"; connectivity = "direct"; nat = "none" },
|
||||
@{ key = "c"; name = "c17z18-node-c-outbound"; roles = @("core-mesh"); port = ($MeshBasePort + 3); transport = "outbound_reverse"; connectivity = "outbound_only"; nat = "symmetric" },
|
||||
@{ key = "idle"; name = "c17z18-node-s-alt-relay"; roles = @("core-mesh", "relay-node"); port = ($MeshBasePort + 4); transport = "direct_tcp_tls"; connectivity = "direct"; nat = "none" }
|
||||
)
|
||||
|
||||
$nodes = @{}
|
||||
foreach ($spec in $nodeSpecs) {
|
||||
$fingerprint = "c17z18-fp-$($spec.key)-$([guid]::NewGuid().ToString('N'))"
|
||||
$publicKey = "c17z18-pub-$($spec.key)-$([guid]::NewGuid().ToString('N'))"
|
||||
$joinRequest = Invoke-Api -Method Post -Path "/clusters/$clusterID/join-requests" -Body @{
|
||||
join_token = $joinToken.join_token.token
|
||||
node_name = $spec.name
|
||||
node_fingerprint = $fingerprint
|
||||
public_key = $publicKey
|
||||
reported_capabilities = @{
|
||||
can_accept_node_ingress = $true
|
||||
can_route_mesh = $true
|
||||
testing_node = $true
|
||||
mesh_rendezvous_relay_control_contract = $true
|
||||
mesh_rendezvous_lease_telemetry = $true
|
||||
mesh_rendezvous_lease_refresh_contract = $true
|
||||
mesh_rendezvous_relay_replacement_contract = $true
|
||||
mesh_route_path_decision_contract = $true
|
||||
mesh_route_generation_tracker = $true
|
||||
}
|
||||
reported_facts = @{
|
||||
os = "linux"
|
||||
runtime = "docker-test"
|
||||
stage = "c17z18"
|
||||
run_id = $runId
|
||||
connectivity_mode = $spec.connectivity
|
||||
nat_type = $spec.nat
|
||||
}
|
||||
requested_roles = $spec.roles
|
||||
}
|
||||
$approved = Invoke-Api -Method Post -Path "/clusters/$clusterID/join-requests/$($joinRequest.join_request.id)/approve" -Body @{
|
||||
actor_user_id = $actorUserID
|
||||
node_key = $fingerprint
|
||||
ownership_type = "platform_managed"
|
||||
owner_organization_id = $null
|
||||
}
|
||||
$nodeID = $approved.node_bootstrap.node_id
|
||||
foreach ($role in $spec.roles) {
|
||||
Invoke-Api -Method Post -Path "/clusters/$clusterID/nodes/$nodeID/roles" -Body @{
|
||||
actor_user_id = $actorUserID
|
||||
role = $role
|
||||
status = "active"
|
||||
policy = @{
|
||||
stage = "c17z18"
|
||||
run_id = $runId
|
||||
synthetic_only = $true
|
||||
production_payload_forwarding = $false
|
||||
}
|
||||
} | Out-Null
|
||||
}
|
||||
$nodes[$spec.key] = [pscustomobject]@{
|
||||
id = $nodeID
|
||||
name = $spec.name
|
||||
fingerprint = $fingerprint
|
||||
public_key = $publicKey
|
||||
port = $spec.port
|
||||
roles = $spec.roles
|
||||
transport = $spec.transport
|
||||
connectivity = $spec.connectivity
|
||||
nat = $spec.nat
|
||||
}
|
||||
}
|
||||
|
||||
$nodeAID = $nodes["a"].id
|
||||
$nodeRID = $nodes["r"].id
|
||||
$nodeBID = $nodes["b"].id
|
||||
$nodeCID = $nodes["c"].id
|
||||
$nodeSID = $nodes["idle"].id
|
||||
$routeExpiresAt = (Get-Date).ToUniversalTime().AddHours(2).ToString("o")
|
||||
$staleRelayEndpoint = "http://127.0.0.1:$($MeshBasePort + 90)"
|
||||
|
||||
$peerEndpointsDirect = @{}
|
||||
$peerEndpointsDirect[$nodeAID] = "http://127.0.0.1:$($nodes["a"].port)"
|
||||
$peerEndpointsDirect[$nodeBID] = "http://127.0.0.1:$($nodes["b"].port)"
|
||||
|
||||
$peerEndpointsRelayControl = @{}
|
||||
$peerEndpointsRelayControl[$nodeAID] = "http://127.0.0.1:$($nodes["a"].port)"
|
||||
$peerEndpointsRelayControl[$nodeRID] = $staleRelayEndpoint
|
||||
$peerEndpointsRelayControl[$nodeSID] = "http://127.0.0.1:$($nodes["idle"].port)"
|
||||
|
||||
$peerEndpointCandidatesRelay = @{}
|
||||
$peerEndpointCandidatesRelay[$nodeRID] = @(
|
||||
New-EndpointCandidate `
|
||||
-EndpointID "relay-r-public" `
|
||||
-NodeID $nodeRID `
|
||||
-Address $staleRelayEndpoint `
|
||||
-Transport "direct_tcp_tls" `
|
||||
-Reachability "public" `
|
||||
-ConnectivityMode "direct" `
|
||||
-NATType "none" `
|
||||
-Priority 10 `
|
||||
-PolicyTags @("relay-control", "same-site")
|
||||
)
|
||||
$peerEndpointCandidatesRelay[$nodeSID] = @(
|
||||
New-EndpointCandidate `
|
||||
-EndpointID "relay-s-alt-fast" `
|
||||
-NodeID $nodeSID `
|
||||
-Address "http://127.0.0.1:$($nodes["idle"].port)" `
|
||||
-Transport "direct_tcp_tls" `
|
||||
-Reachability "public" `
|
||||
-ConnectivityMode "direct" `
|
||||
-NATType "none" `
|
||||
-Priority 1 `
|
||||
-PolicyTags @("relay-control", "same-site", "fast-path")
|
||||
)
|
||||
$peerEndpointCandidatesRelay[$nodeCID] = @(
|
||||
New-EndpointCandidate `
|
||||
-EndpointID "node-c-outbound-only" `
|
||||
-NodeID $nodeCID `
|
||||
-Address "http://127.0.0.1:$($nodes["c"].port)" `
|
||||
-Transport "outbound_reverse" `
|
||||
-Reachability "outbound_only" `
|
||||
-ConnectivityMode "outbound_only" `
|
||||
-NATType "symmetric" `
|
||||
-Priority 5 `
|
||||
-PolicyTags @("nat", "outbound-only")
|
||||
)
|
||||
|
||||
Write-Host "Creating direct baseline and outbound-only relay-control route intents..."
|
||||
$directIntent = Invoke-Api -Method Post -Path "/clusters/$clusterID/mesh/route-intents" -Body @{
|
||||
actor_user_id = $actorUserID
|
||||
source_selector = @{ node_id = $nodeAID }
|
||||
destination_selector = @{ node_id = $nodeBID }
|
||||
service_class = "control"
|
||||
priority = 10
|
||||
policy = @{
|
||||
synthetic_enabled = $true
|
||||
peer_endpoints = $peerEndpointsDirect
|
||||
hops = @($nodeAID, $nodeBID)
|
||||
allowed_channels = @("fabric_control", "route_control")
|
||||
max_ttl = 4
|
||||
max_hops = 4
|
||||
expires_at = $routeExpiresAt
|
||||
route_version = "$runId-direct"
|
||||
policy_version = "$runId-policy"
|
||||
peer_directory_version = "$runId-peers"
|
||||
production_forwarding = $false
|
||||
}
|
||||
}
|
||||
|
||||
$rendezvousIntent = Invoke-Api -Method Post -Path "/clusters/$clusterID/mesh/route-intents" -Body @{
|
||||
actor_user_id = $actorUserID
|
||||
source_selector = @{ node_id = $nodeAID }
|
||||
destination_selector = @{ node_id = $nodeCID }
|
||||
service_class = "control"
|
||||
priority = 20
|
||||
policy = @{
|
||||
synthetic_enabled = $true
|
||||
peer_endpoints = $peerEndpointsRelayControl
|
||||
peer_endpoint_candidates = $peerEndpointCandidatesRelay
|
||||
hops = @($nodeAID, $nodeRID, $nodeSID, $nodeCID)
|
||||
allowed_channels = @("fabric_control", "route_control")
|
||||
max_ttl = 6
|
||||
max_hops = 6
|
||||
rendezvous_leases = @(
|
||||
@{
|
||||
lease_id = "$runId-explicit-stale-relay-lease"
|
||||
peer_node_id = $nodeCID
|
||||
relay_node_id = $nodeRID
|
||||
relay_endpoint = $staleRelayEndpoint
|
||||
transport = "relay_control"
|
||||
connectivity_mode = "relay_required"
|
||||
route_ids = @()
|
||||
allowed_channels = @("fabric_control", "route_control")
|
||||
priority = 4
|
||||
control_plane_only = $true
|
||||
issued_at = (Get-Date).ToUniversalTime().ToString("o")
|
||||
expires_at = $routeExpiresAt
|
||||
reason = "smoke_stale_relay_replacement"
|
||||
metadata = @{
|
||||
stage = "c17z18"
|
||||
run_id = $runId
|
||||
lease_refresh_contract = "node_scoped_synthetic_config_get"
|
||||
relay_replacement_contract = "stale_relay_feedback_policy"
|
||||
production_payload_forwarding = $false
|
||||
}
|
||||
}
|
||||
)
|
||||
expires_at = $routeExpiresAt
|
||||
route_version = "$runId-rendezvous"
|
||||
policy_version = "$runId-policy"
|
||||
peer_directory_version = "$runId-peers"
|
||||
production_forwarding = $false
|
||||
}
|
||||
}
|
||||
|
||||
$configs = @{}
|
||||
foreach ($key in @("a", "r", "b", "c", "idle")) {
|
||||
$configs[$key] = Invoke-Api -Method Get -Path "/clusters/$clusterID/nodes/$($nodes[$key].id)/mesh/synthetic-config"
|
||||
}
|
||||
|
||||
$nodeAPeerCandidates = Get-OptionalProperty -Object (Get-OptionalProperty -Object $configs["a"].synthetic_mesh_config -PropertyName "peer_endpoint_candidates") -PropertyName $nodeCID
|
||||
$nodeAInitialStaleLeases = @(Get-OptionalProperty -Object $configs["a"].synthetic_mesh_config -PropertyName "rendezvous_leases" | Where-Object {
|
||||
$_.peer_node_id -eq $nodeCID -and $_.relay_node_id -eq $nodeRID
|
||||
})
|
||||
$nodeAInitialAltLeases = @(Get-OptionalProperty -Object $configs["a"].synthetic_mesh_config -PropertyName "rendezvous_leases" | Where-Object {
|
||||
$_.peer_node_id -eq $nodeCID -and $_.relay_node_id -eq $nodeSID
|
||||
})
|
||||
$nodeCLeases = @(Get-OptionalProperty -Object $configs["c"].synthetic_mesh_config -PropertyName "rendezvous_leases" | Where-Object {
|
||||
$_.peer_node_id -eq $nodeCID -and ($_.relay_node_id -eq $nodeRID -or $_.relay_node_id -eq $nodeSID)
|
||||
})
|
||||
|
||||
Write-Host "Starting node-agent containers..."
|
||||
foreach ($key in @("r", "idle", "b", "c", "a")) {
|
||||
$node = $nodes[$key]
|
||||
$containerName = "$nodePrefix$key"
|
||||
$remoteStateDir = "/tmp/$runId-$key"
|
||||
Invoke-RemoteShell -Command "rm -rf '$remoteStateDir' && mkdir -p '$remoteStateDir'"
|
||||
$identity = @{
|
||||
node_id = $node.id
|
||||
cluster_id = $clusterID
|
||||
node_name = $node.name
|
||||
node_fingerprint = $node.fingerprint
|
||||
public_key = $node.public_key
|
||||
identity_status = "active"
|
||||
created_at = (Get-Date).ToUniversalTime().ToString("o")
|
||||
updated_at = (Get-Date).ToUniversalTime().ToString("o")
|
||||
} | ConvertTo-Json -Depth 10
|
||||
Send-RemoteFile -RemotePath "$remoteStateDir/identity.json" -Content $identity
|
||||
Invoke-RemoteDocker -Arguments @(
|
||||
"create",
|
||||
"--name", $containerName,
|
||||
"--network", "host",
|
||||
"-e", "RAP_BACKEND_URL=$backendContainerBaseUrl",
|
||||
"-e", "RAP_NODE_STATE_DIR=/tmp/state",
|
||||
"-e", "RAP_HEARTBEAT_INTERVAL_SECONDS=5",
|
||||
"-e", "RAP_MESH_SYNTHETIC_RUNTIME_ENABLED=true",
|
||||
"-e", "RAP_MESH_LISTEN_ADDR=0.0.0.0:$($node.port)",
|
||||
"-e", "RAP_MESH_ADVERTISE_ENDPOINT=http://127.0.0.1:$($node.port)",
|
||||
"-e", "RAP_MESH_ADVERTISE_TRANSPORT=$($node.transport)",
|
||||
"-e", "RAP_MESH_CONNECTIVITY_MODE=$($node.connectivity)",
|
||||
"-e", "RAP_MESH_NAT_TYPE=$($node.nat)",
|
||||
"-e", "RAP_MESH_REGION=docker-test",
|
||||
$NodeAgentImageTag,
|
||||
"-backend-url", $backendContainerBaseUrl,
|
||||
"-state-dir", "/tmp/state",
|
||||
"-heartbeat-interval", "5s",
|
||||
"-mesh-synthetic-runtime-enabled",
|
||||
"-mesh-listen-addr", "0.0.0.0:$($node.port)",
|
||||
"-mesh-advertise-endpoint", "http://127.0.0.1:$($node.port)",
|
||||
"-mesh-advertise-transport", $node.transport,
|
||||
"-mesh-connectivity-mode", $node.connectivity,
|
||||
"-mesh-nat-type", $node.nat,
|
||||
"-mesh-region", "docker-test"
|
||||
) | Out-Null
|
||||
Invoke-RemoteDocker -Arguments @("cp", "$remoteStateDir/.", "$containerName`:/tmp/state")
|
||||
Invoke-RemoteDocker -Arguments @("start", $containerName) | Out-Null
|
||||
}
|
||||
|
||||
Write-Host "Waiting for rendezvous relay-control observations..."
|
||||
Start-Sleep -Seconds 40
|
||||
|
||||
Write-Host "Waiting for replacement route-health effective path..."
|
||||
$meshSnapshot = Wait-C17Z18ReplacementRouteHealthSnapshot -TimeoutSeconds 40
|
||||
$links = $meshSnapshot.links
|
||||
$summary = Invoke-Api -Method Get -Path "/cluster-admin-summaries?actor_user_id=$actorUserID"
|
||||
$nodeALeaseReport = Get-LatestRendezvousLeaseReport -NodeID $nodeAID
|
||||
$nodeRLeaseReport = Get-LatestRendezvousLeaseReport -NodeID $nodeRID
|
||||
$nodeSLeaseReport = Get-LatestRendezvousLeaseReport -NodeID $nodeSID
|
||||
$nodeCLeaseReport = Get-LatestRendezvousLeaseReport -NodeID $nodeCID
|
||||
$nodeAPathDecisionReport = Get-LatestRoutePathDecisionReport -NodeID $nodeAID
|
||||
$nodeARouteGenerationReport = Get-LatestRouteGenerationReport -NodeID $nodeAID
|
||||
$nodeARouteHealthConfigReport = Get-LatestRouteHealthConfigReport -NodeID $nodeAID
|
||||
$nodeARouteHealthFeedbackRefreshReport = Get-LatestRouteHealthFeedbackRefreshReport -NodeID $nodeAID
|
||||
$refreshedNodeAConfig = Invoke-Api -Method Get -Path "/clusters/$clusterID/nodes/$nodeAID/mesh/synthetic-config"
|
||||
$nodeAReportedLeases = @(Get-OptionalProperty -Object $nodeALeaseReport -PropertyName "leases")
|
||||
$nodeAReportedReplacementLeases = @($nodeAReportedLeases | Where-Object {
|
||||
$_.peer_node_id -eq $nodeCID -and $_.relay_node_id -eq $nodeSID -and $_.reason -eq "stale_relay_replacement"
|
||||
})
|
||||
$nodeAReportedStaleRelayLeases = @($nodeAReportedLeases | Where-Object {
|
||||
$_.peer_node_id -eq $nodeCID -and $_.relay_node_id -eq $nodeRID
|
||||
})
|
||||
$nodeAReplacementLeases = @(Get-OptionalProperty -Object $refreshedNodeAConfig.synthetic_mesh_config -PropertyName "rendezvous_leases" | Where-Object {
|
||||
$_.peer_node_id -eq $nodeCID -and $_.relay_node_id -eq $nodeSID -and $_.reason -eq "stale_relay_replacement"
|
||||
})
|
||||
$nodeAWithdrawnStaleLeases = @(Get-OptionalProperty -Object $refreshedNodeAConfig.synthetic_mesh_config -PropertyName "rendezvous_leases" | Where-Object {
|
||||
$_.peer_node_id -eq $nodeCID -and $_.relay_node_id -eq $nodeRID
|
||||
})
|
||||
$nodeARelayPolicy = Get-OptionalProperty -Object $refreshedNodeAConfig.synthetic_mesh_config -PropertyName "rendezvous_relay_policy"
|
||||
$nodeAInitialPathDecisionReport = Get-OptionalProperty -Object $configs["a"].synthetic_mesh_config -PropertyName "route_path_decisions"
|
||||
$nodeAConfigPathDecisionReport = Get-OptionalProperty -Object $refreshedNodeAConfig.synthetic_mesh_config -PropertyName "route_path_decisions"
|
||||
$nodeAReportedPathDecisions = @(Get-OptionalProperty -Object $nodeAPathDecisionReport -PropertyName "decisions")
|
||||
$nodeAReportedReplacementPathDecisions = @($nodeAReportedPathDecisions | Where-Object {
|
||||
$_.route_id -eq $rendezvousIntent.route_intent.id -and
|
||||
$_.selected_relay_id -eq $nodeSID -and
|
||||
$_.decision_source -eq "stale_relay_replacement" -and
|
||||
(@($_.effective_hops) -contains $nodeSID) -and
|
||||
-not (@($_.effective_hops) -contains $nodeRID)
|
||||
})
|
||||
|
||||
$nodeLogs = @{}
|
||||
foreach ($key in @("a", "r", "b", "c", "idle")) {
|
||||
$nodeLogs[$key] = Invoke-RemoteDockerText -Arguments @("logs", "--tail", "120", "$nodePrefix$key")
|
||||
}
|
||||
$backendLogs = Invoke-RemoteDockerText -Arguments @("logs", "--tail", "80", $backendName)
|
||||
|
||||
$meshLinks = @($meshSnapshot.mesh_links)
|
||||
$routeHealthLinks = @($meshSnapshot.route_health_links)
|
||||
$directHealth = @($meshSnapshot.direct_health)
|
||||
$rendezvousHealth = @($meshSnapshot.rendezvous_health)
|
||||
$replacementRouteHealth = @($meshSnapshot.replacement_route_health)
|
||||
$managerLinks = @($meshLinks | Where-Object { $_.metadata.observation_type -eq "peer_connection_manager" })
|
||||
$relayControlLinks = @($managerLinks | Where-Object {
|
||||
$_.source_node_id -eq $nodeAID -and
|
||||
$_.target_node_id -eq $nodeCID -and
|
||||
$_.link_status -eq "reachable" -and
|
||||
$_.metadata.transport_mode -eq "relay_control" -and
|
||||
$_.metadata.rendezvous_resolved -eq $true -and
|
||||
$_.metadata.relay_candidate -eq $true -and
|
||||
$_.metadata.connection_state -eq "relay_ready"
|
||||
})
|
||||
$replacementRelayControlLinks = @($relayControlLinks | Where-Object {
|
||||
$_.metadata.relay_node_id -eq $nodeSID
|
||||
})
|
||||
$replacementRelayReadyFromLeaseReport = (
|
||||
$nodeAReportedReplacementLeases.Count -gt 0 -and
|
||||
(Get-OptionalProperty -Object $nodeAReportedReplacementLeases[0] -PropertyName "relay_ready") -eq $true -and
|
||||
(Get-OptionalProperty -Object $nodeAReportedReplacementLeases[0] -PropertyName "connection_state") -eq "relay_ready"
|
||||
)
|
||||
|
||||
$nodeALog = $nodeLogs["a"] -join "`n"
|
||||
$directRouteDelivered = $nodeALog -match ('"event":"fabric_route_delivery_succeeded","route_id":"' + [regex]::Escape($directIntent.route_intent.id) + '"')
|
||||
$leaseReportBoundaryFlagsDisabled = $true
|
||||
foreach ($report in @($nodeALeaseReport, $nodeRLeaseReport, $nodeSLeaseReport, $nodeCLeaseReport)) {
|
||||
if ($null -eq $report -or
|
||||
(Get-OptionalProperty -Object $report -PropertyName "control_plane_only") -ne $true -or
|
||||
(Get-OptionalProperty -Object $report -PropertyName "relay_payload_forwarding") -ne $false -or
|
||||
(Get-OptionalProperty -Object $report -PropertyName "production_payload_forwarding") -ne $false -or
|
||||
(Get-OptionalProperty -Object $report -PropertyName "service_workload_traffic") -ne $false) {
|
||||
$leaseReportBoundaryFlagsDisabled = $false
|
||||
}
|
||||
}
|
||||
$pathDecisionBoundaryFlagsDisabled = (
|
||||
$null -ne $nodeAPathDecisionReport -and
|
||||
(Get-OptionalProperty -Object $nodeAPathDecisionReport -PropertyName "control_plane_only") -eq $true -and
|
||||
(Get-OptionalProperty -Object $nodeAPathDecisionReport -PropertyName "production_payload_forwarding") -eq $false -and
|
||||
(Get-OptionalProperty -Object $nodeAPathDecisionReport -PropertyName "service_workload_traffic") -eq $false -and
|
||||
(Get-OptionalProperty -Object $nodeAPathDecisionReport -PropertyName "route_path_forwarding_runtime") -eq $false
|
||||
)
|
||||
$routeGenerationBoundaryFlagsDisabled = (
|
||||
$null -ne $nodeARouteGenerationReport -and
|
||||
(Get-OptionalProperty -Object $nodeARouteGenerationReport -PropertyName "control_plane_only") -eq $true -and
|
||||
(Get-OptionalProperty -Object $nodeARouteGenerationReport -PropertyName "production_payload_forwarding") -eq $false -and
|
||||
(Get-OptionalProperty -Object $nodeARouteGenerationReport -PropertyName "service_workload_traffic") -eq $false -and
|
||||
(Get-OptionalProperty -Object $nodeARouteGenerationReport -PropertyName "route_path_forwarding_runtime") -eq $false
|
||||
)
|
||||
$nodeAWithdrawnDecisionCount = Get-OptionalProperty -Object $nodeARouteGenerationReport -PropertyName "withdrawn_decision_count"
|
||||
$nodeATotalWithdrawnDecisionCount = Get-OptionalProperty -Object $nodeARouteGenerationReport -PropertyName "total_withdrawn_decision_count"
|
||||
$nodeAWithdrawnDecisions = @(Get-OptionalProperty -Object $nodeARouteGenerationReport -PropertyName "withdrawn_decisions")
|
||||
$routeHealthConfigBoundaryFlagsDisabled = (
|
||||
$null -ne $nodeARouteHealthConfigReport -and
|
||||
(Get-OptionalProperty -Object $nodeARouteHealthConfigReport -PropertyName "control_plane_only") -eq $true -and
|
||||
(Get-OptionalProperty -Object $nodeARouteHealthConfigReport -PropertyName "route_health_only") -eq $true -and
|
||||
(Get-OptionalProperty -Object $nodeARouteHealthConfigReport -PropertyName "synthetic_route_health_route_path_runtime") -eq $true -and
|
||||
(Get-OptionalProperty -Object $nodeARouteHealthConfigReport -PropertyName "production_route_path_forwarding_runtime") -eq $false -and
|
||||
(Get-OptionalProperty -Object $nodeARouteHealthConfigReport -PropertyName "production_payload_forwarding") -eq $false -and
|
||||
(Get-OptionalProperty -Object $nodeARouteHealthConfigReport -PropertyName "service_workload_traffic") -eq $false -and
|
||||
(Get-OptionalProperty -Object $nodeARouteHealthConfigReport -PropertyName "test_service_route_config_changed") -eq $false
|
||||
)
|
||||
$routeHealthFeedbackRefreshBoundaryFlagsDisabled = (
|
||||
$null -ne $nodeARouteHealthFeedbackRefreshReport -and
|
||||
(Get-OptionalProperty -Object $nodeARouteHealthFeedbackRefreshReport -PropertyName "control_plane_only") -eq $true -and
|
||||
(Get-OptionalProperty -Object $nodeARouteHealthFeedbackRefreshReport -PropertyName "route_health_only") -eq $true -and
|
||||
(Get-OptionalProperty -Object $nodeARouteHealthFeedbackRefreshReport -PropertyName "production_payload_forwarding") -eq $false -and
|
||||
(Get-OptionalProperty -Object $nodeARouteHealthFeedbackRefreshReport -PropertyName "service_workload_traffic") -eq $false
|
||||
)
|
||||
$passMatrix = [ordered]@{
|
||||
backend_ready = $true
|
||||
platform_owner_login = [bool]$actorUserID
|
||||
cluster_created = [bool]$clusterID
|
||||
fabric_testing_flags_enabled = $true
|
||||
node_a_scoped_config_enabled = $configs["a"].synthetic_mesh_config.enabled -eq $true
|
||||
node_a_has_direct_and_rendezvous_routes = @($configs["a"].synthetic_mesh_config.routes).Count -eq 2
|
||||
node_a_has_outbound_peer_candidate = @($nodeAPeerCandidates).Count -gt 0
|
||||
node_a_has_initial_stale_rendezvous_lease = $nodeAInitialStaleLeases.Count -gt 0
|
||||
node_a_initial_lease_is_control_plane_only = ($nodeAInitialStaleLeases.Count -gt 0 -and $nodeAInitialStaleLeases[0].control_plane_only -eq $true)
|
||||
node_a_initial_lease_uses_relay_control = ($nodeAInitialStaleLeases.Count -gt 0 -and $nodeAInitialStaleLeases[0].transport -eq "relay_control")
|
||||
node_a_initial_auto_alt_relay_candidate = $nodeAInitialAltLeases.Count -gt 0
|
||||
node_a_initial_path_decision_report = (Get-OptionalProperty -Object $nodeAInitialPathDecisionReport -PropertyName "schema_version") -eq "c17z18.route_path_decisions.v1"
|
||||
node_a_report_replacement_lease_uses_alt_relay = ($nodeAReportedReplacementLeases.Count -gt 0 -and $nodeAReportedReplacementLeases[0].relay_node_id -eq $nodeSID)
|
||||
node_a_report_stale_relay_lease_withdrawn = $nodeAReportedStaleRelayLeases.Count -eq 0
|
||||
node_a_report_replacement_reason = ($nodeAReportedReplacementLeases.Count -gt 0 -and $nodeAReportedReplacementLeases[0].reason -eq "stale_relay_replacement")
|
||||
node_a_reports_c17z18_path_decisions = (Get-OptionalProperty -Object $nodeAPathDecisionReport -PropertyName "schema_version") -eq "c17z18.mesh_route_path_decision_report.v1"
|
||||
node_a_path_decision_replacement_count = (Get-OptionalProperty -Object $nodeAPathDecisionReport -PropertyName "replacement_decision_count") -gt 0
|
||||
node_a_path_decision_uses_alt_relay = $nodeAReportedReplacementPathDecisions.Count -gt 0
|
||||
node_a_path_decision_boundary_flags_disabled = $pathDecisionBoundaryFlagsDisabled
|
||||
node_a_reports_c17z18_route_generation = (Get-OptionalProperty -Object $nodeARouteGenerationReport -PropertyName "schema_version") -eq "c17z18.mesh_route_generation_report.v1"
|
||||
node_a_route_generation_active = (Get-OptionalProperty -Object $nodeARouteGenerationReport -PropertyName "active_decision_count") -gt 0
|
||||
node_a_route_generation_applied = (Get-OptionalProperty -Object $nodeARouteGenerationReport -PropertyName "applied_decision_count") -gt 0
|
||||
node_a_route_generation_withdrawn = ($nodeAWithdrawnDecisionCount -gt 0 -or $nodeATotalWithdrawnDecisionCount -gt 0 -or $nodeAWithdrawnDecisions.Count -gt 0)
|
||||
node_a_route_generation_changed = (Get-OptionalProperty -Object $nodeARouteGenerationReport -PropertyName "generation_changed") -eq $true
|
||||
node_a_route_generation_boundary_flags_disabled = $routeGenerationBoundaryFlagsDisabled
|
||||
node_a_reports_c17z20_route_health_config = (Get-OptionalProperty -Object $nodeARouteHealthConfigReport -PropertyName "schema_version") -eq "c17z20.mesh_route_health_config_report.v1"
|
||||
node_a_reports_c17z20_route_health_feedback_refresh = (Get-OptionalProperty -Object $nodeARouteHealthFeedbackRefreshReport -PropertyName "schema_version") -eq "c17z20.mesh_route_health_feedback_refresh_report.v1"
|
||||
node_a_route_health_feedback_refresh_supported = (Get-OptionalProperty -Object $nodeARouteHealthFeedbackRefreshReport -PropertyName "feedback_refresh_supported") -eq $true
|
||||
node_a_route_health_feedback_refresh_boundary_flags_disabled = $routeHealthFeedbackRefreshBoundaryFlagsDisabled
|
||||
node_a_route_health_config_applied = (Get-OptionalProperty -Object $nodeARouteHealthConfigReport -PropertyName "route_path_decision_applied_count") -gt 0
|
||||
node_a_route_health_config_replacement = (Get-OptionalProperty -Object $nodeARouteHealthConfigReport -PropertyName "replacement_route_health_route_count") -gt 0
|
||||
node_a_route_health_config_boundary_flags_disabled = $routeHealthConfigBoundaryFlagsDisabled
|
||||
node_a_route_health_uses_effective_alt_relay = $replacementRouteHealth.Count -gt 0
|
||||
node_a_route_health_has_no_effective_path_drift = ($replacementRouteHealth.Count -gt 0 -and (Get-OptionalProperty -Object $replacementRouteHealth[0].metadata -PropertyName "route_path_drift_detected") -eq $false)
|
||||
node_a_route_health_selected_alt_next_hop = $nodeALog -match ('"event":"fabric_route_selected","route_id":"' + [regex]::Escape($rendezvousIntent.route_intent.id) + '".*"next_node_id":"' + [regex]::Escape($nodeSID) + '"')
|
||||
outbound_node_has_relay_lease = $nodeCLeases.Count -gt 0
|
||||
direct_baseline_health_reported = $directRouteDelivered
|
||||
node_a_loaded_c17z20_control_plane_config = ($nodeAReplacementLeases.Count -gt 0 -and (Get-OptionalProperty -Object $nodeARouteHealthConfigReport -PropertyName "schema_version") -eq "c17z20.mesh_route_health_config_report.v1")
|
||||
node_a_resolved_waiting_rendezvous = ($replacementRelayControlLinks.Count -gt 0 -or $replacementRelayReadyFromLeaseReport)
|
||||
relay_control_manager_link_reachable = ($replacementRelayControlLinks.Count -gt 0 -or $replacementRelayReadyFromLeaseReport)
|
||||
relay_ready_recorded = ($replacementRelayControlLinks.Count -gt 0 -or $replacementRelayReadyFromLeaseReport)
|
||||
node_a_reports_c17z18_lease_telemetry = (Get-OptionalProperty -Object $nodeALeaseReport -PropertyName "schema_version") -eq "c17z18.mesh_rendezvous_lease_report.v1"
|
||||
node_a_lease_report_entry_observer = (Get-OptionalProperty -Object $nodeALeaseReport -PropertyName "entry_observer_count") -gt 0
|
||||
node_a_lease_report_relay_ready = (Get-OptionalProperty -Object $nodeALeaseReport -PropertyName "relay_control_ready_count") -gt 0
|
||||
node_a_lease_report_refresh_contract = (Get-OptionalProperty -Object $nodeALeaseReport -PropertyName "refresh_contract") -eq "node_scoped_synthetic_config_get"
|
||||
stale_relay_refresh_succeeded_on_cluster = (
|
||||
((Get-OptionalProperty -Object $nodeRLeaseReport -PropertyName "last_refresh_reason") -eq "stale_relay" -and (Get-OptionalProperty -Object $nodeRLeaseReport -PropertyName "refresh_success_count") -gt 0) -or
|
||||
((Get-OptionalProperty -Object $nodeSLeaseReport -PropertyName "last_refresh_reason") -eq "stale_relay" -and (Get-OptionalProperty -Object $nodeSLeaseReport -PropertyName "refresh_success_count") -gt 0)
|
||||
)
|
||||
alt_relay_node_reports_admitted_relay_lease = (Get-OptionalProperty -Object $nodeSLeaseReport -PropertyName "admitted_as_relay_count") -gt 0
|
||||
outbound_node_reports_admitted_peer_lease = (Get-OptionalProperty -Object $nodeCLeaseReport -PropertyName "admitted_as_peer_count") -gt 0
|
||||
lease_telemetry_boundary_flags_disabled = $leaseReportBoundaryFlagsDisabled
|
||||
production_forwarding_disabled = (
|
||||
$configs["a"].synthetic_mesh_config.production_forwarding -eq $false -and
|
||||
$configs["r"].synthetic_mesh_config.production_forwarding -eq $false -and
|
||||
$configs["b"].synthetic_mesh_config.production_forwarding -eq $false -and
|
||||
$configs["c"].synthetic_mesh_config.production_forwarding -eq $false -and
|
||||
$configs["idle"].synthetic_mesh_config.production_forwarding -eq $false
|
||||
)
|
||||
}
|
||||
|
||||
$result = [pscustomobject]@{
|
||||
stage = "C17Z18 rendezvous relay replacement docker-test smoke"
|
||||
run_id = $runId
|
||||
backend_base_url = $backendPublicBaseUrl
|
||||
cluster_id = $clusterID
|
||||
node_ids = @{
|
||||
a = $nodes["a"].id
|
||||
r = $nodes["r"].id
|
||||
s = $nodes["idle"].id
|
||||
b = $nodes["b"].id
|
||||
c = $nodes["c"].id
|
||||
idle = $nodes["idle"].id
|
||||
}
|
||||
route_intents = @{
|
||||
direct = $directIntent.route_intent.id
|
||||
rendezvous = $rendezvousIntent.route_intent.id
|
||||
}
|
||||
scoped_config_route_counts = @{
|
||||
a = @($configs["a"].synthetic_mesh_config.routes).Count
|
||||
r = @($configs["r"].synthetic_mesh_config.routes).Count
|
||||
b = @($configs["b"].synthetic_mesh_config.routes).Count
|
||||
c = @($configs["c"].synthetic_mesh_config.routes).Count
|
||||
idle = @($configs["idle"].synthetic_mesh_config.routes).Count
|
||||
}
|
||||
rendezvous_lease_counts = @{
|
||||
a = Get-OptionalArrayCount -Object $configs["a"].synthetic_mesh_config -PropertyName "rendezvous_leases"
|
||||
r = Get-OptionalArrayCount -Object $configs["r"].synthetic_mesh_config -PropertyName "rendezvous_leases"
|
||||
b = Get-OptionalArrayCount -Object $configs["b"].synthetic_mesh_config -PropertyName "rendezvous_leases"
|
||||
c = Get-OptionalArrayCount -Object $configs["c"].synthetic_mesh_config -PropertyName "rendezvous_leases"
|
||||
idle = Get-OptionalArrayCount -Object $configs["idle"].synthetic_mesh_config -PropertyName "rendezvous_leases"
|
||||
}
|
||||
node_a_initial_stale_rendezvous_lease = $nodeAInitialStaleLeases | Select-Object -First 1
|
||||
node_a_reported_replacement_rendezvous_lease = $nodeAReportedReplacementLeases | Select-Object -First 1
|
||||
node_a_current_replacement_rendezvous_lease = $nodeAReplacementLeases | Select-Object -First 1
|
||||
node_a_current_rendezvous_relay_policy = $nodeARelayPolicy
|
||||
node_a_initial_route_path_decisions = $nodeAInitialPathDecisionReport
|
||||
node_a_current_route_path_decisions = $nodeAConfigPathDecisionReport
|
||||
node_a_reported_route_path_decision = $nodeAReportedReplacementPathDecisions | Select-Object -First 1
|
||||
route_path_decision_reports = @{
|
||||
a = $nodeAPathDecisionReport
|
||||
}
|
||||
route_generation_reports = @{
|
||||
a = $nodeARouteGenerationReport
|
||||
}
|
||||
route_health_config_reports = @{
|
||||
a = $nodeARouteHealthConfigReport
|
||||
}
|
||||
route_health_feedback_refresh_reports = @{
|
||||
a = $nodeARouteHealthFeedbackRefreshReport
|
||||
}
|
||||
rendezvous_lease_reports = @{
|
||||
a = $nodeALeaseReport
|
||||
r = $nodeRLeaseReport
|
||||
s = $nodeSLeaseReport
|
||||
c = $nodeCLeaseReport
|
||||
}
|
||||
mesh_link_count = $meshLinks.Count
|
||||
route_health_count = $routeHealthLinks.Count
|
||||
peer_connection_manager_link_count = $managerLinks.Count
|
||||
relay_control_link_count = $relayControlLinks.Count
|
||||
direct_route_delivery_succeeded = $directRouteDelivered
|
||||
pass_matrix = $passMatrix
|
||||
direct_route_health = $directHealth | Select-Object -First 3
|
||||
replacement_route_health = $replacementRouteHealth | Select-Object -First 3
|
||||
relay_control_links = $relayControlLinks | Select-Object -First 5
|
||||
replacement_relay_control_links = $replacementRelayControlLinks | Select-Object -First 5
|
||||
cluster_summaries = $summary.cluster_summaries
|
||||
backend_log_tail = $backendLogs
|
||||
node_log_tail = $nodeLogs
|
||||
containers_left_running = [bool]$KeepRunning
|
||||
}
|
||||
|
||||
$failed = @($passMatrix.GetEnumerator() | Where-Object { -not $_.Value })
|
||||
$result | ConvertTo-Json -Depth 60
|
||||
|
||||
if ($failed.Count -gt 0) {
|
||||
throw "C17Z18 rendezvous relay replacement smoke failed: $($failed.Name -join ', ')"
|
||||
}
|
||||
|
||||
if (-not $KeepRunning) {
|
||||
Write-Host "Cleaning up C17Z18 containers..."
|
||||
Remove-C17Z12Containers
|
||||
Invoke-RemoteShell -Command "rm -rf '$remoteBuildDir'"
|
||||
}
|
||||
@@ -0,0 +1,478 @@
|
||||
param(
|
||||
[string]$DockerSshAlias = "test-docker",
|
||||
[string]$BackendImageTag = "rap-backend:c17z19-route-health-feedback-smoke",
|
||||
[string]$AdminEmail = "fabric-owner-c17z19@example.local",
|
||||
[string]$AdminPassword = "SmokePass!123",
|
||||
[int]$ApiPort = 18122,
|
||||
[int]$PostgresPort = 15444,
|
||||
[int]$RedisPort = 16444,
|
||||
[string]$ResultPath = "artifacts\c17z19-route-health-feedback-smoke-result.json",
|
||||
[switch]$KeepRunning
|
||||
)
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
$repoRoot = (Resolve-Path (Join-Path $PSScriptRoot "..\..")).ProviderPath
|
||||
$backendPublicBaseUrl = "http://192.168.200.61:$ApiPort/api/v1"
|
||||
$runId = "c17z19-" + (Get-Date -Format "yyyyMMdd-HHmmss")
|
||||
$remoteBuildDir = "/tmp/rap-c17z19-build-$runId"
|
||||
|
||||
$postgresName = "rap_c17z19_postgres"
|
||||
$redisName = "rap_c17z19_redis"
|
||||
$backendName = "rap_c17z19_backend"
|
||||
|
||||
function Invoke-RemoteDocker {
|
||||
param([string[]]$Arguments)
|
||||
& ssh $DockerSshAlias docker @Arguments
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "ssh $DockerSshAlias docker $($Arguments -join ' ') failed with exit code $LASTEXITCODE"
|
||||
}
|
||||
}
|
||||
|
||||
function Invoke-RemoteDockerText {
|
||||
param([string[]]$Arguments)
|
||||
$previousErrorActionPreference = $ErrorActionPreference
|
||||
$ErrorActionPreference = "Continue"
|
||||
try {
|
||||
$output = & ssh $DockerSshAlias docker @Arguments 2>&1 | ForEach-Object { $_.ToString() }
|
||||
}
|
||||
finally {
|
||||
$ErrorActionPreference = $previousErrorActionPreference
|
||||
}
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "ssh $DockerSshAlias docker $($Arguments -join ' ') failed with exit code $LASTEXITCODE"
|
||||
}
|
||||
return $output
|
||||
}
|
||||
|
||||
function Invoke-RemoteShell {
|
||||
param([string]$Command)
|
||||
& ssh $DockerSshAlias $Command
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "ssh $DockerSshAlias $Command failed with exit code $LASTEXITCODE"
|
||||
}
|
||||
}
|
||||
|
||||
function Send-RemoteBuildContext {
|
||||
Write-Host "Uploading backend build context to $DockerSshAlias..."
|
||||
Invoke-RemoteShell -Command "rm -rf '$remoteBuildDir' && mkdir -p '$remoteBuildDir'"
|
||||
& tar -czf - -C $repoRoot "backend" | & ssh $DockerSshAlias "tar -xzf - -C '$remoteBuildDir'"
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "upload build context failed"
|
||||
}
|
||||
}
|
||||
|
||||
function Invoke-Api {
|
||||
param(
|
||||
[string]$Method,
|
||||
[string]$Path,
|
||||
[object]$Body = $null
|
||||
)
|
||||
$uri = "$backendPublicBaseUrl$Path"
|
||||
try {
|
||||
if ($null -eq $Body) {
|
||||
return Invoke-RestMethod -Method $Method -Uri $uri -TimeoutSec 30
|
||||
}
|
||||
return Invoke-RestMethod -Method $Method -Uri $uri -ContentType "application/json" -Body ($Body | ConvertTo-Json -Depth 80) -TimeoutSec 30
|
||||
}
|
||||
catch {
|
||||
$statusCode = $null
|
||||
if ($_.Exception.Response) {
|
||||
$statusCode = [int]$_.Exception.Response.StatusCode
|
||||
}
|
||||
$details = $_.ErrorDetails.Message
|
||||
if (-not $details) {
|
||||
$details = $_.Exception.Message
|
||||
}
|
||||
throw "$Method $Path failed with HTTP $statusCode`: $details"
|
||||
}
|
||||
}
|
||||
|
||||
function Wait-HttpReady {
|
||||
param([string]$Url)
|
||||
for ($i = 0; $i -lt 90; $i++) {
|
||||
try {
|
||||
$response = Invoke-WebRequest -UseBasicParsing -Uri $Url -TimeoutSec 2
|
||||
if ($response.StatusCode -ge 200 -and $response.StatusCode -lt 300) {
|
||||
return
|
||||
}
|
||||
}
|
||||
catch {
|
||||
Start-Sleep -Seconds 1
|
||||
}
|
||||
}
|
||||
throw "Timed out waiting for $Url"
|
||||
}
|
||||
|
||||
function Remove-SmokeContainers {
|
||||
foreach ($name in @($backendName, $postgresName, $redisName)) {
|
||||
& ssh $DockerSshAlias docker rm -f $name 2>$null | Out-Null
|
||||
}
|
||||
}
|
||||
|
||||
function New-EndpointCandidate {
|
||||
param(
|
||||
[string]$EndpointID,
|
||||
[string]$NodeID,
|
||||
[string]$Address,
|
||||
[string]$Transport,
|
||||
[string]$Reachability,
|
||||
[string]$ConnectivityMode,
|
||||
[int]$Priority,
|
||||
[string[]]$PolicyTags = @()
|
||||
)
|
||||
return @{
|
||||
endpoint_id = $EndpointID
|
||||
node_id = $NodeID
|
||||
transport = $Transport
|
||||
address = $Address
|
||||
address_family = "ipv4"
|
||||
reachability = $Reachability
|
||||
nat_type = "none"
|
||||
connectivity_mode = $ConnectivityMode
|
||||
region = "docker-test"
|
||||
priority = $Priority
|
||||
policy_tags = $PolicyTags
|
||||
last_verified_at = (Get-Date).ToUniversalTime().ToString("o")
|
||||
metadata = @{
|
||||
stage = "c17z19"
|
||||
run_id = $runId
|
||||
production_payload_forwarding = $false
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function New-Node {
|
||||
param(
|
||||
[string]$Key,
|
||||
[string[]]$Roles
|
||||
)
|
||||
$fingerprint = "c17z19-fp-$Key-$([guid]::NewGuid().ToString('N'))"
|
||||
$publicKey = "c17z19-pub-$Key-$([guid]::NewGuid().ToString('N'))"
|
||||
$joinRequest = Invoke-Api -Method Post -Path "/clusters/$clusterID/join-requests" -Body @{
|
||||
join_token = $joinToken.join_token.token
|
||||
node_name = "c17z19-node-$Key"
|
||||
node_fingerprint = $fingerprint
|
||||
public_key = $publicKey
|
||||
reported_capabilities = @{
|
||||
can_route_mesh = $true
|
||||
testing_node = $true
|
||||
mesh_route_health_feedback = $true
|
||||
}
|
||||
reported_facts = @{
|
||||
stage = "c17z19"
|
||||
run_id = $runId
|
||||
}
|
||||
requested_roles = @()
|
||||
}
|
||||
$approved = Invoke-Api -Method Post -Path "/clusters/$clusterID/join-requests/$($joinRequest.join_request.id)/approve" -Body @{
|
||||
actor_user_id = $actorUserID
|
||||
node_key = $fingerprint
|
||||
ownership_type = "platform_managed"
|
||||
owner_organization_id = $null
|
||||
}
|
||||
$nodeID = $approved.node_bootstrap.node_id
|
||||
foreach ($role in $Roles) {
|
||||
Invoke-Api -Method Post -Path "/clusters/$clusterID/nodes/$nodeID/roles" -Body @{
|
||||
actor_user_id = $actorUserID
|
||||
role = $role
|
||||
status = "active"
|
||||
policy = @{
|
||||
stage = "c17z19"
|
||||
run_id = $runId
|
||||
synthetic_only = $true
|
||||
production_payload_forwarding = $false
|
||||
}
|
||||
} | Out-Null
|
||||
}
|
||||
return [pscustomobject]@{
|
||||
id = $nodeID
|
||||
fingerprint = $fingerprint
|
||||
public_key = $publicKey
|
||||
}
|
||||
}
|
||||
|
||||
function Get-OptionalProperty {
|
||||
param(
|
||||
[object]$Object,
|
||||
[string]$PropertyName
|
||||
)
|
||||
if ($null -eq $Object) {
|
||||
return $null
|
||||
}
|
||||
$property = $Object.PSObject.Properties[$PropertyName]
|
||||
if ($null -eq $property) {
|
||||
return $null
|
||||
}
|
||||
return $property.Value
|
||||
}
|
||||
|
||||
Write-Host "C17Z19 route-health feedback smoke run: $runId"
|
||||
Write-Host "Using SSH Docker host: $DockerSshAlias"
|
||||
|
||||
Remove-SmokeContainers
|
||||
Send-RemoteBuildContext
|
||||
|
||||
Write-Host "Building backend image on docker-test..."
|
||||
Invoke-RemoteDocker -Arguments @("build", "-f", "$remoteBuildDir/backend/Dockerfile", "-t", $BackendImageTag, "$remoteBuildDir/backend")
|
||||
|
||||
Write-Host "Starting isolated PostgreSQL and Redis..."
|
||||
Invoke-RemoteDocker -Arguments @(
|
||||
"run", "-d",
|
||||
"--name", $postgresName,
|
||||
"-e", "POSTGRES_DB=remote_access_platform",
|
||||
"-e", "POSTGRES_USER=rap_user",
|
||||
"-e", "POSTGRES_PASSWORD=rap_password",
|
||||
"-p", "$PostgresPort`:5432",
|
||||
"postgres:16"
|
||||
)
|
||||
Invoke-RemoteDocker -Arguments @(
|
||||
"run", "-d",
|
||||
"--name", $redisName,
|
||||
"-p", "$RedisPort`:6379",
|
||||
"redis:7"
|
||||
)
|
||||
|
||||
Invoke-RemoteShell -Command "for i in `$(seq 1 60); do docker exec $postgresName pg_isready -U rap_user -d remote_access_platform >/dev/null 2>&1 && exit 0; sleep 1; done; exit 1"
|
||||
|
||||
Write-Host "Applying migrations..."
|
||||
Invoke-RemoteShell -Command "for f in `$(find '$remoteBuildDir/backend/migrations' -name '*.up.sql' | sort); do docker exec -i $postgresName psql -U rap_user -d remote_access_platform -v ON_ERROR_STOP=1 -f - < `$f; done"
|
||||
|
||||
$secretBytes = New-Object byte[] 32
|
||||
[Security.Cryptography.RandomNumberGenerator]::Fill($secretBytes)
|
||||
$secretKeyB64 = [Convert]::ToBase64String($secretBytes)
|
||||
|
||||
Write-Host "Starting backend..."
|
||||
Invoke-RemoteDocker -Arguments @(
|
||||
"run", "-d",
|
||||
"--name", $backendName,
|
||||
"--network", "host",
|
||||
"-e", "APP_NAME=rap-api",
|
||||
"-e", "APP_ENV=c17z19-smoke",
|
||||
"-e", "HTTP_HOST=0.0.0.0",
|
||||
"-e", "HTTP_PORT=$ApiPort",
|
||||
"-e", "POSTGRES_DSN=postgres://rap_user:rap_password@127.0.0.1:$PostgresPort/remote_access_platform?sslmode=disable",
|
||||
"-e", "REDIS_ADDR=127.0.0.1:$RedisPort",
|
||||
"-e", "AUTH_ACCESS_TOKEN_SECRET=c17z19-access-secret",
|
||||
"-e", "AUTH_REFRESH_HASH_SECRET=c17z19-refresh-secret",
|
||||
"-e", "INSTALLATION_AUTHORITY_MODE=legacy",
|
||||
"-e", "INSTALLATION_INSECURE_BOOTSTRAP_ENABLED=true",
|
||||
"-e", "SECRET_ENCRYPTION_KEY_B64=$secretKeyB64",
|
||||
"-e", "SECRET_ENCRYPTION_KEY_ID=$runId",
|
||||
$BackendImageTag
|
||||
)
|
||||
Wait-HttpReady -Url "http://192.168.200.61:$ApiPort/readyz"
|
||||
|
||||
Write-Host "Bootstrapping owner, cluster, and synthetic route..."
|
||||
Invoke-Api -Method Post -Path "/installation/bootstrap-owner" -Body @{
|
||||
email = $AdminEmail
|
||||
password = $AdminPassword
|
||||
} | Out-Null
|
||||
$login = Invoke-Api -Method Post -Path "/auth/login" -Body @{
|
||||
email = $AdminEmail
|
||||
password = $AdminPassword
|
||||
device_fingerprint = "c17z19-smoke-device"
|
||||
device_label = "C17Z19 route health feedback smoke"
|
||||
trust_device = $true
|
||||
}
|
||||
$actorUserID = $login.user.id
|
||||
$cluster = Invoke-Api -Method Post -Path "/clusters/" -Body @{
|
||||
actor_user_id = $actorUserID
|
||||
slug = "c17z19-$((New-Guid).Guid.Substring(0, 8))"
|
||||
name = "C17Z19 Route Health Feedback Smoke"
|
||||
region = "docker-test"
|
||||
metadata = @{
|
||||
stage = "c17z19"
|
||||
run_id = $runId
|
||||
production_forwarding = $false
|
||||
service_payload_forwarding = $false
|
||||
}
|
||||
}
|
||||
$clusterID = $cluster.cluster.id
|
||||
Invoke-Api -Method Put -Path "/fabric/testing-flags" -Body @{
|
||||
actor_user_id = $actorUserID
|
||||
scope_type = "platform"
|
||||
scope_id = $null
|
||||
cluster_id = $null
|
||||
enabled = $true
|
||||
telemetry_enabled = $true
|
||||
synthetic_links_enabled = $true
|
||||
history_retention_hours = 24
|
||||
metadata = @{
|
||||
stage = "c17z19"
|
||||
run_id = $runId
|
||||
production_forwarding = $false
|
||||
service_payload_forwarding = $false
|
||||
}
|
||||
} | Out-Null
|
||||
|
||||
$joinToken = Invoke-Api -Method Post -Path "/clusters/$clusterID/join-tokens" -Body @{
|
||||
actor_user_id = $actorUserID
|
||||
scope = @{ purpose = "c17z19-route-health-feedback-smoke"; roles = @("core-mesh", "relay-node") }
|
||||
expires_at = (Get-Date).ToUniversalTime().AddHours(2).ToString("o")
|
||||
max_uses = 4
|
||||
}
|
||||
|
||||
$nodeA = New-Node -Key "a" -Roles @("core-mesh")
|
||||
$nodeB = New-Node -Key "b" -Roles @("core-mesh")
|
||||
$nodeS = New-Node -Key "s" -Roles @("core-mesh", "relay-node")
|
||||
$nodeT = New-Node -Key "t" -Roles @("core-mesh", "relay-node")
|
||||
|
||||
$nodeBCandidate = New-EndpointCandidate -EndpointID "node-b-outbound" -NodeID $nodeB.id -Address "node-b.reverse.local" -Transport "outbound_reverse" -Reachability "outbound_only" -ConnectivityMode "outbound_only" -Priority 5
|
||||
$nodeSCandidate = New-EndpointCandidate -EndpointID "node-s-public" -NodeID $nodeS.id -Address "http://node-s:19000" -Transport "direct_tcp_tls" -Reachability "public" -ConnectivityMode "direct" -Priority 1 -PolicyTags @("fast-path")
|
||||
$nodeTCandidate = New-EndpointCandidate -EndpointID "node-t-public" -NodeID $nodeT.id -Address "http://node-t:19000" -Transport "direct_tcp_tls" -Reachability "public" -ConnectivityMode "direct" -Priority 50
|
||||
|
||||
$routeIntent = Invoke-Api -Method Post -Path "/clusters/$clusterID/mesh/route-intents" -Body @{
|
||||
actor_user_id = $actorUserID
|
||||
source_selector = @{ node_id = $nodeA.id }
|
||||
destination_selector = @{ node_id = $nodeB.id }
|
||||
service_class = "synthetic"
|
||||
priority = 100
|
||||
policy = @{
|
||||
synthetic_enabled = $true
|
||||
hops = @($nodeA.id, $nodeS.id, $nodeT.id, $nodeB.id)
|
||||
allowed_channels = @("fabric_control", "route_control")
|
||||
peer_endpoint_candidates = @{
|
||||
"$($nodeB.id)" = @($nodeBCandidate)
|
||||
"$($nodeS.id)" = @($nodeSCandidate)
|
||||
"$($nodeT.id)" = @($nodeTCandidate)
|
||||
}
|
||||
}
|
||||
}
|
||||
$routeID = $routeIntent.route_intent.id
|
||||
|
||||
$initialConfig = Invoke-Api -Method Get -Path "/clusters/$clusterID/nodes/$($nodeA.id)/mesh/synthetic-config"
|
||||
$initialLease = @($initialConfig.synthetic_mesh_config.rendezvous_leases | Where-Object { $_.peer_node_id -eq $nodeB.id }) | Select-Object -First 1
|
||||
|
||||
Write-Host "Injecting drift route-health for initially selected relay..."
|
||||
Invoke-Api -Method Post -Path "/clusters/$clusterID/mesh/links" -Body @{
|
||||
source_node_id = $nodeA.id
|
||||
target_node_id = $nodeB.id
|
||||
link_status = "reachable"
|
||||
metadata = @{
|
||||
stage = "c17z19"
|
||||
observation_type = "synthetic_route_health"
|
||||
route_id = $routeID
|
||||
route_path_decision_applied = $true
|
||||
route_path_decision_selected_relay_id = $nodeS.id
|
||||
route_path_decision_rendezvous_peer_node_id = $nodeB.id
|
||||
route_path_decision_rendezvous_lease_id = "$routeID-rv-$($nodeB.id)-via-$($nodeS.id)"
|
||||
route_path_decision_rendezvous_lease_reason = "auto_rendezvous_required"
|
||||
expected_effective_hops = @($nodeA.id, $nodeS.id, $nodeB.id)
|
||||
observed_ack_path = @($nodeA.id, $nodeT.id, $nodeB.id)
|
||||
route_path_drift_detected = $true
|
||||
control_plane_only = $true
|
||||
production_forwarding = $false
|
||||
production_payload_forwarding = $false
|
||||
route_health_production_payload_forwarding = $false
|
||||
route_health_service_payload_forwarding = $false
|
||||
}
|
||||
} | Out-Null
|
||||
|
||||
$driftConfig = Invoke-Api -Method Get -Path "/clusters/$clusterID/nodes/$($nodeA.id)/mesh/synthetic-config"
|
||||
$driftLease = @($driftConfig.synthetic_mesh_config.rendezvous_leases | Where-Object { $_.peer_node_id -eq $nodeB.id }) | Select-Object -First 1
|
||||
$driftDecision = @($driftConfig.synthetic_mesh_config.route_path_decisions.decisions | Where-Object { $_.route_id -eq $routeID }) | Select-Object -First 1
|
||||
|
||||
Write-Host "Injecting healthy low-latency route-health for alternate relay..."
|
||||
$latency = 5
|
||||
$quality = 99
|
||||
Invoke-Api -Method Post -Path "/clusters/$clusterID/mesh/links" -Body @{
|
||||
source_node_id = $nodeA.id
|
||||
target_node_id = $nodeB.id
|
||||
link_status = "reachable"
|
||||
latency_ms = $latency
|
||||
quality_score = $quality
|
||||
metadata = @{
|
||||
stage = "c17z19"
|
||||
observation_type = "synthetic_route_health"
|
||||
route_id = $routeID
|
||||
route_path_decision_applied = $true
|
||||
route_path_decision_selected_relay_id = $nodeT.id
|
||||
route_path_decision_rendezvous_peer_node_id = $nodeB.id
|
||||
expected_effective_hops = @($nodeA.id, $nodeT.id, $nodeB.id)
|
||||
observed_ack_path = @($nodeA.id, $nodeT.id, $nodeB.id)
|
||||
route_path_drift_detected = $false
|
||||
control_plane_only = $true
|
||||
production_forwarding = $false
|
||||
production_payload_forwarding = $false
|
||||
route_health_production_payload_forwarding = $false
|
||||
route_health_service_payload_forwarding = $false
|
||||
}
|
||||
} | Out-Null
|
||||
|
||||
$healthyConfig = Invoke-Api -Method Get -Path "/clusters/$clusterID/nodes/$($nodeA.id)/mesh/synthetic-config"
|
||||
$healthyLease = @($healthyConfig.synthetic_mesh_config.rendezvous_leases | Where-Object { $_.peer_node_id -eq $nodeB.id }) | Select-Object -First 1
|
||||
$healthyLeaseReasons = @(Get-OptionalProperty -Object $healthyLease.metadata -PropertyName "relay_selection_score_reasons")
|
||||
$meshLinks = Invoke-Api -Method Get -Path "/clusters/$clusterID/mesh/links?actor_user_id=$actorUserID"
|
||||
$backendLogs = Invoke-RemoteDockerText -Arguments @("logs", "--tail", "80", $backendName)
|
||||
|
||||
$passMatrix = [ordered]@{
|
||||
backend_ready = $true
|
||||
owner_login = [bool]$actorUserID
|
||||
cluster_created = [bool]$clusterID
|
||||
route_intent_created = [bool]$routeID
|
||||
initial_fast_path_prefers_node_s = ($null -ne $initialLease -and $initialLease.relay_node_id -eq $nodeS.id)
|
||||
drift_feedback_reselects_node_t = ($null -ne $driftLease -and $driftLease.relay_node_id -eq $nodeT.id -and $driftLease.reason -eq "stale_relay_replacement")
|
||||
drift_route_decision_uses_node_t = ($null -ne $driftDecision -and $driftDecision.selected_relay_id -eq $nodeT.id -and $driftDecision.stale_relay_node_id -eq $nodeS.id)
|
||||
relay_policy_scoring_mode_c17z19 = ($driftConfig.synthetic_mesh_config.rendezvous_relay_policy.scoring_mode -eq "route_adjacency_endpoint_priority_mesh_link_health_synthetic_route_health_feedback")
|
||||
healthy_latency_keeps_node_t_selected = ($null -ne $healthyLease -and $healthyLease.relay_node_id -eq $nodeT.id)
|
||||
healthy_latency_score_reason_present = ($healthyLeaseReasons -contains "route_health_latency" -and $healthyLeaseReasons -contains "route_health_reachable")
|
||||
synthetic_config_signed = [bool]($healthyConfig.synthetic_mesh_config.authority_required -and $healthyConfig.synthetic_mesh_config.authority_payload -and $healthyConfig.synthetic_mesh_config.authority_signature)
|
||||
production_forwarding_disabled = ($healthyConfig.synthetic_mesh_config.production_forwarding -eq $false)
|
||||
}
|
||||
|
||||
$result = [ordered]@{
|
||||
run_id = $runId
|
||||
stage = "C17Z19 route-health feedback scoring smoke"
|
||||
docker_host = $DockerSshAlias
|
||||
backend_base_url = $backendPublicBaseUrl
|
||||
containers = @{
|
||||
backend = $backendName
|
||||
postgres = $postgresName
|
||||
redis = $redisName
|
||||
}
|
||||
cluster_id = $clusterID
|
||||
route_id = $routeID
|
||||
nodes = @{
|
||||
a = $nodeA.id
|
||||
b = $nodeB.id
|
||||
s = $nodeS.id
|
||||
t = $nodeT.id
|
||||
}
|
||||
initial_lease = $initialLease
|
||||
drift_lease = $driftLease
|
||||
drift_route_decision = $driftDecision
|
||||
healthy_lease = $healthyLease
|
||||
mesh_links = $meshLinks.mesh_links
|
||||
pass_matrix = $passMatrix
|
||||
backend_log_tail = $backendLogs
|
||||
containers_left_running = [bool]$KeepRunning
|
||||
}
|
||||
|
||||
$failed = @($passMatrix.GetEnumerator() | Where-Object { -not $_.Value })
|
||||
$resultJson = $result | ConvertTo-Json -Depth 80
|
||||
|
||||
if ($ResultPath -ne "") {
|
||||
if ([System.IO.Path]::IsPathRooted($ResultPath)) {
|
||||
$resultFullPath = $ResultPath
|
||||
} else {
|
||||
$resultFullPath = Join-Path $repoRoot $ResultPath
|
||||
}
|
||||
New-Item -ItemType Directory -Force -Path (Split-Path $resultFullPath) | Out-Null
|
||||
Set-Content -Path $resultFullPath -Value $resultJson -Encoding UTF8
|
||||
Write-Host "Result written to $resultFullPath"
|
||||
}
|
||||
|
||||
$resultJson
|
||||
|
||||
if ($failed.Count -gt 0) {
|
||||
throw "C17Z19 route-health feedback smoke failed: $($failed.Name -join ', ')"
|
||||
}
|
||||
|
||||
Invoke-RemoteShell -Command "rm -rf '$remoteBuildDir'"
|
||||
|
||||
if (-not $KeepRunning) {
|
||||
Write-Host "Cleaning up C17Z19 containers..."
|
||||
Remove-SmokeContainers
|
||||
}
|
||||
@@ -0,0 +1,174 @@
|
||||
param(
|
||||
[string]$BackendBaseUrl = "http://192.168.200.61:8080/api/v1",
|
||||
[string]$DockerContext = "test-ubuntu",
|
||||
[string]$AdminEmail = "windows-smoke@example.local",
|
||||
[string]$AdminPassword = "SmokePass!123",
|
||||
[int]$Count = 3,
|
||||
[string]$ImageTag = "rap-node-agent:control-panel-test"
|
||||
)
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
$repoRoot = (Resolve-Path (Join-Path $PSScriptRoot "..\..")).ProviderPath
|
||||
$stateRoot = Join-Path "C:\work" "rap-fabric-test-nodes"
|
||||
New-Item -ItemType Directory -Force -Path $stateRoot | Out-Null
|
||||
|
||||
function Invoke-Docker {
|
||||
param([string[]]$Arguments)
|
||||
docker @Arguments
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "docker $($Arguments -join ' ') failed with exit code $LASTEXITCODE"
|
||||
}
|
||||
}
|
||||
|
||||
function Invoke-Api {
|
||||
param(
|
||||
[string]$Method,
|
||||
[string]$Path,
|
||||
[object]$Body = $null
|
||||
)
|
||||
$uri = "$BackendBaseUrl$Path"
|
||||
if ($null -eq $Body) {
|
||||
return Invoke-RestMethod -Method $Method -Uri $uri -TimeoutSec 30
|
||||
}
|
||||
return Invoke-RestMethod -Method $Method -Uri $uri -ContentType "application/json" -Body ($Body | ConvertTo-Json -Depth 20) -TimeoutSec 30
|
||||
}
|
||||
|
||||
Write-Host "Building node-agent image on Docker context $DockerContext..."
|
||||
Invoke-Docker -Arguments @("--context", $DockerContext, "build", "-f", "$repoRoot\agents\rap-node-agent\Dockerfile", "-t", $ImageTag, "$repoRoot")
|
||||
|
||||
$login = Invoke-Api -Method Post -Path "/auth/login" -Body @{
|
||||
email = $AdminEmail
|
||||
password = $AdminPassword
|
||||
device_fingerprint = "fabric-test-nodes-script"
|
||||
device_label = "Fabric Test Nodes Script"
|
||||
trust_device = $true
|
||||
}
|
||||
$actorUserID = $login.user.id
|
||||
|
||||
$clusters = Invoke-Api -Method Get -Path "/clusters/?actor_user_id=$actorUserID"
|
||||
$cluster = @($clusters.clusters)[0]
|
||||
if ($null -eq $cluster) {
|
||||
throw "No cluster available for test node deployment."
|
||||
}
|
||||
$clusterID = $cluster.id
|
||||
Write-Host "Using cluster $($cluster.name) $clusterID"
|
||||
|
||||
Invoke-Api -Method Put -Path "/fabric/testing-flags" -Body @{
|
||||
actor_user_id = $actorUserID
|
||||
scope_type = "platform"
|
||||
scope_id = $null
|
||||
cluster_id = $null
|
||||
enabled = $true
|
||||
telemetry_enabled = $true
|
||||
synthetic_links_enabled = $true
|
||||
history_retention_hours = 24
|
||||
metadata = @{ source = "deploy-test-nodes.ps1"; runtime_mesh_enabled = $false }
|
||||
} | Out-Null
|
||||
|
||||
$joinToken = Invoke-Api -Method Post -Path "/clusters/$clusterID/join-tokens" -Body @{
|
||||
actor_user_id = $actorUserID
|
||||
scope = @{ purpose = "fabric-test-nodes"; roles = @("core-mesh", "relay-node") }
|
||||
expires_at = (Get-Date).ToUniversalTime().AddHours(2).ToString("o")
|
||||
max_uses = $Count
|
||||
}
|
||||
|
||||
$nodes = @()
|
||||
for ($i = 1; $i -le $Count; $i++) {
|
||||
$nodeName = "fabric-test-node-$i"
|
||||
$fingerprint = "rap-node-fp_$([guid]::NewGuid().ToString('N'))"
|
||||
$publicKey = "rap-node-pub_$([guid]::NewGuid().ToString('N'))"
|
||||
|
||||
$joinRequest = Invoke-Api -Method Post -Path "/clusters/$clusterID/join-requests" -Body @{
|
||||
join_token = $joinToken.join_token.token
|
||||
node_name = $nodeName
|
||||
node_fingerprint = $fingerprint
|
||||
public_key = $publicKey
|
||||
reported_capabilities = @{
|
||||
can_accept_node_ingress = $true
|
||||
can_route_mesh = $true
|
||||
can_run_rdp_worker = $false
|
||||
testing_node = $true
|
||||
}
|
||||
reported_facts = @{
|
||||
os = "linux"
|
||||
runtime = "docker-test"
|
||||
source = "deploy-test-nodes.ps1"
|
||||
}
|
||||
requested_roles = @("core-mesh", "relay-node")
|
||||
}
|
||||
|
||||
$approved = Invoke-Api -Method Post -Path "/clusters/$clusterID/join-requests/$($joinRequest.join_request.id)/approve" -Body @{
|
||||
actor_user_id = $actorUserID
|
||||
node_key = $fingerprint
|
||||
ownership_type = "platform_managed"
|
||||
owner_organization_id = $null
|
||||
}
|
||||
$nodeID = $approved.node_bootstrap.node_id
|
||||
$nodes += [pscustomobject]@{ name = $nodeName; id = $nodeID; fingerprint = $fingerprint; public_key = $publicKey }
|
||||
|
||||
$nodeStateDir = Join-Path $stateRoot $nodeName
|
||||
New-Item -ItemType Directory -Force -Path $nodeStateDir | Out-Null
|
||||
@{
|
||||
node_id = $nodeID
|
||||
cluster_id = $clusterID
|
||||
node_name = $nodeName
|
||||
node_fingerprint = $fingerprint
|
||||
public_key = $publicKey
|
||||
identity_status = "active"
|
||||
created_at = (Get-Date).ToUniversalTime().ToString("o")
|
||||
updated_at = (Get-Date).ToUniversalTime().ToString("o")
|
||||
} | ConvertTo-Json -Depth 10 | Set-Content -Encoding UTF8 -Path (Join-Path $nodeStateDir "identity.json")
|
||||
|
||||
$containerName = "rap_fabric_test_node_$i"
|
||||
docker --context $DockerContext rm -f $containerName 2>$null | Out-Null
|
||||
Invoke-Docker -Arguments @(
|
||||
"--context", $DockerContext,
|
||||
"create",
|
||||
"--name", $containerName,
|
||||
"--network", "host",
|
||||
"-e", "RAP_BACKEND_URL=$BackendBaseUrl",
|
||||
"-e", "RAP_NODE_STATE_DIR=/tmp/state",
|
||||
"-e", "RAP_HEARTBEAT_INTERVAL_SECONDS=5",
|
||||
$ImageTag,
|
||||
"-backend-url", $BackendBaseUrl,
|
||||
"-state-dir", "/tmp/state",
|
||||
"-heartbeat-interval", "5s"
|
||||
) | Out-Null
|
||||
Invoke-Docker -Arguments @("--context", $DockerContext, "cp", "$nodeStateDir\.", "$containerName`:/tmp/state")
|
||||
Invoke-Docker -Arguments @("--context", $DockerContext, "start", $containerName) | Out-Null
|
||||
Write-Host "Started $containerName for $nodeName $nodeID"
|
||||
}
|
||||
|
||||
Start-Sleep -Seconds 12
|
||||
|
||||
for ($i = 0; $i -lt $nodes.Count; $i++) {
|
||||
for ($j = 0; $j -lt $nodes.Count; $j++) {
|
||||
if ($i -eq $j) { continue }
|
||||
$latency = 2 + (($i + $j) % 7)
|
||||
Invoke-Api -Method Post -Path "/clusters/$clusterID/mesh/links" -Body @{
|
||||
source_node_id = $nodes[$i].id
|
||||
target_node_id = $nodes[$j].id
|
||||
link_status = "reachable"
|
||||
latency_ms = $latency
|
||||
quality_score = 95 - $latency
|
||||
metadata = @{
|
||||
source = "deploy-test-nodes.ps1"
|
||||
synthetic = $true
|
||||
runtime_mesh_enabled = $false
|
||||
}
|
||||
} | Out-Null
|
||||
}
|
||||
}
|
||||
|
||||
$summary = Invoke-Api -Method Get -Path "/cluster-admin-summaries?actor_user_id=$actorUserID"
|
||||
$links = Invoke-Api -Method Get -Path "/clusters/$clusterID/mesh/links?actor_user_id=$actorUserID"
|
||||
|
||||
[pscustomobject]@{
|
||||
cluster_id = $clusterID
|
||||
nodes_started = $nodes.Count
|
||||
cluster_summaries = $summary.cluster_summaries
|
||||
mesh_link_count = @($links.mesh_links).Count
|
||||
state_root = $stateRoot
|
||||
} | ConvertTo-Json -Depth 20
|
||||
@@ -0,0 +1,468 @@
|
||||
param(
|
||||
[string]$DockerSshAlias = "test-docker",
|
||||
[string]$BackendImageTag = "rap-backend:dev-enrollment-bootstrap-smoke",
|
||||
[string]$NodeAgentImageTag = "rap-node-agent:dev-enrollment-bootstrap-smoke",
|
||||
[string]$AdminEmail = "fabric-owner-dev-bootstrap@example.local",
|
||||
[string]$AdminPassword = "SmokePass!123",
|
||||
[int]$ApiPort = 18121,
|
||||
[int]$PostgresPort = 15443,
|
||||
[int]$RedisPort = 16443,
|
||||
[int]$MeshPort = 19131,
|
||||
[string]$ResultPath = "artifacts\dev-cluster-enrollment-bootstrap-smoke-result.json",
|
||||
[switch]$KeepRunning
|
||||
)
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
$repoRoot = (Resolve-Path (Join-Path $PSScriptRoot "..\..")).ProviderPath
|
||||
$backendPublicBaseUrl = "http://192.168.200.61:$ApiPort/api/v1"
|
||||
$backendContainerBaseUrl = "http://127.0.0.1:$ApiPort/api/v1"
|
||||
$runId = "dev-bootstrap-" + (Get-Date -Format "yyyyMMdd-HHmmss")
|
||||
$remoteBuildDir = "/tmp/rap-dev-bootstrap-build-$runId"
|
||||
|
||||
$postgresName = "rap_dev_bootstrap_postgres"
|
||||
$redisName = "rap_dev_bootstrap_redis"
|
||||
$backendName = "rap_dev_bootstrap_backend"
|
||||
$nodeName = "rap-dev-bootstrap-node-core"
|
||||
$nodeContainerName = "rap_dev_bootstrap_node_core"
|
||||
|
||||
function Invoke-RemoteDocker {
|
||||
param([string[]]$Arguments)
|
||||
& ssh $DockerSshAlias docker @Arguments
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "ssh $DockerSshAlias docker $($Arguments -join ' ') failed with exit code $LASTEXITCODE"
|
||||
}
|
||||
}
|
||||
|
||||
function Invoke-RemoteDockerText {
|
||||
param([string[]]$Arguments)
|
||||
$previousErrorActionPreference = $ErrorActionPreference
|
||||
$ErrorActionPreference = "Continue"
|
||||
try {
|
||||
$output = & ssh $DockerSshAlias docker @Arguments 2>&1 | ForEach-Object { $_.ToString() }
|
||||
}
|
||||
finally {
|
||||
$ErrorActionPreference = $previousErrorActionPreference
|
||||
}
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "ssh $DockerSshAlias docker $($Arguments -join ' ') failed with exit code $LASTEXITCODE"
|
||||
}
|
||||
return $output
|
||||
}
|
||||
|
||||
function Invoke-RemoteShell {
|
||||
param([string]$Command)
|
||||
& ssh $DockerSshAlias $Command
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "ssh $DockerSshAlias $Command failed with exit code $LASTEXITCODE"
|
||||
}
|
||||
}
|
||||
|
||||
function Invoke-RemoteShellOptionalText {
|
||||
param([string]$Command)
|
||||
$previousErrorActionPreference = $ErrorActionPreference
|
||||
$ErrorActionPreference = "Continue"
|
||||
try {
|
||||
$output = & ssh $DockerSshAlias $Command 2>&1 | ForEach-Object { $_.ToString() }
|
||||
$exitCode = $LASTEXITCODE
|
||||
}
|
||||
finally {
|
||||
$ErrorActionPreference = $previousErrorActionPreference
|
||||
}
|
||||
return [pscustomobject]@{
|
||||
exit_code = $exitCode
|
||||
output = $output
|
||||
}
|
||||
}
|
||||
|
||||
function Send-RemoteBuildContext {
|
||||
Write-Host "Uploading backend and node-agent build context to $DockerSshAlias..."
|
||||
Invoke-RemoteShell -Command "rm -rf '$remoteBuildDir' && mkdir -p '$remoteBuildDir'"
|
||||
& tar -czf - -C $repoRoot "backend" "agents/rap-node-agent" | & ssh $DockerSshAlias "tar -xzf - -C '$remoteBuildDir'"
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "upload build context failed"
|
||||
}
|
||||
}
|
||||
|
||||
function Invoke-Api {
|
||||
param(
|
||||
[string]$Method,
|
||||
[string]$Path,
|
||||
[object]$Body = $null
|
||||
)
|
||||
$uri = "$backendPublicBaseUrl$Path"
|
||||
if ($null -eq $Body) {
|
||||
return Invoke-RestMethod -Method $Method -Uri $uri -TimeoutSec 30
|
||||
}
|
||||
return Invoke-RestMethod -Method $Method -Uri $uri -ContentType "application/json" -Body ($Body | ConvertTo-Json -Depth 60) -TimeoutSec 30
|
||||
}
|
||||
|
||||
function Wait-HttpReady {
|
||||
param([string]$Url)
|
||||
for ($i = 0; $i -lt 90; $i++) {
|
||||
try {
|
||||
$response = Invoke-WebRequest -UseBasicParsing -Uri $Url -TimeoutSec 2
|
||||
if ($response.StatusCode -ge 200 -and $response.StatusCode -lt 300) {
|
||||
return
|
||||
}
|
||||
}
|
||||
catch {
|
||||
Start-Sleep -Seconds 1
|
||||
}
|
||||
}
|
||||
throw "Timed out waiting for $Url"
|
||||
}
|
||||
|
||||
function Remove-SmokeContainers {
|
||||
foreach ($name in @($nodeContainerName, $backendName, $postgresName, $redisName)) {
|
||||
& ssh $DockerSshAlias docker rm -f $name 2>$null | Out-Null
|
||||
}
|
||||
}
|
||||
|
||||
function Wait-JoinRequest {
|
||||
param(
|
||||
[string]$ClusterID,
|
||||
[string]$ActorUserID,
|
||||
[string]$ExpectedNodeName,
|
||||
[int]$TimeoutSeconds = 60
|
||||
)
|
||||
$deadline = (Get-Date).AddSeconds($TimeoutSeconds)
|
||||
do {
|
||||
$response = Invoke-Api -Method Get -Path "/clusters/$ClusterID/join-requests?actor_user_id=$ActorUserID"
|
||||
$match = @($response.join_requests | Where-Object {
|
||||
$_.node_name -eq $ExpectedNodeName -and $_.status -eq "pending"
|
||||
} | Sort-Object created_at -Descending | Select-Object -First 1)
|
||||
if ($match.Count -gt 0) {
|
||||
return $match[0]
|
||||
}
|
||||
Start-Sleep -Seconds 1
|
||||
} while ((Get-Date) -lt $deadline)
|
||||
throw "Timed out waiting for node-agent join request"
|
||||
}
|
||||
|
||||
function Get-RemoteContainerIdentity {
|
||||
$remoteIdentity = "/tmp/$runId-identity.json"
|
||||
$command = "rm -f '$remoteIdentity'; docker cp '${nodeContainerName}:/tmp/state/identity.json' '$remoteIdentity' >/dev/null 2>&1 && cat '$remoteIdentity'"
|
||||
$result = Invoke-RemoteShellOptionalText -Command $command
|
||||
if ($result.exit_code -ne 0) {
|
||||
return $null
|
||||
}
|
||||
$raw = ($result.output -join "`n").Trim()
|
||||
if ($raw -eq "") {
|
||||
return $null
|
||||
}
|
||||
return $raw | ConvertFrom-Json
|
||||
}
|
||||
|
||||
function Wait-AgentApprovedIdentity {
|
||||
param([int]$TimeoutSeconds = 90)
|
||||
$deadline = (Get-Date).AddSeconds($TimeoutSeconds)
|
||||
do {
|
||||
$identity = Get-RemoteContainerIdentity
|
||||
if ($null -ne $identity -and
|
||||
$identity.node_id -and
|
||||
$identity.identity_status -eq "active" -and
|
||||
$identity.cluster_authority_public_key -and
|
||||
$identity.cluster_authority_fingerprint) {
|
||||
return $identity
|
||||
}
|
||||
Start-Sleep -Seconds 1
|
||||
} while ((Get-Date) -lt $deadline)
|
||||
throw "Timed out waiting for node-agent approved identity state"
|
||||
}
|
||||
|
||||
function Wait-NodeHeartbeat {
|
||||
param(
|
||||
[string]$ClusterID,
|
||||
[string]$NodeID,
|
||||
[string]$ActorUserID,
|
||||
[int]$TimeoutSeconds = 60
|
||||
)
|
||||
$deadline = (Get-Date).AddSeconds($TimeoutSeconds)
|
||||
do {
|
||||
$response = Invoke-Api -Method Get -Path "/clusters/$ClusterID/nodes/$NodeID/heartbeats?actor_user_id=$ActorUserID&limit=5"
|
||||
$heartbeats = @($response.heartbeats)
|
||||
if ($heartbeats.Count -gt 0) {
|
||||
return $heartbeats[0]
|
||||
}
|
||||
Start-Sleep -Seconds 1
|
||||
} while ((Get-Date) -lt $deadline)
|
||||
throw "Timed out waiting for node-agent heartbeat"
|
||||
}
|
||||
|
||||
function Wait-AgentLogContains {
|
||||
param(
|
||||
[string]$Pattern,
|
||||
[int]$TimeoutSeconds = 60
|
||||
)
|
||||
$deadline = (Get-Date).AddSeconds($TimeoutSeconds)
|
||||
do {
|
||||
$logs = Invoke-RemoteDockerText -Arguments @("logs", "--tail", "200", $nodeContainerName)
|
||||
$joined = $logs -join "`n"
|
||||
if ($joined -match $Pattern) {
|
||||
return $logs
|
||||
}
|
||||
Start-Sleep -Seconds 1
|
||||
} while ((Get-Date) -lt $deadline)
|
||||
return Invoke-RemoteDockerText -Arguments @("logs", "--tail", "200", $nodeContainerName)
|
||||
}
|
||||
|
||||
Write-Host "Dev cluster enrollment/bootstrap smoke run: $runId"
|
||||
Write-Host "Using SSH Docker host: $DockerSshAlias"
|
||||
|
||||
Remove-SmokeContainers
|
||||
Send-RemoteBuildContext
|
||||
|
||||
Write-Host "Building backend image on docker-test..."
|
||||
Invoke-RemoteDocker -Arguments @("build", "-f", "$remoteBuildDir/backend/Dockerfile", "-t", $BackendImageTag, "$remoteBuildDir/backend")
|
||||
|
||||
Write-Host "Building node-agent image on docker-test..."
|
||||
Invoke-RemoteDocker -Arguments @("build", "-f", "$remoteBuildDir/agents/rap-node-agent/Dockerfile", "-t", $NodeAgentImageTag, $remoteBuildDir)
|
||||
|
||||
Write-Host "Starting isolated PostgreSQL and Redis..."
|
||||
Invoke-RemoteDocker -Arguments @(
|
||||
"run", "-d",
|
||||
"--name", $postgresName,
|
||||
"-e", "POSTGRES_DB=remote_access_platform",
|
||||
"-e", "POSTGRES_USER=rap_user",
|
||||
"-e", "POSTGRES_PASSWORD=rap_password",
|
||||
"-p", "$PostgresPort`:5432",
|
||||
"postgres:16"
|
||||
)
|
||||
Invoke-RemoteDocker -Arguments @(
|
||||
"run", "-d",
|
||||
"--name", $redisName,
|
||||
"-p", "$RedisPort`:6379",
|
||||
"redis:7"
|
||||
)
|
||||
|
||||
Invoke-RemoteShell -Command "for i in `$(seq 1 60); do docker exec $postgresName pg_isready -U rap_user -d remote_access_platform >/dev/null 2>&1 && exit 0; sleep 1; done; exit 1"
|
||||
|
||||
Write-Host "Applying migrations..."
|
||||
Invoke-RemoteShell -Command "for f in `$(find '$remoteBuildDir/backend/migrations' -name '*.up.sql' | sort); do docker exec -i $postgresName psql -U rap_user -d remote_access_platform -v ON_ERROR_STOP=1 -f - < `$f; done"
|
||||
|
||||
$secretBytes = New-Object byte[] 32
|
||||
[Security.Cryptography.RandomNumberGenerator]::Fill($secretBytes)
|
||||
$secretKeyB64 = [Convert]::ToBase64String($secretBytes)
|
||||
|
||||
Write-Host "Starting backend..."
|
||||
Invoke-RemoteDocker -Arguments @(
|
||||
"run", "-d",
|
||||
"--name", $backendName,
|
||||
"--network", "host",
|
||||
"-e", "APP_NAME=rap-api",
|
||||
"-e", "APP_ENV=dev-bootstrap-smoke",
|
||||
"-e", "HTTP_HOST=0.0.0.0",
|
||||
"-e", "HTTP_PORT=$ApiPort",
|
||||
"-e", "POSTGRES_DSN=postgres://rap_user:rap_password@127.0.0.1:$PostgresPort/remote_access_platform?sslmode=disable",
|
||||
"-e", "REDIS_ADDR=127.0.0.1:$RedisPort",
|
||||
"-e", "AUTH_ACCESS_TOKEN_SECRET=dev-bootstrap-access-secret",
|
||||
"-e", "AUTH_REFRESH_HASH_SECRET=dev-bootstrap-refresh-secret",
|
||||
"-e", "INSTALLATION_AUTHORITY_MODE=legacy",
|
||||
"-e", "INSTALLATION_INSECURE_BOOTSTRAP_ENABLED=true",
|
||||
"-e", "SECRET_ENCRYPTION_KEY_B64=$secretKeyB64",
|
||||
"-e", "SECRET_ENCRYPTION_KEY_ID=$runId",
|
||||
$BackendImageTag
|
||||
)
|
||||
Wait-HttpReady -Url "http://192.168.200.61:$ApiPort/readyz"
|
||||
|
||||
Write-Host "Bootstrapping dev platform owner through installation API..."
|
||||
$bootstrap = Invoke-Api -Method Post -Path "/installation/bootstrap-owner" -Body @{
|
||||
email = $AdminEmail
|
||||
password = $AdminPassword
|
||||
}
|
||||
$installationStatus = Invoke-Api -Method Get -Path "/installation/status"
|
||||
|
||||
Write-Host "Logging in as platform owner..."
|
||||
$login = Invoke-Api -Method Post -Path "/auth/login" -Body @{
|
||||
email = $AdminEmail
|
||||
password = $AdminPassword
|
||||
device_fingerprint = "dev-bootstrap-smoke-device"
|
||||
device_label = "Dev bootstrap lifecycle smoke"
|
||||
trust_device = $true
|
||||
}
|
||||
$actorUserID = $login.user.id
|
||||
|
||||
Write-Host "Creating dev cluster and join token..."
|
||||
$cluster = Invoke-Api -Method Post -Path "/clusters/" -Body @{
|
||||
actor_user_id = $actorUserID
|
||||
slug = "dev-bootstrap-$((New-Guid).Guid.Substring(0, 8))"
|
||||
name = "Dev Enrollment Bootstrap Smoke"
|
||||
region = "docker-test"
|
||||
metadata = @{
|
||||
stage = "dev-enrollment-bootstrap-smoke"
|
||||
run_id = $runId
|
||||
mandatory_roles_only = $true
|
||||
production_forwarding = $false
|
||||
service_payload_forwarding = $false
|
||||
}
|
||||
}
|
||||
$clusterID = $cluster.cluster.id
|
||||
|
||||
Invoke-Api -Method Put -Path "/fabric/testing-flags" -Body @{
|
||||
actor_user_id = $actorUserID
|
||||
scope_type = "platform"
|
||||
scope_id = $null
|
||||
cluster_id = $null
|
||||
enabled = $true
|
||||
telemetry_enabled = $true
|
||||
synthetic_links_enabled = $true
|
||||
history_retention_hours = 24
|
||||
metadata = @{
|
||||
stage = "dev-enrollment-bootstrap-smoke"
|
||||
run_id = $runId
|
||||
production_forwarding = $false
|
||||
service_payload_forwarding = $false
|
||||
}
|
||||
} | Out-Null
|
||||
|
||||
$joinToken = Invoke-Api -Method Post -Path "/clusters/$clusterID/join-tokens" -Body @{
|
||||
actor_user_id = $actorUserID
|
||||
scope = @{
|
||||
purpose = "dev-enrollment-bootstrap-smoke"
|
||||
roles = @("core-mesh")
|
||||
mandatory_only = $true
|
||||
production_forwarding = $false
|
||||
}
|
||||
expires_at = (Get-Date).ToUniversalTime().AddHours(2).ToString("o")
|
||||
max_uses = 1
|
||||
}
|
||||
|
||||
Write-Host "Starting real node-agent enrollment container..."
|
||||
Invoke-RemoteDocker -Arguments @(
|
||||
"run", "-d",
|
||||
"--name", $nodeContainerName,
|
||||
"--network", "host",
|
||||
"-e", "RAP_BACKEND_URL=$backendContainerBaseUrl",
|
||||
"-e", "RAP_CLUSTER_ID=$clusterID",
|
||||
"-e", "RAP_JOIN_TOKEN=$($joinToken.join_token.token)",
|
||||
"-e", "RAP_NODE_NAME=$nodeName",
|
||||
"-e", "RAP_NODE_STATE_DIR=/tmp/state",
|
||||
"-e", "RAP_HEARTBEAT_INTERVAL_SECONDS=2",
|
||||
"-e", "RAP_ENROLLMENT_POLL_INTERVAL_SECONDS=1",
|
||||
"-e", "RAP_ENROLLMENT_POLL_TIMEOUT_SECONDS=90",
|
||||
"-e", "RAP_MESH_SYNTHETIC_RUNTIME_ENABLED=true",
|
||||
"-e", "RAP_MESH_LISTEN_ADDR=0.0.0.0:$MeshPort",
|
||||
"-e", "RAP_MESH_ADVERTISE_ENDPOINT=http://127.0.0.1:$MeshPort",
|
||||
"-e", "RAP_MESH_ADVERTISE_TRANSPORT=direct_tcp_tls",
|
||||
"-e", "RAP_MESH_CONNECTIVITY_MODE=direct",
|
||||
"-e", "RAP_MESH_NAT_TYPE=none",
|
||||
"-e", "RAP_MESH_REGION=docker-test",
|
||||
$NodeAgentImageTag
|
||||
)
|
||||
|
||||
Write-Host "Waiting for pending join request from node-agent..."
|
||||
$joinRequest = Wait-JoinRequest -ClusterID $clusterID -ActorUserID $actorUserID -ExpectedNodeName $nodeName
|
||||
|
||||
Write-Host "Approving join request..."
|
||||
$approved = Invoke-Api -Method Post -Path "/clusters/$clusterID/join-requests/$($joinRequest.id)/approve" -Body @{
|
||||
actor_user_id = $actorUserID
|
||||
node_key = $joinRequest.node_fingerprint
|
||||
ownership_type = "platform_managed"
|
||||
owner_organization_id = $null
|
||||
}
|
||||
|
||||
Write-Host "Waiting for node-agent to persist signed bootstrap authority pin..."
|
||||
$identity = Wait-AgentApprovedIdentity
|
||||
$nodeID = $identity.node_id
|
||||
|
||||
Write-Host "Assigning mandatory core-mesh role after approved identity..."
|
||||
Invoke-Api -Method Post -Path "/clusters/$clusterID/nodes/$nodeID/roles" -Body @{
|
||||
actor_user_id = $actorUserID
|
||||
role = "core-mesh"
|
||||
status = "active"
|
||||
policy = @{
|
||||
stage = "dev-enrollment-bootstrap-smoke"
|
||||
run_id = $runId
|
||||
mandatory = $true
|
||||
production_forwarding = $false
|
||||
service_payload_forwarding = $false
|
||||
}
|
||||
} | Out-Null
|
||||
|
||||
Write-Host "Waiting for heartbeat and signed synthetic config verification..."
|
||||
$heartbeat = Wait-NodeHeartbeat -ClusterID $clusterID -NodeID $nodeID -ActorUserID $actorUserID
|
||||
$configResponse = Invoke-Api -Method Get -Path "/clusters/$clusterID/nodes/$nodeID/mesh/synthetic-config"
|
||||
$syntheticConfig = $configResponse.synthetic_mesh_config
|
||||
$nodeLogs = Wait-AgentLogContains -Pattern "synthetic mesh config loaded: source=control_plane" -TimeoutSeconds 60
|
||||
$backendLogs = Invoke-RemoteDockerText -Arguments @("logs", "--tail", "120", $backendName)
|
||||
$summaries = Invoke-Api -Method Get -Path "/cluster-admin-summaries?actor_user_id=$actorUserID"
|
||||
|
||||
$agentLogText = $nodeLogs -join "`n"
|
||||
$passMatrix = [ordered]@{
|
||||
installation_bootstrapped = [bool]$installationStatus.installation.bootstrapped
|
||||
cluster_created = [bool]$clusterID
|
||||
join_token_has_cluster_authority_signature = [bool]($joinToken.join_token.authority_payload -and $joinToken.join_token.authority_signature)
|
||||
agent_submitted_pending_join_request = [bool]($joinRequest.id -and $joinRequest.status -eq "pending")
|
||||
approval_returned_signed_bootstrap = [bool]($approved.node_bootstrap.cluster_authority -and $approved.node_bootstrap.authority_payload -and $approved.node_bootstrap.authority_signature)
|
||||
agent_persisted_approved_identity = [bool]($identity.node_id -eq $approved.node_bootstrap.node_id -and $identity.identity_status -eq "active")
|
||||
agent_persisted_cluster_authority_pin = [bool]($identity.cluster_authority_public_key -and $identity.cluster_authority_fingerprint)
|
||||
heartbeat_after_auto_bootstrap = [bool]$heartbeat.id
|
||||
synthetic_config_has_cluster_authority_signature = [bool]($syntheticConfig.authority_required -and $syntheticConfig.cluster_authority -and $syntheticConfig.authority_payload -and $syntheticConfig.authority_signature)
|
||||
agent_verified_control_plane_synthetic_config = [bool]($agentLogText -match "synthetic mesh config loaded: source=control_plane")
|
||||
production_forwarding_disabled = [bool](-not $syntheticConfig.production_forwarding)
|
||||
}
|
||||
|
||||
$result = [ordered]@{
|
||||
run_id = $runId
|
||||
stage = "dev cluster enrollment/bootstrap lifecycle smoke"
|
||||
docker_host = $DockerSshAlias
|
||||
backend_base_url = $backendPublicBaseUrl
|
||||
api_port = $ApiPort
|
||||
postgres_port = $PostgresPort
|
||||
redis_port = $RedisPort
|
||||
mesh_port = $MeshPort
|
||||
backend_image = $BackendImageTag
|
||||
node_agent_image = $NodeAgentImageTag
|
||||
containers = @{
|
||||
backend = $backendName
|
||||
postgres = $postgresName
|
||||
redis = $redisName
|
||||
node_agent = $nodeContainerName
|
||||
}
|
||||
installation = $installationStatus.installation
|
||||
admin_user_id = $actorUserID
|
||||
cluster_id = $clusterID
|
||||
node_id = $nodeID
|
||||
join_request_id = $joinRequest.id
|
||||
join_token_signature = $joinToken.join_token.authority_signature
|
||||
approval_authority = $approved.node_bootstrap.cluster_authority
|
||||
identity_authority_fingerprint = $identity.cluster_authority_fingerprint
|
||||
synthetic_config_schema = $syntheticConfig.schema_version
|
||||
synthetic_config_version = $syntheticConfig.config_version
|
||||
synthetic_config_authority = $syntheticConfig.cluster_authority
|
||||
latest_heartbeat = $heartbeat
|
||||
cluster_summaries = $summaries.cluster_summaries
|
||||
pass_matrix = $passMatrix
|
||||
backend_log_tail = $backendLogs
|
||||
node_log_tail = $nodeLogs
|
||||
containers_left_running = [bool]$KeepRunning
|
||||
}
|
||||
|
||||
$failed = @($passMatrix.GetEnumerator() | Where-Object { -not $_.Value })
|
||||
$resultJson = $result | ConvertTo-Json -Depth 70
|
||||
|
||||
if ($ResultPath -ne "") {
|
||||
if ([System.IO.Path]::IsPathRooted($ResultPath)) {
|
||||
$resultFullPath = $ResultPath
|
||||
} else {
|
||||
$resultFullPath = Join-Path $repoRoot $ResultPath
|
||||
}
|
||||
New-Item -ItemType Directory -Force -Path (Split-Path $resultFullPath) | Out-Null
|
||||
Set-Content -Path $resultFullPath -Value $resultJson -Encoding UTF8
|
||||
Write-Host "Result written to $resultFullPath"
|
||||
}
|
||||
|
||||
$resultJson
|
||||
|
||||
if ($failed.Count -gt 0) {
|
||||
throw "Dev enrollment/bootstrap smoke failed: $($failed.Name -join ', ')"
|
||||
}
|
||||
|
||||
Invoke-RemoteShell -Command "rm -rf '$remoteBuildDir'"
|
||||
|
||||
if (-not $KeepRunning) {
|
||||
Write-Host "Cleaning up dev enrollment/bootstrap containers..."
|
||||
Remove-SmokeContainers
|
||||
}
|
||||
Reference in New Issue
Block a user