391 lines
14 KiB
PowerShell
391 lines
14 KiB
PowerShell
param(
|
|
[string]$DockerSshAlias = "test-docker",
|
|
[string]$NodeAgentImageTag = "rap-node-agent:c17h-synthetic-smoke",
|
|
[string]$AdminEmail = "fabric-owner-c17h@example.local",
|
|
[string]$AdminPassword = "SmokePass!123",
|
|
[int]$ApiPort = 18080,
|
|
[int]$MeshBasePort = 19081,
|
|
[switch]$KeepRunning
|
|
)
|
|
|
|
Set-StrictMode -Version Latest
|
|
$ErrorActionPreference = "Stop"
|
|
|
|
$backendPublicBaseUrl = "http://192.168.200.61:$ApiPort/api/v1"
|
|
$backendContainerBaseUrl = "http://127.0.0.1:$ApiPort/api/v1"
|
|
$runId = "c17h-ssh-" + (Get-Date -Format "yyyyMMdd-HHmmss")
|
|
$nodePrefix = "rap_c17h_node_"
|
|
|
|
function Invoke-RemoteDocker {
|
|
param([string[]]$Arguments)
|
|
& ssh $DockerSshAlias docker @Arguments
|
|
if ($LASTEXITCODE -ne 0) {
|
|
throw "ssh $DockerSshAlias docker $($Arguments -join ' ') failed with exit code $LASTEXITCODE"
|
|
}
|
|
}
|
|
|
|
function Invoke-RemoteDockerText {
|
|
param([string[]]$Arguments)
|
|
$previousErrorActionPreference = $ErrorActionPreference
|
|
$ErrorActionPreference = "Continue"
|
|
try {
|
|
$output = & ssh $DockerSshAlias docker @Arguments 2>&1 | ForEach-Object { $_.ToString() }
|
|
}
|
|
finally {
|
|
$ErrorActionPreference = $previousErrorActionPreference
|
|
}
|
|
if ($LASTEXITCODE -ne 0) {
|
|
throw "ssh $DockerSshAlias docker $($Arguments -join ' ') failed with exit code $LASTEXITCODE"
|
|
}
|
|
return $output
|
|
}
|
|
|
|
function Invoke-RemoteShell {
|
|
param([string]$Command)
|
|
& ssh $DockerSshAlias $Command
|
|
if ($LASTEXITCODE -ne 0) {
|
|
throw "ssh $DockerSshAlias $Command failed with exit code $LASTEXITCODE"
|
|
}
|
|
}
|
|
|
|
function Send-RemoteFile {
|
|
param(
|
|
[string]$RemotePath,
|
|
[string]$Content
|
|
)
|
|
$Content | & ssh $DockerSshAlias "cat > '$RemotePath'"
|
|
if ($LASTEXITCODE -ne 0) {
|
|
throw "write remote file failed: $RemotePath"
|
|
}
|
|
}
|
|
|
|
function Invoke-Api {
|
|
param(
|
|
[string]$Method,
|
|
[string]$Path,
|
|
[object]$Body = $null
|
|
)
|
|
$uri = "$backendPublicBaseUrl$Path"
|
|
if ($null -eq $Body) {
|
|
return Invoke-RestMethod -Method $Method -Uri $uri -TimeoutSec 30
|
|
}
|
|
return Invoke-RestMethod -Method $Method -Uri $uri -ContentType "application/json" -Body ($Body | ConvertTo-Json -Depth 40) -TimeoutSec 30
|
|
}
|
|
|
|
function Wait-HttpReady {
|
|
param([string]$Url)
|
|
for ($i = 0; $i -lt 60; $i++) {
|
|
try {
|
|
$response = Invoke-WebRequest -UseBasicParsing -Uri $Url -TimeoutSec 2
|
|
if ($response.StatusCode -ge 200 -and $response.StatusCode -lt 300) {
|
|
return
|
|
}
|
|
}
|
|
catch {
|
|
Start-Sleep -Seconds 1
|
|
}
|
|
}
|
|
throw "Timed out waiting for $Url"
|
|
}
|
|
|
|
function Remove-NodeContainers {
|
|
foreach ($key in @("a", "b", "c", "r", "idle")) {
|
|
& ssh $DockerSshAlias docker rm -f "$nodePrefix$key" 2>$null | Out-Null
|
|
}
|
|
}
|
|
|
|
Wait-HttpReady -Url "http://192.168.200.61:$ApiPort/readyz"
|
|
|
|
$login = Invoke-Api -Method Post -Path "/auth/login" -Body @{
|
|
email = $AdminEmail
|
|
password = $AdminPassword
|
|
device_fingerprint = "c17h-ssh-smoke-device"
|
|
device_label = "C17H SSH synthetic smoke"
|
|
trust_device = $true
|
|
}
|
|
$actorUserID = $login.user.id
|
|
|
|
$cluster = Invoke-Api -Method Post -Path "/clusters/" -Body @{
|
|
actor_user_id = $actorUserID
|
|
slug = "c17h-ssh-$((New-Guid).Guid.Substring(0, 8))"
|
|
name = "C17H SSH Synthetic Mesh Smoke"
|
|
region = "docker-test"
|
|
metadata = @{
|
|
stage = "c17h"
|
|
run_id = $runId
|
|
production_forwarding = $false
|
|
created_by = "c17h-multi-agent-synthetic-smoke-ssh.ps1"
|
|
}
|
|
}
|
|
$clusterID = $cluster.cluster.id
|
|
|
|
Invoke-Api -Method Put -Path "/fabric/testing-flags" -Body @{
|
|
actor_user_id = $actorUserID
|
|
scope_type = "platform"
|
|
scope_id = $null
|
|
cluster_id = $null
|
|
enabled = $true
|
|
telemetry_enabled = $true
|
|
synthetic_links_enabled = $true
|
|
history_retention_hours = 24
|
|
metadata = @{
|
|
stage = "c17h"
|
|
run_id = $runId
|
|
production_forwarding = $false
|
|
service_workload_traffic = $false
|
|
}
|
|
} | Out-Null
|
|
|
|
$joinToken = Invoke-Api -Method Post -Path "/clusters/$clusterID/join-tokens" -Body @{
|
|
actor_user_id = $actorUserID
|
|
scope = @{ purpose = "c17h-synthetic-smoke"; roles = @("core-mesh", "relay-node") }
|
|
expires_at = (Get-Date).ToUniversalTime().AddHours(2).ToString("o")
|
|
max_uses = 5
|
|
}
|
|
|
|
$nodeSpecs = @(
|
|
@{ key = "a"; name = "c17h-node-a"; roles = @("core-mesh"); port = $MeshBasePort },
|
|
@{ key = "r"; name = "c17h-node-r"; roles = @("core-mesh", "relay-node"); port = ($MeshBasePort + 1) },
|
|
@{ key = "b"; name = "c17h-node-b"; roles = @("core-mesh"); port = ($MeshBasePort + 2) },
|
|
@{ key = "c"; name = "c17h-node-c"; roles = @("core-mesh"); port = ($MeshBasePort + 3) },
|
|
@{ key = "idle"; name = "c17h-node-idle"; roles = @("core-mesh"); port = ($MeshBasePort + 4) }
|
|
)
|
|
|
|
$nodes = @{}
|
|
foreach ($spec in $nodeSpecs) {
|
|
$fingerprint = "c17h-fp-$($spec.key)-$([guid]::NewGuid().ToString('N'))"
|
|
$publicKey = "c17h-pub-$($spec.key)-$([guid]::NewGuid().ToString('N'))"
|
|
$joinRequest = Invoke-Api -Method Post -Path "/clusters/$clusterID/join-requests" -Body @{
|
|
join_token = $joinToken.join_token.token
|
|
node_name = $spec.name
|
|
node_fingerprint = $fingerprint
|
|
public_key = $publicKey
|
|
reported_capabilities = @{
|
|
can_accept_node_ingress = $true
|
|
can_route_mesh = $true
|
|
testing_node = $true
|
|
}
|
|
reported_facts = @{
|
|
os = "linux"
|
|
runtime = "docker-test"
|
|
stage = "c17h"
|
|
run_id = $runId
|
|
}
|
|
requested_roles = $spec.roles
|
|
}
|
|
$approved = Invoke-Api -Method Post -Path "/clusters/$clusterID/join-requests/$($joinRequest.join_request.id)/approve" -Body @{
|
|
actor_user_id = $actorUserID
|
|
node_key = $fingerprint
|
|
ownership_type = "platform_managed"
|
|
owner_organization_id = $null
|
|
}
|
|
$nodeID = $approved.node_bootstrap.node_id
|
|
foreach ($role in $spec.roles) {
|
|
Invoke-Api -Method Post -Path "/clusters/$clusterID/nodes/$nodeID/roles" -Body @{
|
|
actor_user_id = $actorUserID
|
|
role = $role
|
|
status = "active"
|
|
policy = @{
|
|
stage = "c17h"
|
|
run_id = $runId
|
|
synthetic_only = $true
|
|
}
|
|
} | Out-Null
|
|
}
|
|
$nodes[$spec.key] = [pscustomobject]@{
|
|
id = $nodeID
|
|
name = $spec.name
|
|
fingerprint = $fingerprint
|
|
public_key = $publicKey
|
|
port = $spec.port
|
|
roles = $spec.roles
|
|
}
|
|
}
|
|
|
|
$nodeAID = $nodes["a"].id
|
|
$nodeRID = $nodes["r"].id
|
|
$nodeBID = $nodes["b"].id
|
|
$nodeCID = $nodes["c"].id
|
|
$routeExpiresAt = (Get-Date).ToUniversalTime().AddHours(2).ToString("o")
|
|
|
|
$peerEndpointsDirect = @{}
|
|
$peerEndpointsDirect[$nodeAID] = "http://127.0.0.1:$($nodes["a"].port)"
|
|
$peerEndpointsDirect[$nodeBID] = "http://127.0.0.1:$($nodes["b"].port)"
|
|
|
|
$peerEndpointsRelay = @{}
|
|
$peerEndpointsRelay[$nodeAID] = "http://127.0.0.1:$($nodes["a"].port)"
|
|
$peerEndpointsRelay[$nodeRID] = "http://127.0.0.1:$($nodes["r"].port)"
|
|
$peerEndpointsRelay[$nodeCID] = "http://127.0.0.1:$($nodes["c"].port)"
|
|
|
|
$directIntent = Invoke-Api -Method Post -Path "/clusters/$clusterID/mesh/route-intents" -Body @{
|
|
actor_user_id = $actorUserID
|
|
source_selector = @{ node_id = $nodeAID }
|
|
destination_selector = @{ node_id = $nodeBID }
|
|
service_class = "control"
|
|
priority = 10
|
|
policy = @{
|
|
synthetic_enabled = $true
|
|
peer_endpoints = $peerEndpointsDirect
|
|
hops = @($nodeAID, $nodeBID)
|
|
allowed_channels = @("fabric_control", "route_control")
|
|
max_ttl = 4
|
|
max_hops = 4
|
|
expires_at = $routeExpiresAt
|
|
route_version = "$runId-direct"
|
|
policy_version = "$runId-policy"
|
|
peer_directory_version = "$runId-peers"
|
|
production_forwarding = $false
|
|
}
|
|
}
|
|
|
|
$relayIntent = Invoke-Api -Method Post -Path "/clusters/$clusterID/mesh/route-intents" -Body @{
|
|
actor_user_id = $actorUserID
|
|
source_selector = @{ node_id = $nodeAID }
|
|
destination_selector = @{ node_id = $nodeCID }
|
|
service_class = "control"
|
|
priority = 20
|
|
policy = @{
|
|
synthetic_enabled = $true
|
|
peer_endpoints = $peerEndpointsRelay
|
|
hops = @($nodeAID, $nodeRID, $nodeCID)
|
|
allowed_channels = @("fabric_control", "route_control")
|
|
max_ttl = 6
|
|
max_hops = 6
|
|
expires_at = $routeExpiresAt
|
|
route_version = "$runId-relay"
|
|
policy_version = "$runId-policy"
|
|
peer_directory_version = "$runId-peers"
|
|
production_forwarding = $false
|
|
}
|
|
}
|
|
|
|
$configs = @{}
|
|
foreach ($key in @("a", "r", "b", "c", "idle")) {
|
|
$configs[$key] = Invoke-Api -Method Get -Path "/clusters/$clusterID/nodes/$($nodes[$key].id)/mesh/synthetic-config"
|
|
}
|
|
|
|
Remove-NodeContainers
|
|
|
|
foreach ($key in @("a", "r", "b", "c", "idle")) {
|
|
$node = $nodes[$key]
|
|
$containerName = "$nodePrefix$key"
|
|
$remoteStateDir = "/tmp/$runId-$key"
|
|
Invoke-RemoteShell -Command "rm -rf '$remoteStateDir' && mkdir -p '$remoteStateDir'"
|
|
$identity = @{
|
|
node_id = $node.id
|
|
cluster_id = $clusterID
|
|
node_name = $node.name
|
|
node_fingerprint = $node.fingerprint
|
|
public_key = $node.public_key
|
|
identity_status = "active"
|
|
created_at = (Get-Date).ToUniversalTime().ToString("o")
|
|
updated_at = (Get-Date).ToUniversalTime().ToString("o")
|
|
} | ConvertTo-Json -Depth 10
|
|
Send-RemoteFile -RemotePath "$remoteStateDir/identity.json" -Content $identity
|
|
Invoke-RemoteDocker -Arguments @(
|
|
"create",
|
|
"--name", $containerName,
|
|
"--network", "host",
|
|
"-e", "RAP_BACKEND_URL=$backendContainerBaseUrl",
|
|
"-e", "RAP_NODE_STATE_DIR=/tmp/state",
|
|
"-e", "RAP_HEARTBEAT_INTERVAL_SECONDS=5",
|
|
"-e", "RAP_MESH_SYNTHETIC_RUNTIME_ENABLED=true",
|
|
"-e", "RAP_MESH_LISTEN_ADDR=0.0.0.0:$($node.port)",
|
|
$NodeAgentImageTag,
|
|
"-backend-url", $backendContainerBaseUrl,
|
|
"-state-dir", "/tmp/state",
|
|
"-heartbeat-interval", "5s",
|
|
"-mesh-synthetic-runtime-enabled",
|
|
"-mesh-listen-addr", "0.0.0.0:$($node.port)"
|
|
) | Out-Null
|
|
Invoke-RemoteDocker -Arguments @("cp", "$remoteStateDir/.", "$containerName`:/tmp/state")
|
|
Invoke-RemoteDocker -Arguments @("start", $containerName) | Out-Null
|
|
}
|
|
|
|
Start-Sleep -Seconds 25
|
|
|
|
$links = Invoke-Api -Method Get -Path "/clusters/$clusterID/mesh/links?actor_user_id=$actorUserID"
|
|
$summary = Invoke-Api -Method Get -Path "/cluster-admin-summaries?actor_user_id=$actorUserID"
|
|
|
|
$nodeLogs = @{}
|
|
foreach ($key in @("a", "r", "b", "c", "idle")) {
|
|
$nodeLogs[$key] = Invoke-RemoteDockerText -Arguments @("logs", "$nodePrefix$key")
|
|
}
|
|
$backendLogs = Invoke-RemoteDockerText -Arguments @("logs", "--tail", "50", "rap_c17h_backend")
|
|
|
|
$meshLinks = @($links.mesh_links)
|
|
$routeHealthLinks = @($meshLinks | Where-Object {
|
|
$_.metadata.observation_type -eq "synthetic_route_health" -and
|
|
$_.metadata.config_source -eq "control_plane"
|
|
})
|
|
$directHealth = @($routeHealthLinks | Where-Object { $_.metadata.route_id -eq $directIntent.route_intent.id -and $_.link_status -eq "reachable" })
|
|
$relayHealth = @($routeHealthLinks | Where-Object { $_.metadata.route_id -eq $relayIntent.route_intent.id -and $_.link_status -eq "reachable" })
|
|
|
|
$passMatrix = [ordered]@{
|
|
backend_ready = $true
|
|
platform_owner_login = [bool]$actorUserID
|
|
cluster_created = [bool]$clusterID
|
|
fabric_testing_flags_enabled = $true
|
|
node_a_scoped_config_enabled = $configs["a"].synthetic_mesh_config.enabled -eq $true
|
|
node_a_has_direct_and_relay_routes = @($configs["a"].synthetic_mesh_config.routes).Count -eq 2
|
|
relay_node_has_only_relay_route = @($configs["r"].synthetic_mesh_config.routes).Count -eq 1
|
|
direct_destination_node_has_only_direct_route = @($configs["b"].synthetic_mesh_config.routes).Count -eq 1
|
|
relay_destination_node_has_only_relay_route = @($configs["c"].synthetic_mesh_config.routes).Count -eq 1
|
|
idle_node_has_no_routes = @($configs["idle"].synthetic_mesh_config.routes).Count -eq 0
|
|
control_plane_config_used = (@("a", "r", "b", "c", "idle") | Where-Object { ($nodeLogs[$_] -join "`n") -match "source=control_plane" }).Count -eq 5
|
|
direct_route_health_reported = $directHealth.Count -gt 0
|
|
relay_route_health_reported = $relayHealth.Count -gt 0
|
|
production_forwarding_disabled = (
|
|
$configs["a"].synthetic_mesh_config.production_forwarding -eq $false -and
|
|
$configs["r"].synthetic_mesh_config.production_forwarding -eq $false -and
|
|
$configs["b"].synthetic_mesh_config.production_forwarding -eq $false -and
|
|
$configs["c"].synthetic_mesh_config.production_forwarding -eq $false -and
|
|
$configs["idle"].synthetic_mesh_config.production_forwarding -eq $false
|
|
)
|
|
}
|
|
|
|
$result = [pscustomobject]@{
|
|
stage = "C17H deployed multi-agent synthetic config smoke"
|
|
run_id = $runId
|
|
backend_base_url = $backendPublicBaseUrl
|
|
cluster_id = $clusterID
|
|
node_ids = @{
|
|
a = $nodes["a"].id
|
|
r = $nodes["r"].id
|
|
b = $nodes["b"].id
|
|
c = $nodes["c"].id
|
|
idle = $nodes["idle"].id
|
|
}
|
|
route_intents = @{
|
|
direct = $directIntent.route_intent.id
|
|
relay = $relayIntent.route_intent.id
|
|
}
|
|
scoped_config_route_counts = @{
|
|
a = @($configs["a"].synthetic_mesh_config.routes).Count
|
|
r = @($configs["r"].synthetic_mesh_config.routes).Count
|
|
b = @($configs["b"].synthetic_mesh_config.routes).Count
|
|
c = @($configs["c"].synthetic_mesh_config.routes).Count
|
|
idle = @($configs["idle"].synthetic_mesh_config.routes).Count
|
|
}
|
|
mesh_link_count = $meshLinks.Count
|
|
route_health_count = $routeHealthLinks.Count
|
|
pass_matrix = $passMatrix
|
|
direct_route_health = $directHealth | Select-Object -First 3
|
|
relay_route_health = $relayHealth | Select-Object -First 3
|
|
cluster_summaries = $summary.cluster_summaries
|
|
backend_log_tail = $backendLogs
|
|
node_log_tail = $nodeLogs
|
|
containers_left_running = [bool]$KeepRunning
|
|
}
|
|
|
|
$failed = @($passMatrix.GetEnumerator() | Where-Object { -not $_.Value })
|
|
$result | ConvertTo-Json -Depth 50
|
|
|
|
if ($failed.Count -gt 0) {
|
|
throw "C17H smoke failed: $($failed.Name -join ', ')"
|
|
}
|
|
|
|
if (-not $KeepRunning) {
|
|
Remove-NodeContainers
|
|
}
|