Initial project snapshot
This commit is contained in:
@@ -0,0 +1,390 @@
|
||||
param(
|
||||
[string]$DockerSshAlias = "test-docker",
|
||||
[string]$NodeAgentImageTag = "rap-node-agent:c17h-synthetic-smoke",
|
||||
[string]$AdminEmail = "fabric-owner-c17h@example.local",
|
||||
[string]$AdminPassword = "SmokePass!123",
|
||||
[int]$ApiPort = 18080,
|
||||
[int]$MeshBasePort = 19081,
|
||||
[switch]$KeepRunning
|
||||
)
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
$backendPublicBaseUrl = "http://192.168.200.61:$ApiPort/api/v1"
|
||||
$backendContainerBaseUrl = "http://127.0.0.1:$ApiPort/api/v1"
|
||||
$runId = "c17h-ssh-" + (Get-Date -Format "yyyyMMdd-HHmmss")
|
||||
$nodePrefix = "rap_c17h_node_"
|
||||
|
||||
function Invoke-RemoteDocker {
|
||||
param([string[]]$Arguments)
|
||||
& ssh $DockerSshAlias docker @Arguments
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "ssh $DockerSshAlias docker $($Arguments -join ' ') failed with exit code $LASTEXITCODE"
|
||||
}
|
||||
}
|
||||
|
||||
function Invoke-RemoteDockerText {
|
||||
param([string[]]$Arguments)
|
||||
$previousErrorActionPreference = $ErrorActionPreference
|
||||
$ErrorActionPreference = "Continue"
|
||||
try {
|
||||
$output = & ssh $DockerSshAlias docker @Arguments 2>&1 | ForEach-Object { $_.ToString() }
|
||||
}
|
||||
finally {
|
||||
$ErrorActionPreference = $previousErrorActionPreference
|
||||
}
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "ssh $DockerSshAlias docker $($Arguments -join ' ') failed with exit code $LASTEXITCODE"
|
||||
}
|
||||
return $output
|
||||
}
|
||||
|
||||
function Invoke-RemoteShell {
|
||||
param([string]$Command)
|
||||
& ssh $DockerSshAlias $Command
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "ssh $DockerSshAlias $Command failed with exit code $LASTEXITCODE"
|
||||
}
|
||||
}
|
||||
|
||||
function Send-RemoteFile {
|
||||
param(
|
||||
[string]$RemotePath,
|
||||
[string]$Content
|
||||
)
|
||||
$Content | & ssh $DockerSshAlias "cat > '$RemotePath'"
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "write remote file failed: $RemotePath"
|
||||
}
|
||||
}
|
||||
|
||||
function Invoke-Api {
|
||||
param(
|
||||
[string]$Method,
|
||||
[string]$Path,
|
||||
[object]$Body = $null
|
||||
)
|
||||
$uri = "$backendPublicBaseUrl$Path"
|
||||
if ($null -eq $Body) {
|
||||
return Invoke-RestMethod -Method $Method -Uri $uri -TimeoutSec 30
|
||||
}
|
||||
return Invoke-RestMethod -Method $Method -Uri $uri -ContentType "application/json" -Body ($Body | ConvertTo-Json -Depth 40) -TimeoutSec 30
|
||||
}
|
||||
|
||||
function Wait-HttpReady {
|
||||
param([string]$Url)
|
||||
for ($i = 0; $i -lt 60; $i++) {
|
||||
try {
|
||||
$response = Invoke-WebRequest -UseBasicParsing -Uri $Url -TimeoutSec 2
|
||||
if ($response.StatusCode -ge 200 -and $response.StatusCode -lt 300) {
|
||||
return
|
||||
}
|
||||
}
|
||||
catch {
|
||||
Start-Sleep -Seconds 1
|
||||
}
|
||||
}
|
||||
throw "Timed out waiting for $Url"
|
||||
}
|
||||
|
||||
function Remove-NodeContainers {
|
||||
foreach ($key in @("a", "b", "c", "r", "idle")) {
|
||||
& ssh $DockerSshAlias docker rm -f "$nodePrefix$key" 2>$null | Out-Null
|
||||
}
|
||||
}
|
||||
|
||||
Wait-HttpReady -Url "http://192.168.200.61:$ApiPort/readyz"
|
||||
|
||||
$login = Invoke-Api -Method Post -Path "/auth/login" -Body @{
|
||||
email = $AdminEmail
|
||||
password = $AdminPassword
|
||||
device_fingerprint = "c17h-ssh-smoke-device"
|
||||
device_label = "C17H SSH synthetic smoke"
|
||||
trust_device = $true
|
||||
}
|
||||
$actorUserID = $login.user.id
|
||||
|
||||
$cluster = Invoke-Api -Method Post -Path "/clusters/" -Body @{
|
||||
actor_user_id = $actorUserID
|
||||
slug = "c17h-ssh-$((New-Guid).Guid.Substring(0, 8))"
|
||||
name = "C17H SSH Synthetic Mesh Smoke"
|
||||
region = "docker-test"
|
||||
metadata = @{
|
||||
stage = "c17h"
|
||||
run_id = $runId
|
||||
production_forwarding = $false
|
||||
created_by = "c17h-multi-agent-synthetic-smoke-ssh.ps1"
|
||||
}
|
||||
}
|
||||
$clusterID = $cluster.cluster.id
|
||||
|
||||
Invoke-Api -Method Put -Path "/fabric/testing-flags" -Body @{
|
||||
actor_user_id = $actorUserID
|
||||
scope_type = "platform"
|
||||
scope_id = $null
|
||||
cluster_id = $null
|
||||
enabled = $true
|
||||
telemetry_enabled = $true
|
||||
synthetic_links_enabled = $true
|
||||
history_retention_hours = 24
|
||||
metadata = @{
|
||||
stage = "c17h"
|
||||
run_id = $runId
|
||||
production_forwarding = $false
|
||||
service_workload_traffic = $false
|
||||
}
|
||||
} | Out-Null
|
||||
|
||||
$joinToken = Invoke-Api -Method Post -Path "/clusters/$clusterID/join-tokens" -Body @{
|
||||
actor_user_id = $actorUserID
|
||||
scope = @{ purpose = "c17h-synthetic-smoke"; roles = @("core-mesh", "relay-node") }
|
||||
expires_at = (Get-Date).ToUniversalTime().AddHours(2).ToString("o")
|
||||
max_uses = 5
|
||||
}
|
||||
|
||||
$nodeSpecs = @(
|
||||
@{ key = "a"; name = "c17h-node-a"; roles = @("core-mesh"); port = $MeshBasePort },
|
||||
@{ key = "r"; name = "c17h-node-r"; roles = @("core-mesh", "relay-node"); port = ($MeshBasePort + 1) },
|
||||
@{ key = "b"; name = "c17h-node-b"; roles = @("core-mesh"); port = ($MeshBasePort + 2) },
|
||||
@{ key = "c"; name = "c17h-node-c"; roles = @("core-mesh"); port = ($MeshBasePort + 3) },
|
||||
@{ key = "idle"; name = "c17h-node-idle"; roles = @("core-mesh"); port = ($MeshBasePort + 4) }
|
||||
)
|
||||
|
||||
$nodes = @{}
|
||||
foreach ($spec in $nodeSpecs) {
|
||||
$fingerprint = "c17h-fp-$($spec.key)-$([guid]::NewGuid().ToString('N'))"
|
||||
$publicKey = "c17h-pub-$($spec.key)-$([guid]::NewGuid().ToString('N'))"
|
||||
$joinRequest = Invoke-Api -Method Post -Path "/clusters/$clusterID/join-requests" -Body @{
|
||||
join_token = $joinToken.join_token.token
|
||||
node_name = $spec.name
|
||||
node_fingerprint = $fingerprint
|
||||
public_key = $publicKey
|
||||
reported_capabilities = @{
|
||||
can_accept_node_ingress = $true
|
||||
can_route_mesh = $true
|
||||
testing_node = $true
|
||||
}
|
||||
reported_facts = @{
|
||||
os = "linux"
|
||||
runtime = "docker-test"
|
||||
stage = "c17h"
|
||||
run_id = $runId
|
||||
}
|
||||
requested_roles = $spec.roles
|
||||
}
|
||||
$approved = Invoke-Api -Method Post -Path "/clusters/$clusterID/join-requests/$($joinRequest.join_request.id)/approve" -Body @{
|
||||
actor_user_id = $actorUserID
|
||||
node_key = $fingerprint
|
||||
ownership_type = "platform_managed"
|
||||
owner_organization_id = $null
|
||||
}
|
||||
$nodeID = $approved.node_bootstrap.node_id
|
||||
foreach ($role in $spec.roles) {
|
||||
Invoke-Api -Method Post -Path "/clusters/$clusterID/nodes/$nodeID/roles" -Body @{
|
||||
actor_user_id = $actorUserID
|
||||
role = $role
|
||||
status = "active"
|
||||
policy = @{
|
||||
stage = "c17h"
|
||||
run_id = $runId
|
||||
synthetic_only = $true
|
||||
}
|
||||
} | Out-Null
|
||||
}
|
||||
$nodes[$spec.key] = [pscustomobject]@{
|
||||
id = $nodeID
|
||||
name = $spec.name
|
||||
fingerprint = $fingerprint
|
||||
public_key = $publicKey
|
||||
port = $spec.port
|
||||
roles = $spec.roles
|
||||
}
|
||||
}
|
||||
|
||||
$nodeAID = $nodes["a"].id
|
||||
$nodeRID = $nodes["r"].id
|
||||
$nodeBID = $nodes["b"].id
|
||||
$nodeCID = $nodes["c"].id
|
||||
$routeExpiresAt = (Get-Date).ToUniversalTime().AddHours(2).ToString("o")
|
||||
|
||||
$peerEndpointsDirect = @{}
|
||||
$peerEndpointsDirect[$nodeAID] = "http://127.0.0.1:$($nodes["a"].port)"
|
||||
$peerEndpointsDirect[$nodeBID] = "http://127.0.0.1:$($nodes["b"].port)"
|
||||
|
||||
$peerEndpointsRelay = @{}
|
||||
$peerEndpointsRelay[$nodeAID] = "http://127.0.0.1:$($nodes["a"].port)"
|
||||
$peerEndpointsRelay[$nodeRID] = "http://127.0.0.1:$($nodes["r"].port)"
|
||||
$peerEndpointsRelay[$nodeCID] = "http://127.0.0.1:$($nodes["c"].port)"
|
||||
|
||||
$directIntent = Invoke-Api -Method Post -Path "/clusters/$clusterID/mesh/route-intents" -Body @{
|
||||
actor_user_id = $actorUserID
|
||||
source_selector = @{ node_id = $nodeAID }
|
||||
destination_selector = @{ node_id = $nodeBID }
|
||||
service_class = "control"
|
||||
priority = 10
|
||||
policy = @{
|
||||
synthetic_enabled = $true
|
||||
peer_endpoints = $peerEndpointsDirect
|
||||
hops = @($nodeAID, $nodeBID)
|
||||
allowed_channels = @("fabric_control", "route_control")
|
||||
max_ttl = 4
|
||||
max_hops = 4
|
||||
expires_at = $routeExpiresAt
|
||||
route_version = "$runId-direct"
|
||||
policy_version = "$runId-policy"
|
||||
peer_directory_version = "$runId-peers"
|
||||
production_forwarding = $false
|
||||
}
|
||||
}
|
||||
|
||||
$relayIntent = Invoke-Api -Method Post -Path "/clusters/$clusterID/mesh/route-intents" -Body @{
|
||||
actor_user_id = $actorUserID
|
||||
source_selector = @{ node_id = $nodeAID }
|
||||
destination_selector = @{ node_id = $nodeCID }
|
||||
service_class = "control"
|
||||
priority = 20
|
||||
policy = @{
|
||||
synthetic_enabled = $true
|
||||
peer_endpoints = $peerEndpointsRelay
|
||||
hops = @($nodeAID, $nodeRID, $nodeCID)
|
||||
allowed_channels = @("fabric_control", "route_control")
|
||||
max_ttl = 6
|
||||
max_hops = 6
|
||||
expires_at = $routeExpiresAt
|
||||
route_version = "$runId-relay"
|
||||
policy_version = "$runId-policy"
|
||||
peer_directory_version = "$runId-peers"
|
||||
production_forwarding = $false
|
||||
}
|
||||
}
|
||||
|
||||
$configs = @{}
|
||||
foreach ($key in @("a", "r", "b", "c", "idle")) {
|
||||
$configs[$key] = Invoke-Api -Method Get -Path "/clusters/$clusterID/nodes/$($nodes[$key].id)/mesh/synthetic-config"
|
||||
}
|
||||
|
||||
Remove-NodeContainers
|
||||
|
||||
foreach ($key in @("a", "r", "b", "c", "idle")) {
|
||||
$node = $nodes[$key]
|
||||
$containerName = "$nodePrefix$key"
|
||||
$remoteStateDir = "/tmp/$runId-$key"
|
||||
Invoke-RemoteShell -Command "rm -rf '$remoteStateDir' && mkdir -p '$remoteStateDir'"
|
||||
$identity = @{
|
||||
node_id = $node.id
|
||||
cluster_id = $clusterID
|
||||
node_name = $node.name
|
||||
node_fingerprint = $node.fingerprint
|
||||
public_key = $node.public_key
|
||||
identity_status = "active"
|
||||
created_at = (Get-Date).ToUniversalTime().ToString("o")
|
||||
updated_at = (Get-Date).ToUniversalTime().ToString("o")
|
||||
} | ConvertTo-Json -Depth 10
|
||||
Send-RemoteFile -RemotePath "$remoteStateDir/identity.json" -Content $identity
|
||||
Invoke-RemoteDocker -Arguments @(
|
||||
"create",
|
||||
"--name", $containerName,
|
||||
"--network", "host",
|
||||
"-e", "RAP_BACKEND_URL=$backendContainerBaseUrl",
|
||||
"-e", "RAP_NODE_STATE_DIR=/tmp/state",
|
||||
"-e", "RAP_HEARTBEAT_INTERVAL_SECONDS=5",
|
||||
"-e", "RAP_MESH_SYNTHETIC_RUNTIME_ENABLED=true",
|
||||
"-e", "RAP_MESH_LISTEN_ADDR=0.0.0.0:$($node.port)",
|
||||
$NodeAgentImageTag,
|
||||
"-backend-url", $backendContainerBaseUrl,
|
||||
"-state-dir", "/tmp/state",
|
||||
"-heartbeat-interval", "5s",
|
||||
"-mesh-synthetic-runtime-enabled",
|
||||
"-mesh-listen-addr", "0.0.0.0:$($node.port)"
|
||||
) | Out-Null
|
||||
Invoke-RemoteDocker -Arguments @("cp", "$remoteStateDir/.", "$containerName`:/tmp/state")
|
||||
Invoke-RemoteDocker -Arguments @("start", $containerName) | Out-Null
|
||||
}
|
||||
|
||||
Start-Sleep -Seconds 25
|
||||
|
||||
$links = Invoke-Api -Method Get -Path "/clusters/$clusterID/mesh/links?actor_user_id=$actorUserID"
|
||||
$summary = Invoke-Api -Method Get -Path "/cluster-admin-summaries?actor_user_id=$actorUserID"
|
||||
|
||||
$nodeLogs = @{}
|
||||
foreach ($key in @("a", "r", "b", "c", "idle")) {
|
||||
$nodeLogs[$key] = Invoke-RemoteDockerText -Arguments @("logs", "$nodePrefix$key")
|
||||
}
|
||||
$backendLogs = Invoke-RemoteDockerText -Arguments @("logs", "--tail", "50", "rap_c17h_backend")
|
||||
|
||||
$meshLinks = @($links.mesh_links)
|
||||
$routeHealthLinks = @($meshLinks | Where-Object {
|
||||
$_.metadata.observation_type -eq "synthetic_route_health" -and
|
||||
$_.metadata.config_source -eq "control_plane"
|
||||
})
|
||||
$directHealth = @($routeHealthLinks | Where-Object { $_.metadata.route_id -eq $directIntent.route_intent.id -and $_.link_status -eq "reachable" })
|
||||
$relayHealth = @($routeHealthLinks | Where-Object { $_.metadata.route_id -eq $relayIntent.route_intent.id -and $_.link_status -eq "reachable" })
|
||||
|
||||
$passMatrix = [ordered]@{
|
||||
backend_ready = $true
|
||||
platform_owner_login = [bool]$actorUserID
|
||||
cluster_created = [bool]$clusterID
|
||||
fabric_testing_flags_enabled = $true
|
||||
node_a_scoped_config_enabled = $configs["a"].synthetic_mesh_config.enabled -eq $true
|
||||
node_a_has_direct_and_relay_routes = @($configs["a"].synthetic_mesh_config.routes).Count -eq 2
|
||||
relay_node_has_only_relay_route = @($configs["r"].synthetic_mesh_config.routes).Count -eq 1
|
||||
direct_destination_node_has_only_direct_route = @($configs["b"].synthetic_mesh_config.routes).Count -eq 1
|
||||
relay_destination_node_has_only_relay_route = @($configs["c"].synthetic_mesh_config.routes).Count -eq 1
|
||||
idle_node_has_no_routes = @($configs["idle"].synthetic_mesh_config.routes).Count -eq 0
|
||||
control_plane_config_used = (@("a", "r", "b", "c", "idle") | Where-Object { ($nodeLogs[$_] -join "`n") -match "source=control_plane" }).Count -eq 5
|
||||
direct_route_health_reported = $directHealth.Count -gt 0
|
||||
relay_route_health_reported = $relayHealth.Count -gt 0
|
||||
production_forwarding_disabled = (
|
||||
$configs["a"].synthetic_mesh_config.production_forwarding -eq $false -and
|
||||
$configs["r"].synthetic_mesh_config.production_forwarding -eq $false -and
|
||||
$configs["b"].synthetic_mesh_config.production_forwarding -eq $false -and
|
||||
$configs["c"].synthetic_mesh_config.production_forwarding -eq $false -and
|
||||
$configs["idle"].synthetic_mesh_config.production_forwarding -eq $false
|
||||
)
|
||||
}
|
||||
|
||||
$result = [pscustomobject]@{
|
||||
stage = "C17H deployed multi-agent synthetic config smoke"
|
||||
run_id = $runId
|
||||
backend_base_url = $backendPublicBaseUrl
|
||||
cluster_id = $clusterID
|
||||
node_ids = @{
|
||||
a = $nodes["a"].id
|
||||
r = $nodes["r"].id
|
||||
b = $nodes["b"].id
|
||||
c = $nodes["c"].id
|
||||
idle = $nodes["idle"].id
|
||||
}
|
||||
route_intents = @{
|
||||
direct = $directIntent.route_intent.id
|
||||
relay = $relayIntent.route_intent.id
|
||||
}
|
||||
scoped_config_route_counts = @{
|
||||
a = @($configs["a"].synthetic_mesh_config.routes).Count
|
||||
r = @($configs["r"].synthetic_mesh_config.routes).Count
|
||||
b = @($configs["b"].synthetic_mesh_config.routes).Count
|
||||
c = @($configs["c"].synthetic_mesh_config.routes).Count
|
||||
idle = @($configs["idle"].synthetic_mesh_config.routes).Count
|
||||
}
|
||||
mesh_link_count = $meshLinks.Count
|
||||
route_health_count = $routeHealthLinks.Count
|
||||
pass_matrix = $passMatrix
|
||||
direct_route_health = $directHealth | Select-Object -First 3
|
||||
relay_route_health = $relayHealth | Select-Object -First 3
|
||||
cluster_summaries = $summary.cluster_summaries
|
||||
backend_log_tail = $backendLogs
|
||||
node_log_tail = $nodeLogs
|
||||
containers_left_running = [bool]$KeepRunning
|
||||
}
|
||||
|
||||
$failed = @($passMatrix.GetEnumerator() | Where-Object { -not $_.Value })
|
||||
$result | ConvertTo-Json -Depth 50
|
||||
|
||||
if ($failed.Count -gt 0) {
|
||||
throw "C17H smoke failed: $($failed.Name -join ', ')"
|
||||
}
|
||||
|
||||
if (-not $KeepRunning) {
|
||||
Remove-NodeContainers
|
||||
}
|
||||
Reference in New Issue
Block a user