param( [int]$PerPair = 20, [int]$TimeoutSeconds = 20, [string]$AgentDir = "agents\rap-node-agent", [string]$DockerHost = "test-docker" ) $ErrorActionPreference = "Stop" $repoRoot = (Resolve-Path (Join-Path $PSScriptRoot "..\..")).ProviderPath $agentRoot = Join-Path $repoRoot $AgentDir $exePath = Join-Path $agentRoot "tmp\fabric-production-smoke.exe" New-Item -ItemType Directory -Force (Split-Path $exePath) | Out-Null Push-Location $agentRoot try { go build -o $exePath ./cmd/fabric-production-smoke } finally { Pop-Location } $clusterID = "cfc0743d-d960-49fb-9de8-96e063d5e4aa" $pairs = @( @{ name = "home-2-to-home-3" srcName = "home-2" route = "a2c2e529-05e6-4e26-9b9e-0ca4f135cbbf" src = "a6777ebe-44b0-4f4f-95ad-d6bd7caceb8e" dst = "fab50dc4-ce2f-4f53-a3c3-2fa210530baa" path = "a6777ebe-44b0-4f4f-95ad-d6bd7caceb8e,fab50dc4-ce2f-4f53-a3c3-2fa210530baa" }, @{ name = "usa-los-1-to-ifcm" srcName = "usa-los-1" route = "e8a7a16e-be85-4129-baa3-70bd2d275aad" src = "b829ffde-690b-47ab-9522-0f22ab42596d" dst = "f3c95cb7-a189-4dbb-b5d7-5ff93ba9c040" path = "b829ffde-690b-47ab-9522-0f22ab42596d,f3c95cb7-a189-4dbb-b5d7-5ff93ba9c040" } ) function Invoke-FabricSqlJson { param([string]$Sql) $encoded = [Convert]::ToBase64String([Text.Encoding]::UTF8.GetBytes($Sql)) $output = ssh $DockerHost "printf '%s' '$encoded' | base64 -d | docker exec -i rap_test_postgres psql -U rap_user -d remote_access_platform -t -A -F ''" $json = ($output -join "`n").Trim() if ([string]::IsNullOrWhiteSpace($json)) { throw "SQL query returned no data" } return $json | ConvertFrom-Json } $nodeNames = (($pairs | ForEach-Object { $_.srcName }) | Sort-Object -Unique | ForEach-Object { "'$($_.Replace("'", "''"))'" }) -join "," $endpointRows = Invoke-FabricSqlJson @" select coalesce(json_agg(row_to_json(t)), '[]'::json) from ( select n.name, h.metadata #>> '{mesh_endpoint_report,peer_endpoint}' as endpoint, h.metadata #>> '{mesh_endpoint_report,endpoint_candidates,0,metadata,tls_cert_sha256}' as cert from nodes n join lateral ( select * from node_heartbeats h where h.node_id = n.id order by observed_at desc limit 1 ) h on true where n.name in ($nodeNames) ) t; "@ $endpointsByName = @{} foreach ($row in @($endpointRows)) { $endpointsByName[$row.name] = $row } foreach ($pair in $pairs) { $endpoint = $endpointsByName[$pair.srcName] if ($null -eq $endpoint -or [string]::IsNullOrWhiteSpace($endpoint.endpoint) -or [string]::IsNullOrWhiteSpace($endpoint.cert)) { throw "Missing live QUIC endpoint or certificate fingerprint for $($pair.srcName)" } $pair["endpoint"] = $endpoint.endpoint $pair["cert"] = $endpoint.cert } $jobs = @() foreach ($pair in $pairs) { for ($i = 0; $i -lt $PerPair; $i++) { $jobs += Start-Job -ScriptBlock { param($exePath, $clusterID, $pair, $index, $timeoutSeconds) $payload = (@{ kind = "fabric-live-production-burst" pair = $pair.name index = $index } | ConvertTo-Json -Compress) $payloadB64 = [Convert]::ToBase64String([Text.Encoding]::UTF8.GetBytes($payload)) $args = @( "-endpoint", $pair.endpoint, "-peer-cert-sha256", $pair.cert, "-cluster-id", $clusterID, "-route-id", $pair.route, "-source-node-id", $pair.src, "-destination-node-id", $pair.dst, "-current-hop-node-id", $pair.src, "-next-hop-node-id", $pair.dst, "-route-path", $pair.path, "-channel", "fabric_control", "-timeout", "$($timeoutSeconds)s", "-payload-b64", $payloadB64 ) $output = & $exePath @args 2>&1 [pscustomobject]@{ pair = $pair.name index = $index exit = $LASTEXITCODE output = ($output -join "`n") } } -ArgumentList $exePath, $clusterID, $pair, $i, $TimeoutSeconds } } $raw = $jobs | Wait-Job | Receive-Job $jobs | Remove-Job $results = foreach ($item in $raw) { $ok = $false $elapsed = $null $errorText = $null try { $json = $item.output | ConvertFrom-Json $ok = [bool]$json.ok $elapsed = [int64]$json.elapsed_ms $errorText = [string]$json.error } catch { $errorText = $item.output } [pscustomobject]@{ pair = $item.pair index = $item.index exit = $item.exit ok = $ok elapsed_ms = $elapsed error = $errorText } } $summary = $results | Group-Object pair | ForEach-Object { $items = @($_.Group) $latencies = @($items | Where-Object { $_.ok -and $null -ne $_.elapsed_ms } | ForEach-Object { $_.elapsed_ms } | Sort-Object) $p95 = $null $max = $null if ($latencies.Count -gt 0) { $p95 = $latencies[[Math]::Min($latencies.Count - 1, [int][Math]::Ceiling($latencies.Count * 0.95) - 1)] $max = $latencies[-1] } [pscustomobject]@{ pair = $_.Name total = $items.Count ok = @($items | Where-Object ok).Count failed = @($items | Where-Object { -not $_.ok }).Count p95_ms = $p95 max_ms = $max } } [pscustomobject]@{ schema_version = "rap.fabric_live_production_burst.v1" generated_at = (Get-Date).ToUniversalTime().ToString("o") total = $results.Count ok = @($results | Where-Object ok).Count failed = @($results | Where-Object { -not $_.ok }).Count summary = $summary failures = @($results | Where-Object { -not $_.ok } | Select-Object -First 10) } | ConvertTo-Json -Depth 6