Refactor RDP proxy handling and update related tests
This commit is contained in:
@@ -0,0 +1,112 @@
|
||||
param(
|
||||
[string]$ArtifactDir = "",
|
||||
[int]$Limit = 20,
|
||||
[string]$OutputPath = ""
|
||||
)
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
$repoRoot = (Resolve-Path (Join-Path $PSScriptRoot "..\..")).ProviderPath
|
||||
if ($ArtifactDir.Trim() -eq "") {
|
||||
$ArtifactDir = Join-Path $repoRoot "artifacts\fabric-loadtest"
|
||||
}
|
||||
if ($OutputPath.Trim() -eq "") {
|
||||
$OutputPath = Join-Path $ArtifactDir ("fabric-acceptance-summary-" + (Get-Date -Format "yyyyMMdd-HHmmss") + ".json")
|
||||
}
|
||||
|
||||
function Get-PropertyValue {
|
||||
param([object]$Item, [string]$Name, $Default = $null)
|
||||
if ($null -eq $Item -or -not ($Item.PSObject.Properties.Name -contains $Name)) {
|
||||
return $Default
|
||||
}
|
||||
return $Item.$Name
|
||||
}
|
||||
|
||||
function Convert-RouteModes {
|
||||
param([object]$TargetStats)
|
||||
$modes = [ordered]@{}
|
||||
if ($null -eq $TargetStats) {
|
||||
return $modes
|
||||
}
|
||||
foreach ($targetName in $TargetStats.PSObject.Properties.Name) {
|
||||
$stats = $TargetStats.$targetName
|
||||
$routeModes = Get-PropertyValue -Item $stats -Name "route_modes"
|
||||
if ($null -eq $routeModes) {
|
||||
continue
|
||||
}
|
||||
foreach ($mode in $routeModes.PSObject.Properties.Name) {
|
||||
if (-not $modes.Contains($mode)) {
|
||||
$modes[$mode] = 0
|
||||
}
|
||||
$modes[$mode] += [int]$routeModes.$mode
|
||||
}
|
||||
}
|
||||
return $modes
|
||||
}
|
||||
|
||||
function Convert-TargetDistribution {
|
||||
param([object]$TargetStreams)
|
||||
$out = [ordered]@{}
|
||||
if ($null -eq $TargetStreams) {
|
||||
return $out
|
||||
}
|
||||
foreach ($name in $TargetStreams.PSObject.Properties.Name) {
|
||||
$out[$name] = [int]$TargetStreams.$name
|
||||
}
|
||||
return $out
|
||||
}
|
||||
|
||||
$files = @(Get-ChildItem -LiteralPath $ArtifactDir -Filter "*-summary.json" -File | Sort-Object LastWriteTime -Descending | Select-Object -First $Limit)
|
||||
$runs = @()
|
||||
foreach ($file in $files) {
|
||||
try {
|
||||
$summary = Get-Content -LiteralPath $file.FullName -Raw | ConvertFrom-Json
|
||||
}
|
||||
catch {
|
||||
continue
|
||||
}
|
||||
$runs += [pscustomobject]@{
|
||||
run_id = Get-PropertyValue -Item $summary -Name "run_id" -Default $file.BaseName
|
||||
verdict = Get-PropertyValue -Item $summary -Name "verdict"
|
||||
verdict_reasons = Get-PropertyValue -Item $summary -Name "verdict_reasons"
|
||||
docker_context = Get-PropertyValue -Item $summary -Name "docker_context"
|
||||
topology_profile = Get-PropertyValue -Item $summary -Name "topology_profile"
|
||||
soak = [bool](Get-PropertyValue -Item $summary -Name "soak" -Default $false)
|
||||
total_streams = [int64](Get-PropertyValue -Item $summary -Name "total_streams" -Default 0)
|
||||
successful_streams = [int64](Get-PropertyValue -Item $summary -Name "successful_streams" -Default 0)
|
||||
failed_streams = [int64](Get-PropertyValue -Item $summary -Name "failed_streams" -Default 0)
|
||||
failover_events = [int64](Get-PropertyValue -Item $summary -Name "failover_events" -Default 0)
|
||||
migration_events = [int64](Get-PropertyValue -Item $summary -Name "migration_events" -Default 0)
|
||||
channel_opens = [int64](Get-PropertyValue -Item $summary -Name "channel_opens" -Default 0)
|
||||
channel_closes = [int64](Get-PropertyValue -Item $summary -Name "channel_closes" -Default 0)
|
||||
channel_leaks = [int64](Get-PropertyValue -Item $summary -Name "channel_leaks" -Default 0)
|
||||
throughput_bps = [int64](Get-PropertyValue -Item $summary -Name "throughput_bps" -Default 0)
|
||||
channel_churn_per_sec = [int64](Get-PropertyValue -Item $summary -Name "channel_churn_per_sec" -Default 0)
|
||||
ack_p95_ms = [int64](Get-PropertyValue -Item $summary -Name "ack_p95_ms" -Default 0)
|
||||
ack_p99_ms = [int64](Get-PropertyValue -Item $summary -Name "ack_p99_ms" -Default 0)
|
||||
setup_latency_p95_ms = [int64](Get-PropertyValue -Item $summary -Name "setup_latency_p95_ms" -Default 0)
|
||||
reroute_latency_p95_ms = [int64](Get-PropertyValue -Item $summary -Name "reroute_latency_p95_ms" -Default 0)
|
||||
route_modes = Convert-RouteModes -TargetStats (Get-PropertyValue -Item $summary -Name "target_stats")
|
||||
target_streams = Convert-TargetDistribution -TargetStreams (Get-PropertyValue -Item $summary -Name "target_streams")
|
||||
container_stats_samples_count = [int](Get-PropertyValue -Item $summary -Name "container_stats_samples_count" -Default 0)
|
||||
summary_path = $file.FullName
|
||||
}
|
||||
}
|
||||
|
||||
$failed = @($runs | Where-Object { $_.verdict -ne "pass" })
|
||||
$report = [pscustomobject]@{
|
||||
schema_version = "rap.fabric_acceptance_summary.v1"
|
||||
generated_at = (Get-Date).ToUniversalTime().ToString("o")
|
||||
artifact_dir = (Resolve-Path $ArtifactDir).ProviderPath
|
||||
runs_considered = $runs.Count
|
||||
pass_count = @($runs | Where-Object { $_.verdict -eq "pass" }).Count
|
||||
fail_count = $failed.Count
|
||||
all_considered_runs_passed = ($failed.Count -eq 0 -and $runs.Count -gt 0)
|
||||
runs = $runs
|
||||
}
|
||||
|
||||
New-Item -ItemType Directory -Force -Path (Split-Path -Parent $OutputPath) | Out-Null
|
||||
$json = $report | ConvertTo-Json -Depth 30
|
||||
Set-Content -LiteralPath $OutputPath -Value $json -Encoding UTF8
|
||||
$json
|
||||
@@ -0,0 +1,172 @@
|
||||
param(
|
||||
[int]$PerPair = 20,
|
||||
[int]$TimeoutSeconds = 20,
|
||||
[string]$AgentDir = "agents\rap-node-agent",
|
||||
[string]$DockerHost = "test-docker"
|
||||
)
|
||||
|
||||
$ErrorActionPreference = "Stop"
|
||||
$repoRoot = (Resolve-Path (Join-Path $PSScriptRoot "..\..")).ProviderPath
|
||||
$agentRoot = Join-Path $repoRoot $AgentDir
|
||||
$exePath = Join-Path $agentRoot "tmp\fabric-production-smoke.exe"
|
||||
New-Item -ItemType Directory -Force (Split-Path $exePath) | Out-Null
|
||||
|
||||
Push-Location $agentRoot
|
||||
try {
|
||||
go build -o $exePath ./cmd/fabric-production-smoke
|
||||
} finally {
|
||||
Pop-Location
|
||||
}
|
||||
|
||||
$clusterID = "cfc0743d-d960-49fb-9de8-96e063d5e4aa"
|
||||
$pairs = @(
|
||||
@{
|
||||
name = "home-2-to-home-3"
|
||||
srcName = "home-2"
|
||||
route = "a2c2e529-05e6-4e26-9b9e-0ca4f135cbbf"
|
||||
src = "a6777ebe-44b0-4f4f-95ad-d6bd7caceb8e"
|
||||
dst = "fab50dc4-ce2f-4f53-a3c3-2fa210530baa"
|
||||
path = "a6777ebe-44b0-4f4f-95ad-d6bd7caceb8e,fab50dc4-ce2f-4f53-a3c3-2fa210530baa"
|
||||
},
|
||||
@{
|
||||
name = "usa-los-1-to-ifcm"
|
||||
srcName = "usa-los-1"
|
||||
route = "e8a7a16e-be85-4129-baa3-70bd2d275aad"
|
||||
src = "b829ffde-690b-47ab-9522-0f22ab42596d"
|
||||
dst = "f3c95cb7-a189-4dbb-b5d7-5ff93ba9c040"
|
||||
path = "b829ffde-690b-47ab-9522-0f22ab42596d,f3c95cb7-a189-4dbb-b5d7-5ff93ba9c040"
|
||||
}
|
||||
)
|
||||
|
||||
function Invoke-FabricSqlJson {
|
||||
param([string]$Sql)
|
||||
|
||||
$encoded = [Convert]::ToBase64String([Text.Encoding]::UTF8.GetBytes($Sql))
|
||||
$output = ssh $DockerHost "printf '%s' '$encoded' | base64 -d | docker exec -i rap_test_postgres psql -U rap_user -d remote_access_platform -t -A -F ''"
|
||||
$json = ($output -join "`n").Trim()
|
||||
if ([string]::IsNullOrWhiteSpace($json)) {
|
||||
throw "SQL query returned no data"
|
||||
}
|
||||
return $json | ConvertFrom-Json
|
||||
}
|
||||
|
||||
$nodeNames = (($pairs | ForEach-Object { $_.srcName }) | Sort-Object -Unique | ForEach-Object { "'$($_.Replace("'", "''"))'" }) -join ","
|
||||
$endpointRows = Invoke-FabricSqlJson @"
|
||||
select coalesce(json_agg(row_to_json(t)), '[]'::json)
|
||||
from (
|
||||
select n.name,
|
||||
h.metadata #>> '{mesh_endpoint_report,peer_endpoint}' as endpoint,
|
||||
h.metadata #>> '{mesh_endpoint_report,endpoint_candidates,0,metadata,tls_cert_sha256}' as cert
|
||||
from nodes n
|
||||
join lateral (
|
||||
select *
|
||||
from node_heartbeats h
|
||||
where h.node_id = n.id
|
||||
order by observed_at desc
|
||||
limit 1
|
||||
) h on true
|
||||
where n.name in ($nodeNames)
|
||||
) t;
|
||||
"@
|
||||
$endpointsByName = @{}
|
||||
foreach ($row in @($endpointRows)) {
|
||||
$endpointsByName[$row.name] = $row
|
||||
}
|
||||
foreach ($pair in $pairs) {
|
||||
$endpoint = $endpointsByName[$pair.srcName]
|
||||
if ($null -eq $endpoint -or [string]::IsNullOrWhiteSpace($endpoint.endpoint) -or [string]::IsNullOrWhiteSpace($endpoint.cert)) {
|
||||
throw "Missing live QUIC endpoint or certificate fingerprint for $($pair.srcName)"
|
||||
}
|
||||
$pair["endpoint"] = $endpoint.endpoint
|
||||
$pair["cert"] = $endpoint.cert
|
||||
}
|
||||
|
||||
$jobs = @()
|
||||
foreach ($pair in $pairs) {
|
||||
for ($i = 0; $i -lt $PerPair; $i++) {
|
||||
$jobs += Start-Job -ScriptBlock {
|
||||
param($exePath, $clusterID, $pair, $index, $timeoutSeconds)
|
||||
$payload = (@{
|
||||
kind = "fabric-live-production-burst"
|
||||
pair = $pair.name
|
||||
index = $index
|
||||
} | ConvertTo-Json -Compress)
|
||||
$payloadB64 = [Convert]::ToBase64String([Text.Encoding]::UTF8.GetBytes($payload))
|
||||
$args = @(
|
||||
"-endpoint", $pair.endpoint,
|
||||
"-peer-cert-sha256", $pair.cert,
|
||||
"-cluster-id", $clusterID,
|
||||
"-route-id", $pair.route,
|
||||
"-source-node-id", $pair.src,
|
||||
"-destination-node-id", $pair.dst,
|
||||
"-current-hop-node-id", $pair.src,
|
||||
"-next-hop-node-id", $pair.dst,
|
||||
"-route-path", $pair.path,
|
||||
"-channel", "fabric_control",
|
||||
"-timeout", "$($timeoutSeconds)s",
|
||||
"-payload-b64", $payloadB64
|
||||
)
|
||||
$output = & $exePath @args 2>&1
|
||||
[pscustomobject]@{
|
||||
pair = $pair.name
|
||||
index = $index
|
||||
exit = $LASTEXITCODE
|
||||
output = ($output -join "`n")
|
||||
}
|
||||
} -ArgumentList $exePath, $clusterID, $pair, $i, $TimeoutSeconds
|
||||
}
|
||||
}
|
||||
|
||||
$raw = $jobs | Wait-Job | Receive-Job
|
||||
$jobs | Remove-Job
|
||||
|
||||
$results = foreach ($item in $raw) {
|
||||
$ok = $false
|
||||
$elapsed = $null
|
||||
$errorText = $null
|
||||
try {
|
||||
$json = $item.output | ConvertFrom-Json
|
||||
$ok = [bool]$json.ok
|
||||
$elapsed = [int64]$json.elapsed_ms
|
||||
$errorText = [string]$json.error
|
||||
} catch {
|
||||
$errorText = $item.output
|
||||
}
|
||||
[pscustomobject]@{
|
||||
pair = $item.pair
|
||||
index = $item.index
|
||||
exit = $item.exit
|
||||
ok = $ok
|
||||
elapsed_ms = $elapsed
|
||||
error = $errorText
|
||||
}
|
||||
}
|
||||
|
||||
$summary = $results | Group-Object pair | ForEach-Object {
|
||||
$items = @($_.Group)
|
||||
$latencies = @($items | Where-Object { $_.ok -and $null -ne $_.elapsed_ms } | ForEach-Object { $_.elapsed_ms } | Sort-Object)
|
||||
$p95 = $null
|
||||
$max = $null
|
||||
if ($latencies.Count -gt 0) {
|
||||
$p95 = $latencies[[Math]::Min($latencies.Count - 1, [int][Math]::Ceiling($latencies.Count * 0.95) - 1)]
|
||||
$max = $latencies[-1]
|
||||
}
|
||||
[pscustomobject]@{
|
||||
pair = $_.Name
|
||||
total = $items.Count
|
||||
ok = @($items | Where-Object ok).Count
|
||||
failed = @($items | Where-Object { -not $_.ok }).Count
|
||||
p95_ms = $p95
|
||||
max_ms = $max
|
||||
}
|
||||
}
|
||||
|
||||
[pscustomobject]@{
|
||||
schema_version = "rap.fabric_live_production_burst.v1"
|
||||
generated_at = (Get-Date).ToUniversalTime().ToString("o")
|
||||
total = $results.Count
|
||||
ok = @($results | Where-Object ok).Count
|
||||
failed = @($results | Where-Object { -not $_.ok }).Count
|
||||
summary = $summary
|
||||
failures = @($results | Where-Object { -not $_.ok } | Select-Object -First 10)
|
||||
} | ConvertTo-Json -Depth 6
|
||||
@@ -0,0 +1,839 @@
|
||||
param(
|
||||
[string]$DockerContext = "test-docker",
|
||||
[string]$ImageTag = "rap-fabric-loadtest:dev",
|
||||
[int]$Nodes = 4,
|
||||
[int]$Streams = 400,
|
||||
[int]$Concurrency = 64,
|
||||
[int64]$BytesPerStream = 262144,
|
||||
[string]$Duration = "",
|
||||
[string]$ClientTimeout = "10m",
|
||||
[string]$ServerTimeout = "",
|
||||
[string]$StreamTimeout = "30s",
|
||||
[string]$AckTimeout = "2s",
|
||||
[string]$TargetQuarantineTTL = "30s",
|
||||
[string]$FailureQuarantineTTL = "5m",
|
||||
[string]$TopologyProfile = "",
|
||||
[switch]$Soak,
|
||||
[int]$ControlEvery = 0,
|
||||
[int64]$ControlBytesPerStream = 4096,
|
||||
[int64]$MaxControlAckP95Ms = 100,
|
||||
[int]$MaxGoroutineDelta = 0,
|
||||
[int64]$MaxHeapDeltaMB = 0,
|
||||
[int]$MaxOpenFDDelta = 0,
|
||||
[int]$MaxOpenFDs = 0,
|
||||
[int64]$MinThroughputMbps = 0,
|
||||
[int64]$MinChannelChurnPerSec = 0,
|
||||
[int64]$MaxContainerMemoryMiB = 0,
|
||||
[int]$MaxContainerPids = 0,
|
||||
[int]$PayloadSize = 16384,
|
||||
[int]$FailTarget = 0,
|
||||
[string]$FailAfter = "1s",
|
||||
[int]$ImpairTarget = -1,
|
||||
[string]$ImpairDelay = "",
|
||||
[string]$ImpairLoss = "",
|
||||
[string]$ImpairRate = "",
|
||||
[string]$ImpairAfter = "",
|
||||
[switch]$ProbeTargets,
|
||||
[int64]$MaxTargetRttMs = 0,
|
||||
[switch]$MigrateSlowStreams,
|
||||
[int64]$MaxAckMs = 0,
|
||||
[int64]$MaxAckP95Ms = 0,
|
||||
[int64]$MaxAckP99Ms = 0,
|
||||
[int64]$MaxTargetAckMs = 0,
|
||||
[int64]$MaxSetupP95Ms = 200,
|
||||
[int64]$MaxSetupP99Ms = 0,
|
||||
[int64]$MaxRerouteP95Ms = 0,
|
||||
[int64]$MaxRerouteP99Ms = 0,
|
||||
[string]$ResourceSampleInterval = "1s",
|
||||
[string]$ContainerStatsSampleInterval = "",
|
||||
[string]$ContainerStatsSamplesPath = "",
|
||||
[int]$UdpBufferBytes = 7500000,
|
||||
[string]$ArtifactDir = "",
|
||||
[string]$ReportPath = "",
|
||||
[string]$SummaryPath = "",
|
||||
[switch]$TuneUdpBuffers,
|
||||
[switch]$KeepRunning,
|
||||
[switch]$SkipBuild
|
||||
)
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
$repoRoot = (Resolve-Path (Join-Path $PSScriptRoot "..\..")).ProviderPath
|
||||
$runId = "fabric-loadtest-" + (Get-Date -Format "yyyyMMdd-HHmmss")
|
||||
$networkName = "rap-$runId-net"
|
||||
$serverPrefix = "rap-$runId-server-"
|
||||
$clientName = "rap-$runId-client"
|
||||
if ($ArtifactDir.Trim() -eq "") {
|
||||
$ArtifactDir = Join-Path $repoRoot "artifacts\fabric-loadtest"
|
||||
}
|
||||
if ($ReportPath.Trim() -eq "") {
|
||||
$ReportPath = Join-Path $ArtifactDir "$runId-report.json"
|
||||
}
|
||||
if ($SummaryPath.Trim() -eq "") {
|
||||
$SummaryPath = Join-Path $ArtifactDir "$runId-summary.json"
|
||||
}
|
||||
if ($ContainerStatsSamplesPath.Trim() -eq "") {
|
||||
$ContainerStatsSamplesPath = Join-Path $ArtifactDir "$runId-container-stats-samples.json"
|
||||
}
|
||||
if ($ServerTimeout.Trim() -eq "") {
|
||||
$ServerTimeout = $ClientTimeout
|
||||
}
|
||||
|
||||
function Invoke-Docker {
|
||||
param([string[]]$Arguments)
|
||||
docker --context $DockerContext @Arguments
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "docker --context $DockerContext $($Arguments -join ' ') failed with exit code $LASTEXITCODE"
|
||||
}
|
||||
}
|
||||
|
||||
function Invoke-DockerText {
|
||||
param([string[]]$Arguments)
|
||||
$previousErrorActionPreference = $ErrorActionPreference
|
||||
$ErrorActionPreference = "Continue"
|
||||
try {
|
||||
$output = docker --context $DockerContext @Arguments 2>&1 | ForEach-Object { $_.ToString() }
|
||||
}
|
||||
finally {
|
||||
$ErrorActionPreference = $previousErrorActionPreference
|
||||
}
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "docker --context $DockerContext $($Arguments -join ' ') failed with exit code $LASTEXITCODE`n$($output -join [Environment]::NewLine)"
|
||||
}
|
||||
return $output
|
||||
}
|
||||
|
||||
function Assert-SharedDockerContext {
|
||||
$normalized = $DockerContext.Trim().ToLowerInvariant()
|
||||
if ($normalized -eq "" -or $normalized -eq "default" -or $normalized -eq "desktop-linux" -or $normalized -eq "docker-desktop") {
|
||||
throw "fabric loadtest must use the shared Docker host context, not local Docker Desktop. Use -DockerContext test-docker, docker-test, or test-ubuntu."
|
||||
}
|
||||
}
|
||||
|
||||
function Assert-QuicTargets {
|
||||
param([string[]]$Targets)
|
||||
$invalid = @()
|
||||
foreach ($target in $Targets) {
|
||||
$trimmed = ([string]$target).Trim()
|
||||
if ($trimmed -eq "" -or -not $trimmed.ToLowerInvariant().StartsWith("quic://")) {
|
||||
if ($trimmed -eq "") {
|
||||
$invalid += "<empty>"
|
||||
}
|
||||
else {
|
||||
$invalid += $trimmed
|
||||
}
|
||||
}
|
||||
}
|
||||
if ($invalid.Count -gt 0) {
|
||||
throw "fabric loadtest targets must be QUIC-only: $($invalid -join ', ')"
|
||||
}
|
||||
}
|
||||
|
||||
function Cleanup {
|
||||
if ($KeepRunning) {
|
||||
return
|
||||
}
|
||||
$previousErrorActionPreference = $ErrorActionPreference
|
||||
$ErrorActionPreference = "Continue"
|
||||
try {
|
||||
docker --context $DockerContext rm -f $clientName 2>$null | Out-Null
|
||||
for ($i = 0; $i -lt $Nodes; $i++) {
|
||||
docker --context $DockerContext rm -f "$serverPrefix$i" 2>$null | Out-Null
|
||||
}
|
||||
docker --context $DockerContext network rm $networkName 2>$null | Out-Null
|
||||
}
|
||||
finally {
|
||||
$ErrorActionPreference = $previousErrorActionPreference
|
||||
}
|
||||
}
|
||||
|
||||
function Invoke-HostSysctl {
|
||||
param([string[]]$Arguments)
|
||||
$dockerArgs = @(
|
||||
"run", "--rm",
|
||||
"--privileged",
|
||||
"--network", "host",
|
||||
"--entrypoint", "sysctl",
|
||||
$ImageTag
|
||||
) + $Arguments
|
||||
return Invoke-DockerText -Arguments $dockerArgs
|
||||
}
|
||||
|
||||
function Set-HostUdpBuffers {
|
||||
Invoke-HostSysctl -Arguments @(
|
||||
"-w",
|
||||
"net.core.rmem_max=$UdpBufferBytes",
|
||||
"net.core.wmem_max=$UdpBufferBytes",
|
||||
"net.core.rmem_default=$UdpBufferBytes",
|
||||
"net.core.wmem_default=$UdpBufferBytes"
|
||||
) | Out-Null
|
||||
}
|
||||
|
||||
function Get-HostUdpBuffers {
|
||||
$lines = Invoke-HostSysctl -Arguments @(
|
||||
"net.core.rmem_max",
|
||||
"net.core.wmem_max",
|
||||
"net.core.rmem_default",
|
||||
"net.core.wmem_default"
|
||||
)
|
||||
$values = [ordered]@{}
|
||||
foreach ($line in $lines) {
|
||||
if ($line -match '^(?<key>[^=]+)=\s*(?<value>\d+)$') {
|
||||
$values[$Matches["key"].Trim()] = [int64]$Matches["value"]
|
||||
}
|
||||
}
|
||||
return $values
|
||||
}
|
||||
|
||||
function Set-ContainerImpairment {
|
||||
param([string]$ContainerName)
|
||||
$netem = @()
|
||||
if ($ImpairDelay.Trim() -ne "") {
|
||||
$netem += @("delay", $ImpairDelay.Trim())
|
||||
}
|
||||
if ($ImpairLoss.Trim() -ne "") {
|
||||
$netem += @("loss", $ImpairLoss.Trim())
|
||||
}
|
||||
if ($ImpairRate.Trim() -ne "") {
|
||||
$netem += @("rate", $ImpairRate.Trim())
|
||||
}
|
||||
if ($netem.Count -eq 0) {
|
||||
return @()
|
||||
}
|
||||
$previousErrorActionPreference = $ErrorActionPreference
|
||||
$ErrorActionPreference = "Continue"
|
||||
try {
|
||||
docker --context $DockerContext exec $ContainerName tc qdisc del dev eth0 root 2>$null | Out-Null
|
||||
}
|
||||
finally {
|
||||
$ErrorActionPreference = $previousErrorActionPreference
|
||||
}
|
||||
Invoke-Docker -Arguments (@("exec", $ContainerName, "tc", "qdisc", "add", "dev", "eth0", "root", "netem") + $netem)
|
||||
return Invoke-DockerText -Arguments @("exec", $ContainerName, "tc", "qdisc", "show", "dev", "eth0")
|
||||
}
|
||||
|
||||
function Get-FabricContainerStats {
|
||||
$names = @($clientName)
|
||||
for ($i = 0; $i -lt $Nodes; $i++) {
|
||||
$names += "$serverPrefix$i"
|
||||
}
|
||||
$stats = @()
|
||||
foreach ($name in $names) {
|
||||
$inspect = $null
|
||||
try {
|
||||
$inspectText = Invoke-DockerText -Arguments @("inspect", $name)
|
||||
$inspect = @(($inspectText -join [Environment]::NewLine) | ConvertFrom-Json)
|
||||
}
|
||||
catch {
|
||||
continue
|
||||
}
|
||||
if ($null -eq $inspect -or $inspect.Count -eq 0 -or $inspect[0].State.Running -ne $true) {
|
||||
continue
|
||||
}
|
||||
try {
|
||||
$line = @(Invoke-DockerText -Arguments @("stats", "--no-stream", "--format", "{{json .}}", $name))
|
||||
if ($line.Count -gt 0) {
|
||||
$item = ($line[0] | ConvertFrom-Json)
|
||||
$memoryUsageMiB = Convert-DockerMemoryUsageToMiB -MemoryUsage $item.MemUsage
|
||||
$pidCount = Convert-DockerPidsToInt -Pids $item.PIDs
|
||||
$stats += [pscustomobject]@{
|
||||
name = $item.Name
|
||||
id = $item.ID
|
||||
cpu_percent = $item.CPUPerc
|
||||
memory_percent = $item.MemPerc
|
||||
memory_usage = $item.MemUsage
|
||||
memory_usage_mib = $memoryUsageMiB
|
||||
net_io = $item.NetIO
|
||||
block_io = $item.BlockIO
|
||||
pids = $item.PIDs
|
||||
pids_count = $pidCount
|
||||
role = $(if ($item.Name -eq $clientName) { "client" } else { "server" })
|
||||
}
|
||||
}
|
||||
}
|
||||
catch {
|
||||
$stats += [pscustomobject]@{
|
||||
name = $name
|
||||
error = $_.Exception.Message
|
||||
}
|
||||
}
|
||||
}
|
||||
return $stats
|
||||
}
|
||||
|
||||
function Convert-DockerMemoryUsageToMiB {
|
||||
param([string]$MemoryUsage)
|
||||
if ($MemoryUsage.Trim() -eq "") {
|
||||
return $null
|
||||
}
|
||||
$used = ($MemoryUsage -split '/')[0].Trim()
|
||||
if ($used -notmatch '^(?<value>[0-9.]+)\s*(?<unit>[A-Za-z]+)$') {
|
||||
return $null
|
||||
}
|
||||
$value = [double]$Matches["value"]
|
||||
switch ($Matches["unit"].ToLowerInvariant()) {
|
||||
"b" { return [math]::Round($value / 1MB, 3) }
|
||||
"kb" { return [math]::Round($value / 1024, 3) }
|
||||
"kib" { return [math]::Round($value / 1024, 3) }
|
||||
"mb" { return [math]::Round($value, 3) }
|
||||
"mib" { return [math]::Round($value, 3) }
|
||||
"gb" { return [math]::Round($value * 1024, 3) }
|
||||
"gib" { return [math]::Round($value * 1024, 3) }
|
||||
default { return $null }
|
||||
}
|
||||
}
|
||||
|
||||
function Convert-DockerPidsToInt {
|
||||
param([string]$Pids)
|
||||
$value = 0
|
||||
if ([int]::TryParse($Pids, [ref]$value)) {
|
||||
return $value
|
||||
}
|
||||
return $null
|
||||
}
|
||||
|
||||
function Convert-DurationToMilliseconds {
|
||||
param([string]$Duration, [int]$DefaultMilliseconds)
|
||||
$value = $Duration.Trim()
|
||||
if ($value -eq "") {
|
||||
return $DefaultMilliseconds
|
||||
}
|
||||
if ($value -match '^(?<n>[0-9]+)ms$') {
|
||||
return [int]$Matches["n"]
|
||||
}
|
||||
if ($value -match '^(?<n>[0-9]+)s$') {
|
||||
return [int]$Matches["n"] * 1000
|
||||
}
|
||||
if ($value -match '^(?<n>[0-9]+)m$') {
|
||||
return [int]$Matches["n"] * 60 * 1000
|
||||
}
|
||||
if ($value -match '^(?<n>[0-9]+)$') {
|
||||
return [int]$Matches["n"] * 1000
|
||||
}
|
||||
return $DefaultMilliseconds
|
||||
}
|
||||
|
||||
function Start-ContainerStatsSampler {
|
||||
param([string[]]$Names)
|
||||
if ($ContainerStatsSampleInterval.Trim() -eq "") {
|
||||
return $null
|
||||
}
|
||||
$intervalMs = Convert-DurationToMilliseconds -Duration $ContainerStatsSampleInterval -DefaultMilliseconds 10000
|
||||
if ($intervalMs -le 0) {
|
||||
return $null
|
||||
}
|
||||
return Start-Job -ScriptBlock {
|
||||
param($dockerContext, $names, $intervalMs)
|
||||
while ($true) {
|
||||
$observedAt = (Get-Date).ToUniversalTime().ToString("o")
|
||||
foreach ($name in $names) {
|
||||
$running = docker --context $dockerContext inspect -f "{{.State.Running}}" $name 2>$null
|
||||
if ($LASTEXITCODE -ne 0 -or $running -ne "true") {
|
||||
continue
|
||||
}
|
||||
$line = docker --context $dockerContext stats --no-stream --format "{{json .}}" $name 2>$null
|
||||
if ($LASTEXITCODE -ne 0 -or $null -eq $line -or $line.Trim() -eq "") {
|
||||
continue
|
||||
}
|
||||
try {
|
||||
$item = $line | ConvertFrom-Json
|
||||
[pscustomobject]@{
|
||||
observed_at = $observedAt
|
||||
name = $item.Name
|
||||
id = $item.ID
|
||||
cpu_percent = $item.CPUPerc
|
||||
memory_percent = $item.MemPerc
|
||||
memory_usage = $item.MemUsage
|
||||
net_io = $item.NetIO
|
||||
block_io = $item.BlockIO
|
||||
pids = $item.PIDs
|
||||
role = $(if ($item.Name -like "*-client") { "client" } else { "server" })
|
||||
} | ConvertTo-Json -Compress
|
||||
}
|
||||
catch {
|
||||
}
|
||||
}
|
||||
Start-Sleep -Milliseconds $intervalMs
|
||||
}
|
||||
} -ArgumentList $DockerContext, $Names, $intervalMs
|
||||
}
|
||||
|
||||
function Stop-ContainerStatsSampler {
|
||||
param($Job)
|
||||
if ($null -eq $Job) {
|
||||
return @()
|
||||
}
|
||||
Stop-Job -Job $Job -ErrorAction SilentlyContinue | Out-Null
|
||||
$lines = @(Receive-Job -Job $Job -ErrorAction SilentlyContinue)
|
||||
Remove-Job -Job $Job -Force -ErrorAction SilentlyContinue | Out-Null
|
||||
$samples = @()
|
||||
foreach ($line in $lines) {
|
||||
if ($null -eq $line -or $line.ToString().Trim() -eq "") {
|
||||
continue
|
||||
}
|
||||
try {
|
||||
$sample = $line.ToString() | ConvertFrom-Json
|
||||
$sample | Add-Member -NotePropertyName memory_usage_mib -NotePropertyValue (Convert-DockerMemoryUsageToMiB -MemoryUsage $sample.memory_usage) -Force
|
||||
$sample | Add-Member -NotePropertyName pids_count -NotePropertyValue (Convert-DockerPidsToInt -Pids $sample.pids) -Force
|
||||
$samples += $sample
|
||||
}
|
||||
catch {
|
||||
}
|
||||
}
|
||||
return $samples
|
||||
}
|
||||
|
||||
function Get-ContainerStatsSampleSummary {
|
||||
param([object[]]$Samples)
|
||||
if ($null -eq $Samples -or $Samples.Count -eq 0) {
|
||||
return $null
|
||||
}
|
||||
$byName = @{}
|
||||
foreach ($sample in $Samples) {
|
||||
if ($sample.name -eq $null) {
|
||||
continue
|
||||
}
|
||||
if (-not $byName.ContainsKey($sample.name)) {
|
||||
$byName[$sample.name] = [ordered]@{
|
||||
name = $sample.name
|
||||
role = $sample.role
|
||||
sample_count = 0
|
||||
memory_usage_mib_start = $sample.memory_usage_mib
|
||||
memory_usage_mib_end = $sample.memory_usage_mib
|
||||
memory_usage_mib_max = $sample.memory_usage_mib
|
||||
pids_start = $sample.pids_count
|
||||
pids_end = $sample.pids_count
|
||||
pids_max = $sample.pids_count
|
||||
}
|
||||
}
|
||||
$entry = $byName[$sample.name]
|
||||
$entry.sample_count++
|
||||
$entry.memory_usage_mib_end = $sample.memory_usage_mib
|
||||
if ($null -ne $sample.memory_usage_mib -and ($null -eq $entry.memory_usage_mib_max -or $sample.memory_usage_mib -gt $entry.memory_usage_mib_max)) {
|
||||
$entry.memory_usage_mib_max = $sample.memory_usage_mib
|
||||
}
|
||||
$entry.pids_end = $sample.pids_count
|
||||
if ($null -ne $sample.pids_count -and ($null -eq $entry.pids_max -or $sample.pids_count -gt $entry.pids_max)) {
|
||||
$entry.pids_max = $sample.pids_count
|
||||
}
|
||||
}
|
||||
$summary = @()
|
||||
foreach ($entry in $byName.Values) {
|
||||
$entry.memory_usage_mib_delta = if ($null -ne $entry.memory_usage_mib_start -and $null -ne $entry.memory_usage_mib_end) { [math]::Round($entry.memory_usage_mib_end - $entry.memory_usage_mib_start, 3) } else { $null }
|
||||
$entry.pids_delta = if ($null -ne $entry.pids_start -and $null -ne $entry.pids_end) { $entry.pids_end - $entry.pids_start } else { $null }
|
||||
$summary += [pscustomobject]$entry
|
||||
}
|
||||
return $summary
|
||||
}
|
||||
|
||||
function Get-ContainerStatsSampleVerdictReasons {
|
||||
param([object[]]$SampleSummary)
|
||||
$reasons = @()
|
||||
if ($null -eq $SampleSummary) {
|
||||
return $reasons
|
||||
}
|
||||
foreach ($stat in $SampleSummary) {
|
||||
if ($MaxContainerMemoryMiB -gt 0 -and $null -ne $stat.memory_usage_mib_max -and $stat.memory_usage_mib_max -gt $MaxContainerMemoryMiB) {
|
||||
$reasons += "container_memory_mib_peak=$($stat.name):$($stat.memory_usage_mib_max)>$MaxContainerMemoryMiB"
|
||||
}
|
||||
if ($MaxContainerPids -gt 0 -and $null -ne $stat.pids_max -and $stat.pids_max -gt $MaxContainerPids) {
|
||||
$reasons += "container_pids_peak=$($stat.name):$($stat.pids_max)>$MaxContainerPids"
|
||||
}
|
||||
}
|
||||
return $reasons
|
||||
}
|
||||
|
||||
function Get-ContainerStatsVerdictReasons {
|
||||
param([object[]]$ContainerStats)
|
||||
$reasons = @()
|
||||
foreach ($stat in $ContainerStats) {
|
||||
if ($stat.PSObject.Properties.Name -contains "error") {
|
||||
$reasons += "container_stats_error=$($stat.name)"
|
||||
continue
|
||||
}
|
||||
if ($MaxContainerMemoryMiB -gt 0 -and $null -ne $stat.memory_usage_mib -and $stat.memory_usage_mib -gt $MaxContainerMemoryMiB) {
|
||||
$reasons += "container_memory_mib=$($stat.name):$($stat.memory_usage_mib)>$MaxContainerMemoryMiB"
|
||||
}
|
||||
if ($MaxContainerPids -gt 0 -and $null -ne $stat.pids_count -and $stat.pids_count -gt $MaxContainerPids) {
|
||||
$reasons += "container_pids=$($stat.name):$($stat.pids_count)>$MaxContainerPids"
|
||||
}
|
||||
}
|
||||
return $reasons
|
||||
}
|
||||
|
||||
try {
|
||||
Assert-SharedDockerContext
|
||||
|
||||
if (-not $SkipBuild) {
|
||||
Invoke-Docker -Arguments @(
|
||||
"build",
|
||||
"-f", "$repoRoot\agents\rap-node-agent\Dockerfile.fabric-loadtest",
|
||||
"-t", $ImageTag,
|
||||
$repoRoot
|
||||
)
|
||||
}
|
||||
|
||||
if ($TuneUdpBuffers) {
|
||||
Set-HostUdpBuffers
|
||||
}
|
||||
$udpBuffers = Get-HostUdpBuffers
|
||||
|
||||
Cleanup
|
||||
Invoke-Docker -Arguments @("network", "create", $networkName) | Out-Null
|
||||
|
||||
for ($i = 0; $i -lt $Nodes; $i++) {
|
||||
$name = "$serverPrefix$i"
|
||||
Invoke-Docker -Arguments @(
|
||||
"run", "-d",
|
||||
"--name", $name,
|
||||
"--network", $networkName,
|
||||
"--cap-add", "NET_ADMIN",
|
||||
$ImageTag,
|
||||
"-mode", "server",
|
||||
"-listen", "0.0.0.0:19443",
|
||||
"-timeout", $ServerTimeout,
|
||||
"-concurrency", ([string]$Concurrency)
|
||||
) | Out-Null
|
||||
}
|
||||
|
||||
Start-Sleep -Seconds 3
|
||||
|
||||
$impairment = @()
|
||||
if ($ImpairTarget -ge 0 -and $ImpairTarget -lt $Nodes) {
|
||||
if ($ImpairAfter.Trim() -eq "") {
|
||||
$impairment = Set-ContainerImpairment -ContainerName "$serverPrefix$ImpairTarget"
|
||||
}
|
||||
else {
|
||||
$impairment = @("scheduled target=$serverPrefix$ImpairTarget after=$ImpairAfter delay=$ImpairDelay loss=$ImpairLoss rate=$ImpairRate")
|
||||
Start-Job -ScriptBlock {
|
||||
param($dockerContext, $containerName, $delay, $impairDelay, $impairLoss, $impairRate)
|
||||
if ($delay -match '^(\d+)ms$') {
|
||||
Start-Sleep -Milliseconds ([int]$Matches[1])
|
||||
}
|
||||
elseif ($delay -match '^(\d+)s$') {
|
||||
Start-Sleep -Seconds ([int]$Matches[1])
|
||||
}
|
||||
elseif ($delay -match '^(\d+)$') {
|
||||
Start-Sleep -Seconds ([int]$Matches[1])
|
||||
}
|
||||
docker --context $dockerContext exec $containerName tc qdisc del dev eth0 root 2>$null | Out-Null
|
||||
$args = @("exec", $containerName, "tc", "qdisc", "add", "dev", "eth0", "root", "netem")
|
||||
if ($impairDelay.Trim() -ne "") { $args += @("delay", $impairDelay.Trim()) }
|
||||
if ($impairLoss.Trim() -ne "") { $args += @("loss", $impairLoss.Trim()) }
|
||||
if ($impairRate.Trim() -ne "") { $args += @("rate", $impairRate.Trim()) }
|
||||
docker --context $dockerContext @args | Out-Null
|
||||
} -ArgumentList $DockerContext, "$serverPrefix$ImpairTarget", $ImpairAfter, $ImpairDelay, $ImpairLoss, $ImpairRate | Out-Null
|
||||
}
|
||||
}
|
||||
|
||||
$targets = @()
|
||||
for ($i = 0; $i -lt $Nodes; $i++) {
|
||||
$targets += "quic://$serverPrefix$i`:19443"
|
||||
}
|
||||
Assert-QuicTargets -Targets $targets
|
||||
$targetList = ($targets -join ",")
|
||||
|
||||
if ($FailTarget -ge 0 -and $FailTarget -lt $Nodes) {
|
||||
Start-Job -ScriptBlock {
|
||||
param($dockerContext, $containerName, $delay)
|
||||
if ($delay -match '^(\d+)ms$') {
|
||||
Start-Sleep -Milliseconds ([int]$Matches[1])
|
||||
}
|
||||
elseif ($delay -match '^(\d+)s$') {
|
||||
Start-Sleep -Seconds ([int]$Matches[1])
|
||||
}
|
||||
elseif ($delay -match '^(\d+)$') {
|
||||
Start-Sleep -Seconds ([int]$Matches[1])
|
||||
}
|
||||
docker --context $dockerContext rm -f $containerName | Out-Null
|
||||
} -ArgumentList $DockerContext, "$serverPrefix$FailTarget", $FailAfter | Out-Null
|
||||
}
|
||||
|
||||
$clientArgs = @(
|
||||
"run", "--rm",
|
||||
"--name", $clientName,
|
||||
"--network", $networkName,
|
||||
$ImageTag,
|
||||
"-mode", "client",
|
||||
"-targets", $targetList,
|
||||
"-topology-profile", $TopologyProfile,
|
||||
"-soak=$($Soak.IsPresent.ToString().ToLowerInvariant())",
|
||||
"-streams", ([string]$Streams),
|
||||
"-concurrency", ([string]$Concurrency),
|
||||
"-bytes-per-stream", ([string]$BytesPerStream),
|
||||
"-control-every", ([string]$ControlEvery),
|
||||
"-control-bytes-per-stream", ([string]$ControlBytesPerStream),
|
||||
"-max-control-ack-p95-ms", ([string]$MaxControlAckP95Ms),
|
||||
"-payload-size", ([string]$PayloadSize),
|
||||
"-resource-sample-interval", $ResourceSampleInterval,
|
||||
"-stream-timeout", $StreamTimeout,
|
||||
"-ack-timeout", $AckTimeout,
|
||||
"-target-quarantine-ttl", $TargetQuarantineTTL,
|
||||
"-failure-quarantine-ttl", $FailureQuarantineTTL,
|
||||
"-fail-target", ([string]$FailTarget),
|
||||
"-impair-target", ([string]$ImpairTarget),
|
||||
"-pool-failover=true",
|
||||
"-timeout", $ClientTimeout
|
||||
)
|
||||
if ($ProbeTargets) {
|
||||
$clientArgs += "-probe-targets=true"
|
||||
}
|
||||
if ($MaxTargetRttMs -gt 0) {
|
||||
$clientArgs += @("-max-target-rtt-ms", ([string]$MaxTargetRttMs))
|
||||
}
|
||||
if ($MigrateSlowStreams) {
|
||||
$clientArgs += "-migrate-slow-streams=true"
|
||||
}
|
||||
if ($MaxAckMs -gt 0) {
|
||||
$clientArgs += @("-max-ack-ms", ([string]$MaxAckMs))
|
||||
}
|
||||
if ($MaxAckP95Ms -gt 0) {
|
||||
$clientArgs += @("-max-ack-p95-ms", ([string]$MaxAckP95Ms))
|
||||
}
|
||||
if ($MaxAckP99Ms -gt 0) {
|
||||
$clientArgs += @("-max-ack-p99-ms", ([string]$MaxAckP99Ms))
|
||||
}
|
||||
if ($MaxTargetAckMs -gt 0) {
|
||||
$clientArgs += @("-max-target-ack-ms", ([string]$MaxTargetAckMs))
|
||||
}
|
||||
if ($MaxSetupP95Ms -gt 0) {
|
||||
$clientArgs += @("-max-setup-p95-ms", ([string]$MaxSetupP95Ms))
|
||||
}
|
||||
if ($MaxSetupP99Ms -gt 0) {
|
||||
$clientArgs += @("-max-setup-p99-ms", ([string]$MaxSetupP99Ms))
|
||||
}
|
||||
if ($MaxRerouteP95Ms -gt 0) {
|
||||
$clientArgs += @("-max-reroute-p95-ms", ([string]$MaxRerouteP95Ms))
|
||||
}
|
||||
if ($MaxRerouteP99Ms -gt 0) {
|
||||
$clientArgs += @("-max-reroute-p99-ms", ([string]$MaxRerouteP99Ms))
|
||||
}
|
||||
if ($MaxGoroutineDelta -gt 0) {
|
||||
$clientArgs += @("-max-goroutine-delta", ([string]$MaxGoroutineDelta))
|
||||
}
|
||||
if ($MaxHeapDeltaMB -gt 0) {
|
||||
$clientArgs += @("-max-heap-delta-mb", ([string]$MaxHeapDeltaMB))
|
||||
}
|
||||
if ($MaxOpenFDDelta -gt 0) {
|
||||
$clientArgs += @("-max-open-fd-delta", ([string]$MaxOpenFDDelta))
|
||||
}
|
||||
if ($MaxOpenFDs -gt 0) {
|
||||
$clientArgs += @("-max-open-fds", ([string]$MaxOpenFDs))
|
||||
}
|
||||
if ($MinThroughputMbps -gt 0) {
|
||||
$clientArgs += @("-min-throughput-mbps", ([string]$MinThroughputMbps))
|
||||
}
|
||||
if ($MinChannelChurnPerSec -gt 0) {
|
||||
$clientArgs += @("-min-channel-churn-per-sec", ([string]$MinChannelChurnPerSec))
|
||||
}
|
||||
if ($Duration.Trim() -ne "") {
|
||||
$clientArgs += @("-duration", $Duration.Trim())
|
||||
}
|
||||
$containerNamesForSampling = @($clientName)
|
||||
for ($i = 0; $i -lt $Nodes; $i++) {
|
||||
$containerNamesForSampling += "$serverPrefix$i"
|
||||
}
|
||||
$containerStatsSamplerJob = Start-ContainerStatsSampler -Names $containerNamesForSampling
|
||||
$containerStatsSamples = @()
|
||||
try {
|
||||
$clientOutput = Invoke-DockerText -Arguments $clientArgs
|
||||
}
|
||||
finally {
|
||||
$containerStatsSamples = @(Stop-ContainerStatsSampler -Job $containerStatsSamplerJob)
|
||||
}
|
||||
|
||||
$rawText = ($clientOutput -join [Environment]::NewLine)
|
||||
$jsonStart = $rawText.IndexOf("{")
|
||||
if ($jsonStart -lt 0) {
|
||||
throw "fabric loadtest client did not emit JSON: $rawText"
|
||||
}
|
||||
$jsonText = $rawText.Substring($jsonStart)
|
||||
New-Item -ItemType Directory -Force -Path (Split-Path -Parent $ReportPath) | Out-Null
|
||||
Set-Content -LiteralPath $ReportPath -Value $jsonText -Encoding UTF8
|
||||
$report = $jsonText | ConvertFrom-Json
|
||||
$verdictReasons = $null
|
||||
if ($report.PSObject.Properties.Name -contains "verdict_reasons") {
|
||||
$verdictReasons = $report.verdict_reasons
|
||||
}
|
||||
$targetStats = $null
|
||||
if ($report.PSObject.Properties.Name -contains "target_stats") {
|
||||
$targetStats = $report.target_stats
|
||||
}
|
||||
$targetProbes = $null
|
||||
if ($report.PSObject.Properties.Name -contains "target_probes") {
|
||||
$targetProbes = $report.target_probes
|
||||
}
|
||||
$excludedTargets = $null
|
||||
if ($report.PSObject.Properties.Name -contains "excluded_targets") {
|
||||
$excludedTargets = $report.excluded_targets
|
||||
}
|
||||
$migrationEvents = 0
|
||||
if ($report.PSObject.Properties.Name -contains "migration_events") {
|
||||
$migrationEvents = $report.migration_events
|
||||
}
|
||||
$routePressure = $null
|
||||
if ($report.PSObject.Properties.Name -contains "route_pressure") {
|
||||
$routePressure = $report.route_pressure
|
||||
}
|
||||
$transportSnapshot = $null
|
||||
if ($report.PSObject.Properties.Name -contains "transport_snapshot") {
|
||||
$transportSnapshot = $report.transport_snapshot
|
||||
}
|
||||
$degradedTargets = $null
|
||||
if ($report.PSObject.Properties.Name -contains "degraded_targets") {
|
||||
$degradedTargets = $report.degraded_targets
|
||||
}
|
||||
$ackP95 = $null
|
||||
if ($report.PSObject.Properties.Name -contains "ack_p95_ms") {
|
||||
$ackP95 = $report.ack_p95_ms
|
||||
}
|
||||
$ackP99 = $null
|
||||
if ($report.PSObject.Properties.Name -contains "ack_p99_ms") {
|
||||
$ackP99 = $report.ack_p99_ms
|
||||
}
|
||||
$rerouteLatencyP95 = $null
|
||||
if ($report.PSObject.Properties.Name -contains "reroute_latency_p95_ms") {
|
||||
$rerouteLatencyP95 = $report.reroute_latency_p95_ms
|
||||
}
|
||||
$rerouteLatencyP99 = $null
|
||||
if ($report.PSObject.Properties.Name -contains "reroute_latency_p99_ms") {
|
||||
$rerouteLatencyP99 = $report.reroute_latency_p99_ms
|
||||
}
|
||||
$rerouteCauses = $null
|
||||
if ($report.PSObject.Properties.Name -contains "reroute_causes") {
|
||||
$rerouteCauses = $report.reroute_causes
|
||||
}
|
||||
$resourceSummary = $null
|
||||
if ($report.PSObject.Properties.Name -contains "resource_summary") {
|
||||
$resourceSummary = $report.resource_summary
|
||||
}
|
||||
$ackMismatchedStreams = 0
|
||||
if ($report.PSObject.Properties.Name -contains "ack_mismatched_streams") {
|
||||
$ackMismatchedStreams = $report.ack_mismatched_streams
|
||||
}
|
||||
$ackIntegrityErrors = 0
|
||||
if ($report.PSObject.Properties.Name -contains "ack_integrity_errors") {
|
||||
$ackIntegrityErrors = $report.ack_integrity_errors
|
||||
}
|
||||
$abandonedFrames = 0
|
||||
if ($report.PSObject.Properties.Name -contains "abandoned_frames") {
|
||||
$abandonedFrames = $report.abandoned_frames
|
||||
}
|
||||
$controlAckP95 = $null
|
||||
if ($report.PSObject.Properties.Name -contains "control_ack_p95_ms") {
|
||||
$controlAckP95 = $report.control_ack_p95_ms
|
||||
}
|
||||
$bulkAckP95 = $null
|
||||
if ($report.PSObject.Properties.Name -contains "bulk_ack_p95_ms") {
|
||||
$bulkAckP95 = $report.bulk_ack_p95_ms
|
||||
}
|
||||
$containerStats = Get-FabricContainerStats
|
||||
if ($ContainerStatsSampleInterval.Trim() -ne "" -and $containerStatsSamples.Count -eq 0 -and $containerStats.Count -gt 0) {
|
||||
$observedAt = (Get-Date).ToUniversalTime().ToString("o")
|
||||
foreach ($stat in $containerStats) {
|
||||
if ($stat.PSObject.Properties.Name -contains "error") {
|
||||
continue
|
||||
}
|
||||
$containerStatsSamples += [pscustomobject]@{
|
||||
observed_at = $observedAt
|
||||
name = $stat.name
|
||||
id = $stat.id
|
||||
cpu_percent = $stat.cpu_percent
|
||||
memory_percent = $stat.memory_percent
|
||||
memory_usage = $stat.memory_usage
|
||||
memory_usage_mib = $stat.memory_usage_mib
|
||||
net_io = $stat.net_io
|
||||
block_io = $stat.block_io
|
||||
pids = $stat.pids
|
||||
pids_count = $stat.pids_count
|
||||
role = $stat.role
|
||||
}
|
||||
}
|
||||
}
|
||||
$containerStatsSampleSummary = Get-ContainerStatsSampleSummary -Samples $containerStatsSamples
|
||||
if ($containerStatsSamples.Count -gt 0) {
|
||||
New-Item -ItemType Directory -Force -Path (Split-Path -Parent $ContainerStatsSamplesPath) | Out-Null
|
||||
Set-Content -LiteralPath $ContainerStatsSamplesPath -Value ($containerStatsSamples | ConvertTo-Json -Depth 20) -Encoding UTF8
|
||||
}
|
||||
$containerVerdictReasons = @(Get-ContainerStatsVerdictReasons -ContainerStats $containerStats)
|
||||
$containerSampleVerdictReasons = @(Get-ContainerStatsSampleVerdictReasons -SampleSummary $containerStatsSampleSummary)
|
||||
$combinedVerdictReasons = @()
|
||||
if ($null -ne $verdictReasons) {
|
||||
$combinedVerdictReasons += $verdictReasons
|
||||
}
|
||||
$combinedVerdictReasons += $containerVerdictReasons
|
||||
$combinedVerdictReasons += $containerSampleVerdictReasons
|
||||
$summaryVerdict = $report.verdict
|
||||
if ($containerVerdictReasons.Count -gt 0 -or $containerSampleVerdictReasons.Count -gt 0) {
|
||||
$summaryVerdict = "fail"
|
||||
}
|
||||
$summary = [pscustomobject]@{
|
||||
run_id = $runId
|
||||
report_path = $ReportPath
|
||||
summary_path = $SummaryPath
|
||||
docker_context = $DockerContext
|
||||
nodes = $Nodes
|
||||
topology_profile = $TopologyProfile
|
||||
soak = $Soak.IsPresent
|
||||
targets = $targets
|
||||
total_streams = $report.total_streams
|
||||
successful_streams = $report.successful_streams
|
||||
failed_streams = $report.failed_streams
|
||||
ack_mismatched_streams = $ackMismatchedStreams
|
||||
ack_integrity_errors = $ackIntegrityErrors
|
||||
abandoned_frames = $abandonedFrames
|
||||
failover_events = $report.failover_events
|
||||
migration_events = $migrationEvents
|
||||
channel_opens = $report.channel_opens
|
||||
channel_closes = $report.channel_closes
|
||||
channel_leaks = $report.channel_leaks
|
||||
channel_churn_per_sec = $report.channel_churn_per_sec
|
||||
bytes_sent = $report.bytes_sent
|
||||
throughput_bps = $report.throughput_bps
|
||||
setup_latency_p95_ms = $report.setup_latency_p95_ms
|
||||
channel_open_p95_ms = $report.channel_open_p95_ms
|
||||
stream_duration_p95_ms = $report.stream_duration_p95_ms
|
||||
ack_p95_ms = $ackP95
|
||||
ack_p99_ms = $ackP99
|
||||
reroute_latency_p95_ms = $rerouteLatencyP95
|
||||
reroute_latency_p99_ms = $rerouteLatencyP99
|
||||
route_attempts_total = $report.route_attempts_total
|
||||
reroute_causes = $rerouteCauses
|
||||
resource_summary = $resourceSummary
|
||||
container_stats = $containerStats
|
||||
container_stats_samples_path = $(if ($containerStatsSamples.Count -gt 0) { $ContainerStatsSamplesPath } else { $null })
|
||||
container_stats_samples_count = $containerStatsSamples.Count
|
||||
container_stats_sample_summary = $containerStatsSampleSummary
|
||||
control_streams = $report.control_streams
|
||||
bulk_streams = $report.bulk_streams
|
||||
control_ack_p95_ms = $controlAckP95
|
||||
bulk_ack_p95_ms = $bulkAckP95
|
||||
verdict = $summaryVerdict
|
||||
verdict_reasons = $combinedVerdictReasons
|
||||
udp_buffers = $udpBuffers
|
||||
impairment = $impairment
|
||||
target_probes = $targetProbes
|
||||
excluded_targets = $excludedTargets
|
||||
degraded_targets = $degradedTargets
|
||||
target_streams = $report.target_streams
|
||||
target_bytes = $report.target_bytes
|
||||
target_stats = $targetStats
|
||||
route_pressure = $routePressure
|
||||
transport_snapshot = $transportSnapshot
|
||||
}
|
||||
$summaryJson = $summary | ConvertTo-Json -Depth 20
|
||||
New-Item -ItemType Directory -Force -Path (Split-Path -Parent $SummaryPath) | Out-Null
|
||||
Set-Content -LiteralPath $SummaryPath -Value $summaryJson -Encoding UTF8
|
||||
$summaryJson
|
||||
if ($summaryVerdict -ne "pass") {
|
||||
$reasonText = ""
|
||||
if ($combinedVerdictReasons.Count -gt 0) {
|
||||
$reasonText = ($combinedVerdictReasons -join "; ")
|
||||
}
|
||||
throw "fabric loadtest verdict failed: $summaryVerdict $reasonText"
|
||||
}
|
||||
}
|
||||
finally {
|
||||
Cleanup
|
||||
}
|
||||
Reference in New Issue
Block a user