param( [string]$DockerContext = "test-docker", [string]$ImageTag = "rap-fabric-loadtest:dev", [int]$Nodes = 4, [int]$Streams = 400, [int]$Concurrency = 64, [int64]$BytesPerStream = 262144, [string]$Duration = "", [string]$ClientTimeout = "10m", [string]$ServerTimeout = "", [string]$StreamTimeout = "30s", [string]$AckTimeout = "2s", [string]$TargetQuarantineTTL = "30s", [string]$FailureQuarantineTTL = "5m", [string]$TopologyProfile = "", [switch]$Soak, [int]$ControlEvery = 0, [int64]$ControlBytesPerStream = 4096, [int64]$MaxControlAckP95Ms = 100, [int]$MaxGoroutineDelta = 0, [int64]$MaxHeapDeltaMB = 0, [int]$MaxOpenFDDelta = 0, [int]$MaxOpenFDs = 0, [int64]$MinThroughputMbps = 0, [int64]$MinChannelChurnPerSec = 0, [int64]$MaxContainerMemoryMiB = 0, [int]$MaxContainerPids = 0, [int]$PayloadSize = 16384, [int]$FailTarget = 0, [string]$FailAfter = "1s", [int]$ImpairTarget = -1, [string]$ImpairDelay = "", [string]$ImpairLoss = "", [string]$ImpairRate = "", [string]$ImpairAfter = "", [switch]$ProbeTargets, [int64]$MaxTargetRttMs = 0, [switch]$MigrateSlowStreams, [int64]$MaxAckMs = 0, [int64]$MaxAckP95Ms = 0, [int64]$MaxAckP99Ms = 0, [int64]$MaxTargetAckMs = 0, [int64]$MaxSetupP95Ms = 200, [int64]$MaxSetupP99Ms = 0, [int64]$MaxRerouteP95Ms = 0, [int64]$MaxRerouteP99Ms = 0, [string]$ResourceSampleInterval = "1s", [string]$ContainerStatsSampleInterval = "", [string]$ContainerStatsSamplesPath = "", [int]$UdpBufferBytes = 7500000, [string]$ArtifactDir = "", [string]$ReportPath = "", [string]$SummaryPath = "", [switch]$TuneUdpBuffers, [switch]$KeepRunning, [switch]$SkipBuild ) Set-StrictMode -Version Latest $ErrorActionPreference = "Stop" $repoRoot = (Resolve-Path (Join-Path $PSScriptRoot "..\..")).ProviderPath $runId = "fabric-loadtest-" + (Get-Date -Format "yyyyMMdd-HHmmss") $networkName = "rap-$runId-net" $serverPrefix = "rap-$runId-server-" $clientName = "rap-$runId-client" if ($ArtifactDir.Trim() -eq "") { $ArtifactDir = Join-Path $repoRoot "artifacts\fabric-loadtest" } if ($ReportPath.Trim() -eq "") { $ReportPath = Join-Path $ArtifactDir "$runId-report.json" } if ($SummaryPath.Trim() -eq "") { $SummaryPath = Join-Path $ArtifactDir "$runId-summary.json" } if ($ContainerStatsSamplesPath.Trim() -eq "") { $ContainerStatsSamplesPath = Join-Path $ArtifactDir "$runId-container-stats-samples.json" } if ($ServerTimeout.Trim() -eq "") { $ServerTimeout = $ClientTimeout } function Invoke-Docker { param([string[]]$Arguments) docker --context $DockerContext @Arguments if ($LASTEXITCODE -ne 0) { throw "docker --context $DockerContext $($Arguments -join ' ') failed with exit code $LASTEXITCODE" } } function Invoke-DockerText { param([string[]]$Arguments) $previousErrorActionPreference = $ErrorActionPreference $ErrorActionPreference = "Continue" try { $output = docker --context $DockerContext @Arguments 2>&1 | ForEach-Object { $_.ToString() } } finally { $ErrorActionPreference = $previousErrorActionPreference } if ($LASTEXITCODE -ne 0) { throw "docker --context $DockerContext $($Arguments -join ' ') failed with exit code $LASTEXITCODE`n$($output -join [Environment]::NewLine)" } return $output } function Assert-SharedDockerContext { $normalized = $DockerContext.Trim().ToLowerInvariant() if ($normalized -eq "" -or $normalized -eq "default" -or $normalized -eq "desktop-linux" -or $normalized -eq "docker-desktop") { throw "fabric loadtest must use the shared Docker host context, not local Docker Desktop. Use -DockerContext test-docker, docker-test, or test-ubuntu." } } function Assert-QuicTargets { param([string[]]$Targets) $invalid = @() foreach ($target in $Targets) { $trimmed = ([string]$target).Trim() if ($trimmed -eq "" -or -not $trimmed.ToLowerInvariant().StartsWith("quic://")) { if ($trimmed -eq "") { $invalid += "" } else { $invalid += $trimmed } } } if ($invalid.Count -gt 0) { throw "fabric loadtest targets must be QUIC-only: $($invalid -join ', ')" } } function Cleanup { if ($KeepRunning) { return } $previousErrorActionPreference = $ErrorActionPreference $ErrorActionPreference = "Continue" try { docker --context $DockerContext rm -f $clientName 2>$null | Out-Null for ($i = 0; $i -lt $Nodes; $i++) { docker --context $DockerContext rm -f "$serverPrefix$i" 2>$null | Out-Null } docker --context $DockerContext network rm $networkName 2>$null | Out-Null } finally { $ErrorActionPreference = $previousErrorActionPreference } } function Invoke-HostSysctl { param([string[]]$Arguments) $dockerArgs = @( "run", "--rm", "--privileged", "--network", "host", "--entrypoint", "sysctl", $ImageTag ) + $Arguments return Invoke-DockerText -Arguments $dockerArgs } function Set-HostUdpBuffers { Invoke-HostSysctl -Arguments @( "-w", "net.core.rmem_max=$UdpBufferBytes", "net.core.wmem_max=$UdpBufferBytes", "net.core.rmem_default=$UdpBufferBytes", "net.core.wmem_default=$UdpBufferBytes" ) | Out-Null } function Get-HostUdpBuffers { $lines = Invoke-HostSysctl -Arguments @( "net.core.rmem_max", "net.core.wmem_max", "net.core.rmem_default", "net.core.wmem_default" ) $values = [ordered]@{} foreach ($line in $lines) { if ($line -match '^(?[^=]+)=\s*(?\d+)$') { $values[$Matches["key"].Trim()] = [int64]$Matches["value"] } } return $values } function Set-ContainerImpairment { param([string]$ContainerName) $netem = @() if ($ImpairDelay.Trim() -ne "") { $netem += @("delay", $ImpairDelay.Trim()) } if ($ImpairLoss.Trim() -ne "") { $netem += @("loss", $ImpairLoss.Trim()) } if ($ImpairRate.Trim() -ne "") { $netem += @("rate", $ImpairRate.Trim()) } if ($netem.Count -eq 0) { return @() } $previousErrorActionPreference = $ErrorActionPreference $ErrorActionPreference = "Continue" try { docker --context $DockerContext exec $ContainerName tc qdisc del dev eth0 root 2>$null | Out-Null } finally { $ErrorActionPreference = $previousErrorActionPreference } Invoke-Docker -Arguments (@("exec", $ContainerName, "tc", "qdisc", "add", "dev", "eth0", "root", "netem") + $netem) return Invoke-DockerText -Arguments @("exec", $ContainerName, "tc", "qdisc", "show", "dev", "eth0") } function Get-FabricContainerStats { $names = @($clientName) for ($i = 0; $i -lt $Nodes; $i++) { $names += "$serverPrefix$i" } $stats = @() foreach ($name in $names) { $inspect = $null try { $inspectText = Invoke-DockerText -Arguments @("inspect", $name) $inspect = @(($inspectText -join [Environment]::NewLine) | ConvertFrom-Json) } catch { continue } if ($null -eq $inspect -or $inspect.Count -eq 0 -or $inspect[0].State.Running -ne $true) { continue } try { $line = @(Invoke-DockerText -Arguments @("stats", "--no-stream", "--format", "{{json .}}", $name)) if ($line.Count -gt 0) { $item = ($line[0] | ConvertFrom-Json) $memoryUsageMiB = Convert-DockerMemoryUsageToMiB -MemoryUsage $item.MemUsage $pidCount = Convert-DockerPidsToInt -Pids $item.PIDs $stats += [pscustomobject]@{ name = $item.Name id = $item.ID cpu_percent = $item.CPUPerc memory_percent = $item.MemPerc memory_usage = $item.MemUsage memory_usage_mib = $memoryUsageMiB net_io = $item.NetIO block_io = $item.BlockIO pids = $item.PIDs pids_count = $pidCount role = $(if ($item.Name -eq $clientName) { "client" } else { "server" }) } } } catch { $stats += [pscustomobject]@{ name = $name error = $_.Exception.Message } } } return $stats } function Convert-DockerMemoryUsageToMiB { param([string]$MemoryUsage) if ($MemoryUsage.Trim() -eq "") { return $null } $used = ($MemoryUsage -split '/')[0].Trim() if ($used -notmatch '^(?[0-9.]+)\s*(?[A-Za-z]+)$') { return $null } $value = [double]$Matches["value"] switch ($Matches["unit"].ToLowerInvariant()) { "b" { return [math]::Round($value / 1MB, 3) } "kb" { return [math]::Round($value / 1024, 3) } "kib" { return [math]::Round($value / 1024, 3) } "mb" { return [math]::Round($value, 3) } "mib" { return [math]::Round($value, 3) } "gb" { return [math]::Round($value * 1024, 3) } "gib" { return [math]::Round($value * 1024, 3) } default { return $null } } } function Convert-DockerPidsToInt { param([string]$Pids) $value = 0 if ([int]::TryParse($Pids, [ref]$value)) { return $value } return $null } function Convert-DurationToMilliseconds { param([string]$Duration, [int]$DefaultMilliseconds) $value = $Duration.Trim() if ($value -eq "") { return $DefaultMilliseconds } if ($value -match '^(?[0-9]+)ms$') { return [int]$Matches["n"] } if ($value -match '^(?[0-9]+)s$') { return [int]$Matches["n"] * 1000 } if ($value -match '^(?[0-9]+)m$') { return [int]$Matches["n"] * 60 * 1000 } if ($value -match '^(?[0-9]+)$') { return [int]$Matches["n"] * 1000 } return $DefaultMilliseconds } function Start-ContainerStatsSampler { param([string[]]$Names) if ($ContainerStatsSampleInterval.Trim() -eq "") { return $null } $intervalMs = Convert-DurationToMilliseconds -Duration $ContainerStatsSampleInterval -DefaultMilliseconds 10000 if ($intervalMs -le 0) { return $null } return Start-Job -ScriptBlock { param($dockerContext, $names, $intervalMs) while ($true) { $observedAt = (Get-Date).ToUniversalTime().ToString("o") foreach ($name in $names) { $running = docker --context $dockerContext inspect -f "{{.State.Running}}" $name 2>$null if ($LASTEXITCODE -ne 0 -or $running -ne "true") { continue } $line = docker --context $dockerContext stats --no-stream --format "{{json .}}" $name 2>$null if ($LASTEXITCODE -ne 0 -or $null -eq $line -or $line.Trim() -eq "") { continue } try { $item = $line | ConvertFrom-Json [pscustomobject]@{ observed_at = $observedAt name = $item.Name id = $item.ID cpu_percent = $item.CPUPerc memory_percent = $item.MemPerc memory_usage = $item.MemUsage net_io = $item.NetIO block_io = $item.BlockIO pids = $item.PIDs role = $(if ($item.Name -like "*-client") { "client" } else { "server" }) } | ConvertTo-Json -Compress } catch { } } Start-Sleep -Milliseconds $intervalMs } } -ArgumentList $DockerContext, $Names, $intervalMs } function Stop-ContainerStatsSampler { param($Job) if ($null -eq $Job) { return @() } Stop-Job -Job $Job -ErrorAction SilentlyContinue | Out-Null $lines = @(Receive-Job -Job $Job -ErrorAction SilentlyContinue) Remove-Job -Job $Job -Force -ErrorAction SilentlyContinue | Out-Null $samples = @() foreach ($line in $lines) { if ($null -eq $line -or $line.ToString().Trim() -eq "") { continue } try { $sample = $line.ToString() | ConvertFrom-Json $sample | Add-Member -NotePropertyName memory_usage_mib -NotePropertyValue (Convert-DockerMemoryUsageToMiB -MemoryUsage $sample.memory_usage) -Force $sample | Add-Member -NotePropertyName pids_count -NotePropertyValue (Convert-DockerPidsToInt -Pids $sample.pids) -Force $samples += $sample } catch { } } return $samples } function Get-ContainerStatsSampleSummary { param([object[]]$Samples) if ($null -eq $Samples -or $Samples.Count -eq 0) { return $null } $byName = @{} foreach ($sample in $Samples) { if ($sample.name -eq $null) { continue } if (-not $byName.ContainsKey($sample.name)) { $byName[$sample.name] = [ordered]@{ name = $sample.name role = $sample.role sample_count = 0 memory_usage_mib_start = $sample.memory_usage_mib memory_usage_mib_end = $sample.memory_usage_mib memory_usage_mib_max = $sample.memory_usage_mib pids_start = $sample.pids_count pids_end = $sample.pids_count pids_max = $sample.pids_count } } $entry = $byName[$sample.name] $entry.sample_count++ $entry.memory_usage_mib_end = $sample.memory_usage_mib if ($null -ne $sample.memory_usage_mib -and ($null -eq $entry.memory_usage_mib_max -or $sample.memory_usage_mib -gt $entry.memory_usage_mib_max)) { $entry.memory_usage_mib_max = $sample.memory_usage_mib } $entry.pids_end = $sample.pids_count if ($null -ne $sample.pids_count -and ($null -eq $entry.pids_max -or $sample.pids_count -gt $entry.pids_max)) { $entry.pids_max = $sample.pids_count } } $summary = @() foreach ($entry in $byName.Values) { $entry.memory_usage_mib_delta = if ($null -ne $entry.memory_usage_mib_start -and $null -ne $entry.memory_usage_mib_end) { [math]::Round($entry.memory_usage_mib_end - $entry.memory_usage_mib_start, 3) } else { $null } $entry.pids_delta = if ($null -ne $entry.pids_start -and $null -ne $entry.pids_end) { $entry.pids_end - $entry.pids_start } else { $null } $summary += [pscustomobject]$entry } return $summary } function Get-ContainerStatsSampleVerdictReasons { param([object[]]$SampleSummary) $reasons = @() if ($null -eq $SampleSummary) { return $reasons } foreach ($stat in $SampleSummary) { if ($MaxContainerMemoryMiB -gt 0 -and $null -ne $stat.memory_usage_mib_max -and $stat.memory_usage_mib_max -gt $MaxContainerMemoryMiB) { $reasons += "container_memory_mib_peak=$($stat.name):$($stat.memory_usage_mib_max)>$MaxContainerMemoryMiB" } if ($MaxContainerPids -gt 0 -and $null -ne $stat.pids_max -and $stat.pids_max -gt $MaxContainerPids) { $reasons += "container_pids_peak=$($stat.name):$($stat.pids_max)>$MaxContainerPids" } } return $reasons } function Get-ContainerStatsVerdictReasons { param([object[]]$ContainerStats) $reasons = @() foreach ($stat in $ContainerStats) { if ($stat.PSObject.Properties.Name -contains "error") { $reasons += "container_stats_error=$($stat.name)" continue } if ($MaxContainerMemoryMiB -gt 0 -and $null -ne $stat.memory_usage_mib -and $stat.memory_usage_mib -gt $MaxContainerMemoryMiB) { $reasons += "container_memory_mib=$($stat.name):$($stat.memory_usage_mib)>$MaxContainerMemoryMiB" } if ($MaxContainerPids -gt 0 -and $null -ne $stat.pids_count -and $stat.pids_count -gt $MaxContainerPids) { $reasons += "container_pids=$($stat.name):$($stat.pids_count)>$MaxContainerPids" } } return $reasons } try { Assert-SharedDockerContext if (-not $SkipBuild) { Invoke-Docker -Arguments @( "build", "-f", "$repoRoot\agents\rap-node-agent\Dockerfile.fabric-loadtest", "-t", $ImageTag, $repoRoot ) } if ($TuneUdpBuffers) { Set-HostUdpBuffers } $udpBuffers = Get-HostUdpBuffers Cleanup Invoke-Docker -Arguments @("network", "create", $networkName) | Out-Null for ($i = 0; $i -lt $Nodes; $i++) { $name = "$serverPrefix$i" Invoke-Docker -Arguments @( "run", "-d", "--name", $name, "--network", $networkName, "--cap-add", "NET_ADMIN", $ImageTag, "-mode", "server", "-listen", "0.0.0.0:19443", "-timeout", $ServerTimeout, "-concurrency", ([string]$Concurrency) ) | Out-Null } Start-Sleep -Seconds 3 $impairment = @() if ($ImpairTarget -ge 0 -and $ImpairTarget -lt $Nodes) { if ($ImpairAfter.Trim() -eq "") { $impairment = Set-ContainerImpairment -ContainerName "$serverPrefix$ImpairTarget" } else { $impairment = @("scheduled target=$serverPrefix$ImpairTarget after=$ImpairAfter delay=$ImpairDelay loss=$ImpairLoss rate=$ImpairRate") Start-Job -ScriptBlock { param($dockerContext, $containerName, $delay, $impairDelay, $impairLoss, $impairRate) if ($delay -match '^(\d+)ms$') { Start-Sleep -Milliseconds ([int]$Matches[1]) } elseif ($delay -match '^(\d+)s$') { Start-Sleep -Seconds ([int]$Matches[1]) } elseif ($delay -match '^(\d+)$') { Start-Sleep -Seconds ([int]$Matches[1]) } docker --context $dockerContext exec $containerName tc qdisc del dev eth0 root 2>$null | Out-Null $args = @("exec", $containerName, "tc", "qdisc", "add", "dev", "eth0", "root", "netem") if ($impairDelay.Trim() -ne "") { $args += @("delay", $impairDelay.Trim()) } if ($impairLoss.Trim() -ne "") { $args += @("loss", $impairLoss.Trim()) } if ($impairRate.Trim() -ne "") { $args += @("rate", $impairRate.Trim()) } docker --context $dockerContext @args | Out-Null } -ArgumentList $DockerContext, "$serverPrefix$ImpairTarget", $ImpairAfter, $ImpairDelay, $ImpairLoss, $ImpairRate | Out-Null } } $targets = @() for ($i = 0; $i -lt $Nodes; $i++) { $targets += "quic://$serverPrefix$i`:19443" } Assert-QuicTargets -Targets $targets $targetList = ($targets -join ",") if ($FailTarget -ge 0 -and $FailTarget -lt $Nodes) { Start-Job -ScriptBlock { param($dockerContext, $containerName, $delay) if ($delay -match '^(\d+)ms$') { Start-Sleep -Milliseconds ([int]$Matches[1]) } elseif ($delay -match '^(\d+)s$') { Start-Sleep -Seconds ([int]$Matches[1]) } elseif ($delay -match '^(\d+)$') { Start-Sleep -Seconds ([int]$Matches[1]) } docker --context $dockerContext rm -f $containerName | Out-Null } -ArgumentList $DockerContext, "$serverPrefix$FailTarget", $FailAfter | Out-Null } $clientArgs = @( "run", "--rm", "--name", $clientName, "--network", $networkName, $ImageTag, "-mode", "client", "-targets", $targetList, "-topology-profile", $TopologyProfile, "-soak=$($Soak.IsPresent.ToString().ToLowerInvariant())", "-streams", ([string]$Streams), "-concurrency", ([string]$Concurrency), "-bytes-per-stream", ([string]$BytesPerStream), "-control-every", ([string]$ControlEvery), "-control-bytes-per-stream", ([string]$ControlBytesPerStream), "-max-control-ack-p95-ms", ([string]$MaxControlAckP95Ms), "-payload-size", ([string]$PayloadSize), "-resource-sample-interval", $ResourceSampleInterval, "-stream-timeout", $StreamTimeout, "-ack-timeout", $AckTimeout, "-target-quarantine-ttl", $TargetQuarantineTTL, "-failure-quarantine-ttl", $FailureQuarantineTTL, "-fail-target", ([string]$FailTarget), "-impair-target", ([string]$ImpairTarget), "-pool-failover=true", "-timeout", $ClientTimeout ) if ($ProbeTargets) { $clientArgs += "-probe-targets=true" } if ($MaxTargetRttMs -gt 0) { $clientArgs += @("-max-target-rtt-ms", ([string]$MaxTargetRttMs)) } if ($MigrateSlowStreams) { $clientArgs += "-migrate-slow-streams=true" } if ($MaxAckMs -gt 0) { $clientArgs += @("-max-ack-ms", ([string]$MaxAckMs)) } if ($MaxAckP95Ms -gt 0) { $clientArgs += @("-max-ack-p95-ms", ([string]$MaxAckP95Ms)) } if ($MaxAckP99Ms -gt 0) { $clientArgs += @("-max-ack-p99-ms", ([string]$MaxAckP99Ms)) } if ($MaxTargetAckMs -gt 0) { $clientArgs += @("-max-target-ack-ms", ([string]$MaxTargetAckMs)) } if ($MaxSetupP95Ms -gt 0) { $clientArgs += @("-max-setup-p95-ms", ([string]$MaxSetupP95Ms)) } if ($MaxSetupP99Ms -gt 0) { $clientArgs += @("-max-setup-p99-ms", ([string]$MaxSetupP99Ms)) } if ($MaxRerouteP95Ms -gt 0) { $clientArgs += @("-max-reroute-p95-ms", ([string]$MaxRerouteP95Ms)) } if ($MaxRerouteP99Ms -gt 0) { $clientArgs += @("-max-reroute-p99-ms", ([string]$MaxRerouteP99Ms)) } if ($MaxGoroutineDelta -gt 0) { $clientArgs += @("-max-goroutine-delta", ([string]$MaxGoroutineDelta)) } if ($MaxHeapDeltaMB -gt 0) { $clientArgs += @("-max-heap-delta-mb", ([string]$MaxHeapDeltaMB)) } if ($MaxOpenFDDelta -gt 0) { $clientArgs += @("-max-open-fd-delta", ([string]$MaxOpenFDDelta)) } if ($MaxOpenFDs -gt 0) { $clientArgs += @("-max-open-fds", ([string]$MaxOpenFDs)) } if ($MinThroughputMbps -gt 0) { $clientArgs += @("-min-throughput-mbps", ([string]$MinThroughputMbps)) } if ($MinChannelChurnPerSec -gt 0) { $clientArgs += @("-min-channel-churn-per-sec", ([string]$MinChannelChurnPerSec)) } if ($Duration.Trim() -ne "") { $clientArgs += @("-duration", $Duration.Trim()) } $containerNamesForSampling = @($clientName) for ($i = 0; $i -lt $Nodes; $i++) { $containerNamesForSampling += "$serverPrefix$i" } $containerStatsSamplerJob = Start-ContainerStatsSampler -Names $containerNamesForSampling $containerStatsSamples = @() try { $clientOutput = Invoke-DockerText -Arguments $clientArgs } finally { $containerStatsSamples = @(Stop-ContainerStatsSampler -Job $containerStatsSamplerJob) } $rawText = ($clientOutput -join [Environment]::NewLine) $jsonStart = $rawText.IndexOf("{") if ($jsonStart -lt 0) { throw "fabric loadtest client did not emit JSON: $rawText" } $jsonText = $rawText.Substring($jsonStart) New-Item -ItemType Directory -Force -Path (Split-Path -Parent $ReportPath) | Out-Null Set-Content -LiteralPath $ReportPath -Value $jsonText -Encoding UTF8 $report = $jsonText | ConvertFrom-Json $verdictReasons = $null if ($report.PSObject.Properties.Name -contains "verdict_reasons") { $verdictReasons = $report.verdict_reasons } $targetStats = $null if ($report.PSObject.Properties.Name -contains "target_stats") { $targetStats = $report.target_stats } $targetProbes = $null if ($report.PSObject.Properties.Name -contains "target_probes") { $targetProbes = $report.target_probes } $excludedTargets = $null if ($report.PSObject.Properties.Name -contains "excluded_targets") { $excludedTargets = $report.excluded_targets } $migrationEvents = 0 if ($report.PSObject.Properties.Name -contains "migration_events") { $migrationEvents = $report.migration_events } $routePressure = $null if ($report.PSObject.Properties.Name -contains "route_pressure") { $routePressure = $report.route_pressure } $transportSnapshot = $null if ($report.PSObject.Properties.Name -contains "transport_snapshot") { $transportSnapshot = $report.transport_snapshot } $degradedTargets = $null if ($report.PSObject.Properties.Name -contains "degraded_targets") { $degradedTargets = $report.degraded_targets } $ackP95 = $null if ($report.PSObject.Properties.Name -contains "ack_p95_ms") { $ackP95 = $report.ack_p95_ms } $ackP99 = $null if ($report.PSObject.Properties.Name -contains "ack_p99_ms") { $ackP99 = $report.ack_p99_ms } $rerouteLatencyP95 = $null if ($report.PSObject.Properties.Name -contains "reroute_latency_p95_ms") { $rerouteLatencyP95 = $report.reroute_latency_p95_ms } $rerouteLatencyP99 = $null if ($report.PSObject.Properties.Name -contains "reroute_latency_p99_ms") { $rerouteLatencyP99 = $report.reroute_latency_p99_ms } $rerouteCauses = $null if ($report.PSObject.Properties.Name -contains "reroute_causes") { $rerouteCauses = $report.reroute_causes } $resourceSummary = $null if ($report.PSObject.Properties.Name -contains "resource_summary") { $resourceSummary = $report.resource_summary } $ackMismatchedStreams = 0 if ($report.PSObject.Properties.Name -contains "ack_mismatched_streams") { $ackMismatchedStreams = $report.ack_mismatched_streams } $ackIntegrityErrors = 0 if ($report.PSObject.Properties.Name -contains "ack_integrity_errors") { $ackIntegrityErrors = $report.ack_integrity_errors } $abandonedFrames = 0 if ($report.PSObject.Properties.Name -contains "abandoned_frames") { $abandonedFrames = $report.abandoned_frames } $controlAckP95 = $null if ($report.PSObject.Properties.Name -contains "control_ack_p95_ms") { $controlAckP95 = $report.control_ack_p95_ms } $bulkAckP95 = $null if ($report.PSObject.Properties.Name -contains "bulk_ack_p95_ms") { $bulkAckP95 = $report.bulk_ack_p95_ms } $containerStats = Get-FabricContainerStats if ($ContainerStatsSampleInterval.Trim() -ne "" -and $containerStatsSamples.Count -eq 0 -and $containerStats.Count -gt 0) { $observedAt = (Get-Date).ToUniversalTime().ToString("o") foreach ($stat in $containerStats) { if ($stat.PSObject.Properties.Name -contains "error") { continue } $containerStatsSamples += [pscustomobject]@{ observed_at = $observedAt name = $stat.name id = $stat.id cpu_percent = $stat.cpu_percent memory_percent = $stat.memory_percent memory_usage = $stat.memory_usage memory_usage_mib = $stat.memory_usage_mib net_io = $stat.net_io block_io = $stat.block_io pids = $stat.pids pids_count = $stat.pids_count role = $stat.role } } } $containerStatsSampleSummary = Get-ContainerStatsSampleSummary -Samples $containerStatsSamples if ($containerStatsSamples.Count -gt 0) { New-Item -ItemType Directory -Force -Path (Split-Path -Parent $ContainerStatsSamplesPath) | Out-Null Set-Content -LiteralPath $ContainerStatsSamplesPath -Value ($containerStatsSamples | ConvertTo-Json -Depth 20) -Encoding UTF8 } $containerVerdictReasons = @(Get-ContainerStatsVerdictReasons -ContainerStats $containerStats) $containerSampleVerdictReasons = @(Get-ContainerStatsSampleVerdictReasons -SampleSummary $containerStatsSampleSummary) $combinedVerdictReasons = @() if ($null -ne $verdictReasons) { $combinedVerdictReasons += $verdictReasons } $combinedVerdictReasons += $containerVerdictReasons $combinedVerdictReasons += $containerSampleVerdictReasons $summaryVerdict = $report.verdict if ($containerVerdictReasons.Count -gt 0 -or $containerSampleVerdictReasons.Count -gt 0) { $summaryVerdict = "fail" } $summary = [pscustomobject]@{ run_id = $runId report_path = $ReportPath summary_path = $SummaryPath docker_context = $DockerContext nodes = $Nodes topology_profile = $TopologyProfile soak = $Soak.IsPresent targets = $targets total_streams = $report.total_streams successful_streams = $report.successful_streams failed_streams = $report.failed_streams ack_mismatched_streams = $ackMismatchedStreams ack_integrity_errors = $ackIntegrityErrors abandoned_frames = $abandonedFrames failover_events = $report.failover_events migration_events = $migrationEvents channel_opens = $report.channel_opens channel_closes = $report.channel_closes channel_leaks = $report.channel_leaks channel_churn_per_sec = $report.channel_churn_per_sec bytes_sent = $report.bytes_sent throughput_bps = $report.throughput_bps setup_latency_p95_ms = $report.setup_latency_p95_ms channel_open_p95_ms = $report.channel_open_p95_ms stream_duration_p95_ms = $report.stream_duration_p95_ms ack_p95_ms = $ackP95 ack_p99_ms = $ackP99 reroute_latency_p95_ms = $rerouteLatencyP95 reroute_latency_p99_ms = $rerouteLatencyP99 route_attempts_total = $report.route_attempts_total reroute_causes = $rerouteCauses resource_summary = $resourceSummary container_stats = $containerStats container_stats_samples_path = $(if ($containerStatsSamples.Count -gt 0) { $ContainerStatsSamplesPath } else { $null }) container_stats_samples_count = $containerStatsSamples.Count container_stats_sample_summary = $containerStatsSampleSummary control_streams = $report.control_streams bulk_streams = $report.bulk_streams control_ack_p95_ms = $controlAckP95 bulk_ack_p95_ms = $bulkAckP95 verdict = $summaryVerdict verdict_reasons = $combinedVerdictReasons udp_buffers = $udpBuffers impairment = $impairment target_probes = $targetProbes excluded_targets = $excludedTargets degraded_targets = $degradedTargets target_streams = $report.target_streams target_bytes = $report.target_bytes target_stats = $targetStats route_pressure = $routePressure transport_snapshot = $transportSnapshot } $summaryJson = $summary | ConvertTo-Json -Depth 20 New-Item -ItemType Directory -Force -Path (Split-Path -Parent $SummaryPath) | Out-Null Set-Content -LiteralPath $SummaryPath -Value $summaryJson -Encoding UTF8 $summaryJson if ($summaryVerdict -ne "pass") { $reasonText = "" if ($combinedVerdictReasons.Count -gt 0) { $reasonText = ($combinedVerdictReasons -join "; ") } throw "fabric loadtest verdict failed: $summaryVerdict $reasonText" } } finally { Cleanup }