840 lines
31 KiB
PowerShell
840 lines
31 KiB
PowerShell
param(
|
|
[string]$DockerContext = "test-docker",
|
|
[string]$ImageTag = "rap-fabric-loadtest:dev",
|
|
[int]$Nodes = 4,
|
|
[int]$Streams = 400,
|
|
[int]$Concurrency = 64,
|
|
[int64]$BytesPerStream = 262144,
|
|
[string]$Duration = "",
|
|
[string]$ClientTimeout = "10m",
|
|
[string]$ServerTimeout = "",
|
|
[string]$StreamTimeout = "30s",
|
|
[string]$AckTimeout = "2s",
|
|
[string]$TargetQuarantineTTL = "30s",
|
|
[string]$FailureQuarantineTTL = "5m",
|
|
[string]$TopologyProfile = "",
|
|
[switch]$Soak,
|
|
[int]$ControlEvery = 0,
|
|
[int64]$ControlBytesPerStream = 4096,
|
|
[int64]$MaxControlAckP95Ms = 100,
|
|
[int]$MaxGoroutineDelta = 0,
|
|
[int64]$MaxHeapDeltaMB = 0,
|
|
[int]$MaxOpenFDDelta = 0,
|
|
[int]$MaxOpenFDs = 0,
|
|
[int64]$MinThroughputMbps = 0,
|
|
[int64]$MinChannelChurnPerSec = 0,
|
|
[int64]$MaxContainerMemoryMiB = 0,
|
|
[int]$MaxContainerPids = 0,
|
|
[int]$PayloadSize = 16384,
|
|
[int]$FailTarget = 0,
|
|
[string]$FailAfter = "1s",
|
|
[int]$ImpairTarget = -1,
|
|
[string]$ImpairDelay = "",
|
|
[string]$ImpairLoss = "",
|
|
[string]$ImpairRate = "",
|
|
[string]$ImpairAfter = "",
|
|
[switch]$ProbeTargets,
|
|
[int64]$MaxTargetRttMs = 0,
|
|
[switch]$MigrateSlowStreams,
|
|
[int64]$MaxAckMs = 0,
|
|
[int64]$MaxAckP95Ms = 0,
|
|
[int64]$MaxAckP99Ms = 0,
|
|
[int64]$MaxTargetAckMs = 0,
|
|
[int64]$MaxSetupP95Ms = 200,
|
|
[int64]$MaxSetupP99Ms = 0,
|
|
[int64]$MaxRerouteP95Ms = 0,
|
|
[int64]$MaxRerouteP99Ms = 0,
|
|
[string]$ResourceSampleInterval = "1s",
|
|
[string]$ContainerStatsSampleInterval = "",
|
|
[string]$ContainerStatsSamplesPath = "",
|
|
[int]$UdpBufferBytes = 7500000,
|
|
[string]$ArtifactDir = "",
|
|
[string]$ReportPath = "",
|
|
[string]$SummaryPath = "",
|
|
[switch]$TuneUdpBuffers,
|
|
[switch]$KeepRunning,
|
|
[switch]$SkipBuild
|
|
)
|
|
|
|
Set-StrictMode -Version Latest
|
|
$ErrorActionPreference = "Stop"
|
|
|
|
$repoRoot = (Resolve-Path (Join-Path $PSScriptRoot "..\..")).ProviderPath
|
|
$runId = "fabric-loadtest-" + (Get-Date -Format "yyyyMMdd-HHmmss")
|
|
$networkName = "rap-$runId-net"
|
|
$serverPrefix = "rap-$runId-server-"
|
|
$clientName = "rap-$runId-client"
|
|
if ($ArtifactDir.Trim() -eq "") {
|
|
$ArtifactDir = Join-Path $repoRoot "artifacts\fabric-loadtest"
|
|
}
|
|
if ($ReportPath.Trim() -eq "") {
|
|
$ReportPath = Join-Path $ArtifactDir "$runId-report.json"
|
|
}
|
|
if ($SummaryPath.Trim() -eq "") {
|
|
$SummaryPath = Join-Path $ArtifactDir "$runId-summary.json"
|
|
}
|
|
if ($ContainerStatsSamplesPath.Trim() -eq "") {
|
|
$ContainerStatsSamplesPath = Join-Path $ArtifactDir "$runId-container-stats-samples.json"
|
|
}
|
|
if ($ServerTimeout.Trim() -eq "") {
|
|
$ServerTimeout = $ClientTimeout
|
|
}
|
|
|
|
function Invoke-Docker {
|
|
param([string[]]$Arguments)
|
|
docker --context $DockerContext @Arguments
|
|
if ($LASTEXITCODE -ne 0) {
|
|
throw "docker --context $DockerContext $($Arguments -join ' ') failed with exit code $LASTEXITCODE"
|
|
}
|
|
}
|
|
|
|
function Invoke-DockerText {
|
|
param([string[]]$Arguments)
|
|
$previousErrorActionPreference = $ErrorActionPreference
|
|
$ErrorActionPreference = "Continue"
|
|
try {
|
|
$output = docker --context $DockerContext @Arguments 2>&1 | ForEach-Object { $_.ToString() }
|
|
}
|
|
finally {
|
|
$ErrorActionPreference = $previousErrorActionPreference
|
|
}
|
|
if ($LASTEXITCODE -ne 0) {
|
|
throw "docker --context $DockerContext $($Arguments -join ' ') failed with exit code $LASTEXITCODE`n$($output -join [Environment]::NewLine)"
|
|
}
|
|
return $output
|
|
}
|
|
|
|
function Assert-SharedDockerContext {
|
|
$normalized = $DockerContext.Trim().ToLowerInvariant()
|
|
if ($normalized -eq "" -or $normalized -eq "default" -or $normalized -eq "desktop-linux" -or $normalized -eq "docker-desktop") {
|
|
throw "fabric loadtest must use the shared Docker host context, not local Docker Desktop. Use -DockerContext test-docker, docker-test, or test-ubuntu."
|
|
}
|
|
}
|
|
|
|
function Assert-QuicTargets {
|
|
param([string[]]$Targets)
|
|
$invalid = @()
|
|
foreach ($target in $Targets) {
|
|
$trimmed = ([string]$target).Trim()
|
|
if ($trimmed -eq "" -or -not $trimmed.ToLowerInvariant().StartsWith("quic://")) {
|
|
if ($trimmed -eq "") {
|
|
$invalid += "<empty>"
|
|
}
|
|
else {
|
|
$invalid += $trimmed
|
|
}
|
|
}
|
|
}
|
|
if ($invalid.Count -gt 0) {
|
|
throw "fabric loadtest targets must be QUIC-only: $($invalid -join ', ')"
|
|
}
|
|
}
|
|
|
|
function Cleanup {
|
|
if ($KeepRunning) {
|
|
return
|
|
}
|
|
$previousErrorActionPreference = $ErrorActionPreference
|
|
$ErrorActionPreference = "Continue"
|
|
try {
|
|
docker --context $DockerContext rm -f $clientName 2>$null | Out-Null
|
|
for ($i = 0; $i -lt $Nodes; $i++) {
|
|
docker --context $DockerContext rm -f "$serverPrefix$i" 2>$null | Out-Null
|
|
}
|
|
docker --context $DockerContext network rm $networkName 2>$null | Out-Null
|
|
}
|
|
finally {
|
|
$ErrorActionPreference = $previousErrorActionPreference
|
|
}
|
|
}
|
|
|
|
function Invoke-HostSysctl {
|
|
param([string[]]$Arguments)
|
|
$dockerArgs = @(
|
|
"run", "--rm",
|
|
"--privileged",
|
|
"--network", "host",
|
|
"--entrypoint", "sysctl",
|
|
$ImageTag
|
|
) + $Arguments
|
|
return Invoke-DockerText -Arguments $dockerArgs
|
|
}
|
|
|
|
function Set-HostUdpBuffers {
|
|
Invoke-HostSysctl -Arguments @(
|
|
"-w",
|
|
"net.core.rmem_max=$UdpBufferBytes",
|
|
"net.core.wmem_max=$UdpBufferBytes",
|
|
"net.core.rmem_default=$UdpBufferBytes",
|
|
"net.core.wmem_default=$UdpBufferBytes"
|
|
) | Out-Null
|
|
}
|
|
|
|
function Get-HostUdpBuffers {
|
|
$lines = Invoke-HostSysctl -Arguments @(
|
|
"net.core.rmem_max",
|
|
"net.core.wmem_max",
|
|
"net.core.rmem_default",
|
|
"net.core.wmem_default"
|
|
)
|
|
$values = [ordered]@{}
|
|
foreach ($line in $lines) {
|
|
if ($line -match '^(?<key>[^=]+)=\s*(?<value>\d+)$') {
|
|
$values[$Matches["key"].Trim()] = [int64]$Matches["value"]
|
|
}
|
|
}
|
|
return $values
|
|
}
|
|
|
|
function Set-ContainerImpairment {
|
|
param([string]$ContainerName)
|
|
$netem = @()
|
|
if ($ImpairDelay.Trim() -ne "") {
|
|
$netem += @("delay", $ImpairDelay.Trim())
|
|
}
|
|
if ($ImpairLoss.Trim() -ne "") {
|
|
$netem += @("loss", $ImpairLoss.Trim())
|
|
}
|
|
if ($ImpairRate.Trim() -ne "") {
|
|
$netem += @("rate", $ImpairRate.Trim())
|
|
}
|
|
if ($netem.Count -eq 0) {
|
|
return @()
|
|
}
|
|
$previousErrorActionPreference = $ErrorActionPreference
|
|
$ErrorActionPreference = "Continue"
|
|
try {
|
|
docker --context $DockerContext exec $ContainerName tc qdisc del dev eth0 root 2>$null | Out-Null
|
|
}
|
|
finally {
|
|
$ErrorActionPreference = $previousErrorActionPreference
|
|
}
|
|
Invoke-Docker -Arguments (@("exec", $ContainerName, "tc", "qdisc", "add", "dev", "eth0", "root", "netem") + $netem)
|
|
return Invoke-DockerText -Arguments @("exec", $ContainerName, "tc", "qdisc", "show", "dev", "eth0")
|
|
}
|
|
|
|
function Get-FabricContainerStats {
|
|
$names = @($clientName)
|
|
for ($i = 0; $i -lt $Nodes; $i++) {
|
|
$names += "$serverPrefix$i"
|
|
}
|
|
$stats = @()
|
|
foreach ($name in $names) {
|
|
$inspect = $null
|
|
try {
|
|
$inspectText = Invoke-DockerText -Arguments @("inspect", $name)
|
|
$inspect = @(($inspectText -join [Environment]::NewLine) | ConvertFrom-Json)
|
|
}
|
|
catch {
|
|
continue
|
|
}
|
|
if ($null -eq $inspect -or $inspect.Count -eq 0 -or $inspect[0].State.Running -ne $true) {
|
|
continue
|
|
}
|
|
try {
|
|
$line = @(Invoke-DockerText -Arguments @("stats", "--no-stream", "--format", "{{json .}}", $name))
|
|
if ($line.Count -gt 0) {
|
|
$item = ($line[0] | ConvertFrom-Json)
|
|
$memoryUsageMiB = Convert-DockerMemoryUsageToMiB -MemoryUsage $item.MemUsage
|
|
$pidCount = Convert-DockerPidsToInt -Pids $item.PIDs
|
|
$stats += [pscustomobject]@{
|
|
name = $item.Name
|
|
id = $item.ID
|
|
cpu_percent = $item.CPUPerc
|
|
memory_percent = $item.MemPerc
|
|
memory_usage = $item.MemUsage
|
|
memory_usage_mib = $memoryUsageMiB
|
|
net_io = $item.NetIO
|
|
block_io = $item.BlockIO
|
|
pids = $item.PIDs
|
|
pids_count = $pidCount
|
|
role = $(if ($item.Name -eq $clientName) { "client" } else { "server" })
|
|
}
|
|
}
|
|
}
|
|
catch {
|
|
$stats += [pscustomobject]@{
|
|
name = $name
|
|
error = $_.Exception.Message
|
|
}
|
|
}
|
|
}
|
|
return $stats
|
|
}
|
|
|
|
function Convert-DockerMemoryUsageToMiB {
|
|
param([string]$MemoryUsage)
|
|
if ($MemoryUsage.Trim() -eq "") {
|
|
return $null
|
|
}
|
|
$used = ($MemoryUsage -split '/')[0].Trim()
|
|
if ($used -notmatch '^(?<value>[0-9.]+)\s*(?<unit>[A-Za-z]+)$') {
|
|
return $null
|
|
}
|
|
$value = [double]$Matches["value"]
|
|
switch ($Matches["unit"].ToLowerInvariant()) {
|
|
"b" { return [math]::Round($value / 1MB, 3) }
|
|
"kb" { return [math]::Round($value / 1024, 3) }
|
|
"kib" { return [math]::Round($value / 1024, 3) }
|
|
"mb" { return [math]::Round($value, 3) }
|
|
"mib" { return [math]::Round($value, 3) }
|
|
"gb" { return [math]::Round($value * 1024, 3) }
|
|
"gib" { return [math]::Round($value * 1024, 3) }
|
|
default { return $null }
|
|
}
|
|
}
|
|
|
|
function Convert-DockerPidsToInt {
|
|
param([string]$Pids)
|
|
$value = 0
|
|
if ([int]::TryParse($Pids, [ref]$value)) {
|
|
return $value
|
|
}
|
|
return $null
|
|
}
|
|
|
|
function Convert-DurationToMilliseconds {
|
|
param([string]$Duration, [int]$DefaultMilliseconds)
|
|
$value = $Duration.Trim()
|
|
if ($value -eq "") {
|
|
return $DefaultMilliseconds
|
|
}
|
|
if ($value -match '^(?<n>[0-9]+)ms$') {
|
|
return [int]$Matches["n"]
|
|
}
|
|
if ($value -match '^(?<n>[0-9]+)s$') {
|
|
return [int]$Matches["n"] * 1000
|
|
}
|
|
if ($value -match '^(?<n>[0-9]+)m$') {
|
|
return [int]$Matches["n"] * 60 * 1000
|
|
}
|
|
if ($value -match '^(?<n>[0-9]+)$') {
|
|
return [int]$Matches["n"] * 1000
|
|
}
|
|
return $DefaultMilliseconds
|
|
}
|
|
|
|
function Start-ContainerStatsSampler {
|
|
param([string[]]$Names)
|
|
if ($ContainerStatsSampleInterval.Trim() -eq "") {
|
|
return $null
|
|
}
|
|
$intervalMs = Convert-DurationToMilliseconds -Duration $ContainerStatsSampleInterval -DefaultMilliseconds 10000
|
|
if ($intervalMs -le 0) {
|
|
return $null
|
|
}
|
|
return Start-Job -ScriptBlock {
|
|
param($dockerContext, $names, $intervalMs)
|
|
while ($true) {
|
|
$observedAt = (Get-Date).ToUniversalTime().ToString("o")
|
|
foreach ($name in $names) {
|
|
$running = docker --context $dockerContext inspect -f "{{.State.Running}}" $name 2>$null
|
|
if ($LASTEXITCODE -ne 0 -or $running -ne "true") {
|
|
continue
|
|
}
|
|
$line = docker --context $dockerContext stats --no-stream --format "{{json .}}" $name 2>$null
|
|
if ($LASTEXITCODE -ne 0 -or $null -eq $line -or $line.Trim() -eq "") {
|
|
continue
|
|
}
|
|
try {
|
|
$item = $line | ConvertFrom-Json
|
|
[pscustomobject]@{
|
|
observed_at = $observedAt
|
|
name = $item.Name
|
|
id = $item.ID
|
|
cpu_percent = $item.CPUPerc
|
|
memory_percent = $item.MemPerc
|
|
memory_usage = $item.MemUsage
|
|
net_io = $item.NetIO
|
|
block_io = $item.BlockIO
|
|
pids = $item.PIDs
|
|
role = $(if ($item.Name -like "*-client") { "client" } else { "server" })
|
|
} | ConvertTo-Json -Compress
|
|
}
|
|
catch {
|
|
}
|
|
}
|
|
Start-Sleep -Milliseconds $intervalMs
|
|
}
|
|
} -ArgumentList $DockerContext, $Names, $intervalMs
|
|
}
|
|
|
|
function Stop-ContainerStatsSampler {
|
|
param($Job)
|
|
if ($null -eq $Job) {
|
|
return @()
|
|
}
|
|
Stop-Job -Job $Job -ErrorAction SilentlyContinue | Out-Null
|
|
$lines = @(Receive-Job -Job $Job -ErrorAction SilentlyContinue)
|
|
Remove-Job -Job $Job -Force -ErrorAction SilentlyContinue | Out-Null
|
|
$samples = @()
|
|
foreach ($line in $lines) {
|
|
if ($null -eq $line -or $line.ToString().Trim() -eq "") {
|
|
continue
|
|
}
|
|
try {
|
|
$sample = $line.ToString() | ConvertFrom-Json
|
|
$sample | Add-Member -NotePropertyName memory_usage_mib -NotePropertyValue (Convert-DockerMemoryUsageToMiB -MemoryUsage $sample.memory_usage) -Force
|
|
$sample | Add-Member -NotePropertyName pids_count -NotePropertyValue (Convert-DockerPidsToInt -Pids $sample.pids) -Force
|
|
$samples += $sample
|
|
}
|
|
catch {
|
|
}
|
|
}
|
|
return $samples
|
|
}
|
|
|
|
function Get-ContainerStatsSampleSummary {
|
|
param([object[]]$Samples)
|
|
if ($null -eq $Samples -or $Samples.Count -eq 0) {
|
|
return $null
|
|
}
|
|
$byName = @{}
|
|
foreach ($sample in $Samples) {
|
|
if ($sample.name -eq $null) {
|
|
continue
|
|
}
|
|
if (-not $byName.ContainsKey($sample.name)) {
|
|
$byName[$sample.name] = [ordered]@{
|
|
name = $sample.name
|
|
role = $sample.role
|
|
sample_count = 0
|
|
memory_usage_mib_start = $sample.memory_usage_mib
|
|
memory_usage_mib_end = $sample.memory_usage_mib
|
|
memory_usage_mib_max = $sample.memory_usage_mib
|
|
pids_start = $sample.pids_count
|
|
pids_end = $sample.pids_count
|
|
pids_max = $sample.pids_count
|
|
}
|
|
}
|
|
$entry = $byName[$sample.name]
|
|
$entry.sample_count++
|
|
$entry.memory_usage_mib_end = $sample.memory_usage_mib
|
|
if ($null -ne $sample.memory_usage_mib -and ($null -eq $entry.memory_usage_mib_max -or $sample.memory_usage_mib -gt $entry.memory_usage_mib_max)) {
|
|
$entry.memory_usage_mib_max = $sample.memory_usage_mib
|
|
}
|
|
$entry.pids_end = $sample.pids_count
|
|
if ($null -ne $sample.pids_count -and ($null -eq $entry.pids_max -or $sample.pids_count -gt $entry.pids_max)) {
|
|
$entry.pids_max = $sample.pids_count
|
|
}
|
|
}
|
|
$summary = @()
|
|
foreach ($entry in $byName.Values) {
|
|
$entry.memory_usage_mib_delta = if ($null -ne $entry.memory_usage_mib_start -and $null -ne $entry.memory_usage_mib_end) { [math]::Round($entry.memory_usage_mib_end - $entry.memory_usage_mib_start, 3) } else { $null }
|
|
$entry.pids_delta = if ($null -ne $entry.pids_start -and $null -ne $entry.pids_end) { $entry.pids_end - $entry.pids_start } else { $null }
|
|
$summary += [pscustomobject]$entry
|
|
}
|
|
return $summary
|
|
}
|
|
|
|
function Get-ContainerStatsSampleVerdictReasons {
|
|
param([object[]]$SampleSummary)
|
|
$reasons = @()
|
|
if ($null -eq $SampleSummary) {
|
|
return $reasons
|
|
}
|
|
foreach ($stat in $SampleSummary) {
|
|
if ($MaxContainerMemoryMiB -gt 0 -and $null -ne $stat.memory_usage_mib_max -and $stat.memory_usage_mib_max -gt $MaxContainerMemoryMiB) {
|
|
$reasons += "container_memory_mib_peak=$($stat.name):$($stat.memory_usage_mib_max)>$MaxContainerMemoryMiB"
|
|
}
|
|
if ($MaxContainerPids -gt 0 -and $null -ne $stat.pids_max -and $stat.pids_max -gt $MaxContainerPids) {
|
|
$reasons += "container_pids_peak=$($stat.name):$($stat.pids_max)>$MaxContainerPids"
|
|
}
|
|
}
|
|
return $reasons
|
|
}
|
|
|
|
function Get-ContainerStatsVerdictReasons {
|
|
param([object[]]$ContainerStats)
|
|
$reasons = @()
|
|
foreach ($stat in $ContainerStats) {
|
|
if ($stat.PSObject.Properties.Name -contains "error") {
|
|
$reasons += "container_stats_error=$($stat.name)"
|
|
continue
|
|
}
|
|
if ($MaxContainerMemoryMiB -gt 0 -and $null -ne $stat.memory_usage_mib -and $stat.memory_usage_mib -gt $MaxContainerMemoryMiB) {
|
|
$reasons += "container_memory_mib=$($stat.name):$($stat.memory_usage_mib)>$MaxContainerMemoryMiB"
|
|
}
|
|
if ($MaxContainerPids -gt 0 -and $null -ne $stat.pids_count -and $stat.pids_count -gt $MaxContainerPids) {
|
|
$reasons += "container_pids=$($stat.name):$($stat.pids_count)>$MaxContainerPids"
|
|
}
|
|
}
|
|
return $reasons
|
|
}
|
|
|
|
try {
|
|
Assert-SharedDockerContext
|
|
|
|
if (-not $SkipBuild) {
|
|
Invoke-Docker -Arguments @(
|
|
"build",
|
|
"-f", "$repoRoot\agents\rap-node-agent\Dockerfile.fabric-loadtest",
|
|
"-t", $ImageTag,
|
|
$repoRoot
|
|
)
|
|
}
|
|
|
|
if ($TuneUdpBuffers) {
|
|
Set-HostUdpBuffers
|
|
}
|
|
$udpBuffers = Get-HostUdpBuffers
|
|
|
|
Cleanup
|
|
Invoke-Docker -Arguments @("network", "create", $networkName) | Out-Null
|
|
|
|
for ($i = 0; $i -lt $Nodes; $i++) {
|
|
$name = "$serverPrefix$i"
|
|
Invoke-Docker -Arguments @(
|
|
"run", "-d",
|
|
"--name", $name,
|
|
"--network", $networkName,
|
|
"--cap-add", "NET_ADMIN",
|
|
$ImageTag,
|
|
"-mode", "server",
|
|
"-listen", "0.0.0.0:19443",
|
|
"-timeout", $ServerTimeout,
|
|
"-concurrency", ([string]$Concurrency)
|
|
) | Out-Null
|
|
}
|
|
|
|
Start-Sleep -Seconds 3
|
|
|
|
$impairment = @()
|
|
if ($ImpairTarget -ge 0 -and $ImpairTarget -lt $Nodes) {
|
|
if ($ImpairAfter.Trim() -eq "") {
|
|
$impairment = Set-ContainerImpairment -ContainerName "$serverPrefix$ImpairTarget"
|
|
}
|
|
else {
|
|
$impairment = @("scheduled target=$serverPrefix$ImpairTarget after=$ImpairAfter delay=$ImpairDelay loss=$ImpairLoss rate=$ImpairRate")
|
|
Start-Job -ScriptBlock {
|
|
param($dockerContext, $containerName, $delay, $impairDelay, $impairLoss, $impairRate)
|
|
if ($delay -match '^(\d+)ms$') {
|
|
Start-Sleep -Milliseconds ([int]$Matches[1])
|
|
}
|
|
elseif ($delay -match '^(\d+)s$') {
|
|
Start-Sleep -Seconds ([int]$Matches[1])
|
|
}
|
|
elseif ($delay -match '^(\d+)$') {
|
|
Start-Sleep -Seconds ([int]$Matches[1])
|
|
}
|
|
docker --context $dockerContext exec $containerName tc qdisc del dev eth0 root 2>$null | Out-Null
|
|
$args = @("exec", $containerName, "tc", "qdisc", "add", "dev", "eth0", "root", "netem")
|
|
if ($impairDelay.Trim() -ne "") { $args += @("delay", $impairDelay.Trim()) }
|
|
if ($impairLoss.Trim() -ne "") { $args += @("loss", $impairLoss.Trim()) }
|
|
if ($impairRate.Trim() -ne "") { $args += @("rate", $impairRate.Trim()) }
|
|
docker --context $dockerContext @args | Out-Null
|
|
} -ArgumentList $DockerContext, "$serverPrefix$ImpairTarget", $ImpairAfter, $ImpairDelay, $ImpairLoss, $ImpairRate | Out-Null
|
|
}
|
|
}
|
|
|
|
$targets = @()
|
|
for ($i = 0; $i -lt $Nodes; $i++) {
|
|
$targets += "quic://$serverPrefix$i`:19443"
|
|
}
|
|
Assert-QuicTargets -Targets $targets
|
|
$targetList = ($targets -join ",")
|
|
|
|
if ($FailTarget -ge 0 -and $FailTarget -lt $Nodes) {
|
|
Start-Job -ScriptBlock {
|
|
param($dockerContext, $containerName, $delay)
|
|
if ($delay -match '^(\d+)ms$') {
|
|
Start-Sleep -Milliseconds ([int]$Matches[1])
|
|
}
|
|
elseif ($delay -match '^(\d+)s$') {
|
|
Start-Sleep -Seconds ([int]$Matches[1])
|
|
}
|
|
elseif ($delay -match '^(\d+)$') {
|
|
Start-Sleep -Seconds ([int]$Matches[1])
|
|
}
|
|
docker --context $dockerContext rm -f $containerName | Out-Null
|
|
} -ArgumentList $DockerContext, "$serverPrefix$FailTarget", $FailAfter | Out-Null
|
|
}
|
|
|
|
$clientArgs = @(
|
|
"run", "--rm",
|
|
"--name", $clientName,
|
|
"--network", $networkName,
|
|
$ImageTag,
|
|
"-mode", "client",
|
|
"-targets", $targetList,
|
|
"-topology-profile", $TopologyProfile,
|
|
"-soak=$($Soak.IsPresent.ToString().ToLowerInvariant())",
|
|
"-streams", ([string]$Streams),
|
|
"-concurrency", ([string]$Concurrency),
|
|
"-bytes-per-stream", ([string]$BytesPerStream),
|
|
"-control-every", ([string]$ControlEvery),
|
|
"-control-bytes-per-stream", ([string]$ControlBytesPerStream),
|
|
"-max-control-ack-p95-ms", ([string]$MaxControlAckP95Ms),
|
|
"-payload-size", ([string]$PayloadSize),
|
|
"-resource-sample-interval", $ResourceSampleInterval,
|
|
"-stream-timeout", $StreamTimeout,
|
|
"-ack-timeout", $AckTimeout,
|
|
"-target-quarantine-ttl", $TargetQuarantineTTL,
|
|
"-failure-quarantine-ttl", $FailureQuarantineTTL,
|
|
"-fail-target", ([string]$FailTarget),
|
|
"-impair-target", ([string]$ImpairTarget),
|
|
"-pool-failover=true",
|
|
"-timeout", $ClientTimeout
|
|
)
|
|
if ($ProbeTargets) {
|
|
$clientArgs += "-probe-targets=true"
|
|
}
|
|
if ($MaxTargetRttMs -gt 0) {
|
|
$clientArgs += @("-max-target-rtt-ms", ([string]$MaxTargetRttMs))
|
|
}
|
|
if ($MigrateSlowStreams) {
|
|
$clientArgs += "-migrate-slow-streams=true"
|
|
}
|
|
if ($MaxAckMs -gt 0) {
|
|
$clientArgs += @("-max-ack-ms", ([string]$MaxAckMs))
|
|
}
|
|
if ($MaxAckP95Ms -gt 0) {
|
|
$clientArgs += @("-max-ack-p95-ms", ([string]$MaxAckP95Ms))
|
|
}
|
|
if ($MaxAckP99Ms -gt 0) {
|
|
$clientArgs += @("-max-ack-p99-ms", ([string]$MaxAckP99Ms))
|
|
}
|
|
if ($MaxTargetAckMs -gt 0) {
|
|
$clientArgs += @("-max-target-ack-ms", ([string]$MaxTargetAckMs))
|
|
}
|
|
if ($MaxSetupP95Ms -gt 0) {
|
|
$clientArgs += @("-max-setup-p95-ms", ([string]$MaxSetupP95Ms))
|
|
}
|
|
if ($MaxSetupP99Ms -gt 0) {
|
|
$clientArgs += @("-max-setup-p99-ms", ([string]$MaxSetupP99Ms))
|
|
}
|
|
if ($MaxRerouteP95Ms -gt 0) {
|
|
$clientArgs += @("-max-reroute-p95-ms", ([string]$MaxRerouteP95Ms))
|
|
}
|
|
if ($MaxRerouteP99Ms -gt 0) {
|
|
$clientArgs += @("-max-reroute-p99-ms", ([string]$MaxRerouteP99Ms))
|
|
}
|
|
if ($MaxGoroutineDelta -gt 0) {
|
|
$clientArgs += @("-max-goroutine-delta", ([string]$MaxGoroutineDelta))
|
|
}
|
|
if ($MaxHeapDeltaMB -gt 0) {
|
|
$clientArgs += @("-max-heap-delta-mb", ([string]$MaxHeapDeltaMB))
|
|
}
|
|
if ($MaxOpenFDDelta -gt 0) {
|
|
$clientArgs += @("-max-open-fd-delta", ([string]$MaxOpenFDDelta))
|
|
}
|
|
if ($MaxOpenFDs -gt 0) {
|
|
$clientArgs += @("-max-open-fds", ([string]$MaxOpenFDs))
|
|
}
|
|
if ($MinThroughputMbps -gt 0) {
|
|
$clientArgs += @("-min-throughput-mbps", ([string]$MinThroughputMbps))
|
|
}
|
|
if ($MinChannelChurnPerSec -gt 0) {
|
|
$clientArgs += @("-min-channel-churn-per-sec", ([string]$MinChannelChurnPerSec))
|
|
}
|
|
if ($Duration.Trim() -ne "") {
|
|
$clientArgs += @("-duration", $Duration.Trim())
|
|
}
|
|
$containerNamesForSampling = @($clientName)
|
|
for ($i = 0; $i -lt $Nodes; $i++) {
|
|
$containerNamesForSampling += "$serverPrefix$i"
|
|
}
|
|
$containerStatsSamplerJob = Start-ContainerStatsSampler -Names $containerNamesForSampling
|
|
$containerStatsSamples = @()
|
|
try {
|
|
$clientOutput = Invoke-DockerText -Arguments $clientArgs
|
|
}
|
|
finally {
|
|
$containerStatsSamples = @(Stop-ContainerStatsSampler -Job $containerStatsSamplerJob)
|
|
}
|
|
|
|
$rawText = ($clientOutput -join [Environment]::NewLine)
|
|
$jsonStart = $rawText.IndexOf("{")
|
|
if ($jsonStart -lt 0) {
|
|
throw "fabric loadtest client did not emit JSON: $rawText"
|
|
}
|
|
$jsonText = $rawText.Substring($jsonStart)
|
|
New-Item -ItemType Directory -Force -Path (Split-Path -Parent $ReportPath) | Out-Null
|
|
Set-Content -LiteralPath $ReportPath -Value $jsonText -Encoding UTF8
|
|
$report = $jsonText | ConvertFrom-Json
|
|
$verdictReasons = $null
|
|
if ($report.PSObject.Properties.Name -contains "verdict_reasons") {
|
|
$verdictReasons = $report.verdict_reasons
|
|
}
|
|
$targetStats = $null
|
|
if ($report.PSObject.Properties.Name -contains "target_stats") {
|
|
$targetStats = $report.target_stats
|
|
}
|
|
$targetProbes = $null
|
|
if ($report.PSObject.Properties.Name -contains "target_probes") {
|
|
$targetProbes = $report.target_probes
|
|
}
|
|
$excludedTargets = $null
|
|
if ($report.PSObject.Properties.Name -contains "excluded_targets") {
|
|
$excludedTargets = $report.excluded_targets
|
|
}
|
|
$migrationEvents = 0
|
|
if ($report.PSObject.Properties.Name -contains "migration_events") {
|
|
$migrationEvents = $report.migration_events
|
|
}
|
|
$routePressure = $null
|
|
if ($report.PSObject.Properties.Name -contains "route_pressure") {
|
|
$routePressure = $report.route_pressure
|
|
}
|
|
$transportSnapshot = $null
|
|
if ($report.PSObject.Properties.Name -contains "transport_snapshot") {
|
|
$transportSnapshot = $report.transport_snapshot
|
|
}
|
|
$degradedTargets = $null
|
|
if ($report.PSObject.Properties.Name -contains "degraded_targets") {
|
|
$degradedTargets = $report.degraded_targets
|
|
}
|
|
$ackP95 = $null
|
|
if ($report.PSObject.Properties.Name -contains "ack_p95_ms") {
|
|
$ackP95 = $report.ack_p95_ms
|
|
}
|
|
$ackP99 = $null
|
|
if ($report.PSObject.Properties.Name -contains "ack_p99_ms") {
|
|
$ackP99 = $report.ack_p99_ms
|
|
}
|
|
$rerouteLatencyP95 = $null
|
|
if ($report.PSObject.Properties.Name -contains "reroute_latency_p95_ms") {
|
|
$rerouteLatencyP95 = $report.reroute_latency_p95_ms
|
|
}
|
|
$rerouteLatencyP99 = $null
|
|
if ($report.PSObject.Properties.Name -contains "reroute_latency_p99_ms") {
|
|
$rerouteLatencyP99 = $report.reroute_latency_p99_ms
|
|
}
|
|
$rerouteCauses = $null
|
|
if ($report.PSObject.Properties.Name -contains "reroute_causes") {
|
|
$rerouteCauses = $report.reroute_causes
|
|
}
|
|
$resourceSummary = $null
|
|
if ($report.PSObject.Properties.Name -contains "resource_summary") {
|
|
$resourceSummary = $report.resource_summary
|
|
}
|
|
$ackMismatchedStreams = 0
|
|
if ($report.PSObject.Properties.Name -contains "ack_mismatched_streams") {
|
|
$ackMismatchedStreams = $report.ack_mismatched_streams
|
|
}
|
|
$ackIntegrityErrors = 0
|
|
if ($report.PSObject.Properties.Name -contains "ack_integrity_errors") {
|
|
$ackIntegrityErrors = $report.ack_integrity_errors
|
|
}
|
|
$abandonedFrames = 0
|
|
if ($report.PSObject.Properties.Name -contains "abandoned_frames") {
|
|
$abandonedFrames = $report.abandoned_frames
|
|
}
|
|
$controlAckP95 = $null
|
|
if ($report.PSObject.Properties.Name -contains "control_ack_p95_ms") {
|
|
$controlAckP95 = $report.control_ack_p95_ms
|
|
}
|
|
$bulkAckP95 = $null
|
|
if ($report.PSObject.Properties.Name -contains "bulk_ack_p95_ms") {
|
|
$bulkAckP95 = $report.bulk_ack_p95_ms
|
|
}
|
|
$containerStats = Get-FabricContainerStats
|
|
if ($ContainerStatsSampleInterval.Trim() -ne "" -and $containerStatsSamples.Count -eq 0 -and $containerStats.Count -gt 0) {
|
|
$observedAt = (Get-Date).ToUniversalTime().ToString("o")
|
|
foreach ($stat in $containerStats) {
|
|
if ($stat.PSObject.Properties.Name -contains "error") {
|
|
continue
|
|
}
|
|
$containerStatsSamples += [pscustomobject]@{
|
|
observed_at = $observedAt
|
|
name = $stat.name
|
|
id = $stat.id
|
|
cpu_percent = $stat.cpu_percent
|
|
memory_percent = $stat.memory_percent
|
|
memory_usage = $stat.memory_usage
|
|
memory_usage_mib = $stat.memory_usage_mib
|
|
net_io = $stat.net_io
|
|
block_io = $stat.block_io
|
|
pids = $stat.pids
|
|
pids_count = $stat.pids_count
|
|
role = $stat.role
|
|
}
|
|
}
|
|
}
|
|
$containerStatsSampleSummary = Get-ContainerStatsSampleSummary -Samples $containerStatsSamples
|
|
if ($containerStatsSamples.Count -gt 0) {
|
|
New-Item -ItemType Directory -Force -Path (Split-Path -Parent $ContainerStatsSamplesPath) | Out-Null
|
|
Set-Content -LiteralPath $ContainerStatsSamplesPath -Value ($containerStatsSamples | ConvertTo-Json -Depth 20) -Encoding UTF8
|
|
}
|
|
$containerVerdictReasons = @(Get-ContainerStatsVerdictReasons -ContainerStats $containerStats)
|
|
$containerSampleVerdictReasons = @(Get-ContainerStatsSampleVerdictReasons -SampleSummary $containerStatsSampleSummary)
|
|
$combinedVerdictReasons = @()
|
|
if ($null -ne $verdictReasons) {
|
|
$combinedVerdictReasons += $verdictReasons
|
|
}
|
|
$combinedVerdictReasons += $containerVerdictReasons
|
|
$combinedVerdictReasons += $containerSampleVerdictReasons
|
|
$summaryVerdict = $report.verdict
|
|
if ($containerVerdictReasons.Count -gt 0 -or $containerSampleVerdictReasons.Count -gt 0) {
|
|
$summaryVerdict = "fail"
|
|
}
|
|
$summary = [pscustomobject]@{
|
|
run_id = $runId
|
|
report_path = $ReportPath
|
|
summary_path = $SummaryPath
|
|
docker_context = $DockerContext
|
|
nodes = $Nodes
|
|
topology_profile = $TopologyProfile
|
|
soak = $Soak.IsPresent
|
|
targets = $targets
|
|
total_streams = $report.total_streams
|
|
successful_streams = $report.successful_streams
|
|
failed_streams = $report.failed_streams
|
|
ack_mismatched_streams = $ackMismatchedStreams
|
|
ack_integrity_errors = $ackIntegrityErrors
|
|
abandoned_frames = $abandonedFrames
|
|
failover_events = $report.failover_events
|
|
migration_events = $migrationEvents
|
|
channel_opens = $report.channel_opens
|
|
channel_closes = $report.channel_closes
|
|
channel_leaks = $report.channel_leaks
|
|
channel_churn_per_sec = $report.channel_churn_per_sec
|
|
bytes_sent = $report.bytes_sent
|
|
throughput_bps = $report.throughput_bps
|
|
setup_latency_p95_ms = $report.setup_latency_p95_ms
|
|
channel_open_p95_ms = $report.channel_open_p95_ms
|
|
stream_duration_p95_ms = $report.stream_duration_p95_ms
|
|
ack_p95_ms = $ackP95
|
|
ack_p99_ms = $ackP99
|
|
reroute_latency_p95_ms = $rerouteLatencyP95
|
|
reroute_latency_p99_ms = $rerouteLatencyP99
|
|
route_attempts_total = $report.route_attempts_total
|
|
reroute_causes = $rerouteCauses
|
|
resource_summary = $resourceSummary
|
|
container_stats = $containerStats
|
|
container_stats_samples_path = $(if ($containerStatsSamples.Count -gt 0) { $ContainerStatsSamplesPath } else { $null })
|
|
container_stats_samples_count = $containerStatsSamples.Count
|
|
container_stats_sample_summary = $containerStatsSampleSummary
|
|
control_streams = $report.control_streams
|
|
bulk_streams = $report.bulk_streams
|
|
control_ack_p95_ms = $controlAckP95
|
|
bulk_ack_p95_ms = $bulkAckP95
|
|
verdict = $summaryVerdict
|
|
verdict_reasons = $combinedVerdictReasons
|
|
udp_buffers = $udpBuffers
|
|
impairment = $impairment
|
|
target_probes = $targetProbes
|
|
excluded_targets = $excludedTargets
|
|
degraded_targets = $degradedTargets
|
|
target_streams = $report.target_streams
|
|
target_bytes = $report.target_bytes
|
|
target_stats = $targetStats
|
|
route_pressure = $routePressure
|
|
transport_snapshot = $transportSnapshot
|
|
}
|
|
$summaryJson = $summary | ConvertTo-Json -Depth 20
|
|
New-Item -ItemType Directory -Force -Path (Split-Path -Parent $SummaryPath) | Out-Null
|
|
Set-Content -LiteralPath $SummaryPath -Value $summaryJson -Encoding UTF8
|
|
$summaryJson
|
|
if ($summaryVerdict -ne "pass") {
|
|
$reasonText = ""
|
|
if ($combinedVerdictReasons.Count -gt 0) {
|
|
$reasonText = ($combinedVerdictReasons -join "; ")
|
|
}
|
|
throw "fabric loadtest verdict failed: $summaryVerdict $reasonText"
|
|
}
|
|
}
|
|
finally {
|
|
Cleanup
|
|
}
|