1306 lines
53 KiB
PowerShell
1306 lines
53 KiB
PowerShell
param(
|
|
[string]$ApiBaseUrl = "http://192.168.200.61:18121/api/v1",
|
|
[string]$ClusterID = "cfc0743d-d960-49fb-9de8-96e063d5e4aa",
|
|
[string]$ActorUserID = "f67d943f-5397-4b3a-a229-695fe67ad700",
|
|
[string]$EntryNodeName = "test-1",
|
|
[string]$ExitNodeName = "test-2",
|
|
[string]$EntryBaseUrl = "http://192.168.200.61:19131",
|
|
[string]$DockerSSH = "test-docker",
|
|
[int]$InteractiveSessionCount = 2,
|
|
[int]$InteractiveRounds = 24,
|
|
[int]$InteractivePacketsPerBatch = 8,
|
|
[int]$AbusivePacketCount = 1300,
|
|
[int]$ExpectedAbusiveDelivered = 1024,
|
|
[int]$BatchDelayMilliseconds = 25,
|
|
[string]$RequiredNodeVersion = "0.2.183",
|
|
[string]$ResultPath = "artifacts\c18z8-live-service-channel-backpressure-isolation-smoke-result.json"
|
|
)
|
|
|
|
Set-StrictMode -Version Latest
|
|
$ErrorActionPreference = "Stop"
|
|
Add-Type -AssemblyName System.Net.Http
|
|
|
|
$repoRoot = (Resolve-Path (Join-Path $PSScriptRoot "..\..")).ProviderPath
|
|
$runId = "c18z8-" + (Get-Date -Format "yyyyMMdd-HHmmss")
|
|
$resourceId = "vpn-$runId"
|
|
|
|
function Invoke-Api {
|
|
param(
|
|
[string]$Method,
|
|
[string]$Path,
|
|
[object]$Body = $null
|
|
)
|
|
$uri = "$ApiBaseUrl$Path"
|
|
try {
|
|
if ($null -eq $Body) {
|
|
return Invoke-RestMethod -Method $Method -Uri $uri -TimeoutSec 30
|
|
}
|
|
return Invoke-RestMethod -Method $Method -Uri $uri -ContentType "application/json" -Body ($Body | ConvertTo-Json -Depth 80) -TimeoutSec 30
|
|
}
|
|
catch {
|
|
$statusCode = $null
|
|
if ($_.Exception.Response) {
|
|
$statusCode = [int]$_.Exception.Response.StatusCode
|
|
}
|
|
$details = $_.ErrorDetails.Message
|
|
if (-not $details) {
|
|
$details = $_.Exception.Message
|
|
}
|
|
throw "$Method $Path failed with HTTP $statusCode`: $details"
|
|
}
|
|
}
|
|
|
|
function Get-NodeByName {
|
|
param([string]$Name)
|
|
$nodes = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/nodes?actor_user_id=$ActorUserID").nodes
|
|
$node = @($nodes | Where-Object { $_.name -eq $Name }) | Select-Object -First 1
|
|
if ($null -eq $node) {
|
|
throw "Node '$Name' was not found in cluster $ClusterID"
|
|
}
|
|
return $node
|
|
}
|
|
|
|
function Get-MeshPort {
|
|
param([string]$Name)
|
|
switch ($Name) {
|
|
"test-1" { return 19131 }
|
|
"test-2" { return 19132 }
|
|
"test-3" { return 19133 }
|
|
default { return 19131 }
|
|
}
|
|
}
|
|
|
|
function Enable-TestMeshListener {
|
|
param([object]$Node)
|
|
$port = Get-MeshPort -Name $Node.name
|
|
Invoke-Api -Method PUT -Path "/clusters/$ClusterID/nodes/$($Node.id)/workloads/mesh-listener/desired" -Body @{
|
|
actor_user_id = $ActorUserID
|
|
desired_state = "enabled"
|
|
runtime_mode = "container"
|
|
version = "c18z8-live-fsc-backpressure-isolation"
|
|
config = @{
|
|
listen_addr = "0.0.0.0:$port"
|
|
listen_port_mode = "manual"
|
|
advertise_endpoint = "http://192.168.200.61:$port"
|
|
advertise_transport = "direct_http"
|
|
connectivity_mode = "private_lan"
|
|
nat_type = "none"
|
|
region = "docker-test"
|
|
production_forwarding = $true
|
|
}
|
|
environment = @{}
|
|
} | Out-Null
|
|
}
|
|
|
|
function Clear-OldSmokeRouteIntents {
|
|
param(
|
|
[string]$SourceNodeID,
|
|
[string]$DestinationNodeID
|
|
)
|
|
$items = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/mesh/route-intents?actor_user_id=$ActorUserID").route_intents
|
|
foreach ($item in @($items)) {
|
|
if ([string]$item.lifecycle_status -ne "active") {
|
|
continue
|
|
}
|
|
if ([string]$item.service_class -ne "vpn_packets") {
|
|
continue
|
|
}
|
|
if ([string]$item.source_selector.node_id -ne $SourceNodeID -or [string]$item.destination_selector.node_id -ne $DestinationNodeID) {
|
|
continue
|
|
}
|
|
$smoke = ""
|
|
if ($null -ne $item.policy -and $null -ne $item.policy.metadata) {
|
|
$prop = $item.policy.metadata.PSObject.Properties["smoke"]
|
|
if ($null -ne $prop) {
|
|
$smoke = [string]$prop.Value
|
|
}
|
|
}
|
|
if ($smoke -ne "c18z1_live_service_channel_ingress" -and $smoke -ne "c18z2_live_service_channel_soak" -and $smoke -ne "c18z3_live_service_channel_entry_ws_fallback" -and $smoke -ne "c18z4_live_service_channel_session_pressure" -and $smoke -ne "c18z5_live_service_channel_exit_restart" -and $smoke -ne "c18z6_live_service_channel_active_rebuild" -and $smoke -ne "c18z7_live_service_channel_concurrent_isolation" -and $smoke -ne "c18z8_live_service_channel_backpressure_isolation") {
|
|
continue
|
|
}
|
|
Invoke-Api -Method POST -Path "/clusters/$ClusterID/mesh/route-intents/$($item.id)/expire" -Body @{ actor_user_id = $ActorUserID } | Out-Null
|
|
}
|
|
}
|
|
|
|
function New-RouteIntent {
|
|
param(
|
|
[string]$SourceNodeID,
|
|
[string]$DestinationNodeID,
|
|
[int]$Priority,
|
|
[string]$Label
|
|
)
|
|
$expiresAt = (Get-Date).ToUniversalTime().AddMinutes(10).ToString("o")
|
|
return Invoke-Api -Method POST -Path "/clusters/$ClusterID/mesh/route-intents" -Body @{
|
|
actor_user_id = $ActorUserID
|
|
source_selector = @{ node_id = $SourceNodeID }
|
|
destination_selector = @{ node_id = $DestinationNodeID }
|
|
service_class = "vpn_packets"
|
|
priority = $Priority
|
|
policy = @{
|
|
synthetic_enabled = $true
|
|
route_version = "$runId-$Label"
|
|
policy_version = "$runId-$Label"
|
|
peer_directory_version = "$runId-$Label"
|
|
hops = @($SourceNodeID, $DestinationNodeID)
|
|
allowed_channels = @("vpn_packet", "fabric_control")
|
|
max_ttl = 8
|
|
max_hops = 8
|
|
expires_at = $expiresAt
|
|
metadata = @{
|
|
smoke = "c18z8_live_service_channel_backpressure_isolation"
|
|
run_id = $runId
|
|
label = $Label
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
function Get-SyntheticConfig {
|
|
param([string]$NodeID)
|
|
return Invoke-Api -Method GET -Path "/clusters/$ClusterID/nodes/$NodeID/mesh/synthetic-config?actor_user_id=$ActorUserID"
|
|
}
|
|
|
|
function Get-LatestHeartbeat {
|
|
param([string]$NodeID)
|
|
return (Invoke-Api -Method GET -Path "/clusters/$ClusterID/nodes/$NodeID/heartbeats?actor_user_id=$ActorUserID&limit=1").heartbeats[0]
|
|
}
|
|
|
|
function Get-LatestRuntimeReport {
|
|
param([string]$NodeID)
|
|
$hb = Get-LatestHeartbeat -NodeID $NodeID
|
|
return @{
|
|
heartbeat = $hb
|
|
report = $hb.metadata.fabric_service_channel_runtime_report
|
|
}
|
|
}
|
|
|
|
function Wait-ForRuntimeReady {
|
|
param(
|
|
[string]$NodeID,
|
|
[int]$MinRoutes,
|
|
[int]$TimeoutSeconds = 90
|
|
)
|
|
$deadline = (Get-Date).AddSeconds($TimeoutSeconds)
|
|
do {
|
|
$latest = Get-LatestRuntimeReport -NodeID $NodeID
|
|
$report = $latest.report
|
|
if ($null -ne $report -and
|
|
$report.enabled -eq $true -and
|
|
$report.production_payload_forwarding -eq $true -and
|
|
[int]$report.route_candidate_total -ge $MinRoutes) {
|
|
return $latest
|
|
}
|
|
Start-Sleep -Seconds 2
|
|
} while ((Get-Date) -lt $deadline)
|
|
throw "Timed out waiting for production service-channel runtime ready on node $NodeID"
|
|
}
|
|
|
|
function Wait-ForRuntimeConfigVersion {
|
|
param(
|
|
[string]$NodeID,
|
|
[string]$ConfigVersion,
|
|
[int]$TimeoutSeconds = 90
|
|
)
|
|
$deadline = (Get-Date).AddSeconds($TimeoutSeconds)
|
|
do {
|
|
$latest = Get-LatestRuntimeReport -NodeID $NodeID
|
|
if ($null -ne $latest.report) {
|
|
$loadedVersion = [string]$latest.report.config_version
|
|
if ($loadedVersion -ge $ConfigVersion) {
|
|
return $latest
|
|
}
|
|
}
|
|
Start-Sleep -Seconds 2
|
|
} while ((Get-Date) -lt $deadline)
|
|
throw "Timed out waiting for node $NodeID to load synthetic config $ConfigVersion"
|
|
}
|
|
|
|
function Wait-ForRouteIntentVisible {
|
|
param(
|
|
[string]$NodeID,
|
|
[string[]]$RouteIDs,
|
|
[int]$TimeoutSeconds = 60
|
|
)
|
|
$deadline = (Get-Date).AddSeconds($TimeoutSeconds)
|
|
do {
|
|
$config = Get-SyntheticConfig -NodeID $NodeID
|
|
$routes = @($config.synthetic_mesh_config.routes)
|
|
$present = @($routes | Where-Object { $RouteIDs -contains $_.route_id })
|
|
if ($present.Count -ge $RouteIDs.Count) {
|
|
return $config
|
|
}
|
|
Start-Sleep -Seconds 2
|
|
} while ((Get-Date) -lt $deadline)
|
|
throw "Timed out waiting for routes '$($RouteIDs -join ",")' in synthetic config for node $NodeID"
|
|
}
|
|
|
|
function Wait-ForRouteIntentNotVisible {
|
|
param(
|
|
[string]$NodeID,
|
|
[string]$RouteID,
|
|
[int]$TimeoutSeconds = 90
|
|
)
|
|
$deadline = (Get-Date).AddSeconds($TimeoutSeconds)
|
|
do {
|
|
$config = Get-SyntheticConfig -NodeID $NodeID
|
|
$routes = @($config.synthetic_mesh_config.routes)
|
|
$present = @($routes | Where-Object { $_.route_id -eq $RouteID })
|
|
if ($present.Count -eq 0) {
|
|
return $config
|
|
}
|
|
Start-Sleep -Seconds 2
|
|
} while ((Get-Date) -lt $deadline)
|
|
throw "Timed out waiting for route '$RouteID' to disappear from synthetic config for node $NodeID"
|
|
}
|
|
|
|
function New-ServiceChannelLease {
|
|
param(
|
|
[string]$EntryNodeID,
|
|
[string]$ExitNodeID,
|
|
[string]$VPNResourceID = $resourceId
|
|
)
|
|
return (Invoke-Api -Method POST -Path "/clusters/$ClusterID/fabric/service-channels/leases" -Body @{
|
|
actor_user_id = $ActorUserID
|
|
organization_id = "org-c18z8-smoke"
|
|
user_id = $ActorUserID
|
|
resource_id = $VPNResourceID
|
|
service_class = "vpn_packets"
|
|
entry_node_ids = @($EntryNodeID)
|
|
exit_node_ids = @($ExitNodeID)
|
|
preferred_entry_node_id = $EntryNodeID
|
|
preferred_exit_node_id = $ExitNodeID
|
|
allowed_channels = @("vpn_packet", "bulk", "control")
|
|
ttl_seconds = 300
|
|
metadata = @{
|
|
smoke = "c18z8_live_service_channel_backpressure_isolation"
|
|
run_id = $runId
|
|
}
|
|
}).fabric_service_channel_lease
|
|
}
|
|
|
|
function ConvertTo-Base64UrlJson {
|
|
param([object]$Value)
|
|
$json = $Value | ConvertTo-Json -Depth 80 -Compress
|
|
$bytes = [System.Text.Encoding]::UTF8.GetBytes($json)
|
|
return [Convert]::ToBase64String($bytes).TrimEnd("=").Replace("+", "-").Replace("/", "_")
|
|
}
|
|
|
|
function Get-ObjectPropertyValue {
|
|
param(
|
|
[object]$Object,
|
|
[string]$Name
|
|
)
|
|
if ($null -eq $Object) {
|
|
return $null
|
|
}
|
|
$prop = $Object.PSObject.Properties[$Name]
|
|
if ($null -eq $prop) {
|
|
return $null
|
|
}
|
|
return $prop.Value
|
|
}
|
|
|
|
function New-TestIPv4UDPPacket {
|
|
param([int]$SourcePort)
|
|
$payload = [System.Text.Encoding]::ASCII.GetBytes("c18z1-$SourcePort")
|
|
$totalLength = 20 + 8 + $payload.Length
|
|
$packet = New-Object byte[] $totalLength
|
|
$packet[0] = 0x45
|
|
$packet[1] = 0
|
|
$packet[2] = [byte](($totalLength -shr 8) -band 0xff)
|
|
$packet[3] = [byte]($totalLength -band 0xff)
|
|
$packet[8] = 64
|
|
$packet[9] = 17
|
|
$packet[12] = 10; $packet[13] = 18; $packet[14] = 1; $packet[15] = 10
|
|
$packet[16] = 10; $packet[17] = 18; $packet[18] = 2; $packet[19] = 20
|
|
$udpOffset = 20
|
|
$destPort = 3389
|
|
$udpLength = 8 + $payload.Length
|
|
$packet[$udpOffset] = [byte](($SourcePort -shr 8) -band 0xff)
|
|
$packet[$udpOffset + 1] = [byte]($SourcePort -band 0xff)
|
|
$packet[$udpOffset + 2] = [byte](($destPort -shr 8) -band 0xff)
|
|
$packet[$udpOffset + 3] = [byte]($destPort -band 0xff)
|
|
$packet[$udpOffset + 4] = [byte](($udpLength -shr 8) -band 0xff)
|
|
$packet[$udpOffset + 5] = [byte]($udpLength -band 0xff)
|
|
[Array]::Copy($payload, 0, $packet, 28, $payload.Length)
|
|
return $packet
|
|
}
|
|
|
|
function New-TestIPv4UDPPacketBatch {
|
|
param(
|
|
[int]$SourcePort,
|
|
[int]$Count
|
|
)
|
|
$packets = @()
|
|
for ($i = 0; $i -lt $Count; $i++) {
|
|
$packets += ,(New-TestIPv4UDPPacket -SourcePort $SourcePort)
|
|
}
|
|
return $packets
|
|
}
|
|
|
|
function New-PacketBatchBody {
|
|
param([byte[][]]$Packets)
|
|
$stream = [System.IO.MemoryStream]::new()
|
|
foreach ($packet in $Packets) {
|
|
$length = $packet.Length
|
|
$stream.WriteByte([byte](($length -shr 24) -band 0xff))
|
|
$stream.WriteByte([byte](($length -shr 16) -band 0xff))
|
|
$stream.WriteByte([byte](($length -shr 8) -band 0xff))
|
|
$stream.WriteByte([byte]($length -band 0xff))
|
|
$stream.Write($packet, 0, $packet.Length)
|
|
}
|
|
return $stream.ToArray()
|
|
}
|
|
|
|
function Read-PacketBatchBody {
|
|
param([byte[]]$Body)
|
|
$packets = @()
|
|
$offset = 0
|
|
while (($offset + 4) -le $Body.Length) {
|
|
$length = ([int]$Body[$offset] -shl 24) -bor ([int]$Body[$offset + 1] -shl 16) -bor ([int]$Body[$offset + 2] -shl 8) -bor [int]$Body[$offset + 3]
|
|
$offset += 4
|
|
if ($length -lt 0 -or ($offset + $length) -gt $Body.Length) {
|
|
throw "Malformed packet batch body: length=$length offset=$offset body_length=$($Body.Length)"
|
|
}
|
|
$packet = New-Object byte[] $length
|
|
[Array]::Copy($Body, $offset, $packet, 0, $length)
|
|
$packets += ,$packet
|
|
$offset += $length
|
|
}
|
|
if ($offset -ne $Body.Length) {
|
|
throw "Malformed packet batch body: trailing_bytes=$($Body.Length - $offset)"
|
|
}
|
|
return $packets
|
|
}
|
|
|
|
function Receive-ExitServiceChannelPacketBatch {
|
|
param(
|
|
[string]$VPNResourceID,
|
|
[int]$TimeoutMilliseconds = 5000
|
|
)
|
|
$exitBaseUrl = "http://192.168.200.61:$(Get-MeshPort -Name $ExitNodeName)"
|
|
$url = "$exitBaseUrl/api/v1/clusters/$ClusterID/vpn-connections/$VPNResourceID/tunnel/client/packets?batch=true&timeout_ms=$TimeoutMilliseconds"
|
|
$client = [System.Net.Http.HttpClient]::new()
|
|
try {
|
|
$client.Timeout = [TimeSpan]::FromSeconds([Math]::Max(10, [Math]::Ceiling($TimeoutMilliseconds / 1000) + 5))
|
|
$response = $client.GetAsync($url).GetAwaiter().GetResult()
|
|
if ([int]$response.StatusCode -eq 204) {
|
|
return @()
|
|
}
|
|
$body = $response.Content.ReadAsByteArrayAsync().GetAwaiter().GetResult()
|
|
if (-not $response.IsSuccessStatusCode) {
|
|
$message = [System.Text.Encoding]::UTF8.GetString($body)
|
|
throw "Exit service-channel GET $url failed with HTTP $([int]$response.StatusCode): $message"
|
|
}
|
|
return @(Read-PacketBatchBody -Body $body)
|
|
}
|
|
finally {
|
|
$client.Dispose()
|
|
}
|
|
}
|
|
|
|
function Invoke-ServiceChannelPost {
|
|
param(
|
|
[object]$Lease,
|
|
[int]$PortStart,
|
|
[string]$VPNResourceID = $resourceId
|
|
)
|
|
$packets = @()
|
|
for ($i = 0; $i -lt 8; $i++) {
|
|
$packets += ,(New-TestIPv4UDPPacket -SourcePort ($PortStart + $i))
|
|
}
|
|
$path = $Lease.entry_http.path_template.
|
|
Replace("{cluster_id}", $ClusterID).
|
|
Replace("{channel_id}", $Lease.channel_id).
|
|
Replace("{resource_id}", $VPNResourceID)
|
|
$url = "$EntryBaseUrl$path`?batch=true"
|
|
$headers = @{
|
|
"X-RAP-Service-Channel-Token" = $Lease.token.token
|
|
"X-RAP-Fabric-Channel-ID" = $Lease.channel_id
|
|
"X-RAP-Service-Class" = "vpn_packets"
|
|
"X-RAP-Channel-Class" = "vpn_packet"
|
|
"X-RAP-Service-Channel-Authority-Payload" = ConvertTo-Base64UrlJson -Value $Lease.authority_payload
|
|
"X-RAP-Service-Channel-Authority-Signature" = ConvertTo-Base64UrlJson -Value $Lease.authority_signature
|
|
}
|
|
$body = New-PacketBatchBody -Packets $packets
|
|
$client = [System.Net.Http.HttpClient]::new()
|
|
try {
|
|
$client.Timeout = [TimeSpan]::FromSeconds(30)
|
|
$request = [System.Net.Http.HttpRequestMessage]::new([System.Net.Http.HttpMethod]::Post, $url)
|
|
foreach ($header in $headers.GetEnumerator()) {
|
|
[void]$request.Headers.TryAddWithoutValidation($header.Key, [string]$header.Value)
|
|
}
|
|
$content = [System.Net.Http.ByteArrayContent]::new($body)
|
|
$content.Headers.ContentType = [System.Net.Http.Headers.MediaTypeHeaderValue]::Parse("application/vnd.rap.vpn-packet-batch.v1")
|
|
$request.Content = $content
|
|
$response = $client.SendAsync($request).GetAwaiter().GetResult()
|
|
$responseBody = $response.Content.ReadAsStringAsync().GetAwaiter().GetResult()
|
|
if (-not $response.IsSuccessStatusCode) {
|
|
throw "Service-channel POST $url failed with HTTP $([int]$response.StatusCode): $responseBody"
|
|
}
|
|
return [pscustomobject]@{
|
|
StatusCode = [int]$response.StatusCode
|
|
Body = $responseBody
|
|
}
|
|
}
|
|
finally {
|
|
$client.Dispose()
|
|
}
|
|
}
|
|
|
|
function Get-IngressSendPackets {
|
|
param([string]$NodeID)
|
|
$latest = Get-LatestRuntimeReport -NodeID $NodeID
|
|
$ingress = $latest.report.ingress
|
|
$sendPackets = Get-ObjectPropertyValue -Object $ingress -Name "send_packets"
|
|
if ($null -eq $sendPackets) {
|
|
return 0
|
|
}
|
|
return [int]$sendPackets
|
|
}
|
|
|
|
function Get-IngressRouteFailures {
|
|
param([string]$NodeID)
|
|
$latest = Get-LatestRuntimeReport -NodeID $NodeID
|
|
$ingress = $latest.report.ingress
|
|
$failures = Get-ObjectPropertyValue -Object $ingress -Name "send_route_failures"
|
|
if ($null -eq $failures) {
|
|
return 0
|
|
}
|
|
return [int]$failures
|
|
}
|
|
|
|
function Get-IngressFlowDropped {
|
|
param([string]$NodeID)
|
|
$latest = Get-LatestRuntimeReport -NodeID $NodeID
|
|
$ingress = $latest.report.ingress
|
|
if ($null -eq $ingress) {
|
|
return 0
|
|
}
|
|
$flowScheduler = Get-ObjectPropertyValue -Object $ingress -Name "flow_scheduler"
|
|
if ($null -eq $flowScheduler) {
|
|
return 0
|
|
}
|
|
$dropped = Get-ObjectPropertyValue -Object $flowScheduler -Name "dropped"
|
|
if ($null -eq $dropped) {
|
|
return 0
|
|
}
|
|
return [int]$dropped
|
|
}
|
|
|
|
function Get-ExitQueueDepth {
|
|
param(
|
|
[string]$NodeID,
|
|
[string]$VPNConnectionID
|
|
)
|
|
$latest = Get-LatestRuntimeReport -NodeID $NodeID
|
|
$queueKey = "$VPNConnectionID`:client_to_gateway"
|
|
$depths = $latest.report.inbox.queue_depths
|
|
if ($null -eq $depths) {
|
|
return 0
|
|
}
|
|
$prop = $depths.PSObject.Properties[$queueKey]
|
|
if ($null -eq $prop) {
|
|
return 0
|
|
}
|
|
return [int]$prop.Value
|
|
}
|
|
|
|
function Wait-ForExitQueueDepth {
|
|
param(
|
|
[string]$NodeID,
|
|
[string]$VPNConnectionID,
|
|
[int]$MinDepth,
|
|
[int]$TimeoutSeconds = 90
|
|
)
|
|
$deadline = (Get-Date).AddSeconds($TimeoutSeconds)
|
|
do {
|
|
$depth = Get-ExitQueueDepth -NodeID $NodeID -VPNConnectionID $VPNConnectionID
|
|
if ($depth -ge $MinDepth) {
|
|
return $depth
|
|
}
|
|
Start-Sleep -Seconds 2
|
|
} while ((Get-Date) -lt $deadline)
|
|
throw "Timed out waiting for exit queue depth >= $MinDepth on node $NodeID"
|
|
}
|
|
|
|
function Invoke-ServiceChannelPostSafe {
|
|
param(
|
|
[object]$Lease,
|
|
[int]$PortStart,
|
|
[string]$VPNResourceID = $resourceId
|
|
)
|
|
try {
|
|
$response = Invoke-ServiceChannelPost -Lease $Lease -PortStart $PortStart -VPNResourceID $VPNResourceID
|
|
return [pscustomobject]@{
|
|
ok = $true
|
|
status_code = [int]$response.StatusCode
|
|
error = ""
|
|
}
|
|
}
|
|
catch {
|
|
return [pscustomobject]@{
|
|
ok = $false
|
|
status_code = 0
|
|
error = $_.Exception.Message
|
|
}
|
|
}
|
|
}
|
|
|
|
function ConvertTo-WebSocketURL {
|
|
param([string]$URL)
|
|
if ($URL.StartsWith("https://")) {
|
|
return "wss://" + $URL.Substring("https://".Length)
|
|
}
|
|
if ($URL.StartsWith("http://")) {
|
|
return "ws://" + $URL.Substring("http://".Length)
|
|
}
|
|
return $URL
|
|
}
|
|
|
|
function Invoke-ServiceChannelWebSocketSend {
|
|
param(
|
|
[object]$Lease,
|
|
[int]$PortStart,
|
|
[string]$VPNResourceID = $resourceId
|
|
)
|
|
$packets = @()
|
|
for ($i = 0; $i -lt 8; $i++) {
|
|
$packets += ,(New-TestIPv4UDPPacket -SourcePort ($PortStart + $i))
|
|
}
|
|
$path = $Lease.entry_http.websocket_path_template.
|
|
Replace("{cluster_id}", $ClusterID).
|
|
Replace("{channel_id}", $Lease.channel_id).
|
|
Replace("{resource_id}", $VPNResourceID)
|
|
$url = ConvertTo-WebSocketURL -URL "$EntryBaseUrl$path"
|
|
$socket = [System.Net.WebSockets.ClientWebSocket]::new()
|
|
$cts = [System.Threading.CancellationTokenSource]::new([TimeSpan]::FromSeconds(20))
|
|
try {
|
|
$null = $socket.Options.SetRequestHeader("X-RAP-Service-Channel-Token", [string]$Lease.token.token)
|
|
$null = $socket.Options.SetRequestHeader("X-RAP-Fabric-Channel-ID", [string]$Lease.channel_id)
|
|
$null = $socket.Options.SetRequestHeader("X-RAP-Service-Class", "vpn_packets")
|
|
$null = $socket.Options.SetRequestHeader("X-RAP-Channel-Class", "vpn_packet")
|
|
$null = $socket.Options.SetRequestHeader("X-RAP-Service-Channel-Authority-Payload", (ConvertTo-Base64UrlJson -Value $Lease.authority_payload))
|
|
$null = $socket.Options.SetRequestHeader("X-RAP-Service-Channel-Authority-Signature", (ConvertTo-Base64UrlJson -Value $Lease.authority_signature))
|
|
$null = $socket.ConnectAsync([Uri]$url, $cts.Token).GetAwaiter().GetResult()
|
|
$body = New-PacketBatchBody -Packets $packets
|
|
$segment = [ArraySegment[byte]]::new($body)
|
|
$null = $socket.SendAsync($segment, [System.Net.WebSockets.WebSocketMessageType]::Binary, $true, $cts.Token).GetAwaiter().GetResult()
|
|
Start-Sleep -Milliseconds 300
|
|
if ($socket.State -eq [System.Net.WebSockets.WebSocketState]::Open) {
|
|
$null = $socket.CloseOutputAsync([System.Net.WebSockets.WebSocketCloseStatus]::NormalClosure, "c18z8 sent", $cts.Token).GetAwaiter().GetResult()
|
|
}
|
|
return [pscustomobject]@{
|
|
ok = $true
|
|
url = $url
|
|
sent_packets = $packets.Count
|
|
state = [string]$socket.State
|
|
error = ""
|
|
}
|
|
}
|
|
catch {
|
|
return [pscustomobject]@{
|
|
ok = $false
|
|
url = $url
|
|
sent_packets = 0
|
|
state = [string]$socket.State
|
|
error = $_.Exception.Message
|
|
}
|
|
}
|
|
finally {
|
|
$socket.Dispose()
|
|
$cts.Dispose()
|
|
}
|
|
}
|
|
|
|
function Invoke-ServiceChannelWebSocketPressure {
|
|
param(
|
|
[object]$Lease,
|
|
[int]$PortStart,
|
|
[int]$PreSwitchBatches,
|
|
[int]$PostSwitchBatches,
|
|
[int]$PacketsInBatch,
|
|
[int]$DelayMilliseconds,
|
|
[scriptblock]$AfterPreSwitchAction,
|
|
[string]$VPNResourceID = $resourceId
|
|
)
|
|
$path = $Lease.entry_http.websocket_path_template.
|
|
Replace("{cluster_id}", $ClusterID).
|
|
Replace("{channel_id}", $Lease.channel_id).
|
|
Replace("{resource_id}", $VPNResourceID)
|
|
$url = ConvertTo-WebSocketURL -URL "$EntryBaseUrl$path"
|
|
$socket = [System.Net.WebSockets.ClientWebSocket]::new()
|
|
$cts = [System.Threading.CancellationTokenSource]::new([TimeSpan]::FromSeconds(120))
|
|
$sentBatches = 0
|
|
$sentPackets = 0
|
|
$switchActionRan = $false
|
|
try {
|
|
$null = $socket.Options.SetRequestHeader("X-RAP-Service-Channel-Token", [string]$Lease.token.token)
|
|
$null = $socket.Options.SetRequestHeader("X-RAP-Fabric-Channel-ID", [string]$Lease.channel_id)
|
|
$null = $socket.Options.SetRequestHeader("X-RAP-Service-Class", "vpn_packets")
|
|
$null = $socket.Options.SetRequestHeader("X-RAP-Channel-Class", "vpn_packet")
|
|
$null = $socket.Options.SetRequestHeader("X-RAP-Service-Channel-Authority-Payload", (ConvertTo-Base64UrlJson -Value $Lease.authority_payload))
|
|
$null = $socket.Options.SetRequestHeader("X-RAP-Service-Channel-Authority-Signature", (ConvertTo-Base64UrlJson -Value $Lease.authority_signature))
|
|
$null = $socket.ConnectAsync([Uri]$url, $cts.Token).GetAwaiter().GetResult()
|
|
|
|
$totalBatches = $PreSwitchBatches + $PostSwitchBatches
|
|
for ($batch = 0; $batch -lt $totalBatches; $batch++) {
|
|
if ($batch -eq $PreSwitchBatches -and $null -ne $AfterPreSwitchAction) {
|
|
& $AfterPreSwitchAction
|
|
$switchActionRan = $true
|
|
}
|
|
$packets = @()
|
|
for ($i = 0; $i -lt $PacketsInBatch; $i++) {
|
|
$packets += ,(New-TestIPv4UDPPacket -SourcePort ($PortStart + ($batch * 100) + $i))
|
|
}
|
|
$body = New-PacketBatchBody -Packets $packets
|
|
$segment = [ArraySegment[byte]]::new($body)
|
|
$null = $socket.SendAsync($segment, [System.Net.WebSockets.WebSocketMessageType]::Binary, $true, $cts.Token).GetAwaiter().GetResult()
|
|
$sentBatches++
|
|
$sentPackets += $packets.Count
|
|
if ($DelayMilliseconds -gt 0) {
|
|
Start-Sleep -Milliseconds $DelayMilliseconds
|
|
}
|
|
}
|
|
Start-Sleep -Milliseconds 500
|
|
if ($socket.State -eq [System.Net.WebSockets.WebSocketState]::Open) {
|
|
$null = $socket.CloseOutputAsync([System.Net.WebSockets.WebSocketCloseStatus]::NormalClosure, "c18z8 sent", $cts.Token).GetAwaiter().GetResult()
|
|
}
|
|
return [pscustomobject]@{
|
|
ok = $true
|
|
url = $url
|
|
sent_batches = $sentBatches
|
|
sent_packets = $sentPackets
|
|
switch_action_ran = $switchActionRan
|
|
state = [string]$socket.State
|
|
error = ""
|
|
}
|
|
}
|
|
catch {
|
|
return [pscustomobject]@{
|
|
ok = $false
|
|
url = $url
|
|
sent_batches = $sentBatches
|
|
sent_packets = $sentPackets
|
|
switch_action_ran = $switchActionRan
|
|
state = [string]$socket.State
|
|
error = $_.Exception.Message
|
|
}
|
|
}
|
|
finally {
|
|
$socket.Dispose()
|
|
$cts.Dispose()
|
|
}
|
|
}
|
|
|
|
function Open-ServiceChannelWebSocket {
|
|
param(
|
|
[object]$Lease,
|
|
[string]$VPNResourceID
|
|
)
|
|
$path = $Lease.entry_http.websocket_path_template.
|
|
Replace("{cluster_id}", $ClusterID).
|
|
Replace("{channel_id}", $Lease.channel_id).
|
|
Replace("{resource_id}", $VPNResourceID)
|
|
$url = ConvertTo-WebSocketURL -URL "$EntryBaseUrl$path"
|
|
$socket = [System.Net.WebSockets.ClientWebSocket]::new()
|
|
$cts = [System.Threading.CancellationTokenSource]::new([TimeSpan]::FromSeconds(180))
|
|
$null = $socket.Options.SetRequestHeader("X-RAP-Service-Channel-Token", [string]$Lease.token.token)
|
|
$null = $socket.Options.SetRequestHeader("X-RAP-Fabric-Channel-ID", [string]$Lease.channel_id)
|
|
$null = $socket.Options.SetRequestHeader("X-RAP-Service-Class", "vpn_packets")
|
|
$null = $socket.Options.SetRequestHeader("X-RAP-Channel-Class", "vpn_packet")
|
|
$null = $socket.Options.SetRequestHeader("X-RAP-Service-Channel-Authority-Payload", (ConvertTo-Base64UrlJson -Value $Lease.authority_payload))
|
|
$null = $socket.Options.SetRequestHeader("X-RAP-Service-Channel-Authority-Signature", (ConvertTo-Base64UrlJson -Value $Lease.authority_signature))
|
|
$null = $socket.ConnectAsync([Uri]$url, $cts.Token).GetAwaiter().GetResult()
|
|
return [pscustomobject]@{
|
|
lease = $Lease
|
|
resource_id = $VPNResourceID
|
|
socket = $socket
|
|
cts = $cts
|
|
url = $url
|
|
sent_batches = 0
|
|
sent_packets = 0
|
|
error = ""
|
|
}
|
|
}
|
|
|
|
function Send-ServiceChannelWebSocketBatch {
|
|
param(
|
|
[object]$Session,
|
|
[int]$PortStart,
|
|
[int]$PacketsInBatch
|
|
)
|
|
$packets = @()
|
|
for ($i = 0; $i -lt $PacketsInBatch; $i++) {
|
|
$packets += ,(New-TestIPv4UDPPacket -SourcePort ($PortStart + $i))
|
|
}
|
|
$body = New-PacketBatchBody -Packets $packets
|
|
$segment = [ArraySegment[byte]]::new($body)
|
|
$null = $Session.socket.SendAsync($segment, [System.Net.WebSockets.WebSocketMessageType]::Binary, $true, $Session.cts.Token).GetAwaiter().GetResult()
|
|
$Session.sent_batches = [int]$Session.sent_batches + 1
|
|
$Session.sent_packets = [int]$Session.sent_packets + $packets.Count
|
|
}
|
|
|
|
function Send-ServiceChannelWebSocketPacketBatch {
|
|
param(
|
|
[object]$Session,
|
|
[byte[][]]$Packets
|
|
)
|
|
$body = New-PacketBatchBody -Packets $Packets
|
|
$segment = [ArraySegment[byte]]::new($body)
|
|
$null = $Session.socket.SendAsync($segment, [System.Net.WebSockets.WebSocketMessageType]::Binary, $true, $Session.cts.Token).GetAwaiter().GetResult()
|
|
$Session.sent_batches = [int]$Session.sent_batches + 1
|
|
$Session.sent_packets = [int]$Session.sent_packets + $Packets.Count
|
|
}
|
|
|
|
function Close-ServiceChannelWebSocket {
|
|
param([object]$Session)
|
|
if ($null -eq $Session) {
|
|
return
|
|
}
|
|
try {
|
|
if ($null -ne $Session.socket -and $Session.socket.State -eq [System.Net.WebSockets.WebSocketState]::Open) {
|
|
$null = $Session.socket.CloseOutputAsync([System.Net.WebSockets.WebSocketCloseStatus]::NormalClosure, "c18z8 sent", $Session.cts.Token).GetAwaiter().GetResult()
|
|
}
|
|
}
|
|
catch {}
|
|
try {
|
|
if ($null -ne $Session.socket) {
|
|
$Session.socket.Dispose()
|
|
}
|
|
}
|
|
catch {}
|
|
try {
|
|
if ($null -ne $Session.cts) {
|
|
$Session.cts.Dispose()
|
|
}
|
|
}
|
|
catch {}
|
|
}
|
|
|
|
function Send-BatchSeries {
|
|
param(
|
|
[object]$Lease,
|
|
[int]$Count,
|
|
[int]$PortBase,
|
|
[int]$DelayMilliseconds = 100,
|
|
[string]$VPNResourceID = $resourceId
|
|
)
|
|
$results = @()
|
|
for ($i = 0; $i -lt $Count; $i++) {
|
|
$results += Invoke-ServiceChannelPostSafe -Lease $Lease -PortStart ($PortBase + ($i * 100)) -VPNResourceID $VPNResourceID
|
|
if ($DelayMilliseconds -gt 0) {
|
|
Start-Sleep -Milliseconds $DelayMilliseconds
|
|
}
|
|
}
|
|
return $results
|
|
}
|
|
|
|
function Invoke-RemoteDocker {
|
|
param([string]$Command)
|
|
& ssh $DockerSSH $Command
|
|
if ($LASTEXITCODE -ne 0) {
|
|
throw "ssh $DockerSSH command failed: $Command"
|
|
}
|
|
}
|
|
|
|
function Stop-TestUpdaters {
|
|
Invoke-RemoteDocker -Command "docker stop rap_host_agent_updater_test-1 rap_host_agent_updater_test-2 rap_host_agent_updater_test-3 >/dev/null 2>&1 || true"
|
|
}
|
|
|
|
function Start-TestUpdaters {
|
|
Invoke-RemoteDocker -Command "docker start rap_host_agent_updater_test-1 rap_host_agent_updater_test-2 rap_host_agent_updater_test-3 >/dev/null 2>&1 || true"
|
|
}
|
|
|
|
function Restart-ExitContainer {
|
|
param([string]$Name)
|
|
$containerName = "rap_test_node_" + $Name.Replace("-", "_")
|
|
Invoke-RemoteDocker -Command "docker restart $containerName >/dev/null"
|
|
}
|
|
|
|
function Restart-NodeContainer {
|
|
param([string]$Name)
|
|
$containerName = "rap_test_node_" + $Name.Replace("-", "_")
|
|
Invoke-RemoteDocker -Command "docker restart $containerName >/dev/null"
|
|
}
|
|
|
|
function Get-BackendClientGatewayDepth {
|
|
param([string]$VPNConnectionID)
|
|
$stats = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/vpn-connections/$VPNConnectionID/tunnel/stats").vpn_packet_stats
|
|
$queue = $stats.client_to_gateway
|
|
if ($null -eq $queue) {
|
|
return 0
|
|
}
|
|
$depthProp = $queue.PSObject.Properties["queue_depth"]
|
|
if ($null -eq $depthProp) {
|
|
return 0
|
|
}
|
|
return [int]$depthProp.Value
|
|
}
|
|
|
|
function Wait-ForIngressRoute {
|
|
param(
|
|
[string]$NodeID,
|
|
[string]$RouteID,
|
|
[int]$MinSendPackets,
|
|
[int]$TimeoutSeconds = 45
|
|
)
|
|
$deadline = (Get-Date).AddSeconds($TimeoutSeconds)
|
|
do {
|
|
$latest = Get-LatestRuntimeReport -NodeID $NodeID
|
|
$ingress = $latest.report.ingress
|
|
$sendPackets = Get-ObjectPropertyValue -Object $ingress -Name "send_packets"
|
|
$selectedRoute = Get-ObjectPropertyValue -Object $ingress -Name "last_selected_route_id"
|
|
if ($null -ne $ingress -and
|
|
[int]$sendPackets -ge $MinSendPackets -and
|
|
[string]$selectedRoute -eq $RouteID) {
|
|
return $latest
|
|
}
|
|
Start-Sleep -Seconds 2
|
|
} while ((Get-Date) -lt $deadline)
|
|
throw "Timed out waiting for ingress telemetry route=$RouteID packets>=$MinSendPackets on node $NodeID"
|
|
}
|
|
|
|
function Wait-ForIngressAnyRoute {
|
|
param(
|
|
[string]$NodeID,
|
|
[string[]]$RouteIDs,
|
|
[int]$MinSendPackets,
|
|
[int]$TimeoutSeconds = 45
|
|
)
|
|
$deadline = (Get-Date).AddSeconds($TimeoutSeconds)
|
|
do {
|
|
$latest = Get-LatestRuntimeReport -NodeID $NodeID
|
|
$ingress = $latest.report.ingress
|
|
$sendPackets = Get-ObjectPropertyValue -Object $ingress -Name "send_packets"
|
|
$selectedRoute = Get-ObjectPropertyValue -Object $ingress -Name "last_selected_route_id"
|
|
if ($null -ne $ingress -and
|
|
[int]$sendPackets -ge $MinSendPackets -and
|
|
$RouteIDs -contains [string]$selectedRoute) {
|
|
return $latest
|
|
}
|
|
Start-Sleep -Seconds 2
|
|
} while ((Get-Date) -lt $deadline)
|
|
throw "Timed out waiting for ingress telemetry routes='$($RouteIDs -join ",")' packets>=$MinSendPackets on node $NodeID"
|
|
}
|
|
|
|
function Wait-ForExitInbox {
|
|
param(
|
|
[string]$NodeID,
|
|
[string]$VPNConnectionID,
|
|
[int]$TimeoutSeconds = 45
|
|
)
|
|
$queueKey = "$VPNConnectionID`:client_to_gateway"
|
|
$deadline = (Get-Date).AddSeconds($TimeoutSeconds)
|
|
do {
|
|
$latest = Get-LatestRuntimeReport -NodeID $NodeID
|
|
$depths = $latest.report.inbox.queue_depths
|
|
if ($null -ne $depths) {
|
|
$prop = $depths.PSObject.Properties[$queueKey]
|
|
if ($null -ne $prop -and [int]$prop.Value -gt 0) {
|
|
return $latest
|
|
}
|
|
}
|
|
Start-Sleep -Seconds 2
|
|
} while ((Get-Date) -lt $deadline)
|
|
throw "Timed out waiting for exit inbox queue '$queueKey' on node $NodeID"
|
|
}
|
|
|
|
function Send-FeedbackHeartbeat {
|
|
param(
|
|
[string]$EntryNodeID,
|
|
[string]$BadRouteID,
|
|
[string]$GoodRouteID
|
|
)
|
|
return Invoke-Api -Method POST -Path "/clusters/$ClusterID/nodes/$EntryNodeID/heartbeats" -Body @{
|
|
health_status = "healthy"
|
|
reported_version = $RequiredNodeVersion
|
|
capabilities = @{
|
|
native_node_agent = $true
|
|
fabric_service_channel_runtime = $true
|
|
fabric_service_channel_route_manager = $true
|
|
smoke_feedback_injection = "c18z8"
|
|
}
|
|
service_states = @{ smoke = "c18z8_backpressure_isolation_feedback" }
|
|
metadata = @{
|
|
fabric_service_channel_runtime_report = @{
|
|
schema_version = "c18l.fabric_service_channel_runtime_report.v1"
|
|
ingress = @{
|
|
flow_scheduler = @{
|
|
channel_stats = @{
|
|
"c18z8-backpressure-isolation-flow" = @{
|
|
last_route_id = $GoodRouteID
|
|
last_failed_route_id = $BadRouteID
|
|
last_error = "c18z8 forced stale route during backpressure isolation smoke"
|
|
consecutive_failures = 3
|
|
stall_count = 1
|
|
last_send_duration_ms = 250
|
|
route_rebuild_recommended = $true
|
|
degraded_fallback_recommended = $false
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
smoke = @{
|
|
name = "c18z8_live_service_channel_backpressure_isolation"
|
|
run_id = $runId
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
function Wait-ForConfigDecision {
|
|
param(
|
|
[string]$NodeID,
|
|
[string]$BadRouteID,
|
|
[string]$ExpectedReplacementID,
|
|
[int]$TimeoutSeconds = 60
|
|
)
|
|
$deadline = (Get-Date).AddSeconds($TimeoutSeconds)
|
|
do {
|
|
$config = Get-SyntheticConfig -NodeID $NodeID
|
|
$decisions = @($config.synthetic_mesh_config.route_path_decisions.decisions)
|
|
$decision = @($decisions | Where-Object {
|
|
$_.route_id -eq $BadRouteID -and
|
|
$_.rebuild_status -eq "applied" -and
|
|
$_.replacement_route_id -eq $ExpectedReplacementID
|
|
}) | Select-Object -First 1
|
|
if ($null -ne $decision) {
|
|
return @{
|
|
config = $config
|
|
decision = $decision
|
|
}
|
|
}
|
|
Start-Sleep -Seconds 2
|
|
} while ((Get-Date) -lt $deadline)
|
|
throw "Timed out waiting for applied rebuild decision $BadRouteID -> $ExpectedReplacementID"
|
|
}
|
|
|
|
function Wait-ForAppliedRebuildTransition {
|
|
param(
|
|
[string]$NodeID,
|
|
[string]$BadRouteID = "",
|
|
[string]$ReplacementRouteID = "",
|
|
[int]$TimeoutSeconds = 90
|
|
)
|
|
$deadline = (Get-Date).AddSeconds($TimeoutSeconds)
|
|
do {
|
|
$latest = Get-LatestRuntimeReport -NodeID $NodeID
|
|
$transition = $null
|
|
if ($null -ne $latest.report -and $null -ne $latest.report.ingress) {
|
|
$prop = $latest.report.ingress.PSObject.Properties["route_manager_transition"]
|
|
if ($null -ne $prop) {
|
|
$transition = $prop.Value
|
|
}
|
|
}
|
|
if ($null -ne $transition -and [string]$transition.status -eq "applied_rebuild") {
|
|
return $latest
|
|
}
|
|
if ($BadRouteID -ne "" -and $ReplacementRouteID -ne "") {
|
|
Send-FeedbackHeartbeat -EntryNodeID $NodeID -BadRouteID $BadRouteID -GoodRouteID $ReplacementRouteID | Out-Null
|
|
}
|
|
Start-Sleep -Seconds 2
|
|
} while ((Get-Date) -lt $deadline)
|
|
throw "Timed out waiting for node route-manager transition applied_rebuild on node $NodeID"
|
|
}
|
|
|
|
$entryNode = Get-NodeByName -Name $EntryNodeName
|
|
$exitNode = Get-NodeByName -Name $ExitNodeName
|
|
$primaryRouteID = ""
|
|
$alternateRouteID = ""
|
|
$updatersStopped = $false
|
|
$sessions = @()
|
|
$result = $null
|
|
|
|
try {
|
|
Stop-TestUpdaters
|
|
$updatersStopped = $true
|
|
|
|
Enable-TestMeshListener -Node $entryNode
|
|
Enable-TestMeshListener -Node $exitNode
|
|
Clear-OldSmokeRouteIntents -SourceNodeID $entryNode.id -DestinationNodeID $exitNode.id
|
|
|
|
$primaryIntent = New-RouteIntent -SourceNodeID $entryNode.id -DestinationNodeID $exitNode.id -Priority 2000000000 -Label "primary"
|
|
$alternateIntent = New-RouteIntent -SourceNodeID $entryNode.id -DestinationNodeID $exitNode.id -Priority 1999999999 -Label "alternate"
|
|
$primaryRouteID = $primaryIntent.route_intent.id
|
|
$alternateRouteID = $alternateIntent.route_intent.id
|
|
$routeIDs = @($primaryRouteID, $alternateRouteID)
|
|
|
|
$visibleConfig = Wait-ForRouteIntentVisible -NodeID $entryNode.id -RouteIDs $routeIDs
|
|
$exitVisibleConfig = Wait-ForRouteIntentVisible -NodeID $exitNode.id -RouteIDs $routeIDs
|
|
$readyBefore = Wait-ForRuntimeReady -NodeID $entryNode.id -MinRoutes 2
|
|
$exitReadyBefore = Wait-ForRuntimeReady -NodeID $exitNode.id -MinRoutes 0
|
|
$loadedConfig = Wait-ForRuntimeConfigVersion -NodeID $entryNode.id -ConfigVersion $visibleConfig.synthetic_mesh_config.config_version
|
|
$exitLoadedConfig = Wait-ForRuntimeConfigVersion -NodeID $exitNode.id -ConfigVersion $exitVisibleConfig.synthetic_mesh_config.config_version
|
|
|
|
$baselineSendPackets = Get-IngressSendPackets -NodeID $entryNode.id
|
|
$baselineRouteFailures = Get-IngressRouteFailures -NodeID $entryNode.id
|
|
$baselineDropped = Get-IngressFlowDropped -NodeID $entryNode.id
|
|
$baselineFlowScheduler = $readyBefore.report.ingress.flow_scheduler
|
|
|
|
$sessionSpecs = @()
|
|
$abusiveResourceID = "$resourceId-abusive"
|
|
$abusiveLease = New-ServiceChannelLease -EntryNodeID $entryNode.id -ExitNodeID $exitNode.id -VPNResourceID $abusiveResourceID
|
|
if ($abusiveLease.status -ne "ready") {
|
|
throw "Lease status for abusive session was '$($abusiveLease.status)', want ready"
|
|
}
|
|
$sessionSpecs += [pscustomobject]@{
|
|
kind = "abusive"
|
|
index = 0
|
|
resource_id = $abusiveResourceID
|
|
lease = $abusiveLease
|
|
baseline_exit_depth = Get-ExitQueueDepth -NodeID $exitNode.id -VPNConnectionID $abusiveResourceID
|
|
baseline_backend_depth = Get-BackendClientGatewayDepth -VPNConnectionID $abusiveResourceID
|
|
expected_packets = $ExpectedAbusiveDelivered
|
|
}
|
|
|
|
for ($index = 0; $index -lt $InteractiveSessionCount; $index++) {
|
|
$sessionResourceID = "$resourceId-i$($index + 1)"
|
|
$lease = New-ServiceChannelLease -EntryNodeID $entryNode.id -ExitNodeID $exitNode.id -VPNResourceID $sessionResourceID
|
|
if ($lease.status -ne "ready") {
|
|
throw "Lease status for interactive session $($index + 1) was '$($lease.status)', want ready"
|
|
}
|
|
$sessionSpecs += [pscustomobject]@{
|
|
kind = "interactive"
|
|
index = $index + 1
|
|
resource_id = $sessionResourceID
|
|
lease = $lease
|
|
baseline_exit_depth = Get-ExitQueueDepth -NodeID $exitNode.id -VPNConnectionID $sessionResourceID
|
|
baseline_backend_depth = Get-BackendClientGatewayDepth -VPNConnectionID $sessionResourceID
|
|
expected_packets = ($InteractiveRounds * $InteractivePacketsPerBatch)
|
|
}
|
|
}
|
|
|
|
foreach ($spec in $sessionSpecs) {
|
|
$session = Open-ServiceChannelWebSocket -Lease $spec.lease -VPNResourceID $spec.resource_id
|
|
$session | Add-Member -NotePropertyName kind -NotePropertyValue $spec.kind
|
|
$session | Add-Member -NotePropertyName index -NotePropertyValue $spec.index
|
|
$sessions += $session
|
|
}
|
|
|
|
$abusiveSession = @($sessions | Where-Object { $_.kind -eq "abusive" }) | Select-Object -First 1
|
|
$interactiveSessions = @($sessions | Where-Object { $_.kind -eq "interactive" })
|
|
$abusiveSent = $false
|
|
for ($round = 0; $round -lt $InteractiveRounds; $round++) {
|
|
foreach ($session in $interactiveSessions) {
|
|
Send-ServiceChannelWebSocketBatch -Session $session -PortStart (24000 + ($round * 100) + ([int]$session.index * 10)) -PacketsInBatch $InteractivePacketsPerBatch
|
|
}
|
|
|
|
if (-not $abusiveSent -and $round -ge [Math]::Floor($InteractiveRounds / 3)) {
|
|
[byte[][]]$abusivePackets = New-TestIPv4UDPPacketBatch -SourcePort 39000 -Count $AbusivePacketCount
|
|
Send-ServiceChannelWebSocketPacketBatch -Session $abusiveSession -Packets $abusivePackets
|
|
$abusiveSent = $true
|
|
}
|
|
|
|
if ($BatchDelayMilliseconds -gt 0) {
|
|
Start-Sleep -Milliseconds $BatchDelayMilliseconds
|
|
}
|
|
}
|
|
if (-not $abusiveSent) {
|
|
[byte[][]]$abusivePackets = New-TestIPv4UDPPacketBatch -SourcePort 39000 -Count $AbusivePacketCount
|
|
Send-ServiceChannelWebSocketPacketBatch -Session $abusiveSession -Packets $abusivePackets
|
|
}
|
|
|
|
Start-Sleep -Milliseconds 500
|
|
foreach ($session in $sessions) {
|
|
Close-ServiceChannelWebSocket -Session $session
|
|
}
|
|
|
|
$expectedInteractivePacketsPerSession = $InteractiveRounds * $InteractivePacketsPerBatch
|
|
$expectedInteractiveTotalPackets = $expectedInteractivePacketsPerSession * $InteractiveSessionCount
|
|
$expectedAbusiveDropDelta = [Math]::Max(1, ($AbusivePacketCount - $ExpectedAbusiveDelivered))
|
|
$minimumAbusiveDelivered = [Math]::Min($ExpectedAbusiveDelivered, 960)
|
|
$expectedTotalDeliveredMinimum = $expectedInteractiveTotalPackets + $minimumAbusiveDelivered
|
|
$postPressureIngress = Wait-ForIngressAnyRoute -NodeID $entryNode.id -RouteIDs $routeIDs -MinSendPackets ($baselineSendPackets + $expectedTotalDeliveredMinimum) -TimeoutSeconds 120
|
|
|
|
$sessionResults = @()
|
|
foreach ($i in 0..($sessionSpecs.Count - 1)) {
|
|
$spec = $sessionSpecs[$i]
|
|
$session = $sessions[$i]
|
|
$minDepth = [int]$spec.baseline_exit_depth + [int]$spec.expected_packets
|
|
if ($spec.kind -eq "abusive") {
|
|
$minDepth = [int]$spec.baseline_exit_depth + 1
|
|
}
|
|
$exitDepth = Wait-ForExitQueueDepth -NodeID $exitNode.id -VPNConnectionID $spec.resource_id -MinDepth $minDepth -TimeoutSeconds 120
|
|
$deliveredPackets = [int]$exitDepth - [int]$spec.baseline_exit_depth
|
|
$backendDepth = Get-BackendClientGatewayDepth -VPNConnectionID $spec.resource_id
|
|
$sessionResults += [pscustomobject]@{
|
|
kind = $spec.kind
|
|
index = $spec.index
|
|
resource_id = $spec.resource_id
|
|
channel_id = $spec.lease.channel_id
|
|
sent_batches = $session.sent_batches
|
|
sent_packets = $session.sent_packets
|
|
expected_packets = $spec.expected_packets
|
|
minimum_expected_depth = $minDepth
|
|
baseline_exit_depth = $spec.baseline_exit_depth
|
|
exit_depth = $exitDepth
|
|
exit_delta = ([int]$exitDepth - [int]$spec.baseline_exit_depth)
|
|
delivered_packets = $deliveredPackets
|
|
baseline_backend_depth = $spec.baseline_backend_depth
|
|
backend_depth = $backendDepth
|
|
backend_delta = ([int]$backendDepth - [int]$spec.baseline_backend_depth)
|
|
final_state = [string]$session.socket.State
|
|
}
|
|
}
|
|
|
|
$finalEntryRuntime = Get-LatestRuntimeReport -NodeID $entryNode.id
|
|
$finalExitRuntime = Get-LatestRuntimeReport -NodeID $exitNode.id
|
|
$finalRouteFailures = Get-IngressRouteFailures -NodeID $entryNode.id
|
|
$finalDropped = Get-IngressFlowDropped -NodeID $entryNode.id
|
|
$expiredAlternate = Invoke-Api -Method POST -Path "/clusters/$ClusterID/mesh/route-intents/$alternateRouteID/expire" -Body @{ actor_user_id = $ActorUserID }
|
|
$expiredPrimary = Invoke-Api -Method POST -Path "/clusters/$ClusterID/mesh/route-intents/$primaryRouteID/expire" -Body @{ actor_user_id = $ActorUserID }
|
|
|
|
$routeFailureDelta = $finalRouteFailures - $baselineRouteFailures
|
|
$droppedDelta = $finalDropped - $baselineDropped
|
|
$finalFlowScheduler = $finalEntryRuntime.report.ingress.flow_scheduler
|
|
$maxChannelDropDelta = 0
|
|
$maxChannelEnqueuedDelta = 0
|
|
$maxChannelHighWatermark = 0
|
|
$dropChannelID = ""
|
|
$enqueuedChannelID = ""
|
|
$finalChannelStats = Get-ObjectPropertyValue -Object $finalFlowScheduler -Name "channel_stats"
|
|
if ($null -ne $finalChannelStats) {
|
|
foreach ($prop in @($finalChannelStats.PSObject.Properties)) {
|
|
$finalDroppedForChannel = [int](Get-ObjectPropertyValue -Object $prop.Value -Name "dropped")
|
|
$finalEnqueuedForChannel = [int](Get-ObjectPropertyValue -Object $prop.Value -Name "enqueued")
|
|
$baselineDroppedForChannel = 0
|
|
$baselineEnqueuedForChannel = 0
|
|
$baselineStats = Get-ObjectPropertyValue -Object $baselineFlowScheduler -Name "channel_stats"
|
|
if ($null -ne $baselineStats) {
|
|
$baselineProp = $baselineStats.PSObject.Properties[$prop.Name]
|
|
if ($null -ne $baselineProp) {
|
|
$baselineDroppedForChannel = [int](Get-ObjectPropertyValue -Object $baselineProp.Value -Name "dropped")
|
|
$baselineEnqueuedForChannel = [int](Get-ObjectPropertyValue -Object $baselineProp.Value -Name "enqueued")
|
|
}
|
|
}
|
|
$deltaForChannel = $finalDroppedForChannel - $baselineDroppedForChannel
|
|
if ($deltaForChannel -gt $maxChannelDropDelta) {
|
|
$maxChannelDropDelta = $deltaForChannel
|
|
$dropChannelID = $prop.Name
|
|
}
|
|
$enqueuedDeltaForChannel = $finalEnqueuedForChannel - $baselineEnqueuedForChannel
|
|
if ($enqueuedDeltaForChannel -gt $maxChannelEnqueuedDelta) {
|
|
$maxChannelEnqueuedDelta = $enqueuedDeltaForChannel
|
|
$enqueuedChannelID = $prop.Name
|
|
}
|
|
$highWatermark = [int](Get-ObjectPropertyValue -Object $prop.Value -Name "high_watermark")
|
|
if ($highWatermark -gt $maxChannelHighWatermark) {
|
|
$maxChannelHighWatermark = $highWatermark
|
|
}
|
|
}
|
|
}
|
|
|
|
$result = [ordered]@{
|
|
schema_version = "c18z8.live_service_channel_backpressure_isolation_smoke.v1"
|
|
run_id = $runId
|
|
base_url = $ApiBaseUrl
|
|
entry_base_url = $EntryBaseUrl
|
|
cluster_id = $ClusterID
|
|
entry_node = @{ name = $entryNode.name; id = $entryNode.id }
|
|
exit_node = @{ name = $exitNode.name; id = $exitNode.id }
|
|
resource_id_prefix = $resourceId
|
|
route_intents = @{
|
|
primary_route_intent_id = $primaryRouteID
|
|
alternate_route_intent_id = $alternateRouteID
|
|
expired_primary_status = $expiredPrimary.route_intent.lifecycle_status
|
|
expired_alternate_status = $expiredAlternate.route_intent.lifecycle_status
|
|
}
|
|
pressure = @{
|
|
interactive_session_count = $InteractiveSessionCount
|
|
interactive_rounds = $InteractiveRounds
|
|
interactive_packets_per_batch = $InteractivePacketsPerBatch
|
|
expected_interactive_packets_per_session = $expectedInteractivePacketsPerSession
|
|
expected_interactive_total_packets = $expectedInteractiveTotalPackets
|
|
abusive_packet_count = $AbusivePacketCount
|
|
expected_abusive_delivered = $ExpectedAbusiveDelivered
|
|
minimum_abusive_delivered = $minimumAbusiveDelivered
|
|
expected_abusive_drop_delta = $expectedAbusiveDropDelta
|
|
expected_total_delivered_minimum = $expectedTotalDeliveredMinimum
|
|
sent_total_batches = (@($sessionResults | Measure-Object -Property sent_batches -Sum).Sum)
|
|
sent_total_packets = (@($sessionResults | Measure-Object -Property sent_packets -Sum).Sum)
|
|
}
|
|
route_failures = @{
|
|
baseline = $baselineRouteFailures
|
|
final = $finalRouteFailures
|
|
delta = $routeFailureDelta
|
|
}
|
|
flow_drops = @{
|
|
baseline = $baselineDropped
|
|
final = $finalDropped
|
|
delta = $droppedDelta
|
|
expected_min_delta = $expectedAbusiveDropDelta
|
|
max_channel_drop_delta = $maxChannelDropDelta
|
|
max_channel_drop_id = $dropChannelID
|
|
max_channel_enqueued_delta = $maxChannelEnqueuedDelta
|
|
max_channel_enqueued_id = $enqueuedChannelID
|
|
}
|
|
exit_queue = @{
|
|
total_depth = (@($sessionResults | Measure-Object -Property exit_depth -Sum).Sum)
|
|
}
|
|
backend_fallback_queue = @{
|
|
total_delta = (@($sessionResults | Measure-Object -Property backend_delta -Sum).Sum)
|
|
}
|
|
sessions = $sessionResults
|
|
passed = $true
|
|
checks = [ordered]@{
|
|
production_forwarding_ready = ($readyBefore.report.production_payload_forwarding -eq $true)
|
|
exit_production_forwarding_ready = ($exitReadyBefore.report.production_payload_forwarding -eq $true)
|
|
route_intents_visible_before_pressure = (@($visibleConfig.synthetic_mesh_config.routes | Where-Object { $routeIDs -contains $_.route_id }).Count -ge 2)
|
|
exit_route_intents_visible_before_pressure = (@($exitVisibleConfig.synthetic_mesh_config.routes | Where-Object { $routeIDs -contains $_.route_id }).Count -ge 2)
|
|
entry_runtime_loaded_visible_config = ([string]$loadedConfig.report.config_version -ge [string]$visibleConfig.synthetic_mesh_config.config_version)
|
|
exit_runtime_loaded_visible_config = ([string]$exitLoadedConfig.report.config_version -ge [string]$exitVisibleConfig.synthetic_mesh_config.config_version)
|
|
signed_leases_ready = (@($sessionSpecs | Where-Object { $_.lease.status -eq "ready" }).Count -eq ($InteractiveSessionCount + 1))
|
|
interactive_sessions_sent_all_packets = (@($sessionResults | Where-Object { $_.kind -eq "interactive" -and $_.sent_packets -eq $_.expected_packets }).Count -eq $InteractiveSessionCount)
|
|
interactive_sessions_exit_inbox_received_all_packets = (@($sessionResults | Where-Object { $_.kind -eq "interactive" -and $_.exit_depth -ge ($_.baseline_exit_depth + $_.expected_packets) }).Count -eq $InteractiveSessionCount)
|
|
abusive_session_sent_all_packets = (@($sessionResults | Where-Object { $_.kind -eq "abusive" -and $_.sent_packets -eq $AbusivePacketCount }).Count -eq 1)
|
|
abusive_session_exit_batch_reached_exit = (@($sessionResults | Where-Object { $_.kind -eq "abusive" -and $_.exit_delta -ge 1 }).Count -eq 1)
|
|
abusive_session_bounded_delivery_scheduled = ($maxChannelEnqueuedDelta -ge $ExpectedAbusiveDelivered -and $maxChannelEnqueuedDelta -le $AbusivePacketCount)
|
|
no_backend_fallback_used = ((@($sessionResults | Measure-Object -Property backend_delta -Sum).Sum) -eq 0)
|
|
backpressure_drops_observed = ($droppedDelta -ge $expectedAbusiveDropDelta)
|
|
abusive_channel_drop_observed = ($maxChannelDropDelta -ge $expectedAbusiveDropDelta)
|
|
backpressure_active_visible = ($finalFlowScheduler.backpressure_active -eq $true)
|
|
scheduler_high_watermark_reached = ($maxChannelHighWatermark -ge 900)
|
|
no_route_failures = ($routeFailureDelta -eq 0)
|
|
route_intents_expired = ($expiredPrimary.route_intent.lifecycle_status -eq "expired" -and $expiredAlternate.route_intent.lifecycle_status -eq "expired")
|
|
}
|
|
telemetry = @{
|
|
final_entry_ingress = $finalEntryRuntime.report.ingress
|
|
final_exit_inbox = $finalExitRuntime.report.inbox
|
|
post_pressure_ingress = $postPressureIngress.report.ingress
|
|
baseline_flow_scheduler = $baselineFlowScheduler
|
|
final_flow_scheduler = $finalFlowScheduler
|
|
sessions = $sessionResults
|
|
}
|
|
}
|
|
|
|
$failedChecks = @($result.checks.GetEnumerator() | Where-Object { $_.Value -ne $true })
|
|
if ($failedChecks.Count -gt 0) {
|
|
throw "C18Z8 failed checks: $($failedChecks.Name -join ', ')"
|
|
}
|
|
}
|
|
finally {
|
|
foreach ($session in @($sessions)) {
|
|
try { Close-ServiceChannelWebSocket -Session $session } catch {}
|
|
}
|
|
if ($primaryRouteID) {
|
|
try { Invoke-Api -Method POST -Path "/clusters/$ClusterID/mesh/route-intents/$primaryRouteID/expire" -Body @{ actor_user_id = $ActorUserID } | Out-Null } catch {}
|
|
}
|
|
if ($alternateRouteID) {
|
|
try { Invoke-Api -Method POST -Path "/clusters/$ClusterID/mesh/route-intents/$alternateRouteID/expire" -Body @{ actor_user_id = $ActorUserID } | Out-Null } catch {}
|
|
}
|
|
if ($updatersStopped) {
|
|
try { Start-TestUpdaters } catch { Write-Warning "Could not restart test updaters: $($_.Exception.Message)" }
|
|
}
|
|
}
|
|
|
|
$resultFullPath = Join-Path $repoRoot $ResultPath
|
|
$resultDir = Split-Path -Parent $resultFullPath
|
|
if (-not (Test-Path $resultDir)) {
|
|
New-Item -ItemType Directory -Path $resultDir | Out-Null
|
|
}
|
|
$result | ConvertTo-Json -Depth 100 | Set-Content -Path $resultFullPath -Encoding UTF8
|
|
Write-Host "C18Z8 live service-channel backpressure isolation smoke passed. Result: $resultFullPath"
|
|
$result
|