param( [string]$ApiBaseUrl = "http://192.168.200.61:18121/api/v1", [string]$ClusterID = "cfc0743d-d960-49fb-9de8-96e063d5e4aa", [string]$ActorUserID = "f67d943f-5397-4b3a-a229-695fe67ad700", [string]$EntryNodeName = "test-1", [string]$ExitNodeName = "test-2", [string]$EntryBaseUrl = "http://192.168.200.61:19131", [string]$DockerSSH = "test-docker", [int]$InteractiveSessionCount = 2, [int]$InteractiveRounds = 24, [int]$InteractivePacketsPerBatch = 8, [int]$AbusivePacketCount = 1300, [int]$ExpectedAbusiveDelivered = 1024, [int]$BatchDelayMilliseconds = 25, [string]$RequiredNodeVersion = "0.2.183", [string]$ResultPath = "artifacts\c18z8-live-service-channel-backpressure-isolation-smoke-result.json" ) Set-StrictMode -Version Latest $ErrorActionPreference = "Stop" Add-Type -AssemblyName System.Net.Http $repoRoot = (Resolve-Path (Join-Path $PSScriptRoot "..\..")).ProviderPath $runId = "c18z8-" + (Get-Date -Format "yyyyMMdd-HHmmss") $resourceId = "vpn-$runId" function Invoke-Api { param( [string]$Method, [string]$Path, [object]$Body = $null ) $uri = "$ApiBaseUrl$Path" try { if ($null -eq $Body) { return Invoke-RestMethod -Method $Method -Uri $uri -TimeoutSec 30 } return Invoke-RestMethod -Method $Method -Uri $uri -ContentType "application/json" -Body ($Body | ConvertTo-Json -Depth 80) -TimeoutSec 30 } catch { $statusCode = $null if ($_.Exception.Response) { $statusCode = [int]$_.Exception.Response.StatusCode } $details = $_.ErrorDetails.Message if (-not $details) { $details = $_.Exception.Message } throw "$Method $Path failed with HTTP $statusCode`: $details" } } function Get-NodeByName { param([string]$Name) $nodes = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/nodes?actor_user_id=$ActorUserID").nodes $node = @($nodes | Where-Object { $_.name -eq $Name }) | Select-Object -First 1 if ($null -eq $node) { throw "Node '$Name' was not found in cluster $ClusterID" } return $node } function Get-MeshPort { param([string]$Name) switch ($Name) { "test-1" { return 19131 } "test-2" { return 19132 } "test-3" { return 19133 } default { return 19131 } } } function Enable-TestMeshListener { param([object]$Node) $port = Get-MeshPort -Name $Node.name Invoke-Api -Method PUT -Path "/clusters/$ClusterID/nodes/$($Node.id)/workloads/mesh-listener/desired" -Body @{ actor_user_id = $ActorUserID desired_state = "enabled" runtime_mode = "container" version = "c18z8-live-fsc-backpressure-isolation" config = @{ listen_addr = "0.0.0.0:$port" listen_port_mode = "manual" advertise_endpoint = "http://192.168.200.61:$port" advertise_transport = "direct_http" connectivity_mode = "private_lan" nat_type = "none" region = "docker-test" production_forwarding = $true } environment = @{} } | Out-Null } function Clear-OldSmokeRouteIntents { param( [string]$SourceNodeID, [string]$DestinationNodeID ) $items = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/mesh/route-intents?actor_user_id=$ActorUserID").route_intents foreach ($item in @($items)) { if ([string]$item.lifecycle_status -ne "active") { continue } if ([string]$item.service_class -ne "vpn_packets") { continue } if ([string]$item.source_selector.node_id -ne $SourceNodeID -or [string]$item.destination_selector.node_id -ne $DestinationNodeID) { continue } $smoke = "" if ($null -ne $item.policy -and $null -ne $item.policy.metadata) { $prop = $item.policy.metadata.PSObject.Properties["smoke"] if ($null -ne $prop) { $smoke = [string]$prop.Value } } if ($smoke -ne "c18z1_live_service_channel_ingress" -and $smoke -ne "c18z2_live_service_channel_soak" -and $smoke -ne "c18z3_live_service_channel_entry_ws_fallback" -and $smoke -ne "c18z4_live_service_channel_session_pressure" -and $smoke -ne "c18z5_live_service_channel_exit_restart" -and $smoke -ne "c18z6_live_service_channel_active_rebuild" -and $smoke -ne "c18z7_live_service_channel_concurrent_isolation" -and $smoke -ne "c18z8_live_service_channel_backpressure_isolation") { continue } Invoke-Api -Method POST -Path "/clusters/$ClusterID/mesh/route-intents/$($item.id)/expire" -Body @{ actor_user_id = $ActorUserID } | Out-Null } } function New-RouteIntent { param( [string]$SourceNodeID, [string]$DestinationNodeID, [int]$Priority, [string]$Label ) $expiresAt = (Get-Date).ToUniversalTime().AddMinutes(10).ToString("o") return Invoke-Api -Method POST -Path "/clusters/$ClusterID/mesh/route-intents" -Body @{ actor_user_id = $ActorUserID source_selector = @{ node_id = $SourceNodeID } destination_selector = @{ node_id = $DestinationNodeID } service_class = "vpn_packets" priority = $Priority policy = @{ synthetic_enabled = $true route_version = "$runId-$Label" policy_version = "$runId-$Label" peer_directory_version = "$runId-$Label" hops = @($SourceNodeID, $DestinationNodeID) allowed_channels = @("vpn_packet", "fabric_control") max_ttl = 8 max_hops = 8 expires_at = $expiresAt metadata = @{ smoke = "c18z8_live_service_channel_backpressure_isolation" run_id = $runId label = $Label } } } } function Get-SyntheticConfig { param([string]$NodeID) return Invoke-Api -Method GET -Path "/clusters/$ClusterID/nodes/$NodeID/mesh/synthetic-config?actor_user_id=$ActorUserID" } function Get-LatestHeartbeat { param([string]$NodeID) return (Invoke-Api -Method GET -Path "/clusters/$ClusterID/nodes/$NodeID/heartbeats?actor_user_id=$ActorUserID&limit=1").heartbeats[0] } function Get-LatestRuntimeReport { param([string]$NodeID) $hb = Get-LatestHeartbeat -NodeID $NodeID return @{ heartbeat = $hb report = $hb.metadata.fabric_service_channel_runtime_report } } function Wait-ForRuntimeReady { param( [string]$NodeID, [int]$MinRoutes, [int]$TimeoutSeconds = 90 ) $deadline = (Get-Date).AddSeconds($TimeoutSeconds) do { $latest = Get-LatestRuntimeReport -NodeID $NodeID $report = $latest.report if ($null -ne $report -and $report.enabled -eq $true -and $report.production_payload_forwarding -eq $true -and [int]$report.route_candidate_total -ge $MinRoutes) { return $latest } Start-Sleep -Seconds 2 } while ((Get-Date) -lt $deadline) throw "Timed out waiting for production service-channel runtime ready on node $NodeID" } function Wait-ForRuntimeConfigVersion { param( [string]$NodeID, [string]$ConfigVersion, [int]$TimeoutSeconds = 90 ) $deadline = (Get-Date).AddSeconds($TimeoutSeconds) do { $latest = Get-LatestRuntimeReport -NodeID $NodeID if ($null -ne $latest.report) { $loadedVersion = [string]$latest.report.config_version if ($loadedVersion -ge $ConfigVersion) { return $latest } } Start-Sleep -Seconds 2 } while ((Get-Date) -lt $deadline) throw "Timed out waiting for node $NodeID to load synthetic config $ConfigVersion" } function Wait-ForRouteIntentVisible { param( [string]$NodeID, [string[]]$RouteIDs, [int]$TimeoutSeconds = 60 ) $deadline = (Get-Date).AddSeconds($TimeoutSeconds) do { $config = Get-SyntheticConfig -NodeID $NodeID $routes = @($config.synthetic_mesh_config.routes) $present = @($routes | Where-Object { $RouteIDs -contains $_.route_id }) if ($present.Count -ge $RouteIDs.Count) { return $config } Start-Sleep -Seconds 2 } while ((Get-Date) -lt $deadline) throw "Timed out waiting for routes '$($RouteIDs -join ",")' in synthetic config for node $NodeID" } function Wait-ForRouteIntentNotVisible { param( [string]$NodeID, [string]$RouteID, [int]$TimeoutSeconds = 90 ) $deadline = (Get-Date).AddSeconds($TimeoutSeconds) do { $config = Get-SyntheticConfig -NodeID $NodeID $routes = @($config.synthetic_mesh_config.routes) $present = @($routes | Where-Object { $_.route_id -eq $RouteID }) if ($present.Count -eq 0) { return $config } Start-Sleep -Seconds 2 } while ((Get-Date) -lt $deadline) throw "Timed out waiting for route '$RouteID' to disappear from synthetic config for node $NodeID" } function New-ServiceChannelLease { param( [string]$EntryNodeID, [string]$ExitNodeID, [string]$VPNResourceID = $resourceId ) return (Invoke-Api -Method POST -Path "/clusters/$ClusterID/fabric/service-channels/leases" -Body @{ actor_user_id = $ActorUserID organization_id = "org-c18z8-smoke" user_id = $ActorUserID resource_id = $VPNResourceID service_class = "vpn_packets" entry_node_ids = @($EntryNodeID) exit_node_ids = @($ExitNodeID) preferred_entry_node_id = $EntryNodeID preferred_exit_node_id = $ExitNodeID allowed_channels = @("vpn_packet", "bulk", "control") ttl_seconds = 300 metadata = @{ smoke = "c18z8_live_service_channel_backpressure_isolation" run_id = $runId } }).fabric_service_channel_lease } function ConvertTo-Base64UrlJson { param([object]$Value) $json = $Value | ConvertTo-Json -Depth 80 -Compress $bytes = [System.Text.Encoding]::UTF8.GetBytes($json) return [Convert]::ToBase64String($bytes).TrimEnd("=").Replace("+", "-").Replace("/", "_") } function Get-ObjectPropertyValue { param( [object]$Object, [string]$Name ) if ($null -eq $Object) { return $null } $prop = $Object.PSObject.Properties[$Name] if ($null -eq $prop) { return $null } return $prop.Value } function New-TestIPv4UDPPacket { param([int]$SourcePort) $payload = [System.Text.Encoding]::ASCII.GetBytes("c18z1-$SourcePort") $totalLength = 20 + 8 + $payload.Length $packet = New-Object byte[] $totalLength $packet[0] = 0x45 $packet[1] = 0 $packet[2] = [byte](($totalLength -shr 8) -band 0xff) $packet[3] = [byte]($totalLength -band 0xff) $packet[8] = 64 $packet[9] = 17 $packet[12] = 10; $packet[13] = 18; $packet[14] = 1; $packet[15] = 10 $packet[16] = 10; $packet[17] = 18; $packet[18] = 2; $packet[19] = 20 $udpOffset = 20 $destPort = 3389 $udpLength = 8 + $payload.Length $packet[$udpOffset] = [byte](($SourcePort -shr 8) -band 0xff) $packet[$udpOffset + 1] = [byte]($SourcePort -band 0xff) $packet[$udpOffset + 2] = [byte](($destPort -shr 8) -band 0xff) $packet[$udpOffset + 3] = [byte]($destPort -band 0xff) $packet[$udpOffset + 4] = [byte](($udpLength -shr 8) -band 0xff) $packet[$udpOffset + 5] = [byte]($udpLength -band 0xff) [Array]::Copy($payload, 0, $packet, 28, $payload.Length) return $packet } function New-TestIPv4UDPPacketBatch { param( [int]$SourcePort, [int]$Count ) $packets = @() for ($i = 0; $i -lt $Count; $i++) { $packets += ,(New-TestIPv4UDPPacket -SourcePort $SourcePort) } return $packets } function New-PacketBatchBody { param([byte[][]]$Packets) $stream = [System.IO.MemoryStream]::new() foreach ($packet in $Packets) { $length = $packet.Length $stream.WriteByte([byte](($length -shr 24) -band 0xff)) $stream.WriteByte([byte](($length -shr 16) -band 0xff)) $stream.WriteByte([byte](($length -shr 8) -band 0xff)) $stream.WriteByte([byte]($length -band 0xff)) $stream.Write($packet, 0, $packet.Length) } return $stream.ToArray() } function Read-PacketBatchBody { param([byte[]]$Body) $packets = @() $offset = 0 while (($offset + 4) -le $Body.Length) { $length = ([int]$Body[$offset] -shl 24) -bor ([int]$Body[$offset + 1] -shl 16) -bor ([int]$Body[$offset + 2] -shl 8) -bor [int]$Body[$offset + 3] $offset += 4 if ($length -lt 0 -or ($offset + $length) -gt $Body.Length) { throw "Malformed packet batch body: length=$length offset=$offset body_length=$($Body.Length)" } $packet = New-Object byte[] $length [Array]::Copy($Body, $offset, $packet, 0, $length) $packets += ,$packet $offset += $length } if ($offset -ne $Body.Length) { throw "Malformed packet batch body: trailing_bytes=$($Body.Length - $offset)" } return $packets } function Receive-ExitServiceChannelPacketBatch { param( [string]$VPNResourceID, [int]$TimeoutMilliseconds = 5000 ) $exitBaseUrl = "http://192.168.200.61:$(Get-MeshPort -Name $ExitNodeName)" $url = "$exitBaseUrl/api/v1/clusters/$ClusterID/vpn-connections/$VPNResourceID/tunnel/client/packets?batch=true&timeout_ms=$TimeoutMilliseconds" $client = [System.Net.Http.HttpClient]::new() try { $client.Timeout = [TimeSpan]::FromSeconds([Math]::Max(10, [Math]::Ceiling($TimeoutMilliseconds / 1000) + 5)) $response = $client.GetAsync($url).GetAwaiter().GetResult() if ([int]$response.StatusCode -eq 204) { return @() } $body = $response.Content.ReadAsByteArrayAsync().GetAwaiter().GetResult() if (-not $response.IsSuccessStatusCode) { $message = [System.Text.Encoding]::UTF8.GetString($body) throw "Exit service-channel GET $url failed with HTTP $([int]$response.StatusCode): $message" } return @(Read-PacketBatchBody -Body $body) } finally { $client.Dispose() } } function Invoke-ServiceChannelPost { param( [object]$Lease, [int]$PortStart, [string]$VPNResourceID = $resourceId ) $packets = @() for ($i = 0; $i -lt 8; $i++) { $packets += ,(New-TestIPv4UDPPacket -SourcePort ($PortStart + $i)) } $path = $Lease.entry_http.path_template. Replace("{cluster_id}", $ClusterID). Replace("{channel_id}", $Lease.channel_id). Replace("{resource_id}", $VPNResourceID) $url = "$EntryBaseUrl$path`?batch=true" $headers = @{ "X-RAP-Service-Channel-Token" = $Lease.token.token "X-RAP-Fabric-Channel-ID" = $Lease.channel_id "X-RAP-Service-Class" = "vpn_packets" "X-RAP-Channel-Class" = "vpn_packet" "X-RAP-Service-Channel-Authority-Payload" = ConvertTo-Base64UrlJson -Value $Lease.authority_payload "X-RAP-Service-Channel-Authority-Signature" = ConvertTo-Base64UrlJson -Value $Lease.authority_signature } $body = New-PacketBatchBody -Packets $packets $client = [System.Net.Http.HttpClient]::new() try { $client.Timeout = [TimeSpan]::FromSeconds(30) $request = [System.Net.Http.HttpRequestMessage]::new([System.Net.Http.HttpMethod]::Post, $url) foreach ($header in $headers.GetEnumerator()) { [void]$request.Headers.TryAddWithoutValidation($header.Key, [string]$header.Value) } $content = [System.Net.Http.ByteArrayContent]::new($body) $content.Headers.ContentType = [System.Net.Http.Headers.MediaTypeHeaderValue]::Parse("application/vnd.rap.vpn-packet-batch.v1") $request.Content = $content $response = $client.SendAsync($request).GetAwaiter().GetResult() $responseBody = $response.Content.ReadAsStringAsync().GetAwaiter().GetResult() if (-not $response.IsSuccessStatusCode) { throw "Service-channel POST $url failed with HTTP $([int]$response.StatusCode): $responseBody" } return [pscustomobject]@{ StatusCode = [int]$response.StatusCode Body = $responseBody } } finally { $client.Dispose() } } function Get-IngressSendPackets { param([string]$NodeID) $latest = Get-LatestRuntimeReport -NodeID $NodeID $ingress = $latest.report.ingress $sendPackets = Get-ObjectPropertyValue -Object $ingress -Name "send_packets" if ($null -eq $sendPackets) { return 0 } return [int]$sendPackets } function Get-IngressRouteFailures { param([string]$NodeID) $latest = Get-LatestRuntimeReport -NodeID $NodeID $ingress = $latest.report.ingress $failures = Get-ObjectPropertyValue -Object $ingress -Name "send_route_failures" if ($null -eq $failures) { return 0 } return [int]$failures } function Get-IngressFlowDropped { param([string]$NodeID) $latest = Get-LatestRuntimeReport -NodeID $NodeID $ingress = $latest.report.ingress if ($null -eq $ingress) { return 0 } $flowScheduler = Get-ObjectPropertyValue -Object $ingress -Name "flow_scheduler" if ($null -eq $flowScheduler) { return 0 } $dropped = Get-ObjectPropertyValue -Object $flowScheduler -Name "dropped" if ($null -eq $dropped) { return 0 } return [int]$dropped } function Get-ExitQueueDepth { param( [string]$NodeID, [string]$VPNConnectionID ) $latest = Get-LatestRuntimeReport -NodeID $NodeID $queueKey = "$VPNConnectionID`:client_to_gateway" $depths = $latest.report.inbox.queue_depths if ($null -eq $depths) { return 0 } $prop = $depths.PSObject.Properties[$queueKey] if ($null -eq $prop) { return 0 } return [int]$prop.Value } function Wait-ForExitQueueDepth { param( [string]$NodeID, [string]$VPNConnectionID, [int]$MinDepth, [int]$TimeoutSeconds = 90 ) $deadline = (Get-Date).AddSeconds($TimeoutSeconds) do { $depth = Get-ExitQueueDepth -NodeID $NodeID -VPNConnectionID $VPNConnectionID if ($depth -ge $MinDepth) { return $depth } Start-Sleep -Seconds 2 } while ((Get-Date) -lt $deadline) throw "Timed out waiting for exit queue depth >= $MinDepth on node $NodeID" } function Invoke-ServiceChannelPostSafe { param( [object]$Lease, [int]$PortStart, [string]$VPNResourceID = $resourceId ) try { $response = Invoke-ServiceChannelPost -Lease $Lease -PortStart $PortStart -VPNResourceID $VPNResourceID return [pscustomobject]@{ ok = $true status_code = [int]$response.StatusCode error = "" } } catch { return [pscustomobject]@{ ok = $false status_code = 0 error = $_.Exception.Message } } } function ConvertTo-WebSocketURL { param([string]$URL) if ($URL.StartsWith("https://")) { return "wss://" + $URL.Substring("https://".Length) } if ($URL.StartsWith("http://")) { return "ws://" + $URL.Substring("http://".Length) } return $URL } function Invoke-ServiceChannelWebSocketSend { param( [object]$Lease, [int]$PortStart, [string]$VPNResourceID = $resourceId ) $packets = @() for ($i = 0; $i -lt 8; $i++) { $packets += ,(New-TestIPv4UDPPacket -SourcePort ($PortStart + $i)) } $path = $Lease.entry_http.websocket_path_template. Replace("{cluster_id}", $ClusterID). Replace("{channel_id}", $Lease.channel_id). Replace("{resource_id}", $VPNResourceID) $url = ConvertTo-WebSocketURL -URL "$EntryBaseUrl$path" $socket = [System.Net.WebSockets.ClientWebSocket]::new() $cts = [System.Threading.CancellationTokenSource]::new([TimeSpan]::FromSeconds(20)) try { $null = $socket.Options.SetRequestHeader("X-RAP-Service-Channel-Token", [string]$Lease.token.token) $null = $socket.Options.SetRequestHeader("X-RAP-Fabric-Channel-ID", [string]$Lease.channel_id) $null = $socket.Options.SetRequestHeader("X-RAP-Service-Class", "vpn_packets") $null = $socket.Options.SetRequestHeader("X-RAP-Channel-Class", "vpn_packet") $null = $socket.Options.SetRequestHeader("X-RAP-Service-Channel-Authority-Payload", (ConvertTo-Base64UrlJson -Value $Lease.authority_payload)) $null = $socket.Options.SetRequestHeader("X-RAP-Service-Channel-Authority-Signature", (ConvertTo-Base64UrlJson -Value $Lease.authority_signature)) $null = $socket.ConnectAsync([Uri]$url, $cts.Token).GetAwaiter().GetResult() $body = New-PacketBatchBody -Packets $packets $segment = [ArraySegment[byte]]::new($body) $null = $socket.SendAsync($segment, [System.Net.WebSockets.WebSocketMessageType]::Binary, $true, $cts.Token).GetAwaiter().GetResult() Start-Sleep -Milliseconds 300 if ($socket.State -eq [System.Net.WebSockets.WebSocketState]::Open) { $null = $socket.CloseOutputAsync([System.Net.WebSockets.WebSocketCloseStatus]::NormalClosure, "c18z8 sent", $cts.Token).GetAwaiter().GetResult() } return [pscustomobject]@{ ok = $true url = $url sent_packets = $packets.Count state = [string]$socket.State error = "" } } catch { return [pscustomobject]@{ ok = $false url = $url sent_packets = 0 state = [string]$socket.State error = $_.Exception.Message } } finally { $socket.Dispose() $cts.Dispose() } } function Invoke-ServiceChannelWebSocketPressure { param( [object]$Lease, [int]$PortStart, [int]$PreSwitchBatches, [int]$PostSwitchBatches, [int]$PacketsInBatch, [int]$DelayMilliseconds, [scriptblock]$AfterPreSwitchAction, [string]$VPNResourceID = $resourceId ) $path = $Lease.entry_http.websocket_path_template. Replace("{cluster_id}", $ClusterID). Replace("{channel_id}", $Lease.channel_id). Replace("{resource_id}", $VPNResourceID) $url = ConvertTo-WebSocketURL -URL "$EntryBaseUrl$path" $socket = [System.Net.WebSockets.ClientWebSocket]::new() $cts = [System.Threading.CancellationTokenSource]::new([TimeSpan]::FromSeconds(120)) $sentBatches = 0 $sentPackets = 0 $switchActionRan = $false try { $null = $socket.Options.SetRequestHeader("X-RAP-Service-Channel-Token", [string]$Lease.token.token) $null = $socket.Options.SetRequestHeader("X-RAP-Fabric-Channel-ID", [string]$Lease.channel_id) $null = $socket.Options.SetRequestHeader("X-RAP-Service-Class", "vpn_packets") $null = $socket.Options.SetRequestHeader("X-RAP-Channel-Class", "vpn_packet") $null = $socket.Options.SetRequestHeader("X-RAP-Service-Channel-Authority-Payload", (ConvertTo-Base64UrlJson -Value $Lease.authority_payload)) $null = $socket.Options.SetRequestHeader("X-RAP-Service-Channel-Authority-Signature", (ConvertTo-Base64UrlJson -Value $Lease.authority_signature)) $null = $socket.ConnectAsync([Uri]$url, $cts.Token).GetAwaiter().GetResult() $totalBatches = $PreSwitchBatches + $PostSwitchBatches for ($batch = 0; $batch -lt $totalBatches; $batch++) { if ($batch -eq $PreSwitchBatches -and $null -ne $AfterPreSwitchAction) { & $AfterPreSwitchAction $switchActionRan = $true } $packets = @() for ($i = 0; $i -lt $PacketsInBatch; $i++) { $packets += ,(New-TestIPv4UDPPacket -SourcePort ($PortStart + ($batch * 100) + $i)) } $body = New-PacketBatchBody -Packets $packets $segment = [ArraySegment[byte]]::new($body) $null = $socket.SendAsync($segment, [System.Net.WebSockets.WebSocketMessageType]::Binary, $true, $cts.Token).GetAwaiter().GetResult() $sentBatches++ $sentPackets += $packets.Count if ($DelayMilliseconds -gt 0) { Start-Sleep -Milliseconds $DelayMilliseconds } } Start-Sleep -Milliseconds 500 if ($socket.State -eq [System.Net.WebSockets.WebSocketState]::Open) { $null = $socket.CloseOutputAsync([System.Net.WebSockets.WebSocketCloseStatus]::NormalClosure, "c18z8 sent", $cts.Token).GetAwaiter().GetResult() } return [pscustomobject]@{ ok = $true url = $url sent_batches = $sentBatches sent_packets = $sentPackets switch_action_ran = $switchActionRan state = [string]$socket.State error = "" } } catch { return [pscustomobject]@{ ok = $false url = $url sent_batches = $sentBatches sent_packets = $sentPackets switch_action_ran = $switchActionRan state = [string]$socket.State error = $_.Exception.Message } } finally { $socket.Dispose() $cts.Dispose() } } function Open-ServiceChannelWebSocket { param( [object]$Lease, [string]$VPNResourceID ) $path = $Lease.entry_http.websocket_path_template. Replace("{cluster_id}", $ClusterID). Replace("{channel_id}", $Lease.channel_id). Replace("{resource_id}", $VPNResourceID) $url = ConvertTo-WebSocketURL -URL "$EntryBaseUrl$path" $socket = [System.Net.WebSockets.ClientWebSocket]::new() $cts = [System.Threading.CancellationTokenSource]::new([TimeSpan]::FromSeconds(180)) $null = $socket.Options.SetRequestHeader("X-RAP-Service-Channel-Token", [string]$Lease.token.token) $null = $socket.Options.SetRequestHeader("X-RAP-Fabric-Channel-ID", [string]$Lease.channel_id) $null = $socket.Options.SetRequestHeader("X-RAP-Service-Class", "vpn_packets") $null = $socket.Options.SetRequestHeader("X-RAP-Channel-Class", "vpn_packet") $null = $socket.Options.SetRequestHeader("X-RAP-Service-Channel-Authority-Payload", (ConvertTo-Base64UrlJson -Value $Lease.authority_payload)) $null = $socket.Options.SetRequestHeader("X-RAP-Service-Channel-Authority-Signature", (ConvertTo-Base64UrlJson -Value $Lease.authority_signature)) $null = $socket.ConnectAsync([Uri]$url, $cts.Token).GetAwaiter().GetResult() return [pscustomobject]@{ lease = $Lease resource_id = $VPNResourceID socket = $socket cts = $cts url = $url sent_batches = 0 sent_packets = 0 error = "" } } function Send-ServiceChannelWebSocketBatch { param( [object]$Session, [int]$PortStart, [int]$PacketsInBatch ) $packets = @() for ($i = 0; $i -lt $PacketsInBatch; $i++) { $packets += ,(New-TestIPv4UDPPacket -SourcePort ($PortStart + $i)) } $body = New-PacketBatchBody -Packets $packets $segment = [ArraySegment[byte]]::new($body) $null = $Session.socket.SendAsync($segment, [System.Net.WebSockets.WebSocketMessageType]::Binary, $true, $Session.cts.Token).GetAwaiter().GetResult() $Session.sent_batches = [int]$Session.sent_batches + 1 $Session.sent_packets = [int]$Session.sent_packets + $packets.Count } function Send-ServiceChannelWebSocketPacketBatch { param( [object]$Session, [byte[][]]$Packets ) $body = New-PacketBatchBody -Packets $Packets $segment = [ArraySegment[byte]]::new($body) $null = $Session.socket.SendAsync($segment, [System.Net.WebSockets.WebSocketMessageType]::Binary, $true, $Session.cts.Token).GetAwaiter().GetResult() $Session.sent_batches = [int]$Session.sent_batches + 1 $Session.sent_packets = [int]$Session.sent_packets + $Packets.Count } function Close-ServiceChannelWebSocket { param([object]$Session) if ($null -eq $Session) { return } try { if ($null -ne $Session.socket -and $Session.socket.State -eq [System.Net.WebSockets.WebSocketState]::Open) { $null = $Session.socket.CloseOutputAsync([System.Net.WebSockets.WebSocketCloseStatus]::NormalClosure, "c18z8 sent", $Session.cts.Token).GetAwaiter().GetResult() } } catch {} try { if ($null -ne $Session.socket) { $Session.socket.Dispose() } } catch {} try { if ($null -ne $Session.cts) { $Session.cts.Dispose() } } catch {} } function Send-BatchSeries { param( [object]$Lease, [int]$Count, [int]$PortBase, [int]$DelayMilliseconds = 100, [string]$VPNResourceID = $resourceId ) $results = @() for ($i = 0; $i -lt $Count; $i++) { $results += Invoke-ServiceChannelPostSafe -Lease $Lease -PortStart ($PortBase + ($i * 100)) -VPNResourceID $VPNResourceID if ($DelayMilliseconds -gt 0) { Start-Sleep -Milliseconds $DelayMilliseconds } } return $results } function Invoke-RemoteDocker { param([string]$Command) & ssh $DockerSSH $Command if ($LASTEXITCODE -ne 0) { throw "ssh $DockerSSH command failed: $Command" } } function Stop-TestUpdaters { Invoke-RemoteDocker -Command "docker stop rap_host_agent_updater_test-1 rap_host_agent_updater_test-2 rap_host_agent_updater_test-3 >/dev/null 2>&1 || true" } function Start-TestUpdaters { Invoke-RemoteDocker -Command "docker start rap_host_agent_updater_test-1 rap_host_agent_updater_test-2 rap_host_agent_updater_test-3 >/dev/null 2>&1 || true" } function Restart-ExitContainer { param([string]$Name) $containerName = "rap_test_node_" + $Name.Replace("-", "_") Invoke-RemoteDocker -Command "docker restart $containerName >/dev/null" } function Restart-NodeContainer { param([string]$Name) $containerName = "rap_test_node_" + $Name.Replace("-", "_") Invoke-RemoteDocker -Command "docker restart $containerName >/dev/null" } function Get-BackendClientGatewayDepth { param([string]$VPNConnectionID) $stats = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/vpn-connections/$VPNConnectionID/tunnel/stats").vpn_packet_stats $queue = $stats.client_to_gateway if ($null -eq $queue) { return 0 } $depthProp = $queue.PSObject.Properties["queue_depth"] if ($null -eq $depthProp) { return 0 } return [int]$depthProp.Value } function Wait-ForIngressRoute { param( [string]$NodeID, [string]$RouteID, [int]$MinSendPackets, [int]$TimeoutSeconds = 45 ) $deadline = (Get-Date).AddSeconds($TimeoutSeconds) do { $latest = Get-LatestRuntimeReport -NodeID $NodeID $ingress = $latest.report.ingress $sendPackets = Get-ObjectPropertyValue -Object $ingress -Name "send_packets" $selectedRoute = Get-ObjectPropertyValue -Object $ingress -Name "last_selected_route_id" if ($null -ne $ingress -and [int]$sendPackets -ge $MinSendPackets -and [string]$selectedRoute -eq $RouteID) { return $latest } Start-Sleep -Seconds 2 } while ((Get-Date) -lt $deadline) throw "Timed out waiting for ingress telemetry route=$RouteID packets>=$MinSendPackets on node $NodeID" } function Wait-ForIngressAnyRoute { param( [string]$NodeID, [string[]]$RouteIDs, [int]$MinSendPackets, [int]$TimeoutSeconds = 45 ) $deadline = (Get-Date).AddSeconds($TimeoutSeconds) do { $latest = Get-LatestRuntimeReport -NodeID $NodeID $ingress = $latest.report.ingress $sendPackets = Get-ObjectPropertyValue -Object $ingress -Name "send_packets" $selectedRoute = Get-ObjectPropertyValue -Object $ingress -Name "last_selected_route_id" if ($null -ne $ingress -and [int]$sendPackets -ge $MinSendPackets -and $RouteIDs -contains [string]$selectedRoute) { return $latest } Start-Sleep -Seconds 2 } while ((Get-Date) -lt $deadline) throw "Timed out waiting for ingress telemetry routes='$($RouteIDs -join ",")' packets>=$MinSendPackets on node $NodeID" } function Wait-ForExitInbox { param( [string]$NodeID, [string]$VPNConnectionID, [int]$TimeoutSeconds = 45 ) $queueKey = "$VPNConnectionID`:client_to_gateway" $deadline = (Get-Date).AddSeconds($TimeoutSeconds) do { $latest = Get-LatestRuntimeReport -NodeID $NodeID $depths = $latest.report.inbox.queue_depths if ($null -ne $depths) { $prop = $depths.PSObject.Properties[$queueKey] if ($null -ne $prop -and [int]$prop.Value -gt 0) { return $latest } } Start-Sleep -Seconds 2 } while ((Get-Date) -lt $deadline) throw "Timed out waiting for exit inbox queue '$queueKey' on node $NodeID" } function Send-FeedbackHeartbeat { param( [string]$EntryNodeID, [string]$BadRouteID, [string]$GoodRouteID ) return Invoke-Api -Method POST -Path "/clusters/$ClusterID/nodes/$EntryNodeID/heartbeats" -Body @{ health_status = "healthy" reported_version = $RequiredNodeVersion capabilities = @{ native_node_agent = $true fabric_service_channel_runtime = $true fabric_service_channel_route_manager = $true smoke_feedback_injection = "c18z8" } service_states = @{ smoke = "c18z8_backpressure_isolation_feedback" } metadata = @{ fabric_service_channel_runtime_report = @{ schema_version = "c18l.fabric_service_channel_runtime_report.v1" ingress = @{ flow_scheduler = @{ channel_stats = @{ "c18z8-backpressure-isolation-flow" = @{ last_route_id = $GoodRouteID last_failed_route_id = $BadRouteID last_error = "c18z8 forced stale route during backpressure isolation smoke" consecutive_failures = 3 stall_count = 1 last_send_duration_ms = 250 route_rebuild_recommended = $true degraded_fallback_recommended = $false } } } } } smoke = @{ name = "c18z8_live_service_channel_backpressure_isolation" run_id = $runId } } } } function Wait-ForConfigDecision { param( [string]$NodeID, [string]$BadRouteID, [string]$ExpectedReplacementID, [int]$TimeoutSeconds = 60 ) $deadline = (Get-Date).AddSeconds($TimeoutSeconds) do { $config = Get-SyntheticConfig -NodeID $NodeID $decisions = @($config.synthetic_mesh_config.route_path_decisions.decisions) $decision = @($decisions | Where-Object { $_.route_id -eq $BadRouteID -and $_.rebuild_status -eq "applied" -and $_.replacement_route_id -eq $ExpectedReplacementID }) | Select-Object -First 1 if ($null -ne $decision) { return @{ config = $config decision = $decision } } Start-Sleep -Seconds 2 } while ((Get-Date) -lt $deadline) throw "Timed out waiting for applied rebuild decision $BadRouteID -> $ExpectedReplacementID" } function Wait-ForAppliedRebuildTransition { param( [string]$NodeID, [string]$BadRouteID = "", [string]$ReplacementRouteID = "", [int]$TimeoutSeconds = 90 ) $deadline = (Get-Date).AddSeconds($TimeoutSeconds) do { $latest = Get-LatestRuntimeReport -NodeID $NodeID $transition = $null if ($null -ne $latest.report -and $null -ne $latest.report.ingress) { $prop = $latest.report.ingress.PSObject.Properties["route_manager_transition"] if ($null -ne $prop) { $transition = $prop.Value } } if ($null -ne $transition -and [string]$transition.status -eq "applied_rebuild") { return $latest } if ($BadRouteID -ne "" -and $ReplacementRouteID -ne "") { Send-FeedbackHeartbeat -EntryNodeID $NodeID -BadRouteID $BadRouteID -GoodRouteID $ReplacementRouteID | Out-Null } Start-Sleep -Seconds 2 } while ((Get-Date) -lt $deadline) throw "Timed out waiting for node route-manager transition applied_rebuild on node $NodeID" } $entryNode = Get-NodeByName -Name $EntryNodeName $exitNode = Get-NodeByName -Name $ExitNodeName $primaryRouteID = "" $alternateRouteID = "" $updatersStopped = $false $sessions = @() $result = $null try { Stop-TestUpdaters $updatersStopped = $true Enable-TestMeshListener -Node $entryNode Enable-TestMeshListener -Node $exitNode Clear-OldSmokeRouteIntents -SourceNodeID $entryNode.id -DestinationNodeID $exitNode.id $primaryIntent = New-RouteIntent -SourceNodeID $entryNode.id -DestinationNodeID $exitNode.id -Priority 2000000000 -Label "primary" $alternateIntent = New-RouteIntent -SourceNodeID $entryNode.id -DestinationNodeID $exitNode.id -Priority 1999999999 -Label "alternate" $primaryRouteID = $primaryIntent.route_intent.id $alternateRouteID = $alternateIntent.route_intent.id $routeIDs = @($primaryRouteID, $alternateRouteID) $visibleConfig = Wait-ForRouteIntentVisible -NodeID $entryNode.id -RouteIDs $routeIDs $exitVisibleConfig = Wait-ForRouteIntentVisible -NodeID $exitNode.id -RouteIDs $routeIDs $readyBefore = Wait-ForRuntimeReady -NodeID $entryNode.id -MinRoutes 2 $exitReadyBefore = Wait-ForRuntimeReady -NodeID $exitNode.id -MinRoutes 0 $loadedConfig = Wait-ForRuntimeConfigVersion -NodeID $entryNode.id -ConfigVersion $visibleConfig.synthetic_mesh_config.config_version $exitLoadedConfig = Wait-ForRuntimeConfigVersion -NodeID $exitNode.id -ConfigVersion $exitVisibleConfig.synthetic_mesh_config.config_version $baselineSendPackets = Get-IngressSendPackets -NodeID $entryNode.id $baselineRouteFailures = Get-IngressRouteFailures -NodeID $entryNode.id $baselineDropped = Get-IngressFlowDropped -NodeID $entryNode.id $baselineFlowScheduler = $readyBefore.report.ingress.flow_scheduler $sessionSpecs = @() $abusiveResourceID = "$resourceId-abusive" $abusiveLease = New-ServiceChannelLease -EntryNodeID $entryNode.id -ExitNodeID $exitNode.id -VPNResourceID $abusiveResourceID if ($abusiveLease.status -ne "ready") { throw "Lease status for abusive session was '$($abusiveLease.status)', want ready" } $sessionSpecs += [pscustomobject]@{ kind = "abusive" index = 0 resource_id = $abusiveResourceID lease = $abusiveLease baseline_exit_depth = Get-ExitQueueDepth -NodeID $exitNode.id -VPNConnectionID $abusiveResourceID baseline_backend_depth = Get-BackendClientGatewayDepth -VPNConnectionID $abusiveResourceID expected_packets = $ExpectedAbusiveDelivered } for ($index = 0; $index -lt $InteractiveSessionCount; $index++) { $sessionResourceID = "$resourceId-i$($index + 1)" $lease = New-ServiceChannelLease -EntryNodeID $entryNode.id -ExitNodeID $exitNode.id -VPNResourceID $sessionResourceID if ($lease.status -ne "ready") { throw "Lease status for interactive session $($index + 1) was '$($lease.status)', want ready" } $sessionSpecs += [pscustomobject]@{ kind = "interactive" index = $index + 1 resource_id = $sessionResourceID lease = $lease baseline_exit_depth = Get-ExitQueueDepth -NodeID $exitNode.id -VPNConnectionID $sessionResourceID baseline_backend_depth = Get-BackendClientGatewayDepth -VPNConnectionID $sessionResourceID expected_packets = ($InteractiveRounds * $InteractivePacketsPerBatch) } } foreach ($spec in $sessionSpecs) { $session = Open-ServiceChannelWebSocket -Lease $spec.lease -VPNResourceID $spec.resource_id $session | Add-Member -NotePropertyName kind -NotePropertyValue $spec.kind $session | Add-Member -NotePropertyName index -NotePropertyValue $spec.index $sessions += $session } $abusiveSession = @($sessions | Where-Object { $_.kind -eq "abusive" }) | Select-Object -First 1 $interactiveSessions = @($sessions | Where-Object { $_.kind -eq "interactive" }) $abusiveSent = $false for ($round = 0; $round -lt $InteractiveRounds; $round++) { foreach ($session in $interactiveSessions) { Send-ServiceChannelWebSocketBatch -Session $session -PortStart (24000 + ($round * 100) + ([int]$session.index * 10)) -PacketsInBatch $InteractivePacketsPerBatch } if (-not $abusiveSent -and $round -ge [Math]::Floor($InteractiveRounds / 3)) { [byte[][]]$abusivePackets = New-TestIPv4UDPPacketBatch -SourcePort 39000 -Count $AbusivePacketCount Send-ServiceChannelWebSocketPacketBatch -Session $abusiveSession -Packets $abusivePackets $abusiveSent = $true } if ($BatchDelayMilliseconds -gt 0) { Start-Sleep -Milliseconds $BatchDelayMilliseconds } } if (-not $abusiveSent) { [byte[][]]$abusivePackets = New-TestIPv4UDPPacketBatch -SourcePort 39000 -Count $AbusivePacketCount Send-ServiceChannelWebSocketPacketBatch -Session $abusiveSession -Packets $abusivePackets } Start-Sleep -Milliseconds 500 foreach ($session in $sessions) { Close-ServiceChannelWebSocket -Session $session } $expectedInteractivePacketsPerSession = $InteractiveRounds * $InteractivePacketsPerBatch $expectedInteractiveTotalPackets = $expectedInteractivePacketsPerSession * $InteractiveSessionCount $expectedAbusiveDropDelta = [Math]::Max(1, ($AbusivePacketCount - $ExpectedAbusiveDelivered)) $minimumAbusiveDelivered = [Math]::Min($ExpectedAbusiveDelivered, 960) $expectedTotalDeliveredMinimum = $expectedInteractiveTotalPackets + $minimumAbusiveDelivered $postPressureIngress = Wait-ForIngressAnyRoute -NodeID $entryNode.id -RouteIDs $routeIDs -MinSendPackets ($baselineSendPackets + $expectedTotalDeliveredMinimum) -TimeoutSeconds 120 $sessionResults = @() foreach ($i in 0..($sessionSpecs.Count - 1)) { $spec = $sessionSpecs[$i] $session = $sessions[$i] $minDepth = [int]$spec.baseline_exit_depth + [int]$spec.expected_packets if ($spec.kind -eq "abusive") { $minDepth = [int]$spec.baseline_exit_depth + 1 } $exitDepth = Wait-ForExitQueueDepth -NodeID $exitNode.id -VPNConnectionID $spec.resource_id -MinDepth $minDepth -TimeoutSeconds 120 $deliveredPackets = [int]$exitDepth - [int]$spec.baseline_exit_depth $backendDepth = Get-BackendClientGatewayDepth -VPNConnectionID $spec.resource_id $sessionResults += [pscustomobject]@{ kind = $spec.kind index = $spec.index resource_id = $spec.resource_id channel_id = $spec.lease.channel_id sent_batches = $session.sent_batches sent_packets = $session.sent_packets expected_packets = $spec.expected_packets minimum_expected_depth = $minDepth baseline_exit_depth = $spec.baseline_exit_depth exit_depth = $exitDepth exit_delta = ([int]$exitDepth - [int]$spec.baseline_exit_depth) delivered_packets = $deliveredPackets baseline_backend_depth = $spec.baseline_backend_depth backend_depth = $backendDepth backend_delta = ([int]$backendDepth - [int]$spec.baseline_backend_depth) final_state = [string]$session.socket.State } } $finalEntryRuntime = Get-LatestRuntimeReport -NodeID $entryNode.id $finalExitRuntime = Get-LatestRuntimeReport -NodeID $exitNode.id $finalRouteFailures = Get-IngressRouteFailures -NodeID $entryNode.id $finalDropped = Get-IngressFlowDropped -NodeID $entryNode.id $expiredAlternate = Invoke-Api -Method POST -Path "/clusters/$ClusterID/mesh/route-intents/$alternateRouteID/expire" -Body @{ actor_user_id = $ActorUserID } $expiredPrimary = Invoke-Api -Method POST -Path "/clusters/$ClusterID/mesh/route-intents/$primaryRouteID/expire" -Body @{ actor_user_id = $ActorUserID } $routeFailureDelta = $finalRouteFailures - $baselineRouteFailures $droppedDelta = $finalDropped - $baselineDropped $finalFlowScheduler = $finalEntryRuntime.report.ingress.flow_scheduler $maxChannelDropDelta = 0 $maxChannelEnqueuedDelta = 0 $maxChannelHighWatermark = 0 $dropChannelID = "" $enqueuedChannelID = "" $finalChannelStats = Get-ObjectPropertyValue -Object $finalFlowScheduler -Name "channel_stats" if ($null -ne $finalChannelStats) { foreach ($prop in @($finalChannelStats.PSObject.Properties)) { $finalDroppedForChannel = [int](Get-ObjectPropertyValue -Object $prop.Value -Name "dropped") $finalEnqueuedForChannel = [int](Get-ObjectPropertyValue -Object $prop.Value -Name "enqueued") $baselineDroppedForChannel = 0 $baselineEnqueuedForChannel = 0 $baselineStats = Get-ObjectPropertyValue -Object $baselineFlowScheduler -Name "channel_stats" if ($null -ne $baselineStats) { $baselineProp = $baselineStats.PSObject.Properties[$prop.Name] if ($null -ne $baselineProp) { $baselineDroppedForChannel = [int](Get-ObjectPropertyValue -Object $baselineProp.Value -Name "dropped") $baselineEnqueuedForChannel = [int](Get-ObjectPropertyValue -Object $baselineProp.Value -Name "enqueued") } } $deltaForChannel = $finalDroppedForChannel - $baselineDroppedForChannel if ($deltaForChannel -gt $maxChannelDropDelta) { $maxChannelDropDelta = $deltaForChannel $dropChannelID = $prop.Name } $enqueuedDeltaForChannel = $finalEnqueuedForChannel - $baselineEnqueuedForChannel if ($enqueuedDeltaForChannel -gt $maxChannelEnqueuedDelta) { $maxChannelEnqueuedDelta = $enqueuedDeltaForChannel $enqueuedChannelID = $prop.Name } $highWatermark = [int](Get-ObjectPropertyValue -Object $prop.Value -Name "high_watermark") if ($highWatermark -gt $maxChannelHighWatermark) { $maxChannelHighWatermark = $highWatermark } } } $result = [ordered]@{ schema_version = "c18z8.live_service_channel_backpressure_isolation_smoke.v1" run_id = $runId base_url = $ApiBaseUrl entry_base_url = $EntryBaseUrl cluster_id = $ClusterID entry_node = @{ name = $entryNode.name; id = $entryNode.id } exit_node = @{ name = $exitNode.name; id = $exitNode.id } resource_id_prefix = $resourceId route_intents = @{ primary_route_intent_id = $primaryRouteID alternate_route_intent_id = $alternateRouteID expired_primary_status = $expiredPrimary.route_intent.lifecycle_status expired_alternate_status = $expiredAlternate.route_intent.lifecycle_status } pressure = @{ interactive_session_count = $InteractiveSessionCount interactive_rounds = $InteractiveRounds interactive_packets_per_batch = $InteractivePacketsPerBatch expected_interactive_packets_per_session = $expectedInteractivePacketsPerSession expected_interactive_total_packets = $expectedInteractiveTotalPackets abusive_packet_count = $AbusivePacketCount expected_abusive_delivered = $ExpectedAbusiveDelivered minimum_abusive_delivered = $minimumAbusiveDelivered expected_abusive_drop_delta = $expectedAbusiveDropDelta expected_total_delivered_minimum = $expectedTotalDeliveredMinimum sent_total_batches = (@($sessionResults | Measure-Object -Property sent_batches -Sum).Sum) sent_total_packets = (@($sessionResults | Measure-Object -Property sent_packets -Sum).Sum) } route_failures = @{ baseline = $baselineRouteFailures final = $finalRouteFailures delta = $routeFailureDelta } flow_drops = @{ baseline = $baselineDropped final = $finalDropped delta = $droppedDelta expected_min_delta = $expectedAbusiveDropDelta max_channel_drop_delta = $maxChannelDropDelta max_channel_drop_id = $dropChannelID max_channel_enqueued_delta = $maxChannelEnqueuedDelta max_channel_enqueued_id = $enqueuedChannelID } exit_queue = @{ total_depth = (@($sessionResults | Measure-Object -Property exit_depth -Sum).Sum) } backend_fallback_queue = @{ total_delta = (@($sessionResults | Measure-Object -Property backend_delta -Sum).Sum) } sessions = $sessionResults passed = $true checks = [ordered]@{ production_forwarding_ready = ($readyBefore.report.production_payload_forwarding -eq $true) exit_production_forwarding_ready = ($exitReadyBefore.report.production_payload_forwarding -eq $true) route_intents_visible_before_pressure = (@($visibleConfig.synthetic_mesh_config.routes | Where-Object { $routeIDs -contains $_.route_id }).Count -ge 2) exit_route_intents_visible_before_pressure = (@($exitVisibleConfig.synthetic_mesh_config.routes | Where-Object { $routeIDs -contains $_.route_id }).Count -ge 2) entry_runtime_loaded_visible_config = ([string]$loadedConfig.report.config_version -ge [string]$visibleConfig.synthetic_mesh_config.config_version) exit_runtime_loaded_visible_config = ([string]$exitLoadedConfig.report.config_version -ge [string]$exitVisibleConfig.synthetic_mesh_config.config_version) signed_leases_ready = (@($sessionSpecs | Where-Object { $_.lease.status -eq "ready" }).Count -eq ($InteractiveSessionCount + 1)) interactive_sessions_sent_all_packets = (@($sessionResults | Where-Object { $_.kind -eq "interactive" -and $_.sent_packets -eq $_.expected_packets }).Count -eq $InteractiveSessionCount) interactive_sessions_exit_inbox_received_all_packets = (@($sessionResults | Where-Object { $_.kind -eq "interactive" -and $_.exit_depth -ge ($_.baseline_exit_depth + $_.expected_packets) }).Count -eq $InteractiveSessionCount) abusive_session_sent_all_packets = (@($sessionResults | Where-Object { $_.kind -eq "abusive" -and $_.sent_packets -eq $AbusivePacketCount }).Count -eq 1) abusive_session_exit_batch_reached_exit = (@($sessionResults | Where-Object { $_.kind -eq "abusive" -and $_.exit_delta -ge 1 }).Count -eq 1) abusive_session_bounded_delivery_scheduled = ($maxChannelEnqueuedDelta -ge $ExpectedAbusiveDelivered -and $maxChannelEnqueuedDelta -le $AbusivePacketCount) no_backend_fallback_used = ((@($sessionResults | Measure-Object -Property backend_delta -Sum).Sum) -eq 0) backpressure_drops_observed = ($droppedDelta -ge $expectedAbusiveDropDelta) abusive_channel_drop_observed = ($maxChannelDropDelta -ge $expectedAbusiveDropDelta) backpressure_active_visible = ($finalFlowScheduler.backpressure_active -eq $true) scheduler_high_watermark_reached = ($maxChannelHighWatermark -ge 900) no_route_failures = ($routeFailureDelta -eq 0) route_intents_expired = ($expiredPrimary.route_intent.lifecycle_status -eq "expired" -and $expiredAlternate.route_intent.lifecycle_status -eq "expired") } telemetry = @{ final_entry_ingress = $finalEntryRuntime.report.ingress final_exit_inbox = $finalExitRuntime.report.inbox post_pressure_ingress = $postPressureIngress.report.ingress baseline_flow_scheduler = $baselineFlowScheduler final_flow_scheduler = $finalFlowScheduler sessions = $sessionResults } } $failedChecks = @($result.checks.GetEnumerator() | Where-Object { $_.Value -ne $true }) if ($failedChecks.Count -gt 0) { throw "C18Z8 failed checks: $($failedChecks.Name -join ', ')" } } finally { foreach ($session in @($sessions)) { try { Close-ServiceChannelWebSocket -Session $session } catch {} } if ($primaryRouteID) { try { Invoke-Api -Method POST -Path "/clusters/$ClusterID/mesh/route-intents/$primaryRouteID/expire" -Body @{ actor_user_id = $ActorUserID } | Out-Null } catch {} } if ($alternateRouteID) { try { Invoke-Api -Method POST -Path "/clusters/$ClusterID/mesh/route-intents/$alternateRouteID/expire" -Body @{ actor_user_id = $ActorUserID } | Out-Null } catch {} } if ($updatersStopped) { try { Start-TestUpdaters } catch { Write-Warning "Could not restart test updaters: $($_.Exception.Message)" } } } $resultFullPath = Join-Path $repoRoot $ResultPath $resultDir = Split-Path -Parent $resultFullPath if (-not (Test-Path $resultDir)) { New-Item -ItemType Directory -Path $resultDir | Out-Null } $result | ConvertTo-Json -Depth 100 | Set-Content -Path $resultFullPath -Encoding UTF8 Write-Host "C18Z8 live service-channel backpressure isolation smoke passed. Result: $resultFullPath" $result