param( [string]$ApiBaseUrl = "http://192.168.200.61:18121/api/v1", [string]$ClusterID = "cfc0743d-d960-49fb-9de8-96e063d5e4aa", [string]$ActorUserID = "f67d943f-5397-4b3a-a229-695fe67ad700", [string]$EntryNodeName = "test-1", [string]$RelayNodeName = "test-3", [string]$ExitNodeName = "test-2", [string]$EntryBaseUrl = "http://192.168.200.61:19131", [string]$DockerSSH = "test-docker", [int]$InitialBatchCount = 12, [int]$LearningBatchCount = 24, [int]$PostChurnBatchCount = 24, [int]$PacketsPerBatch = 8, [int]$BatchDelayMilliseconds = 25, [string]$ResultPath = "artifacts\c18z14-live-service-channel-active-quality-shift-smoke-result.json" ) Set-StrictMode -Version Latest $ErrorActionPreference = "Stop" Add-Type -AssemblyName System.Net.Http $repoRoot = (Resolve-Path (Join-Path $PSScriptRoot "..\..")).ProviderPath $runId = "c18z14-" + (Get-Date -Format "yyyyMMdd-HHmmss") $resourceId = "vpn-$runId" function Invoke-Api { param([string]$Method, [string]$Path, [object]$Body = $null) $uri = "$ApiBaseUrl$Path" try { if ($null -eq $Body) { return Invoke-RestMethod -Method $Method -Uri $uri -TimeoutSec 30 } return Invoke-RestMethod -Method $Method -Uri $uri -ContentType "application/json" -Body ($Body | ConvertTo-Json -Depth 100) -TimeoutSec 30 } catch { $statusCode = $null if ($_.Exception.Response) { $statusCode = [int]$_.Exception.Response.StatusCode } $details = $_.ErrorDetails.Message if (-not $details) { $details = $_.Exception.Message } throw "$Method $Path failed with HTTP $statusCode`: $details" } } function Get-ObjectPropertyValue { param([object]$Object, [string]$Name) if ($null -eq $Object) { return $null } $prop = $Object.PSObject.Properties[$Name] if ($null -eq $prop) { return $null } return $prop.Value } function Get-NodeByName { param([string]$Name) $nodes = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/nodes?actor_user_id=$ActorUserID").nodes $node = @($nodes | Where-Object { $_.name -eq $Name }) | Select-Object -First 1 if ($null -eq $node) { throw "Node '$Name' was not found in cluster $ClusterID" } return $node } function Get-MeshPort { param([string]$Name) switch ($Name) { "test-1" { return 19131 } "test-2" { return 19132 } "test-3" { return 19133 } default { return 19131 } } } function Enable-TestMeshListener { param([object]$Node) $port = Get-MeshPort -Name $Node.name Invoke-Api -Method PUT -Path "/clusters/$ClusterID/nodes/$($Node.id)/workloads/fabric-listener/desired" -Body @{ actor_user_id = $ActorUserID desired_state = "enabled" runtime_mode = "container" version = "c18z14-live-fsc-active-quality-shift" config = @{ listen_addr = "0.0.0.0:$port" listen_port_mode = "manual" advertise_endpoint = "http://192.168.200.61:$port" advertise_transport = "direct_http" connectivity_mode = "private_lan" nat_type = "none" region = "docker-test" production_forwarding = $true } environment = @{} } | Out-Null } function Get-SmokeRouteLabel { param([object]$RouteIntent) if ($null -eq $RouteIntent -or $null -eq $RouteIntent.PSObject.Properties["policy"]) { return "" } if ($null -eq $RouteIntent.policy -or $null -eq $RouteIntent.policy.PSObject.Properties["metadata"]) { return "" } $metadata = $RouteIntent.policy.metadata if ($null -eq $metadata) { return "" } $smoke = $metadata.PSObject.Properties["smoke"] if ($null -eq $smoke) { return "" } return [string]$smoke.Value } function Clear-SmokeRouteIntents { $items = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/mesh/route-intents?actor_user_id=$ActorUserID").route_intents foreach ($item in @($items)) { if ([string]$item.lifecycle_status -ne "active" -or [string]$item.service_class -ne "vpn_packets") { continue } if ((Get-SmokeRouteLabel -RouteIntent $item) -ne "c18z14_live_service_channel_active_quality_shift") { continue } Invoke-Api -Method POST -Path "/clusters/$ClusterID/mesh/route-intents/$($item.id)/expire" -Body @{ actor_user_id = $ActorUserID } | Out-Null } } function New-RouteIntent { param([string]$SourceNodeID, [string]$DestinationNodeID, [string[]]$Hops, [int]$Priority, [string]$Label) $expiresAt = (Get-Date).ToUniversalTime().AddMinutes(10).ToString("o") return Invoke-Api -Method POST -Path "/clusters/$ClusterID/mesh/route-intents" -Body @{ actor_user_id = $ActorUserID source_selector = @{ node_id = $SourceNodeID } destination_selector = @{ node_id = $DestinationNodeID } service_class = "vpn_packets" priority = $Priority policy = @{ synthetic_enabled = $true route_version = "$runId-$Label" policy_version = "$runId-$Label" peer_directory_version = "$runId-$Label" hops = @($Hops) allowed_channels = @("vpn_packet", "fabric_control") max_ttl = 8 max_hops = 8 expires_at = $expiresAt metadata = @{ smoke = "c18z14_live_service_channel_active_quality_shift" run_id = $runId label = $Label active_quality_shift_smoke = $true } } } } function Get-SyntheticConfig { param([string]$NodeID) return Invoke-Api -Method GET -Path "/clusters/$ClusterID/nodes/$NodeID/mesh/synthetic-config?actor_user_id=$ActorUserID" } function Get-LatestHeartbeat { param([string]$NodeID) return (Invoke-Api -Method GET -Path "/clusters/$ClusterID/nodes/$NodeID/heartbeats?actor_user_id=$ActorUserID&limit=1").heartbeats[0] } function Get-LatestRuntimeReport { param([string]$NodeID) $hb = Get-LatestHeartbeat -NodeID $NodeID return @{ heartbeat = $hb; report = $hb.metadata.fabric_service_channel_runtime_report } } function Wait-ForRuntimeReady { param([string]$NodeID, [int]$MinRoutes, [int]$TimeoutSeconds = 90) $deadline = (Get-Date).AddSeconds($TimeoutSeconds) do { $latest = Get-LatestRuntimeReport -NodeID $NodeID if ($null -ne $latest.report -and $latest.report.enabled -eq $true -and $latest.report.production_payload_forwarding -eq $true -and [int]$latest.report.route_candidate_total -ge $MinRoutes) { return $latest } Start-Sleep -Seconds 2 } while ((Get-Date) -lt $deadline) throw "Timed out waiting for production service-channel runtime ready on node $NodeID" } function Wait-ForRuntimeConfigVersion { param([string]$NodeID, [string]$ConfigVersion, [int]$TimeoutSeconds = 90) $deadline = (Get-Date).AddSeconds($TimeoutSeconds) do { $latest = Get-LatestRuntimeReport -NodeID $NodeID if ($null -ne $latest.report -and [string]$latest.report.config_version -ge $ConfigVersion) { return $latest } Start-Sleep -Seconds 2 } while ((Get-Date) -lt $deadline) throw "Timed out waiting for node $NodeID to load synthetic config $ConfigVersion" } function Wait-ForRouteIntentVisible { param([string]$NodeID, [string[]]$RouteIDs, [int]$TimeoutSeconds = 60) $deadline = (Get-Date).AddSeconds($TimeoutSeconds) do { $config = Get-SyntheticConfig -NodeID $NodeID $present = @($config.synthetic_mesh_config.routes | Where-Object { $RouteIDs -contains $_.route_id }) if ($present.Count -ge $RouteIDs.Count) { return $config } Start-Sleep -Seconds 2 } while ((Get-Date) -lt $deadline) throw "Timed out waiting for routes '$($RouteIDs -join ",")' in synthetic config for node $NodeID" } function Wait-ForRouteIntentNotVisible { param([string]$NodeID, [string]$RouteID, [int]$TimeoutSeconds = 90) $deadline = (Get-Date).AddSeconds($TimeoutSeconds) do { $config = Get-SyntheticConfig -NodeID $NodeID $present = @($config.synthetic_mesh_config.routes | Where-Object { $_.route_id -eq $RouteID }) if ($present.Count -eq 0) { return $config } Start-Sleep -Seconds 2 } while ((Get-Date) -lt $deadline) throw "Timed out waiting for route '$RouteID' to disappear from synthetic config for node $NodeID" } function Wait-ForRouteFeedback { param([string]$ReporterNodeID, [string]$RouteID, [int]$TimeoutSeconds = 120) $deadline = (Get-Date).AddSeconds($TimeoutSeconds) do { $response = Invoke-Api -Method GET -Path "/clusters/$ClusterID/fabric/service-channels/route-feedback?actor_user_id=$ActorUserID&reporter_node_id=$ReporterNodeID&route_id=$RouteID&service_class=vpn_packets" $item = @($response.route_feedback | Where-Object { $_.route_id -eq $RouteID -and $_.feedback_status -eq "healthy" }) | Select-Object -First 1 if ($null -ne $item) { return $item } Start-Sleep -Seconds 2 } while ((Get-Date) -lt $deadline) throw "Timed out waiting for live route feedback for route '$RouteID'" } function Wait-ForQualityPreferenceApplied { param([string]$NodeID, [int]$TimeoutSeconds = 120) $deadline = (Get-Date).AddSeconds($TimeoutSeconds) do { $latest = Get-LatestRuntimeReport -NodeID $NodeID $count = Get-ObjectPropertyValue -Object $latest.report.ingress -Name "route_quality_preference_count" if ($null -ne $count -and [int]$count -gt 0) { return $latest } Start-Sleep -Seconds 2 } while ((Get-Date) -lt $deadline) throw "Timed out waiting for route quality preferences on node $NodeID" } function New-ServiceChannelLease { param([string]$EntryNodeID, [string]$ExitNodeID) return (Invoke-Api -Method POST -Path "/clusters/$ClusterID/fabric/service-channels/leases" -Body @{ actor_user_id = $ActorUserID organization_id = "org-c18z14-smoke" user_id = $ActorUserID resource_id = $resourceId service_class = "vpn_packets" entry_node_ids = @($EntryNodeID) exit_node_ids = @($ExitNodeID) preferred_entry_node_id = $EntryNodeID preferred_exit_node_id = $ExitNodeID allowed_channels = @("vpn_packet", "bulk", "control") ttl_seconds = 300 metadata = @{ smoke = "c18z14_live_service_channel_active_quality_shift"; run_id = $runId } }).fabric_service_channel_lease } function ConvertTo-Base64UrlJson { param([object]$Value) $json = $Value | ConvertTo-Json -Depth 100 -Compress $bytes = [System.Text.Encoding]::UTF8.GetBytes($json) return [Convert]::ToBase64String($bytes).TrimEnd("=").Replace("+", "-").Replace("/", "_") } function ConvertTo-WebSocketURL { param([string]$URL) if ($URL.StartsWith("https://")) { return "wss://" + $URL.Substring("https://".Length) } if ($URL.StartsWith("http://")) { return "ws://" + $URL.Substring("http://".Length) } return $URL } function New-TestIPv4UDPPacket { param([int]$SourcePort) $payload = [System.Text.Encoding]::ASCII.GetBytes("c18z14-$SourcePort") $totalLength = 20 + 8 + $payload.Length $packet = New-Object byte[] $totalLength $packet[0] = 0x45 $packet[2] = [byte](($totalLength -shr 8) -band 0xff) $packet[3] = [byte]($totalLength -band 0xff) $packet[8] = 64 $packet[9] = 17 $packet[12] = 10; $packet[13] = 18; $packet[14] = 14; $packet[15] = 10 $packet[16] = 10; $packet[17] = 18; $packet[18] = 14; $packet[19] = 20 $udpOffset = 20 $destPort = 3389 $udpLength = 8 + $payload.Length $packet[$udpOffset] = [byte](($SourcePort -shr 8) -band 0xff) $packet[$udpOffset + 1] = [byte]($SourcePort -band 0xff) $packet[$udpOffset + 2] = [byte](($destPort -shr 8) -band 0xff) $packet[$udpOffset + 3] = [byte]($destPort -band 0xff) $packet[$udpOffset + 4] = [byte](($udpLength -shr 8) -band 0xff) $packet[$udpOffset + 5] = [byte]($udpLength -band 0xff) [Array]::Copy($payload, 0, $packet, 28, $payload.Length) return $packet } function New-PacketBatchBody { param([byte[][]]$Packets) $stream = [System.IO.MemoryStream]::new() foreach ($packet in $Packets) { $length = $packet.Length $stream.WriteByte([byte](($length -shr 24) -band 0xff)) $stream.WriteByte([byte](($length -shr 16) -band 0xff)) $stream.WriteByte([byte](($length -shr 8) -band 0xff)) $stream.WriteByte([byte]($length -band 0xff)) $stream.Write($packet, 0, $packet.Length) } return $stream.ToArray() } function Open-ServiceChannelWebSocket { param([object]$Lease) $path = $Lease.entry_http.websocket_path_template.Replace("{cluster_id}", $ClusterID).Replace("{channel_id}", $Lease.channel_id).Replace("{resource_id}", $resourceId) $url = ConvertTo-WebSocketURL -URL "$EntryBaseUrl$path" $socket = [System.Net.WebSockets.ClientWebSocket]::new() [void]$socket.Options.SetRequestHeader("X-RAP-Service-Channel-Token", [string]$Lease.token.token) [void]$socket.Options.SetRequestHeader("X-RAP-Fabric-Channel-ID", [string]$Lease.channel_id) [void]$socket.Options.SetRequestHeader("X-RAP-Service-Class", "vpn_packets") [void]$socket.Options.SetRequestHeader("X-RAP-Channel-Class", "vpn_packet") [void]$socket.Options.SetRequestHeader("X-RAP-Service-Channel-Authority-Payload", (ConvertTo-Base64UrlJson -Value $Lease.authority_payload)) [void]$socket.Options.SetRequestHeader("X-RAP-Service-Channel-Authority-Signature", (ConvertTo-Base64UrlJson -Value $Lease.authority_signature)) $cts = [System.Threading.CancellationTokenSource]::new([TimeSpan]::FromSeconds(30)) [void]$socket.ConnectAsync([Uri]$url, $cts.Token).GetAwaiter().GetResult() $cts.Dispose() return @{ Socket = $socket Url = $url SentBatches = 0 SentPackets = 0 } } function Send-ServiceChannelWebSocketBatches { param([object]$Session, [int]$Count, [int]$PortBase) for ($batch = 0; $batch -lt $Count; $batch++) { $packets = @() for ($i = 0; $i -lt $PacketsPerBatch; $i++) { $packets += ,(New-TestIPv4UDPPacket -SourcePort ($PortBase + ($batch * 100) + $i)) } $body = New-PacketBatchBody -Packets $packets $segment = [ArraySegment[byte]]::new($body) $cts = [System.Threading.CancellationTokenSource]::new([TimeSpan]::FromSeconds(30)) [void]$Session.Socket.SendAsync($segment, [System.Net.WebSockets.WebSocketMessageType]::Binary, $true, $cts.Token).GetAwaiter().GetResult() $cts.Dispose() $Session.SentBatches++ $Session.SentPackets += $packets.Count if ($BatchDelayMilliseconds -gt 0) { Start-Sleep -Milliseconds $BatchDelayMilliseconds } } Start-Sleep -Milliseconds 500 } function Close-ServiceChannelWebSocket { param([object]$Session) if ($null -eq $Session -or $null -eq $Session.Socket) { return } try { if ($Session.Socket.State -eq [System.Net.WebSockets.WebSocketState]::Open) { $cts = [System.Threading.CancellationTokenSource]::new([TimeSpan]::FromSeconds(10)) [void]$Session.Socket.CloseOutputAsync([System.Net.WebSockets.WebSocketCloseStatus]::NormalClosure, "c18z14 sent", $cts.Token).GetAwaiter().GetResult() $cts.Dispose() } } finally { $Session.Socket.Dispose() } } function Get-ExitQueueDepth { param([string]$NodeID, [string]$VPNConnectionID) $latest = Get-LatestRuntimeReport -NodeID $NodeID $queueKey = "$VPNConnectionID`:client_to_gateway" $depths = $latest.report.inbox.queue_depths if ($null -eq $depths) { return 0 } $prop = $depths.PSObject.Properties[$queueKey] if ($null -eq $prop) { return 0 } return [int]$prop.Value } function Wait-ForExitQueueDepth { param([string]$NodeID, [string]$VPNConnectionID, [int]$MinDepth, [int]$TimeoutSeconds = 90) $deadline = (Get-Date).AddSeconds($TimeoutSeconds) do { $depth = Get-ExitQueueDepth -NodeID $NodeID -VPNConnectionID $VPNConnectionID if ($depth -ge $MinDepth) { return $depth } Start-Sleep -Seconds 2 } while ((Get-Date) -lt $deadline) throw "Timed out waiting for exit queue depth >= $MinDepth on node $NodeID" } function Get-IngressFlowDropped { param([string]$NodeID) $latest = Get-LatestRuntimeReport -NodeID $NodeID $flowScheduler = Get-ObjectPropertyValue -Object $latest.report.ingress -Name "flow_scheduler" $dropped = Get-ObjectPropertyValue -Object $flowScheduler -Name "dropped" if ($null -eq $dropped) { return 0 } return [int]$dropped } function Get-BackendClientGatewayDepth { param([string]$VPNConnectionID) $stats = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/vpn-connections/$VPNConnectionID/tunnel/stats").vpn_packet_stats $queue = $stats.client_to_gateway if ($null -eq $queue) { return 0 } $depth = Get-ObjectPropertyValue -Object $queue -Name "queue_depth" if ($null -eq $depth) { return 0 } return [int]$depth } function Invoke-RemoteDocker { param([string]$Command) & ssh $DockerSSH $Command if ($LASTEXITCODE -ne 0) { throw "ssh $DockerSSH command failed: $Command" } } function Stop-TestUpdaters { Invoke-RemoteDocker -Command "docker stop rap_host_agent_updater_test-1 rap_host_agent_updater_test-2 rap_host_agent_updater_test-3 >/dev/null 2>&1 || true" } function Start-TestUpdaters { Invoke-RemoteDocker -Command "docker start rap_host_agent_updater_test-1 rap_host_agent_updater_test-2 rap_host_agent_updater_test-3 >/dev/null 2>&1 || true" } $entryNode = Get-NodeByName -Name $EntryNodeName $relayNode = Get-NodeByName -Name $RelayNodeName $exitNode = Get-NodeByName -Name $ExitNodeName $slowInitialRouteID = "" $fastRouteID = "" $slowCandidateRouteID = "" $session = $null $updatersStopped = $false $result = $null try { Stop-TestUpdaters $updatersStopped = $true Enable-TestMeshListener -Node $entryNode Enable-TestMeshListener -Node $relayNode Enable-TestMeshListener -Node $exitNode Clear-SmokeRouteIntents $slowInitial = New-RouteIntent -SourceNodeID $entryNode.id -DestinationNodeID $exitNode.id -Hops @($entryNode.id, $relayNode.id, $exitNode.id) -Priority 1999999960 -Label "slow-initial" $fast = New-RouteIntent -SourceNodeID $entryNode.id -DestinationNodeID $exitNode.id -Hops @($entryNode.id, $exitNode.id) -Priority 1999999950 -Label "fast-live" $slowInitialRouteID = $slowInitial.route_intent.id $fastRouteID = $fast.route_intent.id $visibleConfig = Wait-ForRouteIntentVisible -NodeID $entryNode.id -RouteIDs @($slowInitialRouteID, $fastRouteID) $exitVisibleConfig = Wait-ForRouteIntentVisible -NodeID $exitNode.id -RouteIDs @($slowInitialRouteID, $fastRouteID) $readyBefore = Wait-ForRuntimeReady -NodeID $entryNode.id -MinRoutes 2 $exitReadyBefore = Wait-ForRuntimeReady -NodeID $exitNode.id -MinRoutes 0 $loadedConfig = Wait-ForRuntimeConfigVersion -NodeID $entryNode.id -ConfigVersion $visibleConfig.synthetic_mesh_config.config_version $exitLoadedConfig = Wait-ForRuntimeConfigVersion -NodeID $exitNode.id -ConfigVersion $exitVisibleConfig.synthetic_mesh_config.config_version $lease = New-ServiceChannelLease -EntryNodeID $entryNode.id -ExitNodeID $exitNode.id if ($lease.status -ne "ready" -or [string]$lease.primary_route.route_id -ne $slowInitialRouteID) { throw "Initial lease should select slow route '$slowInitialRouteID': status=$($lease.status) route=$($lease.primary_route.route_id)" } $baselineExitDepth = Get-ExitQueueDepth -NodeID $exitNode.id -VPNConnectionID $resourceId $baselineBackendDepth = Get-BackendClientGatewayDepth -VPNConnectionID $resourceId $baselineDropped = Get-IngressFlowDropped -NodeID $entryNode.id $session = Open-ServiceChannelWebSocket -Lease $lease Send-ServiceChannelWebSocketBatches -Session $session -Count $InitialBatchCount -PortBase 62000 $initialPackets = $InitialBatchCount * $PacketsPerBatch $initialExitDepth = Wait-ForExitQueueDepth -NodeID $exitNode.id -VPNConnectionID $resourceId -MinDepth ($baselineExitDepth + $initialPackets) $initialRuntime = Get-LatestRuntimeReport -NodeID $entryNode.id if ([string]$initialRuntime.report.ingress.last_selected_route_id -ne $slowInitialRouteID) { throw "Initial active websocket route was '$($initialRuntime.report.ingress.last_selected_route_id)', want slow route '$slowInitialRouteID'" } $expiredSlowInitial = Invoke-Api -Method POST -Path "/clusters/$ClusterID/mesh/route-intents/$slowInitialRouteID/expire" -Body @{ actor_user_id = $ActorUserID } $notVisibleConfig = Wait-ForRouteIntentNotVisible -NodeID $entryNode.id -RouteID $slowInitialRouteID $fastOnlyRuntime = Wait-ForRuntimeConfigVersion -NodeID $entryNode.id -ConfigVersion $notVisibleConfig.synthetic_mesh_config.config_version Send-ServiceChannelWebSocketBatches -Session $session -Count $LearningBatchCount -PortBase 64000 $learningPackets = $LearningBatchCount * $PacketsPerBatch $learnedExitDepth = Wait-ForExitQueueDepth -NodeID $exitNode.id -VPNConnectionID $resourceId -MinDepth ($baselineExitDepth + $initialPackets + $learningPackets) -TimeoutSeconds 120 $fastFeedback = Wait-ForRouteFeedback -ReporterNodeID $entryNode.id -RouteID $fastRouteID -TimeoutSeconds 120 $slowCandidate = New-RouteIntent -SourceNodeID $entryNode.id -DestinationNodeID $exitNode.id -Hops @($entryNode.id, $relayNode.id, $exitNode.id) -Priority 1999999960 -Label "slow-candidate" $slowCandidateRouteID = $slowCandidate.route_intent.id $candidateVisibleConfig = Wait-ForRouteIntentVisible -NodeID $entryNode.id -RouteIDs @($slowCandidateRouteID, $fastRouteID) $candidateLoadedConfig = Wait-ForRuntimeConfigVersion -NodeID $entryNode.id -ConfigVersion $candidateVisibleConfig.synthetic_mesh_config.config_version $qualityRuntime = Wait-ForQualityPreferenceApplied -NodeID $entryNode.id -TimeoutSeconds 120 Send-ServiceChannelWebSocketBatches -Session $session -Count $PostChurnBatchCount -PortBase 67000 $postChurnPackets = $PostChurnBatchCount * $PacketsPerBatch $finalExitDepth = Wait-ForExitQueueDepth -NodeID $exitNode.id -VPNConnectionID $resourceId -MinDepth ($baselineExitDepth + $initialPackets + $learningPackets + $postChurnPackets) -TimeoutSeconds 120 $finalRuntime = Get-LatestRuntimeReport -NodeID $entryNode.id if ([string]$finalRuntime.report.ingress.last_selected_route_id -ne $fastRouteID) { throw "Post-churn active websocket route was '$($finalRuntime.report.ingress.last_selected_route_id)', want learned fast route '$fastRouteID'" } $sessionURL = $session.Url $sessionSentBatches = $session.SentBatches $sessionSentPackets = $session.SentPackets Close-ServiceChannelWebSocket -Session $session $session = $null $finalDropped = Get-IngressFlowDropped -NodeID $entryNode.id $finalBackendDepth = Get-BackendClientGatewayDepth -VPNConnectionID $resourceId $expiredFastFeedback = Invoke-Api -Method POST -Path "/clusters/$ClusterID/fabric/service-channels/route-feedback/expire" -Body @{ actor_user_id = $ActorUserID reporter_node_id = $entryNode.id route_id = $fastRouteID service_class = "vpn_packets" reason = "c18z14 active quality shift smoke cleanup" } $expiredSlowCandidate = Invoke-Api -Method POST -Path "/clusters/$ClusterID/mesh/route-intents/$slowCandidateRouteID/expire" -Body @{ actor_user_id = $ActorUserID } $expiredFast = Invoke-Api -Method POST -Path "/clusters/$ClusterID/mesh/route-intents/$fastRouteID/expire" -Body @{ actor_user_id = $ActorUserID } $result = [ordered]@{ schema_version = "c18z14.live_service_channel_active_quality_shift_smoke.v1" run_id = $runId base_url = $ApiBaseUrl entry_base_url = $EntryBaseUrl cluster_id = $ClusterID entry_node = @{ name = $entryNode.name; id = $entryNode.id } relay_node = @{ name = $relayNode.name; id = $relayNode.id } exit_node = @{ name = $exitNode.name; id = $exitNode.id } resource_id = $resourceId route_intents = @{ slow_initial_route_id = $slowInitialRouteID fast_route_id = $fastRouteID slow_candidate_route_id = $slowCandidateRouteID slow_initial_status = $expiredSlowInitial.route_intent.lifecycle_status slow_candidate_status = $expiredSlowCandidate.route_intent.lifecycle_status fast_status = $expiredFast.route_intent.lifecycle_status } lease = @{ status = $lease.status channel_id = $lease.channel_id primary_route_id = $lease.primary_route.route_id } active_websocket = @{ url = $sessionURL initial_batches = $InitialBatchCount learning_batches = $LearningBatchCount post_churn_batches = $PostChurnBatchCount packets_per_batch = $PacketsPerBatch sent_batches = $sessionSentBatches sent_packets = $sessionSentPackets initial_route_id = $initialRuntime.report.ingress.last_selected_route_id final_route_id = $finalRuntime.report.ingress.last_selected_route_id } live_feedback = @{ route_id = $fastFeedback.route_id feedback_status = $fastFeedback.feedback_status score_adjustment = $fastFeedback.score_adjustment reasons = $fastFeedback.reasons last_send_duration_ms = $fastFeedback.last_send_duration_ms expire_result = $expiredFastFeedback.route_feedback_expire } exit_queue = @{ baseline_depth = $baselineExitDepth initial_depth = $initialExitDepth learned_depth = $learnedExitDepth final_depth = $finalExitDepth } degraded_route_queue = @{ baseline_depth = $baselineBackendDepth; depth = $finalBackendDepth } flow_drops = @{ baseline = $baselineDropped; final = $finalDropped; delta = ($finalDropped - $baselineDropped) } passed = $true checks = [ordered]@{ production_forwarding_ready = ($readyBefore.report.production_payload_forwarding -eq $true) exit_production_forwarding_ready = ($exitReadyBefore.report.production_payload_forwarding -eq $true) entry_runtime_loaded_visible_config = ([string]$loadedConfig.report.config_version -ge [string]$visibleConfig.synthetic_mesh_config.config_version) exit_runtime_loaded_visible_config = ([string]$exitLoadedConfig.report.config_version -ge [string]$exitVisibleConfig.synthetic_mesh_config.config_version) initial_lease_selected_slow_route = ([string]$lease.primary_route.route_id -eq $slowInitialRouteID) active_websocket_started_on_slow_route = ([string]$initialRuntime.report.ingress.last_selected_route_id -eq $slowInitialRouteID) fast_only_runtime_loaded_after_slow_expire = ([string]$fastOnlyRuntime.report.config_version -ge [string]$notVisibleConfig.synthetic_mesh_config.config_version) backend_persisted_live_fast_feedback = ([string]$fastFeedback.route_id -eq $fastRouteID -and [string]$fastFeedback.feedback_status -eq "healthy") candidate_runtime_loaded = ([string]$candidateLoadedConfig.report.config_version -ge [string]$candidateVisibleConfig.synthetic_mesh_config.config_version) node_applied_quality_preference = ([int]$qualityRuntime.report.ingress.route_quality_preference_count -gt 0) active_websocket_stayed_on_learned_fast_route_after_churn = ([string]$finalRuntime.report.ingress.last_selected_route_id -eq $fastRouteID) all_packets_reached_exit = ($finalExitDepth -ge ($baselineExitDepth + $initialPackets + $learningPackets + $postChurnPackets)) no_degraded_route_used = ($finalBackendDepth -eq $baselineBackendDepth) no_flow_drops = (($finalDropped - $baselineDropped) -eq 0) route_intents_expired = ($expiredFast.route_intent.lifecycle_status -eq "expired" -and $expiredSlowCandidate.route_intent.lifecycle_status -eq "expired") } telemetry = @{ initial_entry_ingress = $initialRuntime.report.ingress quality_entry_ingress = $qualityRuntime.report.ingress final_entry_ingress = $finalRuntime.report.ingress } } $failedChecks = @($result.checks.GetEnumerator() | Where-Object { $_.Value -ne $true }) if ($failedChecks.Count -gt 0) { throw "C18Z14 failed checks: $($failedChecks.Name -join ', ')" } } finally { if ($session) { try { Close-ServiceChannelWebSocket -Session $session } catch {} } if ($slowInitialRouteID) { try { Invoke-Api -Method POST -Path "/clusters/$ClusterID/mesh/route-intents/$slowInitialRouteID/expire" -Body @{ actor_user_id = $ActorUserID } | Out-Null } catch {} } if ($slowCandidateRouteID) { try { Invoke-Api -Method POST -Path "/clusters/$ClusterID/mesh/route-intents/$slowCandidateRouteID/expire" -Body @{ actor_user_id = $ActorUserID } | Out-Null } catch {} } if ($fastRouteID) { try { Invoke-Api -Method POST -Path "/clusters/$ClusterID/mesh/route-intents/$fastRouteID/expire" -Body @{ actor_user_id = $ActorUserID } | Out-Null } catch {} try { Invoke-Api -Method POST -Path "/clusters/$ClusterID/fabric/service-channels/route-feedback/expire" -Body @{ actor_user_id = $ActorUserID reporter_node_id = $entryNode.id route_id = $fastRouteID service_class = "vpn_packets" reason = "c18z14 active quality shift smoke cleanup" } | Out-Null } catch {} } if ($updatersStopped) { try { Start-TestUpdaters } catch { Write-Warning "Could not restart test updaters: $($_.Exception.Message)" } } } $resultFullPath = Join-Path $repoRoot $ResultPath $resultDir = Split-Path -Parent $resultFullPath if (-not (Test-Path $resultDir)) { New-Item -ItemType Directory -Path $resultDir | Out-Null } $result | ConvertTo-Json -Depth 100 | Set-Content -Path $resultFullPath -Encoding UTF8 Write-Host "C18Z14 live service-channel active quality shift smoke passed. Result: $resultFullPath" $result