param( [string]$ApiBaseUrl = "http://192.168.200.61:18121/api/v1", [string]$ClusterID = "cfc0743d-d960-49fb-9de8-96e063d5e4aa", [string]$ActorUserID = "f67d943f-5397-4b3a-a229-695fe67ad700", [string]$EntryNodeName = "test-1", [string]$RelayNodeName = "test-3", [string]$ExitNodeName = "test-2", [string]$EntryBaseUrl = "http://192.168.200.61:19131", [string]$DockerSSH = "test-docker", [string]$ExpectedBackendImage = "rap-backend:fabric-service-channel-0.2.256-c18z82", [string]$ExpectedNodeAgentImage = "rap-node-agent:0.2.270-c18z95", [string]$ResultPath = "artifacts\c18z67-service-channel-concurrent-qos-live-smoke-result.json" ) Set-StrictMode -Version Latest $ErrorActionPreference = "Stop" $scriptDir = Split-Path -Parent $MyInvocation.MyCommand.Path $repoRoot = (Resolve-Path (Join-Path $scriptDir "..\..")).ProviderPath $runId = "c18z67-" + (Get-Date -Format "yyyyMMdd-HHmmss") function Invoke-Api { param( [string]$Method, [string]$Path, [object]$Body = $null ) if ($null -eq $Body) { return Invoke-RestMethod -Method $Method -Uri "$ApiBaseUrl$Path" -TimeoutSec 30 } return Invoke-RestMethod -Method $Method -Uri "$ApiBaseUrl$Path" -ContentType "application/json" -Body ($Body | ConvertTo-Json -Depth 80) -TimeoutSec 30 } function Get-PropertyValue { param( [object]$Item, [string]$Name, [object]$Default = $null ) if ($null -eq $Item) { return $Default } $property = $Item.PSObject.Properties[$Name] if ($null -eq $property) { return $Default } return $property.Value } function Get-NodeByName { param([string]$Name) $nodes = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/nodes?actor_user_id=$ActorUserID").nodes $node = @($nodes | Where-Object { $_.name -eq $Name }) | Select-Object -First 1 if ($null -eq $node) { throw "Node '$Name' was not found in cluster $ClusterID" } return $node } function New-IPv4TcpPacket { param( [byte[]]$Source = @(10, 77, 0, 2), [byte[]]$Destination = @(192, 168, 200, 95), [int]$SourcePort, [int]$DestinationPort = 3389 ) $packet = [byte[]]::new(40) $packet[0] = 0x45 $packet[2] = 0 $packet[3] = 40 $packet[8] = 64 $packet[9] = 6 [Array]::Copy($Source, 0, $packet, 12, 4) [Array]::Copy($Destination, 0, $packet, 16, 4) $packet[20] = [byte](($SourcePort -shr 8) -band 0xff) $packet[21] = [byte]($SourcePort -band 0xff) $packet[22] = [byte](($DestinationPort -shr 8) -band 0xff) $packet[23] = [byte]($DestinationPort -band 0xff) $packet[32] = 0x50 return $packet } function ConvertTo-VPNPacketBatch { param([byte[][]]$Packets) $buffer = New-Object System.Collections.Generic.List[byte] foreach ($packet in $Packets) { if ($null -eq $packet -or $packet.Length -eq 0) { continue } $size = [uint32]$packet.Length $buffer.Add([byte](($size -shr 24) -band 0xff)) $buffer.Add([byte](($size -shr 16) -band 0xff)) $buffer.Add([byte](($size -shr 8) -band 0xff)) $buffer.Add([byte]($size -band 0xff)) foreach ($value in $packet) { $buffer.Add($value) } } return $buffer.ToArray() } function New-RouteIntent { param( [string]$SourceNodeID, [string]$DestinationNodeID, [string[]]$Hops, [int]$Priority, [string]$Label ) $expiresAt = (Get-Date).ToUniversalTime().AddMinutes(5).ToString("o") return Invoke-Api -Method POST -Path "/clusters/$ClusterID/mesh/route-intents" -Body @{ actor_user_id = $ActorUserID source_selector = @{ node_id = $SourceNodeID } destination_selector = @{ node_id = $DestinationNodeID } service_class = "vpn_packets" priority = $Priority policy = @{ synthetic_enabled = $true route_version = "$runId-$Label" policy_version = "$runId-$Label" peer_directory_version = "$runId-$Label" hops = @($Hops) allowed_channels = @("vpn_packet", "fabric_control") max_ttl = 8 max_hops = 8 expires_at = $expiresAt metadata = @{ smoke = "c18z67_service_channel_concurrent_qos_live" run_id = $runId label = $Label } } } } function Disable-ExistingRouteIntents { param([string]$SourceNodeID, [string]$DestinationNodeID) $items = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/mesh/route-intents?actor_user_id=$ActorUserID").route_intents foreach ($item in @($items)) { if ([string](Get-PropertyValue -Item $item -Name "status" -Default "") -ne "active") { continue } if ([string](Get-PropertyValue -Item $item -Name "service_class" -Default "") -ne "vpn_packets") { continue } $sourceSelector = Get-PropertyValue -Item $item -Name "source_selector" -Default $null $destinationSelector = Get-PropertyValue -Item $item -Name "destination_selector" -Default $null if ([string](Get-PropertyValue -Item $sourceSelector -Name "node_id" -Default "") -ne $SourceNodeID) { continue } if ([string](Get-PropertyValue -Item $destinationSelector -Name "node_id" -Default "") -ne $DestinationNodeID) { continue } [void](Invoke-Api -Method POST -Path "/clusters/$ClusterID/mesh/route-intents/$($item.id)/disable" -Body @{ actor_user_id = $ActorUserID reason = "c18z67 isolate concurrent-qos smoke route pair" }) } } function Send-DegradedHeartbeat { param( [string]$EntryNodeID, [string]$PrimaryRouteID ) $observedAt = (Get-Date).ToUniversalTime().ToString("o") return Invoke-Api -Method POST -Path "/clusters/$ClusterID/nodes/$EntryNodeID/heartbeats" -Body @{ health_status = "healthy" reported_version = "0.2.236" capabilities = @{ fabric_service_channel_runtime = $true fabric_service_channel_route_manager = $true fabric_service_channel_route_quality_feedback = $true smoke_feedback_injection = "c18z60" } service_states = @{ smoke = "c18z67_primary_degraded_alternate_available" } metadata = @{ fabric_service_channel_runtime_report = @{ schema_version = "c18l.fabric_service_channel_runtime_report.v1" config_version = "$runId-primary" cluster_id = $ClusterID local_node_id = $EntryNodeID observed_at = $observedAt ingress = @{ flow_scheduler = @{ schema_version = "rap.fabric_flow_scheduler.v1" service_neutral = $true service_mode = "application_protocol_agnostic" channel_stats = @{ "c18z67-primary-degraded" = @{ last_route_id = $PrimaryRouteID last_failed_route_id = $PrimaryRouteID route_generation = "$runId-primary" last_error = "c18z60 primary route degraded; alternate available" last_send_duration_ms = 980 consecutive_failures = 3 stall_count = 2 route_rebuild_recommended = $true degraded_fallback_recommended = $false quality_window_sample_count = 8 quality_window_success_count = 2 quality_window_failure_count = 3 quality_window_slow_count = 2 quality_window_drop_count = 1 quality_window_avg_latency_ms = 980 quality_window_last_updated_at = $observedAt } } } } } smoke = @{ name = "c18z67_service_channel_concurrent_qos_live" run_id = $runId } } } } $entryNode = Get-NodeByName -Name $EntryNodeName $relayNode = Get-NodeByName -Name $RelayNodeName $exitNode = Get-NodeByName -Name $ExitNodeName $primaryRouteID = "" $alternateRouteID = "" $result = $null $failedChecks = @() $expectedPressurePacketCount = 512 $expectedPressureFlowCount = 8 $expectedInteractivePacketCount = 1 $expectedBulkRequestCount = 6 try { [void](Disable-ExistingRouteIntents -SourceNodeID $entryNode.id -DestinationNodeID $exitNode.id) $primary = (New-RouteIntent -SourceNodeID $entryNode.id -DestinationNodeID $exitNode.id -Hops @($entryNode.id, $exitNode.id) -Priority 2100000000 -Label "primary").route_intent $alternate = (New-RouteIntent -SourceNodeID $entryNode.id -DestinationNodeID $exitNode.id -Hops @($entryNode.id, $relayNode.id, $exitNode.id) -Priority 2099999900 -Label "alternate").route_intent $primaryRouteID = [string]$primary.id $alternateRouteID = [string]$alternate.id $resourceID = "c18z67-vpn-smoke" $lease = (Invoke-Api -Method POST -Path "/clusters/$ClusterID/fabric/service-channels/leases" -Body @{ actor_user_id = $ActorUserID organization_id = "smoke-org" user_id = "smoke-user" resource_id = $resourceID service_class = "vpn_packets" entry_node_ids = @([string]$entryNode.id) exit_node_ids = @([string]$exitNode.id) preferred_entry_node_id = [string]$entryNode.id preferred_exit_node_id = [string]$exitNode.id allowed_channels = @("vpn_packet", "fabric_control") ttl_seconds = 180 metadata = @{ smoke = "c18z67_service_channel_concurrent_qos_live" run_id = $runId } }).fabric_service_channel_lease [void](Send-DegradedHeartbeat -EntryNodeID $entryNode.id -PrimaryRouteID $primaryRouteID) $packetPath = $lease.entry_http.path_template. Replace("{cluster_id}", $ClusterID). Replace("{channel_id}", [string]$lease.channel_id). Replace("{resource_id}", $resourceID) $packetUrl = $EntryBaseUrl.TrimEnd("/") + $packetPath $headers = @{ "X-RAP-Service-Channel-Token" = [string]$lease.token.token "X-RAP-Fabric-Channel-ID" = [string]$lease.channel_id "X-RAP-Service-Class" = "vpn_packets" "X-RAP-Channel-Class" = "vpn_packet" } $response = Invoke-WebRequest -Method Post -Uri $packetUrl -Headers $headers -Body ([System.Text.Encoding]::UTF8.GetBytes("c18z67-alternate-remediation")) -ContentType "application/octet-stream" -TimeoutSec 30 $acceptedBy = [string]$response.Headers["X-RAP-Service-Channel-Accepted-By"] $accessTelemetry = $null $matchingChannel = $null for ($i = 0; $i -lt 10; $i++) { Start-Sleep -Seconds 3 $accessTelemetry = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/fabric/service-channels/access-telemetry?actor_user_id=$ActorUserID&limit=20").fabric_service_channel_access_telemetry $channels = @() if ($accessTelemetry.PSObject.Properties.Name -contains "active_channels") { $channels = @($accessTelemetry.active_channels) } $matchingChannel = $channels | Where-Object { $_.channel_id -eq $lease.channel_id } | Select-Object -First 1 if ($null -ne $matchingChannel -and [string](Get-PropertyValue -Item $matchingChannel -Name "remediation_action" -Default "") -eq "prefer_alternate_route") { break } } $backendLine = (& ssh $DockerSSH "docker ps --format '{{.Names}} {{.Image}} {{.Status}}' | grep '^rap_test_backend '") | Out-String $nodeLines = (& ssh $DockerSSH "docker ps --format '{{.Names}} {{.Image}} {{.Status}}' | grep '^rap_test_node_test_'") | Out-String $leasePrimaryRouteID = [string](Get-PropertyValue -Item (Get-PropertyValue -Item $lease -Name "primary_route" -Default $null) -Name "route_id" -Default "") $leaseAlternates = @() if ($lease.PSObject.Properties.Name -contains "alternate_routes") { $leaseAlternates = @($lease.alternate_routes) } $leaseHasAlternate = (@($leaseAlternates | Where-Object { [string]$_.route_id -eq $alternateRouteID }).Count -ge 1) $remediationCommand = Get-PropertyValue -Item $matchingChannel -Name "remediation_command" -Default $null $commandAction = [string](Get-PropertyValue -Item $remediationCommand -Name "action" -Default "") $commandPrimaryRouteID = [string](Get-PropertyValue -Item $remediationCommand -Name "primary_route_id" -Default "") $commandReplacementRouteID = [string](Get-PropertyValue -Item $remediationCommand -Name "replacement_route_id" -Default "") $commandExpiresAt = [string](Get-PropertyValue -Item $remediationCommand -Name "expires_at" -Default "") $expectedReplacementRouteID = $commandReplacementRouteID $syntheticConfig = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/nodes/$($entryNode.id)/mesh/synthetic-config").synthetic_mesh_config $syntheticCommands = @() if ($syntheticConfig.PSObject.Properties.Name -contains "service_channel_remediation_commands") { $syntheticCommands = @($syntheticConfig.service_channel_remediation_commands) } $syntheticCommand = $syntheticCommands | Where-Object { [string]$_.channel_id -eq [string]$lease.channel_id } | Select-Object -First 1 $syntheticCommandAction = [string](Get-PropertyValue -Item $syntheticCommand -Name "action" -Default "") $syntheticCommandReplacementRouteID = [string](Get-PropertyValue -Item $syntheticCommand -Name "replacement_route_id" -Default "") if ([string]::IsNullOrWhiteSpace($expectedReplacementRouteID)) { $expectedReplacementRouteID = $syntheticCommandReplacementRouteID } if ([string]::IsNullOrWhiteSpace($expectedReplacementRouteID)) { $expectedReplacementRouteID = $alternateRouteID } $routeManagerDecision = $null $routeManagerRuntime = $null for ($i = 0; $i -lt 10; $i++) { Start-Sleep -Seconds 5 $heartbeats = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/nodes/$($entryNode.id)/heartbeats?actor_user_id=$ActorUserID&limit=5").heartbeats foreach ($heartbeat in @($heartbeats)) { $runtimeReport = Get-PropertyValue -Item (Get-PropertyValue -Item $heartbeat -Name "metadata" -Default $null) -Name "fabric_service_channel_runtime_report" -Default $null $ingress = Get-PropertyValue -Item $runtimeReport -Name "ingress" -Default $null $routeManager = Get-PropertyValue -Item $ingress -Name "route_manager" -Default $null $routeManagerRuntime = $routeManager $decisions = @() if ($null -ne $routeManager -and $routeManager.PSObject.Properties.Name -contains "decisions") { $decisions = @($routeManager.decisions) } $routeManagerDecision = $decisions | Where-Object { [string]$_.route_id -eq $primaryRouteID -and [string]$_.replacement_route_id -eq $expectedReplacementRouteID -and [string]$_.decision_source -eq "service_channel_remediation_command" } | Select-Object -First 1 if ($null -ne $routeManagerDecision) { break } } if ($null -ne $routeManagerDecision) { break } } $baselineFallbackLocal = 0 $baselineRouteFailures = 0 $baselineFlowDropped = 0 $baselineSchedulerDropped = 0 $baselineHeartbeats = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/nodes/$($entryNode.id)/heartbeats?actor_user_id=$ActorUserID&limit=1").heartbeats if (@($baselineHeartbeats).Count -gt 0) { $baselineRuntimeReport = Get-PropertyValue -Item (Get-PropertyValue -Item $baselineHeartbeats[0] -Name "metadata" -Default $null) -Name "fabric_service_channel_runtime_report" -Default $null $baselineIngress = Get-PropertyValue -Item $baselineRuntimeReport -Name "ingress" -Default $null $baselineFlowScheduler = Get-PropertyValue -Item $baselineIngress -Name "flow_scheduler" -Default $null $baselineFallbackLocal = [int](Get-PropertyValue -Item $baselineIngress -Name "send_fallback_local" -Default 0) $baselineRouteFailures = [int](Get-PropertyValue -Item $baselineIngress -Name "send_route_failures" -Default 0) $baselineFlowDropped = [int](Get-PropertyValue -Item $baselineIngress -Name "send_flow_dropped" -Default 0) $baselineSchedulerDropped = [int](Get-PropertyValue -Item $baselineFlowScheduler -Name "dropped" -Default 0) } $multiFlowPackets = @() foreach ($offset in 0..($expectedPressurePacketCount - 1)) { $port = 52000 + $offset $multiFlowPackets += ,(New-IPv4TcpPacket -SourcePort $port -DestinationPort 3389) } $multiFlowPayload = ConvertTo-VPNPacketBatch -Packets $multiFlowPackets $multiFlowPayloadPath = Join-Path ([System.IO.Path]::GetTempPath()) "$runId-vpn-packet-batch.bin" [System.IO.File]::WriteAllBytes($multiFlowPayloadPath, [byte[]]$multiFlowPayload) $bulkHeaders = @{} + $headers $bulkHeaders["X-RAP-Traffic-Class"] = "bulk" $bulkJobs = @() foreach ($jobIndex in 1..$expectedBulkRequestCount) { $jobPayloadPath = Join-Path ([System.IO.Path]::GetTempPath()) "$runId-vpn-packet-batch-$jobIndex.bin" Copy-Item -Path $multiFlowPayloadPath -Destination $jobPayloadPath -Force $bulkJobs += Start-Job -ArgumentList ($packetUrl + "?batch=true"), $bulkHeaders, $jobPayloadPath -ScriptBlock { param($uri, $headers, $path) try { $started = Get-Date $response = Invoke-WebRequest -Method Post -Uri $uri -Headers $headers -InFile $path -ContentType "application/vnd.rap.vpn-packet-batch.v1" -TimeoutSec 30 [pscustomobject]@{ status_code = [int]$response.StatusCode accepted_by = [string]$response.Headers["X-RAP-Service-Channel-Accepted-By"] duration_ms = [int]((Get-Date) - $started).TotalMilliseconds error = "" } } finally { Remove-Item -Path $path -Force -ErrorAction SilentlyContinue } } } Start-Sleep -Milliseconds 100 $interactivePacket = New-IPv4TcpPacket -SourcePort 53000 -DestinationPort 22 $interactivePayload = ConvertTo-VPNPacketBatch -Packets @($interactivePacket) $interactivePayloadPath = Join-Path ([System.IO.Path]::GetTempPath()) "$runId-vpn-interactive-packet-batch.bin" [System.IO.File]::WriteAllBytes($interactivePayloadPath, [byte[]]$interactivePayload) $interactiveHeaders = @{} + $headers $interactiveHeaders["X-RAP-Traffic-Class"] = "interactive" $interactiveStarted = Get-Date $interactiveResponse = Invoke-WebRequest -Method Post -Uri ($packetUrl + "?batch=true") -Headers $interactiveHeaders -InFile $interactivePayloadPath -ContentType "application/vnd.rap.vpn-packet-batch.v1" -TimeoutSec 30 $interactiveDurationMs = [int]((Get-Date) - $interactiveStarted).TotalMilliseconds $interactiveAcceptedBy = [string]$interactiveResponse.Headers["X-RAP-Service-Channel-Accepted-By"] $bulkResponses = @($bulkJobs | Wait-Job -Timeout 60 | Receive-Job) $bulkJobs | Remove-Job -Force -ErrorAction SilentlyContinue $bulkAcceptedCount = @($bulkResponses | Where-Object { [int]$_.status_code -eq 202 -and [string]$_.accepted_by -eq "introspection" }).Count $postRemediationAcceptedBy = (@($bulkResponses | Select-Object -First 1).accepted_by | Out-String).Trim() $replacementTrafficObserved = $false $replacementLastSelected = "" $replacementFlowStat = $null $replacementFlowStats = @() $interactiveFlowStats = @() $bulkFlowStats = @() $postRemediationFallbackLocal = 0 $postRemediationRouteFailures = 0 $postRemediationFlowDropped = 0 $postRemediationSchedulerDropped = 0 $postRemediationHighWatermark = 0 $postRemediationMaxInFlight = 0 for ($i = 0; $i -lt 12; $i++) { Start-Sleep -Seconds 5 $heartbeats = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/nodes/$($entryNode.id)/heartbeats?actor_user_id=$ActorUserID&limit=5").heartbeats foreach ($heartbeat in @($heartbeats)) { $runtimeReport = Get-PropertyValue -Item (Get-PropertyValue -Item $heartbeat -Name "metadata" -Default $null) -Name "fabric_service_channel_runtime_report" -Default $null $ingress = Get-PropertyValue -Item $runtimeReport -Name "ingress" -Default $null $replacementLastSelected = [string](Get-PropertyValue -Item $ingress -Name "last_selected_route_id" -Default "") $postRemediationFallbackLocal = [int](Get-PropertyValue -Item $ingress -Name "send_fallback_local" -Default 0) $postRemediationRouteFailures = [int](Get-PropertyValue -Item $ingress -Name "send_route_failures" -Default 0) $postRemediationFlowDropped = [int](Get-PropertyValue -Item $ingress -Name "send_flow_dropped" -Default 0) $flowScheduler = Get-PropertyValue -Item $ingress -Name "flow_scheduler" -Default $null $postRemediationSchedulerDropped = [int](Get-PropertyValue -Item $flowScheduler -Name "dropped" -Default 0) $postRemediationHighWatermark = [int](Get-PropertyValue -Item $flowScheduler -Name "high_watermark" -Default 0) $postRemediationMaxInFlight = [int](Get-PropertyValue -Item $flowScheduler -Name "max_in_flight" -Default 0) $channelStats = Get-PropertyValue -Item $flowScheduler -Name "channel_stats" -Default $null if ($null -ne $channelStats) { $replacementFlowStats = @() $interactiveFlowStats = @() $bulkFlowStats = @() foreach ($property in @($channelStats.PSObject.Properties)) { $stat = $property.Value if ([string](Get-PropertyValue -Item $stat -Name "last_route_id" -Default "") -eq $expectedReplacementRouteID) { $replacementFlowStats += $stat $replacementFlowStat = $stat $trafficClass = [string](Get-PropertyValue -Item $stat -Name "traffic_class" -Default "") if ($trafficClass -eq "interactive") { $interactiveFlowStats += $stat } if ($trafficClass -eq "bulk") { $bulkFlowStats += $stat } } } } if ($replacementLastSelected -eq $expectedReplacementRouteID -and $bulkFlowStats.Count -ge $expectedPressureFlowCount -and $interactiveFlowStats.Count -ge 1) { $replacementTrafficObserved = $true break } } if ($replacementTrafficObserved) { break } } $result = [ordered]@{ schema_version = "c18z67.service_channel_concurrent_qos_live_smoke.v1" run_id = $runId cluster_id = $ClusterID primary_route_id = $primaryRouteID alternate_route_id = $alternateRouteID expected_replacement_route_id = $expectedReplacementRouteID channel_id = [string]$lease.channel_id passed = [bool]( $backendLine.Contains($ExpectedBackendImage) -and $nodeLines.Contains($ExpectedNodeAgentImage) -and [string]$lease.status -eq "ready" -and $leasePrimaryRouteID -eq $primaryRouteID -and $leaseHasAlternate -and [int]$response.StatusCode -eq 202 -and $acceptedBy -eq "introspection" -and $null -ne $matchingChannel -and [string](Get-PropertyValue -Item $matchingChannel -Name "primary_route_id" -Default "") -eq $primaryRouteID -and -not [bool](Get-PropertyValue -Item $matchingChannel -Name "force_backend_fallback" -Default $false) -and [string](Get-PropertyValue -Item $matchingChannel -Name "route_feedback_status" -Default "") -eq "fenced" -and [string](Get-PropertyValue -Item $matchingChannel -Name "remediation_action" -Default "") -eq "prefer_alternate_route" -and [string](Get-PropertyValue -Item $matchingChannel -Name "remediation_route_id" -Default "") -eq $expectedReplacementRouteID -and $null -ne $remediationCommand -and $commandAction -eq "prefer_alternate_route" -and $commandPrimaryRouteID -eq $primaryRouteID -and $commandReplacementRouteID -eq $expectedReplacementRouteID -and $commandExpiresAt.Length -gt 0 -and $null -ne $syntheticCommand -and $syntheticCommandAction -eq "prefer_alternate_route" -and $syntheticCommandReplacementRouteID -eq $expectedReplacementRouteID -and $null -ne $routeManagerDecision -and [string](Get-PropertyValue -Item $routeManagerDecision -Name "rebuild_status" -Default "") -eq "applied" -and $bulkResponses.Count -eq $expectedBulkRequestCount -and $bulkAcceptedCount -eq $expectedBulkRequestCount -and [int]$interactiveResponse.StatusCode -eq 202 -and $interactiveAcceptedBy -eq "introspection" -and $interactiveDurationMs -lt 2000 -and $replacementTrafficObserved -and $replacementLastSelected -eq $expectedReplacementRouteID -and $bulkFlowStats.Count -ge $expectedPressureFlowCount -and $interactiveFlowStats.Count -ge 1 -and $multiFlowPackets.Count -eq $expectedPressurePacketCount -and ($postRemediationFallbackLocal - $baselineFallbackLocal) -eq 0 -and ($postRemediationFlowDropped - $baselineFlowDropped) -eq 0 -and ($postRemediationSchedulerDropped - $baselineSchedulerDropped) -eq 0 -and ($postRemediationRouteFailures - $baselineRouteFailures) -eq 0 -and $null -ne $matchingChannel -and -not [bool](Get-PropertyValue -Item $matchingChannel -Name "force_backend_fallback" -Default $false) -and [string](Get-PropertyValue -Item $matchingChannel -Name "remediation_action" -Default "") -ne "use_backend_fallback" ) checks = [ordered]@{ backend_expected_image_deployed = $backendLine.Contains($ExpectedBackendImage) node_agent_expected_image_deployed = $nodeLines.Contains($ExpectedNodeAgentImage) lease_ready = ([string]$lease.status -eq "ready") lease_selected_primary_route = ($leasePrimaryRouteID -eq $primaryRouteID) lease_contains_alternate_route = $leaseHasAlternate packet_accepted = ([int]$response.StatusCode -eq 202) accepted_by_header_is_introspection = ($acceptedBy -eq "introspection") active_channel_visible = ($null -ne $matchingChannel) active_channel_not_backend_fallback = ($null -ne $matchingChannel -and -not [bool](Get-PropertyValue -Item $matchingChannel -Name "force_backend_fallback" -Default $false)) route_quality_fenced = ($null -ne $matchingChannel -and [string](Get-PropertyValue -Item $matchingChannel -Name "route_feedback_status" -Default "") -eq "fenced") remediation_prefers_alternate = ($null -ne $matchingChannel -and [string](Get-PropertyValue -Item $matchingChannel -Name "remediation_action" -Default "") -eq "prefer_alternate_route") remediation_route_is_alternate = ($null -ne $matchingChannel -and [string](Get-PropertyValue -Item $matchingChannel -Name "remediation_route_id" -Default "") -eq $expectedReplacementRouteID) remediation_command_visible = ($null -ne $remediationCommand) remediation_command_prefers_alternate = ($commandAction -eq "prefer_alternate_route") remediation_command_primary_route_matches = ($commandPrimaryRouteID -eq $primaryRouteID) remediation_command_replacement_route_matches = ($commandReplacementRouteID -eq $expectedReplacementRouteID) remediation_command_has_ttl = ($commandExpiresAt.Length -gt 0) synthetic_config_command_visible = ($null -ne $syntheticCommand) synthetic_config_command_prefers_alternate = ($syntheticCommandAction -eq "prefer_alternate_route") synthetic_config_command_replacement_route_matches = ($syntheticCommandReplacementRouteID -eq $expectedReplacementRouteID) node_route_manager_consumed_command = ($null -ne $routeManagerDecision) node_route_manager_applied_replacement = ($null -ne $routeManagerDecision -and [string](Get-PropertyValue -Item $routeManagerDecision -Name "rebuild_status" -Default "") -eq "applied") bulk_request_count_matches = ($bulkResponses.Count -eq $expectedBulkRequestCount) bulk_requests_accepted_by_introspection = ($bulkAcceptedCount -eq $expectedBulkRequestCount) interactive_packet_accepted = ([int]$interactiveResponse.StatusCode -eq 202) interactive_accepted_by_introspection = ($interactiveAcceptedBy -eq "introspection") interactive_completed_under_bulk_pressure = ($interactiveDurationMs -lt 2000) replacement_traffic_observed = $replacementTrafficObserved replacement_last_selected_route_matches = ($replacementLastSelected -eq $expectedReplacementRouteID) replacement_flow_stat_observed = ($null -ne $replacementFlowStat) replacement_pressure_flow_stats_observed = ($bulkFlowStats.Count -ge $expectedPressureFlowCount) replacement_interactive_flow_stat_observed = ($interactiveFlowStats.Count -ge 1) pressure_packet_count_matches = ($multiFlowPackets.Count -eq $expectedPressurePacketCount) no_local_gateway_fallback_after_replacement = (($postRemediationFallbackLocal - $baselineFallbackLocal) -eq 0) no_flow_drops_after_replacement = (($postRemediationFlowDropped - $baselineFlowDropped) -eq 0 -and ($postRemediationSchedulerDropped - $baselineSchedulerDropped) -eq 0) no_route_failures_after_replacement = (($postRemediationRouteFailures - $baselineRouteFailures) -eq 0) backend_fallback_not_recommended = ($null -ne $matchingChannel -and -not [bool](Get-PropertyValue -Item $matchingChannel -Name "force_backend_fallback" -Default $false) -and [string](Get-PropertyValue -Item $matchingChannel -Name "remediation_action" -Default "") -ne "use_backend_fallback") } summary = [ordered]@{ backend_container = $backendLine.Trim() node_containers = $nodeLines.Trim() accepted_by = $acceptedBy lease_status = [string]$lease.status lease_primary_route_id = $leasePrimaryRouteID lease_alternate_route_count = $leaseAlternates.Count access_status = [string]$accessTelemetry.status active_channel_count = [int]$accessTelemetry.active_channel_count correlated_route_count = [int]$accessTelemetry.correlated_route_count degraded_route_count = [int]$accessTelemetry.degraded_route_count degraded_fallback_channel_count = [int]$accessTelemetry.degraded_fallback_channel_count synthetic_command = $syntheticCommand route_manager_decision = $routeManagerDecision route_manager_runtime = $routeManagerRuntime post_remediation_accepted_by = $postRemediationAcceptedBy bulk_request_count = $bulkResponses.Count bulk_accepted_count = $bulkAcceptedCount bulk_responses = $bulkResponses post_remediation_packet_count = ($multiFlowPackets.Count * $bulkResponses.Count) interactive_accepted_by = $interactiveAcceptedBy interactive_duration_ms = $interactiveDurationMs interactive_packet_count = $expectedInteractivePacketCount expected_pressure_flow_count = $expectedPressureFlowCount replacement_last_selected_route_id = $replacementLastSelected replacement_flow_stat_count = $replacementFlowStats.Count replacement_bulk_flow_stat_count = $bulkFlowStats.Count replacement_interactive_flow_stat_count = $interactiveFlowStats.Count replacement_flow_stat = $replacementFlowStat replacement_flow_stats = $replacementFlowStats replacement_bulk_flow_stats = $bulkFlowStats replacement_interactive_flow_stats = $interactiveFlowStats baseline_send_fallback_local = $baselineFallbackLocal baseline_send_route_failures = $baselineRouteFailures baseline_send_flow_dropped = $baselineFlowDropped baseline_scheduler_dropped = $baselineSchedulerDropped fallback_local_delta = ($postRemediationFallbackLocal - $baselineFallbackLocal) route_failure_delta = ($postRemediationRouteFailures - $baselineRouteFailures) flow_drop_delta = ($postRemediationFlowDropped - $baselineFlowDropped) scheduler_drop_delta = ($postRemediationSchedulerDropped - $baselineSchedulerDropped) post_remediation_send_fallback_local = $postRemediationFallbackLocal post_remediation_send_route_failures = $postRemediationRouteFailures post_remediation_send_flow_dropped = $postRemediationFlowDropped post_remediation_scheduler_dropped = $postRemediationSchedulerDropped post_remediation_scheduler_high_watermark = $postRemediationHighWatermark post_remediation_scheduler_max_in_flight = $postRemediationMaxInFlight matching_channel = $matchingChannel } } $failedChecks = @($result.checks.GetEnumerator() | Where-Object { $_.Value -ne $true }) $result.summary.failed_checks = @($failedChecks | ForEach-Object { $_.Key }) } finally { if ($primaryRouteID) { try { Invoke-Api -Method POST -Path "/clusters/$ClusterID/mesh/route-intents/$primaryRouteID/expire" -Body @{ actor_user_id = $ActorUserID } | Out-Null } catch {} } if ($alternateRouteID) { try { Invoke-Api -Method POST -Path "/clusters/$ClusterID/mesh/route-intents/$alternateRouteID/expire" -Body @{ actor_user_id = $ActorUserID } | Out-Null } catch {} } } $resultFullPath = Join-Path $repoRoot $ResultPath $resultDir = Split-Path -Parent $resultFullPath if (-not (Test-Path $resultDir)) { New-Item -ItemType Directory -Path $resultDir | Out-Null } $result | ConvertTo-Json -Depth 100 | Set-Content -Path $resultFullPath -Encoding UTF8 if ($failedChecks.Count -gt 0) { throw "C18Z67 failed checks: $(@($failedChecks | ForEach-Object { $_.Key }) -join ', '). Result: $resultFullPath" } Write-Host "C18Z67 service-channel concurrent qos live smoke passed. Result: $resultFullPath" $result