param( [string]$ApiBaseUrl = "http://192.168.200.61:18121/api/v1", [string]$ClusterID = "cfc0743d-d960-49fb-9de8-96e063d5e4aa", [string]$ActorUserID = "f67d943f-5397-4b3a-a229-695fe67ad700", [string]$EntryNodeName = "test-1", [string]$RelayNodeName = "test-3", [string]$ExitNodeName = "test-2", [string]$EntryBaseUrl = "http://192.168.200.61:19131", [string]$DockerSSH = "test-docker", [string]$ExpectedBackendImage = "rap-backend:fabric-service-channel-0.2.281-c18z109", [string]$ExpectedNodeAgentImage = "rap-node-agent:0.2.270-c18z95", [string]$ResultPath = "artifacts\c18z80-service-channel-post-rebuild-pressure-smoke-result.json" ) Set-StrictMode -Version Latest $ErrorActionPreference = "Stop" $scriptDir = Split-Path -Parent $MyInvocation.MyCommand.Path $repoRoot = (Resolve-Path (Join-Path $scriptDir "..\..")).ProviderPath $runId = "c18z80-" + (Get-Date -Format "yyyyMMdd-HHmmss") function Invoke-Api { param([string]$Method, [string]$Path, [object]$Body = $null) $params = @{ Method = $Method; Uri = "$ApiBaseUrl$Path"; TimeoutSec = 30 } if ($null -ne $Body) { $params.ContentType = "application/json" $params.Body = ($Body | ConvertTo-Json -Depth 80) } return Invoke-RestMethod @params } function Get-PropertyValue { param([object]$Item, [string]$Name, [object]$Default = $null) if ($null -eq $Item) { return $Default } $property = $Item.PSObject.Properties[$Name] if ($null -eq $property) { return $Default } return $property.Value } function Get-NodeByName { param([string]$Name) $nodes = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/nodes?actor_user_id=$ActorUserID").nodes $node = @($nodes | Where-Object { $_.name -eq $Name }) | Select-Object -First 1 if ($null -eq $node) { throw "Node '$Name' was not found" } return $node } function New-IPv4TcpPacket { param( [byte[]]$Source = @(10, 77, 0, 2), [byte[]]$Destination = @(192, 168, 200, 95), [int]$SourcePort, [int]$DestinationPort = 3389 ) $packet = [byte[]]::new(40) $packet[0] = 0x45 $packet[2] = 0 $packet[3] = 40 $packet[8] = 64 $packet[9] = 6 [Array]::Copy($Source, 0, $packet, 12, 4) [Array]::Copy($Destination, 0, $packet, 16, 4) $packet[20] = [byte](($SourcePort -shr 8) -band 0xff) $packet[21] = [byte]($SourcePort -band 0xff) $packet[22] = [byte](($DestinationPort -shr 8) -band 0xff) $packet[23] = [byte]($DestinationPort -band 0xff) $packet[32] = 0x50 return $packet } function ConvertTo-VPNPacketBatch { param([byte[][]]$Packets) $buffer = New-Object System.Collections.Generic.List[byte] foreach ($packet in $Packets) { if ($null -eq $packet -or $packet.Length -eq 0) { continue } $size = [uint32]$packet.Length $buffer.Add([byte](($size -shr 24) -band 0xff)) $buffer.Add([byte](($size -shr 16) -band 0xff)) $buffer.Add([byte](($size -shr 8) -band 0xff)) $buffer.Add([byte]($size -band 0xff)) foreach ($value in $packet) { $buffer.Add($value) } } return $buffer.ToArray() } function New-RouteIntent { param([string]$SourceNodeID, [string]$DestinationNodeID, [string[]]$Hops) $expiresAt = (Get-Date).ToUniversalTime().AddMinutes(5).ToString("o") return (Invoke-Api -Method POST -Path "/clusters/$ClusterID/mesh/route-intents" -Body @{ actor_user_id = $ActorUserID source_selector = @{ node_id = $SourceNodeID } destination_selector = @{ node_id = $DestinationNodeID } service_class = "vpn_packets" priority = 2100000000 policy = @{ synthetic_enabled = $true route_version = "$runId-primary" policy_version = "$runId-primary" peer_directory_version = "$runId-primary" hops = @($Hops) allowed_channels = @("vpn_packet", "fabric_control") max_ttl = 8 max_hops = 8 expires_at = $expiresAt metadata = @{ smoke = "c18z80_service_channel_post_rebuild_pressure"; run_id = $runId } } }).route_intent } function Disable-ExistingRouteIntents { param([string]$SourceNodeID, [string]$DestinationNodeID) $items = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/mesh/route-intents?actor_user_id=$ActorUserID").route_intents foreach ($item in @($items)) { if ([string](Get-PropertyValue -Item $item -Name "status" -Default "") -ne "active") { continue } if ([string](Get-PropertyValue -Item $item -Name "service_class" -Default "") -ne "vpn_packets") { continue } $sourceSelector = Get-PropertyValue -Item $item -Name "source_selector" -Default $null $destinationSelector = Get-PropertyValue -Item $item -Name "destination_selector" -Default $null if ([string](Get-PropertyValue -Item $sourceSelector -Name "node_id" -Default "") -ne $SourceNodeID) { continue } if ([string](Get-PropertyValue -Item $destinationSelector -Name "node_id" -Default "") -ne $DestinationNodeID) { continue } [void](Invoke-Api -Method POST -Path "/clusters/$ClusterID/mesh/route-intents/$($item.id)/disable" -Body @{ actor_user_id = $ActorUserID reason = "c18z80 isolate post-rebuild-pressure smoke route pair" }) } } function Send-DegradedHeartbeat { param([string]$EntryNodeID, [string]$PrimaryRouteID) $observedAt = (Get-Date).ToUniversalTime().ToString("o") return Invoke-Api -Method POST -Path "/clusters/$ClusterID/nodes/$EntryNodeID/heartbeats" -Body @{ health_status = "healthy" reported_version = "0.2.254-c18z80" capabilities = @{ fabric_service_channel_runtime = $true fabric_service_channel_route_quality_feedback = $true smoke_feedback_injection = "c18z77" } service_states = @{ smoke = "c18z80_primary_degraded_alternate_after_lease" } metadata = @{ fabric_service_channel_runtime_report = @{ schema_version = "c18l.fabric_service_channel_runtime_report.v1" config_version = "$runId-primary" cluster_id = $ClusterID local_node_id = $EntryNodeID observed_at = $observedAt ingress = @{ flow_scheduler = @{ schema_version = "rap.fabric_flow_scheduler.v1" service_neutral = $true service_mode = "application_protocol_agnostic" channel_stats = @{ "c18z80-primary-degraded" = @{ last_route_id = $PrimaryRouteID last_failed_route_id = $PrimaryRouteID route_generation = "$runId-primary" last_error = "c18z80 primary route degraded; alternate added after lease" last_send_duration_ms = 1200 consecutive_failures = 3 stall_count = 2 route_rebuild_recommended = $true degraded_fallback_recommended = $false quality_window_sample_count = 8 quality_window_success_count = 2 quality_window_failure_count = 3 quality_window_slow_count = 2 quality_window_drop_count = 0 quality_window_avg_latency_ms = 1200 quality_window_last_updated_at = $observedAt } } } } } smoke = @{ name = "c18z80_service_channel_post_rebuild_pressure"; run_id = $runId } } } } function Get-LatestIngressSnapshot { param([string]$EntryNodeID, [string]$ReplacementRouteID) $heartbeats = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/nodes/$EntryNodeID/heartbeats?actor_user_id=$ActorUserID&limit=5").heartbeats foreach ($heartbeat in @($heartbeats)) { $runtimeReport = Get-PropertyValue -Item (Get-PropertyValue -Item $heartbeat -Name "metadata" -Default $null) -Name "fabric_service_channel_runtime_report" -Default $null $ingress = Get-PropertyValue -Item $runtimeReport -Name "ingress" -Default $null if ($null -eq $ingress) { continue } $flowScheduler = Get-PropertyValue -Item $ingress -Name "flow_scheduler" -Default $null $channelStats = Get-PropertyValue -Item $flowScheduler -Name "channel_stats" -Default $null $replacementFlowStats = @() if ($null -ne $channelStats) { foreach ($property in @($channelStats.PSObject.Properties)) { $stat = $property.Value if ([string](Get-PropertyValue -Item $stat -Name "last_route_id" -Default "") -eq $ReplacementRouteID) { $replacementFlowStats += $stat } } } return [pscustomobject]@{ observed_at = [string](Get-PropertyValue -Item $runtimeReport -Name "observed_at" -Default "") last_selected_route_id = [string](Get-PropertyValue -Item $ingress -Name "last_selected_route_id" -Default "") send_fallback_local = [int](Get-PropertyValue -Item $ingress -Name "send_fallback_local" -Default 0) send_route_failures = [int](Get-PropertyValue -Item $ingress -Name "send_route_failures" -Default 0) send_flow_dropped = [int](Get-PropertyValue -Item $ingress -Name "send_flow_dropped" -Default 0) scheduler_dropped = [int](Get-PropertyValue -Item $flowScheduler -Name "dropped" -Default 0) flow_high_watermark = [int](Get-PropertyValue -Item $flowScheduler -Name "high_watermark" -Default 0) flow_max_in_flight = [int](Get-PropertyValue -Item $flowScheduler -Name "max_in_flight" -Default 0) replacement_flow_stat_count = $replacementFlowStats.Count replacement_flow_stats = $replacementFlowStats } } return $null } $entryNode = Get-NodeByName -Name $EntryNodeName $relayNode = Get-NodeByName -Name $RelayNodeName $exitNode = Get-NodeByName -Name $ExitNodeName [void](Disable-ExistingRouteIntents -SourceNodeID $entryNode.id -DestinationNodeID $exitNode.id) $route = New-RouteIntent -SourceNodeID $entryNode.id -DestinationNodeID $exitNode.id -Hops @($entryNode.id, $exitNode.id) $lease = (Invoke-Api -Method POST -Path "/clusters/$ClusterID/fabric/service-channels/leases" -Body @{ actor_user_id = $ActorUserID organization_id = "smoke-org" user_id = "smoke-user" resource_id = "c18z80-vpn-smoke" service_class = "vpn_packets" entry_node_ids = @([string]$entryNode.id) exit_node_ids = @([string]$exitNode.id) preferred_entry_node_id = [string]$entryNode.id preferred_exit_node_id = [string]$exitNode.id allowed_channels = @("vpn_packet", "fabric_control") ttl_seconds = 90 metadata = @{ smoke = "c18z80_service_channel_post_rebuild_pressure"; run_id = $runId } }).fabric_service_channel_lease $primaryRouteID = [string](Get-PropertyValue -Item (Get-PropertyValue -Item $lease -Name "primary_route" -Default $null) -Name "route_id" -Default $route.id) [void](Send-DegradedHeartbeat -EntryNodeID $entryNode.id -PrimaryRouteID $primaryRouteID) $matchingChannel = $null for ($i = 0; $i -lt 10; $i++) { Start-Sleep -Seconds 3 $accessTelemetry = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/fabric/service-channels/access-telemetry?actor_user_id=$ActorUserID&limit=50").fabric_service_channel_access_telemetry $matchingChannel = @($accessTelemetry.active_channels | Where-Object { $_.channel_id -eq $lease.channel_id }) | Select-Object -First 1 if ($null -ne $matchingChannel -and [string](Get-PropertyValue -Item $matchingChannel -Name "remediation_action" -Default "") -eq "rebuild_route") { break } } $command = Get-PropertyValue -Item $matchingChannel -Name "remediation_command" -Default $null $commandID = [string](Get-PropertyValue -Item $command -Name "command_id" -Default "") $alternate = New-RouteIntent -SourceNodeID $entryNode.id -DestinationNodeID $exitNode.id -Hops @($entryNode.id, $relayNode.id, $exitNode.id) $alternateRouteID = [string]$alternate.id [void](Invoke-Api -Method GET -Path "/clusters/$ClusterID/nodes/$($entryNode.id)/mesh/synthetic-config") $attempts = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/fabric/service-channels/rebuild-attempts?actor_user_id=$ActorUserID&reporter_node_id=$($entryNode.id)&rebuild_request_id=$commandID&limit=10").rebuild_attempts $attempt = @($attempts | Where-Object { $_.rebuild_request_id -eq $commandID }) | Select-Object -First 1 $routeManagerDecision = $null $routeManagerTransition = $null for ($i = 0; $i -lt 12; $i++) { Start-Sleep -Seconds 5 $heartbeats = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/nodes/$($entryNode.id)/heartbeats?actor_user_id=$ActorUserID&limit=5").heartbeats foreach ($heartbeat in @($heartbeats)) { $runtimeReport = Get-PropertyValue -Item (Get-PropertyValue -Item $heartbeat -Name "metadata" -Default $null) -Name "fabric_service_channel_runtime_report" -Default $null $ingress = Get-PropertyValue -Item $runtimeReport -Name "ingress" -Default $null $routeManager = Get-PropertyValue -Item $ingress -Name "route_manager" -Default $null $routeManagerTransition = Get-PropertyValue -Item $ingress -Name "route_manager_transition" -Default $null $decisions = @() if ($null -ne $routeManager -and $routeManager.PSObject.Properties.Name -contains "decisions") { $decisions = @($routeManager.decisions) } $routeManagerDecision = $decisions | Where-Object { [string](Get-PropertyValue -Item $_ -Name "rebuild_request_id" -Default "") -eq $commandID -and [string](Get-PropertyValue -Item $_ -Name "route_id" -Default "") -eq $primaryRouteID -and [string](Get-PropertyValue -Item $_ -Name "replacement_route_id" -Default "") -eq $alternateRouteID } | Select-Object -First 1 if ($null -ne $routeManagerDecision) { break } } if ($null -ne $routeManagerDecision) { break } } $packetPath = $lease.entry_http.path_template. Replace("{cluster_id}", $ClusterID). Replace("{channel_id}", [string]$lease.channel_id). Replace("{resource_id}", "c18z80-vpn-smoke") $packetUrl = $EntryBaseUrl.TrimEnd("/") + $packetPath $baseHeaders = @{ "X-RAP-Service-Channel-Token" = [string]$lease.token.token "X-RAP-Fabric-Channel-ID" = [string]$lease.channel_id "X-RAP-Service-Class" = "vpn_packets" "X-RAP-Channel-Class" = "vpn_packet" } $baselineSnapshot = $null for ($i = 0; $i -lt 10; $i++) { Start-Sleep -Seconds 2 $baselineSnapshot = Get-LatestIngressSnapshot -EntryNodeID $entryNode.id -ReplacementRouteID $alternateRouteID if ($null -ne $baselineSnapshot) { break } } $burstCount = 5 $burstPacketCount = 32 $trafficClasses = @("interactive", "bulk", "reliable", "interactive", "bulk") $burstResults = @() for ($burstIndex = 0; $burstIndex -lt $burstCount; $burstIndex++) { $trafficPackets = @() foreach ($offset in 0..($burstPacketCount - 1)) { $trafficPackets += ,(New-IPv4TcpPacket -SourcePort (54000 + ($burstIndex * 100) + $offset) -DestinationPort 3389) } $trafficPayload = ConvertTo-VPNPacketBatch -Packets $trafficPackets $trafficPayloadPath = Join-Path ([System.IO.Path]::GetTempPath()) "$runId-vpn-packet-batch-$burstIndex.bin" [System.IO.File]::WriteAllBytes($trafficPayloadPath, [byte[]]$trafficPayload) $headers = @{} + $baseHeaders $headers["X-RAP-Traffic-Class"] = $trafficClasses[$burstIndex] $trafficStarted = Get-Date $trafficResponse = Invoke-WebRequest -Method Post -Uri ($packetUrl + "?batch=true") -Headers $headers -InFile $trafficPayloadPath -ContentType "application/vnd.rap.vpn-packet-batch.v1" -TimeoutSec 30 $trafficDurationMs = [int]((Get-Date) - $trafficStarted).TotalMilliseconds $trafficAcceptedBy = [string]$trafficResponse.Headers["X-RAP-Service-Channel-Accepted-By"] Remove-Item -Path $trafficPayloadPath -Force -ErrorAction SilentlyContinue $snapshot = $null for ($i = 0; $i -lt 8; $i++) { Start-Sleep -Seconds 3 $snapshot = Get-LatestIngressSnapshot -EntryNodeID $entryNode.id -ReplacementRouteID $alternateRouteID if ($null -ne $snapshot -and $snapshot.last_selected_route_id -eq $alternateRouteID -and $snapshot.replacement_flow_stat_count -ge 1) { break } } $burstResults += [pscustomobject]@{ burst_index = $burstIndex + 1 traffic_class = $trafficClasses[$burstIndex] packet_count = $trafficPackets.Count status_code = [int]$trafficResponse.StatusCode accepted_by = $trafficAcceptedBy duration_ms = $trafficDurationMs snapshot = $snapshot } } $finalSnapshot = Get-LatestIngressSnapshot -EntryNodeID $entryNode.id -ReplacementRouteID $alternateRouteID $acceptedBurstCount = @($burstResults | Where-Object { $_.status_code -eq 202 -and ($_.accepted_by -eq "introspection" -or $_.accepted_by -eq "signed") }).Count $replacementBurstCount = @($burstResults | Where-Object { $null -ne $_.snapshot -and $_.snapshot.last_selected_route_id -eq $alternateRouteID -and $_.snapshot.replacement_flow_stat_count -ge 1 }).Count $stalePrimaryReselections = @($burstResults | Where-Object { $null -ne $_.snapshot -and $_.snapshot.last_selected_route_id -eq $primaryRouteID }).Count $baselineFallbackLocal = if ($null -ne $baselineSnapshot) { [int]$baselineSnapshot.send_fallback_local } else { 0 } $baselineRouteFailures = if ($null -ne $baselineSnapshot) { [int]$baselineSnapshot.send_route_failures } else { 0 } $baselineFlowDropped = if ($null -ne $baselineSnapshot) { [int]$baselineSnapshot.send_flow_dropped } else { 0 } $baselineSchedulerDropped = if ($null -ne $baselineSnapshot) { [int]$baselineSnapshot.scheduler_dropped } else { 0 } $finalFallbackLocal = if ($null -ne $finalSnapshot) { [int]$finalSnapshot.send_fallback_local } else { 0 } $finalRouteFailures = if ($null -ne $finalSnapshot) { [int]$finalSnapshot.send_route_failures } else { 0 } $finalFlowDropped = if ($null -ne $finalSnapshot) { [int]$finalSnapshot.send_flow_dropped } else { 0 } $finalSchedulerDropped = if ($null -ne $finalSnapshot) { [int]$finalSnapshot.scheduler_dropped } else { 0 } $postAccess = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/fabric/service-channels/access-telemetry?actor_user_id=$ActorUserID&limit=50").fabric_service_channel_access_telemetry $postChannel = @($postAccess.active_channels | Where-Object { $_.channel_id -eq $lease.channel_id }) | Select-Object -First 1 $backendLine = (& ssh $DockerSSH "docker ps --format '{{.Names}} {{.Image}} {{.Status}}' | grep '^rap_test_backend '") | Out-String $nodeLines = (& ssh $DockerSSH "docker ps --format '{{.Names}} {{.Image}} {{.Status}}' | grep '^rap_test_node_test_'") | Out-String $checks = [ordered]@{ backend_expected_image_deployed = $backendLine.Contains($ExpectedBackendImage) node_agent_expected_image_deployed = $nodeLines.Contains($ExpectedNodeAgentImage) lease_ready = ([string]$lease.status -eq "ready") remediation_rebuild_route_visible = ($null -ne $matchingChannel -and [string](Get-PropertyValue -Item $matchingChannel -Name "remediation_action" -Default "") -eq "rebuild_route") remediation_command_visible = ($null -ne $command -and $commandID.Length -gt 0) durable_rebuild_intent_recorded = ($null -ne $attempt) durable_rebuild_intent_resolved_applied = ($null -ne $attempt -and [string](Get-PropertyValue -Item $attempt -Name "rebuild_status" -Default "") -eq "applied") durable_rebuild_intent_outcome_replacement_selected = ($null -ne $attempt -and [string](Get-PropertyValue -Item $attempt -Name "outcome" -Default "") -eq "replacement_selected") durable_rebuild_intent_replacement_matches = ($null -ne $attempt -and [string](Get-PropertyValue -Item $attempt -Name "replacement_route_id" -Default "") -eq $alternateRouteID) durable_rebuild_intent_source_matches = ($null -ne $attempt -and [string](Get-PropertyValue -Item $attempt -Name "decision_source" -Default "") -eq "service_channel_remediation_command") access_telemetry_reports_planner_applied = ($null -ne $postChannel -and [string](Get-PropertyValue -Item $postChannel -Name "remediation_execution_status" -Default "") -eq "rebuild_request_applied") node_route_manager_consumed_same_command = ($null -ne $routeManagerDecision) node_route_manager_applied_rebuild = ($null -ne $routeManagerDecision -and [string](Get-PropertyValue -Item $routeManagerDecision -Name "rebuild_status" -Default "") -eq "applied") node_route_manager_transition_applied_rebuild = ($null -ne $routeManagerTransition -and [string](Get-PropertyValue -Item $routeManagerTransition -Name "status" -Default "") -eq "applied_rebuild") baseline_runtime_snapshot_visible = ($null -ne $baselineSnapshot) all_pressure_bursts_accepted = ($acceptedBurstCount -eq $burstCount) all_pressure_bursts_selected_replacement = ($replacementBurstCount -eq $burstCount) stale_primary_not_reselected = ($stalePrimaryReselections -eq 0) final_last_selected_route_matches_replacement = ($null -ne $finalSnapshot -and $finalSnapshot.last_selected_route_id -eq $alternateRouteID) no_backend_or_local_fallback_after_pressure = (($finalFallbackLocal - $baselineFallbackLocal) -eq 0) no_route_failures_after_pressure = (($finalRouteFailures - $baselineRouteFailures) -eq 0) no_flow_drops_after_pressure = (($finalFlowDropped - $baselineFlowDropped) -eq 0 -and ($finalSchedulerDropped - $baselineSchedulerDropped) -eq 0) } $failed = @($checks.GetEnumerator() | Where-Object { -not $_.Value } | ForEach-Object { $_.Key }) $result = [ordered]@{ schema_version = "c18z80.service_channel_post_rebuild_pressure_smoke.v1" run_id = $runId cluster_id = $ClusterID channel_id = [string]$lease.channel_id primary_route_id = $primaryRouteID alternate_route_id = $alternateRouteID rebuild_request_id = $commandID passed = ($failed.Count -eq 0) checks = $checks failed_checks = $failed summary = [ordered]@{ backend_container = $backendLine.Trim() node_containers = $nodeLines.Trim() remediation_command = $command rebuild_attempt = $attempt route_manager_decision = $routeManagerDecision route_manager_transition = $routeManagerTransition burst_count = $burstCount burst_packet_count = $burstPacketCount accepted_burst_count = $acceptedBurstCount replacement_burst_count = $replacementBurstCount stale_primary_reselection_count = $stalePrimaryReselections baseline_snapshot = $baselineSnapshot final_snapshot = $finalSnapshot fallback_local_delta = ($finalFallbackLocal - $baselineFallbackLocal) route_failure_delta = ($finalRouteFailures - $baselineRouteFailures) flow_drop_delta = ($finalFlowDropped - $baselineFlowDropped) scheduler_drop_delta = ($finalSchedulerDropped - $baselineSchedulerDropped) burst_results = $burstResults post_channel = $postChannel } } $target = Join-Path $repoRoot $ResultPath $result | ConvertTo-Json -Depth 80 | Set-Content -Path $target -Encoding UTF8 try { if ($primaryRouteID) { Invoke-Api -Method POST -Path "/clusters/$ClusterID/mesh/route-intents/$primaryRouteID/expire" -Body @{ actor_user_id = $ActorUserID } | Out-Null } if ($alternateRouteID) { Invoke-Api -Method POST -Path "/clusters/$ClusterID/mesh/route-intents/$alternateRouteID/expire" -Body @{ actor_user_id = $ActorUserID } | Out-Null } Start-Sleep -Seconds 3 Invoke-Api -Method POST -Path "/clusters/$ClusterID/fabric/service-channels/leases/cleanup" -Body @{ actor_user_id = $ActorUserID limit = 50 } | Out-Null } catch { Write-Warning "cleanup failed after c18z80 smoke: $($_.Exception.Message)" } if (-not $result.passed) { throw "C18Z80 post-rebuild-pressure smoke failed: $($failed -join ', ')" } Write-Host "C18Z80 service-channel post-rebuild-pressure smoke passed. Result: $target" $result