param( [string]$ApiBaseUrl = "http://192.168.200.61:18121/api/v1", [string]$ClusterID = "cfc0743d-d960-49fb-9de8-96e063d5e4aa", [string]$ActorUserID = "f67d943f-5397-4b3a-a229-695fe67ad700", [string]$EntryNodeName = "test-1", [string]$ExitNodeName = "test-2", [string]$ExpectedBackendImage = "rap-backend:fabric-service-channel-0.2.281-c18z109", [string]$ExpectedNodeAgentImage = "rap-node-agent:0.2.270-c18z95", [string]$DockerSSH = "test-docker", [string]$ResultPath = "artifacts\c18z100-rebuild-health-feedback-breakdown-smoke-result.json" ) Set-StrictMode -Version Latest $ErrorActionPreference = "Stop" $scriptDir = Split-Path -Parent $MyInvocation.MyCommand.Path $repoRoot = (Resolve-Path (Join-Path $scriptDir "..\..")).ProviderPath $runId = "c18z100-" + (Get-Date -Format "yyyyMMdd-HHmmss") function Invoke-Api { param([string]$Method, [string]$Path, [object]$Body = $null) $params = @{ Method = $Method; Uri = "$ApiBaseUrl$Path"; TimeoutSec = 30 } if ($null -ne $Body) { $params.ContentType = "application/json" $params.Body = ($Body | ConvertTo-Json -Depth 80) } return Invoke-RestMethod @params } function Get-PropertyValue { param([object]$Item, [string]$Name, [object]$Default = $null) if ($null -eq $Item) { return $Default } $property = $Item.PSObject.Properties[$Name] if ($null -eq $property) { return $Default } return $property.Value } function Get-NodeByName { param([string]$Name) $nodes = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/nodes?actor_user_id=$ActorUserID").nodes $node = @($nodes | Where-Object { $_.name -eq $Name }) | Select-Object -First 1 if ($null -eq $node) { throw "Node '$Name' was not found" } return $node } function New-RouteIntent { param([string]$SourceNodeID, [string]$DestinationNodeID, [string]$Suffix, [int]$Priority) $expiresAt = (Get-Date).ToUniversalTime().AddMinutes(5).ToString("o") return (Invoke-Api -Method POST -Path "/clusters/$ClusterID/mesh/route-intents" -Body @{ actor_user_id = $ActorUserID source_selector = @{ node_id = $SourceNodeID } destination_selector = @{ node_id = $DestinationNodeID } service_class = "vpn_packets" priority = $Priority policy = @{ synthetic_enabled = $true route_version = "$runId-$Suffix" policy_version = "$runId-$Suffix" peer_directory_version = "$runId-$Suffix" hops = @($SourceNodeID, $DestinationNodeID) allowed_channels = @("vpn_packet", "fabric_control") max_ttl = 8 max_hops = 8 expires_at = $expiresAt metadata = @{ smoke = "c18z100_rebuild_health_feedback_breakdown"; run_id = $runId; suffix = $Suffix } } }).route_intent } $entryNode = Get-NodeByName -Name $EntryNodeName $exitNode = Get-NodeByName -Name $ExitNodeName $badRoute = New-RouteIntent -SourceNodeID $entryNode.id -DestinationNodeID $exitNode.id -Suffix "bad" -Priority 2100000000 $goodRoute = New-RouteIntent -SourceNodeID $entryNode.id -DestinationNodeID $exitNode.id -Suffix "good" -Priority 100 $lease = (Invoke-Api -Method POST -Path "/clusters/$ClusterID/fabric/service-channels/leases" -Body @{ actor_user_id = $ActorUserID organization_id = "org-home" user_id = "user-m" resource_id = "$runId-vpn" service_class = "vpn_packets" entry_node_ids = @([string]$entryNode.id) exit_node_ids = @([string]$exitNode.id) preferred_entry_node_id = [string]$entryNode.id preferred_exit_node_id = [string]$exitNode.id allowed_channels = @("vpn_packet", "fabric_control") ttl_seconds = 120 metadata = @{ smoke = "c18z100_rebuild_health_feedback_breakdown"; run_id = $runId } }).fabric_service_channel_lease $heartbeatBody = @{ health_status = "healthy" capabilities = @{ fabric_service_channel_access_telemetry = $true } service_states = @{} metadata = @{ fabric_service_channel_access_report = @{ schema_version = "c18z52.fabric_service_channel_access_report.v1" total = 1 signed = 1 degraded_route = 0 degraded_route_blocked = 1 fabric_route_send_failure = 1 data_plane_contract = 1 last_degraded_route_policy = "disabled" last_working_data_transport = "fabric_service_channel" last_steady_state_transport = "fabric_route" last_data_plane_violation_status = "fabric_route_send_failed_degraded_route_blocked" last_data_plane_violation_reason = "synthetic c18z100 route send failure" } } } Invoke-Api -Method POST -Path "/clusters/$ClusterID/nodes/$($entryNode.id)/heartbeats" -Body $heartbeatBody | Out-Null $feedback = $null $config = $null for ($i = 0; $i -lt 8; $i++) { Start-Sleep -Seconds 3 $feedbackItems = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/fabric/service-channels/route-feedback?actor_user_id=$ActorUserID&reporter_node_id=$($entryNode.id)&route_id=$($lease.primary_route.route_id)&service_class=vpn_packets").route_feedback $feedback = @($feedbackItems | Select-Object -First 1) $config = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/nodes/$($entryNode.id)/mesh/synthetic-config").synthetic_mesh_config $replacement = @($config.route_path_decisions.decisions | Where-Object { [string](Get-PropertyValue -Item $_ -Name "route_id" -Default "") -eq [string]$lease.primary_route.route_id -and [string](Get-PropertyValue -Item $_ -Name "replacement_route_id" -Default "") -ne "" }) | Select-Object -First 1 if ($null -ne $feedback -and $null -ne $replacement) { break } } $firstFeedback = $feedback $firstFeedbackID = if ($null -ne $firstFeedback) { [string]$firstFeedback.id } else { "" } $firstObservedAt = if ($null -ne $firstFeedback) { [string]$firstFeedback.observed_at } else { "" } Start-Sleep -Seconds 2 Invoke-Api -Method POST -Path "/clusters/$ClusterID/nodes/$($entryNode.id)/heartbeats" -Body $heartbeatBody | Out-Null Start-Sleep -Seconds 2 $feedbackAfterDuplicateItems = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/fabric/service-channels/route-feedback?actor_user_id=$ActorUserID&reporter_node_id=$($entryNode.id)&route_id=$($lease.primary_route.route_id)&service_class=vpn_packets").route_feedback $feedbackAfterDuplicate = @($feedbackAfterDuplicateItems | Select-Object -First 1) $duplicateFeedbackCount = @($feedbackAfterDuplicateItems).Count $secondFeedbackID = if ($null -ne $feedbackAfterDuplicate) { [string]$feedbackAfterDuplicate.id } else { "" } $secondObservedAt = if ($null -ne $feedbackAfterDuplicate) { [string]$feedbackAfterDuplicate.observed_at } else { "" } $replacement = @($config.route_path_decisions.decisions | Where-Object { [string](Get-PropertyValue -Item $_ -Name "route_id" -Default "") -eq [string]$lease.primary_route.route_id -and [string](Get-PropertyValue -Item $_ -Name "replacement_route_id" -Default "") -ne "" }) | Select-Object -First 1 $ledgerItems = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/fabric/service-channels/rebuild-attempts?actor_user_id=$ActorUserID&reporter_node_id=$($entryNode.id)&route_id=$($lease.primary_route.route_id)&limit=20&enrichment=summary").rebuild_attempts $ledgerAttempt = @($ledgerItems | Where-Object { [string](Get-PropertyValue -Item $_ -Name "rebuild_request_id" -Default "") -eq [string](Get-PropertyValue -Item $replacement -Name "rebuild_request_id" -Default "") }) | Select-Object -First 1 $sourceFilteredLedger = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/fabric/service-channels/rebuild-attempts?actor_user_id=$ActorUserID&feedback_source=fabric_service_channel_access_report&limit=20&enrichment=summary").rebuild_attempts $channelFilteredLedger = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/fabric/service-channels/rebuild-attempts?actor_user_id=$ActorUserID&feedback_channel_id=$($lease.channel_id)&limit=20&enrichment=summary").rebuild_attempts $violationFilteredLedger = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/fabric/service-channels/rebuild-attempts?actor_user_id=$ActorUserID&feedback_violation_status=fabric_route_send_failed_degraded_route_blocked&limit=20&enrichment=summary").rebuild_attempts $combinedFilteredLedger = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/fabric/service-channels/rebuild-attempts?actor_user_id=$ActorUserID&feedback_source=fabric_service_channel_access_report&feedback_channel_id=$($lease.channel_id)&feedback_violation_status=fabric_route_send_failed_degraded_route_blocked&limit=20&enrichment=summary").rebuild_attempts $wrongChannelFilteredLedger = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/fabric/service-channels/rebuild-attempts?actor_user_id=$ActorUserID&feedback_channel_id=00000000-0000-0000-0000-000000000000&limit=20&enrichment=summary").rebuild_attempts $sourceFilteredAttempt = @($sourceFilteredLedger | Where-Object { [string](Get-PropertyValue -Item $_ -Name "rebuild_request_id" -Default "") -eq [string](Get-PropertyValue -Item $replacement -Name "rebuild_request_id" -Default "") }) | Select-Object -First 1 $channelFilteredAttempt = @($channelFilteredLedger | Where-Object { [string](Get-PropertyValue -Item $_ -Name "rebuild_request_id" -Default "") -eq [string](Get-PropertyValue -Item $replacement -Name "rebuild_request_id" -Default "") }) | Select-Object -First 1 $violationFilteredAttempt = @($violationFilteredLedger | Where-Object { [string](Get-PropertyValue -Item $_ -Name "rebuild_request_id" -Default "") -eq [string](Get-PropertyValue -Item $replacement -Name "rebuild_request_id" -Default "") }) | Select-Object -First 1 $combinedFilteredAttempt = @($combinedFilteredLedger | Where-Object { [string](Get-PropertyValue -Item $_ -Name "rebuild_request_id" -Default "") -eq [string](Get-PropertyValue -Item $replacement -Name "rebuild_request_id" -Default "") }) | Select-Object -First 1 $health = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/fabric/service-channels/rebuild-health?actor_user_id=$ActorUserID&limit=50").rebuild_health $feedbackBreakdown = @((Get-PropertyValue -Item $health -Name "feedback_breakdowns" -Default @()) | Where-Object { [string](Get-PropertyValue -Item $_ -Name "feedback_source" -Default "") -eq "fabric_service_channel_access_report" -and [string](Get-PropertyValue -Item $_ -Name "feedback_channel_id" -Default "") -eq [string]$lease.channel_id -and [string](Get-PropertyValue -Item $_ -Name "feedback_violation_status" -Default "") -eq "fabric_route_send_failed_degraded_route_blocked" }) | Select-Object -First 1 $backendLine = (& ssh $DockerSSH "docker ps --format '{{.Names}} {{.Image}} {{.Status}}' | grep '^rap_test_backend '") | Out-String $nodeLines = (& ssh $DockerSSH "docker ps --format '{{.Names}} {{.Image}} {{.Status}}' | grep '^rap_test_node_test_'") | Out-String $checks = [ordered]@{ backend_expected_image_deployed = $backendLine.Contains($ExpectedBackendImage) node_agent_expected_image_deployed = $nodeLines.Contains($ExpectedNodeAgentImage) lease_selected_bad_route_first = ([string]$lease.primary_route.route_id -eq [string]$badRoute.id) route_feedback_recorded_from_access_report = ($null -ne $feedback) route_feedback_is_fenced = ($null -ne $feedback -and [string]$feedback.feedback_status -eq "fenced") route_feedback_contains_blocked_policy_reason = ($null -ne $feedback -and @($feedback.reasons | Where-Object { [string]$_ -eq "degraded_route_blocked_by_policy" }).Count -gt 0) duplicate_heartbeat_kept_single_latest_feedback = ($duplicateFeedbackCount -eq 1) duplicate_heartbeat_kept_feedback_id = ($firstFeedbackID -ne "" -and $firstFeedbackID -eq $secondFeedbackID) duplicate_heartbeat_kept_observed_at = ($firstObservedAt -ne "" -and $firstObservedAt -eq $secondObservedAt) planner_selected_replacement = ($null -ne $replacement) planner_replacement_is_good_route = ($null -ne $replacement -and [string]$replacement.replacement_route_id -eq [string]$goodRoute.id) planner_rebuild_status_applied = ($null -ne $replacement -and [string]$replacement.rebuild_status -eq "applied") planner_decision_links_feedback_observation = ($null -ne $replacement -and [string](Get-PropertyValue -Item $replacement -Name "feedback_observation_id" -Default "") -eq $firstFeedbackID) planner_decision_links_access_report_source = ($null -ne $replacement -and [string](Get-PropertyValue -Item $replacement -Name "feedback_source" -Default "") -eq "fabric_service_channel_access_report") planner_decision_links_channel = ($null -ne $replacement -and [string](Get-PropertyValue -Item $replacement -Name "feedback_channel_id" -Default "") -eq [string]$lease.channel_id) planner_decision_links_violation = ($null -ne $replacement -and [string](Get-PropertyValue -Item $replacement -Name "feedback_violation_status" -Default "") -eq "fabric_route_send_failed_degraded_route_blocked") rebuild_ledger_recorded_correlated_attempt = ($null -ne $ledgerAttempt) rebuild_ledger_links_feedback_observation = ($null -ne $ledgerAttempt -and [string](Get-PropertyValue -Item $ledgerAttempt -Name "feedback_observation_id" -Default "") -eq $firstFeedbackID) rebuild_ledger_links_access_report_source = ($null -ne $ledgerAttempt -and [string](Get-PropertyValue -Item $ledgerAttempt -Name "feedback_source" -Default "") -eq "fabric_service_channel_access_report") rebuild_ledger_payload_links_feedback = ($null -ne $ledgerAttempt -and [string](Get-PropertyValue -Item (Get-PropertyValue -Item $ledgerAttempt -Name "payload" -Default $null) -Name "feedback_observation_id" -Default "") -eq $firstFeedbackID) filter_by_feedback_source_returns_attempt = ($null -ne $sourceFilteredAttempt) filter_by_feedback_channel_returns_attempt = ($null -ne $channelFilteredAttempt) filter_by_feedback_violation_returns_attempt = ($null -ne $violationFilteredAttempt) combined_feedback_filters_return_attempt = ($null -ne $combinedFilteredAttempt) wrong_channel_filter_excludes_attempt = (@($wrongChannelFilteredLedger | Where-Object { [string](Get-PropertyValue -Item $_ -Name "rebuild_request_id" -Default "") -eq [string](Get-PropertyValue -Item $replacement -Name "rebuild_request_id" -Default "") }).Count -eq 0) rebuild_health_returns_feedback_breakdown = ($null -ne $feedbackBreakdown) feedback_breakdown_counts_attempt = ($null -ne $feedbackBreakdown -and [int](Get-PropertyValue -Item $feedbackBreakdown -Name "total_count" -Default 0) -ge 1) feedback_breakdown_lists_reporter = ($null -ne $feedbackBreakdown -and @((Get-PropertyValue -Item $feedbackBreakdown -Name "affected_reporter_node_ids" -Default @()) | Where-Object { [string]$_ -eq [string]$entryNode.id }).Count -gt 0) feedback_breakdown_lists_route = ($null -ne $feedbackBreakdown -and @((Get-PropertyValue -Item $feedbackBreakdown -Name "affected_route_ids" -Default @()) | Where-Object { [string]$_ -eq [string]$lease.primary_route.route_id }).Count -gt 0) } $failed = @($checks.GetEnumerator() | Where-Object { -not $_.Value } | ForEach-Object { $_.Key }) $result = [ordered]@{ schema_version = "c18z100.rebuild_health_feedback_breakdown_smoke.v1" run_id = $runId cluster_id = $ClusterID channel_id = [string]$lease.channel_id primary_route_id = [string]$lease.primary_route.route_id replacement_route_id = if ($null -ne $replacement) { [string]$replacement.replacement_route_id } else { "" } passed = ($failed.Count -eq 0) checks = $checks failed_checks = $failed summary = [ordered]@{ backend_container = $backendLine.Trim() node_containers = $nodeLines.Trim() bad_route = $badRoute good_route = $goodRoute lease = $lease feedback = $feedback feedback_after_duplicate = $feedbackAfterDuplicate replacement = $replacement ledger_attempt = $ledgerAttempt source_filtered_attempt = $sourceFilteredAttempt channel_filtered_attempt = $channelFilteredAttempt violation_filtered_attempt = $violationFilteredAttempt combined_filtered_attempt = $combinedFilteredAttempt wrong_channel_filtered_count = @($wrongChannelFilteredLedger).Count rebuild_health = $health feedback_breakdown = $feedbackBreakdown } } $target = Join-Path $repoRoot $ResultPath $result | ConvertTo-Json -Depth 80 | Set-Content -Path $target -Encoding UTF8 try { foreach ($routeID in @([string]$badRoute.id, [string]$goodRoute.id)) { if ($routeID) { Invoke-Api -Method POST -Path "/clusters/$ClusterID/mesh/route-intents/$routeID/expire" -Body @{ actor_user_id = $ActorUserID } | Out-Null } } Invoke-Api -Method POST -Path "/clusters/$ClusterID/fabric/service-channels/leases/cleanup" -Body @{ actor_user_id = $ActorUserID; limit = 100 } | Out-Null } catch { Write-Warning "cleanup failed after c18z100 smoke: $($_.Exception.Message)" } if (-not $result.passed) { throw "C18Z100 rebuild health feedback breakdown smoke failed: $($failed -join ', ')" } Write-Host "C18Z100 rebuild health feedback breakdown smoke passed. Result: $target" $result