251 lines
17 KiB
PowerShell
251 lines
17 KiB
PowerShell
param(
|
|
[string]$ApiBaseUrl = "http://192.168.200.61:18121/api/v1",
|
|
[string]$ClusterID = "cfc0743d-d960-49fb-9de8-96e063d5e4aa",
|
|
[string]$ActorUserID = "f67d943f-5397-4b3a-a229-695fe67ad700",
|
|
[string]$EntryNodeName = "test-1",
|
|
[string]$ExitNodeName = "test-2",
|
|
[string]$ExpectedBackendImage = "rap-backend:fabric-service-channel-0.2.281-c18z109",
|
|
[string]$ExpectedNodeAgentImage = "rap-node-agent:0.2.270-c18z95",
|
|
[string]$DockerSSH = "test-docker",
|
|
[string]$ResultPath = "artifacts\c18z100-rebuild-health-feedback-breakdown-smoke-result.json"
|
|
)
|
|
|
|
Set-StrictMode -Version Latest
|
|
$ErrorActionPreference = "Stop"
|
|
|
|
$scriptDir = Split-Path -Parent $MyInvocation.MyCommand.Path
|
|
$repoRoot = (Resolve-Path (Join-Path $scriptDir "..\..")).ProviderPath
|
|
$runId = "c18z100-" + (Get-Date -Format "yyyyMMdd-HHmmss")
|
|
|
|
function Invoke-Api {
|
|
param([string]$Method, [string]$Path, [object]$Body = $null)
|
|
$params = @{ Method = $Method; Uri = "$ApiBaseUrl$Path"; TimeoutSec = 30 }
|
|
if ($null -ne $Body) {
|
|
$params.ContentType = "application/json"
|
|
$params.Body = ($Body | ConvertTo-Json -Depth 80)
|
|
}
|
|
return Invoke-RestMethod @params
|
|
}
|
|
|
|
function Get-PropertyValue {
|
|
param([object]$Item, [string]$Name, [object]$Default = $null)
|
|
if ($null -eq $Item) { return $Default }
|
|
$property = $Item.PSObject.Properties[$Name]
|
|
if ($null -eq $property) { return $Default }
|
|
return $property.Value
|
|
}
|
|
|
|
function Get-NodeByName {
|
|
param([string]$Name)
|
|
$nodes = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/nodes?actor_user_id=$ActorUserID").nodes
|
|
$node = @($nodes | Where-Object { $_.name -eq $Name }) | Select-Object -First 1
|
|
if ($null -eq $node) { throw "Node '$Name' was not found" }
|
|
return $node
|
|
}
|
|
|
|
function New-RouteIntent {
|
|
param([string]$SourceNodeID, [string]$DestinationNodeID, [string]$Suffix, [int]$Priority)
|
|
$expiresAt = (Get-Date).ToUniversalTime().AddMinutes(5).ToString("o")
|
|
return (Invoke-Api -Method POST -Path "/clusters/$ClusterID/mesh/route-intents" -Body @{
|
|
actor_user_id = $ActorUserID
|
|
source_selector = @{ node_id = $SourceNodeID }
|
|
destination_selector = @{ node_id = $DestinationNodeID }
|
|
service_class = "vpn_packets"
|
|
priority = $Priority
|
|
policy = @{
|
|
synthetic_enabled = $true
|
|
route_version = "$runId-$Suffix"
|
|
policy_version = "$runId-$Suffix"
|
|
peer_directory_version = "$runId-$Suffix"
|
|
hops = @($SourceNodeID, $DestinationNodeID)
|
|
allowed_channels = @("vpn_packet", "fabric_control")
|
|
max_ttl = 8
|
|
max_hops = 8
|
|
expires_at = $expiresAt
|
|
metadata = @{ smoke = "c18z100_rebuild_health_feedback_breakdown"; run_id = $runId; suffix = $Suffix }
|
|
}
|
|
}).route_intent
|
|
}
|
|
|
|
$entryNode = Get-NodeByName -Name $EntryNodeName
|
|
$exitNode = Get-NodeByName -Name $ExitNodeName
|
|
$badRoute = New-RouteIntent -SourceNodeID $entryNode.id -DestinationNodeID $exitNode.id -Suffix "bad" -Priority 2100000000
|
|
$goodRoute = New-RouteIntent -SourceNodeID $entryNode.id -DestinationNodeID $exitNode.id -Suffix "good" -Priority 100
|
|
|
|
$lease = (Invoke-Api -Method POST -Path "/clusters/$ClusterID/fabric/service-channels/leases" -Body @{
|
|
actor_user_id = $ActorUserID
|
|
organization_id = "org-home"
|
|
user_id = "user-m"
|
|
resource_id = "$runId-vpn"
|
|
service_class = "vpn_packets"
|
|
entry_node_ids = @([string]$entryNode.id)
|
|
exit_node_ids = @([string]$exitNode.id)
|
|
preferred_entry_node_id = [string]$entryNode.id
|
|
preferred_exit_node_id = [string]$exitNode.id
|
|
allowed_channels = @("vpn_packet", "fabric_control")
|
|
ttl_seconds = 120
|
|
metadata = @{ smoke = "c18z100_rebuild_health_feedback_breakdown"; run_id = $runId }
|
|
}).fabric_service_channel_lease
|
|
|
|
$heartbeatBody = @{
|
|
health_status = "healthy"
|
|
capabilities = @{ fabric_service_channel_access_telemetry = $true }
|
|
service_states = @{}
|
|
metadata = @{
|
|
fabric_service_channel_access_report = @{
|
|
schema_version = "c18z52.fabric_service_channel_access_report.v1"
|
|
total = 1
|
|
signed = 1
|
|
backend_fallback = 0
|
|
backend_fallback_blocked = 1
|
|
fabric_route_send_failure = 1
|
|
data_plane_contract = 1
|
|
last_backend_relay_policy = "disabled"
|
|
last_working_data_transport = "fabric_service_channel"
|
|
last_steady_state_transport = "fabric_route"
|
|
last_data_plane_violation_status = "fabric_route_send_failed_backend_fallback_blocked"
|
|
last_data_plane_violation_reason = "synthetic c18z100 route send failure"
|
|
}
|
|
}
|
|
}
|
|
|
|
Invoke-Api -Method POST -Path "/clusters/$ClusterID/nodes/$($entryNode.id)/heartbeats" -Body $heartbeatBody | Out-Null
|
|
|
|
$feedback = $null
|
|
$config = $null
|
|
for ($i = 0; $i -lt 8; $i++) {
|
|
Start-Sleep -Seconds 3
|
|
$feedbackItems = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/fabric/service-channels/route-feedback?actor_user_id=$ActorUserID&reporter_node_id=$($entryNode.id)&route_id=$($lease.primary_route.route_id)&service_class=vpn_packets").route_feedback
|
|
$feedback = @($feedbackItems | Select-Object -First 1)
|
|
$config = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/nodes/$($entryNode.id)/mesh/synthetic-config").synthetic_mesh_config
|
|
$replacement = @($config.route_path_decisions.decisions | Where-Object {
|
|
[string](Get-PropertyValue -Item $_ -Name "route_id" -Default "") -eq [string]$lease.primary_route.route_id -and
|
|
[string](Get-PropertyValue -Item $_ -Name "replacement_route_id" -Default "") -ne ""
|
|
}) | Select-Object -First 1
|
|
if ($null -ne $feedback -and $null -ne $replacement) { break }
|
|
}
|
|
|
|
$firstFeedback = $feedback
|
|
$firstFeedbackID = if ($null -ne $firstFeedback) { [string]$firstFeedback.id } else { "" }
|
|
$firstObservedAt = if ($null -ne $firstFeedback) { [string]$firstFeedback.observed_at } else { "" }
|
|
Start-Sleep -Seconds 2
|
|
Invoke-Api -Method POST -Path "/clusters/$ClusterID/nodes/$($entryNode.id)/heartbeats" -Body $heartbeatBody | Out-Null
|
|
Start-Sleep -Seconds 2
|
|
$feedbackAfterDuplicateItems = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/fabric/service-channels/route-feedback?actor_user_id=$ActorUserID&reporter_node_id=$($entryNode.id)&route_id=$($lease.primary_route.route_id)&service_class=vpn_packets").route_feedback
|
|
$feedbackAfterDuplicate = @($feedbackAfterDuplicateItems | Select-Object -First 1)
|
|
$duplicateFeedbackCount = @($feedbackAfterDuplicateItems).Count
|
|
$secondFeedbackID = if ($null -ne $feedbackAfterDuplicate) { [string]$feedbackAfterDuplicate.id } else { "" }
|
|
$secondObservedAt = if ($null -ne $feedbackAfterDuplicate) { [string]$feedbackAfterDuplicate.observed_at } else { "" }
|
|
|
|
$replacement = @($config.route_path_decisions.decisions | Where-Object {
|
|
[string](Get-PropertyValue -Item $_ -Name "route_id" -Default "") -eq [string]$lease.primary_route.route_id -and
|
|
[string](Get-PropertyValue -Item $_ -Name "replacement_route_id" -Default "") -ne ""
|
|
}) | Select-Object -First 1
|
|
|
|
$ledgerItems = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/fabric/service-channels/rebuild-attempts?actor_user_id=$ActorUserID&reporter_node_id=$($entryNode.id)&route_id=$($lease.primary_route.route_id)&limit=20&enrichment=summary").rebuild_attempts
|
|
$ledgerAttempt = @($ledgerItems | Where-Object {
|
|
[string](Get-PropertyValue -Item $_ -Name "rebuild_request_id" -Default "") -eq [string](Get-PropertyValue -Item $replacement -Name "rebuild_request_id" -Default "")
|
|
}) | Select-Object -First 1
|
|
$sourceFilteredLedger = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/fabric/service-channels/rebuild-attempts?actor_user_id=$ActorUserID&feedback_source=fabric_service_channel_access_report&limit=20&enrichment=summary").rebuild_attempts
|
|
$channelFilteredLedger = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/fabric/service-channels/rebuild-attempts?actor_user_id=$ActorUserID&feedback_channel_id=$($lease.channel_id)&limit=20&enrichment=summary").rebuild_attempts
|
|
$violationFilteredLedger = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/fabric/service-channels/rebuild-attempts?actor_user_id=$ActorUserID&feedback_violation_status=fabric_route_send_failed_backend_fallback_blocked&limit=20&enrichment=summary").rebuild_attempts
|
|
$combinedFilteredLedger = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/fabric/service-channels/rebuild-attempts?actor_user_id=$ActorUserID&feedback_source=fabric_service_channel_access_report&feedback_channel_id=$($lease.channel_id)&feedback_violation_status=fabric_route_send_failed_backend_fallback_blocked&limit=20&enrichment=summary").rebuild_attempts
|
|
$wrongChannelFilteredLedger = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/fabric/service-channels/rebuild-attempts?actor_user_id=$ActorUserID&feedback_channel_id=00000000-0000-0000-0000-000000000000&limit=20&enrichment=summary").rebuild_attempts
|
|
$sourceFilteredAttempt = @($sourceFilteredLedger | Where-Object { [string](Get-PropertyValue -Item $_ -Name "rebuild_request_id" -Default "") -eq [string](Get-PropertyValue -Item $replacement -Name "rebuild_request_id" -Default "") }) | Select-Object -First 1
|
|
$channelFilteredAttempt = @($channelFilteredLedger | Where-Object { [string](Get-PropertyValue -Item $_ -Name "rebuild_request_id" -Default "") -eq [string](Get-PropertyValue -Item $replacement -Name "rebuild_request_id" -Default "") }) | Select-Object -First 1
|
|
$violationFilteredAttempt = @($violationFilteredLedger | Where-Object { [string](Get-PropertyValue -Item $_ -Name "rebuild_request_id" -Default "") -eq [string](Get-PropertyValue -Item $replacement -Name "rebuild_request_id" -Default "") }) | Select-Object -First 1
|
|
$combinedFilteredAttempt = @($combinedFilteredLedger | Where-Object { [string](Get-PropertyValue -Item $_ -Name "rebuild_request_id" -Default "") -eq [string](Get-PropertyValue -Item $replacement -Name "rebuild_request_id" -Default "") }) | Select-Object -First 1
|
|
$health = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/fabric/service-channels/rebuild-health?actor_user_id=$ActorUserID&limit=50").rebuild_health
|
|
$feedbackBreakdown = @((Get-PropertyValue -Item $health -Name "feedback_breakdowns" -Default @()) | Where-Object {
|
|
[string](Get-PropertyValue -Item $_ -Name "feedback_source" -Default "") -eq "fabric_service_channel_access_report" -and
|
|
[string](Get-PropertyValue -Item $_ -Name "feedback_channel_id" -Default "") -eq [string]$lease.channel_id -and
|
|
[string](Get-PropertyValue -Item $_ -Name "feedback_violation_status" -Default "") -eq "fabric_route_send_failed_backend_fallback_blocked"
|
|
}) | Select-Object -First 1
|
|
|
|
$backendLine = (& ssh $DockerSSH "docker ps --format '{{.Names}} {{.Image}} {{.Status}}' | grep '^rap_test_backend '") | Out-String
|
|
$nodeLines = (& ssh $DockerSSH "docker ps --format '{{.Names}} {{.Image}} {{.Status}}' | grep '^rap_test_node_test_'") | Out-String
|
|
|
|
$checks = [ordered]@{
|
|
backend_expected_image_deployed = $backendLine.Contains($ExpectedBackendImage)
|
|
node_agent_expected_image_deployed = $nodeLines.Contains($ExpectedNodeAgentImage)
|
|
lease_selected_bad_route_first = ([string]$lease.primary_route.route_id -eq [string]$badRoute.id)
|
|
route_feedback_recorded_from_access_report = ($null -ne $feedback)
|
|
route_feedback_is_fenced = ($null -ne $feedback -and [string]$feedback.feedback_status -eq "fenced")
|
|
route_feedback_contains_blocked_policy_reason = ($null -ne $feedback -and @($feedback.reasons | Where-Object { [string]$_ -eq "backend_fallback_blocked_by_policy" }).Count -gt 0)
|
|
duplicate_heartbeat_kept_single_latest_feedback = ($duplicateFeedbackCount -eq 1)
|
|
duplicate_heartbeat_kept_feedback_id = ($firstFeedbackID -ne "" -and $firstFeedbackID -eq $secondFeedbackID)
|
|
duplicate_heartbeat_kept_observed_at = ($firstObservedAt -ne "" -and $firstObservedAt -eq $secondObservedAt)
|
|
planner_selected_replacement = ($null -ne $replacement)
|
|
planner_replacement_is_good_route = ($null -ne $replacement -and [string]$replacement.replacement_route_id -eq [string]$goodRoute.id)
|
|
planner_rebuild_status_applied = ($null -ne $replacement -and [string]$replacement.rebuild_status -eq "applied")
|
|
planner_decision_links_feedback_observation = ($null -ne $replacement -and [string](Get-PropertyValue -Item $replacement -Name "feedback_observation_id" -Default "") -eq $firstFeedbackID)
|
|
planner_decision_links_access_report_source = ($null -ne $replacement -and [string](Get-PropertyValue -Item $replacement -Name "feedback_source" -Default "") -eq "fabric_service_channel_access_report")
|
|
planner_decision_links_channel = ($null -ne $replacement -and [string](Get-PropertyValue -Item $replacement -Name "feedback_channel_id" -Default "") -eq [string]$lease.channel_id)
|
|
planner_decision_links_violation = ($null -ne $replacement -and [string](Get-PropertyValue -Item $replacement -Name "feedback_violation_status" -Default "") -eq "fabric_route_send_failed_backend_fallback_blocked")
|
|
rebuild_ledger_recorded_correlated_attempt = ($null -ne $ledgerAttempt)
|
|
rebuild_ledger_links_feedback_observation = ($null -ne $ledgerAttempt -and [string](Get-PropertyValue -Item $ledgerAttempt -Name "feedback_observation_id" -Default "") -eq $firstFeedbackID)
|
|
rebuild_ledger_links_access_report_source = ($null -ne $ledgerAttempt -and [string](Get-PropertyValue -Item $ledgerAttempt -Name "feedback_source" -Default "") -eq "fabric_service_channel_access_report")
|
|
rebuild_ledger_payload_links_feedback = ($null -ne $ledgerAttempt -and [string](Get-PropertyValue -Item (Get-PropertyValue -Item $ledgerAttempt -Name "payload" -Default $null) -Name "feedback_observation_id" -Default "") -eq $firstFeedbackID)
|
|
filter_by_feedback_source_returns_attempt = ($null -ne $sourceFilteredAttempt)
|
|
filter_by_feedback_channel_returns_attempt = ($null -ne $channelFilteredAttempt)
|
|
filter_by_feedback_violation_returns_attempt = ($null -ne $violationFilteredAttempt)
|
|
combined_feedback_filters_return_attempt = ($null -ne $combinedFilteredAttempt)
|
|
wrong_channel_filter_excludes_attempt = (@($wrongChannelFilteredLedger | Where-Object { [string](Get-PropertyValue -Item $_ -Name "rebuild_request_id" -Default "") -eq [string](Get-PropertyValue -Item $replacement -Name "rebuild_request_id" -Default "") }).Count -eq 0)
|
|
rebuild_health_returns_feedback_breakdown = ($null -ne $feedbackBreakdown)
|
|
feedback_breakdown_counts_attempt = ($null -ne $feedbackBreakdown -and [int](Get-PropertyValue -Item $feedbackBreakdown -Name "total_count" -Default 0) -ge 1)
|
|
feedback_breakdown_lists_reporter = ($null -ne $feedbackBreakdown -and @((Get-PropertyValue -Item $feedbackBreakdown -Name "affected_reporter_node_ids" -Default @()) | Where-Object { [string]$_ -eq [string]$entryNode.id }).Count -gt 0)
|
|
feedback_breakdown_lists_route = ($null -ne $feedbackBreakdown -and @((Get-PropertyValue -Item $feedbackBreakdown -Name "affected_route_ids" -Default @()) | Where-Object { [string]$_ -eq [string]$lease.primary_route.route_id }).Count -gt 0)
|
|
}
|
|
$failed = @($checks.GetEnumerator() | Where-Object { -not $_.Value } | ForEach-Object { $_.Key })
|
|
|
|
$result = [ordered]@{
|
|
schema_version = "c18z100.rebuild_health_feedback_breakdown_smoke.v1"
|
|
run_id = $runId
|
|
cluster_id = $ClusterID
|
|
channel_id = [string]$lease.channel_id
|
|
primary_route_id = [string]$lease.primary_route.route_id
|
|
replacement_route_id = if ($null -ne $replacement) { [string]$replacement.replacement_route_id } else { "" }
|
|
passed = ($failed.Count -eq 0)
|
|
checks = $checks
|
|
failed_checks = $failed
|
|
summary = [ordered]@{
|
|
backend_container = $backendLine.Trim()
|
|
node_containers = $nodeLines.Trim()
|
|
bad_route = $badRoute
|
|
good_route = $goodRoute
|
|
lease = $lease
|
|
feedback = $feedback
|
|
feedback_after_duplicate = $feedbackAfterDuplicate
|
|
replacement = $replacement
|
|
ledger_attempt = $ledgerAttempt
|
|
source_filtered_attempt = $sourceFilteredAttempt
|
|
channel_filtered_attempt = $channelFilteredAttempt
|
|
violation_filtered_attempt = $violationFilteredAttempt
|
|
combined_filtered_attempt = $combinedFilteredAttempt
|
|
wrong_channel_filtered_count = @($wrongChannelFilteredLedger).Count
|
|
rebuild_health = $health
|
|
feedback_breakdown = $feedbackBreakdown
|
|
}
|
|
}
|
|
|
|
$target = Join-Path $repoRoot $ResultPath
|
|
$result | ConvertTo-Json -Depth 80 | Set-Content -Path $target -Encoding UTF8
|
|
|
|
try {
|
|
foreach ($routeID in @([string]$badRoute.id, [string]$goodRoute.id)) {
|
|
if ($routeID) {
|
|
Invoke-Api -Method POST -Path "/clusters/$ClusterID/mesh/route-intents/$routeID/expire" -Body @{ actor_user_id = $ActorUserID } | Out-Null
|
|
}
|
|
}
|
|
Invoke-Api -Method POST -Path "/clusters/$ClusterID/fabric/service-channels/leases/cleanup" -Body @{ actor_user_id = $ActorUserID; limit = 100 } | Out-Null
|
|
} catch {
|
|
Write-Warning "cleanup failed after c18z100 smoke: $($_.Exception.Message)"
|
|
}
|
|
|
|
if (-not $result.passed) {
|
|
throw "C18Z100 rebuild health feedback breakdown smoke failed: $($failed -join ', ')"
|
|
}
|
|
|
|
Write-Host "C18Z100 rebuild health feedback breakdown smoke passed. Result: $target"
|
|
$result
|