Record project continuation changes
This commit is contained in:
@@ -0,0 +1,250 @@
|
||||
param(
|
||||
[string]$ApiBaseUrl = "http://192.168.200.61:18121/api/v1",
|
||||
[string]$ClusterID = "cfc0743d-d960-49fb-9de8-96e063d5e4aa",
|
||||
[string]$ActorUserID = "f67d943f-5397-4b3a-a229-695fe67ad700",
|
||||
[string]$EntryNodeName = "test-1",
|
||||
[string]$ExitNodeName = "test-2",
|
||||
[string]$ExpectedBackendImage = "rap-backend:fabric-service-channel-0.2.281-c18z109",
|
||||
[string]$ExpectedNodeAgentImage = "rap-node-agent:0.2.270-c18z95",
|
||||
[string]$DockerSSH = "test-docker",
|
||||
[string]$ResultPath = "artifacts\c18z100-rebuild-health-feedback-breakdown-smoke-result.json"
|
||||
)
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
$scriptDir = Split-Path -Parent $MyInvocation.MyCommand.Path
|
||||
$repoRoot = (Resolve-Path (Join-Path $scriptDir "..\..")).ProviderPath
|
||||
$runId = "c18z100-" + (Get-Date -Format "yyyyMMdd-HHmmss")
|
||||
|
||||
function Invoke-Api {
|
||||
param([string]$Method, [string]$Path, [object]$Body = $null)
|
||||
$params = @{ Method = $Method; Uri = "$ApiBaseUrl$Path"; TimeoutSec = 30 }
|
||||
if ($null -ne $Body) {
|
||||
$params.ContentType = "application/json"
|
||||
$params.Body = ($Body | ConvertTo-Json -Depth 80)
|
||||
}
|
||||
return Invoke-RestMethod @params
|
||||
}
|
||||
|
||||
function Get-PropertyValue {
|
||||
param([object]$Item, [string]$Name, [object]$Default = $null)
|
||||
if ($null -eq $Item) { return $Default }
|
||||
$property = $Item.PSObject.Properties[$Name]
|
||||
if ($null -eq $property) { return $Default }
|
||||
return $property.Value
|
||||
}
|
||||
|
||||
function Get-NodeByName {
|
||||
param([string]$Name)
|
||||
$nodes = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/nodes?actor_user_id=$ActorUserID").nodes
|
||||
$node = @($nodes | Where-Object { $_.name -eq $Name }) | Select-Object -First 1
|
||||
if ($null -eq $node) { throw "Node '$Name' was not found" }
|
||||
return $node
|
||||
}
|
||||
|
||||
function New-RouteIntent {
|
||||
param([string]$SourceNodeID, [string]$DestinationNodeID, [string]$Suffix, [int]$Priority)
|
||||
$expiresAt = (Get-Date).ToUniversalTime().AddMinutes(5).ToString("o")
|
||||
return (Invoke-Api -Method POST -Path "/clusters/$ClusterID/mesh/route-intents" -Body @{
|
||||
actor_user_id = $ActorUserID
|
||||
source_selector = @{ node_id = $SourceNodeID }
|
||||
destination_selector = @{ node_id = $DestinationNodeID }
|
||||
service_class = "vpn_packets"
|
||||
priority = $Priority
|
||||
policy = @{
|
||||
synthetic_enabled = $true
|
||||
route_version = "$runId-$Suffix"
|
||||
policy_version = "$runId-$Suffix"
|
||||
peer_directory_version = "$runId-$Suffix"
|
||||
hops = @($SourceNodeID, $DestinationNodeID)
|
||||
allowed_channels = @("vpn_packet", "fabric_control")
|
||||
max_ttl = 8
|
||||
max_hops = 8
|
||||
expires_at = $expiresAt
|
||||
metadata = @{ smoke = "c18z100_rebuild_health_feedback_breakdown"; run_id = $runId; suffix = $Suffix }
|
||||
}
|
||||
}).route_intent
|
||||
}
|
||||
|
||||
$entryNode = Get-NodeByName -Name $EntryNodeName
|
||||
$exitNode = Get-NodeByName -Name $ExitNodeName
|
||||
$badRoute = New-RouteIntent -SourceNodeID $entryNode.id -DestinationNodeID $exitNode.id -Suffix "bad" -Priority 2100000000
|
||||
$goodRoute = New-RouteIntent -SourceNodeID $entryNode.id -DestinationNodeID $exitNode.id -Suffix "good" -Priority 100
|
||||
|
||||
$lease = (Invoke-Api -Method POST -Path "/clusters/$ClusterID/fabric/service-channels/leases" -Body @{
|
||||
actor_user_id = $ActorUserID
|
||||
organization_id = "org-home"
|
||||
user_id = "user-m"
|
||||
resource_id = "$runId-vpn"
|
||||
service_class = "vpn_packets"
|
||||
entry_node_ids = @([string]$entryNode.id)
|
||||
exit_node_ids = @([string]$exitNode.id)
|
||||
preferred_entry_node_id = [string]$entryNode.id
|
||||
preferred_exit_node_id = [string]$exitNode.id
|
||||
allowed_channels = @("vpn_packet", "fabric_control")
|
||||
ttl_seconds = 120
|
||||
metadata = @{ smoke = "c18z100_rebuild_health_feedback_breakdown"; run_id = $runId }
|
||||
}).fabric_service_channel_lease
|
||||
|
||||
$heartbeatBody = @{
|
||||
health_status = "healthy"
|
||||
capabilities = @{ fabric_service_channel_access_telemetry = $true }
|
||||
service_states = @{}
|
||||
metadata = @{
|
||||
fabric_service_channel_access_report = @{
|
||||
schema_version = "c18z52.fabric_service_channel_access_report.v1"
|
||||
total = 1
|
||||
signed = 1
|
||||
backend_fallback = 0
|
||||
backend_fallback_blocked = 1
|
||||
fabric_route_send_failure = 1
|
||||
data_plane_contract = 1
|
||||
last_backend_relay_policy = "disabled"
|
||||
last_working_data_transport = "fabric_service_channel"
|
||||
last_steady_state_transport = "fabric_route"
|
||||
last_data_plane_violation_status = "fabric_route_send_failed_backend_fallback_blocked"
|
||||
last_data_plane_violation_reason = "synthetic c18z100 route send failure"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Invoke-Api -Method POST -Path "/clusters/$ClusterID/nodes/$($entryNode.id)/heartbeats" -Body $heartbeatBody | Out-Null
|
||||
|
||||
$feedback = $null
|
||||
$config = $null
|
||||
for ($i = 0; $i -lt 8; $i++) {
|
||||
Start-Sleep -Seconds 3
|
||||
$feedbackItems = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/fabric/service-channels/route-feedback?actor_user_id=$ActorUserID&reporter_node_id=$($entryNode.id)&route_id=$($lease.primary_route.route_id)&service_class=vpn_packets").route_feedback
|
||||
$feedback = @($feedbackItems | Select-Object -First 1)
|
||||
$config = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/nodes/$($entryNode.id)/mesh/synthetic-config").synthetic_mesh_config
|
||||
$replacement = @($config.route_path_decisions.decisions | Where-Object {
|
||||
[string](Get-PropertyValue -Item $_ -Name "route_id" -Default "") -eq [string]$lease.primary_route.route_id -and
|
||||
[string](Get-PropertyValue -Item $_ -Name "replacement_route_id" -Default "") -ne ""
|
||||
}) | Select-Object -First 1
|
||||
if ($null -ne $feedback -and $null -ne $replacement) { break }
|
||||
}
|
||||
|
||||
$firstFeedback = $feedback
|
||||
$firstFeedbackID = if ($null -ne $firstFeedback) { [string]$firstFeedback.id } else { "" }
|
||||
$firstObservedAt = if ($null -ne $firstFeedback) { [string]$firstFeedback.observed_at } else { "" }
|
||||
Start-Sleep -Seconds 2
|
||||
Invoke-Api -Method POST -Path "/clusters/$ClusterID/nodes/$($entryNode.id)/heartbeats" -Body $heartbeatBody | Out-Null
|
||||
Start-Sleep -Seconds 2
|
||||
$feedbackAfterDuplicateItems = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/fabric/service-channels/route-feedback?actor_user_id=$ActorUserID&reporter_node_id=$($entryNode.id)&route_id=$($lease.primary_route.route_id)&service_class=vpn_packets").route_feedback
|
||||
$feedbackAfterDuplicate = @($feedbackAfterDuplicateItems | Select-Object -First 1)
|
||||
$duplicateFeedbackCount = @($feedbackAfterDuplicateItems).Count
|
||||
$secondFeedbackID = if ($null -ne $feedbackAfterDuplicate) { [string]$feedbackAfterDuplicate.id } else { "" }
|
||||
$secondObservedAt = if ($null -ne $feedbackAfterDuplicate) { [string]$feedbackAfterDuplicate.observed_at } else { "" }
|
||||
|
||||
$replacement = @($config.route_path_decisions.decisions | Where-Object {
|
||||
[string](Get-PropertyValue -Item $_ -Name "route_id" -Default "") -eq [string]$lease.primary_route.route_id -and
|
||||
[string](Get-PropertyValue -Item $_ -Name "replacement_route_id" -Default "") -ne ""
|
||||
}) | Select-Object -First 1
|
||||
|
||||
$ledgerItems = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/fabric/service-channels/rebuild-attempts?actor_user_id=$ActorUserID&reporter_node_id=$($entryNode.id)&route_id=$($lease.primary_route.route_id)&limit=20&enrichment=summary").rebuild_attempts
|
||||
$ledgerAttempt = @($ledgerItems | Where-Object {
|
||||
[string](Get-PropertyValue -Item $_ -Name "rebuild_request_id" -Default "") -eq [string](Get-PropertyValue -Item $replacement -Name "rebuild_request_id" -Default "")
|
||||
}) | Select-Object -First 1
|
||||
$sourceFilteredLedger = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/fabric/service-channels/rebuild-attempts?actor_user_id=$ActorUserID&feedback_source=fabric_service_channel_access_report&limit=20&enrichment=summary").rebuild_attempts
|
||||
$channelFilteredLedger = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/fabric/service-channels/rebuild-attempts?actor_user_id=$ActorUserID&feedback_channel_id=$($lease.channel_id)&limit=20&enrichment=summary").rebuild_attempts
|
||||
$violationFilteredLedger = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/fabric/service-channels/rebuild-attempts?actor_user_id=$ActorUserID&feedback_violation_status=fabric_route_send_failed_backend_fallback_blocked&limit=20&enrichment=summary").rebuild_attempts
|
||||
$combinedFilteredLedger = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/fabric/service-channels/rebuild-attempts?actor_user_id=$ActorUserID&feedback_source=fabric_service_channel_access_report&feedback_channel_id=$($lease.channel_id)&feedback_violation_status=fabric_route_send_failed_backend_fallback_blocked&limit=20&enrichment=summary").rebuild_attempts
|
||||
$wrongChannelFilteredLedger = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/fabric/service-channels/rebuild-attempts?actor_user_id=$ActorUserID&feedback_channel_id=00000000-0000-0000-0000-000000000000&limit=20&enrichment=summary").rebuild_attempts
|
||||
$sourceFilteredAttempt = @($sourceFilteredLedger | Where-Object { [string](Get-PropertyValue -Item $_ -Name "rebuild_request_id" -Default "") -eq [string](Get-PropertyValue -Item $replacement -Name "rebuild_request_id" -Default "") }) | Select-Object -First 1
|
||||
$channelFilteredAttempt = @($channelFilteredLedger | Where-Object { [string](Get-PropertyValue -Item $_ -Name "rebuild_request_id" -Default "") -eq [string](Get-PropertyValue -Item $replacement -Name "rebuild_request_id" -Default "") }) | Select-Object -First 1
|
||||
$violationFilteredAttempt = @($violationFilteredLedger | Where-Object { [string](Get-PropertyValue -Item $_ -Name "rebuild_request_id" -Default "") -eq [string](Get-PropertyValue -Item $replacement -Name "rebuild_request_id" -Default "") }) | Select-Object -First 1
|
||||
$combinedFilteredAttempt = @($combinedFilteredLedger | Where-Object { [string](Get-PropertyValue -Item $_ -Name "rebuild_request_id" -Default "") -eq [string](Get-PropertyValue -Item $replacement -Name "rebuild_request_id" -Default "") }) | Select-Object -First 1
|
||||
$health = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/fabric/service-channels/rebuild-health?actor_user_id=$ActorUserID&limit=50").rebuild_health
|
||||
$feedbackBreakdown = @((Get-PropertyValue -Item $health -Name "feedback_breakdowns" -Default @()) | Where-Object {
|
||||
[string](Get-PropertyValue -Item $_ -Name "feedback_source" -Default "") -eq "fabric_service_channel_access_report" -and
|
||||
[string](Get-PropertyValue -Item $_ -Name "feedback_channel_id" -Default "") -eq [string]$lease.channel_id -and
|
||||
[string](Get-PropertyValue -Item $_ -Name "feedback_violation_status" -Default "") -eq "fabric_route_send_failed_backend_fallback_blocked"
|
||||
}) | Select-Object -First 1
|
||||
|
||||
$backendLine = (& ssh $DockerSSH "docker ps --format '{{.Names}} {{.Image}} {{.Status}}' | grep '^rap_test_backend '") | Out-String
|
||||
$nodeLines = (& ssh $DockerSSH "docker ps --format '{{.Names}} {{.Image}} {{.Status}}' | grep '^rap_test_node_test_'") | Out-String
|
||||
|
||||
$checks = [ordered]@{
|
||||
backend_expected_image_deployed = $backendLine.Contains($ExpectedBackendImage)
|
||||
node_agent_expected_image_deployed = $nodeLines.Contains($ExpectedNodeAgentImage)
|
||||
lease_selected_bad_route_first = ([string]$lease.primary_route.route_id -eq [string]$badRoute.id)
|
||||
route_feedback_recorded_from_access_report = ($null -ne $feedback)
|
||||
route_feedback_is_fenced = ($null -ne $feedback -and [string]$feedback.feedback_status -eq "fenced")
|
||||
route_feedback_contains_blocked_policy_reason = ($null -ne $feedback -and @($feedback.reasons | Where-Object { [string]$_ -eq "backend_fallback_blocked_by_policy" }).Count -gt 0)
|
||||
duplicate_heartbeat_kept_single_latest_feedback = ($duplicateFeedbackCount -eq 1)
|
||||
duplicate_heartbeat_kept_feedback_id = ($firstFeedbackID -ne "" -and $firstFeedbackID -eq $secondFeedbackID)
|
||||
duplicate_heartbeat_kept_observed_at = ($firstObservedAt -ne "" -and $firstObservedAt -eq $secondObservedAt)
|
||||
planner_selected_replacement = ($null -ne $replacement)
|
||||
planner_replacement_is_good_route = ($null -ne $replacement -and [string]$replacement.replacement_route_id -eq [string]$goodRoute.id)
|
||||
planner_rebuild_status_applied = ($null -ne $replacement -and [string]$replacement.rebuild_status -eq "applied")
|
||||
planner_decision_links_feedback_observation = ($null -ne $replacement -and [string](Get-PropertyValue -Item $replacement -Name "feedback_observation_id" -Default "") -eq $firstFeedbackID)
|
||||
planner_decision_links_access_report_source = ($null -ne $replacement -and [string](Get-PropertyValue -Item $replacement -Name "feedback_source" -Default "") -eq "fabric_service_channel_access_report")
|
||||
planner_decision_links_channel = ($null -ne $replacement -and [string](Get-PropertyValue -Item $replacement -Name "feedback_channel_id" -Default "") -eq [string]$lease.channel_id)
|
||||
planner_decision_links_violation = ($null -ne $replacement -and [string](Get-PropertyValue -Item $replacement -Name "feedback_violation_status" -Default "") -eq "fabric_route_send_failed_backend_fallback_blocked")
|
||||
rebuild_ledger_recorded_correlated_attempt = ($null -ne $ledgerAttempt)
|
||||
rebuild_ledger_links_feedback_observation = ($null -ne $ledgerAttempt -and [string](Get-PropertyValue -Item $ledgerAttempt -Name "feedback_observation_id" -Default "") -eq $firstFeedbackID)
|
||||
rebuild_ledger_links_access_report_source = ($null -ne $ledgerAttempt -and [string](Get-PropertyValue -Item $ledgerAttempt -Name "feedback_source" -Default "") -eq "fabric_service_channel_access_report")
|
||||
rebuild_ledger_payload_links_feedback = ($null -ne $ledgerAttempt -and [string](Get-PropertyValue -Item (Get-PropertyValue -Item $ledgerAttempt -Name "payload" -Default $null) -Name "feedback_observation_id" -Default "") -eq $firstFeedbackID)
|
||||
filter_by_feedback_source_returns_attempt = ($null -ne $sourceFilteredAttempt)
|
||||
filter_by_feedback_channel_returns_attempt = ($null -ne $channelFilteredAttempt)
|
||||
filter_by_feedback_violation_returns_attempt = ($null -ne $violationFilteredAttempt)
|
||||
combined_feedback_filters_return_attempt = ($null -ne $combinedFilteredAttempt)
|
||||
wrong_channel_filter_excludes_attempt = (@($wrongChannelFilteredLedger | Where-Object { [string](Get-PropertyValue -Item $_ -Name "rebuild_request_id" -Default "") -eq [string](Get-PropertyValue -Item $replacement -Name "rebuild_request_id" -Default "") }).Count -eq 0)
|
||||
rebuild_health_returns_feedback_breakdown = ($null -ne $feedbackBreakdown)
|
||||
feedback_breakdown_counts_attempt = ($null -ne $feedbackBreakdown -and [int](Get-PropertyValue -Item $feedbackBreakdown -Name "total_count" -Default 0) -ge 1)
|
||||
feedback_breakdown_lists_reporter = ($null -ne $feedbackBreakdown -and @((Get-PropertyValue -Item $feedbackBreakdown -Name "affected_reporter_node_ids" -Default @()) | Where-Object { [string]$_ -eq [string]$entryNode.id }).Count -gt 0)
|
||||
feedback_breakdown_lists_route = ($null -ne $feedbackBreakdown -and @((Get-PropertyValue -Item $feedbackBreakdown -Name "affected_route_ids" -Default @()) | Where-Object { [string]$_ -eq [string]$lease.primary_route.route_id }).Count -gt 0)
|
||||
}
|
||||
$failed = @($checks.GetEnumerator() | Where-Object { -not $_.Value } | ForEach-Object { $_.Key })
|
||||
|
||||
$result = [ordered]@{
|
||||
schema_version = "c18z100.rebuild_health_feedback_breakdown_smoke.v1"
|
||||
run_id = $runId
|
||||
cluster_id = $ClusterID
|
||||
channel_id = [string]$lease.channel_id
|
||||
primary_route_id = [string]$lease.primary_route.route_id
|
||||
replacement_route_id = if ($null -ne $replacement) { [string]$replacement.replacement_route_id } else { "" }
|
||||
passed = ($failed.Count -eq 0)
|
||||
checks = $checks
|
||||
failed_checks = $failed
|
||||
summary = [ordered]@{
|
||||
backend_container = $backendLine.Trim()
|
||||
node_containers = $nodeLines.Trim()
|
||||
bad_route = $badRoute
|
||||
good_route = $goodRoute
|
||||
lease = $lease
|
||||
feedback = $feedback
|
||||
feedback_after_duplicate = $feedbackAfterDuplicate
|
||||
replacement = $replacement
|
||||
ledger_attempt = $ledgerAttempt
|
||||
source_filtered_attempt = $sourceFilteredAttempt
|
||||
channel_filtered_attempt = $channelFilteredAttempt
|
||||
violation_filtered_attempt = $violationFilteredAttempt
|
||||
combined_filtered_attempt = $combinedFilteredAttempt
|
||||
wrong_channel_filtered_count = @($wrongChannelFilteredLedger).Count
|
||||
rebuild_health = $health
|
||||
feedback_breakdown = $feedbackBreakdown
|
||||
}
|
||||
}
|
||||
|
||||
$target = Join-Path $repoRoot $ResultPath
|
||||
$result | ConvertTo-Json -Depth 80 | Set-Content -Path $target -Encoding UTF8
|
||||
|
||||
try {
|
||||
foreach ($routeID in @([string]$badRoute.id, [string]$goodRoute.id)) {
|
||||
if ($routeID) {
|
||||
Invoke-Api -Method POST -Path "/clusters/$ClusterID/mesh/route-intents/$routeID/expire" -Body @{ actor_user_id = $ActorUserID } | Out-Null
|
||||
}
|
||||
}
|
||||
Invoke-Api -Method POST -Path "/clusters/$ClusterID/fabric/service-channels/leases/cleanup" -Body @{ actor_user_id = $ActorUserID; limit = 100 } | Out-Null
|
||||
} catch {
|
||||
Write-Warning "cleanup failed after c18z100 smoke: $($_.Exception.Message)"
|
||||
}
|
||||
|
||||
if (-not $result.passed) {
|
||||
throw "C18Z100 rebuild health feedback breakdown smoke failed: $($failed -join ', ')"
|
||||
}
|
||||
|
||||
Write-Host "C18Z100 rebuild health feedback breakdown smoke passed. Result: $target"
|
||||
$result
|
||||
Reference in New Issue
Block a user