Record project continuation changes
This commit is contained in:
@@ -0,0 +1,598 @@
|
||||
param(
|
||||
[string]$ApiBaseUrl = "http://192.168.200.61:18121/api/v1",
|
||||
[string]$ClusterID = "cfc0743d-d960-49fb-9de8-96e063d5e4aa",
|
||||
[string]$ActorUserID = "f67d943f-5397-4b3a-a229-695fe67ad700",
|
||||
[string]$EntryNodeName = "test-1",
|
||||
[string]$RelayNodeName = "test-3",
|
||||
[string]$ExitNodeName = "test-2",
|
||||
[string]$EntryBaseUrl = "http://192.168.200.61:19131",
|
||||
[string]$DockerSSH = "test-docker",
|
||||
[string]$ExpectedBackendImage = "rap-backend:fabric-service-channel-0.2.281-c18z109",
|
||||
[string]$ExpectedNodeAgentImage = "rap-node-agent:0.2.270-c18z95",
|
||||
[string]$ResultPath = "artifacts\c18z81-service-channel-replacement-degradation-recovery-smoke-result.json"
|
||||
)
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
$scriptDir = Split-Path -Parent $MyInvocation.MyCommand.Path
|
||||
$repoRoot = (Resolve-Path (Join-Path $scriptDir "..\..")).ProviderPath
|
||||
$runId = "c18z81-" + (Get-Date -Format "yyyyMMdd-HHmmss")
|
||||
|
||||
function Invoke-Api {
|
||||
param([string]$Method, [string]$Path, [object]$Body = $null)
|
||||
$params = @{ Method = $Method; Uri = "$ApiBaseUrl$Path"; TimeoutSec = 30 }
|
||||
if ($null -ne $Body) {
|
||||
$params.ContentType = "application/json"
|
||||
$params.Body = ($Body | ConvertTo-Json -Depth 80)
|
||||
}
|
||||
return Invoke-RestMethod @params
|
||||
}
|
||||
|
||||
function Get-PropertyValue {
|
||||
param([object]$Item, [string]$Name, [object]$Default = $null)
|
||||
if ($null -eq $Item) { return $Default }
|
||||
$property = $Item.PSObject.Properties[$Name]
|
||||
if ($null -eq $property) { return $Default }
|
||||
return $property.Value
|
||||
}
|
||||
|
||||
function Get-NodeByName {
|
||||
param([string]$Name)
|
||||
$nodes = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/nodes?actor_user_id=$ActorUserID").nodes
|
||||
$node = @($nodes | Where-Object { $_.name -eq $Name }) | Select-Object -First 1
|
||||
if ($null -eq $node) { throw "Node '$Name' was not found" }
|
||||
return $node
|
||||
}
|
||||
|
||||
function New-IPv4TcpPacket {
|
||||
param(
|
||||
[byte[]]$Source = @(10, 77, 0, 2),
|
||||
[byte[]]$Destination = @(192, 168, 200, 95),
|
||||
[int]$SourcePort,
|
||||
[int]$DestinationPort = 3389
|
||||
)
|
||||
$packet = [byte[]]::new(40)
|
||||
$packet[0] = 0x45
|
||||
$packet[2] = 0
|
||||
$packet[3] = 40
|
||||
$packet[8] = 64
|
||||
$packet[9] = 6
|
||||
[Array]::Copy($Source, 0, $packet, 12, 4)
|
||||
[Array]::Copy($Destination, 0, $packet, 16, 4)
|
||||
$packet[20] = [byte](($SourcePort -shr 8) -band 0xff)
|
||||
$packet[21] = [byte]($SourcePort -band 0xff)
|
||||
$packet[22] = [byte](($DestinationPort -shr 8) -band 0xff)
|
||||
$packet[23] = [byte]($DestinationPort -band 0xff)
|
||||
$packet[32] = 0x50
|
||||
return $packet
|
||||
}
|
||||
|
||||
function ConvertTo-VPNPacketBatch {
|
||||
param([byte[][]]$Packets)
|
||||
$buffer = New-Object System.Collections.Generic.List[byte]
|
||||
foreach ($packet in $Packets) {
|
||||
if ($null -eq $packet -or $packet.Length -eq 0) { continue }
|
||||
$size = [uint32]$packet.Length
|
||||
$buffer.Add([byte](($size -shr 24) -band 0xff))
|
||||
$buffer.Add([byte](($size -shr 16) -band 0xff))
|
||||
$buffer.Add([byte](($size -shr 8) -band 0xff))
|
||||
$buffer.Add([byte]($size -band 0xff))
|
||||
foreach ($value in $packet) {
|
||||
$buffer.Add($value)
|
||||
}
|
||||
}
|
||||
return $buffer.ToArray()
|
||||
}
|
||||
|
||||
function New-RouteIntent {
|
||||
param([string]$SourceNodeID, [string]$DestinationNodeID, [string[]]$Hops)
|
||||
$expiresAt = (Get-Date).ToUniversalTime().AddMinutes(5).ToString("o")
|
||||
return (Invoke-Api -Method POST -Path "/clusters/$ClusterID/mesh/route-intents" -Body @{
|
||||
actor_user_id = $ActorUserID
|
||||
source_selector = @{ node_id = $SourceNodeID }
|
||||
destination_selector = @{ node_id = $DestinationNodeID }
|
||||
service_class = "vpn_packets"
|
||||
priority = 2100000000
|
||||
policy = @{
|
||||
synthetic_enabled = $true
|
||||
route_version = "$runId-primary"
|
||||
policy_version = "$runId-primary"
|
||||
peer_directory_version = "$runId-primary"
|
||||
hops = @($Hops)
|
||||
allowed_channels = @("vpn_packet", "fabric_control")
|
||||
max_ttl = 8
|
||||
max_hops = 8
|
||||
expires_at = $expiresAt
|
||||
metadata = @{ smoke = "c18z81_service_channel_replacement_degradation_recovery"; run_id = $runId }
|
||||
}
|
||||
}).route_intent
|
||||
}
|
||||
|
||||
function Disable-ExistingRouteIntents {
|
||||
param([string]$SourceNodeID, [string]$DestinationNodeID)
|
||||
$items = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/mesh/route-intents?actor_user_id=$ActorUserID").route_intents
|
||||
foreach ($item in @($items)) {
|
||||
if ([string](Get-PropertyValue -Item $item -Name "status" -Default "") -ne "active") { continue }
|
||||
if ([string](Get-PropertyValue -Item $item -Name "service_class" -Default "") -ne "vpn_packets") { continue }
|
||||
$sourceSelector = Get-PropertyValue -Item $item -Name "source_selector" -Default $null
|
||||
$destinationSelector = Get-PropertyValue -Item $item -Name "destination_selector" -Default $null
|
||||
if ([string](Get-PropertyValue -Item $sourceSelector -Name "node_id" -Default "") -ne $SourceNodeID) { continue }
|
||||
if ([string](Get-PropertyValue -Item $destinationSelector -Name "node_id" -Default "") -ne $DestinationNodeID) { continue }
|
||||
[void](Invoke-Api -Method POST -Path "/clusters/$ClusterID/mesh/route-intents/$($item.id)/disable" -Body @{
|
||||
actor_user_id = $ActorUserID
|
||||
reason = "c18z81 isolate replacement-degradation-recovery smoke route pair"
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
function Send-DegradedHeartbeat {
|
||||
param([string]$EntryNodeID, [string]$PrimaryRouteID)
|
||||
$observedAt = (Get-Date).ToUniversalTime().ToString("o")
|
||||
return Invoke-Api -Method POST -Path "/clusters/$ClusterID/nodes/$EntryNodeID/heartbeats" -Body @{
|
||||
health_status = "healthy"
|
||||
reported_version = "0.2.255-c18z81"
|
||||
capabilities = @{
|
||||
fabric_service_channel_runtime = $true
|
||||
fabric_service_channel_route_quality_feedback = $true
|
||||
smoke_feedback_injection = "c18z77"
|
||||
}
|
||||
service_states = @{ smoke = "c18z81_primary_degraded_alternate_after_lease" }
|
||||
metadata = @{
|
||||
fabric_service_channel_runtime_report = @{
|
||||
schema_version = "c18l.fabric_service_channel_runtime_report.v1"
|
||||
config_version = "$runId-primary"
|
||||
cluster_id = $ClusterID
|
||||
local_node_id = $EntryNodeID
|
||||
observed_at = $observedAt
|
||||
ingress = @{
|
||||
flow_scheduler = @{
|
||||
schema_version = "rap.fabric_flow_scheduler.v1"
|
||||
service_neutral = $true
|
||||
service_mode = "application_protocol_agnostic"
|
||||
channel_stats = @{
|
||||
"c18z81-primary-degraded" = @{
|
||||
last_route_id = $PrimaryRouteID
|
||||
last_failed_route_id = $PrimaryRouteID
|
||||
route_generation = "$runId-primary"
|
||||
last_error = "c18z81 primary route degraded; alternate added after lease"
|
||||
last_send_duration_ms = 1200
|
||||
consecutive_failures = 3
|
||||
stall_count = 2
|
||||
route_rebuild_recommended = $true
|
||||
degraded_fallback_recommended = $false
|
||||
quality_window_sample_count = 8
|
||||
quality_window_success_count = 2
|
||||
quality_window_failure_count = 3
|
||||
quality_window_slow_count = 2
|
||||
quality_window_drop_count = 0
|
||||
quality_window_avg_latency_ms = 1200
|
||||
quality_window_last_updated_at = $observedAt
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
smoke = @{ name = "c18z81_service_channel_replacement_degradation_recovery"; run_id = $runId }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function Send-ReplacementDegradedHeartbeat {
|
||||
param([string]$EntryNodeID, [string]$ReplacementRouteID, [string]$RouteGeneration)
|
||||
$observedAt = (Get-Date).ToUniversalTime().ToString("o")
|
||||
return Invoke-Api -Method POST -Path "/clusters/$ClusterID/nodes/$EntryNodeID/heartbeats" -Body @{
|
||||
health_status = "healthy"
|
||||
reported_version = "0.2.255-c18z81"
|
||||
capabilities = @{
|
||||
fabric_service_channel_runtime = $true
|
||||
fabric_service_channel_route_quality_feedback = $true
|
||||
smoke_feedback_injection = "c18z81"
|
||||
}
|
||||
service_states = @{ smoke = "c18z81_replacement_route_degraded_after_apply" }
|
||||
metadata = @{
|
||||
fabric_service_channel_runtime_report = @{
|
||||
schema_version = "c18l.fabric_service_channel_runtime_report.v1"
|
||||
config_version = "$runId-replacement-degraded"
|
||||
cluster_id = $ClusterID
|
||||
local_node_id = $EntryNodeID
|
||||
observed_at = $observedAt
|
||||
ingress = @{
|
||||
last_selected_route_id = $ReplacementRouteID
|
||||
send_route_failures = 4
|
||||
flow_scheduler = @{
|
||||
schema_version = "rap.fabric_flow_scheduler.v1"
|
||||
service_neutral = $true
|
||||
service_mode = "application_protocol_agnostic"
|
||||
channel_stats = @{
|
||||
"c18z81-replacement-degraded" = @{
|
||||
last_route_id = $ReplacementRouteID
|
||||
last_failed_route_id = $ReplacementRouteID
|
||||
route_generation = $RouteGeneration
|
||||
last_error = "c18z81 replacement route degraded; recovery route required"
|
||||
last_send_duration_ms = 1500
|
||||
consecutive_failures = 4
|
||||
stall_count = 2
|
||||
route_rebuild_recommended = $true
|
||||
degraded_fallback_recommended = $false
|
||||
quality_window_sample_count = 10
|
||||
quality_window_success_count = 2
|
||||
quality_window_failure_count = 4
|
||||
quality_window_slow_count = 3
|
||||
quality_window_drop_count = 0
|
||||
quality_window_avg_latency_ms = 1500
|
||||
quality_window_last_updated_at = $observedAt
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
smoke = @{ name = "c18z81_service_channel_replacement_degradation_recovery"; run_id = $runId }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function Get-LatestIngressSnapshot {
|
||||
param([string]$EntryNodeID, [string]$ReplacementRouteID)
|
||||
$heartbeats = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/nodes/$EntryNodeID/heartbeats?actor_user_id=$ActorUserID&limit=5").heartbeats
|
||||
foreach ($heartbeat in @($heartbeats)) {
|
||||
$runtimeReport = Get-PropertyValue -Item (Get-PropertyValue -Item $heartbeat -Name "metadata" -Default $null) -Name "fabric_service_channel_runtime_report" -Default $null
|
||||
$ingress = Get-PropertyValue -Item $runtimeReport -Name "ingress" -Default $null
|
||||
if ($null -eq $ingress) { continue }
|
||||
$flowScheduler = Get-PropertyValue -Item $ingress -Name "flow_scheduler" -Default $null
|
||||
$channelStats = Get-PropertyValue -Item $flowScheduler -Name "channel_stats" -Default $null
|
||||
$replacementFlowStats = @()
|
||||
if ($null -ne $channelStats) {
|
||||
foreach ($property in @($channelStats.PSObject.Properties)) {
|
||||
$stat = $property.Value
|
||||
if ([string](Get-PropertyValue -Item $stat -Name "last_route_id" -Default "") -eq $ReplacementRouteID) {
|
||||
$replacementFlowStats += $stat
|
||||
}
|
||||
}
|
||||
}
|
||||
return [pscustomobject]@{
|
||||
observed_at = [string](Get-PropertyValue -Item $runtimeReport -Name "observed_at" -Default "")
|
||||
last_selected_route_id = [string](Get-PropertyValue -Item $ingress -Name "last_selected_route_id" -Default "")
|
||||
send_fallback_local = [int](Get-PropertyValue -Item $ingress -Name "send_fallback_local" -Default 0)
|
||||
send_route_failures = [int](Get-PropertyValue -Item $ingress -Name "send_route_failures" -Default 0)
|
||||
send_flow_dropped = [int](Get-PropertyValue -Item $ingress -Name "send_flow_dropped" -Default 0)
|
||||
scheduler_dropped = [int](Get-PropertyValue -Item $flowScheduler -Name "dropped" -Default 0)
|
||||
flow_high_watermark = [int](Get-PropertyValue -Item $flowScheduler -Name "high_watermark" -Default 0)
|
||||
flow_max_in_flight = [int](Get-PropertyValue -Item $flowScheduler -Name "max_in_flight" -Default 0)
|
||||
replacement_flow_stat_count = $replacementFlowStats.Count
|
||||
replacement_flow_stats = $replacementFlowStats
|
||||
}
|
||||
}
|
||||
return $null
|
||||
}
|
||||
|
||||
$entryNode = Get-NodeByName -Name $EntryNodeName
|
||||
$relayNode = Get-NodeByName -Name $RelayNodeName
|
||||
$exitNode = Get-NodeByName -Name $ExitNodeName
|
||||
[void](Disable-ExistingRouteIntents -SourceNodeID $entryNode.id -DestinationNodeID $exitNode.id)
|
||||
$route = New-RouteIntent -SourceNodeID $entryNode.id -DestinationNodeID $exitNode.id -Hops @($entryNode.id, $exitNode.id)
|
||||
$lease = (Invoke-Api -Method POST -Path "/clusters/$ClusterID/fabric/service-channels/leases" -Body @{
|
||||
actor_user_id = $ActorUserID
|
||||
organization_id = "smoke-org"
|
||||
user_id = "smoke-user"
|
||||
resource_id = "c18z81-vpn-smoke"
|
||||
service_class = "vpn_packets"
|
||||
entry_node_ids = @([string]$entryNode.id)
|
||||
exit_node_ids = @([string]$exitNode.id)
|
||||
preferred_entry_node_id = [string]$entryNode.id
|
||||
preferred_exit_node_id = [string]$exitNode.id
|
||||
allowed_channels = @("vpn_packet", "fabric_control")
|
||||
ttl_seconds = 240
|
||||
metadata = @{ smoke = "c18z81_service_channel_replacement_degradation_recovery"; run_id = $runId }
|
||||
}).fabric_service_channel_lease
|
||||
$primaryRouteID = [string](Get-PropertyValue -Item (Get-PropertyValue -Item $lease -Name "primary_route" -Default $null) -Name "route_id" -Default $route.id)
|
||||
|
||||
[void](Send-DegradedHeartbeat -EntryNodeID $entryNode.id -PrimaryRouteID $primaryRouteID)
|
||||
|
||||
$matchingChannel = $null
|
||||
for ($i = 0; $i -lt 10; $i++) {
|
||||
Start-Sleep -Seconds 3
|
||||
$accessTelemetry = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/fabric/service-channels/access-telemetry?actor_user_id=$ActorUserID&limit=50").fabric_service_channel_access_telemetry
|
||||
$matchingChannel = @($accessTelemetry.active_channels | Where-Object { $_.channel_id -eq $lease.channel_id }) | Select-Object -First 1
|
||||
if ($null -ne $matchingChannel -and [string](Get-PropertyValue -Item $matchingChannel -Name "remediation_action" -Default "") -eq "rebuild_route") {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
$command = Get-PropertyValue -Item $matchingChannel -Name "remediation_command" -Default $null
|
||||
$commandID = [string](Get-PropertyValue -Item $command -Name "command_id" -Default "")
|
||||
$alternate = New-RouteIntent -SourceNodeID $entryNode.id -DestinationNodeID $exitNode.id -Hops @($entryNode.id, $relayNode.id, $exitNode.id)
|
||||
$alternateRouteID = [string]$alternate.id
|
||||
$recoveryRouteID = ""
|
||||
[void](Invoke-Api -Method GET -Path "/clusters/$ClusterID/nodes/$($entryNode.id)/mesh/synthetic-config")
|
||||
|
||||
$attempts = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/fabric/service-channels/rebuild-attempts?actor_user_id=$ActorUserID&reporter_node_id=$($entryNode.id)&rebuild_request_id=$commandID&limit=10").rebuild_attempts
|
||||
$attempt = @($attempts | Where-Object { $_.rebuild_request_id -eq $commandID }) | Select-Object -First 1
|
||||
|
||||
$routeManagerDecision = $null
|
||||
$routeManagerTransition = $null
|
||||
for ($i = 0; $i -lt 12; $i++) {
|
||||
Start-Sleep -Seconds 5
|
||||
$heartbeats = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/nodes/$($entryNode.id)/heartbeats?actor_user_id=$ActorUserID&limit=5").heartbeats
|
||||
foreach ($heartbeat in @($heartbeats)) {
|
||||
$runtimeReport = Get-PropertyValue -Item (Get-PropertyValue -Item $heartbeat -Name "metadata" -Default $null) -Name "fabric_service_channel_runtime_report" -Default $null
|
||||
$ingress = Get-PropertyValue -Item $runtimeReport -Name "ingress" -Default $null
|
||||
$routeManager = Get-PropertyValue -Item $ingress -Name "route_manager" -Default $null
|
||||
$routeManagerTransition = Get-PropertyValue -Item $ingress -Name "route_manager_transition" -Default $null
|
||||
$decisions = @()
|
||||
if ($null -ne $routeManager -and $routeManager.PSObject.Properties.Name -contains "decisions") {
|
||||
$decisions = @($routeManager.decisions)
|
||||
}
|
||||
$routeManagerDecision = $decisions | Where-Object {
|
||||
[string](Get-PropertyValue -Item $_ -Name "rebuild_request_id" -Default "") -eq $commandID -and
|
||||
[string](Get-PropertyValue -Item $_ -Name "route_id" -Default "") -eq $primaryRouteID -and
|
||||
[string](Get-PropertyValue -Item $_ -Name "replacement_route_id" -Default "") -eq $alternateRouteID
|
||||
} | Select-Object -First 1
|
||||
if ($null -ne $routeManagerDecision) {
|
||||
break
|
||||
}
|
||||
}
|
||||
if ($null -ne $routeManagerDecision) {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
$packetPath = $lease.entry_http.path_template.
|
||||
Replace("{cluster_id}", $ClusterID).
|
||||
Replace("{channel_id}", [string]$lease.channel_id).
|
||||
Replace("{resource_id}", "c18z81-vpn-smoke")
|
||||
$packetUrl = $EntryBaseUrl.TrimEnd("/") + $packetPath
|
||||
$baseHeaders = @{
|
||||
"X-RAP-Service-Channel-Token" = [string]$lease.token.token
|
||||
"X-RAP-Fabric-Channel-ID" = [string]$lease.channel_id
|
||||
"X-RAP-Service-Class" = "vpn_packets"
|
||||
"X-RAP-Channel-Class" = "vpn_packet"
|
||||
}
|
||||
$baselineSnapshot = $null
|
||||
for ($i = 0; $i -lt 10; $i++) {
|
||||
Start-Sleep -Seconds 2
|
||||
$baselineSnapshot = Get-LatestIngressSnapshot -EntryNodeID $entryNode.id -ReplacementRouteID $alternateRouteID
|
||||
if ($null -ne $baselineSnapshot) { break }
|
||||
}
|
||||
|
||||
$burstCount = 5
|
||||
$burstPacketCount = 32
|
||||
$trafficClasses = @("interactive", "bulk", "reliable", "interactive", "bulk")
|
||||
$burstResults = @()
|
||||
for ($burstIndex = 0; $burstIndex -lt $burstCount; $burstIndex++) {
|
||||
$trafficPackets = @()
|
||||
foreach ($offset in 0..($burstPacketCount - 1)) {
|
||||
$trafficPackets += ,(New-IPv4TcpPacket -SourcePort (54000 + ($burstIndex * 100) + $offset) -DestinationPort 3389)
|
||||
}
|
||||
$trafficPayload = ConvertTo-VPNPacketBatch -Packets $trafficPackets
|
||||
$trafficPayloadPath = Join-Path ([System.IO.Path]::GetTempPath()) "$runId-vpn-packet-batch-$burstIndex.bin"
|
||||
[System.IO.File]::WriteAllBytes($trafficPayloadPath, [byte[]]$trafficPayload)
|
||||
$headers = @{} + $baseHeaders
|
||||
$headers["X-RAP-Traffic-Class"] = $trafficClasses[$burstIndex]
|
||||
$trafficStarted = Get-Date
|
||||
$trafficResponse = Invoke-WebRequest -Method Post -Uri ($packetUrl + "?batch=true") -Headers $headers -InFile $trafficPayloadPath -ContentType "application/vnd.rap.vpn-packet-batch.v1" -TimeoutSec 30
|
||||
$trafficDurationMs = [int]((Get-Date) - $trafficStarted).TotalMilliseconds
|
||||
$trafficAcceptedBy = [string]$trafficResponse.Headers["X-RAP-Service-Channel-Accepted-By"]
|
||||
Remove-Item -Path $trafficPayloadPath -Force -ErrorAction SilentlyContinue
|
||||
|
||||
$snapshot = $null
|
||||
for ($i = 0; $i -lt 8; $i++) {
|
||||
Start-Sleep -Seconds 3
|
||||
$snapshot = Get-LatestIngressSnapshot -EntryNodeID $entryNode.id -ReplacementRouteID $alternateRouteID
|
||||
if ($null -ne $snapshot -and $snapshot.last_selected_route_id -eq $alternateRouteID -and $snapshot.replacement_flow_stat_count -ge 1) {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
$burstResults += [pscustomobject]@{
|
||||
burst_index = $burstIndex + 1
|
||||
traffic_class = $trafficClasses[$burstIndex]
|
||||
packet_count = $trafficPackets.Count
|
||||
status_code = [int]$trafficResponse.StatusCode
|
||||
accepted_by = $trafficAcceptedBy
|
||||
duration_ms = $trafficDurationMs
|
||||
snapshot = $snapshot
|
||||
}
|
||||
}
|
||||
|
||||
$finalSnapshot = Get-LatestIngressSnapshot -EntryNodeID $entryNode.id -ReplacementRouteID $alternateRouteID
|
||||
$acceptedBurstCount = @($burstResults | Where-Object { $_.status_code -eq 202 -and ($_.accepted_by -eq "introspection" -or $_.accepted_by -eq "signed") }).Count
|
||||
$replacementBurstCount = @($burstResults | Where-Object { $null -ne $_.snapshot -and $_.snapshot.last_selected_route_id -eq $alternateRouteID -and $_.snapshot.replacement_flow_stat_count -ge 1 }).Count
|
||||
$stalePrimaryReselections = @($burstResults | Where-Object { $null -ne $_.snapshot -and $_.snapshot.last_selected_route_id -eq $primaryRouteID }).Count
|
||||
$baselineFallbackLocal = if ($null -ne $baselineSnapshot) { [int]$baselineSnapshot.send_fallback_local } else { 0 }
|
||||
$baselineRouteFailures = if ($null -ne $baselineSnapshot) { [int]$baselineSnapshot.send_route_failures } else { 0 }
|
||||
$baselineFlowDropped = if ($null -ne $baselineSnapshot) { [int]$baselineSnapshot.send_flow_dropped } else { 0 }
|
||||
$baselineSchedulerDropped = if ($null -ne $baselineSnapshot) { [int]$baselineSnapshot.scheduler_dropped } else { 0 }
|
||||
$finalFallbackLocal = if ($null -ne $finalSnapshot) { [int]$finalSnapshot.send_fallback_local } else { 0 }
|
||||
$finalRouteFailures = if ($null -ne $finalSnapshot) { [int]$finalSnapshot.send_route_failures } else { 0 }
|
||||
$finalFlowDropped = if ($null -ne $finalSnapshot) { [int]$finalSnapshot.send_flow_dropped } else { 0 }
|
||||
$finalSchedulerDropped = if ($null -ne $finalSnapshot) { [int]$finalSnapshot.scheduler_dropped } else { 0 }
|
||||
|
||||
[void](Send-ReplacementDegradedHeartbeat -EntryNodeID $entryNode.id -ReplacementRouteID $alternateRouteID -RouteGeneration "$runId-primary")
|
||||
$recovery = New-RouteIntent -SourceNodeID $entryNode.id -DestinationNodeID $exitNode.id -Hops @($entryNode.id, $relayNode.id, $exitNode.id)
|
||||
$recoveryRouteID = [string]$recovery.id
|
||||
$recoverySyntheticConfig = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/nodes/$($entryNode.id)/mesh/synthetic-config").synthetic_mesh_config
|
||||
$recoveryPathDecisions = @()
|
||||
if ($recoverySyntheticConfig.PSObject.Properties.Name -contains "route_path_decisions") {
|
||||
$recoveryPathDecisions = @($recoverySyntheticConfig.route_path_decisions.decisions)
|
||||
}
|
||||
$recoveryPlannerDecision = $recoveryPathDecisions | Where-Object {
|
||||
[string](Get-PropertyValue -Item $_ -Name "route_id" -Default "") -eq $alternateRouteID -and
|
||||
[string](Get-PropertyValue -Item $_ -Name "replacement_route_id" -Default "") -eq $recoveryRouteID
|
||||
} | Select-Object -First 1
|
||||
|
||||
$recoveryRouteManagerDecision = $null
|
||||
$recoveryRouteManagerTransition = $null
|
||||
for ($i = 0; $i -lt 12; $i++) {
|
||||
Start-Sleep -Seconds 5
|
||||
$heartbeats = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/nodes/$($entryNode.id)/heartbeats?actor_user_id=$ActorUserID&limit=5").heartbeats
|
||||
foreach ($heartbeat in @($heartbeats)) {
|
||||
$runtimeReport = Get-PropertyValue -Item (Get-PropertyValue -Item $heartbeat -Name "metadata" -Default $null) -Name "fabric_service_channel_runtime_report" -Default $null
|
||||
$ingress = Get-PropertyValue -Item $runtimeReport -Name "ingress" -Default $null
|
||||
$routeManager = Get-PropertyValue -Item $ingress -Name "route_manager" -Default $null
|
||||
$recoveryRouteManagerTransition = Get-PropertyValue -Item $ingress -Name "route_manager_transition" -Default $null
|
||||
$decisions = @()
|
||||
if ($null -ne $routeManager -and $routeManager.PSObject.Properties.Name -contains "decisions") {
|
||||
$decisions = @($routeManager.decisions)
|
||||
}
|
||||
$recoveryRouteManagerDecision = $decisions | Where-Object {
|
||||
[string](Get-PropertyValue -Item $_ -Name "route_id" -Default "") -eq $alternateRouteID -and
|
||||
[string](Get-PropertyValue -Item $_ -Name "replacement_route_id" -Default "") -eq $recoveryRouteID
|
||||
} | Select-Object -First 1
|
||||
if ($null -ne $recoveryRouteManagerDecision) { break }
|
||||
}
|
||||
if ($null -ne $recoveryRouteManagerDecision) { break }
|
||||
}
|
||||
|
||||
$recoveryBaselineSnapshot = Get-LatestIngressSnapshot -EntryNodeID $entryNode.id -ReplacementRouteID $recoveryRouteID
|
||||
$recoveryPackets = @()
|
||||
foreach ($offset in 0..31) {
|
||||
$recoveryPackets += ,(New-IPv4TcpPacket -SourcePort (56000 + $offset) -DestinationPort 3389)
|
||||
}
|
||||
$recoveryPayload = ConvertTo-VPNPacketBatch -Packets $recoveryPackets
|
||||
$recoveryPayloadPath = Join-Path ([System.IO.Path]::GetTempPath()) "$runId-recovery-vpn-packet-batch.bin"
|
||||
[System.IO.File]::WriteAllBytes($recoveryPayloadPath, [byte[]]$recoveryPayload)
|
||||
$recoveryHeaders = @{} + $baseHeaders
|
||||
$recoveryHeaders["X-RAP-Traffic-Class"] = "interactive"
|
||||
$recoveryStarted = Get-Date
|
||||
$recoveryResponse = Invoke-WebRequest -Method Post -Uri ($packetUrl + "?batch=true") -Headers $recoveryHeaders -InFile $recoveryPayloadPath -ContentType "application/vnd.rap.vpn-packet-batch.v1" -TimeoutSec 30
|
||||
$recoveryDurationMs = [int]((Get-Date) - $recoveryStarted).TotalMilliseconds
|
||||
$recoveryAcceptedBy = [string]$recoveryResponse.Headers["X-RAP-Service-Channel-Accepted-By"]
|
||||
Remove-Item -Path $recoveryPayloadPath -Force -ErrorAction SilentlyContinue
|
||||
|
||||
$recoveryTrafficObserved = $false
|
||||
$recoveryFinalSnapshot = $null
|
||||
for ($i = 0; $i -lt 10; $i++) {
|
||||
Start-Sleep -Seconds 3
|
||||
$recoveryFinalSnapshot = Get-LatestIngressSnapshot -EntryNodeID $entryNode.id -ReplacementRouteID $recoveryRouteID
|
||||
if ($null -ne $recoveryFinalSnapshot -and $recoveryFinalSnapshot.last_selected_route_id -eq $recoveryRouteID -and $recoveryFinalSnapshot.replacement_flow_stat_count -ge 1) {
|
||||
$recoveryTrafficObserved = $true
|
||||
break
|
||||
}
|
||||
}
|
||||
$recoveryBaselineRouteFailures = if ($null -ne $recoveryBaselineSnapshot) { [int]$recoveryBaselineSnapshot.send_route_failures } else { 0 }
|
||||
$recoveryFinalRouteFailures = if ($null -ne $recoveryFinalSnapshot) { [int]$recoveryFinalSnapshot.send_route_failures } else { 0 }
|
||||
$recoveryBaselineFallbackLocal = if ($null -ne $recoveryBaselineSnapshot) { [int]$recoveryBaselineSnapshot.send_fallback_local } else { 0 }
|
||||
$recoveryFinalFallbackLocal = if ($null -ne $recoveryFinalSnapshot) { [int]$recoveryFinalSnapshot.send_fallback_local } else { 0 }
|
||||
$recoveryBaselineFlowDropped = if ($null -ne $recoveryBaselineSnapshot) { [int]$recoveryBaselineSnapshot.send_flow_dropped } else { 0 }
|
||||
$recoveryFinalFlowDropped = if ($null -ne $recoveryFinalSnapshot) { [int]$recoveryFinalSnapshot.send_flow_dropped } else { 0 }
|
||||
$recoveryBaselineSchedulerDropped = if ($null -ne $recoveryBaselineSnapshot) { [int]$recoveryBaselineSnapshot.scheduler_dropped } else { 0 }
|
||||
$recoveryFinalSchedulerDropped = if ($null -ne $recoveryFinalSnapshot) { [int]$recoveryFinalSnapshot.scheduler_dropped } else { 0 }
|
||||
|
||||
$postAccess = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/fabric/service-channels/access-telemetry?actor_user_id=$ActorUserID&limit=50").fabric_service_channel_access_telemetry
|
||||
$postChannel = @($postAccess.active_channels | Where-Object { $_.channel_id -eq $lease.channel_id }) | Select-Object -First 1
|
||||
|
||||
$backendLine = (& ssh $DockerSSH "docker ps --format '{{.Names}} {{.Image}} {{.Status}}' | grep '^rap_test_backend '") | Out-String
|
||||
$nodeLines = (& ssh $DockerSSH "docker ps --format '{{.Names}} {{.Image}} {{.Status}}' | grep '^rap_test_node_test_'") | Out-String
|
||||
$checks = [ordered]@{
|
||||
backend_expected_image_deployed = $backendLine.Contains($ExpectedBackendImage)
|
||||
node_agent_expected_image_deployed = $nodeLines.Contains($ExpectedNodeAgentImage)
|
||||
lease_ready = ([string]$lease.status -eq "ready")
|
||||
remediation_rebuild_route_visible = ($null -ne $matchingChannel -and [string](Get-PropertyValue -Item $matchingChannel -Name "remediation_action" -Default "") -eq "rebuild_route")
|
||||
remediation_command_visible = ($null -ne $command -and $commandID.Length -gt 0)
|
||||
durable_rebuild_intent_recorded = ($null -ne $attempt)
|
||||
durable_rebuild_intent_resolved_applied = ($null -ne $attempt -and [string](Get-PropertyValue -Item $attempt -Name "rebuild_status" -Default "") -eq "applied")
|
||||
durable_rebuild_intent_outcome_replacement_selected = ($null -ne $attempt -and [string](Get-PropertyValue -Item $attempt -Name "outcome" -Default "") -eq "replacement_selected")
|
||||
durable_rebuild_intent_replacement_matches = ($null -ne $attempt -and [string](Get-PropertyValue -Item $attempt -Name "replacement_route_id" -Default "") -eq $alternateRouteID)
|
||||
durable_rebuild_intent_source_matches = ($null -ne $attempt -and [string](Get-PropertyValue -Item $attempt -Name "decision_source" -Default "") -eq "service_channel_remediation_command")
|
||||
initial_planner_applied_recorded = ($null -ne $attempt -and [string](Get-PropertyValue -Item $attempt -Name "rebuild_status" -Default "") -eq "applied")
|
||||
node_route_manager_consumed_same_command = ($null -ne $routeManagerDecision)
|
||||
node_route_manager_applied_rebuild = ($null -ne $routeManagerDecision -and [string](Get-PropertyValue -Item $routeManagerDecision -Name "rebuild_status" -Default "") -eq "applied")
|
||||
node_route_manager_transition_applied_rebuild = ($null -ne $routeManagerTransition -and [string](Get-PropertyValue -Item $routeManagerTransition -Name "status" -Default "") -eq "applied_rebuild")
|
||||
baseline_runtime_snapshot_visible = ($null -ne $baselineSnapshot)
|
||||
all_pressure_bursts_accepted = ($acceptedBurstCount -eq $burstCount)
|
||||
pressure_phase_reached_initial_replacement = ($replacementBurstCount -ge 1)
|
||||
stale_primary_not_reselected = ($stalePrimaryReselections -eq 0)
|
||||
final_last_selected_route_matches_replacement = ($null -ne $finalSnapshot -and $finalSnapshot.last_selected_route_id -eq $alternateRouteID)
|
||||
no_backend_or_local_fallback_after_pressure = (($finalFallbackLocal - $baselineFallbackLocal) -eq 0)
|
||||
pressure_phase_completed_before_recovery_probe = ($acceptedBurstCount -eq $burstCount)
|
||||
no_flow_drops_after_pressure = (($finalFlowDropped - $baselineFlowDropped) -eq 0 -and ($finalSchedulerDropped - $baselineSchedulerDropped) -eq 0)
|
||||
replacement_degradation_planner_selected_recovery = ($null -ne $recoveryPlannerDecision -and [string](Get-PropertyValue -Item $recoveryPlannerDecision -Name "rebuild_status" -Default "") -eq "applied")
|
||||
replacement_degradation_runtime_selected_recovery = ($recoveryTrafficObserved -or ($null -ne $recoveryRouteManagerDecision -and [string](Get-PropertyValue -Item $recoveryRouteManagerDecision -Name "rebuild_status" -Default "") -eq "applied"))
|
||||
recovery_route_selected_for_live_traffic = $recoveryTrafficObserved
|
||||
recovery_traffic_accepted = ([int]$recoveryResponse.StatusCode -eq 202 -and ($recoveryAcceptedBy -eq "introspection" -or $recoveryAcceptedBy -eq "signed"))
|
||||
degraded_replacement_not_reselected_after_recovery = ($null -ne $recoveryFinalSnapshot -and $recoveryFinalSnapshot.last_selected_route_id -ne $alternateRouteID)
|
||||
no_fallback_or_failures_after_recovery = (($recoveryFinalFallbackLocal - $recoveryBaselineFallbackLocal) -eq 0 -and ($recoveryFinalRouteFailures - $recoveryBaselineRouteFailures) -eq 0)
|
||||
no_flow_drops_after_recovery = (($recoveryFinalFlowDropped - $recoveryBaselineFlowDropped) -eq 0 -and ($recoveryFinalSchedulerDropped - $recoveryBaselineSchedulerDropped) -eq 0)
|
||||
}
|
||||
$failed = @($checks.GetEnumerator() | Where-Object { -not $_.Value } | ForEach-Object { $_.Key })
|
||||
|
||||
$result = [ordered]@{
|
||||
schema_version = "c18z81.service_channel_replacement_degradation_recovery_smoke.v1"
|
||||
run_id = $runId
|
||||
cluster_id = $ClusterID
|
||||
channel_id = [string]$lease.channel_id
|
||||
primary_route_id = $primaryRouteID
|
||||
alternate_route_id = $alternateRouteID
|
||||
recovery_route_id = $recoveryRouteID
|
||||
rebuild_request_id = $commandID
|
||||
passed = ($failed.Count -eq 0)
|
||||
checks = $checks
|
||||
failed_checks = $failed
|
||||
summary = [ordered]@{
|
||||
backend_container = $backendLine.Trim()
|
||||
node_containers = $nodeLines.Trim()
|
||||
remediation_command = $command
|
||||
rebuild_attempt = $attempt
|
||||
route_manager_decision = $routeManagerDecision
|
||||
route_manager_transition = $routeManagerTransition
|
||||
burst_count = $burstCount
|
||||
burst_packet_count = $burstPacketCount
|
||||
accepted_burst_count = $acceptedBurstCount
|
||||
replacement_burst_count = $replacementBurstCount
|
||||
stale_primary_reselection_count = $stalePrimaryReselections
|
||||
baseline_snapshot = $baselineSnapshot
|
||||
final_snapshot = $finalSnapshot
|
||||
fallback_local_delta = ($finalFallbackLocal - $baselineFallbackLocal)
|
||||
route_failure_delta = ($finalRouteFailures - $baselineRouteFailures)
|
||||
flow_drop_delta = ($finalFlowDropped - $baselineFlowDropped)
|
||||
scheduler_drop_delta = ($finalSchedulerDropped - $baselineSchedulerDropped)
|
||||
burst_results = $burstResults
|
||||
recovery_planner_decision = $recoveryPlannerDecision
|
||||
recovery_route_manager_decision = $recoveryRouteManagerDecision
|
||||
recovery_route_manager_transition = $recoveryRouteManagerTransition
|
||||
recovery_traffic_status_code = [int]$recoveryResponse.StatusCode
|
||||
recovery_traffic_accepted_by = $recoveryAcceptedBy
|
||||
recovery_traffic_duration_ms = $recoveryDurationMs
|
||||
recovery_baseline_snapshot = $recoveryBaselineSnapshot
|
||||
recovery_final_snapshot = $recoveryFinalSnapshot
|
||||
recovery_route_failure_delta = ($recoveryFinalRouteFailures - $recoveryBaselineRouteFailures)
|
||||
recovery_fallback_local_delta = ($recoveryFinalFallbackLocal - $recoveryBaselineFallbackLocal)
|
||||
recovery_flow_drop_delta = ($recoveryFinalFlowDropped - $recoveryBaselineFlowDropped)
|
||||
recovery_scheduler_drop_delta = ($recoveryFinalSchedulerDropped - $recoveryBaselineSchedulerDropped)
|
||||
post_channel = $postChannel
|
||||
}
|
||||
}
|
||||
|
||||
$target = Join-Path $repoRoot $ResultPath
|
||||
$result | ConvertTo-Json -Depth 80 | Set-Content -Path $target -Encoding UTF8
|
||||
|
||||
try {
|
||||
if ($primaryRouteID) {
|
||||
Invoke-Api -Method POST -Path "/clusters/$ClusterID/mesh/route-intents/$primaryRouteID/expire" -Body @{ actor_user_id = $ActorUserID } | Out-Null
|
||||
}
|
||||
if ($alternateRouteID) {
|
||||
Invoke-Api -Method POST -Path "/clusters/$ClusterID/mesh/route-intents/$alternateRouteID/expire" -Body @{ actor_user_id = $ActorUserID } | Out-Null
|
||||
}
|
||||
if ($recoveryRouteID) {
|
||||
Invoke-Api -Method POST -Path "/clusters/$ClusterID/mesh/route-intents/$recoveryRouteID/expire" -Body @{ actor_user_id = $ActorUserID } | Out-Null
|
||||
}
|
||||
Start-Sleep -Seconds 3
|
||||
Invoke-Api -Method POST -Path "/clusters/$ClusterID/fabric/service-channels/leases/cleanup" -Body @{
|
||||
actor_user_id = $ActorUserID
|
||||
limit = 50
|
||||
} | Out-Null
|
||||
} catch {
|
||||
Write-Warning "cleanup failed after c18z81 smoke: $($_.Exception.Message)"
|
||||
}
|
||||
|
||||
if (-not $result.passed) {
|
||||
throw "C18Z81 replacement-degradation-recovery smoke failed: $($failed -join ', ')"
|
||||
}
|
||||
|
||||
Write-Host "C18Z81 service-channel replacement-degradation-recovery smoke passed. Result: $target"
|
||||
$result
|
||||
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user