Record project continuation changes
This commit is contained in:
@@ -0,0 +1,619 @@
|
||||
param(
|
||||
[string]$ApiBaseUrl = "http://192.168.200.61:18121/api/v1",
|
||||
[string]$ClusterID = "cfc0743d-d960-49fb-9de8-96e063d5e4aa",
|
||||
[string]$ActorUserID = "f67d943f-5397-4b3a-a229-695fe67ad700",
|
||||
[string]$EntryNodeName = "test-1",
|
||||
[string]$RelayNodeName = "test-3",
|
||||
[string]$ExitNodeName = "test-2",
|
||||
[string]$EntryBaseUrl = "http://192.168.200.61:19131",
|
||||
[string]$DockerSSH = "test-docker",
|
||||
[string]$ExpectedBackendImage = "rap-backend:fabric-service-channel-0.2.256-c18z82",
|
||||
[string]$ExpectedNodeAgentImage = "rap-node-agent:0.2.270-c18z95",
|
||||
[string]$ResultPath = "artifacts\c18z67-service-channel-concurrent-qos-live-smoke-result.json"
|
||||
)
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
$scriptDir = Split-Path -Parent $MyInvocation.MyCommand.Path
|
||||
$repoRoot = (Resolve-Path (Join-Path $scriptDir "..\..")).ProviderPath
|
||||
$runId = "c18z67-" + (Get-Date -Format "yyyyMMdd-HHmmss")
|
||||
|
||||
function Invoke-Api {
|
||||
param(
|
||||
[string]$Method,
|
||||
[string]$Path,
|
||||
[object]$Body = $null
|
||||
)
|
||||
if ($null -eq $Body) {
|
||||
return Invoke-RestMethod -Method $Method -Uri "$ApiBaseUrl$Path" -TimeoutSec 30
|
||||
}
|
||||
return Invoke-RestMethod -Method $Method -Uri "$ApiBaseUrl$Path" -ContentType "application/json" -Body ($Body | ConvertTo-Json -Depth 80) -TimeoutSec 30
|
||||
}
|
||||
|
||||
function Get-PropertyValue {
|
||||
param(
|
||||
[object]$Item,
|
||||
[string]$Name,
|
||||
[object]$Default = $null
|
||||
)
|
||||
if ($null -eq $Item) { return $Default }
|
||||
$property = $Item.PSObject.Properties[$Name]
|
||||
if ($null -eq $property) { return $Default }
|
||||
return $property.Value
|
||||
}
|
||||
|
||||
function Get-NodeByName {
|
||||
param([string]$Name)
|
||||
$nodes = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/nodes?actor_user_id=$ActorUserID").nodes
|
||||
$node = @($nodes | Where-Object { $_.name -eq $Name }) | Select-Object -First 1
|
||||
if ($null -eq $node) {
|
||||
throw "Node '$Name' was not found in cluster $ClusterID"
|
||||
}
|
||||
return $node
|
||||
}
|
||||
|
||||
function New-IPv4TcpPacket {
|
||||
param(
|
||||
[byte[]]$Source = @(10, 77, 0, 2),
|
||||
[byte[]]$Destination = @(192, 168, 200, 95),
|
||||
[int]$SourcePort,
|
||||
[int]$DestinationPort = 3389
|
||||
)
|
||||
$packet = [byte[]]::new(40)
|
||||
$packet[0] = 0x45
|
||||
$packet[2] = 0
|
||||
$packet[3] = 40
|
||||
$packet[8] = 64
|
||||
$packet[9] = 6
|
||||
[Array]::Copy($Source, 0, $packet, 12, 4)
|
||||
[Array]::Copy($Destination, 0, $packet, 16, 4)
|
||||
$packet[20] = [byte](($SourcePort -shr 8) -band 0xff)
|
||||
$packet[21] = [byte]($SourcePort -band 0xff)
|
||||
$packet[22] = [byte](($DestinationPort -shr 8) -band 0xff)
|
||||
$packet[23] = [byte]($DestinationPort -band 0xff)
|
||||
$packet[32] = 0x50
|
||||
return $packet
|
||||
}
|
||||
|
||||
function ConvertTo-VPNPacketBatch {
|
||||
param([byte[][]]$Packets)
|
||||
$buffer = New-Object System.Collections.Generic.List[byte]
|
||||
foreach ($packet in $Packets) {
|
||||
if ($null -eq $packet -or $packet.Length -eq 0) { continue }
|
||||
$size = [uint32]$packet.Length
|
||||
$buffer.Add([byte](($size -shr 24) -band 0xff))
|
||||
$buffer.Add([byte](($size -shr 16) -band 0xff))
|
||||
$buffer.Add([byte](($size -shr 8) -band 0xff))
|
||||
$buffer.Add([byte]($size -band 0xff))
|
||||
foreach ($value in $packet) {
|
||||
$buffer.Add($value)
|
||||
}
|
||||
}
|
||||
return $buffer.ToArray()
|
||||
}
|
||||
|
||||
function New-RouteIntent {
|
||||
param(
|
||||
[string]$SourceNodeID,
|
||||
[string]$DestinationNodeID,
|
||||
[string[]]$Hops,
|
||||
[int]$Priority,
|
||||
[string]$Label
|
||||
)
|
||||
$expiresAt = (Get-Date).ToUniversalTime().AddMinutes(5).ToString("o")
|
||||
return Invoke-Api -Method POST -Path "/clusters/$ClusterID/mesh/route-intents" -Body @{
|
||||
actor_user_id = $ActorUserID
|
||||
source_selector = @{ node_id = $SourceNodeID }
|
||||
destination_selector = @{ node_id = $DestinationNodeID }
|
||||
service_class = "vpn_packets"
|
||||
priority = $Priority
|
||||
policy = @{
|
||||
synthetic_enabled = $true
|
||||
route_version = "$runId-$Label"
|
||||
policy_version = "$runId-$Label"
|
||||
peer_directory_version = "$runId-$Label"
|
||||
hops = @($Hops)
|
||||
allowed_channels = @("vpn_packet", "fabric_control")
|
||||
max_ttl = 8
|
||||
max_hops = 8
|
||||
expires_at = $expiresAt
|
||||
metadata = @{
|
||||
smoke = "c18z67_service_channel_concurrent_qos_live"
|
||||
run_id = $runId
|
||||
label = $Label
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function Disable-ExistingRouteIntents {
|
||||
param([string]$SourceNodeID, [string]$DestinationNodeID)
|
||||
$items = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/mesh/route-intents?actor_user_id=$ActorUserID").route_intents
|
||||
foreach ($item in @($items)) {
|
||||
if ([string](Get-PropertyValue -Item $item -Name "status" -Default "") -ne "active") { continue }
|
||||
if ([string](Get-PropertyValue -Item $item -Name "service_class" -Default "") -ne "vpn_packets") { continue }
|
||||
$sourceSelector = Get-PropertyValue -Item $item -Name "source_selector" -Default $null
|
||||
$destinationSelector = Get-PropertyValue -Item $item -Name "destination_selector" -Default $null
|
||||
if ([string](Get-PropertyValue -Item $sourceSelector -Name "node_id" -Default "") -ne $SourceNodeID) { continue }
|
||||
if ([string](Get-PropertyValue -Item $destinationSelector -Name "node_id" -Default "") -ne $DestinationNodeID) { continue }
|
||||
[void](Invoke-Api -Method POST -Path "/clusters/$ClusterID/mesh/route-intents/$($item.id)/disable" -Body @{
|
||||
actor_user_id = $ActorUserID
|
||||
reason = "c18z67 isolate concurrent-qos smoke route pair"
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
function Send-DegradedHeartbeat {
|
||||
param(
|
||||
[string]$EntryNodeID,
|
||||
[string]$PrimaryRouteID
|
||||
)
|
||||
$observedAt = (Get-Date).ToUniversalTime().ToString("o")
|
||||
return Invoke-Api -Method POST -Path "/clusters/$ClusterID/nodes/$EntryNodeID/heartbeats" -Body @{
|
||||
health_status = "healthy"
|
||||
reported_version = "0.2.236"
|
||||
capabilities = @{
|
||||
fabric_service_channel_runtime = $true
|
||||
fabric_service_channel_route_manager = $true
|
||||
fabric_service_channel_route_quality_feedback = $true
|
||||
smoke_feedback_injection = "c18z60"
|
||||
}
|
||||
service_states = @{ smoke = "c18z67_primary_degraded_alternate_available" }
|
||||
metadata = @{
|
||||
fabric_service_channel_runtime_report = @{
|
||||
schema_version = "c18l.fabric_service_channel_runtime_report.v1"
|
||||
config_version = "$runId-primary"
|
||||
cluster_id = $ClusterID
|
||||
local_node_id = $EntryNodeID
|
||||
observed_at = $observedAt
|
||||
ingress = @{
|
||||
flow_scheduler = @{
|
||||
schema_version = "rap.fabric_flow_scheduler.v1"
|
||||
service_neutral = $true
|
||||
service_mode = "application_protocol_agnostic"
|
||||
channel_stats = @{
|
||||
"c18z67-primary-degraded" = @{
|
||||
last_route_id = $PrimaryRouteID
|
||||
last_failed_route_id = $PrimaryRouteID
|
||||
route_generation = "$runId-primary"
|
||||
last_error = "c18z60 primary route degraded; alternate available"
|
||||
last_send_duration_ms = 980
|
||||
consecutive_failures = 3
|
||||
stall_count = 2
|
||||
route_rebuild_recommended = $true
|
||||
degraded_fallback_recommended = $false
|
||||
quality_window_sample_count = 8
|
||||
quality_window_success_count = 2
|
||||
quality_window_failure_count = 3
|
||||
quality_window_slow_count = 2
|
||||
quality_window_drop_count = 1
|
||||
quality_window_avg_latency_ms = 980
|
||||
quality_window_last_updated_at = $observedAt
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
smoke = @{
|
||||
name = "c18z67_service_channel_concurrent_qos_live"
|
||||
run_id = $runId
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
$entryNode = Get-NodeByName -Name $EntryNodeName
|
||||
$relayNode = Get-NodeByName -Name $RelayNodeName
|
||||
$exitNode = Get-NodeByName -Name $ExitNodeName
|
||||
$primaryRouteID = ""
|
||||
$alternateRouteID = ""
|
||||
$result = $null
|
||||
$failedChecks = @()
|
||||
$expectedPressurePacketCount = 512
|
||||
$expectedPressureFlowCount = 8
|
||||
$expectedInteractivePacketCount = 1
|
||||
$expectedBulkRequestCount = 6
|
||||
|
||||
try {
|
||||
[void](Disable-ExistingRouteIntents -SourceNodeID $entryNode.id -DestinationNodeID $exitNode.id)
|
||||
$primary = (New-RouteIntent -SourceNodeID $entryNode.id -DestinationNodeID $exitNode.id -Hops @($entryNode.id, $exitNode.id) -Priority 2100000000 -Label "primary").route_intent
|
||||
$alternate = (New-RouteIntent -SourceNodeID $entryNode.id -DestinationNodeID $exitNode.id -Hops @($entryNode.id, $relayNode.id, $exitNode.id) -Priority 2099999900 -Label "alternate").route_intent
|
||||
$primaryRouteID = [string]$primary.id
|
||||
$alternateRouteID = [string]$alternate.id
|
||||
|
||||
$resourceID = "c18z67-vpn-smoke"
|
||||
$lease = (Invoke-Api -Method POST -Path "/clusters/$ClusterID/fabric/service-channels/leases" -Body @{
|
||||
actor_user_id = $ActorUserID
|
||||
organization_id = "smoke-org"
|
||||
user_id = "smoke-user"
|
||||
resource_id = $resourceID
|
||||
service_class = "vpn_packets"
|
||||
entry_node_ids = @([string]$entryNode.id)
|
||||
exit_node_ids = @([string]$exitNode.id)
|
||||
preferred_entry_node_id = [string]$entryNode.id
|
||||
preferred_exit_node_id = [string]$exitNode.id
|
||||
allowed_channels = @("vpn_packet", "fabric_control")
|
||||
ttl_seconds = 180
|
||||
metadata = @{
|
||||
smoke = "c18z67_service_channel_concurrent_qos_live"
|
||||
run_id = $runId
|
||||
}
|
||||
}).fabric_service_channel_lease
|
||||
|
||||
[void](Send-DegradedHeartbeat -EntryNodeID $entryNode.id -PrimaryRouteID $primaryRouteID)
|
||||
|
||||
$packetPath = $lease.entry_http.path_template.
|
||||
Replace("{cluster_id}", $ClusterID).
|
||||
Replace("{channel_id}", [string]$lease.channel_id).
|
||||
Replace("{resource_id}", $resourceID)
|
||||
$packetUrl = $EntryBaseUrl.TrimEnd("/") + $packetPath
|
||||
$headers = @{
|
||||
"X-RAP-Service-Channel-Token" = [string]$lease.token.token
|
||||
"X-RAP-Fabric-Channel-ID" = [string]$lease.channel_id
|
||||
"X-RAP-Service-Class" = "vpn_packets"
|
||||
"X-RAP-Channel-Class" = "vpn_packet"
|
||||
}
|
||||
$response = Invoke-WebRequest -Method Post -Uri $packetUrl -Headers $headers -Body ([System.Text.Encoding]::UTF8.GetBytes("c18z67-alternate-remediation")) -ContentType "application/octet-stream" -TimeoutSec 30
|
||||
$acceptedBy = [string]$response.Headers["X-RAP-Service-Channel-Accepted-By"]
|
||||
|
||||
$accessTelemetry = $null
|
||||
$matchingChannel = $null
|
||||
for ($i = 0; $i -lt 10; $i++) {
|
||||
Start-Sleep -Seconds 3
|
||||
$accessTelemetry = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/fabric/service-channels/access-telemetry?actor_user_id=$ActorUserID&limit=20").fabric_service_channel_access_telemetry
|
||||
$channels = @()
|
||||
if ($accessTelemetry.PSObject.Properties.Name -contains "active_channels") {
|
||||
$channels = @($accessTelemetry.active_channels)
|
||||
}
|
||||
$matchingChannel = $channels | Where-Object { $_.channel_id -eq $lease.channel_id } | Select-Object -First 1
|
||||
if ($null -ne $matchingChannel -and [string](Get-PropertyValue -Item $matchingChannel -Name "remediation_action" -Default "") -eq "prefer_alternate_route") {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
$backendLine = (& ssh $DockerSSH "docker ps --format '{{.Names}} {{.Image}} {{.Status}}' | grep '^rap_test_backend '") | Out-String
|
||||
$nodeLines = (& ssh $DockerSSH "docker ps --format '{{.Names}} {{.Image}} {{.Status}}' | grep '^rap_test_node_test_'") | Out-String
|
||||
$leasePrimaryRouteID = [string](Get-PropertyValue -Item (Get-PropertyValue -Item $lease -Name "primary_route" -Default $null) -Name "route_id" -Default "")
|
||||
$leaseAlternates = @()
|
||||
if ($lease.PSObject.Properties.Name -contains "alternate_routes") {
|
||||
$leaseAlternates = @($lease.alternate_routes)
|
||||
}
|
||||
$leaseHasAlternate = (@($leaseAlternates | Where-Object { [string]$_.route_id -eq $alternateRouteID }).Count -ge 1)
|
||||
$remediationCommand = Get-PropertyValue -Item $matchingChannel -Name "remediation_command" -Default $null
|
||||
$commandAction = [string](Get-PropertyValue -Item $remediationCommand -Name "action" -Default "")
|
||||
$commandPrimaryRouteID = [string](Get-PropertyValue -Item $remediationCommand -Name "primary_route_id" -Default "")
|
||||
$commandReplacementRouteID = [string](Get-PropertyValue -Item $remediationCommand -Name "replacement_route_id" -Default "")
|
||||
$commandExpiresAt = [string](Get-PropertyValue -Item $remediationCommand -Name "expires_at" -Default "")
|
||||
$expectedReplacementRouteID = $commandReplacementRouteID
|
||||
$syntheticConfig = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/nodes/$($entryNode.id)/mesh/synthetic-config").synthetic_mesh_config
|
||||
$syntheticCommands = @()
|
||||
if ($syntheticConfig.PSObject.Properties.Name -contains "service_channel_remediation_commands") {
|
||||
$syntheticCommands = @($syntheticConfig.service_channel_remediation_commands)
|
||||
}
|
||||
$syntheticCommand = $syntheticCommands | Where-Object { [string]$_.channel_id -eq [string]$lease.channel_id } | Select-Object -First 1
|
||||
$syntheticCommandAction = [string](Get-PropertyValue -Item $syntheticCommand -Name "action" -Default "")
|
||||
$syntheticCommandReplacementRouteID = [string](Get-PropertyValue -Item $syntheticCommand -Name "replacement_route_id" -Default "")
|
||||
if ([string]::IsNullOrWhiteSpace($expectedReplacementRouteID)) {
|
||||
$expectedReplacementRouteID = $syntheticCommandReplacementRouteID
|
||||
}
|
||||
if ([string]::IsNullOrWhiteSpace($expectedReplacementRouteID)) {
|
||||
$expectedReplacementRouteID = $alternateRouteID
|
||||
}
|
||||
|
||||
$routeManagerDecision = $null
|
||||
$routeManagerRuntime = $null
|
||||
for ($i = 0; $i -lt 10; $i++) {
|
||||
Start-Sleep -Seconds 5
|
||||
$heartbeats = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/nodes/$($entryNode.id)/heartbeats?actor_user_id=$ActorUserID&limit=5").heartbeats
|
||||
foreach ($heartbeat in @($heartbeats)) {
|
||||
$runtimeReport = Get-PropertyValue -Item (Get-PropertyValue -Item $heartbeat -Name "metadata" -Default $null) -Name "fabric_service_channel_runtime_report" -Default $null
|
||||
$ingress = Get-PropertyValue -Item $runtimeReport -Name "ingress" -Default $null
|
||||
$routeManager = Get-PropertyValue -Item $ingress -Name "route_manager" -Default $null
|
||||
$routeManagerRuntime = $routeManager
|
||||
$decisions = @()
|
||||
if ($null -ne $routeManager -and $routeManager.PSObject.Properties.Name -contains "decisions") {
|
||||
$decisions = @($routeManager.decisions)
|
||||
}
|
||||
$routeManagerDecision = $decisions | Where-Object {
|
||||
[string]$_.route_id -eq $primaryRouteID -and
|
||||
[string]$_.replacement_route_id -eq $expectedReplacementRouteID -and
|
||||
[string]$_.decision_source -eq "service_channel_remediation_command"
|
||||
} | Select-Object -First 1
|
||||
if ($null -ne $routeManagerDecision) {
|
||||
break
|
||||
}
|
||||
}
|
||||
if ($null -ne $routeManagerDecision) {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
$baselineFallbackLocal = 0
|
||||
$baselineRouteFailures = 0
|
||||
$baselineFlowDropped = 0
|
||||
$baselineSchedulerDropped = 0
|
||||
$baselineHeartbeats = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/nodes/$($entryNode.id)/heartbeats?actor_user_id=$ActorUserID&limit=1").heartbeats
|
||||
if (@($baselineHeartbeats).Count -gt 0) {
|
||||
$baselineRuntimeReport = Get-PropertyValue -Item (Get-PropertyValue -Item $baselineHeartbeats[0] -Name "metadata" -Default $null) -Name "fabric_service_channel_runtime_report" -Default $null
|
||||
$baselineIngress = Get-PropertyValue -Item $baselineRuntimeReport -Name "ingress" -Default $null
|
||||
$baselineFlowScheduler = Get-PropertyValue -Item $baselineIngress -Name "flow_scheduler" -Default $null
|
||||
$baselineFallbackLocal = [int](Get-PropertyValue -Item $baselineIngress -Name "send_fallback_local" -Default 0)
|
||||
$baselineRouteFailures = [int](Get-PropertyValue -Item $baselineIngress -Name "send_route_failures" -Default 0)
|
||||
$baselineFlowDropped = [int](Get-PropertyValue -Item $baselineIngress -Name "send_flow_dropped" -Default 0)
|
||||
$baselineSchedulerDropped = [int](Get-PropertyValue -Item $baselineFlowScheduler -Name "dropped" -Default 0)
|
||||
}
|
||||
|
||||
$multiFlowPackets = @()
|
||||
foreach ($offset in 0..($expectedPressurePacketCount - 1)) {
|
||||
$port = 52000 + $offset
|
||||
$multiFlowPackets += ,(New-IPv4TcpPacket -SourcePort $port -DestinationPort 3389)
|
||||
}
|
||||
$multiFlowPayload = ConvertTo-VPNPacketBatch -Packets $multiFlowPackets
|
||||
$multiFlowPayloadPath = Join-Path ([System.IO.Path]::GetTempPath()) "$runId-vpn-packet-batch.bin"
|
||||
[System.IO.File]::WriteAllBytes($multiFlowPayloadPath, [byte[]]$multiFlowPayload)
|
||||
$bulkHeaders = @{} + $headers
|
||||
$bulkHeaders["X-RAP-Traffic-Class"] = "bulk"
|
||||
$bulkJobs = @()
|
||||
foreach ($jobIndex in 1..$expectedBulkRequestCount) {
|
||||
$jobPayloadPath = Join-Path ([System.IO.Path]::GetTempPath()) "$runId-vpn-packet-batch-$jobIndex.bin"
|
||||
Copy-Item -Path $multiFlowPayloadPath -Destination $jobPayloadPath -Force
|
||||
$bulkJobs += Start-Job -ArgumentList ($packetUrl + "?batch=true"), $bulkHeaders, $jobPayloadPath -ScriptBlock {
|
||||
param($uri, $headers, $path)
|
||||
try {
|
||||
$started = Get-Date
|
||||
$response = Invoke-WebRequest -Method Post -Uri $uri -Headers $headers -InFile $path -ContentType "application/vnd.rap.vpn-packet-batch.v1" -TimeoutSec 30
|
||||
[pscustomobject]@{
|
||||
status_code = [int]$response.StatusCode
|
||||
accepted_by = [string]$response.Headers["X-RAP-Service-Channel-Accepted-By"]
|
||||
duration_ms = [int]((Get-Date) - $started).TotalMilliseconds
|
||||
error = ""
|
||||
}
|
||||
} finally {
|
||||
Remove-Item -Path $path -Force -ErrorAction SilentlyContinue
|
||||
}
|
||||
}
|
||||
}
|
||||
Start-Sleep -Milliseconds 100
|
||||
$interactivePacket = New-IPv4TcpPacket -SourcePort 53000 -DestinationPort 22
|
||||
$interactivePayload = ConvertTo-VPNPacketBatch -Packets @($interactivePacket)
|
||||
$interactivePayloadPath = Join-Path ([System.IO.Path]::GetTempPath()) "$runId-vpn-interactive-packet-batch.bin"
|
||||
[System.IO.File]::WriteAllBytes($interactivePayloadPath, [byte[]]$interactivePayload)
|
||||
$interactiveHeaders = @{} + $headers
|
||||
$interactiveHeaders["X-RAP-Traffic-Class"] = "interactive"
|
||||
$interactiveStarted = Get-Date
|
||||
$interactiveResponse = Invoke-WebRequest -Method Post -Uri ($packetUrl + "?batch=true") -Headers $interactiveHeaders -InFile $interactivePayloadPath -ContentType "application/vnd.rap.vpn-packet-batch.v1" -TimeoutSec 30
|
||||
$interactiveDurationMs = [int]((Get-Date) - $interactiveStarted).TotalMilliseconds
|
||||
$interactiveAcceptedBy = [string]$interactiveResponse.Headers["X-RAP-Service-Channel-Accepted-By"]
|
||||
$bulkResponses = @($bulkJobs | Wait-Job -Timeout 60 | Receive-Job)
|
||||
$bulkJobs | Remove-Job -Force -ErrorAction SilentlyContinue
|
||||
$bulkAcceptedCount = @($bulkResponses | Where-Object { [int]$_.status_code -eq 202 -and [string]$_.accepted_by -eq "introspection" }).Count
|
||||
$postRemediationAcceptedBy = (@($bulkResponses | Select-Object -First 1).accepted_by | Out-String).Trim()
|
||||
$replacementTrafficObserved = $false
|
||||
$replacementLastSelected = ""
|
||||
$replacementFlowStat = $null
|
||||
$replacementFlowStats = @()
|
||||
$interactiveFlowStats = @()
|
||||
$bulkFlowStats = @()
|
||||
$postRemediationFallbackLocal = 0
|
||||
$postRemediationRouteFailures = 0
|
||||
$postRemediationFlowDropped = 0
|
||||
$postRemediationSchedulerDropped = 0
|
||||
$postRemediationHighWatermark = 0
|
||||
$postRemediationMaxInFlight = 0
|
||||
for ($i = 0; $i -lt 12; $i++) {
|
||||
Start-Sleep -Seconds 5
|
||||
$heartbeats = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/nodes/$($entryNode.id)/heartbeats?actor_user_id=$ActorUserID&limit=5").heartbeats
|
||||
foreach ($heartbeat in @($heartbeats)) {
|
||||
$runtimeReport = Get-PropertyValue -Item (Get-PropertyValue -Item $heartbeat -Name "metadata" -Default $null) -Name "fabric_service_channel_runtime_report" -Default $null
|
||||
$ingress = Get-PropertyValue -Item $runtimeReport -Name "ingress" -Default $null
|
||||
$replacementLastSelected = [string](Get-PropertyValue -Item $ingress -Name "last_selected_route_id" -Default "")
|
||||
$postRemediationFallbackLocal = [int](Get-PropertyValue -Item $ingress -Name "send_fallback_local" -Default 0)
|
||||
$postRemediationRouteFailures = [int](Get-PropertyValue -Item $ingress -Name "send_route_failures" -Default 0)
|
||||
$postRemediationFlowDropped = [int](Get-PropertyValue -Item $ingress -Name "send_flow_dropped" -Default 0)
|
||||
$flowScheduler = Get-PropertyValue -Item $ingress -Name "flow_scheduler" -Default $null
|
||||
$postRemediationSchedulerDropped = [int](Get-PropertyValue -Item $flowScheduler -Name "dropped" -Default 0)
|
||||
$postRemediationHighWatermark = [int](Get-PropertyValue -Item $flowScheduler -Name "high_watermark" -Default 0)
|
||||
$postRemediationMaxInFlight = [int](Get-PropertyValue -Item $flowScheduler -Name "max_in_flight" -Default 0)
|
||||
$channelStats = Get-PropertyValue -Item $flowScheduler -Name "channel_stats" -Default $null
|
||||
if ($null -ne $channelStats) {
|
||||
$replacementFlowStats = @()
|
||||
$interactiveFlowStats = @()
|
||||
$bulkFlowStats = @()
|
||||
foreach ($property in @($channelStats.PSObject.Properties)) {
|
||||
$stat = $property.Value
|
||||
if ([string](Get-PropertyValue -Item $stat -Name "last_route_id" -Default "") -eq $expectedReplacementRouteID) {
|
||||
$replacementFlowStats += $stat
|
||||
$replacementFlowStat = $stat
|
||||
$trafficClass = [string](Get-PropertyValue -Item $stat -Name "traffic_class" -Default "")
|
||||
if ($trafficClass -eq "interactive") {
|
||||
$interactiveFlowStats += $stat
|
||||
}
|
||||
if ($trafficClass -eq "bulk") {
|
||||
$bulkFlowStats += $stat
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if ($replacementLastSelected -eq $expectedReplacementRouteID -and $bulkFlowStats.Count -ge $expectedPressureFlowCount -and $interactiveFlowStats.Count -ge 1) {
|
||||
$replacementTrafficObserved = $true
|
||||
break
|
||||
}
|
||||
}
|
||||
if ($replacementTrafficObserved) {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
$result = [ordered]@{
|
||||
schema_version = "c18z67.service_channel_concurrent_qos_live_smoke.v1"
|
||||
run_id = $runId
|
||||
cluster_id = $ClusterID
|
||||
primary_route_id = $primaryRouteID
|
||||
alternate_route_id = $alternateRouteID
|
||||
expected_replacement_route_id = $expectedReplacementRouteID
|
||||
channel_id = [string]$lease.channel_id
|
||||
passed = [bool](
|
||||
$backendLine.Contains($ExpectedBackendImage) -and
|
||||
$nodeLines.Contains($ExpectedNodeAgentImage) -and
|
||||
[string]$lease.status -eq "ready" -and
|
||||
$leasePrimaryRouteID -eq $primaryRouteID -and
|
||||
$leaseHasAlternate -and
|
||||
[int]$response.StatusCode -eq 202 -and
|
||||
$acceptedBy -eq "introspection" -and
|
||||
$null -ne $matchingChannel -and
|
||||
[string](Get-PropertyValue -Item $matchingChannel -Name "primary_route_id" -Default "") -eq $primaryRouteID -and
|
||||
-not [bool](Get-PropertyValue -Item $matchingChannel -Name "force_backend_fallback" -Default $false) -and
|
||||
[string](Get-PropertyValue -Item $matchingChannel -Name "route_feedback_status" -Default "") -eq "fenced" -and
|
||||
[string](Get-PropertyValue -Item $matchingChannel -Name "remediation_action" -Default "") -eq "prefer_alternate_route" -and
|
||||
[string](Get-PropertyValue -Item $matchingChannel -Name "remediation_route_id" -Default "") -eq $expectedReplacementRouteID -and
|
||||
$null -ne $remediationCommand -and
|
||||
$commandAction -eq "prefer_alternate_route" -and
|
||||
$commandPrimaryRouteID -eq $primaryRouteID -and
|
||||
$commandReplacementRouteID -eq $expectedReplacementRouteID -and
|
||||
$commandExpiresAt.Length -gt 0 -and
|
||||
$null -ne $syntheticCommand -and
|
||||
$syntheticCommandAction -eq "prefer_alternate_route" -and
|
||||
$syntheticCommandReplacementRouteID -eq $expectedReplacementRouteID -and
|
||||
$null -ne $routeManagerDecision -and
|
||||
[string](Get-PropertyValue -Item $routeManagerDecision -Name "rebuild_status" -Default "") -eq "applied" -and
|
||||
$bulkResponses.Count -eq $expectedBulkRequestCount -and
|
||||
$bulkAcceptedCount -eq $expectedBulkRequestCount -and
|
||||
[int]$interactiveResponse.StatusCode -eq 202 -and
|
||||
$interactiveAcceptedBy -eq "introspection" -and
|
||||
$interactiveDurationMs -lt 2000 -and
|
||||
$replacementTrafficObserved -and
|
||||
$replacementLastSelected -eq $expectedReplacementRouteID -and
|
||||
$bulkFlowStats.Count -ge $expectedPressureFlowCount -and
|
||||
$interactiveFlowStats.Count -ge 1 -and
|
||||
$multiFlowPackets.Count -eq $expectedPressurePacketCount -and
|
||||
($postRemediationFallbackLocal - $baselineFallbackLocal) -eq 0 -and
|
||||
($postRemediationFlowDropped - $baselineFlowDropped) -eq 0 -and
|
||||
($postRemediationSchedulerDropped - $baselineSchedulerDropped) -eq 0 -and
|
||||
($postRemediationRouteFailures - $baselineRouteFailures) -eq 0 -and
|
||||
$null -ne $matchingChannel -and
|
||||
-not [bool](Get-PropertyValue -Item $matchingChannel -Name "force_backend_fallback" -Default $false) -and
|
||||
[string](Get-PropertyValue -Item $matchingChannel -Name "remediation_action" -Default "") -ne "use_backend_fallback"
|
||||
)
|
||||
checks = [ordered]@{
|
||||
backend_expected_image_deployed = $backendLine.Contains($ExpectedBackendImage)
|
||||
node_agent_expected_image_deployed = $nodeLines.Contains($ExpectedNodeAgentImage)
|
||||
lease_ready = ([string]$lease.status -eq "ready")
|
||||
lease_selected_primary_route = ($leasePrimaryRouteID -eq $primaryRouteID)
|
||||
lease_contains_alternate_route = $leaseHasAlternate
|
||||
packet_accepted = ([int]$response.StatusCode -eq 202)
|
||||
accepted_by_header_is_introspection = ($acceptedBy -eq "introspection")
|
||||
active_channel_visible = ($null -ne $matchingChannel)
|
||||
active_channel_not_backend_fallback = ($null -ne $matchingChannel -and -not [bool](Get-PropertyValue -Item $matchingChannel -Name "force_backend_fallback" -Default $false))
|
||||
route_quality_fenced = ($null -ne $matchingChannel -and [string](Get-PropertyValue -Item $matchingChannel -Name "route_feedback_status" -Default "") -eq "fenced")
|
||||
remediation_prefers_alternate = ($null -ne $matchingChannel -and [string](Get-PropertyValue -Item $matchingChannel -Name "remediation_action" -Default "") -eq "prefer_alternate_route")
|
||||
remediation_route_is_alternate = ($null -ne $matchingChannel -and [string](Get-PropertyValue -Item $matchingChannel -Name "remediation_route_id" -Default "") -eq $expectedReplacementRouteID)
|
||||
remediation_command_visible = ($null -ne $remediationCommand)
|
||||
remediation_command_prefers_alternate = ($commandAction -eq "prefer_alternate_route")
|
||||
remediation_command_primary_route_matches = ($commandPrimaryRouteID -eq $primaryRouteID)
|
||||
remediation_command_replacement_route_matches = ($commandReplacementRouteID -eq $expectedReplacementRouteID)
|
||||
remediation_command_has_ttl = ($commandExpiresAt.Length -gt 0)
|
||||
synthetic_config_command_visible = ($null -ne $syntheticCommand)
|
||||
synthetic_config_command_prefers_alternate = ($syntheticCommandAction -eq "prefer_alternate_route")
|
||||
synthetic_config_command_replacement_route_matches = ($syntheticCommandReplacementRouteID -eq $expectedReplacementRouteID)
|
||||
node_route_manager_consumed_command = ($null -ne $routeManagerDecision)
|
||||
node_route_manager_applied_replacement = ($null -ne $routeManagerDecision -and [string](Get-PropertyValue -Item $routeManagerDecision -Name "rebuild_status" -Default "") -eq "applied")
|
||||
bulk_request_count_matches = ($bulkResponses.Count -eq $expectedBulkRequestCount)
|
||||
bulk_requests_accepted_by_introspection = ($bulkAcceptedCount -eq $expectedBulkRequestCount)
|
||||
interactive_packet_accepted = ([int]$interactiveResponse.StatusCode -eq 202)
|
||||
interactive_accepted_by_introspection = ($interactiveAcceptedBy -eq "introspection")
|
||||
interactive_completed_under_bulk_pressure = ($interactiveDurationMs -lt 2000)
|
||||
replacement_traffic_observed = $replacementTrafficObserved
|
||||
replacement_last_selected_route_matches = ($replacementLastSelected -eq $expectedReplacementRouteID)
|
||||
replacement_flow_stat_observed = ($null -ne $replacementFlowStat)
|
||||
replacement_pressure_flow_stats_observed = ($bulkFlowStats.Count -ge $expectedPressureFlowCount)
|
||||
replacement_interactive_flow_stat_observed = ($interactiveFlowStats.Count -ge 1)
|
||||
pressure_packet_count_matches = ($multiFlowPackets.Count -eq $expectedPressurePacketCount)
|
||||
no_local_gateway_fallback_after_replacement = (($postRemediationFallbackLocal - $baselineFallbackLocal) -eq 0)
|
||||
no_flow_drops_after_replacement = (($postRemediationFlowDropped - $baselineFlowDropped) -eq 0 -and ($postRemediationSchedulerDropped - $baselineSchedulerDropped) -eq 0)
|
||||
no_route_failures_after_replacement = (($postRemediationRouteFailures - $baselineRouteFailures) -eq 0)
|
||||
backend_fallback_not_recommended = ($null -ne $matchingChannel -and -not [bool](Get-PropertyValue -Item $matchingChannel -Name "force_backend_fallback" -Default $false) -and [string](Get-PropertyValue -Item $matchingChannel -Name "remediation_action" -Default "") -ne "use_backend_fallback")
|
||||
}
|
||||
summary = [ordered]@{
|
||||
backend_container = $backendLine.Trim()
|
||||
node_containers = $nodeLines.Trim()
|
||||
accepted_by = $acceptedBy
|
||||
lease_status = [string]$lease.status
|
||||
lease_primary_route_id = $leasePrimaryRouteID
|
||||
lease_alternate_route_count = $leaseAlternates.Count
|
||||
access_status = [string]$accessTelemetry.status
|
||||
active_channel_count = [int]$accessTelemetry.active_channel_count
|
||||
correlated_route_count = [int]$accessTelemetry.correlated_route_count
|
||||
degraded_route_count = [int]$accessTelemetry.degraded_route_count
|
||||
degraded_fallback_channel_count = [int]$accessTelemetry.degraded_fallback_channel_count
|
||||
synthetic_command = $syntheticCommand
|
||||
route_manager_decision = $routeManagerDecision
|
||||
route_manager_runtime = $routeManagerRuntime
|
||||
post_remediation_accepted_by = $postRemediationAcceptedBy
|
||||
bulk_request_count = $bulkResponses.Count
|
||||
bulk_accepted_count = $bulkAcceptedCount
|
||||
bulk_responses = $bulkResponses
|
||||
post_remediation_packet_count = ($multiFlowPackets.Count * $bulkResponses.Count)
|
||||
interactive_accepted_by = $interactiveAcceptedBy
|
||||
interactive_duration_ms = $interactiveDurationMs
|
||||
interactive_packet_count = $expectedInteractivePacketCount
|
||||
expected_pressure_flow_count = $expectedPressureFlowCount
|
||||
replacement_last_selected_route_id = $replacementLastSelected
|
||||
replacement_flow_stat_count = $replacementFlowStats.Count
|
||||
replacement_bulk_flow_stat_count = $bulkFlowStats.Count
|
||||
replacement_interactive_flow_stat_count = $interactiveFlowStats.Count
|
||||
replacement_flow_stat = $replacementFlowStat
|
||||
replacement_flow_stats = $replacementFlowStats
|
||||
replacement_bulk_flow_stats = $bulkFlowStats
|
||||
replacement_interactive_flow_stats = $interactiveFlowStats
|
||||
baseline_send_fallback_local = $baselineFallbackLocal
|
||||
baseline_send_route_failures = $baselineRouteFailures
|
||||
baseline_send_flow_dropped = $baselineFlowDropped
|
||||
baseline_scheduler_dropped = $baselineSchedulerDropped
|
||||
fallback_local_delta = ($postRemediationFallbackLocal - $baselineFallbackLocal)
|
||||
route_failure_delta = ($postRemediationRouteFailures - $baselineRouteFailures)
|
||||
flow_drop_delta = ($postRemediationFlowDropped - $baselineFlowDropped)
|
||||
scheduler_drop_delta = ($postRemediationSchedulerDropped - $baselineSchedulerDropped)
|
||||
post_remediation_send_fallback_local = $postRemediationFallbackLocal
|
||||
post_remediation_send_route_failures = $postRemediationRouteFailures
|
||||
post_remediation_send_flow_dropped = $postRemediationFlowDropped
|
||||
post_remediation_scheduler_dropped = $postRemediationSchedulerDropped
|
||||
post_remediation_scheduler_high_watermark = $postRemediationHighWatermark
|
||||
post_remediation_scheduler_max_in_flight = $postRemediationMaxInFlight
|
||||
matching_channel = $matchingChannel
|
||||
}
|
||||
}
|
||||
$failedChecks = @($result.checks.GetEnumerator() | Where-Object { $_.Value -ne $true })
|
||||
$result.summary.failed_checks = @($failedChecks | ForEach-Object { $_.Key })
|
||||
}
|
||||
finally {
|
||||
if ($primaryRouteID) {
|
||||
try { Invoke-Api -Method POST -Path "/clusters/$ClusterID/mesh/route-intents/$primaryRouteID/expire" -Body @{ actor_user_id = $ActorUserID } | Out-Null } catch {}
|
||||
}
|
||||
if ($alternateRouteID) {
|
||||
try { Invoke-Api -Method POST -Path "/clusters/$ClusterID/mesh/route-intents/$alternateRouteID/expire" -Body @{ actor_user_id = $ActorUserID } | Out-Null } catch {}
|
||||
}
|
||||
}
|
||||
|
||||
$resultFullPath = Join-Path $repoRoot $ResultPath
|
||||
$resultDir = Split-Path -Parent $resultFullPath
|
||||
if (-not (Test-Path $resultDir)) {
|
||||
New-Item -ItemType Directory -Path $resultDir | Out-Null
|
||||
}
|
||||
$result | ConvertTo-Json -Depth 100 | Set-Content -Path $resultFullPath -Encoding UTF8
|
||||
|
||||
if ($failedChecks.Count -gt 0) {
|
||||
throw "C18Z67 failed checks: $(@($failedChecks | ForEach-Object { $_.Key }) -join ', '). Result: $resultFullPath"
|
||||
}
|
||||
|
||||
Write-Host "C18Z67 service-channel concurrent qos live smoke passed. Result: $resultFullPath"
|
||||
$result
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user