Record project continuation changes
This commit is contained in:
@@ -0,0 +1,586 @@
|
||||
param(
|
||||
[string]$ApiBaseUrl = "http://192.168.200.61:18121/api/v1",
|
||||
[string]$ClusterID = "cfc0743d-d960-49fb-9de8-96e063d5e4aa",
|
||||
[string]$ActorUserID = "f67d943f-5397-4b3a-a229-695fe67ad700",
|
||||
[string]$EntryNodeName = "test-1",
|
||||
[string]$RelayNodeName = "test-3",
|
||||
[string]$ExitNodeName = "test-2",
|
||||
[string]$EntryBaseUrl = "http://192.168.200.61:19131",
|
||||
[string]$DockerSSH = "test-docker",
|
||||
[int]$InitialBatchCount = 12,
|
||||
[int]$LearningBatchCount = 24,
|
||||
[int]$PostChurnBatchCount = 24,
|
||||
[int]$PacketsPerBatch = 8,
|
||||
[int]$BatchDelayMilliseconds = 25,
|
||||
[string]$ResultPath = "artifacts\c18z14-live-service-channel-active-quality-shift-smoke-result.json"
|
||||
)
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = "Stop"
|
||||
Add-Type -AssemblyName System.Net.Http
|
||||
|
||||
$repoRoot = (Resolve-Path (Join-Path $PSScriptRoot "..\..")).ProviderPath
|
||||
$runId = "c18z14-" + (Get-Date -Format "yyyyMMdd-HHmmss")
|
||||
$resourceId = "vpn-$runId"
|
||||
|
||||
function Invoke-Api {
|
||||
param([string]$Method, [string]$Path, [object]$Body = $null)
|
||||
$uri = "$ApiBaseUrl$Path"
|
||||
try {
|
||||
if ($null -eq $Body) {
|
||||
return Invoke-RestMethod -Method $Method -Uri $uri -TimeoutSec 30
|
||||
}
|
||||
return Invoke-RestMethod -Method $Method -Uri $uri -ContentType "application/json" -Body ($Body | ConvertTo-Json -Depth 100) -TimeoutSec 30
|
||||
} catch {
|
||||
$statusCode = $null
|
||||
if ($_.Exception.Response) { $statusCode = [int]$_.Exception.Response.StatusCode }
|
||||
$details = $_.ErrorDetails.Message
|
||||
if (-not $details) { $details = $_.Exception.Message }
|
||||
throw "$Method $Path failed with HTTP $statusCode`: $details"
|
||||
}
|
||||
}
|
||||
|
||||
function Get-ObjectPropertyValue {
|
||||
param([object]$Object, [string]$Name)
|
||||
if ($null -eq $Object) { return $null }
|
||||
$prop = $Object.PSObject.Properties[$Name]
|
||||
if ($null -eq $prop) { return $null }
|
||||
return $prop.Value
|
||||
}
|
||||
|
||||
function Get-NodeByName {
|
||||
param([string]$Name)
|
||||
$nodes = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/nodes?actor_user_id=$ActorUserID").nodes
|
||||
$node = @($nodes | Where-Object { $_.name -eq $Name }) | Select-Object -First 1
|
||||
if ($null -eq $node) { throw "Node '$Name' was not found in cluster $ClusterID" }
|
||||
return $node
|
||||
}
|
||||
|
||||
function Get-MeshPort {
|
||||
param([string]$Name)
|
||||
switch ($Name) {
|
||||
"test-1" { return 19131 }
|
||||
"test-2" { return 19132 }
|
||||
"test-3" { return 19133 }
|
||||
default { return 19131 }
|
||||
}
|
||||
}
|
||||
|
||||
function Enable-TestMeshListener {
|
||||
param([object]$Node)
|
||||
$port = Get-MeshPort -Name $Node.name
|
||||
Invoke-Api -Method PUT -Path "/clusters/$ClusterID/nodes/$($Node.id)/workloads/mesh-listener/desired" -Body @{
|
||||
actor_user_id = $ActorUserID
|
||||
desired_state = "enabled"
|
||||
runtime_mode = "container"
|
||||
version = "c18z14-live-fsc-active-quality-shift"
|
||||
config = @{
|
||||
listen_addr = "0.0.0.0:$port"
|
||||
listen_port_mode = "manual"
|
||||
advertise_endpoint = "http://192.168.200.61:$port"
|
||||
advertise_transport = "direct_http"
|
||||
connectivity_mode = "private_lan"
|
||||
nat_type = "none"
|
||||
region = "docker-test"
|
||||
production_forwarding = $true
|
||||
}
|
||||
environment = @{}
|
||||
} | Out-Null
|
||||
}
|
||||
|
||||
function Get-SmokeRouteLabel {
|
||||
param([object]$RouteIntent)
|
||||
if ($null -eq $RouteIntent -or $null -eq $RouteIntent.PSObject.Properties["policy"]) { return "" }
|
||||
if ($null -eq $RouteIntent.policy -or $null -eq $RouteIntent.policy.PSObject.Properties["metadata"]) { return "" }
|
||||
$metadata = $RouteIntent.policy.metadata
|
||||
if ($null -eq $metadata) { return "" }
|
||||
$smoke = $metadata.PSObject.Properties["smoke"]
|
||||
if ($null -eq $smoke) { return "" }
|
||||
return [string]$smoke.Value
|
||||
}
|
||||
|
||||
function Clear-SmokeRouteIntents {
|
||||
$items = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/mesh/route-intents?actor_user_id=$ActorUserID").route_intents
|
||||
foreach ($item in @($items)) {
|
||||
if ([string]$item.lifecycle_status -ne "active" -or [string]$item.service_class -ne "vpn_packets") { continue }
|
||||
if ((Get-SmokeRouteLabel -RouteIntent $item) -ne "c18z14_live_service_channel_active_quality_shift") { continue }
|
||||
Invoke-Api -Method POST -Path "/clusters/$ClusterID/mesh/route-intents/$($item.id)/expire" -Body @{ actor_user_id = $ActorUserID } | Out-Null
|
||||
}
|
||||
}
|
||||
|
||||
function New-RouteIntent {
|
||||
param([string]$SourceNodeID, [string]$DestinationNodeID, [string[]]$Hops, [int]$Priority, [string]$Label)
|
||||
$expiresAt = (Get-Date).ToUniversalTime().AddMinutes(10).ToString("o")
|
||||
return Invoke-Api -Method POST -Path "/clusters/$ClusterID/mesh/route-intents" -Body @{
|
||||
actor_user_id = $ActorUserID
|
||||
source_selector = @{ node_id = $SourceNodeID }
|
||||
destination_selector = @{ node_id = $DestinationNodeID }
|
||||
service_class = "vpn_packets"
|
||||
priority = $Priority
|
||||
policy = @{
|
||||
synthetic_enabled = $true
|
||||
route_version = "$runId-$Label"
|
||||
policy_version = "$runId-$Label"
|
||||
peer_directory_version = "$runId-$Label"
|
||||
hops = @($Hops)
|
||||
allowed_channels = @("vpn_packet", "fabric_control")
|
||||
max_ttl = 8
|
||||
max_hops = 8
|
||||
expires_at = $expiresAt
|
||||
metadata = @{
|
||||
smoke = "c18z14_live_service_channel_active_quality_shift"
|
||||
run_id = $runId
|
||||
label = $Label
|
||||
active_quality_shift_smoke = $true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function Get-SyntheticConfig { param([string]$NodeID) return Invoke-Api -Method GET -Path "/clusters/$ClusterID/nodes/$NodeID/mesh/synthetic-config?actor_user_id=$ActorUserID" }
|
||||
function Get-LatestHeartbeat { param([string]$NodeID) return (Invoke-Api -Method GET -Path "/clusters/$ClusterID/nodes/$NodeID/heartbeats?actor_user_id=$ActorUserID&limit=1").heartbeats[0] }
|
||||
function Get-LatestRuntimeReport {
|
||||
param([string]$NodeID)
|
||||
$hb = Get-LatestHeartbeat -NodeID $NodeID
|
||||
return @{ heartbeat = $hb; report = $hb.metadata.fabric_service_channel_runtime_report }
|
||||
}
|
||||
|
||||
function Wait-ForRuntimeReady {
|
||||
param([string]$NodeID, [int]$MinRoutes, [int]$TimeoutSeconds = 90)
|
||||
$deadline = (Get-Date).AddSeconds($TimeoutSeconds)
|
||||
do {
|
||||
$latest = Get-LatestRuntimeReport -NodeID $NodeID
|
||||
if ($null -ne $latest.report -and $latest.report.enabled -eq $true -and $latest.report.production_payload_forwarding -eq $true -and [int]$latest.report.route_candidate_total -ge $MinRoutes) {
|
||||
return $latest
|
||||
}
|
||||
Start-Sleep -Seconds 2
|
||||
} while ((Get-Date) -lt $deadline)
|
||||
throw "Timed out waiting for production service-channel runtime ready on node $NodeID"
|
||||
}
|
||||
|
||||
function Wait-ForRuntimeConfigVersion {
|
||||
param([string]$NodeID, [string]$ConfigVersion, [int]$TimeoutSeconds = 90)
|
||||
$deadline = (Get-Date).AddSeconds($TimeoutSeconds)
|
||||
do {
|
||||
$latest = Get-LatestRuntimeReport -NodeID $NodeID
|
||||
if ($null -ne $latest.report -and [string]$latest.report.config_version -ge $ConfigVersion) { return $latest }
|
||||
Start-Sleep -Seconds 2
|
||||
} while ((Get-Date) -lt $deadline)
|
||||
throw "Timed out waiting for node $NodeID to load synthetic config $ConfigVersion"
|
||||
}
|
||||
|
||||
function Wait-ForRouteIntentVisible {
|
||||
param([string]$NodeID, [string[]]$RouteIDs, [int]$TimeoutSeconds = 60)
|
||||
$deadline = (Get-Date).AddSeconds($TimeoutSeconds)
|
||||
do {
|
||||
$config = Get-SyntheticConfig -NodeID $NodeID
|
||||
$present = @($config.synthetic_mesh_config.routes | Where-Object { $RouteIDs -contains $_.route_id })
|
||||
if ($present.Count -ge $RouteIDs.Count) { return $config }
|
||||
Start-Sleep -Seconds 2
|
||||
} while ((Get-Date) -lt $deadline)
|
||||
throw "Timed out waiting for routes '$($RouteIDs -join ",")' in synthetic config for node $NodeID"
|
||||
}
|
||||
|
||||
function Wait-ForRouteIntentNotVisible {
|
||||
param([string]$NodeID, [string]$RouteID, [int]$TimeoutSeconds = 90)
|
||||
$deadline = (Get-Date).AddSeconds($TimeoutSeconds)
|
||||
do {
|
||||
$config = Get-SyntheticConfig -NodeID $NodeID
|
||||
$present = @($config.synthetic_mesh_config.routes | Where-Object { $_.route_id -eq $RouteID })
|
||||
if ($present.Count -eq 0) { return $config }
|
||||
Start-Sleep -Seconds 2
|
||||
} while ((Get-Date) -lt $deadline)
|
||||
throw "Timed out waiting for route '$RouteID' to disappear from synthetic config for node $NodeID"
|
||||
}
|
||||
|
||||
function Wait-ForRouteFeedback {
|
||||
param([string]$ReporterNodeID, [string]$RouteID, [int]$TimeoutSeconds = 120)
|
||||
$deadline = (Get-Date).AddSeconds($TimeoutSeconds)
|
||||
do {
|
||||
$response = Invoke-Api -Method GET -Path "/clusters/$ClusterID/fabric/service-channels/route-feedback?actor_user_id=$ActorUserID&reporter_node_id=$ReporterNodeID&route_id=$RouteID&service_class=vpn_packets"
|
||||
$item = @($response.route_feedback | Where-Object { $_.route_id -eq $RouteID -and $_.feedback_status -eq "healthy" }) | Select-Object -First 1
|
||||
if ($null -ne $item) { return $item }
|
||||
Start-Sleep -Seconds 2
|
||||
} while ((Get-Date) -lt $deadline)
|
||||
throw "Timed out waiting for live route feedback for route '$RouteID'"
|
||||
}
|
||||
|
||||
function Wait-ForQualityPreferenceApplied {
|
||||
param([string]$NodeID, [int]$TimeoutSeconds = 120)
|
||||
$deadline = (Get-Date).AddSeconds($TimeoutSeconds)
|
||||
do {
|
||||
$latest = Get-LatestRuntimeReport -NodeID $NodeID
|
||||
$count = Get-ObjectPropertyValue -Object $latest.report.ingress -Name "route_quality_preference_count"
|
||||
if ($null -ne $count -and [int]$count -gt 0) { return $latest }
|
||||
Start-Sleep -Seconds 2
|
||||
} while ((Get-Date) -lt $deadline)
|
||||
throw "Timed out waiting for route quality preferences on node $NodeID"
|
||||
}
|
||||
|
||||
function New-ServiceChannelLease {
|
||||
param([string]$EntryNodeID, [string]$ExitNodeID)
|
||||
return (Invoke-Api -Method POST -Path "/clusters/$ClusterID/fabric/service-channels/leases" -Body @{
|
||||
actor_user_id = $ActorUserID
|
||||
organization_id = "org-c18z14-smoke"
|
||||
user_id = $ActorUserID
|
||||
resource_id = $resourceId
|
||||
service_class = "vpn_packets"
|
||||
entry_node_ids = @($EntryNodeID)
|
||||
exit_node_ids = @($ExitNodeID)
|
||||
preferred_entry_node_id = $EntryNodeID
|
||||
preferred_exit_node_id = $ExitNodeID
|
||||
allowed_channels = @("vpn_packet", "bulk", "control")
|
||||
ttl_seconds = 300
|
||||
metadata = @{ smoke = "c18z14_live_service_channel_active_quality_shift"; run_id = $runId }
|
||||
}).fabric_service_channel_lease
|
||||
}
|
||||
|
||||
function ConvertTo-Base64UrlJson {
|
||||
param([object]$Value)
|
||||
$json = $Value | ConvertTo-Json -Depth 100 -Compress
|
||||
$bytes = [System.Text.Encoding]::UTF8.GetBytes($json)
|
||||
return [Convert]::ToBase64String($bytes).TrimEnd("=").Replace("+", "-").Replace("/", "_")
|
||||
}
|
||||
|
||||
function ConvertTo-WebSocketURL {
|
||||
param([string]$URL)
|
||||
if ($URL.StartsWith("https://")) { return "wss://" + $URL.Substring("https://".Length) }
|
||||
if ($URL.StartsWith("http://")) { return "ws://" + $URL.Substring("http://".Length) }
|
||||
return $URL
|
||||
}
|
||||
|
||||
function New-TestIPv4UDPPacket {
|
||||
param([int]$SourcePort)
|
||||
$payload = [System.Text.Encoding]::ASCII.GetBytes("c18z14-$SourcePort")
|
||||
$totalLength = 20 + 8 + $payload.Length
|
||||
$packet = New-Object byte[] $totalLength
|
||||
$packet[0] = 0x45
|
||||
$packet[2] = [byte](($totalLength -shr 8) -band 0xff)
|
||||
$packet[3] = [byte]($totalLength -band 0xff)
|
||||
$packet[8] = 64
|
||||
$packet[9] = 17
|
||||
$packet[12] = 10; $packet[13] = 18; $packet[14] = 14; $packet[15] = 10
|
||||
$packet[16] = 10; $packet[17] = 18; $packet[18] = 14; $packet[19] = 20
|
||||
$udpOffset = 20
|
||||
$destPort = 3389
|
||||
$udpLength = 8 + $payload.Length
|
||||
$packet[$udpOffset] = [byte](($SourcePort -shr 8) -band 0xff)
|
||||
$packet[$udpOffset + 1] = [byte]($SourcePort -band 0xff)
|
||||
$packet[$udpOffset + 2] = [byte](($destPort -shr 8) -band 0xff)
|
||||
$packet[$udpOffset + 3] = [byte]($destPort -band 0xff)
|
||||
$packet[$udpOffset + 4] = [byte](($udpLength -shr 8) -band 0xff)
|
||||
$packet[$udpOffset + 5] = [byte]($udpLength -band 0xff)
|
||||
[Array]::Copy($payload, 0, $packet, 28, $payload.Length)
|
||||
return $packet
|
||||
}
|
||||
|
||||
function New-PacketBatchBody {
|
||||
param([byte[][]]$Packets)
|
||||
$stream = [System.IO.MemoryStream]::new()
|
||||
foreach ($packet in $Packets) {
|
||||
$length = $packet.Length
|
||||
$stream.WriteByte([byte](($length -shr 24) -band 0xff))
|
||||
$stream.WriteByte([byte](($length -shr 16) -band 0xff))
|
||||
$stream.WriteByte([byte](($length -shr 8) -band 0xff))
|
||||
$stream.WriteByte([byte]($length -band 0xff))
|
||||
$stream.Write($packet, 0, $packet.Length)
|
||||
}
|
||||
return $stream.ToArray()
|
||||
}
|
||||
|
||||
function Open-ServiceChannelWebSocket {
|
||||
param([object]$Lease)
|
||||
$path = $Lease.entry_http.websocket_path_template.Replace("{cluster_id}", $ClusterID).Replace("{channel_id}", $Lease.channel_id).Replace("{resource_id}", $resourceId)
|
||||
$url = ConvertTo-WebSocketURL -URL "$EntryBaseUrl$path"
|
||||
$socket = [System.Net.WebSockets.ClientWebSocket]::new()
|
||||
[void]$socket.Options.SetRequestHeader("X-RAP-Service-Channel-Token", [string]$Lease.token.token)
|
||||
[void]$socket.Options.SetRequestHeader("X-RAP-Fabric-Channel-ID", [string]$Lease.channel_id)
|
||||
[void]$socket.Options.SetRequestHeader("X-RAP-Service-Class", "vpn_packets")
|
||||
[void]$socket.Options.SetRequestHeader("X-RAP-Channel-Class", "vpn_packet")
|
||||
[void]$socket.Options.SetRequestHeader("X-RAP-Service-Channel-Authority-Payload", (ConvertTo-Base64UrlJson -Value $Lease.authority_payload))
|
||||
[void]$socket.Options.SetRequestHeader("X-RAP-Service-Channel-Authority-Signature", (ConvertTo-Base64UrlJson -Value $Lease.authority_signature))
|
||||
$cts = [System.Threading.CancellationTokenSource]::new([TimeSpan]::FromSeconds(30))
|
||||
[void]$socket.ConnectAsync([Uri]$url, $cts.Token).GetAwaiter().GetResult()
|
||||
$cts.Dispose()
|
||||
return @{
|
||||
Socket = $socket
|
||||
Url = $url
|
||||
SentBatches = 0
|
||||
SentPackets = 0
|
||||
}
|
||||
}
|
||||
|
||||
function Send-ServiceChannelWebSocketBatches {
|
||||
param([object]$Session, [int]$Count, [int]$PortBase)
|
||||
for ($batch = 0; $batch -lt $Count; $batch++) {
|
||||
$packets = @()
|
||||
for ($i = 0; $i -lt $PacketsPerBatch; $i++) {
|
||||
$packets += ,(New-TestIPv4UDPPacket -SourcePort ($PortBase + ($batch * 100) + $i))
|
||||
}
|
||||
$body = New-PacketBatchBody -Packets $packets
|
||||
$segment = [ArraySegment[byte]]::new($body)
|
||||
$cts = [System.Threading.CancellationTokenSource]::new([TimeSpan]::FromSeconds(30))
|
||||
[void]$Session.Socket.SendAsync($segment, [System.Net.WebSockets.WebSocketMessageType]::Binary, $true, $cts.Token).GetAwaiter().GetResult()
|
||||
$cts.Dispose()
|
||||
$Session.SentBatches++
|
||||
$Session.SentPackets += $packets.Count
|
||||
if ($BatchDelayMilliseconds -gt 0) { Start-Sleep -Milliseconds $BatchDelayMilliseconds }
|
||||
}
|
||||
Start-Sleep -Milliseconds 500
|
||||
}
|
||||
|
||||
function Close-ServiceChannelWebSocket {
|
||||
param([object]$Session)
|
||||
if ($null -eq $Session -or $null -eq $Session.Socket) { return }
|
||||
try {
|
||||
if ($Session.Socket.State -eq [System.Net.WebSockets.WebSocketState]::Open) {
|
||||
$cts = [System.Threading.CancellationTokenSource]::new([TimeSpan]::FromSeconds(10))
|
||||
[void]$Session.Socket.CloseOutputAsync([System.Net.WebSockets.WebSocketCloseStatus]::NormalClosure, "c18z14 sent", $cts.Token).GetAwaiter().GetResult()
|
||||
$cts.Dispose()
|
||||
}
|
||||
} finally {
|
||||
$Session.Socket.Dispose()
|
||||
}
|
||||
}
|
||||
|
||||
function Get-ExitQueueDepth {
|
||||
param([string]$NodeID, [string]$VPNConnectionID)
|
||||
$latest = Get-LatestRuntimeReport -NodeID $NodeID
|
||||
$queueKey = "$VPNConnectionID`:client_to_gateway"
|
||||
$depths = $latest.report.inbox.queue_depths
|
||||
if ($null -eq $depths) { return 0 }
|
||||
$prop = $depths.PSObject.Properties[$queueKey]
|
||||
if ($null -eq $prop) { return 0 }
|
||||
return [int]$prop.Value
|
||||
}
|
||||
|
||||
function Wait-ForExitQueueDepth {
|
||||
param([string]$NodeID, [string]$VPNConnectionID, [int]$MinDepth, [int]$TimeoutSeconds = 90)
|
||||
$deadline = (Get-Date).AddSeconds($TimeoutSeconds)
|
||||
do {
|
||||
$depth = Get-ExitQueueDepth -NodeID $NodeID -VPNConnectionID $VPNConnectionID
|
||||
if ($depth -ge $MinDepth) { return $depth }
|
||||
Start-Sleep -Seconds 2
|
||||
} while ((Get-Date) -lt $deadline)
|
||||
throw "Timed out waiting for exit queue depth >= $MinDepth on node $NodeID"
|
||||
}
|
||||
|
||||
function Get-IngressFlowDropped {
|
||||
param([string]$NodeID)
|
||||
$latest = Get-LatestRuntimeReport -NodeID $NodeID
|
||||
$flowScheduler = Get-ObjectPropertyValue -Object $latest.report.ingress -Name "flow_scheduler"
|
||||
$dropped = Get-ObjectPropertyValue -Object $flowScheduler -Name "dropped"
|
||||
if ($null -eq $dropped) { return 0 }
|
||||
return [int]$dropped
|
||||
}
|
||||
|
||||
function Get-BackendClientGatewayDepth {
|
||||
param([string]$VPNConnectionID)
|
||||
$stats = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/vpn-connections/$VPNConnectionID/tunnel/stats").vpn_packet_stats
|
||||
$queue = $stats.client_to_gateway
|
||||
if ($null -eq $queue) { return 0 }
|
||||
$depth = Get-ObjectPropertyValue -Object $queue -Name "queue_depth"
|
||||
if ($null -eq $depth) { return 0 }
|
||||
return [int]$depth
|
||||
}
|
||||
|
||||
function Invoke-RemoteDocker {
|
||||
param([string]$Command)
|
||||
& ssh $DockerSSH $Command
|
||||
if ($LASTEXITCODE -ne 0) { throw "ssh $DockerSSH command failed: $Command" }
|
||||
}
|
||||
|
||||
function Stop-TestUpdaters { Invoke-RemoteDocker -Command "docker stop rap_host_agent_updater_test-1 rap_host_agent_updater_test-2 rap_host_agent_updater_test-3 >/dev/null 2>&1 || true" }
|
||||
function Start-TestUpdaters { Invoke-RemoteDocker -Command "docker start rap_host_agent_updater_test-1 rap_host_agent_updater_test-2 rap_host_agent_updater_test-3 >/dev/null 2>&1 || true" }
|
||||
|
||||
$entryNode = Get-NodeByName -Name $EntryNodeName
|
||||
$relayNode = Get-NodeByName -Name $RelayNodeName
|
||||
$exitNode = Get-NodeByName -Name $ExitNodeName
|
||||
$slowInitialRouteID = ""
|
||||
$fastRouteID = ""
|
||||
$slowCandidateRouteID = ""
|
||||
$session = $null
|
||||
$updatersStopped = $false
|
||||
$result = $null
|
||||
|
||||
try {
|
||||
Stop-TestUpdaters
|
||||
$updatersStopped = $true
|
||||
Enable-TestMeshListener -Node $entryNode
|
||||
Enable-TestMeshListener -Node $relayNode
|
||||
Enable-TestMeshListener -Node $exitNode
|
||||
Clear-SmokeRouteIntents
|
||||
|
||||
$slowInitial = New-RouteIntent -SourceNodeID $entryNode.id -DestinationNodeID $exitNode.id -Hops @($entryNode.id, $relayNode.id, $exitNode.id) -Priority 1999999960 -Label "slow-initial"
|
||||
$fast = New-RouteIntent -SourceNodeID $entryNode.id -DestinationNodeID $exitNode.id -Hops @($entryNode.id, $exitNode.id) -Priority 1999999950 -Label "fast-live"
|
||||
$slowInitialRouteID = $slowInitial.route_intent.id
|
||||
$fastRouteID = $fast.route_intent.id
|
||||
|
||||
$visibleConfig = Wait-ForRouteIntentVisible -NodeID $entryNode.id -RouteIDs @($slowInitialRouteID, $fastRouteID)
|
||||
$exitVisibleConfig = Wait-ForRouteIntentVisible -NodeID $exitNode.id -RouteIDs @($slowInitialRouteID, $fastRouteID)
|
||||
$readyBefore = Wait-ForRuntimeReady -NodeID $entryNode.id -MinRoutes 2
|
||||
$exitReadyBefore = Wait-ForRuntimeReady -NodeID $exitNode.id -MinRoutes 0
|
||||
$loadedConfig = Wait-ForRuntimeConfigVersion -NodeID $entryNode.id -ConfigVersion $visibleConfig.synthetic_mesh_config.config_version
|
||||
$exitLoadedConfig = Wait-ForRuntimeConfigVersion -NodeID $exitNode.id -ConfigVersion $exitVisibleConfig.synthetic_mesh_config.config_version
|
||||
|
||||
$lease = New-ServiceChannelLease -EntryNodeID $entryNode.id -ExitNodeID $exitNode.id
|
||||
if ($lease.status -ne "ready" -or [string]$lease.primary_route.route_id -ne $slowInitialRouteID) {
|
||||
throw "Initial lease should select slow route '$slowInitialRouteID': status=$($lease.status) route=$($lease.primary_route.route_id)"
|
||||
}
|
||||
|
||||
$baselineExitDepth = Get-ExitQueueDepth -NodeID $exitNode.id -VPNConnectionID $resourceId
|
||||
$baselineBackendDepth = Get-BackendClientGatewayDepth -VPNConnectionID $resourceId
|
||||
$baselineDropped = Get-IngressFlowDropped -NodeID $entryNode.id
|
||||
$session = Open-ServiceChannelWebSocket -Lease $lease
|
||||
|
||||
Send-ServiceChannelWebSocketBatches -Session $session -Count $InitialBatchCount -PortBase 62000
|
||||
$initialPackets = $InitialBatchCount * $PacketsPerBatch
|
||||
$initialExitDepth = Wait-ForExitQueueDepth -NodeID $exitNode.id -VPNConnectionID $resourceId -MinDepth ($baselineExitDepth + $initialPackets)
|
||||
$initialRuntime = Get-LatestRuntimeReport -NodeID $entryNode.id
|
||||
if ([string]$initialRuntime.report.ingress.last_selected_route_id -ne $slowInitialRouteID) {
|
||||
throw "Initial active websocket route was '$($initialRuntime.report.ingress.last_selected_route_id)', want slow route '$slowInitialRouteID'"
|
||||
}
|
||||
|
||||
$expiredSlowInitial = Invoke-Api -Method POST -Path "/clusters/$ClusterID/mesh/route-intents/$slowInitialRouteID/expire" -Body @{ actor_user_id = $ActorUserID }
|
||||
$notVisibleConfig = Wait-ForRouteIntentNotVisible -NodeID $entryNode.id -RouteID $slowInitialRouteID
|
||||
$fastOnlyRuntime = Wait-ForRuntimeConfigVersion -NodeID $entryNode.id -ConfigVersion $notVisibleConfig.synthetic_mesh_config.config_version
|
||||
|
||||
Send-ServiceChannelWebSocketBatches -Session $session -Count $LearningBatchCount -PortBase 64000
|
||||
$learningPackets = $LearningBatchCount * $PacketsPerBatch
|
||||
$learnedExitDepth = Wait-ForExitQueueDepth -NodeID $exitNode.id -VPNConnectionID $resourceId -MinDepth ($baselineExitDepth + $initialPackets + $learningPackets) -TimeoutSeconds 120
|
||||
$fastFeedback = Wait-ForRouteFeedback -ReporterNodeID $entryNode.id -RouteID $fastRouteID -TimeoutSeconds 120
|
||||
|
||||
$slowCandidate = New-RouteIntent -SourceNodeID $entryNode.id -DestinationNodeID $exitNode.id -Hops @($entryNode.id, $relayNode.id, $exitNode.id) -Priority 1999999960 -Label "slow-candidate"
|
||||
$slowCandidateRouteID = $slowCandidate.route_intent.id
|
||||
$candidateVisibleConfig = Wait-ForRouteIntentVisible -NodeID $entryNode.id -RouteIDs @($slowCandidateRouteID, $fastRouteID)
|
||||
$candidateLoadedConfig = Wait-ForRuntimeConfigVersion -NodeID $entryNode.id -ConfigVersion $candidateVisibleConfig.synthetic_mesh_config.config_version
|
||||
$qualityRuntime = Wait-ForQualityPreferenceApplied -NodeID $entryNode.id -TimeoutSeconds 120
|
||||
|
||||
Send-ServiceChannelWebSocketBatches -Session $session -Count $PostChurnBatchCount -PortBase 67000
|
||||
$postChurnPackets = $PostChurnBatchCount * $PacketsPerBatch
|
||||
$finalExitDepth = Wait-ForExitQueueDepth -NodeID $exitNode.id -VPNConnectionID $resourceId -MinDepth ($baselineExitDepth + $initialPackets + $learningPackets + $postChurnPackets) -TimeoutSeconds 120
|
||||
$finalRuntime = Get-LatestRuntimeReport -NodeID $entryNode.id
|
||||
if ([string]$finalRuntime.report.ingress.last_selected_route_id -ne $fastRouteID) {
|
||||
throw "Post-churn active websocket route was '$($finalRuntime.report.ingress.last_selected_route_id)', want learned fast route '$fastRouteID'"
|
||||
}
|
||||
|
||||
$sessionURL = $session.Url
|
||||
$sessionSentBatches = $session.SentBatches
|
||||
$sessionSentPackets = $session.SentPackets
|
||||
Close-ServiceChannelWebSocket -Session $session
|
||||
$session = $null
|
||||
|
||||
$finalDropped = Get-IngressFlowDropped -NodeID $entryNode.id
|
||||
$finalBackendDepth = Get-BackendClientGatewayDepth -VPNConnectionID $resourceId
|
||||
$expiredFastFeedback = Invoke-Api -Method POST -Path "/clusters/$ClusterID/fabric/service-channels/route-feedback/expire" -Body @{
|
||||
actor_user_id = $ActorUserID
|
||||
reporter_node_id = $entryNode.id
|
||||
route_id = $fastRouteID
|
||||
service_class = "vpn_packets"
|
||||
reason = "c18z14 active quality shift smoke cleanup"
|
||||
}
|
||||
$expiredSlowCandidate = Invoke-Api -Method POST -Path "/clusters/$ClusterID/mesh/route-intents/$slowCandidateRouteID/expire" -Body @{ actor_user_id = $ActorUserID }
|
||||
$expiredFast = Invoke-Api -Method POST -Path "/clusters/$ClusterID/mesh/route-intents/$fastRouteID/expire" -Body @{ actor_user_id = $ActorUserID }
|
||||
|
||||
$result = [ordered]@{
|
||||
schema_version = "c18z14.live_service_channel_active_quality_shift_smoke.v1"
|
||||
run_id = $runId
|
||||
base_url = $ApiBaseUrl
|
||||
entry_base_url = $EntryBaseUrl
|
||||
cluster_id = $ClusterID
|
||||
entry_node = @{ name = $entryNode.name; id = $entryNode.id }
|
||||
relay_node = @{ name = $relayNode.name; id = $relayNode.id }
|
||||
exit_node = @{ name = $exitNode.name; id = $exitNode.id }
|
||||
resource_id = $resourceId
|
||||
route_intents = @{
|
||||
slow_initial_route_id = $slowInitialRouteID
|
||||
fast_route_id = $fastRouteID
|
||||
slow_candidate_route_id = $slowCandidateRouteID
|
||||
slow_initial_status = $expiredSlowInitial.route_intent.lifecycle_status
|
||||
slow_candidate_status = $expiredSlowCandidate.route_intent.lifecycle_status
|
||||
fast_status = $expiredFast.route_intent.lifecycle_status
|
||||
}
|
||||
lease = @{
|
||||
status = $lease.status
|
||||
channel_id = $lease.channel_id
|
||||
primary_route_id = $lease.primary_route.route_id
|
||||
}
|
||||
active_websocket = @{
|
||||
url = $sessionURL
|
||||
initial_batches = $InitialBatchCount
|
||||
learning_batches = $LearningBatchCount
|
||||
post_churn_batches = $PostChurnBatchCount
|
||||
packets_per_batch = $PacketsPerBatch
|
||||
sent_batches = $sessionSentBatches
|
||||
sent_packets = $sessionSentPackets
|
||||
initial_route_id = $initialRuntime.report.ingress.last_selected_route_id
|
||||
final_route_id = $finalRuntime.report.ingress.last_selected_route_id
|
||||
}
|
||||
live_feedback = @{
|
||||
route_id = $fastFeedback.route_id
|
||||
feedback_status = $fastFeedback.feedback_status
|
||||
score_adjustment = $fastFeedback.score_adjustment
|
||||
reasons = $fastFeedback.reasons
|
||||
last_send_duration_ms = $fastFeedback.last_send_duration_ms
|
||||
expire_result = $expiredFastFeedback.route_feedback_expire
|
||||
}
|
||||
exit_queue = @{
|
||||
baseline_depth = $baselineExitDepth
|
||||
initial_depth = $initialExitDepth
|
||||
learned_depth = $learnedExitDepth
|
||||
final_depth = $finalExitDepth
|
||||
}
|
||||
backend_fallback_queue = @{ baseline_depth = $baselineBackendDepth; depth = $finalBackendDepth }
|
||||
flow_drops = @{ baseline = $baselineDropped; final = $finalDropped; delta = ($finalDropped - $baselineDropped) }
|
||||
passed = $true
|
||||
checks = [ordered]@{
|
||||
production_forwarding_ready = ($readyBefore.report.production_payload_forwarding -eq $true)
|
||||
exit_production_forwarding_ready = ($exitReadyBefore.report.production_payload_forwarding -eq $true)
|
||||
entry_runtime_loaded_visible_config = ([string]$loadedConfig.report.config_version -ge [string]$visibleConfig.synthetic_mesh_config.config_version)
|
||||
exit_runtime_loaded_visible_config = ([string]$exitLoadedConfig.report.config_version -ge [string]$exitVisibleConfig.synthetic_mesh_config.config_version)
|
||||
initial_lease_selected_slow_route = ([string]$lease.primary_route.route_id -eq $slowInitialRouteID)
|
||||
active_websocket_started_on_slow_route = ([string]$initialRuntime.report.ingress.last_selected_route_id -eq $slowInitialRouteID)
|
||||
fast_only_runtime_loaded_after_slow_expire = ([string]$fastOnlyRuntime.report.config_version -ge [string]$notVisibleConfig.synthetic_mesh_config.config_version)
|
||||
backend_persisted_live_fast_feedback = ([string]$fastFeedback.route_id -eq $fastRouteID -and [string]$fastFeedback.feedback_status -eq "healthy")
|
||||
candidate_runtime_loaded = ([string]$candidateLoadedConfig.report.config_version -ge [string]$candidateVisibleConfig.synthetic_mesh_config.config_version)
|
||||
node_applied_quality_preference = ([int]$qualityRuntime.report.ingress.route_quality_preference_count -gt 0)
|
||||
active_websocket_stayed_on_learned_fast_route_after_churn = ([string]$finalRuntime.report.ingress.last_selected_route_id -eq $fastRouteID)
|
||||
all_packets_reached_exit = ($finalExitDepth -ge ($baselineExitDepth + $initialPackets + $learningPackets + $postChurnPackets))
|
||||
no_backend_fallback_used = ($finalBackendDepth -eq $baselineBackendDepth)
|
||||
no_flow_drops = (($finalDropped - $baselineDropped) -eq 0)
|
||||
route_intents_expired = ($expiredFast.route_intent.lifecycle_status -eq "expired" -and $expiredSlowCandidate.route_intent.lifecycle_status -eq "expired")
|
||||
}
|
||||
telemetry = @{
|
||||
initial_entry_ingress = $initialRuntime.report.ingress
|
||||
quality_entry_ingress = $qualityRuntime.report.ingress
|
||||
final_entry_ingress = $finalRuntime.report.ingress
|
||||
}
|
||||
}
|
||||
$failedChecks = @($result.checks.GetEnumerator() | Where-Object { $_.Value -ne $true })
|
||||
if ($failedChecks.Count -gt 0) { throw "C18Z14 failed checks: $($failedChecks.Name -join ', ')" }
|
||||
}
|
||||
finally {
|
||||
if ($session) { try { Close-ServiceChannelWebSocket -Session $session } catch {} }
|
||||
if ($slowInitialRouteID) { try { Invoke-Api -Method POST -Path "/clusters/$ClusterID/mesh/route-intents/$slowInitialRouteID/expire" -Body @{ actor_user_id = $ActorUserID } | Out-Null } catch {} }
|
||||
if ($slowCandidateRouteID) { try { Invoke-Api -Method POST -Path "/clusters/$ClusterID/mesh/route-intents/$slowCandidateRouteID/expire" -Body @{ actor_user_id = $ActorUserID } | Out-Null } catch {} }
|
||||
if ($fastRouteID) {
|
||||
try { Invoke-Api -Method POST -Path "/clusters/$ClusterID/mesh/route-intents/$fastRouteID/expire" -Body @{ actor_user_id = $ActorUserID } | Out-Null } catch {}
|
||||
try {
|
||||
Invoke-Api -Method POST -Path "/clusters/$ClusterID/fabric/service-channels/route-feedback/expire" -Body @{
|
||||
actor_user_id = $ActorUserID
|
||||
reporter_node_id = $entryNode.id
|
||||
route_id = $fastRouteID
|
||||
service_class = "vpn_packets"
|
||||
reason = "c18z14 active quality shift smoke cleanup"
|
||||
} | Out-Null
|
||||
} catch {}
|
||||
}
|
||||
if ($updatersStopped) { try { Start-TestUpdaters } catch { Write-Warning "Could not restart test updaters: $($_.Exception.Message)" } }
|
||||
}
|
||||
|
||||
$resultFullPath = Join-Path $repoRoot $ResultPath
|
||||
$resultDir = Split-Path -Parent $resultFullPath
|
||||
if (-not (Test-Path $resultDir)) { New-Item -ItemType Directory -Path $resultDir | Out-Null }
|
||||
$result | ConvertTo-Json -Depth 100 | Set-Content -Path $resultFullPath -Encoding UTF8
|
||||
Write-Host "C18Z14 live service-channel active quality shift smoke passed. Result: $resultFullPath"
|
||||
$result
|
||||
Reference in New Issue
Block a user