336 lines
13 KiB
PowerShell
336 lines
13 KiB
PowerShell
param(
|
|
[string]$ApiBaseUrl = "http://192.168.200.61:18121/api/v1",
|
|
[string]$ClusterID = "cfc0743d-d960-49fb-9de8-96e063d5e4aa",
|
|
[string]$ActorUserID = "f67d943f-5397-4b3a-a229-695fe67ad700",
|
|
[string]$EntryNodeName = "test-1",
|
|
[string]$ExitNodeName = "test-2",
|
|
[string]$RequiredNodeVersion = "0.2.177",
|
|
[string]$ResultPath = "artifacts\c18w-service-channel-route-manager-smoke-result.json"
|
|
)
|
|
|
|
Set-StrictMode -Version Latest
|
|
$ErrorActionPreference = "Stop"
|
|
|
|
$repoRoot = (Resolve-Path (Join-Path $PSScriptRoot "..\..")).ProviderPath
|
|
$runId = "c18w-" + (Get-Date -Format "yyyyMMdd-HHmmss")
|
|
|
|
function Invoke-Api {
|
|
param(
|
|
[string]$Method,
|
|
[string]$Path,
|
|
[object]$Body = $null
|
|
)
|
|
$uri = "$ApiBaseUrl$Path"
|
|
try {
|
|
if ($null -eq $Body) {
|
|
return Invoke-RestMethod -Method $Method -Uri $uri -TimeoutSec 30
|
|
}
|
|
return Invoke-RestMethod -Method $Method -Uri $uri -ContentType "application/json" -Body ($Body | ConvertTo-Json -Depth 80) -TimeoutSec 30
|
|
}
|
|
catch {
|
|
$statusCode = $null
|
|
if ($_.Exception.Response) {
|
|
$statusCode = [int]$_.Exception.Response.StatusCode
|
|
}
|
|
$details = $_.ErrorDetails.Message
|
|
if (-not $details) {
|
|
$details = $_.Exception.Message
|
|
}
|
|
throw "$Method $Path failed with HTTP $statusCode`: $details"
|
|
}
|
|
}
|
|
|
|
function Get-NodeByName {
|
|
param([string]$Name)
|
|
$nodes = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/nodes?actor_user_id=$ActorUserID").nodes
|
|
$node = @($nodes | Where-Object { $_.name -eq $Name }) | Select-Object -First 1
|
|
if ($null -eq $node) {
|
|
throw "Node '$Name' was not found in cluster $ClusterID"
|
|
}
|
|
return $node
|
|
}
|
|
|
|
function New-RouteIntent {
|
|
param(
|
|
[string]$SourceNodeID,
|
|
[string]$DestinationNodeID,
|
|
[int]$Priority,
|
|
[string]$Label
|
|
)
|
|
$expiresAt = (Get-Date).ToUniversalTime().AddMinutes(8).ToString("o")
|
|
return Invoke-Api -Method POST -Path "/clusters/$ClusterID/mesh/route-intents" -Body @{
|
|
actor_user_id = $ActorUserID
|
|
source_selector = @{
|
|
node_id = $SourceNodeID
|
|
}
|
|
destination_selector = @{
|
|
node_id = $DestinationNodeID
|
|
}
|
|
service_class = "vpn_packets"
|
|
priority = $Priority
|
|
policy = @{
|
|
synthetic_enabled = $true
|
|
route_version = "$runId-$Label"
|
|
policy_version = "$runId-$Label"
|
|
peer_directory_version = "$runId-$Label"
|
|
hops = @($SourceNodeID, $DestinationNodeID)
|
|
allowed_channels = @("vpn_packet", "fabric_control")
|
|
max_ttl = 8
|
|
max_hops = 8
|
|
expires_at = $expiresAt
|
|
metadata = @{
|
|
smoke = "c18w_service_channel_route_manager"
|
|
run_id = $runId
|
|
label = $Label
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
function Send-FeedbackHeartbeat {
|
|
param(
|
|
[string]$EntryNodeID,
|
|
[string]$BadRouteID,
|
|
[string]$GoodRouteID
|
|
)
|
|
return Invoke-Api -Method POST -Path "/clusters/$ClusterID/nodes/$EntryNodeID/heartbeats" -Body @{
|
|
health_status = "healthy"
|
|
reported_version = $RequiredNodeVersion
|
|
capabilities = @{
|
|
native_node_agent = $true
|
|
fabric_service_channel_runtime = $true
|
|
fabric_service_channel_route_manager = $true
|
|
smoke_feedback_injection = "c18w"
|
|
}
|
|
service_states = @{
|
|
smoke = "c18w_route_manager_feedback"
|
|
}
|
|
metadata = @{
|
|
fabric_service_channel_runtime_report = @{
|
|
schema_version = "c18l.fabric_service_channel_runtime_report.v1"
|
|
ingress = @{
|
|
flow_scheduler = @{
|
|
channel_stats = @{
|
|
"c18w-smoke-flow" = @{
|
|
last_route_id = $GoodRouteID
|
|
last_failed_route_id = $BadRouteID
|
|
last_error = "c18w forced stale route feedback"
|
|
consecutive_failures = 3
|
|
stall_count = 0
|
|
last_send_duration_ms = 250
|
|
route_rebuild_recommended = $true
|
|
degraded_fallback_recommended = $false
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
smoke = @{
|
|
name = "c18w_service_channel_route_manager"
|
|
run_id = $runId
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
function Send-CleanHeartbeat {
|
|
param([string]$EntryNodeID)
|
|
return Invoke-Api -Method POST -Path "/clusters/$ClusterID/nodes/$EntryNodeID/heartbeats" -Body @{
|
|
health_status = "healthy"
|
|
reported_version = $RequiredNodeVersion
|
|
capabilities = @{
|
|
native_node_agent = $true
|
|
fabric_service_channel_runtime = $true
|
|
fabric_service_channel_route_manager = $true
|
|
smoke_feedback_injection = "c18w-clean"
|
|
}
|
|
service_states = @{
|
|
smoke = "c18w_route_manager_restore"
|
|
}
|
|
metadata = @{
|
|
fabric_service_channel_runtime_report = @{
|
|
schema_version = "c18l.fabric_service_channel_runtime_report.v1"
|
|
ingress = @{
|
|
flow_scheduler = @{
|
|
channel_stats = @{}
|
|
}
|
|
}
|
|
}
|
|
smoke = @{
|
|
name = "c18w_service_channel_route_manager"
|
|
run_id = $runId
|
|
phase = "clean_after_expire"
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
function Get-SyntheticConfig {
|
|
param([string]$NodeID)
|
|
return Invoke-Api -Method GET -Path "/clusters/$ClusterID/nodes/$NodeID/mesh/synthetic-config?actor_user_id=$ActorUserID"
|
|
}
|
|
|
|
function Get-LatestTransition {
|
|
param([string]$NodeID)
|
|
$heartbeats = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/nodes/$NodeID/heartbeats?actor_user_id=$ActorUserID&limit=8").heartbeats
|
|
foreach ($heartbeat in $heartbeats) {
|
|
$transition = $null
|
|
$reportProperty = $heartbeat.metadata.PSObject.Properties["fabric_service_channel_runtime_report"]
|
|
if ($null -ne $reportProperty) {
|
|
$ingressProperty = $reportProperty.Value.PSObject.Properties["ingress"]
|
|
if ($null -ne $ingressProperty) {
|
|
$transitionProperty = $ingressProperty.Value.PSObject.Properties["route_manager_transition"]
|
|
if ($null -ne $transitionProperty) {
|
|
$transition = $transitionProperty.Value
|
|
}
|
|
}
|
|
}
|
|
if ($null -ne $transition -and $transition.schema_version -eq "rap.fabric_service_channel_route_manager_transition.v1") {
|
|
return @{
|
|
heartbeat = $heartbeat
|
|
transition = $transition
|
|
}
|
|
}
|
|
}
|
|
return $null
|
|
}
|
|
|
|
function Wait-ForTransitionStatus {
|
|
param(
|
|
[string]$NodeID,
|
|
[string[]]$Statuses,
|
|
[int]$TimeoutSeconds = 90
|
|
)
|
|
$deadline = (Get-Date).AddSeconds($TimeoutSeconds)
|
|
do {
|
|
$latest = Get-LatestTransition -NodeID $NodeID
|
|
if ($null -ne $latest -and $Statuses -contains [string]$latest.transition.status) {
|
|
return $latest
|
|
}
|
|
Start-Sleep -Seconds 2
|
|
} while ((Get-Date) -lt $deadline)
|
|
$last = Get-LatestTransition -NodeID $NodeID
|
|
throw "Timed out waiting for transition status '$($Statuses -join ',')'. Last transition: $($last | ConvertTo-Json -Depth 20 -Compress)"
|
|
}
|
|
|
|
function Wait-ForConfigDecision {
|
|
param(
|
|
[string]$NodeID,
|
|
[string]$BadRouteID,
|
|
[string]$ExpectedStatus,
|
|
[int]$TimeoutSeconds = 45
|
|
)
|
|
$deadline = (Get-Date).AddSeconds($TimeoutSeconds)
|
|
do {
|
|
$config = Get-SyntheticConfig -NodeID $NodeID
|
|
$decisions = @($config.synthetic_mesh_config.route_path_decisions.decisions)
|
|
$decision = @($decisions | Where-Object { $_.route_id -eq $BadRouteID -and $_.rebuild_status -eq $ExpectedStatus }) | Select-Object -First 1
|
|
if ($null -ne $decision) {
|
|
return @{
|
|
config = $config
|
|
decision = $decision
|
|
}
|
|
}
|
|
Start-Sleep -Seconds 2
|
|
} while ((Get-Date) -lt $deadline)
|
|
throw "Timed out waiting for rebuild_status=$ExpectedStatus for route $BadRouteID"
|
|
}
|
|
|
|
function Wait-ForNoRebuildDecision {
|
|
param(
|
|
[string]$NodeID,
|
|
[string]$BadRouteID,
|
|
[int]$TimeoutSeconds = 45
|
|
)
|
|
$deadline = (Get-Date).AddSeconds($TimeoutSeconds)
|
|
do {
|
|
$config = Get-SyntheticConfig -NodeID $NodeID
|
|
$decisions = @($config.synthetic_mesh_config.route_path_decisions.decisions)
|
|
$decision = @($decisions | Where-Object {
|
|
$statusProperty = $_.PSObject.Properties["rebuild_status"]
|
|
$_.route_id -eq $BadRouteID -and $null -ne $statusProperty -and -not [string]::IsNullOrWhiteSpace([string]$statusProperty.Value)
|
|
}) | Select-Object -First 1
|
|
if ($null -eq $decision) {
|
|
$routeDecision = @($decisions | Where-Object { $_.route_id -eq $BadRouteID }) | Select-Object -First 1
|
|
return @{
|
|
config = $config
|
|
decision = $routeDecision
|
|
}
|
|
}
|
|
Start-Sleep -Seconds 2
|
|
} while ((Get-Date) -lt $deadline)
|
|
throw "Timed out waiting for rebuild decision to clear for route $BadRouteID"
|
|
}
|
|
|
|
Write-Host "C18W smoke $runId against $ApiBaseUrl"
|
|
$entryNode = Get-NodeByName -Name $EntryNodeName
|
|
$exitNode = Get-NodeByName -Name $ExitNodeName
|
|
if ($entryNode.reported_version -ne $RequiredNodeVersion) {
|
|
throw "$EntryNodeName reports version $($entryNode.reported_version), want $RequiredNodeVersion"
|
|
}
|
|
|
|
Write-Host "Creating temporary service-channel route intents..."
|
|
$badIntent = (New-RouteIntent -SourceNodeID $entryNode.id -DestinationNodeID $exitNode.id -Priority 9700 -Label "bad").route_intent
|
|
$goodIntent = (New-RouteIntent -SourceNodeID $entryNode.id -DestinationNodeID $exitNode.id -Priority 9600 -Label "good").route_intent
|
|
|
|
Write-Host "Injecting fenced/healthy route feedback through heartbeat..."
|
|
$feedbackHeartbeat = Send-FeedbackHeartbeat -EntryNodeID $entryNode.id -BadRouteID $badIntent.id -GoodRouteID $goodIntent.id
|
|
$appliedDecision = Wait-ForConfigDecision -NodeID $entryNode.id -BadRouteID $badIntent.id -ExpectedStatus "applied"
|
|
$appliedTransition = Wait-ForTransitionStatus -NodeID $entryNode.id -Statuses @("applied_rebuild")
|
|
|
|
Write-Host "Expiring route feedback and waiting for fresh config restore..."
|
|
$expireResult = Invoke-Api -Method POST -Path "/clusters/$ClusterID/fabric/service-channels/route-feedback/expire" -Body @{
|
|
actor_user_id = $ActorUserID
|
|
reporter_node_id = $entryNode.id
|
|
route_id = $badIntent.id
|
|
service_class = "vpn_packets"
|
|
reason = "c18w smoke restore by fresh config"
|
|
}
|
|
$cleanHeartbeat = Send-CleanHeartbeat -EntryNodeID $entryNode.id
|
|
$restoredConfig = Wait-ForNoRebuildDecision -NodeID $entryNode.id -BadRouteID $badIntent.id
|
|
$restoredTransition = Wait-ForTransitionStatus -NodeID $entryNode.id -Statuses @("restored_by_new_config", "empty")
|
|
|
|
$result = [ordered]@{
|
|
schema_version = "c18w.service_channel_route_manager_smoke.v1"
|
|
run_id = $runId
|
|
api_base_url = $ApiBaseUrl
|
|
cluster_id = $ClusterID
|
|
actor_user_id = $ActorUserID
|
|
entry_node = @{
|
|
id = $entryNode.id
|
|
name = $entryNode.name
|
|
reported_version = $entryNode.reported_version
|
|
}
|
|
exit_node = @{
|
|
id = $exitNode.id
|
|
name = $exitNode.name
|
|
reported_version = $exitNode.reported_version
|
|
}
|
|
routes = @{
|
|
bad_route_id = $badIntent.id
|
|
good_route_id = $goodIntent.id
|
|
}
|
|
feedback_heartbeat_id = $feedbackHeartbeat.heartbeat.id
|
|
applied_decision = $appliedDecision.decision
|
|
applied_transition = $appliedTransition.transition
|
|
expire_result = $expireResult.route_feedback_expire
|
|
clean_heartbeat_id = $cleanHeartbeat.heartbeat.id
|
|
restored_config_generation = $restoredConfig.config.synthetic_mesh_config.config_version
|
|
restored_decision = $restoredConfig.decision
|
|
restored_transition = $restoredTransition.transition
|
|
checks = @{
|
|
applied_decision = $appliedDecision.decision.rebuild_status -eq "applied"
|
|
applied_transition = $appliedTransition.transition.status -eq "applied_rebuild"
|
|
feedback_expired = $expireResult.route_feedback_expire.expired_count -ge 1
|
|
restored_config_has_no_rebuild_decision = $true
|
|
restored_transition_seen = @("restored_by_new_config", "empty") -contains [string]$restoredTransition.transition.status
|
|
}
|
|
}
|
|
|
|
$resultDir = Split-Path -Parent (Join-Path $repoRoot $ResultPath)
|
|
New-Item -ItemType Directory -Force -Path $resultDir | Out-Null
|
|
$absoluteResultPath = Join-Path $repoRoot $ResultPath
|
|
$result | ConvertTo-Json -Depth 100 | Set-Content -Encoding UTF8 $absoluteResultPath
|
|
Write-Host "C18W smoke passed. Result: $absoluteResultPath"
|
|
$result
|