244 lines
13 KiB
PowerShell
244 lines
13 KiB
PowerShell
param(
|
|
[string]$ApiBaseUrl = "http://192.168.200.61:18121/api/v1",
|
|
[string]$ClusterID = "cfc0743d-d960-49fb-9de8-96e063d5e4aa",
|
|
[string]$ActorUserID = "f67d943f-5397-4b3a-a229-695fe67ad700",
|
|
[string]$EntryNodeName = "test-3",
|
|
[string]$RelayNodeName = "test-2",
|
|
[string]$ExitNodeName = "test-1",
|
|
[string]$DockerSSH = "test-docker",
|
|
[string]$ExpectedBackendImage = "rap-backend:fabric-service-channel-0.2.256-c18z82",
|
|
[string]$ExpectedNodeAgentImage = "rap-node-agent:0.2.270-c18z95",
|
|
[string]$ResultPath = "artifacts\c18z78-service-channel-rebuild-planner-applied-smoke-result.json"
|
|
)
|
|
|
|
Set-StrictMode -Version Latest
|
|
$ErrorActionPreference = "Stop"
|
|
|
|
$scriptDir = Split-Path -Parent $MyInvocation.MyCommand.Path
|
|
$repoRoot = (Resolve-Path (Join-Path $scriptDir "..\..")).ProviderPath
|
|
$runId = "c18z78-" + (Get-Date -Format "yyyyMMdd-HHmmss")
|
|
|
|
function Invoke-Api {
|
|
param([string]$Method, [string]$Path, [object]$Body = $null)
|
|
$params = @{ Method = $Method; Uri = "$ApiBaseUrl$Path"; TimeoutSec = 30 }
|
|
if ($null -ne $Body) {
|
|
$params.ContentType = "application/json"
|
|
$params.Body = ($Body | ConvertTo-Json -Depth 80)
|
|
}
|
|
return Invoke-RestMethod @params
|
|
}
|
|
|
|
function Get-PropertyValue {
|
|
param([object]$Item, [string]$Name, [object]$Default = $null)
|
|
if ($null -eq $Item) { return $Default }
|
|
$property = $Item.PSObject.Properties[$Name]
|
|
if ($null -eq $property) { return $Default }
|
|
return $property.Value
|
|
}
|
|
|
|
function Get-NodeByName {
|
|
param([string]$Name)
|
|
$nodes = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/nodes?actor_user_id=$ActorUserID").nodes
|
|
$node = @($nodes | Where-Object { $_.name -eq $Name }) | Select-Object -First 1
|
|
if ($null -eq $node) { throw "Node '$Name' was not found" }
|
|
return $node
|
|
}
|
|
|
|
function New-RouteIntent {
|
|
param([string]$SourceNodeID, [string]$DestinationNodeID, [string[]]$Hops)
|
|
$expiresAt = (Get-Date).ToUniversalTime().AddMinutes(5).ToString("o")
|
|
return (Invoke-Api -Method POST -Path "/clusters/$ClusterID/mesh/route-intents" -Body @{
|
|
actor_user_id = $ActorUserID
|
|
source_selector = @{ node_id = $SourceNodeID }
|
|
destination_selector = @{ node_id = $DestinationNodeID }
|
|
service_class = "vpn_packets"
|
|
priority = 2100000000
|
|
policy = @{
|
|
synthetic_enabled = $true
|
|
route_version = "$runId-primary"
|
|
policy_version = "$runId-primary"
|
|
peer_directory_version = "$runId-primary"
|
|
hops = @($Hops)
|
|
allowed_channels = @("vpn_packet", "fabric_control")
|
|
max_ttl = 8
|
|
max_hops = 8
|
|
expires_at = $expiresAt
|
|
metadata = @{ smoke = "c18z78_service_channel_rebuild_planner_applied"; run_id = $runId }
|
|
}
|
|
}).route_intent
|
|
}
|
|
|
|
function Disable-ExistingRouteIntents {
|
|
param([string]$SourceNodeID, [string]$DestinationNodeID)
|
|
$items = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/mesh/route-intents?actor_user_id=$ActorUserID").route_intents
|
|
foreach ($item in @($items)) {
|
|
if ([string](Get-PropertyValue -Item $item -Name "status" -Default "") -ne "active") { continue }
|
|
if ([string](Get-PropertyValue -Item $item -Name "service_class" -Default "") -ne "vpn_packets") { continue }
|
|
$sourceSelector = Get-PropertyValue -Item $item -Name "source_selector" -Default $null
|
|
$destinationSelector = Get-PropertyValue -Item $item -Name "destination_selector" -Default $null
|
|
if ([string](Get-PropertyValue -Item $sourceSelector -Name "node_id" -Default "") -ne $SourceNodeID) { continue }
|
|
if ([string](Get-PropertyValue -Item $destinationSelector -Name "node_id" -Default "") -ne $DestinationNodeID) { continue }
|
|
[void](Invoke-Api -Method POST -Path "/clusters/$ClusterID/mesh/route-intents/$($item.id)/disable" -Body @{
|
|
actor_user_id = $ActorUserID
|
|
reason = "c18z78 isolate rebuild-planner-applied smoke route pair"
|
|
})
|
|
}
|
|
}
|
|
|
|
function Send-DegradedHeartbeat {
|
|
param([string]$EntryNodeID, [string]$PrimaryRouteID)
|
|
$observedAt = (Get-Date).ToUniversalTime().ToString("o")
|
|
return Invoke-Api -Method POST -Path "/clusters/$ClusterID/nodes/$EntryNodeID/heartbeats" -Body @{
|
|
health_status = "healthy"
|
|
reported_version = "0.2.252-c18z78"
|
|
capabilities = @{
|
|
fabric_service_channel_runtime = $true
|
|
fabric_service_channel_route_quality_feedback = $true
|
|
smoke_feedback_injection = "c18z77"
|
|
}
|
|
service_states = @{ smoke = "c18z78_primary_degraded_alternate_after_lease" }
|
|
metadata = @{
|
|
fabric_service_channel_runtime_report = @{
|
|
schema_version = "c18l.fabric_service_channel_runtime_report.v1"
|
|
config_version = "$runId-primary"
|
|
cluster_id = $ClusterID
|
|
local_node_id = $EntryNodeID
|
|
observed_at = $observedAt
|
|
ingress = @{
|
|
flow_scheduler = @{
|
|
schema_version = "rap.fabric_flow_scheduler.v1"
|
|
service_neutral = $true
|
|
service_mode = "application_protocol_agnostic"
|
|
channel_stats = @{
|
|
"c18z78-primary-degraded" = @{
|
|
last_route_id = $PrimaryRouteID
|
|
last_failed_route_id = $PrimaryRouteID
|
|
route_generation = "$runId-primary"
|
|
last_error = "c18z78 primary route degraded; alternate added after lease"
|
|
last_send_duration_ms = 1200
|
|
consecutive_failures = 3
|
|
stall_count = 2
|
|
route_rebuild_recommended = $true
|
|
degraded_fallback_recommended = $false
|
|
quality_window_sample_count = 8
|
|
quality_window_success_count = 2
|
|
quality_window_failure_count = 3
|
|
quality_window_slow_count = 2
|
|
quality_window_drop_count = 0
|
|
quality_window_avg_latency_ms = 1200
|
|
quality_window_last_updated_at = $observedAt
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
smoke = @{ name = "c18z78_service_channel_rebuild_planner_applied"; run_id = $runId }
|
|
}
|
|
}
|
|
}
|
|
|
|
$entryNode = Get-NodeByName -Name $EntryNodeName
|
|
$relayNode = Get-NodeByName -Name $RelayNodeName
|
|
$exitNode = Get-NodeByName -Name $ExitNodeName
|
|
[void](Disable-ExistingRouteIntents -SourceNodeID $entryNode.id -DestinationNodeID $exitNode.id)
|
|
$route = New-RouteIntent -SourceNodeID $entryNode.id -DestinationNodeID $exitNode.id -Hops @($entryNode.id, $exitNode.id)
|
|
$lease = (Invoke-Api -Method POST -Path "/clusters/$ClusterID/fabric/service-channels/leases" -Body @{
|
|
actor_user_id = $ActorUserID
|
|
organization_id = "smoke-org"
|
|
user_id = "smoke-user"
|
|
resource_id = "c18z78-vpn-smoke"
|
|
service_class = "vpn_packets"
|
|
entry_node_ids = @([string]$entryNode.id)
|
|
exit_node_ids = @([string]$exitNode.id)
|
|
preferred_entry_node_id = [string]$entryNode.id
|
|
preferred_exit_node_id = [string]$exitNode.id
|
|
allowed_channels = @("vpn_packet", "fabric_control")
|
|
ttl_seconds = 90
|
|
metadata = @{ smoke = "c18z78_service_channel_rebuild_planner_applied"; run_id = $runId }
|
|
}).fabric_service_channel_lease
|
|
$primaryRouteID = [string](Get-PropertyValue -Item (Get-PropertyValue -Item $lease -Name "primary_route" -Default $null) -Name "route_id" -Default $route.id)
|
|
|
|
[void](Send-DegradedHeartbeat -EntryNodeID $entryNode.id -PrimaryRouteID $primaryRouteID)
|
|
|
|
$matchingChannel = $null
|
|
for ($i = 0; $i -lt 10; $i++) {
|
|
Start-Sleep -Seconds 3
|
|
$accessTelemetry = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/fabric/service-channels/access-telemetry?actor_user_id=$ActorUserID&limit=50").fabric_service_channel_access_telemetry
|
|
$matchingChannel = @($accessTelemetry.active_channels | Where-Object { $_.channel_id -eq $lease.channel_id }) | Select-Object -First 1
|
|
if ($null -ne $matchingChannel -and [string](Get-PropertyValue -Item $matchingChannel -Name "remediation_action" -Default "") -eq "rebuild_route") {
|
|
break
|
|
}
|
|
}
|
|
|
|
$command = Get-PropertyValue -Item $matchingChannel -Name "remediation_command" -Default $null
|
|
$commandID = [string](Get-PropertyValue -Item $command -Name "command_id" -Default "")
|
|
$alternate = New-RouteIntent -SourceNodeID $entryNode.id -DestinationNodeID $exitNode.id -Hops @($entryNode.id, $relayNode.id, $exitNode.id)
|
|
$alternateRouteID = [string]$alternate.id
|
|
[void](Invoke-Api -Method GET -Path "/clusters/$ClusterID/nodes/$($entryNode.id)/mesh/synthetic-config")
|
|
|
|
$attempts = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/fabric/service-channels/rebuild-attempts?actor_user_id=$ActorUserID&reporter_node_id=$($entryNode.id)&rebuild_request_id=$commandID&limit=10").rebuild_attempts
|
|
$attempt = @($attempts | Where-Object { $_.rebuild_request_id -eq $commandID }) | Select-Object -First 1
|
|
|
|
Start-Sleep -Seconds 30
|
|
$postAccess = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/fabric/service-channels/access-telemetry?actor_user_id=$ActorUserID&limit=50").fabric_service_channel_access_telemetry
|
|
$postChannel = @($postAccess.active_channels | Where-Object { $_.channel_id -eq $lease.channel_id }) | Select-Object -First 1
|
|
|
|
$backendLine = (& ssh $DockerSSH "docker ps --format '{{.Names}} {{.Image}} {{.Status}}' | grep '^rap_test_backend '") | Out-String
|
|
$nodeLines = (& ssh $DockerSSH "docker ps --format '{{.Names}} {{.Image}} {{.Status}}' | grep '^rap_test_node_test_'") | Out-String
|
|
$checks = [ordered]@{
|
|
backend_expected_image_deployed = $backendLine.Contains($ExpectedBackendImage)
|
|
node_agent_expected_image_deployed = $nodeLines.Contains($ExpectedNodeAgentImage)
|
|
lease_ready = ([string]$lease.status -eq "ready")
|
|
remediation_rebuild_route_visible = ($null -ne $matchingChannel -and [string](Get-PropertyValue -Item $matchingChannel -Name "remediation_action" -Default "") -eq "rebuild_route")
|
|
remediation_command_visible = ($null -ne $command -and $commandID.Length -gt 0)
|
|
durable_rebuild_intent_recorded = ($null -ne $attempt)
|
|
durable_rebuild_intent_resolved_applied = ($null -ne $attempt -and [string](Get-PropertyValue -Item $attempt -Name "rebuild_status" -Default "") -eq "applied")
|
|
durable_rebuild_intent_outcome_replacement_selected = ($null -ne $attempt -and [string](Get-PropertyValue -Item $attempt -Name "outcome" -Default "") -eq "replacement_selected")
|
|
durable_rebuild_intent_replacement_matches = ($null -ne $attempt -and [string](Get-PropertyValue -Item $attempt -Name "replacement_route_id" -Default "") -eq $alternateRouteID)
|
|
durable_rebuild_intent_source_matches = ($null -ne $attempt -and [string](Get-PropertyValue -Item $attempt -Name "decision_source" -Default "") -eq "service_channel_remediation_command")
|
|
access_telemetry_reports_planner_applied = ($null -ne $postChannel -and [string](Get-PropertyValue -Item $postChannel -Name "remediation_execution_status" -Default "") -eq "rebuild_request_applied")
|
|
}
|
|
$failed = @($checks.GetEnumerator() | Where-Object { -not $_.Value } | ForEach-Object { $_.Key })
|
|
|
|
$result = [ordered]@{
|
|
schema_version = "c18z78.service_channel_rebuild_planner_applied_smoke.v1"
|
|
run_id = $runId
|
|
cluster_id = $ClusterID
|
|
channel_id = [string]$lease.channel_id
|
|
primary_route_id = $primaryRouteID
|
|
alternate_route_id = $alternateRouteID
|
|
rebuild_request_id = $commandID
|
|
passed = ($failed.Count -eq 0)
|
|
checks = $checks
|
|
failed_checks = $failed
|
|
summary = [ordered]@{
|
|
backend_container = $backendLine.Trim()
|
|
node_containers = $nodeLines.Trim()
|
|
remediation_command = $command
|
|
rebuild_attempt = $attempt
|
|
post_channel = $postChannel
|
|
}
|
|
}
|
|
|
|
$target = Join-Path $repoRoot $ResultPath
|
|
$result | ConvertTo-Json -Depth 80 | Set-Content -Path $target -Encoding UTF8
|
|
|
|
try {
|
|
Start-Sleep -Seconds 3
|
|
Invoke-Api -Method POST -Path "/clusters/$ClusterID/fabric/service-channels/leases/cleanup" -Body @{
|
|
actor_user_id = $ActorUserID
|
|
limit = 50
|
|
} | Out-Null
|
|
} catch {
|
|
Write-Warning "cleanup failed after c18z78 smoke: $($_.Exception.Message)"
|
|
}
|
|
|
|
if (-not $result.passed) {
|
|
throw "C18Z78 rebuild planner-applied smoke failed: $($failed -join ', ')"
|
|
}
|
|
|
|
Write-Host "C18Z78 service-channel rebuild planner-applied smoke passed. Result: $target"
|
|
$result
|
|
|
|
|