Record project continuation changes
This commit is contained in:
@@ -0,0 +1,138 @@
|
||||
# Android сборка и публикация APK (локально)
|
||||
|
||||
Эти скрипты позволяют делать локальную сборку Android-клиента, сразу публиковать
|
||||
готовый APK в место, из которого веб-панель его скачивает, и при необходимости
|
||||
быстро проверять состояние окружения.
|
||||
|
||||
## Подготовка окружения
|
||||
|
||||
Сначала проверьте зависимости и переменные окружения:
|
||||
|
||||
```powershell
|
||||
pwsh -ExecutionPolicy Bypass -File scripts\android\prepare-android-build-environment.ps1 -SetEnvironment
|
||||
```
|
||||
|
||||
Скрипт проверит:
|
||||
|
||||
- `java` (JDK 17+)
|
||||
- `gradle`
|
||||
- Android SDK (по умолчанию `C:\Android\Sdk`)
|
||||
- `sdkmanager` из `cmdline-tools`
|
||||
- `platform-tools`, `platforms;android-<compileSdk>`, `build-tools;35.0.1`
|
||||
|
||||
По необходимости можно автосоздать отсутствующие SDK-компоненты:
|
||||
|
||||
```powershell
|
||||
pwsh -ExecutionPolicy Bypass -File scripts\android\prepare-android-build-environment.ps1 -InstallMissing -SetEnvironment
|
||||
```
|
||||
|
||||
## Быстрый запуск после апдейта
|
||||
|
||||
После любого обновления версии соблюдаем правило: **сборка → публикация → проверка manifest**.
|
||||
Любой новый номер версии должен сразу появляться в скачиваемых артефактах.
|
||||
|
||||
После обновления клиента запусти сборку и публикацию одной командой:
|
||||
|
||||
```powershell
|
||||
pwsh -ExecutionPolicy Bypass -File scripts\android\rebuild-and-publish-android-apk.ps1
|
||||
```
|
||||
|
||||
По умолчанию команда делает публикацию в локальные папки и в test-docker (vpn.cin.su):
|
||||
это нужно для корректного обновления узлов и пользователей.
|
||||
Для релиза `-PublishToTestDockerDownloads` включен по умолчанию; отключать публикацию
|
||||
внешне можно только флагами `-PublishToTestDockerDownloads:$false` или `-NoRemotePublish`.
|
||||
|
||||
```powershell
|
||||
pwsh -ExecutionPolicy Bypass -File scripts\android\rebuild-and-publish-android-apk.ps1 `
|
||||
-PublishToTestDockerDownloads `
|
||||
-TestDockerSshAlias test-docker `
|
||||
-TestDockerDownloadPath "/tmp/rap-web-admin/html/downloads"
|
||||
```
|
||||
|
||||
Если нужно сделать полноценный release-пайплайн в один шаг (подготовка окружения +
|
||||
сборка + публикация + верификация доступного манифеста), можно использовать:
|
||||
|
||||
```powershell
|
||||
pwsh -ExecutionPolicy Bypass -File scripts\android\release-android-apk.ps1 `
|
||||
-InstallMissing `
|
||||
-BuildType release `
|
||||
-PublishToTestDockerDownloads:$true `
|
||||
-TestDockerSshAlias test-docker `
|
||||
-TestDockerDownloadPath "/tmp/rap-web-admin/html/downloads" `
|
||||
-PortalVerifyBaseUrl "http://192.168.200.61:18080"
|
||||
```
|
||||
|
||||
Для «одной команды без параметров» добавлен быстрый запускатор:
|
||||
|
||||
```powershell
|
||||
pwsh -ExecutionPolicy Bypass -File scripts\android\fast-release-android-apk.ps1
|
||||
```
|
||||
|
||||
Также можно запускать двойным кликом из Windows:
|
||||
|
||||
```text
|
||||
scripts\android\fast-release-android-apk.cmd
|
||||
```
|
||||
|
||||
Через `.cmd` можно добавлять параметры FastScript, например:
|
||||
|
||||
```text
|
||||
scripts\android\fast-release-android-apk.cmd -NoRemotePublish -NoPrepare
|
||||
```
|
||||
|
||||
Для production-сборок не отключай `-NoRemotePublish` и `-NoPrepare`.
|
||||
Эти флаги только для отладочной локальной проверки и не применяются к выпуску.
|
||||
|
||||
## Полный bootstrap локальной сборочной машины
|
||||
|
||||
Если на новом ПК/ноутбуке нужно собрать проект локально, сначала прогрейте окружение:
|
||||
|
||||
```powershell
|
||||
pwsh -ExecutionPolicy Bypass -File scripts\prepare-local-build-workstation.ps1 -SetEnvironment
|
||||
```
|
||||
|
||||
Если нужно сразу пытаться устанавливать недостающее:
|
||||
|
||||
```powershell
|
||||
pwsh -ExecutionPolicy Bypass -File scripts\prepare-local-build-workstation.ps1 -InstallMissing -SetEnvironment
|
||||
```
|
||||
|
||||
После этого можно делать сборку:
|
||||
|
||||
```powershell
|
||||
pwsh -ExecutionPolicy Bypass -File scripts\android\fast-release-android-apk.ps1
|
||||
```
|
||||
|
||||
По умолчанию результат будет (для release-сборки из CI/рабочего процесса это самые нужные имена):
|
||||
|
||||
- `dist/downloads/rap-android-rdp-vpn-latest-release.apk`
|
||||
- `dist/releases/<version>/rap-android-rdp-vpn-<version>-release.apk`
|
||||
- `dist/downloads/rap-android-rdp-vpn-build.json`
|
||||
- `web-admin/deploy/html/downloads/rap-android-rdp-vpn-latest-release.apk`
|
||||
- `web-admin/deploy/html/downloads/releases/<version>/rap-android-rdp-vpn-<version>-release.apk`
|
||||
- `web-admin/deploy/html/downloads/rap-android-rdp-vpn-build.json`
|
||||
|
||||
Эти файлы и папки игнорируются git для `dist`, поэтому `web-admin` артефакты
|
||||
должны публиковаться отдельным шагом инфраструктуры.
|
||||
|
||||
## Принудительная сборка Release
|
||||
|
||||
```powershell
|
||||
pwsh -ExecutionPolicy Bypass -File scripts\android\build-android-apk.ps1 -BuildType release
|
||||
```
|
||||
|
||||
## Где указывать SDK
|
||||
|
||||
Если SDK лежит не в `C:\Android\Sdk`, укажи явно:
|
||||
|
||||
```powershell
|
||||
pwsh -ExecutionPolicy Bypass -File scripts\android\build-android-apk.ps1 -AndroidHome "D:\SDK\Android"
|
||||
```
|
||||
|
||||
## Что важно для работы с админ-панелью
|
||||
|
||||
- Веб-панель уже ожидает файл:
|
||||
`downloads/rap-android-rdp-vpn-latest-release.apk`
|
||||
- Поэтому скрипт публикует APK в `web-admin/deploy/html/downloads`, чтобы новый
|
||||
артефакт был сразу доступен для скачивания пользователем после сборки и для
|
||||
автообновления узлов.
|
||||
@@ -0,0 +1,403 @@
|
||||
#requires -Version 5
|
||||
|
||||
param(
|
||||
[string]$RepoRoot = (Resolve-Path (Join-Path $PSScriptRoot "..\..")).Path,
|
||||
[string]$BuildType = "release",
|
||||
[string]$AndroidHome = $env:ANDROID_HOME,
|
||||
[switch]$SkipWorkspaceCleanup,
|
||||
[switch]$PrintOnly,
|
||||
[bool]$PublishToTestDockerDownloads = $false,
|
||||
[string]$TestDockerSshAlias = "test-docker",
|
||||
[string]$TestDockerDownloadPath = "/tmp/rap-web-admin/html/downloads"
|
||||
)
|
||||
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
function Fail([string]$Message) {
|
||||
Write-Error $Message
|
||||
exit 1
|
||||
}
|
||||
|
||||
function Resolve-AndroidSdk([string]$CandidateHome) {
|
||||
$candidates = @(
|
||||
$CandidateHome,
|
||||
"C:\Android\Sdk",
|
||||
"$env:LOCALAPPDATA\Android\Sdk",
|
||||
"$env:USERPROFILE\AppData\Local\Android\Sdk"
|
||||
) | Where-Object { $_ -and (Test-Path $_) } | Select-Object -Unique
|
||||
|
||||
foreach ($candidate in $candidates) {
|
||||
if ((Test-Path (Join-Path $candidate "platform-tools")) -and (Test-Path (Join-Path $candidate "platforms"))) {
|
||||
return $candidate
|
||||
}
|
||||
}
|
||||
|
||||
if ($candidates.Count -gt 0) {
|
||||
return $candidates[0]
|
||||
}
|
||||
|
||||
return $null
|
||||
}
|
||||
|
||||
function Get-GradleBinary {
|
||||
$gradleCandidate = Get-Command gradle -ErrorAction SilentlyContinue
|
||||
if ($null -eq $gradleCandidate) {
|
||||
Fail "Gradle не найден в PATH. Установи Gradle (`winget install Gradle.Gradle` или `choco install gradle`)."
|
||||
}
|
||||
return $gradleCandidate.Source
|
||||
}
|
||||
|
||||
function Get-JavaVersion {
|
||||
$javaCandidate = Get-Command java -ErrorAction SilentlyContinue
|
||||
if ($null -eq $javaCandidate) {
|
||||
Fail "Java не найден в PATH. Нужен JDK 17+."
|
||||
}
|
||||
|
||||
$versionLine = & $javaCandidate.Source -version 2>&1 | Select-Object -First 1
|
||||
if ($versionLine -match "version\s+`"(\d+)\.") {
|
||||
return $matches[1]
|
||||
}
|
||||
return "unknown"
|
||||
}
|
||||
|
||||
function Resolve-SdkComponent([string]$Root, [string]$RelativePath) {
|
||||
$candidate = Join-Path $Root $RelativePath
|
||||
if (Test-Path $candidate) {
|
||||
return $candidate
|
||||
}
|
||||
return $null
|
||||
}
|
||||
|
||||
function Get-AndroidVersionInfo([string]$AndroidProjectDir) {
|
||||
$moduleBuildFile = Join-Path $AndroidProjectDir "app\build.gradle"
|
||||
if (-not (Test-Path $moduleBuildFile)) {
|
||||
Fail "Не найден файл app/build.gradle: $moduleBuildFile"
|
||||
}
|
||||
|
||||
$content = Get-Content $moduleBuildFile -Raw
|
||||
$versionCodeMatch = [regex]::Match($content, '(?m)^\s*versionCode\s+([0-9]+)\s*$')
|
||||
$versionNameMatch = [regex]::Match($content, '(?m)^\s*versionName\s+"([^"]+)"\s*$')
|
||||
if (-not $versionCodeMatch.Success -or -not $versionNameMatch.Success) {
|
||||
Fail "Не удалось прочитать versionCode/versionName из app/build.gradle"
|
||||
}
|
||||
|
||||
return @{
|
||||
VersionCode = [int]$versionCodeMatch.Groups[1].Value
|
||||
VersionName = $versionNameMatch.Groups[1].Value.Trim()
|
||||
}
|
||||
}
|
||||
|
||||
function Publish-Artifact {
|
||||
param(
|
||||
[string]$SourcePath,
|
||||
[string]$PublishRoot,
|
||||
[string]$BuildTypeNormalized,
|
||||
[string]$VersionName,
|
||||
[string]$VersionFileName,
|
||||
[string]$LatestFileName,
|
||||
[string]$PublishedPathPrefix,
|
||||
[string]$VersionTag
|
||||
)
|
||||
|
||||
New-Item -ItemType Directory -Force -Path $PublishRoot | Out-Null
|
||||
|
||||
$versionDir = Join-Path $PublishRoot "releases\$VersionName"
|
||||
New-Item -ItemType Directory -Force -Path $versionDir | Out-Null
|
||||
|
||||
$latestPath = Join-Path $PublishRoot $LatestFileName
|
||||
$versionPath = Join-Path $versionDir $VersionFileName
|
||||
$metaPath = Join-Path $PublishRoot "rap-android-rdp-vpn-build.json"
|
||||
|
||||
Copy-Item -Path $SourcePath -Destination $latestPath -Force
|
||||
Copy-Item -Path $SourcePath -Destination $versionPath -Force
|
||||
|
||||
$hash = (Get-FileHash -Algorithm SHA256 -Path $SourcePath).Hash.ToLowerInvariant()
|
||||
$size = (Get-Item $SourcePath).Length
|
||||
$published = [ordered]@{
|
||||
version = @{
|
||||
name = $VersionName
|
||||
code = $VersionTag
|
||||
}
|
||||
build = @{
|
||||
type = $BuildTypeNormalized
|
||||
artifact = "app-$BuildTypeNormalized.apk"
|
||||
}
|
||||
published = @{
|
||||
timestamp_utc = (Get-Date).ToUniversalTime().ToString("o")
|
||||
path = ("$PublishedPathPrefix/$LatestFileName").TrimStart("/")
|
||||
size_bytes = $size
|
||||
sha256 = $hash
|
||||
}
|
||||
release_paths = @{
|
||||
latest = $LatestFileName
|
||||
versioned = "releases/$VersionName/$VersionFileName"
|
||||
}
|
||||
}
|
||||
|
||||
$published | ConvertTo-Json -Depth 6 | Out-File -FilePath $metaPath -Encoding utf8 -Force
|
||||
|
||||
return @{
|
||||
latest = $latestPath
|
||||
versioned = $versionPath
|
||||
summary = $metaPath
|
||||
hash = $hash
|
||||
size = $size
|
||||
artifacts = @(
|
||||
$latestPath,
|
||||
$versionPath,
|
||||
$metaPath
|
||||
)
|
||||
releaseDir = $versionDir
|
||||
}
|
||||
}
|
||||
|
||||
function Resolve-RemoteDirectory([string]$RemotePath) {
|
||||
return $RemotePath.Trim()
|
||||
}
|
||||
|
||||
function Ensure-RemoteDirectory {
|
||||
param(
|
||||
[string]$RemoteHost,
|
||||
[string]$RemotePath
|
||||
)
|
||||
|
||||
if (-not $RemoteHost -or -not $RemotePath) {
|
||||
Fail "Remote host/path must be set for publish sync."
|
||||
}
|
||||
|
||||
$escapedPath = $RemotePath.Replace("'", "''")
|
||||
& ssh $RemoteHost "mkdir -p '$escapedPath'" | Out-Null
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
Fail "Не удалось создать каталог на remote хосте: $RemotePath"
|
||||
}
|
||||
}
|
||||
|
||||
function Publish-ToTestDockerDownloads {
|
||||
param(
|
||||
[string]$RemoteHost,
|
||||
[string]$RemoteRoot,
|
||||
[string]$VersionName,
|
||||
[string]$LatestFile,
|
||||
[string]$VersionFile,
|
||||
[string]$ManifestFile
|
||||
)
|
||||
|
||||
Ensure-RemoteDirectory -RemoteHost $RemoteHost -RemotePath $RemoteRoot
|
||||
|
||||
$remoteVersionDir = "$RemoteRoot/releases/$VersionName"
|
||||
|
||||
Ensure-RemoteDirectory -RemoteHost $RemoteHost -RemotePath $remoteVersionDir
|
||||
|
||||
& scp -q $LatestFile "${RemoteHost}:$($RemoteRoot)/" | Out-Null
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
Fail "Не удалось скопировать latest APK на remote host: ${RemoteHost}:$RemoteRoot"
|
||||
}
|
||||
|
||||
& scp -q $ManifestFile "${RemoteHost}:$($RemoteRoot)/" | Out-Null
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
Fail "Не удалось скопировать манифест на remote host: ${RemoteHost}:$RemoteRoot"
|
||||
}
|
||||
|
||||
& scp -q $VersionFile "${RemoteHost}:$($remoteVersionDir)/" | Out-Null
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
Fail "Не удалось скопировать versioned APK на remote host: ${RemoteHost}:$remoteVersionDir"
|
||||
}
|
||||
|
||||
Write-Host "SCP на ${RemoteHost}:${RemoteRoot} выполнен."
|
||||
}
|
||||
|
||||
function Resolve-AbsoluteProjectPath([string]$path) {
|
||||
if ([string]::IsNullOrWhiteSpace($path)) {
|
||||
return $null
|
||||
}
|
||||
|
||||
if (Test-Path $path) {
|
||||
return (Resolve-Path $path).ProviderPath
|
||||
}
|
||||
return $null
|
||||
}
|
||||
|
||||
$projectDir = Resolve-AbsoluteProjectPath $RepoRoot
|
||||
if (-not $projectDir) {
|
||||
Fail "RepoRoot не найден: $RepoRoot"
|
||||
}
|
||||
|
||||
$androidProject = [System.IO.Path]::GetFullPath((Join-Path $projectDir "clients\android"))
|
||||
if (-not (Test-Path $androidProject)) {
|
||||
Fail "Не найден Android проект: $androidProject"
|
||||
}
|
||||
|
||||
$sdkRoot = Resolve-AndroidSdk $AndroidHome
|
||||
if (-not $sdkRoot) {
|
||||
Fail "Android SDK не найден. Укажи -AndroidHome или установи SDK в C:\Android\Sdk."
|
||||
}
|
||||
|
||||
$cmdlineTools = Resolve-SdkComponent $sdkRoot "cmdline-tools\latest"
|
||||
if (-not $cmdlineTools) {
|
||||
Fail "Android cmdline-tools не найдены. Установи Android SDK command-line tools."
|
||||
}
|
||||
|
||||
if (-not $env:ANDROID_HOME -or $env:ANDROID_HOME -ne $sdkRoot) {
|
||||
Write-Host "Устанавливаю переменные ANDROID_HOME/ANDROID_SDK_ROOT в текущей сессии: $sdkRoot"
|
||||
$env:ANDROID_HOME = $sdkRoot
|
||||
}
|
||||
$env:ANDROID_SDK_ROOT = $sdkRoot
|
||||
|
||||
$gradleBinary = Get-GradleBinary
|
||||
$javaMajor = Get-JavaVersion
|
||||
if ($javaMajor -ne "unknown" -and [int]$javaMajor -lt 17) {
|
||||
Fail "Найден Java $javaMajor, нужен JDK 17+"
|
||||
}
|
||||
|
||||
$versionInfo = Get-AndroidVersionInfo $androidProject
|
||||
$versionName = $versionInfo.VersionName
|
||||
if ([string]::IsNullOrWhiteSpace($versionName)) {
|
||||
$versionName = "0"
|
||||
}
|
||||
$versionCode = $versionInfo.VersionCode
|
||||
if ($versionCode -le 0) {
|
||||
$versionCode = 0
|
||||
}
|
||||
$buildTypeNormalized = $BuildType.Trim().ToLower()
|
||||
if ($buildTypeNormalized -notin @("debug","release")) {
|
||||
Fail "BuildType должен быть debug или release"
|
||||
}
|
||||
|
||||
if (-not $PrintOnly -and $buildTypeNormalized -eq "release" -and -not $PublishToTestDockerDownloads) {
|
||||
Fail "Release сборка запрещена без публикации артефактов. Передай -PublishToTestDockerDownloads:`$true или запускай release-android-apk.ps1/fast-release-android-apk.ps1."
|
||||
}
|
||||
|
||||
$buildTask = "assemble" + (Get-Culture).TextInfo.ToTitleCase($buildTypeNormalized)
|
||||
|
||||
Write-Host "=== RAP Android сборка APK ==="
|
||||
Write-Host "Проект: $androidProject"
|
||||
Write-Host "Версия: $versionName ($versionCode)"
|
||||
Write-Host "SDK: $sdkRoot"
|
||||
Write-Host "Gradle: $gradleBinary"
|
||||
Write-Host "Build: $buildTask"
|
||||
Write-Host "Build task: $buildTask"
|
||||
if (-not (Test-Path (Join-Path $sdkRoot "platforms\android-35"))) {
|
||||
Write-Host "Внимание: не найдено платформы android-35 в $sdkRoot\platforms. Проверь install: platforms;android-35."
|
||||
}
|
||||
if (-not (Test-Path (Join-Path $sdkRoot "build-tools\35.0.1"))) {
|
||||
Write-Host "Внимание: не найдено build-tools 35.0.1 в $sdkRoot\build-tools."
|
||||
}
|
||||
|
||||
$workspace = Join-Path $env:TEMP ("rap-android-build-" + [guid]::NewGuid().ToString("N"))
|
||||
if (-not (Test-Path "C:\\")) {
|
||||
Write-Host "Каталог C:\ недоступен. Использую TEMP: $env:TEMP"
|
||||
} else {
|
||||
$candidateWorkspace = Join-Path "C:\" ("rap-android-build-" + [guid]::NewGuid().ToString("N"))
|
||||
$candidateParent = Split-Path $candidateWorkspace -Parent
|
||||
if (Test-Path $candidateParent) {
|
||||
$workspace = $candidateWorkspace
|
||||
}
|
||||
}
|
||||
New-Item -ItemType Directory -Path $workspace -Force | Out-Null
|
||||
Write-Host "Временная рабочая папка: $workspace"
|
||||
|
||||
$buildSucceeded = $false
|
||||
$builtApk = $null
|
||||
$summary = $null
|
||||
|
||||
try {
|
||||
Write-Host "Копирую проект в локальную рабочую папку (избавление от проблем UNC)."
|
||||
Copy-Item -Path (Join-Path $androidProject "*") -Destination $workspace -Recurse -Force
|
||||
|
||||
if ($PrintOnly) {
|
||||
Write-Host "PrintOnly: сборка не выполнена."
|
||||
return
|
||||
}
|
||||
|
||||
$originalLocation = Get-Location
|
||||
try {
|
||||
Push-Location $workspace
|
||||
$resolvedLocalProperties = Join-Path $workspace "local.properties"
|
||||
if (Test-Path $resolvedLocalProperties) {
|
||||
Remove-Item -Force $resolvedLocalProperties
|
||||
}
|
||||
Write-Host "Запускаю сборку..."
|
||||
& $gradleBinary $buildTask
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
Fail "Gradle сборка завершилась с кодом $LASTEXITCODE"
|
||||
}
|
||||
}
|
||||
finally {
|
||||
Set-Location $originalLocation
|
||||
}
|
||||
|
||||
$builtApkDir = Join-Path $workspace "app\build\outputs\apk\$buildTypeNormalized"
|
||||
$candidateApks = @()
|
||||
if (Test-Path $builtApkDir) {
|
||||
$candidateApks = Get-ChildItem -Path $builtApkDir -Filter "app-${buildTypeNormalized}*.apk" -File -ErrorAction SilentlyContinue |
|
||||
Sort-Object LastWriteTime -Descending |
|
||||
Select-Object -ExpandProperty FullName
|
||||
}
|
||||
|
||||
if (-not $candidateApks -or $candidateApks.Count -eq 0) {
|
||||
$fallbackApk = Join-Path $builtApkDir "app-$buildTypeNormalized.apk"
|
||||
if (Test-Path $fallbackApk) {
|
||||
$candidateApks = @($fallbackApk)
|
||||
} else {
|
||||
$candidateApks = @(Get-ChildItem -Path (Join-Path $workspace "app\build\outputs\apk") -Recurse -Filter "app-*.apk" -File -ErrorAction SilentlyContinue |
|
||||
Sort-Object LastWriteTime -Descending |
|
||||
Select-Object -ExpandProperty FullName)
|
||||
}
|
||||
}
|
||||
|
||||
$builtApk = $candidateApks | Select-Object -First 1
|
||||
if (-not $builtApk) {
|
||||
$expectedPath = Join-Path $builtApkDir "app-${buildTypeNormalized}.apk"
|
||||
Fail "Сборка не дала APK: $expectedPath"
|
||||
}
|
||||
$buildSucceeded = $true
|
||||
|
||||
$latestFileName = "rap-android-rdp-vpn-latest-$buildTypeNormalized.apk"
|
||||
$versionFileName = "rap-android-rdp-vpn-$versionName-$buildTypeNormalized.apk"
|
||||
$publishedPathPrefix = "downloads"
|
||||
|
||||
$publishDirs = @(
|
||||
[System.IO.Path]::GetFullPath((Join-Path $projectDir "dist\downloads")),
|
||||
[System.IO.Path]::GetFullPath((Join-Path $projectDir "web-admin\deploy\html\downloads"))
|
||||
)
|
||||
$publishResults = @()
|
||||
foreach ($publishDir in $publishDirs) {
|
||||
$result = Publish-Artifact -SourcePath $builtApk `
|
||||
-PublishRoot $publishDir `
|
||||
-BuildTypeNormalized $buildTypeNormalized `
|
||||
-VersionName $versionName `
|
||||
-VersionFileName $versionFileName `
|
||||
-LatestFileName $latestFileName `
|
||||
-PublishedPathPrefix $publishedPathPrefix `
|
||||
-VersionTag $versionCode
|
||||
|
||||
$publishResults += $result
|
||||
Write-Host "Опубликован в: $publishDir"
|
||||
}
|
||||
|
||||
$summary = $publishResults[0]
|
||||
if ($PublishToTestDockerDownloads) {
|
||||
$remotePublishSource = if ($publishResults.Count -ge 2) { $publishResults[1] } else { $summary }
|
||||
Write-Host "Публикую артефакты в test-docker ($TestDockerSshAlias): $TestDockerDownloadPath"
|
||||
Publish-ToTestDockerDownloads -RemoteHost $TestDockerSshAlias -RemoteRoot (Resolve-RemoteDirectory $TestDockerDownloadPath) `
|
||||
-VersionName $versionName `
|
||||
-LatestFile $remotePublishSource.latest `
|
||||
-VersionFile $remotePublishSource.versioned `
|
||||
-ManifestFile $remotePublishSource.summary
|
||||
}
|
||||
|
||||
Write-Host "Сборка завершена."
|
||||
Write-Host "APK: $($summary.latest)"
|
||||
Write-Host "Версионная копия: $($summary.versioned)"
|
||||
Write-Host "Размер: $($summary.size) байт"
|
||||
Write-Host "SHA256: $($summary.hash)"
|
||||
}
|
||||
finally {
|
||||
if (-not $SkipWorkspaceCleanup -and (Test-Path $workspace)) {
|
||||
Remove-Item -Recurse -Force $workspace
|
||||
}
|
||||
}
|
||||
|
||||
if (-not $buildSucceeded) {
|
||||
Fail "Сборка не завершена."
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
@echo off
|
||||
setlocal
|
||||
|
||||
pwsh -NoProfile -ExecutionPolicy Bypass -File "%~dp0fast-release-android-apk.ps1" %*
|
||||
if %ERRORLEVEL% neq 0 (
|
||||
exit /b %ERRORLEVEL%
|
||||
)
|
||||
@@ -0,0 +1,35 @@
|
||||
#requires -Version 5
|
||||
|
||||
param(
|
||||
[string]$BuildType = "release",
|
||||
[bool]$PublishToTestDockerDownloads = $true,
|
||||
[switch]$NoRemotePublish,
|
||||
[switch]$NoPrepare,
|
||||
[switch]$SkipPortalVerify,
|
||||
[string]$PortalVerifyBaseUrl = "http://192.168.200.61:18080",
|
||||
[string]$TestDockerSshAlias = "test-docker",
|
||||
[string]$TestDockerDownloadPath = "/tmp/rap-web-admin/html/downloads",
|
||||
[int]$PreparationRetryDelaySeconds = 0
|
||||
)
|
||||
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
$scriptDir = Split-Path -Parent (Resolve-Path $PSCommandPath).ProviderPath
|
||||
$releaseScript = Join-Path $scriptDir "release-android-apk.ps1"
|
||||
|
||||
if (-not (Test-Path $releaseScript)) {
|
||||
Write-Error "Не найден скрипт release-android-apk.ps1: $releaseScript"
|
||||
exit 1
|
||||
}
|
||||
|
||||
$publishToTestDockerDownloads = $PublishToTestDockerDownloads -and -not $NoRemotePublish
|
||||
|
||||
& $releaseScript -BuildType $BuildType `
|
||||
-InstallMissing `
|
||||
-PublishToTestDockerDownloads:$publishToTestDockerDownloads `
|
||||
-TestDockerSshAlias $TestDockerSshAlias `
|
||||
-TestDockerDownloadPath $TestDockerDownloadPath `
|
||||
-PortalVerifyBaseUrl $PortalVerifyBaseUrl `
|
||||
-SkipPortalVerify:$SkipPortalVerify `
|
||||
-PreparationRetryDelaySeconds $PreparationRetryDelaySeconds `
|
||||
-SkipPrepare:$NoPrepare
|
||||
@@ -0,0 +1,166 @@
|
||||
#requires -Version 5
|
||||
|
||||
param(
|
||||
[string]$RepoRoot = (Resolve-Path (Join-Path $PSScriptRoot "..\..")).Path,
|
||||
[string]$AndroidHome = $env:ANDROID_HOME,
|
||||
[switch]$SetEnvironment,
|
||||
[switch]$InstallMissing
|
||||
)
|
||||
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
function Fail([string]$Message) {
|
||||
Write-Error $Message
|
||||
exit 1
|
||||
}
|
||||
|
||||
function Get-CommandPath([string]$Name) {
|
||||
$cmd = Get-Command $Name -ErrorAction SilentlyContinue
|
||||
if ($null -eq $cmd) { return $null }
|
||||
return $cmd.Source
|
||||
}
|
||||
|
||||
function Parse-AndroidCompileVersion([string]$AndroidProjectDir) {
|
||||
$buildFile = Join-Path $AndroidProjectDir "app\build.gradle"
|
||||
if (-not (Test-Path $buildFile)) {
|
||||
return $null
|
||||
}
|
||||
|
||||
$content = Get-Content $buildFile -Raw
|
||||
$compileMatch = [regex]::Match($content, '(?m)^\s*compileSdk\s+([0-9]+)\s*$')
|
||||
if ($compileMatch.Success) { return $compileMatch.Groups[1].Value }
|
||||
return $null
|
||||
}
|
||||
|
||||
function Get-AndroidSdkHome([string]$CandidateHome) {
|
||||
$candidates = @(
|
||||
$CandidateHome,
|
||||
"C:\Android\Sdk",
|
||||
"$env:LOCALAPPDATA\Android\Sdk",
|
||||
"$env:USERPROFILE\AppData\Local\Android\Sdk"
|
||||
) | Where-Object { $_ -and (Test-Path $_) } | Select-Object -Unique
|
||||
|
||||
foreach ($candidate in $candidates) {
|
||||
if ((Test-Path (Join-Path $candidate "platform-tools")) -and (Test-Path (Join-Path $candidate "platforms"))) {
|
||||
return $candidate
|
||||
}
|
||||
}
|
||||
|
||||
if ($candidates.Count -gt 0) { return $candidates[0] }
|
||||
return $null
|
||||
}
|
||||
|
||||
Write-Host "=== RAP: подготовка Android среды сборки ==="
|
||||
$repoRoot = (Resolve-Path $RepoRoot).ProviderPath
|
||||
if (-not $repoRoot) {
|
||||
Fail "RepoRoot не найден: $RepoRoot"
|
||||
}
|
||||
|
||||
$androidProject = Join-Path $repoRoot "clients\android"
|
||||
if (-not (Test-Path $androidProject)) {
|
||||
Fail "Не найден проект clients\android: $androidProject"
|
||||
}
|
||||
|
||||
$compileSdk = Parse-AndroidCompileVersion $androidProject
|
||||
if ([string]::IsNullOrWhiteSpace($compileSdk)) {
|
||||
$compileSdk = "35"
|
||||
}
|
||||
|
||||
$java = Get-CommandPath "java"
|
||||
if (-not $java) {
|
||||
Write-Host "Не найден java в PATH. Установи JDK 17+: winget install EclipseAdoptium.Temurin.17.JDK или choco install temurin17"
|
||||
Fail "Зависимость Java отсутствует."
|
||||
}
|
||||
$javaVersion = (& $java -version 2>&1 | Select-Object -First 1)
|
||||
if ($javaVersion -match 'version\s+"(\d+)\.' ) {
|
||||
$major = [int]$matches[1]
|
||||
Write-Host "Java: $java ($major)"
|
||||
if ($major -lt 17) {
|
||||
Write-Host "Найден Java $major, нужен JDK 17+."
|
||||
Fail "Нужен Java 17 или новее."
|
||||
}
|
||||
} else {
|
||||
Write-Host "Java найден, версия не распознана: $javaVersion"
|
||||
}
|
||||
|
||||
$gradle = Get-CommandPath "gradle"
|
||||
if (-not $gradle) {
|
||||
Write-Host "Не найден gradle в PATH. Установи: winget install Gradle.Gradle / choco install gradle"
|
||||
Fail "Зависимость Gradle отсутствует."
|
||||
}
|
||||
Write-Host "Gradle: $gradle"
|
||||
|
||||
$sdkRoot = Get-AndroidSdkHome $AndroidHome
|
||||
if (-not $sdkRoot) {
|
||||
Write-Host "Не найден Android SDK. Укажи -AndroidHome или установи SDK в C:\Android\Sdk."
|
||||
Fail "Android SDK не найден."
|
||||
}
|
||||
Write-Host "Android SDK: $sdkRoot"
|
||||
|
||||
$cmdlineTools = Join-Path $sdkRoot "cmdline-tools\latest\bin\sdkmanager.bat"
|
||||
if (-not (Test-Path $cmdlineTools)) {
|
||||
Write-Host "Не найден Android cmdline-tools ($cmdlineTools)."
|
||||
Write-Host "Установи Android SDK Command-line Tools через Android Studio или вручную скачай SDK Manager."
|
||||
Fail "cmdline-tools не найдены."
|
||||
}
|
||||
Write-Host "sdkmanager: $cmdlineTools"
|
||||
|
||||
Write-Host "Поддерживаем compileSdk: $compileSdk"
|
||||
|
||||
$requiredComponents = @(
|
||||
"platform-tools",
|
||||
"platforms;android-$compileSdk",
|
||||
"build-tools;35.0.1"
|
||||
)
|
||||
$missingComponents = @()
|
||||
foreach ($component in $requiredComponents) {
|
||||
switch -Regex ($component) {
|
||||
"^platform-tools$" {
|
||||
if (-not (Test-Path (Join-Path $sdkRoot "platform-tools"))) { $missingComponents += $component }
|
||||
}
|
||||
"^platforms;android-(?<api>\d+)$" {
|
||||
if (-not (Test-Path (Join-Path $sdkRoot "platforms\android-$($Matches["api"])"))) { $missingComponents += $component }
|
||||
}
|
||||
"^build-tools;(?<bt>.+)$" {
|
||||
if (-not (Test-Path (Join-Path $sdkRoot "build-tools\$($Matches["bt"])"))) { $missingComponents += $component }
|
||||
}
|
||||
default { }
|
||||
}
|
||||
}
|
||||
|
||||
if ($missingComponents.Count -gt 0) {
|
||||
Write-Host ""
|
||||
Write-Host "Отсутствуют компоненты Android SDK:"
|
||||
$missingComponents | ForEach-Object { Write-Host " - $_" }
|
||||
Write-Host ""
|
||||
$installCmd = "& `"$cmdlineTools`" --sdk_root=`"$sdkRoot`" --install $($missingComponents -join ' ')"
|
||||
Write-Host "Команда установки:"
|
||||
Write-Host " $installCmd"
|
||||
if ($InstallMissing) {
|
||||
Write-Host "Запускаю установку отсутствующих компонентов..."
|
||||
& $cmdlineTools --sdk_root=$sdkRoot --install $missingComponents
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
Fail "Не удалось установить компоненты через sdkmanager."
|
||||
}
|
||||
Write-Host "Компоненты установлены."
|
||||
} else {
|
||||
Write-Host "Добавь -InstallMissing для автo-установки."
|
||||
}
|
||||
} else {
|
||||
Write-Host "Все базовые компоненты SDK присутствуют."
|
||||
}
|
||||
|
||||
if ($SetEnvironment) {
|
||||
[Environment]::SetEnvironmentVariable("ANDROID_HOME", $sdkRoot, "User")
|
||||
[Environment]::SetEnvironmentVariable("ANDROID_SDK_ROOT", $sdkRoot, "User")
|
||||
Write-Host "Персистентные переменные ANDROID_HOME/ANDROID_SDK_ROOT сохранены для текущего пользователя."
|
||||
}
|
||||
|
||||
Write-Host ""
|
||||
Write-Host "Рекомендуется в текущей сессии:"
|
||||
Write-Host " `$env:ANDROID_HOME='$sdkRoot'"
|
||||
Write-Host " `$env:ANDROID_SDK_ROOT='$sdkRoot'"
|
||||
|
||||
Write-Host ""
|
||||
Write-Host "Готово. После этого запуск:"
|
||||
Write-Host " pwsh -ExecutionPolicy Bypass -File scripts\\android\\build-android-apk.ps1"
|
||||
@@ -0,0 +1,85 @@
|
||||
#requires -Version 5
|
||||
|
||||
param(
|
||||
[string]$RepoRoot = (Resolve-Path (Join-Path $PSScriptRoot "..\..")).ProviderPath,
|
||||
[string]$BuildType = "release",
|
||||
[string]$AndroidHome = $env:ANDROID_HOME,
|
||||
[switch]$InstallMissing,
|
||||
[switch]$SkipWorkspaceCleanup,
|
||||
[switch]$PrintOnly,
|
||||
[switch]$SkipPrepare,
|
||||
[bool]$PublishToTestDockerDownloads = $true,
|
||||
[string]$TestDockerSshAlias = "test-docker",
|
||||
[string]$TestDockerDownloadPath = "/tmp/rap-web-admin/html/downloads",
|
||||
[int]$PreparationRetryDelaySeconds = 0
|
||||
)
|
||||
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
function Fail([string]$Message) {
|
||||
Write-Error $Message
|
||||
exit 1
|
||||
}
|
||||
|
||||
function Run-Step([string]$Name, [scriptblock]$Action) {
|
||||
Write-Host "=== $Name ==="
|
||||
$global:LASTEXITCODE = 0
|
||||
try {
|
||||
& $Action
|
||||
} catch {
|
||||
Fail "$Name завершился с ошибкой: $($_.Exception.Message)"
|
||||
}
|
||||
if (-not $?) {
|
||||
Fail "$Name завершился с кодом ошибки"
|
||||
}
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
Fail "$Name завершился с кодом $LASTEXITCODE"
|
||||
}
|
||||
}
|
||||
|
||||
$scriptDir = Split-Path -Parent (Resolve-Path $PSCommandPath).ProviderPath
|
||||
$prepareScript = Join-Path $scriptDir "prepare-android-build-environment.ps1"
|
||||
$buildScript = Join-Path $scriptDir "build-android-apk.ps1"
|
||||
|
||||
if (-not (Test-Path $prepareScript)) {
|
||||
Fail "Не найден скрипт подготовки окружения: $prepareScript"
|
||||
}
|
||||
if (-not (Test-Path $buildScript)) {
|
||||
Fail "Не найден скрипт сборки: $buildScript"
|
||||
}
|
||||
|
||||
if ($PreparationRetryDelaySeconds -lt 0 -or $PreparationRetryDelaySeconds -gt 3600) {
|
||||
Fail "PreparationRetryDelaySeconds должен быть в диапазоне 0..3600."
|
||||
}
|
||||
|
||||
if (-not $SkipPrepare) {
|
||||
Run-Step "Подготовка Android окружения" {
|
||||
& $prepareScript -RepoRoot $RepoRoot -AndroidHome $AndroidHome -SetEnvironment -InstallMissing:$InstallMissing
|
||||
}
|
||||
if ($PreparationRetryDelaySeconds -gt 0) {
|
||||
Start-Sleep -Seconds $PreparationRetryDelaySeconds
|
||||
}
|
||||
} else {
|
||||
Write-Host "Подготовка окружения пропущена (-SkipPrepare)."
|
||||
}
|
||||
|
||||
if ($PrintOnly) {
|
||||
Run-Step "Проверка параметров и печать (без сборки)" {
|
||||
& $buildScript -RepoRoot $RepoRoot -BuildType $BuildType -AndroidHome $AndroidHome -PrintOnly `
|
||||
-SkipWorkspaceCleanup:$SkipWorkspaceCleanup `
|
||||
-PublishToTestDockerDownloads:$PublishToTestDockerDownloads `
|
||||
-TestDockerSshAlias $TestDockerSshAlias `
|
||||
-TestDockerDownloadPath $TestDockerDownloadPath
|
||||
}
|
||||
exit 0
|
||||
}
|
||||
|
||||
Run-Step "Сборка и публикация Android APK" {
|
||||
& $buildScript -RepoRoot $RepoRoot -BuildType $BuildType -AndroidHome $AndroidHome -SkipWorkspaceCleanup:$SkipWorkspaceCleanup `
|
||||
-PublishToTestDockerDownloads:$PublishToTestDockerDownloads `
|
||||
-TestDockerSshAlias $TestDockerSshAlias `
|
||||
-TestDockerDownloadPath $TestDockerDownloadPath
|
||||
}
|
||||
|
||||
Write-Host ""
|
||||
Write-Host "Готово. APK опубликован для веб-панели по ссылке: downloads/rap-android-rdp-vpn-latest-$BuildType.apk"
|
||||
@@ -0,0 +1,134 @@
|
||||
#requires -Version 5
|
||||
|
||||
param(
|
||||
[string]$RepoRoot = (Resolve-Path (Join-Path $PSScriptRoot "..\..")).ProviderPath,
|
||||
[string]$BuildType = "release",
|
||||
[string]$AndroidHome = $env:ANDROID_HOME,
|
||||
[switch]$InstallMissing,
|
||||
[switch]$SkipPrepare,
|
||||
[switch]$SkipWorkspaceCleanup,
|
||||
[switch]$PrintOnly,
|
||||
[bool]$PublishToTestDockerDownloads = $true,
|
||||
[switch]$NoRemotePublish,
|
||||
[string]$TestDockerSshAlias = "test-docker",
|
||||
[string]$TestDockerDownloadPath = "/tmp/rap-web-admin/html/downloads",
|
||||
[int]$PreparationRetryDelaySeconds = 0,
|
||||
[string]$PortalVerifyBaseUrl = "http://192.168.200.61:18080",
|
||||
[switch]$SkipPortalVerify
|
||||
)
|
||||
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
function Fail([string]$message) {
|
||||
Write-Error $message
|
||||
exit 1
|
||||
}
|
||||
|
||||
function Run-Step([string]$name, [scriptblock]$action) {
|
||||
Write-Host "=== $name ==="
|
||||
$global:LASTEXITCODE = 0
|
||||
try {
|
||||
& $action
|
||||
} catch {
|
||||
Fail "$name failed: $($_.Exception.Message)"
|
||||
}
|
||||
if (-not $?) {
|
||||
Fail "$name failed with script error"
|
||||
}
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
Fail "$name failed with exit code $LASTEXITCODE"
|
||||
}
|
||||
}
|
||||
|
||||
$scriptDir = Split-Path -Parent (Resolve-Path $PSCommandPath).ProviderPath
|
||||
$prepareScript = Join-Path $scriptDir "prepare-android-build-environment.ps1"
|
||||
$buildScript = Join-Path $scriptDir "build-android-apk.ps1"
|
||||
$buildTypeNormalized = $BuildType.Trim().ToLower()
|
||||
|
||||
if ($buildTypeNormalized -notin @("debug","release")) {
|
||||
Fail "BuildType должен быть debug или release"
|
||||
}
|
||||
|
||||
if (($NoRemotePublish -or -not $PublishToTestDockerDownloads) -and $buildTypeNormalized -eq "release") {
|
||||
Fail "Для Release сборки в этом сценарии требуется публикация в test-docker. Убери -NoRemotePublish и явно не отключай -PublishToTestDockerDownloads."
|
||||
}
|
||||
|
||||
if (-not (Test-Path $prepareScript)) {
|
||||
Fail "Не найден скрипт подготовки окружения: $prepareScript"
|
||||
}
|
||||
if (-not (Test-Path $buildScript)) {
|
||||
Fail "Не найден скрипт сборки: $buildScript"
|
||||
}
|
||||
|
||||
if (-not $SkipPrepare) {
|
||||
Run-Step "Подготовка Android окружения" {
|
||||
& $prepareScript -RepoRoot $RepoRoot -AndroidHome $AndroidHome -SetEnvironment -InstallMissing:$InstallMissing
|
||||
}
|
||||
if ($PreparationRetryDelaySeconds -gt 0) {
|
||||
Start-Sleep -Seconds $PreparationRetryDelaySeconds
|
||||
}
|
||||
} else {
|
||||
Write-Host "Подготовка окружения пропущена (-SkipPrepare)."
|
||||
}
|
||||
|
||||
if ($PrintOnly) {
|
||||
if ($PublishToTestDockerDownloads -and -not $NoRemotePublish) {
|
||||
& $buildScript -RepoRoot $RepoRoot -BuildType $BuildType -AndroidHome $AndroidHome -PrintOnly `
|
||||
-SkipWorkspaceCleanup:$SkipWorkspaceCleanup `
|
||||
-PublishToTestDockerDownloads:$true `
|
||||
-TestDockerSshAlias $TestDockerSshAlias `
|
||||
-TestDockerDownloadPath $TestDockerDownloadPath
|
||||
} else {
|
||||
& $buildScript -RepoRoot $RepoRoot -BuildType $BuildType -AndroidHome $AndroidHome -PrintOnly `
|
||||
-SkipWorkspaceCleanup:$SkipWorkspaceCleanup
|
||||
}
|
||||
exit 0
|
||||
}
|
||||
|
||||
Run-Step "Сборка и публикация Android APK" {
|
||||
if ($PublishToTestDockerDownloads -and -not $NoRemotePublish) {
|
||||
& $buildScript -RepoRoot $RepoRoot -BuildType $BuildType -AndroidHome $AndroidHome -SkipWorkspaceCleanup:$SkipWorkspaceCleanup `
|
||||
-PublishToTestDockerDownloads:$true `
|
||||
-TestDockerSshAlias $TestDockerSshAlias `
|
||||
-TestDockerDownloadPath $TestDockerDownloadPath
|
||||
} else {
|
||||
& $buildScript -RepoRoot $RepoRoot -BuildType $BuildType -AndroidHome $AndroidHome -SkipWorkspaceCleanup:$SkipWorkspaceCleanup
|
||||
}
|
||||
}
|
||||
|
||||
if ($SkipPortalVerify) {
|
||||
Write-Host "Сборка выполнена. Проверка публикации пропущена (-SkipPortalVerify)."
|
||||
exit 0
|
||||
}
|
||||
|
||||
Run-Step "Проверка манифеста веб-панели" {
|
||||
$manifestPath = Join-Path $RepoRoot "web-admin\deploy\html\downloads\rap-android-rdp-vpn-build.json"
|
||||
if (-not (Test-Path $manifestPath)) {
|
||||
Fail "Локальный манифест не найден: $manifestPath"
|
||||
}
|
||||
|
||||
$manifest = Get-Content $manifestPath -Raw | ConvertFrom-Json
|
||||
if (-not $manifest.version -or -not $manifest.version.name) {
|
||||
Fail "Манифест не содержит version.name: $manifestPath"
|
||||
}
|
||||
|
||||
Write-Host "Локальный манифест APK версии: $($manifest.version.name) ($($manifest.version.code))"
|
||||
Write-Host "Path: $($manifest.published.path)"
|
||||
Write-Host "Sha256: $($manifest.published.sha256)"
|
||||
|
||||
if (-not [string]::IsNullOrWhiteSpace($PortalVerifyBaseUrl)) {
|
||||
$manifestUrl = "$PortalVerifyBaseUrl/downloads/rap-android-rdp-vpn-build.json?_cb=$(Get-Date -Format 'yyyyMMddHHmmss')"
|
||||
try {
|
||||
$remoteManifest = Invoke-RestMethod -Uri $manifestUrl -Method Get
|
||||
if (-not $remoteManifest.version -or -not $remoteManifest.version.name -or $remoteManifest.version.name -ne $manifest.version.name) {
|
||||
Fail "Версия после деплоя не совпадает: локально=$($manifest.version.name), remote=$($remoteManifest.version.name)"
|
||||
}
|
||||
Write-Host "Подтверждено: portal returns version $($remoteManifest.version.name)"
|
||||
} catch {
|
||||
Write-Host "WARN: не удалось прочитать удаленный манифест из $manifestUrl"
|
||||
Write-Host " Проверьте доступность веб-панели и путь/порт вручную."
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Write-Host "Готово: релиз собран, опубликован и проверен."
|
||||
@@ -0,0 +1,335 @@
|
||||
param(
|
||||
[string]$ApiBaseUrl = "http://192.168.200.61:18121/api/v1",
|
||||
[string]$ClusterID = "cfc0743d-d960-49fb-9de8-96e063d5e4aa",
|
||||
[string]$ActorUserID = "f67d943f-5397-4b3a-a229-695fe67ad700",
|
||||
[string]$EntryNodeName = "test-1",
|
||||
[string]$ExitNodeName = "test-2",
|
||||
[string]$RequiredNodeVersion = "0.2.177",
|
||||
[string]$ResultPath = "artifacts\c18w-service-channel-route-manager-smoke-result.json"
|
||||
)
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
$repoRoot = (Resolve-Path (Join-Path $PSScriptRoot "..\..")).ProviderPath
|
||||
$runId = "c18w-" + (Get-Date -Format "yyyyMMdd-HHmmss")
|
||||
|
||||
function Invoke-Api {
|
||||
param(
|
||||
[string]$Method,
|
||||
[string]$Path,
|
||||
[object]$Body = $null
|
||||
)
|
||||
$uri = "$ApiBaseUrl$Path"
|
||||
try {
|
||||
if ($null -eq $Body) {
|
||||
return Invoke-RestMethod -Method $Method -Uri $uri -TimeoutSec 30
|
||||
}
|
||||
return Invoke-RestMethod -Method $Method -Uri $uri -ContentType "application/json" -Body ($Body | ConvertTo-Json -Depth 80) -TimeoutSec 30
|
||||
}
|
||||
catch {
|
||||
$statusCode = $null
|
||||
if ($_.Exception.Response) {
|
||||
$statusCode = [int]$_.Exception.Response.StatusCode
|
||||
}
|
||||
$details = $_.ErrorDetails.Message
|
||||
if (-not $details) {
|
||||
$details = $_.Exception.Message
|
||||
}
|
||||
throw "$Method $Path failed with HTTP $statusCode`: $details"
|
||||
}
|
||||
}
|
||||
|
||||
function Get-NodeByName {
|
||||
param([string]$Name)
|
||||
$nodes = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/nodes?actor_user_id=$ActorUserID").nodes
|
||||
$node = @($nodes | Where-Object { $_.name -eq $Name }) | Select-Object -First 1
|
||||
if ($null -eq $node) {
|
||||
throw "Node '$Name' was not found in cluster $ClusterID"
|
||||
}
|
||||
return $node
|
||||
}
|
||||
|
||||
function New-RouteIntent {
|
||||
param(
|
||||
[string]$SourceNodeID,
|
||||
[string]$DestinationNodeID,
|
||||
[int]$Priority,
|
||||
[string]$Label
|
||||
)
|
||||
$expiresAt = (Get-Date).ToUniversalTime().AddMinutes(8).ToString("o")
|
||||
return Invoke-Api -Method POST -Path "/clusters/$ClusterID/mesh/route-intents" -Body @{
|
||||
actor_user_id = $ActorUserID
|
||||
source_selector = @{
|
||||
node_id = $SourceNodeID
|
||||
}
|
||||
destination_selector = @{
|
||||
node_id = $DestinationNodeID
|
||||
}
|
||||
service_class = "vpn_packets"
|
||||
priority = $Priority
|
||||
policy = @{
|
||||
synthetic_enabled = $true
|
||||
route_version = "$runId-$Label"
|
||||
policy_version = "$runId-$Label"
|
||||
peer_directory_version = "$runId-$Label"
|
||||
hops = @($SourceNodeID, $DestinationNodeID)
|
||||
allowed_channels = @("vpn_packet", "fabric_control")
|
||||
max_ttl = 8
|
||||
max_hops = 8
|
||||
expires_at = $expiresAt
|
||||
metadata = @{
|
||||
smoke = "c18w_service_channel_route_manager"
|
||||
run_id = $runId
|
||||
label = $Label
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function Send-FeedbackHeartbeat {
|
||||
param(
|
||||
[string]$EntryNodeID,
|
||||
[string]$BadRouteID,
|
||||
[string]$GoodRouteID
|
||||
)
|
||||
return Invoke-Api -Method POST -Path "/clusters/$ClusterID/nodes/$EntryNodeID/heartbeats" -Body @{
|
||||
health_status = "healthy"
|
||||
reported_version = $RequiredNodeVersion
|
||||
capabilities = @{
|
||||
native_node_agent = $true
|
||||
fabric_service_channel_runtime = $true
|
||||
fabric_service_channel_route_manager = $true
|
||||
smoke_feedback_injection = "c18w"
|
||||
}
|
||||
service_states = @{
|
||||
smoke = "c18w_route_manager_feedback"
|
||||
}
|
||||
metadata = @{
|
||||
fabric_service_channel_runtime_report = @{
|
||||
schema_version = "c18l.fabric_service_channel_runtime_report.v1"
|
||||
ingress = @{
|
||||
flow_scheduler = @{
|
||||
channel_stats = @{
|
||||
"c18w-smoke-flow" = @{
|
||||
last_route_id = $GoodRouteID
|
||||
last_failed_route_id = $BadRouteID
|
||||
last_error = "c18w forced stale route feedback"
|
||||
consecutive_failures = 3
|
||||
stall_count = 0
|
||||
last_send_duration_ms = 250
|
||||
route_rebuild_recommended = $true
|
||||
degraded_fallback_recommended = $false
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
smoke = @{
|
||||
name = "c18w_service_channel_route_manager"
|
||||
run_id = $runId
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function Send-CleanHeartbeat {
|
||||
param([string]$EntryNodeID)
|
||||
return Invoke-Api -Method POST -Path "/clusters/$ClusterID/nodes/$EntryNodeID/heartbeats" -Body @{
|
||||
health_status = "healthy"
|
||||
reported_version = $RequiredNodeVersion
|
||||
capabilities = @{
|
||||
native_node_agent = $true
|
||||
fabric_service_channel_runtime = $true
|
||||
fabric_service_channel_route_manager = $true
|
||||
smoke_feedback_injection = "c18w-clean"
|
||||
}
|
||||
service_states = @{
|
||||
smoke = "c18w_route_manager_restore"
|
||||
}
|
||||
metadata = @{
|
||||
fabric_service_channel_runtime_report = @{
|
||||
schema_version = "c18l.fabric_service_channel_runtime_report.v1"
|
||||
ingress = @{
|
||||
flow_scheduler = @{
|
||||
channel_stats = @{}
|
||||
}
|
||||
}
|
||||
}
|
||||
smoke = @{
|
||||
name = "c18w_service_channel_route_manager"
|
||||
run_id = $runId
|
||||
phase = "clean_after_expire"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function Get-SyntheticConfig {
|
||||
param([string]$NodeID)
|
||||
return Invoke-Api -Method GET -Path "/clusters/$ClusterID/nodes/$NodeID/mesh/synthetic-config?actor_user_id=$ActorUserID"
|
||||
}
|
||||
|
||||
function Get-LatestTransition {
|
||||
param([string]$NodeID)
|
||||
$heartbeats = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/nodes/$NodeID/heartbeats?actor_user_id=$ActorUserID&limit=8").heartbeats
|
||||
foreach ($heartbeat in $heartbeats) {
|
||||
$transition = $null
|
||||
$reportProperty = $heartbeat.metadata.PSObject.Properties["fabric_service_channel_runtime_report"]
|
||||
if ($null -ne $reportProperty) {
|
||||
$ingressProperty = $reportProperty.Value.PSObject.Properties["ingress"]
|
||||
if ($null -ne $ingressProperty) {
|
||||
$transitionProperty = $ingressProperty.Value.PSObject.Properties["route_manager_transition"]
|
||||
if ($null -ne $transitionProperty) {
|
||||
$transition = $transitionProperty.Value
|
||||
}
|
||||
}
|
||||
}
|
||||
if ($null -ne $transition -and $transition.schema_version -eq "rap.fabric_service_channel_route_manager_transition.v1") {
|
||||
return @{
|
||||
heartbeat = $heartbeat
|
||||
transition = $transition
|
||||
}
|
||||
}
|
||||
}
|
||||
return $null
|
||||
}
|
||||
|
||||
function Wait-ForTransitionStatus {
|
||||
param(
|
||||
[string]$NodeID,
|
||||
[string[]]$Statuses,
|
||||
[int]$TimeoutSeconds = 90
|
||||
)
|
||||
$deadline = (Get-Date).AddSeconds($TimeoutSeconds)
|
||||
do {
|
||||
$latest = Get-LatestTransition -NodeID $NodeID
|
||||
if ($null -ne $latest -and $Statuses -contains [string]$latest.transition.status) {
|
||||
return $latest
|
||||
}
|
||||
Start-Sleep -Seconds 2
|
||||
} while ((Get-Date) -lt $deadline)
|
||||
$last = Get-LatestTransition -NodeID $NodeID
|
||||
throw "Timed out waiting for transition status '$($Statuses -join ',')'. Last transition: $($last | ConvertTo-Json -Depth 20 -Compress)"
|
||||
}
|
||||
|
||||
function Wait-ForConfigDecision {
|
||||
param(
|
||||
[string]$NodeID,
|
||||
[string]$BadRouteID,
|
||||
[string]$ExpectedStatus,
|
||||
[int]$TimeoutSeconds = 45
|
||||
)
|
||||
$deadline = (Get-Date).AddSeconds($TimeoutSeconds)
|
||||
do {
|
||||
$config = Get-SyntheticConfig -NodeID $NodeID
|
||||
$decisions = @($config.synthetic_mesh_config.route_path_decisions.decisions)
|
||||
$decision = @($decisions | Where-Object { $_.route_id -eq $BadRouteID -and $_.rebuild_status -eq $ExpectedStatus }) | Select-Object -First 1
|
||||
if ($null -ne $decision) {
|
||||
return @{
|
||||
config = $config
|
||||
decision = $decision
|
||||
}
|
||||
}
|
||||
Start-Sleep -Seconds 2
|
||||
} while ((Get-Date) -lt $deadline)
|
||||
throw "Timed out waiting for rebuild_status=$ExpectedStatus for route $BadRouteID"
|
||||
}
|
||||
|
||||
function Wait-ForNoRebuildDecision {
|
||||
param(
|
||||
[string]$NodeID,
|
||||
[string]$BadRouteID,
|
||||
[int]$TimeoutSeconds = 45
|
||||
)
|
||||
$deadline = (Get-Date).AddSeconds($TimeoutSeconds)
|
||||
do {
|
||||
$config = Get-SyntheticConfig -NodeID $NodeID
|
||||
$decisions = @($config.synthetic_mesh_config.route_path_decisions.decisions)
|
||||
$decision = @($decisions | Where-Object {
|
||||
$statusProperty = $_.PSObject.Properties["rebuild_status"]
|
||||
$_.route_id -eq $BadRouteID -and $null -ne $statusProperty -and -not [string]::IsNullOrWhiteSpace([string]$statusProperty.Value)
|
||||
}) | Select-Object -First 1
|
||||
if ($null -eq $decision) {
|
||||
$routeDecision = @($decisions | Where-Object { $_.route_id -eq $BadRouteID }) | Select-Object -First 1
|
||||
return @{
|
||||
config = $config
|
||||
decision = $routeDecision
|
||||
}
|
||||
}
|
||||
Start-Sleep -Seconds 2
|
||||
} while ((Get-Date) -lt $deadline)
|
||||
throw "Timed out waiting for rebuild decision to clear for route $BadRouteID"
|
||||
}
|
||||
|
||||
Write-Host "C18W smoke $runId against $ApiBaseUrl"
|
||||
$entryNode = Get-NodeByName -Name $EntryNodeName
|
||||
$exitNode = Get-NodeByName -Name $ExitNodeName
|
||||
if ($entryNode.reported_version -ne $RequiredNodeVersion) {
|
||||
throw "$EntryNodeName reports version $($entryNode.reported_version), want $RequiredNodeVersion"
|
||||
}
|
||||
|
||||
Write-Host "Creating temporary service-channel route intents..."
|
||||
$badIntent = (New-RouteIntent -SourceNodeID $entryNode.id -DestinationNodeID $exitNode.id -Priority 9700 -Label "bad").route_intent
|
||||
$goodIntent = (New-RouteIntent -SourceNodeID $entryNode.id -DestinationNodeID $exitNode.id -Priority 9600 -Label "good").route_intent
|
||||
|
||||
Write-Host "Injecting fenced/healthy route feedback through heartbeat..."
|
||||
$feedbackHeartbeat = Send-FeedbackHeartbeat -EntryNodeID $entryNode.id -BadRouteID $badIntent.id -GoodRouteID $goodIntent.id
|
||||
$appliedDecision = Wait-ForConfigDecision -NodeID $entryNode.id -BadRouteID $badIntent.id -ExpectedStatus "applied"
|
||||
$appliedTransition = Wait-ForTransitionStatus -NodeID $entryNode.id -Statuses @("applied_rebuild")
|
||||
|
||||
Write-Host "Expiring route feedback and waiting for fresh config restore..."
|
||||
$expireResult = Invoke-Api -Method POST -Path "/clusters/$ClusterID/fabric/service-channels/route-feedback/expire" -Body @{
|
||||
actor_user_id = $ActorUserID
|
||||
reporter_node_id = $entryNode.id
|
||||
route_id = $badIntent.id
|
||||
service_class = "vpn_packets"
|
||||
reason = "c18w smoke restore by fresh config"
|
||||
}
|
||||
$cleanHeartbeat = Send-CleanHeartbeat -EntryNodeID $entryNode.id
|
||||
$restoredConfig = Wait-ForNoRebuildDecision -NodeID $entryNode.id -BadRouteID $badIntent.id
|
||||
$restoredTransition = Wait-ForTransitionStatus -NodeID $entryNode.id -Statuses @("restored_by_new_config", "empty")
|
||||
|
||||
$result = [ordered]@{
|
||||
schema_version = "c18w.service_channel_route_manager_smoke.v1"
|
||||
run_id = $runId
|
||||
api_base_url = $ApiBaseUrl
|
||||
cluster_id = $ClusterID
|
||||
actor_user_id = $ActorUserID
|
||||
entry_node = @{
|
||||
id = $entryNode.id
|
||||
name = $entryNode.name
|
||||
reported_version = $entryNode.reported_version
|
||||
}
|
||||
exit_node = @{
|
||||
id = $exitNode.id
|
||||
name = $exitNode.name
|
||||
reported_version = $exitNode.reported_version
|
||||
}
|
||||
routes = @{
|
||||
bad_route_id = $badIntent.id
|
||||
good_route_id = $goodIntent.id
|
||||
}
|
||||
feedback_heartbeat_id = $feedbackHeartbeat.heartbeat.id
|
||||
applied_decision = $appliedDecision.decision
|
||||
applied_transition = $appliedTransition.transition
|
||||
expire_result = $expireResult.route_feedback_expire
|
||||
clean_heartbeat_id = $cleanHeartbeat.heartbeat.id
|
||||
restored_config_generation = $restoredConfig.config.synthetic_mesh_config.config_version
|
||||
restored_decision = $restoredConfig.decision
|
||||
restored_transition = $restoredTransition.transition
|
||||
checks = @{
|
||||
applied_decision = $appliedDecision.decision.rebuild_status -eq "applied"
|
||||
applied_transition = $appliedTransition.transition.status -eq "applied_rebuild"
|
||||
feedback_expired = $expireResult.route_feedback_expire.expired_count -ge 1
|
||||
restored_config_has_no_rebuild_decision = $true
|
||||
restored_transition_seen = @("restored_by_new_config", "empty") -contains [string]$restoredTransition.transition.status
|
||||
}
|
||||
}
|
||||
|
||||
$resultDir = Split-Path -Parent (Join-Path $repoRoot $ResultPath)
|
||||
New-Item -ItemType Directory -Force -Path $resultDir | Out-Null
|
||||
$absoluteResultPath = Join-Path $repoRoot $ResultPath
|
||||
$result | ConvertTo-Json -Depth 100 | Set-Content -Encoding UTF8 $absoluteResultPath
|
||||
Write-Host "C18W smoke passed. Result: $absoluteResultPath"
|
||||
$result
|
||||
@@ -0,0 +1,47 @@
|
||||
param(
|
||||
[string]$ResultPath = "artifacts\c18x-service-channel-logical-channel-smoke-result.json"
|
||||
)
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
$repoRoot = (Resolve-Path (Join-Path $PSScriptRoot "..\..")).ProviderPath
|
||||
$agentRoot = Join-Path $repoRoot "agents\rap-node-agent"
|
||||
$runId = "c18x-" + (Get-Date -Format "yyyyMMdd-HHmmss")
|
||||
$testPattern = "TestFabricClientPacketIngress(IsolatesRouteFailoverPerLogicalChannel|ReportsBoundedBackpressurePerLogicalChannel|SplitsIndependentFlowsIntoLogicalChannels)"
|
||||
|
||||
Push-Location $agentRoot
|
||||
try {
|
||||
$output = & go test ./internal/vpnruntime -run $testPattern -v 2>&1 | ForEach-Object { $_.ToString() }
|
||||
$exitCode = $LASTEXITCODE
|
||||
}
|
||||
finally {
|
||||
Pop-Location
|
||||
}
|
||||
|
||||
$passed = $exitCode -eq 0
|
||||
$result = [ordered]@{
|
||||
schema_version = "c18x.service_channel_logical_channel_smoke.v1"
|
||||
run_id = $runId
|
||||
test_package = "github.com/example/remote-access-platform/agents/rap-node-agent/internal/vpnruntime"
|
||||
test_pattern = $testPattern
|
||||
passed = $passed
|
||||
exit_code = $exitCode
|
||||
checks = [ordered]@{
|
||||
independent_logical_channel_route_failover = [bool]($output -match "PASS: TestFabricClientPacketIngressIsolatesRouteFailoverPerLogicalChannel")
|
||||
bounded_backpressure_telemetry = [bool]($output -match "PASS: TestFabricClientPacketIngressReportsBoundedBackpressurePerLogicalChannel")
|
||||
logical_channel_split = [bool]($output -match "PASS: TestFabricClientPacketIngressSplitsIndependentFlowsIntoLogicalChannels")
|
||||
}
|
||||
output = $output
|
||||
}
|
||||
|
||||
$absoluteResultPath = Join-Path $repoRoot $ResultPath
|
||||
New-Item -ItemType Directory -Force -Path (Split-Path -Parent $absoluteResultPath) | Out-Null
|
||||
$result | ConvertTo-Json -Depth 20 | Set-Content -Encoding UTF8 $absoluteResultPath
|
||||
|
||||
if (-not $passed) {
|
||||
throw "C18X logical-channel smoke failed. Result: $absoluteResultPath"
|
||||
}
|
||||
|
||||
Write-Host "C18X logical-channel smoke passed. Result: $absoluteResultPath"
|
||||
$result
|
||||
@@ -0,0 +1,137 @@
|
||||
param(
|
||||
[string]$BaseUrl = "http://192.168.200.61:18121/api/v1",
|
||||
[string]$ClusterID = "cfc0743d-d960-49fb-9de8-96e063d5e4aa",
|
||||
[string]$ActorUserID = "f67d943f-5397-4b3a-a229-695fe67ad700",
|
||||
[string]$EntryNodeName = "test-1",
|
||||
[string]$ExitNodeName = "test-2",
|
||||
[string]$ResultPath = "artifacts\c18y-route-intent-lifecycle-smoke-result.json"
|
||||
)
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
$repoRoot = (Resolve-Path (Join-Path $PSScriptRoot "..\..")).ProviderPath
|
||||
$runId = "c18y-" + (Get-Date -Format "yyyyMMdd-HHmmss")
|
||||
|
||||
function Invoke-ApiJson {
|
||||
param(
|
||||
[string]$Method = "GET",
|
||||
[string]$Path,
|
||||
[object]$Body = $null
|
||||
)
|
||||
$uri = "$BaseUrl$Path"
|
||||
if ($null -eq $Body) {
|
||||
return Invoke-RestMethod -Method $Method -Uri $uri
|
||||
}
|
||||
$json = $Body | ConvertTo-Json -Depth 20
|
||||
return Invoke-RestMethod -Method $Method -Uri $uri -ContentType "application/json" -Body $json
|
||||
}
|
||||
|
||||
function New-RouteIntent {
|
||||
param(
|
||||
[string]$Label,
|
||||
[int]$Priority
|
||||
)
|
||||
$expiresAt = (Get-Date).ToUniversalTime().AddMinutes(10).ToString("o")
|
||||
return Invoke-ApiJson -Method "POST" -Path "/clusters/$ClusterID/mesh/route-intents" -Body @{
|
||||
actor_user_id = $ActorUserID
|
||||
source_selector = @{ node_id = $entryNode.id }
|
||||
destination_selector = @{ node_id = $exitNode.id }
|
||||
service_class = "vpn_packets"
|
||||
priority = $Priority
|
||||
policy = @{
|
||||
synthetic_enabled = $true
|
||||
allowed_channels = @("vpn_packet", "fabric_control")
|
||||
hops = @($entryNode.id, $exitNode.id)
|
||||
expires_at = $expiresAt
|
||||
max_ttl = 8
|
||||
max_hops = 8
|
||||
route_version = "$runId-$Label"
|
||||
policy_version = "$runId-$Label"
|
||||
peer_directory_version = "$runId-$Label"
|
||||
metadata = @{
|
||||
smoke = "c18y_route_intent_lifecycle"
|
||||
run_id = $runId
|
||||
label = $Label
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function Get-RouteIntent {
|
||||
param([string]$RouteIntentID)
|
||||
$items = (Invoke-ApiJson -Path "/clusters/$ClusterID/mesh/route-intents?actor_user_id=$ActorUserID").route_intents
|
||||
return @($items | Where-Object { $_.id -eq $RouteIntentID })[0]
|
||||
}
|
||||
|
||||
function Get-NodeRouteIDs {
|
||||
param([string]$NodeID)
|
||||
$config = (Invoke-ApiJson -Path "/clusters/$ClusterID/nodes/$NodeID/mesh/synthetic-config").synthetic_mesh_config
|
||||
return @($config.routes | ForEach-Object { $_.route_id })
|
||||
}
|
||||
|
||||
$nodes = (Invoke-ApiJson -Path "/clusters/$ClusterID/nodes?actor_user_id=$ActorUserID").nodes
|
||||
$entryNode = @($nodes | Where-Object { $_.name -eq $EntryNodeName })[0]
|
||||
$exitNode = @($nodes | Where-Object { $_.name -eq $ExitNodeName })[0]
|
||||
if (-not $entryNode -or -not $exitNode) {
|
||||
throw "Required nodes not found: entry=$EntryNodeName exit=$ExitNodeName"
|
||||
}
|
||||
|
||||
$expireIntent = (New-RouteIntent -Label "expire" -Priority 9810).route_intent
|
||||
$disableIntent = (New-RouteIntent -Label "disable" -Priority 9820).route_intent
|
||||
|
||||
$routeIDsBefore = Get-NodeRouteIDs -NodeID $entryNode.id
|
||||
$expireVisibleBefore = $routeIDsBefore -contains $expireIntent.id
|
||||
$disableVisibleBefore = $routeIDsBefore -contains $disableIntent.id
|
||||
|
||||
$expired = (Invoke-ApiJson -Method "POST" -Path "/clusters/$ClusterID/mesh/route-intents/$($expireIntent.id)/expire" -Body @{
|
||||
actor_user_id = $ActorUserID
|
||||
reason = "C18Y smoke expire"
|
||||
}).route_intent
|
||||
|
||||
$disabled = (Invoke-ApiJson -Method "POST" -Path "/clusters/$ClusterID/mesh/route-intents/$($disableIntent.id)/disable" -Body @{
|
||||
actor_user_id = $ActorUserID
|
||||
reason = "C18Y smoke disable"
|
||||
}).route_intent
|
||||
|
||||
$routeIDsAfter = Get-NodeRouteIDs -NodeID $entryNode.id
|
||||
$expireVisibleAfter = $routeIDsAfter -contains $expireIntent.id
|
||||
$disableVisibleAfter = $routeIDsAfter -contains $disableIntent.id
|
||||
$expiredListed = Get-RouteIntent -RouteIntentID $expireIntent.id
|
||||
$disabledListed = Get-RouteIntent -RouteIntentID $disableIntent.id
|
||||
|
||||
$checks = [ordered]@{
|
||||
route_visible_before_expire = [bool]$expireVisibleBefore
|
||||
route_visible_before_disable = [bool]$disableVisibleBefore
|
||||
expire_action_marks_expired = [bool]($expired.lifecycle_status -eq "expired" -and $expired.is_expired -eq $true)
|
||||
disable_action_marks_disabled = [bool]($disabled.lifecycle_status -eq "disabled" -and $disabled.status -eq "disabled")
|
||||
expired_route_removed_from_synthetic_config = [bool](-not $expireVisibleAfter)
|
||||
disabled_route_removed_from_synthetic_config = [bool](-not $disableVisibleAfter)
|
||||
list_reports_expired = [bool]($expiredListed.lifecycle_status -eq "expired")
|
||||
list_reports_disabled = [bool]($disabledListed.lifecycle_status -eq "disabled")
|
||||
}
|
||||
|
||||
$passed = -not ($checks.Values -contains $false)
|
||||
$result = [ordered]@{
|
||||
schema_version = "c18y.route_intent_lifecycle_smoke.v1"
|
||||
run_id = $runId
|
||||
base_url = $BaseUrl
|
||||
cluster_id = $ClusterID
|
||||
entry_node = @{ id = $entryNode.id; name = $entryNode.name }
|
||||
exit_node = @{ id = $exitNode.id; name = $exitNode.name }
|
||||
expire_route_intent_id = $expireIntent.id
|
||||
disable_route_intent_id = $disableIntent.id
|
||||
passed = $passed
|
||||
checks = $checks
|
||||
}
|
||||
|
||||
$absoluteResultPath = Join-Path $repoRoot $ResultPath
|
||||
New-Item -ItemType Directory -Force -Path (Split-Path -Parent $absoluteResultPath) | Out-Null
|
||||
$result | ConvertTo-Json -Depth 20 | Set-Content -Encoding UTF8 $absoluteResultPath
|
||||
|
||||
if (-not $passed) {
|
||||
throw "C18Y route-intent lifecycle smoke failed. Result: $absoluteResultPath"
|
||||
}
|
||||
|
||||
Write-Host "C18Y route-intent lifecycle smoke passed. Result: $absoluteResultPath"
|
||||
$result
|
||||
@@ -0,0 +1,149 @@
|
||||
param(
|
||||
[string]$BaseUrl = "http://192.168.200.61:18121/api/v1",
|
||||
[string]$ClusterID = "cfc0743d-d960-49fb-9de8-96e063d5e4aa",
|
||||
[string]$ActorUserID = "f67d943f-5397-4b3a-a229-695fe67ad700",
|
||||
[string]$EntryNodeName = "test-1",
|
||||
[string]$ExitNodeName = "test-2",
|
||||
[string]$ResultPath = "artifacts\c18z-service-channel-load-smoke-result.json"
|
||||
)
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
$repoRoot = (Resolve-Path (Join-Path $PSScriptRoot "..\..")).ProviderPath
|
||||
$agentRoot = Join-Path $repoRoot "agents\rap-node-agent"
|
||||
$runId = "c18z-" + (Get-Date -Format "yyyyMMdd-HHmmss")
|
||||
$testPattern = "TestFabricClientPacketIngressBoundedLoad(RebuildsAwayFromWithdrawnRoute|ReportsPerChannelDrops)"
|
||||
|
||||
function Invoke-ApiJson {
|
||||
param(
|
||||
[string]$Method = "GET",
|
||||
[string]$Path,
|
||||
[object]$Body = $null
|
||||
)
|
||||
$uri = "$BaseUrl$Path"
|
||||
if ($null -eq $Body) {
|
||||
return Invoke-RestMethod -Method $Method -Uri $uri
|
||||
}
|
||||
$json = $Body | ConvertTo-Json -Depth 20
|
||||
return Invoke-RestMethod -Method $Method -Uri $uri -ContentType "application/json" -Body $json
|
||||
}
|
||||
|
||||
function New-SmokeRouteIntent {
|
||||
param(
|
||||
[string]$Label,
|
||||
[int]$Priority
|
||||
)
|
||||
$expiresAt = (Get-Date).ToUniversalTime().AddMinutes(10).ToString("o")
|
||||
return Invoke-ApiJson -Method "POST" -Path "/clusters/$ClusterID/mesh/route-intents" -Body @{
|
||||
actor_user_id = $ActorUserID
|
||||
source_selector = @{ node_id = $entryNode.id }
|
||||
destination_selector = @{ node_id = $exitNode.id }
|
||||
service_class = "vpn_packets"
|
||||
priority = $Priority
|
||||
policy = @{
|
||||
synthetic_enabled = $true
|
||||
allowed_channels = @("vpn_packet", "fabric_control")
|
||||
hops = @($entryNode.id, $exitNode.id)
|
||||
expires_at = $expiresAt
|
||||
max_ttl = 8
|
||||
max_hops = 8
|
||||
route_version = "$runId-$Label"
|
||||
policy_version = "$runId-$Label"
|
||||
peer_directory_version = "$runId-$Label"
|
||||
metadata = @{
|
||||
smoke = "c18z_service_channel_load"
|
||||
run_id = $runId
|
||||
label = $Label
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function Get-RouteIntent {
|
||||
param([string]$RouteIntentID)
|
||||
$items = (Invoke-ApiJson -Path "/clusters/$ClusterID/mesh/route-intents?actor_user_id=$ActorUserID").route_intents
|
||||
return @($items | Where-Object { $_.id -eq $RouteIntentID })[0]
|
||||
}
|
||||
|
||||
function Get-NodeRouteIDs {
|
||||
param([string]$NodeID)
|
||||
$config = (Invoke-ApiJson -Path "/clusters/$ClusterID/nodes/$NodeID/mesh/synthetic-config").synthetic_mesh_config
|
||||
return @($config.routes | ForEach-Object { $_.route_id })
|
||||
}
|
||||
|
||||
Push-Location $agentRoot
|
||||
try {
|
||||
$testOutput = & go test ./internal/vpnruntime -run $testPattern -v 2>&1 | ForEach-Object { $_.ToString() }
|
||||
$testExitCode = $LASTEXITCODE
|
||||
}
|
||||
finally {
|
||||
Pop-Location
|
||||
}
|
||||
|
||||
$nodes = (Invoke-ApiJson -Path "/clusters/$ClusterID/nodes?actor_user_id=$ActorUserID").nodes
|
||||
$entryNode = @($nodes | Where-Object { $_.name -eq $EntryNodeName })[0]
|
||||
$exitNode = @($nodes | Where-Object { $_.name -eq $ExitNodeName })[0]
|
||||
if (-not $entryNode -or -not $exitNode) {
|
||||
throw "Required nodes not found: entry=$EntryNodeName exit=$ExitNodeName"
|
||||
}
|
||||
|
||||
$expireIntent = (New-SmokeRouteIntent -Label "expire-during-load" -Priority 9830).route_intent
|
||||
$disableIntent = (New-SmokeRouteIntent -Label "disable-during-load" -Priority 9840).route_intent
|
||||
$routeIDsBefore = Get-NodeRouteIDs -NodeID $entryNode.id
|
||||
|
||||
$expired = (Invoke-ApiJson -Method "POST" -Path "/clusters/$ClusterID/mesh/route-intents/$($expireIntent.id)/expire" -Body @{
|
||||
actor_user_id = $ActorUserID
|
||||
reason = "C18Z smoke expire during bounded load"
|
||||
}).route_intent
|
||||
|
||||
$disabled = (Invoke-ApiJson -Method "POST" -Path "/clusters/$ClusterID/mesh/route-intents/$($disableIntent.id)/disable" -Body @{
|
||||
actor_user_id = $ActorUserID
|
||||
reason = "C18Z smoke disable during bounded load"
|
||||
}).route_intent
|
||||
|
||||
$routeIDsAfter = Get-NodeRouteIDs -NodeID $entryNode.id
|
||||
$expiredListed = Get-RouteIntent -RouteIntentID $expireIntent.id
|
||||
$disabledListed = Get-RouteIntent -RouteIntentID $disableIntent.id
|
||||
|
||||
$checks = [ordered]@{
|
||||
bounded_load_rebuild_away_from_withdrawn_route = [bool]($testOutput -match "PASS: TestFabricClientPacketIngressBoundedLoadRebuildsAwayFromWithdrawnRoute")
|
||||
bounded_load_drop_telemetry = [bool]($testOutput -match "PASS: TestFabricClientPacketIngressBoundedLoadReportsPerChannelDrops")
|
||||
live_route_visible_before_expire = [bool]($routeIDsBefore -contains $expireIntent.id)
|
||||
live_route_visible_before_disable = [bool]($routeIDsBefore -contains $disableIntent.id)
|
||||
live_expire_marks_expired = [bool]($expired.lifecycle_status -eq "expired" -and $expired.is_expired -eq $true)
|
||||
live_disable_marks_disabled = [bool]($disabled.lifecycle_status -eq "disabled" -and $disabled.status -eq "disabled")
|
||||
live_expired_route_removed_from_synthetic_config = [bool](-not ($routeIDsAfter -contains $expireIntent.id))
|
||||
live_disabled_route_removed_from_synthetic_config = [bool](-not ($routeIDsAfter -contains $disableIntent.id))
|
||||
live_list_reports_expired = [bool]($expiredListed.lifecycle_status -eq "expired")
|
||||
live_list_reports_disabled = [bool]($disabledListed.lifecycle_status -eq "disabled")
|
||||
}
|
||||
|
||||
$passed = $testExitCode -eq 0 -and -not ($checks.Values -contains $false)
|
||||
$result = [ordered]@{
|
||||
schema_version = "c18z.service_channel_load_smoke.v1"
|
||||
run_id = $runId
|
||||
base_url = $BaseUrl
|
||||
cluster_id = $ClusterID
|
||||
entry_node = @{ id = $entryNode.id; name = $entryNode.name }
|
||||
exit_node = @{ id = $exitNode.id; name = $exitNode.name }
|
||||
test_package = "github.com/example/remote-access-platform/agents/rap-node-agent/internal/vpnruntime"
|
||||
test_pattern = $testPattern
|
||||
test_exit_code = $testExitCode
|
||||
expire_route_intent_id = $expireIntent.id
|
||||
disable_route_intent_id = $disableIntent.id
|
||||
passed = $passed
|
||||
checks = $checks
|
||||
test_output = $testOutput
|
||||
}
|
||||
|
||||
$absoluteResultPath = Join-Path $repoRoot $ResultPath
|
||||
New-Item -ItemType Directory -Force -Path (Split-Path -Parent $absoluteResultPath) | Out-Null
|
||||
$result | ConvertTo-Json -Depth 20 | Set-Content -Encoding UTF8 $absoluteResultPath
|
||||
|
||||
if (-not $passed) {
|
||||
throw "C18Z service-channel load smoke failed. Result: $absoluteResultPath"
|
||||
}
|
||||
|
||||
Write-Host "C18Z service-channel load smoke passed. Result: $absoluteResultPath"
|
||||
$result
|
||||
@@ -0,0 +1,649 @@
|
||||
param(
|
||||
[string]$ApiBaseUrl = "http://192.168.200.61:18121/api/v1",
|
||||
[string]$ClusterID = "cfc0743d-d960-49fb-9de8-96e063d5e4aa",
|
||||
[string]$ActorUserID = "f67d943f-5397-4b3a-a229-695fe67ad700",
|
||||
[string]$EntryNodeName = "test-1",
|
||||
[string]$ExitNodeName = "test-2",
|
||||
[string]$EntryBaseUrl = "http://192.168.200.61:19131",
|
||||
[string]$RequiredNodeVersion = "0.2.182",
|
||||
[string]$ResultPath = "artifacts\c18z1-live-service-channel-ingress-smoke-result.json"
|
||||
)
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = "Stop"
|
||||
Add-Type -AssemblyName System.Net.Http
|
||||
|
||||
$repoRoot = (Resolve-Path (Join-Path $PSScriptRoot "..\..")).ProviderPath
|
||||
$runId = "c18z1-" + (Get-Date -Format "yyyyMMdd-HHmmss")
|
||||
$resourceId = "vpn-$runId"
|
||||
|
||||
function Invoke-Api {
|
||||
param(
|
||||
[string]$Method,
|
||||
[string]$Path,
|
||||
[object]$Body = $null
|
||||
)
|
||||
$uri = "$ApiBaseUrl$Path"
|
||||
try {
|
||||
if ($null -eq $Body) {
|
||||
return Invoke-RestMethod -Method $Method -Uri $uri -TimeoutSec 30
|
||||
}
|
||||
return Invoke-RestMethod -Method $Method -Uri $uri -ContentType "application/json" -Body ($Body | ConvertTo-Json -Depth 80) -TimeoutSec 30
|
||||
}
|
||||
catch {
|
||||
$statusCode = $null
|
||||
if ($_.Exception.Response) {
|
||||
$statusCode = [int]$_.Exception.Response.StatusCode
|
||||
}
|
||||
$details = $_.ErrorDetails.Message
|
||||
if (-not $details) {
|
||||
$details = $_.Exception.Message
|
||||
}
|
||||
throw "$Method $Path failed with HTTP $statusCode`: $details"
|
||||
}
|
||||
}
|
||||
|
||||
function Get-NodeByName {
|
||||
param([string]$Name)
|
||||
$nodes = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/nodes?actor_user_id=$ActorUserID").nodes
|
||||
$node = @($nodes | Where-Object { $_.name -eq $Name }) | Select-Object -First 1
|
||||
if ($null -eq $node) {
|
||||
throw "Node '$Name' was not found in cluster $ClusterID"
|
||||
}
|
||||
return $node
|
||||
}
|
||||
|
||||
function Get-MeshPort {
|
||||
param([string]$Name)
|
||||
switch ($Name) {
|
||||
"test-1" { return 19131 }
|
||||
"test-2" { return 19132 }
|
||||
"test-3" { return 19133 }
|
||||
default { return 19131 }
|
||||
}
|
||||
}
|
||||
|
||||
function Enable-TestMeshListener {
|
||||
param([object]$Node)
|
||||
$port = Get-MeshPort -Name $Node.name
|
||||
Invoke-Api -Method PUT -Path "/clusters/$ClusterID/nodes/$($Node.id)/workloads/mesh-listener/desired" -Body @{
|
||||
actor_user_id = $ActorUserID
|
||||
desired_state = "enabled"
|
||||
runtime_mode = "container"
|
||||
version = "c18z1-live-fsc"
|
||||
config = @{
|
||||
listen_addr = "0.0.0.0:$port"
|
||||
listen_port_mode = "manual"
|
||||
advertise_endpoint = "http://192.168.200.61:$port"
|
||||
advertise_transport = "direct_http"
|
||||
connectivity_mode = "private_lan"
|
||||
nat_type = "none"
|
||||
region = "docker-test"
|
||||
production_forwarding = $true
|
||||
}
|
||||
environment = @{}
|
||||
} | Out-Null
|
||||
}
|
||||
|
||||
function Clear-OldSmokeRouteIntents {
|
||||
param(
|
||||
[string]$SourceNodeID,
|
||||
[string]$DestinationNodeID
|
||||
)
|
||||
$items = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/mesh/route-intents?actor_user_id=$ActorUserID").route_intents
|
||||
foreach ($item in @($items)) {
|
||||
if ([string]$item.lifecycle_status -ne "active") {
|
||||
continue
|
||||
}
|
||||
if ([string]$item.service_class -ne "vpn_packets") {
|
||||
continue
|
||||
}
|
||||
if ([string]$item.source_selector.node_id -ne $SourceNodeID -or [string]$item.destination_selector.node_id -ne $DestinationNodeID) {
|
||||
continue
|
||||
}
|
||||
$smoke = ""
|
||||
if ($null -ne $item.policy -and $null -ne $item.policy.metadata) {
|
||||
$prop = $item.policy.metadata.PSObject.Properties["smoke"]
|
||||
if ($null -ne $prop) {
|
||||
$smoke = [string]$prop.Value
|
||||
}
|
||||
}
|
||||
if ($smoke -ne "c18z1_live_service_channel_ingress") {
|
||||
continue
|
||||
}
|
||||
Invoke-Api -Method POST -Path "/clusters/$ClusterID/mesh/route-intents/$($item.id)/expire" -Body @{ actor_user_id = $ActorUserID } | Out-Null
|
||||
}
|
||||
}
|
||||
|
||||
function New-RouteIntent {
|
||||
param(
|
||||
[string]$SourceNodeID,
|
||||
[string]$DestinationNodeID,
|
||||
[int]$Priority,
|
||||
[string]$Label
|
||||
)
|
||||
$expiresAt = (Get-Date).ToUniversalTime().AddMinutes(10).ToString("o")
|
||||
return Invoke-Api -Method POST -Path "/clusters/$ClusterID/mesh/route-intents" -Body @{
|
||||
actor_user_id = $ActorUserID
|
||||
source_selector = @{ node_id = $SourceNodeID }
|
||||
destination_selector = @{ node_id = $DestinationNodeID }
|
||||
service_class = "vpn_packets"
|
||||
priority = $Priority
|
||||
policy = @{
|
||||
synthetic_enabled = $true
|
||||
route_version = "$runId-$Label"
|
||||
policy_version = "$runId-$Label"
|
||||
peer_directory_version = "$runId-$Label"
|
||||
hops = @($SourceNodeID, $DestinationNodeID)
|
||||
allowed_channels = @("vpn_packet", "fabric_control")
|
||||
max_ttl = 8
|
||||
max_hops = 8
|
||||
expires_at = $expiresAt
|
||||
metadata = @{
|
||||
smoke = "c18z1_live_service_channel_ingress"
|
||||
run_id = $runId
|
||||
label = $Label
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function Get-SyntheticConfig {
|
||||
param([string]$NodeID)
|
||||
return Invoke-Api -Method GET -Path "/clusters/$ClusterID/nodes/$NodeID/mesh/synthetic-config?actor_user_id=$ActorUserID"
|
||||
}
|
||||
|
||||
function Get-LatestHeartbeat {
|
||||
param([string]$NodeID)
|
||||
return (Invoke-Api -Method GET -Path "/clusters/$ClusterID/nodes/$NodeID/heartbeats?actor_user_id=$ActorUserID&limit=1").heartbeats[0]
|
||||
}
|
||||
|
||||
function Get-LatestRuntimeReport {
|
||||
param([string]$NodeID)
|
||||
$hb = Get-LatestHeartbeat -NodeID $NodeID
|
||||
return @{
|
||||
heartbeat = $hb
|
||||
report = $hb.metadata.fabric_service_channel_runtime_report
|
||||
}
|
||||
}
|
||||
|
||||
function Wait-ForRuntimeReady {
|
||||
param(
|
||||
[string]$NodeID,
|
||||
[int]$MinRoutes,
|
||||
[int]$TimeoutSeconds = 90
|
||||
)
|
||||
$deadline = (Get-Date).AddSeconds($TimeoutSeconds)
|
||||
do {
|
||||
$latest = Get-LatestRuntimeReport -NodeID $NodeID
|
||||
$report = $latest.report
|
||||
if ($null -ne $report -and
|
||||
$report.enabled -eq $true -and
|
||||
$report.production_payload_forwarding -eq $true -and
|
||||
[int]$report.route_candidate_total -ge $MinRoutes) {
|
||||
return $latest
|
||||
}
|
||||
Start-Sleep -Seconds 2
|
||||
} while ((Get-Date) -lt $deadline)
|
||||
throw "Timed out waiting for production service-channel runtime ready on node $NodeID"
|
||||
}
|
||||
|
||||
function Wait-ForRuntimeConfigVersion {
|
||||
param(
|
||||
[string]$NodeID,
|
||||
[string]$ConfigVersion,
|
||||
[int]$TimeoutSeconds = 90
|
||||
)
|
||||
$deadline = (Get-Date).AddSeconds($TimeoutSeconds)
|
||||
do {
|
||||
$latest = Get-LatestRuntimeReport -NodeID $NodeID
|
||||
if ($null -ne $latest.report) {
|
||||
$loadedVersion = [string]$latest.report.config_version
|
||||
if ($loadedVersion -ge $ConfigVersion) {
|
||||
return $latest
|
||||
}
|
||||
}
|
||||
Start-Sleep -Seconds 2
|
||||
} while ((Get-Date) -lt $deadline)
|
||||
throw "Timed out waiting for node $NodeID to load synthetic config $ConfigVersion"
|
||||
}
|
||||
|
||||
function Wait-ForRouteIntentVisible {
|
||||
param(
|
||||
[string]$NodeID,
|
||||
[string[]]$RouteIDs,
|
||||
[int]$TimeoutSeconds = 60
|
||||
)
|
||||
$deadline = (Get-Date).AddSeconds($TimeoutSeconds)
|
||||
do {
|
||||
$config = Get-SyntheticConfig -NodeID $NodeID
|
||||
$routes = @($config.synthetic_mesh_config.routes)
|
||||
$present = @($routes | Where-Object { $RouteIDs -contains $_.route_id })
|
||||
if ($present.Count -ge $RouteIDs.Count) {
|
||||
return $config
|
||||
}
|
||||
Start-Sleep -Seconds 2
|
||||
} while ((Get-Date) -lt $deadline)
|
||||
throw "Timed out waiting for routes '$($RouteIDs -join ",")' in synthetic config for node $NodeID"
|
||||
}
|
||||
|
||||
function New-ServiceChannelLease {
|
||||
param(
|
||||
[string]$EntryNodeID,
|
||||
[string]$ExitNodeID
|
||||
)
|
||||
return (Invoke-Api -Method POST -Path "/clusters/$ClusterID/fabric/service-channels/leases" -Body @{
|
||||
actor_user_id = $ActorUserID
|
||||
organization_id = "org-c18z1-smoke"
|
||||
user_id = $ActorUserID
|
||||
resource_id = $resourceId
|
||||
service_class = "vpn_packets"
|
||||
entry_node_ids = @($EntryNodeID)
|
||||
exit_node_ids = @($ExitNodeID)
|
||||
preferred_entry_node_id = $EntryNodeID
|
||||
preferred_exit_node_id = $ExitNodeID
|
||||
allowed_channels = @("vpn_packet", "bulk", "control")
|
||||
ttl_seconds = 300
|
||||
metadata = @{
|
||||
smoke = "c18z1_live_service_channel_ingress"
|
||||
run_id = $runId
|
||||
}
|
||||
}).fabric_service_channel_lease
|
||||
}
|
||||
|
||||
function ConvertTo-Base64UrlJson {
|
||||
param([object]$Value)
|
||||
$json = $Value | ConvertTo-Json -Depth 80 -Compress
|
||||
$bytes = [System.Text.Encoding]::UTF8.GetBytes($json)
|
||||
return [Convert]::ToBase64String($bytes).TrimEnd("=").Replace("+", "-").Replace("/", "_")
|
||||
}
|
||||
|
||||
function Get-ObjectPropertyValue {
|
||||
param(
|
||||
[object]$Object,
|
||||
[string]$Name
|
||||
)
|
||||
if ($null -eq $Object) {
|
||||
return $null
|
||||
}
|
||||
$prop = $Object.PSObject.Properties[$Name]
|
||||
if ($null -eq $prop) {
|
||||
return $null
|
||||
}
|
||||
return $prop.Value
|
||||
}
|
||||
|
||||
function New-TestIPv4UDPPacket {
|
||||
param([int]$SourcePort)
|
||||
$payload = [System.Text.Encoding]::ASCII.GetBytes("c18z1-$SourcePort")
|
||||
$totalLength = 20 + 8 + $payload.Length
|
||||
$packet = New-Object byte[] $totalLength
|
||||
$packet[0] = 0x45
|
||||
$packet[1] = 0
|
||||
$packet[2] = [byte](($totalLength -shr 8) -band 0xff)
|
||||
$packet[3] = [byte]($totalLength -band 0xff)
|
||||
$packet[8] = 64
|
||||
$packet[9] = 17
|
||||
$packet[12] = 10; $packet[13] = 18; $packet[14] = 1; $packet[15] = 10
|
||||
$packet[16] = 10; $packet[17] = 18; $packet[18] = 2; $packet[19] = 20
|
||||
$udpOffset = 20
|
||||
$destPort = 3389
|
||||
$udpLength = 8 + $payload.Length
|
||||
$packet[$udpOffset] = [byte](($SourcePort -shr 8) -band 0xff)
|
||||
$packet[$udpOffset + 1] = [byte]($SourcePort -band 0xff)
|
||||
$packet[$udpOffset + 2] = [byte](($destPort -shr 8) -band 0xff)
|
||||
$packet[$udpOffset + 3] = [byte]($destPort -band 0xff)
|
||||
$packet[$udpOffset + 4] = [byte](($udpLength -shr 8) -band 0xff)
|
||||
$packet[$udpOffset + 5] = [byte]($udpLength -band 0xff)
|
||||
[Array]::Copy($payload, 0, $packet, 28, $payload.Length)
|
||||
return $packet
|
||||
}
|
||||
|
||||
function New-PacketBatchBody {
|
||||
param([byte[][]]$Packets)
|
||||
$stream = [System.IO.MemoryStream]::new()
|
||||
foreach ($packet in $Packets) {
|
||||
$length = $packet.Length
|
||||
$stream.WriteByte([byte](($length -shr 24) -band 0xff))
|
||||
$stream.WriteByte([byte](($length -shr 16) -band 0xff))
|
||||
$stream.WriteByte([byte](($length -shr 8) -band 0xff))
|
||||
$stream.WriteByte([byte]($length -band 0xff))
|
||||
$stream.Write($packet, 0, $packet.Length)
|
||||
}
|
||||
return $stream.ToArray()
|
||||
}
|
||||
|
||||
function Invoke-ServiceChannelPost {
|
||||
param(
|
||||
[object]$Lease,
|
||||
[int]$PortStart
|
||||
)
|
||||
$packets = @()
|
||||
for ($i = 0; $i -lt 8; $i++) {
|
||||
$packets += ,(New-TestIPv4UDPPacket -SourcePort ($PortStart + $i))
|
||||
}
|
||||
$path = $Lease.entry_http.path_template.
|
||||
Replace("{cluster_id}", $ClusterID).
|
||||
Replace("{channel_id}", $Lease.channel_id).
|
||||
Replace("{resource_id}", $resourceId)
|
||||
$url = "$EntryBaseUrl$path`?batch=true"
|
||||
$headers = @{
|
||||
"X-RAP-Service-Channel-Token" = $Lease.token.token
|
||||
"X-RAP-Fabric-Channel-ID" = $Lease.channel_id
|
||||
"X-RAP-Service-Class" = "vpn_packets"
|
||||
"X-RAP-Channel-Class" = "vpn_packet"
|
||||
"X-RAP-Service-Channel-Authority-Payload" = ConvertTo-Base64UrlJson -Value $Lease.authority_payload
|
||||
"X-RAP-Service-Channel-Authority-Signature" = ConvertTo-Base64UrlJson -Value $Lease.authority_signature
|
||||
}
|
||||
$body = New-PacketBatchBody -Packets $packets
|
||||
$client = [System.Net.Http.HttpClient]::new()
|
||||
try {
|
||||
$client.Timeout = [TimeSpan]::FromSeconds(30)
|
||||
$request = [System.Net.Http.HttpRequestMessage]::new([System.Net.Http.HttpMethod]::Post, $url)
|
||||
foreach ($header in $headers.GetEnumerator()) {
|
||||
[void]$request.Headers.TryAddWithoutValidation($header.Key, [string]$header.Value)
|
||||
}
|
||||
$content = [System.Net.Http.ByteArrayContent]::new($body)
|
||||
$content.Headers.ContentType = [System.Net.Http.Headers.MediaTypeHeaderValue]::Parse("application/vnd.rap.vpn-packet-batch.v1")
|
||||
$request.Content = $content
|
||||
$response = $client.SendAsync($request).GetAwaiter().GetResult()
|
||||
$responseBody = $response.Content.ReadAsStringAsync().GetAwaiter().GetResult()
|
||||
if (-not $response.IsSuccessStatusCode) {
|
||||
throw "Service-channel POST $url failed with HTTP $([int]$response.StatusCode): $responseBody"
|
||||
}
|
||||
return [pscustomobject]@{
|
||||
StatusCode = [int]$response.StatusCode
|
||||
Body = $responseBody
|
||||
}
|
||||
}
|
||||
finally {
|
||||
$client.Dispose()
|
||||
}
|
||||
}
|
||||
|
||||
function Get-IngressSendPackets {
|
||||
param([string]$NodeID)
|
||||
$latest = Get-LatestRuntimeReport -NodeID $NodeID
|
||||
$ingress = $latest.report.ingress
|
||||
$sendPackets = Get-ObjectPropertyValue -Object $ingress -Name "send_packets"
|
||||
if ($null -eq $sendPackets) {
|
||||
return 0
|
||||
}
|
||||
return [int]$sendPackets
|
||||
}
|
||||
|
||||
function Wait-ForIngressRoute {
|
||||
param(
|
||||
[string]$NodeID,
|
||||
[string]$RouteID,
|
||||
[int]$MinSendPackets,
|
||||
[int]$TimeoutSeconds = 45
|
||||
)
|
||||
$deadline = (Get-Date).AddSeconds($TimeoutSeconds)
|
||||
do {
|
||||
$latest = Get-LatestRuntimeReport -NodeID $NodeID
|
||||
$ingress = $latest.report.ingress
|
||||
$sendPackets = Get-ObjectPropertyValue -Object $ingress -Name "send_packets"
|
||||
$selectedRoute = Get-ObjectPropertyValue -Object $ingress -Name "last_selected_route_id"
|
||||
if ($null -ne $ingress -and
|
||||
[int]$sendPackets -ge $MinSendPackets -and
|
||||
[string]$selectedRoute -eq $RouteID) {
|
||||
return $latest
|
||||
}
|
||||
Start-Sleep -Seconds 2
|
||||
} while ((Get-Date) -lt $deadline)
|
||||
throw "Timed out waiting for ingress telemetry route=$RouteID packets>=$MinSendPackets on node $NodeID"
|
||||
}
|
||||
|
||||
function Wait-ForIngressAnyRoute {
|
||||
param(
|
||||
[string]$NodeID,
|
||||
[string[]]$RouteIDs,
|
||||
[int]$MinSendPackets,
|
||||
[int]$TimeoutSeconds = 45
|
||||
)
|
||||
$deadline = (Get-Date).AddSeconds($TimeoutSeconds)
|
||||
do {
|
||||
$latest = Get-LatestRuntimeReport -NodeID $NodeID
|
||||
$ingress = $latest.report.ingress
|
||||
$sendPackets = Get-ObjectPropertyValue -Object $ingress -Name "send_packets"
|
||||
$selectedRoute = Get-ObjectPropertyValue -Object $ingress -Name "last_selected_route_id"
|
||||
if ($null -ne $ingress -and
|
||||
[int]$sendPackets -ge $MinSendPackets -and
|
||||
$RouteIDs -contains [string]$selectedRoute) {
|
||||
return $latest
|
||||
}
|
||||
Start-Sleep -Seconds 2
|
||||
} while ((Get-Date) -lt $deadline)
|
||||
throw "Timed out waiting for ingress telemetry routes='$($RouteIDs -join ",")' packets>=$MinSendPackets on node $NodeID"
|
||||
}
|
||||
|
||||
function Wait-ForExitInbox {
|
||||
param(
|
||||
[string]$NodeID,
|
||||
[string]$VPNConnectionID,
|
||||
[int]$TimeoutSeconds = 45
|
||||
)
|
||||
$queueKey = "$VPNConnectionID`:client_to_gateway"
|
||||
$deadline = (Get-Date).AddSeconds($TimeoutSeconds)
|
||||
do {
|
||||
$latest = Get-LatestRuntimeReport -NodeID $NodeID
|
||||
$depths = $latest.report.inbox.queue_depths
|
||||
if ($null -ne $depths) {
|
||||
$prop = $depths.PSObject.Properties[$queueKey]
|
||||
if ($null -ne $prop -and [int]$prop.Value -gt 0) {
|
||||
return $latest
|
||||
}
|
||||
}
|
||||
Start-Sleep -Seconds 2
|
||||
} while ((Get-Date) -lt $deadline)
|
||||
throw "Timed out waiting for exit inbox queue '$queueKey' on node $NodeID"
|
||||
}
|
||||
|
||||
function Send-FeedbackHeartbeat {
|
||||
param(
|
||||
[string]$EntryNodeID,
|
||||
[string]$BadRouteID,
|
||||
[string]$GoodRouteID
|
||||
)
|
||||
return Invoke-Api -Method POST -Path "/clusters/$ClusterID/nodes/$EntryNodeID/heartbeats" -Body @{
|
||||
health_status = "healthy"
|
||||
reported_version = $RequiredNodeVersion
|
||||
capabilities = @{
|
||||
native_node_agent = $true
|
||||
fabric_service_channel_runtime = $true
|
||||
fabric_service_channel_route_manager = $true
|
||||
smoke_feedback_injection = "c18z1"
|
||||
}
|
||||
service_states = @{ smoke = "c18z1_live_ingress_feedback" }
|
||||
metadata = @{
|
||||
fabric_service_channel_runtime_report = @{
|
||||
schema_version = "c18l.fabric_service_channel_runtime_report.v1"
|
||||
ingress = @{
|
||||
flow_scheduler = @{
|
||||
channel_stats = @{
|
||||
"c18z1-live-flow" = @{
|
||||
last_route_id = $GoodRouteID
|
||||
last_failed_route_id = $BadRouteID
|
||||
last_error = "c18z1 forced stale route after live packet ingress"
|
||||
consecutive_failures = 3
|
||||
stall_count = 1
|
||||
last_send_duration_ms = 250
|
||||
route_rebuild_recommended = $true
|
||||
degraded_fallback_recommended = $false
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
smoke = @{
|
||||
name = "c18z1_live_service_channel_ingress"
|
||||
run_id = $runId
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function Wait-ForConfigDecision {
|
||||
param(
|
||||
[string]$NodeID,
|
||||
[string]$BadRouteID,
|
||||
[string]$ExpectedReplacementID,
|
||||
[int]$TimeoutSeconds = 60
|
||||
)
|
||||
$deadline = (Get-Date).AddSeconds($TimeoutSeconds)
|
||||
do {
|
||||
$config = Get-SyntheticConfig -NodeID $NodeID
|
||||
$decisions = @($config.synthetic_mesh_config.route_path_decisions.decisions)
|
||||
$decision = @($decisions | Where-Object {
|
||||
$_.route_id -eq $BadRouteID -and
|
||||
$_.rebuild_status -eq "applied" -and
|
||||
$_.replacement_route_id -eq $ExpectedReplacementID
|
||||
}) | Select-Object -First 1
|
||||
if ($null -ne $decision) {
|
||||
return @{
|
||||
config = $config
|
||||
decision = $decision
|
||||
}
|
||||
}
|
||||
Start-Sleep -Seconds 2
|
||||
} while ((Get-Date) -lt $deadline)
|
||||
throw "Timed out waiting for applied rebuild decision $BadRouteID -> $ExpectedReplacementID"
|
||||
}
|
||||
|
||||
function Wait-ForAppliedRebuildTransition {
|
||||
param(
|
||||
[string]$NodeID,
|
||||
[string]$BadRouteID = "",
|
||||
[string]$ReplacementRouteID = "",
|
||||
[int]$TimeoutSeconds = 90
|
||||
)
|
||||
$deadline = (Get-Date).AddSeconds($TimeoutSeconds)
|
||||
do {
|
||||
$latest = Get-LatestRuntimeReport -NodeID $NodeID
|
||||
$transition = $null
|
||||
if ($null -ne $latest.report -and $null -ne $latest.report.ingress) {
|
||||
$prop = $latest.report.ingress.PSObject.Properties["route_manager_transition"]
|
||||
if ($null -ne $prop) {
|
||||
$transition = $prop.Value
|
||||
}
|
||||
}
|
||||
if ($null -ne $transition -and [string]$transition.status -eq "applied_rebuild") {
|
||||
return $latest
|
||||
}
|
||||
if ($BadRouteID -ne "" -and $ReplacementRouteID -ne "") {
|
||||
Send-FeedbackHeartbeat -EntryNodeID $NodeID -BadRouteID $BadRouteID -GoodRouteID $ReplacementRouteID | Out-Null
|
||||
}
|
||||
Start-Sleep -Seconds 2
|
||||
} while ((Get-Date) -lt $deadline)
|
||||
throw "Timed out waiting for node route-manager transition applied_rebuild on node $NodeID"
|
||||
}
|
||||
|
||||
$entryNode = Get-NodeByName -Name $EntryNodeName
|
||||
$exitNode = Get-NodeByName -Name $ExitNodeName
|
||||
Enable-TestMeshListener -Node $entryNode
|
||||
Enable-TestMeshListener -Node $exitNode
|
||||
Clear-OldSmokeRouteIntents -SourceNodeID $entryNode.id -DestinationNodeID $exitNode.id
|
||||
|
||||
$primaryIntent = New-RouteIntent -SourceNodeID $entryNode.id -DestinationNodeID $exitNode.id -Priority 2000000000 -Label "primary"
|
||||
$alternateIntent = New-RouteIntent -SourceNodeID $entryNode.id -DestinationNodeID $exitNode.id -Priority 1999999999 -Label "alternate"
|
||||
$primaryRouteID = $primaryIntent.route_intent.id
|
||||
$alternateRouteID = $alternateIntent.route_intent.id
|
||||
|
||||
$visibleConfig = Wait-ForRouteIntentVisible -NodeID $entryNode.id -RouteIDs @($primaryRouteID, $alternateRouteID)
|
||||
$exitVisibleConfig = Wait-ForRouteIntentVisible -NodeID $exitNode.id -RouteIDs @($primaryRouteID, $alternateRouteID)
|
||||
$readyBefore = Wait-ForRuntimeReady -NodeID $entryNode.id -MinRoutes 2
|
||||
$exitReadyBefore = Wait-ForRuntimeReady -NodeID $exitNode.id -MinRoutes 0
|
||||
$loadedConfig = Wait-ForRuntimeConfigVersion -NodeID $entryNode.id -ConfigVersion $visibleConfig.synthetic_mesh_config.config_version
|
||||
$exitLoadedConfig = Wait-ForRuntimeConfigVersion -NodeID $exitNode.id -ConfigVersion $exitVisibleConfig.synthetic_mesh_config.config_version
|
||||
$lease = New-ServiceChannelLease -EntryNodeID $entryNode.id -ExitNodeID $exitNode.id
|
||||
if ($lease.status -ne "ready") {
|
||||
throw "Lease status was '$($lease.status)', want ready"
|
||||
}
|
||||
if (@($primaryRouteID, $alternateRouteID) -notcontains [string]$lease.primary_route.route_id) {
|
||||
throw "Lease primary route was '$($lease.primary_route.route_id)', want one of smoke routes"
|
||||
}
|
||||
|
||||
$baselineSendPackets = Get-IngressSendPackets -NodeID $entryNode.id
|
||||
$firstPost = Invoke-ServiceChannelPost -Lease $lease -PortStart 41000
|
||||
if ([int]$firstPost.StatusCode -ne 202) {
|
||||
throw "First service-channel POST returned HTTP $($firstPost.StatusCode), want 202"
|
||||
}
|
||||
$firstIngress = Wait-ForIngressAnyRoute -NodeID $entryNode.id -RouteIDs @($primaryRouteID, $alternateRouteID) -MinSendPackets ($baselineSendPackets + 8)
|
||||
$firstSelectedRouteID = [string]$firstIngress.report.ingress.last_selected_route_id
|
||||
$replacementRouteID = @($primaryRouteID, $alternateRouteID) | Where-Object { $_ -ne $firstSelectedRouteID } | Select-Object -First 1
|
||||
if (-not $replacementRouteID) {
|
||||
throw "Could not determine replacement route for selected route '$firstSelectedRouteID'"
|
||||
}
|
||||
$firstExit = Wait-ForExitInbox -NodeID $exitNode.id -VPNConnectionID $resourceId
|
||||
|
||||
$feedback = Send-FeedbackHeartbeat -EntryNodeID $entryNode.id -BadRouteID $firstSelectedRouteID -GoodRouteID $replacementRouteID
|
||||
$decision = Wait-ForConfigDecision -NodeID $entryNode.id -BadRouteID $firstSelectedRouteID -ExpectedReplacementID $replacementRouteID
|
||||
$transition = Wait-ForAppliedRebuildTransition -NodeID $entryNode.id -BadRouteID $firstSelectedRouteID -ReplacementRouteID $replacementRouteID
|
||||
|
||||
$secondPost = Invoke-ServiceChannelPost -Lease $lease -PortStart 42000
|
||||
if ([int]$secondPost.StatusCode -ne 202) {
|
||||
throw "Second service-channel POST returned HTTP $($secondPost.StatusCode), want 202"
|
||||
}
|
||||
$secondIngress = Wait-ForIngressRoute -NodeID $entryNode.id -RouteID $replacementRouteID -MinSendPackets ($baselineSendPackets + 16)
|
||||
$secondExit = Wait-ForExitInbox -NodeID $exitNode.id -VPNConnectionID $resourceId
|
||||
|
||||
$expiredPrimary = Invoke-Api -Method POST -Path "/clusters/$ClusterID/mesh/route-intents/$primaryRouteID/expire" -Body @{ actor_user_id = $ActorUserID }
|
||||
$expiredAlternate = Invoke-Api -Method POST -Path "/clusters/$ClusterID/mesh/route-intents/$alternateRouteID/expire" -Body @{ actor_user_id = $ActorUserID }
|
||||
|
||||
$result = [ordered]@{
|
||||
schema_version = "c18z1.live_service_channel_ingress_smoke.v1"
|
||||
run_id = $runId
|
||||
base_url = $ApiBaseUrl
|
||||
entry_base_url = $EntryBaseUrl
|
||||
cluster_id = $ClusterID
|
||||
entry_node = @{ name = $entryNode.name; id = $entryNode.id }
|
||||
exit_node = @{ name = $exitNode.name; id = $exitNode.id }
|
||||
resource_id = $resourceId
|
||||
lease = @{
|
||||
channel_id = $lease.channel_id
|
||||
status = $lease.status
|
||||
primary_route_id = $lease.primary_route.route_id
|
||||
}
|
||||
primary_route_intent_id = $primaryRouteID
|
||||
alternate_route_intent_id = $alternateRouteID
|
||||
first_selected_route_id = $firstSelectedRouteID
|
||||
replacement_route_id = $replacementRouteID
|
||||
first_post_status = [int]$firstPost.StatusCode
|
||||
second_post_status = [int]$secondPost.StatusCode
|
||||
feedback_status = $feedback.heartbeat.health_status
|
||||
expired_primary_status = $expiredPrimary.route_intent.lifecycle_status
|
||||
expired_alternate_status = $expiredAlternate.route_intent.lifecycle_status
|
||||
passed = $true
|
||||
checks = [ordered]@{
|
||||
production_forwarding_ready = ($readyBefore.report.production_payload_forwarding -eq $true)
|
||||
exit_production_forwarding_ready = ($exitReadyBefore.report.production_payload_forwarding -eq $true)
|
||||
route_intents_visible_before_post = (@($visibleConfig.synthetic_mesh_config.routes | Where-Object { @($primaryRouteID, $alternateRouteID) -contains $_.route_id }).Count -ge 2)
|
||||
exit_route_intents_visible_before_post = (@($exitVisibleConfig.synthetic_mesh_config.routes | Where-Object { @($primaryRouteID, $alternateRouteID) -contains $_.route_id }).Count -ge 2)
|
||||
entry_runtime_loaded_visible_config = ([string]$loadedConfig.report.config_version -ge [string]$visibleConfig.synthetic_mesh_config.config_version)
|
||||
exit_runtime_loaded_visible_config = ([string]$exitLoadedConfig.report.config_version -ge [string]$exitVisibleConfig.synthetic_mesh_config.config_version)
|
||||
signed_lease_ready = ($lease.status -eq "ready")
|
||||
first_post_accepted = ([int]$firstPost.StatusCode -eq 202)
|
||||
first_post_selected_smoke_route = (@($primaryRouteID, $alternateRouteID) -contains $firstIngress.report.ingress.last_selected_route_id)
|
||||
exit_inbox_received_first_batch = ($null -ne $firstExit)
|
||||
control_plane_applied_rebuild_decision = ($decision.decision.replacement_route_id -eq $replacementRouteID)
|
||||
node_applied_rebuild_transition = ($transition.report.ingress.route_manager_transition.status -eq "applied_rebuild")
|
||||
second_post_accepted = ([int]$secondPost.StatusCode -eq 202)
|
||||
second_post_selected_replacement_route = ($secondIngress.report.ingress.last_selected_route_id -eq $replacementRouteID)
|
||||
exit_inbox_received_second_batch = ($null -ne $secondExit)
|
||||
}
|
||||
telemetry = @{
|
||||
first_ingress = $firstIngress.report.ingress
|
||||
second_ingress = $secondIngress.report.ingress
|
||||
exit_inbox = $secondExit.report.inbox
|
||||
}
|
||||
}
|
||||
|
||||
$resultFullPath = Join-Path $repoRoot $ResultPath
|
||||
$resultDir = Split-Path -Parent $resultFullPath
|
||||
if (-not (Test-Path $resultDir)) {
|
||||
New-Item -ItemType Directory -Path $resultDir | Out-Null
|
||||
}
|
||||
$result | ConvertTo-Json -Depth 100 | Set-Content -Path $resultFullPath -Encoding UTF8
|
||||
Write-Host "C18Z1 live service-channel ingress smoke passed. Result: $resultFullPath"
|
||||
$result
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,250 @@
|
||||
param(
|
||||
[string]$ApiBaseUrl = "http://192.168.200.61:18121/api/v1",
|
||||
[string]$ClusterID = "cfc0743d-d960-49fb-9de8-96e063d5e4aa",
|
||||
[string]$ActorUserID = "f67d943f-5397-4b3a-a229-695fe67ad700",
|
||||
[string]$EntryNodeName = "test-1",
|
||||
[string]$ExitNodeName = "test-2",
|
||||
[string]$ExpectedBackendImage = "rap-backend:fabric-service-channel-0.2.281-c18z109",
|
||||
[string]$ExpectedNodeAgentImage = "rap-node-agent:0.2.270-c18z95",
|
||||
[string]$DockerSSH = "test-docker",
|
||||
[string]$ResultPath = "artifacts\c18z100-rebuild-health-feedback-breakdown-smoke-result.json"
|
||||
)
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
$scriptDir = Split-Path -Parent $MyInvocation.MyCommand.Path
|
||||
$repoRoot = (Resolve-Path (Join-Path $scriptDir "..\..")).ProviderPath
|
||||
$runId = "c18z100-" + (Get-Date -Format "yyyyMMdd-HHmmss")
|
||||
|
||||
function Invoke-Api {
|
||||
param([string]$Method, [string]$Path, [object]$Body = $null)
|
||||
$params = @{ Method = $Method; Uri = "$ApiBaseUrl$Path"; TimeoutSec = 30 }
|
||||
if ($null -ne $Body) {
|
||||
$params.ContentType = "application/json"
|
||||
$params.Body = ($Body | ConvertTo-Json -Depth 80)
|
||||
}
|
||||
return Invoke-RestMethod @params
|
||||
}
|
||||
|
||||
function Get-PropertyValue {
|
||||
param([object]$Item, [string]$Name, [object]$Default = $null)
|
||||
if ($null -eq $Item) { return $Default }
|
||||
$property = $Item.PSObject.Properties[$Name]
|
||||
if ($null -eq $property) { return $Default }
|
||||
return $property.Value
|
||||
}
|
||||
|
||||
function Get-NodeByName {
|
||||
param([string]$Name)
|
||||
$nodes = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/nodes?actor_user_id=$ActorUserID").nodes
|
||||
$node = @($nodes | Where-Object { $_.name -eq $Name }) | Select-Object -First 1
|
||||
if ($null -eq $node) { throw "Node '$Name' was not found" }
|
||||
return $node
|
||||
}
|
||||
|
||||
function New-RouteIntent {
|
||||
param([string]$SourceNodeID, [string]$DestinationNodeID, [string]$Suffix, [int]$Priority)
|
||||
$expiresAt = (Get-Date).ToUniversalTime().AddMinutes(5).ToString("o")
|
||||
return (Invoke-Api -Method POST -Path "/clusters/$ClusterID/mesh/route-intents" -Body @{
|
||||
actor_user_id = $ActorUserID
|
||||
source_selector = @{ node_id = $SourceNodeID }
|
||||
destination_selector = @{ node_id = $DestinationNodeID }
|
||||
service_class = "vpn_packets"
|
||||
priority = $Priority
|
||||
policy = @{
|
||||
synthetic_enabled = $true
|
||||
route_version = "$runId-$Suffix"
|
||||
policy_version = "$runId-$Suffix"
|
||||
peer_directory_version = "$runId-$Suffix"
|
||||
hops = @($SourceNodeID, $DestinationNodeID)
|
||||
allowed_channels = @("vpn_packet", "fabric_control")
|
||||
max_ttl = 8
|
||||
max_hops = 8
|
||||
expires_at = $expiresAt
|
||||
metadata = @{ smoke = "c18z100_rebuild_health_feedback_breakdown"; run_id = $runId; suffix = $Suffix }
|
||||
}
|
||||
}).route_intent
|
||||
}
|
||||
|
||||
$entryNode = Get-NodeByName -Name $EntryNodeName
|
||||
$exitNode = Get-NodeByName -Name $ExitNodeName
|
||||
$badRoute = New-RouteIntent -SourceNodeID $entryNode.id -DestinationNodeID $exitNode.id -Suffix "bad" -Priority 2100000000
|
||||
$goodRoute = New-RouteIntent -SourceNodeID $entryNode.id -DestinationNodeID $exitNode.id -Suffix "good" -Priority 100
|
||||
|
||||
$lease = (Invoke-Api -Method POST -Path "/clusters/$ClusterID/fabric/service-channels/leases" -Body @{
|
||||
actor_user_id = $ActorUserID
|
||||
organization_id = "org-home"
|
||||
user_id = "user-m"
|
||||
resource_id = "$runId-vpn"
|
||||
service_class = "vpn_packets"
|
||||
entry_node_ids = @([string]$entryNode.id)
|
||||
exit_node_ids = @([string]$exitNode.id)
|
||||
preferred_entry_node_id = [string]$entryNode.id
|
||||
preferred_exit_node_id = [string]$exitNode.id
|
||||
allowed_channels = @("vpn_packet", "fabric_control")
|
||||
ttl_seconds = 120
|
||||
metadata = @{ smoke = "c18z100_rebuild_health_feedback_breakdown"; run_id = $runId }
|
||||
}).fabric_service_channel_lease
|
||||
|
||||
$heartbeatBody = @{
|
||||
health_status = "healthy"
|
||||
capabilities = @{ fabric_service_channel_access_telemetry = $true }
|
||||
service_states = @{}
|
||||
metadata = @{
|
||||
fabric_service_channel_access_report = @{
|
||||
schema_version = "c18z52.fabric_service_channel_access_report.v1"
|
||||
total = 1
|
||||
signed = 1
|
||||
backend_fallback = 0
|
||||
backend_fallback_blocked = 1
|
||||
fabric_route_send_failure = 1
|
||||
data_plane_contract = 1
|
||||
last_backend_relay_policy = "disabled"
|
||||
last_working_data_transport = "fabric_service_channel"
|
||||
last_steady_state_transport = "fabric_route"
|
||||
last_data_plane_violation_status = "fabric_route_send_failed_backend_fallback_blocked"
|
||||
last_data_plane_violation_reason = "synthetic c18z100 route send failure"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Invoke-Api -Method POST -Path "/clusters/$ClusterID/nodes/$($entryNode.id)/heartbeats" -Body $heartbeatBody | Out-Null
|
||||
|
||||
$feedback = $null
|
||||
$config = $null
|
||||
for ($i = 0; $i -lt 8; $i++) {
|
||||
Start-Sleep -Seconds 3
|
||||
$feedbackItems = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/fabric/service-channels/route-feedback?actor_user_id=$ActorUserID&reporter_node_id=$($entryNode.id)&route_id=$($lease.primary_route.route_id)&service_class=vpn_packets").route_feedback
|
||||
$feedback = @($feedbackItems | Select-Object -First 1)
|
||||
$config = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/nodes/$($entryNode.id)/mesh/synthetic-config").synthetic_mesh_config
|
||||
$replacement = @($config.route_path_decisions.decisions | Where-Object {
|
||||
[string](Get-PropertyValue -Item $_ -Name "route_id" -Default "") -eq [string]$lease.primary_route.route_id -and
|
||||
[string](Get-PropertyValue -Item $_ -Name "replacement_route_id" -Default "") -ne ""
|
||||
}) | Select-Object -First 1
|
||||
if ($null -ne $feedback -and $null -ne $replacement) { break }
|
||||
}
|
||||
|
||||
$firstFeedback = $feedback
|
||||
$firstFeedbackID = if ($null -ne $firstFeedback) { [string]$firstFeedback.id } else { "" }
|
||||
$firstObservedAt = if ($null -ne $firstFeedback) { [string]$firstFeedback.observed_at } else { "" }
|
||||
Start-Sleep -Seconds 2
|
||||
Invoke-Api -Method POST -Path "/clusters/$ClusterID/nodes/$($entryNode.id)/heartbeats" -Body $heartbeatBody | Out-Null
|
||||
Start-Sleep -Seconds 2
|
||||
$feedbackAfterDuplicateItems = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/fabric/service-channels/route-feedback?actor_user_id=$ActorUserID&reporter_node_id=$($entryNode.id)&route_id=$($lease.primary_route.route_id)&service_class=vpn_packets").route_feedback
|
||||
$feedbackAfterDuplicate = @($feedbackAfterDuplicateItems | Select-Object -First 1)
|
||||
$duplicateFeedbackCount = @($feedbackAfterDuplicateItems).Count
|
||||
$secondFeedbackID = if ($null -ne $feedbackAfterDuplicate) { [string]$feedbackAfterDuplicate.id } else { "" }
|
||||
$secondObservedAt = if ($null -ne $feedbackAfterDuplicate) { [string]$feedbackAfterDuplicate.observed_at } else { "" }
|
||||
|
||||
$replacement = @($config.route_path_decisions.decisions | Where-Object {
|
||||
[string](Get-PropertyValue -Item $_ -Name "route_id" -Default "") -eq [string]$lease.primary_route.route_id -and
|
||||
[string](Get-PropertyValue -Item $_ -Name "replacement_route_id" -Default "") -ne ""
|
||||
}) | Select-Object -First 1
|
||||
|
||||
$ledgerItems = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/fabric/service-channels/rebuild-attempts?actor_user_id=$ActorUserID&reporter_node_id=$($entryNode.id)&route_id=$($lease.primary_route.route_id)&limit=20&enrichment=summary").rebuild_attempts
|
||||
$ledgerAttempt = @($ledgerItems | Where-Object {
|
||||
[string](Get-PropertyValue -Item $_ -Name "rebuild_request_id" -Default "") -eq [string](Get-PropertyValue -Item $replacement -Name "rebuild_request_id" -Default "")
|
||||
}) | Select-Object -First 1
|
||||
$sourceFilteredLedger = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/fabric/service-channels/rebuild-attempts?actor_user_id=$ActorUserID&feedback_source=fabric_service_channel_access_report&limit=20&enrichment=summary").rebuild_attempts
|
||||
$channelFilteredLedger = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/fabric/service-channels/rebuild-attempts?actor_user_id=$ActorUserID&feedback_channel_id=$($lease.channel_id)&limit=20&enrichment=summary").rebuild_attempts
|
||||
$violationFilteredLedger = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/fabric/service-channels/rebuild-attempts?actor_user_id=$ActorUserID&feedback_violation_status=fabric_route_send_failed_backend_fallback_blocked&limit=20&enrichment=summary").rebuild_attempts
|
||||
$combinedFilteredLedger = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/fabric/service-channels/rebuild-attempts?actor_user_id=$ActorUserID&feedback_source=fabric_service_channel_access_report&feedback_channel_id=$($lease.channel_id)&feedback_violation_status=fabric_route_send_failed_backend_fallback_blocked&limit=20&enrichment=summary").rebuild_attempts
|
||||
$wrongChannelFilteredLedger = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/fabric/service-channels/rebuild-attempts?actor_user_id=$ActorUserID&feedback_channel_id=00000000-0000-0000-0000-000000000000&limit=20&enrichment=summary").rebuild_attempts
|
||||
$sourceFilteredAttempt = @($sourceFilteredLedger | Where-Object { [string](Get-PropertyValue -Item $_ -Name "rebuild_request_id" -Default "") -eq [string](Get-PropertyValue -Item $replacement -Name "rebuild_request_id" -Default "") }) | Select-Object -First 1
|
||||
$channelFilteredAttempt = @($channelFilteredLedger | Where-Object { [string](Get-PropertyValue -Item $_ -Name "rebuild_request_id" -Default "") -eq [string](Get-PropertyValue -Item $replacement -Name "rebuild_request_id" -Default "") }) | Select-Object -First 1
|
||||
$violationFilteredAttempt = @($violationFilteredLedger | Where-Object { [string](Get-PropertyValue -Item $_ -Name "rebuild_request_id" -Default "") -eq [string](Get-PropertyValue -Item $replacement -Name "rebuild_request_id" -Default "") }) | Select-Object -First 1
|
||||
$combinedFilteredAttempt = @($combinedFilteredLedger | Where-Object { [string](Get-PropertyValue -Item $_ -Name "rebuild_request_id" -Default "") -eq [string](Get-PropertyValue -Item $replacement -Name "rebuild_request_id" -Default "") }) | Select-Object -First 1
|
||||
$health = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/fabric/service-channels/rebuild-health?actor_user_id=$ActorUserID&limit=50").rebuild_health
|
||||
$feedbackBreakdown = @((Get-PropertyValue -Item $health -Name "feedback_breakdowns" -Default @()) | Where-Object {
|
||||
[string](Get-PropertyValue -Item $_ -Name "feedback_source" -Default "") -eq "fabric_service_channel_access_report" -and
|
||||
[string](Get-PropertyValue -Item $_ -Name "feedback_channel_id" -Default "") -eq [string]$lease.channel_id -and
|
||||
[string](Get-PropertyValue -Item $_ -Name "feedback_violation_status" -Default "") -eq "fabric_route_send_failed_backend_fallback_blocked"
|
||||
}) | Select-Object -First 1
|
||||
|
||||
$backendLine = (& ssh $DockerSSH "docker ps --format '{{.Names}} {{.Image}} {{.Status}}' | grep '^rap_test_backend '") | Out-String
|
||||
$nodeLines = (& ssh $DockerSSH "docker ps --format '{{.Names}} {{.Image}} {{.Status}}' | grep '^rap_test_node_test_'") | Out-String
|
||||
|
||||
$checks = [ordered]@{
|
||||
backend_expected_image_deployed = $backendLine.Contains($ExpectedBackendImage)
|
||||
node_agent_expected_image_deployed = $nodeLines.Contains($ExpectedNodeAgentImage)
|
||||
lease_selected_bad_route_first = ([string]$lease.primary_route.route_id -eq [string]$badRoute.id)
|
||||
route_feedback_recorded_from_access_report = ($null -ne $feedback)
|
||||
route_feedback_is_fenced = ($null -ne $feedback -and [string]$feedback.feedback_status -eq "fenced")
|
||||
route_feedback_contains_blocked_policy_reason = ($null -ne $feedback -and @($feedback.reasons | Where-Object { [string]$_ -eq "backend_fallback_blocked_by_policy" }).Count -gt 0)
|
||||
duplicate_heartbeat_kept_single_latest_feedback = ($duplicateFeedbackCount -eq 1)
|
||||
duplicate_heartbeat_kept_feedback_id = ($firstFeedbackID -ne "" -and $firstFeedbackID -eq $secondFeedbackID)
|
||||
duplicate_heartbeat_kept_observed_at = ($firstObservedAt -ne "" -and $firstObservedAt -eq $secondObservedAt)
|
||||
planner_selected_replacement = ($null -ne $replacement)
|
||||
planner_replacement_is_good_route = ($null -ne $replacement -and [string]$replacement.replacement_route_id -eq [string]$goodRoute.id)
|
||||
planner_rebuild_status_applied = ($null -ne $replacement -and [string]$replacement.rebuild_status -eq "applied")
|
||||
planner_decision_links_feedback_observation = ($null -ne $replacement -and [string](Get-PropertyValue -Item $replacement -Name "feedback_observation_id" -Default "") -eq $firstFeedbackID)
|
||||
planner_decision_links_access_report_source = ($null -ne $replacement -and [string](Get-PropertyValue -Item $replacement -Name "feedback_source" -Default "") -eq "fabric_service_channel_access_report")
|
||||
planner_decision_links_channel = ($null -ne $replacement -and [string](Get-PropertyValue -Item $replacement -Name "feedback_channel_id" -Default "") -eq [string]$lease.channel_id)
|
||||
planner_decision_links_violation = ($null -ne $replacement -and [string](Get-PropertyValue -Item $replacement -Name "feedback_violation_status" -Default "") -eq "fabric_route_send_failed_backend_fallback_blocked")
|
||||
rebuild_ledger_recorded_correlated_attempt = ($null -ne $ledgerAttempt)
|
||||
rebuild_ledger_links_feedback_observation = ($null -ne $ledgerAttempt -and [string](Get-PropertyValue -Item $ledgerAttempt -Name "feedback_observation_id" -Default "") -eq $firstFeedbackID)
|
||||
rebuild_ledger_links_access_report_source = ($null -ne $ledgerAttempt -and [string](Get-PropertyValue -Item $ledgerAttempt -Name "feedback_source" -Default "") -eq "fabric_service_channel_access_report")
|
||||
rebuild_ledger_payload_links_feedback = ($null -ne $ledgerAttempt -and [string](Get-PropertyValue -Item (Get-PropertyValue -Item $ledgerAttempt -Name "payload" -Default $null) -Name "feedback_observation_id" -Default "") -eq $firstFeedbackID)
|
||||
filter_by_feedback_source_returns_attempt = ($null -ne $sourceFilteredAttempt)
|
||||
filter_by_feedback_channel_returns_attempt = ($null -ne $channelFilteredAttempt)
|
||||
filter_by_feedback_violation_returns_attempt = ($null -ne $violationFilteredAttempt)
|
||||
combined_feedback_filters_return_attempt = ($null -ne $combinedFilteredAttempt)
|
||||
wrong_channel_filter_excludes_attempt = (@($wrongChannelFilteredLedger | Where-Object { [string](Get-PropertyValue -Item $_ -Name "rebuild_request_id" -Default "") -eq [string](Get-PropertyValue -Item $replacement -Name "rebuild_request_id" -Default "") }).Count -eq 0)
|
||||
rebuild_health_returns_feedback_breakdown = ($null -ne $feedbackBreakdown)
|
||||
feedback_breakdown_counts_attempt = ($null -ne $feedbackBreakdown -and [int](Get-PropertyValue -Item $feedbackBreakdown -Name "total_count" -Default 0) -ge 1)
|
||||
feedback_breakdown_lists_reporter = ($null -ne $feedbackBreakdown -and @((Get-PropertyValue -Item $feedbackBreakdown -Name "affected_reporter_node_ids" -Default @()) | Where-Object { [string]$_ -eq [string]$entryNode.id }).Count -gt 0)
|
||||
feedback_breakdown_lists_route = ($null -ne $feedbackBreakdown -and @((Get-PropertyValue -Item $feedbackBreakdown -Name "affected_route_ids" -Default @()) | Where-Object { [string]$_ -eq [string]$lease.primary_route.route_id }).Count -gt 0)
|
||||
}
|
||||
$failed = @($checks.GetEnumerator() | Where-Object { -not $_.Value } | ForEach-Object { $_.Key })
|
||||
|
||||
$result = [ordered]@{
|
||||
schema_version = "c18z100.rebuild_health_feedback_breakdown_smoke.v1"
|
||||
run_id = $runId
|
||||
cluster_id = $ClusterID
|
||||
channel_id = [string]$lease.channel_id
|
||||
primary_route_id = [string]$lease.primary_route.route_id
|
||||
replacement_route_id = if ($null -ne $replacement) { [string]$replacement.replacement_route_id } else { "" }
|
||||
passed = ($failed.Count -eq 0)
|
||||
checks = $checks
|
||||
failed_checks = $failed
|
||||
summary = [ordered]@{
|
||||
backend_container = $backendLine.Trim()
|
||||
node_containers = $nodeLines.Trim()
|
||||
bad_route = $badRoute
|
||||
good_route = $goodRoute
|
||||
lease = $lease
|
||||
feedback = $feedback
|
||||
feedback_after_duplicate = $feedbackAfterDuplicate
|
||||
replacement = $replacement
|
||||
ledger_attempt = $ledgerAttempt
|
||||
source_filtered_attempt = $sourceFilteredAttempt
|
||||
channel_filtered_attempt = $channelFilteredAttempt
|
||||
violation_filtered_attempt = $violationFilteredAttempt
|
||||
combined_filtered_attempt = $combinedFilteredAttempt
|
||||
wrong_channel_filtered_count = @($wrongChannelFilteredLedger).Count
|
||||
rebuild_health = $health
|
||||
feedback_breakdown = $feedbackBreakdown
|
||||
}
|
||||
}
|
||||
|
||||
$target = Join-Path $repoRoot $ResultPath
|
||||
$result | ConvertTo-Json -Depth 80 | Set-Content -Path $target -Encoding UTF8
|
||||
|
||||
try {
|
||||
foreach ($routeID in @([string]$badRoute.id, [string]$goodRoute.id)) {
|
||||
if ($routeID) {
|
||||
Invoke-Api -Method POST -Path "/clusters/$ClusterID/mesh/route-intents/$routeID/expire" -Body @{ actor_user_id = $ActorUserID } | Out-Null
|
||||
}
|
||||
}
|
||||
Invoke-Api -Method POST -Path "/clusters/$ClusterID/fabric/service-channels/leases/cleanup" -Body @{ actor_user_id = $ActorUserID; limit = 100 } | Out-Null
|
||||
} catch {
|
||||
Write-Warning "cleanup failed after c18z100 smoke: $($_.Exception.Message)"
|
||||
}
|
||||
|
||||
if (-not $result.passed) {
|
||||
throw "C18Z100 rebuild health feedback breakdown smoke failed: $($failed -join ', ')"
|
||||
}
|
||||
|
||||
Write-Host "C18Z100 rebuild health feedback breakdown smoke passed. Result: $target"
|
||||
$result
|
||||
@@ -0,0 +1,92 @@
|
||||
param(
|
||||
[string]$ApiBaseUrl = "http://192.168.200.61:18121/api/v1",
|
||||
[string]$ClusterID = "cfc0743d-d960-49fb-9de8-96e063d5e4aa",
|
||||
[string]$ActorUserID = "f67d943f-5397-4b3a-a229-695fe67ad700",
|
||||
[string]$ExpectedBackendImage = "rap-backend:fabric-service-channel-0.2.281-c18z109",
|
||||
[string]$DockerSSH = "test-docker",
|
||||
[string]$ResultPath = "artifacts\c18z102-rebuild-health-feedback-drilldown-audit-smoke-result.json"
|
||||
)
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
$scriptDir = Split-Path -Parent $MyInvocation.MyCommand.Path
|
||||
$repoRoot = (Resolve-Path (Join-Path $scriptDir "..\..")).ProviderPath
|
||||
$runId = "c18z102-" + (Get-Date -Format "yyyyMMdd-HHmmss")
|
||||
|
||||
function Invoke-Api {
|
||||
param([string]$Method, [string]$Path, [object]$Body = $null)
|
||||
$params = @{ Method = $Method; Uri = "$ApiBaseUrl$Path"; TimeoutSec = 30 }
|
||||
if ($null -ne $Body) {
|
||||
$params.ContentType = "application/json"
|
||||
$params.Body = ($Body | ConvertTo-Json -Depth 50)
|
||||
}
|
||||
return Invoke-RestMethod @params
|
||||
}
|
||||
|
||||
function Get-PropertyValue {
|
||||
param([object]$Item, [string]$Name, [object]$Default = $null)
|
||||
if ($null -eq $Item) { return $Default }
|
||||
$property = $Item.PSObject.Properties[$Name]
|
||||
if ($null -eq $property) { return $Default }
|
||||
return $property.Value
|
||||
}
|
||||
|
||||
$checks = [ordered]@{}
|
||||
|
||||
$backendImage = (& ssh $DockerSSH "docker inspect rap_test_backend --format '{{.Config.Image}}'").Trim()
|
||||
$checks.backend_expected_image_deployed = ($backendImage -eq $ExpectedBackendImage)
|
||||
|
||||
$feedbackChannelID = "$runId-channel"
|
||||
$feedbackViolationStatus = "fabric_route_send_failed_backend_fallback_blocked"
|
||||
$reason = "synthetic c18z102 rebuild-health feedback drilldown audit"
|
||||
|
||||
Invoke-Api -Method POST -Path "/clusters/$ClusterID/fabric/service-channels/rebuild-incidents/investigations" -Body @{
|
||||
actor_user_id = $ActorUserID
|
||||
feedback_source = "fabric_service_channel_access_report"
|
||||
feedback_channel_id = $feedbackChannelID
|
||||
feedback_violation_status = $feedbackViolationStatus
|
||||
drilldown_source = "rebuild_health_feedback_breakdown"
|
||||
reason = $reason
|
||||
} | Out-Null
|
||||
|
||||
$auditEvents = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/audit?actor_user_id=$ActorUserID&limit=25").audit_events
|
||||
$event = @($auditEvents | Where-Object {
|
||||
$_.event_type -eq "fabric.service_channel_rebuild_feedback_breakdown.investigation_opened" -and
|
||||
(Get-PropertyValue -Item $_.payload -Name "feedback_channel_id" -Default "") -eq $feedbackChannelID
|
||||
}) | Select-Object -First 1
|
||||
|
||||
$checks.audit_event_recorded = ($null -ne $event)
|
||||
$checks.audit_event_target_is_breakdown = ($null -ne $event -and $event.target_type -eq "fabric_service_channel_rebuild_feedback_breakdown" -and $event.target_id -eq $feedbackChannelID)
|
||||
$checks.audit_event_payload_has_feedback_filters = (
|
||||
$null -ne $event -and
|
||||
(Get-PropertyValue -Item $event.payload -Name "feedback_source" -Default "") -eq "fabric_service_channel_access_report" -and
|
||||
(Get-PropertyValue -Item $event.payload -Name "feedback_violation_status" -Default "") -eq $feedbackViolationStatus -and
|
||||
(Get-PropertyValue -Item $event.payload -Name "drilldown_source" -Default "") -eq "rebuild_health_feedback_breakdown" -and
|
||||
(Get-PropertyValue -Item $event.payload -Name "reason" -Default "") -eq $reason
|
||||
)
|
||||
|
||||
$failed = @($checks.GetEnumerator() | Where-Object { -not $_.Value } | ForEach-Object { $_.Key })
|
||||
$result = [ordered]@{
|
||||
schema_version = "c18z102.rebuild_health_feedback_drilldown_audit_smoke.v1"
|
||||
run_id = $runId
|
||||
cluster_id = $ClusterID
|
||||
feedback_channel_id = $feedbackChannelID
|
||||
passed = ($failed.Count -eq 0)
|
||||
checks = $checks
|
||||
failed_checks = $failed
|
||||
summary = [ordered]@{
|
||||
backend_image = $backendImage
|
||||
audit_event_id = if ($null -ne $event) { $event.id } else { $null }
|
||||
}
|
||||
}
|
||||
|
||||
$target = Join-Path $repoRoot $ResultPath
|
||||
$result | ConvertTo-Json -Depth 50 | Set-Content -Path $target -Encoding UTF8
|
||||
|
||||
if ($failed.Count -gt 0) {
|
||||
throw "C18Z102 rebuild-health feedback drilldown audit smoke failed: $($failed -join ', ')"
|
||||
}
|
||||
|
||||
Write-Host "C18Z102 rebuild-health feedback drilldown audit smoke passed. Result: $target"
|
||||
$result
|
||||
@@ -0,0 +1,120 @@
|
||||
param(
|
||||
[string]$ApiBaseUrl = "http://192.168.200.61:18121/api/v1",
|
||||
[string]$ClusterID = "cfc0743d-d960-49fb-9de8-96e063d5e4aa",
|
||||
[string]$ActorUserID = "f67d943f-5397-4b3a-a229-695fe67ad700",
|
||||
[string]$ExpectedBackendImage = "rap-backend:fabric-service-channel-0.2.281-c18z109",
|
||||
[string]$DockerSSH = "test-docker",
|
||||
[string]$ResultPath = "artifacts\c18z104-focused-fabric-audit-smoke-result.json"
|
||||
)
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
$scriptDir = Split-Path -Parent $MyInvocation.MyCommand.Path
|
||||
$repoRoot = (Resolve-Path (Join-Path $scriptDir "..\..")).ProviderPath
|
||||
$runId = "c18z104-" + (Get-Date -Format "yyyyMMdd-HHmmss")
|
||||
|
||||
function Invoke-Api {
|
||||
param([string]$Method, [string]$Path, [object]$Body = $null)
|
||||
$params = @{ Method = $Method; Uri = "$ApiBaseUrl$Path"; TimeoutSec = 30 }
|
||||
if ($null -ne $Body) {
|
||||
$params.ContentType = "application/json"
|
||||
$params.Body = ($Body | ConvertTo-Json -Depth 50)
|
||||
}
|
||||
return Invoke-RestMethod @params
|
||||
}
|
||||
|
||||
function Get-PropertyValue {
|
||||
param([object]$Item, [string]$Name, [object]$Default = $null)
|
||||
if ($null -eq $Item) { return $Default }
|
||||
$property = $Item.PSObject.Properties[$Name]
|
||||
if ($null -eq $property) { return $Default }
|
||||
return $property.Value
|
||||
}
|
||||
|
||||
$checks = [ordered]@{}
|
||||
|
||||
$backendImage = (& ssh $DockerSSH "docker inspect rap_test_backend --format '{{.Config.Image}}'").Trim()
|
||||
$checks.backend_expected_image_deployed = ($backendImage -eq $ExpectedBackendImage)
|
||||
|
||||
$feedbackChannelID = "$runId-channel"
|
||||
$incidentRouteID = "$runId-route"
|
||||
$feedbackViolationStatus = "fabric_route_send_failed_backend_fallback_blocked"
|
||||
|
||||
Invoke-Api -Method POST -Path "/clusters/$ClusterID/fabric/service-channels/rebuild-incidents/investigations" -Body @{
|
||||
actor_user_id = $ActorUserID
|
||||
feedback_source = "fabric_service_channel_access_report"
|
||||
feedback_channel_id = $feedbackChannelID
|
||||
feedback_violation_status = $feedbackViolationStatus
|
||||
drilldown_source = "rebuild_health_feedback_breakdown"
|
||||
reason = "synthetic c18z104 focused feedback audit"
|
||||
} | Out-Null
|
||||
|
||||
Invoke-Api -Method POST -Path "/clusters/$ClusterID/fabric/service-channels/rebuild-incidents/investigations" -Body @{
|
||||
actor_user_id = $ActorUserID
|
||||
reporter_node_id = "test-entry"
|
||||
route_id = $incidentRouteID
|
||||
service_class = "vpn_packets"
|
||||
guard_status = "bad"
|
||||
incident_id = "$runId-incident"
|
||||
reason = "synthetic c18z104 focused incident audit"
|
||||
} | Out-Null
|
||||
|
||||
$eventTypes = @(
|
||||
"fabric.service_channel_rebuild_feedback_breakdown.investigation_opened",
|
||||
"fabric.service_channel_rebuild_incident.investigation_opened"
|
||||
)
|
||||
$focusedQuery = ($eventTypes | ForEach-Object { "event_type=$([uri]::EscapeDataString($_))" }) -join "&"
|
||||
$focusedEvents = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/audit?actor_user_id=$ActorUserID&limit=20&$focusedQuery").audit_events
|
||||
$feedbackOnlyEvents = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/audit?actor_user_id=$ActorUserID&limit=20&event_type=$([uri]::EscapeDataString($eventTypes[0]))").audit_events
|
||||
$targetOnlyEvents = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/audit?actor_user_id=$ActorUserID&limit=20&target_type=fabric_service_channel_rebuild_feedback_breakdown").audit_events
|
||||
|
||||
$focusedFeedback = @($focusedEvents | Where-Object {
|
||||
$_.event_type -eq $eventTypes[0] -and
|
||||
(Get-PropertyValue -Item $_.payload -Name "feedback_channel_id" -Default "") -eq $feedbackChannelID
|
||||
}) | Select-Object -First 1
|
||||
$focusedIncident = @($focusedEvents | Where-Object {
|
||||
$_.event_type -eq $eventTypes[1] -and
|
||||
$_.target_id -eq $incidentRouteID
|
||||
}) | Select-Object -First 1
|
||||
$feedbackOnlyMatch = @($feedbackOnlyEvents | Where-Object {
|
||||
$_.event_type -eq $eventTypes[0] -and
|
||||
(Get-PropertyValue -Item $_.payload -Name "feedback_channel_id" -Default "") -eq $feedbackChannelID
|
||||
}) | Select-Object -First 1
|
||||
$feedbackOnlyWrongType = @($feedbackOnlyEvents | Where-Object { $_.event_type -ne $eventTypes[0] }) | Select-Object -First 1
|
||||
$targetOnlyMatch = @($targetOnlyEvents | Where-Object {
|
||||
$_.target_type -eq "fabric_service_channel_rebuild_feedback_breakdown" -and
|
||||
$_.target_id -eq $feedbackChannelID
|
||||
}) | Select-Object -First 1
|
||||
|
||||
$checks.focused_query_returns_both_investigation_types = ($null -ne $focusedFeedback -and $null -ne $focusedIncident)
|
||||
$checks.event_type_filter_excludes_other_types = ($null -ne $feedbackOnlyMatch -and $null -eq $feedbackOnlyWrongType)
|
||||
$checks.target_type_filter_returns_breakdown = ($null -ne $targetOnlyMatch)
|
||||
|
||||
$failed = @($checks.GetEnumerator() | Where-Object { -not $_.Value } | ForEach-Object { $_.Key })
|
||||
$result = [ordered]@{
|
||||
schema_version = "c18z104.focused_fabric_audit_smoke.v1"
|
||||
run_id = $runId
|
||||
cluster_id = $ClusterID
|
||||
feedback_channel_id = $feedbackChannelID
|
||||
incident_route_id = $incidentRouteID
|
||||
passed = ($failed.Count -eq 0)
|
||||
checks = $checks
|
||||
failed_checks = $failed
|
||||
summary = [ordered]@{
|
||||
backend_image = $backendImage
|
||||
focused_count = @($focusedEvents).Count
|
||||
feedback_only_count = @($feedbackOnlyEvents).Count
|
||||
target_only_count = @($targetOnlyEvents).Count
|
||||
}
|
||||
}
|
||||
|
||||
$target = Join-Path $repoRoot $ResultPath
|
||||
$result | ConvertTo-Json -Depth 50 | Set-Content -Path $target -Encoding UTF8
|
||||
|
||||
if ($failed.Count -gt 0) {
|
||||
throw "C18Z104 focused fabric audit smoke failed: $($failed -join ', ')"
|
||||
}
|
||||
|
||||
Write-Host "C18Z104 focused fabric audit smoke passed. Result: $target"
|
||||
$result
|
||||
@@ -0,0 +1,107 @@
|
||||
param(
|
||||
[string]$ApiBaseUrl = "http://192.168.200.61:18121/api/v1",
|
||||
[string]$ClusterID = "cfc0743d-d960-49fb-9de8-96e063d5e4aa",
|
||||
[string]$ActorUserID = "f67d943f-5397-4b3a-a229-695fe67ad700",
|
||||
[string]$ExpectedBackendImage = "rap-backend:fabric-service-channel-0.2.281-c18z109",
|
||||
[string]$DockerSSH = "test-docker",
|
||||
[string]$ResultPath = "artifacts\c18z106-audit-correlation-hints-smoke-result.json"
|
||||
)
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
$scriptDir = Split-Path -Parent $MyInvocation.MyCommand.Path
|
||||
$repoRoot = (Resolve-Path (Join-Path $scriptDir "..\..")).ProviderPath
|
||||
$runId = "c18z106-" + (Get-Date -Format "yyyyMMdd-HHmmss")
|
||||
|
||||
function Invoke-Api {
|
||||
param([string]$Method, [string]$Path, [object]$Body = $null)
|
||||
$params = @{ Method = $Method; Uri = "$ApiBaseUrl$Path"; TimeoutSec = 30 }
|
||||
if ($null -ne $Body) {
|
||||
$params.ContentType = "application/json"
|
||||
$params.Body = ($Body | ConvertTo-Json -Depth 60)
|
||||
}
|
||||
return Invoke-RestMethod @params
|
||||
}
|
||||
|
||||
function Get-PropertyValue {
|
||||
param([object]$Item, [string]$Name, [object]$Default = $null)
|
||||
if ($null -eq $Item) { return $Default }
|
||||
$property = $Item.PSObject.Properties[$Name]
|
||||
if ($null -eq $property) { return $Default }
|
||||
return $property.Value
|
||||
}
|
||||
|
||||
$checks = [ordered]@{}
|
||||
|
||||
$backendImage = (& ssh $DockerSSH "docker inspect rap_test_backend --format '{{.Config.Image}}'").Trim()
|
||||
$checks.backend_expected_image_deployed = ($backendImage -eq $ExpectedBackendImage)
|
||||
|
||||
$health = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/fabric/service-channels/rebuild-health?actor_user_id=$ActorUserID&limit=50").rebuild_health
|
||||
$breakdown = @($health.feedback_breakdowns | Where-Object {
|
||||
(Get-PropertyValue -Item $_ -Name "feedback_source" -Default "") -ne "" -and
|
||||
(Get-PropertyValue -Item $_ -Name "feedback_channel_id" -Default "") -ne "" -and
|
||||
(Get-PropertyValue -Item $_ -Name "feedback_violation_status" -Default "") -ne ""
|
||||
}) | Select-Object -First 1
|
||||
$checks.rebuild_health_has_feedback_breakdown = ($null -ne $breakdown)
|
||||
|
||||
if ($null -eq $breakdown) {
|
||||
$failed = @($checks.GetEnumerator() | Where-Object { -not $_.Value } | ForEach-Object { $_.Key })
|
||||
throw "C18Z106 audit correlation hints smoke failed before audit: $($failed -join ', ')"
|
||||
}
|
||||
|
||||
$feedbackSource = $breakdown.feedback_source
|
||||
$feedbackChannelID = $breakdown.feedback_channel_id
|
||||
$feedbackViolationStatus = $breakdown.feedback_violation_status
|
||||
|
||||
Invoke-Api -Method POST -Path "/clusters/$ClusterID/fabric/service-channels/rebuild-incidents/investigations" -Body @{
|
||||
actor_user_id = $ActorUserID
|
||||
feedback_source = $feedbackSource
|
||||
feedback_channel_id = $feedbackChannelID
|
||||
feedback_violation_status = $feedbackViolationStatus
|
||||
drilldown_source = "rebuild_health_feedback_breakdown"
|
||||
reason = "synthetic c18z106 audit correlation hints"
|
||||
} | Out-Null
|
||||
|
||||
$eventType = "fabric.service_channel_rebuild_feedback_breakdown.investigation_opened"
|
||||
$auditEvents = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/audit?actor_user_id=$ActorUserID&limit=20&event_type=$([uri]::EscapeDataString($eventType))&correlation=fabric_diagnostics").audit_events
|
||||
$event = @($auditEvents | Where-Object {
|
||||
(Get-PropertyValue -Item $_.payload -Name "reason" -Default "") -eq "synthetic c18z106 audit correlation hints" -and
|
||||
(Get-PropertyValue -Item $_.payload -Name "feedback_channel_id" -Default "") -eq $feedbackChannelID
|
||||
}) | Select-Object -First 1
|
||||
$hint = Get-PropertyValue -Item $event -Name "correlation_hints" -Default $null
|
||||
$hintBreakdown = Get-PropertyValue -Item $hint -Name "feedback_breakdown" -Default $null
|
||||
|
||||
$checks.audit_event_has_correlation_hints = ($null -ne $hint)
|
||||
$checks.correlation_status_is_breakdown_active = ((Get-PropertyValue -Item $hint -Name "current_diagnostic_status" -Default "") -eq "breakdown_active")
|
||||
$checks.correlation_hint_points_to_breakdown = (
|
||||
$null -ne $hintBreakdown -and
|
||||
(Get-PropertyValue -Item $hintBreakdown -Name "feedback_channel_id" -Default "") -eq $feedbackChannelID -and
|
||||
(Get-PropertyValue -Item $hintBreakdown -Name "feedback_violation_status" -Default "") -eq $feedbackViolationStatus
|
||||
)
|
||||
|
||||
$failed = @($checks.GetEnumerator() | Where-Object { -not $_.Value } | ForEach-Object { $_.Key })
|
||||
$result = [ordered]@{
|
||||
schema_version = "c18z106.audit_correlation_hints_smoke.v1"
|
||||
run_id = $runId
|
||||
cluster_id = $ClusterID
|
||||
feedback_channel_id = $feedbackChannelID
|
||||
passed = ($failed.Count -eq 0)
|
||||
checks = $checks
|
||||
failed_checks = $failed
|
||||
summary = [ordered]@{
|
||||
backend_image = $backendImage
|
||||
audit_event_id = if ($null -ne $event) { $event.id } else { $null }
|
||||
hint_status = Get-PropertyValue -Item $hint -Name "current_diagnostic_status" -Default ""
|
||||
}
|
||||
}
|
||||
|
||||
$target = Join-Path $repoRoot $ResultPath
|
||||
$result | ConvertTo-Json -Depth 60 | Set-Content -Path $target -Encoding UTF8
|
||||
|
||||
if ($failed.Count -gt 0) {
|
||||
throw "C18Z106 audit correlation hints smoke failed: $($failed -join ', ')"
|
||||
}
|
||||
|
||||
Write-Host "C18Z106 audit correlation hints smoke passed. Result: $target"
|
||||
$result
|
||||
@@ -0,0 +1,112 @@
|
||||
param(
|
||||
[string]$ApiBaseUrl = "http://192.168.200.61:18121/api/v1",
|
||||
[string]$ClusterID = "cfc0743d-d960-49fb-9de8-96e063d5e4aa",
|
||||
[string]$ActorUserID = "f67d943f-5397-4b3a-a229-695fe67ad700",
|
||||
[string]$ExpectedBackendImage = "rap-backend:fabric-service-channel-0.2.281-c18z109",
|
||||
[string]$DockerSSH = "test-docker",
|
||||
[string]$ResultPath = "artifacts\c18z107-audit-correlation-summary-smoke-result.json"
|
||||
)
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
$scriptDir = Split-Path -Parent $MyInvocation.MyCommand.Path
|
||||
$repoRoot = (Resolve-Path (Join-Path $scriptDir "..\..")).ProviderPath
|
||||
$runId = "c18z107-" + (Get-Date -Format "yyyyMMdd-HHmmss")
|
||||
|
||||
function Invoke-Api {
|
||||
param([string]$Method, [string]$Path, [object]$Body = $null)
|
||||
$params = @{ Method = $Method; Uri = "$ApiBaseUrl$Path"; TimeoutSec = 30 }
|
||||
if ($null -ne $Body) {
|
||||
$params.ContentType = "application/json"
|
||||
$params.Body = ($Body | ConvertTo-Json -Depth 60)
|
||||
}
|
||||
return Invoke-RestMethod @params
|
||||
}
|
||||
|
||||
function Get-PropertyValue {
|
||||
param([object]$Item, [string]$Name, [object]$Default = $null)
|
||||
if ($null -eq $Item) { return $Default }
|
||||
$property = $Item.PSObject.Properties[$Name]
|
||||
if ($null -eq $property) { return $Default }
|
||||
return $property.Value
|
||||
}
|
||||
|
||||
$checks = [ordered]@{}
|
||||
|
||||
$backendImage = (& ssh $DockerSSH "docker inspect rap_test_backend --format '{{.Config.Image}}'").Trim()
|
||||
$checks.backend_expected_image_deployed = ($backendImage -eq $ExpectedBackendImage)
|
||||
|
||||
$health = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/fabric/service-channels/rebuild-health?actor_user_id=$ActorUserID&limit=50").rebuild_health
|
||||
$breakdown = @($health.feedback_breakdowns | Where-Object {
|
||||
(Get-PropertyValue -Item $_ -Name "feedback_source" -Default "") -ne "" -and
|
||||
(Get-PropertyValue -Item $_ -Name "feedback_channel_id" -Default "") -ne "" -and
|
||||
(Get-PropertyValue -Item $_ -Name "feedback_violation_status" -Default "") -ne ""
|
||||
}) | Select-Object -First 1
|
||||
$checks.rebuild_health_has_feedback_breakdown = ($null -ne $breakdown)
|
||||
|
||||
if ($null -eq $breakdown) {
|
||||
$failed = @($checks.GetEnumerator() | Where-Object { -not $_.Value } | ForEach-Object { $_.Key })
|
||||
throw "C18Z107 audit correlation summary smoke failed before audit: $($failed -join ', ')"
|
||||
}
|
||||
|
||||
$feedbackSource = $breakdown.feedback_source
|
||||
$feedbackChannelID = $breakdown.feedback_channel_id
|
||||
$feedbackViolationStatus = $breakdown.feedback_violation_status
|
||||
$reason = "synthetic c18z107 audit correlation summary"
|
||||
|
||||
Invoke-Api -Method POST -Path "/clusters/$ClusterID/fabric/service-channels/rebuild-incidents/investigations" -Body @{
|
||||
actor_user_id = $ActorUserID
|
||||
feedback_source = $feedbackSource
|
||||
feedback_channel_id = $feedbackChannelID
|
||||
feedback_violation_status = $feedbackViolationStatus
|
||||
drilldown_source = "rebuild_health_feedback_breakdown"
|
||||
reason = $reason
|
||||
} | Out-Null
|
||||
|
||||
$eventType = "fabric.service_channel_rebuild_feedback_breakdown.investigation_opened"
|
||||
$payload = Invoke-Api -Method GET -Path "/clusters/$ClusterID/audit?actor_user_id=$ActorUserID&limit=20&event_type=$([uri]::EscapeDataString($eventType))&correlation=fabric_diagnostics"
|
||||
$events = $payload.audit_events
|
||||
$summary = $payload.audit_summary
|
||||
$event = @($events | Where-Object {
|
||||
(Get-PropertyValue -Item $_.payload -Name "reason" -Default "") -eq $reason -and
|
||||
(Get-PropertyValue -Item $_.payload -Name "feedback_channel_id" -Default "") -eq $feedbackChannelID
|
||||
}) | Select-Object -First 1
|
||||
$hint = Get-PropertyValue -Item $event -Name "correlation_hints" -Default $null
|
||||
$statusCounts = Get-PropertyValue -Item $summary -Name "counts_by_current_diagnostic_status" -Default $null
|
||||
$sourceCounts = Get-PropertyValue -Item $summary -Name "counts_by_feedback_source" -Default $null
|
||||
$violationCounts = Get-PropertyValue -Item $summary -Name "counts_by_feedback_violation_status" -Default $null
|
||||
|
||||
$checks.audit_summary_returned = ($null -ne $summary)
|
||||
$checks.summary_total_matches_events = ((Get-PropertyValue -Item $summary -Name "total_count" -Default -1) -eq @($events).Count)
|
||||
$checks.summary_counts_breakdown_active = ((Get-PropertyValue -Item $statusCounts -Name "breakdown_active" -Default 0) -ge 1)
|
||||
$checks.summary_counts_feedback_source = ((Get-PropertyValue -Item $sourceCounts -Name $feedbackSource -Default 0) -ge 1)
|
||||
$checks.summary_counts_feedback_violation = ((Get-PropertyValue -Item $violationCounts -Name $feedbackViolationStatus -Default 0) -ge 1)
|
||||
$checks.event_keeps_correlation_hint = ((Get-PropertyValue -Item $hint -Name "current_diagnostic_status" -Default "") -eq "breakdown_active")
|
||||
|
||||
$failed = @($checks.GetEnumerator() | Where-Object { -not $_.Value } | ForEach-Object { $_.Key })
|
||||
$result = [ordered]@{
|
||||
schema_version = "c18z107.audit_correlation_summary_smoke.v1"
|
||||
run_id = $runId
|
||||
cluster_id = $ClusterID
|
||||
feedback_channel_id = $feedbackChannelID
|
||||
passed = ($failed.Count -eq 0)
|
||||
checks = $checks
|
||||
failed_checks = $failed
|
||||
summary = [ordered]@{
|
||||
backend_image = $backendImage
|
||||
audit_event_id = if ($null -ne $event) { $event.id } else { $null }
|
||||
total_count = Get-PropertyValue -Item $summary -Name "total_count" -Default 0
|
||||
correlated_count = Get-PropertyValue -Item $summary -Name "correlated_count" -Default 0
|
||||
}
|
||||
}
|
||||
|
||||
$target = Join-Path $repoRoot $ResultPath
|
||||
$result | ConvertTo-Json -Depth 60 | Set-Content -Path $target -Encoding UTF8
|
||||
|
||||
if ($failed.Count -gt 0) {
|
||||
throw "C18Z107 audit correlation summary smoke failed: $($failed -join ', ')"
|
||||
}
|
||||
|
||||
Write-Host "C18Z107 audit correlation summary smoke passed. Result: $target"
|
||||
$result
|
||||
@@ -0,0 +1,114 @@
|
||||
param(
|
||||
[string]$ApiBaseUrl = "http://192.168.200.61:18121/api/v1",
|
||||
[string]$ClusterID = "cfc0743d-d960-49fb-9de8-96e063d5e4aa",
|
||||
[string]$ActorUserID = "f67d943f-5397-4b3a-a229-695fe67ad700",
|
||||
[string]$ExpectedBackendImage = "rap-backend:fabric-service-channel-0.2.281-c18z109",
|
||||
[string]$DockerSSH = "test-docker",
|
||||
[string]$ResultPath = "artifacts\c18z108-dedicated-breadcrumbs-smoke-result.json"
|
||||
)
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
$scriptDir = Split-Path -Parent $MyInvocation.MyCommand.Path
|
||||
$repoRoot = (Resolve-Path (Join-Path $scriptDir "..\..")).ProviderPath
|
||||
$runId = "c18z108-" + (Get-Date -Format "yyyyMMdd-HHmmss")
|
||||
|
||||
function Invoke-Api {
|
||||
param([string]$Method, [string]$Path, [object]$Body = $null)
|
||||
$params = @{ Method = $Method; Uri = "$ApiBaseUrl$Path"; TimeoutSec = 30 }
|
||||
if ($null -ne $Body) {
|
||||
$params.ContentType = "application/json"
|
||||
$params.Body = ($Body | ConvertTo-Json -Depth 60)
|
||||
}
|
||||
return Invoke-RestMethod @params
|
||||
}
|
||||
|
||||
function Get-PropertyValue {
|
||||
param([object]$Item, [string]$Name, [object]$Default = $null)
|
||||
if ($null -eq $Item) { return $Default }
|
||||
$property = $Item.PSObject.Properties[$Name]
|
||||
if ($null -eq $property) { return $Default }
|
||||
return $property.Value
|
||||
}
|
||||
|
||||
$checks = [ordered]@{}
|
||||
|
||||
$backendImage = (& ssh $DockerSSH "docker inspect rap_test_backend --format '{{.Config.Image}}'").Trim()
|
||||
$checks.backend_expected_image_deployed = ($backendImage -eq $ExpectedBackendImage)
|
||||
|
||||
$health = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/fabric/service-channels/rebuild-health?actor_user_id=$ActorUserID&limit=50").rebuild_health
|
||||
$breakdown = @($health.feedback_breakdowns | Where-Object {
|
||||
(Get-PropertyValue -Item $_ -Name "feedback_source" -Default "") -ne "" -and
|
||||
(Get-PropertyValue -Item $_ -Name "feedback_channel_id" -Default "") -ne "" -and
|
||||
(Get-PropertyValue -Item $_ -Name "feedback_violation_status" -Default "") -ne ""
|
||||
}) | Select-Object -First 1
|
||||
$checks.rebuild_health_has_feedback_breakdown = ($null -ne $breakdown)
|
||||
|
||||
if ($null -eq $breakdown) {
|
||||
$failed = @($checks.GetEnumerator() | Where-Object { -not $_.Value } | ForEach-Object { $_.Key })
|
||||
throw "C18Z108 dedicated breadcrumbs smoke failed before breadcrumb audit: $($failed -join ', ')"
|
||||
}
|
||||
|
||||
$feedbackSource = $breakdown.feedback_source
|
||||
$feedbackChannelID = $breakdown.feedback_channel_id
|
||||
$feedbackViolationStatus = $breakdown.feedback_violation_status
|
||||
$reason = "synthetic c18z108 dedicated breadcrumbs"
|
||||
|
||||
Invoke-Api -Method POST -Path "/clusters/$ClusterID/fabric/service-channels/rebuild-incidents/investigations" -Body @{
|
||||
actor_user_id = $ActorUserID
|
||||
feedback_source = $feedbackSource
|
||||
feedback_channel_id = $feedbackChannelID
|
||||
feedback_violation_status = $feedbackViolationStatus
|
||||
drilldown_source = "rebuild_health_feedback_breakdown"
|
||||
reason = $reason
|
||||
} | Out-Null
|
||||
|
||||
$payload = Invoke-Api -Method GET -Path "/clusters/$ClusterID/fabric/service-channels/rebuild-investigations/breadcrumbs?actor_user_id=$ActorUserID&limit=20"
|
||||
$breadcrumbs = $payload.rebuild_investigation_breadcrumbs
|
||||
$events = Get-PropertyValue -Item $breadcrumbs -Name "events" -Default @()
|
||||
$summary = Get-PropertyValue -Item $breadcrumbs -Name "summary" -Default $null
|
||||
$event = @($events | Where-Object {
|
||||
(Get-PropertyValue -Item $_.payload -Name "reason" -Default "") -eq $reason -and
|
||||
(Get-PropertyValue -Item $_.payload -Name "feedback_channel_id" -Default "") -eq $feedbackChannelID
|
||||
}) | Select-Object -First 1
|
||||
$hint = Get-PropertyValue -Item $event -Name "correlation_hints" -Default $null
|
||||
$statusCounts = Get-PropertyValue -Item $summary -Name "counts_by_current_diagnostic_status" -Default $null
|
||||
$sourceCounts = Get-PropertyValue -Item $summary -Name "counts_by_feedback_source" -Default $null
|
||||
$violationCounts = Get-PropertyValue -Item $summary -Name "counts_by_feedback_violation_status" -Default $null
|
||||
|
||||
$checks.dedicated_breadcrumbs_returned = ($null -ne $breadcrumbs)
|
||||
$checks.dedicated_event_found = ($null -ne $event)
|
||||
$checks.summary_returned = ($null -ne $summary)
|
||||
$checks.summary_total_matches_events = ((Get-PropertyValue -Item $summary -Name "total_count" -Default -1) -eq @($events).Count)
|
||||
$checks.summary_counts_breakdown_active = ((Get-PropertyValue -Item $statusCounts -Name "breakdown_active" -Default 0) -ge 1)
|
||||
$checks.summary_counts_feedback_source = ((Get-PropertyValue -Item $sourceCounts -Name $feedbackSource -Default 0) -ge 1)
|
||||
$checks.summary_counts_feedback_violation = ((Get-PropertyValue -Item $violationCounts -Name $feedbackViolationStatus -Default 0) -ge 1)
|
||||
$checks.event_keeps_correlation_hint = ((Get-PropertyValue -Item $hint -Name "current_diagnostic_status" -Default "") -eq "breakdown_active")
|
||||
|
||||
$failed = @($checks.GetEnumerator() | Where-Object { -not $_.Value } | ForEach-Object { $_.Key })
|
||||
$result = [ordered]@{
|
||||
schema_version = "c18z108.dedicated_breadcrumbs_smoke.v1"
|
||||
run_id = $runId
|
||||
cluster_id = $ClusterID
|
||||
feedback_channel_id = $feedbackChannelID
|
||||
passed = ($failed.Count -eq 0)
|
||||
checks = $checks
|
||||
failed_checks = $failed
|
||||
summary = [ordered]@{
|
||||
backend_image = $backendImage
|
||||
audit_event_id = if ($null -ne $event) { $event.id } else { $null }
|
||||
total_count = Get-PropertyValue -Item $summary -Name "total_count" -Default 0
|
||||
correlated_count = Get-PropertyValue -Item $summary -Name "correlated_count" -Default 0
|
||||
}
|
||||
}
|
||||
|
||||
$target = Join-Path $repoRoot $ResultPath
|
||||
$result | ConvertTo-Json -Depth 60 | Set-Content -Path $target -Encoding UTF8
|
||||
|
||||
if ($failed.Count -gt 0) {
|
||||
throw "C18Z108 dedicated breadcrumbs smoke failed: $($failed -join ', ')"
|
||||
}
|
||||
|
||||
Write-Host "C18Z108 dedicated breadcrumbs smoke passed. Result: $target"
|
||||
$result
|
||||
@@ -0,0 +1,124 @@
|
||||
param(
|
||||
[string]$ApiBaseUrl = "http://192.168.200.61:18121/api/v1",
|
||||
[string]$ClusterID = "cfc0743d-d960-49fb-9de8-96e063d5e4aa",
|
||||
[string]$ActorUserID = "f67d943f-5397-4b3a-a229-695fe67ad700",
|
||||
[string]$ExpectedBackendImage = "rap-backend:fabric-service-channel-0.2.281-c18z109",
|
||||
[string]$DockerSSH = "test-docker",
|
||||
[string]$ResultPath = "artifacts\c18z109-breadcrumb-freshness-window-smoke-result.json"
|
||||
)
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
$scriptDir = Split-Path -Parent $MyInvocation.MyCommand.Path
|
||||
$repoRoot = (Resolve-Path (Join-Path $scriptDir "..\..")).ProviderPath
|
||||
$runId = "c18z109-" + (Get-Date -Format "yyyyMMdd-HHmmss")
|
||||
|
||||
function Invoke-Api {
|
||||
param([string]$Method, [string]$Path, [object]$Body = $null)
|
||||
$params = @{ Method = $Method; Uri = "$ApiBaseUrl$Path"; TimeoutSec = 30 }
|
||||
if ($null -ne $Body) {
|
||||
$params.ContentType = "application/json"
|
||||
$params.Body = ($Body | ConvertTo-Json -Depth 60)
|
||||
}
|
||||
return Invoke-RestMethod @params
|
||||
}
|
||||
|
||||
function Get-PropertyValue {
|
||||
param([object]$Item, [string]$Name, [object]$Default = $null)
|
||||
if ($null -eq $Item) { return $Default }
|
||||
$property = $Item.PSObject.Properties[$Name]
|
||||
if ($null -eq $property) { return $Default }
|
||||
return $property.Value
|
||||
}
|
||||
|
||||
function Find-EventByReason {
|
||||
param([object[]]$Events, [string]$Reason)
|
||||
return @($Events | Where-Object { (Get-PropertyValue -Item $_.payload -Name "reason" -Default "") -eq $Reason }) | Select-Object -First 1
|
||||
}
|
||||
|
||||
$checks = [ordered]@{}
|
||||
|
||||
$backendImage = (& ssh $DockerSSH "docker inspect rap_test_backend --format '{{.Config.Image}}'").Trim()
|
||||
$checks.backend_expected_image_deployed = ($backendImage -eq $ExpectedBackendImage)
|
||||
|
||||
$health = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/fabric/service-channels/rebuild-health?actor_user_id=$ActorUserID&limit=50").rebuild_health
|
||||
$breakdown = @($health.feedback_breakdowns | Where-Object {
|
||||
(Get-PropertyValue -Item $_ -Name "feedback_source" -Default "") -ne "" -and
|
||||
(Get-PropertyValue -Item $_ -Name "feedback_channel_id" -Default "") -ne "" -and
|
||||
(Get-PropertyValue -Item $_ -Name "feedback_violation_status" -Default "") -ne ""
|
||||
}) | Select-Object -First 1
|
||||
$checks.rebuild_health_has_feedback_breakdown = ($null -ne $breakdown)
|
||||
|
||||
if ($null -eq $breakdown) {
|
||||
$failed = @($checks.GetEnumerator() | Where-Object { -not $_.Value } | ForEach-Object { $_.Key })
|
||||
throw "C18Z109 breadcrumb freshness smoke failed before breadcrumb audit: $($failed -join ', ')"
|
||||
}
|
||||
|
||||
$reason = "synthetic c18z109 breadcrumb freshness $runId"
|
||||
Invoke-Api -Method POST -Path "/clusters/$ClusterID/fabric/service-channels/rebuild-incidents/investigations" -Body @{
|
||||
actor_user_id = $ActorUserID
|
||||
feedback_source = $breakdown.feedback_source
|
||||
feedback_channel_id = $breakdown.feedback_channel_id
|
||||
feedback_violation_status = $breakdown.feedback_violation_status
|
||||
drilldown_source = "rebuild_health_feedback_breakdown"
|
||||
reason = $reason
|
||||
} | Out-Null
|
||||
|
||||
$currentPayload = Invoke-Api -Method GET -Path "/clusters/$ClusterID/fabric/service-channels/rebuild-investigations/breadcrumbs?actor_user_id=$ActorUserID&limit=20¤t_window_seconds=3600&history_window_seconds=86400"
|
||||
$currentBreadcrumbs = $currentPayload.rebuild_investigation_breadcrumbs
|
||||
$currentEvent = Find-EventByReason -Events @($currentBreadcrumbs.events) -Reason $reason
|
||||
$currentHint = Get-PropertyValue -Item $currentEvent -Name "correlation_hints" -Default $null
|
||||
$currentCounts = Get-PropertyValue -Item $currentBreadcrumbs.summary -Name "counts_by_breadcrumb_status" -Default $null
|
||||
|
||||
Start-Sleep -Seconds 2
|
||||
|
||||
$stalePayload = Invoke-Api -Method GET -Path "/clusters/$ClusterID/fabric/service-channels/rebuild-investigations/breadcrumbs?actor_user_id=$ActorUserID&limit=20¤t_window_seconds=1&history_window_seconds=86400"
|
||||
$staleBreadcrumbs = $stalePayload.rebuild_investigation_breadcrumbs
|
||||
$staleEvent = Find-EventByReason -Events @($staleBreadcrumbs.events) -Reason $reason
|
||||
$staleHint = Get-PropertyValue -Item $staleEvent -Name "correlation_hints" -Default $null
|
||||
$staleCounts = Get-PropertyValue -Item $staleBreadcrumbs.summary -Name "counts_by_breadcrumb_status" -Default $null
|
||||
|
||||
$expiredPayload = Invoke-Api -Method GET -Path "/clusters/$ClusterID/fabric/service-channels/rebuild-investigations/breadcrumbs?actor_user_id=$ActorUserID&limit=20¤t_window_seconds=1&history_window_seconds=1"
|
||||
$expiredBreadcrumbs = $expiredPayload.rebuild_investigation_breadcrumbs
|
||||
$expiredEvent = Find-EventByReason -Events @($expiredBreadcrumbs.events) -Reason $reason
|
||||
$expiredHint = Get-PropertyValue -Item $expiredEvent -Name "correlation_hints" -Default $null
|
||||
$expiredCounts = Get-PropertyValue -Item $expiredBreadcrumbs.summary -Name "counts_by_breadcrumb_status" -Default $null
|
||||
|
||||
$checks.current_event_marked_current = ((Get-PropertyValue -Item $currentHint -Name "breadcrumb_status" -Default "") -eq "current")
|
||||
$checks.current_summary_counts_current = ((Get-PropertyValue -Item $currentCounts -Name "current" -Default 0) -ge 1)
|
||||
$checks.stale_event_marked_stale = ((Get-PropertyValue -Item $staleHint -Name "breadcrumb_status" -Default "") -eq "stale")
|
||||
$checks.stale_summary_counts_stale = ((Get-PropertyValue -Item $staleCounts -Name "stale" -Default 0) -ge 1)
|
||||
$checks.expired_event_marked_expired = ((Get-PropertyValue -Item $expiredHint -Name "breadcrumb_status" -Default "") -eq "expired")
|
||||
$checks.expired_summary_counts_expired = ((Get-PropertyValue -Item $expiredCounts -Name "expired" -Default 0) -ge 1)
|
||||
$checks.window_values_returned = (
|
||||
(Get-PropertyValue -Item $currentBreadcrumbs -Name "current_window_seconds" -Default 0) -eq 3600 -and
|
||||
(Get-PropertyValue -Item $currentBreadcrumbs -Name "history_window_seconds" -Default 0) -eq 86400
|
||||
)
|
||||
|
||||
$failed = @($checks.GetEnumerator() | Where-Object { -not $_.Value } | ForEach-Object { $_.Key })
|
||||
$result = [ordered]@{
|
||||
schema_version = "c18z109.breadcrumb_freshness_window_smoke.v1"
|
||||
run_id = $runId
|
||||
cluster_id = $ClusterID
|
||||
passed = ($failed.Count -eq 0)
|
||||
checks = $checks
|
||||
failed_checks = $failed
|
||||
summary = [ordered]@{
|
||||
backend_image = $backendImage
|
||||
current_event_id = if ($null -ne $currentEvent) { $currentEvent.id } else { $null }
|
||||
current_status = Get-PropertyValue -Item $currentHint -Name "breadcrumb_status" -Default ""
|
||||
stale_status = Get-PropertyValue -Item $staleHint -Name "breadcrumb_status" -Default ""
|
||||
expired_status = Get-PropertyValue -Item $expiredHint -Name "breadcrumb_status" -Default ""
|
||||
}
|
||||
}
|
||||
|
||||
$target = Join-Path $repoRoot $ResultPath
|
||||
$result | ConvertTo-Json -Depth 60 | Set-Content -Path $target -Encoding UTF8
|
||||
|
||||
if ($failed.Count -gt 0) {
|
||||
throw "C18Z109 breadcrumb freshness smoke failed: $($failed -join ', ')"
|
||||
}
|
||||
|
||||
Write-Host "C18Z109 breadcrumb freshness smoke passed. Result: $target"
|
||||
$result
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,304 @@
|
||||
param(
|
||||
[string]$ApiBaseUrl = "http://192.168.200.61:18121/api/v1",
|
||||
[string]$ClusterID = "cfc0743d-d960-49fb-9de8-96e063d5e4aa",
|
||||
[string]$ActorUserID = "f67d943f-5397-4b3a-a229-695fe67ad700",
|
||||
[string]$EntryNodeName = "test-1",
|
||||
[string]$RelayNodeName = "test-3",
|
||||
[string]$ExitNodeName = "test-2",
|
||||
[string]$ResultPath = "artifacts\c18z12-service-channel-route-quality-smoke-result.json"
|
||||
)
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
$repoRoot = (Resolve-Path (Join-Path $PSScriptRoot "..\..")).ProviderPath
|
||||
$runId = "c18z12-" + (Get-Date -Format "yyyyMMdd-HHmmss")
|
||||
$resourceId = "vpn-$runId"
|
||||
|
||||
function Invoke-Api {
|
||||
param(
|
||||
[string]$Method,
|
||||
[string]$Path,
|
||||
[object]$Body = $null
|
||||
)
|
||||
$uri = "$ApiBaseUrl$Path"
|
||||
try {
|
||||
if ($null -eq $Body) {
|
||||
return Invoke-RestMethod -Method $Method -Uri $uri -TimeoutSec 30
|
||||
}
|
||||
return Invoke-RestMethod -Method $Method -Uri $uri -ContentType "application/json" -Body ($Body | ConvertTo-Json -Depth 80) -TimeoutSec 30
|
||||
}
|
||||
catch {
|
||||
$statusCode = $null
|
||||
if ($_.Exception.Response) {
|
||||
$statusCode = [int]$_.Exception.Response.StatusCode
|
||||
}
|
||||
$details = $_.ErrorDetails.Message
|
||||
if (-not $details) {
|
||||
$details = $_.Exception.Message
|
||||
}
|
||||
throw "$Method $Path failed with HTTP $statusCode`: $details"
|
||||
}
|
||||
}
|
||||
|
||||
function Get-NodeByName {
|
||||
param([string]$Name)
|
||||
$nodes = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/nodes?actor_user_id=$ActorUserID").nodes
|
||||
$node = @($nodes | Where-Object { $_.name -eq $Name }) | Select-Object -First 1
|
||||
if ($null -eq $node) {
|
||||
throw "Node '$Name' was not found in cluster $ClusterID"
|
||||
}
|
||||
return $node
|
||||
}
|
||||
|
||||
function Get-SmokeRouteLabel {
|
||||
param([object]$RouteIntent)
|
||||
if ($null -eq $RouteIntent -or $null -eq $RouteIntent.PSObject.Properties["policy"]) {
|
||||
return ""
|
||||
}
|
||||
$policy = $RouteIntent.policy
|
||||
if ($null -eq $policy -or $null -eq $policy.PSObject.Properties["metadata"]) {
|
||||
return ""
|
||||
}
|
||||
$metadata = $policy.metadata
|
||||
if ($null -eq $metadata) {
|
||||
return ""
|
||||
}
|
||||
$smoke = $metadata.PSObject.Properties["smoke"]
|
||||
if ($null -eq $smoke) {
|
||||
return ""
|
||||
}
|
||||
return [string]$smoke.Value
|
||||
}
|
||||
|
||||
function Clear-SmokeRouteIntents {
|
||||
$items = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/mesh/route-intents?actor_user_id=$ActorUserID").route_intents
|
||||
foreach ($item in @($items)) {
|
||||
if ([string]$item.lifecycle_status -ne "active") {
|
||||
continue
|
||||
}
|
||||
if ([string]$item.service_class -ne "vpn_packets") {
|
||||
continue
|
||||
}
|
||||
if ((Get-SmokeRouteLabel -RouteIntent $item) -ne "c18z12_service_channel_route_quality") {
|
||||
continue
|
||||
}
|
||||
Invoke-Api -Method POST -Path "/clusters/$ClusterID/mesh/route-intents/$($item.id)/expire" -Body @{ actor_user_id = $ActorUserID } | Out-Null
|
||||
}
|
||||
}
|
||||
|
||||
function New-RouteIntent {
|
||||
param(
|
||||
[string]$SourceNodeID,
|
||||
[string]$DestinationNodeID,
|
||||
[string[]]$Hops,
|
||||
[int]$Priority,
|
||||
[string]$Label
|
||||
)
|
||||
$expiresAt = (Get-Date).ToUniversalTime().AddMinutes(10).ToString("o")
|
||||
return Invoke-Api -Method POST -Path "/clusters/$ClusterID/mesh/route-intents" -Body @{
|
||||
actor_user_id = $ActorUserID
|
||||
source_selector = @{ node_id = $SourceNodeID }
|
||||
destination_selector = @{ node_id = $DestinationNodeID }
|
||||
service_class = "vpn_packets"
|
||||
priority = $Priority
|
||||
policy = @{
|
||||
synthetic_enabled = $true
|
||||
route_version = "$runId-$Label"
|
||||
policy_version = "$runId-$Label"
|
||||
peer_directory_version = "$runId-$Label"
|
||||
hops = @($Hops)
|
||||
allowed_channels = @("vpn_packet", "fabric_control")
|
||||
max_ttl = 8
|
||||
max_hops = 8
|
||||
expires_at = $expiresAt
|
||||
metadata = @{
|
||||
smoke = "c18z12_service_channel_route_quality"
|
||||
run_id = $runId
|
||||
label = $Label
|
||||
route_quality_smoke = $true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function New-ServiceChannelLease {
|
||||
param(
|
||||
[string]$EntryNodeID,
|
||||
[string]$ExitNodeID
|
||||
)
|
||||
return (Invoke-Api -Method POST -Path "/clusters/$ClusterID/fabric/service-channels/leases" -Body @{
|
||||
actor_user_id = $ActorUserID
|
||||
organization_id = "org-c18z12-smoke"
|
||||
user_id = $ActorUserID
|
||||
resource_id = $resourceId
|
||||
service_class = "vpn_packets"
|
||||
entry_node_ids = @($EntryNodeID)
|
||||
exit_node_ids = @($ExitNodeID)
|
||||
preferred_entry_node_id = $EntryNodeID
|
||||
preferred_exit_node_id = $ExitNodeID
|
||||
allowed_channels = @("vpn_packet", "bulk", "control")
|
||||
ttl_seconds = 300
|
||||
metadata = @{
|
||||
smoke = "c18z12_service_channel_route_quality"
|
||||
run_id = $runId
|
||||
}
|
||||
}).fabric_service_channel_lease
|
||||
}
|
||||
|
||||
function Send-QualityHeartbeat {
|
||||
param(
|
||||
[string]$EntryNodeID,
|
||||
[string]$SlowRouteID,
|
||||
[string]$FastRouteID
|
||||
)
|
||||
$observedAt = (Get-Date).ToUniversalTime().ToString("o")
|
||||
return Invoke-Api -Method POST -Path "/clusters/$ClusterID/nodes/$EntryNodeID/heartbeats" -Body @{
|
||||
health_status = "healthy"
|
||||
reported_version = "0.2.185"
|
||||
capabilities = @{
|
||||
fabric_service_channel_route_manager = $true
|
||||
fabric_service_channel_route_quality_feedback = $true
|
||||
}
|
||||
service_states = @{ smoke = "c18z12_route_quality_feedback" }
|
||||
metadata = @{
|
||||
fabric_service_channel_runtime_report = @{
|
||||
schema_version = "c18l.fabric_service_channel_runtime_report.v1"
|
||||
cluster_id = $ClusterID
|
||||
local_node_id = $EntryNodeID
|
||||
observed_at = $observedAt
|
||||
ingress = @{
|
||||
flow_scheduler = @{
|
||||
schema_version = "rap.fabric_flow_scheduler.v1"
|
||||
service_neutral = $true
|
||||
service_mode = "application_protocol_agnostic"
|
||||
channel_stats = @{
|
||||
"quality-fast" = @{
|
||||
last_route_id = $FastRouteID
|
||||
last_next_hop = "fast"
|
||||
last_send_duration_ms = 8
|
||||
consecutive_failures = 0
|
||||
stall_count = 0
|
||||
route_rebuild_recommended = $false
|
||||
degraded_fallback_recommended = $false
|
||||
}
|
||||
"quality-slow" = @{
|
||||
last_route_id = $SlowRouteID
|
||||
last_next_hop = "slow"
|
||||
last_send_duration_ms = 900
|
||||
consecutive_failures = 0
|
||||
stall_count = 0
|
||||
route_rebuild_recommended = $false
|
||||
degraded_fallback_recommended = $false
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
$entryNode = Get-NodeByName -Name $EntryNodeName
|
||||
$relayNode = Get-NodeByName -Name $RelayNodeName
|
||||
$exitNode = Get-NodeByName -Name $ExitNodeName
|
||||
$slowRouteID = ""
|
||||
$fastRouteID = ""
|
||||
$result = $null
|
||||
|
||||
try {
|
||||
Clear-SmokeRouteIntents
|
||||
$slowIntent = New-RouteIntent `
|
||||
-SourceNodeID $entryNode.id `
|
||||
-DestinationNodeID $exitNode.id `
|
||||
-Hops @($entryNode.id, $relayNode.id, $exitNode.id) `
|
||||
-Priority 2000000000 `
|
||||
-Label "slow-high-priority"
|
||||
$fastIntent = New-RouteIntent `
|
||||
-SourceNodeID $entryNode.id `
|
||||
-DestinationNodeID $exitNode.id `
|
||||
-Hops @($entryNode.id, $exitNode.id) `
|
||||
-Priority 1999999950 `
|
||||
-Label "fast-lower-priority"
|
||||
$slowRouteID = $slowIntent.route_intent.id
|
||||
$fastRouteID = $fastIntent.route_intent.id
|
||||
|
||||
$initialLease = New-ServiceChannelLease -EntryNodeID $entryNode.id -ExitNodeID $exitNode.id
|
||||
if ($initialLease.status -ne "ready" -or [string]$initialLease.primary_route.route_id -ne $slowRouteID) {
|
||||
throw "Initial lease should select higher-priority slow route '$slowRouteID': status=$($initialLease.status) route=$($initialLease.primary_route.route_id)"
|
||||
}
|
||||
|
||||
$qualityHeartbeat = Send-QualityHeartbeat -EntryNodeID $entryNode.id -SlowRouteID $slowRouteID -FastRouteID $fastRouteID
|
||||
Start-Sleep -Seconds 2
|
||||
$qualityLease = New-ServiceChannelLease -EntryNodeID $entryNode.id -ExitNodeID $exitNode.id
|
||||
if ($qualityLease.status -ne "ready" -or [string]$qualityLease.primary_route.route_id -ne $fastRouteID) {
|
||||
throw "Quality lease should select fast route '$fastRouteID': status=$($qualityLease.status) route=$($qualityLease.primary_route.route_id) score=$($qualityLease.primary_route.path_score)"
|
||||
}
|
||||
|
||||
$expiredSlow = Invoke-Api -Method POST -Path "/clusters/$ClusterID/mesh/route-intents/$slowRouteID/expire" -Body @{ actor_user_id = $ActorUserID }
|
||||
$expiredFast = Invoke-Api -Method POST -Path "/clusters/$ClusterID/mesh/route-intents/$fastRouteID/expire" -Body @{ actor_user_id = $ActorUserID }
|
||||
|
||||
$result = [ordered]@{
|
||||
schema_version = "c18z12.service_channel_route_quality_smoke.v1"
|
||||
run_id = $runId
|
||||
base_url = $ApiBaseUrl
|
||||
cluster_id = $ClusterID
|
||||
entry_node = @{ name = $entryNode.name; id = $entryNode.id }
|
||||
relay_node = @{ name = $relayNode.name; id = $relayNode.id }
|
||||
exit_node = @{ name = $exitNode.name; id = $exitNode.id }
|
||||
resource_id = $resourceId
|
||||
route_intents = @{
|
||||
slow_route_id = $slowRouteID
|
||||
fast_route_id = $fastRouteID
|
||||
slow_hops = @($entryNode.id, $relayNode.id, $exitNode.id)
|
||||
fast_hops = @($entryNode.id, $exitNode.id)
|
||||
expired_slow_status = $expiredSlow.route_intent.lifecycle_status
|
||||
expired_fast_status = $expiredFast.route_intent.lifecycle_status
|
||||
}
|
||||
initial_lease = @{
|
||||
status = $initialLease.status
|
||||
primary_route_id = $initialLease.primary_route.route_id
|
||||
primary_path_score = $initialLease.primary_route.path_score
|
||||
score_reasons = $initialLease.primary_route.score_reasons
|
||||
}
|
||||
quality_lease = @{
|
||||
status = $qualityLease.status
|
||||
primary_route_id = $qualityLease.primary_route.route_id
|
||||
primary_path_score = $qualityLease.primary_route.path_score
|
||||
score_reasons = $qualityLease.primary_route.score_reasons
|
||||
}
|
||||
feedback = @{
|
||||
heartbeat_status = $qualityHeartbeat.heartbeat.health_status
|
||||
fast_last_send_duration_ms = 8
|
||||
slow_last_send_duration_ms = 900
|
||||
}
|
||||
passed = $true
|
||||
checks = [ordered]@{
|
||||
initial_prefers_high_priority_slow_route = ([string]$initialLease.primary_route.route_id -eq $slowRouteID)
|
||||
quality_prefers_fast_route = ([string]$qualityLease.primary_route.route_id -eq $fastRouteID)
|
||||
fast_route_has_quality_reason = (@($qualityLease.primary_route.score_reasons | Where-Object { $_ -eq "service_channel_quality_latency_le_10ms" }).Count -ge 1)
|
||||
route_intents_expired = ($expiredSlow.route_intent.lifecycle_status -eq "expired" -and $expiredFast.route_intent.lifecycle_status -eq "expired")
|
||||
}
|
||||
}
|
||||
$failedChecks = @($result.checks.GetEnumerator() | Where-Object { $_.Value -ne $true })
|
||||
if ($failedChecks.Count -gt 0) {
|
||||
throw "C18Z12 failed checks: $($failedChecks.Name -join ', ')"
|
||||
}
|
||||
}
|
||||
finally {
|
||||
if ($slowRouteID) {
|
||||
try { Invoke-Api -Method POST -Path "/clusters/$ClusterID/mesh/route-intents/$slowRouteID/expire" -Body @{ actor_user_id = $ActorUserID } | Out-Null } catch {}
|
||||
}
|
||||
if ($fastRouteID) {
|
||||
try { Invoke-Api -Method POST -Path "/clusters/$ClusterID/mesh/route-intents/$fastRouteID/expire" -Body @{ actor_user_id = $ActorUserID } | Out-Null } catch {}
|
||||
}
|
||||
}
|
||||
|
||||
$resultFullPath = Join-Path $repoRoot $ResultPath
|
||||
$resultDir = Split-Path -Parent $resultFullPath
|
||||
if (-not (Test-Path $resultDir)) {
|
||||
New-Item -ItemType Directory -Path $resultDir | Out-Null
|
||||
}
|
||||
$result | ConvertTo-Json -Depth 100 | Set-Content -Path $resultFullPath -Encoding UTF8
|
||||
Write-Host "C18Z12 service-channel route quality smoke passed. Result: $resultFullPath"
|
||||
$result
|
||||
@@ -0,0 +1,713 @@
|
||||
param(
|
||||
[string]$ApiBaseUrl = "http://192.168.200.61:18121/api/v1",
|
||||
[string]$ClusterID = "cfc0743d-d960-49fb-9de8-96e063d5e4aa",
|
||||
[string]$ActorUserID = "f67d943f-5397-4b3a-a229-695fe67ad700",
|
||||
[string]$EntryNodeName = "test-1",
|
||||
[string]$RelayNodeName = "test-3",
|
||||
[string]$ExitNodeName = "test-2",
|
||||
[string]$EntryBaseUrl = "http://192.168.200.61:19131",
|
||||
[string]$DockerSSH = "test-docker",
|
||||
[int]$BatchCount = 24,
|
||||
[int]$PacketsPerBatch = 8,
|
||||
[int]$BatchDelayMilliseconds = 25,
|
||||
[string]$ResultPath = "artifacts\c18z13-live-service-channel-route-quality-smoke-result.json"
|
||||
)
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = "Stop"
|
||||
Add-Type -AssemblyName System.Net.Http
|
||||
|
||||
$repoRoot = (Resolve-Path (Join-Path $PSScriptRoot "..\..")).ProviderPath
|
||||
$runId = "c18z13-" + (Get-Date -Format "yyyyMMdd-HHmmss")
|
||||
$resourceId = "vpn-$runId"
|
||||
|
||||
function Invoke-Api {
|
||||
param(
|
||||
[string]$Method,
|
||||
[string]$Path,
|
||||
[object]$Body = $null
|
||||
)
|
||||
$uri = "$ApiBaseUrl$Path"
|
||||
try {
|
||||
if ($null -eq $Body) {
|
||||
return Invoke-RestMethod -Method $Method -Uri $uri -TimeoutSec 30
|
||||
}
|
||||
return Invoke-RestMethod -Method $Method -Uri $uri -ContentType "application/json" -Body ($Body | ConvertTo-Json -Depth 100) -TimeoutSec 30
|
||||
}
|
||||
catch {
|
||||
$statusCode = $null
|
||||
if ($_.Exception.Response) {
|
||||
$statusCode = [int]$_.Exception.Response.StatusCode
|
||||
}
|
||||
$details = $_.ErrorDetails.Message
|
||||
if (-not $details) {
|
||||
$details = $_.Exception.Message
|
||||
}
|
||||
throw "$Method $Path failed with HTTP $statusCode`: $details"
|
||||
}
|
||||
}
|
||||
|
||||
function Get-ObjectPropertyValue {
|
||||
param(
|
||||
[object]$Object,
|
||||
[string]$Name
|
||||
)
|
||||
if ($null -eq $Object) {
|
||||
return $null
|
||||
}
|
||||
$prop = $Object.PSObject.Properties[$Name]
|
||||
if ($null -eq $prop) {
|
||||
return $null
|
||||
}
|
||||
return $prop.Value
|
||||
}
|
||||
|
||||
function Get-NodeByName {
|
||||
param([string]$Name)
|
||||
$nodes = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/nodes?actor_user_id=$ActorUserID").nodes
|
||||
$node = @($nodes | Where-Object { $_.name -eq $Name }) | Select-Object -First 1
|
||||
if ($null -eq $node) {
|
||||
throw "Node '$Name' was not found in cluster $ClusterID"
|
||||
}
|
||||
return $node
|
||||
}
|
||||
|
||||
function Get-MeshPort {
|
||||
param([string]$Name)
|
||||
switch ($Name) {
|
||||
"test-1" { return 19131 }
|
||||
"test-2" { return 19132 }
|
||||
"test-3" { return 19133 }
|
||||
default { return 19131 }
|
||||
}
|
||||
}
|
||||
|
||||
function Enable-TestMeshListener {
|
||||
param([object]$Node)
|
||||
$port = Get-MeshPort -Name $Node.name
|
||||
Invoke-Api -Method PUT -Path "/clusters/$ClusterID/nodes/$($Node.id)/workloads/mesh-listener/desired" -Body @{
|
||||
actor_user_id = $ActorUserID
|
||||
desired_state = "enabled"
|
||||
runtime_mode = "container"
|
||||
version = "c18z13-live-fsc-route-quality"
|
||||
config = @{
|
||||
listen_addr = "0.0.0.0:$port"
|
||||
listen_port_mode = "manual"
|
||||
advertise_endpoint = "http://192.168.200.61:$port"
|
||||
advertise_transport = "direct_http"
|
||||
connectivity_mode = "private_lan"
|
||||
nat_type = "none"
|
||||
region = "docker-test"
|
||||
production_forwarding = $true
|
||||
}
|
||||
environment = @{}
|
||||
} | Out-Null
|
||||
}
|
||||
|
||||
function Get-SmokeRouteLabel {
|
||||
param([object]$RouteIntent)
|
||||
if ($null -eq $RouteIntent -or $null -eq $RouteIntent.PSObject.Properties["policy"]) {
|
||||
return ""
|
||||
}
|
||||
if ($null -eq $RouteIntent.policy -or $null -eq $RouteIntent.policy.PSObject.Properties["metadata"]) {
|
||||
return ""
|
||||
}
|
||||
$metadata = $RouteIntent.policy.metadata
|
||||
if ($null -eq $metadata) {
|
||||
return ""
|
||||
}
|
||||
$smoke = $metadata.PSObject.Properties["smoke"]
|
||||
if ($null -eq $smoke) {
|
||||
return ""
|
||||
}
|
||||
return [string]$smoke.Value
|
||||
}
|
||||
|
||||
function Clear-SmokeRouteIntents {
|
||||
$items = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/mesh/route-intents?actor_user_id=$ActorUserID").route_intents
|
||||
foreach ($item in @($items)) {
|
||||
if ([string]$item.lifecycle_status -ne "active" -or [string]$item.service_class -ne "vpn_packets") {
|
||||
continue
|
||||
}
|
||||
if ((Get-SmokeRouteLabel -RouteIntent $item) -ne "c18z13_live_service_channel_route_quality") {
|
||||
continue
|
||||
}
|
||||
Invoke-Api -Method POST -Path "/clusters/$ClusterID/mesh/route-intents/$($item.id)/expire" -Body @{ actor_user_id = $ActorUserID } | Out-Null
|
||||
}
|
||||
}
|
||||
|
||||
function New-RouteIntent {
|
||||
param(
|
||||
[string]$SourceNodeID,
|
||||
[string]$DestinationNodeID,
|
||||
[string[]]$Hops,
|
||||
[int]$Priority,
|
||||
[string]$Label
|
||||
)
|
||||
$expiresAt = (Get-Date).ToUniversalTime().AddMinutes(10).ToString("o")
|
||||
return Invoke-Api -Method POST -Path "/clusters/$ClusterID/mesh/route-intents" -Body @{
|
||||
actor_user_id = $ActorUserID
|
||||
source_selector = @{ node_id = $SourceNodeID }
|
||||
destination_selector = @{ node_id = $DestinationNodeID }
|
||||
service_class = "vpn_packets"
|
||||
priority = $Priority
|
||||
policy = @{
|
||||
synthetic_enabled = $true
|
||||
route_version = "$runId-$Label"
|
||||
policy_version = "$runId-$Label"
|
||||
peer_directory_version = "$runId-$Label"
|
||||
hops = @($Hops)
|
||||
allowed_channels = @("vpn_packet", "fabric_control")
|
||||
max_ttl = 8
|
||||
max_hops = 8
|
||||
expires_at = $expiresAt
|
||||
metadata = @{
|
||||
smoke = "c18z13_live_service_channel_route_quality"
|
||||
run_id = $runId
|
||||
label = $Label
|
||||
live_route_quality_smoke = $true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function Get-SyntheticConfig {
|
||||
param([string]$NodeID)
|
||||
return Invoke-Api -Method GET -Path "/clusters/$ClusterID/nodes/$NodeID/mesh/synthetic-config?actor_user_id=$ActorUserID"
|
||||
}
|
||||
|
||||
function Get-LatestHeartbeat {
|
||||
param([string]$NodeID)
|
||||
return (Invoke-Api -Method GET -Path "/clusters/$ClusterID/nodes/$NodeID/heartbeats?actor_user_id=$ActorUserID&limit=1").heartbeats[0]
|
||||
}
|
||||
|
||||
function Get-LatestRuntimeReport {
|
||||
param([string]$NodeID)
|
||||
$hb = Get-LatestHeartbeat -NodeID $NodeID
|
||||
return @{
|
||||
heartbeat = $hb
|
||||
report = $hb.metadata.fabric_service_channel_runtime_report
|
||||
}
|
||||
}
|
||||
|
||||
function Wait-ForRuntimeReady {
|
||||
param(
|
||||
[string]$NodeID,
|
||||
[int]$MinRoutes,
|
||||
[int]$TimeoutSeconds = 90
|
||||
)
|
||||
$deadline = (Get-Date).AddSeconds($TimeoutSeconds)
|
||||
do {
|
||||
$latest = Get-LatestRuntimeReport -NodeID $NodeID
|
||||
$report = $latest.report
|
||||
if ($null -ne $report -and
|
||||
$report.enabled -eq $true -and
|
||||
$report.production_payload_forwarding -eq $true -and
|
||||
[int]$report.route_candidate_total -ge $MinRoutes) {
|
||||
return $latest
|
||||
}
|
||||
Start-Sleep -Seconds 2
|
||||
} while ((Get-Date) -lt $deadline)
|
||||
throw "Timed out waiting for production service-channel runtime ready on node $NodeID"
|
||||
}
|
||||
|
||||
function Wait-ForRuntimeConfigVersion {
|
||||
param(
|
||||
[string]$NodeID,
|
||||
[string]$ConfigVersion,
|
||||
[int]$TimeoutSeconds = 90
|
||||
)
|
||||
$deadline = (Get-Date).AddSeconds($TimeoutSeconds)
|
||||
do {
|
||||
$latest = Get-LatestRuntimeReport -NodeID $NodeID
|
||||
if ($null -ne $latest.report -and [string]$latest.report.config_version -ge $ConfigVersion) {
|
||||
return $latest
|
||||
}
|
||||
Start-Sleep -Seconds 2
|
||||
} while ((Get-Date) -lt $deadline)
|
||||
throw "Timed out waiting for node $NodeID to load synthetic config $ConfigVersion"
|
||||
}
|
||||
|
||||
function Wait-ForRouteIntentVisible {
|
||||
param(
|
||||
[string]$NodeID,
|
||||
[string[]]$RouteIDs,
|
||||
[int]$TimeoutSeconds = 60
|
||||
)
|
||||
$deadline = (Get-Date).AddSeconds($TimeoutSeconds)
|
||||
do {
|
||||
$config = Get-SyntheticConfig -NodeID $NodeID
|
||||
$routes = @($config.synthetic_mesh_config.routes)
|
||||
$present = @($routes | Where-Object { $RouteIDs -contains $_.route_id })
|
||||
if ($present.Count -ge $RouteIDs.Count) {
|
||||
return $config
|
||||
}
|
||||
Start-Sleep -Seconds 2
|
||||
} while ((Get-Date) -lt $deadline)
|
||||
throw "Timed out waiting for routes '$($RouteIDs -join ",")' in synthetic config for node $NodeID"
|
||||
}
|
||||
|
||||
function Wait-ForRouteIntentNotVisible {
|
||||
param(
|
||||
[string]$NodeID,
|
||||
[string]$RouteID,
|
||||
[int]$TimeoutSeconds = 90
|
||||
)
|
||||
$deadline = (Get-Date).AddSeconds($TimeoutSeconds)
|
||||
do {
|
||||
$config = Get-SyntheticConfig -NodeID $NodeID
|
||||
$routes = @($config.synthetic_mesh_config.routes)
|
||||
$present = @($routes | Where-Object { $_.route_id -eq $RouteID })
|
||||
if ($present.Count -eq 0) {
|
||||
return $config
|
||||
}
|
||||
Start-Sleep -Seconds 2
|
||||
} while ((Get-Date) -lt $deadline)
|
||||
throw "Timed out waiting for route '$RouteID' to disappear from synthetic config for node $NodeID"
|
||||
}
|
||||
|
||||
function New-ServiceChannelLease {
|
||||
param(
|
||||
[string]$EntryNodeID,
|
||||
[string]$ExitNodeID
|
||||
)
|
||||
return (Invoke-Api -Method POST -Path "/clusters/$ClusterID/fabric/service-channels/leases" -Body @{
|
||||
actor_user_id = $ActorUserID
|
||||
organization_id = "org-c18z13-smoke"
|
||||
user_id = $ActorUserID
|
||||
resource_id = $resourceId
|
||||
service_class = "vpn_packets"
|
||||
entry_node_ids = @($EntryNodeID)
|
||||
exit_node_ids = @($ExitNodeID)
|
||||
preferred_entry_node_id = $EntryNodeID
|
||||
preferred_exit_node_id = $ExitNodeID
|
||||
allowed_channels = @("vpn_packet", "bulk", "control")
|
||||
ttl_seconds = 300
|
||||
metadata = @{
|
||||
smoke = "c18z13_live_service_channel_route_quality"
|
||||
run_id = $runId
|
||||
}
|
||||
}).fabric_service_channel_lease
|
||||
}
|
||||
|
||||
function ConvertTo-Base64UrlJson {
|
||||
param([object]$Value)
|
||||
$json = $Value | ConvertTo-Json -Depth 100 -Compress
|
||||
$bytes = [System.Text.Encoding]::UTF8.GetBytes($json)
|
||||
return [Convert]::ToBase64String($bytes).TrimEnd("=").Replace("+", "-").Replace("/", "_")
|
||||
}
|
||||
|
||||
function New-TestIPv4UDPPacket {
|
||||
param([int]$SourcePort)
|
||||
$payload = [System.Text.Encoding]::ASCII.GetBytes("c18z13-$SourcePort")
|
||||
$totalLength = 20 + 8 + $payload.Length
|
||||
$packet = New-Object byte[] $totalLength
|
||||
$packet[0] = 0x45
|
||||
$packet[1] = 0
|
||||
$packet[2] = [byte](($totalLength -shr 8) -band 0xff)
|
||||
$packet[3] = [byte]($totalLength -band 0xff)
|
||||
$packet[8] = 64
|
||||
$packet[9] = 17
|
||||
$packet[12] = 10; $packet[13] = 18; $packet[14] = 13; $packet[15] = 10
|
||||
$packet[16] = 10; $packet[17] = 18; $packet[18] = 13; $packet[19] = 20
|
||||
$udpOffset = 20
|
||||
$destPort = 3389
|
||||
$udpLength = 8 + $payload.Length
|
||||
$packet[$udpOffset] = [byte](($SourcePort -shr 8) -band 0xff)
|
||||
$packet[$udpOffset + 1] = [byte]($SourcePort -band 0xff)
|
||||
$packet[$udpOffset + 2] = [byte](($destPort -shr 8) -band 0xff)
|
||||
$packet[$udpOffset + 3] = [byte]($destPort -band 0xff)
|
||||
$packet[$udpOffset + 4] = [byte](($udpLength -shr 8) -band 0xff)
|
||||
$packet[$udpOffset + 5] = [byte]($udpLength -band 0xff)
|
||||
[Array]::Copy($payload, 0, $packet, 28, $payload.Length)
|
||||
return $packet
|
||||
}
|
||||
|
||||
function New-PacketBatchBody {
|
||||
param([byte[][]]$Packets)
|
||||
$stream = [System.IO.MemoryStream]::new()
|
||||
foreach ($packet in $Packets) {
|
||||
$length = $packet.Length
|
||||
$stream.WriteByte([byte](($length -shr 24) -band 0xff))
|
||||
$stream.WriteByte([byte](($length -shr 16) -band 0xff))
|
||||
$stream.WriteByte([byte](($length -shr 8) -band 0xff))
|
||||
$stream.WriteByte([byte]($length -band 0xff))
|
||||
$stream.Write($packet, 0, $packet.Length)
|
||||
}
|
||||
return $stream.ToArray()
|
||||
}
|
||||
|
||||
function Invoke-ServiceChannelPost {
|
||||
param(
|
||||
[object]$Lease,
|
||||
[int]$PortStart
|
||||
)
|
||||
$packets = @()
|
||||
for ($i = 0; $i -lt $PacketsPerBatch; $i++) {
|
||||
$packets += ,(New-TestIPv4UDPPacket -SourcePort ($PortStart + $i))
|
||||
}
|
||||
$path = $Lease.entry_http.path_template.
|
||||
Replace("{cluster_id}", $ClusterID).
|
||||
Replace("{channel_id}", $Lease.channel_id).
|
||||
Replace("{resource_id}", $resourceId)
|
||||
$url = "$EntryBaseUrl$path`?batch=true"
|
||||
$headers = @{
|
||||
"X-RAP-Service-Channel-Token" = $Lease.token.token
|
||||
"X-RAP-Fabric-Channel-ID" = $Lease.channel_id
|
||||
"X-RAP-Service-Class" = "vpn_packets"
|
||||
"X-RAP-Channel-Class" = "vpn_packet"
|
||||
"X-RAP-Service-Channel-Authority-Payload" = ConvertTo-Base64UrlJson -Value $Lease.authority_payload
|
||||
"X-RAP-Service-Channel-Authority-Signature" = ConvertTo-Base64UrlJson -Value $Lease.authority_signature
|
||||
}
|
||||
$body = New-PacketBatchBody -Packets $packets
|
||||
$client = [System.Net.Http.HttpClient]::new()
|
||||
try {
|
||||
$client.Timeout = [TimeSpan]::FromSeconds(30)
|
||||
$request = [System.Net.Http.HttpRequestMessage]::new([System.Net.Http.HttpMethod]::Post, $url)
|
||||
foreach ($header in $headers.GetEnumerator()) {
|
||||
[void]$request.Headers.TryAddWithoutValidation($header.Key, [string]$header.Value)
|
||||
}
|
||||
$content = [System.Net.Http.ByteArrayContent]::new($body)
|
||||
$content.Headers.ContentType = [System.Net.Http.Headers.MediaTypeHeaderValue]::Parse("application/vnd.rap.vpn-packet-batch.v1")
|
||||
$request.Content = $content
|
||||
$response = $client.SendAsync($request).GetAwaiter().GetResult()
|
||||
$responseBody = $response.Content.ReadAsStringAsync().GetAwaiter().GetResult()
|
||||
if (-not $response.IsSuccessStatusCode) {
|
||||
throw "Service-channel POST $url failed with HTTP $([int]$response.StatusCode): $responseBody"
|
||||
}
|
||||
return [pscustomobject]@{ ok = $true; status_code = [int]$response.StatusCode; error = "" }
|
||||
}
|
||||
catch {
|
||||
return [pscustomobject]@{ ok = $false; status_code = 0; error = $_.Exception.Message }
|
||||
}
|
||||
finally {
|
||||
$client.Dispose()
|
||||
}
|
||||
}
|
||||
|
||||
function Send-BatchSeries {
|
||||
param(
|
||||
[object]$Lease,
|
||||
[int]$Count,
|
||||
[int]$PortBase
|
||||
)
|
||||
$results = @()
|
||||
for ($i = 0; $i -lt $Count; $i++) {
|
||||
$results += Invoke-ServiceChannelPost -Lease $Lease -PortStart ($PortBase + ($i * 100))
|
||||
if ($BatchDelayMilliseconds -gt 0) {
|
||||
Start-Sleep -Milliseconds $BatchDelayMilliseconds
|
||||
}
|
||||
}
|
||||
return $results
|
||||
}
|
||||
|
||||
function Get-ExitQueueDepth {
|
||||
param(
|
||||
[string]$NodeID,
|
||||
[string]$VPNConnectionID
|
||||
)
|
||||
$latest = Get-LatestRuntimeReport -NodeID $NodeID
|
||||
$queueKey = "$VPNConnectionID`:client_to_gateway"
|
||||
$depths = $latest.report.inbox.queue_depths
|
||||
if ($null -eq $depths) {
|
||||
return 0
|
||||
}
|
||||
$prop = $depths.PSObject.Properties[$queueKey]
|
||||
if ($null -eq $prop) {
|
||||
return 0
|
||||
}
|
||||
return [int]$prop.Value
|
||||
}
|
||||
|
||||
function Wait-ForExitQueueDepth {
|
||||
param(
|
||||
[string]$NodeID,
|
||||
[string]$VPNConnectionID,
|
||||
[int]$MinDepth,
|
||||
[int]$TimeoutSeconds = 90
|
||||
)
|
||||
$deadline = (Get-Date).AddSeconds($TimeoutSeconds)
|
||||
do {
|
||||
$depth = Get-ExitQueueDepth -NodeID $NodeID -VPNConnectionID $VPNConnectionID
|
||||
if ($depth -ge $MinDepth) {
|
||||
return $depth
|
||||
}
|
||||
Start-Sleep -Seconds 2
|
||||
} while ((Get-Date) -lt $deadline)
|
||||
throw "Timed out waiting for exit queue depth >= $MinDepth on node $NodeID"
|
||||
}
|
||||
|
||||
function Wait-ForRouteFeedback {
|
||||
param(
|
||||
[string]$ReporterNodeID,
|
||||
[string]$RouteID,
|
||||
[int]$TimeoutSeconds = 120
|
||||
)
|
||||
$deadline = (Get-Date).AddSeconds($TimeoutSeconds)
|
||||
do {
|
||||
$response = Invoke-Api -Method GET -Path "/clusters/$ClusterID/fabric/service-channels/route-feedback?actor_user_id=$ActorUserID&reporter_node_id=$ReporterNodeID&route_id=$RouteID&service_class=vpn_packets"
|
||||
$item = @($response.route_feedback | Where-Object {
|
||||
$_.route_id -eq $RouteID -and $_.feedback_status -eq "healthy" -and $_.last_send_duration_ms -ge 0
|
||||
}) | Select-Object -First 1
|
||||
if ($null -ne $item) {
|
||||
return $item
|
||||
}
|
||||
Start-Sleep -Seconds 2
|
||||
} while ((Get-Date) -lt $deadline)
|
||||
throw "Timed out waiting for live route feedback for route '$RouteID'"
|
||||
}
|
||||
|
||||
function Get-IngressFlowDropped {
|
||||
param([string]$NodeID)
|
||||
$latest = Get-LatestRuntimeReport -NodeID $NodeID
|
||||
$flowScheduler = Get-ObjectPropertyValue -Object $latest.report.ingress -Name "flow_scheduler"
|
||||
$dropped = Get-ObjectPropertyValue -Object $flowScheduler -Name "dropped"
|
||||
if ($null -eq $dropped) {
|
||||
return 0
|
||||
}
|
||||
return [int]$dropped
|
||||
}
|
||||
|
||||
function Get-BackendClientGatewayDepth {
|
||||
param([string]$VPNConnectionID)
|
||||
$stats = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/vpn-connections/$VPNConnectionID/tunnel/stats").vpn_packet_stats
|
||||
$queue = $stats.client_to_gateway
|
||||
if ($null -eq $queue) {
|
||||
return 0
|
||||
}
|
||||
$depth = Get-ObjectPropertyValue -Object $queue -Name "queue_depth"
|
||||
if ($null -eq $depth) {
|
||||
return 0
|
||||
}
|
||||
return [int]$depth
|
||||
}
|
||||
|
||||
function Invoke-RemoteDocker {
|
||||
param([string]$Command)
|
||||
& ssh $DockerSSH $Command
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "ssh $DockerSSH command failed: $Command"
|
||||
}
|
||||
}
|
||||
|
||||
function Stop-TestUpdaters {
|
||||
Invoke-RemoteDocker -Command "docker stop rap_host_agent_updater_test-1 rap_host_agent_updater_test-2 rap_host_agent_updater_test-3 >/dev/null 2>&1 || true"
|
||||
}
|
||||
|
||||
function Start-TestUpdaters {
|
||||
Invoke-RemoteDocker -Command "docker start rap_host_agent_updater_test-1 rap_host_agent_updater_test-2 rap_host_agent_updater_test-3 >/dev/null 2>&1 || true"
|
||||
}
|
||||
|
||||
$entryNode = Get-NodeByName -Name $EntryNodeName
|
||||
$relayNode = Get-NodeByName -Name $RelayNodeName
|
||||
$exitNode = Get-NodeByName -Name $ExitNodeName
|
||||
$slowInitialRouteID = ""
|
||||
$fastRouteID = ""
|
||||
$slowCandidateRouteID = ""
|
||||
$updatersStopped = $false
|
||||
$result = $null
|
||||
|
||||
try {
|
||||
Stop-TestUpdaters
|
||||
$updatersStopped = $true
|
||||
|
||||
Enable-TestMeshListener -Node $entryNode
|
||||
Enable-TestMeshListener -Node $relayNode
|
||||
Enable-TestMeshListener -Node $exitNode
|
||||
Clear-SmokeRouteIntents
|
||||
|
||||
$slowInitialIntent = New-RouteIntent `
|
||||
-SourceNodeID $entryNode.id `
|
||||
-DestinationNodeID $exitNode.id `
|
||||
-Hops @($entryNode.id, $relayNode.id, $exitNode.id) `
|
||||
-Priority 1999999960 `
|
||||
-Label "slow-initial-higher-priority"
|
||||
$fastIntent = New-RouteIntent `
|
||||
-SourceNodeID $entryNode.id `
|
||||
-DestinationNodeID $exitNode.id `
|
||||
-Hops @($entryNode.id, $exitNode.id) `
|
||||
-Priority 1999999950 `
|
||||
-Label "fast-live-lower-priority"
|
||||
$slowInitialRouteID = $slowInitialIntent.route_intent.id
|
||||
$fastRouteID = $fastIntent.route_intent.id
|
||||
|
||||
$visibleConfig = Wait-ForRouteIntentVisible -NodeID $entryNode.id -RouteIDs @($slowInitialRouteID, $fastRouteID)
|
||||
$exitVisibleConfig = Wait-ForRouteIntentVisible -NodeID $exitNode.id -RouteIDs @($slowInitialRouteID, $fastRouteID)
|
||||
$readyBefore = Wait-ForRuntimeReady -NodeID $entryNode.id -MinRoutes 2
|
||||
$exitReadyBefore = Wait-ForRuntimeReady -NodeID $exitNode.id -MinRoutes 0
|
||||
$loadedConfig = Wait-ForRuntimeConfigVersion -NodeID $entryNode.id -ConfigVersion $visibleConfig.synthetic_mesh_config.config_version
|
||||
$exitLoadedConfig = Wait-ForRuntimeConfigVersion -NodeID $exitNode.id -ConfigVersion $exitVisibleConfig.synthetic_mesh_config.config_version
|
||||
|
||||
$initialLease = New-ServiceChannelLease -EntryNodeID $entryNode.id -ExitNodeID $exitNode.id
|
||||
if ($initialLease.status -ne "ready" -or [string]$initialLease.primary_route.route_id -ne $slowInitialRouteID) {
|
||||
throw "Initial lease should select higher-priority slow route '$slowInitialRouteID': status=$($initialLease.status) route=$($initialLease.primary_route.route_id)"
|
||||
}
|
||||
|
||||
$expiredSlowInitial = Invoke-Api -Method POST -Path "/clusters/$ClusterID/mesh/route-intents/$slowInitialRouteID/expire" -Body @{ actor_user_id = $ActorUserID }
|
||||
$notVisibleConfig = Wait-ForRouteIntentNotVisible -NodeID $entryNode.id -RouteID $slowInitialRouteID
|
||||
$fastOnlyRuntime = Wait-ForRuntimeConfigVersion -NodeID $entryNode.id -ConfigVersion $notVisibleConfig.synthetic_mesh_config.config_version
|
||||
$fastLease = New-ServiceChannelLease -EntryNodeID $entryNode.id -ExitNodeID $exitNode.id
|
||||
if ($fastLease.status -ne "ready" -or [string]$fastLease.primary_route.route_id -ne $fastRouteID) {
|
||||
throw "Fast-only lease should select live fast route '$fastRouteID': status=$($fastLease.status) route=$($fastLease.primary_route.route_id)"
|
||||
}
|
||||
|
||||
$baselineExitDepth = Get-ExitQueueDepth -NodeID $exitNode.id -VPNConnectionID $resourceId
|
||||
$baselineBackendDepth = Get-BackendClientGatewayDepth -VPNConnectionID $resourceId
|
||||
$baselineDropped = Get-IngressFlowDropped -NodeID $entryNode.id
|
||||
$sendResults = Send-BatchSeries -Lease $fastLease -Count $BatchCount -PortBase 61000
|
||||
$failedSends = @($sendResults | Where-Object { $_.ok -ne $true })
|
||||
if ($failedSends.Count -gt 0) {
|
||||
throw "Live fast-route service-channel sends failed: $($failedSends[0].error)"
|
||||
}
|
||||
$expectedPackets = $BatchCount * $PacketsPerBatch
|
||||
$exitDepthAfterFast = Wait-ForExitQueueDepth -NodeID $exitNode.id -VPNConnectionID $resourceId -MinDepth ($baselineExitDepth + $expectedPackets) -TimeoutSeconds 120
|
||||
$fastFeedback = Wait-ForRouteFeedback -ReporterNodeID $entryNode.id -RouteID $fastRouteID -TimeoutSeconds 120
|
||||
|
||||
$slowCandidateIntent = New-RouteIntent `
|
||||
-SourceNodeID $entryNode.id `
|
||||
-DestinationNodeID $exitNode.id `
|
||||
-Hops @($entryNode.id, $relayNode.id, $exitNode.id) `
|
||||
-Priority 1999999960 `
|
||||
-Label "slow-candidate-higher-priority"
|
||||
$slowCandidateRouteID = $slowCandidateIntent.route_intent.id
|
||||
$candidateVisibleConfig = Wait-ForRouteIntentVisible -NodeID $entryNode.id -RouteIDs @($slowCandidateRouteID, $fastRouteID)
|
||||
$candidateLoadedConfig = Wait-ForRuntimeConfigVersion -NodeID $entryNode.id -ConfigVersion $candidateVisibleConfig.synthetic_mesh_config.config_version
|
||||
Start-Sleep -Seconds 2
|
||||
$qualityLease = New-ServiceChannelLease -EntryNodeID $entryNode.id -ExitNodeID $exitNode.id
|
||||
if ($qualityLease.status -ne "ready" -or [string]$qualityLease.primary_route.route_id -ne $fastRouteID) {
|
||||
throw "Quality lease should select live-learned fast route '$fastRouteID': status=$($qualityLease.status) route=$($qualityLease.primary_route.route_id) score=$($qualityLease.primary_route.path_score)"
|
||||
}
|
||||
|
||||
$finalDropped = Get-IngressFlowDropped -NodeID $entryNode.id
|
||||
$finalBackendDepth = Get-BackendClientGatewayDepth -VPNConnectionID $resourceId
|
||||
$finalEntryRuntime = Get-LatestRuntimeReport -NodeID $entryNode.id
|
||||
$finalExitRuntime = Get-LatestRuntimeReport -NodeID $exitNode.id
|
||||
$expiredFastFeedback = Invoke-Api -Method POST -Path "/clusters/$ClusterID/fabric/service-channels/route-feedback/expire" -Body @{
|
||||
actor_user_id = $ActorUserID
|
||||
reporter_node_id = $entryNode.id
|
||||
route_id = $fastRouteID
|
||||
service_class = "vpn_packets"
|
||||
reason = "c18z13 live route quality smoke cleanup"
|
||||
}
|
||||
$expiredSlowCandidate = Invoke-Api -Method POST -Path "/clusters/$ClusterID/mesh/route-intents/$slowCandidateRouteID/expire" -Body @{ actor_user_id = $ActorUserID }
|
||||
$expiredFast = Invoke-Api -Method POST -Path "/clusters/$ClusterID/mesh/route-intents/$fastRouteID/expire" -Body @{ actor_user_id = $ActorUserID }
|
||||
|
||||
$result = [ordered]@{
|
||||
schema_version = "c18z13.live_service_channel_route_quality_smoke.v1"
|
||||
run_id = $runId
|
||||
base_url = $ApiBaseUrl
|
||||
entry_base_url = $EntryBaseUrl
|
||||
cluster_id = $ClusterID
|
||||
entry_node = @{ name = $entryNode.name; id = $entryNode.id }
|
||||
relay_node = @{ name = $relayNode.name; id = $relayNode.id }
|
||||
exit_node = @{ name = $exitNode.name; id = $exitNode.id }
|
||||
resource_id = $resourceId
|
||||
route_intents = @{
|
||||
slow_initial_route_id = $slowInitialRouteID
|
||||
fast_route_id = $fastRouteID
|
||||
slow_candidate_route_id = $slowCandidateRouteID
|
||||
slow_initial_status = $expiredSlowInitial.route_intent.lifecycle_status
|
||||
slow_candidate_status = $expiredSlowCandidate.route_intent.lifecycle_status
|
||||
fast_status = $expiredFast.route_intent.lifecycle_status
|
||||
}
|
||||
initial_lease = @{
|
||||
status = $initialLease.status
|
||||
primary_route_id = $initialLease.primary_route.route_id
|
||||
primary_path_score = $initialLease.primary_route.path_score
|
||||
score_reasons = $initialLease.primary_route.score_reasons
|
||||
}
|
||||
live_fast_traffic = @{
|
||||
batches = $BatchCount
|
||||
packets_per_batch = $PacketsPerBatch
|
||||
expected_packets = $expectedPackets
|
||||
sent_batches = @($sendResults | Where-Object { $_.ok -eq $true }).Count
|
||||
exit_queue_baseline = $baselineExitDepth
|
||||
exit_queue_depth = $exitDepthAfterFast
|
||||
}
|
||||
live_feedback = @{
|
||||
route_id = $fastFeedback.route_id
|
||||
feedback_status = $fastFeedback.feedback_status
|
||||
score_adjustment = $fastFeedback.score_adjustment
|
||||
reasons = $fastFeedback.reasons
|
||||
last_send_duration_ms = $fastFeedback.last_send_duration_ms
|
||||
observed_at = $fastFeedback.observed_at
|
||||
expire_result = $expiredFastFeedback.route_feedback_expire
|
||||
}
|
||||
quality_lease = @{
|
||||
status = $qualityLease.status
|
||||
primary_route_id = $qualityLease.primary_route.route_id
|
||||
primary_path_score = $qualityLease.primary_route.path_score
|
||||
score_reasons = $qualityLease.primary_route.score_reasons
|
||||
alternate_route_count = @($qualityLease.alternate_routes).Count
|
||||
}
|
||||
backend_fallback_queue = @{
|
||||
baseline_depth = $baselineBackendDepth
|
||||
depth = $finalBackendDepth
|
||||
}
|
||||
flow_drops = @{
|
||||
baseline = $baselineDropped
|
||||
final = $finalDropped
|
||||
delta = ($finalDropped - $baselineDropped)
|
||||
}
|
||||
passed = $true
|
||||
checks = [ordered]@{
|
||||
production_forwarding_ready = ($readyBefore.report.production_payload_forwarding -eq $true)
|
||||
exit_production_forwarding_ready = ($exitReadyBefore.report.production_payload_forwarding -eq $true)
|
||||
route_intents_visible = (@($visibleConfig.synthetic_mesh_config.routes | Where-Object { @($slowInitialRouteID, $fastRouteID) -contains $_.route_id }).Count -ge 2)
|
||||
entry_runtime_loaded_visible_config = ([string]$loadedConfig.report.config_version -ge [string]$visibleConfig.synthetic_mesh_config.config_version)
|
||||
exit_runtime_loaded_visible_config = ([string]$exitLoadedConfig.report.config_version -ge [string]$exitVisibleConfig.synthetic_mesh_config.config_version)
|
||||
initial_prefers_high_priority_slow_route = ([string]$initialLease.primary_route.route_id -eq $slowInitialRouteID)
|
||||
fast_only_runtime_loaded_after_slow_expire = ([string]$fastOnlyRuntime.report.config_version -ge [string]$notVisibleConfig.synthetic_mesh_config.config_version)
|
||||
fast_only_lease_selects_fast_route = ([string]$fastLease.primary_route.route_id -eq $fastRouteID)
|
||||
live_fast_packets_reached_exit = ($exitDepthAfterFast -ge ($baselineExitDepth + $expectedPackets))
|
||||
backend_persisted_live_fast_feedback = ([string]$fastFeedback.route_id -eq $fastRouteID -and [string]$fastFeedback.feedback_status -eq "healthy")
|
||||
live_feedback_has_recent_success_reason = (@($fastFeedback.reasons | Where-Object { $_ -eq "service_channel_recent_success" }).Count -ge 1)
|
||||
candidate_runtime_loaded = ([string]$candidateLoadedConfig.report.config_version -ge [string]$candidateVisibleConfig.synthetic_mesh_config.config_version)
|
||||
quality_prefers_live_learned_fast_route = ([string]$qualityLease.primary_route.route_id -eq $fastRouteID)
|
||||
quality_lease_uses_live_feedback_reason = (@($qualityLease.primary_route.score_reasons | Where-Object { $_ -eq "service_channel_recent_success" }).Count -ge 1)
|
||||
no_backend_fallback_used = ($finalBackendDepth -eq $baselineBackendDepth)
|
||||
no_flow_drops = (($finalDropped - $baselineDropped) -eq 0)
|
||||
route_intents_expired = ($expiredFast.route_intent.lifecycle_status -eq "expired" -and $expiredSlowCandidate.route_intent.lifecycle_status -eq "expired")
|
||||
}
|
||||
telemetry = @{
|
||||
final_entry_ingress = $finalEntryRuntime.report.ingress
|
||||
final_exit_inbox = $finalExitRuntime.report.inbox
|
||||
}
|
||||
}
|
||||
$failedChecks = @($result.checks.GetEnumerator() | Where-Object { $_.Value -ne $true })
|
||||
if ($failedChecks.Count -gt 0) {
|
||||
throw "C18Z13 failed checks: $($failedChecks.Name -join ', ')"
|
||||
}
|
||||
}
|
||||
finally {
|
||||
if ($slowInitialRouteID) {
|
||||
try { Invoke-Api -Method POST -Path "/clusters/$ClusterID/mesh/route-intents/$slowInitialRouteID/expire" -Body @{ actor_user_id = $ActorUserID } | Out-Null } catch {}
|
||||
}
|
||||
if ($slowCandidateRouteID) {
|
||||
try { Invoke-Api -Method POST -Path "/clusters/$ClusterID/mesh/route-intents/$slowCandidateRouteID/expire" -Body @{ actor_user_id = $ActorUserID } | Out-Null } catch {}
|
||||
}
|
||||
if ($fastRouteID) {
|
||||
try { Invoke-Api -Method POST -Path "/clusters/$ClusterID/mesh/route-intents/$fastRouteID/expire" -Body @{ actor_user_id = $ActorUserID } | Out-Null } catch {}
|
||||
try {
|
||||
Invoke-Api -Method POST -Path "/clusters/$ClusterID/fabric/service-channels/route-feedback/expire" -Body @{
|
||||
actor_user_id = $ActorUserID
|
||||
reporter_node_id = $entryNode.id
|
||||
route_id = $fastRouteID
|
||||
service_class = "vpn_packets"
|
||||
reason = "c18z13 live route quality smoke cleanup"
|
||||
} | Out-Null
|
||||
} catch {}
|
||||
}
|
||||
if ($updatersStopped) {
|
||||
try { Start-TestUpdaters } catch { Write-Warning "Could not restart test updaters: $($_.Exception.Message)" }
|
||||
}
|
||||
}
|
||||
|
||||
$resultFullPath = Join-Path $repoRoot $ResultPath
|
||||
$resultDir = Split-Path -Parent $resultFullPath
|
||||
if (-not (Test-Path $resultDir)) {
|
||||
New-Item -ItemType Directory -Path $resultDir | Out-Null
|
||||
}
|
||||
$result | ConvertTo-Json -Depth 100 | Set-Content -Path $resultFullPath -Encoding UTF8
|
||||
Write-Host "C18Z13 live service-channel route quality smoke passed. Result: $resultFullPath"
|
||||
$result
|
||||
@@ -0,0 +1,586 @@
|
||||
param(
|
||||
[string]$ApiBaseUrl = "http://192.168.200.61:18121/api/v1",
|
||||
[string]$ClusterID = "cfc0743d-d960-49fb-9de8-96e063d5e4aa",
|
||||
[string]$ActorUserID = "f67d943f-5397-4b3a-a229-695fe67ad700",
|
||||
[string]$EntryNodeName = "test-1",
|
||||
[string]$RelayNodeName = "test-3",
|
||||
[string]$ExitNodeName = "test-2",
|
||||
[string]$EntryBaseUrl = "http://192.168.200.61:19131",
|
||||
[string]$DockerSSH = "test-docker",
|
||||
[int]$InitialBatchCount = 12,
|
||||
[int]$LearningBatchCount = 24,
|
||||
[int]$PostChurnBatchCount = 24,
|
||||
[int]$PacketsPerBatch = 8,
|
||||
[int]$BatchDelayMilliseconds = 25,
|
||||
[string]$ResultPath = "artifacts\c18z14-live-service-channel-active-quality-shift-smoke-result.json"
|
||||
)
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = "Stop"
|
||||
Add-Type -AssemblyName System.Net.Http
|
||||
|
||||
$repoRoot = (Resolve-Path (Join-Path $PSScriptRoot "..\..")).ProviderPath
|
||||
$runId = "c18z14-" + (Get-Date -Format "yyyyMMdd-HHmmss")
|
||||
$resourceId = "vpn-$runId"
|
||||
|
||||
function Invoke-Api {
|
||||
param([string]$Method, [string]$Path, [object]$Body = $null)
|
||||
$uri = "$ApiBaseUrl$Path"
|
||||
try {
|
||||
if ($null -eq $Body) {
|
||||
return Invoke-RestMethod -Method $Method -Uri $uri -TimeoutSec 30
|
||||
}
|
||||
return Invoke-RestMethod -Method $Method -Uri $uri -ContentType "application/json" -Body ($Body | ConvertTo-Json -Depth 100) -TimeoutSec 30
|
||||
} catch {
|
||||
$statusCode = $null
|
||||
if ($_.Exception.Response) { $statusCode = [int]$_.Exception.Response.StatusCode }
|
||||
$details = $_.ErrorDetails.Message
|
||||
if (-not $details) { $details = $_.Exception.Message }
|
||||
throw "$Method $Path failed with HTTP $statusCode`: $details"
|
||||
}
|
||||
}
|
||||
|
||||
function Get-ObjectPropertyValue {
|
||||
param([object]$Object, [string]$Name)
|
||||
if ($null -eq $Object) { return $null }
|
||||
$prop = $Object.PSObject.Properties[$Name]
|
||||
if ($null -eq $prop) { return $null }
|
||||
return $prop.Value
|
||||
}
|
||||
|
||||
function Get-NodeByName {
|
||||
param([string]$Name)
|
||||
$nodes = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/nodes?actor_user_id=$ActorUserID").nodes
|
||||
$node = @($nodes | Where-Object { $_.name -eq $Name }) | Select-Object -First 1
|
||||
if ($null -eq $node) { throw "Node '$Name' was not found in cluster $ClusterID" }
|
||||
return $node
|
||||
}
|
||||
|
||||
function Get-MeshPort {
|
||||
param([string]$Name)
|
||||
switch ($Name) {
|
||||
"test-1" { return 19131 }
|
||||
"test-2" { return 19132 }
|
||||
"test-3" { return 19133 }
|
||||
default { return 19131 }
|
||||
}
|
||||
}
|
||||
|
||||
function Enable-TestMeshListener {
|
||||
param([object]$Node)
|
||||
$port = Get-MeshPort -Name $Node.name
|
||||
Invoke-Api -Method PUT -Path "/clusters/$ClusterID/nodes/$($Node.id)/workloads/mesh-listener/desired" -Body @{
|
||||
actor_user_id = $ActorUserID
|
||||
desired_state = "enabled"
|
||||
runtime_mode = "container"
|
||||
version = "c18z14-live-fsc-active-quality-shift"
|
||||
config = @{
|
||||
listen_addr = "0.0.0.0:$port"
|
||||
listen_port_mode = "manual"
|
||||
advertise_endpoint = "http://192.168.200.61:$port"
|
||||
advertise_transport = "direct_http"
|
||||
connectivity_mode = "private_lan"
|
||||
nat_type = "none"
|
||||
region = "docker-test"
|
||||
production_forwarding = $true
|
||||
}
|
||||
environment = @{}
|
||||
} | Out-Null
|
||||
}
|
||||
|
||||
function Get-SmokeRouteLabel {
|
||||
param([object]$RouteIntent)
|
||||
if ($null -eq $RouteIntent -or $null -eq $RouteIntent.PSObject.Properties["policy"]) { return "" }
|
||||
if ($null -eq $RouteIntent.policy -or $null -eq $RouteIntent.policy.PSObject.Properties["metadata"]) { return "" }
|
||||
$metadata = $RouteIntent.policy.metadata
|
||||
if ($null -eq $metadata) { return "" }
|
||||
$smoke = $metadata.PSObject.Properties["smoke"]
|
||||
if ($null -eq $smoke) { return "" }
|
||||
return [string]$smoke.Value
|
||||
}
|
||||
|
||||
function Clear-SmokeRouteIntents {
|
||||
$items = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/mesh/route-intents?actor_user_id=$ActorUserID").route_intents
|
||||
foreach ($item in @($items)) {
|
||||
if ([string]$item.lifecycle_status -ne "active" -or [string]$item.service_class -ne "vpn_packets") { continue }
|
||||
if ((Get-SmokeRouteLabel -RouteIntent $item) -ne "c18z14_live_service_channel_active_quality_shift") { continue }
|
||||
Invoke-Api -Method POST -Path "/clusters/$ClusterID/mesh/route-intents/$($item.id)/expire" -Body @{ actor_user_id = $ActorUserID } | Out-Null
|
||||
}
|
||||
}
|
||||
|
||||
function New-RouteIntent {
|
||||
param([string]$SourceNodeID, [string]$DestinationNodeID, [string[]]$Hops, [int]$Priority, [string]$Label)
|
||||
$expiresAt = (Get-Date).ToUniversalTime().AddMinutes(10).ToString("o")
|
||||
return Invoke-Api -Method POST -Path "/clusters/$ClusterID/mesh/route-intents" -Body @{
|
||||
actor_user_id = $ActorUserID
|
||||
source_selector = @{ node_id = $SourceNodeID }
|
||||
destination_selector = @{ node_id = $DestinationNodeID }
|
||||
service_class = "vpn_packets"
|
||||
priority = $Priority
|
||||
policy = @{
|
||||
synthetic_enabled = $true
|
||||
route_version = "$runId-$Label"
|
||||
policy_version = "$runId-$Label"
|
||||
peer_directory_version = "$runId-$Label"
|
||||
hops = @($Hops)
|
||||
allowed_channels = @("vpn_packet", "fabric_control")
|
||||
max_ttl = 8
|
||||
max_hops = 8
|
||||
expires_at = $expiresAt
|
||||
metadata = @{
|
||||
smoke = "c18z14_live_service_channel_active_quality_shift"
|
||||
run_id = $runId
|
||||
label = $Label
|
||||
active_quality_shift_smoke = $true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function Get-SyntheticConfig { param([string]$NodeID) return Invoke-Api -Method GET -Path "/clusters/$ClusterID/nodes/$NodeID/mesh/synthetic-config?actor_user_id=$ActorUserID" }
|
||||
function Get-LatestHeartbeat { param([string]$NodeID) return (Invoke-Api -Method GET -Path "/clusters/$ClusterID/nodes/$NodeID/heartbeats?actor_user_id=$ActorUserID&limit=1").heartbeats[0] }
|
||||
function Get-LatestRuntimeReport {
|
||||
param([string]$NodeID)
|
||||
$hb = Get-LatestHeartbeat -NodeID $NodeID
|
||||
return @{ heartbeat = $hb; report = $hb.metadata.fabric_service_channel_runtime_report }
|
||||
}
|
||||
|
||||
function Wait-ForRuntimeReady {
|
||||
param([string]$NodeID, [int]$MinRoutes, [int]$TimeoutSeconds = 90)
|
||||
$deadline = (Get-Date).AddSeconds($TimeoutSeconds)
|
||||
do {
|
||||
$latest = Get-LatestRuntimeReport -NodeID $NodeID
|
||||
if ($null -ne $latest.report -and $latest.report.enabled -eq $true -and $latest.report.production_payload_forwarding -eq $true -and [int]$latest.report.route_candidate_total -ge $MinRoutes) {
|
||||
return $latest
|
||||
}
|
||||
Start-Sleep -Seconds 2
|
||||
} while ((Get-Date) -lt $deadline)
|
||||
throw "Timed out waiting for production service-channel runtime ready on node $NodeID"
|
||||
}
|
||||
|
||||
function Wait-ForRuntimeConfigVersion {
|
||||
param([string]$NodeID, [string]$ConfigVersion, [int]$TimeoutSeconds = 90)
|
||||
$deadline = (Get-Date).AddSeconds($TimeoutSeconds)
|
||||
do {
|
||||
$latest = Get-LatestRuntimeReport -NodeID $NodeID
|
||||
if ($null -ne $latest.report -and [string]$latest.report.config_version -ge $ConfigVersion) { return $latest }
|
||||
Start-Sleep -Seconds 2
|
||||
} while ((Get-Date) -lt $deadline)
|
||||
throw "Timed out waiting for node $NodeID to load synthetic config $ConfigVersion"
|
||||
}
|
||||
|
||||
function Wait-ForRouteIntentVisible {
|
||||
param([string]$NodeID, [string[]]$RouteIDs, [int]$TimeoutSeconds = 60)
|
||||
$deadline = (Get-Date).AddSeconds($TimeoutSeconds)
|
||||
do {
|
||||
$config = Get-SyntheticConfig -NodeID $NodeID
|
||||
$present = @($config.synthetic_mesh_config.routes | Where-Object { $RouteIDs -contains $_.route_id })
|
||||
if ($present.Count -ge $RouteIDs.Count) { return $config }
|
||||
Start-Sleep -Seconds 2
|
||||
} while ((Get-Date) -lt $deadline)
|
||||
throw "Timed out waiting for routes '$($RouteIDs -join ",")' in synthetic config for node $NodeID"
|
||||
}
|
||||
|
||||
function Wait-ForRouteIntentNotVisible {
|
||||
param([string]$NodeID, [string]$RouteID, [int]$TimeoutSeconds = 90)
|
||||
$deadline = (Get-Date).AddSeconds($TimeoutSeconds)
|
||||
do {
|
||||
$config = Get-SyntheticConfig -NodeID $NodeID
|
||||
$present = @($config.synthetic_mesh_config.routes | Where-Object { $_.route_id -eq $RouteID })
|
||||
if ($present.Count -eq 0) { return $config }
|
||||
Start-Sleep -Seconds 2
|
||||
} while ((Get-Date) -lt $deadline)
|
||||
throw "Timed out waiting for route '$RouteID' to disappear from synthetic config for node $NodeID"
|
||||
}
|
||||
|
||||
function Wait-ForRouteFeedback {
|
||||
param([string]$ReporterNodeID, [string]$RouteID, [int]$TimeoutSeconds = 120)
|
||||
$deadline = (Get-Date).AddSeconds($TimeoutSeconds)
|
||||
do {
|
||||
$response = Invoke-Api -Method GET -Path "/clusters/$ClusterID/fabric/service-channels/route-feedback?actor_user_id=$ActorUserID&reporter_node_id=$ReporterNodeID&route_id=$RouteID&service_class=vpn_packets"
|
||||
$item = @($response.route_feedback | Where-Object { $_.route_id -eq $RouteID -and $_.feedback_status -eq "healthy" }) | Select-Object -First 1
|
||||
if ($null -ne $item) { return $item }
|
||||
Start-Sleep -Seconds 2
|
||||
} while ((Get-Date) -lt $deadline)
|
||||
throw "Timed out waiting for live route feedback for route '$RouteID'"
|
||||
}
|
||||
|
||||
function Wait-ForQualityPreferenceApplied {
|
||||
param([string]$NodeID, [int]$TimeoutSeconds = 120)
|
||||
$deadline = (Get-Date).AddSeconds($TimeoutSeconds)
|
||||
do {
|
||||
$latest = Get-LatestRuntimeReport -NodeID $NodeID
|
||||
$count = Get-ObjectPropertyValue -Object $latest.report.ingress -Name "route_quality_preference_count"
|
||||
if ($null -ne $count -and [int]$count -gt 0) { return $latest }
|
||||
Start-Sleep -Seconds 2
|
||||
} while ((Get-Date) -lt $deadline)
|
||||
throw "Timed out waiting for route quality preferences on node $NodeID"
|
||||
}
|
||||
|
||||
function New-ServiceChannelLease {
|
||||
param([string]$EntryNodeID, [string]$ExitNodeID)
|
||||
return (Invoke-Api -Method POST -Path "/clusters/$ClusterID/fabric/service-channels/leases" -Body @{
|
||||
actor_user_id = $ActorUserID
|
||||
organization_id = "org-c18z14-smoke"
|
||||
user_id = $ActorUserID
|
||||
resource_id = $resourceId
|
||||
service_class = "vpn_packets"
|
||||
entry_node_ids = @($EntryNodeID)
|
||||
exit_node_ids = @($ExitNodeID)
|
||||
preferred_entry_node_id = $EntryNodeID
|
||||
preferred_exit_node_id = $ExitNodeID
|
||||
allowed_channels = @("vpn_packet", "bulk", "control")
|
||||
ttl_seconds = 300
|
||||
metadata = @{ smoke = "c18z14_live_service_channel_active_quality_shift"; run_id = $runId }
|
||||
}).fabric_service_channel_lease
|
||||
}
|
||||
|
||||
function ConvertTo-Base64UrlJson {
|
||||
param([object]$Value)
|
||||
$json = $Value | ConvertTo-Json -Depth 100 -Compress
|
||||
$bytes = [System.Text.Encoding]::UTF8.GetBytes($json)
|
||||
return [Convert]::ToBase64String($bytes).TrimEnd("=").Replace("+", "-").Replace("/", "_")
|
||||
}
|
||||
|
||||
function ConvertTo-WebSocketURL {
|
||||
param([string]$URL)
|
||||
if ($URL.StartsWith("https://")) { return "wss://" + $URL.Substring("https://".Length) }
|
||||
if ($URL.StartsWith("http://")) { return "ws://" + $URL.Substring("http://".Length) }
|
||||
return $URL
|
||||
}
|
||||
|
||||
function New-TestIPv4UDPPacket {
|
||||
param([int]$SourcePort)
|
||||
$payload = [System.Text.Encoding]::ASCII.GetBytes("c18z14-$SourcePort")
|
||||
$totalLength = 20 + 8 + $payload.Length
|
||||
$packet = New-Object byte[] $totalLength
|
||||
$packet[0] = 0x45
|
||||
$packet[2] = [byte](($totalLength -shr 8) -band 0xff)
|
||||
$packet[3] = [byte]($totalLength -band 0xff)
|
||||
$packet[8] = 64
|
||||
$packet[9] = 17
|
||||
$packet[12] = 10; $packet[13] = 18; $packet[14] = 14; $packet[15] = 10
|
||||
$packet[16] = 10; $packet[17] = 18; $packet[18] = 14; $packet[19] = 20
|
||||
$udpOffset = 20
|
||||
$destPort = 3389
|
||||
$udpLength = 8 + $payload.Length
|
||||
$packet[$udpOffset] = [byte](($SourcePort -shr 8) -band 0xff)
|
||||
$packet[$udpOffset + 1] = [byte]($SourcePort -band 0xff)
|
||||
$packet[$udpOffset + 2] = [byte](($destPort -shr 8) -band 0xff)
|
||||
$packet[$udpOffset + 3] = [byte]($destPort -band 0xff)
|
||||
$packet[$udpOffset + 4] = [byte](($udpLength -shr 8) -band 0xff)
|
||||
$packet[$udpOffset + 5] = [byte]($udpLength -band 0xff)
|
||||
[Array]::Copy($payload, 0, $packet, 28, $payload.Length)
|
||||
return $packet
|
||||
}
|
||||
|
||||
function New-PacketBatchBody {
|
||||
param([byte[][]]$Packets)
|
||||
$stream = [System.IO.MemoryStream]::new()
|
||||
foreach ($packet in $Packets) {
|
||||
$length = $packet.Length
|
||||
$stream.WriteByte([byte](($length -shr 24) -band 0xff))
|
||||
$stream.WriteByte([byte](($length -shr 16) -band 0xff))
|
||||
$stream.WriteByte([byte](($length -shr 8) -band 0xff))
|
||||
$stream.WriteByte([byte]($length -band 0xff))
|
||||
$stream.Write($packet, 0, $packet.Length)
|
||||
}
|
||||
return $stream.ToArray()
|
||||
}
|
||||
|
||||
function Open-ServiceChannelWebSocket {
|
||||
param([object]$Lease)
|
||||
$path = $Lease.entry_http.websocket_path_template.Replace("{cluster_id}", $ClusterID).Replace("{channel_id}", $Lease.channel_id).Replace("{resource_id}", $resourceId)
|
||||
$url = ConvertTo-WebSocketURL -URL "$EntryBaseUrl$path"
|
||||
$socket = [System.Net.WebSockets.ClientWebSocket]::new()
|
||||
[void]$socket.Options.SetRequestHeader("X-RAP-Service-Channel-Token", [string]$Lease.token.token)
|
||||
[void]$socket.Options.SetRequestHeader("X-RAP-Fabric-Channel-ID", [string]$Lease.channel_id)
|
||||
[void]$socket.Options.SetRequestHeader("X-RAP-Service-Class", "vpn_packets")
|
||||
[void]$socket.Options.SetRequestHeader("X-RAP-Channel-Class", "vpn_packet")
|
||||
[void]$socket.Options.SetRequestHeader("X-RAP-Service-Channel-Authority-Payload", (ConvertTo-Base64UrlJson -Value $Lease.authority_payload))
|
||||
[void]$socket.Options.SetRequestHeader("X-RAP-Service-Channel-Authority-Signature", (ConvertTo-Base64UrlJson -Value $Lease.authority_signature))
|
||||
$cts = [System.Threading.CancellationTokenSource]::new([TimeSpan]::FromSeconds(30))
|
||||
[void]$socket.ConnectAsync([Uri]$url, $cts.Token).GetAwaiter().GetResult()
|
||||
$cts.Dispose()
|
||||
return @{
|
||||
Socket = $socket
|
||||
Url = $url
|
||||
SentBatches = 0
|
||||
SentPackets = 0
|
||||
}
|
||||
}
|
||||
|
||||
function Send-ServiceChannelWebSocketBatches {
|
||||
param([object]$Session, [int]$Count, [int]$PortBase)
|
||||
for ($batch = 0; $batch -lt $Count; $batch++) {
|
||||
$packets = @()
|
||||
for ($i = 0; $i -lt $PacketsPerBatch; $i++) {
|
||||
$packets += ,(New-TestIPv4UDPPacket -SourcePort ($PortBase + ($batch * 100) + $i))
|
||||
}
|
||||
$body = New-PacketBatchBody -Packets $packets
|
||||
$segment = [ArraySegment[byte]]::new($body)
|
||||
$cts = [System.Threading.CancellationTokenSource]::new([TimeSpan]::FromSeconds(30))
|
||||
[void]$Session.Socket.SendAsync($segment, [System.Net.WebSockets.WebSocketMessageType]::Binary, $true, $cts.Token).GetAwaiter().GetResult()
|
||||
$cts.Dispose()
|
||||
$Session.SentBatches++
|
||||
$Session.SentPackets += $packets.Count
|
||||
if ($BatchDelayMilliseconds -gt 0) { Start-Sleep -Milliseconds $BatchDelayMilliseconds }
|
||||
}
|
||||
Start-Sleep -Milliseconds 500
|
||||
}
|
||||
|
||||
function Close-ServiceChannelWebSocket {
|
||||
param([object]$Session)
|
||||
if ($null -eq $Session -or $null -eq $Session.Socket) { return }
|
||||
try {
|
||||
if ($Session.Socket.State -eq [System.Net.WebSockets.WebSocketState]::Open) {
|
||||
$cts = [System.Threading.CancellationTokenSource]::new([TimeSpan]::FromSeconds(10))
|
||||
[void]$Session.Socket.CloseOutputAsync([System.Net.WebSockets.WebSocketCloseStatus]::NormalClosure, "c18z14 sent", $cts.Token).GetAwaiter().GetResult()
|
||||
$cts.Dispose()
|
||||
}
|
||||
} finally {
|
||||
$Session.Socket.Dispose()
|
||||
}
|
||||
}
|
||||
|
||||
function Get-ExitQueueDepth {
|
||||
param([string]$NodeID, [string]$VPNConnectionID)
|
||||
$latest = Get-LatestRuntimeReport -NodeID $NodeID
|
||||
$queueKey = "$VPNConnectionID`:client_to_gateway"
|
||||
$depths = $latest.report.inbox.queue_depths
|
||||
if ($null -eq $depths) { return 0 }
|
||||
$prop = $depths.PSObject.Properties[$queueKey]
|
||||
if ($null -eq $prop) { return 0 }
|
||||
return [int]$prop.Value
|
||||
}
|
||||
|
||||
function Wait-ForExitQueueDepth {
|
||||
param([string]$NodeID, [string]$VPNConnectionID, [int]$MinDepth, [int]$TimeoutSeconds = 90)
|
||||
$deadline = (Get-Date).AddSeconds($TimeoutSeconds)
|
||||
do {
|
||||
$depth = Get-ExitQueueDepth -NodeID $NodeID -VPNConnectionID $VPNConnectionID
|
||||
if ($depth -ge $MinDepth) { return $depth }
|
||||
Start-Sleep -Seconds 2
|
||||
} while ((Get-Date) -lt $deadline)
|
||||
throw "Timed out waiting for exit queue depth >= $MinDepth on node $NodeID"
|
||||
}
|
||||
|
||||
function Get-IngressFlowDropped {
|
||||
param([string]$NodeID)
|
||||
$latest = Get-LatestRuntimeReport -NodeID $NodeID
|
||||
$flowScheduler = Get-ObjectPropertyValue -Object $latest.report.ingress -Name "flow_scheduler"
|
||||
$dropped = Get-ObjectPropertyValue -Object $flowScheduler -Name "dropped"
|
||||
if ($null -eq $dropped) { return 0 }
|
||||
return [int]$dropped
|
||||
}
|
||||
|
||||
function Get-BackendClientGatewayDepth {
|
||||
param([string]$VPNConnectionID)
|
||||
$stats = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/vpn-connections/$VPNConnectionID/tunnel/stats").vpn_packet_stats
|
||||
$queue = $stats.client_to_gateway
|
||||
if ($null -eq $queue) { return 0 }
|
||||
$depth = Get-ObjectPropertyValue -Object $queue -Name "queue_depth"
|
||||
if ($null -eq $depth) { return 0 }
|
||||
return [int]$depth
|
||||
}
|
||||
|
||||
function Invoke-RemoteDocker {
|
||||
param([string]$Command)
|
||||
& ssh $DockerSSH $Command
|
||||
if ($LASTEXITCODE -ne 0) { throw "ssh $DockerSSH command failed: $Command" }
|
||||
}
|
||||
|
||||
function Stop-TestUpdaters { Invoke-RemoteDocker -Command "docker stop rap_host_agent_updater_test-1 rap_host_agent_updater_test-2 rap_host_agent_updater_test-3 >/dev/null 2>&1 || true" }
|
||||
function Start-TestUpdaters { Invoke-RemoteDocker -Command "docker start rap_host_agent_updater_test-1 rap_host_agent_updater_test-2 rap_host_agent_updater_test-3 >/dev/null 2>&1 || true" }
|
||||
|
||||
$entryNode = Get-NodeByName -Name $EntryNodeName
|
||||
$relayNode = Get-NodeByName -Name $RelayNodeName
|
||||
$exitNode = Get-NodeByName -Name $ExitNodeName
|
||||
$slowInitialRouteID = ""
|
||||
$fastRouteID = ""
|
||||
$slowCandidateRouteID = ""
|
||||
$session = $null
|
||||
$updatersStopped = $false
|
||||
$result = $null
|
||||
|
||||
try {
|
||||
Stop-TestUpdaters
|
||||
$updatersStopped = $true
|
||||
Enable-TestMeshListener -Node $entryNode
|
||||
Enable-TestMeshListener -Node $relayNode
|
||||
Enable-TestMeshListener -Node $exitNode
|
||||
Clear-SmokeRouteIntents
|
||||
|
||||
$slowInitial = New-RouteIntent -SourceNodeID $entryNode.id -DestinationNodeID $exitNode.id -Hops @($entryNode.id, $relayNode.id, $exitNode.id) -Priority 1999999960 -Label "slow-initial"
|
||||
$fast = New-RouteIntent -SourceNodeID $entryNode.id -DestinationNodeID $exitNode.id -Hops @($entryNode.id, $exitNode.id) -Priority 1999999950 -Label "fast-live"
|
||||
$slowInitialRouteID = $slowInitial.route_intent.id
|
||||
$fastRouteID = $fast.route_intent.id
|
||||
|
||||
$visibleConfig = Wait-ForRouteIntentVisible -NodeID $entryNode.id -RouteIDs @($slowInitialRouteID, $fastRouteID)
|
||||
$exitVisibleConfig = Wait-ForRouteIntentVisible -NodeID $exitNode.id -RouteIDs @($slowInitialRouteID, $fastRouteID)
|
||||
$readyBefore = Wait-ForRuntimeReady -NodeID $entryNode.id -MinRoutes 2
|
||||
$exitReadyBefore = Wait-ForRuntimeReady -NodeID $exitNode.id -MinRoutes 0
|
||||
$loadedConfig = Wait-ForRuntimeConfigVersion -NodeID $entryNode.id -ConfigVersion $visibleConfig.synthetic_mesh_config.config_version
|
||||
$exitLoadedConfig = Wait-ForRuntimeConfigVersion -NodeID $exitNode.id -ConfigVersion $exitVisibleConfig.synthetic_mesh_config.config_version
|
||||
|
||||
$lease = New-ServiceChannelLease -EntryNodeID $entryNode.id -ExitNodeID $exitNode.id
|
||||
if ($lease.status -ne "ready" -or [string]$lease.primary_route.route_id -ne $slowInitialRouteID) {
|
||||
throw "Initial lease should select slow route '$slowInitialRouteID': status=$($lease.status) route=$($lease.primary_route.route_id)"
|
||||
}
|
||||
|
||||
$baselineExitDepth = Get-ExitQueueDepth -NodeID $exitNode.id -VPNConnectionID $resourceId
|
||||
$baselineBackendDepth = Get-BackendClientGatewayDepth -VPNConnectionID $resourceId
|
||||
$baselineDropped = Get-IngressFlowDropped -NodeID $entryNode.id
|
||||
$session = Open-ServiceChannelWebSocket -Lease $lease
|
||||
|
||||
Send-ServiceChannelWebSocketBatches -Session $session -Count $InitialBatchCount -PortBase 62000
|
||||
$initialPackets = $InitialBatchCount * $PacketsPerBatch
|
||||
$initialExitDepth = Wait-ForExitQueueDepth -NodeID $exitNode.id -VPNConnectionID $resourceId -MinDepth ($baselineExitDepth + $initialPackets)
|
||||
$initialRuntime = Get-LatestRuntimeReport -NodeID $entryNode.id
|
||||
if ([string]$initialRuntime.report.ingress.last_selected_route_id -ne $slowInitialRouteID) {
|
||||
throw "Initial active websocket route was '$($initialRuntime.report.ingress.last_selected_route_id)', want slow route '$slowInitialRouteID'"
|
||||
}
|
||||
|
||||
$expiredSlowInitial = Invoke-Api -Method POST -Path "/clusters/$ClusterID/mesh/route-intents/$slowInitialRouteID/expire" -Body @{ actor_user_id = $ActorUserID }
|
||||
$notVisibleConfig = Wait-ForRouteIntentNotVisible -NodeID $entryNode.id -RouteID $slowInitialRouteID
|
||||
$fastOnlyRuntime = Wait-ForRuntimeConfigVersion -NodeID $entryNode.id -ConfigVersion $notVisibleConfig.synthetic_mesh_config.config_version
|
||||
|
||||
Send-ServiceChannelWebSocketBatches -Session $session -Count $LearningBatchCount -PortBase 64000
|
||||
$learningPackets = $LearningBatchCount * $PacketsPerBatch
|
||||
$learnedExitDepth = Wait-ForExitQueueDepth -NodeID $exitNode.id -VPNConnectionID $resourceId -MinDepth ($baselineExitDepth + $initialPackets + $learningPackets) -TimeoutSeconds 120
|
||||
$fastFeedback = Wait-ForRouteFeedback -ReporterNodeID $entryNode.id -RouteID $fastRouteID -TimeoutSeconds 120
|
||||
|
||||
$slowCandidate = New-RouteIntent -SourceNodeID $entryNode.id -DestinationNodeID $exitNode.id -Hops @($entryNode.id, $relayNode.id, $exitNode.id) -Priority 1999999960 -Label "slow-candidate"
|
||||
$slowCandidateRouteID = $slowCandidate.route_intent.id
|
||||
$candidateVisibleConfig = Wait-ForRouteIntentVisible -NodeID $entryNode.id -RouteIDs @($slowCandidateRouteID, $fastRouteID)
|
||||
$candidateLoadedConfig = Wait-ForRuntimeConfigVersion -NodeID $entryNode.id -ConfigVersion $candidateVisibleConfig.synthetic_mesh_config.config_version
|
||||
$qualityRuntime = Wait-ForQualityPreferenceApplied -NodeID $entryNode.id -TimeoutSeconds 120
|
||||
|
||||
Send-ServiceChannelWebSocketBatches -Session $session -Count $PostChurnBatchCount -PortBase 67000
|
||||
$postChurnPackets = $PostChurnBatchCount * $PacketsPerBatch
|
||||
$finalExitDepth = Wait-ForExitQueueDepth -NodeID $exitNode.id -VPNConnectionID $resourceId -MinDepth ($baselineExitDepth + $initialPackets + $learningPackets + $postChurnPackets) -TimeoutSeconds 120
|
||||
$finalRuntime = Get-LatestRuntimeReport -NodeID $entryNode.id
|
||||
if ([string]$finalRuntime.report.ingress.last_selected_route_id -ne $fastRouteID) {
|
||||
throw "Post-churn active websocket route was '$($finalRuntime.report.ingress.last_selected_route_id)', want learned fast route '$fastRouteID'"
|
||||
}
|
||||
|
||||
$sessionURL = $session.Url
|
||||
$sessionSentBatches = $session.SentBatches
|
||||
$sessionSentPackets = $session.SentPackets
|
||||
Close-ServiceChannelWebSocket -Session $session
|
||||
$session = $null
|
||||
|
||||
$finalDropped = Get-IngressFlowDropped -NodeID $entryNode.id
|
||||
$finalBackendDepth = Get-BackendClientGatewayDepth -VPNConnectionID $resourceId
|
||||
$expiredFastFeedback = Invoke-Api -Method POST -Path "/clusters/$ClusterID/fabric/service-channels/route-feedback/expire" -Body @{
|
||||
actor_user_id = $ActorUserID
|
||||
reporter_node_id = $entryNode.id
|
||||
route_id = $fastRouteID
|
||||
service_class = "vpn_packets"
|
||||
reason = "c18z14 active quality shift smoke cleanup"
|
||||
}
|
||||
$expiredSlowCandidate = Invoke-Api -Method POST -Path "/clusters/$ClusterID/mesh/route-intents/$slowCandidateRouteID/expire" -Body @{ actor_user_id = $ActorUserID }
|
||||
$expiredFast = Invoke-Api -Method POST -Path "/clusters/$ClusterID/mesh/route-intents/$fastRouteID/expire" -Body @{ actor_user_id = $ActorUserID }
|
||||
|
||||
$result = [ordered]@{
|
||||
schema_version = "c18z14.live_service_channel_active_quality_shift_smoke.v1"
|
||||
run_id = $runId
|
||||
base_url = $ApiBaseUrl
|
||||
entry_base_url = $EntryBaseUrl
|
||||
cluster_id = $ClusterID
|
||||
entry_node = @{ name = $entryNode.name; id = $entryNode.id }
|
||||
relay_node = @{ name = $relayNode.name; id = $relayNode.id }
|
||||
exit_node = @{ name = $exitNode.name; id = $exitNode.id }
|
||||
resource_id = $resourceId
|
||||
route_intents = @{
|
||||
slow_initial_route_id = $slowInitialRouteID
|
||||
fast_route_id = $fastRouteID
|
||||
slow_candidate_route_id = $slowCandidateRouteID
|
||||
slow_initial_status = $expiredSlowInitial.route_intent.lifecycle_status
|
||||
slow_candidate_status = $expiredSlowCandidate.route_intent.lifecycle_status
|
||||
fast_status = $expiredFast.route_intent.lifecycle_status
|
||||
}
|
||||
lease = @{
|
||||
status = $lease.status
|
||||
channel_id = $lease.channel_id
|
||||
primary_route_id = $lease.primary_route.route_id
|
||||
}
|
||||
active_websocket = @{
|
||||
url = $sessionURL
|
||||
initial_batches = $InitialBatchCount
|
||||
learning_batches = $LearningBatchCount
|
||||
post_churn_batches = $PostChurnBatchCount
|
||||
packets_per_batch = $PacketsPerBatch
|
||||
sent_batches = $sessionSentBatches
|
||||
sent_packets = $sessionSentPackets
|
||||
initial_route_id = $initialRuntime.report.ingress.last_selected_route_id
|
||||
final_route_id = $finalRuntime.report.ingress.last_selected_route_id
|
||||
}
|
||||
live_feedback = @{
|
||||
route_id = $fastFeedback.route_id
|
||||
feedback_status = $fastFeedback.feedback_status
|
||||
score_adjustment = $fastFeedback.score_adjustment
|
||||
reasons = $fastFeedback.reasons
|
||||
last_send_duration_ms = $fastFeedback.last_send_duration_ms
|
||||
expire_result = $expiredFastFeedback.route_feedback_expire
|
||||
}
|
||||
exit_queue = @{
|
||||
baseline_depth = $baselineExitDepth
|
||||
initial_depth = $initialExitDepth
|
||||
learned_depth = $learnedExitDepth
|
||||
final_depth = $finalExitDepth
|
||||
}
|
||||
backend_fallback_queue = @{ baseline_depth = $baselineBackendDepth; depth = $finalBackendDepth }
|
||||
flow_drops = @{ baseline = $baselineDropped; final = $finalDropped; delta = ($finalDropped - $baselineDropped) }
|
||||
passed = $true
|
||||
checks = [ordered]@{
|
||||
production_forwarding_ready = ($readyBefore.report.production_payload_forwarding -eq $true)
|
||||
exit_production_forwarding_ready = ($exitReadyBefore.report.production_payload_forwarding -eq $true)
|
||||
entry_runtime_loaded_visible_config = ([string]$loadedConfig.report.config_version -ge [string]$visibleConfig.synthetic_mesh_config.config_version)
|
||||
exit_runtime_loaded_visible_config = ([string]$exitLoadedConfig.report.config_version -ge [string]$exitVisibleConfig.synthetic_mesh_config.config_version)
|
||||
initial_lease_selected_slow_route = ([string]$lease.primary_route.route_id -eq $slowInitialRouteID)
|
||||
active_websocket_started_on_slow_route = ([string]$initialRuntime.report.ingress.last_selected_route_id -eq $slowInitialRouteID)
|
||||
fast_only_runtime_loaded_after_slow_expire = ([string]$fastOnlyRuntime.report.config_version -ge [string]$notVisibleConfig.synthetic_mesh_config.config_version)
|
||||
backend_persisted_live_fast_feedback = ([string]$fastFeedback.route_id -eq $fastRouteID -and [string]$fastFeedback.feedback_status -eq "healthy")
|
||||
candidate_runtime_loaded = ([string]$candidateLoadedConfig.report.config_version -ge [string]$candidateVisibleConfig.synthetic_mesh_config.config_version)
|
||||
node_applied_quality_preference = ([int]$qualityRuntime.report.ingress.route_quality_preference_count -gt 0)
|
||||
active_websocket_stayed_on_learned_fast_route_after_churn = ([string]$finalRuntime.report.ingress.last_selected_route_id -eq $fastRouteID)
|
||||
all_packets_reached_exit = ($finalExitDepth -ge ($baselineExitDepth + $initialPackets + $learningPackets + $postChurnPackets))
|
||||
no_backend_fallback_used = ($finalBackendDepth -eq $baselineBackendDepth)
|
||||
no_flow_drops = (($finalDropped - $baselineDropped) -eq 0)
|
||||
route_intents_expired = ($expiredFast.route_intent.lifecycle_status -eq "expired" -and $expiredSlowCandidate.route_intent.lifecycle_status -eq "expired")
|
||||
}
|
||||
telemetry = @{
|
||||
initial_entry_ingress = $initialRuntime.report.ingress
|
||||
quality_entry_ingress = $qualityRuntime.report.ingress
|
||||
final_entry_ingress = $finalRuntime.report.ingress
|
||||
}
|
||||
}
|
||||
$failedChecks = @($result.checks.GetEnumerator() | Where-Object { $_.Value -ne $true })
|
||||
if ($failedChecks.Count -gt 0) { throw "C18Z14 failed checks: $($failedChecks.Name -join ', ')" }
|
||||
}
|
||||
finally {
|
||||
if ($session) { try { Close-ServiceChannelWebSocket -Session $session } catch {} }
|
||||
if ($slowInitialRouteID) { try { Invoke-Api -Method POST -Path "/clusters/$ClusterID/mesh/route-intents/$slowInitialRouteID/expire" -Body @{ actor_user_id = $ActorUserID } | Out-Null } catch {} }
|
||||
if ($slowCandidateRouteID) { try { Invoke-Api -Method POST -Path "/clusters/$ClusterID/mesh/route-intents/$slowCandidateRouteID/expire" -Body @{ actor_user_id = $ActorUserID } | Out-Null } catch {} }
|
||||
if ($fastRouteID) {
|
||||
try { Invoke-Api -Method POST -Path "/clusters/$ClusterID/mesh/route-intents/$fastRouteID/expire" -Body @{ actor_user_id = $ActorUserID } | Out-Null } catch {}
|
||||
try {
|
||||
Invoke-Api -Method POST -Path "/clusters/$ClusterID/fabric/service-channels/route-feedback/expire" -Body @{
|
||||
actor_user_id = $ActorUserID
|
||||
reporter_node_id = $entryNode.id
|
||||
route_id = $fastRouteID
|
||||
service_class = "vpn_packets"
|
||||
reason = "c18z14 active quality shift smoke cleanup"
|
||||
} | Out-Null
|
||||
} catch {}
|
||||
}
|
||||
if ($updatersStopped) { try { Start-TestUpdaters } catch { Write-Warning "Could not restart test updaters: $($_.Exception.Message)" } }
|
||||
}
|
||||
|
||||
$resultFullPath = Join-Path $repoRoot $ResultPath
|
||||
$resultDir = Split-Path -Parent $resultFullPath
|
||||
if (-not (Test-Path $resultDir)) { New-Item -ItemType Directory -Path $resultDir | Out-Null }
|
||||
$result | ConvertTo-Json -Depth 100 | Set-Content -Path $resultFullPath -Encoding UTF8
|
||||
Write-Host "C18Z14 live service-channel active quality shift smoke passed. Result: $resultFullPath"
|
||||
$result
|
||||
@@ -0,0 +1,72 @@
|
||||
param(
|
||||
[string]$ApiBaseUrl = "http://192.168.200.61:18121/api/v1",
|
||||
[string]$ClusterID = "cfc0743d-d960-49fb-9de8-96e063d5e4aa",
|
||||
[string]$ActorUserID = "f67d943f-5397-4b3a-a229-695fe67ad700",
|
||||
[string]$EntryNodeName = "test-1",
|
||||
[string]$RelayNodeName = "test-3",
|
||||
[string]$ExitNodeName = "test-2",
|
||||
[string]$EntryBaseUrl = "http://192.168.200.61:19131",
|
||||
[string]$DockerSSH = "test-docker",
|
||||
[int]$InitialBatchCount = 12,
|
||||
[int]$LearningBatchCount = 24,
|
||||
[int]$PostChurnBatchCount = 24,
|
||||
[int]$PacketsPerBatch = 8,
|
||||
[int]$BatchDelayMilliseconds = 25,
|
||||
[string]$ResultPath = "artifacts\c18z15-live-service-channel-effective-quality-smoke-result.json"
|
||||
)
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
$scriptDir = Split-Path -Parent $MyInvocation.MyCommand.Path
|
||||
$repoRoot = (Resolve-Path (Join-Path $scriptDir "..\..")).ProviderPath
|
||||
$innerResultPath = "artifacts\c18z15-live-service-channel-effective-quality-inner-result.json"
|
||||
$innerResultFullPath = Join-Path $repoRoot $innerResultPath
|
||||
|
||||
ssh $DockerSSH "docker restart rap_test_node_test_1 rap_test_node_test_2 rap_test_node_test_3 >/dev/null && sleep 5" | Out-Null
|
||||
|
||||
& (Join-Path $scriptDir "c18z14-live-service-channel-active-quality-shift-smoke.ps1") `
|
||||
-ApiBaseUrl $ApiBaseUrl `
|
||||
-ClusterID $ClusterID `
|
||||
-ActorUserID $ActorUserID `
|
||||
-EntryNodeName $EntryNodeName `
|
||||
-RelayNodeName $RelayNodeName `
|
||||
-ExitNodeName $ExitNodeName `
|
||||
-EntryBaseUrl $EntryBaseUrl `
|
||||
-DockerSSH $DockerSSH `
|
||||
-InitialBatchCount $InitialBatchCount `
|
||||
-LearningBatchCount $LearningBatchCount `
|
||||
-PostChurnBatchCount $PostChurnBatchCount `
|
||||
-PacketsPerBatch $PacketsPerBatch `
|
||||
-BatchDelayMilliseconds $BatchDelayMilliseconds `
|
||||
-ResultPath $innerResultPath
|
||||
|
||||
$result = Get-Content -Path $innerResultFullPath -Raw | ConvertFrom-Json
|
||||
$qualityIngress = $result.telemetry.quality_entry_ingress
|
||||
$preferences = @($qualityIngress.route_quality_preferences)
|
||||
$decayedPreferences = @($preferences | Where-Object {
|
||||
$_.raw_score_adjustment -gt $_.score_adjustment -and
|
||||
@($_.reasons) -contains "service_channel_feedback_age_decay"
|
||||
})
|
||||
|
||||
$result.schema_version = "c18z15.live_service_channel_effective_quality_smoke.v1"
|
||||
$result | Add-Member -NotePropertyName c18z15_checks -NotePropertyValue ([ordered]@{
|
||||
route_quality_preferences_exposed = ($preferences.Count -gt 0)
|
||||
effective_score_visible = (@($preferences | Where-Object { $_.raw_score_adjustment -gt 0 -and $_.score_adjustment -gt 0 }).Count -gt 0)
|
||||
decayed_effective_score_visible = ($decayedPreferences.Count -gt 0)
|
||||
}) -Force
|
||||
$result.passed = [bool]($result.passed -and
|
||||
$result.c18z15_checks.route_quality_preferences_exposed -and
|
||||
$result.c18z15_checks.effective_score_visible -and
|
||||
$result.c18z15_checks.decayed_effective_score_visible)
|
||||
|
||||
$resolvedResultPath = Join-Path $repoRoot $ResultPath
|
||||
$result | ConvertTo-Json -Depth 100 | Set-Content -Path $resolvedResultPath -Encoding UTF8
|
||||
Remove-Item -Path $innerResultFullPath -Force -ErrorAction SilentlyContinue
|
||||
|
||||
if (-not $result.passed) {
|
||||
throw "C18Z15 effective route-quality smoke failed. Result: $resolvedResultPath"
|
||||
}
|
||||
|
||||
Write-Host "C18Z15 live service-channel effective quality smoke passed. Result: $resolvedResultPath"
|
||||
$result
|
||||
@@ -0,0 +1,97 @@
|
||||
param(
|
||||
[string]$ApiBaseUrl = "http://192.168.200.61:18121/api/v1",
|
||||
[string]$ClusterID = "cfc0743d-d960-49fb-9de8-96e063d5e4aa",
|
||||
[string]$ActorUserID = "f67d943f-5397-4b3a-a229-695fe67ad700",
|
||||
[string]$EntryNodeName = "test-1",
|
||||
[string]$RelayNodeName = "test-3",
|
||||
[string]$ExitNodeName = "test-2",
|
||||
[string]$EntryBaseUrl = "http://192.168.200.61:19131",
|
||||
[string]$DockerSSH = "test-docker",
|
||||
[int]$InitialBatchCount = 12,
|
||||
[int]$LearningBatchCount = 24,
|
||||
[int]$PostChurnBatchCount = 24,
|
||||
[int]$PacketsPerBatch = 8,
|
||||
[int]$BatchDelayMilliseconds = 25,
|
||||
[string]$ResultPath = "artifacts\c18z16-live-service-channel-quality-fairness-smoke-result.json"
|
||||
)
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
$scriptDir = Split-Path -Parent $MyInvocation.MyCommand.Path
|
||||
$repoRoot = (Resolve-Path (Join-Path $scriptDir "..\..")).ProviderPath
|
||||
$innerResultPath = "artifacts\c18z16-live-service-channel-quality-fairness-inner-result.json"
|
||||
$innerResultFullPath = Join-Path $repoRoot $innerResultPath
|
||||
|
||||
& (Join-Path $scriptDir "c18z15-live-service-channel-effective-quality-smoke.ps1") `
|
||||
-ApiBaseUrl $ApiBaseUrl `
|
||||
-ClusterID $ClusterID `
|
||||
-ActorUserID $ActorUserID `
|
||||
-EntryNodeName $EntryNodeName `
|
||||
-RelayNodeName $RelayNodeName `
|
||||
-ExitNodeName $ExitNodeName `
|
||||
-EntryBaseUrl $EntryBaseUrl `
|
||||
-DockerSSH $DockerSSH `
|
||||
-InitialBatchCount $InitialBatchCount `
|
||||
-LearningBatchCount $LearningBatchCount `
|
||||
-PostChurnBatchCount $PostChurnBatchCount `
|
||||
-PacketsPerBatch $PacketsPerBatch `
|
||||
-BatchDelayMilliseconds $BatchDelayMilliseconds `
|
||||
-ResultPath $innerResultPath
|
||||
|
||||
$result = Get-Content -Path $innerResultFullPath -Raw | ConvertFrom-Json
|
||||
$qualityIngress = $result.telemetry.final_entry_ingress
|
||||
$channelStats = $qualityIngress.flow_scheduler.channel_stats
|
||||
$statItems = @()
|
||||
if ($null -ne $channelStats) {
|
||||
$statItems = @($channelStats.PSObject.Properties | ForEach-Object {
|
||||
$qualityRouteID = ""
|
||||
$qualityScore = 0
|
||||
if ($_.Value.PSObject.Properties["quality_preference_route_id"]) { $qualityRouteID = [string]$_.Value.quality_preference_route_id }
|
||||
if ($_.Value.PSObject.Properties["quality_preference_score"]) { $qualityScore = [int]$_.Value.quality_preference_score }
|
||||
[pscustomobject]@{
|
||||
channel = $_.Name
|
||||
served = [int64]($_.Value.served)
|
||||
dropped = [int64]($_.Value.dropped)
|
||||
last_route_id = [string]($_.Value.last_route_id)
|
||||
quality_preference_route_id = $qualityRouteID
|
||||
quality_preference_score = $qualityScore
|
||||
}
|
||||
})
|
||||
}
|
||||
$qualityChannels = @($statItems | Where-Object {
|
||||
$_.served -gt 0 -and
|
||||
$_.dropped -eq 0 -and
|
||||
$_.quality_preference_route_id -eq [string]$result.active_websocket.final_route_id -and
|
||||
$_.quality_preference_score -gt 0
|
||||
})
|
||||
$servedChannels = @($statItems | Where-Object { $_.served -gt 0 })
|
||||
|
||||
$result.schema_version = "c18z16.live_service_channel_quality_fairness_smoke.v1"
|
||||
$result | Add-Member -NotePropertyName c18z16_checks -NotePropertyValue ([ordered]@{
|
||||
multi_channel_traffic_observed = ($servedChannels.Count -ge 2)
|
||||
quality_preference_applied_per_channel = ($qualityChannels.Count -ge 2)
|
||||
no_channel_drops_under_quality_preference = (@($statItems | Where-Object { $_.dropped -gt 0 }).Count -eq 0)
|
||||
active_session_stayed_on_quality_route = ([string]$result.active_websocket.final_route_id -eq [string]$result.live_feedback.route_id)
|
||||
}) -Force
|
||||
$result | Add-Member -NotePropertyName c18z16_summary -NotePropertyValue ([ordered]@{
|
||||
served_channel_count = $servedChannels.Count
|
||||
quality_preference_channel_count = $qualityChannels.Count
|
||||
final_quality_route_id = [string]$result.active_websocket.final_route_id
|
||||
}) -Force
|
||||
$result.passed = [bool]($result.passed -and
|
||||
$result.c18z16_checks.multi_channel_traffic_observed -and
|
||||
$result.c18z16_checks.quality_preference_applied_per_channel -and
|
||||
$result.c18z16_checks.no_channel_drops_under_quality_preference -and
|
||||
$result.c18z16_checks.active_session_stayed_on_quality_route)
|
||||
|
||||
$resolvedResultPath = Join-Path $repoRoot $ResultPath
|
||||
$result | ConvertTo-Json -Depth 100 | Set-Content -Path $resolvedResultPath -Encoding UTF8
|
||||
Remove-Item -Path $innerResultFullPath -Force -ErrorAction SilentlyContinue
|
||||
|
||||
if (-not $result.passed) {
|
||||
throw "C18Z16 route-quality fairness smoke failed. Result: $resolvedResultPath"
|
||||
}
|
||||
|
||||
Write-Host "C18Z16 live service-channel quality fairness smoke passed. Result: $resolvedResultPath"
|
||||
$result
|
||||
@@ -0,0 +1,82 @@
|
||||
param(
|
||||
[string]$ApiBaseUrl = "http://192.168.200.61:18121/api/v1",
|
||||
[string]$ClusterID = "cfc0743d-d960-49fb-9de8-96e063d5e4aa",
|
||||
[string]$ActorUserID = "f67d943f-5397-4b3a-a229-695fe67ad700",
|
||||
[string]$EntryNodeName = "test-1",
|
||||
[string]$RelayNodeName = "test-3",
|
||||
[string]$ExitNodeName = "test-2",
|
||||
[string]$EntryBaseUrl = "http://192.168.200.61:19131",
|
||||
[string]$DockerSSH = "test-docker",
|
||||
[int]$InitialBatchCount = 12,
|
||||
[int]$LearningBatchCount = 24,
|
||||
[int]$PostChurnBatchCount = 24,
|
||||
[int]$PacketsPerBatch = 8,
|
||||
[int]$BatchDelayMilliseconds = 25,
|
||||
[string]$ResultPath = "artifacts\c18z17-live-service-channel-quality-cleanup-smoke-result.json"
|
||||
)
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
$scriptDir = Split-Path -Parent $MyInvocation.MyCommand.Path
|
||||
$repoRoot = (Resolve-Path (Join-Path $scriptDir "..\..")).ProviderPath
|
||||
$innerResultPath = "artifacts\c18z17-live-service-channel-quality-cleanup-inner-result.json"
|
||||
$innerResultFullPath = Join-Path $repoRoot $innerResultPath
|
||||
|
||||
& (Join-Path $scriptDir "c18z16-live-service-channel-quality-fairness-smoke.ps1") `
|
||||
-ApiBaseUrl $ApiBaseUrl `
|
||||
-ClusterID $ClusterID `
|
||||
-ActorUserID $ActorUserID `
|
||||
-EntryNodeName $EntryNodeName `
|
||||
-RelayNodeName $RelayNodeName `
|
||||
-ExitNodeName $ExitNodeName `
|
||||
-EntryBaseUrl $EntryBaseUrl `
|
||||
-DockerSSH $DockerSSH `
|
||||
-InitialBatchCount $InitialBatchCount `
|
||||
-LearningBatchCount $LearningBatchCount `
|
||||
-PostChurnBatchCount $PostChurnBatchCount `
|
||||
-PacketsPerBatch $PacketsPerBatch `
|
||||
-BatchDelayMilliseconds $BatchDelayMilliseconds `
|
||||
-ResultPath $innerResultPath
|
||||
|
||||
$result = Get-Content -Path $innerResultFullPath -Raw | ConvertFrom-Json
|
||||
$finalIngress = $result.telemetry.final_entry_ingress
|
||||
$stats = @($finalIngress.flow_scheduler.channel_stats.PSObject.Properties | ForEach-Object { $_.Value })
|
||||
$activeMarkers = @($stats | Where-Object {
|
||||
$_.PSObject.Properties["quality_preference_route_id"] -and
|
||||
-not [string]::IsNullOrWhiteSpace([string]$_.quality_preference_route_id)
|
||||
})
|
||||
$validPreferenceIDs = @{}
|
||||
foreach ($preference in @($finalIngress.route_quality_preferences)) {
|
||||
$validPreferenceIDs[[string]$preference.route_id] = $true
|
||||
}
|
||||
$staleMarkers = @($activeMarkers | Where-Object { -not $validPreferenceIDs.ContainsKey([string]$_.quality_preference_route_id) })
|
||||
|
||||
$result.schema_version = "c18z17.live_service_channel_quality_cleanup_smoke.v1"
|
||||
$result | Add-Member -NotePropertyName c18z17_checks -NotePropertyValue ([ordered]@{
|
||||
stale_channel_quality_markers_absent = ($staleMarkers.Count -eq 0)
|
||||
active_markers_reference_visible_preferences = ($activeMarkers.Count -gt 0 -and $staleMarkers.Count -eq 0)
|
||||
expired_route_intents_not_active = ([string]$result.route_intents.fast_status -eq "expired" -and [string]$result.route_intents.slow_candidate_status -eq "expired" -and [string]$result.route_intents.slow_initial_status -eq "expired")
|
||||
session_completed_without_backend_fallback = ([int]$result.backend_fallback_queue.depth -eq 0)
|
||||
}) -Force
|
||||
$result | Add-Member -NotePropertyName c18z17_summary -NotePropertyValue ([ordered]@{
|
||||
active_quality_marker_count = $activeMarkers.Count
|
||||
stale_quality_marker_count = $staleMarkers.Count
|
||||
visible_quality_preference_count = @($finalIngress.route_quality_preferences).Count
|
||||
}) -Force
|
||||
$result.passed = [bool]($result.passed -and
|
||||
$result.c18z17_checks.stale_channel_quality_markers_absent -and
|
||||
$result.c18z17_checks.active_markers_reference_visible_preferences -and
|
||||
$result.c18z17_checks.expired_route_intents_not_active -and
|
||||
$result.c18z17_checks.session_completed_without_backend_fallback)
|
||||
|
||||
$resolvedResultPath = Join-Path $repoRoot $ResultPath
|
||||
$result | ConvertTo-Json -Depth 100 | Set-Content -Path $resolvedResultPath -Encoding UTF8
|
||||
Remove-Item -Path $innerResultFullPath -Force -ErrorAction SilentlyContinue
|
||||
|
||||
if (-not $result.passed) {
|
||||
throw "C18Z17 route-quality cleanup smoke failed. Result: $resolvedResultPath"
|
||||
}
|
||||
|
||||
Write-Host "C18Z17 live service-channel quality cleanup smoke passed. Result: $resolvedResultPath"
|
||||
$result
|
||||
@@ -0,0 +1,108 @@
|
||||
param(
|
||||
[string]$ApiBaseUrl = "http://192.168.200.61:18121/api/v1",
|
||||
[string]$ClusterID = "cfc0743d-d960-49fb-9de8-96e063d5e4aa",
|
||||
[string]$ActorUserID = "f67d943f-5397-4b3a-a229-695fe67ad700",
|
||||
[string]$EntryNodeName = "test-1",
|
||||
[string]$RelayNodeName = "test-3",
|
||||
[string]$ExitNodeName = "test-2",
|
||||
[string]$EntryBaseUrl = "http://192.168.200.61:19131",
|
||||
[string]$DockerSSH = "test-docker",
|
||||
[int]$InitialBatchCount = 12,
|
||||
[int]$LearningBatchCount = 24,
|
||||
[int]$PostChurnBatchCount = 24,
|
||||
[int]$PacketsPerBatch = 8,
|
||||
[int]$BatchDelayMilliseconds = 25,
|
||||
[string]$ResultPath = "artifacts\c18z18-service-channel-session-scoped-fairness-smoke-result.json"
|
||||
)
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
$scriptDir = Split-Path -Parent $MyInvocation.MyCommand.Path
|
||||
$repoRoot = (Resolve-Path (Join-Path $scriptDir "..\..")).ProviderPath
|
||||
$innerResultPath = "artifacts\c18z18-service-channel-session-scoped-fairness-inner-result.json"
|
||||
$innerResultFullPath = Join-Path $repoRoot $innerResultPath
|
||||
|
||||
$agentRoot = Join-Path $repoRoot "agents\rap-node-agent"
|
||||
Push-Location $agentRoot
|
||||
try {
|
||||
$unitTestOutput = & go test ./internal/vpnruntime -run "TestFabricClientPacketIngressIsolatesRouteMemoryPerVPNConnection|TestFabricClientPacketIngressQualityPreferencePreservesMultiChannelFairness" 2>&1
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
$unitText = ($unitTestOutput | Out-String)
|
||||
throw "C18Z18 unit isolation/fairness tests failed:`n$unitText"
|
||||
}
|
||||
} finally {
|
||||
Pop-Location
|
||||
}
|
||||
|
||||
& (Join-Path $scriptDir "c18z17-live-service-channel-quality-cleanup-smoke.ps1") `
|
||||
-ApiBaseUrl $ApiBaseUrl `
|
||||
-ClusterID $ClusterID `
|
||||
-ActorUserID $ActorUserID `
|
||||
-EntryNodeName $EntryNodeName `
|
||||
-RelayNodeName $RelayNodeName `
|
||||
-ExitNodeName $ExitNodeName `
|
||||
-EntryBaseUrl $EntryBaseUrl `
|
||||
-DockerSSH $DockerSSH `
|
||||
-InitialBatchCount $InitialBatchCount `
|
||||
-LearningBatchCount $LearningBatchCount `
|
||||
-PostChurnBatchCount $PostChurnBatchCount `
|
||||
-PacketsPerBatch $PacketsPerBatch `
|
||||
-BatchDelayMilliseconds $BatchDelayMilliseconds `
|
||||
-ResultPath $innerResultPath
|
||||
|
||||
$result = Get-Content -Path $innerResultFullPath -Raw | ConvertFrom-Json
|
||||
$resourceID = [string]$result.resource_id
|
||||
$finalIngress = $result.telemetry.final_entry_ingress
|
||||
$stats = @()
|
||||
if ($null -ne $finalIngress.flow_scheduler.channel_stats) {
|
||||
$stats = @($finalIngress.flow_scheduler.channel_stats.PSObject.Properties | ForEach-Object {
|
||||
[pscustomobject]@{
|
||||
channel = $_.Name
|
||||
served = [int64]$_.Value.served
|
||||
dropped = [int64]$_.Value.dropped
|
||||
quality_preference_route_id = if ($_.Value.PSObject.Properties["quality_preference_route_id"]) { [string]$_.Value.quality_preference_route_id } else { "" }
|
||||
}
|
||||
})
|
||||
}
|
||||
$servedStats = @($stats | Where-Object { $_.served -gt 0 })
|
||||
$sessionPrefix = "vpn:$resourceID`:flow-"
|
||||
$sessionScopedStats = @($servedStats | Where-Object { $_.channel.StartsWith($sessionPrefix) })
|
||||
$unscopedServedStats = @($servedStats | Where-Object { $_.channel -match "^flow-[0-9][0-9]$" })
|
||||
$sessionScopedQualityStats = @($sessionScopedStats | Where-Object { -not [string]::IsNullOrWhiteSpace($_.quality_preference_route_id) })
|
||||
|
||||
$result.schema_version = "c18z18.service_channel_session_scoped_fairness_smoke.v1"
|
||||
$result | Add-Member -NotePropertyName c18z18_checks -NotePropertyValue ([ordered]@{
|
||||
unit_multi_session_route_memory_isolated = ($unitTestOutput -join "`n").Contains("ok")
|
||||
live_served_channels_are_session_scoped = ($servedStats.Count -gt 0 -and $sessionScopedStats.Count -eq $servedStats.Count)
|
||||
live_unscoped_served_channels_absent = ($unscopedServedStats.Count -eq 0)
|
||||
live_quality_markers_are_session_scoped = ($sessionScopedQualityStats.Count -ge 2)
|
||||
live_session_completed_without_backend_fallback = ([int]$result.backend_fallback_queue.depth -eq 0)
|
||||
live_session_completed_without_flow_drops = ([int]$result.flow_drops.delta -eq 0)
|
||||
}) -Force
|
||||
$result | Add-Member -NotePropertyName c18z18_summary -NotePropertyValue ([ordered]@{
|
||||
unit_test_output = ($unitTestOutput | Out-String).Trim()
|
||||
served_channel_count = $servedStats.Count
|
||||
session_scoped_served_channel_count = $sessionScopedStats.Count
|
||||
session_scoped_quality_channel_count = $sessionScopedQualityStats.Count
|
||||
unscoped_served_channel_count = $unscopedServedStats.Count
|
||||
session_prefix = $sessionPrefix
|
||||
}) -Force
|
||||
$result.passed = [bool]($result.passed -and
|
||||
$result.c18z18_checks.unit_multi_session_route_memory_isolated -and
|
||||
$result.c18z18_checks.live_served_channels_are_session_scoped -and
|
||||
$result.c18z18_checks.live_unscoped_served_channels_absent -and
|
||||
$result.c18z18_checks.live_quality_markers_are_session_scoped -and
|
||||
$result.c18z18_checks.live_session_completed_without_backend_fallback -and
|
||||
$result.c18z18_checks.live_session_completed_without_flow_drops)
|
||||
|
||||
$resolvedResultPath = Join-Path $repoRoot $ResultPath
|
||||
$result | ConvertTo-Json -Depth 100 | Set-Content -Path $resolvedResultPath -Encoding UTF8
|
||||
Remove-Item -Path $innerResultFullPath -Force -ErrorAction SilentlyContinue
|
||||
|
||||
if (-not $result.passed) {
|
||||
throw "C18Z18 session-scoped fairness smoke failed. Result: $resolvedResultPath"
|
||||
}
|
||||
|
||||
Write-Host "C18Z18 service-channel session-scoped fairness smoke passed. Result: $resolvedResultPath"
|
||||
$result
|
||||
@@ -0,0 +1,95 @@
|
||||
param(
|
||||
[string]$ApiBaseUrl = "http://192.168.200.61:18121/api/v1",
|
||||
[string]$ClusterID = "cfc0743d-d960-49fb-9de8-96e063d5e4aa",
|
||||
[string]$ActorUserID = "f67d943f-5397-4b3a-a229-695fe67ad700",
|
||||
[string]$EntryNodeName = "test-1",
|
||||
[string]$RelayNodeName = "test-3",
|
||||
[string]$ExitNodeName = "test-2",
|
||||
[string]$EntryBaseUrl = "http://192.168.200.61:19131",
|
||||
[string]$DockerSSH = "test-docker",
|
||||
[int]$InitialBatchCount = 12,
|
||||
[int]$LearningBatchCount = 24,
|
||||
[int]$PostChurnBatchCount = 24,
|
||||
[int]$PacketsPerBatch = 8,
|
||||
[int]$BatchDelayMilliseconds = 25,
|
||||
[string]$ResultPath = "artifacts\c18z19-service-channel-parallel-flow-window-smoke-result.json"
|
||||
)
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
$scriptDir = Split-Path -Parent $MyInvocation.MyCommand.Path
|
||||
$repoRoot = (Resolve-Path (Join-Path $scriptDir "..\..")).ProviderPath
|
||||
$innerResultPath = "artifacts\c18z19-service-channel-parallel-flow-window-inner-result.json"
|
||||
$innerResultFullPath = Join-Path $repoRoot $innerResultPath
|
||||
|
||||
$agentRoot = Join-Path $repoRoot "agents\rap-node-agent"
|
||||
Push-Location $agentRoot
|
||||
try {
|
||||
$unitTestOutput = & go test ./internal/vpnruntime -run "TestFabricClientPacketIngressParallelFlowWindowDoesNotBlockIndependentChannel" 2>&1
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
$unitText = ($unitTestOutput | Out-String)
|
||||
throw "C18Z19 parallel flow window unit test failed:`n$unitText"
|
||||
}
|
||||
} finally {
|
||||
Pop-Location
|
||||
}
|
||||
|
||||
& (Join-Path $scriptDir "c18z18-service-channel-session-scoped-fairness-smoke.ps1") `
|
||||
-ApiBaseUrl $ApiBaseUrl `
|
||||
-ClusterID $ClusterID `
|
||||
-ActorUserID $ActorUserID `
|
||||
-EntryNodeName $EntryNodeName `
|
||||
-RelayNodeName $RelayNodeName `
|
||||
-ExitNodeName $ExitNodeName `
|
||||
-EntryBaseUrl $EntryBaseUrl `
|
||||
-DockerSSH $DockerSSH `
|
||||
-InitialBatchCount $InitialBatchCount `
|
||||
-LearningBatchCount $LearningBatchCount `
|
||||
-PostChurnBatchCount $PostChurnBatchCount `
|
||||
-PacketsPerBatch $PacketsPerBatch `
|
||||
-BatchDelayMilliseconds $BatchDelayMilliseconds `
|
||||
-ResultPath $innerResultPath
|
||||
|
||||
$result = Get-Content -Path $innerResultFullPath -Raw | ConvertFrom-Json
|
||||
$finalIngress = $result.telemetry.final_entry_ingress
|
||||
$maxParallel = 0
|
||||
$parallelBatches = 0
|
||||
if ($finalIngress.PSObject.Properties["max_parallel_flow_sends"]) {
|
||||
$maxParallel = [int]$finalIngress.max_parallel_flow_sends
|
||||
}
|
||||
if ($finalIngress.PSObject.Properties["send_flow_parallel_batches"]) {
|
||||
$parallelBatches = [int64]$finalIngress.send_flow_parallel_batches
|
||||
}
|
||||
|
||||
$result.schema_version = "c18z19.service_channel_parallel_flow_window_smoke.v1"
|
||||
$result | Add-Member -NotePropertyName c18z19_checks -NotePropertyValue ([ordered]@{
|
||||
unit_parallel_flow_window_does_not_block_independent_channel = ($unitTestOutput -join "`n").Contains("ok")
|
||||
live_parallel_window_enabled = ($maxParallel -ge 2)
|
||||
live_parallel_batches_observed = ($parallelBatches -gt 0)
|
||||
live_session_scoped_fairness_still_passed = ([bool]$result.passed)
|
||||
live_session_completed_without_backend_fallback = ([int]$result.backend_fallback_queue.depth -eq 0)
|
||||
live_session_completed_without_flow_drops = ([int]$result.flow_drops.delta -eq 0)
|
||||
}) -Force
|
||||
$result | Add-Member -NotePropertyName c18z19_summary -NotePropertyValue ([ordered]@{
|
||||
unit_test_output = ($unitTestOutput | Out-String).Trim()
|
||||
max_parallel_flow_sends = $maxParallel
|
||||
send_flow_parallel_batches = $parallelBatches
|
||||
}) -Force
|
||||
$result.passed = [bool]($result.passed -and
|
||||
$result.c18z19_checks.unit_parallel_flow_window_does_not_block_independent_channel -and
|
||||
$result.c18z19_checks.live_parallel_window_enabled -and
|
||||
$result.c18z19_checks.live_parallel_batches_observed -and
|
||||
$result.c18z19_checks.live_session_completed_without_backend_fallback -and
|
||||
$result.c18z19_checks.live_session_completed_without_flow_drops)
|
||||
|
||||
$resolvedResultPath = Join-Path $repoRoot $ResultPath
|
||||
$result | ConvertTo-Json -Depth 100 | Set-Content -Path $resolvedResultPath -Encoding UTF8
|
||||
Remove-Item -Path $innerResultFullPath -Force -ErrorAction SilentlyContinue
|
||||
|
||||
if (-not $result.passed) {
|
||||
throw "C18Z19 parallel flow window smoke failed. Result: $resolvedResultPath"
|
||||
}
|
||||
|
||||
Write-Host "C18Z19 service-channel parallel flow window smoke passed. Result: $resolvedResultPath"
|
||||
$result
|
||||
@@ -0,0 +1,820 @@
|
||||
param(
|
||||
[string]$ApiBaseUrl = "http://192.168.200.61:18121/api/v1",
|
||||
[string]$ClusterID = "cfc0743d-d960-49fb-9de8-96e063d5e4aa",
|
||||
[string]$ActorUserID = "f67d943f-5397-4b3a-a229-695fe67ad700",
|
||||
[string]$EntryNodeName = "test-1",
|
||||
[string]$ExitNodeName = "test-2",
|
||||
[string]$EntryBaseUrl = "http://192.168.200.61:19131",
|
||||
[string]$DockerSSH = "test-docker",
|
||||
[int]$WarmBatchCount = 6,
|
||||
[int]$RecoveryBatchCount = 8,
|
||||
[switch]$SkipExitRestart,
|
||||
[string]$RequiredNodeVersion = "0.2.182",
|
||||
[string]$ResultPath = "artifacts\c18z2-live-service-channel-soak-smoke-result.json"
|
||||
)
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = "Stop"
|
||||
Add-Type -AssemblyName System.Net.Http
|
||||
|
||||
$repoRoot = (Resolve-Path (Join-Path $PSScriptRoot "..\..")).ProviderPath
|
||||
$runId = "c18z2-" + (Get-Date -Format "yyyyMMdd-HHmmss")
|
||||
$resourceId = "vpn-$runId"
|
||||
|
||||
function Invoke-Api {
|
||||
param(
|
||||
[string]$Method,
|
||||
[string]$Path,
|
||||
[object]$Body = $null
|
||||
)
|
||||
$uri = "$ApiBaseUrl$Path"
|
||||
try {
|
||||
if ($null -eq $Body) {
|
||||
return Invoke-RestMethod -Method $Method -Uri $uri -TimeoutSec 30
|
||||
}
|
||||
return Invoke-RestMethod -Method $Method -Uri $uri -ContentType "application/json" -Body ($Body | ConvertTo-Json -Depth 80) -TimeoutSec 30
|
||||
}
|
||||
catch {
|
||||
$statusCode = $null
|
||||
if ($_.Exception.Response) {
|
||||
$statusCode = [int]$_.Exception.Response.StatusCode
|
||||
}
|
||||
$details = $_.ErrorDetails.Message
|
||||
if (-not $details) {
|
||||
$details = $_.Exception.Message
|
||||
}
|
||||
throw "$Method $Path failed with HTTP $statusCode`: $details"
|
||||
}
|
||||
}
|
||||
|
||||
function Get-NodeByName {
|
||||
param([string]$Name)
|
||||
$nodes = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/nodes?actor_user_id=$ActorUserID").nodes
|
||||
$node = @($nodes | Where-Object { $_.name -eq $Name }) | Select-Object -First 1
|
||||
if ($null -eq $node) {
|
||||
throw "Node '$Name' was not found in cluster $ClusterID"
|
||||
}
|
||||
return $node
|
||||
}
|
||||
|
||||
function Get-MeshPort {
|
||||
param([string]$Name)
|
||||
switch ($Name) {
|
||||
"test-1" { return 19131 }
|
||||
"test-2" { return 19132 }
|
||||
"test-3" { return 19133 }
|
||||
default { return 19131 }
|
||||
}
|
||||
}
|
||||
|
||||
function Enable-TestMeshListener {
|
||||
param([object]$Node)
|
||||
$port = Get-MeshPort -Name $Node.name
|
||||
Invoke-Api -Method PUT -Path "/clusters/$ClusterID/nodes/$($Node.id)/workloads/mesh-listener/desired" -Body @{
|
||||
actor_user_id = $ActorUserID
|
||||
desired_state = "enabled"
|
||||
runtime_mode = "container"
|
||||
version = "c18z2-live-fsc-soak"
|
||||
config = @{
|
||||
listen_addr = "0.0.0.0:$port"
|
||||
listen_port_mode = "manual"
|
||||
advertise_endpoint = "http://192.168.200.61:$port"
|
||||
advertise_transport = "direct_http"
|
||||
connectivity_mode = "private_lan"
|
||||
nat_type = "none"
|
||||
region = "docker-test"
|
||||
production_forwarding = $true
|
||||
}
|
||||
environment = @{}
|
||||
} | Out-Null
|
||||
}
|
||||
|
||||
function Clear-OldSmokeRouteIntents {
|
||||
param(
|
||||
[string]$SourceNodeID,
|
||||
[string]$DestinationNodeID
|
||||
)
|
||||
$items = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/mesh/route-intents?actor_user_id=$ActorUserID").route_intents
|
||||
foreach ($item in @($items)) {
|
||||
if ([string]$item.lifecycle_status -ne "active") {
|
||||
continue
|
||||
}
|
||||
if ([string]$item.service_class -ne "vpn_packets") {
|
||||
continue
|
||||
}
|
||||
if ([string]$item.source_selector.node_id -ne $SourceNodeID -or [string]$item.destination_selector.node_id -ne $DestinationNodeID) {
|
||||
continue
|
||||
}
|
||||
$smoke = ""
|
||||
if ($null -ne $item.policy -and $null -ne $item.policy.metadata) {
|
||||
$prop = $item.policy.metadata.PSObject.Properties["smoke"]
|
||||
if ($null -ne $prop) {
|
||||
$smoke = [string]$prop.Value
|
||||
}
|
||||
}
|
||||
if ($smoke -ne "c18z1_live_service_channel_ingress" -and $smoke -ne "c18z2_live_service_channel_soak") {
|
||||
continue
|
||||
}
|
||||
Invoke-Api -Method POST -Path "/clusters/$ClusterID/mesh/route-intents/$($item.id)/expire" -Body @{ actor_user_id = $ActorUserID } | Out-Null
|
||||
}
|
||||
}
|
||||
|
||||
function New-RouteIntent {
|
||||
param(
|
||||
[string]$SourceNodeID,
|
||||
[string]$DestinationNodeID,
|
||||
[int]$Priority,
|
||||
[string]$Label
|
||||
)
|
||||
$expiresAt = (Get-Date).ToUniversalTime().AddMinutes(10).ToString("o")
|
||||
return Invoke-Api -Method POST -Path "/clusters/$ClusterID/mesh/route-intents" -Body @{
|
||||
actor_user_id = $ActorUserID
|
||||
source_selector = @{ node_id = $SourceNodeID }
|
||||
destination_selector = @{ node_id = $DestinationNodeID }
|
||||
service_class = "vpn_packets"
|
||||
priority = $Priority
|
||||
policy = @{
|
||||
synthetic_enabled = $true
|
||||
route_version = "$runId-$Label"
|
||||
policy_version = "$runId-$Label"
|
||||
peer_directory_version = "$runId-$Label"
|
||||
hops = @($SourceNodeID, $DestinationNodeID)
|
||||
allowed_channels = @("vpn_packet", "fabric_control")
|
||||
max_ttl = 8
|
||||
max_hops = 8
|
||||
expires_at = $expiresAt
|
||||
metadata = @{
|
||||
smoke = "c18z2_live_service_channel_soak"
|
||||
run_id = $runId
|
||||
label = $Label
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function Get-SyntheticConfig {
|
||||
param([string]$NodeID)
|
||||
return Invoke-Api -Method GET -Path "/clusters/$ClusterID/nodes/$NodeID/mesh/synthetic-config?actor_user_id=$ActorUserID"
|
||||
}
|
||||
|
||||
function Get-LatestHeartbeat {
|
||||
param([string]$NodeID)
|
||||
return (Invoke-Api -Method GET -Path "/clusters/$ClusterID/nodes/$NodeID/heartbeats?actor_user_id=$ActorUserID&limit=1").heartbeats[0]
|
||||
}
|
||||
|
||||
function Get-LatestRuntimeReport {
|
||||
param([string]$NodeID)
|
||||
$hb = Get-LatestHeartbeat -NodeID $NodeID
|
||||
return @{
|
||||
heartbeat = $hb
|
||||
report = $hb.metadata.fabric_service_channel_runtime_report
|
||||
}
|
||||
}
|
||||
|
||||
function Wait-ForRuntimeReady {
|
||||
param(
|
||||
[string]$NodeID,
|
||||
[int]$MinRoutes,
|
||||
[int]$TimeoutSeconds = 90
|
||||
)
|
||||
$deadline = (Get-Date).AddSeconds($TimeoutSeconds)
|
||||
do {
|
||||
$latest = Get-LatestRuntimeReport -NodeID $NodeID
|
||||
$report = $latest.report
|
||||
if ($null -ne $report -and
|
||||
$report.enabled -eq $true -and
|
||||
$report.production_payload_forwarding -eq $true -and
|
||||
[int]$report.route_candidate_total -ge $MinRoutes) {
|
||||
return $latest
|
||||
}
|
||||
Start-Sleep -Seconds 2
|
||||
} while ((Get-Date) -lt $deadline)
|
||||
throw "Timed out waiting for production service-channel runtime ready on node $NodeID"
|
||||
}
|
||||
|
||||
function Wait-ForRuntimeConfigVersion {
|
||||
param(
|
||||
[string]$NodeID,
|
||||
[string]$ConfigVersion,
|
||||
[int]$TimeoutSeconds = 90
|
||||
)
|
||||
$deadline = (Get-Date).AddSeconds($TimeoutSeconds)
|
||||
do {
|
||||
$latest = Get-LatestRuntimeReport -NodeID $NodeID
|
||||
if ($null -ne $latest.report) {
|
||||
$loadedVersion = [string]$latest.report.config_version
|
||||
if ($loadedVersion -ge $ConfigVersion) {
|
||||
return $latest
|
||||
}
|
||||
}
|
||||
Start-Sleep -Seconds 2
|
||||
} while ((Get-Date) -lt $deadline)
|
||||
throw "Timed out waiting for node $NodeID to load synthetic config $ConfigVersion"
|
||||
}
|
||||
|
||||
function Wait-ForRouteIntentVisible {
|
||||
param(
|
||||
[string]$NodeID,
|
||||
[string[]]$RouteIDs,
|
||||
[int]$TimeoutSeconds = 60
|
||||
)
|
||||
$deadline = (Get-Date).AddSeconds($TimeoutSeconds)
|
||||
do {
|
||||
$config = Get-SyntheticConfig -NodeID $NodeID
|
||||
$routes = @($config.synthetic_mesh_config.routes)
|
||||
$present = @($routes | Where-Object { $RouteIDs -contains $_.route_id })
|
||||
if ($present.Count -ge $RouteIDs.Count) {
|
||||
return $config
|
||||
}
|
||||
Start-Sleep -Seconds 2
|
||||
} while ((Get-Date) -lt $deadline)
|
||||
throw "Timed out waiting for routes '$($RouteIDs -join ",")' in synthetic config for node $NodeID"
|
||||
}
|
||||
|
||||
function New-ServiceChannelLease {
|
||||
param(
|
||||
[string]$EntryNodeID,
|
||||
[string]$ExitNodeID
|
||||
)
|
||||
return (Invoke-Api -Method POST -Path "/clusters/$ClusterID/fabric/service-channels/leases" -Body @{
|
||||
actor_user_id = $ActorUserID
|
||||
organization_id = "org-c18z1-smoke"
|
||||
user_id = $ActorUserID
|
||||
resource_id = $resourceId
|
||||
service_class = "vpn_packets"
|
||||
entry_node_ids = @($EntryNodeID)
|
||||
exit_node_ids = @($ExitNodeID)
|
||||
preferred_entry_node_id = $EntryNodeID
|
||||
preferred_exit_node_id = $ExitNodeID
|
||||
allowed_channels = @("vpn_packet", "bulk", "control")
|
||||
ttl_seconds = 300
|
||||
metadata = @{
|
||||
smoke = "c18z2_live_service_channel_soak"
|
||||
run_id = $runId
|
||||
}
|
||||
}).fabric_service_channel_lease
|
||||
}
|
||||
|
||||
function ConvertTo-Base64UrlJson {
|
||||
param([object]$Value)
|
||||
$json = $Value | ConvertTo-Json -Depth 80 -Compress
|
||||
$bytes = [System.Text.Encoding]::UTF8.GetBytes($json)
|
||||
return [Convert]::ToBase64String($bytes).TrimEnd("=").Replace("+", "-").Replace("/", "_")
|
||||
}
|
||||
|
||||
function Get-ObjectPropertyValue {
|
||||
param(
|
||||
[object]$Object,
|
||||
[string]$Name
|
||||
)
|
||||
if ($null -eq $Object) {
|
||||
return $null
|
||||
}
|
||||
$prop = $Object.PSObject.Properties[$Name]
|
||||
if ($null -eq $prop) {
|
||||
return $null
|
||||
}
|
||||
return $prop.Value
|
||||
}
|
||||
|
||||
function New-TestIPv4UDPPacket {
|
||||
param([int]$SourcePort)
|
||||
$payload = [System.Text.Encoding]::ASCII.GetBytes("c18z1-$SourcePort")
|
||||
$totalLength = 20 + 8 + $payload.Length
|
||||
$packet = New-Object byte[] $totalLength
|
||||
$packet[0] = 0x45
|
||||
$packet[1] = 0
|
||||
$packet[2] = [byte](($totalLength -shr 8) -band 0xff)
|
||||
$packet[3] = [byte]($totalLength -band 0xff)
|
||||
$packet[8] = 64
|
||||
$packet[9] = 17
|
||||
$packet[12] = 10; $packet[13] = 18; $packet[14] = 1; $packet[15] = 10
|
||||
$packet[16] = 10; $packet[17] = 18; $packet[18] = 2; $packet[19] = 20
|
||||
$udpOffset = 20
|
||||
$destPort = 3389
|
||||
$udpLength = 8 + $payload.Length
|
||||
$packet[$udpOffset] = [byte](($SourcePort -shr 8) -band 0xff)
|
||||
$packet[$udpOffset + 1] = [byte]($SourcePort -band 0xff)
|
||||
$packet[$udpOffset + 2] = [byte](($destPort -shr 8) -band 0xff)
|
||||
$packet[$udpOffset + 3] = [byte]($destPort -band 0xff)
|
||||
$packet[$udpOffset + 4] = [byte](($udpLength -shr 8) -band 0xff)
|
||||
$packet[$udpOffset + 5] = [byte]($udpLength -band 0xff)
|
||||
[Array]::Copy($payload, 0, $packet, 28, $payload.Length)
|
||||
return $packet
|
||||
}
|
||||
|
||||
function New-PacketBatchBody {
|
||||
param([byte[][]]$Packets)
|
||||
$stream = [System.IO.MemoryStream]::new()
|
||||
foreach ($packet in $Packets) {
|
||||
$length = $packet.Length
|
||||
$stream.WriteByte([byte](($length -shr 24) -band 0xff))
|
||||
$stream.WriteByte([byte](($length -shr 16) -band 0xff))
|
||||
$stream.WriteByte([byte](($length -shr 8) -band 0xff))
|
||||
$stream.WriteByte([byte]($length -band 0xff))
|
||||
$stream.Write($packet, 0, $packet.Length)
|
||||
}
|
||||
return $stream.ToArray()
|
||||
}
|
||||
|
||||
function Invoke-ServiceChannelPost {
|
||||
param(
|
||||
[object]$Lease,
|
||||
[int]$PortStart
|
||||
)
|
||||
$packets = @()
|
||||
for ($i = 0; $i -lt 8; $i++) {
|
||||
$packets += ,(New-TestIPv4UDPPacket -SourcePort ($PortStart + $i))
|
||||
}
|
||||
$path = $Lease.entry_http.path_template.
|
||||
Replace("{cluster_id}", $ClusterID).
|
||||
Replace("{channel_id}", $Lease.channel_id).
|
||||
Replace("{resource_id}", $resourceId)
|
||||
$url = "$EntryBaseUrl$path`?batch=true"
|
||||
$headers = @{
|
||||
"X-RAP-Service-Channel-Token" = $Lease.token.token
|
||||
"X-RAP-Fabric-Channel-ID" = $Lease.channel_id
|
||||
"X-RAP-Service-Class" = "vpn_packets"
|
||||
"X-RAP-Channel-Class" = "vpn_packet"
|
||||
"X-RAP-Service-Channel-Authority-Payload" = ConvertTo-Base64UrlJson -Value $Lease.authority_payload
|
||||
"X-RAP-Service-Channel-Authority-Signature" = ConvertTo-Base64UrlJson -Value $Lease.authority_signature
|
||||
}
|
||||
$body = New-PacketBatchBody -Packets $packets
|
||||
$client = [System.Net.Http.HttpClient]::new()
|
||||
try {
|
||||
$client.Timeout = [TimeSpan]::FromSeconds(30)
|
||||
$request = [System.Net.Http.HttpRequestMessage]::new([System.Net.Http.HttpMethod]::Post, $url)
|
||||
foreach ($header in $headers.GetEnumerator()) {
|
||||
[void]$request.Headers.TryAddWithoutValidation($header.Key, [string]$header.Value)
|
||||
}
|
||||
$content = [System.Net.Http.ByteArrayContent]::new($body)
|
||||
$content.Headers.ContentType = [System.Net.Http.Headers.MediaTypeHeaderValue]::Parse("application/vnd.rap.vpn-packet-batch.v1")
|
||||
$request.Content = $content
|
||||
$response = $client.SendAsync($request).GetAwaiter().GetResult()
|
||||
$responseBody = $response.Content.ReadAsStringAsync().GetAwaiter().GetResult()
|
||||
if (-not $response.IsSuccessStatusCode) {
|
||||
throw "Service-channel POST $url failed with HTTP $([int]$response.StatusCode): $responseBody"
|
||||
}
|
||||
return [pscustomobject]@{
|
||||
StatusCode = [int]$response.StatusCode
|
||||
Body = $responseBody
|
||||
}
|
||||
}
|
||||
finally {
|
||||
$client.Dispose()
|
||||
}
|
||||
}
|
||||
|
||||
function Get-IngressSendPackets {
|
||||
param([string]$NodeID)
|
||||
$latest = Get-LatestRuntimeReport -NodeID $NodeID
|
||||
$ingress = $latest.report.ingress
|
||||
$sendPackets = Get-ObjectPropertyValue -Object $ingress -Name "send_packets"
|
||||
if ($null -eq $sendPackets) {
|
||||
return 0
|
||||
}
|
||||
return [int]$sendPackets
|
||||
}
|
||||
|
||||
function Get-IngressRouteFailures {
|
||||
param([string]$NodeID)
|
||||
$latest = Get-LatestRuntimeReport -NodeID $NodeID
|
||||
$ingress = $latest.report.ingress
|
||||
$failures = Get-ObjectPropertyValue -Object $ingress -Name "send_route_failures"
|
||||
if ($null -eq $failures) {
|
||||
return 0
|
||||
}
|
||||
return [int]$failures
|
||||
}
|
||||
|
||||
function Get-ExitQueueDepth {
|
||||
param(
|
||||
[string]$NodeID,
|
||||
[string]$VPNConnectionID
|
||||
)
|
||||
$latest = Get-LatestRuntimeReport -NodeID $NodeID
|
||||
$queueKey = "$VPNConnectionID`:client_to_gateway"
|
||||
$depths = $latest.report.inbox.queue_depths
|
||||
if ($null -eq $depths) {
|
||||
return 0
|
||||
}
|
||||
$prop = $depths.PSObject.Properties[$queueKey]
|
||||
if ($null -eq $prop) {
|
||||
return 0
|
||||
}
|
||||
return [int]$prop.Value
|
||||
}
|
||||
|
||||
function Wait-ForExitQueueDepth {
|
||||
param(
|
||||
[string]$NodeID,
|
||||
[string]$VPNConnectionID,
|
||||
[int]$MinDepth,
|
||||
[int]$TimeoutSeconds = 90
|
||||
)
|
||||
$deadline = (Get-Date).AddSeconds($TimeoutSeconds)
|
||||
do {
|
||||
$depth = Get-ExitQueueDepth -NodeID $NodeID -VPNConnectionID $VPNConnectionID
|
||||
if ($depth -ge $MinDepth) {
|
||||
return $depth
|
||||
}
|
||||
Start-Sleep -Seconds 2
|
||||
} while ((Get-Date) -lt $deadline)
|
||||
throw "Timed out waiting for exit queue depth >= $MinDepth on node $NodeID"
|
||||
}
|
||||
|
||||
function Invoke-ServiceChannelPostSafe {
|
||||
param(
|
||||
[object]$Lease,
|
||||
[int]$PortStart
|
||||
)
|
||||
try {
|
||||
$response = Invoke-ServiceChannelPost -Lease $Lease -PortStart $PortStart
|
||||
return [pscustomobject]@{
|
||||
ok = $true
|
||||
status_code = [int]$response.StatusCode
|
||||
error = ""
|
||||
}
|
||||
}
|
||||
catch {
|
||||
return [pscustomobject]@{
|
||||
ok = $false
|
||||
status_code = 0
|
||||
error = $_.Exception.Message
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function Send-BatchSeries {
|
||||
param(
|
||||
[object]$Lease,
|
||||
[int]$Count,
|
||||
[int]$PortBase,
|
||||
[int]$DelayMilliseconds = 100
|
||||
)
|
||||
$results = @()
|
||||
for ($i = 0; $i -lt $Count; $i++) {
|
||||
$results += Invoke-ServiceChannelPostSafe -Lease $Lease -PortStart ($PortBase + ($i * 100))
|
||||
if ($DelayMilliseconds -gt 0) {
|
||||
Start-Sleep -Milliseconds $DelayMilliseconds
|
||||
}
|
||||
}
|
||||
return $results
|
||||
}
|
||||
|
||||
function Invoke-RemoteDocker {
|
||||
param([string]$Command)
|
||||
& ssh $DockerSSH $Command
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "ssh $DockerSSH command failed: $Command"
|
||||
}
|
||||
}
|
||||
|
||||
function Stop-TestUpdaters {
|
||||
Invoke-RemoteDocker -Command "docker stop rap_host_agent_updater_test-1 rap_host_agent_updater_test-2 rap_host_agent_updater_test-3 >/dev/null 2>&1 || true"
|
||||
}
|
||||
|
||||
function Start-TestUpdaters {
|
||||
Invoke-RemoteDocker -Command "docker start rap_host_agent_updater_test-1 rap_host_agent_updater_test-2 rap_host_agent_updater_test-3 >/dev/null 2>&1 || true"
|
||||
}
|
||||
|
||||
function Restart-ExitContainer {
|
||||
param([string]$Name)
|
||||
$containerName = "rap_test_node_" + $Name.Replace("-", "_")
|
||||
Invoke-RemoteDocker -Command "docker restart $containerName >/dev/null"
|
||||
}
|
||||
|
||||
function Wait-ForIngressRoute {
|
||||
param(
|
||||
[string]$NodeID,
|
||||
[string]$RouteID,
|
||||
[int]$MinSendPackets,
|
||||
[int]$TimeoutSeconds = 45
|
||||
)
|
||||
$deadline = (Get-Date).AddSeconds($TimeoutSeconds)
|
||||
do {
|
||||
$latest = Get-LatestRuntimeReport -NodeID $NodeID
|
||||
$ingress = $latest.report.ingress
|
||||
$sendPackets = Get-ObjectPropertyValue -Object $ingress -Name "send_packets"
|
||||
$selectedRoute = Get-ObjectPropertyValue -Object $ingress -Name "last_selected_route_id"
|
||||
if ($null -ne $ingress -and
|
||||
[int]$sendPackets -ge $MinSendPackets -and
|
||||
[string]$selectedRoute -eq $RouteID) {
|
||||
return $latest
|
||||
}
|
||||
Start-Sleep -Seconds 2
|
||||
} while ((Get-Date) -lt $deadline)
|
||||
throw "Timed out waiting for ingress telemetry route=$RouteID packets>=$MinSendPackets on node $NodeID"
|
||||
}
|
||||
|
||||
function Wait-ForIngressAnyRoute {
|
||||
param(
|
||||
[string]$NodeID,
|
||||
[string[]]$RouteIDs,
|
||||
[int]$MinSendPackets,
|
||||
[int]$TimeoutSeconds = 45
|
||||
)
|
||||
$deadline = (Get-Date).AddSeconds($TimeoutSeconds)
|
||||
do {
|
||||
$latest = Get-LatestRuntimeReport -NodeID $NodeID
|
||||
$ingress = $latest.report.ingress
|
||||
$sendPackets = Get-ObjectPropertyValue -Object $ingress -Name "send_packets"
|
||||
$selectedRoute = Get-ObjectPropertyValue -Object $ingress -Name "last_selected_route_id"
|
||||
if ($null -ne $ingress -and
|
||||
[int]$sendPackets -ge $MinSendPackets -and
|
||||
$RouteIDs -contains [string]$selectedRoute) {
|
||||
return $latest
|
||||
}
|
||||
Start-Sleep -Seconds 2
|
||||
} while ((Get-Date) -lt $deadline)
|
||||
throw "Timed out waiting for ingress telemetry routes='$($RouteIDs -join ",")' packets>=$MinSendPackets on node $NodeID"
|
||||
}
|
||||
|
||||
function Wait-ForExitInbox {
|
||||
param(
|
||||
[string]$NodeID,
|
||||
[string]$VPNConnectionID,
|
||||
[int]$TimeoutSeconds = 45
|
||||
)
|
||||
$queueKey = "$VPNConnectionID`:client_to_gateway"
|
||||
$deadline = (Get-Date).AddSeconds($TimeoutSeconds)
|
||||
do {
|
||||
$latest = Get-LatestRuntimeReport -NodeID $NodeID
|
||||
$depths = $latest.report.inbox.queue_depths
|
||||
if ($null -ne $depths) {
|
||||
$prop = $depths.PSObject.Properties[$queueKey]
|
||||
if ($null -ne $prop -and [int]$prop.Value -gt 0) {
|
||||
return $latest
|
||||
}
|
||||
}
|
||||
Start-Sleep -Seconds 2
|
||||
} while ((Get-Date) -lt $deadline)
|
||||
throw "Timed out waiting for exit inbox queue '$queueKey' on node $NodeID"
|
||||
}
|
||||
|
||||
function Send-FeedbackHeartbeat {
|
||||
param(
|
||||
[string]$EntryNodeID,
|
||||
[string]$BadRouteID,
|
||||
[string]$GoodRouteID
|
||||
)
|
||||
return Invoke-Api -Method POST -Path "/clusters/$ClusterID/nodes/$EntryNodeID/heartbeats" -Body @{
|
||||
health_status = "healthy"
|
||||
reported_version = $RequiredNodeVersion
|
||||
capabilities = @{
|
||||
native_node_agent = $true
|
||||
fabric_service_channel_runtime = $true
|
||||
fabric_service_channel_route_manager = $true
|
||||
smoke_feedback_injection = "c18z1"
|
||||
}
|
||||
service_states = @{ smoke = "c18z1_live_ingress_feedback" }
|
||||
metadata = @{
|
||||
fabric_service_channel_runtime_report = @{
|
||||
schema_version = "c18l.fabric_service_channel_runtime_report.v1"
|
||||
ingress = @{
|
||||
flow_scheduler = @{
|
||||
channel_stats = @{
|
||||
"c18z1-live-flow" = @{
|
||||
last_route_id = $GoodRouteID
|
||||
last_failed_route_id = $BadRouteID
|
||||
last_error = "c18z1 forced stale route after live packet ingress"
|
||||
consecutive_failures = 3
|
||||
stall_count = 1
|
||||
last_send_duration_ms = 250
|
||||
route_rebuild_recommended = $true
|
||||
degraded_fallback_recommended = $false
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
smoke = @{
|
||||
name = "c18z1_live_service_channel_ingress"
|
||||
run_id = $runId
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function Wait-ForConfigDecision {
|
||||
param(
|
||||
[string]$NodeID,
|
||||
[string]$BadRouteID,
|
||||
[string]$ExpectedReplacementID,
|
||||
[int]$TimeoutSeconds = 60
|
||||
)
|
||||
$deadline = (Get-Date).AddSeconds($TimeoutSeconds)
|
||||
do {
|
||||
$config = Get-SyntheticConfig -NodeID $NodeID
|
||||
$decisions = @($config.synthetic_mesh_config.route_path_decisions.decisions)
|
||||
$decision = @($decisions | Where-Object {
|
||||
$_.route_id -eq $BadRouteID -and
|
||||
$_.rebuild_status -eq "applied" -and
|
||||
$_.replacement_route_id -eq $ExpectedReplacementID
|
||||
}) | Select-Object -First 1
|
||||
if ($null -ne $decision) {
|
||||
return @{
|
||||
config = $config
|
||||
decision = $decision
|
||||
}
|
||||
}
|
||||
Start-Sleep -Seconds 2
|
||||
} while ((Get-Date) -lt $deadline)
|
||||
throw "Timed out waiting for applied rebuild decision $BadRouteID -> $ExpectedReplacementID"
|
||||
}
|
||||
|
||||
function Wait-ForAppliedRebuildTransition {
|
||||
param(
|
||||
[string]$NodeID,
|
||||
[string]$BadRouteID = "",
|
||||
[string]$ReplacementRouteID = "",
|
||||
[int]$TimeoutSeconds = 90
|
||||
)
|
||||
$deadline = (Get-Date).AddSeconds($TimeoutSeconds)
|
||||
do {
|
||||
$latest = Get-LatestRuntimeReport -NodeID $NodeID
|
||||
$transition = $null
|
||||
if ($null -ne $latest.report -and $null -ne $latest.report.ingress) {
|
||||
$prop = $latest.report.ingress.PSObject.Properties["route_manager_transition"]
|
||||
if ($null -ne $prop) {
|
||||
$transition = $prop.Value
|
||||
}
|
||||
}
|
||||
if ($null -ne $transition -and [string]$transition.status -eq "applied_rebuild") {
|
||||
return $latest
|
||||
}
|
||||
if ($BadRouteID -ne "" -and $ReplacementRouteID -ne "") {
|
||||
Send-FeedbackHeartbeat -EntryNodeID $NodeID -BadRouteID $BadRouteID -GoodRouteID $ReplacementRouteID | Out-Null
|
||||
}
|
||||
Start-Sleep -Seconds 2
|
||||
} while ((Get-Date) -lt $deadline)
|
||||
throw "Timed out waiting for node route-manager transition applied_rebuild on node $NodeID"
|
||||
}
|
||||
|
||||
$entryNode = Get-NodeByName -Name $EntryNodeName
|
||||
$exitNode = Get-NodeByName -Name $ExitNodeName
|
||||
$primaryRouteID = ""
|
||||
$alternateRouteID = ""
|
||||
$updatersStopped = $false
|
||||
$result = $null
|
||||
|
||||
try {
|
||||
Stop-TestUpdaters
|
||||
$updatersStopped = $true
|
||||
|
||||
Enable-TestMeshListener -Node $entryNode
|
||||
Enable-TestMeshListener -Node $exitNode
|
||||
Clear-OldSmokeRouteIntents -SourceNodeID $entryNode.id -DestinationNodeID $exitNode.id
|
||||
|
||||
$primaryIntent = New-RouteIntent -SourceNodeID $entryNode.id -DestinationNodeID $exitNode.id -Priority 2000000000 -Label "primary"
|
||||
$alternateIntent = New-RouteIntent -SourceNodeID $entryNode.id -DestinationNodeID $exitNode.id -Priority 1999999999 -Label "alternate"
|
||||
$primaryRouteID = $primaryIntent.route_intent.id
|
||||
$alternateRouteID = $alternateIntent.route_intent.id
|
||||
$routeIDs = @($primaryRouteID, $alternateRouteID)
|
||||
|
||||
$visibleConfig = Wait-ForRouteIntentVisible -NodeID $entryNode.id -RouteIDs $routeIDs
|
||||
$exitVisibleConfig = Wait-ForRouteIntentVisible -NodeID $exitNode.id -RouteIDs $routeIDs
|
||||
$readyBefore = Wait-ForRuntimeReady -NodeID $entryNode.id -MinRoutes 2
|
||||
$exitReadyBefore = Wait-ForRuntimeReady -NodeID $exitNode.id -MinRoutes 0
|
||||
$loadedConfig = Wait-ForRuntimeConfigVersion -NodeID $entryNode.id -ConfigVersion $visibleConfig.synthetic_mesh_config.config_version
|
||||
$exitLoadedConfig = Wait-ForRuntimeConfigVersion -NodeID $exitNode.id -ConfigVersion $exitVisibleConfig.synthetic_mesh_config.config_version
|
||||
$lease = New-ServiceChannelLease -EntryNodeID $entryNode.id -ExitNodeID $exitNode.id
|
||||
if ($lease.status -ne "ready") {
|
||||
throw "Lease status was '$($lease.status)', want ready"
|
||||
}
|
||||
if ($routeIDs -notcontains [string]$lease.primary_route.route_id) {
|
||||
throw "Lease primary route was '$($lease.primary_route.route_id)', want one of smoke routes"
|
||||
}
|
||||
|
||||
$baselineSendPackets = Get-IngressSendPackets -NodeID $entryNode.id
|
||||
$baselineRouteFailures = Get-IngressRouteFailures -NodeID $entryNode.id
|
||||
|
||||
$warmResults = Send-BatchSeries -Lease $lease -Count $WarmBatchCount -PortBase 43000 -DelayMilliseconds 100
|
||||
$warmOk = @($warmResults | Where-Object { $_.ok -eq $true }).Count
|
||||
if ($warmOk -lt $WarmBatchCount) {
|
||||
throw "Warm service-channel soak accepted $warmOk/$WarmBatchCount batches"
|
||||
}
|
||||
$warmIngress = Wait-ForIngressAnyRoute -NodeID $entryNode.id -RouteIDs $routeIDs -MinSendPackets ($baselineSendPackets + ($WarmBatchCount * 8)) -TimeoutSeconds 90
|
||||
$warmExitDepth = Wait-ForExitQueueDepth -NodeID $exitNode.id -VPNConnectionID $resourceId -MinDepth 8 -TimeoutSeconds 90
|
||||
|
||||
$restartAttempted = $false
|
||||
$duringRestartResults = @()
|
||||
$postRestartBaselineExitDepth = $warmExitDepth
|
||||
if (-not $SkipExitRestart) {
|
||||
$restartAttempted = $true
|
||||
Restart-ExitContainer -Name $ExitNodeName
|
||||
$duringRestartResults = Send-BatchSeries -Lease $lease -Count 3 -PortBase 53000 -DelayMilliseconds 150
|
||||
$exitNode = Get-NodeByName -Name $ExitNodeName
|
||||
$postRestartVisibleConfig = Wait-ForRouteIntentVisible -NodeID $exitNode.id -RouteIDs $routeIDs -TimeoutSeconds 120
|
||||
$postRestartReady = Wait-ForRuntimeReady -NodeID $exitNode.id -MinRoutes 0 -TimeoutSeconds 120
|
||||
$postRestartLoadedConfig = Wait-ForRuntimeConfigVersion -NodeID $exitNode.id -ConfigVersion $postRestartVisibleConfig.synthetic_mesh_config.config_version -TimeoutSeconds 120
|
||||
$postRestartBaselineExitDepth = Get-ExitQueueDepth -NodeID $exitNode.id -VPNConnectionID $resourceId
|
||||
}
|
||||
else {
|
||||
$postRestartVisibleConfig = $exitVisibleConfig
|
||||
$postRestartReady = $exitReadyBefore
|
||||
$postRestartLoadedConfig = $exitLoadedConfig
|
||||
}
|
||||
|
||||
$recoveryBaseline = Get-IngressSendPackets -NodeID $entryNode.id
|
||||
$recoveryResults = Send-BatchSeries -Lease $lease -Count $RecoveryBatchCount -PortBase 63000 -DelayMilliseconds 100
|
||||
$recoveryOk = @($recoveryResults | Where-Object { $_.ok -eq $true }).Count
|
||||
if ($recoveryOk -lt $RecoveryBatchCount) {
|
||||
throw "Recovery service-channel soak accepted $recoveryOk/$RecoveryBatchCount batches"
|
||||
}
|
||||
$recoveryIngress = Wait-ForIngressAnyRoute -NodeID $entryNode.id -RouteIDs $routeIDs -MinSendPackets ($recoveryBaseline + ($RecoveryBatchCount * 8)) -TimeoutSeconds 120
|
||||
$recoveryExitDepth = Wait-ForExitQueueDepth -NodeID $exitNode.id -VPNConnectionID $resourceId -MinDepth ($postRestartBaselineExitDepth + 8) -TimeoutSeconds 120
|
||||
$finalExitRuntime = Get-LatestRuntimeReport -NodeID $exitNode.id
|
||||
$finalRouteFailures = Get-IngressRouteFailures -NodeID $entryNode.id
|
||||
|
||||
$expiredPrimary = Invoke-Api -Method POST -Path "/clusters/$ClusterID/mesh/route-intents/$primaryRouteID/expire" -Body @{ actor_user_id = $ActorUserID }
|
||||
$expiredAlternate = Invoke-Api -Method POST -Path "/clusters/$ClusterID/mesh/route-intents/$alternateRouteID/expire" -Body @{ actor_user_id = $ActorUserID }
|
||||
|
||||
$result = [ordered]@{
|
||||
schema_version = "c18z2.live_service_channel_soak_smoke.v1"
|
||||
run_id = $runId
|
||||
base_url = $ApiBaseUrl
|
||||
entry_base_url = $EntryBaseUrl
|
||||
cluster_id = $ClusterID
|
||||
entry_node = @{ name = $entryNode.name; id = $entryNode.id }
|
||||
exit_node = @{ name = $exitNode.name; id = $exitNode.id }
|
||||
resource_id = $resourceId
|
||||
route_intents = @{
|
||||
primary_route_intent_id = $primaryRouteID
|
||||
alternate_route_intent_id = $alternateRouteID
|
||||
expired_primary_status = $expiredPrimary.route_intent.lifecycle_status
|
||||
expired_alternate_status = $expiredAlternate.route_intent.lifecycle_status
|
||||
}
|
||||
lease = @{
|
||||
channel_id = $lease.channel_id
|
||||
status = $lease.status
|
||||
primary_route_id = $lease.primary_route.route_id
|
||||
}
|
||||
batches = @{
|
||||
warm_requested = $WarmBatchCount
|
||||
warm_accepted = $warmOk
|
||||
during_restart_requested = @($duringRestartResults).Count
|
||||
during_restart_accepted = @($duringRestartResults | Where-Object { $_.ok -eq $true }).Count
|
||||
recovery_requested = $RecoveryBatchCount
|
||||
recovery_accepted = $recoveryOk
|
||||
}
|
||||
route_failures = @{
|
||||
baseline = $baselineRouteFailures
|
||||
final = $finalRouteFailures
|
||||
delta = ($finalRouteFailures - $baselineRouteFailures)
|
||||
}
|
||||
exit_queue = @{
|
||||
warm_depth = $warmExitDepth
|
||||
post_restart_baseline_depth = $postRestartBaselineExitDepth
|
||||
recovery_depth = $recoveryExitDepth
|
||||
}
|
||||
restart_attempted = $restartAttempted
|
||||
passed = $true
|
||||
checks = [ordered]@{
|
||||
production_forwarding_ready = ($readyBefore.report.production_payload_forwarding -eq $true)
|
||||
exit_production_forwarding_ready = ($exitReadyBefore.report.production_payload_forwarding -eq $true)
|
||||
route_intents_visible_before_post = (@($visibleConfig.synthetic_mesh_config.routes | Where-Object { $routeIDs -contains $_.route_id }).Count -ge 2)
|
||||
exit_route_intents_visible_before_post = (@($exitVisibleConfig.synthetic_mesh_config.routes | Where-Object { $routeIDs -contains $_.route_id }).Count -ge 2)
|
||||
entry_runtime_loaded_visible_config = ([string]$loadedConfig.report.config_version -ge [string]$visibleConfig.synthetic_mesh_config.config_version)
|
||||
exit_runtime_loaded_visible_config = ([string]$exitLoadedConfig.report.config_version -ge [string]$exitVisibleConfig.synthetic_mesh_config.config_version)
|
||||
signed_lease_ready = ($lease.status -eq "ready")
|
||||
warm_batches_accepted = ($warmOk -eq $WarmBatchCount)
|
||||
warm_exit_inbox_received = ($warmExitDepth -ge 8)
|
||||
exit_restart_recovered = ($postRestartReady.report.production_payload_forwarding -eq $true -and [string]$postRestartLoadedConfig.report.config_version -ge [string]$postRestartVisibleConfig.synthetic_mesh_config.config_version)
|
||||
recovery_batches_accepted = ($recoveryOk -eq $RecoveryBatchCount)
|
||||
recovery_exit_inbox_grew = ($recoveryExitDepth -ge ($postRestartBaselineExitDepth + 8))
|
||||
route_intents_expired = ($expiredPrimary.route_intent.lifecycle_status -eq "expired" -and $expiredAlternate.route_intent.lifecycle_status -eq "expired")
|
||||
}
|
||||
telemetry = @{
|
||||
warm_ingress = $warmIngress.report.ingress
|
||||
recovery_ingress = $recoveryIngress.report.ingress
|
||||
exit_inbox = $finalExitRuntime.report.inbox
|
||||
during_restart_results = $duringRestartResults
|
||||
}
|
||||
}
|
||||
|
||||
$failedChecks = @($result.checks.GetEnumerator() | Where-Object { $_.Value -ne $true })
|
||||
if ($failedChecks.Count -gt 0) {
|
||||
throw "C18Z2 failed checks: $($failedChecks.Name -join ', ')"
|
||||
}
|
||||
}
|
||||
finally {
|
||||
if ($primaryRouteID) {
|
||||
try { Invoke-Api -Method POST -Path "/clusters/$ClusterID/mesh/route-intents/$primaryRouteID/expire" -Body @{ actor_user_id = $ActorUserID } | Out-Null } catch {}
|
||||
}
|
||||
if ($alternateRouteID) {
|
||||
try { Invoke-Api -Method POST -Path "/clusters/$ClusterID/mesh/route-intents/$alternateRouteID/expire" -Body @{ actor_user_id = $ActorUserID } | Out-Null } catch {}
|
||||
}
|
||||
if ($updatersStopped) {
|
||||
try { Start-TestUpdaters } catch { Write-Warning "Could not restart test updaters: $($_.Exception.Message)" }
|
||||
}
|
||||
}
|
||||
|
||||
$resultFullPath = Join-Path $repoRoot $ResultPath
|
||||
$resultDir = Split-Path -Parent $resultFullPath
|
||||
if (-not (Test-Path $resultDir)) {
|
||||
New-Item -ItemType Directory -Path $resultDir | Out-Null
|
||||
}
|
||||
$result | ConvertTo-Json -Depth 100 | Set-Content -Path $resultFullPath -Encoding UTF8
|
||||
Write-Host "C18Z2 live service-channel soak smoke passed. Result: $resultFullPath"
|
||||
$result
|
||||
@@ -0,0 +1,110 @@
|
||||
param(
|
||||
[string]$ApiBaseUrl = "http://192.168.200.61:18121/api/v1",
|
||||
[string]$ClusterID = "cfc0743d-d960-49fb-9de8-96e063d5e4aa",
|
||||
[string]$ActorUserID = "f67d943f-5397-4b3a-a229-695fe67ad700",
|
||||
[string]$EntryNodeName = "test-1",
|
||||
[string]$RelayNodeName = "test-3",
|
||||
[string]$ExitNodeName = "test-2",
|
||||
[string]$EntryBaseUrl = "http://192.168.200.61:19131",
|
||||
[string]$DockerSSH = "test-docker",
|
||||
[int]$InitialBatchCount = 12,
|
||||
[int]$LearningBatchCount = 24,
|
||||
[int]$PostChurnBatchCount = 24,
|
||||
[int]$PacketsPerBatch = 8,
|
||||
[int]$BatchDelayMilliseconds = 25,
|
||||
[string]$ResultPath = "artifacts\c18z20-service-channel-adaptive-window-telemetry-smoke-result.json"
|
||||
)
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
$scriptDir = Split-Path -Parent $MyInvocation.MyCommand.Path
|
||||
$repoRoot = (Resolve-Path (Join-Path $scriptDir "..\..")).ProviderPath
|
||||
$innerResultPath = "artifacts\c18z20-service-channel-adaptive-window-telemetry-inner-result.json"
|
||||
$innerResultFullPath = Join-Path $repoRoot $innerResultPath
|
||||
|
||||
$agentRoot = Join-Path $repoRoot "agents\rap-node-agent"
|
||||
Push-Location $agentRoot
|
||||
try {
|
||||
$unitTestOutput = & go test ./internal/vpnruntime -run "TestFabricFlowSchedulerRecommendsSmallerWindowUnderPressure|TestFabricClientPacketIngressParallelFlowWindowDoesNotBlockIndependentChannel" 2>&1
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
$unitText = ($unitTestOutput | Out-String)
|
||||
throw "C18Z20 adaptive window telemetry unit tests failed:`n$unitText"
|
||||
}
|
||||
} finally {
|
||||
Pop-Location
|
||||
}
|
||||
|
||||
& (Join-Path $scriptDir "c18z19-service-channel-parallel-flow-window-smoke.ps1") `
|
||||
-ApiBaseUrl $ApiBaseUrl `
|
||||
-ClusterID $ClusterID `
|
||||
-ActorUserID $ActorUserID `
|
||||
-EntryNodeName $EntryNodeName `
|
||||
-RelayNodeName $RelayNodeName `
|
||||
-ExitNodeName $ExitNodeName `
|
||||
-EntryBaseUrl $EntryBaseUrl `
|
||||
-DockerSSH $DockerSSH `
|
||||
-InitialBatchCount $InitialBatchCount `
|
||||
-LearningBatchCount $LearningBatchCount `
|
||||
-PostChurnBatchCount $PostChurnBatchCount `
|
||||
-PacketsPerBatch $PacketsPerBatch `
|
||||
-BatchDelayMilliseconds $BatchDelayMilliseconds `
|
||||
-ResultPath $innerResultPath
|
||||
|
||||
$result = Get-Content -Path $innerResultFullPath -Raw | ConvertFrom-Json
|
||||
$ingress = $result.telemetry.final_entry_ingress
|
||||
$scheduler = $ingress.flow_scheduler
|
||||
$stats = @()
|
||||
if ($null -ne $scheduler.channel_stats) {
|
||||
$stats = @($scheduler.channel_stats.PSObject.Properties | ForEach-Object { $_.Value })
|
||||
}
|
||||
$attemptStats = @($stats | Where-Object { $_.PSObject.Properties["send_attempts"] -and [int64]$_.send_attempts -gt 0 })
|
||||
$successStats = @($stats | Where-Object { $_.PSObject.Properties["send_successes"] -and [int64]$_.send_successes -gt 0 })
|
||||
$latencyStats = @($stats | Where-Object {
|
||||
(($_.PSObject.Properties["latency_le_10ms"] -and [int64]$_.latency_le_10ms -gt 0) -or
|
||||
($_.PSObject.Properties["latency_le_100ms"] -and [int64]$_.latency_le_100ms -gt 0) -or
|
||||
($_.PSObject.Properties["latency_le_1000ms"] -and [int64]$_.latency_le_1000ms -gt 0) -or
|
||||
($_.PSObject.Properties["latency_gt_1000ms"] -and [int64]$_.latency_gt_1000ms -gt 0))
|
||||
})
|
||||
$maxParallel = if ($ingress.PSObject.Properties["max_parallel_flow_sends"]) { [int]$ingress.max_parallel_flow_sends } else { 0 }
|
||||
$recommended = if ($ingress.PSObject.Properties["recommended_parallel_flow_sends"]) { [int]$ingress.recommended_parallel_flow_sends } else { 0 }
|
||||
$maxInFlight = if ($scheduler.PSObject.Properties["max_in_flight"]) { [int]$scheduler.max_in_flight } else { 0 }
|
||||
$inFlight = if ($scheduler.PSObject.Properties["in_flight"]) { [int]$scheduler.in_flight } else { -1 }
|
||||
|
||||
$result.schema_version = "c18z20.service_channel_adaptive_window_telemetry_smoke.v1"
|
||||
$result | Add-Member -NotePropertyName c18z20_checks -NotePropertyValue ([ordered]@{
|
||||
unit_adaptive_window_pressure_contract_passed = ($unitTestOutput -join "`n").Contains("ok")
|
||||
live_recommended_window_visible = ($recommended -gt 0 -and $recommended -le $maxParallel)
|
||||
live_inflight_telemetry_visible = ($maxInFlight -ge 2 -and $inFlight -eq 0)
|
||||
live_per_channel_attempt_telemetry_visible = ($attemptStats.Count -ge 2 -and $successStats.Count -ge 2)
|
||||
live_per_channel_latency_buckets_visible = ($latencyStats.Count -ge 2)
|
||||
live_parallel_path_still_clean = ([bool]$result.passed -and [int]$result.backend_fallback_queue.depth -eq 0 -and [int]$result.flow_drops.delta -eq 0)
|
||||
}) -Force
|
||||
$result | Add-Member -NotePropertyName c18z20_summary -NotePropertyValue ([ordered]@{
|
||||
unit_test_output = ($unitTestOutput | Out-String).Trim()
|
||||
max_parallel_flow_sends = $maxParallel
|
||||
recommended_parallel_flow_sends = $recommended
|
||||
scheduler_in_flight = $inFlight
|
||||
scheduler_max_in_flight = $maxInFlight
|
||||
attempt_channel_count = $attemptStats.Count
|
||||
success_channel_count = $successStats.Count
|
||||
latency_channel_count = $latencyStats.Count
|
||||
}) -Force
|
||||
$result.passed = [bool]($result.passed -and
|
||||
$result.c18z20_checks.unit_adaptive_window_pressure_contract_passed -and
|
||||
$result.c18z20_checks.live_recommended_window_visible -and
|
||||
$result.c18z20_checks.live_inflight_telemetry_visible -and
|
||||
$result.c18z20_checks.live_per_channel_attempt_telemetry_visible -and
|
||||
$result.c18z20_checks.live_per_channel_latency_buckets_visible -and
|
||||
$result.c18z20_checks.live_parallel_path_still_clean)
|
||||
|
||||
$resolvedResultPath = Join-Path $repoRoot $ResultPath
|
||||
$result | ConvertTo-Json -Depth 100 | Set-Content -Path $resolvedResultPath -Encoding UTF8
|
||||
Remove-Item -Path $innerResultFullPath -Force -ErrorAction SilentlyContinue
|
||||
|
||||
if (-not $result.passed) {
|
||||
throw "C18Z20 adaptive window telemetry smoke failed. Result: $resolvedResultPath"
|
||||
}
|
||||
|
||||
Write-Host "C18Z20 service-channel adaptive window telemetry smoke passed. Result: $resolvedResultPath"
|
||||
$result
|
||||
@@ -0,0 +1,115 @@
|
||||
param(
|
||||
[string]$ApiBaseUrl = "http://192.168.200.61:18121/api/v1",
|
||||
[string]$ClusterID = "cfc0743d-d960-49fb-9de8-96e063d5e4aa",
|
||||
[string]$ActorUserID = "f67d943f-5397-4b3a-a229-695fe67ad700",
|
||||
[string]$EntryNodeName = "test-1",
|
||||
[string]$RelayNodeName = "test-3",
|
||||
[string]$ExitNodeName = "test-2",
|
||||
[string]$EntryBaseUrl = "http://192.168.200.61:19131",
|
||||
[string]$DockerSSH = "test-docker",
|
||||
[int]$InitialBatchCount = 12,
|
||||
[int]$LearningBatchCount = 24,
|
||||
[int]$PostChurnBatchCount = 24,
|
||||
[int]$PacketsPerBatch = 8,
|
||||
[int]$BatchDelayMilliseconds = 25,
|
||||
[string]$ResultPath = "artifacts\c18z21-service-channel-rolling-quality-window-smoke-result.json"
|
||||
)
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
$scriptDir = Split-Path -Parent $MyInvocation.MyCommand.Path
|
||||
$repoRoot = (Resolve-Path (Join-Path $scriptDir "..\..")).ProviderPath
|
||||
$innerResultPath = "artifacts\c18z21-service-channel-rolling-quality-window-inner-result.json"
|
||||
$innerResultFullPath = Join-Path $repoRoot $innerResultPath
|
||||
|
||||
$agentRoot = Join-Path $repoRoot "agents\rap-node-agent"
|
||||
Push-Location $agentRoot
|
||||
try {
|
||||
$unitTestOutput = & go test ./internal/vpnruntime -run "TestFabricFlowSchedulerRollingQualityWindowForgetsOldPressure|TestFabricFlowSchedulerRecommendsSmallerWindowUnderPressure|TestFabricClientPacketIngressParallelFlowWindowDoesNotBlockIndependentChannel" 2>&1
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
$unitText = ($unitTestOutput | Out-String)
|
||||
throw "C18Z21 rolling quality window unit tests failed:`n$unitText"
|
||||
}
|
||||
} finally {
|
||||
Pop-Location
|
||||
}
|
||||
|
||||
& (Join-Path $scriptDir "c18z20-service-channel-adaptive-window-telemetry-smoke.ps1") `
|
||||
-ApiBaseUrl $ApiBaseUrl `
|
||||
-ClusterID $ClusterID `
|
||||
-ActorUserID $ActorUserID `
|
||||
-EntryNodeName $EntryNodeName `
|
||||
-RelayNodeName $RelayNodeName `
|
||||
-ExitNodeName $ExitNodeName `
|
||||
-EntryBaseUrl $EntryBaseUrl `
|
||||
-DockerSSH $DockerSSH `
|
||||
-InitialBatchCount $InitialBatchCount `
|
||||
-LearningBatchCount $LearningBatchCount `
|
||||
-PostChurnBatchCount $PostChurnBatchCount `
|
||||
-PacketsPerBatch $PacketsPerBatch `
|
||||
-BatchDelayMilliseconds $BatchDelayMilliseconds `
|
||||
-ResultPath $innerResultPath
|
||||
|
||||
$result = Get-Content -Path $innerResultFullPath -Raw | ConvertFrom-Json
|
||||
$ingress = $result.telemetry.final_entry_ingress
|
||||
$scheduler = $ingress.flow_scheduler
|
||||
$stats = @()
|
||||
if ($null -ne $scheduler.channel_stats) {
|
||||
$stats = @($scheduler.channel_stats.PSObject.Properties | ForEach-Object { $_.Value })
|
||||
}
|
||||
$rollingStats = @($stats | Where-Object {
|
||||
$_.PSObject.Properties["quality_window_sample_count"] -and [int]$_.quality_window_sample_count -gt 0
|
||||
})
|
||||
$rollingSuccessStats = @($stats | Where-Object {
|
||||
$_.PSObject.Properties["quality_window_success_count"] -and [int]$_.quality_window_success_count -gt 0
|
||||
})
|
||||
$rollingLatencyStats = @($stats | Where-Object {
|
||||
$_.PSObject.Properties["quality_window_avg_latency_ms"] -and [int64]$_.quality_window_avg_latency_ms -ge 0
|
||||
})
|
||||
$schedulerWindowSamples = if ($scheduler.PSObject.Properties["quality_window_sample_count"]) { [int]$scheduler.quality_window_sample_count } else { 0 }
|
||||
$schedulerWindowFailures = if ($scheduler.PSObject.Properties["quality_window_failure_count"]) { [int]$scheduler.quality_window_failure_count } else { -1 }
|
||||
$schedulerWindowDrops = if ($scheduler.PSObject.Properties["quality_window_drop_count"]) { [int]$scheduler.quality_window_drop_count } else { -1 }
|
||||
$maxParallel = if ($ingress.PSObject.Properties["max_parallel_flow_sends"]) { [int]$ingress.max_parallel_flow_sends } else { 0 }
|
||||
$recommended = if ($ingress.PSObject.Properties["recommended_parallel_flow_sends"]) { [int]$ingress.recommended_parallel_flow_sends } else { 0 }
|
||||
|
||||
$result.schema_version = "c18z21.service_channel_rolling_quality_window_smoke.v1"
|
||||
$result | Add-Member -NotePropertyName c18z21_checks -NotePropertyValue ([ordered]@{
|
||||
unit_rolling_quality_contract_passed = ($unitTestOutput -join "`n").Contains("ok")
|
||||
live_scheduler_window_telemetry_visible = ($schedulerWindowSamples -gt 0 -and $schedulerWindowFailures -ge 0 -and $schedulerWindowDrops -ge 0)
|
||||
live_per_channel_window_samples_visible = ($rollingStats.Count -ge 2)
|
||||
live_per_channel_window_success_visible = ($rollingSuccessStats.Count -ge 2)
|
||||
live_per_channel_window_latency_visible = ($rollingLatencyStats.Count -ge 2)
|
||||
live_adaptive_window_still_open = ($recommended -gt 0 -and $recommended -le $maxParallel)
|
||||
live_parallel_path_still_clean = ([bool]$result.passed -and [int]$result.backend_fallback_queue.depth -eq 0 -and [int]$result.flow_drops.delta -eq 0)
|
||||
}) -Force
|
||||
$result | Add-Member -NotePropertyName c18z21_summary -NotePropertyValue ([ordered]@{
|
||||
unit_test_output = ($unitTestOutput | Out-String).Trim()
|
||||
scheduler_quality_window_sample_count = $schedulerWindowSamples
|
||||
scheduler_quality_window_failure_count = $schedulerWindowFailures
|
||||
scheduler_quality_window_drop_count = $schedulerWindowDrops
|
||||
quality_window_channel_count = $rollingStats.Count
|
||||
quality_window_success_channel_count = $rollingSuccessStats.Count
|
||||
quality_window_latency_channel_count = $rollingLatencyStats.Count
|
||||
max_parallel_flow_sends = $maxParallel
|
||||
recommended_parallel_flow_sends = $recommended
|
||||
}) -Force
|
||||
$result.passed = [bool]($result.passed -and
|
||||
$result.c18z21_checks.unit_rolling_quality_contract_passed -and
|
||||
$result.c18z21_checks.live_scheduler_window_telemetry_visible -and
|
||||
$result.c18z21_checks.live_per_channel_window_samples_visible -and
|
||||
$result.c18z21_checks.live_per_channel_window_success_visible -and
|
||||
$result.c18z21_checks.live_per_channel_window_latency_visible -and
|
||||
$result.c18z21_checks.live_adaptive_window_still_open -and
|
||||
$result.c18z21_checks.live_parallel_path_still_clean)
|
||||
|
||||
$resolvedResultPath = Join-Path $repoRoot $ResultPath
|
||||
$result | ConvertTo-Json -Depth 100 | Set-Content -Path $resolvedResultPath -Encoding UTF8
|
||||
Remove-Item -Path $innerResultFullPath -Force -ErrorAction SilentlyContinue
|
||||
|
||||
if (-not $result.passed) {
|
||||
throw "C18Z21 rolling quality window smoke failed. Result: $resolvedResultPath"
|
||||
}
|
||||
|
||||
Write-Host "C18Z21 service-channel rolling quality window smoke passed. Result: $resolvedResultPath"
|
||||
$result
|
||||
@@ -0,0 +1,102 @@
|
||||
param(
|
||||
[string]$ApiBaseUrl = "http://192.168.200.61:18121/api/v1",
|
||||
[string]$ClusterID = "cfc0743d-d960-49fb-9de8-96e063d5e4aa",
|
||||
[string]$ActorUserID = "f67d943f-5397-4b3a-a229-695fe67ad700",
|
||||
[string]$EntryNodeName = "test-1",
|
||||
[string]$RelayNodeName = "test-3",
|
||||
[string]$ExitNodeName = "test-2",
|
||||
[string]$EntryBaseUrl = "http://192.168.200.61:19131",
|
||||
[string]$DockerSSH = "test-docker",
|
||||
[int]$InitialBatchCount = 12,
|
||||
[int]$LearningBatchCount = 24,
|
||||
[int]$PostChurnBatchCount = 24,
|
||||
[int]$PacketsPerBatch = 8,
|
||||
[int]$BatchDelayMilliseconds = 25,
|
||||
[string]$ResultPath = "artifacts\c18z22-service-channel-rolling-feedback-smoke-result.json"
|
||||
)
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
$scriptDir = Split-Path -Parent $MyInvocation.MyCommand.Path
|
||||
$repoRoot = (Resolve-Path (Join-Path $scriptDir "..\..")).ProviderPath
|
||||
$innerResultPath = "artifacts\c18z22-service-channel-rolling-feedback-inner-result.json"
|
||||
$innerResultFullPath = Join-Path $repoRoot $innerResultPath
|
||||
|
||||
$backendRoot = Join-Path $repoRoot "backend"
|
||||
Push-Location $backendRoot
|
||||
try {
|
||||
$unitTestOutput = & go test ./internal/modules/cluster -run "TestRecordHeartbeatUsesRollingQualityWindowForRouteFeedback|TestRecordHeartbeatPersistsServiceChannelRouteFeedbackForLaterLease" 2>&1
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
$unitText = ($unitTestOutput | Out-String)
|
||||
throw "C18Z22 rolling feedback backend unit tests failed:`n$unitText"
|
||||
}
|
||||
} finally {
|
||||
Pop-Location
|
||||
}
|
||||
|
||||
& (Join-Path $scriptDir "c18z21-service-channel-rolling-quality-window-smoke.ps1") `
|
||||
-ApiBaseUrl $ApiBaseUrl `
|
||||
-ClusterID $ClusterID `
|
||||
-ActorUserID $ActorUserID `
|
||||
-EntryNodeName $EntryNodeName `
|
||||
-RelayNodeName $RelayNodeName `
|
||||
-ExitNodeName $ExitNodeName `
|
||||
-EntryBaseUrl $EntryBaseUrl `
|
||||
-DockerSSH $DockerSSH `
|
||||
-InitialBatchCount $InitialBatchCount `
|
||||
-LearningBatchCount $LearningBatchCount `
|
||||
-PostChurnBatchCount $PostChurnBatchCount `
|
||||
-PacketsPerBatch $PacketsPerBatch `
|
||||
-BatchDelayMilliseconds $BatchDelayMilliseconds `
|
||||
-ResultPath $innerResultPath
|
||||
|
||||
$result = Get-Content -Path $innerResultFullPath -Raw | ConvertFrom-Json
|
||||
$entryNodeID = $result.entry_node.id
|
||||
$liveRouteID = [string]$result.live_feedback.route_id
|
||||
$routeFeedbackUrl = "$ApiBaseUrl/clusters/$ClusterID/fabric/service-channels/route-feedback?actor_user_id=$ActorUserID&reporter_node_id=$entryNodeID&route_id=$liveRouteID&service_class=vpn_packets&include_expired=true"
|
||||
$feedbackResponse = Invoke-RestMethod -Method Get -Uri $routeFeedbackUrl -TimeoutSec 30
|
||||
$feedbackItems = @($feedbackResponse.route_feedback)
|
||||
$rollingFeedbackItems = @($feedbackItems | Where-Object {
|
||||
$_.reasons -and @($_.reasons) -contains "service_channel_rolling_quality_window"
|
||||
})
|
||||
$healthyRollingItems = @($rollingFeedbackItems | Where-Object { $_.feedback_status -eq "healthy" })
|
||||
$rollingPayloadItems = @($rollingFeedbackItems | Where-Object {
|
||||
$_.payload -and $_.payload.PSObject.Properties["quality_window_sample_count"] -and [int]$_.payload.quality_window_sample_count -gt 0
|
||||
})
|
||||
|
||||
$result.schema_version = "c18z22.service_channel_rolling_feedback_smoke.v1"
|
||||
$result | Add-Member -NotePropertyName c18z22_checks -NotePropertyValue ([ordered]@{
|
||||
unit_backend_rolling_feedback_contract_passed = ($unitTestOutput -join "`n").Contains("ok")
|
||||
live_c18z21_still_passed = [bool]$result.passed
|
||||
live_route_feedback_visible = ($feedbackItems.Count -gt 0)
|
||||
live_rolling_feedback_reason_visible = ($rollingFeedbackItems.Count -gt 0)
|
||||
live_healthy_rolling_feedback_visible = ($healthyRollingItems.Count -gt 0)
|
||||
live_rolling_feedback_payload_visible = ($rollingPayloadItems.Count -gt 0)
|
||||
}) -Force
|
||||
$result | Add-Member -NotePropertyName c18z22_summary -NotePropertyValue ([ordered]@{
|
||||
unit_test_output = ($unitTestOutput | Out-String).Trim()
|
||||
route_feedback_count = $feedbackItems.Count
|
||||
rolling_feedback_count = $rollingFeedbackItems.Count
|
||||
healthy_rolling_feedback_count = $healthyRollingItems.Count
|
||||
rolling_payload_count = $rollingPayloadItems.Count
|
||||
backend_image = "rap-backend:fabric-service-channel-0.2.197"
|
||||
node_agent_version = "0.2.196"
|
||||
}) -Force
|
||||
$result.passed = [bool]($result.passed -and
|
||||
$result.c18z22_checks.unit_backend_rolling_feedback_contract_passed -and
|
||||
$result.c18z22_checks.live_route_feedback_visible -and
|
||||
$result.c18z22_checks.live_rolling_feedback_reason_visible -and
|
||||
$result.c18z22_checks.live_healthy_rolling_feedback_visible -and
|
||||
$result.c18z22_checks.live_rolling_feedback_payload_visible)
|
||||
|
||||
$resolvedResultPath = Join-Path $repoRoot $ResultPath
|
||||
$result | ConvertTo-Json -Depth 100 | Set-Content -Path $resolvedResultPath -Encoding UTF8
|
||||
Remove-Item -Path $innerResultFullPath -Force -ErrorAction SilentlyContinue
|
||||
|
||||
if (-not $result.passed) {
|
||||
throw "C18Z22 rolling feedback smoke failed. Result: $resolvedResultPath"
|
||||
}
|
||||
|
||||
Write-Host "C18Z22 service-channel rolling feedback smoke passed. Result: $resolvedResultPath"
|
||||
$result
|
||||
@@ -0,0 +1,88 @@
|
||||
param(
|
||||
[string]$ApiBaseUrl = "http://192.168.200.61:18121/api/v1",
|
||||
[string]$ClusterID = "cfc0743d-d960-49fb-9de8-96e063d5e4aa",
|
||||
[string]$ActorUserID = "f67d943f-5397-4b3a-a229-695fe67ad700",
|
||||
[string]$EntryNodeName = "test-1",
|
||||
[string]$RelayNodeName = "test-3",
|
||||
[string]$ExitNodeName = "test-2",
|
||||
[string]$EntryBaseUrl = "http://192.168.200.61:19131",
|
||||
[string]$DockerSSH = "test-docker",
|
||||
[int]$InitialBatchCount = 12,
|
||||
[int]$LearningBatchCount = 24,
|
||||
[int]$PostChurnBatchCount = 24,
|
||||
[int]$PacketsPerBatch = 8,
|
||||
[int]$BatchDelayMilliseconds = 25,
|
||||
[string]$ResultPath = "artifacts\c18z23-service-channel-recovery-hysteresis-smoke-result.json"
|
||||
)
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
$scriptDir = Split-Path -Parent $MyInvocation.MyCommand.Path
|
||||
$repoRoot = (Resolve-Path (Join-Path $scriptDir "..\..")).ProviderPath
|
||||
$innerResultPath = "artifacts\c18z23-service-channel-recovery-hysteresis-inner-result.json"
|
||||
$innerResultFullPath = Join-Path $repoRoot $innerResultPath
|
||||
|
||||
$backendRoot = Join-Path $repoRoot "backend"
|
||||
Push-Location $backendRoot
|
||||
try {
|
||||
$unitTestOutput = & go test ./internal/modules/cluster -run "TestIssueFabricServiceChannelLeaseDampensRecoveredRouteDuringRetryCooldown|TestRecordHeartbeatUsesRollingQualityWindowForRouteFeedback" 2>&1
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
$unitText = ($unitTestOutput | Out-String)
|
||||
throw "C18Z23 recovery hysteresis backend unit tests failed:`n$unitText"
|
||||
}
|
||||
} finally {
|
||||
Pop-Location
|
||||
}
|
||||
|
||||
& (Join-Path $scriptDir "c18z22-service-channel-rolling-feedback-smoke.ps1") `
|
||||
-ApiBaseUrl $ApiBaseUrl `
|
||||
-ClusterID $ClusterID `
|
||||
-ActorUserID $ActorUserID `
|
||||
-EntryNodeName $EntryNodeName `
|
||||
-RelayNodeName $RelayNodeName `
|
||||
-ExitNodeName $ExitNodeName `
|
||||
-EntryBaseUrl $EntryBaseUrl `
|
||||
-DockerSSH $DockerSSH `
|
||||
-InitialBatchCount $InitialBatchCount `
|
||||
-LearningBatchCount $LearningBatchCount `
|
||||
-PostChurnBatchCount $PostChurnBatchCount `
|
||||
-PacketsPerBatch $PacketsPerBatch `
|
||||
-BatchDelayMilliseconds $BatchDelayMilliseconds `
|
||||
-ResultPath $innerResultPath
|
||||
|
||||
$result = Get-Content -Path $innerResultFullPath -Raw | ConvertFrom-Json
|
||||
$backendLine = (& ssh $DockerSSH "docker ps --format '{{.Names}} {{.Image}} {{.Status}}' | grep '^rap_test_backend '") | Out-String
|
||||
$backendImageOK = $backendLine.Contains("rap-backend:fabric-service-channel-0.2.198")
|
||||
|
||||
$result.schema_version = "c18z23.service_channel_recovery_hysteresis_smoke.v1"
|
||||
$result | Add-Member -NotePropertyName c18z23_checks -NotePropertyValue ([ordered]@{
|
||||
unit_recovery_hysteresis_contract_passed = ($unitTestOutput -join "`n").Contains("ok")
|
||||
live_c18z22_still_passed = [bool]$result.passed
|
||||
backend_0_2_198_deployed = $backendImageOK
|
||||
live_rolling_feedback_still_visible = ($result.c18z22_summary.rolling_feedback_count -gt 0 -and $result.c18z22_summary.rolling_payload_count -gt 0)
|
||||
live_parallel_path_still_clean = ([int]$result.backend_fallback_queue.depth -eq 0 -and [int]$result.flow_drops.delta -eq 0)
|
||||
}) -Force
|
||||
$result | Add-Member -NotePropertyName c18z23_summary -NotePropertyValue ([ordered]@{
|
||||
unit_test_output = ($unitTestOutput | Out-String).Trim()
|
||||
backend_container = $backendLine.Trim()
|
||||
recovery_hysteresis_penalty = 150
|
||||
backend_image = "rap-backend:fabric-service-channel-0.2.198"
|
||||
node_agent_version = "0.2.196"
|
||||
}) -Force
|
||||
$result.passed = [bool]($result.passed -and
|
||||
$result.c18z23_checks.unit_recovery_hysteresis_contract_passed -and
|
||||
$result.c18z23_checks.backend_0_2_198_deployed -and
|
||||
$result.c18z23_checks.live_rolling_feedback_still_visible -and
|
||||
$result.c18z23_checks.live_parallel_path_still_clean)
|
||||
|
||||
$resolvedResultPath = Join-Path $repoRoot $ResultPath
|
||||
$result | ConvertTo-Json -Depth 100 | Set-Content -Path $resolvedResultPath -Encoding UTF8
|
||||
Remove-Item -Path $innerResultFullPath -Force -ErrorAction SilentlyContinue
|
||||
|
||||
if (-not $result.passed) {
|
||||
throw "C18Z23 recovery hysteresis smoke failed. Result: $resolvedResultPath"
|
||||
}
|
||||
|
||||
Write-Host "C18Z23 service-channel recovery hysteresis smoke passed. Result: $resolvedResultPath"
|
||||
$result
|
||||
@@ -0,0 +1,73 @@
|
||||
param(
|
||||
[string]$ApiBaseUrl = "http://192.168.200.61:18121/api/v1",
|
||||
[string]$ClusterID = "cfc0743d-d960-49fb-9de8-96e063d5e4aa",
|
||||
[string]$ActorUserID = "f67d943f-5397-4b3a-a229-695fe67ad700",
|
||||
[string]$DockerSSH = "test-docker",
|
||||
[string]$ExpectedBackendImage = "rap-backend:fabric-service-channel-0.2.199",
|
||||
[string]$ResultPath = "artifacts\c18z24-service-channel-recovery-visibility-smoke-result.json"
|
||||
)
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
$scriptDir = Split-Path -Parent $MyInvocation.MyCommand.Path
|
||||
$repoRoot = (Resolve-Path (Join-Path $scriptDir "..\..")).ProviderPath
|
||||
$backendRoot = Join-Path $repoRoot "backend"
|
||||
|
||||
Push-Location $backendRoot
|
||||
try {
|
||||
$unitTestOutput = & go test ./internal/modules/cluster -run "TestIssueFabricServiceChannelLeaseDampensRecoveredRouteDuringRetryCooldown|TestServiceChannelRouteFeedbackReportExposesRecoveryState|TestRoutePathDecisionReportCountsRecoveryHysteresis" 2>&1
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
$unitText = ($unitTestOutput | Out-String)
|
||||
throw "C18Z24 recovery visibility backend unit tests failed:`n$unitText"
|
||||
}
|
||||
} finally {
|
||||
Pop-Location
|
||||
}
|
||||
|
||||
$feedbackUrl = "$ApiBaseUrl/clusters/$ClusterID/fabric/service-channels/route-feedback?actor_user_id=$ActorUserID&include_expired=true"
|
||||
$feedbackResponse = Invoke-RestMethod -Method Get -Uri $feedbackUrl
|
||||
$feedbackItems = @($feedbackResponse.route_feedback)
|
||||
$feedbackHasRecoveryShape = $true
|
||||
foreach ($item in $feedbackItems) {
|
||||
$properties = @($item.PSObject.Properties.Name)
|
||||
if (-not ($properties -contains "recovery_state")) {
|
||||
$feedbackHasRecoveryShape = $false
|
||||
}
|
||||
}
|
||||
$recoveredItems = @($feedbackItems | Where-Object {
|
||||
$properties = @($_.PSObject.Properties.Name)
|
||||
$_.recovery_state -eq "recovered" -or (($properties -contains "recovery_hysteresis_active") -and $_.recovery_hysteresis_active)
|
||||
})
|
||||
|
||||
$backendLine = (& ssh $DockerSSH "docker ps --format '{{.Names}} {{.Image}} {{.Status}}' | grep '^rap_test_backend '") | Out-String
|
||||
$backendImageOK = $backendLine.Contains($ExpectedBackendImage)
|
||||
|
||||
$result = [ordered]@{
|
||||
schema_version = "c18z24.service_channel_recovery_visibility_smoke.v1"
|
||||
passed = [bool]($backendImageOK -and $feedbackHasRecoveryShape)
|
||||
checks = [ordered]@{
|
||||
unit_recovery_visibility_contract_passed = ($unitTestOutput -join "`n").Contains("ok")
|
||||
backend_expected_image_deployed = $backendImageOK
|
||||
route_feedback_api_has_recovery_shape = $feedbackHasRecoveryShape
|
||||
}
|
||||
summary = [ordered]@{
|
||||
unit_test_output = ($unitTestOutput | Out-String).Trim()
|
||||
backend_container = $backendLine.Trim()
|
||||
expected_backend_image = $ExpectedBackendImage
|
||||
route_feedback_count = $feedbackItems.Count
|
||||
recovered_route_count = $recoveredItems.Count
|
||||
fenced_route_count = @($feedbackItems | Where-Object { $_.recovery_state -eq "fenced" }).Count
|
||||
healthy_route_count = @($feedbackItems | Where-Object { $_.recovery_state -eq "healthy" }).Count
|
||||
}
|
||||
}
|
||||
|
||||
$resolvedResultPath = Join-Path $repoRoot $ResultPath
|
||||
$result | ConvertTo-Json -Depth 100 | Set-Content -Path $resolvedResultPath -Encoding UTF8
|
||||
|
||||
if (-not $result.passed) {
|
||||
throw "C18Z24 recovery visibility smoke failed. Result: $resolvedResultPath"
|
||||
}
|
||||
|
||||
Write-Host "C18Z24 service-channel recovery visibility smoke passed. Result: $resolvedResultPath"
|
||||
$result
|
||||
@@ -0,0 +1,71 @@
|
||||
param(
|
||||
[string]$ApiBaseUrl = "http://192.168.200.61:18121/api/v1",
|
||||
[string]$ClusterID = "cfc0743d-d960-49fb-9de8-96e063d5e4aa",
|
||||
[string]$ActorUserID = "f67d943f-5397-4b3a-a229-695fe67ad700",
|
||||
[string]$DockerSSH = "test-docker",
|
||||
[string]$ExpectedBackendImage = "rap-backend:fabric-service-channel-0.2.200",
|
||||
[string]$ResultPath = "artifacts\c18z25-service-channel-recovery-promotion-smoke-result.json"
|
||||
)
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
$scriptDir = Split-Path -Parent $MyInvocation.MyCommand.Path
|
||||
$repoRoot = (Resolve-Path (Join-Path $scriptDir "..\..")).ProviderPath
|
||||
$backendRoot = Join-Path $repoRoot "backend"
|
||||
|
||||
Push-Location $backendRoot
|
||||
try {
|
||||
$unitTestOutput = & go test ./internal/modules/cluster -run "TestIssueFabricServiceChannelLeaseDampensRecoveredRouteDuringRetryCooldown|TestServiceChannelRouteFeedbackReportExposesRecoveryState|TestFabricServiceChannelRecoveryPromotionRemovesHysteresisPenalty|TestRoutePathDecisionReportCountsRecoveryHysteresis" 2>&1
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
$unitText = ($unitTestOutput | Out-String)
|
||||
throw "C18Z25 recovery promotion backend unit tests failed:`n$unitText"
|
||||
}
|
||||
} finally {
|
||||
Pop-Location
|
||||
}
|
||||
|
||||
$feedbackUrl = "$ApiBaseUrl/clusters/$ClusterID/fabric/service-channels/route-feedback?actor_user_id=$ActorUserID&include_expired=true"
|
||||
$feedbackResponse = Invoke-RestMethod -Method Get -Uri $feedbackUrl
|
||||
$feedbackItems = @($feedbackResponse.route_feedback)
|
||||
$feedbackHasRecoveryState = $true
|
||||
foreach ($item in $feedbackItems) {
|
||||
$properties = @($item.PSObject.Properties.Name)
|
||||
if (-not ($properties -contains "recovery_state")) {
|
||||
$feedbackHasRecoveryState = $false
|
||||
}
|
||||
}
|
||||
|
||||
$backendLine = (& ssh $DockerSSH "docker ps --format '{{.Names}} {{.Image}} {{.Status}}' | grep '^rap_test_backend '") | Out-String
|
||||
$backendImageOK = $backendLine.Contains($ExpectedBackendImage)
|
||||
|
||||
$result = [ordered]@{
|
||||
schema_version = "c18z25.service_channel_recovery_promotion_smoke.v1"
|
||||
passed = [bool]($backendImageOK -and $feedbackHasRecoveryState)
|
||||
checks = [ordered]@{
|
||||
unit_recovery_promotion_contract_passed = ($unitTestOutput -join "`n").Contains("ok")
|
||||
backend_expected_image_deployed = $backendImageOK
|
||||
route_feedback_api_has_recovery_state = $feedbackHasRecoveryState
|
||||
}
|
||||
summary = [ordered]@{
|
||||
unit_test_output = ($unitTestOutput | Out-String).Trim()
|
||||
backend_container = $backendLine.Trim()
|
||||
expected_backend_image = $ExpectedBackendImage
|
||||
promotion_min_samples = 64
|
||||
route_feedback_count = $feedbackItems.Count
|
||||
promoted_route_count = @($feedbackItems | Where-Object {
|
||||
$properties = @($_.PSObject.Properties.Name)
|
||||
($properties -contains "recovery_promoted") -and $_.recovery_promoted
|
||||
}).Count
|
||||
}
|
||||
}
|
||||
|
||||
$resolvedResultPath = Join-Path $repoRoot $ResultPath
|
||||
$result | ConvertTo-Json -Depth 100 | Set-Content -Path $resolvedResultPath -Encoding UTF8
|
||||
|
||||
if (-not $result.passed) {
|
||||
throw "C18Z25 recovery promotion smoke failed. Result: $resolvedResultPath"
|
||||
}
|
||||
|
||||
Write-Host "C18Z25 service-channel recovery promotion smoke passed. Result: $resolvedResultPath"
|
||||
$result
|
||||
@@ -0,0 +1,70 @@
|
||||
param(
|
||||
[string]$ApiBaseUrl = "http://192.168.200.61:18121/api/v1",
|
||||
[string]$ClusterID = "cfc0743d-d960-49fb-9de8-96e063d5e4aa",
|
||||
[string]$ActorUserID = "f67d943f-5397-4b3a-a229-695fe67ad700",
|
||||
[string]$DockerSSH = "test-docker",
|
||||
[string]$ExpectedBackendImage = "rap-backend:fabric-service-channel-0.2.201",
|
||||
[string]$ResultPath = "artifacts\c18z26-service-channel-recovery-demotion-smoke-result.json"
|
||||
)
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
$scriptDir = Split-Path -Parent $MyInvocation.MyCommand.Path
|
||||
$repoRoot = (Resolve-Path (Join-Path $scriptDir "..\..")).ProviderPath
|
||||
$backendRoot = Join-Path $repoRoot "backend"
|
||||
|
||||
Push-Location $backendRoot
|
||||
try {
|
||||
$unitTestOutput = & go test ./internal/modules/cluster -run "TestServiceChannelRouteFeedbackReportExposesRecoveryState|TestFabricServiceChannelRecoveryPromotionRemovesHysteresisPenalty|TestFabricServiceChannelRecoveryDemotionMarksRouteReason|TestRoutePathDecisionReportCountsRecoveryHysteresis" 2>&1
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
$unitText = ($unitTestOutput | Out-String)
|
||||
throw "C18Z26 recovery demotion backend unit tests failed:`n$unitText"
|
||||
}
|
||||
} finally {
|
||||
Pop-Location
|
||||
}
|
||||
|
||||
$feedbackUrl = "$ApiBaseUrl/clusters/$ClusterID/fabric/service-channels/route-feedback?actor_user_id=$ActorUserID&include_expired=true"
|
||||
$feedbackResponse = Invoke-RestMethod -Method Get -Uri $feedbackUrl
|
||||
$feedbackItems = @($feedbackResponse.route_feedback)
|
||||
$feedbackHasRecoveryState = $true
|
||||
foreach ($item in $feedbackItems) {
|
||||
$properties = @($item.PSObject.Properties.Name)
|
||||
if (-not ($properties -contains "recovery_state")) {
|
||||
$feedbackHasRecoveryState = $false
|
||||
}
|
||||
}
|
||||
|
||||
$backendLine = (& ssh $DockerSSH "docker ps --format '{{.Names}} {{.Image}} {{.Status}}' | grep '^rap_test_backend '") | Out-String
|
||||
$backendImageOK = $backendLine.Contains($ExpectedBackendImage)
|
||||
|
||||
$result = [ordered]@{
|
||||
schema_version = "c18z26.service_channel_recovery_demotion_smoke.v1"
|
||||
passed = [bool]($backendImageOK -and $feedbackHasRecoveryState)
|
||||
checks = [ordered]@{
|
||||
unit_recovery_demotion_contract_passed = ($unitTestOutput -join "`n").Contains("ok")
|
||||
backend_expected_image_deployed = $backendImageOK
|
||||
route_feedback_api_has_recovery_state = $feedbackHasRecoveryState
|
||||
}
|
||||
summary = [ordered]@{
|
||||
unit_test_output = ($unitTestOutput | Out-String).Trim()
|
||||
backend_container = $backendLine.Trim()
|
||||
expected_backend_image = $ExpectedBackendImage
|
||||
route_feedback_count = $feedbackItems.Count
|
||||
demoted_route_count = @($feedbackItems | Where-Object {
|
||||
$properties = @($_.PSObject.Properties.Name)
|
||||
($properties -contains "recovery_demoted") -and $_.recovery_demoted
|
||||
}).Count
|
||||
}
|
||||
}
|
||||
|
||||
$resolvedResultPath = Join-Path $repoRoot $ResultPath
|
||||
$result | ConvertTo-Json -Depth 100 | Set-Content -Path $resolvedResultPath -Encoding UTF8
|
||||
|
||||
if (-not $result.passed) {
|
||||
throw "C18Z26 recovery demotion smoke failed. Result: $resolvedResultPath"
|
||||
}
|
||||
|
||||
Write-Host "C18Z26 service-channel recovery demotion smoke passed. Result: $resolvedResultPath"
|
||||
$result
|
||||
@@ -0,0 +1,96 @@
|
||||
param(
|
||||
[string]$ApiBaseUrl = "http://192.168.200.61:18121/api/v1",
|
||||
[string]$ClusterID = "cfc0743d-d960-49fb-9de8-96e063d5e4aa",
|
||||
[string]$ActorUserID = "f67d943f-5397-4b3a-a229-695fe67ad700",
|
||||
[string]$DockerSSH = "test-docker",
|
||||
[string]$ExpectedBackendImage = "rap-backend:fabric-service-channel-0.2.202",
|
||||
[string]$ResultPath = "artifacts\c18z27-service-channel-recovery-policy-smoke-result.json"
|
||||
)
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
$scriptDir = Split-Path -Parent $MyInvocation.MyCommand.Path
|
||||
$repoRoot = (Resolve-Path (Join-Path $scriptDir "..\..")).ProviderPath
|
||||
$backendRoot = Join-Path $repoRoot "backend"
|
||||
|
||||
Push-Location $backendRoot
|
||||
try {
|
||||
$unitTestOutput = & go test ./internal/modules/cluster -run "TestFabricServiceChannelRecoveryPolicyControlsPromotionAndPenalty|TestUpdateFabricServiceChannelRecoveryPolicyPersistsClusterMetadata|TestFabricServiceChannelRecoveryDemotionMarksRouteReason" 2>&1
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
$unitText = ($unitTestOutput | Out-String)
|
||||
throw "C18Z27 recovery policy backend unit tests failed:`n$unitText"
|
||||
}
|
||||
} finally {
|
||||
Pop-Location
|
||||
}
|
||||
|
||||
$policyUrl = "$ApiBaseUrl/clusters/$ClusterID/fabric/service-channels/recovery-policy"
|
||||
$getUrl = "$policyUrl`?actor_user_id=$ActorUserID"
|
||||
$before = Invoke-RestMethod -Method Get -Uri $getUrl
|
||||
$beforePolicy = $before.fabric_service_channel_recovery_policy
|
||||
|
||||
$desired = @{
|
||||
actor_user_id = $ActorUserID
|
||||
hysteresis_penalty = 151
|
||||
promotion_min_samples = 65
|
||||
demotion_failure_threshold = 2
|
||||
demotion_drop_threshold = 2
|
||||
demotion_slow_threshold = 3
|
||||
demotion_rebuild_enabled = $true
|
||||
demotion_fenced_enabled = $true
|
||||
}
|
||||
$updated = Invoke-RestMethod -Method Put -Uri $policyUrl -ContentType "application/json" -Body ($desired | ConvertTo-Json -Depth 20)
|
||||
$updatedPolicy = $updated.fabric_service_channel_recovery_policy
|
||||
|
||||
$restore = @{
|
||||
actor_user_id = $ActorUserID
|
||||
hysteresis_penalty = 150
|
||||
promotion_min_samples = 64
|
||||
demotion_failure_threshold = 1
|
||||
demotion_drop_threshold = 1
|
||||
demotion_slow_threshold = 1
|
||||
demotion_rebuild_enabled = $true
|
||||
demotion_fenced_enabled = $true
|
||||
}
|
||||
$restored = Invoke-RestMethod -Method Put -Uri $policyUrl -ContentType "application/json" -Body ($restore | ConvertTo-Json -Depth 20)
|
||||
$restoredPolicy = $restored.fabric_service_channel_recovery_policy
|
||||
|
||||
$backendLine = (& ssh $DockerSSH "docker ps --format '{{.Names}} {{.Image}} {{.Status}}' | grep '^rap_test_backend '") | Out-String
|
||||
$backendImageOK = $backendLine.Contains($ExpectedBackendImage)
|
||||
$policyUpdatedOK = [int]$updatedPolicy.hysteresis_penalty -eq 151 -and
|
||||
[int]$updatedPolicy.promotion_min_samples -eq 65 -and
|
||||
[int]$updatedPolicy.demotion_failure_threshold -eq 2 -and
|
||||
[int]$updatedPolicy.demotion_slow_threshold -eq 3
|
||||
$policyRestoredOK = [int]$restoredPolicy.hysteresis_penalty -eq 150 -and
|
||||
[int]$restoredPolicy.promotion_min_samples -eq 64 -and
|
||||
[int]$restoredPolicy.demotion_failure_threshold -eq 1
|
||||
|
||||
$result = [ordered]@{
|
||||
schema_version = "c18z27.service_channel_recovery_policy_smoke.v1"
|
||||
passed = [bool]($backendImageOK -and $policyUpdatedOK -and $policyRestoredOK)
|
||||
checks = [ordered]@{
|
||||
unit_recovery_policy_contract_passed = ($unitTestOutput -join "`n").Contains("ok")
|
||||
backend_expected_image_deployed = $backendImageOK
|
||||
recovery_policy_update_api_applied = $policyUpdatedOK
|
||||
recovery_policy_restored_defaults = $policyRestoredOK
|
||||
}
|
||||
summary = [ordered]@{
|
||||
unit_test_output = ($unitTestOutput | Out-String).Trim()
|
||||
backend_container = $backendLine.Trim()
|
||||
expected_backend_image = $ExpectedBackendImage
|
||||
before_policy = $beforePolicy
|
||||
updated_policy = $updatedPolicy
|
||||
restored_policy = $restoredPolicy
|
||||
}
|
||||
}
|
||||
|
||||
$resolvedResultPath = Join-Path $repoRoot $ResultPath
|
||||
$result | ConvertTo-Json -Depth 100 | Set-Content -Path $resolvedResultPath -Encoding UTF8
|
||||
|
||||
if (-not $result.passed) {
|
||||
throw "C18Z27 recovery policy smoke failed. Result: $resolvedResultPath"
|
||||
}
|
||||
|
||||
Write-Host "C18Z27 service-channel recovery policy smoke passed. Result: $resolvedResultPath"
|
||||
$result
|
||||
@@ -0,0 +1,101 @@
|
||||
param(
|
||||
[string]$ApiBaseUrl = "http://192.168.200.61:18121/api/v1",
|
||||
[string]$ClusterID = "cfc0743d-d960-49fb-9de8-96e063d5e4aa",
|
||||
[string]$DockerSSH = "test-docker",
|
||||
[string]$ExpectedBackendImage = "rap-backend:fabric-service-channel-0.2.203",
|
||||
[string]$EntryNodeID = "108a0d66-d65e-4dea-b9a8-135366bf7dba",
|
||||
[string]$ExitNodeID = "830a26de-e7e8-462f-8f37-5189027955d5",
|
||||
[string]$ResultPath = "artifacts\c18z28-service-channel-recovery-policy-provenance-smoke-result.json"
|
||||
)
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
$scriptDir = Split-Path -Parent $MyInvocation.MyCommand.Path
|
||||
$repoRoot = (Resolve-Path (Join-Path $scriptDir "..\..")).ProviderPath
|
||||
$backendRoot = Join-Path $repoRoot "backend"
|
||||
|
||||
Push-Location $backendRoot
|
||||
try {
|
||||
$unitTestOutput = & go test ./internal/modules/cluster -run "TestIssueFabricServiceChannelLeaseSelectsAuthorizedRoute|TestFabricServiceChannelRecoveryPolicyControlsPromotionAndPenalty|TestRoutePathDecisionReportCountsRecoveryHysteresis" 2>&1
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
$unitText = ($unitTestOutput | Out-String)
|
||||
throw "C18Z28 recovery policy provenance backend unit tests failed:`n$unitText"
|
||||
}
|
||||
} finally {
|
||||
Pop-Location
|
||||
}
|
||||
|
||||
$syntheticUrl = "$ApiBaseUrl/clusters/$ClusterID/nodes/$EntryNodeID/mesh/synthetic-config"
|
||||
$synthetic = Invoke-RestMethod -Method Get -Uri $syntheticUrl
|
||||
$syntheticConfig = $synthetic.synthetic_mesh_config
|
||||
$feedbackPolicy = $syntheticConfig.service_channel_route_feedback.recovery_policy
|
||||
$pathPolicy = $null
|
||||
$routePathDecisions = $null
|
||||
$routePathProperty = $syntheticConfig.PSObject.Properties["route_path_decisions"]
|
||||
if ($null -ne $routePathProperty) {
|
||||
$routePathDecisions = $routePathProperty.Value
|
||||
}
|
||||
if ($null -ne $routePathDecisions) {
|
||||
$pathPolicy = $routePathDecisions.recovery_policy
|
||||
}
|
||||
|
||||
$leaseBody = @{
|
||||
organization_id = "org-c18z28-smoke"
|
||||
user_id = "user-c18z28-smoke"
|
||||
resource_id = "vpn-c18z28-smoke"
|
||||
service_class = "vpn_packets"
|
||||
entry_node_ids = @($EntryNodeID)
|
||||
exit_node_ids = @($ExitNodeID)
|
||||
preferred_entry_node_id = $EntryNodeID
|
||||
preferred_exit_node_id = $ExitNodeID
|
||||
allowed_channels = @("vpn_packet")
|
||||
ttl_seconds = 60
|
||||
}
|
||||
$lease = Invoke-RestMethod -Method Post -Uri "$ApiBaseUrl/clusters/$ClusterID/fabric/service-channels/leases" -ContentType "application/json" -Body ($leaseBody | ConvertTo-Json -Depth 20)
|
||||
$leaseItem = $lease.fabric_service_channel_lease
|
||||
|
||||
$backendLine = (& ssh $DockerSSH "docker ps --format '{{.Names}} {{.Image}} {{.Status}}' | grep '^rap_test_backend '") | Out-String
|
||||
$backendImageOK = $backendLine.Contains($ExpectedBackendImage)
|
||||
$feedbackPolicyOK = $null -ne $feedbackPolicy -and [int]$feedbackPolicy.hysteresis_penalty -gt 0 -and [string]$feedbackPolicy.source -ne ""
|
||||
$leasePolicyOK = $null -ne $leaseItem.recovery_policy -and [int]$leaseItem.recovery_policy.promotion_min_samples -gt 0
|
||||
$routePolicyOK = $null -ne $leaseItem.primary_route.recovery_policy -and [int]$leaseItem.primary_route.recovery_policy.demotion_failure_threshold -gt 0
|
||||
$signedPolicyOK = $null -ne $leaseItem.authority_payload.recovery_policy -and [int]$leaseItem.authority_payload.recovery_policy.hysteresis_penalty -gt 0
|
||||
$pathPolicyOK = $true
|
||||
if ($null -ne $routePathDecisions) {
|
||||
$pathPolicyOK = $null -ne $pathPolicy -and [string]$pathPolicy.source -ne ""
|
||||
}
|
||||
|
||||
$result = [ordered]@{
|
||||
schema_version = "c18z28.service_channel_recovery_policy_provenance_smoke.v1"
|
||||
passed = [bool]($backendImageOK -and $feedbackPolicyOK -and $leasePolicyOK -and $routePolicyOK -and $signedPolicyOK -and $pathPolicyOK)
|
||||
checks = [ordered]@{
|
||||
unit_recovery_policy_provenance_passed = ($unitTestOutput -join "`n").Contains("ok")
|
||||
backend_expected_image_deployed = $backendImageOK
|
||||
synthetic_feedback_policy_present = $feedbackPolicyOK
|
||||
synthetic_route_path_policy_present_when_report_exists = $pathPolicyOK
|
||||
lease_policy_present = $leasePolicyOK
|
||||
lease_primary_route_policy_present = $routePolicyOK
|
||||
signed_authority_payload_policy_present = $signedPolicyOK
|
||||
}
|
||||
summary = [ordered]@{
|
||||
unit_test_output = ($unitTestOutput | Out-String).Trim()
|
||||
backend_container = $backendLine.Trim()
|
||||
expected_backend_image = $ExpectedBackendImage
|
||||
synthetic_feedback_policy = $feedbackPolicy
|
||||
synthetic_route_path_policy = $pathPolicy
|
||||
lease_policy = $leaseItem.recovery_policy
|
||||
primary_route_policy = $leaseItem.primary_route.recovery_policy
|
||||
signed_payload_policy = $leaseItem.authority_payload.recovery_policy
|
||||
}
|
||||
}
|
||||
|
||||
$resolvedResultPath = Join-Path $repoRoot $ResultPath
|
||||
$result | ConvertTo-Json -Depth 100 | Set-Content -Path $resolvedResultPath -Encoding UTF8
|
||||
|
||||
if (-not $result.passed) {
|
||||
throw "C18Z28 recovery policy provenance smoke failed. Result: $resolvedResultPath"
|
||||
}
|
||||
|
||||
Write-Host "C18Z28 service-channel recovery policy provenance smoke passed. Result: $resolvedResultPath"
|
||||
$result
|
||||
@@ -0,0 +1,74 @@
|
||||
param(
|
||||
[string]$ApiBaseUrl = "http://192.168.200.61:18121/api/v1",
|
||||
[string]$ClusterID = "cfc0743d-d960-49fb-9de8-96e063d5e4aa",
|
||||
[string]$NodeID = "108a0d66-d65e-4dea-b9a8-135366bf7dba",
|
||||
[string]$DockerSSH = "test-docker",
|
||||
[string]$ExpectedBackendImage = "rap-backend:fabric-service-channel-0.2.204",
|
||||
[string]$ResultPath = "artifacts\c18z29-service-channel-feedback-provenance-guard-smoke-result.json"
|
||||
)
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
$scriptDir = Split-Path -Parent $MyInvocation.MyCommand.Path
|
||||
$repoRoot = (Resolve-Path (Join-Path $scriptDir "..\..")).ProviderPath
|
||||
$backendRoot = Join-Path $repoRoot "backend"
|
||||
|
||||
Push-Location $backendRoot
|
||||
try {
|
||||
$unitTestOutput = & go test ./internal/modules/cluster -run "TestFabricServiceChannelFeedbackStalePolicyIsConservative|TestFabricServiceChannelFeedbackMissingProvenanceIsVisibleButCompatible|TestIssueFabricServiceChannelLeaseSelectsAuthorizedRoute" 2>&1
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
$unitText = ($unitTestOutput | Out-String)
|
||||
throw "C18Z29 feedback provenance guard unit tests failed:`n$unitText"
|
||||
}
|
||||
} finally {
|
||||
Pop-Location
|
||||
}
|
||||
|
||||
$synthetic = Invoke-RestMethod -Method Get -Uri "$ApiBaseUrl/clusters/$ClusterID/nodes/$NodeID/mesh/synthetic-config"
|
||||
$feedback = $synthetic.synthetic_mesh_config.service_channel_route_feedback
|
||||
$policy = $feedback.recovery_policy
|
||||
|
||||
$backendLine = (& ssh $DockerSSH "docker ps --format '{{.Names}} {{.Image}} {{.Status}}' | grep '^rap_test_backend '") | Out-String
|
||||
$backendImageOK = $backendLine.Contains($ExpectedBackendImage)
|
||||
$policyFingerprintOK = $null -ne $policy -and [string]$policy.fingerprint -ne ""
|
||||
$provenanceCountersOK = $feedback.PSObject.Properties.Name -contains "missing_provenance_count" -or
|
||||
$feedback.PSObject.Properties.Name -contains "stale_policy_count" -or
|
||||
$feedback.PSObject.Properties.Name -contains "stale_generation_count" -or
|
||||
[int]$feedback.observation_count -eq 0
|
||||
$missingProvenanceCount = if ($feedback.PSObject.Properties.Name -contains "missing_provenance_count") { $feedback.missing_provenance_count } else { 0 }
|
||||
$stalePolicyCount = if ($feedback.PSObject.Properties.Name -contains "stale_policy_count") { $feedback.stale_policy_count } else { 0 }
|
||||
$staleGenerationCount = if ($feedback.PSObject.Properties.Name -contains "stale_generation_count") { $feedback.stale_generation_count } else { 0 }
|
||||
|
||||
$result = [ordered]@{
|
||||
schema_version = "c18z29.service_channel_feedback_provenance_guard_smoke.v1"
|
||||
passed = [bool]($backendImageOK -and $policyFingerprintOK -and $provenanceCountersOK)
|
||||
checks = [ordered]@{
|
||||
unit_feedback_provenance_guard_passed = ($unitTestOutput -join "`n").Contains("ok")
|
||||
backend_expected_image_deployed = $backendImageOK
|
||||
recovery_policy_fingerprint_present = $policyFingerprintOK
|
||||
feedback_provenance_counters_present_or_no_observations = $provenanceCountersOK
|
||||
}
|
||||
summary = [ordered]@{
|
||||
unit_test_output = ($unitTestOutput | Out-String).Trim()
|
||||
backend_container = $backendLine.Trim()
|
||||
expected_backend_image = $ExpectedBackendImage
|
||||
feedback_counts = [ordered]@{
|
||||
observations = $feedback.observation_count
|
||||
missing_provenance = $missingProvenanceCount
|
||||
stale_policy = $stalePolicyCount
|
||||
stale_generation = $staleGenerationCount
|
||||
}
|
||||
recovery_policy = $policy
|
||||
}
|
||||
}
|
||||
|
||||
$resolvedResultPath = Join-Path $repoRoot $ResultPath
|
||||
$result | ConvertTo-Json -Depth 100 | Set-Content -Path $resolvedResultPath -Encoding UTF8
|
||||
|
||||
if (-not $result.passed) {
|
||||
throw "C18Z29 feedback provenance guard smoke failed. Result: $resolvedResultPath"
|
||||
}
|
||||
|
||||
Write-Host "C18Z29 service-channel feedback provenance guard smoke passed. Result: $resolvedResultPath"
|
||||
$result
|
||||
@@ -0,0 +1,943 @@
|
||||
param(
|
||||
[string]$ApiBaseUrl = "http://192.168.200.61:18121/api/v1",
|
||||
[string]$ClusterID = "cfc0743d-d960-49fb-9de8-96e063d5e4aa",
|
||||
[string]$ActorUserID = "f67d943f-5397-4b3a-a229-695fe67ad700",
|
||||
[string]$EntryNodeName = "test-1",
|
||||
[string]$ExitNodeName = "test-2",
|
||||
[string]$EntryBaseUrl = "http://192.168.200.61:19131",
|
||||
[string]$DockerSSH = "test-docker",
|
||||
[int]$WarmBatchCount = 4,
|
||||
[int]$RecoveryBatchCount = 4,
|
||||
[string]$DegradedExitNodeName = "test-3",
|
||||
[string]$RequiredNodeVersion = "0.2.183",
|
||||
[string]$ResultPath = "artifacts\c18z3-live-service-channel-entry-ws-fallback-smoke-result.json"
|
||||
)
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = "Stop"
|
||||
Add-Type -AssemblyName System.Net.Http
|
||||
|
||||
$repoRoot = (Resolve-Path (Join-Path $PSScriptRoot "..\..")).ProviderPath
|
||||
$runId = "c18z3-" + (Get-Date -Format "yyyyMMdd-HHmmss")
|
||||
$resourceId = "vpn-$runId"
|
||||
$degradedResourceId = "vpn-$runId-degraded"
|
||||
|
||||
function Invoke-Api {
|
||||
param(
|
||||
[string]$Method,
|
||||
[string]$Path,
|
||||
[object]$Body = $null
|
||||
)
|
||||
$uri = "$ApiBaseUrl$Path"
|
||||
try {
|
||||
if ($null -eq $Body) {
|
||||
return Invoke-RestMethod -Method $Method -Uri $uri -TimeoutSec 30
|
||||
}
|
||||
return Invoke-RestMethod -Method $Method -Uri $uri -ContentType "application/json" -Body ($Body | ConvertTo-Json -Depth 80) -TimeoutSec 30
|
||||
}
|
||||
catch {
|
||||
$statusCode = $null
|
||||
if ($_.Exception.Response) {
|
||||
$statusCode = [int]$_.Exception.Response.StatusCode
|
||||
}
|
||||
$details = $_.ErrorDetails.Message
|
||||
if (-not $details) {
|
||||
$details = $_.Exception.Message
|
||||
}
|
||||
throw "$Method $Path failed with HTTP $statusCode`: $details"
|
||||
}
|
||||
}
|
||||
|
||||
function Get-NodeByName {
|
||||
param([string]$Name)
|
||||
$nodes = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/nodes?actor_user_id=$ActorUserID").nodes
|
||||
$node = @($nodes | Where-Object { $_.name -eq $Name }) | Select-Object -First 1
|
||||
if ($null -eq $node) {
|
||||
throw "Node '$Name' was not found in cluster $ClusterID"
|
||||
}
|
||||
return $node
|
||||
}
|
||||
|
||||
function Get-MeshPort {
|
||||
param([string]$Name)
|
||||
switch ($Name) {
|
||||
"test-1" { return 19131 }
|
||||
"test-2" { return 19132 }
|
||||
"test-3" { return 19133 }
|
||||
default { return 19131 }
|
||||
}
|
||||
}
|
||||
|
||||
function Enable-TestMeshListener {
|
||||
param([object]$Node)
|
||||
$port = Get-MeshPort -Name $Node.name
|
||||
Invoke-Api -Method PUT -Path "/clusters/$ClusterID/nodes/$($Node.id)/workloads/mesh-listener/desired" -Body @{
|
||||
actor_user_id = $ActorUserID
|
||||
desired_state = "enabled"
|
||||
runtime_mode = "container"
|
||||
version = "c18z3-live-fsc-entry-ws-fallback"
|
||||
config = @{
|
||||
listen_addr = "0.0.0.0:$port"
|
||||
listen_port_mode = "manual"
|
||||
advertise_endpoint = "http://192.168.200.61:$port"
|
||||
advertise_transport = "direct_http"
|
||||
connectivity_mode = "private_lan"
|
||||
nat_type = "none"
|
||||
region = "docker-test"
|
||||
production_forwarding = $true
|
||||
}
|
||||
environment = @{}
|
||||
} | Out-Null
|
||||
}
|
||||
|
||||
function Clear-OldSmokeRouteIntents {
|
||||
param(
|
||||
[string]$SourceNodeID,
|
||||
[string]$DestinationNodeID
|
||||
)
|
||||
$items = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/mesh/route-intents?actor_user_id=$ActorUserID").route_intents
|
||||
foreach ($item in @($items)) {
|
||||
if ([string]$item.lifecycle_status -ne "active") {
|
||||
continue
|
||||
}
|
||||
if ([string]$item.service_class -ne "vpn_packets") {
|
||||
continue
|
||||
}
|
||||
if ([string]$item.source_selector.node_id -ne $SourceNodeID -or [string]$item.destination_selector.node_id -ne $DestinationNodeID) {
|
||||
continue
|
||||
}
|
||||
$smoke = ""
|
||||
if ($null -ne $item.policy -and $null -ne $item.policy.metadata) {
|
||||
$prop = $item.policy.metadata.PSObject.Properties["smoke"]
|
||||
if ($null -ne $prop) {
|
||||
$smoke = [string]$prop.Value
|
||||
}
|
||||
}
|
||||
if ($smoke -ne "c18z1_live_service_channel_ingress" -and $smoke -ne "c18z2_live_service_channel_soak" -and $smoke -ne "c18z3_live_service_channel_entry_ws_fallback") {
|
||||
continue
|
||||
}
|
||||
Invoke-Api -Method POST -Path "/clusters/$ClusterID/mesh/route-intents/$($item.id)/expire" -Body @{ actor_user_id = $ActorUserID } | Out-Null
|
||||
}
|
||||
}
|
||||
|
||||
function New-RouteIntent {
|
||||
param(
|
||||
[string]$SourceNodeID,
|
||||
[string]$DestinationNodeID,
|
||||
[int]$Priority,
|
||||
[string]$Label
|
||||
)
|
||||
$expiresAt = (Get-Date).ToUniversalTime().AddMinutes(10).ToString("o")
|
||||
return Invoke-Api -Method POST -Path "/clusters/$ClusterID/mesh/route-intents" -Body @{
|
||||
actor_user_id = $ActorUserID
|
||||
source_selector = @{ node_id = $SourceNodeID }
|
||||
destination_selector = @{ node_id = $DestinationNodeID }
|
||||
service_class = "vpn_packets"
|
||||
priority = $Priority
|
||||
policy = @{
|
||||
synthetic_enabled = $true
|
||||
route_version = "$runId-$Label"
|
||||
policy_version = "$runId-$Label"
|
||||
peer_directory_version = "$runId-$Label"
|
||||
hops = @($SourceNodeID, $DestinationNodeID)
|
||||
allowed_channels = @("vpn_packet", "fabric_control")
|
||||
max_ttl = 8
|
||||
max_hops = 8
|
||||
expires_at = $expiresAt
|
||||
metadata = @{
|
||||
smoke = "c18z3_live_service_channel_entry_ws_fallback"
|
||||
run_id = $runId
|
||||
label = $Label
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function Get-SyntheticConfig {
|
||||
param([string]$NodeID)
|
||||
return Invoke-Api -Method GET -Path "/clusters/$ClusterID/nodes/$NodeID/mesh/synthetic-config?actor_user_id=$ActorUserID"
|
||||
}
|
||||
|
||||
function Get-LatestHeartbeat {
|
||||
param([string]$NodeID)
|
||||
return (Invoke-Api -Method GET -Path "/clusters/$ClusterID/nodes/$NodeID/heartbeats?actor_user_id=$ActorUserID&limit=1").heartbeats[0]
|
||||
}
|
||||
|
||||
function Get-LatestRuntimeReport {
|
||||
param([string]$NodeID)
|
||||
$hb = Get-LatestHeartbeat -NodeID $NodeID
|
||||
return @{
|
||||
heartbeat = $hb
|
||||
report = $hb.metadata.fabric_service_channel_runtime_report
|
||||
}
|
||||
}
|
||||
|
||||
function Wait-ForRuntimeReady {
|
||||
param(
|
||||
[string]$NodeID,
|
||||
[int]$MinRoutes,
|
||||
[int]$TimeoutSeconds = 90
|
||||
)
|
||||
$deadline = (Get-Date).AddSeconds($TimeoutSeconds)
|
||||
do {
|
||||
$latest = Get-LatestRuntimeReport -NodeID $NodeID
|
||||
$report = $latest.report
|
||||
if ($null -ne $report -and
|
||||
$report.enabled -eq $true -and
|
||||
$report.production_payload_forwarding -eq $true -and
|
||||
[int]$report.route_candidate_total -ge $MinRoutes) {
|
||||
return $latest
|
||||
}
|
||||
Start-Sleep -Seconds 2
|
||||
} while ((Get-Date) -lt $deadline)
|
||||
throw "Timed out waiting for production service-channel runtime ready on node $NodeID"
|
||||
}
|
||||
|
||||
function Wait-ForRuntimeConfigVersion {
|
||||
param(
|
||||
[string]$NodeID,
|
||||
[string]$ConfigVersion,
|
||||
[int]$TimeoutSeconds = 90
|
||||
)
|
||||
$deadline = (Get-Date).AddSeconds($TimeoutSeconds)
|
||||
do {
|
||||
$latest = Get-LatestRuntimeReport -NodeID $NodeID
|
||||
if ($null -ne $latest.report) {
|
||||
$loadedVersion = [string]$latest.report.config_version
|
||||
if ($loadedVersion -ge $ConfigVersion) {
|
||||
return $latest
|
||||
}
|
||||
}
|
||||
Start-Sleep -Seconds 2
|
||||
} while ((Get-Date) -lt $deadline)
|
||||
throw "Timed out waiting for node $NodeID to load synthetic config $ConfigVersion"
|
||||
}
|
||||
|
||||
function Wait-ForRouteIntentVisible {
|
||||
param(
|
||||
[string]$NodeID,
|
||||
[string[]]$RouteIDs,
|
||||
[int]$TimeoutSeconds = 60
|
||||
)
|
||||
$deadline = (Get-Date).AddSeconds($TimeoutSeconds)
|
||||
do {
|
||||
$config = Get-SyntheticConfig -NodeID $NodeID
|
||||
$routes = @($config.synthetic_mesh_config.routes)
|
||||
$present = @($routes | Where-Object { $RouteIDs -contains $_.route_id })
|
||||
if ($present.Count -ge $RouteIDs.Count) {
|
||||
return $config
|
||||
}
|
||||
Start-Sleep -Seconds 2
|
||||
} while ((Get-Date) -lt $deadline)
|
||||
throw "Timed out waiting for routes '$($RouteIDs -join ",")' in synthetic config for node $NodeID"
|
||||
}
|
||||
|
||||
function New-ServiceChannelLease {
|
||||
param(
|
||||
[string]$EntryNodeID,
|
||||
[string]$ExitNodeID,
|
||||
[string]$VPNResourceID = $resourceId
|
||||
)
|
||||
return (Invoke-Api -Method POST -Path "/clusters/$ClusterID/fabric/service-channels/leases" -Body @{
|
||||
actor_user_id = $ActorUserID
|
||||
organization_id = "org-c18z3-smoke"
|
||||
user_id = $ActorUserID
|
||||
resource_id = $VPNResourceID
|
||||
service_class = "vpn_packets"
|
||||
entry_node_ids = @($EntryNodeID)
|
||||
exit_node_ids = @($ExitNodeID)
|
||||
preferred_entry_node_id = $EntryNodeID
|
||||
preferred_exit_node_id = $ExitNodeID
|
||||
allowed_channels = @("vpn_packet", "bulk", "control")
|
||||
ttl_seconds = 300
|
||||
metadata = @{
|
||||
smoke = "c18z3_live_service_channel_entry_ws_fallback"
|
||||
run_id = $runId
|
||||
}
|
||||
}).fabric_service_channel_lease
|
||||
}
|
||||
|
||||
function ConvertTo-Base64UrlJson {
|
||||
param([object]$Value)
|
||||
$json = $Value | ConvertTo-Json -Depth 80 -Compress
|
||||
$bytes = [System.Text.Encoding]::UTF8.GetBytes($json)
|
||||
return [Convert]::ToBase64String($bytes).TrimEnd("=").Replace("+", "-").Replace("/", "_")
|
||||
}
|
||||
|
||||
function Get-ObjectPropertyValue {
|
||||
param(
|
||||
[object]$Object,
|
||||
[string]$Name
|
||||
)
|
||||
if ($null -eq $Object) {
|
||||
return $null
|
||||
}
|
||||
$prop = $Object.PSObject.Properties[$Name]
|
||||
if ($null -eq $prop) {
|
||||
return $null
|
||||
}
|
||||
return $prop.Value
|
||||
}
|
||||
|
||||
function New-TestIPv4UDPPacket {
|
||||
param([int]$SourcePort)
|
||||
$payload = [System.Text.Encoding]::ASCII.GetBytes("c18z1-$SourcePort")
|
||||
$totalLength = 20 + 8 + $payload.Length
|
||||
$packet = New-Object byte[] $totalLength
|
||||
$packet[0] = 0x45
|
||||
$packet[1] = 0
|
||||
$packet[2] = [byte](($totalLength -shr 8) -band 0xff)
|
||||
$packet[3] = [byte]($totalLength -band 0xff)
|
||||
$packet[8] = 64
|
||||
$packet[9] = 17
|
||||
$packet[12] = 10; $packet[13] = 18; $packet[14] = 1; $packet[15] = 10
|
||||
$packet[16] = 10; $packet[17] = 18; $packet[18] = 2; $packet[19] = 20
|
||||
$udpOffset = 20
|
||||
$destPort = 3389
|
||||
$udpLength = 8 + $payload.Length
|
||||
$packet[$udpOffset] = [byte](($SourcePort -shr 8) -band 0xff)
|
||||
$packet[$udpOffset + 1] = [byte]($SourcePort -band 0xff)
|
||||
$packet[$udpOffset + 2] = [byte](($destPort -shr 8) -band 0xff)
|
||||
$packet[$udpOffset + 3] = [byte]($destPort -band 0xff)
|
||||
$packet[$udpOffset + 4] = [byte](($udpLength -shr 8) -band 0xff)
|
||||
$packet[$udpOffset + 5] = [byte]($udpLength -band 0xff)
|
||||
[Array]::Copy($payload, 0, $packet, 28, $payload.Length)
|
||||
return $packet
|
||||
}
|
||||
|
||||
function New-PacketBatchBody {
|
||||
param([byte[][]]$Packets)
|
||||
$stream = [System.IO.MemoryStream]::new()
|
||||
foreach ($packet in $Packets) {
|
||||
$length = $packet.Length
|
||||
$stream.WriteByte([byte](($length -shr 24) -band 0xff))
|
||||
$stream.WriteByte([byte](($length -shr 16) -band 0xff))
|
||||
$stream.WriteByte([byte](($length -shr 8) -band 0xff))
|
||||
$stream.WriteByte([byte]($length -band 0xff))
|
||||
$stream.Write($packet, 0, $packet.Length)
|
||||
}
|
||||
return $stream.ToArray()
|
||||
}
|
||||
|
||||
function Invoke-ServiceChannelPost {
|
||||
param(
|
||||
[object]$Lease,
|
||||
[int]$PortStart,
|
||||
[string]$VPNResourceID = $resourceId
|
||||
)
|
||||
$packets = @()
|
||||
for ($i = 0; $i -lt 8; $i++) {
|
||||
$packets += ,(New-TestIPv4UDPPacket -SourcePort ($PortStart + $i))
|
||||
}
|
||||
$path = $Lease.entry_http.path_template.
|
||||
Replace("{cluster_id}", $ClusterID).
|
||||
Replace("{channel_id}", $Lease.channel_id).
|
||||
Replace("{resource_id}", $VPNResourceID)
|
||||
$url = "$EntryBaseUrl$path`?batch=true"
|
||||
$headers = @{
|
||||
"X-RAP-Service-Channel-Token" = $Lease.token.token
|
||||
"X-RAP-Fabric-Channel-ID" = $Lease.channel_id
|
||||
"X-RAP-Service-Class" = "vpn_packets"
|
||||
"X-RAP-Channel-Class" = "vpn_packet"
|
||||
"X-RAP-Service-Channel-Authority-Payload" = ConvertTo-Base64UrlJson -Value $Lease.authority_payload
|
||||
"X-RAP-Service-Channel-Authority-Signature" = ConvertTo-Base64UrlJson -Value $Lease.authority_signature
|
||||
}
|
||||
$body = New-PacketBatchBody -Packets $packets
|
||||
$client = [System.Net.Http.HttpClient]::new()
|
||||
try {
|
||||
$client.Timeout = [TimeSpan]::FromSeconds(30)
|
||||
$request = [System.Net.Http.HttpRequestMessage]::new([System.Net.Http.HttpMethod]::Post, $url)
|
||||
foreach ($header in $headers.GetEnumerator()) {
|
||||
[void]$request.Headers.TryAddWithoutValidation($header.Key, [string]$header.Value)
|
||||
}
|
||||
$content = [System.Net.Http.ByteArrayContent]::new($body)
|
||||
$content.Headers.ContentType = [System.Net.Http.Headers.MediaTypeHeaderValue]::Parse("application/vnd.rap.vpn-packet-batch.v1")
|
||||
$request.Content = $content
|
||||
$response = $client.SendAsync($request).GetAwaiter().GetResult()
|
||||
$responseBody = $response.Content.ReadAsStringAsync().GetAwaiter().GetResult()
|
||||
if (-not $response.IsSuccessStatusCode) {
|
||||
throw "Service-channel POST $url failed with HTTP $([int]$response.StatusCode): $responseBody"
|
||||
}
|
||||
return [pscustomobject]@{
|
||||
StatusCode = [int]$response.StatusCode
|
||||
Body = $responseBody
|
||||
}
|
||||
}
|
||||
finally {
|
||||
$client.Dispose()
|
||||
}
|
||||
}
|
||||
|
||||
function Get-IngressSendPackets {
|
||||
param([string]$NodeID)
|
||||
$latest = Get-LatestRuntimeReport -NodeID $NodeID
|
||||
$ingress = $latest.report.ingress
|
||||
$sendPackets = Get-ObjectPropertyValue -Object $ingress -Name "send_packets"
|
||||
if ($null -eq $sendPackets) {
|
||||
return 0
|
||||
}
|
||||
return [int]$sendPackets
|
||||
}
|
||||
|
||||
function Get-IngressRouteFailures {
|
||||
param([string]$NodeID)
|
||||
$latest = Get-LatestRuntimeReport -NodeID $NodeID
|
||||
$ingress = $latest.report.ingress
|
||||
$failures = Get-ObjectPropertyValue -Object $ingress -Name "send_route_failures"
|
||||
if ($null -eq $failures) {
|
||||
return 0
|
||||
}
|
||||
return [int]$failures
|
||||
}
|
||||
|
||||
function Get-ExitQueueDepth {
|
||||
param(
|
||||
[string]$NodeID,
|
||||
[string]$VPNConnectionID
|
||||
)
|
||||
$latest = Get-LatestRuntimeReport -NodeID $NodeID
|
||||
$queueKey = "$VPNConnectionID`:client_to_gateway"
|
||||
$depths = $latest.report.inbox.queue_depths
|
||||
if ($null -eq $depths) {
|
||||
return 0
|
||||
}
|
||||
$prop = $depths.PSObject.Properties[$queueKey]
|
||||
if ($null -eq $prop) {
|
||||
return 0
|
||||
}
|
||||
return [int]$prop.Value
|
||||
}
|
||||
|
||||
function Wait-ForExitQueueDepth {
|
||||
param(
|
||||
[string]$NodeID,
|
||||
[string]$VPNConnectionID,
|
||||
[int]$MinDepth,
|
||||
[int]$TimeoutSeconds = 90
|
||||
)
|
||||
$deadline = (Get-Date).AddSeconds($TimeoutSeconds)
|
||||
do {
|
||||
$depth = Get-ExitQueueDepth -NodeID $NodeID -VPNConnectionID $VPNConnectionID
|
||||
if ($depth -ge $MinDepth) {
|
||||
return $depth
|
||||
}
|
||||
Start-Sleep -Seconds 2
|
||||
} while ((Get-Date) -lt $deadline)
|
||||
throw "Timed out waiting for exit queue depth >= $MinDepth on node $NodeID"
|
||||
}
|
||||
|
||||
function Invoke-ServiceChannelPostSafe {
|
||||
param(
|
||||
[object]$Lease,
|
||||
[int]$PortStart,
|
||||
[string]$VPNResourceID = $resourceId
|
||||
)
|
||||
try {
|
||||
$response = Invoke-ServiceChannelPost -Lease $Lease -PortStart $PortStart -VPNResourceID $VPNResourceID
|
||||
return [pscustomobject]@{
|
||||
ok = $true
|
||||
status_code = [int]$response.StatusCode
|
||||
error = ""
|
||||
}
|
||||
}
|
||||
catch {
|
||||
return [pscustomobject]@{
|
||||
ok = $false
|
||||
status_code = 0
|
||||
error = $_.Exception.Message
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function ConvertTo-WebSocketURL {
|
||||
param([string]$URL)
|
||||
if ($URL.StartsWith("https://")) {
|
||||
return "wss://" + $URL.Substring("https://".Length)
|
||||
}
|
||||
if ($URL.StartsWith("http://")) {
|
||||
return "ws://" + $URL.Substring("http://".Length)
|
||||
}
|
||||
return $URL
|
||||
}
|
||||
|
||||
function Invoke-ServiceChannelWebSocketSend {
|
||||
param(
|
||||
[object]$Lease,
|
||||
[int]$PortStart,
|
||||
[string]$VPNResourceID = $resourceId
|
||||
)
|
||||
$packets = @()
|
||||
for ($i = 0; $i -lt 8; $i++) {
|
||||
$packets += ,(New-TestIPv4UDPPacket -SourcePort ($PortStart + $i))
|
||||
}
|
||||
$path = $Lease.entry_http.websocket_path_template.
|
||||
Replace("{cluster_id}", $ClusterID).
|
||||
Replace("{channel_id}", $Lease.channel_id).
|
||||
Replace("{resource_id}", $VPNResourceID)
|
||||
$url = ConvertTo-WebSocketURL -URL "$EntryBaseUrl$path"
|
||||
$socket = [System.Net.WebSockets.ClientWebSocket]::new()
|
||||
$cts = [System.Threading.CancellationTokenSource]::new([TimeSpan]::FromSeconds(20))
|
||||
try {
|
||||
$null = $socket.Options.SetRequestHeader("X-RAP-Service-Channel-Token", [string]$Lease.token.token)
|
||||
$null = $socket.Options.SetRequestHeader("X-RAP-Fabric-Channel-ID", [string]$Lease.channel_id)
|
||||
$null = $socket.Options.SetRequestHeader("X-RAP-Service-Class", "vpn_packets")
|
||||
$null = $socket.Options.SetRequestHeader("X-RAP-Channel-Class", "vpn_packet")
|
||||
$null = $socket.Options.SetRequestHeader("X-RAP-Service-Channel-Authority-Payload", (ConvertTo-Base64UrlJson -Value $Lease.authority_payload))
|
||||
$null = $socket.Options.SetRequestHeader("X-RAP-Service-Channel-Authority-Signature", (ConvertTo-Base64UrlJson -Value $Lease.authority_signature))
|
||||
$null = $socket.ConnectAsync([Uri]$url, $cts.Token).GetAwaiter().GetResult()
|
||||
$body = New-PacketBatchBody -Packets $packets
|
||||
$segment = [ArraySegment[byte]]::new($body)
|
||||
$null = $socket.SendAsync($segment, [System.Net.WebSockets.WebSocketMessageType]::Binary, $true, $cts.Token).GetAwaiter().GetResult()
|
||||
Start-Sleep -Milliseconds 300
|
||||
if ($socket.State -eq [System.Net.WebSockets.WebSocketState]::Open) {
|
||||
$null = $socket.CloseOutputAsync([System.Net.WebSockets.WebSocketCloseStatus]::NormalClosure, "c18z3 sent", $cts.Token).GetAwaiter().GetResult()
|
||||
}
|
||||
return [pscustomobject]@{
|
||||
ok = $true
|
||||
url = $url
|
||||
sent_packets = $packets.Count
|
||||
state = [string]$socket.State
|
||||
error = ""
|
||||
}
|
||||
}
|
||||
catch {
|
||||
return [pscustomobject]@{
|
||||
ok = $false
|
||||
url = $url
|
||||
sent_packets = 0
|
||||
state = [string]$socket.State
|
||||
error = $_.Exception.Message
|
||||
}
|
||||
}
|
||||
finally {
|
||||
$socket.Dispose()
|
||||
$cts.Dispose()
|
||||
}
|
||||
}
|
||||
|
||||
function Send-BatchSeries {
|
||||
param(
|
||||
[object]$Lease,
|
||||
[int]$Count,
|
||||
[int]$PortBase,
|
||||
[int]$DelayMilliseconds = 100,
|
||||
[string]$VPNResourceID = $resourceId
|
||||
)
|
||||
$results = @()
|
||||
for ($i = 0; $i -lt $Count; $i++) {
|
||||
$results += Invoke-ServiceChannelPostSafe -Lease $Lease -PortStart ($PortBase + ($i * 100)) -VPNResourceID $VPNResourceID
|
||||
if ($DelayMilliseconds -gt 0) {
|
||||
Start-Sleep -Milliseconds $DelayMilliseconds
|
||||
}
|
||||
}
|
||||
return $results
|
||||
}
|
||||
|
||||
function Invoke-RemoteDocker {
|
||||
param([string]$Command)
|
||||
& ssh $DockerSSH $Command
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "ssh $DockerSSH command failed: $Command"
|
||||
}
|
||||
}
|
||||
|
||||
function Stop-TestUpdaters {
|
||||
Invoke-RemoteDocker -Command "docker stop rap_host_agent_updater_test-1 rap_host_agent_updater_test-2 rap_host_agent_updater_test-3 >/dev/null 2>&1 || true"
|
||||
}
|
||||
|
||||
function Start-TestUpdaters {
|
||||
Invoke-RemoteDocker -Command "docker start rap_host_agent_updater_test-1 rap_host_agent_updater_test-2 rap_host_agent_updater_test-3 >/dev/null 2>&1 || true"
|
||||
}
|
||||
|
||||
function Restart-ExitContainer {
|
||||
param([string]$Name)
|
||||
$containerName = "rap_test_node_" + $Name.Replace("-", "_")
|
||||
Invoke-RemoteDocker -Command "docker restart $containerName >/dev/null"
|
||||
}
|
||||
|
||||
function Restart-NodeContainer {
|
||||
param([string]$Name)
|
||||
$containerName = "rap_test_node_" + $Name.Replace("-", "_")
|
||||
Invoke-RemoteDocker -Command "docker restart $containerName >/dev/null"
|
||||
}
|
||||
|
||||
function Get-BackendClientGatewayDepth {
|
||||
param([string]$VPNConnectionID)
|
||||
$stats = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/vpn-connections/$VPNConnectionID/tunnel/stats").vpn_packet_stats
|
||||
$queue = $stats.client_to_gateway
|
||||
if ($null -eq $queue) {
|
||||
return 0
|
||||
}
|
||||
$depthProp = $queue.PSObject.Properties["queue_depth"]
|
||||
if ($null -eq $depthProp) {
|
||||
return 0
|
||||
}
|
||||
return [int]$depthProp.Value
|
||||
}
|
||||
|
||||
function Wait-ForIngressRoute {
|
||||
param(
|
||||
[string]$NodeID,
|
||||
[string]$RouteID,
|
||||
[int]$MinSendPackets,
|
||||
[int]$TimeoutSeconds = 45
|
||||
)
|
||||
$deadline = (Get-Date).AddSeconds($TimeoutSeconds)
|
||||
do {
|
||||
$latest = Get-LatestRuntimeReport -NodeID $NodeID
|
||||
$ingress = $latest.report.ingress
|
||||
$sendPackets = Get-ObjectPropertyValue -Object $ingress -Name "send_packets"
|
||||
$selectedRoute = Get-ObjectPropertyValue -Object $ingress -Name "last_selected_route_id"
|
||||
if ($null -ne $ingress -and
|
||||
[int]$sendPackets -ge $MinSendPackets -and
|
||||
[string]$selectedRoute -eq $RouteID) {
|
||||
return $latest
|
||||
}
|
||||
Start-Sleep -Seconds 2
|
||||
} while ((Get-Date) -lt $deadline)
|
||||
throw "Timed out waiting for ingress telemetry route=$RouteID packets>=$MinSendPackets on node $NodeID"
|
||||
}
|
||||
|
||||
function Wait-ForIngressAnyRoute {
|
||||
param(
|
||||
[string]$NodeID,
|
||||
[string[]]$RouteIDs,
|
||||
[int]$MinSendPackets,
|
||||
[int]$TimeoutSeconds = 45
|
||||
)
|
||||
$deadline = (Get-Date).AddSeconds($TimeoutSeconds)
|
||||
do {
|
||||
$latest = Get-LatestRuntimeReport -NodeID $NodeID
|
||||
$ingress = $latest.report.ingress
|
||||
$sendPackets = Get-ObjectPropertyValue -Object $ingress -Name "send_packets"
|
||||
$selectedRoute = Get-ObjectPropertyValue -Object $ingress -Name "last_selected_route_id"
|
||||
if ($null -ne $ingress -and
|
||||
[int]$sendPackets -ge $MinSendPackets -and
|
||||
$RouteIDs -contains [string]$selectedRoute) {
|
||||
return $latest
|
||||
}
|
||||
Start-Sleep -Seconds 2
|
||||
} while ((Get-Date) -lt $deadline)
|
||||
throw "Timed out waiting for ingress telemetry routes='$($RouteIDs -join ",")' packets>=$MinSendPackets on node $NodeID"
|
||||
}
|
||||
|
||||
function Wait-ForExitInbox {
|
||||
param(
|
||||
[string]$NodeID,
|
||||
[string]$VPNConnectionID,
|
||||
[int]$TimeoutSeconds = 45
|
||||
)
|
||||
$queueKey = "$VPNConnectionID`:client_to_gateway"
|
||||
$deadline = (Get-Date).AddSeconds($TimeoutSeconds)
|
||||
do {
|
||||
$latest = Get-LatestRuntimeReport -NodeID $NodeID
|
||||
$depths = $latest.report.inbox.queue_depths
|
||||
if ($null -ne $depths) {
|
||||
$prop = $depths.PSObject.Properties[$queueKey]
|
||||
if ($null -ne $prop -and [int]$prop.Value -gt 0) {
|
||||
return $latest
|
||||
}
|
||||
}
|
||||
Start-Sleep -Seconds 2
|
||||
} while ((Get-Date) -lt $deadline)
|
||||
throw "Timed out waiting for exit inbox queue '$queueKey' on node $NodeID"
|
||||
}
|
||||
|
||||
function Send-FeedbackHeartbeat {
|
||||
param(
|
||||
[string]$EntryNodeID,
|
||||
[string]$BadRouteID,
|
||||
[string]$GoodRouteID
|
||||
)
|
||||
return Invoke-Api -Method POST -Path "/clusters/$ClusterID/nodes/$EntryNodeID/heartbeats" -Body @{
|
||||
health_status = "healthy"
|
||||
reported_version = $RequiredNodeVersion
|
||||
capabilities = @{
|
||||
native_node_agent = $true
|
||||
fabric_service_channel_runtime = $true
|
||||
fabric_service_channel_route_manager = $true
|
||||
smoke_feedback_injection = "c18z1"
|
||||
}
|
||||
service_states = @{ smoke = "c18z1_live_ingress_feedback" }
|
||||
metadata = @{
|
||||
fabric_service_channel_runtime_report = @{
|
||||
schema_version = "c18l.fabric_service_channel_runtime_report.v1"
|
||||
ingress = @{
|
||||
flow_scheduler = @{
|
||||
channel_stats = @{
|
||||
"c18z1-live-flow" = @{
|
||||
last_route_id = $GoodRouteID
|
||||
last_failed_route_id = $BadRouteID
|
||||
last_error = "c18z1 forced stale route after live packet ingress"
|
||||
consecutive_failures = 3
|
||||
stall_count = 1
|
||||
last_send_duration_ms = 250
|
||||
route_rebuild_recommended = $true
|
||||
degraded_fallback_recommended = $false
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
smoke = @{
|
||||
name = "c18z1_live_service_channel_ingress"
|
||||
run_id = $runId
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function Wait-ForConfigDecision {
|
||||
param(
|
||||
[string]$NodeID,
|
||||
[string]$BadRouteID,
|
||||
[string]$ExpectedReplacementID,
|
||||
[int]$TimeoutSeconds = 60
|
||||
)
|
||||
$deadline = (Get-Date).AddSeconds($TimeoutSeconds)
|
||||
do {
|
||||
$config = Get-SyntheticConfig -NodeID $NodeID
|
||||
$decisions = @($config.synthetic_mesh_config.route_path_decisions.decisions)
|
||||
$decision = @($decisions | Where-Object {
|
||||
$_.route_id -eq $BadRouteID -and
|
||||
$_.rebuild_status -eq "applied" -and
|
||||
$_.replacement_route_id -eq $ExpectedReplacementID
|
||||
}) | Select-Object -First 1
|
||||
if ($null -ne $decision) {
|
||||
return @{
|
||||
config = $config
|
||||
decision = $decision
|
||||
}
|
||||
}
|
||||
Start-Sleep -Seconds 2
|
||||
} while ((Get-Date) -lt $deadline)
|
||||
throw "Timed out waiting for applied rebuild decision $BadRouteID -> $ExpectedReplacementID"
|
||||
}
|
||||
|
||||
function Wait-ForAppliedRebuildTransition {
|
||||
param(
|
||||
[string]$NodeID,
|
||||
[string]$BadRouteID = "",
|
||||
[string]$ReplacementRouteID = "",
|
||||
[int]$TimeoutSeconds = 90
|
||||
)
|
||||
$deadline = (Get-Date).AddSeconds($TimeoutSeconds)
|
||||
do {
|
||||
$latest = Get-LatestRuntimeReport -NodeID $NodeID
|
||||
$transition = $null
|
||||
if ($null -ne $latest.report -and $null -ne $latest.report.ingress) {
|
||||
$prop = $latest.report.ingress.PSObject.Properties["route_manager_transition"]
|
||||
if ($null -ne $prop) {
|
||||
$transition = $prop.Value
|
||||
}
|
||||
}
|
||||
if ($null -ne $transition -and [string]$transition.status -eq "applied_rebuild") {
|
||||
return $latest
|
||||
}
|
||||
if ($BadRouteID -ne "" -and $ReplacementRouteID -ne "") {
|
||||
Send-FeedbackHeartbeat -EntryNodeID $NodeID -BadRouteID $BadRouteID -GoodRouteID $ReplacementRouteID | Out-Null
|
||||
}
|
||||
Start-Sleep -Seconds 2
|
||||
} while ((Get-Date) -lt $deadline)
|
||||
throw "Timed out waiting for node route-manager transition applied_rebuild on node $NodeID"
|
||||
}
|
||||
|
||||
$entryNode = Get-NodeByName -Name $EntryNodeName
|
||||
$exitNode = Get-NodeByName -Name $ExitNodeName
|
||||
$degradedExitNode = Get-NodeByName -Name $DegradedExitNodeName
|
||||
$primaryRouteID = ""
|
||||
$alternateRouteID = ""
|
||||
$updatersStopped = $false
|
||||
$result = $null
|
||||
|
||||
try {
|
||||
Stop-TestUpdaters
|
||||
$updatersStopped = $true
|
||||
|
||||
Enable-TestMeshListener -Node $entryNode
|
||||
Enable-TestMeshListener -Node $exitNode
|
||||
Enable-TestMeshListener -Node $degradedExitNode
|
||||
Clear-OldSmokeRouteIntents -SourceNodeID $entryNode.id -DestinationNodeID $exitNode.id
|
||||
Clear-OldSmokeRouteIntents -SourceNodeID $entryNode.id -DestinationNodeID $degradedExitNode.id
|
||||
|
||||
$primaryIntent = New-RouteIntent -SourceNodeID $entryNode.id -DestinationNodeID $exitNode.id -Priority 2000000000 -Label "primary"
|
||||
$alternateIntent = New-RouteIntent -SourceNodeID $entryNode.id -DestinationNodeID $exitNode.id -Priority 1999999999 -Label "alternate"
|
||||
$primaryRouteID = $primaryIntent.route_intent.id
|
||||
$alternateRouteID = $alternateIntent.route_intent.id
|
||||
$routeIDs = @($primaryRouteID, $alternateRouteID)
|
||||
|
||||
$visibleConfig = Wait-ForRouteIntentVisible -NodeID $entryNode.id -RouteIDs $routeIDs
|
||||
$exitVisibleConfig = Wait-ForRouteIntentVisible -NodeID $exitNode.id -RouteIDs $routeIDs
|
||||
$readyBefore = Wait-ForRuntimeReady -NodeID $entryNode.id -MinRoutes 2
|
||||
$exitReadyBefore = Wait-ForRuntimeReady -NodeID $exitNode.id -MinRoutes 0
|
||||
$loadedConfig = Wait-ForRuntimeConfigVersion -NodeID $entryNode.id -ConfigVersion $visibleConfig.synthetic_mesh_config.config_version
|
||||
$exitLoadedConfig = Wait-ForRuntimeConfigVersion -NodeID $exitNode.id -ConfigVersion $exitVisibleConfig.synthetic_mesh_config.config_version
|
||||
$lease = New-ServiceChannelLease -EntryNodeID $entryNode.id -ExitNodeID $exitNode.id
|
||||
if ($lease.status -ne "ready") {
|
||||
throw "Lease status was '$($lease.status)', want ready"
|
||||
}
|
||||
if ($routeIDs -notcontains [string]$lease.primary_route.route_id) {
|
||||
throw "Lease primary route was '$($lease.primary_route.route_id)', want one of smoke routes"
|
||||
}
|
||||
|
||||
$baselineSendPackets = Get-IngressSendPackets -NodeID $entryNode.id
|
||||
$baselineRouteFailures = Get-IngressRouteFailures -NodeID $entryNode.id
|
||||
|
||||
$warmResults = Send-BatchSeries -Lease $lease -Count $WarmBatchCount -PortBase 43000 -DelayMilliseconds 100
|
||||
$warmOk = @($warmResults | Where-Object { $_.ok -eq $true }).Count
|
||||
if ($warmOk -lt $WarmBatchCount) {
|
||||
throw "Warm service-channel HTTP accepted $warmOk/$WarmBatchCount batches"
|
||||
}
|
||||
$warmIngress = Wait-ForIngressAnyRoute -NodeID $entryNode.id -RouteIDs $routeIDs -MinSendPackets ($baselineSendPackets + ($WarmBatchCount * 8)) -TimeoutSeconds 90
|
||||
$warmExitDepth = Wait-ForExitQueueDepth -NodeID $exitNode.id -VPNConnectionID $resourceId -MinDepth 8 -TimeoutSeconds 90
|
||||
|
||||
$webSocketBaseline = Get-IngressSendPackets -NodeID $entryNode.id
|
||||
$webSocketExitBaseline = Get-ExitQueueDepth -NodeID $exitNode.id -VPNConnectionID $resourceId
|
||||
$webSocketResult = Invoke-ServiceChannelWebSocketSend -Lease $lease -PortStart 53000
|
||||
if (-not $webSocketResult.ok) {
|
||||
throw "WebSocket service-channel send failed: $($webSocketResult.error)"
|
||||
}
|
||||
$webSocketIngress = Wait-ForIngressAnyRoute -NodeID $entryNode.id -RouteIDs $routeIDs -MinSendPackets ($webSocketBaseline + 8) -TimeoutSeconds 90
|
||||
$webSocketExitDepth = Wait-ForExitQueueDepth -NodeID $exitNode.id -VPNConnectionID $resourceId -MinDepth ($webSocketExitBaseline + 8) -TimeoutSeconds 90
|
||||
|
||||
Restart-NodeContainer -Name $EntryNodeName
|
||||
$entryNode = Get-NodeByName -Name $EntryNodeName
|
||||
$postRestartVisibleConfig = Wait-ForRouteIntentVisible -NodeID $entryNode.id -RouteIDs $routeIDs -TimeoutSeconds 120
|
||||
$postRestartReady = Wait-ForRuntimeReady -NodeID $entryNode.id -MinRoutes 2 -TimeoutSeconds 120
|
||||
$postRestartLoadedConfig = Wait-ForRuntimeConfigVersion -NodeID $entryNode.id -ConfigVersion $postRestartVisibleConfig.synthetic_mesh_config.config_version -TimeoutSeconds 120
|
||||
$postRestartExitBaseline = Get-ExitQueueDepth -NodeID $exitNode.id -VPNConnectionID $resourceId
|
||||
|
||||
$recoveryBaseline = Get-IngressSendPackets -NodeID $entryNode.id
|
||||
$recoveryResults = Send-BatchSeries -Lease $lease -Count $RecoveryBatchCount -PortBase 63000 -DelayMilliseconds 100
|
||||
$recoveryOk = @($recoveryResults | Where-Object { $_.ok -eq $true }).Count
|
||||
if ($recoveryOk -lt $RecoveryBatchCount) {
|
||||
throw "Recovery service-channel soak accepted $recoveryOk/$RecoveryBatchCount batches"
|
||||
}
|
||||
$recoveryIngress = Wait-ForIngressAnyRoute -NodeID $entryNode.id -RouteIDs $routeIDs -MinSendPackets ($recoveryBaseline + ($RecoveryBatchCount * 8)) -TimeoutSeconds 120
|
||||
$recoveryExitDepth = Wait-ForExitQueueDepth -NodeID $exitNode.id -VPNConnectionID $resourceId -MinDepth ($postRestartExitBaseline + 8) -TimeoutSeconds 120
|
||||
|
||||
$degradedLease = New-ServiceChannelLease -EntryNodeID $entryNode.id -ExitNodeID $degradedExitNode.id -VPNResourceID $degradedResourceId
|
||||
if ($degradedLease.status -ne "degraded_fallback") {
|
||||
throw "Degraded lease status was '$($degradedLease.status)', want degraded_fallback"
|
||||
}
|
||||
$degradedBackendBaseline = Get-BackendClientGatewayDepth -VPNConnectionID $degradedResourceId
|
||||
$degradedPost = Invoke-ServiceChannelPostSafe -Lease $degradedLease -PortStart 73000 -VPNResourceID $degradedResourceId
|
||||
if (-not $degradedPost.ok) {
|
||||
throw "Degraded fallback POST failed: $($degradedPost.error)"
|
||||
}
|
||||
$degradedBackendDepth = Get-BackendClientGatewayDepth -VPNConnectionID $degradedResourceId
|
||||
$finalExitRuntime = Get-LatestRuntimeReport -NodeID $exitNode.id
|
||||
$finalRouteFailures = Get-IngressRouteFailures -NodeID $entryNode.id
|
||||
|
||||
$expiredPrimary = Invoke-Api -Method POST -Path "/clusters/$ClusterID/mesh/route-intents/$primaryRouteID/expire" -Body @{ actor_user_id = $ActorUserID }
|
||||
$expiredAlternate = Invoke-Api -Method POST -Path "/clusters/$ClusterID/mesh/route-intents/$alternateRouteID/expire" -Body @{ actor_user_id = $ActorUserID }
|
||||
|
||||
$result = [ordered]@{
|
||||
schema_version = "c18z3.live_service_channel_entry_ws_fallback_smoke.v1"
|
||||
run_id = $runId
|
||||
base_url = $ApiBaseUrl
|
||||
entry_base_url = $EntryBaseUrl
|
||||
cluster_id = $ClusterID
|
||||
entry_node = @{ name = $entryNode.name; id = $entryNode.id }
|
||||
exit_node = @{ name = $exitNode.name; id = $exitNode.id }
|
||||
degraded_exit_node = @{ name = $degradedExitNode.name; id = $degradedExitNode.id }
|
||||
resource_id = $resourceId
|
||||
degraded_resource_id = $degradedResourceId
|
||||
route_intents = @{
|
||||
primary_route_intent_id = $primaryRouteID
|
||||
alternate_route_intent_id = $alternateRouteID
|
||||
expired_primary_status = $expiredPrimary.route_intent.lifecycle_status
|
||||
expired_alternate_status = $expiredAlternate.route_intent.lifecycle_status
|
||||
}
|
||||
lease = @{
|
||||
channel_id = $lease.channel_id
|
||||
status = $lease.status
|
||||
primary_route_id = $lease.primary_route.route_id
|
||||
}
|
||||
degraded_lease = @{
|
||||
channel_id = $degradedLease.channel_id
|
||||
status = $degradedLease.status
|
||||
fallback_active = $degradedLease.fallback.active
|
||||
fallback_degraded = $degradedLease.fallback.degraded
|
||||
fallback_reason = $degradedLease.fallback.reason
|
||||
primary_route_status = $degradedLease.primary_route.status
|
||||
}
|
||||
batches = @{
|
||||
warm_requested = $WarmBatchCount
|
||||
warm_accepted = $warmOk
|
||||
websocket_sent = $webSocketResult.sent_packets
|
||||
recovery_requested = $RecoveryBatchCount
|
||||
recovery_accepted = $recoveryOk
|
||||
}
|
||||
route_failures = @{
|
||||
baseline = $baselineRouteFailures
|
||||
final = $finalRouteFailures
|
||||
delta = ($finalRouteFailures - $baselineRouteFailures)
|
||||
}
|
||||
exit_queue = @{
|
||||
warm_depth = $warmExitDepth
|
||||
websocket_baseline_depth = $webSocketExitBaseline
|
||||
websocket_depth = $webSocketExitDepth
|
||||
post_entry_restart_baseline_depth = $postRestartExitBaseline
|
||||
recovery_depth = $recoveryExitDepth
|
||||
}
|
||||
backend_fallback_queue = @{
|
||||
baseline_depth = $degradedBackendBaseline
|
||||
depth = $degradedBackendDepth
|
||||
}
|
||||
entry_restart_attempted = $true
|
||||
passed = $true
|
||||
checks = [ordered]@{
|
||||
production_forwarding_ready = ($readyBefore.report.production_payload_forwarding -eq $true)
|
||||
exit_production_forwarding_ready = ($exitReadyBefore.report.production_payload_forwarding -eq $true)
|
||||
route_intents_visible_before_post = (@($visibleConfig.synthetic_mesh_config.routes | Where-Object { $routeIDs -contains $_.route_id }).Count -ge 2)
|
||||
exit_route_intents_visible_before_post = (@($exitVisibleConfig.synthetic_mesh_config.routes | Where-Object { $routeIDs -contains $_.route_id }).Count -ge 2)
|
||||
entry_runtime_loaded_visible_config = ([string]$loadedConfig.report.config_version -ge [string]$visibleConfig.synthetic_mesh_config.config_version)
|
||||
exit_runtime_loaded_visible_config = ([string]$exitLoadedConfig.report.config_version -ge [string]$exitVisibleConfig.synthetic_mesh_config.config_version)
|
||||
signed_lease_ready = ($lease.status -eq "ready")
|
||||
warm_batches_accepted = ($warmOk -eq $WarmBatchCount)
|
||||
warm_exit_inbox_received = ($warmExitDepth -ge 8)
|
||||
websocket_send_accepted = ($webSocketResult.ok -eq $true)
|
||||
websocket_exit_inbox_grew = ($webSocketExitDepth -ge ($webSocketExitBaseline + 8))
|
||||
entry_restart_recovered = ($postRestartReady.report.production_payload_forwarding -eq $true -and [string]$postRestartLoadedConfig.report.config_version -ge [string]$postRestartVisibleConfig.synthetic_mesh_config.config_version)
|
||||
recovery_batches_accepted = ($recoveryOk -eq $RecoveryBatchCount)
|
||||
recovery_exit_inbox_grew = ($recoveryExitDepth -ge ($postRestartExitBaseline + 8))
|
||||
degraded_lease_visible = ($degradedLease.status -eq "degraded_fallback" -and $degradedLease.fallback.active -eq $true -and $degradedLease.fallback.degraded -eq $true)
|
||||
degraded_backend_fallback_received = ($degradedPost.ok -eq $true -and $degradedBackendDepth -ge ($degradedBackendBaseline + 8))
|
||||
route_intents_expired = ($expiredPrimary.route_intent.lifecycle_status -eq "expired" -and $expiredAlternate.route_intent.lifecycle_status -eq "expired")
|
||||
}
|
||||
telemetry = @{
|
||||
warm_ingress = $warmIngress.report.ingress
|
||||
websocket_ingress = $webSocketIngress.report.ingress
|
||||
recovery_ingress = $recoveryIngress.report.ingress
|
||||
exit_inbox = $finalExitRuntime.report.inbox
|
||||
websocket_result = $webSocketResult
|
||||
degraded_post = $degradedPost
|
||||
}
|
||||
}
|
||||
|
||||
$failedChecks = @($result.checks.GetEnumerator() | Where-Object { $_.Value -ne $true })
|
||||
if ($failedChecks.Count -gt 0) {
|
||||
throw "C18Z3 failed checks: $($failedChecks.Name -join ', ')"
|
||||
}
|
||||
}
|
||||
finally {
|
||||
if ($primaryRouteID) {
|
||||
try { Invoke-Api -Method POST -Path "/clusters/$ClusterID/mesh/route-intents/$primaryRouteID/expire" -Body @{ actor_user_id = $ActorUserID } | Out-Null } catch {}
|
||||
}
|
||||
if ($alternateRouteID) {
|
||||
try { Invoke-Api -Method POST -Path "/clusters/$ClusterID/mesh/route-intents/$alternateRouteID/expire" -Body @{ actor_user_id = $ActorUserID } | Out-Null } catch {}
|
||||
}
|
||||
if ($updatersStopped) {
|
||||
try { Start-TestUpdaters } catch { Write-Warning "Could not restart test updaters: $($_.Exception.Message)" }
|
||||
}
|
||||
}
|
||||
|
||||
$resultFullPath = Join-Path $repoRoot $ResultPath
|
||||
$resultDir = Split-Path -Parent $resultFullPath
|
||||
if (-not (Test-Path $resultDir)) {
|
||||
New-Item -ItemType Directory -Path $resultDir | Out-Null
|
||||
}
|
||||
$result | ConvertTo-Json -Depth 100 | Set-Content -Path $resultFullPath -Encoding UTF8
|
||||
Write-Host "C18Z3 live service-channel entry/ws/fallback smoke passed. Result: $resultFullPath"
|
||||
$result
|
||||
@@ -0,0 +1,96 @@
|
||||
param(
|
||||
[string]$ApiBaseUrl = "http://192.168.200.61:18121/api/v1",
|
||||
[string]$ClusterID = "cfc0743d-d960-49fb-9de8-96e063d5e4aa",
|
||||
[string]$ActorUserID = "f67d943f-5397-4b3a-a229-695fe67ad700",
|
||||
[string]$NodeID = "108a0d66-d65e-4dea-b9a8-135366bf7dba",
|
||||
[string]$ExpectedBackendImage = "rap-backend:fabric-service-channel-0.2.209",
|
||||
[string]$ExpectedNodeAgentImage = "rap-node-agent:0.2.208",
|
||||
[string]$DockerSSH = "test-docker",
|
||||
[string]$ResultPath = "artifacts\c18z30-node-agent-feedback-provenance-smoke-result.json"
|
||||
)
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
$scriptDir = Split-Path -Parent $MyInvocation.MyCommand.Path
|
||||
$repoRoot = (Resolve-Path (Join-Path $scriptDir "..\..")).ProviderPath
|
||||
|
||||
function Invoke-Api {
|
||||
param([string]$Path)
|
||||
return Invoke-RestMethod -Method Get -Uri "$ApiBaseUrl$Path"
|
||||
}
|
||||
|
||||
$backendLine = (& ssh $DockerSSH "docker ps --format '{{.Names}} {{.Image}} {{.Status}}' | grep '^rap_test_backend '") | Out-String
|
||||
$nodeLines = (& ssh $DockerSSH "docker ps --format '{{.Names}} {{.Image}} {{.Status}}' | grep '^rap_test_node_test_'") | Out-String
|
||||
|
||||
$heartbeat = (Invoke-Api -Path "/clusters/$ClusterID/nodes/$NodeID/heartbeats?actor_user_id=$ActorUserID&limit=1").heartbeats[0]
|
||||
$runtime = $heartbeat.metadata.fabric_service_channel_runtime_report
|
||||
$channelStats = @()
|
||||
if ($null -ne $runtime.ingress.flow_scheduler.channel_stats) {
|
||||
$channelStats = @($runtime.ingress.flow_scheduler.channel_stats.PSObject.Properties | ForEach-Object { $_.Value })
|
||||
}
|
||||
$runtimeProvenance = @($channelStats | Where-Object {
|
||||
$_.recovery_policy_fingerprint -and $_.route_policy_version -and $_.route_generation
|
||||
})
|
||||
|
||||
$feedback = (Invoke-Api -Path "/clusters/$ClusterID/nodes/$NodeID/mesh/synthetic-config?actor_user_id=$ActorUserID").synthetic_mesh_config.service_channel_route_feedback
|
||||
$observations = @()
|
||||
if ($null -ne $feedback.observations) {
|
||||
$observations = @($feedback.observations)
|
||||
}
|
||||
$feedbackWithProvenance = @($observations | Where-Object {
|
||||
$_.payload.recovery_policy_fingerprint -and $_.payload.route_policy_version -and $_.payload.route_generation
|
||||
})
|
||||
|
||||
$missing = if ($feedback.PSObject.Properties["missing_provenance_count"]) { [int]$feedback.missing_provenance_count } else { 0 }
|
||||
$stalePolicy = if ($feedback.PSObject.Properties["stale_policy_count"]) { [int]$feedback.stale_policy_count } else { 0 }
|
||||
$staleGeneration = if ($feedback.PSObject.Properties["stale_generation_count"]) { [int]$feedback.stale_generation_count } else { 0 }
|
||||
|
||||
$result = [ordered]@{
|
||||
schema_version = "c18z30.node_agent_feedback_provenance_smoke.v1"
|
||||
passed = [bool](
|
||||
$backendLine.Contains($ExpectedBackendImage) -and
|
||||
$nodeLines.Contains($ExpectedNodeAgentImage) -and
|
||||
[string]$heartbeat.reported_version -eq "0.2.208" -and
|
||||
$channelStats.Count -gt 0 -and
|
||||
$runtimeProvenance.Count -eq $channelStats.Count -and
|
||||
$observations.Count -gt 0 -and
|
||||
$feedbackWithProvenance.Count -eq $observations.Count -and
|
||||
$missing -eq 0 -and
|
||||
$stalePolicy -eq 0 -and
|
||||
$staleGeneration -eq 0
|
||||
)
|
||||
checks = [ordered]@{
|
||||
backend_expected_image_deployed = $backendLine.Contains($ExpectedBackendImage)
|
||||
node_agent_expected_image_deployed = $nodeLines.Contains($ExpectedNodeAgentImage)
|
||||
heartbeat_reported_expected_version = ([string]$heartbeat.reported_version -eq "0.2.208")
|
||||
runtime_channel_stats_have_provenance = ($channelStats.Count -gt 0 -and $runtimeProvenance.Count -eq $channelStats.Count)
|
||||
feedback_observations_have_provenance = ($observations.Count -gt 0 -and $feedbackWithProvenance.Count -eq $observations.Count)
|
||||
feedback_provenance_is_current = ($missing -eq 0 -and $stalePolicy -eq 0 -and $staleGeneration -eq 0)
|
||||
}
|
||||
summary = [ordered]@{
|
||||
backend_container = $backendLine.Trim()
|
||||
node_containers = $nodeLines.Trim()
|
||||
heartbeat_version = $heartbeat.reported_version
|
||||
runtime_channel_stats = $channelStats.Count
|
||||
runtime_provenance_channel_stats = $runtimeProvenance.Count
|
||||
feedback_observations = $observations.Count
|
||||
feedback_observations_with_provenance = $feedbackWithProvenance.Count
|
||||
feedback_counts = [ordered]@{
|
||||
missing_provenance = $missing
|
||||
stale_policy = $stalePolicy
|
||||
stale_generation = $staleGeneration
|
||||
}
|
||||
sample_feedback = ($feedbackWithProvenance | Select-Object -First 1)
|
||||
}
|
||||
}
|
||||
|
||||
$resolvedResultPath = Join-Path $repoRoot $ResultPath
|
||||
$result | ConvertTo-Json -Depth 100 | Set-Content -Path $resolvedResultPath -Encoding UTF8
|
||||
|
||||
if (-not $result.passed) {
|
||||
throw "C18Z30 node-agent feedback provenance smoke failed. Result: $resolvedResultPath"
|
||||
}
|
||||
|
||||
Write-Host "C18Z30 node-agent feedback provenance smoke passed. Result: $resolvedResultPath"
|
||||
$result
|
||||
@@ -0,0 +1,117 @@
|
||||
param(
|
||||
[string]$ApiBaseUrl = "http://192.168.200.61:18121/api/v1",
|
||||
[string]$ClusterID = "cfc0743d-d960-49fb-9de8-96e063d5e4aa",
|
||||
[string]$ActorUserID = "f67d943f-5397-4b3a-a229-695fe67ad700",
|
||||
[string]$EntryNodeName = "test-1",
|
||||
[string]$ExitNodeName = "test-2",
|
||||
[string]$EntryBaseUrl = "http://192.168.200.61:19131",
|
||||
[string]$DockerSSH = "test-docker",
|
||||
[string]$ExpectedBackendImage = "rap-backend:fabric-service-channel-0.2.211",
|
||||
[string]$ExpectedNodeAgentImage = "rap-node-agent:0.2.208",
|
||||
[string]$ResultPath = "artifacts\c18z31-service-channel-rebuild-ledger-smoke-result.json"
|
||||
)
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
$scriptDir = Split-Path -Parent $MyInvocation.MyCommand.Path
|
||||
$repoRoot = (Resolve-Path (Join-Path $scriptDir "..\..")).ProviderPath
|
||||
|
||||
function Invoke-Api {
|
||||
param([string]$Path)
|
||||
return Invoke-RestMethod -Method Get -Uri "$ApiBaseUrl$Path" -TimeoutSec 30
|
||||
}
|
||||
|
||||
function Get-NodeByName {
|
||||
param([string]$Name)
|
||||
$nodes = (Invoke-Api -Path "/clusters/$ClusterID/nodes?actor_user_id=$ActorUserID").nodes
|
||||
$node = @($nodes | Where-Object { $_.name -eq $Name }) | Select-Object -First 1
|
||||
if ($null -eq $node) {
|
||||
throw "Node '$Name' was not found in cluster $ClusterID"
|
||||
}
|
||||
return $node
|
||||
}
|
||||
|
||||
$baseSmokePath = Join-Path $scriptDir "c18z6-live-service-channel-active-rebuild-smoke.ps1"
|
||||
$baseResultPath = "artifacts\c18z31-base-active-rebuild-smoke-result.json"
|
||||
& $baseSmokePath `
|
||||
-ApiBaseUrl $ApiBaseUrl `
|
||||
-ClusterID $ClusterID `
|
||||
-ActorUserID $ActorUserID `
|
||||
-EntryNodeName $EntryNodeName `
|
||||
-ExitNodeName $ExitNodeName `
|
||||
-EntryBaseUrl $EntryBaseUrl `
|
||||
-DockerSSH $DockerSSH `
|
||||
-PreRebuildBatchCount 4 `
|
||||
-PostRebuildBatchCount 8 `
|
||||
-PacketsPerBatch 4 `
|
||||
-BatchDelayMilliseconds 20 `
|
||||
-RequiredNodeVersion "0.2.208" `
|
||||
-ResultPath $baseResultPath | Out-Null
|
||||
|
||||
$baseResultFullPath = Join-Path $repoRoot $baseResultPath
|
||||
$baseResult = Get-Content $baseResultFullPath -Raw | ConvertFrom-Json
|
||||
$entryNode = Get-NodeByName -Name $EntryNodeName
|
||||
$primaryRouteID = [string]$baseResult.route_intents.primary_route_intent_id
|
||||
$alternateRouteID = [string]$baseResult.route_intents.alternate_route_intent_id
|
||||
|
||||
$ledger = (Invoke-Api -Path "/clusters/$ClusterID/fabric/service-channels/rebuild-attempts?actor_user_id=$ActorUserID&reporter_node_id=$($entryNode.id)&route_id=$primaryRouteID&limit=20").rebuild_attempts
|
||||
$matching = @($ledger | Where-Object {
|
||||
$_.route_id -eq $primaryRouteID -and
|
||||
$_.replacement_route_id -eq $alternateRouteID -and
|
||||
$_.rebuild_status -eq "applied" -and
|
||||
$_.outcome -eq "replacement_selected"
|
||||
})
|
||||
|
||||
$backendLine = (& ssh $DockerSSH "docker ps --format '{{.Names}} {{.Image}} {{.Status}}' | grep '^rap_test_backend '") | Out-String
|
||||
$nodeLines = (& ssh $DockerSSH "docker ps --format '{{.Names}} {{.Image}} {{.Status}}' | grep '^rap_test_node_test_'") | Out-String
|
||||
|
||||
$sample = $matching | Select-Object -First 1
|
||||
$result = [ordered]@{
|
||||
schema_version = "c18z31.service_channel_rebuild_ledger_smoke.v1"
|
||||
cluster_id = $ClusterID
|
||||
entry_node = @{ name = $entryNode.name; id = $entryNode.id }
|
||||
primary_route_id = $primaryRouteID
|
||||
alternate_route_id = $alternateRouteID
|
||||
passed = [bool](
|
||||
$backendLine.Contains($ExpectedBackendImage) -and
|
||||
$nodeLines.Contains($ExpectedNodeAgentImage) -and
|
||||
$matching.Count -ge 1 -and
|
||||
[string]$sample.rebuild_request_id -ne "" -and
|
||||
[string]$sample.policy_fingerprint -ne "" -and
|
||||
[string]$sample.generation -ne "" -and
|
||||
@($sample.old_hops).Count -gt 0 -and
|
||||
@($sample.replacement_hops).Count -gt 0
|
||||
)
|
||||
checks = [ordered]@{
|
||||
backend_expected_image_deployed = $backendLine.Contains($ExpectedBackendImage)
|
||||
node_agent_expected_image_deployed = $nodeLines.Contains($ExpectedNodeAgentImage)
|
||||
ledger_has_matching_applied_rebuild = ($matching.Count -ge 1)
|
||||
ledger_has_request_id = ([string]$sample.rebuild_request_id -ne "")
|
||||
ledger_has_policy_fingerprint = ([string]$sample.policy_fingerprint -ne "")
|
||||
ledger_has_generation = ([string]$sample.generation -ne "")
|
||||
ledger_has_old_and_replacement_hops = (@($sample.old_hops).Count -gt 0 -and @($sample.replacement_hops).Count -gt 0)
|
||||
}
|
||||
summary = [ordered]@{
|
||||
backend_container = $backendLine.Trim()
|
||||
node_containers = $nodeLines.Trim()
|
||||
ledger_count = @($ledger).Count
|
||||
matching_count = $matching.Count
|
||||
sample = $sample
|
||||
base_smoke_result = $baseResultFullPath
|
||||
}
|
||||
}
|
||||
|
||||
$resultFullPath = Join-Path $repoRoot $ResultPath
|
||||
$resultDir = Split-Path -Parent $resultFullPath
|
||||
if (-not (Test-Path $resultDir)) {
|
||||
New-Item -ItemType Directory -Path $resultDir | Out-Null
|
||||
}
|
||||
$result | ConvertTo-Json -Depth 100 | Set-Content -Path $resultFullPath -Encoding UTF8
|
||||
|
||||
if (-not $result.passed) {
|
||||
throw "C18Z31 service-channel rebuild ledger smoke failed. Result: $resultFullPath"
|
||||
}
|
||||
|
||||
Write-Host "C18Z31 service-channel rebuild ledger smoke passed. Result: $resultFullPath"
|
||||
$result
|
||||
@@ -0,0 +1,111 @@
|
||||
param(
|
||||
[string]$ApiBaseUrl = "http://192.168.200.61:18121/api/v1",
|
||||
[string]$ClusterID = "cfc0743d-d960-49fb-9de8-96e063d5e4aa",
|
||||
[string]$ActorUserID = "f67d943f-5397-4b3a-a229-695fe67ad700",
|
||||
[string]$EntryNodeName = "test-1",
|
||||
[string]$ExitNodeName = "test-2",
|
||||
[string]$EntryBaseUrl = "http://192.168.200.61:19131",
|
||||
[string]$DockerSSH = "test-docker",
|
||||
[string]$ExpectedBackendImage = "rap-backend:fabric-service-channel-0.2.213",
|
||||
[string]$ExpectedNodeAgentImage = "rap-node-agent:0.2.208",
|
||||
[string]$ResultPath = "artifacts\c18z32-service-channel-rebuild-timeline-smoke-result.json"
|
||||
)
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
$scriptDir = Split-Path -Parent $MyInvocation.MyCommand.Path
|
||||
$repoRoot = (Resolve-Path (Join-Path $scriptDir "..\..")).ProviderPath
|
||||
$baseResultPath = "artifacts\c18z32-base-rebuild-ledger-smoke-result.json"
|
||||
|
||||
& (Join-Path $scriptDir "c18z31-service-channel-rebuild-ledger-smoke.ps1") `
|
||||
-ApiBaseUrl $ApiBaseUrl `
|
||||
-ClusterID $ClusterID `
|
||||
-ActorUserID $ActorUserID `
|
||||
-EntryNodeName $EntryNodeName `
|
||||
-ExitNodeName $ExitNodeName `
|
||||
-EntryBaseUrl $EntryBaseUrl `
|
||||
-DockerSSH $DockerSSH `
|
||||
-ExpectedBackendImage $ExpectedBackendImage `
|
||||
-ExpectedNodeAgentImage $ExpectedNodeAgentImage `
|
||||
-ResultPath $baseResultPath | Out-Null
|
||||
|
||||
$baseResult = Get-Content (Join-Path $repoRoot $baseResultPath) -Raw | ConvertFrom-Json
|
||||
$entryNodeID = [string]$baseResult.entry_node.id
|
||||
$primaryRouteID = [string]$baseResult.primary_route_id
|
||||
$alternateRouteID = [string]$baseResult.alternate_route_id
|
||||
|
||||
function Get-Prop {
|
||||
param([object]$Object, [string]$Name)
|
||||
if ($null -eq $Object) { return $null }
|
||||
$prop = $Object.PSObject.Properties[$Name]
|
||||
if ($null -eq $prop) { return $null }
|
||||
return $prop.Value
|
||||
}
|
||||
|
||||
$ledger = (Invoke-RestMethod -Method Get -Uri "$ApiBaseUrl/clusters/$ClusterID/fabric/service-channels/rebuild-attempts?actor_user_id=$ActorUserID&reporter_node_id=$entryNodeID&route_id=$primaryRouteID&replacement_route_id=$alternateRouteID&limit=20&enrichment=deep" -TimeoutSec 30).rebuild_attempts
|
||||
$matching = @($ledger | Where-Object {
|
||||
$_.route_id -eq $primaryRouteID -and
|
||||
$_.replacement_route_id -eq $alternateRouteID -and
|
||||
$_.rebuild_status -eq "applied" -and
|
||||
(Get-Prop $_ "node_transition_matched") -eq $true -and
|
||||
(Get-Prop $_ "node_route_generation_matched") -eq $true -and
|
||||
(Get-Prop $_ "post_rebuild_selected_route_id")
|
||||
})
|
||||
$sample = $matching | Select-Object -First 1
|
||||
$timelineStages = @()
|
||||
if ($null -ne $sample -and $null -ne $sample.timeline) {
|
||||
$timelineStages = @($sample.timeline | ForEach-Object { [string]$_.stage })
|
||||
}
|
||||
|
||||
$backendLine = (& ssh $DockerSSH "docker ps --format '{{.Names}} {{.Image}} {{.Status}}' | grep '^rap_test_backend '") | Out-String
|
||||
$nodeLines = (& ssh $DockerSSH "docker ps --format '{{.Names}} {{.Image}} {{.Status}}' | grep '^rap_test_node_test_'") | Out-String
|
||||
|
||||
$result = [ordered]@{
|
||||
schema_version = "c18z32.service_channel_rebuild_timeline_smoke.v1"
|
||||
cluster_id = $ClusterID
|
||||
primary_route_id = $primaryRouteID
|
||||
alternate_route_id = $alternateRouteID
|
||||
passed = [bool](
|
||||
$backendLine.Contains($ExpectedBackendImage) -and
|
||||
$nodeLines.Contains($ExpectedNodeAgentImage) -and
|
||||
$matching.Count -ge 1 -and
|
||||
$timelineStages -contains "backend_decision" -and
|
||||
$timelineStages -contains "node_route_manager_transition" -and
|
||||
$timelineStages -contains "node_route_generation_apply" -and
|
||||
$timelineStages -contains "post_rebuild_traffic" -and
|
||||
[string]$sample.node_transition_status -eq "applied_rebuild" -and
|
||||
[string]$sample.post_rebuild_selected_route_id -eq $alternateRouteID
|
||||
)
|
||||
checks = [ordered]@{
|
||||
backend_expected_image_deployed = $backendLine.Contains($ExpectedBackendImage)
|
||||
node_agent_expected_image_deployed = $nodeLines.Contains($ExpectedNodeAgentImage)
|
||||
timeline_has_backend_decision = ($timelineStages -contains "backend_decision")
|
||||
timeline_has_node_route_manager_transition = ($timelineStages -contains "node_route_manager_transition")
|
||||
timeline_has_node_route_generation_apply = ($timelineStages -contains "node_route_generation_apply")
|
||||
timeline_has_post_rebuild_traffic = ($timelineStages -contains "post_rebuild_traffic")
|
||||
node_transition_applied_rebuild = ([string]$sample.node_transition_status -eq "applied_rebuild")
|
||||
post_rebuild_uses_replacement = ([string]$sample.post_rebuild_selected_route_id -eq $alternateRouteID)
|
||||
}
|
||||
summary = [ordered]@{
|
||||
backend_container = $backendLine.Trim()
|
||||
node_containers = $nodeLines.Trim()
|
||||
matching_count = $matching.Count
|
||||
sample = $sample
|
||||
base_smoke_result = (Join-Path $repoRoot $baseResultPath)
|
||||
}
|
||||
}
|
||||
|
||||
$resultFullPath = Join-Path $repoRoot $ResultPath
|
||||
$resultDir = Split-Path -Parent $resultFullPath
|
||||
if (-not (Test-Path $resultDir)) {
|
||||
New-Item -ItemType Directory -Path $resultDir | Out-Null
|
||||
}
|
||||
$result | ConvertTo-Json -Depth 100 | Set-Content -Path $resultFullPath -Encoding UTF8
|
||||
|
||||
if (-not $result.passed) {
|
||||
throw "C18Z32 service-channel rebuild timeline smoke failed. Result: $resultFullPath"
|
||||
}
|
||||
|
||||
Write-Host "C18Z32 service-channel rebuild timeline smoke passed. Result: $resultFullPath"
|
||||
$result
|
||||
@@ -0,0 +1,116 @@
|
||||
param(
|
||||
[string]$ApiBaseUrl = "http://192.168.200.61:18121/api/v1",
|
||||
[string]$ClusterID = "cfc0743d-d960-49fb-9de8-96e063d5e4aa",
|
||||
[string]$ActorUserID = "f67d943f-5397-4b3a-a229-695fe67ad700",
|
||||
[string]$EntryNodeName = "test-1",
|
||||
[string]$ExitNodeName = "test-2",
|
||||
[string]$EntryBaseUrl = "http://192.168.200.61:19131",
|
||||
[string]$DockerSSH = "test-docker",
|
||||
[string]$ExpectedBackendImage = "rap-backend:fabric-service-channel-0.2.214",
|
||||
[string]$ExpectedNodeAgentImage = "rap-node-agent:0.2.208",
|
||||
[string]$ResultPath = "artifacts\c18z33-service-channel-rebuild-guard-smoke-result.json"
|
||||
)
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
$scriptDir = Split-Path -Parent $MyInvocation.MyCommand.Path
|
||||
$repoRoot = (Resolve-Path (Join-Path $scriptDir "..\..")).ProviderPath
|
||||
$baseResultPath = "artifacts\c18z33-base-rebuild-ledger-smoke-result.json"
|
||||
|
||||
& (Join-Path $scriptDir "c18z31-service-channel-rebuild-ledger-smoke.ps1") `
|
||||
-ApiBaseUrl $ApiBaseUrl `
|
||||
-ClusterID $ClusterID `
|
||||
-ActorUserID $ActorUserID `
|
||||
-EntryNodeName $EntryNodeName `
|
||||
-ExitNodeName $ExitNodeName `
|
||||
-EntryBaseUrl $EntryBaseUrl `
|
||||
-DockerSSH $DockerSSH `
|
||||
-ExpectedBackendImage $ExpectedBackendImage `
|
||||
-ExpectedNodeAgentImage $ExpectedNodeAgentImage `
|
||||
-ResultPath $baseResultPath | Out-Null
|
||||
|
||||
$baseResult = Get-Content (Join-Path $repoRoot $baseResultPath) -Raw | ConvertFrom-Json
|
||||
$entryNodeID = [string]$baseResult.entry_node.id
|
||||
$primaryRouteID = [string]$baseResult.primary_route_id
|
||||
$alternateRouteID = [string]$baseResult.alternate_route_id
|
||||
|
||||
function Get-Prop {
|
||||
param([object]$Object, [string]$Name)
|
||||
if ($null -eq $Object) { return $null }
|
||||
$prop = $Object.PSObject.Properties[$Name]
|
||||
if ($null -eq $prop) { return $null }
|
||||
return $prop.Value
|
||||
}
|
||||
|
||||
$ledger = (Invoke-RestMethod -Method Get -Uri "$ApiBaseUrl/clusters/$ClusterID/fabric/service-channels/rebuild-attempts?actor_user_id=$ActorUserID&reporter_node_id=$entryNodeID&route_id=$primaryRouteID&replacement_route_id=$alternateRouteID&limit=20&enrichment=deep" -TimeoutSec 30).rebuild_attempts
|
||||
$matching = @($ledger | Where-Object {
|
||||
$_.route_id -eq $primaryRouteID -and
|
||||
$_.replacement_route_id -eq $alternateRouteID -and
|
||||
$_.rebuild_status -eq "applied" -and
|
||||
(Get-Prop $_ "node_transition_matched") -eq $true -and
|
||||
(Get-Prop $_ "node_route_generation_matched") -eq $true -and
|
||||
(Get-Prop $_ "post_rebuild_selected_route_id") -and
|
||||
[string](Get-Prop $_ "guard_status") -eq "ok"
|
||||
})
|
||||
$sample = $matching | Select-Object -First 1
|
||||
$timelineStages = @()
|
||||
if ($null -ne $sample -and $null -ne $sample.timeline) {
|
||||
$timelineStages = @($sample.timeline | ForEach-Object { [string]$_.stage })
|
||||
}
|
||||
|
||||
$backendLine = (& ssh $DockerSSH "docker ps --format '{{.Names}} {{.Image}} {{.Status}}' | grep '^rap_test_backend '") | Out-String
|
||||
$nodeLines = (& ssh $DockerSSH "docker ps --format '{{.Names}} {{.Image}} {{.Status}}' | grep '^rap_test_node_test_'") | Out-String
|
||||
|
||||
$result = [ordered]@{
|
||||
schema_version = "c18z33.service_channel_rebuild_guard_smoke.v1"
|
||||
cluster_id = $ClusterID
|
||||
primary_route_id = $primaryRouteID
|
||||
alternate_route_id = $alternateRouteID
|
||||
passed = [bool](
|
||||
$backendLine.Contains($ExpectedBackendImage) -and
|
||||
$nodeLines.Contains($ExpectedNodeAgentImage) -and
|
||||
$matching.Count -ge 1 -and
|
||||
$timelineStages -contains "backend_decision" -and
|
||||
$timelineStages -contains "node_route_manager_transition" -and
|
||||
$timelineStages -contains "node_route_generation_apply" -and
|
||||
$timelineStages -contains "post_rebuild_traffic" -and
|
||||
[string]$sample.node_transition_status -eq "applied_rebuild" -and
|
||||
[string]$sample.post_rebuild_selected_route_id -eq $alternateRouteID -and
|
||||
[string]$sample.guard_status -eq "ok" -and
|
||||
[string]$sample.guard_severity -eq "good"
|
||||
)
|
||||
checks = [ordered]@{
|
||||
backend_expected_image_deployed = $backendLine.Contains($ExpectedBackendImage)
|
||||
node_agent_expected_image_deployed = $nodeLines.Contains($ExpectedNodeAgentImage)
|
||||
timeline_has_backend_decision = ($timelineStages -contains "backend_decision")
|
||||
timeline_has_node_route_manager_transition = ($timelineStages -contains "node_route_manager_transition")
|
||||
timeline_has_node_route_generation_apply = ($timelineStages -contains "node_route_generation_apply")
|
||||
timeline_has_post_rebuild_traffic = ($timelineStages -contains "post_rebuild_traffic")
|
||||
node_transition_applied_rebuild = ([string]$sample.node_transition_status -eq "applied_rebuild")
|
||||
post_rebuild_uses_replacement = ([string]$sample.post_rebuild_selected_route_id -eq $alternateRouteID)
|
||||
guard_status_ok = ([string]$sample.guard_status -eq "ok")
|
||||
guard_severity_good = ([string]$sample.guard_severity -eq "good")
|
||||
}
|
||||
summary = [ordered]@{
|
||||
backend_container = $backendLine.Trim()
|
||||
node_containers = $nodeLines.Trim()
|
||||
matching_count = $matching.Count
|
||||
sample = $sample
|
||||
base_smoke_result = (Join-Path $repoRoot $baseResultPath)
|
||||
}
|
||||
}
|
||||
|
||||
$resultFullPath = Join-Path $repoRoot $ResultPath
|
||||
$resultDir = Split-Path -Parent $resultFullPath
|
||||
if (-not (Test-Path $resultDir)) {
|
||||
New-Item -ItemType Directory -Path $resultDir | Out-Null
|
||||
}
|
||||
$result | ConvertTo-Json -Depth 100 | Set-Content -Path $resultFullPath -Encoding UTF8
|
||||
|
||||
if (-not $result.passed) {
|
||||
throw "C18Z33 service-channel rebuild guard smoke failed. Result: $resultFullPath"
|
||||
}
|
||||
|
||||
Write-Host "C18Z33 service-channel rebuild guard smoke passed. Result: $resultFullPath"
|
||||
$result
|
||||
@@ -0,0 +1,97 @@
|
||||
param(
|
||||
[string]$ApiBaseUrl = "http://192.168.200.61:18121/api/v1",
|
||||
[string]$ClusterID = "cfc0743d-d960-49fb-9de8-96e063d5e4aa",
|
||||
[string]$ActorUserID = "f67d943f-5397-4b3a-a229-695fe67ad700",
|
||||
[string]$EntryNodeName = "test-1",
|
||||
[string]$ExitNodeName = "test-2",
|
||||
[string]$EntryBaseUrl = "http://192.168.200.61:19131",
|
||||
[string]$DockerSSH = "test-docker",
|
||||
[string]$ExpectedBackendImage = "rap-backend:fabric-service-channel-0.2.215",
|
||||
[string]$ExpectedNodeAgentImage = "rap-node-agent:0.2.208",
|
||||
[string]$ResultPath = "artifacts\c18z34-service-channel-rebuild-health-smoke-result.json"
|
||||
)
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
$scriptDir = Split-Path -Parent $MyInvocation.MyCommand.Path
|
||||
$repoRoot = (Resolve-Path (Join-Path $scriptDir "..\..")).ProviderPath
|
||||
$baseResultPath = "artifacts\c18z34-base-rebuild-guard-smoke-result.json"
|
||||
|
||||
& (Join-Path $scriptDir "c18z33-service-channel-rebuild-guard-smoke.ps1") `
|
||||
-ApiBaseUrl $ApiBaseUrl `
|
||||
-ClusterID $ClusterID `
|
||||
-ActorUserID $ActorUserID `
|
||||
-EntryNodeName $EntryNodeName `
|
||||
-ExitNodeName $ExitNodeName `
|
||||
-EntryBaseUrl $EntryBaseUrl `
|
||||
-DockerSSH $DockerSSH `
|
||||
-ExpectedBackendImage $ExpectedBackendImage `
|
||||
-ExpectedNodeAgentImage $ExpectedNodeAgentImage `
|
||||
-ResultPath $baseResultPath | Out-Null
|
||||
|
||||
$health = (Invoke-RestMethod -Method Get -Uri "$ApiBaseUrl/clusters/$ClusterID/fabric/service-channels/rebuild-health?actor_user_id=$ActorUserID&limit=5" -TimeoutSec 30).rebuild_health
|
||||
$backendLine = (& ssh $DockerSSH "docker ps --format '{{.Names}} {{.Image}} {{.Status}}' | grep '^rap_test_backend '") | Out-String
|
||||
$nodeLines = (& ssh $DockerSSH "docker ps --format '{{.Names}} {{.Image}} {{.Status}}' | grep '^rap_test_node_test_'") | Out-String
|
||||
|
||||
function Get-Prop {
|
||||
param([object]$Object, [string]$Name)
|
||||
if ($null -eq $Object) { return $null }
|
||||
$prop = $Object.PSObject.Properties[$Name]
|
||||
if ($null -eq $prop) { return $null }
|
||||
return $prop.Value
|
||||
}
|
||||
|
||||
$statusCounts = Get-Prop $health "counts_by_guard_status"
|
||||
$severityCounts = Get-Prop $health "counts_by_guard_severity"
|
||||
$okCount = 0
|
||||
if ($null -ne $statusCounts -and $null -ne $statusCounts.PSObject.Properties["ok"]) {
|
||||
$okCount = [int]$statusCounts.ok
|
||||
}
|
||||
$goodCount = 0
|
||||
if ($null -ne $severityCounts -and $null -ne $severityCounts.PSObject.Properties["good"]) {
|
||||
$goodCount = [int]$severityCounts.good
|
||||
}
|
||||
|
||||
$result = [ordered]@{
|
||||
schema_version = "c18z34.service_channel_rebuild_health_smoke.v1"
|
||||
cluster_id = $ClusterID
|
||||
passed = [bool](
|
||||
$backendLine.Contains($ExpectedBackendImage) -and
|
||||
$nodeLines.Contains($ExpectedNodeAgentImage) -and
|
||||
[int]$health.total_attempts -gt 0 -and
|
||||
[int]$health.good_count -ge 1 -and
|
||||
$okCount -ge 1 -and
|
||||
$goodCount -ge 1 -and
|
||||
[string]$health.recommended_operator_action
|
||||
)
|
||||
checks = [ordered]@{
|
||||
backend_expected_image_deployed = $backendLine.Contains($ExpectedBackendImage)
|
||||
node_agent_expected_image_deployed = $nodeLines.Contains($ExpectedNodeAgentImage)
|
||||
has_attempts = ([int]$health.total_attempts -gt 0)
|
||||
has_good_count = ([int]$health.good_count -ge 1)
|
||||
status_ok_counted = ($okCount -ge 1)
|
||||
severity_good_counted = ($goodCount -ge 1)
|
||||
has_operator_action = ([string]$health.recommended_operator_action).Length -gt 0
|
||||
}
|
||||
summary = [ordered]@{
|
||||
backend_container = $backendLine.Trim()
|
||||
node_containers = $nodeLines.Trim()
|
||||
health = $health
|
||||
base_smoke_result = (Join-Path $repoRoot $baseResultPath)
|
||||
}
|
||||
}
|
||||
|
||||
$resultFullPath = Join-Path $repoRoot $ResultPath
|
||||
$resultDir = Split-Path -Parent $resultFullPath
|
||||
if (-not (Test-Path $resultDir)) {
|
||||
New-Item -ItemType Directory -Path $resultDir | Out-Null
|
||||
}
|
||||
$result | ConvertTo-Json -Depth 100 | Set-Content -Path $resultFullPath -Encoding UTF8
|
||||
|
||||
if (-not $result.passed) {
|
||||
throw "C18Z34 service-channel rebuild health smoke failed. Result: $resultFullPath"
|
||||
}
|
||||
|
||||
Write-Host "C18Z34 service-channel rebuild health smoke passed. Result: $resultFullPath"
|
||||
$result
|
||||
@@ -0,0 +1,102 @@
|
||||
param(
|
||||
[string]$ApiBaseUrl = "http://192.168.200.61:18121/api/v1",
|
||||
[string]$ClusterID = "cfc0743d-d960-49fb-9de8-96e063d5e4aa",
|
||||
[string]$ActorUserID = "f67d943f-5397-4b3a-a229-695fe67ad700",
|
||||
[string]$EntryNodeName = "test-1",
|
||||
[string]$ExitNodeName = "test-2",
|
||||
[string]$EntryBaseUrl = "http://192.168.200.61:19131",
|
||||
[string]$DockerSSH = "test-docker",
|
||||
[string]$ExpectedBackendImage = "rap-backend:fabric-service-channel-0.2.216",
|
||||
[string]$ExpectedNodeAgentImage = "rap-node-agent:0.2.208",
|
||||
[string]$ResultPath = "artifacts\c18z35-service-channel-rebuild-alert-silence-smoke-result.json"
|
||||
)
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
$scriptDir = Split-Path -Parent $MyInvocation.MyCommand.Path
|
||||
$repoRoot = (Resolve-Path (Join-Path $scriptDir "..\..")).ProviderPath
|
||||
$baseResultPath = "artifacts\c18z35-base-rebuild-health-smoke-result.json"
|
||||
|
||||
& (Join-Path $scriptDir "c18z34-service-channel-rebuild-health-smoke.ps1") `
|
||||
-ApiBaseUrl $ApiBaseUrl `
|
||||
-ClusterID $ClusterID `
|
||||
-ActorUserID $ActorUserID `
|
||||
-EntryNodeName $EntryNodeName `
|
||||
-ExitNodeName $ExitNodeName `
|
||||
-EntryBaseUrl $EntryBaseUrl `
|
||||
-DockerSSH $DockerSSH `
|
||||
-ExpectedBackendImage $ExpectedBackendImage `
|
||||
-ExpectedNodeAgentImage $ExpectedNodeAgentImage `
|
||||
-ResultPath $baseResultPath | Out-Null
|
||||
|
||||
$before = (Invoke-RestMethod -Method Get -Uri "$ApiBaseUrl/clusters/$ClusterID/fabric/service-channels/rebuild-health?actor_user_id=$ActorUserID&limit=5" -TimeoutSec 30).rebuild_health
|
||||
$target = @($before.most_recent_bad_attempts | Select-Object -First 1)[0]
|
||||
if ($null -eq $target) {
|
||||
throw "C18Z35 requires at least one active bad rebuild attempt to silence"
|
||||
}
|
||||
|
||||
$payload = @{
|
||||
actor_user_id = $ActorUserID
|
||||
reporter_node_id = [string]$target.reporter_node_id
|
||||
route_id = [string]$target.route_id
|
||||
guard_status = [string]$target.guard_status
|
||||
generation = [string]$target.generation
|
||||
reason = "c18z35 smoke acknowledged known rebuild alert"
|
||||
ttl_seconds = 21600
|
||||
}
|
||||
$silence = (Invoke-RestMethod -Method Post -Uri "$ApiBaseUrl/clusters/$ClusterID/fabric/service-channels/rebuild-health/silences" -ContentType "application/json" -Body ($payload | ConvertTo-Json -Depth 20) -TimeoutSec 30).rebuild_alert_silence
|
||||
$after = (Invoke-RestMethod -Method Get -Uri "$ApiBaseUrl/clusters/$ClusterID/fabric/service-channels/rebuild-health?actor_user_id=$ActorUserID&limit=5" -TimeoutSec 30).rebuild_health
|
||||
$stillActiveSameGeneration = @($after.most_recent_bad_attempts | Where-Object {
|
||||
$_.reporter_node_id -eq $payload.reporter_node_id -and
|
||||
$_.route_id -eq $payload.route_id -and
|
||||
$_.guard_status -eq $payload.guard_status -and
|
||||
([string]$_.generation) -eq $payload.generation
|
||||
})
|
||||
|
||||
$backendLine = (& ssh $DockerSSH "docker ps --format '{{.Names}} {{.Image}} {{.Status}}' | grep '^rap_test_backend '") | Out-String
|
||||
$nodeLines = (& ssh $DockerSSH "docker ps --format '{{.Names}} {{.Image}} {{.Status}}' | grep '^rap_test_node_test_'") | Out-String
|
||||
|
||||
$result = [ordered]@{
|
||||
schema_version = "c18z35.service_channel_rebuild_alert_silence_smoke.v1"
|
||||
cluster_id = $ClusterID
|
||||
passed = [bool](
|
||||
$backendLine.Contains($ExpectedBackendImage) -and
|
||||
$nodeLines.Contains($ExpectedNodeAgentImage) -and
|
||||
[string]$silence.id -and
|
||||
[int]$after.silenced_count -gt [int]$before.silenced_count -and
|
||||
[int]$after.active_bad_count -lt [int]$before.active_bad_count -and
|
||||
$stillActiveSameGeneration.Count -eq 0
|
||||
)
|
||||
checks = [ordered]@{
|
||||
backend_expected_image_deployed = $backendLine.Contains($ExpectedBackendImage)
|
||||
node_agent_expected_image_deployed = $nodeLines.Contains($ExpectedNodeAgentImage)
|
||||
silence_created = ([string]$silence.id).Length -gt 0
|
||||
silenced_count_increased = ([int]$after.silenced_count -gt [int]$before.silenced_count)
|
||||
active_bad_count_decreased = ([int]$after.active_bad_count -lt [int]$before.active_bad_count)
|
||||
exact_generation_removed_from_active_bad = ($stillActiveSameGeneration.Count -eq 0)
|
||||
}
|
||||
summary = [ordered]@{
|
||||
backend_container = $backendLine.Trim()
|
||||
node_containers = $nodeLines.Trim()
|
||||
target = $payload
|
||||
silence = $silence
|
||||
before = $before
|
||||
after = $after
|
||||
base_smoke_result = (Join-Path $repoRoot $baseResultPath)
|
||||
}
|
||||
}
|
||||
|
||||
$resultFullPath = Join-Path $repoRoot $ResultPath
|
||||
$resultDir = Split-Path -Parent $resultFullPath
|
||||
if (-not (Test-Path $resultDir)) {
|
||||
New-Item -ItemType Directory -Path $resultDir | Out-Null
|
||||
}
|
||||
$result | ConvertTo-Json -Depth 100 | Set-Content -Path $resultFullPath -Encoding UTF8
|
||||
|
||||
if (-not $result.passed) {
|
||||
throw "C18Z35 service-channel rebuild alert silence smoke failed. Result: $resultFullPath"
|
||||
}
|
||||
|
||||
Write-Host "C18Z35 service-channel rebuild alert silence smoke passed. Result: $resultFullPath"
|
||||
$result
|
||||
@@ -0,0 +1,87 @@
|
||||
param(
|
||||
[string]$ApiBaseUrl = "http://192.168.200.61:18121/api/v1",
|
||||
[string]$ClusterID = "cfc0743d-d960-49fb-9de8-96e063d5e4aa",
|
||||
[string]$ActorUserID = "f67d943f-5397-4b3a-a229-695fe67ad700",
|
||||
[string]$EntryNodeName = "test-1",
|
||||
[string]$ExitNodeName = "test-2",
|
||||
[string]$EntryBaseUrl = "http://192.168.200.61:19131",
|
||||
[string]$DockerSSH = "test-docker",
|
||||
[string]$ExpectedBackendImage = "rap-backend:fabric-service-channel-0.2.217",
|
||||
[string]$ExpectedNodeAgentImage = "rap-node-agent:0.2.208",
|
||||
[string]$ResultPath = "artifacts\c18z36-service-channel-rebuild-alert-resurface-smoke-result.json"
|
||||
)
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
$scriptDir = Split-Path -Parent $MyInvocation.MyCommand.Path
|
||||
$repoRoot = (Resolve-Path (Join-Path $scriptDir "..\..")).ProviderPath
|
||||
$baseResultPath = "artifacts\c18z36-base-rebuild-alert-silence-smoke-result.json"
|
||||
|
||||
& (Join-Path $scriptDir "c18z35-service-channel-rebuild-alert-silence-smoke.ps1") `
|
||||
-ApiBaseUrl $ApiBaseUrl `
|
||||
-ClusterID $ClusterID `
|
||||
-ActorUserID $ActorUserID `
|
||||
-EntryNodeName $EntryNodeName `
|
||||
-ExitNodeName $ExitNodeName `
|
||||
-EntryBaseUrl $EntryBaseUrl `
|
||||
-DockerSSH $DockerSSH `
|
||||
-ExpectedBackendImage $ExpectedBackendImage `
|
||||
-ExpectedNodeAgentImage $ExpectedNodeAgentImage `
|
||||
-ResultPath $baseResultPath | Out-Null
|
||||
|
||||
$after = (Invoke-RestMethod -Method Get -Uri "$ApiBaseUrl/clusters/$ClusterID/fabric/service-channels/rebuild-health?actor_user_id=$ActorUserID&limit=5" -TimeoutSec 30).rebuild_health
|
||||
function Get-Prop {
|
||||
param([object]$Object, [string]$Name)
|
||||
if ($null -eq $Object) { return $null }
|
||||
$prop = $Object.PSObject.Properties[$Name]
|
||||
if ($null -eq $prop) { return $null }
|
||||
return $prop.Value
|
||||
}
|
||||
$rawResurfacedAttempts = Get-Prop $after "resurfaced_attempts"
|
||||
$resurfacedAttempts = if ($null -eq $rawResurfacedAttempts) { @() } else { @($rawResurfacedAttempts) }
|
||||
$resurfaced = @($resurfacedAttempts | Where-Object {
|
||||
(Get-Prop $_ "alert_resurfaced") -eq $true -and
|
||||
(Get-Prop $_ "alert_resurfaced_previous_generation")
|
||||
})
|
||||
|
||||
$backendLine = (& ssh $DockerSSH "docker ps --format '{{.Names}} {{.Image}} {{.Status}}' | grep '^rap_test_backend '") | Out-String
|
||||
$nodeLines = (& ssh $DockerSSH "docker ps --format '{{.Names}} {{.Image}} {{.Status}}' | grep '^rap_test_node_test_'") | Out-String
|
||||
|
||||
$result = [ordered]@{
|
||||
schema_version = "c18z36.service_channel_rebuild_alert_resurface_smoke.v1"
|
||||
cluster_id = $ClusterID
|
||||
passed = [bool](
|
||||
$backendLine.Contains($ExpectedBackendImage) -and
|
||||
$nodeLines.Contains($ExpectedNodeAgentImage) -and
|
||||
[int](Get-Prop $after "resurfaced_count") -gt 0 -and
|
||||
$resurfaced.Count -gt 0
|
||||
)
|
||||
checks = [ordered]@{
|
||||
backend_expected_image_deployed = $backendLine.Contains($ExpectedBackendImage)
|
||||
node_agent_expected_image_deployed = $nodeLines.Contains($ExpectedNodeAgentImage)
|
||||
resurfaced_count_positive = ([int](Get-Prop $after "resurfaced_count") -gt 0)
|
||||
new_generation_resurfaced = ($resurfaced.Count -gt 0)
|
||||
}
|
||||
summary = [ordered]@{
|
||||
backend_container = $backendLine.Trim()
|
||||
node_containers = $nodeLines.Trim()
|
||||
resurfaced = $resurfaced
|
||||
health = $after
|
||||
base_smoke_result = (Join-Path $repoRoot $baseResultPath)
|
||||
}
|
||||
}
|
||||
|
||||
$resultFullPath = Join-Path $repoRoot $ResultPath
|
||||
$resultDir = Split-Path -Parent $resultFullPath
|
||||
if (-not (Test-Path $resultDir)) {
|
||||
New-Item -ItemType Directory -Path $resultDir | Out-Null
|
||||
}
|
||||
$result | ConvertTo-Json -Depth 100 | Set-Content -Path $resultFullPath -Encoding UTF8
|
||||
|
||||
if (-not $result.passed) {
|
||||
throw "C18Z36 service-channel rebuild alert resurface smoke failed. Result: $resultFullPath"
|
||||
}
|
||||
|
||||
Write-Host "C18Z36 service-channel rebuild alert resurface smoke passed. Result: $resultFullPath"
|
||||
$result
|
||||
@@ -0,0 +1,89 @@
|
||||
param(
|
||||
[string]$ApiBaseUrl = "http://192.168.200.61:18121/api/v1",
|
||||
[string]$ClusterID = "cfc0743d-d960-49fb-9de8-96e063d5e4aa",
|
||||
[string]$ActorUserID = "f67d943f-5397-4b3a-a229-695fe67ad700",
|
||||
[string]$EntryNodeName = "test-1",
|
||||
[string]$ExitNodeName = "test-2",
|
||||
[string]$EntryBaseUrl = "http://192.168.200.61:19131",
|
||||
[string]$DockerSSH = "test-docker",
|
||||
[string]$ExpectedBackendImage = "rap-backend:fabric-service-channel-0.2.218",
|
||||
[string]$ExpectedNodeAgentImage = "rap-node-agent:0.2.208",
|
||||
[string]$ResultPath = "artifacts\c18z37-service-channel-readiness-smoke-result.json"
|
||||
)
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
$scriptDir = Split-Path -Parent $MyInvocation.MyCommand.Path
|
||||
$repoRoot = (Resolve-Path (Join-Path $scriptDir "..\..")).ProviderPath
|
||||
$baseResultPath = "artifacts\c18z37-base-rebuild-health-smoke-result.json"
|
||||
|
||||
& (Join-Path $scriptDir "c18z34-service-channel-rebuild-health-smoke.ps1") `
|
||||
-ApiBaseUrl $ApiBaseUrl `
|
||||
-ClusterID $ClusterID `
|
||||
-ActorUserID $ActorUserID `
|
||||
-EntryNodeName $EntryNodeName `
|
||||
-ExitNodeName $ExitNodeName `
|
||||
-EntryBaseUrl $EntryBaseUrl `
|
||||
-DockerSSH $DockerSSH `
|
||||
-ExpectedBackendImage $ExpectedBackendImage `
|
||||
-ExpectedNodeAgentImage $ExpectedNodeAgentImage `
|
||||
-ResultPath $baseResultPath | Out-Null
|
||||
|
||||
$readiness = (Invoke-RestMethod -Method Get -Uri "$ApiBaseUrl/clusters/$ClusterID/fabric/service-channels/readiness?actor_user_id=$ActorUserID&limit=5" -TimeoutSec 30).fabric_service_channel_readiness
|
||||
$health = (Invoke-RestMethod -Method Get -Uri "$ApiBaseUrl/clusters/$ClusterID/fabric/service-channels/rebuild-health?actor_user_id=$ActorUserID&limit=5" -TimeoutSec 30).rebuild_health
|
||||
$backendLine = (& ssh $DockerSSH "docker ps --format '{{.Names}} {{.Image}} {{.Status}}' | grep '^rap_test_backend '") | Out-String
|
||||
$nodeLines = (& ssh $DockerSSH "docker ps --format '{{.Names}} {{.Image}} {{.Status}}' | grep '^rap_test_node_test_'") | Out-String
|
||||
|
||||
$expectedStatus = if ([int]$health.active_bad_count -gt 0 -or [int]$health.resurfaced_count -gt 0) {
|
||||
"blocked"
|
||||
} elseif ([int]$health.active_warn_count -gt 0 -or [int]$health.silenced_count -gt 0 -or [int]$health.pending_count -gt 0) {
|
||||
"degraded"
|
||||
} else {
|
||||
"clean"
|
||||
}
|
||||
|
||||
$result = [ordered]@{
|
||||
schema_version = "c18z37.service_channel_readiness_smoke.v1"
|
||||
cluster_id = $ClusterID
|
||||
passed = [bool](
|
||||
$backendLine.Contains($ExpectedBackendImage) -and
|
||||
$nodeLines.Contains($ExpectedNodeAgentImage) -and
|
||||
[string]$readiness.status -eq $expectedStatus -and
|
||||
[int]$readiness.active_bad_count -eq [int]$health.active_bad_count -and
|
||||
[int]$readiness.active_warn_count -eq [int]$health.active_warn_count -and
|
||||
[int]$readiness.resurfaced_count -eq [int]$health.resurfaced_count -and
|
||||
[string]$readiness.reason
|
||||
)
|
||||
checks = [ordered]@{
|
||||
backend_expected_image_deployed = $backendLine.Contains($ExpectedBackendImage)
|
||||
node_agent_expected_image_deployed = $nodeLines.Contains($ExpectedNodeAgentImage)
|
||||
status_matches_health = ([string]$readiness.status -eq $expectedStatus)
|
||||
active_bad_matches = ([int]$readiness.active_bad_count -eq [int]$health.active_bad_count)
|
||||
active_warn_matches = ([int]$readiness.active_warn_count -eq [int]$health.active_warn_count)
|
||||
resurfaced_matches = ([int]$readiness.resurfaced_count -eq [int]$health.resurfaced_count)
|
||||
has_reason = ([string]$readiness.reason).Length -gt 0
|
||||
}
|
||||
summary = [ordered]@{
|
||||
backend_container = $backendLine.Trim()
|
||||
node_containers = $nodeLines.Trim()
|
||||
expected_status = $expectedStatus
|
||||
readiness = $readiness
|
||||
health = $health
|
||||
base_smoke_result = (Join-Path $repoRoot $baseResultPath)
|
||||
}
|
||||
}
|
||||
|
||||
$resultFullPath = Join-Path $repoRoot $ResultPath
|
||||
$resultDir = Split-Path -Parent $resultFullPath
|
||||
if (-not (Test-Path $resultDir)) {
|
||||
New-Item -ItemType Directory -Path $resultDir | Out-Null
|
||||
}
|
||||
$result | ConvertTo-Json -Depth 100 | Set-Content -Path $resultFullPath -Encoding UTF8
|
||||
|
||||
if (-not $result.passed) {
|
||||
throw "C18Z37 service-channel readiness smoke failed. Result: $resultFullPath"
|
||||
}
|
||||
|
||||
Write-Host "C18Z37 service-channel readiness smoke passed. Result: $resultFullPath"
|
||||
$result
|
||||
@@ -0,0 +1,109 @@
|
||||
param(
|
||||
[string]$ApiBaseUrl = "http://192.168.200.61:18121/api/v1",
|
||||
[string]$ClusterID = "cfc0743d-d960-49fb-9de8-96e063d5e4aa",
|
||||
[string]$ActorUserID = "f67d943f-5397-4b3a-a229-695fe67ad700",
|
||||
[string]$DockerSSH = "test-docker",
|
||||
[string]$ExpectedBackendImage = "rap-backend:fabric-service-channel-0.2.219",
|
||||
[string]$ResultPath = "artifacts\c18z38-service-channel-rebuild-ledger-enrichment-smoke-result.json"
|
||||
)
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
$scriptDir = Split-Path -Parent $MyInvocation.MyCommand.Path
|
||||
$repoRoot = (Resolve-Path (Join-Path $scriptDir "..\..")).ProviderPath
|
||||
|
||||
function Get-Prop {
|
||||
param([object]$Object, [string]$Name)
|
||||
if ($null -eq $Object) { return $null }
|
||||
$prop = $Object.PSObject.Properties[$Name]
|
||||
if ($null -eq $prop) { return $null }
|
||||
return $prop.Value
|
||||
}
|
||||
|
||||
function Invoke-TimedApi {
|
||||
param([string]$Path, [int]$TimeoutSec = 30)
|
||||
$sw = [System.Diagnostics.Stopwatch]::StartNew()
|
||||
$body = Invoke-RestMethod -Method Get -Uri "$ApiBaseUrl$Path" -TimeoutSec $TimeoutSec
|
||||
$sw.Stop()
|
||||
[pscustomobject]@{
|
||||
body = $body
|
||||
elapsed_ms = [int]$sw.ElapsedMilliseconds
|
||||
}
|
||||
}
|
||||
|
||||
$summaryResponse = Invoke-TimedApi -Path "/clusters/$ClusterID/fabric/service-channels/rebuild-attempts?actor_user_id=$ActorUserID&limit=5&enrichment=summary" -TimeoutSec 15
|
||||
$summaryAttempts = @($summaryResponse.body.rebuild_attempts)
|
||||
if ($summaryAttempts.Count -lt 1) {
|
||||
throw "C18Z38 smoke needs at least one rebuild attempt in the ledger."
|
||||
}
|
||||
|
||||
$sampleSummary = $summaryAttempts | Select-Object -First 1
|
||||
$routeID = [string]$sampleSummary.route_id
|
||||
$reporterNodeID = [string]$sampleSummary.reporter_node_id
|
||||
|
||||
$deepResponse = Invoke-TimedApi -Path "/clusters/$ClusterID/fabric/service-channels/rebuild-attempts?actor_user_id=$ActorUserID&reporter_node_id=$reporterNodeID&route_id=$routeID&limit=5&enrichment=deep" -TimeoutSec 30
|
||||
$deepAttempts = @($deepResponse.body.rebuild_attempts)
|
||||
$sampleDeep = $deepAttempts | Select-Object -First 1
|
||||
|
||||
$readinessResponse = Invoke-TimedApi -Path "/clusters/$ClusterID/fabric/service-channels/readiness?actor_user_id=$ActorUserID&limit=5" -TimeoutSec 30
|
||||
$readiness = $readinessResponse.body.fabric_service_channel_readiness
|
||||
|
||||
$summaryHasTimeline = $null -ne (Get-Prop $sampleSummary "timeline")
|
||||
$summaryHasGuard = [string](Get-Prop $sampleSummary "guard_status") -ne ""
|
||||
$deepHasTimeline = $null -ne (Get-Prop $sampleDeep "timeline")
|
||||
$deepHasGuard = [string](Get-Prop $sampleDeep "guard_status") -ne ""
|
||||
|
||||
$backendLine = (& ssh $DockerSSH "docker ps --format '{{.Names}} {{.Image}} {{.Status}}' | grep '^rap_test_backend '") | Out-String
|
||||
|
||||
$result = [ordered]@{
|
||||
schema_version = "c18z38.service_channel_rebuild_ledger_enrichment_smoke.v1"
|
||||
cluster_id = $ClusterID
|
||||
route_id = $routeID
|
||||
reporter_node_id = $reporterNodeID
|
||||
passed = [bool](
|
||||
$backendLine.Contains($ExpectedBackendImage) -and
|
||||
$summaryAttempts.Count -ge 1 -and
|
||||
$deepAttempts.Count -ge 1 -and
|
||||
-not $summaryHasTimeline -and
|
||||
-not $summaryHasGuard -and
|
||||
($deepHasTimeline -or $deepHasGuard) -and
|
||||
[string](Get-Prop $readiness "status") -ne ""
|
||||
)
|
||||
checks = [ordered]@{
|
||||
backend_expected_image_deployed = $backendLine.Contains($ExpectedBackendImage)
|
||||
summary_returned_attempts = ($summaryAttempts.Count -ge 1)
|
||||
summary_omits_timeline = (-not $summaryHasTimeline)
|
||||
summary_omits_guard = (-not $summaryHasGuard)
|
||||
deep_returned_attempts = ($deepAttempts.Count -ge 1)
|
||||
deep_has_timeline_or_guard = ($deepHasTimeline -or $deepHasGuard)
|
||||
readiness_returned_status = ([string](Get-Prop $readiness "status") -ne "")
|
||||
}
|
||||
timings_ms = [ordered]@{
|
||||
summary = $summaryResponse.elapsed_ms
|
||||
deep = $deepResponse.elapsed_ms
|
||||
readiness = $readinessResponse.elapsed_ms
|
||||
}
|
||||
summary = [ordered]@{
|
||||
backend_container = $backendLine.Trim()
|
||||
summary_attempt_count = $summaryAttempts.Count
|
||||
deep_attempt_count = $deepAttempts.Count
|
||||
readiness_status = [string](Get-Prop $readiness "status")
|
||||
summary_sample = $sampleSummary
|
||||
deep_sample = $sampleDeep
|
||||
}
|
||||
}
|
||||
|
||||
$resultFullPath = Join-Path $repoRoot $ResultPath
|
||||
$resultDir = Split-Path -Parent $resultFullPath
|
||||
if (-not (Test-Path $resultDir)) {
|
||||
New-Item -ItemType Directory -Path $resultDir | Out-Null
|
||||
}
|
||||
$result | ConvertTo-Json -Depth 100 | Set-Content -Path $resultFullPath -Encoding UTF8
|
||||
|
||||
if (-not $result.passed) {
|
||||
throw "C18Z38 service-channel rebuild ledger enrichment smoke failed. Result: $resultFullPath"
|
||||
}
|
||||
|
||||
Write-Host "C18Z38 service-channel rebuild ledger enrichment smoke passed. Result: $resultFullPath"
|
||||
$result
|
||||
@@ -0,0 +1,105 @@
|
||||
param(
|
||||
[string]$ApiBaseUrl = "http://192.168.200.61:18121/api/v1",
|
||||
[string]$ClusterID = "cfc0743d-d960-49fb-9de8-96e063d5e4aa",
|
||||
[string]$ActorUserID = "f67d943f-5397-4b3a-a229-695fe67ad700",
|
||||
[string]$DockerSSH = "test-docker",
|
||||
[string]$ExpectedBackendImage = "rap-backend:fabric-service-channel-0.2.220",
|
||||
[string]$ResultPath = "artifacts\c18z39-service-channel-rebuild-ledger-drilldown-smoke-result.json"
|
||||
)
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
$scriptDir = Split-Path -Parent $MyInvocation.MyCommand.Path
|
||||
$repoRoot = (Resolve-Path (Join-Path $scriptDir "..\..")).ProviderPath
|
||||
|
||||
function Invoke-Api {
|
||||
param([string]$Path, [int]$TimeoutSec = 30)
|
||||
Invoke-RestMethod -Method Get -Uri "$ApiBaseUrl$Path" -TimeoutSec $TimeoutSec
|
||||
}
|
||||
|
||||
$firstPage = @((
|
||||
Invoke-Api -Path "/clusters/$ClusterID/fabric/service-channels/rebuild-attempts?actor_user_id=$ActorUserID&limit=2&offset=0&enrichment=summary" -TimeoutSec 15
|
||||
).rebuild_attempts)
|
||||
if ($firstPage.Count -lt 1) {
|
||||
throw "C18Z39 smoke needs at least one rebuild attempt in the ledger."
|
||||
}
|
||||
|
||||
$sample = $firstPage | Select-Object -First 1
|
||||
$routeID = [string]$sample.route_id
|
||||
$reporterNodeID = [string]$sample.reporter_node_id
|
||||
$serviceClass = [string]$sample.service_class
|
||||
$generation = [string]$sample.generation
|
||||
|
||||
$filteredDeep = @((
|
||||
Invoke-Api -Path "/clusters/$ClusterID/fabric/service-channels/rebuild-attempts?actor_user_id=$ActorUserID&reporter_node_id=$reporterNodeID&route_id=$routeID&service_class=$serviceClass&generation=$generation&limit=5&offset=0&enrichment=deep" -TimeoutSec 30
|
||||
).rebuild_attempts)
|
||||
|
||||
$secondPage = @((
|
||||
Invoke-Api -Path "/clusters/$ClusterID/fabric/service-channels/rebuild-attempts?actor_user_id=$ActorUserID&limit=1&offset=1&enrichment=summary" -TimeoutSec 15
|
||||
).rebuild_attempts)
|
||||
|
||||
$filteredAllMatch = $filteredDeep.Count -ge 1 -and @($filteredDeep | Where-Object {
|
||||
[string]$_.reporter_node_id -eq $reporterNodeID -and
|
||||
[string]$_.route_id -eq $routeID -and
|
||||
[string]$_.service_class -eq $serviceClass -and
|
||||
[string]$_.generation -eq $generation
|
||||
}).Count -eq $filteredDeep.Count
|
||||
|
||||
$offsetMoved = $true
|
||||
if ($firstPage.Count -gt 1 -and $secondPage.Count -gt 0) {
|
||||
$offsetMoved = ([string]$firstPage[1].id -eq [string]$secondPage[0].id) -and ([string]$firstPage[0].id -ne [string]$secondPage[0].id)
|
||||
}
|
||||
|
||||
$deepHasCorrelation = $false
|
||||
if ($filteredDeep.Count -gt 0) {
|
||||
$deepHasCorrelation = $null -ne $filteredDeep[0].timeline -or [string]$filteredDeep[0].guard_status -ne ""
|
||||
}
|
||||
|
||||
$backendLine = (& ssh $DockerSSH "docker ps --format '{{.Names}} {{.Image}} {{.Status}}' | grep '^rap_test_backend '") | Out-String
|
||||
|
||||
$result = [ordered]@{
|
||||
schema_version = "c18z39.service_channel_rebuild_ledger_drilldown_smoke.v1"
|
||||
cluster_id = $ClusterID
|
||||
route_id = $routeID
|
||||
reporter_node_id = $reporterNodeID
|
||||
service_class = $serviceClass
|
||||
generation = $generation
|
||||
passed = [bool](
|
||||
$backendLine.Contains($ExpectedBackendImage) -and
|
||||
$firstPage.Count -ge 1 -and
|
||||
$filteredAllMatch -and
|
||||
$offsetMoved -and
|
||||
$deepHasCorrelation
|
||||
)
|
||||
checks = [ordered]@{
|
||||
backend_expected_image_deployed = $backendLine.Contains($ExpectedBackendImage)
|
||||
first_page_returned = ($firstPage.Count -ge 1)
|
||||
filtered_deep_all_match = $filteredAllMatch
|
||||
offset_page_matches_expected_second_row = $offsetMoved
|
||||
filtered_deep_has_timeline_or_guard = $deepHasCorrelation
|
||||
}
|
||||
summary = [ordered]@{
|
||||
backend_container = $backendLine.Trim()
|
||||
first_page_count = $firstPage.Count
|
||||
second_page_count = $secondPage.Count
|
||||
filtered_deep_count = $filteredDeep.Count
|
||||
first_page_ids = @($firstPage | ForEach-Object { [string]$_.id })
|
||||
second_page_ids = @($secondPage | ForEach-Object { [string]$_.id })
|
||||
filtered_sample = ($filteredDeep | Select-Object -First 1)
|
||||
}
|
||||
}
|
||||
|
||||
$resultFullPath = Join-Path $repoRoot $ResultPath
|
||||
$resultDir = Split-Path -Parent $resultFullPath
|
||||
if (-not (Test-Path $resultDir)) {
|
||||
New-Item -ItemType Directory -Path $resultDir | Out-Null
|
||||
}
|
||||
$result | ConvertTo-Json -Depth 100 | Set-Content -Path $resultFullPath -Encoding UTF8
|
||||
|
||||
if (-not $result.passed) {
|
||||
throw "C18Z39 service-channel rebuild ledger drilldown smoke failed. Result: $resultFullPath"
|
||||
}
|
||||
|
||||
Write-Host "C18Z39 service-channel rebuild ledger drilldown smoke passed. Result: $resultFullPath"
|
||||
$result
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,91 @@
|
||||
param(
|
||||
[string]$ApiBaseUrl = "http://192.168.200.61:18121/api/v1",
|
||||
[string]$ClusterID = "cfc0743d-d960-49fb-9de8-96e063d5e4aa",
|
||||
[string]$ActorUserID = "f67d943f-5397-4b3a-a229-695fe67ad700",
|
||||
[string]$DockerSSH = "test-docker",
|
||||
[string]$ExpectedBackendImage = "rap-backend:fabric-service-channel-0.2.222",
|
||||
[string]$ResultPath = "artifacts\c18z40-service-channel-rebuild-incidents-smoke-result.json"
|
||||
)
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
$scriptDir = Split-Path -Parent $MyInvocation.MyCommand.Path
|
||||
$repoRoot = (Resolve-Path (Join-Path $scriptDir "..\..")).ProviderPath
|
||||
|
||||
function Invoke-Api {
|
||||
param([string]$Path, [int]$TimeoutSec = 30)
|
||||
Invoke-RestMethod -Method Get -Uri "$ApiBaseUrl$Path" -TimeoutSec $TimeoutSec
|
||||
}
|
||||
|
||||
$incidents = @((Invoke-Api -Path "/clusters/$ClusterID/fabric/service-channels/rebuild-incidents?actor_user_id=$ActorUserID&limit=5" -TimeoutSec 30).rebuild_incidents)
|
||||
if ($incidents.Count -lt 1) {
|
||||
throw "C18Z40 smoke needs at least one rebuild incident."
|
||||
}
|
||||
|
||||
$sample = $incidents | Select-Object -First 1
|
||||
$reporterNodeID = [string]$sample.reporter_node_id
|
||||
$routeID = [string]$sample.route_id
|
||||
$serviceClass = [string]$sample.service_class
|
||||
$generation = [string]$sample.generation
|
||||
|
||||
$ledger = @((Invoke-Api -Path "/clusters/$ClusterID/fabric/service-channels/rebuild-attempts?actor_user_id=$ActorUserID&reporter_node_id=$reporterNodeID&route_id=$routeID&service_class=$serviceClass&generation=$generation&limit=20&offset=0&enrichment=deep" -TimeoutSec 30).rebuild_attempts)
|
||||
|
||||
$ledgerMatchesIncident = $ledger.Count -ge 1 -and @($ledger | Where-Object {
|
||||
[string]$_.reporter_node_id -eq $reporterNodeID -and
|
||||
[string]$_.route_id -eq $routeID -and
|
||||
[string]$_.service_class -eq $serviceClass -and
|
||||
[string]$_.generation -eq $generation
|
||||
}).Count -eq $ledger.Count
|
||||
|
||||
$hasRequiredIncidentFields = [string]$sample.fingerprint -ne "" -and
|
||||
[string]$sample.guard_status -ne "" -and
|
||||
[string]$sample.guard_severity -ne "" -and
|
||||
[int]$sample.attempt_count -ge 1 -and
|
||||
[string]$sample.last_seen_at -ne "" -and
|
||||
[string]$sample.recommended_operator_action -ne ""
|
||||
|
||||
$backendLine = (& ssh $DockerSSH "docker ps --format '{{.Names}} {{.Image}} {{.Status}}' | grep '^rap_test_backend '") | Out-String
|
||||
|
||||
$result = [ordered]@{
|
||||
schema_version = "c18z40.service_channel_rebuild_incidents_smoke.v1"
|
||||
cluster_id = $ClusterID
|
||||
incident_fingerprint = [string]$sample.fingerprint
|
||||
route_id = $routeID
|
||||
reporter_node_id = $reporterNodeID
|
||||
service_class = $serviceClass
|
||||
generation = $generation
|
||||
passed = [bool](
|
||||
$backendLine.Contains($ExpectedBackendImage) -and
|
||||
$incidents.Count -ge 1 -and
|
||||
$hasRequiredIncidentFields -and
|
||||
$ledgerMatchesIncident
|
||||
)
|
||||
checks = [ordered]@{
|
||||
backend_expected_image_deployed = $backendLine.Contains($ExpectedBackendImage)
|
||||
incidents_returned = ($incidents.Count -ge 1)
|
||||
required_incident_fields_present = $hasRequiredIncidentFields
|
||||
deep_ledger_filter_matches_incident = $ledgerMatchesIncident
|
||||
}
|
||||
summary = [ordered]@{
|
||||
backend_container = $backendLine.Trim()
|
||||
incident_count = $incidents.Count
|
||||
filtered_ledger_count = $ledger.Count
|
||||
sample_incident = $sample
|
||||
filtered_ledger_sample = ($ledger | Select-Object -First 1)
|
||||
}
|
||||
}
|
||||
|
||||
$resultFullPath = Join-Path $repoRoot $ResultPath
|
||||
$resultDir = Split-Path -Parent $resultFullPath
|
||||
if (-not (Test-Path $resultDir)) {
|
||||
New-Item -ItemType Directory -Path $resultDir | Out-Null
|
||||
}
|
||||
$result | ConvertTo-Json -Depth 100 | Set-Content -Path $resultFullPath -Encoding UTF8
|
||||
|
||||
if (-not $result.passed) {
|
||||
throw "C18Z40 service-channel rebuild incidents smoke failed. Result: $resultFullPath"
|
||||
}
|
||||
|
||||
Write-Host "C18Z40 service-channel rebuild incidents smoke passed. Result: $resultFullPath"
|
||||
$result
|
||||
@@ -0,0 +1,129 @@
|
||||
param(
|
||||
[string]$ApiBaseUrl = "http://192.168.200.61:18121/api/v1",
|
||||
[string]$ClusterID = "cfc0743d-d960-49fb-9de8-96e063d5e4aa",
|
||||
[string]$ActorUserID = "f67d943f-5397-4b3a-a229-695fe67ad700",
|
||||
[string]$DockerSSH = "test-docker",
|
||||
[string]$ExpectedBackendImage = "rap-backend:fabric-service-channel-0.2.223",
|
||||
[string]$ResultPath = "artifacts\c18z41-service-channel-rebuild-incident-actions-smoke-result.json"
|
||||
)
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
$scriptDir = Split-Path -Parent $MyInvocation.MyCommand.Path
|
||||
$repoRoot = (Resolve-Path (Join-Path $scriptDir "..\..")).ProviderPath
|
||||
|
||||
function Invoke-ApiGet {
|
||||
param([string]$Path, [int]$TimeoutSec = 30)
|
||||
Invoke-RestMethod -Method Get -Uri "$ApiBaseUrl$Path" -TimeoutSec $TimeoutSec
|
||||
}
|
||||
|
||||
function Invoke-ApiPost {
|
||||
param([string]$Path, [object]$Body, [int]$TimeoutSec = 30)
|
||||
Invoke-RestMethod -Method Post -Uri "$ApiBaseUrl$Path" -ContentType "application/json" -Body ($Body | ConvertTo-Json -Depth 20) -TimeoutSec $TimeoutSec
|
||||
}
|
||||
|
||||
function Get-Prop {
|
||||
param([object]$Object, [string]$Name)
|
||||
if ($null -eq $Object) { return $null }
|
||||
$prop = $Object.PSObject.Properties[$Name]
|
||||
if ($null -eq $prop) { return $null }
|
||||
return $prop.Value
|
||||
}
|
||||
|
||||
$before = @((Invoke-ApiGet -Path "/clusters/$ClusterID/fabric/service-channels/rebuild-incidents?actor_user_id=$ActorUserID&limit=5" -TimeoutSec 30).rebuild_incidents)
|
||||
if ($before.Count -lt 1) {
|
||||
throw "C18Z41 smoke needs at least one rebuild incident."
|
||||
}
|
||||
|
||||
$target = $before | Where-Object { (Get-Prop $_ "alert_silenced") -ne $true } | Select-Object -First 1
|
||||
if ($null -eq $target) {
|
||||
$target = $before | Select-Object -First 1
|
||||
}
|
||||
|
||||
$investigationPayload = [ordered]@{
|
||||
actor_user_id = $ActorUserID
|
||||
reporter_node_id = [string]$target.reporter_node_id
|
||||
route_id = [string]$target.route_id
|
||||
service_class = [string]$target.service_class
|
||||
generation = [string]$target.generation
|
||||
guard_status = [string]$target.guard_status
|
||||
incident_id = [string]$target.fingerprint
|
||||
reason = "c18z41 smoke opened deep rebuild ledger"
|
||||
}
|
||||
Invoke-ApiPost -Path "/clusters/$ClusterID/fabric/service-channels/rebuild-incidents/investigations" -Body $investigationPayload -TimeoutSec 15 | Out-Null
|
||||
|
||||
$audit = @((Invoke-ApiGet -Path "/clusters/$ClusterID/audit?actor_user_id=$ActorUserID&limit=20" -TimeoutSec 15).audit_events)
|
||||
$auditMatched = @($audit | Where-Object {
|
||||
[string]$_.event_type -eq "fabric.service_channel_rebuild_incident.investigation_opened" -and
|
||||
[string]$_.target_id -eq [string]$target.route_id
|
||||
}).Count -ge 1
|
||||
|
||||
$silencePayload = [ordered]@{
|
||||
actor_user_id = $ActorUserID
|
||||
reporter_node_id = [string]$target.reporter_node_id
|
||||
route_id = [string]$target.route_id
|
||||
guard_status = [string]$target.guard_status
|
||||
generation = [string]$target.generation
|
||||
reason = "c18z41 smoke incident acknowledgement"
|
||||
ttl_seconds = 21600
|
||||
}
|
||||
Invoke-ApiPost -Path "/clusters/$ClusterID/fabric/service-channels/rebuild-health/silences" -Body $silencePayload -TimeoutSec 15 | Out-Null
|
||||
|
||||
$after = @((Invoke-ApiGet -Path "/clusters/$ClusterID/fabric/service-channels/rebuild-incidents?actor_user_id=$ActorUserID&limit=5" -TimeoutSec 30).rebuild_incidents)
|
||||
$afterTarget = $after | Where-Object {
|
||||
[string]$_.reporter_node_id -eq [string]$target.reporter_node_id -and
|
||||
[string]$_.route_id -eq [string]$target.route_id -and
|
||||
[string]$_.service_class -eq [string]$target.service_class -and
|
||||
[string]$_.generation -eq [string]$target.generation -and
|
||||
[string]$_.guard_status -eq [string]$target.guard_status
|
||||
} | Select-Object -First 1
|
||||
|
||||
$health = (Invoke-ApiGet -Path "/clusters/$ClusterID/fabric/service-channels/rebuild-health?actor_user_id=$ActorUserID&limit=5" -TimeoutSec 30).rebuild_health
|
||||
|
||||
$backendLine = (& ssh $DockerSSH "docker ps --format '{{.Names}} {{.Image}} {{.Status}}' | grep '^rap_test_backend '") | Out-String
|
||||
|
||||
$incidentSilenced = $null -ne $afterTarget -and (Get-Prop $afterTarget "alert_silenced") -eq $true
|
||||
|
||||
$result = [ordered]@{
|
||||
schema_version = "c18z41.service_channel_rebuild_incident_actions_smoke.v1"
|
||||
cluster_id = $ClusterID
|
||||
incident_fingerprint = [string]$target.fingerprint
|
||||
route_id = [string]$target.route_id
|
||||
reporter_node_id = [string]$target.reporter_node_id
|
||||
generation = [string]$target.generation
|
||||
passed = [bool](
|
||||
$backendLine.Contains($ExpectedBackendImage) -and
|
||||
$auditMatched -and
|
||||
$incidentSilenced -and
|
||||
[int]$health.silenced_count -ge 1
|
||||
)
|
||||
checks = [ordered]@{
|
||||
backend_expected_image_deployed = $backendLine.Contains($ExpectedBackendImage)
|
||||
investigation_audit_recorded = $auditMatched
|
||||
incident_marked_silenced = $incidentSilenced
|
||||
health_reports_silenced = ([int]$health.silenced_count -ge 1)
|
||||
}
|
||||
summary = [ordered]@{
|
||||
backend_container = $backendLine.Trim()
|
||||
before_incident_count = $before.Count
|
||||
after_incident_count = $after.Count
|
||||
target_before = $target
|
||||
target_after = $afterTarget
|
||||
health_silenced_count = [int]$health.silenced_count
|
||||
}
|
||||
}
|
||||
|
||||
$resultFullPath = Join-Path $repoRoot $ResultPath
|
||||
$resultDir = Split-Path -Parent $resultFullPath
|
||||
if (-not (Test-Path $resultDir)) {
|
||||
New-Item -ItemType Directory -Path $resultDir | Out-Null
|
||||
}
|
||||
$result | ConvertTo-Json -Depth 100 | Set-Content -Path $resultFullPath -Encoding UTF8
|
||||
|
||||
if (-not $result.passed) {
|
||||
throw "C18Z41 service-channel rebuild incident actions smoke failed. Result: $resultFullPath"
|
||||
}
|
||||
|
||||
Write-Host "C18Z41 service-channel rebuild incident actions smoke passed. Result: $resultFullPath"
|
||||
$result
|
||||
@@ -0,0 +1,117 @@
|
||||
param(
|
||||
[string]$ApiBaseUrl = "http://192.168.200.61:18121/api/v1",
|
||||
[string]$ClusterID = "cfc0743d-d960-49fb-9de8-96e063d5e4aa",
|
||||
[string]$ActorUserID = "f67d943f-5397-4b3a-a229-695fe67ad700",
|
||||
[string]$DockerSSH = "test-docker",
|
||||
[string]$ExpectedBackendImage = "rap-backend:fabric-service-channel-0.2.224",
|
||||
[string]$ResultPath = "artifacts\c18z42-service-channel-rebuild-correlation-snapshot-smoke-result.json"
|
||||
)
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
$scriptDir = Split-Path -Parent $MyInvocation.MyCommand.Path
|
||||
$repoRoot = (Resolve-Path (Join-Path $scriptDir "..\..")).ProviderPath
|
||||
|
||||
function Get-Prop {
|
||||
param([object]$Object, [string]$Name)
|
||||
if ($null -eq $Object) { return $null }
|
||||
$prop = $Object.PSObject.Properties[$Name]
|
||||
if ($null -eq $prop) { return $null }
|
||||
return $prop.Value
|
||||
}
|
||||
|
||||
function Invoke-TimedApi {
|
||||
param([string]$Path, [int]$TimeoutSec = 30)
|
||||
$sw = [System.Diagnostics.Stopwatch]::StartNew()
|
||||
$body = Invoke-RestMethod -Method Get -Uri "$ApiBaseUrl$Path" -TimeoutSec $TimeoutSec
|
||||
$sw.Stop()
|
||||
[pscustomobject]@{
|
||||
body = $body
|
||||
elapsed_ms = [int]$sw.ElapsedMilliseconds
|
||||
}
|
||||
}
|
||||
|
||||
$summaryResponse = Invoke-TimedApi -Path "/clusters/$ClusterID/fabric/service-channels/rebuild-attempts?actor_user_id=$ActorUserID&limit=1&enrichment=summary" -TimeoutSec 15
|
||||
$summaryAttempts = @($summaryResponse.body.rebuild_attempts)
|
||||
if ($summaryAttempts.Count -lt 1) {
|
||||
throw "C18Z42 smoke needs at least one rebuild attempt."
|
||||
}
|
||||
|
||||
$sampleSummary = $summaryAttempts | Select-Object -First 1
|
||||
$reporterNodeID = [string]$sampleSummary.reporter_node_id
|
||||
$routeID = [string]$sampleSummary.route_id
|
||||
$serviceClass = [string]$sampleSummary.service_class
|
||||
$generation = [string]$sampleSummary.generation
|
||||
|
||||
$deepPath = "/clusters/$ClusterID/fabric/service-channels/rebuild-attempts?actor_user_id=$ActorUserID&reporter_node_id=$reporterNodeID&route_id=$routeID&service_class=$serviceClass&generation=$generation&limit=1&enrichment=deep"
|
||||
$deepFirst = Invoke-TimedApi -Path $deepPath -TimeoutSec 30
|
||||
$deepFirstAttempts = @($deepFirst.body.rebuild_attempts)
|
||||
$deepFirstSample = $deepFirstAttempts | Select-Object -First 1
|
||||
|
||||
$deepSecond = Invoke-TimedApi -Path $deepPath -TimeoutSec 30
|
||||
$deepSecondAttempts = @($deepSecond.body.rebuild_attempts)
|
||||
$deepSecondSample = $deepSecondAttempts | Select-Object -First 1
|
||||
|
||||
$incidentsResponse = Invoke-TimedApi -Path "/clusters/$ClusterID/fabric/service-channels/rebuild-incidents?actor_user_id=$ActorUserID&limit=5" -TimeoutSec 30
|
||||
$incidents = @($incidentsResponse.body.rebuild_incidents)
|
||||
|
||||
$summaryHasGuard = [string](Get-Prop $sampleSummary "guard_status") -ne ""
|
||||
$summaryHasTimeline = $null -ne (Get-Prop $sampleSummary "timeline")
|
||||
$deepHasSnapshot = [string](Get-Prop $deepSecondSample "correlation_snapshot_at") -ne ""
|
||||
$deepHasGuard = [string](Get-Prop $deepSecondSample "guard_status") -ne ""
|
||||
$deepHasTimeline = $null -ne (Get-Prop $deepSecondSample "timeline")
|
||||
|
||||
$backendLine = (& ssh $DockerSSH "docker ps --format '{{.Names}} {{.Image}} {{.Status}}' | grep '^rap_test_backend '") | Out-String
|
||||
|
||||
$result = [ordered]@{
|
||||
schema_version = "c18z42.service_channel_rebuild_correlation_snapshot_smoke.v1"
|
||||
cluster_id = $ClusterID
|
||||
route_id = $routeID
|
||||
reporter_node_id = $reporterNodeID
|
||||
generation = $generation
|
||||
passed = [bool](
|
||||
$backendLine.Contains($ExpectedBackendImage) -and
|
||||
-not $summaryHasGuard -and
|
||||
-not $summaryHasTimeline -and
|
||||
$deepHasSnapshot -and
|
||||
$deepHasGuard -and
|
||||
$deepHasTimeline -and
|
||||
$incidents.Count -ge 1
|
||||
)
|
||||
checks = [ordered]@{
|
||||
backend_expected_image_deployed = $backendLine.Contains($ExpectedBackendImage)
|
||||
summary_omits_guard = (-not $summaryHasGuard)
|
||||
summary_omits_timeline = (-not $summaryHasTimeline)
|
||||
deep_returns_snapshot_timestamp = $deepHasSnapshot
|
||||
deep_returns_guard = $deepHasGuard
|
||||
deep_returns_timeline = $deepHasTimeline
|
||||
incidents_returned = ($incidents.Count -ge 1)
|
||||
}
|
||||
timings_ms = [ordered]@{
|
||||
summary = $summaryResponse.elapsed_ms
|
||||
deep_first = $deepFirst.elapsed_ms
|
||||
deep_second = $deepSecond.elapsed_ms
|
||||
incidents = $incidentsResponse.elapsed_ms
|
||||
}
|
||||
summary = [ordered]@{
|
||||
backend_container = $backendLine.Trim()
|
||||
deep_first_sample = $deepFirstSample
|
||||
deep_second_sample = $deepSecondSample
|
||||
incident_count = $incidents.Count
|
||||
}
|
||||
}
|
||||
|
||||
$resultFullPath = Join-Path $repoRoot $ResultPath
|
||||
$resultDir = Split-Path -Parent $resultFullPath
|
||||
if (-not (Test-Path $resultDir)) {
|
||||
New-Item -ItemType Directory -Path $resultDir | Out-Null
|
||||
}
|
||||
$result | ConvertTo-Json -Depth 100 | Set-Content -Path $resultFullPath -Encoding UTF8
|
||||
|
||||
if (-not $result.passed) {
|
||||
throw "C18Z42 service-channel rebuild correlation snapshot smoke failed. Result: $resultFullPath"
|
||||
}
|
||||
|
||||
Write-Host "C18Z42 service-channel rebuild correlation snapshot smoke passed. Result: $resultFullPath"
|
||||
$result
|
||||
@@ -0,0 +1,90 @@
|
||||
param(
|
||||
[string]$ApiBaseUrl = "http://192.168.200.61:18121/api/v1",
|
||||
[string]$ClusterID = "cfc0743d-d960-49fb-9de8-96e063d5e4aa",
|
||||
[string]$ActorUserID = "f67d943f-5397-4b3a-a229-695fe67ad700",
|
||||
[string]$DockerSSH = "test-docker",
|
||||
[string]$ExpectedBackendImage = "rap-backend:fabric-service-channel-0.2.225",
|
||||
[string]$ResultPath = "artifacts\c18z43-service-channel-schema-preflight-smoke-result.json"
|
||||
)
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
$scriptDir = Split-Path -Parent $MyInvocation.MyCommand.Path
|
||||
$repoRoot = (Resolve-Path (Join-Path $scriptDir "..\..")).ProviderPath
|
||||
|
||||
function Invoke-Api {
|
||||
param([string]$Path, [int]$TimeoutSec = 30)
|
||||
Invoke-RestMethod -Method Get -Uri "$ApiBaseUrl$Path" -TimeoutSec $TimeoutSec
|
||||
}
|
||||
|
||||
function Get-Prop {
|
||||
param([object]$Object, [string]$Name)
|
||||
if ($null -eq $Object) { return $null }
|
||||
$prop = $Object.PSObject.Properties[$Name]
|
||||
if ($null -eq $prop) { return $null }
|
||||
return $prop.Value
|
||||
}
|
||||
|
||||
$schemaStatus = (Invoke-Api -Path "/clusters/$ClusterID/fabric/service-channels/schema-status?actor_user_id=$ActorUserID" -TimeoutSec 15).fabric_service_channel_schema_status
|
||||
$readiness = (Invoke-Api -Path "/clusters/$ClusterID/fabric/service-channels/readiness?actor_user_id=$ActorUserID&limit=5" -TimeoutSec 30).fabric_service_channel_readiness
|
||||
$health = (Invoke-Api -Path "/clusters/$ClusterID/fabric/service-channels/rebuild-health?actor_user_id=$ActorUserID&limit=5" -TimeoutSec 30).rebuild_health
|
||||
$backendLine = (& ssh $DockerSSH "docker ps --format '{{.Names}} {{.Image}} {{.Status}}' | grep '^rap_test_backend '") | Out-String
|
||||
|
||||
$missingChecksValue = Get-Prop $schemaStatus "missing_checks"
|
||||
$requiredChecksValue = Get-Prop $schemaStatus "required_checks"
|
||||
$missingChecks = @()
|
||||
if ($null -ne $missingChecksValue) {
|
||||
$missingChecks = @($missingChecksValue)
|
||||
}
|
||||
$requiredChecks = @()
|
||||
if ($null -ne $requiredChecksValue) {
|
||||
$requiredChecks = @($requiredChecksValue)
|
||||
}
|
||||
|
||||
$result = [ordered]@{
|
||||
schema_version = "c18z43.service_channel_schema_preflight_smoke.v1"
|
||||
cluster_id = $ClusterID
|
||||
passed = [bool](
|
||||
$backendLine.Contains($ExpectedBackendImage) -and
|
||||
[string]$schemaStatus.status -eq "ready" -and
|
||||
[string]$schemaStatus.reason -eq "schema_ready" -and
|
||||
[int]$schemaStatus.missing_check_count -eq 0 -and
|
||||
$missingChecks.Count -eq 0 -and
|
||||
$requiredChecks.Count -ge 20 -and
|
||||
[string]$readiness.status -ne "" -and
|
||||
[int]$health.total_attempts -ge 0
|
||||
)
|
||||
checks = [ordered]@{
|
||||
backend_expected_image_deployed = $backendLine.Contains($ExpectedBackendImage)
|
||||
schema_ready = ([string]$schemaStatus.status -eq "ready")
|
||||
reason_ready = ([string]$schemaStatus.reason -eq "schema_ready")
|
||||
no_missing_checks = ([int]$schemaStatus.missing_check_count -eq 0 -and $missingChecks.Count -eq 0)
|
||||
required_checks_present = ($requiredChecks.Count -ge 20)
|
||||
readiness_available = ([string]$readiness.status -ne "")
|
||||
rebuild_health_available = ([int]$health.total_attempts -ge 0)
|
||||
}
|
||||
summary = [ordered]@{
|
||||
backend_container = $backendLine.Trim()
|
||||
required_migration = [string]$schemaStatus.required_migration
|
||||
required_check_count = [int]$schemaStatus.required_check_count
|
||||
passed_check_count = [int]$schemaStatus.passed_check_count
|
||||
missing_check_count = [int]$schemaStatus.missing_check_count
|
||||
readiness_status = [string]$readiness.status
|
||||
rebuild_health_total_attempts = [int]$health.total_attempts
|
||||
}
|
||||
}
|
||||
|
||||
$resultFullPath = Join-Path $repoRoot $ResultPath
|
||||
$resultDir = Split-Path -Parent $resultFullPath
|
||||
if (-not (Test-Path $resultDir)) {
|
||||
New-Item -ItemType Directory -Path $resultDir | Out-Null
|
||||
}
|
||||
$result | ConvertTo-Json -Depth 100 | Set-Content -Path $resultFullPath -Encoding UTF8
|
||||
|
||||
if (-not $result.passed) {
|
||||
throw "C18Z43 service-channel schema preflight smoke failed. Result: $resultFullPath"
|
||||
}
|
||||
|
||||
Write-Host "C18Z43 service-channel schema preflight smoke passed. Result: $resultFullPath"
|
||||
$result
|
||||
@@ -0,0 +1,84 @@
|
||||
param(
|
||||
[string]$ApiBaseUrl = "http://192.168.200.61:18121/api/v1",
|
||||
[string]$ClusterID = "cfc0743d-d960-49fb-9de8-96e063d5e4aa",
|
||||
[string]$ActorUserID = "f67d943f-5397-4b3a-a229-695fe67ad700",
|
||||
[string]$DockerSSH = "test-docker",
|
||||
[string]$ExpectedBackendImage = "rap-backend:fabric-service-channel-0.2.226",
|
||||
[string]$ResultPath = "artifacts\c18z44-service-channel-rebuild-snapshot-warmup-smoke-result.json"
|
||||
)
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
$scriptDir = Split-Path -Parent $MyInvocation.MyCommand.Path
|
||||
$repoRoot = (Resolve-Path (Join-Path $scriptDir "..\..")).ProviderPath
|
||||
|
||||
function Invoke-ApiGet {
|
||||
param([string]$Path, [int]$TimeoutSec = 30)
|
||||
Invoke-RestMethod -Method Get -Uri "$ApiBaseUrl$Path" -TimeoutSec $TimeoutSec
|
||||
}
|
||||
|
||||
function Invoke-ApiPost {
|
||||
param([string]$Path, [object]$Body, [int]$TimeoutSec = 30)
|
||||
Invoke-RestMethod -Method Post -Uri "$ApiBaseUrl$Path" -ContentType "application/json" -Body ($Body | ConvertTo-Json -Depth 20) -TimeoutSec $TimeoutSec
|
||||
}
|
||||
|
||||
$warmupPayload = @{
|
||||
actor_user_id = $ActorUserID
|
||||
limit = 10
|
||||
stale_after_seconds = 1
|
||||
}
|
||||
$warmup = (Invoke-ApiPost -Path "/clusters/$ClusterID/fabric/service-channels/rebuild-snapshots/warmup" -Body $warmupPayload -TimeoutSec 30).rebuild_snapshot_warmup
|
||||
$schemaStatus = (Invoke-ApiGet -Path "/clusters/$ClusterID/fabric/service-channels/schema-status?actor_user_id=$ActorUserID" -TimeoutSec 15).fabric_service_channel_schema_status
|
||||
$deep = (Invoke-ApiGet -Path "/clusters/$ClusterID/fabric/service-channels/rebuild-attempts?actor_user_id=$ActorUserID&limit=3&enrichment=deep" -TimeoutSec 30).rebuild_attempts
|
||||
$readiness = (Invoke-ApiGet -Path "/clusters/$ClusterID/fabric/service-channels/readiness?actor_user_id=$ActorUserID&limit=5" -TimeoutSec 30).fabric_service_channel_readiness
|
||||
$backendLine = (& ssh $DockerSSH "docker ps --format '{{.Names}} {{.Image}} {{.Status}}' | grep '^rap_test_backend '") | Out-String
|
||||
|
||||
$deepItems = @($deep)
|
||||
$deepWithSnapshots = @($deepItems | Where-Object { [string]$_.correlation_snapshot_at -ne "" })
|
||||
|
||||
$result = [ordered]@{
|
||||
schema_version = "c18z44.service_channel_rebuild_snapshot_warmup_smoke.v1"
|
||||
cluster_id = $ClusterID
|
||||
passed = [bool](
|
||||
$backendLine.Contains($ExpectedBackendImage) -and
|
||||
[string]$schemaStatus.status -eq "ready" -and
|
||||
[int]$warmup.scanned_count -gt 0 -and
|
||||
([int]$warmup.warmed_count + [int]$warmup.already_fresh_count + [int]$warmup.deferred_stale_count) -gt 0 -and
|
||||
[int]$warmup.error_count -eq 0 -and
|
||||
$deepItems.Count -gt 0 -and
|
||||
$deepWithSnapshots.Count -eq $deepItems.Count -and
|
||||
[string]$readiness.status -ne ""
|
||||
)
|
||||
checks = [ordered]@{
|
||||
backend_expected_image_deployed = $backendLine.Contains($ExpectedBackendImage)
|
||||
schema_ready = ([string]$schemaStatus.status -eq "ready")
|
||||
warmup_scanned_attempts = ([int]$warmup.scanned_count -gt 0)
|
||||
warmup_touched_or_confirmed = (([int]$warmup.warmed_count + [int]$warmup.already_fresh_count + [int]$warmup.deferred_stale_count) -gt 0)
|
||||
warmup_no_errors = ([int]$warmup.error_count -eq 0)
|
||||
deep_returns_snapshots = ($deepItems.Count -gt 0 -and $deepWithSnapshots.Count -eq $deepItems.Count)
|
||||
readiness_available = ([string]$readiness.status -ne "")
|
||||
}
|
||||
summary = [ordered]@{
|
||||
backend_container = $backendLine.Trim()
|
||||
warmup = $warmup
|
||||
schema_status = [string]$schemaStatus.status
|
||||
deep_count = $deepItems.Count
|
||||
deep_with_snapshots = $deepWithSnapshots.Count
|
||||
readiness_status = [string]$readiness.status
|
||||
}
|
||||
}
|
||||
|
||||
$resultFullPath = Join-Path $repoRoot $ResultPath
|
||||
$resultDir = Split-Path -Parent $resultFullPath
|
||||
if (-not (Test-Path $resultDir)) {
|
||||
New-Item -ItemType Directory -Path $resultDir | Out-Null
|
||||
}
|
||||
$result | ConvertTo-Json -Depth 100 | Set-Content -Path $resultFullPath -Encoding UTF8
|
||||
|
||||
if (-not $result.passed) {
|
||||
throw "C18Z44 service-channel rebuild snapshot warmup smoke failed. Result: $resultFullPath"
|
||||
}
|
||||
|
||||
Write-Host "C18Z44 service-channel rebuild snapshot warmup smoke passed. Result: $resultFullPath"
|
||||
$result
|
||||
@@ -0,0 +1,136 @@
|
||||
param(
|
||||
[string]$ApiBaseUrl = "http://192.168.200.61:18121/api/v1",
|
||||
[string]$ClusterID = "cfc0743d-d960-49fb-9de8-96e063d5e4aa",
|
||||
[string]$ActorUserID = "f67d943f-5397-4b3a-a229-695fe67ad700",
|
||||
[string]$EntryNodeName = "test-1",
|
||||
[string]$ExitNodeName = "test-2",
|
||||
[string]$EntryBaseUrl = "http://192.168.200.61:19131",
|
||||
[string]$DockerSSH = "test-docker",
|
||||
[string]$ExpectedBackendImage = "rap-backend:fabric-service-channel-0.2.227",
|
||||
[string]$ExpectedNodeAgentImage = "rap-node-agent:0.2.208",
|
||||
[string]$ResultPath = "artifacts\c18z45-service-channel-rebuild-snapshot-auto-warmup-smoke-result.json"
|
||||
)
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
$scriptDir = Split-Path -Parent $MyInvocation.MyCommand.Path
|
||||
$repoRoot = (Resolve-Path (Join-Path $scriptDir "..\..")).ProviderPath
|
||||
$baseResultPath = "artifacts\c18z45-base-rebuild-ledger-smoke-result.json"
|
||||
|
||||
function Invoke-ApiGet {
|
||||
param([string]$Path, [int]$TimeoutSec = 30)
|
||||
Invoke-RestMethod -Method Get -Uri "$ApiBaseUrl$Path" -TimeoutSec $TimeoutSec
|
||||
}
|
||||
|
||||
function Get-Prop {
|
||||
param([object]$Object, [string]$Name)
|
||||
if ($null -eq $Object) { return $null }
|
||||
$prop = $Object.PSObject.Properties[$Name]
|
||||
if ($null -eq $prop) { return $null }
|
||||
return $prop.Value
|
||||
}
|
||||
|
||||
& (Join-Path $scriptDir "c18z31-service-channel-rebuild-ledger-smoke.ps1") `
|
||||
-ApiBaseUrl $ApiBaseUrl `
|
||||
-ClusterID $ClusterID `
|
||||
-ActorUserID $ActorUserID `
|
||||
-EntryNodeName $EntryNodeName `
|
||||
-ExitNodeName $ExitNodeName `
|
||||
-EntryBaseUrl $EntryBaseUrl `
|
||||
-DockerSSH $DockerSSH `
|
||||
-ExpectedBackendImage $ExpectedBackendImage `
|
||||
-ExpectedNodeAgentImage $ExpectedNodeAgentImage `
|
||||
-ResultPath $baseResultPath | Out-Null
|
||||
|
||||
$baseResult = Get-Content (Join-Path $repoRoot $baseResultPath) -Raw | ConvertFrom-Json
|
||||
$entryNodeID = [string]$baseResult.entry_node.id
|
||||
$primaryRouteID = [string]$baseResult.primary_route_id
|
||||
$alternateRouteID = [string]$baseResult.alternate_route_id
|
||||
$generation = [string]$baseResult.summary.sample.generation
|
||||
|
||||
$auditEvent = $null
|
||||
for ($i = 0; $i -lt 12; $i++) {
|
||||
$audit = @((Invoke-ApiGet -Path "/clusters/$ClusterID/audit?actor_user_id=$ActorUserID&limit=50" -TimeoutSec 15).audit_events)
|
||||
$auditEvent = @($audit | Where-Object {
|
||||
$warmedRouteIDs = @((Get-Prop $_.payload "warmed_route_ids") | ForEach-Object { [string]$_ })
|
||||
$warmedGenerations = @((Get-Prop $_.payload "warmed_generations") | ForEach-Object { [string]$_ })
|
||||
$_.event_type -eq "fabric.service_channel_rebuild_snapshot.auto_warmup" -and
|
||||
[string](Get-Prop $_.payload "reporter_node_id") -eq $entryNodeID -and
|
||||
[int](Get-Prop $_.payload "warmed_count") -gt 0 -and
|
||||
$warmedRouteIDs -contains $primaryRouteID -and
|
||||
$warmedGenerations -contains $generation
|
||||
} | Select-Object -First 1)
|
||||
if ($null -ne $auditEvent -and $auditEvent.Count -gt 0) {
|
||||
$auditEvent = $auditEvent | Select-Object -First 1
|
||||
break
|
||||
}
|
||||
Start-Sleep -Seconds 2
|
||||
}
|
||||
|
||||
$deepPath = "/clusters/$ClusterID/fabric/service-channels/rebuild-attempts?actor_user_id=$ActorUserID&reporter_node_id=$entryNodeID&route_id=$primaryRouteID&replacement_route_id=$alternateRouteID&generation=$generation&limit=5&enrichment=deep"
|
||||
$deepAttempts = @((Invoke-ApiGet -Path $deepPath -TimeoutSec 30).rebuild_attempts)
|
||||
$warmedAttemptIDs = @()
|
||||
if ($null -ne $auditEvent) {
|
||||
$warmedAttemptIDs = @((Get-Prop $auditEvent.payload "warmed_attempt_ids") | ForEach-Object { [string]$_ })
|
||||
}
|
||||
$deepSample = $deepAttempts | Where-Object { $warmedAttemptIDs -contains [string]$_.id } | Select-Object -First 1
|
||||
if ($null -eq $deepSample) {
|
||||
$deepSample = $deepAttempts | Select-Object -First 1
|
||||
}
|
||||
|
||||
$backendLine = (& ssh $DockerSSH "docker ps --format '{{.Names}} {{.Image}} {{.Status}}' | grep '^rap_test_backend '") | Out-String
|
||||
$nodeLines = (& ssh $DockerSSH "docker ps --format '{{.Names}} {{.Image}} {{.Status}}' | grep '^rap_test_node_test_'") | Out-String
|
||||
|
||||
$result = [ordered]@{
|
||||
schema_version = "c18z45.service_channel_rebuild_snapshot_auto_warmup_smoke.v1"
|
||||
cluster_id = $ClusterID
|
||||
entry_node_id = $entryNodeID
|
||||
primary_route_id = $primaryRouteID
|
||||
alternate_route_id = $alternateRouteID
|
||||
generation = $generation
|
||||
passed = [bool](
|
||||
$backendLine.Contains($ExpectedBackendImage) -and
|
||||
$nodeLines.Contains($ExpectedNodeAgentImage) -and
|
||||
$null -ne $auditEvent -and
|
||||
[string](Get-Prop $auditEvent.payload "trigger") -eq "node_heartbeat" -and
|
||||
[int](Get-Prop $auditEvent.payload "warmed_count") -gt 0 -and
|
||||
@((Get-Prop $auditEvent.payload "warmed_route_ids") | ForEach-Object { [string]$_ }) -contains $primaryRouteID -and
|
||||
@((Get-Prop $auditEvent.payload "warmed_generations") | ForEach-Object { [string]$_ }) -contains $generation -and
|
||||
$deepAttempts.Count -ge 1 -and
|
||||
[string](Get-Prop $deepSample "correlation_snapshot_at") -ne "" -and
|
||||
((Get-Prop $deepSample "node_transition_matched") -eq $true -or (Get-Prop $deepSample "node_route_generation_matched") -eq $true -or [string](Get-Prop $deepSample "post_rebuild_selected_route_id") -ne "")
|
||||
)
|
||||
checks = [ordered]@{
|
||||
backend_expected_image_deployed = $backendLine.Contains($ExpectedBackendImage)
|
||||
node_agent_expected_image_deployed = $nodeLines.Contains($ExpectedNodeAgentImage)
|
||||
audit_auto_warmup_found = ($null -ne $auditEvent)
|
||||
audit_triggered_by_heartbeat = ($null -ne $auditEvent -and [string](Get-Prop $auditEvent.payload "trigger") -eq "node_heartbeat")
|
||||
audit_warmed_snapshot = ($null -ne $auditEvent -and [int](Get-Prop $auditEvent.payload "warmed_count") -gt 0)
|
||||
audit_warmed_current_route = ($null -ne $auditEvent -and @((Get-Prop $auditEvent.payload "warmed_route_ids") | ForEach-Object { [string]$_ }) -contains $primaryRouteID)
|
||||
audit_warmed_current_generation = ($null -ne $auditEvent -and @((Get-Prop $auditEvent.payload "warmed_generations") | ForEach-Object { [string]$_ }) -contains $generation)
|
||||
deep_has_snapshot = ($deepAttempts.Count -ge 1 -and [string](Get-Prop $deepSample "correlation_snapshot_at") -ne "")
|
||||
deep_has_runtime_evidence = ($deepAttempts.Count -ge 1 -and ((Get-Prop $deepSample "node_transition_matched") -eq $true -or (Get-Prop $deepSample "node_route_generation_matched") -eq $true -or [string](Get-Prop $deepSample "post_rebuild_selected_route_id") -ne ""))
|
||||
}
|
||||
summary = [ordered]@{
|
||||
backend_container = $backendLine.Trim()
|
||||
node_containers = $nodeLines.Trim()
|
||||
audit_event = $auditEvent
|
||||
deep_sample = $deepSample
|
||||
base_smoke_result = (Join-Path $repoRoot $baseResultPath)
|
||||
}
|
||||
}
|
||||
|
||||
$resultFullPath = Join-Path $repoRoot $ResultPath
|
||||
$resultDir = Split-Path -Parent $resultFullPath
|
||||
if (-not (Test-Path $resultDir)) {
|
||||
New-Item -ItemType Directory -Path $resultDir | Out-Null
|
||||
}
|
||||
$result | ConvertTo-Json -Depth 100 | Set-Content -Path $resultFullPath -Encoding UTF8
|
||||
|
||||
if (-not $result.passed) {
|
||||
throw "C18Z45 service-channel rebuild snapshot auto-warmup smoke failed. Result: $resultFullPath"
|
||||
}
|
||||
|
||||
Write-Host "C18Z45 service-channel rebuild snapshot auto-warmup smoke passed. Result: $resultFullPath"
|
||||
$result
|
||||
@@ -0,0 +1,73 @@
|
||||
param(
|
||||
[string]$ApiBaseUrl = "http://192.168.200.61:18121/api/v1",
|
||||
[string]$ClusterID = "cfc0743d-d960-49fb-9de8-96e063d5e4aa",
|
||||
[string]$ActorUserID = "f67d943f-5397-4b3a-a229-695fe67ad700",
|
||||
[string]$DockerSSH = "test-docker",
|
||||
[string]$ExpectedBackendImage = "rap-backend:fabric-service-channel-0.2.228",
|
||||
[string]$ResultPath = "artifacts\c18z46-service-channel-rebuild-snapshot-health-smoke-result.json"
|
||||
)
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
$scriptDir = Split-Path -Parent $MyInvocation.MyCommand.Path
|
||||
$repoRoot = (Resolve-Path (Join-Path $scriptDir "..\..")).ProviderPath
|
||||
|
||||
function Invoke-ApiGet {
|
||||
param([string]$Path, [int]$TimeoutSec = 30)
|
||||
Invoke-RestMethod -Method Get -Uri "$ApiBaseUrl$Path" -TimeoutSec $TimeoutSec
|
||||
}
|
||||
|
||||
function Get-Prop {
|
||||
param([object]$Object, [string]$Name)
|
||||
if ($null -eq $Object) { return $null }
|
||||
$prop = $Object.PSObject.Properties[$Name]
|
||||
if ($null -eq $prop) { return $null }
|
||||
return $prop.Value
|
||||
}
|
||||
|
||||
$schemaStatus = (Invoke-ApiGet -Path "/clusters/$ClusterID/fabric/service-channels/schema-status?actor_user_id=$ActorUserID" -TimeoutSec 15).fabric_service_channel_schema_status
|
||||
$health = (Invoke-ApiGet -Path "/clusters/$ClusterID/fabric/service-channels/rebuild-snapshots/health?actor_user_id=$ActorUserID&limit=50&min_age_seconds=60&heartbeat_threshold=2" -TimeoutSec 30).rebuild_snapshot_health
|
||||
$backendLine = (& ssh $DockerSSH "docker ps --format '{{.Names}} {{.Image}} {{.Status}}' | grep '^rap_test_backend '") | Out-String
|
||||
|
||||
$result = [ordered]@{
|
||||
schema_version = "c18z46.service_channel_rebuild_snapshot_health_smoke.v1"
|
||||
cluster_id = $ClusterID
|
||||
passed = [bool](
|
||||
$backendLine.Contains($ExpectedBackendImage) -and
|
||||
[string](Get-Prop $schemaStatus "status") -ne "" -and
|
||||
[string](Get-Prop $health "status") -ne "" -and
|
||||
[int](Get-Prop $health "window_limit") -eq 50 -and
|
||||
[int](Get-Prop $health "heartbeat_threshold") -eq 2 -and
|
||||
[int](Get-Prop $health "recent_attempt_count") -ge 0 -and
|
||||
[int](Get-Prop $health "auto_warmup_event_count") -ge 0
|
||||
)
|
||||
checks = [ordered]@{
|
||||
backend_expected_image_deployed = $backendLine.Contains($ExpectedBackendImage)
|
||||
schema_status_present = ([string](Get-Prop $schemaStatus "status") -ne "")
|
||||
health_status_present = ([string](Get-Prop $health "status") -ne "")
|
||||
health_window_limit_honored = ([int](Get-Prop $health "window_limit") -eq 50)
|
||||
health_heartbeat_threshold_honored = ([int](Get-Prop $health "heartbeat_threshold") -eq 2)
|
||||
health_recent_attempt_count_present = ([int](Get-Prop $health "recent_attempt_count") -ge 0)
|
||||
health_auto_warmup_count_present = ([int](Get-Prop $health "auto_warmup_event_count") -ge 0)
|
||||
}
|
||||
summary = [ordered]@{
|
||||
backend_container = $backendLine.Trim()
|
||||
schema_status = $schemaStatus
|
||||
snapshot_health = $health
|
||||
}
|
||||
}
|
||||
|
||||
$resultFullPath = Join-Path $repoRoot $ResultPath
|
||||
$resultDir = Split-Path -Parent $resultFullPath
|
||||
if (-not (Test-Path $resultDir)) {
|
||||
New-Item -ItemType Directory -Path $resultDir | Out-Null
|
||||
}
|
||||
$result | ConvertTo-Json -Depth 100 | Set-Content -Path $resultFullPath -Encoding UTF8
|
||||
|
||||
if (-not $result.passed) {
|
||||
throw "C18Z46 service-channel rebuild snapshot health smoke failed. Result: $resultFullPath"
|
||||
}
|
||||
|
||||
Write-Host "C18Z46 service-channel rebuild snapshot health smoke passed. Result: $resultFullPath"
|
||||
$result
|
||||
@@ -0,0 +1,132 @@
|
||||
param(
|
||||
[string]$ApiBaseUrl = "http://192.168.200.61:18121/api/v1",
|
||||
[string]$ClusterID = "cfc0743d-d960-49fb-9de8-96e063d5e4aa",
|
||||
[string]$ActorUserID = "f67d943f-5397-4b3a-a229-695fe67ad700",
|
||||
[string]$EntryNodeName = "test-1",
|
||||
[string]$ExitNodeName = "test-2",
|
||||
[string]$EntryBaseUrl = "http://192.168.200.61:19131",
|
||||
[string]$DockerSSH = "test-docker",
|
||||
[string]$ExpectedBackendImage = "rap-backend:fabric-service-channel-0.2.228",
|
||||
[string]$ExpectedNodeAgentImage = "rap-node-agent:0.2.230",
|
||||
[string]$ResultPath = "artifacts\c18z47-service-channel-signed-lease-enforcement-smoke-result.json"
|
||||
)
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
$scriptDir = Split-Path -Parent $MyInvocation.MyCommand.Path
|
||||
$repoRoot = (Resolve-Path (Join-Path $scriptDir "..\..")).ProviderPath
|
||||
|
||||
function Invoke-ApiGet {
|
||||
param([string]$Path, [int]$TimeoutSec = 30)
|
||||
Invoke-RestMethod -Method Get -Uri "$ApiBaseUrl$Path" -TimeoutSec $TimeoutSec
|
||||
}
|
||||
|
||||
function ConvertTo-Base64Url {
|
||||
param([string]$Value)
|
||||
$bytes = [System.Text.Encoding]::UTF8.GetBytes($Value)
|
||||
return [Convert]::ToBase64String($bytes).TrimEnd("=").Replace("+", "-").Replace("/", "_")
|
||||
}
|
||||
|
||||
function Invoke-RawPostStatus {
|
||||
param(
|
||||
[string]$Url,
|
||||
[hashtable]$Headers
|
||||
)
|
||||
try {
|
||||
Invoke-WebRequest -Method Post -Uri $Url -Headers $Headers -Body ([System.Text.Encoding]::UTF8.GetBytes("packet")) -ContentType "application/octet-stream" -TimeoutSec 30 | Out-Null
|
||||
return 202
|
||||
} catch {
|
||||
if ($_.Exception.Response -and $_.Exception.Response.StatusCode) {
|
||||
return [int]$_.Exception.Response.StatusCode
|
||||
}
|
||||
throw
|
||||
}
|
||||
}
|
||||
|
||||
$nodes = @((Invoke-ApiGet -Path "/clusters/$ClusterID/nodes?actor_user_id=$ActorUserID").nodes)
|
||||
$entryNode = $nodes | Where-Object { $_.name -eq $EntryNodeName } | Select-Object -First 1
|
||||
$exitNode = $nodes | Where-Object { $_.name -eq $ExitNodeName } | Select-Object -First 1
|
||||
if ($null -eq $entryNode -or $null -eq $exitNode) {
|
||||
throw "Entry or exit node not found: $EntryNodeName / $ExitNodeName"
|
||||
}
|
||||
|
||||
$leaseBody = @{
|
||||
actor_user_id = $ActorUserID
|
||||
organization_id = "smoke-org"
|
||||
user_id = "smoke-user"
|
||||
resource_id = "c18z47-vpn-smoke"
|
||||
service_class = "vpn_packets"
|
||||
entry_node_ids = @([string]$entryNode.id)
|
||||
exit_node_ids = @([string]$exitNode.id)
|
||||
preferred_entry_node_id = [string]$entryNode.id
|
||||
preferred_exit_node_id = [string]$exitNode.id
|
||||
allowed_channels = @("vpn_packet", "fabric_control")
|
||||
ttl_seconds = 120
|
||||
} | ConvertTo-Json -Depth 20
|
||||
$lease = (Invoke-RestMethod -Method Post -Uri "$ApiBaseUrl/clusters/$ClusterID/fabric/service-channels/leases" -ContentType "application/json" -Body $leaseBody -TimeoutSec 30).fabric_service_channel_lease
|
||||
|
||||
$packetPath = $lease.entry_http.path_template.
|
||||
Replace("{cluster_id}", $ClusterID).
|
||||
Replace("{channel_id}", [string]$lease.channel_id).
|
||||
Replace("{resource_id}", "c18z47-vpn-smoke")
|
||||
$packetUrl = $EntryBaseUrl.TrimEnd("/") + $packetPath
|
||||
$unsignedHeaders = @{
|
||||
"X-RAP-Service-Channel-Token" = [string]$lease.token.token
|
||||
"X-RAP-Fabric-Channel-ID" = [string]$lease.channel_id
|
||||
"X-RAP-Service-Class" = "vpn_packets"
|
||||
"X-RAP-Channel-Class" = "vpn_packet"
|
||||
}
|
||||
$signedHeaders = $unsignedHeaders.Clone()
|
||||
$signedHeaders["X-RAP-Service-Channel-Authority-Payload"] = ConvertTo-Base64Url (($lease.authority_payload | ConvertTo-Json -Depth 50 -Compress))
|
||||
$signedHeaders["X-RAP-Service-Channel-Authority-Signature"] = ConvertTo-Base64Url (($lease.authority_signature | ConvertTo-Json -Depth 50 -Compress))
|
||||
|
||||
$unsignedStatus = Invoke-RawPostStatus -Url $packetUrl -Headers $unsignedHeaders
|
||||
$signedStatus = Invoke-RawPostStatus -Url $packetUrl -Headers $signedHeaders
|
||||
$backendLine = (& ssh $DockerSSH "docker ps --format '{{.Names}} {{.Image}} {{.Status}}' | grep '^rap_test_backend '") | Out-String
|
||||
$nodeLines = (& ssh $DockerSSH "docker ps --format '{{.Names}} {{.Image}} {{.Status}}' | grep '^rap_test_node_test_'") | Out-String
|
||||
|
||||
$result = [ordered]@{
|
||||
schema_version = "c18z47.service_channel_signed_lease_enforcement_smoke.v1"
|
||||
cluster_id = $ClusterID
|
||||
entry_node_id = [string]$entryNode.id
|
||||
exit_node_id = [string]$exitNode.id
|
||||
channel_id = [string]$lease.channel_id
|
||||
passed = [bool](
|
||||
$backendLine.Contains($ExpectedBackendImage) -and
|
||||
$nodeLines.Contains($ExpectedNodeAgentImage) -and
|
||||
$unsignedStatus -eq 403 -and
|
||||
$signedStatus -eq 202 -and
|
||||
[string]$lease.authority_payload.schema_version -eq "rap.fabric_service_channel_lease_authority.v1"
|
||||
)
|
||||
checks = [ordered]@{
|
||||
backend_expected_image_deployed = $backendLine.Contains($ExpectedBackendImage)
|
||||
node_agent_expected_image_deployed = $nodeLines.Contains($ExpectedNodeAgentImage)
|
||||
unsigned_rejected = ($unsignedStatus -eq 403)
|
||||
signed_accepted = ($signedStatus -eq 202)
|
||||
lease_authority_payload_present = ([string]$lease.authority_payload.schema_version -eq "rap.fabric_service_channel_lease_authority.v1")
|
||||
}
|
||||
summary = [ordered]@{
|
||||
backend_container = $backendLine.Trim()
|
||||
node_containers = $nodeLines.Trim()
|
||||
packet_url = $packetUrl
|
||||
unsigned_status = $unsignedStatus
|
||||
signed_status = $signedStatus
|
||||
lease_status = [string]$lease.status
|
||||
selected_route = $lease.primary_route
|
||||
}
|
||||
}
|
||||
|
||||
$resultFullPath = Join-Path $repoRoot $ResultPath
|
||||
$resultDir = Split-Path -Parent $resultFullPath
|
||||
if (-not (Test-Path $resultDir)) {
|
||||
New-Item -ItemType Directory -Path $resultDir | Out-Null
|
||||
}
|
||||
$result | ConvertTo-Json -Depth 100 | Set-Content -Path $resultFullPath -Encoding UTF8
|
||||
|
||||
if (-not $result.passed) {
|
||||
throw "C18Z47 service-channel signed lease enforcement smoke failed. Result: $resultFullPath"
|
||||
}
|
||||
|
||||
Write-Host "C18Z47 service-channel signed lease enforcement smoke passed. Result: $resultFullPath"
|
||||
$result
|
||||
@@ -0,0 +1,128 @@
|
||||
param(
|
||||
[string]$ApiBaseUrl = "http://192.168.200.61:18121/api/v1",
|
||||
[string]$ClusterID = "cfc0743d-d960-49fb-9de8-96e063d5e4aa",
|
||||
[string]$ActorUserID = "f67d943f-5397-4b3a-a229-695fe67ad700",
|
||||
[string]$EntryNodeName = "test-1",
|
||||
[string]$ExitNodeName = "test-2",
|
||||
[string]$EntryBaseUrl = "http://192.168.200.61:19131",
|
||||
[string]$DockerSSH = "test-docker",
|
||||
[string]$ExpectedBackendImage = "rap-backend:fabric-service-channel-0.2.231",
|
||||
[string]$ExpectedNodeAgentImage = "rap-node-agent:0.2.232",
|
||||
[string]$ResultPath = "artifacts\c18z48-service-channel-introspection-smoke-result.json"
|
||||
)
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
$scriptDir = Split-Path -Parent $MyInvocation.MyCommand.Path
|
||||
$repoRoot = (Resolve-Path (Join-Path $scriptDir "..\..")).ProviderPath
|
||||
|
||||
function Invoke-ApiGet {
|
||||
param([string]$Path, [int]$TimeoutSec = 30)
|
||||
Invoke-RestMethod -Method Get -Uri "$ApiBaseUrl$Path" -TimeoutSec $TimeoutSec
|
||||
}
|
||||
|
||||
function Invoke-RawPostStatus {
|
||||
param(
|
||||
[string]$Url,
|
||||
[hashtable]$Headers
|
||||
)
|
||||
try {
|
||||
Invoke-WebRequest -Method Post -Uri $Url -Headers $Headers -Body ([System.Text.Encoding]::UTF8.GetBytes("packet")) -ContentType "application/octet-stream" -TimeoutSec 30 | Out-Null
|
||||
return 202
|
||||
} catch {
|
||||
if ($_.Exception.Response -and $_.Exception.Response.StatusCode) {
|
||||
return [int]$_.Exception.Response.StatusCode
|
||||
}
|
||||
throw
|
||||
}
|
||||
}
|
||||
|
||||
$nodes = @((Invoke-ApiGet -Path "/clusters/$ClusterID/nodes?actor_user_id=$ActorUserID").nodes)
|
||||
$entryNode = $nodes | Where-Object { $_.name -eq $EntryNodeName } | Select-Object -First 1
|
||||
$exitNode = $nodes | Where-Object { $_.name -eq $ExitNodeName } | Select-Object -First 1
|
||||
if ($null -eq $entryNode -or $null -eq $exitNode) {
|
||||
throw "Entry or exit node not found: $EntryNodeName / $ExitNodeName"
|
||||
}
|
||||
|
||||
$resourceID = "c18z48-vpn-smoke"
|
||||
$leaseBody = @{
|
||||
actor_user_id = $ActorUserID
|
||||
organization_id = "smoke-org"
|
||||
user_id = "smoke-user"
|
||||
resource_id = $resourceID
|
||||
service_class = "vpn_packets"
|
||||
entry_node_ids = @([string]$entryNode.id)
|
||||
exit_node_ids = @([string]$exitNode.id)
|
||||
preferred_entry_node_id = [string]$entryNode.id
|
||||
preferred_exit_node_id = [string]$exitNode.id
|
||||
allowed_channels = @("vpn_packet", "fabric_control")
|
||||
ttl_seconds = 120
|
||||
} | ConvertTo-Json -Depth 20
|
||||
$lease = (Invoke-RestMethod -Method Post -Uri "$ApiBaseUrl/clusters/$ClusterID/fabric/service-channels/leases" -ContentType "application/json" -Body $leaseBody -TimeoutSec 30).fabric_service_channel_lease
|
||||
|
||||
$packetPath = $lease.entry_http.path_template.
|
||||
Replace("{cluster_id}", $ClusterID).
|
||||
Replace("{channel_id}", [string]$lease.channel_id).
|
||||
Replace("{resource_id}", $resourceID)
|
||||
$packetUrl = $EntryBaseUrl.TrimEnd("/") + $packetPath
|
||||
$headers = @{
|
||||
"X-RAP-Service-Channel-Token" = [string]$lease.token.token
|
||||
"X-RAP-Fabric-Channel-ID" = [string]$lease.channel_id
|
||||
"X-RAP-Service-Class" = "vpn_packets"
|
||||
"X-RAP-Channel-Class" = "vpn_packet"
|
||||
}
|
||||
$badHeaders = $headers.Clone()
|
||||
$badHeaders["X-RAP-Service-Channel-Token"] = "rap_fsc_wrong"
|
||||
|
||||
$introspectionStatus = Invoke-RawPostStatus -Url $packetUrl -Headers $headers
|
||||
$introspectionResponse = Invoke-WebRequest -Method Post -Uri $packetUrl -Headers $headers -Body ([System.Text.Encoding]::UTF8.GetBytes("packet-accepted-by")) -ContentType "application/octet-stream" -TimeoutSec 30
|
||||
$badTokenStatus = Invoke-RawPostStatus -Url $packetUrl -Headers $badHeaders
|
||||
$backendLine = (& ssh $DockerSSH "docker ps --format '{{.Names}} {{.Image}} {{.Status}}' | grep '^rap_test_backend '") | Out-String
|
||||
$nodeLines = (& ssh $DockerSSH "docker ps --format '{{.Names}} {{.Image}} {{.Status}}' | grep '^rap_test_node_test_'") | Out-String
|
||||
|
||||
$result = [ordered]@{
|
||||
schema_version = "c18z48.service_channel_introspection_smoke.v1"
|
||||
cluster_id = $ClusterID
|
||||
entry_node_id = [string]$entryNode.id
|
||||
exit_node_id = [string]$exitNode.id
|
||||
channel_id = [string]$lease.channel_id
|
||||
passed = [bool](
|
||||
$backendLine.Contains($ExpectedBackendImage) -and
|
||||
$nodeLines.Contains($ExpectedNodeAgentImage) -and
|
||||
$introspectionStatus -eq 202 -and
|
||||
[string]$introspectionResponse.Headers["X-RAP-Service-Channel-Accepted-By"] -eq "introspection" -and
|
||||
$badTokenStatus -eq 403
|
||||
)
|
||||
checks = [ordered]@{
|
||||
backend_expected_image_deployed = $backendLine.Contains($ExpectedBackendImage)
|
||||
node_agent_expected_image_deployed = $nodeLines.Contains($ExpectedNodeAgentImage)
|
||||
unsigned_token_accepted_by_introspection = ($introspectionStatus -eq 202)
|
||||
accepted_by_header_is_introspection = ([string]$introspectionResponse.Headers["X-RAP-Service-Channel-Accepted-By"] -eq "introspection")
|
||||
bad_token_rejected = ($badTokenStatus -eq 403)
|
||||
}
|
||||
summary = [ordered]@{
|
||||
backend_container = $backendLine.Trim()
|
||||
node_containers = $nodeLines.Trim()
|
||||
packet_url = $packetUrl
|
||||
introspection_status = $introspectionStatus
|
||||
accepted_by = [string]$introspectionResponse.Headers["X-RAP-Service-Channel-Accepted-By"]
|
||||
bad_token_status = $badTokenStatus
|
||||
lease_status = [string]$lease.status
|
||||
selected_route = $lease.primary_route
|
||||
}
|
||||
}
|
||||
|
||||
$resultFullPath = Join-Path $repoRoot $ResultPath
|
||||
$resultDir = Split-Path -Parent $resultFullPath
|
||||
if (-not (Test-Path $resultDir)) {
|
||||
New-Item -ItemType Directory -Path $resultDir | Out-Null
|
||||
}
|
||||
$result | ConvertTo-Json -Depth 100 | Set-Content -Path $resultFullPath -Encoding UTF8
|
||||
|
||||
if (-not $result.passed) {
|
||||
throw "C18Z48 service-channel introspection smoke failed. Result: $resultFullPath"
|
||||
}
|
||||
|
||||
Write-Host "C18Z48 service-channel introspection smoke passed. Result: $resultFullPath"
|
||||
$result
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,153 @@
|
||||
param(
|
||||
[string]$ApiBaseUrl = "http://192.168.200.61:18121/api/v1",
|
||||
[string]$ClusterID = "cfc0743d-d960-49fb-9de8-96e063d5e4aa",
|
||||
[string]$ActorUserID = "f67d943f-5397-4b3a-a229-695fe67ad700",
|
||||
[string]$EntryNodeName = "test-1",
|
||||
[string]$ExitNodeName = "test-2",
|
||||
[string]$EntryBaseUrl = "http://192.168.200.61:19131",
|
||||
[string]$DockerSSH = "test-docker",
|
||||
[string]$ExpectedBackendImage = "rap-backend:fabric-service-channel-0.2.233",
|
||||
[string]$ExpectedNodeAgentImage = "rap-node-agent:0.2.232",
|
||||
[string]$ResultPath = "artifacts\c18z50-service-channel-durable-introspection-smoke-result.json"
|
||||
)
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
$scriptDir = Split-Path -Parent $MyInvocation.MyCommand.Path
|
||||
$repoRoot = (Resolve-Path (Join-Path $scriptDir "..\..")).ProviderPath
|
||||
|
||||
function Invoke-ApiGet {
|
||||
param([string]$Path, [int]$TimeoutSec = 30)
|
||||
Invoke-RestMethod -Method Get -Uri "$ApiBaseUrl$Path" -TimeoutSec $TimeoutSec
|
||||
}
|
||||
|
||||
function Wait-Api {
|
||||
for ($i = 0; $i -lt 30; $i++) {
|
||||
try {
|
||||
Invoke-ApiGet -Path "/clusters/$ClusterID/nodes?actor_user_id=$ActorUserID" -TimeoutSec 3 | Out-Null
|
||||
return
|
||||
} catch {
|
||||
Start-Sleep -Seconds 1
|
||||
}
|
||||
}
|
||||
throw "API did not become ready after backend restart"
|
||||
}
|
||||
|
||||
function Invoke-RawPost {
|
||||
param(
|
||||
[string]$Url,
|
||||
[hashtable]$Headers,
|
||||
[string]$Body = "packet"
|
||||
)
|
||||
try {
|
||||
return Invoke-WebRequest -Method Post -Uri $Url -Headers $Headers -Body ([System.Text.Encoding]::UTF8.GetBytes($Body)) -ContentType "application/octet-stream" -TimeoutSec 30
|
||||
} catch {
|
||||
if ($_.Exception.Response -and $_.Exception.Response.StatusCode) {
|
||||
return [pscustomobject]@{ StatusCode = [int]$_.Exception.Response.StatusCode; Headers = @{} }
|
||||
}
|
||||
throw
|
||||
}
|
||||
}
|
||||
|
||||
function Invoke-RemoteSqlScalar {
|
||||
param([string]$Sql)
|
||||
$encoded = [Convert]::ToBase64String([System.Text.Encoding]::UTF8.GetBytes($Sql))
|
||||
(& ssh $DockerSSH "printf '%s' '$encoded' | base64 -d | docker exec -i rap_test_postgres psql -U rap_user -d remote_access_platform -t -A -v ON_ERROR_STOP=1") | Out-String
|
||||
}
|
||||
|
||||
$nodes = @((Invoke-ApiGet -Path "/clusters/$ClusterID/nodes?actor_user_id=$ActorUserID").nodes)
|
||||
$entryNode = $nodes | Where-Object { $_.name -eq $EntryNodeName } | Select-Object -First 1
|
||||
$exitNode = $nodes | Where-Object { $_.name -eq $ExitNodeName } | Select-Object -First 1
|
||||
if ($null -eq $entryNode -or $null -eq $exitNode) {
|
||||
throw "Entry or exit node not found: $EntryNodeName / $ExitNodeName"
|
||||
}
|
||||
|
||||
$resourceID = "c18z50-vpn-smoke"
|
||||
$leaseBody = @{
|
||||
actor_user_id = $ActorUserID
|
||||
organization_id = "smoke-org"
|
||||
user_id = "smoke-user"
|
||||
resource_id = $resourceID
|
||||
service_class = "vpn_packets"
|
||||
entry_node_ids = @([string]$entryNode.id)
|
||||
exit_node_ids = @([string]$exitNode.id)
|
||||
preferred_entry_node_id = [string]$entryNode.id
|
||||
preferred_exit_node_id = [string]$exitNode.id
|
||||
allowed_channels = @("vpn_packet", "fabric_control")
|
||||
ttl_seconds = 120
|
||||
} | ConvertTo-Json -Depth 20
|
||||
$lease = (Invoke-RestMethod -Method Post -Uri "$ApiBaseUrl/clusters/$ClusterID/fabric/service-channels/leases" -ContentType "application/json" -Body $leaseBody -TimeoutSec 30).fabric_service_channel_lease
|
||||
|
||||
$storedTokenValue = (Invoke-RemoteSqlScalar -Sql "SELECT COALESCE(lease #>> '{token,token}', '<null>') FROM fabric_service_channel_leases WHERE cluster_id = '$ClusterID'::uuid AND channel_id = '$($lease.channel_id)'::uuid;").Trim()
|
||||
|
||||
& ssh $DockerSSH "docker restart rap_test_backend >/dev/null"
|
||||
Wait-Api
|
||||
|
||||
$packetPath = $lease.entry_http.path_template.
|
||||
Replace("{cluster_id}", $ClusterID).
|
||||
Replace("{channel_id}", [string]$lease.channel_id).
|
||||
Replace("{resource_id}", $resourceID)
|
||||
$packetUrl = $EntryBaseUrl.TrimEnd("/") + $packetPath
|
||||
$headers = @{
|
||||
"X-RAP-Service-Channel-Token" = [string]$lease.token.token
|
||||
"X-RAP-Fabric-Channel-ID" = [string]$lease.channel_id
|
||||
"X-RAP-Service-Class" = "vpn_packets"
|
||||
"X-RAP-Channel-Class" = "vpn_packet"
|
||||
}
|
||||
$badHeaders = $headers.Clone()
|
||||
$badHeaders["X-RAP-Service-Channel-Token"] = "rap_fsc_wrong"
|
||||
|
||||
$acceptedResponse = Invoke-RawPost -Url $packetUrl -Headers $headers -Body "packet-after-backend-restart"
|
||||
$badResponse = Invoke-RawPost -Url $packetUrl -Headers $badHeaders -Body "bad-token"
|
||||
$backendLine = (& ssh $DockerSSH "docker ps --format '{{.Names}} {{.Image}} {{.Status}}' | grep '^rap_test_backend '") | Out-String
|
||||
$nodeLines = (& ssh $DockerSSH "docker ps --format '{{.Names}} {{.Image}} {{.Status}}' | grep '^rap_test_node_test_'") | Out-String
|
||||
|
||||
$result = [ordered]@{
|
||||
schema_version = "c18z50.service_channel_durable_introspection_smoke.v1"
|
||||
cluster_id = $ClusterID
|
||||
entry_node_id = [string]$entryNode.id
|
||||
exit_node_id = [string]$exitNode.id
|
||||
channel_id = [string]$lease.channel_id
|
||||
passed = [bool](
|
||||
$backendLine.Contains($ExpectedBackendImage) -and
|
||||
$nodeLines.Contains($ExpectedNodeAgentImage) -and
|
||||
[int]$acceptedResponse.StatusCode -eq 202 -and
|
||||
[string]$acceptedResponse.Headers["X-RAP-Service-Channel-Accepted-By"] -eq "introspection" -and
|
||||
[int]$badResponse.StatusCode -eq 403 -and
|
||||
$storedTokenValue -eq ""
|
||||
)
|
||||
checks = [ordered]@{
|
||||
backend_expected_image_deployed = $backendLine.Contains($ExpectedBackendImage)
|
||||
node_agent_expected_image_deployed = $nodeLines.Contains($ExpectedNodeAgentImage)
|
||||
durable_introspection_after_backend_restart = ([int]$acceptedResponse.StatusCode -eq 202)
|
||||
accepted_by_header_is_introspection = ([string]$acceptedResponse.Headers["X-RAP-Service-Channel-Accepted-By"] -eq "introspection")
|
||||
bad_token_rejected = ([int]$badResponse.StatusCode -eq 403)
|
||||
durable_lease_omits_raw_token = ($storedTokenValue -eq "")
|
||||
}
|
||||
summary = [ordered]@{
|
||||
backend_container = $backendLine.Trim()
|
||||
node_containers = $nodeLines.Trim()
|
||||
packet_url = $packetUrl
|
||||
accepted_status = [int]$acceptedResponse.StatusCode
|
||||
accepted_by = [string]$acceptedResponse.Headers["X-RAP-Service-Channel-Accepted-By"]
|
||||
bad_token_status = [int]$badResponse.StatusCode
|
||||
stored_token_value = $storedTokenValue
|
||||
lease_status = [string]$lease.status
|
||||
selected_route = $lease.primary_route
|
||||
}
|
||||
}
|
||||
|
||||
$resultFullPath = Join-Path $repoRoot $ResultPath
|
||||
$resultDir = Split-Path -Parent $resultFullPath
|
||||
if (-not (Test-Path $resultDir)) {
|
||||
New-Item -ItemType Directory -Path $resultDir | Out-Null
|
||||
}
|
||||
$result | ConvertTo-Json -Depth 100 | Set-Content -Path $resultFullPath -Encoding UTF8
|
||||
|
||||
if (-not $result.passed) {
|
||||
throw "C18Z50 service-channel durable introspection smoke failed. Result: $resultFullPath"
|
||||
}
|
||||
|
||||
Write-Host "C18Z50 service-channel durable introspection smoke passed. Result: $resultFullPath"
|
||||
$result
|
||||
@@ -0,0 +1,103 @@
|
||||
param(
|
||||
[string]$ApiBaseUrl = "http://192.168.200.61:18121/api/v1",
|
||||
[string]$ClusterID = "cfc0743d-d960-49fb-9de8-96e063d5e4aa",
|
||||
[string]$ActorUserID = "f67d943f-5397-4b3a-a229-695fe67ad700",
|
||||
[string]$EntryNodeName = "test-1",
|
||||
[string]$ExitNodeName = "test-2",
|
||||
[string]$DockerSSH = "test-docker",
|
||||
[string]$ExpectedBackendImage = "rap-backend:fabric-service-channel-0.2.234",
|
||||
[string]$ResultPath = "artifacts\c18z51-service-channel-lease-maintenance-smoke-result.json"
|
||||
)
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
$scriptDir = Split-Path -Parent $MyInvocation.MyCommand.Path
|
||||
$repoRoot = (Resolve-Path (Join-Path $scriptDir "..\..")).ProviderPath
|
||||
|
||||
function Invoke-ApiGet {
|
||||
param([string]$Path, [int]$TimeoutSec = 30)
|
||||
Invoke-RestMethod -Method Get -Uri "$ApiBaseUrl$Path" -TimeoutSec $TimeoutSec
|
||||
}
|
||||
|
||||
$nodes = @((Invoke-ApiGet -Path "/clusters/$ClusterID/nodes?actor_user_id=$ActorUserID").nodes)
|
||||
$entryNode = $nodes | Where-Object { $_.name -eq $EntryNodeName } | Select-Object -First 1
|
||||
$exitNode = $nodes | Where-Object { $_.name -eq $ExitNodeName } | Select-Object -First 1
|
||||
if ($null -eq $entryNode -or $null -eq $exitNode) {
|
||||
throw "Entry or exit node not found: $EntryNodeName / $ExitNodeName"
|
||||
}
|
||||
|
||||
$resourceID = "c18z51-vpn-smoke"
|
||||
$leaseBody = @{
|
||||
actor_user_id = $ActorUserID
|
||||
organization_id = "smoke-org"
|
||||
user_id = "smoke-user"
|
||||
resource_id = $resourceID
|
||||
service_class = "vpn_packets"
|
||||
entry_node_ids = @([string]$entryNode.id)
|
||||
exit_node_ids = @([string]$exitNode.id)
|
||||
preferred_entry_node_id = [string]$entryNode.id
|
||||
preferred_exit_node_id = [string]$exitNode.id
|
||||
allowed_channels = @("vpn_packet", "fabric_control")
|
||||
ttl_seconds = 1
|
||||
} | ConvertTo-Json -Depth 20
|
||||
$lease = (Invoke-RestMethod -Method Post -Uri "$ApiBaseUrl/clusters/$ClusterID/fabric/service-channels/leases" -ContentType "application/json" -Body $leaseBody -TimeoutSec 30).fabric_service_channel_lease
|
||||
Start-Sleep -Seconds 2
|
||||
|
||||
$before = (Invoke-ApiGet -Path "/clusters/$ClusterID/fabric/service-channels/leases?actor_user_id=$ActorUserID&include_expired=true&limit=50").fabric_service_channel_lease_maintenance
|
||||
$cleanupBody = @{ actor_user_id = $ActorUserID; limit = 50 } | ConvertTo-Json -Depth 5
|
||||
$cleanup = (Invoke-RestMethod -Method Post -Uri "$ApiBaseUrl/clusters/$ClusterID/fabric/service-channels/leases/cleanup" -ContentType "application/json" -Body $cleanupBody -TimeoutSec 30).fabric_service_channel_lease_maintenance
|
||||
$after = (Invoke-ApiGet -Path "/clusters/$ClusterID/fabric/service-channels/leases?actor_user_id=$ActorUserID&include_expired=true&limit=50").fabric_service_channel_lease_maintenance
|
||||
$backendLine = (& ssh $DockerSSH "docker ps --format '{{.Names}} {{.Image}} {{.Status}}' | grep '^rap_test_backend '") | Out-String
|
||||
|
||||
$beforeLeases = @()
|
||||
if ($before.PSObject.Properties.Name -contains "leases") {
|
||||
$beforeLeases = @($before.leases)
|
||||
}
|
||||
$afterLeases = @()
|
||||
if ($after.PSObject.Properties.Name -contains "leases") {
|
||||
$afterLeases = @($after.leases)
|
||||
}
|
||||
$expiredBefore = @($beforeLeases | Where-Object { $_.channel_id -eq $lease.channel_id -and $_.expired })
|
||||
$presentAfter = @($afterLeases | Where-Object { $_.channel_id -eq $lease.channel_id })
|
||||
|
||||
$result = [ordered]@{
|
||||
schema_version = "c18z51.service_channel_lease_maintenance_smoke.v1"
|
||||
cluster_id = $ClusterID
|
||||
channel_id = [string]$lease.channel_id
|
||||
passed = [bool](
|
||||
$backendLine.Contains($ExpectedBackendImage) -and
|
||||
$expiredBefore.Count -eq 1 -and
|
||||
[int]$cleanup.deleted_expired_count -ge 1 -and
|
||||
$presentAfter.Count -eq 0
|
||||
)
|
||||
checks = [ordered]@{
|
||||
backend_expected_image_deployed = $backendLine.Contains($ExpectedBackendImage)
|
||||
expired_lease_visible_before_cleanup = ($expiredBefore.Count -eq 1)
|
||||
cleanup_deleted_expired = ([int]$cleanup.deleted_expired_count -ge 1)
|
||||
expired_lease_removed_after_cleanup = ($presentAfter.Count -eq 0)
|
||||
}
|
||||
summary = [ordered]@{
|
||||
backend_container = $backendLine.Trim()
|
||||
before_status = [string]$before.status
|
||||
before_active = [int]$before.active_count
|
||||
before_expired = [int]$before.expired_count
|
||||
cleanup_deleted = [int]$cleanup.deleted_expired_count
|
||||
after_active = [int]$after.active_count
|
||||
after_expired = [int]$after.expired_count
|
||||
}
|
||||
}
|
||||
|
||||
$resultFullPath = Join-Path $repoRoot $ResultPath
|
||||
$resultDir = Split-Path -Parent $resultFullPath
|
||||
if (-not (Test-Path $resultDir)) {
|
||||
New-Item -ItemType Directory -Path $resultDir | Out-Null
|
||||
}
|
||||
$result | ConvertTo-Json -Depth 100 | Set-Content -Path $resultFullPath -Encoding UTF8
|
||||
|
||||
if (-not $result.passed) {
|
||||
throw "C18Z51 service-channel lease maintenance smoke failed. Result: $resultFullPath"
|
||||
}
|
||||
|
||||
Write-Host "C18Z51 service-channel lease maintenance smoke passed. Result: $resultFullPath"
|
||||
$result
|
||||
@@ -0,0 +1,145 @@
|
||||
param(
|
||||
[string]$ApiBaseUrl = "http://192.168.200.61:18121/api/v1",
|
||||
[string]$ClusterID = "cfc0743d-d960-49fb-9de8-96e063d5e4aa",
|
||||
[string]$ActorUserID = "f67d943f-5397-4b3a-a229-695fe67ad700",
|
||||
[string]$EntryNodeName = "test-1",
|
||||
[string]$ExitNodeName = "test-2",
|
||||
[string]$EntryBaseUrl = "http://192.168.200.61:19131",
|
||||
[string]$DockerSSH = "test-docker",
|
||||
[string]$ExpectedBackendImage = "rap-backend:fabric-service-channel-0.2.239",
|
||||
[string]$ExpectedNodeAgentImage = "rap-node-agent:0.2.237",
|
||||
[string]$ResultPath = "artifacts\c18z52-service-channel-access-telemetry-smoke-result.json"
|
||||
)
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
$scriptDir = Split-Path -Parent $MyInvocation.MyCommand.Path
|
||||
$repoRoot = (Resolve-Path (Join-Path $scriptDir "..\..")).ProviderPath
|
||||
|
||||
function Invoke-ApiGet {
|
||||
param([string]$Path, [int]$TimeoutSec = 30)
|
||||
Invoke-RestMethod -Method Get -Uri "$ApiBaseUrl$Path" -TimeoutSec $TimeoutSec
|
||||
}
|
||||
|
||||
function Invoke-RawPostStatus {
|
||||
param(
|
||||
[string]$Url,
|
||||
[hashtable]$Headers,
|
||||
[string]$Body = "packet"
|
||||
)
|
||||
try {
|
||||
Invoke-WebRequest -Method Post -Uri $Url -Headers $Headers -Body ([System.Text.Encoding]::UTF8.GetBytes($Body)) -ContentType "application/octet-stream" -TimeoutSec 30 | Out-Null
|
||||
return 202
|
||||
} catch {
|
||||
if ($_.Exception.Response -and $_.Exception.Response.StatusCode) {
|
||||
return [int]$_.Exception.Response.StatusCode
|
||||
}
|
||||
throw
|
||||
}
|
||||
}
|
||||
|
||||
$nodes = @((Invoke-ApiGet -Path "/clusters/$ClusterID/nodes?actor_user_id=$ActorUserID").nodes)
|
||||
$entryNode = $nodes | Where-Object { $_.name -eq $EntryNodeName } | Select-Object -First 1
|
||||
$exitNode = $nodes | Where-Object { $_.name -eq $ExitNodeName } | Select-Object -First 1
|
||||
if ($null -eq $entryNode -or $null -eq $exitNode) {
|
||||
throw "Entry or exit node not found: $EntryNodeName / $ExitNodeName"
|
||||
}
|
||||
|
||||
$resourceID = "c18z52-vpn-smoke"
|
||||
$leaseBody = @{
|
||||
actor_user_id = $ActorUserID
|
||||
organization_id = "smoke-org"
|
||||
user_id = "smoke-user"
|
||||
resource_id = $resourceID
|
||||
service_class = "vpn_packets"
|
||||
entry_node_ids = @([string]$entryNode.id)
|
||||
exit_node_ids = @([string]$exitNode.id)
|
||||
preferred_entry_node_id = [string]$entryNode.id
|
||||
preferred_exit_node_id = [string]$exitNode.id
|
||||
allowed_channels = @("vpn_packet", "fabric_control")
|
||||
ttl_seconds = 120
|
||||
} | ConvertTo-Json -Depth 20
|
||||
$lease = (Invoke-RestMethod -Method Post -Uri "$ApiBaseUrl/clusters/$ClusterID/fabric/service-channels/leases" -ContentType "application/json" -Body $leaseBody -TimeoutSec 30).fabric_service_channel_lease
|
||||
|
||||
$packetPath = $lease.entry_http.path_template.
|
||||
Replace("{cluster_id}", $ClusterID).
|
||||
Replace("{channel_id}", [string]$lease.channel_id).
|
||||
Replace("{resource_id}", $resourceID)
|
||||
$packetUrl = $EntryBaseUrl.TrimEnd("/") + $packetPath
|
||||
$headers = @{
|
||||
"X-RAP-Service-Channel-Token" = [string]$lease.token.token
|
||||
"X-RAP-Fabric-Channel-ID" = [string]$lease.channel_id
|
||||
"X-RAP-Service-Class" = "vpn_packets"
|
||||
"X-RAP-Channel-Class" = "vpn_packet"
|
||||
}
|
||||
|
||||
$firstStatus = Invoke-RawPostStatus -Url $packetUrl -Headers $headers -Body "c18z52-access-1"
|
||||
$secondResponse = Invoke-WebRequest -Method Post -Uri $packetUrl -Headers $headers -Body ([System.Text.Encoding]::UTF8.GetBytes("c18z52-access-2")) -ContentType "application/octet-stream" -TimeoutSec 30
|
||||
$acceptedBy = [string]$secondResponse.Headers["X-RAP-Service-Channel-Accepted-By"]
|
||||
|
||||
$accessTelemetry = $null
|
||||
for ($i = 0; $i -lt 8; $i++) {
|
||||
Start-Sleep -Seconds 3
|
||||
$accessTelemetry = (Invoke-ApiGet -Path "/clusters/$ClusterID/fabric/service-channels/access-telemetry?actor_user_id=$ActorUserID&limit=20").fabric_service_channel_access_telemetry
|
||||
if ([int]$accessTelemetry.total_accepted -ge 2 -and [int]$accessTelemetry.reporting_node_count -ge 1) {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
$backendLine = (& ssh $DockerSSH "docker ps --format '{{.Names}} {{.Image}} {{.Status}}' | grep '^rap_test_backend '") | Out-String
|
||||
$nodeLines = (& ssh $DockerSSH "docker ps --format '{{.Names}} {{.Image}} {{.Status}}' | grep '^rap_test_node_test_'") | Out-String
|
||||
|
||||
$result = [ordered]@{
|
||||
schema_version = "c18z52.service_channel_access_telemetry_smoke.v1"
|
||||
cluster_id = $ClusterID
|
||||
entry_node_id = [string]$entryNode.id
|
||||
exit_node_id = [string]$exitNode.id
|
||||
channel_id = [string]$lease.channel_id
|
||||
passed = [bool](
|
||||
$backendLine.Contains($ExpectedBackendImage) -and
|
||||
$nodeLines.Contains($ExpectedNodeAgentImage) -and
|
||||
$firstStatus -eq 202 -and
|
||||
$acceptedBy -eq "introspection" -and
|
||||
[int]$accessTelemetry.reporting_node_count -ge 1 -and
|
||||
[int]$accessTelemetry.total_accepted -ge 2 -and
|
||||
[int]$accessTelemetry.introspection_accepted -ge 2
|
||||
)
|
||||
checks = [ordered]@{
|
||||
backend_expected_image_deployed = $backendLine.Contains($ExpectedBackendImage)
|
||||
node_agent_expected_image_deployed = $nodeLines.Contains($ExpectedNodeAgentImage)
|
||||
packet_accepted = ($firstStatus -eq 202)
|
||||
accepted_by_header_is_introspection = ($acceptedBy -eq "introspection")
|
||||
access_telemetry_reporting = ([int]$accessTelemetry.reporting_node_count -ge 1)
|
||||
accepted_count_recorded = ([int]$accessTelemetry.total_accepted -ge 2)
|
||||
introspection_count_recorded = ([int]$accessTelemetry.introspection_accepted -ge 2)
|
||||
}
|
||||
summary = [ordered]@{
|
||||
backend_container = $backendLine.Trim()
|
||||
node_containers = $nodeLines.Trim()
|
||||
packet_url = $packetUrl
|
||||
accepted_by = $acceptedBy
|
||||
telemetry_status = [string]$accessTelemetry.status
|
||||
reporting_nodes = [int]$accessTelemetry.reporting_node_count
|
||||
total_accepted = [int]$accessTelemetry.total_accepted
|
||||
signed_accepted = [int]$accessTelemetry.signed_accepted
|
||||
introspection_accepted = [int]$accessTelemetry.introspection_accepted
|
||||
backend_fallback_count = [int]$accessTelemetry.backend_fallback_count
|
||||
}
|
||||
}
|
||||
|
||||
$resultFullPath = Join-Path $repoRoot $ResultPath
|
||||
$resultDir = Split-Path -Parent $resultFullPath
|
||||
if (-not (Test-Path $resultDir)) {
|
||||
New-Item -ItemType Directory -Path $resultDir | Out-Null
|
||||
}
|
||||
$result | ConvertTo-Json -Depth 100 | Set-Content -Path $resultFullPath -Encoding UTF8
|
||||
|
||||
if (-not $result.passed) {
|
||||
throw "C18Z52 service-channel access telemetry smoke failed. Result: $resultFullPath"
|
||||
}
|
||||
|
||||
Write-Host "C18Z52 service-channel access telemetry smoke passed. Result: $resultFullPath"
|
||||
$result
|
||||
|
||||
|
||||
@@ -0,0 +1,131 @@
|
||||
param(
|
||||
[string]$ApiBaseUrl = "http://192.168.200.61:18121/api/v1",
|
||||
[string]$ClusterID = "cfc0743d-d960-49fb-9de8-96e063d5e4aa",
|
||||
[string]$ActorUserID = "f67d943f-5397-4b3a-a229-695fe67ad700",
|
||||
[string]$EntryNodeName = "test-1",
|
||||
[string]$ExitNodeName = "test-2",
|
||||
[string]$EntryBaseUrl = "http://192.168.200.61:19131",
|
||||
[string]$DockerSSH = "test-docker",
|
||||
[string]$ExpectedBackendImage = "rap-backend:fabric-service-channel-0.2.239",
|
||||
[string]$ExpectedNodeAgentImage = "rap-node-agent:0.2.237",
|
||||
[string]$ResultPath = "artifacts\c18z53-service-channel-access-correlation-smoke-result.json"
|
||||
)
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
$scriptDir = Split-Path -Parent $MyInvocation.MyCommand.Path
|
||||
$repoRoot = (Resolve-Path (Join-Path $scriptDir "..\..")).ProviderPath
|
||||
|
||||
function Invoke-ApiGet {
|
||||
param([string]$Path, [int]$TimeoutSec = 30)
|
||||
Invoke-RestMethod -Method Get -Uri "$ApiBaseUrl$Path" -TimeoutSec $TimeoutSec
|
||||
}
|
||||
|
||||
$nodes = @((Invoke-ApiGet -Path "/clusters/$ClusterID/nodes?actor_user_id=$ActorUserID").nodes)
|
||||
$entryNode = $nodes | Where-Object { $_.name -eq $EntryNodeName } | Select-Object -First 1
|
||||
$exitNode = $nodes | Where-Object { $_.name -eq $ExitNodeName } | Select-Object -First 1
|
||||
if ($null -eq $entryNode -or $null -eq $exitNode) {
|
||||
throw "Entry or exit node not found: $EntryNodeName / $ExitNodeName"
|
||||
}
|
||||
|
||||
$resourceID = "c18z53-vpn-smoke"
|
||||
$leaseBody = @{
|
||||
actor_user_id = $ActorUserID
|
||||
organization_id = "smoke-org"
|
||||
user_id = "smoke-user"
|
||||
resource_id = $resourceID
|
||||
service_class = "vpn_packets"
|
||||
entry_node_ids = @([string]$entryNode.id)
|
||||
exit_node_ids = @([string]$exitNode.id)
|
||||
preferred_entry_node_id = [string]$entryNode.id
|
||||
preferred_exit_node_id = [string]$exitNode.id
|
||||
allowed_channels = @("vpn_packet", "fabric_control")
|
||||
ttl_seconds = 180
|
||||
} | ConvertTo-Json -Depth 20
|
||||
$lease = (Invoke-RestMethod -Method Post -Uri "$ApiBaseUrl/clusters/$ClusterID/fabric/service-channels/leases" -ContentType "application/json" -Body $leaseBody -TimeoutSec 30).fabric_service_channel_lease
|
||||
|
||||
$packetPath = $lease.entry_http.path_template.
|
||||
Replace("{cluster_id}", $ClusterID).
|
||||
Replace("{channel_id}", [string]$lease.channel_id).
|
||||
Replace("{resource_id}", $resourceID)
|
||||
$packetUrl = $EntryBaseUrl.TrimEnd("/") + $packetPath
|
||||
$headers = @{
|
||||
"X-RAP-Service-Channel-Token" = [string]$lease.token.token
|
||||
"X-RAP-Fabric-Channel-ID" = [string]$lease.channel_id
|
||||
"X-RAP-Service-Class" = "vpn_packets"
|
||||
"X-RAP-Channel-Class" = "vpn_packet"
|
||||
}
|
||||
$response = Invoke-WebRequest -Method Post -Uri $packetUrl -Headers $headers -Body ([System.Text.Encoding]::UTF8.GetBytes("c18z53-access-correlation")) -ContentType "application/octet-stream" -TimeoutSec 30
|
||||
$acceptedBy = [string]$response.Headers["X-RAP-Service-Channel-Accepted-By"]
|
||||
|
||||
$accessTelemetry = $null
|
||||
$matchingChannel = $null
|
||||
for ($i = 0; $i -lt 8; $i++) {
|
||||
Start-Sleep -Seconds 3
|
||||
$accessTelemetry = (Invoke-ApiGet -Path "/clusters/$ClusterID/fabric/service-channels/access-telemetry?actor_user_id=$ActorUserID&limit=20").fabric_service_channel_access_telemetry
|
||||
$channels = @()
|
||||
if ($accessTelemetry.PSObject.Properties.Name -contains "active_channels") {
|
||||
$channels = @($accessTelemetry.active_channels)
|
||||
}
|
||||
$matchingChannel = $channels | Where-Object { $_.channel_id -eq $lease.channel_id } | Select-Object -First 1
|
||||
if ($null -ne $matchingChannel -and [int]$matchingChannel.entry_node_total_accepted -ge 1) {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
$backendLine = (& ssh $DockerSSH "docker ps --format '{{.Names}} {{.Image}} {{.Status}}' | grep '^rap_test_backend '") | Out-String
|
||||
$nodeLines = (& ssh $DockerSSH "docker ps --format '{{.Names}} {{.Image}} {{.Status}}' | grep '^rap_test_node_test_'") | Out-String
|
||||
|
||||
$result = [ordered]@{
|
||||
schema_version = "c18z53.service_channel_access_correlation_smoke.v1"
|
||||
cluster_id = $ClusterID
|
||||
channel_id = [string]$lease.channel_id
|
||||
passed = [bool](
|
||||
$backendLine.Contains($ExpectedBackendImage) -and
|
||||
$nodeLines.Contains($ExpectedNodeAgentImage) -and
|
||||
[int]$response.StatusCode -eq 202 -and
|
||||
$acceptedBy -eq "introspection" -and
|
||||
[int]$accessTelemetry.active_channel_count -ge 1 -and
|
||||
$null -ne $matchingChannel -and
|
||||
[string]$matchingChannel.selected_entry_node_id -eq [string]$entryNode.id -and
|
||||
[string]$matchingChannel.selected_exit_node_id -eq [string]$exitNode.id -and
|
||||
[int]$matchingChannel.entry_node_total_accepted -ge 1
|
||||
)
|
||||
checks = [ordered]@{
|
||||
backend_expected_image_deployed = $backendLine.Contains($ExpectedBackendImage)
|
||||
node_agent_expected_image_deployed = $nodeLines.Contains($ExpectedNodeAgentImage)
|
||||
packet_accepted = ([int]$response.StatusCode -eq 202)
|
||||
accepted_by_header_is_introspection = ($acceptedBy -eq "introspection")
|
||||
active_channel_visible = ([int]$accessTelemetry.active_channel_count -ge 1)
|
||||
matching_channel_visible = ($null -ne $matchingChannel)
|
||||
entry_exit_correlated = ($null -ne $matchingChannel -and [string]$matchingChannel.selected_entry_node_id -eq [string]$entryNode.id -and [string]$matchingChannel.selected_exit_node_id -eq [string]$exitNode.id)
|
||||
entry_access_correlated = ($null -ne $matchingChannel -and [int]$matchingChannel.entry_node_total_accepted -ge 1)
|
||||
}
|
||||
summary = [ordered]@{
|
||||
backend_container = $backendLine.Trim()
|
||||
node_containers = $nodeLines.Trim()
|
||||
accepted_by = $acceptedBy
|
||||
active_channel_count = [int]$accessTelemetry.active_channel_count
|
||||
degraded_fallback_channel_count = [int]$accessTelemetry.degraded_fallback_channel_count
|
||||
correlated_route_count = [int]$accessTelemetry.correlated_route_count
|
||||
degraded_route_count = [int]$accessTelemetry.degraded_route_count
|
||||
matching_channel = $matchingChannel
|
||||
}
|
||||
}
|
||||
|
||||
$resultFullPath = Join-Path $repoRoot $ResultPath
|
||||
$resultDir = Split-Path -Parent $resultFullPath
|
||||
if (-not (Test-Path $resultDir)) {
|
||||
New-Item -ItemType Directory -Path $resultDir | Out-Null
|
||||
}
|
||||
$result | ConvertTo-Json -Depth 100 | Set-Content -Path $resultFullPath -Encoding UTF8
|
||||
|
||||
if (-not $result.passed) {
|
||||
throw "C18Z53 service-channel access correlation smoke failed. Result: $resultFullPath"
|
||||
}
|
||||
|
||||
Write-Host "C18Z53 service-channel access correlation smoke passed. Result: $resultFullPath"
|
||||
$result
|
||||
|
||||
|
||||
@@ -0,0 +1,256 @@
|
||||
param(
|
||||
[string]$ApiBaseUrl = "http://192.168.200.61:18121/api/v1",
|
||||
[string]$ClusterID = "cfc0743d-d960-49fb-9de8-96e063d5e4aa",
|
||||
[string]$ActorUserID = "f67d943f-5397-4b3a-a229-695fe67ad700",
|
||||
[string]$EntryNodeName = "test-1",
|
||||
[string]$ExitNodeName = "test-2",
|
||||
[string]$EntryBaseUrl = "http://192.168.200.61:19131",
|
||||
[string]$DockerSSH = "test-docker",
|
||||
[string]$ExpectedBackendImage = "rap-backend:fabric-service-channel-0.2.239",
|
||||
[string]$ExpectedNodeAgentImage = "rap-node-agent:0.2.237",
|
||||
[string]$ResultPath = "artifacts\c18z54-service-channel-normal-route-access-smoke-result.json"
|
||||
)
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
$scriptDir = Split-Path -Parent $MyInvocation.MyCommand.Path
|
||||
$repoRoot = (Resolve-Path (Join-Path $scriptDir "..\..")).ProviderPath
|
||||
$runId = "c18z54-" + (Get-Date -Format "yyyyMMdd-HHmmss")
|
||||
|
||||
function Invoke-Api {
|
||||
param(
|
||||
[string]$Method,
|
||||
[string]$Path,
|
||||
[object]$Body = $null
|
||||
)
|
||||
if ($null -eq $Body) {
|
||||
return Invoke-RestMethod -Method $Method -Uri "$ApiBaseUrl$Path" -TimeoutSec 30
|
||||
}
|
||||
return Invoke-RestMethod -Method $Method -Uri "$ApiBaseUrl$Path" -ContentType "application/json" -Body ($Body | ConvertTo-Json -Depth 80) -TimeoutSec 30
|
||||
}
|
||||
|
||||
function Get-NodeByName {
|
||||
param([string]$Name)
|
||||
$nodes = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/nodes?actor_user_id=$ActorUserID").nodes
|
||||
$node = @($nodes | Where-Object { $_.name -eq $Name }) | Select-Object -First 1
|
||||
if ($null -eq $node) {
|
||||
throw "Node '$Name' was not found in cluster $ClusterID"
|
||||
}
|
||||
return $node
|
||||
}
|
||||
|
||||
function New-RouteIntent {
|
||||
param(
|
||||
[string]$SourceNodeID,
|
||||
[string]$DestinationNodeID
|
||||
)
|
||||
$expiresAt = (Get-Date).ToUniversalTime().AddMinutes(5).ToString("o")
|
||||
return Invoke-Api -Method POST -Path "/clusters/$ClusterID/mesh/route-intents" -Body @{
|
||||
actor_user_id = $ActorUserID
|
||||
source_selector = @{ node_id = $SourceNodeID }
|
||||
destination_selector = @{ node_id = $DestinationNodeID }
|
||||
service_class = "vpn_packets"
|
||||
priority = 2100000000
|
||||
policy = @{
|
||||
synthetic_enabled = $true
|
||||
route_version = "$runId-direct"
|
||||
policy_version = "$runId-direct"
|
||||
peer_directory_version = "$runId-direct"
|
||||
hops = @($SourceNodeID, $DestinationNodeID)
|
||||
allowed_channels = @("vpn_packet", "fabric_control")
|
||||
max_ttl = 8
|
||||
max_hops = 8
|
||||
expires_at = $expiresAt
|
||||
metadata = @{
|
||||
smoke = "c18z54_service_channel_normal_route_access"
|
||||
run_id = $runId
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function Send-QualityHeartbeat {
|
||||
param(
|
||||
[string]$EntryNodeID,
|
||||
[string]$RouteID
|
||||
)
|
||||
$observedAt = (Get-Date).ToUniversalTime().ToString("o")
|
||||
return Invoke-Api -Method POST -Path "/clusters/$ClusterID/nodes/$EntryNodeID/heartbeats" -Body @{
|
||||
health_status = "healthy"
|
||||
reported_version = "0.2.235"
|
||||
capabilities = @{
|
||||
fabric_service_channel_runtime = $true
|
||||
fabric_service_channel_route_manager = $true
|
||||
fabric_service_channel_route_quality_feedback = $true
|
||||
smoke_feedback_injection = "c18z54"
|
||||
}
|
||||
service_states = @{ smoke = "c18z54_normal_route_quality_feedback" }
|
||||
metadata = @{
|
||||
fabric_service_channel_runtime_report = @{
|
||||
schema_version = "c18l.fabric_service_channel_runtime_report.v1"
|
||||
config_version = "$runId-direct"
|
||||
cluster_id = $ClusterID
|
||||
local_node_id = $EntryNodeID
|
||||
observed_at = $observedAt
|
||||
ingress = @{
|
||||
flow_scheduler = @{
|
||||
schema_version = "rap.fabric_flow_scheduler.v1"
|
||||
service_neutral = $true
|
||||
service_mode = "application_protocol_agnostic"
|
||||
channel_stats = @{
|
||||
"c18z54-normal-route" = @{
|
||||
last_route_id = $RouteID
|
||||
route_generation = "$runId-direct"
|
||||
last_send_duration_ms = 11
|
||||
consecutive_failures = 0
|
||||
stall_count = 0
|
||||
route_rebuild_recommended = $false
|
||||
degraded_fallback_recommended = $false
|
||||
quality_window_sample_count = 6
|
||||
quality_window_success_count = 6
|
||||
quality_window_failure_count = 0
|
||||
quality_window_slow_count = 1
|
||||
quality_window_drop_count = 0
|
||||
quality_window_avg_latency_ms = 11
|
||||
quality_window_last_updated_at = $observedAt
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
smoke = @{
|
||||
name = "c18z54_service_channel_normal_route_access"
|
||||
run_id = $runId
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
$entryNode = Get-NodeByName -Name $EntryNodeName
|
||||
$exitNode = Get-NodeByName -Name $ExitNodeName
|
||||
$routeID = ""
|
||||
$result = $null
|
||||
|
||||
try {
|
||||
$routeIntent = (New-RouteIntent -SourceNodeID $entryNode.id -DestinationNodeID $exitNode.id).route_intent
|
||||
$routeID = [string]$routeIntent.id
|
||||
[void](Send-QualityHeartbeat -EntryNodeID $entryNode.id -RouteID $routeID)
|
||||
|
||||
$resourceID = "c18z54-vpn-smoke"
|
||||
$lease = (Invoke-Api -Method POST -Path "/clusters/$ClusterID/fabric/service-channels/leases" -Body @{
|
||||
actor_user_id = $ActorUserID
|
||||
organization_id = "smoke-org"
|
||||
user_id = "smoke-user"
|
||||
resource_id = $resourceID
|
||||
service_class = "vpn_packets"
|
||||
entry_node_ids = @([string]$entryNode.id)
|
||||
exit_node_ids = @([string]$exitNode.id)
|
||||
preferred_entry_node_id = [string]$entryNode.id
|
||||
preferred_exit_node_id = [string]$exitNode.id
|
||||
allowed_channels = @("vpn_packet", "fabric_control")
|
||||
ttl_seconds = 180
|
||||
metadata = @{
|
||||
smoke = "c18z54_service_channel_normal_route_access"
|
||||
run_id = $runId
|
||||
}
|
||||
}).fabric_service_channel_lease
|
||||
|
||||
$packetPath = $lease.entry_http.path_template.
|
||||
Replace("{cluster_id}", $ClusterID).
|
||||
Replace("{channel_id}", [string]$lease.channel_id).
|
||||
Replace("{resource_id}", $resourceID)
|
||||
$packetUrl = $EntryBaseUrl.TrimEnd("/") + $packetPath
|
||||
$headers = @{
|
||||
"X-RAP-Service-Channel-Token" = [string]$lease.token.token
|
||||
"X-RAP-Fabric-Channel-ID" = [string]$lease.channel_id
|
||||
"X-RAP-Service-Class" = "vpn_packets"
|
||||
"X-RAP-Channel-Class" = "vpn_packet"
|
||||
}
|
||||
$response = Invoke-WebRequest -Method Post -Uri $packetUrl -Headers $headers -Body ([System.Text.Encoding]::UTF8.GetBytes("c18z54-normal-route")) -ContentType "application/octet-stream" -TimeoutSec 30
|
||||
$acceptedBy = [string]$response.Headers["X-RAP-Service-Channel-Accepted-By"]
|
||||
|
||||
$accessTelemetry = $null
|
||||
$matchingChannel = $null
|
||||
for ($i = 0; $i -lt 10; $i++) {
|
||||
Start-Sleep -Seconds 3
|
||||
$accessTelemetry = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/fabric/service-channels/access-telemetry?actor_user_id=$ActorUserID&limit=20").fabric_service_channel_access_telemetry
|
||||
$channels = @()
|
||||
if ($accessTelemetry.PSObject.Properties.Name -contains "active_channels") {
|
||||
$channels = @($accessTelemetry.active_channels)
|
||||
}
|
||||
$matchingChannel = $channels | Where-Object { $_.channel_id -eq $lease.channel_id } | Select-Object -First 1
|
||||
if ($null -ne $matchingChannel -and [string]$matchingChannel.route_feedback_status -eq "healthy" -and [int]$matchingChannel.route_quality_window_sample_count -ge 6) {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
$backendLine = (& ssh $DockerSSH "docker ps --format '{{.Names}} {{.Image}} {{.Status}}' | grep '^rap_test_backend '") | Out-String
|
||||
$nodeLines = (& ssh $DockerSSH "docker ps --format '{{.Names}} {{.Image}} {{.Status}}' | grep '^rap_test_node_test_'") | Out-String
|
||||
|
||||
$result = [ordered]@{
|
||||
schema_version = "c18z54.service_channel_normal_route_access_smoke.v1"
|
||||
run_id = $runId
|
||||
cluster_id = $ClusterID
|
||||
route_id = $routeID
|
||||
channel_id = [string]$lease.channel_id
|
||||
passed = [bool](
|
||||
$backendLine.Contains($ExpectedBackendImage) -and
|
||||
$nodeLines.Contains($ExpectedNodeAgentImage) -and
|
||||
[string]$lease.status -eq "ready" -and
|
||||
[string]$lease.primary_route.route_id -eq $routeID -and
|
||||
[int]$response.StatusCode -eq 202 -and
|
||||
$acceptedBy -eq "introspection" -and
|
||||
$null -ne $matchingChannel -and
|
||||
[string]$matchingChannel.primary_route_id -eq $routeID -and
|
||||
-not [bool]$matchingChannel.force_backend_fallback -and
|
||||
[string]$matchingChannel.route_feedback_status -eq "healthy" -and
|
||||
[int]$matchingChannel.route_quality_window_sample_count -ge 6
|
||||
)
|
||||
checks = [ordered]@{
|
||||
backend_expected_image_deployed = $backendLine.Contains($ExpectedBackendImage)
|
||||
node_agent_expected_image_deployed = $nodeLines.Contains($ExpectedNodeAgentImage)
|
||||
lease_ready = ([string]$lease.status -eq "ready")
|
||||
lease_selected_normal_route = ([string]$lease.primary_route.route_id -eq $routeID)
|
||||
packet_accepted = ([int]$response.StatusCode -eq 202)
|
||||
accepted_by_header_is_introspection = ($acceptedBy -eq "introspection")
|
||||
active_channel_visible = ($null -ne $matchingChannel)
|
||||
active_channel_uses_primary_route = ($null -ne $matchingChannel -and [string]$matchingChannel.primary_route_id -eq $routeID)
|
||||
active_channel_not_backend_fallback = ($null -ne $matchingChannel -and -not [bool]$matchingChannel.force_backend_fallback)
|
||||
route_quality_correlated = ($null -ne $matchingChannel -and [string]$matchingChannel.route_feedback_status -eq "healthy" -and [int]$matchingChannel.route_quality_window_sample_count -ge 6)
|
||||
}
|
||||
summary = [ordered]@{
|
||||
backend_container = $backendLine.Trim()
|
||||
node_containers = $nodeLines.Trim()
|
||||
accepted_by = $acceptedBy
|
||||
lease_status = [string]$lease.status
|
||||
lease_primary_route_id = [string]$lease.primary_route.route_id
|
||||
access_status = [string]$accessTelemetry.status
|
||||
active_channel_count = [int]$accessTelemetry.active_channel_count
|
||||
correlated_route_count = [int]$accessTelemetry.correlated_route_count
|
||||
degraded_route_count = [int]$accessTelemetry.degraded_route_count
|
||||
matching_channel = $matchingChannel
|
||||
}
|
||||
}
|
||||
$failedChecks = @($result.checks.GetEnumerator() | Where-Object { $_.Value -ne $true })
|
||||
if ($failedChecks.Count -gt 0) {
|
||||
throw "C18Z54 failed checks: $($failedChecks.Name -join ', ')"
|
||||
}
|
||||
}
|
||||
finally {
|
||||
if ($routeID) {
|
||||
try { Invoke-Api -Method POST -Path "/clusters/$ClusterID/mesh/route-intents/$routeID/expire" -Body @{ actor_user_id = $ActorUserID } | Out-Null } catch {}
|
||||
}
|
||||
}
|
||||
|
||||
$resultFullPath = Join-Path $repoRoot $ResultPath
|
||||
$resultDir = Split-Path -Parent $resultFullPath
|
||||
if (-not (Test-Path $resultDir)) {
|
||||
New-Item -ItemType Directory -Path $resultDir | Out-Null
|
||||
}
|
||||
$result | ConvertTo-Json -Depth 100 | Set-Content -Path $resultFullPath -Encoding UTF8
|
||||
|
||||
Write-Host "C18Z54 service-channel normal route access smoke passed. Result: $resultFullPath"
|
||||
$result
|
||||
|
||||
|
||||
@@ -0,0 +1,283 @@
|
||||
param(
|
||||
[string]$ApiBaseUrl = "http://192.168.200.61:18121/api/v1",
|
||||
[string]$ClusterID = "cfc0743d-d960-49fb-9de8-96e063d5e4aa",
|
||||
[string]$ActorUserID = "f67d943f-5397-4b3a-a229-695fe67ad700",
|
||||
[string]$EntryNodeName = "test-1",
|
||||
[string]$ExitNodeName = "test-2",
|
||||
[string]$EntryBaseUrl = "http://192.168.200.61:19131",
|
||||
[string]$DockerSSH = "test-docker",
|
||||
[string]$ExpectedBackendImage = "rap-backend:fabric-service-channel-0.2.239",
|
||||
[string]$ExpectedNodeAgentImage = "rap-node-agent:0.2.237",
|
||||
[string]$ResultPath = "artifacts\c18z55-service-channel-degraded-route-access-smoke-result.json"
|
||||
)
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
$scriptDir = Split-Path -Parent $MyInvocation.MyCommand.Path
|
||||
$repoRoot = (Resolve-Path (Join-Path $scriptDir "..\..")).ProviderPath
|
||||
$runId = "c18z55-" + (Get-Date -Format "yyyyMMdd-HHmmss")
|
||||
|
||||
function Invoke-Api {
|
||||
param(
|
||||
[string]$Method,
|
||||
[string]$Path,
|
||||
[object]$Body = $null
|
||||
)
|
||||
if ($null -eq $Body) {
|
||||
return Invoke-RestMethod -Method $Method -Uri "$ApiBaseUrl$Path" -TimeoutSec 30
|
||||
}
|
||||
return Invoke-RestMethod -Method $Method -Uri "$ApiBaseUrl$Path" -ContentType "application/json" -Body ($Body | ConvertTo-Json -Depth 80) -TimeoutSec 30
|
||||
}
|
||||
|
||||
function Get-NodeByName {
|
||||
param([string]$Name)
|
||||
$nodes = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/nodes?actor_user_id=$ActorUserID").nodes
|
||||
$node = @($nodes | Where-Object { $_.name -eq $Name }) | Select-Object -First 1
|
||||
if ($null -eq $node) {
|
||||
throw "Node '$Name' was not found in cluster $ClusterID"
|
||||
}
|
||||
return $node
|
||||
}
|
||||
|
||||
function Get-PropertyValue {
|
||||
param(
|
||||
[object]$Item,
|
||||
[string]$Name,
|
||||
[object]$Default = $null
|
||||
)
|
||||
if ($null -eq $Item) {
|
||||
return $Default
|
||||
}
|
||||
$property = $Item.PSObject.Properties[$Name]
|
||||
if ($null -eq $property) {
|
||||
return $Default
|
||||
}
|
||||
return $property.Value
|
||||
}
|
||||
|
||||
function New-RouteIntent {
|
||||
param(
|
||||
[string]$SourceNodeID,
|
||||
[string]$DestinationNodeID
|
||||
)
|
||||
$expiresAt = (Get-Date).ToUniversalTime().AddMinutes(5).ToString("o")
|
||||
return Invoke-Api -Method POST -Path "/clusters/$ClusterID/mesh/route-intents" -Body @{
|
||||
actor_user_id = $ActorUserID
|
||||
source_selector = @{ node_id = $SourceNodeID }
|
||||
destination_selector = @{ node_id = $DestinationNodeID }
|
||||
service_class = "vpn_packets"
|
||||
priority = 2100000000
|
||||
policy = @{
|
||||
synthetic_enabled = $true
|
||||
route_version = "$runId-direct"
|
||||
policy_version = "$runId-direct"
|
||||
peer_directory_version = "$runId-direct"
|
||||
hops = @($SourceNodeID, $DestinationNodeID)
|
||||
allowed_channels = @("vpn_packet", "fabric_control")
|
||||
max_ttl = 8
|
||||
max_hops = 8
|
||||
expires_at = $expiresAt
|
||||
metadata = @{
|
||||
smoke = "c18z55_service_channel_degraded_route_access"
|
||||
run_id = $runId
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function Send-DegradedHeartbeat {
|
||||
param(
|
||||
[string]$EntryNodeID,
|
||||
[string]$RouteID
|
||||
)
|
||||
$observedAt = (Get-Date).ToUniversalTime().ToString("o")
|
||||
return Invoke-Api -Method POST -Path "/clusters/$ClusterID/nodes/$EntryNodeID/heartbeats" -Body @{
|
||||
health_status = "healthy"
|
||||
reported_version = "0.2.235"
|
||||
capabilities = @{
|
||||
fabric_service_channel_runtime = $true
|
||||
fabric_service_channel_route_manager = $true
|
||||
fabric_service_channel_route_quality_feedback = $true
|
||||
smoke_feedback_injection = "c18z55"
|
||||
}
|
||||
service_states = @{ smoke = "c18z55_degraded_route_quality_feedback" }
|
||||
metadata = @{
|
||||
fabric_service_channel_runtime_report = @{
|
||||
schema_version = "c18l.fabric_service_channel_runtime_report.v1"
|
||||
config_version = "$runId-direct"
|
||||
cluster_id = $ClusterID
|
||||
local_node_id = $EntryNodeID
|
||||
observed_at = $observedAt
|
||||
ingress = @{
|
||||
flow_scheduler = @{
|
||||
schema_version = "rap.fabric_flow_scheduler.v1"
|
||||
service_neutral = $true
|
||||
service_mode = "application_protocol_agnostic"
|
||||
channel_stats = @{
|
||||
"c18z55-degraded-route" = @{
|
||||
last_route_id = $RouteID
|
||||
last_failed_route_id = $RouteID
|
||||
route_generation = "$runId-direct"
|
||||
last_error = "c18z55 forced degraded normal route"
|
||||
last_send_duration_ms = 1250
|
||||
consecutive_failures = 3
|
||||
stall_count = 2
|
||||
route_rebuild_recommended = $true
|
||||
degraded_fallback_recommended = $false
|
||||
quality_window_sample_count = 7
|
||||
quality_window_success_count = 1
|
||||
quality_window_failure_count = 3
|
||||
quality_window_slow_count = 2
|
||||
quality_window_drop_count = 1
|
||||
quality_window_avg_latency_ms = 1250
|
||||
quality_window_last_updated_at = $observedAt
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
smoke = @{
|
||||
name = "c18z55_service_channel_degraded_route_access"
|
||||
run_id = $runId
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
$entryNode = Get-NodeByName -Name $EntryNodeName
|
||||
$exitNode = Get-NodeByName -Name $ExitNodeName
|
||||
$routeID = ""
|
||||
$result = $null
|
||||
|
||||
try {
|
||||
$routeIntent = (New-RouteIntent -SourceNodeID $entryNode.id -DestinationNodeID $exitNode.id).route_intent
|
||||
$routeID = [string]$routeIntent.id
|
||||
|
||||
$resourceID = "c18z55-vpn-smoke"
|
||||
$lease = (Invoke-Api -Method POST -Path "/clusters/$ClusterID/fabric/service-channels/leases" -Body @{
|
||||
actor_user_id = $ActorUserID
|
||||
organization_id = "smoke-org"
|
||||
user_id = "smoke-user"
|
||||
resource_id = $resourceID
|
||||
service_class = "vpn_packets"
|
||||
entry_node_ids = @([string]$entryNode.id)
|
||||
exit_node_ids = @([string]$exitNode.id)
|
||||
preferred_entry_node_id = [string]$entryNode.id
|
||||
preferred_exit_node_id = [string]$exitNode.id
|
||||
allowed_channels = @("vpn_packet", "fabric_control")
|
||||
ttl_seconds = 180
|
||||
metadata = @{
|
||||
smoke = "c18z55_service_channel_degraded_route_access"
|
||||
run_id = $runId
|
||||
}
|
||||
}).fabric_service_channel_lease
|
||||
[void](Send-DegradedHeartbeat -EntryNodeID $entryNode.id -RouteID $routeID)
|
||||
|
||||
$packetPath = $lease.entry_http.path_template.
|
||||
Replace("{cluster_id}", $ClusterID).
|
||||
Replace("{channel_id}", [string]$lease.channel_id).
|
||||
Replace("{resource_id}", $resourceID)
|
||||
$packetUrl = $EntryBaseUrl.TrimEnd("/") + $packetPath
|
||||
$headers = @{
|
||||
"X-RAP-Service-Channel-Token" = [string]$lease.token.token
|
||||
"X-RAP-Fabric-Channel-ID" = [string]$lease.channel_id
|
||||
"X-RAP-Service-Class" = "vpn_packets"
|
||||
"X-RAP-Channel-Class" = "vpn_packet"
|
||||
}
|
||||
$response = Invoke-WebRequest -Method Post -Uri $packetUrl -Headers $headers -Body ([System.Text.Encoding]::UTF8.GetBytes("c18z55-degraded-route")) -ContentType "application/octet-stream" -TimeoutSec 30
|
||||
$acceptedBy = [string]$response.Headers["X-RAP-Service-Channel-Accepted-By"]
|
||||
|
||||
$accessTelemetry = $null
|
||||
$matchingChannel = $null
|
||||
for ($i = 0; $i -lt 10; $i++) {
|
||||
Start-Sleep -Seconds 3
|
||||
$accessTelemetry = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/fabric/service-channels/access-telemetry?actor_user_id=$ActorUserID&limit=20").fabric_service_channel_access_telemetry
|
||||
$channels = @()
|
||||
if ($accessTelemetry.PSObject.Properties.Name -contains "active_channels") {
|
||||
$channels = @($accessTelemetry.active_channels)
|
||||
}
|
||||
$matchingChannel = $channels | Where-Object { $_.channel_id -eq $lease.channel_id } | Select-Object -First 1
|
||||
$routeFeedbackStatus = [string](Get-PropertyValue -Item $matchingChannel -Name "route_feedback_status" -Default "")
|
||||
if ($null -ne $matchingChannel -and @("degraded", "fenced") -contains $routeFeedbackStatus -and [int]$accessTelemetry.degraded_route_count -ge 1) {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
$backendLine = (& ssh $DockerSSH "docker ps --format '{{.Names}} {{.Image}} {{.Status}}' | grep '^rap_test_backend '") | Out-String
|
||||
$nodeLines = (& ssh $DockerSSH "docker ps --format '{{.Names}} {{.Image}} {{.Status}}' | grep '^rap_test_node_test_'") | Out-String
|
||||
$leasePrimaryRouteID = [string](Get-PropertyValue -Item (Get-PropertyValue -Item $lease -Name "primary_route" -Default $null) -Name "route_id" -Default "")
|
||||
|
||||
$result = [ordered]@{
|
||||
schema_version = "c18z55.service_channel_degraded_route_access_smoke.v1"
|
||||
run_id = $runId
|
||||
cluster_id = $ClusterID
|
||||
route_id = $routeID
|
||||
channel_id = [string]$lease.channel_id
|
||||
passed = [bool](
|
||||
$backendLine.Contains($ExpectedBackendImage) -and
|
||||
$nodeLines.Contains($ExpectedNodeAgentImage) -and
|
||||
[string]$lease.status -eq "ready" -and
|
||||
$leasePrimaryRouteID -eq $routeID -and
|
||||
[int]$response.StatusCode -eq 202 -and
|
||||
$acceptedBy -eq "introspection" -and
|
||||
$null -ne $matchingChannel -and
|
||||
[string](Get-PropertyValue -Item $matchingChannel -Name "primary_route_id" -Default "") -eq $routeID -and
|
||||
-not [bool](Get-PropertyValue -Item $matchingChannel -Name "force_backend_fallback" -Default $false) -and
|
||||
@("degraded", "fenced") -contains [string](Get-PropertyValue -Item $matchingChannel -Name "route_feedback_status" -Default "") -and
|
||||
[int](Get-PropertyValue -Item $matchingChannel -Name "route_quality_window_failure_count" -Default 0) -ge 3 -and
|
||||
[int](Get-PropertyValue -Item $matchingChannel -Name "route_quality_window_drop_count" -Default 0) -ge 1 -and
|
||||
[string](Get-PropertyValue -Item $matchingChannel -Name "remediation_action" -Default "") -eq "rebuild_route" -and
|
||||
[int]$accessTelemetry.degraded_route_count -ge 1
|
||||
)
|
||||
checks = [ordered]@{
|
||||
backend_expected_image_deployed = $backendLine.Contains($ExpectedBackendImage)
|
||||
node_agent_expected_image_deployed = $nodeLines.Contains($ExpectedNodeAgentImage)
|
||||
lease_ready = ([string]$lease.status -eq "ready")
|
||||
lease_selected_normal_route = ($leasePrimaryRouteID -eq $routeID)
|
||||
packet_accepted = ([int]$response.StatusCode -eq 202)
|
||||
accepted_by_header_is_introspection = ($acceptedBy -eq "introspection")
|
||||
active_channel_visible = ($null -ne $matchingChannel)
|
||||
active_channel_uses_primary_route = ($null -ne $matchingChannel -and [string](Get-PropertyValue -Item $matchingChannel -Name "primary_route_id" -Default "") -eq $routeID)
|
||||
active_channel_not_backend_fallback = ($null -ne $matchingChannel -and -not [bool](Get-PropertyValue -Item $matchingChannel -Name "force_backend_fallback" -Default $false))
|
||||
route_quality_degraded = ($null -ne $matchingChannel -and @("degraded", "fenced") -contains [string](Get-PropertyValue -Item $matchingChannel -Name "route_feedback_status" -Default ""))
|
||||
route_quality_counters_visible = ($null -ne $matchingChannel -and [int](Get-PropertyValue -Item $matchingChannel -Name "route_quality_window_failure_count" -Default 0) -ge 3 -and [int](Get-PropertyValue -Item $matchingChannel -Name "route_quality_window_drop_count" -Default 0) -ge 1)
|
||||
remediation_recommends_rebuild = ($null -ne $matchingChannel -and [string](Get-PropertyValue -Item $matchingChannel -Name "remediation_action" -Default "") -eq "rebuild_route")
|
||||
access_marks_degraded_route = ([int]$accessTelemetry.degraded_route_count -ge 1)
|
||||
}
|
||||
summary = [ordered]@{
|
||||
backend_container = $backendLine.Trim()
|
||||
node_containers = $nodeLines.Trim()
|
||||
accepted_by = $acceptedBy
|
||||
lease_status = [string]$lease.status
|
||||
lease_primary_route_id = $leasePrimaryRouteID
|
||||
access_status = [string]$accessTelemetry.status
|
||||
active_channel_count = [int]$accessTelemetry.active_channel_count
|
||||
correlated_route_count = [int]$accessTelemetry.correlated_route_count
|
||||
degraded_route_count = [int]$accessTelemetry.degraded_route_count
|
||||
degraded_fallback_channel_count = [int]$accessTelemetry.degraded_fallback_channel_count
|
||||
matching_channel = $matchingChannel
|
||||
}
|
||||
}
|
||||
$failedChecks = @($result.checks.GetEnumerator() | Where-Object { $_.Value -ne $true })
|
||||
if ($failedChecks.Count -gt 0) {
|
||||
throw "C18Z55 failed checks: $($failedChecks.Name -join ', ')"
|
||||
}
|
||||
}
|
||||
finally {
|
||||
if ($routeID) {
|
||||
try { Invoke-Api -Method POST -Path "/clusters/$ClusterID/mesh/route-intents/$routeID/expire" -Body @{ actor_user_id = $ActorUserID } | Out-Null } catch {}
|
||||
}
|
||||
}
|
||||
|
||||
$resultFullPath = Join-Path $repoRoot $ResultPath
|
||||
$resultDir = Split-Path -Parent $resultFullPath
|
||||
if (-not (Test-Path $resultDir)) {
|
||||
New-Item -ItemType Directory -Path $resultDir | Out-Null
|
||||
}
|
||||
$result | ConvertTo-Json -Depth 100 | Set-Content -Path $resultFullPath -Encoding UTF8
|
||||
|
||||
Write-Host "C18Z55 service-channel degraded route access smoke passed. Result: $resultFullPath"
|
||||
$result
|
||||
|
||||
|
||||
@@ -0,0 +1,298 @@
|
||||
param(
|
||||
[string]$ApiBaseUrl = "http://192.168.200.61:18121/api/v1",
|
||||
[string]$ClusterID = "cfc0743d-d960-49fb-9de8-96e063d5e4aa",
|
||||
[string]$ActorUserID = "f67d943f-5397-4b3a-a229-695fe67ad700",
|
||||
[string]$EntryNodeName = "test-1",
|
||||
[string]$RelayNodeName = "test-3",
|
||||
[string]$ExitNodeName = "test-2",
|
||||
[string]$EntryBaseUrl = "http://192.168.200.61:19131",
|
||||
[string]$DockerSSH = "test-docker",
|
||||
[string]$ExpectedBackendImage = "rap-backend:fabric-service-channel-0.2.239",
|
||||
[string]$ExpectedNodeAgentImage = "rap-node-agent:0.2.237",
|
||||
[string]$ResultPath = "artifacts\c18z56-service-channel-alternate-remediation-smoke-result.json"
|
||||
)
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
$scriptDir = Split-Path -Parent $MyInvocation.MyCommand.Path
|
||||
$repoRoot = (Resolve-Path (Join-Path $scriptDir "..\..")).ProviderPath
|
||||
$runId = "c18z56-" + (Get-Date -Format "yyyyMMdd-HHmmss")
|
||||
|
||||
function Invoke-Api {
|
||||
param(
|
||||
[string]$Method,
|
||||
[string]$Path,
|
||||
[object]$Body = $null
|
||||
)
|
||||
if ($null -eq $Body) {
|
||||
return Invoke-RestMethod -Method $Method -Uri "$ApiBaseUrl$Path" -TimeoutSec 30
|
||||
}
|
||||
return Invoke-RestMethod -Method $Method -Uri "$ApiBaseUrl$Path" -ContentType "application/json" -Body ($Body | ConvertTo-Json -Depth 80) -TimeoutSec 30
|
||||
}
|
||||
|
||||
function Get-PropertyValue {
|
||||
param(
|
||||
[object]$Item,
|
||||
[string]$Name,
|
||||
[object]$Default = $null
|
||||
)
|
||||
if ($null -eq $Item) { return $Default }
|
||||
$property = $Item.PSObject.Properties[$Name]
|
||||
if ($null -eq $property) { return $Default }
|
||||
return $property.Value
|
||||
}
|
||||
|
||||
function Get-NodeByName {
|
||||
param([string]$Name)
|
||||
$nodes = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/nodes?actor_user_id=$ActorUserID").nodes
|
||||
$node = @($nodes | Where-Object { $_.name -eq $Name }) | Select-Object -First 1
|
||||
if ($null -eq $node) {
|
||||
throw "Node '$Name' was not found in cluster $ClusterID"
|
||||
}
|
||||
return $node
|
||||
}
|
||||
|
||||
function New-RouteIntent {
|
||||
param(
|
||||
[string]$SourceNodeID,
|
||||
[string]$DestinationNodeID,
|
||||
[string[]]$Hops,
|
||||
[int]$Priority,
|
||||
[string]$Label
|
||||
)
|
||||
$expiresAt = (Get-Date).ToUniversalTime().AddMinutes(5).ToString("o")
|
||||
return Invoke-Api -Method POST -Path "/clusters/$ClusterID/mesh/route-intents" -Body @{
|
||||
actor_user_id = $ActorUserID
|
||||
source_selector = @{ node_id = $SourceNodeID }
|
||||
destination_selector = @{ node_id = $DestinationNodeID }
|
||||
service_class = "vpn_packets"
|
||||
priority = $Priority
|
||||
policy = @{
|
||||
synthetic_enabled = $true
|
||||
route_version = "$runId-$Label"
|
||||
policy_version = "$runId-$Label"
|
||||
peer_directory_version = "$runId-$Label"
|
||||
hops = @($Hops)
|
||||
allowed_channels = @("vpn_packet", "fabric_control")
|
||||
max_ttl = 8
|
||||
max_hops = 8
|
||||
expires_at = $expiresAt
|
||||
metadata = @{
|
||||
smoke = "c18z56_service_channel_alternate_remediation"
|
||||
run_id = $runId
|
||||
label = $Label
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function Send-DegradedHeartbeat {
|
||||
param(
|
||||
[string]$EntryNodeID,
|
||||
[string]$PrimaryRouteID
|
||||
)
|
||||
$observedAt = (Get-Date).ToUniversalTime().ToString("o")
|
||||
return Invoke-Api -Method POST -Path "/clusters/$ClusterID/nodes/$EntryNodeID/heartbeats" -Body @{
|
||||
health_status = "healthy"
|
||||
reported_version = "0.2.235"
|
||||
capabilities = @{
|
||||
fabric_service_channel_runtime = $true
|
||||
fabric_service_channel_route_manager = $true
|
||||
fabric_service_channel_route_quality_feedback = $true
|
||||
smoke_feedback_injection = "c18z56"
|
||||
}
|
||||
service_states = @{ smoke = "c18z56_primary_degraded_alternate_available" }
|
||||
metadata = @{
|
||||
fabric_service_channel_runtime_report = @{
|
||||
schema_version = "c18l.fabric_service_channel_runtime_report.v1"
|
||||
config_version = "$runId-primary"
|
||||
cluster_id = $ClusterID
|
||||
local_node_id = $EntryNodeID
|
||||
observed_at = $observedAt
|
||||
ingress = @{
|
||||
flow_scheduler = @{
|
||||
schema_version = "rap.fabric_flow_scheduler.v1"
|
||||
service_neutral = $true
|
||||
service_mode = "application_protocol_agnostic"
|
||||
channel_stats = @{
|
||||
"c18z56-primary-degraded" = @{
|
||||
last_route_id = $PrimaryRouteID
|
||||
last_failed_route_id = $PrimaryRouteID
|
||||
route_generation = "$runId-primary"
|
||||
last_error = "c18z56 primary route degraded; alternate available"
|
||||
last_send_duration_ms = 980
|
||||
consecutive_failures = 3
|
||||
stall_count = 2
|
||||
route_rebuild_recommended = $true
|
||||
degraded_fallback_recommended = $false
|
||||
quality_window_sample_count = 8
|
||||
quality_window_success_count = 2
|
||||
quality_window_failure_count = 3
|
||||
quality_window_slow_count = 2
|
||||
quality_window_drop_count = 1
|
||||
quality_window_avg_latency_ms = 980
|
||||
quality_window_last_updated_at = $observedAt
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
smoke = @{
|
||||
name = "c18z56_service_channel_alternate_remediation"
|
||||
run_id = $runId
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
$entryNode = Get-NodeByName -Name $EntryNodeName
|
||||
$relayNode = Get-NodeByName -Name $RelayNodeName
|
||||
$exitNode = Get-NodeByName -Name $ExitNodeName
|
||||
$primaryRouteID = ""
|
||||
$alternateRouteID = ""
|
||||
$result = $null
|
||||
|
||||
try {
|
||||
$primary = (New-RouteIntent -SourceNodeID $entryNode.id -DestinationNodeID $exitNode.id -Hops @($entryNode.id, $exitNode.id) -Priority 2100000000 -Label "primary").route_intent
|
||||
$alternate = (New-RouteIntent -SourceNodeID $entryNode.id -DestinationNodeID $exitNode.id -Hops @($entryNode.id, $relayNode.id, $exitNode.id) -Priority 2099999900 -Label "alternate").route_intent
|
||||
$primaryRouteID = [string]$primary.id
|
||||
$alternateRouteID = [string]$alternate.id
|
||||
|
||||
$resourceID = "c18z56-vpn-smoke"
|
||||
$lease = (Invoke-Api -Method POST -Path "/clusters/$ClusterID/fabric/service-channels/leases" -Body @{
|
||||
actor_user_id = $ActorUserID
|
||||
organization_id = "smoke-org"
|
||||
user_id = "smoke-user"
|
||||
resource_id = $resourceID
|
||||
service_class = "vpn_packets"
|
||||
entry_node_ids = @([string]$entryNode.id)
|
||||
exit_node_ids = @([string]$exitNode.id)
|
||||
preferred_entry_node_id = [string]$entryNode.id
|
||||
preferred_exit_node_id = [string]$exitNode.id
|
||||
allowed_channels = @("vpn_packet", "fabric_control")
|
||||
ttl_seconds = 180
|
||||
metadata = @{
|
||||
smoke = "c18z56_service_channel_alternate_remediation"
|
||||
run_id = $runId
|
||||
}
|
||||
}).fabric_service_channel_lease
|
||||
|
||||
[void](Send-DegradedHeartbeat -EntryNodeID $entryNode.id -PrimaryRouteID $primaryRouteID)
|
||||
|
||||
$packetPath = $lease.entry_http.path_template.
|
||||
Replace("{cluster_id}", $ClusterID).
|
||||
Replace("{channel_id}", [string]$lease.channel_id).
|
||||
Replace("{resource_id}", $resourceID)
|
||||
$packetUrl = $EntryBaseUrl.TrimEnd("/") + $packetPath
|
||||
$headers = @{
|
||||
"X-RAP-Service-Channel-Token" = [string]$lease.token.token
|
||||
"X-RAP-Fabric-Channel-ID" = [string]$lease.channel_id
|
||||
"X-RAP-Service-Class" = "vpn_packets"
|
||||
"X-RAP-Channel-Class" = "vpn_packet"
|
||||
}
|
||||
$response = Invoke-WebRequest -Method Post -Uri $packetUrl -Headers $headers -Body ([System.Text.Encoding]::UTF8.GetBytes("c18z56-alternate-remediation")) -ContentType "application/octet-stream" -TimeoutSec 30
|
||||
$acceptedBy = [string]$response.Headers["X-RAP-Service-Channel-Accepted-By"]
|
||||
|
||||
$accessTelemetry = $null
|
||||
$matchingChannel = $null
|
||||
for ($i = 0; $i -lt 10; $i++) {
|
||||
Start-Sleep -Seconds 3
|
||||
$accessTelemetry = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/fabric/service-channels/access-telemetry?actor_user_id=$ActorUserID&limit=20").fabric_service_channel_access_telemetry
|
||||
$channels = @()
|
||||
if ($accessTelemetry.PSObject.Properties.Name -contains "active_channels") {
|
||||
$channels = @($accessTelemetry.active_channels)
|
||||
}
|
||||
$matchingChannel = $channels | Where-Object { $_.channel_id -eq $lease.channel_id } | Select-Object -First 1
|
||||
if ($null -ne $matchingChannel -and [string](Get-PropertyValue -Item $matchingChannel -Name "remediation_action" -Default "") -eq "prefer_alternate_route") {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
$backendLine = (& ssh $DockerSSH "docker ps --format '{{.Names}} {{.Image}} {{.Status}}' | grep '^rap_test_backend '") | Out-String
|
||||
$nodeLines = (& ssh $DockerSSH "docker ps --format '{{.Names}} {{.Image}} {{.Status}}' | grep '^rap_test_node_test_'") | Out-String
|
||||
$leasePrimaryRouteID = [string](Get-PropertyValue -Item (Get-PropertyValue -Item $lease -Name "primary_route" -Default $null) -Name "route_id" -Default "")
|
||||
$leaseAlternates = @()
|
||||
if ($lease.PSObject.Properties.Name -contains "alternate_routes") {
|
||||
$leaseAlternates = @($lease.alternate_routes)
|
||||
}
|
||||
$leaseHasAlternate = (@($leaseAlternates | Where-Object { [string]$_.route_id -eq $alternateRouteID }).Count -ge 1)
|
||||
|
||||
$result = [ordered]@{
|
||||
schema_version = "c18z56.service_channel_alternate_remediation_smoke.v1"
|
||||
run_id = $runId
|
||||
cluster_id = $ClusterID
|
||||
primary_route_id = $primaryRouteID
|
||||
alternate_route_id = $alternateRouteID
|
||||
channel_id = [string]$lease.channel_id
|
||||
passed = [bool](
|
||||
$backendLine.Contains($ExpectedBackendImage) -and
|
||||
$nodeLines.Contains($ExpectedNodeAgentImage) -and
|
||||
[string]$lease.status -eq "ready" -and
|
||||
$leasePrimaryRouteID -eq $primaryRouteID -and
|
||||
$leaseHasAlternate -and
|
||||
[int]$response.StatusCode -eq 202 -and
|
||||
$acceptedBy -eq "introspection" -and
|
||||
$null -ne $matchingChannel -and
|
||||
[string](Get-PropertyValue -Item $matchingChannel -Name "primary_route_id" -Default "") -eq $primaryRouteID -and
|
||||
-not [bool](Get-PropertyValue -Item $matchingChannel -Name "force_backend_fallback" -Default $false) -and
|
||||
[string](Get-PropertyValue -Item $matchingChannel -Name "route_feedback_status" -Default "") -eq "fenced" -and
|
||||
[string](Get-PropertyValue -Item $matchingChannel -Name "remediation_action" -Default "") -eq "prefer_alternate_route" -and
|
||||
[string](Get-PropertyValue -Item $matchingChannel -Name "remediation_route_id" -Default "") -eq $alternateRouteID -and
|
||||
[int]$accessTelemetry.degraded_fallback_channel_count -eq 0
|
||||
)
|
||||
checks = [ordered]@{
|
||||
backend_expected_image_deployed = $backendLine.Contains($ExpectedBackendImage)
|
||||
node_agent_expected_image_deployed = $nodeLines.Contains($ExpectedNodeAgentImage)
|
||||
lease_ready = ([string]$lease.status -eq "ready")
|
||||
lease_selected_primary_route = ($leasePrimaryRouteID -eq $primaryRouteID)
|
||||
lease_contains_alternate_route = $leaseHasAlternate
|
||||
packet_accepted = ([int]$response.StatusCode -eq 202)
|
||||
accepted_by_header_is_introspection = ($acceptedBy -eq "introspection")
|
||||
active_channel_visible = ($null -ne $matchingChannel)
|
||||
active_channel_not_backend_fallback = ($null -ne $matchingChannel -and -not [bool](Get-PropertyValue -Item $matchingChannel -Name "force_backend_fallback" -Default $false))
|
||||
route_quality_fenced = ($null -ne $matchingChannel -and [string](Get-PropertyValue -Item $matchingChannel -Name "route_feedback_status" -Default "") -eq "fenced")
|
||||
remediation_prefers_alternate = ($null -ne $matchingChannel -and [string](Get-PropertyValue -Item $matchingChannel -Name "remediation_action" -Default "") -eq "prefer_alternate_route")
|
||||
remediation_route_is_alternate = ($null -ne $matchingChannel -and [string](Get-PropertyValue -Item $matchingChannel -Name "remediation_route_id" -Default "") -eq $alternateRouteID)
|
||||
backend_fallback_not_recommended = ([int]$accessTelemetry.degraded_fallback_channel_count -eq 0)
|
||||
}
|
||||
summary = [ordered]@{
|
||||
backend_container = $backendLine.Trim()
|
||||
node_containers = $nodeLines.Trim()
|
||||
accepted_by = $acceptedBy
|
||||
lease_status = [string]$lease.status
|
||||
lease_primary_route_id = $leasePrimaryRouteID
|
||||
lease_alternate_route_count = $leaseAlternates.Count
|
||||
access_status = [string]$accessTelemetry.status
|
||||
active_channel_count = [int]$accessTelemetry.active_channel_count
|
||||
correlated_route_count = [int]$accessTelemetry.correlated_route_count
|
||||
degraded_route_count = [int]$accessTelemetry.degraded_route_count
|
||||
degraded_fallback_channel_count = [int]$accessTelemetry.degraded_fallback_channel_count
|
||||
matching_channel = $matchingChannel
|
||||
}
|
||||
}
|
||||
$failedChecks = @($result.checks.GetEnumerator() | Where-Object { $_.Value -ne $true })
|
||||
if ($failedChecks.Count -gt 0) {
|
||||
throw "C18Z56 failed checks: $($failedChecks.Name -join ', ')"
|
||||
}
|
||||
}
|
||||
finally {
|
||||
if ($primaryRouteID) {
|
||||
try { Invoke-Api -Method POST -Path "/clusters/$ClusterID/mesh/route-intents/$primaryRouteID/expire" -Body @{ actor_user_id = $ActorUserID } | Out-Null } catch {}
|
||||
}
|
||||
if ($alternateRouteID) {
|
||||
try { Invoke-Api -Method POST -Path "/clusters/$ClusterID/mesh/route-intents/$alternateRouteID/expire" -Body @{ actor_user_id = $ActorUserID } | Out-Null } catch {}
|
||||
}
|
||||
}
|
||||
|
||||
$resultFullPath = Join-Path $repoRoot $ResultPath
|
||||
$resultDir = Split-Path -Parent $resultFullPath
|
||||
if (-not (Test-Path $resultDir)) {
|
||||
New-Item -ItemType Directory -Path $resultDir | Out-Null
|
||||
}
|
||||
$result | ConvertTo-Json -Depth 100 | Set-Content -Path $resultFullPath -Encoding UTF8
|
||||
|
||||
Write-Host "C18Z56 service-channel alternate remediation smoke passed. Result: $resultFullPath"
|
||||
$result
|
||||
|
||||
|
||||
@@ -0,0 +1,313 @@
|
||||
param(
|
||||
[string]$ApiBaseUrl = "http://192.168.200.61:18121/api/v1",
|
||||
[string]$ClusterID = "cfc0743d-d960-49fb-9de8-96e063d5e4aa",
|
||||
[string]$ActorUserID = "f67d943f-5397-4b3a-a229-695fe67ad700",
|
||||
[string]$EntryNodeName = "test-1",
|
||||
[string]$RelayNodeName = "test-3",
|
||||
[string]$ExitNodeName = "test-2",
|
||||
[string]$EntryBaseUrl = "http://192.168.200.61:19131",
|
||||
[string]$DockerSSH = "test-docker",
|
||||
[string]$ExpectedBackendImage = "rap-backend:fabric-service-channel-0.2.239",
|
||||
[string]$ExpectedNodeAgentImage = "rap-node-agent:0.2.237",
|
||||
[string]$ResultPath = "artifacts\c18z57-service-channel-remediation-command-smoke-result.json"
|
||||
)
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
$scriptDir = Split-Path -Parent $MyInvocation.MyCommand.Path
|
||||
$repoRoot = (Resolve-Path (Join-Path $scriptDir "..\..")).ProviderPath
|
||||
$runId = "c18z57-" + (Get-Date -Format "yyyyMMdd-HHmmss")
|
||||
|
||||
function Invoke-Api {
|
||||
param(
|
||||
[string]$Method,
|
||||
[string]$Path,
|
||||
[object]$Body = $null
|
||||
)
|
||||
if ($null -eq $Body) {
|
||||
return Invoke-RestMethod -Method $Method -Uri "$ApiBaseUrl$Path" -TimeoutSec 30
|
||||
}
|
||||
return Invoke-RestMethod -Method $Method -Uri "$ApiBaseUrl$Path" -ContentType "application/json" -Body ($Body | ConvertTo-Json -Depth 80) -TimeoutSec 30
|
||||
}
|
||||
|
||||
function Get-PropertyValue {
|
||||
param(
|
||||
[object]$Item,
|
||||
[string]$Name,
|
||||
[object]$Default = $null
|
||||
)
|
||||
if ($null -eq $Item) { return $Default }
|
||||
$property = $Item.PSObject.Properties[$Name]
|
||||
if ($null -eq $property) { return $Default }
|
||||
return $property.Value
|
||||
}
|
||||
|
||||
function Get-NodeByName {
|
||||
param([string]$Name)
|
||||
$nodes = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/nodes?actor_user_id=$ActorUserID").nodes
|
||||
$node = @($nodes | Where-Object { $_.name -eq $Name }) | Select-Object -First 1
|
||||
if ($null -eq $node) {
|
||||
throw "Node '$Name' was not found in cluster $ClusterID"
|
||||
}
|
||||
return $node
|
||||
}
|
||||
|
||||
function New-RouteIntent {
|
||||
param(
|
||||
[string]$SourceNodeID,
|
||||
[string]$DestinationNodeID,
|
||||
[string[]]$Hops,
|
||||
[int]$Priority,
|
||||
[string]$Label
|
||||
)
|
||||
$expiresAt = (Get-Date).ToUniversalTime().AddMinutes(5).ToString("o")
|
||||
return Invoke-Api -Method POST -Path "/clusters/$ClusterID/mesh/route-intents" -Body @{
|
||||
actor_user_id = $ActorUserID
|
||||
source_selector = @{ node_id = $SourceNodeID }
|
||||
destination_selector = @{ node_id = $DestinationNodeID }
|
||||
service_class = "vpn_packets"
|
||||
priority = $Priority
|
||||
policy = @{
|
||||
synthetic_enabled = $true
|
||||
route_version = "$runId-$Label"
|
||||
policy_version = "$runId-$Label"
|
||||
peer_directory_version = "$runId-$Label"
|
||||
hops = @($Hops)
|
||||
allowed_channels = @("vpn_packet", "fabric_control")
|
||||
max_ttl = 8
|
||||
max_hops = 8
|
||||
expires_at = $expiresAt
|
||||
metadata = @{
|
||||
smoke = "c18z57_service_channel_remediation_command"
|
||||
run_id = $runId
|
||||
label = $Label
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function Send-DegradedHeartbeat {
|
||||
param(
|
||||
[string]$EntryNodeID,
|
||||
[string]$PrimaryRouteID
|
||||
)
|
||||
$observedAt = (Get-Date).ToUniversalTime().ToString("o")
|
||||
return Invoke-Api -Method POST -Path "/clusters/$ClusterID/nodes/$EntryNodeID/heartbeats" -Body @{
|
||||
health_status = "healthy"
|
||||
reported_version = "0.2.235"
|
||||
capabilities = @{
|
||||
fabric_service_channel_runtime = $true
|
||||
fabric_service_channel_route_manager = $true
|
||||
fabric_service_channel_route_quality_feedback = $true
|
||||
smoke_feedback_injection = "c18z57"
|
||||
}
|
||||
service_states = @{ smoke = "c18z57_primary_degraded_alternate_available" }
|
||||
metadata = @{
|
||||
fabric_service_channel_runtime_report = @{
|
||||
schema_version = "c18l.fabric_service_channel_runtime_report.v1"
|
||||
config_version = "$runId-primary"
|
||||
cluster_id = $ClusterID
|
||||
local_node_id = $EntryNodeID
|
||||
observed_at = $observedAt
|
||||
ingress = @{
|
||||
flow_scheduler = @{
|
||||
schema_version = "rap.fabric_flow_scheduler.v1"
|
||||
service_neutral = $true
|
||||
service_mode = "application_protocol_agnostic"
|
||||
channel_stats = @{
|
||||
"c18z57-primary-degraded" = @{
|
||||
last_route_id = $PrimaryRouteID
|
||||
last_failed_route_id = $PrimaryRouteID
|
||||
route_generation = "$runId-primary"
|
||||
last_error = "c18z57 primary route degraded; alternate available"
|
||||
last_send_duration_ms = 980
|
||||
consecutive_failures = 3
|
||||
stall_count = 2
|
||||
route_rebuild_recommended = $true
|
||||
degraded_fallback_recommended = $false
|
||||
quality_window_sample_count = 8
|
||||
quality_window_success_count = 2
|
||||
quality_window_failure_count = 3
|
||||
quality_window_slow_count = 2
|
||||
quality_window_drop_count = 1
|
||||
quality_window_avg_latency_ms = 980
|
||||
quality_window_last_updated_at = $observedAt
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
smoke = @{
|
||||
name = "c18z57_service_channel_remediation_command"
|
||||
run_id = $runId
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
$entryNode = Get-NodeByName -Name $EntryNodeName
|
||||
$relayNode = Get-NodeByName -Name $RelayNodeName
|
||||
$exitNode = Get-NodeByName -Name $ExitNodeName
|
||||
$primaryRouteID = ""
|
||||
$alternateRouteID = ""
|
||||
$result = $null
|
||||
|
||||
try {
|
||||
$primary = (New-RouteIntent -SourceNodeID $entryNode.id -DestinationNodeID $exitNode.id -Hops @($entryNode.id, $exitNode.id) -Priority 2100000000 -Label "primary").route_intent
|
||||
$alternate = (New-RouteIntent -SourceNodeID $entryNode.id -DestinationNodeID $exitNode.id -Hops @($entryNode.id, $relayNode.id, $exitNode.id) -Priority 2099999900 -Label "alternate").route_intent
|
||||
$primaryRouteID = [string]$primary.id
|
||||
$alternateRouteID = [string]$alternate.id
|
||||
|
||||
$resourceID = "c18z57-vpn-smoke"
|
||||
$lease = (Invoke-Api -Method POST -Path "/clusters/$ClusterID/fabric/service-channels/leases" -Body @{
|
||||
actor_user_id = $ActorUserID
|
||||
organization_id = "smoke-org"
|
||||
user_id = "smoke-user"
|
||||
resource_id = $resourceID
|
||||
service_class = "vpn_packets"
|
||||
entry_node_ids = @([string]$entryNode.id)
|
||||
exit_node_ids = @([string]$exitNode.id)
|
||||
preferred_entry_node_id = [string]$entryNode.id
|
||||
preferred_exit_node_id = [string]$exitNode.id
|
||||
allowed_channels = @("vpn_packet", "fabric_control")
|
||||
ttl_seconds = 180
|
||||
metadata = @{
|
||||
smoke = "c18z57_service_channel_remediation_command"
|
||||
run_id = $runId
|
||||
}
|
||||
}).fabric_service_channel_lease
|
||||
|
||||
[void](Send-DegradedHeartbeat -EntryNodeID $entryNode.id -PrimaryRouteID $primaryRouteID)
|
||||
|
||||
$packetPath = $lease.entry_http.path_template.
|
||||
Replace("{cluster_id}", $ClusterID).
|
||||
Replace("{channel_id}", [string]$lease.channel_id).
|
||||
Replace("{resource_id}", $resourceID)
|
||||
$packetUrl = $EntryBaseUrl.TrimEnd("/") + $packetPath
|
||||
$headers = @{
|
||||
"X-RAP-Service-Channel-Token" = [string]$lease.token.token
|
||||
"X-RAP-Fabric-Channel-ID" = [string]$lease.channel_id
|
||||
"X-RAP-Service-Class" = "vpn_packets"
|
||||
"X-RAP-Channel-Class" = "vpn_packet"
|
||||
}
|
||||
$response = Invoke-WebRequest -Method Post -Uri $packetUrl -Headers $headers -Body ([System.Text.Encoding]::UTF8.GetBytes("c18z57-alternate-remediation")) -ContentType "application/octet-stream" -TimeoutSec 30
|
||||
$acceptedBy = [string]$response.Headers["X-RAP-Service-Channel-Accepted-By"]
|
||||
|
||||
$accessTelemetry = $null
|
||||
$matchingChannel = $null
|
||||
for ($i = 0; $i -lt 10; $i++) {
|
||||
Start-Sleep -Seconds 3
|
||||
$accessTelemetry = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/fabric/service-channels/access-telemetry?actor_user_id=$ActorUserID&limit=20").fabric_service_channel_access_telemetry
|
||||
$channels = @()
|
||||
if ($accessTelemetry.PSObject.Properties.Name -contains "active_channels") {
|
||||
$channels = @($accessTelemetry.active_channels)
|
||||
}
|
||||
$matchingChannel = $channels | Where-Object { $_.channel_id -eq $lease.channel_id } | Select-Object -First 1
|
||||
if ($null -ne $matchingChannel -and [string](Get-PropertyValue -Item $matchingChannel -Name "remediation_action" -Default "") -eq "prefer_alternate_route") {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
$backendLine = (& ssh $DockerSSH "docker ps --format '{{.Names}} {{.Image}} {{.Status}}' | grep '^rap_test_backend '") | Out-String
|
||||
$nodeLines = (& ssh $DockerSSH "docker ps --format '{{.Names}} {{.Image}} {{.Status}}' | grep '^rap_test_node_test_'") | Out-String
|
||||
$leasePrimaryRouteID = [string](Get-PropertyValue -Item (Get-PropertyValue -Item $lease -Name "primary_route" -Default $null) -Name "route_id" -Default "")
|
||||
$leaseAlternates = @()
|
||||
if ($lease.PSObject.Properties.Name -contains "alternate_routes") {
|
||||
$leaseAlternates = @($lease.alternate_routes)
|
||||
}
|
||||
$leaseHasAlternate = (@($leaseAlternates | Where-Object { [string]$_.route_id -eq $alternateRouteID }).Count -ge 1)
|
||||
$remediationCommand = Get-PropertyValue -Item $matchingChannel -Name "remediation_command" -Default $null
|
||||
$commandAction = [string](Get-PropertyValue -Item $remediationCommand -Name "action" -Default "")
|
||||
$commandPrimaryRouteID = [string](Get-PropertyValue -Item $remediationCommand -Name "primary_route_id" -Default "")
|
||||
$commandReplacementRouteID = [string](Get-PropertyValue -Item $remediationCommand -Name "replacement_route_id" -Default "")
|
||||
$commandExpiresAt = [string](Get-PropertyValue -Item $remediationCommand -Name "expires_at" -Default "")
|
||||
|
||||
$result = [ordered]@{
|
||||
schema_version = "c18z57.service_channel_remediation_command_smoke.v1"
|
||||
run_id = $runId
|
||||
cluster_id = $ClusterID
|
||||
primary_route_id = $primaryRouteID
|
||||
alternate_route_id = $alternateRouteID
|
||||
channel_id = [string]$lease.channel_id
|
||||
passed = [bool](
|
||||
$backendLine.Contains($ExpectedBackendImage) -and
|
||||
$nodeLines.Contains($ExpectedNodeAgentImage) -and
|
||||
[string]$lease.status -eq "ready" -and
|
||||
$leasePrimaryRouteID -eq $primaryRouteID -and
|
||||
$leaseHasAlternate -and
|
||||
[int]$response.StatusCode -eq 202 -and
|
||||
$acceptedBy -eq "introspection" -and
|
||||
$null -ne $matchingChannel -and
|
||||
[string](Get-PropertyValue -Item $matchingChannel -Name "primary_route_id" -Default "") -eq $primaryRouteID -and
|
||||
-not [bool](Get-PropertyValue -Item $matchingChannel -Name "force_backend_fallback" -Default $false) -and
|
||||
[string](Get-PropertyValue -Item $matchingChannel -Name "route_feedback_status" -Default "") -eq "fenced" -and
|
||||
[string](Get-PropertyValue -Item $matchingChannel -Name "remediation_action" -Default "") -eq "prefer_alternate_route" -and
|
||||
[string](Get-PropertyValue -Item $matchingChannel -Name "remediation_route_id" -Default "") -eq $alternateRouteID -and
|
||||
$null -ne $remediationCommand -and
|
||||
$commandAction -eq "prefer_alternate_route" -and
|
||||
$commandPrimaryRouteID -eq $primaryRouteID -and
|
||||
$commandReplacementRouteID -eq $alternateRouteID -and
|
||||
$commandExpiresAt.Length -gt 0 -and
|
||||
[int]$accessTelemetry.degraded_fallback_channel_count -eq 0
|
||||
)
|
||||
checks = [ordered]@{
|
||||
backend_expected_image_deployed = $backendLine.Contains($ExpectedBackendImage)
|
||||
node_agent_expected_image_deployed = $nodeLines.Contains($ExpectedNodeAgentImage)
|
||||
lease_ready = ([string]$lease.status -eq "ready")
|
||||
lease_selected_primary_route = ($leasePrimaryRouteID -eq $primaryRouteID)
|
||||
lease_contains_alternate_route = $leaseHasAlternate
|
||||
packet_accepted = ([int]$response.StatusCode -eq 202)
|
||||
accepted_by_header_is_introspection = ($acceptedBy -eq "introspection")
|
||||
active_channel_visible = ($null -ne $matchingChannel)
|
||||
active_channel_not_backend_fallback = ($null -ne $matchingChannel -and -not [bool](Get-PropertyValue -Item $matchingChannel -Name "force_backend_fallback" -Default $false))
|
||||
route_quality_fenced = ($null -ne $matchingChannel -and [string](Get-PropertyValue -Item $matchingChannel -Name "route_feedback_status" -Default "") -eq "fenced")
|
||||
remediation_prefers_alternate = ($null -ne $matchingChannel -and [string](Get-PropertyValue -Item $matchingChannel -Name "remediation_action" -Default "") -eq "prefer_alternate_route")
|
||||
remediation_route_is_alternate = ($null -ne $matchingChannel -and [string](Get-PropertyValue -Item $matchingChannel -Name "remediation_route_id" -Default "") -eq $alternateRouteID)
|
||||
remediation_command_visible = ($null -ne $remediationCommand)
|
||||
remediation_command_prefers_alternate = ($commandAction -eq "prefer_alternate_route")
|
||||
remediation_command_primary_route_matches = ($commandPrimaryRouteID -eq $primaryRouteID)
|
||||
remediation_command_replacement_route_matches = ($commandReplacementRouteID -eq $alternateRouteID)
|
||||
remediation_command_has_ttl = ($commandExpiresAt.Length -gt 0)
|
||||
backend_fallback_not_recommended = ([int]$accessTelemetry.degraded_fallback_channel_count -eq 0)
|
||||
}
|
||||
summary = [ordered]@{
|
||||
backend_container = $backendLine.Trim()
|
||||
node_containers = $nodeLines.Trim()
|
||||
accepted_by = $acceptedBy
|
||||
lease_status = [string]$lease.status
|
||||
lease_primary_route_id = $leasePrimaryRouteID
|
||||
lease_alternate_route_count = $leaseAlternates.Count
|
||||
access_status = [string]$accessTelemetry.status
|
||||
active_channel_count = [int]$accessTelemetry.active_channel_count
|
||||
correlated_route_count = [int]$accessTelemetry.correlated_route_count
|
||||
degraded_route_count = [int]$accessTelemetry.degraded_route_count
|
||||
degraded_fallback_channel_count = [int]$accessTelemetry.degraded_fallback_channel_count
|
||||
matching_channel = $matchingChannel
|
||||
}
|
||||
}
|
||||
$failedChecks = @($result.checks.GetEnumerator() | Where-Object { $_.Value -ne $true })
|
||||
if ($failedChecks.Count -gt 0) {
|
||||
throw "C18Z57 failed checks: $($failedChecks.Name -join ', ')"
|
||||
}
|
||||
}
|
||||
finally {
|
||||
if ($primaryRouteID) {
|
||||
try { Invoke-Api -Method POST -Path "/clusters/$ClusterID/mesh/route-intents/$primaryRouteID/expire" -Body @{ actor_user_id = $ActorUserID } | Out-Null } catch {}
|
||||
}
|
||||
if ($alternateRouteID) {
|
||||
try { Invoke-Api -Method POST -Path "/clusters/$ClusterID/mesh/route-intents/$alternateRouteID/expire" -Body @{ actor_user_id = $ActorUserID } | Out-Null } catch {}
|
||||
}
|
||||
}
|
||||
|
||||
$resultFullPath = Join-Path $repoRoot $ResultPath
|
||||
$resultDir = Split-Path -Parent $resultFullPath
|
||||
if (-not (Test-Path $resultDir)) {
|
||||
New-Item -ItemType Directory -Path $resultDir | Out-Null
|
||||
}
|
||||
$result | ConvertTo-Json -Depth 100 | Set-Content -Path $resultFullPath -Encoding UTF8
|
||||
|
||||
Write-Host "C18Z57 service-channel remediation command smoke passed. Result: $resultFullPath"
|
||||
$result
|
||||
|
||||
|
||||
@@ -0,0 +1,359 @@
|
||||
param(
|
||||
[string]$ApiBaseUrl = "http://192.168.200.61:18121/api/v1",
|
||||
[string]$ClusterID = "cfc0743d-d960-49fb-9de8-96e063d5e4aa",
|
||||
[string]$ActorUserID = "f67d943f-5397-4b3a-a229-695fe67ad700",
|
||||
[string]$EntryNodeName = "test-1",
|
||||
[string]$RelayNodeName = "test-3",
|
||||
[string]$ExitNodeName = "test-2",
|
||||
[string]$EntryBaseUrl = "http://192.168.200.61:19131",
|
||||
[string]$DockerSSH = "test-docker",
|
||||
[string]$ExpectedBackendImage = "rap-backend:fabric-service-channel-0.2.239",
|
||||
[string]$ExpectedNodeAgentImage = "rap-node-agent:0.2.237",
|
||||
[string]$ResultPath = "artifacts\c18z58-service-channel-remediation-apply-smoke-result.json"
|
||||
)
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
$scriptDir = Split-Path -Parent $MyInvocation.MyCommand.Path
|
||||
$repoRoot = (Resolve-Path (Join-Path $scriptDir "..\..")).ProviderPath
|
||||
$runId = "c18z58-" + (Get-Date -Format "yyyyMMdd-HHmmss")
|
||||
|
||||
function Invoke-Api {
|
||||
param(
|
||||
[string]$Method,
|
||||
[string]$Path,
|
||||
[object]$Body = $null
|
||||
)
|
||||
if ($null -eq $Body) {
|
||||
return Invoke-RestMethod -Method $Method -Uri "$ApiBaseUrl$Path" -TimeoutSec 30
|
||||
}
|
||||
return Invoke-RestMethod -Method $Method -Uri "$ApiBaseUrl$Path" -ContentType "application/json" -Body ($Body | ConvertTo-Json -Depth 80) -TimeoutSec 30
|
||||
}
|
||||
|
||||
function Get-PropertyValue {
|
||||
param(
|
||||
[object]$Item,
|
||||
[string]$Name,
|
||||
[object]$Default = $null
|
||||
)
|
||||
if ($null -eq $Item) { return $Default }
|
||||
$property = $Item.PSObject.Properties[$Name]
|
||||
if ($null -eq $property) { return $Default }
|
||||
return $property.Value
|
||||
}
|
||||
|
||||
function Get-NodeByName {
|
||||
param([string]$Name)
|
||||
$nodes = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/nodes?actor_user_id=$ActorUserID").nodes
|
||||
$node = @($nodes | Where-Object { $_.name -eq $Name }) | Select-Object -First 1
|
||||
if ($null -eq $node) {
|
||||
throw "Node '$Name' was not found in cluster $ClusterID"
|
||||
}
|
||||
return $node
|
||||
}
|
||||
|
||||
function New-RouteIntent {
|
||||
param(
|
||||
[string]$SourceNodeID,
|
||||
[string]$DestinationNodeID,
|
||||
[string[]]$Hops,
|
||||
[int]$Priority,
|
||||
[string]$Label
|
||||
)
|
||||
$expiresAt = (Get-Date).ToUniversalTime().AddMinutes(5).ToString("o")
|
||||
return Invoke-Api -Method POST -Path "/clusters/$ClusterID/mesh/route-intents" -Body @{
|
||||
actor_user_id = $ActorUserID
|
||||
source_selector = @{ node_id = $SourceNodeID }
|
||||
destination_selector = @{ node_id = $DestinationNodeID }
|
||||
service_class = "vpn_packets"
|
||||
priority = $Priority
|
||||
policy = @{
|
||||
synthetic_enabled = $true
|
||||
route_version = "$runId-$Label"
|
||||
policy_version = "$runId-$Label"
|
||||
peer_directory_version = "$runId-$Label"
|
||||
hops = @($Hops)
|
||||
allowed_channels = @("vpn_packet", "fabric_control")
|
||||
max_ttl = 8
|
||||
max_hops = 8
|
||||
expires_at = $expiresAt
|
||||
metadata = @{
|
||||
smoke = "c18z58_service_channel_remediation_apply"
|
||||
run_id = $runId
|
||||
label = $Label
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function Send-DegradedHeartbeat {
|
||||
param(
|
||||
[string]$EntryNodeID,
|
||||
[string]$PrimaryRouteID
|
||||
)
|
||||
$observedAt = (Get-Date).ToUniversalTime().ToString("o")
|
||||
return Invoke-Api -Method POST -Path "/clusters/$ClusterID/nodes/$EntryNodeID/heartbeats" -Body @{
|
||||
health_status = "healthy"
|
||||
reported_version = "0.2.236"
|
||||
capabilities = @{
|
||||
fabric_service_channel_runtime = $true
|
||||
fabric_service_channel_route_manager = $true
|
||||
fabric_service_channel_route_quality_feedback = $true
|
||||
smoke_feedback_injection = "c18z58"
|
||||
}
|
||||
service_states = @{ smoke = "c18z58_primary_degraded_alternate_available" }
|
||||
metadata = @{
|
||||
fabric_service_channel_runtime_report = @{
|
||||
schema_version = "c18l.fabric_service_channel_runtime_report.v1"
|
||||
config_version = "$runId-primary"
|
||||
cluster_id = $ClusterID
|
||||
local_node_id = $EntryNodeID
|
||||
observed_at = $observedAt
|
||||
ingress = @{
|
||||
flow_scheduler = @{
|
||||
schema_version = "rap.fabric_flow_scheduler.v1"
|
||||
service_neutral = $true
|
||||
service_mode = "application_protocol_agnostic"
|
||||
channel_stats = @{
|
||||
"c18z58-primary-degraded" = @{
|
||||
last_route_id = $PrimaryRouteID
|
||||
last_failed_route_id = $PrimaryRouteID
|
||||
route_generation = "$runId-primary"
|
||||
last_error = "c18z58 primary route degraded; alternate available"
|
||||
last_send_duration_ms = 980
|
||||
consecutive_failures = 3
|
||||
stall_count = 2
|
||||
route_rebuild_recommended = $true
|
||||
degraded_fallback_recommended = $false
|
||||
quality_window_sample_count = 8
|
||||
quality_window_success_count = 2
|
||||
quality_window_failure_count = 3
|
||||
quality_window_slow_count = 2
|
||||
quality_window_drop_count = 1
|
||||
quality_window_avg_latency_ms = 980
|
||||
quality_window_last_updated_at = $observedAt
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
smoke = @{
|
||||
name = "c18z58_service_channel_remediation_apply"
|
||||
run_id = $runId
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
$entryNode = Get-NodeByName -Name $EntryNodeName
|
||||
$relayNode = Get-NodeByName -Name $RelayNodeName
|
||||
$exitNode = Get-NodeByName -Name $ExitNodeName
|
||||
$primaryRouteID = ""
|
||||
$alternateRouteID = ""
|
||||
$result = $null
|
||||
|
||||
try {
|
||||
$primary = (New-RouteIntent -SourceNodeID $entryNode.id -DestinationNodeID $exitNode.id -Hops @($entryNode.id, $exitNode.id) -Priority 2100000000 -Label "primary").route_intent
|
||||
$alternate = (New-RouteIntent -SourceNodeID $entryNode.id -DestinationNodeID $exitNode.id -Hops @($entryNode.id, $relayNode.id, $exitNode.id) -Priority 2099999900 -Label "alternate").route_intent
|
||||
$primaryRouteID = [string]$primary.id
|
||||
$alternateRouteID = [string]$alternate.id
|
||||
|
||||
$resourceID = "c18z58-vpn-smoke"
|
||||
$lease = (Invoke-Api -Method POST -Path "/clusters/$ClusterID/fabric/service-channels/leases" -Body @{
|
||||
actor_user_id = $ActorUserID
|
||||
organization_id = "smoke-org"
|
||||
user_id = "smoke-user"
|
||||
resource_id = $resourceID
|
||||
service_class = "vpn_packets"
|
||||
entry_node_ids = @([string]$entryNode.id)
|
||||
exit_node_ids = @([string]$exitNode.id)
|
||||
preferred_entry_node_id = [string]$entryNode.id
|
||||
preferred_exit_node_id = [string]$exitNode.id
|
||||
allowed_channels = @("vpn_packet", "fabric_control")
|
||||
ttl_seconds = 180
|
||||
metadata = @{
|
||||
smoke = "c18z58_service_channel_remediation_apply"
|
||||
run_id = $runId
|
||||
}
|
||||
}).fabric_service_channel_lease
|
||||
|
||||
[void](Send-DegradedHeartbeat -EntryNodeID $entryNode.id -PrimaryRouteID $primaryRouteID)
|
||||
|
||||
$packetPath = $lease.entry_http.path_template.
|
||||
Replace("{cluster_id}", $ClusterID).
|
||||
Replace("{channel_id}", [string]$lease.channel_id).
|
||||
Replace("{resource_id}", $resourceID)
|
||||
$packetUrl = $EntryBaseUrl.TrimEnd("/") + $packetPath
|
||||
$headers = @{
|
||||
"X-RAP-Service-Channel-Token" = [string]$lease.token.token
|
||||
"X-RAP-Fabric-Channel-ID" = [string]$lease.channel_id
|
||||
"X-RAP-Service-Class" = "vpn_packets"
|
||||
"X-RAP-Channel-Class" = "vpn_packet"
|
||||
}
|
||||
$response = Invoke-WebRequest -Method Post -Uri $packetUrl -Headers $headers -Body ([System.Text.Encoding]::UTF8.GetBytes("c18z58-alternate-remediation")) -ContentType "application/octet-stream" -TimeoutSec 30
|
||||
$acceptedBy = [string]$response.Headers["X-RAP-Service-Channel-Accepted-By"]
|
||||
|
||||
$accessTelemetry = $null
|
||||
$matchingChannel = $null
|
||||
for ($i = 0; $i -lt 10; $i++) {
|
||||
Start-Sleep -Seconds 3
|
||||
$accessTelemetry = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/fabric/service-channels/access-telemetry?actor_user_id=$ActorUserID&limit=20").fabric_service_channel_access_telemetry
|
||||
$channels = @()
|
||||
if ($accessTelemetry.PSObject.Properties.Name -contains "active_channels") {
|
||||
$channels = @($accessTelemetry.active_channels)
|
||||
}
|
||||
$matchingChannel = $channels | Where-Object { $_.channel_id -eq $lease.channel_id } | Select-Object -First 1
|
||||
if ($null -ne $matchingChannel -and [string](Get-PropertyValue -Item $matchingChannel -Name "remediation_action" -Default "") -eq "prefer_alternate_route") {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
$backendLine = (& ssh $DockerSSH "docker ps --format '{{.Names}} {{.Image}} {{.Status}}' | grep '^rap_test_backend '") | Out-String
|
||||
$nodeLines = (& ssh $DockerSSH "docker ps --format '{{.Names}} {{.Image}} {{.Status}}' | grep '^rap_test_node_test_'") | Out-String
|
||||
$leasePrimaryRouteID = [string](Get-PropertyValue -Item (Get-PropertyValue -Item $lease -Name "primary_route" -Default $null) -Name "route_id" -Default "")
|
||||
$leaseAlternates = @()
|
||||
if ($lease.PSObject.Properties.Name -contains "alternate_routes") {
|
||||
$leaseAlternates = @($lease.alternate_routes)
|
||||
}
|
||||
$leaseHasAlternate = (@($leaseAlternates | Where-Object { [string]$_.route_id -eq $alternateRouteID }).Count -ge 1)
|
||||
$remediationCommand = Get-PropertyValue -Item $matchingChannel -Name "remediation_command" -Default $null
|
||||
$commandAction = [string](Get-PropertyValue -Item $remediationCommand -Name "action" -Default "")
|
||||
$commandPrimaryRouteID = [string](Get-PropertyValue -Item $remediationCommand -Name "primary_route_id" -Default "")
|
||||
$commandReplacementRouteID = [string](Get-PropertyValue -Item $remediationCommand -Name "replacement_route_id" -Default "")
|
||||
$commandExpiresAt = [string](Get-PropertyValue -Item $remediationCommand -Name "expires_at" -Default "")
|
||||
$syntheticConfig = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/nodes/$($entryNode.id)/mesh/synthetic-config").synthetic_mesh_config
|
||||
$syntheticCommands = @()
|
||||
if ($syntheticConfig.PSObject.Properties.Name -contains "service_channel_remediation_commands") {
|
||||
$syntheticCommands = @($syntheticConfig.service_channel_remediation_commands)
|
||||
}
|
||||
$syntheticCommand = $syntheticCommands | Where-Object { [string]$_.channel_id -eq [string]$lease.channel_id } | Select-Object -First 1
|
||||
$syntheticCommandAction = [string](Get-PropertyValue -Item $syntheticCommand -Name "action" -Default "")
|
||||
$syntheticCommandReplacementRouteID = [string](Get-PropertyValue -Item $syntheticCommand -Name "replacement_route_id" -Default "")
|
||||
|
||||
$routeManagerDecision = $null
|
||||
for ($i = 0; $i -lt 10; $i++) {
|
||||
Start-Sleep -Seconds 5
|
||||
$heartbeats = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/nodes/$($entryNode.id)/heartbeats?actor_user_id=$ActorUserID&limit=5").heartbeats
|
||||
foreach ($heartbeat in @($heartbeats)) {
|
||||
$runtimeReport = Get-PropertyValue -Item (Get-PropertyValue -Item $heartbeat -Name "metadata" -Default $null) -Name "fabric_service_channel_runtime_report" -Default $null
|
||||
$ingress = Get-PropertyValue -Item $runtimeReport -Name "ingress" -Default $null
|
||||
$routeManager = Get-PropertyValue -Item $ingress -Name "route_manager" -Default $null
|
||||
$decisions = @()
|
||||
if ($null -ne $routeManager -and $routeManager.PSObject.Properties.Name -contains "decisions") {
|
||||
$decisions = @($routeManager.decisions)
|
||||
}
|
||||
$routeManagerDecision = $decisions | Where-Object {
|
||||
[string]$_.route_id -eq $primaryRouteID -and
|
||||
[string]$_.replacement_route_id -eq $alternateRouteID -and
|
||||
[string]$_.decision_source -eq "service_channel_remediation_command"
|
||||
} | Select-Object -First 1
|
||||
if ($null -ne $routeManagerDecision) {
|
||||
break
|
||||
}
|
||||
}
|
||||
if ($null -ne $routeManagerDecision) {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
$result = [ordered]@{
|
||||
schema_version = "c18z58.service_channel_remediation_apply_smoke.v1"
|
||||
run_id = $runId
|
||||
cluster_id = $ClusterID
|
||||
primary_route_id = $primaryRouteID
|
||||
alternate_route_id = $alternateRouteID
|
||||
channel_id = [string]$lease.channel_id
|
||||
passed = [bool](
|
||||
$backendLine.Contains($ExpectedBackendImage) -and
|
||||
$nodeLines.Contains($ExpectedNodeAgentImage) -and
|
||||
[string]$lease.status -eq "ready" -and
|
||||
$leasePrimaryRouteID -eq $primaryRouteID -and
|
||||
$leaseHasAlternate -and
|
||||
[int]$response.StatusCode -eq 202 -and
|
||||
$acceptedBy -eq "introspection" -and
|
||||
$null -ne $matchingChannel -and
|
||||
[string](Get-PropertyValue -Item $matchingChannel -Name "primary_route_id" -Default "") -eq $primaryRouteID -and
|
||||
-not [bool](Get-PropertyValue -Item $matchingChannel -Name "force_backend_fallback" -Default $false) -and
|
||||
[string](Get-PropertyValue -Item $matchingChannel -Name "route_feedback_status" -Default "") -eq "fenced" -and
|
||||
[string](Get-PropertyValue -Item $matchingChannel -Name "remediation_action" -Default "") -eq "prefer_alternate_route" -and
|
||||
[string](Get-PropertyValue -Item $matchingChannel -Name "remediation_route_id" -Default "") -eq $alternateRouteID -and
|
||||
$null -ne $remediationCommand -and
|
||||
$commandAction -eq "prefer_alternate_route" -and
|
||||
$commandPrimaryRouteID -eq $primaryRouteID -and
|
||||
$commandReplacementRouteID -eq $alternateRouteID -and
|
||||
$commandExpiresAt.Length -gt 0 -and
|
||||
$null -ne $syntheticCommand -and
|
||||
$syntheticCommandAction -eq "prefer_alternate_route" -and
|
||||
$syntheticCommandReplacementRouteID -eq $alternateRouteID -and
|
||||
$null -ne $routeManagerDecision -and
|
||||
[string](Get-PropertyValue -Item $routeManagerDecision -Name "rebuild_status" -Default "") -eq "applied" -and
|
||||
[int]$accessTelemetry.degraded_fallback_channel_count -eq 0
|
||||
)
|
||||
checks = [ordered]@{
|
||||
backend_expected_image_deployed = $backendLine.Contains($ExpectedBackendImage)
|
||||
node_agent_expected_image_deployed = $nodeLines.Contains($ExpectedNodeAgentImage)
|
||||
lease_ready = ([string]$lease.status -eq "ready")
|
||||
lease_selected_primary_route = ($leasePrimaryRouteID -eq $primaryRouteID)
|
||||
lease_contains_alternate_route = $leaseHasAlternate
|
||||
packet_accepted = ([int]$response.StatusCode -eq 202)
|
||||
accepted_by_header_is_introspection = ($acceptedBy -eq "introspection")
|
||||
active_channel_visible = ($null -ne $matchingChannel)
|
||||
active_channel_not_backend_fallback = ($null -ne $matchingChannel -and -not [bool](Get-PropertyValue -Item $matchingChannel -Name "force_backend_fallback" -Default $false))
|
||||
route_quality_fenced = ($null -ne $matchingChannel -and [string](Get-PropertyValue -Item $matchingChannel -Name "route_feedback_status" -Default "") -eq "fenced")
|
||||
remediation_prefers_alternate = ($null -ne $matchingChannel -and [string](Get-PropertyValue -Item $matchingChannel -Name "remediation_action" -Default "") -eq "prefer_alternate_route")
|
||||
remediation_route_is_alternate = ($null -ne $matchingChannel -and [string](Get-PropertyValue -Item $matchingChannel -Name "remediation_route_id" -Default "") -eq $alternateRouteID)
|
||||
remediation_command_visible = ($null -ne $remediationCommand)
|
||||
remediation_command_prefers_alternate = ($commandAction -eq "prefer_alternate_route")
|
||||
remediation_command_primary_route_matches = ($commandPrimaryRouteID -eq $primaryRouteID)
|
||||
remediation_command_replacement_route_matches = ($commandReplacementRouteID -eq $alternateRouteID)
|
||||
remediation_command_has_ttl = ($commandExpiresAt.Length -gt 0)
|
||||
synthetic_config_command_visible = ($null -ne $syntheticCommand)
|
||||
synthetic_config_command_prefers_alternate = ($syntheticCommandAction -eq "prefer_alternate_route")
|
||||
synthetic_config_command_replacement_route_matches = ($syntheticCommandReplacementRouteID -eq $alternateRouteID)
|
||||
node_route_manager_consumed_command = ($null -ne $routeManagerDecision)
|
||||
node_route_manager_applied_replacement = ($null -ne $routeManagerDecision -and [string](Get-PropertyValue -Item $routeManagerDecision -Name "rebuild_status" -Default "") -eq "applied")
|
||||
backend_fallback_not_recommended = ([int]$accessTelemetry.degraded_fallback_channel_count -eq 0)
|
||||
}
|
||||
summary = [ordered]@{
|
||||
backend_container = $backendLine.Trim()
|
||||
node_containers = $nodeLines.Trim()
|
||||
accepted_by = $acceptedBy
|
||||
lease_status = [string]$lease.status
|
||||
lease_primary_route_id = $leasePrimaryRouteID
|
||||
lease_alternate_route_count = $leaseAlternates.Count
|
||||
access_status = [string]$accessTelemetry.status
|
||||
active_channel_count = [int]$accessTelemetry.active_channel_count
|
||||
correlated_route_count = [int]$accessTelemetry.correlated_route_count
|
||||
degraded_route_count = [int]$accessTelemetry.degraded_route_count
|
||||
degraded_fallback_channel_count = [int]$accessTelemetry.degraded_fallback_channel_count
|
||||
synthetic_command = $syntheticCommand
|
||||
route_manager_decision = $routeManagerDecision
|
||||
matching_channel = $matchingChannel
|
||||
}
|
||||
}
|
||||
$failedChecks = @($result.checks.GetEnumerator() | Where-Object { $_.Value -ne $true })
|
||||
if ($failedChecks.Count -gt 0) {
|
||||
throw "C18Z58 failed checks: $($failedChecks.Name -join ', ')"
|
||||
}
|
||||
}
|
||||
finally {
|
||||
if ($primaryRouteID) {
|
||||
try { Invoke-Api -Method POST -Path "/clusters/$ClusterID/mesh/route-intents/$primaryRouteID/expire" -Body @{ actor_user_id = $ActorUserID } | Out-Null } catch {}
|
||||
}
|
||||
if ($alternateRouteID) {
|
||||
try { Invoke-Api -Method POST -Path "/clusters/$ClusterID/mesh/route-intents/$alternateRouteID/expire" -Body @{ actor_user_id = $ActorUserID } | Out-Null } catch {}
|
||||
}
|
||||
}
|
||||
|
||||
$resultFullPath = Join-Path $repoRoot $ResultPath
|
||||
$resultDir = Split-Path -Parent $resultFullPath
|
||||
if (-not (Test-Path $resultDir)) {
|
||||
New-Item -ItemType Directory -Path $resultDir | Out-Null
|
||||
}
|
||||
$result | ConvertTo-Json -Depth 100 | Set-Content -Path $resultFullPath -Encoding UTF8
|
||||
|
||||
Write-Host "C18Z58 service-channel remediation apply smoke passed. Result: $resultFullPath"
|
||||
$result
|
||||
|
||||
|
||||
@@ -0,0 +1,416 @@
|
||||
param(
|
||||
[string]$ApiBaseUrl = "http://192.168.200.61:18121/api/v1",
|
||||
[string]$ClusterID = "cfc0743d-d960-49fb-9de8-96e063d5e4aa",
|
||||
[string]$ActorUserID = "f67d943f-5397-4b3a-a229-695fe67ad700",
|
||||
[string]$EntryNodeName = "test-1",
|
||||
[string]$RelayNodeName = "test-3",
|
||||
[string]$ExitNodeName = "test-2",
|
||||
[string]$EntryBaseUrl = "http://192.168.200.61:19131",
|
||||
[string]$DockerSSH = "test-docker",
|
||||
[string]$ExpectedBackendImage = "rap-backend:fabric-service-channel-0.2.239",
|
||||
[string]$ExpectedNodeAgentImage = "rap-node-agent:0.2.237",
|
||||
[string]$ResultPath = "artifacts\c18z59-service-channel-remediation-traffic-smoke-result.json"
|
||||
)
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
$scriptDir = Split-Path -Parent $MyInvocation.MyCommand.Path
|
||||
$repoRoot = (Resolve-Path (Join-Path $scriptDir "..\..")).ProviderPath
|
||||
$runId = "c18z59-" + (Get-Date -Format "yyyyMMdd-HHmmss")
|
||||
|
||||
function Invoke-Api {
|
||||
param(
|
||||
[string]$Method,
|
||||
[string]$Path,
|
||||
[object]$Body = $null
|
||||
)
|
||||
if ($null -eq $Body) {
|
||||
return Invoke-RestMethod -Method $Method -Uri "$ApiBaseUrl$Path" -TimeoutSec 30
|
||||
}
|
||||
return Invoke-RestMethod -Method $Method -Uri "$ApiBaseUrl$Path" -ContentType "application/json" -Body ($Body | ConvertTo-Json -Depth 80) -TimeoutSec 30
|
||||
}
|
||||
|
||||
function Get-PropertyValue {
|
||||
param(
|
||||
[object]$Item,
|
||||
[string]$Name,
|
||||
[object]$Default = $null
|
||||
)
|
||||
if ($null -eq $Item) { return $Default }
|
||||
$property = $Item.PSObject.Properties[$Name]
|
||||
if ($null -eq $property) { return $Default }
|
||||
return $property.Value
|
||||
}
|
||||
|
||||
function Get-NodeByName {
|
||||
param([string]$Name)
|
||||
$nodes = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/nodes?actor_user_id=$ActorUserID").nodes
|
||||
$node = @($nodes | Where-Object { $_.name -eq $Name }) | Select-Object -First 1
|
||||
if ($null -eq $node) {
|
||||
throw "Node '$Name' was not found in cluster $ClusterID"
|
||||
}
|
||||
return $node
|
||||
}
|
||||
|
||||
function New-RouteIntent {
|
||||
param(
|
||||
[string]$SourceNodeID,
|
||||
[string]$DestinationNodeID,
|
||||
[string[]]$Hops,
|
||||
[int]$Priority,
|
||||
[string]$Label
|
||||
)
|
||||
$expiresAt = (Get-Date).ToUniversalTime().AddMinutes(5).ToString("o")
|
||||
return Invoke-Api -Method POST -Path "/clusters/$ClusterID/mesh/route-intents" -Body @{
|
||||
actor_user_id = $ActorUserID
|
||||
source_selector = @{ node_id = $SourceNodeID }
|
||||
destination_selector = @{ node_id = $DestinationNodeID }
|
||||
service_class = "vpn_packets"
|
||||
priority = $Priority
|
||||
policy = @{
|
||||
synthetic_enabled = $true
|
||||
route_version = "$runId-$Label"
|
||||
policy_version = "$runId-$Label"
|
||||
peer_directory_version = "$runId-$Label"
|
||||
hops = @($Hops)
|
||||
allowed_channels = @("vpn_packet", "fabric_control")
|
||||
max_ttl = 8
|
||||
max_hops = 8
|
||||
expires_at = $expiresAt
|
||||
metadata = @{
|
||||
smoke = "c18z59_service_channel_remediation_traffic"
|
||||
run_id = $runId
|
||||
label = $Label
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function Send-DegradedHeartbeat {
|
||||
param(
|
||||
[string]$EntryNodeID,
|
||||
[string]$PrimaryRouteID
|
||||
)
|
||||
$observedAt = (Get-Date).ToUniversalTime().ToString("o")
|
||||
return Invoke-Api -Method POST -Path "/clusters/$ClusterID/nodes/$EntryNodeID/heartbeats" -Body @{
|
||||
health_status = "healthy"
|
||||
reported_version = "0.2.236"
|
||||
capabilities = @{
|
||||
fabric_service_channel_runtime = $true
|
||||
fabric_service_channel_route_manager = $true
|
||||
fabric_service_channel_route_quality_feedback = $true
|
||||
smoke_feedback_injection = "c18z59"
|
||||
}
|
||||
service_states = @{ smoke = "c18z59_primary_degraded_alternate_available" }
|
||||
metadata = @{
|
||||
fabric_service_channel_runtime_report = @{
|
||||
schema_version = "c18l.fabric_service_channel_runtime_report.v1"
|
||||
config_version = "$runId-primary"
|
||||
cluster_id = $ClusterID
|
||||
local_node_id = $EntryNodeID
|
||||
observed_at = $observedAt
|
||||
ingress = @{
|
||||
flow_scheduler = @{
|
||||
schema_version = "rap.fabric_flow_scheduler.v1"
|
||||
service_neutral = $true
|
||||
service_mode = "application_protocol_agnostic"
|
||||
channel_stats = @{
|
||||
"c18z59-primary-degraded" = @{
|
||||
last_route_id = $PrimaryRouteID
|
||||
last_failed_route_id = $PrimaryRouteID
|
||||
route_generation = "$runId-primary"
|
||||
last_error = "c18z59 primary route degraded; alternate available"
|
||||
last_send_duration_ms = 980
|
||||
consecutive_failures = 3
|
||||
stall_count = 2
|
||||
route_rebuild_recommended = $true
|
||||
degraded_fallback_recommended = $false
|
||||
quality_window_sample_count = 8
|
||||
quality_window_success_count = 2
|
||||
quality_window_failure_count = 3
|
||||
quality_window_slow_count = 2
|
||||
quality_window_drop_count = 1
|
||||
quality_window_avg_latency_ms = 980
|
||||
quality_window_last_updated_at = $observedAt
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
smoke = @{
|
||||
name = "c18z59_service_channel_remediation_traffic"
|
||||
run_id = $runId
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
$entryNode = Get-NodeByName -Name $EntryNodeName
|
||||
$relayNode = Get-NodeByName -Name $RelayNodeName
|
||||
$exitNode = Get-NodeByName -Name $ExitNodeName
|
||||
$primaryRouteID = ""
|
||||
$alternateRouteID = ""
|
||||
$result = $null
|
||||
|
||||
try {
|
||||
$primary = (New-RouteIntent -SourceNodeID $entryNode.id -DestinationNodeID $exitNode.id -Hops @($entryNode.id, $exitNode.id) -Priority 2100000000 -Label "primary").route_intent
|
||||
$alternate = (New-RouteIntent -SourceNodeID $entryNode.id -DestinationNodeID $exitNode.id -Hops @($entryNode.id, $relayNode.id, $exitNode.id) -Priority 2099999900 -Label "alternate").route_intent
|
||||
$primaryRouteID = [string]$primary.id
|
||||
$alternateRouteID = [string]$alternate.id
|
||||
|
||||
$resourceID = "c18z59-vpn-smoke"
|
||||
$lease = (Invoke-Api -Method POST -Path "/clusters/$ClusterID/fabric/service-channels/leases" -Body @{
|
||||
actor_user_id = $ActorUserID
|
||||
organization_id = "smoke-org"
|
||||
user_id = "smoke-user"
|
||||
resource_id = $resourceID
|
||||
service_class = "vpn_packets"
|
||||
entry_node_ids = @([string]$entryNode.id)
|
||||
exit_node_ids = @([string]$exitNode.id)
|
||||
preferred_entry_node_id = [string]$entryNode.id
|
||||
preferred_exit_node_id = [string]$exitNode.id
|
||||
allowed_channels = @("vpn_packet", "fabric_control")
|
||||
ttl_seconds = 180
|
||||
metadata = @{
|
||||
smoke = "c18z59_service_channel_remediation_traffic"
|
||||
run_id = $runId
|
||||
}
|
||||
}).fabric_service_channel_lease
|
||||
|
||||
[void](Send-DegradedHeartbeat -EntryNodeID $entryNode.id -PrimaryRouteID $primaryRouteID)
|
||||
|
||||
$packetPath = $lease.entry_http.path_template.
|
||||
Replace("{cluster_id}", $ClusterID).
|
||||
Replace("{channel_id}", [string]$lease.channel_id).
|
||||
Replace("{resource_id}", $resourceID)
|
||||
$packetUrl = $EntryBaseUrl.TrimEnd("/") + $packetPath
|
||||
$headers = @{
|
||||
"X-RAP-Service-Channel-Token" = [string]$lease.token.token
|
||||
"X-RAP-Fabric-Channel-ID" = [string]$lease.channel_id
|
||||
"X-RAP-Service-Class" = "vpn_packets"
|
||||
"X-RAP-Channel-Class" = "vpn_packet"
|
||||
}
|
||||
$response = Invoke-WebRequest -Method Post -Uri $packetUrl -Headers $headers -Body ([System.Text.Encoding]::UTF8.GetBytes("c18z59-alternate-remediation")) -ContentType "application/octet-stream" -TimeoutSec 30
|
||||
$acceptedBy = [string]$response.Headers["X-RAP-Service-Channel-Accepted-By"]
|
||||
|
||||
$accessTelemetry = $null
|
||||
$matchingChannel = $null
|
||||
for ($i = 0; $i -lt 10; $i++) {
|
||||
Start-Sleep -Seconds 3
|
||||
$accessTelemetry = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/fabric/service-channels/access-telemetry?actor_user_id=$ActorUserID&limit=20").fabric_service_channel_access_telemetry
|
||||
$channels = @()
|
||||
if ($accessTelemetry.PSObject.Properties.Name -contains "active_channels") {
|
||||
$channels = @($accessTelemetry.active_channels)
|
||||
}
|
||||
$matchingChannel = $channels | Where-Object { $_.channel_id -eq $lease.channel_id } | Select-Object -First 1
|
||||
if ($null -ne $matchingChannel -and [string](Get-PropertyValue -Item $matchingChannel -Name "remediation_action" -Default "") -eq "prefer_alternate_route") {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
$backendLine = (& ssh $DockerSSH "docker ps --format '{{.Names}} {{.Image}} {{.Status}}' | grep '^rap_test_backend '") | Out-String
|
||||
$nodeLines = (& ssh $DockerSSH "docker ps --format '{{.Names}} {{.Image}} {{.Status}}' | grep '^rap_test_node_test_'") | Out-String
|
||||
$leasePrimaryRouteID = [string](Get-PropertyValue -Item (Get-PropertyValue -Item $lease -Name "primary_route" -Default $null) -Name "route_id" -Default "")
|
||||
$leaseAlternates = @()
|
||||
if ($lease.PSObject.Properties.Name -contains "alternate_routes") {
|
||||
$leaseAlternates = @($lease.alternate_routes)
|
||||
}
|
||||
$leaseHasAlternate = (@($leaseAlternates | Where-Object { [string]$_.route_id -eq $alternateRouteID }).Count -ge 1)
|
||||
$remediationCommand = Get-PropertyValue -Item $matchingChannel -Name "remediation_command" -Default $null
|
||||
$commandAction = [string](Get-PropertyValue -Item $remediationCommand -Name "action" -Default "")
|
||||
$commandPrimaryRouteID = [string](Get-PropertyValue -Item $remediationCommand -Name "primary_route_id" -Default "")
|
||||
$commandReplacementRouteID = [string](Get-PropertyValue -Item $remediationCommand -Name "replacement_route_id" -Default "")
|
||||
$commandExpiresAt = [string](Get-PropertyValue -Item $remediationCommand -Name "expires_at" -Default "")
|
||||
$syntheticConfig = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/nodes/$($entryNode.id)/mesh/synthetic-config").synthetic_mesh_config
|
||||
$syntheticCommands = @()
|
||||
if ($syntheticConfig.PSObject.Properties.Name -contains "service_channel_remediation_commands") {
|
||||
$syntheticCommands = @($syntheticConfig.service_channel_remediation_commands)
|
||||
}
|
||||
$syntheticCommand = $syntheticCommands | Where-Object { [string]$_.channel_id -eq [string]$lease.channel_id } | Select-Object -First 1
|
||||
$syntheticCommandAction = [string](Get-PropertyValue -Item $syntheticCommand -Name "action" -Default "")
|
||||
$syntheticCommandReplacementRouteID = [string](Get-PropertyValue -Item $syntheticCommand -Name "replacement_route_id" -Default "")
|
||||
|
||||
$routeManagerDecision = $null
|
||||
$routeManagerRuntime = $null
|
||||
for ($i = 0; $i -lt 10; $i++) {
|
||||
Start-Sleep -Seconds 5
|
||||
$heartbeats = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/nodes/$($entryNode.id)/heartbeats?actor_user_id=$ActorUserID&limit=5").heartbeats
|
||||
foreach ($heartbeat in @($heartbeats)) {
|
||||
$runtimeReport = Get-PropertyValue -Item (Get-PropertyValue -Item $heartbeat -Name "metadata" -Default $null) -Name "fabric_service_channel_runtime_report" -Default $null
|
||||
$ingress = Get-PropertyValue -Item $runtimeReport -Name "ingress" -Default $null
|
||||
$routeManager = Get-PropertyValue -Item $ingress -Name "route_manager" -Default $null
|
||||
$routeManagerRuntime = $routeManager
|
||||
$decisions = @()
|
||||
if ($null -ne $routeManager -and $routeManager.PSObject.Properties.Name -contains "decisions") {
|
||||
$decisions = @($routeManager.decisions)
|
||||
}
|
||||
$routeManagerDecision = $decisions | Where-Object {
|
||||
[string]$_.route_id -eq $primaryRouteID -and
|
||||
[string]$_.replacement_route_id -eq $alternateRouteID -and
|
||||
[string]$_.decision_source -eq "service_channel_remediation_command"
|
||||
} | Select-Object -First 1
|
||||
if ($null -ne $routeManagerDecision) {
|
||||
break
|
||||
}
|
||||
}
|
||||
if ($null -ne $routeManagerDecision) {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
$postRemediationResponse = Invoke-WebRequest -Method Post -Uri $packetUrl -Headers $headers -Body ([System.Text.Encoding]::UTF8.GetBytes("c18z59-remediation-replacement-traffic")) -ContentType "application/octet-stream" -TimeoutSec 30
|
||||
$postRemediationAcceptedBy = [string]$postRemediationResponse.Headers["X-RAP-Service-Channel-Accepted-By"]
|
||||
$replacementTrafficObserved = $false
|
||||
$replacementLastSelected = ""
|
||||
$replacementFlowStat = $null
|
||||
$postRemediationFallbackLocal = 0
|
||||
$postRemediationRouteFailures = 0
|
||||
for ($i = 0; $i -lt 12; $i++) {
|
||||
Start-Sleep -Seconds 5
|
||||
$heartbeats = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/nodes/$($entryNode.id)/heartbeats?actor_user_id=$ActorUserID&limit=5").heartbeats
|
||||
foreach ($heartbeat in @($heartbeats)) {
|
||||
$runtimeReport = Get-PropertyValue -Item (Get-PropertyValue -Item $heartbeat -Name "metadata" -Default $null) -Name "fabric_service_channel_runtime_report" -Default $null
|
||||
$ingress = Get-PropertyValue -Item $runtimeReport -Name "ingress" -Default $null
|
||||
$replacementLastSelected = [string](Get-PropertyValue -Item $ingress -Name "last_selected_route_id" -Default "")
|
||||
$postRemediationFallbackLocal = [int](Get-PropertyValue -Item $ingress -Name "send_fallback_local" -Default 0)
|
||||
$postRemediationRouteFailures = [int](Get-PropertyValue -Item $ingress -Name "send_route_failures" -Default 0)
|
||||
$flowScheduler = Get-PropertyValue -Item $ingress -Name "flow_scheduler" -Default $null
|
||||
$channelStats = Get-PropertyValue -Item $flowScheduler -Name "channel_stats" -Default $null
|
||||
if ($null -ne $channelStats) {
|
||||
foreach ($property in @($channelStats.PSObject.Properties)) {
|
||||
$stat = $property.Value
|
||||
if ([string](Get-PropertyValue -Item $stat -Name "last_route_id" -Default "") -eq $alternateRouteID) {
|
||||
$replacementFlowStat = $stat
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
if ($replacementLastSelected -eq $alternateRouteID -or $null -ne $replacementFlowStat) {
|
||||
$replacementTrafficObserved = $true
|
||||
break
|
||||
}
|
||||
}
|
||||
if ($replacementTrafficObserved) {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
$result = [ordered]@{
|
||||
schema_version = "c18z59.service_channel_remediation_traffic_smoke.v1"
|
||||
run_id = $runId
|
||||
cluster_id = $ClusterID
|
||||
primary_route_id = $primaryRouteID
|
||||
alternate_route_id = $alternateRouteID
|
||||
channel_id = [string]$lease.channel_id
|
||||
passed = [bool](
|
||||
$backendLine.Contains($ExpectedBackendImage) -and
|
||||
$nodeLines.Contains($ExpectedNodeAgentImage) -and
|
||||
[string]$lease.status -eq "ready" -and
|
||||
$leasePrimaryRouteID -eq $primaryRouteID -and
|
||||
$leaseHasAlternate -and
|
||||
[int]$response.StatusCode -eq 202 -and
|
||||
$acceptedBy -eq "introspection" -and
|
||||
$null -ne $matchingChannel -and
|
||||
[string](Get-PropertyValue -Item $matchingChannel -Name "primary_route_id" -Default "") -eq $primaryRouteID -and
|
||||
-not [bool](Get-PropertyValue -Item $matchingChannel -Name "force_backend_fallback" -Default $false) -and
|
||||
[string](Get-PropertyValue -Item $matchingChannel -Name "route_feedback_status" -Default "") -eq "fenced" -and
|
||||
[string](Get-PropertyValue -Item $matchingChannel -Name "remediation_action" -Default "") -eq "prefer_alternate_route" -and
|
||||
[string](Get-PropertyValue -Item $matchingChannel -Name "remediation_route_id" -Default "") -eq $alternateRouteID -and
|
||||
$null -ne $remediationCommand -and
|
||||
$commandAction -eq "prefer_alternate_route" -and
|
||||
$commandPrimaryRouteID -eq $primaryRouteID -and
|
||||
$commandReplacementRouteID -eq $alternateRouteID -and
|
||||
$commandExpiresAt.Length -gt 0 -and
|
||||
$null -ne $syntheticCommand -and
|
||||
$syntheticCommandAction -eq "prefer_alternate_route" -and
|
||||
$syntheticCommandReplacementRouteID -eq $alternateRouteID -and
|
||||
$null -ne $routeManagerDecision -and
|
||||
[string](Get-PropertyValue -Item $routeManagerDecision -Name "rebuild_status" -Default "") -eq "applied" -and
|
||||
[int]$postRemediationResponse.StatusCode -eq 202 -and
|
||||
$postRemediationAcceptedBy -eq "introspection" -and
|
||||
$replacementTrafficObserved -and
|
||||
$replacementLastSelected -eq $alternateRouteID -and
|
||||
$postRemediationFallbackLocal -eq 0 -and
|
||||
[int]$accessTelemetry.degraded_fallback_channel_count -eq 0
|
||||
)
|
||||
checks = [ordered]@{
|
||||
backend_expected_image_deployed = $backendLine.Contains($ExpectedBackendImage)
|
||||
node_agent_expected_image_deployed = $nodeLines.Contains($ExpectedNodeAgentImage)
|
||||
lease_ready = ([string]$lease.status -eq "ready")
|
||||
lease_selected_primary_route = ($leasePrimaryRouteID -eq $primaryRouteID)
|
||||
lease_contains_alternate_route = $leaseHasAlternate
|
||||
packet_accepted = ([int]$response.StatusCode -eq 202)
|
||||
accepted_by_header_is_introspection = ($acceptedBy -eq "introspection")
|
||||
active_channel_visible = ($null -ne $matchingChannel)
|
||||
active_channel_not_backend_fallback = ($null -ne $matchingChannel -and -not [bool](Get-PropertyValue -Item $matchingChannel -Name "force_backend_fallback" -Default $false))
|
||||
route_quality_fenced = ($null -ne $matchingChannel -and [string](Get-PropertyValue -Item $matchingChannel -Name "route_feedback_status" -Default "") -eq "fenced")
|
||||
remediation_prefers_alternate = ($null -ne $matchingChannel -and [string](Get-PropertyValue -Item $matchingChannel -Name "remediation_action" -Default "") -eq "prefer_alternate_route")
|
||||
remediation_route_is_alternate = ($null -ne $matchingChannel -and [string](Get-PropertyValue -Item $matchingChannel -Name "remediation_route_id" -Default "") -eq $alternateRouteID)
|
||||
remediation_command_visible = ($null -ne $remediationCommand)
|
||||
remediation_command_prefers_alternate = ($commandAction -eq "prefer_alternate_route")
|
||||
remediation_command_primary_route_matches = ($commandPrimaryRouteID -eq $primaryRouteID)
|
||||
remediation_command_replacement_route_matches = ($commandReplacementRouteID -eq $alternateRouteID)
|
||||
remediation_command_has_ttl = ($commandExpiresAt.Length -gt 0)
|
||||
synthetic_config_command_visible = ($null -ne $syntheticCommand)
|
||||
synthetic_config_command_prefers_alternate = ($syntheticCommandAction -eq "prefer_alternate_route")
|
||||
synthetic_config_command_replacement_route_matches = ($syntheticCommandReplacementRouteID -eq $alternateRouteID)
|
||||
node_route_manager_consumed_command = ($null -ne $routeManagerDecision)
|
||||
node_route_manager_applied_replacement = ($null -ne $routeManagerDecision -and [string](Get-PropertyValue -Item $routeManagerDecision -Name "rebuild_status" -Default "") -eq "applied")
|
||||
post_remediation_packet_accepted = ([int]$postRemediationResponse.StatusCode -eq 202)
|
||||
post_remediation_accepted_by_introspection = ($postRemediationAcceptedBy -eq "introspection")
|
||||
replacement_traffic_observed = $replacementTrafficObserved
|
||||
replacement_last_selected_route_matches = ($replacementLastSelected -eq $alternateRouteID)
|
||||
replacement_flow_stat_observed = ($null -ne $replacementFlowStat)
|
||||
no_local_gateway_fallback_after_replacement = ($postRemediationFallbackLocal -eq 0)
|
||||
backend_fallback_not_recommended = ([int]$accessTelemetry.degraded_fallback_channel_count -eq 0)
|
||||
}
|
||||
summary = [ordered]@{
|
||||
backend_container = $backendLine.Trim()
|
||||
node_containers = $nodeLines.Trim()
|
||||
accepted_by = $acceptedBy
|
||||
lease_status = [string]$lease.status
|
||||
lease_primary_route_id = $leasePrimaryRouteID
|
||||
lease_alternate_route_count = $leaseAlternates.Count
|
||||
access_status = [string]$accessTelemetry.status
|
||||
active_channel_count = [int]$accessTelemetry.active_channel_count
|
||||
correlated_route_count = [int]$accessTelemetry.correlated_route_count
|
||||
degraded_route_count = [int]$accessTelemetry.degraded_route_count
|
||||
degraded_fallback_channel_count = [int]$accessTelemetry.degraded_fallback_channel_count
|
||||
synthetic_command = $syntheticCommand
|
||||
route_manager_decision = $routeManagerDecision
|
||||
route_manager_runtime = $routeManagerRuntime
|
||||
post_remediation_accepted_by = $postRemediationAcceptedBy
|
||||
replacement_last_selected_route_id = $replacementLastSelected
|
||||
replacement_flow_stat = $replacementFlowStat
|
||||
post_remediation_send_fallback_local = $postRemediationFallbackLocal
|
||||
post_remediation_send_route_failures = $postRemediationRouteFailures
|
||||
matching_channel = $matchingChannel
|
||||
}
|
||||
}
|
||||
$failedChecks = @($result.checks.GetEnumerator() | Where-Object { $_.Value -ne $true })
|
||||
if ($failedChecks.Count -gt 0) {
|
||||
throw "C18Z59 failed checks: $($failedChecks.Name -join ', ')"
|
||||
}
|
||||
}
|
||||
finally {
|
||||
if ($primaryRouteID) {
|
||||
try { Invoke-Api -Method POST -Path "/clusters/$ClusterID/mesh/route-intents/$primaryRouteID/expire" -Body @{ actor_user_id = $ActorUserID } | Out-Null } catch {}
|
||||
}
|
||||
if ($alternateRouteID) {
|
||||
try { Invoke-Api -Method POST -Path "/clusters/$ClusterID/mesh/route-intents/$alternateRouteID/expire" -Body @{ actor_user_id = $ActorUserID } | Out-Null } catch {}
|
||||
}
|
||||
}
|
||||
|
||||
$resultFullPath = Join-Path $repoRoot $ResultPath
|
||||
$resultDir = Split-Path -Parent $resultFullPath
|
||||
if (-not (Test-Path $resultDir)) {
|
||||
New-Item -ItemType Directory -Path $resultDir | Out-Null
|
||||
}
|
||||
$result | ConvertTo-Json -Depth 100 | Set-Content -Path $resultFullPath -Encoding UTF8
|
||||
|
||||
Write-Host "C18Z59 service-channel remediation traffic smoke passed. Result: $resultFullPath"
|
||||
$result
|
||||
|
||||
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,499 @@
|
||||
param(
|
||||
[string]$ApiBaseUrl = "http://192.168.200.61:18121/api/v1",
|
||||
[string]$ClusterID = "cfc0743d-d960-49fb-9de8-96e063d5e4aa",
|
||||
[string]$ActorUserID = "f67d943f-5397-4b3a-a229-695fe67ad700",
|
||||
[string]$EntryNodeName = "test-1",
|
||||
[string]$RelayNodeName = "test-3",
|
||||
[string]$ExitNodeName = "test-2",
|
||||
[string]$EntryBaseUrl = "http://192.168.200.61:19131",
|
||||
[string]$DockerSSH = "test-docker",
|
||||
[string]$ExpectedBackendImage = "rap-backend:fabric-service-channel-0.2.239",
|
||||
[string]$ExpectedNodeAgentImage = "rap-node-agent:0.2.237",
|
||||
[string]$ResultPath = "artifacts\c18z60-service-channel-remediation-multiflow-smoke-result.json"
|
||||
)
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
$scriptDir = Split-Path -Parent $MyInvocation.MyCommand.Path
|
||||
$repoRoot = (Resolve-Path (Join-Path $scriptDir "..\..")).ProviderPath
|
||||
$runId = "c18z60-" + (Get-Date -Format "yyyyMMdd-HHmmss")
|
||||
|
||||
function Invoke-Api {
|
||||
param(
|
||||
[string]$Method,
|
||||
[string]$Path,
|
||||
[object]$Body = $null
|
||||
)
|
||||
if ($null -eq $Body) {
|
||||
return Invoke-RestMethod -Method $Method -Uri "$ApiBaseUrl$Path" -TimeoutSec 30
|
||||
}
|
||||
return Invoke-RestMethod -Method $Method -Uri "$ApiBaseUrl$Path" -ContentType "application/json" -Body ($Body | ConvertTo-Json -Depth 80) -TimeoutSec 30
|
||||
}
|
||||
|
||||
function Get-PropertyValue {
|
||||
param(
|
||||
[object]$Item,
|
||||
[string]$Name,
|
||||
[object]$Default = $null
|
||||
)
|
||||
if ($null -eq $Item) { return $Default }
|
||||
$property = $Item.PSObject.Properties[$Name]
|
||||
if ($null -eq $property) { return $Default }
|
||||
return $property.Value
|
||||
}
|
||||
|
||||
function Get-NodeByName {
|
||||
param([string]$Name)
|
||||
$nodes = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/nodes?actor_user_id=$ActorUserID").nodes
|
||||
$node = @($nodes | Where-Object { $_.name -eq $Name }) | Select-Object -First 1
|
||||
if ($null -eq $node) {
|
||||
throw "Node '$Name' was not found in cluster $ClusterID"
|
||||
}
|
||||
return $node
|
||||
}
|
||||
|
||||
function New-IPv4TcpPacket {
|
||||
param(
|
||||
[byte[]]$Source = @(10, 77, 0, 2),
|
||||
[byte[]]$Destination = @(192, 168, 200, 95),
|
||||
[int]$SourcePort,
|
||||
[int]$DestinationPort = 3389
|
||||
)
|
||||
$packet = [byte[]]::new(40)
|
||||
$packet[0] = 0x45
|
||||
$packet[2] = 0
|
||||
$packet[3] = 40
|
||||
$packet[8] = 64
|
||||
$packet[9] = 6
|
||||
[Array]::Copy($Source, 0, $packet, 12, 4)
|
||||
[Array]::Copy($Destination, 0, $packet, 16, 4)
|
||||
$packet[20] = [byte](($SourcePort -shr 8) -band 0xff)
|
||||
$packet[21] = [byte]($SourcePort -band 0xff)
|
||||
$packet[22] = [byte](($DestinationPort -shr 8) -band 0xff)
|
||||
$packet[23] = [byte]($DestinationPort -band 0xff)
|
||||
$packet[32] = 0x50
|
||||
return $packet
|
||||
}
|
||||
|
||||
function ConvertTo-VPNPacketBatch {
|
||||
param([byte[][]]$Packets)
|
||||
$buffer = New-Object System.Collections.Generic.List[byte]
|
||||
foreach ($packet in $Packets) {
|
||||
if ($null -eq $packet -or $packet.Length -eq 0) { continue }
|
||||
$size = [uint32]$packet.Length
|
||||
$buffer.Add([byte](($size -shr 24) -band 0xff))
|
||||
$buffer.Add([byte](($size -shr 16) -band 0xff))
|
||||
$buffer.Add([byte](($size -shr 8) -band 0xff))
|
||||
$buffer.Add([byte]($size -band 0xff))
|
||||
foreach ($value in $packet) {
|
||||
$buffer.Add($value)
|
||||
}
|
||||
}
|
||||
return $buffer.ToArray()
|
||||
}
|
||||
|
||||
function New-RouteIntent {
|
||||
param(
|
||||
[string]$SourceNodeID,
|
||||
[string]$DestinationNodeID,
|
||||
[string[]]$Hops,
|
||||
[int]$Priority,
|
||||
[string]$Label
|
||||
)
|
||||
$expiresAt = (Get-Date).ToUniversalTime().AddMinutes(5).ToString("o")
|
||||
return Invoke-Api -Method POST -Path "/clusters/$ClusterID/mesh/route-intents" -Body @{
|
||||
actor_user_id = $ActorUserID
|
||||
source_selector = @{ node_id = $SourceNodeID }
|
||||
destination_selector = @{ node_id = $DestinationNodeID }
|
||||
service_class = "vpn_packets"
|
||||
priority = $Priority
|
||||
policy = @{
|
||||
synthetic_enabled = $true
|
||||
route_version = "$runId-$Label"
|
||||
policy_version = "$runId-$Label"
|
||||
peer_directory_version = "$runId-$Label"
|
||||
hops = @($Hops)
|
||||
allowed_channels = @("vpn_packet", "fabric_control")
|
||||
max_ttl = 8
|
||||
max_hops = 8
|
||||
expires_at = $expiresAt
|
||||
metadata = @{
|
||||
smoke = "c18z60_service_channel_remediation_multiflow"
|
||||
run_id = $runId
|
||||
label = $Label
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function Send-DegradedHeartbeat {
|
||||
param(
|
||||
[string]$EntryNodeID,
|
||||
[string]$PrimaryRouteID
|
||||
)
|
||||
$observedAt = (Get-Date).ToUniversalTime().ToString("o")
|
||||
return Invoke-Api -Method POST -Path "/clusters/$ClusterID/nodes/$EntryNodeID/heartbeats" -Body @{
|
||||
health_status = "healthy"
|
||||
reported_version = "0.2.236"
|
||||
capabilities = @{
|
||||
fabric_service_channel_runtime = $true
|
||||
fabric_service_channel_route_manager = $true
|
||||
fabric_service_channel_route_quality_feedback = $true
|
||||
smoke_feedback_injection = "c18z60"
|
||||
}
|
||||
service_states = @{ smoke = "c18z60_primary_degraded_alternate_available" }
|
||||
metadata = @{
|
||||
fabric_service_channel_runtime_report = @{
|
||||
schema_version = "c18l.fabric_service_channel_runtime_report.v1"
|
||||
config_version = "$runId-primary"
|
||||
cluster_id = $ClusterID
|
||||
local_node_id = $EntryNodeID
|
||||
observed_at = $observedAt
|
||||
ingress = @{
|
||||
flow_scheduler = @{
|
||||
schema_version = "rap.fabric_flow_scheduler.v1"
|
||||
service_neutral = $true
|
||||
service_mode = "application_protocol_agnostic"
|
||||
channel_stats = @{
|
||||
"c18z60-primary-degraded" = @{
|
||||
last_route_id = $PrimaryRouteID
|
||||
last_failed_route_id = $PrimaryRouteID
|
||||
route_generation = "$runId-primary"
|
||||
last_error = "c18z60 primary route degraded; alternate available"
|
||||
last_send_duration_ms = 980
|
||||
consecutive_failures = 3
|
||||
stall_count = 2
|
||||
route_rebuild_recommended = $true
|
||||
degraded_fallback_recommended = $false
|
||||
quality_window_sample_count = 8
|
||||
quality_window_success_count = 2
|
||||
quality_window_failure_count = 3
|
||||
quality_window_slow_count = 2
|
||||
quality_window_drop_count = 1
|
||||
quality_window_avg_latency_ms = 980
|
||||
quality_window_last_updated_at = $observedAt
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
smoke = @{
|
||||
name = "c18z60_service_channel_remediation_multiflow"
|
||||
run_id = $runId
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
$entryNode = Get-NodeByName -Name $EntryNodeName
|
||||
$relayNode = Get-NodeByName -Name $RelayNodeName
|
||||
$exitNode = Get-NodeByName -Name $ExitNodeName
|
||||
$primaryRouteID = ""
|
||||
$alternateRouteID = ""
|
||||
$result = $null
|
||||
$failedChecks = @()
|
||||
|
||||
try {
|
||||
$primary = (New-RouteIntent -SourceNodeID $entryNode.id -DestinationNodeID $exitNode.id -Hops @($entryNode.id, $exitNode.id) -Priority 2100000000 -Label "primary").route_intent
|
||||
$alternate = (New-RouteIntent -SourceNodeID $entryNode.id -DestinationNodeID $exitNode.id -Hops @($entryNode.id, $relayNode.id, $exitNode.id) -Priority 2099999900 -Label "alternate").route_intent
|
||||
$primaryRouteID = [string]$primary.id
|
||||
$alternateRouteID = [string]$alternate.id
|
||||
|
||||
$resourceID = "c18z60-vpn-smoke"
|
||||
$lease = (Invoke-Api -Method POST -Path "/clusters/$ClusterID/fabric/service-channels/leases" -Body @{
|
||||
actor_user_id = $ActorUserID
|
||||
organization_id = "smoke-org"
|
||||
user_id = "smoke-user"
|
||||
resource_id = $resourceID
|
||||
service_class = "vpn_packets"
|
||||
entry_node_ids = @([string]$entryNode.id)
|
||||
exit_node_ids = @([string]$exitNode.id)
|
||||
preferred_entry_node_id = [string]$entryNode.id
|
||||
preferred_exit_node_id = [string]$exitNode.id
|
||||
allowed_channels = @("vpn_packet", "fabric_control")
|
||||
ttl_seconds = 180
|
||||
metadata = @{
|
||||
smoke = "c18z60_service_channel_remediation_multiflow"
|
||||
run_id = $runId
|
||||
}
|
||||
}).fabric_service_channel_lease
|
||||
|
||||
[void](Send-DegradedHeartbeat -EntryNodeID $entryNode.id -PrimaryRouteID $primaryRouteID)
|
||||
|
||||
$packetPath = $lease.entry_http.path_template.
|
||||
Replace("{cluster_id}", $ClusterID).
|
||||
Replace("{channel_id}", [string]$lease.channel_id).
|
||||
Replace("{resource_id}", $resourceID)
|
||||
$packetUrl = $EntryBaseUrl.TrimEnd("/") + $packetPath
|
||||
$headers = @{
|
||||
"X-RAP-Service-Channel-Token" = [string]$lease.token.token
|
||||
"X-RAP-Fabric-Channel-ID" = [string]$lease.channel_id
|
||||
"X-RAP-Service-Class" = "vpn_packets"
|
||||
"X-RAP-Channel-Class" = "vpn_packet"
|
||||
}
|
||||
$response = Invoke-WebRequest -Method Post -Uri $packetUrl -Headers $headers -Body ([System.Text.Encoding]::UTF8.GetBytes("c18z60-alternate-remediation")) -ContentType "application/octet-stream" -TimeoutSec 30
|
||||
$acceptedBy = [string]$response.Headers["X-RAP-Service-Channel-Accepted-By"]
|
||||
|
||||
$accessTelemetry = $null
|
||||
$matchingChannel = $null
|
||||
for ($i = 0; $i -lt 10; $i++) {
|
||||
Start-Sleep -Seconds 3
|
||||
$accessTelemetry = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/fabric/service-channels/access-telemetry?actor_user_id=$ActorUserID&limit=20").fabric_service_channel_access_telemetry
|
||||
$channels = @()
|
||||
if ($accessTelemetry.PSObject.Properties.Name -contains "active_channels") {
|
||||
$channels = @($accessTelemetry.active_channels)
|
||||
}
|
||||
$matchingChannel = $channels | Where-Object { $_.channel_id -eq $lease.channel_id } | Select-Object -First 1
|
||||
if ($null -ne $matchingChannel -and [string](Get-PropertyValue -Item $matchingChannel -Name "remediation_action" -Default "") -eq "prefer_alternate_route") {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
$backendLine = (& ssh $DockerSSH "docker ps --format '{{.Names}} {{.Image}} {{.Status}}' | grep '^rap_test_backend '") | Out-String
|
||||
$nodeLines = (& ssh $DockerSSH "docker ps --format '{{.Names}} {{.Image}} {{.Status}}' | grep '^rap_test_node_test_'") | Out-String
|
||||
$leasePrimaryRouteID = [string](Get-PropertyValue -Item (Get-PropertyValue -Item $lease -Name "primary_route" -Default $null) -Name "route_id" -Default "")
|
||||
$leaseAlternates = @()
|
||||
if ($lease.PSObject.Properties.Name -contains "alternate_routes") {
|
||||
$leaseAlternates = @($lease.alternate_routes)
|
||||
}
|
||||
$leaseHasAlternate = (@($leaseAlternates | Where-Object { [string]$_.route_id -eq $alternateRouteID }).Count -ge 1)
|
||||
$remediationCommand = Get-PropertyValue -Item $matchingChannel -Name "remediation_command" -Default $null
|
||||
$commandAction = [string](Get-PropertyValue -Item $remediationCommand -Name "action" -Default "")
|
||||
$commandPrimaryRouteID = [string](Get-PropertyValue -Item $remediationCommand -Name "primary_route_id" -Default "")
|
||||
$commandReplacementRouteID = [string](Get-PropertyValue -Item $remediationCommand -Name "replacement_route_id" -Default "")
|
||||
$commandExpiresAt = [string](Get-PropertyValue -Item $remediationCommand -Name "expires_at" -Default "")
|
||||
$expectedReplacementRouteID = $commandReplacementRouteID
|
||||
$syntheticConfig = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/nodes/$($entryNode.id)/mesh/synthetic-config").synthetic_mesh_config
|
||||
$syntheticCommands = @()
|
||||
if ($syntheticConfig.PSObject.Properties.Name -contains "service_channel_remediation_commands") {
|
||||
$syntheticCommands = @($syntheticConfig.service_channel_remediation_commands)
|
||||
}
|
||||
$syntheticCommand = $syntheticCommands | Where-Object { [string]$_.channel_id -eq [string]$lease.channel_id } | Select-Object -First 1
|
||||
$syntheticCommandAction = [string](Get-PropertyValue -Item $syntheticCommand -Name "action" -Default "")
|
||||
$syntheticCommandReplacementRouteID = [string](Get-PropertyValue -Item $syntheticCommand -Name "replacement_route_id" -Default "")
|
||||
if ([string]::IsNullOrWhiteSpace($expectedReplacementRouteID)) {
|
||||
$expectedReplacementRouteID = $syntheticCommandReplacementRouteID
|
||||
}
|
||||
if ([string]::IsNullOrWhiteSpace($expectedReplacementRouteID)) {
|
||||
$expectedReplacementRouteID = $alternateRouteID
|
||||
}
|
||||
|
||||
$routeManagerDecision = $null
|
||||
$routeManagerRuntime = $null
|
||||
for ($i = 0; $i -lt 10; $i++) {
|
||||
Start-Sleep -Seconds 5
|
||||
$heartbeats = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/nodes/$($entryNode.id)/heartbeats?actor_user_id=$ActorUserID&limit=5").heartbeats
|
||||
foreach ($heartbeat in @($heartbeats)) {
|
||||
$runtimeReport = Get-PropertyValue -Item (Get-PropertyValue -Item $heartbeat -Name "metadata" -Default $null) -Name "fabric_service_channel_runtime_report" -Default $null
|
||||
$ingress = Get-PropertyValue -Item $runtimeReport -Name "ingress" -Default $null
|
||||
$routeManager = Get-PropertyValue -Item $ingress -Name "route_manager" -Default $null
|
||||
$routeManagerRuntime = $routeManager
|
||||
$decisions = @()
|
||||
if ($null -ne $routeManager -and $routeManager.PSObject.Properties.Name -contains "decisions") {
|
||||
$decisions = @($routeManager.decisions)
|
||||
}
|
||||
$routeManagerDecision = $decisions | Where-Object {
|
||||
[string]$_.route_id -eq $primaryRouteID -and
|
||||
[string]$_.replacement_route_id -eq $expectedReplacementRouteID -and
|
||||
[string]$_.decision_source -eq "service_channel_remediation_command"
|
||||
} | Select-Object -First 1
|
||||
if ($null -ne $routeManagerDecision) {
|
||||
break
|
||||
}
|
||||
}
|
||||
if ($null -ne $routeManagerDecision) {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
$multiFlowPackets = @()
|
||||
foreach ($port in @(51000, 51001, 51002, 51003, 51004, 51005, 51006, 51007, 51008, 51009, 51010, 51011)) {
|
||||
$multiFlowPackets += ,(New-IPv4TcpPacket -SourcePort $port -DestinationPort 3389)
|
||||
}
|
||||
$multiFlowPayload = ConvertTo-VPNPacketBatch -Packets $multiFlowPackets
|
||||
$multiFlowPayloadPath = Join-Path ([System.IO.Path]::GetTempPath()) "$runId-vpn-packet-batch.bin"
|
||||
[System.IO.File]::WriteAllBytes($multiFlowPayloadPath, [byte[]]$multiFlowPayload)
|
||||
$postRemediationResponse = Invoke-WebRequest -Method Post -Uri ($packetUrl + "?batch=true") -Headers $headers -InFile $multiFlowPayloadPath -ContentType "application/vnd.rap.vpn-packet-batch.v1" -TimeoutSec 30
|
||||
$postRemediationAcceptedBy = [string]$postRemediationResponse.Headers["X-RAP-Service-Channel-Accepted-By"]
|
||||
$replacementTrafficObserved = $false
|
||||
$replacementLastSelected = ""
|
||||
$replacementFlowStat = $null
|
||||
$replacementFlowStats = @()
|
||||
$postRemediationFallbackLocal = 0
|
||||
$postRemediationRouteFailures = 0
|
||||
$postRemediationFlowDropped = 0
|
||||
$postRemediationSchedulerDropped = 0
|
||||
$postRemediationHighWatermark = 0
|
||||
$postRemediationMaxInFlight = 0
|
||||
for ($i = 0; $i -lt 12; $i++) {
|
||||
Start-Sleep -Seconds 5
|
||||
$heartbeats = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/nodes/$($entryNode.id)/heartbeats?actor_user_id=$ActorUserID&limit=5").heartbeats
|
||||
foreach ($heartbeat in @($heartbeats)) {
|
||||
$runtimeReport = Get-PropertyValue -Item (Get-PropertyValue -Item $heartbeat -Name "metadata" -Default $null) -Name "fabric_service_channel_runtime_report" -Default $null
|
||||
$ingress = Get-PropertyValue -Item $runtimeReport -Name "ingress" -Default $null
|
||||
$replacementLastSelected = [string](Get-PropertyValue -Item $ingress -Name "last_selected_route_id" -Default "")
|
||||
$postRemediationFallbackLocal = [int](Get-PropertyValue -Item $ingress -Name "send_fallback_local" -Default 0)
|
||||
$postRemediationRouteFailures = [int](Get-PropertyValue -Item $ingress -Name "send_route_failures" -Default 0)
|
||||
$postRemediationFlowDropped = [int](Get-PropertyValue -Item $ingress -Name "send_flow_dropped" -Default 0)
|
||||
$flowScheduler = Get-PropertyValue -Item $ingress -Name "flow_scheduler" -Default $null
|
||||
$postRemediationSchedulerDropped = [int](Get-PropertyValue -Item $flowScheduler -Name "dropped" -Default 0)
|
||||
$postRemediationHighWatermark = [int](Get-PropertyValue -Item $flowScheduler -Name "high_watermark" -Default 0)
|
||||
$postRemediationMaxInFlight = [int](Get-PropertyValue -Item $flowScheduler -Name "max_in_flight" -Default 0)
|
||||
$channelStats = Get-PropertyValue -Item $flowScheduler -Name "channel_stats" -Default $null
|
||||
if ($null -ne $channelStats) {
|
||||
$replacementFlowStats = @()
|
||||
foreach ($property in @($channelStats.PSObject.Properties)) {
|
||||
$stat = $property.Value
|
||||
if ([string](Get-PropertyValue -Item $stat -Name "last_route_id" -Default "") -eq $expectedReplacementRouteID) {
|
||||
$replacementFlowStats += $stat
|
||||
$replacementFlowStat = $stat
|
||||
}
|
||||
}
|
||||
}
|
||||
if ($replacementLastSelected -eq $expectedReplacementRouteID -and $replacementFlowStats.Count -ge 2) {
|
||||
$replacementTrafficObserved = $true
|
||||
break
|
||||
}
|
||||
}
|
||||
if ($replacementTrafficObserved) {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
$result = [ordered]@{
|
||||
schema_version = "c18z60.service_channel_remediation_multiflow_smoke.v1"
|
||||
run_id = $runId
|
||||
cluster_id = $ClusterID
|
||||
primary_route_id = $primaryRouteID
|
||||
alternate_route_id = $alternateRouteID
|
||||
expected_replacement_route_id = $expectedReplacementRouteID
|
||||
channel_id = [string]$lease.channel_id
|
||||
passed = [bool](
|
||||
$backendLine.Contains($ExpectedBackendImage) -and
|
||||
$nodeLines.Contains($ExpectedNodeAgentImage) -and
|
||||
[string]$lease.status -eq "ready" -and
|
||||
$leasePrimaryRouteID -eq $primaryRouteID -and
|
||||
$leaseHasAlternate -and
|
||||
[int]$response.StatusCode -eq 202 -and
|
||||
$acceptedBy -eq "introspection" -and
|
||||
$null -ne $matchingChannel -and
|
||||
[string](Get-PropertyValue -Item $matchingChannel -Name "primary_route_id" -Default "") -eq $primaryRouteID -and
|
||||
-not [bool](Get-PropertyValue -Item $matchingChannel -Name "force_backend_fallback" -Default $false) -and
|
||||
[string](Get-PropertyValue -Item $matchingChannel -Name "route_feedback_status" -Default "") -eq "fenced" -and
|
||||
[string](Get-PropertyValue -Item $matchingChannel -Name "remediation_action" -Default "") -eq "prefer_alternate_route" -and
|
||||
[string](Get-PropertyValue -Item $matchingChannel -Name "remediation_route_id" -Default "") -eq $expectedReplacementRouteID -and
|
||||
$null -ne $remediationCommand -and
|
||||
$commandAction -eq "prefer_alternate_route" -and
|
||||
$commandPrimaryRouteID -eq $primaryRouteID -and
|
||||
$commandReplacementRouteID -eq $expectedReplacementRouteID -and
|
||||
$commandExpiresAt.Length -gt 0 -and
|
||||
$null -ne $syntheticCommand -and
|
||||
$syntheticCommandAction -eq "prefer_alternate_route" -and
|
||||
$syntheticCommandReplacementRouteID -eq $expectedReplacementRouteID -and
|
||||
$null -ne $routeManagerDecision -and
|
||||
[string](Get-PropertyValue -Item $routeManagerDecision -Name "rebuild_status" -Default "") -eq "applied" -and
|
||||
[int]$postRemediationResponse.StatusCode -eq 202 -and
|
||||
$postRemediationAcceptedBy -eq "introspection" -and
|
||||
$replacementTrafficObserved -and
|
||||
$replacementLastSelected -eq $expectedReplacementRouteID -and
|
||||
$replacementFlowStats.Count -ge 2 -and
|
||||
$postRemediationFallbackLocal -eq 0 -and
|
||||
$postRemediationFlowDropped -eq 0 -and
|
||||
$postRemediationSchedulerDropped -eq 0 -and
|
||||
$postRemediationRouteFailures -eq 0 -and
|
||||
[int]$accessTelemetry.degraded_fallback_channel_count -eq 0
|
||||
)
|
||||
checks = [ordered]@{
|
||||
backend_expected_image_deployed = $backendLine.Contains($ExpectedBackendImage)
|
||||
node_agent_expected_image_deployed = $nodeLines.Contains($ExpectedNodeAgentImage)
|
||||
lease_ready = ([string]$lease.status -eq "ready")
|
||||
lease_selected_primary_route = ($leasePrimaryRouteID -eq $primaryRouteID)
|
||||
lease_contains_alternate_route = $leaseHasAlternate
|
||||
packet_accepted = ([int]$response.StatusCode -eq 202)
|
||||
accepted_by_header_is_introspection = ($acceptedBy -eq "introspection")
|
||||
active_channel_visible = ($null -ne $matchingChannel)
|
||||
active_channel_not_backend_fallback = ($null -ne $matchingChannel -and -not [bool](Get-PropertyValue -Item $matchingChannel -Name "force_backend_fallback" -Default $false))
|
||||
route_quality_fenced = ($null -ne $matchingChannel -and [string](Get-PropertyValue -Item $matchingChannel -Name "route_feedback_status" -Default "") -eq "fenced")
|
||||
remediation_prefers_alternate = ($null -ne $matchingChannel -and [string](Get-PropertyValue -Item $matchingChannel -Name "remediation_action" -Default "") -eq "prefer_alternate_route")
|
||||
remediation_route_is_alternate = ($null -ne $matchingChannel -and [string](Get-PropertyValue -Item $matchingChannel -Name "remediation_route_id" -Default "") -eq $expectedReplacementRouteID)
|
||||
remediation_command_visible = ($null -ne $remediationCommand)
|
||||
remediation_command_prefers_alternate = ($commandAction -eq "prefer_alternate_route")
|
||||
remediation_command_primary_route_matches = ($commandPrimaryRouteID -eq $primaryRouteID)
|
||||
remediation_command_replacement_route_matches = ($commandReplacementRouteID -eq $expectedReplacementRouteID)
|
||||
remediation_command_has_ttl = ($commandExpiresAt.Length -gt 0)
|
||||
synthetic_config_command_visible = ($null -ne $syntheticCommand)
|
||||
synthetic_config_command_prefers_alternate = ($syntheticCommandAction -eq "prefer_alternate_route")
|
||||
synthetic_config_command_replacement_route_matches = ($syntheticCommandReplacementRouteID -eq $expectedReplacementRouteID)
|
||||
node_route_manager_consumed_command = ($null -ne $routeManagerDecision)
|
||||
node_route_manager_applied_replacement = ($null -ne $routeManagerDecision -and [string](Get-PropertyValue -Item $routeManagerDecision -Name "rebuild_status" -Default "") -eq "applied")
|
||||
post_remediation_packet_accepted = ([int]$postRemediationResponse.StatusCode -eq 202)
|
||||
post_remediation_accepted_by_introspection = ($postRemediationAcceptedBy -eq "introspection")
|
||||
replacement_traffic_observed = $replacementTrafficObserved
|
||||
replacement_last_selected_route_matches = ($replacementLastSelected -eq $expectedReplacementRouteID)
|
||||
replacement_flow_stat_observed = ($null -ne $replacementFlowStat)
|
||||
replacement_multiflow_stats_observed = ($replacementFlowStats.Count -ge 2)
|
||||
no_local_gateway_fallback_after_replacement = ($postRemediationFallbackLocal -eq 0)
|
||||
no_flow_drops_after_replacement = ($postRemediationFlowDropped -eq 0 -and $postRemediationSchedulerDropped -eq 0)
|
||||
no_route_failures_after_replacement = ($postRemediationRouteFailures -eq 0)
|
||||
backend_fallback_not_recommended = ([int]$accessTelemetry.degraded_fallback_channel_count -eq 0)
|
||||
}
|
||||
summary = [ordered]@{
|
||||
backend_container = $backendLine.Trim()
|
||||
node_containers = $nodeLines.Trim()
|
||||
accepted_by = $acceptedBy
|
||||
lease_status = [string]$lease.status
|
||||
lease_primary_route_id = $leasePrimaryRouteID
|
||||
lease_alternate_route_count = $leaseAlternates.Count
|
||||
access_status = [string]$accessTelemetry.status
|
||||
active_channel_count = [int]$accessTelemetry.active_channel_count
|
||||
correlated_route_count = [int]$accessTelemetry.correlated_route_count
|
||||
degraded_route_count = [int]$accessTelemetry.degraded_route_count
|
||||
degraded_fallback_channel_count = [int]$accessTelemetry.degraded_fallback_channel_count
|
||||
synthetic_command = $syntheticCommand
|
||||
route_manager_decision = $routeManagerDecision
|
||||
route_manager_runtime = $routeManagerRuntime
|
||||
post_remediation_accepted_by = $postRemediationAcceptedBy
|
||||
post_remediation_packet_count = $multiFlowPackets.Count
|
||||
replacement_last_selected_route_id = $replacementLastSelected
|
||||
replacement_flow_stat_count = $replacementFlowStats.Count
|
||||
replacement_flow_stat = $replacementFlowStat
|
||||
replacement_flow_stats = $replacementFlowStats
|
||||
post_remediation_send_fallback_local = $postRemediationFallbackLocal
|
||||
post_remediation_send_route_failures = $postRemediationRouteFailures
|
||||
post_remediation_send_flow_dropped = $postRemediationFlowDropped
|
||||
post_remediation_scheduler_dropped = $postRemediationSchedulerDropped
|
||||
post_remediation_scheduler_high_watermark = $postRemediationHighWatermark
|
||||
post_remediation_scheduler_max_in_flight = $postRemediationMaxInFlight
|
||||
matching_channel = $matchingChannel
|
||||
}
|
||||
}
|
||||
$failedChecks = @($result.checks.GetEnumerator() | Where-Object { $_.Value -ne $true })
|
||||
$result.summary.failed_checks = @($failedChecks | ForEach-Object { $_.Key })
|
||||
}
|
||||
finally {
|
||||
if ($primaryRouteID) {
|
||||
try { Invoke-Api -Method POST -Path "/clusters/$ClusterID/mesh/route-intents/$primaryRouteID/expire" -Body @{ actor_user_id = $ActorUserID } | Out-Null } catch {}
|
||||
}
|
||||
if ($alternateRouteID) {
|
||||
try { Invoke-Api -Method POST -Path "/clusters/$ClusterID/mesh/route-intents/$alternateRouteID/expire" -Body @{ actor_user_id = $ActorUserID } | Out-Null } catch {}
|
||||
}
|
||||
}
|
||||
|
||||
$resultFullPath = Join-Path $repoRoot $ResultPath
|
||||
$resultDir = Split-Path -Parent $resultFullPath
|
||||
if (-not (Test-Path $resultDir)) {
|
||||
New-Item -ItemType Directory -Path $resultDir | Out-Null
|
||||
}
|
||||
$result | ConvertTo-Json -Depth 100 | Set-Content -Path $resultFullPath -Encoding UTF8
|
||||
|
||||
if ($failedChecks.Count -gt 0) {
|
||||
throw "C18Z60 failed checks: $(@($failedChecks | ForEach-Object { $_.Key }) -join ', '). Result: $resultFullPath"
|
||||
}
|
||||
|
||||
Write-Host "C18Z60 service-channel remediation multiflow smoke passed. Result: $resultFullPath"
|
||||
$result
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,506 @@
|
||||
param(
|
||||
[string]$ApiBaseUrl = "http://192.168.200.61:18121/api/v1",
|
||||
[string]$ClusterID = "cfc0743d-d960-49fb-9de8-96e063d5e4aa",
|
||||
[string]$ActorUserID = "f67d943f-5397-4b3a-a229-695fe67ad700",
|
||||
[string]$EntryNodeName = "test-1",
|
||||
[string]$RelayNodeName = "test-3",
|
||||
[string]$ExitNodeName = "test-2",
|
||||
[string]$EntryBaseUrl = "http://192.168.200.61:19131",
|
||||
[string]$DockerSSH = "test-docker",
|
||||
[string]$ExpectedBackendImage = "rap-backend:fabric-service-channel-0.2.239",
|
||||
[string]$ExpectedNodeAgentImage = "rap-node-agent:0.2.237",
|
||||
[string]$ResultPath = "artifacts\c18z61-service-channel-remediation-pressure-smoke-result.json"
|
||||
)
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
$scriptDir = Split-Path -Parent $MyInvocation.MyCommand.Path
|
||||
$repoRoot = (Resolve-Path (Join-Path $scriptDir "..\..")).ProviderPath
|
||||
$runId = "c18z61-" + (Get-Date -Format "yyyyMMdd-HHmmss")
|
||||
|
||||
function Invoke-Api {
|
||||
param(
|
||||
[string]$Method,
|
||||
[string]$Path,
|
||||
[object]$Body = $null
|
||||
)
|
||||
if ($null -eq $Body) {
|
||||
return Invoke-RestMethod -Method $Method -Uri "$ApiBaseUrl$Path" -TimeoutSec 30
|
||||
}
|
||||
return Invoke-RestMethod -Method $Method -Uri "$ApiBaseUrl$Path" -ContentType "application/json" -Body ($Body | ConvertTo-Json -Depth 80) -TimeoutSec 30
|
||||
}
|
||||
|
||||
function Get-PropertyValue {
|
||||
param(
|
||||
[object]$Item,
|
||||
[string]$Name,
|
||||
[object]$Default = $null
|
||||
)
|
||||
if ($null -eq $Item) { return $Default }
|
||||
$property = $Item.PSObject.Properties[$Name]
|
||||
if ($null -eq $property) { return $Default }
|
||||
return $property.Value
|
||||
}
|
||||
|
||||
function Get-NodeByName {
|
||||
param([string]$Name)
|
||||
$nodes = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/nodes?actor_user_id=$ActorUserID").nodes
|
||||
$node = @($nodes | Where-Object { $_.name -eq $Name }) | Select-Object -First 1
|
||||
if ($null -eq $node) {
|
||||
throw "Node '$Name' was not found in cluster $ClusterID"
|
||||
}
|
||||
return $node
|
||||
}
|
||||
|
||||
function New-IPv4TcpPacket {
|
||||
param(
|
||||
[byte[]]$Source = @(10, 77, 0, 2),
|
||||
[byte[]]$Destination = @(192, 168, 200, 95),
|
||||
[int]$SourcePort,
|
||||
[int]$DestinationPort = 3389
|
||||
)
|
||||
$packet = [byte[]]::new(40)
|
||||
$packet[0] = 0x45
|
||||
$packet[2] = 0
|
||||
$packet[3] = 40
|
||||
$packet[8] = 64
|
||||
$packet[9] = 6
|
||||
[Array]::Copy($Source, 0, $packet, 12, 4)
|
||||
[Array]::Copy($Destination, 0, $packet, 16, 4)
|
||||
$packet[20] = [byte](($SourcePort -shr 8) -band 0xff)
|
||||
$packet[21] = [byte]($SourcePort -band 0xff)
|
||||
$packet[22] = [byte](($DestinationPort -shr 8) -band 0xff)
|
||||
$packet[23] = [byte]($DestinationPort -band 0xff)
|
||||
$packet[32] = 0x50
|
||||
return $packet
|
||||
}
|
||||
|
||||
function ConvertTo-VPNPacketBatch {
|
||||
param([byte[][]]$Packets)
|
||||
$buffer = New-Object System.Collections.Generic.List[byte]
|
||||
foreach ($packet in $Packets) {
|
||||
if ($null -eq $packet -or $packet.Length -eq 0) { continue }
|
||||
$size = [uint32]$packet.Length
|
||||
$buffer.Add([byte](($size -shr 24) -band 0xff))
|
||||
$buffer.Add([byte](($size -shr 16) -band 0xff))
|
||||
$buffer.Add([byte](($size -shr 8) -band 0xff))
|
||||
$buffer.Add([byte]($size -band 0xff))
|
||||
foreach ($value in $packet) {
|
||||
$buffer.Add($value)
|
||||
}
|
||||
}
|
||||
return $buffer.ToArray()
|
||||
}
|
||||
|
||||
function New-RouteIntent {
|
||||
param(
|
||||
[string]$SourceNodeID,
|
||||
[string]$DestinationNodeID,
|
||||
[string[]]$Hops,
|
||||
[int]$Priority,
|
||||
[string]$Label
|
||||
)
|
||||
$expiresAt = (Get-Date).ToUniversalTime().AddMinutes(5).ToString("o")
|
||||
return Invoke-Api -Method POST -Path "/clusters/$ClusterID/mesh/route-intents" -Body @{
|
||||
actor_user_id = $ActorUserID
|
||||
source_selector = @{ node_id = $SourceNodeID }
|
||||
destination_selector = @{ node_id = $DestinationNodeID }
|
||||
service_class = "vpn_packets"
|
||||
priority = $Priority
|
||||
policy = @{
|
||||
synthetic_enabled = $true
|
||||
route_version = "$runId-$Label"
|
||||
policy_version = "$runId-$Label"
|
||||
peer_directory_version = "$runId-$Label"
|
||||
hops = @($Hops)
|
||||
allowed_channels = @("vpn_packet", "fabric_control")
|
||||
max_ttl = 8
|
||||
max_hops = 8
|
||||
expires_at = $expiresAt
|
||||
metadata = @{
|
||||
smoke = "c18z61_service_channel_remediation_pressure"
|
||||
run_id = $runId
|
||||
label = $Label
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function Send-DegradedHeartbeat {
|
||||
param(
|
||||
[string]$EntryNodeID,
|
||||
[string]$PrimaryRouteID
|
||||
)
|
||||
$observedAt = (Get-Date).ToUniversalTime().ToString("o")
|
||||
return Invoke-Api -Method POST -Path "/clusters/$ClusterID/nodes/$EntryNodeID/heartbeats" -Body @{
|
||||
health_status = "healthy"
|
||||
reported_version = "0.2.236"
|
||||
capabilities = @{
|
||||
fabric_service_channel_runtime = $true
|
||||
fabric_service_channel_route_manager = $true
|
||||
fabric_service_channel_route_quality_feedback = $true
|
||||
smoke_feedback_injection = "c18z60"
|
||||
}
|
||||
service_states = @{ smoke = "c18z61_primary_degraded_alternate_available" }
|
||||
metadata = @{
|
||||
fabric_service_channel_runtime_report = @{
|
||||
schema_version = "c18l.fabric_service_channel_runtime_report.v1"
|
||||
config_version = "$runId-primary"
|
||||
cluster_id = $ClusterID
|
||||
local_node_id = $EntryNodeID
|
||||
observed_at = $observedAt
|
||||
ingress = @{
|
||||
flow_scheduler = @{
|
||||
schema_version = "rap.fabric_flow_scheduler.v1"
|
||||
service_neutral = $true
|
||||
service_mode = "application_protocol_agnostic"
|
||||
channel_stats = @{
|
||||
"c18z61-primary-degraded" = @{
|
||||
last_route_id = $PrimaryRouteID
|
||||
last_failed_route_id = $PrimaryRouteID
|
||||
route_generation = "$runId-primary"
|
||||
last_error = "c18z60 primary route degraded; alternate available"
|
||||
last_send_duration_ms = 980
|
||||
consecutive_failures = 3
|
||||
stall_count = 2
|
||||
route_rebuild_recommended = $true
|
||||
degraded_fallback_recommended = $false
|
||||
quality_window_sample_count = 8
|
||||
quality_window_success_count = 2
|
||||
quality_window_failure_count = 3
|
||||
quality_window_slow_count = 2
|
||||
quality_window_drop_count = 1
|
||||
quality_window_avg_latency_ms = 980
|
||||
quality_window_last_updated_at = $observedAt
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
smoke = @{
|
||||
name = "c18z61_service_channel_remediation_pressure"
|
||||
run_id = $runId
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
$entryNode = Get-NodeByName -Name $EntryNodeName
|
||||
$relayNode = Get-NodeByName -Name $RelayNodeName
|
||||
$exitNode = Get-NodeByName -Name $ExitNodeName
|
||||
$primaryRouteID = ""
|
||||
$alternateRouteID = ""
|
||||
$result = $null
|
||||
$failedChecks = @()
|
||||
$expectedPressurePacketCount = 128
|
||||
$expectedPressureFlowCount = 8
|
||||
|
||||
try {
|
||||
$primary = (New-RouteIntent -SourceNodeID $entryNode.id -DestinationNodeID $exitNode.id -Hops @($entryNode.id, $exitNode.id) -Priority 2100000000 -Label "primary").route_intent
|
||||
$alternate = (New-RouteIntent -SourceNodeID $entryNode.id -DestinationNodeID $exitNode.id -Hops @($entryNode.id, $relayNode.id, $exitNode.id) -Priority 2099999900 -Label "alternate").route_intent
|
||||
$primaryRouteID = [string]$primary.id
|
||||
$alternateRouteID = [string]$alternate.id
|
||||
|
||||
$resourceID = "c18z61-vpn-smoke"
|
||||
$lease = (Invoke-Api -Method POST -Path "/clusters/$ClusterID/fabric/service-channels/leases" -Body @{
|
||||
actor_user_id = $ActorUserID
|
||||
organization_id = "smoke-org"
|
||||
user_id = "smoke-user"
|
||||
resource_id = $resourceID
|
||||
service_class = "vpn_packets"
|
||||
entry_node_ids = @([string]$entryNode.id)
|
||||
exit_node_ids = @([string]$exitNode.id)
|
||||
preferred_entry_node_id = [string]$entryNode.id
|
||||
preferred_exit_node_id = [string]$exitNode.id
|
||||
allowed_channels = @("vpn_packet", "fabric_control")
|
||||
ttl_seconds = 180
|
||||
metadata = @{
|
||||
smoke = "c18z61_service_channel_remediation_pressure"
|
||||
run_id = $runId
|
||||
}
|
||||
}).fabric_service_channel_lease
|
||||
|
||||
[void](Send-DegradedHeartbeat -EntryNodeID $entryNode.id -PrimaryRouteID $primaryRouteID)
|
||||
|
||||
$packetPath = $lease.entry_http.path_template.
|
||||
Replace("{cluster_id}", $ClusterID).
|
||||
Replace("{channel_id}", [string]$lease.channel_id).
|
||||
Replace("{resource_id}", $resourceID)
|
||||
$packetUrl = $EntryBaseUrl.TrimEnd("/") + $packetPath
|
||||
$headers = @{
|
||||
"X-RAP-Service-Channel-Token" = [string]$lease.token.token
|
||||
"X-RAP-Fabric-Channel-ID" = [string]$lease.channel_id
|
||||
"X-RAP-Service-Class" = "vpn_packets"
|
||||
"X-RAP-Channel-Class" = "vpn_packet"
|
||||
}
|
||||
$response = Invoke-WebRequest -Method Post -Uri $packetUrl -Headers $headers -Body ([System.Text.Encoding]::UTF8.GetBytes("c18z61-alternate-remediation")) -ContentType "application/octet-stream" -TimeoutSec 30
|
||||
$acceptedBy = [string]$response.Headers["X-RAP-Service-Channel-Accepted-By"]
|
||||
|
||||
$accessTelemetry = $null
|
||||
$matchingChannel = $null
|
||||
for ($i = 0; $i -lt 10; $i++) {
|
||||
Start-Sleep -Seconds 3
|
||||
$accessTelemetry = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/fabric/service-channels/access-telemetry?actor_user_id=$ActorUserID&limit=20").fabric_service_channel_access_telemetry
|
||||
$channels = @()
|
||||
if ($accessTelemetry.PSObject.Properties.Name -contains "active_channels") {
|
||||
$channels = @($accessTelemetry.active_channels)
|
||||
}
|
||||
$matchingChannel = $channels | Where-Object { $_.channel_id -eq $lease.channel_id } | Select-Object -First 1
|
||||
if ($null -ne $matchingChannel -and [string](Get-PropertyValue -Item $matchingChannel -Name "remediation_action" -Default "") -eq "prefer_alternate_route") {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
$backendLine = (& ssh $DockerSSH "docker ps --format '{{.Names}} {{.Image}} {{.Status}}' | grep '^rap_test_backend '") | Out-String
|
||||
$nodeLines = (& ssh $DockerSSH "docker ps --format '{{.Names}} {{.Image}} {{.Status}}' | grep '^rap_test_node_test_'") | Out-String
|
||||
$leasePrimaryRouteID = [string](Get-PropertyValue -Item (Get-PropertyValue -Item $lease -Name "primary_route" -Default $null) -Name "route_id" -Default "")
|
||||
$leaseAlternates = @()
|
||||
if ($lease.PSObject.Properties.Name -contains "alternate_routes") {
|
||||
$leaseAlternates = @($lease.alternate_routes)
|
||||
}
|
||||
$leaseHasAlternate = (@($leaseAlternates | Where-Object { [string]$_.route_id -eq $alternateRouteID }).Count -ge 1)
|
||||
$remediationCommand = Get-PropertyValue -Item $matchingChannel -Name "remediation_command" -Default $null
|
||||
$commandAction = [string](Get-PropertyValue -Item $remediationCommand -Name "action" -Default "")
|
||||
$commandPrimaryRouteID = [string](Get-PropertyValue -Item $remediationCommand -Name "primary_route_id" -Default "")
|
||||
$commandReplacementRouteID = [string](Get-PropertyValue -Item $remediationCommand -Name "replacement_route_id" -Default "")
|
||||
$commandExpiresAt = [string](Get-PropertyValue -Item $remediationCommand -Name "expires_at" -Default "")
|
||||
$expectedReplacementRouteID = $commandReplacementRouteID
|
||||
$syntheticConfig = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/nodes/$($entryNode.id)/mesh/synthetic-config").synthetic_mesh_config
|
||||
$syntheticCommands = @()
|
||||
if ($syntheticConfig.PSObject.Properties.Name -contains "service_channel_remediation_commands") {
|
||||
$syntheticCommands = @($syntheticConfig.service_channel_remediation_commands)
|
||||
}
|
||||
$syntheticCommand = $syntheticCommands | Where-Object { [string]$_.channel_id -eq [string]$lease.channel_id } | Select-Object -First 1
|
||||
$syntheticCommandAction = [string](Get-PropertyValue -Item $syntheticCommand -Name "action" -Default "")
|
||||
$syntheticCommandReplacementRouteID = [string](Get-PropertyValue -Item $syntheticCommand -Name "replacement_route_id" -Default "")
|
||||
if ([string]::IsNullOrWhiteSpace($expectedReplacementRouteID)) {
|
||||
$expectedReplacementRouteID = $syntheticCommandReplacementRouteID
|
||||
}
|
||||
if ([string]::IsNullOrWhiteSpace($expectedReplacementRouteID)) {
|
||||
$expectedReplacementRouteID = $alternateRouteID
|
||||
}
|
||||
|
||||
$routeManagerDecision = $null
|
||||
$routeManagerRuntime = $null
|
||||
for ($i = 0; $i -lt 10; $i++) {
|
||||
Start-Sleep -Seconds 5
|
||||
$heartbeats = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/nodes/$($entryNode.id)/heartbeats?actor_user_id=$ActorUserID&limit=5").heartbeats
|
||||
foreach ($heartbeat in @($heartbeats)) {
|
||||
$runtimeReport = Get-PropertyValue -Item (Get-PropertyValue -Item $heartbeat -Name "metadata" -Default $null) -Name "fabric_service_channel_runtime_report" -Default $null
|
||||
$ingress = Get-PropertyValue -Item $runtimeReport -Name "ingress" -Default $null
|
||||
$routeManager = Get-PropertyValue -Item $ingress -Name "route_manager" -Default $null
|
||||
$routeManagerRuntime = $routeManager
|
||||
$decisions = @()
|
||||
if ($null -ne $routeManager -and $routeManager.PSObject.Properties.Name -contains "decisions") {
|
||||
$decisions = @($routeManager.decisions)
|
||||
}
|
||||
$routeManagerDecision = $decisions | Where-Object {
|
||||
[string]$_.route_id -eq $primaryRouteID -and
|
||||
[string]$_.replacement_route_id -eq $expectedReplacementRouteID -and
|
||||
[string]$_.decision_source -eq "service_channel_remediation_command"
|
||||
} | Select-Object -First 1
|
||||
if ($null -ne $routeManagerDecision) {
|
||||
break
|
||||
}
|
||||
}
|
||||
if ($null -ne $routeManagerDecision) {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
$multiFlowPackets = @()
|
||||
foreach ($offset in 0..($expectedPressurePacketCount - 1)) {
|
||||
$port = 52000 + $offset
|
||||
$multiFlowPackets += ,(New-IPv4TcpPacket -SourcePort $port -DestinationPort 3389)
|
||||
}
|
||||
$multiFlowPayload = ConvertTo-VPNPacketBatch -Packets $multiFlowPackets
|
||||
$multiFlowPayloadPath = Join-Path ([System.IO.Path]::GetTempPath()) "$runId-vpn-packet-batch.bin"
|
||||
[System.IO.File]::WriteAllBytes($multiFlowPayloadPath, [byte[]]$multiFlowPayload)
|
||||
$postRemediationResponse = Invoke-WebRequest -Method Post -Uri ($packetUrl + "?batch=true") -Headers $headers -InFile $multiFlowPayloadPath -ContentType "application/vnd.rap.vpn-packet-batch.v1" -TimeoutSec 30
|
||||
$postRemediationAcceptedBy = [string]$postRemediationResponse.Headers["X-RAP-Service-Channel-Accepted-By"]
|
||||
$replacementTrafficObserved = $false
|
||||
$replacementLastSelected = ""
|
||||
$replacementFlowStat = $null
|
||||
$replacementFlowStats = @()
|
||||
$postRemediationFallbackLocal = 0
|
||||
$postRemediationRouteFailures = 0
|
||||
$postRemediationFlowDropped = 0
|
||||
$postRemediationSchedulerDropped = 0
|
||||
$postRemediationHighWatermark = 0
|
||||
$postRemediationMaxInFlight = 0
|
||||
for ($i = 0; $i -lt 12; $i++) {
|
||||
Start-Sleep -Seconds 5
|
||||
$heartbeats = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/nodes/$($entryNode.id)/heartbeats?actor_user_id=$ActorUserID&limit=5").heartbeats
|
||||
foreach ($heartbeat in @($heartbeats)) {
|
||||
$runtimeReport = Get-PropertyValue -Item (Get-PropertyValue -Item $heartbeat -Name "metadata" -Default $null) -Name "fabric_service_channel_runtime_report" -Default $null
|
||||
$ingress = Get-PropertyValue -Item $runtimeReport -Name "ingress" -Default $null
|
||||
$replacementLastSelected = [string](Get-PropertyValue -Item $ingress -Name "last_selected_route_id" -Default "")
|
||||
$postRemediationFallbackLocal = [int](Get-PropertyValue -Item $ingress -Name "send_fallback_local" -Default 0)
|
||||
$postRemediationRouteFailures = [int](Get-PropertyValue -Item $ingress -Name "send_route_failures" -Default 0)
|
||||
$postRemediationFlowDropped = [int](Get-PropertyValue -Item $ingress -Name "send_flow_dropped" -Default 0)
|
||||
$flowScheduler = Get-PropertyValue -Item $ingress -Name "flow_scheduler" -Default $null
|
||||
$postRemediationSchedulerDropped = [int](Get-PropertyValue -Item $flowScheduler -Name "dropped" -Default 0)
|
||||
$postRemediationHighWatermark = [int](Get-PropertyValue -Item $flowScheduler -Name "high_watermark" -Default 0)
|
||||
$postRemediationMaxInFlight = [int](Get-PropertyValue -Item $flowScheduler -Name "max_in_flight" -Default 0)
|
||||
$channelStats = Get-PropertyValue -Item $flowScheduler -Name "channel_stats" -Default $null
|
||||
if ($null -ne $channelStats) {
|
||||
$replacementFlowStats = @()
|
||||
foreach ($property in @($channelStats.PSObject.Properties)) {
|
||||
$stat = $property.Value
|
||||
if ([string](Get-PropertyValue -Item $stat -Name "last_route_id" -Default "") -eq $expectedReplacementRouteID) {
|
||||
$replacementFlowStats += $stat
|
||||
$replacementFlowStat = $stat
|
||||
}
|
||||
}
|
||||
}
|
||||
if ($replacementLastSelected -eq $expectedReplacementRouteID -and $replacementFlowStats.Count -ge $expectedPressureFlowCount) {
|
||||
$replacementTrafficObserved = $true
|
||||
break
|
||||
}
|
||||
}
|
||||
if ($replacementTrafficObserved) {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
$result = [ordered]@{
|
||||
schema_version = "c18z61.service_channel_remediation_pressure_smoke.v1"
|
||||
run_id = $runId
|
||||
cluster_id = $ClusterID
|
||||
primary_route_id = $primaryRouteID
|
||||
alternate_route_id = $alternateRouteID
|
||||
expected_replacement_route_id = $expectedReplacementRouteID
|
||||
channel_id = [string]$lease.channel_id
|
||||
passed = [bool](
|
||||
$backendLine.Contains($ExpectedBackendImage) -and
|
||||
$nodeLines.Contains($ExpectedNodeAgentImage) -and
|
||||
[string]$lease.status -eq "ready" -and
|
||||
$leasePrimaryRouteID -eq $primaryRouteID -and
|
||||
$leaseHasAlternate -and
|
||||
[int]$response.StatusCode -eq 202 -and
|
||||
$acceptedBy -eq "introspection" -and
|
||||
$null -ne $matchingChannel -and
|
||||
[string](Get-PropertyValue -Item $matchingChannel -Name "primary_route_id" -Default "") -eq $primaryRouteID -and
|
||||
-not [bool](Get-PropertyValue -Item $matchingChannel -Name "force_backend_fallback" -Default $false) -and
|
||||
[string](Get-PropertyValue -Item $matchingChannel -Name "route_feedback_status" -Default "") -eq "fenced" -and
|
||||
[string](Get-PropertyValue -Item $matchingChannel -Name "remediation_action" -Default "") -eq "prefer_alternate_route" -and
|
||||
[string](Get-PropertyValue -Item $matchingChannel -Name "remediation_route_id" -Default "") -eq $expectedReplacementRouteID -and
|
||||
$null -ne $remediationCommand -and
|
||||
$commandAction -eq "prefer_alternate_route" -and
|
||||
$commandPrimaryRouteID -eq $primaryRouteID -and
|
||||
$commandReplacementRouteID -eq $expectedReplacementRouteID -and
|
||||
$commandExpiresAt.Length -gt 0 -and
|
||||
$null -ne $syntheticCommand -and
|
||||
$syntheticCommandAction -eq "prefer_alternate_route" -and
|
||||
$syntheticCommandReplacementRouteID -eq $expectedReplacementRouteID -and
|
||||
$null -ne $routeManagerDecision -and
|
||||
[string](Get-PropertyValue -Item $routeManagerDecision -Name "rebuild_status" -Default "") -eq "applied" -and
|
||||
[int]$postRemediationResponse.StatusCode -eq 202 -and
|
||||
$postRemediationAcceptedBy -eq "introspection" -and
|
||||
$replacementTrafficObserved -and
|
||||
$replacementLastSelected -eq $expectedReplacementRouteID -and
|
||||
$replacementFlowStats.Count -ge $expectedPressureFlowCount -and
|
||||
$multiFlowPackets.Count -eq $expectedPressurePacketCount -and
|
||||
$postRemediationFallbackLocal -eq 0 -and
|
||||
$postRemediationFlowDropped -eq 0 -and
|
||||
$postRemediationSchedulerDropped -eq 0 -and
|
||||
$postRemediationRouteFailures -eq 0 -and
|
||||
[int]$accessTelemetry.degraded_fallback_channel_count -eq 0
|
||||
)
|
||||
checks = [ordered]@{
|
||||
backend_expected_image_deployed = $backendLine.Contains($ExpectedBackendImage)
|
||||
node_agent_expected_image_deployed = $nodeLines.Contains($ExpectedNodeAgentImage)
|
||||
lease_ready = ([string]$lease.status -eq "ready")
|
||||
lease_selected_primary_route = ($leasePrimaryRouteID -eq $primaryRouteID)
|
||||
lease_contains_alternate_route = $leaseHasAlternate
|
||||
packet_accepted = ([int]$response.StatusCode -eq 202)
|
||||
accepted_by_header_is_introspection = ($acceptedBy -eq "introspection")
|
||||
active_channel_visible = ($null -ne $matchingChannel)
|
||||
active_channel_not_backend_fallback = ($null -ne $matchingChannel -and -not [bool](Get-PropertyValue -Item $matchingChannel -Name "force_backend_fallback" -Default $false))
|
||||
route_quality_fenced = ($null -ne $matchingChannel -and [string](Get-PropertyValue -Item $matchingChannel -Name "route_feedback_status" -Default "") -eq "fenced")
|
||||
remediation_prefers_alternate = ($null -ne $matchingChannel -and [string](Get-PropertyValue -Item $matchingChannel -Name "remediation_action" -Default "") -eq "prefer_alternate_route")
|
||||
remediation_route_is_alternate = ($null -ne $matchingChannel -and [string](Get-PropertyValue -Item $matchingChannel -Name "remediation_route_id" -Default "") -eq $expectedReplacementRouteID)
|
||||
remediation_command_visible = ($null -ne $remediationCommand)
|
||||
remediation_command_prefers_alternate = ($commandAction -eq "prefer_alternate_route")
|
||||
remediation_command_primary_route_matches = ($commandPrimaryRouteID -eq $primaryRouteID)
|
||||
remediation_command_replacement_route_matches = ($commandReplacementRouteID -eq $expectedReplacementRouteID)
|
||||
remediation_command_has_ttl = ($commandExpiresAt.Length -gt 0)
|
||||
synthetic_config_command_visible = ($null -ne $syntheticCommand)
|
||||
synthetic_config_command_prefers_alternate = ($syntheticCommandAction -eq "prefer_alternate_route")
|
||||
synthetic_config_command_replacement_route_matches = ($syntheticCommandReplacementRouteID -eq $expectedReplacementRouteID)
|
||||
node_route_manager_consumed_command = ($null -ne $routeManagerDecision)
|
||||
node_route_manager_applied_replacement = ($null -ne $routeManagerDecision -and [string](Get-PropertyValue -Item $routeManagerDecision -Name "rebuild_status" -Default "") -eq "applied")
|
||||
post_remediation_packet_accepted = ([int]$postRemediationResponse.StatusCode -eq 202)
|
||||
post_remediation_accepted_by_introspection = ($postRemediationAcceptedBy -eq "introspection")
|
||||
replacement_traffic_observed = $replacementTrafficObserved
|
||||
replacement_last_selected_route_matches = ($replacementLastSelected -eq $expectedReplacementRouteID)
|
||||
replacement_flow_stat_observed = ($null -ne $replacementFlowStat)
|
||||
replacement_pressure_flow_stats_observed = ($replacementFlowStats.Count -ge $expectedPressureFlowCount)
|
||||
pressure_packet_count_matches = ($multiFlowPackets.Count -eq $expectedPressurePacketCount)
|
||||
no_local_gateway_fallback_after_replacement = ($postRemediationFallbackLocal -eq 0)
|
||||
no_flow_drops_after_replacement = ($postRemediationFlowDropped -eq 0 -and $postRemediationSchedulerDropped -eq 0)
|
||||
no_route_failures_after_replacement = ($postRemediationRouteFailures -eq 0)
|
||||
backend_fallback_not_recommended = ([int]$accessTelemetry.degraded_fallback_channel_count -eq 0)
|
||||
}
|
||||
summary = [ordered]@{
|
||||
backend_container = $backendLine.Trim()
|
||||
node_containers = $nodeLines.Trim()
|
||||
accepted_by = $acceptedBy
|
||||
lease_status = [string]$lease.status
|
||||
lease_primary_route_id = $leasePrimaryRouteID
|
||||
lease_alternate_route_count = $leaseAlternates.Count
|
||||
access_status = [string]$accessTelemetry.status
|
||||
active_channel_count = [int]$accessTelemetry.active_channel_count
|
||||
correlated_route_count = [int]$accessTelemetry.correlated_route_count
|
||||
degraded_route_count = [int]$accessTelemetry.degraded_route_count
|
||||
degraded_fallback_channel_count = [int]$accessTelemetry.degraded_fallback_channel_count
|
||||
synthetic_command = $syntheticCommand
|
||||
route_manager_decision = $routeManagerDecision
|
||||
route_manager_runtime = $routeManagerRuntime
|
||||
post_remediation_accepted_by = $postRemediationAcceptedBy
|
||||
post_remediation_packet_count = $multiFlowPackets.Count
|
||||
expected_pressure_flow_count = $expectedPressureFlowCount
|
||||
replacement_last_selected_route_id = $replacementLastSelected
|
||||
replacement_flow_stat_count = $replacementFlowStats.Count
|
||||
replacement_flow_stat = $replacementFlowStat
|
||||
replacement_flow_stats = $replacementFlowStats
|
||||
post_remediation_send_fallback_local = $postRemediationFallbackLocal
|
||||
post_remediation_send_route_failures = $postRemediationRouteFailures
|
||||
post_remediation_send_flow_dropped = $postRemediationFlowDropped
|
||||
post_remediation_scheduler_dropped = $postRemediationSchedulerDropped
|
||||
post_remediation_scheduler_high_watermark = $postRemediationHighWatermark
|
||||
post_remediation_scheduler_max_in_flight = $postRemediationMaxInFlight
|
||||
matching_channel = $matchingChannel
|
||||
}
|
||||
}
|
||||
$failedChecks = @($result.checks.GetEnumerator() | Where-Object { $_.Value -ne $true })
|
||||
$result.summary.failed_checks = @($failedChecks | ForEach-Object { $_.Key })
|
||||
}
|
||||
finally {
|
||||
if ($primaryRouteID) {
|
||||
try { Invoke-Api -Method POST -Path "/clusters/$ClusterID/mesh/route-intents/$primaryRouteID/expire" -Body @{ actor_user_id = $ActorUserID } | Out-Null } catch {}
|
||||
}
|
||||
if ($alternateRouteID) {
|
||||
try { Invoke-Api -Method POST -Path "/clusters/$ClusterID/mesh/route-intents/$alternateRouteID/expire" -Body @{ actor_user_id = $ActorUserID } | Out-Null } catch {}
|
||||
}
|
||||
}
|
||||
|
||||
$resultFullPath = Join-Path $repoRoot $ResultPath
|
||||
$resultDir = Split-Path -Parent $resultFullPath
|
||||
if (-not (Test-Path $resultDir)) {
|
||||
New-Item -ItemType Directory -Path $resultDir | Out-Null
|
||||
}
|
||||
$result | ConvertTo-Json -Depth 100 | Set-Content -Path $resultFullPath -Encoding UTF8
|
||||
|
||||
if ($failedChecks.Count -gt 0) {
|
||||
throw "C18Z61 failed checks: $(@($failedChecks | ForEach-Object { $_.Key }) -join ', '). Result: $resultFullPath"
|
||||
}
|
||||
|
||||
Write-Host "C18Z61 service-channel remediation pressure smoke passed. Result: $resultFullPath"
|
||||
$result
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,541 @@
|
||||
param(
|
||||
[string]$ApiBaseUrl = "http://192.168.200.61:18121/api/v1",
|
||||
[string]$ClusterID = "cfc0743d-d960-49fb-9de8-96e063d5e4aa",
|
||||
[string]$ActorUserID = "f67d943f-5397-4b3a-a229-695fe67ad700",
|
||||
[string]$EntryNodeName = "test-1",
|
||||
[string]$RelayNodeName = "test-3",
|
||||
[string]$ExitNodeName = "test-2",
|
||||
[string]$EntryBaseUrl = "http://192.168.200.61:19131",
|
||||
[string]$DockerSSH = "test-docker",
|
||||
[string]$ExpectedBackendImage = "rap-backend:fabric-service-channel-0.2.239",
|
||||
[string]$ExpectedNodeAgentImage = "rap-node-agent:0.2.238-c18z62",
|
||||
[string]$ResultPath = "artifacts\c18z62-service-channel-remediation-qos-smoke-result.json"
|
||||
)
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
$scriptDir = Split-Path -Parent $MyInvocation.MyCommand.Path
|
||||
$repoRoot = (Resolve-Path (Join-Path $scriptDir "..\..")).ProviderPath
|
||||
$runId = "c18z62-" + (Get-Date -Format "yyyyMMdd-HHmmss")
|
||||
|
||||
function Invoke-Api {
|
||||
param(
|
||||
[string]$Method,
|
||||
[string]$Path,
|
||||
[object]$Body = $null
|
||||
)
|
||||
if ($null -eq $Body) {
|
||||
return Invoke-RestMethod -Method $Method -Uri "$ApiBaseUrl$Path" -TimeoutSec 30
|
||||
}
|
||||
return Invoke-RestMethod -Method $Method -Uri "$ApiBaseUrl$Path" -ContentType "application/json" -Body ($Body | ConvertTo-Json -Depth 80) -TimeoutSec 30
|
||||
}
|
||||
|
||||
function Get-PropertyValue {
|
||||
param(
|
||||
[object]$Item,
|
||||
[string]$Name,
|
||||
[object]$Default = $null
|
||||
)
|
||||
if ($null -eq $Item) { return $Default }
|
||||
$property = $Item.PSObject.Properties[$Name]
|
||||
if ($null -eq $property) { return $Default }
|
||||
return $property.Value
|
||||
}
|
||||
|
||||
function Get-NodeByName {
|
||||
param([string]$Name)
|
||||
$nodes = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/nodes?actor_user_id=$ActorUserID").nodes
|
||||
$node = @($nodes | Where-Object { $_.name -eq $Name }) | Select-Object -First 1
|
||||
if ($null -eq $node) {
|
||||
throw "Node '$Name' was not found in cluster $ClusterID"
|
||||
}
|
||||
return $node
|
||||
}
|
||||
|
||||
function New-IPv4TcpPacket {
|
||||
param(
|
||||
[byte[]]$Source = @(10, 77, 0, 2),
|
||||
[byte[]]$Destination = @(192, 168, 200, 95),
|
||||
[int]$SourcePort,
|
||||
[int]$DestinationPort = 3389
|
||||
)
|
||||
$packet = [byte[]]::new(40)
|
||||
$packet[0] = 0x45
|
||||
$packet[2] = 0
|
||||
$packet[3] = 40
|
||||
$packet[8] = 64
|
||||
$packet[9] = 6
|
||||
[Array]::Copy($Source, 0, $packet, 12, 4)
|
||||
[Array]::Copy($Destination, 0, $packet, 16, 4)
|
||||
$packet[20] = [byte](($SourcePort -shr 8) -band 0xff)
|
||||
$packet[21] = [byte]($SourcePort -band 0xff)
|
||||
$packet[22] = [byte](($DestinationPort -shr 8) -band 0xff)
|
||||
$packet[23] = [byte]($DestinationPort -band 0xff)
|
||||
$packet[32] = 0x50
|
||||
return $packet
|
||||
}
|
||||
|
||||
function ConvertTo-VPNPacketBatch {
|
||||
param([byte[][]]$Packets)
|
||||
$buffer = New-Object System.Collections.Generic.List[byte]
|
||||
foreach ($packet in $Packets) {
|
||||
if ($null -eq $packet -or $packet.Length -eq 0) { continue }
|
||||
$size = [uint32]$packet.Length
|
||||
$buffer.Add([byte](($size -shr 24) -band 0xff))
|
||||
$buffer.Add([byte](($size -shr 16) -band 0xff))
|
||||
$buffer.Add([byte](($size -shr 8) -band 0xff))
|
||||
$buffer.Add([byte]($size -band 0xff))
|
||||
foreach ($value in $packet) {
|
||||
$buffer.Add($value)
|
||||
}
|
||||
}
|
||||
return $buffer.ToArray()
|
||||
}
|
||||
|
||||
function New-RouteIntent {
|
||||
param(
|
||||
[string]$SourceNodeID,
|
||||
[string]$DestinationNodeID,
|
||||
[string[]]$Hops,
|
||||
[int]$Priority,
|
||||
[string]$Label
|
||||
)
|
||||
$expiresAt = (Get-Date).ToUniversalTime().AddMinutes(5).ToString("o")
|
||||
return Invoke-Api -Method POST -Path "/clusters/$ClusterID/mesh/route-intents" -Body @{
|
||||
actor_user_id = $ActorUserID
|
||||
source_selector = @{ node_id = $SourceNodeID }
|
||||
destination_selector = @{ node_id = $DestinationNodeID }
|
||||
service_class = "vpn_packets"
|
||||
priority = $Priority
|
||||
policy = @{
|
||||
synthetic_enabled = $true
|
||||
route_version = "$runId-$Label"
|
||||
policy_version = "$runId-$Label"
|
||||
peer_directory_version = "$runId-$Label"
|
||||
hops = @($Hops)
|
||||
allowed_channels = @("vpn_packet", "fabric_control")
|
||||
max_ttl = 8
|
||||
max_hops = 8
|
||||
expires_at = $expiresAt
|
||||
metadata = @{
|
||||
smoke = "c18z62_service_channel_remediation_qos"
|
||||
run_id = $runId
|
||||
label = $Label
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function Send-DegradedHeartbeat {
|
||||
param(
|
||||
[string]$EntryNodeID,
|
||||
[string]$PrimaryRouteID
|
||||
)
|
||||
$observedAt = (Get-Date).ToUniversalTime().ToString("o")
|
||||
return Invoke-Api -Method POST -Path "/clusters/$ClusterID/nodes/$EntryNodeID/heartbeats" -Body @{
|
||||
health_status = "healthy"
|
||||
reported_version = "0.2.236"
|
||||
capabilities = @{
|
||||
fabric_service_channel_runtime = $true
|
||||
fabric_service_channel_route_manager = $true
|
||||
fabric_service_channel_route_quality_feedback = $true
|
||||
smoke_feedback_injection = "c18z60"
|
||||
}
|
||||
service_states = @{ smoke = "c18z62_primary_degraded_alternate_available" }
|
||||
metadata = @{
|
||||
fabric_service_channel_runtime_report = @{
|
||||
schema_version = "c18l.fabric_service_channel_runtime_report.v1"
|
||||
config_version = "$runId-primary"
|
||||
cluster_id = $ClusterID
|
||||
local_node_id = $EntryNodeID
|
||||
observed_at = $observedAt
|
||||
ingress = @{
|
||||
flow_scheduler = @{
|
||||
schema_version = "rap.fabric_flow_scheduler.v1"
|
||||
service_neutral = $true
|
||||
service_mode = "application_protocol_agnostic"
|
||||
channel_stats = @{
|
||||
"c18z62-primary-degraded" = @{
|
||||
last_route_id = $PrimaryRouteID
|
||||
last_failed_route_id = $PrimaryRouteID
|
||||
route_generation = "$runId-primary"
|
||||
last_error = "c18z60 primary route degraded; alternate available"
|
||||
last_send_duration_ms = 980
|
||||
consecutive_failures = 3
|
||||
stall_count = 2
|
||||
route_rebuild_recommended = $true
|
||||
degraded_fallback_recommended = $false
|
||||
quality_window_sample_count = 8
|
||||
quality_window_success_count = 2
|
||||
quality_window_failure_count = 3
|
||||
quality_window_slow_count = 2
|
||||
quality_window_drop_count = 1
|
||||
quality_window_avg_latency_ms = 980
|
||||
quality_window_last_updated_at = $observedAt
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
smoke = @{
|
||||
name = "c18z62_service_channel_remediation_qos"
|
||||
run_id = $runId
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
$entryNode = Get-NodeByName -Name $EntryNodeName
|
||||
$relayNode = Get-NodeByName -Name $RelayNodeName
|
||||
$exitNode = Get-NodeByName -Name $ExitNodeName
|
||||
$primaryRouteID = ""
|
||||
$alternateRouteID = ""
|
||||
$result = $null
|
||||
$failedChecks = @()
|
||||
$expectedPressurePacketCount = 128
|
||||
$expectedPressureFlowCount = 8
|
||||
$expectedInteractivePacketCount = 1
|
||||
|
||||
try {
|
||||
$primary = (New-RouteIntent -SourceNodeID $entryNode.id -DestinationNodeID $exitNode.id -Hops @($entryNode.id, $exitNode.id) -Priority 2100000000 -Label "primary").route_intent
|
||||
$alternate = (New-RouteIntent -SourceNodeID $entryNode.id -DestinationNodeID $exitNode.id -Hops @($entryNode.id, $relayNode.id, $exitNode.id) -Priority 2099999900 -Label "alternate").route_intent
|
||||
$primaryRouteID = [string]$primary.id
|
||||
$alternateRouteID = [string]$alternate.id
|
||||
|
||||
$resourceID = "c18z62-vpn-smoke"
|
||||
$lease = (Invoke-Api -Method POST -Path "/clusters/$ClusterID/fabric/service-channels/leases" -Body @{
|
||||
actor_user_id = $ActorUserID
|
||||
organization_id = "smoke-org"
|
||||
user_id = "smoke-user"
|
||||
resource_id = $resourceID
|
||||
service_class = "vpn_packets"
|
||||
entry_node_ids = @([string]$entryNode.id)
|
||||
exit_node_ids = @([string]$exitNode.id)
|
||||
preferred_entry_node_id = [string]$entryNode.id
|
||||
preferred_exit_node_id = [string]$exitNode.id
|
||||
allowed_channels = @("vpn_packet", "fabric_control")
|
||||
ttl_seconds = 180
|
||||
metadata = @{
|
||||
smoke = "c18z62_service_channel_remediation_qos"
|
||||
run_id = $runId
|
||||
}
|
||||
}).fabric_service_channel_lease
|
||||
|
||||
[void](Send-DegradedHeartbeat -EntryNodeID $entryNode.id -PrimaryRouteID $primaryRouteID)
|
||||
|
||||
$packetPath = $lease.entry_http.path_template.
|
||||
Replace("{cluster_id}", $ClusterID).
|
||||
Replace("{channel_id}", [string]$lease.channel_id).
|
||||
Replace("{resource_id}", $resourceID)
|
||||
$packetUrl = $EntryBaseUrl.TrimEnd("/") + $packetPath
|
||||
$headers = @{
|
||||
"X-RAP-Service-Channel-Token" = [string]$lease.token.token
|
||||
"X-RAP-Fabric-Channel-ID" = [string]$lease.channel_id
|
||||
"X-RAP-Service-Class" = "vpn_packets"
|
||||
"X-RAP-Channel-Class" = "vpn_packet"
|
||||
}
|
||||
$response = Invoke-WebRequest -Method Post -Uri $packetUrl -Headers $headers -Body ([System.Text.Encoding]::UTF8.GetBytes("c18z62-alternate-remediation")) -ContentType "application/octet-stream" -TimeoutSec 30
|
||||
$acceptedBy = [string]$response.Headers["X-RAP-Service-Channel-Accepted-By"]
|
||||
|
||||
$accessTelemetry = $null
|
||||
$matchingChannel = $null
|
||||
for ($i = 0; $i -lt 10; $i++) {
|
||||
Start-Sleep -Seconds 3
|
||||
$accessTelemetry = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/fabric/service-channels/access-telemetry?actor_user_id=$ActorUserID&limit=20").fabric_service_channel_access_telemetry
|
||||
$channels = @()
|
||||
if ($accessTelemetry.PSObject.Properties.Name -contains "active_channels") {
|
||||
$channels = @($accessTelemetry.active_channels)
|
||||
}
|
||||
$matchingChannel = $channels | Where-Object { $_.channel_id -eq $lease.channel_id } | Select-Object -First 1
|
||||
if ($null -ne $matchingChannel -and [string](Get-PropertyValue -Item $matchingChannel -Name "remediation_action" -Default "") -eq "prefer_alternate_route") {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
$backendLine = (& ssh $DockerSSH "docker ps --format '{{.Names}} {{.Image}} {{.Status}}' | grep '^rap_test_backend '") | Out-String
|
||||
$nodeLines = (& ssh $DockerSSH "docker ps --format '{{.Names}} {{.Image}} {{.Status}}' | grep '^rap_test_node_test_'") | Out-String
|
||||
$leasePrimaryRouteID = [string](Get-PropertyValue -Item (Get-PropertyValue -Item $lease -Name "primary_route" -Default $null) -Name "route_id" -Default "")
|
||||
$leaseAlternates = @()
|
||||
if ($lease.PSObject.Properties.Name -contains "alternate_routes") {
|
||||
$leaseAlternates = @($lease.alternate_routes)
|
||||
}
|
||||
$leaseHasAlternate = (@($leaseAlternates | Where-Object { [string]$_.route_id -eq $alternateRouteID }).Count -ge 1)
|
||||
$remediationCommand = Get-PropertyValue -Item $matchingChannel -Name "remediation_command" -Default $null
|
||||
$commandAction = [string](Get-PropertyValue -Item $remediationCommand -Name "action" -Default "")
|
||||
$commandPrimaryRouteID = [string](Get-PropertyValue -Item $remediationCommand -Name "primary_route_id" -Default "")
|
||||
$commandReplacementRouteID = [string](Get-PropertyValue -Item $remediationCommand -Name "replacement_route_id" -Default "")
|
||||
$commandExpiresAt = [string](Get-PropertyValue -Item $remediationCommand -Name "expires_at" -Default "")
|
||||
$expectedReplacementRouteID = $commandReplacementRouteID
|
||||
$syntheticConfig = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/nodes/$($entryNode.id)/mesh/synthetic-config").synthetic_mesh_config
|
||||
$syntheticCommands = @()
|
||||
if ($syntheticConfig.PSObject.Properties.Name -contains "service_channel_remediation_commands") {
|
||||
$syntheticCommands = @($syntheticConfig.service_channel_remediation_commands)
|
||||
}
|
||||
$syntheticCommand = $syntheticCommands | Where-Object { [string]$_.channel_id -eq [string]$lease.channel_id } | Select-Object -First 1
|
||||
$syntheticCommandAction = [string](Get-PropertyValue -Item $syntheticCommand -Name "action" -Default "")
|
||||
$syntheticCommandReplacementRouteID = [string](Get-PropertyValue -Item $syntheticCommand -Name "replacement_route_id" -Default "")
|
||||
if ([string]::IsNullOrWhiteSpace($expectedReplacementRouteID)) {
|
||||
$expectedReplacementRouteID = $syntheticCommandReplacementRouteID
|
||||
}
|
||||
if ([string]::IsNullOrWhiteSpace($expectedReplacementRouteID)) {
|
||||
$expectedReplacementRouteID = $alternateRouteID
|
||||
}
|
||||
|
||||
$routeManagerDecision = $null
|
||||
$routeManagerRuntime = $null
|
||||
for ($i = 0; $i -lt 10; $i++) {
|
||||
Start-Sleep -Seconds 5
|
||||
$heartbeats = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/nodes/$($entryNode.id)/heartbeats?actor_user_id=$ActorUserID&limit=5").heartbeats
|
||||
foreach ($heartbeat in @($heartbeats)) {
|
||||
$runtimeReport = Get-PropertyValue -Item (Get-PropertyValue -Item $heartbeat -Name "metadata" -Default $null) -Name "fabric_service_channel_runtime_report" -Default $null
|
||||
$ingress = Get-PropertyValue -Item $runtimeReport -Name "ingress" -Default $null
|
||||
$routeManager = Get-PropertyValue -Item $ingress -Name "route_manager" -Default $null
|
||||
$routeManagerRuntime = $routeManager
|
||||
$decisions = @()
|
||||
if ($null -ne $routeManager -and $routeManager.PSObject.Properties.Name -contains "decisions") {
|
||||
$decisions = @($routeManager.decisions)
|
||||
}
|
||||
$routeManagerDecision = $decisions | Where-Object {
|
||||
[string]$_.route_id -eq $primaryRouteID -and
|
||||
[string]$_.replacement_route_id -eq $expectedReplacementRouteID -and
|
||||
[string]$_.decision_source -eq "service_channel_remediation_command"
|
||||
} | Select-Object -First 1
|
||||
if ($null -ne $routeManagerDecision) {
|
||||
break
|
||||
}
|
||||
}
|
||||
if ($null -ne $routeManagerDecision) {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
$multiFlowPackets = @()
|
||||
foreach ($offset in 0..($expectedPressurePacketCount - 1)) {
|
||||
$port = 52000 + $offset
|
||||
$multiFlowPackets += ,(New-IPv4TcpPacket -SourcePort $port -DestinationPort 3389)
|
||||
}
|
||||
$multiFlowPayload = ConvertTo-VPNPacketBatch -Packets $multiFlowPackets
|
||||
$multiFlowPayloadPath = Join-Path ([System.IO.Path]::GetTempPath()) "$runId-vpn-packet-batch.bin"
|
||||
[System.IO.File]::WriteAllBytes($multiFlowPayloadPath, [byte[]]$multiFlowPayload)
|
||||
$bulkHeaders = @{} + $headers
|
||||
$bulkHeaders["X-RAP-Traffic-Class"] = "bulk"
|
||||
$postRemediationResponse = Invoke-WebRequest -Method Post -Uri ($packetUrl + "?batch=true") -Headers $bulkHeaders -InFile $multiFlowPayloadPath -ContentType "application/vnd.rap.vpn-packet-batch.v1" -TimeoutSec 30
|
||||
$postRemediationAcceptedBy = [string]$postRemediationResponse.Headers["X-RAP-Service-Channel-Accepted-By"]
|
||||
$interactivePacket = New-IPv4TcpPacket -SourcePort 53000 -DestinationPort 22
|
||||
$interactivePayload = ConvertTo-VPNPacketBatch -Packets @($interactivePacket)
|
||||
$interactivePayloadPath = Join-Path ([System.IO.Path]::GetTempPath()) "$runId-vpn-interactive-packet-batch.bin"
|
||||
[System.IO.File]::WriteAllBytes($interactivePayloadPath, [byte[]]$interactivePayload)
|
||||
$interactiveHeaders = @{} + $headers
|
||||
$interactiveHeaders["X-RAP-Traffic-Class"] = "interactive"
|
||||
$interactiveResponse = Invoke-WebRequest -Method Post -Uri ($packetUrl + "?batch=true") -Headers $interactiveHeaders -InFile $interactivePayloadPath -ContentType "application/vnd.rap.vpn-packet-batch.v1" -TimeoutSec 30
|
||||
$interactiveAcceptedBy = [string]$interactiveResponse.Headers["X-RAP-Service-Channel-Accepted-By"]
|
||||
$replacementTrafficObserved = $false
|
||||
$replacementLastSelected = ""
|
||||
$replacementFlowStat = $null
|
||||
$replacementFlowStats = @()
|
||||
$interactiveFlowStats = @()
|
||||
$bulkFlowStats = @()
|
||||
$postRemediationFallbackLocal = 0
|
||||
$postRemediationRouteFailures = 0
|
||||
$postRemediationFlowDropped = 0
|
||||
$postRemediationSchedulerDropped = 0
|
||||
$postRemediationHighWatermark = 0
|
||||
$postRemediationMaxInFlight = 0
|
||||
for ($i = 0; $i -lt 12; $i++) {
|
||||
Start-Sleep -Seconds 5
|
||||
$heartbeats = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/nodes/$($entryNode.id)/heartbeats?actor_user_id=$ActorUserID&limit=5").heartbeats
|
||||
foreach ($heartbeat in @($heartbeats)) {
|
||||
$runtimeReport = Get-PropertyValue -Item (Get-PropertyValue -Item $heartbeat -Name "metadata" -Default $null) -Name "fabric_service_channel_runtime_report" -Default $null
|
||||
$ingress = Get-PropertyValue -Item $runtimeReport -Name "ingress" -Default $null
|
||||
$replacementLastSelected = [string](Get-PropertyValue -Item $ingress -Name "last_selected_route_id" -Default "")
|
||||
$postRemediationFallbackLocal = [int](Get-PropertyValue -Item $ingress -Name "send_fallback_local" -Default 0)
|
||||
$postRemediationRouteFailures = [int](Get-PropertyValue -Item $ingress -Name "send_route_failures" -Default 0)
|
||||
$postRemediationFlowDropped = [int](Get-PropertyValue -Item $ingress -Name "send_flow_dropped" -Default 0)
|
||||
$flowScheduler = Get-PropertyValue -Item $ingress -Name "flow_scheduler" -Default $null
|
||||
$postRemediationSchedulerDropped = [int](Get-PropertyValue -Item $flowScheduler -Name "dropped" -Default 0)
|
||||
$postRemediationHighWatermark = [int](Get-PropertyValue -Item $flowScheduler -Name "high_watermark" -Default 0)
|
||||
$postRemediationMaxInFlight = [int](Get-PropertyValue -Item $flowScheduler -Name "max_in_flight" -Default 0)
|
||||
$channelStats = Get-PropertyValue -Item $flowScheduler -Name "channel_stats" -Default $null
|
||||
if ($null -ne $channelStats) {
|
||||
$replacementFlowStats = @()
|
||||
$interactiveFlowStats = @()
|
||||
$bulkFlowStats = @()
|
||||
foreach ($property in @($channelStats.PSObject.Properties)) {
|
||||
$stat = $property.Value
|
||||
if ([string](Get-PropertyValue -Item $stat -Name "last_route_id" -Default "") -eq $expectedReplacementRouteID) {
|
||||
$replacementFlowStats += $stat
|
||||
$replacementFlowStat = $stat
|
||||
$trafficClass = [string](Get-PropertyValue -Item $stat -Name "traffic_class" -Default "")
|
||||
if ($trafficClass -eq "interactive") {
|
||||
$interactiveFlowStats += $stat
|
||||
}
|
||||
if ($trafficClass -eq "bulk") {
|
||||
$bulkFlowStats += $stat
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if ($replacementLastSelected -eq $expectedReplacementRouteID -and $bulkFlowStats.Count -ge $expectedPressureFlowCount -and $interactiveFlowStats.Count -ge 1) {
|
||||
$replacementTrafficObserved = $true
|
||||
break
|
||||
}
|
||||
}
|
||||
if ($replacementTrafficObserved) {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
$result = [ordered]@{
|
||||
schema_version = "c18z62.service_channel_remediation_qos_smoke.v1"
|
||||
run_id = $runId
|
||||
cluster_id = $ClusterID
|
||||
primary_route_id = $primaryRouteID
|
||||
alternate_route_id = $alternateRouteID
|
||||
expected_replacement_route_id = $expectedReplacementRouteID
|
||||
channel_id = [string]$lease.channel_id
|
||||
passed = [bool](
|
||||
$backendLine.Contains($ExpectedBackendImage) -and
|
||||
$nodeLines.Contains($ExpectedNodeAgentImage) -and
|
||||
[string]$lease.status -eq "ready" -and
|
||||
$leasePrimaryRouteID -eq $primaryRouteID -and
|
||||
$leaseHasAlternate -and
|
||||
[int]$response.StatusCode -eq 202 -and
|
||||
$acceptedBy -eq "introspection" -and
|
||||
$null -ne $matchingChannel -and
|
||||
[string](Get-PropertyValue -Item $matchingChannel -Name "primary_route_id" -Default "") -eq $primaryRouteID -and
|
||||
-not [bool](Get-PropertyValue -Item $matchingChannel -Name "force_backend_fallback" -Default $false) -and
|
||||
[string](Get-PropertyValue -Item $matchingChannel -Name "route_feedback_status" -Default "") -eq "fenced" -and
|
||||
[string](Get-PropertyValue -Item $matchingChannel -Name "remediation_action" -Default "") -eq "prefer_alternate_route" -and
|
||||
[string](Get-PropertyValue -Item $matchingChannel -Name "remediation_route_id" -Default "") -eq $expectedReplacementRouteID -and
|
||||
$null -ne $remediationCommand -and
|
||||
$commandAction -eq "prefer_alternate_route" -and
|
||||
$commandPrimaryRouteID -eq $primaryRouteID -and
|
||||
$commandReplacementRouteID -eq $expectedReplacementRouteID -and
|
||||
$commandExpiresAt.Length -gt 0 -and
|
||||
$null -ne $syntheticCommand -and
|
||||
$syntheticCommandAction -eq "prefer_alternate_route" -and
|
||||
$syntheticCommandReplacementRouteID -eq $expectedReplacementRouteID -and
|
||||
$null -ne $routeManagerDecision -and
|
||||
[string](Get-PropertyValue -Item $routeManagerDecision -Name "rebuild_status" -Default "") -eq "applied" -and
|
||||
[int]$postRemediationResponse.StatusCode -eq 202 -and
|
||||
$postRemediationAcceptedBy -eq "introspection" -and
|
||||
[int]$interactiveResponse.StatusCode -eq 202 -and
|
||||
$interactiveAcceptedBy -eq "introspection" -and
|
||||
$replacementTrafficObserved -and
|
||||
$replacementLastSelected -eq $expectedReplacementRouteID -and
|
||||
$bulkFlowStats.Count -ge $expectedPressureFlowCount -and
|
||||
$interactiveFlowStats.Count -ge 1 -and
|
||||
$multiFlowPackets.Count -eq $expectedPressurePacketCount -and
|
||||
$postRemediationFallbackLocal -eq 0 -and
|
||||
$postRemediationFlowDropped -eq 0 -and
|
||||
$postRemediationSchedulerDropped -eq 0 -and
|
||||
$postRemediationRouteFailures -eq 0 -and
|
||||
[int]$accessTelemetry.degraded_fallback_channel_count -eq 0
|
||||
)
|
||||
checks = [ordered]@{
|
||||
backend_expected_image_deployed = $backendLine.Contains($ExpectedBackendImage)
|
||||
node_agent_expected_image_deployed = $nodeLines.Contains($ExpectedNodeAgentImage)
|
||||
lease_ready = ([string]$lease.status -eq "ready")
|
||||
lease_selected_primary_route = ($leasePrimaryRouteID -eq $primaryRouteID)
|
||||
lease_contains_alternate_route = $leaseHasAlternate
|
||||
packet_accepted = ([int]$response.StatusCode -eq 202)
|
||||
accepted_by_header_is_introspection = ($acceptedBy -eq "introspection")
|
||||
active_channel_visible = ($null -ne $matchingChannel)
|
||||
active_channel_not_backend_fallback = ($null -ne $matchingChannel -and -not [bool](Get-PropertyValue -Item $matchingChannel -Name "force_backend_fallback" -Default $false))
|
||||
route_quality_fenced = ($null -ne $matchingChannel -and [string](Get-PropertyValue -Item $matchingChannel -Name "route_feedback_status" -Default "") -eq "fenced")
|
||||
remediation_prefers_alternate = ($null -ne $matchingChannel -and [string](Get-PropertyValue -Item $matchingChannel -Name "remediation_action" -Default "") -eq "prefer_alternate_route")
|
||||
remediation_route_is_alternate = ($null -ne $matchingChannel -and [string](Get-PropertyValue -Item $matchingChannel -Name "remediation_route_id" -Default "") -eq $expectedReplacementRouteID)
|
||||
remediation_command_visible = ($null -ne $remediationCommand)
|
||||
remediation_command_prefers_alternate = ($commandAction -eq "prefer_alternate_route")
|
||||
remediation_command_primary_route_matches = ($commandPrimaryRouteID -eq $primaryRouteID)
|
||||
remediation_command_replacement_route_matches = ($commandReplacementRouteID -eq $expectedReplacementRouteID)
|
||||
remediation_command_has_ttl = ($commandExpiresAt.Length -gt 0)
|
||||
synthetic_config_command_visible = ($null -ne $syntheticCommand)
|
||||
synthetic_config_command_prefers_alternate = ($syntheticCommandAction -eq "prefer_alternate_route")
|
||||
synthetic_config_command_replacement_route_matches = ($syntheticCommandReplacementRouteID -eq $expectedReplacementRouteID)
|
||||
node_route_manager_consumed_command = ($null -ne $routeManagerDecision)
|
||||
node_route_manager_applied_replacement = ($null -ne $routeManagerDecision -and [string](Get-PropertyValue -Item $routeManagerDecision -Name "rebuild_status" -Default "") -eq "applied")
|
||||
post_remediation_packet_accepted = ([int]$postRemediationResponse.StatusCode -eq 202)
|
||||
post_remediation_accepted_by_introspection = ($postRemediationAcceptedBy -eq "introspection")
|
||||
interactive_packet_accepted = ([int]$interactiveResponse.StatusCode -eq 202)
|
||||
interactive_accepted_by_introspection = ($interactiveAcceptedBy -eq "introspection")
|
||||
replacement_traffic_observed = $replacementTrafficObserved
|
||||
replacement_last_selected_route_matches = ($replacementLastSelected -eq $expectedReplacementRouteID)
|
||||
replacement_flow_stat_observed = ($null -ne $replacementFlowStat)
|
||||
replacement_pressure_flow_stats_observed = ($bulkFlowStats.Count -ge $expectedPressureFlowCount)
|
||||
replacement_interactive_flow_stat_observed = ($interactiveFlowStats.Count -ge 1)
|
||||
pressure_packet_count_matches = ($multiFlowPackets.Count -eq $expectedPressurePacketCount)
|
||||
no_local_gateway_fallback_after_replacement = ($postRemediationFallbackLocal -eq 0)
|
||||
no_flow_drops_after_replacement = ($postRemediationFlowDropped -eq 0 -and $postRemediationSchedulerDropped -eq 0)
|
||||
no_route_failures_after_replacement = ($postRemediationRouteFailures -eq 0)
|
||||
backend_fallback_not_recommended = ([int]$accessTelemetry.degraded_fallback_channel_count -eq 0)
|
||||
}
|
||||
summary = [ordered]@{
|
||||
backend_container = $backendLine.Trim()
|
||||
node_containers = $nodeLines.Trim()
|
||||
accepted_by = $acceptedBy
|
||||
lease_status = [string]$lease.status
|
||||
lease_primary_route_id = $leasePrimaryRouteID
|
||||
lease_alternate_route_count = $leaseAlternates.Count
|
||||
access_status = [string]$accessTelemetry.status
|
||||
active_channel_count = [int]$accessTelemetry.active_channel_count
|
||||
correlated_route_count = [int]$accessTelemetry.correlated_route_count
|
||||
degraded_route_count = [int]$accessTelemetry.degraded_route_count
|
||||
degraded_fallback_channel_count = [int]$accessTelemetry.degraded_fallback_channel_count
|
||||
synthetic_command = $syntheticCommand
|
||||
route_manager_decision = $routeManagerDecision
|
||||
route_manager_runtime = $routeManagerRuntime
|
||||
post_remediation_accepted_by = $postRemediationAcceptedBy
|
||||
post_remediation_packet_count = $multiFlowPackets.Count
|
||||
interactive_accepted_by = $interactiveAcceptedBy
|
||||
interactive_packet_count = $expectedInteractivePacketCount
|
||||
expected_pressure_flow_count = $expectedPressureFlowCount
|
||||
replacement_last_selected_route_id = $replacementLastSelected
|
||||
replacement_flow_stat_count = $replacementFlowStats.Count
|
||||
replacement_bulk_flow_stat_count = $bulkFlowStats.Count
|
||||
replacement_interactive_flow_stat_count = $interactiveFlowStats.Count
|
||||
replacement_flow_stat = $replacementFlowStat
|
||||
replacement_flow_stats = $replacementFlowStats
|
||||
replacement_bulk_flow_stats = $bulkFlowStats
|
||||
replacement_interactive_flow_stats = $interactiveFlowStats
|
||||
post_remediation_send_fallback_local = $postRemediationFallbackLocal
|
||||
post_remediation_send_route_failures = $postRemediationRouteFailures
|
||||
post_remediation_send_flow_dropped = $postRemediationFlowDropped
|
||||
post_remediation_scheduler_dropped = $postRemediationSchedulerDropped
|
||||
post_remediation_scheduler_high_watermark = $postRemediationHighWatermark
|
||||
post_remediation_scheduler_max_in_flight = $postRemediationMaxInFlight
|
||||
matching_channel = $matchingChannel
|
||||
}
|
||||
}
|
||||
$failedChecks = @($result.checks.GetEnumerator() | Where-Object { $_.Value -ne $true })
|
||||
$result.summary.failed_checks = @($failedChecks | ForEach-Object { $_.Key })
|
||||
}
|
||||
finally {
|
||||
if ($primaryRouteID) {
|
||||
try { Invoke-Api -Method POST -Path "/clusters/$ClusterID/mesh/route-intents/$primaryRouteID/expire" -Body @{ actor_user_id = $ActorUserID } | Out-Null } catch {}
|
||||
}
|
||||
if ($alternateRouteID) {
|
||||
try { Invoke-Api -Method POST -Path "/clusters/$ClusterID/mesh/route-intents/$alternateRouteID/expire" -Body @{ actor_user_id = $ActorUserID } | Out-Null } catch {}
|
||||
}
|
||||
}
|
||||
|
||||
$resultFullPath = Join-Path $repoRoot $ResultPath
|
||||
$resultDir = Split-Path -Parent $resultFullPath
|
||||
if (-not (Test-Path $resultDir)) {
|
||||
New-Item -ItemType Directory -Path $resultDir | Out-Null
|
||||
}
|
||||
$result | ConvertTo-Json -Depth 100 | Set-Content -Path $resultFullPath -Encoding UTF8
|
||||
|
||||
if ($failedChecks.Count -gt 0) {
|
||||
throw "C18Z62 failed checks: $(@($failedChecks | ForEach-Object { $_.Key }) -join ', '). Result: $resultFullPath"
|
||||
}
|
||||
|
||||
Write-Host "C18Z62 service-channel remediation qos smoke passed. Result: $resultFullPath"
|
||||
$result
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,619 @@
|
||||
param(
|
||||
[string]$ApiBaseUrl = "http://192.168.200.61:18121/api/v1",
|
||||
[string]$ClusterID = "cfc0743d-d960-49fb-9de8-96e063d5e4aa",
|
||||
[string]$ActorUserID = "f67d943f-5397-4b3a-a229-695fe67ad700",
|
||||
[string]$EntryNodeName = "test-1",
|
||||
[string]$RelayNodeName = "test-3",
|
||||
[string]$ExitNodeName = "test-2",
|
||||
[string]$EntryBaseUrl = "http://192.168.200.61:19131",
|
||||
[string]$DockerSSH = "test-docker",
|
||||
[string]$ExpectedBackendImage = "rap-backend:fabric-service-channel-0.2.256-c18z82",
|
||||
[string]$ExpectedNodeAgentImage = "rap-node-agent:0.2.270-c18z95",
|
||||
[string]$ResultPath = "artifacts\c18z67-service-channel-concurrent-qos-live-smoke-result.json"
|
||||
)
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
$scriptDir = Split-Path -Parent $MyInvocation.MyCommand.Path
|
||||
$repoRoot = (Resolve-Path (Join-Path $scriptDir "..\..")).ProviderPath
|
||||
$runId = "c18z67-" + (Get-Date -Format "yyyyMMdd-HHmmss")
|
||||
|
||||
function Invoke-Api {
|
||||
param(
|
||||
[string]$Method,
|
||||
[string]$Path,
|
||||
[object]$Body = $null
|
||||
)
|
||||
if ($null -eq $Body) {
|
||||
return Invoke-RestMethod -Method $Method -Uri "$ApiBaseUrl$Path" -TimeoutSec 30
|
||||
}
|
||||
return Invoke-RestMethod -Method $Method -Uri "$ApiBaseUrl$Path" -ContentType "application/json" -Body ($Body | ConvertTo-Json -Depth 80) -TimeoutSec 30
|
||||
}
|
||||
|
||||
function Get-PropertyValue {
|
||||
param(
|
||||
[object]$Item,
|
||||
[string]$Name,
|
||||
[object]$Default = $null
|
||||
)
|
||||
if ($null -eq $Item) { return $Default }
|
||||
$property = $Item.PSObject.Properties[$Name]
|
||||
if ($null -eq $property) { return $Default }
|
||||
return $property.Value
|
||||
}
|
||||
|
||||
function Get-NodeByName {
|
||||
param([string]$Name)
|
||||
$nodes = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/nodes?actor_user_id=$ActorUserID").nodes
|
||||
$node = @($nodes | Where-Object { $_.name -eq $Name }) | Select-Object -First 1
|
||||
if ($null -eq $node) {
|
||||
throw "Node '$Name' was not found in cluster $ClusterID"
|
||||
}
|
||||
return $node
|
||||
}
|
||||
|
||||
function New-IPv4TcpPacket {
|
||||
param(
|
||||
[byte[]]$Source = @(10, 77, 0, 2),
|
||||
[byte[]]$Destination = @(192, 168, 200, 95),
|
||||
[int]$SourcePort,
|
||||
[int]$DestinationPort = 3389
|
||||
)
|
||||
$packet = [byte[]]::new(40)
|
||||
$packet[0] = 0x45
|
||||
$packet[2] = 0
|
||||
$packet[3] = 40
|
||||
$packet[8] = 64
|
||||
$packet[9] = 6
|
||||
[Array]::Copy($Source, 0, $packet, 12, 4)
|
||||
[Array]::Copy($Destination, 0, $packet, 16, 4)
|
||||
$packet[20] = [byte](($SourcePort -shr 8) -band 0xff)
|
||||
$packet[21] = [byte]($SourcePort -band 0xff)
|
||||
$packet[22] = [byte](($DestinationPort -shr 8) -band 0xff)
|
||||
$packet[23] = [byte]($DestinationPort -band 0xff)
|
||||
$packet[32] = 0x50
|
||||
return $packet
|
||||
}
|
||||
|
||||
function ConvertTo-VPNPacketBatch {
|
||||
param([byte[][]]$Packets)
|
||||
$buffer = New-Object System.Collections.Generic.List[byte]
|
||||
foreach ($packet in $Packets) {
|
||||
if ($null -eq $packet -or $packet.Length -eq 0) { continue }
|
||||
$size = [uint32]$packet.Length
|
||||
$buffer.Add([byte](($size -shr 24) -band 0xff))
|
||||
$buffer.Add([byte](($size -shr 16) -band 0xff))
|
||||
$buffer.Add([byte](($size -shr 8) -band 0xff))
|
||||
$buffer.Add([byte]($size -band 0xff))
|
||||
foreach ($value in $packet) {
|
||||
$buffer.Add($value)
|
||||
}
|
||||
}
|
||||
return $buffer.ToArray()
|
||||
}
|
||||
|
||||
function New-RouteIntent {
|
||||
param(
|
||||
[string]$SourceNodeID,
|
||||
[string]$DestinationNodeID,
|
||||
[string[]]$Hops,
|
||||
[int]$Priority,
|
||||
[string]$Label
|
||||
)
|
||||
$expiresAt = (Get-Date).ToUniversalTime().AddMinutes(5).ToString("o")
|
||||
return Invoke-Api -Method POST -Path "/clusters/$ClusterID/mesh/route-intents" -Body @{
|
||||
actor_user_id = $ActorUserID
|
||||
source_selector = @{ node_id = $SourceNodeID }
|
||||
destination_selector = @{ node_id = $DestinationNodeID }
|
||||
service_class = "vpn_packets"
|
||||
priority = $Priority
|
||||
policy = @{
|
||||
synthetic_enabled = $true
|
||||
route_version = "$runId-$Label"
|
||||
policy_version = "$runId-$Label"
|
||||
peer_directory_version = "$runId-$Label"
|
||||
hops = @($Hops)
|
||||
allowed_channels = @("vpn_packet", "fabric_control")
|
||||
max_ttl = 8
|
||||
max_hops = 8
|
||||
expires_at = $expiresAt
|
||||
metadata = @{
|
||||
smoke = "c18z67_service_channel_concurrent_qos_live"
|
||||
run_id = $runId
|
||||
label = $Label
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function Disable-ExistingRouteIntents {
|
||||
param([string]$SourceNodeID, [string]$DestinationNodeID)
|
||||
$items = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/mesh/route-intents?actor_user_id=$ActorUserID").route_intents
|
||||
foreach ($item in @($items)) {
|
||||
if ([string](Get-PropertyValue -Item $item -Name "status" -Default "") -ne "active") { continue }
|
||||
if ([string](Get-PropertyValue -Item $item -Name "service_class" -Default "") -ne "vpn_packets") { continue }
|
||||
$sourceSelector = Get-PropertyValue -Item $item -Name "source_selector" -Default $null
|
||||
$destinationSelector = Get-PropertyValue -Item $item -Name "destination_selector" -Default $null
|
||||
if ([string](Get-PropertyValue -Item $sourceSelector -Name "node_id" -Default "") -ne $SourceNodeID) { continue }
|
||||
if ([string](Get-PropertyValue -Item $destinationSelector -Name "node_id" -Default "") -ne $DestinationNodeID) { continue }
|
||||
[void](Invoke-Api -Method POST -Path "/clusters/$ClusterID/mesh/route-intents/$($item.id)/disable" -Body @{
|
||||
actor_user_id = $ActorUserID
|
||||
reason = "c18z67 isolate concurrent-qos smoke route pair"
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
function Send-DegradedHeartbeat {
|
||||
param(
|
||||
[string]$EntryNodeID,
|
||||
[string]$PrimaryRouteID
|
||||
)
|
||||
$observedAt = (Get-Date).ToUniversalTime().ToString("o")
|
||||
return Invoke-Api -Method POST -Path "/clusters/$ClusterID/nodes/$EntryNodeID/heartbeats" -Body @{
|
||||
health_status = "healthy"
|
||||
reported_version = "0.2.236"
|
||||
capabilities = @{
|
||||
fabric_service_channel_runtime = $true
|
||||
fabric_service_channel_route_manager = $true
|
||||
fabric_service_channel_route_quality_feedback = $true
|
||||
smoke_feedback_injection = "c18z60"
|
||||
}
|
||||
service_states = @{ smoke = "c18z67_primary_degraded_alternate_available" }
|
||||
metadata = @{
|
||||
fabric_service_channel_runtime_report = @{
|
||||
schema_version = "c18l.fabric_service_channel_runtime_report.v1"
|
||||
config_version = "$runId-primary"
|
||||
cluster_id = $ClusterID
|
||||
local_node_id = $EntryNodeID
|
||||
observed_at = $observedAt
|
||||
ingress = @{
|
||||
flow_scheduler = @{
|
||||
schema_version = "rap.fabric_flow_scheduler.v1"
|
||||
service_neutral = $true
|
||||
service_mode = "application_protocol_agnostic"
|
||||
channel_stats = @{
|
||||
"c18z67-primary-degraded" = @{
|
||||
last_route_id = $PrimaryRouteID
|
||||
last_failed_route_id = $PrimaryRouteID
|
||||
route_generation = "$runId-primary"
|
||||
last_error = "c18z60 primary route degraded; alternate available"
|
||||
last_send_duration_ms = 980
|
||||
consecutive_failures = 3
|
||||
stall_count = 2
|
||||
route_rebuild_recommended = $true
|
||||
degraded_fallback_recommended = $false
|
||||
quality_window_sample_count = 8
|
||||
quality_window_success_count = 2
|
||||
quality_window_failure_count = 3
|
||||
quality_window_slow_count = 2
|
||||
quality_window_drop_count = 1
|
||||
quality_window_avg_latency_ms = 980
|
||||
quality_window_last_updated_at = $observedAt
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
smoke = @{
|
||||
name = "c18z67_service_channel_concurrent_qos_live"
|
||||
run_id = $runId
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
$entryNode = Get-NodeByName -Name $EntryNodeName
|
||||
$relayNode = Get-NodeByName -Name $RelayNodeName
|
||||
$exitNode = Get-NodeByName -Name $ExitNodeName
|
||||
$primaryRouteID = ""
|
||||
$alternateRouteID = ""
|
||||
$result = $null
|
||||
$failedChecks = @()
|
||||
$expectedPressurePacketCount = 512
|
||||
$expectedPressureFlowCount = 8
|
||||
$expectedInteractivePacketCount = 1
|
||||
$expectedBulkRequestCount = 6
|
||||
|
||||
try {
|
||||
[void](Disable-ExistingRouteIntents -SourceNodeID $entryNode.id -DestinationNodeID $exitNode.id)
|
||||
$primary = (New-RouteIntent -SourceNodeID $entryNode.id -DestinationNodeID $exitNode.id -Hops @($entryNode.id, $exitNode.id) -Priority 2100000000 -Label "primary").route_intent
|
||||
$alternate = (New-RouteIntent -SourceNodeID $entryNode.id -DestinationNodeID $exitNode.id -Hops @($entryNode.id, $relayNode.id, $exitNode.id) -Priority 2099999900 -Label "alternate").route_intent
|
||||
$primaryRouteID = [string]$primary.id
|
||||
$alternateRouteID = [string]$alternate.id
|
||||
|
||||
$resourceID = "c18z67-vpn-smoke"
|
||||
$lease = (Invoke-Api -Method POST -Path "/clusters/$ClusterID/fabric/service-channels/leases" -Body @{
|
||||
actor_user_id = $ActorUserID
|
||||
organization_id = "smoke-org"
|
||||
user_id = "smoke-user"
|
||||
resource_id = $resourceID
|
||||
service_class = "vpn_packets"
|
||||
entry_node_ids = @([string]$entryNode.id)
|
||||
exit_node_ids = @([string]$exitNode.id)
|
||||
preferred_entry_node_id = [string]$entryNode.id
|
||||
preferred_exit_node_id = [string]$exitNode.id
|
||||
allowed_channels = @("vpn_packet", "fabric_control")
|
||||
ttl_seconds = 180
|
||||
metadata = @{
|
||||
smoke = "c18z67_service_channel_concurrent_qos_live"
|
||||
run_id = $runId
|
||||
}
|
||||
}).fabric_service_channel_lease
|
||||
|
||||
[void](Send-DegradedHeartbeat -EntryNodeID $entryNode.id -PrimaryRouteID $primaryRouteID)
|
||||
|
||||
$packetPath = $lease.entry_http.path_template.
|
||||
Replace("{cluster_id}", $ClusterID).
|
||||
Replace("{channel_id}", [string]$lease.channel_id).
|
||||
Replace("{resource_id}", $resourceID)
|
||||
$packetUrl = $EntryBaseUrl.TrimEnd("/") + $packetPath
|
||||
$headers = @{
|
||||
"X-RAP-Service-Channel-Token" = [string]$lease.token.token
|
||||
"X-RAP-Fabric-Channel-ID" = [string]$lease.channel_id
|
||||
"X-RAP-Service-Class" = "vpn_packets"
|
||||
"X-RAP-Channel-Class" = "vpn_packet"
|
||||
}
|
||||
$response = Invoke-WebRequest -Method Post -Uri $packetUrl -Headers $headers -Body ([System.Text.Encoding]::UTF8.GetBytes("c18z67-alternate-remediation")) -ContentType "application/octet-stream" -TimeoutSec 30
|
||||
$acceptedBy = [string]$response.Headers["X-RAP-Service-Channel-Accepted-By"]
|
||||
|
||||
$accessTelemetry = $null
|
||||
$matchingChannel = $null
|
||||
for ($i = 0; $i -lt 10; $i++) {
|
||||
Start-Sleep -Seconds 3
|
||||
$accessTelemetry = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/fabric/service-channels/access-telemetry?actor_user_id=$ActorUserID&limit=20").fabric_service_channel_access_telemetry
|
||||
$channels = @()
|
||||
if ($accessTelemetry.PSObject.Properties.Name -contains "active_channels") {
|
||||
$channels = @($accessTelemetry.active_channels)
|
||||
}
|
||||
$matchingChannel = $channels | Where-Object { $_.channel_id -eq $lease.channel_id } | Select-Object -First 1
|
||||
if ($null -ne $matchingChannel -and [string](Get-PropertyValue -Item $matchingChannel -Name "remediation_action" -Default "") -eq "prefer_alternate_route") {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
$backendLine = (& ssh $DockerSSH "docker ps --format '{{.Names}} {{.Image}} {{.Status}}' | grep '^rap_test_backend '") | Out-String
|
||||
$nodeLines = (& ssh $DockerSSH "docker ps --format '{{.Names}} {{.Image}} {{.Status}}' | grep '^rap_test_node_test_'") | Out-String
|
||||
$leasePrimaryRouteID = [string](Get-PropertyValue -Item (Get-PropertyValue -Item $lease -Name "primary_route" -Default $null) -Name "route_id" -Default "")
|
||||
$leaseAlternates = @()
|
||||
if ($lease.PSObject.Properties.Name -contains "alternate_routes") {
|
||||
$leaseAlternates = @($lease.alternate_routes)
|
||||
}
|
||||
$leaseHasAlternate = (@($leaseAlternates | Where-Object { [string]$_.route_id -eq $alternateRouteID }).Count -ge 1)
|
||||
$remediationCommand = Get-PropertyValue -Item $matchingChannel -Name "remediation_command" -Default $null
|
||||
$commandAction = [string](Get-PropertyValue -Item $remediationCommand -Name "action" -Default "")
|
||||
$commandPrimaryRouteID = [string](Get-PropertyValue -Item $remediationCommand -Name "primary_route_id" -Default "")
|
||||
$commandReplacementRouteID = [string](Get-PropertyValue -Item $remediationCommand -Name "replacement_route_id" -Default "")
|
||||
$commandExpiresAt = [string](Get-PropertyValue -Item $remediationCommand -Name "expires_at" -Default "")
|
||||
$expectedReplacementRouteID = $commandReplacementRouteID
|
||||
$syntheticConfig = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/nodes/$($entryNode.id)/mesh/synthetic-config").synthetic_mesh_config
|
||||
$syntheticCommands = @()
|
||||
if ($syntheticConfig.PSObject.Properties.Name -contains "service_channel_remediation_commands") {
|
||||
$syntheticCommands = @($syntheticConfig.service_channel_remediation_commands)
|
||||
}
|
||||
$syntheticCommand = $syntheticCommands | Where-Object { [string]$_.channel_id -eq [string]$lease.channel_id } | Select-Object -First 1
|
||||
$syntheticCommandAction = [string](Get-PropertyValue -Item $syntheticCommand -Name "action" -Default "")
|
||||
$syntheticCommandReplacementRouteID = [string](Get-PropertyValue -Item $syntheticCommand -Name "replacement_route_id" -Default "")
|
||||
if ([string]::IsNullOrWhiteSpace($expectedReplacementRouteID)) {
|
||||
$expectedReplacementRouteID = $syntheticCommandReplacementRouteID
|
||||
}
|
||||
if ([string]::IsNullOrWhiteSpace($expectedReplacementRouteID)) {
|
||||
$expectedReplacementRouteID = $alternateRouteID
|
||||
}
|
||||
|
||||
$routeManagerDecision = $null
|
||||
$routeManagerRuntime = $null
|
||||
for ($i = 0; $i -lt 10; $i++) {
|
||||
Start-Sleep -Seconds 5
|
||||
$heartbeats = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/nodes/$($entryNode.id)/heartbeats?actor_user_id=$ActorUserID&limit=5").heartbeats
|
||||
foreach ($heartbeat in @($heartbeats)) {
|
||||
$runtimeReport = Get-PropertyValue -Item (Get-PropertyValue -Item $heartbeat -Name "metadata" -Default $null) -Name "fabric_service_channel_runtime_report" -Default $null
|
||||
$ingress = Get-PropertyValue -Item $runtimeReport -Name "ingress" -Default $null
|
||||
$routeManager = Get-PropertyValue -Item $ingress -Name "route_manager" -Default $null
|
||||
$routeManagerRuntime = $routeManager
|
||||
$decisions = @()
|
||||
if ($null -ne $routeManager -and $routeManager.PSObject.Properties.Name -contains "decisions") {
|
||||
$decisions = @($routeManager.decisions)
|
||||
}
|
||||
$routeManagerDecision = $decisions | Where-Object {
|
||||
[string]$_.route_id -eq $primaryRouteID -and
|
||||
[string]$_.replacement_route_id -eq $expectedReplacementRouteID -and
|
||||
[string]$_.decision_source -eq "service_channel_remediation_command"
|
||||
} | Select-Object -First 1
|
||||
if ($null -ne $routeManagerDecision) {
|
||||
break
|
||||
}
|
||||
}
|
||||
if ($null -ne $routeManagerDecision) {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
$baselineFallbackLocal = 0
|
||||
$baselineRouteFailures = 0
|
||||
$baselineFlowDropped = 0
|
||||
$baselineSchedulerDropped = 0
|
||||
$baselineHeartbeats = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/nodes/$($entryNode.id)/heartbeats?actor_user_id=$ActorUserID&limit=1").heartbeats
|
||||
if (@($baselineHeartbeats).Count -gt 0) {
|
||||
$baselineRuntimeReport = Get-PropertyValue -Item (Get-PropertyValue -Item $baselineHeartbeats[0] -Name "metadata" -Default $null) -Name "fabric_service_channel_runtime_report" -Default $null
|
||||
$baselineIngress = Get-PropertyValue -Item $baselineRuntimeReport -Name "ingress" -Default $null
|
||||
$baselineFlowScheduler = Get-PropertyValue -Item $baselineIngress -Name "flow_scheduler" -Default $null
|
||||
$baselineFallbackLocal = [int](Get-PropertyValue -Item $baselineIngress -Name "send_fallback_local" -Default 0)
|
||||
$baselineRouteFailures = [int](Get-PropertyValue -Item $baselineIngress -Name "send_route_failures" -Default 0)
|
||||
$baselineFlowDropped = [int](Get-PropertyValue -Item $baselineIngress -Name "send_flow_dropped" -Default 0)
|
||||
$baselineSchedulerDropped = [int](Get-PropertyValue -Item $baselineFlowScheduler -Name "dropped" -Default 0)
|
||||
}
|
||||
|
||||
$multiFlowPackets = @()
|
||||
foreach ($offset in 0..($expectedPressurePacketCount - 1)) {
|
||||
$port = 52000 + $offset
|
||||
$multiFlowPackets += ,(New-IPv4TcpPacket -SourcePort $port -DestinationPort 3389)
|
||||
}
|
||||
$multiFlowPayload = ConvertTo-VPNPacketBatch -Packets $multiFlowPackets
|
||||
$multiFlowPayloadPath = Join-Path ([System.IO.Path]::GetTempPath()) "$runId-vpn-packet-batch.bin"
|
||||
[System.IO.File]::WriteAllBytes($multiFlowPayloadPath, [byte[]]$multiFlowPayload)
|
||||
$bulkHeaders = @{} + $headers
|
||||
$bulkHeaders["X-RAP-Traffic-Class"] = "bulk"
|
||||
$bulkJobs = @()
|
||||
foreach ($jobIndex in 1..$expectedBulkRequestCount) {
|
||||
$jobPayloadPath = Join-Path ([System.IO.Path]::GetTempPath()) "$runId-vpn-packet-batch-$jobIndex.bin"
|
||||
Copy-Item -Path $multiFlowPayloadPath -Destination $jobPayloadPath -Force
|
||||
$bulkJobs += Start-Job -ArgumentList ($packetUrl + "?batch=true"), $bulkHeaders, $jobPayloadPath -ScriptBlock {
|
||||
param($uri, $headers, $path)
|
||||
try {
|
||||
$started = Get-Date
|
||||
$response = Invoke-WebRequest -Method Post -Uri $uri -Headers $headers -InFile $path -ContentType "application/vnd.rap.vpn-packet-batch.v1" -TimeoutSec 30
|
||||
[pscustomobject]@{
|
||||
status_code = [int]$response.StatusCode
|
||||
accepted_by = [string]$response.Headers["X-RAP-Service-Channel-Accepted-By"]
|
||||
duration_ms = [int]((Get-Date) - $started).TotalMilliseconds
|
||||
error = ""
|
||||
}
|
||||
} finally {
|
||||
Remove-Item -Path $path -Force -ErrorAction SilentlyContinue
|
||||
}
|
||||
}
|
||||
}
|
||||
Start-Sleep -Milliseconds 100
|
||||
$interactivePacket = New-IPv4TcpPacket -SourcePort 53000 -DestinationPort 22
|
||||
$interactivePayload = ConvertTo-VPNPacketBatch -Packets @($interactivePacket)
|
||||
$interactivePayloadPath = Join-Path ([System.IO.Path]::GetTempPath()) "$runId-vpn-interactive-packet-batch.bin"
|
||||
[System.IO.File]::WriteAllBytes($interactivePayloadPath, [byte[]]$interactivePayload)
|
||||
$interactiveHeaders = @{} + $headers
|
||||
$interactiveHeaders["X-RAP-Traffic-Class"] = "interactive"
|
||||
$interactiveStarted = Get-Date
|
||||
$interactiveResponse = Invoke-WebRequest -Method Post -Uri ($packetUrl + "?batch=true") -Headers $interactiveHeaders -InFile $interactivePayloadPath -ContentType "application/vnd.rap.vpn-packet-batch.v1" -TimeoutSec 30
|
||||
$interactiveDurationMs = [int]((Get-Date) - $interactiveStarted).TotalMilliseconds
|
||||
$interactiveAcceptedBy = [string]$interactiveResponse.Headers["X-RAP-Service-Channel-Accepted-By"]
|
||||
$bulkResponses = @($bulkJobs | Wait-Job -Timeout 60 | Receive-Job)
|
||||
$bulkJobs | Remove-Job -Force -ErrorAction SilentlyContinue
|
||||
$bulkAcceptedCount = @($bulkResponses | Where-Object { [int]$_.status_code -eq 202 -and [string]$_.accepted_by -eq "introspection" }).Count
|
||||
$postRemediationAcceptedBy = (@($bulkResponses | Select-Object -First 1).accepted_by | Out-String).Trim()
|
||||
$replacementTrafficObserved = $false
|
||||
$replacementLastSelected = ""
|
||||
$replacementFlowStat = $null
|
||||
$replacementFlowStats = @()
|
||||
$interactiveFlowStats = @()
|
||||
$bulkFlowStats = @()
|
||||
$postRemediationFallbackLocal = 0
|
||||
$postRemediationRouteFailures = 0
|
||||
$postRemediationFlowDropped = 0
|
||||
$postRemediationSchedulerDropped = 0
|
||||
$postRemediationHighWatermark = 0
|
||||
$postRemediationMaxInFlight = 0
|
||||
for ($i = 0; $i -lt 12; $i++) {
|
||||
Start-Sleep -Seconds 5
|
||||
$heartbeats = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/nodes/$($entryNode.id)/heartbeats?actor_user_id=$ActorUserID&limit=5").heartbeats
|
||||
foreach ($heartbeat in @($heartbeats)) {
|
||||
$runtimeReport = Get-PropertyValue -Item (Get-PropertyValue -Item $heartbeat -Name "metadata" -Default $null) -Name "fabric_service_channel_runtime_report" -Default $null
|
||||
$ingress = Get-PropertyValue -Item $runtimeReport -Name "ingress" -Default $null
|
||||
$replacementLastSelected = [string](Get-PropertyValue -Item $ingress -Name "last_selected_route_id" -Default "")
|
||||
$postRemediationFallbackLocal = [int](Get-PropertyValue -Item $ingress -Name "send_fallback_local" -Default 0)
|
||||
$postRemediationRouteFailures = [int](Get-PropertyValue -Item $ingress -Name "send_route_failures" -Default 0)
|
||||
$postRemediationFlowDropped = [int](Get-PropertyValue -Item $ingress -Name "send_flow_dropped" -Default 0)
|
||||
$flowScheduler = Get-PropertyValue -Item $ingress -Name "flow_scheduler" -Default $null
|
||||
$postRemediationSchedulerDropped = [int](Get-PropertyValue -Item $flowScheduler -Name "dropped" -Default 0)
|
||||
$postRemediationHighWatermark = [int](Get-PropertyValue -Item $flowScheduler -Name "high_watermark" -Default 0)
|
||||
$postRemediationMaxInFlight = [int](Get-PropertyValue -Item $flowScheduler -Name "max_in_flight" -Default 0)
|
||||
$channelStats = Get-PropertyValue -Item $flowScheduler -Name "channel_stats" -Default $null
|
||||
if ($null -ne $channelStats) {
|
||||
$replacementFlowStats = @()
|
||||
$interactiveFlowStats = @()
|
||||
$bulkFlowStats = @()
|
||||
foreach ($property in @($channelStats.PSObject.Properties)) {
|
||||
$stat = $property.Value
|
||||
if ([string](Get-PropertyValue -Item $stat -Name "last_route_id" -Default "") -eq $expectedReplacementRouteID) {
|
||||
$replacementFlowStats += $stat
|
||||
$replacementFlowStat = $stat
|
||||
$trafficClass = [string](Get-PropertyValue -Item $stat -Name "traffic_class" -Default "")
|
||||
if ($trafficClass -eq "interactive") {
|
||||
$interactiveFlowStats += $stat
|
||||
}
|
||||
if ($trafficClass -eq "bulk") {
|
||||
$bulkFlowStats += $stat
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if ($replacementLastSelected -eq $expectedReplacementRouteID -and $bulkFlowStats.Count -ge $expectedPressureFlowCount -and $interactiveFlowStats.Count -ge 1) {
|
||||
$replacementTrafficObserved = $true
|
||||
break
|
||||
}
|
||||
}
|
||||
if ($replacementTrafficObserved) {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
$result = [ordered]@{
|
||||
schema_version = "c18z67.service_channel_concurrent_qos_live_smoke.v1"
|
||||
run_id = $runId
|
||||
cluster_id = $ClusterID
|
||||
primary_route_id = $primaryRouteID
|
||||
alternate_route_id = $alternateRouteID
|
||||
expected_replacement_route_id = $expectedReplacementRouteID
|
||||
channel_id = [string]$lease.channel_id
|
||||
passed = [bool](
|
||||
$backendLine.Contains($ExpectedBackendImage) -and
|
||||
$nodeLines.Contains($ExpectedNodeAgentImage) -and
|
||||
[string]$lease.status -eq "ready" -and
|
||||
$leasePrimaryRouteID -eq $primaryRouteID -and
|
||||
$leaseHasAlternate -and
|
||||
[int]$response.StatusCode -eq 202 -and
|
||||
$acceptedBy -eq "introspection" -and
|
||||
$null -ne $matchingChannel -and
|
||||
[string](Get-PropertyValue -Item $matchingChannel -Name "primary_route_id" -Default "") -eq $primaryRouteID -and
|
||||
-not [bool](Get-PropertyValue -Item $matchingChannel -Name "force_backend_fallback" -Default $false) -and
|
||||
[string](Get-PropertyValue -Item $matchingChannel -Name "route_feedback_status" -Default "") -eq "fenced" -and
|
||||
[string](Get-PropertyValue -Item $matchingChannel -Name "remediation_action" -Default "") -eq "prefer_alternate_route" -and
|
||||
[string](Get-PropertyValue -Item $matchingChannel -Name "remediation_route_id" -Default "") -eq $expectedReplacementRouteID -and
|
||||
$null -ne $remediationCommand -and
|
||||
$commandAction -eq "prefer_alternate_route" -and
|
||||
$commandPrimaryRouteID -eq $primaryRouteID -and
|
||||
$commandReplacementRouteID -eq $expectedReplacementRouteID -and
|
||||
$commandExpiresAt.Length -gt 0 -and
|
||||
$null -ne $syntheticCommand -and
|
||||
$syntheticCommandAction -eq "prefer_alternate_route" -and
|
||||
$syntheticCommandReplacementRouteID -eq $expectedReplacementRouteID -and
|
||||
$null -ne $routeManagerDecision -and
|
||||
[string](Get-PropertyValue -Item $routeManagerDecision -Name "rebuild_status" -Default "") -eq "applied" -and
|
||||
$bulkResponses.Count -eq $expectedBulkRequestCount -and
|
||||
$bulkAcceptedCount -eq $expectedBulkRequestCount -and
|
||||
[int]$interactiveResponse.StatusCode -eq 202 -and
|
||||
$interactiveAcceptedBy -eq "introspection" -and
|
||||
$interactiveDurationMs -lt 2000 -and
|
||||
$replacementTrafficObserved -and
|
||||
$replacementLastSelected -eq $expectedReplacementRouteID -and
|
||||
$bulkFlowStats.Count -ge $expectedPressureFlowCount -and
|
||||
$interactiveFlowStats.Count -ge 1 -and
|
||||
$multiFlowPackets.Count -eq $expectedPressurePacketCount -and
|
||||
($postRemediationFallbackLocal - $baselineFallbackLocal) -eq 0 -and
|
||||
($postRemediationFlowDropped - $baselineFlowDropped) -eq 0 -and
|
||||
($postRemediationSchedulerDropped - $baselineSchedulerDropped) -eq 0 -and
|
||||
($postRemediationRouteFailures - $baselineRouteFailures) -eq 0 -and
|
||||
$null -ne $matchingChannel -and
|
||||
-not [bool](Get-PropertyValue -Item $matchingChannel -Name "force_backend_fallback" -Default $false) -and
|
||||
[string](Get-PropertyValue -Item $matchingChannel -Name "remediation_action" -Default "") -ne "use_backend_fallback"
|
||||
)
|
||||
checks = [ordered]@{
|
||||
backend_expected_image_deployed = $backendLine.Contains($ExpectedBackendImage)
|
||||
node_agent_expected_image_deployed = $nodeLines.Contains($ExpectedNodeAgentImage)
|
||||
lease_ready = ([string]$lease.status -eq "ready")
|
||||
lease_selected_primary_route = ($leasePrimaryRouteID -eq $primaryRouteID)
|
||||
lease_contains_alternate_route = $leaseHasAlternate
|
||||
packet_accepted = ([int]$response.StatusCode -eq 202)
|
||||
accepted_by_header_is_introspection = ($acceptedBy -eq "introspection")
|
||||
active_channel_visible = ($null -ne $matchingChannel)
|
||||
active_channel_not_backend_fallback = ($null -ne $matchingChannel -and -not [bool](Get-PropertyValue -Item $matchingChannel -Name "force_backend_fallback" -Default $false))
|
||||
route_quality_fenced = ($null -ne $matchingChannel -and [string](Get-PropertyValue -Item $matchingChannel -Name "route_feedback_status" -Default "") -eq "fenced")
|
||||
remediation_prefers_alternate = ($null -ne $matchingChannel -and [string](Get-PropertyValue -Item $matchingChannel -Name "remediation_action" -Default "") -eq "prefer_alternate_route")
|
||||
remediation_route_is_alternate = ($null -ne $matchingChannel -and [string](Get-PropertyValue -Item $matchingChannel -Name "remediation_route_id" -Default "") -eq $expectedReplacementRouteID)
|
||||
remediation_command_visible = ($null -ne $remediationCommand)
|
||||
remediation_command_prefers_alternate = ($commandAction -eq "prefer_alternate_route")
|
||||
remediation_command_primary_route_matches = ($commandPrimaryRouteID -eq $primaryRouteID)
|
||||
remediation_command_replacement_route_matches = ($commandReplacementRouteID -eq $expectedReplacementRouteID)
|
||||
remediation_command_has_ttl = ($commandExpiresAt.Length -gt 0)
|
||||
synthetic_config_command_visible = ($null -ne $syntheticCommand)
|
||||
synthetic_config_command_prefers_alternate = ($syntheticCommandAction -eq "prefer_alternate_route")
|
||||
synthetic_config_command_replacement_route_matches = ($syntheticCommandReplacementRouteID -eq $expectedReplacementRouteID)
|
||||
node_route_manager_consumed_command = ($null -ne $routeManagerDecision)
|
||||
node_route_manager_applied_replacement = ($null -ne $routeManagerDecision -and [string](Get-PropertyValue -Item $routeManagerDecision -Name "rebuild_status" -Default "") -eq "applied")
|
||||
bulk_request_count_matches = ($bulkResponses.Count -eq $expectedBulkRequestCount)
|
||||
bulk_requests_accepted_by_introspection = ($bulkAcceptedCount -eq $expectedBulkRequestCount)
|
||||
interactive_packet_accepted = ([int]$interactiveResponse.StatusCode -eq 202)
|
||||
interactive_accepted_by_introspection = ($interactiveAcceptedBy -eq "introspection")
|
||||
interactive_completed_under_bulk_pressure = ($interactiveDurationMs -lt 2000)
|
||||
replacement_traffic_observed = $replacementTrafficObserved
|
||||
replacement_last_selected_route_matches = ($replacementLastSelected -eq $expectedReplacementRouteID)
|
||||
replacement_flow_stat_observed = ($null -ne $replacementFlowStat)
|
||||
replacement_pressure_flow_stats_observed = ($bulkFlowStats.Count -ge $expectedPressureFlowCount)
|
||||
replacement_interactive_flow_stat_observed = ($interactiveFlowStats.Count -ge 1)
|
||||
pressure_packet_count_matches = ($multiFlowPackets.Count -eq $expectedPressurePacketCount)
|
||||
no_local_gateway_fallback_after_replacement = (($postRemediationFallbackLocal - $baselineFallbackLocal) -eq 0)
|
||||
no_flow_drops_after_replacement = (($postRemediationFlowDropped - $baselineFlowDropped) -eq 0 -and ($postRemediationSchedulerDropped - $baselineSchedulerDropped) -eq 0)
|
||||
no_route_failures_after_replacement = (($postRemediationRouteFailures - $baselineRouteFailures) -eq 0)
|
||||
backend_fallback_not_recommended = ($null -ne $matchingChannel -and -not [bool](Get-PropertyValue -Item $matchingChannel -Name "force_backend_fallback" -Default $false) -and [string](Get-PropertyValue -Item $matchingChannel -Name "remediation_action" -Default "") -ne "use_backend_fallback")
|
||||
}
|
||||
summary = [ordered]@{
|
||||
backend_container = $backendLine.Trim()
|
||||
node_containers = $nodeLines.Trim()
|
||||
accepted_by = $acceptedBy
|
||||
lease_status = [string]$lease.status
|
||||
lease_primary_route_id = $leasePrimaryRouteID
|
||||
lease_alternate_route_count = $leaseAlternates.Count
|
||||
access_status = [string]$accessTelemetry.status
|
||||
active_channel_count = [int]$accessTelemetry.active_channel_count
|
||||
correlated_route_count = [int]$accessTelemetry.correlated_route_count
|
||||
degraded_route_count = [int]$accessTelemetry.degraded_route_count
|
||||
degraded_fallback_channel_count = [int]$accessTelemetry.degraded_fallback_channel_count
|
||||
synthetic_command = $syntheticCommand
|
||||
route_manager_decision = $routeManagerDecision
|
||||
route_manager_runtime = $routeManagerRuntime
|
||||
post_remediation_accepted_by = $postRemediationAcceptedBy
|
||||
bulk_request_count = $bulkResponses.Count
|
||||
bulk_accepted_count = $bulkAcceptedCount
|
||||
bulk_responses = $bulkResponses
|
||||
post_remediation_packet_count = ($multiFlowPackets.Count * $bulkResponses.Count)
|
||||
interactive_accepted_by = $interactiveAcceptedBy
|
||||
interactive_duration_ms = $interactiveDurationMs
|
||||
interactive_packet_count = $expectedInteractivePacketCount
|
||||
expected_pressure_flow_count = $expectedPressureFlowCount
|
||||
replacement_last_selected_route_id = $replacementLastSelected
|
||||
replacement_flow_stat_count = $replacementFlowStats.Count
|
||||
replacement_bulk_flow_stat_count = $bulkFlowStats.Count
|
||||
replacement_interactive_flow_stat_count = $interactiveFlowStats.Count
|
||||
replacement_flow_stat = $replacementFlowStat
|
||||
replacement_flow_stats = $replacementFlowStats
|
||||
replacement_bulk_flow_stats = $bulkFlowStats
|
||||
replacement_interactive_flow_stats = $interactiveFlowStats
|
||||
baseline_send_fallback_local = $baselineFallbackLocal
|
||||
baseline_send_route_failures = $baselineRouteFailures
|
||||
baseline_send_flow_dropped = $baselineFlowDropped
|
||||
baseline_scheduler_dropped = $baselineSchedulerDropped
|
||||
fallback_local_delta = ($postRemediationFallbackLocal - $baselineFallbackLocal)
|
||||
route_failure_delta = ($postRemediationRouteFailures - $baselineRouteFailures)
|
||||
flow_drop_delta = ($postRemediationFlowDropped - $baselineFlowDropped)
|
||||
scheduler_drop_delta = ($postRemediationSchedulerDropped - $baselineSchedulerDropped)
|
||||
post_remediation_send_fallback_local = $postRemediationFallbackLocal
|
||||
post_remediation_send_route_failures = $postRemediationRouteFailures
|
||||
post_remediation_send_flow_dropped = $postRemediationFlowDropped
|
||||
post_remediation_scheduler_dropped = $postRemediationSchedulerDropped
|
||||
post_remediation_scheduler_high_watermark = $postRemediationHighWatermark
|
||||
post_remediation_scheduler_max_in_flight = $postRemediationMaxInFlight
|
||||
matching_channel = $matchingChannel
|
||||
}
|
||||
}
|
||||
$failedChecks = @($result.checks.GetEnumerator() | Where-Object { $_.Value -ne $true })
|
||||
$result.summary.failed_checks = @($failedChecks | ForEach-Object { $_.Key })
|
||||
}
|
||||
finally {
|
||||
if ($primaryRouteID) {
|
||||
try { Invoke-Api -Method POST -Path "/clusters/$ClusterID/mesh/route-intents/$primaryRouteID/expire" -Body @{ actor_user_id = $ActorUserID } | Out-Null } catch {}
|
||||
}
|
||||
if ($alternateRouteID) {
|
||||
try { Invoke-Api -Method POST -Path "/clusters/$ClusterID/mesh/route-intents/$alternateRouteID/expire" -Body @{ actor_user_id = $ActorUserID } | Out-Null } catch {}
|
||||
}
|
||||
}
|
||||
|
||||
$resultFullPath = Join-Path $repoRoot $ResultPath
|
||||
$resultDir = Split-Path -Parent $resultFullPath
|
||||
if (-not (Test-Path $resultDir)) {
|
||||
New-Item -ItemType Directory -Path $resultDir | Out-Null
|
||||
}
|
||||
$result | ConvertTo-Json -Depth 100 | Set-Content -Path $resultFullPath -Encoding UTF8
|
||||
|
||||
if ($failedChecks.Count -gt 0) {
|
||||
throw "C18Z67 failed checks: $(@($failedChecks | ForEach-Object { $_.Key }) -join ', '). Result: $resultFullPath"
|
||||
}
|
||||
|
||||
Write-Host "C18Z67 service-channel concurrent qos live smoke passed. Result: $resultFullPath"
|
||||
$result
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,92 @@
|
||||
param(
|
||||
[string]$ApiBaseUrl = "http://192.168.200.61:18121/api/v1",
|
||||
[string]$ClusterID = "cfc0743d-d960-49fb-9de8-96e063d5e4aa",
|
||||
[string]$ActorUserID = "f67d943f-5397-4b3a-a229-695fe67ad700",
|
||||
[string]$EntryNodeName = "test-1",
|
||||
[string]$ExpectedBackendImage = "rap-backend:fabric-service-channel-0.2.281-c18z109",
|
||||
[string]$ExpectedNodeAgentImage = "rap-node-agent:codex-fabric-stability-20260512b",
|
||||
[string]$ResultPath = "artifacts\c18z69-service-channel-adaptive-backpressure-smoke-result.json"
|
||||
)
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
$scriptDir = Split-Path -Parent $MyInvocation.MyCommand.Path
|
||||
$repoRoot = (Resolve-Path (Join-Path $scriptDir "..\..")).ProviderPath
|
||||
$c18z67 = Join-Path $scriptDir "c18z67-service-channel-concurrent-qos-live-smoke.ps1"
|
||||
|
||||
& powershell -NoProfile -ExecutionPolicy Bypass -File $c18z67 -ExpectedBackendImage $ExpectedBackendImage -ExpectedNodeAgentImage $ExpectedNodeAgentImage | Out-Host
|
||||
|
||||
function Invoke-Api {
|
||||
param([string]$Path)
|
||||
return Invoke-RestMethod -Method GET -Uri "$ApiBaseUrl$Path" -TimeoutSec 30
|
||||
}
|
||||
|
||||
$nodes = (Invoke-Api -Path "/clusters/$ClusterID/nodes?actor_user_id=$ActorUserID").nodes
|
||||
$entryNode = @($nodes | Where-Object { $_.name -eq $EntryNodeName }) | Select-Object -First 1
|
||||
if ($null -eq $entryNode) {
|
||||
throw "entry node '$EntryNodeName' not found"
|
||||
}
|
||||
|
||||
$heartbeats = (Invoke-Api -Path "/clusters/$ClusterID/nodes/$($entryNode.id)/heartbeats?actor_user_id=$ActorUserID&limit=1").heartbeats
|
||||
if ($null -eq $heartbeats -or @($heartbeats).Count -eq 0) {
|
||||
throw "no heartbeat found for entry node '$EntryNodeName'"
|
||||
}
|
||||
|
||||
$flowScheduler = $heartbeats[0].metadata.fabric_service_channel_runtime_report.ingress.flow_scheduler
|
||||
if ($null -eq $flowScheduler) {
|
||||
throw "latest heartbeat does not include fabric service-channel flow scheduler"
|
||||
}
|
||||
|
||||
$windows = $flowScheduler.recommended_parallel_windows
|
||||
$trafficClasses = $flowScheduler.traffic_class_counts
|
||||
$bulkWindow = [int]$windows.bulk
|
||||
$interactiveWindow = [int]$windows.interactive
|
||||
$controlWindow = [int]$windows.control
|
||||
$bulkCount = [int]$trafficClasses.bulk
|
||||
$interactiveCount = [int]$trafficClasses.interactive
|
||||
$adaptiveActive = [bool]$flowScheduler.adaptive_backpressure_active
|
||||
$adaptiveReason = [string]$flowScheduler.adaptive_backpressure_reason
|
||||
$dropped = [int]$flowScheduler.dropped
|
||||
|
||||
$checks = [ordered]@{
|
||||
adaptive_backpressure_active = $adaptiveActive
|
||||
adaptive_reason_protects_interactive = ($adaptiveReason -eq "bulk_window_reduced_to_protect_interactive")
|
||||
bulk_window_reduced = ($bulkWindow -eq 1)
|
||||
interactive_window_preserved = ($interactiveWindow -ge 4)
|
||||
control_window_preserved = ($controlWindow -ge 4)
|
||||
bulk_pressure_observed = ($bulkCount -ge 16)
|
||||
interactive_qos_observed = ($interactiveCount -gt 0)
|
||||
no_scheduler_drops = ($dropped -eq 0)
|
||||
}
|
||||
$failed = @($checks.GetEnumerator() | Where-Object { -not $_.Value } | ForEach-Object { $_.Key })
|
||||
|
||||
$result = [ordered]@{
|
||||
schema_version = "c18z69.service_channel_adaptive_backpressure_smoke.v1"
|
||||
run_id = "c18z69-" + (Get-Date -Format "yyyyMMdd-HHmmss")
|
||||
cluster_id = $ClusterID
|
||||
entry_node_id = $entryNode.id
|
||||
entry_node_name = $entryNode.name
|
||||
passed = ($failed.Count -eq 0)
|
||||
checks = $checks
|
||||
failed_checks = $failed
|
||||
summary = [ordered]@{
|
||||
adaptive_backpressure_active = $adaptiveActive
|
||||
adaptive_backpressure_reason = $adaptiveReason
|
||||
recommended_parallel_windows = $windows
|
||||
traffic_class_counts = $trafficClasses
|
||||
channel_count = [int]$flowScheduler.channel_count
|
||||
high_watermark = [int]$flowScheduler.high_watermark
|
||||
max_in_flight = [int]$flowScheduler.max_in_flight
|
||||
dropped = $dropped
|
||||
}
|
||||
}
|
||||
|
||||
$target = Join-Path $repoRoot $ResultPath
|
||||
$result | ConvertTo-Json -Depth 40 | Set-Content -Path $target -Encoding UTF8
|
||||
if (-not $result.passed) {
|
||||
throw "C18Z69 adaptive backpressure smoke failed: $($failed -join ', ')"
|
||||
}
|
||||
|
||||
Write-Host "C18Z69 service-channel adaptive backpressure smoke passed. Result: $target"
|
||||
$result
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,142 @@
|
||||
param(
|
||||
[string]$ApiBaseUrl = "http://192.168.200.61:18121/api/v1",
|
||||
[string]$ClusterID = "cfc0743d-d960-49fb-9de8-96e063d5e4aa",
|
||||
[string]$ActorUserID = "f67d943f-5397-4b3a-a229-695fe67ad700",
|
||||
[string]$EntryNodeName = "test-1",
|
||||
[string]$ResultPath = "artifacts\c18z71-service-channel-adaptive-policy-smoke-result.json"
|
||||
)
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
$scriptDir = Split-Path -Parent $MyInvocation.MyCommand.Path
|
||||
$repoRoot = (Resolve-Path (Join-Path $scriptDir "..\..")).ProviderPath
|
||||
$c18z67 = Join-Path $scriptDir "c18z67-service-channel-concurrent-qos-live-smoke.ps1"
|
||||
|
||||
function Invoke-Api {
|
||||
param(
|
||||
[string]$Method,
|
||||
[string]$Path,
|
||||
[object]$Body = $null
|
||||
)
|
||||
$params = @{
|
||||
Method = $Method
|
||||
Uri = "$ApiBaseUrl$Path"
|
||||
TimeoutSec = 30
|
||||
}
|
||||
if ($null -ne $Body) {
|
||||
$params.ContentType = "application/json"
|
||||
$params.Body = ($Body | ConvertTo-Json -Depth 30)
|
||||
}
|
||||
return Invoke-RestMethod @params
|
||||
}
|
||||
|
||||
function Get-PropertyValue {
|
||||
param([object]$Item, [string]$Name, [object]$Default = $null)
|
||||
if ($null -eq $Item) { return $Default }
|
||||
$property = $Item.PSObject.Properties[$Name]
|
||||
if ($null -eq $property) { return $Default }
|
||||
return $property.Value
|
||||
}
|
||||
|
||||
$policyBody = @{
|
||||
actor_user_id = $ActorUserID
|
||||
max_parallel_window = 6
|
||||
bulk_pressure_channel_threshold = 8
|
||||
queue_pressure_high_watermark = 8
|
||||
queue_pressure_max_in_flight = 8
|
||||
class_windows = @{
|
||||
control = 6
|
||||
interactive = 6
|
||||
reliable = 4
|
||||
bulk = 2
|
||||
droppable = 1
|
||||
}
|
||||
}
|
||||
$updatedPolicy = (Invoke-Api -Method PUT -Path "/clusters/$ClusterID/fabric/service-channels/adaptive-policy" -Body $policyBody).fabric_service_channel_adaptive_policy
|
||||
$expectedFingerprint = [string]$updatedPolicy.fingerprint
|
||||
|
||||
& powershell -NoProfile -ExecutionPolicy Bypass -File $c18z67 | Out-Host
|
||||
|
||||
$nodes = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/nodes?actor_user_id=$ActorUserID").nodes
|
||||
$entryNode = @($nodes | Where-Object { $_.name -eq $EntryNodeName }) | Select-Object -First 1
|
||||
if ($null -eq $entryNode) {
|
||||
throw "entry node '$EntryNodeName' not found"
|
||||
}
|
||||
|
||||
$flowScheduler = $null
|
||||
for ($i = 0; $i -lt 12; $i++) {
|
||||
Start-Sleep -Seconds 5
|
||||
$heartbeats = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/nodes/$($entryNode.id)/heartbeats?actor_user_id=$ActorUserID&limit=5").heartbeats
|
||||
foreach ($heartbeat in @($heartbeats)) {
|
||||
$runtimeReport = Get-PropertyValue -Item (Get-PropertyValue -Item $heartbeat -Name "metadata" -Default $null) -Name "fabric_service_channel_runtime_report" -Default $null
|
||||
$ingress = Get-PropertyValue -Item $runtimeReport -Name "ingress" -Default $null
|
||||
$candidate = Get-PropertyValue -Item $ingress -Name "flow_scheduler" -Default $null
|
||||
if ($null -eq $candidate) { continue }
|
||||
if ([string](Get-PropertyValue -Item $candidate -Name "adaptive_policy_fingerprint" -Default "") -eq $expectedFingerprint) {
|
||||
$flowScheduler = $candidate
|
||||
break
|
||||
}
|
||||
}
|
||||
if ($null -ne $flowScheduler) { break }
|
||||
}
|
||||
if ($null -eq $flowScheduler) {
|
||||
throw "entry node did not report adaptive policy fingerprint '$expectedFingerprint'"
|
||||
}
|
||||
|
||||
$windows = Get-PropertyValue -Item $flowScheduler -Name "recommended_parallel_windows" -Default $null
|
||||
$checks = [ordered]@{
|
||||
policy_fingerprint_persisted = ($expectedFingerprint.Length -gt 0)
|
||||
node_reported_policy_fingerprint = ([string](Get-PropertyValue -Item $flowScheduler -Name "adaptive_policy_fingerprint" -Default "") -eq $expectedFingerprint)
|
||||
adaptive_backpressure_active = [bool](Get-PropertyValue -Item $flowScheduler -Name "adaptive_backpressure_active" -Default $false)
|
||||
bulk_window_uses_policy = ([int](Get-PropertyValue -Item $windows -Name "bulk" -Default 0) -eq 2)
|
||||
droppable_window_uses_policy = ([int](Get-PropertyValue -Item $windows -Name "droppable" -Default 0) -eq 1)
|
||||
reliable_window_uses_policy = ([int](Get-PropertyValue -Item $windows -Name "reliable" -Default 0) -le 4)
|
||||
interactive_window_uses_policy_ceiling = ([int](Get-PropertyValue -Item $windows -Name "interactive" -Default 0) -eq 6)
|
||||
control_window_uses_policy_ceiling = ([int](Get-PropertyValue -Item $windows -Name "control" -Default 0) -eq 6)
|
||||
}
|
||||
$failed = @($checks.GetEnumerator() | Where-Object { -not $_.Value } | ForEach-Object { $_.Key })
|
||||
|
||||
$result = [ordered]@{
|
||||
schema_version = "c18z71.service_channel_adaptive_policy_smoke.v1"
|
||||
run_id = "c18z71-" + (Get-Date -Format "yyyyMMdd-HHmmss")
|
||||
cluster_id = $ClusterID
|
||||
entry_node_id = $entryNode.id
|
||||
entry_node_name = $entryNode.name
|
||||
passed = ($failed.Count -eq 0)
|
||||
checks = $checks
|
||||
failed_checks = $failed
|
||||
summary = [ordered]@{
|
||||
adaptive_policy = $updatedPolicy
|
||||
flow_scheduler = $flowScheduler
|
||||
}
|
||||
}
|
||||
|
||||
$target = Join-Path $repoRoot $ResultPath
|
||||
$result | ConvertTo-Json -Depth 50 | Set-Content -Path $target -Encoding UTF8
|
||||
|
||||
try {
|
||||
Invoke-Api -Method PUT -Path "/clusters/$ClusterID/fabric/service-channels/adaptive-policy" -Body @{
|
||||
actor_user_id = $ActorUserID
|
||||
max_parallel_window = 4
|
||||
bulk_pressure_channel_threshold = 16
|
||||
queue_pressure_high_watermark = 16
|
||||
queue_pressure_max_in_flight = 16
|
||||
class_windows = @{
|
||||
control = 4
|
||||
interactive = 4
|
||||
reliable = 3
|
||||
bulk = 1
|
||||
droppable = 1
|
||||
}
|
||||
} | Out-Null
|
||||
} catch {
|
||||
Write-Warning "failed to restore default adaptive policy after smoke: $($_.Exception.Message)"
|
||||
}
|
||||
|
||||
if (-not $result.passed) {
|
||||
throw "C18Z71 adaptive policy smoke failed: $($failed -join ', ')"
|
||||
}
|
||||
|
||||
Write-Host "C18Z71 service-channel adaptive policy smoke passed. Result: $target"
|
||||
$result
|
||||
@@ -0,0 +1,136 @@
|
||||
param(
|
||||
[string]$ApiBaseUrl = "http://192.168.200.61:18121/api/v1",
|
||||
[string]$ClusterID = "cfc0743d-d960-49fb-9de8-96e063d5e4aa",
|
||||
[string]$ActorUserID = "f67d943f-5397-4b3a-a229-695fe67ad700",
|
||||
[string]$EntryNodeName = "test-1",
|
||||
[string]$NonPreferredEntryNodeName = "test-2",
|
||||
[string]$ExitNodeName = "test-3",
|
||||
[string]$NonPreferredExitNodeName = "test-2",
|
||||
[string]$ResultPath = "artifacts\c18z72-service-channel-pool-policy-smoke-result.json"
|
||||
)
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
$scriptDir = Split-Path -Parent $MyInvocation.MyCommand.Path
|
||||
$repoRoot = (Resolve-Path (Join-Path $scriptDir "..\..")).ProviderPath
|
||||
|
||||
function Invoke-Api {
|
||||
param(
|
||||
[string]$Method,
|
||||
[string]$Path,
|
||||
[object]$Body = $null
|
||||
)
|
||||
$params = @{
|
||||
Method = $Method
|
||||
Uri = "$ApiBaseUrl$Path"
|
||||
TimeoutSec = 30
|
||||
}
|
||||
if ($null -ne $Body) {
|
||||
$params.ContentType = "application/json"
|
||||
$params.Body = ($Body | ConvertTo-Json -Depth 30)
|
||||
}
|
||||
return Invoke-RestMethod @params
|
||||
}
|
||||
|
||||
function Select-NodeByName {
|
||||
param([object[]]$Nodes, [string]$Name)
|
||||
$node = @($Nodes | Where-Object { $_.name -eq $Name }) | Select-Object -First 1
|
||||
if ($null -eq $node) {
|
||||
throw "node '$Name' not found"
|
||||
}
|
||||
return $node
|
||||
}
|
||||
|
||||
$nodes = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/nodes?actor_user_id=$ActorUserID").nodes
|
||||
$entryNode = Select-NodeByName -Nodes $nodes -Name $EntryNodeName
|
||||
$otherEntryNode = Select-NodeByName -Nodes $nodes -Name $NonPreferredEntryNodeName
|
||||
$exitNode = Select-NodeByName -Nodes $nodes -Name $ExitNodeName
|
||||
$otherExitNode = Select-NodeByName -Nodes $nodes -Name $NonPreferredExitNodeName
|
||||
|
||||
$policy = (Invoke-Api -Method PUT -Path "/clusters/$ClusterID/fabric/service-channels/pool-policy" -Body @{
|
||||
actor_user_id = $ActorUserID
|
||||
entry_pool_node_ids = @($entryNode.id)
|
||||
exit_pool_node_ids = @($exitNode.id)
|
||||
preferred_entry_node_id = $entryNode.id
|
||||
preferred_exit_node_id = $exitNode.id
|
||||
selection_strategy = "preferred_first"
|
||||
route_rebuild = "automatic"
|
||||
entry_failover = "automatic"
|
||||
exit_failover = "automatic"
|
||||
backend_fallback_allowed = $true
|
||||
sticky_session = $true
|
||||
}).fabric_service_channel_pool_policy
|
||||
|
||||
$lease = (Invoke-Api -Method POST -Path "/clusters/$ClusterID/fabric/service-channels/leases" -Body @{
|
||||
actor_user_id = $ActorUserID
|
||||
organization_id = "c18z72-org"
|
||||
user_id = "c18z72-user"
|
||||
resource_id = "c18z72-pool-policy-smoke"
|
||||
service_class = "vpn_packets"
|
||||
entry_node_ids = @($otherEntryNode.id, $entryNode.id)
|
||||
exit_node_ids = @($otherExitNode.id, $exitNode.id)
|
||||
allowed_channels = @("vpn_packet", "fabric_control")
|
||||
ttl_seconds = 5
|
||||
}).fabric_service_channel_lease
|
||||
|
||||
$entryPoolIDs = @($lease.entry_pool | ForEach-Object { $_.node_id })
|
||||
$exitPoolIDs = @($lease.exit_pool | ForEach-Object { $_.node_id })
|
||||
$checks = [ordered]@{
|
||||
policy_fingerprint_persisted = ([string]$policy.fingerprint).Length -gt 0
|
||||
selected_entry_from_policy = ([string]$lease.selected_entry_node_id -eq [string]$entryNode.id)
|
||||
selected_exit_from_policy = ([string]$lease.selected_exit_node_id -eq [string]$exitNode.id)
|
||||
entry_pool_constrained = ($entryPoolIDs.Count -eq 1 -and $entryPoolIDs[0] -eq [string]$entryNode.id)
|
||||
exit_pool_constrained = ($exitPoolIDs.Count -eq 1 -and $exitPoolIDs[0] -eq [string]$exitNode.id)
|
||||
lease_has_pool_policy = ($null -ne $lease.pool_policy -and [string]$lease.pool_policy.fingerprint -eq [string]$policy.fingerprint)
|
||||
authority_payload_present = ([string]$lease.authority_payload).Length -gt 0
|
||||
}
|
||||
$failed = @($checks.GetEnumerator() | Where-Object { -not $_.Value } | ForEach-Object { $_.Key })
|
||||
|
||||
$result = [ordered]@{
|
||||
schema_version = "c18z72.service_channel_pool_policy_smoke.v1"
|
||||
run_id = "c18z72-" + (Get-Date -Format "yyyyMMdd-HHmmss")
|
||||
cluster_id = $ClusterID
|
||||
passed = ($failed.Count -eq 0)
|
||||
checks = $checks
|
||||
failed_checks = $failed
|
||||
summary = [ordered]@{
|
||||
policy = $policy
|
||||
lease = $lease
|
||||
entry_pool_ids = $entryPoolIDs
|
||||
exit_pool_ids = $exitPoolIDs
|
||||
}
|
||||
}
|
||||
|
||||
$target = Join-Path $repoRoot $ResultPath
|
||||
$result | ConvertTo-Json -Depth 60 | Set-Content -Path $target -Encoding UTF8
|
||||
|
||||
try {
|
||||
Start-Sleep -Seconds 6
|
||||
Invoke-Api -Method POST -Path "/clusters/$ClusterID/fabric/service-channels/leases/cleanup" -Body @{
|
||||
actor_user_id = $ActorUserID
|
||||
limit = 50
|
||||
} | Out-Null
|
||||
Invoke-Api -Method PUT -Path "/clusters/$ClusterID/fabric/service-channels/pool-policy" -Body @{
|
||||
actor_user_id = $ActorUserID
|
||||
entry_pool_node_ids = @()
|
||||
exit_pool_node_ids = @()
|
||||
preferred_entry_node_id = ""
|
||||
preferred_exit_node_id = ""
|
||||
selection_strategy = "fastest_healthy"
|
||||
route_rebuild = "automatic"
|
||||
entry_failover = "automatic"
|
||||
exit_failover = "automatic"
|
||||
backend_fallback_allowed = $true
|
||||
sticky_session = $true
|
||||
} | Out-Null
|
||||
} catch {
|
||||
Write-Warning "failed to restore default pool policy after smoke: $($_.Exception.Message)"
|
||||
}
|
||||
|
||||
if (-not $result.passed) {
|
||||
throw "C18Z72 pool policy smoke failed: $($failed -join ', ')"
|
||||
}
|
||||
|
||||
Write-Host "C18Z72 service-channel pool policy smoke passed. Result: $target"
|
||||
$result
|
||||
@@ -0,0 +1,138 @@
|
||||
param(
|
||||
[string]$ApiBaseUrl = "http://192.168.200.61:18121/api/v1",
|
||||
[string]$ClusterID = "cfc0743d-d960-49fb-9de8-96e063d5e4aa",
|
||||
[string]$ActorUserID = "f67d943f-5397-4b3a-a229-695fe67ad700",
|
||||
[string]$EntryNodeName = "test-1",
|
||||
[string]$ExitNodeName = "test-3",
|
||||
[string]$ResultPath = "artifacts\c18z73-service-channel-pool-policy-remediation-guard-smoke-result.json"
|
||||
)
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
$scriptDir = Split-Path -Parent $MyInvocation.MyCommand.Path
|
||||
$repoRoot = (Resolve-Path (Join-Path $scriptDir "..\..")).ProviderPath
|
||||
|
||||
function Invoke-Api {
|
||||
param(
|
||||
[string]$Method,
|
||||
[string]$Path,
|
||||
[object]$Body = $null
|
||||
)
|
||||
$params = @{
|
||||
Method = $Method
|
||||
Uri = "$ApiBaseUrl$Path"
|
||||
TimeoutSec = 30
|
||||
}
|
||||
if ($null -ne $Body) {
|
||||
$params.ContentType = "application/json"
|
||||
$params.Body = ($Body | ConvertTo-Json -Depth 30)
|
||||
}
|
||||
return Invoke-RestMethod @params
|
||||
}
|
||||
|
||||
function Select-NodeByName {
|
||||
param([object[]]$Nodes, [string]$Name)
|
||||
$node = @($Nodes | Where-Object { $_.name -eq $Name }) | Select-Object -First 1
|
||||
if ($null -eq $node) {
|
||||
throw "node '$Name' not found"
|
||||
}
|
||||
return $node
|
||||
}
|
||||
|
||||
function Get-PropertyValue {
|
||||
param([object]$Item, [string]$Name, [object]$Default = $null)
|
||||
if ($null -eq $Item) { return $Default }
|
||||
$property = $Item.PSObject.Properties[$Name]
|
||||
if ($null -eq $property) { return $Default }
|
||||
return $property.Value
|
||||
}
|
||||
|
||||
$nodes = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/nodes?actor_user_id=$ActorUserID").nodes
|
||||
$entryNode = Select-NodeByName -Nodes $nodes -Name $EntryNodeName
|
||||
$exitNode = Select-NodeByName -Nodes $nodes -Name $ExitNodeName
|
||||
|
||||
$policy = (Invoke-Api -Method PUT -Path "/clusters/$ClusterID/fabric/service-channels/pool-policy" -Body @{
|
||||
actor_user_id = $ActorUserID
|
||||
entry_pool_node_ids = @($entryNode.id)
|
||||
exit_pool_node_ids = @($exitNode.id)
|
||||
preferred_entry_node_id = $entryNode.id
|
||||
preferred_exit_node_id = $exitNode.id
|
||||
selection_strategy = "preferred_first"
|
||||
route_rebuild = "automatic"
|
||||
entry_failover = "automatic"
|
||||
exit_failover = "automatic"
|
||||
backend_fallback_allowed = $true
|
||||
sticky_session = $true
|
||||
}).fabric_service_channel_pool_policy
|
||||
|
||||
$lease = (Invoke-Api -Method POST -Path "/clusters/$ClusterID/fabric/service-channels/leases" -Body @{
|
||||
actor_user_id = $ActorUserID
|
||||
organization_id = "c18z73-org"
|
||||
user_id = "c18z73-user"
|
||||
resource_id = "c18z73-pool-policy-remediation-guard-smoke"
|
||||
service_class = "vpn_packets"
|
||||
entry_node_ids = @($entryNode.id)
|
||||
exit_node_ids = @($exitNode.id)
|
||||
allowed_channels = @("vpn_packet", "fabric_control")
|
||||
ttl_seconds = 5
|
||||
}).fabric_service_channel_lease
|
||||
|
||||
$accessTelemetry = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/fabric/service-channels/access-telemetry?actor_user_id=$ActorUserID&limit=50").fabric_service_channel_access_telemetry
|
||||
$channel = @($accessTelemetry.active_channels | Where-Object { $_.channel_id -eq $lease.channel_id }) | Select-Object -First 1
|
||||
|
||||
$checks = [ordered]@{
|
||||
policy_fingerprint_persisted = ([string]$policy.fingerprint).Length -gt 0
|
||||
lease_has_policy_fingerprint = ($null -ne $lease.pool_policy -and [string]$lease.pool_policy.fingerprint -eq [string]$policy.fingerprint)
|
||||
access_channel_found = ($null -ne $channel)
|
||||
access_channel_projects_policy_fingerprint = ($null -ne $channel -and [string](Get-PropertyValue -Item $channel -Name "pool_policy_fingerprint" -Default "") -eq [string]$policy.fingerprint)
|
||||
remediation_guard_absent_on_healthy_route = ($null -ne $channel -and ([string](Get-PropertyValue -Item $channel -Name "remediation_guard_status" -Default "")).Length -eq 0)
|
||||
}
|
||||
$failed = @($checks.GetEnumerator() | Where-Object { -not $_.Value } | ForEach-Object { $_.Key })
|
||||
|
||||
$result = [ordered]@{
|
||||
schema_version = "c18z73.service_channel_pool_policy_remediation_guard_smoke.v1"
|
||||
run_id = "c18z73-" + (Get-Date -Format "yyyyMMdd-HHmmss")
|
||||
cluster_id = $ClusterID
|
||||
passed = ($failed.Count -eq 0)
|
||||
checks = $checks
|
||||
failed_checks = $failed
|
||||
summary = [ordered]@{
|
||||
policy = $policy
|
||||
lease = $lease
|
||||
access_channel = $channel
|
||||
}
|
||||
}
|
||||
|
||||
$target = Join-Path $repoRoot $ResultPath
|
||||
$result | ConvertTo-Json -Depth 60 | Set-Content -Path $target -Encoding UTF8
|
||||
|
||||
try {
|
||||
Start-Sleep -Seconds 6
|
||||
Invoke-Api -Method POST -Path "/clusters/$ClusterID/fabric/service-channels/leases/cleanup" -Body @{
|
||||
actor_user_id = $ActorUserID
|
||||
limit = 50
|
||||
} | Out-Null
|
||||
Invoke-Api -Method PUT -Path "/clusters/$ClusterID/fabric/service-channels/pool-policy" -Body @{
|
||||
actor_user_id = $ActorUserID
|
||||
entry_pool_node_ids = @()
|
||||
exit_pool_node_ids = @()
|
||||
preferred_entry_node_id = ""
|
||||
preferred_exit_node_id = ""
|
||||
selection_strategy = "fastest_healthy"
|
||||
route_rebuild = "automatic"
|
||||
entry_failover = "automatic"
|
||||
exit_failover = "automatic"
|
||||
backend_fallback_allowed = $true
|
||||
sticky_session = $true
|
||||
} | Out-Null
|
||||
} catch {
|
||||
Write-Warning "failed to restore default pool policy after smoke: $($_.Exception.Message)"
|
||||
}
|
||||
|
||||
if (-not $result.passed) {
|
||||
throw "C18Z73 pool-policy remediation guard smoke failed: $($failed -join ', ')"
|
||||
}
|
||||
|
||||
Write-Host "C18Z73 service-channel pool-policy remediation guard smoke passed. Result: $target"
|
||||
$result
|
||||
@@ -0,0 +1,94 @@
|
||||
param(
|
||||
[string]$ApiBaseUrl = "http://192.168.200.61:18121/api/v1",
|
||||
[string]$ClusterID = "cfc0743d-d960-49fb-9de8-96e063d5e4aa",
|
||||
[string]$ActorUserID = "f67d943f-5397-4b3a-a229-695fe67ad700",
|
||||
[string]$ResultPath = "artifacts\c18z74-service-channel-remediation-execution-smoke-result.json"
|
||||
)
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
$scriptDir = Split-Path -Parent $MyInvocation.MyCommand.Path
|
||||
$repoRoot = (Resolve-Path (Join-Path $scriptDir "..\..")).ProviderPath
|
||||
$c18z67 = Join-Path $scriptDir "c18z67-service-channel-concurrent-qos-live-smoke.ps1"
|
||||
|
||||
function Invoke-Api {
|
||||
param(
|
||||
[string]$Method,
|
||||
[string]$Path,
|
||||
[object]$Body = $null
|
||||
)
|
||||
$params = @{
|
||||
Method = $Method
|
||||
Uri = "$ApiBaseUrl$Path"
|
||||
TimeoutSec = 30
|
||||
}
|
||||
if ($null -ne $Body) {
|
||||
$params.ContentType = "application/json"
|
||||
$params.Body = ($Body | ConvertTo-Json -Depth 30)
|
||||
}
|
||||
return Invoke-RestMethod @params
|
||||
}
|
||||
|
||||
function Get-PropertyValue {
|
||||
param([object]$Item, [string]$Name, [object]$Default = $null)
|
||||
if ($null -eq $Item) { return $Default }
|
||||
$property = $Item.PSObject.Properties[$Name]
|
||||
if ($null -eq $property) { return $Default }
|
||||
return $property.Value
|
||||
}
|
||||
|
||||
& powershell -NoProfile -ExecutionPolicy Bypass -File $c18z67 | Out-Host
|
||||
|
||||
$c18z67ResultPath = Join-Path $repoRoot "artifacts\c18z67-service-channel-concurrent-qos-live-smoke-result.json"
|
||||
$c18z67Result = Get-Content -Path $c18z67ResultPath -Raw | ConvertFrom-Json
|
||||
$channelID = [string]$c18z67Result.channel_id
|
||||
$replacementRouteID = [string]$c18z67Result.expected_replacement_route_id
|
||||
|
||||
$accessTelemetry = $null
|
||||
$channel = $null
|
||||
for ($i = 0; $i -lt 12; $i++) {
|
||||
Start-Sleep -Seconds 5
|
||||
$accessTelemetry = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/fabric/service-channels/access-telemetry?actor_user_id=$ActorUserID&limit=50").fabric_service_channel_access_telemetry
|
||||
$channel = @($accessTelemetry.active_channels | Where-Object { $_.channel_id -eq $channelID }) | Select-Object -First 1
|
||||
if ($null -ne $channel -and [string](Get-PropertyValue -Item $channel -Name "remediation_execution_status" -Default "") -eq "applied") {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
$command = Get-PropertyValue -Item $channel -Name "remediation_command" -Default $null
|
||||
$checks = [ordered]@{
|
||||
c18z67_regression_passed = [bool]$c18z67Result.passed
|
||||
active_channel_found = ($null -ne $channel)
|
||||
remediation_prefers_alternate = ($null -ne $channel -and [string](Get-PropertyValue -Item $channel -Name "remediation_action" -Default "") -eq "prefer_alternate_route")
|
||||
remediation_execution_applied = ($null -ne $channel -and [string](Get-PropertyValue -Item $channel -Name "remediation_execution_status" -Default "") -eq "applied")
|
||||
remediation_execution_generation_visible = ($null -ne $channel -and ([string](Get-PropertyValue -Item $channel -Name "remediation_execution_generation" -Default "")).Length -gt 0)
|
||||
command_execution_applied = ($null -ne $command -and [string](Get-PropertyValue -Item $command -Name "execution_status" -Default "") -eq "applied")
|
||||
command_replacement_matches = ($null -ne $command -and [string](Get-PropertyValue -Item $command -Name "replacement_route_id" -Default "") -eq $replacementRouteID)
|
||||
}
|
||||
$failed = @($checks.GetEnumerator() | Where-Object { -not $_.Value } | ForEach-Object { $_.Key })
|
||||
|
||||
$result = [ordered]@{
|
||||
schema_version = "c18z74.service_channel_remediation_execution_smoke.v1"
|
||||
run_id = "c18z74-" + (Get-Date -Format "yyyyMMdd-HHmmss")
|
||||
cluster_id = $ClusterID
|
||||
channel_id = $channelID
|
||||
passed = ($failed.Count -eq 0)
|
||||
checks = $checks
|
||||
failed_checks = $failed
|
||||
summary = [ordered]@{
|
||||
c18z67_run_id = $c18z67Result.run_id
|
||||
access_status = Get-PropertyValue -Item $accessTelemetry -Name "status" -Default ""
|
||||
active_channel = $channel
|
||||
}
|
||||
}
|
||||
|
||||
$target = Join-Path $repoRoot $ResultPath
|
||||
$result | ConvertTo-Json -Depth 60 | Set-Content -Path $target -Encoding UTF8
|
||||
|
||||
if (-not $result.passed) {
|
||||
throw "C18Z74 remediation execution smoke failed: $($failed -join ', ')"
|
||||
}
|
||||
|
||||
Write-Host "C18Z74 service-channel remediation execution smoke passed. Result: $target"
|
||||
$result
|
||||
@@ -0,0 +1,234 @@
|
||||
param(
|
||||
[string]$ApiBaseUrl = "http://192.168.200.61:18121/api/v1",
|
||||
[string]$ClusterID = "cfc0743d-d960-49fb-9de8-96e063d5e4aa",
|
||||
[string]$ActorUserID = "f67d943f-5397-4b3a-a229-695fe67ad700",
|
||||
[string]$EntryNodeName = "test-3",
|
||||
[string]$ExitNodeName = "test-1",
|
||||
[string]$DockerSSH = "test-docker",
|
||||
[string]$ExpectedBackendImage = "rap-backend:fabric-service-channel-0.2.256-c18z82",
|
||||
[string]$ExpectedNodeAgentImage = "rap-node-agent:0.2.270-c18z95",
|
||||
[string]$ResultPath = "artifacts\c18z75-service-channel-rebuild-intent-smoke-result.json"
|
||||
)
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
$scriptDir = Split-Path -Parent $MyInvocation.MyCommand.Path
|
||||
$repoRoot = (Resolve-Path (Join-Path $scriptDir "..\..")).ProviderPath
|
||||
$runId = "c18z75-" + (Get-Date -Format "yyyyMMdd-HHmmss")
|
||||
|
||||
function Invoke-Api {
|
||||
param([string]$Method, [string]$Path, [object]$Body = $null)
|
||||
$params = @{ Method = $Method; Uri = "$ApiBaseUrl$Path"; TimeoutSec = 30 }
|
||||
if ($null -ne $Body) {
|
||||
$params.ContentType = "application/json"
|
||||
$params.Body = ($Body | ConvertTo-Json -Depth 80)
|
||||
}
|
||||
return Invoke-RestMethod @params
|
||||
}
|
||||
|
||||
function Get-PropertyValue {
|
||||
param([object]$Item, [string]$Name, [object]$Default = $null)
|
||||
if ($null -eq $Item) { return $Default }
|
||||
$property = $Item.PSObject.Properties[$Name]
|
||||
if ($null -eq $property) { return $Default }
|
||||
return $property.Value
|
||||
}
|
||||
|
||||
function Get-NodeByName {
|
||||
param([string]$Name)
|
||||
$nodes = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/nodes?actor_user_id=$ActorUserID").nodes
|
||||
$node = @($nodes | Where-Object { $_.name -eq $Name }) | Select-Object -First 1
|
||||
if ($null -eq $node) { throw "Node '$Name' was not found" }
|
||||
return $node
|
||||
}
|
||||
|
||||
function New-RouteIntent {
|
||||
param([string]$SourceNodeID, [string]$DestinationNodeID, [string[]]$Hops)
|
||||
$expiresAt = (Get-Date).ToUniversalTime().AddMinutes(5).ToString("o")
|
||||
return (Invoke-Api -Method POST -Path "/clusters/$ClusterID/mesh/route-intents" -Body @{
|
||||
actor_user_id = $ActorUserID
|
||||
source_selector = @{ node_id = $SourceNodeID }
|
||||
destination_selector = @{ node_id = $DestinationNodeID }
|
||||
service_class = "vpn_packets"
|
||||
priority = 2100000000
|
||||
policy = @{
|
||||
synthetic_enabled = $true
|
||||
route_version = "$runId-primary"
|
||||
policy_version = "$runId-primary"
|
||||
peer_directory_version = "$runId-primary"
|
||||
hops = @($Hops)
|
||||
allowed_channels = @("vpn_packet", "fabric_control")
|
||||
max_ttl = 8
|
||||
max_hops = 8
|
||||
expires_at = $expiresAt
|
||||
metadata = @{ smoke = "c18z75_service_channel_rebuild_intent"; run_id = $runId }
|
||||
}
|
||||
}).route_intent
|
||||
}
|
||||
|
||||
function Disable-ExistingRouteIntents {
|
||||
param([string]$SourceNodeID, [string]$DestinationNodeID)
|
||||
$items = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/mesh/route-intents?actor_user_id=$ActorUserID").route_intents
|
||||
foreach ($item in @($items)) {
|
||||
if ([string](Get-PropertyValue -Item $item -Name "status" -Default "") -ne "active") { continue }
|
||||
if ([string](Get-PropertyValue -Item $item -Name "service_class" -Default "") -ne "vpn_packets") { continue }
|
||||
$sourceSelector = Get-PropertyValue -Item $item -Name "source_selector" -Default $null
|
||||
$destinationSelector = Get-PropertyValue -Item $item -Name "destination_selector" -Default $null
|
||||
if ([string](Get-PropertyValue -Item $sourceSelector -Name "node_id" -Default "") -ne $SourceNodeID) { continue }
|
||||
if ([string](Get-PropertyValue -Item $destinationSelector -Name "node_id" -Default "") -ne $DestinationNodeID) { continue }
|
||||
[void](Invoke-Api -Method POST -Path "/clusters/$ClusterID/mesh/route-intents/$($item.id)/disable" -Body @{
|
||||
actor_user_id = $ActorUserID
|
||||
reason = "c18z75 isolate rebuild-intent smoke route pair"
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
function Send-DegradedHeartbeat {
|
||||
param([string]$EntryNodeID, [string]$PrimaryRouteID)
|
||||
$observedAt = (Get-Date).ToUniversalTime().ToString("o")
|
||||
return Invoke-Api -Method POST -Path "/clusters/$ClusterID/nodes/$EntryNodeID/heartbeats" -Body @{
|
||||
health_status = "healthy"
|
||||
reported_version = "0.2.248-c18z74"
|
||||
capabilities = @{
|
||||
fabric_service_channel_runtime = $true
|
||||
fabric_service_channel_route_quality_feedback = $true
|
||||
smoke_feedback_injection = "c18z75"
|
||||
}
|
||||
service_states = @{ smoke = "c18z75_primary_degraded_no_alternate" }
|
||||
metadata = @{
|
||||
fabric_service_channel_runtime_report = @{
|
||||
schema_version = "c18l.fabric_service_channel_runtime_report.v1"
|
||||
config_version = "$runId-primary"
|
||||
cluster_id = $ClusterID
|
||||
local_node_id = $EntryNodeID
|
||||
observed_at = $observedAt
|
||||
ingress = @{
|
||||
flow_scheduler = @{
|
||||
schema_version = "rap.fabric_flow_scheduler.v1"
|
||||
service_neutral = $true
|
||||
service_mode = "application_protocol_agnostic"
|
||||
channel_stats = @{
|
||||
"c18z75-primary-degraded" = @{
|
||||
last_route_id = $PrimaryRouteID
|
||||
last_failed_route_id = $PrimaryRouteID
|
||||
route_generation = "$runId-primary"
|
||||
last_error = "c18z75 primary route degraded; rebuild requested"
|
||||
last_send_duration_ms = 1200
|
||||
consecutive_failures = 3
|
||||
stall_count = 2
|
||||
route_rebuild_recommended = $true
|
||||
degraded_fallback_recommended = $false
|
||||
quality_window_sample_count = 8
|
||||
quality_window_success_count = 2
|
||||
quality_window_failure_count = 3
|
||||
quality_window_slow_count = 2
|
||||
quality_window_drop_count = 0
|
||||
quality_window_avg_latency_ms = 1200
|
||||
quality_window_last_updated_at = $observedAt
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
smoke = @{ name = "c18z75_service_channel_rebuild_intent"; run_id = $runId }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
$entryNode = Get-NodeByName -Name $EntryNodeName
|
||||
$exitNode = Get-NodeByName -Name $ExitNodeName
|
||||
[void](Disable-ExistingRouteIntents -SourceNodeID $entryNode.id -DestinationNodeID $exitNode.id)
|
||||
$route = New-RouteIntent -SourceNodeID $entryNode.id -DestinationNodeID $exitNode.id -Hops @($entryNode.id, $exitNode.id)
|
||||
$lease = (Invoke-Api -Method POST -Path "/clusters/$ClusterID/fabric/service-channels/leases" -Body @{
|
||||
actor_user_id = $ActorUserID
|
||||
organization_id = "smoke-org"
|
||||
user_id = "smoke-user"
|
||||
resource_id = "c18z75-vpn-smoke"
|
||||
service_class = "vpn_packets"
|
||||
entry_node_ids = @([string]$entryNode.id)
|
||||
exit_node_ids = @([string]$exitNode.id)
|
||||
preferred_entry_node_id = [string]$entryNode.id
|
||||
preferred_exit_node_id = [string]$exitNode.id
|
||||
allowed_channels = @("vpn_packet", "fabric_control")
|
||||
ttl_seconds = 90
|
||||
metadata = @{ smoke = "c18z75_service_channel_rebuild_intent"; run_id = $runId }
|
||||
}).fabric_service_channel_lease
|
||||
$primaryRouteID = [string](Get-PropertyValue -Item (Get-PropertyValue -Item $lease -Name "primary_route" -Default $null) -Name "route_id" -Default $route.id)
|
||||
|
||||
[void](Send-DegradedHeartbeat -EntryNodeID $entryNode.id -PrimaryRouteID $primaryRouteID)
|
||||
|
||||
$matchingChannel = $null
|
||||
for ($i = 0; $i -lt 10; $i++) {
|
||||
Start-Sleep -Seconds 3
|
||||
$accessTelemetry = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/fabric/service-channels/access-telemetry?actor_user_id=$ActorUserID&limit=50").fabric_service_channel_access_telemetry
|
||||
$matchingChannel = @($accessTelemetry.active_channels | Where-Object { $_.channel_id -eq $lease.channel_id }) | Select-Object -First 1
|
||||
if ($null -ne $matchingChannel -and [string](Get-PropertyValue -Item $matchingChannel -Name "remediation_action" -Default "") -eq "rebuild_route") {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
$command = Get-PropertyValue -Item $matchingChannel -Name "remediation_command" -Default $null
|
||||
$commandID = [string](Get-PropertyValue -Item $command -Name "command_id" -Default "")
|
||||
[void](Invoke-Api -Method GET -Path "/clusters/$ClusterID/nodes/$($entryNode.id)/mesh/synthetic-config")
|
||||
|
||||
$attempts = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/fabric/service-channels/rebuild-attempts?actor_user_id=$ActorUserID&reporter_node_id=$($entryNode.id)&rebuild_request_id=$commandID&limit=10").rebuild_attempts
|
||||
$attempt = @($attempts | Where-Object { $_.rebuild_request_id -eq $commandID }) | Select-Object -First 1
|
||||
$postAccess = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/fabric/service-channels/access-telemetry?actor_user_id=$ActorUserID&limit=50").fabric_service_channel_access_telemetry
|
||||
$postChannel = @($postAccess.active_channels | Where-Object { $_.channel_id -eq $lease.channel_id }) | Select-Object -First 1
|
||||
|
||||
$backendLine = (& ssh $DockerSSH "docker ps --format '{{.Names}} {{.Image}} {{.Status}}' | grep '^rap_test_backend '") | Out-String
|
||||
$nodeLines = (& ssh $DockerSSH "docker ps --format '{{.Names}} {{.Image}} {{.Status}}' | grep '^rap_test_node_test_'") | Out-String
|
||||
$checks = [ordered]@{
|
||||
backend_expected_image_deployed = $backendLine.Contains($ExpectedBackendImage)
|
||||
node_agent_expected_image_deployed = $nodeLines.Contains($ExpectedNodeAgentImage)
|
||||
lease_ready = ([string]$lease.status -eq "ready")
|
||||
remediation_rebuild_route_visible = ($null -ne $matchingChannel -and [string](Get-PropertyValue -Item $matchingChannel -Name "remediation_action" -Default "") -eq "rebuild_route")
|
||||
remediation_command_visible = ($null -ne $command -and $commandID.Length -gt 0)
|
||||
durable_rebuild_intent_recorded = ($null -ne $attempt)
|
||||
durable_rebuild_intent_recorded_or_resolved = ($null -ne $attempt -and @("requested", "no_alternate", "applied", "deferred_by_policy", "expired") -contains [string](Get-PropertyValue -Item $attempt -Name "rebuild_status" -Default ""))
|
||||
durable_rebuild_intent_source_matches = ($null -ne $attempt -and [string](Get-PropertyValue -Item $attempt -Name "decision_source" -Default "") -eq "service_channel_remediation_command")
|
||||
access_telemetry_reports_recorded_execution = ($null -ne $postChannel -and @("rebuild_request_recorded", "rebuild_request_no_alternate", "rebuild_request_applied", "rebuild_request_deferred_by_policy", "rebuild_request_expired") -contains [string](Get-PropertyValue -Item $postChannel -Name "remediation_execution_status" -Default ""))
|
||||
}
|
||||
$failed = @($checks.GetEnumerator() | Where-Object { -not $_.Value } | ForEach-Object { $_.Key })
|
||||
|
||||
$result = [ordered]@{
|
||||
schema_version = "c18z75.service_channel_rebuild_intent_smoke.v1"
|
||||
run_id = $runId
|
||||
cluster_id = $ClusterID
|
||||
channel_id = [string]$lease.channel_id
|
||||
primary_route_id = $primaryRouteID
|
||||
rebuild_request_id = $commandID
|
||||
passed = ($failed.Count -eq 0)
|
||||
checks = $checks
|
||||
failed_checks = $failed
|
||||
summary = [ordered]@{
|
||||
backend_container = $backendLine.Trim()
|
||||
node_containers = $nodeLines.Trim()
|
||||
remediation_command = $command
|
||||
rebuild_attempt = $attempt
|
||||
post_channel = $postChannel
|
||||
}
|
||||
}
|
||||
|
||||
$target = Join-Path $repoRoot $ResultPath
|
||||
$result | ConvertTo-Json -Depth 80 | Set-Content -Path $target -Encoding UTF8
|
||||
|
||||
try {
|
||||
Start-Sleep -Seconds 3
|
||||
Invoke-Api -Method POST -Path "/clusters/$ClusterID/fabric/service-channels/leases/cleanup" -Body @{
|
||||
actor_user_id = $ActorUserID
|
||||
limit = 50
|
||||
} | Out-Null
|
||||
} catch {
|
||||
Write-Warning "cleanup failed after c18z75 smoke: $($_.Exception.Message)"
|
||||
}
|
||||
|
||||
if (-not $result.passed) {
|
||||
throw "C18Z75 rebuild intent smoke failed: $($failed -join ', ')"
|
||||
}
|
||||
|
||||
Write-Host "C18Z75 service-channel rebuild intent smoke passed. Result: $target"
|
||||
$result
|
||||
|
||||
|
||||
@@ -0,0 +1,236 @@
|
||||
param(
|
||||
[string]$ApiBaseUrl = "http://192.168.200.61:18121/api/v1",
|
||||
[string]$ClusterID = "cfc0743d-d960-49fb-9de8-96e063d5e4aa",
|
||||
[string]$ActorUserID = "f67d943f-5397-4b3a-a229-695fe67ad700",
|
||||
[string]$EntryNodeName = "test-3",
|
||||
[string]$ExitNodeName = "test-1",
|
||||
[string]$DockerSSH = "test-docker",
|
||||
[string]$ExpectedBackendImage = "rap-backend:fabric-service-channel-0.2.256-c18z82",
|
||||
[string]$ExpectedNodeAgentImage = "rap-node-agent:0.2.270-c18z95",
|
||||
[string]$ResultPath = "artifacts\c18z76-service-channel-rebuild-node-pending-smoke-result.json"
|
||||
)
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
$scriptDir = Split-Path -Parent $MyInvocation.MyCommand.Path
|
||||
$repoRoot = (Resolve-Path (Join-Path $scriptDir "..\..")).ProviderPath
|
||||
$runId = "c18z76-" + (Get-Date -Format "yyyyMMdd-HHmmss")
|
||||
|
||||
function Invoke-Api {
|
||||
param([string]$Method, [string]$Path, [object]$Body = $null)
|
||||
$params = @{ Method = $Method; Uri = "$ApiBaseUrl$Path"; TimeoutSec = 30 }
|
||||
if ($null -ne $Body) {
|
||||
$params.ContentType = "application/json"
|
||||
$params.Body = ($Body | ConvertTo-Json -Depth 80)
|
||||
}
|
||||
return Invoke-RestMethod @params
|
||||
}
|
||||
|
||||
function Get-PropertyValue {
|
||||
param([object]$Item, [string]$Name, [object]$Default = $null)
|
||||
if ($null -eq $Item) { return $Default }
|
||||
$property = $Item.PSObject.Properties[$Name]
|
||||
if ($null -eq $property) { return $Default }
|
||||
return $property.Value
|
||||
}
|
||||
|
||||
function Get-NodeByName {
|
||||
param([string]$Name)
|
||||
$nodes = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/nodes?actor_user_id=$ActorUserID").nodes
|
||||
$node = @($nodes | Where-Object { $_.name -eq $Name }) | Select-Object -First 1
|
||||
if ($null -eq $node) { throw "Node '$Name' was not found" }
|
||||
return $node
|
||||
}
|
||||
|
||||
function New-RouteIntent {
|
||||
param([string]$SourceNodeID, [string]$DestinationNodeID, [string[]]$Hops)
|
||||
$expiresAt = (Get-Date).ToUniversalTime().AddMinutes(5).ToString("o")
|
||||
return (Invoke-Api -Method POST -Path "/clusters/$ClusterID/mesh/route-intents" -Body @{
|
||||
actor_user_id = $ActorUserID
|
||||
source_selector = @{ node_id = $SourceNodeID }
|
||||
destination_selector = @{ node_id = $DestinationNodeID }
|
||||
service_class = "vpn_packets"
|
||||
priority = 2100000000
|
||||
policy = @{
|
||||
synthetic_enabled = $true
|
||||
route_version = "$runId-primary"
|
||||
policy_version = "$runId-primary"
|
||||
peer_directory_version = "$runId-primary"
|
||||
hops = @($Hops)
|
||||
allowed_channels = @("vpn_packet", "fabric_control")
|
||||
max_ttl = 8
|
||||
max_hops = 8
|
||||
expires_at = $expiresAt
|
||||
metadata = @{ smoke = "c18z76_service_channel_rebuild_node_pending"; run_id = $runId }
|
||||
}
|
||||
}).route_intent
|
||||
}
|
||||
|
||||
function Disable-ExistingRouteIntents {
|
||||
param([string]$SourceNodeID, [string]$DestinationNodeID)
|
||||
$items = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/mesh/route-intents?actor_user_id=$ActorUserID").route_intents
|
||||
foreach ($item in @($items)) {
|
||||
if ([string](Get-PropertyValue -Item $item -Name "status" -Default "") -ne "active") { continue }
|
||||
if ([string](Get-PropertyValue -Item $item -Name "service_class" -Default "") -ne "vpn_packets") { continue }
|
||||
$sourceSelector = Get-PropertyValue -Item $item -Name "source_selector" -Default $null
|
||||
$destinationSelector = Get-PropertyValue -Item $item -Name "destination_selector" -Default $null
|
||||
if ([string](Get-PropertyValue -Item $sourceSelector -Name "node_id" -Default "") -ne $SourceNodeID) { continue }
|
||||
if ([string](Get-PropertyValue -Item $destinationSelector -Name "node_id" -Default "") -ne $DestinationNodeID) { continue }
|
||||
[void](Invoke-Api -Method POST -Path "/clusters/$ClusterID/mesh/route-intents/$($item.id)/disable" -Body @{
|
||||
actor_user_id = $ActorUserID
|
||||
reason = "c18z76 isolate rebuild-node-pending smoke route pair"
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
function Send-DegradedHeartbeat {
|
||||
param([string]$EntryNodeID, [string]$PrimaryRouteID)
|
||||
$observedAt = (Get-Date).ToUniversalTime().ToString("o")
|
||||
return Invoke-Api -Method POST -Path "/clusters/$ClusterID/nodes/$EntryNodeID/heartbeats" -Body @{
|
||||
health_status = "healthy"
|
||||
reported_version = "0.2.250-c18z76"
|
||||
capabilities = @{
|
||||
fabric_service_channel_runtime = $true
|
||||
fabric_service_channel_route_quality_feedback = $true
|
||||
smoke_feedback_injection = "c18z76"
|
||||
}
|
||||
service_states = @{ smoke = "c18z76_primary_degraded_no_alternate" }
|
||||
metadata = @{
|
||||
fabric_service_channel_runtime_report = @{
|
||||
schema_version = "c18l.fabric_service_channel_runtime_report.v1"
|
||||
config_version = "$runId-primary"
|
||||
cluster_id = $ClusterID
|
||||
local_node_id = $EntryNodeID
|
||||
observed_at = $observedAt
|
||||
ingress = @{
|
||||
flow_scheduler = @{
|
||||
schema_version = "rap.fabric_flow_scheduler.v1"
|
||||
service_neutral = $true
|
||||
service_mode = "application_protocol_agnostic"
|
||||
channel_stats = @{
|
||||
"c18z76-primary-degraded" = @{
|
||||
last_route_id = $PrimaryRouteID
|
||||
last_failed_route_id = $PrimaryRouteID
|
||||
route_generation = "$runId-primary"
|
||||
last_error = "c18z76 primary route degraded; rebuild requested"
|
||||
last_send_duration_ms = 1200
|
||||
consecutive_failures = 3
|
||||
stall_count = 2
|
||||
route_rebuild_recommended = $true
|
||||
degraded_fallback_recommended = $false
|
||||
quality_window_sample_count = 8
|
||||
quality_window_success_count = 2
|
||||
quality_window_failure_count = 3
|
||||
quality_window_slow_count = 2
|
||||
quality_window_drop_count = 0
|
||||
quality_window_avg_latency_ms = 1200
|
||||
quality_window_last_updated_at = $observedAt
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
smoke = @{ name = "c18z76_service_channel_rebuild_node_pending"; run_id = $runId }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
$entryNode = Get-NodeByName -Name $EntryNodeName
|
||||
$exitNode = Get-NodeByName -Name $ExitNodeName
|
||||
[void](Disable-ExistingRouteIntents -SourceNodeID $entryNode.id -DestinationNodeID $exitNode.id)
|
||||
$route = New-RouteIntent -SourceNodeID $entryNode.id -DestinationNodeID $exitNode.id -Hops @($entryNode.id, $exitNode.id)
|
||||
$lease = (Invoke-Api -Method POST -Path "/clusters/$ClusterID/fabric/service-channels/leases" -Body @{
|
||||
actor_user_id = $ActorUserID
|
||||
organization_id = "smoke-org"
|
||||
user_id = "smoke-user"
|
||||
resource_id = "c18z76-vpn-smoke"
|
||||
service_class = "vpn_packets"
|
||||
entry_node_ids = @([string]$entryNode.id)
|
||||
exit_node_ids = @([string]$exitNode.id)
|
||||
preferred_entry_node_id = [string]$entryNode.id
|
||||
preferred_exit_node_id = [string]$exitNode.id
|
||||
allowed_channels = @("vpn_packet", "fabric_control")
|
||||
ttl_seconds = 90
|
||||
metadata = @{ smoke = "c18z76_service_channel_rebuild_node_pending"; run_id = $runId }
|
||||
}).fabric_service_channel_lease
|
||||
$primaryRouteID = [string](Get-PropertyValue -Item (Get-PropertyValue -Item $lease -Name "primary_route" -Default $null) -Name "route_id" -Default $route.id)
|
||||
|
||||
[void](Send-DegradedHeartbeat -EntryNodeID $entryNode.id -PrimaryRouteID $primaryRouteID)
|
||||
|
||||
$matchingChannel = $null
|
||||
for ($i = 0; $i -lt 10; $i++) {
|
||||
Start-Sleep -Seconds 3
|
||||
$accessTelemetry = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/fabric/service-channels/access-telemetry?actor_user_id=$ActorUserID&limit=50").fabric_service_channel_access_telemetry
|
||||
$matchingChannel = @($accessTelemetry.active_channels | Where-Object { $_.channel_id -eq $lease.channel_id }) | Select-Object -First 1
|
||||
if ($null -ne $matchingChannel -and [string](Get-PropertyValue -Item $matchingChannel -Name "remediation_action" -Default "") -eq "rebuild_route") {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
$command = Get-PropertyValue -Item $matchingChannel -Name "remediation_command" -Default $null
|
||||
$commandID = [string](Get-PropertyValue -Item $command -Name "command_id" -Default "")
|
||||
[void](Invoke-Api -Method GET -Path "/clusters/$ClusterID/nodes/$($entryNode.id)/mesh/synthetic-config")
|
||||
|
||||
$attempts = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/fabric/service-channels/rebuild-attempts?actor_user_id=$ActorUserID&reporter_node_id=$($entryNode.id)&rebuild_request_id=$commandID&limit=10").rebuild_attempts
|
||||
$attempt = @($attempts | Where-Object { $_.rebuild_request_id -eq $commandID }) | Select-Object -First 1
|
||||
|
||||
Start-Sleep -Seconds 30
|
||||
$postAccess = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/fabric/service-channels/access-telemetry?actor_user_id=$ActorUserID&limit=50").fabric_service_channel_access_telemetry
|
||||
$postChannel = @($postAccess.active_channels | Where-Object { $_.channel_id -eq $lease.channel_id }) | Select-Object -First 1
|
||||
|
||||
$backendLine = (& ssh $DockerSSH "docker ps --format '{{.Names}} {{.Image}} {{.Status}}' | grep '^rap_test_backend '") | Out-String
|
||||
$nodeLines = (& ssh $DockerSSH "docker ps --format '{{.Names}} {{.Image}} {{.Status}}' | grep '^rap_test_node_test_'") | Out-String
|
||||
$checks = [ordered]@{
|
||||
backend_expected_image_deployed = $backendLine.Contains($ExpectedBackendImage)
|
||||
node_agent_expected_image_deployed = $nodeLines.Contains($ExpectedNodeAgentImage)
|
||||
lease_ready = ([string]$lease.status -eq "ready")
|
||||
remediation_rebuild_route_visible = ($null -ne $matchingChannel -and [string](Get-PropertyValue -Item $matchingChannel -Name "remediation_action" -Default "") -eq "rebuild_route")
|
||||
remediation_command_visible = ($null -ne $command -and $commandID.Length -gt 0)
|
||||
durable_rebuild_intent_recorded = ($null -ne $attempt)
|
||||
durable_rebuild_intent_requested = ($null -ne $attempt -and [string](Get-PropertyValue -Item $attempt -Name "rebuild_status" -Default "") -eq "requested")
|
||||
durable_rebuild_intent_source_matches = ($null -ne $attempt -and [string](Get-PropertyValue -Item $attempt -Name "decision_source" -Default "") -eq "service_channel_remediation_command")
|
||||
access_telemetry_reports_node_pending_or_resolved_execution = ($null -ne $postChannel -and @("rebuild_request_recorded_node_pending", "rebuild_request_no_alternate", "rebuild_request_applied", "rebuild_request_deferred_by_policy", "rebuild_request_expired") -contains [string](Get-PropertyValue -Item $postChannel -Name "remediation_execution_status" -Default ""))
|
||||
}
|
||||
$failed = @($checks.GetEnumerator() | Where-Object { -not $_.Value } | ForEach-Object { $_.Key })
|
||||
|
||||
$result = [ordered]@{
|
||||
schema_version = "c18z76.service_channel_rebuild_node_pending_smoke.v1"
|
||||
run_id = $runId
|
||||
cluster_id = $ClusterID
|
||||
channel_id = [string]$lease.channel_id
|
||||
primary_route_id = $primaryRouteID
|
||||
rebuild_request_id = $commandID
|
||||
passed = ($failed.Count -eq 0)
|
||||
checks = $checks
|
||||
failed_checks = $failed
|
||||
summary = [ordered]@{
|
||||
backend_container = $backendLine.Trim()
|
||||
node_containers = $nodeLines.Trim()
|
||||
remediation_command = $command
|
||||
rebuild_attempt = $attempt
|
||||
post_channel = $postChannel
|
||||
}
|
||||
}
|
||||
|
||||
$target = Join-Path $repoRoot $ResultPath
|
||||
$result | ConvertTo-Json -Depth 80 | Set-Content -Path $target -Encoding UTF8
|
||||
|
||||
try {
|
||||
Start-Sleep -Seconds 3
|
||||
Invoke-Api -Method POST -Path "/clusters/$ClusterID/fabric/service-channels/leases/cleanup" -Body @{
|
||||
actor_user_id = $ActorUserID
|
||||
limit = 50
|
||||
} | Out-Null
|
||||
} catch {
|
||||
Write-Warning "cleanup failed after c18z76 smoke: $($_.Exception.Message)"
|
||||
}
|
||||
|
||||
if (-not $result.passed) {
|
||||
throw "C18Z76 rebuild node-pending smoke failed: $($failed -join ', ')"
|
||||
}
|
||||
|
||||
Write-Host "C18Z76 service-channel rebuild node-pending smoke passed. Result: $target"
|
||||
$result
|
||||
|
||||
|
||||
@@ -0,0 +1,237 @@
|
||||
param(
|
||||
[string]$ApiBaseUrl = "http://192.168.200.61:18121/api/v1",
|
||||
[string]$ClusterID = "cfc0743d-d960-49fb-9de8-96e063d5e4aa",
|
||||
[string]$ActorUserID = "f67d943f-5397-4b3a-a229-695fe67ad700",
|
||||
[string]$EntryNodeName = "test-3",
|
||||
[string]$ExitNodeName = "test-1",
|
||||
[string]$DockerSSH = "test-docker",
|
||||
[string]$ExpectedBackendImage = "rap-backend:fabric-service-channel-0.2.256-c18z82",
|
||||
[string]$ExpectedNodeAgentImage = "rap-node-agent:0.2.270-c18z95",
|
||||
[string]$ResultPath = "artifacts\c18z77-service-channel-rebuild-planner-resolution-smoke-result.json"
|
||||
)
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
$scriptDir = Split-Path -Parent $MyInvocation.MyCommand.Path
|
||||
$repoRoot = (Resolve-Path (Join-Path $scriptDir "..\..")).ProviderPath
|
||||
$runId = "c18z77-" + (Get-Date -Format "yyyyMMdd-HHmmss")
|
||||
|
||||
function Invoke-Api {
|
||||
param([string]$Method, [string]$Path, [object]$Body = $null)
|
||||
$params = @{ Method = $Method; Uri = "$ApiBaseUrl$Path"; TimeoutSec = 30 }
|
||||
if ($null -ne $Body) {
|
||||
$params.ContentType = "application/json"
|
||||
$params.Body = ($Body | ConvertTo-Json -Depth 80)
|
||||
}
|
||||
return Invoke-RestMethod @params
|
||||
}
|
||||
|
||||
function Get-PropertyValue {
|
||||
param([object]$Item, [string]$Name, [object]$Default = $null)
|
||||
if ($null -eq $Item) { return $Default }
|
||||
$property = $Item.PSObject.Properties[$Name]
|
||||
if ($null -eq $property) { return $Default }
|
||||
return $property.Value
|
||||
}
|
||||
|
||||
function Get-NodeByName {
|
||||
param([string]$Name)
|
||||
$nodes = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/nodes?actor_user_id=$ActorUserID").nodes
|
||||
$node = @($nodes | Where-Object { $_.name -eq $Name }) | Select-Object -First 1
|
||||
if ($null -eq $node) { throw "Node '$Name' was not found" }
|
||||
return $node
|
||||
}
|
||||
|
||||
function New-RouteIntent {
|
||||
param([string]$SourceNodeID, [string]$DestinationNodeID, [string[]]$Hops)
|
||||
$expiresAt = (Get-Date).ToUniversalTime().AddMinutes(5).ToString("o")
|
||||
return (Invoke-Api -Method POST -Path "/clusters/$ClusterID/mesh/route-intents" -Body @{
|
||||
actor_user_id = $ActorUserID
|
||||
source_selector = @{ node_id = $SourceNodeID }
|
||||
destination_selector = @{ node_id = $DestinationNodeID }
|
||||
service_class = "vpn_packets"
|
||||
priority = 2100000000
|
||||
policy = @{
|
||||
synthetic_enabled = $true
|
||||
route_version = "$runId-primary"
|
||||
policy_version = "$runId-primary"
|
||||
peer_directory_version = "$runId-primary"
|
||||
hops = @($Hops)
|
||||
allowed_channels = @("vpn_packet", "fabric_control")
|
||||
max_ttl = 8
|
||||
max_hops = 8
|
||||
expires_at = $expiresAt
|
||||
metadata = @{ smoke = "c18z77_service_channel_rebuild_planner_resolution"; run_id = $runId }
|
||||
}
|
||||
}).route_intent
|
||||
}
|
||||
|
||||
function Disable-ExistingRouteIntents {
|
||||
param([string]$SourceNodeID, [string]$DestinationNodeID)
|
||||
$items = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/mesh/route-intents?actor_user_id=$ActorUserID").route_intents
|
||||
foreach ($item in @($items)) {
|
||||
if ([string](Get-PropertyValue -Item $item -Name "status" -Default "") -ne "active") { continue }
|
||||
if ([string](Get-PropertyValue -Item $item -Name "service_class" -Default "") -ne "vpn_packets") { continue }
|
||||
$sourceSelector = Get-PropertyValue -Item $item -Name "source_selector" -Default $null
|
||||
$destinationSelector = Get-PropertyValue -Item $item -Name "destination_selector" -Default $null
|
||||
if ([string](Get-PropertyValue -Item $sourceSelector -Name "node_id" -Default "") -ne $SourceNodeID) { continue }
|
||||
if ([string](Get-PropertyValue -Item $destinationSelector -Name "node_id" -Default "") -ne $DestinationNodeID) { continue }
|
||||
[void](Invoke-Api -Method POST -Path "/clusters/$ClusterID/mesh/route-intents/$($item.id)/disable" -Body @{
|
||||
actor_user_id = $ActorUserID
|
||||
reason = "c18z77 isolate rebuild-planner-resolution smoke route pair"
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
function Send-DegradedHeartbeat {
|
||||
param([string]$EntryNodeID, [string]$PrimaryRouteID)
|
||||
$observedAt = (Get-Date).ToUniversalTime().ToString("o")
|
||||
return Invoke-Api -Method POST -Path "/clusters/$ClusterID/nodes/$EntryNodeID/heartbeats" -Body @{
|
||||
health_status = "healthy"
|
||||
reported_version = "0.2.251-c18z77"
|
||||
capabilities = @{
|
||||
fabric_service_channel_runtime = $true
|
||||
fabric_service_channel_route_quality_feedback = $true
|
||||
smoke_feedback_injection = "c18z77"
|
||||
}
|
||||
service_states = @{ smoke = "c18z77_primary_degraded_no_alternate" }
|
||||
metadata = @{
|
||||
fabric_service_channel_runtime_report = @{
|
||||
schema_version = "c18l.fabric_service_channel_runtime_report.v1"
|
||||
config_version = "$runId-primary"
|
||||
cluster_id = $ClusterID
|
||||
local_node_id = $EntryNodeID
|
||||
observed_at = $observedAt
|
||||
ingress = @{
|
||||
flow_scheduler = @{
|
||||
schema_version = "rap.fabric_flow_scheduler.v1"
|
||||
service_neutral = $true
|
||||
service_mode = "application_protocol_agnostic"
|
||||
channel_stats = @{
|
||||
"c18z77-primary-degraded" = @{
|
||||
last_route_id = $PrimaryRouteID
|
||||
last_failed_route_id = $PrimaryRouteID
|
||||
route_generation = "$runId-primary"
|
||||
last_error = "c18z77 primary route degraded; rebuild requested"
|
||||
last_send_duration_ms = 1200
|
||||
consecutive_failures = 3
|
||||
stall_count = 2
|
||||
route_rebuild_recommended = $true
|
||||
degraded_fallback_recommended = $false
|
||||
quality_window_sample_count = 8
|
||||
quality_window_success_count = 2
|
||||
quality_window_failure_count = 3
|
||||
quality_window_slow_count = 2
|
||||
quality_window_drop_count = 0
|
||||
quality_window_avg_latency_ms = 1200
|
||||
quality_window_last_updated_at = $observedAt
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
smoke = @{ name = "c18z77_service_channel_rebuild_planner_resolution"; run_id = $runId }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
$entryNode = Get-NodeByName -Name $EntryNodeName
|
||||
$exitNode = Get-NodeByName -Name $ExitNodeName
|
||||
[void](Disable-ExistingRouteIntents -SourceNodeID $entryNode.id -DestinationNodeID $exitNode.id)
|
||||
$route = New-RouteIntent -SourceNodeID $entryNode.id -DestinationNodeID $exitNode.id -Hops @($entryNode.id, $exitNode.id)
|
||||
$lease = (Invoke-Api -Method POST -Path "/clusters/$ClusterID/fabric/service-channels/leases" -Body @{
|
||||
actor_user_id = $ActorUserID
|
||||
organization_id = "smoke-org"
|
||||
user_id = "smoke-user"
|
||||
resource_id = "c18z77-vpn-smoke"
|
||||
service_class = "vpn_packets"
|
||||
entry_node_ids = @([string]$entryNode.id)
|
||||
exit_node_ids = @([string]$exitNode.id)
|
||||
preferred_entry_node_id = [string]$entryNode.id
|
||||
preferred_exit_node_id = [string]$exitNode.id
|
||||
allowed_channels = @("vpn_packet", "fabric_control")
|
||||
ttl_seconds = 90
|
||||
metadata = @{ smoke = "c18z77_service_channel_rebuild_planner_resolution"; run_id = $runId }
|
||||
}).fabric_service_channel_lease
|
||||
$primaryRouteID = [string](Get-PropertyValue -Item (Get-PropertyValue -Item $lease -Name "primary_route" -Default $null) -Name "route_id" -Default $route.id)
|
||||
|
||||
[void](Send-DegradedHeartbeat -EntryNodeID $entryNode.id -PrimaryRouteID $primaryRouteID)
|
||||
|
||||
$matchingChannel = $null
|
||||
for ($i = 0; $i -lt 10; $i++) {
|
||||
Start-Sleep -Seconds 3
|
||||
$accessTelemetry = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/fabric/service-channels/access-telemetry?actor_user_id=$ActorUserID&limit=50").fabric_service_channel_access_telemetry
|
||||
$matchingChannel = @($accessTelemetry.active_channels | Where-Object { $_.channel_id -eq $lease.channel_id }) | Select-Object -First 1
|
||||
if ($null -ne $matchingChannel -and [string](Get-PropertyValue -Item $matchingChannel -Name "remediation_action" -Default "") -eq "rebuild_route") {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
$command = Get-PropertyValue -Item $matchingChannel -Name "remediation_command" -Default $null
|
||||
$commandID = [string](Get-PropertyValue -Item $command -Name "command_id" -Default "")
|
||||
[void](Invoke-Api -Method GET -Path "/clusters/$ClusterID/nodes/$($entryNode.id)/mesh/synthetic-config")
|
||||
|
||||
$attempts = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/fabric/service-channels/rebuild-attempts?actor_user_id=$ActorUserID&reporter_node_id=$($entryNode.id)&rebuild_request_id=$commandID&limit=10").rebuild_attempts
|
||||
$attempt = @($attempts | Where-Object { $_.rebuild_request_id -eq $commandID }) | Select-Object -First 1
|
||||
|
||||
Start-Sleep -Seconds 30
|
||||
$postAccess = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/fabric/service-channels/access-telemetry?actor_user_id=$ActorUserID&limit=50").fabric_service_channel_access_telemetry
|
||||
$postChannel = @($postAccess.active_channels | Where-Object { $_.channel_id -eq $lease.channel_id }) | Select-Object -First 1
|
||||
|
||||
$backendLine = (& ssh $DockerSSH "docker ps --format '{{.Names}} {{.Image}} {{.Status}}' | grep '^rap_test_backend '") | Out-String
|
||||
$nodeLines = (& ssh $DockerSSH "docker ps --format '{{.Names}} {{.Image}} {{.Status}}' | grep '^rap_test_node_test_'") | Out-String
|
||||
$checks = [ordered]@{
|
||||
backend_expected_image_deployed = $backendLine.Contains($ExpectedBackendImage)
|
||||
node_agent_expected_image_deployed = $nodeLines.Contains($ExpectedNodeAgentImage)
|
||||
lease_ready = ([string]$lease.status -eq "ready")
|
||||
remediation_rebuild_route_visible = ($null -ne $matchingChannel -and [string](Get-PropertyValue -Item $matchingChannel -Name "remediation_action" -Default "") -eq "rebuild_route")
|
||||
remediation_command_visible = ($null -ne $command -and $commandID.Length -gt 0)
|
||||
durable_rebuild_intent_recorded = ($null -ne $attempt)
|
||||
durable_rebuild_intent_resolved_no_alternate = ($null -ne $attempt -and [string](Get-PropertyValue -Item $attempt -Name "rebuild_status" -Default "") -eq "no_alternate")
|
||||
durable_rebuild_intent_outcome_no_alternate = ($null -ne $attempt -and [string](Get-PropertyValue -Item $attempt -Name "outcome" -Default "") -eq "no_alternate")
|
||||
durable_rebuild_intent_source_matches = ($null -ne $attempt -and [string](Get-PropertyValue -Item $attempt -Name "decision_source" -Default "") -eq "service_channel_remediation_command")
|
||||
access_telemetry_reports_planner_no_alternate = ($null -ne $postChannel -and [string](Get-PropertyValue -Item $postChannel -Name "remediation_execution_status" -Default "") -eq "rebuild_request_no_alternate")
|
||||
}
|
||||
$failed = @($checks.GetEnumerator() | Where-Object { -not $_.Value } | ForEach-Object { $_.Key })
|
||||
|
||||
$result = [ordered]@{
|
||||
schema_version = "c18z77.service_channel_rebuild_planner_resolution_smoke.v1"
|
||||
run_id = $runId
|
||||
cluster_id = $ClusterID
|
||||
channel_id = [string]$lease.channel_id
|
||||
primary_route_id = $primaryRouteID
|
||||
rebuild_request_id = $commandID
|
||||
passed = ($failed.Count -eq 0)
|
||||
checks = $checks
|
||||
failed_checks = $failed
|
||||
summary = [ordered]@{
|
||||
backend_container = $backendLine.Trim()
|
||||
node_containers = $nodeLines.Trim()
|
||||
remediation_command = $command
|
||||
rebuild_attempt = $attempt
|
||||
post_channel = $postChannel
|
||||
}
|
||||
}
|
||||
|
||||
$target = Join-Path $repoRoot $ResultPath
|
||||
$result | ConvertTo-Json -Depth 80 | Set-Content -Path $target -Encoding UTF8
|
||||
|
||||
try {
|
||||
Start-Sleep -Seconds 3
|
||||
Invoke-Api -Method POST -Path "/clusters/$ClusterID/fabric/service-channels/leases/cleanup" -Body @{
|
||||
actor_user_id = $ActorUserID
|
||||
limit = 50
|
||||
} | Out-Null
|
||||
} catch {
|
||||
Write-Warning "cleanup failed after c18z77 smoke: $($_.Exception.Message)"
|
||||
}
|
||||
|
||||
if (-not $result.passed) {
|
||||
throw "C18Z77 rebuild planner-resolution smoke failed: $($failed -join ', ')"
|
||||
}
|
||||
|
||||
Write-Host "C18Z77 service-channel rebuild planner-resolution smoke passed. Result: $target"
|
||||
$result
|
||||
|
||||
|
||||
@@ -0,0 +1,243 @@
|
||||
param(
|
||||
[string]$ApiBaseUrl = "http://192.168.200.61:18121/api/v1",
|
||||
[string]$ClusterID = "cfc0743d-d960-49fb-9de8-96e063d5e4aa",
|
||||
[string]$ActorUserID = "f67d943f-5397-4b3a-a229-695fe67ad700",
|
||||
[string]$EntryNodeName = "test-3",
|
||||
[string]$RelayNodeName = "test-2",
|
||||
[string]$ExitNodeName = "test-1",
|
||||
[string]$DockerSSH = "test-docker",
|
||||
[string]$ExpectedBackendImage = "rap-backend:fabric-service-channel-0.2.256-c18z82",
|
||||
[string]$ExpectedNodeAgentImage = "rap-node-agent:0.2.270-c18z95",
|
||||
[string]$ResultPath = "artifacts\c18z78-service-channel-rebuild-planner-applied-smoke-result.json"
|
||||
)
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
$scriptDir = Split-Path -Parent $MyInvocation.MyCommand.Path
|
||||
$repoRoot = (Resolve-Path (Join-Path $scriptDir "..\..")).ProviderPath
|
||||
$runId = "c18z78-" + (Get-Date -Format "yyyyMMdd-HHmmss")
|
||||
|
||||
function Invoke-Api {
|
||||
param([string]$Method, [string]$Path, [object]$Body = $null)
|
||||
$params = @{ Method = $Method; Uri = "$ApiBaseUrl$Path"; TimeoutSec = 30 }
|
||||
if ($null -ne $Body) {
|
||||
$params.ContentType = "application/json"
|
||||
$params.Body = ($Body | ConvertTo-Json -Depth 80)
|
||||
}
|
||||
return Invoke-RestMethod @params
|
||||
}
|
||||
|
||||
function Get-PropertyValue {
|
||||
param([object]$Item, [string]$Name, [object]$Default = $null)
|
||||
if ($null -eq $Item) { return $Default }
|
||||
$property = $Item.PSObject.Properties[$Name]
|
||||
if ($null -eq $property) { return $Default }
|
||||
return $property.Value
|
||||
}
|
||||
|
||||
function Get-NodeByName {
|
||||
param([string]$Name)
|
||||
$nodes = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/nodes?actor_user_id=$ActorUserID").nodes
|
||||
$node = @($nodes | Where-Object { $_.name -eq $Name }) | Select-Object -First 1
|
||||
if ($null -eq $node) { throw "Node '$Name' was not found" }
|
||||
return $node
|
||||
}
|
||||
|
||||
function New-RouteIntent {
|
||||
param([string]$SourceNodeID, [string]$DestinationNodeID, [string[]]$Hops)
|
||||
$expiresAt = (Get-Date).ToUniversalTime().AddMinutes(5).ToString("o")
|
||||
return (Invoke-Api -Method POST -Path "/clusters/$ClusterID/mesh/route-intents" -Body @{
|
||||
actor_user_id = $ActorUserID
|
||||
source_selector = @{ node_id = $SourceNodeID }
|
||||
destination_selector = @{ node_id = $DestinationNodeID }
|
||||
service_class = "vpn_packets"
|
||||
priority = 2100000000
|
||||
policy = @{
|
||||
synthetic_enabled = $true
|
||||
route_version = "$runId-primary"
|
||||
policy_version = "$runId-primary"
|
||||
peer_directory_version = "$runId-primary"
|
||||
hops = @($Hops)
|
||||
allowed_channels = @("vpn_packet", "fabric_control")
|
||||
max_ttl = 8
|
||||
max_hops = 8
|
||||
expires_at = $expiresAt
|
||||
metadata = @{ smoke = "c18z78_service_channel_rebuild_planner_applied"; run_id = $runId }
|
||||
}
|
||||
}).route_intent
|
||||
}
|
||||
|
||||
function Disable-ExistingRouteIntents {
|
||||
param([string]$SourceNodeID, [string]$DestinationNodeID)
|
||||
$items = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/mesh/route-intents?actor_user_id=$ActorUserID").route_intents
|
||||
foreach ($item in @($items)) {
|
||||
if ([string](Get-PropertyValue -Item $item -Name "status" -Default "") -ne "active") { continue }
|
||||
if ([string](Get-PropertyValue -Item $item -Name "service_class" -Default "") -ne "vpn_packets") { continue }
|
||||
$sourceSelector = Get-PropertyValue -Item $item -Name "source_selector" -Default $null
|
||||
$destinationSelector = Get-PropertyValue -Item $item -Name "destination_selector" -Default $null
|
||||
if ([string](Get-PropertyValue -Item $sourceSelector -Name "node_id" -Default "") -ne $SourceNodeID) { continue }
|
||||
if ([string](Get-PropertyValue -Item $destinationSelector -Name "node_id" -Default "") -ne $DestinationNodeID) { continue }
|
||||
[void](Invoke-Api -Method POST -Path "/clusters/$ClusterID/mesh/route-intents/$($item.id)/disable" -Body @{
|
||||
actor_user_id = $ActorUserID
|
||||
reason = "c18z78 isolate rebuild-planner-applied smoke route pair"
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
function Send-DegradedHeartbeat {
|
||||
param([string]$EntryNodeID, [string]$PrimaryRouteID)
|
||||
$observedAt = (Get-Date).ToUniversalTime().ToString("o")
|
||||
return Invoke-Api -Method POST -Path "/clusters/$ClusterID/nodes/$EntryNodeID/heartbeats" -Body @{
|
||||
health_status = "healthy"
|
||||
reported_version = "0.2.252-c18z78"
|
||||
capabilities = @{
|
||||
fabric_service_channel_runtime = $true
|
||||
fabric_service_channel_route_quality_feedback = $true
|
||||
smoke_feedback_injection = "c18z77"
|
||||
}
|
||||
service_states = @{ smoke = "c18z78_primary_degraded_alternate_after_lease" }
|
||||
metadata = @{
|
||||
fabric_service_channel_runtime_report = @{
|
||||
schema_version = "c18l.fabric_service_channel_runtime_report.v1"
|
||||
config_version = "$runId-primary"
|
||||
cluster_id = $ClusterID
|
||||
local_node_id = $EntryNodeID
|
||||
observed_at = $observedAt
|
||||
ingress = @{
|
||||
flow_scheduler = @{
|
||||
schema_version = "rap.fabric_flow_scheduler.v1"
|
||||
service_neutral = $true
|
||||
service_mode = "application_protocol_agnostic"
|
||||
channel_stats = @{
|
||||
"c18z78-primary-degraded" = @{
|
||||
last_route_id = $PrimaryRouteID
|
||||
last_failed_route_id = $PrimaryRouteID
|
||||
route_generation = "$runId-primary"
|
||||
last_error = "c18z78 primary route degraded; alternate added after lease"
|
||||
last_send_duration_ms = 1200
|
||||
consecutive_failures = 3
|
||||
stall_count = 2
|
||||
route_rebuild_recommended = $true
|
||||
degraded_fallback_recommended = $false
|
||||
quality_window_sample_count = 8
|
||||
quality_window_success_count = 2
|
||||
quality_window_failure_count = 3
|
||||
quality_window_slow_count = 2
|
||||
quality_window_drop_count = 0
|
||||
quality_window_avg_latency_ms = 1200
|
||||
quality_window_last_updated_at = $observedAt
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
smoke = @{ name = "c18z78_service_channel_rebuild_planner_applied"; run_id = $runId }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
$entryNode = Get-NodeByName -Name $EntryNodeName
|
||||
$relayNode = Get-NodeByName -Name $RelayNodeName
|
||||
$exitNode = Get-NodeByName -Name $ExitNodeName
|
||||
[void](Disable-ExistingRouteIntents -SourceNodeID $entryNode.id -DestinationNodeID $exitNode.id)
|
||||
$route = New-RouteIntent -SourceNodeID $entryNode.id -DestinationNodeID $exitNode.id -Hops @($entryNode.id, $exitNode.id)
|
||||
$lease = (Invoke-Api -Method POST -Path "/clusters/$ClusterID/fabric/service-channels/leases" -Body @{
|
||||
actor_user_id = $ActorUserID
|
||||
organization_id = "smoke-org"
|
||||
user_id = "smoke-user"
|
||||
resource_id = "c18z78-vpn-smoke"
|
||||
service_class = "vpn_packets"
|
||||
entry_node_ids = @([string]$entryNode.id)
|
||||
exit_node_ids = @([string]$exitNode.id)
|
||||
preferred_entry_node_id = [string]$entryNode.id
|
||||
preferred_exit_node_id = [string]$exitNode.id
|
||||
allowed_channels = @("vpn_packet", "fabric_control")
|
||||
ttl_seconds = 90
|
||||
metadata = @{ smoke = "c18z78_service_channel_rebuild_planner_applied"; run_id = $runId }
|
||||
}).fabric_service_channel_lease
|
||||
$primaryRouteID = [string](Get-PropertyValue -Item (Get-PropertyValue -Item $lease -Name "primary_route" -Default $null) -Name "route_id" -Default $route.id)
|
||||
|
||||
[void](Send-DegradedHeartbeat -EntryNodeID $entryNode.id -PrimaryRouteID $primaryRouteID)
|
||||
|
||||
$matchingChannel = $null
|
||||
for ($i = 0; $i -lt 10; $i++) {
|
||||
Start-Sleep -Seconds 3
|
||||
$accessTelemetry = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/fabric/service-channels/access-telemetry?actor_user_id=$ActorUserID&limit=50").fabric_service_channel_access_telemetry
|
||||
$matchingChannel = @($accessTelemetry.active_channels | Where-Object { $_.channel_id -eq $lease.channel_id }) | Select-Object -First 1
|
||||
if ($null -ne $matchingChannel -and [string](Get-PropertyValue -Item $matchingChannel -Name "remediation_action" -Default "") -eq "rebuild_route") {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
$command = Get-PropertyValue -Item $matchingChannel -Name "remediation_command" -Default $null
|
||||
$commandID = [string](Get-PropertyValue -Item $command -Name "command_id" -Default "")
|
||||
$alternate = New-RouteIntent -SourceNodeID $entryNode.id -DestinationNodeID $exitNode.id -Hops @($entryNode.id, $relayNode.id, $exitNode.id)
|
||||
$alternateRouteID = [string]$alternate.id
|
||||
[void](Invoke-Api -Method GET -Path "/clusters/$ClusterID/nodes/$($entryNode.id)/mesh/synthetic-config")
|
||||
|
||||
$attempts = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/fabric/service-channels/rebuild-attempts?actor_user_id=$ActorUserID&reporter_node_id=$($entryNode.id)&rebuild_request_id=$commandID&limit=10").rebuild_attempts
|
||||
$attempt = @($attempts | Where-Object { $_.rebuild_request_id -eq $commandID }) | Select-Object -First 1
|
||||
|
||||
Start-Sleep -Seconds 30
|
||||
$postAccess = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/fabric/service-channels/access-telemetry?actor_user_id=$ActorUserID&limit=50").fabric_service_channel_access_telemetry
|
||||
$postChannel = @($postAccess.active_channels | Where-Object { $_.channel_id -eq $lease.channel_id }) | Select-Object -First 1
|
||||
|
||||
$backendLine = (& ssh $DockerSSH "docker ps --format '{{.Names}} {{.Image}} {{.Status}}' | grep '^rap_test_backend '") | Out-String
|
||||
$nodeLines = (& ssh $DockerSSH "docker ps --format '{{.Names}} {{.Image}} {{.Status}}' | grep '^rap_test_node_test_'") | Out-String
|
||||
$checks = [ordered]@{
|
||||
backend_expected_image_deployed = $backendLine.Contains($ExpectedBackendImage)
|
||||
node_agent_expected_image_deployed = $nodeLines.Contains($ExpectedNodeAgentImage)
|
||||
lease_ready = ([string]$lease.status -eq "ready")
|
||||
remediation_rebuild_route_visible = ($null -ne $matchingChannel -and [string](Get-PropertyValue -Item $matchingChannel -Name "remediation_action" -Default "") -eq "rebuild_route")
|
||||
remediation_command_visible = ($null -ne $command -and $commandID.Length -gt 0)
|
||||
durable_rebuild_intent_recorded = ($null -ne $attempt)
|
||||
durable_rebuild_intent_resolved_applied = ($null -ne $attempt -and [string](Get-PropertyValue -Item $attempt -Name "rebuild_status" -Default "") -eq "applied")
|
||||
durable_rebuild_intent_outcome_replacement_selected = ($null -ne $attempt -and [string](Get-PropertyValue -Item $attempt -Name "outcome" -Default "") -eq "replacement_selected")
|
||||
durable_rebuild_intent_replacement_matches = ($null -ne $attempt -and [string](Get-PropertyValue -Item $attempt -Name "replacement_route_id" -Default "") -eq $alternateRouteID)
|
||||
durable_rebuild_intent_source_matches = ($null -ne $attempt -and [string](Get-PropertyValue -Item $attempt -Name "decision_source" -Default "") -eq "service_channel_remediation_command")
|
||||
access_telemetry_reports_planner_applied = ($null -ne $postChannel -and [string](Get-PropertyValue -Item $postChannel -Name "remediation_execution_status" -Default "") -eq "rebuild_request_applied")
|
||||
}
|
||||
$failed = @($checks.GetEnumerator() | Where-Object { -not $_.Value } | ForEach-Object { $_.Key })
|
||||
|
||||
$result = [ordered]@{
|
||||
schema_version = "c18z78.service_channel_rebuild_planner_applied_smoke.v1"
|
||||
run_id = $runId
|
||||
cluster_id = $ClusterID
|
||||
channel_id = [string]$lease.channel_id
|
||||
primary_route_id = $primaryRouteID
|
||||
alternate_route_id = $alternateRouteID
|
||||
rebuild_request_id = $commandID
|
||||
passed = ($failed.Count -eq 0)
|
||||
checks = $checks
|
||||
failed_checks = $failed
|
||||
summary = [ordered]@{
|
||||
backend_container = $backendLine.Trim()
|
||||
node_containers = $nodeLines.Trim()
|
||||
remediation_command = $command
|
||||
rebuild_attempt = $attempt
|
||||
post_channel = $postChannel
|
||||
}
|
||||
}
|
||||
|
||||
$target = Join-Path $repoRoot $ResultPath
|
||||
$result | ConvertTo-Json -Depth 80 | Set-Content -Path $target -Encoding UTF8
|
||||
|
||||
try {
|
||||
Start-Sleep -Seconds 3
|
||||
Invoke-Api -Method POST -Path "/clusters/$ClusterID/fabric/service-channels/leases/cleanup" -Body @{
|
||||
actor_user_id = $ActorUserID
|
||||
limit = 50
|
||||
} | Out-Null
|
||||
} catch {
|
||||
Write-Warning "cleanup failed after c18z78 smoke: $($_.Exception.Message)"
|
||||
}
|
||||
|
||||
if (-not $result.passed) {
|
||||
throw "C18Z78 rebuild planner-applied smoke failed: $($failed -join ', ')"
|
||||
}
|
||||
|
||||
Write-Host "C18Z78 service-channel rebuild planner-applied smoke passed. Result: $target"
|
||||
$result
|
||||
|
||||
|
||||
@@ -0,0 +1,426 @@
|
||||
param(
|
||||
[string]$ApiBaseUrl = "http://192.168.200.61:18121/api/v1",
|
||||
[string]$ClusterID = "cfc0743d-d960-49fb-9de8-96e063d5e4aa",
|
||||
[string]$ActorUserID = "f67d943f-5397-4b3a-a229-695fe67ad700",
|
||||
[string]$EntryNodeName = "test-1",
|
||||
[string]$RelayNodeName = "test-3",
|
||||
[string]$ExitNodeName = "test-2",
|
||||
[string]$EntryBaseUrl = "http://192.168.200.61:19131",
|
||||
[string]$DockerSSH = "test-docker",
|
||||
[string]$ExpectedBackendImage = "rap-backend:fabric-service-channel-0.2.256-c18z82",
|
||||
[string]$ExpectedNodeAgentImage = "rap-node-agent:0.2.270-c18z95",
|
||||
[string]$ResultPath = "artifacts\c18z79-service-channel-planner-runtime-loop-smoke-result.json"
|
||||
)
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
$scriptDir = Split-Path -Parent $MyInvocation.MyCommand.Path
|
||||
$repoRoot = (Resolve-Path (Join-Path $scriptDir "..\..")).ProviderPath
|
||||
$runId = "c18z79-" + (Get-Date -Format "yyyyMMdd-HHmmss")
|
||||
|
||||
function Invoke-Api {
|
||||
param([string]$Method, [string]$Path, [object]$Body = $null)
|
||||
$params = @{ Method = $Method; Uri = "$ApiBaseUrl$Path"; TimeoutSec = 30 }
|
||||
if ($null -ne $Body) {
|
||||
$params.ContentType = "application/json"
|
||||
$params.Body = ($Body | ConvertTo-Json -Depth 80)
|
||||
}
|
||||
return Invoke-RestMethod @params
|
||||
}
|
||||
|
||||
function Get-PropertyValue {
|
||||
param([object]$Item, [string]$Name, [object]$Default = $null)
|
||||
if ($null -eq $Item) { return $Default }
|
||||
$property = $Item.PSObject.Properties[$Name]
|
||||
if ($null -eq $property) { return $Default }
|
||||
return $property.Value
|
||||
}
|
||||
|
||||
function Get-NodeByName {
|
||||
param([string]$Name)
|
||||
$nodes = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/nodes?actor_user_id=$ActorUserID").nodes
|
||||
$node = @($nodes | Where-Object { $_.name -eq $Name }) | Select-Object -First 1
|
||||
if ($null -eq $node) { throw "Node '$Name' was not found" }
|
||||
return $node
|
||||
}
|
||||
|
||||
function New-IPv4TcpPacket {
|
||||
param(
|
||||
[byte[]]$Source = @(10, 77, 0, 2),
|
||||
[byte[]]$Destination = @(192, 168, 200, 95),
|
||||
[int]$SourcePort,
|
||||
[int]$DestinationPort = 3389
|
||||
)
|
||||
$packet = [byte[]]::new(40)
|
||||
$packet[0] = 0x45
|
||||
$packet[2] = 0
|
||||
$packet[3] = 40
|
||||
$packet[8] = 64
|
||||
$packet[9] = 6
|
||||
[Array]::Copy($Source, 0, $packet, 12, 4)
|
||||
[Array]::Copy($Destination, 0, $packet, 16, 4)
|
||||
$packet[20] = [byte](($SourcePort -shr 8) -band 0xff)
|
||||
$packet[21] = [byte]($SourcePort -band 0xff)
|
||||
$packet[22] = [byte](($DestinationPort -shr 8) -band 0xff)
|
||||
$packet[23] = [byte]($DestinationPort -band 0xff)
|
||||
$packet[32] = 0x50
|
||||
return $packet
|
||||
}
|
||||
|
||||
function ConvertTo-VPNPacketBatch {
|
||||
param([byte[][]]$Packets)
|
||||
$buffer = New-Object System.Collections.Generic.List[byte]
|
||||
foreach ($packet in $Packets) {
|
||||
if ($null -eq $packet -or $packet.Length -eq 0) { continue }
|
||||
$size = [uint32]$packet.Length
|
||||
$buffer.Add([byte](($size -shr 24) -band 0xff))
|
||||
$buffer.Add([byte](($size -shr 16) -band 0xff))
|
||||
$buffer.Add([byte](($size -shr 8) -band 0xff))
|
||||
$buffer.Add([byte]($size -band 0xff))
|
||||
foreach ($value in $packet) {
|
||||
$buffer.Add($value)
|
||||
}
|
||||
}
|
||||
return $buffer.ToArray()
|
||||
}
|
||||
|
||||
function New-RouteIntent {
|
||||
param([string]$SourceNodeID, [string]$DestinationNodeID, [string[]]$Hops)
|
||||
$expiresAt = (Get-Date).ToUniversalTime().AddMinutes(5).ToString("o")
|
||||
return (Invoke-Api -Method POST -Path "/clusters/$ClusterID/mesh/route-intents" -Body @{
|
||||
actor_user_id = $ActorUserID
|
||||
source_selector = @{ node_id = $SourceNodeID }
|
||||
destination_selector = @{ node_id = $DestinationNodeID }
|
||||
service_class = "vpn_packets"
|
||||
priority = 2100000000
|
||||
policy = @{
|
||||
synthetic_enabled = $true
|
||||
route_version = "$runId-primary"
|
||||
policy_version = "$runId-primary"
|
||||
peer_directory_version = "$runId-primary"
|
||||
hops = @($Hops)
|
||||
allowed_channels = @("vpn_packet", "fabric_control")
|
||||
max_ttl = 8
|
||||
max_hops = 8
|
||||
expires_at = $expiresAt
|
||||
metadata = @{ smoke = "c18z79_service_channel_planner_runtime_loop"; run_id = $runId }
|
||||
}
|
||||
}).route_intent
|
||||
}
|
||||
|
||||
function Disable-ExistingRouteIntents {
|
||||
param([string]$SourceNodeID, [string]$DestinationNodeID)
|
||||
$items = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/mesh/route-intents?actor_user_id=$ActorUserID").route_intents
|
||||
foreach ($item in @($items)) {
|
||||
if ([string](Get-PropertyValue -Item $item -Name "status" -Default "") -ne "active") { continue }
|
||||
if ([string](Get-PropertyValue -Item $item -Name "service_class" -Default "") -ne "vpn_packets") { continue }
|
||||
$sourceSelector = Get-PropertyValue -Item $item -Name "source_selector" -Default $null
|
||||
$destinationSelector = Get-PropertyValue -Item $item -Name "destination_selector" -Default $null
|
||||
if ([string](Get-PropertyValue -Item $sourceSelector -Name "node_id" -Default "") -ne $SourceNodeID) { continue }
|
||||
if ([string](Get-PropertyValue -Item $destinationSelector -Name "node_id" -Default "") -ne $DestinationNodeID) { continue }
|
||||
[void](Invoke-Api -Method POST -Path "/clusters/$ClusterID/mesh/route-intents/$($item.id)/disable" -Body @{
|
||||
actor_user_id = $ActorUserID
|
||||
reason = "c18z79 isolate planner-runtime-loop smoke route pair"
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
function Send-DegradedHeartbeat {
|
||||
param([string]$EntryNodeID, [string]$PrimaryRouteID)
|
||||
$observedAt = (Get-Date).ToUniversalTime().ToString("o")
|
||||
return Invoke-Api -Method POST -Path "/clusters/$ClusterID/nodes/$EntryNodeID/heartbeats" -Body @{
|
||||
health_status = "healthy"
|
||||
reported_version = "0.2.253-c18z79"
|
||||
capabilities = @{
|
||||
fabric_service_channel_runtime = $true
|
||||
fabric_service_channel_route_quality_feedback = $true
|
||||
smoke_feedback_injection = "c18z77"
|
||||
}
|
||||
service_states = @{ smoke = "c18z79_primary_degraded_alternate_after_lease" }
|
||||
metadata = @{
|
||||
fabric_service_channel_runtime_report = @{
|
||||
schema_version = "c18l.fabric_service_channel_runtime_report.v1"
|
||||
config_version = "$runId-primary"
|
||||
cluster_id = $ClusterID
|
||||
local_node_id = $EntryNodeID
|
||||
observed_at = $observedAt
|
||||
ingress = @{
|
||||
flow_scheduler = @{
|
||||
schema_version = "rap.fabric_flow_scheduler.v1"
|
||||
service_neutral = $true
|
||||
service_mode = "application_protocol_agnostic"
|
||||
channel_stats = @{
|
||||
"c18z79-primary-degraded" = @{
|
||||
last_route_id = $PrimaryRouteID
|
||||
last_failed_route_id = $PrimaryRouteID
|
||||
route_generation = "$runId-primary"
|
||||
last_error = "c18z79 primary route degraded; alternate added after lease"
|
||||
last_send_duration_ms = 1200
|
||||
consecutive_failures = 3
|
||||
stall_count = 2
|
||||
route_rebuild_recommended = $true
|
||||
degraded_fallback_recommended = $false
|
||||
quality_window_sample_count = 8
|
||||
quality_window_success_count = 2
|
||||
quality_window_failure_count = 3
|
||||
quality_window_slow_count = 2
|
||||
quality_window_drop_count = 0
|
||||
quality_window_avg_latency_ms = 1200
|
||||
quality_window_last_updated_at = $observedAt
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
smoke = @{ name = "c18z79_service_channel_planner_runtime_loop"; run_id = $runId }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
$entryNode = Get-NodeByName -Name $EntryNodeName
|
||||
$relayNode = Get-NodeByName -Name $RelayNodeName
|
||||
$exitNode = Get-NodeByName -Name $ExitNodeName
|
||||
[void](Disable-ExistingRouteIntents -SourceNodeID $entryNode.id -DestinationNodeID $exitNode.id)
|
||||
$route = New-RouteIntent -SourceNodeID $entryNode.id -DestinationNodeID $exitNode.id -Hops @($entryNode.id, $exitNode.id)
|
||||
$lease = (Invoke-Api -Method POST -Path "/clusters/$ClusterID/fabric/service-channels/leases" -Body @{
|
||||
actor_user_id = $ActorUserID
|
||||
organization_id = "smoke-org"
|
||||
user_id = "smoke-user"
|
||||
resource_id = "c18z79-vpn-smoke"
|
||||
service_class = "vpn_packets"
|
||||
entry_node_ids = @([string]$entryNode.id)
|
||||
exit_node_ids = @([string]$exitNode.id)
|
||||
preferred_entry_node_id = [string]$entryNode.id
|
||||
preferred_exit_node_id = [string]$exitNode.id
|
||||
allowed_channels = @("vpn_packet", "fabric_control")
|
||||
ttl_seconds = 90
|
||||
metadata = @{ smoke = "c18z79_service_channel_planner_runtime_loop"; run_id = $runId }
|
||||
}).fabric_service_channel_lease
|
||||
$primaryRouteID = [string](Get-PropertyValue -Item (Get-PropertyValue -Item $lease -Name "primary_route" -Default $null) -Name "route_id" -Default $route.id)
|
||||
|
||||
[void](Send-DegradedHeartbeat -EntryNodeID $entryNode.id -PrimaryRouteID $primaryRouteID)
|
||||
|
||||
$matchingChannel = $null
|
||||
for ($i = 0; $i -lt 10; $i++) {
|
||||
Start-Sleep -Seconds 3
|
||||
$accessTelemetry = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/fabric/service-channels/access-telemetry?actor_user_id=$ActorUserID&limit=50").fabric_service_channel_access_telemetry
|
||||
$matchingChannel = @($accessTelemetry.active_channels | Where-Object { $_.channel_id -eq $lease.channel_id }) | Select-Object -First 1
|
||||
if ($null -ne $matchingChannel -and [string](Get-PropertyValue -Item $matchingChannel -Name "remediation_action" -Default "") -eq "rebuild_route") {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
$command = Get-PropertyValue -Item $matchingChannel -Name "remediation_command" -Default $null
|
||||
$commandID = [string](Get-PropertyValue -Item $command -Name "command_id" -Default "")
|
||||
$alternate = New-RouteIntent -SourceNodeID $entryNode.id -DestinationNodeID $exitNode.id -Hops @($entryNode.id, $relayNode.id, $exitNode.id)
|
||||
$alternateRouteID = [string]$alternate.id
|
||||
[void](Invoke-Api -Method GET -Path "/clusters/$ClusterID/nodes/$($entryNode.id)/mesh/synthetic-config")
|
||||
|
||||
$attempts = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/fabric/service-channels/rebuild-attempts?actor_user_id=$ActorUserID&reporter_node_id=$($entryNode.id)&rebuild_request_id=$commandID&limit=10").rebuild_attempts
|
||||
$attempt = @($attempts | Where-Object { $_.rebuild_request_id -eq $commandID }) | Select-Object -First 1
|
||||
|
||||
$routeManagerDecision = $null
|
||||
$routeManagerTransition = $null
|
||||
for ($i = 0; $i -lt 12; $i++) {
|
||||
Start-Sleep -Seconds 5
|
||||
$heartbeats = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/nodes/$($entryNode.id)/heartbeats?actor_user_id=$ActorUserID&limit=5").heartbeats
|
||||
foreach ($heartbeat in @($heartbeats)) {
|
||||
$runtimeReport = Get-PropertyValue -Item (Get-PropertyValue -Item $heartbeat -Name "metadata" -Default $null) -Name "fabric_service_channel_runtime_report" -Default $null
|
||||
$ingress = Get-PropertyValue -Item $runtimeReport -Name "ingress" -Default $null
|
||||
$routeManager = Get-PropertyValue -Item $ingress -Name "route_manager" -Default $null
|
||||
$routeManagerTransition = Get-PropertyValue -Item $ingress -Name "route_manager_transition" -Default $null
|
||||
$decisions = @()
|
||||
if ($null -ne $routeManager -and $routeManager.PSObject.Properties.Name -contains "decisions") {
|
||||
$decisions = @($routeManager.decisions)
|
||||
}
|
||||
$routeManagerDecision = $decisions | Where-Object {
|
||||
[string](Get-PropertyValue -Item $_ -Name "rebuild_request_id" -Default "") -eq $commandID -and
|
||||
[string](Get-PropertyValue -Item $_ -Name "route_id" -Default "") -eq $primaryRouteID -and
|
||||
[string](Get-PropertyValue -Item $_ -Name "replacement_route_id" -Default "") -eq $alternateRouteID
|
||||
} | Select-Object -First 1
|
||||
if ($null -ne $routeManagerDecision) {
|
||||
break
|
||||
}
|
||||
}
|
||||
if ($null -ne $routeManagerDecision) {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
$packetPath = $lease.entry_http.path_template.
|
||||
Replace("{cluster_id}", $ClusterID).
|
||||
Replace("{channel_id}", [string]$lease.channel_id).
|
||||
Replace("{resource_id}", "c18z79-vpn-smoke")
|
||||
$packetUrl = $EntryBaseUrl.TrimEnd("/") + $packetPath
|
||||
$headers = @{
|
||||
"X-RAP-Service-Channel-Token" = [string]$lease.token.token
|
||||
"X-RAP-Fabric-Channel-ID" = [string]$lease.channel_id
|
||||
"X-RAP-Service-Class" = "vpn_packets"
|
||||
"X-RAP-Channel-Class" = "vpn_packet"
|
||||
"X-RAP-Traffic-Class" = "interactive"
|
||||
}
|
||||
$baselineFallbackLocal = 0
|
||||
$baselineRouteFailures = 0
|
||||
$baselineFlowDropped = 0
|
||||
$baselineSchedulerDropped = 0
|
||||
$baselineHeartbeats = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/nodes/$($entryNode.id)/heartbeats?actor_user_id=$ActorUserID&limit=1").heartbeats
|
||||
if (@($baselineHeartbeats).Count -gt 0) {
|
||||
$baselineRuntimeReport = Get-PropertyValue -Item (Get-PropertyValue -Item $baselineHeartbeats[0] -Name "metadata" -Default $null) -Name "fabric_service_channel_runtime_report" -Default $null
|
||||
$baselineIngress = Get-PropertyValue -Item $baselineRuntimeReport -Name "ingress" -Default $null
|
||||
$baselineFlowScheduler = Get-PropertyValue -Item $baselineIngress -Name "flow_scheduler" -Default $null
|
||||
$baselineFallbackLocal = [int](Get-PropertyValue -Item $baselineIngress -Name "send_fallback_local" -Default 0)
|
||||
$baselineRouteFailures = [int](Get-PropertyValue -Item $baselineIngress -Name "send_route_failures" -Default 0)
|
||||
$baselineFlowDropped = [int](Get-PropertyValue -Item $baselineIngress -Name "send_flow_dropped" -Default 0)
|
||||
$baselineSchedulerDropped = [int](Get-PropertyValue -Item $baselineFlowScheduler -Name "dropped" -Default 0)
|
||||
}
|
||||
$trafficPackets = @()
|
||||
foreach ($offset in 0..15) {
|
||||
$trafficPackets += ,(New-IPv4TcpPacket -SourcePort (54000 + $offset) -DestinationPort 3389)
|
||||
}
|
||||
$trafficPayload = ConvertTo-VPNPacketBatch -Packets $trafficPackets
|
||||
$trafficPayloadPath = Join-Path ([System.IO.Path]::GetTempPath()) "$runId-vpn-packet-batch.bin"
|
||||
[System.IO.File]::WriteAllBytes($trafficPayloadPath, [byte[]]$trafficPayload)
|
||||
$trafficStarted = Get-Date
|
||||
$trafficResponse = Invoke-WebRequest -Method Post -Uri ($packetUrl + "?batch=true") -Headers $headers -InFile $trafficPayloadPath -ContentType "application/vnd.rap.vpn-packet-batch.v1" -TimeoutSec 30
|
||||
$trafficDurationMs = [int]((Get-Date) - $trafficStarted).TotalMilliseconds
|
||||
$trafficAcceptedBy = [string]$trafficResponse.Headers["X-RAP-Service-Channel-Accepted-By"]
|
||||
Remove-Item -Path $trafficPayloadPath -Force -ErrorAction SilentlyContinue
|
||||
|
||||
$replacementTrafficObserved = $false
|
||||
$replacementLastSelected = ""
|
||||
$replacementFlowStats = @()
|
||||
$postRemediationFallbackLocal = 0
|
||||
$postRemediationRouteFailures = 0
|
||||
$postRemediationFlowDropped = 0
|
||||
$postRemediationSchedulerDropped = 0
|
||||
for ($i = 0; $i -lt 12; $i++) {
|
||||
Start-Sleep -Seconds 5
|
||||
$heartbeats = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/nodes/$($entryNode.id)/heartbeats?actor_user_id=$ActorUserID&limit=5").heartbeats
|
||||
foreach ($heartbeat in @($heartbeats)) {
|
||||
$runtimeReport = Get-PropertyValue -Item (Get-PropertyValue -Item $heartbeat -Name "metadata" -Default $null) -Name "fabric_service_channel_runtime_report" -Default $null
|
||||
$ingress = Get-PropertyValue -Item $runtimeReport -Name "ingress" -Default $null
|
||||
$replacementLastSelected = [string](Get-PropertyValue -Item $ingress -Name "last_selected_route_id" -Default "")
|
||||
$postRemediationFallbackLocal = [int](Get-PropertyValue -Item $ingress -Name "send_fallback_local" -Default 0)
|
||||
$postRemediationRouteFailures = [int](Get-PropertyValue -Item $ingress -Name "send_route_failures" -Default 0)
|
||||
$postRemediationFlowDropped = [int](Get-PropertyValue -Item $ingress -Name "send_flow_dropped" -Default 0)
|
||||
$flowScheduler = Get-PropertyValue -Item $ingress -Name "flow_scheduler" -Default $null
|
||||
$postRemediationSchedulerDropped = [int](Get-PropertyValue -Item $flowScheduler -Name "dropped" -Default 0)
|
||||
$channelStats = Get-PropertyValue -Item $flowScheduler -Name "channel_stats" -Default $null
|
||||
if ($null -ne $channelStats) {
|
||||
$replacementFlowStats = @()
|
||||
foreach ($property in @($channelStats.PSObject.Properties)) {
|
||||
$stat = $property.Value
|
||||
if ([string](Get-PropertyValue -Item $stat -Name "last_route_id" -Default "") -eq $alternateRouteID) {
|
||||
$replacementFlowStats += $stat
|
||||
}
|
||||
}
|
||||
}
|
||||
if ($replacementLastSelected -eq $alternateRouteID -and $replacementFlowStats.Count -ge 1) {
|
||||
$replacementTrafficObserved = $true
|
||||
break
|
||||
}
|
||||
}
|
||||
if ($replacementTrafficObserved) {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
$postAccess = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/fabric/service-channels/access-telemetry?actor_user_id=$ActorUserID&limit=50").fabric_service_channel_access_telemetry
|
||||
$postChannel = @($postAccess.active_channels | Where-Object { $_.channel_id -eq $lease.channel_id }) | Select-Object -First 1
|
||||
|
||||
$backendLine = (& ssh $DockerSSH "docker ps --format '{{.Names}} {{.Image}} {{.Status}}' | grep '^rap_test_backend '") | Out-String
|
||||
$nodeLines = (& ssh $DockerSSH "docker ps --format '{{.Names}} {{.Image}} {{.Status}}' | grep '^rap_test_node_test_'") | Out-String
|
||||
$checks = [ordered]@{
|
||||
backend_expected_image_deployed = $backendLine.Contains($ExpectedBackendImage)
|
||||
node_agent_expected_image_deployed = $nodeLines.Contains($ExpectedNodeAgentImage)
|
||||
lease_ready = ([string]$lease.status -eq "ready")
|
||||
remediation_rebuild_route_visible = ($null -ne $matchingChannel -and [string](Get-PropertyValue -Item $matchingChannel -Name "remediation_action" -Default "") -eq "rebuild_route")
|
||||
remediation_command_visible = ($null -ne $command -and $commandID.Length -gt 0)
|
||||
durable_rebuild_intent_recorded = ($null -ne $attempt)
|
||||
durable_rebuild_intent_resolved_applied = ($null -ne $attempt -and [string](Get-PropertyValue -Item $attempt -Name "rebuild_status" -Default "") -eq "applied")
|
||||
durable_rebuild_intent_outcome_replacement_selected = ($null -ne $attempt -and [string](Get-PropertyValue -Item $attempt -Name "outcome" -Default "") -eq "replacement_selected")
|
||||
durable_rebuild_intent_replacement_matches = ($null -ne $attempt -and [string](Get-PropertyValue -Item $attempt -Name "replacement_route_id" -Default "") -eq $alternateRouteID)
|
||||
durable_rebuild_intent_source_matches = ($null -ne $attempt -and [string](Get-PropertyValue -Item $attempt -Name "decision_source" -Default "") -eq "service_channel_remediation_command")
|
||||
access_telemetry_reports_planner_applied = ($null -ne $postChannel -and [string](Get-PropertyValue -Item $postChannel -Name "remediation_execution_status" -Default "") -eq "rebuild_request_applied")
|
||||
node_route_manager_consumed_same_command = ($null -ne $routeManagerDecision)
|
||||
node_route_manager_applied_rebuild = ($null -ne $routeManagerDecision -and [string](Get-PropertyValue -Item $routeManagerDecision -Name "rebuild_status" -Default "") -eq "applied")
|
||||
node_route_manager_transition_applied_rebuild = ($null -ne $routeManagerTransition -and [string](Get-PropertyValue -Item $routeManagerTransition -Name "status" -Default "") -eq "applied_rebuild")
|
||||
traffic_packet_accepted = ([int]$trafficResponse.StatusCode -eq 202)
|
||||
traffic_accepted_by_runtime = ($trafficAcceptedBy -eq "introspection" -or $trafficAcceptedBy -eq "signed")
|
||||
traffic_selected_replacement_route = $replacementTrafficObserved
|
||||
traffic_last_selected_route_matches_replacement = ($replacementLastSelected -eq $alternateRouteID)
|
||||
no_backend_or_local_fallback_after_replacement = (($postRemediationFallbackLocal - $baselineFallbackLocal) -eq 0)
|
||||
no_route_failures_after_replacement = (($postRemediationRouteFailures - $baselineRouteFailures) -eq 0)
|
||||
no_flow_drops_after_replacement = (($postRemediationFlowDropped - $baselineFlowDropped) -eq 0 -and ($postRemediationSchedulerDropped - $baselineSchedulerDropped) -eq 0)
|
||||
}
|
||||
$failed = @($checks.GetEnumerator() | Where-Object { -not $_.Value } | ForEach-Object { $_.Key })
|
||||
|
||||
$result = [ordered]@{
|
||||
schema_version = "c18z79.service_channel_planner_runtime_loop_smoke.v1"
|
||||
run_id = $runId
|
||||
cluster_id = $ClusterID
|
||||
channel_id = [string]$lease.channel_id
|
||||
primary_route_id = $primaryRouteID
|
||||
alternate_route_id = $alternateRouteID
|
||||
rebuild_request_id = $commandID
|
||||
passed = ($failed.Count -eq 0)
|
||||
checks = $checks
|
||||
failed_checks = $failed
|
||||
summary = [ordered]@{
|
||||
backend_container = $backendLine.Trim()
|
||||
node_containers = $nodeLines.Trim()
|
||||
remediation_command = $command
|
||||
rebuild_attempt = $attempt
|
||||
route_manager_decision = $routeManagerDecision
|
||||
route_manager_transition = $routeManagerTransition
|
||||
traffic_status_code = [int]$trafficResponse.StatusCode
|
||||
traffic_accepted_by = $trafficAcceptedBy
|
||||
traffic_duration_ms = $trafficDurationMs
|
||||
traffic_packet_count = $trafficPackets.Count
|
||||
replacement_last_selected_route_id = $replacementLastSelected
|
||||
replacement_flow_stat_count = $replacementFlowStats.Count
|
||||
replacement_flow_stats = $replacementFlowStats
|
||||
baseline_send_fallback_local = $baselineFallbackLocal
|
||||
baseline_send_route_failures = $baselineRouteFailures
|
||||
baseline_send_flow_dropped = $baselineFlowDropped
|
||||
baseline_scheduler_dropped = $baselineSchedulerDropped
|
||||
fallback_local_delta = ($postRemediationFallbackLocal - $baselineFallbackLocal)
|
||||
route_failure_delta = ($postRemediationRouteFailures - $baselineRouteFailures)
|
||||
flow_drop_delta = ($postRemediationFlowDropped - $baselineFlowDropped)
|
||||
scheduler_drop_delta = ($postRemediationSchedulerDropped - $baselineSchedulerDropped)
|
||||
post_remediation_send_fallback_local = $postRemediationFallbackLocal
|
||||
post_remediation_send_route_failures = $postRemediationRouteFailures
|
||||
post_remediation_send_flow_dropped = $postRemediationFlowDropped
|
||||
post_remediation_scheduler_dropped = $postRemediationSchedulerDropped
|
||||
post_channel = $postChannel
|
||||
}
|
||||
}
|
||||
|
||||
$target = Join-Path $repoRoot $ResultPath
|
||||
$result | ConvertTo-Json -Depth 80 | Set-Content -Path $target -Encoding UTF8
|
||||
|
||||
try {
|
||||
if ($primaryRouteID) {
|
||||
Invoke-Api -Method POST -Path "/clusters/$ClusterID/mesh/route-intents/$primaryRouteID/expire" -Body @{ actor_user_id = $ActorUserID } | Out-Null
|
||||
}
|
||||
if ($alternateRouteID) {
|
||||
Invoke-Api -Method POST -Path "/clusters/$ClusterID/mesh/route-intents/$alternateRouteID/expire" -Body @{ actor_user_id = $ActorUserID } | Out-Null
|
||||
}
|
||||
Start-Sleep -Seconds 3
|
||||
Invoke-Api -Method POST -Path "/clusters/$ClusterID/fabric/service-channels/leases/cleanup" -Body @{
|
||||
actor_user_id = $ActorUserID
|
||||
limit = 50
|
||||
} | Out-Null
|
||||
} catch {
|
||||
Write-Warning "cleanup failed after c18z79 smoke: $($_.Exception.Message)"
|
||||
}
|
||||
|
||||
if (-not $result.passed) {
|
||||
throw "C18Z79 planner-runtime-loop smoke failed: $($failed -join ', ')"
|
||||
}
|
||||
|
||||
Write-Host "C18Z79 service-channel planner-runtime-loop smoke passed. Result: $target"
|
||||
$result
|
||||
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,449 @@
|
||||
param(
|
||||
[string]$ApiBaseUrl = "http://192.168.200.61:18121/api/v1",
|
||||
[string]$ClusterID = "cfc0743d-d960-49fb-9de8-96e063d5e4aa",
|
||||
[string]$ActorUserID = "f67d943f-5397-4b3a-a229-695fe67ad700",
|
||||
[string]$EntryNodeName = "test-1",
|
||||
[string]$RelayNodeName = "test-3",
|
||||
[string]$ExitNodeName = "test-2",
|
||||
[string]$EntryBaseUrl = "http://192.168.200.61:19131",
|
||||
[string]$DockerSSH = "test-docker",
|
||||
[string]$ExpectedBackendImage = "rap-backend:fabric-service-channel-0.2.281-c18z109",
|
||||
[string]$ExpectedNodeAgentImage = "rap-node-agent:0.2.270-c18z95",
|
||||
[string]$ResultPath = "artifacts\c18z80-service-channel-post-rebuild-pressure-smoke-result.json"
|
||||
)
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
$scriptDir = Split-Path -Parent $MyInvocation.MyCommand.Path
|
||||
$repoRoot = (Resolve-Path (Join-Path $scriptDir "..\..")).ProviderPath
|
||||
$runId = "c18z80-" + (Get-Date -Format "yyyyMMdd-HHmmss")
|
||||
|
||||
function Invoke-Api {
|
||||
param([string]$Method, [string]$Path, [object]$Body = $null)
|
||||
$params = @{ Method = $Method; Uri = "$ApiBaseUrl$Path"; TimeoutSec = 30 }
|
||||
if ($null -ne $Body) {
|
||||
$params.ContentType = "application/json"
|
||||
$params.Body = ($Body | ConvertTo-Json -Depth 80)
|
||||
}
|
||||
return Invoke-RestMethod @params
|
||||
}
|
||||
|
||||
function Get-PropertyValue {
|
||||
param([object]$Item, [string]$Name, [object]$Default = $null)
|
||||
if ($null -eq $Item) { return $Default }
|
||||
$property = $Item.PSObject.Properties[$Name]
|
||||
if ($null -eq $property) { return $Default }
|
||||
return $property.Value
|
||||
}
|
||||
|
||||
function Get-NodeByName {
|
||||
param([string]$Name)
|
||||
$nodes = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/nodes?actor_user_id=$ActorUserID").nodes
|
||||
$node = @($nodes | Where-Object { $_.name -eq $Name }) | Select-Object -First 1
|
||||
if ($null -eq $node) { throw "Node '$Name' was not found" }
|
||||
return $node
|
||||
}
|
||||
|
||||
function New-IPv4TcpPacket {
|
||||
param(
|
||||
[byte[]]$Source = @(10, 77, 0, 2),
|
||||
[byte[]]$Destination = @(192, 168, 200, 95),
|
||||
[int]$SourcePort,
|
||||
[int]$DestinationPort = 3389
|
||||
)
|
||||
$packet = [byte[]]::new(40)
|
||||
$packet[0] = 0x45
|
||||
$packet[2] = 0
|
||||
$packet[3] = 40
|
||||
$packet[8] = 64
|
||||
$packet[9] = 6
|
||||
[Array]::Copy($Source, 0, $packet, 12, 4)
|
||||
[Array]::Copy($Destination, 0, $packet, 16, 4)
|
||||
$packet[20] = [byte](($SourcePort -shr 8) -band 0xff)
|
||||
$packet[21] = [byte]($SourcePort -band 0xff)
|
||||
$packet[22] = [byte](($DestinationPort -shr 8) -band 0xff)
|
||||
$packet[23] = [byte]($DestinationPort -band 0xff)
|
||||
$packet[32] = 0x50
|
||||
return $packet
|
||||
}
|
||||
|
||||
function ConvertTo-VPNPacketBatch {
|
||||
param([byte[][]]$Packets)
|
||||
$buffer = New-Object System.Collections.Generic.List[byte]
|
||||
foreach ($packet in $Packets) {
|
||||
if ($null -eq $packet -or $packet.Length -eq 0) { continue }
|
||||
$size = [uint32]$packet.Length
|
||||
$buffer.Add([byte](($size -shr 24) -band 0xff))
|
||||
$buffer.Add([byte](($size -shr 16) -band 0xff))
|
||||
$buffer.Add([byte](($size -shr 8) -band 0xff))
|
||||
$buffer.Add([byte]($size -band 0xff))
|
||||
foreach ($value in $packet) {
|
||||
$buffer.Add($value)
|
||||
}
|
||||
}
|
||||
return $buffer.ToArray()
|
||||
}
|
||||
|
||||
function New-RouteIntent {
|
||||
param([string]$SourceNodeID, [string]$DestinationNodeID, [string[]]$Hops)
|
||||
$expiresAt = (Get-Date).ToUniversalTime().AddMinutes(5).ToString("o")
|
||||
return (Invoke-Api -Method POST -Path "/clusters/$ClusterID/mesh/route-intents" -Body @{
|
||||
actor_user_id = $ActorUserID
|
||||
source_selector = @{ node_id = $SourceNodeID }
|
||||
destination_selector = @{ node_id = $DestinationNodeID }
|
||||
service_class = "vpn_packets"
|
||||
priority = 2100000000
|
||||
policy = @{
|
||||
synthetic_enabled = $true
|
||||
route_version = "$runId-primary"
|
||||
policy_version = "$runId-primary"
|
||||
peer_directory_version = "$runId-primary"
|
||||
hops = @($Hops)
|
||||
allowed_channels = @("vpn_packet", "fabric_control")
|
||||
max_ttl = 8
|
||||
max_hops = 8
|
||||
expires_at = $expiresAt
|
||||
metadata = @{ smoke = "c18z80_service_channel_post_rebuild_pressure"; run_id = $runId }
|
||||
}
|
||||
}).route_intent
|
||||
}
|
||||
|
||||
function Disable-ExistingRouteIntents {
|
||||
param([string]$SourceNodeID, [string]$DestinationNodeID)
|
||||
$items = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/mesh/route-intents?actor_user_id=$ActorUserID").route_intents
|
||||
foreach ($item in @($items)) {
|
||||
if ([string](Get-PropertyValue -Item $item -Name "status" -Default "") -ne "active") { continue }
|
||||
if ([string](Get-PropertyValue -Item $item -Name "service_class" -Default "") -ne "vpn_packets") { continue }
|
||||
$sourceSelector = Get-PropertyValue -Item $item -Name "source_selector" -Default $null
|
||||
$destinationSelector = Get-PropertyValue -Item $item -Name "destination_selector" -Default $null
|
||||
if ([string](Get-PropertyValue -Item $sourceSelector -Name "node_id" -Default "") -ne $SourceNodeID) { continue }
|
||||
if ([string](Get-PropertyValue -Item $destinationSelector -Name "node_id" -Default "") -ne $DestinationNodeID) { continue }
|
||||
[void](Invoke-Api -Method POST -Path "/clusters/$ClusterID/mesh/route-intents/$($item.id)/disable" -Body @{
|
||||
actor_user_id = $ActorUserID
|
||||
reason = "c18z80 isolate post-rebuild-pressure smoke route pair"
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
function Send-DegradedHeartbeat {
|
||||
param([string]$EntryNodeID, [string]$PrimaryRouteID)
|
||||
$observedAt = (Get-Date).ToUniversalTime().ToString("o")
|
||||
return Invoke-Api -Method POST -Path "/clusters/$ClusterID/nodes/$EntryNodeID/heartbeats" -Body @{
|
||||
health_status = "healthy"
|
||||
reported_version = "0.2.254-c18z80"
|
||||
capabilities = @{
|
||||
fabric_service_channel_runtime = $true
|
||||
fabric_service_channel_route_quality_feedback = $true
|
||||
smoke_feedback_injection = "c18z77"
|
||||
}
|
||||
service_states = @{ smoke = "c18z80_primary_degraded_alternate_after_lease" }
|
||||
metadata = @{
|
||||
fabric_service_channel_runtime_report = @{
|
||||
schema_version = "c18l.fabric_service_channel_runtime_report.v1"
|
||||
config_version = "$runId-primary"
|
||||
cluster_id = $ClusterID
|
||||
local_node_id = $EntryNodeID
|
||||
observed_at = $observedAt
|
||||
ingress = @{
|
||||
flow_scheduler = @{
|
||||
schema_version = "rap.fabric_flow_scheduler.v1"
|
||||
service_neutral = $true
|
||||
service_mode = "application_protocol_agnostic"
|
||||
channel_stats = @{
|
||||
"c18z80-primary-degraded" = @{
|
||||
last_route_id = $PrimaryRouteID
|
||||
last_failed_route_id = $PrimaryRouteID
|
||||
route_generation = "$runId-primary"
|
||||
last_error = "c18z80 primary route degraded; alternate added after lease"
|
||||
last_send_duration_ms = 1200
|
||||
consecutive_failures = 3
|
||||
stall_count = 2
|
||||
route_rebuild_recommended = $true
|
||||
degraded_fallback_recommended = $false
|
||||
quality_window_sample_count = 8
|
||||
quality_window_success_count = 2
|
||||
quality_window_failure_count = 3
|
||||
quality_window_slow_count = 2
|
||||
quality_window_drop_count = 0
|
||||
quality_window_avg_latency_ms = 1200
|
||||
quality_window_last_updated_at = $observedAt
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
smoke = @{ name = "c18z80_service_channel_post_rebuild_pressure"; run_id = $runId }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function Get-LatestIngressSnapshot {
|
||||
param([string]$EntryNodeID, [string]$ReplacementRouteID)
|
||||
$heartbeats = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/nodes/$EntryNodeID/heartbeats?actor_user_id=$ActorUserID&limit=5").heartbeats
|
||||
foreach ($heartbeat in @($heartbeats)) {
|
||||
$runtimeReport = Get-PropertyValue -Item (Get-PropertyValue -Item $heartbeat -Name "metadata" -Default $null) -Name "fabric_service_channel_runtime_report" -Default $null
|
||||
$ingress = Get-PropertyValue -Item $runtimeReport -Name "ingress" -Default $null
|
||||
if ($null -eq $ingress) { continue }
|
||||
$flowScheduler = Get-PropertyValue -Item $ingress -Name "flow_scheduler" -Default $null
|
||||
$channelStats = Get-PropertyValue -Item $flowScheduler -Name "channel_stats" -Default $null
|
||||
$replacementFlowStats = @()
|
||||
if ($null -ne $channelStats) {
|
||||
foreach ($property in @($channelStats.PSObject.Properties)) {
|
||||
$stat = $property.Value
|
||||
if ([string](Get-PropertyValue -Item $stat -Name "last_route_id" -Default "") -eq $ReplacementRouteID) {
|
||||
$replacementFlowStats += $stat
|
||||
}
|
||||
}
|
||||
}
|
||||
return [pscustomobject]@{
|
||||
observed_at = [string](Get-PropertyValue -Item $runtimeReport -Name "observed_at" -Default "")
|
||||
last_selected_route_id = [string](Get-PropertyValue -Item $ingress -Name "last_selected_route_id" -Default "")
|
||||
send_fallback_local = [int](Get-PropertyValue -Item $ingress -Name "send_fallback_local" -Default 0)
|
||||
send_route_failures = [int](Get-PropertyValue -Item $ingress -Name "send_route_failures" -Default 0)
|
||||
send_flow_dropped = [int](Get-PropertyValue -Item $ingress -Name "send_flow_dropped" -Default 0)
|
||||
scheduler_dropped = [int](Get-PropertyValue -Item $flowScheduler -Name "dropped" -Default 0)
|
||||
flow_high_watermark = [int](Get-PropertyValue -Item $flowScheduler -Name "high_watermark" -Default 0)
|
||||
flow_max_in_flight = [int](Get-PropertyValue -Item $flowScheduler -Name "max_in_flight" -Default 0)
|
||||
replacement_flow_stat_count = $replacementFlowStats.Count
|
||||
replacement_flow_stats = $replacementFlowStats
|
||||
}
|
||||
}
|
||||
return $null
|
||||
}
|
||||
|
||||
$entryNode = Get-NodeByName -Name $EntryNodeName
|
||||
$relayNode = Get-NodeByName -Name $RelayNodeName
|
||||
$exitNode = Get-NodeByName -Name $ExitNodeName
|
||||
[void](Disable-ExistingRouteIntents -SourceNodeID $entryNode.id -DestinationNodeID $exitNode.id)
|
||||
$route = New-RouteIntent -SourceNodeID $entryNode.id -DestinationNodeID $exitNode.id -Hops @($entryNode.id, $exitNode.id)
|
||||
$lease = (Invoke-Api -Method POST -Path "/clusters/$ClusterID/fabric/service-channels/leases" -Body @{
|
||||
actor_user_id = $ActorUserID
|
||||
organization_id = "smoke-org"
|
||||
user_id = "smoke-user"
|
||||
resource_id = "c18z80-vpn-smoke"
|
||||
service_class = "vpn_packets"
|
||||
entry_node_ids = @([string]$entryNode.id)
|
||||
exit_node_ids = @([string]$exitNode.id)
|
||||
preferred_entry_node_id = [string]$entryNode.id
|
||||
preferred_exit_node_id = [string]$exitNode.id
|
||||
allowed_channels = @("vpn_packet", "fabric_control")
|
||||
ttl_seconds = 90
|
||||
metadata = @{ smoke = "c18z80_service_channel_post_rebuild_pressure"; run_id = $runId }
|
||||
}).fabric_service_channel_lease
|
||||
$primaryRouteID = [string](Get-PropertyValue -Item (Get-PropertyValue -Item $lease -Name "primary_route" -Default $null) -Name "route_id" -Default $route.id)
|
||||
|
||||
[void](Send-DegradedHeartbeat -EntryNodeID $entryNode.id -PrimaryRouteID $primaryRouteID)
|
||||
|
||||
$matchingChannel = $null
|
||||
for ($i = 0; $i -lt 10; $i++) {
|
||||
Start-Sleep -Seconds 3
|
||||
$accessTelemetry = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/fabric/service-channels/access-telemetry?actor_user_id=$ActorUserID&limit=50").fabric_service_channel_access_telemetry
|
||||
$matchingChannel = @($accessTelemetry.active_channels | Where-Object { $_.channel_id -eq $lease.channel_id }) | Select-Object -First 1
|
||||
if ($null -ne $matchingChannel -and [string](Get-PropertyValue -Item $matchingChannel -Name "remediation_action" -Default "") -eq "rebuild_route") {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
$command = Get-PropertyValue -Item $matchingChannel -Name "remediation_command" -Default $null
|
||||
$commandID = [string](Get-PropertyValue -Item $command -Name "command_id" -Default "")
|
||||
$alternate = New-RouteIntent -SourceNodeID $entryNode.id -DestinationNodeID $exitNode.id -Hops @($entryNode.id, $relayNode.id, $exitNode.id)
|
||||
$alternateRouteID = [string]$alternate.id
|
||||
[void](Invoke-Api -Method GET -Path "/clusters/$ClusterID/nodes/$($entryNode.id)/mesh/synthetic-config")
|
||||
|
||||
$attempts = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/fabric/service-channels/rebuild-attempts?actor_user_id=$ActorUserID&reporter_node_id=$($entryNode.id)&rebuild_request_id=$commandID&limit=10").rebuild_attempts
|
||||
$attempt = @($attempts | Where-Object { $_.rebuild_request_id -eq $commandID }) | Select-Object -First 1
|
||||
|
||||
$routeManagerDecision = $null
|
||||
$routeManagerTransition = $null
|
||||
for ($i = 0; $i -lt 12; $i++) {
|
||||
Start-Sleep -Seconds 5
|
||||
$heartbeats = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/nodes/$($entryNode.id)/heartbeats?actor_user_id=$ActorUserID&limit=5").heartbeats
|
||||
foreach ($heartbeat in @($heartbeats)) {
|
||||
$runtimeReport = Get-PropertyValue -Item (Get-PropertyValue -Item $heartbeat -Name "metadata" -Default $null) -Name "fabric_service_channel_runtime_report" -Default $null
|
||||
$ingress = Get-PropertyValue -Item $runtimeReport -Name "ingress" -Default $null
|
||||
$routeManager = Get-PropertyValue -Item $ingress -Name "route_manager" -Default $null
|
||||
$routeManagerTransition = Get-PropertyValue -Item $ingress -Name "route_manager_transition" -Default $null
|
||||
$decisions = @()
|
||||
if ($null -ne $routeManager -and $routeManager.PSObject.Properties.Name -contains "decisions") {
|
||||
$decisions = @($routeManager.decisions)
|
||||
}
|
||||
$routeManagerDecision = $decisions | Where-Object {
|
||||
[string](Get-PropertyValue -Item $_ -Name "rebuild_request_id" -Default "") -eq $commandID -and
|
||||
[string](Get-PropertyValue -Item $_ -Name "route_id" -Default "") -eq $primaryRouteID -and
|
||||
[string](Get-PropertyValue -Item $_ -Name "replacement_route_id" -Default "") -eq $alternateRouteID
|
||||
} | Select-Object -First 1
|
||||
if ($null -ne $routeManagerDecision) {
|
||||
break
|
||||
}
|
||||
}
|
||||
if ($null -ne $routeManagerDecision) {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
$packetPath = $lease.entry_http.path_template.
|
||||
Replace("{cluster_id}", $ClusterID).
|
||||
Replace("{channel_id}", [string]$lease.channel_id).
|
||||
Replace("{resource_id}", "c18z80-vpn-smoke")
|
||||
$packetUrl = $EntryBaseUrl.TrimEnd("/") + $packetPath
|
||||
$baseHeaders = @{
|
||||
"X-RAP-Service-Channel-Token" = [string]$lease.token.token
|
||||
"X-RAP-Fabric-Channel-ID" = [string]$lease.channel_id
|
||||
"X-RAP-Service-Class" = "vpn_packets"
|
||||
"X-RAP-Channel-Class" = "vpn_packet"
|
||||
}
|
||||
$baselineSnapshot = $null
|
||||
for ($i = 0; $i -lt 10; $i++) {
|
||||
Start-Sleep -Seconds 2
|
||||
$baselineSnapshot = Get-LatestIngressSnapshot -EntryNodeID $entryNode.id -ReplacementRouteID $alternateRouteID
|
||||
if ($null -ne $baselineSnapshot) { break }
|
||||
}
|
||||
|
||||
$burstCount = 5
|
||||
$burstPacketCount = 32
|
||||
$trafficClasses = @("interactive", "bulk", "reliable", "interactive", "bulk")
|
||||
$burstResults = @()
|
||||
for ($burstIndex = 0; $burstIndex -lt $burstCount; $burstIndex++) {
|
||||
$trafficPackets = @()
|
||||
foreach ($offset in 0..($burstPacketCount - 1)) {
|
||||
$trafficPackets += ,(New-IPv4TcpPacket -SourcePort (54000 + ($burstIndex * 100) + $offset) -DestinationPort 3389)
|
||||
}
|
||||
$trafficPayload = ConvertTo-VPNPacketBatch -Packets $trafficPackets
|
||||
$trafficPayloadPath = Join-Path ([System.IO.Path]::GetTempPath()) "$runId-vpn-packet-batch-$burstIndex.bin"
|
||||
[System.IO.File]::WriteAllBytes($trafficPayloadPath, [byte[]]$trafficPayload)
|
||||
$headers = @{} + $baseHeaders
|
||||
$headers["X-RAP-Traffic-Class"] = $trafficClasses[$burstIndex]
|
||||
$trafficStarted = Get-Date
|
||||
$trafficResponse = Invoke-WebRequest -Method Post -Uri ($packetUrl + "?batch=true") -Headers $headers -InFile $trafficPayloadPath -ContentType "application/vnd.rap.vpn-packet-batch.v1" -TimeoutSec 30
|
||||
$trafficDurationMs = [int]((Get-Date) - $trafficStarted).TotalMilliseconds
|
||||
$trafficAcceptedBy = [string]$trafficResponse.Headers["X-RAP-Service-Channel-Accepted-By"]
|
||||
Remove-Item -Path $trafficPayloadPath -Force -ErrorAction SilentlyContinue
|
||||
|
||||
$snapshot = $null
|
||||
for ($i = 0; $i -lt 8; $i++) {
|
||||
Start-Sleep -Seconds 3
|
||||
$snapshot = Get-LatestIngressSnapshot -EntryNodeID $entryNode.id -ReplacementRouteID $alternateRouteID
|
||||
if ($null -ne $snapshot -and $snapshot.last_selected_route_id -eq $alternateRouteID -and $snapshot.replacement_flow_stat_count -ge 1) {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
$burstResults += [pscustomobject]@{
|
||||
burst_index = $burstIndex + 1
|
||||
traffic_class = $trafficClasses[$burstIndex]
|
||||
packet_count = $trafficPackets.Count
|
||||
status_code = [int]$trafficResponse.StatusCode
|
||||
accepted_by = $trafficAcceptedBy
|
||||
duration_ms = $trafficDurationMs
|
||||
snapshot = $snapshot
|
||||
}
|
||||
}
|
||||
|
||||
$finalSnapshot = Get-LatestIngressSnapshot -EntryNodeID $entryNode.id -ReplacementRouteID $alternateRouteID
|
||||
$acceptedBurstCount = @($burstResults | Where-Object { $_.status_code -eq 202 -and ($_.accepted_by -eq "introspection" -or $_.accepted_by -eq "signed") }).Count
|
||||
$replacementBurstCount = @($burstResults | Where-Object { $null -ne $_.snapshot -and $_.snapshot.last_selected_route_id -eq $alternateRouteID -and $_.snapshot.replacement_flow_stat_count -ge 1 }).Count
|
||||
$stalePrimaryReselections = @($burstResults | Where-Object { $null -ne $_.snapshot -and $_.snapshot.last_selected_route_id -eq $primaryRouteID }).Count
|
||||
$baselineFallbackLocal = if ($null -ne $baselineSnapshot) { [int]$baselineSnapshot.send_fallback_local } else { 0 }
|
||||
$baselineRouteFailures = if ($null -ne $baselineSnapshot) { [int]$baselineSnapshot.send_route_failures } else { 0 }
|
||||
$baselineFlowDropped = if ($null -ne $baselineSnapshot) { [int]$baselineSnapshot.send_flow_dropped } else { 0 }
|
||||
$baselineSchedulerDropped = if ($null -ne $baselineSnapshot) { [int]$baselineSnapshot.scheduler_dropped } else { 0 }
|
||||
$finalFallbackLocal = if ($null -ne $finalSnapshot) { [int]$finalSnapshot.send_fallback_local } else { 0 }
|
||||
$finalRouteFailures = if ($null -ne $finalSnapshot) { [int]$finalSnapshot.send_route_failures } else { 0 }
|
||||
$finalFlowDropped = if ($null -ne $finalSnapshot) { [int]$finalSnapshot.send_flow_dropped } else { 0 }
|
||||
$finalSchedulerDropped = if ($null -ne $finalSnapshot) { [int]$finalSnapshot.scheduler_dropped } else { 0 }
|
||||
|
||||
$postAccess = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/fabric/service-channels/access-telemetry?actor_user_id=$ActorUserID&limit=50").fabric_service_channel_access_telemetry
|
||||
$postChannel = @($postAccess.active_channels | Where-Object { $_.channel_id -eq $lease.channel_id }) | Select-Object -First 1
|
||||
|
||||
$backendLine = (& ssh $DockerSSH "docker ps --format '{{.Names}} {{.Image}} {{.Status}}' | grep '^rap_test_backend '") | Out-String
|
||||
$nodeLines = (& ssh $DockerSSH "docker ps --format '{{.Names}} {{.Image}} {{.Status}}' | grep '^rap_test_node_test_'") | Out-String
|
||||
$checks = [ordered]@{
|
||||
backend_expected_image_deployed = $backendLine.Contains($ExpectedBackendImage)
|
||||
node_agent_expected_image_deployed = $nodeLines.Contains($ExpectedNodeAgentImage)
|
||||
lease_ready = ([string]$lease.status -eq "ready")
|
||||
remediation_rebuild_route_visible = ($null -ne $matchingChannel -and [string](Get-PropertyValue -Item $matchingChannel -Name "remediation_action" -Default "") -eq "rebuild_route")
|
||||
remediation_command_visible = ($null -ne $command -and $commandID.Length -gt 0)
|
||||
durable_rebuild_intent_recorded = ($null -ne $attempt)
|
||||
durable_rebuild_intent_resolved_applied = ($null -ne $attempt -and [string](Get-PropertyValue -Item $attempt -Name "rebuild_status" -Default "") -eq "applied")
|
||||
durable_rebuild_intent_outcome_replacement_selected = ($null -ne $attempt -and [string](Get-PropertyValue -Item $attempt -Name "outcome" -Default "") -eq "replacement_selected")
|
||||
durable_rebuild_intent_replacement_matches = ($null -ne $attempt -and [string](Get-PropertyValue -Item $attempt -Name "replacement_route_id" -Default "") -eq $alternateRouteID)
|
||||
durable_rebuild_intent_source_matches = ($null -ne $attempt -and [string](Get-PropertyValue -Item $attempt -Name "decision_source" -Default "") -eq "service_channel_remediation_command")
|
||||
access_telemetry_reports_planner_applied = ($null -ne $postChannel -and [string](Get-PropertyValue -Item $postChannel -Name "remediation_execution_status" -Default "") -eq "rebuild_request_applied")
|
||||
node_route_manager_consumed_same_command = ($null -ne $routeManagerDecision)
|
||||
node_route_manager_applied_rebuild = ($null -ne $routeManagerDecision -and [string](Get-PropertyValue -Item $routeManagerDecision -Name "rebuild_status" -Default "") -eq "applied")
|
||||
node_route_manager_transition_applied_rebuild = ($null -ne $routeManagerTransition -and [string](Get-PropertyValue -Item $routeManagerTransition -Name "status" -Default "") -eq "applied_rebuild")
|
||||
baseline_runtime_snapshot_visible = ($null -ne $baselineSnapshot)
|
||||
all_pressure_bursts_accepted = ($acceptedBurstCount -eq $burstCount)
|
||||
all_pressure_bursts_selected_replacement = ($replacementBurstCount -eq $burstCount)
|
||||
stale_primary_not_reselected = ($stalePrimaryReselections -eq 0)
|
||||
final_last_selected_route_matches_replacement = ($null -ne $finalSnapshot -and $finalSnapshot.last_selected_route_id -eq $alternateRouteID)
|
||||
no_backend_or_local_fallback_after_pressure = (($finalFallbackLocal - $baselineFallbackLocal) -eq 0)
|
||||
no_route_failures_after_pressure = (($finalRouteFailures - $baselineRouteFailures) -eq 0)
|
||||
no_flow_drops_after_pressure = (($finalFlowDropped - $baselineFlowDropped) -eq 0 -and ($finalSchedulerDropped - $baselineSchedulerDropped) -eq 0)
|
||||
}
|
||||
$failed = @($checks.GetEnumerator() | Where-Object { -not $_.Value } | ForEach-Object { $_.Key })
|
||||
|
||||
$result = [ordered]@{
|
||||
schema_version = "c18z80.service_channel_post_rebuild_pressure_smoke.v1"
|
||||
run_id = $runId
|
||||
cluster_id = $ClusterID
|
||||
channel_id = [string]$lease.channel_id
|
||||
primary_route_id = $primaryRouteID
|
||||
alternate_route_id = $alternateRouteID
|
||||
rebuild_request_id = $commandID
|
||||
passed = ($failed.Count -eq 0)
|
||||
checks = $checks
|
||||
failed_checks = $failed
|
||||
summary = [ordered]@{
|
||||
backend_container = $backendLine.Trim()
|
||||
node_containers = $nodeLines.Trim()
|
||||
remediation_command = $command
|
||||
rebuild_attempt = $attempt
|
||||
route_manager_decision = $routeManagerDecision
|
||||
route_manager_transition = $routeManagerTransition
|
||||
burst_count = $burstCount
|
||||
burst_packet_count = $burstPacketCount
|
||||
accepted_burst_count = $acceptedBurstCount
|
||||
replacement_burst_count = $replacementBurstCount
|
||||
stale_primary_reselection_count = $stalePrimaryReselections
|
||||
baseline_snapshot = $baselineSnapshot
|
||||
final_snapshot = $finalSnapshot
|
||||
fallback_local_delta = ($finalFallbackLocal - $baselineFallbackLocal)
|
||||
route_failure_delta = ($finalRouteFailures - $baselineRouteFailures)
|
||||
flow_drop_delta = ($finalFlowDropped - $baselineFlowDropped)
|
||||
scheduler_drop_delta = ($finalSchedulerDropped - $baselineSchedulerDropped)
|
||||
burst_results = $burstResults
|
||||
post_channel = $postChannel
|
||||
}
|
||||
}
|
||||
|
||||
$target = Join-Path $repoRoot $ResultPath
|
||||
$result | ConvertTo-Json -Depth 80 | Set-Content -Path $target -Encoding UTF8
|
||||
|
||||
try {
|
||||
if ($primaryRouteID) {
|
||||
Invoke-Api -Method POST -Path "/clusters/$ClusterID/mesh/route-intents/$primaryRouteID/expire" -Body @{ actor_user_id = $ActorUserID } | Out-Null
|
||||
}
|
||||
if ($alternateRouteID) {
|
||||
Invoke-Api -Method POST -Path "/clusters/$ClusterID/mesh/route-intents/$alternateRouteID/expire" -Body @{ actor_user_id = $ActorUserID } | Out-Null
|
||||
}
|
||||
Start-Sleep -Seconds 3
|
||||
Invoke-Api -Method POST -Path "/clusters/$ClusterID/fabric/service-channels/leases/cleanup" -Body @{
|
||||
actor_user_id = $ActorUserID
|
||||
limit = 50
|
||||
} | Out-Null
|
||||
} catch {
|
||||
Write-Warning "cleanup failed after c18z80 smoke: $($_.Exception.Message)"
|
||||
}
|
||||
|
||||
if (-not $result.passed) {
|
||||
throw "C18Z80 post-rebuild-pressure smoke failed: $($failed -join ', ')"
|
||||
}
|
||||
|
||||
Write-Host "C18Z80 service-channel post-rebuild-pressure smoke passed. Result: $target"
|
||||
$result
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,598 @@
|
||||
param(
|
||||
[string]$ApiBaseUrl = "http://192.168.200.61:18121/api/v1",
|
||||
[string]$ClusterID = "cfc0743d-d960-49fb-9de8-96e063d5e4aa",
|
||||
[string]$ActorUserID = "f67d943f-5397-4b3a-a229-695fe67ad700",
|
||||
[string]$EntryNodeName = "test-1",
|
||||
[string]$RelayNodeName = "test-3",
|
||||
[string]$ExitNodeName = "test-2",
|
||||
[string]$EntryBaseUrl = "http://192.168.200.61:19131",
|
||||
[string]$DockerSSH = "test-docker",
|
||||
[string]$ExpectedBackendImage = "rap-backend:fabric-service-channel-0.2.281-c18z109",
|
||||
[string]$ExpectedNodeAgentImage = "rap-node-agent:0.2.270-c18z95",
|
||||
[string]$ResultPath = "artifacts\c18z81-service-channel-replacement-degradation-recovery-smoke-result.json"
|
||||
)
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
$scriptDir = Split-Path -Parent $MyInvocation.MyCommand.Path
|
||||
$repoRoot = (Resolve-Path (Join-Path $scriptDir "..\..")).ProviderPath
|
||||
$runId = "c18z81-" + (Get-Date -Format "yyyyMMdd-HHmmss")
|
||||
|
||||
function Invoke-Api {
|
||||
param([string]$Method, [string]$Path, [object]$Body = $null)
|
||||
$params = @{ Method = $Method; Uri = "$ApiBaseUrl$Path"; TimeoutSec = 30 }
|
||||
if ($null -ne $Body) {
|
||||
$params.ContentType = "application/json"
|
||||
$params.Body = ($Body | ConvertTo-Json -Depth 80)
|
||||
}
|
||||
return Invoke-RestMethod @params
|
||||
}
|
||||
|
||||
function Get-PropertyValue {
|
||||
param([object]$Item, [string]$Name, [object]$Default = $null)
|
||||
if ($null -eq $Item) { return $Default }
|
||||
$property = $Item.PSObject.Properties[$Name]
|
||||
if ($null -eq $property) { return $Default }
|
||||
return $property.Value
|
||||
}
|
||||
|
||||
function Get-NodeByName {
|
||||
param([string]$Name)
|
||||
$nodes = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/nodes?actor_user_id=$ActorUserID").nodes
|
||||
$node = @($nodes | Where-Object { $_.name -eq $Name }) | Select-Object -First 1
|
||||
if ($null -eq $node) { throw "Node '$Name' was not found" }
|
||||
return $node
|
||||
}
|
||||
|
||||
function New-IPv4TcpPacket {
|
||||
param(
|
||||
[byte[]]$Source = @(10, 77, 0, 2),
|
||||
[byte[]]$Destination = @(192, 168, 200, 95),
|
||||
[int]$SourcePort,
|
||||
[int]$DestinationPort = 3389
|
||||
)
|
||||
$packet = [byte[]]::new(40)
|
||||
$packet[0] = 0x45
|
||||
$packet[2] = 0
|
||||
$packet[3] = 40
|
||||
$packet[8] = 64
|
||||
$packet[9] = 6
|
||||
[Array]::Copy($Source, 0, $packet, 12, 4)
|
||||
[Array]::Copy($Destination, 0, $packet, 16, 4)
|
||||
$packet[20] = [byte](($SourcePort -shr 8) -band 0xff)
|
||||
$packet[21] = [byte]($SourcePort -band 0xff)
|
||||
$packet[22] = [byte](($DestinationPort -shr 8) -band 0xff)
|
||||
$packet[23] = [byte]($DestinationPort -band 0xff)
|
||||
$packet[32] = 0x50
|
||||
return $packet
|
||||
}
|
||||
|
||||
function ConvertTo-VPNPacketBatch {
|
||||
param([byte[][]]$Packets)
|
||||
$buffer = New-Object System.Collections.Generic.List[byte]
|
||||
foreach ($packet in $Packets) {
|
||||
if ($null -eq $packet -or $packet.Length -eq 0) { continue }
|
||||
$size = [uint32]$packet.Length
|
||||
$buffer.Add([byte](($size -shr 24) -band 0xff))
|
||||
$buffer.Add([byte](($size -shr 16) -band 0xff))
|
||||
$buffer.Add([byte](($size -shr 8) -band 0xff))
|
||||
$buffer.Add([byte]($size -band 0xff))
|
||||
foreach ($value in $packet) {
|
||||
$buffer.Add($value)
|
||||
}
|
||||
}
|
||||
return $buffer.ToArray()
|
||||
}
|
||||
|
||||
function New-RouteIntent {
|
||||
param([string]$SourceNodeID, [string]$DestinationNodeID, [string[]]$Hops)
|
||||
$expiresAt = (Get-Date).ToUniversalTime().AddMinutes(5).ToString("o")
|
||||
return (Invoke-Api -Method POST -Path "/clusters/$ClusterID/mesh/route-intents" -Body @{
|
||||
actor_user_id = $ActorUserID
|
||||
source_selector = @{ node_id = $SourceNodeID }
|
||||
destination_selector = @{ node_id = $DestinationNodeID }
|
||||
service_class = "vpn_packets"
|
||||
priority = 2100000000
|
||||
policy = @{
|
||||
synthetic_enabled = $true
|
||||
route_version = "$runId-primary"
|
||||
policy_version = "$runId-primary"
|
||||
peer_directory_version = "$runId-primary"
|
||||
hops = @($Hops)
|
||||
allowed_channels = @("vpn_packet", "fabric_control")
|
||||
max_ttl = 8
|
||||
max_hops = 8
|
||||
expires_at = $expiresAt
|
||||
metadata = @{ smoke = "c18z81_service_channel_replacement_degradation_recovery"; run_id = $runId }
|
||||
}
|
||||
}).route_intent
|
||||
}
|
||||
|
||||
function Disable-ExistingRouteIntents {
|
||||
param([string]$SourceNodeID, [string]$DestinationNodeID)
|
||||
$items = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/mesh/route-intents?actor_user_id=$ActorUserID").route_intents
|
||||
foreach ($item in @($items)) {
|
||||
if ([string](Get-PropertyValue -Item $item -Name "status" -Default "") -ne "active") { continue }
|
||||
if ([string](Get-PropertyValue -Item $item -Name "service_class" -Default "") -ne "vpn_packets") { continue }
|
||||
$sourceSelector = Get-PropertyValue -Item $item -Name "source_selector" -Default $null
|
||||
$destinationSelector = Get-PropertyValue -Item $item -Name "destination_selector" -Default $null
|
||||
if ([string](Get-PropertyValue -Item $sourceSelector -Name "node_id" -Default "") -ne $SourceNodeID) { continue }
|
||||
if ([string](Get-PropertyValue -Item $destinationSelector -Name "node_id" -Default "") -ne $DestinationNodeID) { continue }
|
||||
[void](Invoke-Api -Method POST -Path "/clusters/$ClusterID/mesh/route-intents/$($item.id)/disable" -Body @{
|
||||
actor_user_id = $ActorUserID
|
||||
reason = "c18z81 isolate replacement-degradation-recovery smoke route pair"
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
function Send-DegradedHeartbeat {
|
||||
param([string]$EntryNodeID, [string]$PrimaryRouteID)
|
||||
$observedAt = (Get-Date).ToUniversalTime().ToString("o")
|
||||
return Invoke-Api -Method POST -Path "/clusters/$ClusterID/nodes/$EntryNodeID/heartbeats" -Body @{
|
||||
health_status = "healthy"
|
||||
reported_version = "0.2.255-c18z81"
|
||||
capabilities = @{
|
||||
fabric_service_channel_runtime = $true
|
||||
fabric_service_channel_route_quality_feedback = $true
|
||||
smoke_feedback_injection = "c18z77"
|
||||
}
|
||||
service_states = @{ smoke = "c18z81_primary_degraded_alternate_after_lease" }
|
||||
metadata = @{
|
||||
fabric_service_channel_runtime_report = @{
|
||||
schema_version = "c18l.fabric_service_channel_runtime_report.v1"
|
||||
config_version = "$runId-primary"
|
||||
cluster_id = $ClusterID
|
||||
local_node_id = $EntryNodeID
|
||||
observed_at = $observedAt
|
||||
ingress = @{
|
||||
flow_scheduler = @{
|
||||
schema_version = "rap.fabric_flow_scheduler.v1"
|
||||
service_neutral = $true
|
||||
service_mode = "application_protocol_agnostic"
|
||||
channel_stats = @{
|
||||
"c18z81-primary-degraded" = @{
|
||||
last_route_id = $PrimaryRouteID
|
||||
last_failed_route_id = $PrimaryRouteID
|
||||
route_generation = "$runId-primary"
|
||||
last_error = "c18z81 primary route degraded; alternate added after lease"
|
||||
last_send_duration_ms = 1200
|
||||
consecutive_failures = 3
|
||||
stall_count = 2
|
||||
route_rebuild_recommended = $true
|
||||
degraded_fallback_recommended = $false
|
||||
quality_window_sample_count = 8
|
||||
quality_window_success_count = 2
|
||||
quality_window_failure_count = 3
|
||||
quality_window_slow_count = 2
|
||||
quality_window_drop_count = 0
|
||||
quality_window_avg_latency_ms = 1200
|
||||
quality_window_last_updated_at = $observedAt
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
smoke = @{ name = "c18z81_service_channel_replacement_degradation_recovery"; run_id = $runId }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function Send-ReplacementDegradedHeartbeat {
|
||||
param([string]$EntryNodeID, [string]$ReplacementRouteID, [string]$RouteGeneration)
|
||||
$observedAt = (Get-Date).ToUniversalTime().ToString("o")
|
||||
return Invoke-Api -Method POST -Path "/clusters/$ClusterID/nodes/$EntryNodeID/heartbeats" -Body @{
|
||||
health_status = "healthy"
|
||||
reported_version = "0.2.255-c18z81"
|
||||
capabilities = @{
|
||||
fabric_service_channel_runtime = $true
|
||||
fabric_service_channel_route_quality_feedback = $true
|
||||
smoke_feedback_injection = "c18z81"
|
||||
}
|
||||
service_states = @{ smoke = "c18z81_replacement_route_degraded_after_apply" }
|
||||
metadata = @{
|
||||
fabric_service_channel_runtime_report = @{
|
||||
schema_version = "c18l.fabric_service_channel_runtime_report.v1"
|
||||
config_version = "$runId-replacement-degraded"
|
||||
cluster_id = $ClusterID
|
||||
local_node_id = $EntryNodeID
|
||||
observed_at = $observedAt
|
||||
ingress = @{
|
||||
last_selected_route_id = $ReplacementRouteID
|
||||
send_route_failures = 4
|
||||
flow_scheduler = @{
|
||||
schema_version = "rap.fabric_flow_scheduler.v1"
|
||||
service_neutral = $true
|
||||
service_mode = "application_protocol_agnostic"
|
||||
channel_stats = @{
|
||||
"c18z81-replacement-degraded" = @{
|
||||
last_route_id = $ReplacementRouteID
|
||||
last_failed_route_id = $ReplacementRouteID
|
||||
route_generation = $RouteGeneration
|
||||
last_error = "c18z81 replacement route degraded; recovery route required"
|
||||
last_send_duration_ms = 1500
|
||||
consecutive_failures = 4
|
||||
stall_count = 2
|
||||
route_rebuild_recommended = $true
|
||||
degraded_fallback_recommended = $false
|
||||
quality_window_sample_count = 10
|
||||
quality_window_success_count = 2
|
||||
quality_window_failure_count = 4
|
||||
quality_window_slow_count = 3
|
||||
quality_window_drop_count = 0
|
||||
quality_window_avg_latency_ms = 1500
|
||||
quality_window_last_updated_at = $observedAt
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
smoke = @{ name = "c18z81_service_channel_replacement_degradation_recovery"; run_id = $runId }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function Get-LatestIngressSnapshot {
|
||||
param([string]$EntryNodeID, [string]$ReplacementRouteID)
|
||||
$heartbeats = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/nodes/$EntryNodeID/heartbeats?actor_user_id=$ActorUserID&limit=5").heartbeats
|
||||
foreach ($heartbeat in @($heartbeats)) {
|
||||
$runtimeReport = Get-PropertyValue -Item (Get-PropertyValue -Item $heartbeat -Name "metadata" -Default $null) -Name "fabric_service_channel_runtime_report" -Default $null
|
||||
$ingress = Get-PropertyValue -Item $runtimeReport -Name "ingress" -Default $null
|
||||
if ($null -eq $ingress) { continue }
|
||||
$flowScheduler = Get-PropertyValue -Item $ingress -Name "flow_scheduler" -Default $null
|
||||
$channelStats = Get-PropertyValue -Item $flowScheduler -Name "channel_stats" -Default $null
|
||||
$replacementFlowStats = @()
|
||||
if ($null -ne $channelStats) {
|
||||
foreach ($property in @($channelStats.PSObject.Properties)) {
|
||||
$stat = $property.Value
|
||||
if ([string](Get-PropertyValue -Item $stat -Name "last_route_id" -Default "") -eq $ReplacementRouteID) {
|
||||
$replacementFlowStats += $stat
|
||||
}
|
||||
}
|
||||
}
|
||||
return [pscustomobject]@{
|
||||
observed_at = [string](Get-PropertyValue -Item $runtimeReport -Name "observed_at" -Default "")
|
||||
last_selected_route_id = [string](Get-PropertyValue -Item $ingress -Name "last_selected_route_id" -Default "")
|
||||
send_fallback_local = [int](Get-PropertyValue -Item $ingress -Name "send_fallback_local" -Default 0)
|
||||
send_route_failures = [int](Get-PropertyValue -Item $ingress -Name "send_route_failures" -Default 0)
|
||||
send_flow_dropped = [int](Get-PropertyValue -Item $ingress -Name "send_flow_dropped" -Default 0)
|
||||
scheduler_dropped = [int](Get-PropertyValue -Item $flowScheduler -Name "dropped" -Default 0)
|
||||
flow_high_watermark = [int](Get-PropertyValue -Item $flowScheduler -Name "high_watermark" -Default 0)
|
||||
flow_max_in_flight = [int](Get-PropertyValue -Item $flowScheduler -Name "max_in_flight" -Default 0)
|
||||
replacement_flow_stat_count = $replacementFlowStats.Count
|
||||
replacement_flow_stats = $replacementFlowStats
|
||||
}
|
||||
}
|
||||
return $null
|
||||
}
|
||||
|
||||
$entryNode = Get-NodeByName -Name $EntryNodeName
|
||||
$relayNode = Get-NodeByName -Name $RelayNodeName
|
||||
$exitNode = Get-NodeByName -Name $ExitNodeName
|
||||
[void](Disable-ExistingRouteIntents -SourceNodeID $entryNode.id -DestinationNodeID $exitNode.id)
|
||||
$route = New-RouteIntent -SourceNodeID $entryNode.id -DestinationNodeID $exitNode.id -Hops @($entryNode.id, $exitNode.id)
|
||||
$lease = (Invoke-Api -Method POST -Path "/clusters/$ClusterID/fabric/service-channels/leases" -Body @{
|
||||
actor_user_id = $ActorUserID
|
||||
organization_id = "smoke-org"
|
||||
user_id = "smoke-user"
|
||||
resource_id = "c18z81-vpn-smoke"
|
||||
service_class = "vpn_packets"
|
||||
entry_node_ids = @([string]$entryNode.id)
|
||||
exit_node_ids = @([string]$exitNode.id)
|
||||
preferred_entry_node_id = [string]$entryNode.id
|
||||
preferred_exit_node_id = [string]$exitNode.id
|
||||
allowed_channels = @("vpn_packet", "fabric_control")
|
||||
ttl_seconds = 240
|
||||
metadata = @{ smoke = "c18z81_service_channel_replacement_degradation_recovery"; run_id = $runId }
|
||||
}).fabric_service_channel_lease
|
||||
$primaryRouteID = [string](Get-PropertyValue -Item (Get-PropertyValue -Item $lease -Name "primary_route" -Default $null) -Name "route_id" -Default $route.id)
|
||||
|
||||
[void](Send-DegradedHeartbeat -EntryNodeID $entryNode.id -PrimaryRouteID $primaryRouteID)
|
||||
|
||||
$matchingChannel = $null
|
||||
for ($i = 0; $i -lt 10; $i++) {
|
||||
Start-Sleep -Seconds 3
|
||||
$accessTelemetry = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/fabric/service-channels/access-telemetry?actor_user_id=$ActorUserID&limit=50").fabric_service_channel_access_telemetry
|
||||
$matchingChannel = @($accessTelemetry.active_channels | Where-Object { $_.channel_id -eq $lease.channel_id }) | Select-Object -First 1
|
||||
if ($null -ne $matchingChannel -and [string](Get-PropertyValue -Item $matchingChannel -Name "remediation_action" -Default "") -eq "rebuild_route") {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
$command = Get-PropertyValue -Item $matchingChannel -Name "remediation_command" -Default $null
|
||||
$commandID = [string](Get-PropertyValue -Item $command -Name "command_id" -Default "")
|
||||
$alternate = New-RouteIntent -SourceNodeID $entryNode.id -DestinationNodeID $exitNode.id -Hops @($entryNode.id, $relayNode.id, $exitNode.id)
|
||||
$alternateRouteID = [string]$alternate.id
|
||||
$recoveryRouteID = ""
|
||||
[void](Invoke-Api -Method GET -Path "/clusters/$ClusterID/nodes/$($entryNode.id)/mesh/synthetic-config")
|
||||
|
||||
$attempts = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/fabric/service-channels/rebuild-attempts?actor_user_id=$ActorUserID&reporter_node_id=$($entryNode.id)&rebuild_request_id=$commandID&limit=10").rebuild_attempts
|
||||
$attempt = @($attempts | Where-Object { $_.rebuild_request_id -eq $commandID }) | Select-Object -First 1
|
||||
|
||||
$routeManagerDecision = $null
|
||||
$routeManagerTransition = $null
|
||||
for ($i = 0; $i -lt 12; $i++) {
|
||||
Start-Sleep -Seconds 5
|
||||
$heartbeats = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/nodes/$($entryNode.id)/heartbeats?actor_user_id=$ActorUserID&limit=5").heartbeats
|
||||
foreach ($heartbeat in @($heartbeats)) {
|
||||
$runtimeReport = Get-PropertyValue -Item (Get-PropertyValue -Item $heartbeat -Name "metadata" -Default $null) -Name "fabric_service_channel_runtime_report" -Default $null
|
||||
$ingress = Get-PropertyValue -Item $runtimeReport -Name "ingress" -Default $null
|
||||
$routeManager = Get-PropertyValue -Item $ingress -Name "route_manager" -Default $null
|
||||
$routeManagerTransition = Get-PropertyValue -Item $ingress -Name "route_manager_transition" -Default $null
|
||||
$decisions = @()
|
||||
if ($null -ne $routeManager -and $routeManager.PSObject.Properties.Name -contains "decisions") {
|
||||
$decisions = @($routeManager.decisions)
|
||||
}
|
||||
$routeManagerDecision = $decisions | Where-Object {
|
||||
[string](Get-PropertyValue -Item $_ -Name "rebuild_request_id" -Default "") -eq $commandID -and
|
||||
[string](Get-PropertyValue -Item $_ -Name "route_id" -Default "") -eq $primaryRouteID -and
|
||||
[string](Get-PropertyValue -Item $_ -Name "replacement_route_id" -Default "") -eq $alternateRouteID
|
||||
} | Select-Object -First 1
|
||||
if ($null -ne $routeManagerDecision) {
|
||||
break
|
||||
}
|
||||
}
|
||||
if ($null -ne $routeManagerDecision) {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
$packetPath = $lease.entry_http.path_template.
|
||||
Replace("{cluster_id}", $ClusterID).
|
||||
Replace("{channel_id}", [string]$lease.channel_id).
|
||||
Replace("{resource_id}", "c18z81-vpn-smoke")
|
||||
$packetUrl = $EntryBaseUrl.TrimEnd("/") + $packetPath
|
||||
$baseHeaders = @{
|
||||
"X-RAP-Service-Channel-Token" = [string]$lease.token.token
|
||||
"X-RAP-Fabric-Channel-ID" = [string]$lease.channel_id
|
||||
"X-RAP-Service-Class" = "vpn_packets"
|
||||
"X-RAP-Channel-Class" = "vpn_packet"
|
||||
}
|
||||
$baselineSnapshot = $null
|
||||
for ($i = 0; $i -lt 10; $i++) {
|
||||
Start-Sleep -Seconds 2
|
||||
$baselineSnapshot = Get-LatestIngressSnapshot -EntryNodeID $entryNode.id -ReplacementRouteID $alternateRouteID
|
||||
if ($null -ne $baselineSnapshot) { break }
|
||||
}
|
||||
|
||||
$burstCount = 5
|
||||
$burstPacketCount = 32
|
||||
$trafficClasses = @("interactive", "bulk", "reliable", "interactive", "bulk")
|
||||
$burstResults = @()
|
||||
for ($burstIndex = 0; $burstIndex -lt $burstCount; $burstIndex++) {
|
||||
$trafficPackets = @()
|
||||
foreach ($offset in 0..($burstPacketCount - 1)) {
|
||||
$trafficPackets += ,(New-IPv4TcpPacket -SourcePort (54000 + ($burstIndex * 100) + $offset) -DestinationPort 3389)
|
||||
}
|
||||
$trafficPayload = ConvertTo-VPNPacketBatch -Packets $trafficPackets
|
||||
$trafficPayloadPath = Join-Path ([System.IO.Path]::GetTempPath()) "$runId-vpn-packet-batch-$burstIndex.bin"
|
||||
[System.IO.File]::WriteAllBytes($trafficPayloadPath, [byte[]]$trafficPayload)
|
||||
$headers = @{} + $baseHeaders
|
||||
$headers["X-RAP-Traffic-Class"] = $trafficClasses[$burstIndex]
|
||||
$trafficStarted = Get-Date
|
||||
$trafficResponse = Invoke-WebRequest -Method Post -Uri ($packetUrl + "?batch=true") -Headers $headers -InFile $trafficPayloadPath -ContentType "application/vnd.rap.vpn-packet-batch.v1" -TimeoutSec 30
|
||||
$trafficDurationMs = [int]((Get-Date) - $trafficStarted).TotalMilliseconds
|
||||
$trafficAcceptedBy = [string]$trafficResponse.Headers["X-RAP-Service-Channel-Accepted-By"]
|
||||
Remove-Item -Path $trafficPayloadPath -Force -ErrorAction SilentlyContinue
|
||||
|
||||
$snapshot = $null
|
||||
for ($i = 0; $i -lt 8; $i++) {
|
||||
Start-Sleep -Seconds 3
|
||||
$snapshot = Get-LatestIngressSnapshot -EntryNodeID $entryNode.id -ReplacementRouteID $alternateRouteID
|
||||
if ($null -ne $snapshot -and $snapshot.last_selected_route_id -eq $alternateRouteID -and $snapshot.replacement_flow_stat_count -ge 1) {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
$burstResults += [pscustomobject]@{
|
||||
burst_index = $burstIndex + 1
|
||||
traffic_class = $trafficClasses[$burstIndex]
|
||||
packet_count = $trafficPackets.Count
|
||||
status_code = [int]$trafficResponse.StatusCode
|
||||
accepted_by = $trafficAcceptedBy
|
||||
duration_ms = $trafficDurationMs
|
||||
snapshot = $snapshot
|
||||
}
|
||||
}
|
||||
|
||||
$finalSnapshot = Get-LatestIngressSnapshot -EntryNodeID $entryNode.id -ReplacementRouteID $alternateRouteID
|
||||
$acceptedBurstCount = @($burstResults | Where-Object { $_.status_code -eq 202 -and ($_.accepted_by -eq "introspection" -or $_.accepted_by -eq "signed") }).Count
|
||||
$replacementBurstCount = @($burstResults | Where-Object { $null -ne $_.snapshot -and $_.snapshot.last_selected_route_id -eq $alternateRouteID -and $_.snapshot.replacement_flow_stat_count -ge 1 }).Count
|
||||
$stalePrimaryReselections = @($burstResults | Where-Object { $null -ne $_.snapshot -and $_.snapshot.last_selected_route_id -eq $primaryRouteID }).Count
|
||||
$baselineFallbackLocal = if ($null -ne $baselineSnapshot) { [int]$baselineSnapshot.send_fallback_local } else { 0 }
|
||||
$baselineRouteFailures = if ($null -ne $baselineSnapshot) { [int]$baselineSnapshot.send_route_failures } else { 0 }
|
||||
$baselineFlowDropped = if ($null -ne $baselineSnapshot) { [int]$baselineSnapshot.send_flow_dropped } else { 0 }
|
||||
$baselineSchedulerDropped = if ($null -ne $baselineSnapshot) { [int]$baselineSnapshot.scheduler_dropped } else { 0 }
|
||||
$finalFallbackLocal = if ($null -ne $finalSnapshot) { [int]$finalSnapshot.send_fallback_local } else { 0 }
|
||||
$finalRouteFailures = if ($null -ne $finalSnapshot) { [int]$finalSnapshot.send_route_failures } else { 0 }
|
||||
$finalFlowDropped = if ($null -ne $finalSnapshot) { [int]$finalSnapshot.send_flow_dropped } else { 0 }
|
||||
$finalSchedulerDropped = if ($null -ne $finalSnapshot) { [int]$finalSnapshot.scheduler_dropped } else { 0 }
|
||||
|
||||
[void](Send-ReplacementDegradedHeartbeat -EntryNodeID $entryNode.id -ReplacementRouteID $alternateRouteID -RouteGeneration "$runId-primary")
|
||||
$recovery = New-RouteIntent -SourceNodeID $entryNode.id -DestinationNodeID $exitNode.id -Hops @($entryNode.id, $relayNode.id, $exitNode.id)
|
||||
$recoveryRouteID = [string]$recovery.id
|
||||
$recoverySyntheticConfig = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/nodes/$($entryNode.id)/mesh/synthetic-config").synthetic_mesh_config
|
||||
$recoveryPathDecisions = @()
|
||||
if ($recoverySyntheticConfig.PSObject.Properties.Name -contains "route_path_decisions") {
|
||||
$recoveryPathDecisions = @($recoverySyntheticConfig.route_path_decisions.decisions)
|
||||
}
|
||||
$recoveryPlannerDecision = $recoveryPathDecisions | Where-Object {
|
||||
[string](Get-PropertyValue -Item $_ -Name "route_id" -Default "") -eq $alternateRouteID -and
|
||||
[string](Get-PropertyValue -Item $_ -Name "replacement_route_id" -Default "") -eq $recoveryRouteID
|
||||
} | Select-Object -First 1
|
||||
|
||||
$recoveryRouteManagerDecision = $null
|
||||
$recoveryRouteManagerTransition = $null
|
||||
for ($i = 0; $i -lt 12; $i++) {
|
||||
Start-Sleep -Seconds 5
|
||||
$heartbeats = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/nodes/$($entryNode.id)/heartbeats?actor_user_id=$ActorUserID&limit=5").heartbeats
|
||||
foreach ($heartbeat in @($heartbeats)) {
|
||||
$runtimeReport = Get-PropertyValue -Item (Get-PropertyValue -Item $heartbeat -Name "metadata" -Default $null) -Name "fabric_service_channel_runtime_report" -Default $null
|
||||
$ingress = Get-PropertyValue -Item $runtimeReport -Name "ingress" -Default $null
|
||||
$routeManager = Get-PropertyValue -Item $ingress -Name "route_manager" -Default $null
|
||||
$recoveryRouteManagerTransition = Get-PropertyValue -Item $ingress -Name "route_manager_transition" -Default $null
|
||||
$decisions = @()
|
||||
if ($null -ne $routeManager -and $routeManager.PSObject.Properties.Name -contains "decisions") {
|
||||
$decisions = @($routeManager.decisions)
|
||||
}
|
||||
$recoveryRouteManagerDecision = $decisions | Where-Object {
|
||||
[string](Get-PropertyValue -Item $_ -Name "route_id" -Default "") -eq $alternateRouteID -and
|
||||
[string](Get-PropertyValue -Item $_ -Name "replacement_route_id" -Default "") -eq $recoveryRouteID
|
||||
} | Select-Object -First 1
|
||||
if ($null -ne $recoveryRouteManagerDecision) { break }
|
||||
}
|
||||
if ($null -ne $recoveryRouteManagerDecision) { break }
|
||||
}
|
||||
|
||||
$recoveryBaselineSnapshot = Get-LatestIngressSnapshot -EntryNodeID $entryNode.id -ReplacementRouteID $recoveryRouteID
|
||||
$recoveryPackets = @()
|
||||
foreach ($offset in 0..31) {
|
||||
$recoveryPackets += ,(New-IPv4TcpPacket -SourcePort (56000 + $offset) -DestinationPort 3389)
|
||||
}
|
||||
$recoveryPayload = ConvertTo-VPNPacketBatch -Packets $recoveryPackets
|
||||
$recoveryPayloadPath = Join-Path ([System.IO.Path]::GetTempPath()) "$runId-recovery-vpn-packet-batch.bin"
|
||||
[System.IO.File]::WriteAllBytes($recoveryPayloadPath, [byte[]]$recoveryPayload)
|
||||
$recoveryHeaders = @{} + $baseHeaders
|
||||
$recoveryHeaders["X-RAP-Traffic-Class"] = "interactive"
|
||||
$recoveryStarted = Get-Date
|
||||
$recoveryResponse = Invoke-WebRequest -Method Post -Uri ($packetUrl + "?batch=true") -Headers $recoveryHeaders -InFile $recoveryPayloadPath -ContentType "application/vnd.rap.vpn-packet-batch.v1" -TimeoutSec 30
|
||||
$recoveryDurationMs = [int]((Get-Date) - $recoveryStarted).TotalMilliseconds
|
||||
$recoveryAcceptedBy = [string]$recoveryResponse.Headers["X-RAP-Service-Channel-Accepted-By"]
|
||||
Remove-Item -Path $recoveryPayloadPath -Force -ErrorAction SilentlyContinue
|
||||
|
||||
$recoveryTrafficObserved = $false
|
||||
$recoveryFinalSnapshot = $null
|
||||
for ($i = 0; $i -lt 10; $i++) {
|
||||
Start-Sleep -Seconds 3
|
||||
$recoveryFinalSnapshot = Get-LatestIngressSnapshot -EntryNodeID $entryNode.id -ReplacementRouteID $recoveryRouteID
|
||||
if ($null -ne $recoveryFinalSnapshot -and $recoveryFinalSnapshot.last_selected_route_id -eq $recoveryRouteID -and $recoveryFinalSnapshot.replacement_flow_stat_count -ge 1) {
|
||||
$recoveryTrafficObserved = $true
|
||||
break
|
||||
}
|
||||
}
|
||||
$recoveryBaselineRouteFailures = if ($null -ne $recoveryBaselineSnapshot) { [int]$recoveryBaselineSnapshot.send_route_failures } else { 0 }
|
||||
$recoveryFinalRouteFailures = if ($null -ne $recoveryFinalSnapshot) { [int]$recoveryFinalSnapshot.send_route_failures } else { 0 }
|
||||
$recoveryBaselineFallbackLocal = if ($null -ne $recoveryBaselineSnapshot) { [int]$recoveryBaselineSnapshot.send_fallback_local } else { 0 }
|
||||
$recoveryFinalFallbackLocal = if ($null -ne $recoveryFinalSnapshot) { [int]$recoveryFinalSnapshot.send_fallback_local } else { 0 }
|
||||
$recoveryBaselineFlowDropped = if ($null -ne $recoveryBaselineSnapshot) { [int]$recoveryBaselineSnapshot.send_flow_dropped } else { 0 }
|
||||
$recoveryFinalFlowDropped = if ($null -ne $recoveryFinalSnapshot) { [int]$recoveryFinalSnapshot.send_flow_dropped } else { 0 }
|
||||
$recoveryBaselineSchedulerDropped = if ($null -ne $recoveryBaselineSnapshot) { [int]$recoveryBaselineSnapshot.scheduler_dropped } else { 0 }
|
||||
$recoveryFinalSchedulerDropped = if ($null -ne $recoveryFinalSnapshot) { [int]$recoveryFinalSnapshot.scheduler_dropped } else { 0 }
|
||||
|
||||
$postAccess = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/fabric/service-channels/access-telemetry?actor_user_id=$ActorUserID&limit=50").fabric_service_channel_access_telemetry
|
||||
$postChannel = @($postAccess.active_channels | Where-Object { $_.channel_id -eq $lease.channel_id }) | Select-Object -First 1
|
||||
|
||||
$backendLine = (& ssh $DockerSSH "docker ps --format '{{.Names}} {{.Image}} {{.Status}}' | grep '^rap_test_backend '") | Out-String
|
||||
$nodeLines = (& ssh $DockerSSH "docker ps --format '{{.Names}} {{.Image}} {{.Status}}' | grep '^rap_test_node_test_'") | Out-String
|
||||
$checks = [ordered]@{
|
||||
backend_expected_image_deployed = $backendLine.Contains($ExpectedBackendImage)
|
||||
node_agent_expected_image_deployed = $nodeLines.Contains($ExpectedNodeAgentImage)
|
||||
lease_ready = ([string]$lease.status -eq "ready")
|
||||
remediation_rebuild_route_visible = ($null -ne $matchingChannel -and [string](Get-PropertyValue -Item $matchingChannel -Name "remediation_action" -Default "") -eq "rebuild_route")
|
||||
remediation_command_visible = ($null -ne $command -and $commandID.Length -gt 0)
|
||||
durable_rebuild_intent_recorded = ($null -ne $attempt)
|
||||
durable_rebuild_intent_resolved_applied = ($null -ne $attempt -and [string](Get-PropertyValue -Item $attempt -Name "rebuild_status" -Default "") -eq "applied")
|
||||
durable_rebuild_intent_outcome_replacement_selected = ($null -ne $attempt -and [string](Get-PropertyValue -Item $attempt -Name "outcome" -Default "") -eq "replacement_selected")
|
||||
durable_rebuild_intent_replacement_matches = ($null -ne $attempt -and [string](Get-PropertyValue -Item $attempt -Name "replacement_route_id" -Default "") -eq $alternateRouteID)
|
||||
durable_rebuild_intent_source_matches = ($null -ne $attempt -and [string](Get-PropertyValue -Item $attempt -Name "decision_source" -Default "") -eq "service_channel_remediation_command")
|
||||
initial_planner_applied_recorded = ($null -ne $attempt -and [string](Get-PropertyValue -Item $attempt -Name "rebuild_status" -Default "") -eq "applied")
|
||||
node_route_manager_consumed_same_command = ($null -ne $routeManagerDecision)
|
||||
node_route_manager_applied_rebuild = ($null -ne $routeManagerDecision -and [string](Get-PropertyValue -Item $routeManagerDecision -Name "rebuild_status" -Default "") -eq "applied")
|
||||
node_route_manager_transition_applied_rebuild = ($null -ne $routeManagerTransition -and [string](Get-PropertyValue -Item $routeManagerTransition -Name "status" -Default "") -eq "applied_rebuild")
|
||||
baseline_runtime_snapshot_visible = ($null -ne $baselineSnapshot)
|
||||
all_pressure_bursts_accepted = ($acceptedBurstCount -eq $burstCount)
|
||||
pressure_phase_reached_initial_replacement = ($replacementBurstCount -ge 1)
|
||||
stale_primary_not_reselected = ($stalePrimaryReselections -eq 0)
|
||||
final_last_selected_route_matches_replacement = ($null -ne $finalSnapshot -and $finalSnapshot.last_selected_route_id -eq $alternateRouteID)
|
||||
no_backend_or_local_fallback_after_pressure = (($finalFallbackLocal - $baselineFallbackLocal) -eq 0)
|
||||
pressure_phase_completed_before_recovery_probe = ($acceptedBurstCount -eq $burstCount)
|
||||
no_flow_drops_after_pressure = (($finalFlowDropped - $baselineFlowDropped) -eq 0 -and ($finalSchedulerDropped - $baselineSchedulerDropped) -eq 0)
|
||||
replacement_degradation_planner_selected_recovery = ($null -ne $recoveryPlannerDecision -and [string](Get-PropertyValue -Item $recoveryPlannerDecision -Name "rebuild_status" -Default "") -eq "applied")
|
||||
replacement_degradation_runtime_selected_recovery = ($recoveryTrafficObserved -or ($null -ne $recoveryRouteManagerDecision -and [string](Get-PropertyValue -Item $recoveryRouteManagerDecision -Name "rebuild_status" -Default "") -eq "applied"))
|
||||
recovery_route_selected_for_live_traffic = $recoveryTrafficObserved
|
||||
recovery_traffic_accepted = ([int]$recoveryResponse.StatusCode -eq 202 -and ($recoveryAcceptedBy -eq "introspection" -or $recoveryAcceptedBy -eq "signed"))
|
||||
degraded_replacement_not_reselected_after_recovery = ($null -ne $recoveryFinalSnapshot -and $recoveryFinalSnapshot.last_selected_route_id -ne $alternateRouteID)
|
||||
no_fallback_or_failures_after_recovery = (($recoveryFinalFallbackLocal - $recoveryBaselineFallbackLocal) -eq 0 -and ($recoveryFinalRouteFailures - $recoveryBaselineRouteFailures) -eq 0)
|
||||
no_flow_drops_after_recovery = (($recoveryFinalFlowDropped - $recoveryBaselineFlowDropped) -eq 0 -and ($recoveryFinalSchedulerDropped - $recoveryBaselineSchedulerDropped) -eq 0)
|
||||
}
|
||||
$failed = @($checks.GetEnumerator() | Where-Object { -not $_.Value } | ForEach-Object { $_.Key })
|
||||
|
||||
$result = [ordered]@{
|
||||
schema_version = "c18z81.service_channel_replacement_degradation_recovery_smoke.v1"
|
||||
run_id = $runId
|
||||
cluster_id = $ClusterID
|
||||
channel_id = [string]$lease.channel_id
|
||||
primary_route_id = $primaryRouteID
|
||||
alternate_route_id = $alternateRouteID
|
||||
recovery_route_id = $recoveryRouteID
|
||||
rebuild_request_id = $commandID
|
||||
passed = ($failed.Count -eq 0)
|
||||
checks = $checks
|
||||
failed_checks = $failed
|
||||
summary = [ordered]@{
|
||||
backend_container = $backendLine.Trim()
|
||||
node_containers = $nodeLines.Trim()
|
||||
remediation_command = $command
|
||||
rebuild_attempt = $attempt
|
||||
route_manager_decision = $routeManagerDecision
|
||||
route_manager_transition = $routeManagerTransition
|
||||
burst_count = $burstCount
|
||||
burst_packet_count = $burstPacketCount
|
||||
accepted_burst_count = $acceptedBurstCount
|
||||
replacement_burst_count = $replacementBurstCount
|
||||
stale_primary_reselection_count = $stalePrimaryReselections
|
||||
baseline_snapshot = $baselineSnapshot
|
||||
final_snapshot = $finalSnapshot
|
||||
fallback_local_delta = ($finalFallbackLocal - $baselineFallbackLocal)
|
||||
route_failure_delta = ($finalRouteFailures - $baselineRouteFailures)
|
||||
flow_drop_delta = ($finalFlowDropped - $baselineFlowDropped)
|
||||
scheduler_drop_delta = ($finalSchedulerDropped - $baselineSchedulerDropped)
|
||||
burst_results = $burstResults
|
||||
recovery_planner_decision = $recoveryPlannerDecision
|
||||
recovery_route_manager_decision = $recoveryRouteManagerDecision
|
||||
recovery_route_manager_transition = $recoveryRouteManagerTransition
|
||||
recovery_traffic_status_code = [int]$recoveryResponse.StatusCode
|
||||
recovery_traffic_accepted_by = $recoveryAcceptedBy
|
||||
recovery_traffic_duration_ms = $recoveryDurationMs
|
||||
recovery_baseline_snapshot = $recoveryBaselineSnapshot
|
||||
recovery_final_snapshot = $recoveryFinalSnapshot
|
||||
recovery_route_failure_delta = ($recoveryFinalRouteFailures - $recoveryBaselineRouteFailures)
|
||||
recovery_fallback_local_delta = ($recoveryFinalFallbackLocal - $recoveryBaselineFallbackLocal)
|
||||
recovery_flow_drop_delta = ($recoveryFinalFlowDropped - $recoveryBaselineFlowDropped)
|
||||
recovery_scheduler_drop_delta = ($recoveryFinalSchedulerDropped - $recoveryBaselineSchedulerDropped)
|
||||
post_channel = $postChannel
|
||||
}
|
||||
}
|
||||
|
||||
$target = Join-Path $repoRoot $ResultPath
|
||||
$result | ConvertTo-Json -Depth 80 | Set-Content -Path $target -Encoding UTF8
|
||||
|
||||
try {
|
||||
if ($primaryRouteID) {
|
||||
Invoke-Api -Method POST -Path "/clusters/$ClusterID/mesh/route-intents/$primaryRouteID/expire" -Body @{ actor_user_id = $ActorUserID } | Out-Null
|
||||
}
|
||||
if ($alternateRouteID) {
|
||||
Invoke-Api -Method POST -Path "/clusters/$ClusterID/mesh/route-intents/$alternateRouteID/expire" -Body @{ actor_user_id = $ActorUserID } | Out-Null
|
||||
}
|
||||
if ($recoveryRouteID) {
|
||||
Invoke-Api -Method POST -Path "/clusters/$ClusterID/mesh/route-intents/$recoveryRouteID/expire" -Body @{ actor_user_id = $ActorUserID } | Out-Null
|
||||
}
|
||||
Start-Sleep -Seconds 3
|
||||
Invoke-Api -Method POST -Path "/clusters/$ClusterID/fabric/service-channels/leases/cleanup" -Body @{
|
||||
actor_user_id = $ActorUserID
|
||||
limit = 50
|
||||
} | Out-Null
|
||||
} catch {
|
||||
Write-Warning "cleanup failed after c18z81 smoke: $($_.Exception.Message)"
|
||||
}
|
||||
|
||||
if (-not $result.passed) {
|
||||
throw "C18Z81 replacement-degradation-recovery smoke failed: $($failed -join ', ')"
|
||||
}
|
||||
|
||||
Write-Host "C18Z81 service-channel replacement-degradation-recovery smoke passed. Result: $target"
|
||||
$result
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,510 @@
|
||||
param(
|
||||
[string]$ApiBaseUrl = "http://192.168.200.61:18121/api/v1",
|
||||
[string]$ClusterID = "cfc0743d-d960-49fb-9de8-96e063d5e4aa",
|
||||
[string]$ActorUserID = "f67d943f-5397-4b3a-a229-695fe67ad700",
|
||||
[string]$EntryNodeName = "test-1",
|
||||
[string]$RelayNodeName = "test-3",
|
||||
[string]$ExitNodeName = "test-2",
|
||||
[string]$EntryBaseUrl = "http://192.168.200.61:19131",
|
||||
[string]$DockerSSH = "test-docker",
|
||||
[string]$ExpectedBackendImage = "rap-backend:fabric-service-channel-0.2.281-c18z109",
|
||||
[string]$ExpectedNodeAgentImage = "rap-node-agent:0.2.270-c18z95",
|
||||
[string]$ResultPath = "artifacts\c18z82-service-channel-no-safe-recovery-smoke-result.json"
|
||||
)
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
$scriptDir = Split-Path -Parent $MyInvocation.MyCommand.Path
|
||||
$repoRoot = (Resolve-Path (Join-Path $scriptDir "..\..")).ProviderPath
|
||||
$runId = "c18z82-" + (Get-Date -Format "yyyyMMdd-HHmmss")
|
||||
|
||||
function Invoke-Api {
|
||||
param([string]$Method, [string]$Path, [object]$Body = $null)
|
||||
$params = @{ Method = $Method; Uri = "$ApiBaseUrl$Path"; TimeoutSec = 30 }
|
||||
if ($null -ne $Body) {
|
||||
$params.ContentType = "application/json"
|
||||
$params.Body = ($Body | ConvertTo-Json -Depth 80)
|
||||
}
|
||||
return Invoke-RestMethod @params
|
||||
}
|
||||
|
||||
function Get-PropertyValue {
|
||||
param([object]$Item, [string]$Name, [object]$Default = $null)
|
||||
if ($null -eq $Item) { return $Default }
|
||||
$property = $Item.PSObject.Properties[$Name]
|
||||
if ($null -eq $property) { return $Default }
|
||||
return $property.Value
|
||||
}
|
||||
|
||||
function Get-NodeByName {
|
||||
param([string]$Name)
|
||||
$nodes = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/nodes?actor_user_id=$ActorUserID").nodes
|
||||
$node = @($nodes | Where-Object { $_.name -eq $Name }) | Select-Object -First 1
|
||||
if ($null -eq $node) { throw "Node '$Name' was not found" }
|
||||
return $node
|
||||
}
|
||||
|
||||
function New-IPv4TcpPacket {
|
||||
param(
|
||||
[byte[]]$Source = @(10, 77, 0, 2),
|
||||
[byte[]]$Destination = @(192, 168, 200, 95),
|
||||
[int]$SourcePort,
|
||||
[int]$DestinationPort = 3389
|
||||
)
|
||||
$packet = [byte[]]::new(40)
|
||||
$packet[0] = 0x45
|
||||
$packet[2] = 0
|
||||
$packet[3] = 40
|
||||
$packet[8] = 64
|
||||
$packet[9] = 6
|
||||
[Array]::Copy($Source, 0, $packet, 12, 4)
|
||||
[Array]::Copy($Destination, 0, $packet, 16, 4)
|
||||
$packet[20] = [byte](($SourcePort -shr 8) -band 0xff)
|
||||
$packet[21] = [byte]($SourcePort -band 0xff)
|
||||
$packet[22] = [byte](($DestinationPort -shr 8) -band 0xff)
|
||||
$packet[23] = [byte]($DestinationPort -band 0xff)
|
||||
$packet[32] = 0x50
|
||||
return $packet
|
||||
}
|
||||
|
||||
function ConvertTo-VPNPacketBatch {
|
||||
param([byte[][]]$Packets)
|
||||
$buffer = New-Object System.Collections.Generic.List[byte]
|
||||
foreach ($packet in $Packets) {
|
||||
if ($null -eq $packet -or $packet.Length -eq 0) { continue }
|
||||
$size = [uint32]$packet.Length
|
||||
$buffer.Add([byte](($size -shr 24) -band 0xff))
|
||||
$buffer.Add([byte](($size -shr 16) -band 0xff))
|
||||
$buffer.Add([byte](($size -shr 8) -band 0xff))
|
||||
$buffer.Add([byte]($size -band 0xff))
|
||||
foreach ($value in $packet) {
|
||||
$buffer.Add($value)
|
||||
}
|
||||
}
|
||||
return $buffer.ToArray()
|
||||
}
|
||||
|
||||
function New-RouteIntent {
|
||||
param([string]$SourceNodeID, [string]$DestinationNodeID, [string[]]$Hops)
|
||||
$expiresAt = (Get-Date).ToUniversalTime().AddMinutes(5).ToString("o")
|
||||
return (Invoke-Api -Method POST -Path "/clusters/$ClusterID/mesh/route-intents" -Body @{
|
||||
actor_user_id = $ActorUserID
|
||||
source_selector = @{ node_id = $SourceNodeID }
|
||||
destination_selector = @{ node_id = $DestinationNodeID }
|
||||
service_class = "vpn_packets"
|
||||
priority = 2100000000
|
||||
policy = @{
|
||||
synthetic_enabled = $true
|
||||
route_version = "$runId-primary"
|
||||
policy_version = "$runId-primary"
|
||||
peer_directory_version = "$runId-primary"
|
||||
hops = @($Hops)
|
||||
allowed_channels = @("vpn_packet", "fabric_control")
|
||||
max_ttl = 8
|
||||
max_hops = 8
|
||||
expires_at = $expiresAt
|
||||
metadata = @{ smoke = "c18z82_service_channel_no_safe_recovery"; run_id = $runId }
|
||||
}
|
||||
}).route_intent
|
||||
}
|
||||
|
||||
function Disable-ExistingRouteIntents {
|
||||
param([string]$SourceNodeID, [string]$DestinationNodeID)
|
||||
$items = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/mesh/route-intents?actor_user_id=$ActorUserID").route_intents
|
||||
foreach ($item in @($items)) {
|
||||
if ([string](Get-PropertyValue -Item $item -Name "status" -Default "") -ne "active") { continue }
|
||||
if ([string](Get-PropertyValue -Item $item -Name "service_class" -Default "") -ne "vpn_packets") { continue }
|
||||
$sourceSelector = Get-PropertyValue -Item $item -Name "source_selector" -Default $null
|
||||
$destinationSelector = Get-PropertyValue -Item $item -Name "destination_selector" -Default $null
|
||||
if ([string](Get-PropertyValue -Item $sourceSelector -Name "node_id" -Default "") -ne $SourceNodeID) { continue }
|
||||
if ([string](Get-PropertyValue -Item $destinationSelector -Name "node_id" -Default "") -ne $DestinationNodeID) { continue }
|
||||
[void](Invoke-Api -Method POST -Path "/clusters/$ClusterID/mesh/route-intents/$($item.id)/disable" -Body @{
|
||||
actor_user_id = $ActorUserID
|
||||
reason = "c18z82 isolate no-safe-recovery smoke route pair"
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
function Send-DegradedHeartbeat {
|
||||
param([string]$EntryNodeID, [string]$PrimaryRouteID)
|
||||
$observedAt = (Get-Date).ToUniversalTime().ToString("o")
|
||||
return Invoke-Api -Method POST -Path "/clusters/$ClusterID/nodes/$EntryNodeID/heartbeats" -Body @{
|
||||
health_status = "healthy"
|
||||
reported_version = "0.2.256-c18z82"
|
||||
capabilities = @{
|
||||
fabric_service_channel_runtime = $true
|
||||
fabric_service_channel_route_quality_feedback = $true
|
||||
smoke_feedback_injection = "c18z77"
|
||||
}
|
||||
service_states = @{ smoke = "c18z82_primary_degraded_alternate_after_lease" }
|
||||
metadata = @{
|
||||
fabric_service_channel_runtime_report = @{
|
||||
schema_version = "c18l.fabric_service_channel_runtime_report.v1"
|
||||
config_version = "$runId-primary"
|
||||
cluster_id = $ClusterID
|
||||
local_node_id = $EntryNodeID
|
||||
observed_at = $observedAt
|
||||
ingress = @{
|
||||
flow_scheduler = @{
|
||||
schema_version = "rap.fabric_flow_scheduler.v1"
|
||||
service_neutral = $true
|
||||
service_mode = "application_protocol_agnostic"
|
||||
channel_stats = @{
|
||||
"c18z82-primary-degraded" = @{
|
||||
last_route_id = $PrimaryRouteID
|
||||
last_failed_route_id = $PrimaryRouteID
|
||||
route_generation = "$runId-primary"
|
||||
last_error = "c18z82 primary route degraded; alternate added after lease"
|
||||
last_send_duration_ms = 1200
|
||||
consecutive_failures = 3
|
||||
stall_count = 2
|
||||
route_rebuild_recommended = $true
|
||||
degraded_fallback_recommended = $false
|
||||
quality_window_sample_count = 8
|
||||
quality_window_success_count = 2
|
||||
quality_window_failure_count = 3
|
||||
quality_window_slow_count = 2
|
||||
quality_window_drop_count = 0
|
||||
quality_window_avg_latency_ms = 1200
|
||||
quality_window_last_updated_at = $observedAt
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
smoke = @{ name = "c18z82_service_channel_no_safe_recovery"; run_id = $runId }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function Send-ReplacementDegradedHeartbeat {
|
||||
param([string]$EntryNodeID, [string]$ReplacementRouteID, [string]$RouteGeneration)
|
||||
$observedAt = (Get-Date).ToUniversalTime().ToString("o")
|
||||
return Invoke-Api -Method POST -Path "/clusters/$ClusterID/nodes/$EntryNodeID/heartbeats" -Body @{
|
||||
health_status = "healthy"
|
||||
reported_version = "0.2.256-c18z82"
|
||||
capabilities = @{
|
||||
fabric_service_channel_runtime = $true
|
||||
fabric_service_channel_route_quality_feedback = $true
|
||||
smoke_feedback_injection = "c18z82"
|
||||
}
|
||||
service_states = @{ smoke = "c18z82_replacement_route_degraded_no_safe_recovery" }
|
||||
metadata = @{
|
||||
fabric_service_channel_runtime_report = @{
|
||||
schema_version = "c18l.fabric_service_channel_runtime_report.v1"
|
||||
config_version = "$runId-replacement-degraded"
|
||||
cluster_id = $ClusterID
|
||||
local_node_id = $EntryNodeID
|
||||
observed_at = $observedAt
|
||||
ingress = @{
|
||||
last_selected_route_id = $ReplacementRouteID
|
||||
send_route_failures = 4
|
||||
flow_scheduler = @{
|
||||
schema_version = "rap.fabric_flow_scheduler.v1"
|
||||
service_neutral = $true
|
||||
service_mode = "application_protocol_agnostic"
|
||||
channel_stats = @{
|
||||
"c18z82-replacement-degraded" = @{
|
||||
last_route_id = $ReplacementRouteID
|
||||
last_failed_route_id = $ReplacementRouteID
|
||||
route_generation = $RouteGeneration
|
||||
last_error = "c18z82 replacement route degraded; no safe recovery route exists"
|
||||
last_send_duration_ms = 1500
|
||||
consecutive_failures = 4
|
||||
stall_count = 2
|
||||
route_rebuild_recommended = $true
|
||||
degraded_fallback_recommended = $true
|
||||
quality_window_sample_count = 10
|
||||
quality_window_success_count = 1
|
||||
quality_window_failure_count = 4
|
||||
quality_window_slow_count = 3
|
||||
quality_window_drop_count = 0
|
||||
quality_window_avg_latency_ms = 1500
|
||||
quality_window_last_updated_at = $observedAt
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
smoke = @{ name = "c18z82_service_channel_no_safe_recovery"; run_id = $runId }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
$entryNode = Get-NodeByName -Name $EntryNodeName
|
||||
$relayNode = Get-NodeByName -Name $RelayNodeName
|
||||
$exitNode = Get-NodeByName -Name $ExitNodeName
|
||||
[void](Disable-ExistingRouteIntents -SourceNodeID $entryNode.id -DestinationNodeID $exitNode.id)
|
||||
$route = New-RouteIntent -SourceNodeID $entryNode.id -DestinationNodeID $exitNode.id -Hops @($entryNode.id, $exitNode.id)
|
||||
$lease = (Invoke-Api -Method POST -Path "/clusters/$ClusterID/fabric/service-channels/leases" -Body @{
|
||||
actor_user_id = $ActorUserID
|
||||
organization_id = "smoke-org"
|
||||
user_id = "smoke-user"
|
||||
resource_id = "c18z82-vpn-smoke"
|
||||
service_class = "vpn_packets"
|
||||
entry_node_ids = @([string]$entryNode.id)
|
||||
exit_node_ids = @([string]$exitNode.id)
|
||||
preferred_entry_node_id = [string]$entryNode.id
|
||||
preferred_exit_node_id = [string]$exitNode.id
|
||||
allowed_channels = @("vpn_packet", "fabric_control")
|
||||
ttl_seconds = 180
|
||||
metadata = @{ smoke = "c18z82_service_channel_no_safe_recovery"; run_id = $runId }
|
||||
}).fabric_service_channel_lease
|
||||
$primaryRouteID = [string](Get-PropertyValue -Item (Get-PropertyValue -Item $lease -Name "primary_route" -Default $null) -Name "route_id" -Default $route.id)
|
||||
|
||||
[void](Send-DegradedHeartbeat -EntryNodeID $entryNode.id -PrimaryRouteID $primaryRouteID)
|
||||
|
||||
$matchingChannel = $null
|
||||
for ($i = 0; $i -lt 10; $i++) {
|
||||
Start-Sleep -Seconds 3
|
||||
$accessTelemetry = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/fabric/service-channels/access-telemetry?actor_user_id=$ActorUserID&limit=50").fabric_service_channel_access_telemetry
|
||||
$matchingChannel = @($accessTelemetry.active_channels | Where-Object { $_.channel_id -eq $lease.channel_id }) | Select-Object -First 1
|
||||
if ($null -ne $matchingChannel -and [string](Get-PropertyValue -Item $matchingChannel -Name "remediation_action" -Default "") -eq "rebuild_route") {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
$command = Get-PropertyValue -Item $matchingChannel -Name "remediation_command" -Default $null
|
||||
$commandID = [string](Get-PropertyValue -Item $command -Name "command_id" -Default "")
|
||||
$alternate = New-RouteIntent -SourceNodeID $entryNode.id -DestinationNodeID $exitNode.id -Hops @($entryNode.id, $relayNode.id, $exitNode.id)
|
||||
$alternateRouteID = [string]$alternate.id
|
||||
[void](Invoke-Api -Method GET -Path "/clusters/$ClusterID/nodes/$($entryNode.id)/mesh/synthetic-config")
|
||||
|
||||
$attempts = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/fabric/service-channels/rebuild-attempts?actor_user_id=$ActorUserID&reporter_node_id=$($entryNode.id)&rebuild_request_id=$commandID&limit=10").rebuild_attempts
|
||||
$attempt = @($attempts | Where-Object { $_.rebuild_request_id -eq $commandID }) | Select-Object -First 1
|
||||
|
||||
$routeManagerDecision = $null
|
||||
$routeManagerTransition = $null
|
||||
for ($i = 0; $i -lt 12; $i++) {
|
||||
Start-Sleep -Seconds 5
|
||||
$heartbeats = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/nodes/$($entryNode.id)/heartbeats?actor_user_id=$ActorUserID&limit=5").heartbeats
|
||||
foreach ($heartbeat in @($heartbeats)) {
|
||||
$runtimeReport = Get-PropertyValue -Item (Get-PropertyValue -Item $heartbeat -Name "metadata" -Default $null) -Name "fabric_service_channel_runtime_report" -Default $null
|
||||
$ingress = Get-PropertyValue -Item $runtimeReport -Name "ingress" -Default $null
|
||||
$routeManager = Get-PropertyValue -Item $ingress -Name "route_manager" -Default $null
|
||||
$routeManagerTransition = Get-PropertyValue -Item $ingress -Name "route_manager_transition" -Default $null
|
||||
$decisions = @()
|
||||
if ($null -ne $routeManager -and $routeManager.PSObject.Properties.Name -contains "decisions") {
|
||||
$decisions = @($routeManager.decisions)
|
||||
}
|
||||
$routeManagerDecision = $decisions | Where-Object {
|
||||
[string](Get-PropertyValue -Item $_ -Name "rebuild_request_id" -Default "") -eq $commandID -and
|
||||
[string](Get-PropertyValue -Item $_ -Name "route_id" -Default "") -eq $primaryRouteID -and
|
||||
[string](Get-PropertyValue -Item $_ -Name "replacement_route_id" -Default "") -eq $alternateRouteID
|
||||
} | Select-Object -First 1
|
||||
if ($null -ne $routeManagerDecision) {
|
||||
break
|
||||
}
|
||||
}
|
||||
if ($null -ne $routeManagerDecision) {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
$packetPath = $lease.entry_http.path_template.
|
||||
Replace("{cluster_id}", $ClusterID).
|
||||
Replace("{channel_id}", [string]$lease.channel_id).
|
||||
Replace("{resource_id}", "c18z82-vpn-smoke")
|
||||
$packetUrl = $EntryBaseUrl.TrimEnd("/") + $packetPath
|
||||
$headers = @{
|
||||
"X-RAP-Service-Channel-Token" = [string]$lease.token.token
|
||||
"X-RAP-Fabric-Channel-ID" = [string]$lease.channel_id
|
||||
"X-RAP-Service-Class" = "vpn_packets"
|
||||
"X-RAP-Channel-Class" = "vpn_packet"
|
||||
"X-RAP-Traffic-Class" = "interactive"
|
||||
}
|
||||
$baselineFallbackLocal = 0
|
||||
$baselineRouteFailures = 0
|
||||
$baselineFlowDropped = 0
|
||||
$baselineSchedulerDropped = 0
|
||||
$baselineHeartbeats = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/nodes/$($entryNode.id)/heartbeats?actor_user_id=$ActorUserID&limit=1").heartbeats
|
||||
if (@($baselineHeartbeats).Count -gt 0) {
|
||||
$baselineRuntimeReport = Get-PropertyValue -Item (Get-PropertyValue -Item $baselineHeartbeats[0] -Name "metadata" -Default $null) -Name "fabric_service_channel_runtime_report" -Default $null
|
||||
$baselineIngress = Get-PropertyValue -Item $baselineRuntimeReport -Name "ingress" -Default $null
|
||||
$baselineFlowScheduler = Get-PropertyValue -Item $baselineIngress -Name "flow_scheduler" -Default $null
|
||||
$baselineFallbackLocal = [int](Get-PropertyValue -Item $baselineIngress -Name "send_fallback_local" -Default 0)
|
||||
$baselineRouteFailures = [int](Get-PropertyValue -Item $baselineIngress -Name "send_route_failures" -Default 0)
|
||||
$baselineFlowDropped = [int](Get-PropertyValue -Item $baselineIngress -Name "send_flow_dropped" -Default 0)
|
||||
$baselineSchedulerDropped = [int](Get-PropertyValue -Item $baselineFlowScheduler -Name "dropped" -Default 0)
|
||||
}
|
||||
$trafficPackets = @()
|
||||
foreach ($offset in 0..15) {
|
||||
$trafficPackets += ,(New-IPv4TcpPacket -SourcePort (54000 + $offset) -DestinationPort 3389)
|
||||
}
|
||||
$trafficPayload = ConvertTo-VPNPacketBatch -Packets $trafficPackets
|
||||
$trafficPayloadPath = Join-Path ([System.IO.Path]::GetTempPath()) "$runId-vpn-packet-batch.bin"
|
||||
[System.IO.File]::WriteAllBytes($trafficPayloadPath, [byte[]]$trafficPayload)
|
||||
$trafficStarted = Get-Date
|
||||
$trafficResponse = Invoke-WebRequest -Method Post -Uri ($packetUrl + "?batch=true") -Headers $headers -InFile $trafficPayloadPath -ContentType "application/vnd.rap.vpn-packet-batch.v1" -TimeoutSec 30
|
||||
$trafficDurationMs = [int]((Get-Date) - $trafficStarted).TotalMilliseconds
|
||||
$trafficAcceptedBy = [string]$trafficResponse.Headers["X-RAP-Service-Channel-Accepted-By"]
|
||||
Remove-Item -Path $trafficPayloadPath -Force -ErrorAction SilentlyContinue
|
||||
|
||||
$replacementTrafficObserved = $false
|
||||
$replacementLastSelected = ""
|
||||
$replacementFlowStats = @()
|
||||
$postRemediationFallbackLocal = 0
|
||||
$postRemediationRouteFailures = 0
|
||||
$postRemediationFlowDropped = 0
|
||||
$postRemediationSchedulerDropped = 0
|
||||
for ($i = 0; $i -lt 12; $i++) {
|
||||
Start-Sleep -Seconds 5
|
||||
$heartbeats = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/nodes/$($entryNode.id)/heartbeats?actor_user_id=$ActorUserID&limit=5").heartbeats
|
||||
foreach ($heartbeat in @($heartbeats)) {
|
||||
$runtimeReport = Get-PropertyValue -Item (Get-PropertyValue -Item $heartbeat -Name "metadata" -Default $null) -Name "fabric_service_channel_runtime_report" -Default $null
|
||||
$ingress = Get-PropertyValue -Item $runtimeReport -Name "ingress" -Default $null
|
||||
$replacementLastSelected = [string](Get-PropertyValue -Item $ingress -Name "last_selected_route_id" -Default "")
|
||||
$postRemediationFallbackLocal = [int](Get-PropertyValue -Item $ingress -Name "send_fallback_local" -Default 0)
|
||||
$postRemediationRouteFailures = [int](Get-PropertyValue -Item $ingress -Name "send_route_failures" -Default 0)
|
||||
$postRemediationFlowDropped = [int](Get-PropertyValue -Item $ingress -Name "send_flow_dropped" -Default 0)
|
||||
$flowScheduler = Get-PropertyValue -Item $ingress -Name "flow_scheduler" -Default $null
|
||||
$postRemediationSchedulerDropped = [int](Get-PropertyValue -Item $flowScheduler -Name "dropped" -Default 0)
|
||||
$channelStats = Get-PropertyValue -Item $flowScheduler -Name "channel_stats" -Default $null
|
||||
if ($null -ne $channelStats) {
|
||||
$replacementFlowStats = @()
|
||||
foreach ($property in @($channelStats.PSObject.Properties)) {
|
||||
$stat = $property.Value
|
||||
if ([string](Get-PropertyValue -Item $stat -Name "last_route_id" -Default "") -eq $alternateRouteID) {
|
||||
$replacementFlowStats += $stat
|
||||
}
|
||||
}
|
||||
}
|
||||
if ($replacementLastSelected -eq $alternateRouteID -and $replacementFlowStats.Count -ge 1) {
|
||||
$replacementTrafficObserved = $true
|
||||
break
|
||||
}
|
||||
}
|
||||
if ($replacementTrafficObserved) {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
$postAccess = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/fabric/service-channels/access-telemetry?actor_user_id=$ActorUserID&limit=50").fabric_service_channel_access_telemetry
|
||||
$postChannel = @($postAccess.active_channels | Where-Object { $_.channel_id -eq $lease.channel_id }) | Select-Object -First 1
|
||||
|
||||
[void](Send-ReplacementDegradedHeartbeat -EntryNodeID $entryNode.id -ReplacementRouteID $alternateRouteID -RouteGeneration "$runId-primary")
|
||||
$noSafeSyntheticConfig = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/nodes/$($entryNode.id)/mesh/synthetic-config").synthetic_mesh_config
|
||||
$noSafePathDecisions = @()
|
||||
if ($noSafeSyntheticConfig.PSObject.Properties.Name -contains "route_path_decisions") {
|
||||
$noSafePathDecisions = @($noSafeSyntheticConfig.route_path_decisions.decisions)
|
||||
}
|
||||
$noSafeDecision = $noSafePathDecisions | Where-Object {
|
||||
[string](Get-PropertyValue -Item $_ -Name "route_id" -Default "") -eq $alternateRouteID -and
|
||||
[string](Get-PropertyValue -Item $_ -Name "decision_source" -Default "") -eq "service_channel_feedback_no_alternate"
|
||||
} | Select-Object -First 1
|
||||
$noSafeFeedback = $null
|
||||
$serviceChannelFeedback = Get-PropertyValue -Item $noSafeSyntheticConfig -Name "service_channel_feedback" -Default $null
|
||||
if ($null -ne $serviceChannelFeedback -and $serviceChannelFeedback.PSObject.Properties.Name -contains "routes") {
|
||||
$noSafeFeedback = @($serviceChannelFeedback.routes | Where-Object { [string](Get-PropertyValue -Item $_ -Name "route_id" -Default "") -eq $alternateRouteID }) | Select-Object -First 1
|
||||
}
|
||||
|
||||
$postNoSafeAccess = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/fabric/service-channels/access-telemetry?actor_user_id=$ActorUserID&limit=50").fabric_service_channel_access_telemetry
|
||||
$postNoSafeChannel = @($postNoSafeAccess.active_channels | Where-Object { $_.channel_id -eq $lease.channel_id }) | Select-Object -First 1
|
||||
|
||||
$backendLine = (& ssh $DockerSSH "docker ps --format '{{.Names}} {{.Image}} {{.Status}}' | grep '^rap_test_backend '") | Out-String
|
||||
$nodeLines = (& ssh $DockerSSH "docker ps --format '{{.Names}} {{.Image}} {{.Status}}' | grep '^rap_test_node_test_'") | Out-String
|
||||
$checks = [ordered]@{
|
||||
backend_expected_image_deployed = $backendLine.Contains($ExpectedBackendImage)
|
||||
node_agent_expected_image_deployed = $nodeLines.Contains($ExpectedNodeAgentImage)
|
||||
lease_ready = ([string]$lease.status -eq "ready")
|
||||
remediation_rebuild_route_visible = ($null -ne $matchingChannel -and [string](Get-PropertyValue -Item $matchingChannel -Name "remediation_action" -Default "") -eq "rebuild_route")
|
||||
remediation_command_visible = ($null -ne $command -and $commandID.Length -gt 0)
|
||||
durable_rebuild_intent_recorded = ($null -ne $attempt)
|
||||
durable_rebuild_intent_resolved_applied = ($null -ne $attempt -and [string](Get-PropertyValue -Item $attempt -Name "rebuild_status" -Default "") -eq "applied")
|
||||
durable_rebuild_intent_outcome_replacement_selected = ($null -ne $attempt -and [string](Get-PropertyValue -Item $attempt -Name "outcome" -Default "") -eq "replacement_selected")
|
||||
durable_rebuild_intent_replacement_matches = ($null -ne $attempt -and [string](Get-PropertyValue -Item $attempt -Name "replacement_route_id" -Default "") -eq $alternateRouteID)
|
||||
durable_rebuild_intent_source_matches = ($null -ne $attempt -and [string](Get-PropertyValue -Item $attempt -Name "decision_source" -Default "") -eq "service_channel_remediation_command")
|
||||
initial_access_telemetry_reports_planner_applied = ($null -ne $postChannel -and [string](Get-PropertyValue -Item $postChannel -Name "remediation_execution_status" -Default "") -eq "rebuild_request_applied")
|
||||
node_route_manager_consumed_same_command = ($null -ne $routeManagerDecision)
|
||||
node_route_manager_applied_rebuild = ($null -ne $routeManagerDecision -and [string](Get-PropertyValue -Item $routeManagerDecision -Name "rebuild_status" -Default "") -eq "applied")
|
||||
node_route_manager_transition_applied_rebuild = ($null -ne $routeManagerTransition -and [string](Get-PropertyValue -Item $routeManagerTransition -Name "status" -Default "") -eq "applied_rebuild")
|
||||
traffic_packet_accepted = ([int]$trafficResponse.StatusCode -eq 202)
|
||||
traffic_accepted_by_runtime = ($trafficAcceptedBy -eq "introspection" -or $trafficAcceptedBy -eq "signed")
|
||||
traffic_selected_replacement_route = $replacementTrafficObserved
|
||||
traffic_last_selected_route_matches_replacement = ($replacementLastSelected -eq $alternateRouteID)
|
||||
no_backend_or_local_fallback_after_replacement = (($postRemediationFallbackLocal - $baselineFallbackLocal) -eq 0)
|
||||
no_route_failures_after_replacement = (($postRemediationRouteFailures - $baselineRouteFailures) -eq 0)
|
||||
no_flow_drops_after_replacement = (($postRemediationFlowDropped - $baselineFlowDropped) -eq 0 -and ($postRemediationSchedulerDropped - $baselineSchedulerDropped) -eq 0)
|
||||
replacement_degradation_feedback_visible = ($null -ne $noSafeDecision -and @($noSafeDecision.score_reasons | Where-Object { [string]$_ -eq "service_channel_fenced_route" }).Count -gt 0)
|
||||
no_safe_recovery_decision_visible = ($null -ne $noSafeDecision)
|
||||
no_safe_recovery_status_pending_fallback = ($null -ne $noSafeDecision -and [string](Get-PropertyValue -Item $noSafeDecision -Name "rebuild_status" -Default "") -eq "pending_degraded_fallback")
|
||||
no_safe_recovery_reason_visible = ($null -ne $noSafeDecision -and @($noSafeDecision.score_reasons | Where-Object { [string]$_ -eq "no_unfenced_alternate_route" }).Count -gt 0)
|
||||
no_silent_stickiness_to_bad_replacement = ($null -ne $noSafeDecision -and [string](Get-PropertyValue -Item $noSafeDecision -Name "decision_source" -Default "") -eq "service_channel_feedback_no_alternate")
|
||||
}
|
||||
$failed = @($checks.GetEnumerator() | Where-Object { -not $_.Value } | ForEach-Object { $_.Key })
|
||||
|
||||
$result = [ordered]@{
|
||||
schema_version = "c18z82.service_channel_no_safe_recovery_smoke.v1"
|
||||
run_id = $runId
|
||||
cluster_id = $ClusterID
|
||||
channel_id = [string]$lease.channel_id
|
||||
primary_route_id = $primaryRouteID
|
||||
alternate_route_id = $alternateRouteID
|
||||
rebuild_request_id = $commandID
|
||||
passed = ($failed.Count -eq 0)
|
||||
checks = $checks
|
||||
failed_checks = $failed
|
||||
summary = [ordered]@{
|
||||
backend_container = $backendLine.Trim()
|
||||
node_containers = $nodeLines.Trim()
|
||||
remediation_command = $command
|
||||
rebuild_attempt = $attempt
|
||||
route_manager_decision = $routeManagerDecision
|
||||
route_manager_transition = $routeManagerTransition
|
||||
traffic_status_code = [int]$trafficResponse.StatusCode
|
||||
traffic_accepted_by = $trafficAcceptedBy
|
||||
traffic_duration_ms = $trafficDurationMs
|
||||
traffic_packet_count = $trafficPackets.Count
|
||||
replacement_last_selected_route_id = $replacementLastSelected
|
||||
replacement_flow_stat_count = $replacementFlowStats.Count
|
||||
replacement_flow_stats = $replacementFlowStats
|
||||
baseline_send_fallback_local = $baselineFallbackLocal
|
||||
baseline_send_route_failures = $baselineRouteFailures
|
||||
baseline_send_flow_dropped = $baselineFlowDropped
|
||||
baseline_scheduler_dropped = $baselineSchedulerDropped
|
||||
fallback_local_delta = ($postRemediationFallbackLocal - $baselineFallbackLocal)
|
||||
route_failure_delta = ($postRemediationRouteFailures - $baselineRouteFailures)
|
||||
flow_drop_delta = ($postRemediationFlowDropped - $baselineFlowDropped)
|
||||
scheduler_drop_delta = ($postRemediationSchedulerDropped - $baselineSchedulerDropped)
|
||||
post_remediation_send_fallback_local = $postRemediationFallbackLocal
|
||||
post_remediation_send_route_failures = $postRemediationRouteFailures
|
||||
post_remediation_send_flow_dropped = $postRemediationFlowDropped
|
||||
post_remediation_scheduler_dropped = $postRemediationSchedulerDropped
|
||||
post_channel = $postChannel
|
||||
no_safe_feedback = $noSafeFeedback
|
||||
no_safe_decision = $noSafeDecision
|
||||
no_safe_route_path_decisions = $noSafePathDecisions
|
||||
post_no_safe_channel = $postNoSafeChannel
|
||||
}
|
||||
}
|
||||
|
||||
$target = Join-Path $repoRoot $ResultPath
|
||||
$result | ConvertTo-Json -Depth 80 | Set-Content -Path $target -Encoding UTF8
|
||||
|
||||
try {
|
||||
if ($primaryRouteID) {
|
||||
Invoke-Api -Method POST -Path "/clusters/$ClusterID/mesh/route-intents/$primaryRouteID/expire" -Body @{ actor_user_id = $ActorUserID } | Out-Null
|
||||
}
|
||||
if ($alternateRouteID) {
|
||||
Invoke-Api -Method POST -Path "/clusters/$ClusterID/mesh/route-intents/$alternateRouteID/expire" -Body @{ actor_user_id = $ActorUserID } | Out-Null
|
||||
}
|
||||
Start-Sleep -Seconds 3
|
||||
Invoke-Api -Method POST -Path "/clusters/$ClusterID/fabric/service-channels/leases/cleanup" -Body @{
|
||||
actor_user_id = $ActorUserID
|
||||
limit = 50
|
||||
} | Out-Null
|
||||
} catch {
|
||||
Write-Warning "cleanup failed after c18z82 smoke: $($_.Exception.Message)"
|
||||
}
|
||||
|
||||
if (-not $result.passed) {
|
||||
throw "C18Z82 no-safe-recovery smoke failed: $($failed -join ', ')"
|
||||
}
|
||||
|
||||
Write-Host "C18Z82 service-channel no-safe-recovery smoke passed. Result: $target"
|
||||
$result
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,522 @@
|
||||
param(
|
||||
[string]$ApiBaseUrl = "http://192.168.200.61:18121/api/v1",
|
||||
[string]$ClusterID = "cfc0743d-d960-49fb-9de8-96e063d5e4aa",
|
||||
[string]$ActorUserID = "f67d943f-5397-4b3a-a229-695fe67ad700",
|
||||
[string]$EntryNodeName = "test-1",
|
||||
[string]$RelayNodeName = "test-3",
|
||||
[string]$ExitNodeName = "test-2",
|
||||
[string]$EntryBaseUrl = "http://192.168.200.61:19131",
|
||||
[string]$DockerSSH = "test-docker",
|
||||
[string]$ExpectedBackendImage = "rap-backend:fabric-service-channel-0.2.281-c18z109",
|
||||
[string]$ExpectedNodeAgentImage = "rap-node-agent:0.2.270-c18z95",
|
||||
[string]$ResultPath = "artifacts\c18z83-service-channel-access-telemetry-no-safe-smoke-result.json"
|
||||
)
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
$scriptDir = Split-Path -Parent $MyInvocation.MyCommand.Path
|
||||
$repoRoot = (Resolve-Path (Join-Path $scriptDir "..\..")).ProviderPath
|
||||
$runId = "c18z83-" + (Get-Date -Format "yyyyMMdd-HHmmss")
|
||||
|
||||
function Invoke-Api {
|
||||
param([string]$Method, [string]$Path, [object]$Body = $null)
|
||||
$params = @{ Method = $Method; Uri = "$ApiBaseUrl$Path"; TimeoutSec = 30 }
|
||||
if ($null -ne $Body) {
|
||||
$params.ContentType = "application/json"
|
||||
$params.Body = ($Body | ConvertTo-Json -Depth 80)
|
||||
}
|
||||
return Invoke-RestMethod @params
|
||||
}
|
||||
|
||||
function Get-PropertyValue {
|
||||
param([object]$Item, [string]$Name, [object]$Default = $null)
|
||||
if ($null -eq $Item) { return $Default }
|
||||
$property = $Item.PSObject.Properties[$Name]
|
||||
if ($null -eq $property) { return $Default }
|
||||
return $property.Value
|
||||
}
|
||||
|
||||
function Get-NodeByName {
|
||||
param([string]$Name)
|
||||
$nodes = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/nodes?actor_user_id=$ActorUserID").nodes
|
||||
$node = @($nodes | Where-Object { $_.name -eq $Name }) | Select-Object -First 1
|
||||
if ($null -eq $node) { throw "Node '$Name' was not found" }
|
||||
return $node
|
||||
}
|
||||
|
||||
function New-IPv4TcpPacket {
|
||||
param(
|
||||
[byte[]]$Source = @(10, 77, 0, 2),
|
||||
[byte[]]$Destination = @(192, 168, 200, 95),
|
||||
[int]$SourcePort,
|
||||
[int]$DestinationPort = 3389
|
||||
)
|
||||
$packet = [byte[]]::new(40)
|
||||
$packet[0] = 0x45
|
||||
$packet[2] = 0
|
||||
$packet[3] = 40
|
||||
$packet[8] = 64
|
||||
$packet[9] = 6
|
||||
[Array]::Copy($Source, 0, $packet, 12, 4)
|
||||
[Array]::Copy($Destination, 0, $packet, 16, 4)
|
||||
$packet[20] = [byte](($SourcePort -shr 8) -band 0xff)
|
||||
$packet[21] = [byte]($SourcePort -band 0xff)
|
||||
$packet[22] = [byte](($DestinationPort -shr 8) -band 0xff)
|
||||
$packet[23] = [byte]($DestinationPort -band 0xff)
|
||||
$packet[32] = 0x50
|
||||
return $packet
|
||||
}
|
||||
|
||||
function ConvertTo-VPNPacketBatch {
|
||||
param([byte[][]]$Packets)
|
||||
$buffer = New-Object System.Collections.Generic.List[byte]
|
||||
foreach ($packet in $Packets) {
|
||||
if ($null -eq $packet -or $packet.Length -eq 0) { continue }
|
||||
$size = [uint32]$packet.Length
|
||||
$buffer.Add([byte](($size -shr 24) -band 0xff))
|
||||
$buffer.Add([byte](($size -shr 16) -band 0xff))
|
||||
$buffer.Add([byte](($size -shr 8) -band 0xff))
|
||||
$buffer.Add([byte]($size -band 0xff))
|
||||
foreach ($value in $packet) {
|
||||
$buffer.Add($value)
|
||||
}
|
||||
}
|
||||
return $buffer.ToArray()
|
||||
}
|
||||
|
||||
function New-RouteIntent {
|
||||
param([string]$SourceNodeID, [string]$DestinationNodeID, [string[]]$Hops)
|
||||
$expiresAt = (Get-Date).ToUniversalTime().AddMinutes(5).ToString("o")
|
||||
return (Invoke-Api -Method POST -Path "/clusters/$ClusterID/mesh/route-intents" -Body @{
|
||||
actor_user_id = $ActorUserID
|
||||
source_selector = @{ node_id = $SourceNodeID }
|
||||
destination_selector = @{ node_id = $DestinationNodeID }
|
||||
service_class = "vpn_packets"
|
||||
priority = 2100000000
|
||||
policy = @{
|
||||
synthetic_enabled = $true
|
||||
route_version = "$runId-primary"
|
||||
policy_version = "$runId-primary"
|
||||
peer_directory_version = "$runId-primary"
|
||||
hops = @($Hops)
|
||||
allowed_channels = @("vpn_packet", "fabric_control")
|
||||
max_ttl = 8
|
||||
max_hops = 8
|
||||
expires_at = $expiresAt
|
||||
metadata = @{ smoke = "c18z82_service_channel_no_safe_recovery"; run_id = $runId }
|
||||
}
|
||||
}).route_intent
|
||||
}
|
||||
|
||||
function Disable-ExistingRouteIntents {
|
||||
param([string]$SourceNodeID, [string]$DestinationNodeID)
|
||||
$items = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/mesh/route-intents?actor_user_id=$ActorUserID").route_intents
|
||||
foreach ($item in @($items)) {
|
||||
if ([string](Get-PropertyValue -Item $item -Name "status" -Default "") -ne "active") { continue }
|
||||
if ([string](Get-PropertyValue -Item $item -Name "service_class" -Default "") -ne "vpn_packets") { continue }
|
||||
$sourceSelector = Get-PropertyValue -Item $item -Name "source_selector" -Default $null
|
||||
$destinationSelector = Get-PropertyValue -Item $item -Name "destination_selector" -Default $null
|
||||
if ([string](Get-PropertyValue -Item $sourceSelector -Name "node_id" -Default "") -ne $SourceNodeID) { continue }
|
||||
if ([string](Get-PropertyValue -Item $destinationSelector -Name "node_id" -Default "") -ne $DestinationNodeID) { continue }
|
||||
[void](Invoke-Api -Method POST -Path "/clusters/$ClusterID/mesh/route-intents/$($item.id)/disable" -Body @{
|
||||
actor_user_id = $ActorUserID
|
||||
reason = "c18z82 isolate no-safe-recovery smoke route pair"
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
function Send-DegradedHeartbeat {
|
||||
param([string]$EntryNodeID, [string]$PrimaryRouteID)
|
||||
$observedAt = (Get-Date).ToUniversalTime().ToString("o")
|
||||
return Invoke-Api -Method POST -Path "/clusters/$ClusterID/nodes/$EntryNodeID/heartbeats" -Body @{
|
||||
health_status = "healthy"
|
||||
reported_version = "0.2.256-c18z82"
|
||||
capabilities = @{
|
||||
fabric_service_channel_runtime = $true
|
||||
fabric_service_channel_route_quality_feedback = $true
|
||||
smoke_feedback_injection = "c18z77"
|
||||
}
|
||||
service_states = @{ smoke = "c18z82_primary_degraded_alternate_after_lease" }
|
||||
metadata = @{
|
||||
fabric_service_channel_runtime_report = @{
|
||||
schema_version = "c18l.fabric_service_channel_runtime_report.v1"
|
||||
config_version = "$runId-primary"
|
||||
cluster_id = $ClusterID
|
||||
local_node_id = $EntryNodeID
|
||||
observed_at = $observedAt
|
||||
ingress = @{
|
||||
flow_scheduler = @{
|
||||
schema_version = "rap.fabric_flow_scheduler.v1"
|
||||
service_neutral = $true
|
||||
service_mode = "application_protocol_agnostic"
|
||||
channel_stats = @{
|
||||
"c18z82-primary-degraded" = @{
|
||||
last_route_id = $PrimaryRouteID
|
||||
last_failed_route_id = $PrimaryRouteID
|
||||
route_generation = "$runId-primary"
|
||||
last_error = "c18z82 primary route degraded; alternate added after lease"
|
||||
last_send_duration_ms = 1200
|
||||
consecutive_failures = 3
|
||||
stall_count = 2
|
||||
route_rebuild_recommended = $true
|
||||
degraded_fallback_recommended = $false
|
||||
quality_window_sample_count = 8
|
||||
quality_window_success_count = 2
|
||||
quality_window_failure_count = 3
|
||||
quality_window_slow_count = 2
|
||||
quality_window_drop_count = 0
|
||||
quality_window_avg_latency_ms = 1200
|
||||
quality_window_last_updated_at = $observedAt
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
smoke = @{ name = "c18z82_service_channel_no_safe_recovery"; run_id = $runId }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function Send-ReplacementDegradedHeartbeat {
|
||||
param([string]$EntryNodeID, [string]$ReplacementRouteID, [string]$RouteGeneration)
|
||||
$observedAt = (Get-Date).ToUniversalTime().ToString("o")
|
||||
return Invoke-Api -Method POST -Path "/clusters/$ClusterID/nodes/$EntryNodeID/heartbeats" -Body @{
|
||||
health_status = "healthy"
|
||||
reported_version = "0.2.256-c18z82"
|
||||
capabilities = @{
|
||||
fabric_service_channel_runtime = $true
|
||||
fabric_service_channel_route_quality_feedback = $true
|
||||
smoke_feedback_injection = "c18z82"
|
||||
}
|
||||
service_states = @{ smoke = "c18z82_replacement_route_degraded_no_safe_recovery" }
|
||||
metadata = @{
|
||||
fabric_service_channel_runtime_report = @{
|
||||
schema_version = "c18l.fabric_service_channel_runtime_report.v1"
|
||||
config_version = "$runId-replacement-degraded"
|
||||
cluster_id = $ClusterID
|
||||
local_node_id = $EntryNodeID
|
||||
observed_at = $observedAt
|
||||
ingress = @{
|
||||
last_selected_route_id = $ReplacementRouteID
|
||||
send_route_failures = 4
|
||||
flow_scheduler = @{
|
||||
schema_version = "rap.fabric_flow_scheduler.v1"
|
||||
service_neutral = $true
|
||||
service_mode = "application_protocol_agnostic"
|
||||
channel_stats = @{
|
||||
"c18z82-replacement-degraded" = @{
|
||||
last_route_id = $ReplacementRouteID
|
||||
last_failed_route_id = $ReplacementRouteID
|
||||
route_generation = $RouteGeneration
|
||||
last_error = "c18z82 replacement route degraded; no safe recovery route exists"
|
||||
last_send_duration_ms = 1500
|
||||
consecutive_failures = 4
|
||||
stall_count = 2
|
||||
route_rebuild_recommended = $true
|
||||
degraded_fallback_recommended = $true
|
||||
quality_window_sample_count = 10
|
||||
quality_window_success_count = 1
|
||||
quality_window_failure_count = 4
|
||||
quality_window_slow_count = 3
|
||||
quality_window_drop_count = 0
|
||||
quality_window_avg_latency_ms = 1500
|
||||
quality_window_last_updated_at = $observedAt
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
smoke = @{ name = "c18z82_service_channel_no_safe_recovery"; run_id = $runId }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
$entryNode = Get-NodeByName -Name $EntryNodeName
|
||||
$relayNode = Get-NodeByName -Name $RelayNodeName
|
||||
$exitNode = Get-NodeByName -Name $ExitNodeName
|
||||
[void](Disable-ExistingRouteIntents -SourceNodeID $entryNode.id -DestinationNodeID $exitNode.id)
|
||||
$route = New-RouteIntent -SourceNodeID $entryNode.id -DestinationNodeID $exitNode.id -Hops @($entryNode.id, $exitNode.id)
|
||||
$lease = (Invoke-Api -Method POST -Path "/clusters/$ClusterID/fabric/service-channels/leases" -Body @{
|
||||
actor_user_id = $ActorUserID
|
||||
organization_id = "smoke-org"
|
||||
user_id = "smoke-user"
|
||||
resource_id = "c18z82-vpn-smoke"
|
||||
service_class = "vpn_packets"
|
||||
entry_node_ids = @([string]$entryNode.id)
|
||||
exit_node_ids = @([string]$exitNode.id)
|
||||
preferred_entry_node_id = [string]$entryNode.id
|
||||
preferred_exit_node_id = [string]$exitNode.id
|
||||
allowed_channels = @("vpn_packet", "fabric_control")
|
||||
ttl_seconds = 180
|
||||
metadata = @{ smoke = "c18z82_service_channel_no_safe_recovery"; run_id = $runId }
|
||||
}).fabric_service_channel_lease
|
||||
$primaryRouteID = [string](Get-PropertyValue -Item (Get-PropertyValue -Item $lease -Name "primary_route" -Default $null) -Name "route_id" -Default $route.id)
|
||||
|
||||
[void](Send-DegradedHeartbeat -EntryNodeID $entryNode.id -PrimaryRouteID $primaryRouteID)
|
||||
|
||||
$matchingChannel = $null
|
||||
for ($i = 0; $i -lt 10; $i++) {
|
||||
Start-Sleep -Seconds 3
|
||||
$accessTelemetry = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/fabric/service-channels/access-telemetry?actor_user_id=$ActorUserID&limit=50").fabric_service_channel_access_telemetry
|
||||
$matchingChannel = @($accessTelemetry.active_channels | Where-Object { $_.channel_id -eq $lease.channel_id }) | Select-Object -First 1
|
||||
if ($null -ne $matchingChannel -and [string](Get-PropertyValue -Item $matchingChannel -Name "remediation_action" -Default "") -eq "rebuild_route") {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
$command = Get-PropertyValue -Item $matchingChannel -Name "remediation_command" -Default $null
|
||||
$commandID = [string](Get-PropertyValue -Item $command -Name "command_id" -Default "")
|
||||
$alternate = New-RouteIntent -SourceNodeID $entryNode.id -DestinationNodeID $exitNode.id -Hops @($entryNode.id, $relayNode.id, $exitNode.id)
|
||||
$alternateRouteID = [string]$alternate.id
|
||||
[void](Invoke-Api -Method GET -Path "/clusters/$ClusterID/nodes/$($entryNode.id)/mesh/synthetic-config")
|
||||
|
||||
$attempts = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/fabric/service-channels/rebuild-attempts?actor_user_id=$ActorUserID&reporter_node_id=$($entryNode.id)&rebuild_request_id=$commandID&limit=10").rebuild_attempts
|
||||
$attempt = @($attempts | Where-Object { $_.rebuild_request_id -eq $commandID }) | Select-Object -First 1
|
||||
|
||||
$routeManagerDecision = $null
|
||||
$routeManagerTransition = $null
|
||||
for ($i = 0; $i -lt 12; $i++) {
|
||||
Start-Sleep -Seconds 5
|
||||
$heartbeats = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/nodes/$($entryNode.id)/heartbeats?actor_user_id=$ActorUserID&limit=5").heartbeats
|
||||
foreach ($heartbeat in @($heartbeats)) {
|
||||
$runtimeReport = Get-PropertyValue -Item (Get-PropertyValue -Item $heartbeat -Name "metadata" -Default $null) -Name "fabric_service_channel_runtime_report" -Default $null
|
||||
$ingress = Get-PropertyValue -Item $runtimeReport -Name "ingress" -Default $null
|
||||
$routeManager = Get-PropertyValue -Item $ingress -Name "route_manager" -Default $null
|
||||
$routeManagerTransition = Get-PropertyValue -Item $ingress -Name "route_manager_transition" -Default $null
|
||||
$decisions = @()
|
||||
if ($null -ne $routeManager -and $routeManager.PSObject.Properties.Name -contains "decisions") {
|
||||
$decisions = @($routeManager.decisions)
|
||||
}
|
||||
$routeManagerDecision = $decisions | Where-Object {
|
||||
[string](Get-PropertyValue -Item $_ -Name "rebuild_request_id" -Default "") -eq $commandID -and
|
||||
[string](Get-PropertyValue -Item $_ -Name "route_id" -Default "") -eq $primaryRouteID -and
|
||||
[string](Get-PropertyValue -Item $_ -Name "replacement_route_id" -Default "") -eq $alternateRouteID
|
||||
} | Select-Object -First 1
|
||||
if ($null -ne $routeManagerDecision) {
|
||||
break
|
||||
}
|
||||
}
|
||||
if ($null -ne $routeManagerDecision) {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
$packetPath = $lease.entry_http.path_template.
|
||||
Replace("{cluster_id}", $ClusterID).
|
||||
Replace("{channel_id}", [string]$lease.channel_id).
|
||||
Replace("{resource_id}", "c18z82-vpn-smoke")
|
||||
$packetUrl = $EntryBaseUrl.TrimEnd("/") + $packetPath
|
||||
$headers = @{
|
||||
"X-RAP-Service-Channel-Token" = [string]$lease.token.token
|
||||
"X-RAP-Fabric-Channel-ID" = [string]$lease.channel_id
|
||||
"X-RAP-Service-Class" = "vpn_packets"
|
||||
"X-RAP-Channel-Class" = "vpn_packet"
|
||||
"X-RAP-Traffic-Class" = "interactive"
|
||||
}
|
||||
$baselineFallbackLocal = 0
|
||||
$baselineRouteFailures = 0
|
||||
$baselineFlowDropped = 0
|
||||
$baselineSchedulerDropped = 0
|
||||
$baselineHeartbeats = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/nodes/$($entryNode.id)/heartbeats?actor_user_id=$ActorUserID&limit=1").heartbeats
|
||||
if (@($baselineHeartbeats).Count -gt 0) {
|
||||
$baselineRuntimeReport = Get-PropertyValue -Item (Get-PropertyValue -Item $baselineHeartbeats[0] -Name "metadata" -Default $null) -Name "fabric_service_channel_runtime_report" -Default $null
|
||||
$baselineIngress = Get-PropertyValue -Item $baselineRuntimeReport -Name "ingress" -Default $null
|
||||
$baselineFlowScheduler = Get-PropertyValue -Item $baselineIngress -Name "flow_scheduler" -Default $null
|
||||
$baselineFallbackLocal = [int](Get-PropertyValue -Item $baselineIngress -Name "send_fallback_local" -Default 0)
|
||||
$baselineRouteFailures = [int](Get-PropertyValue -Item $baselineIngress -Name "send_route_failures" -Default 0)
|
||||
$baselineFlowDropped = [int](Get-PropertyValue -Item $baselineIngress -Name "send_flow_dropped" -Default 0)
|
||||
$baselineSchedulerDropped = [int](Get-PropertyValue -Item $baselineFlowScheduler -Name "dropped" -Default 0)
|
||||
}
|
||||
$trafficPackets = @()
|
||||
foreach ($offset in 0..15) {
|
||||
$trafficPackets += ,(New-IPv4TcpPacket -SourcePort (54000 + $offset) -DestinationPort 3389)
|
||||
}
|
||||
$trafficPayload = ConvertTo-VPNPacketBatch -Packets $trafficPackets
|
||||
$trafficPayloadPath = Join-Path ([System.IO.Path]::GetTempPath()) "$runId-vpn-packet-batch.bin"
|
||||
[System.IO.File]::WriteAllBytes($trafficPayloadPath, [byte[]]$trafficPayload)
|
||||
$trafficStarted = Get-Date
|
||||
$trafficResponse = Invoke-WebRequest -Method Post -Uri ($packetUrl + "?batch=true") -Headers $headers -InFile $trafficPayloadPath -ContentType "application/vnd.rap.vpn-packet-batch.v1" -TimeoutSec 30
|
||||
$trafficDurationMs = [int]((Get-Date) - $trafficStarted).TotalMilliseconds
|
||||
$trafficAcceptedBy = [string]$trafficResponse.Headers["X-RAP-Service-Channel-Accepted-By"]
|
||||
Remove-Item -Path $trafficPayloadPath -Force -ErrorAction SilentlyContinue
|
||||
|
||||
$replacementTrafficObserved = $false
|
||||
$replacementLastSelected = ""
|
||||
$replacementFlowStats = @()
|
||||
$postRemediationFallbackLocal = 0
|
||||
$postRemediationRouteFailures = 0
|
||||
$postRemediationFlowDropped = 0
|
||||
$postRemediationSchedulerDropped = 0
|
||||
for ($i = 0; $i -lt 12; $i++) {
|
||||
Start-Sleep -Seconds 5
|
||||
$heartbeats = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/nodes/$($entryNode.id)/heartbeats?actor_user_id=$ActorUserID&limit=5").heartbeats
|
||||
foreach ($heartbeat in @($heartbeats)) {
|
||||
$runtimeReport = Get-PropertyValue -Item (Get-PropertyValue -Item $heartbeat -Name "metadata" -Default $null) -Name "fabric_service_channel_runtime_report" -Default $null
|
||||
$ingress = Get-PropertyValue -Item $runtimeReport -Name "ingress" -Default $null
|
||||
$replacementLastSelected = [string](Get-PropertyValue -Item $ingress -Name "last_selected_route_id" -Default "")
|
||||
$postRemediationFallbackLocal = [int](Get-PropertyValue -Item $ingress -Name "send_fallback_local" -Default 0)
|
||||
$postRemediationRouteFailures = [int](Get-PropertyValue -Item $ingress -Name "send_route_failures" -Default 0)
|
||||
$postRemediationFlowDropped = [int](Get-PropertyValue -Item $ingress -Name "send_flow_dropped" -Default 0)
|
||||
$flowScheduler = Get-PropertyValue -Item $ingress -Name "flow_scheduler" -Default $null
|
||||
$postRemediationSchedulerDropped = [int](Get-PropertyValue -Item $flowScheduler -Name "dropped" -Default 0)
|
||||
$channelStats = Get-PropertyValue -Item $flowScheduler -Name "channel_stats" -Default $null
|
||||
if ($null -ne $channelStats) {
|
||||
$replacementFlowStats = @()
|
||||
foreach ($property in @($channelStats.PSObject.Properties)) {
|
||||
$stat = $property.Value
|
||||
if ([string](Get-PropertyValue -Item $stat -Name "last_route_id" -Default "") -eq $alternateRouteID) {
|
||||
$replacementFlowStats += $stat
|
||||
}
|
||||
}
|
||||
}
|
||||
if ($replacementLastSelected -eq $alternateRouteID -and $replacementFlowStats.Count -ge 1) {
|
||||
$replacementTrafficObserved = $true
|
||||
break
|
||||
}
|
||||
}
|
||||
if ($replacementTrafficObserved) {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
$postAccess = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/fabric/service-channels/access-telemetry?actor_user_id=$ActorUserID&limit=50").fabric_service_channel_access_telemetry
|
||||
$postChannel = @($postAccess.active_channels | Where-Object { $_.channel_id -eq $lease.channel_id }) | Select-Object -First 1
|
||||
|
||||
[void](Send-ReplacementDegradedHeartbeat -EntryNodeID $entryNode.id -ReplacementRouteID $alternateRouteID -RouteGeneration "$runId-primary")
|
||||
$noSafeSyntheticConfig = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/nodes/$($entryNode.id)/mesh/synthetic-config").synthetic_mesh_config
|
||||
$noSafePathDecisions = @()
|
||||
if ($noSafeSyntheticConfig.PSObject.Properties.Name -contains "route_path_decisions") {
|
||||
$noSafePathDecisions = @($noSafeSyntheticConfig.route_path_decisions.decisions)
|
||||
}
|
||||
$noSafeDecision = $noSafePathDecisions | Where-Object {
|
||||
[string](Get-PropertyValue -Item $_ -Name "route_id" -Default "") -eq $alternateRouteID -and
|
||||
[string](Get-PropertyValue -Item $_ -Name "decision_source" -Default "") -eq "service_channel_feedback_no_alternate"
|
||||
} | Select-Object -First 1
|
||||
$noSafeFeedback = $null
|
||||
$serviceChannelFeedback = Get-PropertyValue -Item $noSafeSyntheticConfig -Name "service_channel_feedback" -Default $null
|
||||
if ($null -ne $serviceChannelFeedback -and $serviceChannelFeedback.PSObject.Properties.Name -contains "routes") {
|
||||
$noSafeFeedback = @($serviceChannelFeedback.routes | Where-Object { [string](Get-PropertyValue -Item $_ -Name "route_id" -Default "") -eq $alternateRouteID }) | Select-Object -First 1
|
||||
}
|
||||
|
||||
$postNoSafeAccess = $null
|
||||
$postNoSafeChannel = $null
|
||||
for ($i = 0; $i -lt 12; $i++) {
|
||||
Start-Sleep -Seconds 3
|
||||
$postNoSafeAccess = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/fabric/service-channels/access-telemetry?actor_user_id=$ActorUserID&limit=50").fabric_service_channel_access_telemetry
|
||||
$postNoSafeChannel = @($postNoSafeAccess.active_channels | Where-Object { $_.channel_id -eq $lease.channel_id }) | Select-Object -First 1
|
||||
if ($null -ne $postNoSafeChannel -and [string](Get-PropertyValue -Item $postNoSafeChannel -Name "route_decision_source" -Default "") -eq "service_channel_feedback_no_alternate") {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
$backendLine = (& ssh $DockerSSH "docker ps --format '{{.Names}} {{.Image}} {{.Status}}' | grep '^rap_test_backend '") | Out-String
|
||||
$nodeLines = (& ssh $DockerSSH "docker ps --format '{{.Names}} {{.Image}} {{.Status}}' | grep '^rap_test_node_test_'") | Out-String
|
||||
$checks = [ordered]@{
|
||||
backend_expected_image_deployed = $backendLine.Contains($ExpectedBackendImage)
|
||||
node_agent_expected_image_deployed = $nodeLines.Contains($ExpectedNodeAgentImage)
|
||||
lease_ready = ([string]$lease.status -eq "ready")
|
||||
remediation_rebuild_route_visible = ($null -ne $matchingChannel -and [string](Get-PropertyValue -Item $matchingChannel -Name "remediation_action" -Default "") -eq "rebuild_route")
|
||||
remediation_command_visible = ($null -ne $command -and $commandID.Length -gt 0)
|
||||
durable_rebuild_intent_recorded = ($null -ne $attempt)
|
||||
durable_rebuild_intent_resolved_applied = ($null -ne $attempt -and [string](Get-PropertyValue -Item $attempt -Name "rebuild_status" -Default "") -eq "applied")
|
||||
durable_rebuild_intent_outcome_replacement_selected = ($null -ne $attempt -and [string](Get-PropertyValue -Item $attempt -Name "outcome" -Default "") -eq "replacement_selected")
|
||||
durable_rebuild_intent_replacement_matches = ($null -ne $attempt -and [string](Get-PropertyValue -Item $attempt -Name "replacement_route_id" -Default "") -eq $alternateRouteID)
|
||||
durable_rebuild_intent_source_matches = ($null -ne $attempt -and [string](Get-PropertyValue -Item $attempt -Name "decision_source" -Default "") -eq "service_channel_remediation_command")
|
||||
initial_access_telemetry_reports_planner_applied = ($null -ne $postChannel -and [string](Get-PropertyValue -Item $postChannel -Name "remediation_execution_status" -Default "") -eq "rebuild_request_applied")
|
||||
node_route_manager_consumed_same_command = ($null -ne $routeManagerDecision)
|
||||
node_route_manager_applied_rebuild = ($null -ne $routeManagerDecision -and [string](Get-PropertyValue -Item $routeManagerDecision -Name "rebuild_status" -Default "") -eq "applied")
|
||||
node_route_manager_transition_applied_rebuild = ($null -ne $routeManagerTransition -and [string](Get-PropertyValue -Item $routeManagerTransition -Name "status" -Default "") -eq "applied_rebuild")
|
||||
traffic_packet_accepted = ([int]$trafficResponse.StatusCode -eq 202)
|
||||
traffic_accepted_by_runtime = ($trafficAcceptedBy -eq "introspection" -or $trafficAcceptedBy -eq "signed")
|
||||
traffic_selected_replacement_route = $replacementTrafficObserved
|
||||
traffic_last_selected_route_matches_replacement = ($replacementLastSelected -eq $alternateRouteID)
|
||||
no_backend_or_local_fallback_after_replacement = (($postRemediationFallbackLocal - $baselineFallbackLocal) -eq 0)
|
||||
no_route_failures_after_replacement = (($postRemediationRouteFailures - $baselineRouteFailures) -eq 0)
|
||||
no_flow_drops_after_replacement = (($postRemediationFlowDropped - $baselineFlowDropped) -eq 0 -and ($postRemediationSchedulerDropped - $baselineSchedulerDropped) -eq 0)
|
||||
replacement_degradation_feedback_visible = ($null -ne $noSafeDecision -and @($noSafeDecision.score_reasons | Where-Object { [string]$_ -eq "service_channel_fenced_route" }).Count -gt 0)
|
||||
no_safe_recovery_decision_visible = ($null -ne $noSafeDecision)
|
||||
no_safe_recovery_status_pending_fallback = ($null -ne $noSafeDecision -and [string](Get-PropertyValue -Item $noSafeDecision -Name "rebuild_status" -Default "") -eq "pending_degraded_fallback")
|
||||
no_safe_recovery_reason_visible = ($null -ne $noSafeDecision -and @($noSafeDecision.score_reasons | Where-Object { [string]$_ -eq "no_unfenced_alternate_route" }).Count -gt 0)
|
||||
no_silent_stickiness_to_bad_replacement = ($null -ne $noSafeDecision -and [string](Get-PropertyValue -Item $noSafeDecision -Name "decision_source" -Default "") -eq "service_channel_feedback_no_alternate")
|
||||
access_telemetry_projects_no_safe_decision = ($null -ne $postNoSafeChannel -and [string](Get-PropertyValue -Item $postNoSafeChannel -Name "route_decision_source" -Default "") -eq "service_channel_feedback_no_alternate")
|
||||
access_telemetry_projects_pending_fallback = ($null -ne $postNoSafeChannel -and [string](Get-PropertyValue -Item $postNoSafeChannel -Name "route_decision_rebuild_status" -Default "") -eq "pending_degraded_fallback")
|
||||
access_telemetry_projects_no_safe_route_id = ($null -ne $postNoSafeChannel -and @($primaryRouteID, $alternateRouteID) -contains [string](Get-PropertyValue -Item $postNoSafeChannel -Name "route_decision_route_id" -Default ""))
|
||||
access_telemetry_projects_operator_state = ($null -ne $postNoSafeChannel -and @("route_rebuild_no_safe_recovery", "rebuild_request_no_alternate") -contains [string](Get-PropertyValue -Item $postNoSafeChannel -Name "remediation_execution_status" -Default ""))
|
||||
}
|
||||
$failed = @($checks.GetEnumerator() | Where-Object { -not $_.Value } | ForEach-Object { $_.Key })
|
||||
|
||||
$result = [ordered]@{
|
||||
schema_version = "c18z83.service_channel_access_telemetry_no_safe_smoke.v1"
|
||||
run_id = $runId
|
||||
cluster_id = $ClusterID
|
||||
channel_id = [string]$lease.channel_id
|
||||
primary_route_id = $primaryRouteID
|
||||
alternate_route_id = $alternateRouteID
|
||||
rebuild_request_id = $commandID
|
||||
passed = ($failed.Count -eq 0)
|
||||
checks = $checks
|
||||
failed_checks = $failed
|
||||
summary = [ordered]@{
|
||||
backend_container = $backendLine.Trim()
|
||||
node_containers = $nodeLines.Trim()
|
||||
remediation_command = $command
|
||||
rebuild_attempt = $attempt
|
||||
route_manager_decision = $routeManagerDecision
|
||||
route_manager_transition = $routeManagerTransition
|
||||
traffic_status_code = [int]$trafficResponse.StatusCode
|
||||
traffic_accepted_by = $trafficAcceptedBy
|
||||
traffic_duration_ms = $trafficDurationMs
|
||||
traffic_packet_count = $trafficPackets.Count
|
||||
replacement_last_selected_route_id = $replacementLastSelected
|
||||
replacement_flow_stat_count = $replacementFlowStats.Count
|
||||
replacement_flow_stats = $replacementFlowStats
|
||||
baseline_send_fallback_local = $baselineFallbackLocal
|
||||
baseline_send_route_failures = $baselineRouteFailures
|
||||
baseline_send_flow_dropped = $baselineFlowDropped
|
||||
baseline_scheduler_dropped = $baselineSchedulerDropped
|
||||
fallback_local_delta = ($postRemediationFallbackLocal - $baselineFallbackLocal)
|
||||
route_failure_delta = ($postRemediationRouteFailures - $baselineRouteFailures)
|
||||
flow_drop_delta = ($postRemediationFlowDropped - $baselineFlowDropped)
|
||||
scheduler_drop_delta = ($postRemediationSchedulerDropped - $baselineSchedulerDropped)
|
||||
post_remediation_send_fallback_local = $postRemediationFallbackLocal
|
||||
post_remediation_send_route_failures = $postRemediationRouteFailures
|
||||
post_remediation_send_flow_dropped = $postRemediationFlowDropped
|
||||
post_remediation_scheduler_dropped = $postRemediationSchedulerDropped
|
||||
post_channel = $postChannel
|
||||
no_safe_feedback = $noSafeFeedback
|
||||
no_safe_decision = $noSafeDecision
|
||||
no_safe_route_path_decisions = $noSafePathDecisions
|
||||
post_no_safe_channel = $postNoSafeChannel
|
||||
}
|
||||
}
|
||||
|
||||
$target = Join-Path $repoRoot $ResultPath
|
||||
$result | ConvertTo-Json -Depth 80 | Set-Content -Path $target -Encoding UTF8
|
||||
|
||||
try {
|
||||
if ($primaryRouteID) {
|
||||
Invoke-Api -Method POST -Path "/clusters/$ClusterID/mesh/route-intents/$primaryRouteID/expire" -Body @{ actor_user_id = $ActorUserID } | Out-Null
|
||||
}
|
||||
if ($alternateRouteID) {
|
||||
Invoke-Api -Method POST -Path "/clusters/$ClusterID/mesh/route-intents/$alternateRouteID/expire" -Body @{ actor_user_id = $ActorUserID } | Out-Null
|
||||
}
|
||||
Start-Sleep -Seconds 3
|
||||
Invoke-Api -Method POST -Path "/clusters/$ClusterID/fabric/service-channels/leases/cleanup" -Body @{
|
||||
actor_user_id = $ActorUserID
|
||||
limit = 50
|
||||
} | Out-Null
|
||||
} catch {
|
||||
Write-Warning "cleanup failed after c18z83 smoke: $($_.Exception.Message)"
|
||||
}
|
||||
|
||||
if (-not $result.passed) {
|
||||
throw "C18Z83 access-telemetry no-safe smoke failed: $($failed -join ', ')"
|
||||
}
|
||||
|
||||
Write-Host "C18Z83 service-channel access-telemetry no-safe smoke passed. Result: $target"
|
||||
$result
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,526 @@
|
||||
param(
|
||||
[string]$ApiBaseUrl = "http://192.168.200.61:18121/api/v1",
|
||||
[string]$ClusterID = "cfc0743d-d960-49fb-9de8-96e063d5e4aa",
|
||||
[string]$ActorUserID = "f67d943f-5397-4b3a-a229-695fe67ad700",
|
||||
[string]$EntryNodeName = "test-1",
|
||||
[string]$RelayNodeName = "test-3",
|
||||
[string]$ExitNodeName = "test-2",
|
||||
[string]$EntryBaseUrl = "http://192.168.200.61:19131",
|
||||
[string]$DockerSSH = "test-docker",
|
||||
[string]$ExpectedBackendImage = "rap-backend:fabric-service-channel-0.2.281-c18z109",
|
||||
[string]$ExpectedNodeAgentImage = "rap-node-agent:0.2.270-c18z95",
|
||||
[string]$ResultPath = "artifacts\c18z84-service-channel-access-decision-aggregate-smoke-result.json"
|
||||
)
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
$scriptDir = Split-Path -Parent $MyInvocation.MyCommand.Path
|
||||
$repoRoot = (Resolve-Path (Join-Path $scriptDir "..\..")).ProviderPath
|
||||
$runId = "c18z84-" + (Get-Date -Format "yyyyMMdd-HHmmss")
|
||||
|
||||
function Invoke-Api {
|
||||
param([string]$Method, [string]$Path, [object]$Body = $null)
|
||||
$params = @{ Method = $Method; Uri = "$ApiBaseUrl$Path"; TimeoutSec = 30 }
|
||||
if ($null -ne $Body) {
|
||||
$params.ContentType = "application/json"
|
||||
$params.Body = ($Body | ConvertTo-Json -Depth 80)
|
||||
}
|
||||
return Invoke-RestMethod @params
|
||||
}
|
||||
|
||||
function Get-PropertyValue {
|
||||
param([object]$Item, [string]$Name, [object]$Default = $null)
|
||||
if ($null -eq $Item) { return $Default }
|
||||
$property = $Item.PSObject.Properties[$Name]
|
||||
if ($null -eq $property) { return $Default }
|
||||
return $property.Value
|
||||
}
|
||||
|
||||
function Get-NodeByName {
|
||||
param([string]$Name)
|
||||
$nodes = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/nodes?actor_user_id=$ActorUserID").nodes
|
||||
$node = @($nodes | Where-Object { $_.name -eq $Name }) | Select-Object -First 1
|
||||
if ($null -eq $node) { throw "Node '$Name' was not found" }
|
||||
return $node
|
||||
}
|
||||
|
||||
function New-IPv4TcpPacket {
|
||||
param(
|
||||
[byte[]]$Source = @(10, 77, 0, 2),
|
||||
[byte[]]$Destination = @(192, 168, 200, 95),
|
||||
[int]$SourcePort,
|
||||
[int]$DestinationPort = 3389
|
||||
)
|
||||
$packet = [byte[]]::new(40)
|
||||
$packet[0] = 0x45
|
||||
$packet[2] = 0
|
||||
$packet[3] = 40
|
||||
$packet[8] = 64
|
||||
$packet[9] = 6
|
||||
[Array]::Copy($Source, 0, $packet, 12, 4)
|
||||
[Array]::Copy($Destination, 0, $packet, 16, 4)
|
||||
$packet[20] = [byte](($SourcePort -shr 8) -band 0xff)
|
||||
$packet[21] = [byte]($SourcePort -band 0xff)
|
||||
$packet[22] = [byte](($DestinationPort -shr 8) -band 0xff)
|
||||
$packet[23] = [byte]($DestinationPort -band 0xff)
|
||||
$packet[32] = 0x50
|
||||
return $packet
|
||||
}
|
||||
|
||||
function ConvertTo-VPNPacketBatch {
|
||||
param([byte[][]]$Packets)
|
||||
$buffer = New-Object System.Collections.Generic.List[byte]
|
||||
foreach ($packet in $Packets) {
|
||||
if ($null -eq $packet -or $packet.Length -eq 0) { continue }
|
||||
$size = [uint32]$packet.Length
|
||||
$buffer.Add([byte](($size -shr 24) -band 0xff))
|
||||
$buffer.Add([byte](($size -shr 16) -band 0xff))
|
||||
$buffer.Add([byte](($size -shr 8) -band 0xff))
|
||||
$buffer.Add([byte]($size -band 0xff))
|
||||
foreach ($value in $packet) {
|
||||
$buffer.Add($value)
|
||||
}
|
||||
}
|
||||
return $buffer.ToArray()
|
||||
}
|
||||
|
||||
function New-RouteIntent {
|
||||
param([string]$SourceNodeID, [string]$DestinationNodeID, [string[]]$Hops)
|
||||
$expiresAt = (Get-Date).ToUniversalTime().AddMinutes(5).ToString("o")
|
||||
return (Invoke-Api -Method POST -Path "/clusters/$ClusterID/mesh/route-intents" -Body @{
|
||||
actor_user_id = $ActorUserID
|
||||
source_selector = @{ node_id = $SourceNodeID }
|
||||
destination_selector = @{ node_id = $DestinationNodeID }
|
||||
service_class = "vpn_packets"
|
||||
priority = 2100000000
|
||||
policy = @{
|
||||
synthetic_enabled = $true
|
||||
route_version = "$runId-primary"
|
||||
policy_version = "$runId-primary"
|
||||
peer_directory_version = "$runId-primary"
|
||||
hops = @($Hops)
|
||||
allowed_channels = @("vpn_packet", "fabric_control")
|
||||
max_ttl = 8
|
||||
max_hops = 8
|
||||
expires_at = $expiresAt
|
||||
metadata = @{ smoke = "c18z82_service_channel_no_safe_recovery"; run_id = $runId }
|
||||
}
|
||||
}).route_intent
|
||||
}
|
||||
|
||||
function Disable-ExistingRouteIntents {
|
||||
param([string]$SourceNodeID, [string]$DestinationNodeID)
|
||||
$items = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/mesh/route-intents?actor_user_id=$ActorUserID").route_intents
|
||||
foreach ($item in @($items)) {
|
||||
if ([string](Get-PropertyValue -Item $item -Name "status" -Default "") -ne "active") { continue }
|
||||
if ([string](Get-PropertyValue -Item $item -Name "service_class" -Default "") -ne "vpn_packets") { continue }
|
||||
$sourceSelector = Get-PropertyValue -Item $item -Name "source_selector" -Default $null
|
||||
$destinationSelector = Get-PropertyValue -Item $item -Name "destination_selector" -Default $null
|
||||
if ([string](Get-PropertyValue -Item $sourceSelector -Name "node_id" -Default "") -ne $SourceNodeID) { continue }
|
||||
if ([string](Get-PropertyValue -Item $destinationSelector -Name "node_id" -Default "") -ne $DestinationNodeID) { continue }
|
||||
[void](Invoke-Api -Method POST -Path "/clusters/$ClusterID/mesh/route-intents/$($item.id)/disable" -Body @{
|
||||
actor_user_id = $ActorUserID
|
||||
reason = "c18z82 isolate no-safe-recovery smoke route pair"
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
function Send-DegradedHeartbeat {
|
||||
param([string]$EntryNodeID, [string]$PrimaryRouteID)
|
||||
$observedAt = (Get-Date).ToUniversalTime().ToString("o")
|
||||
return Invoke-Api -Method POST -Path "/clusters/$ClusterID/nodes/$EntryNodeID/heartbeats" -Body @{
|
||||
health_status = "healthy"
|
||||
reported_version = "0.2.256-c18z82"
|
||||
capabilities = @{
|
||||
fabric_service_channel_runtime = $true
|
||||
fabric_service_channel_route_quality_feedback = $true
|
||||
smoke_feedback_injection = "c18z77"
|
||||
}
|
||||
service_states = @{ smoke = "c18z82_primary_degraded_alternate_after_lease" }
|
||||
metadata = @{
|
||||
fabric_service_channel_runtime_report = @{
|
||||
schema_version = "c18l.fabric_service_channel_runtime_report.v1"
|
||||
config_version = "$runId-primary"
|
||||
cluster_id = $ClusterID
|
||||
local_node_id = $EntryNodeID
|
||||
observed_at = $observedAt
|
||||
ingress = @{
|
||||
flow_scheduler = @{
|
||||
schema_version = "rap.fabric_flow_scheduler.v1"
|
||||
service_neutral = $true
|
||||
service_mode = "application_protocol_agnostic"
|
||||
channel_stats = @{
|
||||
"c18z82-primary-degraded" = @{
|
||||
last_route_id = $PrimaryRouteID
|
||||
last_failed_route_id = $PrimaryRouteID
|
||||
route_generation = "$runId-primary"
|
||||
last_error = "c18z82 primary route degraded; alternate added after lease"
|
||||
last_send_duration_ms = 1200
|
||||
consecutive_failures = 3
|
||||
stall_count = 2
|
||||
route_rebuild_recommended = $true
|
||||
degraded_fallback_recommended = $false
|
||||
quality_window_sample_count = 8
|
||||
quality_window_success_count = 2
|
||||
quality_window_failure_count = 3
|
||||
quality_window_slow_count = 2
|
||||
quality_window_drop_count = 0
|
||||
quality_window_avg_latency_ms = 1200
|
||||
quality_window_last_updated_at = $observedAt
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
smoke = @{ name = "c18z82_service_channel_no_safe_recovery"; run_id = $runId }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function Send-ReplacementDegradedHeartbeat {
|
||||
param([string]$EntryNodeID, [string]$ReplacementRouteID, [string]$RouteGeneration)
|
||||
$observedAt = (Get-Date).ToUniversalTime().ToString("o")
|
||||
return Invoke-Api -Method POST -Path "/clusters/$ClusterID/nodes/$EntryNodeID/heartbeats" -Body @{
|
||||
health_status = "healthy"
|
||||
reported_version = "0.2.256-c18z82"
|
||||
capabilities = @{
|
||||
fabric_service_channel_runtime = $true
|
||||
fabric_service_channel_route_quality_feedback = $true
|
||||
smoke_feedback_injection = "c18z82"
|
||||
}
|
||||
service_states = @{ smoke = "c18z82_replacement_route_degraded_no_safe_recovery" }
|
||||
metadata = @{
|
||||
fabric_service_channel_runtime_report = @{
|
||||
schema_version = "c18l.fabric_service_channel_runtime_report.v1"
|
||||
config_version = "$runId-replacement-degraded"
|
||||
cluster_id = $ClusterID
|
||||
local_node_id = $EntryNodeID
|
||||
observed_at = $observedAt
|
||||
ingress = @{
|
||||
last_selected_route_id = $ReplacementRouteID
|
||||
send_route_failures = 4
|
||||
flow_scheduler = @{
|
||||
schema_version = "rap.fabric_flow_scheduler.v1"
|
||||
service_neutral = $true
|
||||
service_mode = "application_protocol_agnostic"
|
||||
channel_stats = @{
|
||||
"c18z82-replacement-degraded" = @{
|
||||
last_route_id = $ReplacementRouteID
|
||||
last_failed_route_id = $ReplacementRouteID
|
||||
route_generation = $RouteGeneration
|
||||
last_error = "c18z82 replacement route degraded; no safe recovery route exists"
|
||||
last_send_duration_ms = 1500
|
||||
consecutive_failures = 4
|
||||
stall_count = 2
|
||||
route_rebuild_recommended = $true
|
||||
degraded_fallback_recommended = $true
|
||||
quality_window_sample_count = 10
|
||||
quality_window_success_count = 1
|
||||
quality_window_failure_count = 4
|
||||
quality_window_slow_count = 3
|
||||
quality_window_drop_count = 0
|
||||
quality_window_avg_latency_ms = 1500
|
||||
quality_window_last_updated_at = $observedAt
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
smoke = @{ name = "c18z82_service_channel_no_safe_recovery"; run_id = $runId }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
$entryNode = Get-NodeByName -Name $EntryNodeName
|
||||
$relayNode = Get-NodeByName -Name $RelayNodeName
|
||||
$exitNode = Get-NodeByName -Name $ExitNodeName
|
||||
[void](Disable-ExistingRouteIntents -SourceNodeID $entryNode.id -DestinationNodeID $exitNode.id)
|
||||
$route = New-RouteIntent -SourceNodeID $entryNode.id -DestinationNodeID $exitNode.id -Hops @($entryNode.id, $exitNode.id)
|
||||
$lease = (Invoke-Api -Method POST -Path "/clusters/$ClusterID/fabric/service-channels/leases" -Body @{
|
||||
actor_user_id = $ActorUserID
|
||||
organization_id = "smoke-org"
|
||||
user_id = "smoke-user"
|
||||
resource_id = "c18z82-vpn-smoke"
|
||||
service_class = "vpn_packets"
|
||||
entry_node_ids = @([string]$entryNode.id)
|
||||
exit_node_ids = @([string]$exitNode.id)
|
||||
preferred_entry_node_id = [string]$entryNode.id
|
||||
preferred_exit_node_id = [string]$exitNode.id
|
||||
allowed_channels = @("vpn_packet", "fabric_control")
|
||||
ttl_seconds = 180
|
||||
metadata = @{ smoke = "c18z82_service_channel_no_safe_recovery"; run_id = $runId }
|
||||
}).fabric_service_channel_lease
|
||||
$primaryRouteID = [string](Get-PropertyValue -Item (Get-PropertyValue -Item $lease -Name "primary_route" -Default $null) -Name "route_id" -Default $route.id)
|
||||
|
||||
[void](Send-DegradedHeartbeat -EntryNodeID $entryNode.id -PrimaryRouteID $primaryRouteID)
|
||||
|
||||
$matchingChannel = $null
|
||||
for ($i = 0; $i -lt 10; $i++) {
|
||||
Start-Sleep -Seconds 3
|
||||
$accessTelemetry = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/fabric/service-channels/access-telemetry?actor_user_id=$ActorUserID&limit=50").fabric_service_channel_access_telemetry
|
||||
$matchingChannel = @($accessTelemetry.active_channels | Where-Object { $_.channel_id -eq $lease.channel_id }) | Select-Object -First 1
|
||||
if ($null -ne $matchingChannel -and [string](Get-PropertyValue -Item $matchingChannel -Name "remediation_action" -Default "") -eq "rebuild_route") {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
$command = Get-PropertyValue -Item $matchingChannel -Name "remediation_command" -Default $null
|
||||
$commandID = [string](Get-PropertyValue -Item $command -Name "command_id" -Default "")
|
||||
$alternate = New-RouteIntent -SourceNodeID $entryNode.id -DestinationNodeID $exitNode.id -Hops @($entryNode.id, $relayNode.id, $exitNode.id)
|
||||
$alternateRouteID = [string]$alternate.id
|
||||
[void](Invoke-Api -Method GET -Path "/clusters/$ClusterID/nodes/$($entryNode.id)/mesh/synthetic-config")
|
||||
|
||||
$attempts = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/fabric/service-channels/rebuild-attempts?actor_user_id=$ActorUserID&reporter_node_id=$($entryNode.id)&rebuild_request_id=$commandID&limit=10").rebuild_attempts
|
||||
$attempt = @($attempts | Where-Object { $_.rebuild_request_id -eq $commandID }) | Select-Object -First 1
|
||||
|
||||
$routeManagerDecision = $null
|
||||
$routeManagerTransition = $null
|
||||
for ($i = 0; $i -lt 12; $i++) {
|
||||
Start-Sleep -Seconds 5
|
||||
$heartbeats = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/nodes/$($entryNode.id)/heartbeats?actor_user_id=$ActorUserID&limit=5").heartbeats
|
||||
foreach ($heartbeat in @($heartbeats)) {
|
||||
$runtimeReport = Get-PropertyValue -Item (Get-PropertyValue -Item $heartbeat -Name "metadata" -Default $null) -Name "fabric_service_channel_runtime_report" -Default $null
|
||||
$ingress = Get-PropertyValue -Item $runtimeReport -Name "ingress" -Default $null
|
||||
$routeManager = Get-PropertyValue -Item $ingress -Name "route_manager" -Default $null
|
||||
$routeManagerTransition = Get-PropertyValue -Item $ingress -Name "route_manager_transition" -Default $null
|
||||
$decisions = @()
|
||||
if ($null -ne $routeManager -and $routeManager.PSObject.Properties.Name -contains "decisions") {
|
||||
$decisions = @($routeManager.decisions)
|
||||
}
|
||||
$routeManagerDecision = $decisions | Where-Object {
|
||||
[string](Get-PropertyValue -Item $_ -Name "rebuild_request_id" -Default "") -eq $commandID -and
|
||||
[string](Get-PropertyValue -Item $_ -Name "route_id" -Default "") -eq $primaryRouteID -and
|
||||
[string](Get-PropertyValue -Item $_ -Name "replacement_route_id" -Default "") -eq $alternateRouteID
|
||||
} | Select-Object -First 1
|
||||
if ($null -ne $routeManagerDecision) {
|
||||
break
|
||||
}
|
||||
}
|
||||
if ($null -ne $routeManagerDecision) {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
$packetPath = $lease.entry_http.path_template.
|
||||
Replace("{cluster_id}", $ClusterID).
|
||||
Replace("{channel_id}", [string]$lease.channel_id).
|
||||
Replace("{resource_id}", "c18z82-vpn-smoke")
|
||||
$packetUrl = $EntryBaseUrl.TrimEnd("/") + $packetPath
|
||||
$headers = @{
|
||||
"X-RAP-Service-Channel-Token" = [string]$lease.token.token
|
||||
"X-RAP-Fabric-Channel-ID" = [string]$lease.channel_id
|
||||
"X-RAP-Service-Class" = "vpn_packets"
|
||||
"X-RAP-Channel-Class" = "vpn_packet"
|
||||
"X-RAP-Traffic-Class" = "interactive"
|
||||
}
|
||||
$baselineFallbackLocal = 0
|
||||
$baselineRouteFailures = 0
|
||||
$baselineFlowDropped = 0
|
||||
$baselineSchedulerDropped = 0
|
||||
$baselineHeartbeats = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/nodes/$($entryNode.id)/heartbeats?actor_user_id=$ActorUserID&limit=1").heartbeats
|
||||
if (@($baselineHeartbeats).Count -gt 0) {
|
||||
$baselineRuntimeReport = Get-PropertyValue -Item (Get-PropertyValue -Item $baselineHeartbeats[0] -Name "metadata" -Default $null) -Name "fabric_service_channel_runtime_report" -Default $null
|
||||
$baselineIngress = Get-PropertyValue -Item $baselineRuntimeReport -Name "ingress" -Default $null
|
||||
$baselineFlowScheduler = Get-PropertyValue -Item $baselineIngress -Name "flow_scheduler" -Default $null
|
||||
$baselineFallbackLocal = [int](Get-PropertyValue -Item $baselineIngress -Name "send_fallback_local" -Default 0)
|
||||
$baselineRouteFailures = [int](Get-PropertyValue -Item $baselineIngress -Name "send_route_failures" -Default 0)
|
||||
$baselineFlowDropped = [int](Get-PropertyValue -Item $baselineIngress -Name "send_flow_dropped" -Default 0)
|
||||
$baselineSchedulerDropped = [int](Get-PropertyValue -Item $baselineFlowScheduler -Name "dropped" -Default 0)
|
||||
}
|
||||
$trafficPackets = @()
|
||||
foreach ($offset in 0..15) {
|
||||
$trafficPackets += ,(New-IPv4TcpPacket -SourcePort (54000 + $offset) -DestinationPort 3389)
|
||||
}
|
||||
$trafficPayload = ConvertTo-VPNPacketBatch -Packets $trafficPackets
|
||||
$trafficPayloadPath = Join-Path ([System.IO.Path]::GetTempPath()) "$runId-vpn-packet-batch.bin"
|
||||
[System.IO.File]::WriteAllBytes($trafficPayloadPath, [byte[]]$trafficPayload)
|
||||
$trafficStarted = Get-Date
|
||||
$trafficResponse = Invoke-WebRequest -Method Post -Uri ($packetUrl + "?batch=true") -Headers $headers -InFile $trafficPayloadPath -ContentType "application/vnd.rap.vpn-packet-batch.v1" -TimeoutSec 30
|
||||
$trafficDurationMs = [int]((Get-Date) - $trafficStarted).TotalMilliseconds
|
||||
$trafficAcceptedBy = [string]$trafficResponse.Headers["X-RAP-Service-Channel-Accepted-By"]
|
||||
Remove-Item -Path $trafficPayloadPath -Force -ErrorAction SilentlyContinue
|
||||
|
||||
$replacementTrafficObserved = $false
|
||||
$replacementLastSelected = ""
|
||||
$replacementFlowStats = @()
|
||||
$postRemediationFallbackLocal = 0
|
||||
$postRemediationRouteFailures = 0
|
||||
$postRemediationFlowDropped = 0
|
||||
$postRemediationSchedulerDropped = 0
|
||||
for ($i = 0; $i -lt 12; $i++) {
|
||||
Start-Sleep -Seconds 5
|
||||
$heartbeats = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/nodes/$($entryNode.id)/heartbeats?actor_user_id=$ActorUserID&limit=5").heartbeats
|
||||
foreach ($heartbeat in @($heartbeats)) {
|
||||
$runtimeReport = Get-PropertyValue -Item (Get-PropertyValue -Item $heartbeat -Name "metadata" -Default $null) -Name "fabric_service_channel_runtime_report" -Default $null
|
||||
$ingress = Get-PropertyValue -Item $runtimeReport -Name "ingress" -Default $null
|
||||
$replacementLastSelected = [string](Get-PropertyValue -Item $ingress -Name "last_selected_route_id" -Default "")
|
||||
$postRemediationFallbackLocal = [int](Get-PropertyValue -Item $ingress -Name "send_fallback_local" -Default 0)
|
||||
$postRemediationRouteFailures = [int](Get-PropertyValue -Item $ingress -Name "send_route_failures" -Default 0)
|
||||
$postRemediationFlowDropped = [int](Get-PropertyValue -Item $ingress -Name "send_flow_dropped" -Default 0)
|
||||
$flowScheduler = Get-PropertyValue -Item $ingress -Name "flow_scheduler" -Default $null
|
||||
$postRemediationSchedulerDropped = [int](Get-PropertyValue -Item $flowScheduler -Name "dropped" -Default 0)
|
||||
$channelStats = Get-PropertyValue -Item $flowScheduler -Name "channel_stats" -Default $null
|
||||
if ($null -ne $channelStats) {
|
||||
$replacementFlowStats = @()
|
||||
foreach ($property in @($channelStats.PSObject.Properties)) {
|
||||
$stat = $property.Value
|
||||
if ([string](Get-PropertyValue -Item $stat -Name "last_route_id" -Default "") -eq $alternateRouteID) {
|
||||
$replacementFlowStats += $stat
|
||||
}
|
||||
}
|
||||
}
|
||||
if ($replacementLastSelected -eq $alternateRouteID -and $replacementFlowStats.Count -ge 1) {
|
||||
$replacementTrafficObserved = $true
|
||||
break
|
||||
}
|
||||
}
|
||||
if ($replacementTrafficObserved) {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
$postAccess = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/fabric/service-channels/access-telemetry?actor_user_id=$ActorUserID&limit=50").fabric_service_channel_access_telemetry
|
||||
$postChannel = @($postAccess.active_channels | Where-Object { $_.channel_id -eq $lease.channel_id }) | Select-Object -First 1
|
||||
|
||||
[void](Send-ReplacementDegradedHeartbeat -EntryNodeID $entryNode.id -ReplacementRouteID $alternateRouteID -RouteGeneration "$runId-primary")
|
||||
$noSafeSyntheticConfig = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/nodes/$($entryNode.id)/mesh/synthetic-config").synthetic_mesh_config
|
||||
$noSafePathDecisions = @()
|
||||
if ($noSafeSyntheticConfig.PSObject.Properties.Name -contains "route_path_decisions") {
|
||||
$noSafePathDecisions = @($noSafeSyntheticConfig.route_path_decisions.decisions)
|
||||
}
|
||||
$noSafeDecision = $noSafePathDecisions | Where-Object {
|
||||
[string](Get-PropertyValue -Item $_ -Name "route_id" -Default "") -eq $alternateRouteID -and
|
||||
[string](Get-PropertyValue -Item $_ -Name "decision_source" -Default "") -eq "service_channel_feedback_no_alternate"
|
||||
} | Select-Object -First 1
|
||||
$noSafeFeedback = $null
|
||||
$serviceChannelFeedback = Get-PropertyValue -Item $noSafeSyntheticConfig -Name "service_channel_feedback" -Default $null
|
||||
if ($null -ne $serviceChannelFeedback -and $serviceChannelFeedback.PSObject.Properties.Name -contains "routes") {
|
||||
$noSafeFeedback = @($serviceChannelFeedback.routes | Where-Object { [string](Get-PropertyValue -Item $_ -Name "route_id" -Default "") -eq $alternateRouteID }) | Select-Object -First 1
|
||||
}
|
||||
|
||||
$postNoSafeAccess = $null
|
||||
$postNoSafeChannel = $null
|
||||
for ($i = 0; $i -lt 12; $i++) {
|
||||
Start-Sleep -Seconds 3
|
||||
$postNoSafeAccess = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/fabric/service-channels/access-telemetry?actor_user_id=$ActorUserID&limit=50").fabric_service_channel_access_telemetry
|
||||
$postNoSafeChannel = @($postNoSafeAccess.active_channels | Where-Object { $_.channel_id -eq $lease.channel_id }) | Select-Object -First 1
|
||||
if ($null -ne $postNoSafeChannel -and [string](Get-PropertyValue -Item $postNoSafeChannel -Name "route_decision_source" -Default "") -eq "service_channel_feedback_no_alternate") {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
$backendLine = (& ssh $DockerSSH "docker ps --format '{{.Names}} {{.Image}} {{.Status}}' | grep '^rap_test_backend '") | Out-String
|
||||
$nodeLines = (& ssh $DockerSSH "docker ps --format '{{.Names}} {{.Image}} {{.Status}}' | grep '^rap_test_node_test_'") | Out-String
|
||||
$checks = [ordered]@{
|
||||
backend_expected_image_deployed = $backendLine.Contains($ExpectedBackendImage)
|
||||
node_agent_expected_image_deployed = $nodeLines.Contains($ExpectedNodeAgentImage)
|
||||
lease_ready = ([string]$lease.status -eq "ready")
|
||||
remediation_rebuild_route_visible = ($null -ne $matchingChannel -and [string](Get-PropertyValue -Item $matchingChannel -Name "remediation_action" -Default "") -eq "rebuild_route")
|
||||
remediation_command_visible = ($null -ne $command -and $commandID.Length -gt 0)
|
||||
durable_rebuild_intent_recorded = ($null -ne $attempt)
|
||||
durable_rebuild_intent_resolved_applied = ($null -ne $attempt -and [string](Get-PropertyValue -Item $attempt -Name "rebuild_status" -Default "") -eq "applied")
|
||||
durable_rebuild_intent_outcome_replacement_selected = ($null -ne $attempt -and [string](Get-PropertyValue -Item $attempt -Name "outcome" -Default "") -eq "replacement_selected")
|
||||
durable_rebuild_intent_replacement_matches = ($null -ne $attempt -and [string](Get-PropertyValue -Item $attempt -Name "replacement_route_id" -Default "") -eq $alternateRouteID)
|
||||
durable_rebuild_intent_source_matches = ($null -ne $attempt -and [string](Get-PropertyValue -Item $attempt -Name "decision_source" -Default "") -eq "service_channel_remediation_command")
|
||||
initial_access_telemetry_reports_planner_applied = ($null -ne $postChannel -and [string](Get-PropertyValue -Item $postChannel -Name "remediation_execution_status" -Default "") -eq "rebuild_request_applied")
|
||||
node_route_manager_consumed_same_command = ($null -ne $routeManagerDecision)
|
||||
node_route_manager_applied_rebuild = ($null -ne $routeManagerDecision -and [string](Get-PropertyValue -Item $routeManagerDecision -Name "rebuild_status" -Default "") -eq "applied")
|
||||
node_route_manager_transition_applied_rebuild = ($null -ne $routeManagerTransition -and [string](Get-PropertyValue -Item $routeManagerTransition -Name "status" -Default "") -eq "applied_rebuild")
|
||||
traffic_packet_accepted = ([int]$trafficResponse.StatusCode -eq 202)
|
||||
traffic_accepted_by_runtime = ($trafficAcceptedBy -eq "introspection" -or $trafficAcceptedBy -eq "signed")
|
||||
traffic_selected_replacement_route = $replacementTrafficObserved
|
||||
traffic_last_selected_route_matches_replacement = ($replacementLastSelected -eq $alternateRouteID)
|
||||
no_backend_or_local_fallback_after_replacement = (($postRemediationFallbackLocal - $baselineFallbackLocal) -eq 0)
|
||||
no_route_failures_after_replacement = (($postRemediationRouteFailures - $baselineRouteFailures) -eq 0)
|
||||
no_flow_drops_after_replacement = (($postRemediationFlowDropped - $baselineFlowDropped) -eq 0 -and ($postRemediationSchedulerDropped - $baselineSchedulerDropped) -eq 0)
|
||||
replacement_degradation_feedback_visible = ($null -ne $noSafeDecision -and @($noSafeDecision.score_reasons | Where-Object { [string]$_ -eq "service_channel_fenced_route" }).Count -gt 0)
|
||||
no_safe_recovery_decision_visible = ($null -ne $noSafeDecision)
|
||||
no_safe_recovery_status_pending_fallback = ($null -ne $noSafeDecision -and [string](Get-PropertyValue -Item $noSafeDecision -Name "rebuild_status" -Default "") -eq "pending_degraded_fallback")
|
||||
no_safe_recovery_reason_visible = ($null -ne $noSafeDecision -and @($noSafeDecision.score_reasons | Where-Object { [string]$_ -eq "no_unfenced_alternate_route" }).Count -gt 0)
|
||||
no_silent_stickiness_to_bad_replacement = ($null -ne $noSafeDecision -and [string](Get-PropertyValue -Item $noSafeDecision -Name "decision_source" -Default "") -eq "service_channel_feedback_no_alternate")
|
||||
access_telemetry_projects_no_safe_decision = ($null -ne $postNoSafeChannel -and [string](Get-PropertyValue -Item $postNoSafeChannel -Name "route_decision_source" -Default "") -eq "service_channel_feedback_no_alternate")
|
||||
access_telemetry_projects_pending_fallback = ($null -ne $postNoSafeChannel -and [string](Get-PropertyValue -Item $postNoSafeChannel -Name "route_decision_rebuild_status" -Default "") -eq "pending_degraded_fallback")
|
||||
access_telemetry_projects_no_safe_route_id = ($null -ne $postNoSafeChannel -and @($primaryRouteID, $alternateRouteID) -contains [string](Get-PropertyValue -Item $postNoSafeChannel -Name "route_decision_route_id" -Default ""))
|
||||
access_telemetry_projects_operator_state = ($null -ne $postNoSafeChannel -and @("route_rebuild_no_safe_recovery", "rebuild_request_no_alternate") -contains [string](Get-PropertyValue -Item $postNoSafeChannel -Name "remediation_execution_status" -Default ""))
|
||||
access_telemetry_aggregates_route_decision = ($null -ne $postNoSafeAccess -and [int](Get-PropertyValue -Item $postNoSafeAccess -Name "route_decision_channel_count" -Default 0) -ge 1)
|
||||
access_telemetry_aggregates_no_safe_decision = ($null -ne $postNoSafeAccess -and [int](Get-PropertyValue -Item $postNoSafeAccess -Name "no_safe_recovery_decision_count" -Default 0) -ge 1)
|
||||
access_telemetry_aggregate_status_prioritizes_no_safe = ($null -ne $postNoSafeAccess -and [string](Get-PropertyValue -Item $postNoSafeAccess -Name "reason" -Default "") -eq "active_channels_no_safe_recovery")
|
||||
}
|
||||
$failed = @($checks.GetEnumerator() | Where-Object { -not $_.Value } | ForEach-Object { $_.Key })
|
||||
|
||||
$result = [ordered]@{
|
||||
schema_version = "c18z84.service_channel_access_decision_aggregate_smoke.v1"
|
||||
run_id = $runId
|
||||
cluster_id = $ClusterID
|
||||
channel_id = [string]$lease.channel_id
|
||||
primary_route_id = $primaryRouteID
|
||||
alternate_route_id = $alternateRouteID
|
||||
rebuild_request_id = $commandID
|
||||
passed = ($failed.Count -eq 0)
|
||||
checks = $checks
|
||||
failed_checks = $failed
|
||||
summary = [ordered]@{
|
||||
backend_container = $backendLine.Trim()
|
||||
node_containers = $nodeLines.Trim()
|
||||
remediation_command = $command
|
||||
rebuild_attempt = $attempt
|
||||
route_manager_decision = $routeManagerDecision
|
||||
route_manager_transition = $routeManagerTransition
|
||||
traffic_status_code = [int]$trafficResponse.StatusCode
|
||||
traffic_accepted_by = $trafficAcceptedBy
|
||||
traffic_duration_ms = $trafficDurationMs
|
||||
traffic_packet_count = $trafficPackets.Count
|
||||
replacement_last_selected_route_id = $replacementLastSelected
|
||||
replacement_flow_stat_count = $replacementFlowStats.Count
|
||||
replacement_flow_stats = $replacementFlowStats
|
||||
baseline_send_fallback_local = $baselineFallbackLocal
|
||||
baseline_send_route_failures = $baselineRouteFailures
|
||||
baseline_send_flow_dropped = $baselineFlowDropped
|
||||
baseline_scheduler_dropped = $baselineSchedulerDropped
|
||||
fallback_local_delta = ($postRemediationFallbackLocal - $baselineFallbackLocal)
|
||||
route_failure_delta = ($postRemediationRouteFailures - $baselineRouteFailures)
|
||||
flow_drop_delta = ($postRemediationFlowDropped - $baselineFlowDropped)
|
||||
scheduler_drop_delta = ($postRemediationSchedulerDropped - $baselineSchedulerDropped)
|
||||
post_remediation_send_fallback_local = $postRemediationFallbackLocal
|
||||
post_remediation_send_route_failures = $postRemediationRouteFailures
|
||||
post_remediation_send_flow_dropped = $postRemediationFlowDropped
|
||||
post_remediation_scheduler_dropped = $postRemediationSchedulerDropped
|
||||
post_channel = $postChannel
|
||||
no_safe_feedback = $noSafeFeedback
|
||||
no_safe_decision = $noSafeDecision
|
||||
no_safe_route_path_decisions = $noSafePathDecisions
|
||||
post_no_safe_channel = $postNoSafeChannel
|
||||
post_no_safe_access = $postNoSafeAccess
|
||||
}
|
||||
}
|
||||
|
||||
$target = Join-Path $repoRoot $ResultPath
|
||||
$result | ConvertTo-Json -Depth 80 | Set-Content -Path $target -Encoding UTF8
|
||||
|
||||
try {
|
||||
if ($primaryRouteID) {
|
||||
Invoke-Api -Method POST -Path "/clusters/$ClusterID/mesh/route-intents/$primaryRouteID/expire" -Body @{ actor_user_id = $ActorUserID } | Out-Null
|
||||
}
|
||||
if ($alternateRouteID) {
|
||||
Invoke-Api -Method POST -Path "/clusters/$ClusterID/mesh/route-intents/$alternateRouteID/expire" -Body @{ actor_user_id = $ActorUserID } | Out-Null
|
||||
}
|
||||
Start-Sleep -Seconds 3
|
||||
Invoke-Api -Method POST -Path "/clusters/$ClusterID/fabric/service-channels/leases/cleanup" -Body @{
|
||||
actor_user_id = $ActorUserID
|
||||
limit = 50
|
||||
} | Out-Null
|
||||
} catch {
|
||||
Write-Warning "cleanup failed after c18z84 smoke: $($_.Exception.Message)"
|
||||
}
|
||||
|
||||
if (-not $result.passed) {
|
||||
throw "C18Z84 access decision aggregate smoke failed: $($failed -join ', ')"
|
||||
}
|
||||
|
||||
Write-Host "C18Z84 service-channel access decision aggregate smoke passed. Result: $target"
|
||||
$result
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,540 @@
|
||||
param(
|
||||
[string]$ApiBaseUrl = "http://192.168.200.61:18121/api/v1",
|
||||
[string]$ClusterID = "cfc0743d-d960-49fb-9de8-96e063d5e4aa",
|
||||
[string]$ActorUserID = "f67d943f-5397-4b3a-a229-695fe67ad700",
|
||||
[string]$EntryNodeName = "test-1",
|
||||
[string]$RelayNodeName = "test-3",
|
||||
[string]$ExitNodeName = "test-2",
|
||||
[string]$EntryBaseUrl = "http://192.168.200.61:19131",
|
||||
[string]$DockerSSH = "test-docker",
|
||||
[string]$ExpectedBackendImage = "rap-backend:fabric-service-channel-0.2.281-c18z109",
|
||||
[string]$ExpectedNodeAgentImage = "rap-node-agent:0.2.270-c18z95",
|
||||
[string]$ResultPath = "artifacts\c18z85-service-channel-access-decision-incident-smoke-result.json"
|
||||
)
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
$scriptDir = Split-Path -Parent $MyInvocation.MyCommand.Path
|
||||
$repoRoot = (Resolve-Path (Join-Path $scriptDir "..\..")).ProviderPath
|
||||
$runId = "c18z85-" + (Get-Date -Format "yyyyMMdd-HHmmss")
|
||||
|
||||
function Invoke-Api {
|
||||
param([string]$Method, [string]$Path, [object]$Body = $null)
|
||||
$params = @{ Method = $Method; Uri = "$ApiBaseUrl$Path"; TimeoutSec = 30 }
|
||||
if ($null -ne $Body) {
|
||||
$params.ContentType = "application/json"
|
||||
$params.Body = ($Body | ConvertTo-Json -Depth 80)
|
||||
}
|
||||
return Invoke-RestMethod @params
|
||||
}
|
||||
|
||||
function Get-PropertyValue {
|
||||
param([object]$Item, [string]$Name, [object]$Default = $null)
|
||||
if ($null -eq $Item) { return $Default }
|
||||
$property = $Item.PSObject.Properties[$Name]
|
||||
if ($null -eq $property) { return $Default }
|
||||
return $property.Value
|
||||
}
|
||||
|
||||
function Get-NodeByName {
|
||||
param([string]$Name)
|
||||
$nodes = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/nodes?actor_user_id=$ActorUserID").nodes
|
||||
$node = @($nodes | Where-Object { $_.name -eq $Name }) | Select-Object -First 1
|
||||
if ($null -eq $node) { throw "Node '$Name' was not found" }
|
||||
return $node
|
||||
}
|
||||
|
||||
function New-IPv4TcpPacket {
|
||||
param(
|
||||
[byte[]]$Source = @(10, 77, 0, 2),
|
||||
[byte[]]$Destination = @(192, 168, 200, 95),
|
||||
[int]$SourcePort,
|
||||
[int]$DestinationPort = 3389
|
||||
)
|
||||
$packet = [byte[]]::new(40)
|
||||
$packet[0] = 0x45
|
||||
$packet[2] = 0
|
||||
$packet[3] = 40
|
||||
$packet[8] = 64
|
||||
$packet[9] = 6
|
||||
[Array]::Copy($Source, 0, $packet, 12, 4)
|
||||
[Array]::Copy($Destination, 0, $packet, 16, 4)
|
||||
$packet[20] = [byte](($SourcePort -shr 8) -band 0xff)
|
||||
$packet[21] = [byte]($SourcePort -band 0xff)
|
||||
$packet[22] = [byte](($DestinationPort -shr 8) -band 0xff)
|
||||
$packet[23] = [byte]($DestinationPort -band 0xff)
|
||||
$packet[32] = 0x50
|
||||
return $packet
|
||||
}
|
||||
|
||||
function ConvertTo-VPNPacketBatch {
|
||||
param([byte[][]]$Packets)
|
||||
$buffer = New-Object System.Collections.Generic.List[byte]
|
||||
foreach ($packet in $Packets) {
|
||||
if ($null -eq $packet -or $packet.Length -eq 0) { continue }
|
||||
$size = [uint32]$packet.Length
|
||||
$buffer.Add([byte](($size -shr 24) -band 0xff))
|
||||
$buffer.Add([byte](($size -shr 16) -band 0xff))
|
||||
$buffer.Add([byte](($size -shr 8) -band 0xff))
|
||||
$buffer.Add([byte]($size -band 0xff))
|
||||
foreach ($value in $packet) {
|
||||
$buffer.Add($value)
|
||||
}
|
||||
}
|
||||
return $buffer.ToArray()
|
||||
}
|
||||
|
||||
function New-RouteIntent {
|
||||
param([string]$SourceNodeID, [string]$DestinationNodeID, [string[]]$Hops)
|
||||
$expiresAt = (Get-Date).ToUniversalTime().AddMinutes(5).ToString("o")
|
||||
return (Invoke-Api -Method POST -Path "/clusters/$ClusterID/mesh/route-intents" -Body @{
|
||||
actor_user_id = $ActorUserID
|
||||
source_selector = @{ node_id = $SourceNodeID }
|
||||
destination_selector = @{ node_id = $DestinationNodeID }
|
||||
service_class = "vpn_packets"
|
||||
priority = 2100000000
|
||||
policy = @{
|
||||
synthetic_enabled = $true
|
||||
route_version = "$runId-primary"
|
||||
policy_version = "$runId-primary"
|
||||
peer_directory_version = "$runId-primary"
|
||||
hops = @($Hops)
|
||||
allowed_channels = @("vpn_packet", "fabric_control")
|
||||
max_ttl = 8
|
||||
max_hops = 8
|
||||
expires_at = $expiresAt
|
||||
metadata = @{ smoke = "c18z82_service_channel_no_safe_recovery"; run_id = $runId }
|
||||
}
|
||||
}).route_intent
|
||||
}
|
||||
|
||||
function Disable-ExistingRouteIntents {
|
||||
param([string]$SourceNodeID, [string]$DestinationNodeID)
|
||||
$items = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/mesh/route-intents?actor_user_id=$ActorUserID").route_intents
|
||||
foreach ($item in @($items)) {
|
||||
if ([string](Get-PropertyValue -Item $item -Name "status" -Default "") -ne "active") { continue }
|
||||
if ([string](Get-PropertyValue -Item $item -Name "service_class" -Default "") -ne "vpn_packets") { continue }
|
||||
$sourceSelector = Get-PropertyValue -Item $item -Name "source_selector" -Default $null
|
||||
$destinationSelector = Get-PropertyValue -Item $item -Name "destination_selector" -Default $null
|
||||
if ([string](Get-PropertyValue -Item $sourceSelector -Name "node_id" -Default "") -ne $SourceNodeID) { continue }
|
||||
if ([string](Get-PropertyValue -Item $destinationSelector -Name "node_id" -Default "") -ne $DestinationNodeID) { continue }
|
||||
[void](Invoke-Api -Method POST -Path "/clusters/$ClusterID/mesh/route-intents/$($item.id)/disable" -Body @{
|
||||
actor_user_id = $ActorUserID
|
||||
reason = "c18z82 isolate no-safe-recovery smoke route pair"
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
function Send-DegradedHeartbeat {
|
||||
param([string]$EntryNodeID, [string]$PrimaryRouteID)
|
||||
$observedAt = (Get-Date).ToUniversalTime().ToString("o")
|
||||
return Invoke-Api -Method POST -Path "/clusters/$ClusterID/nodes/$EntryNodeID/heartbeats" -Body @{
|
||||
health_status = "healthy"
|
||||
reported_version = "0.2.256-c18z82"
|
||||
capabilities = @{
|
||||
fabric_service_channel_runtime = $true
|
||||
fabric_service_channel_route_quality_feedback = $true
|
||||
smoke_feedback_injection = "c18z77"
|
||||
}
|
||||
service_states = @{ smoke = "c18z82_primary_degraded_alternate_after_lease" }
|
||||
metadata = @{
|
||||
fabric_service_channel_runtime_report = @{
|
||||
schema_version = "c18l.fabric_service_channel_runtime_report.v1"
|
||||
config_version = "$runId-primary"
|
||||
cluster_id = $ClusterID
|
||||
local_node_id = $EntryNodeID
|
||||
observed_at = $observedAt
|
||||
ingress = @{
|
||||
flow_scheduler = @{
|
||||
schema_version = "rap.fabric_flow_scheduler.v1"
|
||||
service_neutral = $true
|
||||
service_mode = "application_protocol_agnostic"
|
||||
channel_stats = @{
|
||||
"c18z82-primary-degraded" = @{
|
||||
last_route_id = $PrimaryRouteID
|
||||
last_failed_route_id = $PrimaryRouteID
|
||||
route_generation = "$runId-primary"
|
||||
last_error = "c18z82 primary route degraded; alternate added after lease"
|
||||
last_send_duration_ms = 1200
|
||||
consecutive_failures = 3
|
||||
stall_count = 2
|
||||
route_rebuild_recommended = $true
|
||||
degraded_fallback_recommended = $false
|
||||
quality_window_sample_count = 8
|
||||
quality_window_success_count = 2
|
||||
quality_window_failure_count = 3
|
||||
quality_window_slow_count = 2
|
||||
quality_window_drop_count = 0
|
||||
quality_window_avg_latency_ms = 1200
|
||||
quality_window_last_updated_at = $observedAt
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
smoke = @{ name = "c18z82_service_channel_no_safe_recovery"; run_id = $runId }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function Send-ReplacementDegradedHeartbeat {
|
||||
param([string]$EntryNodeID, [string]$ReplacementRouteID, [string]$RouteGeneration)
|
||||
$observedAt = (Get-Date).ToUniversalTime().ToString("o")
|
||||
return Invoke-Api -Method POST -Path "/clusters/$ClusterID/nodes/$EntryNodeID/heartbeats" -Body @{
|
||||
health_status = "healthy"
|
||||
reported_version = "0.2.256-c18z82"
|
||||
capabilities = @{
|
||||
fabric_service_channel_runtime = $true
|
||||
fabric_service_channel_route_quality_feedback = $true
|
||||
smoke_feedback_injection = "c18z82"
|
||||
}
|
||||
service_states = @{ smoke = "c18z82_replacement_route_degraded_no_safe_recovery" }
|
||||
metadata = @{
|
||||
fabric_service_channel_runtime_report = @{
|
||||
schema_version = "c18l.fabric_service_channel_runtime_report.v1"
|
||||
config_version = "$runId-replacement-degraded"
|
||||
cluster_id = $ClusterID
|
||||
local_node_id = $EntryNodeID
|
||||
observed_at = $observedAt
|
||||
ingress = @{
|
||||
last_selected_route_id = $ReplacementRouteID
|
||||
send_route_failures = 4
|
||||
flow_scheduler = @{
|
||||
schema_version = "rap.fabric_flow_scheduler.v1"
|
||||
service_neutral = $true
|
||||
service_mode = "application_protocol_agnostic"
|
||||
channel_stats = @{
|
||||
"c18z82-replacement-degraded" = @{
|
||||
last_route_id = $ReplacementRouteID
|
||||
last_failed_route_id = $ReplacementRouteID
|
||||
route_generation = $RouteGeneration
|
||||
last_error = "c18z82 replacement route degraded; no safe recovery route exists"
|
||||
last_send_duration_ms = 1500
|
||||
consecutive_failures = 4
|
||||
stall_count = 2
|
||||
route_rebuild_recommended = $true
|
||||
degraded_fallback_recommended = $true
|
||||
quality_window_sample_count = 10
|
||||
quality_window_success_count = 1
|
||||
quality_window_failure_count = 4
|
||||
quality_window_slow_count = 3
|
||||
quality_window_drop_count = 0
|
||||
quality_window_avg_latency_ms = 1500
|
||||
quality_window_last_updated_at = $observedAt
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
smoke = @{ name = "c18z82_service_channel_no_safe_recovery"; run_id = $runId }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
$entryNode = Get-NodeByName -Name $EntryNodeName
|
||||
$relayNode = Get-NodeByName -Name $RelayNodeName
|
||||
$exitNode = Get-NodeByName -Name $ExitNodeName
|
||||
[void](Disable-ExistingRouteIntents -SourceNodeID $entryNode.id -DestinationNodeID $exitNode.id)
|
||||
$route = New-RouteIntent -SourceNodeID $entryNode.id -DestinationNodeID $exitNode.id -Hops @($entryNode.id, $exitNode.id)
|
||||
$lease = (Invoke-Api -Method POST -Path "/clusters/$ClusterID/fabric/service-channels/leases" -Body @{
|
||||
actor_user_id = $ActorUserID
|
||||
organization_id = "smoke-org"
|
||||
user_id = "smoke-user"
|
||||
resource_id = "c18z82-vpn-smoke"
|
||||
service_class = "vpn_packets"
|
||||
entry_node_ids = @([string]$entryNode.id)
|
||||
exit_node_ids = @([string]$exitNode.id)
|
||||
preferred_entry_node_id = [string]$entryNode.id
|
||||
preferred_exit_node_id = [string]$exitNode.id
|
||||
allowed_channels = @("vpn_packet", "fabric_control")
|
||||
ttl_seconds = 180
|
||||
metadata = @{ smoke = "c18z82_service_channel_no_safe_recovery"; run_id = $runId }
|
||||
}).fabric_service_channel_lease
|
||||
$primaryRouteID = [string](Get-PropertyValue -Item (Get-PropertyValue -Item $lease -Name "primary_route" -Default $null) -Name "route_id" -Default $route.id)
|
||||
|
||||
[void](Send-DegradedHeartbeat -EntryNodeID $entryNode.id -PrimaryRouteID $primaryRouteID)
|
||||
|
||||
$matchingChannel = $null
|
||||
for ($i = 0; $i -lt 10; $i++) {
|
||||
Start-Sleep -Seconds 3
|
||||
$accessTelemetry = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/fabric/service-channels/access-telemetry?actor_user_id=$ActorUserID&limit=50").fabric_service_channel_access_telemetry
|
||||
$matchingChannel = @($accessTelemetry.active_channels | Where-Object { $_.channel_id -eq $lease.channel_id }) | Select-Object -First 1
|
||||
if ($null -ne $matchingChannel -and [string](Get-PropertyValue -Item $matchingChannel -Name "remediation_action" -Default "") -eq "rebuild_route") {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
$command = Get-PropertyValue -Item $matchingChannel -Name "remediation_command" -Default $null
|
||||
$commandID = [string](Get-PropertyValue -Item $command -Name "command_id" -Default "")
|
||||
$alternate = New-RouteIntent -SourceNodeID $entryNode.id -DestinationNodeID $exitNode.id -Hops @($entryNode.id, $relayNode.id, $exitNode.id)
|
||||
$alternateRouteID = [string]$alternate.id
|
||||
[void](Invoke-Api -Method GET -Path "/clusters/$ClusterID/nodes/$($entryNode.id)/mesh/synthetic-config")
|
||||
|
||||
$attempts = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/fabric/service-channels/rebuild-attempts?actor_user_id=$ActorUserID&reporter_node_id=$($entryNode.id)&rebuild_request_id=$commandID&limit=10").rebuild_attempts
|
||||
$attempt = @($attempts | Where-Object { $_.rebuild_request_id -eq $commandID }) | Select-Object -First 1
|
||||
|
||||
$routeManagerDecision = $null
|
||||
$routeManagerTransition = $null
|
||||
for ($i = 0; $i -lt 12; $i++) {
|
||||
Start-Sleep -Seconds 5
|
||||
$heartbeats = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/nodes/$($entryNode.id)/heartbeats?actor_user_id=$ActorUserID&limit=5").heartbeats
|
||||
foreach ($heartbeat in @($heartbeats)) {
|
||||
$runtimeReport = Get-PropertyValue -Item (Get-PropertyValue -Item $heartbeat -Name "metadata" -Default $null) -Name "fabric_service_channel_runtime_report" -Default $null
|
||||
$ingress = Get-PropertyValue -Item $runtimeReport -Name "ingress" -Default $null
|
||||
$routeManager = Get-PropertyValue -Item $ingress -Name "route_manager" -Default $null
|
||||
$routeManagerTransition = Get-PropertyValue -Item $ingress -Name "route_manager_transition" -Default $null
|
||||
$decisions = @()
|
||||
if ($null -ne $routeManager -and $routeManager.PSObject.Properties.Name -contains "decisions") {
|
||||
$decisions = @($routeManager.decisions)
|
||||
}
|
||||
$routeManagerDecision = $decisions | Where-Object {
|
||||
[string](Get-PropertyValue -Item $_ -Name "rebuild_request_id" -Default "") -eq $commandID -and
|
||||
[string](Get-PropertyValue -Item $_ -Name "route_id" -Default "") -eq $primaryRouteID -and
|
||||
[string](Get-PropertyValue -Item $_ -Name "replacement_route_id" -Default "") -eq $alternateRouteID
|
||||
} | Select-Object -First 1
|
||||
if ($null -ne $routeManagerDecision) {
|
||||
break
|
||||
}
|
||||
}
|
||||
if ($null -ne $routeManagerDecision) {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
$packetPath = $lease.entry_http.path_template.
|
||||
Replace("{cluster_id}", $ClusterID).
|
||||
Replace("{channel_id}", [string]$lease.channel_id).
|
||||
Replace("{resource_id}", "c18z82-vpn-smoke")
|
||||
$packetUrl = $EntryBaseUrl.TrimEnd("/") + $packetPath
|
||||
$headers = @{
|
||||
"X-RAP-Service-Channel-Token" = [string]$lease.token.token
|
||||
"X-RAP-Fabric-Channel-ID" = [string]$lease.channel_id
|
||||
"X-RAP-Service-Class" = "vpn_packets"
|
||||
"X-RAP-Channel-Class" = "vpn_packet"
|
||||
"X-RAP-Traffic-Class" = "interactive"
|
||||
}
|
||||
$baselineFallbackLocal = 0
|
||||
$baselineRouteFailures = 0
|
||||
$baselineFlowDropped = 0
|
||||
$baselineSchedulerDropped = 0
|
||||
$baselineHeartbeats = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/nodes/$($entryNode.id)/heartbeats?actor_user_id=$ActorUserID&limit=1").heartbeats
|
||||
if (@($baselineHeartbeats).Count -gt 0) {
|
||||
$baselineRuntimeReport = Get-PropertyValue -Item (Get-PropertyValue -Item $baselineHeartbeats[0] -Name "metadata" -Default $null) -Name "fabric_service_channel_runtime_report" -Default $null
|
||||
$baselineIngress = Get-PropertyValue -Item $baselineRuntimeReport -Name "ingress" -Default $null
|
||||
$baselineFlowScheduler = Get-PropertyValue -Item $baselineIngress -Name "flow_scheduler" -Default $null
|
||||
$baselineFallbackLocal = [int](Get-PropertyValue -Item $baselineIngress -Name "send_fallback_local" -Default 0)
|
||||
$baselineRouteFailures = [int](Get-PropertyValue -Item $baselineIngress -Name "send_route_failures" -Default 0)
|
||||
$baselineFlowDropped = [int](Get-PropertyValue -Item $baselineIngress -Name "send_flow_dropped" -Default 0)
|
||||
$baselineSchedulerDropped = [int](Get-PropertyValue -Item $baselineFlowScheduler -Name "dropped" -Default 0)
|
||||
}
|
||||
$trafficPackets = @()
|
||||
foreach ($offset in 0..15) {
|
||||
$trafficPackets += ,(New-IPv4TcpPacket -SourcePort (54000 + $offset) -DestinationPort 3389)
|
||||
}
|
||||
$trafficPayload = ConvertTo-VPNPacketBatch -Packets $trafficPackets
|
||||
$trafficPayloadPath = Join-Path ([System.IO.Path]::GetTempPath()) "$runId-vpn-packet-batch.bin"
|
||||
[System.IO.File]::WriteAllBytes($trafficPayloadPath, [byte[]]$trafficPayload)
|
||||
$trafficStarted = Get-Date
|
||||
$trafficResponse = Invoke-WebRequest -Method Post -Uri ($packetUrl + "?batch=true") -Headers $headers -InFile $trafficPayloadPath -ContentType "application/vnd.rap.vpn-packet-batch.v1" -TimeoutSec 30
|
||||
$trafficDurationMs = [int]((Get-Date) - $trafficStarted).TotalMilliseconds
|
||||
$trafficAcceptedBy = [string]$trafficResponse.Headers["X-RAP-Service-Channel-Accepted-By"]
|
||||
Remove-Item -Path $trafficPayloadPath -Force -ErrorAction SilentlyContinue
|
||||
|
||||
$replacementTrafficObserved = $false
|
||||
$replacementLastSelected = ""
|
||||
$replacementFlowStats = @()
|
||||
$postRemediationFallbackLocal = 0
|
||||
$postRemediationRouteFailures = 0
|
||||
$postRemediationFlowDropped = 0
|
||||
$postRemediationSchedulerDropped = 0
|
||||
for ($i = 0; $i -lt 12; $i++) {
|
||||
Start-Sleep -Seconds 5
|
||||
$heartbeats = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/nodes/$($entryNode.id)/heartbeats?actor_user_id=$ActorUserID&limit=5").heartbeats
|
||||
foreach ($heartbeat in @($heartbeats)) {
|
||||
$runtimeReport = Get-PropertyValue -Item (Get-PropertyValue -Item $heartbeat -Name "metadata" -Default $null) -Name "fabric_service_channel_runtime_report" -Default $null
|
||||
$ingress = Get-PropertyValue -Item $runtimeReport -Name "ingress" -Default $null
|
||||
$replacementLastSelected = [string](Get-PropertyValue -Item $ingress -Name "last_selected_route_id" -Default "")
|
||||
$postRemediationFallbackLocal = [int](Get-PropertyValue -Item $ingress -Name "send_fallback_local" -Default 0)
|
||||
$postRemediationRouteFailures = [int](Get-PropertyValue -Item $ingress -Name "send_route_failures" -Default 0)
|
||||
$postRemediationFlowDropped = [int](Get-PropertyValue -Item $ingress -Name "send_flow_dropped" -Default 0)
|
||||
$flowScheduler = Get-PropertyValue -Item $ingress -Name "flow_scheduler" -Default $null
|
||||
$postRemediationSchedulerDropped = [int](Get-PropertyValue -Item $flowScheduler -Name "dropped" -Default 0)
|
||||
$channelStats = Get-PropertyValue -Item $flowScheduler -Name "channel_stats" -Default $null
|
||||
if ($null -ne $channelStats) {
|
||||
$replacementFlowStats = @()
|
||||
foreach ($property in @($channelStats.PSObject.Properties)) {
|
||||
$stat = $property.Value
|
||||
if ([string](Get-PropertyValue -Item $stat -Name "last_route_id" -Default "") -eq $alternateRouteID) {
|
||||
$replacementFlowStats += $stat
|
||||
}
|
||||
}
|
||||
}
|
||||
if ($replacementLastSelected -eq $alternateRouteID -and $replacementFlowStats.Count -ge 1) {
|
||||
$replacementTrafficObserved = $true
|
||||
break
|
||||
}
|
||||
}
|
||||
if ($replacementTrafficObserved) {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
$postAccess = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/fabric/service-channels/access-telemetry?actor_user_id=$ActorUserID&limit=50").fabric_service_channel_access_telemetry
|
||||
$postChannel = @($postAccess.active_channels | Where-Object { $_.channel_id -eq $lease.channel_id }) | Select-Object -First 1
|
||||
|
||||
[void](Send-ReplacementDegradedHeartbeat -EntryNodeID $entryNode.id -ReplacementRouteID $alternateRouteID -RouteGeneration "$runId-primary")
|
||||
$noSafeSyntheticConfig = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/nodes/$($entryNode.id)/mesh/synthetic-config").synthetic_mesh_config
|
||||
$noSafePathDecisions = @()
|
||||
if ($noSafeSyntheticConfig.PSObject.Properties.Name -contains "route_path_decisions") {
|
||||
$noSafePathDecisions = @($noSafeSyntheticConfig.route_path_decisions.decisions)
|
||||
}
|
||||
$noSafeDecision = $noSafePathDecisions | Where-Object {
|
||||
[string](Get-PropertyValue -Item $_ -Name "route_id" -Default "") -eq $alternateRouteID -and
|
||||
[string](Get-PropertyValue -Item $_ -Name "decision_source" -Default "") -eq "service_channel_feedback_no_alternate"
|
||||
} | Select-Object -First 1
|
||||
$noSafeFeedback = $null
|
||||
$serviceChannelFeedback = Get-PropertyValue -Item $noSafeSyntheticConfig -Name "service_channel_feedback" -Default $null
|
||||
if ($null -ne $serviceChannelFeedback -and $serviceChannelFeedback.PSObject.Properties.Name -contains "routes") {
|
||||
$noSafeFeedback = @($serviceChannelFeedback.routes | Where-Object { [string](Get-PropertyValue -Item $_ -Name "route_id" -Default "") -eq $alternateRouteID }) | Select-Object -First 1
|
||||
}
|
||||
|
||||
$postNoSafeAccess = $null
|
||||
$postNoSafeChannel = $null
|
||||
for ($i = 0; $i -lt 12; $i++) {
|
||||
Start-Sleep -Seconds 3
|
||||
$postNoSafeAccess = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/fabric/service-channels/access-telemetry?actor_user_id=$ActorUserID&limit=50").fabric_service_channel_access_telemetry
|
||||
$postNoSafeChannel = @($postNoSafeAccess.active_channels | Where-Object { $_.channel_id -eq $lease.channel_id }) | Select-Object -First 1
|
||||
if ($null -ne $postNoSafeChannel -and [string](Get-PropertyValue -Item $postNoSafeChannel -Name "route_decision_source" -Default "") -eq "service_channel_feedback_no_alternate") {
|
||||
break
|
||||
}
|
||||
}
|
||||
$postNoSafeHealth = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/fabric/service-channels/rebuild-health?actor_user_id=$ActorUserID&limit=50").rebuild_health
|
||||
$postNoSafeIncidents = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/fabric/service-channels/rebuild-incidents?actor_user_id=$ActorUserID&limit=10").rebuild_incidents
|
||||
$postNoSafeIncident = @($postNoSafeIncidents | Where-Object {
|
||||
[string](Get-PropertyValue -Item $_ -Name "incident_source" -Default "") -eq "access_decision" -and
|
||||
[string](Get-PropertyValue -Item $_ -Name "guard_status" -Default "") -eq "access_no_safe_recovery" -and
|
||||
[string](Get-PropertyValue -Item $_ -Name "channel_id" -Default "") -eq [string]$lease.channel_id
|
||||
}) | Select-Object -First 1
|
||||
|
||||
$backendLine = (& ssh $DockerSSH "docker ps --format '{{.Names}} {{.Image}} {{.Status}}' | grep '^rap_test_backend '") | Out-String
|
||||
$nodeLines = (& ssh $DockerSSH "docker ps --format '{{.Names}} {{.Image}} {{.Status}}' | grep '^rap_test_node_test_'") | Out-String
|
||||
$checks = [ordered]@{
|
||||
backend_expected_image_deployed = $backendLine.Contains($ExpectedBackendImage)
|
||||
node_agent_expected_image_deployed = $nodeLines.Contains($ExpectedNodeAgentImage)
|
||||
lease_ready = ([string]$lease.status -eq "ready")
|
||||
remediation_rebuild_route_visible = ($null -ne $matchingChannel -and [string](Get-PropertyValue -Item $matchingChannel -Name "remediation_action" -Default "") -eq "rebuild_route")
|
||||
remediation_command_visible = ($null -ne $command -and $commandID.Length -gt 0)
|
||||
durable_rebuild_intent_recorded = ($null -ne $attempt)
|
||||
durable_rebuild_intent_resolved_applied = ($null -ne $attempt -and [string](Get-PropertyValue -Item $attempt -Name "rebuild_status" -Default "") -eq "applied")
|
||||
durable_rebuild_intent_outcome_replacement_selected = ($null -ne $attempt -and [string](Get-PropertyValue -Item $attempt -Name "outcome" -Default "") -eq "replacement_selected")
|
||||
durable_rebuild_intent_replacement_matches = ($null -ne $attempt -and [string](Get-PropertyValue -Item $attempt -Name "replacement_route_id" -Default "") -eq $alternateRouteID)
|
||||
durable_rebuild_intent_source_matches = ($null -ne $attempt -and [string](Get-PropertyValue -Item $attempt -Name "decision_source" -Default "") -eq "service_channel_remediation_command")
|
||||
initial_access_telemetry_reports_planner_applied = ($null -ne $postChannel -and [string](Get-PropertyValue -Item $postChannel -Name "remediation_execution_status" -Default "") -eq "rebuild_request_applied")
|
||||
node_route_manager_consumed_same_command = ($null -ne $routeManagerDecision)
|
||||
node_route_manager_applied_rebuild = ($null -ne $routeManagerDecision -and [string](Get-PropertyValue -Item $routeManagerDecision -Name "rebuild_status" -Default "") -eq "applied")
|
||||
node_route_manager_transition_applied_rebuild = ($null -ne $routeManagerTransition -and [string](Get-PropertyValue -Item $routeManagerTransition -Name "status" -Default "") -eq "applied_rebuild")
|
||||
traffic_packet_accepted = ([int]$trafficResponse.StatusCode -eq 202)
|
||||
traffic_accepted_by_runtime = ($trafficAcceptedBy -eq "introspection" -or $trafficAcceptedBy -eq "signed")
|
||||
traffic_selected_replacement_route = $replacementTrafficObserved
|
||||
traffic_last_selected_route_matches_replacement = ($replacementLastSelected -eq $alternateRouteID)
|
||||
no_backend_or_local_fallback_after_replacement = (($postRemediationFallbackLocal - $baselineFallbackLocal) -eq 0)
|
||||
no_route_failures_after_replacement = (($postRemediationRouteFailures - $baselineRouteFailures) -eq 0)
|
||||
no_flow_drops_after_replacement = (($postRemediationFlowDropped - $baselineFlowDropped) -eq 0 -and ($postRemediationSchedulerDropped - $baselineSchedulerDropped) -eq 0)
|
||||
replacement_degradation_feedback_visible = ($null -ne $noSafeDecision -and @($noSafeDecision.score_reasons | Where-Object { [string]$_ -eq "service_channel_fenced_route" }).Count -gt 0)
|
||||
no_safe_recovery_decision_visible = ($null -ne $noSafeDecision)
|
||||
no_safe_recovery_status_pending_fallback = ($null -ne $noSafeDecision -and [string](Get-PropertyValue -Item $noSafeDecision -Name "rebuild_status" -Default "") -eq "pending_degraded_fallback")
|
||||
no_safe_recovery_reason_visible = ($null -ne $noSafeDecision -and @($noSafeDecision.score_reasons | Where-Object { [string]$_ -eq "no_unfenced_alternate_route" }).Count -gt 0)
|
||||
no_silent_stickiness_to_bad_replacement = ($null -ne $noSafeDecision -and [string](Get-PropertyValue -Item $noSafeDecision -Name "decision_source" -Default "") -eq "service_channel_feedback_no_alternate")
|
||||
access_telemetry_projects_no_safe_decision = ($null -ne $postNoSafeChannel -and [string](Get-PropertyValue -Item $postNoSafeChannel -Name "route_decision_source" -Default "") -eq "service_channel_feedback_no_alternate")
|
||||
access_telemetry_projects_pending_fallback = ($null -ne $postNoSafeChannel -and [string](Get-PropertyValue -Item $postNoSafeChannel -Name "route_decision_rebuild_status" -Default "") -eq "pending_degraded_fallback")
|
||||
access_telemetry_projects_no_safe_route_id = ($null -ne $postNoSafeChannel -and @($primaryRouteID, $alternateRouteID) -contains [string](Get-PropertyValue -Item $postNoSafeChannel -Name "route_decision_route_id" -Default ""))
|
||||
access_telemetry_projects_operator_state = ($null -ne $postNoSafeChannel -and @("route_rebuild_no_safe_recovery", "rebuild_request_no_alternate") -contains [string](Get-PropertyValue -Item $postNoSafeChannel -Name "remediation_execution_status" -Default ""))
|
||||
access_telemetry_aggregates_route_decision = ($null -ne $postNoSafeAccess -and [int](Get-PropertyValue -Item $postNoSafeAccess -Name "route_decision_channel_count" -Default 0) -ge 1)
|
||||
access_telemetry_aggregates_no_safe_decision = ($null -ne $postNoSafeAccess -and [int](Get-PropertyValue -Item $postNoSafeAccess -Name "no_safe_recovery_decision_count" -Default 0) -ge 1)
|
||||
access_telemetry_aggregate_status_prioritizes_no_safe = ($null -ne $postNoSafeAccess -and [string](Get-PropertyValue -Item $postNoSafeAccess -Name "reason" -Default "") -eq "active_channels_no_safe_recovery")
|
||||
rebuild_health_projects_access_no_safe = ($null -ne $postNoSafeHealth -and [int](Get-PropertyValue -Item $postNoSafeHealth -Name "access_no_safe_count" -Default 0) -ge 1)
|
||||
rebuild_health_action_prioritizes_access_no_safe = ($null -ne $postNoSafeHealth -and [string](Get-PropertyValue -Item $postNoSafeHealth -Name "recommended_operator_action" -Default "") -eq "inspect_access_no_safe_recovery_route_pool_and_signed_policy")
|
||||
rebuild_incident_projects_access_no_safe = ($null -ne $postNoSafeIncident)
|
||||
rebuild_incident_access_no_safe_is_bad = ($null -ne $postNoSafeIncident -and [string](Get-PropertyValue -Item $postNoSafeIncident -Name "guard_severity" -Default "") -eq "bad")
|
||||
}
|
||||
$failed = @($checks.GetEnumerator() | Where-Object { -not $_.Value } | ForEach-Object { $_.Key })
|
||||
|
||||
$result = [ordered]@{
|
||||
schema_version = "c18z85.service_channel_access_decision_incident_smoke.v1"
|
||||
run_id = $runId
|
||||
cluster_id = $ClusterID
|
||||
channel_id = [string]$lease.channel_id
|
||||
primary_route_id = $primaryRouteID
|
||||
alternate_route_id = $alternateRouteID
|
||||
rebuild_request_id = $commandID
|
||||
passed = ($failed.Count -eq 0)
|
||||
checks = $checks
|
||||
failed_checks = $failed
|
||||
summary = [ordered]@{
|
||||
backend_container = $backendLine.Trim()
|
||||
node_containers = $nodeLines.Trim()
|
||||
remediation_command = $command
|
||||
rebuild_attempt = $attempt
|
||||
route_manager_decision = $routeManagerDecision
|
||||
route_manager_transition = $routeManagerTransition
|
||||
traffic_status_code = [int]$trafficResponse.StatusCode
|
||||
traffic_accepted_by = $trafficAcceptedBy
|
||||
traffic_duration_ms = $trafficDurationMs
|
||||
traffic_packet_count = $trafficPackets.Count
|
||||
replacement_last_selected_route_id = $replacementLastSelected
|
||||
replacement_flow_stat_count = $replacementFlowStats.Count
|
||||
replacement_flow_stats = $replacementFlowStats
|
||||
baseline_send_fallback_local = $baselineFallbackLocal
|
||||
baseline_send_route_failures = $baselineRouteFailures
|
||||
baseline_send_flow_dropped = $baselineFlowDropped
|
||||
baseline_scheduler_dropped = $baselineSchedulerDropped
|
||||
fallback_local_delta = ($postRemediationFallbackLocal - $baselineFallbackLocal)
|
||||
route_failure_delta = ($postRemediationRouteFailures - $baselineRouteFailures)
|
||||
flow_drop_delta = ($postRemediationFlowDropped - $baselineFlowDropped)
|
||||
scheduler_drop_delta = ($postRemediationSchedulerDropped - $baselineSchedulerDropped)
|
||||
post_remediation_send_fallback_local = $postRemediationFallbackLocal
|
||||
post_remediation_send_route_failures = $postRemediationRouteFailures
|
||||
post_remediation_send_flow_dropped = $postRemediationFlowDropped
|
||||
post_remediation_scheduler_dropped = $postRemediationSchedulerDropped
|
||||
post_channel = $postChannel
|
||||
no_safe_feedback = $noSafeFeedback
|
||||
no_safe_decision = $noSafeDecision
|
||||
no_safe_route_path_decisions = $noSafePathDecisions
|
||||
post_no_safe_channel = $postNoSafeChannel
|
||||
post_no_safe_access = $postNoSafeAccess
|
||||
post_no_safe_health = $postNoSafeHealth
|
||||
post_no_safe_incidents = $postNoSafeIncidents
|
||||
post_no_safe_incident = $postNoSafeIncident
|
||||
}
|
||||
}
|
||||
|
||||
$target = Join-Path $repoRoot $ResultPath
|
||||
$result | ConvertTo-Json -Depth 80 | Set-Content -Path $target -Encoding UTF8
|
||||
|
||||
try {
|
||||
if ($primaryRouteID) {
|
||||
Invoke-Api -Method POST -Path "/clusters/$ClusterID/mesh/route-intents/$primaryRouteID/expire" -Body @{ actor_user_id = $ActorUserID } | Out-Null
|
||||
}
|
||||
if ($alternateRouteID) {
|
||||
Invoke-Api -Method POST -Path "/clusters/$ClusterID/mesh/route-intents/$alternateRouteID/expire" -Body @{ actor_user_id = $ActorUserID } | Out-Null
|
||||
}
|
||||
Start-Sleep -Seconds 3
|
||||
Invoke-Api -Method POST -Path "/clusters/$ClusterID/fabric/service-channels/leases/cleanup" -Body @{
|
||||
actor_user_id = $ActorUserID
|
||||
limit = 50
|
||||
} | Out-Null
|
||||
} catch {
|
||||
Write-Warning "cleanup failed after c18z85 smoke: $($_.Exception.Message)"
|
||||
}
|
||||
|
||||
if (-not $result.passed) {
|
||||
throw "C18Z85 access decision incident smoke failed: $($failed -join ', ')"
|
||||
}
|
||||
|
||||
Write-Host "C18Z85 service-channel access decision incident smoke passed. Result: $target"
|
||||
$result
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,571 @@
|
||||
param(
|
||||
[string]$ApiBaseUrl = "http://192.168.200.61:18121/api/v1",
|
||||
[string]$ClusterID = "cfc0743d-d960-49fb-9de8-96e063d5e4aa",
|
||||
[string]$ActorUserID = "f67d943f-5397-4b3a-a229-695fe67ad700",
|
||||
[string]$EntryNodeName = "test-1",
|
||||
[string]$RelayNodeName = "test-3",
|
||||
[string]$ExitNodeName = "test-2",
|
||||
[string]$EntryBaseUrl = "http://192.168.200.61:19131",
|
||||
[string]$DockerSSH = "test-docker",
|
||||
[string]$ExpectedBackendImage = "rap-backend:fabric-service-channel-0.2.281-c18z109",
|
||||
[string]$ExpectedNodeAgentImage = "rap-node-agent:0.2.270-c18z95",
|
||||
[string]$ResultPath = "artifacts\c18z86-service-channel-access-decision-silence-smoke-result.json"
|
||||
)
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
$scriptDir = Split-Path -Parent $MyInvocation.MyCommand.Path
|
||||
$repoRoot = (Resolve-Path (Join-Path $scriptDir "..\..")).ProviderPath
|
||||
$runId = "c18z86-" + (Get-Date -Format "yyyyMMdd-HHmmss")
|
||||
|
||||
function Invoke-Api {
|
||||
param([string]$Method, [string]$Path, [object]$Body = $null)
|
||||
$params = @{ Method = $Method; Uri = "$ApiBaseUrl$Path"; TimeoutSec = 30 }
|
||||
if ($null -ne $Body) {
|
||||
$params.ContentType = "application/json"
|
||||
$params.Body = ($Body | ConvertTo-Json -Depth 80)
|
||||
}
|
||||
return Invoke-RestMethod @params
|
||||
}
|
||||
|
||||
function Get-PropertyValue {
|
||||
param([object]$Item, [string]$Name, [object]$Default = $null)
|
||||
if ($null -eq $Item) { return $Default }
|
||||
$property = $Item.PSObject.Properties[$Name]
|
||||
if ($null -eq $property) { return $Default }
|
||||
return $property.Value
|
||||
}
|
||||
|
||||
function Get-NodeByName {
|
||||
param([string]$Name)
|
||||
$nodes = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/nodes?actor_user_id=$ActorUserID").nodes
|
||||
$node = @($nodes | Where-Object { $_.name -eq $Name }) | Select-Object -First 1
|
||||
if ($null -eq $node) { throw "Node '$Name' was not found" }
|
||||
return $node
|
||||
}
|
||||
|
||||
function New-IPv4TcpPacket {
|
||||
param(
|
||||
[byte[]]$Source = @(10, 77, 0, 2),
|
||||
[byte[]]$Destination = @(192, 168, 200, 95),
|
||||
[int]$SourcePort,
|
||||
[int]$DestinationPort = 3389
|
||||
)
|
||||
$packet = [byte[]]::new(40)
|
||||
$packet[0] = 0x45
|
||||
$packet[2] = 0
|
||||
$packet[3] = 40
|
||||
$packet[8] = 64
|
||||
$packet[9] = 6
|
||||
[Array]::Copy($Source, 0, $packet, 12, 4)
|
||||
[Array]::Copy($Destination, 0, $packet, 16, 4)
|
||||
$packet[20] = [byte](($SourcePort -shr 8) -band 0xff)
|
||||
$packet[21] = [byte]($SourcePort -band 0xff)
|
||||
$packet[22] = [byte](($DestinationPort -shr 8) -band 0xff)
|
||||
$packet[23] = [byte]($DestinationPort -band 0xff)
|
||||
$packet[32] = 0x50
|
||||
return $packet
|
||||
}
|
||||
|
||||
function ConvertTo-VPNPacketBatch {
|
||||
param([byte[][]]$Packets)
|
||||
$buffer = New-Object System.Collections.Generic.List[byte]
|
||||
foreach ($packet in $Packets) {
|
||||
if ($null -eq $packet -or $packet.Length -eq 0) { continue }
|
||||
$size = [uint32]$packet.Length
|
||||
$buffer.Add([byte](($size -shr 24) -band 0xff))
|
||||
$buffer.Add([byte](($size -shr 16) -band 0xff))
|
||||
$buffer.Add([byte](($size -shr 8) -band 0xff))
|
||||
$buffer.Add([byte]($size -band 0xff))
|
||||
foreach ($value in $packet) {
|
||||
$buffer.Add($value)
|
||||
}
|
||||
}
|
||||
return $buffer.ToArray()
|
||||
}
|
||||
|
||||
function New-RouteIntent {
|
||||
param([string]$SourceNodeID, [string]$DestinationNodeID, [string[]]$Hops)
|
||||
$expiresAt = (Get-Date).ToUniversalTime().AddMinutes(5).ToString("o")
|
||||
return (Invoke-Api -Method POST -Path "/clusters/$ClusterID/mesh/route-intents" -Body @{
|
||||
actor_user_id = $ActorUserID
|
||||
source_selector = @{ node_id = $SourceNodeID }
|
||||
destination_selector = @{ node_id = $DestinationNodeID }
|
||||
service_class = "vpn_packets"
|
||||
priority = 2100000000
|
||||
policy = @{
|
||||
synthetic_enabled = $true
|
||||
route_version = "$runId-primary"
|
||||
policy_version = "$runId-primary"
|
||||
peer_directory_version = "$runId-primary"
|
||||
hops = @($Hops)
|
||||
allowed_channels = @("vpn_packet", "fabric_control")
|
||||
max_ttl = 8
|
||||
max_hops = 8
|
||||
expires_at = $expiresAt
|
||||
metadata = @{ smoke = "c18z82_service_channel_no_safe_recovery"; run_id = $runId }
|
||||
}
|
||||
}).route_intent
|
||||
}
|
||||
|
||||
function Disable-ExistingRouteIntents {
|
||||
param([string]$SourceNodeID, [string]$DestinationNodeID)
|
||||
$items = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/mesh/route-intents?actor_user_id=$ActorUserID").route_intents
|
||||
foreach ($item in @($items)) {
|
||||
if ([string](Get-PropertyValue -Item $item -Name "status" -Default "") -ne "active") { continue }
|
||||
if ([string](Get-PropertyValue -Item $item -Name "service_class" -Default "") -ne "vpn_packets") { continue }
|
||||
$sourceSelector = Get-PropertyValue -Item $item -Name "source_selector" -Default $null
|
||||
$destinationSelector = Get-PropertyValue -Item $item -Name "destination_selector" -Default $null
|
||||
if ([string](Get-PropertyValue -Item $sourceSelector -Name "node_id" -Default "") -ne $SourceNodeID) { continue }
|
||||
if ([string](Get-PropertyValue -Item $destinationSelector -Name "node_id" -Default "") -ne $DestinationNodeID) { continue }
|
||||
[void](Invoke-Api -Method POST -Path "/clusters/$ClusterID/mesh/route-intents/$($item.id)/disable" -Body @{
|
||||
actor_user_id = $ActorUserID
|
||||
reason = "c18z82 isolate no-safe-recovery smoke route pair"
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
function Send-DegradedHeartbeat {
|
||||
param([string]$EntryNodeID, [string]$PrimaryRouteID)
|
||||
$observedAt = (Get-Date).ToUniversalTime().ToString("o")
|
||||
return Invoke-Api -Method POST -Path "/clusters/$ClusterID/nodes/$EntryNodeID/heartbeats" -Body @{
|
||||
health_status = "healthy"
|
||||
reported_version = "0.2.256-c18z82"
|
||||
capabilities = @{
|
||||
fabric_service_channel_runtime = $true
|
||||
fabric_service_channel_route_quality_feedback = $true
|
||||
smoke_feedback_injection = "c18z77"
|
||||
}
|
||||
service_states = @{ smoke = "c18z82_primary_degraded_alternate_after_lease" }
|
||||
metadata = @{
|
||||
fabric_service_channel_runtime_report = @{
|
||||
schema_version = "c18l.fabric_service_channel_runtime_report.v1"
|
||||
config_version = "$runId-primary"
|
||||
cluster_id = $ClusterID
|
||||
local_node_id = $EntryNodeID
|
||||
observed_at = $observedAt
|
||||
ingress = @{
|
||||
flow_scheduler = @{
|
||||
schema_version = "rap.fabric_flow_scheduler.v1"
|
||||
service_neutral = $true
|
||||
service_mode = "application_protocol_agnostic"
|
||||
channel_stats = @{
|
||||
"c18z82-primary-degraded" = @{
|
||||
last_route_id = $PrimaryRouteID
|
||||
last_failed_route_id = $PrimaryRouteID
|
||||
route_generation = "$runId-primary"
|
||||
last_error = "c18z82 primary route degraded; alternate added after lease"
|
||||
last_send_duration_ms = 1200
|
||||
consecutive_failures = 3
|
||||
stall_count = 2
|
||||
route_rebuild_recommended = $true
|
||||
degraded_fallback_recommended = $false
|
||||
quality_window_sample_count = 8
|
||||
quality_window_success_count = 2
|
||||
quality_window_failure_count = 3
|
||||
quality_window_slow_count = 2
|
||||
quality_window_drop_count = 0
|
||||
quality_window_avg_latency_ms = 1200
|
||||
quality_window_last_updated_at = $observedAt
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
smoke = @{ name = "c18z82_service_channel_no_safe_recovery"; run_id = $runId }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function Send-ReplacementDegradedHeartbeat {
|
||||
param([string]$EntryNodeID, [string]$ReplacementRouteID, [string]$RouteGeneration)
|
||||
$observedAt = (Get-Date).ToUniversalTime().ToString("o")
|
||||
return Invoke-Api -Method POST -Path "/clusters/$ClusterID/nodes/$EntryNodeID/heartbeats" -Body @{
|
||||
health_status = "healthy"
|
||||
reported_version = "0.2.256-c18z82"
|
||||
capabilities = @{
|
||||
fabric_service_channel_runtime = $true
|
||||
fabric_service_channel_route_quality_feedback = $true
|
||||
smoke_feedback_injection = "c18z82"
|
||||
}
|
||||
service_states = @{ smoke = "c18z82_replacement_route_degraded_no_safe_recovery" }
|
||||
metadata = @{
|
||||
fabric_service_channel_runtime_report = @{
|
||||
schema_version = "c18l.fabric_service_channel_runtime_report.v1"
|
||||
config_version = "$runId-replacement-degraded"
|
||||
cluster_id = $ClusterID
|
||||
local_node_id = $EntryNodeID
|
||||
observed_at = $observedAt
|
||||
ingress = @{
|
||||
last_selected_route_id = $ReplacementRouteID
|
||||
send_route_failures = 4
|
||||
flow_scheduler = @{
|
||||
schema_version = "rap.fabric_flow_scheduler.v1"
|
||||
service_neutral = $true
|
||||
service_mode = "application_protocol_agnostic"
|
||||
channel_stats = @{
|
||||
"c18z82-replacement-degraded" = @{
|
||||
last_route_id = $ReplacementRouteID
|
||||
last_failed_route_id = $ReplacementRouteID
|
||||
route_generation = $RouteGeneration
|
||||
last_error = "c18z82 replacement route degraded; no safe recovery route exists"
|
||||
last_send_duration_ms = 1500
|
||||
consecutive_failures = 4
|
||||
stall_count = 2
|
||||
route_rebuild_recommended = $true
|
||||
degraded_fallback_recommended = $true
|
||||
quality_window_sample_count = 10
|
||||
quality_window_success_count = 1
|
||||
quality_window_failure_count = 4
|
||||
quality_window_slow_count = 3
|
||||
quality_window_drop_count = 0
|
||||
quality_window_avg_latency_ms = 1500
|
||||
quality_window_last_updated_at = $observedAt
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
smoke = @{ name = "c18z82_service_channel_no_safe_recovery"; run_id = $runId }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
$entryNode = Get-NodeByName -Name $EntryNodeName
|
||||
$relayNode = Get-NodeByName -Name $RelayNodeName
|
||||
$exitNode = Get-NodeByName -Name $ExitNodeName
|
||||
[void](Disable-ExistingRouteIntents -SourceNodeID $entryNode.id -DestinationNodeID $exitNode.id)
|
||||
$route = New-RouteIntent -SourceNodeID $entryNode.id -DestinationNodeID $exitNode.id -Hops @($entryNode.id, $exitNode.id)
|
||||
$lease = (Invoke-Api -Method POST -Path "/clusters/$ClusterID/fabric/service-channels/leases" -Body @{
|
||||
actor_user_id = $ActorUserID
|
||||
organization_id = "smoke-org"
|
||||
user_id = "smoke-user"
|
||||
resource_id = "c18z82-vpn-smoke"
|
||||
service_class = "vpn_packets"
|
||||
entry_node_ids = @([string]$entryNode.id)
|
||||
exit_node_ids = @([string]$exitNode.id)
|
||||
preferred_entry_node_id = [string]$entryNode.id
|
||||
preferred_exit_node_id = [string]$exitNode.id
|
||||
allowed_channels = @("vpn_packet", "fabric_control")
|
||||
ttl_seconds = 180
|
||||
metadata = @{ smoke = "c18z82_service_channel_no_safe_recovery"; run_id = $runId }
|
||||
}).fabric_service_channel_lease
|
||||
$primaryRouteID = [string](Get-PropertyValue -Item (Get-PropertyValue -Item $lease -Name "primary_route" -Default $null) -Name "route_id" -Default $route.id)
|
||||
|
||||
[void](Send-DegradedHeartbeat -EntryNodeID $entryNode.id -PrimaryRouteID $primaryRouteID)
|
||||
|
||||
$matchingChannel = $null
|
||||
for ($i = 0; $i -lt 10; $i++) {
|
||||
Start-Sleep -Seconds 3
|
||||
$accessTelemetry = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/fabric/service-channels/access-telemetry?actor_user_id=$ActorUserID&limit=50").fabric_service_channel_access_telemetry
|
||||
$matchingChannel = @($accessTelemetry.active_channels | Where-Object { $_.channel_id -eq $lease.channel_id }) | Select-Object -First 1
|
||||
if ($null -ne $matchingChannel -and [string](Get-PropertyValue -Item $matchingChannel -Name "remediation_action" -Default "") -eq "rebuild_route") {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
$command = Get-PropertyValue -Item $matchingChannel -Name "remediation_command" -Default $null
|
||||
$commandID = [string](Get-PropertyValue -Item $command -Name "command_id" -Default "")
|
||||
$alternate = New-RouteIntent -SourceNodeID $entryNode.id -DestinationNodeID $exitNode.id -Hops @($entryNode.id, $relayNode.id, $exitNode.id)
|
||||
$alternateRouteID = [string]$alternate.id
|
||||
[void](Invoke-Api -Method GET -Path "/clusters/$ClusterID/nodes/$($entryNode.id)/mesh/synthetic-config")
|
||||
|
||||
$attempts = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/fabric/service-channels/rebuild-attempts?actor_user_id=$ActorUserID&reporter_node_id=$($entryNode.id)&rebuild_request_id=$commandID&limit=10").rebuild_attempts
|
||||
$attempt = @($attempts | Where-Object { $_.rebuild_request_id -eq $commandID }) | Select-Object -First 1
|
||||
|
||||
$routeManagerDecision = $null
|
||||
$routeManagerTransition = $null
|
||||
for ($i = 0; $i -lt 12; $i++) {
|
||||
Start-Sleep -Seconds 5
|
||||
$heartbeats = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/nodes/$($entryNode.id)/heartbeats?actor_user_id=$ActorUserID&limit=5").heartbeats
|
||||
foreach ($heartbeat in @($heartbeats)) {
|
||||
$runtimeReport = Get-PropertyValue -Item (Get-PropertyValue -Item $heartbeat -Name "metadata" -Default $null) -Name "fabric_service_channel_runtime_report" -Default $null
|
||||
$ingress = Get-PropertyValue -Item $runtimeReport -Name "ingress" -Default $null
|
||||
$routeManager = Get-PropertyValue -Item $ingress -Name "route_manager" -Default $null
|
||||
$routeManagerTransition = Get-PropertyValue -Item $ingress -Name "route_manager_transition" -Default $null
|
||||
$decisions = @()
|
||||
if ($null -ne $routeManager -and $routeManager.PSObject.Properties.Name -contains "decisions") {
|
||||
$decisions = @($routeManager.decisions)
|
||||
}
|
||||
$routeManagerDecision = $decisions | Where-Object {
|
||||
[string](Get-PropertyValue -Item $_ -Name "rebuild_request_id" -Default "") -eq $commandID -and
|
||||
[string](Get-PropertyValue -Item $_ -Name "route_id" -Default "") -eq $primaryRouteID -and
|
||||
[string](Get-PropertyValue -Item $_ -Name "replacement_route_id" -Default "") -eq $alternateRouteID
|
||||
} | Select-Object -First 1
|
||||
if ($null -ne $routeManagerDecision) {
|
||||
break
|
||||
}
|
||||
}
|
||||
if ($null -ne $routeManagerDecision) {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
$packetPath = $lease.entry_http.path_template.
|
||||
Replace("{cluster_id}", $ClusterID).
|
||||
Replace("{channel_id}", [string]$lease.channel_id).
|
||||
Replace("{resource_id}", "c18z82-vpn-smoke")
|
||||
$packetUrl = $EntryBaseUrl.TrimEnd("/") + $packetPath
|
||||
$headers = @{
|
||||
"X-RAP-Service-Channel-Token" = [string]$lease.token.token
|
||||
"X-RAP-Fabric-Channel-ID" = [string]$lease.channel_id
|
||||
"X-RAP-Service-Class" = "vpn_packets"
|
||||
"X-RAP-Channel-Class" = "vpn_packet"
|
||||
"X-RAP-Traffic-Class" = "interactive"
|
||||
}
|
||||
$baselineFallbackLocal = 0
|
||||
$baselineRouteFailures = 0
|
||||
$baselineFlowDropped = 0
|
||||
$baselineSchedulerDropped = 0
|
||||
$baselineHeartbeats = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/nodes/$($entryNode.id)/heartbeats?actor_user_id=$ActorUserID&limit=1").heartbeats
|
||||
if (@($baselineHeartbeats).Count -gt 0) {
|
||||
$baselineRuntimeReport = Get-PropertyValue -Item (Get-PropertyValue -Item $baselineHeartbeats[0] -Name "metadata" -Default $null) -Name "fabric_service_channel_runtime_report" -Default $null
|
||||
$baselineIngress = Get-PropertyValue -Item $baselineRuntimeReport -Name "ingress" -Default $null
|
||||
$baselineFlowScheduler = Get-PropertyValue -Item $baselineIngress -Name "flow_scheduler" -Default $null
|
||||
$baselineFallbackLocal = [int](Get-PropertyValue -Item $baselineIngress -Name "send_fallback_local" -Default 0)
|
||||
$baselineRouteFailures = [int](Get-PropertyValue -Item $baselineIngress -Name "send_route_failures" -Default 0)
|
||||
$baselineFlowDropped = [int](Get-PropertyValue -Item $baselineIngress -Name "send_flow_dropped" -Default 0)
|
||||
$baselineSchedulerDropped = [int](Get-PropertyValue -Item $baselineFlowScheduler -Name "dropped" -Default 0)
|
||||
}
|
||||
$trafficPackets = @()
|
||||
foreach ($offset in 0..15) {
|
||||
$trafficPackets += ,(New-IPv4TcpPacket -SourcePort (54000 + $offset) -DestinationPort 3389)
|
||||
}
|
||||
$trafficPayload = ConvertTo-VPNPacketBatch -Packets $trafficPackets
|
||||
$trafficPayloadPath = Join-Path ([System.IO.Path]::GetTempPath()) "$runId-vpn-packet-batch.bin"
|
||||
[System.IO.File]::WriteAllBytes($trafficPayloadPath, [byte[]]$trafficPayload)
|
||||
$trafficStarted = Get-Date
|
||||
$trafficResponse = Invoke-WebRequest -Method Post -Uri ($packetUrl + "?batch=true") -Headers $headers -InFile $trafficPayloadPath -ContentType "application/vnd.rap.vpn-packet-batch.v1" -TimeoutSec 30
|
||||
$trafficDurationMs = [int]((Get-Date) - $trafficStarted).TotalMilliseconds
|
||||
$trafficAcceptedBy = [string]$trafficResponse.Headers["X-RAP-Service-Channel-Accepted-By"]
|
||||
Remove-Item -Path $trafficPayloadPath -Force -ErrorAction SilentlyContinue
|
||||
|
||||
$replacementTrafficObserved = $false
|
||||
$replacementLastSelected = ""
|
||||
$replacementFlowStats = @()
|
||||
$postRemediationFallbackLocal = 0
|
||||
$postRemediationRouteFailures = 0
|
||||
$postRemediationFlowDropped = 0
|
||||
$postRemediationSchedulerDropped = 0
|
||||
for ($i = 0; $i -lt 12; $i++) {
|
||||
Start-Sleep -Seconds 5
|
||||
$heartbeats = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/nodes/$($entryNode.id)/heartbeats?actor_user_id=$ActorUserID&limit=5").heartbeats
|
||||
foreach ($heartbeat in @($heartbeats)) {
|
||||
$runtimeReport = Get-PropertyValue -Item (Get-PropertyValue -Item $heartbeat -Name "metadata" -Default $null) -Name "fabric_service_channel_runtime_report" -Default $null
|
||||
$ingress = Get-PropertyValue -Item $runtimeReport -Name "ingress" -Default $null
|
||||
$replacementLastSelected = [string](Get-PropertyValue -Item $ingress -Name "last_selected_route_id" -Default "")
|
||||
$postRemediationFallbackLocal = [int](Get-PropertyValue -Item $ingress -Name "send_fallback_local" -Default 0)
|
||||
$postRemediationRouteFailures = [int](Get-PropertyValue -Item $ingress -Name "send_route_failures" -Default 0)
|
||||
$postRemediationFlowDropped = [int](Get-PropertyValue -Item $ingress -Name "send_flow_dropped" -Default 0)
|
||||
$flowScheduler = Get-PropertyValue -Item $ingress -Name "flow_scheduler" -Default $null
|
||||
$postRemediationSchedulerDropped = [int](Get-PropertyValue -Item $flowScheduler -Name "dropped" -Default 0)
|
||||
$channelStats = Get-PropertyValue -Item $flowScheduler -Name "channel_stats" -Default $null
|
||||
if ($null -ne $channelStats) {
|
||||
$replacementFlowStats = @()
|
||||
foreach ($property in @($channelStats.PSObject.Properties)) {
|
||||
$stat = $property.Value
|
||||
if ([string](Get-PropertyValue -Item $stat -Name "last_route_id" -Default "") -eq $alternateRouteID) {
|
||||
$replacementFlowStats += $stat
|
||||
}
|
||||
}
|
||||
}
|
||||
if ($replacementLastSelected -eq $alternateRouteID -and $replacementFlowStats.Count -ge 1) {
|
||||
$replacementTrafficObserved = $true
|
||||
break
|
||||
}
|
||||
}
|
||||
if ($replacementTrafficObserved) {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
$postAccess = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/fabric/service-channels/access-telemetry?actor_user_id=$ActorUserID&limit=50").fabric_service_channel_access_telemetry
|
||||
$postChannel = @($postAccess.active_channels | Where-Object { $_.channel_id -eq $lease.channel_id }) | Select-Object -First 1
|
||||
|
||||
[void](Send-ReplacementDegradedHeartbeat -EntryNodeID $entryNode.id -ReplacementRouteID $alternateRouteID -RouteGeneration "$runId-primary")
|
||||
$noSafeSyntheticConfig = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/nodes/$($entryNode.id)/mesh/synthetic-config").synthetic_mesh_config
|
||||
$noSafePathDecisions = @()
|
||||
if ($noSafeSyntheticConfig.PSObject.Properties.Name -contains "route_path_decisions") {
|
||||
$noSafePathDecisions = @($noSafeSyntheticConfig.route_path_decisions.decisions)
|
||||
}
|
||||
$noSafeDecision = $noSafePathDecisions | Where-Object {
|
||||
[string](Get-PropertyValue -Item $_ -Name "route_id" -Default "") -eq $alternateRouteID -and
|
||||
[string](Get-PropertyValue -Item $_ -Name "decision_source" -Default "") -eq "service_channel_feedback_no_alternate"
|
||||
} | Select-Object -First 1
|
||||
$noSafeFeedback = $null
|
||||
$serviceChannelFeedback = Get-PropertyValue -Item $noSafeSyntheticConfig -Name "service_channel_feedback" -Default $null
|
||||
if ($null -ne $serviceChannelFeedback -and $serviceChannelFeedback.PSObject.Properties.Name -contains "routes") {
|
||||
$noSafeFeedback = @($serviceChannelFeedback.routes | Where-Object { [string](Get-PropertyValue -Item $_ -Name "route_id" -Default "") -eq $alternateRouteID }) | Select-Object -First 1
|
||||
}
|
||||
|
||||
$postNoSafeAccess = $null
|
||||
$postNoSafeChannel = $null
|
||||
for ($i = 0; $i -lt 12; $i++) {
|
||||
Start-Sleep -Seconds 3
|
||||
$postNoSafeAccess = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/fabric/service-channels/access-telemetry?actor_user_id=$ActorUserID&limit=50").fabric_service_channel_access_telemetry
|
||||
$postNoSafeChannel = @($postNoSafeAccess.active_channels | Where-Object { $_.channel_id -eq $lease.channel_id }) | Select-Object -First 1
|
||||
if ($null -ne $postNoSafeChannel -and [string](Get-PropertyValue -Item $postNoSafeChannel -Name "route_decision_source" -Default "") -eq "service_channel_feedback_no_alternate") {
|
||||
break
|
||||
}
|
||||
}
|
||||
$postNoSafeHealth = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/fabric/service-channels/rebuild-health?actor_user_id=$ActorUserID&limit=50").rebuild_health
|
||||
$postNoSafeIncidents = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/fabric/service-channels/rebuild-incidents?actor_user_id=$ActorUserID&limit=10").rebuild_incidents
|
||||
$postNoSafeIncident = @($postNoSafeIncidents | Where-Object {
|
||||
[string](Get-PropertyValue -Item $_ -Name "incident_source" -Default "") -eq "access_decision" -and
|
||||
[string](Get-PropertyValue -Item $_ -Name "guard_status" -Default "") -eq "access_no_safe_recovery" -and
|
||||
[string](Get-PropertyValue -Item $_ -Name "channel_id" -Default "") -eq [string]$lease.channel_id
|
||||
}) | Select-Object -First 1
|
||||
$silence = $null
|
||||
$postSilenceHealth = $null
|
||||
$postSilenceIncident = $null
|
||||
if ($null -ne $postNoSafeIncident) {
|
||||
$silence = Invoke-Api -Method POST -Path "/clusters/$ClusterID/fabric/service-channels/rebuild-health/silences" -Body @{
|
||||
actor_user_id = $ActorUserID
|
||||
incident_source = "access_decision"
|
||||
channel_id = [string]$lease.channel_id
|
||||
reporter_node_id = [string](Get-PropertyValue -Item $postNoSafeIncident -Name "reporter_node_id" -Default "")
|
||||
route_id = [string](Get-PropertyValue -Item $postNoSafeIncident -Name "route_id" -Default "")
|
||||
guard_status = "access_no_safe_recovery"
|
||||
generation = [string](Get-PropertyValue -Item $postNoSafeIncident -Name "generation" -Default "")
|
||||
reason = "c18z86 operator acknowledged access no-safe"
|
||||
ttl_seconds = 21600
|
||||
}
|
||||
Start-Sleep -Seconds 2
|
||||
$postSilenceHealth = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/fabric/service-channels/rebuild-health?actor_user_id=$ActorUserID&limit=50").rebuild_health
|
||||
$postSilenceIncidents = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/fabric/service-channels/rebuild-incidents?actor_user_id=$ActorUserID&limit=10").rebuild_incidents
|
||||
$postSilenceIncident = @($postSilenceIncidents | Where-Object {
|
||||
[string](Get-PropertyValue -Item $_ -Name "incident_source" -Default "") -eq "access_decision" -and
|
||||
[string](Get-PropertyValue -Item $_ -Name "guard_status" -Default "") -eq "access_no_safe_recovery" -and
|
||||
[string](Get-PropertyValue -Item $_ -Name "channel_id" -Default "") -eq [string]$lease.channel_id
|
||||
}) | Select-Object -First 1
|
||||
|
||||
}
|
||||
|
||||
$backendLine = (& ssh $DockerSSH "docker ps --format '{{.Names}} {{.Image}} {{.Status}}' | grep '^rap_test_backend '") | Out-String
|
||||
$nodeLines = (& ssh $DockerSSH "docker ps --format '{{.Names}} {{.Image}} {{.Status}}' | grep '^rap_test_node_test_'") | Out-String
|
||||
$checks = [ordered]@{
|
||||
backend_expected_image_deployed = $backendLine.Contains($ExpectedBackendImage)
|
||||
node_agent_expected_image_deployed = $nodeLines.Contains($ExpectedNodeAgentImage)
|
||||
lease_ready = ([string]$lease.status -eq "ready")
|
||||
remediation_rebuild_route_visible = ($null -ne $matchingChannel -and [string](Get-PropertyValue -Item $matchingChannel -Name "remediation_action" -Default "") -eq "rebuild_route")
|
||||
remediation_command_visible = ($null -ne $command -and $commandID.Length -gt 0)
|
||||
durable_rebuild_intent_recorded = ($null -ne $attempt)
|
||||
durable_rebuild_intent_resolved_applied = ($null -ne $attempt -and [string](Get-PropertyValue -Item $attempt -Name "rebuild_status" -Default "") -eq "applied")
|
||||
durable_rebuild_intent_outcome_replacement_selected = ($null -ne $attempt -and [string](Get-PropertyValue -Item $attempt -Name "outcome" -Default "") -eq "replacement_selected")
|
||||
durable_rebuild_intent_replacement_matches = ($null -ne $attempt -and [string](Get-PropertyValue -Item $attempt -Name "replacement_route_id" -Default "") -eq $alternateRouteID)
|
||||
durable_rebuild_intent_source_matches = ($null -ne $attempt -and [string](Get-PropertyValue -Item $attempt -Name "decision_source" -Default "") -eq "service_channel_remediation_command")
|
||||
initial_access_telemetry_reports_planner_applied = ($null -ne $postChannel -and [string](Get-PropertyValue -Item $postChannel -Name "remediation_execution_status" -Default "") -eq "rebuild_request_applied")
|
||||
node_route_manager_consumed_same_command = ($null -ne $routeManagerDecision)
|
||||
node_route_manager_applied_rebuild = ($null -ne $routeManagerDecision -and [string](Get-PropertyValue -Item $routeManagerDecision -Name "rebuild_status" -Default "") -eq "applied")
|
||||
node_route_manager_transition_applied_rebuild = ($null -ne $routeManagerTransition -and [string](Get-PropertyValue -Item $routeManagerTransition -Name "status" -Default "") -eq "applied_rebuild")
|
||||
traffic_packet_accepted = ([int]$trafficResponse.StatusCode -eq 202)
|
||||
traffic_accepted_by_runtime = ($trafficAcceptedBy -eq "introspection" -or $trafficAcceptedBy -eq "signed")
|
||||
traffic_selected_replacement_route = $replacementTrafficObserved
|
||||
traffic_last_selected_route_matches_replacement = ($replacementLastSelected -eq $alternateRouteID)
|
||||
no_backend_or_local_fallback_after_replacement = (($postRemediationFallbackLocal - $baselineFallbackLocal) -eq 0)
|
||||
no_route_failures_after_replacement = (($postRemediationRouteFailures - $baselineRouteFailures) -eq 0)
|
||||
no_flow_drops_after_replacement = (($postRemediationFlowDropped - $baselineFlowDropped) -eq 0 -and ($postRemediationSchedulerDropped - $baselineSchedulerDropped) -eq 0)
|
||||
replacement_degradation_feedback_visible = ($null -ne $noSafeDecision -and @($noSafeDecision.score_reasons | Where-Object { [string]$_ -eq "service_channel_fenced_route" }).Count -gt 0)
|
||||
no_safe_recovery_decision_visible = ($null -ne $noSafeDecision)
|
||||
no_safe_recovery_status_pending_fallback = ($null -ne $noSafeDecision -and [string](Get-PropertyValue -Item $noSafeDecision -Name "rebuild_status" -Default "") -eq "pending_degraded_fallback")
|
||||
no_safe_recovery_reason_visible = ($null -ne $noSafeDecision -and @($noSafeDecision.score_reasons | Where-Object { [string]$_ -eq "no_unfenced_alternate_route" }).Count -gt 0)
|
||||
no_silent_stickiness_to_bad_replacement = ($null -ne $noSafeDecision -and [string](Get-PropertyValue -Item $noSafeDecision -Name "decision_source" -Default "") -eq "service_channel_feedback_no_alternate")
|
||||
access_telemetry_projects_no_safe_decision = ($null -ne $postNoSafeChannel -and [string](Get-PropertyValue -Item $postNoSafeChannel -Name "route_decision_source" -Default "") -eq "service_channel_feedback_no_alternate")
|
||||
access_telemetry_projects_pending_fallback = ($null -ne $postNoSafeChannel -and [string](Get-PropertyValue -Item $postNoSafeChannel -Name "route_decision_rebuild_status" -Default "") -eq "pending_degraded_fallback")
|
||||
access_telemetry_projects_no_safe_route_id = ($null -ne $postNoSafeChannel -and @($primaryRouteID, $alternateRouteID) -contains [string](Get-PropertyValue -Item $postNoSafeChannel -Name "route_decision_route_id" -Default ""))
|
||||
access_telemetry_projects_operator_state = ($null -ne $postNoSafeChannel -and @("route_rebuild_no_safe_recovery", "rebuild_request_no_alternate") -contains [string](Get-PropertyValue -Item $postNoSafeChannel -Name "remediation_execution_status" -Default ""))
|
||||
access_telemetry_aggregates_route_decision = ($null -ne $postNoSafeAccess -and [int](Get-PropertyValue -Item $postNoSafeAccess -Name "route_decision_channel_count" -Default 0) -ge 1)
|
||||
access_telemetry_aggregates_no_safe_decision = ($null -ne $postNoSafeAccess -and [int](Get-PropertyValue -Item $postNoSafeAccess -Name "no_safe_recovery_decision_count" -Default 0) -ge 1)
|
||||
access_telemetry_aggregate_status_prioritizes_no_safe = ($null -ne $postNoSafeAccess -and [string](Get-PropertyValue -Item $postNoSafeAccess -Name "reason" -Default "") -eq "active_channels_no_safe_recovery")
|
||||
rebuild_health_projects_access_no_safe = ($null -ne $postNoSafeHealth -and [int](Get-PropertyValue -Item $postNoSafeHealth -Name "access_no_safe_count" -Default 0) -ge 1)
|
||||
rebuild_health_action_prioritizes_access_no_safe = ($null -ne $postNoSafeHealth -and [string](Get-PropertyValue -Item $postNoSafeHealth -Name "recommended_operator_action" -Default "") -eq "inspect_access_no_safe_recovery_route_pool_and_signed_policy")
|
||||
rebuild_incident_projects_access_no_safe = ($null -ne $postNoSafeIncident)
|
||||
rebuild_incident_access_no_safe_is_bad = ($null -ne $postNoSafeIncident -and [string](Get-PropertyValue -Item $postNoSafeIncident -Name "guard_severity" -Default "") -eq "bad")
|
||||
access_decision_silence_recorded = ($null -ne $silence -and $null -ne (Get-PropertyValue -Item $silence -Name "rebuild_alert_silence" -Default $null))
|
||||
access_decision_incident_silenced = ($null -ne $postSilenceIncident -and [bool](Get-PropertyValue -Item $postSilenceIncident -Name "alert_silenced" -Default $false))
|
||||
access_decision_silence_reduces_active_bad = ($null -ne $postSilenceHealth -and [int](Get-PropertyValue -Item $postSilenceHealth -Name "silenced_count" -Default 0) -ge 1)
|
||||
}
|
||||
$failed = @($checks.GetEnumerator() | Where-Object { -not $_.Value } | ForEach-Object { $_.Key })
|
||||
|
||||
$result = [ordered]@{
|
||||
schema_version = "c18z86.service_channel_access_decision_silence_smoke.v1"
|
||||
run_id = $runId
|
||||
cluster_id = $ClusterID
|
||||
channel_id = [string]$lease.channel_id
|
||||
primary_route_id = $primaryRouteID
|
||||
alternate_route_id = $alternateRouteID
|
||||
rebuild_request_id = $commandID
|
||||
passed = ($failed.Count -eq 0)
|
||||
checks = $checks
|
||||
failed_checks = $failed
|
||||
summary = [ordered]@{
|
||||
backend_container = $backendLine.Trim()
|
||||
node_containers = $nodeLines.Trim()
|
||||
remediation_command = $command
|
||||
rebuild_attempt = $attempt
|
||||
route_manager_decision = $routeManagerDecision
|
||||
route_manager_transition = $routeManagerTransition
|
||||
traffic_status_code = [int]$trafficResponse.StatusCode
|
||||
traffic_accepted_by = $trafficAcceptedBy
|
||||
traffic_duration_ms = $trafficDurationMs
|
||||
traffic_packet_count = $trafficPackets.Count
|
||||
replacement_last_selected_route_id = $replacementLastSelected
|
||||
replacement_flow_stat_count = $replacementFlowStats.Count
|
||||
replacement_flow_stats = $replacementFlowStats
|
||||
baseline_send_fallback_local = $baselineFallbackLocal
|
||||
baseline_send_route_failures = $baselineRouteFailures
|
||||
baseline_send_flow_dropped = $baselineFlowDropped
|
||||
baseline_scheduler_dropped = $baselineSchedulerDropped
|
||||
fallback_local_delta = ($postRemediationFallbackLocal - $baselineFallbackLocal)
|
||||
route_failure_delta = ($postRemediationRouteFailures - $baselineRouteFailures)
|
||||
flow_drop_delta = ($postRemediationFlowDropped - $baselineFlowDropped)
|
||||
scheduler_drop_delta = ($postRemediationSchedulerDropped - $baselineSchedulerDropped)
|
||||
post_remediation_send_fallback_local = $postRemediationFallbackLocal
|
||||
post_remediation_send_route_failures = $postRemediationRouteFailures
|
||||
post_remediation_send_flow_dropped = $postRemediationFlowDropped
|
||||
post_remediation_scheduler_dropped = $postRemediationSchedulerDropped
|
||||
post_channel = $postChannel
|
||||
no_safe_feedback = $noSafeFeedback
|
||||
no_safe_decision = $noSafeDecision
|
||||
no_safe_route_path_decisions = $noSafePathDecisions
|
||||
post_no_safe_channel = $postNoSafeChannel
|
||||
post_no_safe_access = $postNoSafeAccess
|
||||
post_no_safe_health = $postNoSafeHealth
|
||||
post_no_safe_incidents = $postNoSafeIncidents
|
||||
post_no_safe_incident = $postNoSafeIncident
|
||||
silence = $silence
|
||||
post_silence_health = $postSilenceHealth
|
||||
post_silence_incident = $postSilenceIncident
|
||||
}
|
||||
}
|
||||
|
||||
$target = Join-Path $repoRoot $ResultPath
|
||||
$result | ConvertTo-Json -Depth 80 | Set-Content -Path $target -Encoding UTF8
|
||||
|
||||
try {
|
||||
if ($primaryRouteID) {
|
||||
Invoke-Api -Method POST -Path "/clusters/$ClusterID/mesh/route-intents/$primaryRouteID/expire" -Body @{ actor_user_id = $ActorUserID } | Out-Null
|
||||
}
|
||||
if ($alternateRouteID) {
|
||||
Invoke-Api -Method POST -Path "/clusters/$ClusterID/mesh/route-intents/$alternateRouteID/expire" -Body @{ actor_user_id = $ActorUserID } | Out-Null
|
||||
}
|
||||
Start-Sleep -Seconds 3
|
||||
Invoke-Api -Method POST -Path "/clusters/$ClusterID/fabric/service-channels/leases/cleanup" -Body @{
|
||||
actor_user_id = $ActorUserID
|
||||
limit = 50
|
||||
} | Out-Null
|
||||
} catch {
|
||||
Write-Warning "cleanup failed after c18z86 smoke: $($_.Exception.Message)"
|
||||
}
|
||||
|
||||
if (-not $result.passed) {
|
||||
throw "C18Z86 access decision silence smoke failed: $($failed -join ', ')"
|
||||
}
|
||||
|
||||
Write-Host "C18Z86 service-channel access decision silence smoke passed. Result: $target"
|
||||
$result
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,600 @@
|
||||
param(
|
||||
[string]$ApiBaseUrl = "http://192.168.200.61:18121/api/v1",
|
||||
[string]$ClusterID = "cfc0743d-d960-49fb-9de8-96e063d5e4aa",
|
||||
[string]$ActorUserID = "f67d943f-5397-4b3a-a229-695fe67ad700",
|
||||
[string]$EntryNodeName = "test-1",
|
||||
[string]$RelayNodeName = "test-3",
|
||||
[string]$ExitNodeName = "test-2",
|
||||
[string]$EntryBaseUrl = "http://192.168.200.61:19131",
|
||||
[string]$DockerSSH = "test-docker",
|
||||
[string]$ExpectedBackendImage = "rap-backend:fabric-service-channel-0.2.281-c18z109",
|
||||
[string]$ExpectedNodeAgentImage = "rap-node-agent:0.2.270-c18z95",
|
||||
[string]$ResultPath = "artifacts\c18z87-service-channel-access-decision-unsilence-smoke-result.json"
|
||||
)
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
$scriptDir = Split-Path -Parent $MyInvocation.MyCommand.Path
|
||||
$repoRoot = (Resolve-Path (Join-Path $scriptDir "..\..")).ProviderPath
|
||||
$runId = "c18z87-" + (Get-Date -Format "yyyyMMdd-HHmmss")
|
||||
|
||||
function Invoke-Api {
|
||||
param([string]$Method, [string]$Path, [object]$Body = $null)
|
||||
$params = @{ Method = $Method; Uri = "$ApiBaseUrl$Path"; TimeoutSec = 30 }
|
||||
if ($null -ne $Body) {
|
||||
$params.ContentType = "application/json"
|
||||
$params.Body = ($Body | ConvertTo-Json -Depth 80)
|
||||
}
|
||||
return Invoke-RestMethod @params
|
||||
}
|
||||
|
||||
function Get-PropertyValue {
|
||||
param([object]$Item, [string]$Name, [object]$Default = $null)
|
||||
if ($null -eq $Item) { return $Default }
|
||||
$property = $Item.PSObject.Properties[$Name]
|
||||
if ($null -eq $property) { return $Default }
|
||||
return $property.Value
|
||||
}
|
||||
|
||||
function Get-NodeByName {
|
||||
param([string]$Name)
|
||||
$nodes = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/nodes?actor_user_id=$ActorUserID").nodes
|
||||
$node = @($nodes | Where-Object { $_.name -eq $Name }) | Select-Object -First 1
|
||||
if ($null -eq $node) { throw "Node '$Name' was not found" }
|
||||
return $node
|
||||
}
|
||||
|
||||
function New-IPv4TcpPacket {
|
||||
param(
|
||||
[byte[]]$Source = @(10, 77, 0, 2),
|
||||
[byte[]]$Destination = @(192, 168, 200, 95),
|
||||
[int]$SourcePort,
|
||||
[int]$DestinationPort = 3389
|
||||
)
|
||||
$packet = [byte[]]::new(40)
|
||||
$packet[0] = 0x45
|
||||
$packet[2] = 0
|
||||
$packet[3] = 40
|
||||
$packet[8] = 64
|
||||
$packet[9] = 6
|
||||
[Array]::Copy($Source, 0, $packet, 12, 4)
|
||||
[Array]::Copy($Destination, 0, $packet, 16, 4)
|
||||
$packet[20] = [byte](($SourcePort -shr 8) -band 0xff)
|
||||
$packet[21] = [byte]($SourcePort -band 0xff)
|
||||
$packet[22] = [byte](($DestinationPort -shr 8) -band 0xff)
|
||||
$packet[23] = [byte]($DestinationPort -band 0xff)
|
||||
$packet[32] = 0x50
|
||||
return $packet
|
||||
}
|
||||
|
||||
function ConvertTo-VPNPacketBatch {
|
||||
param([byte[][]]$Packets)
|
||||
$buffer = New-Object System.Collections.Generic.List[byte]
|
||||
foreach ($packet in $Packets) {
|
||||
if ($null -eq $packet -or $packet.Length -eq 0) { continue }
|
||||
$size = [uint32]$packet.Length
|
||||
$buffer.Add([byte](($size -shr 24) -band 0xff))
|
||||
$buffer.Add([byte](($size -shr 16) -band 0xff))
|
||||
$buffer.Add([byte](($size -shr 8) -band 0xff))
|
||||
$buffer.Add([byte]($size -band 0xff))
|
||||
foreach ($value in $packet) {
|
||||
$buffer.Add($value)
|
||||
}
|
||||
}
|
||||
return $buffer.ToArray()
|
||||
}
|
||||
|
||||
function New-RouteIntent {
|
||||
param([string]$SourceNodeID, [string]$DestinationNodeID, [string[]]$Hops)
|
||||
$expiresAt = (Get-Date).ToUniversalTime().AddMinutes(5).ToString("o")
|
||||
return (Invoke-Api -Method POST -Path "/clusters/$ClusterID/mesh/route-intents" -Body @{
|
||||
actor_user_id = $ActorUserID
|
||||
source_selector = @{ node_id = $SourceNodeID }
|
||||
destination_selector = @{ node_id = $DestinationNodeID }
|
||||
service_class = "vpn_packets"
|
||||
priority = 2100000000
|
||||
policy = @{
|
||||
synthetic_enabled = $true
|
||||
route_version = "$runId-primary"
|
||||
policy_version = "$runId-primary"
|
||||
peer_directory_version = "$runId-primary"
|
||||
hops = @($Hops)
|
||||
allowed_channels = @("vpn_packet", "fabric_control")
|
||||
max_ttl = 8
|
||||
max_hops = 8
|
||||
expires_at = $expiresAt
|
||||
metadata = @{ smoke = "c18z82_service_channel_no_safe_recovery"; run_id = $runId }
|
||||
}
|
||||
}).route_intent
|
||||
}
|
||||
|
||||
function Disable-ExistingRouteIntents {
|
||||
param([string]$SourceNodeID, [string]$DestinationNodeID)
|
||||
$items = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/mesh/route-intents?actor_user_id=$ActorUserID").route_intents
|
||||
foreach ($item in @($items)) {
|
||||
if ([string](Get-PropertyValue -Item $item -Name "status" -Default "") -ne "active") { continue }
|
||||
if ([string](Get-PropertyValue -Item $item -Name "service_class" -Default "") -ne "vpn_packets") { continue }
|
||||
$sourceSelector = Get-PropertyValue -Item $item -Name "source_selector" -Default $null
|
||||
$destinationSelector = Get-PropertyValue -Item $item -Name "destination_selector" -Default $null
|
||||
if ([string](Get-PropertyValue -Item $sourceSelector -Name "node_id" -Default "") -ne $SourceNodeID) { continue }
|
||||
if ([string](Get-PropertyValue -Item $destinationSelector -Name "node_id" -Default "") -ne $DestinationNodeID) { continue }
|
||||
[void](Invoke-Api -Method POST -Path "/clusters/$ClusterID/mesh/route-intents/$($item.id)/disable" -Body @{
|
||||
actor_user_id = $ActorUserID
|
||||
reason = "c18z82 isolate no-safe-recovery smoke route pair"
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
function Send-DegradedHeartbeat {
|
||||
param([string]$EntryNodeID, [string]$PrimaryRouteID)
|
||||
$observedAt = (Get-Date).ToUniversalTime().ToString("o")
|
||||
return Invoke-Api -Method POST -Path "/clusters/$ClusterID/nodes/$EntryNodeID/heartbeats" -Body @{
|
||||
health_status = "healthy"
|
||||
reported_version = "0.2.256-c18z82"
|
||||
capabilities = @{
|
||||
fabric_service_channel_runtime = $true
|
||||
fabric_service_channel_route_quality_feedback = $true
|
||||
smoke_feedback_injection = "c18z77"
|
||||
}
|
||||
service_states = @{ smoke = "c18z82_primary_degraded_alternate_after_lease" }
|
||||
metadata = @{
|
||||
fabric_service_channel_runtime_report = @{
|
||||
schema_version = "c18l.fabric_service_channel_runtime_report.v1"
|
||||
config_version = "$runId-primary"
|
||||
cluster_id = $ClusterID
|
||||
local_node_id = $EntryNodeID
|
||||
observed_at = $observedAt
|
||||
ingress = @{
|
||||
flow_scheduler = @{
|
||||
schema_version = "rap.fabric_flow_scheduler.v1"
|
||||
service_neutral = $true
|
||||
service_mode = "application_protocol_agnostic"
|
||||
channel_stats = @{
|
||||
"c18z82-primary-degraded" = @{
|
||||
last_route_id = $PrimaryRouteID
|
||||
last_failed_route_id = $PrimaryRouteID
|
||||
route_generation = "$runId-primary"
|
||||
last_error = "c18z82 primary route degraded; alternate added after lease"
|
||||
last_send_duration_ms = 1200
|
||||
consecutive_failures = 3
|
||||
stall_count = 2
|
||||
route_rebuild_recommended = $true
|
||||
degraded_fallback_recommended = $false
|
||||
quality_window_sample_count = 8
|
||||
quality_window_success_count = 2
|
||||
quality_window_failure_count = 3
|
||||
quality_window_slow_count = 2
|
||||
quality_window_drop_count = 0
|
||||
quality_window_avg_latency_ms = 1200
|
||||
quality_window_last_updated_at = $observedAt
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
smoke = @{ name = "c18z82_service_channel_no_safe_recovery"; run_id = $runId }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function Send-ReplacementDegradedHeartbeat {
|
||||
param([string]$EntryNodeID, [string]$ReplacementRouteID, [string]$RouteGeneration)
|
||||
$observedAt = (Get-Date).ToUniversalTime().ToString("o")
|
||||
return Invoke-Api -Method POST -Path "/clusters/$ClusterID/nodes/$EntryNodeID/heartbeats" -Body @{
|
||||
health_status = "healthy"
|
||||
reported_version = "0.2.256-c18z82"
|
||||
capabilities = @{
|
||||
fabric_service_channel_runtime = $true
|
||||
fabric_service_channel_route_quality_feedback = $true
|
||||
smoke_feedback_injection = "c18z82"
|
||||
}
|
||||
service_states = @{ smoke = "c18z82_replacement_route_degraded_no_safe_recovery" }
|
||||
metadata = @{
|
||||
fabric_service_channel_runtime_report = @{
|
||||
schema_version = "c18l.fabric_service_channel_runtime_report.v1"
|
||||
config_version = "$runId-replacement-degraded"
|
||||
cluster_id = $ClusterID
|
||||
local_node_id = $EntryNodeID
|
||||
observed_at = $observedAt
|
||||
ingress = @{
|
||||
last_selected_route_id = $ReplacementRouteID
|
||||
send_route_failures = 4
|
||||
flow_scheduler = @{
|
||||
schema_version = "rap.fabric_flow_scheduler.v1"
|
||||
service_neutral = $true
|
||||
service_mode = "application_protocol_agnostic"
|
||||
channel_stats = @{
|
||||
"c18z82-replacement-degraded" = @{
|
||||
last_route_id = $ReplacementRouteID
|
||||
last_failed_route_id = $ReplacementRouteID
|
||||
route_generation = $RouteGeneration
|
||||
last_error = "c18z82 replacement route degraded; no safe recovery route exists"
|
||||
last_send_duration_ms = 1500
|
||||
consecutive_failures = 4
|
||||
stall_count = 2
|
||||
route_rebuild_recommended = $true
|
||||
degraded_fallback_recommended = $true
|
||||
quality_window_sample_count = 10
|
||||
quality_window_success_count = 1
|
||||
quality_window_failure_count = 4
|
||||
quality_window_slow_count = 3
|
||||
quality_window_drop_count = 0
|
||||
quality_window_avg_latency_ms = 1500
|
||||
quality_window_last_updated_at = $observedAt
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
smoke = @{ name = "c18z82_service_channel_no_safe_recovery"; run_id = $runId }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
$entryNode = Get-NodeByName -Name $EntryNodeName
|
||||
$relayNode = Get-NodeByName -Name $RelayNodeName
|
||||
$exitNode = Get-NodeByName -Name $ExitNodeName
|
||||
[void](Disable-ExistingRouteIntents -SourceNodeID $entryNode.id -DestinationNodeID $exitNode.id)
|
||||
$route = New-RouteIntent -SourceNodeID $entryNode.id -DestinationNodeID $exitNode.id -Hops @($entryNode.id, $exitNode.id)
|
||||
$lease = (Invoke-Api -Method POST -Path "/clusters/$ClusterID/fabric/service-channels/leases" -Body @{
|
||||
actor_user_id = $ActorUserID
|
||||
organization_id = "smoke-org"
|
||||
user_id = "smoke-user"
|
||||
resource_id = "c18z82-vpn-smoke"
|
||||
service_class = "vpn_packets"
|
||||
entry_node_ids = @([string]$entryNode.id)
|
||||
exit_node_ids = @([string]$exitNode.id)
|
||||
preferred_entry_node_id = [string]$entryNode.id
|
||||
preferred_exit_node_id = [string]$exitNode.id
|
||||
allowed_channels = @("vpn_packet", "fabric_control")
|
||||
ttl_seconds = 180
|
||||
metadata = @{ smoke = "c18z82_service_channel_no_safe_recovery"; run_id = $runId }
|
||||
}).fabric_service_channel_lease
|
||||
$primaryRouteID = [string](Get-PropertyValue -Item (Get-PropertyValue -Item $lease -Name "primary_route" -Default $null) -Name "route_id" -Default $route.id)
|
||||
|
||||
[void](Send-DegradedHeartbeat -EntryNodeID $entryNode.id -PrimaryRouteID $primaryRouteID)
|
||||
|
||||
$matchingChannel = $null
|
||||
for ($i = 0; $i -lt 10; $i++) {
|
||||
Start-Sleep -Seconds 3
|
||||
$accessTelemetry = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/fabric/service-channels/access-telemetry?actor_user_id=$ActorUserID&limit=50").fabric_service_channel_access_telemetry
|
||||
$matchingChannel = @($accessTelemetry.active_channels | Where-Object { $_.channel_id -eq $lease.channel_id }) | Select-Object -First 1
|
||||
if ($null -ne $matchingChannel -and [string](Get-PropertyValue -Item $matchingChannel -Name "remediation_action" -Default "") -eq "rebuild_route") {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
$command = Get-PropertyValue -Item $matchingChannel -Name "remediation_command" -Default $null
|
||||
$commandID = [string](Get-PropertyValue -Item $command -Name "command_id" -Default "")
|
||||
$alternate = New-RouteIntent -SourceNodeID $entryNode.id -DestinationNodeID $exitNode.id -Hops @($entryNode.id, $relayNode.id, $exitNode.id)
|
||||
$alternateRouteID = [string]$alternate.id
|
||||
[void](Invoke-Api -Method GET -Path "/clusters/$ClusterID/nodes/$($entryNode.id)/mesh/synthetic-config")
|
||||
|
||||
$attempts = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/fabric/service-channels/rebuild-attempts?actor_user_id=$ActorUserID&reporter_node_id=$($entryNode.id)&rebuild_request_id=$commandID&limit=10").rebuild_attempts
|
||||
$attempt = @($attempts | Where-Object { $_.rebuild_request_id -eq $commandID }) | Select-Object -First 1
|
||||
|
||||
$routeManagerDecision = $null
|
||||
$routeManagerTransition = $null
|
||||
for ($i = 0; $i -lt 12; $i++) {
|
||||
Start-Sleep -Seconds 5
|
||||
$heartbeats = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/nodes/$($entryNode.id)/heartbeats?actor_user_id=$ActorUserID&limit=5").heartbeats
|
||||
foreach ($heartbeat in @($heartbeats)) {
|
||||
$runtimeReport = Get-PropertyValue -Item (Get-PropertyValue -Item $heartbeat -Name "metadata" -Default $null) -Name "fabric_service_channel_runtime_report" -Default $null
|
||||
$ingress = Get-PropertyValue -Item $runtimeReport -Name "ingress" -Default $null
|
||||
$routeManager = Get-PropertyValue -Item $ingress -Name "route_manager" -Default $null
|
||||
$routeManagerTransition = Get-PropertyValue -Item $ingress -Name "route_manager_transition" -Default $null
|
||||
$decisions = @()
|
||||
if ($null -ne $routeManager -and $routeManager.PSObject.Properties.Name -contains "decisions") {
|
||||
$decisions = @($routeManager.decisions)
|
||||
}
|
||||
$routeManagerDecision = $decisions | Where-Object {
|
||||
[string](Get-PropertyValue -Item $_ -Name "rebuild_request_id" -Default "") -eq $commandID -and
|
||||
[string](Get-PropertyValue -Item $_ -Name "route_id" -Default "") -eq $primaryRouteID -and
|
||||
[string](Get-PropertyValue -Item $_ -Name "replacement_route_id" -Default "") -eq $alternateRouteID
|
||||
} | Select-Object -First 1
|
||||
if ($null -ne $routeManagerDecision) {
|
||||
break
|
||||
}
|
||||
}
|
||||
if ($null -ne $routeManagerDecision) {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
$packetPath = $lease.entry_http.path_template.
|
||||
Replace("{cluster_id}", $ClusterID).
|
||||
Replace("{channel_id}", [string]$lease.channel_id).
|
||||
Replace("{resource_id}", "c18z82-vpn-smoke")
|
||||
$packetUrl = $EntryBaseUrl.TrimEnd("/") + $packetPath
|
||||
$headers = @{
|
||||
"X-RAP-Service-Channel-Token" = [string]$lease.token.token
|
||||
"X-RAP-Fabric-Channel-ID" = [string]$lease.channel_id
|
||||
"X-RAP-Service-Class" = "vpn_packets"
|
||||
"X-RAP-Channel-Class" = "vpn_packet"
|
||||
"X-RAP-Traffic-Class" = "interactive"
|
||||
}
|
||||
$baselineFallbackLocal = 0
|
||||
$baselineRouteFailures = 0
|
||||
$baselineFlowDropped = 0
|
||||
$baselineSchedulerDropped = 0
|
||||
$baselineHeartbeats = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/nodes/$($entryNode.id)/heartbeats?actor_user_id=$ActorUserID&limit=1").heartbeats
|
||||
if (@($baselineHeartbeats).Count -gt 0) {
|
||||
$baselineRuntimeReport = Get-PropertyValue -Item (Get-PropertyValue -Item $baselineHeartbeats[0] -Name "metadata" -Default $null) -Name "fabric_service_channel_runtime_report" -Default $null
|
||||
$baselineIngress = Get-PropertyValue -Item $baselineRuntimeReport -Name "ingress" -Default $null
|
||||
$baselineFlowScheduler = Get-PropertyValue -Item $baselineIngress -Name "flow_scheduler" -Default $null
|
||||
$baselineFallbackLocal = [int](Get-PropertyValue -Item $baselineIngress -Name "send_fallback_local" -Default 0)
|
||||
$baselineRouteFailures = [int](Get-PropertyValue -Item $baselineIngress -Name "send_route_failures" -Default 0)
|
||||
$baselineFlowDropped = [int](Get-PropertyValue -Item $baselineIngress -Name "send_flow_dropped" -Default 0)
|
||||
$baselineSchedulerDropped = [int](Get-PropertyValue -Item $baselineFlowScheduler -Name "dropped" -Default 0)
|
||||
}
|
||||
$trafficPackets = @()
|
||||
foreach ($offset in 0..15) {
|
||||
$trafficPackets += ,(New-IPv4TcpPacket -SourcePort (54000 + $offset) -DestinationPort 3389)
|
||||
}
|
||||
$trafficPayload = ConvertTo-VPNPacketBatch -Packets $trafficPackets
|
||||
$trafficPayloadPath = Join-Path ([System.IO.Path]::GetTempPath()) "$runId-vpn-packet-batch.bin"
|
||||
[System.IO.File]::WriteAllBytes($trafficPayloadPath, [byte[]]$trafficPayload)
|
||||
$trafficStarted = Get-Date
|
||||
$trafficResponse = Invoke-WebRequest -Method Post -Uri ($packetUrl + "?batch=true") -Headers $headers -InFile $trafficPayloadPath -ContentType "application/vnd.rap.vpn-packet-batch.v1" -TimeoutSec 30
|
||||
$trafficDurationMs = [int]((Get-Date) - $trafficStarted).TotalMilliseconds
|
||||
$trafficAcceptedBy = [string]$trafficResponse.Headers["X-RAP-Service-Channel-Accepted-By"]
|
||||
Remove-Item -Path $trafficPayloadPath -Force -ErrorAction SilentlyContinue
|
||||
|
||||
$replacementTrafficObserved = $false
|
||||
$replacementLastSelected = ""
|
||||
$replacementFlowStats = @()
|
||||
$postRemediationFallbackLocal = 0
|
||||
$postRemediationRouteFailures = 0
|
||||
$postRemediationFlowDropped = 0
|
||||
$postRemediationSchedulerDropped = 0
|
||||
for ($i = 0; $i -lt 12; $i++) {
|
||||
Start-Sleep -Seconds 5
|
||||
$heartbeats = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/nodes/$($entryNode.id)/heartbeats?actor_user_id=$ActorUserID&limit=5").heartbeats
|
||||
foreach ($heartbeat in @($heartbeats)) {
|
||||
$runtimeReport = Get-PropertyValue -Item (Get-PropertyValue -Item $heartbeat -Name "metadata" -Default $null) -Name "fabric_service_channel_runtime_report" -Default $null
|
||||
$ingress = Get-PropertyValue -Item $runtimeReport -Name "ingress" -Default $null
|
||||
$replacementLastSelected = [string](Get-PropertyValue -Item $ingress -Name "last_selected_route_id" -Default "")
|
||||
$postRemediationFallbackLocal = [int](Get-PropertyValue -Item $ingress -Name "send_fallback_local" -Default 0)
|
||||
$postRemediationRouteFailures = [int](Get-PropertyValue -Item $ingress -Name "send_route_failures" -Default 0)
|
||||
$postRemediationFlowDropped = [int](Get-PropertyValue -Item $ingress -Name "send_flow_dropped" -Default 0)
|
||||
$flowScheduler = Get-PropertyValue -Item $ingress -Name "flow_scheduler" -Default $null
|
||||
$postRemediationSchedulerDropped = [int](Get-PropertyValue -Item $flowScheduler -Name "dropped" -Default 0)
|
||||
$channelStats = Get-PropertyValue -Item $flowScheduler -Name "channel_stats" -Default $null
|
||||
if ($null -ne $channelStats) {
|
||||
$replacementFlowStats = @()
|
||||
foreach ($property in @($channelStats.PSObject.Properties)) {
|
||||
$stat = $property.Value
|
||||
if ([string](Get-PropertyValue -Item $stat -Name "last_route_id" -Default "") -eq $alternateRouteID) {
|
||||
$replacementFlowStats += $stat
|
||||
}
|
||||
}
|
||||
}
|
||||
if ($replacementLastSelected -eq $alternateRouteID -and $replacementFlowStats.Count -ge 1) {
|
||||
$replacementTrafficObserved = $true
|
||||
break
|
||||
}
|
||||
}
|
||||
if ($replacementTrafficObserved) {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
$postAccess = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/fabric/service-channels/access-telemetry?actor_user_id=$ActorUserID&limit=50").fabric_service_channel_access_telemetry
|
||||
$postChannel = @($postAccess.active_channels | Where-Object { $_.channel_id -eq $lease.channel_id }) | Select-Object -First 1
|
||||
|
||||
[void](Send-ReplacementDegradedHeartbeat -EntryNodeID $entryNode.id -ReplacementRouteID $alternateRouteID -RouteGeneration "$runId-primary")
|
||||
$noSafeSyntheticConfig = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/nodes/$($entryNode.id)/mesh/synthetic-config").synthetic_mesh_config
|
||||
$noSafePathDecisions = @()
|
||||
if ($noSafeSyntheticConfig.PSObject.Properties.Name -contains "route_path_decisions") {
|
||||
$noSafePathDecisions = @($noSafeSyntheticConfig.route_path_decisions.decisions)
|
||||
}
|
||||
$noSafeDecision = $noSafePathDecisions | Where-Object {
|
||||
[string](Get-PropertyValue -Item $_ -Name "route_id" -Default "") -eq $alternateRouteID -and
|
||||
[string](Get-PropertyValue -Item $_ -Name "decision_source" -Default "") -eq "service_channel_feedback_no_alternate"
|
||||
} | Select-Object -First 1
|
||||
$noSafeFeedback = $null
|
||||
$serviceChannelFeedback = Get-PropertyValue -Item $noSafeSyntheticConfig -Name "service_channel_feedback" -Default $null
|
||||
if ($null -ne $serviceChannelFeedback -and $serviceChannelFeedback.PSObject.Properties.Name -contains "routes") {
|
||||
$noSafeFeedback = @($serviceChannelFeedback.routes | Where-Object { [string](Get-PropertyValue -Item $_ -Name "route_id" -Default "") -eq $alternateRouteID }) | Select-Object -First 1
|
||||
}
|
||||
|
||||
$postNoSafeAccess = $null
|
||||
$postNoSafeChannel = $null
|
||||
for ($i = 0; $i -lt 12; $i++) {
|
||||
Start-Sleep -Seconds 3
|
||||
$postNoSafeAccess = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/fabric/service-channels/access-telemetry?actor_user_id=$ActorUserID&limit=50").fabric_service_channel_access_telemetry
|
||||
$postNoSafeChannel = @($postNoSafeAccess.active_channels | Where-Object { $_.channel_id -eq $lease.channel_id }) | Select-Object -First 1
|
||||
if ($null -ne $postNoSafeChannel -and [string](Get-PropertyValue -Item $postNoSafeChannel -Name "route_decision_source" -Default "") -eq "service_channel_feedback_no_alternate") {
|
||||
break
|
||||
}
|
||||
}
|
||||
$postNoSafeHealth = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/fabric/service-channels/rebuild-health?actor_user_id=$ActorUserID&limit=50").rebuild_health
|
||||
$postNoSafeIncidents = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/fabric/service-channels/rebuild-incidents?actor_user_id=$ActorUserID&limit=10").rebuild_incidents
|
||||
$postNoSafeIncident = @($postNoSafeIncidents | Where-Object {
|
||||
[string](Get-PropertyValue -Item $_ -Name "incident_source" -Default "") -eq "access_decision" -and
|
||||
[string](Get-PropertyValue -Item $_ -Name "guard_status" -Default "") -eq "access_no_safe_recovery" -and
|
||||
[string](Get-PropertyValue -Item $_ -Name "channel_id" -Default "") -eq [string]$lease.channel_id
|
||||
}) | Select-Object -First 1
|
||||
$silence = $null
|
||||
$listedSilence = $null
|
||||
$unsilence = $null
|
||||
$postSilenceHealth = $null
|
||||
$postSilenceIncident = $null
|
||||
$postUnsilenceHealth = $null
|
||||
$postUnsilenceIncident = $null
|
||||
if ($null -ne $postNoSafeIncident) {
|
||||
$silence = Invoke-Api -Method POST -Path "/clusters/$ClusterID/fabric/service-channels/rebuild-health/silences" -Body @{
|
||||
actor_user_id = $ActorUserID
|
||||
incident_source = "access_decision"
|
||||
channel_id = [string]$lease.channel_id
|
||||
reporter_node_id = [string](Get-PropertyValue -Item $postNoSafeIncident -Name "reporter_node_id" -Default "")
|
||||
route_id = [string](Get-PropertyValue -Item $postNoSafeIncident -Name "route_id" -Default "")
|
||||
guard_status = "access_no_safe_recovery"
|
||||
generation = [string](Get-PropertyValue -Item $postNoSafeIncident -Name "generation" -Default "")
|
||||
reason = "c18z87 operator acknowledged access no-safe"
|
||||
ttl_seconds = 21600
|
||||
}
|
||||
Start-Sleep -Seconds 2
|
||||
$activeSilences = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/fabric/service-channels/rebuild-health/silences?actor_user_id=$ActorUserID").rebuild_alert_silences
|
||||
$silenceID = [string](Get-PropertyValue -Item (Get-PropertyValue -Item $silence -Name "rebuild_alert_silence" -Default $null) -Name "id" -Default "")
|
||||
$listedSilence = @($activeSilences | Where-Object { [string](Get-PropertyValue -Item $_ -Name "id" -Default "") -eq $silenceID }) | Select-Object -First 1
|
||||
$postSilenceHealth = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/fabric/service-channels/rebuild-health?actor_user_id=$ActorUserID&limit=50").rebuild_health
|
||||
$postSilenceIncidents = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/fabric/service-channels/rebuild-incidents?actor_user_id=$ActorUserID&limit=10").rebuild_incidents
|
||||
$postSilenceIncident = @($postSilenceIncidents | Where-Object {
|
||||
[string](Get-PropertyValue -Item $_ -Name "incident_source" -Default "") -eq "access_decision" -and
|
||||
[string](Get-PropertyValue -Item $_ -Name "guard_status" -Default "") -eq "access_no_safe_recovery" -and
|
||||
[string](Get-PropertyValue -Item $_ -Name "channel_id" -Default "") -eq [string]$lease.channel_id
|
||||
}) | Select-Object -First 1
|
||||
|
||||
if ($silenceID.Length -gt 0) {
|
||||
$unsilence = Invoke-Api -Method DELETE -Path "/clusters/$ClusterID/fabric/service-channels/rebuild-health/silences/$silenceID" -Body @{
|
||||
actor_user_id = $ActorUserID
|
||||
reason = "c18z87 operator removed access no-safe silence"
|
||||
}
|
||||
Start-Sleep -Seconds 2
|
||||
$postUnsilenceHealth = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/fabric/service-channels/rebuild-health?actor_user_id=$ActorUserID&limit=50").rebuild_health
|
||||
$postUnsilenceIncidents = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/fabric/service-channels/rebuild-incidents?actor_user_id=$ActorUserID&limit=10").rebuild_incidents
|
||||
$postUnsilenceIncident = @($postUnsilenceIncidents | Where-Object {
|
||||
[string](Get-PropertyValue -Item $_ -Name "incident_source" -Default "") -eq "access_decision" -and
|
||||
[string](Get-PropertyValue -Item $_ -Name "guard_status" -Default "") -eq "access_no_safe_recovery" -and
|
||||
[string](Get-PropertyValue -Item $_ -Name "channel_id" -Default "") -eq [string]$lease.channel_id
|
||||
}) | Select-Object -First 1
|
||||
}
|
||||
}
|
||||
|
||||
$backendLine = (& ssh $DockerSSH "docker ps --format '{{.Names}} {{.Image}} {{.Status}}' | grep '^rap_test_backend '") | Out-String
|
||||
$nodeLines = (& ssh $DockerSSH "docker ps --format '{{.Names}} {{.Image}} {{.Status}}' | grep '^rap_test_node_test_'") | Out-String
|
||||
$checks = [ordered]@{
|
||||
backend_expected_image_deployed = $backendLine.Contains($ExpectedBackendImage)
|
||||
node_agent_expected_image_deployed = $nodeLines.Contains($ExpectedNodeAgentImage)
|
||||
lease_ready = ([string]$lease.status -eq "ready")
|
||||
remediation_rebuild_route_visible = ($null -ne $matchingChannel -and [string](Get-PropertyValue -Item $matchingChannel -Name "remediation_action" -Default "") -eq "rebuild_route")
|
||||
remediation_command_visible = ($null -ne $command -and $commandID.Length -gt 0)
|
||||
durable_rebuild_intent_recorded = ($null -ne $attempt)
|
||||
durable_rebuild_intent_resolved_applied = ($null -ne $attempt -and [string](Get-PropertyValue -Item $attempt -Name "rebuild_status" -Default "") -eq "applied")
|
||||
durable_rebuild_intent_outcome_replacement_selected = ($null -ne $attempt -and [string](Get-PropertyValue -Item $attempt -Name "outcome" -Default "") -eq "replacement_selected")
|
||||
durable_rebuild_intent_replacement_matches = ($null -ne $attempt -and [string](Get-PropertyValue -Item $attempt -Name "replacement_route_id" -Default "") -eq $alternateRouteID)
|
||||
durable_rebuild_intent_source_matches = ($null -ne $attempt -and [string](Get-PropertyValue -Item $attempt -Name "decision_source" -Default "") -eq "service_channel_remediation_command")
|
||||
initial_access_telemetry_reports_planner_applied = ($null -ne $postChannel -and [string](Get-PropertyValue -Item $postChannel -Name "remediation_execution_status" -Default "") -eq "rebuild_request_applied")
|
||||
node_route_manager_consumed_same_command = ($null -ne $routeManagerDecision)
|
||||
node_route_manager_applied_rebuild = ($null -ne $routeManagerDecision -and [string](Get-PropertyValue -Item $routeManagerDecision -Name "rebuild_status" -Default "") -eq "applied")
|
||||
node_route_manager_transition_applied_rebuild = ($null -ne $routeManagerTransition -and [string](Get-PropertyValue -Item $routeManagerTransition -Name "status" -Default "") -eq "applied_rebuild")
|
||||
traffic_packet_accepted = ([int]$trafficResponse.StatusCode -eq 202)
|
||||
traffic_accepted_by_runtime = ($trafficAcceptedBy -eq "introspection" -or $trafficAcceptedBy -eq "signed")
|
||||
traffic_selected_replacement_route = $replacementTrafficObserved
|
||||
traffic_last_selected_route_matches_replacement = ($replacementLastSelected -eq $alternateRouteID)
|
||||
no_backend_or_local_fallback_after_replacement = (($postRemediationFallbackLocal - $baselineFallbackLocal) -eq 0)
|
||||
no_route_failures_after_replacement = (($postRemediationRouteFailures - $baselineRouteFailures) -eq 0)
|
||||
no_flow_drops_after_replacement = (($postRemediationFlowDropped - $baselineFlowDropped) -eq 0 -and ($postRemediationSchedulerDropped - $baselineSchedulerDropped) -eq 0)
|
||||
replacement_degradation_feedback_visible = ($null -ne $noSafeDecision -and @($noSafeDecision.score_reasons | Where-Object { [string]$_ -eq "service_channel_fenced_route" }).Count -gt 0)
|
||||
no_safe_recovery_decision_visible = ($null -ne $noSafeDecision)
|
||||
no_safe_recovery_status_pending_fallback = ($null -ne $noSafeDecision -and [string](Get-PropertyValue -Item $noSafeDecision -Name "rebuild_status" -Default "") -eq "pending_degraded_fallback")
|
||||
no_safe_recovery_reason_visible = ($null -ne $noSafeDecision -and @($noSafeDecision.score_reasons | Where-Object { [string]$_ -eq "no_unfenced_alternate_route" }).Count -gt 0)
|
||||
no_silent_stickiness_to_bad_replacement = ($null -ne $noSafeDecision -and [string](Get-PropertyValue -Item $noSafeDecision -Name "decision_source" -Default "") -eq "service_channel_feedback_no_alternate")
|
||||
access_telemetry_projects_no_safe_decision = ($null -ne $postNoSafeChannel -and [string](Get-PropertyValue -Item $postNoSafeChannel -Name "route_decision_source" -Default "") -eq "service_channel_feedback_no_alternate")
|
||||
access_telemetry_projects_pending_fallback = ($null -ne $postNoSafeChannel -and [string](Get-PropertyValue -Item $postNoSafeChannel -Name "route_decision_rebuild_status" -Default "") -eq "pending_degraded_fallback")
|
||||
access_telemetry_projects_no_safe_route_id = ($null -ne $postNoSafeChannel -and @($primaryRouteID, $alternateRouteID) -contains [string](Get-PropertyValue -Item $postNoSafeChannel -Name "route_decision_route_id" -Default ""))
|
||||
access_telemetry_projects_operator_state = ($null -ne $postNoSafeChannel -and @("route_rebuild_no_safe_recovery", "rebuild_request_no_alternate") -contains [string](Get-PropertyValue -Item $postNoSafeChannel -Name "remediation_execution_status" -Default ""))
|
||||
access_telemetry_aggregates_route_decision = ($null -ne $postNoSafeAccess -and [int](Get-PropertyValue -Item $postNoSafeAccess -Name "route_decision_channel_count" -Default 0) -ge 1)
|
||||
access_telemetry_aggregates_no_safe_decision = ($null -ne $postNoSafeAccess -and [int](Get-PropertyValue -Item $postNoSafeAccess -Name "no_safe_recovery_decision_count" -Default 0) -ge 1)
|
||||
access_telemetry_aggregate_status_prioritizes_no_safe = ($null -ne $postNoSafeAccess -and [string](Get-PropertyValue -Item $postNoSafeAccess -Name "reason" -Default "") -eq "active_channels_no_safe_recovery")
|
||||
rebuild_health_projects_access_no_safe = ($null -ne $postNoSafeHealth -and [int](Get-PropertyValue -Item $postNoSafeHealth -Name "access_no_safe_count" -Default 0) -ge 1)
|
||||
rebuild_health_action_prioritizes_access_no_safe = ($null -ne $postNoSafeHealth -and [string](Get-PropertyValue -Item $postNoSafeHealth -Name "recommended_operator_action" -Default "") -eq "inspect_access_no_safe_recovery_route_pool_and_signed_policy")
|
||||
rebuild_incident_projects_access_no_safe = ($null -ne $postNoSafeIncident)
|
||||
rebuild_incident_access_no_safe_is_bad = ($null -ne $postNoSafeIncident -and [string](Get-PropertyValue -Item $postNoSafeIncident -Name "guard_severity" -Default "") -eq "bad")
|
||||
access_decision_silence_recorded = ($null -ne $silence -and $null -ne (Get-PropertyValue -Item $silence -Name "rebuild_alert_silence" -Default $null))
|
||||
access_decision_silence_listed = ($null -ne $listedSilence -and [string](Get-PropertyValue -Item $listedSilence -Name "incident_source" -Default "") -eq "access_decision" -and [string](Get-PropertyValue -Item $listedSilence -Name "channel_id" -Default "") -eq [string]$lease.channel_id)
|
||||
access_decision_incident_silenced = ($null -ne $postSilenceIncident -and [bool](Get-PropertyValue -Item $postSilenceIncident -Name "alert_silenced" -Default $false))
|
||||
access_decision_silence_reduces_active_bad = ($null -ne $postSilenceHealth -and [int](Get-PropertyValue -Item $postSilenceHealth -Name "silenced_count" -Default 0) -ge 1)
|
||||
access_decision_unsilence_recorded = ($null -ne $unsilence -and $null -ne (Get-PropertyValue -Item $unsilence -Name "rebuild_alert_silence" -Default $null))
|
||||
access_decision_unsilence_restores_active_bad = ($null -ne $postUnsilenceHealth -and [int](Get-PropertyValue -Item $postUnsilenceHealth -Name "active_bad_count" -Default 0) -ge 1 -and [int](Get-PropertyValue -Item $postUnsilenceHealth -Name "silenced_count" -Default 0) -eq 0)
|
||||
access_decision_incident_unsilenced = ($null -ne $postUnsilenceIncident -and -not [bool](Get-PropertyValue -Item $postUnsilenceIncident -Name "alert_silenced" -Default $false))
|
||||
}
|
||||
$failed = @($checks.GetEnumerator() | Where-Object { -not $_.Value } | ForEach-Object { $_.Key })
|
||||
|
||||
$result = [ordered]@{
|
||||
schema_version = "c18z87.service_channel_access_decision_unsilence_smoke.v1"
|
||||
run_id = $runId
|
||||
cluster_id = $ClusterID
|
||||
channel_id = [string]$lease.channel_id
|
||||
primary_route_id = $primaryRouteID
|
||||
alternate_route_id = $alternateRouteID
|
||||
rebuild_request_id = $commandID
|
||||
passed = ($failed.Count -eq 0)
|
||||
checks = $checks
|
||||
failed_checks = $failed
|
||||
summary = [ordered]@{
|
||||
backend_container = $backendLine.Trim()
|
||||
node_containers = $nodeLines.Trim()
|
||||
remediation_command = $command
|
||||
rebuild_attempt = $attempt
|
||||
route_manager_decision = $routeManagerDecision
|
||||
route_manager_transition = $routeManagerTransition
|
||||
traffic_status_code = [int]$trafficResponse.StatusCode
|
||||
traffic_accepted_by = $trafficAcceptedBy
|
||||
traffic_duration_ms = $trafficDurationMs
|
||||
traffic_packet_count = $trafficPackets.Count
|
||||
replacement_last_selected_route_id = $replacementLastSelected
|
||||
replacement_flow_stat_count = $replacementFlowStats.Count
|
||||
replacement_flow_stats = $replacementFlowStats
|
||||
baseline_send_fallback_local = $baselineFallbackLocal
|
||||
baseline_send_route_failures = $baselineRouteFailures
|
||||
baseline_send_flow_dropped = $baselineFlowDropped
|
||||
baseline_scheduler_dropped = $baselineSchedulerDropped
|
||||
fallback_local_delta = ($postRemediationFallbackLocal - $baselineFallbackLocal)
|
||||
route_failure_delta = ($postRemediationRouteFailures - $baselineRouteFailures)
|
||||
flow_drop_delta = ($postRemediationFlowDropped - $baselineFlowDropped)
|
||||
scheduler_drop_delta = ($postRemediationSchedulerDropped - $baselineSchedulerDropped)
|
||||
post_remediation_send_fallback_local = $postRemediationFallbackLocal
|
||||
post_remediation_send_route_failures = $postRemediationRouteFailures
|
||||
post_remediation_send_flow_dropped = $postRemediationFlowDropped
|
||||
post_remediation_scheduler_dropped = $postRemediationSchedulerDropped
|
||||
post_channel = $postChannel
|
||||
no_safe_feedback = $noSafeFeedback
|
||||
no_safe_decision = $noSafeDecision
|
||||
no_safe_route_path_decisions = $noSafePathDecisions
|
||||
post_no_safe_channel = $postNoSafeChannel
|
||||
post_no_safe_access = $postNoSafeAccess
|
||||
post_no_safe_health = $postNoSafeHealth
|
||||
post_no_safe_incidents = $postNoSafeIncidents
|
||||
post_no_safe_incident = $postNoSafeIncident
|
||||
silence = $silence
|
||||
listed_silence = $listedSilence
|
||||
unsilence = $unsilence
|
||||
post_silence_health = $postSilenceHealth
|
||||
post_silence_incident = $postSilenceIncident
|
||||
post_unsilence_health = $postUnsilenceHealth
|
||||
post_unsilence_incident = $postUnsilenceIncident
|
||||
}
|
||||
}
|
||||
|
||||
$target = Join-Path $repoRoot $ResultPath
|
||||
$result | ConvertTo-Json -Depth 80 | Set-Content -Path $target -Encoding UTF8
|
||||
|
||||
try {
|
||||
if ($primaryRouteID) {
|
||||
Invoke-Api -Method POST -Path "/clusters/$ClusterID/mesh/route-intents/$primaryRouteID/expire" -Body @{ actor_user_id = $ActorUserID } | Out-Null
|
||||
}
|
||||
if ($alternateRouteID) {
|
||||
Invoke-Api -Method POST -Path "/clusters/$ClusterID/mesh/route-intents/$alternateRouteID/expire" -Body @{ actor_user_id = $ActorUserID } | Out-Null
|
||||
}
|
||||
Start-Sleep -Seconds 3
|
||||
Invoke-Api -Method POST -Path "/clusters/$ClusterID/fabric/service-channels/leases/cleanup" -Body @{
|
||||
actor_user_id = $ActorUserID
|
||||
limit = 50
|
||||
} | Out-Null
|
||||
} catch {
|
||||
Write-Warning "cleanup failed after c18z87 smoke: $($_.Exception.Message)"
|
||||
}
|
||||
|
||||
if (-not $result.passed) {
|
||||
throw "C18Z87 access decision unsilence smoke failed: $($failed -join ', ')"
|
||||
}
|
||||
|
||||
Write-Host "C18Z87 service-channel access decision unsilence smoke passed. Result: $target"
|
||||
$result
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,595 @@
|
||||
param(
|
||||
[string]$ApiBaseUrl = "http://192.168.200.61:18121/api/v1",
|
||||
[string]$ClusterID = "cfc0743d-d960-49fb-9de8-96e063d5e4aa",
|
||||
[string]$ActorUserID = "f67d943f-5397-4b3a-a229-695fe67ad700",
|
||||
[string]$EntryNodeName = "test-1",
|
||||
[string]$RelayNodeName = "test-3",
|
||||
[string]$ExitNodeName = "test-2",
|
||||
[string]$EntryBaseUrl = "http://192.168.200.61:19131",
|
||||
[string]$DockerSSH = "test-docker",
|
||||
[string]$ExpectedBackendImage = "rap-backend:fabric-service-channel-0.2.281-c18z109",
|
||||
[string]$ExpectedNodeAgentImage = "rap-node-agent:0.2.270-c18z95",
|
||||
[string]$ResultPath = "artifacts\c18z88-service-channel-access-decision-resurface-smoke-result.json"
|
||||
)
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
$scriptDir = Split-Path -Parent $MyInvocation.MyCommand.Path
|
||||
$repoRoot = (Resolve-Path (Join-Path $scriptDir "..\..")).ProviderPath
|
||||
$runId = "c18z88-" + (Get-Date -Format "yyyyMMdd-HHmmss")
|
||||
|
||||
function Invoke-Api {
|
||||
param([string]$Method, [string]$Path, [object]$Body = $null)
|
||||
$params = @{ Method = $Method; Uri = "$ApiBaseUrl$Path"; TimeoutSec = 30 }
|
||||
if ($null -ne $Body) {
|
||||
$params.ContentType = "application/json"
|
||||
$params.Body = ($Body | ConvertTo-Json -Depth 80)
|
||||
}
|
||||
return Invoke-RestMethod @params
|
||||
}
|
||||
|
||||
function Get-PropertyValue {
|
||||
param([object]$Item, [string]$Name, [object]$Default = $null)
|
||||
if ($null -eq $Item) { return $Default }
|
||||
$property = $Item.PSObject.Properties[$Name]
|
||||
if ($null -eq $property) { return $Default }
|
||||
return $property.Value
|
||||
}
|
||||
|
||||
function Get-NodeByName {
|
||||
param([string]$Name)
|
||||
$nodes = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/nodes?actor_user_id=$ActorUserID").nodes
|
||||
$node = @($nodes | Where-Object { $_.name -eq $Name }) | Select-Object -First 1
|
||||
if ($null -eq $node) { throw "Node '$Name' was not found" }
|
||||
return $node
|
||||
}
|
||||
|
||||
function New-IPv4TcpPacket {
|
||||
param(
|
||||
[byte[]]$Source = @(10, 77, 0, 2),
|
||||
[byte[]]$Destination = @(192, 168, 200, 95),
|
||||
[int]$SourcePort,
|
||||
[int]$DestinationPort = 3389
|
||||
)
|
||||
$packet = [byte[]]::new(40)
|
||||
$packet[0] = 0x45
|
||||
$packet[2] = 0
|
||||
$packet[3] = 40
|
||||
$packet[8] = 64
|
||||
$packet[9] = 6
|
||||
[Array]::Copy($Source, 0, $packet, 12, 4)
|
||||
[Array]::Copy($Destination, 0, $packet, 16, 4)
|
||||
$packet[20] = [byte](($SourcePort -shr 8) -band 0xff)
|
||||
$packet[21] = [byte]($SourcePort -band 0xff)
|
||||
$packet[22] = [byte](($DestinationPort -shr 8) -band 0xff)
|
||||
$packet[23] = [byte]($DestinationPort -band 0xff)
|
||||
$packet[32] = 0x50
|
||||
return $packet
|
||||
}
|
||||
|
||||
function ConvertTo-VPNPacketBatch {
|
||||
param([byte[][]]$Packets)
|
||||
$buffer = New-Object System.Collections.Generic.List[byte]
|
||||
foreach ($packet in $Packets) {
|
||||
if ($null -eq $packet -or $packet.Length -eq 0) { continue }
|
||||
$size = [uint32]$packet.Length
|
||||
$buffer.Add([byte](($size -shr 24) -band 0xff))
|
||||
$buffer.Add([byte](($size -shr 16) -band 0xff))
|
||||
$buffer.Add([byte](($size -shr 8) -band 0xff))
|
||||
$buffer.Add([byte]($size -band 0xff))
|
||||
foreach ($value in $packet) {
|
||||
$buffer.Add($value)
|
||||
}
|
||||
}
|
||||
return $buffer.ToArray()
|
||||
}
|
||||
|
||||
function New-RouteIntent {
|
||||
param([string]$SourceNodeID, [string]$DestinationNodeID, [string[]]$Hops)
|
||||
$expiresAt = (Get-Date).ToUniversalTime().AddMinutes(5).ToString("o")
|
||||
return (Invoke-Api -Method POST -Path "/clusters/$ClusterID/mesh/route-intents" -Body @{
|
||||
actor_user_id = $ActorUserID
|
||||
source_selector = @{ node_id = $SourceNodeID }
|
||||
destination_selector = @{ node_id = $DestinationNodeID }
|
||||
service_class = "vpn_packets"
|
||||
priority = 2100000000
|
||||
policy = @{
|
||||
synthetic_enabled = $true
|
||||
route_version = "$runId-primary"
|
||||
policy_version = "$runId-primary"
|
||||
peer_directory_version = "$runId-primary"
|
||||
hops = @($Hops)
|
||||
allowed_channels = @("vpn_packet", "fabric_control")
|
||||
max_ttl = 8
|
||||
max_hops = 8
|
||||
expires_at = $expiresAt
|
||||
metadata = @{ smoke = "c18z82_service_channel_no_safe_recovery"; run_id = $runId }
|
||||
}
|
||||
}).route_intent
|
||||
}
|
||||
|
||||
function Disable-ExistingRouteIntents {
|
||||
param([string]$SourceNodeID, [string]$DestinationNodeID)
|
||||
$items = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/mesh/route-intents?actor_user_id=$ActorUserID").route_intents
|
||||
foreach ($item in @($items)) {
|
||||
if ([string](Get-PropertyValue -Item $item -Name "status" -Default "") -ne "active") { continue }
|
||||
if ([string](Get-PropertyValue -Item $item -Name "service_class" -Default "") -ne "vpn_packets") { continue }
|
||||
$sourceSelector = Get-PropertyValue -Item $item -Name "source_selector" -Default $null
|
||||
$destinationSelector = Get-PropertyValue -Item $item -Name "destination_selector" -Default $null
|
||||
if ([string](Get-PropertyValue -Item $sourceSelector -Name "node_id" -Default "") -ne $SourceNodeID) { continue }
|
||||
if ([string](Get-PropertyValue -Item $destinationSelector -Name "node_id" -Default "") -ne $DestinationNodeID) { continue }
|
||||
[void](Invoke-Api -Method POST -Path "/clusters/$ClusterID/mesh/route-intents/$($item.id)/disable" -Body @{
|
||||
actor_user_id = $ActorUserID
|
||||
reason = "c18z82 isolate no-safe-recovery smoke route pair"
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
function Send-DegradedHeartbeat {
|
||||
param([string]$EntryNodeID, [string]$PrimaryRouteID)
|
||||
$observedAt = (Get-Date).ToUniversalTime().ToString("o")
|
||||
return Invoke-Api -Method POST -Path "/clusters/$ClusterID/nodes/$EntryNodeID/heartbeats" -Body @{
|
||||
health_status = "healthy"
|
||||
reported_version = "0.2.256-c18z82"
|
||||
capabilities = @{
|
||||
fabric_service_channel_runtime = $true
|
||||
fabric_service_channel_route_quality_feedback = $true
|
||||
smoke_feedback_injection = "c18z77"
|
||||
}
|
||||
service_states = @{ smoke = "c18z82_primary_degraded_alternate_after_lease" }
|
||||
metadata = @{
|
||||
fabric_service_channel_runtime_report = @{
|
||||
schema_version = "c18l.fabric_service_channel_runtime_report.v1"
|
||||
config_version = "$runId-primary"
|
||||
cluster_id = $ClusterID
|
||||
local_node_id = $EntryNodeID
|
||||
observed_at = $observedAt
|
||||
ingress = @{
|
||||
flow_scheduler = @{
|
||||
schema_version = "rap.fabric_flow_scheduler.v1"
|
||||
service_neutral = $true
|
||||
service_mode = "application_protocol_agnostic"
|
||||
channel_stats = @{
|
||||
"c18z82-primary-degraded" = @{
|
||||
last_route_id = $PrimaryRouteID
|
||||
last_failed_route_id = $PrimaryRouteID
|
||||
route_generation = "$runId-primary"
|
||||
last_error = "c18z82 primary route degraded; alternate added after lease"
|
||||
last_send_duration_ms = 1200
|
||||
consecutive_failures = 3
|
||||
stall_count = 2
|
||||
route_rebuild_recommended = $true
|
||||
degraded_fallback_recommended = $false
|
||||
quality_window_sample_count = 8
|
||||
quality_window_success_count = 2
|
||||
quality_window_failure_count = 3
|
||||
quality_window_slow_count = 2
|
||||
quality_window_drop_count = 0
|
||||
quality_window_avg_latency_ms = 1200
|
||||
quality_window_last_updated_at = $observedAt
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
smoke = @{ name = "c18z82_service_channel_no_safe_recovery"; run_id = $runId }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function Send-ReplacementDegradedHeartbeat {
|
||||
param([string]$EntryNodeID, [string]$ReplacementRouteID, [string]$RouteGeneration)
|
||||
$observedAt = (Get-Date).ToUniversalTime().ToString("o")
|
||||
return Invoke-Api -Method POST -Path "/clusters/$ClusterID/nodes/$EntryNodeID/heartbeats" -Body @{
|
||||
health_status = "healthy"
|
||||
reported_version = "0.2.256-c18z82"
|
||||
capabilities = @{
|
||||
fabric_service_channel_runtime = $true
|
||||
fabric_service_channel_route_quality_feedback = $true
|
||||
smoke_feedback_injection = "c18z82"
|
||||
}
|
||||
service_states = @{ smoke = "c18z82_replacement_route_degraded_no_safe_recovery" }
|
||||
metadata = @{
|
||||
fabric_service_channel_runtime_report = @{
|
||||
schema_version = "c18l.fabric_service_channel_runtime_report.v1"
|
||||
config_version = "$runId-replacement-degraded"
|
||||
cluster_id = $ClusterID
|
||||
local_node_id = $EntryNodeID
|
||||
observed_at = $observedAt
|
||||
ingress = @{
|
||||
last_selected_route_id = $ReplacementRouteID
|
||||
send_route_failures = 4
|
||||
flow_scheduler = @{
|
||||
schema_version = "rap.fabric_flow_scheduler.v1"
|
||||
service_neutral = $true
|
||||
service_mode = "application_protocol_agnostic"
|
||||
channel_stats = @{
|
||||
"c18z82-replacement-degraded" = @{
|
||||
last_route_id = $ReplacementRouteID
|
||||
last_failed_route_id = $ReplacementRouteID
|
||||
route_generation = $RouteGeneration
|
||||
last_error = "c18z82 replacement route degraded; no safe recovery route exists"
|
||||
last_send_duration_ms = 1500
|
||||
consecutive_failures = 4
|
||||
stall_count = 2
|
||||
route_rebuild_recommended = $true
|
||||
degraded_fallback_recommended = $true
|
||||
quality_window_sample_count = 10
|
||||
quality_window_success_count = 1
|
||||
quality_window_failure_count = 4
|
||||
quality_window_slow_count = 3
|
||||
quality_window_drop_count = 0
|
||||
quality_window_avg_latency_ms = 1500
|
||||
quality_window_last_updated_at = $observedAt
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
smoke = @{ name = "c18z82_service_channel_no_safe_recovery"; run_id = $runId }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
$entryNode = Get-NodeByName -Name $EntryNodeName
|
||||
$relayNode = Get-NodeByName -Name $RelayNodeName
|
||||
$exitNode = Get-NodeByName -Name $ExitNodeName
|
||||
[void](Disable-ExistingRouteIntents -SourceNodeID $entryNode.id -DestinationNodeID $exitNode.id)
|
||||
$route = New-RouteIntent -SourceNodeID $entryNode.id -DestinationNodeID $exitNode.id -Hops @($entryNode.id, $exitNode.id)
|
||||
$lease = (Invoke-Api -Method POST -Path "/clusters/$ClusterID/fabric/service-channels/leases" -Body @{
|
||||
actor_user_id = $ActorUserID
|
||||
organization_id = "smoke-org"
|
||||
user_id = "smoke-user"
|
||||
resource_id = "c18z82-vpn-smoke"
|
||||
service_class = "vpn_packets"
|
||||
entry_node_ids = @([string]$entryNode.id)
|
||||
exit_node_ids = @([string]$exitNode.id)
|
||||
preferred_entry_node_id = [string]$entryNode.id
|
||||
preferred_exit_node_id = [string]$exitNode.id
|
||||
allowed_channels = @("vpn_packet", "fabric_control")
|
||||
ttl_seconds = 180
|
||||
metadata = @{ smoke = "c18z82_service_channel_no_safe_recovery"; run_id = $runId }
|
||||
}).fabric_service_channel_lease
|
||||
$primaryRouteID = [string](Get-PropertyValue -Item (Get-PropertyValue -Item $lease -Name "primary_route" -Default $null) -Name "route_id" -Default $route.id)
|
||||
|
||||
[void](Send-DegradedHeartbeat -EntryNodeID $entryNode.id -PrimaryRouteID $primaryRouteID)
|
||||
|
||||
$matchingChannel = $null
|
||||
for ($i = 0; $i -lt 10; $i++) {
|
||||
Start-Sleep -Seconds 3
|
||||
$accessTelemetry = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/fabric/service-channels/access-telemetry?actor_user_id=$ActorUserID&limit=50").fabric_service_channel_access_telemetry
|
||||
$matchingChannel = @($accessTelemetry.active_channels | Where-Object { $_.channel_id -eq $lease.channel_id }) | Select-Object -First 1
|
||||
if ($null -ne $matchingChannel -and [string](Get-PropertyValue -Item $matchingChannel -Name "remediation_action" -Default "") -eq "rebuild_route") {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
$command = Get-PropertyValue -Item $matchingChannel -Name "remediation_command" -Default $null
|
||||
$commandID = [string](Get-PropertyValue -Item $command -Name "command_id" -Default "")
|
||||
$alternate = New-RouteIntent -SourceNodeID $entryNode.id -DestinationNodeID $exitNode.id -Hops @($entryNode.id, $relayNode.id, $exitNode.id)
|
||||
$alternateRouteID = [string]$alternate.id
|
||||
[void](Invoke-Api -Method GET -Path "/clusters/$ClusterID/nodes/$($entryNode.id)/mesh/synthetic-config")
|
||||
|
||||
$attempts = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/fabric/service-channels/rebuild-attempts?actor_user_id=$ActorUserID&reporter_node_id=$($entryNode.id)&rebuild_request_id=$commandID&limit=10").rebuild_attempts
|
||||
$attempt = @($attempts | Where-Object { $_.rebuild_request_id -eq $commandID }) | Select-Object -First 1
|
||||
|
||||
$routeManagerDecision = $null
|
||||
$routeManagerTransition = $null
|
||||
for ($i = 0; $i -lt 12; $i++) {
|
||||
Start-Sleep -Seconds 5
|
||||
$heartbeats = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/nodes/$($entryNode.id)/heartbeats?actor_user_id=$ActorUserID&limit=5").heartbeats
|
||||
foreach ($heartbeat in @($heartbeats)) {
|
||||
$runtimeReport = Get-PropertyValue -Item (Get-PropertyValue -Item $heartbeat -Name "metadata" -Default $null) -Name "fabric_service_channel_runtime_report" -Default $null
|
||||
$ingress = Get-PropertyValue -Item $runtimeReport -Name "ingress" -Default $null
|
||||
$routeManager = Get-PropertyValue -Item $ingress -Name "route_manager" -Default $null
|
||||
$routeManagerTransition = Get-PropertyValue -Item $ingress -Name "route_manager_transition" -Default $null
|
||||
$decisions = @()
|
||||
if ($null -ne $routeManager -and $routeManager.PSObject.Properties.Name -contains "decisions") {
|
||||
$decisions = @($routeManager.decisions)
|
||||
}
|
||||
$routeManagerDecision = $decisions | Where-Object {
|
||||
[string](Get-PropertyValue -Item $_ -Name "rebuild_request_id" -Default "") -eq $commandID -and
|
||||
[string](Get-PropertyValue -Item $_ -Name "route_id" -Default "") -eq $primaryRouteID -and
|
||||
[string](Get-PropertyValue -Item $_ -Name "replacement_route_id" -Default "") -eq $alternateRouteID
|
||||
} | Select-Object -First 1
|
||||
if ($null -ne $routeManagerDecision) {
|
||||
break
|
||||
}
|
||||
}
|
||||
if ($null -ne $routeManagerDecision) {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
$packetPath = $lease.entry_http.path_template.
|
||||
Replace("{cluster_id}", $ClusterID).
|
||||
Replace("{channel_id}", [string]$lease.channel_id).
|
||||
Replace("{resource_id}", "c18z82-vpn-smoke")
|
||||
$packetUrl = $EntryBaseUrl.TrimEnd("/") + $packetPath
|
||||
$headers = @{
|
||||
"X-RAP-Service-Channel-Token" = [string]$lease.token.token
|
||||
"X-RAP-Fabric-Channel-ID" = [string]$lease.channel_id
|
||||
"X-RAP-Service-Class" = "vpn_packets"
|
||||
"X-RAP-Channel-Class" = "vpn_packet"
|
||||
"X-RAP-Traffic-Class" = "interactive"
|
||||
}
|
||||
$baselineFallbackLocal = 0
|
||||
$baselineRouteFailures = 0
|
||||
$baselineFlowDropped = 0
|
||||
$baselineSchedulerDropped = 0
|
||||
$baselineHeartbeats = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/nodes/$($entryNode.id)/heartbeats?actor_user_id=$ActorUserID&limit=1").heartbeats
|
||||
if (@($baselineHeartbeats).Count -gt 0) {
|
||||
$baselineRuntimeReport = Get-PropertyValue -Item (Get-PropertyValue -Item $baselineHeartbeats[0] -Name "metadata" -Default $null) -Name "fabric_service_channel_runtime_report" -Default $null
|
||||
$baselineIngress = Get-PropertyValue -Item $baselineRuntimeReport -Name "ingress" -Default $null
|
||||
$baselineFlowScheduler = Get-PropertyValue -Item $baselineIngress -Name "flow_scheduler" -Default $null
|
||||
$baselineFallbackLocal = [int](Get-PropertyValue -Item $baselineIngress -Name "send_fallback_local" -Default 0)
|
||||
$baselineRouteFailures = [int](Get-PropertyValue -Item $baselineIngress -Name "send_route_failures" -Default 0)
|
||||
$baselineFlowDropped = [int](Get-PropertyValue -Item $baselineIngress -Name "send_flow_dropped" -Default 0)
|
||||
$baselineSchedulerDropped = [int](Get-PropertyValue -Item $baselineFlowScheduler -Name "dropped" -Default 0)
|
||||
}
|
||||
$trafficPackets = @()
|
||||
foreach ($offset in 0..15) {
|
||||
$trafficPackets += ,(New-IPv4TcpPacket -SourcePort (54000 + $offset) -DestinationPort 3389)
|
||||
}
|
||||
$trafficPayload = ConvertTo-VPNPacketBatch -Packets $trafficPackets
|
||||
$trafficPayloadPath = Join-Path ([System.IO.Path]::GetTempPath()) "$runId-vpn-packet-batch.bin"
|
||||
[System.IO.File]::WriteAllBytes($trafficPayloadPath, [byte[]]$trafficPayload)
|
||||
$trafficStarted = Get-Date
|
||||
$trafficResponse = Invoke-WebRequest -Method Post -Uri ($packetUrl + "?batch=true") -Headers $headers -InFile $trafficPayloadPath -ContentType "application/vnd.rap.vpn-packet-batch.v1" -TimeoutSec 30
|
||||
$trafficDurationMs = [int]((Get-Date) - $trafficStarted).TotalMilliseconds
|
||||
$trafficAcceptedBy = [string]$trafficResponse.Headers["X-RAP-Service-Channel-Accepted-By"]
|
||||
Remove-Item -Path $trafficPayloadPath -Force -ErrorAction SilentlyContinue
|
||||
|
||||
$replacementTrafficObserved = $false
|
||||
$replacementLastSelected = ""
|
||||
$replacementFlowStats = @()
|
||||
$postRemediationFallbackLocal = 0
|
||||
$postRemediationRouteFailures = 0
|
||||
$postRemediationFlowDropped = 0
|
||||
$postRemediationSchedulerDropped = 0
|
||||
for ($i = 0; $i -lt 12; $i++) {
|
||||
Start-Sleep -Seconds 5
|
||||
$heartbeats = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/nodes/$($entryNode.id)/heartbeats?actor_user_id=$ActorUserID&limit=5").heartbeats
|
||||
foreach ($heartbeat in @($heartbeats)) {
|
||||
$runtimeReport = Get-PropertyValue -Item (Get-PropertyValue -Item $heartbeat -Name "metadata" -Default $null) -Name "fabric_service_channel_runtime_report" -Default $null
|
||||
$ingress = Get-PropertyValue -Item $runtimeReport -Name "ingress" -Default $null
|
||||
$replacementLastSelected = [string](Get-PropertyValue -Item $ingress -Name "last_selected_route_id" -Default "")
|
||||
$postRemediationFallbackLocal = [int](Get-PropertyValue -Item $ingress -Name "send_fallback_local" -Default 0)
|
||||
$postRemediationRouteFailures = [int](Get-PropertyValue -Item $ingress -Name "send_route_failures" -Default 0)
|
||||
$postRemediationFlowDropped = [int](Get-PropertyValue -Item $ingress -Name "send_flow_dropped" -Default 0)
|
||||
$flowScheduler = Get-PropertyValue -Item $ingress -Name "flow_scheduler" -Default $null
|
||||
$postRemediationSchedulerDropped = [int](Get-PropertyValue -Item $flowScheduler -Name "dropped" -Default 0)
|
||||
$channelStats = Get-PropertyValue -Item $flowScheduler -Name "channel_stats" -Default $null
|
||||
if ($null -ne $channelStats) {
|
||||
$replacementFlowStats = @()
|
||||
foreach ($property in @($channelStats.PSObject.Properties)) {
|
||||
$stat = $property.Value
|
||||
if ([string](Get-PropertyValue -Item $stat -Name "last_route_id" -Default "") -eq $alternateRouteID) {
|
||||
$replacementFlowStats += $stat
|
||||
}
|
||||
}
|
||||
}
|
||||
if ($replacementLastSelected -eq $alternateRouteID -and $replacementFlowStats.Count -ge 1) {
|
||||
$replacementTrafficObserved = $true
|
||||
break
|
||||
}
|
||||
}
|
||||
if ($replacementTrafficObserved) {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
$postAccess = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/fabric/service-channels/access-telemetry?actor_user_id=$ActorUserID&limit=50").fabric_service_channel_access_telemetry
|
||||
$postChannel = @($postAccess.active_channels | Where-Object { $_.channel_id -eq $lease.channel_id }) | Select-Object -First 1
|
||||
|
||||
[void](Send-ReplacementDegradedHeartbeat -EntryNodeID $entryNode.id -ReplacementRouteID $alternateRouteID -RouteGeneration "$runId-primary")
|
||||
$noSafeSyntheticConfig = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/nodes/$($entryNode.id)/mesh/synthetic-config").synthetic_mesh_config
|
||||
$noSafePathDecisions = @()
|
||||
if ($noSafeSyntheticConfig.PSObject.Properties.Name -contains "route_path_decisions") {
|
||||
$noSafePathDecisions = @($noSafeSyntheticConfig.route_path_decisions.decisions)
|
||||
}
|
||||
$noSafeDecision = $noSafePathDecisions | Where-Object {
|
||||
[string](Get-PropertyValue -Item $_ -Name "route_id" -Default "") -eq $alternateRouteID -and
|
||||
[string](Get-PropertyValue -Item $_ -Name "decision_source" -Default "") -eq "service_channel_feedback_no_alternate"
|
||||
} | Select-Object -First 1
|
||||
$noSafeFeedback = $null
|
||||
$serviceChannelFeedback = Get-PropertyValue -Item $noSafeSyntheticConfig -Name "service_channel_feedback" -Default $null
|
||||
if ($null -ne $serviceChannelFeedback -and $serviceChannelFeedback.PSObject.Properties.Name -contains "routes") {
|
||||
$noSafeFeedback = @($serviceChannelFeedback.routes | Where-Object { [string](Get-PropertyValue -Item $_ -Name "route_id" -Default "") -eq $alternateRouteID }) | Select-Object -First 1
|
||||
}
|
||||
|
||||
$postNoSafeAccess = $null
|
||||
$postNoSafeChannel = $null
|
||||
for ($i = 0; $i -lt 12; $i++) {
|
||||
Start-Sleep -Seconds 3
|
||||
$postNoSafeAccess = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/fabric/service-channels/access-telemetry?actor_user_id=$ActorUserID&limit=50").fabric_service_channel_access_telemetry
|
||||
$postNoSafeChannel = @($postNoSafeAccess.active_channels | Where-Object { $_.channel_id -eq $lease.channel_id }) | Select-Object -First 1
|
||||
if ($null -ne $postNoSafeChannel -and [string](Get-PropertyValue -Item $postNoSafeChannel -Name "route_decision_source" -Default "") -eq "service_channel_feedback_no_alternate") {
|
||||
break
|
||||
}
|
||||
}
|
||||
$postNoSafeHealth = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/fabric/service-channels/rebuild-health?actor_user_id=$ActorUserID&limit=50").rebuild_health
|
||||
$postNoSafeIncidents = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/fabric/service-channels/rebuild-incidents?actor_user_id=$ActorUserID&limit=10").rebuild_incidents
|
||||
$postNoSafeIncident = @($postNoSafeIncidents | Where-Object {
|
||||
[string](Get-PropertyValue -Item $_ -Name "incident_source" -Default "") -eq "access_decision" -and
|
||||
[string](Get-PropertyValue -Item $_ -Name "guard_status" -Default "") -eq "access_no_safe_recovery" -and
|
||||
[string](Get-PropertyValue -Item $_ -Name "channel_id" -Default "") -eq [string]$lease.channel_id
|
||||
}) | Select-Object -First 1
|
||||
$silence = $null
|
||||
$listedSilence = $null
|
||||
$postSilenceHealth = $null
|
||||
$postSilenceIncident = $null
|
||||
$postResurfaceHealth = $null
|
||||
$postResurfaceIncident = $null
|
||||
if ($null -ne $postNoSafeIncident) {
|
||||
$silence = Invoke-Api -Method POST -Path "/clusters/$ClusterID/fabric/service-channels/rebuild-health/silences" -Body @{
|
||||
actor_user_id = $ActorUserID
|
||||
incident_source = "access_decision"
|
||||
channel_id = [string]$lease.channel_id
|
||||
reporter_node_id = [string](Get-PropertyValue -Item $postNoSafeIncident -Name "reporter_node_id" -Default "")
|
||||
route_id = [string](Get-PropertyValue -Item $postNoSafeIncident -Name "route_id" -Default "")
|
||||
guard_status = "access_no_safe_recovery"
|
||||
generation = [string](Get-PropertyValue -Item $postNoSafeIncident -Name "generation" -Default "")
|
||||
reason = "c18z88 operator acknowledged access no-safe before generation change"
|
||||
ttl_seconds = 21600
|
||||
}
|
||||
Start-Sleep -Seconds 2
|
||||
$activeSilences = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/fabric/service-channels/rebuild-health/silences?actor_user_id=$ActorUserID").rebuild_alert_silences
|
||||
$silenceID = [string](Get-PropertyValue -Item (Get-PropertyValue -Item $silence -Name "rebuild_alert_silence" -Default $null) -Name "id" -Default "")
|
||||
$listedSilence = @($activeSilences | Where-Object { [string](Get-PropertyValue -Item $_ -Name "id" -Default "") -eq $silenceID }) | Select-Object -First 1
|
||||
$postSilenceHealth = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/fabric/service-channels/rebuild-health?actor_user_id=$ActorUserID&limit=50").rebuild_health
|
||||
$postSilenceIncidents = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/fabric/service-channels/rebuild-incidents?actor_user_id=$ActorUserID&limit=10").rebuild_incidents
|
||||
$postSilenceIncident = @($postSilenceIncidents | Where-Object {
|
||||
[string](Get-PropertyValue -Item $_ -Name "incident_source" -Default "") -eq "access_decision" -and
|
||||
[string](Get-PropertyValue -Item $_ -Name "guard_status" -Default "") -eq "access_no_safe_recovery" -and
|
||||
[string](Get-PropertyValue -Item $_ -Name "channel_id" -Default "") -eq [string]$lease.channel_id
|
||||
}) | Select-Object -First 1
|
||||
|
||||
for ($i = 0; $i -lt 18; $i++) {
|
||||
Start-Sleep -Seconds 5
|
||||
$postResurfaceHealth = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/fabric/service-channels/rebuild-health?actor_user_id=$ActorUserID&limit=50").rebuild_health
|
||||
$postResurfaceIncidents = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/fabric/service-channels/rebuild-incidents?actor_user_id=$ActorUserID&limit=10").rebuild_incidents
|
||||
$postResurfaceIncident = @($postResurfaceIncidents | Where-Object {
|
||||
[string](Get-PropertyValue -Item $_ -Name "incident_source" -Default "") -eq "access_decision" -and
|
||||
[string](Get-PropertyValue -Item $_ -Name "guard_status" -Default "") -eq "access_no_safe_recovery" -and
|
||||
[string](Get-PropertyValue -Item $_ -Name "channel_id" -Default "") -eq [string]$lease.channel_id -and
|
||||
[bool](Get-PropertyValue -Item $_ -Name "alert_resurfaced" -Default $false)
|
||||
}) | Select-Object -First 1
|
||||
if ($null -ne $postResurfaceIncident) { break }
|
||||
}
|
||||
}
|
||||
$backendLine = (& ssh $DockerSSH "docker ps --format '{{.Names}} {{.Image}} {{.Status}}' | grep '^rap_test_backend '") | Out-String
|
||||
$nodeLines = (& ssh $DockerSSH "docker ps --format '{{.Names}} {{.Image}} {{.Status}}' | grep '^rap_test_node_test_'") | Out-String
|
||||
$checks = [ordered]@{
|
||||
backend_expected_image_deployed = $backendLine.Contains($ExpectedBackendImage)
|
||||
node_agent_expected_image_deployed = $nodeLines.Contains($ExpectedNodeAgentImage)
|
||||
lease_ready = ([string]$lease.status -eq "ready")
|
||||
remediation_rebuild_route_visible = ($null -ne $matchingChannel -and [string](Get-PropertyValue -Item $matchingChannel -Name "remediation_action" -Default "") -eq "rebuild_route")
|
||||
remediation_command_visible = ($null -ne $command -and $commandID.Length -gt 0)
|
||||
durable_rebuild_intent_recorded = ($null -ne $attempt)
|
||||
durable_rebuild_intent_resolved_applied = ($null -ne $attempt -and [string](Get-PropertyValue -Item $attempt -Name "rebuild_status" -Default "") -eq "applied")
|
||||
durable_rebuild_intent_outcome_replacement_selected = ($null -ne $attempt -and [string](Get-PropertyValue -Item $attempt -Name "outcome" -Default "") -eq "replacement_selected")
|
||||
durable_rebuild_intent_replacement_matches = ($null -ne $attempt -and [string](Get-PropertyValue -Item $attempt -Name "replacement_route_id" -Default "") -eq $alternateRouteID)
|
||||
durable_rebuild_intent_source_matches = ($null -ne $attempt -and [string](Get-PropertyValue -Item $attempt -Name "decision_source" -Default "") -eq "service_channel_remediation_command")
|
||||
initial_access_telemetry_reports_planner_applied = ($null -ne $postChannel -and [string](Get-PropertyValue -Item $postChannel -Name "remediation_execution_status" -Default "") -eq "rebuild_request_applied")
|
||||
node_route_manager_consumed_same_command = ($null -ne $routeManagerDecision)
|
||||
node_route_manager_applied_rebuild = ($null -ne $routeManagerDecision -and [string](Get-PropertyValue -Item $routeManagerDecision -Name "rebuild_status" -Default "") -eq "applied")
|
||||
node_route_manager_transition_applied_rebuild = ($null -ne $routeManagerTransition -and [string](Get-PropertyValue -Item $routeManagerTransition -Name "status" -Default "") -eq "applied_rebuild")
|
||||
traffic_packet_accepted = ([int]$trafficResponse.StatusCode -eq 202)
|
||||
traffic_accepted_by_runtime = ($trafficAcceptedBy -eq "introspection" -or $trafficAcceptedBy -eq "signed")
|
||||
traffic_selected_replacement_route = $replacementTrafficObserved
|
||||
traffic_last_selected_route_matches_replacement = ($replacementLastSelected -eq $alternateRouteID)
|
||||
no_backend_or_local_fallback_after_replacement = (($postRemediationFallbackLocal - $baselineFallbackLocal) -eq 0)
|
||||
no_route_failures_after_replacement = (($postRemediationRouteFailures - $baselineRouteFailures) -eq 0)
|
||||
no_flow_drops_after_replacement = (($postRemediationFlowDropped - $baselineFlowDropped) -eq 0 -and ($postRemediationSchedulerDropped - $baselineSchedulerDropped) -eq 0)
|
||||
replacement_degradation_feedback_visible = ($null -ne $noSafeDecision -and @($noSafeDecision.score_reasons | Where-Object { [string]$_ -eq "service_channel_fenced_route" }).Count -gt 0)
|
||||
no_safe_recovery_decision_visible = ($null -ne $noSafeDecision)
|
||||
no_safe_recovery_status_pending_fallback = ($null -ne $noSafeDecision -and [string](Get-PropertyValue -Item $noSafeDecision -Name "rebuild_status" -Default "") -eq "pending_degraded_fallback")
|
||||
no_safe_recovery_reason_visible = ($null -ne $noSafeDecision -and @($noSafeDecision.score_reasons | Where-Object { [string]$_ -eq "no_unfenced_alternate_route" }).Count -gt 0)
|
||||
no_silent_stickiness_to_bad_replacement = ($null -ne $noSafeDecision -and [string](Get-PropertyValue -Item $noSafeDecision -Name "decision_source" -Default "") -eq "service_channel_feedback_no_alternate")
|
||||
access_telemetry_projects_no_safe_decision = ($null -ne $postNoSafeChannel -and [string](Get-PropertyValue -Item $postNoSafeChannel -Name "route_decision_source" -Default "") -eq "service_channel_feedback_no_alternate")
|
||||
access_telemetry_projects_pending_fallback = ($null -ne $postNoSafeChannel -and [string](Get-PropertyValue -Item $postNoSafeChannel -Name "route_decision_rebuild_status" -Default "") -eq "pending_degraded_fallback")
|
||||
access_telemetry_projects_no_safe_route_id = ($null -ne $postNoSafeChannel -and @($primaryRouteID, $alternateRouteID) -contains [string](Get-PropertyValue -Item $postNoSafeChannel -Name "route_decision_route_id" -Default ""))
|
||||
access_telemetry_projects_operator_state = ($null -ne $postNoSafeChannel -and @("route_rebuild_no_safe_recovery", "rebuild_request_no_alternate") -contains [string](Get-PropertyValue -Item $postNoSafeChannel -Name "remediation_execution_status" -Default ""))
|
||||
access_telemetry_aggregates_route_decision = ($null -ne $postNoSafeAccess -and [int](Get-PropertyValue -Item $postNoSafeAccess -Name "route_decision_channel_count" -Default 0) -ge 1)
|
||||
access_telemetry_aggregates_no_safe_decision = ($null -ne $postNoSafeAccess -and [int](Get-PropertyValue -Item $postNoSafeAccess -Name "no_safe_recovery_decision_count" -Default 0) -ge 1)
|
||||
access_telemetry_aggregate_status_prioritizes_no_safe = ($null -ne $postNoSafeAccess -and [string](Get-PropertyValue -Item $postNoSafeAccess -Name "reason" -Default "") -eq "active_channels_no_safe_recovery")
|
||||
rebuild_health_projects_access_no_safe = ($null -ne $postNoSafeHealth -and [int](Get-PropertyValue -Item $postNoSafeHealth -Name "access_no_safe_count" -Default 0) -ge 1)
|
||||
rebuild_health_action_prioritizes_access_no_safe = ($null -ne $postNoSafeHealth -and [string](Get-PropertyValue -Item $postNoSafeHealth -Name "recommended_operator_action" -Default "") -eq "inspect_access_no_safe_recovery_route_pool_and_signed_policy")
|
||||
rebuild_incident_projects_access_no_safe = ($null -ne $postNoSafeIncident)
|
||||
rebuild_incident_access_no_safe_is_bad = ($null -ne $postNoSafeIncident -and [string](Get-PropertyValue -Item $postNoSafeIncident -Name "guard_severity" -Default "") -eq "bad")
|
||||
access_decision_silence_recorded = ($null -ne $silence -and $null -ne (Get-PropertyValue -Item $silence -Name "rebuild_alert_silence" -Default $null))
|
||||
access_decision_silence_listed = ($null -ne $listedSilence -and [string](Get-PropertyValue -Item $listedSilence -Name "incident_source" -Default "") -eq "access_decision" -and [string](Get-PropertyValue -Item $listedSilence -Name "channel_id" -Default "") -eq [string]$lease.channel_id)
|
||||
access_decision_incident_silenced = ($null -ne $postSilenceIncident -and [bool](Get-PropertyValue -Item $postSilenceIncident -Name "alert_silenced" -Default $false))
|
||||
access_decision_silence_reduces_active_bad = ($null -ne $postSilenceHealth -and [int](Get-PropertyValue -Item $postSilenceHealth -Name "silenced_count" -Default 0) -ge 1)
|
||||
access_decision_resurfaced_on_generation_change = ($null -ne $postResurfaceIncident -and [bool](Get-PropertyValue -Item $postResurfaceIncident -Name "alert_resurfaced" -Default $false))
|
||||
access_decision_resurface_restores_active_bad = ($null -ne $postResurfaceHealth -and [int](Get-PropertyValue -Item $postResurfaceHealth -Name "active_bad_count" -Default 0) -ge 1 -and [int](Get-PropertyValue -Item $postResurfaceHealth -Name "resurfaced_count" -Default 0) -ge 1)
|
||||
access_decision_resurface_keeps_previous_generation = ($null -ne $postResurfaceIncident -and [string](Get-PropertyValue -Item $postResurfaceIncident -Name "alert_resurfaced_previous_generation" -Default "") -eq [string](Get-PropertyValue -Item $postNoSafeIncident -Name "generation" -Default ""))
|
||||
}
|
||||
$failed = @($checks.GetEnumerator() | Where-Object { -not $_.Value } | ForEach-Object { $_.Key })
|
||||
|
||||
$result = [ordered]@{
|
||||
schema_version = "c18z88.service_channel_access_decision_resurface_smoke.v1"
|
||||
run_id = $runId
|
||||
cluster_id = $ClusterID
|
||||
channel_id = [string]$lease.channel_id
|
||||
primary_route_id = $primaryRouteID
|
||||
alternate_route_id = $alternateRouteID
|
||||
rebuild_request_id = $commandID
|
||||
passed = ($failed.Count -eq 0)
|
||||
checks = $checks
|
||||
failed_checks = $failed
|
||||
summary = [ordered]@{
|
||||
backend_container = $backendLine.Trim()
|
||||
node_containers = $nodeLines.Trim()
|
||||
remediation_command = $command
|
||||
rebuild_attempt = $attempt
|
||||
route_manager_decision = $routeManagerDecision
|
||||
route_manager_transition = $routeManagerTransition
|
||||
traffic_status_code = [int]$trafficResponse.StatusCode
|
||||
traffic_accepted_by = $trafficAcceptedBy
|
||||
traffic_duration_ms = $trafficDurationMs
|
||||
traffic_packet_count = $trafficPackets.Count
|
||||
replacement_last_selected_route_id = $replacementLastSelected
|
||||
replacement_flow_stat_count = $replacementFlowStats.Count
|
||||
replacement_flow_stats = $replacementFlowStats
|
||||
baseline_send_fallback_local = $baselineFallbackLocal
|
||||
baseline_send_route_failures = $baselineRouteFailures
|
||||
baseline_send_flow_dropped = $baselineFlowDropped
|
||||
baseline_scheduler_dropped = $baselineSchedulerDropped
|
||||
fallback_local_delta = ($postRemediationFallbackLocal - $baselineFallbackLocal)
|
||||
route_failure_delta = ($postRemediationRouteFailures - $baselineRouteFailures)
|
||||
flow_drop_delta = ($postRemediationFlowDropped - $baselineFlowDropped)
|
||||
scheduler_drop_delta = ($postRemediationSchedulerDropped - $baselineSchedulerDropped)
|
||||
post_remediation_send_fallback_local = $postRemediationFallbackLocal
|
||||
post_remediation_send_route_failures = $postRemediationRouteFailures
|
||||
post_remediation_send_flow_dropped = $postRemediationFlowDropped
|
||||
post_remediation_scheduler_dropped = $postRemediationSchedulerDropped
|
||||
post_channel = $postChannel
|
||||
no_safe_feedback = $noSafeFeedback
|
||||
no_safe_decision = $noSafeDecision
|
||||
no_safe_route_path_decisions = $noSafePathDecisions
|
||||
post_no_safe_channel = $postNoSafeChannel
|
||||
post_no_safe_access = $postNoSafeAccess
|
||||
post_no_safe_health = $postNoSafeHealth
|
||||
post_no_safe_incidents = $postNoSafeIncidents
|
||||
post_no_safe_incident = $postNoSafeIncident
|
||||
silence = $silence
|
||||
listed_silence = $listedSilence
|
||||
post_silence_health = $postSilenceHealth
|
||||
post_silence_incident = $postSilenceIncident
|
||||
post_resurface_health = $postResurfaceHealth
|
||||
post_resurface_incident = $postResurfaceIncident
|
||||
}
|
||||
}
|
||||
|
||||
$target = Join-Path $repoRoot $ResultPath
|
||||
$result | ConvertTo-Json -Depth 80 | Set-Content -Path $target -Encoding UTF8
|
||||
|
||||
try {
|
||||
if ($primaryRouteID) {
|
||||
Invoke-Api -Method POST -Path "/clusters/$ClusterID/mesh/route-intents/$primaryRouteID/expire" -Body @{ actor_user_id = $ActorUserID } | Out-Null
|
||||
}
|
||||
if ($alternateRouteID) {
|
||||
Invoke-Api -Method POST -Path "/clusters/$ClusterID/mesh/route-intents/$alternateRouteID/expire" -Body @{ actor_user_id = $ActorUserID } | Out-Null
|
||||
}
|
||||
Start-Sleep -Seconds 3
|
||||
Invoke-Api -Method POST -Path "/clusters/$ClusterID/fabric/service-channels/leases/cleanup" -Body @{
|
||||
actor_user_id = $ActorUserID
|
||||
limit = 50
|
||||
} | Out-Null
|
||||
} catch {
|
||||
Write-Warning "cleanup failed after c18z88 smoke: $($_.Exception.Message)"
|
||||
}
|
||||
|
||||
if (-not $result.passed) {
|
||||
throw "C18Z88 access decision resurface smoke failed: $($failed -join ', ')"
|
||||
}
|
||||
|
||||
Write-Host "C18Z88 service-channel access decision resurface smoke passed. Result: $target"
|
||||
$result
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,637 @@
|
||||
param(
|
||||
[string]$ApiBaseUrl = "http://192.168.200.61:18121/api/v1",
|
||||
[string]$ClusterID = "cfc0743d-d960-49fb-9de8-96e063d5e4aa",
|
||||
[string]$ActorUserID = "f67d943f-5397-4b3a-a229-695fe67ad700",
|
||||
[string]$EntryNodeName = "test-1",
|
||||
[string]$RelayNodeName = "test-3",
|
||||
[string]$ExitNodeName = "test-2",
|
||||
[string]$EntryBaseUrl = "http://192.168.200.61:19131",
|
||||
[string]$DockerSSH = "test-docker",
|
||||
[string]$ExpectedBackendImage = "rap-backend:fabric-service-channel-0.2.281-c18z109",
|
||||
[string]$ExpectedNodeAgentImage = "rap-node-agent:0.2.270-c18z95",
|
||||
[string]$ResultPath = "artifacts\c18z89-service-channel-access-decision-resurface-action-smoke-result.json"
|
||||
)
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
$scriptDir = Split-Path -Parent $MyInvocation.MyCommand.Path
|
||||
$repoRoot = (Resolve-Path (Join-Path $scriptDir "..\..")).ProviderPath
|
||||
$runId = "c18z89-" + (Get-Date -Format "yyyyMMdd-HHmmss")
|
||||
|
||||
function Invoke-Api {
|
||||
param([string]$Method, [string]$Path, [object]$Body = $null)
|
||||
$params = @{ Method = $Method; Uri = "$ApiBaseUrl$Path"; TimeoutSec = 30 }
|
||||
if ($null -ne $Body) {
|
||||
$params.ContentType = "application/json"
|
||||
$params.Body = ($Body | ConvertTo-Json -Depth 80)
|
||||
}
|
||||
return Invoke-RestMethod @params
|
||||
}
|
||||
|
||||
function Get-PropertyValue {
|
||||
param([object]$Item, [string]$Name, [object]$Default = $null)
|
||||
if ($null -eq $Item) { return $Default }
|
||||
$property = $Item.PSObject.Properties[$Name]
|
||||
if ($null -eq $property) { return $Default }
|
||||
return $property.Value
|
||||
}
|
||||
|
||||
function Get-NodeByName {
|
||||
param([string]$Name)
|
||||
$nodes = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/nodes?actor_user_id=$ActorUserID").nodes
|
||||
$node = @($nodes | Where-Object { $_.name -eq $Name }) | Select-Object -First 1
|
||||
if ($null -eq $node) { throw "Node '$Name' was not found" }
|
||||
return $node
|
||||
}
|
||||
|
||||
function New-IPv4TcpPacket {
|
||||
param(
|
||||
[byte[]]$Source = @(10, 77, 0, 2),
|
||||
[byte[]]$Destination = @(192, 168, 200, 95),
|
||||
[int]$SourcePort,
|
||||
[int]$DestinationPort = 3389
|
||||
)
|
||||
$packet = [byte[]]::new(40)
|
||||
$packet[0] = 0x45
|
||||
$packet[2] = 0
|
||||
$packet[3] = 40
|
||||
$packet[8] = 64
|
||||
$packet[9] = 6
|
||||
[Array]::Copy($Source, 0, $packet, 12, 4)
|
||||
[Array]::Copy($Destination, 0, $packet, 16, 4)
|
||||
$packet[20] = [byte](($SourcePort -shr 8) -band 0xff)
|
||||
$packet[21] = [byte]($SourcePort -band 0xff)
|
||||
$packet[22] = [byte](($DestinationPort -shr 8) -band 0xff)
|
||||
$packet[23] = [byte]($DestinationPort -band 0xff)
|
||||
$packet[32] = 0x50
|
||||
return $packet
|
||||
}
|
||||
|
||||
function ConvertTo-VPNPacketBatch {
|
||||
param([byte[][]]$Packets)
|
||||
$buffer = New-Object System.Collections.Generic.List[byte]
|
||||
foreach ($packet in $Packets) {
|
||||
if ($null -eq $packet -or $packet.Length -eq 0) { continue }
|
||||
$size = [uint32]$packet.Length
|
||||
$buffer.Add([byte](($size -shr 24) -band 0xff))
|
||||
$buffer.Add([byte](($size -shr 16) -band 0xff))
|
||||
$buffer.Add([byte](($size -shr 8) -band 0xff))
|
||||
$buffer.Add([byte]($size -band 0xff))
|
||||
foreach ($value in $packet) {
|
||||
$buffer.Add($value)
|
||||
}
|
||||
}
|
||||
return $buffer.ToArray()
|
||||
}
|
||||
|
||||
function New-RouteIntent {
|
||||
param([string]$SourceNodeID, [string]$DestinationNodeID, [string[]]$Hops)
|
||||
$expiresAt = (Get-Date).ToUniversalTime().AddMinutes(5).ToString("o")
|
||||
return (Invoke-Api -Method POST -Path "/clusters/$ClusterID/mesh/route-intents" -Body @{
|
||||
actor_user_id = $ActorUserID
|
||||
source_selector = @{ node_id = $SourceNodeID }
|
||||
destination_selector = @{ node_id = $DestinationNodeID }
|
||||
service_class = "vpn_packets"
|
||||
priority = 2100000000
|
||||
policy = @{
|
||||
synthetic_enabled = $true
|
||||
route_version = "$runId-primary"
|
||||
policy_version = "$runId-primary"
|
||||
peer_directory_version = "$runId-primary"
|
||||
hops = @($Hops)
|
||||
allowed_channels = @("vpn_packet", "fabric_control")
|
||||
max_ttl = 8
|
||||
max_hops = 8
|
||||
expires_at = $expiresAt
|
||||
metadata = @{ smoke = "c18z82_service_channel_no_safe_recovery"; run_id = $runId }
|
||||
}
|
||||
}).route_intent
|
||||
}
|
||||
|
||||
function Disable-ExistingRouteIntents {
|
||||
param([string]$SourceNodeID, [string]$DestinationNodeID)
|
||||
$items = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/mesh/route-intents?actor_user_id=$ActorUserID").route_intents
|
||||
foreach ($item in @($items)) {
|
||||
if ([string](Get-PropertyValue -Item $item -Name "status" -Default "") -ne "active") { continue }
|
||||
if ([string](Get-PropertyValue -Item $item -Name "service_class" -Default "") -ne "vpn_packets") { continue }
|
||||
$sourceSelector = Get-PropertyValue -Item $item -Name "source_selector" -Default $null
|
||||
$destinationSelector = Get-PropertyValue -Item $item -Name "destination_selector" -Default $null
|
||||
if ([string](Get-PropertyValue -Item $sourceSelector -Name "node_id" -Default "") -ne $SourceNodeID) { continue }
|
||||
if ([string](Get-PropertyValue -Item $destinationSelector -Name "node_id" -Default "") -ne $DestinationNodeID) { continue }
|
||||
[void](Invoke-Api -Method POST -Path "/clusters/$ClusterID/mesh/route-intents/$($item.id)/disable" -Body @{
|
||||
actor_user_id = $ActorUserID
|
||||
reason = "c18z82 isolate no-safe-recovery smoke route pair"
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
function Send-DegradedHeartbeat {
|
||||
param([string]$EntryNodeID, [string]$PrimaryRouteID)
|
||||
$observedAt = (Get-Date).ToUniversalTime().ToString("o")
|
||||
return Invoke-Api -Method POST -Path "/clusters/$ClusterID/nodes/$EntryNodeID/heartbeats" -Body @{
|
||||
health_status = "healthy"
|
||||
reported_version = "0.2.256-c18z82"
|
||||
capabilities = @{
|
||||
fabric_service_channel_runtime = $true
|
||||
fabric_service_channel_route_quality_feedback = $true
|
||||
smoke_feedback_injection = "c18z77"
|
||||
}
|
||||
service_states = @{ smoke = "c18z82_primary_degraded_alternate_after_lease" }
|
||||
metadata = @{
|
||||
fabric_service_channel_runtime_report = @{
|
||||
schema_version = "c18l.fabric_service_channel_runtime_report.v1"
|
||||
config_version = "$runId-primary"
|
||||
cluster_id = $ClusterID
|
||||
local_node_id = $EntryNodeID
|
||||
observed_at = $observedAt
|
||||
ingress = @{
|
||||
flow_scheduler = @{
|
||||
schema_version = "rap.fabric_flow_scheduler.v1"
|
||||
service_neutral = $true
|
||||
service_mode = "application_protocol_agnostic"
|
||||
channel_stats = @{
|
||||
"c18z82-primary-degraded" = @{
|
||||
last_route_id = $PrimaryRouteID
|
||||
last_failed_route_id = $PrimaryRouteID
|
||||
route_generation = "$runId-primary"
|
||||
last_error = "c18z82 primary route degraded; alternate added after lease"
|
||||
last_send_duration_ms = 1200
|
||||
consecutive_failures = 3
|
||||
stall_count = 2
|
||||
route_rebuild_recommended = $true
|
||||
degraded_fallback_recommended = $false
|
||||
quality_window_sample_count = 8
|
||||
quality_window_success_count = 2
|
||||
quality_window_failure_count = 3
|
||||
quality_window_slow_count = 2
|
||||
quality_window_drop_count = 0
|
||||
quality_window_avg_latency_ms = 1200
|
||||
quality_window_last_updated_at = $observedAt
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
smoke = @{ name = "c18z82_service_channel_no_safe_recovery"; run_id = $runId }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function Send-ReplacementDegradedHeartbeat {
|
||||
param([string]$EntryNodeID, [string]$ReplacementRouteID, [string]$RouteGeneration)
|
||||
$observedAt = (Get-Date).ToUniversalTime().ToString("o")
|
||||
return Invoke-Api -Method POST -Path "/clusters/$ClusterID/nodes/$EntryNodeID/heartbeats" -Body @{
|
||||
health_status = "healthy"
|
||||
reported_version = "0.2.256-c18z82"
|
||||
capabilities = @{
|
||||
fabric_service_channel_runtime = $true
|
||||
fabric_service_channel_route_quality_feedback = $true
|
||||
smoke_feedback_injection = "c18z82"
|
||||
}
|
||||
service_states = @{ smoke = "c18z82_replacement_route_degraded_no_safe_recovery" }
|
||||
metadata = @{
|
||||
fabric_service_channel_runtime_report = @{
|
||||
schema_version = "c18l.fabric_service_channel_runtime_report.v1"
|
||||
config_version = "$runId-replacement-degraded"
|
||||
cluster_id = $ClusterID
|
||||
local_node_id = $EntryNodeID
|
||||
observed_at = $observedAt
|
||||
ingress = @{
|
||||
last_selected_route_id = $ReplacementRouteID
|
||||
send_route_failures = 4
|
||||
flow_scheduler = @{
|
||||
schema_version = "rap.fabric_flow_scheduler.v1"
|
||||
service_neutral = $true
|
||||
service_mode = "application_protocol_agnostic"
|
||||
channel_stats = @{
|
||||
"c18z82-replacement-degraded" = @{
|
||||
last_route_id = $ReplacementRouteID
|
||||
last_failed_route_id = $ReplacementRouteID
|
||||
route_generation = $RouteGeneration
|
||||
last_error = "c18z82 replacement route degraded; no safe recovery route exists"
|
||||
last_send_duration_ms = 1500
|
||||
consecutive_failures = 4
|
||||
stall_count = 2
|
||||
route_rebuild_recommended = $true
|
||||
degraded_fallback_recommended = $true
|
||||
quality_window_sample_count = 10
|
||||
quality_window_success_count = 1
|
||||
quality_window_failure_count = 4
|
||||
quality_window_slow_count = 3
|
||||
quality_window_drop_count = 0
|
||||
quality_window_avg_latency_ms = 1500
|
||||
quality_window_last_updated_at = $observedAt
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
smoke = @{ name = "c18z82_service_channel_no_safe_recovery"; run_id = $runId }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
$entryNode = Get-NodeByName -Name $EntryNodeName
|
||||
$relayNode = Get-NodeByName -Name $RelayNodeName
|
||||
$exitNode = Get-NodeByName -Name $ExitNodeName
|
||||
[void](Disable-ExistingRouteIntents -SourceNodeID $entryNode.id -DestinationNodeID $exitNode.id)
|
||||
$route = New-RouteIntent -SourceNodeID $entryNode.id -DestinationNodeID $exitNode.id -Hops @($entryNode.id, $exitNode.id)
|
||||
$lease = (Invoke-Api -Method POST -Path "/clusters/$ClusterID/fabric/service-channels/leases" -Body @{
|
||||
actor_user_id = $ActorUserID
|
||||
organization_id = "smoke-org"
|
||||
user_id = "smoke-user"
|
||||
resource_id = "c18z82-vpn-smoke"
|
||||
service_class = "vpn_packets"
|
||||
entry_node_ids = @([string]$entryNode.id)
|
||||
exit_node_ids = @([string]$exitNode.id)
|
||||
preferred_entry_node_id = [string]$entryNode.id
|
||||
preferred_exit_node_id = [string]$exitNode.id
|
||||
allowed_channels = @("vpn_packet", "fabric_control")
|
||||
ttl_seconds = 180
|
||||
metadata = @{ smoke = "c18z82_service_channel_no_safe_recovery"; run_id = $runId }
|
||||
}).fabric_service_channel_lease
|
||||
$primaryRouteID = [string](Get-PropertyValue -Item (Get-PropertyValue -Item $lease -Name "primary_route" -Default $null) -Name "route_id" -Default $route.id)
|
||||
|
||||
[void](Send-DegradedHeartbeat -EntryNodeID $entryNode.id -PrimaryRouteID $primaryRouteID)
|
||||
|
||||
$matchingChannel = $null
|
||||
for ($i = 0; $i -lt 10; $i++) {
|
||||
Start-Sleep -Seconds 3
|
||||
$accessTelemetry = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/fabric/service-channels/access-telemetry?actor_user_id=$ActorUserID&limit=50").fabric_service_channel_access_telemetry
|
||||
$matchingChannel = @($accessTelemetry.active_channels | Where-Object { $_.channel_id -eq $lease.channel_id }) | Select-Object -First 1
|
||||
if ($null -ne $matchingChannel -and [string](Get-PropertyValue -Item $matchingChannel -Name "remediation_action" -Default "") -eq "rebuild_route") {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
$command = Get-PropertyValue -Item $matchingChannel -Name "remediation_command" -Default $null
|
||||
$commandID = [string](Get-PropertyValue -Item $command -Name "command_id" -Default "")
|
||||
$alternate = New-RouteIntent -SourceNodeID $entryNode.id -DestinationNodeID $exitNode.id -Hops @($entryNode.id, $relayNode.id, $exitNode.id)
|
||||
$alternateRouteID = [string]$alternate.id
|
||||
[void](Invoke-Api -Method GET -Path "/clusters/$ClusterID/nodes/$($entryNode.id)/mesh/synthetic-config")
|
||||
|
||||
$attempts = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/fabric/service-channels/rebuild-attempts?actor_user_id=$ActorUserID&reporter_node_id=$($entryNode.id)&rebuild_request_id=$commandID&limit=10").rebuild_attempts
|
||||
$attempt = @($attempts | Where-Object { $_.rebuild_request_id -eq $commandID }) | Select-Object -First 1
|
||||
|
||||
$routeManagerDecision = $null
|
||||
$routeManagerTransition = $null
|
||||
for ($i = 0; $i -lt 12; $i++) {
|
||||
Start-Sleep -Seconds 5
|
||||
$heartbeats = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/nodes/$($entryNode.id)/heartbeats?actor_user_id=$ActorUserID&limit=5").heartbeats
|
||||
foreach ($heartbeat in @($heartbeats)) {
|
||||
$runtimeReport = Get-PropertyValue -Item (Get-PropertyValue -Item $heartbeat -Name "metadata" -Default $null) -Name "fabric_service_channel_runtime_report" -Default $null
|
||||
$ingress = Get-PropertyValue -Item $runtimeReport -Name "ingress" -Default $null
|
||||
$routeManager = Get-PropertyValue -Item $ingress -Name "route_manager" -Default $null
|
||||
$routeManagerTransition = Get-PropertyValue -Item $ingress -Name "route_manager_transition" -Default $null
|
||||
$decisions = @()
|
||||
if ($null -ne $routeManager -and $routeManager.PSObject.Properties.Name -contains "decisions") {
|
||||
$decisions = @($routeManager.decisions)
|
||||
}
|
||||
$routeManagerDecision = $decisions | Where-Object {
|
||||
[string](Get-PropertyValue -Item $_ -Name "rebuild_request_id" -Default "") -eq $commandID -and
|
||||
[string](Get-PropertyValue -Item $_ -Name "route_id" -Default "") -eq $primaryRouteID -and
|
||||
[string](Get-PropertyValue -Item $_ -Name "replacement_route_id" -Default "") -eq $alternateRouteID
|
||||
} | Select-Object -First 1
|
||||
if ($null -ne $routeManagerDecision) {
|
||||
break
|
||||
}
|
||||
}
|
||||
if ($null -ne $routeManagerDecision) {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
$packetPath = $lease.entry_http.path_template.
|
||||
Replace("{cluster_id}", $ClusterID).
|
||||
Replace("{channel_id}", [string]$lease.channel_id).
|
||||
Replace("{resource_id}", "c18z82-vpn-smoke")
|
||||
$packetUrl = $EntryBaseUrl.TrimEnd("/") + $packetPath
|
||||
$headers = @{
|
||||
"X-RAP-Service-Channel-Token" = [string]$lease.token.token
|
||||
"X-RAP-Fabric-Channel-ID" = [string]$lease.channel_id
|
||||
"X-RAP-Service-Class" = "vpn_packets"
|
||||
"X-RAP-Channel-Class" = "vpn_packet"
|
||||
"X-RAP-Traffic-Class" = "interactive"
|
||||
}
|
||||
$baselineFallbackLocal = 0
|
||||
$baselineRouteFailures = 0
|
||||
$baselineFlowDropped = 0
|
||||
$baselineSchedulerDropped = 0
|
||||
$baselineHeartbeats = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/nodes/$($entryNode.id)/heartbeats?actor_user_id=$ActorUserID&limit=1").heartbeats
|
||||
if (@($baselineHeartbeats).Count -gt 0) {
|
||||
$baselineRuntimeReport = Get-PropertyValue -Item (Get-PropertyValue -Item $baselineHeartbeats[0] -Name "metadata" -Default $null) -Name "fabric_service_channel_runtime_report" -Default $null
|
||||
$baselineIngress = Get-PropertyValue -Item $baselineRuntimeReport -Name "ingress" -Default $null
|
||||
$baselineFlowScheduler = Get-PropertyValue -Item $baselineIngress -Name "flow_scheduler" -Default $null
|
||||
$baselineFallbackLocal = [int](Get-PropertyValue -Item $baselineIngress -Name "send_fallback_local" -Default 0)
|
||||
$baselineRouteFailures = [int](Get-PropertyValue -Item $baselineIngress -Name "send_route_failures" -Default 0)
|
||||
$baselineFlowDropped = [int](Get-PropertyValue -Item $baselineIngress -Name "send_flow_dropped" -Default 0)
|
||||
$baselineSchedulerDropped = [int](Get-PropertyValue -Item $baselineFlowScheduler -Name "dropped" -Default 0)
|
||||
}
|
||||
$trafficPackets = @()
|
||||
foreach ($offset in 0..15) {
|
||||
$trafficPackets += ,(New-IPv4TcpPacket -SourcePort (54000 + $offset) -DestinationPort 3389)
|
||||
}
|
||||
$trafficPayload = ConvertTo-VPNPacketBatch -Packets $trafficPackets
|
||||
$trafficPayloadPath = Join-Path ([System.IO.Path]::GetTempPath()) "$runId-vpn-packet-batch.bin"
|
||||
[System.IO.File]::WriteAllBytes($trafficPayloadPath, [byte[]]$trafficPayload)
|
||||
$trafficStarted = Get-Date
|
||||
$trafficResponse = Invoke-WebRequest -Method Post -Uri ($packetUrl + "?batch=true") -Headers $headers -InFile $trafficPayloadPath -ContentType "application/vnd.rap.vpn-packet-batch.v1" -TimeoutSec 30
|
||||
$trafficDurationMs = [int]((Get-Date) - $trafficStarted).TotalMilliseconds
|
||||
$trafficAcceptedBy = [string]$trafficResponse.Headers["X-RAP-Service-Channel-Accepted-By"]
|
||||
Remove-Item -Path $trafficPayloadPath -Force -ErrorAction SilentlyContinue
|
||||
|
||||
$replacementTrafficObserved = $false
|
||||
$replacementLastSelected = ""
|
||||
$replacementFlowStats = @()
|
||||
$postRemediationFallbackLocal = 0
|
||||
$postRemediationRouteFailures = 0
|
||||
$postRemediationFlowDropped = 0
|
||||
$postRemediationSchedulerDropped = 0
|
||||
for ($i = 0; $i -lt 12; $i++) {
|
||||
Start-Sleep -Seconds 5
|
||||
$heartbeats = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/nodes/$($entryNode.id)/heartbeats?actor_user_id=$ActorUserID&limit=5").heartbeats
|
||||
foreach ($heartbeat in @($heartbeats)) {
|
||||
$runtimeReport = Get-PropertyValue -Item (Get-PropertyValue -Item $heartbeat -Name "metadata" -Default $null) -Name "fabric_service_channel_runtime_report" -Default $null
|
||||
$ingress = Get-PropertyValue -Item $runtimeReport -Name "ingress" -Default $null
|
||||
$replacementLastSelected = [string](Get-PropertyValue -Item $ingress -Name "last_selected_route_id" -Default "")
|
||||
$postRemediationFallbackLocal = [int](Get-PropertyValue -Item $ingress -Name "send_fallback_local" -Default 0)
|
||||
$postRemediationRouteFailures = [int](Get-PropertyValue -Item $ingress -Name "send_route_failures" -Default 0)
|
||||
$postRemediationFlowDropped = [int](Get-PropertyValue -Item $ingress -Name "send_flow_dropped" -Default 0)
|
||||
$flowScheduler = Get-PropertyValue -Item $ingress -Name "flow_scheduler" -Default $null
|
||||
$postRemediationSchedulerDropped = [int](Get-PropertyValue -Item $flowScheduler -Name "dropped" -Default 0)
|
||||
$channelStats = Get-PropertyValue -Item $flowScheduler -Name "channel_stats" -Default $null
|
||||
if ($null -ne $channelStats) {
|
||||
$replacementFlowStats = @()
|
||||
foreach ($property in @($channelStats.PSObject.Properties)) {
|
||||
$stat = $property.Value
|
||||
if ([string](Get-PropertyValue -Item $stat -Name "last_route_id" -Default "") -eq $alternateRouteID) {
|
||||
$replacementFlowStats += $stat
|
||||
}
|
||||
}
|
||||
}
|
||||
if ($replacementLastSelected -eq $alternateRouteID -and $replacementFlowStats.Count -ge 1) {
|
||||
$replacementTrafficObserved = $true
|
||||
break
|
||||
}
|
||||
}
|
||||
if ($replacementTrafficObserved) {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
$postAccess = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/fabric/service-channels/access-telemetry?actor_user_id=$ActorUserID&limit=50").fabric_service_channel_access_telemetry
|
||||
$postChannel = @($postAccess.active_channels | Where-Object { $_.channel_id -eq $lease.channel_id }) | Select-Object -First 1
|
||||
|
||||
[void](Send-ReplacementDegradedHeartbeat -EntryNodeID $entryNode.id -ReplacementRouteID $alternateRouteID -RouteGeneration "$runId-primary")
|
||||
$noSafeSyntheticConfig = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/nodes/$($entryNode.id)/mesh/synthetic-config").synthetic_mesh_config
|
||||
$noSafePathDecisions = @()
|
||||
if ($noSafeSyntheticConfig.PSObject.Properties.Name -contains "route_path_decisions") {
|
||||
$noSafePathDecisions = @($noSafeSyntheticConfig.route_path_decisions.decisions)
|
||||
}
|
||||
$noSafeDecision = $noSafePathDecisions | Where-Object {
|
||||
[string](Get-PropertyValue -Item $_ -Name "route_id" -Default "") -eq $alternateRouteID -and
|
||||
[string](Get-PropertyValue -Item $_ -Name "decision_source" -Default "") -eq "service_channel_feedback_no_alternate"
|
||||
} | Select-Object -First 1
|
||||
$noSafeFeedback = $null
|
||||
$serviceChannelFeedback = Get-PropertyValue -Item $noSafeSyntheticConfig -Name "service_channel_feedback" -Default $null
|
||||
if ($null -ne $serviceChannelFeedback -and $serviceChannelFeedback.PSObject.Properties.Name -contains "routes") {
|
||||
$noSafeFeedback = @($serviceChannelFeedback.routes | Where-Object { [string](Get-PropertyValue -Item $_ -Name "route_id" -Default "") -eq $alternateRouteID }) | Select-Object -First 1
|
||||
}
|
||||
|
||||
$postNoSafeAccess = $null
|
||||
$postNoSafeChannel = $null
|
||||
for ($i = 0; $i -lt 12; $i++) {
|
||||
Start-Sleep -Seconds 3
|
||||
$postNoSafeAccess = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/fabric/service-channels/access-telemetry?actor_user_id=$ActorUserID&limit=50").fabric_service_channel_access_telemetry
|
||||
$postNoSafeChannel = @($postNoSafeAccess.active_channels | Where-Object { $_.channel_id -eq $lease.channel_id }) | Select-Object -First 1
|
||||
if ($null -ne $postNoSafeChannel -and [string](Get-PropertyValue -Item $postNoSafeChannel -Name "route_decision_source" -Default "") -eq "service_channel_feedback_no_alternate") {
|
||||
break
|
||||
}
|
||||
}
|
||||
$postNoSafeHealth = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/fabric/service-channels/rebuild-health?actor_user_id=$ActorUserID&limit=50").rebuild_health
|
||||
$postNoSafeIncidents = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/fabric/service-channels/rebuild-incidents?actor_user_id=$ActorUserID&limit=10").rebuild_incidents
|
||||
$postNoSafeIncident = @($postNoSafeIncidents | Where-Object {
|
||||
[string](Get-PropertyValue -Item $_ -Name "incident_source" -Default "") -eq "access_decision" -and
|
||||
[string](Get-PropertyValue -Item $_ -Name "guard_status" -Default "") -eq "access_no_safe_recovery" -and
|
||||
[string](Get-PropertyValue -Item $_ -Name "channel_id" -Default "") -eq [string]$lease.channel_id
|
||||
}) | Select-Object -First 1
|
||||
$silence = $null
|
||||
$listedSilence = $null
|
||||
$postSilenceHealth = $null
|
||||
$postSilenceIncident = $null
|
||||
$postResurfaceHealth = $null
|
||||
$postResurfaceIncident = $null
|
||||
$postResurfaceAccess = $null
|
||||
$postResurfaceChannel = $null
|
||||
$resurfaceReack = $null
|
||||
$postResurfaceReackHealth = $null
|
||||
$postResurfaceReackIncident = $null
|
||||
if ($null -ne $postNoSafeIncident) {
|
||||
$silence = Invoke-Api -Method POST -Path "/clusters/$ClusterID/fabric/service-channels/rebuild-health/silences" -Body @{
|
||||
actor_user_id = $ActorUserID
|
||||
incident_source = "access_decision"
|
||||
channel_id = [string]$lease.channel_id
|
||||
reporter_node_id = [string](Get-PropertyValue -Item $postNoSafeIncident -Name "reporter_node_id" -Default "")
|
||||
route_id = [string](Get-PropertyValue -Item $postNoSafeIncident -Name "route_id" -Default "")
|
||||
guard_status = "access_no_safe_recovery"
|
||||
generation = [string](Get-PropertyValue -Item $postNoSafeIncident -Name "generation" -Default "")
|
||||
reason = "c18z88 operator acknowledged access no-safe before generation change"
|
||||
ttl_seconds = 21600
|
||||
}
|
||||
Start-Sleep -Seconds 2
|
||||
$activeSilences = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/fabric/service-channels/rebuild-health/silences?actor_user_id=$ActorUserID").rebuild_alert_silences
|
||||
$silenceID = [string](Get-PropertyValue -Item (Get-PropertyValue -Item $silence -Name "rebuild_alert_silence" -Default $null) -Name "id" -Default "")
|
||||
$listedSilence = @($activeSilences | Where-Object { [string](Get-PropertyValue -Item $_ -Name "id" -Default "") -eq $silenceID }) | Select-Object -First 1
|
||||
$postSilenceHealth = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/fabric/service-channels/rebuild-health?actor_user_id=$ActorUserID&limit=50").rebuild_health
|
||||
$postSilenceIncidents = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/fabric/service-channels/rebuild-incidents?actor_user_id=$ActorUserID&limit=10").rebuild_incidents
|
||||
$postSilenceIncident = @($postSilenceIncidents | Where-Object {
|
||||
[string](Get-PropertyValue -Item $_ -Name "incident_source" -Default "") -eq "access_decision" -and
|
||||
[string](Get-PropertyValue -Item $_ -Name "guard_status" -Default "") -eq "access_no_safe_recovery" -and
|
||||
[string](Get-PropertyValue -Item $_ -Name "channel_id" -Default "") -eq [string]$lease.channel_id
|
||||
}) | Select-Object -First 1
|
||||
|
||||
for ($i = 0; $i -lt 18; $i++) {
|
||||
Start-Sleep -Seconds 5
|
||||
$postResurfaceHealth = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/fabric/service-channels/rebuild-health?actor_user_id=$ActorUserID&limit=50").rebuild_health
|
||||
$postResurfaceIncidents = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/fabric/service-channels/rebuild-incidents?actor_user_id=$ActorUserID&limit=10").rebuild_incidents
|
||||
$postResurfaceIncident = @($postResurfaceIncidents | Where-Object {
|
||||
[string](Get-PropertyValue -Item $_ -Name "incident_source" -Default "") -eq "access_decision" -and
|
||||
[string](Get-PropertyValue -Item $_ -Name "guard_status" -Default "") -eq "access_no_safe_recovery" -and
|
||||
[string](Get-PropertyValue -Item $_ -Name "channel_id" -Default "") -eq [string]$lease.channel_id -and
|
||||
[bool](Get-PropertyValue -Item $_ -Name "alert_resurfaced" -Default $false)
|
||||
}) | Select-Object -First 1
|
||||
if ($null -ne $postResurfaceIncident) { break }
|
||||
}
|
||||
|
||||
if ($null -ne $postResurfaceIncident) {
|
||||
$postResurfaceAccess = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/fabric/service-channels/access-telemetry?actor_user_id=$ActorUserID&limit=50").fabric_service_channel_access_telemetry
|
||||
$postResurfaceChannel = @($postResurfaceAccess.active_channels | Where-Object { $_.channel_id -eq $lease.channel_id }) | Select-Object -First 1
|
||||
|
||||
$resurfaceReack = Invoke-Api -Method POST -Path "/clusters/$ClusterID/fabric/service-channels/rebuild-health/silences" -Body @{
|
||||
actor_user_id = $ActorUserID
|
||||
incident_source = "access_decision"
|
||||
channel_id = [string]$lease.channel_id
|
||||
reporter_node_id = [string](Get-PropertyValue -Item $postResurfaceIncident -Name "reporter_node_id" -Default "")
|
||||
route_id = [string](Get-PropertyValue -Item $postResurfaceIncident -Name "route_id" -Default "")
|
||||
guard_status = "access_no_safe_recovery"
|
||||
generation = [string](Get-PropertyValue -Item $postResurfaceIncident -Name "generation" -Default "")
|
||||
reason = "c18z89 operator re-acknowledged resurfaced access no-safe"
|
||||
ttl_seconds = 21600
|
||||
}
|
||||
Start-Sleep -Seconds 2
|
||||
$postResurfaceReackHealth = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/fabric/service-channels/rebuild-health?actor_user_id=$ActorUserID&limit=50").rebuild_health
|
||||
$postResurfaceReackIncidents = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/fabric/service-channels/rebuild-incidents?actor_user_id=$ActorUserID&limit=10").rebuild_incidents
|
||||
$postResurfaceReackIncident = @($postResurfaceReackIncidents | Where-Object {
|
||||
[string](Get-PropertyValue -Item $_ -Name "incident_source" -Default "") -eq "access_decision" -and
|
||||
[string](Get-PropertyValue -Item $_ -Name "guard_status" -Default "") -eq "access_no_safe_recovery" -and
|
||||
[string](Get-PropertyValue -Item $_ -Name "channel_id" -Default "") -eq [string]$lease.channel_id
|
||||
}) | Select-Object -First 1
|
||||
}
|
||||
}
|
||||
$backendLine = (& ssh $DockerSSH "docker ps --format '{{.Names}} {{.Image}} {{.Status}}' | grep '^rap_test_backend '") | Out-String
|
||||
$nodeLines = (& ssh $DockerSSH "docker ps --format '{{.Names}} {{.Image}} {{.Status}}' | grep '^rap_test_node_test_'") | Out-String
|
||||
$checks = [ordered]@{
|
||||
backend_expected_image_deployed = $backendLine.Contains($ExpectedBackendImage)
|
||||
node_agent_expected_image_deployed = $nodeLines.Contains($ExpectedNodeAgentImage)
|
||||
lease_ready = ([string]$lease.status -eq "ready")
|
||||
remediation_rebuild_route_visible = ($null -ne $matchingChannel -and [string](Get-PropertyValue -Item $matchingChannel -Name "remediation_action" -Default "") -eq "rebuild_route")
|
||||
remediation_command_visible = ($null -ne $command -and $commandID.Length -gt 0)
|
||||
durable_rebuild_intent_recorded = ($null -ne $attempt)
|
||||
durable_rebuild_intent_resolved_applied = ($null -ne $attempt -and [string](Get-PropertyValue -Item $attempt -Name "rebuild_status" -Default "") -eq "applied")
|
||||
durable_rebuild_intent_outcome_replacement_selected = ($null -ne $attempt -and [string](Get-PropertyValue -Item $attempt -Name "outcome" -Default "") -eq "replacement_selected")
|
||||
durable_rebuild_intent_replacement_matches = ($null -ne $attempt -and [string](Get-PropertyValue -Item $attempt -Name "replacement_route_id" -Default "") -eq $alternateRouteID)
|
||||
durable_rebuild_intent_source_matches = ($null -ne $attempt -and [string](Get-PropertyValue -Item $attempt -Name "decision_source" -Default "") -eq "service_channel_remediation_command")
|
||||
initial_access_telemetry_reports_planner_applied = ($null -ne $postChannel -and [string](Get-PropertyValue -Item $postChannel -Name "remediation_execution_status" -Default "") -eq "rebuild_request_applied")
|
||||
node_route_manager_consumed_same_command = ($null -ne $routeManagerDecision)
|
||||
node_route_manager_applied_rebuild = ($null -ne $routeManagerDecision -and [string](Get-PropertyValue -Item $routeManagerDecision -Name "rebuild_status" -Default "") -eq "applied")
|
||||
node_route_manager_transition_applied_rebuild = ($null -ne $routeManagerTransition -and [string](Get-PropertyValue -Item $routeManagerTransition -Name "status" -Default "") -eq "applied_rebuild")
|
||||
traffic_packet_accepted = ([int]$trafficResponse.StatusCode -eq 202)
|
||||
traffic_accepted_by_runtime = ($trafficAcceptedBy -eq "introspection" -or $trafficAcceptedBy -eq "signed")
|
||||
traffic_selected_replacement_route = $replacementTrafficObserved
|
||||
traffic_last_selected_route_matches_replacement = ($replacementLastSelected -eq $alternateRouteID)
|
||||
no_backend_or_local_fallback_after_replacement = (($postRemediationFallbackLocal - $baselineFallbackLocal) -eq 0)
|
||||
no_route_failures_after_replacement = (($postRemediationRouteFailures - $baselineRouteFailures) -eq 0)
|
||||
no_flow_drops_after_replacement = (($postRemediationFlowDropped - $baselineFlowDropped) -eq 0 -and ($postRemediationSchedulerDropped - $baselineSchedulerDropped) -eq 0)
|
||||
replacement_degradation_feedback_visible = ($null -ne $noSafeDecision -and @($noSafeDecision.score_reasons | Where-Object { [string]$_ -eq "service_channel_fenced_route" }).Count -gt 0)
|
||||
no_safe_recovery_decision_visible = ($null -ne $noSafeDecision)
|
||||
no_safe_recovery_status_pending_fallback = ($null -ne $noSafeDecision -and [string](Get-PropertyValue -Item $noSafeDecision -Name "rebuild_status" -Default "") -eq "pending_degraded_fallback")
|
||||
no_safe_recovery_reason_visible = ($null -ne $noSafeDecision -and @($noSafeDecision.score_reasons | Where-Object { [string]$_ -eq "no_unfenced_alternate_route" }).Count -gt 0)
|
||||
no_silent_stickiness_to_bad_replacement = ($null -ne $noSafeDecision -and [string](Get-PropertyValue -Item $noSafeDecision -Name "decision_source" -Default "") -eq "service_channel_feedback_no_alternate")
|
||||
access_telemetry_projects_no_safe_decision = ($null -ne $postNoSafeChannel -and [string](Get-PropertyValue -Item $postNoSafeChannel -Name "route_decision_source" -Default "") -eq "service_channel_feedback_no_alternate")
|
||||
access_telemetry_projects_pending_fallback = ($null -ne $postNoSafeChannel -and [string](Get-PropertyValue -Item $postNoSafeChannel -Name "route_decision_rebuild_status" -Default "") -eq "pending_degraded_fallback")
|
||||
access_telemetry_projects_no_safe_route_id = ($null -ne $postNoSafeChannel -and @($primaryRouteID, $alternateRouteID) -contains [string](Get-PropertyValue -Item $postNoSafeChannel -Name "route_decision_route_id" -Default ""))
|
||||
access_telemetry_projects_operator_state = ($null -ne $postNoSafeChannel -and @("route_rebuild_no_safe_recovery", "rebuild_request_no_alternate") -contains [string](Get-PropertyValue -Item $postNoSafeChannel -Name "remediation_execution_status" -Default ""))
|
||||
access_telemetry_aggregates_route_decision = ($null -ne $postNoSafeAccess -and [int](Get-PropertyValue -Item $postNoSafeAccess -Name "route_decision_channel_count" -Default 0) -ge 1)
|
||||
access_telemetry_aggregates_no_safe_decision = ($null -ne $postNoSafeAccess -and [int](Get-PropertyValue -Item $postNoSafeAccess -Name "no_safe_recovery_decision_count" -Default 0) -ge 1)
|
||||
access_telemetry_aggregate_status_prioritizes_no_safe = ($null -ne $postNoSafeAccess -and [string](Get-PropertyValue -Item $postNoSafeAccess -Name "reason" -Default "") -eq "active_channels_no_safe_recovery")
|
||||
rebuild_health_projects_access_no_safe = ($null -ne $postNoSafeHealth -and [int](Get-PropertyValue -Item $postNoSafeHealth -Name "access_no_safe_count" -Default 0) -ge 1)
|
||||
rebuild_health_action_prioritizes_access_no_safe = ($null -ne $postNoSafeHealth -and [string](Get-PropertyValue -Item $postNoSafeHealth -Name "recommended_operator_action" -Default "") -eq "inspect_access_no_safe_recovery_route_pool_and_signed_policy")
|
||||
rebuild_incident_projects_access_no_safe = ($null -ne $postNoSafeIncident)
|
||||
rebuild_incident_access_no_safe_is_bad = ($null -ne $postNoSafeIncident -and [string](Get-PropertyValue -Item $postNoSafeIncident -Name "guard_severity" -Default "") -eq "bad")
|
||||
access_decision_silence_recorded = ($null -ne $silence -and $null -ne (Get-PropertyValue -Item $silence -Name "rebuild_alert_silence" -Default $null))
|
||||
access_decision_silence_listed = ($null -ne $listedSilence -and [string](Get-PropertyValue -Item $listedSilence -Name "incident_source" -Default "") -eq "access_decision" -and [string](Get-PropertyValue -Item $listedSilence -Name "channel_id" -Default "") -eq [string]$lease.channel_id)
|
||||
access_decision_incident_silenced = ($null -ne $postSilenceIncident -and [bool](Get-PropertyValue -Item $postSilenceIncident -Name "alert_silenced" -Default $false))
|
||||
access_decision_silence_reduces_active_bad = ($null -ne $postSilenceHealth -and [int](Get-PropertyValue -Item $postSilenceHealth -Name "silenced_count" -Default 0) -ge 1)
|
||||
access_decision_resurfaced_on_generation_change = ($null -ne $postResurfaceIncident -and [bool](Get-PropertyValue -Item $postResurfaceIncident -Name "alert_resurfaced" -Default $false))
|
||||
access_decision_resurface_restores_active_bad = ($null -ne $postResurfaceHealth -and [int](Get-PropertyValue -Item $postResurfaceHealth -Name "active_bad_count" -Default 0) -ge 1 -and [int](Get-PropertyValue -Item $postResurfaceHealth -Name "resurfaced_count" -Default 0) -ge 1)
|
||||
access_decision_resurface_keeps_previous_generation = ($null -ne $postResurfaceIncident -and [string](Get-PropertyValue -Item $postResurfaceIncident -Name "alert_resurfaced_previous_generation" -Default "") -eq [string](Get-PropertyValue -Item $postNoSafeIncident -Name "generation" -Default ""))
|
||||
access_decision_resurface_cause_generation_changed = ($null -ne $postResurfaceIncident -and [string](Get-PropertyValue -Item $postResurfaceIncident -Name "alert_resurfaced_cause" -Default "") -eq "generation_changed")
|
||||
access_decision_resurface_previous_route_visible = ($null -ne $postResurfaceIncident -and [string](Get-PropertyValue -Item $postResurfaceIncident -Name "alert_resurfaced_previous_route_id" -Default "") -eq [string](Get-PropertyValue -Item $postNoSafeIncident -Name "route_id" -Default ""))
|
||||
access_decision_resurface_previous_channel_visible = ($null -ne $postResurfaceIncident -and [string](Get-PropertyValue -Item $postResurfaceIncident -Name "alert_resurfaced_previous_channel_id" -Default "") -eq [string]$lease.channel_id)
|
||||
access_decision_resurface_active_channel_context_visible = ($null -ne $postResurfaceChannel -and [string](Get-PropertyValue -Item $postResurfaceChannel -Name "route_decision_source" -Default "") -eq "service_channel_feedback_no_alternate" -and [string](Get-PropertyValue -Item $postResurfaceChannel -Name "route_decision_generation" -Default "") -eq [string](Get-PropertyValue -Item $postResurfaceIncident -Name "generation" -Default "") -and [string](Get-PropertyValue -Item $postResurfaceChannel -Name "route_decision_route_id" -Default "") -eq [string](Get-PropertyValue -Item $postResurfaceIncident -Name "route_id" -Default ""))
|
||||
access_decision_resurfaced_reack_recorded = ($null -ne $resurfaceReack -and $null -ne (Get-PropertyValue -Item $resurfaceReack -Name "rebuild_alert_silence" -Default $null))
|
||||
access_decision_resurfaced_reack_silences_current_generation = ($null -ne $postResurfaceReackIncident -and [bool](Get-PropertyValue -Item $postResurfaceReackIncident -Name "alert_silenced" -Default $false) -and (-not [bool](Get-PropertyValue -Item $postResurfaceReackIncident -Name "alert_resurfaced" -Default $false)))
|
||||
access_decision_resurfaced_reack_reduces_active_bad = ($null -ne $postResurfaceReackHealth -and [int](Get-PropertyValue -Item $postResurfaceReackHealth -Name "silenced_count" -Default 0) -ge 1)
|
||||
}
|
||||
$failed = @($checks.GetEnumerator() | Where-Object { -not $_.Value } | ForEach-Object { $_.Key })
|
||||
|
||||
$result = [ordered]@{
|
||||
schema_version = "c18z89.service_channel_access_decision_resurface_action_smoke.v1"
|
||||
run_id = $runId
|
||||
cluster_id = $ClusterID
|
||||
channel_id = [string]$lease.channel_id
|
||||
primary_route_id = $primaryRouteID
|
||||
alternate_route_id = $alternateRouteID
|
||||
rebuild_request_id = $commandID
|
||||
passed = ($failed.Count -eq 0)
|
||||
checks = $checks
|
||||
failed_checks = $failed
|
||||
summary = [ordered]@{
|
||||
backend_container = $backendLine.Trim()
|
||||
node_containers = $nodeLines.Trim()
|
||||
remediation_command = $command
|
||||
rebuild_attempt = $attempt
|
||||
route_manager_decision = $routeManagerDecision
|
||||
route_manager_transition = $routeManagerTransition
|
||||
traffic_status_code = [int]$trafficResponse.StatusCode
|
||||
traffic_accepted_by = $trafficAcceptedBy
|
||||
traffic_duration_ms = $trafficDurationMs
|
||||
traffic_packet_count = $trafficPackets.Count
|
||||
replacement_last_selected_route_id = $replacementLastSelected
|
||||
replacement_flow_stat_count = $replacementFlowStats.Count
|
||||
replacement_flow_stats = $replacementFlowStats
|
||||
baseline_send_fallback_local = $baselineFallbackLocal
|
||||
baseline_send_route_failures = $baselineRouteFailures
|
||||
baseline_send_flow_dropped = $baselineFlowDropped
|
||||
baseline_scheduler_dropped = $baselineSchedulerDropped
|
||||
fallback_local_delta = ($postRemediationFallbackLocal - $baselineFallbackLocal)
|
||||
route_failure_delta = ($postRemediationRouteFailures - $baselineRouteFailures)
|
||||
flow_drop_delta = ($postRemediationFlowDropped - $baselineFlowDropped)
|
||||
scheduler_drop_delta = ($postRemediationSchedulerDropped - $baselineSchedulerDropped)
|
||||
post_remediation_send_fallback_local = $postRemediationFallbackLocal
|
||||
post_remediation_send_route_failures = $postRemediationRouteFailures
|
||||
post_remediation_send_flow_dropped = $postRemediationFlowDropped
|
||||
post_remediation_scheduler_dropped = $postRemediationSchedulerDropped
|
||||
post_channel = $postChannel
|
||||
no_safe_feedback = $noSafeFeedback
|
||||
no_safe_decision = $noSafeDecision
|
||||
no_safe_route_path_decisions = $noSafePathDecisions
|
||||
post_no_safe_channel = $postNoSafeChannel
|
||||
post_no_safe_access = $postNoSafeAccess
|
||||
post_no_safe_health = $postNoSafeHealth
|
||||
post_no_safe_incidents = $postNoSafeIncidents
|
||||
post_no_safe_incident = $postNoSafeIncident
|
||||
silence = $silence
|
||||
listed_silence = $listedSilence
|
||||
post_silence_health = $postSilenceHealth
|
||||
post_silence_incident = $postSilenceIncident
|
||||
post_resurface_health = $postResurfaceHealth
|
||||
post_resurface_incident = $postResurfaceIncident
|
||||
post_resurface_access = $postResurfaceAccess
|
||||
post_resurface_channel = $postResurfaceChannel
|
||||
resurface_reack = $resurfaceReack
|
||||
post_resurface_reack_health = $postResurfaceReackHealth
|
||||
post_resurface_reack_incident = $postResurfaceReackIncident
|
||||
}
|
||||
}
|
||||
|
||||
$target = Join-Path $repoRoot $ResultPath
|
||||
$result | ConvertTo-Json -Depth 80 | Set-Content -Path $target -Encoding UTF8
|
||||
|
||||
try {
|
||||
if ($primaryRouteID) {
|
||||
Invoke-Api -Method POST -Path "/clusters/$ClusterID/mesh/route-intents/$primaryRouteID/expire" -Body @{ actor_user_id = $ActorUserID } | Out-Null
|
||||
}
|
||||
if ($alternateRouteID) {
|
||||
Invoke-Api -Method POST -Path "/clusters/$ClusterID/mesh/route-intents/$alternateRouteID/expire" -Body @{ actor_user_id = $ActorUserID } | Out-Null
|
||||
}
|
||||
Start-Sleep -Seconds 3
|
||||
Invoke-Api -Method POST -Path "/clusters/$ClusterID/fabric/service-channels/leases/cleanup" -Body @{
|
||||
actor_user_id = $ActorUserID
|
||||
limit = 50
|
||||
} | Out-Null
|
||||
} catch {
|
||||
Write-Warning "cleanup failed after c18z89 smoke: $($_.Exception.Message)"
|
||||
}
|
||||
|
||||
if (-not $result.passed) {
|
||||
throw "C18Z89 access decision resurface action smoke failed: $($failed -join ', ')"
|
||||
}
|
||||
|
||||
Write-Host "C18Z89 service-channel access decision resurface action smoke passed. Result: $target"
|
||||
$result
|
||||
|
||||
|
||||
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user