рабочий вариант, но скороть 10 МБит
This commit is contained in:
@@ -1,4 +1,4 @@
|
||||
#requires -Version 5
|
||||
#requires -Version 5
|
||||
|
||||
param(
|
||||
[string]$RepoRoot = (Resolve-Path (Join-Path $PSScriptRoot "..\..")).Path,
|
||||
@@ -8,7 +8,8 @@ param(
|
||||
[switch]$PrintOnly,
|
||||
[bool]$PublishToTestDockerDownloads = $false,
|
||||
[string]$TestDockerSshAlias = "test-docker",
|
||||
[string]$TestDockerDownloadPath = "/tmp/rap-web-admin/html/downloads"
|
||||
[string]$TestDockerDownloadPath = "/tmp/rap-web-admin/html/downloads",
|
||||
[string]$TestDockerBackendReleasePath = "/tmp/rap-release-0.2.309-latencyaware"
|
||||
)
|
||||
|
||||
$ErrorActionPreference = "Stop"
|
||||
@@ -53,7 +54,8 @@ function Get-JavaVersion {
|
||||
Fail "Java не найден в PATH. Нужен JDK 17+."
|
||||
}
|
||||
|
||||
$versionLine = & $javaCandidate.Source -version 2>&1 | Select-Object -First 1
|
||||
$escapedJava = $javaCandidate.Source.Replace('"', '\"')
|
||||
$versionLine = & cmd.exe /c "`"$escapedJava`" -version 2>&1" | Select-Object -First 1
|
||||
if ($versionLine -match "version\s+`"(\d+)\.") {
|
||||
return $matches[1]
|
||||
}
|
||||
@@ -134,7 +136,9 @@ function Publish-Artifact {
|
||||
}
|
||||
}
|
||||
|
||||
$published | ConvertTo-Json -Depth 6 | Out-File -FilePath $metaPath -Encoding utf8 -Force
|
||||
$manifestJson = ($published | ConvertTo-Json -Depth 6) + [Environment]::NewLine
|
||||
$utf8NoBom = New-Object System.Text.UTF8Encoding($false)
|
||||
[System.IO.File]::WriteAllText($metaPath, $manifestJson, $utf8NoBom)
|
||||
|
||||
return @{
|
||||
latest = $latestPath
|
||||
@@ -306,6 +310,7 @@ try {
|
||||
|
||||
if ($PrintOnly) {
|
||||
Write-Host "PrintOnly: сборка не выполнена."
|
||||
$global:LASTEXITCODE = 0
|
||||
return
|
||||
}
|
||||
|
||||
@@ -384,6 +389,15 @@ try {
|
||||
-LatestFile $remotePublishSource.latest `
|
||||
-VersionFile $remotePublishSource.versioned `
|
||||
-ManifestFile $remotePublishSource.summary
|
||||
|
||||
if (-not [string]::IsNullOrWhiteSpace($TestDockerBackendReleasePath) -and $TestDockerBackendReleasePath -ne $TestDockerDownloadPath) {
|
||||
Write-Host "Публикую артефакты в backend RAP_RELEASE_DIR ($TestDockerSshAlias): $TestDockerBackendReleasePath"
|
||||
Publish-ToTestDockerDownloads -RemoteHost $TestDockerSshAlias -RemoteRoot (Resolve-RemoteDirectory $TestDockerBackendReleasePath) `
|
||||
-VersionName $versionName `
|
||||
-LatestFile $remotePublishSource.latest `
|
||||
-VersionFile $remotePublishSource.versioned `
|
||||
-ManifestFile $remotePublishSource.summary
|
||||
}
|
||||
}
|
||||
|
||||
Write-Host "Сборка завершена."
|
||||
@@ -401,3 +415,5 @@ finally {
|
||||
if (-not $buildSucceeded) {
|
||||
Fail "Сборка не завершена."
|
||||
}
|
||||
|
||||
$global:LASTEXITCODE = 0
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
#requires -Version 5
|
||||
#requires -Version 5
|
||||
|
||||
param(
|
||||
[string]$BuildType = "release",
|
||||
@@ -9,6 +9,7 @@ param(
|
||||
[string]$PortalVerifyBaseUrl = "http://192.168.200.61:18080",
|
||||
[string]$TestDockerSshAlias = "test-docker",
|
||||
[string]$TestDockerDownloadPath = "/tmp/rap-web-admin/html/downloads",
|
||||
[string]$TestDockerBackendReleasePath = "/tmp/rap-release-0.2.309-latencyaware",
|
||||
[int]$PreparationRetryDelaySeconds = 0
|
||||
)
|
||||
|
||||
@@ -29,6 +30,7 @@ $publishToTestDockerDownloads = $PublishToTestDockerDownloads -and -not $NoRemot
|
||||
-PublishToTestDockerDownloads:$publishToTestDockerDownloads `
|
||||
-TestDockerSshAlias $TestDockerSshAlias `
|
||||
-TestDockerDownloadPath $TestDockerDownloadPath `
|
||||
-TestDockerBackendReleasePath $TestDockerBackendReleasePath `
|
||||
-PortalVerifyBaseUrl $PortalVerifyBaseUrl `
|
||||
-SkipPortalVerify:$SkipPortalVerify `
|
||||
-PreparationRetryDelaySeconds $PreparationRetryDelaySeconds `
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
#requires -Version 5
|
||||
#requires -Version 5
|
||||
|
||||
param(
|
||||
[string]$RepoRoot = (Resolve-Path (Join-Path $PSScriptRoot "..\..")).ProviderPath,
|
||||
@@ -11,6 +11,7 @@ param(
|
||||
[bool]$PublishToTestDockerDownloads = $true,
|
||||
[string]$TestDockerSshAlias = "test-docker",
|
||||
[string]$TestDockerDownloadPath = "/tmp/rap-web-admin/html/downloads",
|
||||
[string]$TestDockerBackendReleasePath = "/tmp/rap-release-0.2.309-latencyaware",
|
||||
[int]$PreparationRetryDelaySeconds = 0
|
||||
)
|
||||
|
||||
@@ -69,7 +70,8 @@ if ($PrintOnly) {
|
||||
-SkipWorkspaceCleanup:$SkipWorkspaceCleanup `
|
||||
-PublishToTestDockerDownloads:$PublishToTestDockerDownloads `
|
||||
-TestDockerSshAlias $TestDockerSshAlias `
|
||||
-TestDockerDownloadPath $TestDockerDownloadPath
|
||||
-TestDockerDownloadPath $TestDockerDownloadPath `
|
||||
-TestDockerBackendReleasePath $TestDockerBackendReleasePath
|
||||
}
|
||||
exit 0
|
||||
}
|
||||
@@ -78,7 +80,8 @@ Run-Step "Сборка и публикация Android APK" {
|
||||
& $buildScript -RepoRoot $RepoRoot -BuildType $BuildType -AndroidHome $AndroidHome -SkipWorkspaceCleanup:$SkipWorkspaceCleanup `
|
||||
-PublishToTestDockerDownloads:$PublishToTestDockerDownloads `
|
||||
-TestDockerSshAlias $TestDockerSshAlias `
|
||||
-TestDockerDownloadPath $TestDockerDownloadPath
|
||||
-TestDockerDownloadPath $TestDockerDownloadPath `
|
||||
-TestDockerBackendReleasePath $TestDockerBackendReleasePath
|
||||
}
|
||||
|
||||
Write-Host ""
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
#requires -Version 5
|
||||
#requires -Version 5
|
||||
|
||||
param(
|
||||
[string]$RepoRoot = (Resolve-Path (Join-Path $PSScriptRoot "..\..")).ProviderPath,
|
||||
@@ -12,6 +12,7 @@ param(
|
||||
[switch]$NoRemotePublish,
|
||||
[string]$TestDockerSshAlias = "test-docker",
|
||||
[string]$TestDockerDownloadPath = "/tmp/rap-web-admin/html/downloads",
|
||||
[string]$TestDockerBackendReleasePath = "/tmp/rap-release-0.2.309-latencyaware",
|
||||
[int]$PreparationRetryDelaySeconds = 0,
|
||||
[string]$PortalVerifyBaseUrl = "http://192.168.200.61:18080",
|
||||
[switch]$SkipPortalVerify
|
||||
@@ -77,7 +78,8 @@ if ($PrintOnly) {
|
||||
-SkipWorkspaceCleanup:$SkipWorkspaceCleanup `
|
||||
-PublishToTestDockerDownloads:$true `
|
||||
-TestDockerSshAlias $TestDockerSshAlias `
|
||||
-TestDockerDownloadPath $TestDockerDownloadPath
|
||||
-TestDockerDownloadPath $TestDockerDownloadPath `
|
||||
-TestDockerBackendReleasePath $TestDockerBackendReleasePath
|
||||
} else {
|
||||
& $buildScript -RepoRoot $RepoRoot -BuildType $BuildType -AndroidHome $AndroidHome -PrintOnly `
|
||||
-SkipWorkspaceCleanup:$SkipWorkspaceCleanup
|
||||
@@ -90,7 +92,8 @@ Run-Step "Сборка и публикация Android APK" {
|
||||
& $buildScript -RepoRoot $RepoRoot -BuildType $BuildType -AndroidHome $AndroidHome -SkipWorkspaceCleanup:$SkipWorkspaceCleanup `
|
||||
-PublishToTestDockerDownloads:$true `
|
||||
-TestDockerSshAlias $TestDockerSshAlias `
|
||||
-TestDockerDownloadPath $TestDockerDownloadPath
|
||||
-TestDockerDownloadPath $TestDockerDownloadPath `
|
||||
-TestDockerBackendReleasePath $TestDockerBackendReleasePath
|
||||
} else {
|
||||
& $buildScript -RepoRoot $RepoRoot -BuildType $BuildType -AndroidHome $AndroidHome -SkipWorkspaceCleanup:$SkipWorkspaceCleanup
|
||||
}
|
||||
@@ -117,7 +120,7 @@ Run-Step "Проверка манифеста веб-панели" {
|
||||
Write-Host "Sha256: $($manifest.published.sha256)"
|
||||
|
||||
if (-not [string]::IsNullOrWhiteSpace($PortalVerifyBaseUrl)) {
|
||||
$manifestUrl = "$PortalVerifyBaseUrl/downloads/rap-android-vpn-build.json?_cb=$(Get-Date -Format 'yyyyMMddHHmmss')"
|
||||
$manifestUrl = "$PortalVerifyBaseUrl/api/v1/downloads/rap-android-vpn-build.json?_cb=$(Get-Date -Format 'yyyyMMddHHmmss')"
|
||||
try {
|
||||
$remoteManifest = Invoke-RestMethod -Uri $manifestUrl -Method Get
|
||||
if (-not $remoteManifest.version -or -not $remoteManifest.version.name -or $remoteManifest.version.name -ne $manifest.version.name) {
|
||||
|
||||
@@ -0,0 +1,174 @@
|
||||
param()
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
$repoRoot = (Resolve-Path (Join-Path $PSScriptRoot "..")).ProviderPath
|
||||
$forbidden = @(
|
||||
('RAP_' + 'BACKEND_URL'),
|
||||
('RAP_' + 'CONTROL_PLANE_ENDPOINTS_JSON'),
|
||||
('RAP_' + 'MESH_LISTEN_ADDR'),
|
||||
('RAP_' + 'MESH_LISTEN_PORT_MODE'),
|
||||
('RAP_' + 'MESH_LISTEN_AUTO_PORT_START'),
|
||||
('RAP_' + 'MESH_LISTEN_AUTO_PORT_END'),
|
||||
('RAP_' + 'MESH_SYNTHETIC_RUNTIME_ENABLED'),
|
||||
('--' + 'backend-url'),
|
||||
('--' + 'control-plane-endpoints-json'),
|
||||
('mesh' + '-listener'),
|
||||
('mesh_' + 'synthetic_runtime_enabled'),
|
||||
('mesh' + '-synthetic-runtime-enabled')
|
||||
)
|
||||
|
||||
$rgArgs = @(
|
||||
'-n',
|
||||
'--glob', '!**/deploy/html/assets/**',
|
||||
'--glob', '!**/dist/**',
|
||||
'--glob', '!**/node_modules/**',
|
||||
($forbidden -join '|'),
|
||||
$repoRoot
|
||||
)
|
||||
|
||||
$matches = & rg @rgArgs
|
||||
if ($LASTEXITCODE -eq 1) {
|
||||
$matches = @()
|
||||
} elseif ($LASTEXITCODE -ne 0) {
|
||||
throw "rg failed with exit code $LASTEXITCODE"
|
||||
}
|
||||
|
||||
$farmActiveForbidden = @(
|
||||
('net' + '/http'),
|
||||
('http' + '://'),
|
||||
('https' + '://'),
|
||||
('ws' + '://'),
|
||||
('wss' + '://'),
|
||||
('http' + '.Client')
|
||||
)
|
||||
$farmStandardForbidden = @(
|
||||
('compat_' + 'fallback'),
|
||||
('compat' + ' fallback'),
|
||||
('compat_' + 'relay'),
|
||||
('last_' + 'compat_relay_policy'),
|
||||
('force_' + 'compat_fallback'),
|
||||
('entry_node_' + 'compat_fallback'),
|
||||
('fabric_route_send_failed_' + 'compat_fallback_blocked'),
|
||||
('compat_' + 'cleanup'),
|
||||
('compat_' + 'control'),
|
||||
('compat_' + 'updater'),
|
||||
('compat_' + 'recovery'),
|
||||
('compat_' + 'contract'),
|
||||
('remove_' + 'compat'),
|
||||
('fallback_' + 'poll_seconds'),
|
||||
('direct_' + 'fallback'),
|
||||
('trigger_direct_' + 'fallback'),
|
||||
('launchDirectUpdater' + 'Fallback'),
|
||||
('runLocalDirectUpdate' + 'Fallback')
|
||||
)
|
||||
$farmRouteForbidden = @(
|
||||
('/node-agents/\{nodeID\}/' + 'health'),
|
||||
('/node-agents/\{nodeID\}/' + 'services/status'),
|
||||
('/node-agents/\{nodeID\}/' + 'update-manifest/request'),
|
||||
('/node-agents/\{nodeID\}/' + 'update-result'),
|
||||
('/node-agents/\{nodeID\}/' + 'rollback-result'),
|
||||
('/node-agents/enrollments/\{requestID\}/' + 'bootstrap'),
|
||||
('docker-' + 'bootstrap-bundle'),
|
||||
('windows-' + 'bootstrap-bundle'),
|
||||
('linux-' + 'bootstrap-bundle')
|
||||
)
|
||||
$farmIngressClassForbidden = @(
|
||||
'platform_admin',
|
||||
'cluster_admin',
|
||||
'organization_portal',
|
||||
'user_portal',
|
||||
'global-admin-runtime',
|
||||
'cluster-admin-runtime',
|
||||
'organization-portal-runtime',
|
||||
'user-portal-runtime',
|
||||
'identity-runtime',
|
||||
'policy-authority',
|
||||
'audit-sink'
|
||||
)
|
||||
$farmActivePaths = @(
|
||||
(Join-Path $repoRoot "agents/rap-node-agent/internal/client"),
|
||||
(Join-Path $repoRoot "agents/rap-node-agent/internal/hostagent"),
|
||||
(Join-Path $repoRoot "agents/rap-node-agent/internal/mesh"),
|
||||
(Join-Path $repoRoot "agents/rap-node-agent/internal/vpnruntime"),
|
||||
(Join-Path $repoRoot "agents/rap-node-agent/cmd/rap-node-agent"),
|
||||
(Join-Path $repoRoot "agents/rap-node-agent/cmd/rap-host-agent")
|
||||
)
|
||||
$farmRgArgs = @(
|
||||
'-n',
|
||||
'--glob', '*.go',
|
||||
'--glob', '!*_test.go',
|
||||
($farmActiveForbidden -join '|')
|
||||
) + $farmActivePaths
|
||||
|
||||
$farmMatches = & rg @farmRgArgs
|
||||
if ($LASTEXITCODE -eq 1) {
|
||||
$farmMatches = @()
|
||||
} elseif ($LASTEXITCODE -ne 0) {
|
||||
throw "active farm rg failed with exit code $LASTEXITCODE"
|
||||
}
|
||||
|
||||
$farmStandardPaths = @(
|
||||
(Join-Path $repoRoot "agents/rap-node-agent/internal/client"),
|
||||
(Join-Path $repoRoot "agents/rap-node-agent/internal/hostagent"),
|
||||
(Join-Path $repoRoot "agents/rap-node-agent/internal/mesh"),
|
||||
(Join-Path $repoRoot "agents/rap-node-agent/internal/vpnruntime"),
|
||||
(Join-Path $repoRoot "agents/rap-node-agent/cmd/rap-node-agent"),
|
||||
(Join-Path $repoRoot "agents/rap-node-agent/cmd/rap-host-agent"),
|
||||
(Join-Path $repoRoot "backend/internal/modules/cluster"),
|
||||
(Join-Path $repoRoot "web-admin/src")
|
||||
)
|
||||
$farmStandardRgArgs = @(
|
||||
'-n',
|
||||
'--glob', '!**/*_test.go',
|
||||
($farmStandardForbidden -join '|')
|
||||
) + $farmStandardPaths
|
||||
|
||||
$farmStandardMatches = & rg @farmStandardRgArgs
|
||||
if ($LASTEXITCODE -eq 1) {
|
||||
$farmStandardMatches = @()
|
||||
} elseif ($LASTEXITCODE -ne 0) {
|
||||
throw "farm standard rg failed with exit code $LASTEXITCODE"
|
||||
}
|
||||
|
||||
$farmRouteRgArgs = @(
|
||||
'-n',
|
||||
'--glob', '!**/deploy/html/assets/**',
|
||||
'--glob', '!**/dist/**',
|
||||
'--glob', '!**/node_modules/**',
|
||||
($farmRouteForbidden -join '|'),
|
||||
$repoRoot
|
||||
)
|
||||
$farmRouteMatches = & rg @farmRouteRgArgs
|
||||
if ($LASTEXITCODE -eq 1) {
|
||||
$farmRouteMatches = @()
|
||||
} elseif ($LASTEXITCODE -ne 0) {
|
||||
throw "farm route rg failed with exit code $LASTEXITCODE"
|
||||
}
|
||||
|
||||
$farmIngressClassPaths = @(
|
||||
(Join-Path $repoRoot "agents/rap-node-agent/internal/webingress"),
|
||||
(Join-Path $repoRoot "agents/rap-node-agent/internal/supervisor"),
|
||||
(Join-Path $repoRoot "agents/rap-node-agent/cmd/rap-node-agent")
|
||||
)
|
||||
$farmIngressClassRgArgs = @(
|
||||
'-n',
|
||||
'--glob', '!**/*_test.go',
|
||||
($farmIngressClassForbidden -join '|')
|
||||
) + $farmIngressClassPaths
|
||||
$farmIngressClassMatches = & rg @farmIngressClassRgArgs
|
||||
if ($LASTEXITCODE -eq 1) {
|
||||
$farmIngressClassMatches = @()
|
||||
} elseif ($LASTEXITCODE -ne 0) {
|
||||
throw "farm ingress class rg failed with exit code $LASTEXITCODE"
|
||||
}
|
||||
|
||||
if (@($matches).Count -eq 0 -and @($farmMatches).Count -eq 0 -and @($farmStandardMatches).Count -eq 0 -and @($farmRouteMatches).Count -eq 0 -and @($farmIngressClassMatches).Count -eq 0) {
|
||||
Write-Host "Fabric standard boundary check passed."
|
||||
exit 0
|
||||
}
|
||||
|
||||
$allMatches = @($matches) + @($farmMatches) + @($farmStandardMatches) + @($farmRouteMatches) + @($farmIngressClassMatches)
|
||||
Write-Error "Fabric standard boundary violated:`n$allMatches"
|
||||
exit 1
|
||||
@@ -0,0 +1,81 @@
|
||||
param(
|
||||
[string]$DockerContext = "test-ubuntu",
|
||||
[string]$PostgresContainer = "rap_test_postgres",
|
||||
[string]$Database = "remote_access_platform",
|
||||
[string]$User = "rap_user",
|
||||
[string]$ExpectedVersion = "0.2.369-fabric-artifact-chunks"
|
||||
)
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
function Invoke-PsqlScalar {
|
||||
param([string]$Sql)
|
||||
$output = & docker --context $DockerContext exec $PostgresContainer psql -U $User -d $Database -At -c $Sql
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "psql failed for query: $Sql"
|
||||
}
|
||||
return [int]($output | Select-Object -First 1)
|
||||
}
|
||||
|
||||
$violations = @()
|
||||
|
||||
$httpReleaseArtifacts = Invoke-PsqlScalar @"
|
||||
select count(*)
|
||||
from release_artifacts
|
||||
where url like 'http://%'
|
||||
or url like 'https://%'
|
||||
or metadata::text like '%http://%'
|
||||
or metadata::text like '%https://%';
|
||||
"@
|
||||
if ($httpReleaseArtifacts -ne 0) {
|
||||
$violations += "release_artifacts contain HTTP references: $httpReleaseArtifacts"
|
||||
}
|
||||
|
||||
$httpUpdateStatuses = Invoke-PsqlScalar @"
|
||||
select count(*)
|
||||
from node_update_status_reports
|
||||
where payload::text like '%http://%'
|
||||
or payload::text like '%https://%';
|
||||
"@
|
||||
if ($httpUpdateStatuses -ne 0) {
|
||||
$violations += "node_update_status_reports contain HTTP references: $httpUpdateStatuses"
|
||||
}
|
||||
|
||||
$nonFabricCurrentArtifacts = Invoke-PsqlScalar @"
|
||||
select count(*)
|
||||
from release_artifacts
|
||||
where version = '$ExpectedVersion'
|
||||
and url not like 'fabric-artifact://%';
|
||||
"@
|
||||
if ($nonFabricCurrentArtifacts -ne 0) {
|
||||
$violations += "current release artifacts are not fabric-artifact URLs: $nonFabricCurrentArtifacts"
|
||||
}
|
||||
|
||||
$unexpectedActiveReleases = Invoke-PsqlScalar @"
|
||||
select count(*)
|
||||
from release_versions
|
||||
where status = 'active'
|
||||
and version <> '$ExpectedVersion';
|
||||
"@
|
||||
if ($unexpectedActiveReleases -ne 0) {
|
||||
$violations += "unexpected active releases outside ${ExpectedVersion}: $unexpectedActiveReleases"
|
||||
}
|
||||
|
||||
$legacyNodeMetadata = Invoke-PsqlScalar @"
|
||||
select count(*)
|
||||
from nodes
|
||||
where metadata::text like '%relay_control%'
|
||||
or metadata::text like '%http://%'
|
||||
or metadata::text like '%https://%';
|
||||
"@
|
||||
if ($legacyNodeMetadata -ne 0) {
|
||||
$violations += "node metadata contains retired transport hints: $legacyNodeMetadata"
|
||||
}
|
||||
|
||||
if ($violations.Count -gt 0) {
|
||||
Write-Error ("Live farm fabric standard violated:`n" + ($violations -join "`n"))
|
||||
exit 1
|
||||
}
|
||||
|
||||
Write-Host "Live farm fabric standard check passed."
|
||||
@@ -1,390 +0,0 @@
|
||||
param(
|
||||
[string]$DockerSshAlias = "test-docker",
|
||||
[string]$NodeAgentImageTag = "rap-node-agent:c17h-synthetic-smoke",
|
||||
[string]$AdminEmail = "fabric-owner-c17h@example.local",
|
||||
[string]$AdminPassword = "SmokePass!123",
|
||||
[int]$ApiPort = 18080,
|
||||
[int]$MeshBasePort = 19081,
|
||||
[switch]$KeepRunning
|
||||
)
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
$backendPublicBaseUrl = "http://192.168.200.61:$ApiPort/api/v1"
|
||||
$backendContainerBaseUrl = "http://127.0.0.1:$ApiPort/api/v1"
|
||||
$runId = "c17h-ssh-" + (Get-Date -Format "yyyyMMdd-HHmmss")
|
||||
$nodePrefix = "rap_c17h_node_"
|
||||
|
||||
function Invoke-RemoteDocker {
|
||||
param([string[]]$Arguments)
|
||||
& ssh $DockerSshAlias docker @Arguments
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "ssh $DockerSshAlias docker $($Arguments -join ' ') failed with exit code $LASTEXITCODE"
|
||||
}
|
||||
}
|
||||
|
||||
function Invoke-RemoteDockerText {
|
||||
param([string[]]$Arguments)
|
||||
$previousErrorActionPreference = $ErrorActionPreference
|
||||
$ErrorActionPreference = "Continue"
|
||||
try {
|
||||
$output = & ssh $DockerSshAlias docker @Arguments 2>&1 | ForEach-Object { $_.ToString() }
|
||||
}
|
||||
finally {
|
||||
$ErrorActionPreference = $previousErrorActionPreference
|
||||
}
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "ssh $DockerSshAlias docker $($Arguments -join ' ') failed with exit code $LASTEXITCODE"
|
||||
}
|
||||
return $output
|
||||
}
|
||||
|
||||
function Invoke-RemoteShell {
|
||||
param([string]$Command)
|
||||
& ssh $DockerSshAlias $Command
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "ssh $DockerSshAlias $Command failed with exit code $LASTEXITCODE"
|
||||
}
|
||||
}
|
||||
|
||||
function Send-RemoteFile {
|
||||
param(
|
||||
[string]$RemotePath,
|
||||
[string]$Content
|
||||
)
|
||||
$Content | & ssh $DockerSshAlias "cat > '$RemotePath'"
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "write remote file failed: $RemotePath"
|
||||
}
|
||||
}
|
||||
|
||||
function Invoke-Api {
|
||||
param(
|
||||
[string]$Method,
|
||||
[string]$Path,
|
||||
[object]$Body = $null
|
||||
)
|
||||
$uri = "$backendPublicBaseUrl$Path"
|
||||
if ($null -eq $Body) {
|
||||
return Invoke-RestMethod -Method $Method -Uri $uri -TimeoutSec 30
|
||||
}
|
||||
return Invoke-RestMethod -Method $Method -Uri $uri -ContentType "application/json" -Body ($Body | ConvertTo-Json -Depth 40) -TimeoutSec 30
|
||||
}
|
||||
|
||||
function Wait-HttpReady {
|
||||
param([string]$Url)
|
||||
for ($i = 0; $i -lt 60; $i++) {
|
||||
try {
|
||||
$response = Invoke-WebRequest -UseBasicParsing -Uri $Url -TimeoutSec 2
|
||||
if ($response.StatusCode -ge 200 -and $response.StatusCode -lt 300) {
|
||||
return
|
||||
}
|
||||
}
|
||||
catch {
|
||||
Start-Sleep -Seconds 1
|
||||
}
|
||||
}
|
||||
throw "Timed out waiting for $Url"
|
||||
}
|
||||
|
||||
function Remove-NodeContainers {
|
||||
foreach ($key in @("a", "b", "c", "r", "idle")) {
|
||||
& ssh $DockerSshAlias docker rm -f "$nodePrefix$key" 2>$null | Out-Null
|
||||
}
|
||||
}
|
||||
|
||||
Wait-HttpReady -Url "http://192.168.200.61:$ApiPort/readyz"
|
||||
|
||||
$login = Invoke-Api -Method Post -Path "/auth/login" -Body @{
|
||||
email = $AdminEmail
|
||||
password = $AdminPassword
|
||||
device_fingerprint = "c17h-ssh-smoke-device"
|
||||
device_label = "C17H SSH synthetic smoke"
|
||||
trust_device = $true
|
||||
}
|
||||
$actorUserID = $login.user.id
|
||||
|
||||
$cluster = Invoke-Api -Method Post -Path "/clusters/" -Body @{
|
||||
actor_user_id = $actorUserID
|
||||
slug = "c17h-ssh-$((New-Guid).Guid.Substring(0, 8))"
|
||||
name = "C17H SSH Synthetic Mesh Smoke"
|
||||
region = "docker-test"
|
||||
metadata = @{
|
||||
stage = "c17h"
|
||||
run_id = $runId
|
||||
production_forwarding = $false
|
||||
created_by = "c17h-multi-agent-synthetic-smoke-ssh.ps1"
|
||||
}
|
||||
}
|
||||
$clusterID = $cluster.cluster.id
|
||||
|
||||
Invoke-Api -Method Put -Path "/fabric/testing-flags" -Body @{
|
||||
actor_user_id = $actorUserID
|
||||
scope_type = "platform"
|
||||
scope_id = $null
|
||||
cluster_id = $null
|
||||
enabled = $true
|
||||
telemetry_enabled = $true
|
||||
synthetic_links_enabled = $true
|
||||
history_retention_hours = 24
|
||||
metadata = @{
|
||||
stage = "c17h"
|
||||
run_id = $runId
|
||||
production_forwarding = $false
|
||||
service_workload_traffic = $false
|
||||
}
|
||||
} | Out-Null
|
||||
|
||||
$joinToken = Invoke-Api -Method Post -Path "/clusters/$clusterID/join-tokens" -Body @{
|
||||
actor_user_id = $actorUserID
|
||||
scope = @{ purpose = "c17h-synthetic-smoke"; roles = @("core-mesh", "relay-node") }
|
||||
expires_at = (Get-Date).ToUniversalTime().AddHours(2).ToString("o")
|
||||
max_uses = 5
|
||||
}
|
||||
|
||||
$nodeSpecs = @(
|
||||
@{ key = "a"; name = "c17h-node-a"; roles = @("core-mesh"); port = $MeshBasePort },
|
||||
@{ key = "r"; name = "c17h-node-r"; roles = @("core-mesh", "relay-node"); port = ($MeshBasePort + 1) },
|
||||
@{ key = "b"; name = "c17h-node-b"; roles = @("core-mesh"); port = ($MeshBasePort + 2) },
|
||||
@{ key = "c"; name = "c17h-node-c"; roles = @("core-mesh"); port = ($MeshBasePort + 3) },
|
||||
@{ key = "idle"; name = "c17h-node-idle"; roles = @("core-mesh"); port = ($MeshBasePort + 4) }
|
||||
)
|
||||
|
||||
$nodes = @{}
|
||||
foreach ($spec in $nodeSpecs) {
|
||||
$fingerprint = "c17h-fp-$($spec.key)-$([guid]::NewGuid().ToString('N'))"
|
||||
$publicKey = "c17h-pub-$($spec.key)-$([guid]::NewGuid().ToString('N'))"
|
||||
$joinRequest = Invoke-Api -Method Post -Path "/clusters/$clusterID/join-requests" -Body @{
|
||||
join_token = $joinToken.join_token.token
|
||||
node_name = $spec.name
|
||||
node_fingerprint = $fingerprint
|
||||
public_key = $publicKey
|
||||
reported_capabilities = @{
|
||||
can_accept_node_ingress = $true
|
||||
can_route_mesh = $true
|
||||
testing_node = $true
|
||||
}
|
||||
reported_facts = @{
|
||||
os = "linux"
|
||||
runtime = "docker-test"
|
||||
stage = "c17h"
|
||||
run_id = $runId
|
||||
}
|
||||
requested_roles = $spec.roles
|
||||
}
|
||||
$approved = Invoke-Api -Method Post -Path "/clusters/$clusterID/join-requests/$($joinRequest.join_request.id)/approve" -Body @{
|
||||
actor_user_id = $actorUserID
|
||||
node_key = $fingerprint
|
||||
ownership_type = "platform_managed"
|
||||
owner_organization_id = $null
|
||||
}
|
||||
$nodeID = $approved.node_bootstrap.node_id
|
||||
foreach ($role in $spec.roles) {
|
||||
Invoke-Api -Method Post -Path "/clusters/$clusterID/nodes/$nodeID/roles" -Body @{
|
||||
actor_user_id = $actorUserID
|
||||
role = $role
|
||||
status = "active"
|
||||
policy = @{
|
||||
stage = "c17h"
|
||||
run_id = $runId
|
||||
synthetic_only = $true
|
||||
}
|
||||
} | Out-Null
|
||||
}
|
||||
$nodes[$spec.key] = [pscustomobject]@{
|
||||
id = $nodeID
|
||||
name = $spec.name
|
||||
fingerprint = $fingerprint
|
||||
public_key = $publicKey
|
||||
port = $spec.port
|
||||
roles = $spec.roles
|
||||
}
|
||||
}
|
||||
|
||||
$nodeAID = $nodes["a"].id
|
||||
$nodeRID = $nodes["r"].id
|
||||
$nodeBID = $nodes["b"].id
|
||||
$nodeCID = $nodes["c"].id
|
||||
$routeExpiresAt = (Get-Date).ToUniversalTime().AddHours(2).ToString("o")
|
||||
|
||||
$peerEndpointsDirect = @{}
|
||||
$peerEndpointsDirect[$nodeAID] = "http://127.0.0.1:$($nodes["a"].port)"
|
||||
$peerEndpointsDirect[$nodeBID] = "http://127.0.0.1:$($nodes["b"].port)"
|
||||
|
||||
$peerEndpointsRelay = @{}
|
||||
$peerEndpointsRelay[$nodeAID] = "http://127.0.0.1:$($nodes["a"].port)"
|
||||
$peerEndpointsRelay[$nodeRID] = "http://127.0.0.1:$($nodes["r"].port)"
|
||||
$peerEndpointsRelay[$nodeCID] = "http://127.0.0.1:$($nodes["c"].port)"
|
||||
|
||||
$directIntent = Invoke-Api -Method Post -Path "/clusters/$clusterID/mesh/route-intents" -Body @{
|
||||
actor_user_id = $actorUserID
|
||||
source_selector = @{ node_id = $nodeAID }
|
||||
destination_selector = @{ node_id = $nodeBID }
|
||||
service_class = "control"
|
||||
priority = 10
|
||||
policy = @{
|
||||
synthetic_enabled = $true
|
||||
peer_endpoints = $peerEndpointsDirect
|
||||
hops = @($nodeAID, $nodeBID)
|
||||
allowed_channels = @("fabric_control", "route_control")
|
||||
max_ttl = 4
|
||||
max_hops = 4
|
||||
expires_at = $routeExpiresAt
|
||||
route_version = "$runId-direct"
|
||||
policy_version = "$runId-policy"
|
||||
peer_directory_version = "$runId-peers"
|
||||
production_forwarding = $false
|
||||
}
|
||||
}
|
||||
|
||||
$relayIntent = Invoke-Api -Method Post -Path "/clusters/$clusterID/mesh/route-intents" -Body @{
|
||||
actor_user_id = $actorUserID
|
||||
source_selector = @{ node_id = $nodeAID }
|
||||
destination_selector = @{ node_id = $nodeCID }
|
||||
service_class = "control"
|
||||
priority = 20
|
||||
policy = @{
|
||||
synthetic_enabled = $true
|
||||
peer_endpoints = $peerEndpointsRelay
|
||||
hops = @($nodeAID, $nodeRID, $nodeCID)
|
||||
allowed_channels = @("fabric_control", "route_control")
|
||||
max_ttl = 6
|
||||
max_hops = 6
|
||||
expires_at = $routeExpiresAt
|
||||
route_version = "$runId-relay"
|
||||
policy_version = "$runId-policy"
|
||||
peer_directory_version = "$runId-peers"
|
||||
production_forwarding = $false
|
||||
}
|
||||
}
|
||||
|
||||
$configs = @{}
|
||||
foreach ($key in @("a", "r", "b", "c", "idle")) {
|
||||
$configs[$key] = Invoke-Api -Method Get -Path "/clusters/$clusterID/nodes/$($nodes[$key].id)/mesh/synthetic-config"
|
||||
}
|
||||
|
||||
Remove-NodeContainers
|
||||
|
||||
foreach ($key in @("a", "r", "b", "c", "idle")) {
|
||||
$node = $nodes[$key]
|
||||
$containerName = "$nodePrefix$key"
|
||||
$remoteStateDir = "/tmp/$runId-$key"
|
||||
Invoke-RemoteShell -Command "rm -rf '$remoteStateDir' && mkdir -p '$remoteStateDir'"
|
||||
$identity = @{
|
||||
node_id = $node.id
|
||||
cluster_id = $clusterID
|
||||
node_name = $node.name
|
||||
node_fingerprint = $node.fingerprint
|
||||
public_key = $node.public_key
|
||||
identity_status = "active"
|
||||
created_at = (Get-Date).ToUniversalTime().ToString("o")
|
||||
updated_at = (Get-Date).ToUniversalTime().ToString("o")
|
||||
} | ConvertTo-Json -Depth 10
|
||||
Send-RemoteFile -RemotePath "$remoteStateDir/identity.json" -Content $identity
|
||||
Invoke-RemoteDocker -Arguments @(
|
||||
"create",
|
||||
"--name", $containerName,
|
||||
"--network", "host",
|
||||
"-e", "RAP_BACKEND_URL=$backendContainerBaseUrl",
|
||||
"-e", "RAP_NODE_STATE_DIR=/tmp/state",
|
||||
"-e", "RAP_HEARTBEAT_INTERVAL_SECONDS=5",
|
||||
"-e", "RAP_MESH_SYNTHETIC_RUNTIME_ENABLED=true",
|
||||
"-e", "RAP_MESH_LISTEN_ADDR=0.0.0.0:$($node.port)",
|
||||
$NodeAgentImageTag,
|
||||
"-backend-url", $backendContainerBaseUrl,
|
||||
"-state-dir", "/tmp/state",
|
||||
"-heartbeat-interval", "5s",
|
||||
"-mesh-synthetic-runtime-enabled",
|
||||
"-mesh-listen-addr", "0.0.0.0:$($node.port)"
|
||||
) | Out-Null
|
||||
Invoke-RemoteDocker -Arguments @("cp", "$remoteStateDir/.", "$containerName`:/tmp/state")
|
||||
Invoke-RemoteDocker -Arguments @("start", $containerName) | Out-Null
|
||||
}
|
||||
|
||||
Start-Sleep -Seconds 25
|
||||
|
||||
$links = Invoke-Api -Method Get -Path "/clusters/$clusterID/mesh/links?actor_user_id=$actorUserID"
|
||||
$summary = Invoke-Api -Method Get -Path "/cluster-admin-summaries?actor_user_id=$actorUserID"
|
||||
|
||||
$nodeLogs = @{}
|
||||
foreach ($key in @("a", "r", "b", "c", "idle")) {
|
||||
$nodeLogs[$key] = Invoke-RemoteDockerText -Arguments @("logs", "$nodePrefix$key")
|
||||
}
|
||||
$backendLogs = Invoke-RemoteDockerText -Arguments @("logs", "--tail", "50", "rap_c17h_backend")
|
||||
|
||||
$meshLinks = @($links.mesh_links)
|
||||
$routeHealthLinks = @($meshLinks | Where-Object {
|
||||
$_.metadata.observation_type -eq "synthetic_route_health" -and
|
||||
$_.metadata.config_source -eq "control_plane"
|
||||
})
|
||||
$directHealth = @($routeHealthLinks | Where-Object { $_.metadata.route_id -eq $directIntent.route_intent.id -and $_.link_status -eq "reachable" })
|
||||
$relayHealth = @($routeHealthLinks | Where-Object { $_.metadata.route_id -eq $relayIntent.route_intent.id -and $_.link_status -eq "reachable" })
|
||||
|
||||
$passMatrix = [ordered]@{
|
||||
backend_ready = $true
|
||||
platform_owner_login = [bool]$actorUserID
|
||||
cluster_created = [bool]$clusterID
|
||||
fabric_testing_flags_enabled = $true
|
||||
node_a_scoped_config_enabled = $configs["a"].synthetic_mesh_config.enabled -eq $true
|
||||
node_a_has_direct_and_relay_routes = @($configs["a"].synthetic_mesh_config.routes).Count -eq 2
|
||||
relay_node_has_only_relay_route = @($configs["r"].synthetic_mesh_config.routes).Count -eq 1
|
||||
direct_destination_node_has_only_direct_route = @($configs["b"].synthetic_mesh_config.routes).Count -eq 1
|
||||
relay_destination_node_has_only_relay_route = @($configs["c"].synthetic_mesh_config.routes).Count -eq 1
|
||||
idle_node_has_no_routes = @($configs["idle"].synthetic_mesh_config.routes).Count -eq 0
|
||||
control_plane_config_used = (@("a", "r", "b", "c", "idle") | Where-Object { ($nodeLogs[$_] -join "`n") -match "source=control_plane" }).Count -eq 5
|
||||
direct_route_health_reported = $directHealth.Count -gt 0
|
||||
relay_route_health_reported = $relayHealth.Count -gt 0
|
||||
production_forwarding_disabled = (
|
||||
$configs["a"].synthetic_mesh_config.production_forwarding -eq $false -and
|
||||
$configs["r"].synthetic_mesh_config.production_forwarding -eq $false -and
|
||||
$configs["b"].synthetic_mesh_config.production_forwarding -eq $false -and
|
||||
$configs["c"].synthetic_mesh_config.production_forwarding -eq $false -and
|
||||
$configs["idle"].synthetic_mesh_config.production_forwarding -eq $false
|
||||
)
|
||||
}
|
||||
|
||||
$result = [pscustomobject]@{
|
||||
stage = "C17H deployed multi-agent synthetic config smoke"
|
||||
run_id = $runId
|
||||
backend_base_url = $backendPublicBaseUrl
|
||||
cluster_id = $clusterID
|
||||
node_ids = @{
|
||||
a = $nodes["a"].id
|
||||
r = $nodes["r"].id
|
||||
b = $nodes["b"].id
|
||||
c = $nodes["c"].id
|
||||
idle = $nodes["idle"].id
|
||||
}
|
||||
route_intents = @{
|
||||
direct = $directIntent.route_intent.id
|
||||
relay = $relayIntent.route_intent.id
|
||||
}
|
||||
scoped_config_route_counts = @{
|
||||
a = @($configs["a"].synthetic_mesh_config.routes).Count
|
||||
r = @($configs["r"].synthetic_mesh_config.routes).Count
|
||||
b = @($configs["b"].synthetic_mesh_config.routes).Count
|
||||
c = @($configs["c"].synthetic_mesh_config.routes).Count
|
||||
idle = @($configs["idle"].synthetic_mesh_config.routes).Count
|
||||
}
|
||||
mesh_link_count = $meshLinks.Count
|
||||
route_health_count = $routeHealthLinks.Count
|
||||
pass_matrix = $passMatrix
|
||||
direct_route_health = $directHealth | Select-Object -First 3
|
||||
relay_route_health = $relayHealth | Select-Object -First 3
|
||||
cluster_summaries = $summary.cluster_summaries
|
||||
backend_log_tail = $backendLogs
|
||||
node_log_tail = $nodeLogs
|
||||
containers_left_running = [bool]$KeepRunning
|
||||
}
|
||||
|
||||
$failed = @($passMatrix.GetEnumerator() | Where-Object { -not $_.Value })
|
||||
$result | ConvertTo-Json -Depth 50
|
||||
|
||||
if ($failed.Count -gt 0) {
|
||||
throw "C17H smoke failed: $($failed.Name -join ', ')"
|
||||
}
|
||||
|
||||
if (-not $KeepRunning) {
|
||||
Remove-NodeContainers
|
||||
}
|
||||
@@ -1,460 +0,0 @@
|
||||
param(
|
||||
[string]$DockerContext = "test-ubuntu",
|
||||
[string]$BackendImageTag = "rap-backend:c17h-synthetic-smoke",
|
||||
[string]$NodeAgentImageTag = "rap-node-agent:c17h-synthetic-smoke",
|
||||
[string]$AdminEmail = "fabric-owner-c17h@example.local",
|
||||
[string]$AdminPassword = "SmokePass!123",
|
||||
[int]$ApiPort = 18080,
|
||||
[int]$PostgresPort = 15432,
|
||||
[int]$RedisPort = 16379,
|
||||
[int]$MeshBasePort = 19081,
|
||||
[switch]$KeepRunning
|
||||
)
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
$repoRoot = (Resolve-Path (Join-Path $PSScriptRoot "..\..")).ProviderPath
|
||||
$backendBaseUrl = "http://127.0.0.1:$ApiPort/api/v1"
|
||||
$backendPublicBaseUrl = "http://192.168.200.61:$ApiPort/api/v1"
|
||||
$runId = "c17h-" + (Get-Date -Format "yyyyMMdd-HHmmss")
|
||||
|
||||
$postgresName = "rap_c17h_postgres"
|
||||
$redisName = "rap_c17h_redis"
|
||||
$backendName = "rap_c17h_backend"
|
||||
$nodePrefix = "rap_c17h_node_"
|
||||
|
||||
function Invoke-Docker {
|
||||
param([string[]]$Arguments)
|
||||
docker --context $DockerContext @Arguments
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "docker --context $DockerContext $($Arguments -join ' ') failed with exit code $LASTEXITCODE"
|
||||
}
|
||||
}
|
||||
|
||||
function Invoke-DockerText {
|
||||
param([string[]]$Arguments)
|
||||
$output = docker --context $DockerContext @Arguments 2>&1 | ForEach-Object { $_.ToString() }
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "docker --context $DockerContext $($Arguments -join ' ') failed with exit code $LASTEXITCODE"
|
||||
}
|
||||
return $output
|
||||
}
|
||||
|
||||
function Invoke-Api {
|
||||
param(
|
||||
[string]$Method,
|
||||
[string]$Path,
|
||||
[object]$Body = $null
|
||||
)
|
||||
$uri = "$backendPublicBaseUrl$Path"
|
||||
if ($null -eq $Body) {
|
||||
return Invoke-RestMethod -Method $Method -Uri $uri -TimeoutSec 30
|
||||
}
|
||||
return Invoke-RestMethod -Method $Method -Uri $uri -ContentType "application/json" -Body ($Body | ConvertTo-Json -Depth 30) -TimeoutSec 30
|
||||
}
|
||||
|
||||
function Wait-HttpReady {
|
||||
param([string]$Url)
|
||||
for ($i = 0; $i -lt 60; $i++) {
|
||||
try {
|
||||
$response = Invoke-WebRequest -UseBasicParsing -Uri $Url -TimeoutSec 2
|
||||
if ($response.StatusCode -ge 200 -and $response.StatusCode -lt 300) {
|
||||
return
|
||||
}
|
||||
}
|
||||
catch {
|
||||
Start-Sleep -Seconds 1
|
||||
}
|
||||
}
|
||||
throw "Timed out waiting for $Url"
|
||||
}
|
||||
|
||||
function Invoke-PostgresSql {
|
||||
param([string]$Sql)
|
||||
$Sql | docker --context $DockerContext exec -i $postgresName psql -U rap_user -d remote_access_platform -v ON_ERROR_STOP=1 -f -
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "psql command failed"
|
||||
}
|
||||
}
|
||||
|
||||
function Remove-C17HContainers {
|
||||
$names = @($backendName, $postgresName, $redisName)
|
||||
foreach ($letter in @("a", "b", "c", "r", "idle")) {
|
||||
$names += "$nodePrefix$letter"
|
||||
}
|
||||
foreach ($name in $names) {
|
||||
docker --context $DockerContext rm -f $name 2>$null | Out-Null
|
||||
}
|
||||
}
|
||||
|
||||
Write-Host "C17H smoke run: $runId"
|
||||
Write-Host "Using Docker context: $DockerContext"
|
||||
|
||||
Remove-C17HContainers
|
||||
|
||||
Write-Host "Building backend image..."
|
||||
Invoke-Docker -Arguments @("build", "-f", "$repoRoot\backend\Dockerfile", "-t", $BackendImageTag, "$repoRoot\backend")
|
||||
|
||||
Write-Host "Building node-agent image..."
|
||||
Invoke-Docker -Arguments @("build", "-f", "$repoRoot\agents\rap-node-agent\Dockerfile", "-t", $NodeAgentImageTag, "$repoRoot")
|
||||
|
||||
Write-Host "Starting PostgreSQL and Redis..."
|
||||
Invoke-Docker -Arguments @(
|
||||
"run", "-d",
|
||||
"--name", $postgresName,
|
||||
"-e", "POSTGRES_DB=remote_access_platform",
|
||||
"-e", "POSTGRES_USER=rap_user",
|
||||
"-e", "POSTGRES_PASSWORD=rap_password",
|
||||
"-p", "$PostgresPort`:5432",
|
||||
"postgres:16"
|
||||
)
|
||||
Invoke-Docker -Arguments @(
|
||||
"run", "-d",
|
||||
"--name", $redisName,
|
||||
"-p", "$RedisPort`:6379",
|
||||
"redis:7"
|
||||
)
|
||||
|
||||
for ($i = 0; $i -lt 60; $i++) {
|
||||
docker --context $DockerContext exec $postgresName pg_isready -U rap_user -d remote_access_platform | Out-Null
|
||||
if ($LASTEXITCODE -eq 0) {
|
||||
break
|
||||
}
|
||||
Start-Sleep -Seconds 1
|
||||
}
|
||||
docker --context $DockerContext exec $postgresName pg_isready -U rap_user -d remote_access_platform | Out-Null
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "PostgreSQL did not become ready"
|
||||
}
|
||||
|
||||
Write-Host "Applying migrations..."
|
||||
$migrationFiles = Get-ChildItem -Path (Join-Path $repoRoot "backend\migrations") -Filter "*.up.sql" | Sort-Object Name
|
||||
foreach ($migration in $migrationFiles) {
|
||||
Write-Host "Applying $($migration.Name)"
|
||||
Get-Content -Raw $migration.FullName | docker --context $DockerContext exec -i $postgresName psql -U rap_user -d remote_access_platform -v ON_ERROR_STOP=1 -f -
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "Migration failed: $($migration.Name)"
|
||||
}
|
||||
}
|
||||
|
||||
Write-Host "Seeding platform owner..."
|
||||
$adminUserID = [guid]::NewGuid().ToString()
|
||||
$adminHash = '$2a$10$AqLRexkI1yXbuiMPU6dHM.KVUhF.t..9NolyK4OOodQTyTsHyG.7u'
|
||||
Invoke-PostgresSql -Sql @"
|
||||
INSERT INTO users (id, email, password_hash, mfa_enabled, platform_role)
|
||||
VALUES ('$adminUserID'::uuid, '$AdminEmail', '$adminHash', FALSE, 'platform_admin')
|
||||
ON CONFLICT (email) DO UPDATE SET
|
||||
password_hash = EXCLUDED.password_hash,
|
||||
platform_role = 'platform_admin',
|
||||
updated_at = NOW();
|
||||
"@
|
||||
|
||||
Write-Host "Starting backend..."
|
||||
Invoke-Docker -Arguments @(
|
||||
"run", "-d",
|
||||
"--name", $backendName,
|
||||
"--network", "host",
|
||||
"-e", "APP_NAME=rap-api",
|
||||
"-e", "APP_ENV=c17h-smoke",
|
||||
"-e", "HTTP_HOST=0.0.0.0",
|
||||
"-e", "HTTP_PORT=$ApiPort",
|
||||
"-e", "POSTGRES_DSN=postgres://rap_user:rap_password@127.0.0.1:$PostgresPort/remote_access_platform?sslmode=disable",
|
||||
"-e", "REDIS_ADDR=127.0.0.1:$RedisPort",
|
||||
"-e", "AUTH_ACCESS_TOKEN_SECRET=c17h-access-secret",
|
||||
"-e", "AUTH_REFRESH_HASH_SECRET=c17h-refresh-secret",
|
||||
$BackendImageTag
|
||||
)
|
||||
Wait-HttpReady -Url "http://192.168.200.61:$ApiPort/readyz"
|
||||
|
||||
Write-Host "Logging in as platform owner..."
|
||||
$login = Invoke-Api -Method Post -Path "/auth/login" -Body @{
|
||||
email = $AdminEmail
|
||||
password = $AdminPassword
|
||||
device_fingerprint = "c17h-smoke-device"
|
||||
device_label = "C17H synthetic smoke"
|
||||
trust_device = $true
|
||||
}
|
||||
$actorUserID = $login.user.id
|
||||
|
||||
Write-Host "Creating C17H cluster..."
|
||||
$cluster = Invoke-Api -Method Post -Path "/clusters/" -Body @{
|
||||
actor_user_id = $actorUserID
|
||||
slug = "c17h-$((New-Guid).Guid.Substring(0, 8))"
|
||||
name = "C17H Synthetic Mesh Smoke"
|
||||
region = "test-docker"
|
||||
metadata = @{
|
||||
stage = "c17h"
|
||||
production_forwarding = $false
|
||||
created_by = "c17h-multi-agent-synthetic-smoke.ps1"
|
||||
}
|
||||
}
|
||||
$clusterID = $cluster.cluster.id
|
||||
|
||||
Write-Host "Enabling test-only Fabric flags..."
|
||||
Invoke-Api -Method Put -Path "/fabric/testing-flags" -Body @{
|
||||
actor_user_id = $actorUserID
|
||||
scope_type = "platform"
|
||||
scope_id = $null
|
||||
cluster_id = $null
|
||||
enabled = $true
|
||||
telemetry_enabled = $true
|
||||
synthetic_links_enabled = $true
|
||||
history_retention_hours = 24
|
||||
metadata = @{
|
||||
stage = "c17h"
|
||||
production_forwarding = $false
|
||||
service_workload_traffic = $false
|
||||
}
|
||||
} | Out-Null
|
||||
|
||||
$joinToken = Invoke-Api -Method Post -Path "/clusters/$clusterID/join-tokens" -Body @{
|
||||
actor_user_id = $actorUserID
|
||||
scope = @{ purpose = "c17h-synthetic-smoke"; roles = @("core-mesh", "relay-node") }
|
||||
expires_at = (Get-Date).ToUniversalTime().AddHours(2).ToString("o")
|
||||
max_uses = 5
|
||||
}
|
||||
|
||||
$nodeSpecs = @(
|
||||
@{ key = "a"; name = "c17h-node-a"; roles = @("core-mesh"); port = $MeshBasePort },
|
||||
@{ key = "r"; name = "c17h-node-r"; roles = @("core-mesh", "relay-node"); port = ($MeshBasePort + 1) },
|
||||
@{ key = "b"; name = "c17h-node-b"; roles = @("core-mesh"); port = ($MeshBasePort + 2) },
|
||||
@{ key = "c"; name = "c17h-node-c"; roles = @("core-mesh"); port = ($MeshBasePort + 3) },
|
||||
@{ key = "idle"; name = "c17h-node-idle"; roles = @("core-mesh"); port = ($MeshBasePort + 4) }
|
||||
)
|
||||
|
||||
$nodes = @{}
|
||||
foreach ($spec in $nodeSpecs) {
|
||||
$fingerprint = "c17h-fp-$($spec.key)-$([guid]::NewGuid().ToString('N'))"
|
||||
$publicKey = "c17h-pub-$($spec.key)-$([guid]::NewGuid().ToString('N'))"
|
||||
$joinRequest = Invoke-Api -Method Post -Path "/clusters/$clusterID/join-requests" -Body @{
|
||||
join_token = $joinToken.join_token.token
|
||||
node_name = $spec.name
|
||||
node_fingerprint = $fingerprint
|
||||
public_key = $publicKey
|
||||
reported_capabilities = @{
|
||||
can_accept_node_ingress = $true
|
||||
can_route_mesh = $true
|
||||
testing_node = $true
|
||||
}
|
||||
reported_facts = @{
|
||||
os = "linux"
|
||||
runtime = "docker-test"
|
||||
stage = "c17h"
|
||||
}
|
||||
requested_roles = $spec.roles
|
||||
}
|
||||
$approved = Invoke-Api -Method Post -Path "/clusters/$clusterID/join-requests/$($joinRequest.join_request.id)/approve" -Body @{
|
||||
actor_user_id = $actorUserID
|
||||
node_key = $fingerprint
|
||||
ownership_type = "platform_managed"
|
||||
owner_organization_id = $null
|
||||
}
|
||||
$nodeID = $approved.node_bootstrap.node_id
|
||||
foreach ($role in $spec.roles) {
|
||||
Invoke-Api -Method Post -Path "/clusters/$clusterID/nodes/$nodeID/roles" -Body @{
|
||||
actor_user_id = $actorUserID
|
||||
role = $role
|
||||
status = "active"
|
||||
policy = @{
|
||||
stage = "c17h"
|
||||
synthetic_only = $true
|
||||
}
|
||||
} | Out-Null
|
||||
}
|
||||
$nodes[$spec.key] = [pscustomobject]@{
|
||||
id = $nodeID
|
||||
name = $spec.name
|
||||
fingerprint = $fingerprint
|
||||
public_key = $publicKey
|
||||
port = $spec.port
|
||||
roles = $spec.roles
|
||||
}
|
||||
}
|
||||
|
||||
$nodeAID = $nodes["a"].id
|
||||
$nodeRID = $nodes["r"].id
|
||||
$nodeBID = $nodes["b"].id
|
||||
$nodeCID = $nodes["c"].id
|
||||
|
||||
$peerEndpointsDirect = @{}
|
||||
$peerEndpointsDirect[$nodeAID] = "http://127.0.0.1:$($nodes["a"].port)"
|
||||
$peerEndpointsDirect[$nodeBID] = "http://127.0.0.1:$($nodes["b"].port)"
|
||||
|
||||
$peerEndpointsRelay = @{}
|
||||
$peerEndpointsRelay[$nodeAID] = "http://127.0.0.1:$($nodes["a"].port)"
|
||||
$peerEndpointsRelay[$nodeRID] = "http://127.0.0.1:$($nodes["r"].port)"
|
||||
$peerEndpointsRelay[$nodeCID] = "http://127.0.0.1:$($nodes["c"].port)"
|
||||
|
||||
Write-Host "Creating synthetic route intents..."
|
||||
$directIntent = Invoke-Api -Method Post -Path "/clusters/$clusterID/mesh/route-intents" -Body @{
|
||||
actor_user_id = $actorUserID
|
||||
source_selector = @{ node_id = $nodeAID }
|
||||
destination_selector = @{ node_id = $nodeBID }
|
||||
service_class = "control"
|
||||
priority = 10
|
||||
policy = @{
|
||||
synthetic_enabled = $true
|
||||
peer_endpoints = $peerEndpointsDirect
|
||||
hops = @($nodeAID, $nodeBID)
|
||||
allowed_channels = @("fabric_control", "route_control")
|
||||
max_ttl = 4
|
||||
max_hops = 4
|
||||
route_version = "$runId-direct"
|
||||
policy_version = "$runId-policy"
|
||||
peer_directory_version = "$runId-peers"
|
||||
production_forwarding = $false
|
||||
}
|
||||
}
|
||||
$relayIntent = Invoke-Api -Method Post -Path "/clusters/$clusterID/mesh/route-intents" -Body @{
|
||||
actor_user_id = $actorUserID
|
||||
source_selector = @{ node_id = $nodeAID }
|
||||
destination_selector = @{ node_id = $nodeCID }
|
||||
service_class = "control"
|
||||
priority = 20
|
||||
policy = @{
|
||||
synthetic_enabled = $true
|
||||
peer_endpoints = $peerEndpointsRelay
|
||||
hops = @($nodeAID, $nodeRID, $nodeCID)
|
||||
allowed_channels = @("fabric_control", "route_control")
|
||||
max_ttl = 6
|
||||
max_hops = 6
|
||||
route_version = "$runId-relay"
|
||||
policy_version = "$runId-policy"
|
||||
peer_directory_version = "$runId-peers"
|
||||
production_forwarding = $false
|
||||
}
|
||||
}
|
||||
|
||||
$configs = @{}
|
||||
foreach ($key in @("a", "r", "b", "c", "idle")) {
|
||||
$configs[$key] = Invoke-Api -Method Get -Path "/clusters/$clusterID/nodes/$($nodes[$key].id)/mesh/synthetic-config"
|
||||
}
|
||||
|
||||
Write-Host "Starting node-agent containers..."
|
||||
foreach ($key in @("a", "r", "b", "c", "idle")) {
|
||||
$node = $nodes[$key]
|
||||
$containerName = "$nodePrefix$key"
|
||||
docker --context $DockerContext rm -f $containerName 2>$null | Out-Null
|
||||
$stateDir = Join-Path $env:TEMP "$runId-$key"
|
||||
New-Item -ItemType Directory -Force -Path $stateDir | Out-Null
|
||||
@{
|
||||
node_id = $node.id
|
||||
cluster_id = $clusterID
|
||||
node_name = $node.name
|
||||
node_fingerprint = $node.fingerprint
|
||||
public_key = $node.public_key
|
||||
identity_status = "active"
|
||||
created_at = (Get-Date).ToUniversalTime().ToString("o")
|
||||
updated_at = (Get-Date).ToUniversalTime().ToString("o")
|
||||
} | ConvertTo-Json -Depth 10 | Set-Content -Encoding UTF8 -Path (Join-Path $stateDir "identity.json")
|
||||
|
||||
Invoke-Docker -Arguments @(
|
||||
"create",
|
||||
"--name", $containerName,
|
||||
"--network", "host",
|
||||
"-e", "RAP_BACKEND_URL=$backendBaseUrl",
|
||||
"-e", "RAP_NODE_STATE_DIR=/tmp/state",
|
||||
"-e", "RAP_HEARTBEAT_INTERVAL_SECONDS=5",
|
||||
"-e", "RAP_MESH_SYNTHETIC_RUNTIME_ENABLED=true",
|
||||
"-e", "RAP_MESH_LISTEN_ADDR=0.0.0.0:$($node.port)",
|
||||
$NodeAgentImageTag,
|
||||
"-backend-url", $backendBaseUrl,
|
||||
"-state-dir", "/tmp/state",
|
||||
"-heartbeat-interval", "5s",
|
||||
"-mesh-synthetic-runtime-enabled",
|
||||
"-mesh-listen-addr", "0.0.0.0:$($node.port)"
|
||||
) | Out-Null
|
||||
Invoke-Docker -Arguments @("cp", "$stateDir\.", "$containerName`:/tmp/state")
|
||||
Invoke-Docker -Arguments @("start", $containerName) | Out-Null
|
||||
}
|
||||
|
||||
Write-Host "Waiting for synthetic observations..."
|
||||
Start-Sleep -Seconds 25
|
||||
|
||||
$links = Invoke-Api -Method Get -Path "/clusters/$clusterID/mesh/links?actor_user_id=$actorUserID"
|
||||
$summary = Invoke-Api -Method Get -Path "/cluster-admin-summaries?actor_user_id=$actorUserID"
|
||||
|
||||
$nodeLogs = @{}
|
||||
foreach ($key in @("a", "r", "b", "c", "idle")) {
|
||||
$nodeLogs[$key] = Invoke-DockerText -Arguments @("logs", "--tail", "30", "$nodePrefix$key")
|
||||
}
|
||||
$backendLogs = Invoke-DockerText -Arguments @("logs", "--tail", "30", $backendName)
|
||||
|
||||
$meshLinks = @($links.mesh_links)
|
||||
$routeHealthLinks = @($meshLinks | Where-Object {
|
||||
$_.metadata.observation_type -eq "synthetic_route_health" -and
|
||||
$_.metadata.config_source -eq "control_plane"
|
||||
})
|
||||
$directHealth = @($routeHealthLinks | Where-Object { $_.metadata.route_id -eq $directIntent.route_intent.id -and $_.link_status -eq "reachable" })
|
||||
$relayHealth = @($routeHealthLinks | Where-Object { $_.metadata.route_id -eq $relayIntent.route_intent.id -and $_.link_status -eq "reachable" })
|
||||
|
||||
$passMatrix = [ordered]@{
|
||||
backend_ready = $true
|
||||
platform_owner_login = [bool]$actorUserID
|
||||
cluster_created = [bool]$clusterID
|
||||
fabric_testing_flags_enabled = $true
|
||||
node_a_scoped_config_enabled = $configs["a"].synthetic_mesh_config.enabled -eq $true
|
||||
node_a_has_direct_and_relay_routes = @($configs["a"].synthetic_mesh_config.routes).Count -eq 2
|
||||
relay_node_has_only_relay_route = @($configs["r"].synthetic_mesh_config.routes).Count -eq 1
|
||||
direct_destination_node_has_only_direct_route = @($configs["b"].synthetic_mesh_config.routes).Count -eq 1
|
||||
relay_destination_node_has_only_relay_route = @($configs["c"].synthetic_mesh_config.routes).Count -eq 1
|
||||
idle_node_has_no_routes = @($configs["idle"].synthetic_mesh_config.routes).Count -eq 0
|
||||
node_agents_started = @("a", "r", "b", "c", "idle").Count -eq 5
|
||||
control_plane_config_used = ($nodeLogs["a"] -join "`n") -match "control_plane"
|
||||
direct_route_health_reported = $directHealth.Count -gt 0
|
||||
relay_route_health_reported = $relayHealth.Count -gt 0
|
||||
production_forwarding_disabled = (
|
||||
$configs["a"].synthetic_mesh_config.production_forwarding -eq $false -and
|
||||
$configs["r"].synthetic_mesh_config.production_forwarding -eq $false -and
|
||||
$configs["b"].synthetic_mesh_config.production_forwarding -eq $false -and
|
||||
$configs["c"].synthetic_mesh_config.production_forwarding -eq $false -and
|
||||
$configs["idle"].synthetic_mesh_config.production_forwarding -eq $false
|
||||
)
|
||||
}
|
||||
|
||||
$failed = @($passMatrix.GetEnumerator() | Where-Object { -not $_.Value })
|
||||
$result = [pscustomobject]@{
|
||||
stage = "C17H deployed multi-agent synthetic config smoke"
|
||||
run_id = $runId
|
||||
backend_base_url = $backendPublicBaseUrl
|
||||
cluster_id = $clusterID
|
||||
node_ids = @{
|
||||
a = $nodes["a"].id
|
||||
r = $nodes["r"].id
|
||||
b = $nodes["b"].id
|
||||
c = $nodes["c"].id
|
||||
idle = $nodes["idle"].id
|
||||
}
|
||||
route_intents = @{
|
||||
direct = $directIntent.route_intent.id
|
||||
relay = $relayIntent.route_intent.id
|
||||
}
|
||||
scoped_config_route_counts = @{
|
||||
a = @($configs["a"].synthetic_mesh_config.routes).Count
|
||||
r = @($configs["r"].synthetic_mesh_config.routes).Count
|
||||
b = @($configs["b"].synthetic_mesh_config.routes).Count
|
||||
c = @($configs["c"].synthetic_mesh_config.routes).Count
|
||||
idle = @($configs["idle"].synthetic_mesh_config.routes).Count
|
||||
}
|
||||
mesh_link_count = $meshLinks.Count
|
||||
route_health_count = $routeHealthLinks.Count
|
||||
pass_matrix = $passMatrix
|
||||
direct_route_health = $directHealth | Select-Object -First 3
|
||||
relay_route_health = $relayHealth | Select-Object -First 3
|
||||
cluster_summaries = $summary.cluster_summaries
|
||||
backend_log_tail = $backendLogs
|
||||
node_log_tail = $nodeLogs
|
||||
containers_left_running = [bool]$KeepRunning
|
||||
}
|
||||
|
||||
$result | ConvertTo-Json -Depth 40
|
||||
|
||||
if ($failed.Count -gt 0) {
|
||||
throw "C17H smoke failed: $($failed.Name -join ', ')"
|
||||
}
|
||||
|
||||
if (-not $KeepRunning) {
|
||||
Write-Host "Cleaning up C17H containers..."
|
||||
Remove-C17HContainers
|
||||
}
|
||||
@@ -1,917 +0,0 @@
|
||||
param(
|
||||
[string]$DockerSshAlias = "test-docker",
|
||||
[string]$BackendImageTag = "rap-backend:c17z18-rendezvous-smoke",
|
||||
[string]$NodeAgentImageTag = "rap-node-agent:c17z18-rendezvous-smoke",
|
||||
[string]$AdminEmail = "fabric-owner-c17z18@example.local",
|
||||
[string]$AdminPassword = "SmokePass!123",
|
||||
[int]$ApiPort = 18120,
|
||||
[int]$PostgresPort = 15442,
|
||||
[int]$RedisPort = 16442,
|
||||
[int]$MeshBasePort = 19120,
|
||||
[switch]$KeepRunning
|
||||
)
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
$repoRoot = (Resolve-Path (Join-Path $PSScriptRoot "..\..")).ProviderPath
|
||||
$backendPublicBaseUrl = "http://192.168.200.61:$ApiPort/api/v1"
|
||||
$backendContainerBaseUrl = "http://127.0.0.1:$ApiPort/api/v1"
|
||||
$runId = "c17z18-" + (Get-Date -Format "yyyyMMdd-HHmmss")
|
||||
$remoteBuildDir = "/tmp/rap-c17z18-build-$runId"
|
||||
|
||||
$postgresName = "rap_c17z12_postgres"
|
||||
$redisName = "rap_c17z12_redis"
|
||||
$backendName = "rap_c17z12_backend"
|
||||
$nodePrefix = "rap_c17z12_node_"
|
||||
|
||||
function Invoke-RemoteDocker {
|
||||
param([string[]]$Arguments)
|
||||
& ssh $DockerSshAlias docker @Arguments
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "ssh $DockerSshAlias docker $($Arguments -join ' ') failed with exit code $LASTEXITCODE"
|
||||
}
|
||||
}
|
||||
|
||||
function Invoke-RemoteDockerText {
|
||||
param([string[]]$Arguments)
|
||||
$previousErrorActionPreference = $ErrorActionPreference
|
||||
$ErrorActionPreference = "Continue"
|
||||
try {
|
||||
$output = & ssh $DockerSshAlias docker @Arguments 2>&1 | ForEach-Object { $_.ToString() }
|
||||
}
|
||||
finally {
|
||||
$ErrorActionPreference = $previousErrorActionPreference
|
||||
}
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "ssh $DockerSshAlias docker $($Arguments -join ' ') failed with exit code $LASTEXITCODE"
|
||||
}
|
||||
return $output
|
||||
}
|
||||
|
||||
function Invoke-RemoteShell {
|
||||
param([string]$Command)
|
||||
& ssh $DockerSshAlias $Command
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "ssh $DockerSshAlias $Command failed with exit code $LASTEXITCODE"
|
||||
}
|
||||
}
|
||||
|
||||
function Invoke-RemotePostgresSql {
|
||||
param([string]$Sql)
|
||||
$Sql | & ssh $DockerSshAlias "docker exec -i $postgresName psql -U rap_user -d remote_access_platform -v ON_ERROR_STOP=1 -f -"
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "remote psql command failed"
|
||||
}
|
||||
}
|
||||
|
||||
function Send-RemoteFile {
|
||||
param(
|
||||
[string]$RemotePath,
|
||||
[string]$Content
|
||||
)
|
||||
$Content | & ssh $DockerSshAlias "cat > '$RemotePath'"
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "write remote file failed: $RemotePath"
|
||||
}
|
||||
}
|
||||
|
||||
function Send-RemoteBuildContext {
|
||||
Write-Host "Uploading backend and node-agent build context to $DockerSshAlias..."
|
||||
Invoke-RemoteShell -Command "rm -rf '$remoteBuildDir' && mkdir -p '$remoteBuildDir'"
|
||||
& tar -czf - -C $repoRoot "backend" "agents/rap-node-agent" | & ssh $DockerSshAlias "tar -xzf - -C '$remoteBuildDir'"
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "upload build context failed"
|
||||
}
|
||||
}
|
||||
|
||||
function Invoke-Api {
|
||||
param(
|
||||
[string]$Method,
|
||||
[string]$Path,
|
||||
[object]$Body = $null
|
||||
)
|
||||
$uri = "$backendPublicBaseUrl$Path"
|
||||
if ($null -eq $Body) {
|
||||
return Invoke-RestMethod -Method $Method -Uri $uri -TimeoutSec 30
|
||||
}
|
||||
return Invoke-RestMethod -Method $Method -Uri $uri -ContentType "application/json" -Body ($Body | ConvertTo-Json -Depth 50) -TimeoutSec 30
|
||||
}
|
||||
|
||||
function Wait-HttpReady {
|
||||
param([string]$Url)
|
||||
for ($i = 0; $i -lt 60; $i++) {
|
||||
try {
|
||||
$response = Invoke-WebRequest -UseBasicParsing -Uri $Url -TimeoutSec 2
|
||||
if ($response.StatusCode -ge 200 -and $response.StatusCode -lt 300) {
|
||||
return
|
||||
}
|
||||
}
|
||||
catch {
|
||||
Start-Sleep -Seconds 1
|
||||
}
|
||||
}
|
||||
throw "Timed out waiting for $Url"
|
||||
}
|
||||
|
||||
function Remove-C17Z12Containers {
|
||||
$names = @($backendName, $postgresName, $redisName)
|
||||
foreach ($key in @("a", "b", "c", "r", "idle")) {
|
||||
$names += "$nodePrefix$key"
|
||||
}
|
||||
foreach ($name in $names) {
|
||||
& ssh $DockerSshAlias docker rm -f $name 2>$null | Out-Null
|
||||
}
|
||||
}
|
||||
|
||||
function New-EndpointCandidate {
|
||||
param(
|
||||
[string]$EndpointID,
|
||||
[string]$NodeID,
|
||||
[string]$Address,
|
||||
[string]$Transport,
|
||||
[string]$Reachability,
|
||||
[string]$ConnectivityMode,
|
||||
[string]$NATType,
|
||||
[int]$Priority,
|
||||
[string[]]$PolicyTags = @()
|
||||
)
|
||||
return @{
|
||||
endpoint_id = $EndpointID
|
||||
node_id = $NodeID
|
||||
transport = $Transport
|
||||
address = $Address
|
||||
address_family = "ipv4"
|
||||
reachability = $Reachability
|
||||
nat_type = $NATType
|
||||
connectivity_mode = $ConnectivityMode
|
||||
region = "docker-test"
|
||||
priority = $Priority
|
||||
policy_tags = $PolicyTags
|
||||
last_verified_at = (Get-Date).ToUniversalTime().ToString("o")
|
||||
metadata = @{
|
||||
stage = "c17z18"
|
||||
run_id = $runId
|
||||
service_workload_traffic = $false
|
||||
production_payload_forwarding = $false
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function Get-OptionalProperty {
|
||||
param(
|
||||
[object]$Object,
|
||||
[string]$PropertyName
|
||||
)
|
||||
if ($null -eq $Object) {
|
||||
return $null
|
||||
}
|
||||
$property = $Object.PSObject.Properties[$PropertyName]
|
||||
if ($null -eq $property) {
|
||||
return $null
|
||||
}
|
||||
return $property.Value
|
||||
}
|
||||
|
||||
function Get-OptionalArrayCount {
|
||||
param(
|
||||
[object]$Object,
|
||||
[string]$PropertyName
|
||||
)
|
||||
$value = Get-OptionalProperty -Object $Object -PropertyName $PropertyName
|
||||
if ($null -eq $value) {
|
||||
return 0
|
||||
}
|
||||
return @($value).Count
|
||||
}
|
||||
|
||||
function Get-LatestHeartbeatMetadataReport {
|
||||
param(
|
||||
[string]$NodeID,
|
||||
[string]$PropertyName
|
||||
)
|
||||
$heartbeats = Invoke-Api -Method Get -Path "/clusters/$clusterID/nodes/$NodeID/heartbeats?actor_user_id=$actorUserID&limit=5"
|
||||
$latest = @($heartbeats.heartbeats) | Select-Object -First 1
|
||||
$metadata = Get-OptionalProperty -Object $latest -PropertyName "metadata"
|
||||
return Get-OptionalProperty -Object $metadata -PropertyName $PropertyName
|
||||
}
|
||||
|
||||
function Get-LatestRendezvousLeaseReport {
|
||||
param([string]$NodeID)
|
||||
return Get-LatestHeartbeatMetadataReport -NodeID $NodeID -PropertyName "mesh_rendezvous_lease_report"
|
||||
}
|
||||
|
||||
function Get-LatestRoutePathDecisionReport {
|
||||
param([string]$NodeID)
|
||||
return Get-LatestHeartbeatMetadataReport -NodeID $NodeID -PropertyName "mesh_route_path_decision_report"
|
||||
}
|
||||
|
||||
function Get-LatestRouteGenerationReport {
|
||||
param([string]$NodeID)
|
||||
return Get-LatestHeartbeatMetadataReport -NodeID $NodeID -PropertyName "mesh_route_generation_report"
|
||||
}
|
||||
|
||||
function Get-LatestRouteHealthConfigReport {
|
||||
param([string]$NodeID)
|
||||
return Get-LatestHeartbeatMetadataReport -NodeID $NodeID -PropertyName "mesh_route_health_config_report"
|
||||
}
|
||||
|
||||
function Get-LatestRouteHealthFeedbackRefreshReport {
|
||||
param([string]$NodeID)
|
||||
return Get-LatestHeartbeatMetadataReport -NodeID $NodeID -PropertyName "mesh_route_health_feedback_refresh_report"
|
||||
}
|
||||
|
||||
function Select-C17Z18RouteHealthSnapshot {
|
||||
param([object[]]$MeshLinks)
|
||||
$routeHealthLinks = @($MeshLinks | Where-Object {
|
||||
$_.metadata.observation_type -eq "synthetic_route_health" -and
|
||||
$_.metadata.config_source -eq "control_plane"
|
||||
})
|
||||
$directHealth = @($routeHealthLinks | Where-Object {
|
||||
$_.metadata.route_id -eq $directIntent.route_intent.id -and
|
||||
$_.link_status -eq "reachable"
|
||||
})
|
||||
$rendezvousHealth = @($routeHealthLinks | Where-Object {
|
||||
$_.source_node_id -eq $nodeAID -and
|
||||
$_.metadata.route_id -eq $rendezvousIntent.route_intent.id -and
|
||||
$_.link_status -eq "reachable"
|
||||
})
|
||||
$replacementRouteHealth = @($rendezvousHealth | Where-Object {
|
||||
$_.metadata.route_path_decision_applied -eq $true -and
|
||||
$_.metadata.route_path_decision_selected_relay_id -eq $nodeSID -and
|
||||
(@($_.metadata.expected_effective_hops) -contains $nodeSID) -and
|
||||
-not (@($_.metadata.expected_effective_hops) -contains $nodeRID) -and
|
||||
(@($_.metadata.observed_ack_path) -contains $nodeSID) -and
|
||||
-not (@($_.metadata.observed_ack_path) -contains $nodeRID) -and
|
||||
$_.metadata.route_path_drift_detected -eq $false
|
||||
})
|
||||
return [pscustomobject]@{
|
||||
route_health_links = $routeHealthLinks
|
||||
direct_health = $directHealth
|
||||
rendezvous_health = $rendezvousHealth
|
||||
replacement_route_health = $replacementRouteHealth
|
||||
}
|
||||
}
|
||||
|
||||
function Get-C17Z18MeshLinkSnapshot {
|
||||
$links = Invoke-Api -Method Get -Path "/clusters/$clusterID/mesh/links?actor_user_id=$actorUserID"
|
||||
$meshLinks = @($links.mesh_links)
|
||||
$routeHealth = Select-C17Z18RouteHealthSnapshot -MeshLinks $meshLinks
|
||||
return [pscustomobject]@{
|
||||
links = $links
|
||||
mesh_links = $meshLinks
|
||||
route_health_links = @($routeHealth.route_health_links)
|
||||
direct_health = @($routeHealth.direct_health)
|
||||
rendezvous_health = @($routeHealth.rendezvous_health)
|
||||
replacement_route_health = @($routeHealth.replacement_route_health)
|
||||
}
|
||||
}
|
||||
|
||||
function Wait-C17Z18ReplacementRouteHealthSnapshot {
|
||||
param([int]$TimeoutSeconds = 40)
|
||||
$deadline = (Get-Date).AddSeconds($TimeoutSeconds)
|
||||
$latest = $null
|
||||
do {
|
||||
$latest = Get-C17Z18MeshLinkSnapshot
|
||||
if (@($latest.replacement_route_health).Count -gt 0) {
|
||||
return $latest
|
||||
}
|
||||
Start-Sleep -Seconds 2
|
||||
} while ((Get-Date) -lt $deadline)
|
||||
return $latest
|
||||
}
|
||||
|
||||
Write-Host "C17Z18 rendezvous relay replacement smoke run: $runId"
|
||||
Write-Host "Using SSH Docker host: $DockerSshAlias"
|
||||
|
||||
Remove-C17Z12Containers
|
||||
Send-RemoteBuildContext
|
||||
|
||||
Write-Host "Building backend image on docker-test..."
|
||||
Invoke-RemoteDocker -Arguments @("build", "-f", "$remoteBuildDir/backend/Dockerfile", "-t", $BackendImageTag, "$remoteBuildDir/backend")
|
||||
|
||||
Write-Host "Building node-agent image on docker-test..."
|
||||
Invoke-RemoteDocker -Arguments @("build", "-f", "$remoteBuildDir/agents/rap-node-agent/Dockerfile", "-t", $NodeAgentImageTag, $remoteBuildDir)
|
||||
|
||||
Write-Host "Starting PostgreSQL and Redis..."
|
||||
Invoke-RemoteDocker -Arguments @(
|
||||
"run", "-d",
|
||||
"--name", $postgresName,
|
||||
"-e", "POSTGRES_DB=remote_access_platform",
|
||||
"-e", "POSTGRES_USER=rap_user",
|
||||
"-e", "POSTGRES_PASSWORD=rap_password",
|
||||
"-p", "$PostgresPort`:5432",
|
||||
"postgres:16"
|
||||
)
|
||||
Invoke-RemoteDocker -Arguments @(
|
||||
"run", "-d",
|
||||
"--name", $redisName,
|
||||
"-p", "$RedisPort`:6379",
|
||||
"redis:7"
|
||||
)
|
||||
|
||||
Invoke-RemoteShell -Command "for i in `$(seq 1 60); do docker exec $postgresName pg_isready -U rap_user -d remote_access_platform >/dev/null 2>&1 && exit 0; sleep 1; done; exit 1"
|
||||
|
||||
Write-Host "Applying migrations..."
|
||||
Invoke-RemoteShell -Command "for f in `$(find '$remoteBuildDir/backend/migrations' -name '*.up.sql' | sort); do docker exec -i $postgresName psql -U rap_user -d remote_access_platform -v ON_ERROR_STOP=1 -f - < `$f; done"
|
||||
|
||||
Write-Host "Seeding platform owner..."
|
||||
$adminUserID = [guid]::NewGuid().ToString()
|
||||
$adminHash = '$2a$10$AqLRexkI1yXbuiMPU6dHM.KVUhF.t..9NolyK4OOodQTyTsHyG.7u'
|
||||
Invoke-RemotePostgresSql -Sql @"
|
||||
INSERT INTO users (id, email, password_hash, mfa_enabled, platform_role)
|
||||
VALUES ('$adminUserID'::uuid, '$AdminEmail', '$adminHash', FALSE, 'platform_admin')
|
||||
ON CONFLICT (email) DO UPDATE SET
|
||||
password_hash = EXCLUDED.password_hash,
|
||||
platform_role = 'platform_admin',
|
||||
updated_at = NOW();
|
||||
"@
|
||||
|
||||
Write-Host "Starting backend..."
|
||||
Invoke-RemoteDocker -Arguments @(
|
||||
"run", "-d",
|
||||
"--name", $backendName,
|
||||
"--network", "host",
|
||||
"-e", "APP_NAME=rap-api",
|
||||
"-e", "APP_ENV=c17z18-smoke",
|
||||
"-e", "HTTP_HOST=0.0.0.0",
|
||||
"-e", "HTTP_PORT=$ApiPort",
|
||||
"-e", "POSTGRES_DSN=postgres://rap_user:rap_password@127.0.0.1:$PostgresPort/remote_access_platform?sslmode=disable",
|
||||
"-e", "REDIS_ADDR=127.0.0.1:$RedisPort",
|
||||
"-e", "AUTH_ACCESS_TOKEN_SECRET=c17z18-access-secret",
|
||||
"-e", "AUTH_REFRESH_HASH_SECRET=c17z18-refresh-secret",
|
||||
$BackendImageTag
|
||||
)
|
||||
Wait-HttpReady -Url "http://192.168.200.61:$ApiPort/readyz"
|
||||
|
||||
Write-Host "Logging in as platform owner..."
|
||||
$login = Invoke-Api -Method Post -Path "/auth/login" -Body @{
|
||||
email = $AdminEmail
|
||||
password = $AdminPassword
|
||||
device_fingerprint = "c17z18-smoke-device"
|
||||
device_label = "C17Z18 rendezvous relay replacement smoke"
|
||||
trust_device = $true
|
||||
}
|
||||
$actorUserID = $login.user.id
|
||||
|
||||
Write-Host "Creating C17Z18 cluster..."
|
||||
$cluster = Invoke-Api -Method Post -Path "/clusters/" -Body @{
|
||||
actor_user_id = $actorUserID
|
||||
slug = "c17z18-$((New-Guid).Guid.Substring(0, 8))"
|
||||
name = "C17Z18 Rendezvous Relay Replacement Smoke"
|
||||
region = "docker-test"
|
||||
metadata = @{
|
||||
stage = "c17z18"
|
||||
run_id = $runId
|
||||
production_forwarding = $false
|
||||
service_workload_traffic = $false
|
||||
created_by = "c17z12-rendezvous-relay-smoke-ssh.ps1:c17z18"
|
||||
}
|
||||
}
|
||||
$clusterID = $cluster.cluster.id
|
||||
|
||||
Write-Host "Enabling test-only Fabric flags..."
|
||||
Invoke-Api -Method Put -Path "/fabric/testing-flags" -Body @{
|
||||
actor_user_id = $actorUserID
|
||||
scope_type = "platform"
|
||||
scope_id = $null
|
||||
cluster_id = $null
|
||||
enabled = $true
|
||||
telemetry_enabled = $true
|
||||
synthetic_links_enabled = $true
|
||||
history_retention_hours = 24
|
||||
metadata = @{
|
||||
stage = "c17z18"
|
||||
run_id = $runId
|
||||
production_forwarding = $false
|
||||
service_workload_traffic = $false
|
||||
}
|
||||
} | Out-Null
|
||||
|
||||
$joinToken = Invoke-Api -Method Post -Path "/clusters/$clusterID/join-tokens" -Body @{
|
||||
actor_user_id = $actorUserID
|
||||
scope = @{ purpose = "c17z18-rendezvous-relay-replacement-smoke"; roles = @("core-mesh", "relay-node") }
|
||||
expires_at = (Get-Date).ToUniversalTime().AddHours(2).ToString("o")
|
||||
max_uses = 5
|
||||
}
|
||||
|
||||
$nodeSpecs = @(
|
||||
@{ key = "a"; name = "c17z18-node-a-entry"; roles = @("core-mesh"); port = $MeshBasePort; transport = "direct_tcp_tls"; connectivity = "direct"; nat = "none" },
|
||||
@{ key = "r"; name = "c17z18-node-r-stale-relay"; roles = @("core-mesh", "relay-node"); port = ($MeshBasePort + 1); transport = "direct_tcp_tls"; connectivity = "direct"; nat = "none" },
|
||||
@{ key = "b"; name = "c17z18-node-b-direct"; roles = @("core-mesh"); port = ($MeshBasePort + 2); transport = "direct_tcp_tls"; connectivity = "direct"; nat = "none" },
|
||||
@{ key = "c"; name = "c17z18-node-c-outbound"; roles = @("core-mesh"); port = ($MeshBasePort + 3); transport = "outbound_reverse"; connectivity = "outbound_only"; nat = "symmetric" },
|
||||
@{ key = "idle"; name = "c17z18-node-s-alt-relay"; roles = @("core-mesh", "relay-node"); port = ($MeshBasePort + 4); transport = "direct_tcp_tls"; connectivity = "direct"; nat = "none" }
|
||||
)
|
||||
|
||||
$nodes = @{}
|
||||
foreach ($spec in $nodeSpecs) {
|
||||
$fingerprint = "c17z18-fp-$($spec.key)-$([guid]::NewGuid().ToString('N'))"
|
||||
$publicKey = "c17z18-pub-$($spec.key)-$([guid]::NewGuid().ToString('N'))"
|
||||
$joinRequest = Invoke-Api -Method Post -Path "/clusters/$clusterID/join-requests" -Body @{
|
||||
join_token = $joinToken.join_token.token
|
||||
node_name = $spec.name
|
||||
node_fingerprint = $fingerprint
|
||||
public_key = $publicKey
|
||||
reported_capabilities = @{
|
||||
can_accept_node_ingress = $true
|
||||
can_route_mesh = $true
|
||||
testing_node = $true
|
||||
mesh_rendezvous_relay_control_contract = $true
|
||||
mesh_rendezvous_lease_telemetry = $true
|
||||
mesh_rendezvous_lease_refresh_contract = $true
|
||||
mesh_rendezvous_relay_replacement_contract = $true
|
||||
mesh_route_path_decision_contract = $true
|
||||
mesh_route_generation_tracker = $true
|
||||
}
|
||||
reported_facts = @{
|
||||
os = "linux"
|
||||
runtime = "docker-test"
|
||||
stage = "c17z18"
|
||||
run_id = $runId
|
||||
connectivity_mode = $spec.connectivity
|
||||
nat_type = $spec.nat
|
||||
}
|
||||
requested_roles = $spec.roles
|
||||
}
|
||||
$approved = Invoke-Api -Method Post -Path "/clusters/$clusterID/join-requests/$($joinRequest.join_request.id)/approve" -Body @{
|
||||
actor_user_id = $actorUserID
|
||||
node_key = $fingerprint
|
||||
ownership_type = "platform_managed"
|
||||
owner_organization_id = $null
|
||||
}
|
||||
$nodeID = $approved.node_bootstrap.node_id
|
||||
foreach ($role in $spec.roles) {
|
||||
Invoke-Api -Method Post -Path "/clusters/$clusterID/nodes/$nodeID/roles" -Body @{
|
||||
actor_user_id = $actorUserID
|
||||
role = $role
|
||||
status = "active"
|
||||
policy = @{
|
||||
stage = "c17z18"
|
||||
run_id = $runId
|
||||
synthetic_only = $true
|
||||
production_payload_forwarding = $false
|
||||
}
|
||||
} | Out-Null
|
||||
}
|
||||
$nodes[$spec.key] = [pscustomobject]@{
|
||||
id = $nodeID
|
||||
name = $spec.name
|
||||
fingerprint = $fingerprint
|
||||
public_key = $publicKey
|
||||
port = $spec.port
|
||||
roles = $spec.roles
|
||||
transport = $spec.transport
|
||||
connectivity = $spec.connectivity
|
||||
nat = $spec.nat
|
||||
}
|
||||
}
|
||||
|
||||
$nodeAID = $nodes["a"].id
|
||||
$nodeRID = $nodes["r"].id
|
||||
$nodeBID = $nodes["b"].id
|
||||
$nodeCID = $nodes["c"].id
|
||||
$nodeSID = $nodes["idle"].id
|
||||
$routeExpiresAt = (Get-Date).ToUniversalTime().AddHours(2).ToString("o")
|
||||
$staleRelayEndpoint = "http://127.0.0.1:$($MeshBasePort + 90)"
|
||||
|
||||
$peerEndpointsDirect = @{}
|
||||
$peerEndpointsDirect[$nodeAID] = "http://127.0.0.1:$($nodes["a"].port)"
|
||||
$peerEndpointsDirect[$nodeBID] = "http://127.0.0.1:$($nodes["b"].port)"
|
||||
|
||||
$peerEndpointsRelayControl = @{}
|
||||
$peerEndpointsRelayControl[$nodeAID] = "http://127.0.0.1:$($nodes["a"].port)"
|
||||
$peerEndpointsRelayControl[$nodeRID] = $staleRelayEndpoint
|
||||
$peerEndpointsRelayControl[$nodeSID] = "http://127.0.0.1:$($nodes["idle"].port)"
|
||||
|
||||
$peerEndpointCandidatesRelay = @{}
|
||||
$peerEndpointCandidatesRelay[$nodeRID] = @(
|
||||
New-EndpointCandidate `
|
||||
-EndpointID "relay-r-public" `
|
||||
-NodeID $nodeRID `
|
||||
-Address $staleRelayEndpoint `
|
||||
-Transport "direct_tcp_tls" `
|
||||
-Reachability "public" `
|
||||
-ConnectivityMode "direct" `
|
||||
-NATType "none" `
|
||||
-Priority 10 `
|
||||
-PolicyTags @("relay-control", "same-site")
|
||||
)
|
||||
$peerEndpointCandidatesRelay[$nodeSID] = @(
|
||||
New-EndpointCandidate `
|
||||
-EndpointID "relay-s-alt-fast" `
|
||||
-NodeID $nodeSID `
|
||||
-Address "http://127.0.0.1:$($nodes["idle"].port)" `
|
||||
-Transport "direct_tcp_tls" `
|
||||
-Reachability "public" `
|
||||
-ConnectivityMode "direct" `
|
||||
-NATType "none" `
|
||||
-Priority 1 `
|
||||
-PolicyTags @("relay-control", "same-site", "fast-path")
|
||||
)
|
||||
$peerEndpointCandidatesRelay[$nodeCID] = @(
|
||||
New-EndpointCandidate `
|
||||
-EndpointID "node-c-outbound-only" `
|
||||
-NodeID $nodeCID `
|
||||
-Address "http://127.0.0.1:$($nodes["c"].port)" `
|
||||
-Transport "outbound_reverse" `
|
||||
-Reachability "outbound_only" `
|
||||
-ConnectivityMode "outbound_only" `
|
||||
-NATType "symmetric" `
|
||||
-Priority 5 `
|
||||
-PolicyTags @("nat", "outbound-only")
|
||||
)
|
||||
|
||||
Write-Host "Creating direct baseline and outbound-only relay-control route intents..."
|
||||
$directIntent = Invoke-Api -Method Post -Path "/clusters/$clusterID/mesh/route-intents" -Body @{
|
||||
actor_user_id = $actorUserID
|
||||
source_selector = @{ node_id = $nodeAID }
|
||||
destination_selector = @{ node_id = $nodeBID }
|
||||
service_class = "control"
|
||||
priority = 10
|
||||
policy = @{
|
||||
synthetic_enabled = $true
|
||||
peer_endpoints = $peerEndpointsDirect
|
||||
hops = @($nodeAID, $nodeBID)
|
||||
allowed_channels = @("fabric_control", "route_control")
|
||||
max_ttl = 4
|
||||
max_hops = 4
|
||||
expires_at = $routeExpiresAt
|
||||
route_version = "$runId-direct"
|
||||
policy_version = "$runId-policy"
|
||||
peer_directory_version = "$runId-peers"
|
||||
production_forwarding = $false
|
||||
}
|
||||
}
|
||||
|
||||
$rendezvousIntent = Invoke-Api -Method Post -Path "/clusters/$clusterID/mesh/route-intents" -Body @{
|
||||
actor_user_id = $actorUserID
|
||||
source_selector = @{ node_id = $nodeAID }
|
||||
destination_selector = @{ node_id = $nodeCID }
|
||||
service_class = "control"
|
||||
priority = 20
|
||||
policy = @{
|
||||
synthetic_enabled = $true
|
||||
peer_endpoints = $peerEndpointsRelayControl
|
||||
peer_endpoint_candidates = $peerEndpointCandidatesRelay
|
||||
hops = @($nodeAID, $nodeRID, $nodeSID, $nodeCID)
|
||||
allowed_channels = @("fabric_control", "route_control")
|
||||
max_ttl = 6
|
||||
max_hops = 6
|
||||
rendezvous_leases = @(
|
||||
@{
|
||||
lease_id = "$runId-explicit-stale-relay-lease"
|
||||
peer_node_id = $nodeCID
|
||||
relay_node_id = $nodeRID
|
||||
relay_endpoint = $staleRelayEndpoint
|
||||
transport = "relay_control"
|
||||
connectivity_mode = "relay_required"
|
||||
route_ids = @()
|
||||
allowed_channels = @("fabric_control", "route_control")
|
||||
priority = 4
|
||||
control_plane_only = $true
|
||||
issued_at = (Get-Date).ToUniversalTime().ToString("o")
|
||||
expires_at = $routeExpiresAt
|
||||
reason = "smoke_stale_relay_replacement"
|
||||
metadata = @{
|
||||
stage = "c17z18"
|
||||
run_id = $runId
|
||||
lease_refresh_contract = "node_scoped_synthetic_config_get"
|
||||
relay_replacement_contract = "stale_relay_feedback_policy"
|
||||
production_payload_forwarding = $false
|
||||
}
|
||||
}
|
||||
)
|
||||
expires_at = $routeExpiresAt
|
||||
route_version = "$runId-rendezvous"
|
||||
policy_version = "$runId-policy"
|
||||
peer_directory_version = "$runId-peers"
|
||||
production_forwarding = $false
|
||||
}
|
||||
}
|
||||
|
||||
$configs = @{}
|
||||
foreach ($key in @("a", "r", "b", "c", "idle")) {
|
||||
$configs[$key] = Invoke-Api -Method Get -Path "/clusters/$clusterID/nodes/$($nodes[$key].id)/mesh/synthetic-config"
|
||||
}
|
||||
|
||||
$nodeAPeerCandidates = Get-OptionalProperty -Object (Get-OptionalProperty -Object $configs["a"].synthetic_mesh_config -PropertyName "peer_endpoint_candidates") -PropertyName $nodeCID
|
||||
$nodeAInitialStaleLeases = @(Get-OptionalProperty -Object $configs["a"].synthetic_mesh_config -PropertyName "rendezvous_leases" | Where-Object {
|
||||
$_.peer_node_id -eq $nodeCID -and $_.relay_node_id -eq $nodeRID
|
||||
})
|
||||
$nodeAInitialAltLeases = @(Get-OptionalProperty -Object $configs["a"].synthetic_mesh_config -PropertyName "rendezvous_leases" | Where-Object {
|
||||
$_.peer_node_id -eq $nodeCID -and $_.relay_node_id -eq $nodeSID
|
||||
})
|
||||
$nodeCLeases = @(Get-OptionalProperty -Object $configs["c"].synthetic_mesh_config -PropertyName "rendezvous_leases" | Where-Object {
|
||||
$_.peer_node_id -eq $nodeCID -and ($_.relay_node_id -eq $nodeRID -or $_.relay_node_id -eq $nodeSID)
|
||||
})
|
||||
|
||||
Write-Host "Starting node-agent containers..."
|
||||
foreach ($key in @("r", "idle", "b", "c", "a")) {
|
||||
$node = $nodes[$key]
|
||||
$containerName = "$nodePrefix$key"
|
||||
$remoteStateDir = "/tmp/$runId-$key"
|
||||
Invoke-RemoteShell -Command "rm -rf '$remoteStateDir' && mkdir -p '$remoteStateDir'"
|
||||
$identity = @{
|
||||
node_id = $node.id
|
||||
cluster_id = $clusterID
|
||||
node_name = $node.name
|
||||
node_fingerprint = $node.fingerprint
|
||||
public_key = $node.public_key
|
||||
identity_status = "active"
|
||||
created_at = (Get-Date).ToUniversalTime().ToString("o")
|
||||
updated_at = (Get-Date).ToUniversalTime().ToString("o")
|
||||
} | ConvertTo-Json -Depth 10
|
||||
Send-RemoteFile -RemotePath "$remoteStateDir/identity.json" -Content $identity
|
||||
Invoke-RemoteDocker -Arguments @(
|
||||
"create",
|
||||
"--name", $containerName,
|
||||
"--network", "host",
|
||||
"-e", "RAP_BACKEND_URL=$backendContainerBaseUrl",
|
||||
"-e", "RAP_NODE_STATE_DIR=/tmp/state",
|
||||
"-e", "RAP_HEARTBEAT_INTERVAL_SECONDS=5",
|
||||
"-e", "RAP_MESH_SYNTHETIC_RUNTIME_ENABLED=true",
|
||||
"-e", "RAP_MESH_LISTEN_ADDR=0.0.0.0:$($node.port)",
|
||||
"-e", "RAP_MESH_ADVERTISE_ENDPOINT=http://127.0.0.1:$($node.port)",
|
||||
"-e", "RAP_MESH_ADVERTISE_TRANSPORT=$($node.transport)",
|
||||
"-e", "RAP_MESH_CONNECTIVITY_MODE=$($node.connectivity)",
|
||||
"-e", "RAP_MESH_NAT_TYPE=$($node.nat)",
|
||||
"-e", "RAP_MESH_REGION=docker-test",
|
||||
$NodeAgentImageTag,
|
||||
"-backend-url", $backendContainerBaseUrl,
|
||||
"-state-dir", "/tmp/state",
|
||||
"-heartbeat-interval", "5s",
|
||||
"-mesh-synthetic-runtime-enabled",
|
||||
"-mesh-listen-addr", "0.0.0.0:$($node.port)",
|
||||
"-mesh-advertise-endpoint", "http://127.0.0.1:$($node.port)",
|
||||
"-mesh-advertise-transport", $node.transport,
|
||||
"-mesh-connectivity-mode", $node.connectivity,
|
||||
"-mesh-nat-type", $node.nat,
|
||||
"-mesh-region", "docker-test"
|
||||
) | Out-Null
|
||||
Invoke-RemoteDocker -Arguments @("cp", "$remoteStateDir/.", "$containerName`:/tmp/state")
|
||||
Invoke-RemoteDocker -Arguments @("start", $containerName) | Out-Null
|
||||
}
|
||||
|
||||
Write-Host "Waiting for rendezvous relay-control observations..."
|
||||
Start-Sleep -Seconds 40
|
||||
|
||||
Write-Host "Waiting for replacement route-health effective path..."
|
||||
$meshSnapshot = Wait-C17Z18ReplacementRouteHealthSnapshot -TimeoutSeconds 40
|
||||
$links = $meshSnapshot.links
|
||||
$summary = Invoke-Api -Method Get -Path "/cluster-admin-summaries?actor_user_id=$actorUserID"
|
||||
$nodeALeaseReport = Get-LatestRendezvousLeaseReport -NodeID $nodeAID
|
||||
$nodeRLeaseReport = Get-LatestRendezvousLeaseReport -NodeID $nodeRID
|
||||
$nodeSLeaseReport = Get-LatestRendezvousLeaseReport -NodeID $nodeSID
|
||||
$nodeCLeaseReport = Get-LatestRendezvousLeaseReport -NodeID $nodeCID
|
||||
$nodeAPathDecisionReport = Get-LatestRoutePathDecisionReport -NodeID $nodeAID
|
||||
$nodeARouteGenerationReport = Get-LatestRouteGenerationReport -NodeID $nodeAID
|
||||
$nodeARouteHealthConfigReport = Get-LatestRouteHealthConfigReport -NodeID $nodeAID
|
||||
$nodeARouteHealthFeedbackRefreshReport = Get-LatestRouteHealthFeedbackRefreshReport -NodeID $nodeAID
|
||||
$refreshedNodeAConfig = Invoke-Api -Method Get -Path "/clusters/$clusterID/nodes/$nodeAID/mesh/synthetic-config"
|
||||
$nodeAReportedLeases = @(Get-OptionalProperty -Object $nodeALeaseReport -PropertyName "leases")
|
||||
$nodeAReportedReplacementLeases = @($nodeAReportedLeases | Where-Object {
|
||||
$_.peer_node_id -eq $nodeCID -and $_.relay_node_id -eq $nodeSID -and $_.reason -eq "stale_relay_replacement"
|
||||
})
|
||||
$nodeAReportedStaleRelayLeases = @($nodeAReportedLeases | Where-Object {
|
||||
$_.peer_node_id -eq $nodeCID -and $_.relay_node_id -eq $nodeRID
|
||||
})
|
||||
$nodeAReplacementLeases = @(Get-OptionalProperty -Object $refreshedNodeAConfig.synthetic_mesh_config -PropertyName "rendezvous_leases" | Where-Object {
|
||||
$_.peer_node_id -eq $nodeCID -and $_.relay_node_id -eq $nodeSID -and $_.reason -eq "stale_relay_replacement"
|
||||
})
|
||||
$nodeAWithdrawnStaleLeases = @(Get-OptionalProperty -Object $refreshedNodeAConfig.synthetic_mesh_config -PropertyName "rendezvous_leases" | Where-Object {
|
||||
$_.peer_node_id -eq $nodeCID -and $_.relay_node_id -eq $nodeRID
|
||||
})
|
||||
$nodeARelayPolicy = Get-OptionalProperty -Object $refreshedNodeAConfig.synthetic_mesh_config -PropertyName "rendezvous_relay_policy"
|
||||
$nodeAInitialPathDecisionReport = Get-OptionalProperty -Object $configs["a"].synthetic_mesh_config -PropertyName "route_path_decisions"
|
||||
$nodeAConfigPathDecisionReport = Get-OptionalProperty -Object $refreshedNodeAConfig.synthetic_mesh_config -PropertyName "route_path_decisions"
|
||||
$nodeAReportedPathDecisions = @(Get-OptionalProperty -Object $nodeAPathDecisionReport -PropertyName "decisions")
|
||||
$nodeAReportedReplacementPathDecisions = @($nodeAReportedPathDecisions | Where-Object {
|
||||
$_.route_id -eq $rendezvousIntent.route_intent.id -and
|
||||
$_.selected_relay_id -eq $nodeSID -and
|
||||
$_.decision_source -eq "stale_relay_replacement" -and
|
||||
(@($_.effective_hops) -contains $nodeSID) -and
|
||||
-not (@($_.effective_hops) -contains $nodeRID)
|
||||
})
|
||||
|
||||
$nodeLogs = @{}
|
||||
foreach ($key in @("a", "r", "b", "c", "idle")) {
|
||||
$nodeLogs[$key] = Invoke-RemoteDockerText -Arguments @("logs", "--tail", "120", "$nodePrefix$key")
|
||||
}
|
||||
$backendLogs = Invoke-RemoteDockerText -Arguments @("logs", "--tail", "80", $backendName)
|
||||
|
||||
$meshLinks = @($meshSnapshot.mesh_links)
|
||||
$routeHealthLinks = @($meshSnapshot.route_health_links)
|
||||
$directHealth = @($meshSnapshot.direct_health)
|
||||
$rendezvousHealth = @($meshSnapshot.rendezvous_health)
|
||||
$replacementRouteHealth = @($meshSnapshot.replacement_route_health)
|
||||
$managerLinks = @($meshLinks | Where-Object { $_.metadata.observation_type -eq "peer_connection_manager" })
|
||||
$relayControlLinks = @($managerLinks | Where-Object {
|
||||
$_.source_node_id -eq $nodeAID -and
|
||||
$_.target_node_id -eq $nodeCID -and
|
||||
$_.link_status -eq "reachable" -and
|
||||
$_.metadata.transport_mode -eq "relay_control" -and
|
||||
$_.metadata.rendezvous_resolved -eq $true -and
|
||||
$_.metadata.relay_candidate -eq $true -and
|
||||
$_.metadata.connection_state -eq "relay_ready"
|
||||
})
|
||||
$replacementRelayControlLinks = @($relayControlLinks | Where-Object {
|
||||
$_.metadata.relay_node_id -eq $nodeSID
|
||||
})
|
||||
$replacementRelayReadyFromLeaseReport = (
|
||||
$nodeAReportedReplacementLeases.Count -gt 0 -and
|
||||
(Get-OptionalProperty -Object $nodeAReportedReplacementLeases[0] -PropertyName "relay_ready") -eq $true -and
|
||||
(Get-OptionalProperty -Object $nodeAReportedReplacementLeases[0] -PropertyName "connection_state") -eq "relay_ready"
|
||||
)
|
||||
|
||||
$nodeALog = $nodeLogs["a"] -join "`n"
|
||||
$directRouteDelivered = $nodeALog -match ('"event":"fabric_route_delivery_succeeded","route_id":"' + [regex]::Escape($directIntent.route_intent.id) + '"')
|
||||
$leaseReportBoundaryFlagsDisabled = $true
|
||||
foreach ($report in @($nodeALeaseReport, $nodeRLeaseReport, $nodeSLeaseReport, $nodeCLeaseReport)) {
|
||||
if ($null -eq $report -or
|
||||
(Get-OptionalProperty -Object $report -PropertyName "control_plane_only") -ne $true -or
|
||||
(Get-OptionalProperty -Object $report -PropertyName "relay_payload_forwarding") -ne $false -or
|
||||
(Get-OptionalProperty -Object $report -PropertyName "production_payload_forwarding") -ne $false -or
|
||||
(Get-OptionalProperty -Object $report -PropertyName "service_workload_traffic") -ne $false) {
|
||||
$leaseReportBoundaryFlagsDisabled = $false
|
||||
}
|
||||
}
|
||||
$pathDecisionBoundaryFlagsDisabled = (
|
||||
$null -ne $nodeAPathDecisionReport -and
|
||||
(Get-OptionalProperty -Object $nodeAPathDecisionReport -PropertyName "control_plane_only") -eq $true -and
|
||||
(Get-OptionalProperty -Object $nodeAPathDecisionReport -PropertyName "production_payload_forwarding") -eq $false -and
|
||||
(Get-OptionalProperty -Object $nodeAPathDecisionReport -PropertyName "service_workload_traffic") -eq $false -and
|
||||
(Get-OptionalProperty -Object $nodeAPathDecisionReport -PropertyName "route_path_forwarding_runtime") -eq $false
|
||||
)
|
||||
$routeGenerationBoundaryFlagsDisabled = (
|
||||
$null -ne $nodeARouteGenerationReport -and
|
||||
(Get-OptionalProperty -Object $nodeARouteGenerationReport -PropertyName "control_plane_only") -eq $true -and
|
||||
(Get-OptionalProperty -Object $nodeARouteGenerationReport -PropertyName "production_payload_forwarding") -eq $false -and
|
||||
(Get-OptionalProperty -Object $nodeARouteGenerationReport -PropertyName "service_workload_traffic") -eq $false -and
|
||||
(Get-OptionalProperty -Object $nodeARouteGenerationReport -PropertyName "route_path_forwarding_runtime") -eq $false
|
||||
)
|
||||
$nodeAWithdrawnDecisionCount = Get-OptionalProperty -Object $nodeARouteGenerationReport -PropertyName "withdrawn_decision_count"
|
||||
$nodeATotalWithdrawnDecisionCount = Get-OptionalProperty -Object $nodeARouteGenerationReport -PropertyName "total_withdrawn_decision_count"
|
||||
$nodeAWithdrawnDecisions = @(Get-OptionalProperty -Object $nodeARouteGenerationReport -PropertyName "withdrawn_decisions")
|
||||
$routeHealthConfigBoundaryFlagsDisabled = (
|
||||
$null -ne $nodeARouteHealthConfigReport -and
|
||||
(Get-OptionalProperty -Object $nodeARouteHealthConfigReport -PropertyName "control_plane_only") -eq $true -and
|
||||
(Get-OptionalProperty -Object $nodeARouteHealthConfigReport -PropertyName "route_health_only") -eq $true -and
|
||||
(Get-OptionalProperty -Object $nodeARouteHealthConfigReport -PropertyName "synthetic_route_health_route_path_runtime") -eq $true -and
|
||||
(Get-OptionalProperty -Object $nodeARouteHealthConfigReport -PropertyName "production_route_path_forwarding_runtime") -eq $false -and
|
||||
(Get-OptionalProperty -Object $nodeARouteHealthConfigReport -PropertyName "production_payload_forwarding") -eq $false -and
|
||||
(Get-OptionalProperty -Object $nodeARouteHealthConfigReport -PropertyName "service_workload_traffic") -eq $false -and
|
||||
(Get-OptionalProperty -Object $nodeARouteHealthConfigReport -PropertyName "test_service_route_config_changed") -eq $false
|
||||
)
|
||||
$routeHealthFeedbackRefreshBoundaryFlagsDisabled = (
|
||||
$null -ne $nodeARouteHealthFeedbackRefreshReport -and
|
||||
(Get-OptionalProperty -Object $nodeARouteHealthFeedbackRefreshReport -PropertyName "control_plane_only") -eq $true -and
|
||||
(Get-OptionalProperty -Object $nodeARouteHealthFeedbackRefreshReport -PropertyName "route_health_only") -eq $true -and
|
||||
(Get-OptionalProperty -Object $nodeARouteHealthFeedbackRefreshReport -PropertyName "production_payload_forwarding") -eq $false -and
|
||||
(Get-OptionalProperty -Object $nodeARouteHealthFeedbackRefreshReport -PropertyName "service_workload_traffic") -eq $false
|
||||
)
|
||||
$passMatrix = [ordered]@{
|
||||
backend_ready = $true
|
||||
platform_owner_login = [bool]$actorUserID
|
||||
cluster_created = [bool]$clusterID
|
||||
fabric_testing_flags_enabled = $true
|
||||
node_a_scoped_config_enabled = $configs["a"].synthetic_mesh_config.enabled -eq $true
|
||||
node_a_has_direct_and_rendezvous_routes = @($configs["a"].synthetic_mesh_config.routes).Count -eq 2
|
||||
node_a_has_outbound_peer_candidate = @($nodeAPeerCandidates).Count -gt 0
|
||||
node_a_has_initial_stale_rendezvous_lease = $nodeAInitialStaleLeases.Count -gt 0
|
||||
node_a_initial_lease_is_control_plane_only = ($nodeAInitialStaleLeases.Count -gt 0 -and $nodeAInitialStaleLeases[0].control_plane_only -eq $true)
|
||||
node_a_initial_lease_uses_relay_control = ($nodeAInitialStaleLeases.Count -gt 0 -and $nodeAInitialStaleLeases[0].transport -eq "relay_control")
|
||||
node_a_initial_auto_alt_relay_candidate = $nodeAInitialAltLeases.Count -gt 0
|
||||
node_a_initial_path_decision_report = (Get-OptionalProperty -Object $nodeAInitialPathDecisionReport -PropertyName "schema_version") -eq "c17z18.route_path_decisions.v1"
|
||||
node_a_report_replacement_lease_uses_alt_relay = ($nodeAReportedReplacementLeases.Count -gt 0 -and $nodeAReportedReplacementLeases[0].relay_node_id -eq $nodeSID)
|
||||
node_a_report_stale_relay_lease_withdrawn = $nodeAReportedStaleRelayLeases.Count -eq 0
|
||||
node_a_report_replacement_reason = ($nodeAReportedReplacementLeases.Count -gt 0 -and $nodeAReportedReplacementLeases[0].reason -eq "stale_relay_replacement")
|
||||
node_a_reports_c17z18_path_decisions = (Get-OptionalProperty -Object $nodeAPathDecisionReport -PropertyName "schema_version") -eq "c17z18.mesh_route_path_decision_report.v1"
|
||||
node_a_path_decision_replacement_count = (Get-OptionalProperty -Object $nodeAPathDecisionReport -PropertyName "replacement_decision_count") -gt 0
|
||||
node_a_path_decision_uses_alt_relay = $nodeAReportedReplacementPathDecisions.Count -gt 0
|
||||
node_a_path_decision_boundary_flags_disabled = $pathDecisionBoundaryFlagsDisabled
|
||||
node_a_reports_c17z18_route_generation = (Get-OptionalProperty -Object $nodeARouteGenerationReport -PropertyName "schema_version") -eq "c17z18.mesh_route_generation_report.v1"
|
||||
node_a_route_generation_active = (Get-OptionalProperty -Object $nodeARouteGenerationReport -PropertyName "active_decision_count") -gt 0
|
||||
node_a_route_generation_applied = (Get-OptionalProperty -Object $nodeARouteGenerationReport -PropertyName "applied_decision_count") -gt 0
|
||||
node_a_route_generation_withdrawn = ($nodeAWithdrawnDecisionCount -gt 0 -or $nodeATotalWithdrawnDecisionCount -gt 0 -or $nodeAWithdrawnDecisions.Count -gt 0)
|
||||
node_a_route_generation_changed = (Get-OptionalProperty -Object $nodeARouteGenerationReport -PropertyName "generation_changed") -eq $true
|
||||
node_a_route_generation_boundary_flags_disabled = $routeGenerationBoundaryFlagsDisabled
|
||||
node_a_reports_c17z20_route_health_config = (Get-OptionalProperty -Object $nodeARouteHealthConfigReport -PropertyName "schema_version") -eq "c17z20.mesh_route_health_config_report.v1"
|
||||
node_a_reports_c17z20_route_health_feedback_refresh = (Get-OptionalProperty -Object $nodeARouteHealthFeedbackRefreshReport -PropertyName "schema_version") -eq "c17z20.mesh_route_health_feedback_refresh_report.v1"
|
||||
node_a_route_health_feedback_refresh_supported = (Get-OptionalProperty -Object $nodeARouteHealthFeedbackRefreshReport -PropertyName "feedback_refresh_supported") -eq $true
|
||||
node_a_route_health_feedback_refresh_boundary_flags_disabled = $routeHealthFeedbackRefreshBoundaryFlagsDisabled
|
||||
node_a_route_health_config_applied = (Get-OptionalProperty -Object $nodeARouteHealthConfigReport -PropertyName "route_path_decision_applied_count") -gt 0
|
||||
node_a_route_health_config_replacement = (Get-OptionalProperty -Object $nodeARouteHealthConfigReport -PropertyName "replacement_route_health_route_count") -gt 0
|
||||
node_a_route_health_config_boundary_flags_disabled = $routeHealthConfigBoundaryFlagsDisabled
|
||||
node_a_route_health_uses_effective_alt_relay = $replacementRouteHealth.Count -gt 0
|
||||
node_a_route_health_has_no_effective_path_drift = ($replacementRouteHealth.Count -gt 0 -and (Get-OptionalProperty -Object $replacementRouteHealth[0].metadata -PropertyName "route_path_drift_detected") -eq $false)
|
||||
node_a_route_health_selected_alt_next_hop = $nodeALog -match ('"event":"fabric_route_selected","route_id":"' + [regex]::Escape($rendezvousIntent.route_intent.id) + '".*"next_node_id":"' + [regex]::Escape($nodeSID) + '"')
|
||||
outbound_node_has_relay_lease = $nodeCLeases.Count -gt 0
|
||||
direct_baseline_health_reported = $directRouteDelivered
|
||||
node_a_loaded_c17z20_control_plane_config = ($nodeAReplacementLeases.Count -gt 0 -and (Get-OptionalProperty -Object $nodeARouteHealthConfigReport -PropertyName "schema_version") -eq "c17z20.mesh_route_health_config_report.v1")
|
||||
node_a_resolved_waiting_rendezvous = ($replacementRelayControlLinks.Count -gt 0 -or $replacementRelayReadyFromLeaseReport)
|
||||
relay_control_manager_link_reachable = ($replacementRelayControlLinks.Count -gt 0 -or $replacementRelayReadyFromLeaseReport)
|
||||
relay_ready_recorded = ($replacementRelayControlLinks.Count -gt 0 -or $replacementRelayReadyFromLeaseReport)
|
||||
node_a_reports_c17z18_lease_telemetry = (Get-OptionalProperty -Object $nodeALeaseReport -PropertyName "schema_version") -eq "c17z18.mesh_rendezvous_lease_report.v1"
|
||||
node_a_lease_report_entry_observer = (Get-OptionalProperty -Object $nodeALeaseReport -PropertyName "entry_observer_count") -gt 0
|
||||
node_a_lease_report_relay_ready = (Get-OptionalProperty -Object $nodeALeaseReport -PropertyName "relay_control_ready_count") -gt 0
|
||||
node_a_lease_report_refresh_contract = (Get-OptionalProperty -Object $nodeALeaseReport -PropertyName "refresh_contract") -eq "node_scoped_synthetic_config_get"
|
||||
stale_relay_refresh_succeeded_on_cluster = (
|
||||
((Get-OptionalProperty -Object $nodeRLeaseReport -PropertyName "last_refresh_reason") -eq "stale_relay" -and (Get-OptionalProperty -Object $nodeRLeaseReport -PropertyName "refresh_success_count") -gt 0) -or
|
||||
((Get-OptionalProperty -Object $nodeSLeaseReport -PropertyName "last_refresh_reason") -eq "stale_relay" -and (Get-OptionalProperty -Object $nodeSLeaseReport -PropertyName "refresh_success_count") -gt 0)
|
||||
)
|
||||
alt_relay_node_reports_admitted_relay_lease = (Get-OptionalProperty -Object $nodeSLeaseReport -PropertyName "admitted_as_relay_count") -gt 0
|
||||
outbound_node_reports_admitted_peer_lease = (Get-OptionalProperty -Object $nodeCLeaseReport -PropertyName "admitted_as_peer_count") -gt 0
|
||||
lease_telemetry_boundary_flags_disabled = $leaseReportBoundaryFlagsDisabled
|
||||
production_forwarding_disabled = (
|
||||
$configs["a"].synthetic_mesh_config.production_forwarding -eq $false -and
|
||||
$configs["r"].synthetic_mesh_config.production_forwarding -eq $false -and
|
||||
$configs["b"].synthetic_mesh_config.production_forwarding -eq $false -and
|
||||
$configs["c"].synthetic_mesh_config.production_forwarding -eq $false -and
|
||||
$configs["idle"].synthetic_mesh_config.production_forwarding -eq $false
|
||||
)
|
||||
}
|
||||
|
||||
$result = [pscustomobject]@{
|
||||
stage = "C17Z18 rendezvous relay replacement docker-test smoke"
|
||||
run_id = $runId
|
||||
backend_base_url = $backendPublicBaseUrl
|
||||
cluster_id = $clusterID
|
||||
node_ids = @{
|
||||
a = $nodes["a"].id
|
||||
r = $nodes["r"].id
|
||||
s = $nodes["idle"].id
|
||||
b = $nodes["b"].id
|
||||
c = $nodes["c"].id
|
||||
idle = $nodes["idle"].id
|
||||
}
|
||||
route_intents = @{
|
||||
direct = $directIntent.route_intent.id
|
||||
rendezvous = $rendezvousIntent.route_intent.id
|
||||
}
|
||||
scoped_config_route_counts = @{
|
||||
a = @($configs["a"].synthetic_mesh_config.routes).Count
|
||||
r = @($configs["r"].synthetic_mesh_config.routes).Count
|
||||
b = @($configs["b"].synthetic_mesh_config.routes).Count
|
||||
c = @($configs["c"].synthetic_mesh_config.routes).Count
|
||||
idle = @($configs["idle"].synthetic_mesh_config.routes).Count
|
||||
}
|
||||
rendezvous_lease_counts = @{
|
||||
a = Get-OptionalArrayCount -Object $configs["a"].synthetic_mesh_config -PropertyName "rendezvous_leases"
|
||||
r = Get-OptionalArrayCount -Object $configs["r"].synthetic_mesh_config -PropertyName "rendezvous_leases"
|
||||
b = Get-OptionalArrayCount -Object $configs["b"].synthetic_mesh_config -PropertyName "rendezvous_leases"
|
||||
c = Get-OptionalArrayCount -Object $configs["c"].synthetic_mesh_config -PropertyName "rendezvous_leases"
|
||||
idle = Get-OptionalArrayCount -Object $configs["idle"].synthetic_mesh_config -PropertyName "rendezvous_leases"
|
||||
}
|
||||
node_a_initial_stale_rendezvous_lease = $nodeAInitialStaleLeases | Select-Object -First 1
|
||||
node_a_reported_replacement_rendezvous_lease = $nodeAReportedReplacementLeases | Select-Object -First 1
|
||||
node_a_current_replacement_rendezvous_lease = $nodeAReplacementLeases | Select-Object -First 1
|
||||
node_a_current_rendezvous_relay_policy = $nodeARelayPolicy
|
||||
node_a_initial_route_path_decisions = $nodeAInitialPathDecisionReport
|
||||
node_a_current_route_path_decisions = $nodeAConfigPathDecisionReport
|
||||
node_a_reported_route_path_decision = $nodeAReportedReplacementPathDecisions | Select-Object -First 1
|
||||
route_path_decision_reports = @{
|
||||
a = $nodeAPathDecisionReport
|
||||
}
|
||||
route_generation_reports = @{
|
||||
a = $nodeARouteGenerationReport
|
||||
}
|
||||
route_health_config_reports = @{
|
||||
a = $nodeARouteHealthConfigReport
|
||||
}
|
||||
route_health_feedback_refresh_reports = @{
|
||||
a = $nodeARouteHealthFeedbackRefreshReport
|
||||
}
|
||||
rendezvous_lease_reports = @{
|
||||
a = $nodeALeaseReport
|
||||
r = $nodeRLeaseReport
|
||||
s = $nodeSLeaseReport
|
||||
c = $nodeCLeaseReport
|
||||
}
|
||||
mesh_link_count = $meshLinks.Count
|
||||
route_health_count = $routeHealthLinks.Count
|
||||
peer_connection_manager_link_count = $managerLinks.Count
|
||||
relay_control_link_count = $relayControlLinks.Count
|
||||
direct_route_delivery_succeeded = $directRouteDelivered
|
||||
pass_matrix = $passMatrix
|
||||
direct_route_health = $directHealth | Select-Object -First 3
|
||||
replacement_route_health = $replacementRouteHealth | Select-Object -First 3
|
||||
relay_control_links = $relayControlLinks | Select-Object -First 5
|
||||
replacement_relay_control_links = $replacementRelayControlLinks | Select-Object -First 5
|
||||
cluster_summaries = $summary.cluster_summaries
|
||||
backend_log_tail = $backendLogs
|
||||
node_log_tail = $nodeLogs
|
||||
containers_left_running = [bool]$KeepRunning
|
||||
}
|
||||
|
||||
$failed = @($passMatrix.GetEnumerator() | Where-Object { -not $_.Value })
|
||||
$result | ConvertTo-Json -Depth 60
|
||||
|
||||
if ($failed.Count -gt 0) {
|
||||
throw "C17Z18 rendezvous relay replacement smoke failed: $($failed.Name -join ', ')"
|
||||
}
|
||||
|
||||
if (-not $KeepRunning) {
|
||||
Write-Host "Cleaning up C17Z18 containers..."
|
||||
Remove-C17Z12Containers
|
||||
Invoke-RemoteShell -Command "rm -rf '$remoteBuildDir'"
|
||||
}
|
||||
@@ -256,7 +256,7 @@ Invoke-RemoteDocker -Arguments @(
|
||||
"-e", "REDIS_ADDR=127.0.0.1:$RedisPort",
|
||||
"-e", "AUTH_ACCESS_TOKEN_SECRET=c17z19-access-secret",
|
||||
"-e", "AUTH_REFRESH_HASH_SECRET=c17z19-refresh-secret",
|
||||
"-e", "INSTALLATION_AUTHORITY_MODE=legacy",
|
||||
"-e", "INSTALLATION_AUTHORITY_MODE=compat",
|
||||
"-e", "INSTALLATION_INSECURE_BOOTSTRAP_ENABLED=true",
|
||||
"-e", "SECRET_ENCRYPTION_KEY_B64=$secretKeyB64",
|
||||
"-e", "SECRET_ENCRYPTION_KEY_ID=$runId",
|
||||
|
||||
@@ -66,7 +66,7 @@ function Get-MeshPort {
|
||||
function Enable-TestMeshListener {
|
||||
param([object]$Node)
|
||||
$port = Get-MeshPort -Name $Node.name
|
||||
Invoke-Api -Method PUT -Path "/clusters/$ClusterID/nodes/$($Node.id)/workloads/mesh-listener/desired" -Body @{
|
||||
Invoke-Api -Method PUT -Path "/clusters/$ClusterID/nodes/$($Node.id)/workloads/fabric-listener/desired" -Body @{
|
||||
actor_user_id = $ActorUserID
|
||||
desired_state = "enabled"
|
||||
runtime_mode = "container"
|
||||
|
||||
@@ -72,7 +72,7 @@ function Get-MeshPort {
|
||||
function Enable-TestMeshListener {
|
||||
param([object]$Node)
|
||||
$port = Get-MeshPort -Name $Node.name
|
||||
Invoke-Api -Method PUT -Path "/clusters/$ClusterID/nodes/$($Node.id)/workloads/mesh-listener/desired" -Body @{
|
||||
Invoke-Api -Method PUT -Path "/clusters/$ClusterID/nodes/$($Node.id)/workloads/fabric-listener/desired" -Body @{
|
||||
actor_user_id = $ActorUserID
|
||||
desired_state = "enabled"
|
||||
runtime_mode = "container"
|
||||
@@ -1067,7 +1067,7 @@ try {
|
||||
baseline_depth = $baselineAlternateExitDepth
|
||||
depth = $alternateExitDepth
|
||||
}
|
||||
backend_fallback_queue = @{
|
||||
degraded_route_queue = @{
|
||||
baseline_depth = $baselineBackendDepth
|
||||
depth = $finalBackendDepth
|
||||
}
|
||||
@@ -1094,7 +1094,7 @@ try {
|
||||
post_rebuild_uses_alternate_exit_route = ([string]$postRebuildIngress.report.ingress.last_selected_route_id -eq $alternateRouteID)
|
||||
primary_exit_inbox_received_pre_rebuild_packets = ($primaryExitDepth -ge ($baselinePrimaryExitDepth + $expectedPrimaryExitPackets))
|
||||
alternate_exit_inbox_received_post_rebuild_packets = ($alternateExitDepth -ge ($baselineAlternateExitDepth + $expectedAlternateExitPackets))
|
||||
no_backend_fallback_used = ($finalBackendDepth -eq $baselineBackendDepth)
|
||||
no_degraded_route_used = ($finalBackendDepth -eq $baselineBackendDepth)
|
||||
no_flow_drops = ($droppedDelta -eq 0)
|
||||
route_intents_expired = ($expiredPrimary.route_intent.lifecycle_status -eq "expired" -and $expiredAlternate.route_intent.lifecycle_status -eq "expired")
|
||||
}
|
||||
@@ -1134,3 +1134,4 @@ if (-not (Test-Path $resultDir)) {
|
||||
$result | ConvertTo-Json -Depth 100 | Set-Content -Path $resultFullPath -Encoding UTF8
|
||||
Write-Host "C18Z10 live service-channel exit pool smoke passed. Result: $resultFullPath"
|
||||
$result
|
||||
|
||||
|
||||
@@ -96,14 +96,14 @@ $heartbeatBody = @{
|
||||
schema_version = "c18z52.fabric_service_channel_access_report.v1"
|
||||
total = 1
|
||||
signed = 1
|
||||
backend_fallback = 0
|
||||
backend_fallback_blocked = 1
|
||||
degraded_route = 0
|
||||
degraded_route_blocked = 1
|
||||
fabric_route_send_failure = 1
|
||||
data_plane_contract = 1
|
||||
last_backend_relay_policy = "disabled"
|
||||
last_degraded_route_policy = "disabled"
|
||||
last_working_data_transport = "fabric_service_channel"
|
||||
last_steady_state_transport = "fabric_route"
|
||||
last_data_plane_violation_status = "fabric_route_send_failed_backend_fallback_blocked"
|
||||
last_data_plane_violation_status = "fabric_route_send_failed_degraded_route_blocked"
|
||||
last_data_plane_violation_reason = "synthetic c18z100 route send failure"
|
||||
}
|
||||
}
|
||||
@@ -148,8 +148,8 @@ $ledgerAttempt = @($ledgerItems | Where-Object {
|
||||
}) | Select-Object -First 1
|
||||
$sourceFilteredLedger = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/fabric/service-channels/rebuild-attempts?actor_user_id=$ActorUserID&feedback_source=fabric_service_channel_access_report&limit=20&enrichment=summary").rebuild_attempts
|
||||
$channelFilteredLedger = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/fabric/service-channels/rebuild-attempts?actor_user_id=$ActorUserID&feedback_channel_id=$($lease.channel_id)&limit=20&enrichment=summary").rebuild_attempts
|
||||
$violationFilteredLedger = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/fabric/service-channels/rebuild-attempts?actor_user_id=$ActorUserID&feedback_violation_status=fabric_route_send_failed_backend_fallback_blocked&limit=20&enrichment=summary").rebuild_attempts
|
||||
$combinedFilteredLedger = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/fabric/service-channels/rebuild-attempts?actor_user_id=$ActorUserID&feedback_source=fabric_service_channel_access_report&feedback_channel_id=$($lease.channel_id)&feedback_violation_status=fabric_route_send_failed_backend_fallback_blocked&limit=20&enrichment=summary").rebuild_attempts
|
||||
$violationFilteredLedger = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/fabric/service-channels/rebuild-attempts?actor_user_id=$ActorUserID&feedback_violation_status=fabric_route_send_failed_degraded_route_blocked&limit=20&enrichment=summary").rebuild_attempts
|
||||
$combinedFilteredLedger = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/fabric/service-channels/rebuild-attempts?actor_user_id=$ActorUserID&feedback_source=fabric_service_channel_access_report&feedback_channel_id=$($lease.channel_id)&feedback_violation_status=fabric_route_send_failed_degraded_route_blocked&limit=20&enrichment=summary").rebuild_attempts
|
||||
$wrongChannelFilteredLedger = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/fabric/service-channels/rebuild-attempts?actor_user_id=$ActorUserID&feedback_channel_id=00000000-0000-0000-0000-000000000000&limit=20&enrichment=summary").rebuild_attempts
|
||||
$sourceFilteredAttempt = @($sourceFilteredLedger | Where-Object { [string](Get-PropertyValue -Item $_ -Name "rebuild_request_id" -Default "") -eq [string](Get-PropertyValue -Item $replacement -Name "rebuild_request_id" -Default "") }) | Select-Object -First 1
|
||||
$channelFilteredAttempt = @($channelFilteredLedger | Where-Object { [string](Get-PropertyValue -Item $_ -Name "rebuild_request_id" -Default "") -eq [string](Get-PropertyValue -Item $replacement -Name "rebuild_request_id" -Default "") }) | Select-Object -First 1
|
||||
@@ -159,7 +159,7 @@ $health = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/fabric/service-cha
|
||||
$feedbackBreakdown = @((Get-PropertyValue -Item $health -Name "feedback_breakdowns" -Default @()) | Where-Object {
|
||||
[string](Get-PropertyValue -Item $_ -Name "feedback_source" -Default "") -eq "fabric_service_channel_access_report" -and
|
||||
[string](Get-PropertyValue -Item $_ -Name "feedback_channel_id" -Default "") -eq [string]$lease.channel_id -and
|
||||
[string](Get-PropertyValue -Item $_ -Name "feedback_violation_status" -Default "") -eq "fabric_route_send_failed_backend_fallback_blocked"
|
||||
[string](Get-PropertyValue -Item $_ -Name "feedback_violation_status" -Default "") -eq "fabric_route_send_failed_degraded_route_blocked"
|
||||
}) | Select-Object -First 1
|
||||
|
||||
$backendLine = (& ssh $DockerSSH "docker ps --format '{{.Names}} {{.Image}} {{.Status}}' | grep '^rap_test_backend '") | Out-String
|
||||
@@ -171,7 +171,7 @@ $checks = [ordered]@{
|
||||
lease_selected_bad_route_first = ([string]$lease.primary_route.route_id -eq [string]$badRoute.id)
|
||||
route_feedback_recorded_from_access_report = ($null -ne $feedback)
|
||||
route_feedback_is_fenced = ($null -ne $feedback -and [string]$feedback.feedback_status -eq "fenced")
|
||||
route_feedback_contains_blocked_policy_reason = ($null -ne $feedback -and @($feedback.reasons | Where-Object { [string]$_ -eq "backend_fallback_blocked_by_policy" }).Count -gt 0)
|
||||
route_feedback_contains_blocked_policy_reason = ($null -ne $feedback -and @($feedback.reasons | Where-Object { [string]$_ -eq "degraded_route_blocked_by_policy" }).Count -gt 0)
|
||||
duplicate_heartbeat_kept_single_latest_feedback = ($duplicateFeedbackCount -eq 1)
|
||||
duplicate_heartbeat_kept_feedback_id = ($firstFeedbackID -ne "" -and $firstFeedbackID -eq $secondFeedbackID)
|
||||
duplicate_heartbeat_kept_observed_at = ($firstObservedAt -ne "" -and $firstObservedAt -eq $secondObservedAt)
|
||||
@@ -181,7 +181,7 @@ $checks = [ordered]@{
|
||||
planner_decision_links_feedback_observation = ($null -ne $replacement -and [string](Get-PropertyValue -Item $replacement -Name "feedback_observation_id" -Default "") -eq $firstFeedbackID)
|
||||
planner_decision_links_access_report_source = ($null -ne $replacement -and [string](Get-PropertyValue -Item $replacement -Name "feedback_source" -Default "") -eq "fabric_service_channel_access_report")
|
||||
planner_decision_links_channel = ($null -ne $replacement -and [string](Get-PropertyValue -Item $replacement -Name "feedback_channel_id" -Default "") -eq [string]$lease.channel_id)
|
||||
planner_decision_links_violation = ($null -ne $replacement -and [string](Get-PropertyValue -Item $replacement -Name "feedback_violation_status" -Default "") -eq "fabric_route_send_failed_backend_fallback_blocked")
|
||||
planner_decision_links_violation = ($null -ne $replacement -and [string](Get-PropertyValue -Item $replacement -Name "feedback_violation_status" -Default "") -eq "fabric_route_send_failed_degraded_route_blocked")
|
||||
rebuild_ledger_recorded_correlated_attempt = ($null -ne $ledgerAttempt)
|
||||
rebuild_ledger_links_feedback_observation = ($null -ne $ledgerAttempt -and [string](Get-PropertyValue -Item $ledgerAttempt -Name "feedback_observation_id" -Default "") -eq $firstFeedbackID)
|
||||
rebuild_ledger_links_access_report_source = ($null -ne $ledgerAttempt -and [string](Get-PropertyValue -Item $ledgerAttempt -Name "feedback_source" -Default "") -eq "fabric_service_channel_access_report")
|
||||
@@ -248,3 +248,4 @@ if (-not $result.passed) {
|
||||
|
||||
Write-Host "C18Z100 rebuild health feedback breakdown smoke passed. Result: $target"
|
||||
$result
|
||||
|
||||
|
||||
@@ -38,7 +38,7 @@ $backendImage = (& ssh $DockerSSH "docker inspect rap_test_backend --format '{{.
|
||||
$checks.backend_expected_image_deployed = ($backendImage -eq $ExpectedBackendImage)
|
||||
|
||||
$feedbackChannelID = "$runId-channel"
|
||||
$feedbackViolationStatus = "fabric_route_send_failed_backend_fallback_blocked"
|
||||
$feedbackViolationStatus = "fabric_route_send_failed_degraded_route_blocked"
|
||||
$reason = "synthetic c18z102 rebuild-health feedback drilldown audit"
|
||||
|
||||
Invoke-Api -Method POST -Path "/clusters/$ClusterID/fabric/service-channels/rebuild-incidents/investigations" -Body @{
|
||||
@@ -90,3 +90,4 @@ if ($failed.Count -gt 0) {
|
||||
|
||||
Write-Host "C18Z102 rebuild-health feedback drilldown audit smoke passed. Result: $target"
|
||||
$result
|
||||
|
||||
|
||||
@@ -39,7 +39,7 @@ $checks.backend_expected_image_deployed = ($backendImage -eq $ExpectedBackendIma
|
||||
|
||||
$feedbackChannelID = "$runId-channel"
|
||||
$incidentRouteID = "$runId-route"
|
||||
$feedbackViolationStatus = "fabric_route_send_failed_backend_fallback_blocked"
|
||||
$feedbackViolationStatus = "fabric_route_send_failed_degraded_route_blocked"
|
||||
|
||||
Invoke-Api -Method POST -Path "/clusters/$ClusterID/fabric/service-channels/rebuild-incidents/investigations" -Body @{
|
||||
actor_user_id = $ActorUserID
|
||||
@@ -118,3 +118,4 @@ if ($failed.Count -gt 0) {
|
||||
|
||||
Write-Host "C18Z104 focused fabric audit smoke passed. Result: $target"
|
||||
$result
|
||||
|
||||
|
||||
@@ -76,7 +76,7 @@ function Get-EntryBaseUrlForNode {
|
||||
function Enable-TestMeshListener {
|
||||
param([object]$Node)
|
||||
$port = Get-MeshPort -Name $Node.name
|
||||
Invoke-Api -Method PUT -Path "/clusters/$ClusterID/nodes/$($Node.id)/workloads/mesh-listener/desired" -Body @{
|
||||
Invoke-Api -Method PUT -Path "/clusters/$ClusterID/nodes/$($Node.id)/workloads/fabric-listener/desired" -Body @{
|
||||
actor_user_id = $ActorUserID
|
||||
desired_state = "enabled"
|
||||
runtime_mode = "container"
|
||||
@@ -1061,7 +1061,7 @@ try {
|
||||
pre_feedback_depth = $preExitDepth
|
||||
final_depth = $finalExitDepth
|
||||
}
|
||||
backend_fallback_queue = @{
|
||||
degraded_route_queue = @{
|
||||
baseline_depth = $baselineBackendDepth
|
||||
depth = $finalBackendDepth
|
||||
}
|
||||
@@ -1083,7 +1083,7 @@ try {
|
||||
pre_feedback_primary_entry_delivered = ($preExitDepth -ge ($baselineExitDepth + $expectedPrePackets))
|
||||
post_feedback_alternate_entry_delivered = ($finalExitDepth -ge ($baselineExitDepth + $expectedPackets))
|
||||
post_feedback_uses_alternate_entry_route = ([string]$postIngress.report.ingress.last_selected_route_id -eq $alternateRouteID)
|
||||
no_backend_fallback_used = ($finalBackendDepth -eq $baselineBackendDepth)
|
||||
no_degraded_route_used = ($finalBackendDepth -eq $baselineBackendDepth)
|
||||
no_flow_drops = ($primaryDroppedDelta -eq 0 -and $alternateDroppedDelta -eq 0)
|
||||
route_intents_expired = ($expiredPrimary.route_intent.lifecycle_status -eq "expired" -and $expiredAlternate.route_intent.lifecycle_status -eq "expired")
|
||||
}
|
||||
@@ -1122,3 +1122,4 @@ if (-not (Test-Path $resultDir)) {
|
||||
$result | ConvertTo-Json -Depth 100 | Set-Content -Path $resultFullPath -Encoding UTF8
|
||||
Write-Host "C18Z11 live service-channel entry pool smoke passed. Result: $resultFullPath"
|
||||
$result
|
||||
|
||||
|
||||
@@ -85,7 +85,7 @@ function Get-MeshPort {
|
||||
function Enable-TestMeshListener {
|
||||
param([object]$Node)
|
||||
$port = Get-MeshPort -Name $Node.name
|
||||
Invoke-Api -Method PUT -Path "/clusters/$ClusterID/nodes/$($Node.id)/workloads/mesh-listener/desired" -Body @{
|
||||
Invoke-Api -Method PUT -Path "/clusters/$ClusterID/nodes/$($Node.id)/workloads/fabric-listener/desired" -Body @{
|
||||
actor_user_id = $ActorUserID
|
||||
desired_state = "enabled"
|
||||
runtime_mode = "container"
|
||||
@@ -640,7 +640,7 @@ try {
|
||||
score_reasons = $qualityLease.primary_route.score_reasons
|
||||
alternate_route_count = @($qualityLease.alternate_routes).Count
|
||||
}
|
||||
backend_fallback_queue = @{
|
||||
degraded_route_queue = @{
|
||||
baseline_depth = $baselineBackendDepth
|
||||
depth = $finalBackendDepth
|
||||
}
|
||||
@@ -665,7 +665,7 @@ try {
|
||||
candidate_runtime_loaded = ([string]$candidateLoadedConfig.report.config_version -ge [string]$candidateVisibleConfig.synthetic_mesh_config.config_version)
|
||||
quality_prefers_live_learned_fast_route = ([string]$qualityLease.primary_route.route_id -eq $fastRouteID)
|
||||
quality_lease_uses_live_feedback_reason = (@($qualityLease.primary_route.score_reasons | Where-Object { $_ -eq "service_channel_recent_success" }).Count -ge 1)
|
||||
no_backend_fallback_used = ($finalBackendDepth -eq $baselineBackendDepth)
|
||||
no_degraded_route_used = ($finalBackendDepth -eq $baselineBackendDepth)
|
||||
no_flow_drops = (($finalDropped - $baselineDropped) -eq 0)
|
||||
route_intents_expired = ($expiredFast.route_intent.lifecycle_status -eq "expired" -and $expiredSlowCandidate.route_intent.lifecycle_status -eq "expired")
|
||||
}
|
||||
@@ -711,3 +711,4 @@ if (-not (Test-Path $resultDir)) {
|
||||
$result | ConvertTo-Json -Depth 100 | Set-Content -Path $resultFullPath -Encoding UTF8
|
||||
Write-Host "C18Z13 live service-channel route quality smoke passed. Result: $resultFullPath"
|
||||
$result
|
||||
|
||||
|
||||
@@ -69,7 +69,7 @@ function Get-MeshPort {
|
||||
function Enable-TestMeshListener {
|
||||
param([object]$Node)
|
||||
$port = Get-MeshPort -Name $Node.name
|
||||
Invoke-Api -Method PUT -Path "/clusters/$ClusterID/nodes/$($Node.id)/workloads/mesh-listener/desired" -Body @{
|
||||
Invoke-Api -Method PUT -Path "/clusters/$ClusterID/nodes/$($Node.id)/workloads/fabric-listener/desired" -Body @{
|
||||
actor_user_id = $ActorUserID
|
||||
desired_state = "enabled"
|
||||
runtime_mode = "container"
|
||||
@@ -530,7 +530,7 @@ try {
|
||||
learned_depth = $learnedExitDepth
|
||||
final_depth = $finalExitDepth
|
||||
}
|
||||
backend_fallback_queue = @{ baseline_depth = $baselineBackendDepth; depth = $finalBackendDepth }
|
||||
degraded_route_queue = @{ baseline_depth = $baselineBackendDepth; depth = $finalBackendDepth }
|
||||
flow_drops = @{ baseline = $baselineDropped; final = $finalDropped; delta = ($finalDropped - $baselineDropped) }
|
||||
passed = $true
|
||||
checks = [ordered]@{
|
||||
@@ -546,7 +546,7 @@ try {
|
||||
node_applied_quality_preference = ([int]$qualityRuntime.report.ingress.route_quality_preference_count -gt 0)
|
||||
active_websocket_stayed_on_learned_fast_route_after_churn = ([string]$finalRuntime.report.ingress.last_selected_route_id -eq $fastRouteID)
|
||||
all_packets_reached_exit = ($finalExitDepth -ge ($baselineExitDepth + $initialPackets + $learningPackets + $postChurnPackets))
|
||||
no_backend_fallback_used = ($finalBackendDepth -eq $baselineBackendDepth)
|
||||
no_degraded_route_used = ($finalBackendDepth -eq $baselineBackendDepth)
|
||||
no_flow_drops = (($finalDropped - $baselineDropped) -eq 0)
|
||||
route_intents_expired = ($expiredFast.route_intent.lifecycle_status -eq "expired" -and $expiredSlowCandidate.route_intent.lifecycle_status -eq "expired")
|
||||
}
|
||||
@@ -584,3 +584,4 @@ if (-not (Test-Path $resultDir)) { New-Item -ItemType Directory -Path $resultDir
|
||||
$result | ConvertTo-Json -Depth 100 | Set-Content -Path $resultFullPath -Encoding UTF8
|
||||
Write-Host "C18Z14 live service-channel active quality shift smoke passed. Result: $resultFullPath"
|
||||
$result
|
||||
|
||||
|
||||
@@ -57,7 +57,7 @@ $result | Add-Member -NotePropertyName c18z17_checks -NotePropertyValue ([ordere
|
||||
stale_channel_quality_markers_absent = ($staleMarkers.Count -eq 0)
|
||||
active_markers_reference_visible_preferences = ($activeMarkers.Count -gt 0 -and $staleMarkers.Count -eq 0)
|
||||
expired_route_intents_not_active = ([string]$result.route_intents.fast_status -eq "expired" -and [string]$result.route_intents.slow_candidate_status -eq "expired" -and [string]$result.route_intents.slow_initial_status -eq "expired")
|
||||
session_completed_without_backend_fallback = ([int]$result.backend_fallback_queue.depth -eq 0)
|
||||
session_completed_without_degraded_route = ([int]$result.degraded_route_queue.depth -eq 0)
|
||||
}) -Force
|
||||
$result | Add-Member -NotePropertyName c18z17_summary -NotePropertyValue ([ordered]@{
|
||||
active_quality_marker_count = $activeMarkers.Count
|
||||
@@ -68,7 +68,7 @@ $result.passed = [bool]($result.passed -and
|
||||
$result.c18z17_checks.stale_channel_quality_markers_absent -and
|
||||
$result.c18z17_checks.active_markers_reference_visible_preferences -and
|
||||
$result.c18z17_checks.expired_route_intents_not_active -and
|
||||
$result.c18z17_checks.session_completed_without_backend_fallback)
|
||||
$result.c18z17_checks.session_completed_without_degraded_route)
|
||||
|
||||
$resolvedResultPath = Join-Path $repoRoot $ResultPath
|
||||
$result | ConvertTo-Json -Depth 100 | Set-Content -Path $resolvedResultPath -Encoding UTF8
|
||||
@@ -80,3 +80,4 @@ if (-not $result.passed) {
|
||||
|
||||
Write-Host "C18Z17 live service-channel quality cleanup smoke passed. Result: $resolvedResultPath"
|
||||
$result
|
||||
|
||||
|
||||
@@ -77,7 +77,7 @@ $result | Add-Member -NotePropertyName c18z18_checks -NotePropertyValue ([ordere
|
||||
live_served_channels_are_session_scoped = ($servedStats.Count -gt 0 -and $sessionScopedStats.Count -eq $servedStats.Count)
|
||||
live_unscoped_served_channels_absent = ($unscopedServedStats.Count -eq 0)
|
||||
live_quality_markers_are_session_scoped = ($sessionScopedQualityStats.Count -ge 2)
|
||||
live_session_completed_without_backend_fallback = ([int]$result.backend_fallback_queue.depth -eq 0)
|
||||
live_session_completed_without_degraded_route = ([int]$result.degraded_route_queue.depth -eq 0)
|
||||
live_session_completed_without_flow_drops = ([int]$result.flow_drops.delta -eq 0)
|
||||
}) -Force
|
||||
$result | Add-Member -NotePropertyName c18z18_summary -NotePropertyValue ([ordered]@{
|
||||
@@ -93,7 +93,7 @@ $result.passed = [bool]($result.passed -and
|
||||
$result.c18z18_checks.live_served_channels_are_session_scoped -and
|
||||
$result.c18z18_checks.live_unscoped_served_channels_absent -and
|
||||
$result.c18z18_checks.live_quality_markers_are_session_scoped -and
|
||||
$result.c18z18_checks.live_session_completed_without_backend_fallback -and
|
||||
$result.c18z18_checks.live_session_completed_without_degraded_route -and
|
||||
$result.c18z18_checks.live_session_completed_without_flow_drops)
|
||||
|
||||
$resolvedResultPath = Join-Path $repoRoot $ResultPath
|
||||
@@ -106,3 +106,4 @@ if (-not $result.passed) {
|
||||
|
||||
Write-Host "C18Z18 service-channel session-scoped fairness smoke passed. Result: $resolvedResultPath"
|
||||
$result
|
||||
|
||||
|
||||
@@ -68,7 +68,7 @@ $result | Add-Member -NotePropertyName c18z19_checks -NotePropertyValue ([ordere
|
||||
live_parallel_window_enabled = ($maxParallel -ge 2)
|
||||
live_parallel_batches_observed = ($parallelBatches -gt 0)
|
||||
live_session_scoped_fairness_still_passed = ([bool]$result.passed)
|
||||
live_session_completed_without_backend_fallback = ([int]$result.backend_fallback_queue.depth -eq 0)
|
||||
live_session_completed_without_degraded_route = ([int]$result.degraded_route_queue.depth -eq 0)
|
||||
live_session_completed_without_flow_drops = ([int]$result.flow_drops.delta -eq 0)
|
||||
}) -Force
|
||||
$result | Add-Member -NotePropertyName c18z19_summary -NotePropertyValue ([ordered]@{
|
||||
@@ -80,7 +80,7 @@ $result.passed = [bool]($result.passed -and
|
||||
$result.c18z19_checks.unit_parallel_flow_window_does_not_block_independent_channel -and
|
||||
$result.c18z19_checks.live_parallel_window_enabled -and
|
||||
$result.c18z19_checks.live_parallel_batches_observed -and
|
||||
$result.c18z19_checks.live_session_completed_without_backend_fallback -and
|
||||
$result.c18z19_checks.live_session_completed_without_degraded_route -and
|
||||
$result.c18z19_checks.live_session_completed_without_flow_drops)
|
||||
|
||||
$resolvedResultPath = Join-Path $repoRoot $ResultPath
|
||||
@@ -93,3 +93,4 @@ if (-not $result.passed) {
|
||||
|
||||
Write-Host "C18Z19 service-channel parallel flow window smoke passed. Result: $resolvedResultPath"
|
||||
$result
|
||||
|
||||
|
||||
@@ -70,7 +70,7 @@ function Get-MeshPort {
|
||||
function Enable-TestMeshListener {
|
||||
param([object]$Node)
|
||||
$port = Get-MeshPort -Name $Node.name
|
||||
Invoke-Api -Method PUT -Path "/clusters/$ClusterID/nodes/$($Node.id)/workloads/mesh-listener/desired" -Body @{
|
||||
Invoke-Api -Method PUT -Path "/clusters/$ClusterID/nodes/$($Node.id)/workloads/fabric-listener/desired" -Body @{
|
||||
actor_user_id = $ActorUserID
|
||||
desired_state = "enabled"
|
||||
runtime_mode = "container"
|
||||
|
||||
@@ -78,7 +78,7 @@ $result | Add-Member -NotePropertyName c18z20_checks -NotePropertyValue ([ordere
|
||||
live_inflight_telemetry_visible = ($maxInFlight -ge 2 -and $inFlight -eq 0)
|
||||
live_per_channel_attempt_telemetry_visible = ($attemptStats.Count -ge 2 -and $successStats.Count -ge 2)
|
||||
live_per_channel_latency_buckets_visible = ($latencyStats.Count -ge 2)
|
||||
live_parallel_path_still_clean = ([bool]$result.passed -and [int]$result.backend_fallback_queue.depth -eq 0 -and [int]$result.flow_drops.delta -eq 0)
|
||||
live_parallel_path_still_clean = ([bool]$result.passed -and [int]$result.degraded_route_queue.depth -eq 0 -and [int]$result.flow_drops.delta -eq 0)
|
||||
}) -Force
|
||||
$result | Add-Member -NotePropertyName c18z20_summary -NotePropertyValue ([ordered]@{
|
||||
unit_test_output = ($unitTestOutput | Out-String).Trim()
|
||||
@@ -108,3 +108,4 @@ if (-not $result.passed) {
|
||||
|
||||
Write-Host "C18Z20 service-channel adaptive window telemetry smoke passed. Result: $resolvedResultPath"
|
||||
$result
|
||||
|
||||
|
||||
@@ -81,7 +81,7 @@ $result | Add-Member -NotePropertyName c18z21_checks -NotePropertyValue ([ordere
|
||||
live_per_channel_window_success_visible = ($rollingSuccessStats.Count -ge 2)
|
||||
live_per_channel_window_latency_visible = ($rollingLatencyStats.Count -ge 2)
|
||||
live_adaptive_window_still_open = ($recommended -gt 0 -and $recommended -le $maxParallel)
|
||||
live_parallel_path_still_clean = ([bool]$result.passed -and [int]$result.backend_fallback_queue.depth -eq 0 -and [int]$result.flow_drops.delta -eq 0)
|
||||
live_parallel_path_still_clean = ([bool]$result.passed -and [int]$result.degraded_route_queue.depth -eq 0 -and [int]$result.flow_drops.delta -eq 0)
|
||||
}) -Force
|
||||
$result | Add-Member -NotePropertyName c18z21_summary -NotePropertyValue ([ordered]@{
|
||||
unit_test_output = ($unitTestOutput | Out-String).Trim()
|
||||
@@ -113,3 +113,4 @@ if (-not $result.passed) {
|
||||
|
||||
Write-Host "C18Z21 service-channel rolling quality window smoke passed. Result: $resolvedResultPath"
|
||||
$result
|
||||
|
||||
|
||||
@@ -61,7 +61,7 @@ $result | Add-Member -NotePropertyName c18z23_checks -NotePropertyValue ([ordere
|
||||
live_c18z22_still_passed = [bool]$result.passed
|
||||
backend_0_2_198_deployed = $backendImageOK
|
||||
live_rolling_feedback_still_visible = ($result.c18z22_summary.rolling_feedback_count -gt 0 -and $result.c18z22_summary.rolling_payload_count -gt 0)
|
||||
live_parallel_path_still_clean = ([int]$result.backend_fallback_queue.depth -eq 0 -and [int]$result.flow_drops.delta -eq 0)
|
||||
live_parallel_path_still_clean = ([int]$result.degraded_route_queue.depth -eq 0 -and [int]$result.flow_drops.delta -eq 0)
|
||||
}) -Force
|
||||
$result | Add-Member -NotePropertyName c18z23_summary -NotePropertyValue ([ordered]@{
|
||||
unit_test_output = ($unitTestOutput | Out-String).Trim()
|
||||
@@ -86,3 +86,4 @@ if (-not $result.passed) {
|
||||
|
||||
Write-Host "C18Z23 service-channel recovery hysteresis smoke passed. Result: $resolvedResultPath"
|
||||
$result
|
||||
|
||||
|
||||
@@ -71,7 +71,7 @@ function Get-MeshPort {
|
||||
function Enable-TestMeshListener {
|
||||
param([object]$Node)
|
||||
$port = Get-MeshPort -Name $Node.name
|
||||
Invoke-Api -Method PUT -Path "/clusters/$ClusterID/nodes/$($Node.id)/workloads/mesh-listener/desired" -Body @{
|
||||
Invoke-Api -Method PUT -Path "/clusters/$ClusterID/nodes/$($Node.id)/workloads/fabric-listener/desired" -Body @{
|
||||
actor_user_id = $ActorUserID
|
||||
desired_state = "enabled"
|
||||
runtime_mode = "container"
|
||||
@@ -881,7 +881,7 @@ try {
|
||||
post_entry_restart_baseline_depth = $postRestartExitBaseline
|
||||
recovery_depth = $recoveryExitDepth
|
||||
}
|
||||
backend_fallback_queue = @{
|
||||
degraded_route_queue = @{
|
||||
baseline_depth = $degradedBackendBaseline
|
||||
depth = $degradedBackendDepth
|
||||
}
|
||||
@@ -903,7 +903,7 @@ try {
|
||||
recovery_batches_accepted = ($recoveryOk -eq $RecoveryBatchCount)
|
||||
recovery_exit_inbox_grew = ($recoveryExitDepth -ge ($postRestartExitBaseline + 8))
|
||||
degraded_lease_visible = ($degradedLease.status -eq "degraded_fallback" -and $degradedLease.fallback.active -eq $true -and $degradedLease.fallback.degraded -eq $true)
|
||||
degraded_backend_fallback_received = ($degradedPost.ok -eq $true -and $degradedBackendDepth -ge ($degradedBackendBaseline + 8))
|
||||
degraded_degraded_route_received = ($degradedPost.ok -eq $true -and $degradedBackendDepth -ge ($degradedBackendBaseline + 8))
|
||||
route_intents_expired = ($expiredPrimary.route_intent.lifecycle_status -eq "expired" -and $expiredAlternate.route_intent.lifecycle_status -eq "expired")
|
||||
}
|
||||
telemetry = @{
|
||||
@@ -941,3 +941,4 @@ if (-not (Test-Path $resultDir)) {
|
||||
$result | ConvertTo-Json -Depth 100 | Set-Content -Path $resultFullPath -Encoding UTF8
|
||||
Write-Host "C18Z3 live service-channel entry/ws/fallback smoke passed. Result: $resultFullPath"
|
||||
$result
|
||||
|
||||
|
||||
@@ -72,7 +72,7 @@ function Get-MeshPort {
|
||||
function Enable-TestMeshListener {
|
||||
param([object]$Node)
|
||||
$port = Get-MeshPort -Name $Node.name
|
||||
Invoke-Api -Method PUT -Path "/clusters/$ClusterID/nodes/$($Node.id)/workloads/mesh-listener/desired" -Body @{
|
||||
Invoke-Api -Method PUT -Path "/clusters/$ClusterID/nodes/$($Node.id)/workloads/fabric-listener/desired" -Body @{
|
||||
actor_user_id = $ActorUserID
|
||||
desired_state = "enabled"
|
||||
runtime_mode = "container"
|
||||
@@ -1051,7 +1051,7 @@ try {
|
||||
baseline_depth = $baselineExitDepth
|
||||
depth = $exitDepth
|
||||
}
|
||||
backend_fallback_queue = @{
|
||||
degraded_route_queue = @{
|
||||
baseline_depth = $baselineBackendDepth
|
||||
depth = $finalBackendDepth
|
||||
}
|
||||
@@ -1071,7 +1071,7 @@ try {
|
||||
entry_runtime_loaded_post_switch_config = ([string]$postSwitchLoadedConfig.report.config_version -ge [string]$entryConfigWithoutPrimary.synthetic_mesh_config.config_version)
|
||||
post_switch_uses_alternate_route = ([bool]$postSwitchRouteStats.matched)
|
||||
exit_inbox_received_all_packets = ($exitDepth -ge ($baselineExitDepth + $expectedPackets))
|
||||
no_backend_fallback_used = ($finalBackendDepth -eq $baselineBackendDepth)
|
||||
no_degraded_route_used = ($finalBackendDepth -eq $baselineBackendDepth)
|
||||
route_failures_within_budget = ($routeFailureDelta -le $MaxAllowedRouteFailures)
|
||||
no_flow_drops = ($droppedDelta -eq 0)
|
||||
route_intents_expired = ($expiredPrimary.route_intent.lifecycle_status -eq "expired" -and $expiredAlternate.route_intent.lifecycle_status -eq "expired")
|
||||
@@ -1109,3 +1109,4 @@ if (-not (Test-Path $resultDir)) {
|
||||
$result | ConvertTo-Json -Depth 100 | Set-Content -Path $resultFullPath -Encoding UTF8
|
||||
Write-Host "C18Z4 live service-channel session pressure smoke passed. Result: $resultFullPath"
|
||||
$result
|
||||
|
||||
|
||||
@@ -73,7 +73,7 @@ function Get-MeshPort {
|
||||
function Enable-TestMeshListener {
|
||||
param([object]$Node)
|
||||
$port = Get-MeshPort -Name $Node.name
|
||||
Invoke-Api -Method PUT -Path "/clusters/$ClusterID/nodes/$($Node.id)/workloads/mesh-listener/desired" -Body @{
|
||||
Invoke-Api -Method PUT -Path "/clusters/$ClusterID/nodes/$($Node.id)/workloads/fabric-listener/desired" -Body @{
|
||||
actor_user_id = $ActorUserID
|
||||
desired_state = "enabled"
|
||||
runtime_mode = "container"
|
||||
@@ -1136,7 +1136,7 @@ try {
|
||||
baseline_depth = $baselineExitDepth
|
||||
depth = $exitDepth
|
||||
}
|
||||
backend_fallback_queue = @{
|
||||
degraded_route_queue = @{
|
||||
baseline_depth = $baselineBackendDepth
|
||||
depth = $finalBackendDepth
|
||||
delta = $backendFallbackDelta
|
||||
@@ -1157,7 +1157,7 @@ try {
|
||||
long_lived_websocket_sent_all_batches = ($webSocketResult.ok -eq $true -and $webSocketResult.sent_batches -eq ($PreOutageBatchCount + $DuringOutageBatchCount + $RecoveryBatchCount) -and $webSocketResult.sent_packets -eq $expectedPackets)
|
||||
exit_restart_attempted_during_session = ($webSocketResult.outage_action_ran -eq $true -and $webSocketResult.recovery_action_ran -eq $true)
|
||||
route_failures_observed = ($routeFailureDelta -gt 0)
|
||||
backend_fallback_visible = ($backendFallbackDelta -gt 0)
|
||||
degraded_route_visible = ($backendFallbackDelta -gt 0)
|
||||
recovery_packets_reached_exit = ($exitDepth -ge ($baselineExitDepth + $recoveryPackets))
|
||||
post_recovery_fabric_send_observed = ([int]$postRecoveryIngress.report.ingress.send_packets -ge ($baselineSendPackets + $expectedPackets))
|
||||
no_flow_drops = ($droppedDelta -eq 0)
|
||||
@@ -1200,3 +1200,4 @@ if (-not (Test-Path $resultDir)) {
|
||||
$result | ConvertTo-Json -Depth 100 | Set-Content -Path $resultFullPath -Encoding UTF8
|
||||
Write-Host "C18Z5 live service-channel exit-restart smoke passed. Result: $resultFullPath"
|
||||
$result
|
||||
|
||||
|
||||
@@ -124,7 +124,7 @@ $result = [ordered]@{
|
||||
total_accepted = [int]$accessTelemetry.total_accepted
|
||||
signed_accepted = [int]$accessTelemetry.signed_accepted
|
||||
introspection_accepted = [int]$accessTelemetry.introspection_accepted
|
||||
backend_fallback_count = [int]$accessTelemetry.backend_fallback_count
|
||||
degraded_route_use_count = [int]$accessTelemetry.degraded_route_use_count
|
||||
}
|
||||
}
|
||||
|
||||
@@ -143,3 +143,4 @@ Write-Host "C18Z52 service-channel access telemetry smoke passed. Result: $resul
|
||||
$result
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -203,7 +203,7 @@ try {
|
||||
$acceptedBy -eq "introspection" -and
|
||||
$null -ne $matchingChannel -and
|
||||
[string]$matchingChannel.primary_route_id -eq $routeID -and
|
||||
-not [bool]$matchingChannel.force_backend_fallback -and
|
||||
-not [bool]$matchingChannel.force_degraded_route -and
|
||||
[string]$matchingChannel.route_feedback_status -eq "healthy" -and
|
||||
[int]$matchingChannel.route_quality_window_sample_count -ge 6
|
||||
)
|
||||
@@ -216,7 +216,7 @@ try {
|
||||
accepted_by_header_is_introspection = ($acceptedBy -eq "introspection")
|
||||
active_channel_visible = ($null -ne $matchingChannel)
|
||||
active_channel_uses_primary_route = ($null -ne $matchingChannel -and [string]$matchingChannel.primary_route_id -eq $routeID)
|
||||
active_channel_not_backend_fallback = ($null -ne $matchingChannel -and -not [bool]$matchingChannel.force_backend_fallback)
|
||||
active_channel_not_degraded_route = ($null -ne $matchingChannel -and -not [bool]$matchingChannel.force_degraded_route)
|
||||
route_quality_correlated = ($null -ne $matchingChannel -and [string]$matchingChannel.route_feedback_status -eq "healthy" -and [int]$matchingChannel.route_quality_window_sample_count -ge 6)
|
||||
}
|
||||
summary = [ordered]@{
|
||||
@@ -254,3 +254,4 @@ Write-Host "C18Z54 service-channel normal route access smoke passed. Result: $re
|
||||
$result
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -223,7 +223,7 @@ try {
|
||||
$acceptedBy -eq "introspection" -and
|
||||
$null -ne $matchingChannel -and
|
||||
[string](Get-PropertyValue -Item $matchingChannel -Name "primary_route_id" -Default "") -eq $routeID -and
|
||||
-not [bool](Get-PropertyValue -Item $matchingChannel -Name "force_backend_fallback" -Default $false) -and
|
||||
-not [bool](Get-PropertyValue -Item $matchingChannel -Name "force_degraded_route" -Default $false) -and
|
||||
@("degraded", "fenced") -contains [string](Get-PropertyValue -Item $matchingChannel -Name "route_feedback_status" -Default "") -and
|
||||
[int](Get-PropertyValue -Item $matchingChannel -Name "route_quality_window_failure_count" -Default 0) -ge 3 -and
|
||||
[int](Get-PropertyValue -Item $matchingChannel -Name "route_quality_window_drop_count" -Default 0) -ge 1 -and
|
||||
@@ -239,7 +239,7 @@ try {
|
||||
accepted_by_header_is_introspection = ($acceptedBy -eq "introspection")
|
||||
active_channel_visible = ($null -ne $matchingChannel)
|
||||
active_channel_uses_primary_route = ($null -ne $matchingChannel -and [string](Get-PropertyValue -Item $matchingChannel -Name "primary_route_id" -Default "") -eq $routeID)
|
||||
active_channel_not_backend_fallback = ($null -ne $matchingChannel -and -not [bool](Get-PropertyValue -Item $matchingChannel -Name "force_backend_fallback" -Default $false))
|
||||
active_channel_not_degraded_route = ($null -ne $matchingChannel -and -not [bool](Get-PropertyValue -Item $matchingChannel -Name "force_degraded_route" -Default $false))
|
||||
route_quality_degraded = ($null -ne $matchingChannel -and @("degraded", "fenced") -contains [string](Get-PropertyValue -Item $matchingChannel -Name "route_feedback_status" -Default ""))
|
||||
route_quality_counters_visible = ($null -ne $matchingChannel -and [int](Get-PropertyValue -Item $matchingChannel -Name "route_quality_window_failure_count" -Default 0) -ge 3 -and [int](Get-PropertyValue -Item $matchingChannel -Name "route_quality_window_drop_count" -Default 0) -ge 1)
|
||||
remediation_recommends_rebuild = ($null -ne $matchingChannel -and [string](Get-PropertyValue -Item $matchingChannel -Name "remediation_action" -Default "") -eq "rebuild_route")
|
||||
@@ -281,3 +281,4 @@ Write-Host "C18Z55 service-channel degraded route access smoke passed. Result: $
|
||||
$result
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -235,7 +235,7 @@ try {
|
||||
$acceptedBy -eq "introspection" -and
|
||||
$null -ne $matchingChannel -and
|
||||
[string](Get-PropertyValue -Item $matchingChannel -Name "primary_route_id" -Default "") -eq $primaryRouteID -and
|
||||
-not [bool](Get-PropertyValue -Item $matchingChannel -Name "force_backend_fallback" -Default $false) -and
|
||||
-not [bool](Get-PropertyValue -Item $matchingChannel -Name "force_degraded_route" -Default $false) -and
|
||||
[string](Get-PropertyValue -Item $matchingChannel -Name "route_feedback_status" -Default "") -eq "fenced" -and
|
||||
[string](Get-PropertyValue -Item $matchingChannel -Name "remediation_action" -Default "") -eq "prefer_alternate_route" -and
|
||||
[string](Get-PropertyValue -Item $matchingChannel -Name "remediation_route_id" -Default "") -eq $alternateRouteID -and
|
||||
@@ -250,11 +250,11 @@ try {
|
||||
packet_accepted = ([int]$response.StatusCode -eq 202)
|
||||
accepted_by_header_is_introspection = ($acceptedBy -eq "introspection")
|
||||
active_channel_visible = ($null -ne $matchingChannel)
|
||||
active_channel_not_backend_fallback = ($null -ne $matchingChannel -and -not [bool](Get-PropertyValue -Item $matchingChannel -Name "force_backend_fallback" -Default $false))
|
||||
active_channel_not_degraded_route = ($null -ne $matchingChannel -and -not [bool](Get-PropertyValue -Item $matchingChannel -Name "force_degraded_route" -Default $false))
|
||||
route_quality_fenced = ($null -ne $matchingChannel -and [string](Get-PropertyValue -Item $matchingChannel -Name "route_feedback_status" -Default "") -eq "fenced")
|
||||
remediation_prefers_alternate = ($null -ne $matchingChannel -and [string](Get-PropertyValue -Item $matchingChannel -Name "remediation_action" -Default "") -eq "prefer_alternate_route")
|
||||
remediation_route_is_alternate = ($null -ne $matchingChannel -and [string](Get-PropertyValue -Item $matchingChannel -Name "remediation_route_id" -Default "") -eq $alternateRouteID)
|
||||
backend_fallback_not_recommended = ([int]$accessTelemetry.degraded_fallback_channel_count -eq 0)
|
||||
degraded_route_not_recommended = ([int]$accessTelemetry.degraded_fallback_channel_count -eq 0)
|
||||
}
|
||||
summary = [ordered]@{
|
||||
backend_container = $backendLine.Trim()
|
||||
@@ -296,3 +296,4 @@ Write-Host "C18Z56 service-channel alternate remediation smoke passed. Result: $
|
||||
$result
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -240,7 +240,7 @@ try {
|
||||
$acceptedBy -eq "introspection" -and
|
||||
$null -ne $matchingChannel -and
|
||||
[string](Get-PropertyValue -Item $matchingChannel -Name "primary_route_id" -Default "") -eq $primaryRouteID -and
|
||||
-not [bool](Get-PropertyValue -Item $matchingChannel -Name "force_backend_fallback" -Default $false) -and
|
||||
-not [bool](Get-PropertyValue -Item $matchingChannel -Name "force_degraded_route" -Default $false) -and
|
||||
[string](Get-PropertyValue -Item $matchingChannel -Name "route_feedback_status" -Default "") -eq "fenced" -and
|
||||
[string](Get-PropertyValue -Item $matchingChannel -Name "remediation_action" -Default "") -eq "prefer_alternate_route" -and
|
||||
[string](Get-PropertyValue -Item $matchingChannel -Name "remediation_route_id" -Default "") -eq $alternateRouteID -and
|
||||
@@ -260,7 +260,7 @@ try {
|
||||
packet_accepted = ([int]$response.StatusCode -eq 202)
|
||||
accepted_by_header_is_introspection = ($acceptedBy -eq "introspection")
|
||||
active_channel_visible = ($null -ne $matchingChannel)
|
||||
active_channel_not_backend_fallback = ($null -ne $matchingChannel -and -not [bool](Get-PropertyValue -Item $matchingChannel -Name "force_backend_fallback" -Default $false))
|
||||
active_channel_not_degraded_route = ($null -ne $matchingChannel -and -not [bool](Get-PropertyValue -Item $matchingChannel -Name "force_degraded_route" -Default $false))
|
||||
route_quality_fenced = ($null -ne $matchingChannel -and [string](Get-PropertyValue -Item $matchingChannel -Name "route_feedback_status" -Default "") -eq "fenced")
|
||||
remediation_prefers_alternate = ($null -ne $matchingChannel -and [string](Get-PropertyValue -Item $matchingChannel -Name "remediation_action" -Default "") -eq "prefer_alternate_route")
|
||||
remediation_route_is_alternate = ($null -ne $matchingChannel -and [string](Get-PropertyValue -Item $matchingChannel -Name "remediation_route_id" -Default "") -eq $alternateRouteID)
|
||||
@@ -269,7 +269,7 @@ try {
|
||||
remediation_command_primary_route_matches = ($commandPrimaryRouteID -eq $primaryRouteID)
|
||||
remediation_command_replacement_route_matches = ($commandReplacementRouteID -eq $alternateRouteID)
|
||||
remediation_command_has_ttl = ($commandExpiresAt.Length -gt 0)
|
||||
backend_fallback_not_recommended = ([int]$accessTelemetry.degraded_fallback_channel_count -eq 0)
|
||||
degraded_route_not_recommended = ([int]$accessTelemetry.degraded_fallback_channel_count -eq 0)
|
||||
}
|
||||
summary = [ordered]@{
|
||||
backend_container = $backendLine.Trim()
|
||||
@@ -311,3 +311,4 @@ Write-Host "C18Z57 service-channel remediation command smoke passed. Result: $re
|
||||
$result
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -274,7 +274,7 @@ try {
|
||||
$acceptedBy -eq "introspection" -and
|
||||
$null -ne $matchingChannel -and
|
||||
[string](Get-PropertyValue -Item $matchingChannel -Name "primary_route_id" -Default "") -eq $primaryRouteID -and
|
||||
-not [bool](Get-PropertyValue -Item $matchingChannel -Name "force_backend_fallback" -Default $false) -and
|
||||
-not [bool](Get-PropertyValue -Item $matchingChannel -Name "force_degraded_route" -Default $false) -and
|
||||
[string](Get-PropertyValue -Item $matchingChannel -Name "route_feedback_status" -Default "") -eq "fenced" -and
|
||||
[string](Get-PropertyValue -Item $matchingChannel -Name "remediation_action" -Default "") -eq "prefer_alternate_route" -and
|
||||
[string](Get-PropertyValue -Item $matchingChannel -Name "remediation_route_id" -Default "") -eq $alternateRouteID -and
|
||||
@@ -299,7 +299,7 @@ try {
|
||||
packet_accepted = ([int]$response.StatusCode -eq 202)
|
||||
accepted_by_header_is_introspection = ($acceptedBy -eq "introspection")
|
||||
active_channel_visible = ($null -ne $matchingChannel)
|
||||
active_channel_not_backend_fallback = ($null -ne $matchingChannel -and -not [bool](Get-PropertyValue -Item $matchingChannel -Name "force_backend_fallback" -Default $false))
|
||||
active_channel_not_degraded_route = ($null -ne $matchingChannel -and -not [bool](Get-PropertyValue -Item $matchingChannel -Name "force_degraded_route" -Default $false))
|
||||
route_quality_fenced = ($null -ne $matchingChannel -and [string](Get-PropertyValue -Item $matchingChannel -Name "route_feedback_status" -Default "") -eq "fenced")
|
||||
remediation_prefers_alternate = ($null -ne $matchingChannel -and [string](Get-PropertyValue -Item $matchingChannel -Name "remediation_action" -Default "") -eq "prefer_alternate_route")
|
||||
remediation_route_is_alternate = ($null -ne $matchingChannel -and [string](Get-PropertyValue -Item $matchingChannel -Name "remediation_route_id" -Default "") -eq $alternateRouteID)
|
||||
@@ -313,7 +313,7 @@ try {
|
||||
synthetic_config_command_replacement_route_matches = ($syntheticCommandReplacementRouteID -eq $alternateRouteID)
|
||||
node_route_manager_consumed_command = ($null -ne $routeManagerDecision)
|
||||
node_route_manager_applied_replacement = ($null -ne $routeManagerDecision -and [string](Get-PropertyValue -Item $routeManagerDecision -Name "rebuild_status" -Default "") -eq "applied")
|
||||
backend_fallback_not_recommended = ([int]$accessTelemetry.degraded_fallback_channel_count -eq 0)
|
||||
degraded_route_not_recommended = ([int]$accessTelemetry.degraded_fallback_channel_count -eq 0)
|
||||
}
|
||||
summary = [ordered]@{
|
||||
backend_container = $backendLine.Trim()
|
||||
@@ -357,3 +357,4 @@ Write-Host "C18Z58 service-channel remediation apply smoke passed. Result: $resu
|
||||
$result
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -313,7 +313,7 @@ try {
|
||||
$acceptedBy -eq "introspection" -and
|
||||
$null -ne $matchingChannel -and
|
||||
[string](Get-PropertyValue -Item $matchingChannel -Name "primary_route_id" -Default "") -eq $primaryRouteID -and
|
||||
-not [bool](Get-PropertyValue -Item $matchingChannel -Name "force_backend_fallback" -Default $false) -and
|
||||
-not [bool](Get-PropertyValue -Item $matchingChannel -Name "force_degraded_route" -Default $false) -and
|
||||
[string](Get-PropertyValue -Item $matchingChannel -Name "route_feedback_status" -Default "") -eq "fenced" -and
|
||||
[string](Get-PropertyValue -Item $matchingChannel -Name "remediation_action" -Default "") -eq "prefer_alternate_route" -and
|
||||
[string](Get-PropertyValue -Item $matchingChannel -Name "remediation_route_id" -Default "") -eq $alternateRouteID -and
|
||||
@@ -343,7 +343,7 @@ try {
|
||||
packet_accepted = ([int]$response.StatusCode -eq 202)
|
||||
accepted_by_header_is_introspection = ($acceptedBy -eq "introspection")
|
||||
active_channel_visible = ($null -ne $matchingChannel)
|
||||
active_channel_not_backend_fallback = ($null -ne $matchingChannel -and -not [bool](Get-PropertyValue -Item $matchingChannel -Name "force_backend_fallback" -Default $false))
|
||||
active_channel_not_degraded_route = ($null -ne $matchingChannel -and -not [bool](Get-PropertyValue -Item $matchingChannel -Name "force_degraded_route" -Default $false))
|
||||
route_quality_fenced = ($null -ne $matchingChannel -and [string](Get-PropertyValue -Item $matchingChannel -Name "route_feedback_status" -Default "") -eq "fenced")
|
||||
remediation_prefers_alternate = ($null -ne $matchingChannel -and [string](Get-PropertyValue -Item $matchingChannel -Name "remediation_action" -Default "") -eq "prefer_alternate_route")
|
||||
remediation_route_is_alternate = ($null -ne $matchingChannel -and [string](Get-PropertyValue -Item $matchingChannel -Name "remediation_route_id" -Default "") -eq $alternateRouteID)
|
||||
@@ -363,7 +363,7 @@ try {
|
||||
replacement_last_selected_route_matches = ($replacementLastSelected -eq $alternateRouteID)
|
||||
replacement_flow_stat_observed = ($null -ne $replacementFlowStat)
|
||||
no_local_gateway_fallback_after_replacement = ($postRemediationFallbackLocal -eq 0)
|
||||
backend_fallback_not_recommended = ([int]$accessTelemetry.degraded_fallback_channel_count -eq 0)
|
||||
degraded_route_not_recommended = ([int]$accessTelemetry.degraded_fallback_channel_count -eq 0)
|
||||
}
|
||||
summary = [ordered]@{
|
||||
backend_container = $backendLine.Trim()
|
||||
@@ -414,3 +414,4 @@ $result
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -71,7 +71,7 @@ function Get-MeshPort {
|
||||
function Enable-TestMeshListener {
|
||||
param([object]$Node)
|
||||
$port = Get-MeshPort -Name $Node.name
|
||||
Invoke-Api -Method PUT -Path "/clusters/$ClusterID/nodes/$($Node.id)/workloads/mesh-listener/desired" -Body @{
|
||||
Invoke-Api -Method PUT -Path "/clusters/$ClusterID/nodes/$($Node.id)/workloads/fabric-listener/desired" -Body @{
|
||||
actor_user_id = $ActorUserID
|
||||
desired_state = "enabled"
|
||||
runtime_mode = "container"
|
||||
@@ -1000,7 +1000,7 @@ try {
|
||||
baseline_depth = $baselineExitDepth
|
||||
depth = $exitDepth
|
||||
}
|
||||
backend_fallback_queue = @{
|
||||
degraded_route_queue = @{
|
||||
baseline_depth = $baselineBackendDepth
|
||||
depth = $finalBackendDepth
|
||||
}
|
||||
@@ -1020,7 +1020,7 @@ try {
|
||||
entry_runtime_loaded_rebuild_config = ([string]$postRebuildLoadedConfig.report.config_version -ge [string]$appliedDecision.config.synthetic_mesh_config.config_version)
|
||||
post_rebuild_uses_alternate_route = ([string]$postRebuildIngress.report.ingress.last_selected_route_id -eq $alternateRouteID)
|
||||
exit_inbox_received_all_packets = ($exitDepth -ge ($baselineExitDepth + $expectedPackets))
|
||||
no_backend_fallback_used = ($finalBackendDepth -eq $baselineBackendDepth)
|
||||
no_degraded_route_used = ($finalBackendDepth -eq $baselineBackendDepth)
|
||||
no_flow_drops = ($droppedDelta -eq 0)
|
||||
route_intents_expired = ($expiredPrimary.route_intent.lifecycle_status -eq "expired" -and $expiredAlternate.route_intent.lifecycle_status -eq "expired")
|
||||
}
|
||||
@@ -1059,3 +1059,4 @@ if (-not (Test-Path $resultDir)) {
|
||||
$result | ConvertTo-Json -Depth 100 | Set-Content -Path $resultFullPath -Encoding UTF8
|
||||
Write-Host "C18Z6 live service-channel active rebuild smoke passed. Result: $resultFullPath"
|
||||
$result
|
||||
|
||||
|
||||
@@ -379,7 +379,7 @@ try {
|
||||
$acceptedBy -eq "introspection" -and
|
||||
$null -ne $matchingChannel -and
|
||||
[string](Get-PropertyValue -Item $matchingChannel -Name "primary_route_id" -Default "") -eq $primaryRouteID -and
|
||||
-not [bool](Get-PropertyValue -Item $matchingChannel -Name "force_backend_fallback" -Default $false) -and
|
||||
-not [bool](Get-PropertyValue -Item $matchingChannel -Name "force_degraded_route" -Default $false) -and
|
||||
[string](Get-PropertyValue -Item $matchingChannel -Name "route_feedback_status" -Default "") -eq "fenced" -and
|
||||
[string](Get-PropertyValue -Item $matchingChannel -Name "remediation_action" -Default "") -eq "prefer_alternate_route" -and
|
||||
[string](Get-PropertyValue -Item $matchingChannel -Name "remediation_route_id" -Default "") -eq $expectedReplacementRouteID -and
|
||||
@@ -413,7 +413,7 @@ try {
|
||||
packet_accepted = ([int]$response.StatusCode -eq 202)
|
||||
accepted_by_header_is_introspection = ($acceptedBy -eq "introspection")
|
||||
active_channel_visible = ($null -ne $matchingChannel)
|
||||
active_channel_not_backend_fallback = ($null -ne $matchingChannel -and -not [bool](Get-PropertyValue -Item $matchingChannel -Name "force_backend_fallback" -Default $false))
|
||||
active_channel_not_degraded_route = ($null -ne $matchingChannel -and -not [bool](Get-PropertyValue -Item $matchingChannel -Name "force_degraded_route" -Default $false))
|
||||
route_quality_fenced = ($null -ne $matchingChannel -and [string](Get-PropertyValue -Item $matchingChannel -Name "route_feedback_status" -Default "") -eq "fenced")
|
||||
remediation_prefers_alternate = ($null -ne $matchingChannel -and [string](Get-PropertyValue -Item $matchingChannel -Name "remediation_action" -Default "") -eq "prefer_alternate_route")
|
||||
remediation_route_is_alternate = ($null -ne $matchingChannel -and [string](Get-PropertyValue -Item $matchingChannel -Name "remediation_route_id" -Default "") -eq $expectedReplacementRouteID)
|
||||
@@ -436,7 +436,7 @@ try {
|
||||
no_local_gateway_fallback_after_replacement = ($postRemediationFallbackLocal -eq 0)
|
||||
no_flow_drops_after_replacement = ($postRemediationFlowDropped -eq 0 -and $postRemediationSchedulerDropped -eq 0)
|
||||
no_route_failures_after_replacement = ($postRemediationRouteFailures -eq 0)
|
||||
backend_fallback_not_recommended = ([int]$accessTelemetry.degraded_fallback_channel_count -eq 0)
|
||||
degraded_route_not_recommended = ([int]$accessTelemetry.degraded_fallback_channel_count -eq 0)
|
||||
}
|
||||
summary = [ordered]@{
|
||||
backend_container = $backendLine.Trim()
|
||||
@@ -497,3 +497,4 @@ $result
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -382,7 +382,7 @@ try {
|
||||
$acceptedBy -eq "introspection" -and
|
||||
$null -ne $matchingChannel -and
|
||||
[string](Get-PropertyValue -Item $matchingChannel -Name "primary_route_id" -Default "") -eq $primaryRouteID -and
|
||||
-not [bool](Get-PropertyValue -Item $matchingChannel -Name "force_backend_fallback" -Default $false) -and
|
||||
-not [bool](Get-PropertyValue -Item $matchingChannel -Name "force_degraded_route" -Default $false) -and
|
||||
[string](Get-PropertyValue -Item $matchingChannel -Name "route_feedback_status" -Default "") -eq "fenced" -and
|
||||
[string](Get-PropertyValue -Item $matchingChannel -Name "remediation_action" -Default "") -eq "prefer_alternate_route" -and
|
||||
[string](Get-PropertyValue -Item $matchingChannel -Name "remediation_route_id" -Default "") -eq $expectedReplacementRouteID -and
|
||||
@@ -417,7 +417,7 @@ try {
|
||||
packet_accepted = ([int]$response.StatusCode -eq 202)
|
||||
accepted_by_header_is_introspection = ($acceptedBy -eq "introspection")
|
||||
active_channel_visible = ($null -ne $matchingChannel)
|
||||
active_channel_not_backend_fallback = ($null -ne $matchingChannel -and -not [bool](Get-PropertyValue -Item $matchingChannel -Name "force_backend_fallback" -Default $false))
|
||||
active_channel_not_degraded_route = ($null -ne $matchingChannel -and -not [bool](Get-PropertyValue -Item $matchingChannel -Name "force_degraded_route" -Default $false))
|
||||
route_quality_fenced = ($null -ne $matchingChannel -and [string](Get-PropertyValue -Item $matchingChannel -Name "route_feedback_status" -Default "") -eq "fenced")
|
||||
remediation_prefers_alternate = ($null -ne $matchingChannel -and [string](Get-PropertyValue -Item $matchingChannel -Name "remediation_action" -Default "") -eq "prefer_alternate_route")
|
||||
remediation_route_is_alternate = ($null -ne $matchingChannel -and [string](Get-PropertyValue -Item $matchingChannel -Name "remediation_route_id" -Default "") -eq $expectedReplacementRouteID)
|
||||
@@ -441,7 +441,7 @@ try {
|
||||
no_local_gateway_fallback_after_replacement = ($postRemediationFallbackLocal -eq 0)
|
||||
no_flow_drops_after_replacement = ($postRemediationFlowDropped -eq 0 -and $postRemediationSchedulerDropped -eq 0)
|
||||
no_route_failures_after_replacement = ($postRemediationRouteFailures -eq 0)
|
||||
backend_fallback_not_recommended = ([int]$accessTelemetry.degraded_fallback_channel_count -eq 0)
|
||||
degraded_route_not_recommended = ([int]$accessTelemetry.degraded_fallback_channel_count -eq 0)
|
||||
}
|
||||
summary = [ordered]@{
|
||||
backend_container = $backendLine.Trim()
|
||||
@@ -504,3 +504,4 @@ $result
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -404,7 +404,7 @@ try {
|
||||
$acceptedBy -eq "introspection" -and
|
||||
$null -ne $matchingChannel -and
|
||||
[string](Get-PropertyValue -Item $matchingChannel -Name "primary_route_id" -Default "") -eq $primaryRouteID -and
|
||||
-not [bool](Get-PropertyValue -Item $matchingChannel -Name "force_backend_fallback" -Default $false) -and
|
||||
-not [bool](Get-PropertyValue -Item $matchingChannel -Name "force_degraded_route" -Default $false) -and
|
||||
[string](Get-PropertyValue -Item $matchingChannel -Name "route_feedback_status" -Default "") -eq "fenced" -and
|
||||
[string](Get-PropertyValue -Item $matchingChannel -Name "remediation_action" -Default "") -eq "prefer_alternate_route" -and
|
||||
[string](Get-PropertyValue -Item $matchingChannel -Name "remediation_route_id" -Default "") -eq $expectedReplacementRouteID -and
|
||||
@@ -442,7 +442,7 @@ try {
|
||||
packet_accepted = ([int]$response.StatusCode -eq 202)
|
||||
accepted_by_header_is_introspection = ($acceptedBy -eq "introspection")
|
||||
active_channel_visible = ($null -ne $matchingChannel)
|
||||
active_channel_not_backend_fallback = ($null -ne $matchingChannel -and -not [bool](Get-PropertyValue -Item $matchingChannel -Name "force_backend_fallback" -Default $false))
|
||||
active_channel_not_degraded_route = ($null -ne $matchingChannel -and -not [bool](Get-PropertyValue -Item $matchingChannel -Name "force_degraded_route" -Default $false))
|
||||
route_quality_fenced = ($null -ne $matchingChannel -and [string](Get-PropertyValue -Item $matchingChannel -Name "route_feedback_status" -Default "") -eq "fenced")
|
||||
remediation_prefers_alternate = ($null -ne $matchingChannel -and [string](Get-PropertyValue -Item $matchingChannel -Name "remediation_action" -Default "") -eq "prefer_alternate_route")
|
||||
remediation_route_is_alternate = ($null -ne $matchingChannel -and [string](Get-PropertyValue -Item $matchingChannel -Name "remediation_route_id" -Default "") -eq $expectedReplacementRouteID)
|
||||
@@ -469,7 +469,7 @@ try {
|
||||
no_local_gateway_fallback_after_replacement = ($postRemediationFallbackLocal -eq 0)
|
||||
no_flow_drops_after_replacement = ($postRemediationFlowDropped -eq 0 -and $postRemediationSchedulerDropped -eq 0)
|
||||
no_route_failures_after_replacement = ($postRemediationRouteFailures -eq 0)
|
||||
backend_fallback_not_recommended = ([int]$accessTelemetry.degraded_fallback_channel_count -eq 0)
|
||||
degraded_route_not_recommended = ([int]$accessTelemetry.degraded_fallback_channel_count -eq 0)
|
||||
}
|
||||
summary = [ordered]@{
|
||||
backend_container = $backendLine.Trim()
|
||||
@@ -539,3 +539,4 @@ $result
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -463,7 +463,7 @@ try {
|
||||
$acceptedBy -eq "introspection" -and
|
||||
$null -ne $matchingChannel -and
|
||||
[string](Get-PropertyValue -Item $matchingChannel -Name "primary_route_id" -Default "") -eq $primaryRouteID -and
|
||||
-not [bool](Get-PropertyValue -Item $matchingChannel -Name "force_backend_fallback" -Default $false) -and
|
||||
-not [bool](Get-PropertyValue -Item $matchingChannel -Name "force_degraded_route" -Default $false) -and
|
||||
[string](Get-PropertyValue -Item $matchingChannel -Name "route_feedback_status" -Default "") -eq "fenced" -and
|
||||
[string](Get-PropertyValue -Item $matchingChannel -Name "remediation_action" -Default "") -eq "prefer_alternate_route" -and
|
||||
[string](Get-PropertyValue -Item $matchingChannel -Name "remediation_route_id" -Default "") -eq $expectedReplacementRouteID -and
|
||||
@@ -492,8 +492,8 @@ try {
|
||||
($postRemediationSchedulerDropped - $baselineSchedulerDropped) -eq 0 -and
|
||||
($postRemediationRouteFailures - $baselineRouteFailures) -eq 0 -and
|
||||
$null -ne $matchingChannel -and
|
||||
-not [bool](Get-PropertyValue -Item $matchingChannel -Name "force_backend_fallback" -Default $false) -and
|
||||
[string](Get-PropertyValue -Item $matchingChannel -Name "remediation_action" -Default "") -ne "use_backend_fallback"
|
||||
-not [bool](Get-PropertyValue -Item $matchingChannel -Name "force_degraded_route" -Default $false) -and
|
||||
[string](Get-PropertyValue -Item $matchingChannel -Name "remediation_action" -Default "") -ne "use_degraded_route"
|
||||
)
|
||||
checks = [ordered]@{
|
||||
backend_expected_image_deployed = $backendLine.Contains($ExpectedBackendImage)
|
||||
@@ -504,7 +504,7 @@ try {
|
||||
packet_accepted = ([int]$response.StatusCode -eq 202)
|
||||
accepted_by_header_is_introspection = ($acceptedBy -eq "introspection")
|
||||
active_channel_visible = ($null -ne $matchingChannel)
|
||||
active_channel_not_backend_fallback = ($null -ne $matchingChannel -and -not [bool](Get-PropertyValue -Item $matchingChannel -Name "force_backend_fallback" -Default $false))
|
||||
active_channel_not_degraded_route = ($null -ne $matchingChannel -and -not [bool](Get-PropertyValue -Item $matchingChannel -Name "force_degraded_route" -Default $false))
|
||||
route_quality_fenced = ($null -ne $matchingChannel -and [string](Get-PropertyValue -Item $matchingChannel -Name "route_feedback_status" -Default "") -eq "fenced")
|
||||
remediation_prefers_alternate = ($null -ne $matchingChannel -and [string](Get-PropertyValue -Item $matchingChannel -Name "remediation_action" -Default "") -eq "prefer_alternate_route")
|
||||
remediation_route_is_alternate = ($null -ne $matchingChannel -and [string](Get-PropertyValue -Item $matchingChannel -Name "remediation_route_id" -Default "") -eq $expectedReplacementRouteID)
|
||||
@@ -532,7 +532,7 @@ try {
|
||||
no_local_gateway_fallback_after_replacement = (($postRemediationFallbackLocal - $baselineFallbackLocal) -eq 0)
|
||||
no_flow_drops_after_replacement = (($postRemediationFlowDropped - $baselineFlowDropped) -eq 0 -and ($postRemediationSchedulerDropped - $baselineSchedulerDropped) -eq 0)
|
||||
no_route_failures_after_replacement = (($postRemediationRouteFailures - $baselineRouteFailures) -eq 0)
|
||||
backend_fallback_not_recommended = ($null -ne $matchingChannel -and -not [bool](Get-PropertyValue -Item $matchingChannel -Name "force_backend_fallback" -Default $false) -and [string](Get-PropertyValue -Item $matchingChannel -Name "remediation_action" -Default "") -ne "use_backend_fallback")
|
||||
degraded_route_not_recommended = ($null -ne $matchingChannel -and -not [bool](Get-PropertyValue -Item $matchingChannel -Name "force_degraded_route" -Default $false) -and [string](Get-PropertyValue -Item $matchingChannel -Name "remediation_action" -Default "") -ne "use_degraded_route")
|
||||
}
|
||||
summary = [ordered]@{
|
||||
backend_container = $backendLine.Trim()
|
||||
@@ -617,3 +617,4 @@ $result
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -72,7 +72,7 @@ function Get-MeshPort {
|
||||
function Enable-TestMeshListener {
|
||||
param([object]$Node)
|
||||
$port = Get-MeshPort -Name $Node.name
|
||||
Invoke-Api -Method PUT -Path "/clusters/$ClusterID/nodes/$($Node.id)/workloads/mesh-listener/desired" -Body @{
|
||||
Invoke-Api -Method PUT -Path "/clusters/$ClusterID/nodes/$($Node.id)/workloads/fabric-listener/desired" -Body @{
|
||||
actor_user_id = $ActorUserID
|
||||
desired_state = "enabled"
|
||||
runtime_mode = "container"
|
||||
@@ -1113,7 +1113,7 @@ try {
|
||||
exit_queue = @{
|
||||
total_depth = (@($sessionResults | Measure-Object -Property exit_depth -Sum).Sum)
|
||||
}
|
||||
backend_fallback_queue = @{
|
||||
degraded_route_queue = @{
|
||||
total_delta = (@($sessionResults | Measure-Object -Property backend_delta -Sum).Sum)
|
||||
}
|
||||
sessions = $sessionResults
|
||||
@@ -1132,7 +1132,7 @@ try {
|
||||
entry_runtime_loaded_rebuild_config = ([string]$postRebuildLoadedConfig.report.config_version -ge [string]$appliedDecision.config.synthetic_mesh_config.config_version)
|
||||
post_rebuild_uses_alternate_route = ([string]$postRebuildIngress.report.ingress.last_selected_route_id -eq $alternateRouteID)
|
||||
every_session_exit_inbox_received_all_packets = (@($sessionResults | Where-Object { $_.exit_depth -ge ($_.baseline_exit_depth + $_.expected_packets) }).Count -eq $SessionCount)
|
||||
no_backend_fallback_used = ((@($sessionResults | Measure-Object -Property backend_delta -Sum).Sum) -eq 0)
|
||||
no_degraded_route_used = ((@($sessionResults | Measure-Object -Property backend_delta -Sum).Sum) -eq 0)
|
||||
no_flow_drops = ($droppedDelta -eq 0)
|
||||
route_intents_expired = ($expiredPrimary.route_intent.lifecycle_status -eq "expired" -and $expiredAlternate.route_intent.lifecycle_status -eq "expired")
|
||||
}
|
||||
@@ -1174,3 +1174,4 @@ if (-not (Test-Path $resultDir)) {
|
||||
$result | ConvertTo-Json -Depth 100 | Set-Content -Path $resultFullPath -Encoding UTF8
|
||||
Write-Host "C18Z7 live service-channel concurrent isolation smoke passed. Result: $resultFullPath"
|
||||
$result
|
||||
|
||||
|
||||
@@ -58,7 +58,7 @@ $policy = (Invoke-Api -Method PUT -Path "/clusters/$ClusterID/fabric/service-cha
|
||||
route_rebuild = "automatic"
|
||||
entry_failover = "automatic"
|
||||
exit_failover = "automatic"
|
||||
backend_fallback_allowed = $true
|
||||
degraded_route_allowed = $true
|
||||
sticky_session = $true
|
||||
}).fabric_service_channel_pool_policy
|
||||
|
||||
@@ -121,7 +121,7 @@ try {
|
||||
route_rebuild = "automatic"
|
||||
entry_failover = "automatic"
|
||||
exit_failover = "automatic"
|
||||
backend_fallback_allowed = $true
|
||||
degraded_route_allowed = $true
|
||||
sticky_session = $true
|
||||
} | Out-Null
|
||||
} catch {
|
||||
@@ -134,3 +134,4 @@ if (-not $result.passed) {
|
||||
|
||||
Write-Host "C18Z72 service-channel pool policy smoke passed. Result: $target"
|
||||
$result
|
||||
|
||||
|
||||
@@ -62,7 +62,7 @@ $policy = (Invoke-Api -Method PUT -Path "/clusters/$ClusterID/fabric/service-cha
|
||||
route_rebuild = "automatic"
|
||||
entry_failover = "automatic"
|
||||
exit_failover = "automatic"
|
||||
backend_fallback_allowed = $true
|
||||
degraded_route_allowed = $true
|
||||
sticky_session = $true
|
||||
}).fabric_service_channel_pool_policy
|
||||
|
||||
@@ -123,7 +123,7 @@ try {
|
||||
route_rebuild = "automatic"
|
||||
entry_failover = "automatic"
|
||||
exit_failover = "automatic"
|
||||
backend_fallback_allowed = $true
|
||||
degraded_route_allowed = $true
|
||||
sticky_session = $true
|
||||
} | Out-Null
|
||||
} catch {
|
||||
@@ -136,3 +136,4 @@ if (-not $result.passed) {
|
||||
|
||||
Write-Host "C18Z73 service-channel pool-policy remediation guard smoke passed. Result: $target"
|
||||
$result
|
||||
|
||||
|
||||
@@ -73,7 +73,7 @@ function Get-MeshPort {
|
||||
function Enable-TestMeshListener {
|
||||
param([object]$Node)
|
||||
$port = Get-MeshPort -Name $Node.name
|
||||
Invoke-Api -Method PUT -Path "/clusters/$ClusterID/nodes/$($Node.id)/workloads/mesh-listener/desired" -Body @{
|
||||
Invoke-Api -Method PUT -Path "/clusters/$ClusterID/nodes/$($Node.id)/workloads/fabric-listener/desired" -Body @{
|
||||
actor_user_id = $ActorUserID
|
||||
desired_state = "enabled"
|
||||
runtime_mode = "container"
|
||||
@@ -1239,7 +1239,7 @@ try {
|
||||
exit_queue = @{
|
||||
total_depth = (@($sessionResults | Measure-Object -Property exit_depth -Sum).Sum)
|
||||
}
|
||||
backend_fallback_queue = @{
|
||||
degraded_route_queue = @{
|
||||
total_delta = (@($sessionResults | Measure-Object -Property backend_delta -Sum).Sum)
|
||||
}
|
||||
sessions = $sessionResults
|
||||
@@ -1257,7 +1257,7 @@ try {
|
||||
abusive_session_sent_all_packets = (@($sessionResults | Where-Object { $_.kind -eq "abusive" -and $_.sent_packets -eq $AbusivePacketCount }).Count -eq 1)
|
||||
abusive_session_exit_batch_reached_exit = (@($sessionResults | Where-Object { $_.kind -eq "abusive" -and $_.exit_delta -ge 1 }).Count -eq 1)
|
||||
abusive_session_bounded_delivery_scheduled = ($maxChannelEnqueuedDelta -ge $ExpectedAbusiveDelivered -and $maxChannelEnqueuedDelta -le $AbusivePacketCount)
|
||||
no_backend_fallback_used = ((@($sessionResults | Measure-Object -Property backend_delta -Sum).Sum) -eq 0)
|
||||
no_degraded_route_used = ((@($sessionResults | Measure-Object -Property backend_delta -Sum).Sum) -eq 0)
|
||||
backpressure_drops_observed = ($droppedDelta -ge $expectedAbusiveDropDelta)
|
||||
abusive_channel_drop_observed = ($maxChannelDropDelta -ge $expectedAbusiveDropDelta)
|
||||
backpressure_active_visible = ($finalFlowScheduler.backpressure_active -eq $true)
|
||||
@@ -1303,3 +1303,4 @@ if (-not (Test-Path $resultDir)) {
|
||||
$result | ConvertTo-Json -Depth 100 | Set-Content -Path $resultFullPath -Encoding UTF8
|
||||
Write-Host "C18Z8 live service-channel backpressure isolation smoke passed. Result: $resultFullPath"
|
||||
$result
|
||||
|
||||
|
||||
@@ -72,7 +72,7 @@ function Get-MeshPort {
|
||||
function Enable-TestMeshListener {
|
||||
param([object]$Node)
|
||||
$port = Get-MeshPort -Name $Node.name
|
||||
Invoke-Api -Method PUT -Path "/clusters/$ClusterID/nodes/$($Node.id)/workloads/mesh-listener/desired" -Body @{
|
||||
Invoke-Api -Method PUT -Path "/clusters/$ClusterID/nodes/$($Node.id)/workloads/fabric-listener/desired" -Body @{
|
||||
actor_user_id = $ActorUserID
|
||||
desired_state = "enabled"
|
||||
runtime_mode = "container"
|
||||
@@ -1029,7 +1029,7 @@ try {
|
||||
baseline_depth = $baselineExitDepth
|
||||
depth = $exitDepth
|
||||
}
|
||||
backend_fallback_queue = @{
|
||||
degraded_route_queue = @{
|
||||
baseline_depth = $baselineBackendDepth
|
||||
depth = $finalBackendDepth
|
||||
}
|
||||
@@ -1054,7 +1054,7 @@ try {
|
||||
entry_runtime_loaded_rebuild_config = ([string]$postRebuildLoadedConfig.report.config_version -ge [string]$appliedDecision.config.synthetic_mesh_config.config_version)
|
||||
post_rebuild_uses_fast_direct_route = ([string]$postRebuildIngress.report.ingress.last_selected_route_id -eq $alternateRouteID)
|
||||
exit_inbox_received_all_packets = ($exitDepth -ge ($baselineExitDepth + $expectedPackets))
|
||||
no_backend_fallback_used = ($finalBackendDepth -eq $baselineBackendDepth)
|
||||
no_degraded_route_used = ($finalBackendDepth -eq $baselineBackendDepth)
|
||||
no_flow_drops = ($droppedDelta -eq 0)
|
||||
route_intents_expired = ($expiredPrimary.route_intent.lifecycle_status -eq "expired" -and $expiredAlternate.route_intent.lifecycle_status -eq "expired")
|
||||
}
|
||||
@@ -1093,3 +1093,4 @@ if (-not (Test-Path $resultDir)) {
|
||||
$result | ConvertTo-Json -Depth 100 | Set-Content -Path $resultFullPath -Encoding UTF8
|
||||
Write-Host "C18Z9 live service-channel route pool smoke passed. Result: $resultFullPath"
|
||||
$result
|
||||
|
||||
|
||||
@@ -192,7 +192,7 @@ $checks = [ordered]@{
|
||||
packet_ingress_accepted_by_signed_contract = ([int]$packetResponse.StatusCode -eq 202 -and $acceptedBy -eq "signed")
|
||||
node_agent_reports_data_plane_contract = ($null -ne $accessReport -and [int](Get-PropertyValue -Item $accessReport -Name "data_plane_contract" -Default 0) -ge 1)
|
||||
node_agent_reports_fabric_route_data_plane = ($null -ne $accessReport -and [string](Get-PropertyValue -Item $accessReport -Name "last_working_data_transport" -Default "") -eq "fabric_service_channel" -and [string](Get-PropertyValue -Item $accessReport -Name "last_steady_state_transport" -Default "") -eq "fabric_route")
|
||||
node_agent_reports_degraded_backend_fallback_policy = ($null -ne $accessReport -and [string](Get-PropertyValue -Item $accessReport -Name "last_backend_relay_policy" -Default "") -eq "degraded_fallback_only")
|
||||
node_agent_reports_degraded_degraded_route_policy = ($null -ne $accessReport -and [string](Get-PropertyValue -Item $accessReport -Name "last_degraded_route_policy" -Default "") -eq "degraded_fallback_only")
|
||||
node_agent_reports_multi_flow_contract = ($null -ne $accessReport -and [string](Get-PropertyValue -Item $accessReport -Name "last_logical_flow_mode" -Default "") -eq "multi_flow_isolated")
|
||||
}
|
||||
$failed = @($checks.GetEnumerator() | Where-Object { -not $_.Value } | ForEach-Object { $_.Key })
|
||||
@@ -241,3 +241,4 @@ Write-Host "C18Z91 node-agent data-plane contract enforcement smoke passed. Resu
|
||||
$result
|
||||
|
||||
|
||||
|
||||
|
||||
+12
-11
@@ -7,7 +7,7 @@ param(
|
||||
[string]$DockerSSH = "test-docker",
|
||||
[string]$ExpectedBackendImage = "rap-backend:fabric-service-channel-0.2.281-c18z109",
|
||||
[string]$ExpectedNodeAgentImage = "rap-node-agent:0.2.270-c18z95",
|
||||
[string]$ResultPath = "artifacts\c18z92-node-agent-disabled-backend-fallback-smoke-result.json"
|
||||
[string]$ResultPath = "artifacts\c18z92-node-agent-disabled-compat-fallback-smoke-result.json"
|
||||
)
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
@@ -57,12 +57,12 @@ function Disable-ExistingRoutePair {
|
||||
if ([string](Get-PropertyValue -Item $destinationSelector -Name "node_id" -Default "") -ne $DestinationNodeID) { continue }
|
||||
[void](Invoke-Api -Method POST -Path "/clusters/$ClusterID/mesh/route-intents/$($item.id)/disable" -Body @{
|
||||
actor_user_id = $ActorUserID
|
||||
reason = "c18z92 isolate disabled backend fallback route pair"
|
||||
reason = "c18z92 isolate disabled degraded route route pair"
|
||||
})
|
||||
}
|
||||
}
|
||||
function Set-PoolPolicy {
|
||||
param([object]$Policy, [bool]$BackendFallbackAllowed)
|
||||
param([object]$Policy, [bool]$DegradedRouteAllowed)
|
||||
return Invoke-Api -Method PUT -Path "/clusters/$ClusterID/fabric/service-channels/pool-policy" -Body @{
|
||||
actor_user_id = $ActorUserID
|
||||
entry_pool_node_ids = @(Get-PropertyValue -Item $Policy -Name "entry_pool_node_ids" -Default @())
|
||||
@@ -73,7 +73,7 @@ function Set-PoolPolicy {
|
||||
route_rebuild = [string](Get-PropertyValue -Item $Policy -Name "route_rebuild" -Default "automatic")
|
||||
entry_failover = [string](Get-PropertyValue -Item $Policy -Name "entry_failover" -Default "automatic")
|
||||
exit_failover = [string](Get-PropertyValue -Item $Policy -Name "exit_failover" -Default "automatic")
|
||||
backend_fallback_allowed = $BackendFallbackAllowed
|
||||
degraded_route_allowed = $DegradedRouteAllowed
|
||||
sticky_session = [bool](Get-PropertyValue -Item $Policy -Name "sticky_session" -Default $true)
|
||||
}
|
||||
}
|
||||
@@ -81,13 +81,13 @@ function Set-PoolPolicy {
|
||||
$entryNode = Get-NodeByName -Name $EntryNodeName
|
||||
$exitNode = Get-NodeByName -Name $ExitNodeName
|
||||
$originalPolicy = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/fabric/service-channels/pool-policy?actor_user_id=$ActorUserID").fabric_service_channel_pool_policy
|
||||
$originalBackendFallback = [bool](Get-PropertyValue -Item $originalPolicy -Name "backend_fallback_allowed" -Default $true)
|
||||
$originalDegradedRoute = [bool](Get-PropertyValue -Item $originalPolicy -Name "degraded_route_allowed" -Default $true)
|
||||
$lease = $null
|
||||
$packetStatus = 0
|
||||
$packetAcceptedBy = ""
|
||||
try {
|
||||
Disable-ExistingRoutePair -SourceNodeID $entryNode.id -DestinationNodeID $exitNode.id
|
||||
[void](Set-PoolPolicy -Policy $originalPolicy -BackendFallbackAllowed $false)
|
||||
[void](Set-PoolPolicy -Policy $originalPolicy -DegradedRouteAllowed $false)
|
||||
$lease = (Invoke-Api -Method POST -Path "/clusters/$ClusterID/fabric/service-channels/leases" -Body @{
|
||||
actor_user_id = $ActorUserID
|
||||
organization_id = "org-home"
|
||||
@@ -100,7 +100,7 @@ try {
|
||||
preferred_exit_node_id = [string]$exitNode.id
|
||||
allowed_channels = @("vpn_packet", "fabric_control")
|
||||
ttl_seconds = 120
|
||||
metadata = @{ smoke = "c18z92_node_agent_disabled_backend_fallback"; run_id = $runId }
|
||||
metadata = @{ smoke = "c18z92_node_agent_disabled_degraded_route"; run_id = $runId }
|
||||
}).fabric_service_channel_lease
|
||||
$decodedAuthority = Get-PropertyValue -Item $lease -Name "authority_payload" -Default $null
|
||||
$authoritySignature = Get-PropertyValue -Item $lease -Name "authority_signature" -Default $null
|
||||
@@ -120,7 +120,7 @@ try {
|
||||
$packetAcceptedBy = [string]$_.Exception.Response.Headers["X-RAP-Service-Channel-Accepted-By"]
|
||||
}
|
||||
} finally {
|
||||
try { [void](Set-PoolPolicy -Policy $originalPolicy -BackendFallbackAllowed $originalBackendFallback) } catch {}
|
||||
try { [void](Set-PoolPolicy -Policy $originalPolicy -DegradedRouteAllowed $originalDegradedRoute) } catch {}
|
||||
}
|
||||
|
||||
$dataPlane = Get-PropertyValue -Item $lease -Name "data_plane" -Default $null
|
||||
@@ -136,7 +136,7 @@ $checks = [ordered]@{
|
||||
}
|
||||
$failed = @($checks.GetEnumerator() | Where-Object { -not $_.Value } | ForEach-Object { $_.Key })
|
||||
$result = [ordered]@{
|
||||
schema_version = "c18z92.node_agent_disabled_backend_fallback_smoke.v1"
|
||||
schema_version = "c18z92.node_agent_disabled_degraded_route_smoke.v1"
|
||||
run_id = $runId
|
||||
cluster_id = $ClusterID
|
||||
channel_id = [string]$lease.channel_id
|
||||
@@ -154,7 +154,8 @@ $result = [ordered]@{
|
||||
$target = Join-Path $repoRoot $ResultPath
|
||||
$result | ConvertTo-Json -Depth 80 | Set-Content -Path $target -Encoding UTF8
|
||||
if (-not $result.passed) {
|
||||
throw "C18Z92 node-agent disabled backend fallback smoke failed: $($failed -join ', ')"
|
||||
throw "C18Z92 node-agent disabled degraded route smoke failed: $($failed -join ', ')"
|
||||
}
|
||||
Write-Host "C18Z92 node-agent disabled backend fallback smoke passed. Result: $target"
|
||||
Write-Host "C18Z92 node-agent disabled degraded route smoke passed. Result: $target"
|
||||
$result
|
||||
|
||||
@@ -157,11 +157,11 @@ $checks = [ordered]@{
|
||||
packet_ingress_accepted_by_signed_contract = ([int]$packetResponse.StatusCode -eq 202 -and $acceptedBy -eq "signed")
|
||||
access_telemetry_reports_data_plane_contract = ($null -ne $accessTelemetry -and [int](Get-PropertyValue -Item $accessTelemetry -Name "data_plane_contract_count" -Default 0) -ge 1)
|
||||
access_telemetry_reports_fabric_data_plane = ($null -ne $accessTelemetry -and [string](Get-PropertyValue -Item $accessTelemetry -Name "last_working_data_transport" -Default "") -eq "fabric_service_channel" -and [string](Get-PropertyValue -Item $accessTelemetry -Name "last_steady_state_transport" -Default "") -eq "fabric_route")
|
||||
access_telemetry_reports_backend_policy = ($null -ne $accessTelemetry -and [string](Get-PropertyValue -Item $accessTelemetry -Name "last_backend_relay_policy" -Default "") -eq "degraded_fallback_only")
|
||||
access_telemetry_reports_backend_policy = ($null -ne $accessTelemetry -and [string](Get-PropertyValue -Item $accessTelemetry -Name "last_degraded_route_policy" -Default "") -eq "degraded_fallback_only")
|
||||
access_telemetry_reports_logical_flow = ($null -ne $accessTelemetry -and [string](Get-PropertyValue -Item $accessTelemetry -Name "last_logical_flow_mode" -Default "") -eq "multi_flow_isolated")
|
||||
active_channel_reports_data_plane_contract = ($null -ne $activeChannel -and [int](Get-PropertyValue -Item $activeChannel -Name "entry_node_data_plane_contract_count" -Default 0) -ge 1)
|
||||
active_channel_reports_fabric_data_plane = ($null -ne $activeChannel -and [string](Get-PropertyValue -Item $activeChannel -Name "entry_node_last_working_data_transport" -Default "") -eq "fabric_service_channel" -and [string](Get-PropertyValue -Item $activeChannel -Name "entry_node_last_steady_state_transport" -Default "") -eq "fabric_route")
|
||||
active_channel_reports_backend_policy = ($null -ne $activeChannel -and [string](Get-PropertyValue -Item $activeChannel -Name "entry_node_last_backend_relay_policy" -Default "") -eq "degraded_fallback_only")
|
||||
active_channel_reports_backend_policy = ($null -ne $activeChannel -and [string](Get-PropertyValue -Item $activeChannel -Name "entry_node_last_degraded_route_policy" -Default "") -eq "degraded_fallback_only")
|
||||
active_channel_reports_logical_flow = ($null -ne $activeChannel -and [string](Get-PropertyValue -Item $activeChannel -Name "entry_node_last_logical_flow_mode" -Default "") -eq "multi_flow_isolated")
|
||||
}
|
||||
$failed = @($checks.GetEnumerator() | Where-Object { -not $_.Value } | ForEach-Object { $_.Key })
|
||||
@@ -206,3 +206,4 @@ if (-not $result.passed) {
|
||||
|
||||
Write-Host "C18Z93 access telemetry data-plane contract smoke passed. Result: $target"
|
||||
$result
|
||||
|
||||
|
||||
@@ -33,7 +33,7 @@ function Get-PropertyValue {
|
||||
return $property.Value
|
||||
}
|
||||
|
||||
& (Join-Path $scriptDir "c18z92-node-agent-disabled-backend-fallback-smoke.ps1") `
|
||||
& (Join-Path $scriptDir "c18z92-node-agent-disabled-compat-fallback-smoke.ps1") `
|
||||
-ApiBaseUrl $ApiBaseUrl `
|
||||
-ClusterID $ClusterID `
|
||||
-ActorUserID $ActorUserID `
|
||||
@@ -54,7 +54,7 @@ $checks = [ordered]@{
|
||||
base_disabled_fallback_smoke_passed = [bool]$baseResult.passed
|
||||
data_plane_incident_emitted = ($null -ne $incident)
|
||||
incident_source_is_data_plane_contract = ($null -ne $incident -and [string]$incident.incident_source -eq "data_plane_contract")
|
||||
incident_status_reports_blocked_backend_relay = ($null -ne $incident -and [string]$incident.guard_status -eq "backend_fallback_blocked_by_policy")
|
||||
incident_status_reports_blocked_backend_relay = ($null -ne $incident -and [string]$incident.guard_status -eq "degraded_route_blocked_by_policy")
|
||||
incident_severity_is_bad = ($null -ne $incident -and [string]$incident.guard_severity -eq "bad")
|
||||
incident_action_is_operator_visible = ($null -ne $incident -and [string]$incident.recommended_operator_action -eq "restore_fabric_route_or_change_signed_backend_relay_policy_before_retry")
|
||||
}
|
||||
@@ -84,3 +84,4 @@ if (-not $result.passed) {
|
||||
|
||||
Write-Host "C18Z94 data-plane contract incident smoke passed. Result: $target"
|
||||
$result
|
||||
|
||||
|
||||
@@ -33,7 +33,7 @@ function Get-PropertyValue {
|
||||
return $property.Value
|
||||
}
|
||||
|
||||
& (Join-Path $scriptDir "c18z92-node-agent-disabled-backend-fallback-smoke.ps1") `
|
||||
& (Join-Path $scriptDir "c18z92-node-agent-disabled-compat-fallback-smoke.ps1") `
|
||||
-ApiBaseUrl $ApiBaseUrl `
|
||||
-ClusterID $ClusterID `
|
||||
-ActorUserID $ActorUserID `
|
||||
@@ -51,7 +51,7 @@ for ($i = 0; $i -lt 8; $i++) {
|
||||
$response = Invoke-Api -Method GET -Path "/clusters/$ClusterID/fabric/service-channels/access-telemetry?actor_user_id=$ActorUserID&limit=50"
|
||||
$accessTelemetry = $response.fabric_service_channel_access_telemetry
|
||||
$activeChannel = @($accessTelemetry.active_channels | Where-Object { [string](Get-PropertyValue -Item $_ -Name "channel_id" -Default "") -eq $channelID }) | Select-Object -First 1
|
||||
if ($null -ne $activeChannel -and [int](Get-PropertyValue -Item $activeChannel -Name "entry_node_backend_fallback_blocked_count" -Default 0) -ge 1) {
|
||||
if ($null -ne $activeChannel -and [int](Get-PropertyValue -Item $activeChannel -Name "entry_node_degraded_route_blocked_count" -Default 0) -ge 1) {
|
||||
break
|
||||
}
|
||||
}
|
||||
@@ -64,10 +64,10 @@ $incident = @($incidents | Where-Object {
|
||||
|
||||
$checks = [ordered]@{
|
||||
base_disabled_fallback_smoke_passed = [bool]$baseResult.passed
|
||||
access_telemetry_reports_blocked_fallback = ($null -ne $accessTelemetry -and [int](Get-PropertyValue -Item $accessTelemetry -Name "backend_fallback_blocked_count" -Default 0) -ge 1)
|
||||
active_channel_reports_blocked_fallback = ($null -ne $activeChannel -and [int](Get-PropertyValue -Item $activeChannel -Name "entry_node_backend_fallback_blocked_count" -Default 0) -ge 1)
|
||||
active_channel_reports_violation_status = ($null -ne $activeChannel -and [string](Get-PropertyValue -Item $activeChannel -Name "entry_node_last_data_plane_violation_status" -Default "") -eq "backend_fallback_blocked_by_policy")
|
||||
incident_reports_blocked_fallback = ($null -ne $incident -and [string](Get-PropertyValue -Item $incident -Name "guard_status" -Default "") -eq "backend_fallback_blocked_by_policy")
|
||||
access_telemetry_reports_blocked_fallback = ($null -ne $accessTelemetry -and [int](Get-PropertyValue -Item $accessTelemetry -Name "degraded_route_blocked_count" -Default 0) -ge 1)
|
||||
active_channel_reports_blocked_fallback = ($null -ne $activeChannel -and [int](Get-PropertyValue -Item $activeChannel -Name "entry_node_degraded_route_blocked_count" -Default 0) -ge 1)
|
||||
active_channel_reports_violation_status = ($null -ne $activeChannel -and [string](Get-PropertyValue -Item $activeChannel -Name "entry_node_last_data_plane_violation_status" -Default "") -eq "degraded_route_blocked_by_policy")
|
||||
incident_reports_blocked_fallback = ($null -ne $incident -and [string](Get-PropertyValue -Item $incident -Name "guard_status" -Default "") -eq "degraded_route_blocked_by_policy")
|
||||
incident_action_reports_restore_route_or_policy = ($null -ne $incident -and [string](Get-PropertyValue -Item $incident -Name "recommended_operator_action" -Default "") -eq "restore_fabric_route_or_change_signed_backend_relay_policy_before_retry")
|
||||
}
|
||||
$failed = @($checks.GetEnumerator() | Where-Object { -not $_.Value } | ForEach-Object { $_.Key })
|
||||
@@ -96,3 +96,4 @@ if (-not $result.passed) {
|
||||
|
||||
Write-Host "C18Z95 node-agent blocked fallback telemetry smoke passed. Result: $target"
|
||||
$result
|
||||
|
||||
|
||||
@@ -96,14 +96,14 @@ Invoke-Api -Method POST -Path "/clusters/$ClusterID/nodes/$($entryNode.id)/heart
|
||||
schema_version = "c18z52.fabric_service_channel_access_report.v1"
|
||||
total = 1
|
||||
signed = 1
|
||||
backend_fallback = 0
|
||||
backend_fallback_blocked = 1
|
||||
degraded_route = 0
|
||||
degraded_route_blocked = 1
|
||||
fabric_route_send_failure = 1
|
||||
data_plane_contract = 1
|
||||
last_backend_relay_policy = "disabled"
|
||||
last_degraded_route_policy = "disabled"
|
||||
last_working_data_transport = "fabric_service_channel"
|
||||
last_steady_state_transport = "fabric_route"
|
||||
last_data_plane_violation_status = "fabric_route_send_failed_backend_fallback_blocked"
|
||||
last_data_plane_violation_status = "fabric_route_send_failed_degraded_route_blocked"
|
||||
last_data_plane_violation_reason = "synthetic c18z96 route send failure"
|
||||
}
|
||||
}
|
||||
@@ -137,7 +137,7 @@ $checks = [ordered]@{
|
||||
lease_selected_bad_route_first = ([string]$lease.primary_route.route_id -eq [string]$badRoute.id)
|
||||
route_feedback_recorded_from_access_report = ($null -ne $feedback)
|
||||
route_feedback_is_fenced = ($null -ne $feedback -and [string]$feedback.feedback_status -eq "fenced")
|
||||
route_feedback_contains_blocked_policy_reason = ($null -ne $feedback -and @($feedback.reasons | Where-Object { [string]$_ -eq "backend_fallback_blocked_by_policy" }).Count -gt 0)
|
||||
route_feedback_contains_blocked_policy_reason = ($null -ne $feedback -and @($feedback.reasons | Where-Object { [string]$_ -eq "degraded_route_blocked_by_policy" }).Count -gt 0)
|
||||
planner_selected_replacement = ($null -ne $replacement)
|
||||
planner_replacement_is_good_route = ($null -ne $replacement -and [string]$replacement.replacement_route_id -eq [string]$goodRoute.id)
|
||||
planner_rebuild_status_applied = ($null -ne $replacement -and [string]$replacement.rebuild_status -eq "applied")
|
||||
@@ -185,3 +185,4 @@ if (-not $result.passed) {
|
||||
|
||||
Write-Host "C18Z96 blocked fallback rebuild feedback smoke passed. Result: $target"
|
||||
$result
|
||||
|
||||
|
||||
@@ -96,14 +96,14 @@ $heartbeatBody = @{
|
||||
schema_version = "c18z52.fabric_service_channel_access_report.v1"
|
||||
total = 1
|
||||
signed = 1
|
||||
backend_fallback = 0
|
||||
backend_fallback_blocked = 1
|
||||
degraded_route = 0
|
||||
degraded_route_blocked = 1
|
||||
fabric_route_send_failure = 1
|
||||
data_plane_contract = 1
|
||||
last_backend_relay_policy = "disabled"
|
||||
last_degraded_route_policy = "disabled"
|
||||
last_working_data_transport = "fabric_service_channel"
|
||||
last_steady_state_transport = "fabric_route"
|
||||
last_data_plane_violation_status = "fabric_route_send_failed_backend_fallback_blocked"
|
||||
last_data_plane_violation_status = "fabric_route_send_failed_degraded_route_blocked"
|
||||
last_data_plane_violation_reason = "synthetic c18z97 duplicate route send failure"
|
||||
}
|
||||
}
|
||||
@@ -151,7 +151,7 @@ $checks = [ordered]@{
|
||||
lease_selected_bad_route_first = ([string]$lease.primary_route.route_id -eq [string]$badRoute.id)
|
||||
route_feedback_recorded_from_access_report = ($null -ne $feedback)
|
||||
route_feedback_is_fenced = ($null -ne $feedback -and [string]$feedback.feedback_status -eq "fenced")
|
||||
route_feedback_contains_blocked_policy_reason = ($null -ne $feedback -and @($feedback.reasons | Where-Object { [string]$_ -eq "backend_fallback_blocked_by_policy" }).Count -gt 0)
|
||||
route_feedback_contains_blocked_policy_reason = ($null -ne $feedback -and @($feedback.reasons | Where-Object { [string]$_ -eq "degraded_route_blocked_by_policy" }).Count -gt 0)
|
||||
duplicate_heartbeat_kept_single_latest_feedback = ($duplicateFeedbackCount -eq 1)
|
||||
duplicate_heartbeat_kept_feedback_id = ($firstFeedbackID -ne "" -and $firstFeedbackID -eq $secondFeedbackID)
|
||||
duplicate_heartbeat_kept_observed_at = ($firstObservedAt -ne "" -and $firstObservedAt -eq $secondObservedAt)
|
||||
@@ -203,3 +203,4 @@ if (-not $result.passed) {
|
||||
|
||||
Write-Host "C18Z97 blocked fallback feedback dedup smoke passed. Result: $target"
|
||||
$result
|
||||
|
||||
|
||||
@@ -96,14 +96,14 @@ $heartbeatBody = @{
|
||||
schema_version = "c18z52.fabric_service_channel_access_report.v1"
|
||||
total = 1
|
||||
signed = 1
|
||||
backend_fallback = 0
|
||||
backend_fallback_blocked = 1
|
||||
degraded_route = 0
|
||||
degraded_route_blocked = 1
|
||||
fabric_route_send_failure = 1
|
||||
data_plane_contract = 1
|
||||
last_backend_relay_policy = "disabled"
|
||||
last_degraded_route_policy = "disabled"
|
||||
last_working_data_transport = "fabric_service_channel"
|
||||
last_steady_state_transport = "fabric_route"
|
||||
last_data_plane_violation_status = "fabric_route_send_failed_backend_fallback_blocked"
|
||||
last_data_plane_violation_status = "fabric_route_send_failed_degraded_route_blocked"
|
||||
last_data_plane_violation_reason = "synthetic c18z98 route send failure"
|
||||
}
|
||||
}
|
||||
@@ -156,7 +156,7 @@ $checks = [ordered]@{
|
||||
lease_selected_bad_route_first = ([string]$lease.primary_route.route_id -eq [string]$badRoute.id)
|
||||
route_feedback_recorded_from_access_report = ($null -ne $feedback)
|
||||
route_feedback_is_fenced = ($null -ne $feedback -and [string]$feedback.feedback_status -eq "fenced")
|
||||
route_feedback_contains_blocked_policy_reason = ($null -ne $feedback -and @($feedback.reasons | Where-Object { [string]$_ -eq "backend_fallback_blocked_by_policy" }).Count -gt 0)
|
||||
route_feedback_contains_blocked_policy_reason = ($null -ne $feedback -and @($feedback.reasons | Where-Object { [string]$_ -eq "degraded_route_blocked_by_policy" }).Count -gt 0)
|
||||
duplicate_heartbeat_kept_single_latest_feedback = ($duplicateFeedbackCount -eq 1)
|
||||
duplicate_heartbeat_kept_feedback_id = ($firstFeedbackID -ne "" -and $firstFeedbackID -eq $secondFeedbackID)
|
||||
duplicate_heartbeat_kept_observed_at = ($firstObservedAt -ne "" -and $firstObservedAt -eq $secondObservedAt)
|
||||
@@ -166,7 +166,7 @@ $checks = [ordered]@{
|
||||
planner_decision_links_feedback_observation = ($null -ne $replacement -and [string](Get-PropertyValue -Item $replacement -Name "feedback_observation_id" -Default "") -eq $firstFeedbackID)
|
||||
planner_decision_links_access_report_source = ($null -ne $replacement -and [string](Get-PropertyValue -Item $replacement -Name "feedback_source" -Default "") -eq "fabric_service_channel_access_report")
|
||||
planner_decision_links_channel = ($null -ne $replacement -and [string](Get-PropertyValue -Item $replacement -Name "feedback_channel_id" -Default "") -eq [string]$lease.channel_id)
|
||||
planner_decision_links_violation = ($null -ne $replacement -and [string](Get-PropertyValue -Item $replacement -Name "feedback_violation_status" -Default "") -eq "fabric_route_send_failed_backend_fallback_blocked")
|
||||
planner_decision_links_violation = ($null -ne $replacement -and [string](Get-PropertyValue -Item $replacement -Name "feedback_violation_status" -Default "") -eq "fabric_route_send_failed_degraded_route_blocked")
|
||||
rebuild_ledger_recorded_correlated_attempt = ($null -ne $ledgerAttempt)
|
||||
rebuild_ledger_links_feedback_observation = ($null -ne $ledgerAttempt -and [string](Get-PropertyValue -Item $ledgerAttempt -Name "feedback_observation_id" -Default "") -eq $firstFeedbackID)
|
||||
rebuild_ledger_links_access_report_source = ($null -ne $ledgerAttempt -and [string](Get-PropertyValue -Item $ledgerAttempt -Name "feedback_source" -Default "") -eq "fabric_service_channel_access_report")
|
||||
@@ -217,3 +217,4 @@ if (-not $result.passed) {
|
||||
|
||||
Write-Host "C18Z98 blocked fallback rebuild correlation smoke passed. Result: $target"
|
||||
$result
|
||||
|
||||
|
||||
@@ -96,14 +96,14 @@ $heartbeatBody = @{
|
||||
schema_version = "c18z52.fabric_service_channel_access_report.v1"
|
||||
total = 1
|
||||
signed = 1
|
||||
backend_fallback = 0
|
||||
backend_fallback_blocked = 1
|
||||
degraded_route = 0
|
||||
degraded_route_blocked = 1
|
||||
fabric_route_send_failure = 1
|
||||
data_plane_contract = 1
|
||||
last_backend_relay_policy = "disabled"
|
||||
last_degraded_route_policy = "disabled"
|
||||
last_working_data_transport = "fabric_service_channel"
|
||||
last_steady_state_transport = "fabric_route"
|
||||
last_data_plane_violation_status = "fabric_route_send_failed_backend_fallback_blocked"
|
||||
last_data_plane_violation_status = "fabric_route_send_failed_degraded_route_blocked"
|
||||
last_data_plane_violation_reason = "synthetic c18z99 route send failure"
|
||||
}
|
||||
}
|
||||
@@ -148,8 +148,8 @@ $ledgerAttempt = @($ledgerItems | Where-Object {
|
||||
}) | Select-Object -First 1
|
||||
$sourceFilteredLedger = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/fabric/service-channels/rebuild-attempts?actor_user_id=$ActorUserID&feedback_source=fabric_service_channel_access_report&limit=20&enrichment=summary").rebuild_attempts
|
||||
$channelFilteredLedger = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/fabric/service-channels/rebuild-attempts?actor_user_id=$ActorUserID&feedback_channel_id=$($lease.channel_id)&limit=20&enrichment=summary").rebuild_attempts
|
||||
$violationFilteredLedger = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/fabric/service-channels/rebuild-attempts?actor_user_id=$ActorUserID&feedback_violation_status=fabric_route_send_failed_backend_fallback_blocked&limit=20&enrichment=summary").rebuild_attempts
|
||||
$combinedFilteredLedger = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/fabric/service-channels/rebuild-attempts?actor_user_id=$ActorUserID&feedback_source=fabric_service_channel_access_report&feedback_channel_id=$($lease.channel_id)&feedback_violation_status=fabric_route_send_failed_backend_fallback_blocked&limit=20&enrichment=summary").rebuild_attempts
|
||||
$violationFilteredLedger = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/fabric/service-channels/rebuild-attempts?actor_user_id=$ActorUserID&feedback_violation_status=fabric_route_send_failed_degraded_route_blocked&limit=20&enrichment=summary").rebuild_attempts
|
||||
$combinedFilteredLedger = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/fabric/service-channels/rebuild-attempts?actor_user_id=$ActorUserID&feedback_source=fabric_service_channel_access_report&feedback_channel_id=$($lease.channel_id)&feedback_violation_status=fabric_route_send_failed_degraded_route_blocked&limit=20&enrichment=summary").rebuild_attempts
|
||||
$wrongChannelFilteredLedger = (Invoke-Api -Method GET -Path "/clusters/$ClusterID/fabric/service-channels/rebuild-attempts?actor_user_id=$ActorUserID&feedback_channel_id=00000000-0000-0000-0000-000000000000&limit=20&enrichment=summary").rebuild_attempts
|
||||
$sourceFilteredAttempt = @($sourceFilteredLedger | Where-Object { [string](Get-PropertyValue -Item $_ -Name "rebuild_request_id" -Default "") -eq [string](Get-PropertyValue -Item $replacement -Name "rebuild_request_id" -Default "") }) | Select-Object -First 1
|
||||
$channelFilteredAttempt = @($channelFilteredLedger | Where-Object { [string](Get-PropertyValue -Item $_ -Name "rebuild_request_id" -Default "") -eq [string](Get-PropertyValue -Item $replacement -Name "rebuild_request_id" -Default "") }) | Select-Object -First 1
|
||||
@@ -165,7 +165,7 @@ $checks = [ordered]@{
|
||||
lease_selected_bad_route_first = ([string]$lease.primary_route.route_id -eq [string]$badRoute.id)
|
||||
route_feedback_recorded_from_access_report = ($null -ne $feedback)
|
||||
route_feedback_is_fenced = ($null -ne $feedback -and [string]$feedback.feedback_status -eq "fenced")
|
||||
route_feedback_contains_blocked_policy_reason = ($null -ne $feedback -and @($feedback.reasons | Where-Object { [string]$_ -eq "backend_fallback_blocked_by_policy" }).Count -gt 0)
|
||||
route_feedback_contains_blocked_policy_reason = ($null -ne $feedback -and @($feedback.reasons | Where-Object { [string]$_ -eq "degraded_route_blocked_by_policy" }).Count -gt 0)
|
||||
duplicate_heartbeat_kept_single_latest_feedback = ($duplicateFeedbackCount -eq 1)
|
||||
duplicate_heartbeat_kept_feedback_id = ($firstFeedbackID -ne "" -and $firstFeedbackID -eq $secondFeedbackID)
|
||||
duplicate_heartbeat_kept_observed_at = ($firstObservedAt -ne "" -and $firstObservedAt -eq $secondObservedAt)
|
||||
@@ -175,7 +175,7 @@ $checks = [ordered]@{
|
||||
planner_decision_links_feedback_observation = ($null -ne $replacement -and [string](Get-PropertyValue -Item $replacement -Name "feedback_observation_id" -Default "") -eq $firstFeedbackID)
|
||||
planner_decision_links_access_report_source = ($null -ne $replacement -and [string](Get-PropertyValue -Item $replacement -Name "feedback_source" -Default "") -eq "fabric_service_channel_access_report")
|
||||
planner_decision_links_channel = ($null -ne $replacement -and [string](Get-PropertyValue -Item $replacement -Name "feedback_channel_id" -Default "") -eq [string]$lease.channel_id)
|
||||
planner_decision_links_violation = ($null -ne $replacement -and [string](Get-PropertyValue -Item $replacement -Name "feedback_violation_status" -Default "") -eq "fabric_route_send_failed_backend_fallback_blocked")
|
||||
planner_decision_links_violation = ($null -ne $replacement -and [string](Get-PropertyValue -Item $replacement -Name "feedback_violation_status" -Default "") -eq "fabric_route_send_failed_degraded_route_blocked")
|
||||
rebuild_ledger_recorded_correlated_attempt = ($null -ne $ledgerAttempt)
|
||||
rebuild_ledger_links_feedback_observation = ($null -ne $ledgerAttempt -and [string](Get-PropertyValue -Item $ledgerAttempt -Name "feedback_observation_id" -Default "") -eq $firstFeedbackID)
|
||||
rebuild_ledger_links_access_report_source = ($null -ne $ledgerAttempt -and [string](Get-PropertyValue -Item $ledgerAttempt -Name "feedback_source" -Default "") -eq "fabric_service_channel_access_report")
|
||||
@@ -236,3 +236,4 @@ if (-not $result.passed) {
|
||||
|
||||
Write-Host "C18Z99 rebuild correlation filter smoke passed. Result: $target"
|
||||
$result
|
||||
|
||||
|
||||
@@ -93,7 +93,7 @@ Invoke-Api -Method PUT -Path "/clusters/$ClusterID/nodes/$($node.id)/workloads/s
|
||||
|
||||
$expected = @{
|
||||
"core-mesh" = "running"
|
||||
"mesh-listener" = "running"
|
||||
"fabric-listener" = "running"
|
||||
"synthetic.echo" = "running"
|
||||
}
|
||||
$statuses = Wait-ForWorkloadStates -NodeID $node.id -Expected $expected
|
||||
@@ -101,7 +101,7 @@ $synthetic = @($statuses | Where-Object { $_.service_type -eq "synthetic.echo" }
|
||||
|
||||
$checks = [ordered]@{
|
||||
core_mesh_running = [bool](@($statuses | Where-Object { $_.service_type -eq "core-mesh" -and $_.reported_state -eq "running" }).Count -gt 0)
|
||||
mesh_listener_running = [bool](@($statuses | Where-Object { $_.service_type -eq "mesh-listener" -and $_.reported_state -eq "running" }).Count -gt 0)
|
||||
fabric_listener_running = [bool](@($statuses | Where-Object { $_.service_type -eq "fabric-listener" -and $_.reported_state -eq "running" }).Count -gt 0)
|
||||
synthetic_echo_running = [bool]($null -ne $synthetic -and [string]$synthetic.reported_state -eq "running")
|
||||
synthetic_echo_builtin = [bool]($null -ne $synthetic -and [string]$synthetic.status_payload.execution_mode -eq "builtin")
|
||||
synthetic_echo_test_only = [bool]($null -ne $synthetic -and [string]$synthetic.status_payload.traffic -eq "test_service_only")
|
||||
|
||||
@@ -1,174 +0,0 @@
|
||||
param(
|
||||
[string]$BackendBaseUrl = "http://192.168.200.61:8080/api/v1",
|
||||
[string]$DockerContext = "test-ubuntu",
|
||||
[string]$AdminEmail = "windows-smoke@example.local",
|
||||
[string]$AdminPassword = "SmokePass!123",
|
||||
[int]$Count = 3,
|
||||
[string]$ImageTag = "rap-node-agent:control-panel-test"
|
||||
)
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
$repoRoot = (Resolve-Path (Join-Path $PSScriptRoot "..\..")).ProviderPath
|
||||
$stateRoot = Join-Path "C:\work" "rap-fabric-test-nodes"
|
||||
New-Item -ItemType Directory -Force -Path $stateRoot | Out-Null
|
||||
|
||||
function Invoke-Docker {
|
||||
param([string[]]$Arguments)
|
||||
docker @Arguments
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "docker $($Arguments -join ' ') failed with exit code $LASTEXITCODE"
|
||||
}
|
||||
}
|
||||
|
||||
function Invoke-Api {
|
||||
param(
|
||||
[string]$Method,
|
||||
[string]$Path,
|
||||
[object]$Body = $null
|
||||
)
|
||||
$uri = "$BackendBaseUrl$Path"
|
||||
if ($null -eq $Body) {
|
||||
return Invoke-RestMethod -Method $Method -Uri $uri -TimeoutSec 30
|
||||
}
|
||||
return Invoke-RestMethod -Method $Method -Uri $uri -ContentType "application/json" -Body ($Body | ConvertTo-Json -Depth 20) -TimeoutSec 30
|
||||
}
|
||||
|
||||
Write-Host "Building node-agent image on Docker context $DockerContext..."
|
||||
Invoke-Docker -Arguments @("--context", $DockerContext, "build", "-f", "$repoRoot\agents\rap-node-agent\Dockerfile", "-t", $ImageTag, "$repoRoot")
|
||||
|
||||
$login = Invoke-Api -Method Post -Path "/auth/login" -Body @{
|
||||
email = $AdminEmail
|
||||
password = $AdminPassword
|
||||
device_fingerprint = "fabric-test-nodes-script"
|
||||
device_label = "Fabric Test Nodes Script"
|
||||
trust_device = $true
|
||||
}
|
||||
$actorUserID = $login.user.id
|
||||
|
||||
$clusters = Invoke-Api -Method Get -Path "/clusters/?actor_user_id=$actorUserID"
|
||||
$cluster = @($clusters.clusters)[0]
|
||||
if ($null -eq $cluster) {
|
||||
throw "No cluster available for test node deployment."
|
||||
}
|
||||
$clusterID = $cluster.id
|
||||
Write-Host "Using cluster $($cluster.name) $clusterID"
|
||||
|
||||
Invoke-Api -Method Put -Path "/fabric/testing-flags" -Body @{
|
||||
actor_user_id = $actorUserID
|
||||
scope_type = "platform"
|
||||
scope_id = $null
|
||||
cluster_id = $null
|
||||
enabled = $true
|
||||
telemetry_enabled = $true
|
||||
synthetic_links_enabled = $true
|
||||
history_retention_hours = 24
|
||||
metadata = @{ source = "deploy-test-nodes.ps1"; runtime_mesh_enabled = $false }
|
||||
} | Out-Null
|
||||
|
||||
$joinToken = Invoke-Api -Method Post -Path "/clusters/$clusterID/join-tokens" -Body @{
|
||||
actor_user_id = $actorUserID
|
||||
scope = @{ purpose = "fabric-test-nodes"; roles = @("core-mesh", "relay-node") }
|
||||
expires_at = (Get-Date).ToUniversalTime().AddHours(2).ToString("o")
|
||||
max_uses = $Count
|
||||
}
|
||||
|
||||
$nodes = @()
|
||||
for ($i = 1; $i -le $Count; $i++) {
|
||||
$nodeName = "fabric-test-node-$i"
|
||||
$fingerprint = "rap-node-fp_$([guid]::NewGuid().ToString('N'))"
|
||||
$publicKey = "rap-node-pub_$([guid]::NewGuid().ToString('N'))"
|
||||
|
||||
$joinRequest = Invoke-Api -Method Post -Path "/clusters/$clusterID/join-requests" -Body @{
|
||||
join_token = $joinToken.join_token.token
|
||||
node_name = $nodeName
|
||||
node_fingerprint = $fingerprint
|
||||
public_key = $publicKey
|
||||
reported_capabilities = @{
|
||||
can_accept_node_ingress = $true
|
||||
can_route_mesh = $true
|
||||
can_run_rdp_worker = $false
|
||||
testing_node = $true
|
||||
}
|
||||
reported_facts = @{
|
||||
os = "linux"
|
||||
runtime = "docker-test"
|
||||
source = "deploy-test-nodes.ps1"
|
||||
}
|
||||
requested_roles = @("core-mesh", "relay-node")
|
||||
}
|
||||
|
||||
$approved = Invoke-Api -Method Post -Path "/clusters/$clusterID/join-requests/$($joinRequest.join_request.id)/approve" -Body @{
|
||||
actor_user_id = $actorUserID
|
||||
node_key = $fingerprint
|
||||
ownership_type = "platform_managed"
|
||||
owner_organization_id = $null
|
||||
}
|
||||
$nodeID = $approved.node_bootstrap.node_id
|
||||
$nodes += [pscustomobject]@{ name = $nodeName; id = $nodeID; fingerprint = $fingerprint; public_key = $publicKey }
|
||||
|
||||
$nodeStateDir = Join-Path $stateRoot $nodeName
|
||||
New-Item -ItemType Directory -Force -Path $nodeStateDir | Out-Null
|
||||
@{
|
||||
node_id = $nodeID
|
||||
cluster_id = $clusterID
|
||||
node_name = $nodeName
|
||||
node_fingerprint = $fingerprint
|
||||
public_key = $publicKey
|
||||
identity_status = "active"
|
||||
created_at = (Get-Date).ToUniversalTime().ToString("o")
|
||||
updated_at = (Get-Date).ToUniversalTime().ToString("o")
|
||||
} | ConvertTo-Json -Depth 10 | Set-Content -Encoding UTF8 -Path (Join-Path $nodeStateDir "identity.json")
|
||||
|
||||
$containerName = "rap_fabric_test_node_$i"
|
||||
docker --context $DockerContext rm -f $containerName 2>$null | Out-Null
|
||||
Invoke-Docker -Arguments @(
|
||||
"--context", $DockerContext,
|
||||
"create",
|
||||
"--name", $containerName,
|
||||
"--network", "host",
|
||||
"-e", "RAP_BACKEND_URL=$BackendBaseUrl",
|
||||
"-e", "RAP_NODE_STATE_DIR=/tmp/state",
|
||||
"-e", "RAP_HEARTBEAT_INTERVAL_SECONDS=5",
|
||||
$ImageTag,
|
||||
"-backend-url", $BackendBaseUrl,
|
||||
"-state-dir", "/tmp/state",
|
||||
"-heartbeat-interval", "5s"
|
||||
) | Out-Null
|
||||
Invoke-Docker -Arguments @("--context", $DockerContext, "cp", "$nodeStateDir\.", "$containerName`:/tmp/state")
|
||||
Invoke-Docker -Arguments @("--context", $DockerContext, "start", $containerName) | Out-Null
|
||||
Write-Host "Started $containerName for $nodeName $nodeID"
|
||||
}
|
||||
|
||||
Start-Sleep -Seconds 12
|
||||
|
||||
for ($i = 0; $i -lt $nodes.Count; $i++) {
|
||||
for ($j = 0; $j -lt $nodes.Count; $j++) {
|
||||
if ($i -eq $j) { continue }
|
||||
$latency = 2 + (($i + $j) % 7)
|
||||
Invoke-Api -Method Post -Path "/clusters/$clusterID/mesh/links" -Body @{
|
||||
source_node_id = $nodes[$i].id
|
||||
target_node_id = $nodes[$j].id
|
||||
link_status = "reachable"
|
||||
latency_ms = $latency
|
||||
quality_score = 95 - $latency
|
||||
metadata = @{
|
||||
source = "deploy-test-nodes.ps1"
|
||||
synthetic = $true
|
||||
runtime_mesh_enabled = $false
|
||||
}
|
||||
} | Out-Null
|
||||
}
|
||||
}
|
||||
|
||||
$summary = Invoke-Api -Method Get -Path "/cluster-admin-summaries?actor_user_id=$actorUserID"
|
||||
$links = Invoke-Api -Method Get -Path "/clusters/$clusterID/mesh/links?actor_user_id=$actorUserID"
|
||||
|
||||
[pscustomobject]@{
|
||||
cluster_id = $clusterID
|
||||
nodes_started = $nodes.Count
|
||||
cluster_summaries = $summary.cluster_summaries
|
||||
mesh_link_count = @($links.mesh_links).Count
|
||||
state_root = $stateRoot
|
||||
} | ConvertTo-Json -Depth 20
|
||||
@@ -1,468 +0,0 @@
|
||||
param(
|
||||
[string]$DockerSshAlias = "test-docker",
|
||||
[string]$BackendImageTag = "rap-backend:dev-enrollment-bootstrap-smoke",
|
||||
[string]$NodeAgentImageTag = "rap-node-agent:dev-enrollment-bootstrap-smoke",
|
||||
[string]$AdminEmail = "fabric-owner-dev-bootstrap@example.local",
|
||||
[string]$AdminPassword = "SmokePass!123",
|
||||
[int]$ApiPort = 18121,
|
||||
[int]$PostgresPort = 15443,
|
||||
[int]$RedisPort = 16443,
|
||||
[int]$MeshPort = 19131,
|
||||
[string]$ResultPath = "artifacts\dev-cluster-enrollment-bootstrap-smoke-result.json",
|
||||
[switch]$KeepRunning
|
||||
)
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
$repoRoot = (Resolve-Path (Join-Path $PSScriptRoot "..\..")).ProviderPath
|
||||
$backendPublicBaseUrl = "http://192.168.200.61:$ApiPort/api/v1"
|
||||
$backendContainerBaseUrl = "http://127.0.0.1:$ApiPort/api/v1"
|
||||
$runId = "dev-bootstrap-" + (Get-Date -Format "yyyyMMdd-HHmmss")
|
||||
$remoteBuildDir = "/tmp/rap-dev-bootstrap-build-$runId"
|
||||
|
||||
$postgresName = "rap_dev_bootstrap_postgres"
|
||||
$redisName = "rap_dev_bootstrap_redis"
|
||||
$backendName = "rap_dev_bootstrap_backend"
|
||||
$nodeName = "rap-dev-bootstrap-node-core"
|
||||
$nodeContainerName = "rap_dev_bootstrap_node_core"
|
||||
|
||||
function Invoke-RemoteDocker {
|
||||
param([string[]]$Arguments)
|
||||
& ssh $DockerSshAlias docker @Arguments
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "ssh $DockerSshAlias docker $($Arguments -join ' ') failed with exit code $LASTEXITCODE"
|
||||
}
|
||||
}
|
||||
|
||||
function Invoke-RemoteDockerText {
|
||||
param([string[]]$Arguments)
|
||||
$previousErrorActionPreference = $ErrorActionPreference
|
||||
$ErrorActionPreference = "Continue"
|
||||
try {
|
||||
$output = & ssh $DockerSshAlias docker @Arguments 2>&1 | ForEach-Object { $_.ToString() }
|
||||
}
|
||||
finally {
|
||||
$ErrorActionPreference = $previousErrorActionPreference
|
||||
}
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "ssh $DockerSshAlias docker $($Arguments -join ' ') failed with exit code $LASTEXITCODE"
|
||||
}
|
||||
return $output
|
||||
}
|
||||
|
||||
function Invoke-RemoteShell {
|
||||
param([string]$Command)
|
||||
& ssh $DockerSshAlias $Command
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "ssh $DockerSshAlias $Command failed with exit code $LASTEXITCODE"
|
||||
}
|
||||
}
|
||||
|
||||
function Invoke-RemoteShellOptionalText {
|
||||
param([string]$Command)
|
||||
$previousErrorActionPreference = $ErrorActionPreference
|
||||
$ErrorActionPreference = "Continue"
|
||||
try {
|
||||
$output = & ssh $DockerSshAlias $Command 2>&1 | ForEach-Object { $_.ToString() }
|
||||
$exitCode = $LASTEXITCODE
|
||||
}
|
||||
finally {
|
||||
$ErrorActionPreference = $previousErrorActionPreference
|
||||
}
|
||||
return [pscustomobject]@{
|
||||
exit_code = $exitCode
|
||||
output = $output
|
||||
}
|
||||
}
|
||||
|
||||
function Send-RemoteBuildContext {
|
||||
Write-Host "Uploading backend and node-agent build context to $DockerSshAlias..."
|
||||
Invoke-RemoteShell -Command "rm -rf '$remoteBuildDir' && mkdir -p '$remoteBuildDir'"
|
||||
& tar -czf - -C $repoRoot "backend" "agents/rap-node-agent" | & ssh $DockerSshAlias "tar -xzf - -C '$remoteBuildDir'"
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "upload build context failed"
|
||||
}
|
||||
}
|
||||
|
||||
function Invoke-Api {
|
||||
param(
|
||||
[string]$Method,
|
||||
[string]$Path,
|
||||
[object]$Body = $null
|
||||
)
|
||||
$uri = "$backendPublicBaseUrl$Path"
|
||||
if ($null -eq $Body) {
|
||||
return Invoke-RestMethod -Method $Method -Uri $uri -TimeoutSec 30
|
||||
}
|
||||
return Invoke-RestMethod -Method $Method -Uri $uri -ContentType "application/json" -Body ($Body | ConvertTo-Json -Depth 60) -TimeoutSec 30
|
||||
}
|
||||
|
||||
function Wait-HttpReady {
|
||||
param([string]$Url)
|
||||
for ($i = 0; $i -lt 90; $i++) {
|
||||
try {
|
||||
$response = Invoke-WebRequest -UseBasicParsing -Uri $Url -TimeoutSec 2
|
||||
if ($response.StatusCode -ge 200 -and $response.StatusCode -lt 300) {
|
||||
return
|
||||
}
|
||||
}
|
||||
catch {
|
||||
Start-Sleep -Seconds 1
|
||||
}
|
||||
}
|
||||
throw "Timed out waiting for $Url"
|
||||
}
|
||||
|
||||
function Remove-SmokeContainers {
|
||||
foreach ($name in @($nodeContainerName, $backendName, $postgresName, $redisName)) {
|
||||
& ssh $DockerSshAlias docker rm -f $name 2>$null | Out-Null
|
||||
}
|
||||
}
|
||||
|
||||
function Wait-JoinRequest {
|
||||
param(
|
||||
[string]$ClusterID,
|
||||
[string]$ActorUserID,
|
||||
[string]$ExpectedNodeName,
|
||||
[int]$TimeoutSeconds = 60
|
||||
)
|
||||
$deadline = (Get-Date).AddSeconds($TimeoutSeconds)
|
||||
do {
|
||||
$response = Invoke-Api -Method Get -Path "/clusters/$ClusterID/join-requests?actor_user_id=$ActorUserID"
|
||||
$match = @($response.join_requests | Where-Object {
|
||||
$_.node_name -eq $ExpectedNodeName -and $_.status -eq "pending"
|
||||
} | Sort-Object created_at -Descending | Select-Object -First 1)
|
||||
if ($match.Count -gt 0) {
|
||||
return $match[0]
|
||||
}
|
||||
Start-Sleep -Seconds 1
|
||||
} while ((Get-Date) -lt $deadline)
|
||||
throw "Timed out waiting for node-agent join request"
|
||||
}
|
||||
|
||||
function Get-RemoteContainerIdentity {
|
||||
$remoteIdentity = "/tmp/$runId-identity.json"
|
||||
$command = "rm -f '$remoteIdentity'; docker cp '${nodeContainerName}:/tmp/state/identity.json' '$remoteIdentity' >/dev/null 2>&1 && cat '$remoteIdentity'"
|
||||
$result = Invoke-RemoteShellOptionalText -Command $command
|
||||
if ($result.exit_code -ne 0) {
|
||||
return $null
|
||||
}
|
||||
$raw = ($result.output -join "`n").Trim()
|
||||
if ($raw -eq "") {
|
||||
return $null
|
||||
}
|
||||
return $raw | ConvertFrom-Json
|
||||
}
|
||||
|
||||
function Wait-AgentApprovedIdentity {
|
||||
param([int]$TimeoutSeconds = 90)
|
||||
$deadline = (Get-Date).AddSeconds($TimeoutSeconds)
|
||||
do {
|
||||
$identity = Get-RemoteContainerIdentity
|
||||
if ($null -ne $identity -and
|
||||
$identity.node_id -and
|
||||
$identity.identity_status -eq "active" -and
|
||||
$identity.cluster_authority_public_key -and
|
||||
$identity.cluster_authority_fingerprint) {
|
||||
return $identity
|
||||
}
|
||||
Start-Sleep -Seconds 1
|
||||
} while ((Get-Date) -lt $deadline)
|
||||
throw "Timed out waiting for node-agent approved identity state"
|
||||
}
|
||||
|
||||
function Wait-NodeHeartbeat {
|
||||
param(
|
||||
[string]$ClusterID,
|
||||
[string]$NodeID,
|
||||
[string]$ActorUserID,
|
||||
[int]$TimeoutSeconds = 60
|
||||
)
|
||||
$deadline = (Get-Date).AddSeconds($TimeoutSeconds)
|
||||
do {
|
||||
$response = Invoke-Api -Method Get -Path "/clusters/$ClusterID/nodes/$NodeID/heartbeats?actor_user_id=$ActorUserID&limit=5"
|
||||
$heartbeats = @($response.heartbeats)
|
||||
if ($heartbeats.Count -gt 0) {
|
||||
return $heartbeats[0]
|
||||
}
|
||||
Start-Sleep -Seconds 1
|
||||
} while ((Get-Date) -lt $deadline)
|
||||
throw "Timed out waiting for node-agent heartbeat"
|
||||
}
|
||||
|
||||
function Wait-AgentLogContains {
|
||||
param(
|
||||
[string]$Pattern,
|
||||
[int]$TimeoutSeconds = 60
|
||||
)
|
||||
$deadline = (Get-Date).AddSeconds($TimeoutSeconds)
|
||||
do {
|
||||
$logs = Invoke-RemoteDockerText -Arguments @("logs", "--tail", "200", $nodeContainerName)
|
||||
$joined = $logs -join "`n"
|
||||
if ($joined -match $Pattern) {
|
||||
return $logs
|
||||
}
|
||||
Start-Sleep -Seconds 1
|
||||
} while ((Get-Date) -lt $deadline)
|
||||
return Invoke-RemoteDockerText -Arguments @("logs", "--tail", "200", $nodeContainerName)
|
||||
}
|
||||
|
||||
Write-Host "Dev cluster enrollment/bootstrap smoke run: $runId"
|
||||
Write-Host "Using SSH Docker host: $DockerSshAlias"
|
||||
|
||||
Remove-SmokeContainers
|
||||
Send-RemoteBuildContext
|
||||
|
||||
Write-Host "Building backend image on docker-test..."
|
||||
Invoke-RemoteDocker -Arguments @("build", "-f", "$remoteBuildDir/backend/Dockerfile", "-t", $BackendImageTag, "$remoteBuildDir/backend")
|
||||
|
||||
Write-Host "Building node-agent image on docker-test..."
|
||||
Invoke-RemoteDocker -Arguments @("build", "-f", "$remoteBuildDir/agents/rap-node-agent/Dockerfile", "-t", $NodeAgentImageTag, $remoteBuildDir)
|
||||
|
||||
Write-Host "Starting isolated PostgreSQL and Redis..."
|
||||
Invoke-RemoteDocker -Arguments @(
|
||||
"run", "-d",
|
||||
"--name", $postgresName,
|
||||
"-e", "POSTGRES_DB=remote_access_platform",
|
||||
"-e", "POSTGRES_USER=rap_user",
|
||||
"-e", "POSTGRES_PASSWORD=rap_password",
|
||||
"-p", "$PostgresPort`:5432",
|
||||
"postgres:16"
|
||||
)
|
||||
Invoke-RemoteDocker -Arguments @(
|
||||
"run", "-d",
|
||||
"--name", $redisName,
|
||||
"-p", "$RedisPort`:6379",
|
||||
"redis:7"
|
||||
)
|
||||
|
||||
Invoke-RemoteShell -Command "for i in `$(seq 1 60); do docker exec $postgresName pg_isready -U rap_user -d remote_access_platform >/dev/null 2>&1 && exit 0; sleep 1; done; exit 1"
|
||||
|
||||
Write-Host "Applying migrations..."
|
||||
Invoke-RemoteShell -Command "for f in `$(find '$remoteBuildDir/backend/migrations' -name '*.up.sql' | sort); do docker exec -i $postgresName psql -U rap_user -d remote_access_platform -v ON_ERROR_STOP=1 -f - < `$f; done"
|
||||
|
||||
$secretBytes = New-Object byte[] 32
|
||||
[Security.Cryptography.RandomNumberGenerator]::Fill($secretBytes)
|
||||
$secretKeyB64 = [Convert]::ToBase64String($secretBytes)
|
||||
|
||||
Write-Host "Starting backend..."
|
||||
Invoke-RemoteDocker -Arguments @(
|
||||
"run", "-d",
|
||||
"--name", $backendName,
|
||||
"--network", "host",
|
||||
"-e", "APP_NAME=rap-api",
|
||||
"-e", "APP_ENV=dev-bootstrap-smoke",
|
||||
"-e", "HTTP_HOST=0.0.0.0",
|
||||
"-e", "HTTP_PORT=$ApiPort",
|
||||
"-e", "POSTGRES_DSN=postgres://rap_user:rap_password@127.0.0.1:$PostgresPort/remote_access_platform?sslmode=disable",
|
||||
"-e", "REDIS_ADDR=127.0.0.1:$RedisPort",
|
||||
"-e", "AUTH_ACCESS_TOKEN_SECRET=dev-bootstrap-access-secret",
|
||||
"-e", "AUTH_REFRESH_HASH_SECRET=dev-bootstrap-refresh-secret",
|
||||
"-e", "INSTALLATION_AUTHORITY_MODE=legacy",
|
||||
"-e", "INSTALLATION_INSECURE_BOOTSTRAP_ENABLED=true",
|
||||
"-e", "SECRET_ENCRYPTION_KEY_B64=$secretKeyB64",
|
||||
"-e", "SECRET_ENCRYPTION_KEY_ID=$runId",
|
||||
$BackendImageTag
|
||||
)
|
||||
Wait-HttpReady -Url "http://192.168.200.61:$ApiPort/readyz"
|
||||
|
||||
Write-Host "Bootstrapping dev platform owner through installation API..."
|
||||
$bootstrap = Invoke-Api -Method Post -Path "/installation/bootstrap-owner" -Body @{
|
||||
email = $AdminEmail
|
||||
password = $AdminPassword
|
||||
}
|
||||
$installationStatus = Invoke-Api -Method Get -Path "/installation/status"
|
||||
|
||||
Write-Host "Logging in as platform owner..."
|
||||
$login = Invoke-Api -Method Post -Path "/auth/login" -Body @{
|
||||
email = $AdminEmail
|
||||
password = $AdminPassword
|
||||
device_fingerprint = "dev-bootstrap-smoke-device"
|
||||
device_label = "Dev bootstrap lifecycle smoke"
|
||||
trust_device = $true
|
||||
}
|
||||
$actorUserID = $login.user.id
|
||||
|
||||
Write-Host "Creating dev cluster and join token..."
|
||||
$cluster = Invoke-Api -Method Post -Path "/clusters/" -Body @{
|
||||
actor_user_id = $actorUserID
|
||||
slug = "dev-bootstrap-$((New-Guid).Guid.Substring(0, 8))"
|
||||
name = "Dev Enrollment Bootstrap Smoke"
|
||||
region = "docker-test"
|
||||
metadata = @{
|
||||
stage = "dev-enrollment-bootstrap-smoke"
|
||||
run_id = $runId
|
||||
mandatory_roles_only = $true
|
||||
production_forwarding = $false
|
||||
service_payload_forwarding = $false
|
||||
}
|
||||
}
|
||||
$clusterID = $cluster.cluster.id
|
||||
|
||||
Invoke-Api -Method Put -Path "/fabric/testing-flags" -Body @{
|
||||
actor_user_id = $actorUserID
|
||||
scope_type = "platform"
|
||||
scope_id = $null
|
||||
cluster_id = $null
|
||||
enabled = $true
|
||||
telemetry_enabled = $true
|
||||
synthetic_links_enabled = $true
|
||||
history_retention_hours = 24
|
||||
metadata = @{
|
||||
stage = "dev-enrollment-bootstrap-smoke"
|
||||
run_id = $runId
|
||||
production_forwarding = $false
|
||||
service_payload_forwarding = $false
|
||||
}
|
||||
} | Out-Null
|
||||
|
||||
$joinToken = Invoke-Api -Method Post -Path "/clusters/$clusterID/join-tokens" -Body @{
|
||||
actor_user_id = $actorUserID
|
||||
scope = @{
|
||||
purpose = "dev-enrollment-bootstrap-smoke"
|
||||
roles = @("core-mesh")
|
||||
mandatory_only = $true
|
||||
production_forwarding = $false
|
||||
}
|
||||
expires_at = (Get-Date).ToUniversalTime().AddHours(2).ToString("o")
|
||||
max_uses = 1
|
||||
}
|
||||
|
||||
Write-Host "Starting real node-agent enrollment container..."
|
||||
Invoke-RemoteDocker -Arguments @(
|
||||
"run", "-d",
|
||||
"--name", $nodeContainerName,
|
||||
"--network", "host",
|
||||
"-e", "RAP_BACKEND_URL=$backendContainerBaseUrl",
|
||||
"-e", "RAP_CLUSTER_ID=$clusterID",
|
||||
"-e", "RAP_JOIN_TOKEN=$($joinToken.join_token.token)",
|
||||
"-e", "RAP_NODE_NAME=$nodeName",
|
||||
"-e", "RAP_NODE_STATE_DIR=/tmp/state",
|
||||
"-e", "RAP_HEARTBEAT_INTERVAL_SECONDS=2",
|
||||
"-e", "RAP_ENROLLMENT_POLL_INTERVAL_SECONDS=1",
|
||||
"-e", "RAP_ENROLLMENT_POLL_TIMEOUT_SECONDS=90",
|
||||
"-e", "RAP_MESH_SYNTHETIC_RUNTIME_ENABLED=true",
|
||||
"-e", "RAP_MESH_LISTEN_ADDR=0.0.0.0:$MeshPort",
|
||||
"-e", "RAP_MESH_ADVERTISE_ENDPOINT=http://127.0.0.1:$MeshPort",
|
||||
"-e", "RAP_MESH_ADVERTISE_TRANSPORT=direct_tcp_tls",
|
||||
"-e", "RAP_MESH_CONNECTIVITY_MODE=direct",
|
||||
"-e", "RAP_MESH_NAT_TYPE=none",
|
||||
"-e", "RAP_MESH_REGION=docker-test",
|
||||
$NodeAgentImageTag
|
||||
)
|
||||
|
||||
Write-Host "Waiting for pending join request from node-agent..."
|
||||
$joinRequest = Wait-JoinRequest -ClusterID $clusterID -ActorUserID $actorUserID -ExpectedNodeName $nodeName
|
||||
|
||||
Write-Host "Approving join request..."
|
||||
$approved = Invoke-Api -Method Post -Path "/clusters/$clusterID/join-requests/$($joinRequest.id)/approve" -Body @{
|
||||
actor_user_id = $actorUserID
|
||||
node_key = $joinRequest.node_fingerprint
|
||||
ownership_type = "platform_managed"
|
||||
owner_organization_id = $null
|
||||
}
|
||||
|
||||
Write-Host "Waiting for node-agent to persist signed bootstrap authority pin..."
|
||||
$identity = Wait-AgentApprovedIdentity
|
||||
$nodeID = $identity.node_id
|
||||
|
||||
Write-Host "Assigning mandatory core-mesh role after approved identity..."
|
||||
Invoke-Api -Method Post -Path "/clusters/$clusterID/nodes/$nodeID/roles" -Body @{
|
||||
actor_user_id = $actorUserID
|
||||
role = "core-mesh"
|
||||
status = "active"
|
||||
policy = @{
|
||||
stage = "dev-enrollment-bootstrap-smoke"
|
||||
run_id = $runId
|
||||
mandatory = $true
|
||||
production_forwarding = $false
|
||||
service_payload_forwarding = $false
|
||||
}
|
||||
} | Out-Null
|
||||
|
||||
Write-Host "Waiting for heartbeat and signed synthetic config verification..."
|
||||
$heartbeat = Wait-NodeHeartbeat -ClusterID $clusterID -NodeID $nodeID -ActorUserID $actorUserID
|
||||
$configResponse = Invoke-Api -Method Get -Path "/clusters/$clusterID/nodes/$nodeID/mesh/synthetic-config"
|
||||
$syntheticConfig = $configResponse.synthetic_mesh_config
|
||||
$nodeLogs = Wait-AgentLogContains -Pattern "synthetic mesh config loaded: source=control_plane" -TimeoutSeconds 60
|
||||
$backendLogs = Invoke-RemoteDockerText -Arguments @("logs", "--tail", "120", $backendName)
|
||||
$summaries = Invoke-Api -Method Get -Path "/cluster-admin-summaries?actor_user_id=$actorUserID"
|
||||
|
||||
$agentLogText = $nodeLogs -join "`n"
|
||||
$passMatrix = [ordered]@{
|
||||
installation_bootstrapped = [bool]$installationStatus.installation.bootstrapped
|
||||
cluster_created = [bool]$clusterID
|
||||
join_token_has_cluster_authority_signature = [bool]($joinToken.join_token.authority_payload -and $joinToken.join_token.authority_signature)
|
||||
agent_submitted_pending_join_request = [bool]($joinRequest.id -and $joinRequest.status -eq "pending")
|
||||
approval_returned_signed_bootstrap = [bool]($approved.node_bootstrap.cluster_authority -and $approved.node_bootstrap.authority_payload -and $approved.node_bootstrap.authority_signature)
|
||||
agent_persisted_approved_identity = [bool]($identity.node_id -eq $approved.node_bootstrap.node_id -and $identity.identity_status -eq "active")
|
||||
agent_persisted_cluster_authority_pin = [bool]($identity.cluster_authority_public_key -and $identity.cluster_authority_fingerprint)
|
||||
heartbeat_after_auto_bootstrap = [bool]$heartbeat.id
|
||||
synthetic_config_has_cluster_authority_signature = [bool]($syntheticConfig.authority_required -and $syntheticConfig.cluster_authority -and $syntheticConfig.authority_payload -and $syntheticConfig.authority_signature)
|
||||
agent_verified_control_plane_synthetic_config = [bool]($agentLogText -match "synthetic mesh config loaded: source=control_plane")
|
||||
production_forwarding_disabled = [bool](-not $syntheticConfig.production_forwarding)
|
||||
}
|
||||
|
||||
$result = [ordered]@{
|
||||
run_id = $runId
|
||||
stage = "dev cluster enrollment/bootstrap lifecycle smoke"
|
||||
docker_host = $DockerSshAlias
|
||||
backend_base_url = $backendPublicBaseUrl
|
||||
api_port = $ApiPort
|
||||
postgres_port = $PostgresPort
|
||||
redis_port = $RedisPort
|
||||
mesh_port = $MeshPort
|
||||
backend_image = $BackendImageTag
|
||||
node_agent_image = $NodeAgentImageTag
|
||||
containers = @{
|
||||
backend = $backendName
|
||||
postgres = $postgresName
|
||||
redis = $redisName
|
||||
node_agent = $nodeContainerName
|
||||
}
|
||||
installation = $installationStatus.installation
|
||||
admin_user_id = $actorUserID
|
||||
cluster_id = $clusterID
|
||||
node_id = $nodeID
|
||||
join_request_id = $joinRequest.id
|
||||
join_token_signature = $joinToken.join_token.authority_signature
|
||||
approval_authority = $approved.node_bootstrap.cluster_authority
|
||||
identity_authority_fingerprint = $identity.cluster_authority_fingerprint
|
||||
synthetic_config_schema = $syntheticConfig.schema_version
|
||||
synthetic_config_version = $syntheticConfig.config_version
|
||||
synthetic_config_authority = $syntheticConfig.cluster_authority
|
||||
latest_heartbeat = $heartbeat
|
||||
cluster_summaries = $summaries.cluster_summaries
|
||||
pass_matrix = $passMatrix
|
||||
backend_log_tail = $backendLogs
|
||||
node_log_tail = $nodeLogs
|
||||
containers_left_running = [bool]$KeepRunning
|
||||
}
|
||||
|
||||
$failed = @($passMatrix.GetEnumerator() | Where-Object { -not $_.Value })
|
||||
$resultJson = $result | ConvertTo-Json -Depth 70
|
||||
|
||||
if ($ResultPath -ne "") {
|
||||
if ([System.IO.Path]::IsPathRooted($ResultPath)) {
|
||||
$resultFullPath = $ResultPath
|
||||
} else {
|
||||
$resultFullPath = Join-Path $repoRoot $ResultPath
|
||||
}
|
||||
New-Item -ItemType Directory -Force -Path (Split-Path $resultFullPath) | Out-Null
|
||||
Set-Content -Path $resultFullPath -Value $resultJson -Encoding UTF8
|
||||
Write-Host "Result written to $resultFullPath"
|
||||
}
|
||||
|
||||
$resultJson
|
||||
|
||||
if ($failed.Count -gt 0) {
|
||||
throw "Dev enrollment/bootstrap smoke failed: $($failed.Name -join ', ')"
|
||||
}
|
||||
|
||||
Invoke-RemoteShell -Command "rm -rf '$remoteBuildDir'"
|
||||
|
||||
if (-not $KeepRunning) {
|
||||
Write-Host "Cleaning up dev enrollment/bootstrap containers..."
|
||||
Remove-SmokeContainers
|
||||
}
|
||||
Reference in New Issue
Block a user