Initial project snapshot
This commit is contained in:
@@ -0,0 +1,390 @@
|
||||
param(
|
||||
[string]$DockerSshAlias = "test-docker",
|
||||
[string]$NodeAgentImageTag = "rap-node-agent:c17h-synthetic-smoke",
|
||||
[string]$AdminEmail = "fabric-owner-c17h@example.local",
|
||||
[string]$AdminPassword = "SmokePass!123",
|
||||
[int]$ApiPort = 18080,
|
||||
[int]$MeshBasePort = 19081,
|
||||
[switch]$KeepRunning
|
||||
)
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
$backendPublicBaseUrl = "http://192.168.200.61:$ApiPort/api/v1"
|
||||
$backendContainerBaseUrl = "http://127.0.0.1:$ApiPort/api/v1"
|
||||
$runId = "c17h-ssh-" + (Get-Date -Format "yyyyMMdd-HHmmss")
|
||||
$nodePrefix = "rap_c17h_node_"
|
||||
|
||||
function Invoke-RemoteDocker {
|
||||
param([string[]]$Arguments)
|
||||
& ssh $DockerSshAlias docker @Arguments
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "ssh $DockerSshAlias docker $($Arguments -join ' ') failed with exit code $LASTEXITCODE"
|
||||
}
|
||||
}
|
||||
|
||||
function Invoke-RemoteDockerText {
|
||||
param([string[]]$Arguments)
|
||||
$previousErrorActionPreference = $ErrorActionPreference
|
||||
$ErrorActionPreference = "Continue"
|
||||
try {
|
||||
$output = & ssh $DockerSshAlias docker @Arguments 2>&1 | ForEach-Object { $_.ToString() }
|
||||
}
|
||||
finally {
|
||||
$ErrorActionPreference = $previousErrorActionPreference
|
||||
}
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "ssh $DockerSshAlias docker $($Arguments -join ' ') failed with exit code $LASTEXITCODE"
|
||||
}
|
||||
return $output
|
||||
}
|
||||
|
||||
function Invoke-RemoteShell {
|
||||
param([string]$Command)
|
||||
& ssh $DockerSshAlias $Command
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "ssh $DockerSshAlias $Command failed with exit code $LASTEXITCODE"
|
||||
}
|
||||
}
|
||||
|
||||
function Send-RemoteFile {
|
||||
param(
|
||||
[string]$RemotePath,
|
||||
[string]$Content
|
||||
)
|
||||
$Content | & ssh $DockerSshAlias "cat > '$RemotePath'"
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "write remote file failed: $RemotePath"
|
||||
}
|
||||
}
|
||||
|
||||
function Invoke-Api {
|
||||
param(
|
||||
[string]$Method,
|
||||
[string]$Path,
|
||||
[object]$Body = $null
|
||||
)
|
||||
$uri = "$backendPublicBaseUrl$Path"
|
||||
if ($null -eq $Body) {
|
||||
return Invoke-RestMethod -Method $Method -Uri $uri -TimeoutSec 30
|
||||
}
|
||||
return Invoke-RestMethod -Method $Method -Uri $uri -ContentType "application/json" -Body ($Body | ConvertTo-Json -Depth 40) -TimeoutSec 30
|
||||
}
|
||||
|
||||
function Wait-HttpReady {
|
||||
param([string]$Url)
|
||||
for ($i = 0; $i -lt 60; $i++) {
|
||||
try {
|
||||
$response = Invoke-WebRequest -UseBasicParsing -Uri $Url -TimeoutSec 2
|
||||
if ($response.StatusCode -ge 200 -and $response.StatusCode -lt 300) {
|
||||
return
|
||||
}
|
||||
}
|
||||
catch {
|
||||
Start-Sleep -Seconds 1
|
||||
}
|
||||
}
|
||||
throw "Timed out waiting for $Url"
|
||||
}
|
||||
|
||||
function Remove-NodeContainers {
|
||||
foreach ($key in @("a", "b", "c", "r", "idle")) {
|
||||
& ssh $DockerSshAlias docker rm -f "$nodePrefix$key" 2>$null | Out-Null
|
||||
}
|
||||
}
|
||||
|
||||
Wait-HttpReady -Url "http://192.168.200.61:$ApiPort/readyz"
|
||||
|
||||
$login = Invoke-Api -Method Post -Path "/auth/login" -Body @{
|
||||
email = $AdminEmail
|
||||
password = $AdminPassword
|
||||
device_fingerprint = "c17h-ssh-smoke-device"
|
||||
device_label = "C17H SSH synthetic smoke"
|
||||
trust_device = $true
|
||||
}
|
||||
$actorUserID = $login.user.id
|
||||
|
||||
$cluster = Invoke-Api -Method Post -Path "/clusters/" -Body @{
|
||||
actor_user_id = $actorUserID
|
||||
slug = "c17h-ssh-$((New-Guid).Guid.Substring(0, 8))"
|
||||
name = "C17H SSH Synthetic Mesh Smoke"
|
||||
region = "docker-test"
|
||||
metadata = @{
|
||||
stage = "c17h"
|
||||
run_id = $runId
|
||||
production_forwarding = $false
|
||||
created_by = "c17h-multi-agent-synthetic-smoke-ssh.ps1"
|
||||
}
|
||||
}
|
||||
$clusterID = $cluster.cluster.id
|
||||
|
||||
Invoke-Api -Method Put -Path "/fabric/testing-flags" -Body @{
|
||||
actor_user_id = $actorUserID
|
||||
scope_type = "platform"
|
||||
scope_id = $null
|
||||
cluster_id = $null
|
||||
enabled = $true
|
||||
telemetry_enabled = $true
|
||||
synthetic_links_enabled = $true
|
||||
history_retention_hours = 24
|
||||
metadata = @{
|
||||
stage = "c17h"
|
||||
run_id = $runId
|
||||
production_forwarding = $false
|
||||
service_workload_traffic = $false
|
||||
}
|
||||
} | Out-Null
|
||||
|
||||
$joinToken = Invoke-Api -Method Post -Path "/clusters/$clusterID/join-tokens" -Body @{
|
||||
actor_user_id = $actorUserID
|
||||
scope = @{ purpose = "c17h-synthetic-smoke"; roles = @("core-mesh", "relay-node") }
|
||||
expires_at = (Get-Date).ToUniversalTime().AddHours(2).ToString("o")
|
||||
max_uses = 5
|
||||
}
|
||||
|
||||
$nodeSpecs = @(
|
||||
@{ key = "a"; name = "c17h-node-a"; roles = @("core-mesh"); port = $MeshBasePort },
|
||||
@{ key = "r"; name = "c17h-node-r"; roles = @("core-mesh", "relay-node"); port = ($MeshBasePort + 1) },
|
||||
@{ key = "b"; name = "c17h-node-b"; roles = @("core-mesh"); port = ($MeshBasePort + 2) },
|
||||
@{ key = "c"; name = "c17h-node-c"; roles = @("core-mesh"); port = ($MeshBasePort + 3) },
|
||||
@{ key = "idle"; name = "c17h-node-idle"; roles = @("core-mesh"); port = ($MeshBasePort + 4) }
|
||||
)
|
||||
|
||||
$nodes = @{}
|
||||
foreach ($spec in $nodeSpecs) {
|
||||
$fingerprint = "c17h-fp-$($spec.key)-$([guid]::NewGuid().ToString('N'))"
|
||||
$publicKey = "c17h-pub-$($spec.key)-$([guid]::NewGuid().ToString('N'))"
|
||||
$joinRequest = Invoke-Api -Method Post -Path "/clusters/$clusterID/join-requests" -Body @{
|
||||
join_token = $joinToken.join_token.token
|
||||
node_name = $spec.name
|
||||
node_fingerprint = $fingerprint
|
||||
public_key = $publicKey
|
||||
reported_capabilities = @{
|
||||
can_accept_node_ingress = $true
|
||||
can_route_mesh = $true
|
||||
testing_node = $true
|
||||
}
|
||||
reported_facts = @{
|
||||
os = "linux"
|
||||
runtime = "docker-test"
|
||||
stage = "c17h"
|
||||
run_id = $runId
|
||||
}
|
||||
requested_roles = $spec.roles
|
||||
}
|
||||
$approved = Invoke-Api -Method Post -Path "/clusters/$clusterID/join-requests/$($joinRequest.join_request.id)/approve" -Body @{
|
||||
actor_user_id = $actorUserID
|
||||
node_key = $fingerprint
|
||||
ownership_type = "platform_managed"
|
||||
owner_organization_id = $null
|
||||
}
|
||||
$nodeID = $approved.node_bootstrap.node_id
|
||||
foreach ($role in $spec.roles) {
|
||||
Invoke-Api -Method Post -Path "/clusters/$clusterID/nodes/$nodeID/roles" -Body @{
|
||||
actor_user_id = $actorUserID
|
||||
role = $role
|
||||
status = "active"
|
||||
policy = @{
|
||||
stage = "c17h"
|
||||
run_id = $runId
|
||||
synthetic_only = $true
|
||||
}
|
||||
} | Out-Null
|
||||
}
|
||||
$nodes[$spec.key] = [pscustomobject]@{
|
||||
id = $nodeID
|
||||
name = $spec.name
|
||||
fingerprint = $fingerprint
|
||||
public_key = $publicKey
|
||||
port = $spec.port
|
||||
roles = $spec.roles
|
||||
}
|
||||
}
|
||||
|
||||
$nodeAID = $nodes["a"].id
|
||||
$nodeRID = $nodes["r"].id
|
||||
$nodeBID = $nodes["b"].id
|
||||
$nodeCID = $nodes["c"].id
|
||||
$routeExpiresAt = (Get-Date).ToUniversalTime().AddHours(2).ToString("o")
|
||||
|
||||
$peerEndpointsDirect = @{}
|
||||
$peerEndpointsDirect[$nodeAID] = "http://127.0.0.1:$($nodes["a"].port)"
|
||||
$peerEndpointsDirect[$nodeBID] = "http://127.0.0.1:$($nodes["b"].port)"
|
||||
|
||||
$peerEndpointsRelay = @{}
|
||||
$peerEndpointsRelay[$nodeAID] = "http://127.0.0.1:$($nodes["a"].port)"
|
||||
$peerEndpointsRelay[$nodeRID] = "http://127.0.0.1:$($nodes["r"].port)"
|
||||
$peerEndpointsRelay[$nodeCID] = "http://127.0.0.1:$($nodes["c"].port)"
|
||||
|
||||
$directIntent = Invoke-Api -Method Post -Path "/clusters/$clusterID/mesh/route-intents" -Body @{
|
||||
actor_user_id = $actorUserID
|
||||
source_selector = @{ node_id = $nodeAID }
|
||||
destination_selector = @{ node_id = $nodeBID }
|
||||
service_class = "control"
|
||||
priority = 10
|
||||
policy = @{
|
||||
synthetic_enabled = $true
|
||||
peer_endpoints = $peerEndpointsDirect
|
||||
hops = @($nodeAID, $nodeBID)
|
||||
allowed_channels = @("fabric_control", "route_control")
|
||||
max_ttl = 4
|
||||
max_hops = 4
|
||||
expires_at = $routeExpiresAt
|
||||
route_version = "$runId-direct"
|
||||
policy_version = "$runId-policy"
|
||||
peer_directory_version = "$runId-peers"
|
||||
production_forwarding = $false
|
||||
}
|
||||
}
|
||||
|
||||
$relayIntent = Invoke-Api -Method Post -Path "/clusters/$clusterID/mesh/route-intents" -Body @{
|
||||
actor_user_id = $actorUserID
|
||||
source_selector = @{ node_id = $nodeAID }
|
||||
destination_selector = @{ node_id = $nodeCID }
|
||||
service_class = "control"
|
||||
priority = 20
|
||||
policy = @{
|
||||
synthetic_enabled = $true
|
||||
peer_endpoints = $peerEndpointsRelay
|
||||
hops = @($nodeAID, $nodeRID, $nodeCID)
|
||||
allowed_channels = @("fabric_control", "route_control")
|
||||
max_ttl = 6
|
||||
max_hops = 6
|
||||
expires_at = $routeExpiresAt
|
||||
route_version = "$runId-relay"
|
||||
policy_version = "$runId-policy"
|
||||
peer_directory_version = "$runId-peers"
|
||||
production_forwarding = $false
|
||||
}
|
||||
}
|
||||
|
||||
$configs = @{}
|
||||
foreach ($key in @("a", "r", "b", "c", "idle")) {
|
||||
$configs[$key] = Invoke-Api -Method Get -Path "/clusters/$clusterID/nodes/$($nodes[$key].id)/mesh/synthetic-config"
|
||||
}
|
||||
|
||||
Remove-NodeContainers
|
||||
|
||||
foreach ($key in @("a", "r", "b", "c", "idle")) {
|
||||
$node = $nodes[$key]
|
||||
$containerName = "$nodePrefix$key"
|
||||
$remoteStateDir = "/tmp/$runId-$key"
|
||||
Invoke-RemoteShell -Command "rm -rf '$remoteStateDir' && mkdir -p '$remoteStateDir'"
|
||||
$identity = @{
|
||||
node_id = $node.id
|
||||
cluster_id = $clusterID
|
||||
node_name = $node.name
|
||||
node_fingerprint = $node.fingerprint
|
||||
public_key = $node.public_key
|
||||
identity_status = "active"
|
||||
created_at = (Get-Date).ToUniversalTime().ToString("o")
|
||||
updated_at = (Get-Date).ToUniversalTime().ToString("o")
|
||||
} | ConvertTo-Json -Depth 10
|
||||
Send-RemoteFile -RemotePath "$remoteStateDir/identity.json" -Content $identity
|
||||
Invoke-RemoteDocker -Arguments @(
|
||||
"create",
|
||||
"--name", $containerName,
|
||||
"--network", "host",
|
||||
"-e", "RAP_BACKEND_URL=$backendContainerBaseUrl",
|
||||
"-e", "RAP_NODE_STATE_DIR=/tmp/state",
|
||||
"-e", "RAP_HEARTBEAT_INTERVAL_SECONDS=5",
|
||||
"-e", "RAP_MESH_SYNTHETIC_RUNTIME_ENABLED=true",
|
||||
"-e", "RAP_MESH_LISTEN_ADDR=0.0.0.0:$($node.port)",
|
||||
$NodeAgentImageTag,
|
||||
"-backend-url", $backendContainerBaseUrl,
|
||||
"-state-dir", "/tmp/state",
|
||||
"-heartbeat-interval", "5s",
|
||||
"-mesh-synthetic-runtime-enabled",
|
||||
"-mesh-listen-addr", "0.0.0.0:$($node.port)"
|
||||
) | Out-Null
|
||||
Invoke-RemoteDocker -Arguments @("cp", "$remoteStateDir/.", "$containerName`:/tmp/state")
|
||||
Invoke-RemoteDocker -Arguments @("start", $containerName) | Out-Null
|
||||
}
|
||||
|
||||
Start-Sleep -Seconds 25
|
||||
|
||||
$links = Invoke-Api -Method Get -Path "/clusters/$clusterID/mesh/links?actor_user_id=$actorUserID"
|
||||
$summary = Invoke-Api -Method Get -Path "/cluster-admin-summaries?actor_user_id=$actorUserID"
|
||||
|
||||
$nodeLogs = @{}
|
||||
foreach ($key in @("a", "r", "b", "c", "idle")) {
|
||||
$nodeLogs[$key] = Invoke-RemoteDockerText -Arguments @("logs", "$nodePrefix$key")
|
||||
}
|
||||
$backendLogs = Invoke-RemoteDockerText -Arguments @("logs", "--tail", "50", "rap_c17h_backend")
|
||||
|
||||
$meshLinks = @($links.mesh_links)
|
||||
$routeHealthLinks = @($meshLinks | Where-Object {
|
||||
$_.metadata.observation_type -eq "synthetic_route_health" -and
|
||||
$_.metadata.config_source -eq "control_plane"
|
||||
})
|
||||
$directHealth = @($routeHealthLinks | Where-Object { $_.metadata.route_id -eq $directIntent.route_intent.id -and $_.link_status -eq "reachable" })
|
||||
$relayHealth = @($routeHealthLinks | Where-Object { $_.metadata.route_id -eq $relayIntent.route_intent.id -and $_.link_status -eq "reachable" })
|
||||
|
||||
$passMatrix = [ordered]@{
|
||||
backend_ready = $true
|
||||
platform_owner_login = [bool]$actorUserID
|
||||
cluster_created = [bool]$clusterID
|
||||
fabric_testing_flags_enabled = $true
|
||||
node_a_scoped_config_enabled = $configs["a"].synthetic_mesh_config.enabled -eq $true
|
||||
node_a_has_direct_and_relay_routes = @($configs["a"].synthetic_mesh_config.routes).Count -eq 2
|
||||
relay_node_has_only_relay_route = @($configs["r"].synthetic_mesh_config.routes).Count -eq 1
|
||||
direct_destination_node_has_only_direct_route = @($configs["b"].synthetic_mesh_config.routes).Count -eq 1
|
||||
relay_destination_node_has_only_relay_route = @($configs["c"].synthetic_mesh_config.routes).Count -eq 1
|
||||
idle_node_has_no_routes = @($configs["idle"].synthetic_mesh_config.routes).Count -eq 0
|
||||
control_plane_config_used = (@("a", "r", "b", "c", "idle") | Where-Object { ($nodeLogs[$_] -join "`n") -match "source=control_plane" }).Count -eq 5
|
||||
direct_route_health_reported = $directHealth.Count -gt 0
|
||||
relay_route_health_reported = $relayHealth.Count -gt 0
|
||||
production_forwarding_disabled = (
|
||||
$configs["a"].synthetic_mesh_config.production_forwarding -eq $false -and
|
||||
$configs["r"].synthetic_mesh_config.production_forwarding -eq $false -and
|
||||
$configs["b"].synthetic_mesh_config.production_forwarding -eq $false -and
|
||||
$configs["c"].synthetic_mesh_config.production_forwarding -eq $false -and
|
||||
$configs["idle"].synthetic_mesh_config.production_forwarding -eq $false
|
||||
)
|
||||
}
|
||||
|
||||
$result = [pscustomobject]@{
|
||||
stage = "C17H deployed multi-agent synthetic config smoke"
|
||||
run_id = $runId
|
||||
backend_base_url = $backendPublicBaseUrl
|
||||
cluster_id = $clusterID
|
||||
node_ids = @{
|
||||
a = $nodes["a"].id
|
||||
r = $nodes["r"].id
|
||||
b = $nodes["b"].id
|
||||
c = $nodes["c"].id
|
||||
idle = $nodes["idle"].id
|
||||
}
|
||||
route_intents = @{
|
||||
direct = $directIntent.route_intent.id
|
||||
relay = $relayIntent.route_intent.id
|
||||
}
|
||||
scoped_config_route_counts = @{
|
||||
a = @($configs["a"].synthetic_mesh_config.routes).Count
|
||||
r = @($configs["r"].synthetic_mesh_config.routes).Count
|
||||
b = @($configs["b"].synthetic_mesh_config.routes).Count
|
||||
c = @($configs["c"].synthetic_mesh_config.routes).Count
|
||||
idle = @($configs["idle"].synthetic_mesh_config.routes).Count
|
||||
}
|
||||
mesh_link_count = $meshLinks.Count
|
||||
route_health_count = $routeHealthLinks.Count
|
||||
pass_matrix = $passMatrix
|
||||
direct_route_health = $directHealth | Select-Object -First 3
|
||||
relay_route_health = $relayHealth | Select-Object -First 3
|
||||
cluster_summaries = $summary.cluster_summaries
|
||||
backend_log_tail = $backendLogs
|
||||
node_log_tail = $nodeLogs
|
||||
containers_left_running = [bool]$KeepRunning
|
||||
}
|
||||
|
||||
$failed = @($passMatrix.GetEnumerator() | Where-Object { -not $_.Value })
|
||||
$result | ConvertTo-Json -Depth 50
|
||||
|
||||
if ($failed.Count -gt 0) {
|
||||
throw "C17H smoke failed: $($failed.Name -join ', ')"
|
||||
}
|
||||
|
||||
if (-not $KeepRunning) {
|
||||
Remove-NodeContainers
|
||||
}
|
||||
@@ -0,0 +1,460 @@
|
||||
param(
|
||||
[string]$DockerContext = "test-ubuntu",
|
||||
[string]$BackendImageTag = "rap-backend:c17h-synthetic-smoke",
|
||||
[string]$NodeAgentImageTag = "rap-node-agent:c17h-synthetic-smoke",
|
||||
[string]$AdminEmail = "fabric-owner-c17h@example.local",
|
||||
[string]$AdminPassword = "SmokePass!123",
|
||||
[int]$ApiPort = 18080,
|
||||
[int]$PostgresPort = 15432,
|
||||
[int]$RedisPort = 16379,
|
||||
[int]$MeshBasePort = 19081,
|
||||
[switch]$KeepRunning
|
||||
)
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
$repoRoot = (Resolve-Path (Join-Path $PSScriptRoot "..\..")).ProviderPath
|
||||
$backendBaseUrl = "http://127.0.0.1:$ApiPort/api/v1"
|
||||
$backendPublicBaseUrl = "http://192.168.200.61:$ApiPort/api/v1"
|
||||
$runId = "c17h-" + (Get-Date -Format "yyyyMMdd-HHmmss")
|
||||
|
||||
$postgresName = "rap_c17h_postgres"
|
||||
$redisName = "rap_c17h_redis"
|
||||
$backendName = "rap_c17h_backend"
|
||||
$nodePrefix = "rap_c17h_node_"
|
||||
|
||||
function Invoke-Docker {
|
||||
param([string[]]$Arguments)
|
||||
docker --context $DockerContext @Arguments
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "docker --context $DockerContext $($Arguments -join ' ') failed with exit code $LASTEXITCODE"
|
||||
}
|
||||
}
|
||||
|
||||
function Invoke-DockerText {
|
||||
param([string[]]$Arguments)
|
||||
$output = docker --context $DockerContext @Arguments 2>&1 | ForEach-Object { $_.ToString() }
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "docker --context $DockerContext $($Arguments -join ' ') failed with exit code $LASTEXITCODE"
|
||||
}
|
||||
return $output
|
||||
}
|
||||
|
||||
function Invoke-Api {
|
||||
param(
|
||||
[string]$Method,
|
||||
[string]$Path,
|
||||
[object]$Body = $null
|
||||
)
|
||||
$uri = "$backendPublicBaseUrl$Path"
|
||||
if ($null -eq $Body) {
|
||||
return Invoke-RestMethod -Method $Method -Uri $uri -TimeoutSec 30
|
||||
}
|
||||
return Invoke-RestMethod -Method $Method -Uri $uri -ContentType "application/json" -Body ($Body | ConvertTo-Json -Depth 30) -TimeoutSec 30
|
||||
}
|
||||
|
||||
function Wait-HttpReady {
|
||||
param([string]$Url)
|
||||
for ($i = 0; $i -lt 60; $i++) {
|
||||
try {
|
||||
$response = Invoke-WebRequest -UseBasicParsing -Uri $Url -TimeoutSec 2
|
||||
if ($response.StatusCode -ge 200 -and $response.StatusCode -lt 300) {
|
||||
return
|
||||
}
|
||||
}
|
||||
catch {
|
||||
Start-Sleep -Seconds 1
|
||||
}
|
||||
}
|
||||
throw "Timed out waiting for $Url"
|
||||
}
|
||||
|
||||
function Invoke-PostgresSql {
|
||||
param([string]$Sql)
|
||||
$Sql | docker --context $DockerContext exec -i $postgresName psql -U rap_user -d remote_access_platform -v ON_ERROR_STOP=1 -f -
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "psql command failed"
|
||||
}
|
||||
}
|
||||
|
||||
function Remove-C17HContainers {
|
||||
$names = @($backendName, $postgresName, $redisName)
|
||||
foreach ($letter in @("a", "b", "c", "r", "idle")) {
|
||||
$names += "$nodePrefix$letter"
|
||||
}
|
||||
foreach ($name in $names) {
|
||||
docker --context $DockerContext rm -f $name 2>$null | Out-Null
|
||||
}
|
||||
}
|
||||
|
||||
Write-Host "C17H smoke run: $runId"
|
||||
Write-Host "Using Docker context: $DockerContext"
|
||||
|
||||
Remove-C17HContainers
|
||||
|
||||
Write-Host "Building backend image..."
|
||||
Invoke-Docker -Arguments @("build", "-f", "$repoRoot\backend\Dockerfile", "-t", $BackendImageTag, "$repoRoot\backend")
|
||||
|
||||
Write-Host "Building node-agent image..."
|
||||
Invoke-Docker -Arguments @("build", "-f", "$repoRoot\agents\rap-node-agent\Dockerfile", "-t", $NodeAgentImageTag, "$repoRoot")
|
||||
|
||||
Write-Host "Starting PostgreSQL and Redis..."
|
||||
Invoke-Docker -Arguments @(
|
||||
"run", "-d",
|
||||
"--name", $postgresName,
|
||||
"-e", "POSTGRES_DB=remote_access_platform",
|
||||
"-e", "POSTGRES_USER=rap_user",
|
||||
"-e", "POSTGRES_PASSWORD=rap_password",
|
||||
"-p", "$PostgresPort`:5432",
|
||||
"postgres:16"
|
||||
)
|
||||
Invoke-Docker -Arguments @(
|
||||
"run", "-d",
|
||||
"--name", $redisName,
|
||||
"-p", "$RedisPort`:6379",
|
||||
"redis:7"
|
||||
)
|
||||
|
||||
for ($i = 0; $i -lt 60; $i++) {
|
||||
docker --context $DockerContext exec $postgresName pg_isready -U rap_user -d remote_access_platform | Out-Null
|
||||
if ($LASTEXITCODE -eq 0) {
|
||||
break
|
||||
}
|
||||
Start-Sleep -Seconds 1
|
||||
}
|
||||
docker --context $DockerContext exec $postgresName pg_isready -U rap_user -d remote_access_platform | Out-Null
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "PostgreSQL did not become ready"
|
||||
}
|
||||
|
||||
Write-Host "Applying migrations..."
|
||||
$migrationFiles = Get-ChildItem -Path (Join-Path $repoRoot "backend\migrations") -Filter "*.up.sql" | Sort-Object Name
|
||||
foreach ($migration in $migrationFiles) {
|
||||
Write-Host "Applying $($migration.Name)"
|
||||
Get-Content -Raw $migration.FullName | docker --context $DockerContext exec -i $postgresName psql -U rap_user -d remote_access_platform -v ON_ERROR_STOP=1 -f -
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "Migration failed: $($migration.Name)"
|
||||
}
|
||||
}
|
||||
|
||||
Write-Host "Seeding platform owner..."
|
||||
$adminUserID = [guid]::NewGuid().ToString()
|
||||
$adminHash = '$2a$10$AqLRexkI1yXbuiMPU6dHM.KVUhF.t..9NolyK4OOodQTyTsHyG.7u'
|
||||
Invoke-PostgresSql -Sql @"
|
||||
INSERT INTO users (id, email, password_hash, mfa_enabled, platform_role)
|
||||
VALUES ('$adminUserID'::uuid, '$AdminEmail', '$adminHash', FALSE, 'platform_admin')
|
||||
ON CONFLICT (email) DO UPDATE SET
|
||||
password_hash = EXCLUDED.password_hash,
|
||||
platform_role = 'platform_admin',
|
||||
updated_at = NOW();
|
||||
"@
|
||||
|
||||
Write-Host "Starting backend..."
|
||||
Invoke-Docker -Arguments @(
|
||||
"run", "-d",
|
||||
"--name", $backendName,
|
||||
"--network", "host",
|
||||
"-e", "APP_NAME=rap-api",
|
||||
"-e", "APP_ENV=c17h-smoke",
|
||||
"-e", "HTTP_HOST=0.0.0.0",
|
||||
"-e", "HTTP_PORT=$ApiPort",
|
||||
"-e", "POSTGRES_DSN=postgres://rap_user:rap_password@127.0.0.1:$PostgresPort/remote_access_platform?sslmode=disable",
|
||||
"-e", "REDIS_ADDR=127.0.0.1:$RedisPort",
|
||||
"-e", "AUTH_ACCESS_TOKEN_SECRET=c17h-access-secret",
|
||||
"-e", "AUTH_REFRESH_HASH_SECRET=c17h-refresh-secret",
|
||||
$BackendImageTag
|
||||
)
|
||||
Wait-HttpReady -Url "http://192.168.200.61:$ApiPort/readyz"
|
||||
|
||||
Write-Host "Logging in as platform owner..."
|
||||
$login = Invoke-Api -Method Post -Path "/auth/login" -Body @{
|
||||
email = $AdminEmail
|
||||
password = $AdminPassword
|
||||
device_fingerprint = "c17h-smoke-device"
|
||||
device_label = "C17H synthetic smoke"
|
||||
trust_device = $true
|
||||
}
|
||||
$actorUserID = $login.user.id
|
||||
|
||||
Write-Host "Creating C17H cluster..."
|
||||
$cluster = Invoke-Api -Method Post -Path "/clusters/" -Body @{
|
||||
actor_user_id = $actorUserID
|
||||
slug = "c17h-$((New-Guid).Guid.Substring(0, 8))"
|
||||
name = "C17H Synthetic Mesh Smoke"
|
||||
region = "test-docker"
|
||||
metadata = @{
|
||||
stage = "c17h"
|
||||
production_forwarding = $false
|
||||
created_by = "c17h-multi-agent-synthetic-smoke.ps1"
|
||||
}
|
||||
}
|
||||
$clusterID = $cluster.cluster.id
|
||||
|
||||
Write-Host "Enabling test-only Fabric flags..."
|
||||
Invoke-Api -Method Put -Path "/fabric/testing-flags" -Body @{
|
||||
actor_user_id = $actorUserID
|
||||
scope_type = "platform"
|
||||
scope_id = $null
|
||||
cluster_id = $null
|
||||
enabled = $true
|
||||
telemetry_enabled = $true
|
||||
synthetic_links_enabled = $true
|
||||
history_retention_hours = 24
|
||||
metadata = @{
|
||||
stage = "c17h"
|
||||
production_forwarding = $false
|
||||
service_workload_traffic = $false
|
||||
}
|
||||
} | Out-Null
|
||||
|
||||
$joinToken = Invoke-Api -Method Post -Path "/clusters/$clusterID/join-tokens" -Body @{
|
||||
actor_user_id = $actorUserID
|
||||
scope = @{ purpose = "c17h-synthetic-smoke"; roles = @("core-mesh", "relay-node") }
|
||||
expires_at = (Get-Date).ToUniversalTime().AddHours(2).ToString("o")
|
||||
max_uses = 5
|
||||
}
|
||||
|
||||
$nodeSpecs = @(
|
||||
@{ key = "a"; name = "c17h-node-a"; roles = @("core-mesh"); port = $MeshBasePort },
|
||||
@{ key = "r"; name = "c17h-node-r"; roles = @("core-mesh", "relay-node"); port = ($MeshBasePort + 1) },
|
||||
@{ key = "b"; name = "c17h-node-b"; roles = @("core-mesh"); port = ($MeshBasePort + 2) },
|
||||
@{ key = "c"; name = "c17h-node-c"; roles = @("core-mesh"); port = ($MeshBasePort + 3) },
|
||||
@{ key = "idle"; name = "c17h-node-idle"; roles = @("core-mesh"); port = ($MeshBasePort + 4) }
|
||||
)
|
||||
|
||||
$nodes = @{}
|
||||
foreach ($spec in $nodeSpecs) {
|
||||
$fingerprint = "c17h-fp-$($spec.key)-$([guid]::NewGuid().ToString('N'))"
|
||||
$publicKey = "c17h-pub-$($spec.key)-$([guid]::NewGuid().ToString('N'))"
|
||||
$joinRequest = Invoke-Api -Method Post -Path "/clusters/$clusterID/join-requests" -Body @{
|
||||
join_token = $joinToken.join_token.token
|
||||
node_name = $spec.name
|
||||
node_fingerprint = $fingerprint
|
||||
public_key = $publicKey
|
||||
reported_capabilities = @{
|
||||
can_accept_node_ingress = $true
|
||||
can_route_mesh = $true
|
||||
testing_node = $true
|
||||
}
|
||||
reported_facts = @{
|
||||
os = "linux"
|
||||
runtime = "docker-test"
|
||||
stage = "c17h"
|
||||
}
|
||||
requested_roles = $spec.roles
|
||||
}
|
||||
$approved = Invoke-Api -Method Post -Path "/clusters/$clusterID/join-requests/$($joinRequest.join_request.id)/approve" -Body @{
|
||||
actor_user_id = $actorUserID
|
||||
node_key = $fingerprint
|
||||
ownership_type = "platform_managed"
|
||||
owner_organization_id = $null
|
||||
}
|
||||
$nodeID = $approved.node_bootstrap.node_id
|
||||
foreach ($role in $spec.roles) {
|
||||
Invoke-Api -Method Post -Path "/clusters/$clusterID/nodes/$nodeID/roles" -Body @{
|
||||
actor_user_id = $actorUserID
|
||||
role = $role
|
||||
status = "active"
|
||||
policy = @{
|
||||
stage = "c17h"
|
||||
synthetic_only = $true
|
||||
}
|
||||
} | Out-Null
|
||||
}
|
||||
$nodes[$spec.key] = [pscustomobject]@{
|
||||
id = $nodeID
|
||||
name = $spec.name
|
||||
fingerprint = $fingerprint
|
||||
public_key = $publicKey
|
||||
port = $spec.port
|
||||
roles = $spec.roles
|
||||
}
|
||||
}
|
||||
|
||||
$nodeAID = $nodes["a"].id
|
||||
$nodeRID = $nodes["r"].id
|
||||
$nodeBID = $nodes["b"].id
|
||||
$nodeCID = $nodes["c"].id
|
||||
|
||||
$peerEndpointsDirect = @{}
|
||||
$peerEndpointsDirect[$nodeAID] = "http://127.0.0.1:$($nodes["a"].port)"
|
||||
$peerEndpointsDirect[$nodeBID] = "http://127.0.0.1:$($nodes["b"].port)"
|
||||
|
||||
$peerEndpointsRelay = @{}
|
||||
$peerEndpointsRelay[$nodeAID] = "http://127.0.0.1:$($nodes["a"].port)"
|
||||
$peerEndpointsRelay[$nodeRID] = "http://127.0.0.1:$($nodes["r"].port)"
|
||||
$peerEndpointsRelay[$nodeCID] = "http://127.0.0.1:$($nodes["c"].port)"
|
||||
|
||||
Write-Host "Creating synthetic route intents..."
|
||||
$directIntent = Invoke-Api -Method Post -Path "/clusters/$clusterID/mesh/route-intents" -Body @{
|
||||
actor_user_id = $actorUserID
|
||||
source_selector = @{ node_id = $nodeAID }
|
||||
destination_selector = @{ node_id = $nodeBID }
|
||||
service_class = "control"
|
||||
priority = 10
|
||||
policy = @{
|
||||
synthetic_enabled = $true
|
||||
peer_endpoints = $peerEndpointsDirect
|
||||
hops = @($nodeAID, $nodeBID)
|
||||
allowed_channels = @("fabric_control", "route_control")
|
||||
max_ttl = 4
|
||||
max_hops = 4
|
||||
route_version = "$runId-direct"
|
||||
policy_version = "$runId-policy"
|
||||
peer_directory_version = "$runId-peers"
|
||||
production_forwarding = $false
|
||||
}
|
||||
}
|
||||
$relayIntent = Invoke-Api -Method Post -Path "/clusters/$clusterID/mesh/route-intents" -Body @{
|
||||
actor_user_id = $actorUserID
|
||||
source_selector = @{ node_id = $nodeAID }
|
||||
destination_selector = @{ node_id = $nodeCID }
|
||||
service_class = "control"
|
||||
priority = 20
|
||||
policy = @{
|
||||
synthetic_enabled = $true
|
||||
peer_endpoints = $peerEndpointsRelay
|
||||
hops = @($nodeAID, $nodeRID, $nodeCID)
|
||||
allowed_channels = @("fabric_control", "route_control")
|
||||
max_ttl = 6
|
||||
max_hops = 6
|
||||
route_version = "$runId-relay"
|
||||
policy_version = "$runId-policy"
|
||||
peer_directory_version = "$runId-peers"
|
||||
production_forwarding = $false
|
||||
}
|
||||
}
|
||||
|
||||
$configs = @{}
|
||||
foreach ($key in @("a", "r", "b", "c", "idle")) {
|
||||
$configs[$key] = Invoke-Api -Method Get -Path "/clusters/$clusterID/nodes/$($nodes[$key].id)/mesh/synthetic-config"
|
||||
}
|
||||
|
||||
Write-Host "Starting node-agent containers..."
|
||||
foreach ($key in @("a", "r", "b", "c", "idle")) {
|
||||
$node = $nodes[$key]
|
||||
$containerName = "$nodePrefix$key"
|
||||
docker --context $DockerContext rm -f $containerName 2>$null | Out-Null
|
||||
$stateDir = Join-Path $env:TEMP "$runId-$key"
|
||||
New-Item -ItemType Directory -Force -Path $stateDir | Out-Null
|
||||
@{
|
||||
node_id = $node.id
|
||||
cluster_id = $clusterID
|
||||
node_name = $node.name
|
||||
node_fingerprint = $node.fingerprint
|
||||
public_key = $node.public_key
|
||||
identity_status = "active"
|
||||
created_at = (Get-Date).ToUniversalTime().ToString("o")
|
||||
updated_at = (Get-Date).ToUniversalTime().ToString("o")
|
||||
} | ConvertTo-Json -Depth 10 | Set-Content -Encoding UTF8 -Path (Join-Path $stateDir "identity.json")
|
||||
|
||||
Invoke-Docker -Arguments @(
|
||||
"create",
|
||||
"--name", $containerName,
|
||||
"--network", "host",
|
||||
"-e", "RAP_BACKEND_URL=$backendBaseUrl",
|
||||
"-e", "RAP_NODE_STATE_DIR=/tmp/state",
|
||||
"-e", "RAP_HEARTBEAT_INTERVAL_SECONDS=5",
|
||||
"-e", "RAP_MESH_SYNTHETIC_RUNTIME_ENABLED=true",
|
||||
"-e", "RAP_MESH_LISTEN_ADDR=0.0.0.0:$($node.port)",
|
||||
$NodeAgentImageTag,
|
||||
"-backend-url", $backendBaseUrl,
|
||||
"-state-dir", "/tmp/state",
|
||||
"-heartbeat-interval", "5s",
|
||||
"-mesh-synthetic-runtime-enabled",
|
||||
"-mesh-listen-addr", "0.0.0.0:$($node.port)"
|
||||
) | Out-Null
|
||||
Invoke-Docker -Arguments @("cp", "$stateDir\.", "$containerName`:/tmp/state")
|
||||
Invoke-Docker -Arguments @("start", $containerName) | Out-Null
|
||||
}
|
||||
|
||||
Write-Host "Waiting for synthetic observations..."
|
||||
Start-Sleep -Seconds 25
|
||||
|
||||
$links = Invoke-Api -Method Get -Path "/clusters/$clusterID/mesh/links?actor_user_id=$actorUserID"
|
||||
$summary = Invoke-Api -Method Get -Path "/cluster-admin-summaries?actor_user_id=$actorUserID"
|
||||
|
||||
$nodeLogs = @{}
|
||||
foreach ($key in @("a", "r", "b", "c", "idle")) {
|
||||
$nodeLogs[$key] = Invoke-DockerText -Arguments @("logs", "--tail", "30", "$nodePrefix$key")
|
||||
}
|
||||
$backendLogs = Invoke-DockerText -Arguments @("logs", "--tail", "30", $backendName)
|
||||
|
||||
$meshLinks = @($links.mesh_links)
|
||||
$routeHealthLinks = @($meshLinks | Where-Object {
|
||||
$_.metadata.observation_type -eq "synthetic_route_health" -and
|
||||
$_.metadata.config_source -eq "control_plane"
|
||||
})
|
||||
$directHealth = @($routeHealthLinks | Where-Object { $_.metadata.route_id -eq $directIntent.route_intent.id -and $_.link_status -eq "reachable" })
|
||||
$relayHealth = @($routeHealthLinks | Where-Object { $_.metadata.route_id -eq $relayIntent.route_intent.id -and $_.link_status -eq "reachable" })
|
||||
|
||||
$passMatrix = [ordered]@{
|
||||
backend_ready = $true
|
||||
platform_owner_login = [bool]$actorUserID
|
||||
cluster_created = [bool]$clusterID
|
||||
fabric_testing_flags_enabled = $true
|
||||
node_a_scoped_config_enabled = $configs["a"].synthetic_mesh_config.enabled -eq $true
|
||||
node_a_has_direct_and_relay_routes = @($configs["a"].synthetic_mesh_config.routes).Count -eq 2
|
||||
relay_node_has_only_relay_route = @($configs["r"].synthetic_mesh_config.routes).Count -eq 1
|
||||
direct_destination_node_has_only_direct_route = @($configs["b"].synthetic_mesh_config.routes).Count -eq 1
|
||||
relay_destination_node_has_only_relay_route = @($configs["c"].synthetic_mesh_config.routes).Count -eq 1
|
||||
idle_node_has_no_routes = @($configs["idle"].synthetic_mesh_config.routes).Count -eq 0
|
||||
node_agents_started = @("a", "r", "b", "c", "idle").Count -eq 5
|
||||
control_plane_config_used = ($nodeLogs["a"] -join "`n") -match "control_plane"
|
||||
direct_route_health_reported = $directHealth.Count -gt 0
|
||||
relay_route_health_reported = $relayHealth.Count -gt 0
|
||||
production_forwarding_disabled = (
|
||||
$configs["a"].synthetic_mesh_config.production_forwarding -eq $false -and
|
||||
$configs["r"].synthetic_mesh_config.production_forwarding -eq $false -and
|
||||
$configs["b"].synthetic_mesh_config.production_forwarding -eq $false -and
|
||||
$configs["c"].synthetic_mesh_config.production_forwarding -eq $false -and
|
||||
$configs["idle"].synthetic_mesh_config.production_forwarding -eq $false
|
||||
)
|
||||
}
|
||||
|
||||
$failed = @($passMatrix.GetEnumerator() | Where-Object { -not $_.Value })
|
||||
$result = [pscustomobject]@{
|
||||
stage = "C17H deployed multi-agent synthetic config smoke"
|
||||
run_id = $runId
|
||||
backend_base_url = $backendPublicBaseUrl
|
||||
cluster_id = $clusterID
|
||||
node_ids = @{
|
||||
a = $nodes["a"].id
|
||||
r = $nodes["r"].id
|
||||
b = $nodes["b"].id
|
||||
c = $nodes["c"].id
|
||||
idle = $nodes["idle"].id
|
||||
}
|
||||
route_intents = @{
|
||||
direct = $directIntent.route_intent.id
|
||||
relay = $relayIntent.route_intent.id
|
||||
}
|
||||
scoped_config_route_counts = @{
|
||||
a = @($configs["a"].synthetic_mesh_config.routes).Count
|
||||
r = @($configs["r"].synthetic_mesh_config.routes).Count
|
||||
b = @($configs["b"].synthetic_mesh_config.routes).Count
|
||||
c = @($configs["c"].synthetic_mesh_config.routes).Count
|
||||
idle = @($configs["idle"].synthetic_mesh_config.routes).Count
|
||||
}
|
||||
mesh_link_count = $meshLinks.Count
|
||||
route_health_count = $routeHealthLinks.Count
|
||||
pass_matrix = $passMatrix
|
||||
direct_route_health = $directHealth | Select-Object -First 3
|
||||
relay_route_health = $relayHealth | Select-Object -First 3
|
||||
cluster_summaries = $summary.cluster_summaries
|
||||
backend_log_tail = $backendLogs
|
||||
node_log_tail = $nodeLogs
|
||||
containers_left_running = [bool]$KeepRunning
|
||||
}
|
||||
|
||||
$result | ConvertTo-Json -Depth 40
|
||||
|
||||
if ($failed.Count -gt 0) {
|
||||
throw "C17H smoke failed: $($failed.Name -join ', ')"
|
||||
}
|
||||
|
||||
if (-not $KeepRunning) {
|
||||
Write-Host "Cleaning up C17H containers..."
|
||||
Remove-C17HContainers
|
||||
}
|
||||
@@ -0,0 +1,917 @@
|
||||
param(
|
||||
[string]$DockerSshAlias = "test-docker",
|
||||
[string]$BackendImageTag = "rap-backend:c17z18-rendezvous-smoke",
|
||||
[string]$NodeAgentImageTag = "rap-node-agent:c17z18-rendezvous-smoke",
|
||||
[string]$AdminEmail = "fabric-owner-c17z18@example.local",
|
||||
[string]$AdminPassword = "SmokePass!123",
|
||||
[int]$ApiPort = 18120,
|
||||
[int]$PostgresPort = 15442,
|
||||
[int]$RedisPort = 16442,
|
||||
[int]$MeshBasePort = 19120,
|
||||
[switch]$KeepRunning
|
||||
)
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
$repoRoot = (Resolve-Path (Join-Path $PSScriptRoot "..\..")).ProviderPath
|
||||
$backendPublicBaseUrl = "http://192.168.200.61:$ApiPort/api/v1"
|
||||
$backendContainerBaseUrl = "http://127.0.0.1:$ApiPort/api/v1"
|
||||
$runId = "c17z18-" + (Get-Date -Format "yyyyMMdd-HHmmss")
|
||||
$remoteBuildDir = "/tmp/rap-c17z18-build-$runId"
|
||||
|
||||
$postgresName = "rap_c17z12_postgres"
|
||||
$redisName = "rap_c17z12_redis"
|
||||
$backendName = "rap_c17z12_backend"
|
||||
$nodePrefix = "rap_c17z12_node_"
|
||||
|
||||
function Invoke-RemoteDocker {
|
||||
param([string[]]$Arguments)
|
||||
& ssh $DockerSshAlias docker @Arguments
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "ssh $DockerSshAlias docker $($Arguments -join ' ') failed with exit code $LASTEXITCODE"
|
||||
}
|
||||
}
|
||||
|
||||
function Invoke-RemoteDockerText {
|
||||
param([string[]]$Arguments)
|
||||
$previousErrorActionPreference = $ErrorActionPreference
|
||||
$ErrorActionPreference = "Continue"
|
||||
try {
|
||||
$output = & ssh $DockerSshAlias docker @Arguments 2>&1 | ForEach-Object { $_.ToString() }
|
||||
}
|
||||
finally {
|
||||
$ErrorActionPreference = $previousErrorActionPreference
|
||||
}
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "ssh $DockerSshAlias docker $($Arguments -join ' ') failed with exit code $LASTEXITCODE"
|
||||
}
|
||||
return $output
|
||||
}
|
||||
|
||||
function Invoke-RemoteShell {
|
||||
param([string]$Command)
|
||||
& ssh $DockerSshAlias $Command
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "ssh $DockerSshAlias $Command failed with exit code $LASTEXITCODE"
|
||||
}
|
||||
}
|
||||
|
||||
function Invoke-RemotePostgresSql {
|
||||
param([string]$Sql)
|
||||
$Sql | & ssh $DockerSshAlias "docker exec -i $postgresName psql -U rap_user -d remote_access_platform -v ON_ERROR_STOP=1 -f -"
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "remote psql command failed"
|
||||
}
|
||||
}
|
||||
|
||||
function Send-RemoteFile {
|
||||
param(
|
||||
[string]$RemotePath,
|
||||
[string]$Content
|
||||
)
|
||||
$Content | & ssh $DockerSshAlias "cat > '$RemotePath'"
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "write remote file failed: $RemotePath"
|
||||
}
|
||||
}
|
||||
|
||||
function Send-RemoteBuildContext {
|
||||
Write-Host "Uploading backend and node-agent build context to $DockerSshAlias..."
|
||||
Invoke-RemoteShell -Command "rm -rf '$remoteBuildDir' && mkdir -p '$remoteBuildDir'"
|
||||
& tar -czf - -C $repoRoot "backend" "agents/rap-node-agent" | & ssh $DockerSshAlias "tar -xzf - -C '$remoteBuildDir'"
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "upload build context failed"
|
||||
}
|
||||
}
|
||||
|
||||
function Invoke-Api {
|
||||
param(
|
||||
[string]$Method,
|
||||
[string]$Path,
|
||||
[object]$Body = $null
|
||||
)
|
||||
$uri = "$backendPublicBaseUrl$Path"
|
||||
if ($null -eq $Body) {
|
||||
return Invoke-RestMethod -Method $Method -Uri $uri -TimeoutSec 30
|
||||
}
|
||||
return Invoke-RestMethod -Method $Method -Uri $uri -ContentType "application/json" -Body ($Body | ConvertTo-Json -Depth 50) -TimeoutSec 30
|
||||
}
|
||||
|
||||
function Wait-HttpReady {
|
||||
param([string]$Url)
|
||||
for ($i = 0; $i -lt 60; $i++) {
|
||||
try {
|
||||
$response = Invoke-WebRequest -UseBasicParsing -Uri $Url -TimeoutSec 2
|
||||
if ($response.StatusCode -ge 200 -and $response.StatusCode -lt 300) {
|
||||
return
|
||||
}
|
||||
}
|
||||
catch {
|
||||
Start-Sleep -Seconds 1
|
||||
}
|
||||
}
|
||||
throw "Timed out waiting for $Url"
|
||||
}
|
||||
|
||||
function Remove-C17Z12Containers {
|
||||
$names = @($backendName, $postgresName, $redisName)
|
||||
foreach ($key in @("a", "b", "c", "r", "idle")) {
|
||||
$names += "$nodePrefix$key"
|
||||
}
|
||||
foreach ($name in $names) {
|
||||
& ssh $DockerSshAlias docker rm -f $name 2>$null | Out-Null
|
||||
}
|
||||
}
|
||||
|
||||
function New-EndpointCandidate {
|
||||
param(
|
||||
[string]$EndpointID,
|
||||
[string]$NodeID,
|
||||
[string]$Address,
|
||||
[string]$Transport,
|
||||
[string]$Reachability,
|
||||
[string]$ConnectivityMode,
|
||||
[string]$NATType,
|
||||
[int]$Priority,
|
||||
[string[]]$PolicyTags = @()
|
||||
)
|
||||
return @{
|
||||
endpoint_id = $EndpointID
|
||||
node_id = $NodeID
|
||||
transport = $Transport
|
||||
address = $Address
|
||||
address_family = "ipv4"
|
||||
reachability = $Reachability
|
||||
nat_type = $NATType
|
||||
connectivity_mode = $ConnectivityMode
|
||||
region = "docker-test"
|
||||
priority = $Priority
|
||||
policy_tags = $PolicyTags
|
||||
last_verified_at = (Get-Date).ToUniversalTime().ToString("o")
|
||||
metadata = @{
|
||||
stage = "c17z18"
|
||||
run_id = $runId
|
||||
service_workload_traffic = $false
|
||||
production_payload_forwarding = $false
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function Get-OptionalProperty {
|
||||
param(
|
||||
[object]$Object,
|
||||
[string]$PropertyName
|
||||
)
|
||||
if ($null -eq $Object) {
|
||||
return $null
|
||||
}
|
||||
$property = $Object.PSObject.Properties[$PropertyName]
|
||||
if ($null -eq $property) {
|
||||
return $null
|
||||
}
|
||||
return $property.Value
|
||||
}
|
||||
|
||||
function Get-OptionalArrayCount {
|
||||
param(
|
||||
[object]$Object,
|
||||
[string]$PropertyName
|
||||
)
|
||||
$value = Get-OptionalProperty -Object $Object -PropertyName $PropertyName
|
||||
if ($null -eq $value) {
|
||||
return 0
|
||||
}
|
||||
return @($value).Count
|
||||
}
|
||||
|
||||
function Get-LatestHeartbeatMetadataReport {
|
||||
param(
|
||||
[string]$NodeID,
|
||||
[string]$PropertyName
|
||||
)
|
||||
$heartbeats = Invoke-Api -Method Get -Path "/clusters/$clusterID/nodes/$NodeID/heartbeats?actor_user_id=$actorUserID&limit=5"
|
||||
$latest = @($heartbeats.heartbeats) | Select-Object -First 1
|
||||
$metadata = Get-OptionalProperty -Object $latest -PropertyName "metadata"
|
||||
return Get-OptionalProperty -Object $metadata -PropertyName $PropertyName
|
||||
}
|
||||
|
||||
function Get-LatestRendezvousLeaseReport {
|
||||
param([string]$NodeID)
|
||||
return Get-LatestHeartbeatMetadataReport -NodeID $NodeID -PropertyName "mesh_rendezvous_lease_report"
|
||||
}
|
||||
|
||||
function Get-LatestRoutePathDecisionReport {
|
||||
param([string]$NodeID)
|
||||
return Get-LatestHeartbeatMetadataReport -NodeID $NodeID -PropertyName "mesh_route_path_decision_report"
|
||||
}
|
||||
|
||||
function Get-LatestRouteGenerationReport {
|
||||
param([string]$NodeID)
|
||||
return Get-LatestHeartbeatMetadataReport -NodeID $NodeID -PropertyName "mesh_route_generation_report"
|
||||
}
|
||||
|
||||
function Get-LatestRouteHealthConfigReport {
|
||||
param([string]$NodeID)
|
||||
return Get-LatestHeartbeatMetadataReport -NodeID $NodeID -PropertyName "mesh_route_health_config_report"
|
||||
}
|
||||
|
||||
function Get-LatestRouteHealthFeedbackRefreshReport {
|
||||
param([string]$NodeID)
|
||||
return Get-LatestHeartbeatMetadataReport -NodeID $NodeID -PropertyName "mesh_route_health_feedback_refresh_report"
|
||||
}
|
||||
|
||||
function Select-C17Z18RouteHealthSnapshot {
|
||||
param([object[]]$MeshLinks)
|
||||
$routeHealthLinks = @($MeshLinks | Where-Object {
|
||||
$_.metadata.observation_type -eq "synthetic_route_health" -and
|
||||
$_.metadata.config_source -eq "control_plane"
|
||||
})
|
||||
$directHealth = @($routeHealthLinks | Where-Object {
|
||||
$_.metadata.route_id -eq $directIntent.route_intent.id -and
|
||||
$_.link_status -eq "reachable"
|
||||
})
|
||||
$rendezvousHealth = @($routeHealthLinks | Where-Object {
|
||||
$_.source_node_id -eq $nodeAID -and
|
||||
$_.metadata.route_id -eq $rendezvousIntent.route_intent.id -and
|
||||
$_.link_status -eq "reachable"
|
||||
})
|
||||
$replacementRouteHealth = @($rendezvousHealth | Where-Object {
|
||||
$_.metadata.route_path_decision_applied -eq $true -and
|
||||
$_.metadata.route_path_decision_selected_relay_id -eq $nodeSID -and
|
||||
(@($_.metadata.expected_effective_hops) -contains $nodeSID) -and
|
||||
-not (@($_.metadata.expected_effective_hops) -contains $nodeRID) -and
|
||||
(@($_.metadata.observed_ack_path) -contains $nodeSID) -and
|
||||
-not (@($_.metadata.observed_ack_path) -contains $nodeRID) -and
|
||||
$_.metadata.route_path_drift_detected -eq $false
|
||||
})
|
||||
return [pscustomobject]@{
|
||||
route_health_links = $routeHealthLinks
|
||||
direct_health = $directHealth
|
||||
rendezvous_health = $rendezvousHealth
|
||||
replacement_route_health = $replacementRouteHealth
|
||||
}
|
||||
}
|
||||
|
||||
function Get-C17Z18MeshLinkSnapshot {
|
||||
$links = Invoke-Api -Method Get -Path "/clusters/$clusterID/mesh/links?actor_user_id=$actorUserID"
|
||||
$meshLinks = @($links.mesh_links)
|
||||
$routeHealth = Select-C17Z18RouteHealthSnapshot -MeshLinks $meshLinks
|
||||
return [pscustomobject]@{
|
||||
links = $links
|
||||
mesh_links = $meshLinks
|
||||
route_health_links = @($routeHealth.route_health_links)
|
||||
direct_health = @($routeHealth.direct_health)
|
||||
rendezvous_health = @($routeHealth.rendezvous_health)
|
||||
replacement_route_health = @($routeHealth.replacement_route_health)
|
||||
}
|
||||
}
|
||||
|
||||
function Wait-C17Z18ReplacementRouteHealthSnapshot {
|
||||
param([int]$TimeoutSeconds = 40)
|
||||
$deadline = (Get-Date).AddSeconds($TimeoutSeconds)
|
||||
$latest = $null
|
||||
do {
|
||||
$latest = Get-C17Z18MeshLinkSnapshot
|
||||
if (@($latest.replacement_route_health).Count -gt 0) {
|
||||
return $latest
|
||||
}
|
||||
Start-Sleep -Seconds 2
|
||||
} while ((Get-Date) -lt $deadline)
|
||||
return $latest
|
||||
}
|
||||
|
||||
Write-Host "C17Z18 rendezvous relay replacement smoke run: $runId"
|
||||
Write-Host "Using SSH Docker host: $DockerSshAlias"
|
||||
|
||||
Remove-C17Z12Containers
|
||||
Send-RemoteBuildContext
|
||||
|
||||
Write-Host "Building backend image on docker-test..."
|
||||
Invoke-RemoteDocker -Arguments @("build", "-f", "$remoteBuildDir/backend/Dockerfile", "-t", $BackendImageTag, "$remoteBuildDir/backend")
|
||||
|
||||
Write-Host "Building node-agent image on docker-test..."
|
||||
Invoke-RemoteDocker -Arguments @("build", "-f", "$remoteBuildDir/agents/rap-node-agent/Dockerfile", "-t", $NodeAgentImageTag, $remoteBuildDir)
|
||||
|
||||
Write-Host "Starting PostgreSQL and Redis..."
|
||||
Invoke-RemoteDocker -Arguments @(
|
||||
"run", "-d",
|
||||
"--name", $postgresName,
|
||||
"-e", "POSTGRES_DB=remote_access_platform",
|
||||
"-e", "POSTGRES_USER=rap_user",
|
||||
"-e", "POSTGRES_PASSWORD=rap_password",
|
||||
"-p", "$PostgresPort`:5432",
|
||||
"postgres:16"
|
||||
)
|
||||
Invoke-RemoteDocker -Arguments @(
|
||||
"run", "-d",
|
||||
"--name", $redisName,
|
||||
"-p", "$RedisPort`:6379",
|
||||
"redis:7"
|
||||
)
|
||||
|
||||
Invoke-RemoteShell -Command "for i in `$(seq 1 60); do docker exec $postgresName pg_isready -U rap_user -d remote_access_platform >/dev/null 2>&1 && exit 0; sleep 1; done; exit 1"
|
||||
|
||||
Write-Host "Applying migrations..."
|
||||
Invoke-RemoteShell -Command "for f in `$(find '$remoteBuildDir/backend/migrations' -name '*.up.sql' | sort); do docker exec -i $postgresName psql -U rap_user -d remote_access_platform -v ON_ERROR_STOP=1 -f - < `$f; done"
|
||||
|
||||
Write-Host "Seeding platform owner..."
|
||||
$adminUserID = [guid]::NewGuid().ToString()
|
||||
$adminHash = '$2a$10$AqLRexkI1yXbuiMPU6dHM.KVUhF.t..9NolyK4OOodQTyTsHyG.7u'
|
||||
Invoke-RemotePostgresSql -Sql @"
|
||||
INSERT INTO users (id, email, password_hash, mfa_enabled, platform_role)
|
||||
VALUES ('$adminUserID'::uuid, '$AdminEmail', '$adminHash', FALSE, 'platform_admin')
|
||||
ON CONFLICT (email) DO UPDATE SET
|
||||
password_hash = EXCLUDED.password_hash,
|
||||
platform_role = 'platform_admin',
|
||||
updated_at = NOW();
|
||||
"@
|
||||
|
||||
Write-Host "Starting backend..."
|
||||
Invoke-RemoteDocker -Arguments @(
|
||||
"run", "-d",
|
||||
"--name", $backendName,
|
||||
"--network", "host",
|
||||
"-e", "APP_NAME=rap-api",
|
||||
"-e", "APP_ENV=c17z18-smoke",
|
||||
"-e", "HTTP_HOST=0.0.0.0",
|
||||
"-e", "HTTP_PORT=$ApiPort",
|
||||
"-e", "POSTGRES_DSN=postgres://rap_user:rap_password@127.0.0.1:$PostgresPort/remote_access_platform?sslmode=disable",
|
||||
"-e", "REDIS_ADDR=127.0.0.1:$RedisPort",
|
||||
"-e", "AUTH_ACCESS_TOKEN_SECRET=c17z18-access-secret",
|
||||
"-e", "AUTH_REFRESH_HASH_SECRET=c17z18-refresh-secret",
|
||||
$BackendImageTag
|
||||
)
|
||||
Wait-HttpReady -Url "http://192.168.200.61:$ApiPort/readyz"
|
||||
|
||||
Write-Host "Logging in as platform owner..."
|
||||
$login = Invoke-Api -Method Post -Path "/auth/login" -Body @{
|
||||
email = $AdminEmail
|
||||
password = $AdminPassword
|
||||
device_fingerprint = "c17z18-smoke-device"
|
||||
device_label = "C17Z18 rendezvous relay replacement smoke"
|
||||
trust_device = $true
|
||||
}
|
||||
$actorUserID = $login.user.id
|
||||
|
||||
Write-Host "Creating C17Z18 cluster..."
|
||||
$cluster = Invoke-Api -Method Post -Path "/clusters/" -Body @{
|
||||
actor_user_id = $actorUserID
|
||||
slug = "c17z18-$((New-Guid).Guid.Substring(0, 8))"
|
||||
name = "C17Z18 Rendezvous Relay Replacement Smoke"
|
||||
region = "docker-test"
|
||||
metadata = @{
|
||||
stage = "c17z18"
|
||||
run_id = $runId
|
||||
production_forwarding = $false
|
||||
service_workload_traffic = $false
|
||||
created_by = "c17z12-rendezvous-relay-smoke-ssh.ps1:c17z18"
|
||||
}
|
||||
}
|
||||
$clusterID = $cluster.cluster.id
|
||||
|
||||
Write-Host "Enabling test-only Fabric flags..."
|
||||
Invoke-Api -Method Put -Path "/fabric/testing-flags" -Body @{
|
||||
actor_user_id = $actorUserID
|
||||
scope_type = "platform"
|
||||
scope_id = $null
|
||||
cluster_id = $null
|
||||
enabled = $true
|
||||
telemetry_enabled = $true
|
||||
synthetic_links_enabled = $true
|
||||
history_retention_hours = 24
|
||||
metadata = @{
|
||||
stage = "c17z18"
|
||||
run_id = $runId
|
||||
production_forwarding = $false
|
||||
service_workload_traffic = $false
|
||||
}
|
||||
} | Out-Null
|
||||
|
||||
$joinToken = Invoke-Api -Method Post -Path "/clusters/$clusterID/join-tokens" -Body @{
|
||||
actor_user_id = $actorUserID
|
||||
scope = @{ purpose = "c17z18-rendezvous-relay-replacement-smoke"; roles = @("core-mesh", "relay-node") }
|
||||
expires_at = (Get-Date).ToUniversalTime().AddHours(2).ToString("o")
|
||||
max_uses = 5
|
||||
}
|
||||
|
||||
$nodeSpecs = @(
|
||||
@{ key = "a"; name = "c17z18-node-a-entry"; roles = @("core-mesh"); port = $MeshBasePort; transport = "direct_tcp_tls"; connectivity = "direct"; nat = "none" },
|
||||
@{ key = "r"; name = "c17z18-node-r-stale-relay"; roles = @("core-mesh", "relay-node"); port = ($MeshBasePort + 1); transport = "direct_tcp_tls"; connectivity = "direct"; nat = "none" },
|
||||
@{ key = "b"; name = "c17z18-node-b-direct"; roles = @("core-mesh"); port = ($MeshBasePort + 2); transport = "direct_tcp_tls"; connectivity = "direct"; nat = "none" },
|
||||
@{ key = "c"; name = "c17z18-node-c-outbound"; roles = @("core-mesh"); port = ($MeshBasePort + 3); transport = "outbound_reverse"; connectivity = "outbound_only"; nat = "symmetric" },
|
||||
@{ key = "idle"; name = "c17z18-node-s-alt-relay"; roles = @("core-mesh", "relay-node"); port = ($MeshBasePort + 4); transport = "direct_tcp_tls"; connectivity = "direct"; nat = "none" }
|
||||
)
|
||||
|
||||
$nodes = @{}
|
||||
foreach ($spec in $nodeSpecs) {
|
||||
$fingerprint = "c17z18-fp-$($spec.key)-$([guid]::NewGuid().ToString('N'))"
|
||||
$publicKey = "c17z18-pub-$($spec.key)-$([guid]::NewGuid().ToString('N'))"
|
||||
$joinRequest = Invoke-Api -Method Post -Path "/clusters/$clusterID/join-requests" -Body @{
|
||||
join_token = $joinToken.join_token.token
|
||||
node_name = $spec.name
|
||||
node_fingerprint = $fingerprint
|
||||
public_key = $publicKey
|
||||
reported_capabilities = @{
|
||||
can_accept_node_ingress = $true
|
||||
can_route_mesh = $true
|
||||
testing_node = $true
|
||||
mesh_rendezvous_relay_control_contract = $true
|
||||
mesh_rendezvous_lease_telemetry = $true
|
||||
mesh_rendezvous_lease_refresh_contract = $true
|
||||
mesh_rendezvous_relay_replacement_contract = $true
|
||||
mesh_route_path_decision_contract = $true
|
||||
mesh_route_generation_tracker = $true
|
||||
}
|
||||
reported_facts = @{
|
||||
os = "linux"
|
||||
runtime = "docker-test"
|
||||
stage = "c17z18"
|
||||
run_id = $runId
|
||||
connectivity_mode = $spec.connectivity
|
||||
nat_type = $spec.nat
|
||||
}
|
||||
requested_roles = $spec.roles
|
||||
}
|
||||
$approved = Invoke-Api -Method Post -Path "/clusters/$clusterID/join-requests/$($joinRequest.join_request.id)/approve" -Body @{
|
||||
actor_user_id = $actorUserID
|
||||
node_key = $fingerprint
|
||||
ownership_type = "platform_managed"
|
||||
owner_organization_id = $null
|
||||
}
|
||||
$nodeID = $approved.node_bootstrap.node_id
|
||||
foreach ($role in $spec.roles) {
|
||||
Invoke-Api -Method Post -Path "/clusters/$clusterID/nodes/$nodeID/roles" -Body @{
|
||||
actor_user_id = $actorUserID
|
||||
role = $role
|
||||
status = "active"
|
||||
policy = @{
|
||||
stage = "c17z18"
|
||||
run_id = $runId
|
||||
synthetic_only = $true
|
||||
production_payload_forwarding = $false
|
||||
}
|
||||
} | Out-Null
|
||||
}
|
||||
$nodes[$spec.key] = [pscustomobject]@{
|
||||
id = $nodeID
|
||||
name = $spec.name
|
||||
fingerprint = $fingerprint
|
||||
public_key = $publicKey
|
||||
port = $spec.port
|
||||
roles = $spec.roles
|
||||
transport = $spec.transport
|
||||
connectivity = $spec.connectivity
|
||||
nat = $spec.nat
|
||||
}
|
||||
}
|
||||
|
||||
$nodeAID = $nodes["a"].id
|
||||
$nodeRID = $nodes["r"].id
|
||||
$nodeBID = $nodes["b"].id
|
||||
$nodeCID = $nodes["c"].id
|
||||
$nodeSID = $nodes["idle"].id
|
||||
$routeExpiresAt = (Get-Date).ToUniversalTime().AddHours(2).ToString("o")
|
||||
$staleRelayEndpoint = "http://127.0.0.1:$($MeshBasePort + 90)"
|
||||
|
||||
$peerEndpointsDirect = @{}
|
||||
$peerEndpointsDirect[$nodeAID] = "http://127.0.0.1:$($nodes["a"].port)"
|
||||
$peerEndpointsDirect[$nodeBID] = "http://127.0.0.1:$($nodes["b"].port)"
|
||||
|
||||
$peerEndpointsRelayControl = @{}
|
||||
$peerEndpointsRelayControl[$nodeAID] = "http://127.0.0.1:$($nodes["a"].port)"
|
||||
$peerEndpointsRelayControl[$nodeRID] = $staleRelayEndpoint
|
||||
$peerEndpointsRelayControl[$nodeSID] = "http://127.0.0.1:$($nodes["idle"].port)"
|
||||
|
||||
$peerEndpointCandidatesRelay = @{}
|
||||
$peerEndpointCandidatesRelay[$nodeRID] = @(
|
||||
New-EndpointCandidate `
|
||||
-EndpointID "relay-r-public" `
|
||||
-NodeID $nodeRID `
|
||||
-Address $staleRelayEndpoint `
|
||||
-Transport "direct_tcp_tls" `
|
||||
-Reachability "public" `
|
||||
-ConnectivityMode "direct" `
|
||||
-NATType "none" `
|
||||
-Priority 10 `
|
||||
-PolicyTags @("relay-control", "same-site")
|
||||
)
|
||||
$peerEndpointCandidatesRelay[$nodeSID] = @(
|
||||
New-EndpointCandidate `
|
||||
-EndpointID "relay-s-alt-fast" `
|
||||
-NodeID $nodeSID `
|
||||
-Address "http://127.0.0.1:$($nodes["idle"].port)" `
|
||||
-Transport "direct_tcp_tls" `
|
||||
-Reachability "public" `
|
||||
-ConnectivityMode "direct" `
|
||||
-NATType "none" `
|
||||
-Priority 1 `
|
||||
-PolicyTags @("relay-control", "same-site", "fast-path")
|
||||
)
|
||||
$peerEndpointCandidatesRelay[$nodeCID] = @(
|
||||
New-EndpointCandidate `
|
||||
-EndpointID "node-c-outbound-only" `
|
||||
-NodeID $nodeCID `
|
||||
-Address "http://127.0.0.1:$($nodes["c"].port)" `
|
||||
-Transport "outbound_reverse" `
|
||||
-Reachability "outbound_only" `
|
||||
-ConnectivityMode "outbound_only" `
|
||||
-NATType "symmetric" `
|
||||
-Priority 5 `
|
||||
-PolicyTags @("nat", "outbound-only")
|
||||
)
|
||||
|
||||
Write-Host "Creating direct baseline and outbound-only relay-control route intents..."
|
||||
$directIntent = Invoke-Api -Method Post -Path "/clusters/$clusterID/mesh/route-intents" -Body @{
|
||||
actor_user_id = $actorUserID
|
||||
source_selector = @{ node_id = $nodeAID }
|
||||
destination_selector = @{ node_id = $nodeBID }
|
||||
service_class = "control"
|
||||
priority = 10
|
||||
policy = @{
|
||||
synthetic_enabled = $true
|
||||
peer_endpoints = $peerEndpointsDirect
|
||||
hops = @($nodeAID, $nodeBID)
|
||||
allowed_channels = @("fabric_control", "route_control")
|
||||
max_ttl = 4
|
||||
max_hops = 4
|
||||
expires_at = $routeExpiresAt
|
||||
route_version = "$runId-direct"
|
||||
policy_version = "$runId-policy"
|
||||
peer_directory_version = "$runId-peers"
|
||||
production_forwarding = $false
|
||||
}
|
||||
}
|
||||
|
||||
$rendezvousIntent = Invoke-Api -Method Post -Path "/clusters/$clusterID/mesh/route-intents" -Body @{
|
||||
actor_user_id = $actorUserID
|
||||
source_selector = @{ node_id = $nodeAID }
|
||||
destination_selector = @{ node_id = $nodeCID }
|
||||
service_class = "control"
|
||||
priority = 20
|
||||
policy = @{
|
||||
synthetic_enabled = $true
|
||||
peer_endpoints = $peerEndpointsRelayControl
|
||||
peer_endpoint_candidates = $peerEndpointCandidatesRelay
|
||||
hops = @($nodeAID, $nodeRID, $nodeSID, $nodeCID)
|
||||
allowed_channels = @("fabric_control", "route_control")
|
||||
max_ttl = 6
|
||||
max_hops = 6
|
||||
rendezvous_leases = @(
|
||||
@{
|
||||
lease_id = "$runId-explicit-stale-relay-lease"
|
||||
peer_node_id = $nodeCID
|
||||
relay_node_id = $nodeRID
|
||||
relay_endpoint = $staleRelayEndpoint
|
||||
transport = "relay_control"
|
||||
connectivity_mode = "relay_required"
|
||||
route_ids = @()
|
||||
allowed_channels = @("fabric_control", "route_control")
|
||||
priority = 4
|
||||
control_plane_only = $true
|
||||
issued_at = (Get-Date).ToUniversalTime().ToString("o")
|
||||
expires_at = $routeExpiresAt
|
||||
reason = "smoke_stale_relay_replacement"
|
||||
metadata = @{
|
||||
stage = "c17z18"
|
||||
run_id = $runId
|
||||
lease_refresh_contract = "node_scoped_synthetic_config_get"
|
||||
relay_replacement_contract = "stale_relay_feedback_policy"
|
||||
production_payload_forwarding = $false
|
||||
}
|
||||
}
|
||||
)
|
||||
expires_at = $routeExpiresAt
|
||||
route_version = "$runId-rendezvous"
|
||||
policy_version = "$runId-policy"
|
||||
peer_directory_version = "$runId-peers"
|
||||
production_forwarding = $false
|
||||
}
|
||||
}
|
||||
|
||||
$configs = @{}
|
||||
foreach ($key in @("a", "r", "b", "c", "idle")) {
|
||||
$configs[$key] = Invoke-Api -Method Get -Path "/clusters/$clusterID/nodes/$($nodes[$key].id)/mesh/synthetic-config"
|
||||
}
|
||||
|
||||
$nodeAPeerCandidates = Get-OptionalProperty -Object (Get-OptionalProperty -Object $configs["a"].synthetic_mesh_config -PropertyName "peer_endpoint_candidates") -PropertyName $nodeCID
|
||||
$nodeAInitialStaleLeases = @(Get-OptionalProperty -Object $configs["a"].synthetic_mesh_config -PropertyName "rendezvous_leases" | Where-Object {
|
||||
$_.peer_node_id -eq $nodeCID -and $_.relay_node_id -eq $nodeRID
|
||||
})
|
||||
$nodeAInitialAltLeases = @(Get-OptionalProperty -Object $configs["a"].synthetic_mesh_config -PropertyName "rendezvous_leases" | Where-Object {
|
||||
$_.peer_node_id -eq $nodeCID -and $_.relay_node_id -eq $nodeSID
|
||||
})
|
||||
$nodeCLeases = @(Get-OptionalProperty -Object $configs["c"].synthetic_mesh_config -PropertyName "rendezvous_leases" | Where-Object {
|
||||
$_.peer_node_id -eq $nodeCID -and ($_.relay_node_id -eq $nodeRID -or $_.relay_node_id -eq $nodeSID)
|
||||
})
|
||||
|
||||
Write-Host "Starting node-agent containers..."
|
||||
foreach ($key in @("r", "idle", "b", "c", "a")) {
|
||||
$node = $nodes[$key]
|
||||
$containerName = "$nodePrefix$key"
|
||||
$remoteStateDir = "/tmp/$runId-$key"
|
||||
Invoke-RemoteShell -Command "rm -rf '$remoteStateDir' && mkdir -p '$remoteStateDir'"
|
||||
$identity = @{
|
||||
node_id = $node.id
|
||||
cluster_id = $clusterID
|
||||
node_name = $node.name
|
||||
node_fingerprint = $node.fingerprint
|
||||
public_key = $node.public_key
|
||||
identity_status = "active"
|
||||
created_at = (Get-Date).ToUniversalTime().ToString("o")
|
||||
updated_at = (Get-Date).ToUniversalTime().ToString("o")
|
||||
} | ConvertTo-Json -Depth 10
|
||||
Send-RemoteFile -RemotePath "$remoteStateDir/identity.json" -Content $identity
|
||||
Invoke-RemoteDocker -Arguments @(
|
||||
"create",
|
||||
"--name", $containerName,
|
||||
"--network", "host",
|
||||
"-e", "RAP_BACKEND_URL=$backendContainerBaseUrl",
|
||||
"-e", "RAP_NODE_STATE_DIR=/tmp/state",
|
||||
"-e", "RAP_HEARTBEAT_INTERVAL_SECONDS=5",
|
||||
"-e", "RAP_MESH_SYNTHETIC_RUNTIME_ENABLED=true",
|
||||
"-e", "RAP_MESH_LISTEN_ADDR=0.0.0.0:$($node.port)",
|
||||
"-e", "RAP_MESH_ADVERTISE_ENDPOINT=http://127.0.0.1:$($node.port)",
|
||||
"-e", "RAP_MESH_ADVERTISE_TRANSPORT=$($node.transport)",
|
||||
"-e", "RAP_MESH_CONNECTIVITY_MODE=$($node.connectivity)",
|
||||
"-e", "RAP_MESH_NAT_TYPE=$($node.nat)",
|
||||
"-e", "RAP_MESH_REGION=docker-test",
|
||||
$NodeAgentImageTag,
|
||||
"-backend-url", $backendContainerBaseUrl,
|
||||
"-state-dir", "/tmp/state",
|
||||
"-heartbeat-interval", "5s",
|
||||
"-mesh-synthetic-runtime-enabled",
|
||||
"-mesh-listen-addr", "0.0.0.0:$($node.port)",
|
||||
"-mesh-advertise-endpoint", "http://127.0.0.1:$($node.port)",
|
||||
"-mesh-advertise-transport", $node.transport,
|
||||
"-mesh-connectivity-mode", $node.connectivity,
|
||||
"-mesh-nat-type", $node.nat,
|
||||
"-mesh-region", "docker-test"
|
||||
) | Out-Null
|
||||
Invoke-RemoteDocker -Arguments @("cp", "$remoteStateDir/.", "$containerName`:/tmp/state")
|
||||
Invoke-RemoteDocker -Arguments @("start", $containerName) | Out-Null
|
||||
}
|
||||
|
||||
Write-Host "Waiting for rendezvous relay-control observations..."
|
||||
Start-Sleep -Seconds 40
|
||||
|
||||
Write-Host "Waiting for replacement route-health effective path..."
|
||||
$meshSnapshot = Wait-C17Z18ReplacementRouteHealthSnapshot -TimeoutSeconds 40
|
||||
$links = $meshSnapshot.links
|
||||
$summary = Invoke-Api -Method Get -Path "/cluster-admin-summaries?actor_user_id=$actorUserID"
|
||||
$nodeALeaseReport = Get-LatestRendezvousLeaseReport -NodeID $nodeAID
|
||||
$nodeRLeaseReport = Get-LatestRendezvousLeaseReport -NodeID $nodeRID
|
||||
$nodeSLeaseReport = Get-LatestRendezvousLeaseReport -NodeID $nodeSID
|
||||
$nodeCLeaseReport = Get-LatestRendezvousLeaseReport -NodeID $nodeCID
|
||||
$nodeAPathDecisionReport = Get-LatestRoutePathDecisionReport -NodeID $nodeAID
|
||||
$nodeARouteGenerationReport = Get-LatestRouteGenerationReport -NodeID $nodeAID
|
||||
$nodeARouteHealthConfigReport = Get-LatestRouteHealthConfigReport -NodeID $nodeAID
|
||||
$nodeARouteHealthFeedbackRefreshReport = Get-LatestRouteHealthFeedbackRefreshReport -NodeID $nodeAID
|
||||
$refreshedNodeAConfig = Invoke-Api -Method Get -Path "/clusters/$clusterID/nodes/$nodeAID/mesh/synthetic-config"
|
||||
$nodeAReportedLeases = @(Get-OptionalProperty -Object $nodeALeaseReport -PropertyName "leases")
|
||||
$nodeAReportedReplacementLeases = @($nodeAReportedLeases | Where-Object {
|
||||
$_.peer_node_id -eq $nodeCID -and $_.relay_node_id -eq $nodeSID -and $_.reason -eq "stale_relay_replacement"
|
||||
})
|
||||
$nodeAReportedStaleRelayLeases = @($nodeAReportedLeases | Where-Object {
|
||||
$_.peer_node_id -eq $nodeCID -and $_.relay_node_id -eq $nodeRID
|
||||
})
|
||||
$nodeAReplacementLeases = @(Get-OptionalProperty -Object $refreshedNodeAConfig.synthetic_mesh_config -PropertyName "rendezvous_leases" | Where-Object {
|
||||
$_.peer_node_id -eq $nodeCID -and $_.relay_node_id -eq $nodeSID -and $_.reason -eq "stale_relay_replacement"
|
||||
})
|
||||
$nodeAWithdrawnStaleLeases = @(Get-OptionalProperty -Object $refreshedNodeAConfig.synthetic_mesh_config -PropertyName "rendezvous_leases" | Where-Object {
|
||||
$_.peer_node_id -eq $nodeCID -and $_.relay_node_id -eq $nodeRID
|
||||
})
|
||||
$nodeARelayPolicy = Get-OptionalProperty -Object $refreshedNodeAConfig.synthetic_mesh_config -PropertyName "rendezvous_relay_policy"
|
||||
$nodeAInitialPathDecisionReport = Get-OptionalProperty -Object $configs["a"].synthetic_mesh_config -PropertyName "route_path_decisions"
|
||||
$nodeAConfigPathDecisionReport = Get-OptionalProperty -Object $refreshedNodeAConfig.synthetic_mesh_config -PropertyName "route_path_decisions"
|
||||
$nodeAReportedPathDecisions = @(Get-OptionalProperty -Object $nodeAPathDecisionReport -PropertyName "decisions")
|
||||
$nodeAReportedReplacementPathDecisions = @($nodeAReportedPathDecisions | Where-Object {
|
||||
$_.route_id -eq $rendezvousIntent.route_intent.id -and
|
||||
$_.selected_relay_id -eq $nodeSID -and
|
||||
$_.decision_source -eq "stale_relay_replacement" -and
|
||||
(@($_.effective_hops) -contains $nodeSID) -and
|
||||
-not (@($_.effective_hops) -contains $nodeRID)
|
||||
})
|
||||
|
||||
$nodeLogs = @{}
|
||||
foreach ($key in @("a", "r", "b", "c", "idle")) {
|
||||
$nodeLogs[$key] = Invoke-RemoteDockerText -Arguments @("logs", "--tail", "120", "$nodePrefix$key")
|
||||
}
|
||||
$backendLogs = Invoke-RemoteDockerText -Arguments @("logs", "--tail", "80", $backendName)
|
||||
|
||||
$meshLinks = @($meshSnapshot.mesh_links)
|
||||
$routeHealthLinks = @($meshSnapshot.route_health_links)
|
||||
$directHealth = @($meshSnapshot.direct_health)
|
||||
$rendezvousHealth = @($meshSnapshot.rendezvous_health)
|
||||
$replacementRouteHealth = @($meshSnapshot.replacement_route_health)
|
||||
$managerLinks = @($meshLinks | Where-Object { $_.metadata.observation_type -eq "peer_connection_manager" })
|
||||
$relayControlLinks = @($managerLinks | Where-Object {
|
||||
$_.source_node_id -eq $nodeAID -and
|
||||
$_.target_node_id -eq $nodeCID -and
|
||||
$_.link_status -eq "reachable" -and
|
||||
$_.metadata.transport_mode -eq "relay_control" -and
|
||||
$_.metadata.rendezvous_resolved -eq $true -and
|
||||
$_.metadata.relay_candidate -eq $true -and
|
||||
$_.metadata.connection_state -eq "relay_ready"
|
||||
})
|
||||
$replacementRelayControlLinks = @($relayControlLinks | Where-Object {
|
||||
$_.metadata.relay_node_id -eq $nodeSID
|
||||
})
|
||||
$replacementRelayReadyFromLeaseReport = (
|
||||
$nodeAReportedReplacementLeases.Count -gt 0 -and
|
||||
(Get-OptionalProperty -Object $nodeAReportedReplacementLeases[0] -PropertyName "relay_ready") -eq $true -and
|
||||
(Get-OptionalProperty -Object $nodeAReportedReplacementLeases[0] -PropertyName "connection_state") -eq "relay_ready"
|
||||
)
|
||||
|
||||
$nodeALog = $nodeLogs["a"] -join "`n"
|
||||
$directRouteDelivered = $nodeALog -match ('"event":"fabric_route_delivery_succeeded","route_id":"' + [regex]::Escape($directIntent.route_intent.id) + '"')
|
||||
$leaseReportBoundaryFlagsDisabled = $true
|
||||
foreach ($report in @($nodeALeaseReport, $nodeRLeaseReport, $nodeSLeaseReport, $nodeCLeaseReport)) {
|
||||
if ($null -eq $report -or
|
||||
(Get-OptionalProperty -Object $report -PropertyName "control_plane_only") -ne $true -or
|
||||
(Get-OptionalProperty -Object $report -PropertyName "relay_payload_forwarding") -ne $false -or
|
||||
(Get-OptionalProperty -Object $report -PropertyName "production_payload_forwarding") -ne $false -or
|
||||
(Get-OptionalProperty -Object $report -PropertyName "service_workload_traffic") -ne $false) {
|
||||
$leaseReportBoundaryFlagsDisabled = $false
|
||||
}
|
||||
}
|
||||
$pathDecisionBoundaryFlagsDisabled = (
|
||||
$null -ne $nodeAPathDecisionReport -and
|
||||
(Get-OptionalProperty -Object $nodeAPathDecisionReport -PropertyName "control_plane_only") -eq $true -and
|
||||
(Get-OptionalProperty -Object $nodeAPathDecisionReport -PropertyName "production_payload_forwarding") -eq $false -and
|
||||
(Get-OptionalProperty -Object $nodeAPathDecisionReport -PropertyName "service_workload_traffic") -eq $false -and
|
||||
(Get-OptionalProperty -Object $nodeAPathDecisionReport -PropertyName "route_path_forwarding_runtime") -eq $false
|
||||
)
|
||||
$routeGenerationBoundaryFlagsDisabled = (
|
||||
$null -ne $nodeARouteGenerationReport -and
|
||||
(Get-OptionalProperty -Object $nodeARouteGenerationReport -PropertyName "control_plane_only") -eq $true -and
|
||||
(Get-OptionalProperty -Object $nodeARouteGenerationReport -PropertyName "production_payload_forwarding") -eq $false -and
|
||||
(Get-OptionalProperty -Object $nodeARouteGenerationReport -PropertyName "service_workload_traffic") -eq $false -and
|
||||
(Get-OptionalProperty -Object $nodeARouteGenerationReport -PropertyName "route_path_forwarding_runtime") -eq $false
|
||||
)
|
||||
$nodeAWithdrawnDecisionCount = Get-OptionalProperty -Object $nodeARouteGenerationReport -PropertyName "withdrawn_decision_count"
|
||||
$nodeATotalWithdrawnDecisionCount = Get-OptionalProperty -Object $nodeARouteGenerationReport -PropertyName "total_withdrawn_decision_count"
|
||||
$nodeAWithdrawnDecisions = @(Get-OptionalProperty -Object $nodeARouteGenerationReport -PropertyName "withdrawn_decisions")
|
||||
$routeHealthConfigBoundaryFlagsDisabled = (
|
||||
$null -ne $nodeARouteHealthConfigReport -and
|
||||
(Get-OptionalProperty -Object $nodeARouteHealthConfigReport -PropertyName "control_plane_only") -eq $true -and
|
||||
(Get-OptionalProperty -Object $nodeARouteHealthConfigReport -PropertyName "route_health_only") -eq $true -and
|
||||
(Get-OptionalProperty -Object $nodeARouteHealthConfigReport -PropertyName "synthetic_route_health_route_path_runtime") -eq $true -and
|
||||
(Get-OptionalProperty -Object $nodeARouteHealthConfigReport -PropertyName "production_route_path_forwarding_runtime") -eq $false -and
|
||||
(Get-OptionalProperty -Object $nodeARouteHealthConfigReport -PropertyName "production_payload_forwarding") -eq $false -and
|
||||
(Get-OptionalProperty -Object $nodeARouteHealthConfigReport -PropertyName "service_workload_traffic") -eq $false -and
|
||||
(Get-OptionalProperty -Object $nodeARouteHealthConfigReport -PropertyName "test_service_route_config_changed") -eq $false
|
||||
)
|
||||
$routeHealthFeedbackRefreshBoundaryFlagsDisabled = (
|
||||
$null -ne $nodeARouteHealthFeedbackRefreshReport -and
|
||||
(Get-OptionalProperty -Object $nodeARouteHealthFeedbackRefreshReport -PropertyName "control_plane_only") -eq $true -and
|
||||
(Get-OptionalProperty -Object $nodeARouteHealthFeedbackRefreshReport -PropertyName "route_health_only") -eq $true -and
|
||||
(Get-OptionalProperty -Object $nodeARouteHealthFeedbackRefreshReport -PropertyName "production_payload_forwarding") -eq $false -and
|
||||
(Get-OptionalProperty -Object $nodeARouteHealthFeedbackRefreshReport -PropertyName "service_workload_traffic") -eq $false
|
||||
)
|
||||
$passMatrix = [ordered]@{
|
||||
backend_ready = $true
|
||||
platform_owner_login = [bool]$actorUserID
|
||||
cluster_created = [bool]$clusterID
|
||||
fabric_testing_flags_enabled = $true
|
||||
node_a_scoped_config_enabled = $configs["a"].synthetic_mesh_config.enabled -eq $true
|
||||
node_a_has_direct_and_rendezvous_routes = @($configs["a"].synthetic_mesh_config.routes).Count -eq 2
|
||||
node_a_has_outbound_peer_candidate = @($nodeAPeerCandidates).Count -gt 0
|
||||
node_a_has_initial_stale_rendezvous_lease = $nodeAInitialStaleLeases.Count -gt 0
|
||||
node_a_initial_lease_is_control_plane_only = ($nodeAInitialStaleLeases.Count -gt 0 -and $nodeAInitialStaleLeases[0].control_plane_only -eq $true)
|
||||
node_a_initial_lease_uses_relay_control = ($nodeAInitialStaleLeases.Count -gt 0 -and $nodeAInitialStaleLeases[0].transport -eq "relay_control")
|
||||
node_a_initial_auto_alt_relay_candidate = $nodeAInitialAltLeases.Count -gt 0
|
||||
node_a_initial_path_decision_report = (Get-OptionalProperty -Object $nodeAInitialPathDecisionReport -PropertyName "schema_version") -eq "c17z18.route_path_decisions.v1"
|
||||
node_a_report_replacement_lease_uses_alt_relay = ($nodeAReportedReplacementLeases.Count -gt 0 -and $nodeAReportedReplacementLeases[0].relay_node_id -eq $nodeSID)
|
||||
node_a_report_stale_relay_lease_withdrawn = $nodeAReportedStaleRelayLeases.Count -eq 0
|
||||
node_a_report_replacement_reason = ($nodeAReportedReplacementLeases.Count -gt 0 -and $nodeAReportedReplacementLeases[0].reason -eq "stale_relay_replacement")
|
||||
node_a_reports_c17z18_path_decisions = (Get-OptionalProperty -Object $nodeAPathDecisionReport -PropertyName "schema_version") -eq "c17z18.mesh_route_path_decision_report.v1"
|
||||
node_a_path_decision_replacement_count = (Get-OptionalProperty -Object $nodeAPathDecisionReport -PropertyName "replacement_decision_count") -gt 0
|
||||
node_a_path_decision_uses_alt_relay = $nodeAReportedReplacementPathDecisions.Count -gt 0
|
||||
node_a_path_decision_boundary_flags_disabled = $pathDecisionBoundaryFlagsDisabled
|
||||
node_a_reports_c17z18_route_generation = (Get-OptionalProperty -Object $nodeARouteGenerationReport -PropertyName "schema_version") -eq "c17z18.mesh_route_generation_report.v1"
|
||||
node_a_route_generation_active = (Get-OptionalProperty -Object $nodeARouteGenerationReport -PropertyName "active_decision_count") -gt 0
|
||||
node_a_route_generation_applied = (Get-OptionalProperty -Object $nodeARouteGenerationReport -PropertyName "applied_decision_count") -gt 0
|
||||
node_a_route_generation_withdrawn = ($nodeAWithdrawnDecisionCount -gt 0 -or $nodeATotalWithdrawnDecisionCount -gt 0 -or $nodeAWithdrawnDecisions.Count -gt 0)
|
||||
node_a_route_generation_changed = (Get-OptionalProperty -Object $nodeARouteGenerationReport -PropertyName "generation_changed") -eq $true
|
||||
node_a_route_generation_boundary_flags_disabled = $routeGenerationBoundaryFlagsDisabled
|
||||
node_a_reports_c17z20_route_health_config = (Get-OptionalProperty -Object $nodeARouteHealthConfigReport -PropertyName "schema_version") -eq "c17z20.mesh_route_health_config_report.v1"
|
||||
node_a_reports_c17z20_route_health_feedback_refresh = (Get-OptionalProperty -Object $nodeARouteHealthFeedbackRefreshReport -PropertyName "schema_version") -eq "c17z20.mesh_route_health_feedback_refresh_report.v1"
|
||||
node_a_route_health_feedback_refresh_supported = (Get-OptionalProperty -Object $nodeARouteHealthFeedbackRefreshReport -PropertyName "feedback_refresh_supported") -eq $true
|
||||
node_a_route_health_feedback_refresh_boundary_flags_disabled = $routeHealthFeedbackRefreshBoundaryFlagsDisabled
|
||||
node_a_route_health_config_applied = (Get-OptionalProperty -Object $nodeARouteHealthConfigReport -PropertyName "route_path_decision_applied_count") -gt 0
|
||||
node_a_route_health_config_replacement = (Get-OptionalProperty -Object $nodeARouteHealthConfigReport -PropertyName "replacement_route_health_route_count") -gt 0
|
||||
node_a_route_health_config_boundary_flags_disabled = $routeHealthConfigBoundaryFlagsDisabled
|
||||
node_a_route_health_uses_effective_alt_relay = $replacementRouteHealth.Count -gt 0
|
||||
node_a_route_health_has_no_effective_path_drift = ($replacementRouteHealth.Count -gt 0 -and (Get-OptionalProperty -Object $replacementRouteHealth[0].metadata -PropertyName "route_path_drift_detected") -eq $false)
|
||||
node_a_route_health_selected_alt_next_hop = $nodeALog -match ('"event":"fabric_route_selected","route_id":"' + [regex]::Escape($rendezvousIntent.route_intent.id) + '".*"next_node_id":"' + [regex]::Escape($nodeSID) + '"')
|
||||
outbound_node_has_relay_lease = $nodeCLeases.Count -gt 0
|
||||
direct_baseline_health_reported = $directRouteDelivered
|
||||
node_a_loaded_c17z20_control_plane_config = ($nodeAReplacementLeases.Count -gt 0 -and (Get-OptionalProperty -Object $nodeARouteHealthConfigReport -PropertyName "schema_version") -eq "c17z20.mesh_route_health_config_report.v1")
|
||||
node_a_resolved_waiting_rendezvous = ($replacementRelayControlLinks.Count -gt 0 -or $replacementRelayReadyFromLeaseReport)
|
||||
relay_control_manager_link_reachable = ($replacementRelayControlLinks.Count -gt 0 -or $replacementRelayReadyFromLeaseReport)
|
||||
relay_ready_recorded = ($replacementRelayControlLinks.Count -gt 0 -or $replacementRelayReadyFromLeaseReport)
|
||||
node_a_reports_c17z18_lease_telemetry = (Get-OptionalProperty -Object $nodeALeaseReport -PropertyName "schema_version") -eq "c17z18.mesh_rendezvous_lease_report.v1"
|
||||
node_a_lease_report_entry_observer = (Get-OptionalProperty -Object $nodeALeaseReport -PropertyName "entry_observer_count") -gt 0
|
||||
node_a_lease_report_relay_ready = (Get-OptionalProperty -Object $nodeALeaseReport -PropertyName "relay_control_ready_count") -gt 0
|
||||
node_a_lease_report_refresh_contract = (Get-OptionalProperty -Object $nodeALeaseReport -PropertyName "refresh_contract") -eq "node_scoped_synthetic_config_get"
|
||||
stale_relay_refresh_succeeded_on_cluster = (
|
||||
((Get-OptionalProperty -Object $nodeRLeaseReport -PropertyName "last_refresh_reason") -eq "stale_relay" -and (Get-OptionalProperty -Object $nodeRLeaseReport -PropertyName "refresh_success_count") -gt 0) -or
|
||||
((Get-OptionalProperty -Object $nodeSLeaseReport -PropertyName "last_refresh_reason") -eq "stale_relay" -and (Get-OptionalProperty -Object $nodeSLeaseReport -PropertyName "refresh_success_count") -gt 0)
|
||||
)
|
||||
alt_relay_node_reports_admitted_relay_lease = (Get-OptionalProperty -Object $nodeSLeaseReport -PropertyName "admitted_as_relay_count") -gt 0
|
||||
outbound_node_reports_admitted_peer_lease = (Get-OptionalProperty -Object $nodeCLeaseReport -PropertyName "admitted_as_peer_count") -gt 0
|
||||
lease_telemetry_boundary_flags_disabled = $leaseReportBoundaryFlagsDisabled
|
||||
production_forwarding_disabled = (
|
||||
$configs["a"].synthetic_mesh_config.production_forwarding -eq $false -and
|
||||
$configs["r"].synthetic_mesh_config.production_forwarding -eq $false -and
|
||||
$configs["b"].synthetic_mesh_config.production_forwarding -eq $false -and
|
||||
$configs["c"].synthetic_mesh_config.production_forwarding -eq $false -and
|
||||
$configs["idle"].synthetic_mesh_config.production_forwarding -eq $false
|
||||
)
|
||||
}
|
||||
|
||||
$result = [pscustomobject]@{
|
||||
stage = "C17Z18 rendezvous relay replacement docker-test smoke"
|
||||
run_id = $runId
|
||||
backend_base_url = $backendPublicBaseUrl
|
||||
cluster_id = $clusterID
|
||||
node_ids = @{
|
||||
a = $nodes["a"].id
|
||||
r = $nodes["r"].id
|
||||
s = $nodes["idle"].id
|
||||
b = $nodes["b"].id
|
||||
c = $nodes["c"].id
|
||||
idle = $nodes["idle"].id
|
||||
}
|
||||
route_intents = @{
|
||||
direct = $directIntent.route_intent.id
|
||||
rendezvous = $rendezvousIntent.route_intent.id
|
||||
}
|
||||
scoped_config_route_counts = @{
|
||||
a = @($configs["a"].synthetic_mesh_config.routes).Count
|
||||
r = @($configs["r"].synthetic_mesh_config.routes).Count
|
||||
b = @($configs["b"].synthetic_mesh_config.routes).Count
|
||||
c = @($configs["c"].synthetic_mesh_config.routes).Count
|
||||
idle = @($configs["idle"].synthetic_mesh_config.routes).Count
|
||||
}
|
||||
rendezvous_lease_counts = @{
|
||||
a = Get-OptionalArrayCount -Object $configs["a"].synthetic_mesh_config -PropertyName "rendezvous_leases"
|
||||
r = Get-OptionalArrayCount -Object $configs["r"].synthetic_mesh_config -PropertyName "rendezvous_leases"
|
||||
b = Get-OptionalArrayCount -Object $configs["b"].synthetic_mesh_config -PropertyName "rendezvous_leases"
|
||||
c = Get-OptionalArrayCount -Object $configs["c"].synthetic_mesh_config -PropertyName "rendezvous_leases"
|
||||
idle = Get-OptionalArrayCount -Object $configs["idle"].synthetic_mesh_config -PropertyName "rendezvous_leases"
|
||||
}
|
||||
node_a_initial_stale_rendezvous_lease = $nodeAInitialStaleLeases | Select-Object -First 1
|
||||
node_a_reported_replacement_rendezvous_lease = $nodeAReportedReplacementLeases | Select-Object -First 1
|
||||
node_a_current_replacement_rendezvous_lease = $nodeAReplacementLeases | Select-Object -First 1
|
||||
node_a_current_rendezvous_relay_policy = $nodeARelayPolicy
|
||||
node_a_initial_route_path_decisions = $nodeAInitialPathDecisionReport
|
||||
node_a_current_route_path_decisions = $nodeAConfigPathDecisionReport
|
||||
node_a_reported_route_path_decision = $nodeAReportedReplacementPathDecisions | Select-Object -First 1
|
||||
route_path_decision_reports = @{
|
||||
a = $nodeAPathDecisionReport
|
||||
}
|
||||
route_generation_reports = @{
|
||||
a = $nodeARouteGenerationReport
|
||||
}
|
||||
route_health_config_reports = @{
|
||||
a = $nodeARouteHealthConfigReport
|
||||
}
|
||||
route_health_feedback_refresh_reports = @{
|
||||
a = $nodeARouteHealthFeedbackRefreshReport
|
||||
}
|
||||
rendezvous_lease_reports = @{
|
||||
a = $nodeALeaseReport
|
||||
r = $nodeRLeaseReport
|
||||
s = $nodeSLeaseReport
|
||||
c = $nodeCLeaseReport
|
||||
}
|
||||
mesh_link_count = $meshLinks.Count
|
||||
route_health_count = $routeHealthLinks.Count
|
||||
peer_connection_manager_link_count = $managerLinks.Count
|
||||
relay_control_link_count = $relayControlLinks.Count
|
||||
direct_route_delivery_succeeded = $directRouteDelivered
|
||||
pass_matrix = $passMatrix
|
||||
direct_route_health = $directHealth | Select-Object -First 3
|
||||
replacement_route_health = $replacementRouteHealth | Select-Object -First 3
|
||||
relay_control_links = $relayControlLinks | Select-Object -First 5
|
||||
replacement_relay_control_links = $replacementRelayControlLinks | Select-Object -First 5
|
||||
cluster_summaries = $summary.cluster_summaries
|
||||
backend_log_tail = $backendLogs
|
||||
node_log_tail = $nodeLogs
|
||||
containers_left_running = [bool]$KeepRunning
|
||||
}
|
||||
|
||||
$failed = @($passMatrix.GetEnumerator() | Where-Object { -not $_.Value })
|
||||
$result | ConvertTo-Json -Depth 60
|
||||
|
||||
if ($failed.Count -gt 0) {
|
||||
throw "C17Z18 rendezvous relay replacement smoke failed: $($failed.Name -join ', ')"
|
||||
}
|
||||
|
||||
if (-not $KeepRunning) {
|
||||
Write-Host "Cleaning up C17Z18 containers..."
|
||||
Remove-C17Z12Containers
|
||||
Invoke-RemoteShell -Command "rm -rf '$remoteBuildDir'"
|
||||
}
|
||||
@@ -0,0 +1,478 @@
|
||||
param(
|
||||
[string]$DockerSshAlias = "test-docker",
|
||||
[string]$BackendImageTag = "rap-backend:c17z19-route-health-feedback-smoke",
|
||||
[string]$AdminEmail = "fabric-owner-c17z19@example.local",
|
||||
[string]$AdminPassword = "SmokePass!123",
|
||||
[int]$ApiPort = 18122,
|
||||
[int]$PostgresPort = 15444,
|
||||
[int]$RedisPort = 16444,
|
||||
[string]$ResultPath = "artifacts\c17z19-route-health-feedback-smoke-result.json",
|
||||
[switch]$KeepRunning
|
||||
)
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
$repoRoot = (Resolve-Path (Join-Path $PSScriptRoot "..\..")).ProviderPath
|
||||
$backendPublicBaseUrl = "http://192.168.200.61:$ApiPort/api/v1"
|
||||
$runId = "c17z19-" + (Get-Date -Format "yyyyMMdd-HHmmss")
|
||||
$remoteBuildDir = "/tmp/rap-c17z19-build-$runId"
|
||||
|
||||
$postgresName = "rap_c17z19_postgres"
|
||||
$redisName = "rap_c17z19_redis"
|
||||
$backendName = "rap_c17z19_backend"
|
||||
|
||||
function Invoke-RemoteDocker {
|
||||
param([string[]]$Arguments)
|
||||
& ssh $DockerSshAlias docker @Arguments
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "ssh $DockerSshAlias docker $($Arguments -join ' ') failed with exit code $LASTEXITCODE"
|
||||
}
|
||||
}
|
||||
|
||||
function Invoke-RemoteDockerText {
|
||||
param([string[]]$Arguments)
|
||||
$previousErrorActionPreference = $ErrorActionPreference
|
||||
$ErrorActionPreference = "Continue"
|
||||
try {
|
||||
$output = & ssh $DockerSshAlias docker @Arguments 2>&1 | ForEach-Object { $_.ToString() }
|
||||
}
|
||||
finally {
|
||||
$ErrorActionPreference = $previousErrorActionPreference
|
||||
}
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "ssh $DockerSshAlias docker $($Arguments -join ' ') failed with exit code $LASTEXITCODE"
|
||||
}
|
||||
return $output
|
||||
}
|
||||
|
||||
function Invoke-RemoteShell {
|
||||
param([string]$Command)
|
||||
& ssh $DockerSshAlias $Command
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "ssh $DockerSshAlias $Command failed with exit code $LASTEXITCODE"
|
||||
}
|
||||
}
|
||||
|
||||
function Send-RemoteBuildContext {
|
||||
Write-Host "Uploading backend build context to $DockerSshAlias..."
|
||||
Invoke-RemoteShell -Command "rm -rf '$remoteBuildDir' && mkdir -p '$remoteBuildDir'"
|
||||
& tar -czf - -C $repoRoot "backend" | & ssh $DockerSshAlias "tar -xzf - -C '$remoteBuildDir'"
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "upload build context failed"
|
||||
}
|
||||
}
|
||||
|
||||
function Invoke-Api {
|
||||
param(
|
||||
[string]$Method,
|
||||
[string]$Path,
|
||||
[object]$Body = $null
|
||||
)
|
||||
$uri = "$backendPublicBaseUrl$Path"
|
||||
try {
|
||||
if ($null -eq $Body) {
|
||||
return Invoke-RestMethod -Method $Method -Uri $uri -TimeoutSec 30
|
||||
}
|
||||
return Invoke-RestMethod -Method $Method -Uri $uri -ContentType "application/json" -Body ($Body | ConvertTo-Json -Depth 80) -TimeoutSec 30
|
||||
}
|
||||
catch {
|
||||
$statusCode = $null
|
||||
if ($_.Exception.Response) {
|
||||
$statusCode = [int]$_.Exception.Response.StatusCode
|
||||
}
|
||||
$details = $_.ErrorDetails.Message
|
||||
if (-not $details) {
|
||||
$details = $_.Exception.Message
|
||||
}
|
||||
throw "$Method $Path failed with HTTP $statusCode`: $details"
|
||||
}
|
||||
}
|
||||
|
||||
function Wait-HttpReady {
|
||||
param([string]$Url)
|
||||
for ($i = 0; $i -lt 90; $i++) {
|
||||
try {
|
||||
$response = Invoke-WebRequest -UseBasicParsing -Uri $Url -TimeoutSec 2
|
||||
if ($response.StatusCode -ge 200 -and $response.StatusCode -lt 300) {
|
||||
return
|
||||
}
|
||||
}
|
||||
catch {
|
||||
Start-Sleep -Seconds 1
|
||||
}
|
||||
}
|
||||
throw "Timed out waiting for $Url"
|
||||
}
|
||||
|
||||
function Remove-SmokeContainers {
|
||||
foreach ($name in @($backendName, $postgresName, $redisName)) {
|
||||
& ssh $DockerSshAlias docker rm -f $name 2>$null | Out-Null
|
||||
}
|
||||
}
|
||||
|
||||
function New-EndpointCandidate {
|
||||
param(
|
||||
[string]$EndpointID,
|
||||
[string]$NodeID,
|
||||
[string]$Address,
|
||||
[string]$Transport,
|
||||
[string]$Reachability,
|
||||
[string]$ConnectivityMode,
|
||||
[int]$Priority,
|
||||
[string[]]$PolicyTags = @()
|
||||
)
|
||||
return @{
|
||||
endpoint_id = $EndpointID
|
||||
node_id = $NodeID
|
||||
transport = $Transport
|
||||
address = $Address
|
||||
address_family = "ipv4"
|
||||
reachability = $Reachability
|
||||
nat_type = "none"
|
||||
connectivity_mode = $ConnectivityMode
|
||||
region = "docker-test"
|
||||
priority = $Priority
|
||||
policy_tags = $PolicyTags
|
||||
last_verified_at = (Get-Date).ToUniversalTime().ToString("o")
|
||||
metadata = @{
|
||||
stage = "c17z19"
|
||||
run_id = $runId
|
||||
production_payload_forwarding = $false
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function New-Node {
|
||||
param(
|
||||
[string]$Key,
|
||||
[string[]]$Roles
|
||||
)
|
||||
$fingerprint = "c17z19-fp-$Key-$([guid]::NewGuid().ToString('N'))"
|
||||
$publicKey = "c17z19-pub-$Key-$([guid]::NewGuid().ToString('N'))"
|
||||
$joinRequest = Invoke-Api -Method Post -Path "/clusters/$clusterID/join-requests" -Body @{
|
||||
join_token = $joinToken.join_token.token
|
||||
node_name = "c17z19-node-$Key"
|
||||
node_fingerprint = $fingerprint
|
||||
public_key = $publicKey
|
||||
reported_capabilities = @{
|
||||
can_route_mesh = $true
|
||||
testing_node = $true
|
||||
mesh_route_health_feedback = $true
|
||||
}
|
||||
reported_facts = @{
|
||||
stage = "c17z19"
|
||||
run_id = $runId
|
||||
}
|
||||
requested_roles = @()
|
||||
}
|
||||
$approved = Invoke-Api -Method Post -Path "/clusters/$clusterID/join-requests/$($joinRequest.join_request.id)/approve" -Body @{
|
||||
actor_user_id = $actorUserID
|
||||
node_key = $fingerprint
|
||||
ownership_type = "platform_managed"
|
||||
owner_organization_id = $null
|
||||
}
|
||||
$nodeID = $approved.node_bootstrap.node_id
|
||||
foreach ($role in $Roles) {
|
||||
Invoke-Api -Method Post -Path "/clusters/$clusterID/nodes/$nodeID/roles" -Body @{
|
||||
actor_user_id = $actorUserID
|
||||
role = $role
|
||||
status = "active"
|
||||
policy = @{
|
||||
stage = "c17z19"
|
||||
run_id = $runId
|
||||
synthetic_only = $true
|
||||
production_payload_forwarding = $false
|
||||
}
|
||||
} | Out-Null
|
||||
}
|
||||
return [pscustomobject]@{
|
||||
id = $nodeID
|
||||
fingerprint = $fingerprint
|
||||
public_key = $publicKey
|
||||
}
|
||||
}
|
||||
|
||||
function Get-OptionalProperty {
|
||||
param(
|
||||
[object]$Object,
|
||||
[string]$PropertyName
|
||||
)
|
||||
if ($null -eq $Object) {
|
||||
return $null
|
||||
}
|
||||
$property = $Object.PSObject.Properties[$PropertyName]
|
||||
if ($null -eq $property) {
|
||||
return $null
|
||||
}
|
||||
return $property.Value
|
||||
}
|
||||
|
||||
Write-Host "C17Z19 route-health feedback smoke run: $runId"
|
||||
Write-Host "Using SSH Docker host: $DockerSshAlias"
|
||||
|
||||
Remove-SmokeContainers
|
||||
Send-RemoteBuildContext
|
||||
|
||||
Write-Host "Building backend image on docker-test..."
|
||||
Invoke-RemoteDocker -Arguments @("build", "-f", "$remoteBuildDir/backend/Dockerfile", "-t", $BackendImageTag, "$remoteBuildDir/backend")
|
||||
|
||||
Write-Host "Starting isolated PostgreSQL and Redis..."
|
||||
Invoke-RemoteDocker -Arguments @(
|
||||
"run", "-d",
|
||||
"--name", $postgresName,
|
||||
"-e", "POSTGRES_DB=remote_access_platform",
|
||||
"-e", "POSTGRES_USER=rap_user",
|
||||
"-e", "POSTGRES_PASSWORD=rap_password",
|
||||
"-p", "$PostgresPort`:5432",
|
||||
"postgres:16"
|
||||
)
|
||||
Invoke-RemoteDocker -Arguments @(
|
||||
"run", "-d",
|
||||
"--name", $redisName,
|
||||
"-p", "$RedisPort`:6379",
|
||||
"redis:7"
|
||||
)
|
||||
|
||||
Invoke-RemoteShell -Command "for i in `$(seq 1 60); do docker exec $postgresName pg_isready -U rap_user -d remote_access_platform >/dev/null 2>&1 && exit 0; sleep 1; done; exit 1"
|
||||
|
||||
Write-Host "Applying migrations..."
|
||||
Invoke-RemoteShell -Command "for f in `$(find '$remoteBuildDir/backend/migrations' -name '*.up.sql' | sort); do docker exec -i $postgresName psql -U rap_user -d remote_access_platform -v ON_ERROR_STOP=1 -f - < `$f; done"
|
||||
|
||||
$secretBytes = New-Object byte[] 32
|
||||
[Security.Cryptography.RandomNumberGenerator]::Fill($secretBytes)
|
||||
$secretKeyB64 = [Convert]::ToBase64String($secretBytes)
|
||||
|
||||
Write-Host "Starting backend..."
|
||||
Invoke-RemoteDocker -Arguments @(
|
||||
"run", "-d",
|
||||
"--name", $backendName,
|
||||
"--network", "host",
|
||||
"-e", "APP_NAME=rap-api",
|
||||
"-e", "APP_ENV=c17z19-smoke",
|
||||
"-e", "HTTP_HOST=0.0.0.0",
|
||||
"-e", "HTTP_PORT=$ApiPort",
|
||||
"-e", "POSTGRES_DSN=postgres://rap_user:rap_password@127.0.0.1:$PostgresPort/remote_access_platform?sslmode=disable",
|
||||
"-e", "REDIS_ADDR=127.0.0.1:$RedisPort",
|
||||
"-e", "AUTH_ACCESS_TOKEN_SECRET=c17z19-access-secret",
|
||||
"-e", "AUTH_REFRESH_HASH_SECRET=c17z19-refresh-secret",
|
||||
"-e", "INSTALLATION_AUTHORITY_MODE=legacy",
|
||||
"-e", "INSTALLATION_INSECURE_BOOTSTRAP_ENABLED=true",
|
||||
"-e", "SECRET_ENCRYPTION_KEY_B64=$secretKeyB64",
|
||||
"-e", "SECRET_ENCRYPTION_KEY_ID=$runId",
|
||||
$BackendImageTag
|
||||
)
|
||||
Wait-HttpReady -Url "http://192.168.200.61:$ApiPort/readyz"
|
||||
|
||||
Write-Host "Bootstrapping owner, cluster, and synthetic route..."
|
||||
Invoke-Api -Method Post -Path "/installation/bootstrap-owner" -Body @{
|
||||
email = $AdminEmail
|
||||
password = $AdminPassword
|
||||
} | Out-Null
|
||||
$login = Invoke-Api -Method Post -Path "/auth/login" -Body @{
|
||||
email = $AdminEmail
|
||||
password = $AdminPassword
|
||||
device_fingerprint = "c17z19-smoke-device"
|
||||
device_label = "C17Z19 route health feedback smoke"
|
||||
trust_device = $true
|
||||
}
|
||||
$actorUserID = $login.user.id
|
||||
$cluster = Invoke-Api -Method Post -Path "/clusters/" -Body @{
|
||||
actor_user_id = $actorUserID
|
||||
slug = "c17z19-$((New-Guid).Guid.Substring(0, 8))"
|
||||
name = "C17Z19 Route Health Feedback Smoke"
|
||||
region = "docker-test"
|
||||
metadata = @{
|
||||
stage = "c17z19"
|
||||
run_id = $runId
|
||||
production_forwarding = $false
|
||||
service_payload_forwarding = $false
|
||||
}
|
||||
}
|
||||
$clusterID = $cluster.cluster.id
|
||||
Invoke-Api -Method Put -Path "/fabric/testing-flags" -Body @{
|
||||
actor_user_id = $actorUserID
|
||||
scope_type = "platform"
|
||||
scope_id = $null
|
||||
cluster_id = $null
|
||||
enabled = $true
|
||||
telemetry_enabled = $true
|
||||
synthetic_links_enabled = $true
|
||||
history_retention_hours = 24
|
||||
metadata = @{
|
||||
stage = "c17z19"
|
||||
run_id = $runId
|
||||
production_forwarding = $false
|
||||
service_payload_forwarding = $false
|
||||
}
|
||||
} | Out-Null
|
||||
|
||||
$joinToken = Invoke-Api -Method Post -Path "/clusters/$clusterID/join-tokens" -Body @{
|
||||
actor_user_id = $actorUserID
|
||||
scope = @{ purpose = "c17z19-route-health-feedback-smoke"; roles = @("core-mesh", "relay-node") }
|
||||
expires_at = (Get-Date).ToUniversalTime().AddHours(2).ToString("o")
|
||||
max_uses = 4
|
||||
}
|
||||
|
||||
$nodeA = New-Node -Key "a" -Roles @("core-mesh")
|
||||
$nodeB = New-Node -Key "b" -Roles @("core-mesh")
|
||||
$nodeS = New-Node -Key "s" -Roles @("core-mesh", "relay-node")
|
||||
$nodeT = New-Node -Key "t" -Roles @("core-mesh", "relay-node")
|
||||
|
||||
$nodeBCandidate = New-EndpointCandidate -EndpointID "node-b-outbound" -NodeID $nodeB.id -Address "node-b.reverse.local" -Transport "outbound_reverse" -Reachability "outbound_only" -ConnectivityMode "outbound_only" -Priority 5
|
||||
$nodeSCandidate = New-EndpointCandidate -EndpointID "node-s-public" -NodeID $nodeS.id -Address "http://node-s:19000" -Transport "direct_tcp_tls" -Reachability "public" -ConnectivityMode "direct" -Priority 1 -PolicyTags @("fast-path")
|
||||
$nodeTCandidate = New-EndpointCandidate -EndpointID "node-t-public" -NodeID $nodeT.id -Address "http://node-t:19000" -Transport "direct_tcp_tls" -Reachability "public" -ConnectivityMode "direct" -Priority 50
|
||||
|
||||
$routeIntent = Invoke-Api -Method Post -Path "/clusters/$clusterID/mesh/route-intents" -Body @{
|
||||
actor_user_id = $actorUserID
|
||||
source_selector = @{ node_id = $nodeA.id }
|
||||
destination_selector = @{ node_id = $nodeB.id }
|
||||
service_class = "synthetic"
|
||||
priority = 100
|
||||
policy = @{
|
||||
synthetic_enabled = $true
|
||||
hops = @($nodeA.id, $nodeS.id, $nodeT.id, $nodeB.id)
|
||||
allowed_channels = @("fabric_control", "route_control")
|
||||
peer_endpoint_candidates = @{
|
||||
"$($nodeB.id)" = @($nodeBCandidate)
|
||||
"$($nodeS.id)" = @($nodeSCandidate)
|
||||
"$($nodeT.id)" = @($nodeTCandidate)
|
||||
}
|
||||
}
|
||||
}
|
||||
$routeID = $routeIntent.route_intent.id
|
||||
|
||||
$initialConfig = Invoke-Api -Method Get -Path "/clusters/$clusterID/nodes/$($nodeA.id)/mesh/synthetic-config"
|
||||
$initialLease = @($initialConfig.synthetic_mesh_config.rendezvous_leases | Where-Object { $_.peer_node_id -eq $nodeB.id }) | Select-Object -First 1
|
||||
|
||||
Write-Host "Injecting drift route-health for initially selected relay..."
|
||||
Invoke-Api -Method Post -Path "/clusters/$clusterID/mesh/links" -Body @{
|
||||
source_node_id = $nodeA.id
|
||||
target_node_id = $nodeB.id
|
||||
link_status = "reachable"
|
||||
metadata = @{
|
||||
stage = "c17z19"
|
||||
observation_type = "synthetic_route_health"
|
||||
route_id = $routeID
|
||||
route_path_decision_applied = $true
|
||||
route_path_decision_selected_relay_id = $nodeS.id
|
||||
route_path_decision_rendezvous_peer_node_id = $nodeB.id
|
||||
route_path_decision_rendezvous_lease_id = "$routeID-rv-$($nodeB.id)-via-$($nodeS.id)"
|
||||
route_path_decision_rendezvous_lease_reason = "auto_rendezvous_required"
|
||||
expected_effective_hops = @($nodeA.id, $nodeS.id, $nodeB.id)
|
||||
observed_ack_path = @($nodeA.id, $nodeT.id, $nodeB.id)
|
||||
route_path_drift_detected = $true
|
||||
control_plane_only = $true
|
||||
production_forwarding = $false
|
||||
production_payload_forwarding = $false
|
||||
route_health_production_payload_forwarding = $false
|
||||
route_health_service_payload_forwarding = $false
|
||||
}
|
||||
} | Out-Null
|
||||
|
||||
$driftConfig = Invoke-Api -Method Get -Path "/clusters/$clusterID/nodes/$($nodeA.id)/mesh/synthetic-config"
|
||||
$driftLease = @($driftConfig.synthetic_mesh_config.rendezvous_leases | Where-Object { $_.peer_node_id -eq $nodeB.id }) | Select-Object -First 1
|
||||
$driftDecision = @($driftConfig.synthetic_mesh_config.route_path_decisions.decisions | Where-Object { $_.route_id -eq $routeID }) | Select-Object -First 1
|
||||
|
||||
Write-Host "Injecting healthy low-latency route-health for alternate relay..."
|
||||
$latency = 5
|
||||
$quality = 99
|
||||
Invoke-Api -Method Post -Path "/clusters/$clusterID/mesh/links" -Body @{
|
||||
source_node_id = $nodeA.id
|
||||
target_node_id = $nodeB.id
|
||||
link_status = "reachable"
|
||||
latency_ms = $latency
|
||||
quality_score = $quality
|
||||
metadata = @{
|
||||
stage = "c17z19"
|
||||
observation_type = "synthetic_route_health"
|
||||
route_id = $routeID
|
||||
route_path_decision_applied = $true
|
||||
route_path_decision_selected_relay_id = $nodeT.id
|
||||
route_path_decision_rendezvous_peer_node_id = $nodeB.id
|
||||
expected_effective_hops = @($nodeA.id, $nodeT.id, $nodeB.id)
|
||||
observed_ack_path = @($nodeA.id, $nodeT.id, $nodeB.id)
|
||||
route_path_drift_detected = $false
|
||||
control_plane_only = $true
|
||||
production_forwarding = $false
|
||||
production_payload_forwarding = $false
|
||||
route_health_production_payload_forwarding = $false
|
||||
route_health_service_payload_forwarding = $false
|
||||
}
|
||||
} | Out-Null
|
||||
|
||||
$healthyConfig = Invoke-Api -Method Get -Path "/clusters/$clusterID/nodes/$($nodeA.id)/mesh/synthetic-config"
|
||||
$healthyLease = @($healthyConfig.synthetic_mesh_config.rendezvous_leases | Where-Object { $_.peer_node_id -eq $nodeB.id }) | Select-Object -First 1
|
||||
$healthyLeaseReasons = @(Get-OptionalProperty -Object $healthyLease.metadata -PropertyName "relay_selection_score_reasons")
|
||||
$meshLinks = Invoke-Api -Method Get -Path "/clusters/$clusterID/mesh/links?actor_user_id=$actorUserID"
|
||||
$backendLogs = Invoke-RemoteDockerText -Arguments @("logs", "--tail", "80", $backendName)
|
||||
|
||||
$passMatrix = [ordered]@{
|
||||
backend_ready = $true
|
||||
owner_login = [bool]$actorUserID
|
||||
cluster_created = [bool]$clusterID
|
||||
route_intent_created = [bool]$routeID
|
||||
initial_fast_path_prefers_node_s = ($null -ne $initialLease -and $initialLease.relay_node_id -eq $nodeS.id)
|
||||
drift_feedback_reselects_node_t = ($null -ne $driftLease -and $driftLease.relay_node_id -eq $nodeT.id -and $driftLease.reason -eq "stale_relay_replacement")
|
||||
drift_route_decision_uses_node_t = ($null -ne $driftDecision -and $driftDecision.selected_relay_id -eq $nodeT.id -and $driftDecision.stale_relay_node_id -eq $nodeS.id)
|
||||
relay_policy_scoring_mode_c17z19 = ($driftConfig.synthetic_mesh_config.rendezvous_relay_policy.scoring_mode -eq "route_adjacency_endpoint_priority_mesh_link_health_synthetic_route_health_feedback")
|
||||
healthy_latency_keeps_node_t_selected = ($null -ne $healthyLease -and $healthyLease.relay_node_id -eq $nodeT.id)
|
||||
healthy_latency_score_reason_present = ($healthyLeaseReasons -contains "route_health_latency" -and $healthyLeaseReasons -contains "route_health_reachable")
|
||||
synthetic_config_signed = [bool]($healthyConfig.synthetic_mesh_config.authority_required -and $healthyConfig.synthetic_mesh_config.authority_payload -and $healthyConfig.synthetic_mesh_config.authority_signature)
|
||||
production_forwarding_disabled = ($healthyConfig.synthetic_mesh_config.production_forwarding -eq $false)
|
||||
}
|
||||
|
||||
$result = [ordered]@{
|
||||
run_id = $runId
|
||||
stage = "C17Z19 route-health feedback scoring smoke"
|
||||
docker_host = $DockerSshAlias
|
||||
backend_base_url = $backendPublicBaseUrl
|
||||
containers = @{
|
||||
backend = $backendName
|
||||
postgres = $postgresName
|
||||
redis = $redisName
|
||||
}
|
||||
cluster_id = $clusterID
|
||||
route_id = $routeID
|
||||
nodes = @{
|
||||
a = $nodeA.id
|
||||
b = $nodeB.id
|
||||
s = $nodeS.id
|
||||
t = $nodeT.id
|
||||
}
|
||||
initial_lease = $initialLease
|
||||
drift_lease = $driftLease
|
||||
drift_route_decision = $driftDecision
|
||||
healthy_lease = $healthyLease
|
||||
mesh_links = $meshLinks.mesh_links
|
||||
pass_matrix = $passMatrix
|
||||
backend_log_tail = $backendLogs
|
||||
containers_left_running = [bool]$KeepRunning
|
||||
}
|
||||
|
||||
$failed = @($passMatrix.GetEnumerator() | Where-Object { -not $_.Value })
|
||||
$resultJson = $result | ConvertTo-Json -Depth 80
|
||||
|
||||
if ($ResultPath -ne "") {
|
||||
if ([System.IO.Path]::IsPathRooted($ResultPath)) {
|
||||
$resultFullPath = $ResultPath
|
||||
} else {
|
||||
$resultFullPath = Join-Path $repoRoot $ResultPath
|
||||
}
|
||||
New-Item -ItemType Directory -Force -Path (Split-Path $resultFullPath) | Out-Null
|
||||
Set-Content -Path $resultFullPath -Value $resultJson -Encoding UTF8
|
||||
Write-Host "Result written to $resultFullPath"
|
||||
}
|
||||
|
||||
$resultJson
|
||||
|
||||
if ($failed.Count -gt 0) {
|
||||
throw "C17Z19 route-health feedback smoke failed: $($failed.Name -join ', ')"
|
||||
}
|
||||
|
||||
Invoke-RemoteShell -Command "rm -rf '$remoteBuildDir'"
|
||||
|
||||
if (-not $KeepRunning) {
|
||||
Write-Host "Cleaning up C17Z19 containers..."
|
||||
Remove-SmokeContainers
|
||||
}
|
||||
@@ -0,0 +1,174 @@
|
||||
param(
|
||||
[string]$BackendBaseUrl = "http://192.168.200.61:8080/api/v1",
|
||||
[string]$DockerContext = "test-ubuntu",
|
||||
[string]$AdminEmail = "windows-smoke@example.local",
|
||||
[string]$AdminPassword = "SmokePass!123",
|
||||
[int]$Count = 3,
|
||||
[string]$ImageTag = "rap-node-agent:control-panel-test"
|
||||
)
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
$repoRoot = (Resolve-Path (Join-Path $PSScriptRoot "..\..")).ProviderPath
|
||||
$stateRoot = Join-Path "C:\work" "rap-fabric-test-nodes"
|
||||
New-Item -ItemType Directory -Force -Path $stateRoot | Out-Null
|
||||
|
||||
function Invoke-Docker {
|
||||
param([string[]]$Arguments)
|
||||
docker @Arguments
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "docker $($Arguments -join ' ') failed with exit code $LASTEXITCODE"
|
||||
}
|
||||
}
|
||||
|
||||
function Invoke-Api {
|
||||
param(
|
||||
[string]$Method,
|
||||
[string]$Path,
|
||||
[object]$Body = $null
|
||||
)
|
||||
$uri = "$BackendBaseUrl$Path"
|
||||
if ($null -eq $Body) {
|
||||
return Invoke-RestMethod -Method $Method -Uri $uri -TimeoutSec 30
|
||||
}
|
||||
return Invoke-RestMethod -Method $Method -Uri $uri -ContentType "application/json" -Body ($Body | ConvertTo-Json -Depth 20) -TimeoutSec 30
|
||||
}
|
||||
|
||||
Write-Host "Building node-agent image on Docker context $DockerContext..."
|
||||
Invoke-Docker -Arguments @("--context", $DockerContext, "build", "-f", "$repoRoot\agents\rap-node-agent\Dockerfile", "-t", $ImageTag, "$repoRoot")
|
||||
|
||||
$login = Invoke-Api -Method Post -Path "/auth/login" -Body @{
|
||||
email = $AdminEmail
|
||||
password = $AdminPassword
|
||||
device_fingerprint = "fabric-test-nodes-script"
|
||||
device_label = "Fabric Test Nodes Script"
|
||||
trust_device = $true
|
||||
}
|
||||
$actorUserID = $login.user.id
|
||||
|
||||
$clusters = Invoke-Api -Method Get -Path "/clusters/?actor_user_id=$actorUserID"
|
||||
$cluster = @($clusters.clusters)[0]
|
||||
if ($null -eq $cluster) {
|
||||
throw "No cluster available for test node deployment."
|
||||
}
|
||||
$clusterID = $cluster.id
|
||||
Write-Host "Using cluster $($cluster.name) $clusterID"
|
||||
|
||||
Invoke-Api -Method Put -Path "/fabric/testing-flags" -Body @{
|
||||
actor_user_id = $actorUserID
|
||||
scope_type = "platform"
|
||||
scope_id = $null
|
||||
cluster_id = $null
|
||||
enabled = $true
|
||||
telemetry_enabled = $true
|
||||
synthetic_links_enabled = $true
|
||||
history_retention_hours = 24
|
||||
metadata = @{ source = "deploy-test-nodes.ps1"; runtime_mesh_enabled = $false }
|
||||
} | Out-Null
|
||||
|
||||
$joinToken = Invoke-Api -Method Post -Path "/clusters/$clusterID/join-tokens" -Body @{
|
||||
actor_user_id = $actorUserID
|
||||
scope = @{ purpose = "fabric-test-nodes"; roles = @("core-mesh", "relay-node") }
|
||||
expires_at = (Get-Date).ToUniversalTime().AddHours(2).ToString("o")
|
||||
max_uses = $Count
|
||||
}
|
||||
|
||||
$nodes = @()
|
||||
for ($i = 1; $i -le $Count; $i++) {
|
||||
$nodeName = "fabric-test-node-$i"
|
||||
$fingerprint = "rap-node-fp_$([guid]::NewGuid().ToString('N'))"
|
||||
$publicKey = "rap-node-pub_$([guid]::NewGuid().ToString('N'))"
|
||||
|
||||
$joinRequest = Invoke-Api -Method Post -Path "/clusters/$clusterID/join-requests" -Body @{
|
||||
join_token = $joinToken.join_token.token
|
||||
node_name = $nodeName
|
||||
node_fingerprint = $fingerprint
|
||||
public_key = $publicKey
|
||||
reported_capabilities = @{
|
||||
can_accept_node_ingress = $true
|
||||
can_route_mesh = $true
|
||||
can_run_rdp_worker = $false
|
||||
testing_node = $true
|
||||
}
|
||||
reported_facts = @{
|
||||
os = "linux"
|
||||
runtime = "docker-test"
|
||||
source = "deploy-test-nodes.ps1"
|
||||
}
|
||||
requested_roles = @("core-mesh", "relay-node")
|
||||
}
|
||||
|
||||
$approved = Invoke-Api -Method Post -Path "/clusters/$clusterID/join-requests/$($joinRequest.join_request.id)/approve" -Body @{
|
||||
actor_user_id = $actorUserID
|
||||
node_key = $fingerprint
|
||||
ownership_type = "platform_managed"
|
||||
owner_organization_id = $null
|
||||
}
|
||||
$nodeID = $approved.node_bootstrap.node_id
|
||||
$nodes += [pscustomobject]@{ name = $nodeName; id = $nodeID; fingerprint = $fingerprint; public_key = $publicKey }
|
||||
|
||||
$nodeStateDir = Join-Path $stateRoot $nodeName
|
||||
New-Item -ItemType Directory -Force -Path $nodeStateDir | Out-Null
|
||||
@{
|
||||
node_id = $nodeID
|
||||
cluster_id = $clusterID
|
||||
node_name = $nodeName
|
||||
node_fingerprint = $fingerprint
|
||||
public_key = $publicKey
|
||||
identity_status = "active"
|
||||
created_at = (Get-Date).ToUniversalTime().ToString("o")
|
||||
updated_at = (Get-Date).ToUniversalTime().ToString("o")
|
||||
} | ConvertTo-Json -Depth 10 | Set-Content -Encoding UTF8 -Path (Join-Path $nodeStateDir "identity.json")
|
||||
|
||||
$containerName = "rap_fabric_test_node_$i"
|
||||
docker --context $DockerContext rm -f $containerName 2>$null | Out-Null
|
||||
Invoke-Docker -Arguments @(
|
||||
"--context", $DockerContext,
|
||||
"create",
|
||||
"--name", $containerName,
|
||||
"--network", "host",
|
||||
"-e", "RAP_BACKEND_URL=$BackendBaseUrl",
|
||||
"-e", "RAP_NODE_STATE_DIR=/tmp/state",
|
||||
"-e", "RAP_HEARTBEAT_INTERVAL_SECONDS=5",
|
||||
$ImageTag,
|
||||
"-backend-url", $BackendBaseUrl,
|
||||
"-state-dir", "/tmp/state",
|
||||
"-heartbeat-interval", "5s"
|
||||
) | Out-Null
|
||||
Invoke-Docker -Arguments @("--context", $DockerContext, "cp", "$nodeStateDir\.", "$containerName`:/tmp/state")
|
||||
Invoke-Docker -Arguments @("--context", $DockerContext, "start", $containerName) | Out-Null
|
||||
Write-Host "Started $containerName for $nodeName $nodeID"
|
||||
}
|
||||
|
||||
Start-Sleep -Seconds 12
|
||||
|
||||
for ($i = 0; $i -lt $nodes.Count; $i++) {
|
||||
for ($j = 0; $j -lt $nodes.Count; $j++) {
|
||||
if ($i -eq $j) { continue }
|
||||
$latency = 2 + (($i + $j) % 7)
|
||||
Invoke-Api -Method Post -Path "/clusters/$clusterID/mesh/links" -Body @{
|
||||
source_node_id = $nodes[$i].id
|
||||
target_node_id = $nodes[$j].id
|
||||
link_status = "reachable"
|
||||
latency_ms = $latency
|
||||
quality_score = 95 - $latency
|
||||
metadata = @{
|
||||
source = "deploy-test-nodes.ps1"
|
||||
synthetic = $true
|
||||
runtime_mesh_enabled = $false
|
||||
}
|
||||
} | Out-Null
|
||||
}
|
||||
}
|
||||
|
||||
$summary = Invoke-Api -Method Get -Path "/cluster-admin-summaries?actor_user_id=$actorUserID"
|
||||
$links = Invoke-Api -Method Get -Path "/clusters/$clusterID/mesh/links?actor_user_id=$actorUserID"
|
||||
|
||||
[pscustomobject]@{
|
||||
cluster_id = $clusterID
|
||||
nodes_started = $nodes.Count
|
||||
cluster_summaries = $summary.cluster_summaries
|
||||
mesh_link_count = @($links.mesh_links).Count
|
||||
state_root = $stateRoot
|
||||
} | ConvertTo-Json -Depth 20
|
||||
@@ -0,0 +1,468 @@
|
||||
param(
|
||||
[string]$DockerSshAlias = "test-docker",
|
||||
[string]$BackendImageTag = "rap-backend:dev-enrollment-bootstrap-smoke",
|
||||
[string]$NodeAgentImageTag = "rap-node-agent:dev-enrollment-bootstrap-smoke",
|
||||
[string]$AdminEmail = "fabric-owner-dev-bootstrap@example.local",
|
||||
[string]$AdminPassword = "SmokePass!123",
|
||||
[int]$ApiPort = 18121,
|
||||
[int]$PostgresPort = 15443,
|
||||
[int]$RedisPort = 16443,
|
||||
[int]$MeshPort = 19131,
|
||||
[string]$ResultPath = "artifacts\dev-cluster-enrollment-bootstrap-smoke-result.json",
|
||||
[switch]$KeepRunning
|
||||
)
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
$repoRoot = (Resolve-Path (Join-Path $PSScriptRoot "..\..")).ProviderPath
|
||||
$backendPublicBaseUrl = "http://192.168.200.61:$ApiPort/api/v1"
|
||||
$backendContainerBaseUrl = "http://127.0.0.1:$ApiPort/api/v1"
|
||||
$runId = "dev-bootstrap-" + (Get-Date -Format "yyyyMMdd-HHmmss")
|
||||
$remoteBuildDir = "/tmp/rap-dev-bootstrap-build-$runId"
|
||||
|
||||
$postgresName = "rap_dev_bootstrap_postgres"
|
||||
$redisName = "rap_dev_bootstrap_redis"
|
||||
$backendName = "rap_dev_bootstrap_backend"
|
||||
$nodeName = "rap-dev-bootstrap-node-core"
|
||||
$nodeContainerName = "rap_dev_bootstrap_node_core"
|
||||
|
||||
function Invoke-RemoteDocker {
|
||||
param([string[]]$Arguments)
|
||||
& ssh $DockerSshAlias docker @Arguments
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "ssh $DockerSshAlias docker $($Arguments -join ' ') failed with exit code $LASTEXITCODE"
|
||||
}
|
||||
}
|
||||
|
||||
function Invoke-RemoteDockerText {
|
||||
param([string[]]$Arguments)
|
||||
$previousErrorActionPreference = $ErrorActionPreference
|
||||
$ErrorActionPreference = "Continue"
|
||||
try {
|
||||
$output = & ssh $DockerSshAlias docker @Arguments 2>&1 | ForEach-Object { $_.ToString() }
|
||||
}
|
||||
finally {
|
||||
$ErrorActionPreference = $previousErrorActionPreference
|
||||
}
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "ssh $DockerSshAlias docker $($Arguments -join ' ') failed with exit code $LASTEXITCODE"
|
||||
}
|
||||
return $output
|
||||
}
|
||||
|
||||
function Invoke-RemoteShell {
|
||||
param([string]$Command)
|
||||
& ssh $DockerSshAlias $Command
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "ssh $DockerSshAlias $Command failed with exit code $LASTEXITCODE"
|
||||
}
|
||||
}
|
||||
|
||||
function Invoke-RemoteShellOptionalText {
|
||||
param([string]$Command)
|
||||
$previousErrorActionPreference = $ErrorActionPreference
|
||||
$ErrorActionPreference = "Continue"
|
||||
try {
|
||||
$output = & ssh $DockerSshAlias $Command 2>&1 | ForEach-Object { $_.ToString() }
|
||||
$exitCode = $LASTEXITCODE
|
||||
}
|
||||
finally {
|
||||
$ErrorActionPreference = $previousErrorActionPreference
|
||||
}
|
||||
return [pscustomobject]@{
|
||||
exit_code = $exitCode
|
||||
output = $output
|
||||
}
|
||||
}
|
||||
|
||||
function Send-RemoteBuildContext {
|
||||
Write-Host "Uploading backend and node-agent build context to $DockerSshAlias..."
|
||||
Invoke-RemoteShell -Command "rm -rf '$remoteBuildDir' && mkdir -p '$remoteBuildDir'"
|
||||
& tar -czf - -C $repoRoot "backend" "agents/rap-node-agent" | & ssh $DockerSshAlias "tar -xzf - -C '$remoteBuildDir'"
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "upload build context failed"
|
||||
}
|
||||
}
|
||||
|
||||
function Invoke-Api {
|
||||
param(
|
||||
[string]$Method,
|
||||
[string]$Path,
|
||||
[object]$Body = $null
|
||||
)
|
||||
$uri = "$backendPublicBaseUrl$Path"
|
||||
if ($null -eq $Body) {
|
||||
return Invoke-RestMethod -Method $Method -Uri $uri -TimeoutSec 30
|
||||
}
|
||||
return Invoke-RestMethod -Method $Method -Uri $uri -ContentType "application/json" -Body ($Body | ConvertTo-Json -Depth 60) -TimeoutSec 30
|
||||
}
|
||||
|
||||
function Wait-HttpReady {
|
||||
param([string]$Url)
|
||||
for ($i = 0; $i -lt 90; $i++) {
|
||||
try {
|
||||
$response = Invoke-WebRequest -UseBasicParsing -Uri $Url -TimeoutSec 2
|
||||
if ($response.StatusCode -ge 200 -and $response.StatusCode -lt 300) {
|
||||
return
|
||||
}
|
||||
}
|
||||
catch {
|
||||
Start-Sleep -Seconds 1
|
||||
}
|
||||
}
|
||||
throw "Timed out waiting for $Url"
|
||||
}
|
||||
|
||||
function Remove-SmokeContainers {
|
||||
foreach ($name in @($nodeContainerName, $backendName, $postgresName, $redisName)) {
|
||||
& ssh $DockerSshAlias docker rm -f $name 2>$null | Out-Null
|
||||
}
|
||||
}
|
||||
|
||||
function Wait-JoinRequest {
|
||||
param(
|
||||
[string]$ClusterID,
|
||||
[string]$ActorUserID,
|
||||
[string]$ExpectedNodeName,
|
||||
[int]$TimeoutSeconds = 60
|
||||
)
|
||||
$deadline = (Get-Date).AddSeconds($TimeoutSeconds)
|
||||
do {
|
||||
$response = Invoke-Api -Method Get -Path "/clusters/$ClusterID/join-requests?actor_user_id=$ActorUserID"
|
||||
$match = @($response.join_requests | Where-Object {
|
||||
$_.node_name -eq $ExpectedNodeName -and $_.status -eq "pending"
|
||||
} | Sort-Object created_at -Descending | Select-Object -First 1)
|
||||
if ($match.Count -gt 0) {
|
||||
return $match[0]
|
||||
}
|
||||
Start-Sleep -Seconds 1
|
||||
} while ((Get-Date) -lt $deadline)
|
||||
throw "Timed out waiting for node-agent join request"
|
||||
}
|
||||
|
||||
function Get-RemoteContainerIdentity {
|
||||
$remoteIdentity = "/tmp/$runId-identity.json"
|
||||
$command = "rm -f '$remoteIdentity'; docker cp '${nodeContainerName}:/tmp/state/identity.json' '$remoteIdentity' >/dev/null 2>&1 && cat '$remoteIdentity'"
|
||||
$result = Invoke-RemoteShellOptionalText -Command $command
|
||||
if ($result.exit_code -ne 0) {
|
||||
return $null
|
||||
}
|
||||
$raw = ($result.output -join "`n").Trim()
|
||||
if ($raw -eq "") {
|
||||
return $null
|
||||
}
|
||||
return $raw | ConvertFrom-Json
|
||||
}
|
||||
|
||||
function Wait-AgentApprovedIdentity {
|
||||
param([int]$TimeoutSeconds = 90)
|
||||
$deadline = (Get-Date).AddSeconds($TimeoutSeconds)
|
||||
do {
|
||||
$identity = Get-RemoteContainerIdentity
|
||||
if ($null -ne $identity -and
|
||||
$identity.node_id -and
|
||||
$identity.identity_status -eq "active" -and
|
||||
$identity.cluster_authority_public_key -and
|
||||
$identity.cluster_authority_fingerprint) {
|
||||
return $identity
|
||||
}
|
||||
Start-Sleep -Seconds 1
|
||||
} while ((Get-Date) -lt $deadline)
|
||||
throw "Timed out waiting for node-agent approved identity state"
|
||||
}
|
||||
|
||||
function Wait-NodeHeartbeat {
|
||||
param(
|
||||
[string]$ClusterID,
|
||||
[string]$NodeID,
|
||||
[string]$ActorUserID,
|
||||
[int]$TimeoutSeconds = 60
|
||||
)
|
||||
$deadline = (Get-Date).AddSeconds($TimeoutSeconds)
|
||||
do {
|
||||
$response = Invoke-Api -Method Get -Path "/clusters/$ClusterID/nodes/$NodeID/heartbeats?actor_user_id=$ActorUserID&limit=5"
|
||||
$heartbeats = @($response.heartbeats)
|
||||
if ($heartbeats.Count -gt 0) {
|
||||
return $heartbeats[0]
|
||||
}
|
||||
Start-Sleep -Seconds 1
|
||||
} while ((Get-Date) -lt $deadline)
|
||||
throw "Timed out waiting for node-agent heartbeat"
|
||||
}
|
||||
|
||||
function Wait-AgentLogContains {
|
||||
param(
|
||||
[string]$Pattern,
|
||||
[int]$TimeoutSeconds = 60
|
||||
)
|
||||
$deadline = (Get-Date).AddSeconds($TimeoutSeconds)
|
||||
do {
|
||||
$logs = Invoke-RemoteDockerText -Arguments @("logs", "--tail", "200", $nodeContainerName)
|
||||
$joined = $logs -join "`n"
|
||||
if ($joined -match $Pattern) {
|
||||
return $logs
|
||||
}
|
||||
Start-Sleep -Seconds 1
|
||||
} while ((Get-Date) -lt $deadline)
|
||||
return Invoke-RemoteDockerText -Arguments @("logs", "--tail", "200", $nodeContainerName)
|
||||
}
|
||||
|
||||
Write-Host "Dev cluster enrollment/bootstrap smoke run: $runId"
|
||||
Write-Host "Using SSH Docker host: $DockerSshAlias"
|
||||
|
||||
Remove-SmokeContainers
|
||||
Send-RemoteBuildContext
|
||||
|
||||
Write-Host "Building backend image on docker-test..."
|
||||
Invoke-RemoteDocker -Arguments @("build", "-f", "$remoteBuildDir/backend/Dockerfile", "-t", $BackendImageTag, "$remoteBuildDir/backend")
|
||||
|
||||
Write-Host "Building node-agent image on docker-test..."
|
||||
Invoke-RemoteDocker -Arguments @("build", "-f", "$remoteBuildDir/agents/rap-node-agent/Dockerfile", "-t", $NodeAgentImageTag, $remoteBuildDir)
|
||||
|
||||
Write-Host "Starting isolated PostgreSQL and Redis..."
|
||||
Invoke-RemoteDocker -Arguments @(
|
||||
"run", "-d",
|
||||
"--name", $postgresName,
|
||||
"-e", "POSTGRES_DB=remote_access_platform",
|
||||
"-e", "POSTGRES_USER=rap_user",
|
||||
"-e", "POSTGRES_PASSWORD=rap_password",
|
||||
"-p", "$PostgresPort`:5432",
|
||||
"postgres:16"
|
||||
)
|
||||
Invoke-RemoteDocker -Arguments @(
|
||||
"run", "-d",
|
||||
"--name", $redisName,
|
||||
"-p", "$RedisPort`:6379",
|
||||
"redis:7"
|
||||
)
|
||||
|
||||
Invoke-RemoteShell -Command "for i in `$(seq 1 60); do docker exec $postgresName pg_isready -U rap_user -d remote_access_platform >/dev/null 2>&1 && exit 0; sleep 1; done; exit 1"
|
||||
|
||||
Write-Host "Applying migrations..."
|
||||
Invoke-RemoteShell -Command "for f in `$(find '$remoteBuildDir/backend/migrations' -name '*.up.sql' | sort); do docker exec -i $postgresName psql -U rap_user -d remote_access_platform -v ON_ERROR_STOP=1 -f - < `$f; done"
|
||||
|
||||
$secretBytes = New-Object byte[] 32
|
||||
[Security.Cryptography.RandomNumberGenerator]::Fill($secretBytes)
|
||||
$secretKeyB64 = [Convert]::ToBase64String($secretBytes)
|
||||
|
||||
Write-Host "Starting backend..."
|
||||
Invoke-RemoteDocker -Arguments @(
|
||||
"run", "-d",
|
||||
"--name", $backendName,
|
||||
"--network", "host",
|
||||
"-e", "APP_NAME=rap-api",
|
||||
"-e", "APP_ENV=dev-bootstrap-smoke",
|
||||
"-e", "HTTP_HOST=0.0.0.0",
|
||||
"-e", "HTTP_PORT=$ApiPort",
|
||||
"-e", "POSTGRES_DSN=postgres://rap_user:rap_password@127.0.0.1:$PostgresPort/remote_access_platform?sslmode=disable",
|
||||
"-e", "REDIS_ADDR=127.0.0.1:$RedisPort",
|
||||
"-e", "AUTH_ACCESS_TOKEN_SECRET=dev-bootstrap-access-secret",
|
||||
"-e", "AUTH_REFRESH_HASH_SECRET=dev-bootstrap-refresh-secret",
|
||||
"-e", "INSTALLATION_AUTHORITY_MODE=legacy",
|
||||
"-e", "INSTALLATION_INSECURE_BOOTSTRAP_ENABLED=true",
|
||||
"-e", "SECRET_ENCRYPTION_KEY_B64=$secretKeyB64",
|
||||
"-e", "SECRET_ENCRYPTION_KEY_ID=$runId",
|
||||
$BackendImageTag
|
||||
)
|
||||
Wait-HttpReady -Url "http://192.168.200.61:$ApiPort/readyz"
|
||||
|
||||
Write-Host "Bootstrapping dev platform owner through installation API..."
|
||||
$bootstrap = Invoke-Api -Method Post -Path "/installation/bootstrap-owner" -Body @{
|
||||
email = $AdminEmail
|
||||
password = $AdminPassword
|
||||
}
|
||||
$installationStatus = Invoke-Api -Method Get -Path "/installation/status"
|
||||
|
||||
Write-Host "Logging in as platform owner..."
|
||||
$login = Invoke-Api -Method Post -Path "/auth/login" -Body @{
|
||||
email = $AdminEmail
|
||||
password = $AdminPassword
|
||||
device_fingerprint = "dev-bootstrap-smoke-device"
|
||||
device_label = "Dev bootstrap lifecycle smoke"
|
||||
trust_device = $true
|
||||
}
|
||||
$actorUserID = $login.user.id
|
||||
|
||||
Write-Host "Creating dev cluster and join token..."
|
||||
$cluster = Invoke-Api -Method Post -Path "/clusters/" -Body @{
|
||||
actor_user_id = $actorUserID
|
||||
slug = "dev-bootstrap-$((New-Guid).Guid.Substring(0, 8))"
|
||||
name = "Dev Enrollment Bootstrap Smoke"
|
||||
region = "docker-test"
|
||||
metadata = @{
|
||||
stage = "dev-enrollment-bootstrap-smoke"
|
||||
run_id = $runId
|
||||
mandatory_roles_only = $true
|
||||
production_forwarding = $false
|
||||
service_payload_forwarding = $false
|
||||
}
|
||||
}
|
||||
$clusterID = $cluster.cluster.id
|
||||
|
||||
Invoke-Api -Method Put -Path "/fabric/testing-flags" -Body @{
|
||||
actor_user_id = $actorUserID
|
||||
scope_type = "platform"
|
||||
scope_id = $null
|
||||
cluster_id = $null
|
||||
enabled = $true
|
||||
telemetry_enabled = $true
|
||||
synthetic_links_enabled = $true
|
||||
history_retention_hours = 24
|
||||
metadata = @{
|
||||
stage = "dev-enrollment-bootstrap-smoke"
|
||||
run_id = $runId
|
||||
production_forwarding = $false
|
||||
service_payload_forwarding = $false
|
||||
}
|
||||
} | Out-Null
|
||||
|
||||
$joinToken = Invoke-Api -Method Post -Path "/clusters/$clusterID/join-tokens" -Body @{
|
||||
actor_user_id = $actorUserID
|
||||
scope = @{
|
||||
purpose = "dev-enrollment-bootstrap-smoke"
|
||||
roles = @("core-mesh")
|
||||
mandatory_only = $true
|
||||
production_forwarding = $false
|
||||
}
|
||||
expires_at = (Get-Date).ToUniversalTime().AddHours(2).ToString("o")
|
||||
max_uses = 1
|
||||
}
|
||||
|
||||
Write-Host "Starting real node-agent enrollment container..."
|
||||
Invoke-RemoteDocker -Arguments @(
|
||||
"run", "-d",
|
||||
"--name", $nodeContainerName,
|
||||
"--network", "host",
|
||||
"-e", "RAP_BACKEND_URL=$backendContainerBaseUrl",
|
||||
"-e", "RAP_CLUSTER_ID=$clusterID",
|
||||
"-e", "RAP_JOIN_TOKEN=$($joinToken.join_token.token)",
|
||||
"-e", "RAP_NODE_NAME=$nodeName",
|
||||
"-e", "RAP_NODE_STATE_DIR=/tmp/state",
|
||||
"-e", "RAP_HEARTBEAT_INTERVAL_SECONDS=2",
|
||||
"-e", "RAP_ENROLLMENT_POLL_INTERVAL_SECONDS=1",
|
||||
"-e", "RAP_ENROLLMENT_POLL_TIMEOUT_SECONDS=90",
|
||||
"-e", "RAP_MESH_SYNTHETIC_RUNTIME_ENABLED=true",
|
||||
"-e", "RAP_MESH_LISTEN_ADDR=0.0.0.0:$MeshPort",
|
||||
"-e", "RAP_MESH_ADVERTISE_ENDPOINT=http://127.0.0.1:$MeshPort",
|
||||
"-e", "RAP_MESH_ADVERTISE_TRANSPORT=direct_tcp_tls",
|
||||
"-e", "RAP_MESH_CONNECTIVITY_MODE=direct",
|
||||
"-e", "RAP_MESH_NAT_TYPE=none",
|
||||
"-e", "RAP_MESH_REGION=docker-test",
|
||||
$NodeAgentImageTag
|
||||
)
|
||||
|
||||
Write-Host "Waiting for pending join request from node-agent..."
|
||||
$joinRequest = Wait-JoinRequest -ClusterID $clusterID -ActorUserID $actorUserID -ExpectedNodeName $nodeName
|
||||
|
||||
Write-Host "Approving join request..."
|
||||
$approved = Invoke-Api -Method Post -Path "/clusters/$clusterID/join-requests/$($joinRequest.id)/approve" -Body @{
|
||||
actor_user_id = $actorUserID
|
||||
node_key = $joinRequest.node_fingerprint
|
||||
ownership_type = "platform_managed"
|
||||
owner_organization_id = $null
|
||||
}
|
||||
|
||||
Write-Host "Waiting for node-agent to persist signed bootstrap authority pin..."
|
||||
$identity = Wait-AgentApprovedIdentity
|
||||
$nodeID = $identity.node_id
|
||||
|
||||
Write-Host "Assigning mandatory core-mesh role after approved identity..."
|
||||
Invoke-Api -Method Post -Path "/clusters/$clusterID/nodes/$nodeID/roles" -Body @{
|
||||
actor_user_id = $actorUserID
|
||||
role = "core-mesh"
|
||||
status = "active"
|
||||
policy = @{
|
||||
stage = "dev-enrollment-bootstrap-smoke"
|
||||
run_id = $runId
|
||||
mandatory = $true
|
||||
production_forwarding = $false
|
||||
service_payload_forwarding = $false
|
||||
}
|
||||
} | Out-Null
|
||||
|
||||
Write-Host "Waiting for heartbeat and signed synthetic config verification..."
|
||||
$heartbeat = Wait-NodeHeartbeat -ClusterID $clusterID -NodeID $nodeID -ActorUserID $actorUserID
|
||||
$configResponse = Invoke-Api -Method Get -Path "/clusters/$clusterID/nodes/$nodeID/mesh/synthetic-config"
|
||||
$syntheticConfig = $configResponse.synthetic_mesh_config
|
||||
$nodeLogs = Wait-AgentLogContains -Pattern "synthetic mesh config loaded: source=control_plane" -TimeoutSeconds 60
|
||||
$backendLogs = Invoke-RemoteDockerText -Arguments @("logs", "--tail", "120", $backendName)
|
||||
$summaries = Invoke-Api -Method Get -Path "/cluster-admin-summaries?actor_user_id=$actorUserID"
|
||||
|
||||
$agentLogText = $nodeLogs -join "`n"
|
||||
$passMatrix = [ordered]@{
|
||||
installation_bootstrapped = [bool]$installationStatus.installation.bootstrapped
|
||||
cluster_created = [bool]$clusterID
|
||||
join_token_has_cluster_authority_signature = [bool]($joinToken.join_token.authority_payload -and $joinToken.join_token.authority_signature)
|
||||
agent_submitted_pending_join_request = [bool]($joinRequest.id -and $joinRequest.status -eq "pending")
|
||||
approval_returned_signed_bootstrap = [bool]($approved.node_bootstrap.cluster_authority -and $approved.node_bootstrap.authority_payload -and $approved.node_bootstrap.authority_signature)
|
||||
agent_persisted_approved_identity = [bool]($identity.node_id -eq $approved.node_bootstrap.node_id -and $identity.identity_status -eq "active")
|
||||
agent_persisted_cluster_authority_pin = [bool]($identity.cluster_authority_public_key -and $identity.cluster_authority_fingerprint)
|
||||
heartbeat_after_auto_bootstrap = [bool]$heartbeat.id
|
||||
synthetic_config_has_cluster_authority_signature = [bool]($syntheticConfig.authority_required -and $syntheticConfig.cluster_authority -and $syntheticConfig.authority_payload -and $syntheticConfig.authority_signature)
|
||||
agent_verified_control_plane_synthetic_config = [bool]($agentLogText -match "synthetic mesh config loaded: source=control_plane")
|
||||
production_forwarding_disabled = [bool](-not $syntheticConfig.production_forwarding)
|
||||
}
|
||||
|
||||
$result = [ordered]@{
|
||||
run_id = $runId
|
||||
stage = "dev cluster enrollment/bootstrap lifecycle smoke"
|
||||
docker_host = $DockerSshAlias
|
||||
backend_base_url = $backendPublicBaseUrl
|
||||
api_port = $ApiPort
|
||||
postgres_port = $PostgresPort
|
||||
redis_port = $RedisPort
|
||||
mesh_port = $MeshPort
|
||||
backend_image = $BackendImageTag
|
||||
node_agent_image = $NodeAgentImageTag
|
||||
containers = @{
|
||||
backend = $backendName
|
||||
postgres = $postgresName
|
||||
redis = $redisName
|
||||
node_agent = $nodeContainerName
|
||||
}
|
||||
installation = $installationStatus.installation
|
||||
admin_user_id = $actorUserID
|
||||
cluster_id = $clusterID
|
||||
node_id = $nodeID
|
||||
join_request_id = $joinRequest.id
|
||||
join_token_signature = $joinToken.join_token.authority_signature
|
||||
approval_authority = $approved.node_bootstrap.cluster_authority
|
||||
identity_authority_fingerprint = $identity.cluster_authority_fingerprint
|
||||
synthetic_config_schema = $syntheticConfig.schema_version
|
||||
synthetic_config_version = $syntheticConfig.config_version
|
||||
synthetic_config_authority = $syntheticConfig.cluster_authority
|
||||
latest_heartbeat = $heartbeat
|
||||
cluster_summaries = $summaries.cluster_summaries
|
||||
pass_matrix = $passMatrix
|
||||
backend_log_tail = $backendLogs
|
||||
node_log_tail = $nodeLogs
|
||||
containers_left_running = [bool]$KeepRunning
|
||||
}
|
||||
|
||||
$failed = @($passMatrix.GetEnumerator() | Where-Object { -not $_.Value })
|
||||
$resultJson = $result | ConvertTo-Json -Depth 70
|
||||
|
||||
if ($ResultPath -ne "") {
|
||||
if ([System.IO.Path]::IsPathRooted($ResultPath)) {
|
||||
$resultFullPath = $ResultPath
|
||||
} else {
|
||||
$resultFullPath = Join-Path $repoRoot $ResultPath
|
||||
}
|
||||
New-Item -ItemType Directory -Force -Path (Split-Path $resultFullPath) | Out-Null
|
||||
Set-Content -Path $resultFullPath -Value $resultJson -Encoding UTF8
|
||||
Write-Host "Result written to $resultFullPath"
|
||||
}
|
||||
|
||||
$resultJson
|
||||
|
||||
if ($failed.Count -gt 0) {
|
||||
throw "Dev enrollment/bootstrap smoke failed: $($failed.Name -join ', ')"
|
||||
}
|
||||
|
||||
Invoke-RemoteShell -Command "rm -rf '$remoteBuildDir'"
|
||||
|
||||
if (-not $KeepRunning) {
|
||||
Write-Host "Cleaning up dev enrollment/bootstrap containers..."
|
||||
Remove-SmokeContainers
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
# Installation Authority Tooling
|
||||
|
||||
The Product Root private key must stay outside the repository and outside the
|
||||
cluster database. The backend stores only the public key and signed activation
|
||||
records.
|
||||
|
||||
Generate a Product Root key pair:
|
||||
|
||||
```powershell
|
||||
go run scripts/installation/product-root-tool.go generate-key
|
||||
```
|
||||
|
||||
Configure production backend nodes with the generated `public_key_b64`:
|
||||
|
||||
```powershell
|
||||
$env:INSTALLATION_AUTHORITY_MODE = "strict"
|
||||
$env:INSTALLATION_PRODUCT_ROOT_PUBLIC_KEY_B64 = "<public_key_b64>"
|
||||
```
|
||||
|
||||
Create a signed first-owner activation manifest:
|
||||
|
||||
```powershell
|
||||
go run scripts/installation/product-root-tool.go activate `
|
||||
-private-key-file C:\secure\rap-product-root.json `
|
||||
-install-id install-prod-001 `
|
||||
-owner-email owner@example.com `
|
||||
-expires-at 2026-05-01T00:00:00Z `
|
||||
-environment production
|
||||
```
|
||||
|
||||
Use the output `activation_payload` and `activation_signature` in the admin
|
||||
panel first-owner screen or in `POST /api/v1/installation/bootstrap-owner`.
|
||||
@@ -0,0 +1,160 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"crypto/ed25519"
|
||||
"crypto/rand"
|
||||
"encoding/base64"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"flag"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
const activationSchemaVersion = "rap.installation.activation.v1"
|
||||
|
||||
func main() {
|
||||
if len(os.Args) < 2 {
|
||||
fail("usage: go run scripts/installation/product-root-tool.go <generate-key|activate> [flags]")
|
||||
}
|
||||
switch os.Args[1] {
|
||||
case "generate-key":
|
||||
generateKey()
|
||||
case "activate":
|
||||
activate(os.Args[2:])
|
||||
default:
|
||||
fail("unknown command %q", os.Args[1])
|
||||
}
|
||||
}
|
||||
|
||||
func generateKey() {
|
||||
publicKey, privateKey, err := ed25519.GenerateKey(rand.Reader)
|
||||
if err != nil {
|
||||
fail("generate key: %v", err)
|
||||
}
|
||||
writeJSON(map[string]string{
|
||||
"key_type": "ed25519",
|
||||
"private_key_b64": base64.StdEncoding.EncodeToString(privateKey),
|
||||
"public_key_b64": base64.StdEncoding.EncodeToString(publicKey),
|
||||
})
|
||||
}
|
||||
|
||||
func activate(args []string) {
|
||||
fs := flag.NewFlagSet("activate", flag.ExitOnError)
|
||||
privateKeyB64 := fs.String("private-key-b64", "", "base64 Ed25519 private key")
|
||||
privateKeyFile := fs.String("private-key-file", "", "file with base64 key or generate-key JSON")
|
||||
installID := fs.String("install-id", "", "installation id; generated when empty")
|
||||
ownerEmail := fs.String("owner-email", "", "first owner email")
|
||||
role := fs.String("role", "platform_admin", "platform_admin or platform_recovery_admin")
|
||||
expiresAt := fs.String("expires-at", "", "RFC3339 expiry time")
|
||||
environment := fs.String("environment", "", "optional environment label")
|
||||
nonce := fs.String("nonce", "", "optional nonce")
|
||||
if err := fs.Parse(args); err != nil {
|
||||
fail("parse flags: %v", err)
|
||||
}
|
||||
|
||||
keyText := strings.TrimSpace(*privateKeyB64)
|
||||
if keyText == "" && strings.TrimSpace(*privateKeyFile) != "" {
|
||||
content, err := os.ReadFile(*privateKeyFile)
|
||||
if err != nil {
|
||||
fail("read private key file: %v", err)
|
||||
}
|
||||
keyText = extractPrivateKeyText(content)
|
||||
}
|
||||
privateKey := decodePrivateKey(keyText)
|
||||
|
||||
email := strings.ToLower(strings.TrimSpace(*ownerEmail))
|
||||
if email == "" || !strings.Contains(email, "@") {
|
||||
fail("owner-email is required")
|
||||
}
|
||||
normalizedRole := strings.TrimSpace(*role)
|
||||
if normalizedRole != "platform_admin" && normalizedRole != "platform_recovery_admin" {
|
||||
fail("role must be platform_admin or platform_recovery_admin")
|
||||
}
|
||||
id := strings.TrimSpace(*installID)
|
||||
if id == "" {
|
||||
id = randomID("install")
|
||||
}
|
||||
|
||||
payload := map[string]any{
|
||||
"schema_version": activationSchemaVersion,
|
||||
"install_id": id,
|
||||
"owner_email": email,
|
||||
"platform_role": normalizedRole,
|
||||
"issued_at": time.Now().UTC().Format(time.RFC3339),
|
||||
}
|
||||
if strings.TrimSpace(*expiresAt) != "" {
|
||||
if _, err := time.Parse(time.RFC3339, strings.TrimSpace(*expiresAt)); err != nil {
|
||||
fail("expires-at must be RFC3339: %v", err)
|
||||
}
|
||||
payload["expires_at"] = strings.TrimSpace(*expiresAt)
|
||||
}
|
||||
if strings.TrimSpace(*environment) != "" {
|
||||
payload["environment"] = strings.TrimSpace(*environment)
|
||||
}
|
||||
if strings.TrimSpace(*nonce) != "" {
|
||||
payload["nonce"] = strings.TrimSpace(*nonce)
|
||||
}
|
||||
|
||||
canonical, err := json.Marshal(payload)
|
||||
if err != nil {
|
||||
fail("canonicalize payload: %v", err)
|
||||
}
|
||||
signature := ed25519.Sign(privateKey, canonical)
|
||||
writeJSON(map[string]any{
|
||||
"activation_payload": payload,
|
||||
"activation_signature": base64.StdEncoding.EncodeToString(signature),
|
||||
})
|
||||
}
|
||||
|
||||
func extractPrivateKeyText(content []byte) string {
|
||||
text := strings.TrimSpace(string(content))
|
||||
var generated struct {
|
||||
PrivateKey string `json:"private_key_b64"`
|
||||
}
|
||||
if err := json.Unmarshal(content, &generated); err == nil && strings.TrimSpace(generated.PrivateKey) != "" {
|
||||
return strings.TrimSpace(generated.PrivateKey)
|
||||
}
|
||||
return text
|
||||
}
|
||||
|
||||
func decodePrivateKey(value string) ed25519.PrivateKey {
|
||||
if strings.TrimSpace(value) == "" {
|
||||
fail("private key is required")
|
||||
}
|
||||
decoded, err := base64.StdEncoding.DecodeString(strings.TrimSpace(value))
|
||||
if err != nil {
|
||||
if raw, rawErr := base64.RawStdEncoding.DecodeString(strings.TrimSpace(value)); rawErr == nil {
|
||||
decoded = raw
|
||||
} else {
|
||||
fail("private key must be base64 encoded: %v", err)
|
||||
}
|
||||
}
|
||||
if len(decoded) != ed25519.PrivateKeySize {
|
||||
fail("private key must decode to %d bytes", ed25519.PrivateKeySize)
|
||||
}
|
||||
return ed25519.PrivateKey(decoded)
|
||||
}
|
||||
|
||||
func randomID(prefix string) string {
|
||||
var bytes [16]byte
|
||||
if _, err := rand.Read(bytes[:]); err != nil {
|
||||
fail("generate id: %v", err)
|
||||
}
|
||||
return fmt.Sprintf("%s-%s", prefix, hex.EncodeToString(bytes[:]))
|
||||
}
|
||||
|
||||
func writeJSON(value any) {
|
||||
encoder := json.NewEncoder(os.Stdout)
|
||||
encoder.SetIndent("", " ")
|
||||
if err := encoder.Encode(value); err != nil {
|
||||
fail("write json: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func fail(format string, args ...any) {
|
||||
fmt.Fprintf(os.Stderr, format+"\n", args...)
|
||||
os.Exit(1)
|
||||
}
|
||||
@@ -0,0 +1,597 @@
|
||||
# Local Smoke Test
|
||||
|
||||
This smoke path is for proving the minimal end-to-end server-side session lifecycle without any UI.
|
||||
|
||||
## Verification matrix
|
||||
|
||||
### Locally proven in this repository work
|
||||
|
||||
- backend `go build ./...` succeeds
|
||||
- worker build environment files exist and are aligned across devcontainer, Docker, and CI
|
||||
- worker Docker image contract now has a deterministic runtime binary path: `/usr/local/bin/rdp-worker`
|
||||
- worker source had minimal compile fixes applied for missing declarations/includes needed by the reproducible build environment
|
||||
|
||||
### Container-proven
|
||||
|
||||
- the canonical worker build environment is [workers/rdp-worker/Dockerfile](/\\?\UNC\192.168.220.200\mst\codex\rdp-proxy\workers\rdp-worker\Dockerfile)
|
||||
- a successful `docker build` in that environment proves CMake configure + compile + install for the worker
|
||||
- Data Plane v1 Stage DP-1C builds the optional worker direct WSS endpoint and
|
||||
installs `/usr/local/bin/rdp-worker-dataplane-token-probe`
|
||||
- Data Plane v1 Stage DP-1D.1 builds the worker direct JSON realtime bridge
|
||||
for the same JSON envelopes used by the backend gateway
|
||||
- DP-1C endpoint validation is proven for malformed-token rejection and
|
||||
valid-token-without-runtime rejection, and replayed `jti` rejection;
|
||||
successful direct attach to a live runtime and direct JSON traffic proof are
|
||||
still live smoke targets
|
||||
|
||||
### CI-defined but not yet executed in this verification pass
|
||||
|
||||
- [`.github/workflows/build.yml`](/\\?\UNC\192.168.220.200\mst\codex\rdp-proxy\.github\workflows\build.yml) builds the backend
|
||||
- the same workflow builds the worker Docker image
|
||||
- the workflow verifies that `/usr/local/bin/rdp-worker` exists inside the image
|
||||
|
||||
### Still not proven automatically
|
||||
|
||||
- behavior on host reboot during an active real RDP session
|
||||
- automated assertion of actual viewer-side rendering correctness
|
||||
- Stage 5.1 file upload is build-proven in this pass, but live upload proof
|
||||
requires the RAP stack to be running on the current test Docker host
|
||||
`192.168.200.61`
|
||||
|
||||
## Data Plane v1C worker WSS validation smoke
|
||||
|
||||
Build the worker image on the test Docker host:
|
||||
|
||||
```powershell
|
||||
$env:DOCKER_BUILDKIT='0'
|
||||
docker --context test-ubuntu build --tag rap-rdp-worker:dp1c-hardened --file workers/rdp-worker/Dockerfile workers/rdp-worker
|
||||
```
|
||||
|
||||
Run the narrow endpoint smoke:
|
||||
|
||||
```powershell
|
||||
pwsh -ExecutionPolicy Bypass -File scripts/smoke/data-plane-v1c-smoke.ps1
|
||||
```
|
||||
|
||||
Expected evidence:
|
||||
|
||||
- worker logs `direct data-plane WSS endpoint listening`
|
||||
- malformed token receives `401 Unauthorized`
|
||||
- valid token with no existing session runtime receives `404 Not Found` with
|
||||
`missing_runtime`
|
||||
- replaying the same token receives `401 Unauthorized` with
|
||||
`jti_replay_rejected`
|
||||
- worker logs `event=token_validation_failed reason=malformed_token`
|
||||
- worker logs `event=data_plane_bind_failed ... reason=missing_runtime`
|
||||
- worker logs `event=jti_replay_rejected`
|
||||
|
||||
This smoke intentionally does not route Windows client traffic through the
|
||||
direct worker WSS endpoint. The backend gateway remains the runtime path until
|
||||
DP-1D.
|
||||
|
||||
## Data Plane v1D.1 direct JSON bridge smoke status
|
||||
|
||||
Build the DP-1D.1 worker image on the test Docker host:
|
||||
|
||||
```powershell
|
||||
$env:DOCKER_BUILDKIT='0'
|
||||
docker --context test-ubuntu build --tag rap-rdp-worker:dp1d1 --file workers/rdp-worker/Dockerfile workers/rdp-worker
|
||||
```
|
||||
|
||||
Backend metadata must remain explicitly gated:
|
||||
|
||||
```text
|
||||
DATA_PLANE_DIRECT_WORKER_JSON_RUNTIME=false # default, client falls back
|
||||
DATA_PLANE_DIRECT_WORKER_JSON_RUNTIME=true # advertise runtime_transport=json_v1
|
||||
```
|
||||
|
||||
Locally/container-proven in DP-1D.1:
|
||||
|
||||
- backend tests prove direct candidate metadata appears only when
|
||||
`DATA_PLANE_DIRECT_WORKER_JSON_RUNTIME=true`
|
||||
- Windows client build still succeeds and remains capability-gated
|
||||
- worker Docker image builds with direct JSON envelope bridge
|
||||
- endpoint/token smoke still proves invalid token, missing runtime, and replay
|
||||
rejection
|
||||
|
||||
Still requiring live RDP smoke:
|
||||
|
||||
- direct WSS connects to an already-running runtime
|
||||
- direct WSS carries input/render/clipboard/file_upload JSON envelopes
|
||||
- direct WSS does not recreate the RDP runtime
|
||||
- backend gateway fallback activates when direct WSS is unavailable
|
||||
- direct vs fallback latency comparison on the same RDP target
|
||||
|
||||
## Prerequisites
|
||||
|
||||
- Docker Desktop or Docker Engine with `docker compose`
|
||||
- Go `1.23.x` for local backend runs
|
||||
- a reachable RDP host for the seeded resource
|
||||
- a machine where the worker Docker image can actually be built and started
|
||||
|
||||
## 0. Verify raw TCP reachability to the target first
|
||||
|
||||
Run this from the same machine or container host that will run the worker:
|
||||
|
||||
```powershell
|
||||
python scripts/smoke/check-rdp-target.py --host 192.168.60.210 --port 60210
|
||||
```
|
||||
|
||||
Expected result:
|
||||
|
||||
- `tcp_connect=ok ...`
|
||||
|
||||
If this step fails, FreeRDP connect proof is blocked by target reachability and the later lifecycle steps cannot be considered proven.
|
||||
|
||||
## Canonical build environments
|
||||
|
||||
- worker devcontainer: [`.devcontainer/devcontainer.json`](/\\?\UNC\192.168.220.200\mst\codex\rdp-proxy\.devcontainer\devcontainer.json)
|
||||
- worker Docker image: [workers/rdp-worker/Dockerfile](/\\?\UNC\192.168.220.200\mst\codex\rdp-proxy\workers\rdp-worker\Dockerfile)
|
||||
- worker CMake preset: [workers/rdp-worker/CMakePresets.json](/\\?\UNC\192.168.220.200\mst\codex\rdp-proxy\workers\rdp-worker\CMakePresets.json)
|
||||
|
||||
## 1. Start infra
|
||||
|
||||
```powershell
|
||||
pwsh -File scripts/smoke/start-infra.ps1
|
||||
```
|
||||
|
||||
Expected result:
|
||||
|
||||
- PostgreSQL is reachable on `127.0.0.1:5432`
|
||||
- Redis is reachable on `127.0.0.1:6379`
|
||||
|
||||
## 2. Apply backend migrations
|
||||
|
||||
```powershell
|
||||
pwsh -File scripts/smoke/apply-migrations.ps1
|
||||
```
|
||||
|
||||
Expected result:
|
||||
|
||||
- backend tables exist in `remote_access_platform`
|
||||
|
||||
## 3. Seed a smoke-test user, trusted device, resource, and policy
|
||||
|
||||
Edit the connection parameters to point to a reachable RDP host:
|
||||
|
||||
```powershell
|
||||
pwsh -File scripts/smoke/seed-resource.ps1 `
|
||||
-RdpHost 10.0.0.10 `
|
||||
-RdpPort 3389 `
|
||||
-RdpUsername Administrator `
|
||||
-RdpPassword secret `
|
||||
-RdpDomain "" `
|
||||
-CertificateVerificationMode strict
|
||||
```
|
||||
|
||||
Expected result:
|
||||
|
||||
- the script creates or reuses the `default` organization created by the v2 migrations
|
||||
- the script creates an active default-organization membership for the seeded smoke user
|
||||
- the script prints `user_id`
|
||||
- the script prints `device_id`
|
||||
- the script prints `resource_id`
|
||||
|
||||
## 4. Start backend
|
||||
|
||||
```powershell
|
||||
pwsh -File scripts/smoke/run-backend.ps1
|
||||
```
|
||||
|
||||
Expected result:
|
||||
|
||||
- backend listens on `http://192.168.200.61:8080` from the Windows client and on `127.0.0.1:8080` inside the Docker host network
|
||||
|
||||
Containerized fallback when the smoke host does not have `go` installed:
|
||||
|
||||
```sh
|
||||
docker run -d --name rap_backend_smoke --network host \
|
||||
-v /absolute/path/to/repo/backend:/workspace/backend \
|
||||
-w /workspace/backend \
|
||||
-e APP_NAME=rap-api \
|
||||
-e APP_ENV=development \
|
||||
-e HTTP_HOST=0.0.0.0 \
|
||||
-e HTTP_PORT=8080 \
|
||||
-e POSTGRES_DSN=postgres://rap_user:rap_password@127.0.0.1:5432/remote_access_platform?sslmode=disable \
|
||||
-e REDIS_ADDR=127.0.0.1:6379 \
|
||||
-e AUTH_ACCESS_TOKEN_SECRET=smoke-access-secret \
|
||||
-e AUTH_REFRESH_HASH_SECRET=smoke-refresh-secret \
|
||||
golang:1.23.8-bookworm /bin/sh -lc '/usr/local/go/bin/go run ./cmd/api'
|
||||
```
|
||||
|
||||
## 5. Build the worker image
|
||||
|
||||
```powershell
|
||||
pwsh -File scripts/smoke/build-worker-image.ps1
|
||||
docker run --rm --entrypoint /bin/sh rap-rdp-worker:dev -lc "test -x /usr/local/bin/rdp-worker"
|
||||
```
|
||||
|
||||
Expected result:
|
||||
|
||||
- image build succeeds
|
||||
- the binary exists at `/usr/local/bin/rdp-worker`
|
||||
|
||||
## 6. Run the worker container
|
||||
|
||||
```powershell
|
||||
pwsh -File scripts/smoke/run-worker-container.ps1
|
||||
```
|
||||
|
||||
Expected result:
|
||||
|
||||
- the worker process starts
|
||||
- Redis contains `worker:registration:rdp-worker-1`
|
||||
|
||||
If the test RDP host uses a self-signed or mismatched certificate and smoke verification needs to continue, set:
|
||||
|
||||
```text
|
||||
RDP_WORKER_INSECURE_SKIP_VERIFY=true
|
||||
```
|
||||
|
||||
This override is worker-runtime-only and is intended strictly for smoke verification.
|
||||
|
||||
## 7. Start a session
|
||||
|
||||
```powershell
|
||||
$body = @{
|
||||
resource_id = "<resource_id>"
|
||||
user_id = "<user_id>"
|
||||
device_id = "<device_id>"
|
||||
} | ConvertTo-Json
|
||||
|
||||
Invoke-RestMethod `
|
||||
-Method Post `
|
||||
-Uri http://192.168.200.61:8080/api/v1/sessions `
|
||||
-ContentType 'application/json' `
|
||||
-Body $body
|
||||
```
|
||||
|
||||
Expected result:
|
||||
|
||||
- the response includes `session.id`
|
||||
- the response includes `attachment.id`
|
||||
- the response includes `attach_token`
|
||||
- the response session payload carries `organization_id`
|
||||
- backend logs a session start and assignment path
|
||||
- worker logs a new assignment and FreeRDP connect attempt
|
||||
- Redis `worker:events` emits `session_connected` and then periodic `session_heartbeat`
|
||||
|
||||
## 8. Detach
|
||||
|
||||
```powershell
|
||||
$body = @{
|
||||
attachment_id = "<attachment_id>"
|
||||
user_id = "<user_id>"
|
||||
reason = "manual_smoke_detach"
|
||||
} | ConvertTo-Json
|
||||
|
||||
Invoke-RestMethod `
|
||||
-Method Post `
|
||||
-Uri http://192.168.200.61:8080/api/v1/sessions/<session_id>/detach `
|
||||
-ContentType 'application/json' `
|
||||
-Body $body
|
||||
```
|
||||
|
||||
Expected result:
|
||||
|
||||
- PostgreSQL session state becomes `detached`
|
||||
- worker keeps the RDP connection alive
|
||||
- worker does not emit `session_terminated`
|
||||
|
||||
## 9. Reattach
|
||||
|
||||
```powershell
|
||||
$body = @{
|
||||
user_id = "<user_id>"
|
||||
device_id = "<device_id>"
|
||||
} | ConvertTo-Json
|
||||
|
||||
Invoke-RestMethod `
|
||||
-Method Post `
|
||||
-Uri http://192.168.200.61:8080/api/v1/sessions/<session_id>/attach `
|
||||
-ContentType 'application/json' `
|
||||
-Body $body
|
||||
```
|
||||
|
||||
Expected result:
|
||||
|
||||
- backend returns a new short-lived `attach_token`
|
||||
- worker does not recreate the remote RDP session
|
||||
- worker continues heartbeating the same `session_id`
|
||||
|
||||
## 10. Takeover
|
||||
|
||||
Create or seed a second trusted device for the same user, then:
|
||||
|
||||
```powershell
|
||||
$body = @{
|
||||
user_id = "<user_id>"
|
||||
device_id = "<second_device_id>"
|
||||
reason = "manual_smoke_takeover"
|
||||
} | ConvertTo-Json
|
||||
|
||||
Invoke-RestMethod `
|
||||
-Method Post `
|
||||
-Uri http://192.168.200.61:8080/api/v1/sessions/<session_id>/takeover `
|
||||
-ContentType 'application/json' `
|
||||
-Body $body
|
||||
```
|
||||
|
||||
Expected result:
|
||||
|
||||
- backend atomically supersedes the previous attachment
|
||||
- previous controller WebSocket session receives `session.taken_over` if connected
|
||||
- worker stays on the same remote RDP session and only updates controller ownership
|
||||
|
||||
## 10A. Prove WebSocket takeover delivery
|
||||
|
||||
Use the real smoke client built into the backend module:
|
||||
|
||||
```sh
|
||||
cd backend
|
||||
go run ./cmd/ws-smoke-client \
|
||||
-attach-token "<controller_a_attach_token>" \
|
||||
-duration 120s
|
||||
```
|
||||
|
||||
Expected result:
|
||||
|
||||
- controller A first receives `session.state`
|
||||
- after takeover from controller B, controller A receives `session.taken_over`
|
||||
- PostgreSQL shows the new attachment for controller B as `active`
|
||||
- worker logs only `updated assignment for existing session ...` and does not log a new runtime start
|
||||
|
||||
## 11. Terminate
|
||||
|
||||
```powershell
|
||||
$body = @{
|
||||
user_id = "<user_id>"
|
||||
reason = "manual_smoke_terminate"
|
||||
} | ConvertTo-Json
|
||||
|
||||
Invoke-RestMethod `
|
||||
-Method Post `
|
||||
-Uri http://192.168.200.61:8080/api/v1/sessions/<session_id>/terminate `
|
||||
-ContentType 'application/json' `
|
||||
-Body $body
|
||||
```
|
||||
|
||||
Expected result:
|
||||
|
||||
- backend marks the session `terminated`
|
||||
- worker receives a `terminate` control envelope
|
||||
- worker disconnects the FreeRDP session and emits `session_terminated`
|
||||
|
||||
## 12. Prove stale lease and worker-death recovery
|
||||
|
||||
With a live active session still running:
|
||||
|
||||
```sh
|
||||
docker rm -f rap_worker_smoke
|
||||
```
|
||||
|
||||
Wait at least:
|
||||
|
||||
- `WORKER_HEARTBEAT_TTL`
|
||||
- plus `WORKER_STALE_LEASE_GRACE_PERIOD`
|
||||
- plus one lease-monitor interval
|
||||
|
||||
With current defaults from [backend/configs/api.example.env](/\\?\UNC\192.168.220.200\mst\codex\rdp-proxy\backend\configs\api.example.env), waiting about `90s` is sufficient for a manual smoke pass.
|
||||
|
||||
Expected result:
|
||||
|
||||
- `worker:registration:<worker_id>` disappears from Redis
|
||||
- `worker:session-lease:<session_id>` is released
|
||||
- `live:session:<session_id>`, `live:binding:<session_id>`, and `live:route:<session_id>` are cleared
|
||||
- PostgreSQL moves the session to `failed`
|
||||
- non-superseded attachments become `closed`
|
||||
- `audit_events` contains `session_failed` with reason `worker_lease_stale_or_worker_missing`
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
- If the worker image build fails, run `docker build --tag rap-rdp-worker:dev --file workers/rdp-worker/Dockerfile workers/rdp-worker` directly and inspect the compiler output.
|
||||
- If raw TCP reachability to the RDP target fails, stop there and fix host/network/firewall/port access before evaluating FreeRDP behavior.
|
||||
- If the worker starts but never receives assignments, verify `worker:registration:<worker_id>` and `worker:control:<worker_id>` in Redis.
|
||||
- If session start returns `access denied` after the v2 migrations, verify that the seeded user has an active membership in the resource organization.
|
||||
- If takeover does not produce `session.taken_over`, confirm that controller A is attached through `/api/v1/gateway/ws` using a still-valid attach token and that the broker binding changed.
|
||||
- If worker death does not transition the session quickly enough, verify `WORKER_HEARTBEAT_TTL`, `WORKER_STALE_LEASE_GRACE_PERIOD`, and the lease monitor interval before treating the session as stuck.
|
||||
- If FreeRDP cannot connect, verify the seeded host, port, username, password, domain, certificate verification mode, and network reachability from Docker to the target host.
|
||||
- If attach tokens expire during manual testing, repeat the attach or takeover call to mint a new token.
|
||||
|
||||
## Stage 5.1 / 5.1.1 File Upload Smoke
|
||||
|
||||
Current target Docker host for this project is `192.168.200.61` via Docker
|
||||
context `test-ubuntu`. Verify before running:
|
||||
|
||||
```powershell
|
||||
docker context use test-ubuntu
|
||||
docker ps
|
||||
```
|
||||
|
||||
Policy setup:
|
||||
|
||||
```sql
|
||||
UPDATE resource_policies
|
||||
SET file_transfer_mode = 'client_to_server',
|
||||
file_transfer_enabled = TRUE,
|
||||
updated_at = now()
|
||||
WHERE resource_id = '<resource_id>';
|
||||
```
|
||||
|
||||
Disabled-policy regression:
|
||||
|
||||
```sql
|
||||
UPDATE resource_policies
|
||||
SET file_transfer_mode = 'disabled',
|
||||
file_transfer_enabled = FALSE,
|
||||
updated_at = now()
|
||||
WHERE resource_id = '<resource_id>';
|
||||
```
|
||||
|
||||
Manual upload proof from the Windows client:
|
||||
|
||||
- start or attach an active RDP session
|
||||
- open the session window
|
||||
- click `Upload File`
|
||||
- choose a small text file, then a small binary file
|
||||
- verify the UI progress reaches 100%
|
||||
- inspect backend logs for `session gateway file upload start accepted` and
|
||||
`session gateway file upload chunk accepted`
|
||||
- inspect worker logs for `file upload completed`
|
||||
- verify the file and hash inside the worker container:
|
||||
|
||||
```powershell
|
||||
docker exec rap_worker_smoke sh -lc `
|
||||
"find /tmp/rap-rdp-worker-transfers -path '*/visible/*' -type f -maxdepth 4 -print -exec wc -c {} \;"
|
||||
```
|
||||
|
||||
Stage 5.1.1 visibility proof:
|
||||
|
||||
Automated smoke command used for the accepted proof:
|
||||
|
||||
```powershell
|
||||
pwsh -ExecutionPolicy Bypass -File scripts\smoke\drive-visibility-smoke.ps1 `
|
||||
-WorkerImage rap-rdp-worker:rdp-p1-region-order2 `
|
||||
-OutputFrame artifacts\stage5-drive-visibility-frame-p1-rerun.bmp
|
||||
```
|
||||
|
||||
- worker logs must show `visible transfer directory ready`
|
||||
- worker logs must show `FreeRDP restricted transfer drive configured name=RAP_Transfers`
|
||||
- inside the remote Windows session, open File Explorer and verify the
|
||||
redirected drive `RAP_Transfers` is present
|
||||
- verify uploaded files are visible under `RAP_Transfers`
|
||||
- open the uploaded text file and verify the content matches
|
||||
- for binary files, verify size and hash through worker storage evidence unless
|
||||
a remote-side hash tool is available
|
||||
- after detach, takeover old-client, and worker failure, upload must be blocked
|
||||
and the visible transfer directory must be cleaned up
|
||||
|
||||
Required PASS cases for accepting Stage 5.1:
|
||||
|
||||
- `disabled` blocks upload
|
||||
- `client_to_server` allows upload
|
||||
- small text file hash matches
|
||||
- small binary file hash matches
|
||||
- file larger than 25 MiB is blocked by client/gateway policy
|
||||
- path traversal names are blocked by gateway/worker validation
|
||||
- upload is blocked after detach, old-client takeover, and worker failure
|
||||
- rendering, mouse input, keyboard input, clipboard, reconnect, and takeover
|
||||
still work
|
||||
|
||||
Accepted Stage 5.1.1 proof artifact:
|
||||
|
||||
- `artifacts/stage5-drive-visibility-frame-p1-rerun.bmp` shows the uploaded
|
||||
`stage5-upload-text.txt` opened inside remote Windows from the restricted
|
||||
`RAP_Transfers` drive.
|
||||
|
||||
Important limitation for Stage 5.1.1: it intentionally exposed only the
|
||||
restricted per-session `visible` directory as `RAP_Transfers`. It must not be
|
||||
expanded to arbitrary paths, full shared folders, SMB/WebDAV, or Windows agent
|
||||
delivery.
|
||||
|
||||
## Stage 5.2 File Download Smoke
|
||||
|
||||
Stage 5.2 server-to-client download has a runtime-proven core data path and
|
||||
lifecycle blocking proof. Manual desktop UI proof remains before full
|
||||
acceptance.
|
||||
|
||||
Build-proven images:
|
||||
|
||||
```text
|
||||
rap-backend-smoke:stage5-2-download
|
||||
rap-rdp-worker:stage5-2-download
|
||||
```
|
||||
|
||||
Headless core data-path proof:
|
||||
|
||||
```powershell
|
||||
pwsh -NoProfile -ExecutionPolicy Bypass `
|
||||
-File scripts\smoke\file-download-smoke.ps1 `
|
||||
-AllowMode server_to_client `
|
||||
-Transport direct_worker_wss `
|
||||
-OutputDirectory artifacts/stage5-2-download-smoke-direct-fixed2
|
||||
|
||||
pwsh -NoProfile -ExecutionPolicy Bypass `
|
||||
-File scripts\smoke\file-download-smoke.ps1 `
|
||||
-AllowMode bidirectional `
|
||||
-Transport direct_worker_wss `
|
||||
-OutputDirectory artifacts/stage5-2-download-smoke-direct-bidirectional
|
||||
|
||||
pwsh -NoProfile -ExecutionPolicy Bypass `
|
||||
-File scripts\smoke\file-download-smoke.ps1 `
|
||||
-AllowMode client_to_server `
|
||||
-Transport direct_worker_wss `
|
||||
-ExpectBlocked `
|
||||
-OutputDirectory artifacts/stage5-2-download-smoke-direct-client-to-server-block-fixed
|
||||
|
||||
pwsh -NoProfile -ExecutionPolicy Bypass `
|
||||
-File scripts\smoke\file-download-smoke.ps1 `
|
||||
-AllowMode disabled `
|
||||
-Transport direct_worker_wss `
|
||||
-ExpectBlocked `
|
||||
-OutputDirectory artifacts/stage5-2-download-smoke-direct-disabled-fixed
|
||||
|
||||
pwsh -NoProfile -ExecutionPolicy Bypass `
|
||||
-File scripts\smoke\file-download-smoke.ps1 `
|
||||
-AllowMode server_to_client `
|
||||
-Transport backend_gateway `
|
||||
-OutputDirectory artifacts/stage5-2-download-smoke-backend-regression-after-direct-block
|
||||
```
|
||||
|
||||
Lifecycle proof:
|
||||
|
||||
```powershell
|
||||
pwsh -NoProfile -ExecutionPolicy Bypass `
|
||||
-File scripts\smoke\file-download-smoke.ps1 `
|
||||
-AllowMode server_to_client `
|
||||
-Transport direct_worker_wss `
|
||||
-LifecycleScenario detach `
|
||||
-OutputDirectory artifacts/stage5-2-download-lifecycle-detach-fixed
|
||||
|
||||
pwsh -NoProfile -ExecutionPolicy Bypass `
|
||||
-File scripts\smoke\file-download-smoke.ps1 `
|
||||
-AllowMode server_to_client `
|
||||
-Transport direct_worker_wss `
|
||||
-LifecycleScenario takeover_old_controller `
|
||||
-OutputDirectory artifacts/stage5-2-download-lifecycle-takeover-fixed
|
||||
|
||||
pwsh -NoProfile -ExecutionPolicy Bypass `
|
||||
-File scripts\smoke\file-download-smoke.ps1 `
|
||||
-AllowMode server_to_client `
|
||||
-Transport direct_worker_wss `
|
||||
-LifecycleScenario worker_failure `
|
||||
-OutputDirectory artifacts/stage5-2-download-lifecycle-worker-failure
|
||||
```
|
||||
|
||||
Accepted core evidence:
|
||||
|
||||
- direct worker WSS `server_to_client`: text and binary size/hash match
|
||||
- direct worker WSS `bidirectional`: text and binary download succeeds
|
||||
- direct worker WSS `client_to_server`: download blocked with `access denied`
|
||||
- direct worker WSS `disabled`: download blocked with `access denied`
|
||||
- backend gateway fallback `server_to_client`: text and binary size/hash match
|
||||
- detach blocks download with `file_download.blocked`
|
||||
- old controller after takeover receives `session.taken_over` and cannot
|
||||
continue download
|
||||
- worker failure transitions PostgreSQL state to `failed`; direct WebSocket
|
||||
closes and download cannot continue
|
||||
|
||||
Report:
|
||||
|
||||
- `artifacts/stage5-2-file-download-runtime-report.md`
|
||||
|
||||
Remaining manual live proof:
|
||||
|
||||
- keep the Stage 5.2 backend and worker images on `docker-test`
|
||||
- set `resource_policies.file_transfer_mode = 'server_to_client'`
|
||||
- start or attach a real RDP session
|
||||
- inside remote Windows, copy a small text file to `RAP_Transfers\ToClient`
|
||||
- verify the Windows client shows `file_download.available`
|
||||
- click `Download File`, choose a local save path, and verify completion
|
||||
- compare size and hash with worker evidence
|
||||
- repeat with a small binary file
|
||||
- verify `disabled` and `client_to_server` block download
|
||||
- verify `bidirectional` allows upload and download
|
||||
- verify rendering, mouse, keyboard, clipboard, upload, reconnect, takeover,
|
||||
and backend gateway fallback do not regress
|
||||
|
||||
@@ -0,0 +1,9 @@
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
$migrationFiles = Get-ChildItem -Path "backend/migrations" -Filter "*.up.sql" | Sort-Object Name
|
||||
|
||||
foreach ($migration in $migrationFiles) {
|
||||
Write-Host "Applying $($migration.Name)..."
|
||||
Get-Content -Raw $migration.FullName | docker exec -i rap_postgres psql -U rap_user -d remote_access_platform -v ON_ERROR_STOP=1 -f -
|
||||
}
|
||||
@@ -0,0 +1,5 @@
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
docker build -t rap-rdp-worker:dev -f workers/rdp-worker/Dockerfile workers/rdp-worker
|
||||
docker run --rm --entrypoint /bin/sh rap-rdp-worker:dev -lc "test -x /usr/local/bin/rdp-worker"
|
||||
@@ -0,0 +1,30 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import argparse
|
||||
import socket
|
||||
import sys
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Check raw TCP reachability to an RDP target.")
|
||||
parser.add_argument("--host", required=True, help="RDP host or IP")
|
||||
parser.add_argument("--port", required=True, type=int, help="RDP TCP port")
|
||||
parser.add_argument("--timeout", type=float, default=5.0, help="connect timeout seconds")
|
||||
args = parser.parse_args()
|
||||
|
||||
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
||||
sock.settimeout(args.timeout)
|
||||
try:
|
||||
sock.connect((args.host, args.port))
|
||||
except OSError as exc:
|
||||
print(f"tcp_connect=failed host={args.host} port={args.port} error={exc}")
|
||||
return 1
|
||||
finally:
|
||||
sock.close()
|
||||
|
||||
print(f"tcp_connect=ok host={args.host} port={args.port}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,84 @@
|
||||
param(
|
||||
[string]$DockerSshAlias = "docker-test",
|
||||
[string]$WorkerImage = "rap-rdp-worker:dp1c-hardened",
|
||||
[string]$WorkerID = "rdp-worker-1",
|
||||
[int]$Port = 18443
|
||||
)
|
||||
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
function ConvertTo-Base64Url([byte[]]$Bytes) {
|
||||
return [Convert]::ToBase64String($Bytes).TrimEnd("=").Replace("+", "-").Replace("/", "_")
|
||||
}
|
||||
|
||||
function New-DataPlaneJwt([hashtable]$Payload) {
|
||||
$encoding = [Text.Encoding]::UTF8
|
||||
$headerJson = @{ alg = "RS256"; typ = "JWT" } | ConvertTo-Json -Compress
|
||||
$payloadJson = $Payload | ConvertTo-Json -Compress -Depth 6
|
||||
$body = "$(ConvertTo-Base64Url $encoding.GetBytes($headerJson)).$(ConvertTo-Base64Url $encoding.GetBytes($payloadJson))"
|
||||
$signature = ConvertTo-Base64Url $script:DataPlaneRsa.SignData(
|
||||
$encoding.GetBytes($body),
|
||||
[System.Security.Cryptography.HashAlgorithmName]::SHA256,
|
||||
[System.Security.Cryptography.RSASignaturePadding]::Pkcs1)
|
||||
return "$body.$signature"
|
||||
}
|
||||
|
||||
function ConvertTo-Pem([string]$Label, [byte[]]$Bytes) {
|
||||
$base64 = [Convert]::ToBase64String($Bytes)
|
||||
$lines = for ($i = 0; $i -lt $base64.Length; $i += 64) {
|
||||
$base64.Substring($i, [Math]::Min(64, $base64.Length - $i))
|
||||
}
|
||||
$joined = $lines -join "`n"
|
||||
return "-----BEGIN $Label-----`n$joined`n-----END $Label-----`n"
|
||||
}
|
||||
|
||||
$now = [int][DateTimeOffset]::UtcNow.ToUnixTimeSeconds()
|
||||
$script:DataPlaneRsa = [System.Security.Cryptography.RSA]::Create(2048)
|
||||
$publicKeyPem = ConvertTo-Pem "PUBLIC KEY" $script:DataPlaneRsa.ExportSubjectPublicKeyInfo()
|
||||
$publicKeyB64 = [Convert]::ToBase64String([Text.Encoding]::UTF8.GetBytes($publicKeyPem))
|
||||
$validPayload = [ordered]@{
|
||||
session_id = "session-without-runtime"
|
||||
attachment_id = "attachment-1"
|
||||
user_id = "user-1"
|
||||
organization_id = "org-1"
|
||||
worker_id = $WorkerID
|
||||
resource_id = "resource-1"
|
||||
allowed_channels = @("control", "input", "render")
|
||||
jti = [guid]::NewGuid().ToString()
|
||||
aud = @("rap-data-plane", "worker:$WorkerID")
|
||||
iat = $now
|
||||
nbf = $now
|
||||
exp = $now + 300
|
||||
}
|
||||
$validToken = New-DataPlaneJwt $validPayload
|
||||
|
||||
$remoteScript = @"
|
||||
set -eu
|
||||
rm -rf /tmp/rap-dp1c && mkdir -p /tmp/rap-dp1c
|
||||
openssl req -x509 -newkey rsa:2048 -nodes -keyout /tmp/rap-dp1c/key.pem -out /tmp/rap-dp1c/cert.pem -subj '/CN=localhost' -days 1 >/tmp/rap-dp1c/openssl.log 2>&1
|
||||
printf '%s' '$publicKeyB64' | base64 -d >/tmp/rap-dp1c/dp-public.pem
|
||||
(docker rm -f rap_worker_dp1c_probe >/dev/null 2>&1 || true)
|
||||
docker run -d --name rap_worker_dp1c_probe --network rdp-proxy_default -p ${Port}:${Port} -v /tmp/rap-dp1c:/certs:ro \
|
||||
-e RDP_WORKER_ID=$WorkerID \
|
||||
-e RDP_WORKER_REDIS_HOST=rap_redis \
|
||||
-e RDP_WORKER_DATA_PLANE_ENABLED=true \
|
||||
-e RDP_WORKER_DATA_PLANE_LISTEN_HOST=0.0.0.0 \
|
||||
-e RDP_WORKER_DATA_PLANE_LISTEN_PORT=$Port \
|
||||
-e RDP_WORKER_DATA_PLANE_PUBLIC_KEY_FILE=/certs/dp-public.pem \
|
||||
-e RDP_WORKER_DATA_PLANE_TLS_CERT_FILE=/certs/cert.pem \
|
||||
-e RDP_WORKER_DATA_PLANE_TLS_KEY_FILE=/certs/key.pem \
|
||||
$WorkerImage >/tmp/rap-dp1c/container.id
|
||||
sleep 2
|
||||
printf 'invalid token response:\n'
|
||||
printf 'GET /rap/v1/data-plane?data_plane_token=bad HTTP/1.1\r\nHost: localhost\r\nUpgrade: websocket\r\nConnection: Upgrade\r\nSec-WebSocket-Key: x3JJHMbDL1EzLkh9GBhXDw==\r\nSec-WebSocket-Version: 13\r\n\r\n' | timeout 5 openssl s_client -connect 127.0.0.1:$Port -servername localhost -quiet 2>/dev/null | head -20
|
||||
printf '\nvalid token without runtime response:\n'
|
||||
printf 'GET /rap/v1/data-plane?data_plane_token=$validToken HTTP/1.1\r\nHost: localhost\r\nUpgrade: websocket\r\nConnection: Upgrade\r\nSec-WebSocket-Key: x3JJHMbDL1EzLkh9GBhXDw==\r\nSec-WebSocket-Version: 13\r\n\r\n' | timeout 5 openssl s_client -connect 127.0.0.1:$Port -servername localhost -quiet 2>/dev/null | head -20
|
||||
printf '\nreplayed jti response:\n'
|
||||
printf 'GET /rap/v1/data-plane?data_plane_token=$validToken HTTP/1.1\r\nHost: localhost\r\nUpgrade: websocket\r\nConnection: Upgrade\r\nSec-WebSocket-Key: x3JJHMbDL1EzLkh9GBhXDw==\r\nSec-WebSocket-Version: 13\r\n\r\n' | timeout 5 openssl s_client -connect 127.0.0.1:$Port -servername localhost -quiet 2>/dev/null | head -20
|
||||
printf '\nworker logs:\n'
|
||||
docker logs rap_worker_dp1c_probe --tail 30
|
||||
docker rm -f rap_worker_dp1c_probe >/dev/null 2>&1 || true
|
||||
rm -rf /tmp/rap-dp1c
|
||||
"@
|
||||
|
||||
ssh $DockerSshAlias $remoteScript
|
||||
@@ -0,0 +1,260 @@
|
||||
param(
|
||||
[string]$BackendApiBase = "http://192.168.200.61:8080/api/v1",
|
||||
[string]$BackendWsBase = "ws://192.168.200.61:8080/api/v1/gateway/ws",
|
||||
[string]$DockerSshAlias = "docker-test",
|
||||
[string]$WorkerContainerName = "rap_worker_smoke",
|
||||
[string]$WorkerImage = "rap-rdp-worker:stage5-drive-visible",
|
||||
[string]$ResourceName = "Windows Smoke Default Resource",
|
||||
[string]$Email = "windows-smoke@example.local",
|
||||
[string]$Password = "SmokePass!123",
|
||||
[string]$OutputFrame = "artifacts/stage5-drive-visibility-frame.bmp"
|
||||
)
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
function Write-SmokeLog([string]$Message) {
|
||||
Write-Host "[$(Get-Date -Format o)] $Message"
|
||||
}
|
||||
|
||||
function Invoke-RemoteSqlScalar([string]$Sql) {
|
||||
$encoded = [Convert]::ToBase64String([Text.Encoding]::UTF8.GetBytes($Sql))
|
||||
$output = ssh $DockerSshAlias "printf '%s' '$encoded' | base64 -d | docker exec -i rap_postgres psql -U rap_user -d remote_access_platform -t -A -v ON_ERROR_STOP=1"
|
||||
return (($output | Select-Object -Last 1) -as [string]).Trim()
|
||||
}
|
||||
|
||||
function Invoke-Api([string]$Method, [string]$Path, [object]$Body = $null) {
|
||||
$uri = "$BackendApiBase/$Path"
|
||||
if ($null -eq $Body) {
|
||||
return Invoke-RestMethod -Method $Method -Uri $uri
|
||||
}
|
||||
return Invoke-RestMethod -Method $Method -Uri $uri -ContentType "application/json" -Body ($Body | ConvertTo-Json -Depth 16)
|
||||
}
|
||||
|
||||
function Restart-SmokeWorker {
|
||||
$cmd = "docker rm -f $WorkerContainerName 2>/dev/null || true; docker run -d --name $WorkerContainerName --network host -e RDP_WORKER_ID=rdp-worker-1 -e RDP_WORKER_REDIS_HOST=127.0.0.1 -e RDP_WORKER_REDIS_PORT=6379 -e RDP_WORKER_TRANSFER_ROOT=/tmp/rap-rdp-worker-transfers -e RDP_WORKER_CAPABILITIES=adaptive-quality,dirty-rects,clipboard,file-transfer $WorkerImage"
|
||||
$output = ssh $DockerSshAlias $cmd
|
||||
Write-SmokeLog "worker restart output=$($output -join ' ')"
|
||||
Start-Sleep -Seconds 5
|
||||
}
|
||||
|
||||
function New-GatewayWebSocket([string]$AttachToken) {
|
||||
$ws = [Net.WebSockets.ClientWebSocket]::new()
|
||||
$uri = [Uri]::new("${BackendWsBase}?attach_token=$([Uri]::EscapeDataString($AttachToken))")
|
||||
$null = $ws.ConnectAsync($uri, [Threading.CancellationToken]::None).GetAwaiter().GetResult()
|
||||
return $ws
|
||||
}
|
||||
|
||||
function Receive-Envelope([Net.WebSockets.ClientWebSocket]$WebSocket, [int]$TimeoutMilliseconds = 30000) {
|
||||
$buffer = [byte[]]::new(1024 * 1024)
|
||||
$segment = [ArraySegment[byte]]::new($buffer)
|
||||
$stream = [IO.MemoryStream]::new()
|
||||
while ($true) {
|
||||
$cts = [Threading.CancellationTokenSource]::new($TimeoutMilliseconds)
|
||||
try {
|
||||
$result = $WebSocket.ReceiveAsync($segment, $cts.Token).GetAwaiter().GetResult()
|
||||
}
|
||||
finally {
|
||||
$cts.Dispose()
|
||||
}
|
||||
if ($result.MessageType -eq [Net.WebSockets.WebSocketMessageType]::Close) {
|
||||
throw "websocket closed"
|
||||
}
|
||||
$stream.Write($buffer, 0, $result.Count)
|
||||
if ($result.EndOfMessage) {
|
||||
break
|
||||
}
|
||||
}
|
||||
return ([Text.Encoding]::UTF8.GetString($stream.ToArray()) | ConvertFrom-Json)
|
||||
}
|
||||
|
||||
function Receive-EnvelopeOfType([Net.WebSockets.ClientWebSocket]$WebSocket, [string[]]$Types, [int]$TimeoutSeconds = 30) {
|
||||
$deadline = (Get-Date).AddSeconds($TimeoutSeconds)
|
||||
while ((Get-Date) -lt $deadline) {
|
||||
$envelope = Receive-Envelope -WebSocket $WebSocket -TimeoutMilliseconds 10000
|
||||
if ($Types -contains [string]$envelope.type) {
|
||||
return $envelope
|
||||
}
|
||||
}
|
||||
throw "timed out waiting for envelope type: $($Types -join ',')"
|
||||
}
|
||||
|
||||
function Send-Envelope([Net.WebSockets.ClientWebSocket]$WebSocket, [object]$Envelope) {
|
||||
$json = $Envelope | ConvertTo-Json -Depth 16 -Compress
|
||||
$bytes = [Text.Encoding]::UTF8.GetBytes($json)
|
||||
$null = $WebSocket.SendAsync([ArraySegment[byte]]::new($bytes), [Net.WebSockets.WebSocketMessageType]::Text, $true, [Threading.CancellationToken]::None).GetAwaiter().GetResult()
|
||||
}
|
||||
|
||||
function Wait-SessionActive([Net.WebSockets.ClientWebSocket]$WebSocket) {
|
||||
$deadline = (Get-Date).AddSeconds(90)
|
||||
while ((Get-Date) -lt $deadline) {
|
||||
$envelope = Receive-Envelope -WebSocket $WebSocket -TimeoutMilliseconds 30000
|
||||
Write-SmokeLog "ws event type=$($envelope.type) state=$($envelope.payload.state)"
|
||||
if ($envelope.type -eq "session.state" -and $envelope.payload.state -eq "active") {
|
||||
return
|
||||
}
|
||||
}
|
||||
throw "session did not become active"
|
||||
}
|
||||
|
||||
function Send-Key([Net.WebSockets.ClientWebSocket]$WebSocket, [int]$ScanCode, [bool]$Extended = $false) {
|
||||
Send-Envelope $WebSocket @{ type = "input"; payload = @{ kind = "keyboard"; action = "key_down"; scan_code = $ScanCode; is_extended = $Extended } }
|
||||
Start-Sleep -Milliseconds 60
|
||||
Send-Envelope $WebSocket @{ type = "input"; payload = @{ kind = "keyboard"; action = "key_up"; scan_code = $ScanCode; is_extended = $Extended } }
|
||||
}
|
||||
|
||||
function Send-Chord([Net.WebSockets.ClientWebSocket]$WebSocket, [int]$ModifierScanCode, [bool]$ModifierExtended, [int]$KeyScanCode) {
|
||||
Send-Envelope $WebSocket @{ type = "input"; payload = @{ kind = "keyboard"; action = "key_down"; scan_code = $ModifierScanCode; is_extended = $ModifierExtended } }
|
||||
Start-Sleep -Milliseconds 80
|
||||
Send-Key $WebSocket $KeyScanCode $false
|
||||
Start-Sleep -Milliseconds 80
|
||||
Send-Envelope $WebSocket @{ type = "input"; payload = @{ kind = "keyboard"; action = "key_up"; scan_code = $ModifierScanCode; is_extended = $ModifierExtended } }
|
||||
}
|
||||
|
||||
function Upload-TextFile([Net.WebSockets.ClientWebSocket]$WebSocket) {
|
||||
$bytes = [Text.Encoding]::UTF8.GetBytes("Stage 5.1.1 RAP_Transfers visible file`r`nрусский текст`r`n")
|
||||
$transferId = [guid]::NewGuid().ToString("D")
|
||||
Send-Envelope $WebSocket @{
|
||||
type = "file_upload.start"
|
||||
payload = @{
|
||||
direction = "client_to_server"
|
||||
transfer_id = $transferId
|
||||
file_name = "stage5-upload-text.txt"
|
||||
file_size = [int64]$bytes.Length
|
||||
total_chunks = 1
|
||||
content_hash = ""
|
||||
}
|
||||
}
|
||||
$started = Receive-EnvelopeOfType $WebSocket @("file_upload.progress", "file_upload.blocked") 20
|
||||
if ($started.type -ne "file_upload.progress") {
|
||||
throw "upload start was blocked"
|
||||
}
|
||||
Send-Envelope $WebSocket @{
|
||||
type = "file_upload.chunk"
|
||||
payload = @{
|
||||
direction = "client_to_server"
|
||||
transfer_id = $transferId
|
||||
chunk_index = 0
|
||||
offset = 0
|
||||
chunk_size = $bytes.Length
|
||||
chunk_bytes = [Convert]::ToBase64String($bytes)
|
||||
}
|
||||
}
|
||||
$done = Receive-EnvelopeOfType $WebSocket @("file_upload.progress", "file_upload.blocked") 20
|
||||
Write-SmokeLog "upload result type=$($done.type) status=$($done.payload.status)"
|
||||
if ($done.type -ne "file_upload.progress" -or $done.payload.status -ne "completed") {
|
||||
throw "upload did not complete"
|
||||
}
|
||||
}
|
||||
|
||||
function Write-BgraFrameToBmp([string]$SessionId, [string]$Path) {
|
||||
$state = $null
|
||||
$data = $null
|
||||
$width = 0
|
||||
$height = 0
|
||||
$imageSize = 0
|
||||
$deadline = (Get-Date).AddSeconds(20)
|
||||
do {
|
||||
$json = ssh $DockerSshAlias "docker exec rap_redis redis-cli --raw GET live:session:$SessionId"
|
||||
if ($json) {
|
||||
$state = ($json -join "`n") | ConvertFrom-Json
|
||||
$width = [int]$state.render_width
|
||||
$height = [int]$state.render_height
|
||||
$stride = $width * 4
|
||||
$imageSize = $stride * $height
|
||||
if ($width -gt 0 -and $height -gt 0 -and -not [string]::IsNullOrWhiteSpace([string]$state.render_frame_data)) {
|
||||
$data = [Convert]::FromBase64String([string]$state.render_frame_data)
|
||||
if ($data.Length -ge $imageSize) {
|
||||
break
|
||||
}
|
||||
Write-SmokeLog "waiting for full framebuffer in live state bytes=$($data.Length) expected=$imageSize sequence=$($state.render_frame_sequence)"
|
||||
}
|
||||
}
|
||||
Start-Sleep -Milliseconds 500
|
||||
} while ((Get-Date) -lt $deadline)
|
||||
|
||||
if ($null -eq $data -or $data.Length -lt $imageSize) {
|
||||
$actualBytes = if ($null -eq $data) { 0 } else { $data.Length }
|
||||
throw "frame data too small or unavailable bytes=$actualBytes expected=$imageSize"
|
||||
}
|
||||
New-Item -ItemType Directory -Force -Path (Split-Path -Parent $Path) | Out-Null
|
||||
$fs = [IO.File]::Open($Path, [IO.FileMode]::Create, [IO.FileAccess]::Write)
|
||||
try {
|
||||
$writer = [IO.BinaryWriter]::new($fs)
|
||||
$fileSize = 14 + 40 + $imageSize
|
||||
$writer.Write([byte][char]'B')
|
||||
$writer.Write([byte][char]'M')
|
||||
$writer.Write([uint32]$fileSize)
|
||||
$writer.Write([uint16]0)
|
||||
$writer.Write([uint16]0)
|
||||
$writer.Write([uint32]54)
|
||||
$writer.Write([uint32]40)
|
||||
$writer.Write([int32]$width)
|
||||
$writer.Write([int32](-$height))
|
||||
$writer.Write([uint16]1)
|
||||
$writer.Write([uint16]32)
|
||||
$writer.Write([uint32]0)
|
||||
$writer.Write([uint32]$imageSize)
|
||||
$writer.Write([int32]2835)
|
||||
$writer.Write([int32]2835)
|
||||
$writer.Write([uint32]0)
|
||||
$writer.Write([uint32]0)
|
||||
$writer.Write($data, 0, $imageSize)
|
||||
}
|
||||
finally {
|
||||
$fs.Dispose()
|
||||
}
|
||||
Write-SmokeLog "wrote frame $Path"
|
||||
}
|
||||
|
||||
Restart-SmokeWorker
|
||||
|
||||
$login = Invoke-Api Post "auth/login" @{
|
||||
email = $Email
|
||||
password = $Password
|
||||
trust_device = $true
|
||||
device_fingerprint = "stage5-drive-visibility"
|
||||
device_label = "stage5-drive-visibility"
|
||||
}
|
||||
$userId = [string]$login.user.id
|
||||
if ([string]::IsNullOrWhiteSpace($userId)) { $userId = [string]$login.user.ID }
|
||||
$deviceId = [string]$login.device.id
|
||||
if ([string]::IsNullOrWhiteSpace($deviceId)) { $deviceId = [string]$login.device.ID }
|
||||
|
||||
$escapedResourceName = $ResourceName.Replace("'", "''")
|
||||
$resourceId = Invoke-RemoteSqlScalar "SELECT id::text FROM resources WHERE name = '$escapedResourceName' LIMIT 1;"
|
||||
$null = Invoke-RemoteSqlScalar "UPDATE resource_policies SET file_transfer_mode = 'client_to_server', file_transfer_enabled = TRUE, clipboard_mode = 'client_to_server', clipboard_enabled = TRUE, updated_at = NOW() WHERE resource_id = '$resourceId'; SELECT 'ok';"
|
||||
|
||||
$session = Invoke-Api Post "sessions" @{
|
||||
resource_id = $resourceId
|
||||
user_id = $userId
|
||||
device_id = $deviceId
|
||||
}
|
||||
$sessionId = [string]$session.session.id
|
||||
if ([string]::IsNullOrWhiteSpace($sessionId)) { $sessionId = [string]$session.session.ID }
|
||||
$attachToken = [string]$session.attach_token.token
|
||||
Write-SmokeLog "visibility session=$sessionId"
|
||||
$ws = New-GatewayWebSocket $attachToken
|
||||
try {
|
||||
Wait-SessionActive $ws
|
||||
Upload-TextFile $ws
|
||||
Start-Sleep -Seconds 2
|
||||
Send-Envelope $ws @{ type = "clipboard"; payload = @{ direction = "client_to_server"; text = "\\tsclient\RAP_Transfers\stage5-upload-text.txt"; sequence_id = [DateTimeOffset]::UtcNow.ToUnixTimeMilliseconds(); origin = "visibility-smoke"; content_hash = "" } }
|
||||
Start-Sleep -Seconds 1
|
||||
Send-Chord $ws 91 $true 19
|
||||
Start-Sleep -Seconds 1
|
||||
Send-Chord $ws 29 $false 47
|
||||
Start-Sleep -Milliseconds 300
|
||||
Send-Key $ws 28 $false
|
||||
Start-Sleep -Seconds 5
|
||||
Write-BgraFrameToBmp $sessionId $OutputFrame
|
||||
}
|
||||
finally {
|
||||
$ws.Dispose()
|
||||
try {
|
||||
Invoke-Api Post "sessions/$sessionId/terminate" @{ user_id = $userId; reason = "stage5_drive_visibility_cleanup" } | Out-Null
|
||||
}
|
||||
catch {
|
||||
Write-SmokeLog "terminate skipped: $($_.Exception.Message)"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,624 @@
|
||||
param(
|
||||
[string]$BackendApiBase = "http://192.168.200.61:8080/api/v1",
|
||||
[string]$BackendWsBase = "ws://192.168.200.61:8080/api/v1/gateway/ws",
|
||||
[string]$DockerSshAlias = "docker-test",
|
||||
[string]$WorkerContainerName = "rap_worker_smoke",
|
||||
[string]$ResourceName = "P3.3 Secret RDP Resource",
|
||||
[string]$UserEmail = "windows-smoke@example.local",
|
||||
[ValidateSet("disabled", "client_to_server", "server_to_client", "bidirectional")]
|
||||
[string]$AllowMode = "server_to_client",
|
||||
[ValidateSet("backend_gateway", "direct_worker_wss")]
|
||||
[string]$Transport = "backend_gateway",
|
||||
[switch]$ExpectBlocked,
|
||||
[ValidateSet("none", "detach", "takeover_old_controller", "worker_failure")]
|
||||
[string]$LifecycleScenario = "none",
|
||||
[string]$OutputDirectory = "artifacts/stage5-2-download-smoke"
|
||||
)
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
Add-Type -TypeDefinition @"
|
||||
using System.Net.Security;
|
||||
using System.Security.Cryptography.X509Certificates;
|
||||
|
||||
public static class RapSmokeTls
|
||||
{
|
||||
public static bool AcceptAll(object sender, X509Certificate certificate, X509Chain chain, SslPolicyErrors sslPolicyErrors)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
}
|
||||
"@
|
||||
|
||||
function Write-Log {
|
||||
param([string]$Message)
|
||||
Write-Host "[$(Get-Date -Format HH:mm:ss)] $Message"
|
||||
}
|
||||
|
||||
function Format-ExceptionChain {
|
||||
param([System.Exception]$Exception)
|
||||
$parts = @()
|
||||
$current = $Exception
|
||||
while ($current -ne $null) {
|
||||
$parts += "$($current.GetType().FullName): $($current.Message)"
|
||||
$current = $current.InnerException
|
||||
}
|
||||
return ($parts -join " -> ")
|
||||
}
|
||||
|
||||
function Invoke-Remote {
|
||||
param([string]$Command)
|
||||
ssh $DockerSshAlias $Command
|
||||
}
|
||||
|
||||
function Invoke-DbRows {
|
||||
param([string]$Sql)
|
||||
$encoded = [Convert]::ToBase64String([Text.Encoding]::UTF8.GetBytes($Sql))
|
||||
$command = "printf '%s' '$encoded' | base64 -d | docker exec -i rap_postgres psql -U rap_user -d remote_access_platform -t -A -F `"|`" -v ON_ERROR_STOP=1"
|
||||
@(Invoke-Remote -Command $command)
|
||||
}
|
||||
|
||||
function Invoke-Api {
|
||||
param(
|
||||
[ValidateSet("Get", "Post")]
|
||||
[string]$Method,
|
||||
[string]$Path,
|
||||
[object]$Body = $null
|
||||
)
|
||||
$uri = "$BackendApiBase/$Path".Replace("//sessions", "/sessions")
|
||||
if ($Body -ne $null) {
|
||||
return Invoke-RestMethod -Method $Method -Uri $uri -ContentType "application/json" -Body ($Body | ConvertTo-Json -Depth 16)
|
||||
}
|
||||
return Invoke-RestMethod -Method $Method -Uri $uri
|
||||
}
|
||||
|
||||
function Send-WsJson {
|
||||
param(
|
||||
[System.Net.WebSockets.ClientWebSocket]$Socket,
|
||||
[object]$Value
|
||||
)
|
||||
$json = $Value | ConvertTo-Json -Depth 20 -Compress
|
||||
$bytes = [Text.Encoding]::UTF8.GetBytes($json)
|
||||
$segment = [ArraySegment[byte]]::new($bytes)
|
||||
[void]$Socket.SendAsync($segment, [System.Net.WebSockets.WebSocketMessageType]::Text, $true, [Threading.CancellationToken]::None).GetAwaiter().GetResult()
|
||||
}
|
||||
|
||||
function Receive-WsJson {
|
||||
param(
|
||||
[System.Net.WebSockets.ClientWebSocket]$Socket,
|
||||
[int]$TimeoutSeconds = 30
|
||||
)
|
||||
$cts = [Threading.CancellationTokenSource]::new([TimeSpan]::FromSeconds($TimeoutSeconds))
|
||||
try {
|
||||
while ($true) {
|
||||
$buffer = New-Object byte[] 65536
|
||||
$stream = [IO.MemoryStream]::new()
|
||||
do {
|
||||
$segment = [ArraySegment[byte]]::new($buffer)
|
||||
$result = $Socket.ReceiveAsync($segment, $cts.Token).GetAwaiter().GetResult()
|
||||
if ($result.MessageType -eq [System.Net.WebSockets.WebSocketMessageType]::Close) {
|
||||
throw "websocket closed by peer"
|
||||
}
|
||||
$stream.Write($buffer, 0, $result.Count)
|
||||
} while (-not $result.EndOfMessage)
|
||||
if ($result.MessageType -eq [System.Net.WebSockets.WebSocketMessageType]::Binary) {
|
||||
continue
|
||||
}
|
||||
$text = [Text.Encoding]::UTF8.GetString($stream.ToArray())
|
||||
return $text | ConvertFrom-Json
|
||||
}
|
||||
} finally {
|
||||
$cts.Dispose()
|
||||
}
|
||||
}
|
||||
|
||||
function Wait-DbValue {
|
||||
param(
|
||||
[string]$Sql,
|
||||
[string]$Expected,
|
||||
[int]$TimeoutSeconds = 90
|
||||
)
|
||||
$deadline = (Get-Date).AddSeconds($TimeoutSeconds)
|
||||
do {
|
||||
$value = (Invoke-DbRows -Sql $Sql | Select-Object -First 1)
|
||||
if ($value -eq $Expected) {
|
||||
return $value
|
||||
}
|
||||
Start-Sleep -Seconds 1
|
||||
} while ((Get-Date) -lt $deadline)
|
||||
throw "Timed out waiting for DB value '$Expected'. Last value: '$value'"
|
||||
}
|
||||
|
||||
function Wait-DbValueIn {
|
||||
param(
|
||||
[string]$Sql,
|
||||
[string[]]$Expected,
|
||||
[int]$TimeoutSeconds = 90
|
||||
)
|
||||
$deadline = (Get-Date).AddSeconds($TimeoutSeconds)
|
||||
do {
|
||||
$value = (Invoke-DbRows -Sql $Sql | Select-Object -First 1)
|
||||
if ($Expected -contains $value) {
|
||||
return $value
|
||||
}
|
||||
Start-Sleep -Seconds 1
|
||||
} while ((Get-Date) -lt $deadline)
|
||||
throw "Timed out waiting for DB value in '$($Expected -join ',')'. Last value: '$value'"
|
||||
}
|
||||
|
||||
function Wait-WorkerPath {
|
||||
param(
|
||||
[string]$SessionId,
|
||||
[int]$TimeoutSeconds = 90
|
||||
)
|
||||
$path = "/tmp/rap-rdp-worker-transfers/$SessionId/visible/ToClient"
|
||||
$deadline = (Get-Date).AddSeconds($TimeoutSeconds)
|
||||
do {
|
||||
$exists = Invoke-Remote -Command "docker exec $WorkerContainerName sh -lc 'test -d ""$path"" && echo yes || true'" | Select-Object -First 1
|
||||
if ($exists -eq "yes") {
|
||||
return $path
|
||||
}
|
||||
Start-Sleep -Seconds 1
|
||||
} while ((Get-Date) -lt $deadline)
|
||||
throw "Timed out waiting for worker ToClient path: $path"
|
||||
}
|
||||
|
||||
function Wait-EnvelopeType {
|
||||
param(
|
||||
[System.Net.WebSockets.ClientWebSocket]$Socket,
|
||||
[string]$Type,
|
||||
[string]$FileName = "",
|
||||
[int]$TimeoutSeconds = 60
|
||||
)
|
||||
$deadline = (Get-Date).AddSeconds($TimeoutSeconds)
|
||||
do {
|
||||
$remaining = [Math]::Max(1, [int]($deadline - (Get-Date)).TotalSeconds)
|
||||
$message = Receive-WsJson -Socket $Socket -TimeoutSeconds $remaining
|
||||
if ($message.type -eq $Type) {
|
||||
if ($FileName -eq "" -or ($message.payload.file_name -eq $FileName)) {
|
||||
return $message
|
||||
}
|
||||
}
|
||||
} while ((Get-Date) -lt $deadline)
|
||||
throw "Timed out waiting for envelope type '$Type' file '$FileName'"
|
||||
}
|
||||
|
||||
function Send-DownloadStartAndWaitOutcome {
|
||||
param(
|
||||
[System.Net.WebSockets.ClientWebSocket]$Socket,
|
||||
[string]$FileId,
|
||||
[int]$TimeoutSeconds = 30
|
||||
)
|
||||
$transferId = [guid]::NewGuid().ToString()
|
||||
Send-WsJson -Socket $Socket -Value @{
|
||||
type = "file_download.start"
|
||||
payload = @{
|
||||
direction = "server_to_client"
|
||||
transfer_id = $transferId
|
||||
file_id = $FileId
|
||||
}
|
||||
}
|
||||
|
||||
$deadline = (Get-Date).AddSeconds($TimeoutSeconds)
|
||||
do {
|
||||
$remaining = [Math]::Max(1, [int]($deadline - (Get-Date)).TotalSeconds)
|
||||
try {
|
||||
$message = Receive-WsJson -Socket $Socket -TimeoutSeconds $remaining
|
||||
} catch {
|
||||
return [ordered]@{
|
||||
type = "websocket.closed"
|
||||
transfer_id = $transferId
|
||||
reason = $_.Exception.Message
|
||||
}
|
||||
}
|
||||
switch ($message.type) {
|
||||
"file_download.blocked" {
|
||||
return [ordered]@{
|
||||
type = [string]$message.type
|
||||
transfer_id = $transferId
|
||||
reason = [string]$message.payload.reason
|
||||
payload = $message.payload
|
||||
}
|
||||
}
|
||||
"file_download.failed" {
|
||||
return [ordered]@{
|
||||
type = [string]$message.type
|
||||
transfer_id = $transferId
|
||||
reason = [string]$message.payload.reason
|
||||
payload = $message.payload
|
||||
}
|
||||
}
|
||||
"session.taken_over" {
|
||||
return [ordered]@{
|
||||
type = [string]$message.type
|
||||
transfer_id = $transferId
|
||||
reason = "controller binding changed"
|
||||
payload = $message.payload
|
||||
}
|
||||
}
|
||||
"session.state" {
|
||||
$state = [string]$message.payload.state
|
||||
if ($state -eq "active" -or $state -eq "reconnecting") {
|
||||
continue
|
||||
}
|
||||
return [ordered]@{
|
||||
type = [string]$message.type
|
||||
transfer_id = $transferId
|
||||
reason = $state
|
||||
payload = $message.payload
|
||||
}
|
||||
}
|
||||
"transport.closed" {
|
||||
return [ordered]@{
|
||||
type = [string]$message.type
|
||||
transfer_id = $transferId
|
||||
reason = [string]$message.payload.reason
|
||||
payload = $message.payload
|
||||
}
|
||||
}
|
||||
"file_download.chunk" {
|
||||
return [ordered]@{
|
||||
type = "unexpected_file_download.chunk"
|
||||
transfer_id = $transferId
|
||||
reason = "download chunk was delivered while scenario expected blocking"
|
||||
payload = $message.payload
|
||||
}
|
||||
}
|
||||
"file_download.completed" {
|
||||
return [ordered]@{
|
||||
type = "unexpected_file_download.completed"
|
||||
transfer_id = $transferId
|
||||
reason = "download completed while scenario expected blocking"
|
||||
payload = $message.payload
|
||||
}
|
||||
}
|
||||
}
|
||||
} while ((Get-Date) -lt $deadline)
|
||||
|
||||
return [ordered]@{
|
||||
type = "timeout_no_download"
|
||||
transfer_id = $transferId
|
||||
reason = "no file download data was delivered before timeout"
|
||||
}
|
||||
}
|
||||
|
||||
function Restart-WorkerContainer {
|
||||
param([string]$ContainerName)
|
||||
Write-Log "Starting worker container $ContainerName"
|
||||
Invoke-Remote -Command "docker start $ContainerName >/dev/null" | Out-Null
|
||||
Start-Sleep -Seconds 5
|
||||
}
|
||||
|
||||
function Download-AvailableFile {
|
||||
param(
|
||||
[System.Net.WebSockets.ClientWebSocket]$Socket,
|
||||
[object]$Available,
|
||||
[string]$Destination
|
||||
)
|
||||
$transferId = [guid]::NewGuid().ToString()
|
||||
$fileId = [string]$Available.payload.file_id
|
||||
$fileName = [string]$Available.payload.file_name
|
||||
$expectedSize = [int64]$Available.payload.file_size
|
||||
$stream = [IO.File]::Open($Destination, [IO.FileMode]::Create, [IO.FileAccess]::Write, [IO.FileShare]::None)
|
||||
try {
|
||||
Write-Log "Starting download file=$fileName file_id=$fileId size=$expectedSize transfer=$transferId"
|
||||
Send-WsJson -Socket $Socket -Value @{
|
||||
type = "file_download.start"
|
||||
payload = @{
|
||||
direction = "server_to_client"
|
||||
transfer_id = $transferId
|
||||
file_id = $fileId
|
||||
}
|
||||
}
|
||||
$completed = $false
|
||||
while (-not $completed) {
|
||||
$message = Receive-WsJson -Socket $Socket -TimeoutSeconds 60
|
||||
switch ($message.type) {
|
||||
"file_download.progress" {
|
||||
Write-Log "Download progress status=$($message.payload.status) received=$($message.payload.received) total=$($message.payload.total)"
|
||||
}
|
||||
"file_download.chunk" {
|
||||
if ([string]$message.payload.transfer_id -ne $transferId) {
|
||||
continue
|
||||
}
|
||||
$offset = [int64]$message.payload.offset
|
||||
if ($offset -ne $stream.Position) {
|
||||
throw "Invalid chunk offset. Expected $($stream.Position), got $offset"
|
||||
}
|
||||
$bytes = [Convert]::FromBase64String([string]$message.payload.chunk_bytes)
|
||||
$stream.Write($bytes, 0, $bytes.Length)
|
||||
$ackOffset = [int64]$stream.Position
|
||||
Send-WsJson -Socket $Socket -Value @{
|
||||
type = "file_download.ack"
|
||||
payload = @{
|
||||
direction = "server_to_client"
|
||||
transfer_id = $transferId
|
||||
offset = $ackOffset
|
||||
sequence = [int64]$message.payload.sequence
|
||||
}
|
||||
}
|
||||
}
|
||||
"file_download.completed" {
|
||||
if ([string]$message.payload.transfer_id -eq $transferId) {
|
||||
$completed = $true
|
||||
}
|
||||
}
|
||||
"file_download.failed" {
|
||||
throw "File download failed: $($message.payload | ConvertTo-Json -Compress)"
|
||||
}
|
||||
"file_download.blocked" {
|
||||
throw "File download blocked: $($message.payload | ConvertTo-Json -Compress)"
|
||||
}
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
$stream.Dispose()
|
||||
}
|
||||
$actualSize = (Get-Item -LiteralPath $Destination).Length
|
||||
if ($actualSize -ne $expectedSize) {
|
||||
throw "Downloaded size mismatch for $fileName. Expected $expectedSize, got $actualSize"
|
||||
}
|
||||
return @{
|
||||
transfer_id = $transferId
|
||||
file_name = $fileName
|
||||
expected_size = $expectedSize
|
||||
actual_size = $actualSize
|
||||
destination = $Destination
|
||||
}
|
||||
}
|
||||
|
||||
New-Item -ItemType Directory -Force -Path $OutputDirectory | Out-Null
|
||||
|
||||
$lookupSql = @"
|
||||
select u.id::text, d.id::text, r.id::text, coalesce(p.file_transfer_mode, 'disabled')
|
||||
from users u
|
||||
join devices d on d.user_id = u.id and d.trust_status = 'trusted' and d.revoked_at is null
|
||||
join resources r on r.name = '$($ResourceName.Replace("'", "''"))'
|
||||
left join resource_policies p on p.resource_id = r.id
|
||||
where u.email = '$($UserEmail.Replace("'", "''"))'
|
||||
order by d.created_at desc
|
||||
limit 1;
|
||||
"@
|
||||
$row = Invoke-DbRows -Sql $lookupSql | Select-Object -First 1
|
||||
if ([string]::IsNullOrWhiteSpace($row)) {
|
||||
throw "Could not find smoke user/device/resource for user=$UserEmail resource=$ResourceName"
|
||||
}
|
||||
$parts = $row -split "\|"
|
||||
$userId = $parts[0]
|
||||
$deviceId = $parts[1]
|
||||
$resourceId = $parts[2]
|
||||
$previousMode = $parts[3]
|
||||
Write-Log "Using user=$userId device=$deviceId resource=$resourceId previous_file_transfer_mode=$previousMode"
|
||||
|
||||
$sessionId = ""
|
||||
$socket = $null
|
||||
$workerStoppedForScenario = $false
|
||||
try {
|
||||
Invoke-DbRows -Sql "update resource_policies set file_transfer_mode = '$AllowMode', updated_at = now() where resource_id = '$resourceId';" | Out-Null
|
||||
Write-Log "Set file_transfer_mode=$AllowMode"
|
||||
|
||||
$activeRows = Invoke-DbRows -Sql "select id::text from remote_sessions where controller_user_id = '$userId' and state in ('starting','active','detached');"
|
||||
foreach ($active in $activeRows) {
|
||||
if (-not [string]::IsNullOrWhiteSpace($active)) {
|
||||
Write-Log "Terminating pre-existing smoke session $active"
|
||||
Invoke-Api -Method Post -Path "sessions/$active/terminate" -Body @{ user_id = $userId; reason = "stage5_2_download_smoke_cleanup" } | Out-Null
|
||||
}
|
||||
}
|
||||
|
||||
$start = Invoke-Api -Method Post -Path "sessions/" -Body @{
|
||||
resource_id = $resourceId
|
||||
user_id = $userId
|
||||
device_id = $deviceId
|
||||
}
|
||||
$sessionId = [string]$start.session.id
|
||||
$attachmentId = [string]$start.attachment.id
|
||||
$attachToken = [string]$start.attach_token.token
|
||||
Write-Log "Started session $sessionId attachment=$attachmentId"
|
||||
|
||||
$socket = [System.Net.WebSockets.ClientWebSocket]::new()
|
||||
if ($Transport -eq "direct_worker_wss") {
|
||||
$callback = [Delegate]::CreateDelegate(
|
||||
[System.Net.Security.RemoteCertificateValidationCallback],
|
||||
[RapSmokeTls].GetMethod("AcceptAll"))
|
||||
$socket.Options.RemoteCertificateValidationCallback = $callback
|
||||
$candidate = @($start.data_plane.candidates | Where-Object { $_.type -eq "direct_worker_wss" } | Select-Object -First 1)
|
||||
if (-not $candidate -or [string]::IsNullOrWhiteSpace([string]$candidate.url)) {
|
||||
throw "Start response did not include direct_worker_wss candidate."
|
||||
}
|
||||
$token = [string]$start.data_plane.token
|
||||
$candidateUrl = [string]$candidate.url
|
||||
$separator = if ($candidateUrl.Contains("?")) { "&" } else { "?" }
|
||||
$wsUri = [Uri]::new("$candidateUrl$separator" +
|
||||
"data_plane_token=$([Uri]::EscapeDataString($token))&render_transport=binary_v1&color_mode=full_color")
|
||||
} else {
|
||||
$wsUri = [Uri]::new("${BackendWsBase}?attach_token=$([Uri]::EscapeDataString($attachToken))")
|
||||
}
|
||||
$redactedWsUri = [regex]::Replace($wsUri.ToString(), '(data_plane_token|attach_token)=([^&]+)', '$1=<redacted>')
|
||||
Write-Log "Connecting websocket transport=$Transport uri=$redactedWsUri"
|
||||
try {
|
||||
[void]$socket.ConnectAsync($wsUri, [Threading.CancellationToken]::None).GetAwaiter().GetResult()
|
||||
} catch {
|
||||
Write-Log "WebSocket connect failed: $(Format-ExceptionChain -Exception $_.Exception)"
|
||||
throw
|
||||
}
|
||||
if ($Transport -eq "direct_worker_wss") {
|
||||
$attached = Receive-WsJson -Socket $socket -TimeoutSeconds 15
|
||||
if ($attached.type -ne "data_plane.attached") {
|
||||
throw "Direct data-plane did not return data_plane.attached. Got: $($attached | ConvertTo-Json -Compress)"
|
||||
}
|
||||
}
|
||||
Write-Log "Connected websocket transport=$Transport"
|
||||
|
||||
Wait-DbValue -Sql "select state from remote_sessions where id = '$sessionId';" -Expected "active" -TimeoutSeconds 120 | Out-Null
|
||||
Write-Log "Session is active"
|
||||
|
||||
if ($LifecycleScenario -ne "none") {
|
||||
$toClientPath = Wait-WorkerPath -SessionId $sessionId -TimeoutSeconds 120
|
||||
Write-Log "Worker ToClient path ready for lifecycle scenario: $toClientPath"
|
||||
|
||||
$lifecycleName = "stage5-2-download-lifecycle.txt"
|
||||
$lifecycleContent = "Stage 5.2 lifecycle download guard`nscenario=$LifecycleScenario`n"
|
||||
$lifecycleBytes = [Text.Encoding]::UTF8.GetBytes($lifecycleContent)
|
||||
$lifecycleB64 = [Convert]::ToBase64String($lifecycleBytes)
|
||||
Invoke-Remote -Command "docker exec $WorkerContainerName sh -lc 'rm -f ""$toClientPath/$lifecycleName"" && printf %s $lifecycleB64 | base64 -d > ""$toClientPath/$lifecycleName"" && sync'" | Out-Null
|
||||
$workerLifecycleEvidence = Invoke-Remote -Command "docker exec $WorkerContainerName sh -lc 'cd ""$toClientPath"" && ls -l $lifecycleName && sha256sum $lifecycleName'"
|
||||
$workerLifecycleEvidence | Set-Content -LiteralPath (Join-Path $OutputDirectory "worker-lifecycle-file.txt") -Encoding UTF8
|
||||
|
||||
$available = Wait-EnvelopeType -Socket $socket -Type "file_download.available" -FileName $lifecycleName -TimeoutSeconds 90
|
||||
$fileId = [string]$available.payload.file_id
|
||||
Write-Log "Lifecycle file is available file_id=$fileId scenario=$LifecycleScenario"
|
||||
|
||||
$stateAfterAction = ""
|
||||
$outcome = $null
|
||||
switch ($LifecycleScenario) {
|
||||
"detach" {
|
||||
Write-Log "Detaching current attachment before attempting download"
|
||||
Invoke-Api -Method Post -Path "sessions/$sessionId/detach" -Body @{
|
||||
attachment_id = $attachmentId
|
||||
user_id = $userId
|
||||
reason = "stage5_2_download_lifecycle_detach"
|
||||
} | Out-Null
|
||||
$stateAfterAction = Wait-DbValue -Sql "select state from remote_sessions where id = '$sessionId';" -Expected "detached" -TimeoutSeconds 60
|
||||
$outcome = Send-DownloadStartAndWaitOutcome -Socket $socket -FileId $fileId -TimeoutSeconds 30
|
||||
}
|
||||
"takeover_old_controller" {
|
||||
$takeoverDeviceId = Invoke-DbRows -Sql "select id::text from devices where user_id = '$userId' and trust_status = 'trusted' and revoked_at is null and id <> '$deviceId' order by created_at desc limit 1;" | Select-Object -First 1
|
||||
if ([string]::IsNullOrWhiteSpace($takeoverDeviceId)) {
|
||||
throw "No second trusted device is available for takeover lifecycle proof."
|
||||
}
|
||||
Write-Log "Taking over session from second trusted device=$takeoverDeviceId"
|
||||
Invoke-Api -Method Post -Path "sessions/$sessionId/takeover" -Body @{
|
||||
user_id = $userId
|
||||
device_id = $takeoverDeviceId
|
||||
reason = "stage5_2_download_lifecycle_takeover"
|
||||
} | Out-Null
|
||||
$stateAfterAction = Wait-DbValue -Sql "select state from remote_sessions where id = '$sessionId';" -Expected "active" -TimeoutSeconds 60
|
||||
$outcome = Send-DownloadStartAndWaitOutcome -Socket $socket -FileId $fileId -TimeoutSeconds 30
|
||||
}
|
||||
"worker_failure" {
|
||||
Write-Log "Stopping worker container to prove download cannot continue after worker failure"
|
||||
Invoke-Remote -Command "docker stop $WorkerContainerName >/dev/null" | Out-Null
|
||||
$workerStoppedForScenario = $true
|
||||
$stateAfterAction = Wait-DbValueIn -Sql "select state from remote_sessions where id = '$sessionId';" -Expected @("failed", "terminated") -TimeoutSeconds 180
|
||||
$outcome = Send-DownloadStartAndWaitOutcome -Socket $socket -FileId $fileId -TimeoutSeconds 30
|
||||
}
|
||||
}
|
||||
|
||||
$acceptableOutcomes = @("file_download.blocked", "file_download.failed", "session.taken_over", "transport.closed", "websocket.closed")
|
||||
$pass = $acceptableOutcomes -contains [string]$outcome.type
|
||||
if ($LifecycleScenario -eq "worker_failure" -and ([string]$outcome.type) -eq "timeout_no_download" -and @("failed", "terminated") -contains $stateAfterAction) {
|
||||
$pass = $true
|
||||
}
|
||||
$result = [ordered]@{
|
||||
session_id = $sessionId
|
||||
attachment_id = $attachmentId
|
||||
mode = $AllowMode
|
||||
transport = $Transport
|
||||
lifecycle_scenario = $LifecycleScenario
|
||||
state_after_action = $stateAfterAction
|
||||
file_id = $fileId
|
||||
outcome = $outcome
|
||||
worker_evidence_file = (Resolve-Path (Join-Path $OutputDirectory "worker-lifecycle-file.txt")).Path
|
||||
pass = $pass
|
||||
}
|
||||
$result | ConvertTo-Json -Depth 10 | Tee-Object -FilePath (Join-Path $OutputDirectory "lifecycle-result.json")
|
||||
if (-not $pass) {
|
||||
throw "Lifecycle scenario '$LifecycleScenario' failed: $($outcome | ConvertTo-Json -Compress)"
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
if ($ExpectBlocked) {
|
||||
$blockedTransferId = [guid]::NewGuid().ToString()
|
||||
Send-WsJson -Socket $socket -Value @{
|
||||
type = "file_download.start"
|
||||
payload = @{
|
||||
direction = "server_to_client"
|
||||
transfer_id = $blockedTransferId
|
||||
file_id = "blocked-file-id"
|
||||
}
|
||||
}
|
||||
$blocked = Wait-EnvelopeType -Socket $socket -Type "file_download.blocked" -TimeoutSeconds 30
|
||||
$result = [ordered]@{
|
||||
session_id = $sessionId
|
||||
mode = $AllowMode
|
||||
transport = $Transport
|
||||
transfer_id = $blockedTransferId
|
||||
blocked_reason = [string]$blocked.payload.reason
|
||||
pass = $true
|
||||
}
|
||||
$result | ConvertTo-Json -Depth 8 | Tee-Object -FilePath (Join-Path $OutputDirectory "blocked-result.json")
|
||||
return
|
||||
}
|
||||
|
||||
$toClientPath = Wait-WorkerPath -SessionId $sessionId -TimeoutSeconds 120
|
||||
Write-Log "Worker ToClient path ready: $toClientPath"
|
||||
|
||||
$textName = "stage5-2-download-text.txt"
|
||||
$binaryName = "stage5-2-download-binary.bin"
|
||||
$textContent = "Stage 5.2 server-to-client download smoke`nрусский текст`n"
|
||||
$textBytes = [Text.Encoding]::UTF8.GetBytes($textContent)
|
||||
$textB64 = [Convert]::ToBase64String($textBytes)
|
||||
Invoke-Remote -Command "docker exec $WorkerContainerName sh -lc 'rm -f ""$toClientPath/$textName"" ""$toClientPath/$binaryName"" && printf %s $textB64 | base64 -d > ""$toClientPath/$textName"" && sync'" | Out-Null
|
||||
$workerTextEvidence = Invoke-Remote -Command "docker exec $WorkerContainerName sh -lc 'cd ""$toClientPath"" && ls -l $textName && sha256sum $textName'"
|
||||
$workerTextEvidence | Set-Content -LiteralPath (Join-Path $OutputDirectory "worker-text-file.txt") -Encoding UTF8
|
||||
Write-Log "Worker text file evidence captured"
|
||||
|
||||
$textAvailable = Wait-EnvelopeType -Socket $socket -Type "file_download.available" -FileName $textName -TimeoutSeconds 90
|
||||
$textDestination = Join-Path $OutputDirectory $textName
|
||||
$textResult = Download-AvailableFile -Socket $socket -Available $textAvailable -Destination $textDestination
|
||||
|
||||
Invoke-Remote -Command "docker exec $WorkerContainerName sh -lc 'head -c 4096 /dev/urandom > ""$toClientPath/$binaryName"" && sync'" | Out-Null
|
||||
$workerBinaryEvidence = Invoke-Remote -Command "docker exec $WorkerContainerName sh -lc 'cd ""$toClientPath"" && ls -l $binaryName && sha256sum $binaryName'"
|
||||
$workerBinaryEvidence | Set-Content -LiteralPath (Join-Path $OutputDirectory "worker-binary-file.txt") -Encoding UTF8
|
||||
Write-Log "Worker binary file evidence captured"
|
||||
|
||||
$binaryAvailable = Wait-EnvelopeType -Socket $socket -Type "file_download.available" -FileName $binaryName -TimeoutSeconds 90
|
||||
$binaryDestination = Join-Path $OutputDirectory $binaryName
|
||||
$binaryResult = Download-AvailableFile -Socket $socket -Available $binaryAvailable -Destination $binaryDestination
|
||||
|
||||
$localHashes = Get-FileHash -Algorithm SHA256 -LiteralPath $textDestination, $binaryDestination |
|
||||
Select-Object Path, Hash
|
||||
$localHashes | ConvertTo-Json -Depth 4 | Set-Content -LiteralPath (Join-Path $OutputDirectory "local-hashes.json") -Encoding UTF8
|
||||
|
||||
$result = [ordered]@{
|
||||
session_id = $sessionId
|
||||
mode = $AllowMode
|
||||
transport = $Transport
|
||||
text = $textResult
|
||||
binary = $binaryResult
|
||||
worker_text_evidence_file = (Resolve-Path (Join-Path $OutputDirectory "worker-text-file.txt")).Path
|
||||
worker_binary_evidence_file = (Resolve-Path (Join-Path $OutputDirectory "worker-binary-file.txt")).Path
|
||||
local_hashes_file = (Resolve-Path (Join-Path $OutputDirectory "local-hashes.json")).Path
|
||||
pass = $true
|
||||
}
|
||||
$result | ConvertTo-Json -Depth 8 | Tee-Object -FilePath (Join-Path $OutputDirectory "result.json")
|
||||
} finally {
|
||||
if ($socket) {
|
||||
try { $socket.Abort(); $socket.Dispose() } catch {}
|
||||
}
|
||||
if ($workerStoppedForScenario) {
|
||||
try {
|
||||
Restart-WorkerContainer -ContainerName $WorkerContainerName
|
||||
} catch {
|
||||
Write-Log "Worker restart failed: $($_.Exception.Message)"
|
||||
}
|
||||
}
|
||||
if ($sessionId) {
|
||||
try {
|
||||
Invoke-Api -Method Post -Path "sessions/$sessionId/terminate" -Body @{ user_id = $userId; reason = "stage5_2_download_smoke_complete" } | Out-Null
|
||||
Write-Log "Terminated session $sessionId"
|
||||
} catch {
|
||||
Write-Log "Session cleanup failed: $($_.Exception.Message)"
|
||||
}
|
||||
}
|
||||
if ($previousMode) {
|
||||
try {
|
||||
Invoke-DbRows -Sql "update resource_policies set file_transfer_mode = '$previousMode', updated_at = now() where resource_id = '$resourceId';" | Out-Null
|
||||
Write-Log "Restored file_transfer_mode=$previousMode"
|
||||
} catch {
|
||||
Write-Log "Policy restore failed: $($_.Exception.Message)"
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,525 @@
|
||||
param(
|
||||
[string]$BackendApiBase = "http://192.168.200.61:8080/api/v1",
|
||||
[string]$BackendWsBase = "ws://192.168.200.61:8080/api/v1/gateway/ws",
|
||||
[string]$DockerSshAlias = "docker-test",
|
||||
[string]$WorkerContainerName = "rap_worker_smoke",
|
||||
[string]$WorkerImage = "rap-rdp-worker:stage5-drive-visible",
|
||||
[string]$ResourceName = "Windows Smoke Default Resource",
|
||||
[string]$Email = "windows-smoke@example.local",
|
||||
[string]$Password = "SmokePass!123",
|
||||
[string]$OutputLog = "artifacts/stage5-file-upload-smoke.log"
|
||||
)
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
$log = [System.Collections.Generic.List[string]]::new()
|
||||
|
||||
function Write-SmokeLog {
|
||||
param([string]$Message)
|
||||
$line = "[$(Get-Date -Format o)] $Message"
|
||||
$log.Add($line)
|
||||
Write-Host $line
|
||||
}
|
||||
|
||||
function Invoke-RemoteSqlScalar {
|
||||
param([string]$Sql)
|
||||
$encoded = [Convert]::ToBase64String([Text.Encoding]::UTF8.GetBytes($Sql))
|
||||
$output = ssh $DockerSshAlias "printf '%s' '$encoded' | base64 -d | docker exec -i rap_postgres psql -U rap_user -d remote_access_platform -t -A -v ON_ERROR_STOP=1"
|
||||
return (($output | Select-Object -Last 1) -as [string]).Trim()
|
||||
}
|
||||
|
||||
function Set-FileTransferMode {
|
||||
param([ValidateSet("disabled", "client_to_server")][string]$Mode)
|
||||
$enabled = if ($Mode -eq "disabled") { "FALSE" } else { "TRUE" }
|
||||
$escapedName = $ResourceName.Replace("'", "''")
|
||||
$sql = @"
|
||||
UPDATE resource_policies rp
|
||||
SET file_transfer_mode = '$Mode',
|
||||
file_transfer_enabled = $enabled,
|
||||
updated_at = NOW()
|
||||
FROM resources r
|
||||
WHERE rp.resource_id = r.id
|
||||
AND r.name = '$escapedName';
|
||||
SELECT COALESCE(MAX(rp.file_transfer_mode), '<missing>')
|
||||
FROM resource_policies rp
|
||||
JOIN resources r ON r.id = rp.resource_id
|
||||
WHERE r.name = '$escapedName';
|
||||
"@
|
||||
$actual = Invoke-RemoteSqlScalar -Sql $sql
|
||||
if ($actual -ne $Mode) {
|
||||
throw "file_transfer_mode was not set. Expected=$Mode actual=$actual"
|
||||
}
|
||||
Write-SmokeLog "policy file_transfer_mode=$Mode"
|
||||
}
|
||||
|
||||
function Invoke-Api {
|
||||
param(
|
||||
[ValidateSet("Get", "Post")][string]$Method,
|
||||
[string]$Path,
|
||||
[object]$Body = $null
|
||||
)
|
||||
$uri = "$BackendApiBase/$Path"
|
||||
if ($null -eq $Body) {
|
||||
return Invoke-RestMethod -Method $Method -Uri $uri
|
||||
}
|
||||
return Invoke-RestMethod -Method $Method -Uri $uri -ContentType "application/json" -Body ($Body | ConvertTo-Json -Depth 12)
|
||||
}
|
||||
|
||||
function New-GatewayWebSocket {
|
||||
param([string]$AttachToken)
|
||||
$ws = [Net.WebSockets.ClientWebSocket]::new()
|
||||
$uri = [Uri]::new("${BackendWsBase}?attach_token=$([Uri]::EscapeDataString($AttachToken))")
|
||||
$null = $ws.ConnectAsync($uri, [Threading.CancellationToken]::None).GetAwaiter().GetResult()
|
||||
return $ws
|
||||
}
|
||||
|
||||
function Receive-Envelope {
|
||||
param(
|
||||
[Net.WebSockets.ClientWebSocket]$WebSocket,
|
||||
[int]$TimeoutMilliseconds = 30000
|
||||
)
|
||||
$buffer = [byte[]]::new(1024 * 1024)
|
||||
$segment = [ArraySegment[byte]]::new($buffer)
|
||||
$stream = [IO.MemoryStream]::new()
|
||||
while ($true) {
|
||||
$cts = [Threading.CancellationTokenSource]::new($TimeoutMilliseconds)
|
||||
try {
|
||||
$result = $WebSocket.ReceiveAsync($segment, $cts.Token).GetAwaiter().GetResult()
|
||||
}
|
||||
finally {
|
||||
$cts.Dispose()
|
||||
}
|
||||
if ($result.MessageType -eq [Net.WebSockets.WebSocketMessageType]::Close) {
|
||||
throw "websocket closed"
|
||||
}
|
||||
$stream.Write($buffer, 0, $result.Count)
|
||||
if ($result.EndOfMessage) {
|
||||
break
|
||||
}
|
||||
}
|
||||
$text = [Text.Encoding]::UTF8.GetString($stream.ToArray())
|
||||
return $text | ConvertFrom-Json
|
||||
}
|
||||
|
||||
function Receive-EnvelopeOfType {
|
||||
param(
|
||||
[Net.WebSockets.ClientWebSocket]$WebSocket,
|
||||
[string[]]$Types,
|
||||
[int]$TimeoutSeconds = 30
|
||||
)
|
||||
$deadline = (Get-Date).AddSeconds($TimeoutSeconds)
|
||||
while ((Get-Date) -lt $deadline) {
|
||||
$envelope = Receive-Envelope -WebSocket $WebSocket -TimeoutMilliseconds 10000
|
||||
if ($Types -contains [string]$envelope.type) {
|
||||
return $envelope
|
||||
}
|
||||
Write-SmokeLog "skipping ws event type=$($envelope.type)"
|
||||
}
|
||||
throw "timed out waiting for envelope type: $($Types -join ',')"
|
||||
}
|
||||
|
||||
function Send-Envelope {
|
||||
param(
|
||||
[Net.WebSockets.ClientWebSocket]$WebSocket,
|
||||
[object]$Envelope
|
||||
)
|
||||
$json = $Envelope | ConvertTo-Json -Depth 16 -Compress
|
||||
$bytes = [Text.Encoding]::UTF8.GetBytes($json)
|
||||
$null = $WebSocket.SendAsync(
|
||||
[ArraySegment[byte]]::new($bytes),
|
||||
[Net.WebSockets.WebSocketMessageType]::Text,
|
||||
$true,
|
||||
[Threading.CancellationToken]::None
|
||||
).GetAwaiter().GetResult()
|
||||
}
|
||||
|
||||
function Get-PayloadProperty {
|
||||
param(
|
||||
[object]$Payload,
|
||||
[string]$Name
|
||||
)
|
||||
if ($null -eq $Payload) {
|
||||
return ""
|
||||
}
|
||||
if ($Payload.PSObject.Properties.Name -contains $Name) {
|
||||
return $Payload.$Name
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
function Wait-SessionActive {
|
||||
param([Net.WebSockets.ClientWebSocket]$WebSocket)
|
||||
$deadline = (Get-Date).AddSeconds(90)
|
||||
while ((Get-Date) -lt $deadline) {
|
||||
$envelope = Receive-Envelope -WebSocket $WebSocket -TimeoutMilliseconds 30000
|
||||
Write-SmokeLog "ws event type=$($envelope.type) state=$($envelope.payload.state)"
|
||||
if ($envelope.type -eq "session.state" -and $envelope.payload.state -eq "active") {
|
||||
return
|
||||
}
|
||||
}
|
||||
throw "session did not become active"
|
||||
}
|
||||
|
||||
function Start-SmokeSession {
|
||||
param([string]$UserId, [string]$DeviceId, [string]$ResourceId)
|
||||
return Invoke-Api -Method Post -Path "sessions" -Body @{
|
||||
resource_id = $ResourceId
|
||||
user_id = $UserId
|
||||
device_id = $DeviceId
|
||||
}
|
||||
}
|
||||
|
||||
function Stop-SmokeSession {
|
||||
param([string]$SessionId, [string]$UserId)
|
||||
try {
|
||||
Invoke-Api -Method Post -Path "sessions/$SessionId/terminate" -Body @{
|
||||
user_id = $UserId
|
||||
reason = "stage5_file_upload_smoke_cleanup"
|
||||
} | Out-Null
|
||||
}
|
||||
catch {
|
||||
Write-SmokeLog "terminate skipped session=$SessionId reason=$($_.Exception.Message)"
|
||||
}
|
||||
}
|
||||
|
||||
function Restart-SmokeWorker {
|
||||
Write-SmokeLog "restarting worker container=$WorkerContainerName"
|
||||
$cmd = "docker rm -f $WorkerContainerName 2>/dev/null || true; docker run -d --name $WorkerContainerName --network host -e RDP_WORKER_ID=rdp-worker-1 -e RDP_WORKER_REDIS_HOST=127.0.0.1 -e RDP_WORKER_REDIS_PORT=6379 -e RDP_WORKER_TRANSFER_ROOT=/tmp/rap-rdp-worker-transfers -e RDP_WORKER_CAPABILITIES=adaptive-quality,dirty-rects,clipboard,file-transfer $WorkerImage"
|
||||
$output = ssh $DockerSshAlias $cmd
|
||||
Write-SmokeLog "worker restart output=$($output -join ' ')"
|
||||
Start-Sleep -Seconds 5
|
||||
}
|
||||
|
||||
function Stop-SmokeWorker {
|
||||
Write-SmokeLog "stopping worker container=$WorkerContainerName"
|
||||
ssh $DockerSshAlias "docker rm -f $WorkerContainerName 2>/dev/null || true" | Out-Null
|
||||
}
|
||||
|
||||
function Wait-SessionState {
|
||||
param(
|
||||
[string]$SessionId,
|
||||
[string]$ExpectedState,
|
||||
[int]$TimeoutSeconds = 90
|
||||
)
|
||||
$deadline = (Get-Date).AddSeconds($TimeoutSeconds)
|
||||
while ((Get-Date) -lt $deadline) {
|
||||
$escaped = $SessionId.Replace("'", "''")
|
||||
$state = Invoke-RemoteSqlScalar -Sql "SELECT state FROM remote_sessions WHERE id = '$escaped' LIMIT 1;"
|
||||
Write-SmokeLog "session_state session=$SessionId state=$state expected=$ExpectedState"
|
||||
if ($state -eq $ExpectedState) {
|
||||
return
|
||||
}
|
||||
Start-Sleep -Seconds 3
|
||||
}
|
||||
throw "session $SessionId did not reach state $ExpectedState"
|
||||
}
|
||||
|
||||
function Assert-UploadBlockedOnConnection {
|
||||
param(
|
||||
[Net.WebSockets.ClientWebSocket]$WebSocket,
|
||||
[string]$Label
|
||||
)
|
||||
try {
|
||||
Send-Envelope -WebSocket $WebSocket -Envelope @{
|
||||
type = "file_upload.start"
|
||||
payload = @{
|
||||
direction = "client_to_server"
|
||||
transfer_id = [guid]::NewGuid().ToString("D")
|
||||
file_name = "$Label.txt"
|
||||
file_size = 5
|
||||
total_chunks = 1
|
||||
content_hash = ""
|
||||
}
|
||||
}
|
||||
$response = Receive-EnvelopeOfType -WebSocket $WebSocket -Types @("file_upload.blocked", "session.taken_over", "transport.closed", "session.state") -TimeoutSeconds 20
|
||||
$reason = Get-PayloadProperty -Payload $response.payload -Name "reason"
|
||||
$state = Get-PayloadProperty -Payload $response.payload -Name "state"
|
||||
Write-SmokeLog "$Label upload gate response type=$($response.type) state=$state reason=$reason"
|
||||
if ($response.type -eq "file_upload.progress") {
|
||||
throw "$Label upload was accepted unexpectedly"
|
||||
}
|
||||
if ($response.type -eq "session.state" -and $state -eq "active") {
|
||||
throw "$Label upload gate returned active session state"
|
||||
}
|
||||
}
|
||||
catch {
|
||||
Write-SmokeLog "$Label upload blocked by closed/failed transport: $($_.Exception.Message)"
|
||||
}
|
||||
}
|
||||
|
||||
function Upload-Bytes {
|
||||
param(
|
||||
[Net.WebSockets.ClientWebSocket]$WebSocket,
|
||||
[string]$FileName,
|
||||
[byte[]]$Bytes
|
||||
)
|
||||
$transferId = [guid]::NewGuid().ToString("D")
|
||||
$chunkSize = 256 * 1024
|
||||
$totalChunks = [int64][Math]::Ceiling($Bytes.Length / $chunkSize)
|
||||
Write-SmokeLog "client upload start transfer_id=$transferId file=$FileName bytes=$($Bytes.Length)"
|
||||
Send-Envelope -WebSocket $WebSocket -Envelope @{
|
||||
type = "file_upload.start"
|
||||
payload = @{
|
||||
direction = "client_to_server"
|
||||
transfer_id = $transferId
|
||||
file_name = $FileName
|
||||
file_size = [int64]$Bytes.Length
|
||||
total_chunks = $totalChunks
|
||||
content_hash = ""
|
||||
}
|
||||
}
|
||||
$startResponse = Receive-EnvelopeOfType -WebSocket $WebSocket -Types @("file_upload.progress", "file_upload.blocked") -TimeoutSeconds 20
|
||||
$startStatus = Get-PayloadProperty -Payload $startResponse.payload -Name "status"
|
||||
$startReason = Get-PayloadProperty -Payload $startResponse.payload -Name "reason"
|
||||
Write-SmokeLog "client upload response type=$($startResponse.type) status=$startStatus reason=$startReason"
|
||||
if ($startResponse.type -ne "file_upload.progress" -or $startResponse.payload.status -ne "started") {
|
||||
throw "upload was not accepted by gateway"
|
||||
}
|
||||
$offset = 0
|
||||
$chunkIndex = 0
|
||||
while ($offset -lt $Bytes.Length) {
|
||||
$take = [Math]::Min($chunkSize, $Bytes.Length - $offset)
|
||||
$chunk = [byte[]]::new($take)
|
||||
[Array]::Copy($Bytes, $offset, $chunk, 0, $take)
|
||||
Send-Envelope -WebSocket $WebSocket -Envelope @{
|
||||
type = "file_upload.chunk"
|
||||
payload = @{
|
||||
direction = "client_to_server"
|
||||
transfer_id = $transferId
|
||||
chunk_index = [int64]$chunkIndex
|
||||
offset = [int64]$offset
|
||||
chunk_size = $take
|
||||
chunk_bytes = [Convert]::ToBase64String($chunk)
|
||||
}
|
||||
}
|
||||
$progress = Receive-EnvelopeOfType -WebSocket $WebSocket -Types @("file_upload.progress", "file_upload.blocked") -TimeoutSeconds 20
|
||||
$progressStatus = Get-PayloadProperty -Payload $progress.payload -Name "status"
|
||||
$progressReceived = Get-PayloadProperty -Payload $progress.payload -Name "received"
|
||||
$progressTotal = Get-PayloadProperty -Payload $progress.payload -Name "total"
|
||||
Write-SmokeLog "client upload progress type=$($progress.type) status=$progressStatus received=$progressReceived total=$progressTotal"
|
||||
$offset += $take
|
||||
$chunkIndex++
|
||||
}
|
||||
return $transferId
|
||||
}
|
||||
|
||||
function Get-RemoteFileEvidence {
|
||||
param(
|
||||
[string]$SessionId,
|
||||
[string]$TransferId,
|
||||
[string]$FileName,
|
||||
[string]$ExpectedSha256
|
||||
)
|
||||
$path = "/tmp/rap-rdp-worker-transfers/$SessionId/visible/$FileName"
|
||||
$remoteScript = "p='$path'; ls -l ""`$p""; wc -c < ""`$p""; sha256sum ""`$p"""
|
||||
$output = $null
|
||||
$deadline = (Get-Date).AddSeconds(20)
|
||||
while ((Get-Date) -lt $deadline) {
|
||||
$output = ssh $DockerSshAlias "docker exec $WorkerContainerName sh -lc '$remoteScript'" 2>$null
|
||||
if ($LASTEXITCODE -eq 0 -and (($output -join "`n") -like "*$ExpectedSha256*")) {
|
||||
break
|
||||
}
|
||||
Start-Sleep -Milliseconds 500
|
||||
}
|
||||
foreach ($line in $output) {
|
||||
Write-SmokeLog "worker_fs $line"
|
||||
}
|
||||
if ((($output -join "`n") -notlike "*$ExpectedSha256*")) {
|
||||
throw "remote sha256 mismatch for $FileName"
|
||||
}
|
||||
}
|
||||
|
||||
function Get-Sha256 {
|
||||
param([byte[]]$Bytes)
|
||||
$stream = [IO.MemoryStream]::new($Bytes)
|
||||
try {
|
||||
return (Get-FileHash -Algorithm SHA256 -InputStream $stream).Hash.ToLowerInvariant()
|
||||
}
|
||||
finally {
|
||||
$stream.Dispose()
|
||||
}
|
||||
}
|
||||
|
||||
New-Item -ItemType Directory -Force -Path (Split-Path -Parent $OutputLog) | Out-Null
|
||||
|
||||
$login = Invoke-Api -Method Post -Path "auth/login" -Body @{
|
||||
email = $Email
|
||||
password = $Password
|
||||
trust_device = $true
|
||||
device_fingerprint = "stage5-file-upload-a"
|
||||
device_label = "stage5-file-upload-a"
|
||||
}
|
||||
$userId = [string]$login.user.ID
|
||||
if ([string]::IsNullOrWhiteSpace($userId)) {
|
||||
$userId = [string]$login.user.id
|
||||
}
|
||||
$deviceId = [string]$login.device.ID
|
||||
if ([string]::IsNullOrWhiteSpace($deviceId)) {
|
||||
$deviceId = [string]$login.device.id
|
||||
}
|
||||
|
||||
$escapedResourceName = $ResourceName.Replace("'", "''")
|
||||
$resourceSql = "SELECT id::text FROM resources WHERE name = '$escapedResourceName' LIMIT 1;"
|
||||
$resourceId = Invoke-RemoteSqlScalar -Sql $resourceSql
|
||||
if ([string]::IsNullOrWhiteSpace($resourceId)) {
|
||||
throw "resource not found: $ResourceName"
|
||||
}
|
||||
Write-SmokeLog "smoke user=$userId device=$deviceId resource=$resourceId"
|
||||
Restart-SmokeWorker
|
||||
|
||||
$sessions = Invoke-Api -Method Get -Path "sessions?user_id=$([Uri]::EscapeDataString($userId))"
|
||||
foreach ($session in @($sessions.sessions)) {
|
||||
if ($session.state -in @("starting", "active", "detached", "reconnecting")) {
|
||||
Stop-SmokeSession -SessionId $session.id -UserId $userId
|
||||
}
|
||||
}
|
||||
|
||||
Set-FileTransferMode -Mode disabled
|
||||
$disabled = Start-SmokeSession -UserId $userId -DeviceId $deviceId -ResourceId $resourceId
|
||||
$disabledSessionId = [string]$disabled.session.ID
|
||||
$disabledAttachToken = [string]$disabled.attach_token.token
|
||||
Write-SmokeLog "disabled session=$disabledSessionId"
|
||||
$disabledWs = New-GatewayWebSocket -AttachToken $disabledAttachToken
|
||||
try {
|
||||
Wait-SessionActive -WebSocket $disabledWs
|
||||
$blockedTransferId = [guid]::NewGuid().ToString("D")
|
||||
Send-Envelope -WebSocket $disabledWs -Envelope @{
|
||||
type = "file_upload.start"
|
||||
payload = @{
|
||||
direction = "client_to_server"
|
||||
transfer_id = $blockedTransferId
|
||||
file_name = "blocked.txt"
|
||||
file_size = 5
|
||||
total_chunks = 1
|
||||
content_hash = ""
|
||||
}
|
||||
}
|
||||
$blocked = Receive-EnvelopeOfType -WebSocket $disabledWs -Types @("file_upload.blocked") -TimeoutSeconds 20
|
||||
$blockedReason = if ($blocked.payload.PSObject.Properties.Name -contains "reason") { $blocked.payload.reason } else { "" }
|
||||
Write-SmokeLog "disabled upload response type=$($blocked.type) reason=$blockedReason"
|
||||
if ($blocked.type -ne "file_upload.blocked") {
|
||||
throw "disabled policy did not block upload"
|
||||
}
|
||||
}
|
||||
finally {
|
||||
$disabledWs.Dispose()
|
||||
Stop-SmokeSession -SessionId $disabledSessionId -UserId $userId
|
||||
}
|
||||
|
||||
Set-FileTransferMode -Mode client_to_server
|
||||
$allowed = Start-SmokeSession -UserId $userId -DeviceId $deviceId -ResourceId $resourceId
|
||||
$sessionId = [string]$allowed.session.ID
|
||||
$attachmentId = [string]$allowed.attachment.ID
|
||||
$attachToken = [string]$allowed.attach_token.token
|
||||
Write-SmokeLog "allowed session=$sessionId attachment=$attachmentId"
|
||||
$ws = New-GatewayWebSocket -AttachToken $attachToken
|
||||
try {
|
||||
Wait-SessionActive -WebSocket $ws
|
||||
Send-Envelope -WebSocket $ws -Envelope @{ type = "input"; payload = @{ kind = "mouse"; action = "move"; normalized_x = 0.5; normalized_y = 0.5; surface_width = 1024; surface_height = 768 } }
|
||||
Send-Envelope -WebSocket $ws -Envelope @{ type = "input"; payload = @{ kind = "keyboard"; action = "key_down"; scan_code = 30; is_extended = $false } }
|
||||
Send-Envelope -WebSocket $ws -Envelope @{ type = "input"; payload = @{ kind = "keyboard"; action = "key_up"; scan_code = 30; is_extended = $false } }
|
||||
Send-Envelope -WebSocket $ws -Envelope @{ type = "clipboard"; payload = @{ direction = "client_to_server"; text = "stage5 clipboard regression"; sequence_id = [DateTimeOffset]::UtcNow.ToUnixTimeMilliseconds(); origin = $attachmentId; content_hash = "" } }
|
||||
Start-Sleep -Seconds 2
|
||||
|
||||
$textBytes = [Text.Encoding]::UTF8.GetBytes("Stage 5.1 text upload`nрусский текст`n")
|
||||
$textSha = Get-Sha256 -Bytes $textBytes
|
||||
$textTransfer = Upload-Bytes -WebSocket $ws -FileName "stage5-upload-text.txt" -Bytes $textBytes
|
||||
Get-RemoteFileEvidence -SessionId $sessionId -TransferId $textTransfer -FileName "stage5-upload-text.txt" -ExpectedSha256 $textSha
|
||||
|
||||
$binaryBytes = [byte[]]::new(4096)
|
||||
[Security.Cryptography.RandomNumberGenerator]::Fill($binaryBytes)
|
||||
$binarySha = Get-Sha256 -Bytes $binaryBytes
|
||||
$binaryTransfer = Upload-Bytes -WebSocket $ws -FileName "stage5-upload-binary.bin" -Bytes $binaryBytes
|
||||
Get-RemoteFileEvidence -SessionId $sessionId -TransferId $binaryTransfer -FileName "stage5-upload-binary.bin" -ExpectedSha256 $binarySha
|
||||
|
||||
Invoke-Api -Method Post -Path "sessions/$sessionId/detach" -Body @{
|
||||
attachment_id = $attachmentId
|
||||
user_id = $userId
|
||||
reason = "stage5_file_upload_detach_gate"
|
||||
} | Out-Null
|
||||
Start-Sleep -Seconds 2
|
||||
try {
|
||||
Send-Envelope -WebSocket $ws -Envelope @{
|
||||
type = "file_upload.start"
|
||||
payload = @{
|
||||
direction = "client_to_server"
|
||||
transfer_id = [guid]::NewGuid().ToString("D")
|
||||
file_name = "after-detach.txt"
|
||||
file_size = 5
|
||||
total_chunks = 1
|
||||
content_hash = ""
|
||||
}
|
||||
}
|
||||
$detachGate = Receive-Envelope -WebSocket $ws -TimeoutMilliseconds 8000
|
||||
Write-SmokeLog "detach upload response type=$($detachGate.type) reason=$($detachGate.payload.reason)"
|
||||
}
|
||||
catch {
|
||||
Write-SmokeLog "detach upload blocked by closed transport: $($_.Exception.Message)"
|
||||
}
|
||||
|
||||
Stop-SmokeSession -SessionId $sessionId -UserId $userId
|
||||
|
||||
$takeoverSession = Start-SmokeSession -UserId $userId -DeviceId $deviceId -ResourceId $resourceId
|
||||
$takeoverSessionId = [string]$takeoverSession.session.ID
|
||||
$takeoverAttachmentId = [string]$takeoverSession.attachment.ID
|
||||
$takeoverAttachToken = [string]$takeoverSession.attach_token.token
|
||||
Write-SmokeLog "takeover session=$takeoverSessionId old_attachment=$takeoverAttachmentId"
|
||||
$takeoverOldWs = New-GatewayWebSocket -AttachToken $takeoverAttachToken
|
||||
$takeoverNewWs = $null
|
||||
try {
|
||||
Wait-SessionActive -WebSocket $takeoverOldWs
|
||||
$loginB = Invoke-Api -Method Post -Path "auth/login" -Body @{
|
||||
email = $Email
|
||||
password = $Password
|
||||
trust_device = $true
|
||||
device_fingerprint = "stage5-file-upload-b"
|
||||
device_label = "stage5-file-upload-b"
|
||||
}
|
||||
$deviceB = [string]$loginB.device.ID
|
||||
if ([string]::IsNullOrWhiteSpace($deviceB)) {
|
||||
$deviceB = [string]$loginB.device.id
|
||||
}
|
||||
$takeoverResult = Invoke-Api -Method Post -Path "sessions/$takeoverSessionId/takeover" -Body @{
|
||||
user_id = $userId
|
||||
device_id = $deviceB
|
||||
reason = "stage5_file_upload_takeover_gate"
|
||||
}
|
||||
$newAttachmentId = [string]$takeoverResult.attachment.ID
|
||||
$newAttachToken = [string]$takeoverResult.attach_token.token
|
||||
Write-SmokeLog "takeover complete new_attachment=$newAttachmentId"
|
||||
$takeoverNewWs = New-GatewayWebSocket -AttachToken $newAttachToken
|
||||
Wait-SessionActive -WebSocket $takeoverNewWs
|
||||
Assert-UploadBlockedOnConnection -WebSocket $takeoverOldWs -Label "takeover_old_client"
|
||||
}
|
||||
finally {
|
||||
if ($null -ne $takeoverNewWs) {
|
||||
$takeoverNewWs.Dispose()
|
||||
}
|
||||
$takeoverOldWs.Dispose()
|
||||
Stop-SmokeSession -SessionId $takeoverSessionId -UserId $userId
|
||||
}
|
||||
|
||||
$failureSession = Start-SmokeSession -UserId $userId -DeviceId $deviceId -ResourceId $resourceId
|
||||
$failureSessionId = [string]$failureSession.session.ID
|
||||
$failureAttachToken = [string]$failureSession.attach_token.token
|
||||
Write-SmokeLog "worker_failure session=$failureSessionId"
|
||||
$failureWs = New-GatewayWebSocket -AttachToken $failureAttachToken
|
||||
try {
|
||||
Wait-SessionActive -WebSocket $failureWs
|
||||
Stop-SmokeWorker
|
||||
Wait-SessionState -SessionId $failureSessionId -ExpectedState "failed" -TimeoutSeconds 120
|
||||
Assert-UploadBlockedOnConnection -WebSocket $failureWs -Label "worker_failure"
|
||||
}
|
||||
finally {
|
||||
$failureWs.Dispose()
|
||||
Restart-SmokeWorker
|
||||
Stop-SmokeSession -SessionId $failureSessionId -UserId $userId
|
||||
}
|
||||
}
|
||||
finally {
|
||||
$ws.Dispose()
|
||||
Stop-SmokeSession -SessionId $sessionId -UserId $userId
|
||||
$log | Set-Content -LiteralPath $OutputLog -Encoding UTF8
|
||||
Write-SmokeLog "wrote $OutputLog"
|
||||
}
|
||||
@@ -0,0 +1,132 @@
|
||||
param(
|
||||
[string]$DockerSshAlias = "docker-test",
|
||||
[string]$RemoteOutputDir = "/tmp/rap-p3-5-platform-ca",
|
||||
[string]$LocalCaOutputPath = "artifacts/p3-5-platform-ca.crt",
|
||||
[string]$WorkerHost = "192.168.200.61",
|
||||
[string]$WorkerDnsName = "",
|
||||
[string]$ClusterId = "default",
|
||||
[string]$WorkerId = "rdp-worker-1",
|
||||
[int]$Days = 30
|
||||
)
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
function Quote-Bash {
|
||||
param([string]$Value)
|
||||
return "'" + $Value.Replace("'", "'\''") + "'"
|
||||
}
|
||||
|
||||
$remoteDir = Quote-Bash $RemoteOutputDir
|
||||
$workerHost = Quote-Bash $WorkerHost
|
||||
$workerDnsName = Quote-Bash $WorkerDnsName
|
||||
$clusterId = Quote-Bash $ClusterId
|
||||
$workerId = Quote-Bash $WorkerId
|
||||
|
||||
$remoteScript = @"
|
||||
set -euo pipefail
|
||||
|
||||
REMOTE_DIR=$remoteDir
|
||||
WORKER_HOST=$workerHost
|
||||
WORKER_DNS_NAME=$workerDnsName
|
||||
CLUSTER_ID=$clusterId
|
||||
WORKER_ID=$workerId
|
||||
DAYS=$Days
|
||||
|
||||
mkdir -p "`$REMOTE_DIR"
|
||||
chmod 700 "`$REMOTE_DIR"
|
||||
rm -f "`$REMOTE_DIR"/ca.crt "`$REMOTE_DIR"/ca.key "`$REMOTE_DIR"/ca.srl \
|
||||
"`$REMOTE_DIR"/worker.crt "`$REMOTE_DIR"/worker.key "`$REMOTE_DIR"/worker.csr \
|
||||
"`$REMOTE_DIR"/ca-openssl.cnf "`$REMOTE_DIR"/worker-openssl.cnf
|
||||
|
||||
cat >"`$REMOTE_DIR/ca-openssl.cnf" <<'EOF_CA'
|
||||
[req]
|
||||
distinguished_name = dn
|
||||
x509_extensions = v3_ca
|
||||
prompt = no
|
||||
|
||||
[dn]
|
||||
CN = RAP Test Platform Root CA
|
||||
|
||||
[v3_ca]
|
||||
basicConstraints = critical,CA:true,pathlen:1
|
||||
keyUsage = critical,keyCertSign,cRLSign
|
||||
subjectKeyIdentifier = hash
|
||||
authorityKeyIdentifier = keyid:always,issuer
|
||||
EOF_CA
|
||||
|
||||
cat >"`$REMOTE_DIR/worker-openssl.cnf" <<EOF_WORKER
|
||||
[req]
|
||||
distinguished_name = dn
|
||||
req_extensions = v3_req
|
||||
prompt = no
|
||||
|
||||
[dn]
|
||||
CN = `$WORKER_ID
|
||||
|
||||
[v3_req]
|
||||
basicConstraints = critical,CA:false
|
||||
keyUsage = critical,digitalSignature,keyEncipherment
|
||||
extendedKeyUsage = serverAuth
|
||||
subjectAltName = @alt_names
|
||||
|
||||
[alt_names]
|
||||
IP.1 = `$WORKER_HOST
|
||||
URI.1 = spiffe://rap/cluster/`$CLUSTER_ID/worker/`$WORKER_ID
|
||||
EOF_WORKER
|
||||
|
||||
if [ -n "`$WORKER_DNS_NAME" ]; then
|
||||
printf 'DNS.1 = %s\n' "`$WORKER_DNS_NAME" >>"`$REMOTE_DIR/worker-openssl.cnf"
|
||||
fi
|
||||
|
||||
openssl genrsa -out "`$REMOTE_DIR/ca.key" 3072 >/dev/null 2>&1
|
||||
openssl req -x509 -new -nodes -key "`$REMOTE_DIR/ca.key" -sha256 -days "`$DAYS" \
|
||||
-out "`$REMOTE_DIR/ca.crt" -config "`$REMOTE_DIR/ca-openssl.cnf" >/dev/null 2>&1
|
||||
|
||||
openssl genrsa -out "`$REMOTE_DIR/worker.key" 3072 >/dev/null 2>&1
|
||||
openssl req -new -key "`$REMOTE_DIR/worker.key" -out "`$REMOTE_DIR/worker.csr" \
|
||||
-config "`$REMOTE_DIR/worker-openssl.cnf" >/dev/null 2>&1
|
||||
openssl x509 -req -in "`$REMOTE_DIR/worker.csr" -CA "`$REMOTE_DIR/ca.crt" -CAkey "`$REMOTE_DIR/ca.key" \
|
||||
-CAcreateserial -out "`$REMOTE_DIR/worker.crt" -days "`$DAYS" -sha256 \
|
||||
-extensions v3_req -extfile "`$REMOTE_DIR/worker-openssl.cnf" >/dev/null 2>&1
|
||||
|
||||
if [ -f /tmp/rap-dp1d1/dp-public.pem ]; then
|
||||
cp /tmp/rap-dp1d1/dp-public.pem "`$REMOTE_DIR/dp-public.pem"
|
||||
elif [ -f /tmp/rap-dp1c/dp-public.pem ]; then
|
||||
cp /tmp/rap-dp1c/dp-public.pem "`$REMOTE_DIR/dp-public.pem"
|
||||
fi
|
||||
|
||||
chmod 600 "`$REMOTE_DIR"/ca.key "`$REMOTE_DIR"/worker.key
|
||||
chmod 644 "`$REMOTE_DIR"/ca.crt "`$REMOTE_DIR"/worker.crt
|
||||
|
||||
echo "remote_dir=`$REMOTE_DIR"
|
||||
echo "ca_cert=`$REMOTE_DIR/ca.crt"
|
||||
echo "worker_cert=`$REMOTE_DIR/worker.crt"
|
||||
echo "worker_key=`$REMOTE_DIR/worker.key"
|
||||
openssl x509 -in "`$REMOTE_DIR/worker.crt" -noout -subject -ext subjectAltName
|
||||
"@
|
||||
|
||||
$remoteScript | & ssh $DockerSshAlias "bash -s"
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "Failed to prepare platform CA and worker certificate on SSH alias '$DockerSshAlias'."
|
||||
}
|
||||
|
||||
$localCaPsPath = if ([System.IO.Path]::IsPathRooted($LocalCaOutputPath)) {
|
||||
$LocalCaOutputPath
|
||||
} else {
|
||||
Join-Path (Resolve-Path -LiteralPath ".").Path $LocalCaOutputPath
|
||||
}
|
||||
$resolvedLocalCaPath = $ExecutionContext.SessionState.Path.GetUnresolvedProviderPathFromPSPath($localCaPsPath)
|
||||
$localCaDirectory = Split-Path -Parent $resolvedLocalCaPath
|
||||
if (-not [string]::IsNullOrWhiteSpace($localCaDirectory)) {
|
||||
New-Item -ItemType Directory -Force -Path $localCaDirectory | Out-Null
|
||||
}
|
||||
|
||||
$remoteCaSource = "$DockerSshAlias`:$RemoteOutputDir/ca.crt"
|
||||
& scp $remoteCaSource $resolvedLocalCaPath
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "Failed to copy platform CA certificate from '$remoteCaSource'."
|
||||
}
|
||||
|
||||
Write-Host "Local platform CA bundle: $resolvedLocalCaPath"
|
||||
Write-Host "Remote worker TLS directory: $RemoteOutputDir"
|
||||
@@ -0,0 +1,19 @@
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
$env:APP_NAME = "rap-api"
|
||||
$env:APP_ENV = "development"
|
||||
$env:HTTP_HOST = "0.0.0.0"
|
||||
$env:HTTP_PORT = "8080"
|
||||
$env:POSTGRES_DSN = "postgres://rap_user:rap_password@127.0.0.1:5432/remote_access_platform?sslmode=disable"
|
||||
$env:REDIS_ADDR = "127.0.0.1:6379"
|
||||
$env:AUTH_ACCESS_TOKEN_SECRET = "local-dev-access-secret"
|
||||
$env:AUTH_REFRESH_HASH_SECRET = "local-dev-refresh-secret"
|
||||
|
||||
Push-Location backend
|
||||
try {
|
||||
go run ./cmd/api
|
||||
}
|
||||
finally {
|
||||
Pop-Location
|
||||
}
|
||||
@@ -0,0 +1,21 @@
|
||||
param(
|
||||
[string]$WorkerId = "rdp-worker-1",
|
||||
[bool]$InsecureSkipVerify = $false
|
||||
)
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
docker rm -f $WorkerId 2>$null | Out-Null
|
||||
|
||||
$envArgs = @(
|
||||
"-e", "RDP_WORKER_ID=$WorkerId",
|
||||
"-e", "RDP_WORKER_REDIS_HOST=host.docker.internal",
|
||||
"-e", "RDP_WORKER_REDIS_PORT=6379"
|
||||
)
|
||||
|
||||
if ($InsecureSkipVerify) {
|
||||
$envArgs += @("-e", "RDP_WORKER_INSECURE_SKIP_VERIFY=true")
|
||||
}
|
||||
|
||||
docker run --rm --name $WorkerId @envArgs rap-rdp-worker:dev
|
||||
@@ -0,0 +1,108 @@
|
||||
param(
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$RdpHost,
|
||||
[int]$RdpPort = 3389,
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$RdpUsername,
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$RdpPassword,
|
||||
[string]$RdpDomain = "",
|
||||
[ValidateSet("strict", "ignore")]
|
||||
[string]$CertificateVerificationMode = "strict"
|
||||
)
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
$userId = [guid]::NewGuid().ToString()
|
||||
$deviceId = [guid]::NewGuid().ToString()
|
||||
$resourceId = [guid]::NewGuid().ToString()
|
||||
$userEmail = "smoke-user-$userId@example.local"
|
||||
|
||||
$metadata = @{
|
||||
rdp_host = $RdpHost
|
||||
rdp_port = $RdpPort
|
||||
username = $RdpUsername
|
||||
password = $RdpPassword
|
||||
domain = $RdpDomain
|
||||
certificate_verification_mode = $CertificateVerificationMode
|
||||
} | ConvertTo-Json -Compress
|
||||
|
||||
$sql = @"
|
||||
INSERT INTO users (id, email, password_hash, mfa_enabled)
|
||||
VALUES ('$userId'::uuid, '$userEmail', '`$2a`$10`$7EqJtq98hPqEX7fNZaFWoOHi6s6i.5NQ32mibXwjlzAIXazhbugzu', FALSE)
|
||||
ON CONFLICT (email) DO NOTHING;
|
||||
|
||||
INSERT INTO organization_memberships (
|
||||
id, organization_id, user_id, role_id, status, invited_by_user_id, created_at, updated_at
|
||||
)
|
||||
SELECT
|
||||
gen_random_uuid(),
|
||||
(SELECT id FROM organizations WHERE slug = 'default'),
|
||||
'$userId'::uuid,
|
||||
'org_member',
|
||||
'active',
|
||||
NULL,
|
||||
NOW(),
|
||||
NOW()
|
||||
ON CONFLICT (organization_id, user_id) DO UPDATE SET
|
||||
status = 'active',
|
||||
updated_at = EXCLUDED.updated_at;
|
||||
|
||||
INSERT INTO devices (id, user_id, device_fingerprint, device_label, trust_status, trusted_at, last_seen_at, created_at, updated_at)
|
||||
VALUES (
|
||||
'$deviceId'::uuid,
|
||||
'$userId'::uuid,
|
||||
'smoke-device-1',
|
||||
'Smoke Device 1',
|
||||
'trusted',
|
||||
NOW(),
|
||||
NOW(),
|
||||
NOW(),
|
||||
NOW()
|
||||
)
|
||||
ON CONFLICT (user_id, device_fingerprint) DO UPDATE SET
|
||||
trust_status = 'trusted',
|
||||
trusted_at = NOW(),
|
||||
last_seen_at = NOW(),
|
||||
updated_at = NOW();
|
||||
|
||||
INSERT INTO resources (id, organization_id, name, address, protocol, certificate_verification_mode, metadata, created_at, updated_at)
|
||||
SELECT
|
||||
'$resourceId'::uuid,
|
||||
(SELECT id FROM organizations WHERE slug = 'default'),
|
||||
'Smoke RDP Resource',
|
||||
'$RdpHost',
|
||||
'rdp',
|
||||
'$CertificateVerificationMode',
|
||||
'$metadata'::jsonb,
|
||||
NOW(),
|
||||
NOW()
|
||||
ON CONFLICT (id) DO NOTHING;
|
||||
|
||||
INSERT INTO resource_policies (
|
||||
resource_id, max_concurrent_sessions, takeover_policy, require_trusted_device,
|
||||
detach_grace_period_seconds, clipboard_enabled, file_transfer_enabled, created_at, updated_at
|
||||
)
|
||||
VALUES (
|
||||
'$resourceId'::uuid,
|
||||
1,
|
||||
'trusted_device',
|
||||
TRUE,
|
||||
1800,
|
||||
FALSE,
|
||||
FALSE,
|
||||
NOW(),
|
||||
NOW()
|
||||
)
|
||||
ON CONFLICT (resource_id) DO UPDATE SET
|
||||
updated_at = NOW();
|
||||
"@
|
||||
|
||||
$sql | docker exec -i rap_postgres psql -U rap_user -d remote_access_platform -v ON_ERROR_STOP=1 -f -
|
||||
|
||||
Write-Host "Seed complete"
|
||||
Write-Host "email=$userEmail"
|
||||
Write-Host "user_id=$userId"
|
||||
Write-Host "device_id=$deviceId"
|
||||
Write-Host "resource_id=$resourceId"
|
||||
@@ -0,0 +1,4 @@
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
docker compose up -d postgres redis
|
||||
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user