Refactor RDP proxy handling and update related tests

This commit is contained in:
2026-05-17 20:38:35 +03:00
parent 8e9402580f
commit d551e57fd5
172 changed files with 22117 additions and 2509 deletions
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,760 @@
package main
import (
"bytes"
"context"
"strings"
"testing"
"time"
"github.com/example/remote-access-platform/agents/rap-node-agent/internal/mesh"
)
func TestRouteModeCoverageVerdictRequiresMixedModes(t *testing.T) {
report := loadtestReport{
Config: loadtestConfig{
TopologyProfile: "mixed-public-nat-lan-relay",
Targets: []string{"a", "b", "c", "d"},
FailTarget: -1,
ImpairTarget: -1,
},
SuccessfulStreams: 4,
TargetStats: map[string]targetStats{
"a": {RouteModes: map[string]int{string(mesh.FabricRouteLAN): 1}},
"b": {RouteModes: map[string]int{string(mesh.FabricRouteICE): 1}},
"c": {RouteModes: map[string]int{string(mesh.FabricRouteReverse): 1}},
"d": {RouteModes: map[string]int{}},
},
}
reasons := routeModeCoverageVerdictReasons(report)
if len(reasons) != 1 || !strings.Contains(reasons[0], string(mesh.FabricRouteRelay)) {
t.Fatalf("reasons = %v, want missing relay route mode", reasons)
}
report.TargetStats["d"] = targetStats{RouteModes: map[string]int{string(mesh.FabricRouteRelay): 1}}
if reasons := routeModeCoverageVerdictReasons(report); len(reasons) != 0 {
t.Fatalf("reasons = %v, want full coverage pass", reasons)
}
}
func TestLegacyRouteModeVerdictRejectsNonQUICModes(t *testing.T) {
report := loadtestReport{
TargetStats: map[string]targetStats{
"a": {RouteModes: map[string]int{
"direct_quic": 4,
"relay": 1,
"outbound_reverse": 2,
"wss": 3,
}},
},
}
reasons := legacyRouteModeVerdictReasons(report)
if len(reasons) != 1 ||
!strings.Contains(reasons[0], "relay:1") ||
!strings.Contains(reasons[0], "outbound_reverse:2") ||
!strings.Contains(reasons[0], "wss:3") {
t.Fatalf("reasons = %v, want legacy route mode failure", reasons)
}
report.TargetStats["a"] = targetStats{RouteModes: map[string]int{
string(mesh.FabricRouteDirect): 1,
string(mesh.FabricRouteLAN): 1,
string(mesh.FabricRouteICE): 1,
string(mesh.FabricRouteReverse): 1,
string(mesh.FabricRouteRelay): 1,
}}
if reasons := legacyRouteModeVerdictReasons(report); len(reasons) != 0 {
t.Fatalf("reasons = %v, want QUIC modes accepted", reasons)
}
}
func TestTargetEndpointPolicyVerdictRejectsNonQUICTargets(t *testing.T) {
report := loadtestReport{
Config: loadtestConfig{
Targets: []string{
"quic://a:19443",
"http://b:19443",
"ws://c:19443",
"d:19443",
"",
},
},
}
reasons := targetEndpointPolicyVerdictReasons(report)
if len(reasons) != 1 ||
!strings.Contains(reasons[0], "http://b:19443") ||
!strings.Contains(reasons[0], "ws://c:19443") ||
!strings.Contains(reasons[0], "d:19443") ||
!strings.Contains(reasons[0], "<empty>") {
t.Fatalf("reasons = %v, want non-QUIC target failure", reasons)
}
report.Config.Targets = []string{"quic://a:19443", " QUIC://b:19443 "}
if reasons := targetEndpointPolicyVerdictReasons(report); len(reasons) != 0 {
t.Fatalf("reasons = %v, want QUIC targets accepted", reasons)
}
}
func TestRunClientRejectsNonQUICTargetBeforeDial(t *testing.T) {
_, err := runClient(context.Background(), loadtestConfig{
Targets: []string{"http://127.0.0.1:19443"},
Streams: 1,
Concurrency: 1,
BytesPerStream: 1,
PayloadSize: 1,
})
if err == nil || !strings.Contains(err.Error(), "non_quic_targets=http://127.0.0.1:19443") {
t.Fatalf("err = %v, want non-QUIC target validation error", err)
}
}
func TestFillLoadtestPayloadVariesByStreamAndSequence(t *testing.T) {
first := make([]byte, 128)
second := make([]byte, 128)
third := make([]byte, 128)
fillLoadtestPayload(first, 7, 9, 1, 0)
fillLoadtestPayload(second, 7, 9, 2, int64(len(first)))
fillLoadtestPayload(third, 8, 10, 1, 0)
if bytes.Equal(first, second) {
t.Fatal("payload did not vary by sequence/offset")
}
if bytes.Equal(first, third) {
t.Fatal("payload did not vary by stream")
}
if bytes.Count(first, []byte{first[0]}) == len(first) {
t.Fatal("payload collapsed to a constant byte")
}
}
func TestFillLoadtestPayloadIsDeterministic(t *testing.T) {
first := make([]byte, 128)
second := make([]byte, 128)
fillLoadtestPayload(first, 7, 9, 1, 0)
fillLoadtestPayload(second, 7, 9, 1, 0)
if !bytes.Equal(first, second) {
t.Fatal("payload is not deterministic")
}
}
func TestFillLoadtestPayloadHandlesShortFinalChunk(t *testing.T) {
chunk := make([]byte, 17)
fillLoadtestPayload(chunk, 7, 9, 3, 256)
if bytes.Equal(chunk, make([]byte, len(chunk))) {
t.Fatal("short payload chunk stayed zeroed")
}
}
func TestVerdictFailsSuccessfulStreamAckMismatch(t *testing.T) {
report := loadtestReport{
Config: loadtestConfig{
FailTarget: -1,
ImpairTarget: -1,
Concurrency: 1,
},
TotalStreams: 1,
SuccessfulStreams: 1,
BytesSent: 1024,
FramesSent: 2,
AcksReceived: 1,
AckMismatchedStreams: 1,
ChannelOpens: 1,
ChannelCloses: 1,
RoutePressure: mesh.FabricRoutePressureSnapshot{AcquiredTotal: 1, ReleasedTotal: 1, MaxActiveTotal: 1},
}
gotVerdict, reasons := verdict(report)
if gotVerdict != "fail" {
t.Fatalf("verdict = %q, want fail", gotVerdict)
}
found := false
for _, reason := range reasons {
if reason == "ack_mismatched_streams=1" {
found = true
}
}
if !found {
t.Fatalf("reasons = %v, want ack mismatch reason", reasons)
}
}
func TestVerdictFailsAckIntegrityError(t *testing.T) {
report := loadtestReport{
Config: loadtestConfig{
FailTarget: -1,
ImpairTarget: -1,
Concurrency: 1,
},
TotalStreams: 1,
FailedStreams: 1,
BytesSent: 1024,
FramesSent: 1,
AcksReceived: 1,
AckIntegrityErrors: 1,
ChannelOpens: 1,
ChannelCloses: 1,
RoutePressure: mesh.FabricRoutePressureSnapshot{AcquiredTotal: 1, ReleasedTotal: 1, MaxActiveTotal: 1},
}
gotVerdict, reasons := verdict(report)
if gotVerdict != "fail" {
t.Fatalf("verdict = %q, want fail", gotVerdict)
}
found := false
for _, reason := range reasons {
if reason == "ack_integrity_errors=1" {
found = true
}
}
if !found {
t.Fatalf("reasons = %v, want ack integrity reason", reasons)
}
}
func TestVerdictFailsBelowMinimumThroughput(t *testing.T) {
report := loadtestReport{
Config: loadtestConfig{
FailTarget: -1,
ImpairTarget: -1,
Concurrency: 1,
MinThroughputMbps: 100,
},
TotalStreams: 1,
SuccessfulStreams: 1,
BytesSent: 1024,
FramesSent: 1,
AcksReceived: 1,
ThroughputBps: 99 * 1000 * 1000,
ChannelOpens: 1,
ChannelCloses: 1,
RoutePressure: mesh.FabricRoutePressureSnapshot{AcquiredTotal: 1, ReleasedTotal: 1, MaxActiveTotal: 1},
}
gotVerdict, reasons := verdict(report)
if gotVerdict != "fail" {
t.Fatalf("verdict = %q, want fail", gotVerdict)
}
found := false
for _, reason := range reasons {
if strings.HasPrefix(reason, "throughput_bps=") {
found = true
}
}
if !found {
t.Fatalf("reasons = %v, want throughput reason", reasons)
}
report.ThroughputBps = 100 * 1000 * 1000
if gotVerdict, reasons := verdict(report); gotVerdict != "pass" {
t.Fatalf("verdict = %q reasons=%v, want pass at threshold", gotVerdict, reasons)
}
}
func TestVerdictFailsBelowMinimumChannelChurn(t *testing.T) {
report := loadtestReport{
Config: loadtestConfig{
FailTarget: -1,
ImpairTarget: -1,
Concurrency: 1,
MinChannelChurn: 1000,
},
TotalStreams: 1,
SuccessfulStreams: 1,
BytesSent: 1024,
FramesSent: 1,
AcksReceived: 1,
ChannelOpens: 1,
ChannelCloses: 1,
ChannelChurnPerSec: 999,
RoutePressure: mesh.FabricRoutePressureSnapshot{AcquiredTotal: 1, ReleasedTotal: 1, MaxActiveTotal: 1},
}
gotVerdict, reasons := verdict(report)
if gotVerdict != "fail" {
t.Fatalf("verdict = %q, want fail", gotVerdict)
}
found := false
for _, reason := range reasons {
if strings.HasPrefix(reason, "channel_churn_per_sec=") {
found = true
}
}
if !found {
t.Fatalf("reasons = %v, want channel churn reason", reasons)
}
report.ChannelChurnPerSec = 1000
if gotVerdict, reasons := verdict(report); gotVerdict != "pass" {
t.Fatalf("verdict = %q reasons=%v, want pass at threshold", gotVerdict, reasons)
}
}
func TestTargetByteDistributionVerdictDetectsSkew(t *testing.T) {
report := loadtestReport{
Config: loadtestConfig{
Targets: []string{"a", "b", "c", "d"},
FailTarget: -1,
ImpairTarget: -1,
Concurrency: 1,
BytesPerStream: 100,
},
SuccessfulStreams: 40,
BytesSent: 4000,
TargetStreams: map[string]int{
"a": 10,
"b": 10,
"c": 10,
"d": 10,
},
TargetBytes: map[string]int64{
"a": 2500,
"b": 500,
"c": 500,
"d": 500,
},
}
reasons := targetByteDistributionVerdictReasons(report)
if len(reasons) != 1 || !strings.HasPrefix(reasons[0], "target_byte_distribution_skew=") {
t.Fatalf("reasons = %v, want byte skew reason", reasons)
}
report.TargetBytes = map[string]int64{
"a": 1000,
"b": 1000,
"c": 1000,
"d": 1000,
}
if reasons := targetByteDistributionVerdictReasons(report); len(reasons) != 0 {
t.Fatalf("reasons = %v, want balanced bytes pass", reasons)
}
}
func TestDistributionVerdictChecksSurvivingTargetsAfterFailure(t *testing.T) {
report := loadtestReport{
Config: loadtestConfig{
Targets: []string{"quic://a:1", "quic://b:1", "quic://c:1", "quic://d:1"},
FailTarget: 0,
ImpairTarget: -1,
Concurrency: 8,
},
SuccessfulStreams: 90,
TargetStreams: map[string]int{
"quic://b:1": 90,
},
}
reasons := targetDistributionVerdictReasons(report)
if len(reasons) != 1 || !strings.HasPrefix(reasons[0], "target_distribution_collapsed=1/3_targets_used") {
t.Fatalf("reasons = %v, want surviving-target collapse", reasons)
}
report.TargetStreams = map[string]int{
"quic://b:1": 30,
"quic://c:1": 30,
"quic://d:1": 30,
}
if reasons := targetDistributionVerdictReasons(report); len(reasons) != 0 {
t.Fatalf("reasons = %v, want balanced surviving targets pass", reasons)
}
}
func TestRoutePressureVerdictChecksSurvivingTargetsAfterFailure(t *testing.T) {
targets := []string{"quic://a:1", "quic://b:1", "quic://c:1", "quic://d:1"}
report := loadtestReport{
Config: loadtestConfig{
Targets: targets,
FailTarget: 0,
ImpairTarget: -1,
Concurrency: 12,
},
RoutePressure: mesh.FabricRoutePressureSnapshot{
MaxActive: map[string]int{
loadtestRouteID(1, targets[1]): 12,
},
MaxActiveTotal: 12,
},
}
reasons := routePressureDistributionVerdictReasons(report)
if len(reasons) != 1 || !strings.HasPrefix(reasons[0], "route_pressure_distribution_collapsed=1/3_targets_used") {
t.Fatalf("reasons = %v, want surviving-route-pressure collapse", reasons)
}
report.RoutePressure.MaxActive = map[string]int{
loadtestRouteID(1, targets[1]): 4,
loadtestRouteID(2, targets[2]): 4,
loadtestRouteID(3, targets[3]): 4,
}
if reasons := routePressureDistributionVerdictReasons(report); len(reasons) != 0 {
t.Fatalf("reasons = %v, want balanced surviving route pressure pass", reasons)
}
}
func TestVerdictFailsOverallAckLatencySLO(t *testing.T) {
report := loadtestReport{
Config: loadtestConfig{
FailTarget: -1,
ImpairTarget: -1,
Concurrency: 1,
MaxAckP95Ms: 10,
MaxAckP99Ms: 20,
},
TotalStreams: 1,
SuccessfulStreams: 1,
BytesSent: 1024,
FramesSent: 1,
AcksReceived: 1,
AckP95Ms: 11,
AckP99Ms: 21,
ChannelOpens: 1,
ChannelCloses: 1,
RoutePressure: mesh.FabricRoutePressureSnapshot{AcquiredTotal: 1, ReleasedTotal: 1, MaxActiveTotal: 1},
}
gotVerdict, reasons := verdict(report)
if gotVerdict != "fail" {
t.Fatalf("verdict = %q, want fail", gotVerdict)
}
foundP95 := false
foundP99 := false
for _, reason := range reasons {
if strings.HasPrefix(reason, "ack_p95_ms=") {
foundP95 = true
}
if strings.HasPrefix(reason, "ack_p99_ms=") {
foundP99 = true
}
}
if !foundP95 || !foundP99 {
t.Fatalf("reasons = %v, want ACK p95 and p99 reasons", reasons)
}
}
func TestTargetAckVerdictDetectsSlowHealthyTarget(t *testing.T) {
report := loadtestReport{
Config: loadtestConfig{
Targets: []string{"a", "b"},
FailTarget: -1,
ImpairTarget: -1,
MaxTargetAckMs: 10,
},
TargetStats: map[string]targetStats{
"a": {Streams: 10, MaxAckMs: 4},
"b": {Streams: 10, MaxAckMs: 11},
},
}
reasons := targetAckVerdictReasons(report)
if len(reasons) != 1 || !strings.HasPrefix(reasons[0], "target_ack_ms=b:11>10") {
t.Fatalf("reasons = %v, want slow target ack reason", reasons)
}
report.TargetStats["b"] = targetStats{Streams: 10, MaxAckMs: 10}
if reasons := targetAckVerdictReasons(report); len(reasons) != 0 {
t.Fatalf("reasons = %v, want target ack pass at threshold", reasons)
}
}
func TestVerdictFailsSetupLatencySLO(t *testing.T) {
report := loadtestReport{
Config: loadtestConfig{
FailTarget: -1,
ImpairTarget: -1,
Concurrency: 1,
MaxSetupP95Ms: 10,
MaxSetupP99Ms: 20,
},
TotalStreams: 1,
SuccessfulStreams: 1,
BytesSent: 1024,
FramesSent: 1,
AcksReceived: 1,
SetupLatencyP95Ms: 11,
SetupLatencyP99Ms: 21,
ChannelOpens: 1,
ChannelCloses: 1,
RoutePressure: mesh.FabricRoutePressureSnapshot{AcquiredTotal: 1, ReleasedTotal: 1, MaxActiveTotal: 1},
}
gotVerdict, reasons := verdict(report)
if gotVerdict != "fail" {
t.Fatalf("verdict = %q, want fail", gotVerdict)
}
foundP95 := false
foundP99 := false
for _, reason := range reasons {
if strings.HasPrefix(reason, "setup_p95_ms=") {
foundP95 = true
}
if strings.HasPrefix(reason, "setup_p99_ms=") {
foundP99 = true
}
}
if !foundP95 || !foundP99 {
t.Fatalf("reasons = %v, want setup p95 and p99 reasons", reasons)
}
}
func TestVerdictFailsRerouteLatencySLO(t *testing.T) {
report := loadtestReport{
Config: loadtestConfig{
FailTarget: -1,
ImpairTarget: -1,
Concurrency: 1,
MaxRerouteP95Ms: 10,
MaxRerouteP99Ms: 20,
},
TotalStreams: 1,
SuccessfulStreams: 1,
BytesSent: 1024,
FramesSent: 1,
AcksReceived: 1,
RerouteLatencyP95Ms: 11,
RerouteLatencyP99Ms: 21,
ChannelOpens: 1,
ChannelCloses: 1,
RoutePressure: mesh.FabricRoutePressureSnapshot{AcquiredTotal: 1, ReleasedTotal: 1, MaxActiveTotal: 1},
}
gotVerdict, reasons := verdict(report)
if gotVerdict != "fail" {
t.Fatalf("verdict = %q, want fail", gotVerdict)
}
foundP95 := false
foundP99 := false
for _, reason := range reasons {
if strings.HasPrefix(reason, "reroute_p95_ms=") {
foundP95 = true
}
if strings.HasPrefix(reason, "reroute_p99_ms=") {
foundP99 = true
}
}
if !foundP95 || !foundP99 {
t.Fatalf("reasons = %v, want reroute p95 and p99 reasons", reasons)
}
}
func TestShouldQuarantineTarget(t *testing.T) {
quarantined := []string{
"ack timeout or session closed",
"deadline exceeded",
"connection refused",
"connection reset by peer",
"no route to host",
}
for _, reason := range quarantined {
if !shouldQuarantineTarget(reason) {
t.Fatalf("shouldQuarantineTarget(%q) = false, want true", reason)
}
}
if shouldQuarantineTarget("ack payload checksum mismatch") {
t.Fatal("checksum mismatch should not quarantine a target")
}
if shouldQuarantineTarget("context deadline exceeded") {
t.Fatal("context deadline should not quarantine a target")
}
}
func TestSpreadStartDistributesQuarantinedSlot(t *testing.T) {
targets := []string{"a", "b", "c", "d"}
health := newTargetHealthTracker()
health.MarkDegraded("a", "connection refused", time.Minute)
counts := map[string]int{}
for index := 0; index < 40; index += len(targets) {
initial, spread := loadtestSpreadStart(index, len(targets))
targetIndex := loadtestPreferredTargetIndex(targets, initial, spread, health, -1)
counts[targets[targetIndex]]++
}
if counts["b"] == 0 || counts["c"] == 0 || counts["d"] == 0 {
t.Fatalf("counts = %v, want degraded slot spread across surviving targets", counts)
}
}
func TestSpreadUsableTargetDistributesRetries(t *testing.T) {
targets := []string{"a", "b", "c", "d"}
health := newTargetHealthTracker()
health.MarkDegraded("a", "connection refused", time.Minute)
counts := map[string]int{}
for cohort := 0; cohort < 90; cohort++ {
targetIndex := loadtestSpreadUsableTargetIndex(targets, cohort, health, 0)
counts[targets[targetIndex]]++
}
if counts["b"] != 30 || counts["c"] != 30 || counts["d"] != 30 {
t.Fatalf("counts = %v, want retry load spread evenly across surviving targets", counts)
}
}
func TestLoadtestLogicalStreamIDAvoidsReservedTransportStreams(t *testing.T) {
for _, index := range []int{-1, 0, 1, 999, 1000, 10_000} {
streamID := loadtestLogicalStreamID(index)
if streamID == mesh.ProductionForwardQUICStreamID || streamID == mesh.SyntheticForwardQUICStreamID {
t.Fatalf("loadtestLogicalStreamID(%d) = %d, collides with reserved transport stream", index, streamID)
}
if streamID < 10_000 {
t.Fatalf("loadtestLogicalStreamID(%d) = %d, want loadtest stream range", index, streamID)
}
}
}
func TestLatencyAwareTargetIndexKeepsSlowWANFromOwningPool(t *testing.T) {
targets := []string{"lan-a", "lan-b", "wan"}
health := newTargetHealthTracker()
health.RecordProbes([]targetProbeResult{
{Target: "lan-a", RTTMs: 4, Usable: true},
{Target: "lan-b", RTTMs: 5, Usable: true},
{Target: "wan", RTTMs: 400, Usable: true},
})
counts := map[string]int{}
for index := 0; index < 300; index++ {
targetIndex := loadtestSpreadUsableTargetIndex(targets, index, health, -1)
counts[targets[targetIndex]]++
}
if counts["wan"] == 0 {
t.Fatalf("counts = %v, want slow WAN to stay represented", counts)
}
if counts["wan"] >= counts["lan-a"] || counts["wan"] >= counts["lan-b"] {
t.Fatalf("counts = %v, want latency-aware placement to prefer LAN capacity", counts)
}
}
func TestLatencyAwarePreferredTargetUsesAbsolutePlacementOrdinal(t *testing.T) {
targets := []string{"lan-a", "lan-b", "lan-c", "wan"}
health := newTargetHealthTracker()
health.RecordProbes([]targetProbeResult{
{Target: "lan-a", RTTMs: 4, Usable: true},
{Target: "lan-b", RTTMs: 4, Usable: true},
{Target: "lan-c", RTTMs: 4, Usable: true},
{Target: "wan", RTTMs: 400, Usable: true},
})
counts := map[string]int{}
for index := 0; index < 500; index++ {
preferred, spread := loadtestSpreadStart(index, len(targets))
targetIndex := loadtestPreferredTargetIndex(targets, preferred, spread, health, -1)
counts[targets[targetIndex]]++
}
if len(counts) < len(targets) {
t.Fatalf("counts = %v, want every probed target represented", counts)
}
if counts["wan"] >= counts["lan-a"] || counts["wan"] >= counts["lan-b"] || counts["wan"] >= counts["lan-c"] {
t.Fatalf("counts = %v, want slow WAN weighted below LAN targets", counts)
}
}
func TestHeterogeneousProbeRTTRelaxesEqualDistributionVerdict(t *testing.T) {
report := loadtestReport{
Config: loadtestConfig{
Targets: []string{"lan", "wan"},
Concurrency: 64,
},
SuccessfulStreams: 100,
BytesSent: 100 * 1024,
TargetStreams: map[string]int{
"lan": 96,
"wan": 4,
},
TargetBytes: map[string]int64{
"lan": 96 * 1024,
"wan": 4 * 1024,
},
TargetProbes: []targetProbeResult{
{Target: "lan", RTTMs: 4, Usable: true},
{Target: "wan", RTTMs: 400, Usable: true},
},
RoutePressure: mesh.FabricRoutePressureSnapshot{
MaxActive: map[string]int{
loadtestRouteID(0, "lan"): 32,
loadtestRouteID(1, "wan"): 1,
},
MaxActiveTotal: 32,
},
}
if reasons := targetDistributionVerdictReasons(report); len(reasons) != 0 {
t.Fatalf("targetDistributionVerdictReasons = %v, want heterogeneous RTT tolerated", reasons)
}
if reasons := targetByteDistributionVerdictReasons(report); len(reasons) != 0 {
t.Fatalf("targetByteDistributionVerdictReasons = %v, want heterogeneous RTT tolerated", reasons)
}
if reasons := routePressureDistributionVerdictReasons(report); len(reasons) != 0 {
t.Fatalf("routePressureDistributionVerdictReasons = %v, want heterogeneous RTT tolerated", reasons)
}
}
func TestTargetHealthQuarantineExpiresButSnapshotKeepsObservation(t *testing.T) {
health := newTargetHealthTracker()
health.MarkDegraded("a", "ack timeout", time.Nanosecond)
if !health.IsDegraded("a") {
t.Fatal("target should be degraded immediately")
}
time.Sleep(time.Millisecond)
if health.IsDegraded("a") {
t.Fatal("target quarantine did not expire")
}
snapshot := health.Snapshot()
if snapshot["a"] != "ack timeout" {
t.Fatalf("snapshot = %v, want historical degraded observation", snapshot)
}
}
func TestRoutePressureDistributionVerdictDetectsCollapse(t *testing.T) {
report := loadtestReport{
Config: loadtestConfig{
Targets: []string{"a", "b", "c", "d"},
FailTarget: -1,
ImpairTarget: -1,
Concurrency: 16,
},
RoutePressure: mesh.FabricRoutePressureSnapshot{
MaxActive: map[string]int{
loadtestRouteID(0, "a"): 16,
},
MaxActiveTotal: 16,
},
}
reasons := routePressureDistributionVerdictReasons(report)
if len(reasons) != 1 || !strings.HasPrefix(reasons[0], "route_pressure_distribution_collapsed=") {
t.Fatalf("reasons = %v, want collapsed route pressure reason", reasons)
}
}
func TestRoutePressureDistributionVerdictDetectsSkew(t *testing.T) {
report := loadtestReport{
Config: loadtestConfig{
Targets: []string{"a", "b", "c", "d"},
FailTarget: -1,
ImpairTarget: -1,
Concurrency: 16,
},
RoutePressure: mesh.FabricRoutePressureSnapshot{
MaxActive: map[string]int{
loadtestRouteID(0, "a"): 14,
loadtestRouteID(1, "b"): 2,
loadtestRouteID(2, "c"): 2,
loadtestRouteID(3, "d"): 2,
},
MaxActiveTotal: 16,
},
}
reasons := routePressureDistributionVerdictReasons(report)
if len(reasons) != 1 || !strings.HasPrefix(reasons[0], "route_pressure_distribution_skew=") {
t.Fatalf("reasons = %v, want route pressure skew reason", reasons)
}
report.RoutePressure.MaxActive = map[string]int{
loadtestRouteID(0, "a"): 6,
loadtestRouteID(1, "b"): 6,
loadtestRouteID(2, "c"): 5,
loadtestRouteID(3, "d"): 5,
}
if reasons := routePressureDistributionVerdictReasons(report); len(reasons) != 0 {
t.Fatalf("reasons = %v, want balanced route pressure pass", reasons)
}
}
@@ -0,0 +1,199 @@
package main
import (
"context"
"crypto/sha256"
"encoding/base64"
"encoding/hex"
"encoding/json"
"errors"
"flag"
"fmt"
"os"
"strings"
"time"
"github.com/example/remote-access-platform/agents/rap-node-agent/internal/fabricproto"
"github.com/example/remote-access-platform/agents/rap-node-agent/internal/mesh"
)
type smokeOutput struct {
OK bool `json:"ok"`
Endpoint string `json:"endpoint"`
EntryNodeID string `json:"entry_node_id"`
NextHopID string `json:"next_hop_node_id"`
RouteID string `json:"route_id"`
ElapsedMS int64 `json:"elapsed_ms"`
Result mesh.ProductionForwardResult `json:"result"`
Error string `json:"error,omitempty"`
EnvelopePath []string `json:"envelope_path,omitempty"`
}
type productionForwardResponse struct {
Result mesh.ProductionForwardResult `json:"result,omitempty"`
Error string `json:"error,omitempty"`
}
func main() {
var (
endpoint = flag.String("endpoint", "", "QUIC fabric endpoint for the entry node, for example quic://host:19131.")
peerCert = flag.String("peer-cert-sha256", "", "Expected entry node QUIC TLS certificate SHA-256 fingerprint.")
clusterID = flag.String("cluster-id", "", "Cluster ID.")
routeID = flag.String("route-id", "", "Configured production route ID.")
sourceNodeID = flag.String("source-node-id", "", "Route source node ID.")
destNodeID = flag.String("destination-node-id", "", "Route destination node ID.")
currentNodeID = flag.String("current-hop-node-id", "", "Current hop node ID expected by the entry node.")
nextHopNodeID = flag.String("next-hop-node-id", "", "Next hop node ID from the entry node.")
routePath = flag.String("route-path", "", "Comma-separated route path.")
channel = flag.String("channel", mesh.ProductionChannelFabricControl, "Production channel class.")
timeout = flag.Duration("timeout", 10*time.Second, "Smoke request timeout.")
payloadText = flag.String("payload", `{"kind":"fabric-production-smoke"}`, "JSON payload string.")
payloadB64 = flag.String("payload-b64", "", "Base64-encoded JSON payload string.")
)
flag.Parse()
if *endpoint == "" || *clusterID == "" || *routeID == "" || *sourceNodeID == "" || *destNodeID == "" || *currentNodeID == "" || *nextHopNodeID == "" {
writeOutput(smokeOutput{OK: false, Error: "endpoint, cluster-id, route-id, source-node-id, destination-node-id, current-hop-node-id and next-hop-node-id are required"})
os.Exit(2)
}
path := splitRoutePath(*routePath)
payloadSource := strings.TrimSpace(*payloadText)
if strings.TrimSpace(*payloadB64) != "" {
decoded, err := base64.StdEncoding.DecodeString(strings.TrimSpace(*payloadB64))
if err != nil {
writeOutput(smokeOutput{OK: false, Error: "payload-b64 must be valid base64"})
os.Exit(2)
}
payloadSource = string(decoded)
}
payload := json.RawMessage(strings.TrimSpace(payloadSource))
if !json.Valid(payload) {
writeOutput(smokeOutput{OK: false, Error: "payload must be valid JSON"})
os.Exit(2)
}
now := time.Now().UTC()
messageType := mesh.ProductionMessageFabricControl
if strings.TrimSpace(*channel) == mesh.ProductionChannelVPNPacket {
messageType = mesh.ProductionMessageVPNPacketBatch
}
sum := sha256.Sum256(payload)
envelope := mesh.ProductionEnvelope{
FabricProtocolVersion: mesh.ProtocolVersion,
MessageID: fmt.Sprintf("fabric-production-smoke-%d", now.UnixNano()),
RouteID: strings.TrimSpace(*routeID),
ClusterID: strings.TrimSpace(*clusterID),
SourceNodeID: strings.TrimSpace(*sourceNodeID),
DestinationNodeID: strings.TrimSpace(*destNodeID),
CurrentHopNodeID: strings.TrimSpace(*currentNodeID),
NextHopNodeID: strings.TrimSpace(*nextHopNodeID),
RoutePath: path,
ChannelClass: strings.TrimSpace(*channel),
MessageType: messageType,
TTL: 8,
HopCount: 0,
CreatedAt: now,
ExpiresAt: now.Add(time.Minute),
PayloadLength: len(payload),
PayloadHash: hex.EncodeToString(sum[:]),
Payload: payload,
}
transport := mesh.NewQUICFabricTransport(nil)
ctx, cancel := context.WithTimeout(context.Background(), *timeout)
defer cancel()
started := time.Now()
result, err := sendProductionEnvelope(ctx, transport, mesh.FabricTransportTarget{
EndpointID: "fabric-production-smoke-entry",
PeerID: envelope.CurrentHopNodeID,
Endpoint: strings.TrimSpace(*endpoint),
Transport: "quic",
PeerCertSHA256: strings.TrimSpace(*peerCert),
Timeout: *timeout,
InboundBuffer: 8,
ErrorBuffer: 4,
}, envelope)
output := smokeOutput{
OK: err == nil && result.Accepted,
Endpoint: *endpoint,
EntryNodeID: envelope.CurrentHopNodeID,
NextHopID: envelope.NextHopNodeID,
RouteID: envelope.RouteID,
ElapsedMS: time.Since(started).Milliseconds(),
Result: result,
EnvelopePath: path,
}
if err != nil {
output.Error = err.Error()
writeOutput(output)
os.Exit(1)
}
writeOutput(output)
}
func sendProductionEnvelope(ctx context.Context, transport *mesh.QUICFabricTransport, target mesh.FabricTransportTarget, envelope mesh.ProductionEnvelope) (mesh.ProductionForwardResult, error) {
session, err := transport.Connect(ctx, target)
if err != nil {
return mesh.ProductionForwardResult{}, err
}
defer session.Close()
payload, err := json.Marshal(envelope)
if err != nil {
return mesh.ProductionForwardResult{}, err
}
if err := session.Send(ctx, fabricproto.Frame{
Type: fabricproto.FrameData,
TrafficClass: fabricproto.TrafficClassReliable,
StreamID: mesh.ProductionForwardQUICStreamID,
Sequence: 1,
Payload: payload,
}); err != nil {
return mesh.ProductionForwardResult{}, err
}
for {
select {
case <-ctx.Done():
return mesh.ProductionForwardResult{}, ctx.Err()
case err := <-session.Errors():
if err != nil {
return mesh.ProductionForwardResult{}, err
}
case frame := <-session.Frames():
if frame.Type != fabricproto.FrameData || frame.StreamID != mesh.ProductionForwardQUICStreamID || frame.Sequence != 1 {
continue
}
var response productionForwardResponse
if err := json.Unmarshal(frame.Payload, &response); err != nil {
return mesh.ProductionForwardResult{}, err
}
if strings.TrimSpace(response.Error) != "" {
return mesh.ProductionForwardResult{}, errors.New(response.Error)
}
return response.Result, nil
}
}
}
func splitRoutePath(value string) []string {
value = strings.TrimSpace(value)
if value == "" {
return nil
}
parts := strings.Split(value, ",")
out := make([]string, 0, len(parts))
for _, part := range parts {
part = strings.TrimSpace(part)
if part != "" {
out = append(out, part)
}
}
return out
}
func writeOutput(output smokeOutput) {
payload, err := json.MarshalIndent(output, "", " ")
if err != nil {
fmt.Fprintf(os.Stderr, "marshal smoke output: %v\n", err)
return
}
fmt.Println(string(payload))
}
@@ -28,6 +28,18 @@ type smokeNode struct {
server *httptest.Server
}
type smokeSyntheticTransport struct {
peers map[string]string
}
func (t smokeSyntheticTransport) SendSynthetic(ctx context.Context, nextNodeID string, envelope mesh.SyntheticEnvelope) (mesh.SyntheticEnvelope, error) {
baseURL := t.peers[nextNodeID]
if baseURL == "" {
return mesh.SyntheticEnvelope{}, mesh.ErrSyntheticPeerUnavailable
}
return mesh.NewClient(baseURL).SendSynthetic(ctx, envelope)
}
type smokeReport struct {
Stage string `json:"stage"`
ProductionForwarding bool `json:"production_forwarding"`
@@ -433,7 +445,7 @@ func writeSmokeScopedConfig(local mesh.PeerIdentity, peers map[string]string, ro
func newSmokeNode(local mesh.PeerIdentity) *smokeNode {
node := &smokeNode{Local: local}
node.server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
mesh.Server{Local: node.Local, SyntheticRuntime: node.Runtime, FabricSessionEnabled: true}.Handler().ServeHTTP(w, r)
mesh.Server{Local: node.Local, SyntheticRuntime: node.Runtime, FabricSessionEnabled: true, FabricSessionWebSocketEnabled: true}.Handler().ServeHTTP(w, r)
}))
node.URL = node.server.URL
return node
@@ -454,7 +466,7 @@ func smokeRuntime(local mesh.PeerIdentity, routes []mesh.SyntheticRoute, peers m
mesh.SyntheticChannelFabricControl,
mesh.SyntheticChannelRouteControl,
},
Transport: mesh.NewHTTPPeerTransport(peers),
Transport: smokeSyntheticTransport{peers: peers},
})
}
@@ -217,7 +217,7 @@ func runInstallLinux(ctx context.Context, args []string) error {
fs.BoolVar(&cfg.RuntimeConfig.WorkloadSupervisionEnabled, "workload-supervision-enabled", getenvBool("RAP_WORKLOAD_SUPERVISION_ENABLED", false), "Enable node-agent workload status reporting.")
fs.BoolVar(&cfg.RuntimeConfig.MeshSyntheticRuntimeEnabled, "mesh-synthetic-runtime-enabled", getenvBool("RAP_MESH_SYNTHETIC_RUNTIME_ENABLED", true), "Enable synthetic mesh runtime.")
fs.BoolVar(&cfg.RuntimeConfig.MeshProductionForwardingEnabled, "mesh-production-forwarding-enabled", getenvBool("RAP_MESH_PRODUCTION_FORWARDING_ENABLED", false), "Enable production forwarding gate; runtime still fail-closed if unavailable.")
fs.BoolVar(&cfg.RuntimeConfig.MeshFabricSessionEnabled, "mesh-fabric-session-enabled", getenvBool("RAP_MESH_FABRIC_SESSION_ENABLED", false), "Enable authenticated fabric session WebSocket endpoint.")
fs.BoolVar(&cfg.RuntimeConfig.MeshFabricSessionEnabled, "mesh-fabric-session-enabled", getenvBool("RAP_MESH_FABRIC_SESSION_ENABLED", false), "Enable authenticated fabric session endpoint.")
fs.BoolVar(&cfg.RuntimeConfig.VPNFabricSessionTransportEnabled, "vpn-fabric-session-transport-enabled", getenvBool("RAP_VPN_FABRIC_SESSION_TRANSPORT_ENABLED", false), "Route VPN packet transport over persistent fabric sessions.")
fs.BoolVar(&cfg.RuntimeConfig.MeshQUICFabricEnabled, "mesh-quic-fabric-enabled", getenvBool("RAP_MESH_QUIC_FABRIC_ENABLED", false), "Enable QUIC/UDP fabric listener.")
fs.StringVar(&cfg.RuntimeConfig.MeshQUICFabricListenAddr, "mesh-quic-fabric-listen-addr", getenv("RAP_MESH_QUIC_FABRIC_LISTEN_ADDR", ""), "QUIC/UDP fabric listen address.")
@@ -230,7 +230,7 @@ func runInstallLinux(ctx context.Context, args []string) error {
fs.IntVar(&cfg.RuntimeConfig.MeshListenAutoPortEnd, "mesh-listen-auto-port-end", getenvInt("RAP_MESH_LISTEN_AUTO_PORT_END", 19231), "Last port used when mesh listen port mode is auto.")
fs.StringVar(&cfg.RuntimeConfig.MeshAdvertiseEndpoint, "mesh-advertise-endpoint", getenv("RAP_MESH_ADVERTISE_ENDPOINT", ""), "Advertised mesh endpoint.")
fs.StringVar(&cfg.RuntimeConfig.MeshAdvertiseEndpointsJSON, "mesh-advertise-endpoints-json", getenv("RAP_MESH_ADVERTISE_ENDPOINTS_JSON", ""), "Advertised endpoint candidates JSON.")
fs.StringVar(&cfg.RuntimeConfig.MeshAdvertiseTransport, "mesh-advertise-transport", getenv("RAP_MESH_ADVERTISE_TRANSPORT", "direct_http"), "Advertised transport.")
fs.StringVar(&cfg.RuntimeConfig.MeshAdvertiseTransport, "mesh-advertise-transport", getenv("RAP_MESH_ADVERTISE_TRANSPORT", "quic"), "Advertised transport.")
fs.StringVar(&cfg.RuntimeConfig.MeshConnectivityMode, "mesh-connectivity-mode", getenv("RAP_MESH_CONNECTIVITY_MODE", "outbound_only"), "Connectivity mode hint.")
fs.StringVar(&cfg.RuntimeConfig.MeshNATType, "mesh-nat-type", getenv("RAP_MESH_NAT_TYPE", "unknown"), "NAT type hint.")
fs.StringVar(&cfg.RuntimeConfig.MeshRegion, "mesh-region", getenv("RAP_MESH_REGION", "linux"), "Region/site hint.")
@@ -305,7 +305,7 @@ func runInstallWindows(ctx context.Context, args []string) error {
fs.BoolVar(&cfg.RuntimeConfig.WorkloadSupervisionEnabled, "workload-supervision-enabled", getenvBool("RAP_WORKLOAD_SUPERVISION_ENABLED", false), "Enable node-agent workload status reporting.")
fs.BoolVar(&cfg.RuntimeConfig.MeshSyntheticRuntimeEnabled, "mesh-synthetic-runtime-enabled", getenvBool("RAP_MESH_SYNTHETIC_RUNTIME_ENABLED", true), "Enable synthetic mesh runtime.")
fs.BoolVar(&cfg.RuntimeConfig.MeshProductionForwardingEnabled, "mesh-production-forwarding-enabled", getenvBool("RAP_MESH_PRODUCTION_FORWARDING_ENABLED", false), "Enable production forwarding gate; runtime still fail-closed if unavailable.")
fs.BoolVar(&cfg.RuntimeConfig.MeshFabricSessionEnabled, "mesh-fabric-session-enabled", getenvBool("RAP_MESH_FABRIC_SESSION_ENABLED", false), "Enable authenticated fabric session WebSocket endpoint.")
fs.BoolVar(&cfg.RuntimeConfig.MeshFabricSessionEnabled, "mesh-fabric-session-enabled", getenvBool("RAP_MESH_FABRIC_SESSION_ENABLED", false), "Enable authenticated fabric session endpoint.")
fs.BoolVar(&cfg.RuntimeConfig.VPNFabricSessionTransportEnabled, "vpn-fabric-session-transport-enabled", getenvBool("RAP_VPN_FABRIC_SESSION_TRANSPORT_ENABLED", false), "Route VPN packet transport over persistent fabric sessions.")
fs.BoolVar(&cfg.RuntimeConfig.MeshQUICFabricEnabled, "mesh-quic-fabric-enabled", getenvBool("RAP_MESH_QUIC_FABRIC_ENABLED", false), "Enable QUIC/UDP fabric listener.")
fs.StringVar(&cfg.RuntimeConfig.MeshQUICFabricListenAddr, "mesh-quic-fabric-listen-addr", getenv("RAP_MESH_QUIC_FABRIC_LISTEN_ADDR", ""), "QUIC/UDP fabric listen address.")
@@ -318,7 +318,7 @@ func runInstallWindows(ctx context.Context, args []string) error {
fs.IntVar(&cfg.RuntimeConfig.MeshListenAutoPortEnd, "mesh-listen-auto-port-end", getenvInt("RAP_MESH_LISTEN_AUTO_PORT_END", 19231), "Last port used when mesh listen port mode is auto.")
fs.StringVar(&cfg.RuntimeConfig.MeshAdvertiseEndpoint, "mesh-advertise-endpoint", getenv("RAP_MESH_ADVERTISE_ENDPOINT", ""), "Advertised mesh endpoint.")
fs.StringVar(&cfg.RuntimeConfig.MeshAdvertiseEndpointsJSON, "mesh-advertise-endpoints-json", getenv("RAP_MESH_ADVERTISE_ENDPOINTS_JSON", ""), "Advertised endpoint candidates JSON.")
fs.StringVar(&cfg.RuntimeConfig.MeshAdvertiseTransport, "mesh-advertise-transport", getenv("RAP_MESH_ADVERTISE_TRANSPORT", "direct_http"), "Advertised transport.")
fs.StringVar(&cfg.RuntimeConfig.MeshAdvertiseTransport, "mesh-advertise-transport", getenv("RAP_MESH_ADVERTISE_TRANSPORT", "quic"), "Advertised transport.")
fs.StringVar(&cfg.RuntimeConfig.MeshConnectivityMode, "mesh-connectivity-mode", getenv("RAP_MESH_CONNECTIVITY_MODE", "outbound_only"), "Connectivity mode hint.")
fs.StringVar(&cfg.RuntimeConfig.MeshNATType, "mesh-nat-type", getenv("RAP_MESH_NAT_TYPE", "unknown"), "NAT type hint.")
fs.StringVar(&cfg.RuntimeConfig.MeshRegion, "mesh-region", getenv("RAP_MESH_REGION", "windows"), "Region/site hint.")
@@ -799,7 +799,7 @@ func parseInstall(args []string) (installCommandConfig, error) {
fs.BoolVar(&cfg.WorkloadSupervisionEnabled, "workload-supervision-enabled", getenvBool("RAP_WORKLOAD_SUPERVISION_ENABLED", false), "Enable node-agent workload status reporting.")
fs.BoolVar(&cfg.MeshSyntheticRuntimeEnabled, "mesh-synthetic-runtime-enabled", getenvBool("RAP_MESH_SYNTHETIC_RUNTIME_ENABLED", false), "Enable synthetic mesh runtime.")
fs.BoolVar(&cfg.MeshProductionForwardingEnabled, "mesh-production-forwarding-enabled", getenvBool("RAP_MESH_PRODUCTION_FORWARDING_ENABLED", false), "Enable production forwarding gate; runtime still fail-closed if unavailable.")
fs.BoolVar(&cfg.MeshFabricSessionEnabled, "mesh-fabric-session-enabled", getenvBool("RAP_MESH_FABRIC_SESSION_ENABLED", false), "Enable authenticated fabric session WebSocket endpoint.")
fs.BoolVar(&cfg.MeshFabricSessionEnabled, "mesh-fabric-session-enabled", getenvBool("RAP_MESH_FABRIC_SESSION_ENABLED", false), "Enable authenticated fabric session endpoint.")
fs.BoolVar(&cfg.VPNFabricSessionTransportEnabled, "vpn-fabric-session-transport-enabled", getenvBool("RAP_VPN_FABRIC_SESSION_TRANSPORT_ENABLED", false), "Route VPN packet transport over persistent fabric sessions.")
fs.BoolVar(&cfg.MeshQUICFabricEnabled, "mesh-quic-fabric-enabled", getenvBool("RAP_MESH_QUIC_FABRIC_ENABLED", false), "Enable QUIC/UDP fabric listener.")
fs.StringVar(&cfg.MeshQUICFabricListenAddr, "mesh-quic-fabric-listen-addr", getenv("RAP_MESH_QUIC_FABRIC_LISTEN_ADDR", ""), "QUIC/UDP fabric listen address.")
@@ -812,7 +812,7 @@ func parseInstall(args []string) (installCommandConfig, error) {
fs.IntVar(&cfg.MeshListenAutoPortEnd, "mesh-listen-auto-port-end", getenvInt("RAP_MESH_LISTEN_AUTO_PORT_END", 0), "Last port used when mesh listen port mode is auto.")
fs.StringVar(&cfg.MeshAdvertiseEndpoint, "mesh-advertise-endpoint", getenv("RAP_MESH_ADVERTISE_ENDPOINT", ""), "Advertised mesh endpoint.")
fs.StringVar(&cfg.MeshAdvertiseEndpointsJSON, "mesh-advertise-endpoints-json", getenv("RAP_MESH_ADVERTISE_ENDPOINTS_JSON", ""), "Advertised endpoint candidates JSON.")
fs.StringVar(&cfg.MeshAdvertiseTransport, "mesh-advertise-transport", getenv("RAP_MESH_ADVERTISE_TRANSPORT", ""), "Advertised transport.")
fs.StringVar(&cfg.MeshAdvertiseTransport, "mesh-advertise-transport", getenv("RAP_MESH_ADVERTISE_TRANSPORT", "quic"), "Advertised transport.")
fs.StringVar(&cfg.MeshConnectivityMode, "mesh-connectivity-mode", getenv("RAP_MESH_CONNECTIVITY_MODE", ""), "Connectivity mode hint.")
fs.StringVar(&cfg.MeshNATType, "mesh-nat-type", getenv("RAP_MESH_NAT_TYPE", ""), "NAT type hint.")
fs.StringVar(&cfg.MeshRegion, "mesh-region", getenv("RAP_MESH_REGION", ""), "Region/site hint.")
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff