Refactor RDP proxy handling and update related tests
This commit is contained in:
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,760 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/example/remote-access-platform/agents/rap-node-agent/internal/mesh"
|
||||
)
|
||||
|
||||
func TestRouteModeCoverageVerdictRequiresMixedModes(t *testing.T) {
|
||||
report := loadtestReport{
|
||||
Config: loadtestConfig{
|
||||
TopologyProfile: "mixed-public-nat-lan-relay",
|
||||
Targets: []string{"a", "b", "c", "d"},
|
||||
FailTarget: -1,
|
||||
ImpairTarget: -1,
|
||||
},
|
||||
SuccessfulStreams: 4,
|
||||
TargetStats: map[string]targetStats{
|
||||
"a": {RouteModes: map[string]int{string(mesh.FabricRouteLAN): 1}},
|
||||
"b": {RouteModes: map[string]int{string(mesh.FabricRouteICE): 1}},
|
||||
"c": {RouteModes: map[string]int{string(mesh.FabricRouteReverse): 1}},
|
||||
"d": {RouteModes: map[string]int{}},
|
||||
},
|
||||
}
|
||||
|
||||
reasons := routeModeCoverageVerdictReasons(report)
|
||||
if len(reasons) != 1 || !strings.Contains(reasons[0], string(mesh.FabricRouteRelay)) {
|
||||
t.Fatalf("reasons = %v, want missing relay route mode", reasons)
|
||||
}
|
||||
|
||||
report.TargetStats["d"] = targetStats{RouteModes: map[string]int{string(mesh.FabricRouteRelay): 1}}
|
||||
if reasons := routeModeCoverageVerdictReasons(report); len(reasons) != 0 {
|
||||
t.Fatalf("reasons = %v, want full coverage pass", reasons)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLegacyRouteModeVerdictRejectsNonQUICModes(t *testing.T) {
|
||||
report := loadtestReport{
|
||||
TargetStats: map[string]targetStats{
|
||||
"a": {RouteModes: map[string]int{
|
||||
"direct_quic": 4,
|
||||
"relay": 1,
|
||||
"outbound_reverse": 2,
|
||||
"wss": 3,
|
||||
}},
|
||||
},
|
||||
}
|
||||
reasons := legacyRouteModeVerdictReasons(report)
|
||||
if len(reasons) != 1 ||
|
||||
!strings.Contains(reasons[0], "relay:1") ||
|
||||
!strings.Contains(reasons[0], "outbound_reverse:2") ||
|
||||
!strings.Contains(reasons[0], "wss:3") {
|
||||
t.Fatalf("reasons = %v, want legacy route mode failure", reasons)
|
||||
}
|
||||
|
||||
report.TargetStats["a"] = targetStats{RouteModes: map[string]int{
|
||||
string(mesh.FabricRouteDirect): 1,
|
||||
string(mesh.FabricRouteLAN): 1,
|
||||
string(mesh.FabricRouteICE): 1,
|
||||
string(mesh.FabricRouteReverse): 1,
|
||||
string(mesh.FabricRouteRelay): 1,
|
||||
}}
|
||||
if reasons := legacyRouteModeVerdictReasons(report); len(reasons) != 0 {
|
||||
t.Fatalf("reasons = %v, want QUIC modes accepted", reasons)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTargetEndpointPolicyVerdictRejectsNonQUICTargets(t *testing.T) {
|
||||
report := loadtestReport{
|
||||
Config: loadtestConfig{
|
||||
Targets: []string{
|
||||
"quic://a:19443",
|
||||
"http://b:19443",
|
||||
"ws://c:19443",
|
||||
"d:19443",
|
||||
"",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
reasons := targetEndpointPolicyVerdictReasons(report)
|
||||
if len(reasons) != 1 ||
|
||||
!strings.Contains(reasons[0], "http://b:19443") ||
|
||||
!strings.Contains(reasons[0], "ws://c:19443") ||
|
||||
!strings.Contains(reasons[0], "d:19443") ||
|
||||
!strings.Contains(reasons[0], "<empty>") {
|
||||
t.Fatalf("reasons = %v, want non-QUIC target failure", reasons)
|
||||
}
|
||||
|
||||
report.Config.Targets = []string{"quic://a:19443", " QUIC://b:19443 "}
|
||||
if reasons := targetEndpointPolicyVerdictReasons(report); len(reasons) != 0 {
|
||||
t.Fatalf("reasons = %v, want QUIC targets accepted", reasons)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunClientRejectsNonQUICTargetBeforeDial(t *testing.T) {
|
||||
_, err := runClient(context.Background(), loadtestConfig{
|
||||
Targets: []string{"http://127.0.0.1:19443"},
|
||||
Streams: 1,
|
||||
Concurrency: 1,
|
||||
BytesPerStream: 1,
|
||||
PayloadSize: 1,
|
||||
})
|
||||
if err == nil || !strings.Contains(err.Error(), "non_quic_targets=http://127.0.0.1:19443") {
|
||||
t.Fatalf("err = %v, want non-QUIC target validation error", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFillLoadtestPayloadVariesByStreamAndSequence(t *testing.T) {
|
||||
first := make([]byte, 128)
|
||||
second := make([]byte, 128)
|
||||
third := make([]byte, 128)
|
||||
|
||||
fillLoadtestPayload(first, 7, 9, 1, 0)
|
||||
fillLoadtestPayload(second, 7, 9, 2, int64(len(first)))
|
||||
fillLoadtestPayload(third, 8, 10, 1, 0)
|
||||
|
||||
if bytes.Equal(first, second) {
|
||||
t.Fatal("payload did not vary by sequence/offset")
|
||||
}
|
||||
if bytes.Equal(first, third) {
|
||||
t.Fatal("payload did not vary by stream")
|
||||
}
|
||||
if bytes.Count(first, []byte{first[0]}) == len(first) {
|
||||
t.Fatal("payload collapsed to a constant byte")
|
||||
}
|
||||
}
|
||||
|
||||
func TestFillLoadtestPayloadIsDeterministic(t *testing.T) {
|
||||
first := make([]byte, 128)
|
||||
second := make([]byte, 128)
|
||||
|
||||
fillLoadtestPayload(first, 7, 9, 1, 0)
|
||||
fillLoadtestPayload(second, 7, 9, 1, 0)
|
||||
|
||||
if !bytes.Equal(first, second) {
|
||||
t.Fatal("payload is not deterministic")
|
||||
}
|
||||
}
|
||||
|
||||
func TestFillLoadtestPayloadHandlesShortFinalChunk(t *testing.T) {
|
||||
chunk := make([]byte, 17)
|
||||
fillLoadtestPayload(chunk, 7, 9, 3, 256)
|
||||
if bytes.Equal(chunk, make([]byte, len(chunk))) {
|
||||
t.Fatal("short payload chunk stayed zeroed")
|
||||
}
|
||||
}
|
||||
|
||||
func TestVerdictFailsSuccessfulStreamAckMismatch(t *testing.T) {
|
||||
report := loadtestReport{
|
||||
Config: loadtestConfig{
|
||||
FailTarget: -1,
|
||||
ImpairTarget: -1,
|
||||
Concurrency: 1,
|
||||
},
|
||||
TotalStreams: 1,
|
||||
SuccessfulStreams: 1,
|
||||
BytesSent: 1024,
|
||||
FramesSent: 2,
|
||||
AcksReceived: 1,
|
||||
AckMismatchedStreams: 1,
|
||||
ChannelOpens: 1,
|
||||
ChannelCloses: 1,
|
||||
RoutePressure: mesh.FabricRoutePressureSnapshot{AcquiredTotal: 1, ReleasedTotal: 1, MaxActiveTotal: 1},
|
||||
}
|
||||
|
||||
gotVerdict, reasons := verdict(report)
|
||||
if gotVerdict != "fail" {
|
||||
t.Fatalf("verdict = %q, want fail", gotVerdict)
|
||||
}
|
||||
found := false
|
||||
for _, reason := range reasons {
|
||||
if reason == "ack_mismatched_streams=1" {
|
||||
found = true
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Fatalf("reasons = %v, want ack mismatch reason", reasons)
|
||||
}
|
||||
}
|
||||
|
||||
func TestVerdictFailsAckIntegrityError(t *testing.T) {
|
||||
report := loadtestReport{
|
||||
Config: loadtestConfig{
|
||||
FailTarget: -1,
|
||||
ImpairTarget: -1,
|
||||
Concurrency: 1,
|
||||
},
|
||||
TotalStreams: 1,
|
||||
FailedStreams: 1,
|
||||
BytesSent: 1024,
|
||||
FramesSent: 1,
|
||||
AcksReceived: 1,
|
||||
AckIntegrityErrors: 1,
|
||||
ChannelOpens: 1,
|
||||
ChannelCloses: 1,
|
||||
RoutePressure: mesh.FabricRoutePressureSnapshot{AcquiredTotal: 1, ReleasedTotal: 1, MaxActiveTotal: 1},
|
||||
}
|
||||
|
||||
gotVerdict, reasons := verdict(report)
|
||||
if gotVerdict != "fail" {
|
||||
t.Fatalf("verdict = %q, want fail", gotVerdict)
|
||||
}
|
||||
found := false
|
||||
for _, reason := range reasons {
|
||||
if reason == "ack_integrity_errors=1" {
|
||||
found = true
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Fatalf("reasons = %v, want ack integrity reason", reasons)
|
||||
}
|
||||
}
|
||||
|
||||
func TestVerdictFailsBelowMinimumThroughput(t *testing.T) {
|
||||
report := loadtestReport{
|
||||
Config: loadtestConfig{
|
||||
FailTarget: -1,
|
||||
ImpairTarget: -1,
|
||||
Concurrency: 1,
|
||||
MinThroughputMbps: 100,
|
||||
},
|
||||
TotalStreams: 1,
|
||||
SuccessfulStreams: 1,
|
||||
BytesSent: 1024,
|
||||
FramesSent: 1,
|
||||
AcksReceived: 1,
|
||||
ThroughputBps: 99 * 1000 * 1000,
|
||||
ChannelOpens: 1,
|
||||
ChannelCloses: 1,
|
||||
RoutePressure: mesh.FabricRoutePressureSnapshot{AcquiredTotal: 1, ReleasedTotal: 1, MaxActiveTotal: 1},
|
||||
}
|
||||
|
||||
gotVerdict, reasons := verdict(report)
|
||||
if gotVerdict != "fail" {
|
||||
t.Fatalf("verdict = %q, want fail", gotVerdict)
|
||||
}
|
||||
found := false
|
||||
for _, reason := range reasons {
|
||||
if strings.HasPrefix(reason, "throughput_bps=") {
|
||||
found = true
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Fatalf("reasons = %v, want throughput reason", reasons)
|
||||
}
|
||||
|
||||
report.ThroughputBps = 100 * 1000 * 1000
|
||||
if gotVerdict, reasons := verdict(report); gotVerdict != "pass" {
|
||||
t.Fatalf("verdict = %q reasons=%v, want pass at threshold", gotVerdict, reasons)
|
||||
}
|
||||
}
|
||||
|
||||
func TestVerdictFailsBelowMinimumChannelChurn(t *testing.T) {
|
||||
report := loadtestReport{
|
||||
Config: loadtestConfig{
|
||||
FailTarget: -1,
|
||||
ImpairTarget: -1,
|
||||
Concurrency: 1,
|
||||
MinChannelChurn: 1000,
|
||||
},
|
||||
TotalStreams: 1,
|
||||
SuccessfulStreams: 1,
|
||||
BytesSent: 1024,
|
||||
FramesSent: 1,
|
||||
AcksReceived: 1,
|
||||
ChannelOpens: 1,
|
||||
ChannelCloses: 1,
|
||||
ChannelChurnPerSec: 999,
|
||||
RoutePressure: mesh.FabricRoutePressureSnapshot{AcquiredTotal: 1, ReleasedTotal: 1, MaxActiveTotal: 1},
|
||||
}
|
||||
|
||||
gotVerdict, reasons := verdict(report)
|
||||
if gotVerdict != "fail" {
|
||||
t.Fatalf("verdict = %q, want fail", gotVerdict)
|
||||
}
|
||||
found := false
|
||||
for _, reason := range reasons {
|
||||
if strings.HasPrefix(reason, "channel_churn_per_sec=") {
|
||||
found = true
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Fatalf("reasons = %v, want channel churn reason", reasons)
|
||||
}
|
||||
|
||||
report.ChannelChurnPerSec = 1000
|
||||
if gotVerdict, reasons := verdict(report); gotVerdict != "pass" {
|
||||
t.Fatalf("verdict = %q reasons=%v, want pass at threshold", gotVerdict, reasons)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTargetByteDistributionVerdictDetectsSkew(t *testing.T) {
|
||||
report := loadtestReport{
|
||||
Config: loadtestConfig{
|
||||
Targets: []string{"a", "b", "c", "d"},
|
||||
FailTarget: -1,
|
||||
ImpairTarget: -1,
|
||||
Concurrency: 1,
|
||||
BytesPerStream: 100,
|
||||
},
|
||||
SuccessfulStreams: 40,
|
||||
BytesSent: 4000,
|
||||
TargetStreams: map[string]int{
|
||||
"a": 10,
|
||||
"b": 10,
|
||||
"c": 10,
|
||||
"d": 10,
|
||||
},
|
||||
TargetBytes: map[string]int64{
|
||||
"a": 2500,
|
||||
"b": 500,
|
||||
"c": 500,
|
||||
"d": 500,
|
||||
},
|
||||
}
|
||||
|
||||
reasons := targetByteDistributionVerdictReasons(report)
|
||||
if len(reasons) != 1 || !strings.HasPrefix(reasons[0], "target_byte_distribution_skew=") {
|
||||
t.Fatalf("reasons = %v, want byte skew reason", reasons)
|
||||
}
|
||||
|
||||
report.TargetBytes = map[string]int64{
|
||||
"a": 1000,
|
||||
"b": 1000,
|
||||
"c": 1000,
|
||||
"d": 1000,
|
||||
}
|
||||
if reasons := targetByteDistributionVerdictReasons(report); len(reasons) != 0 {
|
||||
t.Fatalf("reasons = %v, want balanced bytes pass", reasons)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDistributionVerdictChecksSurvivingTargetsAfterFailure(t *testing.T) {
|
||||
report := loadtestReport{
|
||||
Config: loadtestConfig{
|
||||
Targets: []string{"quic://a:1", "quic://b:1", "quic://c:1", "quic://d:1"},
|
||||
FailTarget: 0,
|
||||
ImpairTarget: -1,
|
||||
Concurrency: 8,
|
||||
},
|
||||
SuccessfulStreams: 90,
|
||||
TargetStreams: map[string]int{
|
||||
"quic://b:1": 90,
|
||||
},
|
||||
}
|
||||
|
||||
reasons := targetDistributionVerdictReasons(report)
|
||||
if len(reasons) != 1 || !strings.HasPrefix(reasons[0], "target_distribution_collapsed=1/3_targets_used") {
|
||||
t.Fatalf("reasons = %v, want surviving-target collapse", reasons)
|
||||
}
|
||||
|
||||
report.TargetStreams = map[string]int{
|
||||
"quic://b:1": 30,
|
||||
"quic://c:1": 30,
|
||||
"quic://d:1": 30,
|
||||
}
|
||||
if reasons := targetDistributionVerdictReasons(report); len(reasons) != 0 {
|
||||
t.Fatalf("reasons = %v, want balanced surviving targets pass", reasons)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRoutePressureVerdictChecksSurvivingTargetsAfterFailure(t *testing.T) {
|
||||
targets := []string{"quic://a:1", "quic://b:1", "quic://c:1", "quic://d:1"}
|
||||
report := loadtestReport{
|
||||
Config: loadtestConfig{
|
||||
Targets: targets,
|
||||
FailTarget: 0,
|
||||
ImpairTarget: -1,
|
||||
Concurrency: 12,
|
||||
},
|
||||
RoutePressure: mesh.FabricRoutePressureSnapshot{
|
||||
MaxActive: map[string]int{
|
||||
loadtestRouteID(1, targets[1]): 12,
|
||||
},
|
||||
MaxActiveTotal: 12,
|
||||
},
|
||||
}
|
||||
|
||||
reasons := routePressureDistributionVerdictReasons(report)
|
||||
if len(reasons) != 1 || !strings.HasPrefix(reasons[0], "route_pressure_distribution_collapsed=1/3_targets_used") {
|
||||
t.Fatalf("reasons = %v, want surviving-route-pressure collapse", reasons)
|
||||
}
|
||||
|
||||
report.RoutePressure.MaxActive = map[string]int{
|
||||
loadtestRouteID(1, targets[1]): 4,
|
||||
loadtestRouteID(2, targets[2]): 4,
|
||||
loadtestRouteID(3, targets[3]): 4,
|
||||
}
|
||||
if reasons := routePressureDistributionVerdictReasons(report); len(reasons) != 0 {
|
||||
t.Fatalf("reasons = %v, want balanced surviving route pressure pass", reasons)
|
||||
}
|
||||
}
|
||||
|
||||
func TestVerdictFailsOverallAckLatencySLO(t *testing.T) {
|
||||
report := loadtestReport{
|
||||
Config: loadtestConfig{
|
||||
FailTarget: -1,
|
||||
ImpairTarget: -1,
|
||||
Concurrency: 1,
|
||||
MaxAckP95Ms: 10,
|
||||
MaxAckP99Ms: 20,
|
||||
},
|
||||
TotalStreams: 1,
|
||||
SuccessfulStreams: 1,
|
||||
BytesSent: 1024,
|
||||
FramesSent: 1,
|
||||
AcksReceived: 1,
|
||||
AckP95Ms: 11,
|
||||
AckP99Ms: 21,
|
||||
ChannelOpens: 1,
|
||||
ChannelCloses: 1,
|
||||
RoutePressure: mesh.FabricRoutePressureSnapshot{AcquiredTotal: 1, ReleasedTotal: 1, MaxActiveTotal: 1},
|
||||
}
|
||||
|
||||
gotVerdict, reasons := verdict(report)
|
||||
if gotVerdict != "fail" {
|
||||
t.Fatalf("verdict = %q, want fail", gotVerdict)
|
||||
}
|
||||
foundP95 := false
|
||||
foundP99 := false
|
||||
for _, reason := range reasons {
|
||||
if strings.HasPrefix(reason, "ack_p95_ms=") {
|
||||
foundP95 = true
|
||||
}
|
||||
if strings.HasPrefix(reason, "ack_p99_ms=") {
|
||||
foundP99 = true
|
||||
}
|
||||
}
|
||||
if !foundP95 || !foundP99 {
|
||||
t.Fatalf("reasons = %v, want ACK p95 and p99 reasons", reasons)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTargetAckVerdictDetectsSlowHealthyTarget(t *testing.T) {
|
||||
report := loadtestReport{
|
||||
Config: loadtestConfig{
|
||||
Targets: []string{"a", "b"},
|
||||
FailTarget: -1,
|
||||
ImpairTarget: -1,
|
||||
MaxTargetAckMs: 10,
|
||||
},
|
||||
TargetStats: map[string]targetStats{
|
||||
"a": {Streams: 10, MaxAckMs: 4},
|
||||
"b": {Streams: 10, MaxAckMs: 11},
|
||||
},
|
||||
}
|
||||
|
||||
reasons := targetAckVerdictReasons(report)
|
||||
if len(reasons) != 1 || !strings.HasPrefix(reasons[0], "target_ack_ms=b:11>10") {
|
||||
t.Fatalf("reasons = %v, want slow target ack reason", reasons)
|
||||
}
|
||||
|
||||
report.TargetStats["b"] = targetStats{Streams: 10, MaxAckMs: 10}
|
||||
if reasons := targetAckVerdictReasons(report); len(reasons) != 0 {
|
||||
t.Fatalf("reasons = %v, want target ack pass at threshold", reasons)
|
||||
}
|
||||
}
|
||||
|
||||
func TestVerdictFailsSetupLatencySLO(t *testing.T) {
|
||||
report := loadtestReport{
|
||||
Config: loadtestConfig{
|
||||
FailTarget: -1,
|
||||
ImpairTarget: -1,
|
||||
Concurrency: 1,
|
||||
MaxSetupP95Ms: 10,
|
||||
MaxSetupP99Ms: 20,
|
||||
},
|
||||
TotalStreams: 1,
|
||||
SuccessfulStreams: 1,
|
||||
BytesSent: 1024,
|
||||
FramesSent: 1,
|
||||
AcksReceived: 1,
|
||||
SetupLatencyP95Ms: 11,
|
||||
SetupLatencyP99Ms: 21,
|
||||
ChannelOpens: 1,
|
||||
ChannelCloses: 1,
|
||||
RoutePressure: mesh.FabricRoutePressureSnapshot{AcquiredTotal: 1, ReleasedTotal: 1, MaxActiveTotal: 1},
|
||||
}
|
||||
|
||||
gotVerdict, reasons := verdict(report)
|
||||
if gotVerdict != "fail" {
|
||||
t.Fatalf("verdict = %q, want fail", gotVerdict)
|
||||
}
|
||||
foundP95 := false
|
||||
foundP99 := false
|
||||
for _, reason := range reasons {
|
||||
if strings.HasPrefix(reason, "setup_p95_ms=") {
|
||||
foundP95 = true
|
||||
}
|
||||
if strings.HasPrefix(reason, "setup_p99_ms=") {
|
||||
foundP99 = true
|
||||
}
|
||||
}
|
||||
if !foundP95 || !foundP99 {
|
||||
t.Fatalf("reasons = %v, want setup p95 and p99 reasons", reasons)
|
||||
}
|
||||
}
|
||||
|
||||
func TestVerdictFailsRerouteLatencySLO(t *testing.T) {
|
||||
report := loadtestReport{
|
||||
Config: loadtestConfig{
|
||||
FailTarget: -1,
|
||||
ImpairTarget: -1,
|
||||
Concurrency: 1,
|
||||
MaxRerouteP95Ms: 10,
|
||||
MaxRerouteP99Ms: 20,
|
||||
},
|
||||
TotalStreams: 1,
|
||||
SuccessfulStreams: 1,
|
||||
BytesSent: 1024,
|
||||
FramesSent: 1,
|
||||
AcksReceived: 1,
|
||||
RerouteLatencyP95Ms: 11,
|
||||
RerouteLatencyP99Ms: 21,
|
||||
ChannelOpens: 1,
|
||||
ChannelCloses: 1,
|
||||
RoutePressure: mesh.FabricRoutePressureSnapshot{AcquiredTotal: 1, ReleasedTotal: 1, MaxActiveTotal: 1},
|
||||
}
|
||||
|
||||
gotVerdict, reasons := verdict(report)
|
||||
if gotVerdict != "fail" {
|
||||
t.Fatalf("verdict = %q, want fail", gotVerdict)
|
||||
}
|
||||
foundP95 := false
|
||||
foundP99 := false
|
||||
for _, reason := range reasons {
|
||||
if strings.HasPrefix(reason, "reroute_p95_ms=") {
|
||||
foundP95 = true
|
||||
}
|
||||
if strings.HasPrefix(reason, "reroute_p99_ms=") {
|
||||
foundP99 = true
|
||||
}
|
||||
}
|
||||
if !foundP95 || !foundP99 {
|
||||
t.Fatalf("reasons = %v, want reroute p95 and p99 reasons", reasons)
|
||||
}
|
||||
}
|
||||
|
||||
func TestShouldQuarantineTarget(t *testing.T) {
|
||||
quarantined := []string{
|
||||
"ack timeout or session closed",
|
||||
"deadline exceeded",
|
||||
"connection refused",
|
||||
"connection reset by peer",
|
||||
"no route to host",
|
||||
}
|
||||
for _, reason := range quarantined {
|
||||
if !shouldQuarantineTarget(reason) {
|
||||
t.Fatalf("shouldQuarantineTarget(%q) = false, want true", reason)
|
||||
}
|
||||
}
|
||||
if shouldQuarantineTarget("ack payload checksum mismatch") {
|
||||
t.Fatal("checksum mismatch should not quarantine a target")
|
||||
}
|
||||
if shouldQuarantineTarget("context deadline exceeded") {
|
||||
t.Fatal("context deadline should not quarantine a target")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSpreadStartDistributesQuarantinedSlot(t *testing.T) {
|
||||
targets := []string{"a", "b", "c", "d"}
|
||||
health := newTargetHealthTracker()
|
||||
health.MarkDegraded("a", "connection refused", time.Minute)
|
||||
counts := map[string]int{}
|
||||
for index := 0; index < 40; index += len(targets) {
|
||||
initial, spread := loadtestSpreadStart(index, len(targets))
|
||||
targetIndex := loadtestPreferredTargetIndex(targets, initial, spread, health, -1)
|
||||
counts[targets[targetIndex]]++
|
||||
}
|
||||
if counts["b"] == 0 || counts["c"] == 0 || counts["d"] == 0 {
|
||||
t.Fatalf("counts = %v, want degraded slot spread across surviving targets", counts)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSpreadUsableTargetDistributesRetries(t *testing.T) {
|
||||
targets := []string{"a", "b", "c", "d"}
|
||||
health := newTargetHealthTracker()
|
||||
health.MarkDegraded("a", "connection refused", time.Minute)
|
||||
counts := map[string]int{}
|
||||
for cohort := 0; cohort < 90; cohort++ {
|
||||
targetIndex := loadtestSpreadUsableTargetIndex(targets, cohort, health, 0)
|
||||
counts[targets[targetIndex]]++
|
||||
}
|
||||
if counts["b"] != 30 || counts["c"] != 30 || counts["d"] != 30 {
|
||||
t.Fatalf("counts = %v, want retry load spread evenly across surviving targets", counts)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadtestLogicalStreamIDAvoidsReservedTransportStreams(t *testing.T) {
|
||||
for _, index := range []int{-1, 0, 1, 999, 1000, 10_000} {
|
||||
streamID := loadtestLogicalStreamID(index)
|
||||
if streamID == mesh.ProductionForwardQUICStreamID || streamID == mesh.SyntheticForwardQUICStreamID {
|
||||
t.Fatalf("loadtestLogicalStreamID(%d) = %d, collides with reserved transport stream", index, streamID)
|
||||
}
|
||||
if streamID < 10_000 {
|
||||
t.Fatalf("loadtestLogicalStreamID(%d) = %d, want loadtest stream range", index, streamID)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestLatencyAwareTargetIndexKeepsSlowWANFromOwningPool(t *testing.T) {
|
||||
targets := []string{"lan-a", "lan-b", "wan"}
|
||||
health := newTargetHealthTracker()
|
||||
health.RecordProbes([]targetProbeResult{
|
||||
{Target: "lan-a", RTTMs: 4, Usable: true},
|
||||
{Target: "lan-b", RTTMs: 5, Usable: true},
|
||||
{Target: "wan", RTTMs: 400, Usable: true},
|
||||
})
|
||||
counts := map[string]int{}
|
||||
for index := 0; index < 300; index++ {
|
||||
targetIndex := loadtestSpreadUsableTargetIndex(targets, index, health, -1)
|
||||
counts[targets[targetIndex]]++
|
||||
}
|
||||
if counts["wan"] == 0 {
|
||||
t.Fatalf("counts = %v, want slow WAN to stay represented", counts)
|
||||
}
|
||||
if counts["wan"] >= counts["lan-a"] || counts["wan"] >= counts["lan-b"] {
|
||||
t.Fatalf("counts = %v, want latency-aware placement to prefer LAN capacity", counts)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLatencyAwarePreferredTargetUsesAbsolutePlacementOrdinal(t *testing.T) {
|
||||
targets := []string{"lan-a", "lan-b", "lan-c", "wan"}
|
||||
health := newTargetHealthTracker()
|
||||
health.RecordProbes([]targetProbeResult{
|
||||
{Target: "lan-a", RTTMs: 4, Usable: true},
|
||||
{Target: "lan-b", RTTMs: 4, Usable: true},
|
||||
{Target: "lan-c", RTTMs: 4, Usable: true},
|
||||
{Target: "wan", RTTMs: 400, Usable: true},
|
||||
})
|
||||
counts := map[string]int{}
|
||||
for index := 0; index < 500; index++ {
|
||||
preferred, spread := loadtestSpreadStart(index, len(targets))
|
||||
targetIndex := loadtestPreferredTargetIndex(targets, preferred, spread, health, -1)
|
||||
counts[targets[targetIndex]]++
|
||||
}
|
||||
if len(counts) < len(targets) {
|
||||
t.Fatalf("counts = %v, want every probed target represented", counts)
|
||||
}
|
||||
if counts["wan"] >= counts["lan-a"] || counts["wan"] >= counts["lan-b"] || counts["wan"] >= counts["lan-c"] {
|
||||
t.Fatalf("counts = %v, want slow WAN weighted below LAN targets", counts)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHeterogeneousProbeRTTRelaxesEqualDistributionVerdict(t *testing.T) {
|
||||
report := loadtestReport{
|
||||
Config: loadtestConfig{
|
||||
Targets: []string{"lan", "wan"},
|
||||
Concurrency: 64,
|
||||
},
|
||||
SuccessfulStreams: 100,
|
||||
BytesSent: 100 * 1024,
|
||||
TargetStreams: map[string]int{
|
||||
"lan": 96,
|
||||
"wan": 4,
|
||||
},
|
||||
TargetBytes: map[string]int64{
|
||||
"lan": 96 * 1024,
|
||||
"wan": 4 * 1024,
|
||||
},
|
||||
TargetProbes: []targetProbeResult{
|
||||
{Target: "lan", RTTMs: 4, Usable: true},
|
||||
{Target: "wan", RTTMs: 400, Usable: true},
|
||||
},
|
||||
RoutePressure: mesh.FabricRoutePressureSnapshot{
|
||||
MaxActive: map[string]int{
|
||||
loadtestRouteID(0, "lan"): 32,
|
||||
loadtestRouteID(1, "wan"): 1,
|
||||
},
|
||||
MaxActiveTotal: 32,
|
||||
},
|
||||
}
|
||||
if reasons := targetDistributionVerdictReasons(report); len(reasons) != 0 {
|
||||
t.Fatalf("targetDistributionVerdictReasons = %v, want heterogeneous RTT tolerated", reasons)
|
||||
}
|
||||
if reasons := targetByteDistributionVerdictReasons(report); len(reasons) != 0 {
|
||||
t.Fatalf("targetByteDistributionVerdictReasons = %v, want heterogeneous RTT tolerated", reasons)
|
||||
}
|
||||
if reasons := routePressureDistributionVerdictReasons(report); len(reasons) != 0 {
|
||||
t.Fatalf("routePressureDistributionVerdictReasons = %v, want heterogeneous RTT tolerated", reasons)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTargetHealthQuarantineExpiresButSnapshotKeepsObservation(t *testing.T) {
|
||||
health := newTargetHealthTracker()
|
||||
health.MarkDegraded("a", "ack timeout", time.Nanosecond)
|
||||
if !health.IsDegraded("a") {
|
||||
t.Fatal("target should be degraded immediately")
|
||||
}
|
||||
time.Sleep(time.Millisecond)
|
||||
if health.IsDegraded("a") {
|
||||
t.Fatal("target quarantine did not expire")
|
||||
}
|
||||
snapshot := health.Snapshot()
|
||||
if snapshot["a"] != "ack timeout" {
|
||||
t.Fatalf("snapshot = %v, want historical degraded observation", snapshot)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRoutePressureDistributionVerdictDetectsCollapse(t *testing.T) {
|
||||
report := loadtestReport{
|
||||
Config: loadtestConfig{
|
||||
Targets: []string{"a", "b", "c", "d"},
|
||||
FailTarget: -1,
|
||||
ImpairTarget: -1,
|
||||
Concurrency: 16,
|
||||
},
|
||||
RoutePressure: mesh.FabricRoutePressureSnapshot{
|
||||
MaxActive: map[string]int{
|
||||
loadtestRouteID(0, "a"): 16,
|
||||
},
|
||||
MaxActiveTotal: 16,
|
||||
},
|
||||
}
|
||||
|
||||
reasons := routePressureDistributionVerdictReasons(report)
|
||||
if len(reasons) != 1 || !strings.HasPrefix(reasons[0], "route_pressure_distribution_collapsed=") {
|
||||
t.Fatalf("reasons = %v, want collapsed route pressure reason", reasons)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRoutePressureDistributionVerdictDetectsSkew(t *testing.T) {
|
||||
report := loadtestReport{
|
||||
Config: loadtestConfig{
|
||||
Targets: []string{"a", "b", "c", "d"},
|
||||
FailTarget: -1,
|
||||
ImpairTarget: -1,
|
||||
Concurrency: 16,
|
||||
},
|
||||
RoutePressure: mesh.FabricRoutePressureSnapshot{
|
||||
MaxActive: map[string]int{
|
||||
loadtestRouteID(0, "a"): 14,
|
||||
loadtestRouteID(1, "b"): 2,
|
||||
loadtestRouteID(2, "c"): 2,
|
||||
loadtestRouteID(3, "d"): 2,
|
||||
},
|
||||
MaxActiveTotal: 16,
|
||||
},
|
||||
}
|
||||
|
||||
reasons := routePressureDistributionVerdictReasons(report)
|
||||
if len(reasons) != 1 || !strings.HasPrefix(reasons[0], "route_pressure_distribution_skew=") {
|
||||
t.Fatalf("reasons = %v, want route pressure skew reason", reasons)
|
||||
}
|
||||
|
||||
report.RoutePressure.MaxActive = map[string]int{
|
||||
loadtestRouteID(0, "a"): 6,
|
||||
loadtestRouteID(1, "b"): 6,
|
||||
loadtestRouteID(2, "c"): 5,
|
||||
loadtestRouteID(3, "d"): 5,
|
||||
}
|
||||
if reasons := routePressureDistributionVerdictReasons(report); len(reasons) != 0 {
|
||||
t.Fatalf("reasons = %v, want balanced route pressure pass", reasons)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user