761 lines
22 KiB
Go
761 lines
22 KiB
Go
package main
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/example/remote-access-platform/agents/rap-node-agent/internal/mesh"
|
|
)
|
|
|
|
func TestRouteModeCoverageVerdictRequiresMixedModes(t *testing.T) {
|
|
report := loadtestReport{
|
|
Config: loadtestConfig{
|
|
TopologyProfile: "mixed-public-nat-lan-relay",
|
|
Targets: []string{"a", "b", "c", "d"},
|
|
FailTarget: -1,
|
|
ImpairTarget: -1,
|
|
},
|
|
SuccessfulStreams: 4,
|
|
TargetStats: map[string]targetStats{
|
|
"a": {RouteModes: map[string]int{string(mesh.FabricRouteLAN): 1}},
|
|
"b": {RouteModes: map[string]int{string(mesh.FabricRouteICE): 1}},
|
|
"c": {RouteModes: map[string]int{string(mesh.FabricRouteReverse): 1}},
|
|
"d": {RouteModes: map[string]int{}},
|
|
},
|
|
}
|
|
|
|
reasons := routeModeCoverageVerdictReasons(report)
|
|
if len(reasons) != 1 || !strings.Contains(reasons[0], string(mesh.FabricRouteRelay)) {
|
|
t.Fatalf("reasons = %v, want missing relay route mode", reasons)
|
|
}
|
|
|
|
report.TargetStats["d"] = targetStats{RouteModes: map[string]int{string(mesh.FabricRouteRelay): 1}}
|
|
if reasons := routeModeCoverageVerdictReasons(report); len(reasons) != 0 {
|
|
t.Fatalf("reasons = %v, want full coverage pass", reasons)
|
|
}
|
|
}
|
|
|
|
func TestDisallowedRouteModeVerdictRejectsNonQUICModes(t *testing.T) {
|
|
report := loadtestReport{
|
|
TargetStats: map[string]targetStats{
|
|
"a": {RouteModes: map[string]int{
|
|
"direct_quic": 4,
|
|
"relay": 1,
|
|
"outbound_reverse": 2,
|
|
"wss": 3,
|
|
}},
|
|
},
|
|
}
|
|
reasons := disallowedRouteModeVerdictReasons(report)
|
|
if len(reasons) != 1 ||
|
|
!strings.Contains(reasons[0], "relay:1") ||
|
|
!strings.Contains(reasons[0], "outbound_reverse:2") ||
|
|
!strings.Contains(reasons[0], "wss:3") {
|
|
t.Fatalf("reasons = %v, want compat route mode failure", reasons)
|
|
}
|
|
|
|
report.TargetStats["a"] = targetStats{RouteModes: map[string]int{
|
|
string(mesh.FabricRouteDirect): 1,
|
|
string(mesh.FabricRouteLAN): 1,
|
|
string(mesh.FabricRouteICE): 1,
|
|
string(mesh.FabricRouteReverse): 1,
|
|
string(mesh.FabricRouteRelay): 1,
|
|
}}
|
|
if reasons := disallowedRouteModeVerdictReasons(report); len(reasons) != 0 {
|
|
t.Fatalf("reasons = %v, want QUIC modes accepted", reasons)
|
|
}
|
|
}
|
|
|
|
func TestTargetEndpointPolicyVerdictRejectsNonQUICTargets(t *testing.T) {
|
|
report := loadtestReport{
|
|
Config: loadtestConfig{
|
|
Targets: []string{
|
|
"quic://a:19443",
|
|
"http://b:19443",
|
|
"ws://c:19443",
|
|
"d:19443",
|
|
"",
|
|
},
|
|
},
|
|
}
|
|
|
|
reasons := targetEndpointPolicyVerdictReasons(report)
|
|
if len(reasons) != 1 ||
|
|
!strings.Contains(reasons[0], "http://b:19443") ||
|
|
!strings.Contains(reasons[0], "ws://c:19443") ||
|
|
!strings.Contains(reasons[0], "d:19443") ||
|
|
!strings.Contains(reasons[0], "<empty>") {
|
|
t.Fatalf("reasons = %v, want non-QUIC target failure", reasons)
|
|
}
|
|
|
|
report.Config.Targets = []string{"quic://a:19443", " QUIC://b:19443 "}
|
|
if reasons := targetEndpointPolicyVerdictReasons(report); len(reasons) != 0 {
|
|
t.Fatalf("reasons = %v, want QUIC targets accepted", reasons)
|
|
}
|
|
}
|
|
|
|
func TestRunClientRejectsNonQUICTargetBeforeDial(t *testing.T) {
|
|
_, err := runClient(context.Background(), loadtestConfig{
|
|
Targets: []string{"http://127.0.0.1:19443"},
|
|
Streams: 1,
|
|
Concurrency: 1,
|
|
BytesPerStream: 1,
|
|
PayloadSize: 1,
|
|
})
|
|
if err == nil || !strings.Contains(err.Error(), "non_quic_targets=http://127.0.0.1:19443") {
|
|
t.Fatalf("err = %v, want non-QUIC target validation error", err)
|
|
}
|
|
}
|
|
|
|
func TestFillLoadtestPayloadVariesByStreamAndSequence(t *testing.T) {
|
|
first := make([]byte, 128)
|
|
second := make([]byte, 128)
|
|
third := make([]byte, 128)
|
|
|
|
fillLoadtestPayload(first, 7, 9, 1, 0)
|
|
fillLoadtestPayload(second, 7, 9, 2, int64(len(first)))
|
|
fillLoadtestPayload(third, 8, 10, 1, 0)
|
|
|
|
if bytes.Equal(first, second) {
|
|
t.Fatal("payload did not vary by sequence/offset")
|
|
}
|
|
if bytes.Equal(first, third) {
|
|
t.Fatal("payload did not vary by stream")
|
|
}
|
|
if bytes.Count(first, []byte{first[0]}) == len(first) {
|
|
t.Fatal("payload collapsed to a constant byte")
|
|
}
|
|
}
|
|
|
|
func TestFillLoadtestPayloadIsDeterministic(t *testing.T) {
|
|
first := make([]byte, 128)
|
|
second := make([]byte, 128)
|
|
|
|
fillLoadtestPayload(first, 7, 9, 1, 0)
|
|
fillLoadtestPayload(second, 7, 9, 1, 0)
|
|
|
|
if !bytes.Equal(first, second) {
|
|
t.Fatal("payload is not deterministic")
|
|
}
|
|
}
|
|
|
|
func TestFillLoadtestPayloadHandlesShortFinalChunk(t *testing.T) {
|
|
chunk := make([]byte, 17)
|
|
fillLoadtestPayload(chunk, 7, 9, 3, 256)
|
|
if bytes.Equal(chunk, make([]byte, len(chunk))) {
|
|
t.Fatal("short payload chunk stayed zeroed")
|
|
}
|
|
}
|
|
|
|
func TestVerdictFailsSuccessfulStreamAckMismatch(t *testing.T) {
|
|
report := loadtestReport{
|
|
Config: loadtestConfig{
|
|
FailTarget: -1,
|
|
ImpairTarget: -1,
|
|
Concurrency: 1,
|
|
},
|
|
TotalStreams: 1,
|
|
SuccessfulStreams: 1,
|
|
BytesSent: 1024,
|
|
FramesSent: 2,
|
|
AcksReceived: 1,
|
|
AckMismatchedStreams: 1,
|
|
ChannelOpens: 1,
|
|
ChannelCloses: 1,
|
|
RoutePressure: mesh.FabricRoutePressureSnapshot{AcquiredTotal: 1, ReleasedTotal: 1, MaxActiveTotal: 1},
|
|
}
|
|
|
|
gotVerdict, reasons := verdict(report)
|
|
if gotVerdict != "fail" {
|
|
t.Fatalf("verdict = %q, want fail", gotVerdict)
|
|
}
|
|
found := false
|
|
for _, reason := range reasons {
|
|
if reason == "ack_mismatched_streams=1" {
|
|
found = true
|
|
}
|
|
}
|
|
if !found {
|
|
t.Fatalf("reasons = %v, want ack mismatch reason", reasons)
|
|
}
|
|
}
|
|
|
|
func TestVerdictFailsAckIntegrityError(t *testing.T) {
|
|
report := loadtestReport{
|
|
Config: loadtestConfig{
|
|
FailTarget: -1,
|
|
ImpairTarget: -1,
|
|
Concurrency: 1,
|
|
},
|
|
TotalStreams: 1,
|
|
FailedStreams: 1,
|
|
BytesSent: 1024,
|
|
FramesSent: 1,
|
|
AcksReceived: 1,
|
|
AckIntegrityErrors: 1,
|
|
ChannelOpens: 1,
|
|
ChannelCloses: 1,
|
|
RoutePressure: mesh.FabricRoutePressureSnapshot{AcquiredTotal: 1, ReleasedTotal: 1, MaxActiveTotal: 1},
|
|
}
|
|
|
|
gotVerdict, reasons := verdict(report)
|
|
if gotVerdict != "fail" {
|
|
t.Fatalf("verdict = %q, want fail", gotVerdict)
|
|
}
|
|
found := false
|
|
for _, reason := range reasons {
|
|
if reason == "ack_integrity_errors=1" {
|
|
found = true
|
|
}
|
|
}
|
|
if !found {
|
|
t.Fatalf("reasons = %v, want ack integrity reason", reasons)
|
|
}
|
|
}
|
|
|
|
func TestVerdictFailsBelowMinimumThroughput(t *testing.T) {
|
|
report := loadtestReport{
|
|
Config: loadtestConfig{
|
|
FailTarget: -1,
|
|
ImpairTarget: -1,
|
|
Concurrency: 1,
|
|
MinThroughputMbps: 100,
|
|
},
|
|
TotalStreams: 1,
|
|
SuccessfulStreams: 1,
|
|
BytesSent: 1024,
|
|
FramesSent: 1,
|
|
AcksReceived: 1,
|
|
ThroughputBps: 99 * 1000 * 1000,
|
|
ChannelOpens: 1,
|
|
ChannelCloses: 1,
|
|
RoutePressure: mesh.FabricRoutePressureSnapshot{AcquiredTotal: 1, ReleasedTotal: 1, MaxActiveTotal: 1},
|
|
}
|
|
|
|
gotVerdict, reasons := verdict(report)
|
|
if gotVerdict != "fail" {
|
|
t.Fatalf("verdict = %q, want fail", gotVerdict)
|
|
}
|
|
found := false
|
|
for _, reason := range reasons {
|
|
if strings.HasPrefix(reason, "throughput_bps=") {
|
|
found = true
|
|
}
|
|
}
|
|
if !found {
|
|
t.Fatalf("reasons = %v, want throughput reason", reasons)
|
|
}
|
|
|
|
report.ThroughputBps = 100 * 1000 * 1000
|
|
if gotVerdict, reasons := verdict(report); gotVerdict != "pass" {
|
|
t.Fatalf("verdict = %q reasons=%v, want pass at threshold", gotVerdict, reasons)
|
|
}
|
|
}
|
|
|
|
func TestVerdictFailsBelowMinimumChannelChurn(t *testing.T) {
|
|
report := loadtestReport{
|
|
Config: loadtestConfig{
|
|
FailTarget: -1,
|
|
ImpairTarget: -1,
|
|
Concurrency: 1,
|
|
MinChannelChurn: 1000,
|
|
},
|
|
TotalStreams: 1,
|
|
SuccessfulStreams: 1,
|
|
BytesSent: 1024,
|
|
FramesSent: 1,
|
|
AcksReceived: 1,
|
|
ChannelOpens: 1,
|
|
ChannelCloses: 1,
|
|
ChannelChurnPerSec: 999,
|
|
RoutePressure: mesh.FabricRoutePressureSnapshot{AcquiredTotal: 1, ReleasedTotal: 1, MaxActiveTotal: 1},
|
|
}
|
|
|
|
gotVerdict, reasons := verdict(report)
|
|
if gotVerdict != "fail" {
|
|
t.Fatalf("verdict = %q, want fail", gotVerdict)
|
|
}
|
|
found := false
|
|
for _, reason := range reasons {
|
|
if strings.HasPrefix(reason, "channel_churn_per_sec=") {
|
|
found = true
|
|
}
|
|
}
|
|
if !found {
|
|
t.Fatalf("reasons = %v, want channel churn reason", reasons)
|
|
}
|
|
|
|
report.ChannelChurnPerSec = 1000
|
|
if gotVerdict, reasons := verdict(report); gotVerdict != "pass" {
|
|
t.Fatalf("verdict = %q reasons=%v, want pass at threshold", gotVerdict, reasons)
|
|
}
|
|
}
|
|
|
|
func TestTargetByteDistributionVerdictDetectsSkew(t *testing.T) {
|
|
report := loadtestReport{
|
|
Config: loadtestConfig{
|
|
Targets: []string{"a", "b", "c", "d"},
|
|
FailTarget: -1,
|
|
ImpairTarget: -1,
|
|
Concurrency: 1,
|
|
BytesPerStream: 100,
|
|
},
|
|
SuccessfulStreams: 40,
|
|
BytesSent: 4000,
|
|
TargetStreams: map[string]int{
|
|
"a": 10,
|
|
"b": 10,
|
|
"c": 10,
|
|
"d": 10,
|
|
},
|
|
TargetBytes: map[string]int64{
|
|
"a": 2500,
|
|
"b": 500,
|
|
"c": 500,
|
|
"d": 500,
|
|
},
|
|
}
|
|
|
|
reasons := targetByteDistributionVerdictReasons(report)
|
|
if len(reasons) != 1 || !strings.HasPrefix(reasons[0], "target_byte_distribution_skew=") {
|
|
t.Fatalf("reasons = %v, want byte skew reason", reasons)
|
|
}
|
|
|
|
report.TargetBytes = map[string]int64{
|
|
"a": 1000,
|
|
"b": 1000,
|
|
"c": 1000,
|
|
"d": 1000,
|
|
}
|
|
if reasons := targetByteDistributionVerdictReasons(report); len(reasons) != 0 {
|
|
t.Fatalf("reasons = %v, want balanced bytes pass", reasons)
|
|
}
|
|
}
|
|
|
|
func TestDistributionVerdictChecksSurvivingTargetsAfterFailure(t *testing.T) {
|
|
report := loadtestReport{
|
|
Config: loadtestConfig{
|
|
Targets: []string{"quic://a:1", "quic://b:1", "quic://c:1", "quic://d:1"},
|
|
FailTarget: 0,
|
|
ImpairTarget: -1,
|
|
Concurrency: 8,
|
|
},
|
|
SuccessfulStreams: 90,
|
|
TargetStreams: map[string]int{
|
|
"quic://b:1": 90,
|
|
},
|
|
}
|
|
|
|
reasons := targetDistributionVerdictReasons(report)
|
|
if len(reasons) != 1 || !strings.HasPrefix(reasons[0], "target_distribution_collapsed=1/3_targets_used") {
|
|
t.Fatalf("reasons = %v, want surviving-target collapse", reasons)
|
|
}
|
|
|
|
report.TargetStreams = map[string]int{
|
|
"quic://b:1": 30,
|
|
"quic://c:1": 30,
|
|
"quic://d:1": 30,
|
|
}
|
|
if reasons := targetDistributionVerdictReasons(report); len(reasons) != 0 {
|
|
t.Fatalf("reasons = %v, want balanced surviving targets pass", reasons)
|
|
}
|
|
}
|
|
|
|
func TestRoutePressureVerdictChecksSurvivingTargetsAfterFailure(t *testing.T) {
|
|
targets := []string{"quic://a:1", "quic://b:1", "quic://c:1", "quic://d:1"}
|
|
report := loadtestReport{
|
|
Config: loadtestConfig{
|
|
Targets: targets,
|
|
FailTarget: 0,
|
|
ImpairTarget: -1,
|
|
Concurrency: 12,
|
|
},
|
|
RoutePressure: mesh.FabricRoutePressureSnapshot{
|
|
MaxActive: map[string]int{
|
|
loadtestRouteID(1, targets[1]): 12,
|
|
},
|
|
MaxActiveTotal: 12,
|
|
},
|
|
}
|
|
|
|
reasons := routePressureDistributionVerdictReasons(report)
|
|
if len(reasons) != 1 || !strings.HasPrefix(reasons[0], "route_pressure_distribution_collapsed=1/3_targets_used") {
|
|
t.Fatalf("reasons = %v, want surviving-route-pressure collapse", reasons)
|
|
}
|
|
|
|
report.RoutePressure.MaxActive = map[string]int{
|
|
loadtestRouteID(1, targets[1]): 4,
|
|
loadtestRouteID(2, targets[2]): 4,
|
|
loadtestRouteID(3, targets[3]): 4,
|
|
}
|
|
if reasons := routePressureDistributionVerdictReasons(report); len(reasons) != 0 {
|
|
t.Fatalf("reasons = %v, want balanced surviving route pressure pass", reasons)
|
|
}
|
|
}
|
|
|
|
func TestVerdictFailsOverallAckLatencySLO(t *testing.T) {
|
|
report := loadtestReport{
|
|
Config: loadtestConfig{
|
|
FailTarget: -1,
|
|
ImpairTarget: -1,
|
|
Concurrency: 1,
|
|
MaxAckP95Ms: 10,
|
|
MaxAckP99Ms: 20,
|
|
},
|
|
TotalStreams: 1,
|
|
SuccessfulStreams: 1,
|
|
BytesSent: 1024,
|
|
FramesSent: 1,
|
|
AcksReceived: 1,
|
|
AckP95Ms: 11,
|
|
AckP99Ms: 21,
|
|
ChannelOpens: 1,
|
|
ChannelCloses: 1,
|
|
RoutePressure: mesh.FabricRoutePressureSnapshot{AcquiredTotal: 1, ReleasedTotal: 1, MaxActiveTotal: 1},
|
|
}
|
|
|
|
gotVerdict, reasons := verdict(report)
|
|
if gotVerdict != "fail" {
|
|
t.Fatalf("verdict = %q, want fail", gotVerdict)
|
|
}
|
|
foundP95 := false
|
|
foundP99 := false
|
|
for _, reason := range reasons {
|
|
if strings.HasPrefix(reason, "ack_p95_ms=") {
|
|
foundP95 = true
|
|
}
|
|
if strings.HasPrefix(reason, "ack_p99_ms=") {
|
|
foundP99 = true
|
|
}
|
|
}
|
|
if !foundP95 || !foundP99 {
|
|
t.Fatalf("reasons = %v, want ACK p95 and p99 reasons", reasons)
|
|
}
|
|
}
|
|
|
|
func TestTargetAckVerdictDetectsSlowHealthyTarget(t *testing.T) {
|
|
report := loadtestReport{
|
|
Config: loadtestConfig{
|
|
Targets: []string{"a", "b"},
|
|
FailTarget: -1,
|
|
ImpairTarget: -1,
|
|
MaxTargetAckMs: 10,
|
|
},
|
|
TargetStats: map[string]targetStats{
|
|
"a": {Streams: 10, MaxAckMs: 4},
|
|
"b": {Streams: 10, MaxAckMs: 11},
|
|
},
|
|
}
|
|
|
|
reasons := targetAckVerdictReasons(report)
|
|
if len(reasons) != 1 || !strings.HasPrefix(reasons[0], "target_ack_ms=b:11>10") {
|
|
t.Fatalf("reasons = %v, want slow target ack reason", reasons)
|
|
}
|
|
|
|
report.TargetStats["b"] = targetStats{Streams: 10, MaxAckMs: 10}
|
|
if reasons := targetAckVerdictReasons(report); len(reasons) != 0 {
|
|
t.Fatalf("reasons = %v, want target ack pass at threshold", reasons)
|
|
}
|
|
}
|
|
|
|
func TestVerdictFailsSetupLatencySLO(t *testing.T) {
|
|
report := loadtestReport{
|
|
Config: loadtestConfig{
|
|
FailTarget: -1,
|
|
ImpairTarget: -1,
|
|
Concurrency: 1,
|
|
MaxSetupP95Ms: 10,
|
|
MaxSetupP99Ms: 20,
|
|
},
|
|
TotalStreams: 1,
|
|
SuccessfulStreams: 1,
|
|
BytesSent: 1024,
|
|
FramesSent: 1,
|
|
AcksReceived: 1,
|
|
SetupLatencyP95Ms: 11,
|
|
SetupLatencyP99Ms: 21,
|
|
ChannelOpens: 1,
|
|
ChannelCloses: 1,
|
|
RoutePressure: mesh.FabricRoutePressureSnapshot{AcquiredTotal: 1, ReleasedTotal: 1, MaxActiveTotal: 1},
|
|
}
|
|
|
|
gotVerdict, reasons := verdict(report)
|
|
if gotVerdict != "fail" {
|
|
t.Fatalf("verdict = %q, want fail", gotVerdict)
|
|
}
|
|
foundP95 := false
|
|
foundP99 := false
|
|
for _, reason := range reasons {
|
|
if strings.HasPrefix(reason, "setup_p95_ms=") {
|
|
foundP95 = true
|
|
}
|
|
if strings.HasPrefix(reason, "setup_p99_ms=") {
|
|
foundP99 = true
|
|
}
|
|
}
|
|
if !foundP95 || !foundP99 {
|
|
t.Fatalf("reasons = %v, want setup p95 and p99 reasons", reasons)
|
|
}
|
|
}
|
|
|
|
func TestVerdictFailsRerouteLatencySLO(t *testing.T) {
|
|
report := loadtestReport{
|
|
Config: loadtestConfig{
|
|
FailTarget: -1,
|
|
ImpairTarget: -1,
|
|
Concurrency: 1,
|
|
MaxRerouteP95Ms: 10,
|
|
MaxRerouteP99Ms: 20,
|
|
},
|
|
TotalStreams: 1,
|
|
SuccessfulStreams: 1,
|
|
BytesSent: 1024,
|
|
FramesSent: 1,
|
|
AcksReceived: 1,
|
|
RerouteLatencyP95Ms: 11,
|
|
RerouteLatencyP99Ms: 21,
|
|
ChannelOpens: 1,
|
|
ChannelCloses: 1,
|
|
RoutePressure: mesh.FabricRoutePressureSnapshot{AcquiredTotal: 1, ReleasedTotal: 1, MaxActiveTotal: 1},
|
|
}
|
|
|
|
gotVerdict, reasons := verdict(report)
|
|
if gotVerdict != "fail" {
|
|
t.Fatalf("verdict = %q, want fail", gotVerdict)
|
|
}
|
|
foundP95 := false
|
|
foundP99 := false
|
|
for _, reason := range reasons {
|
|
if strings.HasPrefix(reason, "reroute_p95_ms=") {
|
|
foundP95 = true
|
|
}
|
|
if strings.HasPrefix(reason, "reroute_p99_ms=") {
|
|
foundP99 = true
|
|
}
|
|
}
|
|
if !foundP95 || !foundP99 {
|
|
t.Fatalf("reasons = %v, want reroute p95 and p99 reasons", reasons)
|
|
}
|
|
}
|
|
|
|
func TestShouldQuarantineTarget(t *testing.T) {
|
|
quarantined := []string{
|
|
"ack timeout or session closed",
|
|
"deadline exceeded",
|
|
"connection refused",
|
|
"connection reset by peer",
|
|
"no route to host",
|
|
}
|
|
for _, reason := range quarantined {
|
|
if !shouldQuarantineTarget(reason) {
|
|
t.Fatalf("shouldQuarantineTarget(%q) = false, want true", reason)
|
|
}
|
|
}
|
|
if shouldQuarantineTarget("ack payload checksum mismatch") {
|
|
t.Fatal("checksum mismatch should not quarantine a target")
|
|
}
|
|
if shouldQuarantineTarget("context deadline exceeded") {
|
|
t.Fatal("context deadline should not quarantine a target")
|
|
}
|
|
}
|
|
|
|
func TestSpreadStartDistributesQuarantinedSlot(t *testing.T) {
|
|
targets := []string{"a", "b", "c", "d"}
|
|
health := newTargetHealthTracker()
|
|
health.MarkDegraded("a", "connection refused", time.Minute)
|
|
counts := map[string]int{}
|
|
for index := 0; index < 40; index += len(targets) {
|
|
initial, spread := loadtestSpreadStart(index, len(targets))
|
|
targetIndex := loadtestPreferredTargetIndex(targets, initial, spread, health, -1)
|
|
counts[targets[targetIndex]]++
|
|
}
|
|
if counts["b"] == 0 || counts["c"] == 0 || counts["d"] == 0 {
|
|
t.Fatalf("counts = %v, want degraded slot spread across surviving targets", counts)
|
|
}
|
|
}
|
|
|
|
func TestSpreadUsableTargetDistributesRetries(t *testing.T) {
|
|
targets := []string{"a", "b", "c", "d"}
|
|
health := newTargetHealthTracker()
|
|
health.MarkDegraded("a", "connection refused", time.Minute)
|
|
counts := map[string]int{}
|
|
for cohort := 0; cohort < 90; cohort++ {
|
|
targetIndex := loadtestSpreadUsableTargetIndex(targets, cohort, health, 0)
|
|
counts[targets[targetIndex]]++
|
|
}
|
|
if counts["b"] != 30 || counts["c"] != 30 || counts["d"] != 30 {
|
|
t.Fatalf("counts = %v, want retry load spread evenly across surviving targets", counts)
|
|
}
|
|
}
|
|
|
|
func TestLoadtestLogicalStreamIDAvoidsReservedTransportStreams(t *testing.T) {
|
|
for _, index := range []int{-1, 0, 1, 999, 1000, 10_000} {
|
|
streamID := loadtestLogicalStreamID(index)
|
|
if streamID == mesh.ProductionForwardQUICStreamID || streamID == mesh.SyntheticForwardQUICStreamID {
|
|
t.Fatalf("loadtestLogicalStreamID(%d) = %d, collides with reserved transport stream", index, streamID)
|
|
}
|
|
if streamID < 10_000 {
|
|
t.Fatalf("loadtestLogicalStreamID(%d) = %d, want loadtest stream range", index, streamID)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestLatencyAwareTargetIndexKeepsSlowWANFromOwningPool(t *testing.T) {
|
|
targets := []string{"lan-a", "lan-b", "wan"}
|
|
health := newTargetHealthTracker()
|
|
health.RecordProbes([]targetProbeResult{
|
|
{Target: "lan-a", RTTMs: 4, Usable: true},
|
|
{Target: "lan-b", RTTMs: 5, Usable: true},
|
|
{Target: "wan", RTTMs: 400, Usable: true},
|
|
})
|
|
counts := map[string]int{}
|
|
for index := 0; index < 300; index++ {
|
|
targetIndex := loadtestSpreadUsableTargetIndex(targets, index, health, -1)
|
|
counts[targets[targetIndex]]++
|
|
}
|
|
if counts["wan"] == 0 {
|
|
t.Fatalf("counts = %v, want slow WAN to stay represented", counts)
|
|
}
|
|
if counts["wan"] >= counts["lan-a"] || counts["wan"] >= counts["lan-b"] {
|
|
t.Fatalf("counts = %v, want latency-aware placement to prefer LAN capacity", counts)
|
|
}
|
|
}
|
|
|
|
func TestLatencyAwarePreferredTargetUsesAbsolutePlacementOrdinal(t *testing.T) {
|
|
targets := []string{"lan-a", "lan-b", "lan-c", "wan"}
|
|
health := newTargetHealthTracker()
|
|
health.RecordProbes([]targetProbeResult{
|
|
{Target: "lan-a", RTTMs: 4, Usable: true},
|
|
{Target: "lan-b", RTTMs: 4, Usable: true},
|
|
{Target: "lan-c", RTTMs: 4, Usable: true},
|
|
{Target: "wan", RTTMs: 400, Usable: true},
|
|
})
|
|
counts := map[string]int{}
|
|
for index := 0; index < 500; index++ {
|
|
preferred, spread := loadtestSpreadStart(index, len(targets))
|
|
targetIndex := loadtestPreferredTargetIndex(targets, preferred, spread, health, -1)
|
|
counts[targets[targetIndex]]++
|
|
}
|
|
if len(counts) < len(targets) {
|
|
t.Fatalf("counts = %v, want every probed target represented", counts)
|
|
}
|
|
if counts["wan"] >= counts["lan-a"] || counts["wan"] >= counts["lan-b"] || counts["wan"] >= counts["lan-c"] {
|
|
t.Fatalf("counts = %v, want slow WAN weighted below LAN targets", counts)
|
|
}
|
|
}
|
|
|
|
func TestHeterogeneousProbeRTTRelaxesEqualDistributionVerdict(t *testing.T) {
|
|
report := loadtestReport{
|
|
Config: loadtestConfig{
|
|
Targets: []string{"lan", "wan"},
|
|
Concurrency: 64,
|
|
},
|
|
SuccessfulStreams: 100,
|
|
BytesSent: 100 * 1024,
|
|
TargetStreams: map[string]int{
|
|
"lan": 96,
|
|
"wan": 4,
|
|
},
|
|
TargetBytes: map[string]int64{
|
|
"lan": 96 * 1024,
|
|
"wan": 4 * 1024,
|
|
},
|
|
TargetProbes: []targetProbeResult{
|
|
{Target: "lan", RTTMs: 4, Usable: true},
|
|
{Target: "wan", RTTMs: 400, Usable: true},
|
|
},
|
|
RoutePressure: mesh.FabricRoutePressureSnapshot{
|
|
MaxActive: map[string]int{
|
|
loadtestRouteID(0, "lan"): 32,
|
|
loadtestRouteID(1, "wan"): 1,
|
|
},
|
|
MaxActiveTotal: 32,
|
|
},
|
|
}
|
|
if reasons := targetDistributionVerdictReasons(report); len(reasons) != 0 {
|
|
t.Fatalf("targetDistributionVerdictReasons = %v, want heterogeneous RTT tolerated", reasons)
|
|
}
|
|
if reasons := targetByteDistributionVerdictReasons(report); len(reasons) != 0 {
|
|
t.Fatalf("targetByteDistributionVerdictReasons = %v, want heterogeneous RTT tolerated", reasons)
|
|
}
|
|
if reasons := routePressureDistributionVerdictReasons(report); len(reasons) != 0 {
|
|
t.Fatalf("routePressureDistributionVerdictReasons = %v, want heterogeneous RTT tolerated", reasons)
|
|
}
|
|
}
|
|
|
|
func TestTargetHealthQuarantineExpiresButSnapshotKeepsObservation(t *testing.T) {
|
|
health := newTargetHealthTracker()
|
|
health.MarkDegraded("a", "ack timeout", time.Nanosecond)
|
|
if !health.IsDegraded("a") {
|
|
t.Fatal("target should be degraded immediately")
|
|
}
|
|
time.Sleep(time.Millisecond)
|
|
if health.IsDegraded("a") {
|
|
t.Fatal("target quarantine did not expire")
|
|
}
|
|
snapshot := health.Snapshot()
|
|
if snapshot["a"] != "ack timeout" {
|
|
t.Fatalf("snapshot = %v, want historical degraded observation", snapshot)
|
|
}
|
|
}
|
|
|
|
func TestRoutePressureDistributionVerdictDetectsCollapse(t *testing.T) {
|
|
report := loadtestReport{
|
|
Config: loadtestConfig{
|
|
Targets: []string{"a", "b", "c", "d"},
|
|
FailTarget: -1,
|
|
ImpairTarget: -1,
|
|
Concurrency: 16,
|
|
},
|
|
RoutePressure: mesh.FabricRoutePressureSnapshot{
|
|
MaxActive: map[string]int{
|
|
loadtestRouteID(0, "a"): 16,
|
|
},
|
|
MaxActiveTotal: 16,
|
|
},
|
|
}
|
|
|
|
reasons := routePressureDistributionVerdictReasons(report)
|
|
if len(reasons) != 1 || !strings.HasPrefix(reasons[0], "route_pressure_distribution_collapsed=") {
|
|
t.Fatalf("reasons = %v, want collapsed route pressure reason", reasons)
|
|
}
|
|
}
|
|
|
|
func TestRoutePressureDistributionVerdictDetectsSkew(t *testing.T) {
|
|
report := loadtestReport{
|
|
Config: loadtestConfig{
|
|
Targets: []string{"a", "b", "c", "d"},
|
|
FailTarget: -1,
|
|
ImpairTarget: -1,
|
|
Concurrency: 16,
|
|
},
|
|
RoutePressure: mesh.FabricRoutePressureSnapshot{
|
|
MaxActive: map[string]int{
|
|
loadtestRouteID(0, "a"): 14,
|
|
loadtestRouteID(1, "b"): 2,
|
|
loadtestRouteID(2, "c"): 2,
|
|
loadtestRouteID(3, "d"): 2,
|
|
},
|
|
MaxActiveTotal: 16,
|
|
},
|
|
}
|
|
|
|
reasons := routePressureDistributionVerdictReasons(report)
|
|
if len(reasons) != 1 || !strings.HasPrefix(reasons[0], "route_pressure_distribution_skew=") {
|
|
t.Fatalf("reasons = %v, want route pressure skew reason", reasons)
|
|
}
|
|
|
|
report.RoutePressure.MaxActive = map[string]int{
|
|
loadtestRouteID(0, "a"): 6,
|
|
loadtestRouteID(1, "b"): 6,
|
|
loadtestRouteID(2, "c"): 5,
|
|
loadtestRouteID(3, "d"): 5,
|
|
}
|
|
if reasons := routePressureDistributionVerdictReasons(report); len(reasons) != 0 {
|
|
t.Fatalf("reasons = %v, want balanced route pressure pass", reasons)
|
|
}
|
|
}
|