266 lines
12 KiB
Go
266 lines
12 KiB
Go
package mesh
|
|
|
|
import (
|
|
"testing"
|
|
"time"
|
|
)
|
|
|
|
func TestPeerRecoveryPlanMaintainsBoundedReadyPeers(t *testing.T) {
|
|
now := time.Date(2026, 4, 28, 12, 0, 0, 0, time.UTC)
|
|
plan := PlanPeerRecovery(PeerRecoveryPlanConfig{
|
|
PeerCache: PeerCacheSnapshot{
|
|
Entries: []PeerCacheEntry{
|
|
recoveryPlanPeer("node-a", true, false, "route_adjacent"),
|
|
recoveryPlanPeer("node-b", true, false, "route_adjacent"),
|
|
recoveryPlanPeer("node-c", true, false, "peer_endpoint"),
|
|
recoveryPlanPeer("node-d", true, false, "peer_endpoint"),
|
|
},
|
|
},
|
|
Connections: PeerConnectionSnapshot{Entries: []PeerConnectionState{
|
|
{NodeID: "node-a", State: PeerConnectionReady, LastLatencyMs: 40},
|
|
{NodeID: "node-b", State: PeerConnectionReady, LastLatencyMs: 20},
|
|
{NodeID: "node-c", State: PeerConnectionReady, LastLatencyMs: 30},
|
|
{NodeID: "node-d", State: PeerConnectionReady, LastLatencyMs: 10},
|
|
}},
|
|
Now: now,
|
|
})
|
|
|
|
if plan.Mode != PeerRecoveryModeSteady || !plan.Healthy {
|
|
t.Fatalf("unexpected plan health: %+v", plan)
|
|
}
|
|
if plan.TargetReadyPeers != DefaultStablePeerTarget || len(plan.Candidates) != DefaultStablePeerTarget {
|
|
t.Fatalf("unexpected bounded candidates: %+v", plan)
|
|
}
|
|
for _, candidate := range plan.Candidates {
|
|
if candidate.Reason != "maintain_ready" {
|
|
t.Fatalf("unexpected candidate reason: %+v", candidate)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestPeerRecoveryPlanAddsRecoverySeedWhenReadyDeficit(t *testing.T) {
|
|
now := time.Date(2026, 4, 28, 12, 0, 0, 0, time.UTC)
|
|
plan := PlanPeerRecovery(PeerRecoveryPlanConfig{
|
|
PeerCache: PeerCacheSnapshot{
|
|
Entries: []PeerCacheEntry{
|
|
recoveryPlanPeer("node-a", true, false, "route_adjacent"),
|
|
recoveryPlanPeer("node-b", true, false, "route_adjacent"),
|
|
recoveryPlanPeer("node-seed", false, true, ""),
|
|
},
|
|
},
|
|
Connections: PeerConnectionSnapshot{Entries: []PeerConnectionState{
|
|
{NodeID: "node-a", State: PeerConnectionReady, LastLatencyMs: 20},
|
|
{NodeID: "node-b", State: PeerConnectionBackoff, BackoffUntil: now.Add(time.Minute)},
|
|
}},
|
|
Now: now,
|
|
})
|
|
|
|
if plan.Mode != PeerRecoveryModeRecovery || plan.Healthy {
|
|
t.Fatalf("unexpected recovery mode: %+v", plan)
|
|
}
|
|
if plan.Deficit != 2 || plan.RecoverySeedCandidateCount != 1 {
|
|
t.Fatalf("unexpected deficit/seed count: %+v", plan)
|
|
}
|
|
if !recoveryPlanHasCandidate(plan, "node-seed", "recover_seed") {
|
|
t.Fatalf("recovery seed was not selected: %+v", plan.Candidates)
|
|
}
|
|
if recoveryPlanHasCandidate(plan, "node-b", "") {
|
|
t.Fatalf("active backoff peer should not be selected: %+v", plan.Candidates)
|
|
}
|
|
}
|
|
|
|
func TestPeerRecoveryPlanTreatsRelayReadyPeersAsRecoveryGap(t *testing.T) {
|
|
now := time.Date(2026, 4, 28, 12, 0, 0, 0, time.UTC)
|
|
plan := PlanPeerRecovery(PeerRecoveryPlanConfig{
|
|
PeerCache: PeerCacheSnapshot{
|
|
Entries: []PeerCacheEntry{
|
|
{
|
|
NodeID: "node-c",
|
|
Endpoint: "quic://relay:19443",
|
|
Warm: true,
|
|
WarmReason: "rendezvous_lease",
|
|
RendezvousLeaseID: "lease-1",
|
|
RelayNodeID: "node-r",
|
|
RelayEndpoint: "quic://relay:19443",
|
|
RelayQUIC: true,
|
|
},
|
|
},
|
|
},
|
|
Connections: PeerConnectionSnapshot{Entries: []PeerConnectionState{
|
|
{NodeID: "node-c", State: PeerConnectionRelayReady, LastLatencyMs: 15},
|
|
}},
|
|
Now: now,
|
|
})
|
|
|
|
if plan.Mode != PeerRecoveryModeRecovery || plan.Healthy {
|
|
t.Fatalf("unexpected relay-ready recovery plan: %+v", plan)
|
|
}
|
|
if !recoveryPlanHasCandidate(plan, "node-c", "maintain_ready") {
|
|
t.Fatalf("relay-ready peer was not maintained: %+v", plan.Candidates)
|
|
}
|
|
if plan.ReadyPeerCount != 0 || plan.Deficit != 1 {
|
|
t.Fatalf("relay-ready peer should not satisfy direct-ready target: %+v", plan)
|
|
}
|
|
}
|
|
|
|
func TestPeerRecoveryPlanCapsTargetByConnectablePeers(t *testing.T) {
|
|
now := time.Date(2026, 4, 28, 12, 0, 0, 0, time.UTC)
|
|
plan := PlanPeerRecovery(PeerRecoveryPlanConfig{
|
|
PeerCache: PeerCacheSnapshot{Entries: []PeerCacheEntry{
|
|
{NodeID: "node-a", Warm: true, WarmReason: "route_adjacent"},
|
|
recoveryPlanPeer("node-b", true, false, "route_adjacent"),
|
|
}},
|
|
Connections: PeerConnectionSnapshot{Entries: []PeerConnectionState{
|
|
{NodeID: "node-b", State: PeerConnectionReady},
|
|
}},
|
|
Now: now,
|
|
})
|
|
|
|
if plan.TargetReadyPeers != 1 || !plan.Healthy {
|
|
t.Fatalf("target should be capped by connectable peers: %+v", plan)
|
|
}
|
|
}
|
|
|
|
func TestPeerRecoveryPlanPrefersExternalRegionsWhenTrimmingReadyPeers(t *testing.T) {
|
|
now := time.Date(2026, 5, 18, 12, 0, 0, 0, time.UTC)
|
|
plan := PlanPeerRecovery(PeerRecoveryPlanConfig{
|
|
PeerCache: PeerCacheSnapshot{
|
|
Entries: []PeerCacheEntry{
|
|
{NodeID: "node-home-a", Endpoint: "quic://node-home-a:19443", Warm: true, WarmReason: "route_adjacent", BestRegion: "home"},
|
|
{NodeID: "node-home-b", Endpoint: "quic://node-home-b:19443", Warm: true, WarmReason: "route_adjacent", BestRegion: "home"},
|
|
{NodeID: "node-usa", Endpoint: "quic://node-usa:19443", Warm: true, WarmReason: "route_adjacent", BestRegion: "usa"},
|
|
{NodeID: "node-ifcm", Endpoint: "quic://node-ifcm:19443", Warm: true, WarmReason: "route_adjacent", BestRegion: "ifcm"},
|
|
},
|
|
},
|
|
Connections: PeerConnectionSnapshot{Entries: []PeerConnectionState{
|
|
{NodeID: "node-home-a", State: PeerConnectionReady, LastLatencyMs: 20},
|
|
{NodeID: "node-home-b", State: PeerConnectionReady, LastLatencyMs: 20},
|
|
{NodeID: "node-usa", State: PeerConnectionReady, LastLatencyMs: 20},
|
|
{NodeID: "node-ifcm", State: PeerConnectionReady, LastLatencyMs: 20},
|
|
}},
|
|
PreferredRegion: "home",
|
|
Now: now,
|
|
})
|
|
|
|
if len(plan.Candidates) != DefaultStablePeerTarget {
|
|
t.Fatalf("candidate count = %d, want %d", len(plan.Candidates), DefaultStablePeerTarget)
|
|
}
|
|
if !recoveryPlanHasCandidate(plan, "node-usa", "maintain_ready") || !recoveryPlanHasCandidate(plan, "node-ifcm", "maintain_ready") {
|
|
t.Fatalf("expected external-region peers to be retained: %+v", plan.Candidates)
|
|
}
|
|
}
|
|
|
|
func TestPeerRecoveryPlanPrefersPublicIngressAtSameRegion(t *testing.T) {
|
|
now := time.Date(2026, 5, 18, 12, 0, 0, 0, time.UTC)
|
|
plan := PlanPeerRecovery(PeerRecoveryPlanConfig{
|
|
PeerCache: PeerCacheSnapshot{
|
|
Entries: []PeerCacheEntry{
|
|
{NodeID: "node-home-private-a", Endpoint: "quic://10.0.0.2:19443", Warm: true, WarmReason: "route_adjacent", BestRegion: "home"},
|
|
{NodeID: "node-home-private-b", Endpoint: "quic://10.0.0.3:19443", Warm: true, WarmReason: "route_adjacent", BestRegion: "home"},
|
|
{NodeID: "node-home-public", Endpoint: "quic://94.141.118.222:19199", Warm: true, WarmReason: "route_adjacent", BestRegion: "home", PublicIngressCount: 1},
|
|
{NodeID: "node-usa", Endpoint: "quic://195.123.240.88:19131", Warm: true, WarmReason: "route_adjacent", BestRegion: "usa", PublicIngressCount: 1},
|
|
},
|
|
},
|
|
Connections: PeerConnectionSnapshot{Entries: []PeerConnectionState{
|
|
{NodeID: "node-home-private-a", State: PeerConnectionReady, LastLatencyMs: 20},
|
|
{NodeID: "node-home-private-b", State: PeerConnectionReady, LastLatencyMs: 20},
|
|
{NodeID: "node-home-public", State: PeerConnectionReady, LastLatencyMs: 20},
|
|
{NodeID: "node-usa", State: PeerConnectionReady, LastLatencyMs: 20},
|
|
}},
|
|
PreferredRegion: "home",
|
|
Now: now,
|
|
})
|
|
|
|
if len(plan.Candidates) != DefaultStablePeerTarget {
|
|
t.Fatalf("candidate count = %d, want %d", len(plan.Candidates), DefaultStablePeerTarget)
|
|
}
|
|
if !recoveryPlanHasCandidate(plan, "node-home-public", "maintain_ready") {
|
|
t.Fatalf("expected public-ingress home peer to be retained: %+v", plan.Candidates)
|
|
}
|
|
}
|
|
|
|
func TestPeerRecoveryPlanRetainsDistinctExternalRegionsWhenAvailable(t *testing.T) {
|
|
now := time.Date(2026, 5, 19, 12, 0, 0, 0, time.UTC)
|
|
plan := PlanPeerRecovery(PeerRecoveryPlanConfig{
|
|
PeerCache: PeerCacheSnapshot{
|
|
Entries: []PeerCacheEntry{
|
|
{NodeID: "node-home-a", Endpoint: "quic://node-home-a:19443", Warm: true, WarmReason: "route_adjacent", BestRegion: "home"},
|
|
{NodeID: "node-home-b", Endpoint: "quic://node-home-b:19443", Warm: true, WarmReason: "route_adjacent", BestRegion: "home"},
|
|
{NodeID: "node-home-c", Endpoint: "quic://node-home-c:19443", Warm: true, WarmReason: "route_adjacent", BestRegion: "home"},
|
|
{NodeID: "node-usa-a", Endpoint: "quic://node-usa-a:19443", Warm: true, WarmReason: "route_adjacent", BestRegion: "usa", PublicIngressCount: 1},
|
|
{NodeID: "node-usa-b", Endpoint: "quic://node-usa-b:19443", Warm: true, WarmReason: "route_adjacent", BestRegion: "usa", PublicIngressCount: 1},
|
|
{NodeID: "node-ifcm", Endpoint: "quic://node-ifcm:19443", Warm: true, WarmReason: "route_adjacent", BestRegion: "ifcm", PublicIngressCount: 1},
|
|
},
|
|
},
|
|
Connections: PeerConnectionSnapshot{Entries: []PeerConnectionState{
|
|
{NodeID: "node-home-a", State: PeerConnectionReady, LastLatencyMs: 20},
|
|
{NodeID: "node-home-b", State: PeerConnectionReady, LastLatencyMs: 20},
|
|
{NodeID: "node-home-c", State: PeerConnectionReady, LastLatencyMs: 20},
|
|
{NodeID: "node-usa-a", State: PeerConnectionReady, LastLatencyMs: 20},
|
|
{NodeID: "node-usa-b", State: PeerConnectionReady, LastLatencyMs: 20},
|
|
{NodeID: "node-ifcm", State: PeerConnectionReady, LastLatencyMs: 20},
|
|
}},
|
|
PreferredRegion: "home",
|
|
Now: now,
|
|
})
|
|
|
|
if len(plan.Candidates) != DefaultStablePeerTarget {
|
|
t.Fatalf("candidate count = %d, want %d", len(plan.Candidates), DefaultStablePeerTarget)
|
|
}
|
|
if !recoveryPlanHasCandidate(plan, "node-usa-a", "maintain_ready") && !recoveryPlanHasCandidate(plan, "node-usa-b", "maintain_ready") {
|
|
t.Fatalf("expected at least one usa candidate to be retained: %+v", plan.Candidates)
|
|
}
|
|
if !recoveryPlanHasCandidate(plan, "node-ifcm", "maintain_ready") {
|
|
t.Fatalf("expected ifcm candidate to be retained for area diversity: %+v", plan.Candidates)
|
|
}
|
|
}
|
|
|
|
func TestPeerRecoveryPlanSteadyModeAddsMissingExternalAreaCandidate(t *testing.T) {
|
|
now := time.Date(2026, 5, 19, 12, 0, 0, 0, time.UTC)
|
|
plan := PlanPeerRecovery(PeerRecoveryPlanConfig{
|
|
PeerCache: PeerCacheSnapshot{
|
|
Entries: []PeerCacheEntry{
|
|
{NodeID: "node-test-a", Endpoint: "quic://node-test-a:19443", Warm: true, WarmReason: "route_adjacent", BestRegion: "test"},
|
|
{NodeID: "node-test-b", Endpoint: "quic://node-test-b:19443", Warm: true, WarmReason: "route_adjacent", BestRegion: "test"},
|
|
{NodeID: "node-usa", Endpoint: "quic://node-usa:19443", Warm: true, WarmReason: "route_adjacent", BestRegion: "usa", PublicIngressCount: 1},
|
|
{NodeID: "node-home", Endpoint: "quic://node-home:19443", Warm: true, WarmReason: "route_adjacent", BestRegion: "home", PublicIngressCount: 1},
|
|
},
|
|
},
|
|
Connections: PeerConnectionSnapshot{Entries: []PeerConnectionState{
|
|
{NodeID: "node-test-a", State: PeerConnectionReady, LastLatencyMs: 10},
|
|
{NodeID: "node-test-b", State: PeerConnectionReady, LastLatencyMs: 10},
|
|
{NodeID: "node-usa", State: PeerConnectionReady, LastLatencyMs: 10},
|
|
{NodeID: "node-home", State: PeerConnectionDegraded, LastLatencyMs: 20},
|
|
}},
|
|
PreferredRegion: "test",
|
|
Now: now,
|
|
})
|
|
|
|
if len(plan.Candidates) != DefaultStablePeerTarget {
|
|
t.Fatalf("candidate count = %d, want %d", len(plan.Candidates), DefaultStablePeerTarget)
|
|
}
|
|
if !recoveryPlanHasCandidate(plan, "node-home", "recover_external_area") {
|
|
t.Fatalf("expected missing external area candidate to be retained: %+v", plan.Candidates)
|
|
}
|
|
}
|
|
|
|
func recoveryPlanPeer(nodeID string, warm bool, recoverySeed bool, warmReason string) PeerCacheEntry {
|
|
return PeerCacheEntry{
|
|
NodeID: nodeID,
|
|
Endpoint: "quic://" + nodeID + ":19443",
|
|
Warm: warm,
|
|
WarmReason: warmReason,
|
|
RecoverySeed: recoverySeed,
|
|
}
|
|
}
|
|
|
|
func recoveryPlanHasCandidate(plan PeerRecoveryPlan, nodeID string, reason string) bool {
|
|
for _, candidate := range plan.Candidates {
|
|
if candidate.NodeID != nodeID {
|
|
continue
|
|
}
|
|
return reason == "" || candidate.Reason == reason
|
|
}
|
|
return false
|
|
}
|