8713 lines
321 KiB
Go
8713 lines
321 KiB
Go
package cluster
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"reflect"
|
|
"sort"
|
|
"strconv"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/example/remote-access-platform/backend/internal/platform/clusterauth"
|
|
"github.com/example/remote-access-platform/backend/internal/platform/secrets"
|
|
"github.com/jackc/pgx/v5"
|
|
)
|
|
|
|
func TestHashJoinTokenDoesNotStoreRawToken(t *testing.T) {
|
|
raw := "rap_join_example"
|
|
hashed, err := hashJoinToken(raw)
|
|
if err != nil {
|
|
t.Fatalf("hash join token: %v", err)
|
|
}
|
|
if hashed == raw {
|
|
t.Fatal("hash must not equal raw token")
|
|
}
|
|
if got, wantPrefix := hashed[:len(joinTokenHashPrefix)], joinTokenHashPrefix; got != wantPrefix {
|
|
t.Fatalf("hash prefix = %q, want %q", got, wantPrefix)
|
|
}
|
|
hashedAgain, err := hashJoinToken(raw)
|
|
if err != nil {
|
|
t.Fatalf("hash join token again: %v", err)
|
|
}
|
|
if hashed != hashedAgain {
|
|
t.Fatal("hash must be deterministic")
|
|
}
|
|
}
|
|
|
|
func TestClusterAuthorityPrivateKeyEncodingUsesSecretEncryptor(t *testing.T) {
|
|
encryptor, err := secrets.NewEncryptor("MDEyMzQ1Njc4OWFiY2RlZjAxMjM0NTY3ODlhYmNkZWY=", "test-key")
|
|
if err != nil {
|
|
t.Fatalf("NewEncryptor: %v", err)
|
|
}
|
|
store := (&PostgresStore{}).WithClusterKeyEncryptor(encryptor)
|
|
|
|
encoded, err := store.encodeClusterAuthorityPrivateKey("cluster-1", "private-key")
|
|
if err != nil {
|
|
t.Fatalf("encodeClusterAuthorityPrivateKey: %v", err)
|
|
}
|
|
if encoded == "private-key" || !strings.HasPrefix(encoded, encryptedClusterAuthorityKeyPrefix) {
|
|
t.Fatalf("private key was not encrypted: %q", encoded)
|
|
}
|
|
decoded, err := store.decodeClusterAuthorityPrivateKey("cluster-1", encoded)
|
|
if err != nil {
|
|
t.Fatalf("decodeClusterAuthorityPrivateKey: %v", err)
|
|
}
|
|
if decoded != "private-key" {
|
|
t.Fatalf("decoded private key = %q", decoded)
|
|
}
|
|
if _, err := store.decodeClusterAuthorityPrivateKey("cluster-2", encoded); err == nil {
|
|
t.Fatal("expected wrong cluster AAD to fail")
|
|
}
|
|
}
|
|
|
|
func TestNodeUpdateHintAssignsUpdateServiceSubscription(t *testing.T) {
|
|
targetVersion := "0.2.15"
|
|
now := time.Date(2026, 5, 2, 8, 0, 0, 0, time.UTC)
|
|
repo := &fakeRepository{
|
|
nodeUpdatePolicies: map[string]NodeUpdatePolicy{
|
|
"node-1|rap-node-agent": {
|
|
ClusterID: "cluster-1",
|
|
NodeID: "node-1",
|
|
Product: "rap-node-agent",
|
|
Channel: "dev",
|
|
TargetVersion: &targetVersion,
|
|
Enabled: true,
|
|
UpdatedAt: now,
|
|
},
|
|
},
|
|
updateServiceCandidates: []NodeUpdateServiceCandidate{{
|
|
NodeID: "update-1",
|
|
NodeName: "update-cache-1",
|
|
Endpoint: "http://10.0.0.5:19131",
|
|
Region: "office",
|
|
}},
|
|
}
|
|
service := NewService(repo)
|
|
service.now = func() time.Time { return now }
|
|
|
|
hint := service.GetNodeUpdateHint(context.Background(), "cluster-1", "node-1")
|
|
|
|
if !hint.CheckNow || hint.Generation == "" {
|
|
t.Fatalf("expected update hint generation, got %+v", hint)
|
|
}
|
|
if hint.DeliveryMode != "update_service_subscription" || hint.SubscriptionStatus != "subscribed" {
|
|
t.Fatalf("unexpected subscription state: %+v", hint)
|
|
}
|
|
if hint.FallbackPollSeconds != 21600 {
|
|
t.Fatalf("fallback poll seconds = %d", hint.FallbackPollSeconds)
|
|
}
|
|
if hint.UpdateService == nil || hint.UpdateService.NodeID != "update-1" || hint.UpdateService.Status != "assigned" {
|
|
t.Fatalf("unexpected update service assignment: %+v", hint.UpdateService)
|
|
}
|
|
}
|
|
|
|
func TestNodeUpdateHintFallsBackWhenNoUpdateServiceHealthy(t *testing.T) {
|
|
targetVersion := "0.2.15"
|
|
now := time.Date(2026, 5, 2, 8, 0, 0, 0, time.UTC)
|
|
repo := &fakeRepository{
|
|
nodeUpdatePolicies: map[string]NodeUpdatePolicy{
|
|
"node-1|rap-host-agent": {
|
|
ClusterID: "cluster-1",
|
|
NodeID: "node-1",
|
|
Product: "rap-host-agent",
|
|
Channel: "dev",
|
|
TargetVersion: &targetVersion,
|
|
Enabled: true,
|
|
UpdatedAt: now,
|
|
},
|
|
},
|
|
fabricRebuildAttempts: []FabricServiceChannelRouteRebuildAttempt{{
|
|
ID: "fsc-rebuild-guard-1",
|
|
ClusterID: "cluster-1",
|
|
ReporterNodeID: "entry-1",
|
|
ServiceClass: FabricServiceClassVPNPackets,
|
|
RouteID: "route-bad",
|
|
ReplacementRouteID: "route-outside-exit",
|
|
RebuildRequestID: "fsc-remediation:channel-guard:rebuild_route:route-outside-exit",
|
|
RebuildStatus: "rejected",
|
|
RebuildReason: "replacement_exit_outside_signed_pool_policy",
|
|
DecisionSource: "service_channel_remediation_command",
|
|
Outcome: "policy_guard_rejected",
|
|
PolicyFingerprint: "pool-fingerprint-1",
|
|
CreatedAt: now,
|
|
UpdatedAt: now,
|
|
}},
|
|
}
|
|
service := NewService(repo)
|
|
service.now = func() time.Time { return now }
|
|
|
|
hint := service.GetNodeUpdateHint(context.Background(), "cluster-1", "node-1")
|
|
|
|
if !hint.CheckNow || hint.UpdateService == nil {
|
|
t.Fatalf("expected fallback hint with update service object, got %+v", hint)
|
|
}
|
|
if hint.UpdateService.Status != "control_plane_fallback" {
|
|
t.Fatalf("fallback status = %q", hint.UpdateService.Status)
|
|
}
|
|
}
|
|
|
|
func TestCreateJoinTokenRequiresPlatformAdmin(t *testing.T) {
|
|
store := &fakeRepository{platformRole: "user"}
|
|
service := NewService(store)
|
|
|
|
_, err := service.CreateJoinToken(context.Background(), CreateJoinTokenInput{
|
|
ActorUserID: "user-1",
|
|
ClusterID: "cluster-1",
|
|
})
|
|
if !errors.Is(err, ErrAccessDenied) {
|
|
t.Fatalf("err = %v, want ErrAccessDenied", err)
|
|
}
|
|
}
|
|
|
|
func TestCreateJoinTokenStoresHashOnlyAndReturnsRawOnce(t *testing.T) {
|
|
store := &fakeRepository{platformRole: PlatformRoleAdmin}
|
|
service := NewService(store)
|
|
service.now = func() time.Time { return time.Date(2026, 4, 26, 12, 0, 0, 0, time.UTC) }
|
|
|
|
created, err := service.CreateJoinToken(context.Background(), CreateJoinTokenInput{
|
|
ActorUserID: "admin-1",
|
|
ClusterID: "cluster-1",
|
|
Scope: json.RawMessage(`{"roles":["rdp-worker"]}`),
|
|
MaxUses: 1,
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("create join token: %v", err)
|
|
}
|
|
if created.Token == "" {
|
|
t.Fatal("raw token must be returned to caller once")
|
|
}
|
|
if store.lastTokenHash == "" || store.lastTokenHash == created.Token {
|
|
t.Fatalf("stored token hash = %q, raw token = %q", store.lastTokenHash, created.Token)
|
|
}
|
|
if created.AuthoritySignature == nil || len(created.AuthorityPayload) == 0 {
|
|
t.Fatalf("created token missing authority signature: %+v", created.NodeJoinToken)
|
|
}
|
|
if err := clusterauth.VerifyRaw(store.clusterAuthority.PublicKey, created.AuthorityPayload, *created.AuthoritySignature); err != nil {
|
|
t.Fatalf("verify token authority signature: %v", err)
|
|
}
|
|
}
|
|
|
|
func TestUpdateClusterRequiresMutableAuthority(t *testing.T) {
|
|
store := &fakeRepository{
|
|
platformRole: PlatformRoleAdmin,
|
|
authorityState: ClusterAuthorityState{
|
|
ClusterID: "cluster-1",
|
|
AuthorityState: "minority",
|
|
MutationMode: "read_only",
|
|
},
|
|
}
|
|
service := NewService(store)
|
|
|
|
_, err := service.UpdateCluster(context.Background(), UpdateClusterInput{
|
|
ActorUserID: "admin-1",
|
|
ClusterID: "cluster-1",
|
|
Name: "Cluster One",
|
|
Status: ClusterStatusActive,
|
|
Metadata: json.RawMessage(`{}`),
|
|
})
|
|
if !errors.Is(err, ErrClusterReadOnly) {
|
|
t.Fatalf("err = %v, want ErrClusterReadOnly", err)
|
|
}
|
|
}
|
|
|
|
func TestUpdateClusterValidatesStatusAndMetadata(t *testing.T) {
|
|
store := &fakeRepository{platformRole: PlatformRoleAdmin}
|
|
service := NewService(store)
|
|
|
|
_, err := service.UpdateCluster(context.Background(), UpdateClusterInput{
|
|
ActorUserID: "admin-1",
|
|
ClusterID: "cluster-1",
|
|
Name: "Cluster One",
|
|
Status: "unknown",
|
|
Metadata: json.RawMessage(`{}`),
|
|
})
|
|
if !errors.Is(err, ErrInvalidPayload) {
|
|
t.Fatalf("err = %v, want ErrInvalidPayload", err)
|
|
}
|
|
|
|
_, err = service.UpdateCluster(context.Background(), UpdateClusterInput{
|
|
ActorUserID: "admin-1",
|
|
ClusterID: "cluster-1",
|
|
Name: "Cluster One",
|
|
Status: ClusterStatusActive,
|
|
Metadata: json.RawMessage(`{`),
|
|
})
|
|
if err == nil || !strings.Contains(err.Error(), "metadata") {
|
|
t.Fatalf("err = %v, want metadata validation error", err)
|
|
}
|
|
}
|
|
|
|
func TestCreateNodeGroupValidatesNameAndMetadata(t *testing.T) {
|
|
store := &fakeRepository{platformRole: PlatformRoleAdmin}
|
|
service := NewService(store)
|
|
|
|
_, err := service.CreateNodeGroup(context.Background(), CreateNodeGroupInput{
|
|
ActorUserID: "admin-1",
|
|
ClusterID: "cluster-1",
|
|
Name: " ",
|
|
})
|
|
if !errors.Is(err, ErrInvalidPayload) {
|
|
t.Fatalf("err = %v, want ErrInvalidPayload", err)
|
|
}
|
|
|
|
_, err = service.CreateNodeGroup(context.Background(), CreateNodeGroupInput{
|
|
ActorUserID: "admin-1",
|
|
ClusterID: "cluster-1",
|
|
Name: "DC-1",
|
|
Metadata: json.RawMessage(`{`),
|
|
})
|
|
if err == nil || !strings.Contains(err.Error(), "metadata") {
|
|
t.Fatalf("err = %v, want metadata validation error", err)
|
|
}
|
|
}
|
|
|
|
func TestAssignNodeToGroupPreservesConcreteMembership(t *testing.T) {
|
|
groupID := "group-1"
|
|
store := &fakeRepository{platformRole: PlatformRoleAdmin}
|
|
service := NewService(store)
|
|
|
|
node, err := service.AssignNodeToGroup(context.Background(), AssignNodeGroupInput{
|
|
ActorUserID: "admin-1",
|
|
ClusterID: "cluster-1",
|
|
NodeID: "node-1",
|
|
GroupID: &groupID,
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("assign node group: %v", err)
|
|
}
|
|
if node.ID != "node-1" || node.NodeGroupID == nil || *node.NodeGroupID != groupID {
|
|
t.Fatalf("unexpected node group assignment: %+v", node)
|
|
}
|
|
if store.lastAssignGroupInput.NodeID != "node-1" || store.lastAssignGroupInput.GroupID == nil {
|
|
t.Fatalf("assignment input not preserved: %+v", store.lastAssignGroupInput)
|
|
}
|
|
}
|
|
|
|
func TestCreateFabricEntryPointValidatesControlPlanePayload(t *testing.T) {
|
|
store := &fakeRepository{platformRole: PlatformRoleAdmin}
|
|
service := NewService(store)
|
|
|
|
_, err := service.CreateFabricEntryPoint(context.Background(), CreateFabricEntryPointInput{
|
|
ActorUserID: "admin-1",
|
|
ClusterID: "cluster-1",
|
|
Name: " ",
|
|
EndpointType: "client_access",
|
|
})
|
|
if !errors.Is(err, ErrInvalidPayload) {
|
|
t.Fatalf("err = %v, want ErrInvalidPayload", err)
|
|
}
|
|
|
|
_, err = service.CreateFabricEntryPoint(context.Background(), CreateFabricEntryPointInput{
|
|
ActorUserID: "admin-1",
|
|
ClusterID: "cluster-1",
|
|
Name: "Main Entry",
|
|
EndpointType: "client_access",
|
|
Policy: json.RawMessage(`{`),
|
|
})
|
|
if err == nil || !strings.Contains(err.Error(), "valid json") {
|
|
t.Fatalf("err = %v, want json validation error", err)
|
|
}
|
|
}
|
|
|
|
func TestCreateFabricEgressPoolDefaultsAndAudits(t *testing.T) {
|
|
store := &fakeRepository{platformRole: PlatformRoleAdmin}
|
|
service := NewService(store)
|
|
|
|
item, err := service.CreateFabricEgressPool(context.Background(), CreateFabricEgressPoolInput{
|
|
ActorUserID: "admin-1",
|
|
ClusterID: "cluster-1",
|
|
Name: "Office Moscow",
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("create egress pool: %v", err)
|
|
}
|
|
if item.Status != "active" || string(item.RouteScope) != "{}" {
|
|
t.Fatalf("unexpected egress pool defaults: %+v", item)
|
|
}
|
|
if len(store.auditEvents) == 0 || store.auditEvents[len(store.auditEvents)-1].EventType != "fabric.egress_pool.created" {
|
|
t.Fatalf("missing egress pool audit event: %+v", store.auditEvents)
|
|
}
|
|
}
|
|
|
|
func TestAssignNodeRoleRejectsUnknownRole(t *testing.T) {
|
|
store := &fakeRepository{platformRole: PlatformRoleAdmin}
|
|
service := NewService(store)
|
|
|
|
_, err := service.AssignNodeRole(context.Background(), AssignNodeRoleInput{
|
|
ActorUserID: "admin-1",
|
|
ClusterID: "cluster-1",
|
|
NodeID: "node-1",
|
|
Role: "can_run_rdp_worker",
|
|
})
|
|
if !errors.Is(err, ErrInvalidNodeRole) {
|
|
t.Fatalf("err = %v, want ErrInvalidNodeRole", err)
|
|
}
|
|
}
|
|
|
|
func TestAttachExistingNodeRequiresPlatformAdmin(t *testing.T) {
|
|
store := &fakeRepository{platformRole: "user"}
|
|
service := NewService(store)
|
|
|
|
_, err := service.AttachExistingNodeToCluster(context.Background(), AttachExistingNodeInput{
|
|
ActorUserID: "user-1",
|
|
ClusterID: "cluster-1",
|
|
NodeID: "node-1",
|
|
})
|
|
if !errors.Is(err, ErrAccessDenied) {
|
|
t.Fatalf("err = %v, want ErrAccessDenied", err)
|
|
}
|
|
}
|
|
|
|
func TestAttachExistingNodeRejectsUnknownRole(t *testing.T) {
|
|
store := &fakeRepository{platformRole: PlatformRoleAdmin}
|
|
service := NewService(store)
|
|
|
|
_, err := service.AttachExistingNodeToCluster(context.Background(), AttachExistingNodeInput{
|
|
ActorUserID: "admin-1",
|
|
ClusterID: "cluster-1",
|
|
NodeID: "node-1",
|
|
Roles: []string{"can_run_rdp_worker"},
|
|
})
|
|
if !errors.Is(err, ErrInvalidNodeRole) {
|
|
t.Fatalf("err = %v, want ErrInvalidNodeRole", err)
|
|
}
|
|
}
|
|
|
|
func TestAttachExistingNodeUsesConcreteNodeAndRoles(t *testing.T) {
|
|
store := &fakeRepository{platformRole: PlatformRoleAdmin}
|
|
service := NewService(store)
|
|
|
|
node, err := service.AttachExistingNodeToCluster(context.Background(), AttachExistingNodeInput{
|
|
ActorUserID: "admin-1",
|
|
ClusterID: "cluster-1",
|
|
NodeID: "node-1",
|
|
Roles: []string{"entry-node", "rdp-worker"},
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("attach existing node: %v", err)
|
|
}
|
|
if node.ID != "node-1" || node.MembershipStatus != "active" {
|
|
t.Fatalf("unexpected node: %+v", node)
|
|
}
|
|
if store.lastAttachInput.NodeID != "node-1" || len(store.lastAttachInput.Roles) != 2 {
|
|
t.Fatalf("attach input not preserved: %+v", store.lastAttachInput)
|
|
}
|
|
}
|
|
|
|
func TestGetDockerInstallProfileBuildsRuntimeProfileFromTokenScope(t *testing.T) {
|
|
rawToken := "rap_join_profile"
|
|
tokenHash, err := hashJoinToken(rawToken)
|
|
if err != nil {
|
|
t.Fatalf("hash token: %v", err)
|
|
}
|
|
store := &fakeRepository{
|
|
validJoinToken: NodeJoinToken{
|
|
ID: "token-1",
|
|
ClusterID: "cluster-1",
|
|
Status: "active",
|
|
Scope: json.RawMessage(`{
|
|
"backend_url": "https://control.example.test/api/v1/",
|
|
"roles": ["core-mesh"],
|
|
"image": "registry.example.test/rap-node-agent:1",
|
|
"artifact_endpoints": ["https://cache-a.example.test/artifacts/"],
|
|
"docker_image_artifact_sha256": "abc123",
|
|
"mesh_connectivity_mode": "outbound_only",
|
|
"mesh_region": "customer-a",
|
|
"pull_image": true
|
|
}`),
|
|
},
|
|
}
|
|
service := NewService(store)
|
|
|
|
profile, err := service.GetDockerInstallProfile(context.Background(), DockerInstallProfileRequest{
|
|
ClusterID: "cluster-1",
|
|
InstallToken: rawToken,
|
|
NodeName: "Customer Node 1",
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("profile: %v", err)
|
|
}
|
|
if store.lastLookupTokenHash != tokenHash {
|
|
t.Fatalf("token hash lookup = %q, want %q", store.lastLookupTokenHash, tokenHash)
|
|
}
|
|
if profile.BackendURL != "https://control.example.test/api/v1" ||
|
|
profile.JoinToken != rawToken ||
|
|
profile.NodeName != "Customer Node 1" ||
|
|
profile.ContainerName != "rap-node-agent-customer-node-1" ||
|
|
profile.MeshConnectivityMode != "outbound_only" ||
|
|
profile.MeshRegion != "customer-a" ||
|
|
len(profile.ArtifactEndpoints) != 1 ||
|
|
profile.DockerImageArtifact == nil ||
|
|
profile.DockerImageArtifact.FileName != "registry.example.test-rap-node-agent-1.tar" ||
|
|
profile.DockerImageArtifact.URLs[0] != "https://cache-a.example.test/artifacts/registry.example.test-rap-node-agent-1.tar" ||
|
|
profile.DockerImageArtifact.SHA256 != "abc123" ||
|
|
profile.EnrollmentPollTimeoutSeconds != 0 ||
|
|
!profile.PullImage ||
|
|
!profile.MeshSyntheticRuntimeEnabled ||
|
|
profile.MeshProductionForwardingEnabled {
|
|
t.Fatalf("unexpected profile: %+v", profile)
|
|
}
|
|
}
|
|
|
|
func TestGetDockerInstallProfileDefaultsArtifactEndpointFromBackendURL(t *testing.T) {
|
|
rawToken := "rap_join_profile"
|
|
store := &fakeRepository{
|
|
validJoinToken: NodeJoinToken{
|
|
ID: "token-1",
|
|
ClusterID: "cluster-1",
|
|
Status: "active",
|
|
Scope: json.RawMessage(`{
|
|
"backend_url": "https://control.example.test/api/v1",
|
|
"image": "rap-node-agent:dev"
|
|
}`),
|
|
},
|
|
}
|
|
service := NewService(store)
|
|
|
|
profile, err := service.GetDockerInstallProfile(context.Background(), DockerInstallProfileRequest{
|
|
ClusterID: "cluster-1",
|
|
InstallToken: rawToken,
|
|
NodeName: "node-a",
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("profile: %v", err)
|
|
}
|
|
if len(profile.ArtifactEndpoints) != 1 || profile.ArtifactEndpoints[0] != "https://control.example.test/downloads" {
|
|
t.Fatalf("artifact endpoints = %#v", profile.ArtifactEndpoints)
|
|
}
|
|
if profile.DockerImageArtifact == nil || profile.DockerImageArtifact.URLs[0] != "https://control.example.test/downloads/rap-node-agent-dev.tar" {
|
|
t.Fatalf("unexpected artifact: %+v", profile.DockerImageArtifact)
|
|
}
|
|
}
|
|
|
|
func TestGetDockerInstallProfileDoesNotPinFloatingDevArtifactMetadata(t *testing.T) {
|
|
rawToken := "rap_join_profile"
|
|
store := &fakeRepository{
|
|
validJoinToken: NodeJoinToken{
|
|
ID: "token-1",
|
|
ClusterID: "cluster-1",
|
|
Status: "active",
|
|
Scope: json.RawMessage(`{
|
|
"backend_url": "https://control.example.test/api/v1",
|
|
"image": "rap-node-agent:dev-enrollment-bootstrap-smoke"
|
|
}`),
|
|
},
|
|
}
|
|
service := NewService(store)
|
|
|
|
profile, err := service.GetDockerInstallProfile(context.Background(), DockerInstallProfileRequest{
|
|
ClusterID: "cluster-1",
|
|
InstallToken: rawToken,
|
|
NodeName: "node-a",
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("profile: %v", err)
|
|
}
|
|
if profile.DockerImageArtifact == nil ||
|
|
profile.DockerImageArtifact.SHA256 != "" ||
|
|
profile.DockerImageArtifact.SizeBytes != 0 {
|
|
t.Fatalf("unexpected artifact metadata: %+v", profile.DockerImageArtifact)
|
|
}
|
|
}
|
|
|
|
func TestCreateJoinRequestRejectsExpiredOrRevokedToken(t *testing.T) {
|
|
store := &fakeRepository{validTokenErr: ErrInvalidJoinToken}
|
|
service := NewService(store)
|
|
|
|
_, err := service.CreateJoinRequest(context.Background(), CreateJoinRequestInput{
|
|
ClusterID: "cluster-1",
|
|
JoinToken: "rap_join_invalid",
|
|
NodeName: "node-a",
|
|
NodeFingerprint: "fingerprint-a",
|
|
PublicKey: "public-key",
|
|
})
|
|
if !errors.Is(err, ErrInvalidJoinToken) {
|
|
t.Fatalf("err = %v, want ErrInvalidJoinToken", err)
|
|
}
|
|
}
|
|
|
|
func TestRevokeJoinTokenRequiresPlatformAdmin(t *testing.T) {
|
|
store := &fakeRepository{platformRole: "user"}
|
|
service := NewService(store)
|
|
|
|
_, err := service.RevokeJoinToken(context.Background(), RevokeJoinTokenInput{
|
|
ActorUserID: "user-1",
|
|
ClusterID: "cluster-1",
|
|
TokenID: "token-1",
|
|
})
|
|
if !errors.Is(err, ErrAccessDenied) {
|
|
t.Fatalf("err = %v, want ErrAccessDenied", err)
|
|
}
|
|
}
|
|
|
|
func TestApproveJoinRequestReturnsBootstrapContract(t *testing.T) {
|
|
store := &fakeRepository{platformRole: PlatformRoleAdmin}
|
|
service := NewService(store)
|
|
|
|
approved, err := service.ApproveJoinRequest(context.Background(), ApproveJoinRequestInput{
|
|
ActorUserID: "admin-1",
|
|
ClusterID: "cluster-1",
|
|
JoinRequestID: "join-request-1",
|
|
NodeKey: "node-key-1",
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("approve join request: %v", err)
|
|
}
|
|
if approved.Bootstrap.ClusterID != "cluster-1" || approved.Bootstrap.IdentityStatus == "" {
|
|
t.Fatalf("unexpected bootstrap contract: %+v", approved.Bootstrap)
|
|
}
|
|
if approved.Bootstrap.ClusterAuthority == nil || approved.Bootstrap.AuthoritySignature == nil || len(approved.Bootstrap.AuthorityPayload) == 0 {
|
|
t.Fatalf("bootstrap missing authority contract: %+v", approved.Bootstrap)
|
|
}
|
|
if err := clusterauth.VerifyRaw(store.clusterAuthority.PublicKey, approved.Bootstrap.AuthorityPayload, *approved.Bootstrap.AuthoritySignature); err != nil {
|
|
t.Fatalf("verify approval authority signature: %v", err)
|
|
}
|
|
}
|
|
|
|
func TestGetJoinRequestBootstrapReturnsSignedApproval(t *testing.T) {
|
|
nodeID := "node-1"
|
|
store := &fakeRepository{
|
|
platformRole: PlatformRoleAdmin,
|
|
bootstrapJoinRequest: NodeJoinRequest{
|
|
ID: "join-request-1",
|
|
ClusterID: "cluster-1",
|
|
NodeFingerprint: "node-fp",
|
|
PublicKey: "node-public-key",
|
|
Status: JoinRequestStatusApproved,
|
|
ApprovedNodeID: &nodeID,
|
|
},
|
|
}
|
|
service := NewService(store)
|
|
|
|
result, err := service.GetJoinRequestBootstrap(context.Background(), GetJoinRequestBootstrapInput{
|
|
ClusterID: "cluster-1",
|
|
JoinRequestID: "join-request-1",
|
|
NodeFingerprint: "node-fp",
|
|
PublicKey: "node-public-key",
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("get join request bootstrap: %v", err)
|
|
}
|
|
if result.Bootstrap == nil || result.Bootstrap.NodeID != nodeID || result.Bootstrap.ClusterAuthority == nil {
|
|
t.Fatalf("unexpected bootstrap result: %+v", result)
|
|
}
|
|
if result.Bootstrap.AuthoritySignature == nil || len(result.Bootstrap.AuthorityPayload) == 0 {
|
|
t.Fatalf("bootstrap missing authority signature: %+v", result.Bootstrap)
|
|
}
|
|
if err := clusterauth.VerifyRaw(store.clusterAuthority.PublicKey, result.Bootstrap.AuthorityPayload, *result.Bootstrap.AuthoritySignature); err != nil {
|
|
t.Fatalf("verify bootstrap authority signature: %v", err)
|
|
}
|
|
}
|
|
|
|
func TestSetDesiredWorkloadRequiresPlatformAdmin(t *testing.T) {
|
|
store := &fakeRepository{platformRole: "user"}
|
|
service := NewService(store)
|
|
|
|
_, err := service.SetDesiredWorkload(context.Background(), SetDesiredWorkloadInput{
|
|
ActorUserID: "user-1",
|
|
ClusterID: "cluster-1",
|
|
NodeID: "node-1",
|
|
ServiceType: "rdp-worker",
|
|
})
|
|
if !errors.Is(err, ErrAccessDenied) {
|
|
t.Fatalf("err = %v, want ErrAccessDenied", err)
|
|
}
|
|
}
|
|
|
|
func TestListDesiredWorkloadsAllowsNodeScopedAgentReadWithoutActor(t *testing.T) {
|
|
store := &fakeRepository{
|
|
desiredWorkloads: []NodeWorkloadDesiredState{{
|
|
ClusterID: "cluster-1",
|
|
NodeID: "node-1",
|
|
ServiceType: "synthetic.echo",
|
|
DesiredState: "enabled",
|
|
RuntimeMode: "native",
|
|
Config: json.RawMessage(`{}`),
|
|
Environment: json.RawMessage(`{}`),
|
|
}},
|
|
}
|
|
service := NewService(store)
|
|
|
|
items, err := service.ListDesiredWorkloads(context.Background(), "", "cluster-1", "node-1")
|
|
if err != nil {
|
|
t.Fatalf("list desired workloads: %v", err)
|
|
}
|
|
if len(items) != 1 || items[0].ServiceType != "synthetic.echo" {
|
|
t.Fatalf("unexpected desired workloads: %+v", items)
|
|
}
|
|
}
|
|
|
|
func TestReportWorkloadStatusDefaultsToSafeStubState(t *testing.T) {
|
|
store := &fakeRepository{}
|
|
service := NewService(store)
|
|
|
|
status, err := service.ReportWorkloadStatus(context.Background(), ReportWorkloadStatusInput{
|
|
ClusterID: "cluster-1",
|
|
NodeID: "node-1",
|
|
ServiceType: "rdp-worker",
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("report workload status: %v", err)
|
|
}
|
|
if status.ReportedState != "unknown" || status.RuntimeMode != "container" {
|
|
t.Fatalf("unexpected status defaults: %+v", status)
|
|
}
|
|
}
|
|
|
|
func TestReportMeshLinkDoesNotRequirePlatformAdmin(t *testing.T) {
|
|
store := &fakeRepository{}
|
|
service := NewService(store)
|
|
|
|
link, err := service.ReportMeshLink(context.Background(), ReportMeshLinkInput{
|
|
ClusterID: "cluster-1",
|
|
SourceNodeID: "node-a",
|
|
TargetNodeID: "node-b",
|
|
LinkStatus: "reachable",
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("report mesh link: %v", err)
|
|
}
|
|
if link.LinkStatus != "reachable" {
|
|
t.Fatalf("LinkStatus = %q", link.LinkStatus)
|
|
}
|
|
}
|
|
|
|
func TestCreateRouteIntentRequiresPlatformAdmin(t *testing.T) {
|
|
store := &fakeRepository{platformRole: "user"}
|
|
service := NewService(store)
|
|
|
|
_, err := service.CreateRouteIntent(context.Background(), CreateRouteIntentInput{
|
|
ActorUserID: "user-1",
|
|
ClusterID: "cluster-1",
|
|
ServiceClass: "input",
|
|
})
|
|
if !errors.Is(err, ErrAccessDenied) {
|
|
t.Fatalf("err = %v, want ErrAccessDenied", err)
|
|
}
|
|
}
|
|
|
|
func TestGetVPNClientProfileEnsuresFabricVPNPacketRouteIntents(t *testing.T) {
|
|
repo := &fakeRepository{
|
|
vpnClientProfile: VPNClientProfile{
|
|
SchemaVersion: "rap.vpn_client_profile.v1",
|
|
Connections: []VPNClientConnection{{
|
|
ID: "vpn-1",
|
|
ClientConfig: json.RawMessage(`{
|
|
"vpn_fabric_route": {
|
|
"status": "planned",
|
|
"selected_entry_node_id": "entry-1",
|
|
"selected_exit_node_id": "exit-1"
|
|
},
|
|
"vpn_entry_endpoint_candidates": [{
|
|
"node_id": "entry-1",
|
|
"endpoint_id": "public-http",
|
|
"transport": "direct_http",
|
|
"address": "http://entry.example.test:19131",
|
|
"api_base_url": "http://entry.example.test:19131/api/v1",
|
|
"reachability": "public",
|
|
"priority": 0
|
|
}]
|
|
}`),
|
|
}},
|
|
},
|
|
}
|
|
service := NewService(repo)
|
|
service.now = func() time.Time { return time.Date(2026, 5, 3, 12, 0, 0, 0, time.UTC) }
|
|
|
|
profile, err := service.GetVPNClientProfile(context.Background(), "cluster-1", "org-1", "user-1", "entry-1")
|
|
if err != nil {
|
|
t.Fatalf("GetVPNClientProfile: %v", err)
|
|
}
|
|
if len(profile.Connections) != 1 {
|
|
t.Fatalf("profile connections = %d, want 1", len(profile.Connections))
|
|
}
|
|
var cfg map[string]any
|
|
if err := json.Unmarshal(profile.Connections[0].ClientConfig, &cfg); err != nil {
|
|
t.Fatalf("unmarshal client config: %v", err)
|
|
}
|
|
session, ok := cfg["vpn_dataplane_session"].(map[string]any)
|
|
if !ok {
|
|
t.Fatalf("missing vpn_dataplane_session in %#v", cfg)
|
|
}
|
|
if session["preferred_transport"] != "fabric_packet_quic_v1" || session["fallback_transport"] != "backend_http_packet_relay" {
|
|
t.Fatalf("unexpected dataplane session transports: %#v", session)
|
|
}
|
|
if session["entry_node_id"] != "entry-1" || session["exit_node_id"] != "exit-1" {
|
|
t.Fatalf("unexpected dataplane session route: %#v", session)
|
|
}
|
|
entryCandidates := session["entry_candidates"].([]any)
|
|
if len(entryCandidates) != 1 {
|
|
t.Fatalf("entry candidate count = %d, want 1", len(entryCandidates))
|
|
}
|
|
entryCandidate := entryCandidates[0].(map[string]any)
|
|
if entryCandidate["api_base_url"] != "http://entry.example.test:19131/api/v1" || entryCandidate["status"] != "selected_endpoint_public" {
|
|
t.Fatalf("unexpected entry candidate: %#v", entryCandidate)
|
|
}
|
|
transportCandidates := session["transport_candidates"].([]any)
|
|
var foundDirect bool
|
|
for _, rawCandidate := range transportCandidates {
|
|
candidate := rawCandidate.(map[string]any)
|
|
if candidate["type"] == "entry_direct_http_v1" {
|
|
foundDirect = true
|
|
if candidate["status"] != "available" || candidate["safe_client_switch"] != true {
|
|
t.Fatalf("unexpected direct entry transport candidate: %#v", candidate)
|
|
}
|
|
}
|
|
}
|
|
if !foundDirect {
|
|
t.Fatalf("missing entry_direct_http_v1 in %#v", transportCandidates)
|
|
}
|
|
auth := session["auth"].(map[string]any)
|
|
if auth["type"] != "control_plane_issued_bearer" || auth["node_validation"] != "entry_node_calls_control_plane_introspection" {
|
|
t.Fatalf("unexpected dataplane session auth: %#v", auth)
|
|
}
|
|
|
|
if got := len(repo.createdRouteIntents); got != 2 {
|
|
t.Fatalf("created route intents = %d, want 2", got)
|
|
}
|
|
for _, input := range repo.createdRouteIntents {
|
|
if input.ClusterID != "cluster-1" || input.ServiceClass != "vpn_packets" || input.Priority != 10 {
|
|
t.Fatalf("unexpected route intent input: %+v", input)
|
|
}
|
|
var policy syntheticRoutePolicy
|
|
if err := json.Unmarshal(input.Policy, &policy); err != nil {
|
|
t.Fatalf("unmarshal policy: %v", err)
|
|
}
|
|
if !policy.SyntheticEnabled || !containsString(policy.AllowedChannels, "vpn_packet") || !containsString(policy.AllowedChannels, "fabric_control") || len(policy.Hops) != 2 {
|
|
t.Fatalf("policy = %+v", policy)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestGetVPNClientProfileForwardsPreferredExit(t *testing.T) {
|
|
repo := &fakeRepository{
|
|
vpnClientProfile: VPNClientProfile{
|
|
SchemaVersion: "rap.vpn_client_profile.v1",
|
|
Connections: []VPNClientConnection{{
|
|
ID: "vpn-1",
|
|
ClientConfig: json.RawMessage(`{
|
|
"vpn_fabric_route": {
|
|
"status": "planned",
|
|
"selected_entry_node_id": "entry-1",
|
|
"selected_exit_node_id": "exit-1"
|
|
}
|
|
}`),
|
|
}},
|
|
},
|
|
}
|
|
service := NewService(repo)
|
|
|
|
if _, err := service.GetVPNClientProfile(context.Background(), "cluster-1", "org-1", "user-1", "entry-1", "exit-2"); err != nil {
|
|
t.Fatalf("GetVPNClientProfile: %v", err)
|
|
}
|
|
if repo.lastPreferredEntryNodeID != "entry-1" {
|
|
t.Fatalf("preferred entry = %q, want entry-1", repo.lastPreferredEntryNodeID)
|
|
}
|
|
if repo.lastPreferredExitNodeID != "exit-2" {
|
|
t.Fatalf("preferred exit = %q, want exit-2", repo.lastPreferredExitNodeID)
|
|
}
|
|
}
|
|
|
|
func TestVPNDirectHTTPEntryTransportWaitsForLocalGatewayShortcutWhenEntryIsExit(t *testing.T) {
|
|
candidate := vpnDirectHTTPEntryTransportCandidate(vpnClientFabricRoute{
|
|
SelectedEntryNodeID: "node-1",
|
|
SelectedExitNodeID: "node-1",
|
|
}, []map[string]any{{
|
|
"node_id": "node-1",
|
|
"api_base_url": "http://node.example.test:19131/api/v1",
|
|
"reachability": "public",
|
|
}})
|
|
if candidate == nil {
|
|
t.Fatal("candidate is nil")
|
|
}
|
|
if candidate["safe_client_switch"] != false || candidate["status"] != "available_local_gateway_shortcut_pending" {
|
|
t.Fatalf("unexpected local shortcut guard: %#v", candidate)
|
|
}
|
|
}
|
|
|
|
func TestVPNDirectHTTPEntryTransportAllowsLocalGatewayShortcutWhenReported(t *testing.T) {
|
|
candidate := vpnDirectHTTPEntryTransportCandidate(vpnClientFabricRoute{
|
|
SelectedEntryNodeID: "node-1",
|
|
SelectedExitNodeID: "node-1",
|
|
}, []map[string]any{{
|
|
"node_id": "node-1",
|
|
"api_base_url": "http://node.example.test:19131/api/v1",
|
|
"reachability": "public",
|
|
"local_gateway_shortcut": true,
|
|
}})
|
|
if candidate == nil {
|
|
t.Fatal("candidate is nil")
|
|
}
|
|
if candidate["safe_client_switch"] != true || candidate["status"] != "available_local_gateway_shortcut" {
|
|
t.Fatalf("unexpected local shortcut candidate: %#v", candidate)
|
|
}
|
|
}
|
|
|
|
func TestGetNodeSyntheticMeshConfigRequiresTestingFlag(t *testing.T) {
|
|
service := NewService(&fakeRepository{})
|
|
|
|
cfg, err := service.GetNodeSyntheticMeshConfig(context.Background(), GetNodeSyntheticMeshConfigInput{
|
|
ClusterID: "cluster-1",
|
|
NodeID: "node-a",
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("get synthetic config: %v", err)
|
|
}
|
|
if cfg.Enabled {
|
|
t.Fatal("config must be disabled when synthetic testing flag is off")
|
|
}
|
|
if len(cfg.Routes) != 0 || len(cfg.PeerEndpoints) != 0 {
|
|
t.Fatalf("disabled config must not leak topology: %+v", cfg)
|
|
}
|
|
}
|
|
|
|
func TestNodeUpdatePlanSelectsMatchingReleaseArtifact(t *testing.T) {
|
|
store := &fakeRepository{
|
|
platformRole: PlatformRoleAdmin,
|
|
releaseVersions: []ReleaseVersion{
|
|
{
|
|
ID: "release-1",
|
|
ClusterID: "cluster-1",
|
|
Product: "rap-node-agent",
|
|
Version: "0.1.0-c17z26",
|
|
Channel: "dev",
|
|
Status: "active",
|
|
Artifacts: []ReleaseArtifact{
|
|
{ID: "linux", ClusterID: "cluster-1", Product: "rap-node-agent", Version: "0.1.0-c17z26", OS: "linux", Arch: "amd64", InstallType: "service", Kind: "binary", URL: "https://cache/agent", SHA256: "linux-sha"},
|
|
{ID: "docker", ClusterID: "cluster-1", Product: "rap-node-agent", Version: "0.1.0-c17z26", OS: "linux", Arch: "amd64", InstallType: "docker", Kind: "docker_image_tar", URL: "https://cache/agent.tar", SHA256: "docker-sha"},
|
|
},
|
|
},
|
|
},
|
|
nodeUpdatePolicies: map[string]NodeUpdatePolicy{
|
|
"node-1|rap-node-agent": {
|
|
ClusterID: "cluster-1",
|
|
NodeID: "node-1",
|
|
Product: "rap-node-agent",
|
|
Channel: "dev",
|
|
Strategy: "manual",
|
|
Enabled: true,
|
|
RollbackAllowed: true,
|
|
HealthWindowSec: 180,
|
|
},
|
|
},
|
|
}
|
|
service := NewService(store)
|
|
|
|
plan, err := service.GetNodeUpdatePlan(context.Background(), GetNodeUpdatePlanInput{
|
|
ClusterID: "cluster-1",
|
|
NodeID: "node-1",
|
|
Product: "rap-node-agent",
|
|
CurrentVersion: "0.1.0-c17z25",
|
|
OS: "linux",
|
|
Arch: "amd64",
|
|
InstallType: "docker",
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("update plan: %v", err)
|
|
}
|
|
if plan.Action != "update" ||
|
|
plan.TargetVersion != "0.1.0-c17z26" ||
|
|
plan.Artifact == nil ||
|
|
plan.Artifact.ID != "docker" ||
|
|
plan.ProductionForwarding {
|
|
t.Fatalf("unexpected update plan: %+v", plan)
|
|
}
|
|
if plan.AuthoritySignature == nil || len(plan.AuthorityPayload) == 0 {
|
|
t.Fatalf("update plan must be signed: %+v", plan)
|
|
}
|
|
}
|
|
|
|
func TestNodeUpdatePlanAbsolutizesRelativeArtifactURLs(t *testing.T) {
|
|
store := &fakeRepository{
|
|
platformRole: PlatformRoleAdmin,
|
|
releaseVersions: []ReleaseVersion{
|
|
{
|
|
ID: "release-1",
|
|
ClusterID: "cluster-1",
|
|
Product: "rap-node-agent",
|
|
Version: "0.2.93",
|
|
Channel: "stable",
|
|
Status: "active",
|
|
Artifacts: []ReleaseArtifact{
|
|
{
|
|
ID: "docker",
|
|
ClusterID: "cluster-1",
|
|
Product: "rap-node-agent",
|
|
Version: "0.2.93",
|
|
OS: "linux",
|
|
Arch: "amd64",
|
|
InstallType: "docker",
|
|
Kind: "docker_image_tar",
|
|
URL: "/downloads/rap-node-agent-0.2.93-docker-amd64.tar",
|
|
SHA256: "docker-sha",
|
|
Metadata: json.RawMessage(`{"urls":["/downloads/mirror.tar","https://cdn.example.test/agent.tar"]}`),
|
|
},
|
|
},
|
|
},
|
|
},
|
|
nodeUpdatePolicies: map[string]NodeUpdatePolicy{
|
|
"node-1|rap-node-agent": {
|
|
ClusterID: "cluster-1",
|
|
NodeID: "node-1",
|
|
Product: "rap-node-agent",
|
|
Channel: "stable",
|
|
Strategy: "rolling",
|
|
Enabled: true,
|
|
RollbackAllowed: true,
|
|
HealthWindowSec: 180,
|
|
},
|
|
},
|
|
}
|
|
service := NewService(store)
|
|
|
|
plan, err := service.GetNodeUpdatePlan(context.Background(), GetNodeUpdatePlanInput{
|
|
ClusterID: "cluster-1",
|
|
NodeID: "node-1",
|
|
Product: "rap-node-agent",
|
|
CurrentVersion: "0.2.92",
|
|
OS: "linux",
|
|
Arch: "amd64",
|
|
InstallType: "docker",
|
|
ArtifactOrigin: "http://vpn.cin.su:19191/api/v1",
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("update plan: %v", err)
|
|
}
|
|
if plan.Artifact == nil {
|
|
t.Fatal("expected artifact")
|
|
}
|
|
if plan.Artifact.URL != "http://vpn.cin.su:19191/downloads/rap-node-agent-0.2.93-docker-amd64.tar" {
|
|
t.Fatalf("artifact URL was not absolutized: %q", plan.Artifact.URL)
|
|
}
|
|
wantMirror := "http://vpn.cin.su:19191/downloads/mirror.tar"
|
|
if len(plan.Artifact.URLs) < 2 || plan.Artifact.URLs[1] != wantMirror || plan.Artifact.URLs[2] != "https://cdn.example.test/agent.tar" {
|
|
t.Fatalf("artifact URLs were not preserved/absolutized: %#v", plan.Artifact.URLs)
|
|
}
|
|
}
|
|
|
|
func TestHostAgentUpdatePlanRejectsLinuxArtifactForObservedWindowsNode(t *testing.T) {
|
|
store := &fakeRepository{
|
|
platformRole: PlatformRoleAdmin,
|
|
releaseVersions: []ReleaseVersion{
|
|
{
|
|
ID: "release-host",
|
|
ClusterID: "cluster-1",
|
|
Product: "rap-host-agent",
|
|
Version: "0.2.95",
|
|
Channel: "stable",
|
|
Status: "active",
|
|
Artifacts: []ReleaseArtifact{
|
|
{ID: "linux", ClusterID: "cluster-1", Product: "rap-host-agent", Version: "0.2.95", OS: "linux", Arch: "amd64", InstallType: "linux_binary", Kind: "binary", URL: "/downloads/rap-host-agent-linux", SHA256: "linux-sha"},
|
|
},
|
|
},
|
|
},
|
|
nodeUpdatePolicies: map[string]NodeUpdatePolicy{
|
|
"node-1|rap-host-agent": {
|
|
ClusterID: "cluster-1",
|
|
NodeID: "node-1",
|
|
Product: "rap-host-agent",
|
|
Channel: "stable",
|
|
Strategy: "rolling",
|
|
Enabled: true,
|
|
RollbackAllowed: true,
|
|
HealthWindowSec: 180,
|
|
},
|
|
},
|
|
updateStatuses: []NodeUpdateStatus{
|
|
{
|
|
ClusterID: "cluster-1",
|
|
NodeID: "node-1",
|
|
Product: "rap-node-agent",
|
|
Phase: "plan",
|
|
Status: "noop",
|
|
Payload: json.RawMessage(`{"binary_path":"C:\\Program Files\\RAP\\node\\rap-node-agent.exe","task":"RAP Node Agent node"}`),
|
|
},
|
|
},
|
|
}
|
|
service := NewService(store)
|
|
|
|
plan, err := service.GetNodeUpdatePlan(context.Background(), GetNodeUpdatePlanInput{
|
|
ClusterID: "cluster-1",
|
|
NodeID: "node-1",
|
|
Product: "rap-host-agent",
|
|
CurrentVersion: "0.2.92",
|
|
OS: "linux",
|
|
Arch: "amd64",
|
|
InstallType: "linux_binary",
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("update plan: %v", err)
|
|
}
|
|
if plan.Action != "none" || plan.Reason != "host_agent_artifact_platform_mismatch" || plan.Artifact != nil {
|
|
t.Fatalf("unexpected mismatch plan: %+v", plan)
|
|
}
|
|
}
|
|
|
|
func TestHostAgentUpdatePlanAllowsWindowsArtifactForObservedWindowsNode(t *testing.T) {
|
|
store := &fakeRepository{
|
|
platformRole: PlatformRoleAdmin,
|
|
releaseVersions: []ReleaseVersion{
|
|
{
|
|
ID: "release-host",
|
|
ClusterID: "cluster-1",
|
|
Product: "rap-host-agent",
|
|
Version: "0.2.95",
|
|
Channel: "stable",
|
|
Status: "active",
|
|
Artifacts: []ReleaseArtifact{
|
|
{ID: "windows", ClusterID: "cluster-1", Product: "rap-host-agent", Version: "0.2.95", OS: "windows", Arch: "amd64", InstallType: "windows_binary", Kind: "binary", URL: "/downloads/rap-host-agent.exe", SHA256: "windows-sha"},
|
|
},
|
|
},
|
|
},
|
|
nodeUpdatePolicies: map[string]NodeUpdatePolicy{
|
|
"node-1|rap-host-agent": {
|
|
ClusterID: "cluster-1",
|
|
NodeID: "node-1",
|
|
Product: "rap-host-agent",
|
|
Channel: "stable",
|
|
Strategy: "rolling",
|
|
Enabled: true,
|
|
RollbackAllowed: true,
|
|
HealthWindowSec: 180,
|
|
},
|
|
},
|
|
updateStatuses: []NodeUpdateStatus{
|
|
{
|
|
ClusterID: "cluster-1",
|
|
NodeID: "node-1",
|
|
Product: "rap-node-agent",
|
|
Phase: "plan",
|
|
Status: "noop",
|
|
Payload: json.RawMessage(`{"binary_path":"C:\\Program Files\\RAP\\node\\rap-node-agent.exe"}`),
|
|
},
|
|
},
|
|
}
|
|
service := NewService(store)
|
|
|
|
plan, err := service.GetNodeUpdatePlan(context.Background(), GetNodeUpdatePlanInput{
|
|
ClusterID: "cluster-1",
|
|
NodeID: "node-1",
|
|
Product: "rap-host-agent",
|
|
CurrentVersion: "0.2.92",
|
|
OS: "windows",
|
|
Arch: "amd64",
|
|
InstallType: "windows_binary",
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("update plan: %v", err)
|
|
}
|
|
if plan.Action != "update" || plan.Artifact == nil || plan.Artifact.ID != "windows" {
|
|
t.Fatalf("unexpected windows plan: %+v", plan)
|
|
}
|
|
}
|
|
|
|
func TestNodeUpdatePlanNoopsWhenPolicyMissing(t *testing.T) {
|
|
service := NewService(&fakeRepository{})
|
|
|
|
plan, err := service.GetNodeUpdatePlan(context.Background(), GetNodeUpdatePlanInput{
|
|
ClusterID: "cluster-1",
|
|
NodeID: "node-1",
|
|
Product: "rap-node-agent",
|
|
CurrentVersion: "0.1.0-c17z25",
|
|
OS: "linux",
|
|
Arch: "amd64",
|
|
InstallType: "docker",
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("update plan: %v", err)
|
|
}
|
|
if plan.Action != "none" || plan.Reason != "no_update_policy" || plan.ProductionForwarding {
|
|
t.Fatalf("unexpected missing-policy plan: %+v", plan)
|
|
}
|
|
}
|
|
|
|
func TestGetNodeSyntheticMeshConfigIncludesDesiredMeshListener(t *testing.T) {
|
|
now := time.Date(2026, 4, 30, 6, 0, 0, 0, time.UTC)
|
|
version := "listener-v1"
|
|
updatedBy := "admin-1"
|
|
service := NewService(&fakeRepository{
|
|
desiredWorkloads: []NodeWorkloadDesiredState{
|
|
{
|
|
ClusterID: "cluster-1",
|
|
NodeID: "node-a",
|
|
ServiceType: "mesh-listener",
|
|
DesiredState: "enabled",
|
|
Version: &version,
|
|
Config: json.RawMessage(`{"listen_addr":":19140","listen_port_mode":"manual","auto_port_start":19140,"auto_port_end":19149,"connectivity_mode":"private_lan","nat_type":"none","region":"site-a"}`),
|
|
UpdatedByUserID: &updatedBy,
|
|
UpdatedAt: now,
|
|
},
|
|
},
|
|
})
|
|
service.now = func() time.Time { return now }
|
|
|
|
cfg, err := service.GetNodeSyntheticMeshConfig(context.Background(), GetNodeSyntheticMeshConfigInput{
|
|
ClusterID: "cluster-1",
|
|
NodeID: "node-a",
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("get synthetic config: %v", err)
|
|
}
|
|
if cfg.MeshListener == nil {
|
|
t.Fatal("expected mesh listener desired config")
|
|
}
|
|
if cfg.MeshListener.ListenAddr != ":19140" ||
|
|
cfg.MeshListener.ListenPortMode != "manual" ||
|
|
cfg.MeshListener.ConnectivityMode != "private_lan" ||
|
|
cfg.MeshListener.ConfigVersion != "listener-v1" ||
|
|
!cfg.MeshListener.ControlPlaneOnly ||
|
|
cfg.MeshListener.ProductionForwarding {
|
|
t.Fatalf("unexpected listener config: %+v", cfg.MeshListener)
|
|
}
|
|
if cfg.AuthoritySignature == nil || len(cfg.AuthorityPayload) == 0 {
|
|
t.Fatal("listener-bearing synthetic config must remain signed")
|
|
}
|
|
}
|
|
|
|
func TestGetNodeSyntheticMeshConfigIsNodeScoped(t *testing.T) {
|
|
now := time.Date(2026, 4, 27, 12, 0, 0, 0, time.UTC)
|
|
service := NewService(&fakeRepository{
|
|
testingFlags: EffectiveNodeTestingFlags{
|
|
Enabled: true,
|
|
SyntheticLinksEnabled: true,
|
|
},
|
|
routeIntents: []MeshRouteIntent{
|
|
{
|
|
ID: "route-a-b",
|
|
ClusterID: "cluster-1",
|
|
SourceSelector: json.RawMessage(`{"node_id":"node-a"}`),
|
|
DestinationSelector: json.RawMessage(`{"node_id":"node-b"}`),
|
|
ServiceClass: "synthetic",
|
|
Status: "active",
|
|
Policy: json.RawMessage(`{
|
|
"synthetic_enabled": true,
|
|
"hops": ["node-a", "node-r", "node-b"],
|
|
"allowed_channels": ["fabric_control", "route_control"],
|
|
"peer_endpoints": {
|
|
"node-r": "http://node-r:19000",
|
|
"node-b": "http://node-b:19000",
|
|
"node-y": "http://node-y:19000"
|
|
},
|
|
"peer_endpoint_candidates": {
|
|
"node-r": [
|
|
{
|
|
"endpoint_id": "node-r-public",
|
|
"node_id": "node-r",
|
|
"transport": "direct_tcp_tls",
|
|
"address": "203.0.113.10:443",
|
|
"address_family": "ipv4",
|
|
"reachability": "public",
|
|
"nat_type": "none",
|
|
"connectivity_mode": "direct",
|
|
"region": "eu",
|
|
"priority": 10,
|
|
"policy_tags": ["fast-path"],
|
|
"metadata": {"source":"test"}
|
|
}
|
|
],
|
|
"node-b": [
|
|
{
|
|
"endpoint_id": "node-b-outbound",
|
|
"node_id": "node-b",
|
|
"transport": "outbound_reverse",
|
|
"address": "node-b.reverse.local",
|
|
"reachability": "outbound_only",
|
|
"nat_type": "symmetric",
|
|
"connectivity_mode": "outbound_only",
|
|
"priority": 20
|
|
}
|
|
]
|
|
},
|
|
"recovery_seeds": [
|
|
{
|
|
"node_id": "node-r",
|
|
"endpoint": "https://node-r.example.test:443",
|
|
"transport": "direct_tcp_tls",
|
|
"connectivity_mode": "direct",
|
|
"region": "eu",
|
|
"priority": 10,
|
|
"metadata": {"role":"stable-recovery"}
|
|
},
|
|
{
|
|
"node_id": "node-seed",
|
|
"endpoint": "wss://seed.example.test/mesh",
|
|
"transport": "wss",
|
|
"connectivity_mode": "direct",
|
|
"priority": 20
|
|
}
|
|
],
|
|
"route_version": "route-v1",
|
|
"policy_version": "policy-v1",
|
|
"peer_directory_version": "peers-v1"
|
|
}`),
|
|
UpdatedAt: now,
|
|
},
|
|
{
|
|
ID: "route-x-y",
|
|
ClusterID: "cluster-1",
|
|
SourceSelector: json.RawMessage(`{"node_id":"node-x"}`),
|
|
DestinationSelector: json.RawMessage(`{"node_id":"node-y"}`),
|
|
ServiceClass: "synthetic",
|
|
Status: "active",
|
|
Policy: json.RawMessage(`{
|
|
"synthetic_enabled": true,
|
|
"hops": ["node-x", "node-y"],
|
|
"peer_endpoints": {"node-y": "http://node-y:19000"}
|
|
}`),
|
|
UpdatedAt: now,
|
|
},
|
|
},
|
|
})
|
|
service.now = func() time.Time { return now }
|
|
|
|
cfg, err := service.GetNodeSyntheticMeshConfig(context.Background(), GetNodeSyntheticMeshConfigInput{
|
|
ClusterID: "cluster-1",
|
|
NodeID: "node-a",
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("get synthetic config: %v", err)
|
|
}
|
|
if !cfg.Enabled {
|
|
t.Fatal("config should be enabled")
|
|
}
|
|
if len(cfg.Routes) != 1 || cfg.Routes[0].RouteID != "route-a-b" {
|
|
t.Fatalf("routes = %+v", cfg.Routes)
|
|
}
|
|
if cfg.PeerEndpoints["node-r"] == "" || cfg.PeerEndpoints["node-b"] == "" {
|
|
t.Fatalf("peer endpoints missing: %+v", cfg.PeerEndpoints)
|
|
}
|
|
if _, leaked := cfg.PeerEndpoints["node-y"]; leaked {
|
|
t.Fatalf("unrelated topology leaked: %+v", cfg.PeerEndpoints)
|
|
}
|
|
nodeRCandidates := cfg.PeerEndpointCandidates["node-r"]
|
|
if len(nodeRCandidates) != 1 {
|
|
t.Fatalf("node-r candidates = %+v", cfg.PeerEndpointCandidates)
|
|
}
|
|
if got := nodeRCandidates[0]; got.EndpointID != "node-r-public" ||
|
|
got.Transport != "direct_tcp_tls" ||
|
|
got.Reachability != "public" ||
|
|
got.NATType != "none" ||
|
|
got.ConnectivityMode != "direct" ||
|
|
got.Priority != 10 {
|
|
t.Fatalf("unexpected node-r candidate: %+v", got)
|
|
}
|
|
if _, leaked := cfg.PeerEndpointCandidates["node-y"]; leaked {
|
|
t.Fatalf("unrelated candidate topology leaked: %+v", cfg.PeerEndpointCandidates)
|
|
}
|
|
if len(cfg.RecoverySeeds) != 2 || cfg.RecoverySeeds[0].NodeID != "node-r" || cfg.RecoverySeeds[1].NodeID != "node-seed" {
|
|
t.Fatalf("unexpected recovery seeds: %+v", cfg.RecoverySeeds)
|
|
}
|
|
nodeRDirectory, ok := findPeerDirectoryEntry(cfg.PeerDirectory, "node-r")
|
|
if !ok || nodeRDirectory.CandidateCount != 1 || !nodeRDirectory.RecoverySeed {
|
|
t.Fatalf("node-r peer directory missing recovery/candidate metadata: %+v", cfg.PeerDirectory)
|
|
}
|
|
if _, ok := findPeerDirectoryEntry(cfg.PeerDirectory, "node-a"); ok {
|
|
t.Fatalf("local node leaked into peer directory: %+v", cfg.PeerDirectory)
|
|
}
|
|
if _, ok := findPeerDirectoryEntry(cfg.PeerDirectory, "node-y"); ok {
|
|
t.Fatalf("unrelated node leaked into peer directory: %+v", cfg.PeerDirectory)
|
|
}
|
|
if cfg.ProductionForwarding {
|
|
t.Fatal("production forwarding must remain false")
|
|
}
|
|
}
|
|
|
|
func TestGetNodeSyntheticMeshConfigSkipsExpiredRouteIntent(t *testing.T) {
|
|
now := time.Date(2026, 5, 7, 18, 20, 0, 0, time.UTC)
|
|
service := NewService(&fakeRepository{
|
|
testingFlags: EffectiveNodeTestingFlags{
|
|
Enabled: true,
|
|
SyntheticLinksEnabled: true,
|
|
},
|
|
routeIntents: []MeshRouteIntent{
|
|
{
|
|
ID: "expired-route",
|
|
ClusterID: "cluster-1",
|
|
SourceSelector: json.RawMessage(`{"node_id":"node-a"}`),
|
|
DestinationSelector: json.RawMessage(`{"node_id":"node-b"}`),
|
|
ServiceClass: "vpn_packets",
|
|
Status: "active",
|
|
Policy: json.RawMessage(`{
|
|
"synthetic_enabled": true,
|
|
"hops": ["node-a", "node-b"],
|
|
"allowed_channels": ["vpn_packet"],
|
|
"expires_at": "2026-05-07T18:19:00Z"
|
|
}`),
|
|
UpdatedAt: now.Add(-time.Minute),
|
|
},
|
|
{
|
|
ID: "fresh-route",
|
|
ClusterID: "cluster-1",
|
|
SourceSelector: json.RawMessage(`{"node_id":"node-a"}`),
|
|
DestinationSelector: json.RawMessage(`{"node_id":"node-b"}`),
|
|
ServiceClass: "vpn_packets",
|
|
Status: "active",
|
|
Policy: json.RawMessage(`{
|
|
"synthetic_enabled": true,
|
|
"hops": ["node-a", "node-b"],
|
|
"allowed_channels": ["vpn_packet"],
|
|
"expires_at": "2026-05-07T18:25:00Z"
|
|
}`),
|
|
UpdatedAt: now,
|
|
},
|
|
},
|
|
})
|
|
service.now = func() time.Time { return now }
|
|
|
|
cfg, err := service.GetNodeSyntheticMeshConfig(context.Background(), GetNodeSyntheticMeshConfigInput{
|
|
ClusterID: "cluster-1",
|
|
NodeID: "node-a",
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("get synthetic config: %v", err)
|
|
}
|
|
if containsRouteID(cfg.Routes, "expired-route") {
|
|
t.Fatalf("expired route leaked into synthetic config: %+v", cfg.Routes)
|
|
}
|
|
if !containsRouteID(cfg.Routes, "fresh-route") {
|
|
t.Fatalf("fresh route missing from synthetic config: %+v", cfg.Routes)
|
|
}
|
|
}
|
|
|
|
func TestRouteIntentLifecycleActionsMarkExpiredAndDisabled(t *testing.T) {
|
|
now := time.Date(2026, 5, 7, 18, 30, 0, 0, time.UTC)
|
|
repo := &fakeRepository{
|
|
platformRole: PlatformRoleAdmin,
|
|
authorityState: ClusterAuthorityState{
|
|
ClusterID: "cluster-1",
|
|
AuthorityState: "authoritative",
|
|
MutationMode: "normal",
|
|
},
|
|
routeIntents: []MeshRouteIntent{
|
|
{
|
|
ID: "route-a",
|
|
ClusterID: "cluster-1",
|
|
SourceSelector: json.RawMessage(`{"node_id":"node-a"}`),
|
|
DestinationSelector: json.RawMessage(`{"node_id":"node-b"}`),
|
|
ServiceClass: "vpn_packets",
|
|
Status: "active",
|
|
Policy: json.RawMessage(`{"synthetic_enabled":true}`),
|
|
UpdatedAt: now,
|
|
},
|
|
{
|
|
ID: "route-b",
|
|
ClusterID: "cluster-1",
|
|
SourceSelector: json.RawMessage(`{"node_id":"node-a"}`),
|
|
DestinationSelector: json.RawMessage(`{"node_id":"node-c"}`),
|
|
ServiceClass: "vpn_packets",
|
|
Status: "active",
|
|
Policy: json.RawMessage(`{"synthetic_enabled":true}`),
|
|
UpdatedAt: now,
|
|
},
|
|
},
|
|
}
|
|
service := NewService(repo)
|
|
service.now = func() time.Time { return now }
|
|
|
|
expired, err := service.ExpireRouteIntent(context.Background(), RouteIntentLifecycleInput{
|
|
ActorUserID: "admin",
|
|
ClusterID: "cluster-1",
|
|
RouteIntentID: "route-a",
|
|
Reason: "test cleanup",
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("expire route intent: %v", err)
|
|
}
|
|
if expired.LifecycleStatus != "expired" || !expired.IsExpired || expired.PolicyExpiresAt == nil {
|
|
t.Fatalf("expired lifecycle = %+v", expired)
|
|
}
|
|
|
|
disabled, err := service.DisableRouteIntent(context.Background(), RouteIntentLifecycleInput{
|
|
ActorUserID: "admin",
|
|
ClusterID: "cluster-1",
|
|
RouteIntentID: "route-b",
|
|
Reason: "test cleanup",
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("disable route intent: %v", err)
|
|
}
|
|
if disabled.Status != "disabled" || disabled.LifecycleStatus != "disabled" {
|
|
t.Fatalf("disabled lifecycle = %+v", disabled)
|
|
}
|
|
|
|
items, err := service.ListRouteIntents(context.Background(), "admin", "cluster-1")
|
|
if err != nil {
|
|
t.Fatalf("list route intents: %v", err)
|
|
}
|
|
if len(items) != 2 || items[0].LifecycleStatus == "" || items[1].LifecycleStatus == "" {
|
|
t.Fatalf("list lifecycle enrichment missing: %+v", items)
|
|
}
|
|
}
|
|
|
|
func TestGetNodeSyntheticMeshConfigUsesReportedMeshEndpoint(t *testing.T) {
|
|
now := time.Date(2026, 4, 28, 12, 0, 0, 0, time.UTC)
|
|
service := NewService(&fakeRepository{
|
|
testingFlags: EffectiveNodeTestingFlags{
|
|
Enabled: true,
|
|
SyntheticLinksEnabled: true,
|
|
},
|
|
routeIntents: []MeshRouteIntent{
|
|
{
|
|
ID: "route-a-b",
|
|
ClusterID: "cluster-1",
|
|
SourceSelector: json.RawMessage(`{"node_id":"node-a"}`),
|
|
DestinationSelector: json.RawMessage(`{"node_id":"node-b"}`),
|
|
ServiceClass: "synthetic",
|
|
Status: "active",
|
|
Policy: json.RawMessage(`{
|
|
"synthetic_enabled": true,
|
|
"hops": ["node-a", "node-b"]
|
|
}`),
|
|
UpdatedAt: now,
|
|
},
|
|
},
|
|
heartbeats: map[string][]NodeHeartbeat{
|
|
"node-b": {
|
|
{
|
|
ClusterID: "cluster-1",
|
|
NodeID: "node-b",
|
|
Metadata: json.RawMessage(`{
|
|
"mesh_endpoint_report": {
|
|
"schema_version": "c17z6.mesh_endpoint_report.v1",
|
|
"cluster_id": "cluster-1",
|
|
"node_id": "node-b",
|
|
"peer_endpoint": "https://node-b.dynamic.example.test:443",
|
|
"transport": "direct_tcp_tls",
|
|
"connectivity_mode": "direct",
|
|
"nat_type": "none",
|
|
"endpoint_candidates": [
|
|
{
|
|
"endpoint_id": "node-b-dynamic",
|
|
"node_id": "node-b",
|
|
"transport": "direct_tcp_tls",
|
|
"address": "https://node-b.dynamic.example.test:443",
|
|
"reachability": "public",
|
|
"connectivity_mode": "direct",
|
|
"nat_type": "none",
|
|
"priority": 1,
|
|
"metadata": {"source":"heartbeat"}
|
|
}
|
|
]
|
|
}
|
|
}`),
|
|
ObservedAt: now,
|
|
},
|
|
},
|
|
},
|
|
})
|
|
service.now = func() time.Time { return now }
|
|
|
|
cfg, err := service.GetNodeSyntheticMeshConfig(context.Background(), GetNodeSyntheticMeshConfigInput{
|
|
ClusterID: "cluster-1",
|
|
NodeID: "node-a",
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("get synthetic config: %v", err)
|
|
}
|
|
if cfg.PeerEndpoints["node-b"] != "https://node-b.dynamic.example.test:443" {
|
|
t.Fatalf("reported endpoint not projected: %+v", cfg.PeerEndpoints)
|
|
}
|
|
if got := cfg.PeerEndpointCandidates["node-b"]; len(got) != 1 || got[0].EndpointID != "node-b-dynamic" {
|
|
t.Fatalf("reported candidates not projected: %+v", cfg.PeerEndpointCandidates)
|
|
}
|
|
entry, ok := findPeerDirectoryEntry(cfg.PeerDirectory, "node-b")
|
|
if !ok || entry.EndpointCount != 1 || entry.CandidateCount != 1 {
|
|
t.Fatalf("peer directory did not include reported endpoint/candidate: %+v", cfg.PeerDirectory)
|
|
}
|
|
}
|
|
|
|
func TestGetNodeSyntheticMeshConfigUsesDesiredMeshListenerAdvertiseEndpointForPeer(t *testing.T) {
|
|
now := time.Date(2026, 5, 1, 9, 0, 0, 0, time.UTC)
|
|
version := "home-1-external-19199"
|
|
service := NewService(&fakeRepository{
|
|
testingFlags: EffectiveNodeTestingFlags{
|
|
Enabled: true,
|
|
SyntheticLinksEnabled: true,
|
|
},
|
|
routeIntents: []MeshRouteIntent{
|
|
{
|
|
ID: "route-a-home",
|
|
ClusterID: "cluster-1",
|
|
SourceSelector: json.RawMessage(`{"node_id":"node-a"}`),
|
|
DestinationSelector: json.RawMessage(`{"node_id":"home-1"}`),
|
|
ServiceClass: "synthetic",
|
|
Status: "active",
|
|
Policy: json.RawMessage(`{"synthetic_enabled":true,"hops":["node-a","home-1"]}`),
|
|
UpdatedAt: now,
|
|
},
|
|
},
|
|
desiredWorkloads: []NodeWorkloadDesiredState{
|
|
{
|
|
ClusterID: "cluster-1",
|
|
NodeID: "home-1",
|
|
ServiceType: "mesh-listener",
|
|
DesiredState: "enabled",
|
|
Version: &version,
|
|
Config: json.RawMessage(`{
|
|
"listen_addr":"0.0.0.0:19131",
|
|
"listen_port_mode":"manual",
|
|
"advertise_endpoint":"http://94.141.118.222:19199",
|
|
"advertise_transport":"direct_http",
|
|
"connectivity_mode":"direct",
|
|
"nat_type":"port_restricted",
|
|
"region":"home"
|
|
}`),
|
|
UpdatedAt: now,
|
|
},
|
|
},
|
|
heartbeats: map[string][]NodeHeartbeat{
|
|
"home-1": {
|
|
{
|
|
ClusterID: "cluster-1",
|
|
NodeID: "home-1",
|
|
Metadata: json.RawMessage(`{
|
|
"mesh_endpoint_report": {
|
|
"cluster_id": "cluster-1",
|
|
"node_id": "home-1",
|
|
"peer_endpoint": "http://192.168.200.85:19131",
|
|
"transport": "direct_http",
|
|
"connectivity_mode": "private_lan",
|
|
"nat_type": "none",
|
|
"endpoint_candidates": [
|
|
{
|
|
"endpoint_id": "home-1-private",
|
|
"node_id": "home-1",
|
|
"transport": "direct_http",
|
|
"address": "http://192.168.200.85:19131",
|
|
"reachability": "private",
|
|
"connectivity_mode": "private_lan",
|
|
"nat_type": "none",
|
|
"priority": 35
|
|
}
|
|
]
|
|
}
|
|
}`),
|
|
ObservedAt: now,
|
|
},
|
|
},
|
|
},
|
|
})
|
|
service.now = func() time.Time { return now }
|
|
|
|
cfg, err := service.GetNodeSyntheticMeshConfig(context.Background(), GetNodeSyntheticMeshConfigInput{
|
|
ClusterID: "cluster-1",
|
|
NodeID: "node-a",
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("get synthetic config: %v", err)
|
|
}
|
|
if cfg.PeerEndpoints["home-1"] != "http://94.141.118.222:19199" {
|
|
t.Fatalf("desired advertise endpoint should win over private heartbeat endpoint: %+v", cfg.PeerEndpoints)
|
|
}
|
|
got := cfg.PeerEndpointCandidates["home-1"]
|
|
if len(got) != 2 {
|
|
t.Fatalf("expected desired and reported candidates: %+v", got)
|
|
}
|
|
if got[0].EndpointID != "home-1-desired-mesh-listener" ||
|
|
got[0].Address != "http://94.141.118.222:19199" ||
|
|
got[0].Reachability != "public" ||
|
|
got[0].ConnectivityMode != "direct" ||
|
|
got[0].NATType != "port_restricted" ||
|
|
got[0].Priority != 0 {
|
|
t.Fatalf("unexpected desired candidate: %+v", got[0])
|
|
}
|
|
}
|
|
|
|
func TestGetNodeSyntheticMeshConfigKeepsOperatorPublicBootstrapPeerBeyondWarmPeerTarget(t *testing.T) {
|
|
now := time.Date(2026, 5, 1, 9, 30, 0, 0, time.UTC)
|
|
version := "home-1-external-19199"
|
|
privateHeartbeat := func(nodeID string, port string) []NodeHeartbeat {
|
|
return []NodeHeartbeat{{
|
|
ClusterID: "cluster-1",
|
|
NodeID: nodeID,
|
|
ObservedAt: now,
|
|
Metadata: json.RawMessage(`{
|
|
"mesh_endpoint_report": {
|
|
"cluster_id": "cluster-1",
|
|
"node_id": "` + nodeID + `",
|
|
"peer_endpoint": "http://192.168.200.61:` + port + `",
|
|
"transport": "direct_http",
|
|
"connectivity_mode": "private_lan",
|
|
"nat_type": "none",
|
|
"endpoint_candidates": [{
|
|
"endpoint_id": "` + nodeID + `-private",
|
|
"node_id": "` + nodeID + `",
|
|
"transport": "direct_http",
|
|
"address": "http://192.168.200.61:` + port + `",
|
|
"reachability": "private",
|
|
"connectivity_mode": "private_lan",
|
|
"nat_type": "none",
|
|
"priority": 35
|
|
}]
|
|
}
|
|
}`),
|
|
}}
|
|
}
|
|
service := NewService(&fakeRepository{
|
|
testingFlags: EffectiveNodeTestingFlags{
|
|
Enabled: true,
|
|
SyntheticLinksEnabled: true,
|
|
},
|
|
clusterNodes: []ClusterNode{
|
|
{ID: "remote", RegistrationStatus: NodeRegistrationActive, HealthStatus: "healthy", MembershipStatus: "active", CreatedAt: now.Add(-5 * time.Hour), LastSeenAt: ptrTime(now)},
|
|
{ID: "test-1", RegistrationStatus: NodeRegistrationActive, HealthStatus: "healthy", MembershipStatus: "active", CreatedAt: now.Add(-4 * time.Hour), LastSeenAt: ptrTime(now.Add(-time.Second))},
|
|
{ID: "test-2", RegistrationStatus: NodeRegistrationActive, HealthStatus: "healthy", MembershipStatus: "active", CreatedAt: now.Add(-3 * time.Hour), LastSeenAt: ptrTime(now.Add(-2 * time.Second))},
|
|
{ID: "test-3", RegistrationStatus: NodeRegistrationActive, HealthStatus: "healthy", MembershipStatus: "active", CreatedAt: now.Add(-2 * time.Hour), LastSeenAt: ptrTime(now.Add(-3 * time.Second))},
|
|
{ID: "home-1", RegistrationStatus: NodeRegistrationActive, HealthStatus: "healthy", MembershipStatus: "active", CreatedAt: now.Add(-time.Hour), LastSeenAt: ptrTime(now.Add(-4 * time.Second))},
|
|
},
|
|
nodeRoles: map[string][]NodeRoleAssignment{
|
|
"remote": {{NodeID: "remote", Role: "core-mesh", Status: "active"}},
|
|
},
|
|
heartbeats: map[string][]NodeHeartbeat{
|
|
"remote": {{
|
|
ClusterID: "cluster-1",
|
|
NodeID: "remote",
|
|
ObservedAt: now,
|
|
Metadata: json.RawMessage(`{
|
|
"mesh_endpoint_report": {
|
|
"cluster_id": "cluster-1",
|
|
"node_id": "remote",
|
|
"connectivity_mode": "outbound_only",
|
|
"region": "office"
|
|
},
|
|
"mesh_listener_report": {
|
|
"inbound_reachability": "outbound_only",
|
|
"one_way_connectivity": true
|
|
},
|
|
"mesh_outbound_session_report": {
|
|
"status": "ready",
|
|
"control_plane_url": "https://control.example.test/api/v1"
|
|
}
|
|
}`),
|
|
}},
|
|
"test-1": privateHeartbeat("test-1", "19131"),
|
|
"test-2": privateHeartbeat("test-2", "19132"),
|
|
"test-3": privateHeartbeat("test-3", "19133"),
|
|
"home-1": privateHeartbeat("home-1", "19131"),
|
|
},
|
|
desiredWorkloads: []NodeWorkloadDesiredState{{
|
|
ClusterID: "cluster-1",
|
|
NodeID: "home-1",
|
|
ServiceType: "mesh-listener",
|
|
DesiredState: "enabled",
|
|
Version: &version,
|
|
Config: json.RawMessage(`{
|
|
"listen_addr":"0.0.0.0:19131",
|
|
"listen_port_mode":"manual",
|
|
"advertise_endpoint":"http://94.141.118.222:19199",
|
|
"advertise_transport":"direct_http",
|
|
"connectivity_mode":"direct",
|
|
"nat_type":"port_restricted",
|
|
"region":"home"
|
|
}`),
|
|
UpdatedAt: now,
|
|
}},
|
|
})
|
|
service.now = func() time.Time { return now }
|
|
|
|
cfg, err := service.GetNodeSyntheticMeshConfig(context.Background(), GetNodeSyntheticMeshConfigInput{
|
|
ClusterID: "cluster-1",
|
|
NodeID: "remote",
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("get synthetic config: %v", err)
|
|
}
|
|
if cfg.PeerEndpoints["home-1"] != "http://94.141.118.222:19199" {
|
|
t.Fatalf("operator public home peer should survive warm-peer target: %+v", cfg.PeerEndpoints)
|
|
}
|
|
homeCandidates := cfg.PeerEndpointCandidates["home-1"]
|
|
if len(homeCandidates) == 0 || homeCandidates[0].EndpointID != "home-1-desired-mesh-listener" {
|
|
t.Fatalf("home desired public candidate missing: %+v", homeCandidates)
|
|
}
|
|
if _, ok := findPeerDirectoryEntry(cfg.PeerDirectory, "home-1"); !ok {
|
|
t.Fatalf("home peer directory entry missing: %+v", cfg.PeerDirectory)
|
|
}
|
|
}
|
|
|
|
func TestGetNodeSyntheticMeshConfigFiltersLoopbackReportedMeshEndpoint(t *testing.T) {
|
|
now := time.Date(2026, 4, 28, 12, 0, 0, 0, time.UTC)
|
|
service := NewService(&fakeRepository{
|
|
testingFlags: EffectiveNodeTestingFlags{
|
|
Enabled: true,
|
|
SyntheticLinksEnabled: true,
|
|
},
|
|
routeIntents: []MeshRouteIntent{
|
|
{
|
|
ID: "route-a-b",
|
|
ClusterID: "cluster-1",
|
|
SourceSelector: json.RawMessage(`{"node_id":"node-a"}`),
|
|
DestinationSelector: json.RawMessage(`{"node_id":"node-b"}`),
|
|
ServiceClass: "synthetic",
|
|
Status: "active",
|
|
Policy: json.RawMessage(`{
|
|
"synthetic_enabled": true,
|
|
"hops": ["node-a", "node-b"]
|
|
}`),
|
|
UpdatedAt: now,
|
|
},
|
|
},
|
|
heartbeats: map[string][]NodeHeartbeat{
|
|
"node-b": {
|
|
{
|
|
ClusterID: "cluster-1",
|
|
NodeID: "node-b",
|
|
Metadata: json.RawMessage(`{
|
|
"mesh_endpoint_report": {
|
|
"schema_version": "c17z25.mesh_endpoint_report.v1",
|
|
"cluster_id": "cluster-1",
|
|
"node_id": "node-b",
|
|
"peer_endpoint": "http://127.0.0.1:19131",
|
|
"transport": "direct_http",
|
|
"connectivity_mode": "private_lan",
|
|
"nat_type": "none",
|
|
"endpoint_candidates": [
|
|
{
|
|
"endpoint_id": "node-b-loopback",
|
|
"node_id": "node-b",
|
|
"transport": "direct_http",
|
|
"address": "http://127.0.0.1:19131",
|
|
"reachability": "private",
|
|
"connectivity_mode": "private_lan",
|
|
"nat_type": "none",
|
|
"priority": 1
|
|
},
|
|
{
|
|
"endpoint_id": "node-b-lan",
|
|
"node_id": "node-b",
|
|
"transport": "direct_http",
|
|
"address": "http://192.168.10.20:19131",
|
|
"reachability": "private",
|
|
"connectivity_mode": "private_lan",
|
|
"nat_type": "none",
|
|
"priority": 2
|
|
}
|
|
]
|
|
}
|
|
}`),
|
|
ObservedAt: now,
|
|
},
|
|
},
|
|
},
|
|
})
|
|
service.now = func() time.Time { return now }
|
|
|
|
cfg, err := service.GetNodeSyntheticMeshConfig(context.Background(), GetNodeSyntheticMeshConfigInput{
|
|
ClusterID: "cluster-1",
|
|
NodeID: "node-a",
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("get synthetic config: %v", err)
|
|
}
|
|
if _, leaked := cfg.PeerEndpoints["node-b"]; leaked {
|
|
t.Fatalf("loopback peer endpoint leaked: %+v", cfg.PeerEndpoints)
|
|
}
|
|
if got := cfg.PeerEndpointCandidates["node-b"]; len(got) != 1 || got[0].EndpointID != "node-b-lan" || got[0].Address != "http://192.168.10.20:19131" {
|
|
t.Fatalf("loopback candidates not filtered correctly: %+v", cfg.PeerEndpointCandidates)
|
|
}
|
|
entry, ok := findPeerDirectoryEntry(cfg.PeerDirectory, "node-b")
|
|
if !ok || entry.EndpointCount != 0 || entry.CandidateCount != 1 {
|
|
t.Fatalf("peer directory should expose only usable candidate: %+v", cfg.PeerDirectory)
|
|
}
|
|
}
|
|
|
|
func TestScopedPeerEndpointsFiltersLoopbackPolicyEndpoints(t *testing.T) {
|
|
got := scopedPeerEndpoints(map[string]string{
|
|
"node-a": "http://127.0.0.1:19131",
|
|
"node-b": "http://0.0.0.0:19132",
|
|
"node-c": "http://192.168.10.20:19133",
|
|
"node-d": "http://localhost:19134",
|
|
}, []string{"node-a", "node-b", "node-c", "node-d"})
|
|
|
|
if len(got) != 1 || got["node-c"] != "http://192.168.10.20:19133" {
|
|
t.Fatalf("loopback/wildcard policy endpoints leaked: %+v", got)
|
|
}
|
|
}
|
|
|
|
func TestGetNodeSyntheticMeshConfigBootstrapsCoreMeshPeersFromHealthyNodes(t *testing.T) {
|
|
now := time.Date(2026, 4, 28, 12, 0, 0, 0, time.UTC)
|
|
service := NewService(&fakeRepository{
|
|
testingFlags: EffectiveNodeTestingFlags{
|
|
Enabled: true,
|
|
SyntheticLinksEnabled: true,
|
|
},
|
|
clusterNodes: []ClusterNode{
|
|
{ID: "node-a", RegistrationStatus: NodeRegistrationActive, HealthStatus: "healthy", MembershipStatus: "active", CreatedAt: now.Add(-2 * time.Hour), LastSeenAt: ptrTime(now)},
|
|
{ID: "node-b", RegistrationStatus: NodeRegistrationActive, HealthStatus: "healthy", MembershipStatus: "active", CreatedAt: now.Add(-time.Hour), LastSeenAt: ptrTime(now.Add(-time.Second))},
|
|
{ID: "node-c", RegistrationStatus: NodeRegistrationActive, HealthStatus: "healthy", MembershipStatus: "active", CreatedAt: now.Add(-30 * time.Minute), LastSeenAt: ptrTime(now.Add(-2 * time.Second))},
|
|
},
|
|
nodeRoles: map[string][]NodeRoleAssignment{
|
|
"node-a": {{NodeID: "node-a", Role: "core-mesh", Status: "active"}},
|
|
},
|
|
heartbeats: map[string][]NodeHeartbeat{
|
|
"node-b": {{
|
|
ClusterID: "cluster-1",
|
|
NodeID: "node-b",
|
|
ObservedAt: now,
|
|
Metadata: json.RawMessage(`{
|
|
"mesh_endpoint_report": {
|
|
"cluster_id": "cluster-1",
|
|
"node_id": "node-b",
|
|
"peer_endpoint": "http://10.0.0.2:19131",
|
|
"transport": "direct_http",
|
|
"connectivity_mode": "private_lan"
|
|
}
|
|
}`),
|
|
}},
|
|
"node-c": {{
|
|
ClusterID: "cluster-1",
|
|
NodeID: "node-c",
|
|
ObservedAt: now,
|
|
Metadata: json.RawMessage(`{
|
|
"mesh_endpoint_report": {
|
|
"cluster_id": "cluster-1",
|
|
"node_id": "node-c",
|
|
"endpoint_candidates": [{
|
|
"endpoint_id": "node-c-lan",
|
|
"node_id": "node-c",
|
|
"transport": "direct_http",
|
|
"address": "http://10.0.0.3:19131",
|
|
"reachability": "private",
|
|
"connectivity_mode": "private_lan",
|
|
"priority": 1
|
|
}]
|
|
}
|
|
}`),
|
|
}},
|
|
},
|
|
})
|
|
service.now = func() time.Time { return now }
|
|
|
|
cfg, err := service.GetNodeSyntheticMeshConfig(context.Background(), GetNodeSyntheticMeshConfigInput{
|
|
ClusterID: "cluster-1",
|
|
NodeID: "node-a",
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("get synthetic config: %v", err)
|
|
}
|
|
if cfg.PeerEndpoints["node-b"] != "http://10.0.0.2:19131" {
|
|
t.Fatalf("reported peer endpoint not bootstrapped: %+v", cfg.PeerEndpoints)
|
|
}
|
|
if got := cfg.PeerEndpointCandidates["node-c"]; len(got) != 1 || got[0].EndpointID != "node-c-lan" {
|
|
t.Fatalf("reported peer candidates not bootstrapped: %+v", cfg.PeerEndpointCandidates)
|
|
}
|
|
if len(cfg.RecoverySeeds) != 2 {
|
|
t.Fatalf("RecoverySeeds = %+v, want two core mesh bootstrap seeds", cfg.RecoverySeeds)
|
|
}
|
|
if _, ok := findPeerDirectoryEntry(cfg.PeerDirectory, "node-b"); !ok {
|
|
t.Fatalf("peer directory missing node-b: %+v", cfg.PeerDirectory)
|
|
}
|
|
if _, ok := findPeerDirectoryEntry(cfg.PeerDirectory, "node-c"); !ok {
|
|
t.Fatalf("peer directory missing node-c: %+v", cfg.PeerDirectory)
|
|
}
|
|
}
|
|
|
|
func TestGetNodeSyntheticMeshConfigScopesPrivateBootstrapPeersForOutboundOnlyNode(t *testing.T) {
|
|
now := time.Date(2026, 4, 28, 12, 0, 0, 0, time.UTC)
|
|
service := NewService(&fakeRepository{
|
|
testingFlags: EffectiveNodeTestingFlags{
|
|
Enabled: true,
|
|
SyntheticLinksEnabled: true,
|
|
},
|
|
clusterNodes: []ClusterNode{
|
|
{ID: "node-local", RegistrationStatus: NodeRegistrationActive, HealthStatus: "healthy", MembershipStatus: "active", CreatedAt: now.Add(-2 * time.Hour), LastSeenAt: ptrTime(now)},
|
|
{ID: "node-peer", RegistrationStatus: NodeRegistrationActive, HealthStatus: "healthy", MembershipStatus: "active", CreatedAt: now.Add(-time.Hour), LastSeenAt: ptrTime(now.Add(-time.Second))},
|
|
},
|
|
nodeRoles: map[string][]NodeRoleAssignment{
|
|
"node-local": {{NodeID: "node-local", Role: "core-mesh", Status: "active"}},
|
|
},
|
|
heartbeats: map[string][]NodeHeartbeat{
|
|
"node-local": {{
|
|
ClusterID: "cluster-1",
|
|
NodeID: "node-local",
|
|
ObservedAt: now,
|
|
Metadata: json.RawMessage(`{
|
|
"mesh_endpoint_report": {
|
|
"cluster_id": "cluster-1",
|
|
"node_id": "node-local",
|
|
"connectivity_mode": "outbound_only",
|
|
"region": "office"
|
|
},
|
|
"mesh_listener_report": {
|
|
"inbound_reachability": "outbound_only",
|
|
"one_way_connectivity": true
|
|
},
|
|
"mesh_outbound_session_report": {
|
|
"status": "ready",
|
|
"control_plane_url": "https://control.example.test/api/v1"
|
|
}
|
|
}`),
|
|
}},
|
|
"node-peer": {{
|
|
ClusterID: "cluster-1",
|
|
NodeID: "node-peer",
|
|
ObservedAt: now,
|
|
Metadata: json.RawMessage(`{
|
|
"mesh_endpoint_report": {
|
|
"cluster_id": "cluster-1",
|
|
"node_id": "node-peer",
|
|
"peer_endpoint": "http://192.168.200.61:19133",
|
|
"transport": "direct_http",
|
|
"connectivity_mode": "private_lan",
|
|
"endpoint_candidates": [{
|
|
"endpoint_id": "node-peer-lan",
|
|
"node_id": "node-peer",
|
|
"transport": "direct_http",
|
|
"address": "http://192.168.200.61:19133",
|
|
"reachability": "private",
|
|
"connectivity_mode": "private_lan",
|
|
"priority": 1
|
|
}]
|
|
}
|
|
}`),
|
|
}},
|
|
},
|
|
})
|
|
service.now = func() time.Time { return now }
|
|
|
|
cfg, err := service.GetNodeSyntheticMeshConfig(context.Background(), GetNodeSyntheticMeshConfigInput{
|
|
ClusterID: "cluster-1",
|
|
NodeID: "node-local",
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("get synthetic config: %v", err)
|
|
}
|
|
if endpoint := cfg.PeerEndpoints["node-peer"]; endpoint != "" {
|
|
t.Fatalf("private peer endpoint leaked to outbound-only node: %q", endpoint)
|
|
}
|
|
candidates := cfg.PeerEndpointCandidates["node-peer"]
|
|
if len(candidates) != 1 {
|
|
t.Fatalf("peer candidates = %+v, want relay-required candidate", cfg.PeerEndpointCandidates)
|
|
}
|
|
candidate := candidates[0]
|
|
if candidate.Transport != "relay" || candidate.Reachability != "relay" || candidate.ConnectivityMode != "relay_required" {
|
|
t.Fatalf("candidate not converted to relay required: %+v", candidate)
|
|
}
|
|
if !containsString(candidate.PolicyTags, "offsite-private-lan-blocked") || !containsString(candidate.PolicyTags, "relay-required") {
|
|
t.Fatalf("candidate missing offsite relay tags: %+v", candidate.PolicyTags)
|
|
}
|
|
for _, seed := range cfg.RecoverySeeds {
|
|
if seed.NodeID == "node-peer" {
|
|
t.Fatalf("private recovery seed leaked to outbound-only node: %+v", cfg.RecoverySeeds)
|
|
}
|
|
}
|
|
entry, ok := findPeerDirectoryEntry(cfg.PeerDirectory, "node-peer")
|
|
if !ok || entry.EndpointCount != 0 || entry.CandidateCount != 2 {
|
|
t.Fatalf("peer directory should show relay-required candidate and bootstrap lease: %+v", cfg.PeerDirectory)
|
|
}
|
|
if len(cfg.RendezvousLeases) != 1 {
|
|
t.Fatalf("rendezvous leases = %+v, want one control-plane bootstrap lease", cfg.RendezvousLeases)
|
|
}
|
|
lease := cfg.RendezvousLeases[0]
|
|
if lease.PeerNodeID != "node-peer" ||
|
|
lease.RelayNodeID != "control-plane-relay" ||
|
|
lease.RelayEndpoint != "https://control.example.test" ||
|
|
lease.Transport != "relay_control" ||
|
|
lease.Reason != "control_plane_bootstrap_relay" ||
|
|
!lease.ControlPlaneOnly {
|
|
t.Fatalf("unexpected bootstrap rendezvous lease: %+v", lease)
|
|
}
|
|
}
|
|
|
|
func TestGetNodeSyntheticMeshConfigIssuesRendezvousRelayLeases(t *testing.T) {
|
|
now := time.Date(2026, 4, 28, 12, 0, 0, 0, time.UTC)
|
|
service := NewService(&fakeRepository{
|
|
testingFlags: EffectiveNodeTestingFlags{
|
|
Enabled: true,
|
|
SyntheticLinksEnabled: true,
|
|
},
|
|
routeIntents: []MeshRouteIntent{
|
|
{
|
|
ID: "route-a-b",
|
|
ClusterID: "cluster-1",
|
|
SourceSelector: json.RawMessage(`{"node_id":"node-a"}`),
|
|
DestinationSelector: json.RawMessage(`{"node_id":"node-b"}`),
|
|
ServiceClass: "synthetic",
|
|
Status: "active",
|
|
Policy: json.RawMessage(`{
|
|
"synthetic_enabled": true,
|
|
"hops": ["node-a", "node-r", "node-b"],
|
|
"allowed_channels": ["fabric_control", "route_control", "service_payload"],
|
|
"peer_endpoints": {
|
|
"node-r": "http://node-r:19000"
|
|
},
|
|
"peer_endpoint_candidates": {
|
|
"node-b": [
|
|
{
|
|
"endpoint_id": "node-b-outbound",
|
|
"node_id": "node-b",
|
|
"transport": "outbound_reverse",
|
|
"address": "node-b.reverse.local",
|
|
"reachability": "outbound_only",
|
|
"nat_type": "symmetric",
|
|
"connectivity_mode": "outbound_only",
|
|
"priority": 20
|
|
}
|
|
]
|
|
},
|
|
"rendezvous_leases": [
|
|
{
|
|
"peer_node_id": "node-b",
|
|
"relay_node_id": "node-r",
|
|
"relay_endpoint": "http://node-r:19000",
|
|
"priority": 5
|
|
}
|
|
]
|
|
}`),
|
|
UpdatedAt: now,
|
|
},
|
|
{
|
|
ID: "route-x-y",
|
|
ClusterID: "cluster-1",
|
|
SourceSelector: json.RawMessage(`{"node_id":"node-x"}`),
|
|
DestinationSelector: json.RawMessage(`{"node_id":"node-y"}`),
|
|
ServiceClass: "synthetic",
|
|
Status: "active",
|
|
Policy: json.RawMessage(`{
|
|
"synthetic_enabled": true,
|
|
"hops": ["node-x", "node-y"],
|
|
"peer_endpoints": {"node-x": "http://node-x:19000"},
|
|
"rendezvous_leases": [
|
|
{
|
|
"peer_node_id": "node-y",
|
|
"relay_node_id": "node-x",
|
|
"relay_endpoint": "http://node-x:19000"
|
|
}
|
|
]
|
|
}`),
|
|
UpdatedAt: now,
|
|
},
|
|
},
|
|
})
|
|
service.now = func() time.Time { return now }
|
|
|
|
cfg, err := service.GetNodeSyntheticMeshConfig(context.Background(), GetNodeSyntheticMeshConfigInput{
|
|
ClusterID: "cluster-1",
|
|
NodeID: "node-a",
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("get synthetic config: %v", err)
|
|
}
|
|
if cfg.SchemaVersion != "c17z18.synthetic.v1" {
|
|
t.Fatalf("schema version = %s, want c17z18.synthetic.v1", cfg.SchemaVersion)
|
|
}
|
|
if len(cfg.RendezvousLeases) != 1 {
|
|
t.Fatalf("unexpected rendezvous leases: %+v", cfg.RendezvousLeases)
|
|
}
|
|
lease := cfg.RendezvousLeases[0]
|
|
if lease.LeaseID != "route-a-b-rv-node-b-via-node-r" ||
|
|
lease.PeerNodeID != "node-b" ||
|
|
lease.RelayNodeID != "node-r" ||
|
|
lease.RelayEndpoint != "http://node-r:19000" ||
|
|
lease.Transport != "relay_control" ||
|
|
lease.Priority != 5 ||
|
|
!lease.ControlPlaneOnly ||
|
|
!containsString(lease.AllowedChannels, "fabric_control") ||
|
|
containsString(lease.AllowedChannels, "service_payload") {
|
|
t.Fatalf("unexpected rendezvous lease contract: %+v", lease)
|
|
}
|
|
if _, ok := findPeerDirectoryEntry(cfg.PeerDirectory, "node-y"); ok {
|
|
t.Fatalf("unrelated rendezvous lease leaked into peer directory: %+v", cfg.PeerDirectory)
|
|
}
|
|
nodeB, ok := findPeerDirectoryEntry(cfg.PeerDirectory, "node-b")
|
|
if !ok || !containsString(nodeB.ConnectivityModes, "relay_required") {
|
|
t.Fatalf("peer directory missing rendezvous peer mode: %+v", cfg.PeerDirectory)
|
|
}
|
|
nodeR, ok := findPeerDirectoryEntry(cfg.PeerDirectory, "node-r")
|
|
if !ok || !containsString(nodeR.ConnectivityModes, "relay_control") {
|
|
t.Fatalf("peer directory missing relay control mode: %+v", cfg.PeerDirectory)
|
|
}
|
|
}
|
|
|
|
func TestGetNodeSyntheticMeshConfigReplacesStaleRendezvousRelay(t *testing.T) {
|
|
now := time.Date(2026, 4, 28, 12, 30, 0, 0, time.UTC)
|
|
staleHeartbeatMetadata, err := json.Marshal(map[string]any{
|
|
"mesh_rendezvous_lease_report": map[string]any{
|
|
"schema_version": "c17z18.mesh_rendezvous_lease_report.v1",
|
|
"cluster_id": "cluster-1",
|
|
"node_id": "node-a",
|
|
"observed_at": now.Format(time.RFC3339Nano),
|
|
"leases": []map[string]any{
|
|
{
|
|
"lease_id": "route-a-b-rv-node-b-via-node-r-old",
|
|
"peer_node_id": "node-b",
|
|
"relay_node_id": "node-r-old",
|
|
"route_ids": []string{"route-a-b"},
|
|
"stale_relay": true,
|
|
"reselection_needed": true,
|
|
"connection_state": "degraded",
|
|
"reason": "auto_outbound_only",
|
|
},
|
|
},
|
|
},
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("marshal heartbeat metadata: %v", err)
|
|
}
|
|
service := NewService(&fakeRepository{
|
|
testingFlags: EffectiveNodeTestingFlags{
|
|
Enabled: true,
|
|
SyntheticLinksEnabled: true,
|
|
},
|
|
heartbeats: map[string][]NodeHeartbeat{
|
|
"node-a": {
|
|
{
|
|
ClusterID: "cluster-1",
|
|
NodeID: "node-a",
|
|
Metadata: staleHeartbeatMetadata,
|
|
ObservedAt: now.Add(-10 * time.Second),
|
|
},
|
|
},
|
|
},
|
|
routeIntents: []MeshRouteIntent{
|
|
{
|
|
ID: "route-a-b",
|
|
ClusterID: "cluster-1",
|
|
SourceSelector: json.RawMessage(`{"node_id":"node-a"}`),
|
|
DestinationSelector: json.RawMessage(`{"node_id":"node-b"}`),
|
|
ServiceClass: "synthetic",
|
|
Status: "active",
|
|
Policy: json.RawMessage(`{
|
|
"synthetic_enabled": true,
|
|
"hops": ["node-a", "node-r-old", "node-r-new", "node-b"],
|
|
"allowed_channels": ["fabric_control", "route_control"],
|
|
"peer_endpoints": {
|
|
"node-r-old": "http://node-r-old:19000",
|
|
"node-r-new": "http://node-r-new:19000"
|
|
},
|
|
"peer_endpoint_candidates": {
|
|
"node-b": [
|
|
{
|
|
"endpoint_id": "node-b-outbound",
|
|
"node_id": "node-b",
|
|
"transport": "outbound_reverse",
|
|
"address": "node-b.reverse.local",
|
|
"reachability": "outbound_only",
|
|
"nat_type": "symmetric",
|
|
"connectivity_mode": "outbound_only",
|
|
"priority": 5
|
|
}
|
|
]
|
|
},
|
|
"rendezvous_leases": [
|
|
{
|
|
"lease_id": "route-a-b-rv-node-b-via-node-r-old",
|
|
"peer_node_id": "node-b",
|
|
"relay_node_id": "node-r-old",
|
|
"relay_endpoint": "http://node-r-old:19000",
|
|
"priority": 4
|
|
}
|
|
]
|
|
}`),
|
|
UpdatedAt: now,
|
|
},
|
|
},
|
|
})
|
|
service.now = func() time.Time { return now }
|
|
|
|
cfg, err := service.GetNodeSyntheticMeshConfig(context.Background(), GetNodeSyntheticMeshConfigInput{
|
|
ClusterID: "cluster-1",
|
|
NodeID: "node-a",
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("get synthetic config: %v", err)
|
|
}
|
|
if len(cfg.RendezvousLeases) != 1 {
|
|
t.Fatalf("unexpected rendezvous leases: %+v", cfg.RendezvousLeases)
|
|
}
|
|
lease := cfg.RendezvousLeases[0]
|
|
if lease.RelayNodeID != "node-r-new" ||
|
|
lease.LeaseID != "route-a-b-rv-node-b-via-node-r-new" ||
|
|
lease.Reason != "stale_relay_replacement" {
|
|
t.Fatalf("stale relay was not replaced: %+v", lease)
|
|
}
|
|
var metadata map[string]any
|
|
if err := json.Unmarshal(lease.Metadata, &metadata); err != nil {
|
|
t.Fatalf("unmarshal lease metadata: %v", err)
|
|
}
|
|
if metadata["replacement_for_stale_relay"] != true ||
|
|
metadata["relay_replacement_contract"] != "stale_relay_feedback_policy" {
|
|
t.Fatalf("replacement metadata missing: %+v", metadata)
|
|
}
|
|
if cfg.RendezvousRelayPolicy == nil ||
|
|
cfg.RendezvousRelayPolicy.StaleRelayCount != 1 ||
|
|
cfg.RendezvousRelayPolicy.WithdrawnLeaseCount != 1 ||
|
|
cfg.RendezvousRelayPolicy.ReplacementLeaseCount != 1 {
|
|
t.Fatalf("unexpected relay policy report: %+v", cfg.RendezvousRelayPolicy)
|
|
}
|
|
var decision RendezvousRelayPolicyDecision
|
|
for _, item := range cfg.RendezvousRelayPolicy.Decisions {
|
|
if item.Reason == "stale_relay_replacement" {
|
|
decision = item
|
|
break
|
|
}
|
|
}
|
|
if decision.SelectedRelayID != "node-r-new" || decision.StaleRelayNodeID != "node-r-old" {
|
|
t.Fatalf("unexpected relay replacement decision: %+v", cfg.RendezvousRelayPolicy.Decisions)
|
|
}
|
|
if cfg.RoutePathDecisions == nil ||
|
|
cfg.RoutePathDecisions.SchemaVersion != "c17z18.route_path_decisions.v1" ||
|
|
cfg.RoutePathDecisions.DecisionCount != 1 ||
|
|
cfg.RoutePathDecisions.ReplacementDecisionCount != 1 {
|
|
t.Fatalf("unexpected route path decisions: %+v", cfg.RoutePathDecisions)
|
|
}
|
|
pathDecision := cfg.RoutePathDecisions.Decisions[0]
|
|
if pathDecision.DecisionSource != "stale_relay_replacement" ||
|
|
pathDecision.SelectedRelayID != "node-r-new" ||
|
|
pathDecision.StaleRelayNodeID != "node-r-old" ||
|
|
pathDecision.RendezvousPeerNodeID != "node-b" ||
|
|
pathDecision.RendezvousLeaseID != "route-a-b-rv-node-b-via-node-r-new" ||
|
|
pathDecision.NextHopID != "node-r-new" ||
|
|
pathDecision.ProductionForwarding ||
|
|
!pathDecision.ControlPlaneOnly ||
|
|
strings.Join(pathDecision.EffectiveHops, ",") != "node-a,node-r-new,node-b" {
|
|
t.Fatalf("unexpected route path decision: %+v", pathDecision)
|
|
}
|
|
}
|
|
|
|
func TestGetNodeSyntheticMeshConfigAppliesReplacementPathHintForExit(t *testing.T) {
|
|
now := time.Date(2026, 4, 28, 12, 30, 0, 0, time.UTC)
|
|
hintMetadata, err := json.Marshal(map[string]any{
|
|
"mesh_route_path_decision_report": map[string]any{
|
|
"cluster_id": "cluster-1",
|
|
"node_id": "node-a",
|
|
"decisions": []map[string]any{
|
|
{
|
|
"decision_id": "route-a-b-path-node-a-via-node-r-new",
|
|
"route_id": "route-a-b",
|
|
"cluster_id": "cluster-1",
|
|
"local_node_id": "node-a",
|
|
"source_node_id": "node-a",
|
|
"destination_node_id": "node-b",
|
|
"original_hops": []string{"node-a", "node-r-old", "node-r-new", "node-b"},
|
|
"effective_hops": []string{"node-a", "node-r-new", "node-b"},
|
|
"next_hop_id": "node-r-new",
|
|
"local_role": "entry",
|
|
"selected_relay_id": "node-r-new",
|
|
"selected_relay_endpoint": "http://node-r-new:19000",
|
|
"stale_relay_node_id": "node-r-old",
|
|
"rendezvous_peer_node_id": "node-b",
|
|
"rendezvous_lease_id": "route-a-b-rv-node-b-via-node-r-new",
|
|
"rendezvous_lease_reason": "stale_relay_replacement",
|
|
"decision_source": "stale_relay_replacement",
|
|
"generation": "hint-generation",
|
|
"path_score": 900,
|
|
"score_reasons": []string{"route_path_decision_hint"},
|
|
"control_plane_only": true,
|
|
"production_forwarding": false,
|
|
"expires_at": now.Add(time.Hour).UTC().Format(time.RFC3339Nano),
|
|
},
|
|
},
|
|
},
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("marshal hint metadata: %v", err)
|
|
}
|
|
service := NewService(&fakeRepository{
|
|
testingFlags: EffectiveNodeTestingFlags{
|
|
Enabled: true,
|
|
SyntheticLinksEnabled: true,
|
|
},
|
|
heartbeats: map[string][]NodeHeartbeat{
|
|
"node-a": {
|
|
{
|
|
ClusterID: "cluster-1",
|
|
NodeID: "node-a",
|
|
Metadata: hintMetadata,
|
|
ObservedAt: now.Add(-10 * time.Second),
|
|
},
|
|
},
|
|
},
|
|
routeIntents: []MeshRouteIntent{
|
|
{
|
|
ID: "route-a-b",
|
|
ClusterID: "cluster-1",
|
|
SourceSelector: json.RawMessage(`{"node_id":"node-a"}`),
|
|
DestinationSelector: json.RawMessage(`{"node_id":"node-b"}`),
|
|
ServiceClass: "synthetic",
|
|
Status: "active",
|
|
Policy: json.RawMessage(`{
|
|
"synthetic_enabled": true,
|
|
"hops": ["node-a", "node-r-old", "node-r-new", "node-b"],
|
|
"allowed_channels": ["fabric_control", "route_control"],
|
|
"peer_endpoints": {
|
|
"node-r-old": "http://node-r-old:19000",
|
|
"node-r-new": "http://node-r-new:19000"
|
|
},
|
|
"peer_endpoint_candidates": {
|
|
"node-b": [
|
|
{
|
|
"endpoint_id": "node-b-outbound",
|
|
"node_id": "node-b",
|
|
"transport": "outbound_reverse",
|
|
"address": "node-b.reverse.local",
|
|
"reachability": "outbound_only",
|
|
"nat_type": "symmetric",
|
|
"connectivity_mode": "outbound_only",
|
|
"priority": 5
|
|
}
|
|
]
|
|
},
|
|
"rendezvous_leases": [
|
|
{
|
|
"lease_id": "route-a-b-rv-node-b-via-node-r-old",
|
|
"peer_node_id": "node-b",
|
|
"relay_node_id": "node-r-old",
|
|
"relay_endpoint": "http://node-r-old:19000",
|
|
"priority": 4
|
|
}
|
|
]
|
|
}`),
|
|
UpdatedAt: now,
|
|
},
|
|
},
|
|
})
|
|
service.now = func() time.Time { return now }
|
|
|
|
cfg, err := service.GetNodeSyntheticMeshConfig(context.Background(), GetNodeSyntheticMeshConfigInput{
|
|
ClusterID: "cluster-1",
|
|
NodeID: "node-b",
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("get synthetic config: %v", err)
|
|
}
|
|
if len(cfg.RendezvousLeases) != 1 ||
|
|
cfg.RendezvousLeases[0].RelayNodeID != "node-r-new" ||
|
|
cfg.RendezvousLeases[0].Reason != "stale_relay_replacement" {
|
|
t.Fatalf("replacement hint did not withdraw stale relay lease: %+v", cfg.RendezvousLeases)
|
|
}
|
|
if cfg.RoutePathDecisions == nil ||
|
|
cfg.RoutePathDecisions.ReplacementDecisionCount != 1 ||
|
|
len(cfg.RoutePathDecisions.Decisions) != 1 {
|
|
t.Fatalf("unexpected route path decisions: %+v", cfg.RoutePathDecisions)
|
|
}
|
|
decision := cfg.RoutePathDecisions.Decisions[0]
|
|
if decision.DecisionSource != "stale_relay_replacement" ||
|
|
decision.LocalRole != "exit" ||
|
|
decision.PreviousHopID != "node-r-new" ||
|
|
decision.SelectedRelayID != "node-r-new" ||
|
|
decision.StaleRelayNodeID != "node-r-old" ||
|
|
decision.RendezvousPeerNodeID != "node-b" ||
|
|
strings.Join(decision.EffectiveHops, ",") != "node-a,node-r-new,node-b" {
|
|
t.Fatalf("unexpected hinted route path decision: %+v", decision)
|
|
}
|
|
}
|
|
|
|
func TestGetNodeSyntheticMeshConfigUsesRouteHealthDriftToReselectRelay(t *testing.T) {
|
|
now := time.Date(2026, 4, 28, 12, 30, 0, 0, time.UTC)
|
|
routeHealthMetadata, err := json.Marshal(map[string]any{
|
|
"observation_type": "synthetic_route_health",
|
|
"route_id": "route-a-b",
|
|
"route_path_decision_applied": true,
|
|
"route_path_decision_selected_relay_id": "node-s",
|
|
"route_path_decision_rendezvous_peer_node_id": "node-b",
|
|
"route_path_decision_rendezvous_lease_id": "route-a-b-rv-node-b-via-node-s",
|
|
"route_path_decision_rendezvous_lease_reason": "auto_rendezvous_required",
|
|
"expected_effective_hops": []string{"node-a", "node-s", "node-b"},
|
|
"observed_ack_path": []string{"node-a", "node-t", "node-b"},
|
|
"route_path_drift_detected": true,
|
|
"control_plane_only": true,
|
|
"production_forwarding": false,
|
|
"production_payload_forwarding": false,
|
|
"route_health_production_payload_forwarding": false,
|
|
"route_health_service_payload_forwarding": false,
|
|
"synthetic_route_health_route_path_runtime": true,
|
|
"production_route_path_forwarding_runtime": false,
|
|
"route_health_route_config_contract": "control_plane_route_path_decisions_to_synthetic_route_health",
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("marshal route health metadata: %v", err)
|
|
}
|
|
service := NewService(&fakeRepository{
|
|
testingFlags: EffectiveNodeTestingFlags{
|
|
Enabled: true,
|
|
SyntheticLinksEnabled: true,
|
|
},
|
|
meshLinks: []MeshLinkObservation{
|
|
{
|
|
ClusterID: "cluster-1",
|
|
SourceNodeID: "node-a",
|
|
TargetNodeID: "node-b",
|
|
LinkStatus: "reachable",
|
|
Metadata: routeHealthMetadata,
|
|
ObservedAt: now.Add(-10 * time.Second),
|
|
},
|
|
},
|
|
routeIntents: []MeshRouteIntent{
|
|
{
|
|
ID: "route-a-b",
|
|
ClusterID: "cluster-1",
|
|
SourceSelector: json.RawMessage(`{"node_id":"node-a"}`),
|
|
DestinationSelector: json.RawMessage(`{"node_id":"node-b"}`),
|
|
ServiceClass: "synthetic",
|
|
Status: "active",
|
|
Policy: json.RawMessage(`{
|
|
"synthetic_enabled": true,
|
|
"hops": ["node-a", "node-s", "node-t", "node-b"],
|
|
"allowed_channels": ["fabric_control", "route_control"],
|
|
"peer_endpoint_candidates": {
|
|
"node-b": [
|
|
{
|
|
"endpoint_id": "node-b-outbound",
|
|
"node_id": "node-b",
|
|
"transport": "outbound_reverse",
|
|
"address": "node-b.reverse.local",
|
|
"reachability": "outbound_only",
|
|
"nat_type": "symmetric",
|
|
"connectivity_mode": "outbound_only",
|
|
"priority": 5
|
|
}
|
|
],
|
|
"node-s": [
|
|
{
|
|
"endpoint_id": "node-s-public",
|
|
"node_id": "node-s",
|
|
"transport": "direct_tcp_tls",
|
|
"address": "http://node-s:19000",
|
|
"reachability": "public",
|
|
"nat_type": "none",
|
|
"connectivity_mode": "direct",
|
|
"priority": 1,
|
|
"policy_tags": ["fast-path"]
|
|
}
|
|
],
|
|
"node-t": [
|
|
{
|
|
"endpoint_id": "node-t-public",
|
|
"node_id": "node-t",
|
|
"transport": "direct_tcp_tls",
|
|
"address": "http://node-t:19000",
|
|
"reachability": "public",
|
|
"nat_type": "none",
|
|
"connectivity_mode": "direct",
|
|
"priority": 50
|
|
}
|
|
]
|
|
}
|
|
}`),
|
|
UpdatedAt: now,
|
|
},
|
|
},
|
|
})
|
|
service.now = func() time.Time { return now }
|
|
|
|
cfg, err := service.GetNodeSyntheticMeshConfig(context.Background(), GetNodeSyntheticMeshConfigInput{
|
|
ClusterID: "cluster-1",
|
|
NodeID: "node-a",
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("get synthetic config: %v", err)
|
|
}
|
|
if len(cfg.RendezvousLeases) != 1 {
|
|
t.Fatalf("unexpected rendezvous leases: %+v", cfg.RendezvousLeases)
|
|
}
|
|
lease := cfg.RendezvousLeases[0]
|
|
if lease.RelayNodeID != "node-t" || lease.Reason != "stale_relay_replacement" {
|
|
t.Fatalf("route health drift did not reselect relay: %+v", lease)
|
|
}
|
|
if cfg.RendezvousRelayPolicy == nil ||
|
|
cfg.RendezvousRelayPolicy.StaleRelayCount != 1 ||
|
|
cfg.RendezvousRelayPolicy.ReplacementLeaseCount != 1 ||
|
|
cfg.RendezvousRelayPolicy.ScoringMode != "route_adjacency_endpoint_priority_mesh_link_health_synthetic_route_health_feedback" {
|
|
t.Fatalf("unexpected relay policy report: %+v", cfg.RendezvousRelayPolicy)
|
|
}
|
|
var policyDecision RendezvousRelayPolicyDecision
|
|
for _, item := range cfg.RendezvousRelayPolicy.Decisions {
|
|
if item.Reason == "stale_relay_replacement" {
|
|
policyDecision = item
|
|
break
|
|
}
|
|
}
|
|
if policyDecision.StaleRelayNodeID != "node-s" || policyDecision.SelectedRelayID != "node-t" || policyDecision.PeerNodeID != "node-b" {
|
|
t.Fatalf("unexpected route health replacement decision: %+v", cfg.RendezvousRelayPolicy.Decisions)
|
|
}
|
|
if cfg.RoutePathDecisions == nil || cfg.RoutePathDecisions.ReplacementDecisionCount != 1 {
|
|
t.Fatalf("expected replacement route path decision: %+v", cfg.RoutePathDecisions)
|
|
}
|
|
decision := cfg.RoutePathDecisions.Decisions[0]
|
|
if decision.SelectedRelayID != "node-t" ||
|
|
decision.StaleRelayNodeID != "node-s" ||
|
|
decision.RendezvousPeerNodeID != "node-b" ||
|
|
strings.Join(decision.EffectiveHops, ",") != "node-a,node-t,node-b" ||
|
|
decision.ProductionForwarding ||
|
|
!decision.ControlPlaneOnly {
|
|
t.Fatalf("unexpected route path decision from route health feedback: %+v", decision)
|
|
}
|
|
}
|
|
|
|
func TestGetNodeSyntheticMeshConfigUsesRouteHealthLatencyForRelayScore(t *testing.T) {
|
|
now := time.Date(2026, 4, 28, 12, 30, 0, 0, time.UTC)
|
|
routeHealthMetadata, err := json.Marshal(map[string]any{
|
|
"observation_type": "synthetic_route_health",
|
|
"route_id": "route-a-b",
|
|
"route_path_decision_applied": true,
|
|
"route_path_decision_selected_relay_id": "node-t",
|
|
"route_path_decision_rendezvous_peer_node_id": "node-b",
|
|
"expected_effective_hops": []string{"node-a", "node-t", "node-b"},
|
|
"observed_ack_path": []string{"node-a", "node-t", "node-b"},
|
|
"route_path_drift_detected": false,
|
|
"control_plane_only": true,
|
|
"production_forwarding": false,
|
|
"production_payload_forwarding": false,
|
|
"route_health_production_payload_forwarding": false,
|
|
"route_health_service_payload_forwarding": false,
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("marshal route health metadata: %v", err)
|
|
}
|
|
latency := 5
|
|
quality := 99
|
|
service := NewService(&fakeRepository{
|
|
testingFlags: EffectiveNodeTestingFlags{
|
|
Enabled: true,
|
|
SyntheticLinksEnabled: true,
|
|
},
|
|
meshLinks: []MeshLinkObservation{
|
|
{
|
|
ClusterID: "cluster-1",
|
|
SourceNodeID: "node-a",
|
|
TargetNodeID: "node-b",
|
|
LinkStatus: "reachable",
|
|
LatencyMs: &latency,
|
|
QualityScore: &quality,
|
|
Metadata: routeHealthMetadata,
|
|
ObservedAt: now.Add(-10 * time.Second),
|
|
},
|
|
},
|
|
routeIntents: []MeshRouteIntent{
|
|
{
|
|
ID: "route-a-b",
|
|
ClusterID: "cluster-1",
|
|
SourceSelector: json.RawMessage(`{"node_id":"node-a"}`),
|
|
DestinationSelector: json.RawMessage(`{"node_id":"node-b"}`),
|
|
ServiceClass: "synthetic",
|
|
Status: "active",
|
|
Policy: json.RawMessage(`{
|
|
"synthetic_enabled": true,
|
|
"hops": ["node-a", "node-s", "node-t", "node-b"],
|
|
"allowed_channels": ["fabric_control", "route_control"],
|
|
"peer_endpoint_candidates": {
|
|
"node-b": [
|
|
{
|
|
"endpoint_id": "node-b-outbound",
|
|
"node_id": "node-b",
|
|
"transport": "outbound_reverse",
|
|
"address": "node-b.reverse.local",
|
|
"reachability": "outbound_only",
|
|
"nat_type": "symmetric",
|
|
"connectivity_mode": "outbound_only",
|
|
"priority": 5
|
|
}
|
|
],
|
|
"node-s": [
|
|
{
|
|
"endpoint_id": "node-s-public",
|
|
"node_id": "node-s",
|
|
"transport": "direct_tcp_tls",
|
|
"address": "http://node-s:19000",
|
|
"reachability": "public",
|
|
"nat_type": "none",
|
|
"connectivity_mode": "direct",
|
|
"priority": 1,
|
|
"policy_tags": ["fast-path"]
|
|
}
|
|
],
|
|
"node-t": [
|
|
{
|
|
"endpoint_id": "node-t-public",
|
|
"node_id": "node-t",
|
|
"transport": "direct_tcp_tls",
|
|
"address": "http://node-t:19000",
|
|
"reachability": "public",
|
|
"nat_type": "none",
|
|
"connectivity_mode": "direct",
|
|
"priority": 50
|
|
}
|
|
]
|
|
}
|
|
}`),
|
|
UpdatedAt: now,
|
|
},
|
|
},
|
|
})
|
|
service.now = func() time.Time { return now }
|
|
|
|
cfg, err := service.GetNodeSyntheticMeshConfig(context.Background(), GetNodeSyntheticMeshConfigInput{
|
|
ClusterID: "cluster-1",
|
|
NodeID: "node-a",
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("get synthetic config: %v", err)
|
|
}
|
|
if len(cfg.RendezvousLeases) != 1 {
|
|
t.Fatalf("unexpected rendezvous leases: %+v", cfg.RendezvousLeases)
|
|
}
|
|
lease := cfg.RendezvousLeases[0]
|
|
if lease.RelayNodeID != "node-t" || lease.Reason == "stale_relay_replacement" {
|
|
t.Fatalf("route health latency did not influence relay score: %+v", lease)
|
|
}
|
|
var metadata map[string]any
|
|
if err := json.Unmarshal(lease.Metadata, &metadata); err != nil {
|
|
t.Fatalf("unmarshal lease metadata: %v", err)
|
|
}
|
|
reasons, _ := metadata["relay_selection_score_reasons"].([]any)
|
|
if !anyString(reasons, "route_health_reachable") ||
|
|
!anyString(reasons, "route_health_no_drift") ||
|
|
!anyString(reasons, "route_health_latency") {
|
|
t.Fatalf("route health score reasons missing: %+v", metadata)
|
|
}
|
|
}
|
|
|
|
func anyString(values []any, want string) bool {
|
|
for _, value := range values {
|
|
if text, ok := value.(string); ok && text == want {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
func findPeerDirectoryEntry(entries []PeerDirectoryEntry, nodeID string) (PeerDirectoryEntry, bool) {
|
|
for _, entry := range entries {
|
|
if entry.NodeID == nodeID {
|
|
return entry, true
|
|
}
|
|
}
|
|
return PeerDirectoryEntry{}, false
|
|
}
|
|
|
|
func TestValidatePeerEndpointCandidates(t *testing.T) {
|
|
valid := map[string][]PeerEndpointCandidate{
|
|
"node-b": {
|
|
{
|
|
EndpointID: "node-b-public",
|
|
NodeID: "node-b",
|
|
Transport: "direct_tcp_tls",
|
|
Address: "203.0.113.20:443",
|
|
AddressFamily: "ipv4",
|
|
Reachability: "public",
|
|
NATType: "restricted",
|
|
ConnectivityMode: "direct",
|
|
Priority: 10,
|
|
Metadata: json.RawMessage(`{"source":"test"}`),
|
|
},
|
|
},
|
|
}
|
|
if err := validatePeerEndpointCandidates(valid, []string{"node-a", "node-b"}); err != nil {
|
|
t.Fatalf("validate valid candidates: %v", err)
|
|
}
|
|
|
|
tests := []struct {
|
|
name string
|
|
candidates map[string][]PeerEndpointCandidate
|
|
}{
|
|
{
|
|
name: "unknown transport",
|
|
candidates: map[string][]PeerEndpointCandidate{"node-b": {{
|
|
EndpointID: "node-b-public",
|
|
NodeID: "node-b",
|
|
Transport: "udp-hole-punch",
|
|
Address: "203.0.113.20:443",
|
|
Reachability: "public",
|
|
ConnectivityMode: "direct",
|
|
}}},
|
|
},
|
|
{
|
|
name: "unknown nat",
|
|
candidates: map[string][]PeerEndpointCandidate{"node-b": {{
|
|
EndpointID: "node-b-public",
|
|
NodeID: "node-b",
|
|
Transport: "direct_tcp_tls",
|
|
Address: "203.0.113.20:443",
|
|
Reachability: "public",
|
|
NATType: "mystery_nat",
|
|
ConnectivityMode: "direct",
|
|
}}},
|
|
},
|
|
{
|
|
name: "node outside route path",
|
|
candidates: map[string][]PeerEndpointCandidate{"node-y": {{
|
|
EndpointID: "node-y-public",
|
|
NodeID: "node-y",
|
|
Transport: "direct_tcp_tls",
|
|
Address: "203.0.113.30:443",
|
|
Reachability: "public",
|
|
ConnectivityMode: "direct",
|
|
}}},
|
|
},
|
|
{
|
|
name: "node mismatch",
|
|
candidates: map[string][]PeerEndpointCandidate{"node-b": {{
|
|
EndpointID: "node-b-public",
|
|
NodeID: "node-c",
|
|
Transport: "direct_tcp_tls",
|
|
Address: "203.0.113.20:443",
|
|
Reachability: "public",
|
|
ConnectivityMode: "direct",
|
|
}}},
|
|
},
|
|
{
|
|
name: "invalid metadata",
|
|
candidates: map[string][]PeerEndpointCandidate{"node-b": {{
|
|
EndpointID: "node-b-public",
|
|
NodeID: "node-b",
|
|
Transport: "direct_tcp_tls",
|
|
Address: "203.0.113.20:443",
|
|
Reachability: "public",
|
|
ConnectivityMode: "direct",
|
|
Metadata: json.RawMessage(`{`),
|
|
}}},
|
|
},
|
|
}
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
err := validatePeerEndpointCandidates(tt.candidates, []string{"node-a", "node-b"})
|
|
if !errors.Is(err, ErrInvalidPayload) {
|
|
t.Fatalf("err = %v, want ErrInvalidPayload", err)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestMinorityClusterBlocksPolicyMutation(t *testing.T) {
|
|
store := &fakeRepository{
|
|
platformRole: PlatformRoleAdmin,
|
|
authorityState: ClusterAuthorityState{
|
|
ClusterID: "cluster-1",
|
|
AuthorityState: "minority",
|
|
MutationMode: "read_only",
|
|
},
|
|
}
|
|
service := NewService(store)
|
|
|
|
_, err := service.AssignNodeRole(context.Background(), AssignNodeRoleInput{
|
|
ActorUserID: "admin-1",
|
|
ClusterID: "cluster-1",
|
|
NodeID: "node-1",
|
|
Role: "rdp-worker",
|
|
})
|
|
if !errors.Is(err, ErrClusterReadOnly) {
|
|
t.Fatalf("err = %v, want ErrClusterReadOnly", err)
|
|
}
|
|
}
|
|
|
|
func TestRecoveryAdminCanMutateReadOnlyCluster(t *testing.T) {
|
|
store := &fakeRepository{
|
|
platformRole: PlatformRoleRecoveryAdmin,
|
|
authorityState: ClusterAuthorityState{
|
|
ClusterID: "cluster-1",
|
|
AuthorityState: "isolated",
|
|
MutationMode: "read_only",
|
|
},
|
|
}
|
|
service := NewService(store)
|
|
|
|
_, err := service.AssignNodeRole(context.Background(), AssignNodeRoleInput{
|
|
ActorUserID: "recovery-1",
|
|
ClusterID: "cluster-1",
|
|
NodeID: "node-1",
|
|
Role: "rdp-worker",
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("recovery admin mutate: %v", err)
|
|
}
|
|
}
|
|
|
|
func TestCreateVPNConnectionRequiresPlatformAdmin(t *testing.T) {
|
|
store := &fakeRepository{platformRole: "user"}
|
|
service := NewService(store)
|
|
|
|
_, err := service.CreateVPNConnection(context.Background(), CreateVPNConnectionInput{
|
|
ActorUserID: "user-1",
|
|
ClusterID: "cluster-1",
|
|
OrganizationID: "org-1",
|
|
Name: "office-a",
|
|
})
|
|
if !errors.Is(err, ErrAccessDenied) {
|
|
t.Fatalf("err = %v, want ErrAccessDenied", err)
|
|
}
|
|
}
|
|
|
|
func TestCreateVPNConnectionDefaultsToDisabledSingleActive(t *testing.T) {
|
|
store := &fakeRepository{platformRole: PlatformRoleAdmin}
|
|
service := NewService(store)
|
|
|
|
item, err := service.CreateVPNConnection(context.Background(), CreateVPNConnectionInput{
|
|
ActorUserID: "admin-1",
|
|
ClusterID: "cluster-1",
|
|
OrganizationID: "org-1",
|
|
Name: "office-a",
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("create vpn connection: %v", err)
|
|
}
|
|
if item.Mode != VPNConnectionModeSingleActive || item.DesiredState != VPNConnectionDesiredDisabled {
|
|
t.Fatalf("unexpected defaults: %+v", item)
|
|
}
|
|
if string(store.lastVPNConnectionInput.AllowedNodePolicy) == "" || string(store.lastVPNConnectionInput.RoutingUsage) == "" {
|
|
t.Fatalf("expected default json policies, got %+v", store.lastVPNConnectionInput)
|
|
}
|
|
}
|
|
|
|
func TestCreateVPNConnectionRequiresClusterAndOrganizationScope(t *testing.T) {
|
|
store := &fakeRepository{platformRole: PlatformRoleAdmin}
|
|
service := NewService(store)
|
|
|
|
_, err := service.CreateVPNConnection(context.Background(), CreateVPNConnectionInput{
|
|
ActorUserID: "admin-1",
|
|
ClusterID: "cluster-1",
|
|
Name: "office-a",
|
|
})
|
|
if !errors.Is(err, ErrInvalidPayload) {
|
|
t.Fatalf("err = %v, want ErrInvalidPayload", err)
|
|
}
|
|
}
|
|
|
|
func TestCreateVPNConnectionBlockedInReadOnlyCluster(t *testing.T) {
|
|
store := &fakeRepository{
|
|
platformRole: PlatformRoleAdmin,
|
|
authorityState: ClusterAuthorityState{
|
|
ClusterID: "cluster-1",
|
|
AuthorityState: "minority",
|
|
MutationMode: "read_only",
|
|
},
|
|
}
|
|
service := NewService(store)
|
|
|
|
_, err := service.CreateVPNConnection(context.Background(), CreateVPNConnectionInput{
|
|
ActorUserID: "admin-1",
|
|
ClusterID: "cluster-1",
|
|
OrganizationID: "org-1",
|
|
Name: "office-a",
|
|
})
|
|
if !errors.Is(err, ErrClusterReadOnly) {
|
|
t.Fatalf("err = %v, want ErrClusterReadOnly", err)
|
|
}
|
|
}
|
|
|
|
func TestAcquireVPNLeaseRequiresEnabledConnection(t *testing.T) {
|
|
store := &fakeRepository{
|
|
platformRole: PlatformRoleAdmin,
|
|
vpnConnection: VPNConnection{
|
|
ID: "vpn-1",
|
|
ClusterID: "cluster-1",
|
|
Mode: VPNConnectionModeSingleActive,
|
|
DesiredState: VPNConnectionDesiredDisabled,
|
|
},
|
|
}
|
|
service := NewService(store)
|
|
|
|
_, err := service.AcquireVPNConnectionLease(context.Background(), AcquireVPNConnectionLeaseInput{
|
|
ActorUserID: "admin-1",
|
|
ClusterID: "cluster-1",
|
|
VPNConnectionID: "vpn-1",
|
|
OwnerNodeID: "node-1",
|
|
})
|
|
if err == nil || !strings.Contains(err.Error(), "enabled single_active") {
|
|
t.Fatalf("err = %v, want enabled single_active validation", err)
|
|
}
|
|
}
|
|
|
|
func TestAcquireVPNLeaseRejectsSecondActiveOwner(t *testing.T) {
|
|
store := &fakeRepository{
|
|
platformRole: PlatformRoleAdmin,
|
|
vpnConnection: VPNConnection{
|
|
ID: "vpn-1",
|
|
ClusterID: "cluster-1",
|
|
Mode: VPNConnectionModeSingleActive,
|
|
DesiredState: VPNConnectionDesiredEnabled,
|
|
},
|
|
acquireVPNLeaseErr: ErrVPNLeaseAlreadyActive,
|
|
}
|
|
service := NewService(store)
|
|
|
|
_, err := service.AcquireVPNConnectionLease(context.Background(), AcquireVPNConnectionLeaseInput{
|
|
ActorUserID: "admin-1",
|
|
ClusterID: "cluster-1",
|
|
VPNConnectionID: "vpn-1",
|
|
OwnerNodeID: "node-2",
|
|
})
|
|
if !errors.Is(err, ErrVPNLeaseAlreadyActive) {
|
|
t.Fatalf("err = %v, want ErrVPNLeaseAlreadyActive", err)
|
|
}
|
|
}
|
|
|
|
func TestAcquireVPNLeaseRejectsOwnerOutsideAllowedPolicy(t *testing.T) {
|
|
store := &fakeRepository{
|
|
platformRole: PlatformRoleAdmin,
|
|
vpnConnection: VPNConnection{
|
|
ID: "vpn-1",
|
|
ClusterID: "cluster-1",
|
|
Mode: VPNConnectionModeSingleActive,
|
|
DesiredState: VPNConnectionDesiredEnabled,
|
|
},
|
|
ownerEligibility: VPNLeaseOwnerEligibility{
|
|
VPNConnectionID: "vpn-1",
|
|
ClusterID: "cluster-1",
|
|
OwnerNodeID: "node-1",
|
|
MembershipStatus: "active",
|
|
NodeRegistrationStatus: NodeRegistrationActive,
|
|
AllowedByPolicy: false,
|
|
HasAuthorizedRole: true,
|
|
},
|
|
}
|
|
service := NewService(store)
|
|
|
|
_, err := service.AcquireVPNConnectionLease(context.Background(), AcquireVPNConnectionLeaseInput{
|
|
ActorUserID: "admin-1",
|
|
ClusterID: "cluster-1",
|
|
VPNConnectionID: "vpn-1",
|
|
OwnerNodeID: "node-1",
|
|
})
|
|
if !errors.Is(err, ErrVPNLeaseOwnerNotAllowed) {
|
|
t.Fatalf("err = %v, want ErrVPNLeaseOwnerNotAllowed", err)
|
|
}
|
|
}
|
|
|
|
func TestAcquireVPNLeaseRejectsOwnerWithoutVPNRole(t *testing.T) {
|
|
store := &fakeRepository{
|
|
platformRole: PlatformRoleAdmin,
|
|
vpnConnection: VPNConnection{
|
|
ID: "vpn-1",
|
|
ClusterID: "cluster-1",
|
|
Mode: VPNConnectionModeSingleActive,
|
|
DesiredState: VPNConnectionDesiredEnabled,
|
|
},
|
|
ownerEligibility: VPNLeaseOwnerEligibility{
|
|
VPNConnectionID: "vpn-1",
|
|
ClusterID: "cluster-1",
|
|
OwnerNodeID: "node-1",
|
|
MembershipStatus: "active",
|
|
NodeRegistrationStatus: NodeRegistrationActive,
|
|
AllowedByPolicy: true,
|
|
HasAuthorizedRole: false,
|
|
},
|
|
}
|
|
service := NewService(store)
|
|
|
|
_, err := service.AcquireVPNConnectionLease(context.Background(), AcquireVPNConnectionLeaseInput{
|
|
ActorUserID: "admin-1",
|
|
ClusterID: "cluster-1",
|
|
VPNConnectionID: "vpn-1",
|
|
OwnerNodeID: "node-1",
|
|
})
|
|
if !errors.Is(err, ErrVPNLeaseOwnerRoleRequired) {
|
|
t.Fatalf("err = %v, want ErrVPNLeaseOwnerRoleRequired", err)
|
|
}
|
|
}
|
|
|
|
func TestAcquireVPNLeaseRejectsWrongCluster(t *testing.T) {
|
|
store := &fakeRepository{
|
|
platformRole: PlatformRoleAdmin,
|
|
vpnConnection: VPNConnection{
|
|
ID: "vpn-1",
|
|
ClusterID: "cluster-1",
|
|
Mode: VPNConnectionModeSingleActive,
|
|
DesiredState: VPNConnectionDesiredEnabled,
|
|
},
|
|
ownerEligibilityErr: pgx.ErrNoRows,
|
|
}
|
|
service := NewService(store)
|
|
|
|
_, err := service.AcquireVPNConnectionLease(context.Background(), AcquireVPNConnectionLeaseInput{
|
|
ActorUserID: "admin-1",
|
|
ClusterID: "cluster-other",
|
|
VPNConnectionID: "vpn-1",
|
|
OwnerNodeID: "node-1",
|
|
})
|
|
if !errors.Is(err, ErrInvalidVPNConnection) {
|
|
t.Fatalf("err = %v, want ErrInvalidVPNConnection", err)
|
|
}
|
|
}
|
|
|
|
func TestRenewVPNLeaseRejectsExpiredLease(t *testing.T) {
|
|
store := &fakeRepository{
|
|
platformRole: PlatformRoleAdmin,
|
|
ownerEligibility: VPNLeaseOwnerEligibility{
|
|
VPNConnectionID: "vpn-1",
|
|
ClusterID: "cluster-1",
|
|
OwnerNodeID: "node-1",
|
|
MembershipStatus: "active",
|
|
NodeRegistrationStatus: NodeRegistrationActive,
|
|
AllowedByPolicy: true,
|
|
HasAuthorizedRole: true,
|
|
},
|
|
renewVPNLeaseErr: pgx.ErrNoRows,
|
|
}
|
|
service := NewService(store)
|
|
|
|
_, err := service.RenewVPNConnectionLease(context.Background(), RenewVPNConnectionLeaseInput{
|
|
ActorUserID: "admin-1",
|
|
ClusterID: "cluster-1",
|
|
VPNConnectionID: "vpn-1",
|
|
LeaseID: "lease-1",
|
|
OwnerNodeID: "node-1",
|
|
FencingToken: "token-1",
|
|
})
|
|
if !errors.Is(err, ErrInvalidVPNLease) {
|
|
t.Fatalf("err = %v, want ErrInvalidVPNLease", err)
|
|
}
|
|
}
|
|
|
|
func TestFenceVPNLeaseRequiresRecoveryAdmin(t *testing.T) {
|
|
store := &fakeRepository{platformRole: PlatformRoleAdmin}
|
|
service := NewService(store)
|
|
|
|
_, err := service.FenceVPNConnectionLease(context.Background(), FenceVPNConnectionLeaseInput{
|
|
ActorUserID: "admin-1",
|
|
ClusterID: "cluster-1",
|
|
VPNConnectionID: "vpn-1",
|
|
LeaseID: "lease-1",
|
|
})
|
|
if !errors.Is(err, ErrAccessDenied) {
|
|
t.Fatalf("err = %v, want ErrAccessDenied", err)
|
|
}
|
|
}
|
|
|
|
func TestExpireStaleVPNConnectionLeasesAuditsEachExpiredLease(t *testing.T) {
|
|
store := &fakeRepository{
|
|
platformRole: PlatformRoleAdmin,
|
|
expiredVPNLeases: []VPNConnectionLease{
|
|
{ID: "lease-1", ClusterID: "cluster-1", VPNConnectionID: "vpn-1", Status: VPNLeaseStatusExpired},
|
|
{ID: "lease-2", ClusterID: "cluster-1", VPNConnectionID: "vpn-2", Status: VPNLeaseStatusExpired},
|
|
},
|
|
}
|
|
service := NewService(store)
|
|
|
|
items, err := service.ExpireStaleVPNConnectionLeases(context.Background(), ExpireStaleVPNConnectionLeasesInput{
|
|
ActorUserID: "admin-1",
|
|
ClusterID: "cluster-1",
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("expire stale vpn leases: %v", err)
|
|
}
|
|
if got, want := len(items), 2; got != want {
|
|
t.Fatalf("expired leases = %d, want %d", got, want)
|
|
}
|
|
var auditCount int
|
|
for _, event := range store.auditEvents {
|
|
if event.EventType == "vpn_connection.lease_expired" {
|
|
auditCount++
|
|
}
|
|
}
|
|
if got, want := auditCount, 2; got != want {
|
|
t.Fatalf("lease_expired audit count = %d, want %d", got, want)
|
|
}
|
|
}
|
|
|
|
func TestSetVPNConnectionAllowedNodesDeduplicatesScope(t *testing.T) {
|
|
store := &fakeRepository{platformRole: PlatformRoleAdmin}
|
|
service := NewService(store)
|
|
|
|
items, err := service.SetVPNConnectionAllowedNodes(context.Background(), SetVPNConnectionAllowedNodesInput{
|
|
ActorUserID: "admin-1",
|
|
ClusterID: "cluster-1",
|
|
VPNConnectionID: "vpn-1",
|
|
NodeIDs: []string{"node-1", "node-1", " ", "node-2"},
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("set allowed nodes: %v", err)
|
|
}
|
|
if got, want := len(store.lastAllowedNodesInput.NodeIDs), 2; got != want {
|
|
t.Fatalf("deduped nodes = %d, want %d", got, want)
|
|
}
|
|
if got, want := len(items), 2; got != want {
|
|
t.Fatalf("allowed nodes returned = %d, want %d", got, want)
|
|
}
|
|
}
|
|
|
|
func TestUpsertVPNRoutePolicyRejectsInvalidType(t *testing.T) {
|
|
store := &fakeRepository{platformRole: PlatformRoleAdmin}
|
|
service := NewService(store)
|
|
|
|
_, err := service.UpsertVPNConnectionRoutePolicy(context.Background(), UpsertVPNConnectionRoutePolicyInput{
|
|
ActorUserID: "admin-1",
|
|
ClusterID: "cluster-1",
|
|
VPNConnectionID: "vpn-1",
|
|
RouteType: "submarine",
|
|
Destination: "10.0.0.0/24",
|
|
})
|
|
if !errors.Is(err, ErrInvalidPayload) {
|
|
t.Fatalf("err = %v, want ErrInvalidPayload", err)
|
|
}
|
|
}
|
|
|
|
func TestListNodeVPNAssignmentsDoesNotRequirePlatformAdmin(t *testing.T) {
|
|
store := &fakeRepository{
|
|
platformRole: "user",
|
|
nodeVPNAssignments: []NodeVPNAssignment{
|
|
{VPNConnectionID: "vpn-1", ClusterID: "cluster-1", OrganizationID: "org-1", AssignmentReason: "eligible_candidate"},
|
|
},
|
|
}
|
|
service := NewService(store)
|
|
|
|
items, err := service.ListNodeVPNAssignments(context.Background(), "cluster-1", "node-1")
|
|
if err != nil {
|
|
t.Fatalf("list node vpn assignments: %v", err)
|
|
}
|
|
if got, want := len(items), 1; got != want {
|
|
t.Fatalf("assignments = %d, want %d", got, want)
|
|
}
|
|
}
|
|
|
|
func TestRenewNodeVPNAssignmentLeaseAllowsActiveOwnerWithoutPlatformAdmin(t *testing.T) {
|
|
store := &fakeRepository{
|
|
platformRole: "user",
|
|
nodeVPNAssignments: []NodeVPNAssignment{
|
|
{
|
|
VPNConnectionID: "vpn-1",
|
|
ClusterID: "cluster-1",
|
|
OrganizationID: "org-1",
|
|
AssignmentReason: "active_owner",
|
|
ActiveLease: &NodeVPNAssignmentLease{
|
|
LeaseID: "lease-1",
|
|
OwnerNodeID: "node-1",
|
|
Status: VPNLeaseStatusActive,
|
|
},
|
|
},
|
|
},
|
|
}
|
|
service := NewService(store)
|
|
|
|
lease, err := service.RenewNodeVPNAssignmentLease(context.Background(), RenewNodeVPNAssignmentLeaseInput{
|
|
ClusterID: "cluster-1",
|
|
VPNConnectionID: "vpn-1",
|
|
LeaseID: "lease-1",
|
|
OwnerNodeID: "node-1",
|
|
TTL: time.Minute,
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("renew node vpn assignment lease: %v", err)
|
|
}
|
|
if lease.ID != "lease-1" {
|
|
t.Fatalf("lease.ID = %q, want lease-1", lease.ID)
|
|
}
|
|
}
|
|
|
|
func TestRenewNodeVPNAssignmentLeaseRejectsNonOwner(t *testing.T) {
|
|
store := &fakeRepository{
|
|
nodeVPNAssignments: []NodeVPNAssignment{
|
|
{VPNConnectionID: "vpn-1", ClusterID: "cluster-1", OrganizationID: "org-1", AssignmentReason: "eligible_candidate"},
|
|
},
|
|
}
|
|
service := NewService(store)
|
|
|
|
_, err := service.RenewNodeVPNAssignmentLease(context.Background(), RenewNodeVPNAssignmentLeaseInput{
|
|
ClusterID: "cluster-1",
|
|
VPNConnectionID: "vpn-1",
|
|
LeaseID: "lease-1",
|
|
OwnerNodeID: "node-1",
|
|
})
|
|
if !errors.Is(err, ErrVPNLeaseOwnerNotAllowed) {
|
|
t.Fatalf("err = %v, want ErrVPNLeaseOwnerNotAllowed", err)
|
|
}
|
|
}
|
|
|
|
func TestReportNodeVPNAssignmentStatusRejectsInvisibleAssignment(t *testing.T) {
|
|
store := &fakeRepository{}
|
|
service := NewService(store)
|
|
|
|
_, err := service.ReportNodeVPNAssignmentStatus(context.Background(), ReportNodeVPNAssignmentStatusInput{
|
|
ClusterID: "cluster-1",
|
|
NodeID: "node-1",
|
|
VPNConnectionID: "vpn-foreign",
|
|
ObservedStatus: VPNAssignmentStatusAssigned,
|
|
})
|
|
if !errors.Is(err, ErrVPNLeaseOwnerNotAllowed) {
|
|
t.Fatalf("err = %v, want ErrVPNLeaseOwnerNotAllowed", err)
|
|
}
|
|
}
|
|
|
|
func TestReportNodeVPNAssignmentStatusAcceptsExplicitStates(t *testing.T) {
|
|
store := &fakeRepository{
|
|
nodeVPNAssignments: []NodeVPNAssignment{
|
|
{VPNConnectionID: "vpn-1", ClusterID: "cluster-1", OrganizationID: "org-1"},
|
|
},
|
|
}
|
|
service := NewService(store)
|
|
|
|
status, err := service.ReportNodeVPNAssignmentStatus(context.Background(), ReportNodeVPNAssignmentStatusInput{
|
|
ClusterID: "cluster-1",
|
|
NodeID: "node-1",
|
|
VPNConnectionID: "vpn-1",
|
|
ObservedStatus: VPNAssignmentStatusLeaseRequired,
|
|
StatusPayload: json.RawMessage(`{"reason":"no_lease"}`),
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("report node vpn assignment status: %v", err)
|
|
}
|
|
if status.ObservedStatus != VPNAssignmentStatusLeaseRequired {
|
|
t.Fatalf("ObservedStatus = %q, want %q", status.ObservedStatus, VPNAssignmentStatusLeaseRequired)
|
|
}
|
|
}
|
|
|
|
func TestReportNodeVPNAssignmentStatusRejectsInvalidStatus(t *testing.T) {
|
|
store := &fakeRepository{
|
|
nodeVPNAssignments: []NodeVPNAssignment{
|
|
{VPNConnectionID: "vpn-1", ClusterID: "cluster-1", OrganizationID: "org-1"},
|
|
},
|
|
}
|
|
service := NewService(store)
|
|
|
|
_, err := service.ReportNodeVPNAssignmentStatus(context.Background(), ReportNodeVPNAssignmentStatusInput{
|
|
ClusterID: "cluster-1",
|
|
NodeID: "node-1",
|
|
VPNConnectionID: "vpn-1",
|
|
ObservedStatus: "running_tunnel",
|
|
})
|
|
if !errors.Is(err, ErrInvalidPayload) {
|
|
t.Fatalf("err = %v, want ErrInvalidPayload", err)
|
|
}
|
|
}
|
|
|
|
func TestIssueFabricServiceChannelLeaseSelectsAuthorizedRoute(t *testing.T) {
|
|
now := time.Date(2026, 5, 7, 12, 0, 0, 0, time.UTC)
|
|
service := NewService(&fakeRepository{
|
|
routeIntents: []MeshRouteIntent{
|
|
{
|
|
ID: "route-usa-home",
|
|
ClusterID: "cluster-1",
|
|
SourceSelector: json.RawMessage(`{"node_id":"usa-los-1"}`),
|
|
DestinationSelector: json.RawMessage(`{"node_id":"home-1"}`),
|
|
ServiceClass: FabricServiceClassVPNPackets,
|
|
Priority: 20,
|
|
Status: "active",
|
|
Policy: json.RawMessage(`{
|
|
"synthetic_enabled": true,
|
|
"hops": ["usa-los-1", "relay-1", "home-1"],
|
|
"allowed_channels": ["vpn_packet", "fabric_control"],
|
|
"route_version": "rv-1",
|
|
"policy_version": "pv-1"
|
|
}`),
|
|
UpdatedAt: now,
|
|
},
|
|
{
|
|
ID: "route-home-home",
|
|
ClusterID: "cluster-1",
|
|
SourceSelector: json.RawMessage(`{"node_id":"home-1"}`),
|
|
DestinationSelector: json.RawMessage(`{"node_id":"home-1"}`),
|
|
ServiceClass: FabricServiceClassVPNPackets,
|
|
Priority: 5,
|
|
Status: "active",
|
|
Policy: json.RawMessage(`{
|
|
"synthetic_enabled": true,
|
|
"hops": ["home-1", "home-1"],
|
|
"allowed_channels": ["vpn_packet", "fabric_control"]
|
|
}`),
|
|
UpdatedAt: now,
|
|
},
|
|
},
|
|
})
|
|
service.now = func() time.Time { return now }
|
|
|
|
lease, err := service.IssueFabricServiceChannelLease(context.Background(), IssueFabricServiceChannelLeaseInput{
|
|
ClusterID: "cluster-1",
|
|
OrganizationID: "org-home",
|
|
UserID: "user-m",
|
|
ResourceID: "vpn-home",
|
|
ServiceClass: FabricServiceClassVPNPackets,
|
|
EntryNodeIDs: []string{"home-1", "usa-los-1"},
|
|
ExitNodeIDs: []string{"home-1", "ifcm-1"},
|
|
PreferredEntryNodeID: "usa-los-1",
|
|
PreferredExitNodeID: "home-1",
|
|
TTL: 90 * time.Second,
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("issue lease: %v", err)
|
|
}
|
|
if lease.Status != FabricServiceChannelStatusReady {
|
|
t.Fatalf("lease.Status = %q, want ready", lease.Status)
|
|
}
|
|
if lease.SelectedEntryNodeID != "usa-los-1" || lease.SelectedExitNodeID != "home-1" {
|
|
t.Fatalf("selected nodes = %s -> %s", lease.SelectedEntryNodeID, lease.SelectedExitNodeID)
|
|
}
|
|
if lease.PrimaryRoute.RouteID != "route-usa-home" {
|
|
t.Fatalf("primary route = %q, want route-usa-home", lease.PrimaryRoute.RouteID)
|
|
}
|
|
if lease.RecoveryPolicy == nil || lease.RecoveryPolicy.HysteresisPenalty != fabricServiceChannelRecoveryHysteresisPenalty {
|
|
t.Fatalf("lease recovery policy provenance = %+v", lease.RecoveryPolicy)
|
|
}
|
|
if lease.PrimaryRoute.RecoveryPolicy == nil || lease.PrimaryRoute.RecoveryPolicy.PromotionMinSamples != fabricServiceChannelRecoveryPromotionMinSamples {
|
|
t.Fatalf("primary route recovery policy provenance = %+v", lease.PrimaryRoute.RecoveryPolicy)
|
|
}
|
|
if lease.Fallback.Active || lease.Fallback.Degraded {
|
|
t.Fatalf("fallback should be available but inactive: %+v", lease.Fallback)
|
|
}
|
|
if !containsString(lease.AllowedChannels, "vpn_packet") || !containsString(lease.RequiredRoles, "vpn-exit") {
|
|
t.Fatalf("unexpected channel/role defaults: channels=%v roles=%v", lease.AllowedChannels, lease.RequiredRoles)
|
|
}
|
|
if lease.Token.Token == "" || lease.Token.TTLSeconds != 90 {
|
|
t.Fatalf("unexpected token contract: %+v", lease.Token)
|
|
}
|
|
if lease.EntryHTTP.PathTemplate == "" || lease.EntryHTTP.WebSocketPathTemplate == "" {
|
|
t.Fatalf("entry http contract must include packet endpoints: %+v", lease.EntryHTTP)
|
|
}
|
|
if lease.DataPlane.SchemaVersion != "rap.fabric_service_channel_data_plane.v1" ||
|
|
lease.DataPlane.Mode != "fabric_primary" ||
|
|
lease.DataPlane.WorkingDataTransport != "fabric_service_channel" ||
|
|
lease.DataPlane.SteadyStateTransport != "fabric_route" ||
|
|
lease.DataPlane.BackendRelayPolicy != "degraded_fallback_only" ||
|
|
!lease.DataPlane.ProductionForwardingRequired ||
|
|
!lease.DataPlane.ServiceNeutral ||
|
|
!lease.DataPlane.ProtocolAgnostic ||
|
|
lease.DataPlane.LogicalFlowMode != "multi_flow_isolated" ||
|
|
!containsString(lease.DataPlane.RequiredFlowIsolationClasses, "vpn_packet") {
|
|
t.Fatalf("unexpected data-plane contract: %+v", lease.DataPlane)
|
|
}
|
|
if lease.AuthoritySignature == nil || len(lease.AuthorityPayload) == 0 {
|
|
t.Fatalf("lease must be signed: payload=%s signature=%+v", string(lease.AuthorityPayload), lease.AuthoritySignature)
|
|
}
|
|
var signedPayload FabricServiceChannelLeaseAuthorityPayload
|
|
if err := json.Unmarshal(lease.AuthorityPayload, &signedPayload); err != nil {
|
|
t.Fatalf("unmarshal signed payload: %v", err)
|
|
}
|
|
if signedPayload.TokenHash != fabricServiceChannelTokenHash(lease.Token.Token) || signedPayload.ChannelID != lease.ChannelID {
|
|
t.Fatalf("signed payload does not bind token/channel: %+v", signedPayload)
|
|
}
|
|
if signedPayload.RecoveryPolicy == nil || signedPayload.RecoveryPolicy.Source != "defaults" {
|
|
t.Fatalf("signed payload recovery policy provenance = %+v", signedPayload.RecoveryPolicy)
|
|
}
|
|
if signedPayload.DataPlane.SchemaVersion != lease.DataPlane.SchemaVersion ||
|
|
signedPayload.DataPlane.WorkingDataTransport != "fabric_service_channel" ||
|
|
signedPayload.DataPlane.BackendRelayPolicy != "degraded_fallback_only" {
|
|
t.Fatalf("signed payload data-plane contract = %+v", signedPayload.DataPlane)
|
|
}
|
|
store := service.store.(*fakeRepository)
|
|
if err := clusterauth.VerifyRaw(store.clusterAuthority.PublicKey, lease.AuthorityPayload, *lease.AuthoritySignature); err != nil {
|
|
t.Fatalf("verify lease authority: %v", err)
|
|
}
|
|
}
|
|
|
|
func TestFabricServiceChannelLeaseIntrospectionAllowsFreshToken(t *testing.T) {
|
|
store := &fakeRepository{}
|
|
service := NewService(store)
|
|
service.now = func() time.Time { return time.Date(2026, 5, 8, 14, 0, 0, 0, time.UTC) }
|
|
store.routeIntents = []MeshRouteIntent{{
|
|
ID: "route-usa-home",
|
|
ClusterID: "cluster-1",
|
|
ServiceClass: FabricServiceClassVPNPackets,
|
|
Status: "active",
|
|
Policy: json.RawMessage(`{
|
|
"schema_version":"rap.synthetic_route_policy.v1",
|
|
"source_node_id":"usa-1",
|
|
"destination_node_id":"home-1",
|
|
"hops":["usa-1","home-1"],
|
|
"allowed_channels":["vpn_packet","fabric_control"],
|
|
"synthetic_enabled":true
|
|
}`),
|
|
CreatedAt: service.now(),
|
|
UpdatedAt: service.now(),
|
|
}}
|
|
lease, err := service.IssueFabricServiceChannelLease(context.Background(), IssueFabricServiceChannelLeaseInput{
|
|
ActorUserID: "admin-1",
|
|
ClusterID: "cluster-1",
|
|
OrganizationID: "org-1",
|
|
UserID: "user-1",
|
|
ResourceID: "vpn-1",
|
|
ServiceClass: FabricServiceClassVPNPackets,
|
|
EntryNodeIDs: []string{"usa-1"},
|
|
ExitNodeIDs: []string{"home-1"},
|
|
AllowedChannels: []string{
|
|
"vpn_packet",
|
|
FabricChannelControl,
|
|
},
|
|
TTL: 90 * time.Second,
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("issue lease: %v", err)
|
|
}
|
|
result, err := service.IntrospectFabricServiceChannelLease(context.Background(), IntrospectFabricServiceChannelLeaseInput{
|
|
ClusterID: "cluster-1",
|
|
ChannelID: lease.ChannelID,
|
|
ResourceID: "vpn-1",
|
|
ServiceClass: FabricServiceClassVPNPackets,
|
|
ChannelClass: "vpn_packet",
|
|
Token: lease.Token.Token,
|
|
EntryNodeID: "usa-1",
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("introspect lease: %v", err)
|
|
}
|
|
if !result.Allowed || result.AcceptedBy != "introspection" || result.PreferredRouteID != "route-usa-home" || result.ForceBackendFallback {
|
|
t.Fatalf("unexpected introspection result: %+v", result)
|
|
}
|
|
if result.DataPlane.SchemaVersion != "rap.fabric_service_channel_data_plane.v1" ||
|
|
result.DataPlane.WorkingDataTransport != "fabric_service_channel" ||
|
|
result.DataPlane.SteadyStateTransport != "fabric_route" ||
|
|
result.DataPlane.BackendRelayPolicy != "degraded_fallback_only" {
|
|
t.Fatalf("unexpected introspection data-plane contract: %+v", result.DataPlane)
|
|
}
|
|
}
|
|
|
|
func TestFabricServiceChannelLeaseIntrospectionRejectsWrongToken(t *testing.T) {
|
|
store := &fakeRepository{}
|
|
service := NewService(store)
|
|
service.now = func() time.Time { return time.Date(2026, 5, 8, 14, 0, 0, 0, time.UTC) }
|
|
lease, err := service.IssueFabricServiceChannelLease(context.Background(), IssueFabricServiceChannelLeaseInput{
|
|
ActorUserID: "admin-1",
|
|
ClusterID: "cluster-1",
|
|
OrganizationID: "org-1",
|
|
UserID: "user-1",
|
|
ResourceID: "vpn-1",
|
|
ServiceClass: FabricServiceClassVPNPackets,
|
|
EntryNodeIDs: []string{"usa-1"},
|
|
ExitNodeIDs: []string{"home-1"},
|
|
TTL: 90 * time.Second,
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("issue lease: %v", err)
|
|
}
|
|
result, err := service.IntrospectFabricServiceChannelLease(context.Background(), IntrospectFabricServiceChannelLeaseInput{
|
|
ClusterID: "cluster-1",
|
|
ChannelID: lease.ChannelID,
|
|
ResourceID: "vpn-1",
|
|
ServiceClass: FabricServiceClassVPNPackets,
|
|
ChannelClass: "vpn_packet",
|
|
Token: "rap_fsc_wrong",
|
|
EntryNodeID: "usa-1",
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("introspect lease: %v", err)
|
|
}
|
|
if result.Allowed || result.Reason != "lease_token_mismatch" {
|
|
t.Fatalf("unexpected introspection result: %+v", result)
|
|
}
|
|
}
|
|
|
|
func TestFabricServiceChannelLeaseIntrospectionSurvivesServiceRestart(t *testing.T) {
|
|
store := &fakeRepository{}
|
|
now := time.Date(2026, 5, 8, 14, 30, 0, 0, time.UTC)
|
|
service := NewService(store)
|
|
service.now = func() time.Time { return now }
|
|
lease, err := service.IssueFabricServiceChannelLease(context.Background(), IssueFabricServiceChannelLeaseInput{
|
|
ActorUserID: "admin-1",
|
|
ClusterID: "cluster-1",
|
|
OrganizationID: "org-1",
|
|
UserID: "user-1",
|
|
ResourceID: "vpn-1",
|
|
ServiceClass: FabricServiceClassVPNPackets,
|
|
EntryNodeIDs: []string{"usa-1"},
|
|
ExitNodeIDs: []string{"home-1"},
|
|
TTL: 90 * time.Second,
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("issue lease: %v", err)
|
|
}
|
|
|
|
restarted := NewService(store)
|
|
restarted.now = func() time.Time { return now.Add(5 * time.Second) }
|
|
result, err := restarted.IntrospectFabricServiceChannelLease(context.Background(), IntrospectFabricServiceChannelLeaseInput{
|
|
ClusterID: "cluster-1",
|
|
ChannelID: lease.ChannelID,
|
|
ResourceID: "vpn-1",
|
|
ServiceClass: FabricServiceClassVPNPackets,
|
|
ChannelClass: "vpn_packet",
|
|
Token: lease.Token.Token,
|
|
EntryNodeID: "usa-1",
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("introspect lease after restart: %v", err)
|
|
}
|
|
if !result.Allowed || result.Reason != "lease_introspection_allowed" {
|
|
t.Fatalf("unexpected introspection result: %+v", result)
|
|
}
|
|
if stored := store.fabricLeases[fabricServiceChannelLeaseCacheKey("cluster-1", lease.ChannelID)]; stored.Lease.Token.Token != "" {
|
|
t.Fatalf("stored durable lease must not include raw bearer token: %+v", stored.Lease.Token)
|
|
}
|
|
}
|
|
|
|
func TestFabricServiceChannelLeaseMaintenanceListsAndCleansExpired(t *testing.T) {
|
|
store := &fakeRepository{platformRole: PlatformRoleAdmin}
|
|
now := time.Date(2026, 5, 8, 15, 0, 0, 0, time.UTC)
|
|
activeLease := FabricServiceChannelLease{
|
|
ChannelID: "channel-active",
|
|
ClusterID: "cluster-1",
|
|
ResourceID: "vpn-active",
|
|
ServiceClass: FabricServiceClassVPNPackets,
|
|
Status: FabricServiceChannelStatusReady,
|
|
SelectedEntryNodeID: "entry-1",
|
|
SelectedExitNodeID: "exit-1",
|
|
AllowedChannels: []string{"vpn_packet"},
|
|
PrimaryRoute: FabricServiceChannelRoute{RouteID: "route-active", Status: "ready"},
|
|
Token: FabricServiceChannelToken{Token: "rap_fsc_active"},
|
|
IssuedAt: now.Add(-time.Minute),
|
|
ExpiresAt: now.Add(time.Minute),
|
|
}
|
|
expiredLease := activeLease
|
|
expiredLease.ChannelID = "channel-expired"
|
|
expiredLease.ResourceID = "vpn-expired"
|
|
expiredLease.Token = FabricServiceChannelToken{Token: "rap_fsc_expired"}
|
|
expiredLease.ExpiresAt = now.Add(-time.Second)
|
|
if _, err := store.StoreFabricServiceChannelLease(context.Background(), StoreFabricServiceChannelLeaseInput{Lease: activeLease, TokenHash: fabricServiceChannelTokenHash(activeLease.Token.Token)}); err != nil {
|
|
t.Fatalf("store active lease: %v", err)
|
|
}
|
|
if _, err := store.StoreFabricServiceChannelLease(context.Background(), StoreFabricServiceChannelLeaseInput{Lease: expiredLease, TokenHash: fabricServiceChannelTokenHash(expiredLease.Token.Token)}); err != nil {
|
|
t.Fatalf("store expired lease: %v", err)
|
|
}
|
|
service := NewService(store)
|
|
service.now = func() time.Time { return now }
|
|
health, err := service.ListFabricServiceChannelLeases(context.Background(), "admin-1", ListFabricServiceChannelLeasesInput{
|
|
ClusterID: "cluster-1",
|
|
IncludeExpired: true,
|
|
Limit: 10,
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("list leases: %v", err)
|
|
}
|
|
if health.ActiveCount != 1 || health.ExpiredCount != 1 || health.Status != "degraded" {
|
|
t.Fatalf("unexpected lease maintenance health: %+v", health)
|
|
}
|
|
cleanup, err := service.CleanupFabricServiceChannelLeases(context.Background(), CleanupFabricServiceChannelLeasesInput{
|
|
ActorUserID: "admin-1",
|
|
ClusterID: "cluster-1",
|
|
Limit: 10,
|
|
Now: now,
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("cleanup leases: %v", err)
|
|
}
|
|
if cleanup.DeletedExpiredCount != 1 || cleanup.ExpiredCount != 0 || cleanup.ActiveCount != 1 || cleanup.Status != "ready" {
|
|
t.Fatalf("unexpected cleanup result: %+v", cleanup)
|
|
}
|
|
}
|
|
|
|
func TestFabricServiceChannelAccessTelemetryAggregatesNodeReports(t *testing.T) {
|
|
now := time.Date(2026, 5, 8, 15, 20, 0, 0, time.UTC)
|
|
store := &fakeRepository{
|
|
platformRole: PlatformRoleAdmin,
|
|
clusterNodes: []ClusterNode{
|
|
{ID: "node-1", Name: "entry-1"},
|
|
{ID: "node-2", Name: "entry-2"},
|
|
},
|
|
heartbeats: map[string][]NodeHeartbeat{
|
|
"node-1": {
|
|
{
|
|
ClusterID: "cluster-1",
|
|
NodeID: "node-1",
|
|
ObservedAt: now.Add(-2 * time.Second),
|
|
Metadata: json.RawMessage(`{
|
|
"fabric_service_channel_runtime_report": {
|
|
"schema_version": "c18z64.fabric_service_channel_runtime_report.v1",
|
|
"ingress": {
|
|
"flow_scheduler": {
|
|
"traffic_class_counts": {"bulk": 32, "interactive": 12},
|
|
"recommended_parallel_windows": {"bulk": 1, "interactive": 4, "control": 4, "reliable": 3, "droppable": 1},
|
|
"adaptive_backpressure_active": true,
|
|
"adaptive_backpressure_reason": "bulk_window_reduced_to_protect_interactive",
|
|
"channel_count": 44,
|
|
"dropped": 0,
|
|
"high_watermark": 25,
|
|
"max_in_flight": 4,
|
|
"channel_stats": {}
|
|
}
|
|
}
|
|
}
|
|
}`),
|
|
},
|
|
},
|
|
},
|
|
}
|
|
service := NewService(store)
|
|
service.now = func() time.Time { return now }
|
|
_, err := store.RecordNodeTelemetry(context.Background(), RecordNodeTelemetryInput{
|
|
ClusterID: "cluster-1",
|
|
NodeID: "node-1",
|
|
Payload: json.RawMessage(`{
|
|
"fabric_service_channel_access_report": {
|
|
"schema_version": "c18z52.fabric_service_channel_access_report.v1",
|
|
"total": 7,
|
|
"signed": 3,
|
|
"introspection": 4,
|
|
"legacy_unsigned": 0,
|
|
"backend_fallback": 2,
|
|
"data_plane_contract": 5,
|
|
"last_data_plane_mode": "fabric_primary",
|
|
"last_working_data_transport": "fabric_service_channel",
|
|
"last_steady_state_transport": "fabric_route",
|
|
"last_backend_relay_policy": "degraded_fallback_only",
|
|
"last_logical_flow_mode": "multi_flow_isolated",
|
|
"last_accepted_at": "2026-05-08T15:19:59Z"
|
|
}
|
|
}`),
|
|
ObservedAt: now,
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("record telemetry: %v", err)
|
|
}
|
|
expiresAt := now.Add(5 * time.Minute)
|
|
store.fabricLeases = map[string]FabricServiceChannelLeaseRecord{
|
|
fabricServiceChannelLeaseCacheKey("cluster-1", "channel-1"): {
|
|
ClusterID: "cluster-1",
|
|
ChannelID: "channel-1",
|
|
ResourceID: "vpn-home",
|
|
ServiceClass: FabricServiceClassVPNPackets,
|
|
SelectedEntryNodeID: "node-1",
|
|
ExpiresAt: expiresAt,
|
|
Lease: FabricServiceChannelLease{
|
|
ClusterID: "cluster-1",
|
|
ChannelID: "channel-1",
|
|
ResourceID: "vpn-home",
|
|
ServiceClass: FabricServiceClassVPNPackets,
|
|
Status: FabricServiceChannelStatusReady,
|
|
SelectedEntryNodeID: "node-1",
|
|
SelectedExitNodeID: "node-2",
|
|
PrimaryRoute: FabricServiceChannelRoute{
|
|
RouteID: "route-1",
|
|
Status: "ready",
|
|
},
|
|
ExpiresAt: expiresAt,
|
|
},
|
|
},
|
|
}
|
|
_, err = store.RecordFabricServiceChannelRouteFeedback(context.Background(), RecordFabricServiceChannelRouteFeedbackInput{
|
|
ClusterID: "cluster-1",
|
|
ReporterNodeID: "node-1",
|
|
RouteID: "route-1",
|
|
ServiceClass: FabricServiceClassVPNPackets,
|
|
FeedbackStatus: "healthy",
|
|
ScoreAdjustment: 15,
|
|
LastSendDurationMs: 42,
|
|
Payload: json.RawMessage(`{"quality_window_sample_count":5,"quality_window_failure_count":0,"quality_window_drop_count":0,"quality_window_slow_count":1}`),
|
|
ObservedAt: now,
|
|
ExpiresAt: expiresAt,
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("record route feedback: %v", err)
|
|
}
|
|
report, err := service.GetFabricServiceChannelAccessTelemetry(context.Background(), "admin-1", GetFabricServiceChannelAccessTelemetryInput{
|
|
ClusterID: "cluster-1",
|
|
Limit: 10,
|
|
Now: now,
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("get access telemetry: %v", err)
|
|
}
|
|
if report.ReportingNodeCount != 1 || report.TotalAccepted != 7 || report.SignedAccepted != 3 || report.IntrospectionAccepted != 4 || report.BackendFallbackCount != 2 {
|
|
t.Fatalf("unexpected access telemetry: %+v", report)
|
|
}
|
|
if report.DataPlaneContractCount != 5 || report.LastDataPlaneMode != "fabric_primary" || report.LastWorkingDataTransport != "fabric_service_channel" || report.LastSteadyStateTransport != "fabric_route" || report.LastBackendRelayPolicy != "degraded_fallback_only" || report.LastLogicalFlowMode != "multi_flow_isolated" {
|
|
t.Fatalf("unexpected aggregate data-plane telemetry: %+v", report)
|
|
}
|
|
if report.Nodes[0].DataPlaneContractCount != 5 || report.Nodes[0].LastWorkingDataTransport != "fabric_service_channel" || report.Nodes[0].LastBackendRelayPolicy != "degraded_fallback_only" || report.Nodes[0].LastLogicalFlowMode != "multi_flow_isolated" {
|
|
t.Fatalf("unexpected node data-plane telemetry: %+v", report.Nodes[0])
|
|
}
|
|
if got := report.Nodes[0].TrafficClassCounts["bulk"]; got != 32 {
|
|
t.Fatalf("bulk traffic class count = %d, want 32: %+v", got, report.Nodes[0])
|
|
}
|
|
if report.TrafficClassCounts["bulk"] != 32 || report.TrafficClassCounts["interactive"] != 12 || report.FlowChannelCount != 44 || report.FlowMaxInFlight != 4 {
|
|
t.Fatalf("unexpected aggregate flow telemetry: %+v", report)
|
|
}
|
|
if report.FlowHealthStatus != "degraded" || report.FlowHealthReason != "backend_fallback_observed" {
|
|
t.Fatalf("unexpected aggregate flow health: %+v", report)
|
|
}
|
|
if !report.AdaptiveBackpressureActive || report.AdaptiveBackpressureReason != "bulk_window_reduced_to_protect_interactive" || report.RecommendedParallelWindows["bulk"] != 1 || report.RecommendedParallelWindows["interactive"] != 4 {
|
|
t.Fatalf("unexpected aggregate adaptive backpressure: %+v", report)
|
|
}
|
|
if report.Nodes[0].FlowChannelCount != 44 || report.Nodes[0].FlowHighWatermark != 25 || report.Nodes[0].FlowMaxInFlight != 4 {
|
|
t.Fatalf("unexpected flow telemetry on node: %+v", report.Nodes[0])
|
|
}
|
|
if report.Nodes[0].FlowHealthStatus != "degraded" || report.Nodes[0].FlowHealthReason != "backend_fallback_observed" {
|
|
t.Fatalf("unexpected node flow health: %+v", report.Nodes[0])
|
|
}
|
|
if !report.Nodes[0].AdaptiveBackpressureActive || report.Nodes[0].RecommendedParallelWindows["control"] != 4 || report.Nodes[0].RecommendedParallelWindows["droppable"] != 1 {
|
|
t.Fatalf("unexpected node adaptive backpressure: %+v", report.Nodes[0])
|
|
}
|
|
if report.ActiveChannelCount != 1 || report.CorrelatedRouteCount != 1 || report.DegradedRouteCount != 0 {
|
|
t.Fatalf("unexpected channel correlation counters: %+v", report)
|
|
}
|
|
if len(report.ActiveChannels) != 1 {
|
|
t.Fatalf("expected one active channel, got %d", len(report.ActiveChannels))
|
|
}
|
|
channel := report.ActiveChannels[0]
|
|
if channel.ChannelID != "channel-1" || channel.EntryNodeTotalAccepted != 7 || channel.RouteFeedbackStatus != "healthy" || channel.RouteQualityWindowSampleCount != 5 || channel.LastSendDurationMs != 42 {
|
|
t.Fatalf("unexpected active channel correlation: %+v", channel)
|
|
}
|
|
if channel.EntryNodeDataPlaneContractCount != 5 || channel.EntryNodeLastDataPlaneMode != "fabric_primary" || channel.EntryNodeLastWorkingDataTransport != "fabric_service_channel" || channel.EntryNodeLastSteadyStateTransport != "fabric_route" || channel.EntryNodeLastBackendRelayPolicy != "degraded_fallback_only" || channel.EntryNodeLastLogicalFlowMode != "multi_flow_isolated" {
|
|
t.Fatalf("unexpected active channel data-plane telemetry: %+v", channel)
|
|
}
|
|
if channel.EntryNodeTrafficClassCounts["interactive"] != 12 || channel.EntryNodeFlowChannelCount != 44 || channel.EntryNodeFlowMaxInFlight != 4 {
|
|
t.Fatalf("unexpected active channel flow telemetry: %+v", channel)
|
|
}
|
|
if channel.EntryNodeFlowHealthStatus != "degraded" || channel.EntryNodeFlowHealthReason != "backend_fallback_observed" {
|
|
t.Fatalf("unexpected channel flow health: %+v", channel)
|
|
}
|
|
if !channel.EntryNodeAdaptiveBackpressureActive || channel.EntryNodeAdaptiveBackpressureReason != "bulk_window_reduced_to_protect_interactive" || channel.EntryNodeRecommendedParallelWindows["bulk"] != 1 {
|
|
t.Fatalf("unexpected channel adaptive backpressure: %+v", channel)
|
|
}
|
|
if channel.RemediationAction != "none" {
|
|
t.Fatalf("healthy route should not need remediation: %+v", channel)
|
|
}
|
|
incidents, err := service.ListFabricServiceChannelRouteRebuildIncidents(context.Background(), "admin-1", ListFabricServiceChannelRouteRebuildIncidentsInput{
|
|
ClusterID: "cluster-1",
|
|
Limit: 10,
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("list rebuild incidents: %v", err)
|
|
}
|
|
if len(incidents) == 0 ||
|
|
incidents[0].IncidentSource != "data_plane_contract" ||
|
|
incidents[0].ChannelID != "channel-1" ||
|
|
incidents[0].GuardStatus != "data_plane_degraded_backend_relay_observed" ||
|
|
incidents[0].GuardSeverity != "warn" ||
|
|
incidents[0].RecommendedOperatorAction != "restore_fabric_route_and_treat_backend_relay_as_degraded_only" {
|
|
t.Fatalf("unexpected data-plane incident projection: %+v", incidents)
|
|
}
|
|
}
|
|
|
|
func TestFabricServiceChannelFlowHealthPolicyClassifiesPressure(t *testing.T) {
|
|
status, reason, action := fabricServiceChannelFlowHealth(map[string]int{"bulk": 32, "interactive": 12}, 0, 25, 4, 0, 132, 0, 0, 0)
|
|
if status != "watch" || reason != "bulk_pressure_with_interactive_qos_observed" || action == "" {
|
|
t.Fatalf("unexpected healthy pressure classification: status=%q reason=%q action=%q", status, reason, action)
|
|
}
|
|
status, reason, _ = fabricServiceChannelFlowHealth(map[string]int{"bulk": 32}, 1, 25, 4, 0, 0, 0, 0, 0)
|
|
if status != "critical" || reason != "flow_drops_reported" {
|
|
t.Fatalf("unexpected drop classification: status=%q reason=%q", status, reason)
|
|
}
|
|
status, reason, _ = fabricServiceChannelFlowHealth(map[string]int{"bulk": 2}, 0, 4, 1, 0, 1500, 0, 0, 0)
|
|
if status != "degraded" || reason != "route_send_latency_high" {
|
|
t.Fatalf("unexpected latency classification: status=%q reason=%q", status, reason)
|
|
}
|
|
}
|
|
|
|
func TestFabricServiceChannelAccessTelemetryRecommendsAlternateForDegradedRoute(t *testing.T) {
|
|
now := time.Date(2026, 5, 8, 15, 55, 0, 0, time.UTC)
|
|
expiresAt := now.Add(5 * time.Minute)
|
|
store := &fakeRepository{
|
|
platformRole: PlatformRoleAdmin,
|
|
clusterNodes: []ClusterNode{
|
|
{ID: "node-1", Name: "entry-1"},
|
|
{ID: "node-2", Name: "exit-1"},
|
|
},
|
|
fabricLeases: map[string]FabricServiceChannelLeaseRecord{
|
|
fabricServiceChannelLeaseCacheKey("cluster-1", "channel-1"): {
|
|
ClusterID: "cluster-1",
|
|
ChannelID: "channel-1",
|
|
ResourceID: "vpn-home",
|
|
ServiceClass: FabricServiceClassVPNPackets,
|
|
SelectedEntryNodeID: "node-1",
|
|
ExpiresAt: expiresAt,
|
|
Lease: FabricServiceChannelLease{
|
|
ClusterID: "cluster-1",
|
|
ChannelID: "channel-1",
|
|
ResourceID: "vpn-home",
|
|
ServiceClass: FabricServiceClassVPNPackets,
|
|
Status: FabricServiceChannelStatusReady,
|
|
SelectedEntryNodeID: "node-1",
|
|
SelectedExitNodeID: "node-2",
|
|
PrimaryRoute: FabricServiceChannelRoute{
|
|
RouteID: "route-bad",
|
|
Status: "authorized",
|
|
},
|
|
AlternateRoutes: []FabricServiceChannelRoute{{
|
|
RouteID: "route-alt",
|
|
Status: "authorized",
|
|
}},
|
|
ExpiresAt: expiresAt,
|
|
},
|
|
},
|
|
},
|
|
}
|
|
service := NewService(store)
|
|
service.now = func() time.Time { return now }
|
|
_, err := store.RecordNodeTelemetry(context.Background(), RecordNodeTelemetryInput{
|
|
ClusterID: "cluster-1",
|
|
NodeID: "node-1",
|
|
Payload: json.RawMessage(`{
|
|
"fabric_service_channel_access_report": {
|
|
"total": 4,
|
|
"introspection": 4,
|
|
"backend_fallback": 0
|
|
}
|
|
}`),
|
|
ObservedAt: now,
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("record telemetry: %v", err)
|
|
}
|
|
_, err = store.RecordFabricServiceChannelRouteFeedback(context.Background(), RecordFabricServiceChannelRouteFeedbackInput{
|
|
ClusterID: "cluster-1",
|
|
ReporterNodeID: "node-1",
|
|
RouteID: "route-bad",
|
|
ServiceClass: FabricServiceClassVPNPackets,
|
|
FeedbackStatus: "fenced",
|
|
ScoreAdjustment: -1030,
|
|
Reasons: []string{"service_channel_route_rebuild_recommended"},
|
|
LastSendDurationMs: 1200,
|
|
Payload: json.RawMessage(`{"quality_window_sample_count":7,"quality_window_failure_count":3,"quality_window_drop_count":1}`),
|
|
ObservedAt: now,
|
|
ExpiresAt: expiresAt,
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("record route feedback: %v", err)
|
|
}
|
|
_, err = store.RecordHeartbeat(context.Background(), RecordHeartbeatInput{
|
|
ClusterID: "cluster-1",
|
|
NodeID: "node-1",
|
|
HealthStatus: "healthy",
|
|
Metadata: json.RawMessage(`{
|
|
"fabric_service_channel_runtime_report": {
|
|
"ingress": {
|
|
"route_manager": {
|
|
"last_applied_at": "2026-05-08T15:55:01Z",
|
|
"decisions": [{
|
|
"route_id": "route-bad",
|
|
"replacement_route_id": "route-alt",
|
|
"rebuild_request_id": "fsc-remediation:channel-1:prefer_alternate_route:route-alt",
|
|
"rebuild_status": "applied",
|
|
"rebuild_reason": "authorized_alternate_route_available",
|
|
"decision_source": "service_channel_remediation_command",
|
|
"generation": "config-c18z74"
|
|
}]
|
|
},
|
|
"route_manager_transition": {
|
|
"status": "applied_rebuild",
|
|
"generation": "config-c18z74",
|
|
"observed_at": "2026-05-08T15:55:01Z"
|
|
}
|
|
}
|
|
}
|
|
}`),
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("record heartbeat: %v", err)
|
|
}
|
|
report, err := service.GetFabricServiceChannelAccessTelemetry(context.Background(), "admin-1", GetFabricServiceChannelAccessTelemetryInput{
|
|
ClusterID: "cluster-1",
|
|
Limit: 10,
|
|
Now: now,
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("get access telemetry: %v", err)
|
|
}
|
|
if report.DegradedRouteCount != 1 || report.Status != "degraded" {
|
|
t.Fatalf("expected degraded route aggregate: %+v", report)
|
|
}
|
|
if len(report.ActiveChannels) != 1 {
|
|
t.Fatalf("expected one active channel, got %d", len(report.ActiveChannels))
|
|
}
|
|
channel := report.ActiveChannels[0]
|
|
if channel.RemediationAction != "prefer_alternate_route" || channel.RemediationRouteID != "route-alt" {
|
|
t.Fatalf("expected alternate remediation, got %+v", channel)
|
|
}
|
|
if channel.RemediationCommand == nil {
|
|
t.Fatalf("expected bounded remediation command, got %+v", channel)
|
|
}
|
|
if channel.RemediationCommand.Action != "prefer_alternate_route" ||
|
|
channel.RemediationCommand.ReplacementRouteID != "route-alt" ||
|
|
channel.RemediationCommand.PrimaryRouteID != "route-bad" ||
|
|
channel.RemediationCommand.ClusterID != "cluster-1" {
|
|
t.Fatalf("unexpected remediation command: %+v", channel.RemediationCommand)
|
|
}
|
|
if channel.RemediationExecutionStatus != "applied" ||
|
|
channel.RemediationExecutionReason != "authorized_alternate_route_available" ||
|
|
channel.RemediationExecutionGeneration != "config-c18z74" ||
|
|
channel.RemediationCommand.ExecutionStatus != "applied" {
|
|
t.Fatalf("unexpected remediation execution: channel=%+v command=%+v", channel, channel.RemediationCommand)
|
|
}
|
|
if !channel.RemediationCommand.IssuedAt.Equal(now) || channel.RemediationCommand.ExpiresAt.After(expiresAt) || !channel.RemediationCommand.ExpiresAt.After(now) {
|
|
t.Fatalf("unexpected remediation command ttl: %+v", channel.RemediationCommand)
|
|
}
|
|
}
|
|
|
|
func TestFabricServiceChannelAccessTelemetryRejectsAlternateOutsideSignedPoolPolicy(t *testing.T) {
|
|
now := time.Date(2026, 5, 8, 16, 15, 0, 0, time.UTC)
|
|
expiresAt := now.Add(5 * time.Minute)
|
|
store := &fakeRepository{
|
|
platformRole: PlatformRoleAdmin,
|
|
clusterNodes: []ClusterNode{
|
|
{ID: "entry-1", Name: "entry-1"},
|
|
{ID: "exit-1", Name: "exit-1"},
|
|
{ID: "exit-2", Name: "exit-2"},
|
|
},
|
|
fabricLeases: map[string]FabricServiceChannelLeaseRecord{
|
|
fabricServiceChannelLeaseCacheKey("cluster-1", "channel-guard"): {
|
|
ClusterID: "cluster-1",
|
|
ChannelID: "channel-guard",
|
|
ResourceID: "vpn-home",
|
|
ServiceClass: FabricServiceClassVPNPackets,
|
|
SelectedEntryNodeID: "entry-1",
|
|
ExpiresAt: expiresAt,
|
|
Lease: FabricServiceChannelLease{
|
|
ClusterID: "cluster-1",
|
|
ChannelID: "channel-guard",
|
|
ResourceID: "vpn-home",
|
|
ServiceClass: FabricServiceClassVPNPackets,
|
|
Status: FabricServiceChannelStatusReady,
|
|
SelectedEntryNodeID: "entry-1",
|
|
SelectedExitNodeID: "exit-1",
|
|
EntryPool: []FabricServiceChannelNodeCandidate{{
|
|
NodeID: "entry-1",
|
|
Status: "selected",
|
|
}},
|
|
ExitPool: []FabricServiceChannelNodeCandidate{{
|
|
NodeID: "exit-1",
|
|
Status: "selected",
|
|
}},
|
|
PoolPolicy: &FabricServiceChannelPoolPolicy{
|
|
SchemaVersion: "rap.fabric_service_channel_pool_policy.v1",
|
|
Fingerprint: "pool-fingerprint-1",
|
|
EntryPoolNodeIDs: []string{"entry-1"},
|
|
ExitPoolNodeIDs: []string{"exit-1"},
|
|
SelectionStrategy: "fastest_healthy",
|
|
RouteRebuild: "automatic",
|
|
EntryFailover: "automatic",
|
|
ExitFailover: "automatic",
|
|
BackendFallbackAllowed: true,
|
|
StickySession: true,
|
|
Source: "cluster_metadata",
|
|
},
|
|
PrimaryRoute: FabricServiceChannelRoute{
|
|
RouteID: "route-bad",
|
|
ClusterID: "cluster-1",
|
|
ServiceClass: FabricServiceClassVPNPackets,
|
|
SourceNodeID: "entry-1",
|
|
DestinationNodeID: "exit-1",
|
|
Status: "authorized",
|
|
},
|
|
AlternateRoutes: []FabricServiceChannelRoute{{
|
|
RouteID: "route-outside-exit",
|
|
ClusterID: "cluster-1",
|
|
ServiceClass: FabricServiceClassVPNPackets,
|
|
SourceNodeID: "entry-1",
|
|
DestinationNodeID: "exit-2",
|
|
Status: "authorized",
|
|
}},
|
|
ExpiresAt: expiresAt,
|
|
},
|
|
},
|
|
},
|
|
fabricRebuildAttempts: []FabricServiceChannelRouteRebuildAttempt{{
|
|
ID: "fsc-rebuild-guard-1",
|
|
ClusterID: "cluster-1",
|
|
ReporterNodeID: "entry-1",
|
|
ServiceClass: FabricServiceClassVPNPackets,
|
|
RouteID: "route-bad",
|
|
ReplacementRouteID: "route-outside-exit",
|
|
RebuildRequestID: "fsc-remediation:channel-guard:rebuild_route:route-outside-exit",
|
|
RebuildStatus: "rejected",
|
|
RebuildReason: "replacement_exit_outside_signed_pool_policy",
|
|
DecisionSource: "service_channel_remediation_command",
|
|
Outcome: "policy_guard_rejected",
|
|
PolicyFingerprint: "pool-fingerprint-1",
|
|
CreatedAt: now,
|
|
UpdatedAt: now,
|
|
}},
|
|
}
|
|
service := NewService(store)
|
|
service.now = func() time.Time { return now }
|
|
_, err := store.RecordFabricServiceChannelRouteFeedback(context.Background(), RecordFabricServiceChannelRouteFeedbackInput{
|
|
ClusterID: "cluster-1",
|
|
ReporterNodeID: "entry-1",
|
|
RouteID: "route-bad",
|
|
ServiceClass: FabricServiceClassVPNPackets,
|
|
FeedbackStatus: "fenced",
|
|
ScoreAdjustment: -1030,
|
|
Reasons: []string{"service_channel_route_rebuild_recommended"},
|
|
LastSendDurationMs: 1200,
|
|
ObservedAt: now,
|
|
ExpiresAt: expiresAt,
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("record route feedback: %v", err)
|
|
}
|
|
report, err := service.GetFabricServiceChannelAccessTelemetry(context.Background(), "admin-1", GetFabricServiceChannelAccessTelemetryInput{
|
|
ClusterID: "cluster-1",
|
|
Limit: 10,
|
|
Now: now,
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("get access telemetry: %v", err)
|
|
}
|
|
if len(report.ActiveChannels) != 1 {
|
|
t.Fatalf("expected one active channel, got %d", len(report.ActiveChannels))
|
|
}
|
|
channel := report.ActiveChannels[0]
|
|
if channel.RemediationAction != "rebuild_route" ||
|
|
channel.RemediationReason != "alternate_route_rejected_by_pool_policy" ||
|
|
channel.RemediationRouteID != "route-outside-exit" ||
|
|
channel.RemediationGuardStatus != "rejected" ||
|
|
channel.RemediationGuardReason != "replacement_exit_outside_signed_pool_policy" ||
|
|
channel.PoolPolicyFingerprint != "pool-fingerprint-1" {
|
|
t.Fatalf("expected guarded rebuild remediation, got %+v", channel)
|
|
}
|
|
if channel.RemediationCommand == nil {
|
|
t.Fatalf("expected guarded remediation command, got %+v", channel)
|
|
}
|
|
if channel.RemediationCommand.Action != "rebuild_route" ||
|
|
channel.RemediationCommand.GuardStatus != "rejected" ||
|
|
channel.RemediationCommand.GuardReason != "replacement_exit_outside_signed_pool_policy" ||
|
|
channel.RemediationCommand.PoolPolicyFingerprint != "pool-fingerprint-1" ||
|
|
channel.RemediationCommand.ExecutionStatus != "rebuild_request_rejected" {
|
|
t.Fatalf("unexpected guarded remediation command: %+v", channel.RemediationCommand)
|
|
}
|
|
}
|
|
|
|
func TestFabricServiceChannelAccessTelemetryShowsRebuildRouteNodePending(t *testing.T) {
|
|
now := time.Date(2026, 5, 8, 16, 50, 0, 0, time.UTC)
|
|
expiresAt := now.Add(5 * time.Minute)
|
|
commandID := "fsc-remediation:channel-pending:rebuild_route:route-bad"
|
|
store := &fakeRepository{
|
|
platformRole: PlatformRoleAdmin,
|
|
clusterNodes: []ClusterNode{
|
|
{ID: "entry-1", Name: "entry-1"},
|
|
{ID: "exit-1", Name: "exit-1"},
|
|
},
|
|
fabricLeases: map[string]FabricServiceChannelLeaseRecord{
|
|
fabricServiceChannelLeaseCacheKey("cluster-1", "channel-pending"): {
|
|
ClusterID: "cluster-1",
|
|
ChannelID: "channel-pending",
|
|
ResourceID: "vpn-home",
|
|
ServiceClass: FabricServiceClassVPNPackets,
|
|
SelectedEntryNodeID: "entry-1",
|
|
ExpiresAt: expiresAt,
|
|
Lease: FabricServiceChannelLease{
|
|
ClusterID: "cluster-1",
|
|
ChannelID: "channel-pending",
|
|
ResourceID: "vpn-home",
|
|
ServiceClass: FabricServiceClassVPNPackets,
|
|
Status: FabricServiceChannelStatusReady,
|
|
SelectedEntryNodeID: "entry-1",
|
|
SelectedExitNodeID: "exit-1",
|
|
PrimaryRoute: FabricServiceChannelRoute{
|
|
RouteID: "route-bad",
|
|
ClusterID: "cluster-1",
|
|
ServiceClass: FabricServiceClassVPNPackets,
|
|
SourceNodeID: "entry-1",
|
|
DestinationNodeID: "exit-1",
|
|
Status: "authorized",
|
|
},
|
|
ExpiresAt: expiresAt,
|
|
},
|
|
},
|
|
},
|
|
fabricRebuildAttempts: []FabricServiceChannelRouteRebuildAttempt{{
|
|
ID: "fsc-rebuild-pending-1",
|
|
ClusterID: "cluster-1",
|
|
ReporterNodeID: "entry-1",
|
|
ServiceClass: FabricServiceClassVPNPackets,
|
|
RouteID: "route-bad",
|
|
RebuildRequestID: commandID,
|
|
RebuildStatus: "requested",
|
|
RebuildReason: "route_feedback_recommends_rebuild",
|
|
DecisionSource: "service_channel_remediation_command",
|
|
Outcome: "rebuild_requested",
|
|
Generation: commandID,
|
|
CreatedAt: now,
|
|
UpdatedAt: now,
|
|
}},
|
|
}
|
|
service := NewService(store)
|
|
service.now = func() time.Time { return now }
|
|
_, err := store.RecordFabricServiceChannelRouteFeedback(context.Background(), RecordFabricServiceChannelRouteFeedbackInput{
|
|
ClusterID: "cluster-1",
|
|
ReporterNodeID: "entry-1",
|
|
RouteID: "route-bad",
|
|
ServiceClass: FabricServiceClassVPNPackets,
|
|
FeedbackStatus: "fenced",
|
|
ScoreAdjustment: -1030,
|
|
Reasons: []string{"service_channel_route_rebuild_recommended"},
|
|
ObservedAt: now,
|
|
ExpiresAt: expiresAt,
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("record route feedback: %v", err)
|
|
}
|
|
_, err = store.RecordHeartbeat(context.Background(), RecordHeartbeatInput{
|
|
ClusterID: "cluster-1",
|
|
NodeID: "entry-1",
|
|
HealthStatus: "healthy",
|
|
Metadata: json.RawMessage(`{
|
|
"fabric_service_channel_runtime_report": {
|
|
"ingress": {
|
|
"route_manager": {
|
|
"last_applied_at": "2026-05-08T16:50:01Z",
|
|
"decisions": [{
|
|
"route_id": "route-bad",
|
|
"rebuild_request_id": "fsc-remediation:channel-pending:rebuild_route:route-bad",
|
|
"rebuild_status": "pending_degraded_fallback",
|
|
"rebuild_reason": "route_feedback_recommends_rebuild",
|
|
"decision_source": "service_channel_remediation_command",
|
|
"generation": "fsc-remediation:channel-pending:rebuild_route:route-bad"
|
|
}]
|
|
},
|
|
"route_manager_transition": {
|
|
"status": "pending_degraded_fallback",
|
|
"generation": "fsc-remediation:channel-pending:rebuild_route:route-bad",
|
|
"observed_at": "2026-05-08T16:50:01Z"
|
|
}
|
|
}
|
|
}
|
|
}`),
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("record heartbeat: %v", err)
|
|
}
|
|
report, err := service.GetFabricServiceChannelAccessTelemetry(context.Background(), "admin-1", GetFabricServiceChannelAccessTelemetryInput{
|
|
ClusterID: "cluster-1",
|
|
Limit: 10,
|
|
Now: now,
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("get access telemetry: %v", err)
|
|
}
|
|
if len(report.ActiveChannels) != 1 {
|
|
t.Fatalf("expected one active channel, got %d", len(report.ActiveChannels))
|
|
}
|
|
channel := report.ActiveChannels[0]
|
|
if channel.RemediationAction != "rebuild_route" ||
|
|
channel.RemediationExecutionStatus != "rebuild_request_recorded_node_pending" ||
|
|
channel.RemediationExecutionGeneration != commandID ||
|
|
channel.RouteDecisionSource != "service_channel_remediation_command" ||
|
|
channel.RouteDecisionRebuildStatus != "pending_degraded_fallback" ||
|
|
channel.RemediationCommand == nil ||
|
|
channel.RemediationCommand.ExecutionStatus != "rebuild_request_recorded_node_pending" {
|
|
t.Fatalf("unexpected rebuild route execution: channel=%+v command=%+v", channel, channel.RemediationCommand)
|
|
}
|
|
}
|
|
|
|
func TestFabricServiceChannelAccessTelemetryProjectsNoSafeRecoveryDecision(t *testing.T) {
|
|
now := time.Date(2026, 5, 9, 3, 10, 0, 0, time.UTC)
|
|
expiresAt := now.Add(5 * time.Minute)
|
|
store := &fakeRepository{
|
|
platformRole: PlatformRoleAdmin,
|
|
clusterNodes: []ClusterNode{
|
|
{ID: "entry-1", Name: "entry-1"},
|
|
{ID: "exit-1", Name: "exit-1"},
|
|
},
|
|
fabricLeases: map[string]FabricServiceChannelLeaseRecord{
|
|
fabricServiceChannelLeaseCacheKey("cluster-1", "channel-no-safe"): {
|
|
ClusterID: "cluster-1",
|
|
ChannelID: "channel-no-safe",
|
|
ResourceID: "vpn-home",
|
|
ServiceClass: FabricServiceClassVPNPackets,
|
|
SelectedEntryNodeID: "entry-1",
|
|
ExpiresAt: expiresAt,
|
|
Lease: FabricServiceChannelLease{
|
|
ClusterID: "cluster-1",
|
|
ChannelID: "channel-no-safe",
|
|
ResourceID: "vpn-home",
|
|
ServiceClass: FabricServiceClassVPNPackets,
|
|
Status: FabricServiceChannelStatusReady,
|
|
SelectedEntryNodeID: "entry-1",
|
|
SelectedExitNodeID: "exit-1",
|
|
PrimaryRoute: FabricServiceChannelRoute{
|
|
RouteID: "route-primary",
|
|
ClusterID: "cluster-1",
|
|
ServiceClass: FabricServiceClassVPNPackets,
|
|
SourceNodeID: "entry-1",
|
|
DestinationNodeID: "exit-1",
|
|
Status: "authorized",
|
|
},
|
|
ExpiresAt: expiresAt,
|
|
},
|
|
},
|
|
},
|
|
}
|
|
service := NewService(store)
|
|
service.now = func() time.Time { return now }
|
|
_, err := store.RecordHeartbeat(context.Background(), RecordHeartbeatInput{
|
|
ClusterID: "cluster-1",
|
|
NodeID: "entry-1",
|
|
HealthStatus: "healthy",
|
|
Metadata: json.RawMessage(`{
|
|
"fabric_service_channel_runtime_report": {
|
|
"ingress": {
|
|
"route_manager": {
|
|
"last_applied_at": "2026-05-09T03:10:01Z",
|
|
"decisions": [{
|
|
"route_id": "route-replacement",
|
|
"source_node_id": "entry-1",
|
|
"destination_node_id": "exit-1",
|
|
"local_node_id": "entry-1",
|
|
"decision_source": "service_channel_feedback_no_alternate",
|
|
"rebuild_status": "pending_degraded_fallback",
|
|
"rebuild_reason": "service_channel_feedback_rebuild_requested",
|
|
"generation": "c18z82-generation",
|
|
"score_reasons": [
|
|
"service_channel_fenced_route",
|
|
"no_unfenced_alternate_route",
|
|
"backend_relay_degraded_fallback_until_rebuild"
|
|
]
|
|
}]
|
|
},
|
|
"route_manager_transition": {
|
|
"status": "pending_degraded_fallback",
|
|
"generation": "c18z82-generation",
|
|
"observed_at": "2026-05-09T03:10:01Z"
|
|
}
|
|
}
|
|
}
|
|
}`),
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("record heartbeat: %v", err)
|
|
}
|
|
report, err := service.GetFabricServiceChannelAccessTelemetry(context.Background(), "admin-1", GetFabricServiceChannelAccessTelemetryInput{
|
|
ClusterID: "cluster-1",
|
|
Limit: 10,
|
|
Now: now,
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("get access telemetry: %v", err)
|
|
}
|
|
if len(report.ActiveChannels) != 1 {
|
|
t.Fatalf("expected one active channel, got %d", len(report.ActiveChannels))
|
|
}
|
|
channel := report.ActiveChannels[0]
|
|
if channel.RouteDecisionSource != "service_channel_feedback_no_alternate" ||
|
|
channel.RouteDecisionRouteID != "route-replacement" ||
|
|
channel.RouteDecisionRebuildStatus != "pending_degraded_fallback" ||
|
|
!containsString(channel.RouteDecisionScoreReasons, "no_unfenced_alternate_route") ||
|
|
channel.RemediationAction != "use_backend_fallback" ||
|
|
channel.RemediationExecutionStatus != "route_rebuild_no_safe_recovery" {
|
|
t.Fatalf("unexpected no-safe route decision projection: %+v", channel)
|
|
}
|
|
if report.RouteDecisionChannelCount != 1 ||
|
|
report.NoSafeRecoveryDecisionCount != 1 ||
|
|
report.ReplacementDecisionCount != 0 ||
|
|
report.AppliedRebuildDecisionCount != 0 ||
|
|
report.Status != "degraded" ||
|
|
report.Reason != "active_channels_no_safe_recovery" {
|
|
t.Fatalf("unexpected no-safe route decision aggregate: %+v", report)
|
|
}
|
|
health, err := service.GetFabricServiceChannelRouteRebuildHealthSummary(context.Background(), "admin-1", GetFabricServiceChannelRouteRebuildHealthSummaryInput{
|
|
ClusterID: "cluster-1",
|
|
Limit: 10,
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("get rebuild health: %v", err)
|
|
}
|
|
if health.AccessRouteDecisionCount != 1 ||
|
|
health.AccessNoSafeCount != 1 ||
|
|
health.ActiveBadCount != 1 ||
|
|
health.RecommendedOperatorAction != "inspect_access_no_safe_recovery_route_pool_and_signed_policy" {
|
|
t.Fatalf("unexpected rebuild health access decision projection: %+v", health)
|
|
}
|
|
incidents, err := service.ListFabricServiceChannelRouteRebuildIncidents(context.Background(), "admin-1", ListFabricServiceChannelRouteRebuildIncidentsInput{
|
|
ClusterID: "cluster-1",
|
|
Limit: 10,
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("list rebuild incidents: %v", err)
|
|
}
|
|
if len(incidents) == 0 ||
|
|
incidents[0].IncidentSource != "access_decision" ||
|
|
incidents[0].ChannelID != "channel-no-safe" ||
|
|
incidents[0].GuardStatus != "access_no_safe_recovery" ||
|
|
incidents[0].GuardSeverity != "bad" {
|
|
t.Fatalf("unexpected access decision incident projection: %+v", incidents)
|
|
}
|
|
silence, err := service.SilenceFabricServiceChannelRouteRebuildAlert(context.Background(), SilenceFabricServiceChannelRouteRebuildAlertInput{
|
|
ActorUserID: "admin-1",
|
|
ClusterID: "cluster-1",
|
|
IncidentSource: "access_decision",
|
|
ChannelID: incidents[0].ChannelID,
|
|
ReporterNodeID: incidents[0].ReporterNodeID,
|
|
RouteID: incidents[0].RouteID,
|
|
GuardStatus: incidents[0].GuardStatus,
|
|
Generation: incidents[0].Generation,
|
|
Reason: "operator acknowledged access no-safe",
|
|
TTL: 6 * time.Hour,
|
|
Now: now,
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("silence access decision incident: %v", err)
|
|
}
|
|
health, err = service.GetFabricServiceChannelRouteRebuildHealthSummary(context.Background(), "admin-1", GetFabricServiceChannelRouteRebuildHealthSummaryInput{
|
|
ClusterID: "cluster-1",
|
|
Limit: 10,
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("get silenced rebuild health: %v", err)
|
|
}
|
|
if health.AccessNoSafeCount != 1 || health.ActiveBadCount != 0 || health.SilencedCount != 1 {
|
|
t.Fatalf("unexpected silenced access decision health: %+v", health)
|
|
}
|
|
incidents, err = service.ListFabricServiceChannelRouteRebuildIncidents(context.Background(), "admin-1", ListFabricServiceChannelRouteRebuildIncidentsInput{
|
|
ClusterID: "cluster-1",
|
|
Limit: 10,
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("list silenced rebuild incidents: %v", err)
|
|
}
|
|
if len(incidents) == 0 || !incidents[0].AlertSilenced {
|
|
t.Fatalf("expected silenced access decision incident: %+v", incidents)
|
|
}
|
|
silences, err := service.ListFabricServiceChannelRouteRebuildAlertSilences(context.Background(), "admin-1", "cluster-1", now)
|
|
if err != nil {
|
|
t.Fatalf("list rebuild alert silences: %v", err)
|
|
}
|
|
if len(silences) != 1 ||
|
|
silences[0].ID != silence.ID ||
|
|
silences[0].IncidentSource != "access_decision" ||
|
|
silences[0].ChannelID != "channel-no-safe" ||
|
|
silences[0].DisplayRouteID != "route-replacement" {
|
|
t.Fatalf("unexpected listed access decision silence: %+v", silences)
|
|
}
|
|
_, err = service.UnsilenceFabricServiceChannelRouteRebuildAlert(context.Background(), UnsilenceFabricServiceChannelRouteRebuildAlertInput{
|
|
ActorUserID: "admin-1",
|
|
ClusterID: "cluster-1",
|
|
SilenceID: silence.ID,
|
|
Reason: "operator reopened access no-safe",
|
|
Now: now.Add(time.Minute),
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("unsilence access decision incident: %v", err)
|
|
}
|
|
health, err = service.GetFabricServiceChannelRouteRebuildHealthSummary(context.Background(), "admin-1", GetFabricServiceChannelRouteRebuildHealthSummaryInput{
|
|
ClusterID: "cluster-1",
|
|
Limit: 10,
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("get unsilenced rebuild health: %v", err)
|
|
}
|
|
if health.ActiveBadCount != 1 || health.SilencedCount != 0 {
|
|
t.Fatalf("unexpected unsilenced access decision health: %+v", health)
|
|
}
|
|
silence, err = service.SilenceFabricServiceChannelRouteRebuildAlert(context.Background(), SilenceFabricServiceChannelRouteRebuildAlertInput{
|
|
ActorUserID: "admin-1",
|
|
ClusterID: "cluster-1",
|
|
IncidentSource: "access_decision",
|
|
ChannelID: incidents[0].ChannelID,
|
|
ReporterNodeID: incidents[0].ReporterNodeID,
|
|
RouteID: incidents[0].RouteID,
|
|
GuardStatus: incidents[0].GuardStatus,
|
|
Generation: incidents[0].Generation,
|
|
Reason: "operator acknowledged access no-safe again",
|
|
TTL: 6 * time.Hour,
|
|
Now: now,
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("resilence access decision incident: %v", err)
|
|
}
|
|
_, err = store.RecordHeartbeat(context.Background(), RecordHeartbeatInput{
|
|
ClusterID: "cluster-1",
|
|
NodeID: "entry-1",
|
|
HealthStatus: "healthy",
|
|
Metadata: json.RawMessage(`{
|
|
"fabric_service_channel_runtime_report": {
|
|
"ingress": {
|
|
"route_manager": {
|
|
"last_applied_at": "2026-05-09T03:11:01Z",
|
|
"decisions": [{
|
|
"route_id": "route-replacement",
|
|
"source_node_id": "entry-1",
|
|
"destination_node_id": "exit-1",
|
|
"local_node_id": "entry-1",
|
|
"decision_source": "service_channel_feedback_no_alternate",
|
|
"rebuild_status": "pending_degraded_fallback",
|
|
"rebuild_reason": "service_channel_feedback_rebuild_requested",
|
|
"generation": "c18z82-generation-next",
|
|
"score_reasons": ["service_channel_fenced_route", "no_unfenced_alternate_route"]
|
|
}]
|
|
}
|
|
}
|
|
}
|
|
}`),
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("record resurfaced heartbeat: %v", err)
|
|
}
|
|
incidents, err = service.ListFabricServiceChannelRouteRebuildIncidents(context.Background(), "admin-1", ListFabricServiceChannelRouteRebuildIncidentsInput{
|
|
ClusterID: "cluster-1",
|
|
Limit: 10,
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("list resurfaced rebuild incidents: %v", err)
|
|
}
|
|
if len(incidents) == 0 || incidents[0].AlertSilenced || !incidents[0].AlertResurfaced || incidents[0].Generation != "c18z82-generation-next" ||
|
|
incidents[0].AlertResurfacedCause != "generation_changed" ||
|
|
incidents[0].AlertResurfacedPreviousGeneration != "c18z82-generation" ||
|
|
incidents[0].AlertResurfacedPreviousRouteID != "route-replacement" ||
|
|
incidents[0].AlertResurfacedPreviousChannelID != "channel-no-safe" {
|
|
t.Fatalf("expected resurfaced access decision incident on new generation: %+v", incidents)
|
|
}
|
|
}
|
|
|
|
func TestRecordFabricServiceChannelRemediationRebuildIntentsPersistsRequestedAndRejected(t *testing.T) {
|
|
now := time.Date(2026, 5, 8, 16, 45, 0, 0, time.UTC)
|
|
store := &fakeRepository{}
|
|
service := NewService(store)
|
|
err := service.recordFabricServiceChannelRemediationRebuildIntents(context.Background(), "cluster-1", "entry-1", []FabricServiceChannelAccessRemediationCommand{
|
|
{
|
|
CommandID: "cmd-requested",
|
|
Action: "rebuild_route",
|
|
ChannelID: "channel-1",
|
|
ServiceClass: FabricServiceClassVPNPackets,
|
|
PrimaryRouteID: "route-a",
|
|
PoolPolicyFingerprint: "pool-fp-1",
|
|
GuardStatus: "allowed",
|
|
GuardReason: "lease_pool_policy_allows_route",
|
|
Reason: "route_feedback_recommends_rebuild",
|
|
ExpiresAt: now.Add(time.Minute),
|
|
},
|
|
{
|
|
CommandID: "cmd-rejected",
|
|
Action: "rebuild_route",
|
|
ChannelID: "channel-2",
|
|
ServiceClass: FabricServiceClassVPNPackets,
|
|
PrimaryRouteID: "route-b",
|
|
ReplacementRouteID: "route-outside",
|
|
PoolPolicyFingerprint: "pool-fp-2",
|
|
GuardStatus: "rejected",
|
|
GuardReason: "replacement_exit_outside_signed_pool_policy",
|
|
Reason: "alternate_route_rejected_by_pool_policy",
|
|
ExpiresAt: now.Add(time.Minute),
|
|
},
|
|
}, now)
|
|
if err != nil {
|
|
t.Fatalf("record rebuild intents: %v", err)
|
|
}
|
|
if len(store.fabricRebuildAttempts) != 2 {
|
|
t.Fatalf("rebuild attempts = %+v, want two", store.fabricRebuildAttempts)
|
|
}
|
|
first := store.fabricRebuildAttempts[0]
|
|
if first.RebuildRequestID != "cmd-requested" ||
|
|
first.RebuildStatus != "requested" ||
|
|
first.Outcome != "rebuild_requested" ||
|
|
first.DecisionSource != "service_channel_remediation_command" ||
|
|
first.PolicyFingerprint != "pool-fp-1" {
|
|
t.Fatalf("unexpected requested rebuild intent: %+v", first)
|
|
}
|
|
second := store.fabricRebuildAttempts[1]
|
|
if second.RebuildRequestID != "cmd-rejected" ||
|
|
second.RebuildStatus != "rejected" ||
|
|
second.Outcome != "policy_guard_rejected" ||
|
|
second.ReplacementRouteID != "route-outside" ||
|
|
second.PolicyFingerprint != "pool-fp-2" {
|
|
t.Fatalf("unexpected rejected rebuild intent: %+v", second)
|
|
}
|
|
}
|
|
|
|
func TestResolveFabricServiceChannelRemediationRebuildIntentsRecordsNoAlternate(t *testing.T) {
|
|
now := time.Date(2026, 5, 9, 1, 10, 0, 0, time.UTC)
|
|
store := &fakeRepository{
|
|
fabricLeases: map[string]FabricServiceChannelLeaseRecord{
|
|
fabricServiceChannelLeaseCacheKey("cluster-1", "channel-no-alt"): {
|
|
ClusterID: "cluster-1",
|
|
ChannelID: "channel-no-alt",
|
|
ResourceID: "vpn-home",
|
|
ServiceClass: FabricServiceClassVPNPackets,
|
|
SelectedEntryNodeID: "entry-1",
|
|
ExpiresAt: now.Add(time.Minute),
|
|
Lease: FabricServiceChannelLease{
|
|
ClusterID: "cluster-1",
|
|
ChannelID: "channel-no-alt",
|
|
ResourceID: "vpn-home",
|
|
ServiceClass: FabricServiceClassVPNPackets,
|
|
Status: FabricServiceChannelStatusReady,
|
|
SelectedEntryNodeID: "entry-1",
|
|
SelectedExitNodeID: "exit-1",
|
|
EntryPool: []FabricServiceChannelNodeCandidate{{NodeID: "entry-1", Status: "selected"}},
|
|
ExitPool: []FabricServiceChannelNodeCandidate{{NodeID: "exit-1", Status: "selected"}},
|
|
ExpiresAt: now.Add(time.Minute),
|
|
},
|
|
},
|
|
},
|
|
}
|
|
service := NewService(store)
|
|
decisions, err := service.resolveFabricServiceChannelRemediationRebuildIntents(context.Background(), GetNodeSyntheticMeshConfigInput{
|
|
ClusterID: "cluster-1",
|
|
NodeID: "entry-1",
|
|
}, []FabricServiceChannelAccessRemediationCommand{{
|
|
CommandID: "cmd-no-alt",
|
|
Action: "rebuild_route",
|
|
ChannelID: "channel-no-alt",
|
|
ResourceID: "vpn-home",
|
|
ServiceClass: FabricServiceClassVPNPackets,
|
|
PrimaryRouteID: "route-bad",
|
|
GuardStatus: "allowed",
|
|
Reason: "route_feedback_recommends_rebuild",
|
|
ExpiresAt: now.Add(time.Minute),
|
|
}}, []MeshRouteIntent{{
|
|
ID: "route-bad",
|
|
ClusterID: "cluster-1",
|
|
SourceSelector: json.RawMessage(`{"node_id":"entry-1"}`),
|
|
DestinationSelector: json.RawMessage(`{"node_id":"exit-1"}`),
|
|
ServiceClass: FabricServiceClassVPNPackets,
|
|
Priority: 100,
|
|
Status: "active",
|
|
Policy: json.RawMessage(`{
|
|
"synthetic_enabled": true,
|
|
"hops": ["entry-1", "exit-1"],
|
|
"allowed_channels": ["vpn_packet", "fabric_control"]
|
|
}`),
|
|
UpdatedAt: now,
|
|
}}, map[string]fabricServiceChannelRouteFeedback{
|
|
"route-bad": {
|
|
RouteID: "route-bad",
|
|
Fenced: true,
|
|
RouteRebuildRecommended: true,
|
|
ScoreAdjustment: -1030,
|
|
Reasons: []string{"service_channel_route_rebuild_recommended"},
|
|
ConsecutiveFailures: 3,
|
|
},
|
|
}, "config-c18z77", now)
|
|
if err != nil {
|
|
t.Fatalf("resolve rebuild intents: %v", err)
|
|
}
|
|
if len(decisions) != 0 {
|
|
t.Fatalf("decisions = %+v, want none without alternate", decisions)
|
|
}
|
|
if len(store.fabricRebuildAttempts) != 1 {
|
|
t.Fatalf("rebuild attempts = %+v, want one", store.fabricRebuildAttempts)
|
|
}
|
|
attempt := store.fabricRebuildAttempts[0]
|
|
if attempt.RebuildStatus != "no_alternate" ||
|
|
attempt.Outcome != "no_alternate" ||
|
|
attempt.RebuildReason != "no_unfenced_alternate_route" ||
|
|
attempt.ConsecutiveFailures != 3 {
|
|
t.Fatalf("unexpected no-alternate rebuild resolution: %+v", attempt)
|
|
}
|
|
}
|
|
|
|
func TestResolveFabricServiceChannelRemediationRebuildIntentsAppliesAlternateDecision(t *testing.T) {
|
|
now := time.Date(2026, 5, 9, 1, 15, 0, 0, time.UTC)
|
|
store := &fakeRepository{
|
|
fabricLeases: map[string]FabricServiceChannelLeaseRecord{
|
|
fabricServiceChannelLeaseCacheKey("cluster-1", "channel-apply"): {
|
|
ClusterID: "cluster-1",
|
|
ChannelID: "channel-apply",
|
|
ResourceID: "vpn-home",
|
|
ServiceClass: FabricServiceClassVPNPackets,
|
|
SelectedEntryNodeID: "entry-1",
|
|
ExpiresAt: now.Add(time.Minute),
|
|
Lease: FabricServiceChannelLease{
|
|
ClusterID: "cluster-1",
|
|
ChannelID: "channel-apply",
|
|
ResourceID: "vpn-home",
|
|
ServiceClass: FabricServiceClassVPNPackets,
|
|
Status: FabricServiceChannelStatusReady,
|
|
SelectedEntryNodeID: "entry-1",
|
|
SelectedExitNodeID: "exit-1",
|
|
EntryPool: []FabricServiceChannelNodeCandidate{{NodeID: "entry-1", Status: "selected"}},
|
|
ExitPool: []FabricServiceChannelNodeCandidate{{NodeID: "exit-1", Status: "selected"}},
|
|
ExpiresAt: now.Add(time.Minute),
|
|
},
|
|
},
|
|
},
|
|
}
|
|
service := NewService(store)
|
|
intents := []MeshRouteIntent{
|
|
{
|
|
ID: "route-bad",
|
|
ClusterID: "cluster-1",
|
|
SourceSelector: json.RawMessage(`{"node_id":"entry-1"}`),
|
|
DestinationSelector: json.RawMessage(`{"node_id":"exit-1"}`),
|
|
ServiceClass: FabricServiceClassVPNPackets,
|
|
Priority: 100,
|
|
Status: "active",
|
|
Policy: json.RawMessage(`{
|
|
"synthetic_enabled": true,
|
|
"hops": ["entry-1", "exit-1"],
|
|
"allowed_channels": ["vpn_packet", "fabric_control"]
|
|
}`),
|
|
UpdatedAt: now,
|
|
},
|
|
{
|
|
ID: "route-good",
|
|
ClusterID: "cluster-1",
|
|
SourceSelector: json.RawMessage(`{"node_id":"entry-1"}`),
|
|
DestinationSelector: json.RawMessage(`{"node_id":"exit-1"}`),
|
|
ServiceClass: FabricServiceClassVPNPackets,
|
|
Priority: 90,
|
|
Status: "active",
|
|
Policy: json.RawMessage(`{
|
|
"synthetic_enabled": true,
|
|
"hops": ["entry-1", "exit-1"],
|
|
"allowed_channels": ["vpn_packet", "fabric_control"]
|
|
}`),
|
|
UpdatedAt: now,
|
|
},
|
|
}
|
|
decisions, err := service.resolveFabricServiceChannelRemediationRebuildIntents(context.Background(), GetNodeSyntheticMeshConfigInput{
|
|
ClusterID: "cluster-1",
|
|
NodeID: "entry-1",
|
|
}, []FabricServiceChannelAccessRemediationCommand{{
|
|
CommandID: "cmd-apply",
|
|
Action: "rebuild_route",
|
|
ChannelID: "channel-apply",
|
|
ResourceID: "vpn-home",
|
|
ServiceClass: FabricServiceClassVPNPackets,
|
|
PrimaryRouteID: "route-bad",
|
|
GuardStatus: "allowed",
|
|
Reason: "route_feedback_recommends_rebuild",
|
|
ExpiresAt: now.Add(time.Minute),
|
|
}}, intents, map[string]fabricServiceChannelRouteFeedback{
|
|
"route-bad": {
|
|
RouteID: "route-bad",
|
|
Fenced: true,
|
|
RouteRebuildRecommended: true,
|
|
ScoreAdjustment: -1030,
|
|
Reasons: []string{"service_channel_route_rebuild_recommended"},
|
|
},
|
|
"route-good": {
|
|
RouteID: "route-good",
|
|
ScoreAdjustment: 100,
|
|
Reasons: []string{"service_channel_recent_success"},
|
|
},
|
|
}, "config-c18z77", now)
|
|
if err != nil {
|
|
t.Fatalf("resolve rebuild intents: %v", err)
|
|
}
|
|
if len(decisions) != 1 {
|
|
t.Fatalf("decisions = %+v, want one applied alternate", decisions)
|
|
}
|
|
decision := decisions[0]
|
|
if decision.RebuildRequestID != "cmd-apply" ||
|
|
decision.RebuildStatus != "applied" ||
|
|
decision.ReplacementRouteID != "route-good" ||
|
|
decision.DecisionSource != "service_channel_remediation_command" {
|
|
t.Fatalf("unexpected applied remediation decision: %+v", decision)
|
|
}
|
|
attempt := store.fabricRebuildAttempts[0]
|
|
if attempt.RebuildStatus != "applied" ||
|
|
attempt.Outcome != "replacement_selected" ||
|
|
attempt.ReplacementRouteID != "route-good" ||
|
|
!reflect.DeepEqual(attempt.OldHops, []string{"entry-1", "exit-1"}) ||
|
|
!reflect.DeepEqual(attempt.ReplacementHops, []string{"entry-1", "exit-1"}) {
|
|
t.Fatalf("unexpected applied rebuild resolution: %+v", attempt)
|
|
}
|
|
}
|
|
|
|
func TestIssueFabricServiceChannelLeaseMarksBackendRelayAsDegradedFallbackWhenRouteMissing(t *testing.T) {
|
|
now := time.Date(2026, 5, 7, 12, 30, 0, 0, time.UTC)
|
|
service := NewService(&fakeRepository{})
|
|
service.now = func() time.Time { return now }
|
|
|
|
lease, err := service.IssueFabricServiceChannelLease(context.Background(), IssueFabricServiceChannelLeaseInput{
|
|
ClusterID: "cluster-1",
|
|
OrganizationID: "org-home",
|
|
UserID: "user-m",
|
|
ServiceClass: FabricServiceClassRemoteWorkspace,
|
|
EntryNodeIDs: []string{"entry-a"},
|
|
ExitNodeIDs: []string{"exit-b"},
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("issue lease: %v", err)
|
|
}
|
|
if lease.Status != FabricServiceChannelStatusDegradedFallback {
|
|
t.Fatalf("lease.Status = %q, want degraded_fallback", lease.Status)
|
|
}
|
|
if lease.PrimaryRoute.Status != "missing_route_intent" || lease.PrimaryRoute.RouteID != "" {
|
|
t.Fatalf("unexpected primary route fallback: %+v", lease.PrimaryRoute)
|
|
}
|
|
if lease.PrimaryRoute.RecoveryPolicy == nil {
|
|
t.Fatalf("fallback primary route must include recovery policy provenance")
|
|
}
|
|
if !lease.Fallback.Active || !lease.Fallback.Degraded || !lease.Fallback.BackendRelay {
|
|
t.Fatalf("fallback should be active degraded backend relay: %+v", lease.Fallback)
|
|
}
|
|
if !containsString(lease.AllowedChannels, FabricChannelInteractive) || !containsString(lease.RequiredRoles, "rdp-worker") {
|
|
t.Fatalf("remote workspace defaults not applied: channels=%v roles=%v", lease.AllowedChannels, lease.RequiredRoles)
|
|
}
|
|
if strings.Contains(lease.EntryHTTP.PathTemplate, "vpn-connections") ||
|
|
!strings.Contains(lease.EntryHTTP.PathTemplate, "remote-workspaces") ||
|
|
lease.EntryHTTP.PacketBatchFormat != "application/vnd.rap.remote-workspace-frame-batch.v1" {
|
|
t.Fatalf("remote workspace ingress should not be vpn-specific: %+v", lease.EntryHTTP)
|
|
}
|
|
if lease.DataPlane.StableContractForServiceClass != FabricServiceClassRemoteWorkspace ||
|
|
!lease.DataPlane.ServiceNeutral ||
|
|
!lease.DataPlane.ProtocolAgnostic ||
|
|
!containsString(lease.DataPlane.RequiredFlowIsolationClasses, FabricChannelInteractive) {
|
|
t.Fatalf("unexpected remote workspace data-plane contract: %+v", lease.DataPlane)
|
|
}
|
|
}
|
|
|
|
func TestIssueFabricServiceChannelLeaseUsesServiceClassAwareIngressDescriptors(t *testing.T) {
|
|
now := time.Date(2026, 5, 12, 14, 10, 0, 0, time.UTC)
|
|
service := NewService(&fakeRepository{})
|
|
service.now = func() time.Time { return now }
|
|
|
|
tests := []struct {
|
|
name string
|
|
service string
|
|
pathNeedle string
|
|
packetMedia string
|
|
}{
|
|
{name: "vpn", service: FabricServiceClassVPNPackets, pathNeedle: "vpn-connections", packetMedia: "application/vnd.rap.vpn-packet-batch.v1"},
|
|
{name: "remote workspace", service: FabricServiceClassRemoteWorkspace, pathNeedle: "remote-workspaces", packetMedia: "application/vnd.rap.remote-workspace-frame-batch.v1"},
|
|
{name: "file transfer", service: FabricServiceClassFileTransfer, pathNeedle: "file-transfers", packetMedia: "application/vnd.rap.file-transfer-chunk-batch.v1"},
|
|
{name: "video", service: FabricServiceClassVideo, pathNeedle: "video-sessions", packetMedia: "application/vnd.rap.video-frame-batch.v1"},
|
|
}
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
lease, err := service.IssueFabricServiceChannelLease(context.Background(), IssueFabricServiceChannelLeaseInput{
|
|
ClusterID: "cluster-1",
|
|
OrganizationID: "org-home",
|
|
UserID: "user-m",
|
|
ResourceID: "resource-1",
|
|
ServiceClass: tt.service,
|
|
EntryNodeIDs: []string{"entry-a"},
|
|
ExitNodeIDs: []string{"exit-b"},
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("issue lease: %v", err)
|
|
}
|
|
if !strings.Contains(lease.EntryHTTP.PathTemplate, tt.pathNeedle) {
|
|
t.Fatalf("PathTemplate = %q, want %q", lease.EntryHTTP.PathTemplate, tt.pathNeedle)
|
|
}
|
|
if lease.EntryHTTP.PacketBatchFormat != tt.packetMedia {
|
|
t.Fatalf("PacketBatchFormat = %q, want %q", lease.EntryHTTP.PacketBatchFormat, tt.packetMedia)
|
|
}
|
|
if lease.DataPlane.StableContractForServiceClass != tt.service {
|
|
t.Fatalf("StableContractForServiceClass = %q, want %q", lease.DataPlane.StableContractForServiceClass, tt.service)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestIssueFabricServiceChannelLeaseFencesRouteFromFlowFeedback(t *testing.T) {
|
|
now := time.Date(2026, 5, 7, 13, 0, 0, 0, time.UTC)
|
|
store := &fakeRepository{
|
|
routeIntents: []MeshRouteIntent{
|
|
{
|
|
ID: "route-bad",
|
|
ClusterID: "cluster-1",
|
|
SourceSelector: json.RawMessage(`{"node_id":"entry-1"}`),
|
|
DestinationSelector: json.RawMessage(`{"node_id":"exit-1"}`),
|
|
ServiceClass: FabricServiceClassVPNPackets,
|
|
Priority: 50,
|
|
Status: "active",
|
|
Policy: json.RawMessage(`{
|
|
"hops": ["entry-1", "relay-bad", "exit-1"],
|
|
"allowed_channels": ["vpn_packet", "fabric_control"]
|
|
}`),
|
|
UpdatedAt: now,
|
|
},
|
|
{
|
|
ID: "route-good",
|
|
ClusterID: "cluster-1",
|
|
SourceSelector: json.RawMessage(`{"node_id":"entry-1"}`),
|
|
DestinationSelector: json.RawMessage(`{"node_id":"exit-1"}`),
|
|
ServiceClass: FabricServiceClassVPNPackets,
|
|
Priority: 10,
|
|
Status: "active",
|
|
Policy: json.RawMessage(`{
|
|
"hops": ["entry-1", "relay-good", "exit-1"],
|
|
"allowed_channels": ["vpn_packet", "fabric_control"]
|
|
}`),
|
|
UpdatedAt: now,
|
|
},
|
|
},
|
|
heartbeats: map[string][]NodeHeartbeat{
|
|
"entry-1": {
|
|
{
|
|
ClusterID: "cluster-1",
|
|
NodeID: "entry-1",
|
|
ObservedAt: now.Add(-15 * time.Second),
|
|
Metadata: json.RawMessage(`{
|
|
"fabric_service_channel_runtime_report": {
|
|
"schema_version": "c18l.fabric_service_channel_runtime_report.v1",
|
|
"ingress": {
|
|
"flow_scheduler": {
|
|
"channel_stats": {
|
|
"flow-7": {
|
|
"last_failed_route_id": "route-bad",
|
|
"last_error": "forward peer unavailable",
|
|
"consecutive_failures": 2,
|
|
"route_rebuild_recommended": true,
|
|
"degraded_fallback_recommended": true
|
|
},
|
|
"flow-9": {
|
|
"last_route_id": "route-good",
|
|
"last_next_hop": "relay-good",
|
|
"consecutive_failures": 0
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}`),
|
|
},
|
|
},
|
|
},
|
|
}
|
|
service := NewService(store)
|
|
service.now = func() time.Time { return now }
|
|
|
|
lease, err := service.IssueFabricServiceChannelLease(context.Background(), IssueFabricServiceChannelLeaseInput{
|
|
ClusterID: "cluster-1",
|
|
OrganizationID: "org-home",
|
|
UserID: "user-m",
|
|
ResourceID: "vpn-home",
|
|
ServiceClass: FabricServiceClassVPNPackets,
|
|
EntryNodeIDs: []string{"entry-1"},
|
|
ExitNodeIDs: []string{"exit-1"},
|
|
PreferredEntryNodeID: "entry-1",
|
|
PreferredExitNodeID: "exit-1",
|
|
TTL: 90 * time.Second,
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("issue lease: %v", err)
|
|
}
|
|
if lease.PrimaryRoute.RouteID != "route-good" {
|
|
t.Fatalf("primary route = %q, want route-good after route-bad feedback fence", lease.PrimaryRoute.RouteID)
|
|
}
|
|
if !containsString(lease.PrimaryRoute.ScoreReasons, "service_channel_recent_success") {
|
|
t.Fatalf("primary route should include service-channel success feedback: %+v", lease.PrimaryRoute)
|
|
}
|
|
for _, alternate := range lease.AlternateRoutes {
|
|
if alternate.RouteID == "route-bad" {
|
|
t.Fatalf("fenced route must not be offered as alternate: %+v", lease.AlternateRoutes)
|
|
}
|
|
}
|
|
if lease.Fallback.Active || lease.Fallback.Degraded {
|
|
t.Fatalf("healthy alternate should avoid degraded fallback: %+v", lease.Fallback)
|
|
}
|
|
}
|
|
|
|
func TestIssueFabricServiceChannelLeasePrefersFastHealthyRouteFeedback(t *testing.T) {
|
|
now := time.Date(2026, 5, 7, 16, 10, 0, 0, time.UTC)
|
|
store := &fakeRepository{
|
|
routeIntents: []MeshRouteIntent{
|
|
{
|
|
ID: "route-slow",
|
|
ClusterID: "cluster-1",
|
|
SourceSelector: json.RawMessage(`{"node_id":"entry-1"}`),
|
|
DestinationSelector: json.RawMessage(`{"node_id":"exit-1"}`),
|
|
ServiceClass: FabricServiceClassVPNPackets,
|
|
Priority: 120,
|
|
Status: "active",
|
|
Policy: json.RawMessage(`{
|
|
"hops": ["entry-1", "relay-slow", "exit-1"],
|
|
"allowed_channels": ["vpn_packet", "fabric_control"]
|
|
}`),
|
|
UpdatedAt: now,
|
|
},
|
|
{
|
|
ID: "route-fast",
|
|
ClusterID: "cluster-1",
|
|
SourceSelector: json.RawMessage(`{"node_id":"entry-1"}`),
|
|
DestinationSelector: json.RawMessage(`{"node_id":"exit-1"}`),
|
|
ServiceClass: FabricServiceClassVPNPackets,
|
|
Priority: 80,
|
|
Status: "active",
|
|
Policy: json.RawMessage(`{
|
|
"hops": ["entry-1", "relay-fast", "exit-1"],
|
|
"allowed_channels": ["vpn_packet", "fabric_control"]
|
|
}`),
|
|
UpdatedAt: now,
|
|
},
|
|
},
|
|
heartbeats: map[string][]NodeHeartbeat{
|
|
"entry-1": {
|
|
{
|
|
ClusterID: "cluster-1",
|
|
NodeID: "entry-1",
|
|
ObservedAt: now.Add(-10 * time.Second),
|
|
Metadata: json.RawMessage(`{
|
|
"fabric_service_channel_runtime_report": {
|
|
"schema_version": "c18l.fabric_service_channel_runtime_report.v1",
|
|
"ingress": {
|
|
"flow_scheduler": {
|
|
"channel_stats": {
|
|
"flow-fast": {
|
|
"last_route_id": "route-fast",
|
|
"last_next_hop": "relay-fast",
|
|
"last_send_duration_ms": 8,
|
|
"consecutive_failures": 0,
|
|
"stall_count": 0
|
|
},
|
|
"flow-slow": {
|
|
"last_route_id": "route-slow",
|
|
"last_next_hop": "relay-slow",
|
|
"last_send_duration_ms": 900,
|
|
"consecutive_failures": 0,
|
|
"stall_count": 0
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}`),
|
|
},
|
|
},
|
|
},
|
|
}
|
|
service := NewService(store)
|
|
service.now = func() time.Time { return now }
|
|
|
|
lease, err := service.IssueFabricServiceChannelLease(context.Background(), IssueFabricServiceChannelLeaseInput{
|
|
ClusterID: "cluster-1",
|
|
OrganizationID: "org-home",
|
|
UserID: "user-m",
|
|
ResourceID: "vpn-home",
|
|
ServiceClass: FabricServiceClassVPNPackets,
|
|
EntryNodeIDs: []string{"entry-1"},
|
|
ExitNodeIDs: []string{"exit-1"},
|
|
PreferredEntryNodeID: "entry-1",
|
|
PreferredExitNodeID: "exit-1",
|
|
TTL: 90 * time.Second,
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("issue lease: %v", err)
|
|
}
|
|
if lease.PrimaryRoute.RouteID != "route-fast" {
|
|
t.Fatalf("primary route = %q, want route-fast from quality feedback; route=%+v alternates=%+v", lease.PrimaryRoute.RouteID, lease.PrimaryRoute, lease.AlternateRoutes)
|
|
}
|
|
if !containsString(lease.PrimaryRoute.ScoreReasons, "service_channel_quality_latency_le_10ms") {
|
|
t.Fatalf("fast route should include latency quality reason: %+v", lease.PrimaryRoute)
|
|
}
|
|
var slow FabricServiceChannelRoute
|
|
for _, route := range lease.AlternateRoutes {
|
|
if route.RouteID == "route-slow" {
|
|
slow = route
|
|
break
|
|
}
|
|
}
|
|
if slow.RouteID == "" || !containsString(slow.ScoreReasons, "service_channel_quality_latency_very_slow") {
|
|
t.Fatalf("slow alternate should retain quality penalty reason: %+v", lease.AlternateRoutes)
|
|
}
|
|
}
|
|
|
|
func TestIssueFabricServiceChannelLeaseDecaysOlderHealthyRouteFeedback(t *testing.T) {
|
|
now := time.Date(2026, 5, 8, 9, 0, 0, 0, time.UTC)
|
|
store := &fakeRepository{
|
|
routeIntents: []MeshRouteIntent{
|
|
{
|
|
ID: "route-old-fast",
|
|
ClusterID: "cluster-1",
|
|
SourceSelector: json.RawMessage(`{"node_id":"entry-1"}`),
|
|
DestinationSelector: json.RawMessage(`{"node_id":"exit-1"}`),
|
|
ServiceClass: FabricServiceClassVPNPackets,
|
|
Priority: 80,
|
|
Status: "active",
|
|
Policy: json.RawMessage(`{
|
|
"hops": ["entry-1", "exit-1"],
|
|
"allowed_channels": ["vpn_packet", "fabric_control"]
|
|
}`),
|
|
UpdatedAt: now,
|
|
},
|
|
{
|
|
ID: "route-fresh",
|
|
ClusterID: "cluster-1",
|
|
SourceSelector: json.RawMessage(`{"node_id":"entry-1"}`),
|
|
DestinationSelector: json.RawMessage(`{"node_id":"exit-1"}`),
|
|
ServiceClass: FabricServiceClassVPNPackets,
|
|
Priority: 80,
|
|
Status: "active",
|
|
Policy: json.RawMessage(`{
|
|
"hops": ["entry-1", "relay-fresh", "exit-1"],
|
|
"allowed_channels": ["vpn_packet", "fabric_control"]
|
|
}`),
|
|
UpdatedAt: now,
|
|
},
|
|
},
|
|
fabricRouteFeedback: []FabricServiceChannelRouteFeedbackObservation{
|
|
{
|
|
ID: "feedback-old",
|
|
ClusterID: "cluster-1",
|
|
ReporterNodeID: "entry-1",
|
|
RouteID: "route-old-fast",
|
|
ServiceClass: FabricServiceClassVPNPackets,
|
|
FeedbackStatus: "healthy",
|
|
ScoreAdjustment: 90,
|
|
Reasons: []string{"service_channel_recent_success", "service_channel_quality_latency_le_10ms"},
|
|
LastSendDurationMs: 1,
|
|
ObservedAt: now.Add(-90 * time.Second),
|
|
ExpiresAt: now.Add(30 * time.Second),
|
|
},
|
|
{
|
|
ID: "feedback-fresh",
|
|
ClusterID: "cluster-1",
|
|
ReporterNodeID: "entry-1",
|
|
RouteID: "route-fresh",
|
|
ServiceClass: FabricServiceClassVPNPackets,
|
|
FeedbackStatus: "healthy",
|
|
ScoreAdjustment: 40,
|
|
Reasons: []string{"service_channel_recent_success", "service_channel_quality_latency_le_50ms"},
|
|
LastSendDurationMs: 40,
|
|
ObservedAt: now.Add(-5 * time.Second),
|
|
ExpiresAt: now.Add(115 * time.Second),
|
|
},
|
|
},
|
|
}
|
|
service := NewService(store)
|
|
service.now = func() time.Time { return now }
|
|
|
|
lease, err := service.IssueFabricServiceChannelLease(context.Background(), IssueFabricServiceChannelLeaseInput{
|
|
ClusterID: "cluster-1",
|
|
OrganizationID: "org-home",
|
|
UserID: "user-m",
|
|
ResourceID: "vpn-home",
|
|
ServiceClass: FabricServiceClassVPNPackets,
|
|
EntryNodeIDs: []string{"entry-1"},
|
|
ExitNodeIDs: []string{"exit-1"},
|
|
PreferredEntryNodeID: "entry-1",
|
|
PreferredExitNodeID: "exit-1",
|
|
TTL: 90 * time.Second,
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("issue lease: %v", err)
|
|
}
|
|
if lease.PrimaryRoute.RouteID != "route-fresh" {
|
|
t.Fatalf("primary route = %q, want fresher feedback route after age decay; route=%+v alternates=%+v", lease.PrimaryRoute.RouteID, lease.PrimaryRoute, lease.AlternateRoutes)
|
|
}
|
|
var oldRoute FabricServiceChannelRoute
|
|
for _, route := range lease.AlternateRoutes {
|
|
if route.RouteID == "route-old-fast" {
|
|
oldRoute = route
|
|
break
|
|
}
|
|
}
|
|
if oldRoute.RouteID == "" || !containsString(oldRoute.ScoreReasons, "service_channel_feedback_age_decay") {
|
|
t.Fatalf("old route should carry age decay reason: %+v", lease.AlternateRoutes)
|
|
}
|
|
}
|
|
|
|
func TestServiceChannelRouteFeedbackReportIncludesEffectiveDecayedScore(t *testing.T) {
|
|
now := time.Date(2026, 5, 8, 9, 3, 0, 0, time.UTC)
|
|
report := serviceChannelRouteFeedbackReport([]FabricServiceChannelRouteFeedbackObservation{{
|
|
ID: "feedback-old",
|
|
ClusterID: "cluster-1",
|
|
ReporterNodeID: "entry-1",
|
|
RouteID: "route-old-fast",
|
|
ServiceClass: FabricServiceClassVPNPackets,
|
|
FeedbackStatus: "healthy",
|
|
ScoreAdjustment: 90,
|
|
Reasons: []string{"service_channel_recent_success"},
|
|
LastSendDurationMs: 1,
|
|
ObservedAt: now.Add(-90 * time.Second),
|
|
ExpiresAt: now.Add(30 * time.Second),
|
|
}}, now)
|
|
if report == nil || len(report.Observations) != 1 {
|
|
t.Fatalf("report observations = %+v, want one observation", report)
|
|
}
|
|
observation := report.Observations[0]
|
|
if observation.ScoreAdjustment != 90 || observation.EffectiveScoreAdjustment != 23 {
|
|
t.Fatalf("scores raw/effective = %d/%d, want 90/23", observation.ScoreAdjustment, observation.EffectiveScoreAdjustment)
|
|
}
|
|
if !containsString(observation.Reasons, "service_channel_feedback_age_decay") {
|
|
t.Fatalf("reasons = %+v, want age decay reason", observation.Reasons)
|
|
}
|
|
}
|
|
|
|
func TestIssueFabricServiceChannelLeaseFallsBackWhenOnlyRouteFencedByFlowFeedback(t *testing.T) {
|
|
now := time.Date(2026, 5, 7, 13, 30, 0, 0, time.UTC)
|
|
store := &fakeRepository{
|
|
routeIntents: []MeshRouteIntent{
|
|
{
|
|
ID: "route-bad",
|
|
ClusterID: "cluster-1",
|
|
SourceSelector: json.RawMessage(`{"node_id":"entry-1"}`),
|
|
DestinationSelector: json.RawMessage(`{"node_id":"exit-1"}`),
|
|
ServiceClass: FabricServiceClassVPNPackets,
|
|
Priority: 50,
|
|
Status: "active",
|
|
Policy: json.RawMessage(`{
|
|
"hops": ["entry-1", "relay-bad", "exit-1"],
|
|
"allowed_channels": ["vpn_packet", "fabric_control"]
|
|
}`),
|
|
UpdatedAt: now,
|
|
},
|
|
},
|
|
heartbeats: map[string][]NodeHeartbeat{
|
|
"entry-1": {
|
|
{
|
|
ClusterID: "cluster-1",
|
|
NodeID: "entry-1",
|
|
ObservedAt: now.Add(-10 * time.Second),
|
|
Metadata: json.RawMessage(`{
|
|
"fabric_service_channel_runtime_report": {
|
|
"schema_version": "c18l.fabric_service_channel_runtime_report.v1",
|
|
"ingress": {
|
|
"flow_scheduler": {
|
|
"channel_stats": {
|
|
"flow-7": {
|
|
"last_failed_route_id": "route-bad",
|
|
"consecutive_failures": 2,
|
|
"route_rebuild_recommended": true
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}`),
|
|
},
|
|
},
|
|
},
|
|
}
|
|
service := NewService(store)
|
|
service.now = func() time.Time { return now }
|
|
|
|
lease, err := service.IssueFabricServiceChannelLease(context.Background(), IssueFabricServiceChannelLeaseInput{
|
|
ClusterID: "cluster-1",
|
|
OrganizationID: "org-home",
|
|
UserID: "user-m",
|
|
ResourceID: "vpn-home",
|
|
ServiceClass: FabricServiceClassVPNPackets,
|
|
EntryNodeIDs: []string{"entry-1"},
|
|
ExitNodeIDs: []string{"exit-1"},
|
|
PreferredEntryNodeID: "entry-1",
|
|
PreferredExitNodeID: "exit-1",
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("issue lease: %v", err)
|
|
}
|
|
if lease.Status != FabricServiceChannelStatusDegradedFallback ||
|
|
lease.Fallback.Reason != "fabric_route_rebuild_pending_backend_relay" {
|
|
t.Fatalf("lease should degrade because the only route is fenced: status=%s fallback=%+v", lease.Status, lease.Fallback)
|
|
}
|
|
}
|
|
|
|
func TestIssueFabricServiceChannelLeaseSelectsHealthyAlternateExitFromPool(t *testing.T) {
|
|
now := time.Date(2026, 5, 7, 13, 45, 0, 0, time.UTC)
|
|
store := &fakeRepository{
|
|
routeIntents: []MeshRouteIntent{
|
|
{
|
|
ID: "route-entry-exit-a",
|
|
ClusterID: "cluster-1",
|
|
SourceSelector: json.RawMessage(`{"node_id":"entry-1"}`),
|
|
DestinationSelector: json.RawMessage(`{"node_id":"exit-a"}`),
|
|
ServiceClass: FabricServiceClassVPNPackets,
|
|
Priority: 100,
|
|
Status: "active",
|
|
Policy: json.RawMessage(`{
|
|
"hops": ["entry-1", "exit-a"],
|
|
"allowed_channels": ["vpn_packet", "fabric_control"],
|
|
"metadata": {"exit_pool_id": "pool-home"}
|
|
}`),
|
|
UpdatedAt: now,
|
|
},
|
|
{
|
|
ID: "route-entry-exit-b",
|
|
ClusterID: "cluster-1",
|
|
SourceSelector: json.RawMessage(`{"node_id":"entry-1"}`),
|
|
DestinationSelector: json.RawMessage(`{"node_id":"exit-b"}`),
|
|
ServiceClass: FabricServiceClassVPNPackets,
|
|
Priority: 30,
|
|
Status: "active",
|
|
Policy: json.RawMessage(`{
|
|
"hops": ["entry-1", "exit-b"],
|
|
"allowed_channels": ["vpn_packet", "fabric_control"],
|
|
"metadata": {"exit_pool_id": "pool-home"}
|
|
}`),
|
|
UpdatedAt: now,
|
|
},
|
|
},
|
|
fabricRouteFeedback: []FabricServiceChannelRouteFeedbackObservation{
|
|
{
|
|
ClusterID: "cluster-1",
|
|
ReporterNodeID: "entry-1",
|
|
RouteID: "route-entry-exit-a",
|
|
ServiceClass: FabricServiceClassVPNPackets,
|
|
FeedbackStatus: "fenced",
|
|
ScoreAdjustment: -1030,
|
|
Reasons: []string{"service_channel_route_rebuild_recommended"},
|
|
ConsecutiveFailures: 2,
|
|
ObservedAt: now,
|
|
ExpiresAt: now.Add(time.Minute),
|
|
},
|
|
{
|
|
ClusterID: "cluster-1",
|
|
ReporterNodeID: "entry-1",
|
|
RouteID: "route-entry-exit-b",
|
|
ServiceClass: FabricServiceClassVPNPackets,
|
|
FeedbackStatus: "healthy",
|
|
ScoreAdjustment: 10,
|
|
Reasons: []string{"service_channel_recent_success"},
|
|
ObservedAt: now,
|
|
ExpiresAt: now.Add(time.Minute),
|
|
},
|
|
},
|
|
}
|
|
service := NewService(store)
|
|
service.now = func() time.Time { return now }
|
|
|
|
lease, err := service.IssueFabricServiceChannelLease(context.Background(), IssueFabricServiceChannelLeaseInput{
|
|
ClusterID: "cluster-1",
|
|
OrganizationID: "org-home",
|
|
UserID: "user-m",
|
|
ResourceID: "vpn-home",
|
|
ServiceClass: FabricServiceClassVPNPackets,
|
|
EntryNodeIDs: []string{"entry-1"},
|
|
ExitNodeIDs: []string{"exit-a", "exit-b"},
|
|
PreferredEntryNodeID: "entry-1",
|
|
PreferredExitNodeID: "exit-a",
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("issue lease: %v", err)
|
|
}
|
|
if lease.PrimaryRoute.RouteID != "route-entry-exit-b" || lease.SelectedExitNodeID != "exit-b" {
|
|
t.Fatalf("lease should select alternate exit from pool: selected_exit=%s primary=%+v", lease.SelectedExitNodeID, lease.PrimaryRoute)
|
|
}
|
|
for _, candidate := range lease.ExitPool {
|
|
if candidate.NodeID == "exit-b" && candidate.Status != "selected" {
|
|
t.Fatalf("alternate exit should be marked selected in exit pool: %+v", lease.ExitPool)
|
|
}
|
|
}
|
|
var signedPayload FabricServiceChannelLeaseAuthorityPayload
|
|
if err := json.Unmarshal(lease.AuthorityPayload, &signedPayload); err != nil {
|
|
t.Fatalf("unmarshal signed payload: %v", err)
|
|
}
|
|
if signedPayload.SelectedExitNodeID != "exit-b" || len(signedPayload.ExitPool) != 2 {
|
|
t.Fatalf("signed payload must bind selected exit and authorized exit pool: %+v", signedPayload)
|
|
}
|
|
if lease.Fallback.Active || lease.Fallback.Degraded {
|
|
t.Fatalf("healthy exit-pool alternate should avoid degraded fallback: %+v", lease.Fallback)
|
|
}
|
|
}
|
|
|
|
func TestIssueFabricServiceChannelLeaseSelectsHealthyAlternateEntryFromPool(t *testing.T) {
|
|
now := time.Date(2026, 5, 7, 14, 45, 0, 0, time.UTC)
|
|
store := &fakeRepository{
|
|
routeIntents: []MeshRouteIntent{
|
|
{
|
|
ID: "route-entry-a",
|
|
ClusterID: "cluster-1",
|
|
SourceSelector: json.RawMessage(`{"node_id":"entry-a"}`),
|
|
DestinationSelector: json.RawMessage(`{"node_id":"exit-1"}`),
|
|
ServiceClass: FabricServiceClassVPNPackets,
|
|
Priority: 100,
|
|
Status: "active",
|
|
Policy: json.RawMessage(`{
|
|
"hops": ["entry-a", "exit-1"],
|
|
"allowed_channels": ["vpn_packet", "fabric_control"],
|
|
"metadata": {"entry_pool_id": "pool-edge"}
|
|
}`),
|
|
UpdatedAt: now,
|
|
},
|
|
{
|
|
ID: "route-entry-b",
|
|
ClusterID: "cluster-1",
|
|
SourceSelector: json.RawMessage(`{"node_id":"entry-b"}`),
|
|
DestinationSelector: json.RawMessage(`{"node_id":"exit-1"}`),
|
|
ServiceClass: FabricServiceClassVPNPackets,
|
|
Priority: 30,
|
|
Status: "active",
|
|
Policy: json.RawMessage(`{
|
|
"hops": ["entry-b", "exit-1"],
|
|
"allowed_channels": ["vpn_packet", "fabric_control"],
|
|
"metadata": {"entry_pool_id": "pool-edge"}
|
|
}`),
|
|
UpdatedAt: now,
|
|
},
|
|
},
|
|
fabricRouteFeedback: []FabricServiceChannelRouteFeedbackObservation{
|
|
{
|
|
ClusterID: "cluster-1",
|
|
ReporterNodeID: "entry-a",
|
|
RouteID: "route-entry-a",
|
|
ServiceClass: FabricServiceClassVPNPackets,
|
|
FeedbackStatus: "fenced",
|
|
ScoreAdjustment: -1030,
|
|
Reasons: []string{"service_channel_entry_unreachable"},
|
|
ConsecutiveFailures: 3,
|
|
ObservedAt: now,
|
|
ExpiresAt: now.Add(time.Minute),
|
|
},
|
|
{
|
|
ClusterID: "cluster-1",
|
|
ReporterNodeID: "entry-b",
|
|
RouteID: "route-entry-b",
|
|
ServiceClass: FabricServiceClassVPNPackets,
|
|
FeedbackStatus: "healthy",
|
|
ScoreAdjustment: 10,
|
|
Reasons: []string{"service_channel_recent_success"},
|
|
ObservedAt: now,
|
|
ExpiresAt: now.Add(time.Minute),
|
|
},
|
|
},
|
|
}
|
|
service := NewService(store)
|
|
service.now = func() time.Time { return now }
|
|
|
|
lease, err := service.IssueFabricServiceChannelLease(context.Background(), IssueFabricServiceChannelLeaseInput{
|
|
ClusterID: "cluster-1",
|
|
OrganizationID: "org-home",
|
|
UserID: "user-m",
|
|
ResourceID: "vpn-home",
|
|
ServiceClass: FabricServiceClassVPNPackets,
|
|
EntryNodeIDs: []string{"entry-a", "entry-b"},
|
|
ExitNodeIDs: []string{"exit-1"},
|
|
PreferredEntryNodeID: "entry-a",
|
|
PreferredExitNodeID: "exit-1",
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("issue lease: %v", err)
|
|
}
|
|
if lease.PrimaryRoute.RouteID != "route-entry-b" || lease.SelectedEntryNodeID != "entry-b" {
|
|
t.Fatalf("lease should select alternate entry from pool: selected_entry=%s primary=%+v", lease.SelectedEntryNodeID, lease.PrimaryRoute)
|
|
}
|
|
for _, candidate := range lease.EntryPool {
|
|
if candidate.NodeID == "entry-b" && candidate.Status != "selected" {
|
|
t.Fatalf("alternate entry should be marked selected in entry pool: %+v", lease.EntryPool)
|
|
}
|
|
}
|
|
var signedPayload FabricServiceChannelLeaseAuthorityPayload
|
|
if err := json.Unmarshal(lease.AuthorityPayload, &signedPayload); err != nil {
|
|
t.Fatalf("unmarshal signed payload: %v", err)
|
|
}
|
|
if signedPayload.SelectedEntryNodeID != "entry-b" || len(signedPayload.EntryPool) != 2 {
|
|
t.Fatalf("signed payload must bind selected entry and authorized entry pool: %+v", signedPayload)
|
|
}
|
|
if lease.Fallback.Active || lease.Fallback.Degraded {
|
|
t.Fatalf("healthy entry-pool alternate should avoid degraded fallback: %+v", lease.Fallback)
|
|
}
|
|
}
|
|
|
|
func TestIssueFabricServiceChannelLeaseAppliesClusterPoolPolicy(t *testing.T) {
|
|
now := time.Date(2026, 5, 8, 20, 10, 0, 0, time.UTC)
|
|
policy := defaultFabricServiceChannelPoolPolicy()
|
|
policy.Source = "cluster_metadata"
|
|
policy.EntryPoolNodeIDs = []string{"entry-b"}
|
|
policy.ExitPoolNodeIDs = []string{"exit-b"}
|
|
policy.PreferredEntryNodeID = "entry-b"
|
|
policy.PreferredExitNodeID = "exit-b"
|
|
policy.SelectionStrategy = "preferred_first"
|
|
policy.RouteRebuild = "automatic"
|
|
policy.EntryFailover = "automatic"
|
|
policy.ExitFailover = "automatic"
|
|
policy.BackendFallbackAllowed = true
|
|
policy.StickySession = true
|
|
policy = normalizeFabricServiceChannelPoolPolicy(policy, defaultFabricServiceChannelPoolPolicy())
|
|
metadata, err := upsertFabricServiceChannelPoolPolicyMetadata(json.RawMessage(`{}`), policy)
|
|
if err != nil {
|
|
t.Fatalf("policy metadata: %v", err)
|
|
}
|
|
store := &fakeRepository{
|
|
cluster: Cluster{
|
|
ID: "cluster-1",
|
|
Slug: "cluster-1",
|
|
Name: "Cluster 1",
|
|
Status: ClusterStatusActive,
|
|
Metadata: metadata,
|
|
},
|
|
routeIntents: []MeshRouteIntent{
|
|
{
|
|
ID: "route-a",
|
|
ClusterID: "cluster-1",
|
|
SourceSelector: json.RawMessage(`{"node_id":"entry-a"}`),
|
|
DestinationSelector: json.RawMessage(`{"node_id":"exit-a"}`),
|
|
ServiceClass: FabricServiceClassVPNPackets,
|
|
Priority: 100,
|
|
Status: "active",
|
|
Policy: json.RawMessage(`{"hops":["entry-a","exit-a"],"allowed_channels":["vpn_packet","fabric_control"]}`),
|
|
UpdatedAt: now,
|
|
},
|
|
{
|
|
ID: "route-b",
|
|
ClusterID: "cluster-1",
|
|
SourceSelector: json.RawMessage(`{"node_id":"entry-b"}`),
|
|
DestinationSelector: json.RawMessage(`{"node_id":"exit-b"}`),
|
|
ServiceClass: FabricServiceClassVPNPackets,
|
|
Priority: 10,
|
|
Status: "active",
|
|
Policy: json.RawMessage(`{"hops":["entry-b","exit-b"],"allowed_channels":["vpn_packet","fabric_control"]}`),
|
|
UpdatedAt: now,
|
|
},
|
|
},
|
|
}
|
|
service := NewService(store)
|
|
service.now = func() time.Time { return now }
|
|
|
|
lease, err := service.IssueFabricServiceChannelLease(context.Background(), IssueFabricServiceChannelLeaseInput{
|
|
ClusterID: "cluster-1",
|
|
OrganizationID: "org-home",
|
|
UserID: "user-m",
|
|
ResourceID: "vpn-home",
|
|
ServiceClass: FabricServiceClassVPNPackets,
|
|
EntryNodeIDs: []string{"entry-a", "entry-b"},
|
|
ExitNodeIDs: []string{"exit-a", "exit-b"},
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("issue lease: %v", err)
|
|
}
|
|
if lease.SelectedEntryNodeID != "entry-b" || lease.SelectedExitNodeID != "exit-b" || lease.PrimaryRoute.RouteID != "route-b" {
|
|
t.Fatalf("lease did not apply pool policy: selected_entry=%s selected_exit=%s primary=%+v", lease.SelectedEntryNodeID, lease.SelectedExitNodeID, lease.PrimaryRoute)
|
|
}
|
|
if len(lease.EntryPool) != 1 || lease.EntryPool[0].NodeID != "entry-b" || len(lease.ExitPool) != 1 || lease.ExitPool[0].NodeID != "exit-b" {
|
|
t.Fatalf("lease pools should be constrained by pool policy: entry=%+v exit=%+v", lease.EntryPool, lease.ExitPool)
|
|
}
|
|
if lease.PoolPolicy == nil || lease.PoolPolicy.Fingerprint != policy.Fingerprint {
|
|
t.Fatalf("lease missing pool policy provenance: %+v want %s", lease.PoolPolicy, policy.Fingerprint)
|
|
}
|
|
var signedPayload FabricServiceChannelLeaseAuthorityPayload
|
|
if err := json.Unmarshal(lease.AuthorityPayload, &signedPayload); err != nil {
|
|
t.Fatalf("unmarshal signed payload: %v", err)
|
|
}
|
|
if signedPayload.PoolPolicy == nil || signedPayload.PoolPolicy.Fingerprint != policy.Fingerprint {
|
|
t.Fatalf("signed payload missing pool policy provenance: %+v want %s", signedPayload.PoolPolicy, policy.Fingerprint)
|
|
}
|
|
}
|
|
|
|
func TestRecordHeartbeatPersistsServiceChannelRouteFeedbackForLaterLease(t *testing.T) {
|
|
now := time.Now().UTC().Truncate(time.Second)
|
|
store := &fakeRepository{
|
|
routeIntents: []MeshRouteIntent{
|
|
{
|
|
ID: "route-bad",
|
|
ClusterID: "cluster-1",
|
|
SourceSelector: json.RawMessage(`{"node_id":"entry-1"}`),
|
|
DestinationSelector: json.RawMessage(`{"node_id":"exit-1"}`),
|
|
ServiceClass: FabricServiceClassVPNPackets,
|
|
Priority: 100,
|
|
Status: "active",
|
|
Policy: json.RawMessage(`{
|
|
"hops": ["entry-1", "relay-bad", "exit-1"],
|
|
"allowed_channels": ["vpn_packet", "fabric_control"]
|
|
}`),
|
|
UpdatedAt: now,
|
|
},
|
|
{
|
|
ID: "route-good",
|
|
ClusterID: "cluster-1",
|
|
SourceSelector: json.RawMessage(`{"node_id":"entry-1"}`),
|
|
DestinationSelector: json.RawMessage(`{"node_id":"exit-1"}`),
|
|
ServiceClass: FabricServiceClassVPNPackets,
|
|
Priority: 10,
|
|
Status: "active",
|
|
Policy: json.RawMessage(`{
|
|
"hops": ["entry-1", "relay-good", "exit-1"],
|
|
"allowed_channels": ["vpn_packet", "fabric_control"]
|
|
}`),
|
|
UpdatedAt: now,
|
|
},
|
|
},
|
|
}
|
|
service := NewService(store)
|
|
service.now = func() time.Time { return now }
|
|
|
|
_, err := service.RecordHeartbeat(context.Background(), RecordHeartbeatInput{
|
|
ClusterID: "cluster-1",
|
|
NodeID: "entry-1",
|
|
HealthStatus: "healthy",
|
|
Metadata: json.RawMessage(`{
|
|
"fabric_service_channel_runtime_report": {
|
|
"schema_version": "c18l.fabric_service_channel_runtime_report.v1",
|
|
"ingress": {
|
|
"flow_scheduler": {
|
|
"channel_stats": {
|
|
"flow-1": {
|
|
"last_failed_route_id": "route-bad",
|
|
"consecutive_failures": 2,
|
|
"route_rebuild_recommended": true
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}`),
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("record heartbeat: %v", err)
|
|
}
|
|
if len(store.fabricRouteFeedback) != 1 || store.fabricRouteFeedback[0].RouteID != "route-bad" ||
|
|
store.fabricRouteFeedback[0].FeedbackStatus != "fenced" {
|
|
t.Fatalf("service-channel route feedback was not persisted: %+v", store.fabricRouteFeedback)
|
|
}
|
|
|
|
lease, err := service.IssueFabricServiceChannelLease(context.Background(), IssueFabricServiceChannelLeaseInput{
|
|
ClusterID: "cluster-1",
|
|
OrganizationID: "org-home",
|
|
UserID: "user-m",
|
|
ResourceID: "vpn-home",
|
|
ServiceClass: FabricServiceClassVPNPackets,
|
|
EntryNodeIDs: []string{"entry-1"},
|
|
ExitNodeIDs: []string{"exit-1"},
|
|
PreferredEntryNodeID: "entry-1",
|
|
PreferredExitNodeID: "exit-1",
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("issue lease: %v", err)
|
|
}
|
|
if lease.PrimaryRoute.RouteID != "route-good" {
|
|
t.Fatalf("primary route = %q, want durable feedback to fence route-bad and select route-good", lease.PrimaryRoute.RouteID)
|
|
}
|
|
}
|
|
|
|
func TestRecordHeartbeatTurnsBlockedFallbackSendFailureIntoRebuildFeedback(t *testing.T) {
|
|
now := time.Now().UTC().Truncate(time.Second)
|
|
store := &fakeRepository{
|
|
platformRole: PlatformRoleAdmin,
|
|
testingFlags: EffectiveNodeTestingFlags{
|
|
Enabled: true,
|
|
SyntheticLinksEnabled: true,
|
|
},
|
|
routeIntents: []MeshRouteIntent{
|
|
{
|
|
ID: "route-bad",
|
|
ClusterID: "cluster-1",
|
|
SourceSelector: json.RawMessage(`{"node_id":"entry-1"}`),
|
|
DestinationSelector: json.RawMessage(`{"node_id":"exit-1"}`),
|
|
ServiceClass: FabricServiceClassVPNPackets,
|
|
Priority: 100,
|
|
Status: "active",
|
|
Policy: json.RawMessage(`{"synthetic_enabled":true,"hops":["entry-1","exit-1"],"allowed_channels":["vpn_packet","fabric_control"]}`),
|
|
UpdatedAt: now,
|
|
},
|
|
{
|
|
ID: "route-good",
|
|
ClusterID: "cluster-1",
|
|
SourceSelector: json.RawMessage(`{"node_id":"entry-1"}`),
|
|
DestinationSelector: json.RawMessage(`{"node_id":"exit-1"}`),
|
|
ServiceClass: FabricServiceClassVPNPackets,
|
|
Priority: 10,
|
|
Status: "active",
|
|
Policy: json.RawMessage(`{"synthetic_enabled":true,"hops":["entry-1","exit-1"],"allowed_channels":["vpn_packet","fabric_control"]}`),
|
|
UpdatedAt: now,
|
|
},
|
|
},
|
|
fabricLeases: map[string]FabricServiceChannelLeaseRecord{
|
|
fabricServiceChannelLeaseCacheKey("cluster-1", "channel-1"): {
|
|
ClusterID: "cluster-1",
|
|
ChannelID: "channel-1",
|
|
ResourceID: "vpn-home",
|
|
ServiceClass: FabricServiceClassVPNPackets,
|
|
SelectedEntryNodeID: "entry-1",
|
|
ExpiresAt: now.Add(time.Minute),
|
|
Lease: FabricServiceChannelLease{
|
|
ClusterID: "cluster-1",
|
|
ChannelID: "channel-1",
|
|
ResourceID: "vpn-home",
|
|
ServiceClass: FabricServiceClassVPNPackets,
|
|
Status: FabricServiceChannelStatusReady,
|
|
SelectedEntryNodeID: "entry-1",
|
|
SelectedExitNodeID: "exit-1",
|
|
PrimaryRoute: FabricServiceChannelRoute{
|
|
RouteID: "route-bad",
|
|
Status: "authorized",
|
|
},
|
|
AlternateRoutes: []FabricServiceChannelRoute{{
|
|
RouteID: "route-good",
|
|
Status: "authorized",
|
|
}},
|
|
ExpiresAt: now.Add(time.Minute),
|
|
},
|
|
},
|
|
},
|
|
}
|
|
service := NewService(store)
|
|
service.now = func() time.Time { return now }
|
|
|
|
_, err := service.RecordHeartbeat(context.Background(), RecordHeartbeatInput{
|
|
ClusterID: "cluster-1",
|
|
NodeID: "entry-1",
|
|
HealthStatus: "healthy",
|
|
Metadata: json.RawMessage(`{
|
|
"fabric_service_channel_access_report": {
|
|
"schema_version": "c18z52.fabric_service_channel_access_report.v1",
|
|
"total": 1,
|
|
"signed": 1,
|
|
"backend_fallback": 0,
|
|
"backend_fallback_blocked": 1,
|
|
"fabric_route_send_failure": 1,
|
|
"data_plane_contract": 1,
|
|
"last_backend_relay_policy": "disabled",
|
|
"last_working_data_transport": "fabric_service_channel",
|
|
"last_steady_state_transport": "fabric_route",
|
|
"last_data_plane_violation_status": "fabric_route_send_failed_backend_fallback_blocked",
|
|
"last_data_plane_violation_reason": "mesh synthetic route not found"
|
|
}
|
|
}`),
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("record heartbeat: %v", err)
|
|
}
|
|
if len(store.fabricRouteFeedback) != 1 {
|
|
t.Fatalf("route feedback count = %d, want one blocked fallback feedback: %+v", len(store.fabricRouteFeedback), store.fabricRouteFeedback)
|
|
}
|
|
feedback := store.fabricRouteFeedback[0]
|
|
if feedback.RouteID != "route-bad" ||
|
|
feedback.FeedbackStatus != "fenced" ||
|
|
feedback.ScoreAdjustment != -1030 ||
|
|
!containsString(feedback.Reasons, "data_plane_fabric_route_send_failed") ||
|
|
!containsString(feedback.Reasons, "backend_fallback_blocked_by_policy") {
|
|
t.Fatalf("unexpected route feedback from blocked fallback: %+v", feedback)
|
|
}
|
|
|
|
cfg, err := service.GetNodeSyntheticMeshConfig(context.Background(), GetNodeSyntheticMeshConfigInput{
|
|
ClusterID: "cluster-1",
|
|
NodeID: "entry-1",
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("synthetic config: %v", err)
|
|
}
|
|
if cfg.RoutePathDecisions == nil || cfg.RoutePathDecisions.ReplacementDecisionCount != 1 {
|
|
t.Fatalf("expected blocked fallback feedback to drive replacement decision: %+v", cfg.RoutePathDecisions)
|
|
}
|
|
decision := cfg.RoutePathDecisions.Decisions[0]
|
|
if decision.RouteID != "route-bad" ||
|
|
decision.ReplacementRouteID != "route-good" ||
|
|
decision.RebuildStatus != "applied" ||
|
|
decision.FeedbackObservationID == "" ||
|
|
decision.FeedbackSource != "fabric_service_channel_access_report" ||
|
|
decision.FeedbackChannelID != "channel-1" ||
|
|
decision.FeedbackResourceID != "vpn-home" ||
|
|
decision.FeedbackViolationStatus != "fabric_route_send_failed_backend_fallback_blocked" ||
|
|
!containsString(decision.ScoreReasons, "service_channel_fenced_route") ||
|
|
!containsString(decision.ScoreReasons, "service_channel_rebuild_applied") {
|
|
t.Fatalf("unexpected replacement decision: %+v", decision)
|
|
}
|
|
if len(store.fabricRebuildAttempts) != 1 {
|
|
t.Fatalf("rebuild attempt count = %d, want one correlated attempt: %+v", len(store.fabricRebuildAttempts), store.fabricRebuildAttempts)
|
|
}
|
|
attempt := store.fabricRebuildAttempts[0]
|
|
if attempt.FeedbackObservationID != decision.FeedbackObservationID ||
|
|
attempt.FeedbackSource != "fabric_service_channel_access_report" ||
|
|
attempt.FeedbackChannelID != "channel-1" ||
|
|
attempt.FeedbackResourceID != "vpn-home" ||
|
|
attempt.FeedbackViolationStatus != "fabric_route_send_failed_backend_fallback_blocked" {
|
|
t.Fatalf("unexpected rebuild attempt feedback correlation: %+v", attempt)
|
|
}
|
|
if jsonString(jsonObject(attempt.Payload), "feedback_observation_id") != decision.FeedbackObservationID ||
|
|
jsonString(jsonObject(attempt.Payload), "feedback_source") != "fabric_service_channel_access_report" {
|
|
t.Fatalf("rebuild attempt payload missing feedback correlation: %s", string(attempt.Payload))
|
|
}
|
|
health, err := service.GetFabricServiceChannelRouteRebuildHealthSummary(context.Background(), "admin-1", GetFabricServiceChannelRouteRebuildHealthSummaryInput{
|
|
ClusterID: "cluster-1",
|
|
Limit: 10,
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("get rebuild health: %v", err)
|
|
}
|
|
if len(health.FeedbackBreakdowns) != 1 {
|
|
t.Fatalf("feedback breakdowns = %+v, want one access-report group", health.FeedbackBreakdowns)
|
|
}
|
|
breakdown := health.FeedbackBreakdowns[0]
|
|
if breakdown.FeedbackSource != "fabric_service_channel_access_report" ||
|
|
breakdown.FeedbackChannelID != "channel-1" ||
|
|
breakdown.FeedbackViolationStatus != "fabric_route_send_failed_backend_fallback_blocked" ||
|
|
breakdown.TotalCount != 1 ||
|
|
len(breakdown.AffectedReporterNodeIDs) != 1 ||
|
|
breakdown.AffectedReporterNodeIDs[0] != "entry-1" ||
|
|
len(breakdown.AffectedRouteIDs) != 1 ||
|
|
breakdown.AffectedRouteIDs[0] != "route-bad" {
|
|
t.Fatalf("unexpected feedback breakdown: %+v", breakdown)
|
|
}
|
|
}
|
|
|
|
func TestRecordHeartbeatDeduplicatesBlockedFallbackAccessFeedback(t *testing.T) {
|
|
now := time.Now().UTC().Truncate(time.Second)
|
|
store := &fakeRepository{
|
|
platformRole: PlatformRoleAdmin,
|
|
testingFlags: EffectiveNodeTestingFlags{
|
|
Enabled: true,
|
|
SyntheticLinksEnabled: true,
|
|
},
|
|
routeIntents: []MeshRouteIntent{
|
|
{
|
|
ID: "route-bad",
|
|
ClusterID: "cluster-1",
|
|
SourceSelector: json.RawMessage(`{"node_id":"entry-1"}`),
|
|
DestinationSelector: json.RawMessage(`{"node_id":"exit-1"}`),
|
|
ServiceClass: FabricServiceClassVPNPackets,
|
|
Priority: 100,
|
|
Status: "active",
|
|
Policy: json.RawMessage(`{"synthetic_enabled":true,"hops":["entry-1","exit-1"],"allowed_channels":["vpn_packet","fabric_control"]}`),
|
|
UpdatedAt: now,
|
|
},
|
|
},
|
|
fabricLeases: map[string]FabricServiceChannelLeaseRecord{
|
|
fabricServiceChannelLeaseCacheKey("cluster-1", "channel-1"): {
|
|
ClusterID: "cluster-1",
|
|
ChannelID: "channel-1",
|
|
ResourceID: "vpn-home",
|
|
ServiceClass: FabricServiceClassVPNPackets,
|
|
SelectedEntryNodeID: "entry-1",
|
|
ExpiresAt: now.Add(time.Minute),
|
|
Lease: FabricServiceChannelLease{
|
|
ClusterID: "cluster-1",
|
|
ChannelID: "channel-1",
|
|
ResourceID: "vpn-home",
|
|
ServiceClass: FabricServiceClassVPNPackets,
|
|
Status: FabricServiceChannelStatusReady,
|
|
SelectedEntryNodeID: "entry-1",
|
|
SelectedExitNodeID: "exit-1",
|
|
PrimaryRoute: FabricServiceChannelRoute{
|
|
RouteID: "route-bad",
|
|
Status: "authorized",
|
|
},
|
|
ExpiresAt: now.Add(time.Minute),
|
|
},
|
|
},
|
|
},
|
|
}
|
|
service := NewService(store)
|
|
|
|
heartbeat := RecordHeartbeatInput{
|
|
ClusterID: "cluster-1",
|
|
NodeID: "entry-1",
|
|
HealthStatus: "healthy",
|
|
Metadata: json.RawMessage(`{
|
|
"fabric_service_channel_access_report": {
|
|
"schema_version": "c18z52.fabric_service_channel_access_report.v1",
|
|
"total": 1,
|
|
"signed": 1,
|
|
"backend_fallback": 0,
|
|
"backend_fallback_blocked": 1,
|
|
"fabric_route_send_failure": 1,
|
|
"data_plane_contract": 1,
|
|
"last_backend_relay_policy": "disabled",
|
|
"last_working_data_transport": "fabric_service_channel",
|
|
"last_steady_state_transport": "fabric_route",
|
|
"last_data_plane_violation_status": "fabric_route_send_failed_backend_fallback_blocked",
|
|
"last_data_plane_violation_reason": "mesh synthetic route not found"
|
|
}
|
|
}`),
|
|
}
|
|
if _, err := service.RecordHeartbeat(context.Background(), heartbeat); err != nil {
|
|
t.Fatalf("record first heartbeat: %v", err)
|
|
}
|
|
if _, err := service.RecordHeartbeat(context.Background(), heartbeat); err != nil {
|
|
t.Fatalf("record duplicate heartbeat: %v", err)
|
|
}
|
|
if len(store.fabricRouteFeedback) != 1 {
|
|
t.Fatalf("route feedback count = %d, want duplicate access-report feedback suppressed: %+v", len(store.fabricRouteFeedback), store.fabricRouteFeedback)
|
|
}
|
|
feedback := store.fabricRouteFeedback[0]
|
|
if feedback.RouteID != "route-bad" ||
|
|
feedback.FeedbackStatus != "fenced" ||
|
|
!containsString(feedback.Reasons, "data_plane_fabric_route_send_failed") ||
|
|
jsonString(jsonObject(feedback.Payload), "source") != "fabric_service_channel_access_report" {
|
|
t.Fatalf("unexpected deduplicated feedback: %+v", feedback)
|
|
}
|
|
}
|
|
|
|
func TestRecordHeartbeatUsesRollingQualityWindowForRouteFeedback(t *testing.T) {
|
|
now := time.Now().UTC().Truncate(time.Second)
|
|
store := &fakeRepository{}
|
|
service := NewService(store)
|
|
service.now = func() time.Time { return now }
|
|
|
|
_, err := service.RecordHeartbeat(context.Background(), RecordHeartbeatInput{
|
|
ClusterID: "cluster-1",
|
|
NodeID: "entry-1",
|
|
HealthStatus: "healthy",
|
|
Metadata: json.RawMessage(`{
|
|
"fabric_service_channel_runtime_report": {
|
|
"schema_version": "c18z21.fabric_service_channel_runtime_report.v1",
|
|
"ingress": {
|
|
"flow_scheduler": {
|
|
"channel_stats": {
|
|
"vpn:vpn-1:flow-01": {
|
|
"last_route_id": "route-good",
|
|
"last_failed_route_id": "route-bad",
|
|
"last_error": "old failure",
|
|
"consecutive_failures": 2,
|
|
"stall_count": 2,
|
|
"last_send_duration_ms": 1500,
|
|
"route_rebuild_recommended": true,
|
|
"degraded_fallback_recommended": true,
|
|
"quality_window_sample_count": 32,
|
|
"quality_window_success_count": 32,
|
|
"quality_window_failure_count": 0,
|
|
"quality_window_slow_count": 0,
|
|
"quality_window_drop_count": 0,
|
|
"quality_window_avg_latency_ms": 1
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}`),
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("record heartbeat: %v", err)
|
|
}
|
|
if len(store.fabricRouteFeedback) != 1 {
|
|
t.Fatalf("route feedback count = %d, want one healthy fresh observation: %+v", len(store.fabricRouteFeedback), store.fabricRouteFeedback)
|
|
}
|
|
observation := store.fabricRouteFeedback[0]
|
|
if observation.RouteID != "route-good" || observation.FeedbackStatus != "healthy" {
|
|
t.Fatalf("route feedback = %+v, want rolling window to ignore old failed route", observation)
|
|
}
|
|
if observation.ConsecutiveFailures != 0 || observation.StallCount != 0 || observation.LastSendDurationMs != 1 {
|
|
t.Fatalf("rolling counters = failures:%d stalls:%d latency:%d, want fresh window values", observation.ConsecutiveFailures, observation.StallCount, observation.LastSendDurationMs)
|
|
}
|
|
if !containsString(observation.Reasons, "service_channel_rolling_quality_window") || !containsString(observation.Reasons, "service_channel_quality_latency_le_10ms") {
|
|
t.Fatalf("feedback reasons = %+v, want rolling window quality reasons", observation.Reasons)
|
|
}
|
|
}
|
|
|
|
func TestGetNodeSyntheticMeshConfigSkipsFencedServiceChannelRoute(t *testing.T) {
|
|
now := time.Now().UTC().Truncate(time.Second)
|
|
store := &fakeRepository{
|
|
platformRole: PlatformRoleAdmin,
|
|
testingFlags: EffectiveNodeTestingFlags{
|
|
Enabled: true,
|
|
SyntheticLinksEnabled: true,
|
|
},
|
|
routeIntents: []MeshRouteIntent{
|
|
{
|
|
ID: "route-bad",
|
|
ClusterID: "cluster-1",
|
|
SourceSelector: json.RawMessage(`{"node_id":"entry-1"}`),
|
|
DestinationSelector: json.RawMessage(`{"node_id":"exit-1"}`),
|
|
ServiceClass: FabricServiceClassVPNPackets,
|
|
Priority: 100,
|
|
Status: "active",
|
|
Policy: json.RawMessage(`{
|
|
"synthetic_enabled": true,
|
|
"hops": ["entry-1", "exit-1"],
|
|
"allowed_channels": ["vpn_packet", "fabric_control"]
|
|
}`),
|
|
UpdatedAt: now,
|
|
},
|
|
{
|
|
ID: "route-good",
|
|
ClusterID: "cluster-1",
|
|
SourceSelector: json.RawMessage(`{"node_id":"entry-1"}`),
|
|
DestinationSelector: json.RawMessage(`{"node_id":"exit-1"}`),
|
|
ServiceClass: FabricServiceClassVPNPackets,
|
|
Priority: 10,
|
|
Status: "active",
|
|
Policy: json.RawMessage(`{
|
|
"synthetic_enabled": true,
|
|
"hops": ["entry-1", "exit-1"],
|
|
"allowed_channels": ["vpn_packet", "fabric_control"]
|
|
}`),
|
|
UpdatedAt: now,
|
|
},
|
|
{
|
|
ID: "route-unproven",
|
|
ClusterID: "cluster-1",
|
|
SourceSelector: json.RawMessage(`{"node_id":"entry-1"}`),
|
|
DestinationSelector: json.RawMessage(`{"node_id":"exit-1"}`),
|
|
ServiceClass: FabricServiceClassVPNPackets,
|
|
Priority: 900,
|
|
Status: "active",
|
|
Policy: json.RawMessage(`{
|
|
"synthetic_enabled": true,
|
|
"hops": ["entry-1", "exit-1"],
|
|
"allowed_channels": ["vpn_packet", "fabric_control"]
|
|
}`),
|
|
UpdatedAt: now,
|
|
},
|
|
},
|
|
fabricRouteFeedback: []FabricServiceChannelRouteFeedbackObservation{
|
|
{
|
|
ClusterID: "cluster-1",
|
|
ReporterNodeID: "entry-1",
|
|
RouteID: "route-bad",
|
|
ServiceClass: FabricServiceClassVPNPackets,
|
|
FeedbackStatus: "fenced",
|
|
ScoreAdjustment: -1030,
|
|
Reasons: []string{"service_channel_route_rebuild_recommended"},
|
|
ConsecutiveFailures: 2,
|
|
ObservedAt: now,
|
|
ExpiresAt: now.Add(time.Minute),
|
|
},
|
|
{
|
|
ClusterID: "cluster-1",
|
|
ReporterNodeID: "entry-1",
|
|
RouteID: "route-good",
|
|
ServiceClass: FabricServiceClassVPNPackets,
|
|
FeedbackStatus: "healthy",
|
|
ScoreAdjustment: 10,
|
|
Reasons: []string{"service_channel_recent_success"},
|
|
ObservedAt: now,
|
|
ExpiresAt: now.Add(time.Minute),
|
|
},
|
|
},
|
|
fabricLeases: map[string]FabricServiceChannelLeaseRecord{
|
|
fabricServiceChannelLeaseCacheKey("cluster-1", "channel-1"): {
|
|
ClusterID: "cluster-1",
|
|
ChannelID: "channel-1",
|
|
ResourceID: "vpn-home",
|
|
ServiceClass: FabricServiceClassVPNPackets,
|
|
SelectedEntryNodeID: "entry-1",
|
|
ExpiresAt: now.Add(time.Minute),
|
|
Lease: FabricServiceChannelLease{
|
|
ClusterID: "cluster-1",
|
|
ChannelID: "channel-1",
|
|
ResourceID: "vpn-home",
|
|
ServiceClass: FabricServiceClassVPNPackets,
|
|
Status: FabricServiceChannelStatusReady,
|
|
SelectedEntryNodeID: "entry-1",
|
|
SelectedExitNodeID: "exit-1",
|
|
PrimaryRoute: FabricServiceChannelRoute{
|
|
RouteID: "route-bad",
|
|
Status: "authorized",
|
|
},
|
|
AlternateRoutes: []FabricServiceChannelRoute{{
|
|
RouteID: "route-good",
|
|
Status: "authorized",
|
|
}},
|
|
ExpiresAt: now.Add(time.Minute),
|
|
},
|
|
},
|
|
},
|
|
}
|
|
service := NewService(store)
|
|
service.now = func() time.Time { return now }
|
|
|
|
cfg, err := service.GetNodeSyntheticMeshConfig(context.Background(), GetNodeSyntheticMeshConfigInput{
|
|
ClusterID: "cluster-1",
|
|
NodeID: "entry-1",
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("synthetic config: %v", err)
|
|
}
|
|
if len(cfg.Routes) != 2 || containsRouteID(cfg.Routes, "route-bad") || !containsRouteID(cfg.Routes, "route-good") {
|
|
t.Fatalf("routes = %+v, want route-bad excluded and route-good retained", cfg.Routes)
|
|
}
|
|
if cfg.ServiceChannelFeedback == nil || cfg.ServiceChannelFeedback.FencedRouteCount != 1 || cfg.ServiceChannelFeedback.HealthyRouteCount != 1 {
|
|
t.Fatalf("feedback report missing fenced count: %+v", cfg.ServiceChannelFeedback)
|
|
}
|
|
if cfg.RoutePathDecisions == nil || cfg.RoutePathDecisions.ReplacementDecisionCount != 1 {
|
|
t.Fatalf("expected one service-channel replacement decision: %+v", cfg.RoutePathDecisions)
|
|
}
|
|
if len(cfg.ServiceChannelRemediationCommands) != 1 {
|
|
t.Fatalf("remediation commands = %+v, want one", cfg.ServiceChannelRemediationCommands)
|
|
}
|
|
command := cfg.ServiceChannelRemediationCommands[0]
|
|
if command.Action != "prefer_alternate_route" ||
|
|
command.PrimaryRouteID != "route-bad" ||
|
|
command.ReplacementRouteID != "route-good" ||
|
|
command.ChannelID != "channel-1" ||
|
|
!command.ExpiresAt.After(now) {
|
|
t.Fatalf("unexpected remediation command: %+v", command)
|
|
}
|
|
var replacement RoutePathDecision
|
|
for _, decision := range cfg.RoutePathDecisions.Decisions {
|
|
if decision.DecisionSource == "service_channel_feedback_replacement" {
|
|
replacement = decision
|
|
break
|
|
}
|
|
}
|
|
if replacement.RouteID != "route-bad" || replacement.ReplacementRouteID != "route-good" ||
|
|
replacement.RebuildStatus != "applied" ||
|
|
replacement.RebuildRequestID == "" ||
|
|
!containsString(replacement.ScoreReasons, "selected_unfenced_alternate_route") ||
|
|
!containsString(replacement.ScoreReasons, "service_channel_rebuild_applied") ||
|
|
!containsString(replacement.ScoreReasons, "active_healthy_feedback_dampening_window") {
|
|
t.Fatalf("unexpected replacement decision: %+v", replacement)
|
|
}
|
|
attempts, err := service.ListFabricServiceChannelRouteRebuildAttempts(context.Background(), "admin-1", ListFabricServiceChannelRouteRebuildAttemptsInput{
|
|
ClusterID: "cluster-1",
|
|
ReporterNodeID: "entry-1",
|
|
RouteID: "route-bad",
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("list rebuild attempts: %v", err)
|
|
}
|
|
if len(attempts) != 1 {
|
|
t.Fatalf("rebuild attempts = %+v, want one", attempts)
|
|
}
|
|
attempt := attempts[0]
|
|
if attempt.RebuildStatus != "applied" ||
|
|
attempt.Outcome != "replacement_selected" ||
|
|
attempt.ReplacementRouteID != "route-good" ||
|
|
attempt.RebuildRequestID != replacement.RebuildRequestID ||
|
|
attempt.FeedbackStatus != "fenced" ||
|
|
attempt.ConsecutiveFailures != 2 ||
|
|
!containsString(attempt.FeedbackReasons, "service_channel_route_rebuild_recommended") ||
|
|
!reflect.DeepEqual(attempt.OldHops, []string{"entry-1", "exit-1"}) ||
|
|
!reflect.DeepEqual(attempt.ReplacementHops, []string{"entry-1", "exit-1"}) {
|
|
t.Fatalf("unexpected rebuild ledger attempt: %+v", attempt)
|
|
}
|
|
}
|
|
|
|
func TestGetNodeSyntheticMeshConfigReportsRebuildPendingWhenNoAlternateExists(t *testing.T) {
|
|
now := time.Date(2026, 5, 7, 14, 0, 0, 0, time.UTC)
|
|
store := &fakeRepository{
|
|
testingFlags: EffectiveNodeTestingFlags{
|
|
Enabled: true,
|
|
SyntheticLinksEnabled: true,
|
|
},
|
|
routeIntents: []MeshRouteIntent{
|
|
{
|
|
ID: "route-bad",
|
|
ClusterID: "cluster-1",
|
|
SourceSelector: json.RawMessage(`{"node_id":"entry-1"}`),
|
|
DestinationSelector: json.RawMessage(`{"node_id":"exit-1"}`),
|
|
ServiceClass: FabricServiceClassVPNPackets,
|
|
Priority: 100,
|
|
Status: "active",
|
|
Policy: json.RawMessage(`{
|
|
"synthetic_enabled": true,
|
|
"hops": ["entry-1", "exit-1"],
|
|
"allowed_channels": ["vpn_packet", "fabric_control"]
|
|
}`),
|
|
UpdatedAt: now,
|
|
},
|
|
},
|
|
fabricRouteFeedback: []FabricServiceChannelRouteFeedbackObservation{
|
|
{
|
|
ClusterID: "cluster-1",
|
|
ReporterNodeID: "entry-1",
|
|
RouteID: "route-bad",
|
|
ServiceClass: FabricServiceClassVPNPackets,
|
|
FeedbackStatus: "fenced",
|
|
ScoreAdjustment: -1030,
|
|
Reasons: []string{"service_channel_route_rebuild_recommended", "service_channel_degraded_fallback_recommended"},
|
|
ConsecutiveFailures: 3,
|
|
ObservedAt: now,
|
|
ExpiresAt: now.Add(time.Minute),
|
|
},
|
|
},
|
|
}
|
|
service := NewService(store)
|
|
service.now = func() time.Time { return now }
|
|
|
|
cfg, err := service.GetNodeSyntheticMeshConfig(context.Background(), GetNodeSyntheticMeshConfigInput{
|
|
ClusterID: "cluster-1",
|
|
NodeID: "entry-1",
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("synthetic config: %v", err)
|
|
}
|
|
if containsRouteID(cfg.Routes, "route-bad") {
|
|
t.Fatalf("fenced route should be withheld while rebuild is pending: %+v", cfg.Routes)
|
|
}
|
|
if cfg.RoutePathDecisions == nil || cfg.RoutePathDecisions.RebuildRequestCount != 1 || cfg.RoutePathDecisions.DegradedDecisionCount != 1 {
|
|
t.Fatalf("expected rebuild/degraded decision counts: %+v", cfg.RoutePathDecisions)
|
|
}
|
|
decision := cfg.RoutePathDecisions.Decisions[0]
|
|
if decision.DecisionSource != "service_channel_feedback_no_alternate" ||
|
|
decision.RebuildStatus != "pending_degraded_fallback" ||
|
|
decision.RebuildRequestID == "" ||
|
|
decision.RebuildAttempt != 3 ||
|
|
!containsString(decision.ScoreReasons, "backend_relay_degraded_fallback_until_rebuild") {
|
|
t.Fatalf("unexpected rebuild decision: %+v", decision)
|
|
}
|
|
}
|
|
|
|
func TestGetNodeSyntheticMeshConfigReplacesFencedServiceChannelRouteAcrossExitPool(t *testing.T) {
|
|
now := time.Date(2026, 5, 7, 14, 15, 0, 0, time.UTC)
|
|
store := &fakeRepository{
|
|
testingFlags: EffectiveNodeTestingFlags{
|
|
Enabled: true,
|
|
SyntheticLinksEnabled: true,
|
|
},
|
|
routeIntents: []MeshRouteIntent{
|
|
{
|
|
ID: "route-exit-a",
|
|
ClusterID: "cluster-1",
|
|
SourceSelector: json.RawMessage(`{"node_id":"entry-1"}`),
|
|
DestinationSelector: json.RawMessage(`{"node_id":"exit-a"}`),
|
|
ServiceClass: FabricServiceClassVPNPackets,
|
|
Priority: 100,
|
|
Status: "active",
|
|
Policy: json.RawMessage(`{
|
|
"synthetic_enabled": true,
|
|
"hops": ["entry-1", "exit-a"],
|
|
"allowed_channels": ["vpn_packet", "fabric_control"],
|
|
"metadata": {"exit_pool_id": "pool-home"}
|
|
}`),
|
|
UpdatedAt: now,
|
|
},
|
|
{
|
|
ID: "route-exit-b",
|
|
ClusterID: "cluster-1",
|
|
SourceSelector: json.RawMessage(`{"node_id":"entry-1"}`),
|
|
DestinationSelector: json.RawMessage(`{"node_id":"exit-b"}`),
|
|
ServiceClass: FabricServiceClassVPNPackets,
|
|
Priority: 20,
|
|
Status: "active",
|
|
Policy: json.RawMessage(`{
|
|
"synthetic_enabled": true,
|
|
"hops": ["entry-1", "exit-b"],
|
|
"allowed_channels": ["vpn_packet", "fabric_control"],
|
|
"metadata": {"exit_pool_id": "pool-home"}
|
|
}`),
|
|
UpdatedAt: now,
|
|
},
|
|
{
|
|
ID: "route-other-pool",
|
|
ClusterID: "cluster-1",
|
|
SourceSelector: json.RawMessage(`{"node_id":"entry-1"}`),
|
|
DestinationSelector: json.RawMessage(`{"node_id":"exit-c"}`),
|
|
ServiceClass: FabricServiceClassVPNPackets,
|
|
Priority: 900,
|
|
Status: "active",
|
|
Policy: json.RawMessage(`{
|
|
"synthetic_enabled": true,
|
|
"hops": ["entry-1", "exit-c"],
|
|
"allowed_channels": ["vpn_packet", "fabric_control"],
|
|
"metadata": {"exit_pool_id": "pool-other"}
|
|
}`),
|
|
UpdatedAt: now,
|
|
},
|
|
},
|
|
fabricRouteFeedback: []FabricServiceChannelRouteFeedbackObservation{
|
|
{
|
|
ClusterID: "cluster-1",
|
|
ReporterNodeID: "entry-1",
|
|
RouteID: "route-exit-a",
|
|
ServiceClass: FabricServiceClassVPNPackets,
|
|
FeedbackStatus: "fenced",
|
|
ScoreAdjustment: -1030,
|
|
Reasons: []string{"service_channel_route_rebuild_recommended"},
|
|
ConsecutiveFailures: 2,
|
|
ObservedAt: now,
|
|
ExpiresAt: now.Add(time.Minute),
|
|
},
|
|
{
|
|
ClusterID: "cluster-1",
|
|
ReporterNodeID: "entry-1",
|
|
RouteID: "route-exit-b",
|
|
ServiceClass: FabricServiceClassVPNPackets,
|
|
FeedbackStatus: "healthy",
|
|
ScoreAdjustment: 10,
|
|
Reasons: []string{"service_channel_recent_success"},
|
|
ObservedAt: now,
|
|
ExpiresAt: now.Add(time.Minute),
|
|
},
|
|
},
|
|
}
|
|
service := NewService(store)
|
|
service.now = func() time.Time { return now }
|
|
|
|
cfg, err := service.GetNodeSyntheticMeshConfig(context.Background(), GetNodeSyntheticMeshConfigInput{
|
|
ClusterID: "cluster-1",
|
|
NodeID: "entry-1",
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("synthetic config: %v", err)
|
|
}
|
|
if cfg.RoutePathDecisions == nil || cfg.RoutePathDecisions.ReplacementDecisionCount != 1 {
|
|
t.Fatalf("expected one exit-pool replacement decision: %+v", cfg.RoutePathDecisions)
|
|
}
|
|
var replacement RoutePathDecision
|
|
for _, decision := range cfg.RoutePathDecisions.Decisions {
|
|
if decision.RouteID == "route-exit-a" {
|
|
replacement = decision
|
|
break
|
|
}
|
|
}
|
|
if replacement.ReplacementRouteID != "route-exit-b" ||
|
|
replacement.DecisionSource != "service_channel_feedback_exit_pool_replacement" ||
|
|
replacement.RebuildStatus != "applied" ||
|
|
!containsString(replacement.ScoreReasons, "selected_unfenced_exit_pool_route") {
|
|
t.Fatalf("unexpected exit-pool replacement decision: %+v", replacement)
|
|
}
|
|
}
|
|
|
|
func TestGetNodeSyntheticMeshConfigReplacesFencedServiceChannelRouteAcrossEntryPool(t *testing.T) {
|
|
now := time.Date(2026, 5, 7, 15, 5, 0, 0, time.UTC)
|
|
store := &fakeRepository{
|
|
testingFlags: EffectiveNodeTestingFlags{
|
|
Enabled: true,
|
|
SyntheticLinksEnabled: true,
|
|
},
|
|
routeIntents: []MeshRouteIntent{
|
|
{
|
|
ID: "route-entry-a",
|
|
ClusterID: "cluster-1",
|
|
SourceSelector: json.RawMessage(`{"node_id":"entry-a"}`),
|
|
DestinationSelector: json.RawMessage(`{"node_id":"exit-1"}`),
|
|
ServiceClass: FabricServiceClassVPNPackets,
|
|
Priority: 100,
|
|
Status: "active",
|
|
Policy: json.RawMessage(`{
|
|
"synthetic_enabled": true,
|
|
"hops": ["entry-a", "exit-1"],
|
|
"allowed_channels": ["vpn_packet", "fabric_control"],
|
|
"metadata": {"entry_pool_id": "pool-edge"}
|
|
}`),
|
|
UpdatedAt: now,
|
|
},
|
|
{
|
|
ID: "route-entry-b",
|
|
ClusterID: "cluster-1",
|
|
SourceSelector: json.RawMessage(`{"node_id":"entry-b"}`),
|
|
DestinationSelector: json.RawMessage(`{"node_id":"exit-1"}`),
|
|
ServiceClass: FabricServiceClassVPNPackets,
|
|
Priority: 20,
|
|
Status: "active",
|
|
Policy: json.RawMessage(`{
|
|
"synthetic_enabled": true,
|
|
"hops": ["entry-b", "exit-1"],
|
|
"allowed_channels": ["vpn_packet", "fabric_control"],
|
|
"metadata": {"entry_pool_id": "pool-edge"}
|
|
}`),
|
|
UpdatedAt: now,
|
|
},
|
|
{
|
|
ID: "route-other-pool",
|
|
ClusterID: "cluster-1",
|
|
SourceSelector: json.RawMessage(`{"node_id":"entry-c"}`),
|
|
DestinationSelector: json.RawMessage(`{"node_id":"exit-1"}`),
|
|
ServiceClass: FabricServiceClassVPNPackets,
|
|
Priority: 900,
|
|
Status: "active",
|
|
Policy: json.RawMessage(`{
|
|
"synthetic_enabled": true,
|
|
"hops": ["entry-c", "exit-1"],
|
|
"allowed_channels": ["vpn_packet", "fabric_control"],
|
|
"metadata": {"entry_pool_id": "pool-other"}
|
|
}`),
|
|
UpdatedAt: now,
|
|
},
|
|
},
|
|
fabricRouteFeedback: []FabricServiceChannelRouteFeedbackObservation{
|
|
{
|
|
ClusterID: "cluster-1",
|
|
ReporterNodeID: "exit-1",
|
|
RouteID: "route-entry-a",
|
|
ServiceClass: FabricServiceClassVPNPackets,
|
|
FeedbackStatus: "fenced",
|
|
ScoreAdjustment: -1030,
|
|
Reasons: []string{"service_channel_route_rebuild_recommended"},
|
|
ConsecutiveFailures: 2,
|
|
ObservedAt: now,
|
|
ExpiresAt: now.Add(time.Minute),
|
|
},
|
|
{
|
|
ClusterID: "cluster-1",
|
|
ReporterNodeID: "exit-1",
|
|
RouteID: "route-entry-b",
|
|
ServiceClass: FabricServiceClassVPNPackets,
|
|
FeedbackStatus: "healthy",
|
|
ScoreAdjustment: 10,
|
|
Reasons: []string{"service_channel_recent_success"},
|
|
ObservedAt: now,
|
|
ExpiresAt: now.Add(time.Minute),
|
|
},
|
|
},
|
|
}
|
|
service := NewService(store)
|
|
service.now = func() time.Time { return now }
|
|
|
|
cfg, err := service.GetNodeSyntheticMeshConfig(context.Background(), GetNodeSyntheticMeshConfigInput{
|
|
ClusterID: "cluster-1",
|
|
NodeID: "exit-1",
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("synthetic config: %v", err)
|
|
}
|
|
if cfg.RoutePathDecisions == nil || cfg.RoutePathDecisions.ReplacementDecisionCount != 1 {
|
|
t.Fatalf("expected one entry-pool replacement decision: %+v", cfg.RoutePathDecisions)
|
|
}
|
|
var replacement RoutePathDecision
|
|
for _, decision := range cfg.RoutePathDecisions.Decisions {
|
|
if decision.RouteID == "route-entry-a" {
|
|
replacement = decision
|
|
break
|
|
}
|
|
}
|
|
if replacement.ReplacementRouteID != "route-entry-b" ||
|
|
replacement.DecisionSource != "service_channel_feedback_entry_pool_replacement" ||
|
|
replacement.RebuildStatus != "applied" ||
|
|
!containsString(replacement.ScoreReasons, "selected_unfenced_entry_pool_route") {
|
|
t.Fatalf("unexpected entry-pool replacement decision: %+v", replacement)
|
|
}
|
|
if replacement.LocalRole != "exit" || replacement.PreviousHopID != "entry-b" {
|
|
t.Fatalf("entry-pool replacement should be visible from shared exit perspective: %+v", replacement)
|
|
}
|
|
}
|
|
|
|
func TestExpireFabricServiceChannelRouteFeedbackRemovesActiveFeedback(t *testing.T) {
|
|
now := time.Date(2026, 5, 7, 12, 0, 0, 0, time.UTC)
|
|
store := &fakeRepository{
|
|
platformRole: PlatformRoleAdmin,
|
|
authorityState: ClusterAuthorityState{
|
|
ClusterID: "cluster-1",
|
|
AuthorityState: "authoritative",
|
|
MutationMode: "normal",
|
|
},
|
|
fabricRouteFeedback: []FabricServiceChannelRouteFeedbackObservation{
|
|
{
|
|
ClusterID: "cluster-1",
|
|
ReporterNodeID: "entry-1",
|
|
RouteID: "route-bad",
|
|
ServiceClass: FabricServiceClassVPNPackets,
|
|
FeedbackStatus: "fenced",
|
|
ObservedAt: now.Add(-time.Minute),
|
|
ExpiresAt: now.Add(time.Minute),
|
|
},
|
|
{
|
|
ClusterID: "cluster-1",
|
|
ReporterNodeID: "entry-2",
|
|
RouteID: "route-other",
|
|
ServiceClass: FabricServiceClassVPNPackets,
|
|
FeedbackStatus: "healthy",
|
|
ObservedAt: now.Add(-time.Minute),
|
|
ExpiresAt: now.Add(time.Minute),
|
|
},
|
|
},
|
|
}
|
|
service := NewService(store)
|
|
service.now = func() time.Time { return now }
|
|
|
|
result, err := service.ExpireFabricServiceChannelRouteFeedback(context.Background(), ExpireFabricServiceChannelRouteFeedbackInput{
|
|
ActorUserID: "admin-1",
|
|
ClusterID: "cluster-1",
|
|
ReporterNodeID: "entry-1",
|
|
RouteID: "route-bad",
|
|
ServiceClass: FabricServiceClassVPNPackets,
|
|
Reason: "operator verified route is healthy",
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("expire feedback: %v", err)
|
|
}
|
|
if result.ExpiredCount != 1 || !result.ExpiredAt.Equal(now) || !result.CooldownUntil.Equal(now.Add(fabricServiceChannelOperatorExpireCooldown)) {
|
|
t.Fatalf("unexpected expire result: %+v", result)
|
|
}
|
|
if len(store.auditEvents) == 0 || store.auditEvents[len(store.auditEvents)-1].EventType != "fabric.service_channel_route_feedback.expired" {
|
|
t.Fatalf("missing feedback expire audit event: %+v", store.auditEvents)
|
|
}
|
|
active, err := service.ListFabricServiceChannelRouteFeedback(context.Background(), "admin-1", ListFabricServiceChannelRouteFeedbackInput{
|
|
ClusterID: "cluster-1",
|
|
Now: now,
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("list feedback: %v", err)
|
|
}
|
|
if len(active) != 1 || active[0].RouteID != "route-other" {
|
|
t.Fatalf("active feedback = %+v, want only route-other", active)
|
|
}
|
|
expired, err := service.ListFabricServiceChannelRouteFeedback(context.Background(), "admin-1", ListFabricServiceChannelRouteFeedbackInput{
|
|
ClusterID: "cluster-1",
|
|
RouteID: "route-bad",
|
|
IncludeExpired: true,
|
|
Now: now,
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("list expired feedback: %v", err)
|
|
}
|
|
if len(expired) != 1 || !expired[0].ExpiresAt.Equal(now) {
|
|
t.Fatalf("expired feedback = %+v, want route-bad expired at now", expired)
|
|
}
|
|
}
|
|
|
|
func TestRecordFabricServiceChannelRouteRebuildFeedbackBreakdownInvestigationAudit(t *testing.T) {
|
|
now := time.Date(2026, 5, 9, 13, 30, 0, 0, time.UTC)
|
|
store := &fakeRepository{platformRole: "platform_admin"}
|
|
service := NewService(store)
|
|
service.now = func() time.Time { return now }
|
|
|
|
err := service.RecordFabricServiceChannelRouteRebuildInvestigation(context.Background(), RecordFabricServiceChannelRouteRebuildInvestigationInput{
|
|
ActorUserID: "admin-1",
|
|
ClusterID: "cluster-1",
|
|
FeedbackSource: "fabric_service_channel_access_report",
|
|
FeedbackChannelID: "channel-1",
|
|
FeedbackViolationStatus: "fabric_route_send_failed_backend_fallback_blocked",
|
|
DrilldownSource: "rebuild_health_feedback_breakdown",
|
|
Reason: "operator opened rebuild-health feedback breakdown ledger",
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("record investigation: %v", err)
|
|
}
|
|
if len(store.auditEvents) != 1 {
|
|
t.Fatalf("audit events = %d, want 1", len(store.auditEvents))
|
|
}
|
|
event := store.auditEvents[0]
|
|
if event.EventType != "fabric.service_channel_rebuild_feedback_breakdown.investigation_opened" {
|
|
t.Fatalf("event type = %q", event.EventType)
|
|
}
|
|
if event.TargetType != "fabric_service_channel_rebuild_feedback_breakdown" || event.TargetID == nil || *event.TargetID != "channel-1" {
|
|
t.Fatalf("unexpected target: type=%q id=%v", event.TargetType, event.TargetID)
|
|
}
|
|
payload := jsonObject(event.Payload)
|
|
if jsonString(payload, "feedback_source") != "fabric_service_channel_access_report" ||
|
|
jsonString(payload, "feedback_channel_id") != "channel-1" ||
|
|
jsonString(payload, "feedback_violation_status") != "fabric_route_send_failed_backend_fallback_blocked" ||
|
|
jsonString(payload, "drilldown_source") != "rebuild_health_feedback_breakdown" {
|
|
t.Fatalf("unexpected audit payload: %s", string(event.Payload))
|
|
}
|
|
if !event.CreatedAt.Equal(now) {
|
|
t.Fatalf("created_at = %s, want %s", event.CreatedAt, now)
|
|
}
|
|
}
|
|
|
|
func TestListAuditEventsFiltersFabricInvestigationBreadcrumbs(t *testing.T) {
|
|
clusterID := "cluster-1"
|
|
otherClusterID := "cluster-other"
|
|
now := time.Date(2026, 5, 9, 14, 20, 0, 0, time.UTC)
|
|
store := &fakeRepository{
|
|
platformRole: "platform_admin",
|
|
fabricRebuildAttempts: []FabricServiceChannelRouteRebuildAttempt{
|
|
{
|
|
ID: "attempt-1",
|
|
ClusterID: clusterID,
|
|
ReporterNodeID: "entry-1",
|
|
RouteID: "route-1",
|
|
ServiceClass: FabricServiceClassVPNPackets,
|
|
RebuildStatus: "applied",
|
|
Outcome: "replacement_selected",
|
|
FeedbackSource: "fabric_service_channel_access_report",
|
|
FeedbackChannelID: "channel-1",
|
|
FeedbackViolationStatus: "fabric_route_send_failed_backend_fallback_blocked",
|
|
FeedbackObservedAt: &now,
|
|
UpdatedAt: now,
|
|
CreatedAt: now,
|
|
Payload: json.RawMessage(`{}`),
|
|
},
|
|
},
|
|
auditEvents: []ClusterAuditEvent{
|
|
{
|
|
ID: "audit-1",
|
|
ClusterID: &clusterID,
|
|
EventType: "fabric.service_channel_rebuild_feedback_breakdown.investigation_opened",
|
|
TargetType: "fabric_service_channel_rebuild_feedback_breakdown",
|
|
TargetID: stringPtr("channel-1"),
|
|
Payload: json.RawMessage(`{"feedback_source":"fabric_service_channel_access_report","feedback_channel_id":"channel-1","feedback_violation_status":"fabric_route_send_failed_backend_fallback_blocked"}`),
|
|
CreatedAt: now.Add(-5 * time.Minute),
|
|
},
|
|
{
|
|
ID: "audit-2",
|
|
ClusterID: &clusterID,
|
|
EventType: "fabric.service_channel_rebuild_incident.investigation_opened",
|
|
TargetType: "fabric_service_channel_route_rebuild_incident",
|
|
TargetID: stringPtr("route-1"),
|
|
Payload: json.RawMessage(`{}`),
|
|
CreatedAt: now.Add(-2 * time.Hour),
|
|
},
|
|
{
|
|
ID: "audit-3",
|
|
ClusterID: &clusterID,
|
|
EventType: "fabric.service_channel_route_feedback.expired",
|
|
TargetType: "fabric_service_channel_route_feedback",
|
|
TargetID: stringPtr("route-2"),
|
|
Payload: json.RawMessage(`{}`),
|
|
CreatedAt: now,
|
|
},
|
|
{
|
|
ID: "audit-4",
|
|
ClusterID: &otherClusterID,
|
|
EventType: "fabric.service_channel_rebuild_feedback_breakdown.investigation_opened",
|
|
TargetType: "fabric_service_channel_rebuild_feedback_breakdown",
|
|
TargetID: stringPtr("channel-other"),
|
|
Payload: json.RawMessage(`{}`),
|
|
CreatedAt: now,
|
|
},
|
|
},
|
|
}
|
|
service := NewService(store)
|
|
service.now = func() time.Time { return now }
|
|
|
|
events, err := service.ListAuditEvents(context.Background(), "admin-1", ListAuditEventsInput{
|
|
ClusterID: clusterID,
|
|
EventTypes: []string{
|
|
"fabric.service_channel_rebuild_feedback_breakdown.investigation_opened",
|
|
"fabric.service_channel_rebuild_incident.investigation_opened",
|
|
},
|
|
Correlation: "fabric_diagnostics",
|
|
Limit: 10,
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("list audit events: %v", err)
|
|
}
|
|
if len(events) != 2 || events[0].ID != "audit-1" || events[1].ID != "audit-2" {
|
|
t.Fatalf("events = %+v, want only fabric investigation breadcrumbs", events)
|
|
}
|
|
if events[0].CorrelationHints == nil || events[0].CorrelationHints.CurrentDiagnosticStatus != "breakdown_active" ||
|
|
events[0].CorrelationHints.FeedbackBreakdown == nil || events[0].CorrelationHints.FeedbackBreakdown.FeedbackChannelID != "channel-1" {
|
|
t.Fatalf("feedback breadcrumb correlation hints = %+v", events[0].CorrelationHints)
|
|
}
|
|
|
|
breadcrumbs, err := service.ListFabricServiceChannelRebuildInvestigationBreadcrumbs(context.Background(), "admin-1", ListFabricServiceChannelRebuildInvestigationBreadcrumbsInput{
|
|
ClusterID: clusterID,
|
|
Limit: 10,
|
|
CurrentWindowSeconds: int64((30 * time.Minute).Seconds()),
|
|
HistoryWindowSeconds: int64((24 * time.Hour).Seconds()),
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("list breadcrumbs: %v", err)
|
|
}
|
|
if len(breadcrumbs.Events) != 2 || breadcrumbs.Summary.TotalCount != 2 || breadcrumbs.Summary.CorrelatedCount != 2 {
|
|
t.Fatalf("breadcrumbs = %+v", breadcrumbs)
|
|
}
|
|
if breadcrumbs.Summary.CountsByCurrentDiagnosticStatus["breakdown_active"] != 1 ||
|
|
breadcrumbs.Summary.CountsByCurrentDiagnosticStatus["incident_visible"] != 1 {
|
|
t.Fatalf("breadcrumb summary statuses = %+v", breadcrumbs.Summary.CountsByCurrentDiagnosticStatus)
|
|
}
|
|
if breadcrumbs.CurrentCount != 1 || breadcrumbs.StaleCount != 1 || breadcrumbs.ExpiredCount != 0 ||
|
|
breadcrumbs.Summary.CountsByBreadcrumbStatus["current"] != 1 ||
|
|
breadcrumbs.Summary.CountsByBreadcrumbStatus["stale"] != 1 {
|
|
t.Fatalf("breadcrumb freshness = %+v summary=%+v", breadcrumbs, breadcrumbs.Summary.CountsByBreadcrumbStatus)
|
|
}
|
|
}
|
|
|
|
func TestRebuildHealthSilenceIsGenerationScoped(t *testing.T) {
|
|
now := time.Date(2026, 5, 8, 12, 0, 0, 0, time.UTC)
|
|
store := &fakeRepository{
|
|
platformRole: "platform_admin",
|
|
fabricRebuildAttempts: []FabricServiceChannelRouteRebuildAttempt{
|
|
{
|
|
ID: "attempt-old",
|
|
ClusterID: "cluster-1",
|
|
ReporterNodeID: "entry-1",
|
|
ServiceClass: FabricServiceClassVPNPackets,
|
|
RouteID: "route-1",
|
|
RebuildStatus: "applied",
|
|
Generation: "gen-old",
|
|
UpdatedAt: now.Add(-5 * time.Minute),
|
|
CreatedAt: now.Add(-5 * time.Minute),
|
|
Payload: json.RawMessage(`{}`),
|
|
},
|
|
{
|
|
ID: "attempt-new",
|
|
ClusterID: "cluster-1",
|
|
ReporterNodeID: "entry-1",
|
|
ServiceClass: FabricServiceClassVPNPackets,
|
|
RouteID: "route-1",
|
|
RebuildStatus: "applied",
|
|
Generation: "gen-new",
|
|
UpdatedAt: now.Add(-4 * time.Minute),
|
|
CreatedAt: now.Add(-4 * time.Minute),
|
|
Payload: json.RawMessage(`{}`),
|
|
},
|
|
},
|
|
}
|
|
service := NewService(store)
|
|
service.now = func() time.Time { return now }
|
|
_, err := service.SilenceFabricServiceChannelRouteRebuildAlert(context.Background(), SilenceFabricServiceChannelRouteRebuildAlertInput{
|
|
ActorUserID: "admin-1",
|
|
ClusterID: "cluster-1",
|
|
ReporterNodeID: "entry-1",
|
|
RouteID: "route-1",
|
|
GuardStatus: "missing_node_transition",
|
|
Generation: "gen-old",
|
|
Reason: "known old test route",
|
|
TTL: time.Hour,
|
|
Now: now,
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("silence rebuild alert: %v", err)
|
|
}
|
|
summary, err := service.GetFabricServiceChannelRouteRebuildHealthSummary(context.Background(), "admin-1", GetFabricServiceChannelRouteRebuildHealthSummaryInput{
|
|
ClusterID: "cluster-1",
|
|
Limit: 20,
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("get rebuild health: %v", err)
|
|
}
|
|
if summary.BadCount != 2 || summary.SilencedCount != 1 || summary.ActiveBadCount != 1 {
|
|
t.Fatalf("summary counts = %+v, want bad=2 silenced=1 active_bad=1", summary)
|
|
}
|
|
if len(summary.MostRecentBadAttempts) != 1 || summary.MostRecentBadAttempts[0].Generation != "gen-new" {
|
|
t.Fatalf("active bad attempts = %+v, want only fresh generation", summary.MostRecentBadAttempts)
|
|
}
|
|
if summary.ResurfacedCount != 1 || len(summary.ResurfacedAttempts) != 1 || summary.ResurfacedAttempts[0].AlertResurfacedPreviousGeneration != "gen-old" {
|
|
t.Fatalf("resurfaced attempts = %+v / count %d, want gen-new resurfaced from gen-old", summary.ResurfacedAttempts, summary.ResurfacedCount)
|
|
}
|
|
readiness, err := service.GetFabricServiceChannelReadiness(context.Background(), "admin-1", GetFabricServiceChannelReadinessInput{
|
|
ClusterID: "cluster-1",
|
|
Limit: 20,
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("get readiness: %v", err)
|
|
}
|
|
if readiness.Status != "blocked" || readiness.Reason != "resurfaced_rebuild_alert" || readiness.ResurfacedCount != 1 {
|
|
t.Fatalf("readiness = %+v, want blocked by resurfaced alert", readiness)
|
|
}
|
|
}
|
|
|
|
func TestOperatorExpiredFabricServiceChannelFeedbackAllowsRetryAndSuppressesImmediateChurn(t *testing.T) {
|
|
now := time.Date(2026, 5, 7, 12, 30, 0, 0, time.UTC)
|
|
cooldownUntil := now.Add(fabricServiceChannelOperatorExpireCooldown)
|
|
store := &fakeRepository{
|
|
platformRole: PlatformRoleAdmin,
|
|
authorityState: ClusterAuthorityState{
|
|
ClusterID: "cluster-1",
|
|
AuthorityState: "authoritative",
|
|
MutationMode: "normal",
|
|
},
|
|
testingFlags: EffectiveNodeTestingFlags{
|
|
Enabled: true,
|
|
SyntheticLinksEnabled: true,
|
|
},
|
|
routeIntents: []MeshRouteIntent{
|
|
{
|
|
ID: "route-retry",
|
|
ClusterID: "cluster-1",
|
|
SourceSelector: json.RawMessage(`{"node_id":"entry-1"}`),
|
|
DestinationSelector: json.RawMessage(`{"node_id":"exit-1"}`),
|
|
ServiceClass: FabricServiceClassVPNPackets,
|
|
Priority: 100,
|
|
Status: "active",
|
|
Policy: json.RawMessage(`{
|
|
"synthetic_enabled": true,
|
|
"hops": ["entry-1", "exit-1"],
|
|
"allowed_channels": ["vpn_packet", "fabric_control"]
|
|
}`),
|
|
UpdatedAt: now,
|
|
},
|
|
},
|
|
fabricRouteFeedback: []FabricServiceChannelRouteFeedbackObservation{
|
|
{
|
|
ClusterID: "cluster-1",
|
|
ReporterNodeID: "entry-1",
|
|
RouteID: "route-retry",
|
|
ServiceClass: FabricServiceClassVPNPackets,
|
|
FeedbackStatus: "fenced",
|
|
ScoreAdjustment: -1030,
|
|
Reasons: []string{"service_channel_route_rebuild_recommended"},
|
|
ObservedAt: now.Add(-time.Minute),
|
|
ExpiresAt: now,
|
|
RetryCooldownUntil: &cooldownUntil,
|
|
},
|
|
},
|
|
}
|
|
service := NewService(store)
|
|
service.now = func() time.Time { return now }
|
|
|
|
cfg, err := service.GetNodeSyntheticMeshConfig(context.Background(), GetNodeSyntheticMeshConfigInput{
|
|
ClusterID: "cluster-1",
|
|
NodeID: "entry-1",
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("synthetic config: %v", err)
|
|
}
|
|
if !containsRouteID(cfg.Routes, "route-retry") {
|
|
t.Fatalf("route-retry should be retried during operator cooldown: %+v", cfg.Routes)
|
|
}
|
|
if cfg.RoutePathDecisions == nil || len(cfg.RoutePathDecisions.Decisions) != 1 ||
|
|
!containsString(cfg.RoutePathDecisions.Decisions[0].ScoreReasons, "service_channel_route_retry_after_operator_expire") {
|
|
t.Fatalf("missing manual retry decision reason: %+v", cfg.RoutePathDecisions)
|
|
}
|
|
|
|
_, err = store.RecordFabricServiceChannelRouteFeedback(context.Background(), RecordFabricServiceChannelRouteFeedbackInput{
|
|
ClusterID: "cluster-1",
|
|
ReporterNodeID: "entry-1",
|
|
RouteID: "route-retry",
|
|
ServiceClass: FabricServiceClassVPNPackets,
|
|
FeedbackStatus: "fenced",
|
|
ScoreAdjustment: -1030,
|
|
Reasons: []string{"service_channel_route_rebuild_recommended"},
|
|
ObservedAt: now.Add(30 * time.Second),
|
|
ExpiresAt: now.Add(3 * time.Minute),
|
|
Payload: json.RawMessage(`{"last_error":"retry failed"}`),
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("record feedback: %v", err)
|
|
}
|
|
active, err := service.ListFabricServiceChannelRouteFeedback(context.Background(), "admin-1", ListFabricServiceChannelRouteFeedbackInput{
|
|
ClusterID: "cluster-1",
|
|
RouteID: "route-retry",
|
|
Now: now.Add(30 * time.Second),
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("list feedback: %v", err)
|
|
}
|
|
if len(active) != 1 || active[0].FeedbackStatus != "operator_retry_cooldown" || active[0].ScoreAdjustment != 0 {
|
|
t.Fatalf("feedback not suppressed during cooldown: %+v", active)
|
|
}
|
|
}
|
|
|
|
func TestIssueFabricServiceChannelLeaseDampensRecoveredRouteDuringRetryCooldown(t *testing.T) {
|
|
now := time.Now().UTC().Truncate(time.Second)
|
|
cooldownUntil := now.Add(2 * time.Minute)
|
|
store := &fakeRepository{
|
|
routeIntents: []MeshRouteIntent{
|
|
{
|
|
ID: "route-recovered",
|
|
ClusterID: "cluster-1",
|
|
SourceSelector: json.RawMessage(`{"node_id":"entry-1"}`),
|
|
DestinationSelector: json.RawMessage(`{"node_id":"exit-1"}`),
|
|
ServiceClass: FabricServiceClassVPNPackets,
|
|
Priority: 100,
|
|
Status: "active",
|
|
Policy: json.RawMessage(`{
|
|
"hops": ["entry-1", "exit-1"],
|
|
"allowed_channels": ["vpn_packet", "fabric_control"]
|
|
}`),
|
|
UpdatedAt: now,
|
|
},
|
|
{
|
|
ID: "route-steady",
|
|
ClusterID: "cluster-1",
|
|
SourceSelector: json.RawMessage(`{"node_id":"entry-1"}`),
|
|
DestinationSelector: json.RawMessage(`{"node_id":"exit-1"}`),
|
|
ServiceClass: FabricServiceClassVPNPackets,
|
|
Priority: 80,
|
|
Status: "active",
|
|
Policy: json.RawMessage(`{
|
|
"hops": ["entry-1", "exit-1"],
|
|
"allowed_channels": ["vpn_packet", "fabric_control"]
|
|
}`),
|
|
UpdatedAt: now,
|
|
},
|
|
},
|
|
fabricRouteFeedback: []FabricServiceChannelRouteFeedbackObservation{
|
|
{
|
|
ClusterID: "cluster-1",
|
|
ReporterNodeID: "entry-1",
|
|
RouteID: "route-recovered",
|
|
ServiceClass: FabricServiceClassVPNPackets,
|
|
FeedbackStatus: "fenced",
|
|
ScoreAdjustment: -1030,
|
|
Reasons: []string{"service_channel_route_rebuild_recommended"},
|
|
ObservedAt: now.Add(-time.Minute),
|
|
ExpiresAt: now,
|
|
RetryCooldownUntil: &cooldownUntil,
|
|
},
|
|
{
|
|
ClusterID: "cluster-1",
|
|
ReporterNodeID: "entry-1",
|
|
RouteID: "route-recovered",
|
|
ServiceClass: FabricServiceClassVPNPackets,
|
|
FeedbackStatus: "healthy",
|
|
ScoreAdjustment: 90,
|
|
Reasons: []string{"service_channel_recent_success", "service_channel_quality_latency_le_10ms", "service_channel_rolling_quality_window"},
|
|
LastSendDurationMs: 1,
|
|
Payload: json.RawMessage(`{"quality_window_sample_count":32,"quality_window_success_count":32,"quality_window_failure_count":0,"quality_window_drop_count":0,"quality_window_avg_latency_ms":1}`),
|
|
ObservedAt: now,
|
|
ExpiresAt: now.Add(2 * time.Minute),
|
|
},
|
|
},
|
|
}
|
|
service := NewService(store)
|
|
service.now = func() time.Time { return now }
|
|
|
|
lease, err := service.IssueFabricServiceChannelLease(context.Background(), IssueFabricServiceChannelLeaseInput{
|
|
ClusterID: "cluster-1",
|
|
OrganizationID: "org-home",
|
|
UserID: "user-m",
|
|
ResourceID: "vpn-home",
|
|
ServiceClass: FabricServiceClassVPNPackets,
|
|
EntryNodeIDs: []string{"entry-1"},
|
|
ExitNodeIDs: []string{"exit-1"},
|
|
PreferredEntryNodeID: "entry-1",
|
|
PreferredExitNodeID: "exit-1",
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("issue lease: %v", err)
|
|
}
|
|
if lease.PrimaryRoute.RouteID != "route-steady" {
|
|
t.Fatalf("primary route = %q, want steady route while recovered route is dampened", lease.PrimaryRoute.RouteID)
|
|
}
|
|
var recovered FabricServiceChannelRoute
|
|
for _, route := range append([]FabricServiceChannelRoute{lease.PrimaryRoute}, lease.AlternateRoutes...) {
|
|
if route.RouteID == "route-recovered" {
|
|
recovered = route
|
|
break
|
|
}
|
|
}
|
|
if recovered.RouteID == "" || recovered.Status != "authorized" {
|
|
t.Fatalf("recovered route should be authorized alternate during hysteresis: primary=%+v alternates=%+v", lease.PrimaryRoute, lease.AlternateRoutes)
|
|
}
|
|
if recovered.RecoveryState != "recovered" || recovered.RecoveryPenalty != fabricServiceChannelRecoveryHysteresisPenalty {
|
|
t.Fatalf("recovered telemetry state=%q penalty=%d, want recovered penalty %d", recovered.RecoveryState, recovered.RecoveryPenalty, fabricServiceChannelRecoveryHysteresisPenalty)
|
|
}
|
|
if !containsString(recovered.ScoreReasons, "service_channel_recovery_hysteresis") ||
|
|
!containsString(recovered.ScoreReasons, "service_channel_rolling_quality_window") ||
|
|
!containsString(recovered.ScoreReasons, "manual_feedback_expired_retry_cooldown") {
|
|
t.Fatalf("recovered route score reasons = %+v, want hysteresis + rolling feedback reasons", recovered.ScoreReasons)
|
|
}
|
|
if recovered.PathScore >= lease.PrimaryRoute.PathScore {
|
|
t.Fatalf("recovered score = %d primary score = %d, want recovered route dampened below steady primary", recovered.PathScore, lease.PrimaryRoute.PathScore)
|
|
}
|
|
}
|
|
|
|
func TestServiceChannelRouteFeedbackReportExposesRecoveryState(t *testing.T) {
|
|
now := time.Now().UTC().Truncate(time.Second)
|
|
cooldownUntil := now.Add(2 * time.Minute)
|
|
report := serviceChannelRouteFeedbackReport([]FabricServiceChannelRouteFeedbackObservation{
|
|
{
|
|
ClusterID: "cluster-1",
|
|
ReporterNodeID: "entry-1",
|
|
RouteID: "route-recovered",
|
|
ServiceClass: FabricServiceClassVPNPackets,
|
|
FeedbackStatus: "healthy",
|
|
ScoreAdjustment: 90,
|
|
Reasons: []string{"service_channel_recent_success", "service_channel_rolling_quality_window"},
|
|
ObservedAt: now,
|
|
ExpiresAt: now.Add(2 * time.Minute),
|
|
RetryCooldownUntil: &cooldownUntil,
|
|
},
|
|
{
|
|
ClusterID: "cluster-1",
|
|
ReporterNodeID: "entry-1",
|
|
RouteID: "route-promoted",
|
|
ServiceClass: FabricServiceClassVPNPackets,
|
|
FeedbackStatus: "healthy",
|
|
ScoreAdjustment: 90,
|
|
Reasons: []string{"service_channel_recent_success", "service_channel_rolling_quality_window"},
|
|
Payload: json.RawMessage(`{"quality_window_sample_count":96,"quality_window_success_count":96,"quality_window_failure_count":0,"quality_window_slow_count":0,"quality_window_drop_count":0}`),
|
|
ObservedAt: now,
|
|
ExpiresAt: now.Add(2 * time.Minute),
|
|
RetryCooldownUntil: &cooldownUntil,
|
|
},
|
|
{
|
|
ClusterID: "cluster-1",
|
|
ReporterNodeID: "entry-1",
|
|
RouteID: "route-demoted",
|
|
ServiceClass: FabricServiceClassVPNPackets,
|
|
FeedbackStatus: "degraded",
|
|
ScoreAdjustment: -30,
|
|
Reasons: []string{"service_channel_recent_route_failure", "service_channel_rolling_quality_window"},
|
|
Payload: json.RawMessage(`{"quality_window_sample_count":96,"quality_window_success_count":95,"quality_window_failure_count":1,"quality_window_slow_count":0,"quality_window_drop_count":0}`),
|
|
ObservedAt: now,
|
|
ExpiresAt: now.Add(2 * time.Minute),
|
|
RetryCooldownUntil: &cooldownUntil,
|
|
},
|
|
}, now)
|
|
if report.RecoveredRouteCount != 1 || report.RecoveryHysteresisCount != 1 || report.RecoveryPromotedCount != 1 || report.RecoveryDemotedCount != 1 {
|
|
t.Fatalf("recovery counters = recovered:%d hysteresis:%d promoted:%d demoted:%d, want 1/1/1/1", report.RecoveredRouteCount, report.RecoveryHysteresisCount, report.RecoveryPromotedCount, report.RecoveryDemotedCount)
|
|
}
|
|
if len(report.Observations) != 3 {
|
|
t.Fatalf("observations = %d, want 3", len(report.Observations))
|
|
}
|
|
observation := report.Observations[0]
|
|
if observation.RecoveryState != "recovered" || !observation.RecoveryHysteresisActive || observation.RecoveryHysteresisPenalty != fabricServiceChannelRecoveryHysteresisPenalty {
|
|
t.Fatalf("observation recovery telemetry = state:%q active:%t penalty:%d", observation.RecoveryState, observation.RecoveryHysteresisActive, observation.RecoveryHysteresisPenalty)
|
|
}
|
|
promoted := report.Observations[1]
|
|
if promoted.RecoveryState != "healthy" || !promoted.RecoveryPromoted || promoted.RecoveryHysteresisActive {
|
|
t.Fatalf("promoted recovery telemetry = state:%q promoted:%t hysteresis:%t", promoted.RecoveryState, promoted.RecoveryPromoted, promoted.RecoveryHysteresisActive)
|
|
}
|
|
demoted := report.Observations[2]
|
|
if demoted.RecoveryState != "degraded" || !demoted.RecoveryDemoted || demoted.RecoveryReason != "service_channel_recovery_demoted_failure" {
|
|
t.Fatalf("demoted recovery telemetry = state:%q demoted:%t reason:%q", demoted.RecoveryState, demoted.RecoveryDemoted, demoted.RecoveryReason)
|
|
}
|
|
}
|
|
|
|
func TestFabricServiceChannelRecoveryPromotionRemovesHysteresisPenalty(t *testing.T) {
|
|
now := time.Now().UTC().Truncate(time.Second)
|
|
route, ok := fabricServiceChannelRouteFromIntent(MeshRouteIntent{
|
|
ID: "route-promoted",
|
|
ClusterID: "cluster-1",
|
|
SourceSelector: json.RawMessage(`{"node_id":"entry-1"}`),
|
|
DestinationSelector: json.RawMessage(`{"node_id":"exit-1"}`),
|
|
ServiceClass: FabricServiceClassVPNPackets,
|
|
Priority: 100,
|
|
Status: "active",
|
|
Policy: json.RawMessage(`{
|
|
"hops": ["entry-1", "exit-1"],
|
|
"allowed_channels": ["vpn_packet", "fabric_control"]
|
|
}`),
|
|
UpdatedAt: now,
|
|
}, FabricServiceClassVPNPackets, []string{"entry-1"}, []string{"exit-1"}, []string{"vpn_packet"}, "generation-1", now, now.Add(time.Minute), map[string]fabricServiceChannelRouteFeedback{
|
|
"route-promoted": {
|
|
RouteID: "route-promoted",
|
|
ManualRetry: true,
|
|
ScoreAdjustment: 90,
|
|
Reasons: []string{"service_channel_recent_success", "service_channel_rolling_quality_window", "manual_feedback_expired_retry_cooldown"},
|
|
QualityWindowSampleCount: 96,
|
|
QualityWindowSuccessCount: 96,
|
|
},
|
|
}, defaultFabricServiceChannelRecoveryPolicy())
|
|
if !ok {
|
|
t.Fatal("route was not built")
|
|
}
|
|
if route.RecoveryState != "healthy" || !route.RecoveryPromoted || route.RecoveryPenalty != 0 {
|
|
t.Fatalf("promoted route recovery = state:%q promoted:%t penalty:%d", route.RecoveryState, route.RecoveryPromoted, route.RecoveryPenalty)
|
|
}
|
|
if containsString(route.ScoreReasons, "service_channel_recovery_hysteresis") || !containsString(route.ScoreReasons, "service_channel_recovery_promoted") {
|
|
t.Fatalf("promoted route reasons = %+v", route.ScoreReasons)
|
|
}
|
|
}
|
|
|
|
func TestFabricServiceChannelRecoveryDemotionMarksRouteReason(t *testing.T) {
|
|
now := time.Now().UTC().Truncate(time.Second)
|
|
route, ok := fabricServiceChannelRouteFromIntent(MeshRouteIntent{
|
|
ID: "route-demoted",
|
|
ClusterID: "cluster-1",
|
|
SourceSelector: json.RawMessage(`{"node_id":"entry-1"}`),
|
|
DestinationSelector: json.RawMessage(`{"node_id":"exit-1"}`),
|
|
ServiceClass: FabricServiceClassVPNPackets,
|
|
Priority: 100,
|
|
Status: "active",
|
|
Policy: json.RawMessage(`{
|
|
"hops": ["entry-1", "exit-1"],
|
|
"allowed_channels": ["vpn_packet", "fabric_control"]
|
|
}`),
|
|
UpdatedAt: now,
|
|
}, FabricServiceClassVPNPackets, []string{"entry-1"}, []string{"exit-1"}, []string{"vpn_packet"}, "generation-1", now, now.Add(time.Minute), map[string]fabricServiceChannelRouteFeedback{
|
|
"route-demoted": {
|
|
RouteID: "route-demoted",
|
|
ManualRetry: true,
|
|
ScoreAdjustment: -30,
|
|
Reasons: []string{"service_channel_recent_route_failure", "service_channel_rolling_quality_window", "manual_feedback_expired_retry_cooldown"},
|
|
QualityWindowSampleCount: 96,
|
|
QualityWindowSuccessCount: 95,
|
|
QualityWindowFailureCount: 1,
|
|
},
|
|
}, defaultFabricServiceChannelRecoveryPolicy())
|
|
if !ok {
|
|
t.Fatal("route was not built")
|
|
}
|
|
if !route.RecoveryDemoted || route.RecoveryReason != "service_channel_recovery_demoted_failure" || route.RecoveryPromoted {
|
|
t.Fatalf("demoted route recovery = demoted:%t reason:%q promoted:%t", route.RecoveryDemoted, route.RecoveryReason, route.RecoveryPromoted)
|
|
}
|
|
if !containsString(route.ScoreReasons, "service_channel_recovery_demoted") || !containsString(route.ScoreReasons, "service_channel_recovery_demoted_failure") {
|
|
t.Fatalf("demoted route reasons = %+v", route.ScoreReasons)
|
|
}
|
|
}
|
|
|
|
func TestFabricServiceChannelRecoveryPolicyControlsPromotionAndPenalty(t *testing.T) {
|
|
now := time.Now().UTC().Truncate(time.Second)
|
|
policy := defaultFabricServiceChannelRecoveryPolicy()
|
|
policy.HysteresisPenalty = 40
|
|
policy.PromotionMinSamples = 4
|
|
cooldownUntil := now.Add(2 * time.Minute)
|
|
report := serviceChannelRouteFeedbackReportWithPolicy([]FabricServiceChannelRouteFeedbackObservation{
|
|
{
|
|
ClusterID: "cluster-1",
|
|
ReporterNodeID: "entry-1",
|
|
RouteID: "route-promoted",
|
|
ServiceClass: FabricServiceClassVPNPackets,
|
|
FeedbackStatus: "healthy",
|
|
ScoreAdjustment: 90,
|
|
Reasons: []string{"service_channel_recent_success", "service_channel_rolling_quality_window"},
|
|
Payload: json.RawMessage(`{"quality_window_sample_count":4,"quality_window_success_count":4,"quality_window_failure_count":0,"quality_window_slow_count":0,"quality_window_drop_count":0}`),
|
|
ObservedAt: now,
|
|
ExpiresAt: now.Add(2 * time.Minute),
|
|
RetryCooldownUntil: &cooldownUntil,
|
|
},
|
|
{
|
|
ClusterID: "cluster-1",
|
|
ReporterNodeID: "entry-1",
|
|
RouteID: "route-recovered",
|
|
ServiceClass: FabricServiceClassVPNPackets,
|
|
FeedbackStatus: "healthy",
|
|
ScoreAdjustment: 90,
|
|
Reasons: []string{"service_channel_recent_success", "service_channel_rolling_quality_window"},
|
|
ObservedAt: now,
|
|
ExpiresAt: now.Add(2 * time.Minute),
|
|
RetryCooldownUntil: &cooldownUntil,
|
|
},
|
|
}, now, policy)
|
|
if report.RecoveryPromotedCount != 1 || report.RecoveryHysteresisCount != 1 {
|
|
t.Fatalf("policy counters promoted/hysteresis = %d/%d, want 1/1", report.RecoveryPromotedCount, report.RecoveryHysteresisCount)
|
|
}
|
|
if report.Observations[1].RecoveryHysteresisPenalty != 40 {
|
|
t.Fatalf("hysteresis penalty = %d, want policy penalty 40", report.Observations[1].RecoveryHysteresisPenalty)
|
|
}
|
|
if report.RecoveryPolicy == nil || report.RecoveryPolicy.HysteresisPenalty != 40 || report.RecoveryPolicy.PromotionMinSamples != 4 {
|
|
t.Fatalf("report recovery policy provenance = %+v", report.RecoveryPolicy)
|
|
}
|
|
}
|
|
|
|
func TestFabricServiceChannelFeedbackStalePolicyIsConservative(t *testing.T) {
|
|
now := time.Now().UTC().Truncate(time.Second)
|
|
policy := defaultFabricServiceChannelRecoveryPolicy()
|
|
policy.HysteresisPenalty = 44
|
|
policy = normalizeFabricServiceChannelRecoveryPolicy(policy, defaultFabricServiceChannelRecoveryPolicy())
|
|
routeProvenance := map[string]fabricServiceChannelRouteProvenance{
|
|
"route-1": {RouteID: "route-1", RouteGeneration: "policy-v2", PolicyVersion: "policy-v2", RouteVersion: "route-v2"},
|
|
}
|
|
report := serviceChannelRouteFeedbackReportWithPolicyAndProvenance([]FabricServiceChannelRouteFeedbackObservation{
|
|
{
|
|
ClusterID: "cluster-1",
|
|
ReporterNodeID: "entry-1",
|
|
RouteID: "route-1",
|
|
ServiceClass: FabricServiceClassVPNPackets,
|
|
FeedbackStatus: "fenced",
|
|
ScoreAdjustment: -1030,
|
|
Reasons: []string{"service_channel_recent_route_failure", "service_channel_route_rebuild_recommended"},
|
|
Payload: json.RawMessage(`{"recovery_policy_fingerprint":"old-policy","route_policy_version":"policy-v1","quality_window_failure_count":3}`),
|
|
ObservedAt: now,
|
|
ExpiresAt: now.Add(time.Minute),
|
|
ConsecutiveFailures: 3,
|
|
LastSendDurationMs: 900,
|
|
},
|
|
}, now, policy, routeProvenance)
|
|
if report == nil || report.StalePolicyCount != 1 || report.StaleGenerationCount != 1 {
|
|
t.Fatalf("stale counters = %+v, want policy/generation stale", report)
|
|
}
|
|
if report.Observations[0].EffectiveScoreAdjustment != -10 || !report.Observations[0].StalePolicy || !report.Observations[0].StaleGeneration {
|
|
t.Fatalf("stale observation = %+v", report.Observations[0])
|
|
}
|
|
feedback := fabricServiceChannelRouteFeedbackFromObservationsWithProvenance(report.Observations, now, policy, routeProvenance)
|
|
item := feedback["route-1"]
|
|
if item.Fenced || item.RouteRebuildRecommended || item.ScoreAdjustment != -10 {
|
|
t.Fatalf("stale feedback should not fence/rebuild current route: %+v", item)
|
|
}
|
|
}
|
|
|
|
func TestFabricServiceChannelFeedbackMissingProvenanceIsVisibleButCompatible(t *testing.T) {
|
|
now := time.Now().UTC().Truncate(time.Second)
|
|
policy := defaultFabricServiceChannelRecoveryPolicy()
|
|
routeProvenance := map[string]fabricServiceChannelRouteProvenance{
|
|
"route-1": {RouteID: "route-1", RouteGeneration: "policy-v2", PolicyVersion: "policy-v2", RouteVersion: "route-v2"},
|
|
}
|
|
report := serviceChannelRouteFeedbackReportWithPolicyAndProvenance([]FabricServiceChannelRouteFeedbackObservation{
|
|
{
|
|
ClusterID: "cluster-1",
|
|
ReporterNodeID: "entry-1",
|
|
RouteID: "route-1",
|
|
ServiceClass: FabricServiceClassVPNPackets,
|
|
FeedbackStatus: "healthy",
|
|
ScoreAdjustment: 42,
|
|
Reasons: []string{"service_channel_recent_success"},
|
|
Payload: json.RawMessage(`{"quality_window_success_count":8}`),
|
|
ObservedAt: now,
|
|
ExpiresAt: now.Add(time.Minute),
|
|
},
|
|
}, now, policy, routeProvenance)
|
|
if report == nil || report.MissingProvenanceCount != 1 || report.StalePolicyCount != 0 || report.StaleGenerationCount != 0 {
|
|
t.Fatalf("missing provenance counters = %+v", report)
|
|
}
|
|
feedback := fabricServiceChannelRouteFeedbackFromObservationsWithProvenance(report.Observations, now, policy, routeProvenance)
|
|
if feedback["route-1"].ScoreAdjustment != 42 || !feedback["route-1"].ProvenanceMissing {
|
|
t.Fatalf("missing provenance should stay compatible for old agents: %+v", feedback["route-1"])
|
|
}
|
|
}
|
|
|
|
func TestUpdateFabricServiceChannelRecoveryPolicyPersistsClusterMetadata(t *testing.T) {
|
|
store := &fakeRepository{
|
|
platformRole: "platform_admin",
|
|
cluster: Cluster{
|
|
ID: "cluster-1",
|
|
Slug: "cluster-1",
|
|
Name: "Cluster 1",
|
|
Status: ClusterStatusActive,
|
|
Metadata: json.RawMessage(`{"existing":true}`),
|
|
},
|
|
}
|
|
service := NewService(store)
|
|
enabled := true
|
|
policy, err := service.UpdateFabricServiceChannelRecoveryPolicy(context.Background(), UpdateFabricServiceChannelRecoveryPolicyInput{
|
|
ActorUserID: "admin-1",
|
|
ClusterID: "cluster-1",
|
|
HysteresisPenalty: 42,
|
|
PromotionMinSamples: 7,
|
|
DemotionFailureThreshold: 3,
|
|
DemotionDropThreshold: 2,
|
|
DemotionSlowThreshold: 5,
|
|
DemotionRebuildEnabled: &enabled,
|
|
DemotionFencedEnabled: &enabled,
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("update recovery policy: %v", err)
|
|
}
|
|
if policy.HysteresisPenalty != 42 || policy.PromotionMinSamples != 7 || policy.DemotionFailureThreshold != 3 {
|
|
t.Fatalf("policy = %+v, want configured values", policy)
|
|
}
|
|
var metadata map[string]any
|
|
if err := json.Unmarshal(store.cluster.Metadata, &metadata); err != nil {
|
|
t.Fatalf("metadata json: %v", err)
|
|
}
|
|
if metadata["existing"] != true || metadata["fabric_service_channel_recovery_policy"] == nil {
|
|
t.Fatalf("metadata = %+v, want existing value plus policy", metadata)
|
|
}
|
|
}
|
|
|
|
func TestUpdateFabricServiceChannelAdaptivePolicyPersistsClusterMetadata(t *testing.T) {
|
|
store := &fakeRepository{
|
|
platformRole: "platform_admin",
|
|
cluster: Cluster{
|
|
ID: "cluster-1",
|
|
Slug: "cluster-1",
|
|
Name: "Cluster 1",
|
|
Status: ClusterStatusActive,
|
|
Metadata: json.RawMessage(`{"existing":true}`),
|
|
},
|
|
}
|
|
service := NewService(store)
|
|
policy, err := service.UpdateFabricServiceChannelAdaptivePolicy(context.Background(), UpdateFabricServiceChannelAdaptivePolicyInput{
|
|
ActorUserID: "admin-1",
|
|
ClusterID: "cluster-1",
|
|
MaxParallelWindow: 6,
|
|
BulkPressureChannelThreshold: 8,
|
|
QueuePressureHighWatermark: 9,
|
|
QueuePressureMaxInFlight: 10,
|
|
ClassWindows: map[string]int{
|
|
"control": 6,
|
|
"interactive": 6,
|
|
"reliable": 4,
|
|
"bulk": 2,
|
|
"droppable": 1,
|
|
},
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("update adaptive policy: %v", err)
|
|
}
|
|
if policy.MaxParallelWindow != 6 || policy.ClassWindows["bulk"] != 2 || policy.QueuePressureHighWatermark != 9 {
|
|
t.Fatalf("policy = %+v, want configured values", policy)
|
|
}
|
|
if policy.Fingerprint == "" || policy.Source != "cluster_metadata" {
|
|
t.Fatalf("policy provenance = %+v", policy)
|
|
}
|
|
var metadata map[string]any
|
|
if err := json.Unmarshal(store.cluster.Metadata, &metadata); err != nil {
|
|
t.Fatalf("metadata json: %v", err)
|
|
}
|
|
if metadata["existing"] != true || metadata["fabric_service_channel_adaptive_policy"] == nil {
|
|
t.Fatalf("metadata = %+v, want existing value plus adaptive policy", metadata)
|
|
}
|
|
}
|
|
|
|
func TestUpdateFabricServiceChannelPoolPolicyPersistsClusterMetadata(t *testing.T) {
|
|
store := &fakeRepository{
|
|
platformRole: "platform_admin",
|
|
cluster: Cluster{
|
|
ID: "cluster-1",
|
|
Slug: "cluster-1",
|
|
Name: "Cluster 1",
|
|
Status: ClusterStatusActive,
|
|
Metadata: json.RawMessage(`{"existing":true}`),
|
|
},
|
|
}
|
|
service := NewService(store)
|
|
enabled := true
|
|
sticky := false
|
|
policy, err := service.UpdateFabricServiceChannelPoolPolicy(context.Background(), UpdateFabricServiceChannelPoolPolicyInput{
|
|
ActorUserID: "admin-1",
|
|
ClusterID: "cluster-1",
|
|
EntryPoolNodeIDs: []string{"entry-a", "entry-b"},
|
|
ExitPoolNodeIDs: []string{"exit-b"},
|
|
PreferredEntryNodeID: "entry-b",
|
|
PreferredExitNodeID: "exit-b",
|
|
SelectionStrategy: "preferred_first",
|
|
RouteRebuild: "automatic",
|
|
EntryFailover: "automatic",
|
|
ExitFailover: "manual",
|
|
BackendFallbackAllowed: &enabled,
|
|
StickySession: &sticky,
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("update pool policy: %v", err)
|
|
}
|
|
if policy.PreferredEntryNodeID != "entry-b" || policy.PreferredExitNodeID != "exit-b" || policy.ExitFailover != "manual" || policy.StickySession {
|
|
t.Fatalf("policy = %+v, want configured values", policy)
|
|
}
|
|
if policy.Fingerprint == "" || policy.Source != "cluster_metadata" {
|
|
t.Fatalf("policy provenance = %+v", policy)
|
|
}
|
|
var metadata map[string]any
|
|
if err := json.Unmarshal(store.cluster.Metadata, &metadata); err != nil {
|
|
t.Fatalf("metadata json: %v", err)
|
|
}
|
|
if metadata["existing"] != true || metadata["fabric_service_channel_pool_policy"] == nil {
|
|
t.Fatalf("metadata = %+v, want existing value plus pool policy", metadata)
|
|
}
|
|
}
|
|
|
|
func TestUpdateFabricServiceChannelBreadcrumbWindowPolicyPersistsClusterMetadata(t *testing.T) {
|
|
store := &fakeRepository{
|
|
platformRole: "platform_admin",
|
|
cluster: Cluster{
|
|
ID: "cluster-1",
|
|
Slug: "cluster-1",
|
|
Name: "Cluster 1",
|
|
Status: ClusterStatusActive,
|
|
Metadata: json.RawMessage(`{"existing":true}`),
|
|
},
|
|
}
|
|
service := NewService(store)
|
|
policy, err := service.UpdateFabricServiceChannelBreadcrumbWindowPolicy(context.Background(), UpdateFabricServiceChannelBreadcrumbWindowPolicyInput{
|
|
ActorUserID: "admin-1",
|
|
ClusterID: "cluster-1",
|
|
CurrentWindowSeconds: 600,
|
|
HistoryWindowSeconds: 7200,
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("update breadcrumb window policy: %v", err)
|
|
}
|
|
if policy.CurrentWindowSeconds != 600 || policy.HistoryWindowSeconds != 7200 {
|
|
t.Fatalf("policy = %+v, want configured windows", policy)
|
|
}
|
|
if policy.Fingerprint == "" || policy.Source != "cluster_metadata" {
|
|
t.Fatalf("policy provenance = %+v", policy)
|
|
}
|
|
var metadata map[string]any
|
|
if err := json.Unmarshal(store.cluster.Metadata, &metadata); err != nil {
|
|
t.Fatalf("metadata json: %v", err)
|
|
}
|
|
if metadata["existing"] != true || metadata["fabric_service_channel_breadcrumb_window_policy"] == nil {
|
|
t.Fatalf("metadata = %+v, want existing value plus breadcrumb window policy", metadata)
|
|
}
|
|
}
|
|
|
|
func TestListFabricBreadcrumbsUsesClusterDefaultWindowPolicy(t *testing.T) {
|
|
clusterID := "cluster-1"
|
|
now := time.Date(2026, 5, 9, 14, 20, 0, 0, time.UTC)
|
|
policy := defaultFabricServiceChannelBreadcrumbWindowPolicy()
|
|
policy.Source = "cluster_metadata"
|
|
policy.CurrentWindowSeconds = 600
|
|
policy.HistoryWindowSeconds = 1800
|
|
policy = normalizeFabricServiceChannelBreadcrumbWindowPolicy(policy, defaultFabricServiceChannelBreadcrumbWindowPolicy())
|
|
metadata, err := upsertFabricServiceChannelBreadcrumbWindowPolicyMetadata(json.RawMessage(`{}`), policy)
|
|
if err != nil {
|
|
t.Fatalf("policy metadata: %v", err)
|
|
}
|
|
store := &fakeRepository{
|
|
platformRole: "platform_admin",
|
|
cluster: Cluster{
|
|
ID: clusterID,
|
|
Slug: "cluster-1",
|
|
Name: "Cluster 1",
|
|
Status: ClusterStatusActive,
|
|
Metadata: metadata,
|
|
},
|
|
auditEvents: []ClusterAuditEvent{
|
|
{
|
|
ID: "audit-current",
|
|
ClusterID: &clusterID,
|
|
EventType: "fabric.service_channel_rebuild_incident.investigation_opened",
|
|
TargetType: "fabric_service_channel_route_rebuild_incident",
|
|
Payload: json.RawMessage(`{}`),
|
|
CreatedAt: now.Add(-5 * time.Minute),
|
|
},
|
|
{
|
|
ID: "audit-stale",
|
|
ClusterID: &clusterID,
|
|
EventType: "fabric.service_channel_rebuild_incident.investigation_opened",
|
|
TargetType: "fabric_service_channel_route_rebuild_incident",
|
|
Payload: json.RawMessage(`{}`),
|
|
CreatedAt: now.Add(-20 * time.Minute),
|
|
},
|
|
{
|
|
ID: "audit-expired",
|
|
ClusterID: &clusterID,
|
|
EventType: "fabric.service_channel_rebuild_incident.investigation_opened",
|
|
TargetType: "fabric_service_channel_route_rebuild_incident",
|
|
Payload: json.RawMessage(`{}`),
|
|
CreatedAt: now.Add(-40 * time.Minute),
|
|
},
|
|
},
|
|
}
|
|
service := NewService(store)
|
|
service.now = func() time.Time { return now }
|
|
|
|
breadcrumbs, err := service.ListFabricServiceChannelRebuildInvestigationBreadcrumbs(context.Background(), "admin-1", ListFabricServiceChannelRebuildInvestigationBreadcrumbsInput{
|
|
ClusterID: clusterID,
|
|
Limit: 10,
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("list breadcrumbs: %v", err)
|
|
}
|
|
if breadcrumbs.CurrentWindowSeconds != 600 || breadcrumbs.HistoryWindowSeconds != 1800 {
|
|
t.Fatalf("breadcrumb windows = %d/%d, want cluster policy", breadcrumbs.CurrentWindowSeconds, breadcrumbs.HistoryWindowSeconds)
|
|
}
|
|
if breadcrumbs.CurrentCount != 1 || breadcrumbs.StaleCount != 1 || breadcrumbs.ExpiredCount != 1 {
|
|
t.Fatalf("breadcrumb freshness counts = current %d stale %d expired %d", breadcrumbs.CurrentCount, breadcrumbs.StaleCount, breadcrumbs.ExpiredCount)
|
|
}
|
|
}
|
|
|
|
func TestListFabricBreadcrumbsKeepsQueryWindowOverrides(t *testing.T) {
|
|
clusterID := "cluster-1"
|
|
now := time.Date(2026, 5, 9, 14, 20, 0, 0, time.UTC)
|
|
policy := defaultFabricServiceChannelBreadcrumbWindowPolicy()
|
|
policy.Source = "cluster_metadata"
|
|
policy.CurrentWindowSeconds = 3600
|
|
policy.HistoryWindowSeconds = 7200
|
|
policy = normalizeFabricServiceChannelBreadcrumbWindowPolicy(policy, defaultFabricServiceChannelBreadcrumbWindowPolicy())
|
|
metadata, err := upsertFabricServiceChannelBreadcrumbWindowPolicyMetadata(json.RawMessage(`{}`), policy)
|
|
if err != nil {
|
|
t.Fatalf("policy metadata: %v", err)
|
|
}
|
|
store := &fakeRepository{
|
|
platformRole: "platform_admin",
|
|
cluster: Cluster{
|
|
ID: clusterID,
|
|
Slug: "cluster-1",
|
|
Name: "Cluster 1",
|
|
Status: ClusterStatusActive,
|
|
Metadata: metadata,
|
|
},
|
|
auditEvents: []ClusterAuditEvent{
|
|
{
|
|
ID: "audit-stale-by-override",
|
|
ClusterID: &clusterID,
|
|
EventType: "fabric.service_channel_rebuild_incident.investigation_opened",
|
|
TargetType: "fabric_service_channel_route_rebuild_incident",
|
|
Payload: json.RawMessage(`{}`),
|
|
CreatedAt: now.Add(-20 * time.Minute),
|
|
},
|
|
},
|
|
}
|
|
service := NewService(store)
|
|
service.now = func() time.Time { return now }
|
|
|
|
breadcrumbs, err := service.ListFabricServiceChannelRebuildInvestigationBreadcrumbs(context.Background(), "admin-1", ListFabricServiceChannelRebuildInvestigationBreadcrumbsInput{
|
|
ClusterID: clusterID,
|
|
Limit: 10,
|
|
CurrentWindowSeconds: 600,
|
|
HistoryWindowSeconds: 1800,
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("list breadcrumbs: %v", err)
|
|
}
|
|
if breadcrumbs.CurrentWindowSeconds != 600 || breadcrumbs.HistoryWindowSeconds != 1800 {
|
|
t.Fatalf("breadcrumb windows = %d/%d, want query override", breadcrumbs.CurrentWindowSeconds, breadcrumbs.HistoryWindowSeconds)
|
|
}
|
|
if breadcrumbs.CurrentCount != 0 || breadcrumbs.StaleCount != 1 || breadcrumbs.ExpiredCount != 0 {
|
|
t.Fatalf("breadcrumb override freshness counts = current %d stale %d expired %d", breadcrumbs.CurrentCount, breadcrumbs.StaleCount, breadcrumbs.ExpiredCount)
|
|
}
|
|
}
|
|
|
|
func TestRoutePathDecisionReportCountsRecoveryHysteresis(t *testing.T) {
|
|
now := time.Now().UTC().Truncate(time.Second)
|
|
policy := defaultFabricServiceChannelRecoveryPolicy()
|
|
policy.Source = "cluster_metadata"
|
|
policy.HysteresisPenalty = 33
|
|
report := routePathDecisionReportWithRecoveryPolicy("generation-1", []RoutePathDecision{
|
|
{
|
|
DecisionID: "decision-1",
|
|
RouteID: "route-recovered",
|
|
ClusterID: "cluster-1",
|
|
LocalNodeID: "entry-1",
|
|
SourceNodeID: "entry-1",
|
|
DestinationNodeID: "exit-1",
|
|
OriginalHops: []string{"entry-1", "exit-1"},
|
|
EffectiveHops: []string{"entry-1", "exit-1"},
|
|
LocalRole: "entry",
|
|
DecisionSource: "service_channel_feedback_replacement",
|
|
Generation: "generation-1",
|
|
ScoreReasons: []string{"service_channel_recovery_hysteresis"},
|
|
ControlPlaneOnly: true,
|
|
ProductionForwarding: false,
|
|
ExpiresAt: now.Add(time.Minute),
|
|
},
|
|
{
|
|
DecisionID: "decision-2",
|
|
RouteID: "route-promoted",
|
|
ClusterID: "cluster-1",
|
|
LocalNodeID: "entry-1",
|
|
SourceNodeID: "entry-1",
|
|
DestinationNodeID: "exit-1",
|
|
OriginalHops: []string{"entry-1", "exit-1"},
|
|
EffectiveHops: []string{"entry-1", "exit-1"},
|
|
LocalRole: "entry",
|
|
DecisionSource: "service_channel_feedback_replacement",
|
|
Generation: "generation-1",
|
|
ScoreReasons: []string{"service_channel_recovery_promoted"},
|
|
ControlPlaneOnly: true,
|
|
ProductionForwarding: false,
|
|
ExpiresAt: now.Add(time.Minute),
|
|
},
|
|
{
|
|
DecisionID: "decision-3",
|
|
RouteID: "route-demoted",
|
|
ClusterID: "cluster-1",
|
|
LocalNodeID: "entry-1",
|
|
SourceNodeID: "entry-1",
|
|
DestinationNodeID: "exit-1",
|
|
OriginalHops: []string{"entry-1", "exit-1"},
|
|
EffectiveHops: []string{"entry-1", "exit-1"},
|
|
LocalRole: "entry",
|
|
DecisionSource: "service_channel_feedback_replacement",
|
|
Generation: "generation-1",
|
|
ScoreReasons: []string{"service_channel_recovery_demoted", "service_channel_recovery_demoted_failure"},
|
|
ControlPlaneOnly: true,
|
|
ProductionForwarding: false,
|
|
ExpiresAt: now.Add(time.Minute),
|
|
},
|
|
}, policy)
|
|
if report == nil || report.RecoveryHysteresisCount != 1 || report.RecoveryPromotedCount != 1 || report.RecoveryDemotedCount != 1 {
|
|
t.Fatalf("recovery counts = %+v, want hysteresis/promoted/demoted 1/1/1", report)
|
|
}
|
|
if report.RecoveryPolicy == nil || report.RecoveryPolicy.Source != "cluster_metadata" || report.RecoveryPolicy.HysteresisPenalty != 33 {
|
|
t.Fatalf("route path decision recovery policy provenance = %+v", report.RecoveryPolicy)
|
|
}
|
|
}
|
|
|
|
func containsRouteID(routes []SyntheticMeshRouteConfig, routeID string) bool {
|
|
for _, route := range routes {
|
|
if route.RouteID == routeID {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
func ptrTime(value time.Time) *time.Time {
|
|
return &value
|
|
}
|
|
|
|
type fakeRepository struct {
|
|
platformRole string
|
|
lastTokenHash string
|
|
lastLookupTokenHash string
|
|
validJoinToken NodeJoinToken
|
|
validTokenErr error
|
|
createJoinRequestID string
|
|
bootstrapJoinRequest NodeJoinRequest
|
|
clusterAuthority ClusterAuthorityKey
|
|
lastTokenAuthority json.RawMessage
|
|
lastApprovalAuthority json.RawMessage
|
|
authorityState ClusterAuthorityState
|
|
vpnConnection VPNConnection
|
|
lastVPNConnectionInput CreateVPNConnectionInput
|
|
lastAllowedNodesInput SetVPNConnectionAllowedNodesInput
|
|
lastAttachInput AttachExistingNodeInput
|
|
lastNodeGroupInput CreateNodeGroupInput
|
|
lastAssignGroupInput AssignNodeGroupInput
|
|
lastEntryPointInput CreateFabricEntryPointInput
|
|
lastEgressPoolInput CreateFabricEgressPoolInput
|
|
acquireVPNLeaseErr error
|
|
ownerEligibility VPNLeaseOwnerEligibility
|
|
ownerEligibilityErr error
|
|
renewVPNLeaseErr error
|
|
expiredVPNLeases []VPNConnectionLease
|
|
nodeVPNAssignments []NodeVPNAssignment
|
|
vpnClientProfile VPNClientProfile
|
|
testingFlags EffectiveNodeTestingFlags
|
|
routeIntents []MeshRouteIntent
|
|
createdRouteIntents []CreateRouteIntentInput
|
|
clusterNodes []ClusterNode
|
|
nodeRoles map[string][]NodeRoleAssignment
|
|
releaseVersions []ReleaseVersion
|
|
updateServiceCandidates []NodeUpdateServiceCandidate
|
|
nodeUpdatePolicies map[string]NodeUpdatePolicy
|
|
updateStatuses []NodeUpdateStatus
|
|
meshLinks []MeshLinkObservation
|
|
fabricRouteFeedback []FabricServiceChannelRouteFeedbackObservation
|
|
fabricLeases map[string]FabricServiceChannelLeaseRecord
|
|
fabricRebuildAttempts []FabricServiceChannelRouteRebuildAttempt
|
|
fabricRebuildSilences []FabricServiceChannelRouteRebuildAlertSilence
|
|
heartbeats map[string][]NodeHeartbeat
|
|
nodeTelemetry map[string][]NodeTelemetryObservation
|
|
desiredWorkloads []NodeWorkloadDesiredState
|
|
auditEvents []ClusterAuditEvent
|
|
cluster Cluster
|
|
lastPreferredEntryNodeID string
|
|
lastPreferredExitNodeID string
|
|
}
|
|
|
|
func (f *fakeRepository) GetPlatformRole(context.Context, string) (string, error) {
|
|
return f.platformRole, nil
|
|
}
|
|
|
|
func (f *fakeRepository) ListClusters(context.Context) ([]Cluster, error) {
|
|
return nil, nil
|
|
}
|
|
|
|
func (f *fakeRepository) GetCluster(context.Context, string) (Cluster, error) {
|
|
if f.cluster.ID != "" {
|
|
return f.cluster, nil
|
|
}
|
|
return Cluster{
|
|
ID: "cluster-1",
|
|
Slug: "cluster-1",
|
|
Name: "Cluster 1",
|
|
Status: ClusterStatusActive,
|
|
Metadata: json.RawMessage(`{}`),
|
|
}, nil
|
|
}
|
|
|
|
func (f *fakeRepository) CreateCluster(context.Context, CreateClusterInput) (Cluster, error) {
|
|
return Cluster{}, nil
|
|
}
|
|
|
|
func (f *fakeRepository) UpdateCluster(_ context.Context, input UpdateClusterInput) (Cluster, error) {
|
|
f.cluster = Cluster{
|
|
ID: input.ClusterID,
|
|
Slug: "cluster-1",
|
|
Name: input.Name,
|
|
Status: input.Status,
|
|
Region: input.Region,
|
|
Metadata: input.Metadata,
|
|
}
|
|
return f.cluster, nil
|
|
}
|
|
|
|
func (f *fakeRepository) GetClusterAuthority(_ context.Context, clusterID string) (ClusterAuthorityKey, error) {
|
|
if f.clusterAuthority.PrivateKey == "" {
|
|
keys, err := clusterauth.GenerateKeyPair()
|
|
if err != nil {
|
|
return ClusterAuthorityKey{}, err
|
|
}
|
|
f.clusterAuthority = ClusterAuthorityKey{
|
|
ClusterAuthorityDescriptor: ClusterAuthorityDescriptor{
|
|
SchemaVersion: clusterauth.AuthoritySchemaVersion,
|
|
ClusterID: clusterID,
|
|
AuthorityState: "active",
|
|
KeyAlgorithm: clusterauth.AlgorithmEd25519,
|
|
PublicKey: keys.PublicKeyB64,
|
|
PublicKeyFingerprint: keys.Fingerprint,
|
|
CreatedAt: time.Now().UTC(),
|
|
UpdatedAt: time.Now().UTC(),
|
|
},
|
|
PrivateKey: keys.PrivateKeyB64,
|
|
}
|
|
}
|
|
if f.clusterAuthority.ClusterID == "" {
|
|
f.clusterAuthority.ClusterID = clusterID
|
|
}
|
|
return f.clusterAuthority, nil
|
|
}
|
|
|
|
func (f *fakeRepository) EnsureClusterAuthority(ctx context.Context, clusterID string, _ *string) (ClusterAuthorityKey, error) {
|
|
return f.GetClusterAuthority(ctx, clusterID)
|
|
}
|
|
|
|
func (f *fakeRepository) ListClusterNodes(context.Context, string) ([]ClusterNode, error) {
|
|
return f.clusterNodes, nil
|
|
}
|
|
|
|
func (f *fakeRepository) ListNodeGroups(context.Context, string) ([]ClusterNodeGroup, error) {
|
|
return nil, nil
|
|
}
|
|
|
|
func (f *fakeRepository) CreateNodeGroup(_ context.Context, input CreateNodeGroupInput) (ClusterNodeGroup, error) {
|
|
f.lastNodeGroupInput = input
|
|
return ClusterNodeGroup{
|
|
ID: "group-1",
|
|
ClusterID: input.ClusterID,
|
|
ParentGroupID: input.ParentGroupID,
|
|
Name: input.Name,
|
|
Description: input.Description,
|
|
SortOrder: input.SortOrder,
|
|
Metadata: input.Metadata,
|
|
}, nil
|
|
}
|
|
|
|
func (f *fakeRepository) AssignNodeToGroup(_ context.Context, input AssignNodeGroupInput) (ClusterNode, error) {
|
|
f.lastAssignGroupInput = input
|
|
return ClusterNode{
|
|
ID: input.NodeID,
|
|
NodeKey: "node-key-1",
|
|
Name: "Node One",
|
|
RegistrationStatus: NodeRegistrationActive,
|
|
MembershipStatus: "active",
|
|
NodeGroupID: input.GroupID,
|
|
}, nil
|
|
}
|
|
|
|
func (f *fakeRepository) CreateJoinToken(_ context.Context, input CreateJoinTokenInput, tokenHash string) (NodeJoinToken, error) {
|
|
f.lastTokenHash = tokenHash
|
|
return NodeJoinToken{
|
|
ID: "token-1",
|
|
ClusterID: input.ClusterID,
|
|
Scope: input.Scope,
|
|
ExpiresAt: input.ExpiresAt,
|
|
MaxUses: input.MaxUses,
|
|
Status: "active",
|
|
CreatedByUserID: &input.ActorUserID,
|
|
CreatedAt: time.Now().UTC(),
|
|
}, nil
|
|
}
|
|
|
|
func (f *fakeRepository) SetJoinTokenAuthority(_ context.Context, clusterID, tokenID string, payload json.RawMessage, signature ClusterSignature) (NodeJoinToken, error) {
|
|
f.lastTokenAuthority = payload
|
|
return NodeJoinToken{
|
|
ID: tokenID,
|
|
ClusterID: clusterID,
|
|
Scope: json.RawMessage(`{"roles":["rdp-worker"]}`),
|
|
ExpiresAt: time.Now().UTC().Add(time.Hour),
|
|
MaxUses: 1,
|
|
Status: "active",
|
|
AuthorityPayload: payload,
|
|
AuthoritySignature: &signature,
|
|
}, nil
|
|
}
|
|
|
|
func (f *fakeRepository) GetValidJoinTokenByHash(_ context.Context, _ string, tokenHash string) (NodeJoinToken, error) {
|
|
f.lastLookupTokenHash = tokenHash
|
|
if f.validTokenErr != nil {
|
|
return NodeJoinToken{}, f.validTokenErr
|
|
}
|
|
if f.validJoinToken.ID != "" {
|
|
return f.validJoinToken, nil
|
|
}
|
|
return NodeJoinToken{ID: "token-1", Status: "active", ExpiresAt: time.Now().Add(time.Hour), MaxUses: 1}, nil
|
|
}
|
|
|
|
func (f *fakeRepository) RevokeJoinToken(context.Context, RevokeJoinTokenInput) (NodeJoinToken, error) {
|
|
return NodeJoinToken{ID: "token-1", Status: "revoked"}, nil
|
|
}
|
|
|
|
func (f *fakeRepository) ListJoinTokens(context.Context, string) ([]NodeJoinToken, error) {
|
|
return []NodeJoinToken{{ID: "token-1", Status: "active", ExpiresAt: time.Now().Add(time.Hour), MaxUses: 1}}, nil
|
|
}
|
|
|
|
func (f *fakeRepository) ExpireJoinTokens(context.Context, string) error {
|
|
return nil
|
|
}
|
|
|
|
func (f *fakeRepository) CreateJoinRequest(_ context.Context, input CreateJoinRequestInput, joinTokenID string) (NodeJoinRequest, error) {
|
|
id := f.createJoinRequestID
|
|
if id == "" {
|
|
id = "join-request-1"
|
|
}
|
|
return NodeJoinRequest{
|
|
ID: id,
|
|
ClusterID: input.ClusterID,
|
|
JoinTokenID: &joinTokenID,
|
|
NodeName: input.NodeName,
|
|
NodeFingerprint: input.NodeFingerprint,
|
|
PublicKey: input.PublicKey,
|
|
ReportedCapabilities: input.ReportedCapabilities,
|
|
ReportedFacts: input.ReportedFacts,
|
|
RequestedRoles: input.RequestedRoles,
|
|
Status: JoinRequestStatusPending,
|
|
}, nil
|
|
}
|
|
|
|
func (f *fakeRepository) GetJoinRequestForBootstrap(context.Context, GetJoinRequestBootstrapInput) (NodeJoinRequest, error) {
|
|
if f.bootstrapJoinRequest.ID != "" {
|
|
return f.bootstrapJoinRequest, nil
|
|
}
|
|
return NodeJoinRequest{ID: "join-request-1", ClusterID: "cluster-1", Status: JoinRequestStatusPending}, nil
|
|
}
|
|
|
|
func (f *fakeRepository) ListJoinRequests(context.Context, string) ([]NodeJoinRequest, error) {
|
|
return nil, nil
|
|
}
|
|
|
|
func (f *fakeRepository) ApproveJoinRequest(_ context.Context, input ApproveJoinRequestInput) (ApprovedJoinRequest, error) {
|
|
return ApprovedJoinRequest{
|
|
JoinRequest: NodeJoinRequest{ID: input.JoinRequestID, ClusterID: input.ClusterID, Status: JoinRequestStatusApproved, ApprovedNodeID: &input.NodeKey},
|
|
Bootstrap: NodeBootstrap{NodeID: input.NodeKey, ClusterID: input.ClusterID, IdentityStatus: "active"},
|
|
}, nil
|
|
}
|
|
|
|
func (f *fakeRepository) SetJoinRequestApprovalAuthority(_ context.Context, clusterID, joinRequestID string, payload json.RawMessage, signature ClusterSignature) (NodeJoinRequest, error) {
|
|
f.lastApprovalAuthority = payload
|
|
signatureRaw, _ := json.Marshal(signature)
|
|
nodeID := "node-1"
|
|
return NodeJoinRequest{
|
|
ID: joinRequestID,
|
|
ClusterID: clusterID,
|
|
Status: JoinRequestStatusApproved,
|
|
ApprovedNodeID: &nodeID,
|
|
ApprovalPayload: payload,
|
|
ApprovalSignature: signatureRaw,
|
|
}, nil
|
|
}
|
|
|
|
func (f *fakeRepository) RejectJoinRequest(context.Context, RejectJoinRequestInput) (NodeJoinRequest, error) {
|
|
return NodeJoinRequest{}, nil
|
|
}
|
|
|
|
func (f *fakeRepository) AssignNodeRole(_ context.Context, input AssignNodeRoleInput) (NodeRoleAssignment, error) {
|
|
return NodeRoleAssignment{ClusterID: input.ClusterID, NodeID: input.NodeID, Role: input.Role}, nil
|
|
}
|
|
|
|
func (f *fakeRepository) ListNodeRoleAssignments(_ context.Context, _ string, nodeID string) ([]NodeRoleAssignment, error) {
|
|
return f.nodeRoles[nodeID], nil
|
|
}
|
|
|
|
func (f *fakeRepository) AttachExistingNodeToCluster(_ context.Context, input AttachExistingNodeInput) (ClusterNode, error) {
|
|
f.lastAttachInput = input
|
|
return ClusterNode{
|
|
ID: input.NodeID,
|
|
NodeKey: "node-key-1",
|
|
Name: "Node One",
|
|
RegistrationStatus: NodeRegistrationActive,
|
|
MembershipStatus: "active",
|
|
}, nil
|
|
}
|
|
|
|
func (f *fakeRepository) RecordHeartbeat(_ context.Context, input RecordHeartbeatInput) (NodeHeartbeat, error) {
|
|
now := time.Now().UTC()
|
|
item := NodeHeartbeat{
|
|
ID: "heartbeat-" + input.NodeID,
|
|
ClusterID: input.ClusterID,
|
|
NodeID: input.NodeID,
|
|
HealthStatus: input.HealthStatus,
|
|
ReportedVersion: input.ReportedVersion,
|
|
Capabilities: input.Capabilities,
|
|
ServiceStates: input.ServiceStates,
|
|
Metadata: input.Metadata,
|
|
ObservedAt: now,
|
|
}
|
|
if f.heartbeats == nil {
|
|
f.heartbeats = map[string][]NodeHeartbeat{}
|
|
}
|
|
f.heartbeats[input.NodeID] = append([]NodeHeartbeat{item}, f.heartbeats[input.NodeID]...)
|
|
return item, nil
|
|
}
|
|
|
|
func (f *fakeRepository) ListNodeHeartbeats(_ context.Context, _ string, nodeID string, _ int) ([]NodeHeartbeat, error) {
|
|
return f.heartbeats[nodeID], nil
|
|
}
|
|
|
|
func (f *fakeRepository) CreateReleaseVersion(_ context.Context, input CreateReleaseVersionInput) (ReleaseVersion, error) {
|
|
item := ReleaseVersion{
|
|
ID: "release-" + input.Version,
|
|
ClusterID: input.ClusterID,
|
|
Product: input.Product,
|
|
Version: input.Version,
|
|
Channel: input.Channel,
|
|
Status: input.Status,
|
|
Compatibility: input.Compatibility,
|
|
Changelog: input.Changelog,
|
|
CreatedByUserID: &input.ActorUserID,
|
|
CreatedAt: time.Now().UTC(),
|
|
}
|
|
for i, artifact := range input.Artifacts {
|
|
item.Artifacts = append(item.Artifacts, ReleaseArtifact{
|
|
ID: item.ID + "-artifact",
|
|
ReleaseID: item.ID,
|
|
ClusterID: input.ClusterID,
|
|
Product: input.Product,
|
|
Version: input.Version,
|
|
OS: artifact.OS,
|
|
Arch: artifact.Arch,
|
|
InstallType: artifact.InstallType,
|
|
Kind: artifact.Kind,
|
|
URL: artifact.URL,
|
|
SHA256: artifact.SHA256,
|
|
SizeBytes: artifact.SizeBytes,
|
|
Signature: artifact.Signature,
|
|
Metadata: artifact.Metadata,
|
|
CreatedAt: time.Now().UTC().Add(time.Duration(i) * time.Second),
|
|
})
|
|
}
|
|
f.releaseVersions = append([]ReleaseVersion{item}, f.releaseVersions...)
|
|
return item, nil
|
|
}
|
|
|
|
func (f *fakeRepository) ListReleaseVersions(_ context.Context, clusterID, product, channel string) ([]ReleaseVersion, error) {
|
|
var out []ReleaseVersion
|
|
for _, item := range f.releaseVersions {
|
|
if item.ClusterID != clusterID {
|
|
continue
|
|
}
|
|
if product != "" && item.Product != product {
|
|
continue
|
|
}
|
|
if channel != "" && item.Channel != channel {
|
|
continue
|
|
}
|
|
out = append(out, item)
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
func (f *fakeRepository) ListNodeUpdateServiceCandidates(context.Context, string) ([]NodeUpdateServiceCandidate, error) {
|
|
return f.updateServiceCandidates, nil
|
|
}
|
|
|
|
func (f *fakeRepository) UpsertNodeUpdatePolicy(_ context.Context, input UpsertNodeUpdatePolicyInput) (NodeUpdatePolicy, error) {
|
|
item := NodeUpdatePolicy{
|
|
ClusterID: input.ClusterID,
|
|
NodeID: input.NodeID,
|
|
Product: input.Product,
|
|
Channel: input.Channel,
|
|
TargetVersion: input.TargetVersion,
|
|
Strategy: input.Strategy,
|
|
Enabled: input.Enabled,
|
|
RollbackAllowed: input.RollbackAllowed,
|
|
HealthWindowSec: input.HealthWindowSec,
|
|
UpdatedByUserID: &input.ActorUserID,
|
|
UpdatedAt: time.Now().UTC(),
|
|
}
|
|
if f.nodeUpdatePolicies == nil {
|
|
f.nodeUpdatePolicies = map[string]NodeUpdatePolicy{}
|
|
}
|
|
f.nodeUpdatePolicies[input.NodeID+"|"+input.Product] = item
|
|
return item, nil
|
|
}
|
|
|
|
func (f *fakeRepository) GetNodeUpdatePolicy(_ context.Context, _ string, nodeID, product string) (NodeUpdatePolicy, error) {
|
|
if f.nodeUpdatePolicies == nil {
|
|
return NodeUpdatePolicy{}, pgx.ErrNoRows
|
|
}
|
|
item, ok := f.nodeUpdatePolicies[nodeID+"|"+product]
|
|
if !ok {
|
|
return NodeUpdatePolicy{}, pgx.ErrNoRows
|
|
}
|
|
return item, nil
|
|
}
|
|
|
|
func (f *fakeRepository) ReportNodeUpdateStatus(_ context.Context, input ReportNodeUpdateStatusInput) (NodeUpdateStatus, error) {
|
|
item := NodeUpdateStatus{
|
|
ID: "status-1",
|
|
ClusterID: input.ClusterID,
|
|
NodeID: input.NodeID,
|
|
Product: input.Product,
|
|
CurrentVersion: input.CurrentVersion,
|
|
TargetVersion: input.TargetVersion,
|
|
Phase: input.Phase,
|
|
Status: input.Status,
|
|
AttemptID: input.AttemptID,
|
|
ErrorMessage: input.ErrorMessage,
|
|
RollbackVersion: input.RollbackVersion,
|
|
Payload: input.Payload,
|
|
ObservedAt: input.ObservedAt,
|
|
}
|
|
f.updateStatuses = append(f.updateStatuses, item)
|
|
return item, nil
|
|
}
|
|
|
|
func (f *fakeRepository) ListNodeUpdateStatuses(_ context.Context, clusterID, nodeID string, limit int) ([]NodeUpdateStatus, error) {
|
|
out := []NodeUpdateStatus{}
|
|
for _, item := range f.updateStatuses {
|
|
if item.ClusterID == clusterID && item.NodeID == nodeID {
|
|
out = append(out, item)
|
|
}
|
|
}
|
|
if limit > 0 && len(out) > limit {
|
|
out = out[:limit]
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
func (f *fakeRepository) RevokeNodeIdentity(context.Context, RevokeNodeIdentityInput) error {
|
|
return nil
|
|
}
|
|
|
|
func (f *fakeRepository) DisableClusterMembership(context.Context, DisableMembershipInput) error {
|
|
return nil
|
|
}
|
|
|
|
func (f *fakeRepository) DeleteClusterNode(context.Context, DeleteClusterNodeInput) error {
|
|
return nil
|
|
}
|
|
|
|
func (f *fakeRepository) UpsertFabricTestingFlag(_ context.Context, input UpsertFabricTestingFlagInput) (FabricTestingFlag, error) {
|
|
return FabricTestingFlag{
|
|
ScopeType: input.ScopeType,
|
|
ScopeID: input.ScopeID,
|
|
ClusterID: input.ClusterID,
|
|
Enabled: input.Enabled,
|
|
TelemetryEnabled: input.TelemetryEnabled,
|
|
SyntheticLinksEnabled: input.SyntheticLinksEnabled,
|
|
HistoryRetentionHours: input.HistoryRetentionHours,
|
|
Metadata: input.Metadata,
|
|
}, nil
|
|
}
|
|
|
|
func (f *fakeRepository) ListFabricTestingFlags(context.Context) ([]FabricTestingFlag, error) {
|
|
return nil, nil
|
|
}
|
|
|
|
func (f *fakeRepository) GetEffectiveNodeTestingFlags(context.Context, string, string) (EffectiveNodeTestingFlags, error) {
|
|
return f.testingFlags, nil
|
|
}
|
|
|
|
func (f *fakeRepository) RecordNodeTelemetry(_ context.Context, input RecordNodeTelemetryInput) (NodeTelemetryObservation, error) {
|
|
item := NodeTelemetryObservation{
|
|
ClusterID: input.ClusterID,
|
|
NodeID: input.NodeID,
|
|
Payload: input.Payload,
|
|
ObservedAt: input.ObservedAt,
|
|
}
|
|
if item.ObservedAt.IsZero() {
|
|
item.ObservedAt = time.Now().UTC()
|
|
}
|
|
if f.nodeTelemetry == nil {
|
|
f.nodeTelemetry = map[string][]NodeTelemetryObservation{}
|
|
}
|
|
f.nodeTelemetry[input.NodeID] = append([]NodeTelemetryObservation{item}, f.nodeTelemetry[input.NodeID]...)
|
|
return item, nil
|
|
}
|
|
|
|
func (f *fakeRepository) ListNodeTelemetry(_ context.Context, _ string, nodeID string, limit int) ([]NodeTelemetryObservation, error) {
|
|
items := append([]NodeTelemetryObservation{}, f.nodeTelemetry[nodeID]...)
|
|
if limit > 0 && len(items) > limit {
|
|
items = items[:limit]
|
|
}
|
|
return items, nil
|
|
}
|
|
|
|
func (f *fakeRepository) SetDesiredWorkload(_ context.Context, input SetDesiredWorkloadInput) (NodeWorkloadDesiredState, error) {
|
|
return NodeWorkloadDesiredState{
|
|
ClusterID: input.ClusterID,
|
|
NodeID: input.NodeID,
|
|
ServiceType: input.ServiceType,
|
|
DesiredState: input.DesiredState,
|
|
RuntimeMode: input.RuntimeMode,
|
|
Config: input.Config,
|
|
Environment: input.Environment,
|
|
}, nil
|
|
}
|
|
|
|
func (f *fakeRepository) ListDesiredWorkloads(_ context.Context, clusterID, nodeID string) ([]NodeWorkloadDesiredState, error) {
|
|
out := []NodeWorkloadDesiredState{}
|
|
for _, item := range f.desiredWorkloads {
|
|
if item.ClusterID == clusterID && item.NodeID == nodeID {
|
|
out = append(out, item)
|
|
}
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
func (f *fakeRepository) ReportWorkloadStatus(_ context.Context, input ReportWorkloadStatusInput) (NodeWorkloadStatus, error) {
|
|
return NodeWorkloadStatus{
|
|
ClusterID: input.ClusterID,
|
|
NodeID: input.NodeID,
|
|
ServiceType: input.ServiceType,
|
|
ReportedState: input.ReportedState,
|
|
RuntimeMode: input.RuntimeMode,
|
|
StatusPayload: input.StatusPayload,
|
|
}, nil
|
|
}
|
|
|
|
func (f *fakeRepository) ListLatestWorkloadStatuses(context.Context, string, string) ([]NodeWorkloadStatus, error) {
|
|
return nil, nil
|
|
}
|
|
|
|
func (f *fakeRepository) ReportMeshLink(_ context.Context, input ReportMeshLinkInput) (MeshLinkObservation, error) {
|
|
return MeshLinkObservation{
|
|
ClusterID: input.ClusterID,
|
|
SourceNodeID: input.SourceNodeID,
|
|
TargetNodeID: input.TargetNodeID,
|
|
LinkStatus: input.LinkStatus,
|
|
Metadata: input.Metadata,
|
|
}, nil
|
|
}
|
|
|
|
func (f *fakeRepository) ListMeshLinks(context.Context, string) ([]MeshLinkObservation, error) {
|
|
return f.meshLinks, nil
|
|
}
|
|
|
|
func (f *fakeRepository) CreateRouteIntent(_ context.Context, input CreateRouteIntentInput) (MeshRouteIntent, error) {
|
|
f.createdRouteIntents = append(f.createdRouteIntents, input)
|
|
item := MeshRouteIntent{
|
|
ID: "route-intent-" + strconv.Itoa(len(f.createdRouteIntents)),
|
|
ClusterID: input.ClusterID,
|
|
SourceSelector: input.SourceSelector,
|
|
DestinationSelector: input.DestinationSelector,
|
|
ServiceClass: input.ServiceClass,
|
|
Priority: input.Priority,
|
|
Status: "active",
|
|
Policy: input.Policy,
|
|
UpdatedAt: time.Now().UTC(),
|
|
}
|
|
f.routeIntents = append(f.routeIntents, item)
|
|
return item, nil
|
|
}
|
|
|
|
func (f *fakeRepository) ListRouteIntents(context.Context, string) ([]MeshRouteIntent, error) {
|
|
return f.routeIntents, nil
|
|
}
|
|
|
|
func (f *fakeRepository) ExpireRouteIntent(_ context.Context, input RouteIntentLifecycleInput, expiresAt time.Time) (MeshRouteIntent, error) {
|
|
for index, item := range f.routeIntents {
|
|
if item.ClusterID != input.ClusterID || item.ID != input.RouteIntentID {
|
|
continue
|
|
}
|
|
var policy map[string]any
|
|
_ = json.Unmarshal(item.Policy, &policy)
|
|
if policy == nil {
|
|
policy = map[string]any{}
|
|
}
|
|
policy["expires_at"] = expiresAt.UTC().Format(time.RFC3339Nano)
|
|
policy["operator_expire"] = map[string]any{
|
|
"expired_at": expiresAt.UTC().Format(time.RFC3339Nano),
|
|
"reason": input.Reason,
|
|
}
|
|
item.Policy = mustJSONRaw(policy)
|
|
item.UpdatedAt = expiresAt.UTC()
|
|
f.routeIntents[index] = item
|
|
return item, nil
|
|
}
|
|
return MeshRouteIntent{}, pgx.ErrNoRows
|
|
}
|
|
|
|
func (f *fakeRepository) DisableRouteIntent(_ context.Context, input RouteIntentLifecycleInput) (MeshRouteIntent, error) {
|
|
for index, item := range f.routeIntents {
|
|
if item.ClusterID != input.ClusterID || item.ID != input.RouteIntentID {
|
|
continue
|
|
}
|
|
var policy map[string]any
|
|
_ = json.Unmarshal(item.Policy, &policy)
|
|
if policy == nil {
|
|
policy = map[string]any{}
|
|
}
|
|
policy["operator_disable"] = map[string]any{
|
|
"reason": input.Reason,
|
|
}
|
|
item.Status = "disabled"
|
|
item.Policy = mustJSONRaw(policy)
|
|
item.UpdatedAt = time.Now().UTC()
|
|
f.routeIntents[index] = item
|
|
return item, nil
|
|
}
|
|
return MeshRouteIntent{}, pgx.ErrNoRows
|
|
}
|
|
|
|
func (f *fakeRepository) RecordFabricServiceChannelRouteFeedback(_ context.Context, input RecordFabricServiceChannelRouteFeedbackInput) (FabricServiceChannelRouteFeedbackObservation, error) {
|
|
observedAt := input.ObservedAt.UTC()
|
|
if observedAt.IsZero() {
|
|
observedAt = time.Now().UTC()
|
|
}
|
|
if input.FeedbackStatus != "healthy" {
|
|
for _, current := range f.fabricRouteFeedback {
|
|
if current.ClusterID != input.ClusterID || current.ReporterNodeID != input.ReporterNodeID || current.RouteID != input.RouteID {
|
|
continue
|
|
}
|
|
if current.RetryCooldownUntil == nil || !current.RetryCooldownUntil.After(observedAt) {
|
|
continue
|
|
}
|
|
input = fabricServiceChannelFeedbackSuppressedByOperatorCooldown(input, *current.RetryCooldownUntil, observedAt)
|
|
break
|
|
}
|
|
}
|
|
item := FabricServiceChannelRouteFeedbackObservation{
|
|
ID: "fsc-feedback-" + strconv.Itoa(len(f.fabricRouteFeedback)+1),
|
|
ClusterID: input.ClusterID,
|
|
ReporterNodeID: input.ReporterNodeID,
|
|
RouteID: input.RouteID,
|
|
ServiceClass: input.ServiceClass,
|
|
FeedbackStatus: input.FeedbackStatus,
|
|
ScoreAdjustment: input.ScoreAdjustment,
|
|
Reasons: append([]string{}, input.Reasons...),
|
|
LastError: input.LastError,
|
|
ConsecutiveFailures: input.ConsecutiveFailures,
|
|
StallCount: input.StallCount,
|
|
LastSendDurationMs: input.LastSendDurationMs,
|
|
Payload: input.Payload,
|
|
ObservedAt: observedAt,
|
|
ExpiresAt: input.ExpiresAt,
|
|
RetryCooldownUntil: fabricServiceChannelRetryCooldownUntil(input.Payload),
|
|
}
|
|
f.fabricRouteFeedback = append(f.fabricRouteFeedback, item)
|
|
return item, nil
|
|
}
|
|
|
|
func (f *fakeRepository) ListFabricServiceChannelRouteFeedback(_ context.Context, input ListFabricServiceChannelRouteFeedbackInput) ([]FabricServiceChannelRouteFeedbackObservation, error) {
|
|
now := input.Now.UTC()
|
|
out := []FabricServiceChannelRouteFeedbackObservation{}
|
|
for _, item := range f.fabricRouteFeedback {
|
|
if item.ClusterID != input.ClusterID {
|
|
continue
|
|
}
|
|
if input.ReporterNodeID != "" && item.ReporterNodeID != input.ReporterNodeID {
|
|
continue
|
|
}
|
|
if input.RouteID != "" && item.RouteID != input.RouteID {
|
|
continue
|
|
}
|
|
if input.ServiceClass != "" && item.ServiceClass != input.ServiceClass {
|
|
continue
|
|
}
|
|
if input.FeedbackStatus != "" && item.FeedbackStatus != input.FeedbackStatus {
|
|
continue
|
|
}
|
|
if !input.IncludeExpired && !item.ExpiresAt.IsZero() && !item.ExpiresAt.After(now) {
|
|
continue
|
|
}
|
|
out = append(out, item)
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
func (f *fakeRepository) ExpireFabricServiceChannelRouteFeedback(_ context.Context, input ExpireFabricServiceChannelRouteFeedbackInput) (ExpireFabricServiceChannelRouteFeedbackResult, error) {
|
|
now := input.Now.UTC()
|
|
if now.IsZero() {
|
|
now = time.Now().UTC()
|
|
}
|
|
cooldownUntil := now.Add(fabricServiceChannelOperatorExpireCooldown)
|
|
expired := 0
|
|
for idx, item := range f.fabricRouteFeedback {
|
|
if item.ClusterID != input.ClusterID || item.RouteID != input.RouteID {
|
|
continue
|
|
}
|
|
if input.ReporterNodeID != "" && item.ReporterNodeID != input.ReporterNodeID {
|
|
continue
|
|
}
|
|
if input.ServiceClass != "" && item.ServiceClass != input.ServiceClass {
|
|
continue
|
|
}
|
|
if !item.ExpiresAt.IsZero() && !item.ExpiresAt.After(now) {
|
|
continue
|
|
}
|
|
f.fabricRouteFeedback[idx].ExpiresAt = now
|
|
f.fabricRouteFeedback[idx].RetryCooldownUntil = &cooldownUntil
|
|
expired++
|
|
}
|
|
return ExpireFabricServiceChannelRouteFeedbackResult{
|
|
ClusterID: input.ClusterID,
|
|
ReporterNodeID: input.ReporterNodeID,
|
|
RouteID: input.RouteID,
|
|
ServiceClass: input.ServiceClass,
|
|
ExpiredCount: expired,
|
|
ExpiredAt: now,
|
|
CooldownUntil: cooldownUntil,
|
|
}, nil
|
|
}
|
|
|
|
func (f *fakeRepository) StoreFabricServiceChannelLease(_ context.Context, input StoreFabricServiceChannelLeaseInput) (FabricServiceChannelLeaseRecord, error) {
|
|
lease := input.Lease
|
|
storedLease := lease
|
|
storedLease.Token.Token = ""
|
|
item := FabricServiceChannelLeaseRecord{
|
|
ClusterID: lease.ClusterID,
|
|
ChannelID: lease.ChannelID,
|
|
TokenHash: input.TokenHash,
|
|
ResourceID: lease.ResourceID,
|
|
ServiceClass: lease.ServiceClass,
|
|
SelectedEntryNodeID: lease.SelectedEntryNodeID,
|
|
ExpiresAt: lease.ExpiresAt,
|
|
Lease: storedLease,
|
|
CreatedAt: lease.IssuedAt,
|
|
UpdatedAt: lease.IssuedAt,
|
|
}
|
|
if f.fabricLeases == nil {
|
|
f.fabricLeases = map[string]FabricServiceChannelLeaseRecord{}
|
|
}
|
|
f.fabricLeases[fabricServiceChannelLeaseCacheKey(lease.ClusterID, lease.ChannelID)] = item
|
|
return item, nil
|
|
}
|
|
|
|
func (f *fakeRepository) GetFabricServiceChannelLease(_ context.Context, clusterID, channelID string) (FabricServiceChannelLeaseRecord, error) {
|
|
if f.fabricLeases == nil {
|
|
return FabricServiceChannelLeaseRecord{}, pgx.ErrNoRows
|
|
}
|
|
item, ok := f.fabricLeases[fabricServiceChannelLeaseCacheKey(clusterID, channelID)]
|
|
if !ok {
|
|
return FabricServiceChannelLeaseRecord{}, pgx.ErrNoRows
|
|
}
|
|
return item, nil
|
|
}
|
|
|
|
func (f *fakeRepository) ListFabricServiceChannelLeases(_ context.Context, input ListFabricServiceChannelLeasesInput) ([]FabricServiceChannelLeaseRecord, error) {
|
|
now := input.Now
|
|
if now.IsZero() {
|
|
now = time.Now().UTC()
|
|
}
|
|
out := []FabricServiceChannelLeaseRecord{}
|
|
for _, item := range f.fabricLeases {
|
|
if item.ClusterID != input.ClusterID {
|
|
continue
|
|
}
|
|
if input.ServiceClass != "" && item.ServiceClass != input.ServiceClass {
|
|
continue
|
|
}
|
|
if input.EntryNodeID != "" && item.SelectedEntryNodeID != input.EntryNodeID {
|
|
continue
|
|
}
|
|
if input.ResourceID != "" && item.ResourceID != input.ResourceID {
|
|
continue
|
|
}
|
|
if !input.IncludeExpired && !item.ExpiresAt.IsZero() && !item.ExpiresAt.After(now) {
|
|
continue
|
|
}
|
|
out = append(out, item)
|
|
}
|
|
sort.Slice(out, func(i, j int) bool {
|
|
return out[i].ExpiresAt.After(out[j].ExpiresAt)
|
|
})
|
|
if input.Limit > 0 && len(out) > input.Limit {
|
|
out = out[:input.Limit]
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
func (f *fakeRepository) CleanupExpiredFabricServiceChannelLeases(_ context.Context, clusterID string, now time.Time, limit int) (int, error) {
|
|
if f.fabricLeases == nil {
|
|
return 0, nil
|
|
}
|
|
if now.IsZero() {
|
|
now = time.Now().UTC()
|
|
}
|
|
if limit <= 0 {
|
|
limit = 100
|
|
}
|
|
deleted := 0
|
|
for key, item := range f.fabricLeases {
|
|
if deleted >= limit {
|
|
break
|
|
}
|
|
if item.ClusterID == clusterID && !item.ExpiresAt.IsZero() && !item.ExpiresAt.After(now) {
|
|
delete(f.fabricLeases, key)
|
|
deleted++
|
|
}
|
|
}
|
|
return deleted, nil
|
|
}
|
|
|
|
func (f *fakeRepository) RecordFabricServiceChannelRouteRebuildAttempt(_ context.Context, input RecordFabricServiceChannelRouteRebuildAttemptInput) (FabricServiceChannelRouteRebuildAttempt, error) {
|
|
item := FabricServiceChannelRouteRebuildAttempt{
|
|
ID: "fsc-rebuild-" + strconv.Itoa(len(f.fabricRebuildAttempts)+1),
|
|
ClusterID: input.ClusterID,
|
|
ReporterNodeID: input.ReporterNodeID,
|
|
ServiceClass: input.ServiceClass,
|
|
RouteID: input.RouteID,
|
|
ReplacementRouteID: input.ReplacementRouteID,
|
|
RebuildRequestID: input.RebuildRequestID,
|
|
RebuildStatus: input.RebuildStatus,
|
|
RebuildReason: input.RebuildReason,
|
|
RebuildAttempt: input.RebuildAttempt,
|
|
DecisionSource: input.DecisionSource,
|
|
Outcome: input.Outcome,
|
|
Generation: input.Generation,
|
|
PolicyFingerprint: input.PolicyFingerprint,
|
|
ObservedPolicyFingerprint: input.ObservedPolicyFingerprint,
|
|
ObservedRouteGeneration: input.ObservedRouteGeneration,
|
|
EffectiveRouteGeneration: input.EffectiveRouteGeneration,
|
|
FeedbackStatus: input.FeedbackStatus,
|
|
FeedbackObservationID: input.FeedbackObservationID,
|
|
FeedbackSource: input.FeedbackSource,
|
|
FeedbackObservedAt: input.FeedbackObservedAt,
|
|
FeedbackExpiresAt: input.FeedbackExpiresAt,
|
|
FeedbackChannelID: input.FeedbackChannelID,
|
|
FeedbackResourceID: input.FeedbackResourceID,
|
|
FeedbackViolationStatus: input.FeedbackViolationStatus,
|
|
FeedbackViolationReason: input.FeedbackViolationReason,
|
|
FeedbackScoreAdjustment: input.FeedbackScoreAdjustment,
|
|
FeedbackEffectiveScoreAdjustment: input.FeedbackEffectiveScoreAdjustment,
|
|
FeedbackReasons: append([]string{}, input.FeedbackReasons...),
|
|
LastError: input.LastError,
|
|
ConsecutiveFailures: input.ConsecutiveFailures,
|
|
StallCount: input.StallCount,
|
|
LastSendDurationMs: input.LastSendDurationMs,
|
|
QualityWindowSampleCount: input.QualityWindowSampleCount,
|
|
QualityWindowFailureCount: input.QualityWindowFailureCount,
|
|
QualityWindowDropCount: input.QualityWindowDropCount,
|
|
QualityWindowSlowCount: input.QualityWindowSlowCount,
|
|
OldHops: append([]string{}, input.OldHops...),
|
|
ReplacementHops: append([]string{}, input.ReplacementHops...),
|
|
Payload: input.Payload,
|
|
CreatedAt: time.Now().UTC(),
|
|
UpdatedAt: time.Now().UTC(),
|
|
}
|
|
for idx, current := range f.fabricRebuildAttempts {
|
|
if current.ClusterID == item.ClusterID &&
|
|
current.ReporterNodeID == item.ReporterNodeID &&
|
|
current.ServiceClass == item.ServiceClass &&
|
|
current.RouteID == item.RouteID &&
|
|
current.RebuildRequestID == item.RebuildRequestID {
|
|
item.ID = current.ID
|
|
item.CreatedAt = current.CreatedAt
|
|
f.fabricRebuildAttempts[idx] = item
|
|
return item, nil
|
|
}
|
|
}
|
|
f.fabricRebuildAttempts = append(f.fabricRebuildAttempts, item)
|
|
return item, nil
|
|
}
|
|
|
|
func (f *fakeRepository) ListFabricServiceChannelRouteRebuildAttempts(_ context.Context, input ListFabricServiceChannelRouteRebuildAttemptsInput) ([]FabricServiceChannelRouteRebuildAttempt, error) {
|
|
out := []FabricServiceChannelRouteRebuildAttempt{}
|
|
for _, item := range f.fabricRebuildAttempts {
|
|
if item.ClusterID != input.ClusterID {
|
|
continue
|
|
}
|
|
if input.ReporterNodeID != "" && item.ReporterNodeID != input.ReporterNodeID {
|
|
continue
|
|
}
|
|
if input.RouteID != "" && item.RouteID != input.RouteID {
|
|
continue
|
|
}
|
|
if input.ReplacementRouteID != "" && item.ReplacementRouteID != input.ReplacementRouteID {
|
|
continue
|
|
}
|
|
if input.ServiceClass != "" && item.ServiceClass != input.ServiceClass {
|
|
continue
|
|
}
|
|
if input.RebuildStatus != "" && item.RebuildStatus != input.RebuildStatus {
|
|
continue
|
|
}
|
|
if input.RebuildRequestID != "" && item.RebuildRequestID != input.RebuildRequestID {
|
|
continue
|
|
}
|
|
payload := jsonObject(item.Payload)
|
|
if input.FeedbackSource != "" && firstNonEmptyString(item.FeedbackSource, jsonString(payload, "feedback_source")) != input.FeedbackSource {
|
|
continue
|
|
}
|
|
if input.FeedbackChannelID != "" && firstNonEmptyString(item.FeedbackChannelID, jsonString(payload, "feedback_channel_id")) != input.FeedbackChannelID {
|
|
continue
|
|
}
|
|
if input.FeedbackViolationStatus != "" && firstNonEmptyString(item.FeedbackViolationStatus, jsonString(payload, "feedback_violation_status")) != input.FeedbackViolationStatus {
|
|
continue
|
|
}
|
|
out = append(out, item)
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
func (f *fakeRepository) UpdateFabricServiceChannelRouteRebuildCorrelationSnapshot(_ context.Context, input UpdateFabricServiceChannelRouteRebuildCorrelationSnapshotInput) error {
|
|
for idx := range f.fabricRebuildAttempts {
|
|
if f.fabricRebuildAttempts[idx].ID != input.ID {
|
|
continue
|
|
}
|
|
f.fabricRebuildAttempts[idx].NodeTransitionStatus = input.NodeTransitionStatus
|
|
f.fabricRebuildAttempts[idx].NodeTransitionGeneration = input.NodeTransitionGeneration
|
|
f.fabricRebuildAttempts[idx].NodeTransitionObservedAt = input.NodeTransitionObservedAt
|
|
f.fabricRebuildAttempts[idx].NodeTransitionMatched = input.NodeTransitionMatched
|
|
f.fabricRebuildAttempts[idx].NodeRouteGenerationStatus = input.NodeRouteGenerationStatus
|
|
f.fabricRebuildAttempts[idx].NodeRouteGenerationAppliedAt = input.NodeRouteGenerationAppliedAt
|
|
f.fabricRebuildAttempts[idx].NodeRouteGenerationWithdrawnAt = input.NodeRouteGenerationWithdrawnAt
|
|
f.fabricRebuildAttempts[idx].NodeRouteGenerationMatched = input.NodeRouteGenerationMatched
|
|
f.fabricRebuildAttempts[idx].PostRebuildSelectedRouteID = input.PostRebuildSelectedRouteID
|
|
f.fabricRebuildAttempts[idx].PostRebuildSendPackets = input.PostRebuildSendPackets
|
|
f.fabricRebuildAttempts[idx].PostRebuildSendFailures = input.PostRebuildSendFailures
|
|
f.fabricRebuildAttempts[idx].PostRebuildSendFlowPackets = input.PostRebuildSendFlowPackets
|
|
f.fabricRebuildAttempts[idx].PostRebuildSendFlowDropped = input.PostRebuildSendFlowDropped
|
|
f.fabricRebuildAttempts[idx].GuardStatus = input.GuardStatus
|
|
f.fabricRebuildAttempts[idx].GuardSeverity = input.GuardSeverity
|
|
f.fabricRebuildAttempts[idx].GuardReason = input.GuardReason
|
|
f.fabricRebuildAttempts[idx].GuardTransitionDeadlineSeconds = input.GuardTransitionDeadlineSeconds
|
|
f.fabricRebuildAttempts[idx].GuardTrafficDeadlineSeconds = input.GuardTrafficDeadlineSeconds
|
|
f.fabricRebuildAttempts[idx].Timeline = append([]FabricServiceChannelRouteRebuildTimelineEvent{}, input.Timeline...)
|
|
snapshotAt := input.CorrelationSnapshotAt
|
|
f.fabricRebuildAttempts[idx].CorrelationSnapshotAt = &snapshotAt
|
|
return nil
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (f *fakeRepository) GetFabricServiceChannelSchemaStatus(_ context.Context, input GetFabricServiceChannelSchemaStatusInput) (FabricServiceChannelSchemaStatus, error) {
|
|
return FabricServiceChannelSchemaStatus{
|
|
ClusterID: input.ClusterID,
|
|
ObservedAt: time.Now().UTC(),
|
|
Status: "ready",
|
|
Reason: "schema_ready",
|
|
RequiredMigration: "000028_fabric_service_channel_rebuild_correlation_snapshot",
|
|
RequiredCheckCount: 1,
|
|
PassedCheckCount: 1,
|
|
RequiredChecks: []FabricServiceChannelSchemaCheck{{
|
|
CheckID: "fabric_service_channel_route_rebuild_attempts",
|
|
RelationName: "fabric_service_channel_route_rebuild_attempts",
|
|
Status: "present",
|
|
RequiredBy: "000028_fabric_service_channel_rebuild_correlation_snapshot",
|
|
}},
|
|
}, nil
|
|
}
|
|
|
|
func (f *fakeRepository) UpsertFabricServiceChannelRouteRebuildAlertSilence(_ context.Context, input SilenceFabricServiceChannelRouteRebuildAlertInput, expiresAt time.Time) (FabricServiceChannelRouteRebuildAlertSilence, error) {
|
|
createdAt := input.Now
|
|
if createdAt.IsZero() {
|
|
createdAt = time.Now().UTC()
|
|
}
|
|
item := FabricServiceChannelRouteRebuildAlertSilence{
|
|
ID: "fsc-rebuild-silence-" + strconv.Itoa(len(f.fabricRebuildSilences)+1),
|
|
ClusterID: input.ClusterID,
|
|
IncidentSource: input.IncidentSource,
|
|
ChannelID: input.ChannelID,
|
|
ReporterNodeID: input.ReporterNodeID,
|
|
RouteID: input.RouteID,
|
|
DisplayRouteID: input.RouteID,
|
|
GuardStatus: input.GuardStatus,
|
|
Generation: input.Generation,
|
|
Reason: input.Reason,
|
|
CreatedByUserID: &input.ActorUserID,
|
|
CreatedAt: createdAt,
|
|
ExpiresAt: expiresAt,
|
|
Payload: mustJSONRaw(map[string]any{
|
|
"schema_version": "rap.fabric_service_channel_rebuild_alert_silence.v1",
|
|
"reason": input.Reason,
|
|
"incident_source": input.IncidentSource,
|
|
"channel_id": input.ChannelID,
|
|
}),
|
|
}
|
|
if channelID, routeID, ok := fabricServiceChannelParseAccessDecisionSilenceRouteID(input.RouteID); ok {
|
|
item.IncidentSource = firstNonEmptyString(item.IncidentSource, "access_decision")
|
|
item.ChannelID = firstNonEmptyString(item.ChannelID, channelID)
|
|
item.DisplayRouteID = routeID
|
|
}
|
|
for idx, current := range f.fabricRebuildSilences {
|
|
if current.ClusterID == item.ClusterID && current.ReporterNodeID == item.ReporterNodeID && current.RouteID == item.RouteID && current.GuardStatus == item.GuardStatus && current.Generation == item.Generation {
|
|
f.fabricRebuildSilences[idx] = item
|
|
return item, nil
|
|
}
|
|
}
|
|
f.fabricRebuildSilences = append(f.fabricRebuildSilences, item)
|
|
return item, nil
|
|
}
|
|
|
|
func (f *fakeRepository) ListFabricServiceChannelRouteRebuildAlertSilences(_ context.Context, clusterID string, now time.Time) ([]FabricServiceChannelRouteRebuildAlertSilence, error) {
|
|
out := []FabricServiceChannelRouteRebuildAlertSilence{}
|
|
for _, item := range f.fabricRebuildSilences {
|
|
if item.ClusterID == clusterID && item.ExpiresAt.After(now) {
|
|
out = append(out, item)
|
|
}
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
func (f *fakeRepository) DeleteFabricServiceChannelRouteRebuildAlertSilence(_ context.Context, input UnsilenceFabricServiceChannelRouteRebuildAlertInput) (FabricServiceChannelRouteRebuildAlertSilence, error) {
|
|
for idx, item := range f.fabricRebuildSilences {
|
|
if item.ClusterID == input.ClusterID && item.ID == input.SilenceID {
|
|
f.fabricRebuildSilences = append(f.fabricRebuildSilences[:idx], f.fabricRebuildSilences[idx+1:]...)
|
|
return item, nil
|
|
}
|
|
}
|
|
return FabricServiceChannelRouteRebuildAlertSilence{}, pgx.ErrNoRows
|
|
}
|
|
|
|
func (f *fakeRepository) ListQoSPolicies(context.Context, string) ([]MeshQoSPolicy, error) {
|
|
return nil, nil
|
|
}
|
|
|
|
func (f *fakeRepository) ListFabricEntryPoints(context.Context, string) ([]FabricEntryPoint, error) {
|
|
return nil, nil
|
|
}
|
|
|
|
func (f *fakeRepository) CreateFabricEntryPoint(_ context.Context, input CreateFabricEntryPointInput) (FabricEntryPoint, error) {
|
|
f.lastEntryPointInput = input
|
|
return FabricEntryPoint{
|
|
ID: "entry-1",
|
|
ClusterID: input.ClusterID,
|
|
Name: input.Name,
|
|
Status: input.Status,
|
|
EndpointType: input.EndpointType,
|
|
PublicEndpoint: input.PublicEndpoint,
|
|
Policy: input.Policy,
|
|
Metadata: input.Metadata,
|
|
}, nil
|
|
}
|
|
|
|
func (f *fakeRepository) SetFabricEntryPointNode(_ context.Context, input SetFabricEntryPointNodeInput) (FabricEntryPointNode, error) {
|
|
return FabricEntryPointNode{
|
|
EntryPointID: input.EntryPointID,
|
|
ClusterID: input.ClusterID,
|
|
NodeID: input.NodeID,
|
|
Status: input.Status,
|
|
Priority: input.Priority,
|
|
Metadata: input.Metadata,
|
|
}, nil
|
|
}
|
|
|
|
func (f *fakeRepository) ListFabricEntryPointNodes(context.Context, string, string) ([]FabricEntryPointNode, error) {
|
|
return []FabricEntryPointNode{}, nil
|
|
}
|
|
|
|
func (f *fakeRepository) ListFabricEgressPools(context.Context, string) ([]FabricEgressPool, error) {
|
|
return nil, nil
|
|
}
|
|
|
|
func (f *fakeRepository) CreateFabricEgressPool(_ context.Context, input CreateFabricEgressPoolInput) (FabricEgressPool, error) {
|
|
f.lastEgressPoolInput = input
|
|
return FabricEgressPool{
|
|
ID: "egress-1",
|
|
ClusterID: input.ClusterID,
|
|
Name: input.Name,
|
|
Status: input.Status,
|
|
Description: input.Description,
|
|
RouteScope: input.RouteScope,
|
|
Policy: input.Policy,
|
|
Metadata: input.Metadata,
|
|
}, nil
|
|
}
|
|
|
|
func (f *fakeRepository) SetFabricEgressPoolNode(_ context.Context, input SetFabricEgressPoolNodeInput) (FabricEgressPoolNode, error) {
|
|
return FabricEgressPoolNode{
|
|
EgressPoolID: input.EgressPoolID,
|
|
ClusterID: input.ClusterID,
|
|
NodeID: input.NodeID,
|
|
Status: input.Status,
|
|
Priority: input.Priority,
|
|
Metadata: input.Metadata,
|
|
}, nil
|
|
}
|
|
|
|
func (f *fakeRepository) ListFabricEgressPoolNodes(context.Context, string, string) ([]FabricEgressPoolNode, error) {
|
|
return []FabricEgressPoolNode{}, nil
|
|
}
|
|
|
|
func (f *fakeRepository) GetClusterAuthorityState(context.Context, string) (ClusterAuthorityState, error) {
|
|
if f.authorityState.ClusterID == "" {
|
|
return ClusterAuthorityState{ClusterID: "cluster-1", AuthorityState: "authoritative", MutationMode: "normal"}, nil
|
|
}
|
|
return f.authorityState, nil
|
|
}
|
|
|
|
func (f *fakeRepository) UpdateClusterAuthorityState(_ context.Context, input UpdateClusterAuthorityInput) (ClusterAuthorityState, error) {
|
|
return ClusterAuthorityState{
|
|
ClusterID: input.ClusterID,
|
|
AuthorityState: input.AuthorityState,
|
|
MutationMode: input.MutationMode,
|
|
Notes: input.Notes,
|
|
}, nil
|
|
}
|
|
|
|
func (f *fakeRepository) ListClusterAdminSummaries(context.Context) ([]ClusterAdminSummary, error) {
|
|
return nil, nil
|
|
}
|
|
|
|
func (f *fakeRepository) CreateVPNConnection(_ context.Context, input CreateVPNConnectionInput) (VPNConnection, error) {
|
|
f.lastVPNConnectionInput = input
|
|
return VPNConnection{
|
|
ID: "vpn-1",
|
|
ClusterID: input.ClusterID,
|
|
OrganizationID: input.OrganizationID,
|
|
Name: input.Name,
|
|
TargetEndpoint: input.TargetEndpoint,
|
|
ProtocolFamily: input.ProtocolFamily,
|
|
CredentialRef: input.CredentialRef,
|
|
Mode: input.Mode,
|
|
DesiredState: input.DesiredState,
|
|
AllowedNodePolicy: input.AllowedNodePolicy,
|
|
RoutingUsage: input.RoutingUsage,
|
|
RoutePolicy: input.RoutePolicy,
|
|
QoSPolicy: input.QoSPolicy,
|
|
PlacementPolicy: input.PlacementPolicy,
|
|
Status: VPNConnectionStatusDisabled,
|
|
Metadata: input.Metadata,
|
|
}, nil
|
|
}
|
|
|
|
func (f *fakeRepository) ListVPNConnections(context.Context, string) ([]VPNConnection, error) {
|
|
return nil, nil
|
|
}
|
|
|
|
func (f *fakeRepository) GetVPNConnection(context.Context, string, string) (VPNConnection, error) {
|
|
if f.vpnConnection.ID != "" {
|
|
return f.vpnConnection, nil
|
|
}
|
|
return VPNConnection{
|
|
ID: "vpn-1",
|
|
ClusterID: "cluster-1",
|
|
Mode: VPNConnectionModeSingleActive,
|
|
DesiredState: VPNConnectionDesiredEnabled,
|
|
}, nil
|
|
}
|
|
|
|
func (f *fakeRepository) UpdateVPNConnectionDesiredState(_ context.Context, input UpdateVPNConnectionDesiredStateInput) (VPNConnection, error) {
|
|
return VPNConnection{ID: input.VPNConnectionID, ClusterID: input.ClusterID, DesiredState: input.DesiredState}, nil
|
|
}
|
|
|
|
func (f *fakeRepository) UpsertVPNConnectionRoutePolicy(_ context.Context, input UpsertVPNConnectionRoutePolicyInput) (VPNConnectionRoutePolicy, error) {
|
|
return VPNConnectionRoutePolicy{
|
|
ID: "route-policy-1",
|
|
VPNConnectionID: input.VPNConnectionID,
|
|
ClusterID: input.ClusterID,
|
|
RouteType: input.RouteType,
|
|
Destination: input.Destination,
|
|
Action: input.Action,
|
|
ServiceType: input.ServiceType,
|
|
Priority: input.Priority,
|
|
Policy: input.Policy,
|
|
Status: input.Status,
|
|
}, nil
|
|
}
|
|
|
|
func (f *fakeRepository) ListVPNConnectionRoutePolicies(context.Context, string, string) ([]VPNConnectionRoutePolicy, error) {
|
|
return nil, nil
|
|
}
|
|
|
|
func (f *fakeRepository) SetVPNConnectionAllowedNodes(_ context.Context, input SetVPNConnectionAllowedNodesInput) ([]VPNConnectionAllowedNode, error) {
|
|
f.lastAllowedNodesInput = input
|
|
items := make([]VPNConnectionAllowedNode, 0, len(input.NodeIDs))
|
|
for _, nodeID := range input.NodeIDs {
|
|
items = append(items, VPNConnectionAllowedNode{
|
|
VPNConnectionID: input.VPNConnectionID,
|
|
ClusterID: input.ClusterID,
|
|
NodeID: nodeID,
|
|
RolePreference: input.RolePreference,
|
|
Status: "active",
|
|
Metadata: input.Metadata,
|
|
})
|
|
}
|
|
return items, nil
|
|
}
|
|
|
|
func (f *fakeRepository) ListVPNConnectionAllowedNodes(context.Context, string, string) ([]VPNConnectionAllowedNode, error) {
|
|
return nil, nil
|
|
}
|
|
|
|
func (f *fakeRepository) AcquireVPNConnectionLease(_ context.Context, input AcquireVPNConnectionLeaseInput, expiresAt time.Time, fencingToken string) (VPNConnectionLease, error) {
|
|
if f.acquireVPNLeaseErr != nil {
|
|
return VPNConnectionLease{}, f.acquireVPNLeaseErr
|
|
}
|
|
return VPNConnectionLease{
|
|
ID: "lease-1",
|
|
VPNConnectionID: input.VPNConnectionID,
|
|
ClusterID: input.ClusterID,
|
|
OwnerNodeID: input.OwnerNodeID,
|
|
LeaseGeneration: 1,
|
|
FencingToken: fencingToken,
|
|
Status: VPNLeaseStatusActive,
|
|
ExpiresAt: expiresAt,
|
|
Metadata: input.Metadata,
|
|
}, nil
|
|
}
|
|
|
|
func (f *fakeRepository) RenewVPNConnectionLease(_ context.Context, input RenewVPNConnectionLeaseInput, expiresAt time.Time) (VPNConnectionLease, error) {
|
|
if f.renewVPNLeaseErr != nil {
|
|
return VPNConnectionLease{}, f.renewVPNLeaseErr
|
|
}
|
|
return VPNConnectionLease{ID: input.LeaseID, VPNConnectionID: input.VPNConnectionID, ClusterID: input.ClusterID, OwnerNodeID: input.OwnerNodeID, FencingToken: input.FencingToken, Status: VPNLeaseStatusActive, ExpiresAt: expiresAt}, nil
|
|
}
|
|
|
|
func (f *fakeRepository) RenewNodeVPNAssignmentLease(_ context.Context, input RenewNodeVPNAssignmentLeaseInput, expiresAt time.Time) (VPNConnectionLease, error) {
|
|
if f.renewVPNLeaseErr != nil {
|
|
return VPNConnectionLease{}, f.renewVPNLeaseErr
|
|
}
|
|
return VPNConnectionLease{ID: input.LeaseID, VPNConnectionID: input.VPNConnectionID, ClusterID: input.ClusterID, OwnerNodeID: input.OwnerNodeID, Status: VPNLeaseStatusActive, ExpiresAt: expiresAt}, nil
|
|
}
|
|
|
|
func (f *fakeRepository) ReleaseVPNConnectionLease(_ context.Context, input ReleaseVPNConnectionLeaseInput) (VPNConnectionLease, error) {
|
|
return VPNConnectionLease{ID: input.LeaseID, VPNConnectionID: input.VPNConnectionID, ClusterID: input.ClusterID, OwnerNodeID: input.OwnerNodeID, FencingToken: input.FencingToken, Status: VPNLeaseStatusReleased}, nil
|
|
}
|
|
|
|
func (f *fakeRepository) FenceVPNConnectionLease(_ context.Context, input FenceVPNConnectionLeaseInput) (VPNConnectionLease, error) {
|
|
return VPNConnectionLease{ID: input.LeaseID, VPNConnectionID: input.VPNConnectionID, ClusterID: input.ClusterID, Status: VPNLeaseStatusFenced}, nil
|
|
}
|
|
|
|
func (f *fakeRepository) GetActiveVPNConnectionLease(context.Context, string, string) (VPNConnectionLease, error) {
|
|
return VPNConnectionLease{ID: "lease-1", Status: VPNLeaseStatusActive}, nil
|
|
}
|
|
|
|
func (f *fakeRepository) CheckVPNLeaseOwnerEligibility(context.Context, string, string, string) (VPNLeaseOwnerEligibility, error) {
|
|
if f.ownerEligibilityErr != nil {
|
|
return VPNLeaseOwnerEligibility{}, f.ownerEligibilityErr
|
|
}
|
|
if f.ownerEligibility.VPNConnectionID != "" {
|
|
return f.ownerEligibility, nil
|
|
}
|
|
return VPNLeaseOwnerEligibility{
|
|
VPNConnectionID: "vpn-1",
|
|
ClusterID: "cluster-1",
|
|
OrganizationID: "org-1",
|
|
OwnerNodeID: "node-1",
|
|
MembershipStatus: "active",
|
|
NodeRegistrationStatus: NodeRegistrationActive,
|
|
AllowedByPolicy: true,
|
|
HasAuthorizedRole: true,
|
|
}, nil
|
|
}
|
|
|
|
func (f *fakeRepository) ExpireStaleVPNConnectionLeases(context.Context, string, time.Time) ([]VPNConnectionLease, error) {
|
|
return f.expiredVPNLeases, nil
|
|
}
|
|
|
|
func (f *fakeRepository) ListNodeVPNAssignments(context.Context, string, string) ([]NodeVPNAssignment, error) {
|
|
return f.nodeVPNAssignments, nil
|
|
}
|
|
|
|
func (f *fakeRepository) ReportNodeVPNAssignmentStatus(_ context.Context, input ReportNodeVPNAssignmentStatusInput) (NodeVPNAssignmentStatus, error) {
|
|
return NodeVPNAssignmentStatus{
|
|
ID: "status-1",
|
|
VPNConnectionID: input.VPNConnectionID,
|
|
ClusterID: input.ClusterID,
|
|
NodeID: input.NodeID,
|
|
ObservedStatus: input.ObservedStatus,
|
|
StatusPayload: input.StatusPayload,
|
|
ObservedAt: input.ObservedAt,
|
|
}, nil
|
|
}
|
|
|
|
func (f *fakeRepository) GetVPNClientProfile(
|
|
_ context.Context,
|
|
clusterID, organizationID, userID, preferredEntryNodeID, preferredExitNodeID string,
|
|
generatedAt time.Time,
|
|
) (VPNClientProfile, error) {
|
|
f.lastPreferredEntryNodeID = preferredEntryNodeID
|
|
f.lastPreferredExitNodeID = preferredExitNodeID
|
|
if f.vpnClientProfile.SchemaVersion != "" {
|
|
profile := f.vpnClientProfile
|
|
profile.ClusterID = clusterID
|
|
profile.OrganizationID = organizationID
|
|
profile.UserID = userID
|
|
profile.GeneratedAt = generatedAt
|
|
return profile, nil
|
|
}
|
|
return VPNClientProfile{
|
|
SchemaVersion: "rap.vpn_client_profile.v1",
|
|
ClusterID: clusterID,
|
|
OrganizationID: organizationID,
|
|
UserID: userID,
|
|
GeneratedAt: generatedAt,
|
|
}, nil
|
|
}
|
|
|
|
func (f *fakeRepository) RecordAudit(_ context.Context, event ClusterAuditEvent) error {
|
|
f.auditEvents = append(f.auditEvents, event)
|
|
return nil
|
|
}
|
|
|
|
func (f *fakeRepository) ListAuditEvents(_ context.Context, input ListAuditEventsInput) ([]ClusterAuditEvent, error) {
|
|
limit := input.Limit
|
|
if limit <= 0 || limit > 200 {
|
|
limit = 100
|
|
}
|
|
eventTypes := map[string]bool{}
|
|
for _, eventType := range trimStringSlice(input.EventTypes) {
|
|
eventTypes[eventType] = true
|
|
}
|
|
targetTypes := map[string]bool{}
|
|
for _, targetType := range trimStringSlice(input.TargetTypes) {
|
|
targetTypes[targetType] = true
|
|
}
|
|
out := []ClusterAuditEvent{}
|
|
for _, event := range f.auditEvents {
|
|
if event.ClusterID != nil && input.ClusterID != "" && *event.ClusterID != input.ClusterID {
|
|
continue
|
|
}
|
|
if len(eventTypes) > 0 && !eventTypes[event.EventType] {
|
|
continue
|
|
}
|
|
if len(targetTypes) > 0 && !targetTypes[event.TargetType] {
|
|
continue
|
|
}
|
|
out = append(out, event)
|
|
if len(out) >= limit {
|
|
break
|
|
}
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
func stringPtr(value string) *string {
|
|
return &value
|
|
}
|