Files
rdp-proxy/agents/rap-node-agent/internal/mesh/server_test.go
T
2026-05-15 16:38:00 +03:00

4799 lines
189 KiB
Go

package mesh
import (
"bytes"
"context"
"crypto/ed25519"
"crypto/sha256"
"encoding/base64"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"net/http/httptest"
"strings"
"sync"
"testing"
"time"
"github.com/example/remote-access-platform/agents/rap-node-agent/internal/authority"
"github.com/gorilla/websocket"
)
func TestMeshHealthAcceptsSameCluster(t *testing.T) {
local := PeerIdentity{ClusterID: "cluster-1", NodeID: "node-b"}
server := httptest.NewServer(Server{Local: local}.Handler())
defer server.Close()
client := NewClient(server.URL)
ack, err := client.SendHealth(context.Background(), NewHealthMessage(
PeerIdentity{ClusterID: "cluster-1", NodeID: "node-a"},
local,
))
if err != nil {
t.Fatalf("send health: %v", err)
}
if !ack.Accepted || ack.By.NodeID != "node-b" {
t.Fatalf("unexpected ack: %+v", ack)
}
}
func TestMeshHealthRejectsClusterMismatch(t *testing.T) {
local := PeerIdentity{ClusterID: "cluster-1", NodeID: "node-b"}
server := httptest.NewServer(Server{Local: local}.Handler())
defer server.Close()
message := NewHealthMessage(PeerIdentity{ClusterID: "cluster-2", NodeID: "node-a"}, local)
payload, err := json.Marshal(message)
if err != nil {
t.Fatalf("marshal message: %v", err)
}
resp, err := http.Post(server.URL+"/mesh/v1/health", "application/json", bytes.NewReader(payload))
if err != nil {
t.Fatalf("post health: %v", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusForbidden {
t.Fatalf("status = %d, want %d", resp.StatusCode, http.StatusForbidden)
}
}
func TestMeshForwardingDisabled(t *testing.T) {
server := httptest.NewServer(Server{Local: PeerIdentity{ClusterID: "cluster-1", NodeID: "node-b"}}.Handler())
defer server.Close()
resp, err := http.Post(server.URL+"/mesh/v1/forward", "application/octet-stream", bytes.NewReader([]byte("payload")))
if err != nil {
t.Fatalf("post forward: %v", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusNotImplemented {
t.Fatalf("status = %d, want %d", resp.StatusCode, http.StatusNotImplemented)
}
}
func TestMeshForwardingGateEnabledStillHasNoProductionRuntime(t *testing.T) {
local := PeerIdentity{ClusterID: "cluster-1", NodeID: "node-b"}
server := httptest.NewServer(Server{
Local: local,
ProductionForwardingEnabled: true,
}.Handler())
defer server.Close()
payload, err := json.Marshal(validProductionEnvelope(local))
if err != nil {
t.Fatalf("marshal envelope: %v", err)
}
resp, err := http.Post(server.URL+"/mesh/v1/forward", "application/json", bytes.NewReader(payload))
if err != nil {
t.Fatalf("post forward: %v", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusNotImplemented {
t.Fatalf("status = %d, want %d", resp.StatusCode, http.StatusNotImplemented)
}
}
func TestMeshForwardingGateDeliversFabricControlAtDestination(t *testing.T) {
local := PeerIdentity{ClusterID: "cluster-1", NodeID: "node-c"}
var events []ProductionForwardLogEntry
server := httptest.NewServer(Server{
Local: local,
ProductionForwardingEnabled: true,
ProductionForwardLogger: func(entry ProductionForwardLogEntry) {
events = append(events, entry)
},
}.Handler())
defer server.Close()
envelope := validProductionEnvelope(local)
envelope.SourceNodeID = "node-a"
envelope.DestinationNodeID = local.NodeID
envelope.CurrentHopNodeID = local.NodeID
envelope.NextHopNodeID = local.NodeID
payload, err := json.Marshal(envelope)
if err != nil {
t.Fatalf("marshal envelope: %v", err)
}
resp, err := http.Post(server.URL+"/mesh/v1/forward", "application/json", bytes.NewReader(payload))
if err != nil {
t.Fatalf("post forward: %v", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
t.Fatalf("status = %d, want %d", resp.StatusCode, http.StatusOK)
}
var result ProductionForwardResult
if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
t.Fatalf("decode result: %v", err)
}
if !result.Accepted || !result.Delivered || result.Forwarded || result.By.NodeID != local.NodeID {
t.Fatalf("unexpected result: %+v", result)
}
if !hasProductionForwardEvent(events, "production_forward_accepted") || !hasProductionForwardEvent(events, "production_forward_delivered") {
t.Fatalf("missing production forward events: %+v", events)
}
}
func TestMeshForwardingGateForwardsDirectFabricControlToNextHop(t *testing.T) {
nodeC := PeerIdentity{ClusterID: "cluster-1", NodeID: "node-c"}
var deliveredObservation ProductionEnvelopeObservation
serverC := httptest.NewServer(Server{
Local: nodeC,
ProductionForwardingEnabled: true,
ProductionEnvelopeObserver: func(_ context.Context, observation ProductionEnvelopeObservation) error {
deliveredObservation = observation
return nil
},
}.Handler())
defer serverC.Close()
nodeB := PeerIdentity{ClusterID: "cluster-1", NodeID: "node-b"}
serverB := httptest.NewServer(Server{
Local: nodeB,
ProductionForwardingEnabled: true,
ProductionForwardTransport: NewHTTPProductionForwardTransport(map[string]string{
nodeC.NodeID: serverC.URL,
}),
}.Handler())
defer serverB.Close()
envelope := validProductionEnvelope(nodeB)
envelope.SourceNodeID = "node-a"
envelope.DestinationNodeID = nodeC.NodeID
envelope.CurrentHopNodeID = nodeB.NodeID
envelope.NextHopNodeID = nodeC.NodeID
payload, err := json.Marshal(envelope)
if err != nil {
t.Fatalf("marshal envelope: %v", err)
}
resp, err := http.Post(serverB.URL+"/mesh/v1/forward", "application/json", bytes.NewReader(payload))
if err != nil {
t.Fatalf("post forward: %v", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
t.Fatalf("status = %d, want %d", resp.StatusCode, http.StatusOK)
}
var result ProductionForwardResult
if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
t.Fatalf("decode result: %v", err)
}
if !result.Accepted || !result.Forwarded || !result.Delivered || result.NextNodeID != nodeC.NodeID || result.By.NodeID != nodeB.NodeID {
t.Fatalf("unexpected forward result: %+v", result)
}
if deliveredObservation.CurrentHopNodeID != nodeC.NodeID || deliveredObservation.MessageID != envelope.MessageID {
t.Fatalf("destination did not observe forwarded envelope: %+v", deliveredObservation)
}
}
func TestMeshForwardingGateForwardsMultiHopFabricControlByRoutePath(t *testing.T) {
nodeC := PeerIdentity{ClusterID: "cluster-1", NodeID: "node-c"}
var deliveredObservation ProductionEnvelopeObservation
var nodeREvents []ProductionForwardLogEntry
var nodeBEvents []ProductionForwardLogEntry
serverC := httptest.NewServer(Server{
Local: nodeC,
ProductionForwardingEnabled: true,
ProductionEnvelopeObserver: func(_ context.Context, observation ProductionEnvelopeObservation) error {
deliveredObservation = observation
return nil
},
}.Handler())
defer serverC.Close()
nodeR := PeerIdentity{ClusterID: "cluster-1", NodeID: "node-r"}
serverR := httptest.NewServer(Server{
Local: nodeR,
ProductionForwardingEnabled: true,
ProductionForwardTransport: NewHTTPProductionForwardTransport(map[string]string{
nodeC.NodeID: serverC.URL,
}),
ProductionForwardLogger: func(entry ProductionForwardLogEntry) {
nodeREvents = append(nodeREvents, entry)
},
}.Handler())
defer serverR.Close()
nodeB := PeerIdentity{ClusterID: "cluster-1", NodeID: "node-b"}
serverB := httptest.NewServer(Server{
Local: nodeB,
ProductionForwardingEnabled: true,
ProductionForwardTransport: NewHTTPProductionForwardTransport(map[string]string{
nodeR.NodeID: serverR.URL,
}),
ProductionForwardLogger: func(entry ProductionForwardLogEntry) {
nodeBEvents = append(nodeBEvents, entry)
},
}.Handler())
defer serverB.Close()
envelope := validProductionEnvelope(nodeB)
envelope.SourceNodeID = "node-a"
envelope.DestinationNodeID = nodeC.NodeID
envelope.CurrentHopNodeID = nodeB.NodeID
envelope.NextHopNodeID = nodeR.NodeID
envelope.RoutePath = []string{"node-a", nodeB.NodeID, nodeR.NodeID, nodeC.NodeID}
payload, err := json.Marshal(envelope)
if err != nil {
t.Fatalf("marshal envelope: %v", err)
}
resp, err := http.Post(serverB.URL+"/mesh/v1/forward", "application/json", bytes.NewReader(payload))
if err != nil {
t.Fatalf("post forward: %v", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
t.Fatalf("status = %d, want %d", resp.StatusCode, http.StatusOK)
}
var result ProductionForwardResult
if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
t.Fatalf("decode result: %v", err)
}
if !result.Accepted || !result.Forwarded || !result.Delivered || result.NextNodeID != nodeR.NodeID || result.By.NodeID != nodeB.NodeID {
t.Fatalf("unexpected multi-hop result: %+v", result)
}
if deliveredObservation.CurrentHopNodeID != nodeC.NodeID || deliveredObservation.NextHopNodeID != nodeC.NodeID {
t.Fatalf("destination did not observe final hop: %+v", deliveredObservation)
}
if len(deliveredObservation.VisitedNodeIDs) != 2 || deliveredObservation.VisitedNodeIDs[0] != nodeB.NodeID || deliveredObservation.VisitedNodeIDs[1] != nodeR.NodeID {
t.Fatalf("visited path not propagated: %+v", deliveredObservation.VisitedNodeIDs)
}
if !hasProductionForwardEvent(nodeBEvents, "production_forward_forwarded") || !hasProductionForwardEvent(nodeREvents, "production_forward_forwarded") {
t.Fatalf("missing relay forward events: nodeB=%+v nodeR=%+v", nodeBEvents, nodeREvents)
}
}
func TestMeshForwardingGateForwardsConfiguredProductionRoute(t *testing.T) {
nodeC := PeerIdentity{ClusterID: "cluster-1", NodeID: "node-c"}
route := configuredProductionRoute("route-1", []string{"node-a", "node-b", "node-r", nodeC.NodeID})
var deliveredObservation ProductionEnvelopeObservation
serverC := httptest.NewServer(Server{
Local: nodeC,
ProductionForwardingEnabled: true,
ProductionRoutes: []SyntheticRoute{route},
ProductionEnvelopeObserver: func(_ context.Context, observation ProductionEnvelopeObservation) error {
deliveredObservation = observation
return nil
},
}.Handler())
defer serverC.Close()
nodeR := PeerIdentity{ClusterID: "cluster-1", NodeID: "node-r"}
serverR := httptest.NewServer(Server{
Local: nodeR,
ProductionForwardingEnabled: true,
ProductionRoutes: []SyntheticRoute{route},
ProductionForwardTransport: NewHTTPProductionForwardTransport(map[string]string{
nodeC.NodeID: serverC.URL,
}),
}.Handler())
defer serverR.Close()
nodeB := PeerIdentity{ClusterID: "cluster-1", NodeID: "node-b"}
serverB := httptest.NewServer(Server{
Local: nodeB,
ProductionForwardingEnabled: true,
ProductionRoutes: []SyntheticRoute{route},
ProductionForwardTransport: NewHTTPProductionForwardTransport(map[string]string{
nodeR.NodeID: serverR.URL,
}),
}.Handler())
defer serverB.Close()
envelope := validProductionEnvelope(nodeB)
envelope.SourceNodeID = "node-a"
envelope.DestinationNodeID = nodeC.NodeID
envelope.CurrentHopNodeID = nodeB.NodeID
envelope.NextHopNodeID = nodeR.NodeID
envelope.RoutePath = route.Hops
payload, err := json.Marshal(envelope)
if err != nil {
t.Fatalf("marshal envelope: %v", err)
}
resp, err := http.Post(serverB.URL+"/mesh/v1/forward", "application/json", bytes.NewReader(payload))
if err != nil {
t.Fatalf("post forward: %v", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
t.Fatalf("status = %d, want %d", resp.StatusCode, http.StatusOK)
}
if deliveredObservation.RouteID != route.RouteID || deliveredObservation.CurrentHopNodeID != nodeC.NodeID {
t.Fatalf("configured route was not delivered: %+v", deliveredObservation)
}
}
func TestMeshForwardingGateRejectsUnknownConfiguredProductionRoute(t *testing.T) {
local := PeerIdentity{ClusterID: "cluster-1", NodeID: "node-b"}
server := httptest.NewServer(Server{
Local: local,
ProductionForwardingEnabled: true,
ProductionRoutes: []SyntheticRoute{
configuredProductionRoute("route-other", []string{"node-a", local.NodeID, "node-c"}),
},
}.Handler())
defer server.Close()
envelope := validProductionEnvelope(local)
payload, err := json.Marshal(envelope)
if err != nil {
t.Fatalf("marshal envelope: %v", err)
}
resp, err := http.Post(server.URL+"/mesh/v1/forward", "application/json", bytes.NewReader(payload))
if err != nil {
t.Fatalf("post forward: %v", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusNotFound {
t.Fatalf("status = %d, want %d", resp.StatusCode, http.StatusNotFound)
}
}
func TestMeshForwardingGateRejectsConfiguredProductionRouteWrongNextHop(t *testing.T) {
local := PeerIdentity{ClusterID: "cluster-1", NodeID: "node-b"}
route := configuredProductionRoute("route-1", []string{"node-a", local.NodeID, "node-r", "node-c"})
server := httptest.NewServer(Server{
Local: local,
ProductionForwardingEnabled: true,
ProductionRoutes: []SyntheticRoute{route},
}.Handler())
defer server.Close()
envelope := validProductionEnvelope(local)
envelope.SourceNodeID = "node-a"
envelope.DestinationNodeID = "node-c"
envelope.CurrentHopNodeID = local.NodeID
envelope.NextHopNodeID = "node-c"
envelope.RoutePath = route.Hops
payload, err := json.Marshal(envelope)
if err != nil {
t.Fatalf("marshal envelope: %v", err)
}
resp, err := http.Post(server.URL+"/mesh/v1/forward", "application/json", bytes.NewReader(payload))
if err != nil {
t.Fatalf("post forward: %v", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusBadRequest {
t.Fatalf("status = %d, want %d", resp.StatusCode, http.StatusBadRequest)
}
}
func TestMeshForwardingGateRejectsRoutePathWrongNextHop(t *testing.T) {
local := PeerIdentity{ClusterID: "cluster-1", NodeID: "node-b"}
var events []ProductionForwardLogEntry
server := httptest.NewServer(Server{
Local: local,
ProductionForwardingEnabled: true,
ProductionForwardLogger: func(entry ProductionForwardLogEntry) {
events = append(events, entry)
},
}.Handler())
defer server.Close()
envelope := validProductionEnvelope(local)
envelope.SourceNodeID = "node-a"
envelope.DestinationNodeID = "node-c"
envelope.CurrentHopNodeID = local.NodeID
envelope.NextHopNodeID = "node-x"
envelope.RoutePath = []string{"node-a", local.NodeID, "node-r", "node-c"}
payload, err := json.Marshal(envelope)
if err != nil {
t.Fatalf("marshal envelope: %v", err)
}
resp, err := http.Post(server.URL+"/mesh/v1/forward", "application/json", bytes.NewReader(payload))
if err != nil {
t.Fatalf("post forward: %v", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusBadRequest {
t.Fatalf("status = %d, want %d", resp.StatusCode, http.StatusBadRequest)
}
if !hasProductionForwardEvent(events, "production_forward_rejected") {
t.Fatalf("missing reject event: %+v", events)
}
}
func TestMeshForwardingGateRejectsRoutePathLoop(t *testing.T) {
local := PeerIdentity{ClusterID: "cluster-1", NodeID: "node-b"}
server := httptest.NewServer(Server{
Local: local,
ProductionForwardingEnabled: true,
}.Handler())
defer server.Close()
envelope := validProductionEnvelope(local)
envelope.SourceNodeID = "node-a"
envelope.DestinationNodeID = "node-c"
envelope.CurrentHopNodeID = local.NodeID
envelope.NextHopNodeID = "node-r"
envelope.RoutePath = []string{"node-a", local.NodeID, "node-r", local.NodeID, "node-c"}
payload, err := json.Marshal(envelope)
if err != nil {
t.Fatalf("marshal envelope: %v", err)
}
resp, err := http.Post(server.URL+"/mesh/v1/forward", "application/json", bytes.NewReader(payload))
if err != nil {
t.Fatalf("post forward: %v", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusForbidden {
t.Fatalf("status = %d, want %d", resp.StatusCode, http.StatusForbidden)
}
}
func TestMeshForwardingGateRejectsInvalidProductionEnvelope(t *testing.T) {
local := PeerIdentity{ClusterID: "cluster-1", NodeID: "node-b"}
server := httptest.NewServer(Server{
Local: local,
ProductionForwardingEnabled: true,
}.Handler())
defer server.Close()
envelope := validProductionEnvelope(local)
envelope.PayloadHash = "bad-hash"
payload, err := json.Marshal(envelope)
if err != nil {
t.Fatalf("marshal envelope: %v", err)
}
resp, err := http.Post(server.URL+"/mesh/v1/forward", "application/json", bytes.NewReader(payload))
if err != nil {
t.Fatalf("post forward: %v", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusBadRequest {
t.Fatalf("status = %d, want %d", resp.StatusCode, http.StatusBadRequest)
}
}
func TestMeshForwardingGateRejectsOversizedProductionEnvelopePayload(t *testing.T) {
local := PeerIdentity{ClusterID: "cluster-1", NodeID: "node-b"}
observed := false
server := httptest.NewServer(Server{
Local: local,
ProductionForwardingEnabled: true,
ProductionEnvelopeObserver: func(context.Context, ProductionEnvelopeObservation) error {
observed = true
return nil
},
}.Handler())
defer server.Close()
envelope := validProductionEnvelope(local)
envelope.Payload = json.RawMessage(`"` + string(bytes.Repeat([]byte("a"), MaxProductionEnvelopePayloadBytes+1)) + `"`)
sum := sha256.Sum256(envelope.Payload)
envelope.PayloadLength = len(envelope.Payload)
envelope.PayloadHash = hex.EncodeToString(sum[:])
payload, err := json.Marshal(envelope)
if err != nil {
t.Fatalf("marshal envelope: %v", err)
}
resp, err := http.Post(server.URL+"/mesh/v1/forward", "application/json", bytes.NewReader(payload))
if err != nil {
t.Fatalf("post forward: %v", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusBadRequest {
t.Fatalf("status = %d, want %d", resp.StatusCode, http.StatusBadRequest)
}
if observed {
t.Fatal("observer called for oversized envelope")
}
}
func TestMeshForwardingGateRejectsFutureCreatedAt(t *testing.T) {
local := PeerIdentity{ClusterID: "cluster-1", NodeID: "node-b"}
observed := false
server := httptest.NewServer(Server{
Local: local,
ProductionForwardingEnabled: true,
ProductionEnvelopeObserver: func(context.Context, ProductionEnvelopeObservation) error {
observed = true
return nil
},
}.Handler())
defer server.Close()
envelope := validProductionEnvelope(local)
envelope.CreatedAt = time.Now().UTC().Add(MaxProductionEnvelopeFutureSkew + time.Second)
envelope.ExpiresAt = envelope.CreatedAt.Add(time.Minute)
payload, err := json.Marshal(envelope)
if err != nil {
t.Fatalf("marshal envelope: %v", err)
}
resp, err := http.Post(server.URL+"/mesh/v1/forward", "application/json", bytes.NewReader(payload))
if err != nil {
t.Fatalf("post forward: %v", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusBadRequest {
t.Fatalf("status = %d, want %d", resp.StatusCode, http.StatusBadRequest)
}
if observed {
t.Fatal("observer called for future-created envelope")
}
}
func TestMeshForwardingGateObservesValidEnvelopeWithoutPayload(t *testing.T) {
local := PeerIdentity{ClusterID: "cluster-1", NodeID: "node-b"}
var observed ProductionEnvelopeObservation
server := httptest.NewServer(Server{
Local: local,
ProductionForwardingEnabled: true,
ProductionEnvelopeObserver: func(_ context.Context, observation ProductionEnvelopeObservation) error {
observed = observation
return nil
},
}.Handler())
defer server.Close()
envelope := validProductionEnvelope(local)
payload, err := json.Marshal(envelope)
if err != nil {
t.Fatalf("marshal envelope: %v", err)
}
resp, err := http.Post(server.URL+"/mesh/v1/forward", "application/json", bytes.NewReader(payload))
if err != nil {
t.Fatalf("post forward: %v", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusNotImplemented {
t.Fatalf("status = %d, want %d", resp.StatusCode, http.StatusNotImplemented)
}
if observed.MessageID != envelope.MessageID || observed.RouteID != envelope.RouteID {
t.Fatalf("unexpected observation: %+v", observed)
}
if observed.PayloadHash != envelope.PayloadHash || observed.PayloadLength != envelope.PayloadLength {
t.Fatalf("payload metadata missing from observation: %+v", observed)
}
}
func TestMeshForwardingGateDoesNotObserveRejectedEnvelope(t *testing.T) {
local := PeerIdentity{ClusterID: "cluster-1", NodeID: "node-b"}
observed := false
server := httptest.NewServer(Server{
Local: local,
ProductionForwardingEnabled: true,
ProductionEnvelopeObserver: func(context.Context, ProductionEnvelopeObservation) error {
observed = true
return nil
},
}.Handler())
defer server.Close()
envelope := validProductionEnvelope(local)
envelope.ClusterID = "wrong-cluster"
payload, err := json.Marshal(envelope)
if err != nil {
t.Fatalf("marshal envelope: %v", err)
}
resp, err := http.Post(server.URL+"/mesh/v1/forward", "application/json", bytes.NewReader(payload))
if err != nil {
t.Fatalf("post forward: %v", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusForbidden {
t.Fatalf("status = %d, want %d", resp.StatusCode, http.StatusForbidden)
}
if observed {
t.Fatal("observer called for rejected envelope")
}
}
func TestMeshForwardingGateFailsClosedWhenObservationFails(t *testing.T) {
local := PeerIdentity{ClusterID: "cluster-1", NodeID: "node-b"}
server := httptest.NewServer(Server{
Local: local,
ProductionForwardingEnabled: true,
ProductionEnvelopeObserver: func(context.Context, ProductionEnvelopeObservation) error {
return errors.New("observer down")
},
}.Handler())
defer server.Close()
payload, err := json.Marshal(validProductionEnvelope(local))
if err != nil {
t.Fatalf("marshal envelope: %v", err)
}
resp, err := http.Post(server.URL+"/mesh/v1/forward", "application/json", bytes.NewReader(payload))
if err != nil {
t.Fatalf("post forward: %v", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusInternalServerError {
t.Fatalf("status = %d, want %d", resp.StatusCode, http.StatusInternalServerError)
}
}
func TestMeshForwardingGateFailsClosedWhenObservationPanics(t *testing.T) {
local := PeerIdentity{ClusterID: "cluster-1", NodeID: "node-b"}
server := httptest.NewServer(Server{
Local: local,
ProductionForwardingEnabled: true,
ProductionEnvelopeObserver: func(context.Context, ProductionEnvelopeObservation) error {
panic("observer panic")
},
}.Handler())
defer server.Close()
payload, err := json.Marshal(validProductionEnvelope(local))
if err != nil {
t.Fatalf("marshal envelope: %v", err)
}
resp, err := http.Post(server.URL+"/mesh/v1/forward", "application/json", bytes.NewReader(payload))
if err != nil {
t.Fatalf("post forward: %v", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusInternalServerError {
t.Fatalf("status = %d, want %d", resp.StatusCode, http.StatusInternalServerError)
}
}
func TestObserveProductionEnvelopeAllowsNilObserver(t *testing.T) {
if err := observeProductionEnvelope(context.Background(), nil, ProductionEnvelopeObservation{}); err != nil {
t.Fatalf("observeProductionEnvelope nil observer err = %v", err)
}
}
func TestProductionEnvelopeObservationSinkKeepsBoundedMetadata(t *testing.T) {
local := PeerIdentity{ClusterID: "cluster-1", NodeID: "node-b"}
sink := NewProductionEnvelopeObservationSink(2)
server := httptest.NewServer(Server{
Local: local,
ProductionForwardingEnabled: true,
ProductionEnvelopeObserver: sink.Observe,
}.Handler())
defer server.Close()
for i := 1; i <= 3; i++ {
envelope := validProductionEnvelope(local)
envelope.MessageID = "message-" + string(rune('0'+i))
payload, err := json.Marshal(envelope)
if err != nil {
t.Fatalf("marshal envelope: %v", err)
}
resp, err := http.Post(server.URL+"/mesh/v1/forward", "application/json", bytes.NewReader(payload))
if err != nil {
t.Fatalf("post forward: %v", err)
}
resp.Body.Close()
if resp.StatusCode != http.StatusNotImplemented {
t.Fatalf("status = %d, want %d", resp.StatusCode, http.StatusNotImplemented)
}
}
observations := sink.Snapshot()
if len(observations) != 2 {
t.Fatalf("observation count = %d, want 2", len(observations))
}
if observations[0].MessageID != "message-2" || observations[1].MessageID != "message-3" {
t.Fatalf("unexpected bounded observations: %+v", observations)
}
if observations[0].PayloadHash == "" || observations[0].PayloadLength == 0 {
t.Fatalf("payload metadata missing from bounded observation: %+v", observations[0])
}
metrics := sink.Metrics()
if metrics.Capacity != 2 || metrics.CurrentDepth != 2 || metrics.AcceptedTotal != 3 || metrics.DroppedOldest != 1 {
t.Fatalf("unexpected sink metrics: %+v", metrics)
}
}
func TestProductionEnvelopeObservationSinkMetricsStartEmpty(t *testing.T) {
sink := NewProductionEnvelopeObservationSink(3)
metrics := sink.Metrics()
if metrics.Capacity != 3 || metrics.CurrentDepth != 0 || metrics.AcceptedTotal != 0 || metrics.DroppedOldest != 0 {
t.Fatalf("unexpected empty metrics: %+v", metrics)
}
}
func TestMeshForwardingGateRejectsServiceChannel(t *testing.T) {
local := PeerIdentity{ClusterID: "cluster-1", NodeID: "node-b"}
server := httptest.NewServer(Server{
Local: local,
ProductionForwardingEnabled: true,
}.Handler())
defer server.Close()
envelope := validProductionEnvelope(local)
envelope.ChannelClass = "render"
payload, err := json.Marshal(envelope)
if err != nil {
t.Fatalf("marshal envelope: %v", err)
}
resp, err := http.Post(server.URL+"/mesh/v1/forward", "application/json", bytes.NewReader(payload))
if err != nil {
t.Fatalf("post forward: %v", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusForbidden {
t.Fatalf("status = %d, want %d", resp.StatusCode, http.StatusForbidden)
}
}
func TestMeshForwardingRequiresPost(t *testing.T) {
server := httptest.NewServer(Server{Local: PeerIdentity{ClusterID: "cluster-1", NodeID: "node-b"}}.Handler())
defer server.Close()
resp, err := http.Get(server.URL + "/mesh/v1/forward")
if err != nil {
t.Fatalf("get forward: %v", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusMethodNotAllowed {
t.Fatalf("status = %d, want %d", resp.StatusCode, http.StatusMethodNotAllowed)
}
}
func validProductionEnvelope(local PeerIdentity) ProductionEnvelope {
payload := json.RawMessage(`{"kind":"control"}`)
sum := sha256.Sum256(payload)
now := time.Now().UTC()
return ProductionEnvelope{
FabricProtocolVersion: ProtocolVersion,
MessageID: "message-1",
RouteID: "route-1",
ClusterID: local.ClusterID,
SourceNodeID: "node-a",
DestinationNodeID: "node-c",
CurrentHopNodeID: local.NodeID,
NextHopNodeID: "node-c",
ChannelClass: ProductionChannelFabricControl,
MessageType: ProductionMessageFabricControl,
TTL: 4,
HopCount: 1,
CreatedAt: now,
ExpiresAt: now.Add(time.Minute),
PayloadLength: len(payload),
PayloadHash: hex.EncodeToString(sum[:]),
Payload: payload,
}
}
func configuredProductionRoute(routeID string, hops []string) SyntheticRoute {
return SyntheticRoute{
RouteID: routeID,
ClusterID: "cluster-1",
SourceNodeID: hops[0],
DestinationNodeID: hops[len(hops)-1],
Hops: append([]string{}, hops...),
AllowedChannels: []string{ProductionChannelFabricControl},
ExpiresAt: time.Now().UTC().Add(time.Hour),
MaxTTL: 8,
MaxHops: 8,
}
}
func TestProductionForwardDeliversVPNPacketBatchOnAuthorizedVPNChannel(t *testing.T) {
local := PeerIdentity{ClusterID: "cluster-1", NodeID: "node-c"}
payload := json.RawMessage(`{"vpn_connection_id":"vpn-1","packets":["AAAA"]}`)
sum := sha256.Sum256(payload)
now := time.Now().UTC()
var delivered ProductionEnvelope
server := httptest.NewServer(Server{
Local: local,
ProductionForwardingEnabled: true,
ProductionRoutes: []SyntheticRoute{{
RouteID: "route-vpn-1",
ClusterID: "cluster-1",
SourceNodeID: "node-a",
DestinationNodeID: "node-c",
Hops: []string{"node-a", "node-c"},
AllowedChannels: []string{ProductionChannelVPNPacket},
ExpiresAt: now.Add(time.Hour),
MaxTTL: 8,
MaxHops: 8,
}},
ProductionEnvelopeDelivery: func(_ context.Context, envelope ProductionEnvelope) error {
delivered = envelope
return nil
},
}.Handler())
defer server.Close()
envelope := ProductionEnvelope{
FabricProtocolVersion: ProtocolVersion,
MessageID: "vpn-message-1",
RouteID: "route-vpn-1",
ClusterID: "cluster-1",
SourceNodeID: "node-a",
DestinationNodeID: "node-c",
CurrentHopNodeID: "node-c",
NextHopNodeID: "node-c",
RoutePath: []string{"node-a", "node-c"},
ChannelClass: ProductionChannelVPNPacket,
MessageType: ProductionMessageVPNPacketBatch,
TTL: 4,
HopCount: 1,
CreatedAt: now,
ExpiresAt: now.Add(time.Minute),
PayloadLength: len(payload),
PayloadHash: hex.EncodeToString(sum[:]),
Payload: payload,
}
body, err := json.Marshal(envelope)
if err != nil {
t.Fatalf("marshal envelope: %v", err)
}
resp, err := http.Post(server.URL+"/mesh/v1/forward", "application/json", bytes.NewReader(body))
if err != nil {
t.Fatalf("post forward: %v", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
t.Fatalf("status = %d, want %d", resp.StatusCode, http.StatusOK)
}
if delivered.MessageID != envelope.MessageID || string(delivered.Payload) != string(payload) {
t.Fatalf("delivered envelope = %+v", delivered)
}
}
func TestProductionForwardRejectsVPNPacketOnFabricControlRoute(t *testing.T) {
local := PeerIdentity{ClusterID: "cluster-1", NodeID: "node-c"}
envelope := validProductionEnvelope(local)
envelope.RouteID = "route-vpn-blocked"
envelope.RoutePath = []string{"node-a", "node-c"}
envelope.ChannelClass = ProductionChannelVPNPacket
envelope.MessageType = ProductionMessageVPNPacketBatch
payload := json.RawMessage(`{"vpn_connection_id":"vpn-1","packets":["AAAA"]}`)
sum := sha256.Sum256(payload)
envelope.Payload = payload
envelope.PayloadLength = len(payload)
envelope.PayloadHash = hex.EncodeToString(sum[:])
server := httptest.NewServer(Server{
Local: local,
ProductionForwardingEnabled: true,
ProductionRoutes: []SyntheticRoute{configuredProductionRoute("route-vpn-blocked", []string{"node-a", "node-c"})},
}.Handler())
defer server.Close()
body, err := json.Marshal(envelope)
if err != nil {
t.Fatalf("marshal envelope: %v", err)
}
resp, err := http.Post(server.URL+"/mesh/v1/forward", "application/json", bytes.NewReader(body))
if err != nil {
t.Fatalf("post forward: %v", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusForbidden {
t.Fatalf("status = %d, want %d", resp.StatusCode, http.StatusForbidden)
}
}
func TestVPNPacketIngressFallsBackToBackendRelayWhenFabricPeerUnavailable(t *testing.T) {
var backendBody []byte
backend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path != "/api/v1/clusters/cluster-1/vpn-connections/vpn-1/tunnel/client/packets" {
t.Fatalf("backend path = %s", r.URL.Path)
}
if r.Header.Get("X-RAP-Entry-Node") != "entry-1" {
t.Fatalf("entry header = %q", r.Header.Get("X-RAP-Entry-Node"))
}
var err error
backendBody, err = io.ReadAll(r.Body)
if err != nil {
t.Fatalf("read backend body: %v", err)
}
w.WriteHeader(http.StatusAccepted)
}))
defer backend.Close()
server := httptest.NewServer(Server{
Local: PeerIdentity{ClusterID: "cluster-1", NodeID: "entry-1"},
VPNPacketIngress: failingVPNPacketIngress{sendErr: ErrForwardPeerUnavailable},
BackendProxyBaseURL: backend.URL + "/api/v1",
}.Handler())
defer server.Close()
resp, err := http.Post(server.URL+"/api/v1/clusters/cluster-1/vpn-connections/vpn-1/tunnel/client/packets", "application/octet-stream", bytes.NewReader([]byte("packet")))
if err != nil {
t.Fatalf("post vpn packet: %v", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusAccepted {
t.Fatalf("status = %d, want %d", resp.StatusCode, http.StatusAccepted)
}
if string(backendBody) != "packet" {
t.Fatalf("backend body = %q", string(backendBody))
}
}
func TestVPNPacketIngressFallsBackToBackendRelayWhenFabricInboxIsEmpty(t *testing.T) {
backend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
w.Header().Set("Content-Type", "application/octet-stream")
_, _ = w.Write([]byte("reply"))
}))
defer backend.Close()
server := httptest.NewServer(Server{
Local: PeerIdentity{ClusterID: "cluster-1", NodeID: "entry-1"},
VPNPacketIngress: failingVPNPacketIngress{},
BackendProxyBaseURL: backend.URL + "/api/v1",
}.Handler())
defer server.Close()
resp, err := http.Get(server.URL + "/api/v1/clusters/cluster-1/vpn-connections/vpn-1/tunnel/client/packets?timeout_ms=2")
if err != nil {
t.Fatalf("get vpn packet: %v", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
t.Fatalf("status = %d, want %d", resp.StatusCode, http.StatusOK)
}
body, err := io.ReadAll(resp.Body)
if err != nil {
t.Fatalf("read body: %v", err)
}
if string(body) != "reply" {
t.Fatalf("body = %q", string(body))
}
}
func TestFabricServiceChannelVPNPacketIngressRequiresLeaseToken(t *testing.T) {
ingress := &recordingVPNPacketIngress{}
server := httptest.NewServer(Server{
Local: PeerIdentity{ClusterID: "cluster-1", NodeID: "entry-1"},
VPNPacketIngress: ingress,
}.Handler())
defer server.Close()
resp, err := http.Post(server.URL+"/api/v1/clusters/cluster-1/fabric/service-channels/channel-1/vpn-connections/vpn-1/packets", "application/octet-stream", bytes.NewReader([]byte("packet")))
if err != nil {
t.Fatalf("post service channel packet: %v", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusUnauthorized {
t.Fatalf("status = %d, want %d", resp.StatusCode, http.StatusUnauthorized)
}
ingress.mu.Lock()
defer ingress.mu.Unlock()
if len(ingress.sent) != 0 {
t.Fatalf("unexpected sent packets = %#v", ingress.sent)
}
}
func TestFabricServiceChannelVPNPacketIngressMovesBatchOverFabricRuntime(t *testing.T) {
ingress := &recordingVPNPacketIngress{
receive: [][]byte{[]byte("reply-1"), []byte("reply-2")},
}
server := httptest.NewServer(Server{
Local: PeerIdentity{ClusterID: "cluster-1", NodeID: "entry-1"},
VPNPacketIngress: ingress,
}.Handler())
defer server.Close()
body := encodeVPNIngressPacketBatch([][]byte{[]byte("packet-1"), []byte("packet-2")})
req, err := http.NewRequest(http.MethodPost, server.URL+"/api/v1/clusters/cluster-1/fabric/service-channels/channel-1/vpn-connections/vpn-1/packets?batch=true", bytes.NewReader(body))
if err != nil {
t.Fatalf("new request: %v", err)
}
req.Header.Set("Authorization", "Bearer rap_fsc_testtoken")
req.Header.Set("X-RAP-Service-Class", FabricServiceClassVPNPackets)
req.Header.Set("X-RAP-Channel-Class", ProductionChannelVPNPacket)
req.Header.Set("X-RAP-Traffic-Class", "interactive")
req.Header.Set("X-RAP-Fabric-Channel-ID", "channel-1")
resp, err := http.DefaultClient.Do(req)
if err != nil {
t.Fatalf("post service channel packet batch: %v", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusAccepted {
t.Fatalf("post status = %d, want %d", resp.StatusCode, http.StatusAccepted)
}
ingress.mu.Lock()
if ingress.clusterID != "cluster-1" || ingress.vpnConnectionID != "vpn-1" {
t.Fatalf("ingress ids = %s %s", ingress.clusterID, ingress.vpnConnectionID)
}
if len(ingress.sent) != 2 || string(ingress.sent[0]) != "packet-1" || string(ingress.sent[1]) != "packet-2" {
t.Fatalf("sent packets = %#v", ingress.sent)
}
if ingress.trafficClass != "interactive" {
t.Fatalf("traffic class = %q, want interactive", ingress.trafficClass)
}
ingress.mu.Unlock()
req, err = http.NewRequest(http.MethodGet, server.URL+"/api/v1/clusters/cluster-1/fabric/service-channels/channel-1/vpn-connections/vpn-1/packets?batch=true&timeout_ms=2", nil)
if err != nil {
t.Fatalf("new get request: %v", err)
}
req.Header.Set("X-RAP-Service-Channel-Token", "rap_fsc_testtoken")
resp, err = http.DefaultClient.Do(req)
if err != nil {
t.Fatalf("get service channel packet batch: %v", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
t.Fatalf("get status = %d, want %d", resp.StatusCode, http.StatusOK)
}
payload, err := io.ReadAll(resp.Body)
if err != nil {
t.Fatalf("read get body: %v", err)
}
packets, err := decodeVPNIngressPacketBatch(payload)
if err != nil {
t.Fatalf("decode get batch: %v", err)
}
if len(packets) != 2 || string(packets[0]) != "reply-1" || string(packets[1]) != "reply-2" {
t.Fatalf("reply packets = %#v", packets)
}
}
func TestFabricServiceChannelVPNPacketIngressRequiresSignedLeaseWhenAuthorityPinned(t *testing.T) {
publicKey, _, err := ed25519.GenerateKey(nil)
if err != nil {
t.Fatalf("generate key: %v", err)
}
ingress := &recordingVPNPacketIngress{}
server := httptest.NewServer(Server{
Local: PeerIdentity{ClusterID: "cluster-1", NodeID: "entry-1"},
VPNPacketIngress: ingress,
ClusterAuthorityPublicKey: base64.StdEncoding.EncodeToString(publicKey),
}.Handler())
defer server.Close()
req, err := http.NewRequest(http.MethodPost, server.URL+"/api/v1/clusters/cluster-1/fabric/service-channels/channel-1/vpn-connections/vpn-1/packets", bytes.NewReader([]byte("packet")))
if err != nil {
t.Fatalf("new request: %v", err)
}
req.Header.Set("X-RAP-Service-Channel-Token", "rap_fsc_unsigned")
resp, err := http.DefaultClient.Do(req)
if err != nil {
t.Fatalf("post service channel packet: %v", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusForbidden {
t.Fatalf("status = %d, want %d", resp.StatusCode, http.StatusForbidden)
}
ingress.mu.Lock()
defer ingress.mu.Unlock()
if len(ingress.sent) != 0 {
t.Fatalf("unexpected sent packets = %#v", ingress.sent)
}
}
func TestFabricServiceChannelVPNPacketIngressUsesBackendIntrospectionWhenUnsigned(t *testing.T) {
publicKey, _, err := ed25519.GenerateKey(nil)
if err != nil {
t.Fatalf("generate key: %v", err)
}
var introspected bool
backend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path != "/api/v1/clusters/cluster-1/fabric/service-channels/channel-1/introspect" {
t.Fatalf("introspection path = %s", r.URL.Path)
}
introspected = true
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(`{"fabric_service_channel_introspection":{"allowed":true,"status":"allowed","selected_entry_node_id":"entry-1","allowed_channels":["vpn_packet"],"preferred_route_id":"route-1","lease_status":"ready","primary_route":{"route_id":"route-1","status":"ready"},"data_plane":{"schema_version":"rap.fabric_service_channel_data_plane.v1","mode":"fabric_primary","working_data_transport":"fabric_service_channel","steady_state_transport":"fabric_route","backend_relay_policy":"degraded_fallback_only","service_neutral":true,"protocol_agnostic":true,"logical_flow_mode":"multi_flow_isolated","required_flow_isolation_classes":["control","vpn_packet"]},"expires_at":"2099-01-01T00:00:00Z"}}`))
}))
defer backend.Close()
ingress := &recordingVPNPacketIngress{}
var accessEvents []FabricServiceChannelAccessLogEntry
server := httptest.NewServer(Server{
Local: PeerIdentity{ClusterID: "cluster-1", NodeID: "entry-1"},
VPNPacketIngress: ingress,
BackendProxyBaseURL: backend.URL + "/api/v1",
ClusterAuthorityPublicKey: base64.StdEncoding.EncodeToString(publicKey),
FabricServiceChannelLogger: func(entry FabricServiceChannelAccessLogEntry) {
accessEvents = append(accessEvents, entry)
},
}.Handler())
defer server.Close()
req, err := http.NewRequest(http.MethodPost, server.URL+"/api/v1/clusters/cluster-1/fabric/service-channels/channel-1/vpn-connections/vpn-1/packets", bytes.NewReader([]byte("packet")))
if err != nil {
t.Fatalf("new request: %v", err)
}
req.Header.Set("X-RAP-Service-Channel-Token", "rap_fsc_unsigned")
resp, err := http.DefaultClient.Do(req)
if err != nil {
t.Fatalf("post service channel packet: %v", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusAccepted {
t.Fatalf("status = %d, want %d", resp.StatusCode, http.StatusAccepted)
}
if got := resp.Header.Get("X-RAP-Service-Channel-Accepted-By"); got != "introspection" {
t.Fatalf("accepted-by header = %q, want introspection", got)
}
if !introspected {
t.Fatal("backend introspection was not called")
}
if len(accessEvents) != 1 || accessEvents[0].AcceptedBy != "introspection" || accessEvents[0].PreferredRouteID != "route-1" ||
!accessEvents[0].DataPlaneValid ||
accessEvents[0].WorkingDataTransport != "fabric_service_channel" ||
accessEvents[0].SteadyStateTransport != "fabric_route" ||
accessEvents[0].BackendRelayPolicy != "degraded_fallback_only" {
t.Fatalf("unexpected access events: %+v", accessEvents)
}
ingress.mu.Lock()
defer ingress.mu.Unlock()
if len(ingress.sent) != 1 || string(ingress.sent[0]) != "packet" {
t.Fatalf("sent packets = %#v", ingress.sent)
}
}
func TestFabricServiceChannelVPNPacketIngressVerifiesSignedLeaseAuthority(t *testing.T) {
publicKey, privateKey, err := ed25519.GenerateKey(nil)
if err != nil {
t.Fatalf("generate key: %v", err)
}
token := "rap_fsc_signedtest"
payload := fabricServiceChannelLeaseAuthorityPayload{
SchemaVersion: "rap.fabric_service_channel_lease_authority.v1",
ChannelID: "channel-1",
ClusterID: "cluster-1",
ResourceID: "vpn-1",
ServiceClass: FabricServiceClassVPNPackets,
SelectedEntryNodeID: "entry-1",
SelectedExitNodeID: "exit-1",
AllowedChannels: []string{ProductionChannelVPNPacket},
RouteGeneration: "rg-1",
FencingEpoch: 7,
TokenHash: fabricServiceChannelTokenHash(token),
IssuedAt: time.Now().UTC().Add(-time.Minute),
ExpiresAt: time.Now().UTC().Add(time.Minute),
DataPlane: fabricServiceChannelDataPlaneContract{
SchemaVersion: "rap.fabric_service_channel_data_plane.v1",
Mode: "fabric_primary",
WorkingDataTransport: "fabric_service_channel",
SteadyStateTransport: "fabric_route",
BackendRelayPolicy: "degraded_fallback_only",
ProductionForwardingRequired: true,
ServiceNeutral: true,
ProtocolAgnostic: true,
LogicalFlowMode: "multi_flow_isolated",
RequiredFlowIsolationClasses: []string{"control", ProductionChannelVPNPacket},
},
}
payload.PrimaryRoute.RouteID = "route-signed"
payload.PrimaryRoute.Status = "authorized"
rawPayload, err := json.Marshal(payload)
if err != nil {
t.Fatalf("marshal payload: %v", err)
}
canonical, err := authority.CanonicalJSON(rawPayload)
if err != nil {
t.Fatalf("canonical payload: %v", err)
}
signature := authority.Signature{
SchemaVersion: authority.SignatureSchemaVersion,
Algorithm: authority.AlgorithmEd25519,
KeyFingerprint: authority.Fingerprint(publicKey),
Signature: base64.StdEncoding.EncodeToString(ed25519.Sign(privateKey, canonical)),
}
rawSignature, err := json.Marshal(signature)
if err != nil {
t.Fatalf("marshal signature: %v", err)
}
ingress := &recordingVPNPacketIngress{}
var accessEvents []FabricServiceChannelAccessLogEntry
server := httptest.NewServer(Server{
Local: PeerIdentity{ClusterID: "cluster-1", NodeID: "entry-1"},
VPNPacketIngress: ingress,
ClusterAuthorityPublicKey: base64.StdEncoding.EncodeToString(publicKey),
FabricServiceChannelLogger: func(entry FabricServiceChannelAccessLogEntry) {
accessEvents = append(accessEvents, entry)
},
}.Handler())
defer server.Close()
req, err := http.NewRequest(http.MethodPost, server.URL+"/api/v1/clusters/cluster-1/fabric/service-channels/channel-1/vpn-connections/vpn-1/packets", bytes.NewReader([]byte("packet")))
if err != nil {
t.Fatalf("new request: %v", err)
}
req.Header.Set("X-RAP-Service-Channel-Token", token)
req.Header.Set("X-RAP-Service-Channel-Authority-Payload", base64.RawURLEncoding.EncodeToString(rawPayload))
req.Header.Set("X-RAP-Service-Channel-Authority-Signature", base64.RawURLEncoding.EncodeToString(rawSignature))
resp, err := http.DefaultClient.Do(req)
if err != nil {
t.Fatalf("post service channel packet: %v", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusAccepted {
t.Fatalf("status = %d, want %d", resp.StatusCode, http.StatusAccepted)
}
if len(accessEvents) != 1 ||
accessEvents[0].AcceptedBy != "signed" ||
accessEvents[0].PreferredRouteID != "route-signed" ||
!accessEvents[0].DataPlaneValid ||
accessEvents[0].WorkingDataTransport != "fabric_service_channel" ||
accessEvents[0].SteadyStateTransport != "fabric_route" ||
accessEvents[0].BackendRelayPolicy != "degraded_fallback_only" {
t.Fatalf("unexpected signed data-plane access events: %+v", accessEvents)
}
}
func TestFabricServiceChannelRemoteWorkspaceIngressValidatesSignedLeaseAuthority(t *testing.T) {
publicKey, privateKey, err := ed25519.GenerateKey(nil)
if err != nil {
t.Fatalf("generate key: %v", err)
}
token := "rap_fsc_remoteworkspace"
payload := fabricServiceChannelLeaseAuthorityPayload{
SchemaVersion: "rap.fabric_service_channel_lease_authority.v1",
ChannelID: "channel-rw",
ClusterID: "cluster-1",
ResourceID: "workspace-1",
ServiceClass: FabricServiceClassRemoteWorkspace,
SelectedEntryNodeID: "entry-1",
SelectedExitNodeID: "exit-1",
AllowedChannels: []string{FabricServiceChannelControl, FabricServiceChannelInteractive, FabricServiceChannelReliable, FabricServiceChannelDroppable},
RouteGeneration: "rg-rw",
FencingEpoch: 11,
TokenHash: fabricServiceChannelTokenHash(token),
IssuedAt: time.Now().UTC().Add(-time.Minute),
ExpiresAt: time.Now().UTC().Add(time.Minute),
DataPlane: fabricServiceChannelDataPlaneContract{
SchemaVersion: "rap.fabric_service_channel_data_plane.v1",
Mode: "fabric_primary",
WorkingDataTransport: "fabric_service_channel",
SteadyStateTransport: "fabric_route",
BackendRelayPolicy: "degraded_fallback_only",
ProductionForwardingRequired: true,
ServiceNeutral: true,
ProtocolAgnostic: true,
LogicalFlowMode: "multi_flow_isolated",
RequiredFlowIsolationClasses: []string{FabricServiceChannelControl, FabricServiceChannelInteractive, FabricServiceChannelReliable, FabricServiceChannelDroppable},
},
}
payload.PrimaryRoute.RouteID = "route-rw"
payload.PrimaryRoute.Status = "authorized"
rawPayload, err := json.Marshal(payload)
if err != nil {
t.Fatalf("marshal payload: %v", err)
}
canonical, err := authority.CanonicalJSON(rawPayload)
if err != nil {
t.Fatalf("canonical payload: %v", err)
}
signature := authority.Signature{
SchemaVersion: authority.SignatureSchemaVersion,
Algorithm: authority.AlgorithmEd25519,
KeyFingerprint: authority.Fingerprint(publicKey),
Signature: base64.StdEncoding.EncodeToString(ed25519.Sign(privateKey, canonical)),
}
rawSignature, err := json.Marshal(signature)
if err != nil {
t.Fatalf("marshal signature: %v", err)
}
var accessEvents []FabricServiceChannelAccessLogEntry
server := httptest.NewServer(Server{
Local: PeerIdentity{ClusterID: "cluster-1", NodeID: "entry-1"},
VPNPacketIngress: &recordingVPNPacketIngress{},
ClusterAuthorityPublicKey: base64.StdEncoding.EncodeToString(publicKey),
FabricServiceChannelLogger: func(entry FabricServiceChannelAccessLogEntry) {
accessEvents = append(accessEvents, entry)
},
}.Handler())
defer server.Close()
req, err := http.NewRequest(http.MethodPost, server.URL+"/api/v1/clusters/cluster-1/fabric/service-channels/channel-rw/remote-workspaces/workspace-1/streams/interactive", nil)
if err != nil {
t.Fatalf("new request: %v", err)
}
req.Header.Set("X-RAP-Service-Channel-Token", token)
req.Header.Set("X-RAP-Service-Class", FabricServiceClassRemoteWorkspace)
req.Header.Set("X-RAP-Channel-Class", FabricServiceChannelInteractive)
req.Header.Set("X-RAP-Service-Channel-Authority-Payload", base64.RawURLEncoding.EncodeToString(rawPayload))
req.Header.Set("X-RAP-Service-Channel-Authority-Signature", base64.RawURLEncoding.EncodeToString(rawSignature))
resp, err := http.DefaultClient.Do(req)
if err != nil {
t.Fatalf("post remote workspace probe: %v", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusAccepted {
t.Fatalf("status = %d, want %d", resp.StatusCode, http.StatusAccepted)
}
var decoded map[string]any
if err := json.NewDecoder(resp.Body).Decode(&decoded); err != nil {
t.Fatalf("decode response: %v", err)
}
if decoded["service_class"] != FabricServiceClassRemoteWorkspace || decoded["channel_class"] != FabricServiceChannelInteractive || decoded["payload_flow"] != "not_implemented" {
t.Fatalf("unexpected response: %+v", decoded)
}
if len(accessEvents) != 1 ||
accessEvents[0].AcceptedBy != "signed" ||
accessEvents[0].ServiceClass != FabricServiceClassRemoteWorkspace ||
accessEvents[0].ChannelClass != FabricServiceChannelInteractive ||
accessEvents[0].PreferredRouteID != "route-rw" ||
!accessEvents[0].DataPlaneValid ||
accessEvents[0].WorkingDataTransport != "fabric_service_channel" ||
accessEvents[0].SteadyStateTransport != "fabric_route" {
t.Fatalf("unexpected remote workspace access events: %+v", accessEvents)
}
}
func TestFabricServiceChannelRemoteWorkspaceIngressAcceptsFrameBatchProbeOnly(t *testing.T) {
publicKey, privateKey, err := ed25519.GenerateKey(nil)
if err != nil {
t.Fatalf("generate key: %v", err)
}
token := "rap_fsc_remoteworkspace_frames"
payload := fabricServiceChannelLeaseAuthorityPayload{
SchemaVersion: "rap.fabric_service_channel_lease_authority.v1",
ChannelID: "channel-rw",
ClusterID: "cluster-1",
ResourceID: "workspace-1",
ServiceClass: FabricServiceClassRemoteWorkspace,
SelectedEntryNodeID: "entry-1",
AllowedChannels: []string{FabricServiceChannelInteractive, FabricServiceChannelDroppable},
TokenHash: fabricServiceChannelTokenHash(token),
ExpiresAt: time.Now().UTC().Add(time.Minute),
DataPlane: fabricServiceChannelDataPlaneContract{
SchemaVersion: "rap.fabric_service_channel_data_plane.v1",
Mode: "fabric_primary",
WorkingDataTransport: "fabric_service_channel",
SteadyStateTransport: "fabric_route",
BackendRelayPolicy: "degraded_fallback_only",
ServiceNeutral: true,
ProtocolAgnostic: true,
LogicalFlowMode: "multi_flow_isolated",
RequiredFlowIsolationClasses: []string{FabricServiceChannelInteractive, FabricServiceChannelDroppable},
},
}
payload.PrimaryRoute.RouteID = "route-rw"
payload.PrimaryRoute.Status = "authorized"
rawPayload, err := json.Marshal(payload)
if err != nil {
t.Fatalf("marshal payload: %v", err)
}
canonical, err := authority.CanonicalJSON(rawPayload)
if err != nil {
t.Fatalf("canonical payload: %v", err)
}
signature := authority.Signature{
SchemaVersion: authority.SignatureSchemaVersion,
Algorithm: authority.AlgorithmEd25519,
KeyFingerprint: authority.Fingerprint(publicKey),
Signature: base64.StdEncoding.EncodeToString(ed25519.Sign(privateKey, canonical)),
}
rawSignature, err := json.Marshal(signature)
if err != nil {
t.Fatalf("marshal signature: %v", err)
}
server := httptest.NewServer(Server{
Local: PeerIdentity{ClusterID: "cluster-1", NodeID: "entry-1"},
VPNPacketIngress: &recordingVPNPacketIngress{},
RemoteWorkspaceFrameSink: NewRemoteWorkspaceFrameProbeSink(),
ClusterAuthorityPublicKey: base64.StdEncoding.EncodeToString(publicKey),
}.Handler())
defer server.Close()
frameBatch := map[string]any{
"schema_version": "rap.remote_workspace_frame_batch.v1",
"probe_only": true,
"service_class": FabricServiceClassRemoteWorkspace,
"channel_class": FabricServiceChannelInteractive,
"adapter_contract_id": "rap.rdp_worker.remote_workspace_adapter_contract_probe.v1",
"frames": []map[string]any{
{"channel": "input", "direction": "client_to_adapter", "payload_encoding": "none", "payload_length": 0, "droppable": true},
{"channel": "display", "direction": "adapter_to_client", "payload_encoding": "none", "payload_length": 0, "droppable": true},
},
}
rawFrameBatch, err := json.Marshal(frameBatch)
if err != nil {
t.Fatalf("marshal frame batch: %v", err)
}
req, err := http.NewRequest(http.MethodPost, server.URL+"/api/v1/clusters/cluster-1/fabric/service-channels/channel-rw/remote-workspaces/workspace-1/streams/interactive", bytes.NewReader(rawFrameBatch))
if err != nil {
t.Fatalf("new request: %v", err)
}
req.Header.Set("X-RAP-Service-Channel-Token", token)
req.Header.Set("X-RAP-Service-Class", FabricServiceClassRemoteWorkspace)
req.Header.Set("X-RAP-Channel-Class", FabricServiceChannelInteractive)
req.Header.Set("X-RAP-Service-Channel-Authority-Payload", base64.RawURLEncoding.EncodeToString(rawPayload))
req.Header.Set("X-RAP-Service-Channel-Authority-Signature", base64.RawURLEncoding.EncodeToString(rawSignature))
resp, err := http.DefaultClient.Do(req)
if err != nil {
t.Fatalf("post remote workspace frame probe: %v", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusAccepted {
t.Fatalf("status = %d, want %d", resp.StatusCode, http.StatusAccepted)
}
var decoded map[string]any
if err := json.NewDecoder(resp.Body).Decode(&decoded); err != nil {
t.Fatalf("decode response: %v", err)
}
if decoded["payload_flow"] != "delivered_probe_only" || decoded["frame_batch_schema"] != "rap.remote_workspace_frame_batch.v1" || int(decoded["frame_count"].(float64)) != 2 {
t.Fatalf("unexpected response: %+v", decoded)
}
adapterSessionID, ok := decoded["adapter_session_id"].(string)
if !ok || !strings.HasPrefix(adapterSessionID, "rap-rw-adapter-session-") {
t.Fatalf("adapter_session_id = %#v", decoded["adapter_session_id"])
}
delivery, ok := decoded["adapter_delivery"].(map[string]any)
if !ok || delivery["sink"] != "node_agent_rdp_worker_contract_probe" || delivery["schema_version"] != "rap.remote_workspace_frame_batch_delivery.v1" {
t.Fatalf("unexpected adapter delivery: %+v", decoded["adapter_delivery"])
}
if delivery["adapter_session_id"] != adapterSessionID || delivery["adapter_runtime_id"] != "node_agent_rdp_worker_contract_probe" || delivery["session_state"] != "probe_bound" {
t.Fatalf("unexpected adapter session delivery fields: %+v", delivery)
}
if int(delivery["accepted_frames"].(float64)) != 2 || int(delivery["acked_frames"].(float64)) != 2 || int(delivery["queue_depth"].(float64)) != 0 {
t.Fatalf("unexpected queue delivery fields: %+v", delivery)
}
}
func TestRemoteWorkspaceFrameBatchProbeRejectsGuardrailViolations(t *testing.T) {
valid := map[string]any{
"schema_version": "rap.remote_workspace_frame_batch.v1",
"probe_only": true,
"service_class": FabricServiceClassRemoteWorkspace,
"channel_class": FabricServiceChannelInteractive,
"adapter_contract_id": "rap.rdp_worker.remote_workspace_adapter_contract_probe.v1",
"frames": []map[string]any{
{"channel": "input", "direction": "client_to_adapter", "payload_encoding": "none", "payload_length": 0, "droppable": true},
},
}
tests := []struct {
name string
mutate func(map[string]any)
wantErr string
}{
{
name: "production payload forwarding disabled",
mutate: func(item map[string]any) {
item["probe_only"] = false
},
wantErr: "remote workspace payload forwarding is not implemented",
},
{
name: "unknown logical channel",
mutate: func(item map[string]any) {
item["frames"] = []map[string]any{{"channel": "unknown", "direction": "client_to_adapter"}}
},
wantErr: "unsupported remote workspace adapter frame channel",
},
{
name: "wrong direction",
mutate: func(item map[string]any) {
item["frames"] = []map[string]any{{"channel": "display", "direction": "client_to_adapter"}}
},
wantErr: "unsupported remote workspace adapter frame direction",
},
{
name: "service mismatch",
mutate: func(item map[string]any) {
item["service_class"] = FabricServiceClassVPNPackets
},
wantErr: "remote workspace frame batch service class mismatch",
},
{
name: "channel mismatch",
mutate: func(item map[string]any) {
item["channel_class"] = FabricServiceChannelReliable
},
wantErr: "remote workspace frame batch channel class mismatch",
},
{
name: "unsupported encoding",
mutate: func(item map[string]any) {
item["frames"] = []map[string]any{{"channel": "input", "direction": "client_to_adapter", "payload_encoding": "raw"}}
},
wantErr: "unsupported remote workspace frame payload encoding",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
item := map[string]any{}
for key, value := range valid {
item[key] = value
}
tt.mutate(item)
raw, err := json.Marshal(item)
if err != nil {
t.Fatalf("marshal frame batch: %v", err)
}
_, err = validateRemoteWorkspaceFrameBatchProbe(raw, FabricServiceChannelInteractive)
if err == nil || !strings.Contains(err.Error(), tt.wantErr) {
t.Fatalf("err = %v, want contains %q", err, tt.wantErr)
}
})
}
}
func TestRemoteWorkspaceFrameProbeSinkAppliesBoundedQueuePolicy(t *testing.T) {
sink := NewRemoteWorkspaceFrameProbeSink()
droppable := RemoteWorkspaceFrameBatchDelivery{
ServiceClass: FabricServiceClassRemoteWorkspace,
ChannelClass: FabricServiceChannelInteractive,
AdapterContractID: "rap.rdp_worker.remote_workspace_adapter_contract_probe.v1",
AdapterSessionID: "rap-rw-adapter-session-test",
}
for i := 0; i < DefaultRemoteWorkspaceFrameProbeSinkQueueCapacity+3; i++ {
droppable.Frames = append(droppable.Frames, RemoteWorkspaceFrameProbeRecord{
Channel: "display",
Direction: "adapter_to_client",
Droppable: true,
})
}
receipt, err := sink.AcceptRemoteWorkspaceFrameBatchProbe(context.Background(), droppable)
if err != nil {
t.Fatalf("accept droppable overflow: %v", err)
}
if !receipt.Accepted || receipt.AcceptedFrames != DefaultRemoteWorkspaceFrameProbeSinkQueueCapacity || receipt.DroppedFrames != 3 || receipt.AckedFrames != receipt.AcceptedFrames || receipt.QueueDepth != 0 {
t.Fatalf("droppable overflow receipt = %+v", receipt)
}
report := sink.Report(time.Unix(10, 0).UTC())
if report["queue_capacity"] != DefaultRemoteWorkspaceFrameProbeSinkQueueCapacity ||
report["queue_depth"] != 0 ||
report["total_accepted_frames"] != int64(DefaultRemoteWorkspaceFrameProbeSinkQueueCapacity) ||
report["total_dropped_frames"] != int64(3) ||
report["total_acked_frames"] != int64(DefaultRemoteWorkspaceFrameProbeSinkQueueCapacity) ||
report["active_session_count"] != 1 ||
report["session_created_total"] != int64(1) ||
report["session_bound_total"] != int64(1) ||
report["current_session_lifecycle_state"] != "probe_bound" ||
report["current_session_delivery_count"] != int64(1) ||
report["current_session_dropped_frames"] != int64(3) {
t.Fatalf("droppable overflow report = %+v", report)
}
reliable := RemoteWorkspaceFrameBatchDelivery{
ServiceClass: FabricServiceClassRemoteWorkspace,
ChannelClass: FabricServiceChannelInteractive,
AdapterContractID: "rap.rdp_worker.remote_workspace_adapter_contract_probe.v1",
AdapterSessionID: "rap-rw-adapter-session-test",
}
for i := 0; i < DefaultRemoteWorkspaceFrameProbeSinkQueueCapacity+1; i++ {
reliable.Frames = append(reliable.Frames, RemoteWorkspaceFrameProbeRecord{
Channel: "input",
Direction: "client_to_adapter",
Droppable: false,
})
}
if _, err := sink.AcceptRemoteWorkspaceFrameBatchProbe(context.Background(), reliable); err == nil || !strings.Contains(err.Error(), "backpressure") {
t.Fatalf("reliable overflow err = %v, want backpressure", err)
}
report = sink.Report(time.Unix(11, 0).UTC())
if report["backpressure_count"] != int64(1) ||
report["last_rejected_frame_count"] != DefaultRemoteWorkspaceFrameProbeSinkQueueCapacity+1 ||
report["last_rejected_queue_capacity"] != DefaultRemoteWorkspaceFrameProbeSinkQueueCapacity ||
report["last_rejected_queue_depth"] != DefaultRemoteWorkspaceFrameProbeSinkQueueCapacity ||
report["last_rejected_adapter_session_id"] != "rap-rw-adapter-session-test" ||
report["last_rejected_channel_class"] != FabricServiceChannelInteractive ||
report["last_rejected_adapter_contract_id"] != "rap.rdp_worker.remote_workspace_adapter_contract_probe.v1" ||
report["session_backpressure_total"] != int64(1) ||
report["current_session_lifecycle_state"] != "backpressure" ||
report["current_session_backpressure_count"] != int64(1) {
t.Fatalf("backpressure report = %+v", report)
}
report = sink.Report(time.Now().UTC().Add(DefaultRemoteWorkspaceFrameProbeSinkSessionTTL + time.Second))
if report["active_session_count"] != 0 ||
report["session_expired_total"] != int64(1) ||
report["session_closed_total"] != int64(1) {
t.Fatalf("expired lifecycle report = %+v", report)
}
}
func TestRemoteWorkspaceAdapterSessionControlEndpointClosesSession(t *testing.T) {
sink := NewRemoteWorkspaceFrameProbeSink()
delivery := RemoteWorkspaceFrameBatchDelivery{
ClusterID: "cluster-1",
ChannelID: "channel-rw",
ResourceID: "workspace-1",
ServiceClass: FabricServiceClassRemoteWorkspace,
ChannelClass: FabricServiceChannelInteractive,
AdapterContractID: "rap.rdp_worker.remote_workspace_adapter_contract_probe.v1",
AdapterSessionID: "rap-rw-adapter-session-aaaaaaaaaaaaaaaaaaaaaaaa",
Frames: []RemoteWorkspaceFrameProbeRecord{{
Channel: "display",
Direction: "adapter_to_client",
Droppable: true,
}},
}
if _, err := sink.AcceptRemoteWorkspaceFrameBatchProbe(context.Background(), delivery); err != nil {
t.Fatalf("accept frame batch: %v", err)
}
server := httptest.NewServer(Server{RemoteWorkspaceFrameSink: sink}.Handler())
defer server.Close()
body := bytes.NewReader([]byte(`{"action":"close","reason":"unit test close"}`))
controlURL := server.URL + "/mesh/v1/remote-workspace/adapter-sessions/rap-rw-adapter-session-aaaaaaaaaaaaaaaaaaaaaaaa/control"
resp, err := http.Post(controlURL, "application/json", body)
if err != nil {
t.Fatalf("post control: %v", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
raw, _ := io.ReadAll(resp.Body)
t.Fatalf("status = %d body=%s", resp.StatusCode, string(raw))
}
var result RemoteWorkspaceAdapterSessionControlResult
if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
t.Fatalf("decode control result: %v", err)
}
if !result.Accepted ||
result.Action != "close" ||
result.AdapterSessionID != "rap-rw-adapter-session-aaaaaaaaaaaaaaaaaaaaaaaa" ||
result.PreviousState != "probe_bound" ||
result.SessionState != "closed" ||
result.ActiveSessions != 0 {
t.Fatalf("control result = %+v", result)
}
report := sink.Report(time.Now().UTC())
if report["active_session_count"] != 0 ||
report["session_control_total"] != int64(1) ||
report["session_closed_total"] != int64(1) ||
report["last_controlled_adapter_session_id"] != "rap-rw-adapter-session-aaaaaaaaaaaaaaaaaaaaaaaa" ||
report["last_session_control_action"] != "close" ||
report["last_session_control_state"] != "closed" {
t.Fatalf("control report = %+v", report)
}
readiness, ok := report["adapter_runtime_readiness"].(map[string]any)
if !ok {
t.Fatalf("adapter runtime readiness missing from control report = %+v", report)
}
if readiness["schema_version"] != "rap.remote_workspace_adapter_runtime_readiness.v1" ||
readiness["status"] != "idle" ||
readiness["diagnostic_state"] != "last_session_terminal_or_expired" ||
readiness["ready"] != false ||
readiness["active_session_count"] != 0 ||
readiness["last_adapter_session_id"] != "rap-rw-adapter-session-aaaaaaaaaaaaaaaaaaaaaaaa" ||
readiness["last_session_state"] != "closed" {
t.Fatalf("invalid no-active-session readiness after close = %+v", readiness)
}
if _, ok := readiness["adapter_session_id"]; ok {
t.Fatalf("adapter_session_id should be absent without active session = %+v", readiness)
}
if _, ok := readiness["last_preflight"]; ok {
t.Fatalf("last_preflight should be absent without active session = %+v", readiness)
}
terminalSummary, ok := readiness["terminal_session_summary"].(map[string]any)
if !ok {
t.Fatalf("terminal session summary missing after close = %+v", readiness)
}
if terminalSummary["adapter_session_id"] != "rap-rw-adapter-session-aaaaaaaaaaaaaaaaaaaaaaaa" ||
terminalSummary["schema_version"] != "rap.remote_workspace_adapter_terminal_session_summary.v1" ||
!stringAnySliceContains(terminalSummary["summary_contract"], "adapter_session_id") ||
!stringAnySliceContains(terminalSummary["summary_contract"], "session_state") ||
!stringAnySliceContains(terminalSummary["summary_contract"], "reason") ||
!stringAnySliceContains(terminalSummary["summary_contract"], "controlled_at") ||
!boolMapValue(terminalSummary["summary_features"], "adapter_session_id") ||
!boolMapValue(terminalSummary["summary_features"], "session_state") ||
!boolMapValue(terminalSummary["summary_features"], "reason") ||
!boolMapValue(terminalSummary["summary_features"], "controlled_at") ||
terminalSummary["session_state"] != "closed" ||
terminalSummary["reason"] != "unit test close" ||
terminalSummary["controlled_at"] == "" {
t.Fatalf("invalid terminal session summary after close = %+v", terminalSummary)
}
resp, err = http.Post(controlURL, "application/json", bytes.NewReader([]byte(`{"action":"close","reason":"repeat close"}`)))
if err != nil {
t.Fatalf("post repeat control: %v", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
raw, _ := io.ReadAll(resp.Body)
t.Fatalf("repeat status = %d body=%s", resp.StatusCode, string(raw))
}
if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
t.Fatalf("decode repeat control result: %v", err)
}
if result.PreviousState != "closed" || result.SessionState != "closed" {
t.Fatalf("repeat control result = %+v", result)
}
report = sink.Report(time.Now().UTC())
if report["session_control_total"] != int64(2) || report["session_closed_total"] != int64(1) {
t.Fatalf("repeat control report = %+v", report)
}
}
func TestRemoteWorkspaceAdapterReadinessBeforeAnySessionHasNoTerminalSummary(t *testing.T) {
sink := NewRemoteWorkspaceFrameProbeSink()
report := sink.Report(time.Now().UTC())
readiness, ok := report["adapter_runtime_readiness"].(map[string]any)
if !ok {
t.Fatalf("adapter runtime readiness missing from report = %+v", report)
}
if readiness["schema_version"] != "rap.remote_workspace_adapter_runtime_readiness.v1" ||
readiness["status"] != "idle" ||
readiness["diagnostic_state"] != "waiting_for_session" ||
readiness["ready"] != false ||
readiness["active_session_count"] != 0 ||
readiness["terminal_session_count"] != 0 {
t.Fatalf("invalid empty readiness = %+v", readiness)
}
if _, ok := readiness["last_adapter_session_id"]; ok {
t.Fatalf("last_adapter_session_id should be absent before any session = %+v", readiness)
}
if _, ok := readiness["last_session_state"]; ok {
t.Fatalf("last_session_state should be absent before any session = %+v", readiness)
}
if _, ok := readiness["terminal_session_summary"]; ok {
t.Fatalf("terminal_session_summary should be absent before terminal history = %+v", readiness)
}
noSessionSummary, ok := readiness["no_session_summary"].(map[string]any)
if !ok {
t.Fatalf("no_session_summary should be present before any session = %+v", readiness)
}
if noSessionSummary["schema_version"] != "rap.remote_workspace_adapter_no_session_summary.v1" ||
!stringAnySliceContains(noSessionSummary["summary_contract"], "status") ||
!stringAnySliceContains(noSessionSummary["summary_contract"], "diagnostic_state") ||
!stringAnySliceContains(noSessionSummary["summary_contract"], "active_session_count") ||
!stringAnySliceContains(noSessionSummary["summary_contract"], "terminal_session_count") ||
!boolMapValue(noSessionSummary["summary_features"], "status") ||
!boolMapValue(noSessionSummary["summary_features"], "diagnostic_state") ||
!boolMapValue(noSessionSummary["summary_features"], "active_session_count") ||
!boolMapValue(noSessionSummary["summary_features"], "terminal_session_count") ||
noSessionSummary["status"] != "idle" ||
noSessionSummary["diagnostic_state"] != "waiting_for_session" ||
noSessionSummary["active_session_count"] != 0 ||
noSessionSummary["terminal_session_count"] != 0 {
t.Fatalf("invalid no-session summary before any session = %+v", noSessionSummary)
}
if _, ok := readiness["last_preflight"]; ok {
t.Fatalf("last_preflight should be absent before any session = %+v", readiness)
}
}
func TestRemoteWorkspaceAdapterReadinessSummaryExclusivity(t *testing.T) {
sink := NewRemoteWorkspaceFrameProbeSink()
freshReport := sink.Report(time.Now().UTC())
freshReadiness, ok := freshReport["adapter_runtime_readiness"].(map[string]any)
if !ok {
t.Fatalf("fresh readiness missing from report = %+v", freshReport)
}
if _, ok := freshReadiness["no_session_summary"]; !ok {
t.Fatalf("fresh readiness should include no_session_summary = %+v", freshReadiness)
}
if _, ok := freshReadiness["terminal_session_summary"]; ok {
t.Fatalf("fresh readiness should not include terminal_session_summary = %+v", freshReadiness)
}
sessionID := "rap-rw-adapter-session-d1d1d1d1d1d1d1d1d1d1d1d1"
delivery := RemoteWorkspaceFrameBatchDelivery{
ClusterID: "cluster-1",
ChannelID: "channel-rw",
ResourceID: "workspace-exclusivity",
ServiceClass: FabricServiceClassRemoteWorkspace,
ChannelClass: FabricServiceChannelInteractive,
AdapterContractID: "rap.rdp_worker.remote_workspace_adapter_contract_probe.v1",
AdapterSessionID: sessionID,
Frames: []RemoteWorkspaceFrameProbeRecord{{
Channel: "display",
Direction: "adapter_to_client",
Droppable: true,
}},
}
if _, err := sink.AcceptRemoteWorkspaceFrameBatchProbe(context.Background(), delivery); err != nil {
t.Fatalf("accept frame batch: %v", err)
}
activeReport := sink.Report(time.Now().UTC())
activeReadiness, ok := activeReport["adapter_runtime_readiness"].(map[string]any)
if !ok {
t.Fatalf("active readiness missing from report = %+v", activeReport)
}
if activeReadiness["adapter_session_id"] != sessionID ||
activeReadiness["active_session_count"] != 1 {
t.Fatalf("invalid active readiness = %+v", activeReadiness)
}
if _, ok := activeReadiness["no_session_summary"]; ok {
t.Fatalf("active readiness should not include no_session_summary = %+v", activeReadiness)
}
if _, ok := activeReadiness["terminal_session_summary"]; ok {
t.Fatalf("active readiness should not include terminal_session_summary = %+v", activeReadiness)
}
server := httptest.NewServer(Server{RemoteWorkspaceFrameSink: sink}.Handler())
defer server.Close()
body := bytes.NewReader([]byte(`{"action":"close","reason":"unit summary exclusivity close"}`))
resp, err := http.Post(server.URL+"/mesh/v1/remote-workspace/adapter-sessions/"+sessionID+"/control", "application/json", body)
if err != nil {
t.Fatalf("post control: %v", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
raw, _ := io.ReadAll(resp.Body)
t.Fatalf("status = %d body=%s", resp.StatusCode, string(raw))
}
terminalReport := sink.Report(time.Now().UTC())
terminalReadiness, ok := terminalReport["adapter_runtime_readiness"].(map[string]any)
if !ok {
t.Fatalf("terminal readiness missing from report = %+v", terminalReport)
}
if _, ok := terminalReadiness["terminal_session_summary"]; !ok {
t.Fatalf("terminal readiness should include terminal_session_summary = %+v", terminalReadiness)
}
if _, ok := terminalReadiness["no_session_summary"]; ok {
t.Fatalf("terminal readiness should not include no_session_summary = %+v", terminalReadiness)
}
}
func TestRemoteWorkspaceAdapterSessionControlTerminalReadinessStates(t *testing.T) {
tests := []struct {
action string
sessionID string
wantState string
wantClosed int64
wantExpired int64
wantReset int64
wantPrevState string
}{
{
action: "expire",
sessionID: "rap-rw-adapter-session-b0b0b0b0b0b0b0b0b0b0b0b0",
wantState: "expired",
wantClosed: 1,
wantExpired: 1,
wantPrevState: "probe_bound",
},
{
action: "reset",
sessionID: "rap-rw-adapter-session-c0c0c0c0c0c0c0c0c0c0c0c0",
wantState: "reset",
wantClosed: 1,
wantReset: 1,
wantPrevState: "probe_bound",
},
}
for _, tt := range tests {
t.Run(tt.action, func(t *testing.T) {
sink := NewRemoteWorkspaceFrameProbeSink()
delivery := RemoteWorkspaceFrameBatchDelivery{
ClusterID: "cluster-1",
ChannelID: "channel-rw",
ResourceID: "workspace-" + tt.action,
ServiceClass: FabricServiceClassRemoteWorkspace,
ChannelClass: FabricServiceChannelInteractive,
AdapterContractID: "rap.rdp_worker.remote_workspace_adapter_contract_probe.v1",
AdapterSessionID: tt.sessionID,
Frames: []RemoteWorkspaceFrameProbeRecord{{
Channel: "display",
Direction: "adapter_to_client",
Droppable: true,
}},
}
if _, err := sink.AcceptRemoteWorkspaceFrameBatchProbe(context.Background(), delivery); err != nil {
t.Fatalf("accept frame batch: %v", err)
}
server := httptest.NewServer(Server{RemoteWorkspaceFrameSink: sink}.Handler())
defer server.Close()
body := bytes.NewReader([]byte(fmt.Sprintf(`{"action":%q,"reason":"unit terminal readiness"}`, tt.action)))
resp, err := http.Post(server.URL+"/mesh/v1/remote-workspace/adapter-sessions/"+tt.sessionID+"/control", "application/json", body)
if err != nil {
t.Fatalf("post control: %v", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
raw, _ := io.ReadAll(resp.Body)
t.Fatalf("status = %d body=%s", resp.StatusCode, string(raw))
}
var result RemoteWorkspaceAdapterSessionControlResult
if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
t.Fatalf("decode control result: %v", err)
}
if !result.Accepted ||
result.Action != tt.action ||
result.AdapterSessionID != tt.sessionID ||
result.PreviousState != tt.wantPrevState ||
result.SessionState != tt.wantState ||
result.ActiveSessions != 0 {
t.Fatalf("control result = %+v", result)
}
report := sink.Report(time.Now().UTC())
if report["active_session_count"] != 0 ||
report["session_closed_total"] != tt.wantClosed ||
report["session_expired_total"] != tt.wantExpired ||
report["session_reset_total"] != tt.wantReset ||
report["last_controlled_adapter_session_id"] != tt.sessionID ||
report["last_session_control_action"] != tt.action ||
report["last_session_control_state"] != tt.wantState {
t.Fatalf("terminal control report = %+v", report)
}
readiness, ok := report["adapter_runtime_readiness"].(map[string]any)
if !ok {
t.Fatalf("adapter runtime readiness missing from report = %+v", report)
}
if readiness["status"] != "idle" ||
readiness["diagnostic_state"] != "last_session_terminal_or_expired" ||
readiness["ready"] != false ||
readiness["active_session_count"] != 0 ||
readiness["last_adapter_session_id"] != tt.sessionID ||
readiness["last_session_state"] != tt.wantState {
t.Fatalf("invalid terminal readiness = %+v", readiness)
}
if _, ok := readiness["adapter_session_id"]; ok {
t.Fatalf("adapter_session_id should be absent without active session = %+v", readiness)
}
if _, ok := readiness["last_preflight"]; ok {
t.Fatalf("last_preflight should be absent without active session = %+v", readiness)
}
if _, ok := readiness["no_session_summary"]; ok {
t.Fatalf("no_session_summary should be absent for terminal session history = %+v", readiness)
}
terminalSummary, ok := readiness["terminal_session_summary"].(map[string]any)
if !ok {
t.Fatalf("terminal session summary missing = %+v", readiness)
}
if terminalSummary["adapter_session_id"] != tt.sessionID ||
terminalSummary["schema_version"] != "rap.remote_workspace_adapter_terminal_session_summary.v1" ||
!stringAnySliceContains(terminalSummary["summary_contract"], "adapter_session_id") ||
!stringAnySliceContains(terminalSummary["summary_contract"], "session_state") ||
!stringAnySliceContains(terminalSummary["summary_contract"], "reason") ||
!stringAnySliceContains(terminalSummary["summary_contract"], "controlled_at") ||
!boolMapValue(terminalSummary["summary_features"], "adapter_session_id") ||
!boolMapValue(terminalSummary["summary_features"], "session_state") ||
!boolMapValue(terminalSummary["summary_features"], "reason") ||
!boolMapValue(terminalSummary["summary_features"], "controlled_at") ||
terminalSummary["session_state"] != tt.wantState ||
terminalSummary["reason"] != "unit terminal readiness" ||
terminalSummary["controlled_at"] == "" {
t.Fatalf("invalid terminal session summary = %+v", terminalSummary)
}
})
}
}
func TestRemoteWorkspaceAdapterSessionControlRejectsInvalidRequests(t *testing.T) {
sink := NewRemoteWorkspaceFrameProbeSink()
server := httptest.NewServer(Server{RemoteWorkspaceFrameSink: sink}.Handler())
defer server.Close()
tests := []struct {
name string
path string
body string
statusCode int
want string
}{
{
name: "unknown action",
path: "/mesh/v1/remote-workspace/adapter-sessions/rap-rw-adapter-session-aaaaaaaaaaaaaaaaaaaaaaaa/control",
body: `{"action":"launch"}`,
statusCode: http.StatusBadRequest,
want: "unsupported remote workspace adapter session control action",
},
{
name: "invalid id",
path: "/mesh/v1/remote-workspace/adapter-sessions/rap-rw-adapter-session-nothex/control",
body: `{"action":"close"}`,
statusCode: http.StatusBadRequest,
want: "invalid remote workspace adapter session id",
},
{
name: "unknown session",
path: "/mesh/v1/remote-workspace/adapter-sessions/rap-rw-adapter-session-bbbbbbbbbbbbbbbbbbbbbbbb/control",
body: `{"action":"close"}`,
statusCode: http.StatusBadRequest,
want: "remote workspace adapter session not found",
},
{
name: "bad json",
path: "/mesh/v1/remote-workspace/adapter-sessions/rap-rw-adapter-session-aaaaaaaaaaaaaaaaaaaaaaaa/control",
body: `{`,
statusCode: http.StatusBadRequest,
want: "invalid remote workspace adapter session control payload",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
resp, err := http.Post(server.URL+tt.path, "application/json", strings.NewReader(tt.body))
if err != nil {
t.Fatalf("post control: %v", err)
}
defer resp.Body.Close()
raw, _ := io.ReadAll(resp.Body)
if resp.StatusCode != tt.statusCode || !strings.Contains(string(raw), tt.want) {
t.Fatalf("status=%d body=%s, want %d containing %q", resp.StatusCode, string(raw), tt.statusCode, tt.want)
}
})
}
}
func TestRemoteWorkspaceAdapterSessionSnapshotEndpointListsActiveAndTerminalSessions(t *testing.T) {
sink := NewRemoteWorkspaceFrameProbeSink()
delivery := RemoteWorkspaceFrameBatchDelivery{
ClusterID: "cluster-1",
ChannelID: "channel-rw",
ResourceID: "workspace-1",
ServiceClass: FabricServiceClassRemoteWorkspace,
ChannelClass: FabricServiceChannelInteractive,
AdapterContractID: "rap.rdp_worker.remote_workspace_adapter_contract_probe.v1",
AdapterSessionID: "rap-rw-adapter-session-cccccccccccccccccccccccc",
Frames: []RemoteWorkspaceFrameProbeRecord{{
Channel: "display",
Direction: "adapter_to_client",
Droppable: true,
}},
}
if _, err := sink.AcceptRemoteWorkspaceFrameBatchProbe(context.Background(), delivery); err != nil {
t.Fatalf("accept frame batch: %v", err)
}
server := httptest.NewServer(Server{RemoteWorkspaceFrameSink: sink}.Handler())
defer server.Close()
resp, err := http.Get(server.URL + "/mesh/v1/remote-workspace/adapter-sessions?include_terminal=true")
if err != nil {
t.Fatalf("get snapshot: %v", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
raw, _ := io.ReadAll(resp.Body)
t.Fatalf("status = %d body=%s", resp.StatusCode, string(raw))
}
var snapshot RemoteWorkspaceAdapterSessionSnapshot
if err := json.NewDecoder(resp.Body).Decode(&snapshot); err != nil {
t.Fatalf("decode snapshot: %v", err)
}
if snapshot.SchemaVersion != "rap.remote_workspace_adapter_session_snapshot.v1" ||
snapshot.ActiveSessionCount != 1 ||
len(snapshot.Sessions) != 1 ||
snapshot.Sessions[0].AdapterSessionID != "rap-rw-adapter-session-cccccccccccccccccccccccc" ||
snapshot.Sessions[0].SessionState != "probe_bound" {
t.Fatalf("active snapshot = %+v", snapshot)
}
controlBody := bytes.NewReader([]byte(`{"action":"close","reason":"snapshot terminal"}`))
controlResp, err := http.Post(server.URL+"/mesh/v1/remote-workspace/adapter-sessions/rap-rw-adapter-session-cccccccccccccccccccccccc/control", "application/json", controlBody)
if err != nil {
t.Fatalf("post control: %v", err)
}
controlResp.Body.Close()
if controlResp.StatusCode != http.StatusOK {
t.Fatalf("control status = %d", controlResp.StatusCode)
}
resp, err = http.Get(server.URL + "/mesh/v1/remote-workspace/adapter-sessions?include_terminal=true")
if err != nil {
t.Fatalf("get terminal snapshot: %v", err)
}
defer resp.Body.Close()
if err := json.NewDecoder(resp.Body).Decode(&snapshot); err != nil {
t.Fatalf("decode terminal snapshot: %v", err)
}
if snapshot.ActiveSessionCount != 0 ||
snapshot.TerminalSessionCount != 1 ||
len(snapshot.TerminalSessions) != 1 ||
snapshot.TerminalSessions[0].AdapterSessionID != "rap-rw-adapter-session-cccccccccccccccccccccccc" ||
snapshot.TerminalSessions[0].SessionState != "closed" {
t.Fatalf("terminal snapshot = %+v", snapshot)
}
}
func TestRemoteWorkspaceAdapterSessionMailboxEndpointReadsAndDrainsEvents(t *testing.T) {
sink := NewRemoteWorkspaceFrameProbeSink()
sessionID := "rap-rw-adapter-session-dddddddddddddddddddddddd"
delivery := RemoteWorkspaceFrameBatchDelivery{
ClusterID: "cluster-1",
ChannelID: "channel-rw",
ResourceID: "workspace-1",
ServiceClass: FabricServiceClassRemoteWorkspace,
ChannelClass: FabricServiceChannelInteractive,
AdapterContractID: "rap.rdp_worker.remote_workspace_adapter_contract_probe.v1",
AdapterSessionID: sessionID,
Frames: []RemoteWorkspaceFrameProbeRecord{{
Channel: "display",
Direction: "adapter_to_client",
Droppable: true,
}},
}
if _, err := sink.AcceptRemoteWorkspaceFrameBatchProbe(context.Background(), delivery); err != nil {
t.Fatalf("accept frame batch: %v", err)
}
reliable := delivery
reliable.Frames = nil
for i := 0; i < DefaultRemoteWorkspaceFrameProbeSinkQueueCapacity+1; i++ {
reliable.Frames = append(reliable.Frames, RemoteWorkspaceFrameProbeRecord{
Channel: "input",
Direction: "client_to_adapter",
Droppable: false,
})
}
if _, err := sink.AcceptRemoteWorkspaceFrameBatchProbe(context.Background(), reliable); err == nil || !strings.Contains(err.Error(), "backpressure") {
t.Fatalf("reliable overflow err = %v, want backpressure", err)
}
server := httptest.NewServer(Server{RemoteWorkspaceFrameSink: sink}.Handler())
defer server.Close()
resp, err := http.Get(server.URL + "/mesh/v1/remote-workspace/adapter-sessions/" + sessionID + "/mailbox?limit=10")
if err != nil {
t.Fatalf("get mailbox: %v", err)
}
defer resp.Body.Close()
var mailbox RemoteWorkspaceAdapterMailboxSnapshot
if err := json.NewDecoder(resp.Body).Decode(&mailbox); err != nil {
t.Fatalf("decode mailbox: %v", err)
}
if mailbox.SchemaVersion != "rap.remote_workspace_adapter_mailbox_snapshot.v1" ||
mailbox.MailboxDepth != 2 ||
mailbox.DepthAfter != 2 ||
len(mailbox.Events) != 2 ||
mailbox.Events[0].Event != "frame_batch_probe_delivered" ||
mailbox.Events[1].Event != "backpressure" {
t.Fatalf("mailbox = %+v", mailbox)
}
resp, err = http.Get(server.URL + "/mesh/v1/remote-workspace/adapter-sessions/" + sessionID + "/mailbox?drain=true&limit=10")
if err != nil {
t.Fatalf("drain mailbox: %v", err)
}
defer resp.Body.Close()
if err := json.NewDecoder(resp.Body).Decode(&mailbox); err != nil {
t.Fatalf("decode drained mailbox: %v", err)
}
if !mailbox.Drained || mailbox.MailboxDepth != 2 || mailbox.DepthAfter != 0 || mailbox.DrainedTotal != 2 {
t.Fatalf("drained mailbox = %+v", mailbox)
}
report := sink.Report(time.Now().UTC())
if report["current_session_mailbox_depth"] != 0 ||
report["current_session_mailbox_enqueued_total"] != int64(2) ||
report["current_session_mailbox_drained_total"] != int64(2) {
t.Fatalf("mailbox report = %+v", report)
}
}
func TestRemoteWorkspaceAdapterSessionMailboxEndpointRejectsInvalidRequests(t *testing.T) {
sink := NewRemoteWorkspaceFrameProbeSink()
sessionID := "rap-rw-adapter-session-eeeeeeeeeeeeeeeeeeeeeeee"
delivery := RemoteWorkspaceFrameBatchDelivery{
ClusterID: "cluster-1",
ChannelID: "channel-rw",
ResourceID: "workspace-1",
ServiceClass: FabricServiceClassRemoteWorkspace,
ChannelClass: FabricServiceChannelInteractive,
AdapterContractID: "rap.rdp_worker.remote_workspace_adapter_contract_probe.v1",
AdapterSessionID: sessionID,
Frames: []RemoteWorkspaceFrameProbeRecord{{
Channel: "display",
Direction: "adapter_to_client",
Droppable: true,
}},
}
if _, err := sink.AcceptRemoteWorkspaceFrameBatchProbe(context.Background(), delivery); err != nil {
t.Fatalf("accept frame batch: %v", err)
}
server := httptest.NewServer(Server{RemoteWorkspaceFrameSink: sink}.Handler())
defer server.Close()
tests := []struct {
name string
path string
statusCode int
want string
}{
{
name: "invalid id",
path: "/mesh/v1/remote-workspace/adapter-sessions/rap-rw-adapter-session-nothex/mailbox",
statusCode: http.StatusBadRequest,
want: "invalid remote workspace adapter session id",
},
{
name: "unknown session",
path: "/mesh/v1/remote-workspace/adapter-sessions/rap-rw-adapter-session-ffffffffffffffffffffffff/mailbox",
statusCode: http.StatusBadRequest,
want: "remote workspace adapter session not found",
},
{
name: "bad limit",
path: "/mesh/v1/remote-workspace/adapter-sessions/" + sessionID + "/mailbox?limit=bad",
statusCode: http.StatusBadRequest,
want: "invalid remote workspace adapter session mailbox limit",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
resp, err := http.Get(server.URL + tt.path)
if err != nil {
t.Fatalf("get mailbox: %v", err)
}
defer resp.Body.Close()
raw, _ := io.ReadAll(resp.Body)
if resp.StatusCode != tt.statusCode || !strings.Contains(string(raw), tt.want) {
t.Fatalf("status=%d body=%s, want %d containing %q", resp.StatusCode, string(raw), tt.statusCode, tt.want)
}
})
}
report := sink.Report(time.Now().UTC())
if report["current_session_mailbox_depth"] != 1 || report["mailbox_drained_total"] != int64(0) {
t.Fatalf("invalid mailbox requests mutated report = %+v", report)
}
}
func TestRemoteWorkspaceAdapterSessionMailboxEndpointDrainsWithLimit(t *testing.T) {
sink := NewRemoteWorkspaceFrameProbeSink()
sessionID := "rap-rw-adapter-session-111111111111111111111111"
delivery := RemoteWorkspaceFrameBatchDelivery{
ClusterID: "cluster-1",
ChannelID: "channel-rw",
ResourceID: "workspace-1",
ServiceClass: FabricServiceClassRemoteWorkspace,
ChannelClass: FabricServiceChannelInteractive,
AdapterContractID: "rap.rdp_worker.remote_workspace_adapter_contract_probe.v1",
AdapterSessionID: sessionID,
Frames: []RemoteWorkspaceFrameProbeRecord{{
Channel: "display",
Direction: "adapter_to_client",
Droppable: true,
}},
}
for i := 0; i < 3; i++ {
if _, err := sink.AcceptRemoteWorkspaceFrameBatchProbe(context.Background(), delivery); err != nil {
t.Fatalf("accept frame batch %d: %v", i, err)
}
}
server := httptest.NewServer(Server{RemoteWorkspaceFrameSink: sink}.Handler())
defer server.Close()
resp, err := http.Get(server.URL + "/mesh/v1/remote-workspace/adapter-sessions/" + sessionID + "/mailbox?drain=true&limit=1")
if err != nil {
t.Fatalf("drain mailbox: %v", err)
}
defer resp.Body.Close()
var mailbox RemoteWorkspaceAdapterMailboxSnapshot
if err := json.NewDecoder(resp.Body).Decode(&mailbox); err != nil {
t.Fatalf("decode drained mailbox: %v", err)
}
if !mailbox.Drained || mailbox.MailboxDepth != 3 || mailbox.DepthAfter != 2 || len(mailbox.Events) != 1 || mailbox.DrainedTotal != 1 {
t.Fatalf("partial drained mailbox = %+v", mailbox)
}
resp, err = http.Get(server.URL + "/mesh/v1/remote-workspace/adapter-sessions/" + sessionID + "/mailbox?limit=10")
if err != nil {
t.Fatalf("read mailbox: %v", err)
}
defer resp.Body.Close()
if err := json.NewDecoder(resp.Body).Decode(&mailbox); err != nil {
t.Fatalf("decode mailbox: %v", err)
}
if mailbox.MailboxDepth != 2 || mailbox.DepthAfter != 2 || len(mailbox.Events) != 2 || mailbox.DrainedTotal != 1 {
t.Fatalf("remaining mailbox = %+v", mailbox)
}
}
func TestRemoteWorkspaceAdapterSessionMailboxDropsOldestWhenFull(t *testing.T) {
sink := NewRemoteWorkspaceFrameProbeSink()
sessionID := "rap-rw-adapter-session-eeeeeeeeeeeeeeeeeeeeeeee"
delivery := RemoteWorkspaceFrameBatchDelivery{
ClusterID: "cluster-1",
ChannelID: "channel-rw",
ResourceID: "workspace-1",
ServiceClass: FabricServiceClassRemoteWorkspace,
ChannelClass: FabricServiceChannelInteractive,
AdapterContractID: "rap.rdp_worker.remote_workspace_adapter_contract_probe.v1",
AdapterSessionID: sessionID,
Frames: []RemoteWorkspaceFrameProbeRecord{{
Channel: "display",
Direction: "adapter_to_client",
Droppable: true,
}},
}
for i := 0; i < DefaultRemoteWorkspaceAdapterMailboxCapacity+2; i++ {
delivery.ResourceID = fmt.Sprintf("workspace-%d", i)
if _, err := sink.AcceptRemoteWorkspaceFrameBatchProbe(context.Background(), delivery); err != nil {
t.Fatalf("accept frame batch %d: %v", i, err)
}
}
mailbox, err := sink.ReadAdapterSessionMailbox(sessionID, false, 50, 0, time.Now().UTC())
if err != nil {
t.Fatalf("read mailbox: %v", err)
}
if mailbox.MailboxCapacity != DefaultRemoteWorkspaceAdapterMailboxCapacity ||
mailbox.MailboxDepth != DefaultRemoteWorkspaceAdapterMailboxCapacity ||
mailbox.DepthAfter != DefaultRemoteWorkspaceAdapterMailboxCapacity ||
mailbox.EnqueuedTotal != int64(DefaultRemoteWorkspaceAdapterMailboxCapacity+2) ||
mailbox.DroppedTotal != 2 ||
len(mailbox.Events) != DefaultRemoteWorkspaceAdapterMailboxCapacity {
t.Fatalf("mailbox overflow snapshot = %+v", mailbox)
}
if mailbox.Events[0].Sequence != 3 ||
mailbox.Events[len(mailbox.Events)-1].Sequence != int64(DefaultRemoteWorkspaceAdapterMailboxCapacity+2) {
t.Fatalf("mailbox event window = first %d last %d", mailbox.Events[0].Sequence, mailbox.Events[len(mailbox.Events)-1].Sequence)
}
report := sink.Report(time.Now().UTC())
if report["mailbox_dropped_total"] != int64(2) ||
report["current_session_mailbox_dropped_total"] != int64(2) {
t.Fatalf("mailbox drop report = %+v", report)
}
}
func TestRemoteWorkspaceAdapterSessionMailboxGuardrailsAndClosedSession(t *testing.T) {
sink := NewRemoteWorkspaceFrameProbeSink()
sessionID := "rap-rw-adapter-session-ffffffffffffffffffffffff"
delivery := RemoteWorkspaceFrameBatchDelivery{
ClusterID: "cluster-1",
ChannelID: "channel-rw",
ResourceID: "workspace-1",
ServiceClass: FabricServiceClassRemoteWorkspace,
ChannelClass: FabricServiceChannelInteractive,
AdapterContractID: "rap.rdp_worker.remote_workspace_adapter_contract_probe.v1",
AdapterSessionID: sessionID,
Frames: []RemoteWorkspaceFrameProbeRecord{{
Channel: "display",
Direction: "adapter_to_client",
Droppable: true,
}},
}
if _, err := sink.AcceptRemoteWorkspaceFrameBatchProbe(context.Background(), delivery); err != nil {
t.Fatalf("accept frame batch: %v", err)
}
server := httptest.NewServer(Server{RemoteWorkspaceFrameSink: sink}.Handler())
defer server.Close()
resp, err := http.Get(server.URL + "/mesh/v1/remote-workspace/adapter-sessions/rap-rw-adapter-session-nothex/mailbox")
if err != nil {
t.Fatalf("get invalid mailbox: %v", err)
}
resp.Body.Close()
if resp.StatusCode != http.StatusBadRequest {
t.Fatalf("invalid mailbox status = %d, want 400", resp.StatusCode)
}
resp, err = http.Get(server.URL + "/mesh/v1/remote-workspace/adapter-sessions/" + sessionID + "/mailbox?limit=0")
if err != nil {
t.Fatalf("get invalid limit mailbox: %v", err)
}
resp.Body.Close()
if resp.StatusCode != http.StatusBadRequest {
t.Fatalf("invalid limit mailbox status = %d, want 400", resp.StatusCode)
}
controlBody := bytes.NewReader([]byte(`{"action":"close","reason":"mailbox closed"}`))
controlResp, err := http.Post(server.URL+"/mesh/v1/remote-workspace/adapter-sessions/"+sessionID+"/control", "application/json", controlBody)
if err != nil {
t.Fatalf("post control: %v", err)
}
controlResp.Body.Close()
if controlResp.StatusCode != http.StatusOK {
t.Fatalf("control status = %d", controlResp.StatusCode)
}
resp, err = http.Get(server.URL + "/mesh/v1/remote-workspace/adapter-sessions/" + sessionID + "/mailbox")
if err != nil {
t.Fatalf("get closed mailbox: %v", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusBadRequest {
t.Fatalf("closed mailbox status = %d, want 400", resp.StatusCode)
}
}
func TestRemoteWorkspaceAdapterSessionMailboxEndpointLongPollsUntilEvent(t *testing.T) {
sink := NewRemoteWorkspaceFrameProbeSink()
sessionID := "rap-rw-adapter-session-222222222222222222222222"
delivery := RemoteWorkspaceFrameBatchDelivery{
ClusterID: "cluster-1",
ChannelID: "channel-rw",
ResourceID: "workspace-1",
ServiceClass: FabricServiceClassRemoteWorkspace,
ChannelClass: FabricServiceChannelInteractive,
AdapterContractID: "rap.rdp_worker.remote_workspace_adapter_contract_probe.v1",
AdapterSessionID: sessionID,
Frames: []RemoteWorkspaceFrameProbeRecord{{
Channel: "display",
Direction: "adapter_to_client",
Droppable: true,
}},
}
if _, err := sink.AcceptRemoteWorkspaceFrameBatchProbe(context.Background(), delivery); err != nil {
t.Fatalf("accept frame batch: %v", err)
}
if _, err := sink.ReadAdapterSessionMailbox(sessionID, true, 10, 0, time.Now().UTC()); err != nil {
t.Fatalf("drain initial mailbox: %v", err)
}
server := httptest.NewServer(Server{RemoteWorkspaceFrameSink: sink}.Handler())
defer server.Close()
result := make(chan RemoteWorkspaceAdapterMailboxSnapshot, 1)
errs := make(chan error, 1)
go func() {
resp, err := http.Get(server.URL + "/mesh/v1/remote-workspace/adapter-sessions/" + sessionID + "/mailbox?wait_ms=250&limit=10")
if err != nil {
errs <- err
return
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
raw, _ := io.ReadAll(resp.Body)
errs <- fmt.Errorf("status=%d body=%s", resp.StatusCode, string(raw))
return
}
var mailbox RemoteWorkspaceAdapterMailboxSnapshot
if err := json.NewDecoder(resp.Body).Decode(&mailbox); err != nil {
errs <- err
return
}
result <- mailbox
}()
time.Sleep(50 * time.Millisecond)
if _, err := sink.AcceptRemoteWorkspaceFrameBatchProbe(context.Background(), delivery); err != nil {
t.Fatalf("accept delayed frame batch: %v", err)
}
select {
case err := <-errs:
t.Fatalf("long poll failed: %v", err)
case mailbox := <-result:
if !mailbox.Waited || mailbox.WaitTimeout || mailbox.WaitMs != 250 || mailbox.Empty ||
mailbox.MailboxDepth != 1 || len(mailbox.Events) != 1 || mailbox.Events[0].Event != "frame_batch_probe_delivered" {
t.Fatalf("long-poll mailbox = %+v", mailbox)
}
report := sink.Report(time.Now().UTC())
if report["mailbox_read_total"] != int64(1) ||
report["mailbox_wait_total"] != int64(1) ||
report["mailbox_wait_timeout_total"] != int64(0) ||
report["mailbox_empty_read_total"] != int64(0) ||
report["current_session_mailbox_wait_total"] != int64(1) ||
report["current_session_last_mailbox_wait_ms"] != 250 {
t.Fatalf("long-poll report = %+v", report)
}
case <-time.After(time.Second):
t.Fatal("timed out waiting for long-poll mailbox")
}
}
func TestRemoteWorkspaceAdapterSessionMailboxEndpointReturnsEmptyAfterWaitTimeout(t *testing.T) {
sink := NewRemoteWorkspaceFrameProbeSink()
sessionID := "rap-rw-adapter-session-333333333333333333333333"
delivery := RemoteWorkspaceFrameBatchDelivery{
ClusterID: "cluster-1",
ChannelID: "channel-rw",
ResourceID: "workspace-1",
ServiceClass: FabricServiceClassRemoteWorkspace,
ChannelClass: FabricServiceChannelInteractive,
AdapterContractID: "rap.rdp_worker.remote_workspace_adapter_contract_probe.v1",
AdapterSessionID: sessionID,
Frames: []RemoteWorkspaceFrameProbeRecord{{
Channel: "display",
Direction: "adapter_to_client",
Droppable: true,
}},
}
if _, err := sink.AcceptRemoteWorkspaceFrameBatchProbe(context.Background(), delivery); err != nil {
t.Fatalf("accept frame batch: %v", err)
}
if _, err := sink.ReadAdapterSessionMailbox(sessionID, true, 10, 0, time.Now().UTC()); err != nil {
t.Fatalf("drain initial mailbox: %v", err)
}
server := httptest.NewServer(Server{RemoteWorkspaceFrameSink: sink}.Handler())
defer server.Close()
resp, err := http.Get(server.URL + "/mesh/v1/remote-workspace/adapter-sessions/" + sessionID + "/mailbox?wait_ms=20&limit=10")
if err != nil {
t.Fatalf("get mailbox: %v", err)
}
defer resp.Body.Close()
var mailbox RemoteWorkspaceAdapterMailboxSnapshot
if err := json.NewDecoder(resp.Body).Decode(&mailbox); err != nil {
t.Fatalf("decode mailbox: %v", err)
}
if !mailbox.Empty || !mailbox.Waited || !mailbox.WaitTimeout || mailbox.WaitMs != 20 ||
mailbox.MailboxDepth != 0 || mailbox.DepthAfter != 0 || len(mailbox.Events) != 0 {
t.Fatalf("empty timeout mailbox = %+v", mailbox)
}
report := sink.Report(time.Now().UTC())
if report["mailbox_read_total"] != int64(1) ||
report["mailbox_wait_total"] != int64(1) ||
report["mailbox_wait_timeout_total"] != int64(1) ||
report["mailbox_empty_read_total"] != int64(1) ||
report["current_session_mailbox_read_total"] != int64(1) ||
report["current_session_mailbox_wait_timeout_total"] != int64(1) ||
report["current_session_mailbox_empty_read_total"] != int64(1) ||
report["current_session_last_mailbox_wait_timeout"] != true ||
report["current_session_last_mailbox_empty"] != true {
t.Fatalf("empty timeout report = %+v", report)
}
resp, err = http.Get(server.URL + "/mesh/v1/remote-workspace/adapter-sessions/" + sessionID + "/mailbox?wait_ms=bad")
if err != nil {
t.Fatalf("get bad wait mailbox: %v", err)
}
defer resp.Body.Close()
raw, _ := io.ReadAll(resp.Body)
if resp.StatusCode != http.StatusBadRequest || !strings.Contains(string(raw), "invalid remote workspace adapter session mailbox wait") {
t.Fatalf("bad wait status=%d body=%s", resp.StatusCode, string(raw))
}
}
func TestRemoteWorkspaceAdapterSessionMailboxEndpointFiltersAfterSequence(t *testing.T) {
sink := NewRemoteWorkspaceFrameProbeSink()
sessionID := "rap-rw-adapter-session-aaaaaaaaaaaaaaaaaaaaaaaa"
delivery := RemoteWorkspaceFrameBatchDelivery{
ClusterID: "cluster-1",
ChannelID: "channel-rw",
ResourceID: "workspace-1",
ServiceClass: FabricServiceClassRemoteWorkspace,
ChannelClass: FabricServiceChannelInteractive,
AdapterContractID: "rap.rdp_worker.remote_workspace_adapter_contract_probe.v1",
AdapterSessionID: sessionID,
Frames: []RemoteWorkspaceFrameProbeRecord{{
Channel: "display",
Direction: "adapter_to_client",
Droppable: true,
}},
}
for i := 0; i < 3; i++ {
delivery.ResourceID = fmt.Sprintf("workspace-%d", i)
if _, err := sink.AcceptRemoteWorkspaceFrameBatchProbe(context.Background(), delivery); err != nil {
t.Fatalf("accept frame batch %d: %v", i, err)
}
}
server := httptest.NewServer(Server{RemoteWorkspaceFrameSink: sink}.Handler())
defer server.Close()
resp, err := http.Get(server.URL + "/mesh/v1/remote-workspace/adapter-sessions/" + sessionID + "/mailbox?after_sequence=1&limit=10")
if err != nil {
t.Fatalf("get filtered mailbox: %v", err)
}
defer resp.Body.Close()
var mailbox RemoteWorkspaceAdapterMailboxSnapshot
if err := json.NewDecoder(resp.Body).Decode(&mailbox); err != nil {
t.Fatalf("decode filtered mailbox: %v", err)
}
if mailbox.AfterSequence != 1 ||
mailbox.SkippedCount != 1 ||
mailbox.ReturnedCount != 2 ||
mailbox.MailboxDepth != 3 ||
mailbox.DepthAfter != 3 ||
mailbox.Empty ||
len(mailbox.Events) != 2 ||
mailbox.Events[0].Sequence != 2 ||
mailbox.Events[1].Sequence != 3 {
t.Fatalf("filtered mailbox = %+v", mailbox)
}
report := sink.Report(time.Now().UTC())
if report["mailbox_read_total"] != int64(1) ||
report["current_session_mailbox_depth"] != 3 {
t.Fatalf("filtered mailbox report = %+v", report)
}
}
func TestRemoteWorkspaceAdapterSessionMailboxEndpointLongPollsAfterSequence(t *testing.T) {
sink := NewRemoteWorkspaceFrameProbeSink()
sessionID := "rap-rw-adapter-session-bbbbbbbbbbbbbbbbbbbbbbbb"
delivery := RemoteWorkspaceFrameBatchDelivery{
ClusterID: "cluster-1",
ChannelID: "channel-rw",
ResourceID: "workspace-1",
ServiceClass: FabricServiceClassRemoteWorkspace,
ChannelClass: FabricServiceChannelInteractive,
AdapterContractID: "rap.rdp_worker.remote_workspace_adapter_contract_probe.v1",
AdapterSessionID: sessionID,
Frames: []RemoteWorkspaceFrameProbeRecord{{
Channel: "display",
Direction: "adapter_to_client",
Droppable: true,
}},
}
if _, err := sink.AcceptRemoteWorkspaceFrameBatchProbe(context.Background(), delivery); err != nil {
t.Fatalf("accept initial frame batch: %v", err)
}
server := httptest.NewServer(Server{RemoteWorkspaceFrameSink: sink}.Handler())
defer server.Close()
result := make(chan RemoteWorkspaceAdapterMailboxSnapshot, 1)
errs := make(chan error, 1)
go func() {
resp, err := http.Get(server.URL + "/mesh/v1/remote-workspace/adapter-sessions/" + sessionID + "/mailbox?after_sequence=1&wait_ms=250&limit=10")
if err != nil {
errs <- err
return
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
raw, _ := io.ReadAll(resp.Body)
errs <- fmt.Errorf("status=%d body=%s", resp.StatusCode, string(raw))
return
}
var mailbox RemoteWorkspaceAdapterMailboxSnapshot
if err := json.NewDecoder(resp.Body).Decode(&mailbox); err != nil {
errs <- err
return
}
result <- mailbox
}()
time.Sleep(50 * time.Millisecond)
delivery.ResourceID = "workspace-delayed"
if _, err := sink.AcceptRemoteWorkspaceFrameBatchProbe(context.Background(), delivery); err != nil {
t.Fatalf("accept delayed frame batch: %v", err)
}
select {
case err := <-errs:
t.Fatalf("after-sequence long poll failed: %v", err)
case mailbox := <-result:
if mailbox.AfterSequence != 1 ||
!mailbox.Waited ||
mailbox.WaitTimeout ||
mailbox.ReturnedCount != 1 ||
len(mailbox.Events) != 1 ||
mailbox.Events[0].Sequence != 2 ||
mailbox.MailboxDepth != 2 ||
mailbox.DepthAfter != 2 {
t.Fatalf("after-sequence long-poll mailbox = %+v", mailbox)
}
case <-time.After(time.Second):
t.Fatal("timed out waiting for after-sequence long-poll mailbox")
}
}
func TestRemoteWorkspaceAdapterSessionMailboxEndpointRejectsInvalidAfterSequence(t *testing.T) {
sink := NewRemoteWorkspaceFrameProbeSink()
sessionID := "rap-rw-adapter-session-cccccccccccccccccccccccc"
delivery := RemoteWorkspaceFrameBatchDelivery{
ClusterID: "cluster-1",
ChannelID: "channel-rw",
ResourceID: "workspace-1",
ServiceClass: FabricServiceClassRemoteWorkspace,
ChannelClass: FabricServiceChannelInteractive,
AdapterContractID: "rap.rdp_worker.remote_workspace_adapter_contract_probe.v1",
AdapterSessionID: sessionID,
Frames: []RemoteWorkspaceFrameProbeRecord{{
Channel: "display",
Direction: "adapter_to_client",
Droppable: true,
}},
}
if _, err := sink.AcceptRemoteWorkspaceFrameBatchProbe(context.Background(), delivery); err != nil {
t.Fatalf("accept frame batch: %v", err)
}
server := httptest.NewServer(Server{RemoteWorkspaceFrameSink: sink}.Handler())
defer server.Close()
tests := []struct {
name string
path string
want string
}{
{
name: "bad after sequence",
path: "/mesh/v1/remote-workspace/adapter-sessions/" + sessionID + "/mailbox?after_sequence=bad",
want: "invalid remote workspace adapter session mailbox after sequence",
},
{
name: "drain after sequence",
path: "/mesh/v1/remote-workspace/adapter-sessions/" + sessionID + "/mailbox?after_sequence=1&drain=true",
want: "remote workspace adapter session mailbox after sequence cannot drain",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
resp, err := http.Get(server.URL + tt.path)
if err != nil {
t.Fatalf("get mailbox: %v", err)
}
defer resp.Body.Close()
raw, _ := io.ReadAll(resp.Body)
if resp.StatusCode != http.StatusBadRequest || !strings.Contains(string(raw), tt.want) {
t.Fatalf("status=%d body=%s, want 400 containing %q", resp.StatusCode, string(raw), tt.want)
}
})
}
report := sink.Report(time.Now().UTC())
if report["mailbox_read_total"] != int64(0) ||
report["current_session_mailbox_depth"] != 1 {
t.Fatalf("invalid after-sequence requests mutated report = %+v", report)
}
}
func TestRemoteWorkspaceAdapterSessionMailboxEndpointResumesFromConsumerCursor(t *testing.T) {
sink := NewRemoteWorkspaceFrameProbeSink()
sessionID := "rap-rw-adapter-session-dddddddddddddddddddddddd"
delivery := RemoteWorkspaceFrameBatchDelivery{
ClusterID: "cluster-1",
ChannelID: "channel-rw",
ResourceID: "workspace-1",
ServiceClass: FabricServiceClassRemoteWorkspace,
ChannelClass: FabricServiceChannelInteractive,
AdapterContractID: "rap.rdp_worker.remote_workspace_adapter_contract_probe.v1",
AdapterSessionID: sessionID,
Frames: []RemoteWorkspaceFrameProbeRecord{{
Channel: "display",
Direction: "adapter_to_client",
Droppable: true,
}},
}
for i := 0; i < 3; i++ {
delivery.ResourceID = fmt.Sprintf("workspace-%d", i)
if _, err := sink.AcceptRemoteWorkspaceFrameBatchProbe(context.Background(), delivery); err != nil {
t.Fatalf("accept frame batch %d: %v", i, err)
}
}
server := httptest.NewServer(Server{RemoteWorkspaceFrameSink: sink}.Handler())
defer server.Close()
resp, err := http.Get(server.URL + "/mesh/v1/remote-workspace/adapter-sessions/" + sessionID + "/mailbox?consumer_id=rdp-worker-probe&limit=2")
if err != nil {
t.Fatalf("get consumer mailbox: %v", err)
}
defer resp.Body.Close()
var mailbox RemoteWorkspaceAdapterMailboxSnapshot
if err := json.NewDecoder(resp.Body).Decode(&mailbox); err != nil {
t.Fatalf("decode consumer mailbox: %v", err)
}
if mailbox.ConsumerCheckpointSequence != 2 {
t.Fatalf("checkpoint mailbox = %+v", mailbox)
}
resp, err = http.Get(server.URL + "/mesh/v1/remote-workspace/adapter-sessions/" + sessionID + "/mailbox?consumer_id=rdp-worker-probe&resume_from=checkpoint&limit=10")
if err != nil {
t.Fatalf("resume checkpoint mailbox: %v", err)
}
defer resp.Body.Close()
mailbox = RemoteWorkspaceAdapterMailboxSnapshot{}
if err := json.NewDecoder(resp.Body).Decode(&mailbox); err != nil {
t.Fatalf("decode resume checkpoint mailbox: %v", err)
}
if mailbox.ResumeFrom != "checkpoint" ||
mailbox.ResumeSequence != 2 ||
mailbox.AfterSequence != 2 ||
mailbox.SkippedCount != 2 ||
mailbox.ReturnedCount != 1 ||
len(mailbox.Events) != 1 ||
mailbox.Events[0].Sequence != 3 ||
mailbox.ConsumerCheckpointSequence != 3 {
t.Fatalf("resume checkpoint mailbox = %+v", mailbox)
}
resp, err = http.Get(server.URL + "/mesh/v1/remote-workspace/adapter-sessions/" + sessionID + "/mailbox?consumer_id=rdp-worker-probe&ack_sequence=2&resume_from=ack&limit=10")
if err != nil {
t.Fatalf("resume ack mailbox: %v", err)
}
defer resp.Body.Close()
mailbox = RemoteWorkspaceAdapterMailboxSnapshot{}
if err := json.NewDecoder(resp.Body).Decode(&mailbox); err != nil {
t.Fatalf("decode resume ack mailbox: %v", err)
}
if mailbox.ResumeFrom != "ack" ||
mailbox.ResumeSequence != 0 ||
mailbox.AfterSequence != 0 ||
mailbox.ReturnedCount != 3 ||
mailbox.ConsumerAckSequence != 2 {
t.Fatalf("resume ack mailbox = %+v", mailbox)
}
report := sink.Report(time.Now().UTC())
if report["mailbox_resume_read_total"] != int64(2) ||
report["mailbox_after_sequence_read_total"] != int64(1) ||
report["mailbox_returned_total"] != int64(6) ||
report["mailbox_skipped_total"] != int64(2) ||
report["last_mailbox_resume_from"] != "ack" ||
report["last_mailbox_resume_sequence"] != int64(0) ||
report["last_mailbox_resume_consumer_id"] != "rdp-worker-probe" ||
report["last_mailbox_after_sequence"] != int64(0) ||
report["last_mailbox_skipped_count"] != 0 ||
report["last_mailbox_returned_count"] != 3 ||
report["current_session_mailbox_resume_read_total"] != int64(2) ||
report["current_session_mailbox_after_sequence_read_total"] != int64(1) ||
report["current_session_mailbox_returned_total"] != int64(6) ||
report["current_session_mailbox_skipped_total"] != int64(2) ||
report["current_session_last_mailbox_resume_from"] != "ack" ||
report["current_session_last_mailbox_resume_sequence"] != int64(0) ||
report["current_session_last_mailbox_resume_consumer_id"] != "rdp-worker-probe" ||
report["current_session_last_mailbox_after_sequence"] != int64(0) ||
report["current_session_last_mailbox_skipped_count"] != 0 ||
report["current_session_last_mailbox_returned_count"] != 3 {
t.Fatalf("invalid resume telemetry report = %+v", report)
}
readiness, ok := report["adapter_runtime_readiness"].(map[string]any)
if !ok {
t.Fatalf("adapter runtime readiness missing from report = %+v", report)
}
if readiness["schema_version"] != "rap.remote_workspace_adapter_runtime_readiness.v1" ||
readiness["status"] != "cursor_ready" ||
readiness["diagnostic_state"] != "adapter_cursor_ready" ||
readiness["ready"] != true ||
readiness["adapter_session_id"] != sessionID ||
readiness["session_state"] != "probe_bound" ||
readiness["mailbox_depth"] != 3 ||
readiness["consumer_count"] != 1 ||
readiness["last_consumer_id"] != "rdp-worker-probe" ||
readiness["last_consumer_checkpoint_sequence"] != int64(3) ||
readiness["last_consumer_ack_sequence"] != int64(2) ||
readiness["last_consumer_lag_count"] != 1 ||
readiness["last_resume_from"] != "ack" ||
readiness["last_resume_sequence"] != int64(0) ||
readiness["last_resume_consumer_id"] != "rdp-worker-probe" ||
readiness["last_returned_count"] != 3 ||
readiness["last_skipped_count"] != 0 {
t.Fatalf("invalid adapter runtime readiness = %+v", readiness)
}
}
func TestRemoteWorkspaceAdapterSessionMailboxEndpointRejectsInvalidResumeFrom(t *testing.T) {
sink := NewRemoteWorkspaceFrameProbeSink()
sessionID := "rap-rw-adapter-session-eeeeeeeeeeeeeeeeeeeeeeee"
delivery := RemoteWorkspaceFrameBatchDelivery{
ClusterID: "cluster-1",
ChannelID: "channel-rw",
ResourceID: "workspace-1",
ServiceClass: FabricServiceClassRemoteWorkspace,
ChannelClass: FabricServiceChannelInteractive,
AdapterContractID: "rap.rdp_worker.remote_workspace_adapter_contract_probe.v1",
AdapterSessionID: sessionID,
Frames: []RemoteWorkspaceFrameProbeRecord{{
Channel: "display",
Direction: "adapter_to_client",
Droppable: true,
}},
}
if _, err := sink.AcceptRemoteWorkspaceFrameBatchProbe(context.Background(), delivery); err != nil {
t.Fatalf("accept frame batch: %v", err)
}
server := httptest.NewServer(Server{RemoteWorkspaceFrameSink: sink}.Handler())
defer server.Close()
tests := []struct {
name string
query string
want string
}{
{
name: "invalid resume cursor",
query: "consumer_id=rdp-worker-probe&resume_from=bad",
want: "invalid remote workspace adapter mailbox resume cursor",
},
{
name: "resume without consumer",
query: "resume_from=ack",
want: "remote workspace adapter mailbox consumer required for resume",
},
{
name: "resume with after",
query: "consumer_id=rdp-worker-probe&resume_from=ack&after_sequence=1",
want: "remote workspace adapter mailbox resume cannot combine with after sequence",
},
{
name: "resume with drain",
query: "consumer_id=rdp-worker-probe&resume_from=ack&drain=true",
want: "remote workspace adapter mailbox resume cannot drain",
},
{
name: "resume with reset",
query: "consumer_id=rdp-worker-probe&resume_from=ack&reset_consumer=true",
want: "remote workspace adapter mailbox resume cannot reset consumer",
},
{
name: "unknown consumer",
query: "consumer_id=rdp-worker-probe&resume_from=ack",
want: "remote workspace adapter mailbox consumer not found",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
resp, err := http.Get(server.URL + "/mesh/v1/remote-workspace/adapter-sessions/" + sessionID + "/mailbox?" + tt.query)
if err != nil {
t.Fatalf("get mailbox: %v", err)
}
defer resp.Body.Close()
raw, _ := io.ReadAll(resp.Body)
if resp.StatusCode != http.StatusBadRequest || !strings.Contains(string(raw), tt.want) {
t.Fatalf("status=%d body=%s, want 400 containing %q", resp.StatusCode, string(raw), tt.want)
}
})
}
report := sink.Report(time.Now().UTC())
if report["mailbox_read_total"] != int64(0) ||
report["mailbox_consumer_read_total"] != int64(0) ||
report["current_session_mailbox_depth"] != 1 {
t.Fatalf("invalid resume requests mutated report = %+v", report)
}
}
func TestRemoteWorkspaceAdapterSessionMailboxConsumerCheckpointAndAck(t *testing.T) {
sink := NewRemoteWorkspaceFrameProbeSink()
sessionID := "rap-rw-adapter-session-444444444444444444444444"
delivery := RemoteWorkspaceFrameBatchDelivery{
ClusterID: "cluster-1",
ChannelID: "channel-rw",
ResourceID: "workspace-1",
ServiceClass: FabricServiceClassRemoteWorkspace,
ChannelClass: FabricServiceChannelInteractive,
AdapterContractID: "rap.rdp_worker.remote_workspace_adapter_contract_probe.v1",
AdapterSessionID: sessionID,
Frames: []RemoteWorkspaceFrameProbeRecord{{
Channel: "display",
Direction: "adapter_to_client",
Droppable: true,
}},
}
for i := 0; i < 2; i++ {
delivery.ResourceID = fmt.Sprintf("workspace-%d", i)
if _, err := sink.AcceptRemoteWorkspaceFrameBatchProbe(context.Background(), delivery); err != nil {
t.Fatalf("accept frame batch %d: %v", i, err)
}
}
server := httptest.NewServer(Server{RemoteWorkspaceFrameSink: sink}.Handler())
defer server.Close()
resp, err := http.Get(server.URL + "/mesh/v1/remote-workspace/adapter-sessions/" + sessionID + "/mailbox?consumer_id=rdp-worker-probe&limit=10")
if err != nil {
t.Fatalf("get consumer mailbox: %v", err)
}
defer resp.Body.Close()
var mailbox RemoteWorkspaceAdapterMailboxSnapshot
if err := json.NewDecoder(resp.Body).Decode(&mailbox); err != nil {
t.Fatalf("decode consumer mailbox: %v", err)
}
if mailbox.ConsumerID != "rdp-worker-probe" ||
mailbox.ConsumerReadTotal != 1 ||
mailbox.ConsumerAckTotal != 0 ||
mailbox.ConsumerCheckpointSequence != mailbox.Events[len(mailbox.Events)-1].Sequence ||
mailbox.ConsumerAckSequence != 0 ||
mailbox.ConsumerLagCount != 2 ||
mailbox.ConsumerCount != 1 ||
mailbox.ConsumerCapacity != DefaultRemoteWorkspaceAdapterMailboxConsumerCapacity ||
!mailbox.ConsumerCreated ||
mailbox.ConsumerCreatedAt == "" ||
mailbox.ConsumerLastReadAt == "" {
t.Fatalf("consumer mailbox = %+v", mailbox)
}
resp, err = http.Get(fmt.Sprintf("%s/mesh/v1/remote-workspace/adapter-sessions/%s/mailbox?consumer_id=rdp-worker-probe&ack_sequence=%d&limit=10", server.URL, sessionID, mailbox.ConsumerCheckpointSequence))
if err != nil {
t.Fatalf("ack consumer mailbox: %v", err)
}
defer resp.Body.Close()
if err := json.NewDecoder(resp.Body).Decode(&mailbox); err != nil {
t.Fatalf("decode acked consumer mailbox: %v", err)
}
if mailbox.ConsumerReadTotal != 2 ||
mailbox.ConsumerAckTotal != 1 ||
mailbox.ConsumerAckSequence != mailbox.ConsumerCheckpointSequence ||
mailbox.ConsumerLagCount != 0 {
t.Fatalf("acked consumer mailbox = %+v", mailbox)
}
report := sink.Report(time.Now().UTC())
if report["mailbox_consumer_count"] != 1 ||
report["mailbox_consumer_read_total"] != int64(2) ||
report["mailbox_consumer_ack_total"] != int64(1) ||
report["last_mailbox_consumer_id"] != "rdp-worker-probe" ||
report["last_mailbox_consumer_checkpoint_sequence"] != mailbox.ConsumerCheckpointSequence ||
report["last_mailbox_consumer_ack_sequence"] != mailbox.ConsumerAckSequence ||
report["current_session_mailbox_consumer_count"] != 1 ||
report["current_session_mailbox_consumer_read_total"] != int64(2) ||
report["current_session_mailbox_consumer_ack_total"] != int64(1) {
t.Fatalf("consumer report = %+v", report)
}
}
func TestRemoteWorkspaceAdapterSessionMailboxConsumerRejectsInvalidCursorInputs(t *testing.T) {
sink := NewRemoteWorkspaceFrameProbeSink()
sessionID := "rap-rw-adapter-session-555555555555555555555555"
delivery := RemoteWorkspaceFrameBatchDelivery{
ClusterID: "cluster-1",
ChannelID: "channel-rw",
ResourceID: "workspace-1",
ServiceClass: FabricServiceClassRemoteWorkspace,
ChannelClass: FabricServiceChannelInteractive,
AdapterContractID: "rap.rdp_worker.remote_workspace_adapter_contract_probe.v1",
AdapterSessionID: sessionID,
Frames: []RemoteWorkspaceFrameProbeRecord{{
Channel: "display",
Direction: "adapter_to_client",
Droppable: true,
}},
}
if _, err := sink.AcceptRemoteWorkspaceFrameBatchProbe(context.Background(), delivery); err != nil {
t.Fatalf("accept frame batch: %v", err)
}
server := httptest.NewServer(Server{RemoteWorkspaceFrameSink: sink}.Handler())
defer server.Close()
tests := []struct {
name string
query string
want string
}{
{
name: "invalid consumer",
query: "consumer_id=bad%2Fconsumer",
want: "invalid remote workspace adapter mailbox consumer",
},
{
name: "invalid ack",
query: "consumer_id=rdp-worker-probe&ack_sequence=-1",
want: "invalid remote workspace adapter mailbox ack sequence",
},
{
name: "invalid reset",
query: "consumer_id=rdp-worker-probe&reset_consumer=maybe",
want: "invalid remote workspace adapter mailbox consumer reset",
},
{
name: "reset without consumer",
query: "reset_consumer=true",
want: "remote workspace adapter mailbox consumer required for reset",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
resp, err := http.Get(server.URL + "/mesh/v1/remote-workspace/adapter-sessions/" + sessionID + "/mailbox?" + tt.query)
if err != nil {
t.Fatalf("get mailbox: %v", err)
}
defer resp.Body.Close()
raw, _ := io.ReadAll(resp.Body)
if resp.StatusCode != http.StatusBadRequest || !strings.Contains(string(raw), tt.want) {
t.Fatalf("status=%d body=%s, want 400 containing %q", resp.StatusCode, string(raw), tt.want)
}
})
}
report := sink.Report(time.Now().UTC())
if report["mailbox_read_total"] != int64(0) ||
report["mailbox_consumer_read_total"] != int64(0) ||
report["current_session_mailbox_depth"] != 1 {
t.Fatalf("invalid consumer requests mutated report = %+v", report)
}
}
func TestRemoteWorkspaceAdapterSessionMailboxConsumerResetCursor(t *testing.T) {
sink := NewRemoteWorkspaceFrameProbeSink()
sessionID := "rap-rw-adapter-session-777777777777777777777777"
delivery := RemoteWorkspaceFrameBatchDelivery{
ClusterID: "cluster-1",
ChannelID: "channel-rw",
ResourceID: "workspace-1",
ServiceClass: FabricServiceClassRemoteWorkspace,
ChannelClass: FabricServiceChannelInteractive,
AdapterContractID: "rap.rdp_worker.remote_workspace_adapter_contract_probe.v1",
AdapterSessionID: sessionID,
Frames: []RemoteWorkspaceFrameProbeRecord{{
Channel: "display",
Direction: "adapter_to_client",
Droppable: true,
}},
}
if _, err := sink.AcceptRemoteWorkspaceFrameBatchProbe(context.Background(), delivery); err != nil {
t.Fatalf("accept frame batch: %v", err)
}
server := httptest.NewServer(Server{RemoteWorkspaceFrameSink: sink}.Handler())
defer server.Close()
resp, err := http.Get(server.URL + "/mesh/v1/remote-workspace/adapter-sessions/" + sessionID + "/mailbox?consumer_id=rdp-worker-probe&ack_sequence=1")
if err != nil {
t.Fatalf("ack consumer mailbox: %v", err)
}
defer resp.Body.Close()
var mailbox RemoteWorkspaceAdapterMailboxSnapshot
if err := json.NewDecoder(resp.Body).Decode(&mailbox); err != nil {
t.Fatalf("decode ack mailbox: %v", err)
}
if mailbox.ConsumerReadTotal != 1 || mailbox.ConsumerAckTotal != 1 || mailbox.ConsumerAckSequence != 1 {
t.Fatalf("acked consumer mailbox = %+v", mailbox)
}
resp, err = http.Get(server.URL + "/mesh/v1/remote-workspace/adapter-sessions/" + sessionID + "/mailbox?consumer_id=rdp-worker-probe&reset_consumer=true")
if err != nil {
t.Fatalf("reset consumer mailbox: %v", err)
}
defer resp.Body.Close()
if err := json.NewDecoder(resp.Body).Decode(&mailbox); err != nil {
t.Fatalf("decode reset mailbox: %v", err)
}
if !mailbox.ConsumerReset ||
!mailbox.ConsumerCreated ||
mailbox.ConsumerReadTotal != 1 ||
mailbox.ConsumerAckTotal != 0 ||
mailbox.ConsumerAckSequence != 0 ||
mailbox.ConsumerResetTotal != 1 ||
mailbox.ConsumerCount != 1 {
t.Fatalf("reset consumer mailbox = %+v", mailbox)
}
report := sink.Report(time.Now().UTC())
if report["mailbox_consumer_reset_total"] != int64(1) ||
report["current_session_mailbox_consumer_reset_total"] != int64(1) {
t.Fatalf("reset consumer report = %+v", report)
}
}
func TestRemoteWorkspaceAdapterSessionMailboxConsumerStateIsBounded(t *testing.T) {
sink := NewRemoteWorkspaceFrameProbeSink()
sessionID := "rap-rw-adapter-session-666666666666666666666666"
delivery := RemoteWorkspaceFrameBatchDelivery{
ClusterID: "cluster-1",
ChannelID: "channel-rw",
ResourceID: "workspace-1",
ServiceClass: FabricServiceClassRemoteWorkspace,
ChannelClass: FabricServiceChannelInteractive,
AdapterContractID: "rap.rdp_worker.remote_workspace_adapter_contract_probe.v1",
AdapterSessionID: sessionID,
Frames: []RemoteWorkspaceFrameProbeRecord{{
Channel: "display",
Direction: "adapter_to_client",
Droppable: true,
}},
}
if _, err := sink.AcceptRemoteWorkspaceFrameBatchProbe(context.Background(), delivery); err != nil {
t.Fatalf("accept frame batch: %v", err)
}
server := httptest.NewServer(Server{RemoteWorkspaceFrameSink: sink}.Handler())
defer server.Close()
var mailbox RemoteWorkspaceAdapterMailboxSnapshot
for i := 0; i < DefaultRemoteWorkspaceAdapterMailboxConsumerCapacity+1; i++ {
resp, err := http.Get(fmt.Sprintf("%s/mesh/v1/remote-workspace/adapter-sessions/%s/mailbox?consumer_id=consumer-%02d", server.URL, sessionID, i))
if err != nil {
t.Fatalf("get consumer mailbox %d: %v", i, err)
}
if resp.StatusCode != http.StatusOK {
resp.Body.Close()
t.Fatalf("consumer mailbox %d status = %d", i, resp.StatusCode)
}
if err := json.NewDecoder(resp.Body).Decode(&mailbox); err != nil {
resp.Body.Close()
t.Fatalf("decode consumer mailbox %d: %v", i, err)
}
resp.Body.Close()
}
report := sink.Report(time.Now().UTC())
if report["mailbox_consumer_count"] != DefaultRemoteWorkspaceAdapterMailboxConsumerCapacity ||
report["current_session_mailbox_consumer_count"] != DefaultRemoteWorkspaceAdapterMailboxConsumerCapacity ||
report["mailbox_consumer_evicted_total"] != int64(1) ||
report["current_session_mailbox_consumer_evicted_total"] != int64(1) ||
!mailbox.ConsumerEvicted ||
mailbox.ConsumerEvictedTotal != 1 ||
mailbox.ConsumerCount != DefaultRemoteWorkspaceAdapterMailboxConsumerCapacity {
t.Fatalf("bounded consumer report = %+v", report)
}
}
func TestRemoteWorkspaceAdapterSessionMailboxConsumerSnapshotIsReadOnly(t *testing.T) {
sink := NewRemoteWorkspaceFrameProbeSink()
sessionID := "rap-rw-adapter-session-888888888888888888888888"
delivery := RemoteWorkspaceFrameBatchDelivery{
ClusterID: "cluster-1",
ChannelID: "channel-rw",
ResourceID: "workspace-1",
ServiceClass: FabricServiceClassRemoteWorkspace,
ChannelClass: FabricServiceChannelInteractive,
AdapterContractID: "rap.rdp_worker.remote_workspace_adapter_contract_probe.v1",
AdapterSessionID: sessionID,
Frames: []RemoteWorkspaceFrameProbeRecord{{
Channel: "display",
Direction: "adapter_to_client",
Droppable: true,
}},
}
if _, err := sink.AcceptRemoteWorkspaceFrameBatchProbe(context.Background(), delivery); err != nil {
t.Fatalf("accept frame batch: %v", err)
}
server := httptest.NewServer(Server{RemoteWorkspaceFrameSink: sink}.Handler())
defer server.Close()
resp, err := http.Get(server.URL + "/mesh/v1/remote-workspace/adapter-sessions/" + sessionID + "/mailbox?consumer_id=consumer-b&ack_sequence=1")
if err != nil {
t.Fatalf("ack consumer b: %v", err)
}
resp.Body.Close()
resp, err = http.Get(server.URL + "/mesh/v1/remote-workspace/adapter-sessions/" + sessionID + "/mailbox?consumer_id=consumer-a")
if err != nil {
t.Fatalf("read consumer a: %v", err)
}
resp.Body.Close()
reportBefore := sink.Report(time.Now().UTC())
resp, err = http.Get(server.URL + "/mesh/v1/remote-workspace/adapter-sessions/" + sessionID + "/mailbox/consumers?limit=1")
if err != nil {
t.Fatalf("get consumer snapshot: %v", err)
}
defer resp.Body.Close()
var snapshot RemoteWorkspaceAdapterMailboxConsumerSnapshot
if err := json.NewDecoder(resp.Body).Decode(&snapshot); err != nil {
t.Fatalf("decode consumer snapshot: %v", err)
}
if snapshot.SchemaVersion != "rap.remote_workspace_adapter_mailbox_consumer_snapshot.v1" ||
snapshot.AdapterSessionID != sessionID ||
snapshot.ConsumerCapacity != DefaultRemoteWorkspaceAdapterMailboxConsumerCapacity ||
snapshot.ConsumerCount != 2 ||
snapshot.ConsumerReadTotal != 2 ||
snapshot.ConsumerAckTotal != 1 ||
len(snapshot.Consumers) != 1 ||
snapshot.Consumers[0].ConsumerID != "consumer-a" ||
snapshot.Consumers[0].CheckpointSequence != 1 ||
snapshot.Consumers[0].LagCount != 1 {
t.Fatalf("consumer snapshot = %+v", snapshot)
}
reportAfter := sink.Report(time.Now().UTC())
if reportAfter["mailbox_read_total"] != reportBefore["mailbox_read_total"] ||
reportAfter["mailbox_consumer_read_total"] != reportBefore["mailbox_consumer_read_total"] ||
reportAfter["mailbox_consumer_ack_total"] != reportBefore["mailbox_consumer_ack_total"] {
t.Fatalf("consumer snapshot mutated report before=%+v after=%+v", reportBefore, reportAfter)
}
}
func TestRemoteWorkspaceAdapterSessionMailboxConsumerSnapshotRejectsInvalidRequests(t *testing.T) {
sink := NewRemoteWorkspaceFrameProbeSink()
sessionID := "rap-rw-adapter-session-999999999999999999999999"
delivery := RemoteWorkspaceFrameBatchDelivery{
ClusterID: "cluster-1",
ChannelID: "channel-rw",
ResourceID: "workspace-1",
ServiceClass: FabricServiceClassRemoteWorkspace,
ChannelClass: FabricServiceChannelInteractive,
AdapterContractID: "rap.rdp_worker.remote_workspace_adapter_contract_probe.v1",
AdapterSessionID: sessionID,
Frames: []RemoteWorkspaceFrameProbeRecord{{
Channel: "display",
Direction: "adapter_to_client",
Droppable: true,
}},
}
if _, err := sink.AcceptRemoteWorkspaceFrameBatchProbe(context.Background(), delivery); err != nil {
t.Fatalf("accept frame batch: %v", err)
}
server := httptest.NewServer(Server{RemoteWorkspaceFrameSink: sink}.Handler())
defer server.Close()
tests := []struct {
name string
path string
want string
}{
{
name: "bad limit",
path: "/mesh/v1/remote-workspace/adapter-sessions/" + sessionID + "/mailbox/consumers?limit=bad",
want: "invalid remote workspace adapter mailbox consumer snapshot limit",
},
{
name: "invalid id",
path: "/mesh/v1/remote-workspace/adapter-sessions/rap-rw-adapter-session-nothex/mailbox/consumers",
want: "invalid remote workspace adapter session id",
},
{
name: "unknown session",
path: "/mesh/v1/remote-workspace/adapter-sessions/rap-rw-adapter-session-aaaaaaaaaaaaaaaaaaaaaaaa/mailbox/consumers",
want: "remote workspace adapter session not found",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
resp, err := http.Get(server.URL + tt.path)
if err != nil {
t.Fatalf("get consumer snapshot: %v", err)
}
defer resp.Body.Close()
raw, _ := io.ReadAll(resp.Body)
if resp.StatusCode != http.StatusBadRequest || !strings.Contains(string(raw), tt.want) {
t.Fatalf("status=%d body=%s, want 400 containing %q", resp.StatusCode, string(raw), tt.want)
}
})
}
}
func TestRemoteWorkspaceAdapterSessionMailboxPreflightIsReadOnly(t *testing.T) {
sink := NewRemoteWorkspaceFrameProbeSink()
sessionID := "rap-rw-adapter-session-abababababababababababab"
delivery := RemoteWorkspaceFrameBatchDelivery{
ClusterID: "cluster-1",
ChannelID: "channel-rw",
ResourceID: "workspace-1",
ServiceClass: FabricServiceClassRemoteWorkspace,
ChannelClass: FabricServiceChannelInteractive,
AdapterContractID: "rap.rdp_worker.remote_workspace_adapter_contract_probe.v1",
AdapterSessionID: sessionID,
Frames: []RemoteWorkspaceFrameProbeRecord{{
Channel: "display",
Direction: "adapter_to_client",
Droppable: true,
}},
}
for i := 0; i < 3; i++ {
delivery.ResourceID = fmt.Sprintf("workspace-preflight-%d", i)
if _, err := sink.AcceptRemoteWorkspaceFrameBatchProbe(context.Background(), delivery); err != nil {
t.Fatalf("accept frame batch %d: %v", i, err)
}
}
server := httptest.NewServer(Server{RemoteWorkspaceFrameSink: sink}.Handler())
defer server.Close()
resp, err := http.Get(server.URL + "/mesh/v1/remote-workspace/adapter-sessions/" + sessionID + "/mailbox?consumer_id=rdp-worker-probe&limit=2")
if err != nil {
t.Fatalf("seed checkpoint: %v", err)
}
resp.Body.Close()
resp, err = http.Get(server.URL + "/mesh/v1/remote-workspace/adapter-sessions/" + sessionID + "/mailbox?consumer_id=rdp-worker-probe&ack_sequence=1&limit=1")
if err != nil {
t.Fatalf("seed ack: %v", err)
}
resp.Body.Close()
reportBefore := sink.Report(time.Now().UTC())
resp, err = http.Get(server.URL + "/mesh/v1/remote-workspace/adapter-sessions/" + sessionID + "/mailbox/preflight?consumer_id=rdp-worker-probe&resume_from=ack&limit=1")
if err != nil {
t.Fatalf("get preflight ack: %v", err)
}
defer resp.Body.Close()
var preflight RemoteWorkspaceAdapterMailboxPreflightSnapshot
if err := json.NewDecoder(resp.Body).Decode(&preflight); err != nil {
t.Fatalf("decode preflight ack: %v", err)
}
if preflight.SchemaVersion != "rap.remote_workspace_adapter_mailbox_preflight.v1" ||
preflight.AdapterSessionID != sessionID ||
!preflight.ReadOnly ||
preflight.ConsumerID != "rdp-worker-probe" ||
preflight.ResumeFrom != "ack" ||
preflight.ResumeSequence != 1 ||
preflight.AfterSequence != 1 ||
preflight.Limit != 1 ||
preflight.MailboxDepth != 3 ||
preflight.ConsumerCheckpointSequence != 2 ||
preflight.ConsumerAckSequence != 1 ||
preflight.ConsumerLagCount != 2 ||
preflight.ExpectedAvailableCount != 2 ||
preflight.ExpectedReturnedCount != 1 ||
preflight.ExpectedSkippedCount != 1 ||
preflight.FirstExpectedSequence != 2 ||
preflight.LastExpectedSequence != 2 {
t.Fatalf("preflight ack = %+v", preflight)
}
resp, err = http.Get(server.URL + "/mesh/v1/remote-workspace/adapter-sessions/" + sessionID + "/mailbox/preflight?consumer_id=rdp-worker-probe&resume_from=checkpoint&limit=10")
if err != nil {
t.Fatalf("get preflight checkpoint: %v", err)
}
defer resp.Body.Close()
preflight = RemoteWorkspaceAdapterMailboxPreflightSnapshot{}
if err := json.NewDecoder(resp.Body).Decode(&preflight); err != nil {
t.Fatalf("decode preflight checkpoint: %v", err)
}
if preflight.ResumeFrom != "checkpoint" ||
preflight.ResumeSequence != 2 ||
preflight.DiagnosticState != "ready" ||
preflight.RecommendedAction != "resume_from_cursor" ||
preflight.ActionReason != "cursor_window_available" ||
preflight.OperatorSummary != "consumer cursor can resume from requested window" ||
preflight.OperatorStatus != "ready_to_resume" ||
preflight.OperatorSeverity != "ok" ||
anyInt64(preflight.ActionContext["resume_sequence"]) != 2 ||
anyInt64(preflight.ActionContext["first_retained_sequence"]) != 1 ||
preflight.OperatorSummaryFields["diagnostic_state"] != "ready" ||
preflight.OperatorSummaryFields["recommended_action"] != "resume_from_cursor" ||
preflight.OperatorSummaryFields["operator_status"] != "ready_to_resume" ||
preflight.OperatorSummaryFields["operator_severity"] != "ok" ||
!stringSliceContains(preflight.ActionHints, "resume_from_requested_cursor") ||
preflight.ExpectedAvailableCount != 1 ||
preflight.ExpectedReturnedCount != 1 ||
preflight.ExpectedSkippedCount != 2 ||
preflight.FirstExpectedSequence != 3 ||
preflight.LastExpectedSequence != 3 {
t.Fatalf("preflight checkpoint = %+v", preflight)
}
reportAfter := sink.Report(time.Now().UTC())
if reportAfter["mailbox_read_total"] != reportBefore["mailbox_read_total"] ||
reportAfter["mailbox_consumer_read_total"] != reportBefore["mailbox_consumer_read_total"] ||
reportAfter["mailbox_consumer_ack_total"] != reportBefore["mailbox_consumer_ack_total"] ||
reportAfter["current_session_mailbox_consumer_read_total"] != reportBefore["current_session_mailbox_consumer_read_total"] ||
reportAfter["current_session_mailbox_consumer_ack_total"] != reportBefore["current_session_mailbox_consumer_ack_total"] {
t.Fatalf("preflight mutated report before=%+v after=%+v", reportBefore, reportAfter)
}
if reportAfter["mailbox_preflight_total"] != int64(2) ||
reportAfter["mailbox_preflight_ack_total"] != int64(1) ||
reportAfter["mailbox_preflight_checkpoint_total"] != int64(1) ||
reportAfter["last_mailbox_preflight_adapter_session_id"] != sessionID ||
reportAfter["last_mailbox_preflight_consumer_id"] != "rdp-worker-probe" ||
reportAfter["last_mailbox_preflight_resume_from"] != "checkpoint" ||
reportAfter["last_mailbox_preflight_resume_sequence"] != int64(2) ||
reportAfter["last_mailbox_preflight_available_count"] != 1 ||
reportAfter["last_mailbox_preflight_returned_count"] != 1 ||
reportAfter["last_mailbox_preflight_skipped_count"] != 2 ||
reportAfter["current_session_mailbox_preflight_total"] != int64(2) ||
reportAfter["current_session_mailbox_preflight_ack_total"] != int64(1) ||
reportAfter["current_session_mailbox_preflight_checkpoint_total"] != int64(1) ||
mapInt64Value(reportAfter["current_session_mailbox_preflight_operator_status_counts"], "ready_to_resume") != 2 ||
mapInt64Value(reportAfter["current_session_mailbox_preflight_operator_severity_counts"], "ok") != 2 ||
reportAfter["current_session_last_mailbox_preflight_resume_from"] != "checkpoint" ||
reportAfter["current_session_last_mailbox_preflight_resume_sequence"] != int64(2) ||
reportAfter["current_session_last_mailbox_preflight_returned_count"] != 1 ||
reportAfter["current_session_last_mailbox_preflight_recommended_action"] != "resume_from_cursor" ||
reportAfter["last_mailbox_preflight_operator_summary"] != "consumer cursor can resume from requested window" ||
reportAfter["last_mailbox_preflight_operator_status"] != "ready_to_resume" ||
reportAfter["last_mailbox_preflight_operator_severity"] != "ok" ||
reportAfter["current_session_last_mailbox_preflight_operator_summary"] != "consumer cursor can resume from requested window" ||
reportAfter["current_session_last_mailbox_preflight_operator_status"] != "ready_to_resume" ||
reportAfter["current_session_last_mailbox_preflight_operator_severity"] != "ok" {
t.Fatalf("invalid preflight telemetry report = %+v", reportAfter)
}
readiness, ok := reportAfter["adapter_runtime_readiness"].(map[string]any)
if !ok {
t.Fatalf("adapter runtime readiness missing from report = %+v", reportAfter)
}
if readiness["mailbox_preflight_total"] != int64(2) ||
readiness["last_preflight_consumer_id"] != "rdp-worker-probe" ||
readiness["last_preflight_resume_from"] != "checkpoint" ||
readiness["last_preflight_resume_sequence"] != int64(2) ||
readiness["last_preflight_returned_count"] != 1 ||
readiness["last_preflight_skipped_count"] != 2 ||
readiness["last_preflight_recommended_action"] != "resume_from_cursor" ||
readiness["last_preflight_action_reason"] != "cursor_window_available" ||
readiness["last_preflight_operator_summary"] != "consumer cursor can resume from requested window" ||
readiness["last_preflight_operator_status"] != "ready_to_resume" ||
readiness["last_preflight_operator_severity"] != "ok" ||
mapInt64Value(readiness["mailbox_preflight_operator_status_counts"], "ready_to_resume") != 2 ||
mapInt64Value(readiness["mailbox_preflight_operator_severity_counts"], "ok") != 2 ||
readiness["preflight_attention_status"] != "clean" ||
readiness["preflight_attention_reason"] != "no_resync_required_preflight_observed" {
t.Fatalf("invalid preflight readiness = %+v", readiness)
}
lastPreflight, ok := readiness["last_preflight"].(map[string]any)
if !ok {
t.Fatalf("last preflight rollup missing from readiness = %+v", readiness)
}
if lastPreflight["consumer_id"] != "rdp-worker-probe" ||
lastPreflight["diagnostics_schema_version"] != "rap.remote_workspace_adapter_mailbox_preflight_diagnostics.v1" ||
!stringAnySliceContains(lastPreflight["diagnostics_contract"], "retained_window") ||
!stringAnySliceContains(lastPreflight["diagnostics_contract"], "remediation_checklist") ||
!stringAnySliceContains(lastPreflight["diagnostics_contract"], "attention") ||
!stringAnySliceContains(lastPreflight["diagnostics_contract"], "operator_counts") ||
!boolMapValue(lastPreflight["diagnostics_features"], "retained_window") ||
!boolMapValue(lastPreflight["diagnostics_features"], "remediation_checklist") ||
!boolMapValue(lastPreflight["diagnostics_features"], "attention") ||
!boolMapValue(lastPreflight["diagnostics_features"], "operator_counts") ||
lastPreflight["resume_from"] != "checkpoint" ||
lastPreflight["operator_status"] != "ready_to_resume" ||
lastPreflight["operator_severity"] != "ok" ||
lastPreflight["recommended_action"] != "resume_from_cursor" ||
!preflightChecklistContains(lastPreflight["remediation_checklist"], "resume_from_requested_cursor", true, true) ||
lastPreflight["remediation_checklist_status"] != "ready" ||
anyInt64(preflightChecklistCountsValue(lastPreflight["remediation_checklist_counts"], "required_count")) != 1 ||
anyInt64(preflightChecklistCountsValue(lastPreflight["remediation_checklist_counts"], "satisfied_count")) != 1 ||
anyInt64(preflightChecklistCountsValue(lastPreflight["remediation_checklist_counts"], "pending_count")) != 0 ||
mapInt64Value(lastPreflight["operator_status_counts"], "ready_to_resume") != 2 ||
mapInt64Value(lastPreflight["operator_severity_counts"], "ok") != 2 ||
lastPreflight["preflight_attention_status"] != "clean" ||
lastPreflight["preflight_attention_reason"] != "no_resync_required_preflight_observed" ||
anyInt64(lastPreflight["resume_sequence"]) != 2 ||
anyInt64(lastPreflight["first_retained_sequence"]) != 1 ||
anyInt64(lastPreflight["last_retained_sequence"]) != 3 ||
anyInt64(lastPreflight["mailbox_dropped_total"]) != 0 ||
anyInt64(lastPreflight["mailbox_preflight_total"]) != 2 {
t.Fatalf("invalid last preflight rollup = %+v", lastPreflight)
}
}
func TestRemoteWorkspaceAdapterSessionReadinessBeforePreflight(t *testing.T) {
sink := NewRemoteWorkspaceFrameProbeSink()
sessionID := "rap-rw-adapter-session-a0a0a0a0a0a0a0a0a0a0a0a0"
delivery := RemoteWorkspaceFrameBatchDelivery{
ClusterID: "cluster-1",
ChannelID: "channel-rw",
ResourceID: "workspace-before-preflight",
ServiceClass: FabricServiceClassRemoteWorkspace,
ChannelClass: FabricServiceChannelInteractive,
AdapterContractID: "rap.rdp_worker.remote_workspace_adapter_contract_probe.v1",
AdapterSessionID: sessionID,
Frames: []RemoteWorkspaceFrameProbeRecord{{
Channel: "display",
Direction: "adapter_to_client",
Droppable: true,
}},
}
if _, err := sink.AcceptRemoteWorkspaceFrameBatchProbe(context.Background(), delivery); err != nil {
t.Fatalf("accept frame batch: %v", err)
}
report := sink.Report(time.Now().UTC())
if report["mailbox_preflight_total"] != int64(0) ||
report["current_session_mailbox_preflight_total"] != int64(0) {
t.Fatalf("unexpected preflight totals before preflight = %+v", report)
}
readiness, ok := report["adapter_runtime_readiness"].(map[string]any)
if !ok {
t.Fatalf("adapter runtime readiness missing from report = %+v", report)
}
if readiness["adapter_session_id"] != sessionID ||
readiness["mailbox_preflight_total"] != int64(0) ||
readiness["preflight_attention_status"] != "unknown" ||
readiness["preflight_attention_reason"] != "no_preflight_observed" {
t.Fatalf("invalid no-preflight readiness = %+v", readiness)
}
if _, ok := readiness["last_preflight"]; ok {
t.Fatalf("last preflight rollup should be absent before preflight = %+v", readiness["last_preflight"])
}
if readiness["last_preflight_diagnostic_state"] != "" ||
readiness["last_preflight_recommended_action"] != "" ||
len(readiness["last_preflight_action_hints"].([]string)) != 0 {
t.Fatalf("last preflight flat fields should be empty before preflight = %+v", readiness)
}
}
func TestRemoteWorkspaceAdapterSessionMailboxPreflightReportsStaleCursorGap(t *testing.T) {
sink := NewRemoteWorkspaceFrameProbeSink()
sessionID := "rap-rw-adapter-session-adadadadadadadadadadadad"
delivery := RemoteWorkspaceFrameBatchDelivery{
ClusterID: "cluster-1",
ChannelID: "channel-rw",
ResourceID: "workspace-stale-0",
ServiceClass: FabricServiceClassRemoteWorkspace,
ChannelClass: FabricServiceChannelInteractive,
AdapterContractID: "rap.rdp_worker.remote_workspace_adapter_contract_probe.v1",
AdapterSessionID: sessionID,
Frames: []RemoteWorkspaceFrameProbeRecord{{
Channel: "display",
Direction: "adapter_to_client",
Droppable: true,
}},
}
if _, err := sink.AcceptRemoteWorkspaceFrameBatchProbe(context.Background(), delivery); err != nil {
t.Fatalf("accept initial frame batch: %v", err)
}
server := httptest.NewServer(Server{RemoteWorkspaceFrameSink: sink}.Handler())
defer server.Close()
resp, err := http.Get(server.URL + "/mesh/v1/remote-workspace/adapter-sessions/" + sessionID + "/mailbox?consumer_id=rdp-worker-probe&ack_sequence=1&limit=1")
if err != nil {
t.Fatalf("seed ack cursor: %v", err)
}
resp.Body.Close()
for i := 1; i <= DefaultRemoteWorkspaceAdapterMailboxCapacity+2; i++ {
delivery.ResourceID = fmt.Sprintf("workspace-stale-%d", i)
if _, err := sink.AcceptRemoteWorkspaceFrameBatchProbe(context.Background(), delivery); err != nil {
t.Fatalf("accept overflow frame batch %d: %v", i, err)
}
}
resp, err = http.Get(server.URL + "/mesh/v1/remote-workspace/adapter-sessions/" + sessionID + "/mailbox/preflight?consumer_id=rdp-worker-probe&resume_from=ack&limit=3")
if err != nil {
t.Fatalf("get stale preflight: %v", err)
}
defer resp.Body.Close()
var preflight RemoteWorkspaceAdapterMailboxPreflightSnapshot
if err := json.NewDecoder(resp.Body).Decode(&preflight); err != nil {
t.Fatalf("decode stale preflight: %v", err)
}
if preflight.ResumeFrom != "ack" ||
preflight.ResumeSequence != 1 ||
preflight.MailboxDepth != DefaultRemoteWorkspaceAdapterMailboxCapacity ||
preflight.MailboxDropped != 3 ||
preflight.ExpectedAvailableCount != DefaultRemoteWorkspaceAdapterMailboxCapacity ||
preflight.ExpectedReturnedCount != 3 ||
preflight.ExpectedSkippedCount != 0 ||
preflight.FirstExpectedSequence != 4 ||
preflight.LastExpectedSequence != 6 ||
preflight.FirstRetainedSequence != 4 ||
preflight.LastRetainedSequence != 19 ||
preflight.DiagnosticState != "stale_cursor_gap" ||
!preflight.StaleCursor ||
preflight.MissingDroppedCount != 2 ||
preflight.RecommendedAction != "reset_consumer_and_resync" ||
preflight.ActionReason != "consumer_cursor_before_first_retained_sequence" ||
preflight.OperatorSummary != "stale cursor gap: reset consumer and resync before resume" ||
preflight.OperatorStatus != "resync_required" ||
preflight.OperatorSeverity != "warn" ||
anyInt64(preflight.ActionContext["resume_sequence"]) != 1 ||
anyInt64(preflight.ActionContext["first_retained_sequence"]) != 4 ||
anyInt64(preflight.ActionContext["missing_dropped_count"]) != 2 ||
preflight.OperatorSummaryFields["diagnostic_state"] != "stale_cursor_gap" ||
preflight.OperatorSummaryFields["recommended_action"] != "reset_consumer_and_resync" ||
preflight.OperatorSummaryFields["operator_status"] != "resync_required" ||
preflight.OperatorSummaryFields["operator_severity"] != "warn" ||
anyInt64(preflight.OperatorSummaryFields["missing_dropped_count"]) != 2 ||
!stringSliceContains(preflight.ActionHints, "reset_consumer_cursor") ||
!stringSliceContains(preflight.ActionHints, "request_full_adapter_resync") ||
!stringSliceContains(preflight.ActionHints, "resume_from_checkpoint_after_resync") {
t.Fatalf("stale preflight = %+v", preflight)
}
resp, err = http.Get(server.URL + "/mesh/v1/remote-workspace/adapter-sessions/" + sessionID + "/mailbox/preflight?consumer_id=rdp-worker-probe&resume_from=ack&limit=3")
if err != nil {
t.Fatalf("get repeated stale preflight: %v", err)
}
resp.Body.Close()
report := sink.Report(time.Now().UTC())
if report["last_mailbox_preflight_diagnostic_state"] != "stale_cursor_gap" ||
report["last_mailbox_preflight_stale_cursor"] != true ||
report["last_mailbox_preflight_missing_dropped_count"] != 2 ||
report["last_mailbox_preflight_recommended_action"] != "reset_consumer_and_resync" ||
report["last_mailbox_preflight_action_reason"] != "consumer_cursor_before_first_retained_sequence" ||
report["last_mailbox_preflight_operator_summary"] != "stale cursor gap: reset consumer and resync before resume" ||
report["last_mailbox_preflight_operator_status"] != "resync_required" ||
report["last_mailbox_preflight_operator_severity"] != "warn" ||
report["current_session_last_mailbox_preflight_diagnostic_state"] != "stale_cursor_gap" ||
report["current_session_last_mailbox_preflight_stale_cursor"] != true ||
report["current_session_last_mailbox_preflight_missing_dropped_count"] != 2 ||
report["current_session_last_mailbox_preflight_recommended_action"] != "reset_consumer_and_resync" ||
report["current_session_last_mailbox_preflight_operator_summary"] != "stale cursor gap: reset consumer and resync before resume" ||
report["current_session_last_mailbox_preflight_operator_status"] != "resync_required" ||
report["current_session_last_mailbox_preflight_operator_severity"] != "warn" ||
mapInt64Value(report["current_session_mailbox_preflight_operator_status_counts"], "resync_required") != 2 ||
mapInt64Value(report["current_session_mailbox_preflight_operator_severity_counts"], "warn") != 2 {
t.Fatalf("stale preflight report = %+v", report)
}
readiness, ok := report["adapter_runtime_readiness"].(map[string]any)
if !ok {
t.Fatalf("adapter runtime readiness missing from report = %+v", report)
}
if readiness["last_preflight_diagnostic_state"] != "stale_cursor_gap" ||
readiness["last_preflight_stale_cursor"] != true ||
readiness["last_preflight_missing_dropped_count"] != 2 ||
readiness["last_preflight_recommended_action"] != "reset_consumer_and_resync" ||
readiness["last_preflight_action_reason"] != "consumer_cursor_before_first_retained_sequence" ||
readiness["last_preflight_operator_summary"] != "stale cursor gap: reset consumer and resync before resume" ||
readiness["last_preflight_operator_status"] != "resync_required" ||
readiness["last_preflight_operator_severity"] != "warn" ||
mapInt64Value(readiness["mailbox_preflight_operator_status_counts"], "resync_required") != 2 ||
mapInt64Value(readiness["mailbox_preflight_operator_severity_counts"], "warn") != 2 ||
readiness["preflight_attention_status"] != "repeated_resync_required" ||
readiness["preflight_attention_reason"] != "resync_required_preflight_repeated" {
t.Fatalf("stale preflight readiness = %+v", readiness)
}
lastPreflight, ok := readiness["last_preflight"].(map[string]any)
if !ok {
t.Fatalf("stale last preflight rollup missing from readiness = %+v", readiness)
}
if lastPreflight["diagnostic_state"] != "stale_cursor_gap" ||
lastPreflight["diagnostics_schema_version"] != "rap.remote_workspace_adapter_mailbox_preflight_diagnostics.v1" ||
!stringAnySliceContains(lastPreflight["diagnostics_contract"], "retained_window") ||
!stringAnySliceContains(lastPreflight["diagnostics_contract"], "remediation_checklist") ||
!stringAnySliceContains(lastPreflight["diagnostics_contract"], "attention") ||
!stringAnySliceContains(lastPreflight["diagnostics_contract"], "operator_counts") ||
!boolMapValue(lastPreflight["diagnostics_features"], "retained_window") ||
!boolMapValue(lastPreflight["diagnostics_features"], "remediation_checklist") ||
!boolMapValue(lastPreflight["diagnostics_features"], "attention") ||
!boolMapValue(lastPreflight["diagnostics_features"], "operator_counts") ||
lastPreflight["operator_status"] != "resync_required" ||
lastPreflight["operator_severity"] != "warn" ||
lastPreflight["recommended_action"] != "reset_consumer_and_resync" ||
lastPreflight["action_reason"] != "consumer_cursor_before_first_retained_sequence" ||
!preflightChecklistContains(lastPreflight["remediation_checklist"], "reset_consumer_cursor", true, false) ||
!preflightChecklistContains(lastPreflight["remediation_checklist"], "request_full_adapter_resync", true, false) ||
!preflightChecklistContains(lastPreflight["remediation_checklist"], "resume_from_checkpoint_after_resync", true, false) ||
lastPreflight["remediation_checklist_status"] != "action_required" ||
anyInt64(preflightChecklistCountsValue(lastPreflight["remediation_checklist_counts"], "required_count")) != 3 ||
anyInt64(preflightChecklistCountsValue(lastPreflight["remediation_checklist_counts"], "satisfied_count")) != 0 ||
anyInt64(preflightChecklistCountsValue(lastPreflight["remediation_checklist_counts"], "pending_count")) != 3 ||
mapInt64Value(lastPreflight["operator_status_counts"], "resync_required") != 2 ||
mapInt64Value(lastPreflight["operator_severity_counts"], "warn") != 2 ||
lastPreflight["preflight_attention_status"] != "repeated_resync_required" ||
lastPreflight["preflight_attention_reason"] != "resync_required_preflight_repeated" ||
anyInt64(lastPreflight["missing_dropped_count"]) != 2 ||
anyInt64(lastPreflight["first_retained_sequence"]) != 4 ||
anyInt64(lastPreflight["last_retained_sequence"]) != 19 ||
anyInt64(lastPreflight["mailbox_dropped_total"]) != 3 ||
anyInt64(lastPreflight["resume_sequence"]) != 1 {
t.Fatalf("invalid stale last preflight rollup = %+v", lastPreflight)
}
}
func preflightChecklistCountsValue(value any, key string) any {
switch counts := value.(type) {
case map[string]any:
return counts[key]
default:
return nil
}
}
func mapInt64Value(value any, key string) int64 {
switch items := value.(type) {
case map[string]int64:
return items[key]
case map[string]any:
return anyInt64(items[key])
default:
return 0
}
}
func boolMapValue(value any, key string) bool {
switch items := value.(type) {
case map[string]bool:
return items[key]
case map[string]any:
item, _ := items[key].(bool)
return item
default:
return false
}
}
func preflightDiagnosticsContractCompatible(rollup map[string]any) bool {
for _, feature := range []string{"retained_window", "remediation_checklist", "attention", "operator_counts"} {
if !stringAnySliceContains(rollup["diagnostics_contract"], feature) || !boolMapValue(rollup["diagnostics_features"], feature) {
return false
}
}
return true
}
func terminalSessionSummaryContractCompatible(summary map[string]any) bool {
for _, feature := range []string{"adapter_session_id", "session_state", "reason", "controlled_at"} {
if !stringAnySliceContains(summary["summary_contract"], feature) || !boolMapValue(summary["summary_features"], feature) {
return false
}
}
return true
}
func noSessionSummaryContractCompatible(summary map[string]any) bool {
for _, feature := range []string{"status", "diagnostic_state", "active_session_count", "terminal_session_count"} {
if !stringAnySliceContains(summary["summary_contract"], feature) || !boolMapValue(summary["summary_features"], feature) {
return false
}
}
return true
}
func stringAnySliceContains(value any, want string) bool {
switch items := value.(type) {
case []string:
for _, item := range items {
if item == want {
return true
}
}
case []any:
for _, item := range items {
if item == want {
return true
}
}
}
return false
}
func preflightChecklistContains(value any, step string, required bool, satisfied bool) bool {
switch items := value.(type) {
case []map[string]any:
for _, item := range items {
if item["step"] == step && item["required"] == required && item["satisfied"] == satisfied && item["source_hint"] == true {
return true
}
}
case []any:
for _, raw := range items {
item, ok := raw.(map[string]any)
if !ok {
continue
}
if item["step"] == step && item["required"] == required && item["satisfied"] == satisfied && item["source_hint"] == true {
return true
}
}
}
return false
}
func stringSliceContains(items []string, want string) bool {
for _, item := range items {
if item == want {
return true
}
}
return false
}
func anyInt64(value any) int64 {
switch v := value.(type) {
case int:
return int64(v)
case int64:
return v
case float64:
return int64(v)
default:
return 0
}
}
func TestRemoteWorkspacePreflightDiagnosticsContractCompatibility(t *testing.T) {
compatible := map[string]any{
"diagnostics_contract": []string{"retained_window", "remediation_checklist", "attention", "operator_counts"},
"diagnostics_features": map[string]bool{"retained_window": true, "remediation_checklist": true, "attention": true, "operator_counts": true},
}
if !preflightDiagnosticsContractCompatible(compatible) {
t.Fatalf("expected contract/features to be compatible")
}
tests := []struct {
name string
rollup map[string]any
}{
{
name: "missing contract item",
rollup: map[string]any{
"diagnostics_contract": []string{"retained_window", "remediation_checklist", "attention"},
"diagnostics_features": map[string]bool{"retained_window": true, "remediation_checklist": true, "attention": true, "operator_counts": true},
},
},
{
name: "missing feature flag",
rollup: map[string]any{
"diagnostics_contract": []string{"retained_window", "remediation_checklist", "attention", "operator_counts"},
"diagnostics_features": map[string]bool{"retained_window": true, "remediation_checklist": true, "attention": true},
},
},
{
name: "false feature flag",
rollup: map[string]any{
"diagnostics_contract": []string{"retained_window", "remediation_checklist", "attention", "operator_counts"},
"diagnostics_features": map[string]bool{"retained_window": true, "remediation_checklist": true, "attention": true, "operator_counts": false},
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if preflightDiagnosticsContractCompatible(tt.rollup) {
t.Fatalf("expected incompatible contract/features for %+v", tt.rollup)
}
})
}
}
func TestRemoteWorkspaceTerminalSessionSummaryContractCompatibility(t *testing.T) {
compatible := map[string]any{
"summary_contract": []string{"adapter_session_id", "session_state", "reason", "controlled_at"},
"summary_features": map[string]bool{"adapter_session_id": true, "session_state": true, "reason": true, "controlled_at": true},
}
if !terminalSessionSummaryContractCompatible(compatible) {
t.Fatalf("expected summary contract/features to be compatible")
}
tests := []struct {
name string
summary map[string]any
}{
{
name: "missing contract item",
summary: map[string]any{
"summary_contract": []string{"adapter_session_id", "session_state", "reason"},
"summary_features": map[string]bool{"adapter_session_id": true, "session_state": true, "reason": true, "controlled_at": true},
},
},
{
name: "missing feature flag",
summary: map[string]any{
"summary_contract": []string{"adapter_session_id", "session_state", "reason", "controlled_at"},
"summary_features": map[string]bool{"adapter_session_id": true, "session_state": true, "reason": true},
},
},
{
name: "false feature flag",
summary: map[string]any{
"summary_contract": []string{"adapter_session_id", "session_state", "reason", "controlled_at"},
"summary_features": map[string]bool{"adapter_session_id": true, "session_state": true, "reason": true, "controlled_at": false},
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if terminalSessionSummaryContractCompatible(tt.summary) {
t.Fatalf("expected incompatible summary contract/features for %+v", tt.summary)
}
})
}
}
func TestRemoteWorkspaceNoSessionSummaryContractCompatibility(t *testing.T) {
compatible := map[string]any{
"summary_contract": []string{"status", "diagnostic_state", "active_session_count", "terminal_session_count"},
"summary_features": map[string]bool{"status": true, "diagnostic_state": true, "active_session_count": true, "terminal_session_count": true},
}
if !noSessionSummaryContractCompatible(compatible) {
t.Fatalf("expected no-session summary contract/features to be compatible")
}
tests := []struct {
name string
summary map[string]any
}{
{
name: "missing contract item",
summary: map[string]any{
"summary_contract": []string{"status", "diagnostic_state", "active_session_count"},
"summary_features": map[string]bool{"status": true, "diagnostic_state": true, "active_session_count": true, "terminal_session_count": true},
},
},
{
name: "missing feature flag",
summary: map[string]any{
"summary_contract": []string{"status", "diagnostic_state", "active_session_count", "terminal_session_count"},
"summary_features": map[string]bool{"status": true, "diagnostic_state": true, "active_session_count": true},
},
},
{
name: "false feature flag",
summary: map[string]any{
"summary_contract": []string{"status", "diagnostic_state", "active_session_count", "terminal_session_count"},
"summary_features": map[string]bool{"status": true, "diagnostic_state": true, "active_session_count": true, "terminal_session_count": false},
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if noSessionSummaryContractCompatible(tt.summary) {
t.Fatalf("expected incompatible no-session summary contract/features for %+v", tt.summary)
}
})
}
}
func TestRemoteWorkspaceAdapterSessionMailboxPreflightRejectsInvalidRequests(t *testing.T) {
sink := NewRemoteWorkspaceFrameProbeSink()
sessionID := "rap-rw-adapter-session-acacacacacacacacacacacac"
delivery := RemoteWorkspaceFrameBatchDelivery{
ClusterID: "cluster-1",
ChannelID: "channel-rw",
ResourceID: "workspace-1",
ServiceClass: FabricServiceClassRemoteWorkspace,
ChannelClass: FabricServiceChannelInteractive,
AdapterContractID: "rap.rdp_worker.remote_workspace_adapter_contract_probe.v1",
AdapterSessionID: sessionID,
Frames: []RemoteWorkspaceFrameProbeRecord{{
Channel: "display",
Direction: "adapter_to_client",
Droppable: true,
}},
}
if _, err := sink.AcceptRemoteWorkspaceFrameBatchProbe(context.Background(), delivery); err != nil {
t.Fatalf("accept frame batch: %v", err)
}
server := httptest.NewServer(Server{RemoteWorkspaceFrameSink: sink}.Handler())
defer server.Close()
tests := []struct {
name string
path string
want string
}{
{
name: "missing consumer",
path: "/mesh/v1/remote-workspace/adapter-sessions/" + sessionID + "/mailbox/preflight",
want: "remote workspace adapter mailbox consumer required for preflight",
},
{
name: "invalid resume",
path: "/mesh/v1/remote-workspace/adapter-sessions/" + sessionID + "/mailbox/preflight?consumer_id=consumer-a&resume_from=bogus",
want: "invalid remote workspace adapter mailbox resume cursor",
},
{
name: "bad limit",
path: "/mesh/v1/remote-workspace/adapter-sessions/" + sessionID + "/mailbox/preflight?consumer_id=consumer-a&limit=bad",
want: "invalid remote workspace adapter mailbox preflight limit",
},
{
name: "unknown consumer",
path: "/mesh/v1/remote-workspace/adapter-sessions/" + sessionID + "/mailbox/preflight?consumer_id=consumer-a",
want: "remote workspace adapter mailbox consumer not found",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
resp, err := http.Get(server.URL + tt.path)
if err != nil {
t.Fatalf("get preflight: %v", err)
}
defer resp.Body.Close()
raw, _ := io.ReadAll(resp.Body)
if resp.StatusCode != http.StatusBadRequest || !strings.Contains(string(raw), tt.want) {
t.Fatalf("status=%d body=%s, want 400 containing %q", resp.StatusCode, string(raw), tt.want)
}
})
}
}
func TestRemoteWorkspacePreflightAttentionReasonSummaries(t *testing.T) {
tests := []struct {
name string
statusCounts map[string]int64
severityCounts map[string]int64
wantStatus string
wantReason string
}{
{
name: "clean ready",
statusCounts: map[string]int64{"ready_to_resume": 1},
severityCounts: map[string]int64{"ok": 1},
wantStatus: "clean",
wantReason: "no_resync_required_preflight_observed",
},
{
name: "single resync",
statusCounts: map[string]int64{"resync_required": 1},
severityCounts: map[string]int64{"warn": 1},
wantStatus: "needs_attention",
wantReason: "resync_required_preflight_observed",
},
{
name: "repeated resync",
statusCounts: map[string]int64{"resync_required": 2},
severityCounts: map[string]int64{"warn": 2},
wantStatus: "repeated_resync_required",
wantReason: "resync_required_preflight_repeated",
},
{
name: "none observed",
statusCounts: map[string]int64{},
severityCounts: map[string]int64{},
wantStatus: "unknown",
wantReason: "no_preflight_observed",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
status := remoteWorkspacePreflightAttentionStatus(tt.statusCounts, tt.severityCounts)
if status != tt.wantStatus {
t.Fatalf("status=%q want %q", status, tt.wantStatus)
}
reason := remoteWorkspacePreflightAttentionReason(status, tt.statusCounts, tt.severityCounts)
if reason != tt.wantReason {
t.Fatalf("reason=%q want %q", reason, tt.wantReason)
}
})
}
}
func TestFabricServiceChannelVPNPacketIngressHonorsDisabledBackendRelayPolicy(t *testing.T) {
publicKey, privateKey, err := ed25519.GenerateKey(nil)
if err != nil {
t.Fatalf("generate key: %v", err)
}
var backendCalled bool
backend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
backendCalled = true
w.WriteHeader(http.StatusAccepted)
}))
defer backend.Close()
token := "rap_fsc_nobackend"
payload := fabricServiceChannelLeaseAuthorityPayload{
SchemaVersion: "rap.fabric_service_channel_lease_authority.v1",
ChannelID: "channel-1",
ClusterID: "cluster-1",
ResourceID: "vpn-1",
ServiceClass: FabricServiceClassVPNPackets,
SelectedEntryNodeID: "entry-1",
SelectedExitNodeID: "exit-1",
AllowedChannels: []string{ProductionChannelVPNPacket},
TokenHash: fabricServiceChannelTokenHash(token),
ExpiresAt: time.Now().UTC().Add(time.Minute),
DataPlane: fabricServiceChannelDataPlaneContract{
SchemaVersion: "rap.fabric_service_channel_data_plane.v1",
Mode: "fabric_primary",
WorkingDataTransport: "fabric_service_channel",
SteadyStateTransport: "fabric_route",
BackendRelayPolicy: "disabled",
ProductionForwardingRequired: true,
ServiceNeutral: true,
ProtocolAgnostic: true,
LogicalFlowMode: "multi_flow_isolated",
RequiredFlowIsolationClasses: []string{"control", ProductionChannelVPNPacket},
},
}
payload.PrimaryRoute.RouteID = "route-signed"
payload.PrimaryRoute.Status = "authorized"
rawPayload, err := json.Marshal(payload)
if err != nil {
t.Fatalf("marshal payload: %v", err)
}
canonical, err := authority.CanonicalJSON(rawPayload)
if err != nil {
t.Fatalf("canonical payload: %v", err)
}
signature := authority.Signature{
SchemaVersion: authority.SignatureSchemaVersion,
Algorithm: authority.AlgorithmEd25519,
KeyFingerprint: authority.Fingerprint(publicKey),
Signature: base64.StdEncoding.EncodeToString(ed25519.Sign(privateKey, canonical)),
}
rawSignature, err := json.Marshal(signature)
if err != nil {
t.Fatalf("marshal signature: %v", err)
}
server := httptest.NewServer(Server{
Local: PeerIdentity{ClusterID: "cluster-1", NodeID: "entry-1"},
VPNPacketIngress: failingVPNPacketIngress{sendErr: ErrRouteNotFound},
BackendProxyBaseURL: backend.URL + "/api/v1",
ClusterAuthorityPublicKey: base64.StdEncoding.EncodeToString(publicKey),
}.Handler())
defer server.Close()
req, err := http.NewRequest(http.MethodPost, server.URL+"/api/v1/clusters/cluster-1/fabric/service-channels/channel-1/vpn-connections/vpn-1/packets", bytes.NewReader([]byte("packet")))
if err != nil {
t.Fatalf("new request: %v", err)
}
req.Header.Set("X-RAP-Service-Channel-Token", token)
req.Header.Set("X-RAP-Service-Channel-Authority-Payload", base64.RawURLEncoding.EncodeToString(rawPayload))
req.Header.Set("X-RAP-Service-Channel-Authority-Signature", base64.RawURLEncoding.EncodeToString(rawSignature))
resp, err := http.DefaultClient.Do(req)
if err != nil {
t.Fatalf("post service channel packet: %v", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusServiceUnavailable {
t.Fatalf("status = %d, want %d", resp.StatusCode, http.StatusServiceUnavailable)
}
if backendCalled {
t.Fatal("backend relay was called despite disabled data-plane policy")
}
}
func TestFabricServiceChannelVPNPacketWebSocketHonorsDisabledBackendRelayPolicy(t *testing.T) {
publicKey, privateKey, err := ed25519.GenerateKey(nil)
if err != nil {
t.Fatalf("generate key: %v", err)
}
backendCalled := make(chan struct{}, 1)
backend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
select {
case backendCalled <- struct{}{}:
default:
}
w.WriteHeader(http.StatusAccepted)
}))
defer backend.Close()
token := "rap_fsc_nobackend_ws"
payload := fabricServiceChannelLeaseAuthorityPayload{
SchemaVersion: "rap.fabric_service_channel_lease_authority.v1",
ChannelID: "channel-1",
ClusterID: "cluster-1",
ResourceID: "vpn-1",
ServiceClass: FabricServiceClassVPNPackets,
SelectedEntryNodeID: "entry-1",
SelectedExitNodeID: "exit-1",
AllowedChannels: []string{ProductionChannelVPNPacket},
TokenHash: fabricServiceChannelTokenHash(token),
ExpiresAt: time.Now().UTC().Add(time.Minute),
DataPlane: fabricServiceChannelDataPlaneContract{
SchemaVersion: "rap.fabric_service_channel_data_plane.v1",
Mode: "fabric_primary",
WorkingDataTransport: "fabric_service_channel",
SteadyStateTransport: "fabric_route",
BackendRelayPolicy: "disabled",
ProductionForwardingRequired: true,
ServiceNeutral: true,
ProtocolAgnostic: true,
LogicalFlowMode: "multi_flow_isolated",
RequiredFlowIsolationClasses: []string{"control", ProductionChannelVPNPacket},
},
}
payload.PrimaryRoute.RouteID = "route-signed"
payload.PrimaryRoute.Status = "authorized"
rawPayload, err := json.Marshal(payload)
if err != nil {
t.Fatalf("marshal payload: %v", err)
}
canonical, err := authority.CanonicalJSON(rawPayload)
if err != nil {
t.Fatalf("canonical payload: %v", err)
}
signature := authority.Signature{
SchemaVersion: authority.SignatureSchemaVersion,
Algorithm: authority.AlgorithmEd25519,
KeyFingerprint: authority.Fingerprint(publicKey),
Signature: base64.StdEncoding.EncodeToString(ed25519.Sign(privateKey, canonical)),
}
rawSignature, err := json.Marshal(signature)
if err != nil {
t.Fatalf("marshal signature: %v", err)
}
violations := make(chan FabricServiceChannelAccessLogEntry, 2)
server := httptest.NewServer(Server{
Local: PeerIdentity{ClusterID: "cluster-1", NodeID: "entry-1"},
VPNPacketIngress: failingVPNPacketIngress{sendErr: ErrRouteNotFound},
BackendProxyBaseURL: backend.URL + "/api/v1",
ClusterAuthorityPublicKey: base64.StdEncoding.EncodeToString(publicKey),
FabricServiceChannelLogger: func(entry FabricServiceChannelAccessLogEntry) {
if entry.Event == "fabric_service_channel_data_plane_violation" {
violations <- entry
}
},
}.Handler())
defer server.Close()
wsURL := "ws" + strings.TrimPrefix(server.URL, "http") + "/api/v1/clusters/cluster-1/fabric/service-channels/channel-1/vpn-connections/vpn-1/packets/ws"
headers := http.Header{}
headers.Set("X-RAP-Service-Channel-Token", token)
headers.Set("X-RAP-Service-Channel-Authority-Payload", base64.RawURLEncoding.EncodeToString(rawPayload))
headers.Set("X-RAP-Service-Channel-Authority-Signature", base64.RawURLEncoding.EncodeToString(rawSignature))
conn, _, err := websocket.DefaultDialer.Dial(wsURL, headers)
if err != nil {
t.Fatalf("dial websocket: %v", err)
}
defer conn.Close()
if err := conn.WriteMessage(websocket.BinaryMessage, encodeVPNIngressPacketBatch([][]byte{[]byte("packet")})); err != nil {
t.Fatalf("write packet batch: %v", err)
}
select {
case entry := <-violations:
if entry.ViolationStatus != "fabric_route_send_failed_backend_fallback_blocked" ||
entry.BackendRelayPolicy != "disabled" ||
entry.ChannelID != "channel-1" ||
entry.ResourceID != "vpn-1" {
t.Fatalf("violation = %+v", entry)
}
case <-time.After(2 * time.Second):
t.Fatal("blocked fallback violation was not logged")
}
select {
case <-backendCalled:
t.Fatal("backend relay was called despite disabled data-plane policy")
default:
}
}
func TestFabricServiceChannelVPNPacketIngressRejectsSignedLeaseForDifferentEntry(t *testing.T) {
publicKey, privateKey, err := ed25519.GenerateKey(nil)
if err != nil {
t.Fatalf("generate key: %v", err)
}
token := "rap_fsc_signedtest"
payload := fabricServiceChannelLeaseAuthorityPayload{
SchemaVersion: "rap.fabric_service_channel_lease_authority.v1",
ChannelID: "channel-1",
ClusterID: "cluster-1",
ResourceID: "vpn-1",
ServiceClass: FabricServiceClassVPNPackets,
SelectedEntryNodeID: "other-entry",
SelectedExitNodeID: "exit-1",
AllowedChannels: []string{ProductionChannelVPNPacket},
TokenHash: fabricServiceChannelTokenHash(token),
ExpiresAt: time.Now().UTC().Add(time.Minute),
}
rawPayload, err := json.Marshal(payload)
if err != nil {
t.Fatalf("marshal payload: %v", err)
}
canonical, err := authority.CanonicalJSON(rawPayload)
if err != nil {
t.Fatalf("canonical payload: %v", err)
}
signature := authority.Signature{
SchemaVersion: authority.SignatureSchemaVersion,
Algorithm: authority.AlgorithmEd25519,
KeyFingerprint: authority.Fingerprint(publicKey),
Signature: base64.StdEncoding.EncodeToString(ed25519.Sign(privateKey, canonical)),
}
rawSignature, err := json.Marshal(signature)
if err != nil {
t.Fatalf("marshal signature: %v", err)
}
server := httptest.NewServer(Server{
Local: PeerIdentity{ClusterID: "cluster-1", NodeID: "entry-1"},
VPNPacketIngress: &recordingVPNPacketIngress{},
ClusterAuthorityPublicKey: base64.StdEncoding.EncodeToString(publicKey),
}.Handler())
defer server.Close()
req, err := http.NewRequest(http.MethodPost, server.URL+"/api/v1/clusters/cluster-1/fabric/service-channels/channel-1/vpn-connections/vpn-1/packets", bytes.NewReader([]byte("packet")))
if err != nil {
t.Fatalf("new request: %v", err)
}
req.Header.Set("X-RAP-Service-Channel-Token", token)
req.Header.Set("X-RAP-Service-Channel-Authority-Payload", base64.RawURLEncoding.EncodeToString(rawPayload))
req.Header.Set("X-RAP-Service-Channel-Authority-Signature", base64.RawURLEncoding.EncodeToString(rawSignature))
resp, err := http.DefaultClient.Do(req)
if err != nil {
t.Fatalf("post service channel packet: %v", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusForbidden {
t.Fatalf("status = %d, want %d", resp.StatusCode, http.StatusForbidden)
}
}
func TestFabricServiceChannelVPNPacketIngressFallsBackToBackendRelay(t *testing.T) {
var backendPath string
var backendBody []byte
backend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
backendPath = r.URL.Path
var err error
backendBody, err = io.ReadAll(r.Body)
if err != nil {
t.Fatalf("read backend body: %v", err)
}
w.WriteHeader(http.StatusAccepted)
}))
defer backend.Close()
server := httptest.NewServer(Server{
Local: PeerIdentity{ClusterID: "cluster-1", NodeID: "entry-1"},
VPNPacketIngress: failingVPNPacketIngress{sendErr: ErrRouteNotFound},
BackendProxyBaseURL: backend.URL + "/api/v1",
}.Handler())
defer server.Close()
req, err := http.NewRequest(http.MethodPost, server.URL+"/api/v1/clusters/cluster-1/fabric/service-channels/channel-1/vpn-connections/vpn-1/packets", bytes.NewReader([]byte("packet")))
if err != nil {
t.Fatalf("new request: %v", err)
}
req.Header.Set("X-RAP-Service-Channel-Token", "rap_fsc_testtoken")
resp, err := http.DefaultClient.Do(req)
if err != nil {
t.Fatalf("post service channel packet: %v", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusAccepted {
t.Fatalf("status = %d, want %d", resp.StatusCode, http.StatusAccepted)
}
if backendPath != "/api/v1/clusters/cluster-1/vpn-connections/vpn-1/tunnel/client/packets" {
t.Fatalf("backend path = %s", backendPath)
}
if string(backendBody) != "packet" {
t.Fatalf("backend body = %q", string(backendBody))
}
}
func TestFabricServiceChannelVPNPacketIngressUsesSignedDegradedFallback(t *testing.T) {
publicKey, privateKey, err := ed25519.GenerateKey(nil)
if err != nil {
t.Fatalf("generate key: %v", err)
}
token := "rap_fsc_degradedtest"
payload := fabricServiceChannelLeaseAuthorityPayload{
SchemaVersion: "rap.fabric_service_channel_lease_authority.v1",
ChannelID: "channel-1",
ClusterID: "cluster-1",
ResourceID: "vpn-1",
ServiceClass: FabricServiceClassVPNPackets,
Status: "degraded_fallback",
SelectedEntryNodeID: "entry-1",
SelectedExitNodeID: "exit-1",
AllowedChannels: []string{ProductionChannelVPNPacket},
TokenHash: fabricServiceChannelTokenHash(token),
ExpiresAt: time.Now().UTC().Add(time.Minute),
}
payload.PrimaryRoute.Status = "missing_route_intent"
rawPayload, err := json.Marshal(payload)
if err != nil {
t.Fatalf("marshal payload: %v", err)
}
canonical, err := authority.CanonicalJSON(rawPayload)
if err != nil {
t.Fatalf("canonical payload: %v", err)
}
signature := authority.Signature{
SchemaVersion: authority.SignatureSchemaVersion,
Algorithm: authority.AlgorithmEd25519,
KeyFingerprint: authority.Fingerprint(publicKey),
Signature: base64.StdEncoding.EncodeToString(ed25519.Sign(privateKey, canonical)),
}
rawSignature, err := json.Marshal(signature)
if err != nil {
t.Fatalf("marshal signature: %v", err)
}
var backendBody []byte
backend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path != "/api/v1/clusters/cluster-1/vpn-connections/vpn-1/tunnel/client/packets" {
t.Fatalf("backend path = %s", r.URL.Path)
}
var err error
backendBody, err = io.ReadAll(r.Body)
if err != nil {
t.Fatalf("read backend body: %v", err)
}
w.WriteHeader(http.StatusAccepted)
}))
defer backend.Close()
ingress := &recordingVPNPacketIngress{}
server := httptest.NewServer(Server{
Local: PeerIdentity{ClusterID: "cluster-1", NodeID: "entry-1"},
VPNPacketIngress: ingress,
BackendProxyBaseURL: backend.URL + "/api/v1",
ClusterAuthorityPublicKey: base64.StdEncoding.EncodeToString(publicKey),
}.Handler())
defer server.Close()
req, err := http.NewRequest(http.MethodPost, server.URL+"/api/v1/clusters/cluster-1/fabric/service-channels/channel-1/vpn-connections/vpn-1/packets", bytes.NewReader([]byte("packet")))
if err != nil {
t.Fatalf("new request: %v", err)
}
req.Header.Set("X-RAP-Service-Channel-Token", token)
req.Header.Set("X-RAP-Service-Channel-Authority-Payload", base64.RawURLEncoding.EncodeToString(rawPayload))
req.Header.Set("X-RAP-Service-Channel-Authority-Signature", base64.RawURLEncoding.EncodeToString(rawSignature))
resp, err := http.DefaultClient.Do(req)
if err != nil {
t.Fatalf("post service channel packet: %v", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusAccepted {
t.Fatalf("status = %d, want %d", resp.StatusCode, http.StatusAccepted)
}
if string(backendBody) != "packet" {
t.Fatalf("backend body = %q", string(backendBody))
}
ingress.mu.Lock()
defer ingress.mu.Unlock()
if len(ingress.sent) != 0 {
t.Fatalf("fabric ingress should not receive degraded fallback packets: %#v", ingress.sent)
}
}
func TestVPNPacketIngressWebSocketMovesBatchesBothDirections(t *testing.T) {
ingress := &recordingVPNPacketIngress{
receive: [][]byte{[]byte("reply-1"), []byte("reply-2")},
}
server := httptest.NewServer(Server{
Local: PeerIdentity{ClusterID: "cluster-1", NodeID: "entry-1"},
VPNPacketIngress: ingress,
}.Handler())
defer server.Close()
wsURL := "ws" + strings.TrimPrefix(server.URL, "http") + "/api/v1/clusters/cluster-1/vpn-connections/vpn-1/tunnel/client/packets/ws"
conn, _, err := websocket.DefaultDialer.Dial(wsURL, nil)
if err != nil {
t.Fatalf("dial websocket: %v", err)
}
defer conn.Close()
if err := conn.WriteMessage(websocket.BinaryMessage, encodeVPNIngressPacketBatch([][]byte{[]byte("packet-1"), []byte("packet-2")})); err != nil {
t.Fatalf("write packet batch: %v", err)
}
if err := conn.SetReadDeadline(time.Now().Add(2 * time.Second)); err != nil {
t.Fatalf("set read deadline: %v", err)
}
messageType, payload, err := conn.ReadMessage()
if err != nil {
t.Fatalf("read packet batch: %v", err)
}
if messageType != websocket.BinaryMessage {
t.Fatalf("message type = %d, want binary", messageType)
}
packets, err := decodeVPNIngressPacketBatch(payload)
if err != nil {
t.Fatalf("decode reply batch: %v", err)
}
if len(packets) != 2 || string(packets[0]) != "reply-1" || string(packets[1]) != "reply-2" {
t.Fatalf("reply packets = %#v", packets)
}
deadline := time.Now().Add(2 * time.Second)
for {
ingress.mu.Lock()
sent := append([][]byte(nil), ingress.sent...)
clusterID := ingress.clusterID
vpnConnectionID := ingress.vpnConnectionID
ingress.mu.Unlock()
if len(sent) == 2 {
if clusterID != "cluster-1" || vpnConnectionID != "vpn-1" {
t.Fatalf("ingress ids = %s %s", clusterID, vpnConnectionID)
}
if string(sent[0]) != "packet-1" || string(sent[1]) != "packet-2" {
t.Fatalf("sent packets = %#v", sent)
}
break
}
if time.Now().After(deadline) {
t.Fatalf("sent packets = %#v", sent)
}
time.Sleep(10 * time.Millisecond)
}
}
func TestFabricServiceChannelVPNPacketWebSocketPreservesTrafficClass(t *testing.T) {
ingress := &recordingVPNPacketIngress{
receive: [][]byte{[]byte("reply")},
}
server := httptest.NewServer(Server{
Local: PeerIdentity{ClusterID: "cluster-1", NodeID: "entry-1"},
VPNPacketIngress: ingress,
}.Handler())
defer server.Close()
wsURL := "ws" + strings.TrimPrefix(server.URL, "http") + "/api/v1/clusters/cluster-1/fabric/service-channels/channel-1/vpn-connections/vpn-1/packets/ws"
headers := http.Header{}
headers.Set("Authorization", "Bearer rap_fsc_testtoken")
headers.Set("X-RAP-Service-Class", FabricServiceClassVPNPackets)
headers.Set("X-RAP-Channel-Class", ProductionChannelVPNPacket)
headers.Set("X-RAP-Traffic-Class", "interactive")
headers.Set("X-RAP-Fabric-Channel-ID", "channel-1")
conn, _, err := websocket.DefaultDialer.Dial(wsURL, headers)
if err != nil {
t.Fatalf("dial websocket: %v", err)
}
defer conn.Close()
if err := conn.WriteMessage(websocket.BinaryMessage, encodeVPNIngressPacketBatch([][]byte{[]byte("packet")})); err != nil {
t.Fatalf("write packet batch: %v", err)
}
if err := conn.SetReadDeadline(time.Now().Add(2 * time.Second)); err != nil {
t.Fatalf("set read deadline: %v", err)
}
if _, _, err := conn.ReadMessage(); err != nil {
t.Fatalf("read packet batch: %v", err)
}
deadline := time.Now().Add(2 * time.Second)
for {
ingress.mu.Lock()
trafficClass := ingress.trafficClass
sent := append([][]byte(nil), ingress.sent...)
ingress.mu.Unlock()
if trafficClass == "interactive" && len(sent) == 1 && string(sent[0]) == "packet" {
break
}
if time.Now().After(deadline) {
t.Fatalf("traffic class = %q sent packets = %#v, want interactive packet", trafficClass, sent)
}
time.Sleep(10 * time.Millisecond)
}
}
func TestFabricServiceChannelVPNPacketWebSocketInfersInteractiveTrafficClass(t *testing.T) {
ingress := &recordingVPNPacketIngress{
receive: [][]byte{[]byte("reply")},
}
server := httptest.NewServer(Server{
Local: PeerIdentity{ClusterID: "cluster-1", NodeID: "entry-1"},
VPNPacketIngress: ingress,
}.Handler())
defer server.Close()
wsURL := "ws" + strings.TrimPrefix(server.URL, "http") + "/api/v1/clusters/cluster-1/fabric/service-channels/channel-1/vpn-connections/vpn-1/packets/ws"
headers := http.Header{}
headers.Set("Authorization", "Bearer rap_fsc_testtoken")
headers.Set("X-RAP-Service-Class", FabricServiceClassVPNPackets)
headers.Set("X-RAP-Channel-Class", ProductionChannelVPNPacket)
headers.Set("X-RAP-Fabric-Channel-ID", "channel-1")
conn, _, err := websocket.DefaultDialer.Dial(wsURL, headers)
if err != nil {
t.Fatalf("dial websocket: %v", err)
}
defer conn.Close()
packet := testVPNIPv4TCPPacket([4]byte{10, 77, 0, 2}, [4]byte{192, 168, 200, 95}, 51000, 3389, 0x02)
if err := conn.WriteMessage(websocket.BinaryMessage, encodeVPNIngressPacketBatch([][]byte{packet})); err != nil {
t.Fatalf("write packet batch: %v", err)
}
if err := conn.SetReadDeadline(time.Now().Add(2 * time.Second)); err != nil {
t.Fatalf("set read deadline: %v", err)
}
if _, _, err := conn.ReadMessage(); err != nil {
t.Fatalf("read packet batch: %v", err)
}
deadline := time.Now().Add(2 * time.Second)
for {
ingress.mu.Lock()
trafficClass := ingress.trafficClass
sent := append([][]byte(nil), ingress.sent...)
ingress.mu.Unlock()
if trafficClass == "interactive" && len(sent) == 1 {
break
}
if time.Now().After(deadline) {
t.Fatalf("traffic class = %q sent packets = %#v, want inferred interactive packet", trafficClass, sent)
}
time.Sleep(10 * time.Millisecond)
}
}
func TestVPNPacketIngressWebSocketFallsBackToBackendRelay(t *testing.T) {
var backendBody []byte
postSeen := make(chan struct{}, 1)
backend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch r.Method {
case http.MethodPost:
if r.URL.Path != "/api/v1/clusters/cluster-1/vpn-connections/vpn-1/tunnel/client/packets" || r.URL.Query().Get("batch") != "true" {
t.Fatalf("backend post target = %s?%s", r.URL.Path, r.URL.RawQuery)
}
var err error
backendBody, err = io.ReadAll(r.Body)
if err != nil {
t.Fatalf("read backend body: %v", err)
}
select {
case postSeen <- struct{}{}:
default:
}
w.WriteHeader(http.StatusAccepted)
case http.MethodGet:
if r.URL.Path != "/api/v1/clusters/cluster-1/vpn-connections/vpn-1/tunnel/client/packets" || r.URL.Query().Get("batch") != "true" {
t.Fatalf("backend get target = %s?%s", r.URL.Path, r.URL.RawQuery)
}
w.Header().Set("Content-Type", "application/vnd.rap.vpn-packet-batch.v1")
_, _ = w.Write(encodeVPNIngressPacketBatch([][]byte{[]byte("backend-reply")}))
default:
w.WriteHeader(http.StatusMethodNotAllowed)
}
}))
defer backend.Close()
server := httptest.NewServer(Server{
Local: PeerIdentity{ClusterID: "cluster-1", NodeID: "entry-1"},
VPNPacketIngress: failingVPNPacketIngress{sendErr: ErrRouteNotFound, receiveErr: ErrRouteNotFound},
BackendProxyBaseURL: backend.URL + "/api/v1",
}.Handler())
defer server.Close()
wsURL := "ws" + strings.TrimPrefix(server.URL, "http") + "/api/v1/clusters/cluster-1/vpn-connections/vpn-1/tunnel/client/packets/ws"
conn, _, err := websocket.DefaultDialer.Dial(wsURL, nil)
if err != nil {
t.Fatalf("dial websocket: %v", err)
}
defer conn.Close()
sentPayload := encodeVPNIngressPacketBatch([][]byte{[]byte("packet")})
if err := conn.WriteMessage(websocket.BinaryMessage, sentPayload); err != nil {
t.Fatalf("write packet batch: %v", err)
}
if err := conn.SetReadDeadline(time.Now().Add(2 * time.Second)); err != nil {
t.Fatalf("set read deadline: %v", err)
}
_, payload, err := conn.ReadMessage()
if err != nil {
t.Fatalf("read backend packet batch: %v", err)
}
packets, err := decodeVPNIngressPacketBatch(payload)
if err != nil {
t.Fatalf("decode backend batch: %v", err)
}
if len(packets) != 1 || string(packets[0]) != "backend-reply" {
t.Fatalf("backend reply packets = %#v", packets)
}
select {
case <-postSeen:
case <-time.After(2 * time.Second):
t.Fatal("backend POST was not observed")
}
if !bytes.Equal(backendBody, sentPayload) {
t.Fatalf("backend body = %q want %q", string(backendBody), string(sentPayload))
}
}
func TestNewProductionVPNPacketBatchEnvelopeRoundTripsPayload(t *testing.T) {
now := time.Now().UTC()
envelope, err := NewProductionVPNPacketBatchEnvelope(ProductionVPNPacketEnvelopeInput{
MessageID: "vpn-message-builder",
RouteID: "route-vpn-1",
ClusterID: "cluster-1",
SourceNodeID: "entry-1",
DestinationNodeID: "exit-1",
CurrentHopNodeID: "entry-1",
NextHopNodeID: "exit-1",
RoutePath: []string{"entry-1", "exit-1"},
VPNConnectionID: "vpn-1",
Direction: "client_to_gateway",
Packets: [][]byte{[]byte("packet-1"), []byte("packet-2")},
Now: now,
})
if err != nil {
t.Fatalf("new vpn packet envelope: %v", err)
}
if err := ValidateProductionEnvelope(PeerIdentity{ClusterID: "cluster-1", NodeID: "entry-1"}, envelope, now); err != nil {
t.Fatalf("validate envelope: %v", err)
}
payload, err := DecodeProductionVPNPacketBatch(envelope)
if err != nil {
t.Fatalf("decode vpn packet batch: %v", err)
}
if payload.VPNConnectionID != "vpn-1" || payload.Direction != "client_to_gateway" || string(payload.Packets[1]) != "packet-2" {
t.Fatalf("payload = %+v", payload)
}
}
func TestNewProductionVPNPacketBatchEnvelopeRejectsEmptyPackets(t *testing.T) {
_, err := NewProductionVPNPacketBatchEnvelope(ProductionVPNPacketEnvelopeInput{
MessageID: "vpn-message-builder",
RouteID: "route-vpn-1",
ClusterID: "cluster-1",
SourceNodeID: "entry-1",
DestinationNodeID: "exit-1",
CurrentHopNodeID: "entry-1",
NextHopNodeID: "exit-1",
RoutePath: []string{"entry-1", "exit-1"},
VPNConnectionID: "vpn-1",
Direction: "client_to_gateway",
Packets: [][]byte{nil, []byte{}},
})
if err == nil {
t.Fatal("expected empty packet batch to be rejected")
}
}
func TestEncodeVPNIngressPacketBatchSkipsEmptyPackets(t *testing.T) {
encoded := encodeVPNIngressPacketBatch([][]byte{nil, []byte("reply"), []byte{}})
decoded, err := decodeVPNIngressPacketBatch(encoded)
if err != nil {
t.Fatalf("decode batch: %v", err)
}
if len(decoded) != 1 || string(decoded[0]) != "reply" {
t.Fatalf("decoded = %#v", decoded)
}
}
type failingVPNPacketIngress struct {
sendErr error
receiveErr error
}
func (i failingVPNPacketIngress) SendClientPacketBatch(context.Context, string, string, [][]byte) error {
return i.sendErr
}
func (i failingVPNPacketIngress) ReceiveClientPacketBatch(context.Context, string, string, time.Duration) ([][]byte, error) {
return nil, i.receiveErr
}
type recordingVPNPacketIngress struct {
mu sync.Mutex
clusterID string
vpnConnectionID string
trafficClass string
sent [][]byte
receive [][]byte
}
func (i *recordingVPNPacketIngress) SendClientPacketBatch(_ context.Context, clusterID string, vpnConnectionID string, packets [][]byte) error {
i.mu.Lock()
defer i.mu.Unlock()
i.clusterID = clusterID
i.vpnConnectionID = vpnConnectionID
i.sent = cleanVPNIngressPacketBatch(packets)
return nil
}
func (i *recordingVPNPacketIngress) SendClientPacketBatchWithTrafficClass(_ context.Context, clusterID string, vpnConnectionID string, trafficClass string, packets [][]byte) error {
i.mu.Lock()
defer i.mu.Unlock()
i.clusterID = clusterID
i.vpnConnectionID = vpnConnectionID
i.trafficClass = trafficClass
i.sent = cleanVPNIngressPacketBatch(packets)
return nil
}
func (i *recordingVPNPacketIngress) ReceiveClientPacketBatch(_ context.Context, clusterID string, vpnConnectionID string, _ time.Duration) ([][]byte, error) {
i.mu.Lock()
defer i.mu.Unlock()
i.clusterID = clusterID
i.vpnConnectionID = vpnConnectionID
packets := i.receive
i.receive = nil
return packets, nil
}
func testVPNIPv4TCPPacket(src [4]byte, dst [4]byte, srcPort uint16, dstPort uint16, flags byte) []byte {
packet := make([]byte, 40)
packet[0] = 0x45
packet[2] = 0
packet[3] = 40
packet[8] = 64
packet[9] = 6
copy(packet[12:16], src[:])
copy(packet[16:20], dst[:])
packet[20] = byte(srcPort >> 8)
packet[21] = byte(srcPort)
packet[22] = byte(dstPort >> 8)
packet[23] = byte(dstPort)
packet[32] = 0x50
packet[33] = flags
return packet
}
func hasProductionForwardEvent(events []ProductionForwardLogEntry, event string) bool {
for _, item := range events {
if item.Event == event {
return true
}
}
return false
}
func TestSyntheticEndpointDisabledByDefault(t *testing.T) {
server := httptest.NewServer(Server{Local: PeerIdentity{ClusterID: "cluster-1", NodeID: "node-b"}}.Handler())
defer server.Close()
resp, err := http.Post(server.URL+"/mesh/v1/synthetic/probe", "application/json", bytes.NewReader([]byte(`{}`)))
if err != nil {
t.Fatalf("post synthetic probe: %v", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusServiceUnavailable {
t.Fatalf("status = %d, want %d", resp.StatusCode, http.StatusServiceUnavailable)
}
}