package mesh import ( "context" "crypto/rand" "crypto/rsa" "crypto/sha256" "crypto/tls" "crypto/x509" "crypto/x509/pkix" "encoding/hex" "encoding/json" "encoding/pem" "math/big" "strings" "testing" "time" "github.com/example/remote-access-platform/agents/rap-node-agent/internal/fabricproto" "github.com/quic-go/quic-go" ) func TestQUICFabricTransportPingPong(t *testing.T) { listener := startQUICFabricEchoServer(t) defer listener.Close() ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) defer cancel() transport := NewQUICFabricTransport(&quic.Config{EnableDatagrams: true}) session, err := transport.Connect(ctx, FabricTransportTarget{ Endpoint: listener.Addr().String(), TLSConfig: &tls.Config{ InsecureSkipVerify: true, NextProtos: []string{fabricQUICNextProto}, }, Timeout: time.Second, InboundBuffer: 4, ErrorBuffer: 4, }) if err != nil { t.Fatalf("connect quic fabric: %v", err) } defer session.Close() if err := session.Send(ctx, fabricproto.Frame{Type: fabricproto.FramePing, Sequence: 42, Payload: []byte("quic")}); err != nil { t.Fatalf("send ping: %v", err) } select { case frame := <-session.Frames(): if frame.Type != fabricproto.FramePong || frame.Sequence != 42 || string(frame.Payload) != "quic" { t.Fatalf("frame = %+v", frame) } case err := <-session.Errors(): t.Fatalf("session error: %v", err) case <-ctx.Done(): t.Fatal(ctx.Err()) } } func TestQUICFabricTransportDataAck(t *testing.T) { listener := startQUICFabricEchoServer(t) defer listener.Close() ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) defer cancel() session, err := NewQUICFabricTransport(nil).Connect(ctx, FabricTransportTarget{ Endpoint: listener.Addr().String(), TLSConfig: &tls.Config{ InsecureSkipVerify: true, NextProtos: []string{fabricQUICNextProto}, }, Timeout: time.Second, InboundBuffer: 4, ErrorBuffer: 4, }) if err != nil { t.Fatalf("connect quic fabric: %v", err) } defer session.Close() if err := session.Send(ctx, fabricproto.Frame{ Type: fabricproto.FrameOpenStream, StreamID: 9, TrafficClass: fabricproto.TrafficClassInteractive, }); err != nil { t.Fatalf("open stream: %v", err) } if err := session.Send(ctx, fabricproto.Frame{ Type: fabricproto.FrameData, StreamID: 9, Sequence: 7, TrafficClass: fabricproto.TrafficClassInteractive, Payload: []byte("packet"), }); err != nil { t.Fatalf("send data: %v", err) } select { case frame := <-session.Frames(): if frame.Type != fabricproto.FrameAck || frame.StreamID != 9 || frame.Sequence != 7 { t.Fatalf("frame = %+v", frame) } case err := <-session.Errors(): t.Fatalf("session error: %v", err) case <-ctx.Done(): t.Fatal(ctx.Err()) } } func TestQUICFabricTransportVerifiesPinnedCertificate(t *testing.T) { tlsConfig := testQUICTLSConfig(t) listener := startQUICFabricEchoServerWithTLS(t, tlsConfig) defer listener.Close() ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) defer cancel() session, err := NewQUICFabricTransport(nil).Connect(ctx, FabricTransportTarget{ Endpoint: listener.Addr().String(), PeerCertSHA256: testQUICCertSHA256(t, tlsConfig), Timeout: time.Second, InboundBuffer: 4, ErrorBuffer: 4, }) if err != nil { t.Fatalf("connect quic fabric with pinned certificate: %v", err) } defer session.Close() if err := session.Send(ctx, fabricproto.Frame{Type: fabricproto.FramePing, Sequence: 43, Payload: []byte("pin")}); err != nil { t.Fatalf("send ping: %v", err) } select { case frame := <-session.Frames(): if frame.Type != fabricproto.FramePong || frame.Sequence != 43 || string(frame.Payload) != "pin" { t.Fatalf("frame = %+v", frame) } case err := <-session.Errors(): t.Fatalf("session error: %v", err) case <-ctx.Done(): t.Fatal(ctx.Err()) } } func TestQUICFabricTransportRejectsPinnedCertificateMismatch(t *testing.T) { listener := startQUICFabricEchoServer(t) defer listener.Close() ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) defer cancel() _, err := NewQUICFabricTransport(nil).Connect(ctx, FabricTransportTarget{ Endpoint: listener.Addr().String(), PeerCertSHA256: strings.Repeat("0", 64), Timeout: time.Second, }) if err == nil { t.Fatal("connect succeeded with mismatched certificate pin") } } func TestQUICFabricTransportReusesConnectionForPeerEndpoint(t *testing.T) { server, err := StartQUICFabricServer(context.Background(), QUICFabricServerConfig{ ListenAddr: "127.0.0.1:0", TLSConfig: testQUICTLSConfig(t), }) if err != nil { t.Fatalf("start quic fabric server: %v", err) } defer server.Close() ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) defer cancel() transport := NewQUICFabricTransport(nil) defer transport.Close() target := FabricTransportTarget{ PeerID: "node-b", Endpoint: server.Addr().String(), TLSConfig: &tls.Config{ InsecureSkipVerify: true, NextProtos: []string{fabricQUICNextProto}, }, Timeout: time.Second, InboundBuffer: 4, ErrorBuffer: 4, } first, err := transport.Connect(ctx, target) if err != nil { t.Fatalf("first connect: %v", err) } defer first.Close() second, err := transport.Connect(ctx, target) if err != nil { t.Fatalf("second connect: %v", err) } defer second.Close() snapshot := transport.Snapshot() if snapshot.ActiveCount != 1 || snapshot.Stats.Opens != 1 || snapshot.Stats.Reuses != 1 { t.Fatalf("unexpected quic transport snapshot: %+v", snapshot) } if len(snapshot.Connections) != 1 || snapshot.Connections[0].PeerID != "node-b" || snapshot.Connections[0].Endpoint != server.Addr().String() || snapshot.Connections[0].ActiveStreams != 2 || snapshot.Connections[0].MaxStreams != defaultQUICFabricMaxStreamsPerConn { t.Fatalf("unexpected quic connection snapshot: %+v", snapshot.Connections) } } func TestQUICFabricTransportPrunesIdleConnections(t *testing.T) { server, err := StartQUICFabricServer(context.Background(), QUICFabricServerConfig{ ListenAddr: "127.0.0.1:0", TLSConfig: testQUICTLSConfig(t), }) if err != nil { t.Fatalf("start quic fabric server: %v", err) } defer server.Close() ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) defer cancel() transport := NewQUICFabricTransport(nil) transport.IdleTTL = time.Nanosecond defer transport.Close() session, err := transport.Connect(ctx, FabricTransportTarget{ PeerID: "node-b", Endpoint: server.Addr().String(), TLSConfig: &tls.Config{ InsecureSkipVerify: true, NextProtos: []string{fabricQUICNextProto}, }, Timeout: time.Second, }) if err != nil { t.Fatalf("connect: %v", err) } if err := session.Close(); err != nil { t.Fatalf("close session: %v", err) } time.Sleep(time.Millisecond) snapshot := transport.Snapshot() if snapshot.ActiveCount != 0 || snapshot.Stats.IdleEvicted != 1 { t.Fatalf("idle connection was not pruned: %+v", snapshot) } } func TestQUICFabricTransportSnapshotPersistsClosedEvictions(t *testing.T) { server, err := StartQUICFabricServer(context.Background(), QUICFabricServerConfig{ ListenAddr: "127.0.0.1:0", TLSConfig: testQUICTLSConfig(t), }) if err != nil { t.Fatalf("start quic fabric server: %v", err) } defer server.Close() ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) defer cancel() transport := NewQUICFabricTransport(nil) defer transport.Close() target := FabricTransportTarget{ PeerID: "node-b", Endpoint: server.Addr().String(), TLSConfig: &tls.Config{ InsecureSkipVerify: true, NextProtos: []string{fabricQUICNextProto}, }, Timeout: time.Second, } session, err := transport.Connect(ctx, target) if err != nil { t.Fatalf("connect: %v", err) } defer session.Close() key := quicFabricConnKey(target) transport.mu.Lock() entry := transport.conns[key] transport.mu.Unlock() if entry == nil || entry.conn == nil { t.Fatalf("cached connection missing") } _ = entry.conn.CloseWithError(0, "test closed") <-entry.conn.Context().Done() first := transport.Snapshot() second := transport.Snapshot() if first.Stats.ClosedEvicted != 1 || second.Stats.ClosedEvicted != 1 { t.Fatalf("closed eviction stats were not persisted: first=%+v second=%+v", first, second) } } func TestQUICFabricTransportLimitsStreamsPerConnection(t *testing.T) { server, err := StartQUICFabricServer(context.Background(), QUICFabricServerConfig{ ListenAddr: "127.0.0.1:0", TLSConfig: testQUICTLSConfig(t), }) if err != nil { t.Fatalf("start quic fabric server: %v", err) } defer server.Close() ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) defer cancel() transport := NewQUICFabricTransport(nil) transport.MaxStreamsPerConn = 1 defer transport.Close() target := FabricTransportTarget{ PeerID: "node-b", Endpoint: server.Addr().String(), TLSConfig: &tls.Config{ InsecureSkipVerify: true, NextProtos: []string{fabricQUICNextProto}, }, Timeout: time.Second, } first, err := transport.Connect(ctx, target) if err != nil { t.Fatalf("first connect: %v", err) } if _, err := transport.Connect(ctx, target); err == nil { t.Fatal("second connect succeeded past stream limit") } snapshot := transport.Snapshot() if snapshot.ActiveStreams != 1 || snapshot.MaxStreamsPerConn != 1 || snapshot.SaturatedConnections != 1 || snapshot.CapacityPressurePercent != 100 || snapshot.Stats.StreamLimitRejects != 1 { t.Fatalf("unexpected stream limit snapshot: %+v", snapshot) } if len(snapshot.Connections) != 1 || !snapshot.Connections[0].Saturated || snapshot.Connections[0].CapacityPressurePercent != 100 { t.Fatalf("unexpected saturated connection snapshot: %+v", snapshot.Connections) } if err := first.Close(); err != nil { t.Fatalf("close first stream: %v", err) } second, err := transport.Connect(ctx, target) if err != nil { t.Fatalf("connect after release: %v", err) } defer second.Close() } func TestQUICFabricTransportReusesInboundConnectionForReverseStream(t *testing.T) { reverseTransport := NewQUICFabricTransport(nil) defer reverseTransport.Close() server, err := StartQUICFabricServer(context.Background(), QUICFabricServerConfig{ ListenAddr: "127.0.0.1:0", TLSConfig: testQUICTLSConfig(t), ReverseTransport: reverseTransport, SyntheticForwardHandler: func(_ context.Context, envelope SyntheticEnvelope) (SyntheticEnvelope, error) { envelope.To, envelope.From = envelope.From, PeerIdentity{ClusterID: envelope.ClusterID, NodeID: "node-r"} return envelope, nil }, }) if err != nil { t.Fatalf("start quic fabric server: %v", err) } defer server.Close() clientTransport := NewQUICFabricTransport(nil) defer clientTransport.Close() clientTransport.SetLocalPeerID("node-a") clientTransport.SetInboundHandlers(func(_ context.Context, envelope ProductionEnvelope) (ProductionForwardResult, error) { return ProductionForwardResult{ Accepted: true, Delivered: true, Forwarded: true, By: PeerIdentity{ClusterID: envelope.ClusterID, NodeID: "node-a"}, MessageID: envelope.MessageID, RouteID: envelope.RouteID, }, nil }, nil, nil) ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) defer cancel() session, err := clientTransport.Connect(ctx, FabricTransportTarget{ PeerID: "node-r", Endpoint: server.Addr().String(), TLSConfig: &tls.Config{ InsecureSkipVerify: true, NextProtos: []string{fabricQUICNextProto}, }, Timeout: time.Second, InboundBuffer: 4, ErrorBuffer: 4, }) if err != nil { t.Fatalf("client connect: %v", err) } defer session.Close() deadline := time.Now().Add(time.Second) for { if reverseTransport.Snapshot().Stats.ReverseRegisters > 0 { break } if time.Now().After(deadline) { t.Fatalf("reverse hello did not register connection: %+v", reverseTransport.Snapshot()) } time.Sleep(10 * time.Millisecond) } reverseSession, err := reverseTransport.Connect(ctx, FabricTransportTarget{ PeerID: "node-a", Endpoint: "10.0.0.2:19443", Transport: "relay_quic", Timeout: time.Second, InboundBuffer: 4, ErrorBuffer: 4, }) if err != nil { t.Fatalf("reverse connect: %v", err) } defer reverseSession.Close() productionPayload, err := json.Marshal(ProductionEnvelope{ FabricProtocolVersion: ProtocolVersion, MessageID: "msg-1", RouteID: "route-r-a", ClusterID: "cluster-1", SourceNodeID: "node-r", DestinationNodeID: "node-a", CurrentHopNodeID: "node-a", NextHopNodeID: "node-a", ChannelClass: ProductionChannelFabricControl, MessageType: ProductionMessageFabricControl, TTL: 4, CreatedAt: time.Now().UTC(), ExpiresAt: time.Now().UTC().Add(time.Minute), PayloadHash: "unused-by-test-handler", }) if err != nil { t.Fatalf("marshal production: %v", err) } if err := reverseSession.Send(ctx, fabricproto.Frame{Type: fabricproto.FrameData, TrafficClass: fabricproto.TrafficClassReliable, StreamID: ProductionForwardQUICStreamID, Sequence: 2, Payload: productionPayload}); err != nil { t.Fatalf("send reverse production: %v", err) } select { case frame := <-reverseSession.Frames(): var response quicProductionForwardResponse if err := json.Unmarshal(frame.Payload, &response); err != nil { t.Fatalf("decode response: %v", err) } if !response.Result.Accepted || !response.Result.Delivered || response.Result.By.NodeID != "node-a" { t.Fatalf("response = %+v", response) } case err := <-reverseSession.Errors(): t.Fatalf("reverse session error: %v", err) case <-ctx.Done(): t.Fatal(ctx.Err()) } snapshot := reverseTransport.Snapshot() if snapshot.Stats.ReverseRegisters == 0 || snapshot.Stats.ReverseReuses == 0 { t.Fatalf("reverse connection was not registered/reused: %+v", snapshot) } } func TestQUICFabricServerHandlesFabricFrames(t *testing.T) { var events []FabricSessionEventLogEntry server, err := StartQUICFabricServer(context.Background(), QUICFabricServerConfig{ ListenAddr: "127.0.0.1:0", TLSConfig: testQUICTLSConfig(t), Logger: func(entry FabricSessionEventLogEntry) { events = append(events, entry) }, }) if err != nil { t.Fatalf("start quic fabric server: %v", err) } defer server.Close() ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) defer cancel() session, err := NewQUICFabricTransport(nil).Connect(ctx, FabricTransportTarget{ Endpoint: server.Addr().String(), TLSConfig: &tls.Config{ InsecureSkipVerify: true, NextProtos: []string{fabricQUICNextProto}, }, Timeout: time.Second, InboundBuffer: 4, ErrorBuffer: 4, }) if err != nil { t.Fatalf("connect quic fabric: %v", err) } defer session.Close() if err := session.Send(ctx, fabricproto.Frame{Type: fabricproto.FramePing, Sequence: 77, Payload: []byte("server")}); err != nil { t.Fatalf("send ping: %v", err) } select { case frame := <-session.Frames(): if frame.Type != fabricproto.FramePong || frame.Sequence != 77 || string(frame.Payload) != "server" { t.Fatalf("frame = %+v", frame) } case err := <-session.Errors(): t.Fatalf("session error: %v", err) case <-ctx.Done(): t.Fatal(ctx.Err()) } if len(events) < 2 || events[0].Event != "fabric_session_quic_stream_opened" { t.Fatalf("events = %+v", events) } } func TestQUICFabricServerHandlesWebIngressForwardFrames(t *testing.T) { var received []byte server, err := StartQUICFabricServer(context.Background(), QUICFabricServerConfig{ ListenAddr: "127.0.0.1:0", TLSConfig: testQUICTLSConfig(t), WebIngressForwardHandler: func(_ context.Context, payload []byte) ([]byte, error) { received = append([]byte(nil), payload...) return []byte(`{"schema_version":"rap.web_ingress.fabric_runtime_response.v1","status_code":200,"body_b64":"b2s="}`), nil }, }) if err != nil { t.Fatalf("start quic fabric server: %v", err) } defer server.Close() ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) defer cancel() session, err := NewQUICFabricTransport(nil).Connect(ctx, FabricTransportTarget{ Endpoint: server.Addr().String(), TLSConfig: &tls.Config{ InsecureSkipVerify: true, NextProtos: []string{fabricQUICNextProto}, }, Timeout: time.Second, InboundBuffer: 4, ErrorBuffer: 4, }) if err != nil { t.Fatalf("connect quic fabric: %v", err) } defer session.Close() if err := session.Send(ctx, fabricproto.Frame{ Type: fabricproto.FrameData, TrafficClass: fabricproto.TrafficClassReliable, StreamID: WebIngressForwardQUICStreamID, Sequence: 44, Payload: []byte(`{"envelope":true}`), }); err != nil { t.Fatalf("send web ingress frame: %v", err) } select { case frame := <-session.Frames(): if frame.Type != fabricproto.FrameData || frame.StreamID != WebIngressForwardQUICStreamID || frame.Sequence != 44 { t.Fatalf("frame = %+v", frame) } var response quicWebIngressForwardResponse if err := json.Unmarshal(frame.Payload, &response); err != nil { t.Fatalf("decode response: %v", err) } if string(response.Payload) != `{"schema_version":"rap.web_ingress.fabric_runtime_response.v1","status_code":200,"body_b64":"b2s="}` || response.Error != "" { t.Fatalf("response = %+v", response) } case err := <-session.Errors(): t.Fatalf("session error: %v", err) case <-ctx.Done(): t.Fatal(ctx.Err()) } if string(received) != `{"envelope":true}` { t.Fatalf("received = %s", string(received)) } } func TestSendFabricControlForwardUsesQUICStream(t *testing.T) { tlsConfig := testQUICTLSConfig(t) server, err := StartQUICFabricServer(context.Background(), QUICFabricServerConfig{ ListenAddr: "127.0.0.1:0", TLSConfig: tlsConfig, FabricControlHandler: func(_ context.Context, payload []byte) ([]byte, error) { if string(payload) != `{"method":"GET","path":"/auth/login"}` { return nil, ErrForwardRuntimeUnavailable } return []byte(`{"status_code":200,"body":{"ok":true}}`), nil }, }) if err != nil { t.Fatalf("start quic fabric server: %v", err) } defer server.Close() ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) defer cancel() result, err := SendFabricControlForward(ctx, NewQUICFabricTransport(nil), FabricRegistryEndpoint{ EndpointID: "control-a", Address: "quic://" + server.Addr().String(), Transport: "direct_quic", PeerCertSHA256: testQUICCertSHA256(t, tlsConfig), }, []byte(`{"method":"GET","path":"/auth/login"}`), time.Second) if err != nil { t.Fatalf("send fabric control forward: %v", err) } var response quicFabricControlForwardResponse if err := json.Unmarshal(result.Payload, &response); err != nil { t.Fatalf("decode response: %v", err) } if response.Error != "" || string(response.Payload) != `{"status_code":200,"body":{"ok":true}}` { t.Fatalf("response = %+v", response) } } func startQUICFabricEchoServer(t *testing.T) *quic.Listener { t.Helper() return startQUICFabricEchoServerWithTLS(t, testQUICTLSConfig(t)) } func startQUICFabricEchoServerWithTLS(t *testing.T, tlsConfig *tls.Config) *quic.Listener { t.Helper() listener, err := quic.ListenAddr("127.0.0.1:0", tlsConfig, &quic.Config{EnableDatagrams: true}) if err != nil { t.Fatalf("listen quic: %v", err) } go func() { conn, err := listener.Accept(context.Background()) if err != nil { return } stream, err := conn.AcceptStream(context.Background()) if err != nil { _ = conn.CloseWithError(1, "accept stream failed") return } session := fabricproto.NewSession(fabricproto.SessionConfig{}) for { frame, err := fabricproto.ReadFrame(stream, fabricproto.DefaultMaxPayload) if err != nil { _ = conn.CloseWithError(0, "closed") return } _, responses, err := session.HandleFrame(frame) if err != nil { _ = conn.CloseWithError(2, err.Error()) return } for _, response := range responses { if err := fabricproto.WriteFrame(stream, response); err != nil { _ = conn.CloseWithError(3, err.Error()) return } } } }() return listener } func testQUICCertSHA256(t *testing.T, tlsConfig *tls.Config) string { t.Helper() if len(tlsConfig.Certificates) == 0 || len(tlsConfig.Certificates[0].Certificate) == 0 { t.Fatal("test tls config has no certificate") } sum := sha256.Sum256(tlsConfig.Certificates[0].Certificate[0]) return hex.EncodeToString(sum[:]) } func testQUICTLSConfig(t *testing.T) *tls.Config { t.Helper() key, err := rsa.GenerateKey(rand.Reader, 2048) if err != nil { t.Fatalf("generate key: %v", err) } template := x509.Certificate{ SerialNumber: big.NewInt(1), Subject: pkix.Name{CommonName: "rap-fabric-test"}, NotBefore: time.Now().Add(-time.Minute), NotAfter: time.Now().Add(time.Hour), KeyUsage: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature, ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, DNSNames: []string{"localhost"}, } certDER, err := x509.CreateCertificate(rand.Reader, &template, &template, &key.PublicKey, key) if err != nil { t.Fatalf("create certificate: %v", err) } keyDER := x509.MarshalPKCS1PrivateKey(key) cert, err := tls.X509KeyPair( pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: certDER}), pem.EncodeToMemory(&pem.Block{Type: "RSA PRIVATE KEY", Bytes: keyDER}), ) if err != nil { t.Fatalf("key pair: %v", err) } return &tls.Config{ Certificates: []tls.Certificate{cert}, NextProtos: []string{fabricQUICNextProto}, } }