package main import ( "context" "crypto/rand" "crypto/rsa" "crypto/tls" "crypto/x509" "crypto/x509/pkix" "encoding/json" "fmt" "math/big" "os" "path/filepath" "time" "github.com/example/remote-access-platform/agents/rap-node-agent/internal/fabricproto" "github.com/example/remote-access-platform/agents/rap-node-agent/internal/mesh" "github.com/example/remote-access-platform/agents/rap-node-agent/internal/vpnruntime" ) type smokeNode struct { Local mesh.PeerIdentity Runtime *mesh.SyntheticRuntime Endpoint string } type smokeSyntheticTransport struct { peers map[string]*mesh.SyntheticRuntime } func (t smokeSyntheticTransport) SendSynthetic(ctx context.Context, nextNodeID string, envelope mesh.SyntheticEnvelope) (mesh.SyntheticEnvelope, error) { runtime := t.peers[nextNodeID] if runtime == nil { return mesh.SyntheticEnvelope{}, mesh.ErrSyntheticPeerUnavailable } return runtime.Receive(ctx, envelope) } type smokeReport struct { Stage string `json:"stage"` ProductionForwarding bool `json:"production_forwarding"` ScopedConfigLoaded bool `json:"scoped_config_loaded"` DirectProbeAccepted bool `json:"direct_probe_accepted"` DirectPath []string `json:"direct_path"` RelayProbeAccepted bool `json:"relay_probe_accepted"` RelayPath []string `json:"relay_path"` TestServiceAccepted bool `json:"test_service_accepted"` TestServiceEchoPayload string `json:"test_service_echo_payload"` FabricSessionAccepted bool `json:"fabric_session_accepted"` FabricSessionRoundTrips int `json:"fabric_session_round_trips"` FabricVPNPacketAccepted bool `json:"fabric_vpn_packet_accepted"` FabricVPNPacketSharded bool `json:"fabric_vpn_packet_sharded"` FabricVPNPacketFanout int `json:"fabric_vpn_packet_fanout"` FabricVPNBulkPressure bool `json:"fabric_vpn_bulk_pressure_active"` FabricVPNBulkChannels int `json:"fabric_vpn_bulk_pressure_channels"` FabricVPNInteractive int `json:"fabric_vpn_interactive_or_control_channels"` FabricVPNBulkWindow int `json:"fabric_vpn_bulk_parallel_window"` FabricVPNInteractiveWin int `json:"fabric_vpn_interactive_parallel_window"` FabricVPNPressureLevel string `json:"fabric_vpn_pressure_level"` FabricVPNPressureScore int `json:"fabric_vpn_pressure_score"` FabricVPNPressureReason []string `json:"fabric_vpn_pressure_reasons"` FabricVPNPressureAction string `json:"fabric_vpn_pressure_action"` FabricVPNRouteRecovered bool `json:"fabric_vpn_route_recovered"` FabricVPNRouteSwitches uint64 `json:"fabric_vpn_route_switch_count"` FabricVPNRecoveryMS int64 `json:"fabric_vpn_route_recovery_ms"` FabricVPNRecoveryMaxMS int64 `json:"fabric_vpn_route_recovery_max_ms"` FabricVPNRecoveryAvgMS int64 `json:"fabric_vpn_route_recovery_avg_ms"` FabricVPNRecoveryReason string `json:"fabric_vpn_route_recovery_reason"` FabricQUICAccepted bool `json:"fabric_quic_accepted"` FabricQUICEndpoint string `json:"fabric_quic_endpoint"` FabricQUICPressure int `json:"fabric_quic_capacity_pressure_percent"` FabricSessionLatencyMS int64 `json:"fabric_session_latency_ms"` FabricSessionEndpoint string `json:"fabric_session_endpoint"` PeerEndpoints map[string]any `json:"peer_endpoints"` } func main() { report, err := run(context.Background()) if err != nil { fmt.Fprintf(os.Stderr, "mesh live smoke failed: %v\n", err) os.Exit(1) } payload, err := json.MarshalIndent(report, "", " ") if err != nil { fmt.Fprintf(os.Stderr, "marshal report: %v\n", err) os.Exit(1) } fmt.Println(string(payload)) } func run(ctx context.Context) (smokeReport, error) { nodeA := newSmokeNode(mesh.PeerIdentity{ClusterID: "cluster-1", NodeID: "node-a"}) defer nodeA.Close() nodeR := newSmokeNode(mesh.PeerIdentity{ClusterID: "cluster-1", NodeID: "node-r"}) defer nodeR.Close() nodeB := newSmokeNode(mesh.PeerIdentity{ClusterID: "cluster-1", NodeID: "node-b"}) defer nodeB.Close() directRoute := smokeRoute("route-direct", []string{"node-a", "node-b"}) relayRoute := smokeRoute("route-relay", []string{"node-a", "node-r", "node-b"}) routes := []mesh.SyntheticRoute{directRoute, relayRoute} nodeAConfigPath, err := writeSmokeScopedConfig(nodeA.Local, map[string]string{ "node-r": nodeR.Endpoint, "node-b": nodeB.Endpoint, }, routes) if err != nil { return smokeReport{}, err } nodeAConfig, err := mesh.LoadScopedSyntheticConfig(nodeAConfigPath, nodeA.Local) if err != nil { return smokeReport{}, fmt.Errorf("load node-a scoped config: %w", err) } nodeA.Runtime = smokeRuntime(nodeA.Local, nodeAConfig.Routes, nodeAConfig.PeerEndpoints) nodeR.Runtime = smokeRuntime(nodeR.Local, routes, map[string]string{ "node-b": nodeB.Endpoint, }) nodeB.Runtime = smokeRuntime(nodeB.Local, routes, map[string]string{}) nodeA.Runtime = smokeRuntimeWithPeers(nodeA.Local, nodeAConfig.Routes, map[string]*mesh.SyntheticRuntime{ "node-r": nodeR.Runtime, "node-b": nodeB.Runtime, }) nodeR.Runtime = smokeRuntimeWithPeers(nodeR.Local, routes, map[string]*mesh.SyntheticRuntime{ "node-b": nodeB.Runtime, }) nodeB.Runtime = smokeRuntimeWithPeers(nodeB.Local, routes, map[string]*mesh.SyntheticRuntime{}) directAck, err := nodeA.Runtime.SendProbe(ctx, directRoute.RouteID, mesh.SyntheticChannelFabricControl, "smoke-direct") if err != nil { return smokeReport{}, fmt.Errorf("direct probe: %w", err) } relayAck, err := nodeA.Runtime.SendProbe(ctx, relayRoute.RouteID, mesh.SyntheticChannelFabricControl, "smoke-relay") if err != nil { return smokeReport{}, fmt.Errorf("relay probe: %w", err) } testService, err := nodeA.Runtime.SendTestService(ctx, relayRoute.RouteID, mesh.SyntheticChannelRouteControl, mesh.SyntheticTestServiceRequest{ RequestID: "smoke-test-service", OrganizationID: mesh.SyntheticDefaultTestOrganizationID, ServiceType: mesh.SyntheticTestServiceType, Payload: "hello-c17e", SentAt: time.Now().UTC(), }) if err != nil { return smokeReport{}, fmt.Errorf("test service: %w", err) } fabricSessionStartedAt := time.Now() fabricSession, fabricQUICEndpoint, fabricQUICPressure, err := smokeQUICFabricSession(ctx) if err != nil { return smokeReport{}, fmt.Errorf("fabric quic session open: %w", err) } defer fabricSession.Close() firstFabricSessionResponse, err := smokeFabricSessionRoundTrip(ctx, fabricSession, fabricproto.Frame{ Type: fabricproto.FramePing, Sequence: uint64(fabricSessionStartedAt.UnixNano()), Payload: []byte("mesh-live-smoke-fabric-session"), }) if err != nil { return smokeReport{}, fmt.Errorf("fabric session first round trip: %w", err) } secondFabricSessionResponse, err := smokeFabricSessionRoundTrip(ctx, fabricSession, fabricproto.Frame{ Type: fabricproto.FramePing, Sequence: uint64(fabricSessionStartedAt.UnixNano()) + 1, Payload: []byte("mesh-live-smoke-fabric-session-2"), }) if err != nil { return smokeReport{}, fmt.Errorf("fabric session second round trip: %w", err) } fabricSessionLatency := time.Since(fabricSessionStartedAt) fabricSessionAccepted := firstFabricSessionResponse.Type == fabricproto.FramePong && string(firstFabricSessionResponse.Payload) == "mesh-live-smoke-fabric-session" && secondFabricSessionResponse.Type == fabricproto.FramePong && string(secondFabricSessionResponse.Payload) == "mesh-live-smoke-fabric-session-2" fabricVPNPacketAccepted, fabricVPNPacketSharded, fabricVPNPacketFanout, err := smokeFabricVPNPacketOverSession(ctx, fabricSession) if err != nil { return smokeReport{}, fmt.Errorf("fabric vpn packet session smoke: %w", err) } fabricVPNBulkPressure, fabricVPNBulkChannels, fabricVPNInteractiveChannels, fabricVPNBulkWindow, fabricVPNInteractiveWindow, fabricVPNPressureLevel, fabricVPNPressureScore, fabricVPNPressureReasons, fabricVPNPressureAction := smokeVPNFlowSchedulerBulkPressure() fabricVPNRouteRecovered, fabricVPNRouteSwitches, fabricVPNRecoveryMS, fabricVPNRecoveryMaxMS, fabricVPNRecoveryAvgMS, fabricVPNRecoveryReason := smokeVPNFlowSchedulerRouteRecovery() return smokeReport{ Stage: "C17F scoped synthetic config plus live QUIC fabric transport", ProductionForwarding: false, ScopedConfigLoaded: nodeAConfig.ConfigVersion == "smoke-config-v1", DirectProbeAccepted: directAck.MessageType == mesh.SyntheticMessageProbeAck, DirectPath: decodeProbePath(directAck), RelayProbeAccepted: relayAck.MessageType == mesh.SyntheticMessageProbeAck, RelayPath: decodeProbePath(relayAck), TestServiceAccepted: testService.Ack.MessageType == mesh.SyntheticMessageTestServiceAck, TestServiceEchoPayload: testService.Response.EchoPayload, FabricSessionAccepted: fabricSessionAccepted, FabricSessionRoundTrips: 2, FabricVPNPacketAccepted: fabricVPNPacketAccepted, FabricVPNPacketSharded: fabricVPNPacketSharded, FabricVPNPacketFanout: fabricVPNPacketFanout, FabricVPNBulkPressure: fabricVPNBulkPressure, FabricVPNBulkChannels: fabricVPNBulkChannels, FabricVPNInteractive: fabricVPNInteractiveChannels, FabricVPNBulkWindow: fabricVPNBulkWindow, FabricVPNInteractiveWin: fabricVPNInteractiveWindow, FabricVPNPressureLevel: fabricVPNPressureLevel, FabricVPNPressureScore: fabricVPNPressureScore, FabricVPNPressureReason: fabricVPNPressureReasons, FabricVPNPressureAction: fabricVPNPressureAction, FabricVPNRouteRecovered: fabricVPNRouteRecovered, FabricVPNRouteSwitches: fabricVPNRouteSwitches, FabricVPNRecoveryMS: fabricVPNRecoveryMS, FabricVPNRecoveryMaxMS: fabricVPNRecoveryMaxMS, FabricVPNRecoveryAvgMS: fabricVPNRecoveryAvgMS, FabricVPNRecoveryReason: fabricVPNRecoveryReason, FabricQUICAccepted: fabricSessionAccepted, FabricQUICEndpoint: fabricQUICEndpoint, FabricQUICPressure: fabricQUICPressure, FabricSessionLatencyMS: fabricSessionLatency.Milliseconds(), FabricSessionEndpoint: "quic://" + fabricQUICEndpoint, PeerEndpoints: map[string]any{ "node-a": nodeA.Endpoint, "node-r": nodeR.Endpoint, "node-b": nodeB.Endpoint, }, }, nil } func smokeVPNFlowSchedulerBulkPressure() (bool, int, int, int, int, string, int, []string, string) { scheduler := vpnruntime.NewFabricFlowScheduler(32, 16) bulkPacket := []byte("bulk") interactivePacket := []byte("interactive-rdp-like") for i := 0; i < 16; i++ { scheduler.ScheduleClientPacketsForConnectionClass( fmt.Sprintf("vpn-bulk-%02d", i), vpnruntime.FabricTrafficClassBulk, [][]byte{bulkPacket}, ) } scheduler.ScheduleClientPacketsForConnectionClass( "vpn-interactive", vpnruntime.FabricTrafficClassInteractive, [][]byte{interactivePacket}, ) snapshot := scheduler.Snapshot() return snapshot.BulkPressureActive, snapshot.BulkPressureChannelCount, snapshot.InteractiveOrControlCount, snapshot.RecommendedParallelWindows[vpnruntime.FabricTrafficClassBulk], snapshot.RecommendedParallelWindows[vpnruntime.FabricTrafficClassInteractive], snapshot.PressureLevel, snapshot.PressureScore, snapshot.PressureReasons, snapshot.RecommendedAction } func smokeVPNFlowSchedulerRouteRecovery() (bool, uint64, int64, int64, int64, string) { scheduler := vpnruntime.NewFabricFlowScheduler(8, 16) channelID := "vpn-smoke-flow-0" scheduler.RecordRouteFailure(channelID, "route-primary", "node-primary", fmt.Errorf("smoke primary unavailable"), time.Millisecond) time.Sleep(time.Millisecond) scheduler.RecordRouteSuccess(channelID, "route-alternate", "node-alternate", time.Millisecond) snapshot := scheduler.Snapshot() stat := snapshot.ChannelStats[channelID] return stat.LastRecoveredFromRouteID == "route-primary" && stat.LastRouteID == "route-alternate" && snapshot.RouteRecoveredChannelCount == 1, snapshot.RouteSwitchCount, stat.LastRouteRecoveryMillis, snapshot.RouteRecoveryMaxMillis, snapshot.RouteRecoveryAvgMillis, stat.LastRouteSwitchReason } func smokeQUICFabricSession(ctx context.Context) (mesh.FabricTransportSession, string, int, error) { server, err := mesh.StartQUICFabricServer(ctx, mesh.QUICFabricServerConfig{ ListenAddr: "127.0.0.1:0", TLSConfig: smokeQUICTLSConfig(), }) if err != nil { return nil, "", 0, err } endpoint := server.Addr().String() transport := mesh.NewQUICFabricTransport(nil) session, err := transport.Connect(ctx, mesh.FabricTransportTarget{ PeerID: "node-b", Endpoint: endpoint, TLSConfig: &tls.Config{ InsecureSkipVerify: true, NextProtos: []string{"rap-fabric-data-session-v1"}, }, Timeout: 3 * time.Second, InboundBuffer: 4, ErrorBuffer: 4, }) if err != nil { _ = transport.Close() _ = server.Close() return nil, endpoint, 0, err } snapshot := transport.Snapshot() return &smokeManagedFabricSession{session: session, transport: transport, server: server}, endpoint, snapshot.CapacityPressurePercent, nil } func smokeQUICTLSConfig() *tls.Config { key, _ := rsa.GenerateKey(rand.Reader, 2048) template := x509.Certificate{ SerialNumber: big.NewInt(time.Now().UnixNano()), Subject: pkix.Name{CommonName: "mesh-live-smoke"}, NotBefore: time.Now().Add(-time.Minute), NotAfter: time.Now().Add(time.Hour), KeyUsage: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature, ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, DNSNames: []string{"localhost"}, } certDER, _ := x509.CreateCertificate(rand.Reader, &template, &template, &key.PublicKey, key) return &tls.Config{ Certificates: []tls.Certificate{{ Certificate: [][]byte{certDER}, PrivateKey: key, }}, NextProtos: []string{"rap-fabric-data-session-v1"}, } } func smokeFabricVPNPacketOverSession(ctx context.Context, fabricSession mesh.FabricTransportSession) (bool, bool, int, error) { const interactiveStreamID uint64 = 4400 const bulkStreamID uint64 = 4401 for _, frame := range []fabricproto.Frame{ {Type: fabricproto.FrameOpenStream, StreamID: interactiveStreamID, TrafficClass: fabricproto.TrafficClassInteractive}, {Type: fabricproto.FrameOpenStream, StreamID: bulkStreamID, TrafficClass: fabricproto.TrafficClassBulk}, } { if err := fabricSession.Send(ctx, frame); err != nil { return false, false, 0, err } } transport := &vpnruntime.FabricSessionPacketTransport{ Sender: fabricSession, Receiver: fabricSession, StreamID: interactiveStreamID, VPNConnectionID: "vpn-smoke", SendDirection: vpnruntime.FabricDirectionGatewayToClient, StreamIDsByTrafficClass: map[string][]uint64{ vpnruntime.FabricTrafficClassInteractive: []uint64{interactiveStreamID}, vpnruntime.FabricTrafficClassBulk: []uint64{bulkStreamID}, }, } bulkPacket := smokeIPv4TCPPacket([4]byte{10, 77, 0, 2}, [4]byte{192, 168, 200, 95}, 51000, 443, 0) controlPacket := smokeIPv4TCPPacket([4]byte{10, 77, 0, 2}, [4]byte{192, 168, 200, 95}, 51001, 3389, 0x02) if err := transport.SendGatewayPacketBatch(ctx, [][]byte{bulkPacket, controlPacket}); err != nil { return false, false, 0, err } timer := time.NewTimer(3 * time.Second) defer timer.Stop() acked := map[uint64]bool{} for { select { case frame := <-fabricSession.Frames(): if frame.Type == fabricproto.FrameAck && frame.Sequence == 1 { acked[frame.StreamID] = true if acked[interactiveStreamID] && acked[bulkStreamID] { snapshot := transport.Snapshot() framesByClass, _ := snapshot["send_frames_by_class"].(map[string]uint64) sharded := framesByClass[vpnruntime.FabricTrafficClassInteractive] == 1 && framesByClass[vpnruntime.FabricTrafficClassBulk] == 1 && snapshot["sharding_active"] == true && snapshot["send_class_count"] == 2 && snapshot["send_stream_count"] == 2 fanout, _ := snapshot["last_batch_frame_count"].(uint64) return true, sharded, int(fanout), nil } } case err := <-fabricSession.Errors(): return false, false, 0, err case <-timer.C: return false, false, 0, fmt.Errorf("timed out waiting for fabric vpn packet ack") case <-ctx.Done(): return false, false, 0, ctx.Err() } } } type smokeManagedFabricSession struct { session mesh.FabricTransportSession transport *mesh.QUICFabricTransport server *mesh.QUICFabricServer } func (s *smokeManagedFabricSession) Send(ctx context.Context, frame fabricproto.Frame) error { return s.session.Send(ctx, frame) } func (s *smokeManagedFabricSession) Frames() <-chan fabricproto.Frame { return s.session.Frames() } func (s *smokeManagedFabricSession) Errors() <-chan error { return s.session.Errors() } func (s *smokeManagedFabricSession) Closed() bool { return s.session.Closed() } func (s *smokeManagedFabricSession) Close() error { var firstErr error if s.session != nil { firstErr = s.session.Close() } if s.transport != nil { if err := s.transport.Close(); firstErr == nil { firstErr = err } } if s.server != nil { if err := s.server.Close(); firstErr == nil { firstErr = err } } return firstErr } func smokeFabricSessionRoundTrip(ctx context.Context, session mesh.FabricTransportSession, frame fabricproto.Frame) (fabricproto.Frame, error) { if err := session.Send(ctx, frame); err != nil { return fabricproto.Frame{}, err } timer := time.NewTimer(3 * time.Second) defer timer.Stop() for { select { case response := <-session.Frames(): if response.Sequence == frame.Sequence { return response, nil } case err := <-session.Errors(): return fabricproto.Frame{}, err case <-timer.C: return fabricproto.Frame{}, fmt.Errorf("timed out waiting for fabric session response") case <-ctx.Done(): return fabricproto.Frame{}, ctx.Err() } } } func smokeIPv4TCPPacket(src [4]byte, dst [4]byte, srcPort uint16, dstPort uint16, flags byte) []byte { packet := make([]byte, 40) packet[0] = 0x45 packet[2] = 0 packet[3] = 40 packet[8] = 64 packet[9] = 6 copy(packet[12:16], src[:]) copy(packet[16:20], dst[:]) packet[20] = byte(srcPort >> 8) packet[21] = byte(srcPort) packet[22] = byte(dstPort >> 8) packet[23] = byte(dstPort) packet[32] = 0x50 packet[33] = flags return packet } func writeSmokeScopedConfig(local mesh.PeerIdentity, peers map[string]string, routes []mesh.SyntheticRoute) (string, error) { path := filepath.Join(os.TempDir(), "rap-c17e-node-a-scoped-mesh.json") payload, err := json.Marshal(mesh.ScopedSyntheticConfig{ SchemaVersion: "c17f.synthetic.v1", ClusterID: local.ClusterID, LocalNodeID: local.NodeID, ConfigVersion: "smoke-config-v1", PeerDirectoryVersion: "smoke-peers-v1", PolicyVersion: "smoke-policy-v1", PeerEndpoints: peers, Routes: routes, }) if err != nil { return "", err } if err := os.WriteFile(path, payload, 0o600); err != nil { return "", err } return path, nil } func newSmokeNode(local mesh.PeerIdentity) *smokeNode { return &smokeNode{ Local: local, Endpoint: "quic://smoke-" + local.NodeID, } } func (n *smokeNode) Close() { } func smokeRuntime(local mesh.PeerIdentity, routes []mesh.SyntheticRoute, peers map[string]string) *mesh.SyntheticRuntime { _ = peers return smokeRuntimeWithPeers(local, routes, map[string]*mesh.SyntheticRuntime{}) } func smokeRuntimeWithPeers(local mesh.PeerIdentity, routes []mesh.SyntheticRoute, peers map[string]*mesh.SyntheticRuntime) *mesh.SyntheticRuntime { return mesh.NewSyntheticRuntime(mesh.SyntheticRuntimeConfig{ Enabled: true, Local: local, Routes: routes, AllowedChannels: []string{ mesh.SyntheticChannelFabricControl, mesh.SyntheticChannelRouteControl, }, Transport: smokeSyntheticTransport{peers: peers}, }) } func smokeRoute(routeID string, hops []string) mesh.SyntheticRoute { return mesh.SyntheticRoute{ RouteID: routeID, ClusterID: "cluster-1", SourceNodeID: hops[0], DestinationNodeID: hops[len(hops)-1], Hops: hops, AllowedChannels: []string{mesh.SyntheticChannelFabricControl, mesh.SyntheticChannelRouteControl}, MaxTTL: 8, MaxHops: 8, ExpiresAt: time.Now().UTC().Add(time.Hour), RouteVersion: "route-v1", PolicyVersion: "policy-v1", PeerDirectoryVersion: "peers-v1", } } func decodeProbePath(envelope mesh.SyntheticEnvelope) []string { var payload mesh.SyntheticProbeAckPayload _ = json.Unmarshal(envelope.Payload, &payload) return payload.Path }