This commit is contained in:
2026-05-18 21:33:39 +03:00
parent 5096155d83
commit 469fa0e860
94 changed files with 8761 additions and 8003 deletions
+74 -743
View File
@@ -20,7 +20,6 @@ import (
"github.com/example/remote-access-platform/agents/rap-node-agent/internal/authority"
"github.com/example/remote-access-platform/agents/rap-node-agent/internal/fabricproto"
"github.com/gorilla/websocket"
)
type ProductionEnvelopeObserver func(context.Context, ProductionEnvelopeObservation) error
@@ -55,6 +54,22 @@ type RemoteWorkspaceFrameSinkSessionMailboxConsumerResume interface {
type RemoteWorkspaceFrameSinkSessionMailboxPreflight interface {
PreflightAdapterSessionMailboxConsumerResume(adapterSessionID string, consumerID string, resumeFrom string, limit int, now time.Time) (RemoteWorkspaceAdapterMailboxPreflightSnapshot, error)
}
type FabricSessionEventLogEntry struct {
Event string `json:"event"`
ClusterID string `json:"cluster_id,omitempty"`
NodeID string `json:"node_id,omitempty"`
PeerID string `json:"peer_id,omitempty"`
AcceptedBy string `json:"accepted_by,omitempty"`
SessionID string `json:"session_id,omitempty"`
SessionEvent fabricproto.SessionEventType `json:"session_event,omitempty"`
StreamID uint64 `json:"stream_id,omitempty"`
Sequence uint64 `json:"sequence,omitempty"`
TrafficClass fabricproto.TrafficClass `json:"traffic_class,omitempty"`
RemoteAddr string `json:"remote_addr,omitempty"`
Reason string `json:"reason,omitempty"`
ObservedAt time.Time `json:"observed_at"`
}
type VPNPacketIngress interface {
SendClientPacketBatch(ctx context.Context, clusterID string, vpnConnectionID string, packets [][]byte) error
ReceiveClientPacketBatch(ctx context.Context, clusterID string, vpnConnectionID string, timeout time.Duration) ([][]byte, error)
@@ -69,24 +84,21 @@ type VPNPacketIngressRoutePreference interface {
}
type Server struct {
Local PeerIdentity
SyntheticRuntime *SyntheticRuntime
ProductionForwardingEnabled bool
ProductionEnvelopeObserver ProductionEnvelopeObserver
ProductionEnvelopeDelivery ProductionEnvelopeDelivery
ProductionForwardTransport ProductionForwardTransport
ProductionForwardLogger ProductionForwardLogger
DisableHTTPDataPlane bool
FabricServiceChannelLogger FabricServiceChannelAccessLogger
RemoteWorkspaceFrameSink RemoteWorkspaceFrameSink
ProductionRoutes []SyntheticRoute
VPNPacketIngress VPNPacketIngress
BackendProxyBaseURL string
ClusterAuthorityPublicKey string
ServiceChannelIntrospection bool
FabricSessionEnabled bool
FabricSessionWebSocketEnabled bool
FabricSessionLogger FabricSessionEventLogger
Local PeerIdentity
SyntheticRuntime *SyntheticRuntime
ProductionForwardingEnabled bool
ProductionEnvelopeObserver ProductionEnvelopeObserver
ProductionEnvelopeDelivery ProductionEnvelopeDelivery
ProductionForwardTransport ProductionForwardTransport
ProductionForwardLogger ProductionForwardLogger
DisableHTTPDataPlane bool
FabricServiceChannelLogger FabricServiceChannelAccessLogger
RemoteWorkspaceFrameSink RemoteWorkspaceFrameSink
ProductionRoutes []SyntheticRoute
VPNPacketIngress VPNPacketIngress
BackendProxyBaseURL string
ClusterAuthorityPublicKey string
ServiceChannelIntrospection bool
}
func (s Server) Handler() http.Handler {
@@ -94,9 +106,6 @@ func (s Server) Handler() http.Handler {
mux.HandleFunc("/mesh/v1/health", s.handleHealth)
mux.HandleFunc("/mesh/v1/forward", s.handleForward)
mux.HandleFunc("/mesh/v1/synthetic/probe", s.handleSyntheticProbe)
if s.FabricSessionEnabled && s.FabricSessionWebSocketEnabled {
mux.HandleFunc("/mesh/v1/fabric/session/ws", s.handleFabricSessionWebSocket)
}
if s.RemoteWorkspaceFrameSink != nil {
mux.HandleFunc("/mesh/v1/remote-workspace/adapter-sessions/", s.handleRemoteWorkspaceAdapterSessionControl)
}
@@ -196,185 +205,6 @@ func (s Server) handleRemoteWorkspaceAdapterSessionSnapshot(w http.ResponseWrite
_ = json.NewEncoder(w).Encode(snapshotter.SnapshotAdapterSessions(includeTerminal, limit, time.Now().UTC()))
}
type FabricSessionEventLogEntry struct {
Event string `json:"event"`
ClusterID string `json:"cluster_id,omitempty"`
NodeID string `json:"node_id,omitempty"`
PeerID string `json:"peer_id,omitempty"`
AcceptedBy string `json:"accepted_by,omitempty"`
SessionID string `json:"session_id,omitempty"`
SessionEvent fabricproto.SessionEventType `json:"session_event,omitempty"`
StreamID uint64 `json:"stream_id,omitempty"`
Sequence uint64 `json:"sequence,omitempty"`
TrafficClass fabricproto.TrafficClass `json:"traffic_class,omitempty"`
RemoteAddr string `json:"remote_addr,omitempty"`
Reason string `json:"reason,omitempty"`
ObservedAt time.Time `json:"observed_at"`
}
type fabricSessionAuthorityPayload struct {
SchemaVersion string `json:"schema_version"`
ClusterID string `json:"cluster_id"`
SessionID string `json:"session_id"`
SourceNodeID string `json:"source_node_id,omitempty"`
SelectedEntryNodeID string `json:"selected_entry_node_id,omitempty"`
TokenHash string `json:"token_hash"`
IssuedAt time.Time `json:"issued_at"`
ExpiresAt time.Time `json:"expires_at"`
}
type fabricSessionAuthDecision struct {
AcceptedBy string
SessionID string
}
func (s Server) handleFabricSessionWebSocket(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodGet {
w.WriteHeader(http.StatusMethodNotAllowed)
return
}
decision, ok := s.validateFabricSessionRequest(w, r)
if !ok {
return
}
upgrader := websocket.Upgrader{
CheckOrigin: func(_ *http.Request) bool { return true },
}
conn, err := upgrader.Upgrade(w, r, nil)
if err != nil {
return
}
defer conn.Close()
s.logFabricSession(FabricSessionEventLogEntry{
Event: "fabric_session_websocket_opened",
ClusterID: s.Local.ClusterID,
NodeID: s.Local.NodeID,
AcceptedBy: decision.AcceptedBy,
SessionID: decision.SessionID,
RemoteAddr: r.RemoteAddr,
ObservedAt: time.Now().UTC(),
})
loop := fabricproto.TransportLoop{
Session: fabricproto.NewSession(fabricproto.SessionConfig{}),
OnEvent: func(event fabricproto.SessionEvent) ([]fabricproto.Frame, error) {
s.logFabricSession(FabricSessionEventLogEntry{
Event: "fabric_session_event",
ClusterID: s.Local.ClusterID,
NodeID: s.Local.NodeID,
AcceptedBy: decision.AcceptedBy,
SessionID: decision.SessionID,
SessionEvent: event.Type,
StreamID: event.StreamID,
Sequence: event.Sequence,
TrafficClass: event.TrafficClass,
RemoteAddr: r.RemoteAddr,
ObservedAt: time.Now().UTC(),
})
return nil, nil
},
}
err = loop.RunWebSocket(r.Context(), conn, fabricproto.WebSocketTransportConfig{})
if err != nil && !errors.Is(err, context.Canceled) {
s.logFabricSession(FabricSessionEventLogEntry{
Event: "fabric_session_websocket_closed",
ClusterID: s.Local.ClusterID,
NodeID: s.Local.NodeID,
AcceptedBy: decision.AcceptedBy,
SessionID: decision.SessionID,
RemoteAddr: r.RemoteAddr,
Reason: err.Error(),
ObservedAt: time.Now().UTC(),
})
return
}
s.logFabricSession(FabricSessionEventLogEntry{
Event: "fabric_session_websocket_closed",
ClusterID: s.Local.ClusterID,
NodeID: s.Local.NodeID,
AcceptedBy: decision.AcceptedBy,
SessionID: decision.SessionID,
RemoteAddr: r.RemoteAddr,
ObservedAt: time.Now().UTC(),
})
}
func (s Server) validateFabricSessionRequest(w http.ResponseWriter, r *http.Request) (fabricSessionAuthDecision, bool) {
var decision fabricSessionAuthDecision
token := fabricSessionBearerToken(r)
if !strings.HasPrefix(token, "rap_fsn_") {
http.Error(w, "fabric session token is required", http.StatusUnauthorized)
return decision, false
}
payload, err := s.verifyFabricSessionAuthority(r, token)
if err != nil {
http.Error(w, err.Error(), http.StatusForbidden)
return decision, false
}
decision.AcceptedBy = "legacy_unsigned"
if payload != nil {
decision.AcceptedBy = "signed"
decision.SessionID = strings.TrimSpace(payload.SessionID)
}
return decision, true
}
func (s Server) verifyFabricSessionAuthority(r *http.Request, token string) (*fabricSessionAuthorityPayload, error) {
publicKey := strings.TrimSpace(s.ClusterAuthorityPublicKey)
payloadHeader := strings.TrimSpace(r.Header.Get("X-RAP-Fabric-Session-Authority-Payload"))
signatureHeader := strings.TrimSpace(r.Header.Get("X-RAP-Fabric-Session-Authority-Signature"))
if payloadHeader == "" && signatureHeader == "" {
if publicKey != "" {
return nil, fmt.Errorf("%w: signed fabric session authority is required", ErrUnauthorizedChannel)
}
return nil, nil
}
if publicKey == "" {
return nil, ErrUnauthorizedChannel
}
if payloadHeader == "" || signatureHeader == "" {
return nil, fmt.Errorf("%w: fabric session authority payload and signature are required together", ErrUnauthorizedChannel)
}
payloadRaw, err := decodeHeaderJSON(payloadHeader)
if err != nil {
return nil, fmt.Errorf("%w: invalid fabric session authority payload", ErrUnauthorizedChannel)
}
signatureRaw, err := decodeHeaderJSON(signatureHeader)
if err != nil {
return nil, fmt.Errorf("%w: invalid fabric session authority signature", ErrUnauthorizedChannel)
}
var signature authority.Signature
if err := json.Unmarshal(signatureRaw, &signature); err != nil {
return nil, fmt.Errorf("%w: invalid fabric session authority signature", ErrUnauthorizedChannel)
}
if err := authority.VerifyRaw(publicKey, payloadRaw, signature); err != nil {
return nil, fmt.Errorf("%w: fabric session authority signature rejected", ErrUnauthorizedChannel)
}
var payload fabricSessionAuthorityPayload
if err := json.Unmarshal(payloadRaw, &payload); err != nil {
return nil, fmt.Errorf("%w: invalid fabric session authority payload", ErrUnauthorizedChannel)
}
if payload.SchemaVersion != "rap.fabric_session_authority.v1" ||
payload.ClusterID != s.Local.ClusterID ||
payload.TokenHash != fabricSessionTokenHash(token) ||
strings.TrimSpace(payload.SessionID) == "" {
return nil, fmt.Errorf("%w: fabric session authority payload mismatch", ErrUnauthorizedChannel)
}
if payload.SelectedEntryNodeID != "" && s.Local.NodeID != "" && payload.SelectedEntryNodeID != s.Local.NodeID {
return nil, fmt.Errorf("%w: fabric session entry node mismatch", ErrUnauthorizedChannel)
}
if !payload.ExpiresAt.IsZero() && !payload.ExpiresAt.After(time.Now().UTC()) {
return nil, fmt.Errorf("%w: fabric session lease expired", ErrUnauthorizedChannel)
}
return &payload, nil
}
func (s Server) logFabricSession(entry FabricSessionEventLogEntry) {
if s.FabricSessionLogger != nil {
s.FabricSessionLogger(entry)
}
}
func (s Server) handleRemoteWorkspaceAdapterSessionMailbox(w http.ResponseWriter, r *http.Request) {
reader, ok := s.RemoteWorkspaceFrameSink.(RemoteWorkspaceFrameSinkSessionMailbox)
if !ok {
@@ -711,15 +541,15 @@ func parseRemoteWorkspaceAdapterSessionControlPath(path string) (string, bool) {
}
func (s Server) handleVPNPacketIngress(w http.ResponseWriter, r *http.Request) bool {
if clusterID, vpnConnectionID, ok := parseVPNClientPacketWebSocketPath(r.URL.Path); ok {
s.handleVPNPacketWebSocket(w, r, clusterID, "", vpnConnectionID, false, true, "")
if isVPNClientPacketWebSocketPath(r.URL.Path) {
http.Error(w, "legacy VPN WebSocket dataplane is removed; use QUIC fabric route", http.StatusGone)
return true
}
clusterID, vpnConnectionID, ok := parseVPNClientPacketPath(r.URL.Path)
if !ok {
if _, _, ok := parseVPNClientPacketPath(r.URL.Path); !ok {
return false
}
return s.handleVPNPacketHTTP(w, r, clusterID, "", vpnConnectionID, "", false, true, "")
http.Error(w, "legacy VPN HTTP dataplane is removed; use QUIC fabric route", http.StatusGone)
return true
}
func (s Server) handleFabricServiceChannelRemoteWorkspaceIngress(w http.ResponseWriter, r *http.Request) bool {
@@ -728,7 +558,7 @@ func (s Server) handleFabricServiceChannelRemoteWorkspaceIngress(w http.Response
return false
}
if webSocket {
http.Error(w, "remote workspace service-channel websocket forwarding is not implemented", http.StatusNotImplemented)
http.Error(w, "remote workspace service-channel websocket ingress is removed; use QUIC fabric route", http.StatusGone)
return true
}
decision, valid := s.validateFabricServiceChannelRequest(w, r, clusterID, channelID, resourceID, FabricServiceClassRemoteWorkspace, channelClass)
@@ -809,7 +639,7 @@ func (s Server) handleFabricServiceChannelRemoteWorkspaceIngress(w http.Response
"channel_id": channelID,
"resource_id": resourceID,
"data_plane": "validated",
"payload_flow": "not_implemented",
"payload_flow": "validated_only",
})
return true
}
@@ -898,7 +728,7 @@ func validateRemoteWorkspaceFrameBatchProbe(payload []byte, requiredChannelClass
return decoded, fmt.Errorf("unsupported remote workspace frame batch schema")
}
if !decoded.ProbeOnly {
return decoded, fmt.Errorf("remote workspace payload forwarding is not implemented")
return decoded, fmt.Errorf("remote workspace production payload forwarding is disabled; probe_only required")
}
if strings.TrimSpace(strings.ToLower(decoded.ServiceClass)) != FabricServiceClassRemoteWorkspace {
return decoded, fmt.Errorf("remote workspace frame batch service class mismatch")
@@ -952,438 +782,6 @@ func isAllowedRemoteWorkspaceAdapterFrameDirection(channel string, direction str
}
}
func (s Server) handleFabricServiceChannelVPNPacketIngress(w http.ResponseWriter, r *http.Request) bool {
if clusterID, channelID, vpnConnectionID, ok := parseFabricServiceChannelVPNPacketWebSocketPath(r.URL.Path); ok {
decision, valid := s.validateFabricServiceChannelVPNRequest(w, r, clusterID, channelID, vpnConnectionID)
if !valid {
return true
}
s.logFabricServiceChannelAccess(r, clusterID, channelID, vpnConnectionID, decision)
s.preferVPNPacketIngressRoute(decision.PreferredRouteID)
s.handleVPNPacketWebSocket(w, r, clusterID, channelID, vpnConnectionID, decision.ForceBackendFallback, decision.BackendFallbackAllowed(), decision.BackendRelayPolicy)
return true
}
clusterID, channelID, vpnConnectionID, ok := parseFabricServiceChannelVPNPacketPath(r.URL.Path)
if !ok {
return false
}
decision, valid := s.validateFabricServiceChannelVPNRequest(w, r, clusterID, channelID, vpnConnectionID)
if !valid {
return true
}
w.Header().Set("X-RAP-Service-Channel-Accepted-By", decision.AcceptedBy)
s.logFabricServiceChannelAccess(r, clusterID, channelID, vpnConnectionID, decision)
s.preferVPNPacketIngressRoute(decision.PreferredRouteID)
backendPath := "/api/v1/clusters/" + clusterID + "/vpn-connections/" + vpnConnectionID + "/tunnel/client/packets"
return s.handleVPNPacketHTTP(w, r, clusterID, channelID, vpnConnectionID, backendPath, decision.ForceBackendFallback, decision.BackendFallbackAllowed(), decision.BackendRelayPolicy)
}
func (s Server) preferVPNPacketIngressRoute(routeID string) {
routeID = strings.TrimSpace(routeID)
if routeID == "" || s.VPNPacketIngress == nil {
return
}
if preferred, ok := s.VPNPacketIngress.(VPNPacketIngressRoutePreference); ok {
preferred.PreferClientRoute(routeID)
}
}
func (s Server) handleVPNPacketHTTP(w http.ResponseWriter, r *http.Request, clusterID string, channelID string, vpnConnectionID string, backendFallbackPath string, forceBackendFallback bool, backendFallbackAllowed bool, backendRelayPolicy string) bool {
switch r.Method {
case http.MethodPost:
body, err := io.ReadAll(http.MaxBytesReader(w, r.Body, MaxProductionVPNPacketPayloadBytes))
if err != nil {
http.Error(w, "invalid vpn packet payload", http.StatusBadRequest)
return true
}
if r.URL.Query().Get("batch") != "true" && len(body) == 0 {
http.Error(w, "empty vpn packet payload", http.StatusBadRequest)
return true
}
packets := [][]byte{body}
if r.URL.Query().Get("batch") == "true" {
packets, err = decodeVPNIngressPacketBatch(body)
if err != nil {
http.Error(w, "invalid vpn packet batch", http.StatusBadRequest)
return true
}
}
packets = cleanVPNIngressPacketBatch(packets)
if len(packets) == 0 {
http.Error(w, "empty vpn packet batch", http.StatusBadRequest)
return true
}
if forceBackendFallback {
if backendFallbackAllowed && s.proxyVPNPacketIngressToBackendPath(w, r, body, backendFallbackPath) {
return true
}
s.logFabricServiceChannelViolation(r, clusterID, channelID, vpnConnectionID, backendRelayPolicy, "backend_fallback_blocked_by_policy", ErrRouteNotFound.Error())
http.Error(w, ErrRouteNotFound.Error(), vpnIngressStatusCode(ErrRouteNotFound))
return true
}
trafficClass := inferVPNPacketTrafficClass(r.Header.Get("X-RAP-Traffic-Class"), packets)
var sendErr error
if classIngress, ok := s.VPNPacketIngress.(VPNPacketIngressTrafficClass); ok {
sendErr = classIngress.SendClientPacketBatchWithTrafficClass(r.Context(), clusterID, vpnConnectionID, trafficClass, packets)
} else {
sendErr = s.VPNPacketIngress.SendClientPacketBatch(r.Context(), clusterID, vpnConnectionID, packets)
}
if sendErr != nil {
if backendFallbackAllowed && s.proxyVPNPacketIngressToBackendPath(w, r, body, backendFallbackPath) {
return true
}
s.logFabricServiceChannelViolation(r, clusterID, channelID, vpnConnectionID, backendRelayPolicy, "fabric_route_send_failed_backend_fallback_blocked", sendErr.Error())
http.Error(w, sendErr.Error(), vpnIngressStatusCode(sendErr))
return true
}
w.WriteHeader(http.StatusAccepted)
return true
case http.MethodGet:
if forceBackendFallback {
if backendFallbackAllowed && s.proxyVPNPacketIngressToBackendPath(w, r, nil, backendFallbackPath) {
return true
}
s.logFabricServiceChannelViolation(r, clusterID, channelID, vpnConnectionID, backendRelayPolicy, "backend_fallback_blocked_by_policy", ErrRouteNotFound.Error())
w.WriteHeader(http.StatusNoContent)
return true
}
timeout := vpnIngressTimeout(r)
packets, err := s.VPNPacketIngress.ReceiveClientPacketBatch(r.Context(), clusterID, vpnConnectionID, timeout)
if err != nil {
http.Error(w, err.Error(), vpnIngressStatusCode(err))
return true
}
packets = cleanVPNIngressPacketBatch(packets)
if len(packets) == 0 {
if backendFallbackAllowed && s.proxyVPNPacketIngressToBackendPath(w, r, nil, backendFallbackPath) {
return true
}
w.WriteHeader(http.StatusNoContent)
return true
}
if r.URL.Query().Get("batch") == "true" {
w.Header().Set("Content-Type", "application/vnd.rap.vpn-packet-batch.v1")
_, _ = w.Write(encodeVPNIngressPacketBatch(packets))
return true
}
w.Header().Set("Content-Type", "application/octet-stream")
_, _ = w.Write(packets[0])
return true
default:
w.WriteHeader(http.StatusMethodNotAllowed)
return true
}
}
func (s Server) handleVPNPacketWebSocket(w http.ResponseWriter, r *http.Request, clusterID string, channelID string, vpnConnectionID string, forceBackendFallback bool, backendFallbackAllowed bool, backendRelayPolicy string) {
if r.Method != http.MethodGet {
w.WriteHeader(http.StatusMethodNotAllowed)
return
}
if s.VPNPacketIngress == nil {
http.Error(w, ErrForwardRuntimeUnavailable.Error(), http.StatusServiceUnavailable)
return
}
upgrader := websocket.Upgrader{
CheckOrigin: func(_ *http.Request) bool { return true },
}
conn, err := upgrader.Upgrade(w, r, nil)
if err != nil {
return
}
defer conn.Close()
conn.SetReadLimit(MaxProductionVPNPacketPayloadBytes)
ctx, cancel := context.WithCancel(r.Context())
defer cancel()
trafficClass := r.Header.Get("X-RAP-Traffic-Class")
errCh := make(chan error, 2)
go func() {
errCh <- s.readVPNPacketWebSocket(ctx, conn, clusterID, channelID, vpnConnectionID, trafficClass, forceBackendFallback, backendFallbackAllowed, backendRelayPolicy)
}()
go func() {
errCh <- s.writeVPNPacketWebSocket(ctx, conn, clusterID, channelID, vpnConnectionID, forceBackendFallback, backendFallbackAllowed, backendRelayPolicy)
}()
select {
case <-ctx.Done():
case <-errCh:
cancel()
}
}
func (s Server) readVPNPacketWebSocket(ctx context.Context, conn *websocket.Conn, clusterID string, channelID string, vpnConnectionID string, trafficClass string, forceBackendFallback bool, backendFallbackAllowed bool, backendRelayPolicy string) error {
for {
messageType, payload, err := conn.ReadMessage()
if err != nil {
return err
}
if messageType != websocket.BinaryMessage {
continue
}
packets, err := decodeVPNIngressPacketBatch(payload)
if err != nil {
return err
}
packets = cleanVPNIngressPacketBatch(packets)
if len(packets) == 0 {
continue
}
if forceBackendFallback {
if !backendFallbackAllowed {
s.logFabricServiceChannelViolation(nil, clusterID, channelID, vpnConnectionID, backendRelayPolicy, "backend_fallback_blocked_by_policy", ErrRouteNotFound.Error())
return ErrRouteNotFound
}
if proxyErr := s.backendVPNPacketPost(ctx, clusterID, vpnConnectionID, payload); proxyErr != nil {
return proxyErr
}
continue
}
sendErr := s.sendVPNPacketWebSocketBatch(ctx, clusterID, vpnConnectionID, inferVPNPacketTrafficClass(trafficClass, packets), packets, !backendFallbackAllowed)
if sendErr != nil {
if !backendFallbackAllowed {
s.logFabricServiceChannelViolation(nil, clusterID, channelID, vpnConnectionID, backendRelayPolicy, "fabric_route_send_failed_backend_fallback_blocked", sendErr.Error())
if isRetryableVPNPacketIngressError(sendErr) {
continue
}
return sendErr
}
if proxyErr := s.backendVPNPacketPost(ctx, clusterID, vpnConnectionID, payload); proxyErr != nil {
return sendErr
}
}
}
}
func (s Server) sendVPNPacketWebSocketBatch(ctx context.Context, clusterID string, vpnConnectionID string, trafficClass string, packets [][]byte, retryRouteErrors bool) error {
const maxAttempts = 6
var lastErr error
for attempt := 0; attempt < maxAttempts; attempt++ {
if err := ctx.Err(); err != nil {
return err
}
var sendErr error
if classIngress, ok := s.VPNPacketIngress.(VPNPacketIngressTrafficClass); ok {
sendErr = classIngress.SendClientPacketBatchWithTrafficClass(ctx, clusterID, vpnConnectionID, trafficClass, packets)
} else {
sendErr = s.VPNPacketIngress.SendClientPacketBatch(ctx, clusterID, vpnConnectionID, packets)
}
if sendErr == nil {
return nil
}
lastErr = sendErr
if !retryRouteErrors || !isRetryableVPNPacketIngressError(sendErr) {
return sendErr
}
timer := time.NewTimer(time.Duration(75+attempt*50) * time.Millisecond)
select {
case <-ctx.Done():
timer.Stop()
return ctx.Err()
case <-timer.C:
}
}
return lastErr
}
func isRetryableVPNPacketIngressError(err error) bool {
return errors.Is(err, ErrRouteNotFound) ||
errors.Is(err, ErrForwardRuntimeUnavailable) ||
errors.Is(err, ErrForwardPeerUnavailable) ||
errors.Is(err, ErrSyntheticPeerUnavailable)
}
func (s Server) receiveVPNPacketWebSocketBatch(ctx context.Context, clusterID string, vpnConnectionID string, timeout time.Duration, retryRouteErrors bool) ([][]byte, error) {
const maxAttempts = 4
var lastErr error
for attempt := 0; attempt < maxAttempts; attempt++ {
if err := ctx.Err(); err != nil {
return nil, err
}
packets, err := s.VPNPacketIngress.ReceiveClientPacketBatch(ctx, clusterID, vpnConnectionID, timeout)
if err == nil {
return packets, nil
}
lastErr = err
if !retryRouteErrors || !isRetryableVPNPacketIngressError(err) {
return nil, err
}
timer := time.NewTimer(time.Duration(75+attempt*50) * time.Millisecond)
select {
case <-ctx.Done():
timer.Stop()
return nil, ctx.Err()
case <-timer.C:
}
}
if retryRouteErrors && isRetryableVPNPacketIngressError(lastErr) {
return nil, nil
}
return nil, lastErr
}
func (s Server) writeVPNPacketWebSocket(ctx context.Context, conn *websocket.Conn, clusterID string, channelID string, vpnConnectionID string, forceBackendFallback bool, backendFallbackAllowed bool, backendRelayPolicy string) error {
lastPing := time.Now()
for {
select {
case <-ctx.Done():
return ctx.Err()
default:
}
var packets [][]byte
var err error
if !forceBackendFallback {
packets, err = s.receiveVPNPacketWebSocketBatch(ctx, clusterID, vpnConnectionID, 50*time.Millisecond, !backendFallbackAllowed)
}
if forceBackendFallback && !backendFallbackAllowed {
s.logFabricServiceChannelViolation(nil, clusterID, channelID, vpnConnectionID, backendRelayPolicy, "backend_fallback_blocked_by_policy", ErrRouteNotFound.Error())
return ErrRouteNotFound
}
if err != nil && !backendFallbackAllowed {
s.logFabricServiceChannelViolation(nil, clusterID, channelID, vpnConnectionID, backendRelayPolicy, "fabric_route_receive_failed_backend_fallback_blocked", err.Error())
return err
}
if backendFallbackAllowed && (forceBackendFallback || err != nil || len(packets) == 0) {
backendPackets, proxyErr := s.backendVPNPacketGet(ctx, clusterID, vpnConnectionID, 50*time.Millisecond)
if proxyErr != nil && err != nil {
return err
}
if len(backendPackets) > 0 {
packets = backendPackets
}
}
if len(packets) > 0 {
if err := conn.SetWriteDeadline(time.Now().Add(5 * time.Second)); err != nil {
return err
}
if err := conn.WriteMessage(websocket.BinaryMessage, encodeVPNIngressPacketBatch(packets)); err != nil {
return err
}
continue
}
if time.Since(lastPing) >= 15*time.Second {
if err := conn.SetWriteDeadline(time.Now().Add(5 * time.Second)); err != nil {
return err
}
if err := conn.WriteMessage(websocket.PingMessage, []byte("rap-vpn")); err != nil {
return err
}
lastPing = time.Now()
}
}
}
func (s Server) backendVPNPacketPost(ctx context.Context, clusterID string, vpnConnectionID string, batchPayload []byte) error {
target := strings.TrimRight(strings.TrimSpace(s.BackendProxyBaseURL), "/")
if target == "" {
return ErrRouteNotFound
}
req, err := http.NewRequestWithContext(ctx, http.MethodPost, target+"/clusters/"+clusterID+"/vpn-connections/"+vpnConnectionID+"/tunnel/client/packets?batch=true", bytes.NewReader(batchPayload))
if err != nil {
return err
}
req.Header.Set("Content-Type", "application/octet-stream")
req.Header.Set("X-RAP-Entry-Node", s.Local.NodeID)
req.Header.Set("X-RAP-Entry-Cluster", s.Local.ClusterID)
resp, err := http.DefaultClient.Do(req)
if err != nil {
return err
}
defer resp.Body.Close()
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
return fmt.Errorf("backend vpn packet post failed: status=%d", resp.StatusCode)
}
return nil
}
func (s Server) backendVPNPacketGet(ctx context.Context, clusterID string, vpnConnectionID string, timeout time.Duration) ([][]byte, error) {
target := strings.TrimRight(strings.TrimSpace(s.BackendProxyBaseURL), "/")
if target == "" {
return nil, ErrRouteNotFound
}
if timeout <= 0 {
timeout = 50 * time.Millisecond
}
req, err := http.NewRequestWithContext(ctx, http.MethodGet, target+"/clusters/"+clusterID+"/vpn-connections/"+vpnConnectionID+"/tunnel/client/packets?batch=true&timeout_ms="+strconv.FormatInt(timeout.Milliseconds(), 10), nil)
if err != nil {
return nil, err
}
req.Header.Set("Accept", "application/vnd.rap.vpn-packet-batch.v1")
req.Header.Set("X-RAP-Entry-Node", s.Local.NodeID)
req.Header.Set("X-RAP-Entry-Cluster", s.Local.ClusterID)
resp, err := http.DefaultClient.Do(req)
if err != nil {
return nil, err
}
defer resp.Body.Close()
if resp.StatusCode == http.StatusNoContent {
return nil, nil
}
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
return nil, fmt.Errorf("backend vpn packet get failed: status=%d", resp.StatusCode)
}
body, err := io.ReadAll(io.LimitReader(resp.Body, MaxProductionVPNPacketPayloadBytes))
if err != nil {
return nil, err
}
if len(body) == 0 {
return nil, nil
}
return decodeVPNIngressPacketBatch(body)
}
func (s Server) proxyVPNPacketIngressToBackend(w http.ResponseWriter, r *http.Request, body []byte) bool {
return s.proxyVPNPacketIngressToBackendPath(w, r, body, "")
}
func (s Server) proxyVPNPacketIngressToBackendPath(w http.ResponseWriter, r *http.Request, body []byte, backendPath string) bool {
if strings.TrimSpace(s.BackendProxyBaseURL) == "" {
return false
}
target, err := url.Parse(s.BackendProxyBaseURL)
if err != nil || target.Scheme == "" || target.Host == "" {
return false
}
if strings.EqualFold(target.Host, r.Host) {
return false
}
var reader io.Reader
if body != nil {
reader = bytes.NewReader(body)
}
requestURI := r.URL.RequestURI()
if backendPath != "" {
requestURI = backendPath
if r.URL.RawQuery != "" {
requestURI += "?" + r.URL.RawQuery
}
}
req, err := http.NewRequestWithContext(r.Context(), r.Method, target.Scheme+"://"+target.Host+requestURI, reader)
if err != nil {
return false
}
for _, key := range []string{"Accept", "Content-Type"} {
if value := r.Header.Get(key); value != "" {
req.Header.Set(key, value)
}
}
req.Header.Set("X-RAP-Entry-Node", s.Local.NodeID)
req.Header.Set("X-RAP-Entry-Cluster", s.Local.ClusterID)
resp, err := http.DefaultClient.Do(req)
if err != nil {
return false
}
defer resp.Body.Close()
for _, key := range []string{"Content-Type"} {
if value := resp.Header.Get(key); value != "" {
w.Header().Set(key, value)
}
}
w.WriteHeader(resp.StatusCode)
_, _ = io.Copy(w, resp.Body)
return true
}
type fabricServiceChannelLeaseAuthorityPayload struct {
SchemaVersion string `json:"schema_version"`
ChannelID string `json:"channel_id"`
@@ -1443,10 +841,6 @@ func (d fabricServiceChannelRequestDecision) BackendFallbackAllowed() bool {
return strings.TrimSpace(d.BackendRelayPolicy) != "disabled"
}
func (s Server) validateFabricServiceChannelVPNRequest(w http.ResponseWriter, r *http.Request, clusterID string, channelID string, vpnConnectionID string) (fabricServiceChannelRequestDecision, bool) {
return s.validateFabricServiceChannelRequest(w, r, clusterID, channelID, vpnConnectionID, FabricServiceClassVPNPackets, ProductionChannelVPNPacket)
}
func (s Server) validateFabricServiceChannelRequest(w http.ResponseWriter, r *http.Request, clusterID string, channelID string, resourceID string, expectedServiceClass string, defaultChannelClass string) (fabricServiceChannelRequestDecision, bool) {
var decision fabricServiceChannelRequestDecision
expectedServiceClass = strings.TrimSpace(strings.ToLower(expectedServiceClass))
@@ -1485,7 +879,7 @@ func (s Server) validateFabricServiceChannelRequest(w http.ResponseWriter, r *ht
http.Error(w, err.Error(), http.StatusForbidden)
return decision, false
}
decision.AcceptedBy = "legacy_unsigned"
decision.AcceptedBy = "token_authorized"
decision.ServiceClass = serviceClass
decision.ChannelClass = channelClass
if payload != nil && (payload.Status == "degraded_fallback" || payload.PrimaryRoute.Status == "missing_route_intent") {
@@ -1571,30 +965,6 @@ func (s Server) logFabricServiceChannelAccess(r *http.Request, clusterID string,
s.FabricServiceChannelLogger(entry)
}
func (s Server) logFabricServiceChannelViolation(r *http.Request, clusterID string, channelID string, resourceID string, backendRelayPolicy string, status string, reason string) {
if s.FabricServiceChannelLogger == nil || strings.TrimSpace(channelID) == "" {
return
}
entry := FabricServiceChannelAccessLogEntry{
Event: "fabric_service_channel_data_plane_violation",
ClusterID: clusterID,
ChannelID: channelID,
ResourceID: resourceID,
LocalNodeID: s.Local.NodeID,
BackendRelayPolicy: strings.TrimSpace(backendRelayPolicy),
ViolationStatus: strings.TrimSpace(status),
ViolationReason: strings.TrimSpace(reason),
OccurredAt: time.Now().UTC(),
}
if r != nil {
entry.Method = r.Method
if r.URL != nil {
entry.Path = r.URL.Path
}
}
s.FabricServiceChannelLogger(entry)
}
func (s Server) verifyFabricServiceChannelLeaseAuthority(r *http.Request, clusterID string, channelID string, resourceID string, serviceClass string, channelClass string, token string) (*fabricServiceChannelLeaseAuthorityPayload, error) {
publicKey := strings.TrimSpace(s.ClusterAuthorityPublicKey)
payloadHeader := strings.TrimSpace(r.Header.Get("X-RAP-Service-Channel-Authority-Payload"))
@@ -1657,15 +1027,15 @@ func validateFabricServiceChannelDataPlaneContract(contract fabricServiceChannel
}
requiredFlowClass = strings.TrimSpace(strings.ToLower(requiredFlowClass))
if contract.SchemaVersion != "rap.fabric_service_channel_data_plane.v1" ||
contract.WorkingDataTransport != "fabric_service_channel" ||
contract.WorkingDataTransport != "fabric_quic_route" ||
contract.SteadyStateTransport != "fabric_route" ||
(contract.BackendRelayPolicy != "degraded_fallback_only" && contract.BackendRelayPolicy != "disabled") ||
contract.BackendRelayPolicy != "disabled" ||
!contract.ServiceNeutral ||
!contract.ProtocolAgnostic ||
contract.LogicalFlowMode != "multi_flow_isolated" {
return fmt.Errorf("%w: unsupported service channel data-plane contract", ErrUnauthorizedChannel)
}
if contract.Mode != "" && contract.Mode != "fabric_primary" && contract.Mode != "degraded_backend_fallback" {
if contract.Mode != "" && contract.Mode != "fabric_primary" && contract.Mode != "fabric_quic_only" {
return fmt.Errorf("%w: unsupported service channel data-plane mode", ErrUnauthorizedChannel)
}
if requiredFlowClass != "" && len(contract.RequiredFlowIsolationClasses) > 0 && !containsString(contract.RequiredFlowIsolationClasses, requiredFlowClass) {
@@ -1796,29 +1166,6 @@ func fabricServiceChannelBearerToken(r *http.Request) string {
return strings.TrimSpace(r.URL.Query().Get("service_channel_token"))
}
func fabricSessionTokenHash(token string) string {
sum := sha256.Sum256([]byte(strings.TrimSpace(token)))
return hex.EncodeToString(sum[:])
}
func fabricSessionBearerToken(r *http.Request) string {
if r == nil {
return ""
}
if token := strings.TrimSpace(r.Header.Get("X-RAP-Fabric-Session-Token")); token != "" {
return token
}
auth := strings.TrimSpace(r.Header.Get("Authorization"))
if len(auth) > len("Bearer ") && strings.EqualFold(auth[:len("Bearer ")], "Bearer ") {
return strings.TrimSpace(auth[len("Bearer "):])
}
return strings.TrimSpace(r.URL.Query().Get("fabric_session_token"))
}
func isAllowedFabricServiceVPNChannel(channel string) bool {
return isAllowedFabricServiceChannelForClass(FabricServiceClassVPNPackets, channel)
}
func isAllowedFabricServiceChannelForClass(serviceClass string, channel string) bool {
serviceClass = strings.TrimSpace(strings.ToLower(serviceClass))
channel = strings.TrimSpace(strings.ToLower(channel))
@@ -1846,25 +1193,6 @@ func containsString(values []string, target string) bool {
return false
}
func parseFabricServiceChannelVPNPacketWebSocketPath(path string) (string, string, string, bool) {
parts := strings.Split(strings.Trim(path, "/"), "/")
if len(parts) != 11 ||
parts[0] != "api" ||
parts[1] != "v1" ||
parts[2] != "clusters" ||
parts[4] != "fabric" ||
parts[5] != "service-channels" ||
parts[7] != "vpn-connections" ||
parts[9] != "packets" ||
parts[10] != "ws" {
return "", "", "", false
}
if parts[3] == "" || parts[6] == "" || parts[8] == "" {
return "", "", "", false
}
return parts[3], parts[6], parts[8], true
}
func parseFabricServiceChannelRemoteWorkspacePath(path string) (string, string, string, string, bool, bool) {
parts := strings.Split(strings.Trim(path, "/"), "/")
if len(parts) == 11 &&
@@ -1897,6 +1225,34 @@ func parseFabricServiceChannelRemoteWorkspacePath(path string) (string, string,
return parts[3], parts[6], parts[8], strings.TrimSpace(strings.ToLower(parts[10])), false, true
}
func (s Server) handleFabricServiceChannelVPNPacketIngress(w http.ResponseWriter, r *http.Request) bool {
if isFabricServiceChannelVPNPacketWebSocketPath(r.URL.Path) {
http.Error(w, "fabric service-channel WebSocket dataplane is removed; use QUIC fabric route", http.StatusGone)
return true
}
if _, _, _, ok := parseFabricServiceChannelVPNPacketPath(r.URL.Path); !ok {
return false
}
http.Error(w, "fabric service-channel HTTP dataplane is removed; use QUIC fabric route", http.StatusGone)
return true
}
func isFabricServiceChannelVPNPacketWebSocketPath(path string) bool {
parts := strings.Split(strings.Trim(path, "/"), "/")
if len(parts) != 11 ||
parts[0] != "api" ||
parts[1] != "v1" ||
parts[2] != "clusters" ||
parts[4] != "fabric" ||
parts[5] != "service-channels" ||
parts[7] != "vpn-connections" ||
parts[9] != "packets" ||
parts[10] != "ws" {
return false
}
return parts[3] != "" && parts[6] != "" && parts[8] != ""
}
func parseFabricServiceChannelVPNPacketPath(path string) (string, string, string, bool) {
parts := strings.Split(strings.Trim(path, "/"), "/")
if len(parts) != 10 ||
@@ -1915,7 +1271,7 @@ func parseFabricServiceChannelVPNPacketPath(path string) (string, string, string
return parts[3], parts[6], parts[8], true
}
func parseVPNClientPacketWebSocketPath(path string) (string, string, bool) {
func isVPNClientPacketWebSocketPath(path string) bool {
parts := strings.Split(strings.Trim(path, "/"), "/")
if len(parts) != 10 ||
parts[0] != "api" ||
@@ -1926,12 +1282,9 @@ func parseVPNClientPacketWebSocketPath(path string) (string, string, bool) {
parts[7] != "client" ||
parts[8] != "packets" ||
parts[9] != "ws" {
return "", "", false
return false
}
if parts[3] == "" || parts[5] == "" {
return "", "", false
}
return parts[3], parts[5], true
return parts[3] != "" && parts[5] != ""
}
func parseVPNClientPacketPath(path string) (string, string, bool) {
@@ -1952,28 +1305,6 @@ func parseVPNClientPacketPath(path string) (string, string, bool) {
return parts[3], parts[5], true
}
func vpnIngressTimeout(r *http.Request) time.Duration {
timeoutMs, _ := strconv.Atoi(r.URL.Query().Get("timeout_ms"))
if timeoutMs <= 0 {
timeoutMs = 25000
}
if timeoutMs > 30000 {
timeoutMs = 30000
}
return time.Duration(timeoutMs) * time.Millisecond
}
func vpnIngressStatusCode(err error) int {
switch err {
case ErrForwardRuntimeUnavailable, ErrRouteNotFound, ErrForwardPeerUnavailable:
return http.StatusServiceUnavailable
case ErrUnauthorizedChannel, ErrClusterMismatch, ErrNodeMismatch:
return http.StatusForbidden
default:
return http.StatusBadGateway
}
}
func encodeVPNIngressPacketBatch(packets [][]byte) []byte {
packets = cleanVPNIngressPacketBatch(packets)
total := 0