Stabilize VPN farm WebSocket dataplane

agents/rap-node-agent/internal/mesh/server.go

@@ -1002,6 +1002,35 @@ func isRetryableVPNPacketIngressError(err error) bool {
 		errors.Is(err, ErrSyntheticPeerUnavailable)
 }
 
+func (s Server) receiveVPNPacketWebSocketBatch(ctx context.Context, clusterID string, vpnConnectionID string, timeout time.Duration, retryRouteErrors bool) ([][]byte, error) {
+	const maxAttempts = 4
+	var lastErr error
+	for attempt := 0; attempt < maxAttempts; attempt++ {
+		if err := ctx.Err(); err != nil {
+			return nil, err
+		}
+		packets, err := s.VPNPacketIngress.ReceiveClientPacketBatch(ctx, clusterID, vpnConnectionID, timeout)
+		if err == nil {
+			return packets, nil
+		}
+		lastErr = err
+		if !retryRouteErrors || !isRetryableVPNPacketIngressError(err) {
+			return nil, err
+		}
+		timer := time.NewTimer(time.Duration(75+attempt*50) * time.Millisecond)
+		select {
+		case <-ctx.Done():
+			timer.Stop()
+			return nil, ctx.Err()
+		case <-timer.C:
+		}
+	}
+	if retryRouteErrors && isRetryableVPNPacketIngressError(lastErr) {
+		return nil, nil
+	}
+	return nil, lastErr
+}
+
 func (s Server) writeVPNPacketWebSocket(ctx context.Context, conn *websocket.Conn, clusterID string, channelID string, vpnConnectionID string, forceBackendFallback bool, backendFallbackAllowed bool, backendRelayPolicy string) error {
 	lastPing := time.Now()
 	for {
@@ -1013,7 +1042,7 @@ func (s Server) writeVPNPacketWebSocket(ctx context.Context, conn *websocket.Con
 		var packets [][]byte
 		var err error
 		if !forceBackendFallback {
-			packets, err = s.VPNPacketIngress.ReceiveClientPacketBatch(ctx, clusterID, vpnConnectionID, 50*time.Millisecond)
+			packets, err = s.receiveVPNPacketWebSocketBatch(ctx, clusterID, vpnConnectionID, 50*time.Millisecond, !backendFallbackAllowed)
 		}
 		if forceBackendFallback && !backendFallbackAllowed {
 			s.logFabricServiceChannelViolation(nil, clusterID, channelID, vpnConnectionID, backendRelayPolicy, "backend_fallback_blocked_by_policy", ErrRouteNotFound.Error())