324 lines
12 KiB
Go
324 lines
12 KiB
Go
package mesh
|
|
|
|
import (
|
|
"encoding/json"
|
|
"fmt"
|
|
"strings"
|
|
"time"
|
|
)
|
|
|
|
const (
|
|
FabricCandidateReachabilityPublic = "public"
|
|
FabricCandidateReachabilityPrivate = "private"
|
|
FabricCandidateReachabilityRelay = "relay"
|
|
FabricCandidateReachabilityOutboundOnly = "outbound_only"
|
|
|
|
FabricConnectivityDirect = "direct"
|
|
FabricConnectivityOutboundOnly = "outbound_only"
|
|
FabricConnectivityRelayRequired = "relay_required"
|
|
)
|
|
|
|
type FabricRoutePlannerConfig struct {
|
|
ClusterID string
|
|
LocalNodeID string
|
|
LocalSegmentID string
|
|
LocalNATGroupID string
|
|
DefaultCapacity int
|
|
RelayCapacity int
|
|
ReverseCapacity int
|
|
Observations map[string]EndpointCandidateHealthObservation
|
|
CapacityPressure map[string]EndpointCandidateCapacityPressure
|
|
Now time.Time
|
|
MaxObservationAge time.Duration
|
|
MaxCapacityPressureAge time.Duration
|
|
}
|
|
|
|
type FabricCandidateMetadata struct {
|
|
LocalSegmentID string `json:"local_segment_id,omitempty"`
|
|
NATGroupID string `json:"nat_group_id,omitempty"`
|
|
RelayNodeID string `json:"relay_node_id,omitempty"`
|
|
RelayEndpoint string `json:"relay_endpoint,omitempty"`
|
|
ViaNodeID string `json:"via_node_id,omitempty"`
|
|
STUNServer string `json:"stun_server,omitempty"`
|
|
ICEFoundation string `json:"ice_foundation,omitempty"`
|
|
}
|
|
|
|
func FabricRouteSetForPeerEndpointCandidates(targetNodeID string, candidates []PeerEndpointCandidate, cfg FabricRoutePlannerConfig) FabricRouteSet {
|
|
targetNodeID = strings.TrimSpace(targetNodeID)
|
|
if targetNodeID == "" && len(candidates) > 0 {
|
|
targetNodeID = strings.TrimSpace(candidates[0].NodeID)
|
|
}
|
|
routeSet := FabricRouteSet{TargetKind: FabricChannelTargetNode, TargetID: targetNodeID}
|
|
if len(candidates) == 0 {
|
|
return routeSet
|
|
}
|
|
now := cfg.Now
|
|
if now.IsZero() {
|
|
now = time.Now().UTC()
|
|
}
|
|
ranked := RankPeerEndpointCandidates(candidates, EndpointCandidateScoreOptions{
|
|
Now: now,
|
|
Observations: cfg.Observations,
|
|
MaxObservationAge: firstNonZeroDuration(cfg.MaxObservationAge, 30*time.Second),
|
|
CapacityPressure: cfg.CapacityPressure,
|
|
MaxCapacityPressureAge: firstNonZeroDuration(cfg.MaxCapacityPressureAge, 10*time.Second),
|
|
})
|
|
routes := make([]FabricRoute, 0, len(ranked))
|
|
for index, scored := range ranked {
|
|
route, ok := fabricRouteForPeerEndpointCandidate(scored.Candidate, cfg, scored.Score, index, now)
|
|
if ok {
|
|
routes = append(routes, route)
|
|
}
|
|
}
|
|
return routeSetFromRoutes(routeSet, routes)
|
|
}
|
|
|
|
func FabricRouteSetsForPeerEndpointCandidates(candidatesByNode map[string][]PeerEndpointCandidate, cfg FabricRoutePlannerConfig) map[string]FabricRouteSet {
|
|
out := make(map[string]FabricRouteSet, len(candidatesByNode))
|
|
for nodeID, candidates := range candidatesByNode {
|
|
nodeID = strings.TrimSpace(nodeID)
|
|
if nodeID == "" {
|
|
continue
|
|
}
|
|
routeSet := FabricRouteSetForPeerEndpointCandidates(nodeID, candidates, cfg)
|
|
if strings.TrimSpace(routeSet.Primary.RouteID) != "" || len(routeSet.WarmStandby) > 0 || len(routeSet.ColdFallbacks) > 0 {
|
|
out[nodeID] = routeSet
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
func fabricRouteForPeerEndpointCandidate(candidate PeerEndpointCandidate, cfg FabricRoutePlannerConfig, score int, index int, now time.Time) (FabricRoute, bool) {
|
|
candidate.EndpointID = strings.TrimSpace(candidate.EndpointID)
|
|
candidate.NodeID = strings.TrimSpace(candidate.NodeID)
|
|
candidate.Address = strings.TrimRight(strings.TrimSpace(candidate.Address), "/")
|
|
if candidate.EndpointID == "" || candidate.NodeID == "" || candidate.Address == "" || !isQUICOnlyCandidateTransport(candidate.Transport) {
|
|
return FabricRoute{}, false
|
|
}
|
|
metadata := decodeFabricCandidateMetadata(candidate.Metadata)
|
|
mode := fabricRouteModeForPeerEndpointCandidate(candidate, metadata, cfg)
|
|
hops := fabricRouteHopsForCandidate(candidate, metadata, mode, cfg)
|
|
if len(hops) == 0 {
|
|
return FabricRoute{}, false
|
|
}
|
|
relayCount := 0
|
|
for _, hop := range hops {
|
|
if hop.Mode == FabricRouteRelay {
|
|
relayCount++
|
|
}
|
|
}
|
|
latency := fabricRouteLatencyFromCandidate(candidate, cfg, score, index)
|
|
capacity := fabricRouteCapacityForMode(mode, cfg)
|
|
if capacity <= 0 {
|
|
capacity = 100
|
|
}
|
|
healthy := true
|
|
degraded := false
|
|
if observation, ok := cfg.Observations[candidate.EndpointID]; ok {
|
|
healthy = observation.ReliabilityScore == 0 || observation.ReliabilityScore >= 50
|
|
degraded = observation.LastLatencyMs > 0 && observation.LastLatencyMs >= 250
|
|
}
|
|
return FabricRoute{
|
|
RouteID: candidate.EndpointID,
|
|
ClusterID: strings.TrimSpace(cfg.ClusterID),
|
|
SourceNodeID: strings.TrimSpace(cfg.LocalNodeID),
|
|
DestinationNodeID: candidate.NodeID,
|
|
Hops: hops,
|
|
BaseLatencyMs: latency,
|
|
Capacity: capacity,
|
|
ActiveChannels: int(candidatePressureCount(candidate.EndpointID, cfg)),
|
|
RelayCount: relayCount,
|
|
Healthy: healthy,
|
|
Degraded: degraded,
|
|
LastUpdatedAt: now,
|
|
}, true
|
|
}
|
|
|
|
func fabricRouteModeForPeerEndpointCandidate(candidate PeerEndpointCandidate, metadata FabricCandidateMetadata, cfg FabricRoutePlannerConfig) FabricRouteMode {
|
|
transportMode := fabricRouteModeForTransportTarget(FabricTransportTarget{Transport: candidate.Transport})
|
|
if transportMode == FabricRouteRelay || transportMode == FabricRouteReverse || transportMode == FabricRouteICE || transportMode == FabricRouteLAN {
|
|
return transportMode
|
|
}
|
|
reachability := strings.ToLower(strings.TrimSpace(candidate.Reachability))
|
|
connectivity := strings.ToLower(strings.TrimSpace(candidate.ConnectivityMode))
|
|
if sameLocalSegment(metadata, cfg) || sameNATGroup(metadata, cfg) {
|
|
return FabricRouteLAN
|
|
}
|
|
if reachability == FabricCandidateReachabilityRelay || connectivity == FabricConnectivityRelayRequired || strings.TrimSpace(metadata.RelayEndpoint) != "" {
|
|
return FabricRouteRelay
|
|
}
|
|
if connectivity == FabricConnectivityOutboundOnly || reachability == FabricCandidateReachabilityOutboundOnly {
|
|
return FabricRouteReverse
|
|
}
|
|
if strings.TrimSpace(metadata.STUNServer) != "" || strings.TrimSpace(metadata.ICEFoundation) != "" || candidate.NATType != "" {
|
|
return FabricRouteICE
|
|
}
|
|
return FabricRouteDirect
|
|
}
|
|
|
|
func fabricRouteHopsForCandidate(candidate PeerEndpointCandidate, metadata FabricCandidateMetadata, mode FabricRouteMode, cfg FabricRoutePlannerConfig) []FabricRouteHop {
|
|
localNodeID := strings.TrimSpace(cfg.LocalNodeID)
|
|
targetNodeID := strings.TrimSpace(candidate.NodeID)
|
|
endpoint := strings.TrimRight(strings.TrimSpace(candidate.Address), "/")
|
|
switch mode {
|
|
case FabricRouteRelay:
|
|
relayNodeID := firstNonEmpty(strings.TrimSpace(metadata.RelayNodeID), strings.TrimSpace(metadata.ViaNodeID))
|
|
relayEndpoint := firstNonEmpty(strings.TrimRight(strings.TrimSpace(metadata.RelayEndpoint), "/"), endpoint)
|
|
relayPeerCertSHA256 := candidatePeerCertSHA256(candidate)
|
|
hops := []FabricRouteHop{}
|
|
if localNodeID != "" {
|
|
hops = append(hops, FabricRouteHop{NodeID: localNodeID, Mode: FabricRouteDirect})
|
|
}
|
|
if relayNodeID == "" {
|
|
hops = append(hops, FabricRouteHop{NodeID: targetNodeID, Mode: FabricRouteRelay, EndpointID: candidate.EndpointID, Address: endpoint, PeerCertSHA256: candidatePeerCertSHA256(candidate)})
|
|
return hops
|
|
}
|
|
hops = append(hops,
|
|
FabricRouteHop{NodeID: relayNodeID, Mode: FabricRouteRelay, EndpointID: candidate.EndpointID + ":relay", Address: relayEndpoint, PeerCertSHA256: relayPeerCertSHA256},
|
|
FabricRouteHop{NodeID: targetNodeID, Mode: FabricRouteRelay, EndpointID: candidate.EndpointID, Address: endpoint, PeerCertSHA256: candidatePeerCertSHA256(candidate)},
|
|
)
|
|
return hops
|
|
case FabricRouteLAN, FabricRouteICE, FabricRouteReverse, FabricRouteDirect:
|
|
hops := []FabricRouteHop{}
|
|
if localNodeID != "" {
|
|
hops = append(hops, FabricRouteHop{NodeID: localNodeID, Mode: mode})
|
|
}
|
|
hops = append(hops, FabricRouteHop{NodeID: targetNodeID, Mode: mode, EndpointID: candidate.EndpointID, Address: endpoint, PeerCertSHA256: candidatePeerCertSHA256(candidate)})
|
|
return hops
|
|
default:
|
|
return nil
|
|
}
|
|
}
|
|
|
|
func isQUICOnlyCandidateTransport(transport string) bool {
|
|
switch strings.ToLower(strings.TrimSpace(transport)) {
|
|
case "quic", "direct_quic", "udp_quic", "quic_udp",
|
|
string(FabricRouteLAN), string(FabricRouteReverse), string(FabricRouteRelay), string(FabricRouteICE):
|
|
return true
|
|
default:
|
|
return false
|
|
}
|
|
}
|
|
|
|
func fabricRouteLatencyFromCandidate(candidate PeerEndpointCandidate, cfg FabricRoutePlannerConfig, score int, index int) int {
|
|
if observation, ok := cfg.Observations[candidate.EndpointID]; ok && observation.LastLatencyMs > 0 {
|
|
if observation.LastLatencyMs > int64(^uint(0)>>1) {
|
|
return int(^uint(0) >> 1)
|
|
}
|
|
return int(observation.LastLatencyMs)
|
|
}
|
|
base := 10 + index
|
|
switch strings.ToLower(strings.TrimSpace(candidate.Reachability)) {
|
|
case FabricCandidateReachabilityPrivate:
|
|
base = 3 + index
|
|
case FabricCandidateReachabilityOutboundOnly:
|
|
base = 25 + index
|
|
case FabricCandidateReachabilityRelay:
|
|
base = 40 + index
|
|
}
|
|
if score < 100 {
|
|
base += (100 - score) / 10
|
|
}
|
|
return base
|
|
}
|
|
|
|
func fabricRouteCapacityForMode(mode FabricRouteMode, cfg FabricRoutePlannerConfig) int {
|
|
switch mode {
|
|
case FabricRouteRelay:
|
|
return firstPositiveInt(cfg.RelayCapacity, cfg.DefaultCapacity, 100)
|
|
case FabricRouteReverse:
|
|
return firstPositiveInt(cfg.ReverseCapacity, cfg.DefaultCapacity, 100)
|
|
default:
|
|
return firstPositiveInt(cfg.DefaultCapacity, 100)
|
|
}
|
|
}
|
|
|
|
func candidatePressureCount(endpointID string, cfg FabricRoutePlannerConfig) int64 {
|
|
if pressure, ok := cfg.CapacityPressure[endpointID]; ok {
|
|
return pressure.Count
|
|
}
|
|
return 0
|
|
}
|
|
|
|
func sameLocalSegment(metadata FabricCandidateMetadata, cfg FabricRoutePlannerConfig) bool {
|
|
localSegment := strings.TrimSpace(cfg.LocalSegmentID)
|
|
if localSegment == "" {
|
|
return false
|
|
}
|
|
return strings.EqualFold(strings.TrimSpace(metadata.LocalSegmentID), localSegment)
|
|
}
|
|
|
|
func sameNATGroup(metadata FabricCandidateMetadata, cfg FabricRoutePlannerConfig) bool {
|
|
localNATGroup := strings.TrimSpace(cfg.LocalNATGroupID)
|
|
if localNATGroup == "" {
|
|
return false
|
|
}
|
|
return strings.EqualFold(strings.TrimSpace(metadata.NATGroupID), localNATGroup)
|
|
}
|
|
|
|
func decodeFabricCandidateMetadata(raw json.RawMessage) FabricCandidateMetadata {
|
|
if len(raw) == 0 {
|
|
return FabricCandidateMetadata{}
|
|
}
|
|
var metadata FabricCandidateMetadata
|
|
if err := json.Unmarshal(raw, &metadata); err != nil {
|
|
return FabricCandidateMetadata{}
|
|
}
|
|
return metadata
|
|
}
|
|
|
|
func candidatePeerCertSHA256(candidate PeerEndpointCandidate) string {
|
|
var metadata struct {
|
|
PeerCertSHA256 string `json:"peer_cert_sha256,omitempty"`
|
|
TLSCertSHA256 string `json:"tls_cert_sha256,omitempty"`
|
|
}
|
|
if len(candidate.Metadata) == 0 {
|
|
return ""
|
|
}
|
|
if err := json.Unmarshal(candidate.Metadata, &metadata); err != nil {
|
|
return ""
|
|
}
|
|
return firstNonEmpty(strings.TrimSpace(metadata.PeerCertSHA256), strings.TrimSpace(metadata.TLSCertSHA256))
|
|
}
|
|
|
|
func firstPositiveInt(values ...int) int {
|
|
for _, value := range values {
|
|
if value > 0 {
|
|
return value
|
|
}
|
|
}
|
|
return 0
|
|
}
|
|
|
|
func firstNonZeroDuration(values ...time.Duration) time.Duration {
|
|
for _, value := range values {
|
|
if value > 0 {
|
|
return value
|
|
}
|
|
}
|
|
return 0
|
|
}
|
|
|
|
func FabricRouteSetForRelayFallback(clusterID string, sourceNodeID string, targetNodeID string, relayNodeID string, relayEndpoint string, targetEndpoint string) FabricRouteSet {
|
|
relayEndpoint = strings.TrimRight(strings.TrimSpace(relayEndpoint), "/")
|
|
targetEndpoint = strings.TrimRight(strings.TrimSpace(targetEndpoint), "/")
|
|
candidate := PeerEndpointCandidate{
|
|
EndpointID: fmt.Sprintf("%s-via-%s-relay", strings.TrimSpace(targetNodeID), strings.TrimSpace(relayNodeID)),
|
|
NodeID: strings.TrimSpace(targetNodeID),
|
|
Transport: string(FabricRouteRelay),
|
|
Address: targetEndpoint,
|
|
Reachability: FabricCandidateReachabilityRelay,
|
|
ConnectivityMode: FabricConnectivityRelayRequired,
|
|
Metadata: mustMarshalFabricCandidateMetadata(FabricCandidateMetadata{RelayNodeID: relayNodeID, RelayEndpoint: relayEndpoint}),
|
|
}
|
|
return FabricRouteSetForPeerEndpointCandidates(targetNodeID, []PeerEndpointCandidate{candidate}, FabricRoutePlannerConfig{
|
|
ClusterID: clusterID,
|
|
LocalNodeID: sourceNodeID,
|
|
})
|
|
}
|
|
|
|
func mustMarshalFabricCandidateMetadata(metadata FabricCandidateMetadata) json.RawMessage {
|
|
raw, _ := json.Marshal(metadata)
|
|
return raw
|
|
}
|