129 lines
3.0 KiB
Go
129 lines
3.0 KiB
Go
package mesh
|
|
|
|
import (
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
)
|
|
|
|
type FabricRouteHealthTracker struct {
|
|
mu sync.Mutex
|
|
QuarantineTTL time.Duration
|
|
routes map[string]FabricRouteHealthEntry
|
|
}
|
|
|
|
type FabricRouteHealthEntry struct {
|
|
Reason string `json:"reason,omitempty"`
|
|
Failures uint64 `json:"failures"`
|
|
LastFailure time.Time `json:"last_failure,omitempty"`
|
|
RetryAfter time.Time `json:"retry_after,omitempty"`
|
|
}
|
|
|
|
type FabricRouteHealthSnapshot struct {
|
|
Quarantined map[string]FabricRouteHealthEntry `json:"quarantined,omitempty"`
|
|
}
|
|
|
|
func NewFabricRouteHealthTracker(ttl time.Duration) *FabricRouteHealthTracker {
|
|
if ttl <= 0 {
|
|
ttl = 30 * time.Second
|
|
}
|
|
return &FabricRouteHealthTracker{QuarantineTTL: ttl, routes: map[string]FabricRouteHealthEntry{}}
|
|
}
|
|
|
|
func (t *FabricRouteHealthTracker) MarkFailure(routeID string, reason string, now time.Time) {
|
|
routeID = strings.TrimSpace(routeID)
|
|
if t == nil || routeID == "" {
|
|
return
|
|
}
|
|
if now.IsZero() {
|
|
now = time.Now().UTC()
|
|
}
|
|
ttl := t.QuarantineTTL
|
|
if ttl <= 0 {
|
|
ttl = 30 * time.Second
|
|
}
|
|
t.mu.Lock()
|
|
entry := t.routes[routeID]
|
|
entry.Failures++
|
|
entry.Reason = strings.TrimSpace(reason)
|
|
entry.LastFailure = now
|
|
entry.RetryAfter = now.Add(ttl)
|
|
if t.routes == nil {
|
|
t.routes = map[string]FabricRouteHealthEntry{}
|
|
}
|
|
t.routes[routeID] = entry
|
|
t.mu.Unlock()
|
|
}
|
|
|
|
func (t *FabricRouteHealthTracker) MarkSuccess(routeID string) {
|
|
routeID = strings.TrimSpace(routeID)
|
|
if t == nil || routeID == "" {
|
|
return
|
|
}
|
|
t.mu.Lock()
|
|
delete(t.routes, routeID)
|
|
t.mu.Unlock()
|
|
}
|
|
|
|
func (t *FabricRouteHealthTracker) Apply(routeSet FabricRouteSet, now time.Time) FabricRouteSet {
|
|
if t == nil {
|
|
return routeSet
|
|
}
|
|
if now.IsZero() {
|
|
now = time.Now().UTC()
|
|
}
|
|
t.mu.Lock()
|
|
defer t.mu.Unlock()
|
|
if len(t.routes) == 0 {
|
|
return routeSet
|
|
}
|
|
return mapFabricRouteSet(routeSet, func(route FabricRoute) FabricRoute {
|
|
entry, ok := t.routes[route.RouteID]
|
|
if !ok {
|
|
return route
|
|
}
|
|
if !entry.RetryAfter.IsZero() && !now.Before(entry.RetryAfter) {
|
|
delete(t.routes, route.RouteID)
|
|
return route
|
|
}
|
|
route.Healthy = false
|
|
route.Degraded = true
|
|
return route
|
|
})
|
|
}
|
|
|
|
func (t *FabricRouteHealthTracker) Snapshot(now time.Time) FabricRouteHealthSnapshot {
|
|
if t == nil {
|
|
return FabricRouteHealthSnapshot{}
|
|
}
|
|
if now.IsZero() {
|
|
now = time.Now().UTC()
|
|
}
|
|
t.mu.Lock()
|
|
defer t.mu.Unlock()
|
|
out := map[string]FabricRouteHealthEntry{}
|
|
for routeID, entry := range t.routes {
|
|
if !entry.RetryAfter.IsZero() && !now.Before(entry.RetryAfter) {
|
|
continue
|
|
}
|
|
out[routeID] = entry
|
|
}
|
|
if len(out) == 0 {
|
|
return FabricRouteHealthSnapshot{}
|
|
}
|
|
return FabricRouteHealthSnapshot{Quarantined: out}
|
|
}
|
|
|
|
func mapFabricRouteSet(routeSet FabricRouteSet, fn func(FabricRoute) FabricRoute) FabricRouteSet {
|
|
if strings.TrimSpace(routeSet.Primary.RouteID) != "" {
|
|
routeSet.Primary = fn(routeSet.Primary)
|
|
}
|
|
for i := range routeSet.WarmStandby {
|
|
routeSet.WarmStandby[i] = fn(routeSet.WarmStandby[i])
|
|
}
|
|
for i := range routeSet.ColdFallbacks {
|
|
routeSet.ColdFallbacks[i] = fn(routeSet.ColdFallbacks[i])
|
|
}
|
|
return routeSet
|
|
}
|