307 lines
9.1 KiB
Go
307 lines
9.1 KiB
Go
package supervisor
|
|
|
|
import "sort"
|
|
|
|
type ReleaseState string
|
|
|
|
const (
|
|
ReleaseCandidate ReleaseState = "CANDIDATE"
|
|
ReleaseHealthy ReleaseState = "HEALTHY"
|
|
ReleaseFailed ReleaseState = "FAILED"
|
|
ReleaseRolledBack ReleaseState = "ROLLED_BACK"
|
|
)
|
|
|
|
type TrafficMode string
|
|
|
|
const (
|
|
TrafficStable TrafficMode = "STABLE"
|
|
TrafficCanaryHeader TrafficMode = "CANARY_HEADER"
|
|
TrafficCandidate TrafficMode = "CANDIDATE"
|
|
)
|
|
|
|
type Release struct {
|
|
ID string `json:"id"`
|
|
Version string `json:"version"`
|
|
State ReleaseState `json:"state"`
|
|
LastKnownGood bool `json:"last_known_good"`
|
|
Set *ReleaseSet `json:"release_set,omitempty"`
|
|
}
|
|
|
|
const (
|
|
ComponentCreatorWeb = "creator-web"
|
|
ComponentCoreRuntime = "core-runtime"
|
|
ComponentCognitiveWorker = "cognitive-worker"
|
|
ComponentResearchRuntime = "research-runtime"
|
|
)
|
|
|
|
// ComponentRelease is operational deployment evidence. It is deliberately
|
|
// separate from semantic/domain state and contains no credentials.
|
|
type ComponentRelease struct {
|
|
Name string `json:"name"`
|
|
Revision string `json:"revision"`
|
|
Upstream string `json:"upstream"`
|
|
HealthURL string `json:"health_url"`
|
|
ReadyURL string `json:"ready_url"`
|
|
}
|
|
|
|
// ReleaseSet is the atomic deployment unit supervised by the control plane.
|
|
type ReleaseSet struct {
|
|
ID string `json:"id"`
|
|
// Slot identifies the independently managed deployment slot. An empty value
|
|
// remains readable for release state recorded before slot isolation.
|
|
Slot string `json:"slot,omitempty"`
|
|
Revision string `json:"revision"`
|
|
Components map[string]ComponentRelease `json:"components"`
|
|
}
|
|
|
|
var requiredComponents = []string{ComponentCreatorWeb, ComponentCoreRuntime, ComponentCognitiveWorker, ComponentResearchRuntime}
|
|
|
|
func (s ReleaseSet) Validate() bool {
|
|
if s.ID == "" || s.Revision == "" || len(s.Components) != len(requiredComponents) {
|
|
return false
|
|
}
|
|
for _, name := range requiredComponents {
|
|
component, ok := s.Components[name]
|
|
if !ok || component.Name != name || component.Revision != s.Revision || component.Upstream == "" || component.HealthURL == "" || component.ReadyURL == "" {
|
|
return false
|
|
}
|
|
}
|
|
return true
|
|
}
|
|
|
|
func (s ReleaseSet) ComponentNames() []string {
|
|
names := make([]string, 0, len(s.Components))
|
|
for name := range s.Components {
|
|
names = append(names, name)
|
|
}
|
|
sort.Strings(names)
|
|
return names
|
|
}
|
|
|
|
type Observation struct {
|
|
ReleaseID string `json:"release_id"`
|
|
Ready bool `json:"ready"`
|
|
Reason string `json:"reason"`
|
|
ObservedAtUnixMillis int64 `json:"observed_at_unix_millis"`
|
|
}
|
|
|
|
type ReleaseSetObservation struct {
|
|
ReleaseID string `json:"release_id"`
|
|
Ready bool `json:"ready"`
|
|
Reason string `json:"reason"`
|
|
ObservedAtUnixMillis int64 `json:"observed_at_unix_millis"`
|
|
Components map[string]ComponentObservation `json:"components"`
|
|
}
|
|
|
|
type ComponentObservation struct {
|
|
Revision string `json:"revision"`
|
|
Health bool `json:"health"`
|
|
Ready bool `json:"ready"`
|
|
}
|
|
type Lifeboat struct {
|
|
Current Release `json:"current"`
|
|
LastKnownGood Release `json:"last_known_good"`
|
|
RollbackTarget Release `json:"rollback_target"`
|
|
Observations []Observation `json:"observations"`
|
|
TrafficMode TrafficMode `json:"traffic_mode"`
|
|
StableUpstream string `json:"stable_upstream"`
|
|
CandidateUpstream string `json:"candidate_upstream"`
|
|
StableReleaseSet *ReleaseSet `json:"stable_release_set,omitempty"`
|
|
CandidateReleaseSet *ReleaseSet `json:"candidate_release_set,omitempty"`
|
|
LastKnownGoodReleaseSet *ReleaseSet `json:"last_known_good_release_set,omitempty"`
|
|
RollbackTargetReleaseSet *ReleaseSet `json:"rollback_target_release_set,omitempty"`
|
|
}
|
|
|
|
func (l *Lifeboat) StartSet(candidate ReleaseSet) bool {
|
|
if !candidate.Validate() {
|
|
return false
|
|
}
|
|
copySet := candidate
|
|
if l.LastKnownGoodReleaseSet != nil {
|
|
previous := *l.LastKnownGoodReleaseSet
|
|
l.RollbackTargetReleaseSet = &previous
|
|
}
|
|
l.CandidateReleaseSet = ©Set
|
|
l.Current = Release{ID: candidate.ID, Version: candidate.Revision, State: ReleaseCandidate, Set: ©Set}
|
|
return true
|
|
}
|
|
|
|
func (l *Lifeboat) EstablishBaselineSet(stable ReleaseSet) bool {
|
|
if !stable.Validate() {
|
|
return false
|
|
}
|
|
copySet := stable
|
|
l.StableReleaseSet = ©Set
|
|
l.LastKnownGoodReleaseSet = ©Set
|
|
l.Current = Release{ID: stable.ID, Version: stable.Revision, State: ReleaseHealthy, LastKnownGood: true, Set: ©Set}
|
|
l.LastKnownGood = l.Current
|
|
l.RollbackTarget = Release{}
|
|
l.RollbackTargetReleaseSet = nil
|
|
l.TrafficMode = TrafficStable
|
|
return l.setSetUpstreams(©Set, nil)
|
|
}
|
|
|
|
func (l *Lifeboat) setSetUpstreams(stable, candidate *ReleaseSet) bool {
|
|
if stable == nil {
|
|
return false
|
|
}
|
|
stableWeb, ok := stable.Components[ComponentCreatorWeb]
|
|
if !ok {
|
|
return false
|
|
}
|
|
l.StableUpstream = stableWeb.Upstream
|
|
if candidate != nil {
|
|
candidateWeb, exists := candidate.Components[ComponentCreatorWeb]
|
|
if !exists {
|
|
return false
|
|
}
|
|
l.CandidateUpstream = candidateWeb.Upstream
|
|
}
|
|
return true
|
|
}
|
|
|
|
func (l *Lifeboat) RecordSet(o ReleaseSetObservation) bool {
|
|
if l.CandidateReleaseSet == nil || o.ReleaseID == "" || o.ReleaseID != l.CandidateReleaseSet.ID || !o.Ready {
|
|
l.Current.State = ReleaseFailed
|
|
return false
|
|
}
|
|
if len(o.Components) != len(requiredComponents) {
|
|
l.Current.State = ReleaseFailed
|
|
return false
|
|
}
|
|
for _, name := range requiredComponents {
|
|
component, ok := o.Components[name]
|
|
expected := l.CandidateReleaseSet.Components[name]
|
|
if !ok || !component.Health || !component.Ready || component.Revision != expected.Revision {
|
|
l.Current.State = ReleaseFailed
|
|
return false
|
|
}
|
|
}
|
|
l.Current.State = ReleaseHealthy
|
|
return true
|
|
}
|
|
|
|
func (l *Lifeboat) SetTrafficForSets(mode TrafficMode) bool {
|
|
if l.StableReleaseSet == nil || l.CandidateReleaseSet == nil {
|
|
return false
|
|
}
|
|
if !l.setSetUpstreams(l.StableReleaseSet, l.CandidateReleaseSet) {
|
|
return false
|
|
}
|
|
l.TrafficMode = mode
|
|
return true
|
|
}
|
|
|
|
func (l *Lifeboat) PromoteSet() bool {
|
|
if l.CandidateReleaseSet == nil || l.Current.State != ReleaseHealthy {
|
|
return false
|
|
}
|
|
if l.StableReleaseSet != nil {
|
|
previous := *l.StableReleaseSet
|
|
l.RollbackTargetReleaseSet = &previous
|
|
l.RollbackTarget = l.LastKnownGood
|
|
}
|
|
set := *l.CandidateReleaseSet
|
|
l.StableReleaseSet = &set
|
|
l.LastKnownGoodReleaseSet = &set
|
|
l.LastKnownGood = l.Current
|
|
l.LastKnownGood.Set = &set
|
|
l.LastKnownGood.LastKnownGood = true
|
|
l.Current.Set = &set
|
|
l.Current.LastKnownGood = true
|
|
l.CandidateReleaseSet = nil
|
|
// The promoted ReleaseSet is now the stable target. Leaving header-canary
|
|
// mode active would keep an obsolete candidate route in the public proxy.
|
|
l.TrafficMode = TrafficStable
|
|
return l.setSetUpstreams(&set, nil)
|
|
}
|
|
|
|
func (l *Lifeboat) RollbackSet() bool {
|
|
if l.RollbackTargetReleaseSet == nil && l.LastKnownGoodReleaseSet == nil {
|
|
return false
|
|
}
|
|
target := l.RollbackTargetReleaseSet
|
|
if target == nil {
|
|
target = l.LastKnownGoodReleaseSet
|
|
}
|
|
copySet := *target
|
|
l.StableReleaseSet = ©Set
|
|
l.Current = Release{ID: copySet.ID, Version: copySet.Revision, State: ReleaseRolledBack, LastKnownGood: true, Set: ©Set}
|
|
l.LastKnownGood = l.Current
|
|
l.LastKnownGoodReleaseSet = ©Set
|
|
l.CandidateReleaseSet = nil
|
|
l.RollbackTargetReleaseSet = nil
|
|
l.TrafficMode = TrafficStable
|
|
return l.setSetUpstreams(©Set, nil)
|
|
}
|
|
|
|
func (l *Lifeboat) Start(candidate Release) bool {
|
|
if candidate.ID == "" || candidate.Version == "" {
|
|
return false
|
|
}
|
|
candidate.State = ReleaseCandidate
|
|
if l.LastKnownGood.ID != "" {
|
|
l.RollbackTarget = l.LastKnownGood
|
|
}
|
|
l.Current = candidate
|
|
return true
|
|
}
|
|
|
|
func (l *Lifeboat) EstablishBaseline(release Release) bool {
|
|
if release.ID == "" || release.Version == "" {
|
|
return false
|
|
}
|
|
release.State = ReleaseHealthy
|
|
release.LastKnownGood = true
|
|
l.Current = release
|
|
l.LastKnownGood = release
|
|
l.RollbackTarget = Release{}
|
|
l.TrafficMode = TrafficStable
|
|
return true
|
|
}
|
|
func (l *Lifeboat) Record(o Observation) bool {
|
|
if o.ReleaseID == "" {
|
|
return false
|
|
}
|
|
l.Observations = append(l.Observations, o)
|
|
if o.ReleaseID != l.Current.ID {
|
|
return true
|
|
}
|
|
if o.Ready {
|
|
l.Current.State = ReleaseHealthy
|
|
l.Current.LastKnownGood = true
|
|
l.LastKnownGood = l.Current
|
|
} else {
|
|
l.Current.State = ReleaseFailed
|
|
}
|
|
return true
|
|
}
|
|
func (l *Lifeboat) Rollback() bool {
|
|
target := l.RollbackTarget
|
|
if target.ID == "" {
|
|
target = l.LastKnownGood
|
|
}
|
|
if target.ID == "" {
|
|
return false
|
|
}
|
|
l.Current = target
|
|
l.Current.State = ReleaseRolledBack
|
|
l.Current.LastKnownGood = true
|
|
l.LastKnownGood = l.Current
|
|
l.RollbackTarget = Release{}
|
|
return true
|
|
}
|
|
|
|
func (l *Lifeboat) SetTraffic(mode TrafficMode, stableUpstream, candidateUpstream string) bool {
|
|
if mode != TrafficStable && mode != TrafficCanaryHeader && mode != TrafficCandidate {
|
|
return false
|
|
}
|
|
if stableUpstream == "" || (mode != TrafficStable && candidateUpstream == "") {
|
|
return false
|
|
}
|
|
l.TrafficMode = mode
|
|
l.StableUpstream = stableUpstream
|
|
l.CandidateUpstream = candidateUpstream
|
|
return true
|
|
}
|