diff --git a/cmd/supervisor/main.go b/cmd/supervisor/main.go index 9824547..010421f 100644 --- a/cmd/supervisor/main.go +++ b/cmd/supervisor/main.go @@ -32,7 +32,7 @@ func main() { lifeboat := supervisor.NewPersistentLifeboat(store) if _, err := lifeboat.Snapshot(); errors.Is(err, os.ErrNotExist) { baseline := supervisor.Release{ID: envOr("SUPERVISOR_BASELINE_RELEASE", "bootstrap"), Version: envOr("SUPERVISOR_BASELINE_RELEASE", "bootstrap")} - if ok, startErr := lifeboat.Start(baseline); startErr != nil || !ok { + if ok, startErr := lifeboat.EstablishBaseline(baseline); startErr != nil || !ok { fatal(fmt.Errorf("initialize baseline: %v", startErr)) } if ok, observeErr := lifeboat.Record(supervisor.Observation{ReleaseID: baseline.ID, Ready: true, Reason: "supervisor baseline"}); observeErr != nil || !ok { @@ -53,6 +53,7 @@ func main() { control := http.NewServeMux() control.HandleFunc("GET /v1/state", s.state) control.HandleFunc("POST /v1/releases/start", s.start) + control.HandleFunc("POST /v1/releases/baseline", s.baseline) control.HandleFunc("POST /v1/releases/observe", s.observe) control.HandleFunc("POST /v1/releases/promote", s.promote) control.HandleFunc("POST /v1/releases/rollback", s.rollback) @@ -141,6 +142,14 @@ func (s *server) start(w http.ResponseWriter, r *http.Request) { ok, err := s.lifeboat.Start(release) respond(w, ok, err) } +func (s *server) baseline(w http.ResponseWriter, r *http.Request) { + var release supervisor.Release + if !decode(w, r, &release) { + return + } + ok, err := s.lifeboat.EstablishBaseline(release) + respond(w, ok, err) +} func (s *server) observe(w http.ResponseWriter, r *http.Request) { var observation supervisor.Observation if !decode(w, r, &observation) { diff --git a/deploy/run-production-canary.sh b/deploy/run-production-canary.sh index f7690a5..a638aa4 100644 --- a/deploy/run-production-canary.sh +++ b/deploy/run-production-canary.sh @@ -29,6 +29,7 @@ for attempt in $(seq 1 30); do sleep 2 done +control /v1/releases/baseline -H 'Content-Type: application/json' --data "{\"id\":\"stable-$REVISION\",\"version\":\"$REVISION\"}" || fail_rollback control /v1/releases/start -H 'Content-Type: application/json' --data "{\"id\":\"candidate-$REVISION\",\"version\":\"$REVISION\"}" || fail_rollback control /v1/releases/observe -H 'Content-Type: application/json' --data "{\"release_id\":\"candidate-$REVISION\",\"ready\":true,\"reason\":\"candidate healthcheck passed\",\"observed_at_unix_millis\":$(date +%s%3N)}" || fail_rollback control /v1/traffic/canary || fail_rollback diff --git a/runtime/supervisor/durable_store.go b/runtime/supervisor/durable_store.go index 1226be6..1f732f6 100644 --- a/runtime/supervisor/durable_store.go +++ b/runtime/supervisor/durable_store.go @@ -91,6 +91,10 @@ func (p *PersistentLifeboat) transition(fn func(*Lifeboat) bool) (bool, error) { func (p *PersistentLifeboat) Start(candidate Release) (bool, error) { return p.transition(func(value *Lifeboat) bool { return value.Start(candidate) }) } + +func (p *PersistentLifeboat) EstablishBaseline(release Release) (bool, error) { + return p.transition(func(value *Lifeboat) bool { return value.EstablishBaseline(release) }) +} func (p *PersistentLifeboat) Record(observation Observation) (bool, error) { return p.transition(func(value *Lifeboat) bool { return value.Record(observation) }) } diff --git a/runtime/supervisor/supervisor.go b/runtime/supervisor/supervisor.go index 218ae3e..efaaa26 100644 --- a/runtime/supervisor/supervisor.go +++ b/runtime/supervisor/supervisor.go @@ -32,6 +32,7 @@ type Observation struct { type Lifeboat struct { Current Release `json:"current"` LastKnownGood Release `json:"last_known_good"` + RollbackTarget Release `json:"rollback_target"` Observations []Observation `json:"observations"` TrafficMode TrafficMode `json:"traffic_mode"` StableUpstream string `json:"stable_upstream"` @@ -43,9 +44,25 @@ func (l *Lifeboat) Start(candidate Release) bool { return false } candidate.State = ReleaseCandidate + if l.LastKnownGood.ID != "" { + l.RollbackTarget = l.LastKnownGood + } l.Current = candidate return true } + +func (l *Lifeboat) EstablishBaseline(release Release) bool { + if release.ID == "" || release.Version == "" { + return false + } + release.State = ReleaseHealthy + release.LastKnownGood = true + l.Current = release + l.LastKnownGood = release + l.RollbackTarget = Release{} + l.TrafficMode = TrafficStable + return true +} func (l *Lifeboat) Record(o Observation) bool { if o.ReleaseID == "" { return false @@ -64,11 +81,18 @@ func (l *Lifeboat) Record(o Observation) bool { return true } func (l *Lifeboat) Rollback() bool { - if l.LastKnownGood.ID == "" { + target := l.RollbackTarget + if target.ID == "" { + target = l.LastKnownGood + } + if target.ID == "" { return false } - l.Current = l.LastKnownGood + l.Current = target l.Current.State = ReleaseRolledBack + l.Current.LastKnownGood = true + l.LastKnownGood = l.Current + l.RollbackTarget = Release{} return true }