fix(supervisor): preserve stable rollback target

cmd/supervisor/main.go

@@ -32,7 +32,7 @@ func main() {
 	lifeboat := supervisor.NewPersistentLifeboat(store)
 	if _, err := lifeboat.Snapshot(); errors.Is(err, os.ErrNotExist) {
 		baseline := supervisor.Release{ID: envOr("SUPERVISOR_BASELINE_RELEASE", "bootstrap"), Version: envOr("SUPERVISOR_BASELINE_RELEASE", "bootstrap")}
-		if ok, startErr := lifeboat.Start(baseline); startErr != nil || !ok {
+		if ok, startErr := lifeboat.EstablishBaseline(baseline); startErr != nil || !ok {
 			fatal(fmt.Errorf("initialize baseline: %v", startErr))
 		}
 		if ok, observeErr := lifeboat.Record(supervisor.Observation{ReleaseID: baseline.ID, Ready: true, Reason: "supervisor baseline"}); observeErr != nil || !ok {
@@ -53,6 +53,7 @@ func main() {
 	control := http.NewServeMux()
 	control.HandleFunc("GET /v1/state", s.state)
 	control.HandleFunc("POST /v1/releases/start", s.start)
+	control.HandleFunc("POST /v1/releases/baseline", s.baseline)
 	control.HandleFunc("POST /v1/releases/observe", s.observe)
 	control.HandleFunc("POST /v1/releases/promote", s.promote)
 	control.HandleFunc("POST /v1/releases/rollback", s.rollback)
@@ -141,6 +142,14 @@ func (s *server) start(w http.ResponseWriter, r *http.Request) {
 	ok, err := s.lifeboat.Start(release)
 	respond(w, ok, err)
 }
+func (s *server) baseline(w http.ResponseWriter, r *http.Request) {
+	var release supervisor.Release
+	if !decode(w, r, &release) {
+		return
+	}
+	ok, err := s.lifeboat.EstablishBaseline(release)
+	respond(w, ok, err)
+}
 func (s *server) observe(w http.ResponseWriter, r *http.Request) {
 	var observation supervisor.Observation
 	if !decode(w, r, &observation) {

deploy/images/runtime.Dockerfile

@@ -11,6 +11,7 @@ ARG SOURCE_REVISION=unknown
 LABEL org.opencontainers.image.revision=$SOURCE_REVISION \
       org.opencontainers.image.source="https://git.cin.su/m/Assistant.git"
 RUN addgroup -S assistant && adduser -S -G assistant assistant
+RUN mkdir -p /var/lib/supervisor && chown assistant:assistant /var/lib/supervisor
 USER assistant
 COPY --from=build /out/runtime /usr/local/bin/assistant-runtime
 ENTRYPOINT ["/usr/local/bin/assistant-runtime"]

deploy/run-production-canary.sh

@@ -29,6 +29,7 @@ for attempt in $(seq 1 30); do
   sleep 2
 done
 
+control /v1/releases/baseline -H 'Content-Type: application/json' --data "{\"id\":\"stable-$REVISION\",\"version\":\"$REVISION\"}" || fail_rollback
 control /v1/releases/start -H 'Content-Type: application/json' --data "{\"id\":\"candidate-$REVISION\",\"version\":\"$REVISION\"}" || fail_rollback
 control /v1/releases/observe -H 'Content-Type: application/json' --data "{\"release_id\":\"candidate-$REVISION\",\"ready\":true,\"reason\":\"candidate healthcheck passed\",\"observed_at_unix_millis\":$(date +%s%3N)}" || fail_rollback
 control /v1/traffic/canary || fail_rollback

runtime/supervisor/durable_store.go

@@ -91,6 +91,10 @@ func (p *PersistentLifeboat) transition(fn func(*Lifeboat) bool) (bool, error) {
 func (p *PersistentLifeboat) Start(candidate Release) (bool, error) {
 	return p.transition(func(value *Lifeboat) bool { return value.Start(candidate) })
 }
+
+func (p *PersistentLifeboat) EstablishBaseline(release Release) (bool, error) {
+	return p.transition(func(value *Lifeboat) bool { return value.EstablishBaseline(release) })
+}
 func (p *PersistentLifeboat) Record(observation Observation) (bool, error) {
 	return p.transition(func(value *Lifeboat) bool { return value.Record(observation) })
 }

runtime/supervisor/supervisor.go

@@ -18,24 +18,25 @@ const (
 )
 
 type Release struct {
-	ID            string
-	Version       string
-	State         ReleaseState
-	LastKnownGood bool
+	ID            string       `json:"id"`
+	Version       string       `json:"version"`
+	State         ReleaseState `json:"state"`
+	LastKnownGood bool         `json:"last_known_good"`
 }
 type Observation struct {
-	ReleaseID            string
-	Ready                bool
-	Reason               string
-	ObservedAtUnixMillis int64
+	ReleaseID            string `json:"release_id"`
+	Ready                bool   `json:"ready"`
+	Reason               string `json:"reason"`
+	ObservedAtUnixMillis int64  `json:"observed_at_unix_millis"`
 }
 type Lifeboat struct {
-	Current           Release
-	LastKnownGood     Release
-	Observations      []Observation
-	TrafficMode       TrafficMode
-	StableUpstream    string
-	CandidateUpstream string
+	Current           Release       `json:"current"`
+	LastKnownGood     Release       `json:"last_known_good"`
+	RollbackTarget    Release       `json:"rollback_target"`
+	Observations      []Observation `json:"observations"`
+	TrafficMode       TrafficMode   `json:"traffic_mode"`
+	StableUpstream    string        `json:"stable_upstream"`
+	CandidateUpstream string        `json:"candidate_upstream"`
 }
 
 func (l *Lifeboat) Start(candidate Release) bool {
@@ -43,9 +44,25 @@ func (l *Lifeboat) Start(candidate Release) bool {
 		return false
 	}
 	candidate.State = ReleaseCandidate
+	if l.LastKnownGood.ID != "" {
+		l.RollbackTarget = l.LastKnownGood
+	}
 	l.Current = candidate
 	return true
 }
+
+func (l *Lifeboat) EstablishBaseline(release Release) bool {
+	if release.ID == "" || release.Version == "" {
+		return false
+	}
+	release.State = ReleaseHealthy
+	release.LastKnownGood = true
+	l.Current = release
+	l.LastKnownGood = release
+	l.RollbackTarget = Release{}
+	l.TrafficMode = TrafficStable
+	return true
+}
 func (l *Lifeboat) Record(o Observation) bool {
 	if o.ReleaseID == "" {
 		return false
@@ -64,11 +81,18 @@ func (l *Lifeboat) Record(o Observation) bool {
 	return true
 }
 func (l *Lifeboat) Rollback() bool {
-	if l.LastKnownGood.ID == "" {
+	target := l.RollbackTarget
+	if target.ID == "" {
+		target = l.LastKnownGood
+	}
+	if target.ID == "" {
 		return false
 	}
-	l.Current = l.LastKnownGood
+	l.Current = target
 	l.Current.State = ReleaseRolledBack
+	l.Current.LastKnownGood = true
+	l.LastKnownGood = l.Current
+	l.RollbackTarget = Release{}
 	return true
 }