This commit is contained in:
@@ -5,6 +5,7 @@ import (
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
)
|
||||
|
||||
// DurableStore is supervisor-owned external control-plane state. It is not a
|
||||
@@ -63,12 +64,17 @@ func (s *DurableStore) Load() (Lifeboat, error) {
|
||||
}
|
||||
|
||||
// PersistentLifeboat reloads before every transition and commits after it.
|
||||
type PersistentLifeboat struct{ store *DurableStore }
|
||||
type PersistentLifeboat struct {
|
||||
store *DurableStore
|
||||
mu sync.Mutex
|
||||
}
|
||||
|
||||
func NewPersistentLifeboat(store *DurableStore) *PersistentLifeboat {
|
||||
return &PersistentLifeboat{store: store}
|
||||
}
|
||||
func (p *PersistentLifeboat) transition(fn func(*Lifeboat) bool) (bool, error) {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
value, err := p.store.Load()
|
||||
if err != nil && !os.IsNotExist(err) {
|
||||
return false, err
|
||||
@@ -91,4 +97,30 @@ func (p *PersistentLifeboat) Record(observation Observation) (bool, error) {
|
||||
func (p *PersistentLifeboat) Rollback() (bool, error) {
|
||||
return p.transition(func(value *Lifeboat) bool { return value.Rollback() })
|
||||
}
|
||||
func (p *PersistentLifeboat) Snapshot() (Lifeboat, error) { return p.store.Load() }
|
||||
func (p *PersistentLifeboat) SetTraffic(mode TrafficMode, stableUpstream, candidateUpstream string) (bool, error) {
|
||||
return p.transition(func(value *Lifeboat) bool { return value.SetTraffic(mode, stableUpstream, candidateUpstream) })
|
||||
}
|
||||
|
||||
func (p *PersistentLifeboat) Promote() (bool, error) {
|
||||
return p.transition(func(value *Lifeboat) bool {
|
||||
if value.Current.State != ReleaseHealthy || value.Current.ID == "" {
|
||||
return false
|
||||
}
|
||||
return value.SetTraffic(TrafficCandidate, value.StableUpstream, value.CandidateUpstream)
|
||||
})
|
||||
}
|
||||
|
||||
func (p *PersistentLifeboat) RollbackTraffic() (bool, error) {
|
||||
return p.transition(func(value *Lifeboat) bool {
|
||||
if !value.Rollback() {
|
||||
return false
|
||||
}
|
||||
return value.SetTraffic(TrafficStable, value.StableUpstream, value.CandidateUpstream)
|
||||
})
|
||||
}
|
||||
|
||||
func (p *PersistentLifeboat) Snapshot() (Lifeboat, error) {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
return p.store.Load()
|
||||
}
|
||||
|
||||
@@ -9,6 +9,14 @@ const (
|
||||
ReleaseRolledBack ReleaseState = "ROLLED_BACK"
|
||||
)
|
||||
|
||||
type TrafficMode string
|
||||
|
||||
const (
|
||||
TrafficStable TrafficMode = "STABLE"
|
||||
TrafficCanaryHeader TrafficMode = "CANARY_HEADER"
|
||||
TrafficCandidate TrafficMode = "CANDIDATE"
|
||||
)
|
||||
|
||||
type Release struct {
|
||||
ID string
|
||||
Version string
|
||||
@@ -22,9 +30,12 @@ type Observation struct {
|
||||
ObservedAtUnixMillis int64
|
||||
}
|
||||
type Lifeboat struct {
|
||||
Current Release
|
||||
LastKnownGood Release
|
||||
Observations []Observation
|
||||
Current Release
|
||||
LastKnownGood Release
|
||||
Observations []Observation
|
||||
TrafficMode TrafficMode
|
||||
StableUpstream string
|
||||
CandidateUpstream string
|
||||
}
|
||||
|
||||
func (l *Lifeboat) Start(candidate Release) bool {
|
||||
@@ -60,3 +71,16 @@ func (l *Lifeboat) Rollback() bool {
|
||||
l.Current.State = ReleaseRolledBack
|
||||
return true
|
||||
}
|
||||
|
||||
func (l *Lifeboat) SetTraffic(mode TrafficMode, stableUpstream, candidateUpstream string) bool {
|
||||
if mode != TrafficStable && mode != TrafficCanaryHeader && mode != TrafficCandidate {
|
||||
return false
|
||||
}
|
||||
if stableUpstream == "" || (mode != TrafficStable && candidateUpstream == "") {
|
||||
return false
|
||||
}
|
||||
l.TrafficMode = mode
|
||||
l.StableUpstream = stableUpstream
|
||||
l.CandidateUpstream = candidateUpstream
|
||||
return true
|
||||
}
|
||||
|
||||
@@ -47,3 +47,38 @@ func TestPersistentLifeboatRestoresLastKnownGoodAfterCandidateFailure(t *testing
|
||||
t.Fatalf("state=%+v", state)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPersistentSupervisorRestoresTrafficModeAfterRestart(t *testing.T) {
|
||||
store, err := supervisor.NewDurableStore(filepath.Join(t.TempDir(), "control", "release-state.json"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
first := supervisor.NewPersistentLifeboat(store)
|
||||
if ok, err := first.Start(supervisor.Release{ID: "candidate", Version: "2"}); err != nil || !ok {
|
||||
t.Fatalf("start: %v %v", ok, err)
|
||||
}
|
||||
if ok, err := first.Record(supervisor.Observation{ReleaseID: "candidate", Ready: true, ObservedAtUnixMillis: 1}); err != nil || !ok {
|
||||
t.Fatalf("observe: %v %v", ok, err)
|
||||
}
|
||||
if ok, err := first.SetTraffic(supervisor.TrafficCanaryHeader, "http://stable:8084", "http://candidate:8084"); err != nil || !ok {
|
||||
t.Fatalf("canary: %v %v", ok, err)
|
||||
}
|
||||
second := supervisor.NewPersistentLifeboat(store)
|
||||
state, err := second.Snapshot()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if state.TrafficMode != supervisor.TrafficCanaryHeader || state.CandidateUpstream != "http://candidate:8084" || state.LastKnownGood.ID != "candidate" {
|
||||
t.Fatalf("state=%+v", state)
|
||||
}
|
||||
if ok, err := second.RollbackTraffic(); err != nil || !ok {
|
||||
t.Fatalf("rollback traffic: %v %v", ok, err)
|
||||
}
|
||||
state, err = second.Snapshot()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if state.TrafficMode != supervisor.TrafficStable || state.Current.State != supervisor.ReleaseRolledBack {
|
||||
t.Fatalf("rollback state=%+v", state)
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user