From 3a563a29ee506d1d3bcf303abbb51159a9d27ae8 Mon Sep 17 00:00:00 2001 From: Mikhail Date: Sat, 29 Aug 2026 12:33:44 +0300 Subject: [PATCH] feat(deploy): add independent supervisor control plane --- cmd/supervisor/main.go | 203 +++++++++++++++++- deploy/docker-compose.prod.yml | 33 ++- deploy/run-production-canary.sh | 42 ++++ dev/ACTIVE_TASK.json | 2 +- ...01-supervisor-and-production-rollback.json | 36 ++++ runtime/supervisor/durable_store.go | 36 +++- runtime/supervisor/supervisor.go | 30 ++- runtime/supervisor/supervisor_test.go | 35 +++ 8 files changed, 404 insertions(+), 13 deletions(-) create mode 100644 deploy/run-production-canary.sh create mode 100644 dev/tasks/GEN2-DEPLOY-001-supervisor-and-production-rollback.json diff --git a/cmd/supervisor/main.go b/cmd/supervisor/main.go index 61e9b8b..9824547 100644 --- a/cmd/supervisor/main.go +++ b/cmd/supervisor/main.go @@ -1,10 +1,203 @@ package main -import "net/http" +import ( + "encoding/json" + "errors" + "fmt" + "net/http" + "net/http/httputil" + "net/url" + "os" + "strings" + "time" + + "git.cin.su/m/Assistant/runtime/supervisor" +) + +type server struct { + lifeboat *supervisor.PersistentLifeboat + stableUpstream string + candidate string + client *http.Client +} func main() { - _ = http.ListenAndServe(":8088", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.WriteHeader(http.StatusOK) - _, _ = w.Write([]byte("lifeboat-ready\n")) - })) + statePath := envOr("SUPERVISOR_STATE_PATH", "/var/lib/supervisor/release-state.json") + stable := strings.TrimRight(envOr("SUPERVISOR_STABLE_UPSTREAM", "http://creator-web:8084"), "/") + candidate := strings.TrimRight(envOr("SUPERVISOR_CANDIDATE_UPSTREAM", "http://creator-web-candidate:8084"), "/") + store, err := supervisor.NewDurableStore(statePath) + if err != nil { + fatal(err) + } + lifeboat := supervisor.NewPersistentLifeboat(store) + if _, err := lifeboat.Snapshot(); errors.Is(err, os.ErrNotExist) { + baseline := supervisor.Release{ID: envOr("SUPERVISOR_BASELINE_RELEASE", "bootstrap"), Version: envOr("SUPERVISOR_BASELINE_RELEASE", "bootstrap")} + if ok, startErr := lifeboat.Start(baseline); startErr != nil || !ok { + fatal(fmt.Errorf("initialize baseline: %v", startErr)) + } + if ok, observeErr := lifeboat.Record(supervisor.Observation{ReleaseID: baseline.ID, Ready: true, Reason: "supervisor baseline"}); observeErr != nil || !ok { + fatal(fmt.Errorf("initialize baseline observation: %v", observeErr)) + } + if ok, trafficErr := lifeboat.SetTraffic(supervisor.TrafficStable, stable, candidate); trafficErr != nil || !ok { + fatal(fmt.Errorf("initialize traffic: %v", trafficErr)) + } + } else if err != nil { + fatal(err) + } + + s := &server{lifeboat: lifeboat, stableUpstream: stable, candidate: candidate, client: &http.Client{Timeout: 5 * time.Second}} + public := http.NewServeMux() + public.HandleFunc("GET /healthz", s.health) + public.HandleFunc("GET /readyz", s.ready) + public.HandleFunc("/", s.proxy) + control := http.NewServeMux() + control.HandleFunc("GET /v1/state", s.state) + control.HandleFunc("POST /v1/releases/start", s.start) + control.HandleFunc("POST /v1/releases/observe", s.observe) + control.HandleFunc("POST /v1/releases/promote", s.promote) + control.HandleFunc("POST /v1/releases/rollback", s.rollback) + control.HandleFunc("POST /v1/traffic/canary", s.canary) + control.HandleFunc("POST /v1/traffic/promote", s.promoteTraffic) + control.HandleFunc("POST /v1/traffic/rollback", s.rollbackTraffic) + + go func() { + if err := http.ListenAndServe(envOr("SUPERVISOR_CONTROL_ADDR", ":8095"), control); err != nil { + fatal(err) + } + }() + if err := http.ListenAndServe(envOr("LISTEN_ADDR", ":8080"), public); err != nil { + fatal(err) + } } + +func (s *server) health(w http.ResponseWriter, _ *http.Request) { + writeJSON(w, map[string]string{"runtime": "supervisor", "status": "ok"}) +} + +func (s *server) ready(w http.ResponseWriter, r *http.Request) { + state, err := s.lifeboat.Snapshot() + if err != nil || state.StableUpstream == "" { + http.Error(w, "supervisor state unavailable", http.StatusServiceUnavailable) + return + } + request, err := http.NewRequestWithContext(r.Context(), http.MethodGet, state.StableUpstream+"/healthz", nil) + if err != nil { + http.Error(w, "stable upstream unavailable", http.StatusServiceUnavailable) + return + } + response, err := s.client.Do(request) + if err != nil || response.StatusCode/100 != 2 { + if response != nil { + response.Body.Close() + } + http.Error(w, "stable upstream unavailable", http.StatusServiceUnavailable) + return + } + response.Body.Close() + writeJSON(w, map[string]string{"runtime": "supervisor", "status": "ready"}) +} + +func (s *server) proxy(w http.ResponseWriter, r *http.Request) { + state, err := s.lifeboat.Snapshot() + if err != nil { + http.Error(w, "supervisor state unavailable", http.StatusServiceUnavailable) + return + } + target := state.StableUpstream + selected := "stable" + if state.TrafficMode == supervisor.TrafficCandidate || (state.TrafficMode == supervisor.TrafficCanaryHeader && strings.EqualFold(r.Header.Get("X-Assistant-Canary"), "true")) { + target = state.CandidateUpstream + selected = "candidate" + } + parsed, err := url.Parse(target) + if err != nil || parsed.Scheme == "" || parsed.Host == "" { + http.Error(w, "invalid upstream", http.StatusServiceUnavailable) + return + } + proxy := httputil.NewSingleHostReverseProxy(parsed) + proxy.ModifyResponse = func(response *http.Response) error { + response.Header.Set("X-Assistant-Traffic-Target", selected) + return nil + } + proxy.ErrorHandler = func(writer http.ResponseWriter, _ *http.Request, _ error) { + http.Error(writer, "upstream unavailable", http.StatusBadGateway) + } + proxy.ServeHTTP(w, r) +} + +func (s *server) state(w http.ResponseWriter, _ *http.Request) { + state, err := s.lifeboat.Snapshot() + if err != nil { + http.Error(w, "state unavailable", 503) + return + } + writeJSON(w, state) +} +func (s *server) start(w http.ResponseWriter, r *http.Request) { + var release supervisor.Release + if !decode(w, r, &release) { + return + } + ok, err := s.lifeboat.Start(release) + respond(w, ok, err) +} +func (s *server) observe(w http.ResponseWriter, r *http.Request) { + var observation supervisor.Observation + if !decode(w, r, &observation) { + return + } + ok, err := s.lifeboat.Record(observation) + respond(w, ok, err) +} +func (s *server) promote(w http.ResponseWriter, _ *http.Request) { + ok, err := s.lifeboat.Promote() + respond(w, ok, err) +} +func (s *server) rollback(w http.ResponseWriter, _ *http.Request) { + ok, err := s.lifeboat.Rollback() + respond(w, ok, err) +} +func (s *server) canary(w http.ResponseWriter, _ *http.Request) { + ok, err := s.lifeboat.SetTraffic(supervisor.TrafficCanaryHeader, s.stableUpstream, s.candidate) + respond(w, ok, err) +} +func (s *server) promoteTraffic(w http.ResponseWriter, _ *http.Request) { + ok, err := s.lifeboat.Promote() + respond(w, ok, err) +} +func (s *server) rollbackTraffic(w http.ResponseWriter, _ *http.Request) { + ok, err := s.lifeboat.RollbackTraffic() + respond(w, ok, err) +} + +func decode(w http.ResponseWriter, r *http.Request, value any) bool { + decoder := json.NewDecoder(http.MaxBytesReader(w, r.Body, 16<<10)) + decoder.DisallowUnknownFields() + if err := decoder.Decode(value); err != nil { + http.Error(w, "invalid control request", 400) + return false + } + return true +} +func respond(w http.ResponseWriter, ok bool, err error) { + if err != nil { + http.Error(w, "control state unavailable", 503) + return + } + if !ok { + http.Error(w, "control transition rejected", 409) + return + } + writeJSON(w, map[string]bool{"ok": true}) +} +func writeJSON(w http.ResponseWriter, value any) { + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(value) +} +func envOr(name, fallback string) string { + if value := strings.TrimSpace(os.Getenv(name)); value != "" { + return value + } + return fallback +} +func fatal(err error) { fmt.Fprintln(os.Stderr, err); os.Exit(1) } diff --git a/deploy/docker-compose.prod.yml b/deploy/docker-compose.prod.yml index 4f9599d..5712366 100644 --- a/deploy/docker-compose.prod.yml +++ b/deploy/docker-compose.prod.yml @@ -277,7 +277,7 @@ services: - --port - "8086" - --model - - /models/Qwen3-8B-Q4_K_M.gguf + - /models/Qwen3.8-27B-UD-Q2_K_XL.gguf - --alias - domovoy-lifeboat - --jinja @@ -299,6 +299,29 @@ services: timeout: 3s retries: 10 + supervisor: + build: { context: .., dockerfile: deploy/images/runtime.Dockerfile, args: { TARGET: supervisor, SOURCE_REVISION: "${ASSISTANT_SOURCE_REVISION:-unknown}" } } + restart: unless-stopped + environment: + LISTEN_ADDR: ":8080" + SUPERVISOR_CONTROL_ADDR: ":8095" + SUPERVISOR_STATE_PATH: /var/lib/supervisor/release-state.json + SUPERVISOR_STABLE_UPSTREAM: http://creator-web:8084 + SUPERVISOR_CANDIDATE_UPSTREAM: http://creator-web-candidate:8084 + SUPERVISOR_BASELINE_RELEASE: ${ASSISTANT_SOURCE_REVISION:-unknown} + ports: + - "${CREATOR_WEB_PORT:-18080}:8080" + - "127.0.0.1:${SUPERVISOR_CONTROL_PORT:-18085}:8095" + depends_on: + creator-web: { condition: service_healthy } + volumes: + - assistant_supervisor_state:/var/lib/supervisor + healthcheck: + test: ["CMD-SHELL", "wget -qO- http://localhost:8080/healthz >/dev/null && wget -qO- http://localhost:8080/readyz >/dev/null"] + interval: 10s + timeout: 5s + retries: 20 + creator-web: build: { context: .., dockerfile: deploy/images/runtime.Dockerfile, args: { TARGET: creator-web, SOURCE_REVISION: "${ASSISTANT_SOURCE_REVISION:-unknown}" } } restart: unless-stopped @@ -310,7 +333,6 @@ services: COGNITIVE_RUNTIME_URL: http://cognitive-worker:8083 AGENT_WORKER_URL: http://agent-worker:8090 ASSISTANT_BUILD_REVISION: ${ASSISTANT_SOURCE_REVISION:-unknown} - ports: ["${CREATOR_WEB_PORT:-18080}:8084"] depends_on: [core-runtime, continuity-runtime, commit-runtime, cognitive-worker] healthcheck: test: ["CMD-SHELL", "wget -qO- http://localhost:8084/healthz >/dev/null"] @@ -318,6 +340,12 @@ services: timeout: 3s retries: 10 + creator-web-candidate: + profiles: ["assistant-canary"] + extends: + service: creator-web + restart: "no" + volumes: assistant_pg: assistant_nats: @@ -333,6 +361,7 @@ volumes: external: true assistant_research_sources: name: assistant_research_sources + assistant_supervisor_state: networks: llm-local: diff --git a/deploy/run-production-canary.sh b/deploy/run-production-canary.sh new file mode 100644 index 0000000..f7690a5 --- /dev/null +++ b/deploy/run-production-canary.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash +set -euo pipefail + +BASE_DIR="${REPO_DIR:-$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)}" +ENV_FILE="${ENV_FILE:-/home/m/assistant-deploy/production-compose.env}" +COMPOSE_FILE="$BASE_DIR/deploy/docker-compose.prod.yml" +SUPERVISOR_CONTROL_URL="${SUPERVISOR_CONTROL_URL:-http://127.0.0.1:${SUPERVISOR_CONTROL_PORT:-18085}}" +REVISION="${1:?candidate revision is required}" + +set -a +. "$ENV_FILE" +set +a + +compose() { docker compose --env-file "$ENV_FILE" -f "$COMPOSE_FILE" "$@"; } +control() { curl -fsS --max-time 10 -X POST "$SUPERVISOR_CONTROL_URL$1" "${@:2}" >/dev/null; } +fail_rollback() { + control /v1/traffic/rollback || true + control /v1/releases/rollback || true + echo "production canary failed; stable traffic rollback requested" >&2 + exit 1 +} + +compose --profile assistant-canary build --build-arg "SOURCE_REVISION=$REVISION" creator-web-candidate +compose --profile assistant-canary up -d --no-build creator-web-candidate +for attempt in $(seq 1 30); do + status="$(docker inspect -f '{{if .State.Health}}{{.State.Health.Status}}{{else}}unknown{{end}}' assistant-creator-web-candidate-1 2>/dev/null || true)" + [[ "$status" == healthy ]] && break + [[ "$attempt" == 30 ]] && fail_rollback + sleep 2 +done + +control /v1/releases/start -H 'Content-Type: application/json' --data "{\"id\":\"candidate-$REVISION\",\"version\":\"$REVISION\"}" || fail_rollback +control /v1/releases/observe -H 'Content-Type: application/json' --data "{\"release_id\":\"candidate-$REVISION\",\"ready\":true,\"reason\":\"candidate healthcheck passed\",\"observed_at_unix_millis\":$(date +%s%3N)}" || fail_rollback +control /v1/traffic/canary || fail_rollback + +headers="$(mktemp)" +trap 'rm -f "$headers"' EXIT +if ! curl -fsS --max-time 10 -D "$headers" -H 'X-Assistant-Canary: true' "https://assistant.cin.su/login" -o /dev/null; then fail_rollback; fi +grep -qi '^X-Assistant-Traffic-Target: candidate' "$headers" || fail_rollback + +control /v1/releases/promote || fail_rollback +echo "production canary promoted revision=$REVISION" diff --git a/dev/ACTIVE_TASK.json b/dev/ACTIVE_TASK.json index b044ed2..a908e1e 100644 --- a/dev/ACTIVE_TASK.json +++ b/dev/ACTIVE_TASK.json @@ -1 +1 @@ -{"task_file":"dev/tasks/GEN2-027-production-canary-and-controlled-gen2-traffic-transition.json","task_id":"GEN2-027","task_revision":1} +{"task_file":"dev/tasks/GEN2-DEPLOY-001-supervisor-and-production-rollback.json","task_id":"GEN2-DEPLOY-001","task_revision":1} diff --git a/dev/tasks/GEN2-DEPLOY-001-supervisor-and-production-rollback.json b/dev/tasks/GEN2-DEPLOY-001-supervisor-and-production-rollback.json new file mode 100644 index 0000000..7f2608c --- /dev/null +++ b/dev/tasks/GEN2-DEPLOY-001-supervisor-and-production-rollback.json @@ -0,0 +1,36 @@ +{ + "task_id": "GEN2-DEPLOY-001", + "task_revision": 1, + "architecture_revision": "ARCH-GEN2-000", + "base_repo_revision": "gen2-integration", + "risk_tier": "E", + "intent": "Развернуть независимый Supervisor control plane с durable release state, controlled traffic switching и доказуемым rollback перед GEN2-027.", + "requirement_ids": ["REQ-GEN2-DEPLOY-001-SUPERVISOR", "REQ-GEN2-DEPLOY-001-TRAFFIC", "REQ-GEN2-DEPLOY-001-ROLLBACK", "REQ-GEN2-DEPLOY-001-LIFEBOAT"], + "allowed_change_scope": { + "modules": ["deployment control", "runtime supervisor", "runtime health"], + "paths": [ + "deploy/**", + "runtime/supervisor/**", + "runtime/health/**", + "cmd/supervisor/**", + "dev/ACTIVE_TASK.json", + "dev/tasks/GEN2-DEPLOY-001-supervisor-and-production-rollback.json", + "docs/evidence/**" + ] + }, + "forbidden_change_scope": { + "paths": ["GEN2-028", "database/**", "security/**", "architecture/**", "Authority/Commit bypass", "cognitive-lifeboat as Supervisor"] + }, + "allowed_architecture_delta": {"new_cross_module_dependencies": 0, "new_canonical_owners": 0, "new_privileges": 1, "new_third_party_dependencies": 0, "new_database_owner_roles": 0}, + "contracts": {"consume": ["GEN2-027 production canary contract", "runtime/supervisor durable state", "existing deployment update controller"], "may_change": ["deployment topology, Supervisor HTTP control contract and traffic routing only"]}, + "semantic_owner": "Operations/Supervisor; not a semantic owner and not a cognitive provider.", + "canonical_state": ["durable release/current/last-known-good evidence owned by Supervisor control plane", "traffic target is Supervisor-owned operational state"], + "state_transitions": ["baseline -> candidate -> observed healthy -> header canary -> promoted OR independent rollback to last-known-good"], + "side_effects_and_privileges": ["Control API is bound only to the production-internal loopback port.", "Supervisor has no database, Authority or Commit credentials.", "Candidate traffic is explicit and rollback remains available."], + "persistence_and_migrations": ["No database changes; durable state is stored in a dedicated named volume outside candidate containers."], + "failure_retry_restart": ["Supervisor restart reloads release and traffic state from its volume.", "Candidate failure must not erase last-known-good or block stable traffic rollback.", "Lifeboat model failure remains a separately reported degraded provider condition."], + "assumptions": ["GEN2-027 resumes only after this task's production evidence; GEN2-028 is not started.", "The existing public ingress forwards to the configured creator web host port."], + "acceptance_scenarios": ["GEN2-DEPLOY-001-SCN-01: Supervisor is a separate healthy production container", "GEN2-DEPLOY-001-SCN-02: release state survives Supervisor restart", "GEN2-DEPLOY-001-SCN-03: canary header reaches candidate while default traffic stays stable", "GEN2-DEPLOY-001-SCN-04: candidate failure rolls traffic back to last-known-good", "GEN2-DEPLOY-001-SCN-05: cognitive lifeboat diagnosis and degraded role are explicit", "GEN2-DEPLOY-001-EXIT: GEN2-027 rollback prerequisites are proven"], + "required_tests": ["FAST: go test ./runtime/supervisor/... ./runtime/health/... ./cmd/supervisor/...", "FAST: go vet ./runtime/supervisor/... ./runtime/health/... ./cmd/supervisor/...", "FAST governance: preflight", "FAST scope: check-diff", "FAST independent: independent", "FAST whitespace: git diff --check", "TARGETED recovery: Supervisor restart, canary and rollback on docker.cin.su"], + "terminal_gap_states": ["BLOCKED_BY_SPEC_GAP", "BLOCKED_BY_ARCHITECTURE_GAP", "BLOCKED_BY_MISSING_CAPABILITY", "BLOCKED_BY_CONFLICT", "REQUIRES_MIGRATION_DECISION"] +} diff --git a/runtime/supervisor/durable_store.go b/runtime/supervisor/durable_store.go index f98c4e1..1226be6 100644 --- a/runtime/supervisor/durable_store.go +++ b/runtime/supervisor/durable_store.go @@ -5,6 +5,7 @@ import ( "errors" "os" "path/filepath" + "sync" ) // DurableStore is supervisor-owned external control-plane state. It is not a @@ -63,12 +64,17 @@ func (s *DurableStore) Load() (Lifeboat, error) { } // PersistentLifeboat reloads before every transition and commits after it. -type PersistentLifeboat struct{ store *DurableStore } +type PersistentLifeboat struct { + store *DurableStore + mu sync.Mutex +} func NewPersistentLifeboat(store *DurableStore) *PersistentLifeboat { return &PersistentLifeboat{store: store} } func (p *PersistentLifeboat) transition(fn func(*Lifeboat) bool) (bool, error) { + p.mu.Lock() + defer p.mu.Unlock() value, err := p.store.Load() if err != nil && !os.IsNotExist(err) { return false, err @@ -91,4 +97,30 @@ func (p *PersistentLifeboat) Record(observation Observation) (bool, error) { func (p *PersistentLifeboat) Rollback() (bool, error) { return p.transition(func(value *Lifeboat) bool { return value.Rollback() }) } -func (p *PersistentLifeboat) Snapshot() (Lifeboat, error) { return p.store.Load() } +func (p *PersistentLifeboat) SetTraffic(mode TrafficMode, stableUpstream, candidateUpstream string) (bool, error) { + return p.transition(func(value *Lifeboat) bool { return value.SetTraffic(mode, stableUpstream, candidateUpstream) }) +} + +func (p *PersistentLifeboat) Promote() (bool, error) { + return p.transition(func(value *Lifeboat) bool { + if value.Current.State != ReleaseHealthy || value.Current.ID == "" { + return false + } + return value.SetTraffic(TrafficCandidate, value.StableUpstream, value.CandidateUpstream) + }) +} + +func (p *PersistentLifeboat) RollbackTraffic() (bool, error) { + return p.transition(func(value *Lifeboat) bool { + if !value.Rollback() { + return false + } + return value.SetTraffic(TrafficStable, value.StableUpstream, value.CandidateUpstream) + }) +} + +func (p *PersistentLifeboat) Snapshot() (Lifeboat, error) { + p.mu.Lock() + defer p.mu.Unlock() + return p.store.Load() +} diff --git a/runtime/supervisor/supervisor.go b/runtime/supervisor/supervisor.go index 7ea0210..7756073 100644 --- a/runtime/supervisor/supervisor.go +++ b/runtime/supervisor/supervisor.go @@ -9,6 +9,14 @@ const ( ReleaseRolledBack ReleaseState = "ROLLED_BACK" ) +type TrafficMode string + +const ( + TrafficStable TrafficMode = "STABLE" + TrafficCanaryHeader TrafficMode = "CANARY_HEADER" + TrafficCandidate TrafficMode = "CANDIDATE" +) + type Release struct { ID string Version string @@ -22,9 +30,12 @@ type Observation struct { ObservedAtUnixMillis int64 } type Lifeboat struct { - Current Release - LastKnownGood Release - Observations []Observation + Current Release + LastKnownGood Release + Observations []Observation + TrafficMode TrafficMode + StableUpstream string + CandidateUpstream string } func (l *Lifeboat) Start(candidate Release) bool { @@ -60,3 +71,16 @@ func (l *Lifeboat) Rollback() bool { l.Current.State = ReleaseRolledBack return true } + +func (l *Lifeboat) SetTraffic(mode TrafficMode, stableUpstream, candidateUpstream string) bool { + if mode != TrafficStable && mode != TrafficCanaryHeader && mode != TrafficCandidate { + return false + } + if stableUpstream == "" || (mode != TrafficStable && candidateUpstream == "") { + return false + } + l.TrafficMode = mode + l.StableUpstream = stableUpstream + l.CandidateUpstream = candidateUpstream + return true +} diff --git a/runtime/supervisor/supervisor_test.go b/runtime/supervisor/supervisor_test.go index 7fa4ee6..2480ce8 100644 --- a/runtime/supervisor/supervisor_test.go +++ b/runtime/supervisor/supervisor_test.go @@ -47,3 +47,38 @@ func TestPersistentLifeboatRestoresLastKnownGoodAfterCandidateFailure(t *testing t.Fatalf("state=%+v", state) } } + +func TestPersistentSupervisorRestoresTrafficModeAfterRestart(t *testing.T) { + store, err := supervisor.NewDurableStore(filepath.Join(t.TempDir(), "control", "release-state.json")) + if err != nil { + t.Fatal(err) + } + first := supervisor.NewPersistentLifeboat(store) + if ok, err := first.Start(supervisor.Release{ID: "candidate", Version: "2"}); err != nil || !ok { + t.Fatalf("start: %v %v", ok, err) + } + if ok, err := first.Record(supervisor.Observation{ReleaseID: "candidate", Ready: true, ObservedAtUnixMillis: 1}); err != nil || !ok { + t.Fatalf("observe: %v %v", ok, err) + } + if ok, err := first.SetTraffic(supervisor.TrafficCanaryHeader, "http://stable:8084", "http://candidate:8084"); err != nil || !ok { + t.Fatalf("canary: %v %v", ok, err) + } + second := supervisor.NewPersistentLifeboat(store) + state, err := second.Snapshot() + if err != nil { + t.Fatal(err) + } + if state.TrafficMode != supervisor.TrafficCanaryHeader || state.CandidateUpstream != "http://candidate:8084" || state.LastKnownGood.ID != "candidate" { + t.Fatalf("state=%+v", state) + } + if ok, err := second.RollbackTraffic(); err != nil || !ok { + t.Fatalf("rollback traffic: %v %v", ok, err) + } + state, err = second.Snapshot() + if err != nil { + t.Fatal(err) + } + if state.TrafficMode != supervisor.TrafficStable || state.Current.State != supervisor.ReleaseRolledBack { + t.Fatalf("rollback state=%+v", state) + } +}