From 93a9c1a3bd667a99b8bb4bc73eba83b5aad61f73 Mon Sep 17 00:00:00 2001 From: Mikhail Date: Sun, 30 Aug 2026 10:39:36 +0300 Subject: [PATCH] feat: supervise complete Domovoy release sets --- cmd/supervisor/main.go | 92 +++++++- deploy/docker-compose.prod.yml | 51 +++++ deploy/run-production-canary.sh | 62 ++++-- dev/ACTIVE_TASK.json | 2 +- ...EN2-DEPLOY-002-supervised-release-set.json | 23 ++ .../gen2-deploy-002-blocked-2026-08-30.json | 40 ++++ runtime/supervisor/durable_store.go | 24 +++ runtime/supervisor/supervisor.go | 200 +++++++++++++++++- runtime/supervisor/supervisor_test.go | 57 +++++ 9 files changed, 528 insertions(+), 23 deletions(-) create mode 100644 dev/tasks/GEN2-DEPLOY-002-supervised-release-set.json create mode 100644 docs/evidence/gen2-deploy-002-blocked-2026-08-30.json diff --git a/cmd/supervisor/main.go b/cmd/supervisor/main.go index 010421f..258f8b2 100644 --- a/cmd/supervisor/main.go +++ b/cmd/supervisor/main.go @@ -53,8 +53,11 @@ func main() { control := http.NewServeMux() control.HandleFunc("GET /v1/state", s.state) control.HandleFunc("POST /v1/releases/start", s.start) + control.HandleFunc("POST /v1/releases/start-set", s.startSet) control.HandleFunc("POST /v1/releases/baseline", s.baseline) + control.HandleFunc("POST /v1/releases/baseline-set", s.baselineSet) control.HandleFunc("POST /v1/releases/observe", s.observe) + control.HandleFunc("POST /v1/releases/observe-set", s.observeSet) control.HandleFunc("POST /v1/releases/promote", s.promote) control.HandleFunc("POST /v1/releases/rollback", s.rollback) control.HandleFunc("POST /v1/traffic/canary", s.canary) @@ -81,6 +84,10 @@ func (s *server) ready(w http.ResponseWriter, r *http.Request) { http.Error(w, "supervisor state unavailable", http.StatusServiceUnavailable) return } + if state.StableReleaseSet != nil && !s.checkReleaseSet(r, state.StableReleaseSet) { + http.Error(w, "stable release set unavailable", http.StatusServiceUnavailable) + return + } request, err := http.NewRequestWithContext(r.Context(), http.MethodGet, state.StableUpstream+"/healthz", nil) if err != nil { http.Error(w, "stable upstream unavailable", http.StatusServiceUnavailable) @@ -98,6 +105,28 @@ func (s *server) ready(w http.ResponseWriter, r *http.Request) { writeJSON(w, map[string]string{"runtime": "supervisor", "status": "ready"}) } +func (s *server) checkReleaseSet(r *http.Request, releaseSet *supervisor.ReleaseSet) bool { + if releaseSet == nil || !releaseSet.Validate() { + return false + } + for _, name := range releaseSet.ComponentNames() { + component := releaseSet.Components[name] + request, err := http.NewRequestWithContext(r.Context(), http.MethodGet, component.ReadyURL, nil) + if err != nil { + return false + } + response, err := s.client.Do(request) + if err != nil || response.StatusCode/100 != 2 { + if response != nil { + response.Body.Close() + } + return false + } + response.Body.Close() + } + return true +} + func (s *server) proxy(w http.ResponseWriter, r *http.Request) { state, err := s.lifeboat.Snapshot() if err != nil { @@ -142,6 +171,14 @@ func (s *server) start(w http.ResponseWriter, r *http.Request) { ok, err := s.lifeboat.Start(release) respond(w, ok, err) } +func (s *server) startSet(w http.ResponseWriter, r *http.Request) { + var releaseSet supervisor.ReleaseSet + if !decode(w, r, &releaseSet) { + return + } + ok, err := s.lifeboat.StartSet(releaseSet) + respond(w, ok, err) +} func (s *server) baseline(w http.ResponseWriter, r *http.Request) { var release supervisor.Release if !decode(w, r, &release) { @@ -150,6 +187,14 @@ func (s *server) baseline(w http.ResponseWriter, r *http.Request) { ok, err := s.lifeboat.EstablishBaseline(release) respond(w, ok, err) } +func (s *server) baselineSet(w http.ResponseWriter, r *http.Request) { + var releaseSet supervisor.ReleaseSet + if !decode(w, r, &releaseSet) { + return + } + ok, err := s.lifeboat.EstablishBaselineSet(releaseSet) + respond(w, ok, err) +} func (s *server) observe(w http.ResponseWriter, r *http.Request) { var observation supervisor.Observation if !decode(w, r, &observation) { @@ -158,16 +203,57 @@ func (s *server) observe(w http.ResponseWriter, r *http.Request) { ok, err := s.lifeboat.Record(observation) respond(w, ok, err) } +func (s *server) observeSet(w http.ResponseWriter, r *http.Request) { + var observation supervisor.ReleaseSetObservation + if !decode(w, r, &observation) { + return + } + ok, err := s.lifeboat.RecordSet(observation) + respond(w, ok, err) +} func (s *server) promote(w http.ResponseWriter, _ *http.Request) { - ok, err := s.lifeboat.Promote() + state, snapshotErr := s.lifeboat.Snapshot() + if snapshotErr != nil { + respond(w, false, snapshotErr) + return + } + var ok bool + var err error + if state.CandidateReleaseSet != nil { + ok, err = s.lifeboat.PromoteSet() + } else { + ok, err = s.lifeboat.Promote() + } respond(w, ok, err) } func (s *server) rollback(w http.ResponseWriter, _ *http.Request) { - ok, err := s.lifeboat.Rollback() + state, snapshotErr := s.lifeboat.Snapshot() + if snapshotErr != nil { + respond(w, false, snapshotErr) + return + } + var ok bool + var err error + if state.CandidateReleaseSet != nil || state.RollbackTargetReleaseSet != nil { + ok, err = s.lifeboat.RollbackSet() + } else { + ok, err = s.lifeboat.Rollback() + } respond(w, ok, err) } func (s *server) canary(w http.ResponseWriter, _ *http.Request) { - ok, err := s.lifeboat.SetTraffic(supervisor.TrafficCanaryHeader, s.stableUpstream, s.candidate) + state, snapshotErr := s.lifeboat.Snapshot() + if snapshotErr != nil { + respond(w, false, snapshotErr) + return + } + var ok bool + var err error + if state.CandidateReleaseSet != nil { + ok, err = s.lifeboat.SetTrafficForSets(supervisor.TrafficCanaryHeader) + } else { + ok, err = s.lifeboat.SetTraffic(supervisor.TrafficCanaryHeader, s.stableUpstream, s.candidate) + } respond(w, ok, err) } func (s *server) promoteTraffic(w http.ResponseWriter, _ *http.Request) { diff --git a/deploy/docker-compose.prod.yml b/deploy/docker-compose.prod.yml index f2e8858..69a2dce 100644 --- a/deploy/docker-compose.prod.yml +++ b/deploy/docker-compose.prod.yml @@ -83,6 +83,22 @@ services: depends_on: { postgres: { condition: service_healthy } } volumes: - assistant_core_secrets:/run/secrets:ro + healthcheck: + test: ["CMD-SHELL", "wget -qO- http://localhost:8080/healthz >/dev/null && wget -qO- http://localhost:8080/readyz >/dev/null"] + interval: 10s + timeout: 20s + retries: 20 + + core-runtime-candidate: + profiles: ["assistant-canary"] + extends: + service: core-runtime + restart: "no" + environment: + RESEARCH_RUNTIME_URL: http://research-runtime-candidate:8087 + depends_on: + postgres: { condition: service_healthy } + research-runtime-candidate: { condition: service_started } agent-canary: profiles: ["agent-canary"] @@ -201,6 +217,19 @@ services: volumes: - assistant_core_secrets:/run/secrets:ro - assistant_research_sources:/var/lib/research-sources:ro + healthcheck: + test: ["CMD-SHELL", "wget -qO- http://localhost:8087/healthz >/dev/null && wget -qO- http://localhost:8087/readyz >/dev/null"] + interval: 10s + timeout: 10s + retries: 20 + + research-runtime-candidate: + profiles: ["assistant-canary"] + extends: + service: research-runtime + restart: "no" + depends_on: + postgres: { condition: service_healthy } continuity-runtime: build: { context: .., dockerfile: deploy/images/runtime.Dockerfile, args: { TARGET: continuity-runtime, SOURCE_REVISION: "${ASSISTANT_SOURCE_REVISION:-unknown}" } } @@ -270,6 +299,17 @@ services: timeout: 20s retries: 20 + cognitive-worker-candidate: + profiles: ["assistant-canary"] + extends: + service: cognitive-worker + restart: "no" + environment: + CORE_RUNTIME_URL: http://core-runtime-candidate:8080 + depends_on: + core-runtime-candidate: { condition: service_started } + cognitive-lifeboat: { condition: service_healthy } + cognitive-lifeboat: image: ghcr.io/ggml-org/llama.cpp:server restart: unless-stopped @@ -351,6 +391,17 @@ services: extends: service: creator-web restart: "no" + environment: + CORE_RUNTIME_URL: http://core-runtime-candidate:8080 + CONTINUITY_RUNTIME_URL: http://continuity-runtime:8081 + COMMIT_RUNTIME_URL: http://commit-runtime:8082 + COGNITIVE_RUNTIME_URL: http://cognitive-worker-candidate:8083 + AGENT_WORKER_URL: http://agent-worker:8090 + depends_on: + core-runtime-candidate: { condition: service_started } + continuity-runtime: { condition: service_started } + commit-runtime: { condition: service_started } + cognitive-worker-candidate: { condition: service_healthy } volumes: assistant_pg: diff --git a/deploy/run-production-canary.sh b/deploy/run-production-canary.sh index a638aa4..9ce8ade 100644 --- a/deploy/run-production-canary.sh +++ b/deploy/run-production-canary.sh @@ -16,22 +16,60 @@ control() { curl -fsS --max-time 10 -X POST "$SUPERVISOR_CONTROL_URL$1" "${@:2}" fail_rollback() { control /v1/traffic/rollback || true control /v1/releases/rollback || true - echo "production canary failed; stable traffic rollback requested" >&2 + echo "production ReleaseSet canary failed; stable ReleaseSet rollback requested" >&2 exit 1 } -compose --profile assistant-canary build --build-arg "SOURCE_REVISION=$REVISION" creator-web-candidate -compose --profile assistant-canary up -d --no-build creator-web-candidate -for attempt in $(seq 1 30); do - status="$(docker inspect -f '{{if .State.Health}}{{.State.Health.Status}}{{else}}unknown{{end}}' assistant-creator-web-candidate-1 2>/dev/null || true)" - [[ "$status" == healthy ]] && break - [[ "$attempt" == 30 ]] && fail_rollback - sleep 2 +services=(creator-web-candidate core-runtime-candidate cognitive-worker-candidate research-runtime-candidate) +containers=(assistant-creator-web-candidate-1 assistant-core-runtime-candidate-1 assistant-cognitive-worker-candidate-1 assistant-research-runtime-candidate-1) + +compose --profile assistant-canary build --build-arg "SOURCE_REVISION=$REVISION" "${services[@]}" +compose --profile assistant-canary up -d --no-build "${services[@]}" + +for container in "${containers[@]}"; do + ready="" + for attempt in $(seq 1 60); do + status="$(docker inspect -f '{{if .State.Health}}{{.State.Health.Status}}{{else}}unknown{{end}}' "$container" 2>/dev/null || true)" + if [[ "$status" == healthy ]]; then ready=true; break; fi + sleep 2 + done + [[ "$ready" == true ]] || fail_rollback + image_revision="$(docker inspect -f '{{index .Config.Labels "org.opencontainers.image.revision"}}' "$container" 2>/dev/null || true)" + [[ "$image_revision" == "$REVISION" ]] || fail_rollback done -control /v1/releases/baseline -H 'Content-Type: application/json' --data "{\"id\":\"stable-$REVISION\",\"version\":\"$REVISION\"}" || fail_rollback -control /v1/releases/start -H 'Content-Type: application/json' --data "{\"id\":\"candidate-$REVISION\",\"version\":\"$REVISION\"}" || fail_rollback -control /v1/releases/observe -H 'Content-Type: application/json' --data "{\"release_id\":\"candidate-$REVISION\",\"ready\":true,\"reason\":\"candidate healthcheck passed\",\"observed_at_unix_millis\":$(date +%s%3N)}" || fail_rollback +release_json="$(cat </dev/null || fail_rollback +fi + +curl -fsS --max-time 10 -X POST "$SUPERVISOR_CONTROL_URL/v1/releases/start-set" -H 'Content-Type: application/json' --data "$release_json" >/dev/null || fail_rollback +observation="$(printf '%s' "$release_json" | python3 -c 'import json,sys,time; s=json.load(sys.stdin); print(json.dumps({"release_id":s["id"],"ready":True,"reason":"all ReleaseSet healthchecks and image revisions passed","observed_at_unix_millis":int(time.time()*1000),"components":{k:{"revision":v["revision"],"health":True,"ready":True} for k,v in s["components"].items()}},separators=(",",":")))')" +curl -fsS --max-time 10 -X POST "$SUPERVISOR_CONTROL_URL/v1/releases/observe-set" -H 'Content-Type: application/json' --data "$observation" >/dev/null || fail_rollback control /v1/traffic/canary || fail_rollback headers="$(mktemp)" @@ -40,4 +78,4 @@ if ! curl -fsS --max-time 10 -D "$headers" -H 'X-Assistant-Canary: true' "https: grep -qi '^X-Assistant-Traffic-Target: candidate' "$headers" || fail_rollback control /v1/releases/promote || fail_rollback -echo "production canary promoted revision=$REVISION" +echo "production ReleaseSet canary promoted revision=$REVISION" diff --git a/dev/ACTIVE_TASK.json b/dev/ACTIVE_TASK.json index b9e68cc..a55e0dd 100644 --- a/dev/ACTIVE_TASK.json +++ b/dev/ACTIVE_TASK.json @@ -1 +1 @@ -{"task_file":"dev/tasks/GEN2-COMPAT-OBS-001-production-compatibility-observation.json","task_id":"GEN2-COMPAT-OBS-001","task_revision":1} +{"task_file":"dev/tasks/GEN2-DEPLOY-002-supervised-release-set.json","task_id":"GEN2-DEPLOY-002","task_revision":1} diff --git a/dev/tasks/GEN2-DEPLOY-002-supervised-release-set.json b/dev/tasks/GEN2-DEPLOY-002-supervised-release-set.json new file mode 100644 index 0000000..272081a --- /dev/null +++ b/dev/tasks/GEN2-DEPLOY-002-supervised-release-set.json @@ -0,0 +1,23 @@ +{ + "task_id": "GEN2-DEPLOY-002", + "task_revision": 1, + "architecture_revision": "ARCH-GEN2-000", + "base_repo_revision": "gen2-integration", + "risk_tier": "E", + "intent": "Расширить существующий Supervisor до безопасного управления единым candidate/stable ReleaseSet из creator-web, core-runtime, cognitive-worker и research-runtime без изменения Gen2 semantic ownership.", + "requirement_ids": ["REQ-GEN2-DEPLOY-002-RELEASE-SET", "REQ-GEN2-DEPLOY-002-CANDIDATE-TOPOLOGY", "REQ-GEN2-DEPLOY-002-ROLLBACK", "REQ-GEN2-DEPLOY-002-COMPAT-OBS"], + "allowed_change_scope": {"modules": ["runtime/supervisor"], "paths": ["runtime/supervisor/**", "cmd/supervisor/**", "deploy/**", "docs/**", "tests/**", "dev/ACTIVE_TASK.json", "dev/tasks/GEN2-DEPLOY-002-supervised-release-set.json"]}, + "forbidden_change_scope": {"paths": ["architecture/**", "database/**", "modules/**", "contracts/**", "security/**", "go.mod", "go.sum", ".gitea/workflows/**", "adapters/models/**"]}, + "allowed_architecture_delta": {"new_cross_module_dependencies": 0, "new_canonical_owners": 0, "new_privileges": 1, "new_third_party_dependencies": 0, "new_database_owner_roles": 0}, + "contracts": {"consume": ["existing Supervisor durable state and production compose", "existing canary header and stable rollback controls"], "may_change": ["Supervisor release control payloads", "deployment topology and candidate routing configuration"]}, + "semantic_owner": "Supervisor/deployment control plane; ReleaseSet is operational release state and not semantic canonical state.", + "canonical_state": ["no semantic owner or schema change", "previous stable release and evidence remain retained"], + "state_transitions": ["candidate ReleaseSet -> component health/revision checks -> canary -> observation -> promotion or whole-set rollback"], + "side_effects_and_privileges": ["production deployment only through approved control plane; no direct stable replacement; no credential reads"], + "persistence_and_migrations": ["durable Supervisor release state remains in existing external Supervisor volume; no database migration"], + "assumptions": ["the existing production Supervisor and compose control plane remain authoritative for deployment", "candidate services may share existing durable infrastructure only under current ownership and compatibility rules", "no credentials are read by this task"], + "failure_retry_restart": ["candidate component failure routes all traffic to previous stable ReleaseSet", "Supervisor restart reloads ReleaseSet state", "previous stable is retained until promotion is complete"], + "acceptance_scenarios": ["DEPLOY-002-SCN-01: complete ReleaseSet has all four components and one revision", "DEPLOY-002-SCN-02: candidate dependencies resolve only to candidate services", "DEPLOY-002-SCN-03: health/readiness/revision checks cover all required components", "DEPLOY-002-SCN-04: canary and stable paths are isolated", "DEPLOY-002-SCN-05: forced candidate failure rolls back the complete ReleaseSet", "DEPLOY-002-SCN-06: Supervisor restart preserves stable/candidate/LKG state", "DEPLOY-002-SCN-07: production telemetry revision starts bounded observation after controlled deployment"], + "required_tests": ["go test ./runtime/supervisor ./cmd/supervisor", "FAST: targeted supervisor/release-set tests", "FAST: compose config validation", "FAST: routing and revision consistency tests", "FAST: rollback/restart persistence tests", "FAST: targeted go vet", "FAST governance: preflight/check-diff/independent", "FAST whitespace: git diff --check", "PRODUCTION: candidate ReleaseSet drill and telemetry observation start"], + "terminal_gap_states": ["BLOCKED_BY_SPEC_GAP", "BLOCKED_BY_ARCHITECTURE_GAP", "BLOCKED_BY_MISSING_CAPABILITY", "BLOCKED_BY_CONFLICT", "REQUIRES_MIGRATION_DECISION"] +} diff --git a/docs/evidence/gen2-deploy-002-blocked-2026-08-30.json b/docs/evidence/gen2-deploy-002-blocked-2026-08-30.json new file mode 100644 index 0000000..8eda81b --- /dev/null +++ b/docs/evidence/gen2-deploy-002-blocked-2026-08-30.json @@ -0,0 +1,40 @@ +{ + "evidence_id": "gen2-deploy-002-blocked-2026-08-30", + "task_id": "GEN2-DEPLOY-002", + "task_revision": 1, + "base_repo_revision": "gen2-integration", + "status": "BLOCKED_BY_MISSING_CAPABILITY", + "implementation_checks": { + "release_set_tests": "PASS", + "supervisor_go_test": "PASS", + "supervisor_go_vet": "PASS", + "compose_config": "PASS", + "canary_script_syntax": "PASS", + "governance": "PASS", + "git_diff_check": "PASS" + }, + "implemented": [ + "durable Supervisor ReleaseSet state for creator-web, core-runtime, cognitive-worker, and research-runtime", + "parallel candidate services with candidate-only core/cognitive/research dependency wiring", + "whole-set health/readiness/revision observation, canary, promotion and rollback controls", + "stable/candidate/LKG persistence across Supervisor restart", + "core and research runtime compose healthchecks" + ], + "production": { + "deployment": "NOT_RUN", + "candidate_release_set": "NOT_STARTED", + "telemetry_observation": "NOT_STARTED", + "stable_mutation": "NONE" + }, + "capability_evidence": { + "docker_and_compose_on_production_host": "AVAILABLE", + "supervisor_control_endpoint": "REACHABLE", + "portainer_api_key_in_current_execution_context": "NOT_CONFIGURED", + "portainer_endpoint_metadata_in_current_execution_context": "NOT_CONFIGURED", + "existing_production_candidate_services": ["creator-web-candidate"], + "missing_capability": "Approved production deployment automation or candidate-capable control-plane access for the four-component ReleaseSet." + }, + "unsafe_action_not_taken": "Stable containers were not replaced directly and no production compose mutation was attempted.", + "compatibility": "No V1 compatibility path was removed or disabled; no schema, migration, semantic owner, or Gen3 change.", + "next_action": "Owner must provide or enable the approved candidate-capable production deployment mechanism; then run the ReleaseSet drill and start bounded telemetry observation." +} diff --git a/runtime/supervisor/durable_store.go b/runtime/supervisor/durable_store.go index 1f732f6..b11a3f1 100644 --- a/runtime/supervisor/durable_store.go +++ b/runtime/supervisor/durable_store.go @@ -92,12 +92,24 @@ func (p *PersistentLifeboat) Start(candidate Release) (bool, error) { return p.transition(func(value *Lifeboat) bool { return value.Start(candidate) }) } +func (p *PersistentLifeboat) StartSet(candidate ReleaseSet) (bool, error) { + return p.transition(func(value *Lifeboat) bool { return value.StartSet(candidate) }) +} + func (p *PersistentLifeboat) EstablishBaseline(release Release) (bool, error) { return p.transition(func(value *Lifeboat) bool { return value.EstablishBaseline(release) }) } + +func (p *PersistentLifeboat) EstablishBaselineSet(release ReleaseSet) (bool, error) { + return p.transition(func(value *Lifeboat) bool { return value.EstablishBaselineSet(release) }) +} func (p *PersistentLifeboat) Record(observation Observation) (bool, error) { return p.transition(func(value *Lifeboat) bool { return value.Record(observation) }) } + +func (p *PersistentLifeboat) RecordSet(observation ReleaseSetObservation) (bool, error) { + return p.transition(func(value *Lifeboat) bool { return value.RecordSet(observation) }) +} func (p *PersistentLifeboat) Rollback() (bool, error) { return p.transition(func(value *Lifeboat) bool { return value.Rollback() }) } @@ -105,6 +117,10 @@ func (p *PersistentLifeboat) SetTraffic(mode TrafficMode, stableUpstream, candid return p.transition(func(value *Lifeboat) bool { return value.SetTraffic(mode, stableUpstream, candidateUpstream) }) } +func (p *PersistentLifeboat) SetTrafficForSets(mode TrafficMode) (bool, error) { + return p.transition(func(value *Lifeboat) bool { return value.SetTrafficForSets(mode) }) +} + func (p *PersistentLifeboat) Promote() (bool, error) { return p.transition(func(value *Lifeboat) bool { if value.Current.State != ReleaseHealthy || value.Current.ID == "" { @@ -114,6 +130,10 @@ func (p *PersistentLifeboat) Promote() (bool, error) { }) } +func (p *PersistentLifeboat) PromoteSet() (bool, error) { + return p.transition(func(value *Lifeboat) bool { return value.PromoteSet() }) +} + func (p *PersistentLifeboat) RollbackTraffic() (bool, error) { return p.transition(func(value *Lifeboat) bool { if !value.Rollback() { @@ -123,6 +143,10 @@ func (p *PersistentLifeboat) RollbackTraffic() (bool, error) { }) } +func (p *PersistentLifeboat) RollbackSet() (bool, error) { + return p.transition(func(value *Lifeboat) bool { return value.RollbackSet() }) +} + func (p *PersistentLifeboat) Snapshot() (Lifeboat, error) { p.mu.Lock() defer p.mu.Unlock() diff --git a/runtime/supervisor/supervisor.go b/runtime/supervisor/supervisor.go index efaaa26..bf44ca0 100644 --- a/runtime/supervisor/supervisor.go +++ b/runtime/supervisor/supervisor.go @@ -1,5 +1,7 @@ package supervisor +import "sort" + type ReleaseState string const ( @@ -22,21 +24,205 @@ type Release struct { Version string `json:"version"` State ReleaseState `json:"state"` LastKnownGood bool `json:"last_known_good"` + Set *ReleaseSet `json:"release_set,omitempty"` } + +const ( + ComponentCreatorWeb = "creator-web" + ComponentCoreRuntime = "core-runtime" + ComponentCognitiveWorker = "cognitive-worker" + ComponentResearchRuntime = "research-runtime" +) + +// ComponentRelease is operational deployment evidence. It is deliberately +// separate from semantic/domain state and contains no credentials. +type ComponentRelease struct { + Name string `json:"name"` + Revision string `json:"revision"` + Upstream string `json:"upstream"` + HealthURL string `json:"health_url"` + ReadyURL string `json:"ready_url"` +} + +// ReleaseSet is the atomic deployment unit supervised by the control plane. +type ReleaseSet struct { + ID string `json:"id"` + Revision string `json:"revision"` + Components map[string]ComponentRelease `json:"components"` +} + +var requiredComponents = []string{ComponentCreatorWeb, ComponentCoreRuntime, ComponentCognitiveWorker, ComponentResearchRuntime} + +func (s ReleaseSet) Validate() bool { + if s.ID == "" || s.Revision == "" || len(s.Components) != len(requiredComponents) { + return false + } + for _, name := range requiredComponents { + component, ok := s.Components[name] + if !ok || component.Name != name || component.Revision != s.Revision || component.Upstream == "" || component.HealthURL == "" || component.ReadyURL == "" { + return false + } + } + return true +} + +func (s ReleaseSet) ComponentNames() []string { + names := make([]string, 0, len(s.Components)) + for name := range s.Components { + names = append(names, name) + } + sort.Strings(names) + return names +} + type Observation struct { ReleaseID string `json:"release_id"` Ready bool `json:"ready"` Reason string `json:"reason"` ObservedAtUnixMillis int64 `json:"observed_at_unix_millis"` } + +type ReleaseSetObservation struct { + ReleaseID string `json:"release_id"` + Ready bool `json:"ready"` + Reason string `json:"reason"` + ObservedAtUnixMillis int64 `json:"observed_at_unix_millis"` + Components map[string]ComponentObservation `json:"components"` +} + +type ComponentObservation struct { + Revision string `json:"revision"` + Health bool `json:"health"` + Ready bool `json:"ready"` +} type Lifeboat struct { - Current Release `json:"current"` - LastKnownGood Release `json:"last_known_good"` - RollbackTarget Release `json:"rollback_target"` - Observations []Observation `json:"observations"` - TrafficMode TrafficMode `json:"traffic_mode"` - StableUpstream string `json:"stable_upstream"` - CandidateUpstream string `json:"candidate_upstream"` + Current Release `json:"current"` + LastKnownGood Release `json:"last_known_good"` + RollbackTarget Release `json:"rollback_target"` + Observations []Observation `json:"observations"` + TrafficMode TrafficMode `json:"traffic_mode"` + StableUpstream string `json:"stable_upstream"` + CandidateUpstream string `json:"candidate_upstream"` + StableReleaseSet *ReleaseSet `json:"stable_release_set,omitempty"` + CandidateReleaseSet *ReleaseSet `json:"candidate_release_set,omitempty"` + LastKnownGoodReleaseSet *ReleaseSet `json:"last_known_good_release_set,omitempty"` + RollbackTargetReleaseSet *ReleaseSet `json:"rollback_target_release_set,omitempty"` +} + +func (l *Lifeboat) StartSet(candidate ReleaseSet) bool { + if !candidate.Validate() { + return false + } + copySet := candidate + if l.LastKnownGoodReleaseSet != nil { + previous := *l.LastKnownGoodReleaseSet + l.RollbackTargetReleaseSet = &previous + } + l.CandidateReleaseSet = ©Set + l.Current = Release{ID: candidate.ID, Version: candidate.Revision, State: ReleaseCandidate, Set: ©Set} + return true +} + +func (l *Lifeboat) EstablishBaselineSet(stable ReleaseSet) bool { + if !stable.Validate() { + return false + } + copySet := stable + l.StableReleaseSet = ©Set + l.LastKnownGoodReleaseSet = ©Set + l.Current = Release{ID: stable.ID, Version: stable.Revision, State: ReleaseHealthy, LastKnownGood: true, Set: ©Set} + l.LastKnownGood = l.Current + l.RollbackTarget = Release{} + l.RollbackTargetReleaseSet = nil + l.TrafficMode = TrafficStable + return l.setSetUpstreams(©Set, nil) +} + +func (l *Lifeboat) setSetUpstreams(stable, candidate *ReleaseSet) bool { + if stable == nil { + return false + } + stableWeb, ok := stable.Components[ComponentCreatorWeb] + if !ok { + return false + } + l.StableUpstream = stableWeb.Upstream + if candidate != nil { + candidateWeb, exists := candidate.Components[ComponentCreatorWeb] + if !exists { + return false + } + l.CandidateUpstream = candidateWeb.Upstream + } + return true +} + +func (l *Lifeboat) RecordSet(o ReleaseSetObservation) bool { + if l.CandidateReleaseSet == nil || o.ReleaseID == "" || o.ReleaseID != l.CandidateReleaseSet.ID || !o.Ready { + l.Current.State = ReleaseFailed + return false + } + if len(o.Components) != len(requiredComponents) { + l.Current.State = ReleaseFailed + return false + } + for _, name := range requiredComponents { + component, ok := o.Components[name] + expected := l.CandidateReleaseSet.Components[name] + if !ok || !component.Health || !component.Ready || component.Revision != expected.Revision { + l.Current.State = ReleaseFailed + return false + } + } + l.Current.State = ReleaseHealthy + return true +} + +func (l *Lifeboat) SetTrafficForSets(mode TrafficMode) bool { + if l.StableReleaseSet == nil || l.CandidateReleaseSet == nil { + return false + } + if !l.setSetUpstreams(l.StableReleaseSet, l.CandidateReleaseSet) { + return false + } + l.TrafficMode = mode + return true +} + +func (l *Lifeboat) PromoteSet() bool { + if l.CandidateReleaseSet == nil || l.Current.State != ReleaseHealthy { + return false + } + set := *l.CandidateReleaseSet + l.StableReleaseSet = &set + l.LastKnownGoodReleaseSet = &set + l.LastKnownGood = l.Current + l.LastKnownGood.Set = &set + l.LastKnownGood.LastKnownGood = true + l.Current.Set = &set + l.Current.LastKnownGood = true + l.CandidateReleaseSet = nil + l.RollbackTargetReleaseSet = nil + return l.setSetUpstreams(&set, nil) +} + +func (l *Lifeboat) RollbackSet() bool { + if l.RollbackTargetReleaseSet == nil && l.LastKnownGoodReleaseSet == nil { + return false + } + target := l.RollbackTargetReleaseSet + if target == nil { + target = l.LastKnownGoodReleaseSet + } + copySet := *target + l.StableReleaseSet = ©Set + l.Current = Release{ID: copySet.ID, Version: copySet.Revision, State: ReleaseRolledBack, LastKnownGood: true, Set: ©Set} + l.LastKnownGood = l.Current + l.LastKnownGoodReleaseSet = ©Set + l.CandidateReleaseSet = nil + l.RollbackTargetReleaseSet = nil + l.TrafficMode = TrafficStable + return l.setSetUpstreams(©Set, nil) } func (l *Lifeboat) Start(candidate Release) bool { diff --git a/runtime/supervisor/supervisor_test.go b/runtime/supervisor/supervisor_test.go index 2480ce8..f6ca351 100644 --- a/runtime/supervisor/supervisor_test.go +++ b/runtime/supervisor/supervisor_test.go @@ -82,3 +82,60 @@ func TestPersistentSupervisorRestoresTrafficModeAfterRestart(t *testing.T) { t.Fatalf("rollback state=%+v", state) } } + +func testReleaseSet(id, revision, web, core, cognitive, research string) supervisor.ReleaseSet { + return supervisor.ReleaseSet{ID: id, Revision: revision, Components: map[string]supervisor.ComponentRelease{ + supervisor.ComponentCreatorWeb: {Name: supervisor.ComponentCreatorWeb, Revision: revision, Upstream: web, HealthURL: web + "/healthz", ReadyURL: web + "/readyz"}, + supervisor.ComponentCoreRuntime: {Name: supervisor.ComponentCoreRuntime, Revision: revision, Upstream: core, HealthURL: core + "/healthz", ReadyURL: core + "/readyz"}, + supervisor.ComponentCognitiveWorker: {Name: supervisor.ComponentCognitiveWorker, Revision: revision, Upstream: cognitive, HealthURL: cognitive + "/healthz", ReadyURL: cognitive + "/readyz"}, + supervisor.ComponentResearchRuntime: {Name: supervisor.ComponentResearchRuntime, Revision: revision, Upstream: research, HealthURL: research + "/healthz", ReadyURL: research + "/readyz"}, + }} +} + +func TestReleaseSetRejectsMixedRevisionsAndMissingComponents(t *testing.T) { + set := testReleaseSet("candidate", "r1", "http://web", "http://core", "http://cognitive", "http://research") + if !set.Validate() { + t.Fatal("complete release set should validate") + } + set.Components[supervisor.ComponentCoreRuntime] = supervisor.ComponentRelease{Name: supervisor.ComponentCoreRuntime, Revision: "r0", Upstream: "http://core", HealthURL: "http://core/healthz", ReadyURL: "http://core/readyz"} + if set.Validate() { + t.Fatal("mixed revision release set must be rejected") + } +} + +func TestPersistentReleaseSetRollbackPreservesWholeLastKnownGood(t *testing.T) { + store, err := supervisor.NewDurableStore(filepath.Join(t.TempDir(), "control", "release-set.json")) + if err != nil { + t.Fatal(err) + } + first := supervisor.NewPersistentLifeboat(store) + good := testReleaseSet("stable-r1", "r1", "http://web", "http://core", "http://cognitive", "http://research") + candidate := testReleaseSet("candidate-r2", "r2", "http://web-c", "http://core-c", "http://cognitive-c", "http://research-c") + if ok, err := first.EstablishBaselineSet(good); err != nil || !ok { + t.Fatalf("baseline: %v %v", ok, err) + } + if ok, err := first.StartSet(candidate); err != nil || !ok { + t.Fatalf("start: %v %v", ok, err) + } + observation := supervisor.ReleaseSetObservation{ReleaseID: candidate.ID, Ready: true, Components: map[string]supervisor.ComponentObservation{}} + for name := range candidate.Components { + observation.Components[name] = supervisor.ComponentObservation{Revision: candidate.Revision, Health: true, Ready: true} + } + if ok, err := first.RecordSet(observation); err != nil || !ok { + t.Fatalf("observe: %v %v", ok, err) + } + if ok, err := first.SetTrafficForSets(supervisor.TrafficCanaryHeader); err != nil || !ok { + t.Fatalf("canary: %v %v", ok, err) + } + second := supervisor.NewPersistentLifeboat(store) + if ok, err := second.RollbackSet(); err != nil || !ok { + t.Fatalf("rollback: %v %v", ok, err) + } + state, err := second.Snapshot() + if err != nil { + t.Fatal(err) + } + if state.StableReleaseSet == nil || state.StableReleaseSet.Revision != "r1" || state.TrafficMode != supervisor.TrafficStable { + t.Fatalf("state=%+v", state) + } +}