merge GEN2-DEPLOY-002 supervised release set
CI / test (push) Failing after 31s

This commit is contained in:
m
2026-08-30 11:57:07 +03:00
9 changed files with 528 additions and 23 deletions
+89 -3
View File
@@ -53,8 +53,11 @@ func main() {
control := http.NewServeMux()
control.HandleFunc("GET /v1/state", s.state)
control.HandleFunc("POST /v1/releases/start", s.start)
control.HandleFunc("POST /v1/releases/start-set", s.startSet)
control.HandleFunc("POST /v1/releases/baseline", s.baseline)
control.HandleFunc("POST /v1/releases/baseline-set", s.baselineSet)
control.HandleFunc("POST /v1/releases/observe", s.observe)
control.HandleFunc("POST /v1/releases/observe-set", s.observeSet)
control.HandleFunc("POST /v1/releases/promote", s.promote)
control.HandleFunc("POST /v1/releases/rollback", s.rollback)
control.HandleFunc("POST /v1/traffic/canary", s.canary)
@@ -81,6 +84,10 @@ func (s *server) ready(w http.ResponseWriter, r *http.Request) {
http.Error(w, "supervisor state unavailable", http.StatusServiceUnavailable)
return
}
if state.StableReleaseSet != nil && !s.checkReleaseSet(r, state.StableReleaseSet) {
http.Error(w, "stable release set unavailable", http.StatusServiceUnavailable)
return
}
request, err := http.NewRequestWithContext(r.Context(), http.MethodGet, state.StableUpstream+"/healthz", nil)
if err != nil {
http.Error(w, "stable upstream unavailable", http.StatusServiceUnavailable)
@@ -98,6 +105,28 @@ func (s *server) ready(w http.ResponseWriter, r *http.Request) {
writeJSON(w, map[string]string{"runtime": "supervisor", "status": "ready"})
}
func (s *server) checkReleaseSet(r *http.Request, releaseSet *supervisor.ReleaseSet) bool {
if releaseSet == nil || !releaseSet.Validate() {
return false
}
for _, name := range releaseSet.ComponentNames() {
component := releaseSet.Components[name]
request, err := http.NewRequestWithContext(r.Context(), http.MethodGet, component.ReadyURL, nil)
if err != nil {
return false
}
response, err := s.client.Do(request)
if err != nil || response.StatusCode/100 != 2 {
if response != nil {
response.Body.Close()
}
return false
}
response.Body.Close()
}
return true
}
func (s *server) proxy(w http.ResponseWriter, r *http.Request) {
state, err := s.lifeboat.Snapshot()
if err != nil {
@@ -142,6 +171,14 @@ func (s *server) start(w http.ResponseWriter, r *http.Request) {
ok, err := s.lifeboat.Start(release)
respond(w, ok, err)
}
func (s *server) startSet(w http.ResponseWriter, r *http.Request) {
var releaseSet supervisor.ReleaseSet
if !decode(w, r, &releaseSet) {
return
}
ok, err := s.lifeboat.StartSet(releaseSet)
respond(w, ok, err)
}
func (s *server) baseline(w http.ResponseWriter, r *http.Request) {
var release supervisor.Release
if !decode(w, r, &release) {
@@ -150,6 +187,14 @@ func (s *server) baseline(w http.ResponseWriter, r *http.Request) {
ok, err := s.lifeboat.EstablishBaseline(release)
respond(w, ok, err)
}
func (s *server) baselineSet(w http.ResponseWriter, r *http.Request) {
var releaseSet supervisor.ReleaseSet
if !decode(w, r, &releaseSet) {
return
}
ok, err := s.lifeboat.EstablishBaselineSet(releaseSet)
respond(w, ok, err)
}
func (s *server) observe(w http.ResponseWriter, r *http.Request) {
var observation supervisor.Observation
if !decode(w, r, &observation) {
@@ -158,16 +203,57 @@ func (s *server) observe(w http.ResponseWriter, r *http.Request) {
ok, err := s.lifeboat.Record(observation)
respond(w, ok, err)
}
func (s *server) observeSet(w http.ResponseWriter, r *http.Request) {
var observation supervisor.ReleaseSetObservation
if !decode(w, r, &observation) {
return
}
ok, err := s.lifeboat.RecordSet(observation)
respond(w, ok, err)
}
func (s *server) promote(w http.ResponseWriter, _ *http.Request) {
ok, err := s.lifeboat.Promote()
state, snapshotErr := s.lifeboat.Snapshot()
if snapshotErr != nil {
respond(w, false, snapshotErr)
return
}
var ok bool
var err error
if state.CandidateReleaseSet != nil {
ok, err = s.lifeboat.PromoteSet()
} else {
ok, err = s.lifeboat.Promote()
}
respond(w, ok, err)
}
func (s *server) rollback(w http.ResponseWriter, _ *http.Request) {
ok, err := s.lifeboat.Rollback()
state, snapshotErr := s.lifeboat.Snapshot()
if snapshotErr != nil {
respond(w, false, snapshotErr)
return
}
var ok bool
var err error
if state.CandidateReleaseSet != nil || state.RollbackTargetReleaseSet != nil {
ok, err = s.lifeboat.RollbackSet()
} else {
ok, err = s.lifeboat.Rollback()
}
respond(w, ok, err)
}
func (s *server) canary(w http.ResponseWriter, _ *http.Request) {
ok, err := s.lifeboat.SetTraffic(supervisor.TrafficCanaryHeader, s.stableUpstream, s.candidate)
state, snapshotErr := s.lifeboat.Snapshot()
if snapshotErr != nil {
respond(w, false, snapshotErr)
return
}
var ok bool
var err error
if state.CandidateReleaseSet != nil {
ok, err = s.lifeboat.SetTrafficForSets(supervisor.TrafficCanaryHeader)
} else {
ok, err = s.lifeboat.SetTraffic(supervisor.TrafficCanaryHeader, s.stableUpstream, s.candidate)
}
respond(w, ok, err)
}
func (s *server) promoteTraffic(w http.ResponseWriter, _ *http.Request) {
+51
View File
@@ -83,6 +83,22 @@ services:
depends_on: { postgres: { condition: service_healthy } }
volumes:
- assistant_core_secrets:/run/secrets:ro
healthcheck:
test: ["CMD-SHELL", "wget -qO- http://localhost:8080/healthz >/dev/null && wget -qO- http://localhost:8080/readyz >/dev/null"]
interval: 10s
timeout: 20s
retries: 20
core-runtime-candidate:
profiles: ["assistant-canary"]
extends:
service: core-runtime
restart: "no"
environment:
RESEARCH_RUNTIME_URL: http://research-runtime-candidate:8087
depends_on:
postgres: { condition: service_healthy }
research-runtime-candidate: { condition: service_started }
agent-canary:
profiles: ["agent-canary"]
@@ -201,6 +217,19 @@ services:
volumes:
- assistant_core_secrets:/run/secrets:ro
- assistant_research_sources:/var/lib/research-sources:ro
healthcheck:
test: ["CMD-SHELL", "wget -qO- http://localhost:8087/healthz >/dev/null && wget -qO- http://localhost:8087/readyz >/dev/null"]
interval: 10s
timeout: 10s
retries: 20
research-runtime-candidate:
profiles: ["assistant-canary"]
extends:
service: research-runtime
restart: "no"
depends_on:
postgres: { condition: service_healthy }
continuity-runtime:
build: { context: .., dockerfile: deploy/images/runtime.Dockerfile, args: { TARGET: continuity-runtime, SOURCE_REVISION: "${ASSISTANT_SOURCE_REVISION:-unknown}" } }
@@ -270,6 +299,17 @@ services:
timeout: 20s
retries: 20
cognitive-worker-candidate:
profiles: ["assistant-canary"]
extends:
service: cognitive-worker
restart: "no"
environment:
CORE_RUNTIME_URL: http://core-runtime-candidate:8080
depends_on:
core-runtime-candidate: { condition: service_started }
cognitive-lifeboat: { condition: service_healthy }
cognitive-lifeboat:
image: ghcr.io/ggml-org/llama.cpp:server
restart: unless-stopped
@@ -351,6 +391,17 @@ services:
extends:
service: creator-web
restart: "no"
environment:
CORE_RUNTIME_URL: http://core-runtime-candidate:8080
CONTINUITY_RUNTIME_URL: http://continuity-runtime:8081
COMMIT_RUNTIME_URL: http://commit-runtime:8082
COGNITIVE_RUNTIME_URL: http://cognitive-worker-candidate:8083
AGENT_WORKER_URL: http://agent-worker:8090
depends_on:
core-runtime-candidate: { condition: service_started }
continuity-runtime: { condition: service_started }
commit-runtime: { condition: service_started }
cognitive-worker-candidate: { condition: service_healthy }
volumes:
assistant_pg:
+50 -12
View File
@@ -16,22 +16,60 @@ control() { curl -fsS --max-time 10 -X POST "$SUPERVISOR_CONTROL_URL$1" "${@:2}"
fail_rollback() {
control /v1/traffic/rollback || true
control /v1/releases/rollback || true
echo "production canary failed; stable traffic rollback requested" >&2
echo "production ReleaseSet canary failed; stable ReleaseSet rollback requested" >&2
exit 1
}
compose --profile assistant-canary build --build-arg "SOURCE_REVISION=$REVISION" creator-web-candidate
compose --profile assistant-canary up -d --no-build creator-web-candidate
for attempt in $(seq 1 30); do
status="$(docker inspect -f '{{if .State.Health}}{{.State.Health.Status}}{{else}}unknown{{end}}' assistant-creator-web-candidate-1 2>/dev/null || true)"
[[ "$status" == healthy ]] && break
[[ "$attempt" == 30 ]] && fail_rollback
sleep 2
services=(creator-web-candidate core-runtime-candidate cognitive-worker-candidate research-runtime-candidate)
containers=(assistant-creator-web-candidate-1 assistant-core-runtime-candidate-1 assistant-cognitive-worker-candidate-1 assistant-research-runtime-candidate-1)
compose --profile assistant-canary build --build-arg "SOURCE_REVISION=$REVISION" "${services[@]}"
compose --profile assistant-canary up -d --no-build "${services[@]}"
for container in "${containers[@]}"; do
ready=""
for attempt in $(seq 1 60); do
status="$(docker inspect -f '{{if .State.Health}}{{.State.Health.Status}}{{else}}unknown{{end}}' "$container" 2>/dev/null || true)"
if [[ "$status" == healthy ]]; then ready=true; break; fi
sleep 2
done
[[ "$ready" == true ]] || fail_rollback
image_revision="$(docker inspect -f '{{index .Config.Labels "org.opencontainers.image.revision"}}' "$container" 2>/dev/null || true)"
[[ "$image_revision" == "$REVISION" ]] || fail_rollback
done
control /v1/releases/baseline -H 'Content-Type: application/json' --data "{\"id\":\"stable-$REVISION\",\"version\":\"$REVISION\"}" || fail_rollback
control /v1/releases/start -H 'Content-Type: application/json' --data "{\"id\":\"candidate-$REVISION\",\"version\":\"$REVISION\"}" || fail_rollback
control /v1/releases/observe -H 'Content-Type: application/json' --data "{\"release_id\":\"candidate-$REVISION\",\"ready\":true,\"reason\":\"candidate healthcheck passed\",\"observed_at_unix_millis\":$(date +%s%3N)}" || fail_rollback
release_json="$(cat <<JSON
{
"id":"candidate-$REVISION",
"revision":"$REVISION",
"components":{
"creator-web":{"name":"creator-web","revision":"$REVISION","upstream":"http://creator-web-candidate:8084","health_url":"http://creator-web-candidate:8084/healthz","ready_url":"http://creator-web-candidate:8084/readyz"},
"core-runtime":{"name":"core-runtime","revision":"$REVISION","upstream":"http://core-runtime-candidate:8080","health_url":"http://core-runtime-candidate:8080/healthz","ready_url":"http://core-runtime-candidate:8080/readyz"},
"cognitive-worker":{"name":"cognitive-worker","revision":"$REVISION","upstream":"http://cognitive-worker-candidate:8083","health_url":"http://cognitive-worker-candidate:8083/healthz","ready_url":"http://cognitive-worker-candidate:8083/readyz"},
"research-runtime":{"name":"research-runtime","revision":"$REVISION","upstream":"http://research-runtime-candidate:8087","health_url":"http://research-runtime-candidate:8087/healthz","ready_url":"http://research-runtime-candidate:8087/readyz"}
}
}
JSON
)
stable_json="$(curl -fsS --max-time 10 "$SUPERVISOR_CONTROL_URL/v1/state")"
stable_set="$(printf '%s' "$stable_json" | python3 -c 'import json,sys; s=json.load(sys.stdin).get("stable_release_set"); print(json.dumps(s,separators=(",",":")) if s else "")')"
if [[ -z "$stable_set" ]]; then
stable_revision="$(printf '%s' "$stable_json" | python3 -c 'import json,sys; print(json.load(sys.stdin).get("current",{}).get("version", "unknown"))')"
stable_set="$(cat <<JSON
{"id":"stable-$stable_revision","revision":"$stable_revision","components":{
"creator-web":{"name":"creator-web","revision":"$stable_revision","upstream":"http://creator-web:8084","health_url":"http://creator-web:8084/healthz","ready_url":"http://creator-web:8084/readyz"},
"core-runtime":{"name":"core-runtime","revision":"$stable_revision","upstream":"http://core-runtime:8080","health_url":"http://core-runtime:8080/healthz","ready_url":"http://core-runtime:8080/readyz"},
"cognitive-worker":{"name":"cognitive-worker","revision":"$stable_revision","upstream":"http://cognitive-worker:8083","health_url":"http://cognitive-worker:8083/healthz","ready_url":"http://cognitive-worker:8083/readyz"},
"research-runtime":{"name":"research-runtime","revision":"$stable_revision","upstream":"http://research-runtime:8087","health_url":"http://research-runtime:8087/healthz","ready_url":"http://research-runtime:8087/readyz"}}}
JSON
)"
curl -fsS --max-time 10 -X POST "$SUPERVISOR_CONTROL_URL/v1/releases/baseline-set" -H 'Content-Type: application/json' --data "$stable_set" >/dev/null || fail_rollback
fi
curl -fsS --max-time 10 -X POST "$SUPERVISOR_CONTROL_URL/v1/releases/start-set" -H 'Content-Type: application/json' --data "$release_json" >/dev/null || fail_rollback
observation="$(printf '%s' "$release_json" | python3 -c 'import json,sys,time; s=json.load(sys.stdin); print(json.dumps({"release_id":s["id"],"ready":True,"reason":"all ReleaseSet healthchecks and image revisions passed","observed_at_unix_millis":int(time.time()*1000),"components":{k:{"revision":v["revision"],"health":True,"ready":True} for k,v in s["components"].items()}},separators=(",",":")))')"
curl -fsS --max-time 10 -X POST "$SUPERVISOR_CONTROL_URL/v1/releases/observe-set" -H 'Content-Type: application/json' --data "$observation" >/dev/null || fail_rollback
control /v1/traffic/canary || fail_rollback
headers="$(mktemp)"
@@ -40,4 +78,4 @@ if ! curl -fsS --max-time 10 -D "$headers" -H 'X-Assistant-Canary: true' "https:
grep -qi '^X-Assistant-Traffic-Target: candidate' "$headers" || fail_rollback
control /v1/releases/promote || fail_rollback
echo "production canary promoted revision=$REVISION"
echo "production ReleaseSet canary promoted revision=$REVISION"
+1 -1
View File
@@ -1 +1 @@
{"task_file":"dev/tasks/GEN2-COMPAT-OBS-001-production-compatibility-observation.json","task_id":"GEN2-COMPAT-OBS-001","task_revision":1}
{"task_file":"dev/tasks/GEN2-DEPLOY-002-supervised-release-set.json","task_id":"GEN2-DEPLOY-002","task_revision":1}
@@ -0,0 +1,23 @@
{
"task_id": "GEN2-DEPLOY-002",
"task_revision": 1,
"architecture_revision": "ARCH-GEN2-000",
"base_repo_revision": "gen2-integration",
"risk_tier": "E",
"intent": "Расширить существующий Supervisor до безопасного управления единым candidate/stable ReleaseSet из creator-web, core-runtime, cognitive-worker и research-runtime без изменения Gen2 semantic ownership.",
"requirement_ids": ["REQ-GEN2-DEPLOY-002-RELEASE-SET", "REQ-GEN2-DEPLOY-002-CANDIDATE-TOPOLOGY", "REQ-GEN2-DEPLOY-002-ROLLBACK", "REQ-GEN2-DEPLOY-002-COMPAT-OBS"],
"allowed_change_scope": {"modules": ["runtime/supervisor"], "paths": ["runtime/supervisor/**", "cmd/supervisor/**", "deploy/**", "docs/**", "tests/**", "dev/ACTIVE_TASK.json", "dev/tasks/GEN2-DEPLOY-002-supervised-release-set.json"]},
"forbidden_change_scope": {"paths": ["architecture/**", "database/**", "modules/**", "contracts/**", "security/**", "go.mod", "go.sum", ".gitea/workflows/**", "adapters/models/**"]},
"allowed_architecture_delta": {"new_cross_module_dependencies": 0, "new_canonical_owners": 0, "new_privileges": 1, "new_third_party_dependencies": 0, "new_database_owner_roles": 0},
"contracts": {"consume": ["existing Supervisor durable state and production compose", "existing canary header and stable rollback controls"], "may_change": ["Supervisor release control payloads", "deployment topology and candidate routing configuration"]},
"semantic_owner": "Supervisor/deployment control plane; ReleaseSet is operational release state and not semantic canonical state.",
"canonical_state": ["no semantic owner or schema change", "previous stable release and evidence remain retained"],
"state_transitions": ["candidate ReleaseSet -> component health/revision checks -> canary -> observation -> promotion or whole-set rollback"],
"side_effects_and_privileges": ["production deployment only through approved control plane; no direct stable replacement; no credential reads"],
"persistence_and_migrations": ["durable Supervisor release state remains in existing external Supervisor volume; no database migration"],
"assumptions": ["the existing production Supervisor and compose control plane remain authoritative for deployment", "candidate services may share existing durable infrastructure only under current ownership and compatibility rules", "no credentials are read by this task"],
"failure_retry_restart": ["candidate component failure routes all traffic to previous stable ReleaseSet", "Supervisor restart reloads ReleaseSet state", "previous stable is retained until promotion is complete"],
"acceptance_scenarios": ["DEPLOY-002-SCN-01: complete ReleaseSet has all four components and one revision", "DEPLOY-002-SCN-02: candidate dependencies resolve only to candidate services", "DEPLOY-002-SCN-03: health/readiness/revision checks cover all required components", "DEPLOY-002-SCN-04: canary and stable paths are isolated", "DEPLOY-002-SCN-05: forced candidate failure rolls back the complete ReleaseSet", "DEPLOY-002-SCN-06: Supervisor restart preserves stable/candidate/LKG state", "DEPLOY-002-SCN-07: production telemetry revision starts bounded observation after controlled deployment"],
"required_tests": ["go test ./runtime/supervisor ./cmd/supervisor", "FAST: targeted supervisor/release-set tests", "FAST: compose config validation", "FAST: routing and revision consistency tests", "FAST: rollback/restart persistence tests", "FAST: targeted go vet", "FAST governance: preflight/check-diff/independent", "FAST whitespace: git diff --check", "PRODUCTION: candidate ReleaseSet drill and telemetry observation start"],
"terminal_gap_states": ["BLOCKED_BY_SPEC_GAP", "BLOCKED_BY_ARCHITECTURE_GAP", "BLOCKED_BY_MISSING_CAPABILITY", "BLOCKED_BY_CONFLICT", "REQUIRES_MIGRATION_DECISION"]
}
@@ -0,0 +1,40 @@
{
"evidence_id": "gen2-deploy-002-blocked-2026-08-30",
"task_id": "GEN2-DEPLOY-002",
"task_revision": 1,
"base_repo_revision": "gen2-integration",
"status": "BLOCKED_BY_MISSING_CAPABILITY",
"implementation_checks": {
"release_set_tests": "PASS",
"supervisor_go_test": "PASS",
"supervisor_go_vet": "PASS",
"compose_config": "PASS",
"canary_script_syntax": "PASS",
"governance": "PASS",
"git_diff_check": "PASS"
},
"implemented": [
"durable Supervisor ReleaseSet state for creator-web, core-runtime, cognitive-worker, and research-runtime",
"parallel candidate services with candidate-only core/cognitive/research dependency wiring",
"whole-set health/readiness/revision observation, canary, promotion and rollback controls",
"stable/candidate/LKG persistence across Supervisor restart",
"core and research runtime compose healthchecks"
],
"production": {
"deployment": "NOT_RUN",
"candidate_release_set": "NOT_STARTED",
"telemetry_observation": "NOT_STARTED",
"stable_mutation": "NONE"
},
"capability_evidence": {
"docker_and_compose_on_production_host": "AVAILABLE",
"supervisor_control_endpoint": "REACHABLE",
"portainer_api_key_in_current_execution_context": "NOT_CONFIGURED",
"portainer_endpoint_metadata_in_current_execution_context": "NOT_CONFIGURED",
"existing_production_candidate_services": ["creator-web-candidate"],
"missing_capability": "Approved production deployment automation or candidate-capable control-plane access for the four-component ReleaseSet."
},
"unsafe_action_not_taken": "Stable containers were not replaced directly and no production compose mutation was attempted.",
"compatibility": "No V1 compatibility path was removed or disabled; no schema, migration, semantic owner, or Gen3 change.",
"next_action": "Owner must provide or enable the approved candidate-capable production deployment mechanism; then run the ReleaseSet drill and start bounded telemetry observation."
}
+24
View File
@@ -92,12 +92,24 @@ func (p *PersistentLifeboat) Start(candidate Release) (bool, error) {
return p.transition(func(value *Lifeboat) bool { return value.Start(candidate) })
}
func (p *PersistentLifeboat) StartSet(candidate ReleaseSet) (bool, error) {
return p.transition(func(value *Lifeboat) bool { return value.StartSet(candidate) })
}
func (p *PersistentLifeboat) EstablishBaseline(release Release) (bool, error) {
return p.transition(func(value *Lifeboat) bool { return value.EstablishBaseline(release) })
}
func (p *PersistentLifeboat) EstablishBaselineSet(release ReleaseSet) (bool, error) {
return p.transition(func(value *Lifeboat) bool { return value.EstablishBaselineSet(release) })
}
func (p *PersistentLifeboat) Record(observation Observation) (bool, error) {
return p.transition(func(value *Lifeboat) bool { return value.Record(observation) })
}
func (p *PersistentLifeboat) RecordSet(observation ReleaseSetObservation) (bool, error) {
return p.transition(func(value *Lifeboat) bool { return value.RecordSet(observation) })
}
func (p *PersistentLifeboat) Rollback() (bool, error) {
return p.transition(func(value *Lifeboat) bool { return value.Rollback() })
}
@@ -105,6 +117,10 @@ func (p *PersistentLifeboat) SetTraffic(mode TrafficMode, stableUpstream, candid
return p.transition(func(value *Lifeboat) bool { return value.SetTraffic(mode, stableUpstream, candidateUpstream) })
}
func (p *PersistentLifeboat) SetTrafficForSets(mode TrafficMode) (bool, error) {
return p.transition(func(value *Lifeboat) bool { return value.SetTrafficForSets(mode) })
}
func (p *PersistentLifeboat) Promote() (bool, error) {
return p.transition(func(value *Lifeboat) bool {
if value.Current.State != ReleaseHealthy || value.Current.ID == "" {
@@ -114,6 +130,10 @@ func (p *PersistentLifeboat) Promote() (bool, error) {
})
}
func (p *PersistentLifeboat) PromoteSet() (bool, error) {
return p.transition(func(value *Lifeboat) bool { return value.PromoteSet() })
}
func (p *PersistentLifeboat) RollbackTraffic() (bool, error) {
return p.transition(func(value *Lifeboat) bool {
if !value.Rollback() {
@@ -123,6 +143,10 @@ func (p *PersistentLifeboat) RollbackTraffic() (bool, error) {
})
}
func (p *PersistentLifeboat) RollbackSet() (bool, error) {
return p.transition(func(value *Lifeboat) bool { return value.RollbackSet() })
}
func (p *PersistentLifeboat) Snapshot() (Lifeboat, error) {
p.mu.Lock()
defer p.mu.Unlock()
+193 -7
View File
@@ -1,5 +1,7 @@
package supervisor
import "sort"
type ReleaseState string
const (
@@ -22,21 +24,205 @@ type Release struct {
Version string `json:"version"`
State ReleaseState `json:"state"`
LastKnownGood bool `json:"last_known_good"`
Set *ReleaseSet `json:"release_set,omitempty"`
}
const (
ComponentCreatorWeb = "creator-web"
ComponentCoreRuntime = "core-runtime"
ComponentCognitiveWorker = "cognitive-worker"
ComponentResearchRuntime = "research-runtime"
)
// ComponentRelease is operational deployment evidence. It is deliberately
// separate from semantic/domain state and contains no credentials.
type ComponentRelease struct {
Name string `json:"name"`
Revision string `json:"revision"`
Upstream string `json:"upstream"`
HealthURL string `json:"health_url"`
ReadyURL string `json:"ready_url"`
}
// ReleaseSet is the atomic deployment unit supervised by the control plane.
type ReleaseSet struct {
ID string `json:"id"`
Revision string `json:"revision"`
Components map[string]ComponentRelease `json:"components"`
}
var requiredComponents = []string{ComponentCreatorWeb, ComponentCoreRuntime, ComponentCognitiveWorker, ComponentResearchRuntime}
func (s ReleaseSet) Validate() bool {
if s.ID == "" || s.Revision == "" || len(s.Components) != len(requiredComponents) {
return false
}
for _, name := range requiredComponents {
component, ok := s.Components[name]
if !ok || component.Name != name || component.Revision != s.Revision || component.Upstream == "" || component.HealthURL == "" || component.ReadyURL == "" {
return false
}
}
return true
}
func (s ReleaseSet) ComponentNames() []string {
names := make([]string, 0, len(s.Components))
for name := range s.Components {
names = append(names, name)
}
sort.Strings(names)
return names
}
type Observation struct {
ReleaseID string `json:"release_id"`
Ready bool `json:"ready"`
Reason string `json:"reason"`
ObservedAtUnixMillis int64 `json:"observed_at_unix_millis"`
}
type ReleaseSetObservation struct {
ReleaseID string `json:"release_id"`
Ready bool `json:"ready"`
Reason string `json:"reason"`
ObservedAtUnixMillis int64 `json:"observed_at_unix_millis"`
Components map[string]ComponentObservation `json:"components"`
}
type ComponentObservation struct {
Revision string `json:"revision"`
Health bool `json:"health"`
Ready bool `json:"ready"`
}
type Lifeboat struct {
Current Release `json:"current"`
LastKnownGood Release `json:"last_known_good"`
RollbackTarget Release `json:"rollback_target"`
Observations []Observation `json:"observations"`
TrafficMode TrafficMode `json:"traffic_mode"`
StableUpstream string `json:"stable_upstream"`
CandidateUpstream string `json:"candidate_upstream"`
Current Release `json:"current"`
LastKnownGood Release `json:"last_known_good"`
RollbackTarget Release `json:"rollback_target"`
Observations []Observation `json:"observations"`
TrafficMode TrafficMode `json:"traffic_mode"`
StableUpstream string `json:"stable_upstream"`
CandidateUpstream string `json:"candidate_upstream"`
StableReleaseSet *ReleaseSet `json:"stable_release_set,omitempty"`
CandidateReleaseSet *ReleaseSet `json:"candidate_release_set,omitempty"`
LastKnownGoodReleaseSet *ReleaseSet `json:"last_known_good_release_set,omitempty"`
RollbackTargetReleaseSet *ReleaseSet `json:"rollback_target_release_set,omitempty"`
}
func (l *Lifeboat) StartSet(candidate ReleaseSet) bool {
if !candidate.Validate() {
return false
}
copySet := candidate
if l.LastKnownGoodReleaseSet != nil {
previous := *l.LastKnownGoodReleaseSet
l.RollbackTargetReleaseSet = &previous
}
l.CandidateReleaseSet = &copySet
l.Current = Release{ID: candidate.ID, Version: candidate.Revision, State: ReleaseCandidate, Set: &copySet}
return true
}
func (l *Lifeboat) EstablishBaselineSet(stable ReleaseSet) bool {
if !stable.Validate() {
return false
}
copySet := stable
l.StableReleaseSet = &copySet
l.LastKnownGoodReleaseSet = &copySet
l.Current = Release{ID: stable.ID, Version: stable.Revision, State: ReleaseHealthy, LastKnownGood: true, Set: &copySet}
l.LastKnownGood = l.Current
l.RollbackTarget = Release{}
l.RollbackTargetReleaseSet = nil
l.TrafficMode = TrafficStable
return l.setSetUpstreams(&copySet, nil)
}
func (l *Lifeboat) setSetUpstreams(stable, candidate *ReleaseSet) bool {
if stable == nil {
return false
}
stableWeb, ok := stable.Components[ComponentCreatorWeb]
if !ok {
return false
}
l.StableUpstream = stableWeb.Upstream
if candidate != nil {
candidateWeb, exists := candidate.Components[ComponentCreatorWeb]
if !exists {
return false
}
l.CandidateUpstream = candidateWeb.Upstream
}
return true
}
func (l *Lifeboat) RecordSet(o ReleaseSetObservation) bool {
if l.CandidateReleaseSet == nil || o.ReleaseID == "" || o.ReleaseID != l.CandidateReleaseSet.ID || !o.Ready {
l.Current.State = ReleaseFailed
return false
}
if len(o.Components) != len(requiredComponents) {
l.Current.State = ReleaseFailed
return false
}
for _, name := range requiredComponents {
component, ok := o.Components[name]
expected := l.CandidateReleaseSet.Components[name]
if !ok || !component.Health || !component.Ready || component.Revision != expected.Revision {
l.Current.State = ReleaseFailed
return false
}
}
l.Current.State = ReleaseHealthy
return true
}
func (l *Lifeboat) SetTrafficForSets(mode TrafficMode) bool {
if l.StableReleaseSet == nil || l.CandidateReleaseSet == nil {
return false
}
if !l.setSetUpstreams(l.StableReleaseSet, l.CandidateReleaseSet) {
return false
}
l.TrafficMode = mode
return true
}
func (l *Lifeboat) PromoteSet() bool {
if l.CandidateReleaseSet == nil || l.Current.State != ReleaseHealthy {
return false
}
set := *l.CandidateReleaseSet
l.StableReleaseSet = &set
l.LastKnownGoodReleaseSet = &set
l.LastKnownGood = l.Current
l.LastKnownGood.Set = &set
l.LastKnownGood.LastKnownGood = true
l.Current.Set = &set
l.Current.LastKnownGood = true
l.CandidateReleaseSet = nil
l.RollbackTargetReleaseSet = nil
return l.setSetUpstreams(&set, nil)
}
func (l *Lifeboat) RollbackSet() bool {
if l.RollbackTargetReleaseSet == nil && l.LastKnownGoodReleaseSet == nil {
return false
}
target := l.RollbackTargetReleaseSet
if target == nil {
target = l.LastKnownGoodReleaseSet
}
copySet := *target
l.StableReleaseSet = &copySet
l.Current = Release{ID: copySet.ID, Version: copySet.Revision, State: ReleaseRolledBack, LastKnownGood: true, Set: &copySet}
l.LastKnownGood = l.Current
l.LastKnownGoodReleaseSet = &copySet
l.CandidateReleaseSet = nil
l.RollbackTargetReleaseSet = nil
l.TrafficMode = TrafficStable
return l.setSetUpstreams(&copySet, nil)
}
func (l *Lifeboat) Start(candidate Release) bool {
+57
View File
@@ -82,3 +82,60 @@ func TestPersistentSupervisorRestoresTrafficModeAfterRestart(t *testing.T) {
t.Fatalf("rollback state=%+v", state)
}
}
func testReleaseSet(id, revision, web, core, cognitive, research string) supervisor.ReleaseSet {
return supervisor.ReleaseSet{ID: id, Revision: revision, Components: map[string]supervisor.ComponentRelease{
supervisor.ComponentCreatorWeb: {Name: supervisor.ComponentCreatorWeb, Revision: revision, Upstream: web, HealthURL: web + "/healthz", ReadyURL: web + "/readyz"},
supervisor.ComponentCoreRuntime: {Name: supervisor.ComponentCoreRuntime, Revision: revision, Upstream: core, HealthURL: core + "/healthz", ReadyURL: core + "/readyz"},
supervisor.ComponentCognitiveWorker: {Name: supervisor.ComponentCognitiveWorker, Revision: revision, Upstream: cognitive, HealthURL: cognitive + "/healthz", ReadyURL: cognitive + "/readyz"},
supervisor.ComponentResearchRuntime: {Name: supervisor.ComponentResearchRuntime, Revision: revision, Upstream: research, HealthURL: research + "/healthz", ReadyURL: research + "/readyz"},
}}
}
func TestReleaseSetRejectsMixedRevisionsAndMissingComponents(t *testing.T) {
set := testReleaseSet("candidate", "r1", "http://web", "http://core", "http://cognitive", "http://research")
if !set.Validate() {
t.Fatal("complete release set should validate")
}
set.Components[supervisor.ComponentCoreRuntime] = supervisor.ComponentRelease{Name: supervisor.ComponentCoreRuntime, Revision: "r0", Upstream: "http://core", HealthURL: "http://core/healthz", ReadyURL: "http://core/readyz"}
if set.Validate() {
t.Fatal("mixed revision release set must be rejected")
}
}
func TestPersistentReleaseSetRollbackPreservesWholeLastKnownGood(t *testing.T) {
store, err := supervisor.NewDurableStore(filepath.Join(t.TempDir(), "control", "release-set.json"))
if err != nil {
t.Fatal(err)
}
first := supervisor.NewPersistentLifeboat(store)
good := testReleaseSet("stable-r1", "r1", "http://web", "http://core", "http://cognitive", "http://research")
candidate := testReleaseSet("candidate-r2", "r2", "http://web-c", "http://core-c", "http://cognitive-c", "http://research-c")
if ok, err := first.EstablishBaselineSet(good); err != nil || !ok {
t.Fatalf("baseline: %v %v", ok, err)
}
if ok, err := first.StartSet(candidate); err != nil || !ok {
t.Fatalf("start: %v %v", ok, err)
}
observation := supervisor.ReleaseSetObservation{ReleaseID: candidate.ID, Ready: true, Components: map[string]supervisor.ComponentObservation{}}
for name := range candidate.Components {
observation.Components[name] = supervisor.ComponentObservation{Revision: candidate.Revision, Health: true, Ready: true}
}
if ok, err := first.RecordSet(observation); err != nil || !ok {
t.Fatalf("observe: %v %v", ok, err)
}
if ok, err := first.SetTrafficForSets(supervisor.TrafficCanaryHeader); err != nil || !ok {
t.Fatalf("canary: %v %v", ok, err)
}
second := supervisor.NewPersistentLifeboat(store)
if ok, err := second.RollbackSet(); err != nil || !ok {
t.Fatalf("rollback: %v %v", ok, err)
}
state, err := second.Snapshot()
if err != nil {
t.Fatal(err)
}
if state.StableReleaseSet == nil || state.StableReleaseSet.Revision != "r1" || state.TrafficMode != supervisor.TrafficStable {
t.Fatalf("state=%+v", state)
}
}