This commit is contained in:
+89
-3
@@ -53,8 +53,11 @@ func main() {
|
||||
control := http.NewServeMux()
|
||||
control.HandleFunc("GET /v1/state", s.state)
|
||||
control.HandleFunc("POST /v1/releases/start", s.start)
|
||||
control.HandleFunc("POST /v1/releases/start-set", s.startSet)
|
||||
control.HandleFunc("POST /v1/releases/baseline", s.baseline)
|
||||
control.HandleFunc("POST /v1/releases/baseline-set", s.baselineSet)
|
||||
control.HandleFunc("POST /v1/releases/observe", s.observe)
|
||||
control.HandleFunc("POST /v1/releases/observe-set", s.observeSet)
|
||||
control.HandleFunc("POST /v1/releases/promote", s.promote)
|
||||
control.HandleFunc("POST /v1/releases/rollback", s.rollback)
|
||||
control.HandleFunc("POST /v1/traffic/canary", s.canary)
|
||||
@@ -81,6 +84,10 @@ func (s *server) ready(w http.ResponseWriter, r *http.Request) {
|
||||
http.Error(w, "supervisor state unavailable", http.StatusServiceUnavailable)
|
||||
return
|
||||
}
|
||||
if state.StableReleaseSet != nil && !s.checkReleaseSet(r, state.StableReleaseSet) {
|
||||
http.Error(w, "stable release set unavailable", http.StatusServiceUnavailable)
|
||||
return
|
||||
}
|
||||
request, err := http.NewRequestWithContext(r.Context(), http.MethodGet, state.StableUpstream+"/healthz", nil)
|
||||
if err != nil {
|
||||
http.Error(w, "stable upstream unavailable", http.StatusServiceUnavailable)
|
||||
@@ -98,6 +105,28 @@ func (s *server) ready(w http.ResponseWriter, r *http.Request) {
|
||||
writeJSON(w, map[string]string{"runtime": "supervisor", "status": "ready"})
|
||||
}
|
||||
|
||||
func (s *server) checkReleaseSet(r *http.Request, releaseSet *supervisor.ReleaseSet) bool {
|
||||
if releaseSet == nil || !releaseSet.Validate() {
|
||||
return false
|
||||
}
|
||||
for _, name := range releaseSet.ComponentNames() {
|
||||
component := releaseSet.Components[name]
|
||||
request, err := http.NewRequestWithContext(r.Context(), http.MethodGet, component.ReadyURL, nil)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
response, err := s.client.Do(request)
|
||||
if err != nil || response.StatusCode/100 != 2 {
|
||||
if response != nil {
|
||||
response.Body.Close()
|
||||
}
|
||||
return false
|
||||
}
|
||||
response.Body.Close()
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func (s *server) proxy(w http.ResponseWriter, r *http.Request) {
|
||||
state, err := s.lifeboat.Snapshot()
|
||||
if err != nil {
|
||||
@@ -142,6 +171,14 @@ func (s *server) start(w http.ResponseWriter, r *http.Request) {
|
||||
ok, err := s.lifeboat.Start(release)
|
||||
respond(w, ok, err)
|
||||
}
|
||||
func (s *server) startSet(w http.ResponseWriter, r *http.Request) {
|
||||
var releaseSet supervisor.ReleaseSet
|
||||
if !decode(w, r, &releaseSet) {
|
||||
return
|
||||
}
|
||||
ok, err := s.lifeboat.StartSet(releaseSet)
|
||||
respond(w, ok, err)
|
||||
}
|
||||
func (s *server) baseline(w http.ResponseWriter, r *http.Request) {
|
||||
var release supervisor.Release
|
||||
if !decode(w, r, &release) {
|
||||
@@ -150,6 +187,14 @@ func (s *server) baseline(w http.ResponseWriter, r *http.Request) {
|
||||
ok, err := s.lifeboat.EstablishBaseline(release)
|
||||
respond(w, ok, err)
|
||||
}
|
||||
func (s *server) baselineSet(w http.ResponseWriter, r *http.Request) {
|
||||
var releaseSet supervisor.ReleaseSet
|
||||
if !decode(w, r, &releaseSet) {
|
||||
return
|
||||
}
|
||||
ok, err := s.lifeboat.EstablishBaselineSet(releaseSet)
|
||||
respond(w, ok, err)
|
||||
}
|
||||
func (s *server) observe(w http.ResponseWriter, r *http.Request) {
|
||||
var observation supervisor.Observation
|
||||
if !decode(w, r, &observation) {
|
||||
@@ -158,16 +203,57 @@ func (s *server) observe(w http.ResponseWriter, r *http.Request) {
|
||||
ok, err := s.lifeboat.Record(observation)
|
||||
respond(w, ok, err)
|
||||
}
|
||||
func (s *server) observeSet(w http.ResponseWriter, r *http.Request) {
|
||||
var observation supervisor.ReleaseSetObservation
|
||||
if !decode(w, r, &observation) {
|
||||
return
|
||||
}
|
||||
ok, err := s.lifeboat.RecordSet(observation)
|
||||
respond(w, ok, err)
|
||||
}
|
||||
func (s *server) promote(w http.ResponseWriter, _ *http.Request) {
|
||||
ok, err := s.lifeboat.Promote()
|
||||
state, snapshotErr := s.lifeboat.Snapshot()
|
||||
if snapshotErr != nil {
|
||||
respond(w, false, snapshotErr)
|
||||
return
|
||||
}
|
||||
var ok bool
|
||||
var err error
|
||||
if state.CandidateReleaseSet != nil {
|
||||
ok, err = s.lifeboat.PromoteSet()
|
||||
} else {
|
||||
ok, err = s.lifeboat.Promote()
|
||||
}
|
||||
respond(w, ok, err)
|
||||
}
|
||||
func (s *server) rollback(w http.ResponseWriter, _ *http.Request) {
|
||||
ok, err := s.lifeboat.Rollback()
|
||||
state, snapshotErr := s.lifeboat.Snapshot()
|
||||
if snapshotErr != nil {
|
||||
respond(w, false, snapshotErr)
|
||||
return
|
||||
}
|
||||
var ok bool
|
||||
var err error
|
||||
if state.CandidateReleaseSet != nil || state.RollbackTargetReleaseSet != nil {
|
||||
ok, err = s.lifeboat.RollbackSet()
|
||||
} else {
|
||||
ok, err = s.lifeboat.Rollback()
|
||||
}
|
||||
respond(w, ok, err)
|
||||
}
|
||||
func (s *server) canary(w http.ResponseWriter, _ *http.Request) {
|
||||
ok, err := s.lifeboat.SetTraffic(supervisor.TrafficCanaryHeader, s.stableUpstream, s.candidate)
|
||||
state, snapshotErr := s.lifeboat.Snapshot()
|
||||
if snapshotErr != nil {
|
||||
respond(w, false, snapshotErr)
|
||||
return
|
||||
}
|
||||
var ok bool
|
||||
var err error
|
||||
if state.CandidateReleaseSet != nil {
|
||||
ok, err = s.lifeboat.SetTrafficForSets(supervisor.TrafficCanaryHeader)
|
||||
} else {
|
||||
ok, err = s.lifeboat.SetTraffic(supervisor.TrafficCanaryHeader, s.stableUpstream, s.candidate)
|
||||
}
|
||||
respond(w, ok, err)
|
||||
}
|
||||
func (s *server) promoteTraffic(w http.ResponseWriter, _ *http.Request) {
|
||||
|
||||
@@ -83,6 +83,22 @@ services:
|
||||
depends_on: { postgres: { condition: service_healthy } }
|
||||
volumes:
|
||||
- assistant_core_secrets:/run/secrets:ro
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "wget -qO- http://localhost:8080/healthz >/dev/null && wget -qO- http://localhost:8080/readyz >/dev/null"]
|
||||
interval: 10s
|
||||
timeout: 20s
|
||||
retries: 20
|
||||
|
||||
core-runtime-candidate:
|
||||
profiles: ["assistant-canary"]
|
||||
extends:
|
||||
service: core-runtime
|
||||
restart: "no"
|
||||
environment:
|
||||
RESEARCH_RUNTIME_URL: http://research-runtime-candidate:8087
|
||||
depends_on:
|
||||
postgres: { condition: service_healthy }
|
||||
research-runtime-candidate: { condition: service_started }
|
||||
|
||||
agent-canary:
|
||||
profiles: ["agent-canary"]
|
||||
@@ -201,6 +217,19 @@ services:
|
||||
volumes:
|
||||
- assistant_core_secrets:/run/secrets:ro
|
||||
- assistant_research_sources:/var/lib/research-sources:ro
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "wget -qO- http://localhost:8087/healthz >/dev/null && wget -qO- http://localhost:8087/readyz >/dev/null"]
|
||||
interval: 10s
|
||||
timeout: 10s
|
||||
retries: 20
|
||||
|
||||
research-runtime-candidate:
|
||||
profiles: ["assistant-canary"]
|
||||
extends:
|
||||
service: research-runtime
|
||||
restart: "no"
|
||||
depends_on:
|
||||
postgres: { condition: service_healthy }
|
||||
|
||||
continuity-runtime:
|
||||
build: { context: .., dockerfile: deploy/images/runtime.Dockerfile, args: { TARGET: continuity-runtime, SOURCE_REVISION: "${ASSISTANT_SOURCE_REVISION:-unknown}" } }
|
||||
@@ -270,6 +299,17 @@ services:
|
||||
timeout: 20s
|
||||
retries: 20
|
||||
|
||||
cognitive-worker-candidate:
|
||||
profiles: ["assistant-canary"]
|
||||
extends:
|
||||
service: cognitive-worker
|
||||
restart: "no"
|
||||
environment:
|
||||
CORE_RUNTIME_URL: http://core-runtime-candidate:8080
|
||||
depends_on:
|
||||
core-runtime-candidate: { condition: service_started }
|
||||
cognitive-lifeboat: { condition: service_healthy }
|
||||
|
||||
cognitive-lifeboat:
|
||||
image: ghcr.io/ggml-org/llama.cpp:server
|
||||
restart: unless-stopped
|
||||
@@ -351,6 +391,17 @@ services:
|
||||
extends:
|
||||
service: creator-web
|
||||
restart: "no"
|
||||
environment:
|
||||
CORE_RUNTIME_URL: http://core-runtime-candidate:8080
|
||||
CONTINUITY_RUNTIME_URL: http://continuity-runtime:8081
|
||||
COMMIT_RUNTIME_URL: http://commit-runtime:8082
|
||||
COGNITIVE_RUNTIME_URL: http://cognitive-worker-candidate:8083
|
||||
AGENT_WORKER_URL: http://agent-worker:8090
|
||||
depends_on:
|
||||
core-runtime-candidate: { condition: service_started }
|
||||
continuity-runtime: { condition: service_started }
|
||||
commit-runtime: { condition: service_started }
|
||||
cognitive-worker-candidate: { condition: service_healthy }
|
||||
|
||||
volumes:
|
||||
assistant_pg:
|
||||
|
||||
@@ -16,22 +16,60 @@ control() { curl -fsS --max-time 10 -X POST "$SUPERVISOR_CONTROL_URL$1" "${@:2}"
|
||||
fail_rollback() {
|
||||
control /v1/traffic/rollback || true
|
||||
control /v1/releases/rollback || true
|
||||
echo "production canary failed; stable traffic rollback requested" >&2
|
||||
echo "production ReleaseSet canary failed; stable ReleaseSet rollback requested" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
compose --profile assistant-canary build --build-arg "SOURCE_REVISION=$REVISION" creator-web-candidate
|
||||
compose --profile assistant-canary up -d --no-build creator-web-candidate
|
||||
for attempt in $(seq 1 30); do
|
||||
status="$(docker inspect -f '{{if .State.Health}}{{.State.Health.Status}}{{else}}unknown{{end}}' assistant-creator-web-candidate-1 2>/dev/null || true)"
|
||||
[[ "$status" == healthy ]] && break
|
||||
[[ "$attempt" == 30 ]] && fail_rollback
|
||||
sleep 2
|
||||
services=(creator-web-candidate core-runtime-candidate cognitive-worker-candidate research-runtime-candidate)
|
||||
containers=(assistant-creator-web-candidate-1 assistant-core-runtime-candidate-1 assistant-cognitive-worker-candidate-1 assistant-research-runtime-candidate-1)
|
||||
|
||||
compose --profile assistant-canary build --build-arg "SOURCE_REVISION=$REVISION" "${services[@]}"
|
||||
compose --profile assistant-canary up -d --no-build "${services[@]}"
|
||||
|
||||
for container in "${containers[@]}"; do
|
||||
ready=""
|
||||
for attempt in $(seq 1 60); do
|
||||
status="$(docker inspect -f '{{if .State.Health}}{{.State.Health.Status}}{{else}}unknown{{end}}' "$container" 2>/dev/null || true)"
|
||||
if [[ "$status" == healthy ]]; then ready=true; break; fi
|
||||
sleep 2
|
||||
done
|
||||
[[ "$ready" == true ]] || fail_rollback
|
||||
image_revision="$(docker inspect -f '{{index .Config.Labels "org.opencontainers.image.revision"}}' "$container" 2>/dev/null || true)"
|
||||
[[ "$image_revision" == "$REVISION" ]] || fail_rollback
|
||||
done
|
||||
|
||||
control /v1/releases/baseline -H 'Content-Type: application/json' --data "{\"id\":\"stable-$REVISION\",\"version\":\"$REVISION\"}" || fail_rollback
|
||||
control /v1/releases/start -H 'Content-Type: application/json' --data "{\"id\":\"candidate-$REVISION\",\"version\":\"$REVISION\"}" || fail_rollback
|
||||
control /v1/releases/observe -H 'Content-Type: application/json' --data "{\"release_id\":\"candidate-$REVISION\",\"ready\":true,\"reason\":\"candidate healthcheck passed\",\"observed_at_unix_millis\":$(date +%s%3N)}" || fail_rollback
|
||||
release_json="$(cat <<JSON
|
||||
{
|
||||
"id":"candidate-$REVISION",
|
||||
"revision":"$REVISION",
|
||||
"components":{
|
||||
"creator-web":{"name":"creator-web","revision":"$REVISION","upstream":"http://creator-web-candidate:8084","health_url":"http://creator-web-candidate:8084/healthz","ready_url":"http://creator-web-candidate:8084/readyz"},
|
||||
"core-runtime":{"name":"core-runtime","revision":"$REVISION","upstream":"http://core-runtime-candidate:8080","health_url":"http://core-runtime-candidate:8080/healthz","ready_url":"http://core-runtime-candidate:8080/readyz"},
|
||||
"cognitive-worker":{"name":"cognitive-worker","revision":"$REVISION","upstream":"http://cognitive-worker-candidate:8083","health_url":"http://cognitive-worker-candidate:8083/healthz","ready_url":"http://cognitive-worker-candidate:8083/readyz"},
|
||||
"research-runtime":{"name":"research-runtime","revision":"$REVISION","upstream":"http://research-runtime-candidate:8087","health_url":"http://research-runtime-candidate:8087/healthz","ready_url":"http://research-runtime-candidate:8087/readyz"}
|
||||
}
|
||||
}
|
||||
JSON
|
||||
)
|
||||
|
||||
stable_json="$(curl -fsS --max-time 10 "$SUPERVISOR_CONTROL_URL/v1/state")"
|
||||
stable_set="$(printf '%s' "$stable_json" | python3 -c 'import json,sys; s=json.load(sys.stdin).get("stable_release_set"); print(json.dumps(s,separators=(",",":")) if s else "")')"
|
||||
if [[ -z "$stable_set" ]]; then
|
||||
stable_revision="$(printf '%s' "$stable_json" | python3 -c 'import json,sys; print(json.load(sys.stdin).get("current",{}).get("version", "unknown"))')"
|
||||
stable_set="$(cat <<JSON
|
||||
{"id":"stable-$stable_revision","revision":"$stable_revision","components":{
|
||||
"creator-web":{"name":"creator-web","revision":"$stable_revision","upstream":"http://creator-web:8084","health_url":"http://creator-web:8084/healthz","ready_url":"http://creator-web:8084/readyz"},
|
||||
"core-runtime":{"name":"core-runtime","revision":"$stable_revision","upstream":"http://core-runtime:8080","health_url":"http://core-runtime:8080/healthz","ready_url":"http://core-runtime:8080/readyz"},
|
||||
"cognitive-worker":{"name":"cognitive-worker","revision":"$stable_revision","upstream":"http://cognitive-worker:8083","health_url":"http://cognitive-worker:8083/healthz","ready_url":"http://cognitive-worker:8083/readyz"},
|
||||
"research-runtime":{"name":"research-runtime","revision":"$stable_revision","upstream":"http://research-runtime:8087","health_url":"http://research-runtime:8087/healthz","ready_url":"http://research-runtime:8087/readyz"}}}
|
||||
JSON
|
||||
)"
|
||||
curl -fsS --max-time 10 -X POST "$SUPERVISOR_CONTROL_URL/v1/releases/baseline-set" -H 'Content-Type: application/json' --data "$stable_set" >/dev/null || fail_rollback
|
||||
fi
|
||||
|
||||
curl -fsS --max-time 10 -X POST "$SUPERVISOR_CONTROL_URL/v1/releases/start-set" -H 'Content-Type: application/json' --data "$release_json" >/dev/null || fail_rollback
|
||||
observation="$(printf '%s' "$release_json" | python3 -c 'import json,sys,time; s=json.load(sys.stdin); print(json.dumps({"release_id":s["id"],"ready":True,"reason":"all ReleaseSet healthchecks and image revisions passed","observed_at_unix_millis":int(time.time()*1000),"components":{k:{"revision":v["revision"],"health":True,"ready":True} for k,v in s["components"].items()}},separators=(",",":")))')"
|
||||
curl -fsS --max-time 10 -X POST "$SUPERVISOR_CONTROL_URL/v1/releases/observe-set" -H 'Content-Type: application/json' --data "$observation" >/dev/null || fail_rollback
|
||||
control /v1/traffic/canary || fail_rollback
|
||||
|
||||
headers="$(mktemp)"
|
||||
@@ -40,4 +78,4 @@ if ! curl -fsS --max-time 10 -D "$headers" -H 'X-Assistant-Canary: true' "https:
|
||||
grep -qi '^X-Assistant-Traffic-Target: candidate' "$headers" || fail_rollback
|
||||
|
||||
control /v1/releases/promote || fail_rollback
|
||||
echo "production canary promoted revision=$REVISION"
|
||||
echo "production ReleaseSet canary promoted revision=$REVISION"
|
||||
|
||||
@@ -1 +1 @@
|
||||
{"task_file":"dev/tasks/GEN2-COMPAT-OBS-001-production-compatibility-observation.json","task_id":"GEN2-COMPAT-OBS-001","task_revision":1}
|
||||
{"task_file":"dev/tasks/GEN2-DEPLOY-002-supervised-release-set.json","task_id":"GEN2-DEPLOY-002","task_revision":1}
|
||||
|
||||
@@ -0,0 +1,23 @@
|
||||
{
|
||||
"task_id": "GEN2-DEPLOY-002",
|
||||
"task_revision": 1,
|
||||
"architecture_revision": "ARCH-GEN2-000",
|
||||
"base_repo_revision": "gen2-integration",
|
||||
"risk_tier": "E",
|
||||
"intent": "Расширить существующий Supervisor до безопасного управления единым candidate/stable ReleaseSet из creator-web, core-runtime, cognitive-worker и research-runtime без изменения Gen2 semantic ownership.",
|
||||
"requirement_ids": ["REQ-GEN2-DEPLOY-002-RELEASE-SET", "REQ-GEN2-DEPLOY-002-CANDIDATE-TOPOLOGY", "REQ-GEN2-DEPLOY-002-ROLLBACK", "REQ-GEN2-DEPLOY-002-COMPAT-OBS"],
|
||||
"allowed_change_scope": {"modules": ["runtime/supervisor"], "paths": ["runtime/supervisor/**", "cmd/supervisor/**", "deploy/**", "docs/**", "tests/**", "dev/ACTIVE_TASK.json", "dev/tasks/GEN2-DEPLOY-002-supervised-release-set.json"]},
|
||||
"forbidden_change_scope": {"paths": ["architecture/**", "database/**", "modules/**", "contracts/**", "security/**", "go.mod", "go.sum", ".gitea/workflows/**", "adapters/models/**"]},
|
||||
"allowed_architecture_delta": {"new_cross_module_dependencies": 0, "new_canonical_owners": 0, "new_privileges": 1, "new_third_party_dependencies": 0, "new_database_owner_roles": 0},
|
||||
"contracts": {"consume": ["existing Supervisor durable state and production compose", "existing canary header and stable rollback controls"], "may_change": ["Supervisor release control payloads", "deployment topology and candidate routing configuration"]},
|
||||
"semantic_owner": "Supervisor/deployment control plane; ReleaseSet is operational release state and not semantic canonical state.",
|
||||
"canonical_state": ["no semantic owner or schema change", "previous stable release and evidence remain retained"],
|
||||
"state_transitions": ["candidate ReleaseSet -> component health/revision checks -> canary -> observation -> promotion or whole-set rollback"],
|
||||
"side_effects_and_privileges": ["production deployment only through approved control plane; no direct stable replacement; no credential reads"],
|
||||
"persistence_and_migrations": ["durable Supervisor release state remains in existing external Supervisor volume; no database migration"],
|
||||
"assumptions": ["the existing production Supervisor and compose control plane remain authoritative for deployment", "candidate services may share existing durable infrastructure only under current ownership and compatibility rules", "no credentials are read by this task"],
|
||||
"failure_retry_restart": ["candidate component failure routes all traffic to previous stable ReleaseSet", "Supervisor restart reloads ReleaseSet state", "previous stable is retained until promotion is complete"],
|
||||
"acceptance_scenarios": ["DEPLOY-002-SCN-01: complete ReleaseSet has all four components and one revision", "DEPLOY-002-SCN-02: candidate dependencies resolve only to candidate services", "DEPLOY-002-SCN-03: health/readiness/revision checks cover all required components", "DEPLOY-002-SCN-04: canary and stable paths are isolated", "DEPLOY-002-SCN-05: forced candidate failure rolls back the complete ReleaseSet", "DEPLOY-002-SCN-06: Supervisor restart preserves stable/candidate/LKG state", "DEPLOY-002-SCN-07: production telemetry revision starts bounded observation after controlled deployment"],
|
||||
"required_tests": ["go test ./runtime/supervisor ./cmd/supervisor", "FAST: targeted supervisor/release-set tests", "FAST: compose config validation", "FAST: routing and revision consistency tests", "FAST: rollback/restart persistence tests", "FAST: targeted go vet", "FAST governance: preflight/check-diff/independent", "FAST whitespace: git diff --check", "PRODUCTION: candidate ReleaseSet drill and telemetry observation start"],
|
||||
"terminal_gap_states": ["BLOCKED_BY_SPEC_GAP", "BLOCKED_BY_ARCHITECTURE_GAP", "BLOCKED_BY_MISSING_CAPABILITY", "BLOCKED_BY_CONFLICT", "REQUIRES_MIGRATION_DECISION"]
|
||||
}
|
||||
@@ -0,0 +1,40 @@
|
||||
{
|
||||
"evidence_id": "gen2-deploy-002-blocked-2026-08-30",
|
||||
"task_id": "GEN2-DEPLOY-002",
|
||||
"task_revision": 1,
|
||||
"base_repo_revision": "gen2-integration",
|
||||
"status": "BLOCKED_BY_MISSING_CAPABILITY",
|
||||
"implementation_checks": {
|
||||
"release_set_tests": "PASS",
|
||||
"supervisor_go_test": "PASS",
|
||||
"supervisor_go_vet": "PASS",
|
||||
"compose_config": "PASS",
|
||||
"canary_script_syntax": "PASS",
|
||||
"governance": "PASS",
|
||||
"git_diff_check": "PASS"
|
||||
},
|
||||
"implemented": [
|
||||
"durable Supervisor ReleaseSet state for creator-web, core-runtime, cognitive-worker, and research-runtime",
|
||||
"parallel candidate services with candidate-only core/cognitive/research dependency wiring",
|
||||
"whole-set health/readiness/revision observation, canary, promotion and rollback controls",
|
||||
"stable/candidate/LKG persistence across Supervisor restart",
|
||||
"core and research runtime compose healthchecks"
|
||||
],
|
||||
"production": {
|
||||
"deployment": "NOT_RUN",
|
||||
"candidate_release_set": "NOT_STARTED",
|
||||
"telemetry_observation": "NOT_STARTED",
|
||||
"stable_mutation": "NONE"
|
||||
},
|
||||
"capability_evidence": {
|
||||
"docker_and_compose_on_production_host": "AVAILABLE",
|
||||
"supervisor_control_endpoint": "REACHABLE",
|
||||
"portainer_api_key_in_current_execution_context": "NOT_CONFIGURED",
|
||||
"portainer_endpoint_metadata_in_current_execution_context": "NOT_CONFIGURED",
|
||||
"existing_production_candidate_services": ["creator-web-candidate"],
|
||||
"missing_capability": "Approved production deployment automation or candidate-capable control-plane access for the four-component ReleaseSet."
|
||||
},
|
||||
"unsafe_action_not_taken": "Stable containers were not replaced directly and no production compose mutation was attempted.",
|
||||
"compatibility": "No V1 compatibility path was removed or disabled; no schema, migration, semantic owner, or Gen3 change.",
|
||||
"next_action": "Owner must provide or enable the approved candidate-capable production deployment mechanism; then run the ReleaseSet drill and start bounded telemetry observation."
|
||||
}
|
||||
@@ -92,12 +92,24 @@ func (p *PersistentLifeboat) Start(candidate Release) (bool, error) {
|
||||
return p.transition(func(value *Lifeboat) bool { return value.Start(candidate) })
|
||||
}
|
||||
|
||||
func (p *PersistentLifeboat) StartSet(candidate ReleaseSet) (bool, error) {
|
||||
return p.transition(func(value *Lifeboat) bool { return value.StartSet(candidate) })
|
||||
}
|
||||
|
||||
func (p *PersistentLifeboat) EstablishBaseline(release Release) (bool, error) {
|
||||
return p.transition(func(value *Lifeboat) bool { return value.EstablishBaseline(release) })
|
||||
}
|
||||
|
||||
func (p *PersistentLifeboat) EstablishBaselineSet(release ReleaseSet) (bool, error) {
|
||||
return p.transition(func(value *Lifeboat) bool { return value.EstablishBaselineSet(release) })
|
||||
}
|
||||
func (p *PersistentLifeboat) Record(observation Observation) (bool, error) {
|
||||
return p.transition(func(value *Lifeboat) bool { return value.Record(observation) })
|
||||
}
|
||||
|
||||
func (p *PersistentLifeboat) RecordSet(observation ReleaseSetObservation) (bool, error) {
|
||||
return p.transition(func(value *Lifeboat) bool { return value.RecordSet(observation) })
|
||||
}
|
||||
func (p *PersistentLifeboat) Rollback() (bool, error) {
|
||||
return p.transition(func(value *Lifeboat) bool { return value.Rollback() })
|
||||
}
|
||||
@@ -105,6 +117,10 @@ func (p *PersistentLifeboat) SetTraffic(mode TrafficMode, stableUpstream, candid
|
||||
return p.transition(func(value *Lifeboat) bool { return value.SetTraffic(mode, stableUpstream, candidateUpstream) })
|
||||
}
|
||||
|
||||
func (p *PersistentLifeboat) SetTrafficForSets(mode TrafficMode) (bool, error) {
|
||||
return p.transition(func(value *Lifeboat) bool { return value.SetTrafficForSets(mode) })
|
||||
}
|
||||
|
||||
func (p *PersistentLifeboat) Promote() (bool, error) {
|
||||
return p.transition(func(value *Lifeboat) bool {
|
||||
if value.Current.State != ReleaseHealthy || value.Current.ID == "" {
|
||||
@@ -114,6 +130,10 @@ func (p *PersistentLifeboat) Promote() (bool, error) {
|
||||
})
|
||||
}
|
||||
|
||||
func (p *PersistentLifeboat) PromoteSet() (bool, error) {
|
||||
return p.transition(func(value *Lifeboat) bool { return value.PromoteSet() })
|
||||
}
|
||||
|
||||
func (p *PersistentLifeboat) RollbackTraffic() (bool, error) {
|
||||
return p.transition(func(value *Lifeboat) bool {
|
||||
if !value.Rollback() {
|
||||
@@ -123,6 +143,10 @@ func (p *PersistentLifeboat) RollbackTraffic() (bool, error) {
|
||||
})
|
||||
}
|
||||
|
||||
func (p *PersistentLifeboat) RollbackSet() (bool, error) {
|
||||
return p.transition(func(value *Lifeboat) bool { return value.RollbackSet() })
|
||||
}
|
||||
|
||||
func (p *PersistentLifeboat) Snapshot() (Lifeboat, error) {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
package supervisor
|
||||
|
||||
import "sort"
|
||||
|
||||
type ReleaseState string
|
||||
|
||||
const (
|
||||
@@ -22,21 +24,205 @@ type Release struct {
|
||||
Version string `json:"version"`
|
||||
State ReleaseState `json:"state"`
|
||||
LastKnownGood bool `json:"last_known_good"`
|
||||
Set *ReleaseSet `json:"release_set,omitempty"`
|
||||
}
|
||||
|
||||
const (
|
||||
ComponentCreatorWeb = "creator-web"
|
||||
ComponentCoreRuntime = "core-runtime"
|
||||
ComponentCognitiveWorker = "cognitive-worker"
|
||||
ComponentResearchRuntime = "research-runtime"
|
||||
)
|
||||
|
||||
// ComponentRelease is operational deployment evidence. It is deliberately
|
||||
// separate from semantic/domain state and contains no credentials.
|
||||
type ComponentRelease struct {
|
||||
Name string `json:"name"`
|
||||
Revision string `json:"revision"`
|
||||
Upstream string `json:"upstream"`
|
||||
HealthURL string `json:"health_url"`
|
||||
ReadyURL string `json:"ready_url"`
|
||||
}
|
||||
|
||||
// ReleaseSet is the atomic deployment unit supervised by the control plane.
|
||||
type ReleaseSet struct {
|
||||
ID string `json:"id"`
|
||||
Revision string `json:"revision"`
|
||||
Components map[string]ComponentRelease `json:"components"`
|
||||
}
|
||||
|
||||
var requiredComponents = []string{ComponentCreatorWeb, ComponentCoreRuntime, ComponentCognitiveWorker, ComponentResearchRuntime}
|
||||
|
||||
func (s ReleaseSet) Validate() bool {
|
||||
if s.ID == "" || s.Revision == "" || len(s.Components) != len(requiredComponents) {
|
||||
return false
|
||||
}
|
||||
for _, name := range requiredComponents {
|
||||
component, ok := s.Components[name]
|
||||
if !ok || component.Name != name || component.Revision != s.Revision || component.Upstream == "" || component.HealthURL == "" || component.ReadyURL == "" {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func (s ReleaseSet) ComponentNames() []string {
|
||||
names := make([]string, 0, len(s.Components))
|
||||
for name := range s.Components {
|
||||
names = append(names, name)
|
||||
}
|
||||
sort.Strings(names)
|
||||
return names
|
||||
}
|
||||
|
||||
type Observation struct {
|
||||
ReleaseID string `json:"release_id"`
|
||||
Ready bool `json:"ready"`
|
||||
Reason string `json:"reason"`
|
||||
ObservedAtUnixMillis int64 `json:"observed_at_unix_millis"`
|
||||
}
|
||||
|
||||
type ReleaseSetObservation struct {
|
||||
ReleaseID string `json:"release_id"`
|
||||
Ready bool `json:"ready"`
|
||||
Reason string `json:"reason"`
|
||||
ObservedAtUnixMillis int64 `json:"observed_at_unix_millis"`
|
||||
Components map[string]ComponentObservation `json:"components"`
|
||||
}
|
||||
|
||||
type ComponentObservation struct {
|
||||
Revision string `json:"revision"`
|
||||
Health bool `json:"health"`
|
||||
Ready bool `json:"ready"`
|
||||
}
|
||||
type Lifeboat struct {
|
||||
Current Release `json:"current"`
|
||||
LastKnownGood Release `json:"last_known_good"`
|
||||
RollbackTarget Release `json:"rollback_target"`
|
||||
Observations []Observation `json:"observations"`
|
||||
TrafficMode TrafficMode `json:"traffic_mode"`
|
||||
StableUpstream string `json:"stable_upstream"`
|
||||
CandidateUpstream string `json:"candidate_upstream"`
|
||||
Current Release `json:"current"`
|
||||
LastKnownGood Release `json:"last_known_good"`
|
||||
RollbackTarget Release `json:"rollback_target"`
|
||||
Observations []Observation `json:"observations"`
|
||||
TrafficMode TrafficMode `json:"traffic_mode"`
|
||||
StableUpstream string `json:"stable_upstream"`
|
||||
CandidateUpstream string `json:"candidate_upstream"`
|
||||
StableReleaseSet *ReleaseSet `json:"stable_release_set,omitempty"`
|
||||
CandidateReleaseSet *ReleaseSet `json:"candidate_release_set,omitempty"`
|
||||
LastKnownGoodReleaseSet *ReleaseSet `json:"last_known_good_release_set,omitempty"`
|
||||
RollbackTargetReleaseSet *ReleaseSet `json:"rollback_target_release_set,omitempty"`
|
||||
}
|
||||
|
||||
func (l *Lifeboat) StartSet(candidate ReleaseSet) bool {
|
||||
if !candidate.Validate() {
|
||||
return false
|
||||
}
|
||||
copySet := candidate
|
||||
if l.LastKnownGoodReleaseSet != nil {
|
||||
previous := *l.LastKnownGoodReleaseSet
|
||||
l.RollbackTargetReleaseSet = &previous
|
||||
}
|
||||
l.CandidateReleaseSet = ©Set
|
||||
l.Current = Release{ID: candidate.ID, Version: candidate.Revision, State: ReleaseCandidate, Set: ©Set}
|
||||
return true
|
||||
}
|
||||
|
||||
func (l *Lifeboat) EstablishBaselineSet(stable ReleaseSet) bool {
|
||||
if !stable.Validate() {
|
||||
return false
|
||||
}
|
||||
copySet := stable
|
||||
l.StableReleaseSet = ©Set
|
||||
l.LastKnownGoodReleaseSet = ©Set
|
||||
l.Current = Release{ID: stable.ID, Version: stable.Revision, State: ReleaseHealthy, LastKnownGood: true, Set: ©Set}
|
||||
l.LastKnownGood = l.Current
|
||||
l.RollbackTarget = Release{}
|
||||
l.RollbackTargetReleaseSet = nil
|
||||
l.TrafficMode = TrafficStable
|
||||
return l.setSetUpstreams(©Set, nil)
|
||||
}
|
||||
|
||||
func (l *Lifeboat) setSetUpstreams(stable, candidate *ReleaseSet) bool {
|
||||
if stable == nil {
|
||||
return false
|
||||
}
|
||||
stableWeb, ok := stable.Components[ComponentCreatorWeb]
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
l.StableUpstream = stableWeb.Upstream
|
||||
if candidate != nil {
|
||||
candidateWeb, exists := candidate.Components[ComponentCreatorWeb]
|
||||
if !exists {
|
||||
return false
|
||||
}
|
||||
l.CandidateUpstream = candidateWeb.Upstream
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func (l *Lifeboat) RecordSet(o ReleaseSetObservation) bool {
|
||||
if l.CandidateReleaseSet == nil || o.ReleaseID == "" || o.ReleaseID != l.CandidateReleaseSet.ID || !o.Ready {
|
||||
l.Current.State = ReleaseFailed
|
||||
return false
|
||||
}
|
||||
if len(o.Components) != len(requiredComponents) {
|
||||
l.Current.State = ReleaseFailed
|
||||
return false
|
||||
}
|
||||
for _, name := range requiredComponents {
|
||||
component, ok := o.Components[name]
|
||||
expected := l.CandidateReleaseSet.Components[name]
|
||||
if !ok || !component.Health || !component.Ready || component.Revision != expected.Revision {
|
||||
l.Current.State = ReleaseFailed
|
||||
return false
|
||||
}
|
||||
}
|
||||
l.Current.State = ReleaseHealthy
|
||||
return true
|
||||
}
|
||||
|
||||
func (l *Lifeboat) SetTrafficForSets(mode TrafficMode) bool {
|
||||
if l.StableReleaseSet == nil || l.CandidateReleaseSet == nil {
|
||||
return false
|
||||
}
|
||||
if !l.setSetUpstreams(l.StableReleaseSet, l.CandidateReleaseSet) {
|
||||
return false
|
||||
}
|
||||
l.TrafficMode = mode
|
||||
return true
|
||||
}
|
||||
|
||||
func (l *Lifeboat) PromoteSet() bool {
|
||||
if l.CandidateReleaseSet == nil || l.Current.State != ReleaseHealthy {
|
||||
return false
|
||||
}
|
||||
set := *l.CandidateReleaseSet
|
||||
l.StableReleaseSet = &set
|
||||
l.LastKnownGoodReleaseSet = &set
|
||||
l.LastKnownGood = l.Current
|
||||
l.LastKnownGood.Set = &set
|
||||
l.LastKnownGood.LastKnownGood = true
|
||||
l.Current.Set = &set
|
||||
l.Current.LastKnownGood = true
|
||||
l.CandidateReleaseSet = nil
|
||||
l.RollbackTargetReleaseSet = nil
|
||||
return l.setSetUpstreams(&set, nil)
|
||||
}
|
||||
|
||||
func (l *Lifeboat) RollbackSet() bool {
|
||||
if l.RollbackTargetReleaseSet == nil && l.LastKnownGoodReleaseSet == nil {
|
||||
return false
|
||||
}
|
||||
target := l.RollbackTargetReleaseSet
|
||||
if target == nil {
|
||||
target = l.LastKnownGoodReleaseSet
|
||||
}
|
||||
copySet := *target
|
||||
l.StableReleaseSet = ©Set
|
||||
l.Current = Release{ID: copySet.ID, Version: copySet.Revision, State: ReleaseRolledBack, LastKnownGood: true, Set: ©Set}
|
||||
l.LastKnownGood = l.Current
|
||||
l.LastKnownGoodReleaseSet = ©Set
|
||||
l.CandidateReleaseSet = nil
|
||||
l.RollbackTargetReleaseSet = nil
|
||||
l.TrafficMode = TrafficStable
|
||||
return l.setSetUpstreams(©Set, nil)
|
||||
}
|
||||
|
||||
func (l *Lifeboat) Start(candidate Release) bool {
|
||||
|
||||
@@ -82,3 +82,60 @@ func TestPersistentSupervisorRestoresTrafficModeAfterRestart(t *testing.T) {
|
||||
t.Fatalf("rollback state=%+v", state)
|
||||
}
|
||||
}
|
||||
|
||||
func testReleaseSet(id, revision, web, core, cognitive, research string) supervisor.ReleaseSet {
|
||||
return supervisor.ReleaseSet{ID: id, Revision: revision, Components: map[string]supervisor.ComponentRelease{
|
||||
supervisor.ComponentCreatorWeb: {Name: supervisor.ComponentCreatorWeb, Revision: revision, Upstream: web, HealthURL: web + "/healthz", ReadyURL: web + "/readyz"},
|
||||
supervisor.ComponentCoreRuntime: {Name: supervisor.ComponentCoreRuntime, Revision: revision, Upstream: core, HealthURL: core + "/healthz", ReadyURL: core + "/readyz"},
|
||||
supervisor.ComponentCognitiveWorker: {Name: supervisor.ComponentCognitiveWorker, Revision: revision, Upstream: cognitive, HealthURL: cognitive + "/healthz", ReadyURL: cognitive + "/readyz"},
|
||||
supervisor.ComponentResearchRuntime: {Name: supervisor.ComponentResearchRuntime, Revision: revision, Upstream: research, HealthURL: research + "/healthz", ReadyURL: research + "/readyz"},
|
||||
}}
|
||||
}
|
||||
|
||||
func TestReleaseSetRejectsMixedRevisionsAndMissingComponents(t *testing.T) {
|
||||
set := testReleaseSet("candidate", "r1", "http://web", "http://core", "http://cognitive", "http://research")
|
||||
if !set.Validate() {
|
||||
t.Fatal("complete release set should validate")
|
||||
}
|
||||
set.Components[supervisor.ComponentCoreRuntime] = supervisor.ComponentRelease{Name: supervisor.ComponentCoreRuntime, Revision: "r0", Upstream: "http://core", HealthURL: "http://core/healthz", ReadyURL: "http://core/readyz"}
|
||||
if set.Validate() {
|
||||
t.Fatal("mixed revision release set must be rejected")
|
||||
}
|
||||
}
|
||||
|
||||
func TestPersistentReleaseSetRollbackPreservesWholeLastKnownGood(t *testing.T) {
|
||||
store, err := supervisor.NewDurableStore(filepath.Join(t.TempDir(), "control", "release-set.json"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
first := supervisor.NewPersistentLifeboat(store)
|
||||
good := testReleaseSet("stable-r1", "r1", "http://web", "http://core", "http://cognitive", "http://research")
|
||||
candidate := testReleaseSet("candidate-r2", "r2", "http://web-c", "http://core-c", "http://cognitive-c", "http://research-c")
|
||||
if ok, err := first.EstablishBaselineSet(good); err != nil || !ok {
|
||||
t.Fatalf("baseline: %v %v", ok, err)
|
||||
}
|
||||
if ok, err := first.StartSet(candidate); err != nil || !ok {
|
||||
t.Fatalf("start: %v %v", ok, err)
|
||||
}
|
||||
observation := supervisor.ReleaseSetObservation{ReleaseID: candidate.ID, Ready: true, Components: map[string]supervisor.ComponentObservation{}}
|
||||
for name := range candidate.Components {
|
||||
observation.Components[name] = supervisor.ComponentObservation{Revision: candidate.Revision, Health: true, Ready: true}
|
||||
}
|
||||
if ok, err := first.RecordSet(observation); err != nil || !ok {
|
||||
t.Fatalf("observe: %v %v", ok, err)
|
||||
}
|
||||
if ok, err := first.SetTrafficForSets(supervisor.TrafficCanaryHeader); err != nil || !ok {
|
||||
t.Fatalf("canary: %v %v", ok, err)
|
||||
}
|
||||
second := supervisor.NewPersistentLifeboat(store)
|
||||
if ok, err := second.RollbackSet(); err != nil || !ok {
|
||||
t.Fatalf("rollback: %v %v", ok, err)
|
||||
}
|
||||
state, err := second.Snapshot()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if state.StableReleaseSet == nil || state.StableReleaseSet.Revision != "r1" || state.TrafficMode != supervisor.TrafficStable {
|
||||
t.Fatalf("state=%+v", state)
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user