Compare commits
8
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
b30eb38eca | ||
|
|
2083096aed | ||
|
|
671ecb4968 | ||
|
|
af9d5809e9 | ||
|
|
990928618d | ||
|
|
9fac8284f5 | ||
|
|
7d9a5edee5 | ||
|
|
32d3f318a5 |
+10
-1
@@ -32,7 +32,7 @@ func main() {
|
||||
lifeboat := supervisor.NewPersistentLifeboat(store)
|
||||
if _, err := lifeboat.Snapshot(); errors.Is(err, os.ErrNotExist) {
|
||||
baseline := supervisor.Release{ID: envOr("SUPERVISOR_BASELINE_RELEASE", "bootstrap"), Version: envOr("SUPERVISOR_BASELINE_RELEASE", "bootstrap")}
|
||||
if ok, startErr := lifeboat.Start(baseline); startErr != nil || !ok {
|
||||
if ok, startErr := lifeboat.EstablishBaseline(baseline); startErr != nil || !ok {
|
||||
fatal(fmt.Errorf("initialize baseline: %v", startErr))
|
||||
}
|
||||
if ok, observeErr := lifeboat.Record(supervisor.Observation{ReleaseID: baseline.ID, Ready: true, Reason: "supervisor baseline"}); observeErr != nil || !ok {
|
||||
@@ -53,6 +53,7 @@ func main() {
|
||||
control := http.NewServeMux()
|
||||
control.HandleFunc("GET /v1/state", s.state)
|
||||
control.HandleFunc("POST /v1/releases/start", s.start)
|
||||
control.HandleFunc("POST /v1/releases/baseline", s.baseline)
|
||||
control.HandleFunc("POST /v1/releases/observe", s.observe)
|
||||
control.HandleFunc("POST /v1/releases/promote", s.promote)
|
||||
control.HandleFunc("POST /v1/releases/rollback", s.rollback)
|
||||
@@ -141,6 +142,14 @@ func (s *server) start(w http.ResponseWriter, r *http.Request) {
|
||||
ok, err := s.lifeboat.Start(release)
|
||||
respond(w, ok, err)
|
||||
}
|
||||
func (s *server) baseline(w http.ResponseWriter, r *http.Request) {
|
||||
var release supervisor.Release
|
||||
if !decode(w, r, &release) {
|
||||
return
|
||||
}
|
||||
ok, err := s.lifeboat.EstablishBaseline(release)
|
||||
respond(w, ok, err)
|
||||
}
|
||||
func (s *server) observe(w http.ResponseWriter, r *http.Request) {
|
||||
var observation supervisor.Observation
|
||||
if !decode(w, r, &observation) {
|
||||
|
||||
@@ -11,6 +11,7 @@ ARG SOURCE_REVISION=unknown
|
||||
LABEL org.opencontainers.image.revision=$SOURCE_REVISION \
|
||||
org.opencontainers.image.source="https://git.cin.su/m/Assistant.git"
|
||||
RUN addgroup -S assistant && adduser -S -G assistant assistant
|
||||
RUN mkdir -p /var/lib/supervisor && chown assistant:assistant /var/lib/supervisor
|
||||
USER assistant
|
||||
COPY --from=build /out/runtime /usr/local/bin/assistant-runtime
|
||||
ENTRYPOINT ["/usr/local/bin/assistant-runtime"]
|
||||
|
||||
@@ -29,6 +29,7 @@ for attempt in $(seq 1 30); do
|
||||
sleep 2
|
||||
done
|
||||
|
||||
control /v1/releases/baseline -H 'Content-Type: application/json' --data "{\"id\":\"stable-$REVISION\",\"version\":\"$REVISION\"}" || fail_rollback
|
||||
control /v1/releases/start -H 'Content-Type: application/json' --data "{\"id\":\"candidate-$REVISION\",\"version\":\"$REVISION\"}" || fail_rollback
|
||||
control /v1/releases/observe -H 'Content-Type: application/json' --data "{\"release_id\":\"candidate-$REVISION\",\"ready\":true,\"reason\":\"candidate healthcheck passed\",\"observed_at_unix_millis\":$(date +%s%3N)}" || fail_rollback
|
||||
control /v1/traffic/canary || fail_rollback
|
||||
|
||||
@@ -1 +1 @@
|
||||
{"task_file":"dev/tasks/GEN2-DEPLOY-001-supervisor-and-production-rollback.json","task_id":"GEN2-DEPLOY-001","task_revision":1}
|
||||
{"task_file":"dev/tasks/GEN2-027-production-canary-and-controlled-gen2-traffic-transition.json","task_id":"GEN2-027","task_revision":1}
|
||||
|
||||
@@ -0,0 +1,44 @@
|
||||
{
|
||||
"evidence_id": "gen2-deploy-001-production-recovery-2026-08-29",
|
||||
"task_id": "GEN2-DEPLOY-001",
|
||||
"production_revision_before": "36fc8ec",
|
||||
"production_revision_after": "2083096aedaec73191e0bc9e2566f596e1810d82",
|
||||
"topology": {
|
||||
"supervisor": "assistant-supervisor-1",
|
||||
"public_traffic": "Supervisor reverse proxy on creator web host port",
|
||||
"control_api": "Supervisor loopback-only host control port",
|
||||
"durable_state": "assistant_assistant_supervisor_state:/var/lib/supervisor/release-state.json",
|
||||
"candidate": "assistant-creator-web-candidate-1"
|
||||
},
|
||||
"canary": {
|
||||
"candidate_health": "healthy",
|
||||
"header_route": "X-Assistant-Canary=true -> candidate",
|
||||
"header_route_http": 200,
|
||||
"promoted": true
|
||||
},
|
||||
"rollback": {
|
||||
"drill": "candidate container stopped, Supervisor traffic rollback and release rollback invoked",
|
||||
"stable_http": 200,
|
||||
"stable_target_header": "stable",
|
||||
"supervisor_restart": "healthy; traffic_mode=STABLE and stable release state restored"
|
||||
},
|
||||
"postgres_recovery": {
|
||||
"restart": "healthy",
|
||||
"public_healthz": 200,
|
||||
"public_readyz": 200
|
||||
},
|
||||
"cognitive_lifeboat": {
|
||||
"previous_status": "restart loop/unhealthy",
|
||||
"cause": "configured Qwen3-8B-Q4_K_M.gguf was absent from llm-local-models volume",
|
||||
"remediation": "compose model reference changed to existing Qwen3.8-27B-UD-Q2_K_XL.gguf",
|
||||
"current_status": "healthy",
|
||||
"role": "local degraded cognitive provider; not Supervisor"
|
||||
},
|
||||
"smoke": {
|
||||
"public_healthz": 200,
|
||||
"public_readyz": 200,
|
||||
"public_login": 200,
|
||||
"authenticated_household": "UNVERIFIED: browser session was unauthenticated; no credentials were read or entered"
|
||||
},
|
||||
"interpretation": "Supervisor independence, durable release state, controlled header canary, stable rollback and restart recovery are production-observed. Authenticated household acceptance remains an explicit gap."
|
||||
}
|
||||
@@ -91,6 +91,10 @@ func (p *PersistentLifeboat) transition(fn func(*Lifeboat) bool) (bool, error) {
|
||||
func (p *PersistentLifeboat) Start(candidate Release) (bool, error) {
|
||||
return p.transition(func(value *Lifeboat) bool { return value.Start(candidate) })
|
||||
}
|
||||
|
||||
func (p *PersistentLifeboat) EstablishBaseline(release Release) (bool, error) {
|
||||
return p.transition(func(value *Lifeboat) bool { return value.EstablishBaseline(release) })
|
||||
}
|
||||
func (p *PersistentLifeboat) Record(observation Observation) (bool, error) {
|
||||
return p.transition(func(value *Lifeboat) bool { return value.Record(observation) })
|
||||
}
|
||||
|
||||
@@ -18,24 +18,25 @@ const (
|
||||
)
|
||||
|
||||
type Release struct {
|
||||
ID string
|
||||
Version string
|
||||
State ReleaseState
|
||||
LastKnownGood bool
|
||||
ID string `json:"id"`
|
||||
Version string `json:"version"`
|
||||
State ReleaseState `json:"state"`
|
||||
LastKnownGood bool `json:"last_known_good"`
|
||||
}
|
||||
type Observation struct {
|
||||
ReleaseID string
|
||||
Ready bool
|
||||
Reason string
|
||||
ObservedAtUnixMillis int64
|
||||
ReleaseID string `json:"release_id"`
|
||||
Ready bool `json:"ready"`
|
||||
Reason string `json:"reason"`
|
||||
ObservedAtUnixMillis int64 `json:"observed_at_unix_millis"`
|
||||
}
|
||||
type Lifeboat struct {
|
||||
Current Release
|
||||
LastKnownGood Release
|
||||
Observations []Observation
|
||||
TrafficMode TrafficMode
|
||||
StableUpstream string
|
||||
CandidateUpstream string
|
||||
Current Release `json:"current"`
|
||||
LastKnownGood Release `json:"last_known_good"`
|
||||
RollbackTarget Release `json:"rollback_target"`
|
||||
Observations []Observation `json:"observations"`
|
||||
TrafficMode TrafficMode `json:"traffic_mode"`
|
||||
StableUpstream string `json:"stable_upstream"`
|
||||
CandidateUpstream string `json:"candidate_upstream"`
|
||||
}
|
||||
|
||||
func (l *Lifeboat) Start(candidate Release) bool {
|
||||
@@ -43,9 +44,25 @@ func (l *Lifeboat) Start(candidate Release) bool {
|
||||
return false
|
||||
}
|
||||
candidate.State = ReleaseCandidate
|
||||
if l.LastKnownGood.ID != "" {
|
||||
l.RollbackTarget = l.LastKnownGood
|
||||
}
|
||||
l.Current = candidate
|
||||
return true
|
||||
}
|
||||
|
||||
func (l *Lifeboat) EstablishBaseline(release Release) bool {
|
||||
if release.ID == "" || release.Version == "" {
|
||||
return false
|
||||
}
|
||||
release.State = ReleaseHealthy
|
||||
release.LastKnownGood = true
|
||||
l.Current = release
|
||||
l.LastKnownGood = release
|
||||
l.RollbackTarget = Release{}
|
||||
l.TrafficMode = TrafficStable
|
||||
return true
|
||||
}
|
||||
func (l *Lifeboat) Record(o Observation) bool {
|
||||
if o.ReleaseID == "" {
|
||||
return false
|
||||
@@ -64,11 +81,18 @@ func (l *Lifeboat) Record(o Observation) bool {
|
||||
return true
|
||||
}
|
||||
func (l *Lifeboat) Rollback() bool {
|
||||
if l.LastKnownGood.ID == "" {
|
||||
target := l.RollbackTarget
|
||||
if target.ID == "" {
|
||||
target = l.LastKnownGood
|
||||
}
|
||||
if target.ID == "" {
|
||||
return false
|
||||
}
|
||||
l.Current = l.LastKnownGood
|
||||
l.Current = target
|
||||
l.Current.State = ReleaseRolledBack
|
||||
l.Current.LastKnownGood = true
|
||||
l.LastKnownGood = l.Current
|
||||
l.RollbackTarget = Release{}
|
||||
return true
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user