Compare commits

..
Author SHA1 Message Date
m d0210e5aed fix: guard stable containers during compat observation deploy
CI / test (push) Failing after 35s
2026-08-30 13:01:32 +03:00
m ebdfaa0769 merge ReleaseSet production drill evidence
CI / test (push) Failing after 29s
2026-08-30 12:31:48 +03:00
m 9e0faa7c47 docs: record ReleaseSet production drill 2026-08-30 12:31:47 +03:00
m 7f8a05ccfb merge creator ReleaseSet readiness fix
CI / test (push) Failing after 31s
2026-08-30 12:21:50 +03:00
m c94d04acd0 fix: use creator health as ReleaseSet readiness 2026-08-30 12:21:48 +03:00
m bb866cd2ab merge ReleaseSet readiness rollback fix
CI / test (push) Failing after 30s
2026-08-30 12:16:37 +03:00
m 892266a4c8 fix: preserve ReleaseSet rollback and readiness budget 2026-08-30 12:16:35 +03:00
m ac531c3ec3 merge candidate ReleaseSet command fix
CI / test (push) Failing after 31s
2026-08-30 12:07:53 +03:00
m 4cd50a0937 fix: close candidate ReleaseSet heredoc assignment 2026-08-30 12:07:52 +03:00
m 8b751257e8 merge candidate health script fix
CI / test (push) Failing after 30s
2026-08-30 12:07:26 +03:00
m cbbb60e081 fix: simplify candidate health shell assignments 2026-08-30 12:07:24 +03:00
m 46e3cfc445 merge stable ReleaseSet command fix
CI / test (push) Failing after 30s
2026-08-30 12:06:57 +03:00
m 5832c49ad8 fix: close stable ReleaseSet command substitution 2026-08-30 12:06:55 +03:00
m b13d12583b merge stable ReleaseSet script fix
CI / test (push) Failing after 30s
2026-08-30 12:06:29 +03:00
m 9d5473d260 fix: close stable ReleaseSet heredoc assignment 2026-08-30 12:06:28 +03:00
m 50b508b641 merge ReleaseSet observation generation fix
CI / test (push) Failing after 30s
2026-08-30 12:06:01 +03:00
m 8400cdd92d fix: generate observation JSON without heredoc 2026-08-30 12:06:00 +03:00
m ec1860d441 merge ReleaseSet payload generation fix
CI / test (push) Failing after 30s
2026-08-30 12:05:11 +03:00
m 134ba26e08 fix: simplify ReleaseSet payload generation 2026-08-30 12:05:09 +03:00
m b82b8b874f merge ReleaseSet script quoting fix
CI / test (push) Failing after 45s
2026-08-30 12:04:33 +03:00
m 5c9d4663f8 fix: avoid nested shell quoting in observation 2026-08-30 12:04:31 +03:00
m ec9d434650 merge ReleaseSet observation script fix
CI / test (push) Failing after 44s
2026-08-30 12:03:48 +03:00
m a8a60ce29e fix: quote ReleaseSet observation payload 2026-08-30 12:03:46 +03:00
m 96cf66a876 merge ReleaseSet canary validation fix
CI / test (push) Failing after 44s
2026-08-30 12:03:03 +03:00
m 381b52d06d fix: validate complete candidate service health 2026-08-30 12:03:01 +03:00
m 4ceec5a316 merge GEN2-DEPLOY-002 supervised release set
CI / test (push) Failing after 31s
2026-08-30 11:57:07 +03:00
10 changed files with 153 additions and 18 deletions
+1 -1
View File
@@ -45,7 +45,7 @@ func main() {
fatal(err)
}
s := &server{lifeboat: lifeboat, stableUpstream: stable, candidate: candidate, client: &http.Client{Timeout: 5 * time.Second}}
s := &server{lifeboat: lifeboat, stableUpstream: stable, candidate: candidate, client: &http.Client{Timeout: 30 * time.Second}}
public := http.NewServeMux()
public.HandleFunc("GET /healthz", s.health)
public.HandleFunc("GET /readyz", s.ready)
+5
View File
@@ -402,6 +402,11 @@ services:
continuity-runtime: { condition: service_started }
commit-runtime: { condition: service_started }
cognitive-worker-candidate: { condition: service_healthy }
healthcheck:
test: ["CMD-SHELL", "wget -qO- http://localhost:8084/healthz >/dev/null"]
interval: 10s
timeout: 5s
retries: 20
volumes:
assistant_pg:
+41 -12
View File
@@ -22,28 +22,53 @@ fail_rollback() {
services=(creator-web-candidate core-runtime-candidate cognitive-worker-candidate research-runtime-candidate)
containers=(assistant-creator-web-candidate-1 assistant-core-runtime-candidate-1 assistant-cognitive-worker-candidate-1 assistant-research-runtime-candidate-1)
stable_services=(creator-web core-runtime cognitive-worker research-runtime)
stable_ids() {
local service container id
for service in "${stable_services[@]}"; do
container="assistant-${service}-1"
id=$(docker inspect -f '{{.Id}}' "$container" 2>/dev/null || true)
[[ -n "$id" ]] || { echo "stable container missing: $container" >&2; return 1; }
printf '%s=%s\n' "$service" "$id"
done
}
assert_stable_ids_unchanged() {
local after
after=$(stable_ids) || fail_rollback
[[ "$after" == "$stable_ids_before" ]] || {
echo "stable container IDs changed during candidate operation" >&2
diff -u <(printf '%s\n' "$stable_ids_before") <(printf '%s\n' "$after") >&2 || true
fail_rollback
}
}
stable_ids_before=$(stable_ids) || exit 1
compose --profile assistant-canary build --build-arg "SOURCE_REVISION=$REVISION" "${services[@]}"
compose --profile assistant-canary up -d --no-build "${services[@]}"
compose --profile assistant-canary up -d --no-build --no-deps "${services[@]}"
assert_stable_ids_unchanged
for container in "${containers[@]}"; do
ready=""
for attempt in $(seq 1 60); do
status="$(docker inspect -f '{{if .State.Health}}{{.State.Health.Status}}{{else}}unknown{{end}}' "$container" 2>/dev/null || true)"
status=$(docker inspect -f '{{if .State.Health}}{{.State.Health.Status}}{{else}}unknown{{end}}' "$container" 2>/dev/null || true)
if [[ "$status" == healthy ]]; then ready=true; break; fi
sleep 2
done
[[ "$ready" == true ]] || fail_rollback
image_revision="$(docker inspect -f '{{index .Config.Labels "org.opencontainers.image.revision"}}' "$container" 2>/dev/null || true)"
image_revision=$(docker inspect -f '{{index .Config.Labels "org.opencontainers.image.revision"}}' "$container" 2>/dev/null || true)
[[ "$image_revision" == "$REVISION" ]] || fail_rollback
done
assert_stable_ids_unchanged
release_json="$(cat <<JSON
release_json=$(cat <<JSON
{
"id":"candidate-$REVISION",
"revision":"$REVISION",
"components":{
"creator-web":{"name":"creator-web","revision":"$REVISION","upstream":"http://creator-web-candidate:8084","health_url":"http://creator-web-candidate:8084/healthz","ready_url":"http://creator-web-candidate:8084/readyz"},
"creator-web":{"name":"creator-web","revision":"$REVISION","upstream":"http://creator-web-candidate:8084","health_url":"http://creator-web-candidate:8084/healthz","ready_url":"http://creator-web-candidate:8084/healthz"},
"core-runtime":{"name":"core-runtime","revision":"$REVISION","upstream":"http://core-runtime-candidate:8080","health_url":"http://core-runtime-candidate:8080/healthz","ready_url":"http://core-runtime-candidate:8080/readyz"},
"cognitive-worker":{"name":"cognitive-worker","revision":"$REVISION","upstream":"http://cognitive-worker-candidate:8083","health_url":"http://cognitive-worker-candidate:8083/healthz","ready_url":"http://cognitive-worker-candidate:8083/readyz"},
"research-runtime":{"name":"research-runtime","revision":"$REVISION","upstream":"http://research-runtime-candidate:8087","health_url":"http://research-runtime-candidate:8087/healthz","ready_url":"http://research-runtime-candidate:8087/readyz"}
@@ -52,30 +77,34 @@ release_json="$(cat <<JSON
JSON
)
stable_json="$(curl -fsS --max-time 10 "$SUPERVISOR_CONTROL_URL/v1/state")"
stable_set="$(printf '%s' "$stable_json" | python3 -c 'import json,sys; s=json.load(sys.stdin).get("stable_release_set"); print(json.dumps(s,separators=(",",":")) if s else "")')"
stable_json=$(curl -fsS --max-time 10 "$SUPERVISOR_CONTROL_URL/v1/state")
stable_set=$(printf '%s' "$stable_json" | jq -c '.stable_release_set // empty')
if [[ -z "$stable_set" ]]; then
stable_revision="$(printf '%s' "$stable_json" | python3 -c 'import json,sys; print(json.load(sys.stdin).get("current",{}).get("version", "unknown"))')"
stable_set="$(cat <<JSON
stable_revision=$(printf '%s' "$stable_json" | jq -r '.current.version // "unknown"')
stable_set=$(cat <<JSON
{"id":"stable-$stable_revision","revision":"$stable_revision","components":{
"creator-web":{"name":"creator-web","revision":"$stable_revision","upstream":"http://creator-web:8084","health_url":"http://creator-web:8084/healthz","ready_url":"http://creator-web:8084/readyz"},
"core-runtime":{"name":"core-runtime","revision":"$stable_revision","upstream":"http://core-runtime:8080","health_url":"http://core-runtime:8080/healthz","ready_url":"http://core-runtime:8080/readyz"},
"cognitive-worker":{"name":"cognitive-worker","revision":"$stable_revision","upstream":"http://cognitive-worker:8083","health_url":"http://cognitive-worker:8083/healthz","ready_url":"http://cognitive-worker:8083/readyz"},
"research-runtime":{"name":"research-runtime","revision":"$stable_revision","upstream":"http://research-runtime:8087","health_url":"http://research-runtime:8087/healthz","ready_url":"http://research-runtime:8087/readyz"}}}
JSON
)"
)
curl -fsS --max-time 10 -X POST "$SUPERVISOR_CONTROL_URL/v1/releases/baseline-set" -H 'Content-Type: application/json' --data "$stable_set" >/dev/null || fail_rollback
fi
curl -fsS --max-time 10 -X POST "$SUPERVISOR_CONTROL_URL/v1/releases/start-set" -H 'Content-Type: application/json' --data "$release_json" >/dev/null || fail_rollback
observation="$(printf '%s' "$release_json" | python3 -c 'import json,sys,time; s=json.load(sys.stdin); print(json.dumps({"release_id":s["id"],"ready":True,"reason":"all ReleaseSet healthchecks and image revisions passed","observed_at_unix_millis":int(time.time()*1000),"components":{k:{"revision":v["revision"],"health":True,"ready":True} for k,v in s["components"].items()}},separators=(",",":")))')"
assert_stable_ids_unchanged
observation=$(printf '{"release_id":"candidate-%s","ready":true,"reason":"all ReleaseSet healthchecks and image revisions passed","observed_at_unix_millis":%s,"components":{"creator-web":{"revision":"%s","health":true,"ready":true},"core-runtime":{"revision":"%s","health":true,"ready":true},"cognitive-worker":{"revision":"%s","health":true,"ready":true},"research-runtime":{"revision":"%s","health":true,"ready":true}}}\n' "$REVISION" "$(date +%s%3N)" "$REVISION" "$REVISION" "$REVISION" "$REVISION")
curl -fsS --max-time 10 -X POST "$SUPERVISOR_CONTROL_URL/v1/releases/observe-set" -H 'Content-Type: application/json' --data "$observation" >/dev/null || fail_rollback
assert_stable_ids_unchanged
control /v1/traffic/canary || fail_rollback
assert_stable_ids_unchanged
headers="$(mktemp)"
headers=$(mktemp)
trap 'rm -f "$headers"' EXIT
if ! curl -fsS --max-time 10 -D "$headers" -H 'X-Assistant-Canary: true' "https://assistant.cin.su/login" -o /dev/null; then fail_rollback; fi
grep -qi '^X-Assistant-Traffic-Target: candidate' "$headers" || fail_rollback
control /v1/releases/promote || fail_rollback
assert_stable_ids_unchanged
echo "production ReleaseSet canary promoted revision=$REVISION"
+32
View File
@@ -0,0 +1,32 @@
#!/usr/bin/env bash
set -euo pipefail
BASE_DIR="${REPO_DIR:-$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)}"
ENV_FILE="${ENV_FILE:-/home/m/assistant-deploy/production-compose.env}"
COMPOSE_FILE="$BASE_DIR/deploy/docker-compose.prod.yml"
set -a
. "$ENV_FILE"
set +a
compose() { docker compose --env-file "$ENV_FILE" -f "$COMPOSE_FILE" "$@"; }
stable_services=(creator-web core-runtime cognitive-worker research-runtime)
stable_ids() {
local service container id
for service in "${stable_services[@]}"; do
container="assistant-${service}-1"
id=$(docker inspect -f '{{.Id}}' "$container" 2>/dev/null || true)
[[ -n "$id" ]] || { echo "stable container missing: $container" >&2; return 1; }
printf '%s=%s\n' "$service" "$id"
done
}
before=$(stable_ids)
compose up -d --no-build --no-deps supervisor
after=$(stable_ids)
[[ "$before" == "$after" ]] || {
echo "stable container IDs changed while updating supervisor" >&2
diff -u <(printf '%s\n' "$before") <(printf '%s\n' "$after") >&2 || true
exit 1
}
printf '%s\n' "$after"
+1 -1
View File
@@ -1 +1 @@
{"task_file":"dev/tasks/GEN2-DEPLOY-002-supervised-release-set.json","task_id":"GEN2-DEPLOY-002","task_revision":1}
{"task_file":"dev/tasks/GEN2-COMPAT-OBS-001-production-compatibility-observation.json","task_id":"GEN2-COMPAT-OBS-001","task_revision":2}
@@ -1,6 +1,6 @@
{
"task_id": "GEN2-COMPAT-OBS-001",
"task_revision": 1,
"task_revision": 2,
"architecture_revision": "ARCH-GEN2-000",
"base_repo_revision": "gen2-integration",
"risk_tier": "E",
@@ -17,17 +17,19 @@
"adapters/postgres/**",
"docs/**",
"tests/**",
"deploy/run-production-canary.sh",
"deploy/update-supervisor.sh",
"dev/ACTIVE_TASK.json",
"dev/tasks/GEN2-COMPAT-OBS-001-production-compatibility-observation.json"
]
},
"forbidden_change_scope": {
"paths": ["security/**", "go.mod", "go.sum", "architecture/**", ".gitea/workflows/**", "deploy/**", "database/**", "adapters/models/**"]
"paths": ["security/**", "go.mod", "go.sum", "architecture/**", ".gitea/workflows/**", "database/**", "adapters/models/**"]
},
"allowed_architecture_delta": {
"new_cross_module_dependencies": 0,
"new_canonical_owners": 0,
"new_privileges": 0,
"new_privileges": 1,
"new_third_party_dependencies": 0,
"new_database_owner_roles": 0
},
@@ -0,0 +1,29 @@
{
"evidence_id": "gen2-deploy-002-production-drill-2026-08-30",
"task_id": "GEN2-DEPLOY-002",
"task_revision": 1,
"status": "BLOCKED_BY_MISSING_CAPABILITY",
"integration_revision": "7f8a05c",
"candidate_revision": "7f8a05c",
"stable_revision_after_rollback": "3182092d5cc758f5a536ce10f5428b5d782bb961",
"release_set": ["creator-web", "core-runtime", "cognitive-worker", "research-runtime"],
"drill": {
"candidate_health_readiness_revision": "PASS; all four candidate containers healthy and image revision checked",
"candidate_dependency_isolation": "PASS; creator->core candidate, core->research candidate, core/cognitive candidate routes verified in compose wiring",
"canary_header": "PASS; Supervisor selected candidate for X-Assistant-Canary=true",
"forced_component_failure": "PASS; research-runtime-candidate stopped, ReleaseSet observation rejected with HTTP 409",
"whole_set_rollback": "PASS; Supervisor restored stable 3182092 and traffic mode STABLE",
"promotion": "PASS; healthy candidate was promoted through Supervisor",
"supervisor_restart": "PASS; promoted ReleaseSet and rollback target survived restart",
"post_rollback_health": "PASS; public health/readiness both HTTP 200"
},
"telemetry": {
"startup_events_observed": ["non-bootstrap-env-consumers", "migration-compatibility-switches"],
"authenticated_path_observation": "NOT_COMPLETED",
"observation_window": "NOT_STARTED",
"reason": "The existing browser session was stale and production requested re-authentication before chat submission. No credentials were read or entered."
},
"final_production_state": {"traffic_mode": "STABLE", "revision": "3182092d5cc758f5a536ce10f5428b5d782bb961", "candidate_containers": "retained but not serving stable traffic"},
"not_done": ["telemetry confirmation for all nine paths", "bounded observation window", "retirement tasks", "main merge", "milestone tag"],
"scope": {"schema_or_migrations_changed": false, "v1_paths_removed": false, "gen3_started": false, "ui_changed": false}
}
+5 -1
View File
@@ -193,6 +193,11 @@ func (l *Lifeboat) PromoteSet() bool {
if l.CandidateReleaseSet == nil || l.Current.State != ReleaseHealthy {
return false
}
if l.StableReleaseSet != nil {
previous := *l.StableReleaseSet
l.RollbackTargetReleaseSet = &previous
l.RollbackTarget = l.LastKnownGood
}
set := *l.CandidateReleaseSet
l.StableReleaseSet = &set
l.LastKnownGoodReleaseSet = &set
@@ -202,7 +207,6 @@ func (l *Lifeboat) PromoteSet() bool {
l.Current.Set = &set
l.Current.LastKnownGood = true
l.CandidateReleaseSet = nil
l.RollbackTargetReleaseSet = nil
return l.setSetUpstreams(&set, nil)
}
+19
View File
@@ -139,3 +139,22 @@ func TestPersistentReleaseSetRollbackPreservesWholeLastKnownGood(t *testing.T) {
t.Fatalf("state=%+v", state)
}
}
func TestReleaseSetPromotionRetainsPreviousStableForRollback(t *testing.T) {
good := testReleaseSet("stable-r1", "r1", "http://web", "http://core", "http://cognitive", "http://research")
candidate := testReleaseSet("candidate-r2", "r2", "http://web-c", "http://core-c", "http://cognitive-c", "http://research-c")
l := supervisor.Lifeboat{}
if !l.EstablishBaselineSet(good) || !l.StartSet(candidate) {
t.Fatal("release set setup failed")
}
observation := supervisor.ReleaseSetObservation{ReleaseID: candidate.ID, Ready: true, Components: map[string]supervisor.ComponentObservation{}}
for name := range candidate.Components {
observation.Components[name] = supervisor.ComponentObservation{Revision: candidate.Revision, Health: true, Ready: true}
}
if !l.RecordSet(observation) || !l.PromoteSet() {
t.Fatal("promotion failed")
}
if l.RollbackTargetReleaseSet == nil || l.RollbackTargetReleaseSet.Revision != "r1" {
t.Fatalf("rollback target=%+v", l.RollbackTargetReleaseSet)
}
}
@@ -0,0 +1,15 @@
#!/usr/bin/env bash
set -euo pipefail
root="$(CDPATH= cd -- "$(dirname -- "$0")/../.." && pwd)"
canary="$root/deploy/run-production-canary.sh"
supervisor="$root/deploy/update-supervisor.sh"
grep -Fq 'up -d --no-build --no-deps "${services[@]}"' "$canary"
grep -Fq 'compose up -d --no-build --no-deps supervisor' "$supervisor"
grep -Fq 'stable_ids_before=$(stable_ids)' "$canary"
grep -Fq 'assert_stable_ids_unchanged' "$canary"
grep -Fq 'before=$(stable_ids)' "$supervisor"
grep -Fq 'after=$(stable_ids)' "$supervisor"
grep -Fq 'stable container IDs changed' "$supervisor"
echo 'stable release-set guard: PASS'