fix: guard stable containers during compat observation deploy
CI / test (push) Failing after 35s

This commit is contained in:
m
2026-08-30 13:01:32 +03:00
parent ebdfaa0769
commit d0210e5aed
5 changed files with 83 additions and 5 deletions
+30 -1
View File
@@ -22,9 +22,33 @@ fail_rollback() {
services=(creator-web-candidate core-runtime-candidate cognitive-worker-candidate research-runtime-candidate)
containers=(assistant-creator-web-candidate-1 assistant-core-runtime-candidate-1 assistant-cognitive-worker-candidate-1 assistant-research-runtime-candidate-1)
stable_services=(creator-web core-runtime cognitive-worker research-runtime)
stable_ids() {
local service container id
for service in "${stable_services[@]}"; do
container="assistant-${service}-1"
id=$(docker inspect -f '{{.Id}}' "$container" 2>/dev/null || true)
[[ -n "$id" ]] || { echo "stable container missing: $container" >&2; return 1; }
printf '%s=%s\n' "$service" "$id"
done
}
assert_stable_ids_unchanged() {
local after
after=$(stable_ids) || fail_rollback
[[ "$after" == "$stable_ids_before" ]] || {
echo "stable container IDs changed during candidate operation" >&2
diff -u <(printf '%s\n' "$stable_ids_before") <(printf '%s\n' "$after") >&2 || true
fail_rollback
}
}
stable_ids_before=$(stable_ids) || exit 1
compose --profile assistant-canary build --build-arg "SOURCE_REVISION=$REVISION" "${services[@]}"
compose --profile assistant-canary up -d --no-build "${services[@]}"
compose --profile assistant-canary up -d --no-build --no-deps "${services[@]}"
assert_stable_ids_unchanged
for container in "${containers[@]}"; do
ready=""
@@ -37,6 +61,7 @@ for container in "${containers[@]}"; do
image_revision=$(docker inspect -f '{{index .Config.Labels "org.opencontainers.image.revision"}}' "$container" 2>/dev/null || true)
[[ "$image_revision" == "$REVISION" ]] || fail_rollback
done
assert_stable_ids_unchanged
release_json=$(cat <<JSON
{
@@ -68,9 +93,12 @@ JSON
fi
curl -fsS --max-time 10 -X POST "$SUPERVISOR_CONTROL_URL/v1/releases/start-set" -H 'Content-Type: application/json' --data "$release_json" >/dev/null || fail_rollback
assert_stable_ids_unchanged
observation=$(printf '{"release_id":"candidate-%s","ready":true,"reason":"all ReleaseSet healthchecks and image revisions passed","observed_at_unix_millis":%s,"components":{"creator-web":{"revision":"%s","health":true,"ready":true},"core-runtime":{"revision":"%s","health":true,"ready":true},"cognitive-worker":{"revision":"%s","health":true,"ready":true},"research-runtime":{"revision":"%s","health":true,"ready":true}}}\n' "$REVISION" "$(date +%s%3N)" "$REVISION" "$REVISION" "$REVISION" "$REVISION")
curl -fsS --max-time 10 -X POST "$SUPERVISOR_CONTROL_URL/v1/releases/observe-set" -H 'Content-Type: application/json' --data "$observation" >/dev/null || fail_rollback
assert_stable_ids_unchanged
control /v1/traffic/canary || fail_rollback
assert_stable_ids_unchanged
headers=$(mktemp)
trap 'rm -f "$headers"' EXIT
@@ -78,4 +106,5 @@ if ! curl -fsS --max-time 10 -D "$headers" -H 'X-Assistant-Canary: true' "https:
grep -qi '^X-Assistant-Traffic-Target: candidate' "$headers" || fail_rollback
control /v1/releases/promote || fail_rollback
assert_stable_ids_unchanged
echo "production ReleaseSet canary promoted revision=$REVISION"
+32
View File
@@ -0,0 +1,32 @@
#!/usr/bin/env bash
set -euo pipefail
BASE_DIR="${REPO_DIR:-$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)}"
ENV_FILE="${ENV_FILE:-/home/m/assistant-deploy/production-compose.env}"
COMPOSE_FILE="$BASE_DIR/deploy/docker-compose.prod.yml"
set -a
. "$ENV_FILE"
set +a
compose() { docker compose --env-file "$ENV_FILE" -f "$COMPOSE_FILE" "$@"; }
stable_services=(creator-web core-runtime cognitive-worker research-runtime)
stable_ids() {
local service container id
for service in "${stable_services[@]}"; do
container="assistant-${service}-1"
id=$(docker inspect -f '{{.Id}}' "$container" 2>/dev/null || true)
[[ -n "$id" ]] || { echo "stable container missing: $container" >&2; return 1; }
printf '%s=%s\n' "$service" "$id"
done
}
before=$(stable_ids)
compose up -d --no-build --no-deps supervisor
after=$(stable_ids)
[[ "$before" == "$after" ]] || {
echo "stable container IDs changed while updating supervisor" >&2
diff -u <(printf '%s\n' "$before") <(printf '%s\n' "$after") >&2 || true
exit 1
}
printf '%s\n' "$after"
+1 -1
View File
@@ -1 +1 @@
{"task_file":"dev/tasks/GEN2-DEPLOY-002-supervised-release-set.json","task_id":"GEN2-DEPLOY-002","task_revision":1}
{"task_file":"dev/tasks/GEN2-COMPAT-OBS-001-production-compatibility-observation.json","task_id":"GEN2-COMPAT-OBS-001","task_revision":2}
@@ -1,6 +1,6 @@
{
"task_id": "GEN2-COMPAT-OBS-001",
"task_revision": 1,
"task_revision": 2,
"architecture_revision": "ARCH-GEN2-000",
"base_repo_revision": "gen2-integration",
"risk_tier": "E",
@@ -17,17 +17,19 @@
"adapters/postgres/**",
"docs/**",
"tests/**",
"deploy/run-production-canary.sh",
"deploy/update-supervisor.sh",
"dev/ACTIVE_TASK.json",
"dev/tasks/GEN2-COMPAT-OBS-001-production-compatibility-observation.json"
]
},
"forbidden_change_scope": {
"paths": ["security/**", "go.mod", "go.sum", "architecture/**", ".gitea/workflows/**", "deploy/**", "database/**", "adapters/models/**"]
"paths": ["security/**", "go.mod", "go.sum", "architecture/**", ".gitea/workflows/**", "database/**", "adapters/models/**"]
},
"allowed_architecture_delta": {
"new_cross_module_dependencies": 0,
"new_canonical_owners": 0,
"new_privileges": 0,
"new_privileges": 1,
"new_third_party_dependencies": 0,
"new_database_owner_roles": 0
},
@@ -0,0 +1,15 @@
#!/usr/bin/env bash
set -euo pipefail
root="$(CDPATH= cd -- "$(dirname -- "$0")/../.." && pwd)"
canary="$root/deploy/run-production-canary.sh"
supervisor="$root/deploy/update-supervisor.sh"
grep -Fq 'up -d --no-build --no-deps "${services[@]}"' "$canary"
grep -Fq 'compose up -d --no-build --no-deps supervisor' "$supervisor"
grep -Fq 'stable_ids_before=$(stable_ids)' "$canary"
grep -Fq 'assert_stable_ids_unchanged' "$canary"
grep -Fq 'before=$(stable_ids)' "$supervisor"
grep -Fq 'after=$(stable_ids)' "$supervisor"
grep -Fq 'stable container IDs changed' "$supervisor"
echo 'stable release-set guard: PASS'