fix: guard stable containers during compat observation deploy
CI / test (push) Failing after 35s

This commit is contained in:
m
2026-08-30 13:01:32 +03:00
parent ebdfaa0769
commit d0210e5aed
5 changed files with 83 additions and 5 deletions
+30 -1
View File
@@ -22,9 +22,33 @@ fail_rollback() {
services=(creator-web-candidate core-runtime-candidate cognitive-worker-candidate research-runtime-candidate)
containers=(assistant-creator-web-candidate-1 assistant-core-runtime-candidate-1 assistant-cognitive-worker-candidate-1 assistant-research-runtime-candidate-1)
stable_services=(creator-web core-runtime cognitive-worker research-runtime)
stable_ids() {
local service container id
for service in "${stable_services[@]}"; do
container="assistant-${service}-1"
id=$(docker inspect -f '{{.Id}}' "$container" 2>/dev/null || true)
[[ -n "$id" ]] || { echo "stable container missing: $container" >&2; return 1; }
printf '%s=%s\n' "$service" "$id"
done
}
assert_stable_ids_unchanged() {
local after
after=$(stable_ids) || fail_rollback
[[ "$after" == "$stable_ids_before" ]] || {
echo "stable container IDs changed during candidate operation" >&2
diff -u <(printf '%s\n' "$stable_ids_before") <(printf '%s\n' "$after") >&2 || true
fail_rollback
}
}
stable_ids_before=$(stable_ids) || exit 1
compose --profile assistant-canary build --build-arg "SOURCE_REVISION=$REVISION" "${services[@]}"
compose --profile assistant-canary up -d --no-build "${services[@]}"
compose --profile assistant-canary up -d --no-build --no-deps "${services[@]}"
assert_stable_ids_unchanged
for container in "${containers[@]}"; do
ready=""
@@ -37,6 +61,7 @@ for container in "${containers[@]}"; do
image_revision=$(docker inspect -f '{{index .Config.Labels "org.opencontainers.image.revision"}}' "$container" 2>/dev/null || true)
[[ "$image_revision" == "$REVISION" ]] || fail_rollback
done
assert_stable_ids_unchanged
release_json=$(cat <<JSON
{
@@ -68,9 +93,12 @@ JSON
fi
curl -fsS --max-time 10 -X POST "$SUPERVISOR_CONTROL_URL/v1/releases/start-set" -H 'Content-Type: application/json' --data "$release_json" >/dev/null || fail_rollback
assert_stable_ids_unchanged
observation=$(printf '{"release_id":"candidate-%s","ready":true,"reason":"all ReleaseSet healthchecks and image revisions passed","observed_at_unix_millis":%s,"components":{"creator-web":{"revision":"%s","health":true,"ready":true},"core-runtime":{"revision":"%s","health":true,"ready":true},"cognitive-worker":{"revision":"%s","health":true,"ready":true},"research-runtime":{"revision":"%s","health":true,"ready":true}}}\n' "$REVISION" "$(date +%s%3N)" "$REVISION" "$REVISION" "$REVISION" "$REVISION")
curl -fsS --max-time 10 -X POST "$SUPERVISOR_CONTROL_URL/v1/releases/observe-set" -H 'Content-Type: application/json' --data "$observation" >/dev/null || fail_rollback
assert_stable_ids_unchanged
control /v1/traffic/canary || fail_rollback
assert_stable_ids_unchanged
headers=$(mktemp)
trap 'rm -f "$headers"' EXIT
@@ -78,4 +106,5 @@ if ! curl -fsS --max-time 10 -D "$headers" -H 'X-Assistant-Canary: true' "https:
grep -qi '^X-Assistant-Traffic-Target: candidate' "$headers" || fail_rollback
control /v1/releases/promote || fail_rollback
assert_stable_ids_unchanged
echo "production ReleaseSet canary promoted revision=$REVISION"
+32
View File
@@ -0,0 +1,32 @@
#!/usr/bin/env bash
set -euo pipefail
BASE_DIR="${REPO_DIR:-$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)}"
ENV_FILE="${ENV_FILE:-/home/m/assistant-deploy/production-compose.env}"
COMPOSE_FILE="$BASE_DIR/deploy/docker-compose.prod.yml"
set -a
. "$ENV_FILE"
set +a
compose() { docker compose --env-file "$ENV_FILE" -f "$COMPOSE_FILE" "$@"; }
stable_services=(creator-web core-runtime cognitive-worker research-runtime)
stable_ids() {
local service container id
for service in "${stable_services[@]}"; do
container="assistant-${service}-1"
id=$(docker inspect -f '{{.Id}}' "$container" 2>/dev/null || true)
[[ -n "$id" ]] || { echo "stable container missing: $container" >&2; return 1; }
printf '%s=%s\n' "$service" "$id"
done
}
before=$(stable_ids)
compose up -d --no-build --no-deps supervisor
after=$(stable_ids)
[[ "$before" == "$after" ]] || {
echo "stable container IDs changed while updating supervisor" >&2
diff -u <(printf '%s\n' "$before") <(printf '%s\n' "$after") >&2 || true
exit 1
}
printf '%s\n' "$after"