128 lines
6.9 KiB
Bash
128 lines
6.9 KiB
Bash
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
|
|
BASE_DIR="${REPO_DIR:-$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)}"
|
|
ENV_FILE="${ENV_FILE:-/home/m/assistant-deploy/production-compose.env}"
|
|
COMPOSE_FILE="$BASE_DIR/deploy/docker-compose.prod.yml"
|
|
SUPERVISOR_CONTROL_URL="${SUPERVISOR_CONTROL_URL:-http://127.0.0.1:${SUPERVISOR_CONTROL_PORT:-18085}}"
|
|
REVISION="${1:?candidate revision is required}"
|
|
HEALTH_ATTEMPTS="${HEALTH_ATTEMPTS:-360}"
|
|
|
|
set -a
|
|
. "$ENV_FILE"
|
|
set +a
|
|
|
|
compose() { docker compose --env-file "$ENV_FILE" -f "$COMPOSE_FILE" "$@"; }
|
|
control() { curl -fsS --max-time 10 -X POST "$SUPERVISOR_CONTROL_URL$1" "${@:2}" >/dev/null; }
|
|
fail_rollback() {
|
|
control /v1/traffic/rollback || true
|
|
control /v1/releases/rollback || true
|
|
echo "production ReleaseSet canary failed; stable ReleaseSet rollback requested" >&2
|
|
exit 1
|
|
}
|
|
|
|
services=(creator-web-candidate core-runtime-candidate cognitive-worker-candidate research-runtime-candidate)
|
|
containers=(assistant-creator-web-candidate-1 assistant-core-runtime-candidate-1 assistant-cognitive-worker-candidate-1 assistant-research-runtime-candidate-1)
|
|
stable_services=(creator-web core-runtime cognitive-worker research-runtime)
|
|
|
|
stable_ids() {
|
|
local service container id
|
|
for service in "${stable_services[@]}"; do
|
|
container="assistant-${service}-1"
|
|
id=$(docker inspect -f '{{.Id}}' "$container" 2>/dev/null || true)
|
|
[[ -n "$id" ]] || { echo "stable container missing: $container" >&2; return 1; }
|
|
printf '%s=%s\n' "$service" "$id"
|
|
done
|
|
}
|
|
|
|
assert_stable_ids_unchanged() {
|
|
local after
|
|
after=$(stable_ids) || fail_rollback
|
|
[[ "$after" == "$stable_ids_before" ]] || {
|
|
echo "stable container IDs changed during candidate operation" >&2
|
|
diff -u <(printf '%s\n' "$stable_ids_before") <(printf '%s\n' "$after") >&2 || true
|
|
fail_rollback
|
|
}
|
|
}
|
|
|
|
stable_ids_before=$(stable_ids) || exit 1
|
|
|
|
compose --profile assistant-canary build --build-arg "SOURCE_REVISION=$REVISION" "${services[@]}"
|
|
|
|
wait_healthy() {
|
|
local container="$1" status="" ready=""
|
|
ready=""
|
|
for attempt in $(seq 1 "$HEALTH_ATTEMPTS"); do
|
|
status=$(docker inspect -f '{{if .State.Health}}{{.State.Health.Status}}{{else}}unknown{{end}}' "$container" 2>/dev/null || true)
|
|
if [[ "$status" == healthy ]]; then ready=true; break; fi
|
|
sleep 2
|
|
done
|
|
[[ "$ready" == true ]] || fail_rollback
|
|
}
|
|
|
|
compose --profile assistant-canary up -d --no-build --no-deps research-runtime-candidate core-runtime-candidate
|
|
wait_healthy assistant-research-runtime-candidate-1
|
|
wait_healthy assistant-core-runtime-candidate-1
|
|
compose --profile assistant-canary up -d --no-build --no-deps cognitive-worker-candidate
|
|
wait_healthy assistant-cognitive-worker-candidate-1
|
|
compose --profile assistant-canary up -d --no-build --no-deps creator-web-candidate
|
|
wait_healthy assistant-creator-web-candidate-1
|
|
assert_stable_ids_unchanged
|
|
|
|
for container in "${containers[@]}"; do
|
|
image_revision=$(docker inspect -f '{{index .Config.Labels "org.opencontainers.image.revision"}}' "$container" 2>/dev/null || true)
|
|
[[ "$image_revision" == "$REVISION" ]] || fail_rollback
|
|
done
|
|
assert_stable_ids_unchanged
|
|
|
|
if [[ "${PREPARE_ONLY:-false}" == true ]]; then
|
|
echo "production ReleaseSet candidate prepared revision=$REVISION"
|
|
exit 0
|
|
fi
|
|
|
|
release_json=$(cat <<JSON
|
|
{
|
|
"id":"candidate-$REVISION",
|
|
"revision":"$REVISION",
|
|
"components":{
|
|
"creator-web":{"name":"creator-web","revision":"$REVISION","upstream":"http://creator-web-candidate:8084","health_url":"http://creator-web-candidate:8084/healthz","ready_url":"http://creator-web-candidate:8084/healthz"},
|
|
"core-runtime":{"name":"core-runtime","revision":"$REVISION","upstream":"http://core-runtime-candidate:8080","health_url":"http://core-runtime-candidate:8080/healthz","ready_url":"http://core-runtime-candidate:8080/readyz"},
|
|
"cognitive-worker":{"name":"cognitive-worker","revision":"$REVISION","upstream":"http://cognitive-worker-candidate:8083","health_url":"http://cognitive-worker-candidate:8083/healthz","ready_url":"http://cognitive-worker-candidate:8083/readyz"},
|
|
"research-runtime":{"name":"research-runtime","revision":"$REVISION","upstream":"http://research-runtime-candidate:8087","health_url":"http://research-runtime-candidate:8087/healthz","ready_url":"http://research-runtime-candidate:8087/readyz"}
|
|
}
|
|
}
|
|
JSON
|
|
)
|
|
|
|
stable_json=$(curl -fsS --max-time 10 "$SUPERVISOR_CONTROL_URL/v1/state")
|
|
stable_set=$(printf '%s' "$stable_json" | jq -c '.stable_release_set // empty')
|
|
if [[ -z "$stable_set" ]]; then
|
|
stable_revision=$(printf '%s' "$stable_json" | jq -r '.current.version // "unknown"')
|
|
stable_set=$(cat <<JSON
|
|
{"id":"stable-$stable_revision","revision":"$stable_revision","components":{
|
|
"creator-web":{"name":"creator-web","revision":"$stable_revision","upstream":"http://creator-web:8084","health_url":"http://creator-web:8084/healthz","ready_url":"http://creator-web:8084/readyz"},
|
|
"core-runtime":{"name":"core-runtime","revision":"$stable_revision","upstream":"http://core-runtime:8080","health_url":"http://core-runtime:8080/healthz","ready_url":"http://core-runtime:8080/readyz"},
|
|
"cognitive-worker":{"name":"cognitive-worker","revision":"$stable_revision","upstream":"http://cognitive-worker:8083","health_url":"http://cognitive-worker:8083/healthz","ready_url":"http://cognitive-worker:8083/readyz"},
|
|
"research-runtime":{"name":"research-runtime","revision":"$stable_revision","upstream":"http://research-runtime:8087","health_url":"http://research-runtime:8087/healthz","ready_url":"http://research-runtime:8087/readyz"}}}
|
|
JSON
|
|
)
|
|
curl -fsS --max-time 10 -X POST "$SUPERVISOR_CONTROL_URL/v1/releases/baseline-set" -H 'Content-Type: application/json' --data "$stable_set" >/dev/null || fail_rollback
|
|
fi
|
|
|
|
curl -fsS --max-time 10 -X POST "$SUPERVISOR_CONTROL_URL/v1/releases/start-set" -H 'Content-Type: application/json' --data "$release_json" >/dev/null || fail_rollback
|
|
assert_stable_ids_unchanged
|
|
observation=$(printf '{"release_id":"candidate-%s","ready":true,"reason":"all ReleaseSet healthchecks and image revisions passed","observed_at_unix_millis":%s,"components":{"creator-web":{"revision":"%s","health":true,"ready":true},"core-runtime":{"revision":"%s","health":true,"ready":true},"cognitive-worker":{"revision":"%s","health":true,"ready":true},"research-runtime":{"revision":"%s","health":true,"ready":true}}}\n' "$REVISION" "$(date +%s%3N)" "$REVISION" "$REVISION" "$REVISION" "$REVISION")
|
|
curl -fsS --max-time 10 -X POST "$SUPERVISOR_CONTROL_URL/v1/releases/observe-set" -H 'Content-Type: application/json' --data "$observation" >/dev/null || fail_rollback
|
|
assert_stable_ids_unchanged
|
|
control /v1/traffic/canary || fail_rollback
|
|
assert_stable_ids_unchanged
|
|
|
|
headers=$(mktemp)
|
|
trap 'rm -f "$headers"' EXIT
|
|
if ! curl -fsS --max-time 10 -D "$headers" -H 'X-Assistant-Canary: true' "https://assistant.cin.su/login" -o /dev/null; then fail_rollback; fi
|
|
grep -qi '^X-Assistant-Traffic-Target: candidate' "$headers" || fail_rollback
|
|
|
|
control /v1/releases/promote || fail_rollback
|
|
assert_stable_ids_unchanged
|
|
echo "production ReleaseSet canary promoted revision=$REVISION"
|