Files
Assistant/deploy/run-production-canary.sh
T

82 lines
5.4 KiB
Bash

#!/usr/bin/env bash
set -euo pipefail
BASE_DIR="${REPO_DIR:-$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)}"
ENV_FILE="${ENV_FILE:-/home/m/assistant-deploy/production-compose.env}"
COMPOSE_FILE="$BASE_DIR/deploy/docker-compose.prod.yml"
SUPERVISOR_CONTROL_URL="${SUPERVISOR_CONTROL_URL:-http://127.0.0.1:${SUPERVISOR_CONTROL_PORT:-18085}}"
REVISION="${1:?candidate revision is required}"
set -a
. "$ENV_FILE"
set +a
compose() { docker compose --env-file "$ENV_FILE" -f "$COMPOSE_FILE" "$@"; }
control() { curl -fsS --max-time 10 -X POST "$SUPERVISOR_CONTROL_URL$1" "${@:2}" >/dev/null; }
fail_rollback() {
control /v1/traffic/rollback || true
control /v1/releases/rollback || true
echo "production ReleaseSet canary failed; stable ReleaseSet rollback requested" >&2
exit 1
}
services=(creator-web-candidate core-runtime-candidate cognitive-worker-candidate research-runtime-candidate)
containers=(assistant-creator-web-candidate-1 assistant-core-runtime-candidate-1 assistant-cognitive-worker-candidate-1 assistant-research-runtime-candidate-1)
compose --profile assistant-canary build --build-arg "SOURCE_REVISION=$REVISION" "${services[@]}"
compose --profile assistant-canary up -d --no-build "${services[@]}"
for container in "${containers[@]}"; do
ready=""
for attempt in $(seq 1 60); do
status="$(docker inspect -f '{{if .State.Health}}{{.State.Health.Status}}{{else}}unknown{{end}}' "$container" 2>/dev/null || true)"
if [[ "$status" == healthy ]]; then ready=true; break; fi
sleep 2
done
[[ "$ready" == true ]] || fail_rollback
image_revision="$(docker inspect -f '{{index .Config.Labels "org.opencontainers.image.revision"}}' "$container" 2>/dev/null || true)"
[[ "$image_revision" == "$REVISION" ]] || fail_rollback
done
release_json="$(cat <<JSON
{
"id":"candidate-$REVISION",
"revision":"$REVISION",
"components":{
"creator-web":{"name":"creator-web","revision":"$REVISION","upstream":"http://creator-web-candidate:8084","health_url":"http://creator-web-candidate:8084/healthz","ready_url":"http://creator-web-candidate:8084/readyz"},
"core-runtime":{"name":"core-runtime","revision":"$REVISION","upstream":"http://core-runtime-candidate:8080","health_url":"http://core-runtime-candidate:8080/healthz","ready_url":"http://core-runtime-candidate:8080/readyz"},
"cognitive-worker":{"name":"cognitive-worker","revision":"$REVISION","upstream":"http://cognitive-worker-candidate:8083","health_url":"http://cognitive-worker-candidate:8083/healthz","ready_url":"http://cognitive-worker-candidate:8083/readyz"},
"research-runtime":{"name":"research-runtime","revision":"$REVISION","upstream":"http://research-runtime-candidate:8087","health_url":"http://research-runtime-candidate:8087/healthz","ready_url":"http://research-runtime-candidate:8087/readyz"}
}
}
JSON
)
stable_json="$(curl -fsS --max-time 10 "$SUPERVISOR_CONTROL_URL/v1/state")"
stable_set="$(printf '%s' "$stable_json" | python3 -c "import json,sys; s=json.load(sys.stdin).get('stable_release_set'); print(json.dumps(s,separators=(',',':')) if s else '')")"
if [[ -z "$stable_set" ]]; then
stable_revision="$(printf '%s' "$stable_json" | python3 -c "import json,sys; print(json.load(sys.stdin).get('current',{}).get('version','unknown'))")"
stable_set="$(cat <<JSON
{"id":"stable-$stable_revision","revision":"$stable_revision","components":{
"creator-web":{"name":"creator-web","revision":"$stable_revision","upstream":"http://creator-web:8084","health_url":"http://creator-web:8084/healthz","ready_url":"http://creator-web:8084/readyz"},
"core-runtime":{"name":"core-runtime","revision":"$stable_revision","upstream":"http://core-runtime:8080","health_url":"http://core-runtime:8080/healthz","ready_url":"http://core-runtime:8080/readyz"},
"cognitive-worker":{"name":"cognitive-worker","revision":"$stable_revision","upstream":"http://cognitive-worker:8083","health_url":"http://cognitive-worker:8083/healthz","ready_url":"http://cognitive-worker:8083/readyz"},
"research-runtime":{"name":"research-runtime","revision":"$stable_revision","upstream":"http://research-runtime:8087","health_url":"http://research-runtime:8087/healthz","ready_url":"http://research-runtime:8087/readyz"}}}
JSON
)"
curl -fsS --max-time 10 -X POST "$SUPERVISOR_CONTROL_URL/v1/releases/baseline-set" -H 'Content-Type: application/json' --data "$stable_set" >/dev/null || fail_rollback
fi
curl -fsS --max-time 10 -X POST "$SUPERVISOR_CONTROL_URL/v1/releases/start-set" -H 'Content-Type: application/json' --data "$release_json" >/dev/null || fail_rollback
observation="$(printf '%s' "$release_json" | python3 -c "import json,sys,time; s=json.load(sys.stdin); print(json.dumps({'release_id':s['id'],'ready':True,'reason':'all ReleaseSet healthchecks and image revisions passed','observed_at_unix_millis':int(time.time()*1000),'components':{k:{'revision':v['revision'],'health':True,'ready':True} for k,v in s['components'].items()}},separators=(',',':')))" )"
curl -fsS --max-time 10 -X POST "$SUPERVISOR_CONTROL_URL/v1/releases/observe-set" -H 'Content-Type: application/json' --data "$observation" >/dev/null || fail_rollback
control /v1/traffic/canary || fail_rollback
headers="$(mktemp)"
trap 'rm -f "$headers"' EXIT
if ! curl -fsS --max-time 10 -D "$headers" -H 'X-Assistant-Canary: true' "https://assistant.cin.su/login" -o /dev/null; then fail_rollback; fi
grep -qi '^X-Assistant-Traffic-Target: candidate' "$headers" || fail_rollback
control /v1/releases/promote || fail_rollback
echo "production ReleaseSet canary promoted revision=$REVISION"