82 lines
5.4 KiB
Bash
82 lines
5.4 KiB
Bash
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
|
|
BASE_DIR="${REPO_DIR:-$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)}"
|
|
ENV_FILE="${ENV_FILE:-/home/m/assistant-deploy/production-compose.env}"
|
|
COMPOSE_FILE="$BASE_DIR/deploy/docker-compose.prod.yml"
|
|
SUPERVISOR_CONTROL_URL="${SUPERVISOR_CONTROL_URL:-http://127.0.0.1:${SUPERVISOR_CONTROL_PORT:-18085}}"
|
|
REVISION="${1:?candidate revision is required}"
|
|
|
|
set -a
|
|
. "$ENV_FILE"
|
|
set +a
|
|
|
|
compose() { docker compose --env-file "$ENV_FILE" -f "$COMPOSE_FILE" "$@"; }
|
|
control() { curl -fsS --max-time 10 -X POST "$SUPERVISOR_CONTROL_URL$1" "${@:2}" >/dev/null; }
|
|
fail_rollback() {
|
|
control /v1/traffic/rollback || true
|
|
control /v1/releases/rollback || true
|
|
echo "production ReleaseSet canary failed; stable ReleaseSet rollback requested" >&2
|
|
exit 1
|
|
}
|
|
|
|
services=(creator-web-candidate core-runtime-candidate cognitive-worker-candidate research-runtime-candidate)
|
|
containers=(assistant-creator-web-candidate-1 assistant-core-runtime-candidate-1 assistant-cognitive-worker-candidate-1 assistant-research-runtime-candidate-1)
|
|
|
|
compose --profile assistant-canary build --build-arg "SOURCE_REVISION=$REVISION" "${services[@]}"
|
|
compose --profile assistant-canary up -d --no-build "${services[@]}"
|
|
|
|
for container in "${containers[@]}"; do
|
|
ready=""
|
|
for attempt in $(seq 1 60); do
|
|
status="$(docker inspect -f '{{if .State.Health}}{{.State.Health.Status}}{{else}}unknown{{end}}' "$container" 2>/dev/null || true)"
|
|
if [[ "$status" == healthy ]]; then ready=true; break; fi
|
|
sleep 2
|
|
done
|
|
[[ "$ready" == true ]] || fail_rollback
|
|
image_revision="$(docker inspect -f '{{index .Config.Labels "org.opencontainers.image.revision"}}' "$container" 2>/dev/null || true)"
|
|
[[ "$image_revision" == "$REVISION" ]] || fail_rollback
|
|
done
|
|
|
|
release_json="$(cat <<JSON
|
|
{
|
|
"id":"candidate-$REVISION",
|
|
"revision":"$REVISION",
|
|
"components":{
|
|
"creator-web":{"name":"creator-web","revision":"$REVISION","upstream":"http://creator-web-candidate:8084","health_url":"http://creator-web-candidate:8084/healthz","ready_url":"http://creator-web-candidate:8084/readyz"},
|
|
"core-runtime":{"name":"core-runtime","revision":"$REVISION","upstream":"http://core-runtime-candidate:8080","health_url":"http://core-runtime-candidate:8080/healthz","ready_url":"http://core-runtime-candidate:8080/readyz"},
|
|
"cognitive-worker":{"name":"cognitive-worker","revision":"$REVISION","upstream":"http://cognitive-worker-candidate:8083","health_url":"http://cognitive-worker-candidate:8083/healthz","ready_url":"http://cognitive-worker-candidate:8083/readyz"},
|
|
"research-runtime":{"name":"research-runtime","revision":"$REVISION","upstream":"http://research-runtime-candidate:8087","health_url":"http://research-runtime-candidate:8087/healthz","ready_url":"http://research-runtime-candidate:8087/readyz"}
|
|
}
|
|
}
|
|
JSON
|
|
)
|
|
|
|
stable_json="$(curl -fsS --max-time 10 "$SUPERVISOR_CONTROL_URL/v1/state")"
|
|
stable_set="$(printf '%s' "$stable_json" | python3 -c "import json,sys; s=json.load(sys.stdin).get('stable_release_set'); print(json.dumps(s,separators=(',',':')) if s else '')")"
|
|
if [[ -z "$stable_set" ]]; then
|
|
stable_revision="$(printf '%s' "$stable_json" | python3 -c "import json,sys; print(json.load(sys.stdin).get('current',{}).get('version','unknown'))")"
|
|
stable_set="$(cat <<JSON
|
|
{"id":"stable-$stable_revision","revision":"$stable_revision","components":{
|
|
"creator-web":{"name":"creator-web","revision":"$stable_revision","upstream":"http://creator-web:8084","health_url":"http://creator-web:8084/healthz","ready_url":"http://creator-web:8084/readyz"},
|
|
"core-runtime":{"name":"core-runtime","revision":"$stable_revision","upstream":"http://core-runtime:8080","health_url":"http://core-runtime:8080/healthz","ready_url":"http://core-runtime:8080/readyz"},
|
|
"cognitive-worker":{"name":"cognitive-worker","revision":"$stable_revision","upstream":"http://cognitive-worker:8083","health_url":"http://cognitive-worker:8083/healthz","ready_url":"http://cognitive-worker:8083/readyz"},
|
|
"research-runtime":{"name":"research-runtime","revision":"$stable_revision","upstream":"http://research-runtime:8087","health_url":"http://research-runtime:8087/healthz","ready_url":"http://research-runtime:8087/readyz"}}}
|
|
JSON
|
|
)"
|
|
curl -fsS --max-time 10 -X POST "$SUPERVISOR_CONTROL_URL/v1/releases/baseline-set" -H 'Content-Type: application/json' --data "$stable_set" >/dev/null || fail_rollback
|
|
fi
|
|
|
|
curl -fsS --max-time 10 -X POST "$SUPERVISOR_CONTROL_URL/v1/releases/start-set" -H 'Content-Type: application/json' --data "$release_json" >/dev/null || fail_rollback
|
|
observation="$(printf '%s' "$release_json" | python3 -c "import json,sys,time; s=json.load(sys.stdin); print(json.dumps({'release_id':s['id'],'ready':True,'reason':'all ReleaseSet healthchecks and image revisions passed','observed_at_unix_millis':int(time.time()*1000),'components':{k:{'revision':v['revision'],'health':True,'ready':True} for k,v in s['components'].items()}},separators=(',',':')))" )"
|
|
curl -fsS --max-time 10 -X POST "$SUPERVISOR_CONTROL_URL/v1/releases/observe-set" -H 'Content-Type: application/json' --data "$observation" >/dev/null || fail_rollback
|
|
control /v1/traffic/canary || fail_rollback
|
|
|
|
headers="$(mktemp)"
|
|
trap 'rm -f "$headers"' EXIT
|
|
if ! curl -fsS --max-time 10 -D "$headers" -H 'X-Assistant-Canary: true' "https://assistant.cin.su/login" -o /dev/null; then fail_rollback; fi
|
|
grep -qi '^X-Assistant-Traffic-Target: candidate' "$headers" || fail_rollback
|
|
|
|
control /v1/releases/promote || fail_rollback
|
|
echo "production ReleaseSet canary promoted revision=$REVISION"
|