Files
Assistant/deploy/run-production-canary.sh
2026-08-30 13:55:35 +03:00

114 lines
7.6 KiB
Bash

#!/usr/bin/env bash
set -euo pipefail
# Candidate services are never part of the stable Compose project. The slot
# Compose file may use only declared external networks and durable resources.
BASE_DIR="${REPO_DIR:-$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)}"
ENV_FILE="${ENV_FILE:-/home/m/assistant-deploy/production-compose.env}"
CANDIDATE_FILE="$BASE_DIR/deploy/docker-compose.candidate-slot.yml"
SUPERVISOR_CONTROL_URL="${SUPERVISOR_CONTROL_URL:-http://127.0.0.1:${SUPERVISOR_CONTROL_PORT:-18085}}"
REVISION="${1:?candidate revision is required}"
SLOT="${CANDIDATE_SLOT:-b}"
STABLE_PROJECT="${STABLE_PROJECT:-assistant}"
CANDIDATE_PROJECT="${CANDIDATE_PROJECT:-assistant-slot-$SLOT}"
HEALTH_ATTEMPTS="${HEALTH_ATTEMPTS:-360}"
PROMOTE_AFTER_CANARY="${PROMOTE_AFTER_CANARY:-true}"
[[ "$SLOT" =~ ^[a-z0-9-]+$ ]] || { echo "invalid candidate slot" >&2; exit 2; }
[[ "$CANDIDATE_PROJECT" != "$STABLE_PROJECT" ]] || { echo "candidate and stable Compose projects must differ" >&2; exit 2; }
candidate_compose() { docker compose -p "$CANDIDATE_PROJECT" --env-file "$ENV_FILE" -f "$CANDIDATE_FILE" "$@"; }
control() { curl -fsS --max-time 10 -X POST "$SUPERVISOR_CONTROL_URL$1" "${@:2}" >/dev/null; }
fail_rollback() {
control /v1/traffic/rollback || true
control /v1/releases/rollback || true
echo "isolated candidate ReleaseSet failed; stable ReleaseSet rollback requested" >&2
exit 1
}
services=(creator-web core-runtime cognitive-worker research-runtime)
stable_services=(creator-web core-runtime cognitive-worker research-runtime)
stable_ids() {
local service container id
for service in "${stable_services[@]}"; do
container="${STABLE_PROJECT}-${service}-1"
id=$(docker inspect -f '{{.Id}}' "$container" 2>/dev/null || true)
[[ -n "$id" ]] || { echo "stable container missing: $container" >&2; return 1; }
printf '%s=%s\n' "$service" "$id"
done
}
assert_stable_ids_unchanged() {
local after
after=$(stable_ids) || fail_rollback
[[ "$after" == "$stable_ids_before" ]] || {
echo "stable container IDs changed during isolated candidate operation" >&2
diff -u <(printf '%s\n' "$stable_ids_before") <(printf '%s\n' "$after") >&2 || true
fail_rollback
}
}
candidate_container() { printf '%s-%s-1' "$CANDIDATE_PROJECT" "$1"; }
wait_healthy() {
local container="$1" status="" ready=""
for attempt in $(seq 1 "$HEALTH_ATTEMPTS"); do
status=$(docker inspect -f '{{if .State.Health}}{{.State.Health.Status}}{{else}}unknown{{end}}' "$container" 2>/dev/null || true)
if [[ "$status" == healthy ]]; then ready=true; break; fi
sleep 2
done
[[ "$ready" == true ]] || fail_rollback
}
stable_ids_before=$(stable_ids) || exit 1
candidate_compose config >/dev/null
candidate_compose build --build-arg "SOURCE_REVISION=$REVISION" "${services[@]}"
assert_stable_ids_unchanged
candidate_compose up -d --no-build --no-deps research-runtime core-runtime
wait_healthy "$(candidate_container research-runtime)"
wait_healthy "$(candidate_container core-runtime)"
candidate_compose up -d --no-build --no-deps cognitive-worker
wait_healthy "$(candidate_container cognitive-worker)"
candidate_compose up -d --no-build --no-deps creator-web
wait_healthy "$(candidate_container creator-web)"
assert_stable_ids_unchanged
for service in "${services[@]}"; do
image_revision=$(docker inspect -f '{{index .Config.Labels "org.opencontainers.image.revision"}}' "$(candidate_container "$service")" 2>/dev/null || true)
[[ "$image_revision" == "$REVISION" ]] || fail_rollback
done
assert_stable_ids_unchanged
if [[ "${PREPARE_ONLY:-false}" == true ]]; then
echo "isolated candidate ReleaseSet prepared slot=$SLOT project=$CANDIDATE_PROJECT revision=$REVISION"
exit 0
fi
release_json=$(cat <<JSON
{"id":"candidate-$SLOT-$REVISION","slot":"$SLOT","revision":"$REVISION","components":{"creator-web":{"name":"creator-web","revision":"$REVISION","upstream":"http://creator-web-candidate:8084","health_url":"http://creator-web-candidate:8084/healthz","ready_url":"http://creator-web-candidate:8084/healthz"},"core-runtime":{"name":"core-runtime","revision":"$REVISION","upstream":"http://core-runtime-candidate:8080","health_url":"http://core-runtime-candidate:8080/healthz","ready_url":"http://core-runtime-candidate:8080/readyz"},"cognitive-worker":{"name":"cognitive-worker","revision":"$REVISION","upstream":"http://cognitive-worker-candidate:8083","health_url":"http://cognitive-worker-candidate:8083/healthz","ready_url":"http://cognitive-worker-candidate:8083/readyz"},"research-runtime":{"name":"research-runtime","revision":"$REVISION","upstream":"http://research-runtime-candidate:8087","health_url":"http://research-runtime-candidate:8087/healthz","ready_url":"http://research-runtime-candidate:8087/readyz"}}}
JSON
)
stable_json=$(curl -fsS --max-time 10 "$SUPERVISOR_CONTROL_URL/v1/state")
stable_set=$(printf '%s' "$stable_json" | jq -c '.stable_release_set // empty')
if [[ -z "$stable_set" ]]; then
stable_revision=$(printf '%s' "$stable_json" | jq -r '.current.version // "unknown"')
stable_set=$(cat <<JSON
{"id":"stable-$stable_revision","slot":"legacy-stable","revision":"$stable_revision","components":{"creator-web":{"name":"creator-web","revision":"$stable_revision","upstream":"http://creator-web:8084","health_url":"http://creator-web:8084/healthz","ready_url":"http://creator-web:8084/healthz"},"core-runtime":{"name":"core-runtime","revision":"$stable_revision","upstream":"http://core-runtime:8080","health_url":"http://core-runtime:8080/healthz","ready_url":"http://core-runtime:8080/readyz"},"cognitive-worker":{"name":"cognitive-worker","revision":"$stable_revision","upstream":"http://cognitive-worker:8083","health_url":"http://cognitive-worker:8083/healthz","ready_url":"http://cognitive-worker:8083/readyz"},"research-runtime":{"name":"research-runtime","revision":"$stable_revision","upstream":"http://research-runtime:8087","health_url":"http://research-runtime:8087/healthz","ready_url":"http://research-runtime:8087/readyz"}}}
JSON
)
curl -fsS --max-time 10 -X POST "$SUPERVISOR_CONTROL_URL/v1/releases/baseline-set" -H 'Content-Type: application/json' --data "$stable_set" >/dev/null || fail_rollback
fi
curl -fsS --max-time 10 -X POST "$SUPERVISOR_CONTROL_URL/v1/releases/start-set" -H 'Content-Type: application/json' --data "$release_json" >/dev/null || fail_rollback
assert_stable_ids_unchanged
observation=$(printf '{"release_id":"candidate-%s-%s","ready":true,"reason":"isolated slot health and image revisions passed","observed_at_unix_millis":%s,"components":{"creator-web":{"revision":"%s","health":true,"ready":true},"core-runtime":{"revision":"%s","health":true,"ready":true},"cognitive-worker":{"revision":"%s","health":true,"ready":true},"research-runtime":{"revision":"%s","health":true,"ready":true}}}\n' "$SLOT" "$REVISION" "$(date +%s%3N)" "$REVISION" "$REVISION" "$REVISION" "$REVISION")
curl -fsS --max-time 10 -X POST "$SUPERVISOR_CONTROL_URL/v1/releases/observe-set" -H 'Content-Type: application/json' --data "$observation" >/dev/null || fail_rollback
assert_stable_ids_unchanged
control /v1/traffic/canary || fail_rollback
assert_stable_ids_unchanged
headers=$(mktemp)
trap 'rm -f "$headers"' EXIT
if ! curl -fsS --max-time 10 -D "$headers" -H 'X-Assistant-Canary: true' "https://assistant.cin.su/login" -o /dev/null; then fail_rollback; fi
grep -qi '^X-Assistant-Traffic-Target: candidate' "$headers" || fail_rollback
if [[ "$PROMOTE_AFTER_CANARY" != true ]]; then
echo "isolated candidate ReleaseSet canary ready slot=$SLOT project=$CANDIDATE_PROJECT revision=$REVISION"
exit 0
fi
control /v1/releases/promote || fail_rollback
assert_stable_ids_unchanged
echo "isolated candidate ReleaseSet promoted slot=$SLOT project=$CANDIDATE_PROJECT revision=$REVISION"