114 lines
7.6 KiB
Bash
114 lines
7.6 KiB
Bash
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
|
|
# Candidate services are never part of the stable Compose project. The slot
|
|
# Compose file may use only declared external networks and durable resources.
|
|
BASE_DIR="${REPO_DIR:-$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)}"
|
|
ENV_FILE="${ENV_FILE:-/home/m/assistant-deploy/production-compose.env}"
|
|
CANDIDATE_FILE="$BASE_DIR/deploy/docker-compose.candidate-slot.yml"
|
|
SUPERVISOR_CONTROL_URL="${SUPERVISOR_CONTROL_URL:-http://127.0.0.1:${SUPERVISOR_CONTROL_PORT:-18085}}"
|
|
REVISION="${1:?candidate revision is required}"
|
|
SLOT="${CANDIDATE_SLOT:-b}"
|
|
STABLE_PROJECT="${STABLE_PROJECT:-assistant}"
|
|
CANDIDATE_PROJECT="${CANDIDATE_PROJECT:-assistant-slot-$SLOT}"
|
|
HEALTH_ATTEMPTS="${HEALTH_ATTEMPTS:-360}"
|
|
PROMOTE_AFTER_CANARY="${PROMOTE_AFTER_CANARY:-true}"
|
|
|
|
[[ "$SLOT" =~ ^[a-z0-9-]+$ ]] || { echo "invalid candidate slot" >&2; exit 2; }
|
|
[[ "$CANDIDATE_PROJECT" != "$STABLE_PROJECT" ]] || { echo "candidate and stable Compose projects must differ" >&2; exit 2; }
|
|
|
|
candidate_compose() { docker compose -p "$CANDIDATE_PROJECT" --env-file "$ENV_FILE" -f "$CANDIDATE_FILE" "$@"; }
|
|
control() { curl -fsS --max-time 10 -X POST "$SUPERVISOR_CONTROL_URL$1" "${@:2}" >/dev/null; }
|
|
fail_rollback() {
|
|
control /v1/traffic/rollback || true
|
|
control /v1/releases/rollback || true
|
|
echo "isolated candidate ReleaseSet failed; stable ReleaseSet rollback requested" >&2
|
|
exit 1
|
|
}
|
|
|
|
services=(creator-web core-runtime cognitive-worker research-runtime)
|
|
stable_services=(creator-web core-runtime cognitive-worker research-runtime)
|
|
stable_ids() {
|
|
local service container id
|
|
for service in "${stable_services[@]}"; do
|
|
container="${STABLE_PROJECT}-${service}-1"
|
|
id=$(docker inspect -f '{{.Id}}' "$container" 2>/dev/null || true)
|
|
[[ -n "$id" ]] || { echo "stable container missing: $container" >&2; return 1; }
|
|
printf '%s=%s\n' "$service" "$id"
|
|
done
|
|
}
|
|
assert_stable_ids_unchanged() {
|
|
local after
|
|
after=$(stable_ids) || fail_rollback
|
|
[[ "$after" == "$stable_ids_before" ]] || {
|
|
echo "stable container IDs changed during isolated candidate operation" >&2
|
|
diff -u <(printf '%s\n' "$stable_ids_before") <(printf '%s\n' "$after") >&2 || true
|
|
fail_rollback
|
|
}
|
|
}
|
|
candidate_container() { printf '%s-%s-1' "$CANDIDATE_PROJECT" "$1"; }
|
|
wait_healthy() {
|
|
local container="$1" status="" ready=""
|
|
for attempt in $(seq 1 "$HEALTH_ATTEMPTS"); do
|
|
status=$(docker inspect -f '{{if .State.Health}}{{.State.Health.Status}}{{else}}unknown{{end}}' "$container" 2>/dev/null || true)
|
|
if [[ "$status" == healthy ]]; then ready=true; break; fi
|
|
sleep 2
|
|
done
|
|
[[ "$ready" == true ]] || fail_rollback
|
|
}
|
|
|
|
stable_ids_before=$(stable_ids) || exit 1
|
|
candidate_compose config >/dev/null
|
|
candidate_compose build --build-arg "SOURCE_REVISION=$REVISION" "${services[@]}"
|
|
assert_stable_ids_unchanged
|
|
candidate_compose up -d --no-build --no-deps research-runtime core-runtime
|
|
wait_healthy "$(candidate_container research-runtime)"
|
|
wait_healthy "$(candidate_container core-runtime)"
|
|
candidate_compose up -d --no-build --no-deps cognitive-worker
|
|
wait_healthy "$(candidate_container cognitive-worker)"
|
|
candidate_compose up -d --no-build --no-deps creator-web
|
|
wait_healthy "$(candidate_container creator-web)"
|
|
assert_stable_ids_unchanged
|
|
for service in "${services[@]}"; do
|
|
image_revision=$(docker inspect -f '{{index .Config.Labels "org.opencontainers.image.revision"}}' "$(candidate_container "$service")" 2>/dev/null || true)
|
|
[[ "$image_revision" == "$REVISION" ]] || fail_rollback
|
|
done
|
|
assert_stable_ids_unchanged
|
|
if [[ "${PREPARE_ONLY:-false}" == true ]]; then
|
|
echo "isolated candidate ReleaseSet prepared slot=$SLOT project=$CANDIDATE_PROJECT revision=$REVISION"
|
|
exit 0
|
|
fi
|
|
|
|
release_json=$(cat <<JSON
|
|
{"id":"candidate-$SLOT-$REVISION","slot":"$SLOT","revision":"$REVISION","components":{"creator-web":{"name":"creator-web","revision":"$REVISION","upstream":"http://creator-web-candidate:8084","health_url":"http://creator-web-candidate:8084/healthz","ready_url":"http://creator-web-candidate:8084/healthz"},"core-runtime":{"name":"core-runtime","revision":"$REVISION","upstream":"http://core-runtime-candidate:8080","health_url":"http://core-runtime-candidate:8080/healthz","ready_url":"http://core-runtime-candidate:8080/readyz"},"cognitive-worker":{"name":"cognitive-worker","revision":"$REVISION","upstream":"http://cognitive-worker-candidate:8083","health_url":"http://cognitive-worker-candidate:8083/healthz","ready_url":"http://cognitive-worker-candidate:8083/readyz"},"research-runtime":{"name":"research-runtime","revision":"$REVISION","upstream":"http://research-runtime-candidate:8087","health_url":"http://research-runtime-candidate:8087/healthz","ready_url":"http://research-runtime-candidate:8087/readyz"}}}
|
|
JSON
|
|
)
|
|
stable_json=$(curl -fsS --max-time 10 "$SUPERVISOR_CONTROL_URL/v1/state")
|
|
stable_set=$(printf '%s' "$stable_json" | jq -c '.stable_release_set // empty')
|
|
if [[ -z "$stable_set" ]]; then
|
|
stable_revision=$(printf '%s' "$stable_json" | jq -r '.current.version // "unknown"')
|
|
stable_set=$(cat <<JSON
|
|
{"id":"stable-$stable_revision","slot":"legacy-stable","revision":"$stable_revision","components":{"creator-web":{"name":"creator-web","revision":"$stable_revision","upstream":"http://creator-web:8084","health_url":"http://creator-web:8084/healthz","ready_url":"http://creator-web:8084/healthz"},"core-runtime":{"name":"core-runtime","revision":"$stable_revision","upstream":"http://core-runtime:8080","health_url":"http://core-runtime:8080/healthz","ready_url":"http://core-runtime:8080/readyz"},"cognitive-worker":{"name":"cognitive-worker","revision":"$stable_revision","upstream":"http://cognitive-worker:8083","health_url":"http://cognitive-worker:8083/healthz","ready_url":"http://cognitive-worker:8083/readyz"},"research-runtime":{"name":"research-runtime","revision":"$stable_revision","upstream":"http://research-runtime:8087","health_url":"http://research-runtime:8087/healthz","ready_url":"http://research-runtime:8087/readyz"}}}
|
|
JSON
|
|
)
|
|
curl -fsS --max-time 10 -X POST "$SUPERVISOR_CONTROL_URL/v1/releases/baseline-set" -H 'Content-Type: application/json' --data "$stable_set" >/dev/null || fail_rollback
|
|
fi
|
|
curl -fsS --max-time 10 -X POST "$SUPERVISOR_CONTROL_URL/v1/releases/start-set" -H 'Content-Type: application/json' --data "$release_json" >/dev/null || fail_rollback
|
|
assert_stable_ids_unchanged
|
|
observation=$(printf '{"release_id":"candidate-%s-%s","ready":true,"reason":"isolated slot health and image revisions passed","observed_at_unix_millis":%s,"components":{"creator-web":{"revision":"%s","health":true,"ready":true},"core-runtime":{"revision":"%s","health":true,"ready":true},"cognitive-worker":{"revision":"%s","health":true,"ready":true},"research-runtime":{"revision":"%s","health":true,"ready":true}}}\n' "$SLOT" "$REVISION" "$(date +%s%3N)" "$REVISION" "$REVISION" "$REVISION" "$REVISION")
|
|
curl -fsS --max-time 10 -X POST "$SUPERVISOR_CONTROL_URL/v1/releases/observe-set" -H 'Content-Type: application/json' --data "$observation" >/dev/null || fail_rollback
|
|
assert_stable_ids_unchanged
|
|
control /v1/traffic/canary || fail_rollback
|
|
assert_stable_ids_unchanged
|
|
headers=$(mktemp)
|
|
trap 'rm -f "$headers"' EXIT
|
|
if ! curl -fsS --max-time 10 -D "$headers" -H 'X-Assistant-Canary: true' "https://assistant.cin.su/login" -o /dev/null; then fail_rollback; fi
|
|
grep -qi '^X-Assistant-Traffic-Target: candidate' "$headers" || fail_rollback
|
|
if [[ "$PROMOTE_AFTER_CANARY" != true ]]; then
|
|
echo "isolated candidate ReleaseSet canary ready slot=$SLOT project=$CANDIDATE_PROJECT revision=$REVISION"
|
|
exit 0
|
|
fi
|
|
control /v1/releases/promote || fail_rollback
|
|
assert_stable_ids_unchanged
|
|
echo "isolated candidate ReleaseSet promoted slot=$SLOT project=$CANDIDATE_PROJECT revision=$REVISION"
|