From d120cc09583c8f5f5e54e5abede23ab59544be7a Mon Sep 17 00:00:00 2001 From: Mikhail Date: Sun, 30 Aug 2026 13:51:32 +0300 Subject: [PATCH] fix: isolate candidate release slot --- deploy/docker-compose.candidate-slot.yml | 148 ++++++++++++++++++ deploy/docker-compose.prod.yml | 52 ------ deploy/production-stable-updater.sh | 55 +++++++ deploy/run-production-canary.sh | 83 ++++------ dev/ACTIVE_TASK.json | 2 +- ...EN2-DEPLOY-003-isolated-release-slots.json | 59 +++++++ runtime/supervisor/supervisor.go | 5 +- runtime/supervisor/supervisor_test.go | 4 +- .../stable-release-set-guard_test.sh | 17 +- 9 files changed, 316 insertions(+), 109 deletions(-) create mode 100644 deploy/docker-compose.candidate-slot.yml create mode 100644 deploy/production-stable-updater.sh create mode 100644 dev/tasks/GEN2-DEPLOY-003-isolated-release-slots.json diff --git a/deploy/docker-compose.candidate-slot.yml b/deploy/docker-compose.candidate-slot.yml new file mode 100644 index 0000000..7a15d91 --- /dev/null +++ b/deploy/docker-compose.candidate-slot.yml @@ -0,0 +1,148 @@ +# This file deliberately owns only one inactive release slot. It joins the +# stable project's shared network as an external resource, but has a distinct +# Compose project name supplied by run-production-canary.sh. Do not add +# durable services or stable services here. +services: + research-runtime: + build: + context: .. + dockerfile: deploy/images/runtime.Dockerfile + args: { TARGET: research-runtime, SOURCE_REVISION: "${ASSISTANT_SOURCE_REVISION:-unknown}" } + restart: "no" + environment: + LISTEN_ADDR: :8087 + RESEARCH_SOURCE_ROOT: /var/lib/research-sources + RESEARCH_HTTP_ALLOWED_HOSTS: assistant.cin.su,git.cin.su + RESEARCH_SEARCH_URL_TEMPLATE: ${RESEARCH_SEARCH_URL_TEMPLATE:-https://www.bing.com/search?q=%s} + RESEARCH_SEARCH_ALLOWED_HOSTS: ${RESEARCH_SEARCH_ALLOWED_HOSTS:-www.bing.com} + POSTGRES_HOST: postgres + POSTGRES_PORT: "5432" + POSTGRES_DATABASE: ${POSTGRES_DB:-assistant} + EPISTEMICS_POSTGRES_USER: epistemics_role + EPISTEMICS_POSTGRES_CREDENTIAL_REF: credential://domovoy/epistemics/db-password/v1 + SECRET_BACKEND_ROOT: /run/secrets + volumes: + - assistant_core_secrets:/run/secrets:ro + - assistant_research_sources:/var/lib/research-sources:ro + networks: + shared: + aliases: [research-runtime-candidate] + healthcheck: + test: ["CMD-SHELL", "wget -qO- http://localhost:8087/healthz >/dev/null && wget -qO- http://localhost:8087/readyz >/dev/null"] + interval: 10s + timeout: 10s + retries: 20 + + core-runtime: + build: + context: .. + dockerfile: deploy/images/runtime.Dockerfile + args: { TARGET: core-runtime, SOURCE_REVISION: "${ASSISTANT_SOURCE_REVISION:-unknown}" } + restart: "no" + environment: + LISTEN_ADDR: ":8080" + POSTGRES_HOST: postgres + POSTGRES_PORT: "5432" + POSTGRES_DATABASE: ${POSTGRES_DB:-assistant} + POSTGRES_USER: situation_role + POSTGRES_PASSWORD: ${SITUATION_DB_PASSWORD} + IDENTITY_POSTGRES_USER: identity_role + IDENTITY_POSTGRES_PASSWORD: ${IDENTITY_DB_PASSWORD} + CREATOR_USERNAME: ${CREATOR_USERNAME} + WORK_POSTGRES_USER: work_role + WORK_POSTGRES_PASSWORD: ${WORK_DB_PASSWORD} + CAPABILITY_POSTGRES_USER: capability_role + CAPABILITY_POSTGRES_PASSWORD: ${CAPABILITY_DB_PASSWORD} + EPISTEMICS_POSTGRES_USER: epistemics_role + EPISTEMICS_POSTGRES_CREDENTIAL_REF: credential://domovoy/epistemics/db-password/v1 + INTERACTION_POSTGRES_USER: interaction_role + INTERACTION_POSTGRES_CREDENTIAL_REF: credential://domovoy/interaction/db-password/v1 + SECRET_BACKEND_ROOT: /run/secrets + COGNITIVE_ADMISSION_POLICY_ID: creator-advisory-local-v1 + COGNITIVE_ADMISSION_MAX_CONCURRENT: ${COGNITIVE_ADMISSION_MAX_CONCURRENT:-1} + COGNITIVE_ADMISSION_MAX_ACCEPTED_PER_HOUR: ${COGNITIVE_ADMISSION_MAX_ACCEPTED_PER_HOUR:-60} + COGNITIVE_ADMISSION_MAX_INPUT_BYTES: ${COGNITIVE_ADMISSION_MAX_INPUT_BYTES:-81920} + COGNITIVE_ADMISSION_MAX_OUTPUT_BYTES: ${COGNITIVE_ADMISSION_MAX_OUTPUT_BYTES:-16384} + RESEARCH_RUNTIME_URL: http://research-runtime-candidate:8087 + volumes: + - assistant_core_secrets:/run/secrets:ro + networks: + shared: + aliases: [core-runtime-candidate] + healthcheck: + test: ["CMD-SHELL", "wget -qO- http://localhost:8080/healthz >/dev/null && wget -qO- http://localhost:8080/readyz >/dev/null"] + interval: 10s + timeout: 20s + retries: 20 + + cognitive-worker: + build: + context: .. + dockerfile: deploy/images/runtime.Dockerfile + args: { TARGET: cognitive-worker, SOURCE_REVISION: "${ASSISTANT_SOURCE_REVISION:-unknown}" } + restart: "no" + environment: + LISTEN_ADDR: ":8083" + CORE_RUNTIME_URL: http://core-runtime-candidate:8080 + LOCAL_CPU_PROVIDER_URL: http://docker.cin.su:18082/v1 + LOCAL_CPU_PROVIDER_API_KEY_FILE: ${LOCAL_CPU_PROVIDER_API_KEY_FILE:-} + LOCAL_CPU_PROVIDER_CREDENTIAL_REF: ${LOCAL_CPU_PROVIDER_CREDENTIAL_REF:-credential://domovoy/local-cpu/provider-api-key/v1} + LOCAL_MEDIUM_PROVIDER_URL: ${LOCAL_MEDIUM_PROVIDER_URL:-http://llm-local-gateway:8080/v1} + LOCAL_MEDIUM_PROVIDER_API_KEY_FILE: ${LOCAL_MEDIUM_PROVIDER_API_KEY_FILE:-/run/secrets/llm_local_gateway_api_key} + LOCAL_MEDIUM_PROVIDER_CREDENTIAL_REF: ${LOCAL_MEDIUM_PROVIDER_CREDENTIAL_REF:-credential://domovoy/local-medium/provider-api-key/v1} + LOCAL_GPU_PROVIDER_TIMEOUT: ${LOCAL_GPU_PROVIDER_TIMEOUT:-60s} + LOCAL_LIFEBOAT_PROVIDER_URL: http://cognitive-lifeboat:8086/v1 + LOCAL_IMAGE_PROVIDER_KIND: ${LOCAL_IMAGE_PROVIDER_KIND:-comfyui} + LOCAL_IMAGE_PROVIDER_URL: ${LOCAL_IMAGE_PROVIDER_URL:-http://comfyui:8188} + LOCAL_IMAGE_MODEL: ${LOCAL_IMAGE_MODEL:-z_image_turbo_bf16.safetensors} + COGNITIVE_OWNER_PRINCIPAL_ID: domovoy-creator-principal + COGNITIVE_ALLOW_CANARY: "true" + volumes: + - assistant_core_secrets:/run/secrets:ro + networks: + shared: + aliases: [cognitive-worker-candidate] + llm-local: {} + healthcheck: + test: ["CMD-SHELL", "wget -qO- http://localhost:8083/healthz >/dev/null && wget -qO- http://localhost:8083/readyz >/dev/null"] + interval: 10s + timeout: 20s + retries: 20 + + creator-web: + build: + context: .. + dockerfile: deploy/images/runtime.Dockerfile + args: { TARGET: creator-web, SOURCE_REVISION: "${ASSISTANT_SOURCE_REVISION:-unknown}" } + restart: "no" + environment: + LISTEN_ADDR: ":8084" + CORE_RUNTIME_URL: http://core-runtime-candidate:8080 + CONTINUITY_RUNTIME_URL: http://continuity-runtime:8081 + COMMIT_RUNTIME_URL: http://commit-runtime:8082 + COGNITIVE_RUNTIME_URL: http://cognitive-worker-candidate:8083 + AGENT_WORKER_URL: http://agent-worker:8090 + ASSISTANT_BUILD_REVISION: ${ASSISTANT_SOURCE_REVISION:-unknown} + networks: + shared: + aliases: [creator-web-candidate] + healthcheck: + test: ["CMD-SHELL", "wget -qO- http://localhost:8084/healthz >/dev/null"] + interval: 10s + timeout: 5s + retries: 20 + +volumes: + assistant_core_secrets: + external: true + assistant_research_sources: + name: assistant_research_sources + external: true + +networks: + shared: + name: assistant_default + external: true + llm-local: + name: llm-local_default + external: true diff --git a/deploy/docker-compose.prod.yml b/deploy/docker-compose.prod.yml index 3af961b..19f6029 100644 --- a/deploy/docker-compose.prod.yml +++ b/deploy/docker-compose.prod.yml @@ -89,17 +89,6 @@ services: timeout: 20s retries: 20 - core-runtime-candidate: - profiles: ["assistant-canary"] - extends: - service: core-runtime - restart: "no" - environment: - RESEARCH_RUNTIME_URL: http://research-runtime-candidate:8087 - depends_on: - postgres: { condition: service_healthy } - research-runtime-candidate: { condition: service_started } - agent-canary: profiles: ["agent-canary"] build: { context: .., dockerfile: deploy/images/runtime.Dockerfile, args: { TARGET: agent-canary, SOURCE_REVISION: "${ASSISTANT_SOURCE_REVISION:-unknown}" } } @@ -223,14 +212,6 @@ services: timeout: 10s retries: 20 - research-runtime-candidate: - profiles: ["assistant-canary"] - extends: - service: research-runtime - restart: "no" - depends_on: - postgres: { condition: service_healthy } - continuity-runtime: build: { context: .., dockerfile: deploy/images/runtime.Dockerfile, args: { TARGET: continuity-runtime, SOURCE_REVISION: "${ASSISTANT_SOURCE_REVISION:-unknown}" } } restart: unless-stopped @@ -299,17 +280,6 @@ services: timeout: 20s retries: 20 - cognitive-worker-candidate: - profiles: ["assistant-canary"] - extends: - service: cognitive-worker - restart: "no" - environment: - CORE_RUNTIME_URL: http://core-runtime-candidate:8080 - depends_on: - core-runtime-candidate: { condition: service_started } - cognitive-lifeboat: { condition: service_healthy } - cognitive-lifeboat: image: ghcr.io/ggml-org/llama.cpp:server restart: unless-stopped @@ -386,28 +356,6 @@ services: timeout: 3s retries: 10 - creator-web-candidate: - profiles: ["assistant-canary"] - extends: - service: creator-web - restart: "no" - environment: - CORE_RUNTIME_URL: http://core-runtime-candidate:8080 - CONTINUITY_RUNTIME_URL: http://continuity-runtime:8081 - COMMIT_RUNTIME_URL: http://commit-runtime:8082 - COGNITIVE_RUNTIME_URL: http://cognitive-worker-candidate:8083 - AGENT_WORKER_URL: http://agent-worker:8090 - depends_on: - core-runtime-candidate: { condition: service_started } - continuity-runtime: { condition: service_started } - commit-runtime: { condition: service_started } - cognitive-worker-candidate: { condition: service_healthy } - healthcheck: - test: ["CMD-SHELL", "wget -qO- http://localhost:8084/healthz >/dev/null"] - interval: 10s - timeout: 5s - retries: 20 - volumes: assistant_pg: assistant_nats: diff --git a/deploy/production-stable-updater.sh b/deploy/production-stable-updater.sh new file mode 100644 index 0000000..8411399 --- /dev/null +++ b/deploy/production-stable-updater.sh @@ -0,0 +1,55 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Stable-only production updater. Candidate slots are managed exclusively by +# run-production-canary.sh and Supervisor; this controller never starts or +# reconciles a candidate service. +BASE_DIR="${BASE_DIR:-/home/m/assistant-deploy}" +REPO_DIR="${REPO_DIR:-$BASE_DIR/repo}" +ENV_FILE="${ENV_FILE:-$BASE_DIR/production-compose.env}" +COMPOSE_FILE="$REPO_DIR/deploy/docker-compose.prod.yml" +LOG_FILE="${LOG_FILE:-$BASE_DIR/deploy.log}" +LOCK_FILE="${LOCK_FILE:-/tmp/assistant-stable-deploy.lock}" +stable_services=(postgres core-runtime continuity-runtime commit-runtime cognitive-worker creator-web notification-receiver research-runtime agent-worker work-graph-runtime) + +exec 9>"$LOCK_FILE" +flock -n 9 || exit 0 +log() { printf '%s %s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$*" >>"$LOG_FILE"; } +fail() { log "FAILED: $*"; exit 1; } +compose() { docker compose --env-file "$ENV_FILE" -f "$COMPOSE_FILE" "$@"; } + +cd "$REPO_DIR" +git fetch --prune origin main >>"$LOG_FILE" 2>&1 +git merge --ff-only origin/main >>"$LOG_FILE" 2>&1 || fail "stable checkout is not a fast-forward to origin/main" +revision="$(git rev-parse HEAD)" +[[ "$revision" =~ ^[0-9a-f]{40}$ ]] || fail "invalid source revision" + +if ! REPO_DIR="$REPO_DIR" ENV_FILE="$ENV_FILE" bash "$REPO_DIR/deploy/apply-migrations.sh" >>"$LOG_FILE" 2>&1; then + fail "migration application failed" +fi +compose config >/dev/null 2>>"$LOG_FILE" || fail "stable compose config validation failed" + +all_current=1 +for service in "${stable_services[@]}"; do + container="assistant-${service}-1" + label="$(docker inspect "$container" --format '{{index .Config.Labels "org.opencontainers.image.revision"}}' 2>/dev/null || true)" + [[ "$label" == "$revision" ]] || all_current=0 +done +if [[ "$all_current" == 1 ]]; then log "SKIP stable revision=$revision"; exit 0; fi + +log "BUILD stable revision=$revision" +ASSISTANT_SOURCE_REVISION="$revision" compose build --build-arg "SOURCE_REVISION=$revision" "${stable_services[@]}" >>"$LOG_FILE" 2>&1 +log "UP stable revision=$revision" +ASSISTANT_SOURCE_REVISION="$revision" compose up -d --no-build "${stable_services[@]}" >>"$LOG_FILE" 2>&1 + +for attempt in $(seq 1 30); do + if curl -fsS --max-time 5 https://assistant.cin.su/healthz >/dev/null 2>&1; then break; fi + [[ "$attempt" == 30 ]] && fail "public health failed" + sleep 10 +done +for service in "${stable_services[@]}"; do + container="assistant-${service}-1" + label="$(docker inspect "$container" --format '{{index .Config.Labels "org.opencontainers.image.revision"}}' 2>/dev/null || true)" + [[ "$label" == "$revision" ]] || fail "artifact drift container=$container" +done +log "OK stable revision=$revision" diff --git a/deploy/run-production-canary.sh b/deploy/run-production-canary.sh index 0c89b0f..7b39b4b 100644 --- a/deploy/run-production-canary.sh +++ b/deploy/run-production-canary.sh @@ -1,57 +1,53 @@ #!/usr/bin/env bash set -euo pipefail +# Candidate services are never part of the stable Compose project. The slot +# Compose file may use only declared external networks and durable resources. BASE_DIR="${REPO_DIR:-$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)}" ENV_FILE="${ENV_FILE:-/home/m/assistant-deploy/production-compose.env}" -COMPOSE_FILE="$BASE_DIR/deploy/docker-compose.prod.yml" +CANDIDATE_FILE="$BASE_DIR/deploy/docker-compose.candidate-slot.yml" SUPERVISOR_CONTROL_URL="${SUPERVISOR_CONTROL_URL:-http://127.0.0.1:${SUPERVISOR_CONTROL_PORT:-18085}}" REVISION="${1:?candidate revision is required}" +SLOT="${CANDIDATE_SLOT:-b}" +STABLE_PROJECT="${STABLE_PROJECT:-assistant}" +CANDIDATE_PROJECT="${CANDIDATE_PROJECT:-assistant-slot-$SLOT}" HEALTH_ATTEMPTS="${HEALTH_ATTEMPTS:-360}" -set -a -. "$ENV_FILE" -set +a +[[ "$SLOT" =~ ^[a-z0-9-]+$ ]] || { echo "invalid candidate slot" >&2; exit 2; } +[[ "$CANDIDATE_PROJECT" != "$STABLE_PROJECT" ]] || { echo "candidate and stable Compose projects must differ" >&2; exit 2; } -compose() { docker compose --env-file "$ENV_FILE" -f "$COMPOSE_FILE" "$@"; } +candidate_compose() { docker compose -p "$CANDIDATE_PROJECT" --env-file "$ENV_FILE" -f "$CANDIDATE_FILE" "$@"; } control() { curl -fsS --max-time 10 -X POST "$SUPERVISOR_CONTROL_URL$1" "${@:2}" >/dev/null; } fail_rollback() { control /v1/traffic/rollback || true control /v1/releases/rollback || true - echo "production ReleaseSet canary failed; stable ReleaseSet rollback requested" >&2 + echo "isolated candidate ReleaseSet failed; stable ReleaseSet rollback requested" >&2 exit 1 } -services=(creator-web-candidate core-runtime-candidate cognitive-worker-candidate research-runtime-candidate) -containers=(assistant-creator-web-candidate-1 assistant-core-runtime-candidate-1 assistant-cognitive-worker-candidate-1 assistant-research-runtime-candidate-1) +services=(creator-web core-runtime cognitive-worker research-runtime) stable_services=(creator-web core-runtime cognitive-worker research-runtime) - stable_ids() { local service container id for service in "${stable_services[@]}"; do - container="assistant-${service}-1" + container="${STABLE_PROJECT}-${service}-1" id=$(docker inspect -f '{{.Id}}' "$container" 2>/dev/null || true) [[ -n "$id" ]] || { echo "stable container missing: $container" >&2; return 1; } printf '%s=%s\n' "$service" "$id" done } - assert_stable_ids_unchanged() { local after after=$(stable_ids) || fail_rollback [[ "$after" == "$stable_ids_before" ]] || { - echo "stable container IDs changed during candidate operation" >&2 + echo "stable container IDs changed during isolated candidate operation" >&2 diff -u <(printf '%s\n' "$stable_ids_before") <(printf '%s\n' "$after") >&2 || true fail_rollback } } - -stable_ids_before=$(stable_ids) || exit 1 - -compose --profile assistant-canary build --build-arg "SOURCE_REVISION=$REVISION" "${services[@]}" - +candidate_container() { printf '%s-%s-1' "$CANDIDATE_PROJECT" "$1"; } wait_healthy() { local container="$1" status="" ready="" - ready="" for attempt in $(seq 1 "$HEALTH_ATTEMPTS"); do status=$(docker inspect -f '{{if .State.Health}}{{.State.Health.Status}}{{else}}unknown{{end}}' "$container" 2>/dev/null || true) if [[ "$status" == healthy ]]; then ready=true; break; fi @@ -60,68 +56,53 @@ wait_healthy() { [[ "$ready" == true ]] || fail_rollback } -compose --profile assistant-canary up -d --no-build --no-deps research-runtime-candidate core-runtime-candidate -wait_healthy assistant-research-runtime-candidate-1 -wait_healthy assistant-core-runtime-candidate-1 -compose --profile assistant-canary up -d --no-build --no-deps cognitive-worker-candidate -wait_healthy assistant-cognitive-worker-candidate-1 -compose --profile assistant-canary up -d --no-build --no-deps creator-web-candidate -wait_healthy assistant-creator-web-candidate-1 +stable_ids_before=$(stable_ids) || exit 1 +candidate_compose config >/dev/null +candidate_compose build --build-arg "SOURCE_REVISION=$REVISION" "${services[@]}" assert_stable_ids_unchanged - -for container in "${containers[@]}"; do - image_revision=$(docker inspect -f '{{index .Config.Labels "org.opencontainers.image.revision"}}' "$container" 2>/dev/null || true) +candidate_compose up -d --no-build --no-deps research-runtime core-runtime +wait_healthy "$(candidate_container research-runtime)" +wait_healthy "$(candidate_container core-runtime)" +candidate_compose up -d --no-build --no-deps cognitive-worker +wait_healthy "$(candidate_container cognitive-worker)" +candidate_compose up -d --no-build --no-deps creator-web +wait_healthy "$(candidate_container creator-web)" +assert_stable_ids_unchanged +for service in "${services[@]}"; do + image_revision=$(docker inspect -f '{{index .Config.Labels "org.opencontainers.image.revision"}}' "$(candidate_container "$service")" 2>/dev/null || true) [[ "$image_revision" == "$REVISION" ]] || fail_rollback done assert_stable_ids_unchanged - if [[ "${PREPARE_ONLY:-false}" == true ]]; then - echo "production ReleaseSet candidate prepared revision=$REVISION" + echo "isolated candidate ReleaseSet prepared slot=$SLOT project=$CANDIDATE_PROJECT revision=$REVISION" exit 0 fi release_json=$(cat </dev/null || fail_rollback fi - curl -fsS --max-time 10 -X POST "$SUPERVISOR_CONTROL_URL/v1/releases/start-set" -H 'Content-Type: application/json' --data "$release_json" >/dev/null || fail_rollback assert_stable_ids_unchanged -observation=$(printf '{"release_id":"candidate-%s","ready":true,"reason":"all ReleaseSet healthchecks and image revisions passed","observed_at_unix_millis":%s,"components":{"creator-web":{"revision":"%s","health":true,"ready":true},"core-runtime":{"revision":"%s","health":true,"ready":true},"cognitive-worker":{"revision":"%s","health":true,"ready":true},"research-runtime":{"revision":"%s","health":true,"ready":true}}}\n' "$REVISION" "$(date +%s%3N)" "$REVISION" "$REVISION" "$REVISION" "$REVISION") +observation=$(printf '{"release_id":"candidate-%s-%s","ready":true,"reason":"isolated slot health and image revisions passed","observed_at_unix_millis":%s,"components":{"creator-web":{"revision":"%s","health":true,"ready":true},"core-runtime":{"revision":"%s","health":true,"ready":true},"cognitive-worker":{"revision":"%s","health":true,"ready":true},"research-runtime":{"revision":"%s","health":true,"ready":true}}}\n' "$SLOT" "$REVISION" "$(date +%s%3N)" "$REVISION" "$REVISION" "$REVISION" "$REVISION") curl -fsS --max-time 10 -X POST "$SUPERVISOR_CONTROL_URL/v1/releases/observe-set" -H 'Content-Type: application/json' --data "$observation" >/dev/null || fail_rollback assert_stable_ids_unchanged control /v1/traffic/canary || fail_rollback assert_stable_ids_unchanged - headers=$(mktemp) trap 'rm -f "$headers"' EXIT if ! curl -fsS --max-time 10 -D "$headers" -H 'X-Assistant-Canary: true' "https://assistant.cin.su/login" -o /dev/null; then fail_rollback; fi grep -qi '^X-Assistant-Traffic-Target: candidate' "$headers" || fail_rollback - control /v1/releases/promote || fail_rollback assert_stable_ids_unchanged -echo "production ReleaseSet canary promoted revision=$REVISION" +echo "isolated candidate ReleaseSet promoted slot=$SLOT project=$CANDIDATE_PROJECT revision=$REVISION" diff --git a/dev/ACTIVE_TASK.json b/dev/ACTIVE_TASK.json index e6ad59c..386c9a6 100644 --- a/dev/ACTIVE_TASK.json +++ b/dev/ACTIVE_TASK.json @@ -1 +1 @@ -{"task_file":"dev/tasks/GEN2-COMPAT-OBS-001-production-compatibility-observation.json","task_id":"GEN2-COMPAT-OBS-001","task_revision":2} +{"task_file":"dev/tasks/GEN2-DEPLOY-003-isolated-release-slots.json","task_id":"GEN2-DEPLOY-003","task_revision":1} diff --git a/dev/tasks/GEN2-DEPLOY-003-isolated-release-slots.json b/dev/tasks/GEN2-DEPLOY-003-isolated-release-slots.json new file mode 100644 index 0000000..ade27e1 --- /dev/null +++ b/dev/tasks/GEN2-DEPLOY-003-isolated-release-slots.json @@ -0,0 +1,59 @@ +{ + "task_id": "GEN2-DEPLOY-003", + "task_revision": 1, + "architecture_revision": "ARCH-GEN2-000", + "base_repo_revision": "gen2-integration", + "risk_tier": "E", + "intent": "Изолировать stable и candidate ReleaseSet в независимые Compose deployment projects/slots, чтобы подготовка candidate не могла пересоздать stable containers, сохранив Supervisor как единственный traffic/promotion/rollback control plane.", + "requirement_ids": ["REQ-GEN2-DEPLOY-003-SLOT-ISOLATION", "REQ-GEN2-DEPLOY-003-UPDATER-OWNERSHIP", "REQ-GEN2-DEPLOY-003-ROLLBACK"], + "allowed_change_scope": { + "modules": ["deployment control", "runtime supervisor"], + "paths": [ + "deploy/**", + "runtime/supervisor/**", + "cmd/supervisor/**", + "docs/evidence/**", + "docs/DEPLOYMENT.md", + "tests/deployment/**", + "dev/ACTIVE_TASK.json", + "dev/tasks/GEN2-DEPLOY-003-isolated-release-slots.json" + ] + }, + "forbidden_change_scope": { + "paths": ["architecture/**", "database/**", "modules/**", "contracts/**", "security/**", "adapters/**", "go.mod", "go.sum", ".gitea/workflows/**", "GEN2-028", "GEN2-029", "GEN2-030"] + }, + "allowed_architecture_delta": {"new_cross_module_dependencies": 0, "new_canonical_owners": 0, "new_privileges": 1, "new_third_party_dependencies": 0, "new_database_owner_roles": 0}, + "contracts": { + "consume": ["existing Supervisor durable ReleaseSet state", "existing production compose stack", "existing shared durable PostgreSQL/volumes"], + "may_change": ["deployment slot topology", "deployment-controller scripts", "Supervisor operational ReleaseSet identity fields"] + }, + "semantic_owner": "Operations/Supervisor; slot and ReleaseSet identity are runtime-only operational control-plane state, not semantic canonical state.", + "canonical_state": ["no semantic owner, canonical writer, schema or data migration change", "Supervisor durable release evidence/LKG remains retained outside candidate containers"], + "state_transitions": ["stable slot retained -> isolated candidate slot start -> four-component health/revision validation -> Supervisor canary -> promote or traffic rollback -> inactive slot retained for rollback"], + "side_effects_and_privileges": ["production operations only through Docker/Compose and Supervisor control plane explicitly authorized for this remediation", "candidate preparation must never directly replace/recreate stable", "no credential reads or outputs", "external updater pause is allowed only as an explicit bounded operation with restore evidence"], + "persistence_and_migrations": ["no database migration", "shared PostgreSQL, named volumes and durable services are attached as existing external resources and are never duplicated or deleted by candidate cleanup"], + "failure_retry_restart": ["capture stable container IDs before every candidate operation and abort before canary when any changes", "candidate component failure returns traffic to LKG without container recreation", "Supervisor restart reloads ReleaseSet/slot/LKG evidence", "updater ownership must be project-scoped so it cannot reconcile the other slot"], + "assumptions": ["current stable may remain in its existing project during the one-time transition; it must not be recreated merely to rename a slot", "external deployment controller can be changed only when its exact owner/process is evidenced", "GEN2-COMPAT-OBS-001, V1 retirement and Gen3 remain stopped until this task passes"], + "acceptance_scenarios": [ + "DEPLOY-003-SCN-01: stable and candidate Compose projects, service/container identities and updater ownership are distinct", + "DEPLOY-003-SCN-02: candidate dependencies resolve only to candidate services plus declared shared external resources", + "DEPLOY-003-SCN-03: candidate operation leaves every captured stable container ID unchanged; deviation aborts before canary", + "DEPLOY-003-SCN-04: all four candidate components report health/readiness and one revision", + "DEPLOY-003-SCN-05: Supervisor records ReleaseSet slot identity/revision/LKG and canaries the candidate", + "DEPLOY-003-SCN-06: forced candidate component failure rolls all traffic back to previous stable without stable recreate", + "DEPLOY-003-SCN-07: Supervisor restart preserves LKG/rollback target and candidate recovery does not alter stable IDs", + "DEPLOY-003-EXIT: isolated production ReleaseSet drill passes, after which GEN2-COMPAT-OBS-001 may resume without V1 retirement" + ], + "required_tests": [ + "FAST: compose config for stable and candidate slots", + "FAST: targeted go test ./runtime/supervisor ./cmd/supervisor", + "FAST: targeted go vet ./runtime/supervisor ./cmd/supervisor", + "FAST: deployment shell regression for project/identity isolation and stable ID invariance", + "FAST governance: go run ./cmd/devctl preflight -active dev/ACTIVE_TASK.json", + "FAST scope: go run ./cmd/devctl check-diff -active dev/ACTIVE_TASK.json", + "FAST independent: go run ./cmd/devctl/independent", + "FAST whitespace: git diff --check gen2-integration --", + "PRODUCTION: isolated candidate ReleaseSet health, canary, forced rollback, Supervisor restart/LKG and stable-ID drill" + ], + "terminal_gap_states": ["BLOCKED_BY_DEPLOYMENT_CONTROL", "BLOCKED_BY_SPEC_GAP", "BLOCKED_BY_ARCHITECTURE_GAP", "BLOCKED_BY_MISSING_CAPABILITY", "BLOCKED_BY_CONFLICT", "REQUIRES_MIGRATION_DECISION"] +} diff --git a/runtime/supervisor/supervisor.go b/runtime/supervisor/supervisor.go index 2b915fa..ae93a77 100644 --- a/runtime/supervisor/supervisor.go +++ b/runtime/supervisor/supervisor.go @@ -46,7 +46,10 @@ type ComponentRelease struct { // ReleaseSet is the atomic deployment unit supervised by the control plane. type ReleaseSet struct { - ID string `json:"id"` + ID string `json:"id"` + // Slot identifies the independently managed deployment slot. An empty value + // remains readable for release state recorded before slot isolation. + Slot string `json:"slot,omitempty"` Revision string `json:"revision"` Components map[string]ComponentRelease `json:"components"` } diff --git a/runtime/supervisor/supervisor_test.go b/runtime/supervisor/supervisor_test.go index eb2b77e..5e6683b 100644 --- a/runtime/supervisor/supervisor_test.go +++ b/runtime/supervisor/supervisor_test.go @@ -110,7 +110,9 @@ func TestPersistentReleaseSetRollbackPreservesWholeLastKnownGood(t *testing.T) { } first := supervisor.NewPersistentLifeboat(store) good := testReleaseSet("stable-r1", "r1", "http://web", "http://core", "http://cognitive", "http://research") + good.Slot = "a" candidate := testReleaseSet("candidate-r2", "r2", "http://web-c", "http://core-c", "http://cognitive-c", "http://research-c") + candidate.Slot = "b" if ok, err := first.EstablishBaselineSet(good); err != nil || !ok { t.Fatalf("baseline: %v %v", ok, err) } @@ -135,7 +137,7 @@ func TestPersistentReleaseSetRollbackPreservesWholeLastKnownGood(t *testing.T) { if err != nil { t.Fatal(err) } - if state.StableReleaseSet == nil || state.StableReleaseSet.Revision != "r1" || state.TrafficMode != supervisor.TrafficStable { + if state.StableReleaseSet == nil || state.StableReleaseSet.Revision != "r1" || state.StableReleaseSet.Slot != "a" || state.TrafficMode != supervisor.TrafficStable { t.Fatalf("state=%+v", state) } } diff --git a/tests/deployment/stable-release-set-guard_test.sh b/tests/deployment/stable-release-set-guard_test.sh index 9916590..36b0a50 100644 --- a/tests/deployment/stable-release-set-guard_test.sh +++ b/tests/deployment/stable-release-set-guard_test.sh @@ -4,10 +4,16 @@ set -euo pipefail root="$(CDPATH= cd -- "$(dirname -- "$0")/../.." && pwd)" canary="$root/deploy/run-production-canary.sh" supervisor="$root/deploy/update-supervisor.sh" +candidate_compose="$root/deploy/docker-compose.candidate-slot.yml" +stable_compose="$root/deploy/docker-compose.prod.yml" +stable_updater="$root/deploy/production-stable-updater.sh" -grep -Fq 'up -d --no-build --no-deps research-runtime-candidate core-runtime-candidate' "$canary" -grep -Fq 'up -d --no-build --no-deps cognitive-worker-candidate' "$canary" -grep -Fq 'up -d --no-build --no-deps creator-web-candidate' "$canary" +grep -Fq 'CANDIDATE_PROJECT="${CANDIDATE_PROJECT:-assistant-slot-$SLOT}"' "$canary" +grep -Fq '[[ "$CANDIDATE_PROJECT" != "$STABLE_PROJECT" ]]' "$canary" +grep -Fq 'docker compose -p "$CANDIDATE_PROJECT"' "$canary" +grep -Fq 'up -d --no-build --no-deps research-runtime core-runtime' "$canary" +grep -Fq 'up -d --no-build --no-deps cognitive-worker' "$canary" +grep -Fq 'up -d --no-build --no-deps creator-web' "$canary" grep -Fq 'compose up -d --no-build --no-deps supervisor' "$supervisor" grep -Fq 'stable_ids_before=$(stable_ids)' "$canary" grep -Fq 'assert_stable_ids_unchanged' "$canary" @@ -16,4 +22,9 @@ grep -Fq 'after=$(stable_ids)' "$supervisor" grep -Fq 'stable container IDs changed' "$supervisor" grep -Fq 'HEALTH_ATTEMPTS="${HEALTH_ATTEMPTS:-360}"' "$canary" grep -Fq 'seq 1 "$HEALTH_ATTEMPTS"' "$canary" +grep -Fq 'name: assistant_default' "$candidate_compose" +grep -Fq 'external: true' "$candidate_compose" +! grep -Fq 'candidate:' "$stable_compose" +! grep -Fq 'run-production-canary.sh' "$stable_updater" +grep -Fq 'compose up -d --no-build "${stable_services[@]}"' "$stable_updater" echo 'stable release-set guard: PASS'