feat(deploy): close candidate release set topology
CI / test (push) Failing after 34s

This commit is contained in:
m
2026-08-31 16:35:08 +03:00
parent e21eae7a99
commit 42285a55be
7 changed files with 234 additions and 28 deletions
+152 -3
View File
@@ -69,6 +69,20 @@ services:
COGNITIVE_ADMISSION_MAX_OUTPUT_BYTES: ${COGNITIVE_ADMISSION_MAX_OUTPUT_BYTES:-16384}
COGNITIVE_RUNTIME_URL: http://cognitive-worker-candidate:8083
RESEARCH_RUNTIME_URL: http://research-runtime-candidate:8087
CONTINUITY_RUNTIME_URL: http://continuity-runtime-candidate:8081
COMMIT_RUNTIME_URL: http://commit-runtime-candidate:8082
CANDIDATE_RELEASE_SET_ID: assistant-slot-b
CANDIDATE_SLOT: b
CANDIDATE_TASK_ID: GEN2-CAPABILITY-001
CANDIDATE_TASK_REVISION: "7"
CANDIDATE_PRINCIPAL_ID: domovoy-creator-principal
CANDIDATE_CAPABILITY_ID: prospective-notification-delivery
CANDIDATE_RESOURCE_SCOPE: synthetic-shadow-notification-v1
CANDIDATE_BINDING_MODE: SHADOW
CANDIDATE_TARGET_KIND: NOTIFICATION_SHADOW_RECEIVER
CANDIDATE_TARGET_REF: notification-receiver-candidate
CANDIDATE_TARGET_DIGEST: f9f9fa49a0727444c280c092d5bb340f3a8f33cf325c5396f5ae177a6666da35
CANDIDATE_AUTHORITY_GRANT_ID: gen2-capability-001-shadow-notification-grant
volumes:
- assistant_core_secrets:/run/secrets:ro
- assistant_control_plane_service_secrets:/run/control-plane-secrets:ro
@@ -124,10 +138,10 @@ services:
environment:
LISTEN_ADDR: ":8084"
CORE_RUNTIME_URL: http://core-runtime-candidate:8080
CONTINUITY_RUNTIME_URL: http://continuity-runtime:8081
COMMIT_RUNTIME_URL: http://commit-runtime:8082
CONTINUITY_RUNTIME_URL: http://continuity-runtime-candidate:8081
COMMIT_RUNTIME_URL: http://commit-runtime-candidate:8082
COGNITIVE_RUNTIME_URL: http://cognitive-worker-candidate:8083
AGENT_WORKER_URL: http://agent-worker:8090
AGENT_WORKER_URL: http://agent-worker-candidate:8090
ASSISTANT_BUILD_REVISION: ${ASSISTANT_SOURCE_REVISION:-unknown}
networks:
shared:
@@ -138,6 +152,133 @@ services:
timeout: 5s
retries: 20
notification-receiver:
build: { context: .., dockerfile: deploy/images/runtime.Dockerfile, args: { TARGET: notification-receiver, SOURCE_REVISION: "${ASSISTANT_SOURCE_REVISION:-unknown}" } }
restart: "no"
environment:
LISTEN_ADDR: ":8085"
SHADOW_RECEIVER_MODE: "1"
CANDIDATE_RELEASE_SET_ID: assistant-slot-b
CANDIDATE_TASK_ID: GEN2-CAPABILITY-001
RECEIPT_DIRECTORY: /var/lib/notification-shadow-receiver
RECEIVER_WRITE_TOKEN_FILE: /run/notification-shadow-writer/token
RECEIVER_READ_TOKEN_FILE: /run/notification-shadow-reader/token
volumes:
- assistant_candidate_notification_shadow_receipts:/var/lib/notification-shadow-receiver
- assistant_candidate_notification_shadow_writer_secret:/run/notification-shadow-writer:ro
- assistant_candidate_notification_shadow_reader_secret:/run/notification-shadow-reader:ro
networks: { shared: { aliases: [notification-receiver-candidate] } }
healthcheck: { test: ["CMD-SHELL", "wget -qO- http://localhost:8085/healthz >/dev/null"], interval: 10s, timeout: 3s, retries: 10 }
continuity-runtime:
build: { context: .., dockerfile: deploy/images/runtime.Dockerfile, args: { TARGET: continuity-runtime, SOURCE_REVISION: "${ASSISTANT_SOURCE_REVISION:-unknown}" } }
restart: "no"
environment:
LISTEN_ADDR: ":8081"
POSTGRES_HOST: postgres
POSTGRES_PORT: "5432"
POSTGRES_DATABASE: ${POSTGRES_DB:-assistant}
POSTGRES_USER: continuity_role
POSTGRES_PASSWORD: ${CONTINUITY_DB_PASSWORD}
CORE_RUNTIME_URL: http://core-runtime-candidate:8080
COMMIT_RUNTIME_URL: http://commit-runtime-candidate:8082
COMMIT_EXECUTOR_TOKEN: ${COMMIT_EXECUTOR_TOKEN}
networks: { shared: { aliases: [continuity-runtime-candidate] } }
healthcheck: { test: ["CMD-SHELL", "wget -qO- http://localhost:8081/healthz >/dev/null"], interval: 10s, timeout: 5s, retries: 20 }
commit-runtime:
build: { context: .., dockerfile: deploy/images/runtime.Dockerfile, args: { TARGET: commit-runtime, SOURCE_REVISION: "${ASSISTANT_SOURCE_REVISION:-unknown}" } }
restart: "no"
environment:
LISTEN_ADDR: ":8082"
POSTGRES_HOST: postgres
POSTGRES_PORT: "5432"
POSTGRES_DATABASE: ${POSTGRES_DB:-assistant}
POSTGRES_USER: commit_role
POSTGRES_PASSWORD: ${COMMIT_DB_PASSWORD}
AUTHORITY_POSTGRES_USER: authority_role
AUTHORITY_POSTGRES_PASSWORD: ${AUTHORITY_DB_PASSWORD}
CONTROL_PLANE_SERVICE_CREDENTIAL_REF: credential://domovoy/control-plane/service-token/v1
CONTROL_PLANE_SERVICE_SECRET_ROOT: /run/control-plane-secrets
CORE_RUNTIME_URL: http://core-runtime-candidate:8080
CONTINUITY_RUNTIME_URL: http://continuity-runtime-candidate:8081
COMMIT_EXECUTOR_TOKEN: ${COMMIT_EXECUTOR_TOKEN}
CANDIDATE_NOTIFICATION_WRITE_TOKEN_FILE: /run/notification-shadow-writer/token
CANDIDATE_NOTIFICATION_WRITE_SECRET_REF: credential://domovoy/candidate/notification-write/v1
CANDIDATE_RELEASE_SET_ID: assistant-slot-b
CANDIDATE_SLOT: b
CANDIDATE_TASK_ID: GEN2-CAPABILITY-001
CANDIDATE_TASK_REVISION: "7"
CANDIDATE_PRINCIPAL_ID: domovoy-creator-principal
CANDIDATE_CAPABILITY_ID: prospective-notification-delivery
CANDIDATE_RESOURCE_SCOPE: synthetic-shadow-notification-v1
CANDIDATE_BINDING_MODE: SHADOW
CANDIDATE_TARGET_KIND: NOTIFICATION_SHADOW_RECEIVER
CANDIDATE_TARGET_REF: notification-receiver-candidate
CANDIDATE_TARGET_DIGEST: f9f9fa49a0727444c280c092d5bb340f3a8f33cf325c5396f5ae177a6666da35
CANDIDATE_AUTHORITY_GRANT_ID: gen2-capability-001-shadow-notification-grant
volumes:
- assistant_core_secrets:/run/secrets:ro
- assistant_control_plane_service_secrets:/run/control-plane-secrets:ro
- assistant_candidate_notification_shadow_writer_secret:/run/notification-shadow-writer:ro
networks: { shared: { aliases: [commit-runtime-candidate] } }
healthcheck: { test: ["CMD-SHELL", "wget -qO- http://localhost:8082/healthz >/dev/null && wget -qO- http://localhost:8082/readyz >/dev/null"], interval: 10s, timeout: 5s, retries: 20 }
task-verifier:
build: { context: .., dockerfile: deploy/images/runtime.Dockerfile, args: { TARGET: task-verifier, SOURCE_REVISION: "${ASSISTANT_SOURCE_REVISION:-unknown}" } }
restart: "no"
environment: { LISTEN_ADDR: ":8091" }
networks: { shared: { aliases: [task-verifier-candidate] } }
healthcheck: { test: ["CMD-SHELL", "wget -qO- http://localhost:8091/healthz >/dev/null"], interval: 10s, timeout: 3s, retries: 10 }
work-graph-runtime:
build: { context: .., dockerfile: deploy/images/runtime.Dockerfile, args: { TARGET: work-graph-runtime, SOURCE_REVISION: "${ASSISTANT_SOURCE_REVISION:-unknown}" } }
restart: "no"
environment:
LISTEN_ADDR: ":8088"
POSTGRES_HOST: postgres
POSTGRES_PORT: "5432"
POSTGRES_DATABASE: ${POSTGRES_DB:-assistant}
WORK_POSTGRES_USER: work_role
WORK_POSTGRES_CREDENTIAL_REF: credential://domovoy/work/db-password/v1
WORK_POSTGRES_PASSWORD_FILE: /run/secrets/work_db_password
SECRET_BACKEND_ROOT: /run/secrets
CORE_RUNTIME_URL: http://core-runtime-candidate:8080
volumes: [assistant_core_secrets:/run/secrets:ro]
networks: { shared: { aliases: [work-graph-runtime-candidate] } }
healthcheck: { test: ["CMD-SHELL", "wget -qO- http://localhost:8088/healthz >/dev/null"], interval: 10s, timeout: 3s, retries: 10 }
agent-execution-bridge:
build: { context: .., dockerfile: deploy/images/runtime.Dockerfile, args: { TARGET: agent-execution-bridge, SOURCE_REVISION: "${ASSISTANT_SOURCE_REVISION:-unknown}" } }
restart: "no"
environment:
LISTEN_ADDR: ":8092"
COGNITIVE_WORKER_URL: http://cognitive-worker-candidate:8083
WORK_GRAPH_RUNTIME_URL: http://work-graph-runtime-candidate:8088
TASK_VERIFIER_URL: http://task-verifier-candidate:8091
networks: { shared: { aliases: [agent-execution-bridge-candidate] } }
healthcheck: { test: ["CMD-SHELL", "wget -qO- http://localhost:8092/healthz >/dev/null"], interval: 10s, timeout: 3s, retries: 10 }
agent-worker:
build: { context: .., dockerfile: deploy/images/runtime.Dockerfile, args: { TARGET: agent-worker, SOURCE_REVISION: "${ASSISTANT_SOURCE_REVISION:-unknown}" } }
restart: "no"
command: ["-worker-id", "agent-worker-candidate", "-status-addr", ":8090"]
environment:
POSTGRES_HOST: postgres
POSTGRES_PORT: "5432"
POSTGRES_DATABASE: ${POSTGRES_DB:-assistant}
WORK_POSTGRES_USER: work_role
WORK_POSTGRES_CREDENTIAL_REF: credential://domovoy/work/db-password/v1
WORK_POSTGRES_PASSWORD_FILE: /run/secrets/work_db_password
SECRET_BACKEND_ROOT: /run/secrets
COGNITIVE_WORKER_URL: http://cognitive-worker-candidate:8083
WORK_GRAPH_RUNTIME_URL: http://work-graph-runtime-candidate:8088
TASK_VERIFIER_URL: http://task-verifier-candidate:8091
AGENT_EXECUTION_BRIDGE_URL: http://agent-execution-bridge-candidate:8092
volumes: [assistant_core_secrets:/run/secrets:ro]
networks: { shared: { aliases: [agent-worker-candidate] } }
healthcheck: { test: ["CMD-SHELL", "wget -qO- http://localhost:8090/healthz >/dev/null"], interval: 10s, timeout: 3s, retries: 10 }
volumes:
assistant_core_secrets:
external: true
@@ -147,6 +288,14 @@ volumes:
assistant_research_sources:
name: assistant_research_sources
external: true
assistant_candidate_notification_shadow_receipts:
name: assistant_candidate_notification_shadow_receipts
assistant_candidate_notification_shadow_writer_secret:
name: assistant_candidate_notification_shadow_writer_secret
external: true
assistant_candidate_notification_shadow_reader_secret:
name: assistant_candidate_notification_shadow_reader_secret
external: true
networks:
shared:
+28 -14
View File
@@ -26,8 +26,8 @@ fail_rollback() {
exit 1
}
services=(creator-web core-runtime cognitive-worker research-runtime)
stable_services=(creator-web core-runtime cognitive-worker research-runtime)
services=(creator-web core-runtime cognitive-worker research-runtime commit-runtime continuity-runtime agent-worker agent-execution-bridge work-graph-runtime task-verifier notification-receiver)
stable_services=(creator-web core-runtime cognitive-worker research-runtime commit-runtime continuity-runtime agent-worker agent-execution-bridge work-graph-runtime task-verifier notification-receiver)
stable_ids() {
local service container id
for service in "${stable_services[@]}"; do
@@ -61,13 +61,10 @@ stable_ids_before=$(stable_ids) || exit 1
candidate_compose config >/dev/null
candidate_compose build --build-arg "SOURCE_REVISION=$REVISION" "${services[@]}"
assert_stable_ids_unchanged
candidate_compose up -d --no-build --no-deps research-runtime core-runtime
wait_healthy "$(candidate_container research-runtime)"
wait_healthy "$(candidate_container core-runtime)"
candidate_compose up -d --no-build --no-deps cognitive-worker
wait_healthy "$(candidate_container cognitive-worker)"
candidate_compose up -d --no-build --no-deps creator-web
wait_healthy "$(candidate_container creator-web)"
candidate_compose up -d --no-build --no-deps "${services[@]}"
for service in "${services[@]}"; do
wait_healthy "$(candidate_container "$service")"
done
assert_stable_ids_unchanged
for service in "${services[@]}"; do
image_revision=$(docker inspect -f '{{index .Config.Labels "org.opencontainers.image.revision"}}' "$(candidate_container "$service")" 2>/dev/null || true)
@@ -79,10 +76,23 @@ if [[ "${PREPARE_ONLY:-false}" == true ]]; then
exit 0
fi
release_json=$(cat <<JSON
{"id":"candidate-$SLOT-$REVISION","slot":"$SLOT","revision":"$REVISION","components":{"creator-web":{"name":"creator-web","revision":"$REVISION","upstream":"http://creator-web-candidate:8084","health_url":"http://creator-web-candidate:8084/healthz","ready_url":"http://creator-web-candidate:8084/healthz"},"core-runtime":{"name":"core-runtime","revision":"$REVISION","upstream":"http://core-runtime-candidate:8080","health_url":"http://core-runtime-candidate:8080/healthz","ready_url":"http://core-runtime-candidate:8080/readyz"},"cognitive-worker":{"name":"cognitive-worker","revision":"$REVISION","upstream":"http://cognitive-worker-candidate:8083","health_url":"http://cognitive-worker-candidate:8083/healthz","ready_url":"http://cognitive-worker-candidate:8083/readyz"},"research-runtime":{"name":"research-runtime","revision":"$REVISION","upstream":"http://research-runtime-candidate:8087","health_url":"http://research-runtime-candidate:8087/healthz","ready_url":"http://research-runtime-candidate:8087/readyz"}}}
JSON
)
component_port() {
case "$1" in
creator-web) echo 8084 ;; core-runtime) echo 8080 ;; cognitive-worker) echo 8083 ;; research-runtime) echo 8087 ;;
commit-runtime) echo 8082 ;; continuity-runtime) echo 8081 ;; agent-worker) echo 8090 ;; agent-execution-bridge) echo 8092 ;;
work-graph-runtime) echo 8088 ;; task-verifier) echo 8091 ;; notification-receiver) echo 8085 ;;
*) return 1 ;;
esac
}
components='{}'
for service in "${services[@]}"; do
port=$(component_port "$service") || fail_rollback
upstream="http://${service}-candidate:${port}"
ready_url="$upstream/readyz"
case "$service" in creator-web|agent-worker|agent-execution-bridge|work-graph-runtime|task-verifier|notification-receiver) ready_url="$upstream/healthz" ;; esac
components=$(jq -c --arg name "$service" --arg revision "$REVISION" --arg upstream "$upstream" --arg health "$upstream/healthz" --arg ready "$ready_url" '. + {($name): {name:$name,revision:$revision,upstream:$upstream,health_url:$health,ready_url:$ready}}' <<<"$components")
done
release_json=$(jq -cn --arg id "candidate-$SLOT-$REVISION" --arg slot "$SLOT" --arg revision "$REVISION" --argjson components "$components" '{id:$id,slot:$slot,revision:$revision,components:$components}')
stable_json=$(curl -fsS --max-time 10 "$SUPERVISOR_CONTROL_URL/v1/state")
stable_set=$(printf '%s' "$stable_json" | jq -c '.stable_release_set // empty')
if [[ -z "$stable_set" ]]; then
@@ -95,7 +105,11 @@ JSON
fi
curl -fsS --max-time 10 -X POST "$SUPERVISOR_CONTROL_URL/v1/releases/start-set" -H 'Content-Type: application/json' --data "$release_json" >/dev/null || fail_rollback
assert_stable_ids_unchanged
observation=$(printf '{"release_id":"candidate-%s-%s","ready":true,"reason":"isolated slot health and image revisions passed","observed_at_unix_millis":%s,"components":{"creator-web":{"revision":"%s","health":true,"ready":true},"core-runtime":{"revision":"%s","health":true,"ready":true},"cognitive-worker":{"revision":"%s","health":true,"ready":true},"research-runtime":{"revision":"%s","health":true,"ready":true}}}\n' "$SLOT" "$REVISION" "$(date +%s%3N)" "$REVISION" "$REVISION" "$REVISION" "$REVISION")
observed_components='{}'
for service in "${services[@]}"; do
observed_components=$(jq -c --arg name "$service" --arg revision "$REVISION" '. + {($name): {revision:$revision,health:true,ready:true}}' <<<"$observed_components")
done
observation=$(jq -cn --arg release "candidate-$SLOT-$REVISION" --argjson observed_at "$(date +%s%3N)" --argjson components "$observed_components" '{release_id:$release,ready:true,reason:"isolated slot health and image revisions passed",observed_at_unix_millis:$observed_at,components:$components}')
curl -fsS --max-time 10 -X POST "$SUPERVISOR_CONTROL_URL/v1/releases/observe-set" -H 'Content-Type: application/json' --data "$observation" >/dev/null || fail_rollback
assert_stable_ids_unchanged
control /v1/traffic/canary || fail_rollback