Compare commits

...
Author SHA1 Message Date
m 42285a55be feat(deploy): close candidate release set topology
CI / test (push) Failing after 34s
2026-08-31 16:35:08 +03:00
7 changed files with 234 additions and 28 deletions
+152 -3
View File
@@ -69,6 +69,20 @@ services:
COGNITIVE_ADMISSION_MAX_OUTPUT_BYTES: ${COGNITIVE_ADMISSION_MAX_OUTPUT_BYTES:-16384}
COGNITIVE_RUNTIME_URL: http://cognitive-worker-candidate:8083
RESEARCH_RUNTIME_URL: http://research-runtime-candidate:8087
CONTINUITY_RUNTIME_URL: http://continuity-runtime-candidate:8081
COMMIT_RUNTIME_URL: http://commit-runtime-candidate:8082
CANDIDATE_RELEASE_SET_ID: assistant-slot-b
CANDIDATE_SLOT: b
CANDIDATE_TASK_ID: GEN2-CAPABILITY-001
CANDIDATE_TASK_REVISION: "7"
CANDIDATE_PRINCIPAL_ID: domovoy-creator-principal
CANDIDATE_CAPABILITY_ID: prospective-notification-delivery
CANDIDATE_RESOURCE_SCOPE: synthetic-shadow-notification-v1
CANDIDATE_BINDING_MODE: SHADOW
CANDIDATE_TARGET_KIND: NOTIFICATION_SHADOW_RECEIVER
CANDIDATE_TARGET_REF: notification-receiver-candidate
CANDIDATE_TARGET_DIGEST: f9f9fa49a0727444c280c092d5bb340f3a8f33cf325c5396f5ae177a6666da35
CANDIDATE_AUTHORITY_GRANT_ID: gen2-capability-001-shadow-notification-grant
volumes:
- assistant_core_secrets:/run/secrets:ro
- assistant_control_plane_service_secrets:/run/control-plane-secrets:ro
@@ -124,10 +138,10 @@ services:
environment:
LISTEN_ADDR: ":8084"
CORE_RUNTIME_URL: http://core-runtime-candidate:8080
CONTINUITY_RUNTIME_URL: http://continuity-runtime:8081
COMMIT_RUNTIME_URL: http://commit-runtime:8082
CONTINUITY_RUNTIME_URL: http://continuity-runtime-candidate:8081
COMMIT_RUNTIME_URL: http://commit-runtime-candidate:8082
COGNITIVE_RUNTIME_URL: http://cognitive-worker-candidate:8083
AGENT_WORKER_URL: http://agent-worker:8090
AGENT_WORKER_URL: http://agent-worker-candidate:8090
ASSISTANT_BUILD_REVISION: ${ASSISTANT_SOURCE_REVISION:-unknown}
networks:
shared:
@@ -138,6 +152,133 @@ services:
timeout: 5s
retries: 20
notification-receiver:
build: { context: .., dockerfile: deploy/images/runtime.Dockerfile, args: { TARGET: notification-receiver, SOURCE_REVISION: "${ASSISTANT_SOURCE_REVISION:-unknown}" } }
restart: "no"
environment:
LISTEN_ADDR: ":8085"
SHADOW_RECEIVER_MODE: "1"
CANDIDATE_RELEASE_SET_ID: assistant-slot-b
CANDIDATE_TASK_ID: GEN2-CAPABILITY-001
RECEIPT_DIRECTORY: /var/lib/notification-shadow-receiver
RECEIVER_WRITE_TOKEN_FILE: /run/notification-shadow-writer/token
RECEIVER_READ_TOKEN_FILE: /run/notification-shadow-reader/token
volumes:
- assistant_candidate_notification_shadow_receipts:/var/lib/notification-shadow-receiver
- assistant_candidate_notification_shadow_writer_secret:/run/notification-shadow-writer:ro
- assistant_candidate_notification_shadow_reader_secret:/run/notification-shadow-reader:ro
networks: { shared: { aliases: [notification-receiver-candidate] } }
healthcheck: { test: ["CMD-SHELL", "wget -qO- http://localhost:8085/healthz >/dev/null"], interval: 10s, timeout: 3s, retries: 10 }
continuity-runtime:
build: { context: .., dockerfile: deploy/images/runtime.Dockerfile, args: { TARGET: continuity-runtime, SOURCE_REVISION: "${ASSISTANT_SOURCE_REVISION:-unknown}" } }
restart: "no"
environment:
LISTEN_ADDR: ":8081"
POSTGRES_HOST: postgres
POSTGRES_PORT: "5432"
POSTGRES_DATABASE: ${POSTGRES_DB:-assistant}
POSTGRES_USER: continuity_role
POSTGRES_PASSWORD: ${CONTINUITY_DB_PASSWORD}
CORE_RUNTIME_URL: http://core-runtime-candidate:8080
COMMIT_RUNTIME_URL: http://commit-runtime-candidate:8082
COMMIT_EXECUTOR_TOKEN: ${COMMIT_EXECUTOR_TOKEN}
networks: { shared: { aliases: [continuity-runtime-candidate] } }
healthcheck: { test: ["CMD-SHELL", "wget -qO- http://localhost:8081/healthz >/dev/null"], interval: 10s, timeout: 5s, retries: 20 }
commit-runtime:
build: { context: .., dockerfile: deploy/images/runtime.Dockerfile, args: { TARGET: commit-runtime, SOURCE_REVISION: "${ASSISTANT_SOURCE_REVISION:-unknown}" } }
restart: "no"
environment:
LISTEN_ADDR: ":8082"
POSTGRES_HOST: postgres
POSTGRES_PORT: "5432"
POSTGRES_DATABASE: ${POSTGRES_DB:-assistant}
POSTGRES_USER: commit_role
POSTGRES_PASSWORD: ${COMMIT_DB_PASSWORD}
AUTHORITY_POSTGRES_USER: authority_role
AUTHORITY_POSTGRES_PASSWORD: ${AUTHORITY_DB_PASSWORD}
CONTROL_PLANE_SERVICE_CREDENTIAL_REF: credential://domovoy/control-plane/service-token/v1
CONTROL_PLANE_SERVICE_SECRET_ROOT: /run/control-plane-secrets
CORE_RUNTIME_URL: http://core-runtime-candidate:8080
CONTINUITY_RUNTIME_URL: http://continuity-runtime-candidate:8081
COMMIT_EXECUTOR_TOKEN: ${COMMIT_EXECUTOR_TOKEN}
CANDIDATE_NOTIFICATION_WRITE_TOKEN_FILE: /run/notification-shadow-writer/token
CANDIDATE_NOTIFICATION_WRITE_SECRET_REF: credential://domovoy/candidate/notification-write/v1
CANDIDATE_RELEASE_SET_ID: assistant-slot-b
CANDIDATE_SLOT: b
CANDIDATE_TASK_ID: GEN2-CAPABILITY-001
CANDIDATE_TASK_REVISION: "7"
CANDIDATE_PRINCIPAL_ID: domovoy-creator-principal
CANDIDATE_CAPABILITY_ID: prospective-notification-delivery
CANDIDATE_RESOURCE_SCOPE: synthetic-shadow-notification-v1
CANDIDATE_BINDING_MODE: SHADOW
CANDIDATE_TARGET_KIND: NOTIFICATION_SHADOW_RECEIVER
CANDIDATE_TARGET_REF: notification-receiver-candidate
CANDIDATE_TARGET_DIGEST: f9f9fa49a0727444c280c092d5bb340f3a8f33cf325c5396f5ae177a6666da35
CANDIDATE_AUTHORITY_GRANT_ID: gen2-capability-001-shadow-notification-grant
volumes:
- assistant_core_secrets:/run/secrets:ro
- assistant_control_plane_service_secrets:/run/control-plane-secrets:ro
- assistant_candidate_notification_shadow_writer_secret:/run/notification-shadow-writer:ro
networks: { shared: { aliases: [commit-runtime-candidate] } }
healthcheck: { test: ["CMD-SHELL", "wget -qO- http://localhost:8082/healthz >/dev/null && wget -qO- http://localhost:8082/readyz >/dev/null"], interval: 10s, timeout: 5s, retries: 20 }
task-verifier:
build: { context: .., dockerfile: deploy/images/runtime.Dockerfile, args: { TARGET: task-verifier, SOURCE_REVISION: "${ASSISTANT_SOURCE_REVISION:-unknown}" } }
restart: "no"
environment: { LISTEN_ADDR: ":8091" }
networks: { shared: { aliases: [task-verifier-candidate] } }
healthcheck: { test: ["CMD-SHELL", "wget -qO- http://localhost:8091/healthz >/dev/null"], interval: 10s, timeout: 3s, retries: 10 }
work-graph-runtime:
build: { context: .., dockerfile: deploy/images/runtime.Dockerfile, args: { TARGET: work-graph-runtime, SOURCE_REVISION: "${ASSISTANT_SOURCE_REVISION:-unknown}" } }
restart: "no"
environment:
LISTEN_ADDR: ":8088"
POSTGRES_HOST: postgres
POSTGRES_PORT: "5432"
POSTGRES_DATABASE: ${POSTGRES_DB:-assistant}
WORK_POSTGRES_USER: work_role
WORK_POSTGRES_CREDENTIAL_REF: credential://domovoy/work/db-password/v1
WORK_POSTGRES_PASSWORD_FILE: /run/secrets/work_db_password
SECRET_BACKEND_ROOT: /run/secrets
CORE_RUNTIME_URL: http://core-runtime-candidate:8080
volumes: [assistant_core_secrets:/run/secrets:ro]
networks: { shared: { aliases: [work-graph-runtime-candidate] } }
healthcheck: { test: ["CMD-SHELL", "wget -qO- http://localhost:8088/healthz >/dev/null"], interval: 10s, timeout: 3s, retries: 10 }
agent-execution-bridge:
build: { context: .., dockerfile: deploy/images/runtime.Dockerfile, args: { TARGET: agent-execution-bridge, SOURCE_REVISION: "${ASSISTANT_SOURCE_REVISION:-unknown}" } }
restart: "no"
environment:
LISTEN_ADDR: ":8092"
COGNITIVE_WORKER_URL: http://cognitive-worker-candidate:8083
WORK_GRAPH_RUNTIME_URL: http://work-graph-runtime-candidate:8088
TASK_VERIFIER_URL: http://task-verifier-candidate:8091
networks: { shared: { aliases: [agent-execution-bridge-candidate] } }
healthcheck: { test: ["CMD-SHELL", "wget -qO- http://localhost:8092/healthz >/dev/null"], interval: 10s, timeout: 3s, retries: 10 }
agent-worker:
build: { context: .., dockerfile: deploy/images/runtime.Dockerfile, args: { TARGET: agent-worker, SOURCE_REVISION: "${ASSISTANT_SOURCE_REVISION:-unknown}" } }
restart: "no"
command: ["-worker-id", "agent-worker-candidate", "-status-addr", ":8090"]
environment:
POSTGRES_HOST: postgres
POSTGRES_PORT: "5432"
POSTGRES_DATABASE: ${POSTGRES_DB:-assistant}
WORK_POSTGRES_USER: work_role
WORK_POSTGRES_CREDENTIAL_REF: credential://domovoy/work/db-password/v1
WORK_POSTGRES_PASSWORD_FILE: /run/secrets/work_db_password
SECRET_BACKEND_ROOT: /run/secrets
COGNITIVE_WORKER_URL: http://cognitive-worker-candidate:8083
WORK_GRAPH_RUNTIME_URL: http://work-graph-runtime-candidate:8088
TASK_VERIFIER_URL: http://task-verifier-candidate:8091
AGENT_EXECUTION_BRIDGE_URL: http://agent-execution-bridge-candidate:8092
volumes: [assistant_core_secrets:/run/secrets:ro]
networks: { shared: { aliases: [agent-worker-candidate] } }
healthcheck: { test: ["CMD-SHELL", "wget -qO- http://localhost:8090/healthz >/dev/null"], interval: 10s, timeout: 3s, retries: 10 }
volumes:
assistant_core_secrets:
external: true
@@ -147,6 +288,14 @@ volumes:
assistant_research_sources:
name: assistant_research_sources
external: true
assistant_candidate_notification_shadow_receipts:
name: assistant_candidate_notification_shadow_receipts
assistant_candidate_notification_shadow_writer_secret:
name: assistant_candidate_notification_shadow_writer_secret
external: true
assistant_candidate_notification_shadow_reader_secret:
name: assistant_candidate_notification_shadow_reader_secret
external: true
networks:
shared:
+28 -14
View File
@@ -26,8 +26,8 @@ fail_rollback() {
exit 1
}
services=(creator-web core-runtime cognitive-worker research-runtime)
stable_services=(creator-web core-runtime cognitive-worker research-runtime)
services=(creator-web core-runtime cognitive-worker research-runtime commit-runtime continuity-runtime agent-worker agent-execution-bridge work-graph-runtime task-verifier notification-receiver)
stable_services=(creator-web core-runtime cognitive-worker research-runtime commit-runtime continuity-runtime agent-worker agent-execution-bridge work-graph-runtime task-verifier notification-receiver)
stable_ids() {
local service container id
for service in "${stable_services[@]}"; do
@@ -61,13 +61,10 @@ stable_ids_before=$(stable_ids) || exit 1
candidate_compose config >/dev/null
candidate_compose build --build-arg "SOURCE_REVISION=$REVISION" "${services[@]}"
assert_stable_ids_unchanged
candidate_compose up -d --no-build --no-deps research-runtime core-runtime
wait_healthy "$(candidate_container research-runtime)"
wait_healthy "$(candidate_container core-runtime)"
candidate_compose up -d --no-build --no-deps cognitive-worker
wait_healthy "$(candidate_container cognitive-worker)"
candidate_compose up -d --no-build --no-deps creator-web
wait_healthy "$(candidate_container creator-web)"
candidate_compose up -d --no-build --no-deps "${services[@]}"
for service in "${services[@]}"; do
wait_healthy "$(candidate_container "$service")"
done
assert_stable_ids_unchanged
for service in "${services[@]}"; do
image_revision=$(docker inspect -f '{{index .Config.Labels "org.opencontainers.image.revision"}}' "$(candidate_container "$service")" 2>/dev/null || true)
@@ -79,10 +76,23 @@ if [[ "${PREPARE_ONLY:-false}" == true ]]; then
exit 0
fi
release_json=$(cat <<JSON
{"id":"candidate-$SLOT-$REVISION","slot":"$SLOT","revision":"$REVISION","components":{"creator-web":{"name":"creator-web","revision":"$REVISION","upstream":"http://creator-web-candidate:8084","health_url":"http://creator-web-candidate:8084/healthz","ready_url":"http://creator-web-candidate:8084/healthz"},"core-runtime":{"name":"core-runtime","revision":"$REVISION","upstream":"http://core-runtime-candidate:8080","health_url":"http://core-runtime-candidate:8080/healthz","ready_url":"http://core-runtime-candidate:8080/readyz"},"cognitive-worker":{"name":"cognitive-worker","revision":"$REVISION","upstream":"http://cognitive-worker-candidate:8083","health_url":"http://cognitive-worker-candidate:8083/healthz","ready_url":"http://cognitive-worker-candidate:8083/readyz"},"research-runtime":{"name":"research-runtime","revision":"$REVISION","upstream":"http://research-runtime-candidate:8087","health_url":"http://research-runtime-candidate:8087/healthz","ready_url":"http://research-runtime-candidate:8087/readyz"}}}
JSON
)
component_port() {
case "$1" in
creator-web) echo 8084 ;; core-runtime) echo 8080 ;; cognitive-worker) echo 8083 ;; research-runtime) echo 8087 ;;
commit-runtime) echo 8082 ;; continuity-runtime) echo 8081 ;; agent-worker) echo 8090 ;; agent-execution-bridge) echo 8092 ;;
work-graph-runtime) echo 8088 ;; task-verifier) echo 8091 ;; notification-receiver) echo 8085 ;;
*) return 1 ;;
esac
}
components='{}'
for service in "${services[@]}"; do
port=$(component_port "$service") || fail_rollback
upstream="http://${service}-candidate:${port}"
ready_url="$upstream/readyz"
case "$service" in creator-web|agent-worker|agent-execution-bridge|work-graph-runtime|task-verifier|notification-receiver) ready_url="$upstream/healthz" ;; esac
components=$(jq -c --arg name "$service" --arg revision "$REVISION" --arg upstream "$upstream" --arg health "$upstream/healthz" --arg ready "$ready_url" '. + {($name): {name:$name,revision:$revision,upstream:$upstream,health_url:$health,ready_url:$ready}}' <<<"$components")
done
release_json=$(jq -cn --arg id "candidate-$SLOT-$REVISION" --arg slot "$SLOT" --arg revision "$REVISION" --argjson components "$components" '{id:$id,slot:$slot,revision:$revision,components:$components}')
stable_json=$(curl -fsS --max-time 10 "$SUPERVISOR_CONTROL_URL/v1/state")
stable_set=$(printf '%s' "$stable_json" | jq -c '.stable_release_set // empty')
if [[ -z "$stable_set" ]]; then
@@ -95,7 +105,11 @@ JSON
fi
curl -fsS --max-time 10 -X POST "$SUPERVISOR_CONTROL_URL/v1/releases/start-set" -H 'Content-Type: application/json' --data "$release_json" >/dev/null || fail_rollback
assert_stable_ids_unchanged
observation=$(printf '{"release_id":"candidate-%s-%s","ready":true,"reason":"isolated slot health and image revisions passed","observed_at_unix_millis":%s,"components":{"creator-web":{"revision":"%s","health":true,"ready":true},"core-runtime":{"revision":"%s","health":true,"ready":true},"cognitive-worker":{"revision":"%s","health":true,"ready":true},"research-runtime":{"revision":"%s","health":true,"ready":true}}}\n' "$SLOT" "$REVISION" "$(date +%s%3N)" "$REVISION" "$REVISION" "$REVISION" "$REVISION")
observed_components='{}'
for service in "${services[@]}"; do
observed_components=$(jq -c --arg name "$service" --arg revision "$REVISION" '. + {($name): {revision:$revision,health:true,ready:true}}' <<<"$observed_components")
done
observation=$(jq -cn --arg release "candidate-$SLOT-$REVISION" --argjson observed_at "$(date +%s%3N)" --argjson components "$observed_components" '{release_id:$release,ready:true,reason:"isolated slot health and image revisions passed",observed_at_unix_millis:$observed_at,components:$components}')
curl -fsS --max-time 10 -X POST "$SUPERVISOR_CONTROL_URL/v1/releases/observe-set" -H 'Content-Type: application/json' --data "$observation" >/dev/null || fail_rollback
assert_stable_ids_unchanged
control /v1/traffic/canary || fail_rollback
+1 -1
View File
@@ -1 +1 @@
{"task_file":"dev/tasks/GEN2-CAPABILITY-001-release-scoped-candidate-shadow-bindings.json","task_id":"GEN2-CAPABILITY-001","task_revision":7}
{"task_file":"dev/tasks/GEN2-DEPLOY-004-candidate-commit-runtime-releaseset.json","task_id":"GEN2-DEPLOY-004","task_revision":3}
@@ -0,0 +1,17 @@
# GEN2-DEPLOY-004 candidate shadow secret capability gap
Date: 2026-08-31
The candidate Compose slot requires two distinct external, read-only Docker
volumes: `assistant_candidate_notification_shadow_writer_secret` and
`assistant_candidate_notification_shadow_reader_secret`. They are deliberately
not aliases for the production notification writer or reader volumes.
A read-only Docker volume inspection on `docker.cin.su` confirmed that both
candidate-only volumes are absent. No secret values were read or emitted.
The candidate ReleaseSet must not be started until the existing production
secret control plane provisions both bounded artifacts with distinct values.
This preserves the fail-closed, zero-external-delivery property of the shadow
notification path. Stable containers and production notification artifacts
were not modified.
+24 -5
View File
@@ -28,10 +28,17 @@ type Release struct {
}
const (
ComponentCreatorWeb = "creator-web"
ComponentCoreRuntime = "core-runtime"
ComponentCognitiveWorker = "cognitive-worker"
ComponentResearchRuntime = "research-runtime"
ComponentCreatorWeb = "creator-web"
ComponentCoreRuntime = "core-runtime"
ComponentCognitiveWorker = "cognitive-worker"
ComponentResearchRuntime = "research-runtime"
ComponentCommitRuntime = "commit-runtime"
ComponentContinuityRuntime = "continuity-runtime"
ComponentAgentWorker = "agent-worker"
ComponentAgentExecutionBridge = "agent-execution-bridge"
ComponentWorkGraphRuntime = "work-graph-runtime"
ComponentTaskVerifier = "task-verifier"
ComponentNotificationReceiver = "notification-receiver"
)
// ComponentRelease is operational deployment evidence. It is deliberately
@@ -54,7 +61,19 @@ type ReleaseSet struct {
Components map[string]ComponentRelease `json:"components"`
}
var requiredComponents = []string{ComponentCreatorWeb, ComponentCoreRuntime, ComponentCognitiveWorker, ComponentResearchRuntime}
var requiredComponents = []string{
ComponentCreatorWeb,
ComponentCoreRuntime,
ComponentCognitiveWorker,
ComponentResearchRuntime,
ComponentCommitRuntime,
ComponentContinuityRuntime,
ComponentAgentWorker,
ComponentAgentExecutionBridge,
ComponentWorkGraphRuntime,
ComponentTaskVerifier,
ComponentNotificationReceiver,
}
func (s ReleaseSet) Validate() bool {
if s.ID == "" || s.Revision == "" || len(s.Components) != len(requiredComponents) {
+7 -2
View File
@@ -84,12 +84,17 @@ func TestPersistentSupervisorRestoresTrafficModeAfterRestart(t *testing.T) {
}
func testReleaseSet(id, revision, web, core, cognitive, research string) supervisor.ReleaseSet {
return supervisor.ReleaseSet{ID: id, Revision: revision, Components: map[string]supervisor.ComponentRelease{
components := map[string]supervisor.ComponentRelease{
supervisor.ComponentCreatorWeb: {Name: supervisor.ComponentCreatorWeb, Revision: revision, Upstream: web, HealthURL: web + "/healthz", ReadyURL: web + "/readyz"},
supervisor.ComponentCoreRuntime: {Name: supervisor.ComponentCoreRuntime, Revision: revision, Upstream: core, HealthURL: core + "/healthz", ReadyURL: core + "/readyz"},
supervisor.ComponentCognitiveWorker: {Name: supervisor.ComponentCognitiveWorker, Revision: revision, Upstream: cognitive, HealthURL: cognitive + "/healthz", ReadyURL: cognitive + "/readyz"},
supervisor.ComponentResearchRuntime: {Name: supervisor.ComponentResearchRuntime, Revision: revision, Upstream: research, HealthURL: research + "/healthz", ReadyURL: research + "/readyz"},
}}
}
for _, name := range []string{supervisor.ComponentCommitRuntime, supervisor.ComponentContinuityRuntime, supervisor.ComponentAgentWorker, supervisor.ComponentAgentExecutionBridge, supervisor.ComponentWorkGraphRuntime, supervisor.ComponentTaskVerifier, supervisor.ComponentNotificationReceiver} {
upstream := "http://" + name
components[name] = supervisor.ComponentRelease{Name: name, Revision: revision, Upstream: upstream, HealthURL: upstream + "/healthz", ReadyURL: upstream + "/readyz"}
}
return supervisor.ReleaseSet{ID: id, Revision: revision, Components: components}
}
func TestReleaseSetRejectsMixedRevisionsAndMissingComponents(t *testing.T) {
@@ -11,9 +11,11 @@ stable_updater="$root/deploy/production-stable-updater.sh"
grep -Fq 'CANDIDATE_PROJECT="${CANDIDATE_PROJECT:-assistant-slot-$SLOT}"' "$canary"
grep -Fq '[[ "$CANDIDATE_PROJECT" != "$STABLE_PROJECT" ]]' "$canary"
grep -Fq 'docker compose -p "$CANDIDATE_PROJECT"' "$canary"
grep -Fq 'up -d --no-build --no-deps research-runtime core-runtime' "$canary"
grep -Fq 'up -d --no-build --no-deps cognitive-worker' "$canary"
grep -Fq 'up -d --no-build --no-deps creator-web' "$canary"
grep -Fq 'services=(creator-web core-runtime cognitive-worker research-runtime commit-runtime continuity-runtime agent-worker agent-execution-bridge work-graph-runtime task-verifier notification-receiver)' "$canary"
grep -Fq 'candidate_compose up -d --no-build --no-deps "${services[@]}"' "$canary"
grep -Fq 'component_port()' "$canary"
grep -Fq 'notification-receiver-candidate' "$candidate_compose"
grep -Fq 'assistant_candidate_notification_shadow_writer_secret' "$candidate_compose"
grep -Fq 'compose up -d --no-build --no-deps supervisor' "$supervisor"
grep -Fq 'stable_ids_before=$(stable_ids)' "$canary"
grep -Fq 'assert_stable_ids_unchanged' "$canary"