Compare commits
8
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
2ee5da8114 | ||
|
|
91f0c9e799 | ||
|
|
cb66a60db2 | ||
|
|
6a4f973e38 | ||
|
|
87ec7efd0f | ||
|
|
51e33e7283 | ||
|
|
e8f35f6393 | ||
|
|
8babf6ae1f |
@@ -6,6 +6,7 @@ ENV_FILE="${ENV_FILE:-/home/m/assistant-deploy/production-compose.env}"
|
||||
COMPOSE_FILE="$BASE_DIR/deploy/docker-compose.prod.yml"
|
||||
SUPERVISOR_CONTROL_URL="${SUPERVISOR_CONTROL_URL:-http://127.0.0.1:${SUPERVISOR_CONTROL_PORT:-18085}}"
|
||||
REVISION="${1:?candidate revision is required}"
|
||||
HEALTH_ATTEMPTS="${HEALTH_ATTEMPTS:-360}"
|
||||
|
||||
set -a
|
||||
. "$ENV_FILE"
|
||||
@@ -47,22 +48,38 @@ assert_stable_ids_unchanged() {
|
||||
stable_ids_before=$(stable_ids) || exit 1
|
||||
|
||||
compose --profile assistant-canary build --build-arg "SOURCE_REVISION=$REVISION" "${services[@]}"
|
||||
compose --profile assistant-canary up -d --no-build --no-deps "${services[@]}"
|
||||
assert_stable_ids_unchanged
|
||||
|
||||
for container in "${containers[@]}"; do
|
||||
wait_healthy() {
|
||||
local container="$1" status="" ready=""
|
||||
ready=""
|
||||
for attempt in $(seq 1 60); do
|
||||
for attempt in $(seq 1 "$HEALTH_ATTEMPTS"); do
|
||||
status=$(docker inspect -f '{{if .State.Health}}{{.State.Health.Status}}{{else}}unknown{{end}}' "$container" 2>/dev/null || true)
|
||||
if [[ "$status" == healthy ]]; then ready=true; break; fi
|
||||
sleep 2
|
||||
done
|
||||
[[ "$ready" == true ]] || fail_rollback
|
||||
}
|
||||
|
||||
compose --profile assistant-canary up -d --no-build --no-deps research-runtime-candidate core-runtime-candidate
|
||||
wait_healthy assistant-research-runtime-candidate-1
|
||||
wait_healthy assistant-core-runtime-candidate-1
|
||||
compose --profile assistant-canary up -d --no-build --no-deps cognitive-worker-candidate
|
||||
wait_healthy assistant-cognitive-worker-candidate-1
|
||||
compose --profile assistant-canary up -d --no-build --no-deps creator-web-candidate
|
||||
wait_healthy assistant-creator-web-candidate-1
|
||||
assert_stable_ids_unchanged
|
||||
|
||||
for container in "${containers[@]}"; do
|
||||
image_revision=$(docker inspect -f '{{index .Config.Labels "org.opencontainers.image.revision"}}' "$container" 2>/dev/null || true)
|
||||
[[ "$image_revision" == "$REVISION" ]] || fail_rollback
|
||||
done
|
||||
assert_stable_ids_unchanged
|
||||
|
||||
if [[ "${PREPARE_ONLY:-false}" == true ]]; then
|
||||
echo "production ReleaseSet candidate prepared revision=$REVISION"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
release_json=$(cat <<JSON
|
||||
{
|
||||
"id":"candidate-$REVISION",
|
||||
|
||||
@@ -0,0 +1,48 @@
|
||||
{
|
||||
"task_id": "GEN2-COMPAT-OBS-001",
|
||||
"task_revision": 2,
|
||||
"status": "BLOCKED_BY_MISSING_CAPABILITY",
|
||||
"observed_at_utc": "2026-08-30T10:13:06.5676592Z",
|
||||
"deployment_revision": "87ec7efd0fa7bdbed22c1c0c67d671907029d7bb",
|
||||
"deployment_scope": "candidate-only; no Supervisor traffic change or promotion",
|
||||
"stable_release_set": {
|
||||
"traffic_mode": "STABLE",
|
||||
"revision": "3182092d5cc758f5a536ce10f5428b5d782bb961",
|
||||
"container_ids_before_protected_operation": {
|
||||
"creator-web": "9578e7c7e0137ba36a7fc75ffaf9fce33be046e38531c9de99c0b75c2ce465f8",
|
||||
"core-runtime": "8281ba60b234825d6ee965887e8eef10e29a0783264c05d39cbe937f1b780530",
|
||||
"cognitive-worker": "987640770cad44a0b05a47dbf08a4b47a9225f8030f8bc81721fdc0834e2ee4c",
|
||||
"research-runtime": "7e7d6e5fc050b5bde5073f0939e5a9652a1f68be0a75104cc0b486ec0cd745a6"
|
||||
},
|
||||
"container_ids_after_protected_operation": {
|
||||
"creator-web": "9578e7c7e0137ba36a7fc75ffaf9fce33be046e38531c9de99c0b75c2ce465f8",
|
||||
"core-runtime": "8281ba60b234825d6ee965887e8eef10e29a0783264c05d39cbe937f1b780530",
|
||||
"cognitive-worker": "987640770cad44a0b05a47dbf08a4b47a9225f8030f8bc81721fdc0834e2ee4c",
|
||||
"research-runtime": "7e7d6e5fc050b5bde5073f0939e5a9652a1f68be0a75104cc0b486ec0cd745a6"
|
||||
},
|
||||
"stable_ids_unchanged": true
|
||||
},
|
||||
"candidate_components": {
|
||||
"creator-web": {"health": "healthy", "revision": "87ec7efd0fa7bdbed22c1c0c67d671907029d7bb"},
|
||||
"core-runtime": {"health": "healthy", "revision": "87ec7efd0fa7bdbed22c1c0c67d671907029d7bb"},
|
||||
"cognitive-worker": {"health": "healthy", "revision": "87ec7efd0fa7bdbed22c1c0c67d671907029d7bb"},
|
||||
"research-runtime": {"health": "healthy", "revision": "87ec7efd0fa7bdbed22c1c0c67d671907029d7bb"}
|
||||
},
|
||||
"auth_harness": {
|
||||
"approved_harness_found": false,
|
||||
"available_mechanisms": ["/api/session", "tests/creator_web/verify-remote.ps1", "tests/cognitive_advisory/verify-remote.ps1"],
|
||||
"gap": "available remote scripts require production password or bootstrap token; no sanctioned test principal/service harness without reading a credential was found",
|
||||
"browser_or_secret_bypass_used": false
|
||||
},
|
||||
"telemetry_observation": {
|
||||
"started": false,
|
||||
"reason": "no legitimate authenticated generation traffic; candidate was intentionally not promoted",
|
||||
"v1_paths_deleted": false
|
||||
},
|
||||
"deployment_incident": {
|
||||
"occurred": true,
|
||||
"cause": "first attempted checkout used stale local origin/gen2-integration ref and ran pre-guard script",
|
||||
"effect": "core stable container was recreated once; image revision remained 3182092 and stable traffic stayed active",
|
||||
"recovery": "subsequent protected candidate-only deployment used FETCH_HEAD 87ec7ef and proved stable IDs unchanged"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,46 @@
|
||||
{
|
||||
"task_id": "GEN2-COMPAT-OBS-001",
|
||||
"status": "BLOCKED_BY_DEPLOYMENT_CONTROL",
|
||||
"observed_at_utc": "2026-08-30T10:37:07.7219511Z",
|
||||
"browser_connector": {
|
||||
"authenticated_tab_visible": true,
|
||||
"stale_login_tab_also_present": true,
|
||||
"cookies_tokens_or_passwords_read": false
|
||||
},
|
||||
"requested_candidate_revision": "87ec7efd0fa7bdbed22c1c0c67d671907029d7bb",
|
||||
"candidate_prepare": "not_completed_as_a_safe_release_set_promotion",
|
||||
"failure": {
|
||||
"phase": "candidate creator startup after cognitive readiness",
|
||||
"result": "stable container IDs changed despite candidate-only --no-deps; Supervisor control became temporarily unavailable",
|
||||
"stable_ids_before": {
|
||||
"creator-web": "4df348752f1a6def05acb1934bfc23048902ad6f235a7da967806d481e9ce7b7",
|
||||
"core-runtime": "a739d2d5c713dde575f571b38e58ec33143994aa9e3bcc2f5d5fc765a50e1796",
|
||||
"cognitive-worker": "a33d61b32116711287a166e0cce6362dc10de321f920549ef8a344fe0b5efb2d",
|
||||
"research-runtime": "88cfade5cdce288576b1c73493f83c84d915ac82c3d16dc6d3fedb9579a4ba8c"
|
||||
},
|
||||
"stable_ids_after": {
|
||||
"creator-web": "5181e5c6b48eaed6c6ea85eff5668c0e675b72bb982eb9425d6f39f899378ec9",
|
||||
"core-runtime": "641e8f7ae8177e4638cf806d9be1bb6f8b5ac028deb0c0c410c6ff4139032bd5",
|
||||
"cognitive-worker": "2e6d46f5b533cfeda261ceafd65ad5b5873b8aa8166bffc4ae91033a6c994d0a",
|
||||
"research-runtime": "b15d9b0e116fbd18fa726e33541ef11fe1bafe4a85210ab5ee500466a20b400b"
|
||||
},
|
||||
"stable_ids_unchanged": false,
|
||||
"cause": "external production updater/Compose reconcile reacts to production checkout or candidate Compose operation; --no-deps alone does not isolate stable in this control plane"
|
||||
},
|
||||
"recovery": {
|
||||
"traffic_mode": "STABLE",
|
||||
"revision": "91f0c9e7999907a4ee7665b77742dd452c18b17b",
|
||||
"healthz": 200,
|
||||
"readyz": 200,
|
||||
"supervisor_baseline_aligned": true,
|
||||
"data_or_volumes_deleted": false
|
||||
},
|
||||
"observation": {
|
||||
"authenticated_workload_sent": false,
|
||||
"telemetry_promoted": false,
|
||||
"baseline_started": false,
|
||||
"tag_created": false,
|
||||
"v1_paths_deleted": false
|
||||
},
|
||||
"required_follow_up": "deployment control plane must isolate candidate Compose project or disable updater reconcile before safe ReleaseSet promotion"
|
||||
}
|
||||
@@ -5,11 +5,15 @@ root="$(CDPATH= cd -- "$(dirname -- "$0")/../.." && pwd)"
|
||||
canary="$root/deploy/run-production-canary.sh"
|
||||
supervisor="$root/deploy/update-supervisor.sh"
|
||||
|
||||
grep -Fq 'up -d --no-build --no-deps "${services[@]}"' "$canary"
|
||||
grep -Fq 'up -d --no-build --no-deps research-runtime-candidate core-runtime-candidate' "$canary"
|
||||
grep -Fq 'up -d --no-build --no-deps cognitive-worker-candidate' "$canary"
|
||||
grep -Fq 'up -d --no-build --no-deps creator-web-candidate' "$canary"
|
||||
grep -Fq 'compose up -d --no-build --no-deps supervisor' "$supervisor"
|
||||
grep -Fq 'stable_ids_before=$(stable_ids)' "$canary"
|
||||
grep -Fq 'assert_stable_ids_unchanged' "$canary"
|
||||
grep -Fq 'before=$(stable_ids)' "$supervisor"
|
||||
grep -Fq 'after=$(stable_ids)' "$supervisor"
|
||||
grep -Fq 'stable container IDs changed' "$supervisor"
|
||||
grep -Fq 'HEALTH_ATTEMPTS="${HEALTH_ATTEMPTS:-360}"' "$canary"
|
||||
grep -Fq 'seq 1 "$HEALTH_ATTEMPTS"' "$canary"
|
||||
echo 'stable release-set guard: PASS'
|
||||
|
||||
Reference in New Issue
Block a user