Compare commits
34
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
2ee5da8114 | ||
|
|
91f0c9e799 | ||
|
|
cb66a60db2 | ||
|
|
6a4f973e38 | ||
|
|
87ec7efd0f | ||
|
|
51e33e7283 | ||
|
|
e8f35f6393 | ||
|
|
8babf6ae1f | ||
|
|
d0210e5aed | ||
|
|
ebdfaa0769 | ||
|
|
9e0faa7c47 | ||
|
|
7f8a05ccfb | ||
|
|
c94d04acd0 | ||
|
|
bb866cd2ab | ||
|
|
892266a4c8 | ||
|
|
ac531c3ec3 | ||
|
|
4cd50a0937 | ||
|
|
8b751257e8 | ||
|
|
cbbb60e081 | ||
|
|
46e3cfc445 | ||
|
|
5832c49ad8 | ||
|
|
b13d12583b | ||
|
|
9d5473d260 | ||
|
|
50b508b641 | ||
|
|
8400cdd92d | ||
|
|
ec1860d441 | ||
|
|
134ba26e08 | ||
|
|
b82b8b874f | ||
|
|
5c9d4663f8 | ||
|
|
ec9d434650 | ||
|
|
a8a60ce29e | ||
|
|
96cf66a876 | ||
|
|
381b52d06d | ||
|
|
4ceec5a316 |
@@ -45,7 +45,7 @@ func main() {
|
||||
fatal(err)
|
||||
}
|
||||
|
||||
s := &server{lifeboat: lifeboat, stableUpstream: stable, candidate: candidate, client: &http.Client{Timeout: 5 * time.Second}}
|
||||
s := &server{lifeboat: lifeboat, stableUpstream: stable, candidate: candidate, client: &http.Client{Timeout: 30 * time.Second}}
|
||||
public := http.NewServeMux()
|
||||
public.HandleFunc("GET /healthz", s.health)
|
||||
public.HandleFunc("GET /readyz", s.ready)
|
||||
|
||||
@@ -402,6 +402,11 @@ services:
|
||||
continuity-runtime: { condition: service_started }
|
||||
commit-runtime: { condition: service_started }
|
||||
cognitive-worker-candidate: { condition: service_healthy }
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "wget -qO- http://localhost:8084/healthz >/dev/null"]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 20
|
||||
|
||||
volumes:
|
||||
assistant_pg:
|
||||
|
||||
@@ -6,6 +6,7 @@ ENV_FILE="${ENV_FILE:-/home/m/assistant-deploy/production-compose.env}"
|
||||
COMPOSE_FILE="$BASE_DIR/deploy/docker-compose.prod.yml"
|
||||
SUPERVISOR_CONTROL_URL="${SUPERVISOR_CONTROL_URL:-http://127.0.0.1:${SUPERVISOR_CONTROL_PORT:-18085}}"
|
||||
REVISION="${1:?candidate revision is required}"
|
||||
HEALTH_ATTEMPTS="${HEALTH_ATTEMPTS:-360}"
|
||||
|
||||
set -a
|
||||
. "$ENV_FILE"
|
||||
@@ -22,28 +23,69 @@ fail_rollback() {
|
||||
|
||||
services=(creator-web-candidate core-runtime-candidate cognitive-worker-candidate research-runtime-candidate)
|
||||
containers=(assistant-creator-web-candidate-1 assistant-core-runtime-candidate-1 assistant-cognitive-worker-candidate-1 assistant-research-runtime-candidate-1)
|
||||
stable_services=(creator-web core-runtime cognitive-worker research-runtime)
|
||||
|
||||
stable_ids() {
|
||||
local service container id
|
||||
for service in "${stable_services[@]}"; do
|
||||
container="assistant-${service}-1"
|
||||
id=$(docker inspect -f '{{.Id}}' "$container" 2>/dev/null || true)
|
||||
[[ -n "$id" ]] || { echo "stable container missing: $container" >&2; return 1; }
|
||||
printf '%s=%s\n' "$service" "$id"
|
||||
done
|
||||
}
|
||||
|
||||
assert_stable_ids_unchanged() {
|
||||
local after
|
||||
after=$(stable_ids) || fail_rollback
|
||||
[[ "$after" == "$stable_ids_before" ]] || {
|
||||
echo "stable container IDs changed during candidate operation" >&2
|
||||
diff -u <(printf '%s\n' "$stable_ids_before") <(printf '%s\n' "$after") >&2 || true
|
||||
fail_rollback
|
||||
}
|
||||
}
|
||||
|
||||
stable_ids_before=$(stable_ids) || exit 1
|
||||
|
||||
compose --profile assistant-canary build --build-arg "SOURCE_REVISION=$REVISION" "${services[@]}"
|
||||
compose --profile assistant-canary up -d --no-build "${services[@]}"
|
||||
|
||||
for container in "${containers[@]}"; do
|
||||
wait_healthy() {
|
||||
local container="$1" status="" ready=""
|
||||
ready=""
|
||||
for attempt in $(seq 1 60); do
|
||||
status="$(docker inspect -f '{{if .State.Health}}{{.State.Health.Status}}{{else}}unknown{{end}}' "$container" 2>/dev/null || true)"
|
||||
for attempt in $(seq 1 "$HEALTH_ATTEMPTS"); do
|
||||
status=$(docker inspect -f '{{if .State.Health}}{{.State.Health.Status}}{{else}}unknown{{end}}' "$container" 2>/dev/null || true)
|
||||
if [[ "$status" == healthy ]]; then ready=true; break; fi
|
||||
sleep 2
|
||||
done
|
||||
[[ "$ready" == true ]] || fail_rollback
|
||||
image_revision="$(docker inspect -f '{{index .Config.Labels "org.opencontainers.image.revision"}}' "$container" 2>/dev/null || true)"
|
||||
}
|
||||
|
||||
compose --profile assistant-canary up -d --no-build --no-deps research-runtime-candidate core-runtime-candidate
|
||||
wait_healthy assistant-research-runtime-candidate-1
|
||||
wait_healthy assistant-core-runtime-candidate-1
|
||||
compose --profile assistant-canary up -d --no-build --no-deps cognitive-worker-candidate
|
||||
wait_healthy assistant-cognitive-worker-candidate-1
|
||||
compose --profile assistant-canary up -d --no-build --no-deps creator-web-candidate
|
||||
wait_healthy assistant-creator-web-candidate-1
|
||||
assert_stable_ids_unchanged
|
||||
|
||||
for container in "${containers[@]}"; do
|
||||
image_revision=$(docker inspect -f '{{index .Config.Labels "org.opencontainers.image.revision"}}' "$container" 2>/dev/null || true)
|
||||
[[ "$image_revision" == "$REVISION" ]] || fail_rollback
|
||||
done
|
||||
assert_stable_ids_unchanged
|
||||
|
||||
release_json="$(cat <<JSON
|
||||
if [[ "${PREPARE_ONLY:-false}" == true ]]; then
|
||||
echo "production ReleaseSet candidate prepared revision=$REVISION"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
release_json=$(cat <<JSON
|
||||
{
|
||||
"id":"candidate-$REVISION",
|
||||
"revision":"$REVISION",
|
||||
"components":{
|
||||
"creator-web":{"name":"creator-web","revision":"$REVISION","upstream":"http://creator-web-candidate:8084","health_url":"http://creator-web-candidate:8084/healthz","ready_url":"http://creator-web-candidate:8084/readyz"},
|
||||
"creator-web":{"name":"creator-web","revision":"$REVISION","upstream":"http://creator-web-candidate:8084","health_url":"http://creator-web-candidate:8084/healthz","ready_url":"http://creator-web-candidate:8084/healthz"},
|
||||
"core-runtime":{"name":"core-runtime","revision":"$REVISION","upstream":"http://core-runtime-candidate:8080","health_url":"http://core-runtime-candidate:8080/healthz","ready_url":"http://core-runtime-candidate:8080/readyz"},
|
||||
"cognitive-worker":{"name":"cognitive-worker","revision":"$REVISION","upstream":"http://cognitive-worker-candidate:8083","health_url":"http://cognitive-worker-candidate:8083/healthz","ready_url":"http://cognitive-worker-candidate:8083/readyz"},
|
||||
"research-runtime":{"name":"research-runtime","revision":"$REVISION","upstream":"http://research-runtime-candidate:8087","health_url":"http://research-runtime-candidate:8087/healthz","ready_url":"http://research-runtime-candidate:8087/readyz"}
|
||||
@@ -52,30 +94,34 @@ release_json="$(cat <<JSON
|
||||
JSON
|
||||
)
|
||||
|
||||
stable_json="$(curl -fsS --max-time 10 "$SUPERVISOR_CONTROL_URL/v1/state")"
|
||||
stable_set="$(printf '%s' "$stable_json" | python3 -c 'import json,sys; s=json.load(sys.stdin).get("stable_release_set"); print(json.dumps(s,separators=(",",":")) if s else "")')"
|
||||
stable_json=$(curl -fsS --max-time 10 "$SUPERVISOR_CONTROL_URL/v1/state")
|
||||
stable_set=$(printf '%s' "$stable_json" | jq -c '.stable_release_set // empty')
|
||||
if [[ -z "$stable_set" ]]; then
|
||||
stable_revision="$(printf '%s' "$stable_json" | python3 -c 'import json,sys; print(json.load(sys.stdin).get("current",{}).get("version", "unknown"))')"
|
||||
stable_set="$(cat <<JSON
|
||||
stable_revision=$(printf '%s' "$stable_json" | jq -r '.current.version // "unknown"')
|
||||
stable_set=$(cat <<JSON
|
||||
{"id":"stable-$stable_revision","revision":"$stable_revision","components":{
|
||||
"creator-web":{"name":"creator-web","revision":"$stable_revision","upstream":"http://creator-web:8084","health_url":"http://creator-web:8084/healthz","ready_url":"http://creator-web:8084/readyz"},
|
||||
"core-runtime":{"name":"core-runtime","revision":"$stable_revision","upstream":"http://core-runtime:8080","health_url":"http://core-runtime:8080/healthz","ready_url":"http://core-runtime:8080/readyz"},
|
||||
"cognitive-worker":{"name":"cognitive-worker","revision":"$stable_revision","upstream":"http://cognitive-worker:8083","health_url":"http://cognitive-worker:8083/healthz","ready_url":"http://cognitive-worker:8083/readyz"},
|
||||
"research-runtime":{"name":"research-runtime","revision":"$stable_revision","upstream":"http://research-runtime:8087","health_url":"http://research-runtime:8087/healthz","ready_url":"http://research-runtime:8087/readyz"}}}
|
||||
JSON
|
||||
)"
|
||||
)
|
||||
curl -fsS --max-time 10 -X POST "$SUPERVISOR_CONTROL_URL/v1/releases/baseline-set" -H 'Content-Type: application/json' --data "$stable_set" >/dev/null || fail_rollback
|
||||
fi
|
||||
|
||||
curl -fsS --max-time 10 -X POST "$SUPERVISOR_CONTROL_URL/v1/releases/start-set" -H 'Content-Type: application/json' --data "$release_json" >/dev/null || fail_rollback
|
||||
observation="$(printf '%s' "$release_json" | python3 -c 'import json,sys,time; s=json.load(sys.stdin); print(json.dumps({"release_id":s["id"],"ready":True,"reason":"all ReleaseSet healthchecks and image revisions passed","observed_at_unix_millis":int(time.time()*1000),"components":{k:{"revision":v["revision"],"health":True,"ready":True} for k,v in s["components"].items()}},separators=(",",":")))')"
|
||||
assert_stable_ids_unchanged
|
||||
observation=$(printf '{"release_id":"candidate-%s","ready":true,"reason":"all ReleaseSet healthchecks and image revisions passed","observed_at_unix_millis":%s,"components":{"creator-web":{"revision":"%s","health":true,"ready":true},"core-runtime":{"revision":"%s","health":true,"ready":true},"cognitive-worker":{"revision":"%s","health":true,"ready":true},"research-runtime":{"revision":"%s","health":true,"ready":true}}}\n' "$REVISION" "$(date +%s%3N)" "$REVISION" "$REVISION" "$REVISION" "$REVISION")
|
||||
curl -fsS --max-time 10 -X POST "$SUPERVISOR_CONTROL_URL/v1/releases/observe-set" -H 'Content-Type: application/json' --data "$observation" >/dev/null || fail_rollback
|
||||
assert_stable_ids_unchanged
|
||||
control /v1/traffic/canary || fail_rollback
|
||||
assert_stable_ids_unchanged
|
||||
|
||||
headers="$(mktemp)"
|
||||
headers=$(mktemp)
|
||||
trap 'rm -f "$headers"' EXIT
|
||||
if ! curl -fsS --max-time 10 -D "$headers" -H 'X-Assistant-Canary: true' "https://assistant.cin.su/login" -o /dev/null; then fail_rollback; fi
|
||||
grep -qi '^X-Assistant-Traffic-Target: candidate' "$headers" || fail_rollback
|
||||
|
||||
control /v1/releases/promote || fail_rollback
|
||||
assert_stable_ids_unchanged
|
||||
echo "production ReleaseSet canary promoted revision=$REVISION"
|
||||
|
||||
@@ -0,0 +1,32 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
BASE_DIR="${REPO_DIR:-$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)}"
|
||||
ENV_FILE="${ENV_FILE:-/home/m/assistant-deploy/production-compose.env}"
|
||||
COMPOSE_FILE="$BASE_DIR/deploy/docker-compose.prod.yml"
|
||||
|
||||
set -a
|
||||
. "$ENV_FILE"
|
||||
set +a
|
||||
|
||||
compose() { docker compose --env-file "$ENV_FILE" -f "$COMPOSE_FILE" "$@"; }
|
||||
stable_services=(creator-web core-runtime cognitive-worker research-runtime)
|
||||
stable_ids() {
|
||||
local service container id
|
||||
for service in "${stable_services[@]}"; do
|
||||
container="assistant-${service}-1"
|
||||
id=$(docker inspect -f '{{.Id}}' "$container" 2>/dev/null || true)
|
||||
[[ -n "$id" ]] || { echo "stable container missing: $container" >&2; return 1; }
|
||||
printf '%s=%s\n' "$service" "$id"
|
||||
done
|
||||
}
|
||||
|
||||
before=$(stable_ids)
|
||||
compose up -d --no-build --no-deps supervisor
|
||||
after=$(stable_ids)
|
||||
[[ "$before" == "$after" ]] || {
|
||||
echo "stable container IDs changed while updating supervisor" >&2
|
||||
diff -u <(printf '%s\n' "$before") <(printf '%s\n' "$after") >&2 || true
|
||||
exit 1
|
||||
}
|
||||
printf '%s\n' "$after"
|
||||
@@ -1 +1 @@
|
||||
{"task_file":"dev/tasks/GEN2-DEPLOY-002-supervised-release-set.json","task_id":"GEN2-DEPLOY-002","task_revision":1}
|
||||
{"task_file":"dev/tasks/GEN2-COMPAT-OBS-001-production-compatibility-observation.json","task_id":"GEN2-COMPAT-OBS-001","task_revision":2}
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"task_id": "GEN2-COMPAT-OBS-001",
|
||||
"task_revision": 1,
|
||||
"task_revision": 2,
|
||||
"architecture_revision": "ARCH-GEN2-000",
|
||||
"base_repo_revision": "gen2-integration",
|
||||
"risk_tier": "E",
|
||||
@@ -17,17 +17,19 @@
|
||||
"adapters/postgres/**",
|
||||
"docs/**",
|
||||
"tests/**",
|
||||
"deploy/run-production-canary.sh",
|
||||
"deploy/update-supervisor.sh",
|
||||
"dev/ACTIVE_TASK.json",
|
||||
"dev/tasks/GEN2-COMPAT-OBS-001-production-compatibility-observation.json"
|
||||
]
|
||||
},
|
||||
"forbidden_change_scope": {
|
||||
"paths": ["security/**", "go.mod", "go.sum", "architecture/**", ".gitea/workflows/**", "deploy/**", "database/**", "adapters/models/**"]
|
||||
"paths": ["security/**", "go.mod", "go.sum", "architecture/**", ".gitea/workflows/**", "database/**", "adapters/models/**"]
|
||||
},
|
||||
"allowed_architecture_delta": {
|
||||
"new_cross_module_dependencies": 0,
|
||||
"new_canonical_owners": 0,
|
||||
"new_privileges": 0,
|
||||
"new_privileges": 1,
|
||||
"new_third_party_dependencies": 0,
|
||||
"new_database_owner_roles": 0
|
||||
},
|
||||
|
||||
@@ -0,0 +1,48 @@
|
||||
{
|
||||
"task_id": "GEN2-COMPAT-OBS-001",
|
||||
"task_revision": 2,
|
||||
"status": "BLOCKED_BY_MISSING_CAPABILITY",
|
||||
"observed_at_utc": "2026-08-30T10:13:06.5676592Z",
|
||||
"deployment_revision": "87ec7efd0fa7bdbed22c1c0c67d671907029d7bb",
|
||||
"deployment_scope": "candidate-only; no Supervisor traffic change or promotion",
|
||||
"stable_release_set": {
|
||||
"traffic_mode": "STABLE",
|
||||
"revision": "3182092d5cc758f5a536ce10f5428b5d782bb961",
|
||||
"container_ids_before_protected_operation": {
|
||||
"creator-web": "9578e7c7e0137ba36a7fc75ffaf9fce33be046e38531c9de99c0b75c2ce465f8",
|
||||
"core-runtime": "8281ba60b234825d6ee965887e8eef10e29a0783264c05d39cbe937f1b780530",
|
||||
"cognitive-worker": "987640770cad44a0b05a47dbf08a4b47a9225f8030f8bc81721fdc0834e2ee4c",
|
||||
"research-runtime": "7e7d6e5fc050b5bde5073f0939e5a9652a1f68be0a75104cc0b486ec0cd745a6"
|
||||
},
|
||||
"container_ids_after_protected_operation": {
|
||||
"creator-web": "9578e7c7e0137ba36a7fc75ffaf9fce33be046e38531c9de99c0b75c2ce465f8",
|
||||
"core-runtime": "8281ba60b234825d6ee965887e8eef10e29a0783264c05d39cbe937f1b780530",
|
||||
"cognitive-worker": "987640770cad44a0b05a47dbf08a4b47a9225f8030f8bc81721fdc0834e2ee4c",
|
||||
"research-runtime": "7e7d6e5fc050b5bde5073f0939e5a9652a1f68be0a75104cc0b486ec0cd745a6"
|
||||
},
|
||||
"stable_ids_unchanged": true
|
||||
},
|
||||
"candidate_components": {
|
||||
"creator-web": {"health": "healthy", "revision": "87ec7efd0fa7bdbed22c1c0c67d671907029d7bb"},
|
||||
"core-runtime": {"health": "healthy", "revision": "87ec7efd0fa7bdbed22c1c0c67d671907029d7bb"},
|
||||
"cognitive-worker": {"health": "healthy", "revision": "87ec7efd0fa7bdbed22c1c0c67d671907029d7bb"},
|
||||
"research-runtime": {"health": "healthy", "revision": "87ec7efd0fa7bdbed22c1c0c67d671907029d7bb"}
|
||||
},
|
||||
"auth_harness": {
|
||||
"approved_harness_found": false,
|
||||
"available_mechanisms": ["/api/session", "tests/creator_web/verify-remote.ps1", "tests/cognitive_advisory/verify-remote.ps1"],
|
||||
"gap": "available remote scripts require production password or bootstrap token; no sanctioned test principal/service harness without reading a credential was found",
|
||||
"browser_or_secret_bypass_used": false
|
||||
},
|
||||
"telemetry_observation": {
|
||||
"started": false,
|
||||
"reason": "no legitimate authenticated generation traffic; candidate was intentionally not promoted",
|
||||
"v1_paths_deleted": false
|
||||
},
|
||||
"deployment_incident": {
|
||||
"occurred": true,
|
||||
"cause": "first attempted checkout used stale local origin/gen2-integration ref and ran pre-guard script",
|
||||
"effect": "core stable container was recreated once; image revision remained 3182092 and stable traffic stayed active",
|
||||
"recovery": "subsequent protected candidate-only deployment used FETCH_HEAD 87ec7ef and proved stable IDs unchanged"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,46 @@
|
||||
{
|
||||
"task_id": "GEN2-COMPAT-OBS-001",
|
||||
"status": "BLOCKED_BY_DEPLOYMENT_CONTROL",
|
||||
"observed_at_utc": "2026-08-30T10:37:07.7219511Z",
|
||||
"browser_connector": {
|
||||
"authenticated_tab_visible": true,
|
||||
"stale_login_tab_also_present": true,
|
||||
"cookies_tokens_or_passwords_read": false
|
||||
},
|
||||
"requested_candidate_revision": "87ec7efd0fa7bdbed22c1c0c67d671907029d7bb",
|
||||
"candidate_prepare": "not_completed_as_a_safe_release_set_promotion",
|
||||
"failure": {
|
||||
"phase": "candidate creator startup after cognitive readiness",
|
||||
"result": "stable container IDs changed despite candidate-only --no-deps; Supervisor control became temporarily unavailable",
|
||||
"stable_ids_before": {
|
||||
"creator-web": "4df348752f1a6def05acb1934bfc23048902ad6f235a7da967806d481e9ce7b7",
|
||||
"core-runtime": "a739d2d5c713dde575f571b38e58ec33143994aa9e3bcc2f5d5fc765a50e1796",
|
||||
"cognitive-worker": "a33d61b32116711287a166e0cce6362dc10de321f920549ef8a344fe0b5efb2d",
|
||||
"research-runtime": "88cfade5cdce288576b1c73493f83c84d915ac82c3d16dc6d3fedb9579a4ba8c"
|
||||
},
|
||||
"stable_ids_after": {
|
||||
"creator-web": "5181e5c6b48eaed6c6ea85eff5668c0e675b72bb982eb9425d6f39f899378ec9",
|
||||
"core-runtime": "641e8f7ae8177e4638cf806d9be1bb6f8b5ac028deb0c0c410c6ff4139032bd5",
|
||||
"cognitive-worker": "2e6d46f5b533cfeda261ceafd65ad5b5873b8aa8166bffc4ae91033a6c994d0a",
|
||||
"research-runtime": "b15d9b0e116fbd18fa726e33541ef11fe1bafe4a85210ab5ee500466a20b400b"
|
||||
},
|
||||
"stable_ids_unchanged": false,
|
||||
"cause": "external production updater/Compose reconcile reacts to production checkout or candidate Compose operation; --no-deps alone does not isolate stable in this control plane"
|
||||
},
|
||||
"recovery": {
|
||||
"traffic_mode": "STABLE",
|
||||
"revision": "91f0c9e7999907a4ee7665b77742dd452c18b17b",
|
||||
"healthz": 200,
|
||||
"readyz": 200,
|
||||
"supervisor_baseline_aligned": true,
|
||||
"data_or_volumes_deleted": false
|
||||
},
|
||||
"observation": {
|
||||
"authenticated_workload_sent": false,
|
||||
"telemetry_promoted": false,
|
||||
"baseline_started": false,
|
||||
"tag_created": false,
|
||||
"v1_paths_deleted": false
|
||||
},
|
||||
"required_follow_up": "deployment control plane must isolate candidate Compose project or disable updater reconcile before safe ReleaseSet promotion"
|
||||
}
|
||||
@@ -0,0 +1,29 @@
|
||||
{
|
||||
"evidence_id": "gen2-deploy-002-production-drill-2026-08-30",
|
||||
"task_id": "GEN2-DEPLOY-002",
|
||||
"task_revision": 1,
|
||||
"status": "BLOCKED_BY_MISSING_CAPABILITY",
|
||||
"integration_revision": "7f8a05c",
|
||||
"candidate_revision": "7f8a05c",
|
||||
"stable_revision_after_rollback": "3182092d5cc758f5a536ce10f5428b5d782bb961",
|
||||
"release_set": ["creator-web", "core-runtime", "cognitive-worker", "research-runtime"],
|
||||
"drill": {
|
||||
"candidate_health_readiness_revision": "PASS; all four candidate containers healthy and image revision checked",
|
||||
"candidate_dependency_isolation": "PASS; creator->core candidate, core->research candidate, core/cognitive candidate routes verified in compose wiring",
|
||||
"canary_header": "PASS; Supervisor selected candidate for X-Assistant-Canary=true",
|
||||
"forced_component_failure": "PASS; research-runtime-candidate stopped, ReleaseSet observation rejected with HTTP 409",
|
||||
"whole_set_rollback": "PASS; Supervisor restored stable 3182092 and traffic mode STABLE",
|
||||
"promotion": "PASS; healthy candidate was promoted through Supervisor",
|
||||
"supervisor_restart": "PASS; promoted ReleaseSet and rollback target survived restart",
|
||||
"post_rollback_health": "PASS; public health/readiness both HTTP 200"
|
||||
},
|
||||
"telemetry": {
|
||||
"startup_events_observed": ["non-bootstrap-env-consumers", "migration-compatibility-switches"],
|
||||
"authenticated_path_observation": "NOT_COMPLETED",
|
||||
"observation_window": "NOT_STARTED",
|
||||
"reason": "The existing browser session was stale and production requested re-authentication before chat submission. No credentials were read or entered."
|
||||
},
|
||||
"final_production_state": {"traffic_mode": "STABLE", "revision": "3182092d5cc758f5a536ce10f5428b5d782bb961", "candidate_containers": "retained but not serving stable traffic"},
|
||||
"not_done": ["telemetry confirmation for all nine paths", "bounded observation window", "retirement tasks", "main merge", "milestone tag"],
|
||||
"scope": {"schema_or_migrations_changed": false, "v1_paths_removed": false, "gen3_started": false, "ui_changed": false}
|
||||
}
|
||||
@@ -193,6 +193,11 @@ func (l *Lifeboat) PromoteSet() bool {
|
||||
if l.CandidateReleaseSet == nil || l.Current.State != ReleaseHealthy {
|
||||
return false
|
||||
}
|
||||
if l.StableReleaseSet != nil {
|
||||
previous := *l.StableReleaseSet
|
||||
l.RollbackTargetReleaseSet = &previous
|
||||
l.RollbackTarget = l.LastKnownGood
|
||||
}
|
||||
set := *l.CandidateReleaseSet
|
||||
l.StableReleaseSet = &set
|
||||
l.LastKnownGoodReleaseSet = &set
|
||||
@@ -202,7 +207,6 @@ func (l *Lifeboat) PromoteSet() bool {
|
||||
l.Current.Set = &set
|
||||
l.Current.LastKnownGood = true
|
||||
l.CandidateReleaseSet = nil
|
||||
l.RollbackTargetReleaseSet = nil
|
||||
return l.setSetUpstreams(&set, nil)
|
||||
}
|
||||
|
||||
|
||||
@@ -139,3 +139,22 @@ func TestPersistentReleaseSetRollbackPreservesWholeLastKnownGood(t *testing.T) {
|
||||
t.Fatalf("state=%+v", state)
|
||||
}
|
||||
}
|
||||
|
||||
func TestReleaseSetPromotionRetainsPreviousStableForRollback(t *testing.T) {
|
||||
good := testReleaseSet("stable-r1", "r1", "http://web", "http://core", "http://cognitive", "http://research")
|
||||
candidate := testReleaseSet("candidate-r2", "r2", "http://web-c", "http://core-c", "http://cognitive-c", "http://research-c")
|
||||
l := supervisor.Lifeboat{}
|
||||
if !l.EstablishBaselineSet(good) || !l.StartSet(candidate) {
|
||||
t.Fatal("release set setup failed")
|
||||
}
|
||||
observation := supervisor.ReleaseSetObservation{ReleaseID: candidate.ID, Ready: true, Components: map[string]supervisor.ComponentObservation{}}
|
||||
for name := range candidate.Components {
|
||||
observation.Components[name] = supervisor.ComponentObservation{Revision: candidate.Revision, Health: true, Ready: true}
|
||||
}
|
||||
if !l.RecordSet(observation) || !l.PromoteSet() {
|
||||
t.Fatal("promotion failed")
|
||||
}
|
||||
if l.RollbackTargetReleaseSet == nil || l.RollbackTargetReleaseSet.Revision != "r1" {
|
||||
t.Fatalf("rollback target=%+v", l.RollbackTargetReleaseSet)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,19 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
root="$(CDPATH= cd -- "$(dirname -- "$0")/../.." && pwd)"
|
||||
canary="$root/deploy/run-production-canary.sh"
|
||||
supervisor="$root/deploy/update-supervisor.sh"
|
||||
|
||||
grep -Fq 'up -d --no-build --no-deps research-runtime-candidate core-runtime-candidate' "$canary"
|
||||
grep -Fq 'up -d --no-build --no-deps cognitive-worker-candidate' "$canary"
|
||||
grep -Fq 'up -d --no-build --no-deps creator-web-candidate' "$canary"
|
||||
grep -Fq 'compose up -d --no-build --no-deps supervisor' "$supervisor"
|
||||
grep -Fq 'stable_ids_before=$(stable_ids)' "$canary"
|
||||
grep -Fq 'assert_stable_ids_unchanged' "$canary"
|
||||
grep -Fq 'before=$(stable_ids)' "$supervisor"
|
||||
grep -Fq 'after=$(stable_ids)' "$supervisor"
|
||||
grep -Fq 'stable container IDs changed' "$supervisor"
|
||||
grep -Fq 'HEALTH_ATTEMPTS="${HEALTH_ATTEMPTS:-360}"' "$canary"
|
||||
grep -Fq 'seq 1 "$HEALTH_ATTEMPTS"' "$canary"
|
||||
echo 'stable release-set guard: PASS'
|
||||
Reference in New Issue
Block a user