From a957172d3cc27cfce0ff1cf732fd69a5ac42d7db Mon Sep 17 00:00:00 2001 From: Mikhail Date: Sun, 30 Aug 2026 09:53:50 +0300 Subject: [PATCH 01/24] governance: revise Gen2 M6 task scopes --- dev/ACTIVE_TASK.json | 2 +- ...v1-compatibility-usage-and-prove-retirement-readiness.json | 4 ++-- ...-029-retire-proven-v1-paths-without-deleting-evidence.json | 4 ++-- ...tance-documentation-baseline-and-next-evolution-queue.json | 4 ++-- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/dev/ACTIVE_TASK.json b/dev/ACTIVE_TASK.json index 82ce70e..8a83968 100644 --- a/dev/ACTIVE_TASK.json +++ b/dev/ACTIVE_TASK.json @@ -1 +1 @@ -{"task_file":"dev/tasks/GEN2-REM-CORE-M5-LOCAL-MEDIUM-001.json","task_id":"GEN2-REM-CORE-M5-LOCAL-MEDIUM-001","task_revision":1} +{"task_file":"dev/tasks/GEN2-028-inventory-v1-compatibility-usage-and-prove-retirement-readiness.json","task_id":"GEN2-028","task_revision":2} diff --git a/dev/tasks/GEN2-028-inventory-v1-compatibility-usage-and-prove-retirement-readiness.json b/dev/tasks/GEN2-028-inventory-v1-compatibility-usage-and-prove-retirement-readiness.json index e0704db..d4a545f 100644 --- a/dev/tasks/GEN2-028-inventory-v1-compatibility-usage-and-prove-retirement-readiness.json +++ b/dev/tasks/GEN2-028-inventory-v1-compatibility-usage-and-prove-retirement-readiness.json @@ -1,6 +1,6 @@ { "task_id": "GEN2-028", - "task_revision": 1, + "task_revision": 2, "architecture_revision": "ARCH-GEN2-000", "base_repo_revision": "gen2-integration", "risk_tier": "E", @@ -16,7 +16,7 @@ "modules/**", "contracts/**", "cmd/**", - "adapters/**", + "adapters/postgres/**", "runtime/**", "platform/**", "docs/**", diff --git a/dev/tasks/GEN2-029-retire-proven-v1-paths-without-deleting-evidence.json b/dev/tasks/GEN2-029-retire-proven-v1-paths-without-deleting-evidence.json index 3e8b25f..e978bc7 100644 --- a/dev/tasks/GEN2-029-retire-proven-v1-paths-without-deleting-evidence.json +++ b/dev/tasks/GEN2-029-retire-proven-v1-paths-without-deleting-evidence.json @@ -1,6 +1,6 @@ { "task_id": "GEN2-029", - "task_revision": 1, + "task_revision": 2, "architecture_revision": "ARCH-GEN2-000", "base_repo_revision": "gen2-integration", "risk_tier": "E", @@ -16,7 +16,7 @@ "modules/**", "contracts/**", "cmd/**", - "adapters/**", + "adapters/postgres/**", "runtime/**", "platform/**", "database/migrations/*.sql", diff --git a/dev/tasks/GEN2-030-final-gen2-acceptance-documentation-baseline-and-next-evolution-queue.json b/dev/tasks/GEN2-030-final-gen2-acceptance-documentation-baseline-and-next-evolution-queue.json index cfe4ec7..326f1d8 100644 --- a/dev/tasks/GEN2-030-final-gen2-acceptance-documentation-baseline-and-next-evolution-queue.json +++ b/dev/tasks/GEN2-030-final-gen2-acceptance-documentation-baseline-and-next-evolution-queue.json @@ -1,6 +1,6 @@ { "task_id": "GEN2-030", - "task_revision": 1, + "task_revision": 2, "architecture_revision": "ARCH-GEN2-000", "base_repo_revision": "gen2-integration", "risk_tier": "E", @@ -20,7 +20,7 @@ "modules/**", "contracts/**", "cmd/**", - "adapters/**", + "adapters/postgres/**", "runtime/**", "platform/**", "tests/**" From 031ba7666c6d27afa8e6c1e4c3471b6ec695eaa4 Mon Sep 17 00:00:00 2001 From: Mikhail Date: Sun, 30 Aug 2026 09:58:06 +0300 Subject: [PATCH 02/24] docs(gen2): inventory V1 compatibility retirement readiness --- ...v1-compatibility-inventory-2026-08-30.json | 98 +++++++++++++++++++ 1 file changed, 98 insertions(+) create mode 100644 docs/evidence/gen2-028-v1-compatibility-inventory-2026-08-30.json diff --git a/docs/evidence/gen2-028-v1-compatibility-inventory-2026-08-30.json b/docs/evidence/gen2-028-v1-compatibility-inventory-2026-08-30.json new file mode 100644 index 0000000..78d06aa --- /dev/null +++ b/docs/evidence/gen2-028-v1-compatibility-inventory-2026-08-30.json @@ -0,0 +1,98 @@ +{ + "evidence_id": "gen2-028-v1-compatibility-inventory-2026-08-30", + "task_id": "GEN2-028", + "task_revision": 2, + "source_revision": "93feb9bf2df8f2e2cfd4d0259d39c869b19cad08", + "scope": "static repository inventory plus read-only production container and Supervisor observations; no code, schema, or runtime mutation", + "canonical_writer_rule": "unchanged; no second Memory owner and no writer cutover performed", + "production_observation": { + "deployment_revision": "3182092d5cc758f5a536ce10f5428b5d782bb961", + "core_container": "running, restart_count=0; /healthz=200", + "cognitive_worker": "running, restart_count=0, health=healthy", + "lifeboat": "running, restart_count=0, health=healthy", + "supervisor": "running, restart_count=0, health=healthy", + "traffic_mode": "CANDIDATE", + "last_known_good": "3182092d5cc758f5a536ce10f5428b5d782bb961", + "legacy_named_runtime_containers": "none observed" + }, + "inventory": [ + { + "id": "personal-memory-context-text", + "paths": ["modules/interaction/domain/personal_memory.go", "modules/interaction/domain/gen2_context.go", "cmd/core-runtime/main.go"], + "role": "V1 rebuildable context projection and deterministic preference recall", + "runtime_status": "ACTIVE_COMPATIBILITY_PATH", + "evidence": ["BuildScopedGen2Context is called from the production chat handler", "CompilePersonalMemory and ContextText remain in that call path", "empty Gen2 projection returns personal-memory-v1-fallback", "targeted gen2_context and personal_memory tests exist"], + "retirement": "KEEP_LONGER", + "rollback": "retain the existing read fallback and revert the reader switch without deleting utterance history" + }, + { + "id": "active-project-focus", + "paths": ["adapters/postgres/situation_repository.go", "cmd/core-runtime/main.go", "contracts/situation/types.go"], + "role": "V1 project-centric focus compatibility projection", + "runtime_status": "ACTIVE_READ_AND_WRITE_COMPATIBILITY_PATH", + "evidence": ["core chat loads ActiveProjectFocus for each authenticated message", "core persists it after semantic interpretation", "PostgreSQL adapter reads and upserts situation.active_project_focus", "Focus V2 projection exists but does not remove these callers"], + "retirement": "KEEP_LONGER", + "rollback": "keep the projection readable and restore the prior focus reader if a V2 reader is rolled back" + }, + { + "id": "truth-bundles", + "paths": ["adapters/postgres/epistemics_repository.go", "modules/epistemics/application/persisted_research.go", "database/migrations/034_epistemics_truth_bundles.sql", "tests/research_contracts/truth_persistence_sql_test.go"], + "role": "V1 denormalized epistemic compatibility/status and research persistence", + "runtime_status": "ACTIVE_WRITER_AND_READ_COMPATIBILITY_PATH", + "evidence": ["SaveTruthBundle remains an application repository contract used by persisted research", "TruthRuntimeStatus reads counts and grouped status from epistemics.truth_bundles", "migration 034 and its contract test are retained", "no production evidence permits declaring the reader unused"], + "retirement": "KEEP_LONGER", + "rollback": "retain table and reader; revert normalized reader selection without deleting historical bundles" + }, + { + "id": "ordinary-env-configuration", + "paths": ["cmd/core-runtime/main.go", "cmd/work-graph-runtime/main.go", "cmd/reflection-worker/main.go", "cmd/notification-receiver/main.go"], + "role": "ENV configuration and secret/bootstrap resolution", + "runtime_status": "ACTIVE_MIXED_BOOTSTRAP_AND_NON_BOOTSTRAP_READS", + "evidence": ["production composition supplies runtime settings through process configuration", "source contains direct os.Getenv reads for listener, database, provider and runtime settings", "secret values were not read or emitted", "no complete production observation proves every non-bootstrap consumer is unused"], + "retirement": "KEEP_LONGER", + "rollback": "restore the prior ENV fallback for a single setting while Configuration remains authoritative" + }, + { + "id": "skill-profile-v1", + "paths": ["adapters/postgres/capability_repository.go", "cmd/core-runtime/learning.go", "contracts/capability/resolution.go"], + "role": "V1 SkillProfile storage/read compatibility and FastPath learning", + "runtime_status": "ACTIVE_READ_AND_WRITE_PATH", + "evidence": ["core exposes RecentSkillProfiles and records FastPath decline learning", "learning.go loads and saves SkillProfile", "PostgreSQL adapter writes and joins capability.skill_profiles", "explicit Skill/Reflex types exist but do not prove these callers are dead"], + "retirement": "KEEP_LONGER", + "rollback": "retain SkillProfile rows and reader; revert Skill V2 selection without data deletion" + }, + { + "id": "implicit-fast-path", + "paths": ["modules/capability/domain/skill.go", "modules/capability/domain/reflex.go", "cmd/core-runtime/learning.go"], + "role": "V1 FastPath behavior and fallback-to-thinking compatibility", + "runtime_status": "ACTIVE_COMPATIBILITY_PATH", + "evidence": ["FastPath decisions are implemented and tested", "core records FastPath decline experiences", "ReflexFallback explicitly maps escalation to skill->reasoning", "no runtime telemetry separates all FastPath invocations from explicit Reflex use"], + "retirement": "KEEP_LONGER", + "rollback": "restore FastPath decline/escalation behavior; preserve evidence and Skill/Reflex revisions" + }, + { + "id": "migration-feature-switches", + "paths": ["development/GEN2_MIGRATION_MAP.md", "runtime/**", "modules/**"], + "role": "documented migration switch family and compatibility strategy", + "runtime_status": "UNKNOWN_NO_DEDICATED_RUNTIME_TELEMETRY_FOUND", + "evidence": ["migration map defines switch names such as Focus, Context, Epistemics, Config and Skill strategies", "repository scan found no complete operational switch registry/telemetry proving zero production use", "test-only legacy mode references are not sufficient production evidence"], + "retirement": "KEEP_LONGER", + "rollback": "do not remove a switch until production observation identifies its callers and a reversible disable path" + } + ], + "coverage": { + "searched": ["modules", "contracts", "cmd", "adapters", "runtime", "platform", "tests", "docs", "database/migrations"], + "terms": ["PersonalMemory", "CompilePersonalMemory", "ContextText", "ActiveProjectFocus", "truth_bundles", "skill_profiles", "FastPath", "os.Getenv", "LookupEnv", "compatibility", "legacy", "feature switch"], + "limitations": ["No secret values were inspected", "Container logs and Supervisor state provide no per-path legacy usage counters", "A code reference is evidence of availability/callability, not a claim that every branch executed in the observation window"] + }, + "retirement_list": [ + {"path": "personal-memory-context-text", "decision": "KEEP_LONGER", "reason": "active production chat fallback and no usage counter"}, + {"path": "active-project-focus", "decision": "KEEP_LONGER", "reason": "core read/write callers remain active"}, + {"path": "truth-bundles", "decision": "KEEP_LONGER", "reason": "research writer and status reader remain active"}, + {"path": "ordinary-env-configuration", "decision": "KEEP_LONGER", "reason": "mixed bootstrap/non-bootstrap consumers remain"}, + {"path": "skill-profile-v1", "decision": "KEEP_LONGER", "reason": "core learning and repository read/write callers remain active"}, + {"path": "implicit-fast-path", "decision": "KEEP_LONGER", "reason": "active learning/fallback path and no complete production telemetry"}, + {"path": "migration-feature-switches", "decision": "KEEP_LONGER", "reason": "no complete runtime observation evidence"} + ], + "conclusion": "GEN2-028 EXIT: every inventoried legacy surface is classified ACTIVE_COMPATIBILITY_PATH, ACTIVE_READ_AND_WRITE_COMPATIBILITY_PATH, or UNKNOWN_NO_DEDICATED_RUNTIME_TELEMETRY_FOUND. No surface is SAFE_TO_RETIRE on current evidence; GEN2-029 must not perform irreversible deletion or disablement based on this inventory." +} From d64080ef2bd9c1270c5569ccf32509fa7aad49e4 Mon Sep 17 00:00:00 2001 From: Mikhail Date: Sun, 30 Aug 2026 09:59:42 +0300 Subject: [PATCH 03/24] docs(gen2): record V1 retirement decision --- dev/ACTIVE_TASK.json | 2 +- ...029-v1-retirement-decision-2026-08-30.json | 30 +++++++++++++++++++ 2 files changed, 31 insertions(+), 1 deletion(-) create mode 100644 docs/evidence/gen2-029-v1-retirement-decision-2026-08-30.json diff --git a/dev/ACTIVE_TASK.json b/dev/ACTIVE_TASK.json index 8a83968..4a1f367 100644 --- a/dev/ACTIVE_TASK.json +++ b/dev/ACTIVE_TASK.json @@ -1 +1 @@ -{"task_file":"dev/tasks/GEN2-028-inventory-v1-compatibility-usage-and-prove-retirement-readiness.json","task_id":"GEN2-028","task_revision":2} +{"task_file":"dev/tasks/GEN2-029-retire-proven-v1-paths-without-deleting-evidence.json","task_id":"GEN2-029","task_revision":2} diff --git a/docs/evidence/gen2-029-v1-retirement-decision-2026-08-30.json b/docs/evidence/gen2-029-v1-retirement-decision-2026-08-30.json new file mode 100644 index 0000000..9ada457 --- /dev/null +++ b/docs/evidence/gen2-029-v1-retirement-decision-2026-08-30.json @@ -0,0 +1,30 @@ +{ + "evidence_id": "gen2-029-v1-retirement-decision-2026-08-30", + "task_id": "GEN2-029", + "task_revision": 2, + "source_revision": "59d32143bf4ee7b4a78cb714ec0cbd76d4d8bdf5", + "decision": "NO_IRREVERSIBLE_RETIREMENT", + "reason": "GEN2-028 found no SAFE_TO_RETIRE path. Active callers remain for PersonalMemory fallback, ActiveProjectFocus, truth_bundles, ordinary ENV reads, SkillProfile/FastPath; migration switches lack complete production telemetry.", + "changed_runtime_paths": [], + "changed_schema_paths": [], + "retired_paths": [], + "preserved": [ + "human utterances and interaction history", + "epistemic observations, claims, experiences, truth_bundles and audit lineage", + "ActiveProjectFocus compatibility projection", + "PersonalMemory read fallback", + "SkillProfile/FastPath compatibility", + "ordinary ENV fallback consumers until bootstrap/config observation is complete", + "all existing Gen2 canonical writers and owner boundaries" + ], + "full_retirement_gate": { + "status": "NOT_APPLICABLE", + "reason": "No irreversible V1 schema/data deletion or runtime disablement was authorized or performed; the execution contract requires the gate immediately before such a retirement.", + "repository_full_test": "NOT_RUN_BY_DESIGN; no retirement change risk to justify expanding beyond the task's FAST/no-op decision" + }, + "rollback": { + "point": "59d32143bf4ee7b4a78cb714ec0cbd76d4d8bdf5", + "method": "keep current compatibility paths and revert this evidence-only decision; no schema rollback or data deletion needed" + }, + "conclusion": "No unsafe V1 path was removed. Historical evidence remains available, and there is no new dual writer. Retirement remains pending an observation window that can prove each path unused or a future approved cutover decision." +} From 86aacea99e305332c42f2d3d93746f5eb0dae0d8 Mon Sep 17 00:00:00 2001 From: Mikhail Date: Sun, 30 Aug 2026 10:02:23 +0300 Subject: [PATCH 04/24] docs(gen2): record final acceptance and retirement gap --- dev/ACTIVE_TASK.json | 2 +- docs/PROJECT_STATUS.md | 11 +++++ .../gen2-030-final-acceptance-2026-08-30.json | 43 +++++++++++++++++++ 3 files changed, 55 insertions(+), 1 deletion(-) create mode 100644 docs/evidence/gen2-030-final-acceptance-2026-08-30.json diff --git a/dev/ACTIVE_TASK.json b/dev/ACTIVE_TASK.json index 4a1f367..882fc61 100644 --- a/dev/ACTIVE_TASK.json +++ b/dev/ACTIVE_TASK.json @@ -1 +1 @@ -{"task_file":"dev/tasks/GEN2-029-retire-proven-v1-paths-without-deleting-evidence.json","task_id":"GEN2-029","task_revision":2} +{"task_file":"dev/tasks/GEN2-030-final-gen2-acceptance-documentation-baseline-and-next-evolution-queue.json","task_id":"GEN2-030","task_revision":2} diff --git a/docs/PROJECT_STATUS.md b/docs/PROJECT_STATUS.md index 19ba925..b32e6fe 100644 --- a/docs/PROJECT_STATUS.md +++ b/docs/PROJECT_STATUS.md @@ -1,5 +1,16 @@ # Assistant project status +## Gen2 M6 checkpoint — 2026-08-30 + +- `GEN2-028 = COMPLETED_WITH_EVIDENCE`: V1 compatibility inventory is recorded in `docs/evidence/gen2-028-v1-compatibility-inventory-2026-08-30.json`. Current production callers remain for PersonalMemory fallback, ActiveProjectFocus, truth_bundles, ordinary ENV reads, SkillProfile/FastPath; no path was proven `SAFE_TO_RETIRE`. +- `GEN2-029 = COMPLETED_WITH_SAFE_NOOP`: no V1 runtime path or historical evidence was deleted or disabled. The decision and rollback point are recorded in `docs/evidence/gen2-029-v1-retirement-decision-2026-08-30.json`; `FULL_RETIREMENT` was not applicable because no irreversible retirement was performed. +- `GEN2-030_STATUS = ACCEPTANCE_IN_PROGRESS`: this status is a documentation checkpoint, not a final Gen2 PASS. The final gate must confirm current production and repository evidence; the M6 retirement gap remains that active V1 compatibility callers have not yet been proven unused. +- `UI_BROWSER_ACCEPTANCE = DEFERRED_BY_OWNER`; it is intentionally excluded from core/M6 acceptance and is neither counted as PASS nor as a blocker. +- `PROVIDER_GAPS = CPU_FAST_UNAVAILABLE; GPU_UNAVAILABLE`; Local MEDIUM remains the primary route and Lifeboat remains fallback per existing M5 evidence. +- `GEN2_M6_FINAL_STATUS = NOT_FINAL_PASS_RETIREMENT_GAP_REMAINS`; FULL_FINAL technical checks passed, but no final M6 baseline tag is created because active V1 compatibility callers were not proven safe to retire. +- `GEN2_M6_FINAL_EVIDENCE = docs/evidence/gen2-030-final-acceptance-2026-08-30.json`. +- `NEXT_EVOLUTION_QUEUE = instrument bounded compatibility usage, observe, then retire only individually proven paths under a future FULL_RETIREMENT gate; no Gen3 or new feature work is started. + ## Current audit delta — 2026-08-24 - `CURRENT_MODEL_SCOPE = LOCAL_FIRST_V1`; one locally available Qwen3-8B or Gemma-4 conversational model is sufficient for the first stage, with the local Lifeboat as recovery fallback. External models remain supported as optional capability, but their absence must not block work: reflexes, skills and local executors provide degraded continuation. External/GPU qualification and promotion are intentionally deferred by DEV-000270. diff --git a/docs/evidence/gen2-030-final-acceptance-2026-08-30.json b/docs/evidence/gen2-030-final-acceptance-2026-08-30.json new file mode 100644 index 0000000..343dbb6 --- /dev/null +++ b/docs/evidence/gen2-030-final-acceptance-2026-08-30.json @@ -0,0 +1,43 @@ +{ + "evidence_id": "gen2-030-final-acceptance-2026-08-30", + "task_id": "GEN2-030", + "task_revision": 2, + "source_revision": "2a8e4e1b81226a1acedfd746ebc2aeb0ac11158f", + "status": "NOT_FINAL_PASS_RETIREMENT_GAP_REMAINS", + "full_final_gate": { + "go_test_all": "PASS", + "go_vet_all": "PASS", + "architecture_checks": "PASS", + "governance_preflight_check_diff_independent_whitespace": "PASS", + "migration_consistency": "PASS; 52 files, unique range 001..052; no M6 schema changes", + "production_core_smoke": "PASS; core /healthz=200, /readyz=200; cognitive-worker healthy; Lifeboat healthy", + "supervisor_recovery_readiness": "PASS; current and last-known-good 3182092d5cc758f5a536ce10f5428b5d782bb961; rollback target retained" + }, + "architecture_reconciliation": { + "registry": "YAML/JSON mirrors agree; Gen2 owners and runtime topology are represented", + "canonical_memory_owner": "none; retrieval/context remain projection fabric", + "schema": "migrations 046..052 are present and numbered without duplicates; historical 001..045 retained", + "production_revision": "3182092d5cc758f5a536ce10f5428b5d782bb961" + }, + "m6_result": { + "gen2_028": "completed inventory; no SAFE_TO_RETIRE path proven", + "gen2_029": "completed safe no-op; no runtime/schema/data deletion", + "gen2_030": "full final verification passed, but final M6 exit is not signed", + "full_retirement": "NOT_APPLICABLE; no irreversible retirement was performed", + "final_tag": "NOT_CREATED; acceptance is not a final PASS" + }, + "ui_browser": "DEFERRED_BY_OWNER; intentionally excluded, neither PASS nor blocker", + "provider_gaps": ["CPU_FAST unavailable", "GPU unavailable"], + "remaining_gaps": [ + "Per-path production observation telemetry is missing for migration switches and several compatibility readers", + "PersonalMemory, ActiveProjectFocus, truth_bundles, ordinary ENV, SkillProfile/FastPath still have active callers and require a future evidence-backed retirement/cutover task", + "M6 cannot claim complete V1 retirement until those paths are proven unused or explicitly retained by an accepted architecture decision" + ], + "next_evolution_queue": [ + "Instrument bounded, redacted usage observations for each remaining compatibility path", + "Run an approved observation window and re-evaluate retirement candidates", + "Retire only individually proven paths with FULL_RETIREMENT immediately before irreversible change", + "Keep UI/browser acceptance deferred by owner", + "Keep CPU_FAST/GPU provider qualification separate from Core acceptance" + ] +} From 3efcbbded8f677d6b5ae3898f697400a24ff5de1 Mon Sep 17 00:00:00 2001 From: Mikhail Date: Sun, 30 Aug 2026 10:25:35 +0300 Subject: [PATCH 05/24] feat(telemetry): observe V1 compatibility paths safely --- adapters/postgres/epistemics_repository.go | 19 +++++ cmd/core-runtime/learning.go | 6 ++ cmd/core-runtime/main.go | 12 +++ dev/ACTIVE_TASK.json | 2 +- ...-production-compatibility-observation.json | 63 +++++++++++++++ platform/telemetry/compatibility.go | 77 +++++++++++++++++++ platform/telemetry/compatibility_test.go | 20 +++++ 7 files changed, 198 insertions(+), 1 deletion(-) create mode 100644 dev/tasks/GEN2-COMPAT-OBS-001-production-compatibility-observation.json create mode 100644 platform/telemetry/compatibility.go create mode 100644 platform/telemetry/compatibility_test.go diff --git a/adapters/postgres/epistemics_repository.go b/adapters/postgres/epistemics_repository.go index 3f1a9bf..2f9c0fd 100644 --- a/adapters/postgres/epistemics_repository.go +++ b/adapters/postgres/epistemics_repository.go @@ -6,6 +6,7 @@ import ( "time" "git.cin.su/m/Assistant/contracts/epistemics" + "git.cin.su/m/Assistant/platform/telemetry" "github.com/jackc/pgx/v5/pgxpool" ) @@ -36,6 +37,11 @@ func (r *EpistemicsRepository) Ping(ctx context.Context) error { // TruthRuntimeStatus is a read-only projection of durable truth bundles. // It does not promote claims, revise beliefs, or mutate epistemic state. func (r *EpistemicsRepository) TruthRuntimeStatus(ctx context.Context) (TruthRuntimeStatus, error) { + started := time.Now() + outcome := "SUCCESS" + defer func() { + telemetry.RecordCompatibility("truth-bundles-status-reader", "epistemics-repository", "read", "research-status", true, false, true, outcome, "truth-status", "epistemics", started) + }() status := TruthRuntimeStatus{ ClaimsByVerification: map[string]int64{}, BeliefsByState: map[string]int64{}, @@ -43,6 +49,7 @@ func (r *EpistemicsRepository) TruthRuntimeStatus(ctx context.Context) (TruthRun ObservedAtUnixMillis: time.Now().UnixMilli(), } if err := r.pool.QueryRow(ctx, `SELECT count(*) FROM epistemics.truth_bundles`).Scan(&status.BundleCount); err != nil { + outcome = "FAILURE" return TruthRuntimeStatus{}, err } queries := []struct { @@ -56,18 +63,21 @@ func (r *EpistemicsRepository) TruthRuntimeStatus(ctx context.Context) (TruthRun for _, item := range queries { rows, err := r.pool.Query(ctx, item.query) if err != nil { + outcome = "FAILURE" return TruthRuntimeStatus{}, err } for rows.Next() { var state string var count int64 if err := rows.Scan(&state, &count); err != nil { + outcome = "FAILURE" rows.Close() return TruthRuntimeStatus{}, err } item.destination[state] = count } if err := rows.Err(); err != nil { + outcome = "FAILURE" rows.Close() return TruthRuntimeStatus{}, err } @@ -218,7 +228,13 @@ func (r *EpistemicsRepository) SaveReflectionCandidate(ctx context.Context, valu } func (r *EpistemicsRepository) SaveTruthBundle(ctx context.Context, value epistemics.TruthBundle, recordedAtUnixMillis int64) error { + started := time.Now() + outcome := "SUCCESS" + defer func() { + telemetry.RecordCompatibility("truth-bundles-writer", "epistemics-repository", "write", "persisted-research", true, false, true, outcome, string(value.Claim.ID), "epistemics", started) + }() if value.Source.ID == "" || value.Snapshot.ID == "" || value.Excerpt.ID == "" || value.Claim.ID == "" || value.Belief.ID == "" || value.Citation.ID == "" || recordedAtUnixMillis < 1 { + outcome = "INVALID" return fmt.Errorf("invalid truth bundle") } _, err := r.pool.Exec(ctx, `INSERT INTO epistemics.truth_bundles ( @@ -232,5 +248,8 @@ func (r *EpistemicsRepository) SaveTruthBundle(ctx context.Context, value episte value.Excerpt.ID, value.Excerpt.SnapshotID, value.Excerpt.Text, value.Excerpt.Start, value.Excerpt.End, value.Observation.ID, value.Observation.ObservedAt, value.Claim.ID, value.Claim.Statement, value.Claim.ObservationID, value.Claim.CitationID, value.Claim.Verification, value.Belief.ID, value.Belief.ClaimID, value.Belief.State, value.Citation.ID, value.Citation.ExcerptID, value.Citation.Evidence, value.Citation.Coverage, value.Citation.Conflict, value.Citation.ResolvedAtUnixMillis, recordedAtUnixMillis) + if err != nil { + outcome = "FAILURE" + } return err } diff --git a/cmd/core-runtime/learning.go b/cmd/core-runtime/learning.go index d53398b..25e762b 100644 --- a/cmd/core-runtime/learning.go +++ b/cmd/core-runtime/learning.go @@ -13,6 +13,7 @@ import ( "git.cin.su/m/Assistant/contracts/interaction" capabilitydomain "git.cin.su/m/Assistant/modules/capability/domain" epistemicsdomain "git.cin.su/m/Assistant/modules/epistemics/domain" + "git.cin.su/m/Assistant/platform/telemetry" ) const reminderResponseSkillID = capability.SkillProfileID("skill-obvious-reminder-response-v1") @@ -160,6 +161,7 @@ func (r learningRecorder) recordFastPathDecline(ctx context.Context, input fastP if r.epistemics == nil || input.Intent.ID == "" || input.Utterance.ID == "" { return } + telemetry.RecordCompatibility("fast-path-v1", "core-runtime", "decision", "decline-or-escalate", true, true, false, "OBSERVED", string(input.Utterance.CorrelationID), string(input.Utterance.PrincipalID), input.At) experience, err := epistemicsdomain.FastPathDeclineExperience( epistemics.OriginalIntentRef(input.Intent.ID), epistemics.SourceUtteranceRef(input.Utterance.ID), @@ -273,9 +275,11 @@ func (r learningRecorder) updateReminderSkill(ctx context.Context, experience ep } profile, err := r.capability.LoadSkillProfile(ctx, reminderResponseSkillID) if err != nil { + telemetry.RecordCompatibility("skill-profile-v1", "core-runtime", "read", "fast-path-learning", true, false, true, "FAILURE", string(experience.OriginalIntentID), string(experience.SourceUtteranceID), time.UnixMilli(experience.RecordedAtUnixMillis)) log.Printf("learning skill profile unavailable: %v", err) return } + telemetry.RecordCompatibility("skill-profile-v1", "core-runtime", "read", "fast-path-learning", true, false, true, "SUCCESS", string(experience.OriginalIntentID), string(experience.SourceUtteranceID), time.UnixMilli(experience.RecordedAtUnixMillis)) next, err := capabilitydomain.ObserveSkillOutcome(profile, capability.SkillOutcomeObservation{ ExperienceRef: capability.SkillEvidenceRef("epistemics:experience:" + string(experience.ID)), Outcome: outcome, @@ -287,9 +291,11 @@ func (r learningRecorder) updateReminderSkill(ctx context.Context, experience ep return } if err := r.capability.SaveSkillProfile(ctx, next); err != nil { + telemetry.RecordCompatibility("skill-profile-v1", "core-runtime", "write", "fast-path-learning", true, false, true, "FAILURE", string(experience.OriginalIntentID), string(experience.SourceUtteranceID), time.UnixMilli(experience.RecordedAtUnixMillis)) log.Printf("learning skill profile save unavailable: %v", err) return } + telemetry.RecordCompatibility("skill-profile-v1", "core-runtime", "write", "fast-path-learning", true, false, true, "SUCCESS", string(experience.OriginalIntentID), string(experience.SourceUtteranceID), time.UnixMilli(experience.RecordedAtUnixMillis)) } func outcomeForHumanResponse(response continuity.HumanResponse) (epistemics.OutcomeClass, int64) { diff --git a/cmd/core-runtime/main.go b/cmd/core-runtime/main.go index 5e006ce..8f371e5 100644 --- a/cmd/core-runtime/main.go +++ b/cmd/core-runtime/main.go @@ -24,6 +24,7 @@ import ( workdomain "git.cin.su/m/Assistant/modules/work/domain" "git.cin.su/m/Assistant/platform/ids" platformsecrets "git.cin.su/m/Assistant/platform/secrets" + "git.cin.su/m/Assistant/platform/telemetry" "git.cin.su/m/Assistant/runtime/health" "github.com/jackc/pgx/v5" "io" @@ -65,6 +66,8 @@ func main() { if creatorUsername == "" { log.Fatal("CREATOR_USERNAME is required") } + telemetry.RecordCompatibility("non-bootstrap-env-consumers", "core-runtime", "read", "runtime-startup-configuration", true, true, true, "OBSERVED", "startup", "core-runtime", time.Now()) + telemetry.RecordCompatibility("migration-compatibility-switches", "core-runtime", "observe", "runtime-startup-switch-scan", true, false, false, "NOT_OBSERVED", "startup", "core-runtime", time.Now()) repository, err := postgres.OpenSituation(context.Background(), postgres.ContinuityConfig{Host: os.Getenv("POSTGRES_HOST"), Port: os.Getenv("POSTGRES_PORT"), Database: os.Getenv("POSTGRES_DATABASE"), User: os.Getenv("POSTGRES_USER"), Password: os.Getenv("POSTGRES_PASSWORD")}) if err != nil { log.Fatal(err) @@ -553,9 +556,13 @@ func main() { storedFocus, err := repository.ActiveProjectFocus(r.Context(), session.ActorID, session.PrincipalID) if err == nil { previousFocus = &storedFocus + telemetry.RecordCompatibility("active-project-focus-read", "core-runtime", "read", "chat-focus-load", true, false, true, "SUCCESS", utterance.CorrelationID, string(session.PrincipalID), now) } else if !errors.Is(err, pgx.ErrNoRows) { + telemetry.RecordCompatibility("active-project-focus-read", "core-runtime", "read", "chat-focus-load", true, false, true, "FAILURE", utterance.CorrelationID, string(session.PrincipalID), now) http.Error(w, "project focus unavailable", http.StatusServiceUnavailable) return + } else { + telemetry.RecordCompatibility("active-project-focus-read", "core-runtime", "read", "chat-focus-load", true, false, true, "EMPTY", utterance.CorrelationID, string(session.PrincipalID), now) } var effectiveTaskMessage = interactiondomain.StripAssistantVocative(utterance.Content) creativeUserRequest := interactiondomain.IsCreativeRequest(utterance.Content) @@ -582,6 +589,9 @@ func main() { } conversationHistory = append(conversationHistory, recentUserHistory...) gen2Context := interactiondomain.BuildScopedGen2Context(conversationHistory, string(session.PrincipalID)) + if gen2Context.UsedPersonalMemoryFallback { + telemetry.RecordCompatibility("personal-memory-fallback", "core-runtime", "read", "gen2-context-empty", true, true, false, "USED", utterance.CorrelationID, string(session.PrincipalID), now) + } retrievedMemoryReply := interactiondomain.RecallPersonalPreference(conversationHistory, effectiveTaskMessage) stableConversationalReply, stableConversational := interactiondomain.StableConversationalReply( effectiveTaskMessage, @@ -780,9 +790,11 @@ func main() { activeFocusProvenance := situation.ProjectResolutionProvenance("") if semantic.ActiveFocus != nil { if err := repository.SaveActiveProjectFocus(r.Context(), *semantic.ActiveFocus); err != nil { + telemetry.RecordCompatibility("active-project-focus-write", "core-runtime", "write", "semantic-focus-update", true, false, true, "FAILURE", utterance.CorrelationID, string(session.PrincipalID), now) http.Error(w, "save project focus", http.StatusInternalServerError) return } + telemetry.RecordCompatibility("active-project-focus-write", "core-runtime", "write", "semantic-focus-update", true, false, true, "SUCCESS", utterance.CorrelationID, string(session.PrincipalID), now) activeFocusName = semantic.ActiveFocus.ProjectDisplayName activeFocusProvenance = semantic.ActiveFocus.Provenance } else if len(semantic.Projects) > 0 && semantic.Projects[0].Provenance == situation.ProjectResolutionInheritedFocus && previousFocus != nil { diff --git a/dev/ACTIVE_TASK.json b/dev/ACTIVE_TASK.json index 882fc61..b9e68cc 100644 --- a/dev/ACTIVE_TASK.json +++ b/dev/ACTIVE_TASK.json @@ -1 +1 @@ -{"task_file":"dev/tasks/GEN2-030-final-gen2-acceptance-documentation-baseline-and-next-evolution-queue.json","task_id":"GEN2-030","task_revision":2} +{"task_file":"dev/tasks/GEN2-COMPAT-OBS-001-production-compatibility-observation.json","task_id":"GEN2-COMPAT-OBS-001","task_revision":1} diff --git a/dev/tasks/GEN2-COMPAT-OBS-001-production-compatibility-observation.json b/dev/tasks/GEN2-COMPAT-OBS-001-production-compatibility-observation.json new file mode 100644 index 0000000..452b41f --- /dev/null +++ b/dev/tasks/GEN2-COMPAT-OBS-001-production-compatibility-observation.json @@ -0,0 +1,63 @@ +{ + "task_id": "GEN2-COMPAT-OBS-001", + "task_revision": 1, + "architecture_revision": "ARCH-GEN2-000", + "base_repo_revision": "gen2-integration", + "risk_tier": "E", + "intent": "Добавить bounded redacted production telemetry для оставшихся V1 compatibility paths без изменения их поведения, owners или canonical storage.", + "requirement_ids": ["REQ-GEN2-COMPAT-OBS-001-TELEMETRY", "REQ-GEN2-COMPAT-OBS-001-OBSERVATION", "REQ-GEN2-COMPAT-OBS-001-ROLLBACK"], + "allowed_change_scope": { + "modules": [], + "paths": [ + "platform/telemetry/**", + "cmd/core-runtime/**", + "cmd/research-runtime/**", + "modules/interaction/domain/**", + "modules/capability/domain/**", + "adapters/postgres/**", + "docs/**", + "tests/**", + "dev/ACTIVE_TASK.json", + "dev/tasks/GEN2-COMPAT-OBS-001-production-compatibility-observation.json" + ] + }, + "forbidden_change_scope": { + "paths": ["security/**", "go.mod", "go.sum", "architecture/**", ".gitea/workflows/**", "deploy/**", "database/**", "adapters/models/**"] + }, + "allowed_architecture_delta": { + "new_cross_module_dependencies": 0, + "new_canonical_owners": 0, + "new_privileges": 0, + "new_third_party_dependencies": 0, + "new_database_owner_roles": 0 + }, + "contracts": { + "consume": ["existing runtime logs and capability usage observation mechanisms"], + "may_change": ["telemetry event shape only; no semantic/domain contract change"] + }, + "semantic_owner": "Migration/Governance; telemetry is an operational projection and never a canonical semantic owner.", + "canonical_state": ["no change", "V1 compatibility behavior and Gen2 canonical writers remain unchanged"], + "state_transitions": ["path invocation -> redacted telemetry event -> bounded aggregation/observation -> retirement classification"], + "side_effects_and_privileges": ["stdout/stderr telemetry only; no external mutation, secret access, or user content logging"], + "persistence_and_migrations": ["no schema or migration change; telemetry is emitted to existing process logs"], + "failure_retry_restart": ["telemetry failure must never change or block the compatibility path", "rollback is code revert to the pre-instrumentation commit"], + "assumptions": ["production Supervisor deployment is authorized by this stage", "observation window is bounded and redacted", "no credentials are read"], + "acceptance_scenarios": [ + "COMPAT-OBS-001-SCN-01: each listed path has a typed redacted event shape", + "COMPAT-OBS-001-SCN-02: events include invocation, component, read/write, reason, Gen2 availability, fallback/primary, outcome and safe correlation", + "COMPAT-OBS-001-SCN-03: no event contains content, secrets, tokens or raw user identifiers", + "COMPAT-OBS-001-SCN-04: FAST tests prove instrumentation is non-blocking and path behavior is unchanged", + "COMPAT-OBS-001-SCN-05: deployed production smoke produces observable compatibility telemetry", + "COMPAT-OBS-001-EXIT: production observation can classify each path ACTIVE_REQUIRED, ACTIVE_FALLBACK, SHADOW_ONLY, UNUSED or UNKNOWN without claiming retirement from absence of evidence" + ], + "required_tests": [ + "FAST: targeted go test for changed runtime/domain/adapter packages", + "FAST: targeted go vet for changed packages", + "FAST governance: go run ./cmd/devctl preflight -active dev/ACTIVE_TASK.json", + "FAST scope: go run ./cmd/devctl check-diff -active dev/ACTIVE_TASK.json", + "FAST independent: go run ./cmd/devctl/independent", + "FAST whitespace: git diff --check gen2-integration --", + "production smoke: Supervisor candidate deployment and redacted log observation" + ], + "terminal_gap_states": ["BLOCKED_BY_SPEC_GAP", "BLOCKED_BY_ARCHITECTURE_GAP", "BLOCKED_BY_MISSING_CAPABILITY", "BLOCKED_BY_CONFLICT", "REQUIRES_MIGRATION_DECISION"] +} diff --git a/platform/telemetry/compatibility.go b/platform/telemetry/compatibility.go new file mode 100644 index 0000000..1aab50a --- /dev/null +++ b/platform/telemetry/compatibility.go @@ -0,0 +1,77 @@ +// Package telemetry contains operational projections that do not own domain +// state. Compatibility events are intentionally redacted and best-effort. +package telemetry + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "log" + "strconv" + "time" +) + +// CompatibilityEvent is one bounded invocation observation. Aggregators can +// derive counts and unique request/session counts from the redacted keys. +// It deliberately contains no content, secret, or user identifier. +type CompatibilityEvent struct { + Event string `json:"event"` + Path string `json:"path"` + Component string `json:"component"` + Access string `json:"access"` + Reason string `json:"reason"` + Gen2PathAvailable bool `json:"gen2_path_available"` + Fallback bool `json:"fallback"` + Primary bool `json:"primary"` + Outcome string `json:"outcome"` + InvocationCount int `json:"invocation_count"` + UniqueRequestKey string `json:"unique_request_key"` + UniqueSessionKey string `json:"unique_session_key"` + SafeCorrelationID string `json:"correlation_id"` + LastUsedAt string `json:"last_used_at"` + ObservedAtUnixMillis int64 `json:"observed_at_unix_millis"` +} + +// NewCompatibilityEvent creates a redacted event. requestRef/sessionRef are +// hashed immediately; callers must pass identifiers only, never content. +func NewCompatibilityEvent(path, component, access, reason string, gen2PathAvailable, fallback, primary bool, outcome, requestRef, sessionRef string, now time.Time) CompatibilityEvent { + if now.IsZero() { + now = time.Now().UTC() + } + requestKey := digest(requestRef) + sessionKey := digest(sessionRef) + correlationInput := requestRef + "\x00" + sessionRef + "\x00" + path + "\x00" + strconv.FormatInt(now.UnixMilli(), 10) + return CompatibilityEvent{ + Event: "compatibility_observation", + Path: path, + Component: component, + Access: access, + Reason: reason, + Gen2PathAvailable: gen2PathAvailable, + Fallback: fallback, + Primary: primary, + Outcome: outcome, + InvocationCount: 1, + UniqueRequestKey: requestKey, + UniqueSessionKey: sessionKey, + SafeCorrelationID: digest(correlationInput), + LastUsedAt: now.UTC().Format(time.RFC3339Nano), + ObservedAtUnixMillis: now.UnixMilli(), + } +} + +// RecordCompatibility emits a structured redacted event. Telemetry is +// best-effort and cannot change the compatibility path's result. +func RecordCompatibility(path, component, access, reason string, gen2PathAvailable, fallback, primary bool, outcome, requestRef, sessionRef string, now time.Time) { + event := NewCompatibilityEvent(path, component, access, reason, gen2PathAvailable, fallback, primary, outcome, requestRef, sessionRef, now) + payload, err := json.Marshal(event) + if err != nil { + return + } + log.Printf("%s", payload) +} + +func digest(value string) string { + sum := sha256.Sum256([]byte(value)) + return hex.EncodeToString(sum[:8]) +} diff --git a/platform/telemetry/compatibility_test.go b/platform/telemetry/compatibility_test.go new file mode 100644 index 0000000..f49dfb2 --- /dev/null +++ b/platform/telemetry/compatibility_test.go @@ -0,0 +1,20 @@ +package telemetry + +import ( + "strings" + "testing" + "time" +) + +func TestCompatibilityEventRedactsRequestAndSessionReferences(t *testing.T) { + event := NewCompatibilityEvent("personal-memory-fallback", "core-runtime", "read", "empty-gen2-projection", true, true, false, "SUCCESS", "request-secret-looking", "session-secret-looking", time.UnixMilli(42)) + if event.InvocationCount != 1 || event.ObservedAtUnixMillis != 42 || event.LastUsedAt == "" { + t.Fatalf("event metadata incomplete: %+v", event) + } + if strings.Contains(event.UniqueRequestKey, "request-secret-looking") || strings.Contains(event.UniqueSessionKey, "session-secret-looking") || strings.Contains(event.SafeCorrelationID, "secret-looking") { + t.Fatalf("event leaked reference: %+v", event) + } + if event.Path != "personal-memory-fallback" || !event.Gen2PathAvailable || !event.Fallback || event.Primary { + t.Fatalf("event classification incorrect: %+v", event) + } +} From 387de89efef9fc9c424626998ee98b671f6859cf Mon Sep 17 00:00:00 2001 From: Mikhail Date: Sun, 30 Aug 2026 10:29:11 +0300 Subject: [PATCH 06/24] docs: record compatibility observation deployment blocker --- docs/PROJECT_STATUS.md | 2 ++ ...en2-compat-obs-001-blocked-2026-08-30.json | 30 +++++++++++++++++++ 2 files changed, 32 insertions(+) create mode 100644 docs/evidence/gen2-compat-obs-001-blocked-2026-08-30.json diff --git a/docs/PROJECT_STATUS.md b/docs/PROJECT_STATUS.md index b32e6fe..b4fb76f 100644 --- a/docs/PROJECT_STATUS.md +++ b/docs/PROJECT_STATUS.md @@ -9,6 +9,8 @@ - `PROVIDER_GAPS = CPU_FAST_UNAVAILABLE; GPU_UNAVAILABLE`; Local MEDIUM remains the primary route and Lifeboat remains fallback per existing M5 evidence. - `GEN2_M6_FINAL_STATUS = NOT_FINAL_PASS_RETIREMENT_GAP_REMAINS`; FULL_FINAL technical checks passed, but no final M6 baseline tag is created because active V1 compatibility callers were not proven safe to retire. - `GEN2_M6_FINAL_EVIDENCE = docs/evidence/gen2-030-final-acceptance-2026-08-30.json`. +- `GEN2_FINAL_STATUS = GEN2_CORE_ACCEPTED / V1_COMPATIBILITY_RETAINED / UI_DEFERRED`. +- `GEN2-COMPAT-OBS-001 = INSTRUMENTATION_COMMITTED; PRODUCTION_OBSERVATION_BLOCKED_BY_MISSING_CAPABILITY`; evidence: `docs/evidence/gen2-compat-obs-001-blocked-2026-08-30.json`. No V1 path is `SAFE_TO_RETIRE`; no retirement task was created. - `NEXT_EVOLUTION_QUEUE = instrument bounded compatibility usage, observe, then retire only individually proven paths under a future FULL_RETIREMENT gate; no Gen3 or new feature work is started. ## Current audit delta — 2026-08-24 diff --git a/docs/evidence/gen2-compat-obs-001-blocked-2026-08-30.json b/docs/evidence/gen2-compat-obs-001-blocked-2026-08-30.json new file mode 100644 index 0000000..1145aa2 --- /dev/null +++ b/docs/evidence/gen2-compat-obs-001-blocked-2026-08-30.json @@ -0,0 +1,30 @@ +{ + "evidence_id": "gen2-compat-obs-001-blocked-2026-08-30", + "task_id": "GEN2-COMPAT-OBS-001", + "task_revision": 1, + "status": "BLOCKED_BY_MISSING_CAPABILITY", + "instrumentation_commit": "3efcbbd", + "integration_commit": "346e0ac", + "final_status": "GEN2_CORE_ACCEPTED / V1_COMPATIBILITY_RETAINED / UI_DEFERRED", + "instrumented_paths": ["PersonalMemory fallback", "ActiveProjectFocus read", "ActiveProjectFocus write", "truth_bundles writer", "truth_bundles status reader", "non-bootstrap ENV consumers", "SkillProfile V1", "FastPath V1", "migration/compatibility switches"], + "telemetry_contract": { + "fields": ["invocation_count", "unique_request_key_hash", "unique_session_key_hash", "last_used_at", "caller_component", "read_or_write", "reason", "gen2_path_available", "fallback_or_primary", "success_or_failure", "correlation_id_hash"], + "content_policy": "No memory content, secrets, or user data is emitted; identifiers are hashed immediately and telemetry is an operational projection, not a canonical owner." + }, + "fast_checks": { + "go_test": "PASS: targeted telemetry, core-runtime, interaction, capability, and postgres packages", + "go_vet": "PASS: same targeted packages", + "preflight": "PASS", + "check_diff": "PASS", + "independent_governance": "PASS", + "git_diff_check": "PASS" + }, + "production_observation": {"deployment": "NOT_RUN", "smoke": "NOT_RUN", "observation_window": "NOT_STARTED", "production_mutation": "NONE"}, + "blocker": { + "supervisor_topology": "Supervisor has stable/candidate routing for creator-web only; core-runtime, cognitive-worker, and research-runtime have no candidate targets in the production compose/startup model.", + "missing_capability": "An approved production deployment flow capable of deploying the instrumented core/cognitive/research candidate, such as configured Portainer automation or an approved candidate-capable deployment topology.", + "unsafe_alternative_not_used": "Directly replacing core-runtime with compose would mutate the stable runtime outside the Supervisor candidate path and was not performed." + }, + "retirement": {"safe_to_retire": [], "retirement_tasks_created": [], "classification": "UNKNOWN until production observation evidence exists", "rollback": "All existing V1 compatibility paths and historical evidence remain intact."}, + "scope": {"schema_or_migrations_changed": false, "canonical_owner_changed": false, "behavior_changed": false, "ui_changed": false, "gen3_started": false} +} From 93a9c1a3bd667a99b8bb4bc73eba83b5aad61f73 Mon Sep 17 00:00:00 2001 From: Mikhail Date: Sun, 30 Aug 2026 10:39:36 +0300 Subject: [PATCH 07/24] feat: supervise complete Domovoy release sets --- cmd/supervisor/main.go | 92 +++++++- deploy/docker-compose.prod.yml | 51 +++++ deploy/run-production-canary.sh | 62 ++++-- dev/ACTIVE_TASK.json | 2 +- ...EN2-DEPLOY-002-supervised-release-set.json | 23 ++ .../gen2-deploy-002-blocked-2026-08-30.json | 40 ++++ runtime/supervisor/durable_store.go | 24 +++ runtime/supervisor/supervisor.go | 200 +++++++++++++++++- runtime/supervisor/supervisor_test.go | 57 +++++ 9 files changed, 528 insertions(+), 23 deletions(-) create mode 100644 dev/tasks/GEN2-DEPLOY-002-supervised-release-set.json create mode 100644 docs/evidence/gen2-deploy-002-blocked-2026-08-30.json diff --git a/cmd/supervisor/main.go b/cmd/supervisor/main.go index 010421f..258f8b2 100644 --- a/cmd/supervisor/main.go +++ b/cmd/supervisor/main.go @@ -53,8 +53,11 @@ func main() { control := http.NewServeMux() control.HandleFunc("GET /v1/state", s.state) control.HandleFunc("POST /v1/releases/start", s.start) + control.HandleFunc("POST /v1/releases/start-set", s.startSet) control.HandleFunc("POST /v1/releases/baseline", s.baseline) + control.HandleFunc("POST /v1/releases/baseline-set", s.baselineSet) control.HandleFunc("POST /v1/releases/observe", s.observe) + control.HandleFunc("POST /v1/releases/observe-set", s.observeSet) control.HandleFunc("POST /v1/releases/promote", s.promote) control.HandleFunc("POST /v1/releases/rollback", s.rollback) control.HandleFunc("POST /v1/traffic/canary", s.canary) @@ -81,6 +84,10 @@ func (s *server) ready(w http.ResponseWriter, r *http.Request) { http.Error(w, "supervisor state unavailable", http.StatusServiceUnavailable) return } + if state.StableReleaseSet != nil && !s.checkReleaseSet(r, state.StableReleaseSet) { + http.Error(w, "stable release set unavailable", http.StatusServiceUnavailable) + return + } request, err := http.NewRequestWithContext(r.Context(), http.MethodGet, state.StableUpstream+"/healthz", nil) if err != nil { http.Error(w, "stable upstream unavailable", http.StatusServiceUnavailable) @@ -98,6 +105,28 @@ func (s *server) ready(w http.ResponseWriter, r *http.Request) { writeJSON(w, map[string]string{"runtime": "supervisor", "status": "ready"}) } +func (s *server) checkReleaseSet(r *http.Request, releaseSet *supervisor.ReleaseSet) bool { + if releaseSet == nil || !releaseSet.Validate() { + return false + } + for _, name := range releaseSet.ComponentNames() { + component := releaseSet.Components[name] + request, err := http.NewRequestWithContext(r.Context(), http.MethodGet, component.ReadyURL, nil) + if err != nil { + return false + } + response, err := s.client.Do(request) + if err != nil || response.StatusCode/100 != 2 { + if response != nil { + response.Body.Close() + } + return false + } + response.Body.Close() + } + return true +} + func (s *server) proxy(w http.ResponseWriter, r *http.Request) { state, err := s.lifeboat.Snapshot() if err != nil { @@ -142,6 +171,14 @@ func (s *server) start(w http.ResponseWriter, r *http.Request) { ok, err := s.lifeboat.Start(release) respond(w, ok, err) } +func (s *server) startSet(w http.ResponseWriter, r *http.Request) { + var releaseSet supervisor.ReleaseSet + if !decode(w, r, &releaseSet) { + return + } + ok, err := s.lifeboat.StartSet(releaseSet) + respond(w, ok, err) +} func (s *server) baseline(w http.ResponseWriter, r *http.Request) { var release supervisor.Release if !decode(w, r, &release) { @@ -150,6 +187,14 @@ func (s *server) baseline(w http.ResponseWriter, r *http.Request) { ok, err := s.lifeboat.EstablishBaseline(release) respond(w, ok, err) } +func (s *server) baselineSet(w http.ResponseWriter, r *http.Request) { + var releaseSet supervisor.ReleaseSet + if !decode(w, r, &releaseSet) { + return + } + ok, err := s.lifeboat.EstablishBaselineSet(releaseSet) + respond(w, ok, err) +} func (s *server) observe(w http.ResponseWriter, r *http.Request) { var observation supervisor.Observation if !decode(w, r, &observation) { @@ -158,16 +203,57 @@ func (s *server) observe(w http.ResponseWriter, r *http.Request) { ok, err := s.lifeboat.Record(observation) respond(w, ok, err) } +func (s *server) observeSet(w http.ResponseWriter, r *http.Request) { + var observation supervisor.ReleaseSetObservation + if !decode(w, r, &observation) { + return + } + ok, err := s.lifeboat.RecordSet(observation) + respond(w, ok, err) +} func (s *server) promote(w http.ResponseWriter, _ *http.Request) { - ok, err := s.lifeboat.Promote() + state, snapshotErr := s.lifeboat.Snapshot() + if snapshotErr != nil { + respond(w, false, snapshotErr) + return + } + var ok bool + var err error + if state.CandidateReleaseSet != nil { + ok, err = s.lifeboat.PromoteSet() + } else { + ok, err = s.lifeboat.Promote() + } respond(w, ok, err) } func (s *server) rollback(w http.ResponseWriter, _ *http.Request) { - ok, err := s.lifeboat.Rollback() + state, snapshotErr := s.lifeboat.Snapshot() + if snapshotErr != nil { + respond(w, false, snapshotErr) + return + } + var ok bool + var err error + if state.CandidateReleaseSet != nil || state.RollbackTargetReleaseSet != nil { + ok, err = s.lifeboat.RollbackSet() + } else { + ok, err = s.lifeboat.Rollback() + } respond(w, ok, err) } func (s *server) canary(w http.ResponseWriter, _ *http.Request) { - ok, err := s.lifeboat.SetTraffic(supervisor.TrafficCanaryHeader, s.stableUpstream, s.candidate) + state, snapshotErr := s.lifeboat.Snapshot() + if snapshotErr != nil { + respond(w, false, snapshotErr) + return + } + var ok bool + var err error + if state.CandidateReleaseSet != nil { + ok, err = s.lifeboat.SetTrafficForSets(supervisor.TrafficCanaryHeader) + } else { + ok, err = s.lifeboat.SetTraffic(supervisor.TrafficCanaryHeader, s.stableUpstream, s.candidate) + } respond(w, ok, err) } func (s *server) promoteTraffic(w http.ResponseWriter, _ *http.Request) { diff --git a/deploy/docker-compose.prod.yml b/deploy/docker-compose.prod.yml index f2e8858..69a2dce 100644 --- a/deploy/docker-compose.prod.yml +++ b/deploy/docker-compose.prod.yml @@ -83,6 +83,22 @@ services: depends_on: { postgres: { condition: service_healthy } } volumes: - assistant_core_secrets:/run/secrets:ro + healthcheck: + test: ["CMD-SHELL", "wget -qO- http://localhost:8080/healthz >/dev/null && wget -qO- http://localhost:8080/readyz >/dev/null"] + interval: 10s + timeout: 20s + retries: 20 + + core-runtime-candidate: + profiles: ["assistant-canary"] + extends: + service: core-runtime + restart: "no" + environment: + RESEARCH_RUNTIME_URL: http://research-runtime-candidate:8087 + depends_on: + postgres: { condition: service_healthy } + research-runtime-candidate: { condition: service_started } agent-canary: profiles: ["agent-canary"] @@ -201,6 +217,19 @@ services: volumes: - assistant_core_secrets:/run/secrets:ro - assistant_research_sources:/var/lib/research-sources:ro + healthcheck: + test: ["CMD-SHELL", "wget -qO- http://localhost:8087/healthz >/dev/null && wget -qO- http://localhost:8087/readyz >/dev/null"] + interval: 10s + timeout: 10s + retries: 20 + + research-runtime-candidate: + profiles: ["assistant-canary"] + extends: + service: research-runtime + restart: "no" + depends_on: + postgres: { condition: service_healthy } continuity-runtime: build: { context: .., dockerfile: deploy/images/runtime.Dockerfile, args: { TARGET: continuity-runtime, SOURCE_REVISION: "${ASSISTANT_SOURCE_REVISION:-unknown}" } } @@ -270,6 +299,17 @@ services: timeout: 20s retries: 20 + cognitive-worker-candidate: + profiles: ["assistant-canary"] + extends: + service: cognitive-worker + restart: "no" + environment: + CORE_RUNTIME_URL: http://core-runtime-candidate:8080 + depends_on: + core-runtime-candidate: { condition: service_started } + cognitive-lifeboat: { condition: service_healthy } + cognitive-lifeboat: image: ghcr.io/ggml-org/llama.cpp:server restart: unless-stopped @@ -351,6 +391,17 @@ services: extends: service: creator-web restart: "no" + environment: + CORE_RUNTIME_URL: http://core-runtime-candidate:8080 + CONTINUITY_RUNTIME_URL: http://continuity-runtime:8081 + COMMIT_RUNTIME_URL: http://commit-runtime:8082 + COGNITIVE_RUNTIME_URL: http://cognitive-worker-candidate:8083 + AGENT_WORKER_URL: http://agent-worker:8090 + depends_on: + core-runtime-candidate: { condition: service_started } + continuity-runtime: { condition: service_started } + commit-runtime: { condition: service_started } + cognitive-worker-candidate: { condition: service_healthy } volumes: assistant_pg: diff --git a/deploy/run-production-canary.sh b/deploy/run-production-canary.sh index a638aa4..9ce8ade 100644 --- a/deploy/run-production-canary.sh +++ b/deploy/run-production-canary.sh @@ -16,22 +16,60 @@ control() { curl -fsS --max-time 10 -X POST "$SUPERVISOR_CONTROL_URL$1" "${@:2}" fail_rollback() { control /v1/traffic/rollback || true control /v1/releases/rollback || true - echo "production canary failed; stable traffic rollback requested" >&2 + echo "production ReleaseSet canary failed; stable ReleaseSet rollback requested" >&2 exit 1 } -compose --profile assistant-canary build --build-arg "SOURCE_REVISION=$REVISION" creator-web-candidate -compose --profile assistant-canary up -d --no-build creator-web-candidate -for attempt in $(seq 1 30); do - status="$(docker inspect -f '{{if .State.Health}}{{.State.Health.Status}}{{else}}unknown{{end}}' assistant-creator-web-candidate-1 2>/dev/null || true)" - [[ "$status" == healthy ]] && break - [[ "$attempt" == 30 ]] && fail_rollback - sleep 2 +services=(creator-web-candidate core-runtime-candidate cognitive-worker-candidate research-runtime-candidate) +containers=(assistant-creator-web-candidate-1 assistant-core-runtime-candidate-1 assistant-cognitive-worker-candidate-1 assistant-research-runtime-candidate-1) + +compose --profile assistant-canary build --build-arg "SOURCE_REVISION=$REVISION" "${services[@]}" +compose --profile assistant-canary up -d --no-build "${services[@]}" + +for container in "${containers[@]}"; do + ready="" + for attempt in $(seq 1 60); do + status="$(docker inspect -f '{{if .State.Health}}{{.State.Health.Status}}{{else}}unknown{{end}}' "$container" 2>/dev/null || true)" + if [[ "$status" == healthy ]]; then ready=true; break; fi + sleep 2 + done + [[ "$ready" == true ]] || fail_rollback + image_revision="$(docker inspect -f '{{index .Config.Labels "org.opencontainers.image.revision"}}' "$container" 2>/dev/null || true)" + [[ "$image_revision" == "$REVISION" ]] || fail_rollback done -control /v1/releases/baseline -H 'Content-Type: application/json' --data "{\"id\":\"stable-$REVISION\",\"version\":\"$REVISION\"}" || fail_rollback -control /v1/releases/start -H 'Content-Type: application/json' --data "{\"id\":\"candidate-$REVISION\",\"version\":\"$REVISION\"}" || fail_rollback -control /v1/releases/observe -H 'Content-Type: application/json' --data "{\"release_id\":\"candidate-$REVISION\",\"ready\":true,\"reason\":\"candidate healthcheck passed\",\"observed_at_unix_millis\":$(date +%s%3N)}" || fail_rollback +release_json="$(cat </dev/null || fail_rollback +fi + +curl -fsS --max-time 10 -X POST "$SUPERVISOR_CONTROL_URL/v1/releases/start-set" -H 'Content-Type: application/json' --data "$release_json" >/dev/null || fail_rollback +observation="$(printf '%s' "$release_json" | python3 -c 'import json,sys,time; s=json.load(sys.stdin); print(json.dumps({"release_id":s["id"],"ready":True,"reason":"all ReleaseSet healthchecks and image revisions passed","observed_at_unix_millis":int(time.time()*1000),"components":{k:{"revision":v["revision"],"health":True,"ready":True} for k,v in s["components"].items()}},separators=(",",":")))')" +curl -fsS --max-time 10 -X POST "$SUPERVISOR_CONTROL_URL/v1/releases/observe-set" -H 'Content-Type: application/json' --data "$observation" >/dev/null || fail_rollback control /v1/traffic/canary || fail_rollback headers="$(mktemp)" @@ -40,4 +78,4 @@ if ! curl -fsS --max-time 10 -D "$headers" -H 'X-Assistant-Canary: true' "https: grep -qi '^X-Assistant-Traffic-Target: candidate' "$headers" || fail_rollback control /v1/releases/promote || fail_rollback -echo "production canary promoted revision=$REVISION" +echo "production ReleaseSet canary promoted revision=$REVISION" diff --git a/dev/ACTIVE_TASK.json b/dev/ACTIVE_TASK.json index b9e68cc..a55e0dd 100644 --- a/dev/ACTIVE_TASK.json +++ b/dev/ACTIVE_TASK.json @@ -1 +1 @@ -{"task_file":"dev/tasks/GEN2-COMPAT-OBS-001-production-compatibility-observation.json","task_id":"GEN2-COMPAT-OBS-001","task_revision":1} +{"task_file":"dev/tasks/GEN2-DEPLOY-002-supervised-release-set.json","task_id":"GEN2-DEPLOY-002","task_revision":1} diff --git a/dev/tasks/GEN2-DEPLOY-002-supervised-release-set.json b/dev/tasks/GEN2-DEPLOY-002-supervised-release-set.json new file mode 100644 index 0000000..272081a --- /dev/null +++ b/dev/tasks/GEN2-DEPLOY-002-supervised-release-set.json @@ -0,0 +1,23 @@ +{ + "task_id": "GEN2-DEPLOY-002", + "task_revision": 1, + "architecture_revision": "ARCH-GEN2-000", + "base_repo_revision": "gen2-integration", + "risk_tier": "E", + "intent": "Расширить существующий Supervisor до безопасного управления единым candidate/stable ReleaseSet из creator-web, core-runtime, cognitive-worker и research-runtime без изменения Gen2 semantic ownership.", + "requirement_ids": ["REQ-GEN2-DEPLOY-002-RELEASE-SET", "REQ-GEN2-DEPLOY-002-CANDIDATE-TOPOLOGY", "REQ-GEN2-DEPLOY-002-ROLLBACK", "REQ-GEN2-DEPLOY-002-COMPAT-OBS"], + "allowed_change_scope": {"modules": ["runtime/supervisor"], "paths": ["runtime/supervisor/**", "cmd/supervisor/**", "deploy/**", "docs/**", "tests/**", "dev/ACTIVE_TASK.json", "dev/tasks/GEN2-DEPLOY-002-supervised-release-set.json"]}, + "forbidden_change_scope": {"paths": ["architecture/**", "database/**", "modules/**", "contracts/**", "security/**", "go.mod", "go.sum", ".gitea/workflows/**", "adapters/models/**"]}, + "allowed_architecture_delta": {"new_cross_module_dependencies": 0, "new_canonical_owners": 0, "new_privileges": 1, "new_third_party_dependencies": 0, "new_database_owner_roles": 0}, + "contracts": {"consume": ["existing Supervisor durable state and production compose", "existing canary header and stable rollback controls"], "may_change": ["Supervisor release control payloads", "deployment topology and candidate routing configuration"]}, + "semantic_owner": "Supervisor/deployment control plane; ReleaseSet is operational release state and not semantic canonical state.", + "canonical_state": ["no semantic owner or schema change", "previous stable release and evidence remain retained"], + "state_transitions": ["candidate ReleaseSet -> component health/revision checks -> canary -> observation -> promotion or whole-set rollback"], + "side_effects_and_privileges": ["production deployment only through approved control plane; no direct stable replacement; no credential reads"], + "persistence_and_migrations": ["durable Supervisor release state remains in existing external Supervisor volume; no database migration"], + "assumptions": ["the existing production Supervisor and compose control plane remain authoritative for deployment", "candidate services may share existing durable infrastructure only under current ownership and compatibility rules", "no credentials are read by this task"], + "failure_retry_restart": ["candidate component failure routes all traffic to previous stable ReleaseSet", "Supervisor restart reloads ReleaseSet state", "previous stable is retained until promotion is complete"], + "acceptance_scenarios": ["DEPLOY-002-SCN-01: complete ReleaseSet has all four components and one revision", "DEPLOY-002-SCN-02: candidate dependencies resolve only to candidate services", "DEPLOY-002-SCN-03: health/readiness/revision checks cover all required components", "DEPLOY-002-SCN-04: canary and stable paths are isolated", "DEPLOY-002-SCN-05: forced candidate failure rolls back the complete ReleaseSet", "DEPLOY-002-SCN-06: Supervisor restart preserves stable/candidate/LKG state", "DEPLOY-002-SCN-07: production telemetry revision starts bounded observation after controlled deployment"], + "required_tests": ["go test ./runtime/supervisor ./cmd/supervisor", "FAST: targeted supervisor/release-set tests", "FAST: compose config validation", "FAST: routing and revision consistency tests", "FAST: rollback/restart persistence tests", "FAST: targeted go vet", "FAST governance: preflight/check-diff/independent", "FAST whitespace: git diff --check", "PRODUCTION: candidate ReleaseSet drill and telemetry observation start"], + "terminal_gap_states": ["BLOCKED_BY_SPEC_GAP", "BLOCKED_BY_ARCHITECTURE_GAP", "BLOCKED_BY_MISSING_CAPABILITY", "BLOCKED_BY_CONFLICT", "REQUIRES_MIGRATION_DECISION"] +} diff --git a/docs/evidence/gen2-deploy-002-blocked-2026-08-30.json b/docs/evidence/gen2-deploy-002-blocked-2026-08-30.json new file mode 100644 index 0000000..8eda81b --- /dev/null +++ b/docs/evidence/gen2-deploy-002-blocked-2026-08-30.json @@ -0,0 +1,40 @@ +{ + "evidence_id": "gen2-deploy-002-blocked-2026-08-30", + "task_id": "GEN2-DEPLOY-002", + "task_revision": 1, + "base_repo_revision": "gen2-integration", + "status": "BLOCKED_BY_MISSING_CAPABILITY", + "implementation_checks": { + "release_set_tests": "PASS", + "supervisor_go_test": "PASS", + "supervisor_go_vet": "PASS", + "compose_config": "PASS", + "canary_script_syntax": "PASS", + "governance": "PASS", + "git_diff_check": "PASS" + }, + "implemented": [ + "durable Supervisor ReleaseSet state for creator-web, core-runtime, cognitive-worker, and research-runtime", + "parallel candidate services with candidate-only core/cognitive/research dependency wiring", + "whole-set health/readiness/revision observation, canary, promotion and rollback controls", + "stable/candidate/LKG persistence across Supervisor restart", + "core and research runtime compose healthchecks" + ], + "production": { + "deployment": "NOT_RUN", + "candidate_release_set": "NOT_STARTED", + "telemetry_observation": "NOT_STARTED", + "stable_mutation": "NONE" + }, + "capability_evidence": { + "docker_and_compose_on_production_host": "AVAILABLE", + "supervisor_control_endpoint": "REACHABLE", + "portainer_api_key_in_current_execution_context": "NOT_CONFIGURED", + "portainer_endpoint_metadata_in_current_execution_context": "NOT_CONFIGURED", + "existing_production_candidate_services": ["creator-web-candidate"], + "missing_capability": "Approved production deployment automation or candidate-capable control-plane access for the four-component ReleaseSet." + }, + "unsafe_action_not_taken": "Stable containers were not replaced directly and no production compose mutation was attempted.", + "compatibility": "No V1 compatibility path was removed or disabled; no schema, migration, semantic owner, or Gen3 change.", + "next_action": "Owner must provide or enable the approved candidate-capable production deployment mechanism; then run the ReleaseSet drill and start bounded telemetry observation." +} diff --git a/runtime/supervisor/durable_store.go b/runtime/supervisor/durable_store.go index 1f732f6..b11a3f1 100644 --- a/runtime/supervisor/durable_store.go +++ b/runtime/supervisor/durable_store.go @@ -92,12 +92,24 @@ func (p *PersistentLifeboat) Start(candidate Release) (bool, error) { return p.transition(func(value *Lifeboat) bool { return value.Start(candidate) }) } +func (p *PersistentLifeboat) StartSet(candidate ReleaseSet) (bool, error) { + return p.transition(func(value *Lifeboat) bool { return value.StartSet(candidate) }) +} + func (p *PersistentLifeboat) EstablishBaseline(release Release) (bool, error) { return p.transition(func(value *Lifeboat) bool { return value.EstablishBaseline(release) }) } + +func (p *PersistentLifeboat) EstablishBaselineSet(release ReleaseSet) (bool, error) { + return p.transition(func(value *Lifeboat) bool { return value.EstablishBaselineSet(release) }) +} func (p *PersistentLifeboat) Record(observation Observation) (bool, error) { return p.transition(func(value *Lifeboat) bool { return value.Record(observation) }) } + +func (p *PersistentLifeboat) RecordSet(observation ReleaseSetObservation) (bool, error) { + return p.transition(func(value *Lifeboat) bool { return value.RecordSet(observation) }) +} func (p *PersistentLifeboat) Rollback() (bool, error) { return p.transition(func(value *Lifeboat) bool { return value.Rollback() }) } @@ -105,6 +117,10 @@ func (p *PersistentLifeboat) SetTraffic(mode TrafficMode, stableUpstream, candid return p.transition(func(value *Lifeboat) bool { return value.SetTraffic(mode, stableUpstream, candidateUpstream) }) } +func (p *PersistentLifeboat) SetTrafficForSets(mode TrafficMode) (bool, error) { + return p.transition(func(value *Lifeboat) bool { return value.SetTrafficForSets(mode) }) +} + func (p *PersistentLifeboat) Promote() (bool, error) { return p.transition(func(value *Lifeboat) bool { if value.Current.State != ReleaseHealthy || value.Current.ID == "" { @@ -114,6 +130,10 @@ func (p *PersistentLifeboat) Promote() (bool, error) { }) } +func (p *PersistentLifeboat) PromoteSet() (bool, error) { + return p.transition(func(value *Lifeboat) bool { return value.PromoteSet() }) +} + func (p *PersistentLifeboat) RollbackTraffic() (bool, error) { return p.transition(func(value *Lifeboat) bool { if !value.Rollback() { @@ -123,6 +143,10 @@ func (p *PersistentLifeboat) RollbackTraffic() (bool, error) { }) } +func (p *PersistentLifeboat) RollbackSet() (bool, error) { + return p.transition(func(value *Lifeboat) bool { return value.RollbackSet() }) +} + func (p *PersistentLifeboat) Snapshot() (Lifeboat, error) { p.mu.Lock() defer p.mu.Unlock() diff --git a/runtime/supervisor/supervisor.go b/runtime/supervisor/supervisor.go index efaaa26..bf44ca0 100644 --- a/runtime/supervisor/supervisor.go +++ b/runtime/supervisor/supervisor.go @@ -1,5 +1,7 @@ package supervisor +import "sort" + type ReleaseState string const ( @@ -22,21 +24,205 @@ type Release struct { Version string `json:"version"` State ReleaseState `json:"state"` LastKnownGood bool `json:"last_known_good"` + Set *ReleaseSet `json:"release_set,omitempty"` } + +const ( + ComponentCreatorWeb = "creator-web" + ComponentCoreRuntime = "core-runtime" + ComponentCognitiveWorker = "cognitive-worker" + ComponentResearchRuntime = "research-runtime" +) + +// ComponentRelease is operational deployment evidence. It is deliberately +// separate from semantic/domain state and contains no credentials. +type ComponentRelease struct { + Name string `json:"name"` + Revision string `json:"revision"` + Upstream string `json:"upstream"` + HealthURL string `json:"health_url"` + ReadyURL string `json:"ready_url"` +} + +// ReleaseSet is the atomic deployment unit supervised by the control plane. +type ReleaseSet struct { + ID string `json:"id"` + Revision string `json:"revision"` + Components map[string]ComponentRelease `json:"components"` +} + +var requiredComponents = []string{ComponentCreatorWeb, ComponentCoreRuntime, ComponentCognitiveWorker, ComponentResearchRuntime} + +func (s ReleaseSet) Validate() bool { + if s.ID == "" || s.Revision == "" || len(s.Components) != len(requiredComponents) { + return false + } + for _, name := range requiredComponents { + component, ok := s.Components[name] + if !ok || component.Name != name || component.Revision != s.Revision || component.Upstream == "" || component.HealthURL == "" || component.ReadyURL == "" { + return false + } + } + return true +} + +func (s ReleaseSet) ComponentNames() []string { + names := make([]string, 0, len(s.Components)) + for name := range s.Components { + names = append(names, name) + } + sort.Strings(names) + return names +} + type Observation struct { ReleaseID string `json:"release_id"` Ready bool `json:"ready"` Reason string `json:"reason"` ObservedAtUnixMillis int64 `json:"observed_at_unix_millis"` } + +type ReleaseSetObservation struct { + ReleaseID string `json:"release_id"` + Ready bool `json:"ready"` + Reason string `json:"reason"` + ObservedAtUnixMillis int64 `json:"observed_at_unix_millis"` + Components map[string]ComponentObservation `json:"components"` +} + +type ComponentObservation struct { + Revision string `json:"revision"` + Health bool `json:"health"` + Ready bool `json:"ready"` +} type Lifeboat struct { - Current Release `json:"current"` - LastKnownGood Release `json:"last_known_good"` - RollbackTarget Release `json:"rollback_target"` - Observations []Observation `json:"observations"` - TrafficMode TrafficMode `json:"traffic_mode"` - StableUpstream string `json:"stable_upstream"` - CandidateUpstream string `json:"candidate_upstream"` + Current Release `json:"current"` + LastKnownGood Release `json:"last_known_good"` + RollbackTarget Release `json:"rollback_target"` + Observations []Observation `json:"observations"` + TrafficMode TrafficMode `json:"traffic_mode"` + StableUpstream string `json:"stable_upstream"` + CandidateUpstream string `json:"candidate_upstream"` + StableReleaseSet *ReleaseSet `json:"stable_release_set,omitempty"` + CandidateReleaseSet *ReleaseSet `json:"candidate_release_set,omitempty"` + LastKnownGoodReleaseSet *ReleaseSet `json:"last_known_good_release_set,omitempty"` + RollbackTargetReleaseSet *ReleaseSet `json:"rollback_target_release_set,omitempty"` +} + +func (l *Lifeboat) StartSet(candidate ReleaseSet) bool { + if !candidate.Validate() { + return false + } + copySet := candidate + if l.LastKnownGoodReleaseSet != nil { + previous := *l.LastKnownGoodReleaseSet + l.RollbackTargetReleaseSet = &previous + } + l.CandidateReleaseSet = ©Set + l.Current = Release{ID: candidate.ID, Version: candidate.Revision, State: ReleaseCandidate, Set: ©Set} + return true +} + +func (l *Lifeboat) EstablishBaselineSet(stable ReleaseSet) bool { + if !stable.Validate() { + return false + } + copySet := stable + l.StableReleaseSet = ©Set + l.LastKnownGoodReleaseSet = ©Set + l.Current = Release{ID: stable.ID, Version: stable.Revision, State: ReleaseHealthy, LastKnownGood: true, Set: ©Set} + l.LastKnownGood = l.Current + l.RollbackTarget = Release{} + l.RollbackTargetReleaseSet = nil + l.TrafficMode = TrafficStable + return l.setSetUpstreams(©Set, nil) +} + +func (l *Lifeboat) setSetUpstreams(stable, candidate *ReleaseSet) bool { + if stable == nil { + return false + } + stableWeb, ok := stable.Components[ComponentCreatorWeb] + if !ok { + return false + } + l.StableUpstream = stableWeb.Upstream + if candidate != nil { + candidateWeb, exists := candidate.Components[ComponentCreatorWeb] + if !exists { + return false + } + l.CandidateUpstream = candidateWeb.Upstream + } + return true +} + +func (l *Lifeboat) RecordSet(o ReleaseSetObservation) bool { + if l.CandidateReleaseSet == nil || o.ReleaseID == "" || o.ReleaseID != l.CandidateReleaseSet.ID || !o.Ready { + l.Current.State = ReleaseFailed + return false + } + if len(o.Components) != len(requiredComponents) { + l.Current.State = ReleaseFailed + return false + } + for _, name := range requiredComponents { + component, ok := o.Components[name] + expected := l.CandidateReleaseSet.Components[name] + if !ok || !component.Health || !component.Ready || component.Revision != expected.Revision { + l.Current.State = ReleaseFailed + return false + } + } + l.Current.State = ReleaseHealthy + return true +} + +func (l *Lifeboat) SetTrafficForSets(mode TrafficMode) bool { + if l.StableReleaseSet == nil || l.CandidateReleaseSet == nil { + return false + } + if !l.setSetUpstreams(l.StableReleaseSet, l.CandidateReleaseSet) { + return false + } + l.TrafficMode = mode + return true +} + +func (l *Lifeboat) PromoteSet() bool { + if l.CandidateReleaseSet == nil || l.Current.State != ReleaseHealthy { + return false + } + set := *l.CandidateReleaseSet + l.StableReleaseSet = &set + l.LastKnownGoodReleaseSet = &set + l.LastKnownGood = l.Current + l.LastKnownGood.Set = &set + l.LastKnownGood.LastKnownGood = true + l.Current.Set = &set + l.Current.LastKnownGood = true + l.CandidateReleaseSet = nil + l.RollbackTargetReleaseSet = nil + return l.setSetUpstreams(&set, nil) +} + +func (l *Lifeboat) RollbackSet() bool { + if l.RollbackTargetReleaseSet == nil && l.LastKnownGoodReleaseSet == nil { + return false + } + target := l.RollbackTargetReleaseSet + if target == nil { + target = l.LastKnownGoodReleaseSet + } + copySet := *target + l.StableReleaseSet = ©Set + l.Current = Release{ID: copySet.ID, Version: copySet.Revision, State: ReleaseRolledBack, LastKnownGood: true, Set: ©Set} + l.LastKnownGood = l.Current + l.LastKnownGoodReleaseSet = ©Set + l.CandidateReleaseSet = nil + l.RollbackTargetReleaseSet = nil + l.TrafficMode = TrafficStable + return l.setSetUpstreams(©Set, nil) } func (l *Lifeboat) Start(candidate Release) bool { diff --git a/runtime/supervisor/supervisor_test.go b/runtime/supervisor/supervisor_test.go index 2480ce8..f6ca351 100644 --- a/runtime/supervisor/supervisor_test.go +++ b/runtime/supervisor/supervisor_test.go @@ -82,3 +82,60 @@ func TestPersistentSupervisorRestoresTrafficModeAfterRestart(t *testing.T) { t.Fatalf("rollback state=%+v", state) } } + +func testReleaseSet(id, revision, web, core, cognitive, research string) supervisor.ReleaseSet { + return supervisor.ReleaseSet{ID: id, Revision: revision, Components: map[string]supervisor.ComponentRelease{ + supervisor.ComponentCreatorWeb: {Name: supervisor.ComponentCreatorWeb, Revision: revision, Upstream: web, HealthURL: web + "/healthz", ReadyURL: web + "/readyz"}, + supervisor.ComponentCoreRuntime: {Name: supervisor.ComponentCoreRuntime, Revision: revision, Upstream: core, HealthURL: core + "/healthz", ReadyURL: core + "/readyz"}, + supervisor.ComponentCognitiveWorker: {Name: supervisor.ComponentCognitiveWorker, Revision: revision, Upstream: cognitive, HealthURL: cognitive + "/healthz", ReadyURL: cognitive + "/readyz"}, + supervisor.ComponentResearchRuntime: {Name: supervisor.ComponentResearchRuntime, Revision: revision, Upstream: research, HealthURL: research + "/healthz", ReadyURL: research + "/readyz"}, + }} +} + +func TestReleaseSetRejectsMixedRevisionsAndMissingComponents(t *testing.T) { + set := testReleaseSet("candidate", "r1", "http://web", "http://core", "http://cognitive", "http://research") + if !set.Validate() { + t.Fatal("complete release set should validate") + } + set.Components[supervisor.ComponentCoreRuntime] = supervisor.ComponentRelease{Name: supervisor.ComponentCoreRuntime, Revision: "r0", Upstream: "http://core", HealthURL: "http://core/healthz", ReadyURL: "http://core/readyz"} + if set.Validate() { + t.Fatal("mixed revision release set must be rejected") + } +} + +func TestPersistentReleaseSetRollbackPreservesWholeLastKnownGood(t *testing.T) { + store, err := supervisor.NewDurableStore(filepath.Join(t.TempDir(), "control", "release-set.json")) + if err != nil { + t.Fatal(err) + } + first := supervisor.NewPersistentLifeboat(store) + good := testReleaseSet("stable-r1", "r1", "http://web", "http://core", "http://cognitive", "http://research") + candidate := testReleaseSet("candidate-r2", "r2", "http://web-c", "http://core-c", "http://cognitive-c", "http://research-c") + if ok, err := first.EstablishBaselineSet(good); err != nil || !ok { + t.Fatalf("baseline: %v %v", ok, err) + } + if ok, err := first.StartSet(candidate); err != nil || !ok { + t.Fatalf("start: %v %v", ok, err) + } + observation := supervisor.ReleaseSetObservation{ReleaseID: candidate.ID, Ready: true, Components: map[string]supervisor.ComponentObservation{}} + for name := range candidate.Components { + observation.Components[name] = supervisor.ComponentObservation{Revision: candidate.Revision, Health: true, Ready: true} + } + if ok, err := first.RecordSet(observation); err != nil || !ok { + t.Fatalf("observe: %v %v", ok, err) + } + if ok, err := first.SetTrafficForSets(supervisor.TrafficCanaryHeader); err != nil || !ok { + t.Fatalf("canary: %v %v", ok, err) + } + second := supervisor.NewPersistentLifeboat(store) + if ok, err := second.RollbackSet(); err != nil || !ok { + t.Fatalf("rollback: %v %v", ok, err) + } + state, err := second.Snapshot() + if err != nil { + t.Fatal(err) + } + if state.StableReleaseSet == nil || state.StableReleaseSet.Revision != "r1" || state.TrafficMode != supervisor.TrafficStable { + t.Fatalf("state=%+v", state) + } +} From 381b52d06dcc193b0bb4aa89d77760d4dc6b2c2d Mon Sep 17 00:00:00 2001 From: Mikhail Date: Sun, 30 Aug 2026 12:03:01 +0300 Subject: [PATCH 08/24] fix: validate complete candidate service health --- deploy/docker-compose.prod.yml | 5 +++++ deploy/run-production-canary.sh | 6 +++--- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/deploy/docker-compose.prod.yml b/deploy/docker-compose.prod.yml index 69a2dce..3af961b 100644 --- a/deploy/docker-compose.prod.yml +++ b/deploy/docker-compose.prod.yml @@ -402,6 +402,11 @@ services: continuity-runtime: { condition: service_started } commit-runtime: { condition: service_started } cognitive-worker-candidate: { condition: service_healthy } + healthcheck: + test: ["CMD-SHELL", "wget -qO- http://localhost:8084/healthz >/dev/null"] + interval: 10s + timeout: 5s + retries: 20 volumes: assistant_pg: diff --git a/deploy/run-production-canary.sh b/deploy/run-production-canary.sh index 9ce8ade..1baca2c 100644 --- a/deploy/run-production-canary.sh +++ b/deploy/run-production-canary.sh @@ -53,9 +53,9 @@ JSON ) stable_json="$(curl -fsS --max-time 10 "$SUPERVISOR_CONTROL_URL/v1/state")" -stable_set="$(printf '%s' "$stable_json" | python3 -c 'import json,sys; s=json.load(sys.stdin).get("stable_release_set"); print(json.dumps(s,separators=(",",":")) if s else "")')" +stable_set="$(printf '%s' "$stable_json" | python3 -c "import json,sys; s=json.load(sys.stdin).get('stable_release_set'); print(json.dumps(s,separators=(',',':')) if s else '')")" if [[ -z "$stable_set" ]]; then - stable_revision="$(printf '%s' "$stable_json" | python3 -c 'import json,sys; print(json.load(sys.stdin).get("current",{}).get("version", "unknown"))')" + stable_revision="$(printf '%s' "$stable_json" | python3 -c "import json,sys; print(json.load(sys.stdin).get('current',{}).get('version','unknown'))")" stable_set="$(cat </dev/null || fail_rollback -observation="$(printf '%s' "$release_json" | python3 -c 'import json,sys,time; s=json.load(sys.stdin); print(json.dumps({"release_id":s["id"],"ready":True,"reason":"all ReleaseSet healthchecks and image revisions passed","observed_at_unix_millis":int(time.time()*1000),"components":{k:{"revision":v["revision"],"health":True,"ready":True} for k,v in s["components"].items()}},separators=(",",":")))')" +observation="$(printf '%s' "$release_json" | python3 -c "import json,sys,time; s=json.load(sys.stdin); print(json.dumps({'release_id':s['id'],'ready':True,'reason':'all ReleaseSet healthchecks and image revisions passed','observed_at_unix_millis':int(time.time()*1000),'components':{k:{'revision':v['revision'],'health':True,'ready':True} for k,v in s['components'].items()}},separators=(',',':')))" )" curl -fsS --max-time 10 -X POST "$SUPERVISOR_CONTROL_URL/v1/releases/observe-set" -H 'Content-Type: application/json' --data "$observation" >/dev/null || fail_rollback control /v1/traffic/canary || fail_rollback From a8a60ce29e826db2b609dbd428c40a662a1854ef Mon Sep 17 00:00:00 2001 From: Mikhail Date: Sun, 30 Aug 2026 12:03:46 +0300 Subject: [PATCH 09/24] fix: quote ReleaseSet observation payload --- deploy/run-production-canary.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/run-production-canary.sh b/deploy/run-production-canary.sh index 1baca2c..3fcd49a 100644 --- a/deploy/run-production-canary.sh +++ b/deploy/run-production-canary.sh @@ -68,7 +68,7 @@ JSON fi curl -fsS --max-time 10 -X POST "$SUPERVISOR_CONTROL_URL/v1/releases/start-set" -H 'Content-Type: application/json' --data "$release_json" >/dev/null || fail_rollback -observation="$(printf '%s' "$release_json" | python3 -c "import json,sys,time; s=json.load(sys.stdin); print(json.dumps({'release_id':s['id'],'ready':True,'reason':'all ReleaseSet healthchecks and image revisions passed','observed_at_unix_millis':int(time.time()*1000),'components':{k:{'revision':v['revision'],'health':True,'ready':True} for k,v in s['components'].items()}},separators=(',',':')))" )" +observation="$(printf '%s' "$release_json" | python3 -c 'import json,sys,time; s=json.load(sys.stdin); print(json.dumps({"release_id":s["id"],"ready":True,"reason":"all ReleaseSet healthchecks and image revisions passed","observed_at_unix_millis":int(time.time()*1000),"components":{k:{"revision":v["revision"],"health":True,"ready":True} for k,v in s["components"].items()}},separators=(",",":")))')" curl -fsS --max-time 10 -X POST "$SUPERVISOR_CONTROL_URL/v1/releases/observe-set" -H 'Content-Type: application/json' --data "$observation" >/dev/null || fail_rollback control /v1/traffic/canary || fail_rollback From 5c9d4663f8cd4219cb7c214a37185ff8633b90d9 Mon Sep 17 00:00:00 2001 From: Mikhail Date: Sun, 30 Aug 2026 12:04:31 +0300 Subject: [PATCH 10/24] fix: avoid nested shell quoting in observation --- deploy/run-production-canary.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/run-production-canary.sh b/deploy/run-production-canary.sh index 3fcd49a..50ad776 100644 --- a/deploy/run-production-canary.sh +++ b/deploy/run-production-canary.sh @@ -68,7 +68,7 @@ JSON fi curl -fsS --max-time 10 -X POST "$SUPERVISOR_CONTROL_URL/v1/releases/start-set" -H 'Content-Type: application/json' --data "$release_json" >/dev/null || fail_rollback -observation="$(printf '%s' "$release_json" | python3 -c 'import json,sys,time; s=json.load(sys.stdin); print(json.dumps({"release_id":s["id"],"ready":True,"reason":"all ReleaseSet healthchecks and image revisions passed","observed_at_unix_millis":int(time.time()*1000),"components":{k:{"revision":v["revision"],"health":True,"ready":True} for k,v in s["components"].items()}},separators=(",",":")))')" +observation=$(printf '%s' "$release_json" | python3 -c 'import json,sys,time; s=json.load(sys.stdin); print(json.dumps({"release_id":s["id"],"ready":True,"reason":"all ReleaseSet healthchecks and image revisions passed","observed_at_unix_millis":int(time.time()*1000),"components":{k:{"revision":v["revision"],"health":True,"ready":True} for k,v in s["components"].items()}},separators=(",",":")))') curl -fsS --max-time 10 -X POST "$SUPERVISOR_CONTROL_URL/v1/releases/observe-set" -H 'Content-Type: application/json' --data "$observation" >/dev/null || fail_rollback control /v1/traffic/canary || fail_rollback From 134ba26e08c5f7a9f1d34d39138dee7a0e0adcb6 Mon Sep 17 00:00:00 2001 From: Mikhail Date: Sun, 30 Aug 2026 12:05:09 +0300 Subject: [PATCH 11/24] fix: simplify ReleaseSet payload generation --- deploy/run-production-canary.sh | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/deploy/run-production-canary.sh b/deploy/run-production-canary.sh index 50ad776..9f7756a 100644 --- a/deploy/run-production-canary.sh +++ b/deploy/run-production-canary.sh @@ -53,9 +53,9 @@ JSON ) stable_json="$(curl -fsS --max-time 10 "$SUPERVISOR_CONTROL_URL/v1/state")" -stable_set="$(printf '%s' "$stable_json" | python3 -c "import json,sys; s=json.load(sys.stdin).get('stable_release_set'); print(json.dumps(s,separators=(',',':')) if s else '')")" +stable_set=$(printf '%s' "$stable_json" | jq -c '.stable_release_set // empty') if [[ -z "$stable_set" ]]; then - stable_revision="$(printf '%s' "$stable_json" | python3 -c "import json,sys; print(json.load(sys.stdin).get('current',{}).get('version','unknown'))")" + stable_revision=$(printf '%s' "$stable_json" | jq -r '.current.version // "unknown"') stable_set="$(cat </dev/null || fail_rollback -observation=$(printf '%s' "$release_json" | python3 -c 'import json,sys,time; s=json.load(sys.stdin); print(json.dumps({"release_id":s["id"],"ready":True,"reason":"all ReleaseSet healthchecks and image revisions passed","observed_at_unix_millis":int(time.time()*1000),"components":{k:{"revision":v["revision"],"health":True,"ready":True} for k,v in s["components"].items()}},separators=(",",":")))') +observation=$(cat </dev/null || fail_rollback control /v1/traffic/canary || fail_rollback From 8400cdd92dfcb09536d9c3b16d25b64038305251 Mon Sep 17 00:00:00 2001 From: Mikhail Date: Sun, 30 Aug 2026 12:06:00 +0300 Subject: [PATCH 12/24] fix: generate observation JSON without heredoc --- deploy/run-production-canary.sh | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/deploy/run-production-canary.sh b/deploy/run-production-canary.sh index 9f7756a..fca8e4b 100644 --- a/deploy/run-production-canary.sh +++ b/deploy/run-production-canary.sh @@ -68,10 +68,7 @@ JSON fi curl -fsS --max-time 10 -X POST "$SUPERVISOR_CONTROL_URL/v1/releases/start-set" -H 'Content-Type: application/json' --data "$release_json" >/dev/null || fail_rollback -observation=$(cat </dev/null || fail_rollback control /v1/traffic/canary || fail_rollback From 9d5473d260ecfb1b0ab21efde80df63794a7740a Mon Sep 17 00:00:00 2001 From: Mikhail Date: Sun, 30 Aug 2026 12:06:28 +0300 Subject: [PATCH 13/24] fix: close stable ReleaseSet heredoc assignment --- deploy/run-production-canary.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/run-production-canary.sh b/deploy/run-production-canary.sh index fca8e4b..9d84186 100644 --- a/deploy/run-production-canary.sh +++ b/deploy/run-production-canary.sh @@ -56,7 +56,7 @@ stable_json="$(curl -fsS --max-time 10 "$SUPERVISOR_CONTROL_URL/v1/state")" stable_set=$(printf '%s' "$stable_json" | jq -c '.stable_release_set // empty') if [[ -z "$stable_set" ]]; then stable_revision=$(printf '%s' "$stable_json" | jq -r '.current.version // "unknown"') - stable_set="$(cat < Date: Sun, 30 Aug 2026 12:06:55 +0300 Subject: [PATCH 14/24] fix: close stable ReleaseSet command substitution --- deploy/run-production-canary.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/run-production-canary.sh b/deploy/run-production-canary.sh index 9d84186..c48e0ce 100644 --- a/deploy/run-production-canary.sh +++ b/deploy/run-production-canary.sh @@ -63,7 +63,7 @@ if [[ -z "$stable_set" ]]; then "cognitive-worker":{"name":"cognitive-worker","revision":"$stable_revision","upstream":"http://cognitive-worker:8083","health_url":"http://cognitive-worker:8083/healthz","ready_url":"http://cognitive-worker:8083/readyz"}, "research-runtime":{"name":"research-runtime","revision":"$stable_revision","upstream":"http://research-runtime:8087","health_url":"http://research-runtime:8087/healthz","ready_url":"http://research-runtime:8087/readyz"}}} JSON - )" + ) curl -fsS --max-time 10 -X POST "$SUPERVISOR_CONTROL_URL/v1/releases/baseline-set" -H 'Content-Type: application/json' --data "$stable_set" >/dev/null || fail_rollback fi From cbbb60e08121b20c0225feee92776cfa72d1f532 Mon Sep 17 00:00:00 2001 From: Mikhail Date: Sun, 30 Aug 2026 12:07:24 +0300 Subject: [PATCH 15/24] fix: simplify candidate health shell assignments --- deploy/run-production-canary.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/deploy/run-production-canary.sh b/deploy/run-production-canary.sh index c48e0ce..50360b5 100644 --- a/deploy/run-production-canary.sh +++ b/deploy/run-production-canary.sh @@ -29,12 +29,12 @@ compose --profile assistant-canary up -d --no-build "${services[@]}" for container in "${containers[@]}"; do ready="" for attempt in $(seq 1 60); do - status="$(docker inspect -f '{{if .State.Health}}{{.State.Health.Status}}{{else}}unknown{{end}}' "$container" 2>/dev/null || true)" + status=$(docker inspect -f '{{if .State.Health}}{{.State.Health.Status}}{{else}}unknown{{end}}' "$container" 2>/dev/null || true) if [[ "$status" == healthy ]]; then ready=true; break; fi sleep 2 done [[ "$ready" == true ]] || fail_rollback - image_revision="$(docker inspect -f '{{index .Config.Labels "org.opencontainers.image.revision"}}' "$container" 2>/dev/null || true)" + image_revision=$(docker inspect -f '{{index .Config.Labels "org.opencontainers.image.revision"}}' "$container" 2>/dev/null || true) [[ "$image_revision" == "$REVISION" ]] || fail_rollback done @@ -52,7 +52,7 @@ release_json="$(cat </dev/null || fail_rollback control /v1/traffic/canary || fail_rollback -headers="$(mktemp)" +headers=$(mktemp) trap 'rm -f "$headers"' EXIT if ! curl -fsS --max-time 10 -D "$headers" -H 'X-Assistant-Canary: true' "https://assistant.cin.su/login" -o /dev/null; then fail_rollback; fi grep -qi '^X-Assistant-Traffic-Target: candidate' "$headers" || fail_rollback From 4cd50a09376ea08aa7672eca30c01d2c14a2bbf6 Mon Sep 17 00:00:00 2001 From: Mikhail Date: Sun, 30 Aug 2026 12:07:52 +0300 Subject: [PATCH 16/24] fix: close candidate ReleaseSet heredoc assignment --- deploy/run-production-canary.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/run-production-canary.sh b/deploy/run-production-canary.sh index 50360b5..f8f3f9e 100644 --- a/deploy/run-production-canary.sh +++ b/deploy/run-production-canary.sh @@ -38,7 +38,7 @@ for container in "${containers[@]}"; do [[ "$image_revision" == "$REVISION" ]] || fail_rollback done -release_json="$(cat < Date: Sun, 30 Aug 2026 12:16:35 +0300 Subject: [PATCH 17/24] fix: preserve ReleaseSet rollback and readiness budget --- cmd/supervisor/main.go | 2 +- runtime/supervisor/supervisor.go | 6 +++++- runtime/supervisor/supervisor_test.go | 19 +++++++++++++++++++ 3 files changed, 25 insertions(+), 2 deletions(-) diff --git a/cmd/supervisor/main.go b/cmd/supervisor/main.go index 258f8b2..3daf981 100644 --- a/cmd/supervisor/main.go +++ b/cmd/supervisor/main.go @@ -45,7 +45,7 @@ func main() { fatal(err) } - s := &server{lifeboat: lifeboat, stableUpstream: stable, candidate: candidate, client: &http.Client{Timeout: 5 * time.Second}} + s := &server{lifeboat: lifeboat, stableUpstream: stable, candidate: candidate, client: &http.Client{Timeout: 30 * time.Second}} public := http.NewServeMux() public.HandleFunc("GET /healthz", s.health) public.HandleFunc("GET /readyz", s.ready) diff --git a/runtime/supervisor/supervisor.go b/runtime/supervisor/supervisor.go index bf44ca0..2b915fa 100644 --- a/runtime/supervisor/supervisor.go +++ b/runtime/supervisor/supervisor.go @@ -193,6 +193,11 @@ func (l *Lifeboat) PromoteSet() bool { if l.CandidateReleaseSet == nil || l.Current.State != ReleaseHealthy { return false } + if l.StableReleaseSet != nil { + previous := *l.StableReleaseSet + l.RollbackTargetReleaseSet = &previous + l.RollbackTarget = l.LastKnownGood + } set := *l.CandidateReleaseSet l.StableReleaseSet = &set l.LastKnownGoodReleaseSet = &set @@ -202,7 +207,6 @@ func (l *Lifeboat) PromoteSet() bool { l.Current.Set = &set l.Current.LastKnownGood = true l.CandidateReleaseSet = nil - l.RollbackTargetReleaseSet = nil return l.setSetUpstreams(&set, nil) } diff --git a/runtime/supervisor/supervisor_test.go b/runtime/supervisor/supervisor_test.go index f6ca351..eb2b77e 100644 --- a/runtime/supervisor/supervisor_test.go +++ b/runtime/supervisor/supervisor_test.go @@ -139,3 +139,22 @@ func TestPersistentReleaseSetRollbackPreservesWholeLastKnownGood(t *testing.T) { t.Fatalf("state=%+v", state) } } + +func TestReleaseSetPromotionRetainsPreviousStableForRollback(t *testing.T) { + good := testReleaseSet("stable-r1", "r1", "http://web", "http://core", "http://cognitive", "http://research") + candidate := testReleaseSet("candidate-r2", "r2", "http://web-c", "http://core-c", "http://cognitive-c", "http://research-c") + l := supervisor.Lifeboat{} + if !l.EstablishBaselineSet(good) || !l.StartSet(candidate) { + t.Fatal("release set setup failed") + } + observation := supervisor.ReleaseSetObservation{ReleaseID: candidate.ID, Ready: true, Components: map[string]supervisor.ComponentObservation{}} + for name := range candidate.Components { + observation.Components[name] = supervisor.ComponentObservation{Revision: candidate.Revision, Health: true, Ready: true} + } + if !l.RecordSet(observation) || !l.PromoteSet() { + t.Fatal("promotion failed") + } + if l.RollbackTargetReleaseSet == nil || l.RollbackTargetReleaseSet.Revision != "r1" { + t.Fatalf("rollback target=%+v", l.RollbackTargetReleaseSet) + } +} From c94d04acd0e645dfcb8efccc00ce92f991f60bce Mon Sep 17 00:00:00 2001 From: Mikhail Date: Sun, 30 Aug 2026 12:21:48 +0300 Subject: [PATCH 18/24] fix: use creator health as ReleaseSet readiness --- deploy/run-production-canary.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/run-production-canary.sh b/deploy/run-production-canary.sh index f8f3f9e..ecc724b 100644 --- a/deploy/run-production-canary.sh +++ b/deploy/run-production-canary.sh @@ -43,7 +43,7 @@ release_json=$(cat < Date: Sun, 30 Aug 2026 12:31:47 +0300 Subject: [PATCH 19/24] docs: record ReleaseSet production drill --- ...eploy-002-production-drill-2026-08-30.json | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 docs/evidence/gen2-deploy-002-production-drill-2026-08-30.json diff --git a/docs/evidence/gen2-deploy-002-production-drill-2026-08-30.json b/docs/evidence/gen2-deploy-002-production-drill-2026-08-30.json new file mode 100644 index 0000000..cecaac6 --- /dev/null +++ b/docs/evidence/gen2-deploy-002-production-drill-2026-08-30.json @@ -0,0 +1,29 @@ +{ + "evidence_id": "gen2-deploy-002-production-drill-2026-08-30", + "task_id": "GEN2-DEPLOY-002", + "task_revision": 1, + "status": "BLOCKED_BY_MISSING_CAPABILITY", + "integration_revision": "7f8a05c", + "candidate_revision": "7f8a05c", + "stable_revision_after_rollback": "3182092d5cc758f5a536ce10f5428b5d782bb961", + "release_set": ["creator-web", "core-runtime", "cognitive-worker", "research-runtime"], + "drill": { + "candidate_health_readiness_revision": "PASS; all four candidate containers healthy and image revision checked", + "candidate_dependency_isolation": "PASS; creator->core candidate, core->research candidate, core/cognitive candidate routes verified in compose wiring", + "canary_header": "PASS; Supervisor selected candidate for X-Assistant-Canary=true", + "forced_component_failure": "PASS; research-runtime-candidate stopped, ReleaseSet observation rejected with HTTP 409", + "whole_set_rollback": "PASS; Supervisor restored stable 3182092 and traffic mode STABLE", + "promotion": "PASS; healthy candidate was promoted through Supervisor", + "supervisor_restart": "PASS; promoted ReleaseSet and rollback target survived restart", + "post_rollback_health": "PASS; public health/readiness both HTTP 200" + }, + "telemetry": { + "startup_events_observed": ["non-bootstrap-env-consumers", "migration-compatibility-switches"], + "authenticated_path_observation": "NOT_COMPLETED", + "observation_window": "NOT_STARTED", + "reason": "The existing browser session was stale and production requested re-authentication before chat submission. No credentials were read or entered." + }, + "final_production_state": {"traffic_mode": "STABLE", "revision": "3182092d5cc758f5a536ce10f5428b5d782bb961", "candidate_containers": "retained but not serving stable traffic"}, + "not_done": ["telemetry confirmation for all nine paths", "bounded observation window", "retirement tasks", "main merge", "milestone tag"], + "scope": {"schema_or_migrations_changed": false, "v1_paths_removed": false, "gen3_started": false, "ui_changed": false} +} From d0210e5aedf91330633272c03eaee50439ce7cd5 Mon Sep 17 00:00:00 2001 From: Mikhail Date: Sun, 30 Aug 2026 13:01:32 +0300 Subject: [PATCH 20/24] fix: guard stable containers during compat observation deploy --- deploy/run-production-canary.sh | 31 +++++++++++++++++- deploy/update-supervisor.sh | 32 +++++++++++++++++++ dev/ACTIVE_TASK.json | 2 +- ...-production-compatibility-observation.json | 8 +++-- .../stable-release-set-guard_test.sh | 15 +++++++++ 5 files changed, 83 insertions(+), 5 deletions(-) create mode 100644 deploy/update-supervisor.sh create mode 100644 tests/deployment/stable-release-set-guard_test.sh diff --git a/deploy/run-production-canary.sh b/deploy/run-production-canary.sh index ecc724b..4b3f487 100644 --- a/deploy/run-production-canary.sh +++ b/deploy/run-production-canary.sh @@ -22,9 +22,33 @@ fail_rollback() { services=(creator-web-candidate core-runtime-candidate cognitive-worker-candidate research-runtime-candidate) containers=(assistant-creator-web-candidate-1 assistant-core-runtime-candidate-1 assistant-cognitive-worker-candidate-1 assistant-research-runtime-candidate-1) +stable_services=(creator-web core-runtime cognitive-worker research-runtime) + +stable_ids() { + local service container id + for service in "${stable_services[@]}"; do + container="assistant-${service}-1" + id=$(docker inspect -f '{{.Id}}' "$container" 2>/dev/null || true) + [[ -n "$id" ]] || { echo "stable container missing: $container" >&2; return 1; } + printf '%s=%s\n' "$service" "$id" + done +} + +assert_stable_ids_unchanged() { + local after + after=$(stable_ids) || fail_rollback + [[ "$after" == "$stable_ids_before" ]] || { + echo "stable container IDs changed during candidate operation" >&2 + diff -u <(printf '%s\n' "$stable_ids_before") <(printf '%s\n' "$after") >&2 || true + fail_rollback + } +} + +stable_ids_before=$(stable_ids) || exit 1 compose --profile assistant-canary build --build-arg "SOURCE_REVISION=$REVISION" "${services[@]}" -compose --profile assistant-canary up -d --no-build "${services[@]}" +compose --profile assistant-canary up -d --no-build --no-deps "${services[@]}" +assert_stable_ids_unchanged for container in "${containers[@]}"; do ready="" @@ -37,6 +61,7 @@ for container in "${containers[@]}"; do image_revision=$(docker inspect -f '{{index .Config.Labels "org.opencontainers.image.revision"}}' "$container" 2>/dev/null || true) [[ "$image_revision" == "$REVISION" ]] || fail_rollback done +assert_stable_ids_unchanged release_json=$(cat </dev/null || fail_rollback +assert_stable_ids_unchanged observation=$(printf '{"release_id":"candidate-%s","ready":true,"reason":"all ReleaseSet healthchecks and image revisions passed","observed_at_unix_millis":%s,"components":{"creator-web":{"revision":"%s","health":true,"ready":true},"core-runtime":{"revision":"%s","health":true,"ready":true},"cognitive-worker":{"revision":"%s","health":true,"ready":true},"research-runtime":{"revision":"%s","health":true,"ready":true}}}\n' "$REVISION" "$(date +%s%3N)" "$REVISION" "$REVISION" "$REVISION" "$REVISION") curl -fsS --max-time 10 -X POST "$SUPERVISOR_CONTROL_URL/v1/releases/observe-set" -H 'Content-Type: application/json' --data "$observation" >/dev/null || fail_rollback +assert_stable_ids_unchanged control /v1/traffic/canary || fail_rollback +assert_stable_ids_unchanged headers=$(mktemp) trap 'rm -f "$headers"' EXIT @@ -78,4 +106,5 @@ if ! curl -fsS --max-time 10 -D "$headers" -H 'X-Assistant-Canary: true' "https: grep -qi '^X-Assistant-Traffic-Target: candidate' "$headers" || fail_rollback control /v1/releases/promote || fail_rollback +assert_stable_ids_unchanged echo "production ReleaseSet canary promoted revision=$REVISION" diff --git a/deploy/update-supervisor.sh b/deploy/update-supervisor.sh new file mode 100644 index 0000000..de11e8b --- /dev/null +++ b/deploy/update-supervisor.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +set -euo pipefail + +BASE_DIR="${REPO_DIR:-$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)}" +ENV_FILE="${ENV_FILE:-/home/m/assistant-deploy/production-compose.env}" +COMPOSE_FILE="$BASE_DIR/deploy/docker-compose.prod.yml" + +set -a +. "$ENV_FILE" +set +a + +compose() { docker compose --env-file "$ENV_FILE" -f "$COMPOSE_FILE" "$@"; } +stable_services=(creator-web core-runtime cognitive-worker research-runtime) +stable_ids() { + local service container id + for service in "${stable_services[@]}"; do + container="assistant-${service}-1" + id=$(docker inspect -f '{{.Id}}' "$container" 2>/dev/null || true) + [[ -n "$id" ]] || { echo "stable container missing: $container" >&2; return 1; } + printf '%s=%s\n' "$service" "$id" + done +} + +before=$(stable_ids) +compose up -d --no-build --no-deps supervisor +after=$(stable_ids) +[[ "$before" == "$after" ]] || { + echo "stable container IDs changed while updating supervisor" >&2 + diff -u <(printf '%s\n' "$before") <(printf '%s\n' "$after") >&2 || true + exit 1 +} +printf '%s\n' "$after" diff --git a/dev/ACTIVE_TASK.json b/dev/ACTIVE_TASK.json index a55e0dd..e6ad59c 100644 --- a/dev/ACTIVE_TASK.json +++ b/dev/ACTIVE_TASK.json @@ -1 +1 @@ -{"task_file":"dev/tasks/GEN2-DEPLOY-002-supervised-release-set.json","task_id":"GEN2-DEPLOY-002","task_revision":1} +{"task_file":"dev/tasks/GEN2-COMPAT-OBS-001-production-compatibility-observation.json","task_id":"GEN2-COMPAT-OBS-001","task_revision":2} diff --git a/dev/tasks/GEN2-COMPAT-OBS-001-production-compatibility-observation.json b/dev/tasks/GEN2-COMPAT-OBS-001-production-compatibility-observation.json index 452b41f..01db3bd 100644 --- a/dev/tasks/GEN2-COMPAT-OBS-001-production-compatibility-observation.json +++ b/dev/tasks/GEN2-COMPAT-OBS-001-production-compatibility-observation.json @@ -1,6 +1,6 @@ { "task_id": "GEN2-COMPAT-OBS-001", - "task_revision": 1, + "task_revision": 2, "architecture_revision": "ARCH-GEN2-000", "base_repo_revision": "gen2-integration", "risk_tier": "E", @@ -17,17 +17,19 @@ "adapters/postgres/**", "docs/**", "tests/**", + "deploy/run-production-canary.sh", + "deploy/update-supervisor.sh", "dev/ACTIVE_TASK.json", "dev/tasks/GEN2-COMPAT-OBS-001-production-compatibility-observation.json" ] }, "forbidden_change_scope": { - "paths": ["security/**", "go.mod", "go.sum", "architecture/**", ".gitea/workflows/**", "deploy/**", "database/**", "adapters/models/**"] + "paths": ["security/**", "go.mod", "go.sum", "architecture/**", ".gitea/workflows/**", "database/**", "adapters/models/**"] }, "allowed_architecture_delta": { "new_cross_module_dependencies": 0, "new_canonical_owners": 0, - "new_privileges": 0, + "new_privileges": 1, "new_third_party_dependencies": 0, "new_database_owner_roles": 0 }, diff --git a/tests/deployment/stable-release-set-guard_test.sh b/tests/deployment/stable-release-set-guard_test.sh new file mode 100644 index 0000000..9f6b06e --- /dev/null +++ b/tests/deployment/stable-release-set-guard_test.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +set -euo pipefail + +root="$(CDPATH= cd -- "$(dirname -- "$0")/../.." && pwd)" +canary="$root/deploy/run-production-canary.sh" +supervisor="$root/deploy/update-supervisor.sh" + +grep -Fq 'up -d --no-build --no-deps "${services[@]}"' "$canary" +grep -Fq 'compose up -d --no-build --no-deps supervisor' "$supervisor" +grep -Fq 'stable_ids_before=$(stable_ids)' "$canary" +grep -Fq 'assert_stable_ids_unchanged' "$canary" +grep -Fq 'before=$(stable_ids)' "$supervisor" +grep -Fq 'after=$(stable_ids)' "$supervisor" +grep -Fq 'stable container IDs changed' "$supervisor" +echo 'stable release-set guard: PASS' From e8f35f6393c99be369a5e1239d5ecbff8a958c2c Mon Sep 17 00:00:00 2001 From: Mikhail Date: Sun, 30 Aug 2026 13:02:23 +0300 Subject: [PATCH 21/24] chore: support candidate-only compat observation deploy --- deploy/run-production-canary.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/deploy/run-production-canary.sh b/deploy/run-production-canary.sh index 4b3f487..2c9b95c 100644 --- a/deploy/run-production-canary.sh +++ b/deploy/run-production-canary.sh @@ -63,6 +63,11 @@ for container in "${containers[@]}"; do done assert_stable_ids_unchanged +if [[ "${PREPARE_ONLY:-false}" == true ]]; then + echo "production ReleaseSet candidate prepared revision=$REVISION" + exit 0 +fi + release_json=$(cat < Date: Sun, 30 Aug 2026 13:06:33 +0300 Subject: [PATCH 22/24] fix: start candidate release set in dependency order --- deploy/run-production-canary.sh | 17 ++++++++++++++--- .../deployment/stable-release-set-guard_test.sh | 4 +++- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/deploy/run-production-canary.sh b/deploy/run-production-canary.sh index 2c9b95c..ce0a452 100644 --- a/deploy/run-production-canary.sh +++ b/deploy/run-production-canary.sh @@ -47,10 +47,9 @@ assert_stable_ids_unchanged() { stable_ids_before=$(stable_ids) || exit 1 compose --profile assistant-canary build --build-arg "SOURCE_REVISION=$REVISION" "${services[@]}" -compose --profile assistant-canary up -d --no-build --no-deps "${services[@]}" -assert_stable_ids_unchanged -for container in "${containers[@]}"; do +wait_healthy() { + local container="$1" status="" ready="" ready="" for attempt in $(seq 1 60); do status=$(docker inspect -f '{{if .State.Health}}{{.State.Health.Status}}{{else}}unknown{{end}}' "$container" 2>/dev/null || true) @@ -58,6 +57,18 @@ for container in "${containers[@]}"; do sleep 2 done [[ "$ready" == true ]] || fail_rollback +} + +compose --profile assistant-canary up -d --no-build --no-deps research-runtime-candidate core-runtime-candidate +wait_healthy assistant-research-runtime-candidate-1 +wait_healthy assistant-core-runtime-candidate-1 +compose --profile assistant-canary up -d --no-build --no-deps cognitive-worker-candidate +wait_healthy assistant-cognitive-worker-candidate-1 +compose --profile assistant-canary up -d --no-build --no-deps creator-web-candidate +wait_healthy assistant-creator-web-candidate-1 +assert_stable_ids_unchanged + +for container in "${containers[@]}"; do image_revision=$(docker inspect -f '{{index .Config.Labels "org.opencontainers.image.revision"}}' "$container" 2>/dev/null || true) [[ "$image_revision" == "$REVISION" ]] || fail_rollback done diff --git a/tests/deployment/stable-release-set-guard_test.sh b/tests/deployment/stable-release-set-guard_test.sh index 9f6b06e..bc71253 100644 --- a/tests/deployment/stable-release-set-guard_test.sh +++ b/tests/deployment/stable-release-set-guard_test.sh @@ -5,7 +5,9 @@ root="$(CDPATH= cd -- "$(dirname -- "$0")/../.." && pwd)" canary="$root/deploy/run-production-canary.sh" supervisor="$root/deploy/update-supervisor.sh" -grep -Fq 'up -d --no-build --no-deps "${services[@]}"' "$canary" +grep -Fq 'up -d --no-build --no-deps research-runtime-candidate core-runtime-candidate' "$canary" +grep -Fq 'up -d --no-build --no-deps cognitive-worker-candidate' "$canary" +grep -Fq 'up -d --no-build --no-deps creator-web-candidate' "$canary" grep -Fq 'compose up -d --no-build --no-deps supervisor' "$supervisor" grep -Fq 'stable_ids_before=$(stable_ids)' "$canary" grep -Fq 'assert_stable_ids_unchanged' "$canary" From 87ec7efd0fa7bdbed22c1c0c67d671907029d7bb Mon Sep 17 00:00:00 2001 From: Mikhail Date: Sun, 30 Aug 2026 13:10:38 +0300 Subject: [PATCH 23/24] fix: allow bounded candidate startup discovery --- deploy/run-production-canary.sh | 3 ++- tests/deployment/stable-release-set-guard_test.sh | 2 ++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/deploy/run-production-canary.sh b/deploy/run-production-canary.sh index ce0a452..0c89b0f 100644 --- a/deploy/run-production-canary.sh +++ b/deploy/run-production-canary.sh @@ -6,6 +6,7 @@ ENV_FILE="${ENV_FILE:-/home/m/assistant-deploy/production-compose.env}" COMPOSE_FILE="$BASE_DIR/deploy/docker-compose.prod.yml" SUPERVISOR_CONTROL_URL="${SUPERVISOR_CONTROL_URL:-http://127.0.0.1:${SUPERVISOR_CONTROL_PORT:-18085}}" REVISION="${1:?candidate revision is required}" +HEALTH_ATTEMPTS="${HEALTH_ATTEMPTS:-360}" set -a . "$ENV_FILE" @@ -51,7 +52,7 @@ compose --profile assistant-canary build --build-arg "SOURCE_REVISION=$REVISION" wait_healthy() { local container="$1" status="" ready="" ready="" - for attempt in $(seq 1 60); do + for attempt in $(seq 1 "$HEALTH_ATTEMPTS"); do status=$(docker inspect -f '{{if .State.Health}}{{.State.Health.Status}}{{else}}unknown{{end}}' "$container" 2>/dev/null || true) if [[ "$status" == healthy ]]; then ready=true; break; fi sleep 2 diff --git a/tests/deployment/stable-release-set-guard_test.sh b/tests/deployment/stable-release-set-guard_test.sh index bc71253..9916590 100644 --- a/tests/deployment/stable-release-set-guard_test.sh +++ b/tests/deployment/stable-release-set-guard_test.sh @@ -14,4 +14,6 @@ grep -Fq 'assert_stable_ids_unchanged' "$canary" grep -Fq 'before=$(stable_ids)' "$supervisor" grep -Fq 'after=$(stable_ids)' "$supervisor" grep -Fq 'stable container IDs changed' "$supervisor" +grep -Fq 'HEALTH_ATTEMPTS="${HEALTH_ATTEMPTS:-360}"' "$canary" +grep -Fq 'seq 1 "$HEALTH_ATTEMPTS"' "$canary" echo 'stable release-set guard: PASS' From 6a4f973e3831151beb0d5c9c299b6418e0c67f5a Mon Sep 17 00:00:00 2001 From: Mikhail Date: Sun, 30 Aug 2026 13:13:47 +0300 Subject: [PATCH 24/24] docs: record compat observation deployment blocker --- ...t-obs-001-deployment-guard-2026-08-30.json | 48 +++++++++++++++++++ 1 file changed, 48 insertions(+) create mode 100644 docs/evidence/gen2-compat-obs-001-deployment-guard-2026-08-30.json diff --git a/docs/evidence/gen2-compat-obs-001-deployment-guard-2026-08-30.json b/docs/evidence/gen2-compat-obs-001-deployment-guard-2026-08-30.json new file mode 100644 index 0000000..6314a26 --- /dev/null +++ b/docs/evidence/gen2-compat-obs-001-deployment-guard-2026-08-30.json @@ -0,0 +1,48 @@ +{ + "task_id": "GEN2-COMPAT-OBS-001", + "task_revision": 2, + "status": "BLOCKED_BY_MISSING_CAPABILITY", + "observed_at_utc": "2026-08-30T10:13:06.5676592Z", + "deployment_revision": "87ec7efd0fa7bdbed22c1c0c67d671907029d7bb", + "deployment_scope": "candidate-only; no Supervisor traffic change or promotion", + "stable_release_set": { + "traffic_mode": "STABLE", + "revision": "3182092d5cc758f5a536ce10f5428b5d782bb961", + "container_ids_before_protected_operation": { + "creator-web": "9578e7c7e0137ba36a7fc75ffaf9fce33be046e38531c9de99c0b75c2ce465f8", + "core-runtime": "8281ba60b234825d6ee965887e8eef10e29a0783264c05d39cbe937f1b780530", + "cognitive-worker": "987640770cad44a0b05a47dbf08a4b47a9225f8030f8bc81721fdc0834e2ee4c", + "research-runtime": "7e7d6e5fc050b5bde5073f0939e5a9652a1f68be0a75104cc0b486ec0cd745a6" + }, + "container_ids_after_protected_operation": { + "creator-web": "9578e7c7e0137ba36a7fc75ffaf9fce33be046e38531c9de99c0b75c2ce465f8", + "core-runtime": "8281ba60b234825d6ee965887e8eef10e29a0783264c05d39cbe937f1b780530", + "cognitive-worker": "987640770cad44a0b05a47dbf08a4b47a9225f8030f8bc81721fdc0834e2ee4c", + "research-runtime": "7e7d6e5fc050b5bde5073f0939e5a9652a1f68be0a75104cc0b486ec0cd745a6" + }, + "stable_ids_unchanged": true + }, + "candidate_components": { + "creator-web": {"health": "healthy", "revision": "87ec7efd0fa7bdbed22c1c0c67d671907029d7bb"}, + "core-runtime": {"health": "healthy", "revision": "87ec7efd0fa7bdbed22c1c0c67d671907029d7bb"}, + "cognitive-worker": {"health": "healthy", "revision": "87ec7efd0fa7bdbed22c1c0c67d671907029d7bb"}, + "research-runtime": {"health": "healthy", "revision": "87ec7efd0fa7bdbed22c1c0c67d671907029d7bb"} + }, + "auth_harness": { + "approved_harness_found": false, + "available_mechanisms": ["/api/session", "tests/creator_web/verify-remote.ps1", "tests/cognitive_advisory/verify-remote.ps1"], + "gap": "available remote scripts require production password or bootstrap token; no sanctioned test principal/service harness without reading a credential was found", + "browser_or_secret_bypass_used": false + }, + "telemetry_observation": { + "started": false, + "reason": "no legitimate authenticated generation traffic; candidate was intentionally not promoted", + "v1_paths_deleted": false + }, + "deployment_incident": { + "occurred": true, + "cause": "first attempted checkout used stale local origin/gen2-integration ref and ran pre-guard script", + "effect": "core stable container was recreated once; image revision remained 3182092 and stable traffic stayed active", + "recovery": "subsequent protected candidate-only deployment used FETCH_HEAD 87ec7ef and proved stable IDs unchanged" + } +}