Compare commits
60
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
2ee5da8114 | ||
|
|
91f0c9e799 | ||
|
|
cb66a60db2 | ||
|
|
6a4f973e38 | ||
|
|
87ec7efd0f | ||
|
|
51e33e7283 | ||
|
|
e8f35f6393 | ||
|
|
8babf6ae1f | ||
|
|
d0210e5aed | ||
|
|
ebdfaa0769 | ||
|
|
9e0faa7c47 | ||
|
|
7f8a05ccfb | ||
|
|
c94d04acd0 | ||
|
|
bb866cd2ab | ||
|
|
892266a4c8 | ||
|
|
ac531c3ec3 | ||
|
|
4cd50a0937 | ||
|
|
8b751257e8 | ||
|
|
cbbb60e081 | ||
|
|
46e3cfc445 | ||
|
|
5832c49ad8 | ||
|
|
b13d12583b | ||
|
|
9d5473d260 | ||
|
|
50b508b641 | ||
|
|
8400cdd92d | ||
|
|
ec1860d441 | ||
|
|
134ba26e08 | ||
|
|
b82b8b874f | ||
|
|
5c9d4663f8 | ||
|
|
ec9d434650 | ||
|
|
a8a60ce29e | ||
|
|
96cf66a876 | ||
|
|
381b52d06d | ||
|
|
4ceec5a316 | ||
|
|
93a9c1a3bd | ||
|
|
f3ad7d682b | ||
|
|
387de89efe | ||
|
|
346e0acb6b | ||
|
|
3efcbbded8 | ||
|
|
a0b742b1d5 | ||
|
|
86aacea99e | ||
|
|
2a8e4e1b81 | ||
|
|
d64080ef2b | ||
|
|
59d32143bf | ||
|
|
031ba7666c | ||
|
|
93feb9bf2d | ||
|
|
a957172d3c | ||
|
|
3182092d5c | ||
|
|
8aa5179d47 | ||
|
|
5c95916648 | ||
|
|
5b5875d0f4 | ||
|
|
550738f3ca | ||
|
|
ad851a98aa | ||
|
|
62fd265bb7 | ||
|
|
e1c2e5f59c | ||
|
|
41b5c57d05 | ||
|
|
d0103bbf8e | ||
|
|
e9c3df26ae | ||
|
|
916b93517a | ||
|
|
44ac0203a9 |
@@ -6,6 +6,7 @@ import (
|
||||
"time"
|
||||
|
||||
"git.cin.su/m/Assistant/contracts/epistemics"
|
||||
"git.cin.su/m/Assistant/platform/telemetry"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
)
|
||||
|
||||
@@ -36,6 +37,11 @@ func (r *EpistemicsRepository) Ping(ctx context.Context) error {
|
||||
// TruthRuntimeStatus is a read-only projection of durable truth bundles.
|
||||
// It does not promote claims, revise beliefs, or mutate epistemic state.
|
||||
func (r *EpistemicsRepository) TruthRuntimeStatus(ctx context.Context) (TruthRuntimeStatus, error) {
|
||||
started := time.Now()
|
||||
outcome := "SUCCESS"
|
||||
defer func() {
|
||||
telemetry.RecordCompatibility("truth-bundles-status-reader", "epistemics-repository", "read", "research-status", true, false, true, outcome, "truth-status", "epistemics", started)
|
||||
}()
|
||||
status := TruthRuntimeStatus{
|
||||
ClaimsByVerification: map[string]int64{},
|
||||
BeliefsByState: map[string]int64{},
|
||||
@@ -43,6 +49,7 @@ func (r *EpistemicsRepository) TruthRuntimeStatus(ctx context.Context) (TruthRun
|
||||
ObservedAtUnixMillis: time.Now().UnixMilli(),
|
||||
}
|
||||
if err := r.pool.QueryRow(ctx, `SELECT count(*) FROM epistemics.truth_bundles`).Scan(&status.BundleCount); err != nil {
|
||||
outcome = "FAILURE"
|
||||
return TruthRuntimeStatus{}, err
|
||||
}
|
||||
queries := []struct {
|
||||
@@ -56,18 +63,21 @@ func (r *EpistemicsRepository) TruthRuntimeStatus(ctx context.Context) (TruthRun
|
||||
for _, item := range queries {
|
||||
rows, err := r.pool.Query(ctx, item.query)
|
||||
if err != nil {
|
||||
outcome = "FAILURE"
|
||||
return TruthRuntimeStatus{}, err
|
||||
}
|
||||
for rows.Next() {
|
||||
var state string
|
||||
var count int64
|
||||
if err := rows.Scan(&state, &count); err != nil {
|
||||
outcome = "FAILURE"
|
||||
rows.Close()
|
||||
return TruthRuntimeStatus{}, err
|
||||
}
|
||||
item.destination[state] = count
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
outcome = "FAILURE"
|
||||
rows.Close()
|
||||
return TruthRuntimeStatus{}, err
|
||||
}
|
||||
@@ -218,7 +228,13 @@ func (r *EpistemicsRepository) SaveReflectionCandidate(ctx context.Context, valu
|
||||
}
|
||||
|
||||
func (r *EpistemicsRepository) SaveTruthBundle(ctx context.Context, value epistemics.TruthBundle, recordedAtUnixMillis int64) error {
|
||||
started := time.Now()
|
||||
outcome := "SUCCESS"
|
||||
defer func() {
|
||||
telemetry.RecordCompatibility("truth-bundles-writer", "epistemics-repository", "write", "persisted-research", true, false, true, outcome, string(value.Claim.ID), "epistemics", started)
|
||||
}()
|
||||
if value.Source.ID == "" || value.Snapshot.ID == "" || value.Excerpt.ID == "" || value.Claim.ID == "" || value.Belief.ID == "" || value.Citation.ID == "" || recordedAtUnixMillis < 1 {
|
||||
outcome = "INVALID"
|
||||
return fmt.Errorf("invalid truth bundle")
|
||||
}
|
||||
_, err := r.pool.Exec(ctx, `INSERT INTO epistemics.truth_bundles (
|
||||
@@ -232,5 +248,8 @@ func (r *EpistemicsRepository) SaveTruthBundle(ctx context.Context, value episte
|
||||
value.Excerpt.ID, value.Excerpt.SnapshotID, value.Excerpt.Text, value.Excerpt.Start, value.Excerpt.End,
|
||||
value.Observation.ID, value.Observation.ObservedAt, value.Claim.ID, value.Claim.Statement, value.Claim.ObservationID, value.Claim.CitationID, value.Claim.Verification,
|
||||
value.Belief.ID, value.Belief.ClaimID, value.Belief.State, value.Citation.ID, value.Citation.ExcerptID, value.Citation.Evidence, value.Citation.Coverage, value.Citation.Conflict, value.Citation.ResolvedAtUnixMillis, recordedAtUnixMillis)
|
||||
if err != nil {
|
||||
outcome = "FAILURE"
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -13,6 +13,7 @@ import (
|
||||
"git.cin.su/m/Assistant/contracts/interaction"
|
||||
capabilitydomain "git.cin.su/m/Assistant/modules/capability/domain"
|
||||
epistemicsdomain "git.cin.su/m/Assistant/modules/epistemics/domain"
|
||||
"git.cin.su/m/Assistant/platform/telemetry"
|
||||
)
|
||||
|
||||
const reminderResponseSkillID = capability.SkillProfileID("skill-obvious-reminder-response-v1")
|
||||
@@ -160,6 +161,7 @@ func (r learningRecorder) recordFastPathDecline(ctx context.Context, input fastP
|
||||
if r.epistemics == nil || input.Intent.ID == "" || input.Utterance.ID == "" {
|
||||
return
|
||||
}
|
||||
telemetry.RecordCompatibility("fast-path-v1", "core-runtime", "decision", "decline-or-escalate", true, true, false, "OBSERVED", string(input.Utterance.CorrelationID), string(input.Utterance.PrincipalID), input.At)
|
||||
experience, err := epistemicsdomain.FastPathDeclineExperience(
|
||||
epistemics.OriginalIntentRef(input.Intent.ID),
|
||||
epistemics.SourceUtteranceRef(input.Utterance.ID),
|
||||
@@ -273,9 +275,11 @@ func (r learningRecorder) updateReminderSkill(ctx context.Context, experience ep
|
||||
}
|
||||
profile, err := r.capability.LoadSkillProfile(ctx, reminderResponseSkillID)
|
||||
if err != nil {
|
||||
telemetry.RecordCompatibility("skill-profile-v1", "core-runtime", "read", "fast-path-learning", true, false, true, "FAILURE", string(experience.OriginalIntentID), string(experience.SourceUtteranceID), time.UnixMilli(experience.RecordedAtUnixMillis))
|
||||
log.Printf("learning skill profile unavailable: %v", err)
|
||||
return
|
||||
}
|
||||
telemetry.RecordCompatibility("skill-profile-v1", "core-runtime", "read", "fast-path-learning", true, false, true, "SUCCESS", string(experience.OriginalIntentID), string(experience.SourceUtteranceID), time.UnixMilli(experience.RecordedAtUnixMillis))
|
||||
next, err := capabilitydomain.ObserveSkillOutcome(profile, capability.SkillOutcomeObservation{
|
||||
ExperienceRef: capability.SkillEvidenceRef("epistemics:experience:" + string(experience.ID)),
|
||||
Outcome: outcome,
|
||||
@@ -287,9 +291,11 @@ func (r learningRecorder) updateReminderSkill(ctx context.Context, experience ep
|
||||
return
|
||||
}
|
||||
if err := r.capability.SaveSkillProfile(ctx, next); err != nil {
|
||||
telemetry.RecordCompatibility("skill-profile-v1", "core-runtime", "write", "fast-path-learning", true, false, true, "FAILURE", string(experience.OriginalIntentID), string(experience.SourceUtteranceID), time.UnixMilli(experience.RecordedAtUnixMillis))
|
||||
log.Printf("learning skill profile save unavailable: %v", err)
|
||||
return
|
||||
}
|
||||
telemetry.RecordCompatibility("skill-profile-v1", "core-runtime", "write", "fast-path-learning", true, false, true, "SUCCESS", string(experience.OriginalIntentID), string(experience.SourceUtteranceID), time.UnixMilli(experience.RecordedAtUnixMillis))
|
||||
}
|
||||
|
||||
func outcomeForHumanResponse(response continuity.HumanResponse) (epistemics.OutcomeClass, int64) {
|
||||
|
||||
@@ -24,6 +24,7 @@ import (
|
||||
workdomain "git.cin.su/m/Assistant/modules/work/domain"
|
||||
"git.cin.su/m/Assistant/platform/ids"
|
||||
platformsecrets "git.cin.su/m/Assistant/platform/secrets"
|
||||
"git.cin.su/m/Assistant/platform/telemetry"
|
||||
"git.cin.su/m/Assistant/runtime/health"
|
||||
"github.com/jackc/pgx/v5"
|
||||
"io"
|
||||
@@ -65,6 +66,8 @@ func main() {
|
||||
if creatorUsername == "" {
|
||||
log.Fatal("CREATOR_USERNAME is required")
|
||||
}
|
||||
telemetry.RecordCompatibility("non-bootstrap-env-consumers", "core-runtime", "read", "runtime-startup-configuration", true, true, true, "OBSERVED", "startup", "core-runtime", time.Now())
|
||||
telemetry.RecordCompatibility("migration-compatibility-switches", "core-runtime", "observe", "runtime-startup-switch-scan", true, false, false, "NOT_OBSERVED", "startup", "core-runtime", time.Now())
|
||||
repository, err := postgres.OpenSituation(context.Background(), postgres.ContinuityConfig{Host: os.Getenv("POSTGRES_HOST"), Port: os.Getenv("POSTGRES_PORT"), Database: os.Getenv("POSTGRES_DATABASE"), User: os.Getenv("POSTGRES_USER"), Password: os.Getenv("POSTGRES_PASSWORD")})
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
@@ -553,9 +556,13 @@ func main() {
|
||||
storedFocus, err := repository.ActiveProjectFocus(r.Context(), session.ActorID, session.PrincipalID)
|
||||
if err == nil {
|
||||
previousFocus = &storedFocus
|
||||
telemetry.RecordCompatibility("active-project-focus-read", "core-runtime", "read", "chat-focus-load", true, false, true, "SUCCESS", utterance.CorrelationID, string(session.PrincipalID), now)
|
||||
} else if !errors.Is(err, pgx.ErrNoRows) {
|
||||
telemetry.RecordCompatibility("active-project-focus-read", "core-runtime", "read", "chat-focus-load", true, false, true, "FAILURE", utterance.CorrelationID, string(session.PrincipalID), now)
|
||||
http.Error(w, "project focus unavailable", http.StatusServiceUnavailable)
|
||||
return
|
||||
} else {
|
||||
telemetry.RecordCompatibility("active-project-focus-read", "core-runtime", "read", "chat-focus-load", true, false, true, "EMPTY", utterance.CorrelationID, string(session.PrincipalID), now)
|
||||
}
|
||||
var effectiveTaskMessage = interactiondomain.StripAssistantVocative(utterance.Content)
|
||||
creativeUserRequest := interactiondomain.IsCreativeRequest(utterance.Content)
|
||||
@@ -582,6 +589,9 @@ func main() {
|
||||
}
|
||||
conversationHistory = append(conversationHistory, recentUserHistory...)
|
||||
gen2Context := interactiondomain.BuildScopedGen2Context(conversationHistory, string(session.PrincipalID))
|
||||
if gen2Context.UsedPersonalMemoryFallback {
|
||||
telemetry.RecordCompatibility("personal-memory-fallback", "core-runtime", "read", "gen2-context-empty", true, true, false, "USED", utterance.CorrelationID, string(session.PrincipalID), now)
|
||||
}
|
||||
retrievedMemoryReply := interactiondomain.RecallPersonalPreference(conversationHistory, effectiveTaskMessage)
|
||||
stableConversationalReply, stableConversational := interactiondomain.StableConversationalReply(
|
||||
effectiveTaskMessage,
|
||||
@@ -780,9 +790,11 @@ func main() {
|
||||
activeFocusProvenance := situation.ProjectResolutionProvenance("")
|
||||
if semantic.ActiveFocus != nil {
|
||||
if err := repository.SaveActiveProjectFocus(r.Context(), *semantic.ActiveFocus); err != nil {
|
||||
telemetry.RecordCompatibility("active-project-focus-write", "core-runtime", "write", "semantic-focus-update", true, false, true, "FAILURE", utterance.CorrelationID, string(session.PrincipalID), now)
|
||||
http.Error(w, "save project focus", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
telemetry.RecordCompatibility("active-project-focus-write", "core-runtime", "write", "semantic-focus-update", true, false, true, "SUCCESS", utterance.CorrelationID, string(session.PrincipalID), now)
|
||||
activeFocusName = semantic.ActiveFocus.ProjectDisplayName
|
||||
activeFocusProvenance = semantic.ActiveFocus.Provenance
|
||||
} else if len(semantic.Projects) > 0 && semantic.Projects[0].Provenance == situation.ProjectResolutionInheritedFocus && previousFocus != nil {
|
||||
|
||||
+90
-4
@@ -45,7 +45,7 @@ func main() {
|
||||
fatal(err)
|
||||
}
|
||||
|
||||
s := &server{lifeboat: lifeboat, stableUpstream: stable, candidate: candidate, client: &http.Client{Timeout: 5 * time.Second}}
|
||||
s := &server{lifeboat: lifeboat, stableUpstream: stable, candidate: candidate, client: &http.Client{Timeout: 30 * time.Second}}
|
||||
public := http.NewServeMux()
|
||||
public.HandleFunc("GET /healthz", s.health)
|
||||
public.HandleFunc("GET /readyz", s.ready)
|
||||
@@ -53,8 +53,11 @@ func main() {
|
||||
control := http.NewServeMux()
|
||||
control.HandleFunc("GET /v1/state", s.state)
|
||||
control.HandleFunc("POST /v1/releases/start", s.start)
|
||||
control.HandleFunc("POST /v1/releases/start-set", s.startSet)
|
||||
control.HandleFunc("POST /v1/releases/baseline", s.baseline)
|
||||
control.HandleFunc("POST /v1/releases/baseline-set", s.baselineSet)
|
||||
control.HandleFunc("POST /v1/releases/observe", s.observe)
|
||||
control.HandleFunc("POST /v1/releases/observe-set", s.observeSet)
|
||||
control.HandleFunc("POST /v1/releases/promote", s.promote)
|
||||
control.HandleFunc("POST /v1/releases/rollback", s.rollback)
|
||||
control.HandleFunc("POST /v1/traffic/canary", s.canary)
|
||||
@@ -81,6 +84,10 @@ func (s *server) ready(w http.ResponseWriter, r *http.Request) {
|
||||
http.Error(w, "supervisor state unavailable", http.StatusServiceUnavailable)
|
||||
return
|
||||
}
|
||||
if state.StableReleaseSet != nil && !s.checkReleaseSet(r, state.StableReleaseSet) {
|
||||
http.Error(w, "stable release set unavailable", http.StatusServiceUnavailable)
|
||||
return
|
||||
}
|
||||
request, err := http.NewRequestWithContext(r.Context(), http.MethodGet, state.StableUpstream+"/healthz", nil)
|
||||
if err != nil {
|
||||
http.Error(w, "stable upstream unavailable", http.StatusServiceUnavailable)
|
||||
@@ -98,6 +105,28 @@ func (s *server) ready(w http.ResponseWriter, r *http.Request) {
|
||||
writeJSON(w, map[string]string{"runtime": "supervisor", "status": "ready"})
|
||||
}
|
||||
|
||||
func (s *server) checkReleaseSet(r *http.Request, releaseSet *supervisor.ReleaseSet) bool {
|
||||
if releaseSet == nil || !releaseSet.Validate() {
|
||||
return false
|
||||
}
|
||||
for _, name := range releaseSet.ComponentNames() {
|
||||
component := releaseSet.Components[name]
|
||||
request, err := http.NewRequestWithContext(r.Context(), http.MethodGet, component.ReadyURL, nil)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
response, err := s.client.Do(request)
|
||||
if err != nil || response.StatusCode/100 != 2 {
|
||||
if response != nil {
|
||||
response.Body.Close()
|
||||
}
|
||||
return false
|
||||
}
|
||||
response.Body.Close()
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func (s *server) proxy(w http.ResponseWriter, r *http.Request) {
|
||||
state, err := s.lifeboat.Snapshot()
|
||||
if err != nil {
|
||||
@@ -142,6 +171,14 @@ func (s *server) start(w http.ResponseWriter, r *http.Request) {
|
||||
ok, err := s.lifeboat.Start(release)
|
||||
respond(w, ok, err)
|
||||
}
|
||||
func (s *server) startSet(w http.ResponseWriter, r *http.Request) {
|
||||
var releaseSet supervisor.ReleaseSet
|
||||
if !decode(w, r, &releaseSet) {
|
||||
return
|
||||
}
|
||||
ok, err := s.lifeboat.StartSet(releaseSet)
|
||||
respond(w, ok, err)
|
||||
}
|
||||
func (s *server) baseline(w http.ResponseWriter, r *http.Request) {
|
||||
var release supervisor.Release
|
||||
if !decode(w, r, &release) {
|
||||
@@ -150,6 +187,14 @@ func (s *server) baseline(w http.ResponseWriter, r *http.Request) {
|
||||
ok, err := s.lifeboat.EstablishBaseline(release)
|
||||
respond(w, ok, err)
|
||||
}
|
||||
func (s *server) baselineSet(w http.ResponseWriter, r *http.Request) {
|
||||
var releaseSet supervisor.ReleaseSet
|
||||
if !decode(w, r, &releaseSet) {
|
||||
return
|
||||
}
|
||||
ok, err := s.lifeboat.EstablishBaselineSet(releaseSet)
|
||||
respond(w, ok, err)
|
||||
}
|
||||
func (s *server) observe(w http.ResponseWriter, r *http.Request) {
|
||||
var observation supervisor.Observation
|
||||
if !decode(w, r, &observation) {
|
||||
@@ -158,16 +203,57 @@ func (s *server) observe(w http.ResponseWriter, r *http.Request) {
|
||||
ok, err := s.lifeboat.Record(observation)
|
||||
respond(w, ok, err)
|
||||
}
|
||||
func (s *server) observeSet(w http.ResponseWriter, r *http.Request) {
|
||||
var observation supervisor.ReleaseSetObservation
|
||||
if !decode(w, r, &observation) {
|
||||
return
|
||||
}
|
||||
ok, err := s.lifeboat.RecordSet(observation)
|
||||
respond(w, ok, err)
|
||||
}
|
||||
func (s *server) promote(w http.ResponseWriter, _ *http.Request) {
|
||||
ok, err := s.lifeboat.Promote()
|
||||
state, snapshotErr := s.lifeboat.Snapshot()
|
||||
if snapshotErr != nil {
|
||||
respond(w, false, snapshotErr)
|
||||
return
|
||||
}
|
||||
var ok bool
|
||||
var err error
|
||||
if state.CandidateReleaseSet != nil {
|
||||
ok, err = s.lifeboat.PromoteSet()
|
||||
} else {
|
||||
ok, err = s.lifeboat.Promote()
|
||||
}
|
||||
respond(w, ok, err)
|
||||
}
|
||||
func (s *server) rollback(w http.ResponseWriter, _ *http.Request) {
|
||||
ok, err := s.lifeboat.Rollback()
|
||||
state, snapshotErr := s.lifeboat.Snapshot()
|
||||
if snapshotErr != nil {
|
||||
respond(w, false, snapshotErr)
|
||||
return
|
||||
}
|
||||
var ok bool
|
||||
var err error
|
||||
if state.CandidateReleaseSet != nil || state.RollbackTargetReleaseSet != nil {
|
||||
ok, err = s.lifeboat.RollbackSet()
|
||||
} else {
|
||||
ok, err = s.lifeboat.Rollback()
|
||||
}
|
||||
respond(w, ok, err)
|
||||
}
|
||||
func (s *server) canary(w http.ResponseWriter, _ *http.Request) {
|
||||
ok, err := s.lifeboat.SetTraffic(supervisor.TrafficCanaryHeader, s.stableUpstream, s.candidate)
|
||||
state, snapshotErr := s.lifeboat.Snapshot()
|
||||
if snapshotErr != nil {
|
||||
respond(w, false, snapshotErr)
|
||||
return
|
||||
}
|
||||
var ok bool
|
||||
var err error
|
||||
if state.CandidateReleaseSet != nil {
|
||||
ok, err = s.lifeboat.SetTrafficForSets(supervisor.TrafficCanaryHeader)
|
||||
} else {
|
||||
ok, err = s.lifeboat.SetTraffic(supervisor.TrafficCanaryHeader, s.stableUpstream, s.candidate)
|
||||
}
|
||||
respond(w, ok, err)
|
||||
}
|
||||
func (s *server) promoteTraffic(w http.ResponseWriter, _ *http.Request) {
|
||||
|
||||
@@ -83,6 +83,22 @@ services:
|
||||
depends_on: { postgres: { condition: service_healthy } }
|
||||
volumes:
|
||||
- assistant_core_secrets:/run/secrets:ro
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "wget -qO- http://localhost:8080/healthz >/dev/null && wget -qO- http://localhost:8080/readyz >/dev/null"]
|
||||
interval: 10s
|
||||
timeout: 20s
|
||||
retries: 20
|
||||
|
||||
core-runtime-candidate:
|
||||
profiles: ["assistant-canary"]
|
||||
extends:
|
||||
service: core-runtime
|
||||
restart: "no"
|
||||
environment:
|
||||
RESEARCH_RUNTIME_URL: http://research-runtime-candidate:8087
|
||||
depends_on:
|
||||
postgres: { condition: service_healthy }
|
||||
research-runtime-candidate: { condition: service_started }
|
||||
|
||||
agent-canary:
|
||||
profiles: ["agent-canary"]
|
||||
@@ -201,6 +217,19 @@ services:
|
||||
volumes:
|
||||
- assistant_core_secrets:/run/secrets:ro
|
||||
- assistant_research_sources:/var/lib/research-sources:ro
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "wget -qO- http://localhost:8087/healthz >/dev/null && wget -qO- http://localhost:8087/readyz >/dev/null"]
|
||||
interval: 10s
|
||||
timeout: 10s
|
||||
retries: 20
|
||||
|
||||
research-runtime-candidate:
|
||||
profiles: ["assistant-canary"]
|
||||
extends:
|
||||
service: research-runtime
|
||||
restart: "no"
|
||||
depends_on:
|
||||
postgres: { condition: service_healthy }
|
||||
|
||||
continuity-runtime:
|
||||
build: { context: .., dockerfile: deploy/images/runtime.Dockerfile, args: { TARGET: continuity-runtime, SOURCE_REVISION: "${ASSISTANT_SOURCE_REVISION:-unknown}" } }
|
||||
@@ -270,6 +299,17 @@ services:
|
||||
timeout: 20s
|
||||
retries: 20
|
||||
|
||||
cognitive-worker-candidate:
|
||||
profiles: ["assistant-canary"]
|
||||
extends:
|
||||
service: cognitive-worker
|
||||
restart: "no"
|
||||
environment:
|
||||
CORE_RUNTIME_URL: http://core-runtime-candidate:8080
|
||||
depends_on:
|
||||
core-runtime-candidate: { condition: service_started }
|
||||
cognitive-lifeboat: { condition: service_healthy }
|
||||
|
||||
cognitive-lifeboat:
|
||||
image: ghcr.io/ggml-org/llama.cpp:server
|
||||
restart: unless-stopped
|
||||
@@ -351,6 +391,22 @@ services:
|
||||
extends:
|
||||
service: creator-web
|
||||
restart: "no"
|
||||
environment:
|
||||
CORE_RUNTIME_URL: http://core-runtime-candidate:8080
|
||||
CONTINUITY_RUNTIME_URL: http://continuity-runtime:8081
|
||||
COMMIT_RUNTIME_URL: http://commit-runtime:8082
|
||||
COGNITIVE_RUNTIME_URL: http://cognitive-worker-candidate:8083
|
||||
AGENT_WORKER_URL: http://agent-worker:8090
|
||||
depends_on:
|
||||
core-runtime-candidate: { condition: service_started }
|
||||
continuity-runtime: { condition: service_started }
|
||||
commit-runtime: { condition: service_started }
|
||||
cognitive-worker-candidate: { condition: service_healthy }
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "wget -qO- http://localhost:8084/healthz >/dev/null"]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 20
|
||||
|
||||
volumes:
|
||||
assistant_pg:
|
||||
|
||||
@@ -6,6 +6,7 @@ ENV_FILE="${ENV_FILE:-/home/m/assistant-deploy/production-compose.env}"
|
||||
COMPOSE_FILE="$BASE_DIR/deploy/docker-compose.prod.yml"
|
||||
SUPERVISOR_CONTROL_URL="${SUPERVISOR_CONTROL_URL:-http://127.0.0.1:${SUPERVISOR_CONTROL_PORT:-18085}}"
|
||||
REVISION="${1:?candidate revision is required}"
|
||||
HEALTH_ATTEMPTS="${HEALTH_ATTEMPTS:-360}"
|
||||
|
||||
set -a
|
||||
. "$ENV_FILE"
|
||||
@@ -16,28 +17,111 @@ control() { curl -fsS --max-time 10 -X POST "$SUPERVISOR_CONTROL_URL$1" "${@:2}"
|
||||
fail_rollback() {
|
||||
control /v1/traffic/rollback || true
|
||||
control /v1/releases/rollback || true
|
||||
echo "production canary failed; stable traffic rollback requested" >&2
|
||||
echo "production ReleaseSet canary failed; stable ReleaseSet rollback requested" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
compose --profile assistant-canary build --build-arg "SOURCE_REVISION=$REVISION" creator-web-candidate
|
||||
compose --profile assistant-canary up -d --no-build creator-web-candidate
|
||||
for attempt in $(seq 1 30); do
|
||||
status="$(docker inspect -f '{{if .State.Health}}{{.State.Health.Status}}{{else}}unknown{{end}}' assistant-creator-web-candidate-1 2>/dev/null || true)"
|
||||
[[ "$status" == healthy ]] && break
|
||||
[[ "$attempt" == 30 ]] && fail_rollback
|
||||
sleep 2
|
||||
services=(creator-web-candidate core-runtime-candidate cognitive-worker-candidate research-runtime-candidate)
|
||||
containers=(assistant-creator-web-candidate-1 assistant-core-runtime-candidate-1 assistant-cognitive-worker-candidate-1 assistant-research-runtime-candidate-1)
|
||||
stable_services=(creator-web core-runtime cognitive-worker research-runtime)
|
||||
|
||||
stable_ids() {
|
||||
local service container id
|
||||
for service in "${stable_services[@]}"; do
|
||||
container="assistant-${service}-1"
|
||||
id=$(docker inspect -f '{{.Id}}' "$container" 2>/dev/null || true)
|
||||
[[ -n "$id" ]] || { echo "stable container missing: $container" >&2; return 1; }
|
||||
printf '%s=%s\n' "$service" "$id"
|
||||
done
|
||||
}
|
||||
|
||||
assert_stable_ids_unchanged() {
|
||||
local after
|
||||
after=$(stable_ids) || fail_rollback
|
||||
[[ "$after" == "$stable_ids_before" ]] || {
|
||||
echo "stable container IDs changed during candidate operation" >&2
|
||||
diff -u <(printf '%s\n' "$stable_ids_before") <(printf '%s\n' "$after") >&2 || true
|
||||
fail_rollback
|
||||
}
|
||||
}
|
||||
|
||||
stable_ids_before=$(stable_ids) || exit 1
|
||||
|
||||
compose --profile assistant-canary build --build-arg "SOURCE_REVISION=$REVISION" "${services[@]}"
|
||||
|
||||
wait_healthy() {
|
||||
local container="$1" status="" ready=""
|
||||
ready=""
|
||||
for attempt in $(seq 1 "$HEALTH_ATTEMPTS"); do
|
||||
status=$(docker inspect -f '{{if .State.Health}}{{.State.Health.Status}}{{else}}unknown{{end}}' "$container" 2>/dev/null || true)
|
||||
if [[ "$status" == healthy ]]; then ready=true; break; fi
|
||||
sleep 2
|
||||
done
|
||||
[[ "$ready" == true ]] || fail_rollback
|
||||
}
|
||||
|
||||
compose --profile assistant-canary up -d --no-build --no-deps research-runtime-candidate core-runtime-candidate
|
||||
wait_healthy assistant-research-runtime-candidate-1
|
||||
wait_healthy assistant-core-runtime-candidate-1
|
||||
compose --profile assistant-canary up -d --no-build --no-deps cognitive-worker-candidate
|
||||
wait_healthy assistant-cognitive-worker-candidate-1
|
||||
compose --profile assistant-canary up -d --no-build --no-deps creator-web-candidate
|
||||
wait_healthy assistant-creator-web-candidate-1
|
||||
assert_stable_ids_unchanged
|
||||
|
||||
for container in "${containers[@]}"; do
|
||||
image_revision=$(docker inspect -f '{{index .Config.Labels "org.opencontainers.image.revision"}}' "$container" 2>/dev/null || true)
|
||||
[[ "$image_revision" == "$REVISION" ]] || fail_rollback
|
||||
done
|
||||
assert_stable_ids_unchanged
|
||||
|
||||
control /v1/releases/baseline -H 'Content-Type: application/json' --data "{\"id\":\"stable-$REVISION\",\"version\":\"$REVISION\"}" || fail_rollback
|
||||
control /v1/releases/start -H 'Content-Type: application/json' --data "{\"id\":\"candidate-$REVISION\",\"version\":\"$REVISION\"}" || fail_rollback
|
||||
control /v1/releases/observe -H 'Content-Type: application/json' --data "{\"release_id\":\"candidate-$REVISION\",\"ready\":true,\"reason\":\"candidate healthcheck passed\",\"observed_at_unix_millis\":$(date +%s%3N)}" || fail_rollback
|
||||
if [[ "${PREPARE_ONLY:-false}" == true ]]; then
|
||||
echo "production ReleaseSet candidate prepared revision=$REVISION"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
release_json=$(cat <<JSON
|
||||
{
|
||||
"id":"candidate-$REVISION",
|
||||
"revision":"$REVISION",
|
||||
"components":{
|
||||
"creator-web":{"name":"creator-web","revision":"$REVISION","upstream":"http://creator-web-candidate:8084","health_url":"http://creator-web-candidate:8084/healthz","ready_url":"http://creator-web-candidate:8084/healthz"},
|
||||
"core-runtime":{"name":"core-runtime","revision":"$REVISION","upstream":"http://core-runtime-candidate:8080","health_url":"http://core-runtime-candidate:8080/healthz","ready_url":"http://core-runtime-candidate:8080/readyz"},
|
||||
"cognitive-worker":{"name":"cognitive-worker","revision":"$REVISION","upstream":"http://cognitive-worker-candidate:8083","health_url":"http://cognitive-worker-candidate:8083/healthz","ready_url":"http://cognitive-worker-candidate:8083/readyz"},
|
||||
"research-runtime":{"name":"research-runtime","revision":"$REVISION","upstream":"http://research-runtime-candidate:8087","health_url":"http://research-runtime-candidate:8087/healthz","ready_url":"http://research-runtime-candidate:8087/readyz"}
|
||||
}
|
||||
}
|
||||
JSON
|
||||
)
|
||||
|
||||
stable_json=$(curl -fsS --max-time 10 "$SUPERVISOR_CONTROL_URL/v1/state")
|
||||
stable_set=$(printf '%s' "$stable_json" | jq -c '.stable_release_set // empty')
|
||||
if [[ -z "$stable_set" ]]; then
|
||||
stable_revision=$(printf '%s' "$stable_json" | jq -r '.current.version // "unknown"')
|
||||
stable_set=$(cat <<JSON
|
||||
{"id":"stable-$stable_revision","revision":"$stable_revision","components":{
|
||||
"creator-web":{"name":"creator-web","revision":"$stable_revision","upstream":"http://creator-web:8084","health_url":"http://creator-web:8084/healthz","ready_url":"http://creator-web:8084/readyz"},
|
||||
"core-runtime":{"name":"core-runtime","revision":"$stable_revision","upstream":"http://core-runtime:8080","health_url":"http://core-runtime:8080/healthz","ready_url":"http://core-runtime:8080/readyz"},
|
||||
"cognitive-worker":{"name":"cognitive-worker","revision":"$stable_revision","upstream":"http://cognitive-worker:8083","health_url":"http://cognitive-worker:8083/healthz","ready_url":"http://cognitive-worker:8083/readyz"},
|
||||
"research-runtime":{"name":"research-runtime","revision":"$stable_revision","upstream":"http://research-runtime:8087","health_url":"http://research-runtime:8087/healthz","ready_url":"http://research-runtime:8087/readyz"}}}
|
||||
JSON
|
||||
)
|
||||
curl -fsS --max-time 10 -X POST "$SUPERVISOR_CONTROL_URL/v1/releases/baseline-set" -H 'Content-Type: application/json' --data "$stable_set" >/dev/null || fail_rollback
|
||||
fi
|
||||
|
||||
curl -fsS --max-time 10 -X POST "$SUPERVISOR_CONTROL_URL/v1/releases/start-set" -H 'Content-Type: application/json' --data "$release_json" >/dev/null || fail_rollback
|
||||
assert_stable_ids_unchanged
|
||||
observation=$(printf '{"release_id":"candidate-%s","ready":true,"reason":"all ReleaseSet healthchecks and image revisions passed","observed_at_unix_millis":%s,"components":{"creator-web":{"revision":"%s","health":true,"ready":true},"core-runtime":{"revision":"%s","health":true,"ready":true},"cognitive-worker":{"revision":"%s","health":true,"ready":true},"research-runtime":{"revision":"%s","health":true,"ready":true}}}\n' "$REVISION" "$(date +%s%3N)" "$REVISION" "$REVISION" "$REVISION" "$REVISION")
|
||||
curl -fsS --max-time 10 -X POST "$SUPERVISOR_CONTROL_URL/v1/releases/observe-set" -H 'Content-Type: application/json' --data "$observation" >/dev/null || fail_rollback
|
||||
assert_stable_ids_unchanged
|
||||
control /v1/traffic/canary || fail_rollback
|
||||
assert_stable_ids_unchanged
|
||||
|
||||
headers="$(mktemp)"
|
||||
headers=$(mktemp)
|
||||
trap 'rm -f "$headers"' EXIT
|
||||
if ! curl -fsS --max-time 10 -D "$headers" -H 'X-Assistant-Canary: true' "https://assistant.cin.su/login" -o /dev/null; then fail_rollback; fi
|
||||
grep -qi '^X-Assistant-Traffic-Target: candidate' "$headers" || fail_rollback
|
||||
|
||||
control /v1/releases/promote || fail_rollback
|
||||
echo "production canary promoted revision=$REVISION"
|
||||
assert_stable_ids_unchanged
|
||||
echo "production ReleaseSet canary promoted revision=$REVISION"
|
||||
|
||||
@@ -0,0 +1,32 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
BASE_DIR="${REPO_DIR:-$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)}"
|
||||
ENV_FILE="${ENV_FILE:-/home/m/assistant-deploy/production-compose.env}"
|
||||
COMPOSE_FILE="$BASE_DIR/deploy/docker-compose.prod.yml"
|
||||
|
||||
set -a
|
||||
. "$ENV_FILE"
|
||||
set +a
|
||||
|
||||
compose() { docker compose --env-file "$ENV_FILE" -f "$COMPOSE_FILE" "$@"; }
|
||||
stable_services=(creator-web core-runtime cognitive-worker research-runtime)
|
||||
stable_ids() {
|
||||
local service container id
|
||||
for service in "${stable_services[@]}"; do
|
||||
container="assistant-${service}-1"
|
||||
id=$(docker inspect -f '{{.Id}}' "$container" 2>/dev/null || true)
|
||||
[[ -n "$id" ]] || { echo "stable container missing: $container" >&2; return 1; }
|
||||
printf '%s=%s\n' "$service" "$id"
|
||||
done
|
||||
}
|
||||
|
||||
before=$(stable_ids)
|
||||
compose up -d --no-build --no-deps supervisor
|
||||
after=$(stable_ids)
|
||||
[[ "$before" == "$after" ]] || {
|
||||
echo "stable container IDs changed while updating supervisor" >&2
|
||||
diff -u <(printf '%s\n' "$before") <(printf '%s\n' "$after") >&2 || true
|
||||
exit 1
|
||||
}
|
||||
printf '%s\n' "$after"
|
||||
@@ -1 +1 @@
|
||||
{"task_file":"dev/tasks/GEN2-REM-CORE-M5-LOCAL-MEDIUM-001.json","task_id":"GEN2-REM-CORE-M5-LOCAL-MEDIUM-001","task_revision":1}
|
||||
{"task_file":"dev/tasks/GEN2-COMPAT-OBS-001-production-compatibility-observation.json","task_id":"GEN2-COMPAT-OBS-001","task_revision":2}
|
||||
|
||||
+2
-2
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"task_id": "GEN2-028",
|
||||
"task_revision": 1,
|
||||
"task_revision": 2,
|
||||
"architecture_revision": "ARCH-GEN2-000",
|
||||
"base_repo_revision": "gen2-integration",
|
||||
"risk_tier": "E",
|
||||
@@ -16,7 +16,7 @@
|
||||
"modules/**",
|
||||
"contracts/**",
|
||||
"cmd/**",
|
||||
"adapters/**",
|
||||
"adapters/postgres/**",
|
||||
"runtime/**",
|
||||
"platform/**",
|
||||
"docs/**",
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"task_id": "GEN2-029",
|
||||
"task_revision": 1,
|
||||
"task_revision": 2,
|
||||
"architecture_revision": "ARCH-GEN2-000",
|
||||
"base_repo_revision": "gen2-integration",
|
||||
"risk_tier": "E",
|
||||
@@ -16,7 +16,7 @@
|
||||
"modules/**",
|
||||
"contracts/**",
|
||||
"cmd/**",
|
||||
"adapters/**",
|
||||
"adapters/postgres/**",
|
||||
"runtime/**",
|
||||
"platform/**",
|
||||
"database/migrations/*.sql",
|
||||
|
||||
+2
-2
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"task_id": "GEN2-030",
|
||||
"task_revision": 1,
|
||||
"task_revision": 2,
|
||||
"architecture_revision": "ARCH-GEN2-000",
|
||||
"base_repo_revision": "gen2-integration",
|
||||
"risk_tier": "E",
|
||||
@@ -20,7 +20,7 @@
|
||||
"modules/**",
|
||||
"contracts/**",
|
||||
"cmd/**",
|
||||
"adapters/**",
|
||||
"adapters/postgres/**",
|
||||
"runtime/**",
|
||||
"platform/**",
|
||||
"tests/**"
|
||||
|
||||
@@ -0,0 +1,65 @@
|
||||
{
|
||||
"task_id": "GEN2-COMPAT-OBS-001",
|
||||
"task_revision": 2,
|
||||
"architecture_revision": "ARCH-GEN2-000",
|
||||
"base_repo_revision": "gen2-integration",
|
||||
"risk_tier": "E",
|
||||
"intent": "Добавить bounded redacted production telemetry для оставшихся V1 compatibility paths без изменения их поведения, owners или canonical storage.",
|
||||
"requirement_ids": ["REQ-GEN2-COMPAT-OBS-001-TELEMETRY", "REQ-GEN2-COMPAT-OBS-001-OBSERVATION", "REQ-GEN2-COMPAT-OBS-001-ROLLBACK"],
|
||||
"allowed_change_scope": {
|
||||
"modules": [],
|
||||
"paths": [
|
||||
"platform/telemetry/**",
|
||||
"cmd/core-runtime/**",
|
||||
"cmd/research-runtime/**",
|
||||
"modules/interaction/domain/**",
|
||||
"modules/capability/domain/**",
|
||||
"adapters/postgres/**",
|
||||
"docs/**",
|
||||
"tests/**",
|
||||
"deploy/run-production-canary.sh",
|
||||
"deploy/update-supervisor.sh",
|
||||
"dev/ACTIVE_TASK.json",
|
||||
"dev/tasks/GEN2-COMPAT-OBS-001-production-compatibility-observation.json"
|
||||
]
|
||||
},
|
||||
"forbidden_change_scope": {
|
||||
"paths": ["security/**", "go.mod", "go.sum", "architecture/**", ".gitea/workflows/**", "database/**", "adapters/models/**"]
|
||||
},
|
||||
"allowed_architecture_delta": {
|
||||
"new_cross_module_dependencies": 0,
|
||||
"new_canonical_owners": 0,
|
||||
"new_privileges": 1,
|
||||
"new_third_party_dependencies": 0,
|
||||
"new_database_owner_roles": 0
|
||||
},
|
||||
"contracts": {
|
||||
"consume": ["existing runtime logs and capability usage observation mechanisms"],
|
||||
"may_change": ["telemetry event shape only; no semantic/domain contract change"]
|
||||
},
|
||||
"semantic_owner": "Migration/Governance; telemetry is an operational projection and never a canonical semantic owner.",
|
||||
"canonical_state": ["no change", "V1 compatibility behavior and Gen2 canonical writers remain unchanged"],
|
||||
"state_transitions": ["path invocation -> redacted telemetry event -> bounded aggregation/observation -> retirement classification"],
|
||||
"side_effects_and_privileges": ["stdout/stderr telemetry only; no external mutation, secret access, or user content logging"],
|
||||
"persistence_and_migrations": ["no schema or migration change; telemetry is emitted to existing process logs"],
|
||||
"failure_retry_restart": ["telemetry failure must never change or block the compatibility path", "rollback is code revert to the pre-instrumentation commit"],
|
||||
"assumptions": ["production Supervisor deployment is authorized by this stage", "observation window is bounded and redacted", "no credentials are read"],
|
||||
"acceptance_scenarios": [
|
||||
"COMPAT-OBS-001-SCN-01: each listed path has a typed redacted event shape",
|
||||
"COMPAT-OBS-001-SCN-02: events include invocation, component, read/write, reason, Gen2 availability, fallback/primary, outcome and safe correlation",
|
||||
"COMPAT-OBS-001-SCN-03: no event contains content, secrets, tokens or raw user identifiers",
|
||||
"COMPAT-OBS-001-SCN-04: FAST tests prove instrumentation is non-blocking and path behavior is unchanged",
|
||||
"COMPAT-OBS-001-SCN-05: deployed production smoke produces observable compatibility telemetry",
|
||||
"COMPAT-OBS-001-EXIT: production observation can classify each path ACTIVE_REQUIRED, ACTIVE_FALLBACK, SHADOW_ONLY, UNUSED or UNKNOWN without claiming retirement from absence of evidence"
|
||||
],
|
||||
"required_tests": [
|
||||
"FAST: targeted go test for changed runtime/domain/adapter packages",
|
||||
"FAST: targeted go vet for changed packages",
|
||||
"FAST governance: go run ./cmd/devctl preflight -active dev/ACTIVE_TASK.json",
|
||||
"FAST scope: go run ./cmd/devctl check-diff -active dev/ACTIVE_TASK.json",
|
||||
"FAST independent: go run ./cmd/devctl/independent",
|
||||
"FAST whitespace: git diff --check gen2-integration --",
|
||||
"production smoke: Supervisor candidate deployment and redacted log observation"
|
||||
],
|
||||
"terminal_gap_states": ["BLOCKED_BY_SPEC_GAP", "BLOCKED_BY_ARCHITECTURE_GAP", "BLOCKED_BY_MISSING_CAPABILITY", "BLOCKED_BY_CONFLICT", "REQUIRES_MIGRATION_DECISION"]
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
{
|
||||
"task_id": "GEN2-DEPLOY-002",
|
||||
"task_revision": 1,
|
||||
"architecture_revision": "ARCH-GEN2-000",
|
||||
"base_repo_revision": "gen2-integration",
|
||||
"risk_tier": "E",
|
||||
"intent": "Расширить существующий Supervisor до безопасного управления единым candidate/stable ReleaseSet из creator-web, core-runtime, cognitive-worker и research-runtime без изменения Gen2 semantic ownership.",
|
||||
"requirement_ids": ["REQ-GEN2-DEPLOY-002-RELEASE-SET", "REQ-GEN2-DEPLOY-002-CANDIDATE-TOPOLOGY", "REQ-GEN2-DEPLOY-002-ROLLBACK", "REQ-GEN2-DEPLOY-002-COMPAT-OBS"],
|
||||
"allowed_change_scope": {"modules": ["runtime/supervisor"], "paths": ["runtime/supervisor/**", "cmd/supervisor/**", "deploy/**", "docs/**", "tests/**", "dev/ACTIVE_TASK.json", "dev/tasks/GEN2-DEPLOY-002-supervised-release-set.json"]},
|
||||
"forbidden_change_scope": {"paths": ["architecture/**", "database/**", "modules/**", "contracts/**", "security/**", "go.mod", "go.sum", ".gitea/workflows/**", "adapters/models/**"]},
|
||||
"allowed_architecture_delta": {"new_cross_module_dependencies": 0, "new_canonical_owners": 0, "new_privileges": 1, "new_third_party_dependencies": 0, "new_database_owner_roles": 0},
|
||||
"contracts": {"consume": ["existing Supervisor durable state and production compose", "existing canary header and stable rollback controls"], "may_change": ["Supervisor release control payloads", "deployment topology and candidate routing configuration"]},
|
||||
"semantic_owner": "Supervisor/deployment control plane; ReleaseSet is operational release state and not semantic canonical state.",
|
||||
"canonical_state": ["no semantic owner or schema change", "previous stable release and evidence remain retained"],
|
||||
"state_transitions": ["candidate ReleaseSet -> component health/revision checks -> canary -> observation -> promotion or whole-set rollback"],
|
||||
"side_effects_and_privileges": ["production deployment only through approved control plane; no direct stable replacement; no credential reads"],
|
||||
"persistence_and_migrations": ["durable Supervisor release state remains in existing external Supervisor volume; no database migration"],
|
||||
"assumptions": ["the existing production Supervisor and compose control plane remain authoritative for deployment", "candidate services may share existing durable infrastructure only under current ownership and compatibility rules", "no credentials are read by this task"],
|
||||
"failure_retry_restart": ["candidate component failure routes all traffic to previous stable ReleaseSet", "Supervisor restart reloads ReleaseSet state", "previous stable is retained until promotion is complete"],
|
||||
"acceptance_scenarios": ["DEPLOY-002-SCN-01: complete ReleaseSet has all four components and one revision", "DEPLOY-002-SCN-02: candidate dependencies resolve only to candidate services", "DEPLOY-002-SCN-03: health/readiness/revision checks cover all required components", "DEPLOY-002-SCN-04: canary and stable paths are isolated", "DEPLOY-002-SCN-05: forced candidate failure rolls back the complete ReleaseSet", "DEPLOY-002-SCN-06: Supervisor restart preserves stable/candidate/LKG state", "DEPLOY-002-SCN-07: production telemetry revision starts bounded observation after controlled deployment"],
|
||||
"required_tests": ["go test ./runtime/supervisor ./cmd/supervisor", "FAST: targeted supervisor/release-set tests", "FAST: compose config validation", "FAST: routing and revision consistency tests", "FAST: rollback/restart persistence tests", "FAST: targeted go vet", "FAST governance: preflight/check-diff/independent", "FAST whitespace: git diff --check", "PRODUCTION: candidate ReleaseSet drill and telemetry observation start"],
|
||||
"terminal_gap_states": ["BLOCKED_BY_SPEC_GAP", "BLOCKED_BY_ARCHITECTURE_GAP", "BLOCKED_BY_MISSING_CAPABILITY", "BLOCKED_BY_CONFLICT", "REQUIRES_MIGRATION_DECISION"]
|
||||
}
|
||||
@@ -1,5 +1,18 @@
|
||||
# Assistant project status
|
||||
|
||||
## Gen2 M6 checkpoint — 2026-08-30
|
||||
|
||||
- `GEN2-028 = COMPLETED_WITH_EVIDENCE`: V1 compatibility inventory is recorded in `docs/evidence/gen2-028-v1-compatibility-inventory-2026-08-30.json`. Current production callers remain for PersonalMemory fallback, ActiveProjectFocus, truth_bundles, ordinary ENV reads, SkillProfile/FastPath; no path was proven `SAFE_TO_RETIRE`.
|
||||
- `GEN2-029 = COMPLETED_WITH_SAFE_NOOP`: no V1 runtime path or historical evidence was deleted or disabled. The decision and rollback point are recorded in `docs/evidence/gen2-029-v1-retirement-decision-2026-08-30.json`; `FULL_RETIREMENT` was not applicable because no irreversible retirement was performed.
|
||||
- `GEN2-030_STATUS = ACCEPTANCE_IN_PROGRESS`: this status is a documentation checkpoint, not a final Gen2 PASS. The final gate must confirm current production and repository evidence; the M6 retirement gap remains that active V1 compatibility callers have not yet been proven unused.
|
||||
- `UI_BROWSER_ACCEPTANCE = DEFERRED_BY_OWNER`; it is intentionally excluded from core/M6 acceptance and is neither counted as PASS nor as a blocker.
|
||||
- `PROVIDER_GAPS = CPU_FAST_UNAVAILABLE; GPU_UNAVAILABLE`; Local MEDIUM remains the primary route and Lifeboat remains fallback per existing M5 evidence.
|
||||
- `GEN2_M6_FINAL_STATUS = NOT_FINAL_PASS_RETIREMENT_GAP_REMAINS`; FULL_FINAL technical checks passed, but no final M6 baseline tag is created because active V1 compatibility callers were not proven safe to retire.
|
||||
- `GEN2_M6_FINAL_EVIDENCE = docs/evidence/gen2-030-final-acceptance-2026-08-30.json`.
|
||||
- `GEN2_FINAL_STATUS = GEN2_CORE_ACCEPTED / V1_COMPATIBILITY_RETAINED / UI_DEFERRED`.
|
||||
- `GEN2-COMPAT-OBS-001 = INSTRUMENTATION_COMMITTED; PRODUCTION_OBSERVATION_BLOCKED_BY_MISSING_CAPABILITY`; evidence: `docs/evidence/gen2-compat-obs-001-blocked-2026-08-30.json`. No V1 path is `SAFE_TO_RETIRE`; no retirement task was created.
|
||||
- `NEXT_EVOLUTION_QUEUE = instrument bounded compatibility usage, observe, then retire only individually proven paths under a future FULL_RETIREMENT gate; no Gen3 or new feature work is started.
|
||||
|
||||
## Current audit delta — 2026-08-24
|
||||
|
||||
- `CURRENT_MODEL_SCOPE = LOCAL_FIRST_V1`; one locally available Qwen3-8B or Gemma-4 conversational model is sufficient for the first stage, with the local Lifeboat as recovery fallback. External models remain supported as optional capability, but their absence must not block work: reflexes, skills and local executors provide degraded continuation. External/GPU qualification and promotion are intentionally deferred by DEV-000270.
|
||||
|
||||
@@ -0,0 +1,98 @@
|
||||
{
|
||||
"evidence_id": "gen2-028-v1-compatibility-inventory-2026-08-30",
|
||||
"task_id": "GEN2-028",
|
||||
"task_revision": 2,
|
||||
"source_revision": "93feb9bf2df8f2e2cfd4d0259d39c869b19cad08",
|
||||
"scope": "static repository inventory plus read-only production container and Supervisor observations; no code, schema, or runtime mutation",
|
||||
"canonical_writer_rule": "unchanged; no second Memory owner and no writer cutover performed",
|
||||
"production_observation": {
|
||||
"deployment_revision": "3182092d5cc758f5a536ce10f5428b5d782bb961",
|
||||
"core_container": "running, restart_count=0; /healthz=200",
|
||||
"cognitive_worker": "running, restart_count=0, health=healthy",
|
||||
"lifeboat": "running, restart_count=0, health=healthy",
|
||||
"supervisor": "running, restart_count=0, health=healthy",
|
||||
"traffic_mode": "CANDIDATE",
|
||||
"last_known_good": "3182092d5cc758f5a536ce10f5428b5d782bb961",
|
||||
"legacy_named_runtime_containers": "none observed"
|
||||
},
|
||||
"inventory": [
|
||||
{
|
||||
"id": "personal-memory-context-text",
|
||||
"paths": ["modules/interaction/domain/personal_memory.go", "modules/interaction/domain/gen2_context.go", "cmd/core-runtime/main.go"],
|
||||
"role": "V1 rebuildable context projection and deterministic preference recall",
|
||||
"runtime_status": "ACTIVE_COMPATIBILITY_PATH",
|
||||
"evidence": ["BuildScopedGen2Context is called from the production chat handler", "CompilePersonalMemory and ContextText remain in that call path", "empty Gen2 projection returns personal-memory-v1-fallback", "targeted gen2_context and personal_memory tests exist"],
|
||||
"retirement": "KEEP_LONGER",
|
||||
"rollback": "retain the existing read fallback and revert the reader switch without deleting utterance history"
|
||||
},
|
||||
{
|
||||
"id": "active-project-focus",
|
||||
"paths": ["adapters/postgres/situation_repository.go", "cmd/core-runtime/main.go", "contracts/situation/types.go"],
|
||||
"role": "V1 project-centric focus compatibility projection",
|
||||
"runtime_status": "ACTIVE_READ_AND_WRITE_COMPATIBILITY_PATH",
|
||||
"evidence": ["core chat loads ActiveProjectFocus for each authenticated message", "core persists it after semantic interpretation", "PostgreSQL adapter reads and upserts situation.active_project_focus", "Focus V2 projection exists but does not remove these callers"],
|
||||
"retirement": "KEEP_LONGER",
|
||||
"rollback": "keep the projection readable and restore the prior focus reader if a V2 reader is rolled back"
|
||||
},
|
||||
{
|
||||
"id": "truth-bundles",
|
||||
"paths": ["adapters/postgres/epistemics_repository.go", "modules/epistemics/application/persisted_research.go", "database/migrations/034_epistemics_truth_bundles.sql", "tests/research_contracts/truth_persistence_sql_test.go"],
|
||||
"role": "V1 denormalized epistemic compatibility/status and research persistence",
|
||||
"runtime_status": "ACTIVE_WRITER_AND_READ_COMPATIBILITY_PATH",
|
||||
"evidence": ["SaveTruthBundle remains an application repository contract used by persisted research", "TruthRuntimeStatus reads counts and grouped status from epistemics.truth_bundles", "migration 034 and its contract test are retained", "no production evidence permits declaring the reader unused"],
|
||||
"retirement": "KEEP_LONGER",
|
||||
"rollback": "retain table and reader; revert normalized reader selection without deleting historical bundles"
|
||||
},
|
||||
{
|
||||
"id": "ordinary-env-configuration",
|
||||
"paths": ["cmd/core-runtime/main.go", "cmd/work-graph-runtime/main.go", "cmd/reflection-worker/main.go", "cmd/notification-receiver/main.go"],
|
||||
"role": "ENV configuration and secret/bootstrap resolution",
|
||||
"runtime_status": "ACTIVE_MIXED_BOOTSTRAP_AND_NON_BOOTSTRAP_READS",
|
||||
"evidence": ["production composition supplies runtime settings through process configuration", "source contains direct os.Getenv reads for listener, database, provider and runtime settings", "secret values were not read or emitted", "no complete production observation proves every non-bootstrap consumer is unused"],
|
||||
"retirement": "KEEP_LONGER",
|
||||
"rollback": "restore the prior ENV fallback for a single setting while Configuration remains authoritative"
|
||||
},
|
||||
{
|
||||
"id": "skill-profile-v1",
|
||||
"paths": ["adapters/postgres/capability_repository.go", "cmd/core-runtime/learning.go", "contracts/capability/resolution.go"],
|
||||
"role": "V1 SkillProfile storage/read compatibility and FastPath learning",
|
||||
"runtime_status": "ACTIVE_READ_AND_WRITE_PATH",
|
||||
"evidence": ["core exposes RecentSkillProfiles and records FastPath decline learning", "learning.go loads and saves SkillProfile", "PostgreSQL adapter writes and joins capability.skill_profiles", "explicit Skill/Reflex types exist but do not prove these callers are dead"],
|
||||
"retirement": "KEEP_LONGER",
|
||||
"rollback": "retain SkillProfile rows and reader; revert Skill V2 selection without data deletion"
|
||||
},
|
||||
{
|
||||
"id": "implicit-fast-path",
|
||||
"paths": ["modules/capability/domain/skill.go", "modules/capability/domain/reflex.go", "cmd/core-runtime/learning.go"],
|
||||
"role": "V1 FastPath behavior and fallback-to-thinking compatibility",
|
||||
"runtime_status": "ACTIVE_COMPATIBILITY_PATH",
|
||||
"evidence": ["FastPath decisions are implemented and tested", "core records FastPath decline experiences", "ReflexFallback explicitly maps escalation to skill->reasoning", "no runtime telemetry separates all FastPath invocations from explicit Reflex use"],
|
||||
"retirement": "KEEP_LONGER",
|
||||
"rollback": "restore FastPath decline/escalation behavior; preserve evidence and Skill/Reflex revisions"
|
||||
},
|
||||
{
|
||||
"id": "migration-feature-switches",
|
||||
"paths": ["development/GEN2_MIGRATION_MAP.md", "runtime/**", "modules/**"],
|
||||
"role": "documented migration switch family and compatibility strategy",
|
||||
"runtime_status": "UNKNOWN_NO_DEDICATED_RUNTIME_TELEMETRY_FOUND",
|
||||
"evidence": ["migration map defines switch names such as Focus, Context, Epistemics, Config and Skill strategies", "repository scan found no complete operational switch registry/telemetry proving zero production use", "test-only legacy mode references are not sufficient production evidence"],
|
||||
"retirement": "KEEP_LONGER",
|
||||
"rollback": "do not remove a switch until production observation identifies its callers and a reversible disable path"
|
||||
}
|
||||
],
|
||||
"coverage": {
|
||||
"searched": ["modules", "contracts", "cmd", "adapters", "runtime", "platform", "tests", "docs", "database/migrations"],
|
||||
"terms": ["PersonalMemory", "CompilePersonalMemory", "ContextText", "ActiveProjectFocus", "truth_bundles", "skill_profiles", "FastPath", "os.Getenv", "LookupEnv", "compatibility", "legacy", "feature switch"],
|
||||
"limitations": ["No secret values were inspected", "Container logs and Supervisor state provide no per-path legacy usage counters", "A code reference is evidence of availability/callability, not a claim that every branch executed in the observation window"]
|
||||
},
|
||||
"retirement_list": [
|
||||
{"path": "personal-memory-context-text", "decision": "KEEP_LONGER", "reason": "active production chat fallback and no usage counter"},
|
||||
{"path": "active-project-focus", "decision": "KEEP_LONGER", "reason": "core read/write callers remain active"},
|
||||
{"path": "truth-bundles", "decision": "KEEP_LONGER", "reason": "research writer and status reader remain active"},
|
||||
{"path": "ordinary-env-configuration", "decision": "KEEP_LONGER", "reason": "mixed bootstrap/non-bootstrap consumers remain"},
|
||||
{"path": "skill-profile-v1", "decision": "KEEP_LONGER", "reason": "core learning and repository read/write callers remain active"},
|
||||
{"path": "implicit-fast-path", "decision": "KEEP_LONGER", "reason": "active learning/fallback path and no complete production telemetry"},
|
||||
{"path": "migration-feature-switches", "decision": "KEEP_LONGER", "reason": "no complete runtime observation evidence"}
|
||||
],
|
||||
"conclusion": "GEN2-028 EXIT: every inventoried legacy surface is classified ACTIVE_COMPATIBILITY_PATH, ACTIVE_READ_AND_WRITE_COMPATIBILITY_PATH, or UNKNOWN_NO_DEDICATED_RUNTIME_TELEMETRY_FOUND. No surface is SAFE_TO_RETIRE on current evidence; GEN2-029 must not perform irreversible deletion or disablement based on this inventory."
|
||||
}
|
||||
@@ -0,0 +1,30 @@
|
||||
{
|
||||
"evidence_id": "gen2-029-v1-retirement-decision-2026-08-30",
|
||||
"task_id": "GEN2-029",
|
||||
"task_revision": 2,
|
||||
"source_revision": "59d32143bf4ee7b4a78cb714ec0cbd76d4d8bdf5",
|
||||
"decision": "NO_IRREVERSIBLE_RETIREMENT",
|
||||
"reason": "GEN2-028 found no SAFE_TO_RETIRE path. Active callers remain for PersonalMemory fallback, ActiveProjectFocus, truth_bundles, ordinary ENV reads, SkillProfile/FastPath; migration switches lack complete production telemetry.",
|
||||
"changed_runtime_paths": [],
|
||||
"changed_schema_paths": [],
|
||||
"retired_paths": [],
|
||||
"preserved": [
|
||||
"human utterances and interaction history",
|
||||
"epistemic observations, claims, experiences, truth_bundles and audit lineage",
|
||||
"ActiveProjectFocus compatibility projection",
|
||||
"PersonalMemory read fallback",
|
||||
"SkillProfile/FastPath compatibility",
|
||||
"ordinary ENV fallback consumers until bootstrap/config observation is complete",
|
||||
"all existing Gen2 canonical writers and owner boundaries"
|
||||
],
|
||||
"full_retirement_gate": {
|
||||
"status": "NOT_APPLICABLE",
|
||||
"reason": "No irreversible V1 schema/data deletion or runtime disablement was authorized or performed; the execution contract requires the gate immediately before such a retirement.",
|
||||
"repository_full_test": "NOT_RUN_BY_DESIGN; no retirement change risk to justify expanding beyond the task's FAST/no-op decision"
|
||||
},
|
||||
"rollback": {
|
||||
"point": "59d32143bf4ee7b4a78cb714ec0cbd76d4d8bdf5",
|
||||
"method": "keep current compatibility paths and revert this evidence-only decision; no schema rollback or data deletion needed"
|
||||
},
|
||||
"conclusion": "No unsafe V1 path was removed. Historical evidence remains available, and there is no new dual writer. Retirement remains pending an observation window that can prove each path unused or a future approved cutover decision."
|
||||
}
|
||||
@@ -0,0 +1,43 @@
|
||||
{
|
||||
"evidence_id": "gen2-030-final-acceptance-2026-08-30",
|
||||
"task_id": "GEN2-030",
|
||||
"task_revision": 2,
|
||||
"source_revision": "2a8e4e1b81226a1acedfd746ebc2aeb0ac11158f",
|
||||
"status": "NOT_FINAL_PASS_RETIREMENT_GAP_REMAINS",
|
||||
"full_final_gate": {
|
||||
"go_test_all": "PASS",
|
||||
"go_vet_all": "PASS",
|
||||
"architecture_checks": "PASS",
|
||||
"governance_preflight_check_diff_independent_whitespace": "PASS",
|
||||
"migration_consistency": "PASS; 52 files, unique range 001..052; no M6 schema changes",
|
||||
"production_core_smoke": "PASS; core /healthz=200, /readyz=200; cognitive-worker healthy; Lifeboat healthy",
|
||||
"supervisor_recovery_readiness": "PASS; current and last-known-good 3182092d5cc758f5a536ce10f5428b5d782bb961; rollback target retained"
|
||||
},
|
||||
"architecture_reconciliation": {
|
||||
"registry": "YAML/JSON mirrors agree; Gen2 owners and runtime topology are represented",
|
||||
"canonical_memory_owner": "none; retrieval/context remain projection fabric",
|
||||
"schema": "migrations 046..052 are present and numbered without duplicates; historical 001..045 retained",
|
||||
"production_revision": "3182092d5cc758f5a536ce10f5428b5d782bb961"
|
||||
},
|
||||
"m6_result": {
|
||||
"gen2_028": "completed inventory; no SAFE_TO_RETIRE path proven",
|
||||
"gen2_029": "completed safe no-op; no runtime/schema/data deletion",
|
||||
"gen2_030": "full final verification passed, but final M6 exit is not signed",
|
||||
"full_retirement": "NOT_APPLICABLE; no irreversible retirement was performed",
|
||||
"final_tag": "NOT_CREATED; acceptance is not a final PASS"
|
||||
},
|
||||
"ui_browser": "DEFERRED_BY_OWNER; intentionally excluded, neither PASS nor blocker",
|
||||
"provider_gaps": ["CPU_FAST unavailable", "GPU unavailable"],
|
||||
"remaining_gaps": [
|
||||
"Per-path production observation telemetry is missing for migration switches and several compatibility readers",
|
||||
"PersonalMemory, ActiveProjectFocus, truth_bundles, ordinary ENV, SkillProfile/FastPath still have active callers and require a future evidence-backed retirement/cutover task",
|
||||
"M6 cannot claim complete V1 retirement until those paths are proven unused or explicitly retained by an accepted architecture decision"
|
||||
],
|
||||
"next_evolution_queue": [
|
||||
"Instrument bounded, redacted usage observations for each remaining compatibility path",
|
||||
"Run an approved observation window and re-evaluate retirement candidates",
|
||||
"Retire only individually proven paths with FULL_RETIREMENT immediately before irreversible change",
|
||||
"Keep UI/browser acceptance deferred by owner",
|
||||
"Keep CPU_FAST/GPU provider qualification separate from Core acceptance"
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,30 @@
|
||||
{
|
||||
"evidence_id": "gen2-compat-obs-001-blocked-2026-08-30",
|
||||
"task_id": "GEN2-COMPAT-OBS-001",
|
||||
"task_revision": 1,
|
||||
"status": "BLOCKED_BY_MISSING_CAPABILITY",
|
||||
"instrumentation_commit": "3efcbbd",
|
||||
"integration_commit": "346e0ac",
|
||||
"final_status": "GEN2_CORE_ACCEPTED / V1_COMPATIBILITY_RETAINED / UI_DEFERRED",
|
||||
"instrumented_paths": ["PersonalMemory fallback", "ActiveProjectFocus read", "ActiveProjectFocus write", "truth_bundles writer", "truth_bundles status reader", "non-bootstrap ENV consumers", "SkillProfile V1", "FastPath V1", "migration/compatibility switches"],
|
||||
"telemetry_contract": {
|
||||
"fields": ["invocation_count", "unique_request_key_hash", "unique_session_key_hash", "last_used_at", "caller_component", "read_or_write", "reason", "gen2_path_available", "fallback_or_primary", "success_or_failure", "correlation_id_hash"],
|
||||
"content_policy": "No memory content, secrets, or user data is emitted; identifiers are hashed immediately and telemetry is an operational projection, not a canonical owner."
|
||||
},
|
||||
"fast_checks": {
|
||||
"go_test": "PASS: targeted telemetry, core-runtime, interaction, capability, and postgres packages",
|
||||
"go_vet": "PASS: same targeted packages",
|
||||
"preflight": "PASS",
|
||||
"check_diff": "PASS",
|
||||
"independent_governance": "PASS",
|
||||
"git_diff_check": "PASS"
|
||||
},
|
||||
"production_observation": {"deployment": "NOT_RUN", "smoke": "NOT_RUN", "observation_window": "NOT_STARTED", "production_mutation": "NONE"},
|
||||
"blocker": {
|
||||
"supervisor_topology": "Supervisor has stable/candidate routing for creator-web only; core-runtime, cognitive-worker, and research-runtime have no candidate targets in the production compose/startup model.",
|
||||
"missing_capability": "An approved production deployment flow capable of deploying the instrumented core/cognitive/research candidate, such as configured Portainer automation or an approved candidate-capable deployment topology.",
|
||||
"unsafe_alternative_not_used": "Directly replacing core-runtime with compose would mutate the stable runtime outside the Supervisor candidate path and was not performed."
|
||||
},
|
||||
"retirement": {"safe_to_retire": [], "retirement_tasks_created": [], "classification": "UNKNOWN until production observation evidence exists", "rollback": "All existing V1 compatibility paths and historical evidence remain intact."},
|
||||
"scope": {"schema_or_migrations_changed": false, "canonical_owner_changed": false, "behavior_changed": false, "ui_changed": false, "gen3_started": false}
|
||||
}
|
||||
@@ -0,0 +1,48 @@
|
||||
{
|
||||
"task_id": "GEN2-COMPAT-OBS-001",
|
||||
"task_revision": 2,
|
||||
"status": "BLOCKED_BY_MISSING_CAPABILITY",
|
||||
"observed_at_utc": "2026-08-30T10:13:06.5676592Z",
|
||||
"deployment_revision": "87ec7efd0fa7bdbed22c1c0c67d671907029d7bb",
|
||||
"deployment_scope": "candidate-only; no Supervisor traffic change or promotion",
|
||||
"stable_release_set": {
|
||||
"traffic_mode": "STABLE",
|
||||
"revision": "3182092d5cc758f5a536ce10f5428b5d782bb961",
|
||||
"container_ids_before_protected_operation": {
|
||||
"creator-web": "9578e7c7e0137ba36a7fc75ffaf9fce33be046e38531c9de99c0b75c2ce465f8",
|
||||
"core-runtime": "8281ba60b234825d6ee965887e8eef10e29a0783264c05d39cbe937f1b780530",
|
||||
"cognitive-worker": "987640770cad44a0b05a47dbf08a4b47a9225f8030f8bc81721fdc0834e2ee4c",
|
||||
"research-runtime": "7e7d6e5fc050b5bde5073f0939e5a9652a1f68be0a75104cc0b486ec0cd745a6"
|
||||
},
|
||||
"container_ids_after_protected_operation": {
|
||||
"creator-web": "9578e7c7e0137ba36a7fc75ffaf9fce33be046e38531c9de99c0b75c2ce465f8",
|
||||
"core-runtime": "8281ba60b234825d6ee965887e8eef10e29a0783264c05d39cbe937f1b780530",
|
||||
"cognitive-worker": "987640770cad44a0b05a47dbf08a4b47a9225f8030f8bc81721fdc0834e2ee4c",
|
||||
"research-runtime": "7e7d6e5fc050b5bde5073f0939e5a9652a1f68be0a75104cc0b486ec0cd745a6"
|
||||
},
|
||||
"stable_ids_unchanged": true
|
||||
},
|
||||
"candidate_components": {
|
||||
"creator-web": {"health": "healthy", "revision": "87ec7efd0fa7bdbed22c1c0c67d671907029d7bb"},
|
||||
"core-runtime": {"health": "healthy", "revision": "87ec7efd0fa7bdbed22c1c0c67d671907029d7bb"},
|
||||
"cognitive-worker": {"health": "healthy", "revision": "87ec7efd0fa7bdbed22c1c0c67d671907029d7bb"},
|
||||
"research-runtime": {"health": "healthy", "revision": "87ec7efd0fa7bdbed22c1c0c67d671907029d7bb"}
|
||||
},
|
||||
"auth_harness": {
|
||||
"approved_harness_found": false,
|
||||
"available_mechanisms": ["/api/session", "tests/creator_web/verify-remote.ps1", "tests/cognitive_advisory/verify-remote.ps1"],
|
||||
"gap": "available remote scripts require production password or bootstrap token; no sanctioned test principal/service harness without reading a credential was found",
|
||||
"browser_or_secret_bypass_used": false
|
||||
},
|
||||
"telemetry_observation": {
|
||||
"started": false,
|
||||
"reason": "no legitimate authenticated generation traffic; candidate was intentionally not promoted",
|
||||
"v1_paths_deleted": false
|
||||
},
|
||||
"deployment_incident": {
|
||||
"occurred": true,
|
||||
"cause": "first attempted checkout used stale local origin/gen2-integration ref and ran pre-guard script",
|
||||
"effect": "core stable container was recreated once; image revision remained 3182092 and stable traffic stayed active",
|
||||
"recovery": "subsequent protected candidate-only deployment used FETCH_HEAD 87ec7ef and proved stable IDs unchanged"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,46 @@
|
||||
{
|
||||
"task_id": "GEN2-COMPAT-OBS-001",
|
||||
"status": "BLOCKED_BY_DEPLOYMENT_CONTROL",
|
||||
"observed_at_utc": "2026-08-30T10:37:07.7219511Z",
|
||||
"browser_connector": {
|
||||
"authenticated_tab_visible": true,
|
||||
"stale_login_tab_also_present": true,
|
||||
"cookies_tokens_or_passwords_read": false
|
||||
},
|
||||
"requested_candidate_revision": "87ec7efd0fa7bdbed22c1c0c67d671907029d7bb",
|
||||
"candidate_prepare": "not_completed_as_a_safe_release_set_promotion",
|
||||
"failure": {
|
||||
"phase": "candidate creator startup after cognitive readiness",
|
||||
"result": "stable container IDs changed despite candidate-only --no-deps; Supervisor control became temporarily unavailable",
|
||||
"stable_ids_before": {
|
||||
"creator-web": "4df348752f1a6def05acb1934bfc23048902ad6f235a7da967806d481e9ce7b7",
|
||||
"core-runtime": "a739d2d5c713dde575f571b38e58ec33143994aa9e3bcc2f5d5fc765a50e1796",
|
||||
"cognitive-worker": "a33d61b32116711287a166e0cce6362dc10de321f920549ef8a344fe0b5efb2d",
|
||||
"research-runtime": "88cfade5cdce288576b1c73493f83c84d915ac82c3d16dc6d3fedb9579a4ba8c"
|
||||
},
|
||||
"stable_ids_after": {
|
||||
"creator-web": "5181e5c6b48eaed6c6ea85eff5668c0e675b72bb982eb9425d6f39f899378ec9",
|
||||
"core-runtime": "641e8f7ae8177e4638cf806d9be1bb6f8b5ac028deb0c0c410c6ff4139032bd5",
|
||||
"cognitive-worker": "2e6d46f5b533cfeda261ceafd65ad5b5873b8aa8166bffc4ae91033a6c994d0a",
|
||||
"research-runtime": "b15d9b0e116fbd18fa726e33541ef11fe1bafe4a85210ab5ee500466a20b400b"
|
||||
},
|
||||
"stable_ids_unchanged": false,
|
||||
"cause": "external production updater/Compose reconcile reacts to production checkout or candidate Compose operation; --no-deps alone does not isolate stable in this control plane"
|
||||
},
|
||||
"recovery": {
|
||||
"traffic_mode": "STABLE",
|
||||
"revision": "91f0c9e7999907a4ee7665b77742dd452c18b17b",
|
||||
"healthz": 200,
|
||||
"readyz": 200,
|
||||
"supervisor_baseline_aligned": true,
|
||||
"data_or_volumes_deleted": false
|
||||
},
|
||||
"observation": {
|
||||
"authenticated_workload_sent": false,
|
||||
"telemetry_promoted": false,
|
||||
"baseline_started": false,
|
||||
"tag_created": false,
|
||||
"v1_paths_deleted": false
|
||||
},
|
||||
"required_follow_up": "deployment control plane must isolate candidate Compose project or disable updater reconcile before safe ReleaseSet promotion"
|
||||
}
|
||||
@@ -0,0 +1,40 @@
|
||||
{
|
||||
"evidence_id": "gen2-deploy-002-blocked-2026-08-30",
|
||||
"task_id": "GEN2-DEPLOY-002",
|
||||
"task_revision": 1,
|
||||
"base_repo_revision": "gen2-integration",
|
||||
"status": "BLOCKED_BY_MISSING_CAPABILITY",
|
||||
"implementation_checks": {
|
||||
"release_set_tests": "PASS",
|
||||
"supervisor_go_test": "PASS",
|
||||
"supervisor_go_vet": "PASS",
|
||||
"compose_config": "PASS",
|
||||
"canary_script_syntax": "PASS",
|
||||
"governance": "PASS",
|
||||
"git_diff_check": "PASS"
|
||||
},
|
||||
"implemented": [
|
||||
"durable Supervisor ReleaseSet state for creator-web, core-runtime, cognitive-worker, and research-runtime",
|
||||
"parallel candidate services with candidate-only core/cognitive/research dependency wiring",
|
||||
"whole-set health/readiness/revision observation, canary, promotion and rollback controls",
|
||||
"stable/candidate/LKG persistence across Supervisor restart",
|
||||
"core and research runtime compose healthchecks"
|
||||
],
|
||||
"production": {
|
||||
"deployment": "NOT_RUN",
|
||||
"candidate_release_set": "NOT_STARTED",
|
||||
"telemetry_observation": "NOT_STARTED",
|
||||
"stable_mutation": "NONE"
|
||||
},
|
||||
"capability_evidence": {
|
||||
"docker_and_compose_on_production_host": "AVAILABLE",
|
||||
"supervisor_control_endpoint": "REACHABLE",
|
||||
"portainer_api_key_in_current_execution_context": "NOT_CONFIGURED",
|
||||
"portainer_endpoint_metadata_in_current_execution_context": "NOT_CONFIGURED",
|
||||
"existing_production_candidate_services": ["creator-web-candidate"],
|
||||
"missing_capability": "Approved production deployment automation or candidate-capable control-plane access for the four-component ReleaseSet."
|
||||
},
|
||||
"unsafe_action_not_taken": "Stable containers were not replaced directly and no production compose mutation was attempted.",
|
||||
"compatibility": "No V1 compatibility path was removed or disabled; no schema, migration, semantic owner, or Gen3 change.",
|
||||
"next_action": "Owner must provide or enable the approved candidate-capable production deployment mechanism; then run the ReleaseSet drill and start bounded telemetry observation."
|
||||
}
|
||||
@@ -0,0 +1,29 @@
|
||||
{
|
||||
"evidence_id": "gen2-deploy-002-production-drill-2026-08-30",
|
||||
"task_id": "GEN2-DEPLOY-002",
|
||||
"task_revision": 1,
|
||||
"status": "BLOCKED_BY_MISSING_CAPABILITY",
|
||||
"integration_revision": "7f8a05c",
|
||||
"candidate_revision": "7f8a05c",
|
||||
"stable_revision_after_rollback": "3182092d5cc758f5a536ce10f5428b5d782bb961",
|
||||
"release_set": ["creator-web", "core-runtime", "cognitive-worker", "research-runtime"],
|
||||
"drill": {
|
||||
"candidate_health_readiness_revision": "PASS; all four candidate containers healthy and image revision checked",
|
||||
"candidate_dependency_isolation": "PASS; creator->core candidate, core->research candidate, core/cognitive candidate routes verified in compose wiring",
|
||||
"canary_header": "PASS; Supervisor selected candidate for X-Assistant-Canary=true",
|
||||
"forced_component_failure": "PASS; research-runtime-candidate stopped, ReleaseSet observation rejected with HTTP 409",
|
||||
"whole_set_rollback": "PASS; Supervisor restored stable 3182092 and traffic mode STABLE",
|
||||
"promotion": "PASS; healthy candidate was promoted through Supervisor",
|
||||
"supervisor_restart": "PASS; promoted ReleaseSet and rollback target survived restart",
|
||||
"post_rollback_health": "PASS; public health/readiness both HTTP 200"
|
||||
},
|
||||
"telemetry": {
|
||||
"startup_events_observed": ["non-bootstrap-env-consumers", "migration-compatibility-switches"],
|
||||
"authenticated_path_observation": "NOT_COMPLETED",
|
||||
"observation_window": "NOT_STARTED",
|
||||
"reason": "The existing browser session was stale and production requested re-authentication before chat submission. No credentials were read or entered."
|
||||
},
|
||||
"final_production_state": {"traffic_mode": "STABLE", "revision": "3182092d5cc758f5a536ce10f5428b5d782bb961", "candidate_containers": "retained but not serving stable traffic"},
|
||||
"not_done": ["telemetry confirmation for all nine paths", "bounded observation window", "retirement tasks", "main merge", "milestone tag"],
|
||||
"scope": {"schema_or_migrations_changed": false, "v1_paths_removed": false, "gen3_started": false, "ui_changed": false}
|
||||
}
|
||||
@@ -0,0 +1,77 @@
|
||||
// Package telemetry contains operational projections that do not own domain
|
||||
// state. Compatibility events are intentionally redacted and best-effort.
|
||||
package telemetry
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"log"
|
||||
"strconv"
|
||||
"time"
|
||||
)
|
||||
|
||||
// CompatibilityEvent is one bounded invocation observation. Aggregators can
|
||||
// derive counts and unique request/session counts from the redacted keys.
|
||||
// It deliberately contains no content, secret, or user identifier.
|
||||
type CompatibilityEvent struct {
|
||||
Event string `json:"event"`
|
||||
Path string `json:"path"`
|
||||
Component string `json:"component"`
|
||||
Access string `json:"access"`
|
||||
Reason string `json:"reason"`
|
||||
Gen2PathAvailable bool `json:"gen2_path_available"`
|
||||
Fallback bool `json:"fallback"`
|
||||
Primary bool `json:"primary"`
|
||||
Outcome string `json:"outcome"`
|
||||
InvocationCount int `json:"invocation_count"`
|
||||
UniqueRequestKey string `json:"unique_request_key"`
|
||||
UniqueSessionKey string `json:"unique_session_key"`
|
||||
SafeCorrelationID string `json:"correlation_id"`
|
||||
LastUsedAt string `json:"last_used_at"`
|
||||
ObservedAtUnixMillis int64 `json:"observed_at_unix_millis"`
|
||||
}
|
||||
|
||||
// NewCompatibilityEvent creates a redacted event. requestRef/sessionRef are
|
||||
// hashed immediately; callers must pass identifiers only, never content.
|
||||
func NewCompatibilityEvent(path, component, access, reason string, gen2PathAvailable, fallback, primary bool, outcome, requestRef, sessionRef string, now time.Time) CompatibilityEvent {
|
||||
if now.IsZero() {
|
||||
now = time.Now().UTC()
|
||||
}
|
||||
requestKey := digest(requestRef)
|
||||
sessionKey := digest(sessionRef)
|
||||
correlationInput := requestRef + "\x00" + sessionRef + "\x00" + path + "\x00" + strconv.FormatInt(now.UnixMilli(), 10)
|
||||
return CompatibilityEvent{
|
||||
Event: "compatibility_observation",
|
||||
Path: path,
|
||||
Component: component,
|
||||
Access: access,
|
||||
Reason: reason,
|
||||
Gen2PathAvailable: gen2PathAvailable,
|
||||
Fallback: fallback,
|
||||
Primary: primary,
|
||||
Outcome: outcome,
|
||||
InvocationCount: 1,
|
||||
UniqueRequestKey: requestKey,
|
||||
UniqueSessionKey: sessionKey,
|
||||
SafeCorrelationID: digest(correlationInput),
|
||||
LastUsedAt: now.UTC().Format(time.RFC3339Nano),
|
||||
ObservedAtUnixMillis: now.UnixMilli(),
|
||||
}
|
||||
}
|
||||
|
||||
// RecordCompatibility emits a structured redacted event. Telemetry is
|
||||
// best-effort and cannot change the compatibility path's result.
|
||||
func RecordCompatibility(path, component, access, reason string, gen2PathAvailable, fallback, primary bool, outcome, requestRef, sessionRef string, now time.Time) {
|
||||
event := NewCompatibilityEvent(path, component, access, reason, gen2PathAvailable, fallback, primary, outcome, requestRef, sessionRef, now)
|
||||
payload, err := json.Marshal(event)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
log.Printf("%s", payload)
|
||||
}
|
||||
|
||||
func digest(value string) string {
|
||||
sum := sha256.Sum256([]byte(value))
|
||||
return hex.EncodeToString(sum[:8])
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
package telemetry
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestCompatibilityEventRedactsRequestAndSessionReferences(t *testing.T) {
|
||||
event := NewCompatibilityEvent("personal-memory-fallback", "core-runtime", "read", "empty-gen2-projection", true, true, false, "SUCCESS", "request-secret-looking", "session-secret-looking", time.UnixMilli(42))
|
||||
if event.InvocationCount != 1 || event.ObservedAtUnixMillis != 42 || event.LastUsedAt == "" {
|
||||
t.Fatalf("event metadata incomplete: %+v", event)
|
||||
}
|
||||
if strings.Contains(event.UniqueRequestKey, "request-secret-looking") || strings.Contains(event.UniqueSessionKey, "session-secret-looking") || strings.Contains(event.SafeCorrelationID, "secret-looking") {
|
||||
t.Fatalf("event leaked reference: %+v", event)
|
||||
}
|
||||
if event.Path != "personal-memory-fallback" || !event.Gen2PathAvailable || !event.Fallback || event.Primary {
|
||||
t.Fatalf("event classification incorrect: %+v", event)
|
||||
}
|
||||
}
|
||||
@@ -92,12 +92,24 @@ func (p *PersistentLifeboat) Start(candidate Release) (bool, error) {
|
||||
return p.transition(func(value *Lifeboat) bool { return value.Start(candidate) })
|
||||
}
|
||||
|
||||
func (p *PersistentLifeboat) StartSet(candidate ReleaseSet) (bool, error) {
|
||||
return p.transition(func(value *Lifeboat) bool { return value.StartSet(candidate) })
|
||||
}
|
||||
|
||||
func (p *PersistentLifeboat) EstablishBaseline(release Release) (bool, error) {
|
||||
return p.transition(func(value *Lifeboat) bool { return value.EstablishBaseline(release) })
|
||||
}
|
||||
|
||||
func (p *PersistentLifeboat) EstablishBaselineSet(release ReleaseSet) (bool, error) {
|
||||
return p.transition(func(value *Lifeboat) bool { return value.EstablishBaselineSet(release) })
|
||||
}
|
||||
func (p *PersistentLifeboat) Record(observation Observation) (bool, error) {
|
||||
return p.transition(func(value *Lifeboat) bool { return value.Record(observation) })
|
||||
}
|
||||
|
||||
func (p *PersistentLifeboat) RecordSet(observation ReleaseSetObservation) (bool, error) {
|
||||
return p.transition(func(value *Lifeboat) bool { return value.RecordSet(observation) })
|
||||
}
|
||||
func (p *PersistentLifeboat) Rollback() (bool, error) {
|
||||
return p.transition(func(value *Lifeboat) bool { return value.Rollback() })
|
||||
}
|
||||
@@ -105,6 +117,10 @@ func (p *PersistentLifeboat) SetTraffic(mode TrafficMode, stableUpstream, candid
|
||||
return p.transition(func(value *Lifeboat) bool { return value.SetTraffic(mode, stableUpstream, candidateUpstream) })
|
||||
}
|
||||
|
||||
func (p *PersistentLifeboat) SetTrafficForSets(mode TrafficMode) (bool, error) {
|
||||
return p.transition(func(value *Lifeboat) bool { return value.SetTrafficForSets(mode) })
|
||||
}
|
||||
|
||||
func (p *PersistentLifeboat) Promote() (bool, error) {
|
||||
return p.transition(func(value *Lifeboat) bool {
|
||||
if value.Current.State != ReleaseHealthy || value.Current.ID == "" {
|
||||
@@ -114,6 +130,10 @@ func (p *PersistentLifeboat) Promote() (bool, error) {
|
||||
})
|
||||
}
|
||||
|
||||
func (p *PersistentLifeboat) PromoteSet() (bool, error) {
|
||||
return p.transition(func(value *Lifeboat) bool { return value.PromoteSet() })
|
||||
}
|
||||
|
||||
func (p *PersistentLifeboat) RollbackTraffic() (bool, error) {
|
||||
return p.transition(func(value *Lifeboat) bool {
|
||||
if !value.Rollback() {
|
||||
@@ -123,6 +143,10 @@ func (p *PersistentLifeboat) RollbackTraffic() (bool, error) {
|
||||
})
|
||||
}
|
||||
|
||||
func (p *PersistentLifeboat) RollbackSet() (bool, error) {
|
||||
return p.transition(func(value *Lifeboat) bool { return value.RollbackSet() })
|
||||
}
|
||||
|
||||
func (p *PersistentLifeboat) Snapshot() (Lifeboat, error) {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
package supervisor
|
||||
|
||||
import "sort"
|
||||
|
||||
type ReleaseState string
|
||||
|
||||
const (
|
||||
@@ -22,21 +24,209 @@ type Release struct {
|
||||
Version string `json:"version"`
|
||||
State ReleaseState `json:"state"`
|
||||
LastKnownGood bool `json:"last_known_good"`
|
||||
Set *ReleaseSet `json:"release_set,omitempty"`
|
||||
}
|
||||
|
||||
const (
|
||||
ComponentCreatorWeb = "creator-web"
|
||||
ComponentCoreRuntime = "core-runtime"
|
||||
ComponentCognitiveWorker = "cognitive-worker"
|
||||
ComponentResearchRuntime = "research-runtime"
|
||||
)
|
||||
|
||||
// ComponentRelease is operational deployment evidence. It is deliberately
|
||||
// separate from semantic/domain state and contains no credentials.
|
||||
type ComponentRelease struct {
|
||||
Name string `json:"name"`
|
||||
Revision string `json:"revision"`
|
||||
Upstream string `json:"upstream"`
|
||||
HealthURL string `json:"health_url"`
|
||||
ReadyURL string `json:"ready_url"`
|
||||
}
|
||||
|
||||
// ReleaseSet is the atomic deployment unit supervised by the control plane.
|
||||
type ReleaseSet struct {
|
||||
ID string `json:"id"`
|
||||
Revision string `json:"revision"`
|
||||
Components map[string]ComponentRelease `json:"components"`
|
||||
}
|
||||
|
||||
var requiredComponents = []string{ComponentCreatorWeb, ComponentCoreRuntime, ComponentCognitiveWorker, ComponentResearchRuntime}
|
||||
|
||||
func (s ReleaseSet) Validate() bool {
|
||||
if s.ID == "" || s.Revision == "" || len(s.Components) != len(requiredComponents) {
|
||||
return false
|
||||
}
|
||||
for _, name := range requiredComponents {
|
||||
component, ok := s.Components[name]
|
||||
if !ok || component.Name != name || component.Revision != s.Revision || component.Upstream == "" || component.HealthURL == "" || component.ReadyURL == "" {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func (s ReleaseSet) ComponentNames() []string {
|
||||
names := make([]string, 0, len(s.Components))
|
||||
for name := range s.Components {
|
||||
names = append(names, name)
|
||||
}
|
||||
sort.Strings(names)
|
||||
return names
|
||||
}
|
||||
|
||||
type Observation struct {
|
||||
ReleaseID string `json:"release_id"`
|
||||
Ready bool `json:"ready"`
|
||||
Reason string `json:"reason"`
|
||||
ObservedAtUnixMillis int64 `json:"observed_at_unix_millis"`
|
||||
}
|
||||
|
||||
type ReleaseSetObservation struct {
|
||||
ReleaseID string `json:"release_id"`
|
||||
Ready bool `json:"ready"`
|
||||
Reason string `json:"reason"`
|
||||
ObservedAtUnixMillis int64 `json:"observed_at_unix_millis"`
|
||||
Components map[string]ComponentObservation `json:"components"`
|
||||
}
|
||||
|
||||
type ComponentObservation struct {
|
||||
Revision string `json:"revision"`
|
||||
Health bool `json:"health"`
|
||||
Ready bool `json:"ready"`
|
||||
}
|
||||
type Lifeboat struct {
|
||||
Current Release `json:"current"`
|
||||
LastKnownGood Release `json:"last_known_good"`
|
||||
RollbackTarget Release `json:"rollback_target"`
|
||||
Observations []Observation `json:"observations"`
|
||||
TrafficMode TrafficMode `json:"traffic_mode"`
|
||||
StableUpstream string `json:"stable_upstream"`
|
||||
CandidateUpstream string `json:"candidate_upstream"`
|
||||
Current Release `json:"current"`
|
||||
LastKnownGood Release `json:"last_known_good"`
|
||||
RollbackTarget Release `json:"rollback_target"`
|
||||
Observations []Observation `json:"observations"`
|
||||
TrafficMode TrafficMode `json:"traffic_mode"`
|
||||
StableUpstream string `json:"stable_upstream"`
|
||||
CandidateUpstream string `json:"candidate_upstream"`
|
||||
StableReleaseSet *ReleaseSet `json:"stable_release_set,omitempty"`
|
||||
CandidateReleaseSet *ReleaseSet `json:"candidate_release_set,omitempty"`
|
||||
LastKnownGoodReleaseSet *ReleaseSet `json:"last_known_good_release_set,omitempty"`
|
||||
RollbackTargetReleaseSet *ReleaseSet `json:"rollback_target_release_set,omitempty"`
|
||||
}
|
||||
|
||||
func (l *Lifeboat) StartSet(candidate ReleaseSet) bool {
|
||||
if !candidate.Validate() {
|
||||
return false
|
||||
}
|
||||
copySet := candidate
|
||||
if l.LastKnownGoodReleaseSet != nil {
|
||||
previous := *l.LastKnownGoodReleaseSet
|
||||
l.RollbackTargetReleaseSet = &previous
|
||||
}
|
||||
l.CandidateReleaseSet = ©Set
|
||||
l.Current = Release{ID: candidate.ID, Version: candidate.Revision, State: ReleaseCandidate, Set: ©Set}
|
||||
return true
|
||||
}
|
||||
|
||||
func (l *Lifeboat) EstablishBaselineSet(stable ReleaseSet) bool {
|
||||
if !stable.Validate() {
|
||||
return false
|
||||
}
|
||||
copySet := stable
|
||||
l.StableReleaseSet = ©Set
|
||||
l.LastKnownGoodReleaseSet = ©Set
|
||||
l.Current = Release{ID: stable.ID, Version: stable.Revision, State: ReleaseHealthy, LastKnownGood: true, Set: ©Set}
|
||||
l.LastKnownGood = l.Current
|
||||
l.RollbackTarget = Release{}
|
||||
l.RollbackTargetReleaseSet = nil
|
||||
l.TrafficMode = TrafficStable
|
||||
return l.setSetUpstreams(©Set, nil)
|
||||
}
|
||||
|
||||
func (l *Lifeboat) setSetUpstreams(stable, candidate *ReleaseSet) bool {
|
||||
if stable == nil {
|
||||
return false
|
||||
}
|
||||
stableWeb, ok := stable.Components[ComponentCreatorWeb]
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
l.StableUpstream = stableWeb.Upstream
|
||||
if candidate != nil {
|
||||
candidateWeb, exists := candidate.Components[ComponentCreatorWeb]
|
||||
if !exists {
|
||||
return false
|
||||
}
|
||||
l.CandidateUpstream = candidateWeb.Upstream
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func (l *Lifeboat) RecordSet(o ReleaseSetObservation) bool {
|
||||
if l.CandidateReleaseSet == nil || o.ReleaseID == "" || o.ReleaseID != l.CandidateReleaseSet.ID || !o.Ready {
|
||||
l.Current.State = ReleaseFailed
|
||||
return false
|
||||
}
|
||||
if len(o.Components) != len(requiredComponents) {
|
||||
l.Current.State = ReleaseFailed
|
||||
return false
|
||||
}
|
||||
for _, name := range requiredComponents {
|
||||
component, ok := o.Components[name]
|
||||
expected := l.CandidateReleaseSet.Components[name]
|
||||
if !ok || !component.Health || !component.Ready || component.Revision != expected.Revision {
|
||||
l.Current.State = ReleaseFailed
|
||||
return false
|
||||
}
|
||||
}
|
||||
l.Current.State = ReleaseHealthy
|
||||
return true
|
||||
}
|
||||
|
||||
func (l *Lifeboat) SetTrafficForSets(mode TrafficMode) bool {
|
||||
if l.StableReleaseSet == nil || l.CandidateReleaseSet == nil {
|
||||
return false
|
||||
}
|
||||
if !l.setSetUpstreams(l.StableReleaseSet, l.CandidateReleaseSet) {
|
||||
return false
|
||||
}
|
||||
l.TrafficMode = mode
|
||||
return true
|
||||
}
|
||||
|
||||
func (l *Lifeboat) PromoteSet() bool {
|
||||
if l.CandidateReleaseSet == nil || l.Current.State != ReleaseHealthy {
|
||||
return false
|
||||
}
|
||||
if l.StableReleaseSet != nil {
|
||||
previous := *l.StableReleaseSet
|
||||
l.RollbackTargetReleaseSet = &previous
|
||||
l.RollbackTarget = l.LastKnownGood
|
||||
}
|
||||
set := *l.CandidateReleaseSet
|
||||
l.StableReleaseSet = &set
|
||||
l.LastKnownGoodReleaseSet = &set
|
||||
l.LastKnownGood = l.Current
|
||||
l.LastKnownGood.Set = &set
|
||||
l.LastKnownGood.LastKnownGood = true
|
||||
l.Current.Set = &set
|
||||
l.Current.LastKnownGood = true
|
||||
l.CandidateReleaseSet = nil
|
||||
return l.setSetUpstreams(&set, nil)
|
||||
}
|
||||
|
||||
func (l *Lifeboat) RollbackSet() bool {
|
||||
if l.RollbackTargetReleaseSet == nil && l.LastKnownGoodReleaseSet == nil {
|
||||
return false
|
||||
}
|
||||
target := l.RollbackTargetReleaseSet
|
||||
if target == nil {
|
||||
target = l.LastKnownGoodReleaseSet
|
||||
}
|
||||
copySet := *target
|
||||
l.StableReleaseSet = ©Set
|
||||
l.Current = Release{ID: copySet.ID, Version: copySet.Revision, State: ReleaseRolledBack, LastKnownGood: true, Set: ©Set}
|
||||
l.LastKnownGood = l.Current
|
||||
l.LastKnownGoodReleaseSet = ©Set
|
||||
l.CandidateReleaseSet = nil
|
||||
l.RollbackTargetReleaseSet = nil
|
||||
l.TrafficMode = TrafficStable
|
||||
return l.setSetUpstreams(©Set, nil)
|
||||
}
|
||||
|
||||
func (l *Lifeboat) Start(candidate Release) bool {
|
||||
|
||||
@@ -82,3 +82,79 @@ func TestPersistentSupervisorRestoresTrafficModeAfterRestart(t *testing.T) {
|
||||
t.Fatalf("rollback state=%+v", state)
|
||||
}
|
||||
}
|
||||
|
||||
func testReleaseSet(id, revision, web, core, cognitive, research string) supervisor.ReleaseSet {
|
||||
return supervisor.ReleaseSet{ID: id, Revision: revision, Components: map[string]supervisor.ComponentRelease{
|
||||
supervisor.ComponentCreatorWeb: {Name: supervisor.ComponentCreatorWeb, Revision: revision, Upstream: web, HealthURL: web + "/healthz", ReadyURL: web + "/readyz"},
|
||||
supervisor.ComponentCoreRuntime: {Name: supervisor.ComponentCoreRuntime, Revision: revision, Upstream: core, HealthURL: core + "/healthz", ReadyURL: core + "/readyz"},
|
||||
supervisor.ComponentCognitiveWorker: {Name: supervisor.ComponentCognitiveWorker, Revision: revision, Upstream: cognitive, HealthURL: cognitive + "/healthz", ReadyURL: cognitive + "/readyz"},
|
||||
supervisor.ComponentResearchRuntime: {Name: supervisor.ComponentResearchRuntime, Revision: revision, Upstream: research, HealthURL: research + "/healthz", ReadyURL: research + "/readyz"},
|
||||
}}
|
||||
}
|
||||
|
||||
func TestReleaseSetRejectsMixedRevisionsAndMissingComponents(t *testing.T) {
|
||||
set := testReleaseSet("candidate", "r1", "http://web", "http://core", "http://cognitive", "http://research")
|
||||
if !set.Validate() {
|
||||
t.Fatal("complete release set should validate")
|
||||
}
|
||||
set.Components[supervisor.ComponentCoreRuntime] = supervisor.ComponentRelease{Name: supervisor.ComponentCoreRuntime, Revision: "r0", Upstream: "http://core", HealthURL: "http://core/healthz", ReadyURL: "http://core/readyz"}
|
||||
if set.Validate() {
|
||||
t.Fatal("mixed revision release set must be rejected")
|
||||
}
|
||||
}
|
||||
|
||||
func TestPersistentReleaseSetRollbackPreservesWholeLastKnownGood(t *testing.T) {
|
||||
store, err := supervisor.NewDurableStore(filepath.Join(t.TempDir(), "control", "release-set.json"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
first := supervisor.NewPersistentLifeboat(store)
|
||||
good := testReleaseSet("stable-r1", "r1", "http://web", "http://core", "http://cognitive", "http://research")
|
||||
candidate := testReleaseSet("candidate-r2", "r2", "http://web-c", "http://core-c", "http://cognitive-c", "http://research-c")
|
||||
if ok, err := first.EstablishBaselineSet(good); err != nil || !ok {
|
||||
t.Fatalf("baseline: %v %v", ok, err)
|
||||
}
|
||||
if ok, err := first.StartSet(candidate); err != nil || !ok {
|
||||
t.Fatalf("start: %v %v", ok, err)
|
||||
}
|
||||
observation := supervisor.ReleaseSetObservation{ReleaseID: candidate.ID, Ready: true, Components: map[string]supervisor.ComponentObservation{}}
|
||||
for name := range candidate.Components {
|
||||
observation.Components[name] = supervisor.ComponentObservation{Revision: candidate.Revision, Health: true, Ready: true}
|
||||
}
|
||||
if ok, err := first.RecordSet(observation); err != nil || !ok {
|
||||
t.Fatalf("observe: %v %v", ok, err)
|
||||
}
|
||||
if ok, err := first.SetTrafficForSets(supervisor.TrafficCanaryHeader); err != nil || !ok {
|
||||
t.Fatalf("canary: %v %v", ok, err)
|
||||
}
|
||||
second := supervisor.NewPersistentLifeboat(store)
|
||||
if ok, err := second.RollbackSet(); err != nil || !ok {
|
||||
t.Fatalf("rollback: %v %v", ok, err)
|
||||
}
|
||||
state, err := second.Snapshot()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if state.StableReleaseSet == nil || state.StableReleaseSet.Revision != "r1" || state.TrafficMode != supervisor.TrafficStable {
|
||||
t.Fatalf("state=%+v", state)
|
||||
}
|
||||
}
|
||||
|
||||
func TestReleaseSetPromotionRetainsPreviousStableForRollback(t *testing.T) {
|
||||
good := testReleaseSet("stable-r1", "r1", "http://web", "http://core", "http://cognitive", "http://research")
|
||||
candidate := testReleaseSet("candidate-r2", "r2", "http://web-c", "http://core-c", "http://cognitive-c", "http://research-c")
|
||||
l := supervisor.Lifeboat{}
|
||||
if !l.EstablishBaselineSet(good) || !l.StartSet(candidate) {
|
||||
t.Fatal("release set setup failed")
|
||||
}
|
||||
observation := supervisor.ReleaseSetObservation{ReleaseID: candidate.ID, Ready: true, Components: map[string]supervisor.ComponentObservation{}}
|
||||
for name := range candidate.Components {
|
||||
observation.Components[name] = supervisor.ComponentObservation{Revision: candidate.Revision, Health: true, Ready: true}
|
||||
}
|
||||
if !l.RecordSet(observation) || !l.PromoteSet() {
|
||||
t.Fatal("promotion failed")
|
||||
}
|
||||
if l.RollbackTargetReleaseSet == nil || l.RollbackTargetReleaseSet.Revision != "r1" {
|
||||
t.Fatalf("rollback target=%+v", l.RollbackTargetReleaseSet)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,19 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
root="$(CDPATH= cd -- "$(dirname -- "$0")/../.." && pwd)"
|
||||
canary="$root/deploy/run-production-canary.sh"
|
||||
supervisor="$root/deploy/update-supervisor.sh"
|
||||
|
||||
grep -Fq 'up -d --no-build --no-deps research-runtime-candidate core-runtime-candidate' "$canary"
|
||||
grep -Fq 'up -d --no-build --no-deps cognitive-worker-candidate' "$canary"
|
||||
grep -Fq 'up -d --no-build --no-deps creator-web-candidate' "$canary"
|
||||
grep -Fq 'compose up -d --no-build --no-deps supervisor' "$supervisor"
|
||||
grep -Fq 'stable_ids_before=$(stable_ids)' "$canary"
|
||||
grep -Fq 'assert_stable_ids_unchanged' "$canary"
|
||||
grep -Fq 'before=$(stable_ids)' "$supervisor"
|
||||
grep -Fq 'after=$(stable_ids)' "$supervisor"
|
||||
grep -Fq 'stable container IDs changed' "$supervisor"
|
||||
grep -Fq 'HEALTH_ATTEMPTS="${HEALTH_ATTEMPTS:-360}"' "$canary"
|
||||
grep -Fq 'seq 1 "$HEALTH_ATTEMPTS"' "$canary"
|
||||
echo 'stable release-set guard: PASS'
|
||||
Reference in New Issue
Block a user