Compare commits

..
Author SHA1 Message Date
m 42285a55be feat(deploy): close candidate release set topology
CI / test (push) Failing after 34s
2026-08-31 16:35:08 +03:00
13 changed files with 264 additions and 370 deletions
+23 -134
View File
@@ -4,11 +4,8 @@ import (
"context"
"encoding/json"
"fmt"
"log"
"net/http"
"slices"
"sort"
"strings"
"sync"
"time"
@@ -56,83 +53,8 @@ type modelQualificationResponse struct {
Results []modelQualificationResult `json:"results"`
}
const qualificationControlPlaneTimeout = 45 * time.Second
// qualificationCoordinator bounds duplicate control-plane probes within one
// cognitive-worker process. It is deliberately not canonical state: the
// Capability owner remains the sole owner of qualification evidence and
// activation. A restart may repeat an incomplete probe, but cannot create a
// second semantic writer or bypass independent evaluation.
type qualificationCoordinator struct {
mu sync.Mutex
flights map[string]*qualificationFlight
}
type qualificationFlight struct {
done chan struct{}
response modelQualificationResponse
err error
waiters int
}
func (c *qualificationCoordinator) do(ctx context.Context, key string, operation func() (modelQualificationResponse, error)) (modelQualificationResponse, bool, error) {
c.mu.Lock()
if c.flights == nil {
c.flights = make(map[string]*qualificationFlight)
}
if current, exists := c.flights[key]; exists {
current.waiters++
c.mu.Unlock()
select {
case <-current.done:
return current.response, false, current.err
case <-ctx.Done():
return modelQualificationResponse{}, false, ctx.Err()
}
}
current := &qualificationFlight{done: make(chan struct{})}
c.flights[key] = current
c.mu.Unlock()
current.response, current.err = operation()
close(current.done)
c.mu.Lock()
delete(c.flights, key)
c.mu.Unlock()
return current.response, true, current.err
}
type qualificationPhaseTrace struct {
correlationID string
started time.Time
}
func newQualificationPhaseTrace() qualificationPhaseTrace {
correlationID, err := ids.New()
if err != nil {
correlationID = ids.ID(fmt.Sprintf("qualification-%d", time.Now().UnixNano()))
}
return qualificationPhaseTrace{correlationID: string(correlationID), started: time.Now()}
}
func (t qualificationPhaseTrace) phase(name string, fields ...string) {
log.Printf("qualification_phase correlation_id=%s phase=%s elapsed_ms=%d %s", t.correlationID, name, time.Since(t.started).Milliseconds(), strings.Join(fields, " "))
}
func qualificationFlightKey(provider string, modelIDs []capability.ProviderModelID) string {
ids := make([]string, 0, len(modelIDs))
for _, modelID := range modelIDs {
ids = append(ids, string(modelID))
}
sort.Strings(ids)
return provider + "|" + strings.Join(ids, ",")
}
func modelQualificationHandler(providers map[string]providerRegistration, owner string, observers ...qualificationObserver) http.Handler {
coordinator := &qualificationCoordinator{}
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
trace := newQualificationPhaseTrace()
trace.phase("Q0", "event=request_accepted")
name := r.PathValue("provider")
registration, ok := providers[name]
if !ok || registration.Discovery == nil || registration.Provider == nil {
@@ -150,65 +72,32 @@ func modelQualificationHandler(providers map[string]providerRegistration, owner
http.Error(w, "max_output_bytes exceeds qualification bound", http.StatusBadRequest)
return
}
controlCtx, cancel := context.WithTimeout(r.Context(), qualificationControlPlaneTimeout)
defer cancel()
response, leader, err := coordinator.do(controlCtx, qualificationFlightKey(name, request.ModelIDs), func() (modelQualificationResponse, error) {
trace.phase("Q1", "event=binding_candidate_loaded")
modelsFound, probe := registration.Discovery.Discover(controlCtx)
if probe.Health != capability.HealthAvailable || len(modelsFound) == 0 {
return modelQualificationResponse{}, fmt.Errorf("provider unavailable for qualification")
}
if len(request.ModelIDs) > 0 {
modelsFound = slices.DeleteFunc(modelsFound, func(model models.DiscoveredModel) bool {
return !slices.Contains(request.ModelIDs, model.ProviderModelID)
})
}
if len(modelsFound) == 0 {
return modelQualificationResponse{}, fmt.Errorf("requested model not discovered")
}
if len(modelsFound) > 16 {
modelsFound = modelsFound[:16]
}
trace.phase("Q2", fmt.Sprintf("event=provider_model_resolved model_count=%d", len(modelsFound)))
trace.phase("Q3", "event=bounded_invocation_started")
results := qualifyModels(controlCtx, registration, owner, request.Prompt, request.MaxOutputBytes, modelsFound)
trace.phase("Q4", "event=bounded_invocation_completed")
trace.phase("Q5", fmt.Sprintf("event=invocation_evidence_built result_count=%d", len(results)))
if len(observers) > 0 && observers[0] != nil {
trace.phase("Q6", "event=observation_persistence_started")
if err := persistQualificationResults(controlCtx, observers[0], registration, results); err != nil {
return modelQualificationResponse{}, fmt.Errorf("persist model qualification observation: %w", err)
}
trace.phase("Q7", "event=observation_persistence_completed")
}
// Q8-Q17 are intentionally outside this worker's authority. Capability
// owns independent evaluation and governed promotion; this trace makes
// that hand-off explicit instead of disguising it as activation.
trace.phase("Q8", "event=independent_evaluation_not_started owner=capability")
return modelQualificationResponse{Provider: registration.ProviderInstanceID, ObservedAtUnixMillis: time.Now().UnixMilli(), Results: results}, nil
})
if !leader {
trace.phase("Q0", "event=joined_in_progress_qualification")
}
if err != nil {
if controlCtx.Err() != nil {
http.Error(w, "qualification timeout", http.StatusGatewayTimeout)
return
}
if strings.Contains(err.Error(), "requested model not discovered") {
http.Error(w, "requested model not discovered", http.StatusNotFound)
return
}
if strings.Contains(err.Error(), "provider unavailable") {
http.Error(w, "provider unavailable for qualification", http.StatusServiceUnavailable)
return
}
http.Error(w, "persist model qualification observation", http.StatusBadGateway)
modelsFound, probe := registration.Discovery.Discover(r.Context())
if probe.Health != capability.HealthAvailable || len(modelsFound) == 0 {
http.Error(w, "provider unavailable for qualification", http.StatusServiceUnavailable)
return
}
trace.phase("Q17", "event=qualification_response_returned activation=not_attempted")
if len(request.ModelIDs) > 0 {
modelsFound = slices.DeleteFunc(modelsFound, func(model models.DiscoveredModel) bool {
return !slices.Contains(request.ModelIDs, model.ProviderModelID)
})
}
if len(modelsFound) == 0 {
http.Error(w, "requested model not discovered", http.StatusNotFound)
return
}
if len(modelsFound) > 16 {
modelsFound = modelsFound[:16]
}
results := qualifyModels(r.Context(), registration, owner, request.Prompt, request.MaxOutputBytes, modelsFound)
if len(observers) > 0 && observers[0] != nil {
if err := persistQualificationResults(r.Context(), observers[0], registration, results); err != nil {
http.Error(w, "persist model qualification observation", http.StatusBadGateway)
return
}
}
w.Header().Set("Content-Type", "application/json")
_ = jsonEncode(w, response)
_ = jsonEncode(w, modelQualificationResponse{Provider: registration.ProviderInstanceID, ObservedAtUnixMillis: time.Now().UnixMilli(), Results: results})
})
}
@@ -6,7 +6,6 @@ import (
"net/http/httptest"
"strings"
"testing"
"time"
"git.cin.su/m/Assistant/adapters/models"
"git.cin.su/m/Assistant/contracts/capability"
@@ -18,7 +17,6 @@ type qualificationProvider struct{}
func (qualificationProvider) Probe(context.Context) models.ProviderProbe {
return models.ProviderProbe{Health: capability.HealthAvailable}
}
func (qualificationProvider) Discover(context.Context) ([]models.DiscoveredModel, models.ProviderProbe) {
return []models.DiscoveredModel{{ProviderModelID: "model-a"}, {ProviderModelID: "model-b"}}, models.ProviderProbe{Health: capability.HealthAvailable}
}
@@ -85,63 +83,3 @@ func TestModelQualificationPersistsReachabilityWithoutQuality(t *testing.T) {
t.Fatalf("invalid qualification observation: %+v err=%v", observer.observations[0], err)
}
}
func TestModelQualificationDeduplicatesSameProviderModelFlight(t *testing.T) {
coordinator := &qualificationCoordinator{}
started := make(chan struct{})
release := make(chan struct{})
firstDone := make(chan struct{})
secondDone := make(chan struct{})
result := modelQualificationResponse{Provider: "medium"}
go func() {
_, _, err := coordinator.do(context.Background(), "medium|model-a", func() (modelQualificationResponse, error) {
close(started)
<-release
return result, nil
})
if err != nil {
t.Errorf("leader qualification failed: %v", err)
}
close(firstDone)
}()
select {
case <-started:
case <-time.After(time.Second):
t.Fatal("qualification invocation did not start")
}
go func() {
actual, leader, err := coordinator.do(context.Background(), "medium|model-a", func() (modelQualificationResponse, error) {
t.Error("duplicate qualification operation ran")
return modelQualificationResponse{}, nil
})
if err != nil || leader || actual.Provider != result.Provider {
t.Errorf("joined qualification response=%+v leader=%t err=%v", actual, leader, err)
}
close(secondDone)
}()
deadline := time.Now().Add(time.Second)
for {
coordinator.mu.Lock()
flight := coordinator.flights["medium|model-a"]
waiters := 0
if flight != nil {
waiters = flight.waiters
}
coordinator.mu.Unlock()
if waiters == 1 {
break
}
if time.Now().After(deadline) {
t.Fatal("second qualification did not join the in-progress flight")
}
time.Sleep(time.Millisecond)
}
close(release)
for _, done := range []chan struct{}{firstDone, secondDone} {
select {
case <-done:
case <-time.After(time.Second):
t.Fatal("qualification flight did not complete")
}
}
}
+152 -3
View File
@@ -69,6 +69,20 @@ services:
COGNITIVE_ADMISSION_MAX_OUTPUT_BYTES: ${COGNITIVE_ADMISSION_MAX_OUTPUT_BYTES:-16384}
COGNITIVE_RUNTIME_URL: http://cognitive-worker-candidate:8083
RESEARCH_RUNTIME_URL: http://research-runtime-candidate:8087
CONTINUITY_RUNTIME_URL: http://continuity-runtime-candidate:8081
COMMIT_RUNTIME_URL: http://commit-runtime-candidate:8082
CANDIDATE_RELEASE_SET_ID: assistant-slot-b
CANDIDATE_SLOT: b
CANDIDATE_TASK_ID: GEN2-CAPABILITY-001
CANDIDATE_TASK_REVISION: "7"
CANDIDATE_PRINCIPAL_ID: domovoy-creator-principal
CANDIDATE_CAPABILITY_ID: prospective-notification-delivery
CANDIDATE_RESOURCE_SCOPE: synthetic-shadow-notification-v1
CANDIDATE_BINDING_MODE: SHADOW
CANDIDATE_TARGET_KIND: NOTIFICATION_SHADOW_RECEIVER
CANDIDATE_TARGET_REF: notification-receiver-candidate
CANDIDATE_TARGET_DIGEST: f9f9fa49a0727444c280c092d5bb340f3a8f33cf325c5396f5ae177a6666da35
CANDIDATE_AUTHORITY_GRANT_ID: gen2-capability-001-shadow-notification-grant
volumes:
- assistant_core_secrets:/run/secrets:ro
- assistant_control_plane_service_secrets:/run/control-plane-secrets:ro
@@ -124,10 +138,10 @@ services:
environment:
LISTEN_ADDR: ":8084"
CORE_RUNTIME_URL: http://core-runtime-candidate:8080
CONTINUITY_RUNTIME_URL: http://continuity-runtime:8081
COMMIT_RUNTIME_URL: http://commit-runtime:8082
CONTINUITY_RUNTIME_URL: http://continuity-runtime-candidate:8081
COMMIT_RUNTIME_URL: http://commit-runtime-candidate:8082
COGNITIVE_RUNTIME_URL: http://cognitive-worker-candidate:8083
AGENT_WORKER_URL: http://agent-worker:8090
AGENT_WORKER_URL: http://agent-worker-candidate:8090
ASSISTANT_BUILD_REVISION: ${ASSISTANT_SOURCE_REVISION:-unknown}
networks:
shared:
@@ -138,6 +152,133 @@ services:
timeout: 5s
retries: 20
notification-receiver:
build: { context: .., dockerfile: deploy/images/runtime.Dockerfile, args: { TARGET: notification-receiver, SOURCE_REVISION: "${ASSISTANT_SOURCE_REVISION:-unknown}" } }
restart: "no"
environment:
LISTEN_ADDR: ":8085"
SHADOW_RECEIVER_MODE: "1"
CANDIDATE_RELEASE_SET_ID: assistant-slot-b
CANDIDATE_TASK_ID: GEN2-CAPABILITY-001
RECEIPT_DIRECTORY: /var/lib/notification-shadow-receiver
RECEIVER_WRITE_TOKEN_FILE: /run/notification-shadow-writer/token
RECEIVER_READ_TOKEN_FILE: /run/notification-shadow-reader/token
volumes:
- assistant_candidate_notification_shadow_receipts:/var/lib/notification-shadow-receiver
- assistant_candidate_notification_shadow_writer_secret:/run/notification-shadow-writer:ro
- assistant_candidate_notification_shadow_reader_secret:/run/notification-shadow-reader:ro
networks: { shared: { aliases: [notification-receiver-candidate] } }
healthcheck: { test: ["CMD-SHELL", "wget -qO- http://localhost:8085/healthz >/dev/null"], interval: 10s, timeout: 3s, retries: 10 }
continuity-runtime:
build: { context: .., dockerfile: deploy/images/runtime.Dockerfile, args: { TARGET: continuity-runtime, SOURCE_REVISION: "${ASSISTANT_SOURCE_REVISION:-unknown}" } }
restart: "no"
environment:
LISTEN_ADDR: ":8081"
POSTGRES_HOST: postgres
POSTGRES_PORT: "5432"
POSTGRES_DATABASE: ${POSTGRES_DB:-assistant}
POSTGRES_USER: continuity_role
POSTGRES_PASSWORD: ${CONTINUITY_DB_PASSWORD}
CORE_RUNTIME_URL: http://core-runtime-candidate:8080
COMMIT_RUNTIME_URL: http://commit-runtime-candidate:8082
COMMIT_EXECUTOR_TOKEN: ${COMMIT_EXECUTOR_TOKEN}
networks: { shared: { aliases: [continuity-runtime-candidate] } }
healthcheck: { test: ["CMD-SHELL", "wget -qO- http://localhost:8081/healthz >/dev/null"], interval: 10s, timeout: 5s, retries: 20 }
commit-runtime:
build: { context: .., dockerfile: deploy/images/runtime.Dockerfile, args: { TARGET: commit-runtime, SOURCE_REVISION: "${ASSISTANT_SOURCE_REVISION:-unknown}" } }
restart: "no"
environment:
LISTEN_ADDR: ":8082"
POSTGRES_HOST: postgres
POSTGRES_PORT: "5432"
POSTGRES_DATABASE: ${POSTGRES_DB:-assistant}
POSTGRES_USER: commit_role
POSTGRES_PASSWORD: ${COMMIT_DB_PASSWORD}
AUTHORITY_POSTGRES_USER: authority_role
AUTHORITY_POSTGRES_PASSWORD: ${AUTHORITY_DB_PASSWORD}
CONTROL_PLANE_SERVICE_CREDENTIAL_REF: credential://domovoy/control-plane/service-token/v1
CONTROL_PLANE_SERVICE_SECRET_ROOT: /run/control-plane-secrets
CORE_RUNTIME_URL: http://core-runtime-candidate:8080
CONTINUITY_RUNTIME_URL: http://continuity-runtime-candidate:8081
COMMIT_EXECUTOR_TOKEN: ${COMMIT_EXECUTOR_TOKEN}
CANDIDATE_NOTIFICATION_WRITE_TOKEN_FILE: /run/notification-shadow-writer/token
CANDIDATE_NOTIFICATION_WRITE_SECRET_REF: credential://domovoy/candidate/notification-write/v1
CANDIDATE_RELEASE_SET_ID: assistant-slot-b
CANDIDATE_SLOT: b
CANDIDATE_TASK_ID: GEN2-CAPABILITY-001
CANDIDATE_TASK_REVISION: "7"
CANDIDATE_PRINCIPAL_ID: domovoy-creator-principal
CANDIDATE_CAPABILITY_ID: prospective-notification-delivery
CANDIDATE_RESOURCE_SCOPE: synthetic-shadow-notification-v1
CANDIDATE_BINDING_MODE: SHADOW
CANDIDATE_TARGET_KIND: NOTIFICATION_SHADOW_RECEIVER
CANDIDATE_TARGET_REF: notification-receiver-candidate
CANDIDATE_TARGET_DIGEST: f9f9fa49a0727444c280c092d5bb340f3a8f33cf325c5396f5ae177a6666da35
CANDIDATE_AUTHORITY_GRANT_ID: gen2-capability-001-shadow-notification-grant
volumes:
- assistant_core_secrets:/run/secrets:ro
- assistant_control_plane_service_secrets:/run/control-plane-secrets:ro
- assistant_candidate_notification_shadow_writer_secret:/run/notification-shadow-writer:ro
networks: { shared: { aliases: [commit-runtime-candidate] } }
healthcheck: { test: ["CMD-SHELL", "wget -qO- http://localhost:8082/healthz >/dev/null && wget -qO- http://localhost:8082/readyz >/dev/null"], interval: 10s, timeout: 5s, retries: 20 }
task-verifier:
build: { context: .., dockerfile: deploy/images/runtime.Dockerfile, args: { TARGET: task-verifier, SOURCE_REVISION: "${ASSISTANT_SOURCE_REVISION:-unknown}" } }
restart: "no"
environment: { LISTEN_ADDR: ":8091" }
networks: { shared: { aliases: [task-verifier-candidate] } }
healthcheck: { test: ["CMD-SHELL", "wget -qO- http://localhost:8091/healthz >/dev/null"], interval: 10s, timeout: 3s, retries: 10 }
work-graph-runtime:
build: { context: .., dockerfile: deploy/images/runtime.Dockerfile, args: { TARGET: work-graph-runtime, SOURCE_REVISION: "${ASSISTANT_SOURCE_REVISION:-unknown}" } }
restart: "no"
environment:
LISTEN_ADDR: ":8088"
POSTGRES_HOST: postgres
POSTGRES_PORT: "5432"
POSTGRES_DATABASE: ${POSTGRES_DB:-assistant}
WORK_POSTGRES_USER: work_role
WORK_POSTGRES_CREDENTIAL_REF: credential://domovoy/work/db-password/v1
WORK_POSTGRES_PASSWORD_FILE: /run/secrets/work_db_password
SECRET_BACKEND_ROOT: /run/secrets
CORE_RUNTIME_URL: http://core-runtime-candidate:8080
volumes: [assistant_core_secrets:/run/secrets:ro]
networks: { shared: { aliases: [work-graph-runtime-candidate] } }
healthcheck: { test: ["CMD-SHELL", "wget -qO- http://localhost:8088/healthz >/dev/null"], interval: 10s, timeout: 3s, retries: 10 }
agent-execution-bridge:
build: { context: .., dockerfile: deploy/images/runtime.Dockerfile, args: { TARGET: agent-execution-bridge, SOURCE_REVISION: "${ASSISTANT_SOURCE_REVISION:-unknown}" } }
restart: "no"
environment:
LISTEN_ADDR: ":8092"
COGNITIVE_WORKER_URL: http://cognitive-worker-candidate:8083
WORK_GRAPH_RUNTIME_URL: http://work-graph-runtime-candidate:8088
TASK_VERIFIER_URL: http://task-verifier-candidate:8091
networks: { shared: { aliases: [agent-execution-bridge-candidate] } }
healthcheck: { test: ["CMD-SHELL", "wget -qO- http://localhost:8092/healthz >/dev/null"], interval: 10s, timeout: 3s, retries: 10 }
agent-worker:
build: { context: .., dockerfile: deploy/images/runtime.Dockerfile, args: { TARGET: agent-worker, SOURCE_REVISION: "${ASSISTANT_SOURCE_REVISION:-unknown}" } }
restart: "no"
command: ["-worker-id", "agent-worker-candidate", "-status-addr", ":8090"]
environment:
POSTGRES_HOST: postgres
POSTGRES_PORT: "5432"
POSTGRES_DATABASE: ${POSTGRES_DB:-assistant}
WORK_POSTGRES_USER: work_role
WORK_POSTGRES_CREDENTIAL_REF: credential://domovoy/work/db-password/v1
WORK_POSTGRES_PASSWORD_FILE: /run/secrets/work_db_password
SECRET_BACKEND_ROOT: /run/secrets
COGNITIVE_WORKER_URL: http://cognitive-worker-candidate:8083
WORK_GRAPH_RUNTIME_URL: http://work-graph-runtime-candidate:8088
TASK_VERIFIER_URL: http://task-verifier-candidate:8091
AGENT_EXECUTION_BRIDGE_URL: http://agent-execution-bridge-candidate:8092
volumes: [assistant_core_secrets:/run/secrets:ro]
networks: { shared: { aliases: [agent-worker-candidate] } }
healthcheck: { test: ["CMD-SHELL", "wget -qO- http://localhost:8090/healthz >/dev/null"], interval: 10s, timeout: 3s, retries: 10 }
volumes:
assistant_core_secrets:
external: true
@@ -147,6 +288,14 @@ volumes:
assistant_research_sources:
name: assistant_research_sources
external: true
assistant_candidate_notification_shadow_receipts:
name: assistant_candidate_notification_shadow_receipts
assistant_candidate_notification_shadow_writer_secret:
name: assistant_candidate_notification_shadow_writer_secret
external: true
assistant_candidate_notification_shadow_reader_secret:
name: assistant_candidate_notification_shadow_reader_secret
external: true
networks:
shared:
+28 -14
View File
@@ -26,8 +26,8 @@ fail_rollback() {
exit 1
}
services=(creator-web core-runtime cognitive-worker research-runtime)
stable_services=(creator-web core-runtime cognitive-worker research-runtime)
services=(creator-web core-runtime cognitive-worker research-runtime commit-runtime continuity-runtime agent-worker agent-execution-bridge work-graph-runtime task-verifier notification-receiver)
stable_services=(creator-web core-runtime cognitive-worker research-runtime commit-runtime continuity-runtime agent-worker agent-execution-bridge work-graph-runtime task-verifier notification-receiver)
stable_ids() {
local service container id
for service in "${stable_services[@]}"; do
@@ -61,13 +61,10 @@ stable_ids_before=$(stable_ids) || exit 1
candidate_compose config >/dev/null
candidate_compose build --build-arg "SOURCE_REVISION=$REVISION" "${services[@]}"
assert_stable_ids_unchanged
candidate_compose up -d --no-build --no-deps research-runtime core-runtime
wait_healthy "$(candidate_container research-runtime)"
wait_healthy "$(candidate_container core-runtime)"
candidate_compose up -d --no-build --no-deps cognitive-worker
wait_healthy "$(candidate_container cognitive-worker)"
candidate_compose up -d --no-build --no-deps creator-web
wait_healthy "$(candidate_container creator-web)"
candidate_compose up -d --no-build --no-deps "${services[@]}"
for service in "${services[@]}"; do
wait_healthy "$(candidate_container "$service")"
done
assert_stable_ids_unchanged
for service in "${services[@]}"; do
image_revision=$(docker inspect -f '{{index .Config.Labels "org.opencontainers.image.revision"}}' "$(candidate_container "$service")" 2>/dev/null || true)
@@ -79,10 +76,23 @@ if [[ "${PREPARE_ONLY:-false}" == true ]]; then
exit 0
fi
release_json=$(cat <<JSON
{"id":"candidate-$SLOT-$REVISION","slot":"$SLOT","revision":"$REVISION","components":{"creator-web":{"name":"creator-web","revision":"$REVISION","upstream":"http://creator-web-candidate:8084","health_url":"http://creator-web-candidate:8084/healthz","ready_url":"http://creator-web-candidate:8084/healthz"},"core-runtime":{"name":"core-runtime","revision":"$REVISION","upstream":"http://core-runtime-candidate:8080","health_url":"http://core-runtime-candidate:8080/healthz","ready_url":"http://core-runtime-candidate:8080/readyz"},"cognitive-worker":{"name":"cognitive-worker","revision":"$REVISION","upstream":"http://cognitive-worker-candidate:8083","health_url":"http://cognitive-worker-candidate:8083/healthz","ready_url":"http://cognitive-worker-candidate:8083/readyz"},"research-runtime":{"name":"research-runtime","revision":"$REVISION","upstream":"http://research-runtime-candidate:8087","health_url":"http://research-runtime-candidate:8087/healthz","ready_url":"http://research-runtime-candidate:8087/readyz"}}}
JSON
)
component_port() {
case "$1" in
creator-web) echo 8084 ;; core-runtime) echo 8080 ;; cognitive-worker) echo 8083 ;; research-runtime) echo 8087 ;;
commit-runtime) echo 8082 ;; continuity-runtime) echo 8081 ;; agent-worker) echo 8090 ;; agent-execution-bridge) echo 8092 ;;
work-graph-runtime) echo 8088 ;; task-verifier) echo 8091 ;; notification-receiver) echo 8085 ;;
*) return 1 ;;
esac
}
components='{}'
for service in "${services[@]}"; do
port=$(component_port "$service") || fail_rollback
upstream="http://${service}-candidate:${port}"
ready_url="$upstream/readyz"
case "$service" in creator-web|agent-worker|agent-execution-bridge|work-graph-runtime|task-verifier|notification-receiver) ready_url="$upstream/healthz" ;; esac
components=$(jq -c --arg name "$service" --arg revision "$REVISION" --arg upstream "$upstream" --arg health "$upstream/healthz" --arg ready "$ready_url" '. + {($name): {name:$name,revision:$revision,upstream:$upstream,health_url:$health,ready_url:$ready}}' <<<"$components")
done
release_json=$(jq -cn --arg id "candidate-$SLOT-$REVISION" --arg slot "$SLOT" --arg revision "$REVISION" --argjson components "$components" '{id:$id,slot:$slot,revision:$revision,components:$components}')
stable_json=$(curl -fsS --max-time 10 "$SUPERVISOR_CONTROL_URL/v1/state")
stable_set=$(printf '%s' "$stable_json" | jq -c '.stable_release_set // empty')
if [[ -z "$stable_set" ]]; then
@@ -95,7 +105,11 @@ JSON
fi
curl -fsS --max-time 10 -X POST "$SUPERVISOR_CONTROL_URL/v1/releases/start-set" -H 'Content-Type: application/json' --data "$release_json" >/dev/null || fail_rollback
assert_stable_ids_unchanged
observation=$(printf '{"release_id":"candidate-%s-%s","ready":true,"reason":"isolated slot health and image revisions passed","observed_at_unix_millis":%s,"components":{"creator-web":{"revision":"%s","health":true,"ready":true},"core-runtime":{"revision":"%s","health":true,"ready":true},"cognitive-worker":{"revision":"%s","health":true,"ready":true},"research-runtime":{"revision":"%s","health":true,"ready":true}}}\n' "$SLOT" "$REVISION" "$(date +%s%3N)" "$REVISION" "$REVISION" "$REVISION" "$REVISION")
observed_components='{}'
for service in "${services[@]}"; do
observed_components=$(jq -c --arg name "$service" --arg revision "$REVISION" '. + {($name): {revision:$revision,health:true,ready:true}}' <<<"$observed_components")
done
observation=$(jq -cn --arg release "candidate-$SLOT-$REVISION" --argjson observed_at "$(date +%s%3N)" --argjson components "$observed_components" '{release_id:$release,ready:true,reason:"isolated slot health and image revisions passed",observed_at_unix_millis:$observed_at,components:$components}')
curl -fsS --max-time 10 -X POST "$SUPERVISOR_CONTROL_URL/v1/releases/observe-set" -H 'Content-Type: application/json' --data "$observation" >/dev/null || fail_rollback
assert_stable_ids_unchanged
control /v1/traffic/canary || fail_rollback
+1 -1
View File
@@ -1 +1 @@
{"task_file":"dev/tasks/GEN2-COGNITIVE-003-restore-local-medium-invocation.json","task_id":"GEN2-COGNITIVE-003","task_revision":4}
{"task_file":"dev/tasks/GEN2-DEPLOY-004-candidate-commit-runtime-releaseset.json","task_id":"GEN2-DEPLOY-004","task_revision":3}
@@ -1,18 +1,17 @@
{
"task_id": "GEN2-COGNITIVE-003",
"task_revision": 4,
"task_revision": 3,
"architecture_revision": "ARCH-GEN2-000",
"base_repo_revision": "gen2-integration",
"risk_tier": "E",
"intent": "Restore and phase-trace bounded Local MEDIUM qualification and invocation without changing Capability ownership, Core household budget, or activation authority.",
"requirement_ids": ["REQ-GEN2-EVOLVE-004-LOCAL-MEDIUM-INVOCATION", "REQ-GEN2-COGNITIVE-003-QUALIFICATION-CONTROL-PLANE"],
"intent": "Restore bounded OpenAI-compatible Local MEDIUM invocation without changing canonical ownership or Core semantics.",
"requirement_ids": ["REQ-GEN2-EVOLVE-004-LOCAL-MEDIUM-INVOCATION"],
"allowed_change_scope": {
"modules": ["cognitive-worker provider invocation", "existing local OpenAI-compatible adapter", "targeted evidence"],
"paths": [
"cmd/cognitive-worker/main.go",
"cmd/cognitive-worker/advisory.go",
"cmd/cognitive-worker/gateway_registry.go",
"cmd/cognitive-worker/model_qualification.go",
"cmd/cognitive-worker/*_test.go",
"adapters/models/openai_compatible_local.go",
"adapters/models/openai_compatible_local_test.go",
@@ -67,13 +66,13 @@
"may_change": ["provider request normalization and typed immediate failure mapping only"]
},
"semantic_owner": "Cognitive gateway remains the sole owner of provider invocation and route selection.",
"canonical_state": ["discovery is not invocation qualification", "qualification evidence is not activation", "optional provider parameters are capability-gated", "immediate provider rejection is not reported as timeout"],
"state_transitions": ["binding selection -> normalized request -> provider invocation -> typed result", "discovery -> bounded qualification evidence -> independent capability evaluation -> governed canary/activation"],
"canonical_state": ["discovery is not invocation qualification", "optional provider parameters are capability-gated", "immediate provider rejection is not reported as timeout"],
"state_transitions": ["binding selection -> normalized request -> provider invocation -> typed result"],
"side_effects_and_privileges": ["no secrets or authorization headers in evidence", "no direct database access", "no provider calls outside the existing gateway boundary"],
"persistence_and_migrations": ["no schema change"],
"failure_retry_restart": ["household invocation remains bounded by its existing 12-second Core budget", "qualification/revalidation is a separately bounded control-plane operation", "single-flight qualification per provider/model avoids duplicate work", "no timeout inflation; no infinite retry; preserve fail-closed/fallback semantics"],
"failure_retry_restart": ["bounded invocation; no timeout inflation; no infinite retry; preserve fail-closed/fallback semantics"],
"assumptions": ["Local MEDIUM gateway remains the existing provider endpoint", "candidate slot B is available for isolated verification", "cmd/core-runtime/learning_test.go is a preserved carry-forward regression from the preceding runtime task"],
"acceptance_scenarios": ["safe phase trace emits correlation IDs and durations without prompts, answers, credentials, secrets, or user memory", "minimal invocation completes", "ordinary and bounded-context invocation complete", "immediate provider 4xx maps to typed rejection without timeout", "qualification requires bounded invocation evidence", "duplicate qualification is bounded and deduplicated", "fresh validated binding is not synchronously requalified on an ordinary household request"],
"acceptance_scenarios": ["minimal invocation completes", "ordinary and bounded-context invocation complete", "immediate provider 4xx maps to typed rejection without timeout", "qualification requires bounded invocation evidence"],
"required_tests": ["targeted go test", "targeted go vet", "preflight", "check-diff", "independent", "architecture checks", "git diff --check"],
"terminal_gap_states": ["BLOCKED_BY_ARCHITECTURE_GAP", "BLOCKED_BY_SPEC_GAP", "BLOCKED_BY_MISSING_CAPABILITY", "REQUIRES_MIGRATION_DECISION", "BLOCKED_BY_DEPLOYMENT_CONTROL"]
}
@@ -1,47 +0,0 @@
{
"evidence_id": "gen2-cognitive-003-boundary-2026-08-31",
"task_id": "GEN2-COGNITIVE-003",
"task_revision": 3,
"status": "PARTIAL_CORE_VERIFIED / ACTIVATION_REQUIRES_INDEPENDENT_EVALUATION",
"owner_amendment": {
"browser_authenticated_acceptance": "DEFERRED_BY_OWNER",
"ui": "OUT_OF_SCOPE",
"v1_retirement": "NOT_AUTHORIZED",
"gen3": "NOT_STARTED"
},
"gateway_runtime": {
"revision": "a23c36eb9d0c7c73c643b932c184ec1e0adf57ff",
"stable_health": "healthy",
"bounded_qwen_smoke": "HTTP 200 in 4.310 seconds",
"warm_gateway_trace": {
"admission_millis": 13,
"upstream_millis": 1011,
"end_to_end_millis": 1144
}
},
"candidate_slot_b": {
"revision": "cb50b7357b4c37d2add40db6776355b68a074dd0",
"components": "creator-web, core-runtime, cognitive-worker, research-runtime healthy",
"local_medium_discovery": "AVAILABLE",
"bounded_qualification": "HTTP 200 in 1.390 seconds with one-token output budget",
"advisory": "HTTP 200 through the candidate cognitive route within the 12-second budget"
},
"canonical_lifecycle_state": {
"provider": "local-medium-primary",
"qualification": "DISCOVERED",
"activation": "INACTIVE",
"health": "AVAILABLE",
"interpretation": "Discovery and reachability evidence do not self-activate a cognitive binding."
},
"boundary": {
"required_next_transition": "independent evaluation/canary promotion through the protected Capability/Core owner path",
"not_performed": [
"direct database update",
"manual admission lease reset",
"fallback substitution for Local MEDIUM",
"automatic activation",
"browser or authentication bypass"
],
"why_task_cannot_complete": "The active packet permits provider invocation and typed immediate failure mapping, but does not authorize the independent Core/Capability activation decision required to reach VALIDATED/ACTIVE."
}
}
@@ -1,44 +0,0 @@
{
"evidence_id": "gen2-cognitive-003-production-invocation-2026-09-01",
"task_id": "GEN2-COGNITIVE-003",
"task_revision": 4,
"recorded_at": "2026-09-01T00:11:44.8896901Z",
"environment": "production isolated candidate slot B",
"candidate_revision": "dd9bca3ace51e1c0af50fd67dba20caad17cf391",
"candidate_components": {
"creator-web": "healthy",
"core-runtime": "healthy",
"cognitive-worker": "healthy",
"research-runtime": "healthy"
},
"binding": {
"provider_instance": "local-medium-primary",
"qualification": "VALIDATED",
"activation": "ACTIVE",
"health": "AVAILABLE",
"model": "gemma-4-26B-A4B-it-UD-Q4_K_M"
},
"invocation": {
"path": "candidate core network -> candidate cognitive-worker -> Capability resolver -> local-medium-primary",
"before_restart": {
"finish_reason": "COMPLETED",
"response_observed": true
},
"after_restart": {
"finish_reason": "COMPLETED",
"response_observed": true
}
},
"restart": {
"components": ["core-runtime", "cognitive-worker"],
"readiness": "healthy",
"binding_reselected": true,
"stable_untouched": true
},
"security": {
"credentials_or_secrets_logged": false,
"provider_payload_logged": false,
"manual_state_mutation": false
},
"status": "LOCAL_MEDIUM_INVOCATION_AND_RESTART_RECOVERY_VERIFIED"
}
@@ -1,47 +0,0 @@
{
"task_id": "GEN2-COGNITIVE-003",
"task_revision": 4,
"recorded_at": "2026-08-31T07:28:00Z",
"revision": "dd9bca3ace51e1c0af50fd67dba20caad17cf391",
"environment": "isolated candidate slot B",
"candidate_components": {
"creator-web": "healthy",
"core-runtime": "healthy",
"cognitive-worker": "healthy",
"research-runtime": "healthy"
},
"qualification_phase_trace": {
"correlation_id": "92cfec5eca6c57c59cf48da6d1644005",
"provider": "local-medium-primary",
"model": "Qwen3.6-27B-UD-Q4_K_XL",
"result": "REACHABLE",
"finish_reason": "COMPLETED",
"invocation_latency_millis": 3025,
"phases": {
"Q0_request_accepted": 0,
"Q1_binding_candidate_loaded": 0,
"Q2_provider_model_resolved": 342,
"Q3_bounded_invocation_started": 342,
"Q4_bounded_invocation_completed": 3367,
"Q5_invocation_evidence_built": 3368,
"Q6_observation_persistence_started": 3368,
"Q7_observation_persistence_completed": 3371,
"Q8_independent_evaluation_not_started": 3371,
"Q17_response_returned": 3371
}
},
"findings": [
"The bounded Local MEDIUM provider invocation completed in 3.025 seconds.",
"The canonical qualification-observation persistence segment consumed 3 milliseconds and did not approach the 12-second household request budget.",
"The worker intentionally stops after operational qualification evidence. It does not create independent evaluation, CANARY, or ACTIVE state.",
"The existing governed activation path is Core POST /v1/creator/cognitive-canary followed by POST /v1/creator/cognitive-activate; both require a creator session and Capability-owned evaluation evidence."
],
"security": {
"prompts_logged": false,
"model_answers_logged": false,
"credentials_or_secrets_logged": false,
"direct_database_state_change": false,
"manual_admission_lease_change": false
},
"status": "QUALIFICATION_EVIDENCE_RECORDED_ACTIVATION_REQUIRES_EXISTING_GOVERNED_AUTHORIZED_PATH"
}
@@ -0,0 +1,17 @@
# GEN2-DEPLOY-004 candidate shadow secret capability gap
Date: 2026-08-31
The candidate Compose slot requires two distinct external, read-only Docker
volumes: `assistant_candidate_notification_shadow_writer_secret` and
`assistant_candidate_notification_shadow_reader_secret`. They are deliberately
not aliases for the production notification writer or reader volumes.
A read-only Docker volume inspection on `docker.cin.su` confirmed that both
candidate-only volumes are absent. No secret values were read or emitted.
The candidate ReleaseSet must not be started until the existing production
secret control plane provisions both bounded artifacts with distinct values.
This preserves the fail-closed, zero-external-delivery property of the shadow
notification path. Stable containers and production notification artifacts
were not modified.
+24 -5
View File
@@ -28,10 +28,17 @@ type Release struct {
}
const (
ComponentCreatorWeb = "creator-web"
ComponentCoreRuntime = "core-runtime"
ComponentCognitiveWorker = "cognitive-worker"
ComponentResearchRuntime = "research-runtime"
ComponentCreatorWeb = "creator-web"
ComponentCoreRuntime = "core-runtime"
ComponentCognitiveWorker = "cognitive-worker"
ComponentResearchRuntime = "research-runtime"
ComponentCommitRuntime = "commit-runtime"
ComponentContinuityRuntime = "continuity-runtime"
ComponentAgentWorker = "agent-worker"
ComponentAgentExecutionBridge = "agent-execution-bridge"
ComponentWorkGraphRuntime = "work-graph-runtime"
ComponentTaskVerifier = "task-verifier"
ComponentNotificationReceiver = "notification-receiver"
)
// ComponentRelease is operational deployment evidence. It is deliberately
@@ -54,7 +61,19 @@ type ReleaseSet struct {
Components map[string]ComponentRelease `json:"components"`
}
var requiredComponents = []string{ComponentCreatorWeb, ComponentCoreRuntime, ComponentCognitiveWorker, ComponentResearchRuntime}
var requiredComponents = []string{
ComponentCreatorWeb,
ComponentCoreRuntime,
ComponentCognitiveWorker,
ComponentResearchRuntime,
ComponentCommitRuntime,
ComponentContinuityRuntime,
ComponentAgentWorker,
ComponentAgentExecutionBridge,
ComponentWorkGraphRuntime,
ComponentTaskVerifier,
ComponentNotificationReceiver,
}
func (s ReleaseSet) Validate() bool {
if s.ID == "" || s.Revision == "" || len(s.Components) != len(requiredComponents) {
+7 -2
View File
@@ -84,12 +84,17 @@ func TestPersistentSupervisorRestoresTrafficModeAfterRestart(t *testing.T) {
}
func testReleaseSet(id, revision, web, core, cognitive, research string) supervisor.ReleaseSet {
return supervisor.ReleaseSet{ID: id, Revision: revision, Components: map[string]supervisor.ComponentRelease{
components := map[string]supervisor.ComponentRelease{
supervisor.ComponentCreatorWeb: {Name: supervisor.ComponentCreatorWeb, Revision: revision, Upstream: web, HealthURL: web + "/healthz", ReadyURL: web + "/readyz"},
supervisor.ComponentCoreRuntime: {Name: supervisor.ComponentCoreRuntime, Revision: revision, Upstream: core, HealthURL: core + "/healthz", ReadyURL: core + "/readyz"},
supervisor.ComponentCognitiveWorker: {Name: supervisor.ComponentCognitiveWorker, Revision: revision, Upstream: cognitive, HealthURL: cognitive + "/healthz", ReadyURL: cognitive + "/readyz"},
supervisor.ComponentResearchRuntime: {Name: supervisor.ComponentResearchRuntime, Revision: revision, Upstream: research, HealthURL: research + "/healthz", ReadyURL: research + "/readyz"},
}}
}
for _, name := range []string{supervisor.ComponentCommitRuntime, supervisor.ComponentContinuityRuntime, supervisor.ComponentAgentWorker, supervisor.ComponentAgentExecutionBridge, supervisor.ComponentWorkGraphRuntime, supervisor.ComponentTaskVerifier, supervisor.ComponentNotificationReceiver} {
upstream := "http://" + name
components[name] = supervisor.ComponentRelease{Name: name, Revision: revision, Upstream: upstream, HealthURL: upstream + "/healthz", ReadyURL: upstream + "/readyz"}
}
return supervisor.ReleaseSet{ID: id, Revision: revision, Components: components}
}
func TestReleaseSetRejectsMixedRevisionsAndMissingComponents(t *testing.T) {
@@ -11,9 +11,11 @@ stable_updater="$root/deploy/production-stable-updater.sh"
grep -Fq 'CANDIDATE_PROJECT="${CANDIDATE_PROJECT:-assistant-slot-$SLOT}"' "$canary"
grep -Fq '[[ "$CANDIDATE_PROJECT" != "$STABLE_PROJECT" ]]' "$canary"
grep -Fq 'docker compose -p "$CANDIDATE_PROJECT"' "$canary"
grep -Fq 'up -d --no-build --no-deps research-runtime core-runtime' "$canary"
grep -Fq 'up -d --no-build --no-deps cognitive-worker' "$canary"
grep -Fq 'up -d --no-build --no-deps creator-web' "$canary"
grep -Fq 'services=(creator-web core-runtime cognitive-worker research-runtime commit-runtime continuity-runtime agent-worker agent-execution-bridge work-graph-runtime task-verifier notification-receiver)' "$canary"
grep -Fq 'candidate_compose up -d --no-build --no-deps "${services[@]}"' "$canary"
grep -Fq 'component_port()' "$canary"
grep -Fq 'notification-receiver-candidate' "$candidate_compose"
grep -Fq 'assistant_candidate_notification_shadow_writer_secret' "$candidate_compose"
grep -Fq 'compose up -d --no-build --no-deps supervisor' "$supervisor"
grep -Fq 'stable_ids_before=$(stable_ids)' "$canary"
grep -Fq 'assert_stable_ids_unchanged' "$canary"