fix(readiness): degrade optional fleet providers without blocking fallback #108

Merged
m merged 1 commits from codex/wp-002-recovery into main 2026-08-22 12:06:16 +00:00
5 changed files with 56 additions and 8 deletions
+1 -1
View File
@@ -78,7 +78,7 @@ func main() {
health.HTTPProbe(readinessClient, strings.TrimRight(envOr("CORE_RUNTIME_URL", "http://core-runtime:8080"), "/")+"/healthz"),
health.HTTPProbe(readinessClient, strings.TrimRight(envOr("LOCAL_CPU_PROVIDER_URL", "http://docker.cin.su:18082/v1"), "/")+"/models"),
health.HTTPProbe(readinessClient, lifeboatURL),
health.HTTPProbe(readinessClient, strings.TrimRight(envOr("LOCAL_GPU_PROVIDER_URL", "http://192.168.220.91:11434"), "/")+"/api/tags"),
health.OptionalHTTPProbe(readinessClient, strings.TrimRight(envOr("LOCAL_GPU_PROVIDER_URL", "http://192.168.220.91:11434"), "/")+"/api/tags"),
}})
mux.HandleFunc("POST /v1/proposals/deterministic", deterministicProposal)
mux.HandleFunc("POST /v1/advisory-policy", gateway.policyHandler)
+3 -3
View File
@@ -1,5 +1,5 @@
{
"task_file": "dev/tasks/DEV-000115-cognitive-admission-control.json",
"task_id": "DEV-000115",
"task_revision": 6
"task_file": "dev/tasks/DEV-000141-optional-fleet-readiness.json",
"task_id": "DEV-000141",
"task_revision": 1
}
@@ -0,0 +1,23 @@
{
"task_id": "DEV-000141",
"task_revision": 1,
"architecture_revision": "ADR-0018",
"base_repo_revision": "38cff16",
"risk_tier": "E",
"intent": "Keep cognitive readiness available when an optional opportunistic fleet member is unavailable while preserving typed degraded evidence.",
"requirement_ids": ["WP-003", "WP-203", "Fleet failover readiness"],
"allowed_change_scope": {"modules": ["runtime-health", "cognitive-worker"], "paths": ["runtime/health/**", "cmd/cognitive-worker/main.go", "runtime/health/server_test.go", "dev/ACTIVE_TASK.json", "dev/tasks/DEV-000141-optional-fleet-readiness.json"]},
"forbidden_change_scope": {"paths": ["architecture/**", "database/**", "deploy/**", "security/**", ".gitea/**"]},
"allowed_architecture_delta": {"new_cross_module_dependencies": 0, "new_canonical_owners": 0, "new_privileges": 0, "new_third_party_dependencies": 0, "new_database_owner_roles": 0},
"contracts": {"consume": ["Existing readiness Probe and typed readiness response"], "may_change": ["Optional dependency probe behavior"]},
"semantic_owner": "runtime-health and cognitive-worker",
"canonical_state": [],
"state_transitions": ["optional provider unavailable -> readiness remains ready with typed degraded check", "required dependency unavailable -> readiness remains degraded"],
"side_effects_and_privileges": ["No external mutation; no provider invocation"],
"persistence_and_migrations": ["None"],
"failure_retry_restart": ["Optional provider can recover on subsequent readiness probe"],
"assumptions": ["CPU/Lifeboat remains a qualified fallback when opportunistic GPU is unavailable"],
"acceptance_scenarios": ["GPU timeout does not hide degraded evidence or make fallback unavailable", "required dependency failure still returns 503"],
"required_tests": ["go test ./...", "go vet ./...", "preflight", "check-diff", "check-architecture", "independent", "git diff --check", "docker compose config"],
"terminal_gap_states": ["BLOCKED_BY_SPEC_GAP", "BLOCKED_BY_CONFLICT"]
}
+16 -4
View File
@@ -26,8 +26,9 @@ func (h Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
}
type Probe struct {
Name string
Check func(context.Context) error
Name string
Check func(context.Context) error
Optional bool
}
type Result struct {
@@ -59,11 +60,13 @@ func (h ReadinessHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
err := probe.Check(ctx)
cancel()
if err != nil {
ready = false
result.State, result.Reason, result.Detail = "DEGRADED", "DEPENDENCY_UNAVAILABLE", err.Error()
if !probe.Optional {
ready = false
}
}
}
if result.State != "READY" {
if result.State != "READY" && !probe.Optional {
ready = false
}
results = append(results, result)
@@ -96,6 +99,15 @@ func HTTPProbe(client *http.Client, endpoint string) Probe {
}}
}
// OptionalHTTPProbe reports dependency degradation without making the whole
// runtime unavailable. This is used for opportunistic fleet members when a
// qualified local fallback remains available.
func OptionalHTTPProbe(client *http.Client, endpoint string) Probe {
probe := HTTPProbe(client, endpoint)
probe.Optional = true
return probe
}
func TCPProbe(name, host, port string) Probe {
return Probe{Name: name, Check: func(ctx context.Context) error {
connection, err := (&net.Dialer{}).DialContext(ctx, "tcp", net.JoinHostPort(host, port))
+13
View File
@@ -26,3 +26,16 @@ func TestReadinessReturns200WhenAllProbesPass(t *testing.T) {
t.Fatalf("readiness response = %d %s", recorder.Code, recorder.Body.String())
}
}
func TestReadinessStaysAvailableWithOptionalDependencyDegraded(t *testing.T) {
handler := ReadinessHandler{Runtime: "test", Probes: []Probe{
{Name: "lifeboat", Check: func(context.Context) error { return nil }},
{Name: "opportunistic-gpu", Optional: true, Check: func(context.Context) error { return errors.New("timeout") }},
}}
recorder := httptest.NewRecorder()
handler.ServeHTTP(recorder, httptest.NewRequest(http.MethodGet, "/readyz", nil))
body := recorder.Body.String()
if recorder.Code != http.StatusOK || !strings.Contains(body, `"status":"ready"`) || !strings.Contains(body, `"state":"DEGRADED"`) {
t.Fatalf("readiness response = %d %s", recorder.Code, body)
}
}