fix(readiness): degrade optional fleet providers without blocking fallback #108
@@ -78,7 +78,7 @@ func main() {
|
||||
health.HTTPProbe(readinessClient, strings.TrimRight(envOr("CORE_RUNTIME_URL", "http://core-runtime:8080"), "/")+"/healthz"),
|
||||
health.HTTPProbe(readinessClient, strings.TrimRight(envOr("LOCAL_CPU_PROVIDER_URL", "http://docker.cin.su:18082/v1"), "/")+"/models"),
|
||||
health.HTTPProbe(readinessClient, lifeboatURL),
|
||||
health.HTTPProbe(readinessClient, strings.TrimRight(envOr("LOCAL_GPU_PROVIDER_URL", "http://192.168.220.91:11434"), "/")+"/api/tags"),
|
||||
health.OptionalHTTPProbe(readinessClient, strings.TrimRight(envOr("LOCAL_GPU_PROVIDER_URL", "http://192.168.220.91:11434"), "/")+"/api/tags"),
|
||||
}})
|
||||
mux.HandleFunc("POST /v1/proposals/deterministic", deterministicProposal)
|
||||
mux.HandleFunc("POST /v1/advisory-policy", gateway.policyHandler)
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
{
|
||||
"task_file": "dev/tasks/DEV-000115-cognitive-admission-control.json",
|
||||
"task_id": "DEV-000115",
|
||||
"task_revision": 6
|
||||
"task_file": "dev/tasks/DEV-000141-optional-fleet-readiness.json",
|
||||
"task_id": "DEV-000141",
|
||||
"task_revision": 1
|
||||
}
|
||||
|
||||
@@ -0,0 +1,23 @@
|
||||
{
|
||||
"task_id": "DEV-000141",
|
||||
"task_revision": 1,
|
||||
"architecture_revision": "ADR-0018",
|
||||
"base_repo_revision": "38cff16",
|
||||
"risk_tier": "E",
|
||||
"intent": "Keep cognitive readiness available when an optional opportunistic fleet member is unavailable while preserving typed degraded evidence.",
|
||||
"requirement_ids": ["WP-003", "WP-203", "Fleet failover readiness"],
|
||||
"allowed_change_scope": {"modules": ["runtime-health", "cognitive-worker"], "paths": ["runtime/health/**", "cmd/cognitive-worker/main.go", "runtime/health/server_test.go", "dev/ACTIVE_TASK.json", "dev/tasks/DEV-000141-optional-fleet-readiness.json"]},
|
||||
"forbidden_change_scope": {"paths": ["architecture/**", "database/**", "deploy/**", "security/**", ".gitea/**"]},
|
||||
"allowed_architecture_delta": {"new_cross_module_dependencies": 0, "new_canonical_owners": 0, "new_privileges": 0, "new_third_party_dependencies": 0, "new_database_owner_roles": 0},
|
||||
"contracts": {"consume": ["Existing readiness Probe and typed readiness response"], "may_change": ["Optional dependency probe behavior"]},
|
||||
"semantic_owner": "runtime-health and cognitive-worker",
|
||||
"canonical_state": [],
|
||||
"state_transitions": ["optional provider unavailable -> readiness remains ready with typed degraded check", "required dependency unavailable -> readiness remains degraded"],
|
||||
"side_effects_and_privileges": ["No external mutation; no provider invocation"],
|
||||
"persistence_and_migrations": ["None"],
|
||||
"failure_retry_restart": ["Optional provider can recover on subsequent readiness probe"],
|
||||
"assumptions": ["CPU/Lifeboat remains a qualified fallback when opportunistic GPU is unavailable"],
|
||||
"acceptance_scenarios": ["GPU timeout does not hide degraded evidence or make fallback unavailable", "required dependency failure still returns 503"],
|
||||
"required_tests": ["go test ./...", "go vet ./...", "preflight", "check-diff", "check-architecture", "independent", "git diff --check", "docker compose config"],
|
||||
"terminal_gap_states": ["BLOCKED_BY_SPEC_GAP", "BLOCKED_BY_CONFLICT"]
|
||||
}
|
||||
@@ -26,8 +26,9 @@ func (h Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
|
||||
type Probe struct {
|
||||
Name string
|
||||
Check func(context.Context) error
|
||||
Name string
|
||||
Check func(context.Context) error
|
||||
Optional bool
|
||||
}
|
||||
|
||||
type Result struct {
|
||||
@@ -59,11 +60,13 @@ func (h ReadinessHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
err := probe.Check(ctx)
|
||||
cancel()
|
||||
if err != nil {
|
||||
ready = false
|
||||
result.State, result.Reason, result.Detail = "DEGRADED", "DEPENDENCY_UNAVAILABLE", err.Error()
|
||||
if !probe.Optional {
|
||||
ready = false
|
||||
}
|
||||
}
|
||||
}
|
||||
if result.State != "READY" {
|
||||
if result.State != "READY" && !probe.Optional {
|
||||
ready = false
|
||||
}
|
||||
results = append(results, result)
|
||||
@@ -96,6 +99,15 @@ func HTTPProbe(client *http.Client, endpoint string) Probe {
|
||||
}}
|
||||
}
|
||||
|
||||
// OptionalHTTPProbe reports dependency degradation without making the whole
|
||||
// runtime unavailable. This is used for opportunistic fleet members when a
|
||||
// qualified local fallback remains available.
|
||||
func OptionalHTTPProbe(client *http.Client, endpoint string) Probe {
|
||||
probe := HTTPProbe(client, endpoint)
|
||||
probe.Optional = true
|
||||
return probe
|
||||
}
|
||||
|
||||
func TCPProbe(name, host, port string) Probe {
|
||||
return Probe{Name: name, Check: func(ctx context.Context) error {
|
||||
connection, err := (&net.Dialer{}).DialContext(ctx, "tcp", net.JoinHostPort(host, port))
|
||||
|
||||
@@ -26,3 +26,16 @@ func TestReadinessReturns200WhenAllProbesPass(t *testing.T) {
|
||||
t.Fatalf("readiness response = %d %s", recorder.Code, recorder.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestReadinessStaysAvailableWithOptionalDependencyDegraded(t *testing.T) {
|
||||
handler := ReadinessHandler{Runtime: "test", Probes: []Probe{
|
||||
{Name: "lifeboat", Check: func(context.Context) error { return nil }},
|
||||
{Name: "opportunistic-gpu", Optional: true, Check: func(context.Context) error { return errors.New("timeout") }},
|
||||
}}
|
||||
recorder := httptest.NewRecorder()
|
||||
handler.ServeHTTP(recorder, httptest.NewRequest(http.MethodGet, "/readyz", nil))
|
||||
body := recorder.Body.String()
|
||||
if recorder.Code != http.StatusOK || !strings.Contains(body, `"status":"ready"`) || !strings.Contains(body, `"state":"DEGRADED"`) {
|
||||
t.Fatalf("readiness response = %d %s", recorder.Code, body)
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user