diff --git a/cmd/cognitive-worker/fleet_status.go b/cmd/cognitive-worker/fleet_status.go index 15a3c45..bd007cf 100644 --- a/cmd/cognitive-worker/fleet_status.go +++ b/cmd/cognitive-worker/fleet_status.go @@ -18,6 +18,7 @@ type fleetEndpoint struct { Tier capability.FleetTier URL string Headers map[string]string + Client *http.Client } type fleetStatusResponse struct { @@ -41,7 +42,11 @@ func fleetReadinessHandler(client *http.Client, preferred capability.FleetTier, for name, value := range endpoint.Headers { request.Header.Set(name, value) } - response, requestErr := client.Do(request) + probeClient := endpoint.Client + if probeClient == nil { + probeClient = client + } + response, requestErr := probeClient.Do(request) if requestErr == nil { response.Body.Close() if response.StatusCode >= 200 && response.StatusCode < 300 { diff --git a/cmd/cognitive-worker/main.go b/cmd/cognitive-worker/main.go index d70ff5c..8a825f4 100644 --- a/cmd/cognitive-worker/main.go +++ b/cmd/cognitive-worker/main.go @@ -92,7 +92,9 @@ func main() { if cpuAPIKey != "" { cpuFleetHeaders["Authorization"] = "Bearer " + cpuAPIKey } - cpuHealthProbe := health.OptionalHTTPProbeWithHeaders(readinessClient, strings.TrimRight(envOr("LOCAL_CPU_PROVIDER_URL", "http://docker.cin.su:18082/v1"), "/")+"/models", cpuFleetHeaders) + cpuEndpoint := strings.TrimRight(envOr("LOCAL_CPU_PROVIDER_URL", "http://docker.cin.su:18082/v1"), "/") + "/models" + cpuHealthProbe := health.OptionalHTTPProbeWithHeaders(readinessClient, cpuEndpoint, cpuFleetHeaders) + cpuHealthProbe.Timeout = 10 * time.Second readinessProbes := []health.Probe{ health.HTTPProbe(readinessClient, strings.TrimRight(envOr("CORE_RUNTIME_URL", "http://core-runtime:8080"), "/")+"/healthz"), cpuHealthProbe, @@ -101,7 +103,7 @@ func main() { } mux.Handle("/readyz", health.ReadinessHandler{Runtime: "cognitive-worker", Probes: readinessProbes}) mux.Handle("/v1/fleet/readiness", fleetReadinessHandler(fleetClient, preferredFleet, []fleetEndpoint{ - {ProviderID: "local-cpu-primary", ModelID: "local-cpu-model", Tier: capability.FleetTierFast, URL: strings.TrimRight(envOr("LOCAL_CPU_PROVIDER_URL", "http://docker.cin.su:18082/v1"), "/") + "/models", Headers: cpuFleetHeaders}, + {ProviderID: "local-cpu-primary", ModelID: "local-cpu-model", Tier: capability.FleetTierFast, URL: cpuEndpoint, Headers: cpuFleetHeaders, Client: &http.Client{Timeout: 10 * time.Second}}, {ProviderID: "local-medium-unconfigured", ModelID: "medium-unconfigured", Tier: capability.FleetTierMedium}, {ProviderID: "local-gpu-opportunistic", ModelID: "gpu-opportunistic", Tier: capability.FleetTierHeavy, URL: strings.TrimRight(envOr("LOCAL_GPU_PROVIDER_URL", "http://192.168.220.91:11434"), "/") + "/api/tags"}, {ProviderID: "cognitive-lifeboat", ModelID: "qwen3-8b-lifeboat", Tier: capability.FleetTierLifeboat, URL: lifeboatURL}, diff --git a/dev/ACTIVE_TASK.json b/dev/ACTIVE_TASK.json index b930a3c..54927b7 100644 --- a/dev/ACTIVE_TASK.json +++ b/dev/ACTIVE_TASK.json @@ -1,5 +1,5 @@ { - "task_file": "dev/tasks/DEV-000165-authorized-cpu-health-probe.json", - "task_id": "DEV-000165", + "task_file": "dev/tasks/DEV-000166-per-provider-readiness-timeouts.json", + "task_id": "DEV-000166", "task_revision": 1 } diff --git a/dev/tasks/DEV-000166-per-provider-readiness-timeouts.json b/dev/tasks/DEV-000166-per-provider-readiness-timeouts.json new file mode 100644 index 0000000..4e3d17b --- /dev/null +++ b/dev/tasks/DEV-000166-per-provider-readiness-timeouts.json @@ -0,0 +1,23 @@ +{ + "task_id": "DEV-000166", + "task_revision": 1, + "architecture_revision": "ADR-0018", + "base_repo_revision": "4086b7522460a78afc341d996ca7874505018871", + "risk_tier": "E", + "intent": "Give the authenticated CPU readiness probe the provider adapter timeout while retaining short opportunistic GPU probing.", + "requirement_ids": ["WP-601", "Model fleet/failover"], + "allowed_change_scope": {"modules": ["cognitive-worker", "platform-health"], "paths": ["runtime/health/server.go", "runtime/health/server_test.go", "cmd/cognitive-worker/main.go", "cmd/cognitive-worker/fleet_status.go", "dev/ACTIVE_TASK.json", "dev/tasks/DEV-000166-per-provider-readiness-timeouts.json"]}, + "forbidden_change_scope": {"paths": ["architecture/**", "security/**", "database/**", "deploy/**", "contracts/**", "adapters/**", ".gitea/**"]}, + "allowed_architecture_delta": {"new_cross_module_dependencies": 0, "new_canonical_owners": 0, "new_privileges": 0, "new_third_party_dependencies": 0, "new_database_owner_roles": 0}, + "contracts": {"consume": ["existing health Probe contract", "existing fleet readiness probe"], "may_change": ["per-probe timeout metadata"]}, + "semantic_owner": "cognitive-worker/platform health", + "canonical_state": [], + "state_transitions": ["provider-specific probe deadline -> truthful readiness result"], + "side_effects_and_privileges": ["no new privilege; existing provider auth only"], + "persistence_and_migrations": ["none"], + "failure_retry_restart": ["CPU gets bounded 10-second probe; GPU retains 2-second opportunistic timeout"], + "assumptions": ["CPU provider catalog may take longer than opportunistic probes while remaining within adapter bounds"], + "acceptance_scenarios": ["per-probe health timeout is honored", "CPU fleet client uses 10 seconds", "GPU remains bounded", "Lifeboat fallback remains unchanged"], + "required_tests": ["go test ./...", "go vet ./...", "preflight", "check-diff", "check-architecture", "independent", "git diff --check", "CI", "production /readyz and fleet readiness"], + "terminal_gap_states": ["BLOCKED_BY_MISSING_CAPABILITY", "BLOCKED_BY_CONFLICT"] +} diff --git a/runtime/health/server.go b/runtime/health/server.go index 1f81555..dcd1c19 100644 --- a/runtime/health/server.go +++ b/runtime/health/server.go @@ -29,6 +29,7 @@ type Probe struct { Name string Check func(context.Context) error Optional bool + Timeout time.Duration } type Result struct { @@ -56,7 +57,11 @@ func (h ReadinessHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { if probe.Check == nil { result.State, result.Reason, result.Detail = "DEGRADED", "PROBE_NOT_CONFIGURED", "probe is not configured" } else { - ctx, cancel := context.WithTimeout(r.Context(), timeout) + probeTimeout := timeout + if probe.Timeout > 0 { + probeTimeout = probe.Timeout + } + ctx, cancel := context.WithTimeout(r.Context(), probeTimeout) err := probe.Check(ctx) cancel() if err != nil { diff --git a/runtime/health/server_test.go b/runtime/health/server_test.go index 2d8caaf..c7afba1 100644 --- a/runtime/health/server_test.go +++ b/runtime/health/server_test.go @@ -7,6 +7,7 @@ import ( "net/http/httptest" "strings" "testing" + "time" ) func TestReadinessReturns503WithTypedDependencyReason(t *testing.T) { @@ -41,6 +42,17 @@ func TestHTTPProbeWithHeadersSendsHeaders(t *testing.T) { } } +func TestReadinessUsesPerProbeTimeout(t *testing.T) { + handler := ReadinessHandler{Runtime: "test", Timeout: time.Millisecond, Probes: []Probe{{ + Name: "slow-but-required", Timeout: time.Second, Check: func(context.Context) error { return nil }, + }}} + recorder := httptest.NewRecorder() + handler.ServeHTTP(recorder, httptest.NewRequest(http.MethodGet, "/readyz", nil)) + if recorder.Code != http.StatusOK { + t.Fatalf("readiness response = %d %s", recorder.Code, recorder.Body.String()) + } +} + func TestReadinessStaysAvailableWithOptionalDependencyDegraded(t *testing.T) { handler := ReadinessHandler{Runtime: "test", Probes: []Probe{ {Name: "lifeboat", Check: func(context.Context) error { return nil }},