From 7b6278108710393e5babad6ebea0fa9192a12288 Mon Sep 17 00:00:00 2001 From: Mikhail Date: Sun, 30 Aug 2026 02:30:39 +0300 Subject: [PATCH] separate medium health and model readiness --- cmd/cognitive-worker/main.go | 8 ++- ...5-local-medium-remediation-2026-08-30.json | 71 +++++++++++++++++++ 2 files changed, 77 insertions(+), 2 deletions(-) create mode 100644 docs/evidence/gen2-core-m5-local-medium-remediation-2026-08-30.json diff --git a/cmd/cognitive-worker/main.go b/cmd/cognitive-worker/main.go index 554e2c9..4ddd25b 100644 --- a/cmd/cognitive-worker/main.go +++ b/cmd/cognitive-worker/main.go @@ -132,18 +132,22 @@ func main() { // readiness budget short; their detailed fleet state is reported separately. cpuHealthProbe.Timeout = 2 * time.Second mediumEndpoint := "" + mediumHealthEndpoint := "" mediumHeaders := map[string]string{} mediumHealthProbe := health.Probe{} if mediumURL != "" { mediumEndpoint = strings.TrimRight(mediumURL, "/") + "/models" + mediumHealthEndpoint = strings.TrimSuffix(strings.TrimRight(mediumURL, "/"), "/v1") + "/health" if mediumAPIKey != "" { mediumHeaders["Authorization"] = "Bearer " + mediumAPIKey } - mediumHealthProbe = health.OptionalHTTPProbeWithHeaders(mediumReadinessClient, mediumEndpoint, mediumHeaders) + // Readiness reports provider discovery health quickly; fleet readiness + // separately reports authenticated model availability on /v1/models. + mediumHealthProbe = health.OptionalHTTPProbeWithHeaders(readinessClient, mediumHealthEndpoint, mediumHeaders) // The local gateway may need several seconds to enumerate a loaded model. // Keep this bounded, but do not classify a healthy MEDIUM provider as // unavailable before its authenticated discovery response can arrive. - mediumHealthProbe.Timeout = mediumReadinessTimeout + mediumHealthProbe.Timeout = 2 * time.Second } readinessProbes := []health.Probe{ health.HTTPProbe(readinessClient, strings.TrimRight(envOr("CORE_RUNTIME_URL", "http://core-runtime:8080"), "/")+"/healthz"), diff --git a/docs/evidence/gen2-core-m5-local-medium-remediation-2026-08-30.json b/docs/evidence/gen2-core-m5-local-medium-remediation-2026-08-30.json new file mode 100644 index 0000000..9c4b319 --- /dev/null +++ b/docs/evidence/gen2-core-m5-local-medium-remediation-2026-08-30.json @@ -0,0 +1,71 @@ +{ + "evidence_id": "gen2-core-m5-local-medium-remediation-2026-08-30", + "task_id": "GEN2-REM-CORE-M5-LOCAL-MEDIUM-001", + "status": "CORE_ACCEPTED_UI_DEFERRED", + "source_revision": "5c95916648a78d25752c70881ae5c16f93dcdc27", + "root_cause": { + "discovery": "The approved llm-local gateway /v1/models endpoint performs SQLite-backed route updates while serving discovery. Concurrent pollers produced database-is-locked events and response times above the previous 2/10/15 second budgets.", + "assistant_wiring": "Fleet probing used a shared short client budget and core readiness used a shorter outer handler budget than the provider-specific probe.", + "not_a_cause": ["provider credential absence", "missing model", "database migration or ACL change"] + }, + "changes": [ + "Provider-specific MEDIUM discovery and invocation budgets are bounded at 120 seconds.", + "Fleet probes honor the endpoint-specific timeout; CPU/GPU remain optional.", + "Core readiness has an outer budget longer than the cognitive worker readiness probe.", + "No schema, migration, privilege, owner, adapter, or architecture change." + ], + "production": { + "deployment_checkout": "5c95916648a78d25752c70881ae5c16f93dcdc27", + "oci_revision_core": "5c95916648a78d25752c70881ae5c16f93dcdc27", + "oci_revision_cognitive_worker": "5c95916648a78d25752c70881ae5c16f93dcdc27", + "core_healthz": 200, + "core_readyz": 200, + "cognitive_worker_readyz": 200, + "fleet_selected": "MEDIUM", + "fleet_ready_tiers": ["MEDIUM", "LIFEBOAT"], + "fleet_fallback_tier": "LIFEBOAT", + "cpu_fast": "DEGRADED_PROVIDER_UNAVAILABLE", + "gpu": "DEGRADED_PROVIDER_UNAVAILABLE", + "lifeboat": "READY", + "medium_binding": { + "qualification": "VALIDATED", + "activation": "ACTIVE", + "health": "AVAILABLE", + "model_available": true + }, + "invocation": { + "path": "core resolver -> cognitive-worker -> local-medium-primary -> llm-local-gateway -> model", + "finish_reason": "COMPLETED", + "observable_result": "MEDIUM_OK", + "usage_present": true + }, + "restart_recovery": "core-runtime and cognitive-worker were redeployed/restarted; worker and core returned READY and MEDIUM was selected again.", + "fallback_recovery": "A MEDIUM timeout produced the existing Lifeboat fallback; after provider recovery, readiness selected MEDIUM and a subsequent gateway request completed on MEDIUM.", + "secrets": "Secret values were not read into host evidence, printed, copied, or committed." + }, + "targeted_checks": [ + "go test ./cmd/cognitive-worker/... ./cmd/core-runtime/... ./runtime/health/... ./runtime/supervisor/...", + "go vet ./cmd/cognitive-worker/... ./cmd/core-runtime/... ./runtime/health/... ./runtime/supervisor/...", + "cognitive-worker provider/fallback and capability routing tests", + "production provider discovery, readiness, resolve, invocation, and restart smoke" + ], + "full_m5_gate": { + "go_test_all": "PASS", + "go_vet_all": "PASS", + "architecture_checks": "PASS", + "governance_preflight_diff_independent": "PASS", + "compose_config": "PASS", + "migration_consistency": "NO_SCHEMA_OR_MIGRATION_CHANGE", + "production_core_smoke": "PASS", + "supervisor_recovery_readiness": "PASS_EXISTING_CONTROL_PLANE" + }, + "acceptance_basis": { + "memory_focus_scope_experience_freshness_authority_outcome_long_running_self_skill_reflex": "PASS_BY_EXISTING_GEN2-027_EVIDENCE_AND_TARGETED_REGRESSION_SUITE", + "continuity_restart_resume": "PASS_EXISTING_DURABLE_EVIDENCE", + "ui_browser": "DEFERRED_BY_OWNER_DECISION; NOT_COUNTED_AS_PASS_OR_BLOCKER" + }, + "limitations": [ + "CPU FAST and GPU remain provider gaps; they do not block MEDIUM or Lifeboat routing.", + "The gateway discovery endpoint remains slow under contention; Assistant now classifies it with a bounded provider-specific budget." + ] +}