diff --git a/cmd/cognitive-worker/fleet_status_test.go b/cmd/cognitive-worker/fleet_status_test.go index c30d976..45f1296 100644 --- a/cmd/cognitive-worker/fleet_status_test.go +++ b/cmd/cognitive-worker/fleet_status_test.go @@ -9,7 +9,7 @@ import ( "git.cin.su/m/Assistant/contracts/capability" ) -func TestFleetReadinessSelectsLifeboatWhenPrimaryTiersUnavailable(t *testing.T) { +func TestFleetReadinessSelectsBestAvailablePrimaryTierBeforeLifeboat(t *testing.T) { server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusOK) })) defer server.Close() handler := fleetReadinessHandler(server.Client(), capability.FleetTierHeavy, []fleetEndpoint{ @@ -21,7 +21,7 @@ func TestFleetReadinessSelectsLifeboatWhenPrimaryTiersUnavailable(t *testing.T) recorder := httptest.NewRecorder() request := httptest.NewRequest(http.MethodGet, "/v1/fleet/readiness", nil) handler.ServeHTTP(recorder, request) - if recorder.Code != http.StatusOK || !strings.Contains(recorder.Body.String(), `"selected_tier":"LIFEBOAT"`) || !strings.Contains(recorder.Body.String(), `"tier":"MEDIUM"`) { + if recorder.Code != http.StatusOK || !strings.Contains(recorder.Body.String(), `"selected_tier":"FAST"`) || !strings.Contains(recorder.Body.String(), `"tier":"MEDIUM"`) { t.Fatalf("status=%d body=%s", recorder.Code, recorder.Body.String()) } } diff --git a/dev/ACTIVE_TASK.json b/dev/ACTIVE_TASK.json index f997dce..2faef62 100644 --- a/dev/ACTIVE_TASK.json +++ b/dev/ACTIVE_TASK.json @@ -1,5 +1,5 @@ { - "task_file": "dev/tasks/DEV-000171-connect-existing-medium-runtime.json", - "task_id": "DEV-000171", + "task_file": "dev/tasks/DEV-000172-fleet-failover-priority.json", + "task_id": "DEV-000172", "task_revision": 1 } diff --git a/dev/tasks/DEV-000172-fleet-failover-priority.json b/dev/tasks/DEV-000172-fleet-failover-priority.json new file mode 100644 index 0000000..31d4ca0 --- /dev/null +++ b/dev/tasks/DEV-000172-fleet-failover-priority.json @@ -0,0 +1,23 @@ +{ + "task_id": "DEV-000172", + "task_revision": 1, + "architecture_revision": "ADR-0018", + "base_repo_revision": "2c7a85779b530e3147fbf47b71742759cc010d58", + "risk_tier": "E", + "intent": "Make fleet failover choose the best available non-Lifeboat tier below the preferred tier before using Lifeboat.", + "requirement_ids": ["WP-301", "Model fleet/failover"], + "allowed_change_scope": {"modules": ["capability", "cognitive-worker"], "paths": ["modules/capability/domain/fleet.go", "modules/capability/domain/fleet_test.go", "cmd/cognitive-worker/fleet_status_test.go", "dev/ACTIVE_TASK.json", "dev/tasks/DEV-000172-fleet-failover-priority.json"]}, + "forbidden_change_scope": {"paths": ["architecture/**", "security/**", "database/**", "deploy/**", "contracts/**", "adapters/**", ".gitea/**"]}, + "allowed_architecture_delta": {"new_cross_module_dependencies": 0, "new_canonical_owners": 0, "new_privileges": 0, "new_third_party_dependencies": 0, "new_database_owner_roles": 0}, + "contracts": {"consume": ["existing FleetReadiness contract", "existing FleetTier selection contract"], "may_change": []}, + "semantic_owner": "capability", + "canonical_state": [], + "state_transitions": ["preferred unavailable -> next lower ready tier -> Lifeboat fallback"], + "side_effects_and_privileges": ["none"], + "persistence_and_migrations": ["none"], + "failure_retry_restart": ["selection remains deterministic from observed readiness; invalid preferred tier fails closed"], + "assumptions": ["HEAVY > MEDIUM > FAST is the quality/capability fallback order; LIFEBOAT is last-resort recovery"], + "acceptance_scenarios": ["HEAVY unavailable and MEDIUM ready selects MEDIUM", "HEAVY and MEDIUM unavailable and FAST ready selects FAST", "all primary tiers unavailable selects Lifeboat", "invalid preferred tier is rejected"], + "required_tests": ["go test ./...", "go vet ./...", "preflight", "check-diff", "check-architecture", "independent", "git diff --check", "CI", "production fleet readiness"], + "terminal_gap_states": ["BLOCKED_BY_MISSING_CAPABILITY", "BLOCKED_BY_CONFLICT"] +} diff --git a/modules/capability/domain/fleet.go b/modules/capability/domain/fleet.go index 1baee78..cadcf42 100644 --- a/modules/capability/domain/fleet.go +++ b/modules/capability/domain/fleet.go @@ -37,8 +37,13 @@ func SelectFleetTier(readiness capability.FleetReadiness, preferred capability.F if readiness.ObservedAtUnixMillis < 1 || len(readiness.Members) == 0 || !validFleetTier(readiness.FallbackTier) { return "", ErrInvalidFleetReadiness } - if slices.Contains(readiness.ReadyTiers, preferred) { - return preferred, nil + if !validFleetTier(preferred) { + return "", ErrInvalidFleetReadiness + } + for _, tier := range fleetFailoverOrder(preferred) { + if slices.Contains(readiness.ReadyTiers, tier) { + return tier, nil + } } if slices.Contains(readiness.ReadyTiers, readiness.FallbackTier) { return readiness.FallbackTier, nil @@ -46,6 +51,21 @@ func SelectFleetTier(readiness capability.FleetReadiness, preferred capability.F return "", ErrInvalidFleetReadiness } +func fleetFailoverOrder(preferred capability.FleetTier) []capability.FleetTier { + switch preferred { + case capability.FleetTierHeavy: + return []capability.FleetTier{capability.FleetTierHeavy, capability.FleetTierMedium, capability.FleetTierFast} + case capability.FleetTierMedium: + return []capability.FleetTier{capability.FleetTierMedium, capability.FleetTierFast} + case capability.FleetTierFast: + return []capability.FleetTier{capability.FleetTierFast} + case capability.FleetTierLifeboat: + return []capability.FleetTier{capability.FleetTierLifeboat} + default: + return nil + } +} + func validFleetTier(value capability.FleetTier) bool { return value == capability.FleetTierFast || value == capability.FleetTierMedium || value == capability.FleetTierHeavy || value == capability.FleetTierLifeboat } diff --git a/modules/capability/domain/fleet_test.go b/modules/capability/domain/fleet_test.go index a8b5957..fa575e1 100644 --- a/modules/capability/domain/fleet_test.go +++ b/modules/capability/domain/fleet_test.go @@ -16,3 +16,33 @@ func TestFleetReadinessFallsBackToLifeboatWhenPrimaryTiersUnavailable(t *testing t.Fatalf("selected=%q err=%v", selected, err) } } + +func TestFleetReadinessFallsBackFromHeavyToMediumBeforeLifeboat(t *testing.T) { + readiness, err := BuildFleetReadiness(100, []capability.FleetMember{ + {ProviderInstanceID: "medium", ModelDescriptorID: "qwen-medium", Tier: capability.FleetTierMedium, Ready: true, Reason: "available"}, + {ProviderInstanceID: "heavy", ModelDescriptorID: "gpu", Tier: capability.FleetTierHeavy, Ready: false, Reason: "timeout"}, + {ProviderInstanceID: "lifeboat", ModelDescriptorID: "qwen-lifeboat", Tier: capability.FleetTierLifeboat, Ready: true, Reason: "available"}, + }) + if err != nil { + t.Fatal(err) + } + selected, err := SelectFleetTier(readiness, capability.FleetTierHeavy) + if err != nil || selected != capability.FleetTierMedium { + t.Fatalf("selected=%q err=%v", selected, err) + } +} + +func TestFleetReadinessFallsBackFromHeavyToFastBeforeLifeboat(t *testing.T) { + readiness, err := BuildFleetReadiness(100, []capability.FleetMember{ + {ProviderInstanceID: "fast", ModelDescriptorID: "qwen-fast", Tier: capability.FleetTierFast, Ready: true, Reason: "available"}, + {ProviderInstanceID: "heavy", ModelDescriptorID: "gpu", Tier: capability.FleetTierHeavy, Ready: false, Reason: "timeout"}, + {ProviderInstanceID: "lifeboat", ModelDescriptorID: "qwen-lifeboat", Tier: capability.FleetTierLifeboat, Ready: true, Reason: "available"}, + }) + if err != nil { + t.Fatal(err) + } + selected, err := SelectFleetTier(readiness, capability.FleetTierHeavy) + if err != nil || selected != capability.FleetTierFast { + t.Fatalf("selected=%q err=%v", selected, err) + } +}