Compare commits
4
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
2f9e9bdc9f | ||
|
|
2c7a85779b | ||
|
|
b4c3027e6d | ||
|
|
503fe84074 |
@@ -9,7 +9,7 @@ import (
|
||||
"git.cin.su/m/Assistant/contracts/capability"
|
||||
)
|
||||
|
||||
func TestFleetReadinessSelectsLifeboatWhenPrimaryTiersUnavailable(t *testing.T) {
|
||||
func TestFleetReadinessSelectsBestAvailablePrimaryTierBeforeLifeboat(t *testing.T) {
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusOK) }))
|
||||
defer server.Close()
|
||||
handler := fleetReadinessHandler(server.Client(), capability.FleetTierHeavy, []fleetEndpoint{
|
||||
@@ -21,7 +21,7 @@ func TestFleetReadinessSelectsLifeboatWhenPrimaryTiersUnavailable(t *testing.T)
|
||||
recorder := httptest.NewRecorder()
|
||||
request := httptest.NewRequest(http.MethodGet, "/v1/fleet/readiness", nil)
|
||||
handler.ServeHTTP(recorder, request)
|
||||
if recorder.Code != http.StatusOK || !strings.Contains(recorder.Body.String(), `"selected_tier":"LIFEBOAT"`) || !strings.Contains(recorder.Body.String(), `"tier":"MEDIUM"`) {
|
||||
if recorder.Code != http.StatusOK || !strings.Contains(recorder.Body.String(), `"selected_tier":"FAST"`) || !strings.Contains(recorder.Body.String(), `"tier":"MEDIUM"`) {
|
||||
t.Fatalf("status=%d body=%s", recorder.Code, recorder.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -214,7 +214,7 @@ services:
|
||||
LOCAL_CPU_PROVIDER_URL: http://docker.cin.su:18082/v1
|
||||
LOCAL_CPU_PROVIDER_API_KEY_FILE: ${LOCAL_CPU_PROVIDER_API_KEY_FILE:-}
|
||||
LOCAL_CPU_PROVIDER_CREDENTIAL_REF: ${LOCAL_CPU_PROVIDER_CREDENTIAL_REF:-credential://domovoy/local-cpu/provider-api-key/v1}
|
||||
LOCAL_MEDIUM_PROVIDER_URL: ${LOCAL_MEDIUM_PROVIDER_URL:-}
|
||||
LOCAL_MEDIUM_PROVIDER_URL: ${LOCAL_MEDIUM_PROVIDER_URL:-http://llm-local-api:8080/v1}
|
||||
LOCAL_MEDIUM_PROVIDER_API_KEY_FILE: ${LOCAL_MEDIUM_PROVIDER_API_KEY_FILE:-}
|
||||
LOCAL_MEDIUM_PROVIDER_CREDENTIAL_REF: ${LOCAL_MEDIUM_PROVIDER_CREDENTIAL_REF:-credential://domovoy/local-medium/provider-api-key/v1}
|
||||
LOCAL_LIFEBOAT_PROVIDER_URL: http://cognitive-lifeboat:8086/v1
|
||||
@@ -224,6 +224,7 @@ services:
|
||||
depends_on: [core-runtime, cognitive-lifeboat]
|
||||
volumes:
|
||||
- assistant_core_secrets:/run/secrets:ro
|
||||
networks: [default, llm-local]
|
||||
|
||||
cognitive-lifeboat:
|
||||
image: ghcr.io/ggml-org/llama.cpp:server
|
||||
@@ -290,3 +291,8 @@ volumes:
|
||||
external: true
|
||||
assistant_research_sources:
|
||||
name: assistant_research_sources
|
||||
|
||||
networks:
|
||||
llm-local:
|
||||
external: true
|
||||
name: llm-local_default
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
{
|
||||
"task_file": "dev/tasks/DEV-000170-refresh-fleet-audit-evidence.json",
|
||||
"task_id": "DEV-000170",
|
||||
"task_file": "dev/tasks/DEV-000172-fleet-failover-priority.json",
|
||||
"task_id": "DEV-000172",
|
||||
"task_revision": 1
|
||||
}
|
||||
|
||||
@@ -0,0 +1,23 @@
|
||||
{
|
||||
"task_id": "DEV-000171",
|
||||
"task_revision": 1,
|
||||
"architecture_revision": "ADR-0018",
|
||||
"base_repo_revision": "503fe840741ac99499e06bd93df977acf1baad41",
|
||||
"risk_tier": "E",
|
||||
"intent": "Connect cognitive-worker to the already-operated llm-local CPU model runtime as the configured MEDIUM fleet member through an explicit external Docker network.",
|
||||
"requirement_ids": ["WP-200", "WP-301", "Model fleet/failover"],
|
||||
"allowed_change_scope": {"modules": ["deployment-topology", "cognitive-worker"], "paths": ["deploy/docker-compose.prod.yml", "dev/ACTIVE_TASK.json", "dev/tasks/DEV-000171-connect-existing-medium-runtime.json"]},
|
||||
"forbidden_change_scope": {"paths": ["architecture/**", "security/**", "database/**", "contracts/**", "adapters/**", ".gitea/**"]},
|
||||
"allowed_architecture_delta": {"new_cross_module_dependencies": 0, "new_canonical_owners": 0, "new_privileges": 1, "new_third_party_dependencies": 0, "new_database_owner_roles": 0},
|
||||
"contracts": {"consume": ["existing OpenAI-compatible local provider adapter", "existing MEDIUM fleet readiness contract", "existing llm-local Docker network"], "may_change": []},
|
||||
"semantic_owner": "deployment topology/cognitive-worker registration",
|
||||
"canonical_state": [],
|
||||
"state_transitions": ["MEDIUM unconfigured -> live provider discovery/readiness", "preferred HEAVY unavailable -> highest available fallback selection"],
|
||||
"side_effects_and_privileges": ["one explicit read-only network attachment from cognitive-worker to existing llm-local runtime; no secret or database privilege added"],
|
||||
"persistence_and_migrations": ["none"],
|
||||
"failure_retry_restart": ["if external llm-local network is absent deployment fails closed; provider timeout remains bounded; Lifeboat remains fallback"],
|
||||
"assumptions": ["production llm-local_default network and llm-local-api are independently operated and present", "llm-local-api exposes OpenAI-compatible /v1/models and chat routes on port 8080"],
|
||||
"acceptance_scenarios": ["compose config validates with external network declaration", "cognitive-worker resolves llm-local-api", "MEDIUM fleet member is AVAILABLE live", "runtime /readyz remains 200", "Lifeboat fallback remains available", "no secret value is added"],
|
||||
"required_tests": ["docker compose config", "go test ./...", "go vet ./...", "preflight", "check-diff", "check-architecture", "independent", "git diff --check", "CI", "production /readyz and fleet readiness"],
|
||||
"terminal_gap_states": ["BLOCKED_BY_MISSING_CAPABILITY", "BLOCKED_BY_CONFLICT"]
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
{
|
||||
"task_id": "DEV-000172",
|
||||
"task_revision": 1,
|
||||
"architecture_revision": "ADR-0018",
|
||||
"base_repo_revision": "2c7a85779b530e3147fbf47b71742759cc010d58",
|
||||
"risk_tier": "E",
|
||||
"intent": "Make fleet failover choose the best available non-Lifeboat tier below the preferred tier before using Lifeboat.",
|
||||
"requirement_ids": ["WP-301", "Model fleet/failover"],
|
||||
"allowed_change_scope": {"modules": ["capability", "cognitive-worker"], "paths": ["modules/capability/domain/fleet.go", "modules/capability/domain/fleet_test.go", "cmd/cognitive-worker/fleet_status_test.go", "dev/ACTIVE_TASK.json", "dev/tasks/DEV-000172-fleet-failover-priority.json"]},
|
||||
"forbidden_change_scope": {"paths": ["architecture/**", "security/**", "database/**", "deploy/**", "contracts/**", "adapters/**", ".gitea/**"]},
|
||||
"allowed_architecture_delta": {"new_cross_module_dependencies": 0, "new_canonical_owners": 0, "new_privileges": 0, "new_third_party_dependencies": 0, "new_database_owner_roles": 0},
|
||||
"contracts": {"consume": ["existing FleetReadiness contract", "existing FleetTier selection contract"], "may_change": []},
|
||||
"semantic_owner": "capability",
|
||||
"canonical_state": [],
|
||||
"state_transitions": ["preferred unavailable -> next lower ready tier -> Lifeboat fallback"],
|
||||
"side_effects_and_privileges": ["none"],
|
||||
"persistence_and_migrations": ["none"],
|
||||
"failure_retry_restart": ["selection remains deterministic from observed readiness; invalid preferred tier fails closed"],
|
||||
"assumptions": ["HEAVY > MEDIUM > FAST is the quality/capability fallback order; LIFEBOAT is last-resort recovery"],
|
||||
"acceptance_scenarios": ["HEAVY unavailable and MEDIUM ready selects MEDIUM", "HEAVY and MEDIUM unavailable and FAST ready selects FAST", "all primary tiers unavailable selects Lifeboat", "invalid preferred tier is rejected"],
|
||||
"required_tests": ["go test ./...", "go vet ./...", "preflight", "check-diff", "check-architecture", "independent", "git diff --check", "CI", "production fleet readiness"],
|
||||
"terminal_gap_states": ["BLOCKED_BY_MISSING_CAPABILITY", "BLOCKED_BY_CONFLICT"]
|
||||
}
|
||||
@@ -37,8 +37,13 @@ func SelectFleetTier(readiness capability.FleetReadiness, preferred capability.F
|
||||
if readiness.ObservedAtUnixMillis < 1 || len(readiness.Members) == 0 || !validFleetTier(readiness.FallbackTier) {
|
||||
return "", ErrInvalidFleetReadiness
|
||||
}
|
||||
if slices.Contains(readiness.ReadyTiers, preferred) {
|
||||
return preferred, nil
|
||||
if !validFleetTier(preferred) {
|
||||
return "", ErrInvalidFleetReadiness
|
||||
}
|
||||
for _, tier := range fleetFailoverOrder(preferred) {
|
||||
if slices.Contains(readiness.ReadyTiers, tier) {
|
||||
return tier, nil
|
||||
}
|
||||
}
|
||||
if slices.Contains(readiness.ReadyTiers, readiness.FallbackTier) {
|
||||
return readiness.FallbackTier, nil
|
||||
@@ -46,6 +51,21 @@ func SelectFleetTier(readiness capability.FleetReadiness, preferred capability.F
|
||||
return "", ErrInvalidFleetReadiness
|
||||
}
|
||||
|
||||
func fleetFailoverOrder(preferred capability.FleetTier) []capability.FleetTier {
|
||||
switch preferred {
|
||||
case capability.FleetTierHeavy:
|
||||
return []capability.FleetTier{capability.FleetTierHeavy, capability.FleetTierMedium, capability.FleetTierFast}
|
||||
case capability.FleetTierMedium:
|
||||
return []capability.FleetTier{capability.FleetTierMedium, capability.FleetTierFast}
|
||||
case capability.FleetTierFast:
|
||||
return []capability.FleetTier{capability.FleetTierFast}
|
||||
case capability.FleetTierLifeboat:
|
||||
return []capability.FleetTier{capability.FleetTierLifeboat}
|
||||
default:
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
func validFleetTier(value capability.FleetTier) bool {
|
||||
return value == capability.FleetTierFast || value == capability.FleetTierMedium || value == capability.FleetTierHeavy || value == capability.FleetTierLifeboat
|
||||
}
|
||||
|
||||
@@ -16,3 +16,33 @@ func TestFleetReadinessFallsBackToLifeboatWhenPrimaryTiersUnavailable(t *testing
|
||||
t.Fatalf("selected=%q err=%v", selected, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFleetReadinessFallsBackFromHeavyToMediumBeforeLifeboat(t *testing.T) {
|
||||
readiness, err := BuildFleetReadiness(100, []capability.FleetMember{
|
||||
{ProviderInstanceID: "medium", ModelDescriptorID: "qwen-medium", Tier: capability.FleetTierMedium, Ready: true, Reason: "available"},
|
||||
{ProviderInstanceID: "heavy", ModelDescriptorID: "gpu", Tier: capability.FleetTierHeavy, Ready: false, Reason: "timeout"},
|
||||
{ProviderInstanceID: "lifeboat", ModelDescriptorID: "qwen-lifeboat", Tier: capability.FleetTierLifeboat, Ready: true, Reason: "available"},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
selected, err := SelectFleetTier(readiness, capability.FleetTierHeavy)
|
||||
if err != nil || selected != capability.FleetTierMedium {
|
||||
t.Fatalf("selected=%q err=%v", selected, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFleetReadinessFallsBackFromHeavyToFastBeforeLifeboat(t *testing.T) {
|
||||
readiness, err := BuildFleetReadiness(100, []capability.FleetMember{
|
||||
{ProviderInstanceID: "fast", ModelDescriptorID: "qwen-fast", Tier: capability.FleetTierFast, Ready: true, Reason: "available"},
|
||||
{ProviderInstanceID: "heavy", ModelDescriptorID: "gpu", Tier: capability.FleetTierHeavy, Ready: false, Reason: "timeout"},
|
||||
{ProviderInstanceID: "lifeboat", ModelDescriptorID: "qwen-lifeboat", Tier: capability.FleetTierLifeboat, Ready: true, Reason: "available"},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
selected, err := SelectFleetTier(readiness, capability.FleetTierHeavy)
|
||||
if err != nil || selected != capability.FleetTierFast {
|
||||
t.Fatalf("selected=%q err=%v", selected, err)
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user