From 30443027a65c9617137c0d47a128b4022cfafd8f Mon Sep 17 00:00:00 2001 From: Mikhail Date: Sat, 22 Aug 2026 23:42:10 +0300 Subject: [PATCH 1/2] feat: add evidence-based adaptive model rating --- cmd/cognitive-worker/main.go | 2 +- contracts/capability/cognitive_registry.go | 18 ++++++ ...182-model-rating-and-adaptive-routing.json | 23 ++++++++ docs/MODEL_FLEET_POLICY.md | 24 ++++++++ .../capability/domain/cognitive_registry.go | 55 +++++++++++++++++++ modules/capability/domain/fleet.go | 2 +- modules/capability/domain/fleet_test.go | 14 +++++ .../capability/domain/model_rating_test.go | 40 ++++++++++++++ 8 files changed, 176 insertions(+), 2 deletions(-) create mode 100644 dev/tasks/DEV-000182-model-rating-and-adaptive-routing.json create mode 100644 docs/MODEL_FLEET_POLICY.md create mode 100644 modules/capability/domain/model_rating_test.go diff --git a/cmd/cognitive-worker/main.go b/cmd/cognitive-worker/main.go index 210a338..dd1d4e7 100644 --- a/cmd/cognitive-worker/main.go +++ b/cmd/cognitive-worker/main.go @@ -111,7 +111,7 @@ func main() { mediumReadinessClient := &http.Client{Timeout: 10 * time.Second} lifeboatURL := strings.TrimSuffix(strings.TrimRight(envOr("LOCAL_LIFEBOAT_PROVIDER_URL", "http://cognitive-lifeboat:8086/v1"), "/"), "/v1") + "/health" fleetClient := &http.Client{Timeout: 2 * time.Second} - preferredFleet := capability.FleetTier(envOr("COGNITIVE_PREFERRED_FLEET_TIER", string(capability.FleetTierHeavy))) + preferredFleet := capability.FleetTier(envOr("COGNITIVE_PREFERRED_FLEET_TIER", string(capability.FleetTierFast))) cpuFleetHeaders := map[string]string{} if cpuAPIKey != "" { cpuFleetHeaders["Authorization"] = "Bearer " + cpuAPIKey diff --git a/contracts/capability/cognitive_registry.go b/contracts/capability/cognitive_registry.go index 32b1314..9097abf 100644 --- a/contracts/capability/cognitive_registry.go +++ b/contracts/capability/cognitive_registry.go @@ -26,6 +26,7 @@ type PerformanceEvidenceID ids.ID type BillingMode string type AuthMode string type SelectionObjective string +type ModelRatingCategory string const ( QualificationDiscovered QualificationState = "DISCOVERED" @@ -73,6 +74,11 @@ const ( SelectionQuality SelectionObjective = "QUALITY" SelectionSpeed SelectionObjective = "SPEED" SelectionEconomy SelectionObjective = "ECONOMY" + + ModelRatingUnrated ModelRatingCategory = "UNRATED" + ModelRatingFast ModelRatingCategory = "FAST" + ModelRatingBalanced ModelRatingCategory = "BALANCED" + ModelRatingQuality ModelRatingCategory = "QUALITY" ) // ProviderInstance contains only opaque references to endpoint and credentials. @@ -164,6 +170,18 @@ type ResolvedCognitiveBinding struct { Model ModelDescriptor Evaluation *EvaluatedCapability Performance *PerformanceEvidence + Rating *ModelRating +} + +// ModelRating is a derived, provider-neutral explanation of why an eligible +// model is routed. It never grants qualification or activation authority. +type ModelRating struct { + Category ModelRatingCategory `json:"category"` + OverallScoreMillis int64 `json:"overall_score_millis"` + QualityScoreMillis int64 `json:"quality_score_millis"` + ReliabilityScoreMillis int64 `json:"reliability_score_millis"` + LatencyScoreMillis int64 `json:"latency_score_millis"` + EvidenceScoreMillis int64 `json:"evidence_score_millis"` } type CognitiveResultClass string diff --git a/dev/tasks/DEV-000182-model-rating-and-adaptive-routing.json b/dev/tasks/DEV-000182-model-rating-and-adaptive-routing.json new file mode 100644 index 0000000..2da6348 --- /dev/null +++ b/dev/tasks/DEV-000182-model-rating-and-adaptive-routing.json @@ -0,0 +1,23 @@ +{ + "task_id": "DEV-000182", + "task_revision": 1, + "architecture_revision": "ADR-0018", + "base_repo_revision": "4709ef46076cf5c22522bbfea785fdb3c6220c14", + "risk_tier": "E", + "intent": "Expose an evidence-based model rating and adaptive routing policy so model category follows qualification and observed performance rather than provider or hardware names.", + "requirement_ids": ["WP-200", "WP-601", "adaptive model routing", "CPU-first current operation"], + "allowed_change_scope": {"modules": ["capability", "cognitive-worker"], "paths": ["contracts/capability/cognitive_registry.go", "modules/capability/domain/cognitive_registry.go", "modules/capability/domain/fleet.go", "modules/capability/domain/fleet_test.go", "cmd/cognitive-worker/main.go", "cmd/cognitive-worker/fleet_status_test.go", "modules/capability/domain/model_rating_test.go", "docs/MODEL_FLEET_POLICY.md", "dev/tasks/DEV-000182-model-rating-and-adaptive-routing.json"]}, + "forbidden_change_scope": {"paths": ["architecture/**", "security/**", "database/**", "deploy/**", ".gitea/**"]}, + "allowed_architecture_delta": {"new_cross_module_dependencies": 0, "new_canonical_owners": 0, "new_privileges": 0, "new_third_party_dependencies": 0, "new_database_owner_roles": 0}, + "contracts": {"consume": ["EvaluatedCapability", "PerformanceEvidence", "CognitiveRequirement"], "may_change": ["provider-neutral model rating projection"]}, + "semantic_owner": "capability", + "canonical_state": ["existing owner-local qualification and performance evidence; rating is a derived projection"], + "state_transitions": ["unrated -> rated only after supported qualification evidence; category is recomputed from current evidence; routing chooses an eligible rated candidate and falls back on invocation failure"], + "side_effects_and_privileges": ["none; no provider invocation or external mutation"], + "persistence_and_migrations": ["no migration; rating is derived and serializable"], + "failure_retry_restart": ["missing or stale evidence yields UNRATED and cannot silently claim quality; existing gateway fallback remains authoritative"], + "assumptions": ["CPU is the current operational choice, but routing remains provider- and hardware-neutral", "discovery never activates a binding"], + "acceptance_scenarios": ["supported evidence produces a visible rating and category", "missing evidence remains UNRATED", "quality objective prefers higher evidence rating", "speed objective prefers lower observed latency", "provider identity alone cannot raise a rating", "CPU is preferred now but all ready capability classes remain valid failover candidates"], + "required_tests": ["go test ./...", "go vet ./...", "preflight", "check-diff", "check-architecture", "independent", "git diff --check", "CI"], + "terminal_gap_states": ["BLOCKED_BY_SPEC_GAP", "BLOCKED_BY_CONFLICT"] +} diff --git a/docs/MODEL_FLEET_POLICY.md b/docs/MODEL_FLEET_POLICY.md new file mode 100644 index 0000000..6fa5151 --- /dev/null +++ b/docs/MODEL_FLEET_POLICY.md @@ -0,0 +1,24 @@ +# Model fleet policy + +Domovoy does not equate intelligence with GPU, model name, or provider kind. +Hardware and endpoint identity are operational observations only. A model is +eligible for routing only after the existing qualification and activation +gates pass. + +`FAST`, `MEDIUM`, and `HEAVY` are capability service classes, not hardware +claims. A CPU model may be the best available quality model, and a GPU model +may be unavailable, degraded, or unrated. + +The Capability owner derives a provider-neutral `ModelRating` from evidence: + +- `QUALITY`: high measured quality and zero critical failures; +- `BALANCED`: supported model with usable quality/reliability evidence; +- `FAST`: supported model whose observed latency is strong enough for quick work; +- `UNRATED`: discovery or incomplete evidence; it cannot be promoted by name. + +The score combines quality, reliability, latency and evidence coverage. Missing +measurements remain explicit. A model can change category after a new immutable +qualification/performance revision. The current CPU route is an operational +choice, not a permanent hardware assumption. CPU is preferred in the current +deployment; if it is unavailable, the policy tries another ready class before +using the Lifeboat recovery path. diff --git a/modules/capability/domain/cognitive_registry.go b/modules/capability/domain/cognitive_registry.go index a7f1653..bb36138 100644 --- a/modules/capability/domain/cognitive_registry.go +++ b/modules/capability/domain/cognitive_registry.go @@ -148,6 +148,8 @@ func SelectCognitiveBinding(requirement capability.CognitiveRequirement, candida if candidate.Evaluation != nil && candidate.Evaluation.Result == capability.EvaluationUnsupported { continue } + rating := RateCognitiveBinding(candidate) + candidate.Rating = &rating if requirement.MaxCostMicros != nil { cost, known := estimatedCost(candidate.Provider, requirement.InputBytes, requirement.MaxOutputBytes) if known && cost > *requirement.MaxCostMicros { @@ -165,6 +167,56 @@ func SelectCognitiveBinding(requirement capability.CognitiveRequirement, candida return capability.ResolvedCognitiveBinding{}, ErrNoInvocableCognitiveBinding } +// RateCognitiveBinding derives a transparent routing category from measured +// evidence. Provider kind, endpoint name and hardware are intentionally not +// inputs to the score. +func RateCognitiveBinding(candidate capability.ResolvedCognitiveBinding) capability.ModelRating { + if candidate.Evaluation == nil || candidate.Evaluation.Result != capability.EvaluationSupported || candidate.Evaluation.QualityScoreMillis == nil || candidate.Evaluation.CriticalFailureMillis == nil { + return capability.ModelRating{Category: capability.ModelRatingUnrated} + } + quality := clampMillis(*candidate.Evaluation.QualityScoreMillis) + reliability := 1000 - clampMillis(*candidate.Evaluation.CriticalFailureMillis) + latency := int64(500) + if candidate.Performance != nil && candidate.Performance.LatencyMicros != nil { + latency = latencyScore(*candidate.Performance.LatencyMicros) + } + evidence := int64(700) + if candidate.Performance != nil { + evidence = 1000 + } + overall := (quality*55 + reliability*25 + latency*15 + evidence*5) / 100 + category := capability.ModelRatingBalanced + if overall >= 850 && quality >= 800 { + category = capability.ModelRatingQuality + } else if latency >= 800 && overall >= 700 { + category = capability.ModelRatingFast + } + return capability.ModelRating{Category: category, OverallScoreMillis: overall, QualityScoreMillis: quality, ReliabilityScoreMillis: reliability, LatencyScoreMillis: latency, EvidenceScoreMillis: evidence} +} + +func clampMillis(value int64) int64 { + if value < 0 { + return 0 + } + if value > 1000 { + return 1000 + } + return value +} + +func latencyScore(latencyMicros int64) int64 { + if latencyMicros <= 500_000 { + return 1000 + } + if latencyMicros <= 2_000_000 { + return 800 + } + if latencyMicros <= 10_000_000 { + return 500 + } + return 200 +} + func dataPolicySatisfies(actual, allowed capability.DataPolicy) bool { return actual == capability.DataPolicyLocalOnly || actual == allowed } @@ -198,6 +250,9 @@ func betterCandidate(left, right capability.ResolvedCognitiveBinding, objective } } if objective == capability.SelectionQuality || objective == capability.SelectionBalanced || objective == "" { + if left.Rating != nil && right.Rating != nil && left.Rating.OverallScoreMillis != right.Rating.OverallScoreMillis { + return left.Rating.OverallScoreMillis > right.Rating.OverallScoreMillis + } li, ri := left.Model.IntelligenceScoreMillis, right.Model.IntelligenceScoreMillis if li != nil && ri != nil && *li != *ri { return *li > *ri diff --git a/modules/capability/domain/fleet.go b/modules/capability/domain/fleet.go index cadcf42..8fc4541 100644 --- a/modules/capability/domain/fleet.go +++ b/modules/capability/domain/fleet.go @@ -58,7 +58,7 @@ func fleetFailoverOrder(preferred capability.FleetTier) []capability.FleetTier { case capability.FleetTierMedium: return []capability.FleetTier{capability.FleetTierMedium, capability.FleetTierFast} case capability.FleetTierFast: - return []capability.FleetTier{capability.FleetTierFast} + return []capability.FleetTier{capability.FleetTierFast, capability.FleetTierMedium, capability.FleetTierHeavy} case capability.FleetTierLifeboat: return []capability.FleetTier{capability.FleetTierLifeboat} default: diff --git a/modules/capability/domain/fleet_test.go b/modules/capability/domain/fleet_test.go index fa575e1..d95f07d 100644 --- a/modules/capability/domain/fleet_test.go +++ b/modules/capability/domain/fleet_test.go @@ -46,3 +46,17 @@ func TestFleetReadinessFallsBackFromHeavyToFastBeforeLifeboat(t *testing.T) { t.Fatalf("selected=%q err=%v", selected, err) } } + +func TestFleetReadinessCpuPreferenceCanUseOtherReadyClass(t *testing.T) { + readiness, err := BuildFleetReadiness(100, []capability.FleetMember{ + {ProviderInstanceID: "medium", ModelDescriptorID: "medium-model", Tier: capability.FleetTierMedium, Ready: true, Reason: "available"}, + {ProviderInstanceID: "lifeboat", ModelDescriptorID: "lifeboat-model", Tier: capability.FleetTierLifeboat, Ready: true, Reason: "available"}, + }) + if err != nil { + t.Fatal(err) + } + selected, err := SelectFleetTier(readiness, capability.FleetTierFast) + if err != nil || selected != capability.FleetTierMedium { + t.Fatalf("selected=%q err=%v", selected, err) + } +} diff --git a/modules/capability/domain/model_rating_test.go b/modules/capability/domain/model_rating_test.go new file mode 100644 index 0000000..820c72b --- /dev/null +++ b/modules/capability/domain/model_rating_test.go @@ -0,0 +1,40 @@ +package domain + +import ( + "testing" + + "git.cin.su/m/Assistant/contracts/capability" +) + +func TestRateCognitiveBindingRequiresSupportedEvidence(t *testing.T) { + rating := RateCognitiveBinding(capability.ResolvedCognitiveBinding{}) + if rating.Category != capability.ModelRatingUnrated || rating.OverallScoreMillis != 0 { + t.Fatalf("rating=%+v", rating) + } +} + +func TestRateCognitiveBindingUsesEvidenceNotProviderName(t *testing.T) { + quality, critical, latency := int64(900), int64(0), int64(400_000) + base := capability.ResolvedCognitiveBinding{ + Provider: capability.ProviderInstance{Kind: "openai-compatible-chat-v1"}, + Evaluation: &capability.EvaluatedCapability{Result: capability.EvaluationSupported, QualityScoreMillis: &quality, CriticalFailureMillis: &critical}, + Performance: &capability.PerformanceEvidence{LatencyMicros: &latency}, + } + other := base + other.Provider.Kind = "ollama-v1" + left, right := RateCognitiveBinding(base), RateCognitiveBinding(other) + if left != right || left.Category != capability.ModelRatingQuality { + t.Fatalf("provider changed rating: left=%+v right=%+v", left, right) + } +} + +func TestRateCognitiveBindingClassifiesSlowEvidenceAsBalanced(t *testing.T) { + quality, critical, latency := int64(800), int64(0), int64(20_000_000) + rating := RateCognitiveBinding(capability.ResolvedCognitiveBinding{ + Evaluation: &capability.EvaluatedCapability{Result: capability.EvaluationSupported, QualityScoreMillis: &quality, CriticalFailureMillis: &critical}, + Performance: &capability.PerformanceEvidence{LatencyMicros: &latency}, + }) + if rating.Category != capability.ModelRatingBalanced || rating.LatencyScoreMillis != 200 { + t.Fatalf("rating=%+v", rating) + } +} -- 2.54.0 From 6b3a6b2dfb320413fc671f7316cd4f713d033954 Mon Sep 17 00:00:00 2001 From: Mikhail Date: Sat, 22 Aug 2026 23:43:54 +0300 Subject: [PATCH 2/2] chore: activate adaptive fleet task --- dev/ACTIVE_TASK.json | 4 ++-- dev/tasks/DEV-000182-model-rating-and-adaptive-routing.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/dev/ACTIVE_TASK.json b/dev/ACTIVE_TASK.json index 7169f97..df51977 100644 --- a/dev/ACTIVE_TASK.json +++ b/dev/ACTIVE_TASK.json @@ -1,5 +1,5 @@ { - "task_file": "dev/tasks/DEV-000181-medium-bounded-retest-evidence.json", - "task_id": "DEV-000181", + "task_file": "dev/tasks/DEV-000182-model-rating-and-adaptive-routing.json", + "task_id": "DEV-000182", "task_revision": 1 } diff --git a/dev/tasks/DEV-000182-model-rating-and-adaptive-routing.json b/dev/tasks/DEV-000182-model-rating-and-adaptive-routing.json index 2da6348..be51711 100644 --- a/dev/tasks/DEV-000182-model-rating-and-adaptive-routing.json +++ b/dev/tasks/DEV-000182-model-rating-and-adaptive-routing.json @@ -6,7 +6,7 @@ "risk_tier": "E", "intent": "Expose an evidence-based model rating and adaptive routing policy so model category follows qualification and observed performance rather than provider or hardware names.", "requirement_ids": ["WP-200", "WP-601", "adaptive model routing", "CPU-first current operation"], - "allowed_change_scope": {"modules": ["capability", "cognitive-worker"], "paths": ["contracts/capability/cognitive_registry.go", "modules/capability/domain/cognitive_registry.go", "modules/capability/domain/fleet.go", "modules/capability/domain/fleet_test.go", "cmd/cognitive-worker/main.go", "cmd/cognitive-worker/fleet_status_test.go", "modules/capability/domain/model_rating_test.go", "docs/MODEL_FLEET_POLICY.md", "dev/tasks/DEV-000182-model-rating-and-adaptive-routing.json"]}, + "allowed_change_scope": {"modules": ["capability", "cognitive-worker"], "paths": ["contracts/capability/cognitive_registry.go", "modules/capability/domain/cognitive_registry.go", "modules/capability/domain/fleet.go", "modules/capability/domain/fleet_test.go", "cmd/cognitive-worker/main.go", "cmd/cognitive-worker/fleet_status_test.go", "modules/capability/domain/model_rating_test.go", "docs/MODEL_FLEET_POLICY.md", "dev/ACTIVE_TASK.json", "dev/tasks/DEV-000182-model-rating-and-adaptive-routing.json"]}, "forbidden_change_scope": {"paths": ["architecture/**", "security/**", "database/**", "deploy/**", ".gitea/**"]}, "allowed_architecture_delta": {"new_cross_module_dependencies": 0, "new_canonical_owners": 0, "new_privileges": 0, "new_third_party_dependencies": 0, "new_database_owner_roles": 0}, "contracts": {"consume": ["EvaluatedCapability", "PerformanceEvidence", "CognitiveRequirement"], "may_change": ["provider-neutral model rating projection"]}, -- 2.54.0