feat: add evidence-based adaptive model rating #155

Merged
m merged 3 commits from codex/model-rating-adaptive-routing into main 2026-08-22 20:47:02 +00:00
9 changed files with 178 additions and 4 deletions
+1 -1
View File
@@ -111,7 +111,7 @@ func main() {
mediumReadinessClient := &http.Client{Timeout: 10 * time.Second}
lifeboatURL := strings.TrimSuffix(strings.TrimRight(envOr("LOCAL_LIFEBOAT_PROVIDER_URL", "http://cognitive-lifeboat:8086/v1"), "/"), "/v1") + "/health"
fleetClient := &http.Client{Timeout: 2 * time.Second}
preferredFleet := capability.FleetTier(envOr("COGNITIVE_PREFERRED_FLEET_TIER", string(capability.FleetTierHeavy)))
preferredFleet := capability.FleetTier(envOr("COGNITIVE_PREFERRED_FLEET_TIER", string(capability.FleetTierFast)))
cpuFleetHeaders := map[string]string{}
if cpuAPIKey != "" {
cpuFleetHeaders["Authorization"] = "Bearer " + cpuAPIKey
@@ -26,6 +26,7 @@ type PerformanceEvidenceID ids.ID
type BillingMode string
type AuthMode string
type SelectionObjective string
type ModelRatingCategory string
const (
QualificationDiscovered QualificationState = "DISCOVERED"
@@ -73,6 +74,11 @@ const (
SelectionQuality SelectionObjective = "QUALITY"
SelectionSpeed SelectionObjective = "SPEED"
SelectionEconomy SelectionObjective = "ECONOMY"
ModelRatingUnrated ModelRatingCategory = "UNRATED"
ModelRatingFast ModelRatingCategory = "FAST"
ModelRatingBalanced ModelRatingCategory = "BALANCED"
ModelRatingQuality ModelRatingCategory = "QUALITY"
)
// ProviderInstance contains only opaque references to endpoint and credentials.
@@ -164,6 +170,18 @@ type ResolvedCognitiveBinding struct {
Model ModelDescriptor
Evaluation *EvaluatedCapability
Performance *PerformanceEvidence
Rating *ModelRating
}
// ModelRating is a derived, provider-neutral explanation of why an eligible
// model is routed. It never grants qualification or activation authority.
type ModelRating struct {
Category ModelRatingCategory `json:"category"`
OverallScoreMillis int64 `json:"overall_score_millis"`
QualityScoreMillis int64 `json:"quality_score_millis"`
ReliabilityScoreMillis int64 `json:"reliability_score_millis"`
LatencyScoreMillis int64 `json:"latency_score_millis"`
EvidenceScoreMillis int64 `json:"evidence_score_millis"`
}
type CognitiveResultClass string
+2 -2
View File
@@ -1,5 +1,5 @@
{
"task_file": "dev/tasks/DEV-000181-medium-bounded-retest-evidence.json",
"task_id": "DEV-000181",
"task_file": "dev/tasks/DEV-000182-model-rating-and-adaptive-routing.json",
"task_id": "DEV-000182",
"task_revision": 1
}
@@ -0,0 +1,23 @@
{
"task_id": "DEV-000182",
"task_revision": 1,
"architecture_revision": "ADR-0018",
"base_repo_revision": "4709ef46076cf5c22522bbfea785fdb3c6220c14",
"risk_tier": "E",
"intent": "Expose an evidence-based model rating and adaptive routing policy so model category follows qualification and observed performance rather than provider or hardware names.",
"requirement_ids": ["WP-200", "WP-601", "adaptive model routing", "CPU-first current operation"],
"allowed_change_scope": {"modules": ["capability", "cognitive-worker"], "paths": ["contracts/capability/cognitive_registry.go", "modules/capability/domain/cognitive_registry.go", "modules/capability/domain/fleet.go", "modules/capability/domain/fleet_test.go", "cmd/cognitive-worker/main.go", "cmd/cognitive-worker/fleet_status_test.go", "modules/capability/domain/model_rating_test.go", "docs/MODEL_FLEET_POLICY.md", "dev/ACTIVE_TASK.json", "dev/tasks/DEV-000182-model-rating-and-adaptive-routing.json"]},
"forbidden_change_scope": {"paths": ["architecture/**", "security/**", "database/**", "deploy/**", ".gitea/**"]},
"allowed_architecture_delta": {"new_cross_module_dependencies": 0, "new_canonical_owners": 0, "new_privileges": 0, "new_third_party_dependencies": 0, "new_database_owner_roles": 0},
"contracts": {"consume": ["EvaluatedCapability", "PerformanceEvidence", "CognitiveRequirement"], "may_change": ["provider-neutral model rating projection"]},
"semantic_owner": "capability",
"canonical_state": ["existing owner-local qualification and performance evidence; rating is a derived projection"],
"state_transitions": ["unrated -> rated only after supported qualification evidence; category is recomputed from current evidence; routing chooses an eligible rated candidate and falls back on invocation failure"],
"side_effects_and_privileges": ["none; no provider invocation or external mutation"],
"persistence_and_migrations": ["no migration; rating is derived and serializable"],
"failure_retry_restart": ["missing or stale evidence yields UNRATED and cannot silently claim quality; existing gateway fallback remains authoritative"],
"assumptions": ["CPU is the current operational choice, but routing remains provider- and hardware-neutral", "discovery never activates a binding"],
"acceptance_scenarios": ["supported evidence produces a visible rating and category", "missing evidence remains UNRATED", "quality objective prefers higher evidence rating", "speed objective prefers lower observed latency", "provider identity alone cannot raise a rating", "CPU is preferred now but all ready capability classes remain valid failover candidates"],
"required_tests": ["go test ./...", "go vet ./...", "preflight", "check-diff", "check-architecture", "independent", "git diff --check", "CI"],
"terminal_gap_states": ["BLOCKED_BY_SPEC_GAP", "BLOCKED_BY_CONFLICT"]
}
+24
View File
@@ -0,0 +1,24 @@
# Model fleet policy
Domovoy does not equate intelligence with GPU, model name, or provider kind.
Hardware and endpoint identity are operational observations only. A model is
eligible for routing only after the existing qualification and activation
gates pass.
`FAST`, `MEDIUM`, and `HEAVY` are capability service classes, not hardware
claims. A CPU model may be the best available quality model, and a GPU model
may be unavailable, degraded, or unrated.
The Capability owner derives a provider-neutral `ModelRating` from evidence:
- `QUALITY`: high measured quality and zero critical failures;
- `BALANCED`: supported model with usable quality/reliability evidence;
- `FAST`: supported model whose observed latency is strong enough for quick work;
- `UNRATED`: discovery or incomplete evidence; it cannot be promoted by name.
The score combines quality, reliability, latency and evidence coverage. Missing
measurements remain explicit. A model can change category after a new immutable
qualification/performance revision. The current CPU route is an operational
choice, not a permanent hardware assumption. CPU is preferred in the current
deployment; if it is unavailable, the policy tries another ready class before
using the Lifeboat recovery path.
@@ -148,6 +148,8 @@ func SelectCognitiveBinding(requirement capability.CognitiveRequirement, candida
if candidate.Evaluation != nil && candidate.Evaluation.Result == capability.EvaluationUnsupported {
continue
}
rating := RateCognitiveBinding(candidate)
candidate.Rating = &rating
if requirement.MaxCostMicros != nil {
cost, known := estimatedCost(candidate.Provider, requirement.InputBytes, requirement.MaxOutputBytes)
if known && cost > *requirement.MaxCostMicros {
@@ -165,6 +167,56 @@ func SelectCognitiveBinding(requirement capability.CognitiveRequirement, candida
return capability.ResolvedCognitiveBinding{}, ErrNoInvocableCognitiveBinding
}
// RateCognitiveBinding derives a transparent routing category from measured
// evidence. Provider kind, endpoint name and hardware are intentionally not
// inputs to the score.
func RateCognitiveBinding(candidate capability.ResolvedCognitiveBinding) capability.ModelRating {
if candidate.Evaluation == nil || candidate.Evaluation.Result != capability.EvaluationSupported || candidate.Evaluation.QualityScoreMillis == nil || candidate.Evaluation.CriticalFailureMillis == nil {
return capability.ModelRating{Category: capability.ModelRatingUnrated}
}
quality := clampMillis(*candidate.Evaluation.QualityScoreMillis)
reliability := 1000 - clampMillis(*candidate.Evaluation.CriticalFailureMillis)
latency := int64(500)
if candidate.Performance != nil && candidate.Performance.LatencyMicros != nil {
latency = latencyScore(*candidate.Performance.LatencyMicros)
}
evidence := int64(700)
if candidate.Performance != nil {
evidence = 1000
}
overall := (quality*55 + reliability*25 + latency*15 + evidence*5) / 100
category := capability.ModelRatingBalanced
if overall >= 850 && quality >= 800 {
category = capability.ModelRatingQuality
} else if latency >= 800 && overall >= 700 {
category = capability.ModelRatingFast
}
return capability.ModelRating{Category: category, OverallScoreMillis: overall, QualityScoreMillis: quality, ReliabilityScoreMillis: reliability, LatencyScoreMillis: latency, EvidenceScoreMillis: evidence}
}
func clampMillis(value int64) int64 {
if value < 0 {
return 0
}
if value > 1000 {
return 1000
}
return value
}
func latencyScore(latencyMicros int64) int64 {
if latencyMicros <= 500_000 {
return 1000
}
if latencyMicros <= 2_000_000 {
return 800
}
if latencyMicros <= 10_000_000 {
return 500
}
return 200
}
func dataPolicySatisfies(actual, allowed capability.DataPolicy) bool {
return actual == capability.DataPolicyLocalOnly || actual == allowed
}
@@ -198,6 +250,9 @@ func betterCandidate(left, right capability.ResolvedCognitiveBinding, objective
}
}
if objective == capability.SelectionQuality || objective == capability.SelectionBalanced || objective == "" {
if left.Rating != nil && right.Rating != nil && left.Rating.OverallScoreMillis != right.Rating.OverallScoreMillis {
return left.Rating.OverallScoreMillis > right.Rating.OverallScoreMillis
}
li, ri := left.Model.IntelligenceScoreMillis, right.Model.IntelligenceScoreMillis
if li != nil && ri != nil && *li != *ri {
return *li > *ri
+1 -1
View File
@@ -58,7 +58,7 @@ func fleetFailoverOrder(preferred capability.FleetTier) []capability.FleetTier {
case capability.FleetTierMedium:
return []capability.FleetTier{capability.FleetTierMedium, capability.FleetTierFast}
case capability.FleetTierFast:
return []capability.FleetTier{capability.FleetTierFast}
return []capability.FleetTier{capability.FleetTierFast, capability.FleetTierMedium, capability.FleetTierHeavy}
case capability.FleetTierLifeboat:
return []capability.FleetTier{capability.FleetTierLifeboat}
default:
+14
View File
@@ -46,3 +46,17 @@ func TestFleetReadinessFallsBackFromHeavyToFastBeforeLifeboat(t *testing.T) {
t.Fatalf("selected=%q err=%v", selected, err)
}
}
func TestFleetReadinessCpuPreferenceCanUseOtherReadyClass(t *testing.T) {
readiness, err := BuildFleetReadiness(100, []capability.FleetMember{
{ProviderInstanceID: "medium", ModelDescriptorID: "medium-model", Tier: capability.FleetTierMedium, Ready: true, Reason: "available"},
{ProviderInstanceID: "lifeboat", ModelDescriptorID: "lifeboat-model", Tier: capability.FleetTierLifeboat, Ready: true, Reason: "available"},
})
if err != nil {
t.Fatal(err)
}
selected, err := SelectFleetTier(readiness, capability.FleetTierFast)
if err != nil || selected != capability.FleetTierMedium {
t.Fatalf("selected=%q err=%v", selected, err)
}
}
@@ -0,0 +1,40 @@
package domain
import (
"testing"
"git.cin.su/m/Assistant/contracts/capability"
)
func TestRateCognitiveBindingRequiresSupportedEvidence(t *testing.T) {
rating := RateCognitiveBinding(capability.ResolvedCognitiveBinding{})
if rating.Category != capability.ModelRatingUnrated || rating.OverallScoreMillis != 0 {
t.Fatalf("rating=%+v", rating)
}
}
func TestRateCognitiveBindingUsesEvidenceNotProviderName(t *testing.T) {
quality, critical, latency := int64(900), int64(0), int64(400_000)
base := capability.ResolvedCognitiveBinding{
Provider: capability.ProviderInstance{Kind: "openai-compatible-chat-v1"},
Evaluation: &capability.EvaluatedCapability{Result: capability.EvaluationSupported, QualityScoreMillis: &quality, CriticalFailureMillis: &critical},
Performance: &capability.PerformanceEvidence{LatencyMicros: &latency},
}
other := base
other.Provider.Kind = "ollama-v1"
left, right := RateCognitiveBinding(base), RateCognitiveBinding(other)
if left != right || left.Category != capability.ModelRatingQuality {
t.Fatalf("provider changed rating: left=%+v right=%+v", left, right)
}
}
func TestRateCognitiveBindingClassifiesSlowEvidenceAsBalanced(t *testing.T) {
quality, critical, latency := int64(800), int64(0), int64(20_000_000)
rating := RateCognitiveBinding(capability.ResolvedCognitiveBinding{
Evaluation: &capability.EvaluatedCapability{Result: capability.EvaluationSupported, QualityScoreMillis: &quality, CriticalFailureMillis: &critical},
Performance: &capability.PerformanceEvidence{LatencyMicros: &latency},
})
if rating.Category != capability.ModelRatingBalanced || rating.LatencyScoreMillis != 200 {
t.Fatalf("rating=%+v", rating)
}
}