feat: add evidence-based adaptive model rating
CI / test (push) Failing after 37s
CI / test (pull_request) Failing after 32s

This commit is contained in:
m
2026-08-22 23:42:10 +03:00
parent 4709ef4607
commit 30443027a6
8 changed files with 176 additions and 2 deletions
@@ -148,6 +148,8 @@ func SelectCognitiveBinding(requirement capability.CognitiveRequirement, candida
if candidate.Evaluation != nil && candidate.Evaluation.Result == capability.EvaluationUnsupported {
continue
}
rating := RateCognitiveBinding(candidate)
candidate.Rating = &rating
if requirement.MaxCostMicros != nil {
cost, known := estimatedCost(candidate.Provider, requirement.InputBytes, requirement.MaxOutputBytes)
if known && cost > *requirement.MaxCostMicros {
@@ -165,6 +167,56 @@ func SelectCognitiveBinding(requirement capability.CognitiveRequirement, candida
return capability.ResolvedCognitiveBinding{}, ErrNoInvocableCognitiveBinding
}
// RateCognitiveBinding derives a transparent routing category from measured
// evidence. Provider kind, endpoint name and hardware are intentionally not
// inputs to the score.
func RateCognitiveBinding(candidate capability.ResolvedCognitiveBinding) capability.ModelRating {
if candidate.Evaluation == nil || candidate.Evaluation.Result != capability.EvaluationSupported || candidate.Evaluation.QualityScoreMillis == nil || candidate.Evaluation.CriticalFailureMillis == nil {
return capability.ModelRating{Category: capability.ModelRatingUnrated}
}
quality := clampMillis(*candidate.Evaluation.QualityScoreMillis)
reliability := 1000 - clampMillis(*candidate.Evaluation.CriticalFailureMillis)
latency := int64(500)
if candidate.Performance != nil && candidate.Performance.LatencyMicros != nil {
latency = latencyScore(*candidate.Performance.LatencyMicros)
}
evidence := int64(700)
if candidate.Performance != nil {
evidence = 1000
}
overall := (quality*55 + reliability*25 + latency*15 + evidence*5) / 100
category := capability.ModelRatingBalanced
if overall >= 850 && quality >= 800 {
category = capability.ModelRatingQuality
} else if latency >= 800 && overall >= 700 {
category = capability.ModelRatingFast
}
return capability.ModelRating{Category: category, OverallScoreMillis: overall, QualityScoreMillis: quality, ReliabilityScoreMillis: reliability, LatencyScoreMillis: latency, EvidenceScoreMillis: evidence}
}
func clampMillis(value int64) int64 {
if value < 0 {
return 0
}
if value > 1000 {
return 1000
}
return value
}
func latencyScore(latencyMicros int64) int64 {
if latencyMicros <= 500_000 {
return 1000
}
if latencyMicros <= 2_000_000 {
return 800
}
if latencyMicros <= 10_000_000 {
return 500
}
return 200
}
func dataPolicySatisfies(actual, allowed capability.DataPolicy) bool {
return actual == capability.DataPolicyLocalOnly || actual == allowed
}
@@ -198,6 +250,9 @@ func betterCandidate(left, right capability.ResolvedCognitiveBinding, objective
}
}
if objective == capability.SelectionQuality || objective == capability.SelectionBalanced || objective == "" {
if left.Rating != nil && right.Rating != nil && left.Rating.OverallScoreMillis != right.Rating.OverallScoreMillis {
return left.Rating.OverallScoreMillis > right.Rating.OverallScoreMillis
}
li, ri := left.Model.IntelligenceScoreMillis, right.Model.IntelligenceScoreMillis
if li != nil && ri != nil && *li != *ri {
return *li > *ri
+1 -1
View File
@@ -58,7 +58,7 @@ func fleetFailoverOrder(preferred capability.FleetTier) []capability.FleetTier {
case capability.FleetTierMedium:
return []capability.FleetTier{capability.FleetTierMedium, capability.FleetTierFast}
case capability.FleetTierFast:
return []capability.FleetTier{capability.FleetTierFast}
return []capability.FleetTier{capability.FleetTierFast, capability.FleetTierMedium, capability.FleetTierHeavy}
case capability.FleetTierLifeboat:
return []capability.FleetTier{capability.FleetTierLifeboat}
default:
+14
View File
@@ -46,3 +46,17 @@ func TestFleetReadinessFallsBackFromHeavyToFastBeforeLifeboat(t *testing.T) {
t.Fatalf("selected=%q err=%v", selected, err)
}
}
func TestFleetReadinessCpuPreferenceCanUseOtherReadyClass(t *testing.T) {
readiness, err := BuildFleetReadiness(100, []capability.FleetMember{
{ProviderInstanceID: "medium", ModelDescriptorID: "medium-model", Tier: capability.FleetTierMedium, Ready: true, Reason: "available"},
{ProviderInstanceID: "lifeboat", ModelDescriptorID: "lifeboat-model", Tier: capability.FleetTierLifeboat, Ready: true, Reason: "available"},
})
if err != nil {
t.Fatal(err)
}
selected, err := SelectFleetTier(readiness, capability.FleetTierFast)
if err != nil || selected != capability.FleetTierMedium {
t.Fatalf("selected=%q err=%v", selected, err)
}
}
@@ -0,0 +1,40 @@
package domain
import (
"testing"
"git.cin.su/m/Assistant/contracts/capability"
)
func TestRateCognitiveBindingRequiresSupportedEvidence(t *testing.T) {
rating := RateCognitiveBinding(capability.ResolvedCognitiveBinding{})
if rating.Category != capability.ModelRatingUnrated || rating.OverallScoreMillis != 0 {
t.Fatalf("rating=%+v", rating)
}
}
func TestRateCognitiveBindingUsesEvidenceNotProviderName(t *testing.T) {
quality, critical, latency := int64(900), int64(0), int64(400_000)
base := capability.ResolvedCognitiveBinding{
Provider: capability.ProviderInstance{Kind: "openai-compatible-chat-v1"},
Evaluation: &capability.EvaluatedCapability{Result: capability.EvaluationSupported, QualityScoreMillis: &quality, CriticalFailureMillis: &critical},
Performance: &capability.PerformanceEvidence{LatencyMicros: &latency},
}
other := base
other.Provider.Kind = "ollama-v1"
left, right := RateCognitiveBinding(base), RateCognitiveBinding(other)
if left != right || left.Category != capability.ModelRatingQuality {
t.Fatalf("provider changed rating: left=%+v right=%+v", left, right)
}
}
func TestRateCognitiveBindingClassifiesSlowEvidenceAsBalanced(t *testing.T) {
quality, critical, latency := int64(800), int64(0), int64(20_000_000)
rating := RateCognitiveBinding(capability.ResolvedCognitiveBinding{
Evaluation: &capability.EvaluatedCapability{Result: capability.EvaluationSupported, QualityScoreMillis: &quality, CriticalFailureMillis: &critical},
Performance: &capability.PerformanceEvidence{LatencyMicros: &latency},
})
if rating.Category != capability.ModelRatingBalanced || rating.LatencyScoreMillis != 200 {
t.Fatalf("rating=%+v", rating)
}
}