feat: add evidence-based adaptive model rating
This commit is contained in:
@@ -148,6 +148,8 @@ func SelectCognitiveBinding(requirement capability.CognitiveRequirement, candida
|
||||
if candidate.Evaluation != nil && candidate.Evaluation.Result == capability.EvaluationUnsupported {
|
||||
continue
|
||||
}
|
||||
rating := RateCognitiveBinding(candidate)
|
||||
candidate.Rating = &rating
|
||||
if requirement.MaxCostMicros != nil {
|
||||
cost, known := estimatedCost(candidate.Provider, requirement.InputBytes, requirement.MaxOutputBytes)
|
||||
if known && cost > *requirement.MaxCostMicros {
|
||||
@@ -165,6 +167,56 @@ func SelectCognitiveBinding(requirement capability.CognitiveRequirement, candida
|
||||
return capability.ResolvedCognitiveBinding{}, ErrNoInvocableCognitiveBinding
|
||||
}
|
||||
|
||||
// RateCognitiveBinding derives a transparent routing category from measured
|
||||
// evidence. Provider kind, endpoint name and hardware are intentionally not
|
||||
// inputs to the score.
|
||||
func RateCognitiveBinding(candidate capability.ResolvedCognitiveBinding) capability.ModelRating {
|
||||
if candidate.Evaluation == nil || candidate.Evaluation.Result != capability.EvaluationSupported || candidate.Evaluation.QualityScoreMillis == nil || candidate.Evaluation.CriticalFailureMillis == nil {
|
||||
return capability.ModelRating{Category: capability.ModelRatingUnrated}
|
||||
}
|
||||
quality := clampMillis(*candidate.Evaluation.QualityScoreMillis)
|
||||
reliability := 1000 - clampMillis(*candidate.Evaluation.CriticalFailureMillis)
|
||||
latency := int64(500)
|
||||
if candidate.Performance != nil && candidate.Performance.LatencyMicros != nil {
|
||||
latency = latencyScore(*candidate.Performance.LatencyMicros)
|
||||
}
|
||||
evidence := int64(700)
|
||||
if candidate.Performance != nil {
|
||||
evidence = 1000
|
||||
}
|
||||
overall := (quality*55 + reliability*25 + latency*15 + evidence*5) / 100
|
||||
category := capability.ModelRatingBalanced
|
||||
if overall >= 850 && quality >= 800 {
|
||||
category = capability.ModelRatingQuality
|
||||
} else if latency >= 800 && overall >= 700 {
|
||||
category = capability.ModelRatingFast
|
||||
}
|
||||
return capability.ModelRating{Category: category, OverallScoreMillis: overall, QualityScoreMillis: quality, ReliabilityScoreMillis: reliability, LatencyScoreMillis: latency, EvidenceScoreMillis: evidence}
|
||||
}
|
||||
|
||||
func clampMillis(value int64) int64 {
|
||||
if value < 0 {
|
||||
return 0
|
||||
}
|
||||
if value > 1000 {
|
||||
return 1000
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
func latencyScore(latencyMicros int64) int64 {
|
||||
if latencyMicros <= 500_000 {
|
||||
return 1000
|
||||
}
|
||||
if latencyMicros <= 2_000_000 {
|
||||
return 800
|
||||
}
|
||||
if latencyMicros <= 10_000_000 {
|
||||
return 500
|
||||
}
|
||||
return 200
|
||||
}
|
||||
|
||||
func dataPolicySatisfies(actual, allowed capability.DataPolicy) bool {
|
||||
return actual == capability.DataPolicyLocalOnly || actual == allowed
|
||||
}
|
||||
@@ -198,6 +250,9 @@ func betterCandidate(left, right capability.ResolvedCognitiveBinding, objective
|
||||
}
|
||||
}
|
||||
if objective == capability.SelectionQuality || objective == capability.SelectionBalanced || objective == "" {
|
||||
if left.Rating != nil && right.Rating != nil && left.Rating.OverallScoreMillis != right.Rating.OverallScoreMillis {
|
||||
return left.Rating.OverallScoreMillis > right.Rating.OverallScoreMillis
|
||||
}
|
||||
li, ri := left.Model.IntelligenceScoreMillis, right.Model.IntelligenceScoreMillis
|
||||
if li != nil && ri != nil && *li != *ri {
|
||||
return *li > *ri
|
||||
|
||||
@@ -58,7 +58,7 @@ func fleetFailoverOrder(preferred capability.FleetTier) []capability.FleetTier {
|
||||
case capability.FleetTierMedium:
|
||||
return []capability.FleetTier{capability.FleetTierMedium, capability.FleetTierFast}
|
||||
case capability.FleetTierFast:
|
||||
return []capability.FleetTier{capability.FleetTierFast}
|
||||
return []capability.FleetTier{capability.FleetTierFast, capability.FleetTierMedium, capability.FleetTierHeavy}
|
||||
case capability.FleetTierLifeboat:
|
||||
return []capability.FleetTier{capability.FleetTierLifeboat}
|
||||
default:
|
||||
|
||||
@@ -46,3 +46,17 @@ func TestFleetReadinessFallsBackFromHeavyToFastBeforeLifeboat(t *testing.T) {
|
||||
t.Fatalf("selected=%q err=%v", selected, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFleetReadinessCpuPreferenceCanUseOtherReadyClass(t *testing.T) {
|
||||
readiness, err := BuildFleetReadiness(100, []capability.FleetMember{
|
||||
{ProviderInstanceID: "medium", ModelDescriptorID: "medium-model", Tier: capability.FleetTierMedium, Ready: true, Reason: "available"},
|
||||
{ProviderInstanceID: "lifeboat", ModelDescriptorID: "lifeboat-model", Tier: capability.FleetTierLifeboat, Ready: true, Reason: "available"},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
selected, err := SelectFleetTier(readiness, capability.FleetTierFast)
|
||||
if err != nil || selected != capability.FleetTierMedium {
|
||||
t.Fatalf("selected=%q err=%v", selected, err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,40 @@
|
||||
package domain
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"git.cin.su/m/Assistant/contracts/capability"
|
||||
)
|
||||
|
||||
func TestRateCognitiveBindingRequiresSupportedEvidence(t *testing.T) {
|
||||
rating := RateCognitiveBinding(capability.ResolvedCognitiveBinding{})
|
||||
if rating.Category != capability.ModelRatingUnrated || rating.OverallScoreMillis != 0 {
|
||||
t.Fatalf("rating=%+v", rating)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRateCognitiveBindingUsesEvidenceNotProviderName(t *testing.T) {
|
||||
quality, critical, latency := int64(900), int64(0), int64(400_000)
|
||||
base := capability.ResolvedCognitiveBinding{
|
||||
Provider: capability.ProviderInstance{Kind: "openai-compatible-chat-v1"},
|
||||
Evaluation: &capability.EvaluatedCapability{Result: capability.EvaluationSupported, QualityScoreMillis: &quality, CriticalFailureMillis: &critical},
|
||||
Performance: &capability.PerformanceEvidence{LatencyMicros: &latency},
|
||||
}
|
||||
other := base
|
||||
other.Provider.Kind = "ollama-v1"
|
||||
left, right := RateCognitiveBinding(base), RateCognitiveBinding(other)
|
||||
if left != right || left.Category != capability.ModelRatingQuality {
|
||||
t.Fatalf("provider changed rating: left=%+v right=%+v", left, right)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRateCognitiveBindingClassifiesSlowEvidenceAsBalanced(t *testing.T) {
|
||||
quality, critical, latency := int64(800), int64(0), int64(20_000_000)
|
||||
rating := RateCognitiveBinding(capability.ResolvedCognitiveBinding{
|
||||
Evaluation: &capability.EvaluatedCapability{Result: capability.EvaluationSupported, QualityScoreMillis: &quality, CriticalFailureMillis: &critical},
|
||||
Performance: &capability.PerformanceEvidence{LatencyMicros: &latency},
|
||||
})
|
||||
if rating.Category != capability.ModelRatingBalanced || rating.LatencyScoreMillis != 200 {
|
||||
t.Fatalf("rating=%+v", rating)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user