feat(capability): consume qualification evidence in selection
This commit is contained in:
@@ -148,6 +148,9 @@ func SelectCognitiveBinding(requirement capability.CognitiveRequirement, candida
|
||||
if candidate.Evaluation != nil && candidate.Evaluation.Result == capability.EvaluationUnsupported {
|
||||
continue
|
||||
}
|
||||
if candidate.Qualification != nil && candidate.Qualification.Status == capability.ModelQualificationUnavailable && candidate.Qualification.ObservedAtUnixMillis >= maxObservedAt(candidate.Provider.ObservedAtUnixMillis, candidate.Model.ObservedAtUnixMillis) {
|
||||
continue
|
||||
}
|
||||
if requirement.TaskProfileID != "" && (candidate.TaskEvaluation == nil || candidate.TaskEvaluation.ProfileID != requirement.TaskProfileID || candidate.TaskEvaluation.Result != capability.EvaluationSupported) {
|
||||
continue
|
||||
}
|
||||
@@ -195,10 +198,14 @@ func RateCognitiveBinding(candidate capability.ResolvedCognitiveBinding) capabil
|
||||
latency := int64(500)
|
||||
if candidate.Performance != nil && candidate.Performance.LatencyMicros != nil {
|
||||
latency = latencyScore(*candidate.Performance.LatencyMicros)
|
||||
} else if candidate.Qualification != nil && candidate.Qualification.Status == capability.ModelQualificationReachable {
|
||||
latency = latencyScore(candidate.Qualification.LatencyMillis * 1000)
|
||||
}
|
||||
evidence := int64(700)
|
||||
if candidate.Performance != nil {
|
||||
evidence = 1000
|
||||
} else if candidate.Qualification != nil && candidate.Qualification.Status == capability.ModelQualificationReachable {
|
||||
evidence = 750
|
||||
}
|
||||
overall := (quality*55 + reliability*25 + latency*15 + evidence*5) / 100
|
||||
category := capability.ModelRatingBalanced
|
||||
@@ -210,6 +217,13 @@ func RateCognitiveBinding(candidate capability.ResolvedCognitiveBinding) capabil
|
||||
return capability.ModelRating{Category: category, OverallScoreMillis: overall, QualityScoreMillis: quality, ReliabilityScoreMillis: reliability, LatencyScoreMillis: latency, EvidenceScoreMillis: evidence}
|
||||
}
|
||||
|
||||
func maxObservedAt(left, right int64) int64 {
|
||||
if left > right {
|
||||
return left
|
||||
}
|
||||
return right
|
||||
}
|
||||
|
||||
func clampMillis(value int64) int64 {
|
||||
if value < 0 {
|
||||
return 0
|
||||
|
||||
@@ -38,3 +38,22 @@ func TestRateCognitiveBindingClassifiesSlowEvidenceAsBalanced(t *testing.T) {
|
||||
t.Fatalf("rating=%+v", rating)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSelectionExcludesFreshUnavailableQualificationButNotStaleEvidence(t *testing.T) {
|
||||
quality, critical := int64(900), int64(0)
|
||||
base := capability.ResolvedCognitiveBinding{
|
||||
Binding: capability.CognitiveBinding{ID: "binding-qualification", Revision: 1, CapabilityID: "creator-advisory", OwnerPrincipalID: "owner", ProviderInstanceID: "provider", ProviderRevision: 1, ModelDescriptorID: "model", ModelRevision: 1, DataPolicy: capability.DataPolicyLocalOnly, Qualification: capability.QualificationValidated, Activation: capability.ActivationActive, Health: capability.HealthAvailable, LimitPolicyRef: "limits"},
|
||||
Provider: capability.ProviderInstance{ID: "provider", Revision: 1, Kind: "local", EndpointRef: "endpoint", Enabled: true, Qualification: capability.QualificationValidated, Health: capability.HealthAvailable, ObservedAtUnixMillis: 20},
|
||||
Model: capability.ModelDescriptor{ID: "model", Revision: 1, ProviderInstanceID: "provider", ProviderModelID: "model", MaxContextBytes: 1000, Modalities: []capability.ModelModality{capability.ModalityText}, Capabilities: []capability.ModelCapability{capability.CapabilityAdvisory}, Qualification: capability.QualificationValidated, Health: capability.HealthAvailable, ObservedAtUnixMillis: 20},
|
||||
Evaluation: &capability.EvaluatedCapability{ID: "evaluation", BindingID: "binding-qualification", BindingRevision: 1, CapabilityID: "creator-advisory", Result: capability.EvaluationSupported, QualityScoreMillis: &quality, CriticalFailureMillis: &critical},
|
||||
Qualification: &capability.ModelQualificationObservation{ID: "observation", ProviderInstanceID: "provider", ProviderModelID: "model", Status: capability.ModelQualificationUnavailable, FinishReason: "TIMEOUT", ObservedAtUnixMillis: 20, Category: capability.ModelRatingUnrated},
|
||||
}
|
||||
requirement := capability.CognitiveRequirement{CapabilityID: "creator-advisory", OwnerPrincipalID: "owner", DataPolicy: capability.DataPolicyLocalOnly, RequiredModalities: []capability.ModelModality{capability.ModalityText}, RequiredCapabilities: []capability.ModelCapability{capability.CapabilityAdvisory}, MaxOutputBytes: 10}
|
||||
if _, err := SelectCognitiveBinding(requirement, []capability.ResolvedCognitiveBinding{base}); err == nil {
|
||||
t.Fatal("fresh unavailable qualification should exclude candidate")
|
||||
}
|
||||
base.Qualification.ObservedAtUnixMillis = 19
|
||||
if _, err := SelectCognitiveBinding(requirement, []capability.ResolvedCognitiveBinding{base}); err != nil {
|
||||
t.Fatalf("stale unavailable qualification should not exclude candidate: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user