feat(capability): consume qualification evidence in selection
CI / test (push) Successful in 1m16s
CI / test (pull_request) Successful in 1m26s

This commit is contained in:
m
2026-08-23 19:28:13 +03:00
parent 47327cc6d8
commit c81e2269e1
6 changed files with 67 additions and 2 deletions
@@ -148,6 +148,9 @@ func SelectCognitiveBinding(requirement capability.CognitiveRequirement, candida
if candidate.Evaluation != nil && candidate.Evaluation.Result == capability.EvaluationUnsupported {
continue
}
if candidate.Qualification != nil && candidate.Qualification.Status == capability.ModelQualificationUnavailable && candidate.Qualification.ObservedAtUnixMillis >= maxObservedAt(candidate.Provider.ObservedAtUnixMillis, candidate.Model.ObservedAtUnixMillis) {
continue
}
if requirement.TaskProfileID != "" && (candidate.TaskEvaluation == nil || candidate.TaskEvaluation.ProfileID != requirement.TaskProfileID || candidate.TaskEvaluation.Result != capability.EvaluationSupported) {
continue
}
@@ -195,10 +198,14 @@ func RateCognitiveBinding(candidate capability.ResolvedCognitiveBinding) capabil
latency := int64(500)
if candidate.Performance != nil && candidate.Performance.LatencyMicros != nil {
latency = latencyScore(*candidate.Performance.LatencyMicros)
} else if candidate.Qualification != nil && candidate.Qualification.Status == capability.ModelQualificationReachable {
latency = latencyScore(candidate.Qualification.LatencyMillis * 1000)
}
evidence := int64(700)
if candidate.Performance != nil {
evidence = 1000
} else if candidate.Qualification != nil && candidate.Qualification.Status == capability.ModelQualificationReachable {
evidence = 750
}
overall := (quality*55 + reliability*25 + latency*15 + evidence*5) / 100
category := capability.ModelRatingBalanced
@@ -210,6 +217,13 @@ func RateCognitiveBinding(candidate capability.ResolvedCognitiveBinding) capabil
return capability.ModelRating{Category: category, OverallScoreMillis: overall, QualityScoreMillis: quality, ReliabilityScoreMillis: reliability, LatencyScoreMillis: latency, EvidenceScoreMillis: evidence}
}
func maxObservedAt(left, right int64) int64 {
if left > right {
return left
}
return right
}
func clampMillis(value int64) int64 {
if value < 0 {
return 0
@@ -38,3 +38,22 @@ func TestRateCognitiveBindingClassifiesSlowEvidenceAsBalanced(t *testing.T) {
t.Fatalf("rating=%+v", rating)
}
}
func TestSelectionExcludesFreshUnavailableQualificationButNotStaleEvidence(t *testing.T) {
quality, critical := int64(900), int64(0)
base := capability.ResolvedCognitiveBinding{
Binding: capability.CognitiveBinding{ID: "binding-qualification", Revision: 1, CapabilityID: "creator-advisory", OwnerPrincipalID: "owner", ProviderInstanceID: "provider", ProviderRevision: 1, ModelDescriptorID: "model", ModelRevision: 1, DataPolicy: capability.DataPolicyLocalOnly, Qualification: capability.QualificationValidated, Activation: capability.ActivationActive, Health: capability.HealthAvailable, LimitPolicyRef: "limits"},
Provider: capability.ProviderInstance{ID: "provider", Revision: 1, Kind: "local", EndpointRef: "endpoint", Enabled: true, Qualification: capability.QualificationValidated, Health: capability.HealthAvailable, ObservedAtUnixMillis: 20},
Model: capability.ModelDescriptor{ID: "model", Revision: 1, ProviderInstanceID: "provider", ProviderModelID: "model", MaxContextBytes: 1000, Modalities: []capability.ModelModality{capability.ModalityText}, Capabilities: []capability.ModelCapability{capability.CapabilityAdvisory}, Qualification: capability.QualificationValidated, Health: capability.HealthAvailable, ObservedAtUnixMillis: 20},
Evaluation: &capability.EvaluatedCapability{ID: "evaluation", BindingID: "binding-qualification", BindingRevision: 1, CapabilityID: "creator-advisory", Result: capability.EvaluationSupported, QualityScoreMillis: &quality, CriticalFailureMillis: &critical},
Qualification: &capability.ModelQualificationObservation{ID: "observation", ProviderInstanceID: "provider", ProviderModelID: "model", Status: capability.ModelQualificationUnavailable, FinishReason: "TIMEOUT", ObservedAtUnixMillis: 20, Category: capability.ModelRatingUnrated},
}
requirement := capability.CognitiveRequirement{CapabilityID: "creator-advisory", OwnerPrincipalID: "owner", DataPolicy: capability.DataPolicyLocalOnly, RequiredModalities: []capability.ModelModality{capability.ModalityText}, RequiredCapabilities: []capability.ModelCapability{capability.CapabilityAdvisory}, MaxOutputBytes: 10}
if _, err := SelectCognitiveBinding(requirement, []capability.ResolvedCognitiveBinding{base}); err == nil {
t.Fatal("fresh unavailable qualification should exclude candidate")
}
base.Qualification.ObservedAtUnixMillis = 19
if _, err := SelectCognitiveBinding(requirement, []capability.ResolvedCognitiveBinding{base}); err != nil {
t.Fatalf("stale unavailable qualification should not exclude candidate: %v", err)
}
}