feat(capability): add task-aware model evaluation evidence
CI / test (push) Successful in 57s
CI / test (pull_request) Successful in 1m0s

This commit is contained in:
m
2026-08-23 11:57:47 +03:00
parent b78a026ee1
commit feea909d69
11 changed files with 383 additions and 17 deletions
@@ -148,6 +148,9 @@ func SelectCognitiveBinding(requirement capability.CognitiveRequirement, candida
if candidate.Evaluation != nil && candidate.Evaluation.Result == capability.EvaluationUnsupported {
continue
}
if requirement.TaskProfileID != "" && (candidate.TaskEvaluation == nil || candidate.TaskEvaluation.ProfileID != requirement.TaskProfileID || candidate.TaskEvaluation.Result != capability.EvaluationSupported) {
continue
}
rating := RateCognitiveBinding(candidate)
candidate.Rating = &rating
if requirement.MaxCostMicros != nil {
@@ -171,6 +174,19 @@ func SelectCognitiveBinding(requirement capability.CognitiveRequirement, candida
// evidence. Provider kind, endpoint name and hardware are intentionally not
// inputs to the score.
func RateCognitiveBinding(candidate capability.ResolvedCognitiveBinding) capability.ModelRating {
if candidate.TaskEvaluation != nil {
evaluation := candidate.TaskEvaluation
if evaluation.Result != capability.EvaluationSupported {
return capability.ModelRating{TaskProfileID: evaluation.ProfileID, Category: capability.ModelRatingUnrated}
}
category := capability.ModelRatingBalanced
if evaluation.OverallScoreMillis >= 850 && evaluation.QualityScoreMillis >= 800 {
category = capability.ModelRatingQuality
} else if evaluation.LatencyScoreMillis >= 800 && evaluation.OverallScoreMillis >= 700 {
category = capability.ModelRatingFast
}
return capability.ModelRating{TaskProfileID: evaluation.ProfileID, TaskScoreMillis: evaluation.OverallScoreMillis, Category: category, OverallScoreMillis: evaluation.OverallScoreMillis, QualityScoreMillis: evaluation.QualityScoreMillis, ReliabilityScoreMillis: evaluation.ReliabilityScoreMillis, LatencyScoreMillis: evaluation.LatencyScoreMillis, EvidenceScoreMillis: 1000}
}
if candidate.Evaluation == nil || candidate.Evaluation.Result != capability.EvaluationSupported || candidate.Evaluation.QualityScoreMillis == nil || candidate.Evaluation.CriticalFailureMillis == nil {
return capability.ModelRating{Category: capability.ModelRatingUnrated}
}
@@ -0,0 +1,106 @@
package domain
import (
"errors"
"git.cin.su/m/Assistant/contracts/artifact"
"git.cin.su/m/Assistant/contracts/capability"
)
var ErrInvalidTaskEvaluation = errors.New("invalid task evaluation")
func BuildAdditionalTaskEvaluationProfile(parent capability.TaskEvaluationProfile, request capability.TaskEvaluationExpansionRequest, profileID capability.TaskEvaluationProfileID, revision int64) (capability.TaskEvaluationProfile, error) {
if err := parent.Validate(); err != nil || parent.Stage != capability.TaskEvaluationGeneral || request.ParentProfileID != parent.ID || request.CapabilityID != parent.CapabilityID || request.TriggerEvidenceRef == "" || len(request.NewCaseIDs) == 0 || profileID == "" || revision < 1 || request.BenchmarkVersion == "" {
return capability.TaskEvaluationProfile{}, ErrInvalidTaskEvaluation
}
known := make(map[string]struct{}, len(parent.RequiredCaseIDs))
for _, caseID := range parent.RequiredCaseIDs {
known[caseID] = struct{}{}
}
seen := make(map[string]struct{}, len(request.NewCaseIDs))
for _, caseID := range request.NewCaseIDs {
if caseID == "" {
return capability.TaskEvaluationProfile{}, ErrInvalidTaskEvaluation
}
if _, exists := known[caseID]; exists {
return capability.TaskEvaluationProfile{}, ErrInvalidTaskEvaluation
}
if _, exists := seen[caseID]; exists {
return capability.TaskEvaluationProfile{}, ErrInvalidTaskEvaluation
}
seen[caseID] = struct{}{}
}
return capability.TaskEvaluationProfile{ID: profileID, Revision: revision, CapabilityID: parent.CapabilityID, Stage: capability.TaskEvaluationAdditional, ParentProfileID: parent.ID, TriggerEvidenceRef: request.TriggerEvidenceRef, BenchmarkVersion: request.BenchmarkVersion, RequiredCaseIDs: append([]string(nil), request.NewCaseIDs...), QualityWeightMillis: parent.QualityWeightMillis, ReliabilityWeightMillis: parent.ReliabilityWeightMillis, LatencyWeightMillis: parent.LatencyWeightMillis}, nil
}
// BuildTaskEvaluation aggregates independently observed case results. It is
// intentionally deterministic: no provider name, hardware label or caller
// supplied aggregate score can influence the result.
func BuildTaskEvaluation(profile capability.TaskEvaluationProfile, evaluationID capability.TaskEvaluationID, bindingID capability.CapabilityBindingID, bindingRevision int64, capabilityID capability.CapabilityID, observations []capability.TaskCaseObservation, artifactID artifact.ArtifactID, observedAtUnixMillis int64) (capability.TaskEvaluation, error) {
if err := profile.Validate(); err != nil || evaluationID == "" || bindingID == "" || bindingRevision < 1 || capabilityID == "" || capabilityID != profile.CapabilityID || artifactID == "" || observedAtUnixMillis < 1 || len(observations) != len(profile.RequiredCaseIDs) {
return capability.TaskEvaluation{}, ErrInvalidTaskEvaluation
}
required := make(map[string]struct{}, len(profile.RequiredCaseIDs))
for _, caseID := range profile.RequiredCaseIDs {
required[caseID] = struct{}{}
}
seen := make(map[string]struct{}, len(observations))
qualityTotal, latencyTotal, passed, critical := int64(0), int64(0), int64(0), int64(0)
unknown := false
unsupported := false
for _, observation := range observations {
if _, ok := required[observation.CaseID]; !ok {
return capability.TaskEvaluation{}, ErrInvalidTaskEvaluation
}
if _, ok := seen[observation.CaseID]; ok || observation.EvidenceArtifactRef == "" || observation.QualityScoreMillis < 0 || observation.QualityScoreMillis > 1000 || observation.LatencyMicros < 0 {
return capability.TaskEvaluation{}, ErrInvalidTaskEvaluation
}
seen[observation.CaseID] = struct{}{}
qualityTotal += observation.QualityScoreMillis
latencyTotal += observation.LatencyMicros
if observation.CriticalFailure {
critical++
}
switch observation.Result {
case capability.EvaluationSupported:
passed++
case capability.EvaluationUnsupported:
unsupported = true
case capability.EvaluationUnknown:
unknown = true
default:
return capability.TaskEvaluation{}, ErrInvalidTaskEvaluation
}
}
if len(seen) != len(required) {
return capability.TaskEvaluation{}, ErrInvalidTaskEvaluation
}
count := int64(len(observations))
quality := qualityTotal / count
reliability := passed * 1000 / count
latency := latencyScore(latencyTotal / count)
criticalMillis := critical * 1000 / count
overall := (quality*profile.QualityWeightMillis + reliability*profile.ReliabilityWeightMillis + latency*profile.LatencyWeightMillis) / 1000
result := capability.EvaluationSupported
if unknown {
result = capability.EvaluationUnknown
} else if unsupported {
result = capability.EvaluationUnsupported
}
return capability.TaskEvaluation{ID: evaluationID, ProfileID: profile.ID, BindingID: bindingID, BindingRevision: bindingRevision, CapabilityID: capabilityID, Result: result, QualityScoreMillis: quality, ReliabilityScoreMillis: reliability, LatencyScoreMillis: latency, OverallScoreMillis: overall, CriticalFailureMillis: criticalMillis, BenchmarkVersion: profile.BenchmarkVersion, ArtifactID: artifactID, ObservedAtUnixMillis: observedAtUnixMillis}, nil
}
func ValidateTaskEvaluation(value capability.TaskEvaluation) error {
if value.ID == "" || value.ProfileID == "" || value.BindingID == "" || value.BindingRevision < 1 || value.CapabilityID == "" || value.BenchmarkVersion == "" || value.ArtifactID == "" || value.ObservedAtUnixMillis < 1 {
return ErrInvalidTaskEvaluation
}
if value.Result != capability.EvaluationUnknown && value.Result != capability.EvaluationSupported && value.Result != capability.EvaluationUnsupported {
return ErrInvalidTaskEvaluation
}
for _, score := range []int64{value.QualityScoreMillis, value.ReliabilityScoreMillis, value.LatencyScoreMillis, value.OverallScoreMillis, value.CriticalFailureMillis} {
if score < 0 || score > 1000 {
return ErrInvalidTaskEvaluation
}
}
return nil
}
@@ -0,0 +1,35 @@
package domain
import (
"testing"
"git.cin.su/m/Assistant/contracts/artifact"
"git.cin.su/m/Assistant/contracts/capability"
)
func TestBuildTaskEvaluationAggregatesOnlyProfileCases(t *testing.T) {
profile := capability.TaskEvaluationProfile{ID: "research-v1", Revision: 1, CapabilityID: "creator-advisory", Stage: capability.TaskEvaluationGeneral, BenchmarkVersion: "research-v1", RequiredCaseIDs: []string{"claim", "uncertainty"}, QualityWeightMillis: 600, ReliabilityWeightMillis: 300, LatencyWeightMillis: 100}
value, err := BuildTaskEvaluation(profile, "evaluation-1", "binding-1", 2, "creator-advisory", []capability.TaskCaseObservation{
{CaseID: "claim", Result: capability.EvaluationSupported, QualityScoreMillis: 900, LatencyMicros: 400_000, EvidenceArtifactRef: "artifact-case-1"},
{CaseID: "uncertainty", Result: capability.EvaluationSupported, QualityScoreMillis: 800, LatencyMicros: 600_000, EvidenceArtifactRef: "artifact-case-2"},
}, artifact.ArtifactID("artifact-summary"), 10)
if err != nil || value.Result != capability.EvaluationSupported || value.OverallScoreMillis == 0 || value.ProfileID != profile.ID {
t.Fatalf("evaluation=%+v err=%v", value, err)
}
}
func TestBuildTaskEvaluationDoesNotPromoteUnknown(t *testing.T) {
profile := capability.TaskEvaluationProfile{ID: "coding-v1", Revision: 1, CapabilityID: "creator-advisory", Stage: capability.TaskEvaluationGeneral, BenchmarkVersion: "coding-v1", RequiredCaseIDs: []string{"compile"}, QualityWeightMillis: 700, ReliabilityWeightMillis: 200, LatencyWeightMillis: 100}
value, err := BuildTaskEvaluation(profile, "evaluation-2", "binding-1", 2, "creator-advisory", []capability.TaskCaseObservation{{CaseID: "compile", Result: capability.EvaluationUnknown, QualityScoreMillis: 0, LatencyMicros: 0, EvidenceArtifactRef: "artifact-case"}}, artifact.ArtifactID("artifact-summary"), 11)
if err != nil || value.Result != capability.EvaluationUnknown {
t.Fatalf("evaluation=%+v err=%v", value, err)
}
}
func TestBuildAdditionalProfileRequiresUnresolvedEvidenceAndPreservesHistory(t *testing.T) {
parent := capability.TaskEvaluationProfile{ID: "general-v1", Revision: 1, CapabilityID: "creator-advisory", Stage: capability.TaskEvaluationGeneral, BenchmarkVersion: "general-v1", RequiredCaseIDs: []string{"general"}, QualityWeightMillis: 700, ReliabilityWeightMillis: 200, LatencyWeightMillis: 100}
profile, err := BuildAdditionalTaskEvaluationProfile(parent, capability.TaskEvaluationExpansionRequest{ParentProfileID: parent.ID, CapabilityID: parent.CapabilityID, TriggerEvidenceRef: "experience-unsolved-1", NewCaseIDs: []string{"edge-case"}, BenchmarkVersion: "additional-v1"}, "additional-v1", 1)
if err != nil || profile.Stage != capability.TaskEvaluationAdditional || profile.ParentProfileID != parent.ID || profile.RequiredCaseIDs[0] != "edge-case" {
t.Fatalf("profile=%+v err=%v", profile, err)
}
}
+2
View File
@@ -18,6 +18,7 @@ type Repository interface {
SaveModelDescriptor(context.Context, capability.ModelDescriptor) error
SaveCognitiveBinding(context.Context, capability.CognitiveBinding) error
LoadCognitiveCandidates(context.Context, capability.CapabilityID, identity.PrincipalID) ([]capability.ResolvedCognitiveBinding, error)
LoadCognitiveCandidatesForProfile(context.Context, capability.CapabilityID, identity.PrincipalID, capability.TaskEvaluationProfileID) ([]capability.ResolvedCognitiveBinding, error)
LoadCognitiveCandidate(context.Context, capability.CapabilityBindingID, int64) (capability.ResolvedCognitiveBinding, error)
SaveCognitiveCanaryPromotion(context.Context, capability.CognitiveCanaryPromotion) error
SaveCognitiveActivationPromotion(context.Context, capability.CognitiveActivationPromotion) error
@@ -25,4 +26,5 @@ type Repository interface {
SaveCognitiveDiscovery(context.Context, capability.CognitiveDiscoveryRegistration) error
SaveEvaluatedCapability(context.Context, capability.EvaluatedCapability) error
SavePerformanceEvidence(context.Context, capability.PerformanceEvidence) error
SaveTaskEvaluation(context.Context, capability.TaskEvaluation) error
}