feat(capability): gate additional model evaluations
CI / test (push) Successful in 1m8s
CI / test (pull_request) Successful in 59s

This commit is contained in:
m
2026-08-23 13:56:16 +03:00
parent ab840878fe
commit be29ed5776
4 changed files with 73 additions and 2 deletions
@@ -2,6 +2,7 @@ package domain
import (
"errors"
"fmt"
"git.cin.su/m/Assistant/contracts/artifact"
"git.cin.su/m/Assistant/contracts/capability"
@@ -9,6 +10,35 @@ import (
var ErrInvalidTaskEvaluation = errors.New("invalid task evaluation")
const automaticExpansionMinimumOverallScore int64 = 700
// AutomaticTaskEvaluationExpansion decides whether a completed general
// evaluation has evidence that warrants a bounded additional profile. It is
// deliberately conservative: one supported candidate above the quality
// threshold suppresses expansion, while unknown/unsupported/low-scoring or
// critically failing candidates keep the unresolved path explicit.
func AutomaticTaskEvaluationExpansion(parent capability.TaskEvaluationProfile, parentGraphState string, evaluations []capability.TaskEvaluation, request capability.TaskEvaluationExpansionRequest, profileID capability.TaskEvaluationProfileID, revision int64) (capability.TaskEvaluationProfile, bool, error) {
if parentGraphState != "COMPLETED" && parentGraphState != "PARTIAL" {
return capability.TaskEvaluationProfile{}, false, nil
}
if len(evaluations) == 0 {
return capability.TaskEvaluationProfile{}, false, nil
}
for _, evaluation := range evaluations {
if evaluation.Result == capability.EvaluationSupported && evaluation.OverallScoreMillis >= automaticExpansionMinimumOverallScore && evaluation.CriticalFailureMillis == 0 {
return capability.TaskEvaluationProfile{}, false, nil
}
}
if request.TriggerEvidenceRef == "" || len(request.NewCaseIDs) == 0 {
return capability.TaskEvaluationProfile{}, false, fmt.Errorf("unresolved task evaluation requires trigger evidence and additional cases")
}
profile, err := BuildAdditionalTaskEvaluationProfile(parent, request, profileID, revision)
if err != nil {
return capability.TaskEvaluationProfile{}, false, err
}
return profile, true, nil
}
func BuildAdditionalTaskEvaluationProfile(parent capability.TaskEvaluationProfile, request capability.TaskEvaluationExpansionRequest, profileID capability.TaskEvaluationProfileID, revision int64) (capability.TaskEvaluationProfile, error) {
if err := parent.Validate(); err != nil || parent.Stage != capability.TaskEvaluationGeneral || request.ParentProfileID != parent.ID || request.CapabilityID != parent.CapabilityID || request.TriggerEvidenceRef == "" || len(request.NewCaseIDs) == 0 || profileID == "" || revision < 1 || request.BenchmarkVersion == "" {
return capability.TaskEvaluationProfile{}, ErrInvalidTaskEvaluation
@@ -33,3 +33,21 @@ func TestBuildAdditionalProfileRequiresUnresolvedEvidenceAndPreservesHistory(t *
t.Fatalf("profile=%+v err=%v", profile, err)
}
}
func TestAutomaticExpansionWaitsForGeneralGraphAndUnresolvedCandidates(t *testing.T) {
parent := capability.TaskEvaluationProfile{ID: "general-v1", Revision: 1, CapabilityID: "creator-advisory", Stage: capability.TaskEvaluationGeneral, BenchmarkVersion: "general-v1", RequiredCaseIDs: []string{"general"}, QualityWeightMillis: 700, ReliabilityWeightMillis: 200, LatencyWeightMillis: 100}
request := capability.TaskEvaluationExpansionRequest{ParentProfileID: parent.ID, CapabilityID: parent.CapabilityID, TriggerEvidenceRef: "outcome-unsolved", NewCaseIDs: []string{"edge-case"}, BenchmarkVersion: "additional-v1"}
value, expanded, err := AutomaticTaskEvaluationExpansion(parent, "COMPLETED", []capability.TaskEvaluation{{ID: "evaluation-1", ProfileID: parent.ID, BindingID: "binding-1", BindingRevision: 1, CapabilityID: parent.CapabilityID, Result: capability.EvaluationUnknown, OverallScoreMillis: 0, CriticalFailureMillis: 1000, BenchmarkVersion: parent.BenchmarkVersion, ArtifactID: "artifact-1", ObservedAtUnixMillis: 1}}, request, "additional-v1", 1)
if err != nil || !expanded || value.Stage != capability.TaskEvaluationAdditional || value.ParentProfileID != parent.ID {
t.Fatalf("expected bounded expansion, profile=%+v expanded=%t err=%v", value, expanded, err)
}
}
func TestAutomaticExpansionDoesNotRunWhenAQualifiedCandidateExists(t *testing.T) {
parent := capability.TaskEvaluationProfile{ID: "general-v1", Revision: 1, CapabilityID: "creator-advisory", Stage: capability.TaskEvaluationGeneral, BenchmarkVersion: "general-v1", RequiredCaseIDs: []string{"general"}, QualityWeightMillis: 700, ReliabilityWeightMillis: 200, LatencyWeightMillis: 100}
request := capability.TaskEvaluationExpansionRequest{ParentProfileID: parent.ID, CapabilityID: parent.CapabilityID, TriggerEvidenceRef: "outcome-unsolved", NewCaseIDs: []string{"edge-case"}, BenchmarkVersion: "additional-v1"}
value, expanded, err := AutomaticTaskEvaluationExpansion(parent, "PARTIAL", []capability.TaskEvaluation{{ID: "evaluation-1", ProfileID: parent.ID, BindingID: "binding-1", BindingRevision: 1, CapabilityID: parent.CapabilityID, Result: capability.EvaluationSupported, OverallScoreMillis: 850, CriticalFailureMillis: 0, BenchmarkVersion: parent.BenchmarkVersion, ArtifactID: "artifact-1", ObservedAtUnixMillis: 1}}, request, "additional-v1", 1)
if err != nil || expanded || value.ID != "" {
t.Fatalf("qualified candidate must suppress expansion: profile=%+v expanded=%t err=%v", value, expanded, err)
}
}