feat(capability): gate additional model evaluations
This commit is contained in:
@@ -2,6 +2,7 @@ package domain
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
|
||||
"git.cin.su/m/Assistant/contracts/artifact"
|
||||
"git.cin.su/m/Assistant/contracts/capability"
|
||||
@@ -9,6 +10,35 @@ import (
|
||||
|
||||
var ErrInvalidTaskEvaluation = errors.New("invalid task evaluation")
|
||||
|
||||
const automaticExpansionMinimumOverallScore int64 = 700
|
||||
|
||||
// AutomaticTaskEvaluationExpansion decides whether a completed general
|
||||
// evaluation has evidence that warrants a bounded additional profile. It is
|
||||
// deliberately conservative: one supported candidate above the quality
|
||||
// threshold suppresses expansion, while unknown/unsupported/low-scoring or
|
||||
// critically failing candidates keep the unresolved path explicit.
|
||||
func AutomaticTaskEvaluationExpansion(parent capability.TaskEvaluationProfile, parentGraphState string, evaluations []capability.TaskEvaluation, request capability.TaskEvaluationExpansionRequest, profileID capability.TaskEvaluationProfileID, revision int64) (capability.TaskEvaluationProfile, bool, error) {
|
||||
if parentGraphState != "COMPLETED" && parentGraphState != "PARTIAL" {
|
||||
return capability.TaskEvaluationProfile{}, false, nil
|
||||
}
|
||||
if len(evaluations) == 0 {
|
||||
return capability.TaskEvaluationProfile{}, false, nil
|
||||
}
|
||||
for _, evaluation := range evaluations {
|
||||
if evaluation.Result == capability.EvaluationSupported && evaluation.OverallScoreMillis >= automaticExpansionMinimumOverallScore && evaluation.CriticalFailureMillis == 0 {
|
||||
return capability.TaskEvaluationProfile{}, false, nil
|
||||
}
|
||||
}
|
||||
if request.TriggerEvidenceRef == "" || len(request.NewCaseIDs) == 0 {
|
||||
return capability.TaskEvaluationProfile{}, false, fmt.Errorf("unresolved task evaluation requires trigger evidence and additional cases")
|
||||
}
|
||||
profile, err := BuildAdditionalTaskEvaluationProfile(parent, request, profileID, revision)
|
||||
if err != nil {
|
||||
return capability.TaskEvaluationProfile{}, false, err
|
||||
}
|
||||
return profile, true, nil
|
||||
}
|
||||
|
||||
func BuildAdditionalTaskEvaluationProfile(parent capability.TaskEvaluationProfile, request capability.TaskEvaluationExpansionRequest, profileID capability.TaskEvaluationProfileID, revision int64) (capability.TaskEvaluationProfile, error) {
|
||||
if err := parent.Validate(); err != nil || parent.Stage != capability.TaskEvaluationGeneral || request.ParentProfileID != parent.ID || request.CapabilityID != parent.CapabilityID || request.TriggerEvidenceRef == "" || len(request.NewCaseIDs) == 0 || profileID == "" || revision < 1 || request.BenchmarkVersion == "" {
|
||||
return capability.TaskEvaluationProfile{}, ErrInvalidTaskEvaluation
|
||||
|
||||
@@ -33,3 +33,21 @@ func TestBuildAdditionalProfileRequiresUnresolvedEvidenceAndPreservesHistory(t *
|
||||
t.Fatalf("profile=%+v err=%v", profile, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAutomaticExpansionWaitsForGeneralGraphAndUnresolvedCandidates(t *testing.T) {
|
||||
parent := capability.TaskEvaluationProfile{ID: "general-v1", Revision: 1, CapabilityID: "creator-advisory", Stage: capability.TaskEvaluationGeneral, BenchmarkVersion: "general-v1", RequiredCaseIDs: []string{"general"}, QualityWeightMillis: 700, ReliabilityWeightMillis: 200, LatencyWeightMillis: 100}
|
||||
request := capability.TaskEvaluationExpansionRequest{ParentProfileID: parent.ID, CapabilityID: parent.CapabilityID, TriggerEvidenceRef: "outcome-unsolved", NewCaseIDs: []string{"edge-case"}, BenchmarkVersion: "additional-v1"}
|
||||
value, expanded, err := AutomaticTaskEvaluationExpansion(parent, "COMPLETED", []capability.TaskEvaluation{{ID: "evaluation-1", ProfileID: parent.ID, BindingID: "binding-1", BindingRevision: 1, CapabilityID: parent.CapabilityID, Result: capability.EvaluationUnknown, OverallScoreMillis: 0, CriticalFailureMillis: 1000, BenchmarkVersion: parent.BenchmarkVersion, ArtifactID: "artifact-1", ObservedAtUnixMillis: 1}}, request, "additional-v1", 1)
|
||||
if err != nil || !expanded || value.Stage != capability.TaskEvaluationAdditional || value.ParentProfileID != parent.ID {
|
||||
t.Fatalf("expected bounded expansion, profile=%+v expanded=%t err=%v", value, expanded, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAutomaticExpansionDoesNotRunWhenAQualifiedCandidateExists(t *testing.T) {
|
||||
parent := capability.TaskEvaluationProfile{ID: "general-v1", Revision: 1, CapabilityID: "creator-advisory", Stage: capability.TaskEvaluationGeneral, BenchmarkVersion: "general-v1", RequiredCaseIDs: []string{"general"}, QualityWeightMillis: 700, ReliabilityWeightMillis: 200, LatencyWeightMillis: 100}
|
||||
request := capability.TaskEvaluationExpansionRequest{ParentProfileID: parent.ID, CapabilityID: parent.CapabilityID, TriggerEvidenceRef: "outcome-unsolved", NewCaseIDs: []string{"edge-case"}, BenchmarkVersion: "additional-v1"}
|
||||
value, expanded, err := AutomaticTaskEvaluationExpansion(parent, "PARTIAL", []capability.TaskEvaluation{{ID: "evaluation-1", ProfileID: parent.ID, BindingID: "binding-1", BindingRevision: 1, CapabilityID: parent.CapabilityID, Result: capability.EvaluationSupported, OverallScoreMillis: 850, CriticalFailureMillis: 0, BenchmarkVersion: parent.BenchmarkVersion, ArtifactID: "artifact-1", ObservedAtUnixMillis: 1}}, request, "additional-v1", 1)
|
||||
if err != nil || expanded || value.ID != "" {
|
||||
t.Fatalf("qualified candidate must suppress expansion: profile=%+v expanded=%t err=%v", value, expanded, err)
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user