70 lines
6.7 KiB
Go
70 lines
6.7 KiB
Go
package domain
|
|
|
|
import (
|
|
"strings"
|
|
"testing"
|
|
|
|
"git.cin.su/m/Assistant/contracts/artifact"
|
|
"git.cin.su/m/Assistant/contracts/capability"
|
|
)
|
|
|
|
func TestBuildTaskEvaluationAggregatesOnlyProfileCases(t *testing.T) {
|
|
profile := capability.TaskEvaluationProfile{ID: "research-v1", Revision: 1, CapabilityID: "creator-advisory", Stage: capability.TaskEvaluationGeneral, BenchmarkVersion: "research-v1", RequiredCaseIDs: []string{"claim", "uncertainty"}, QualityWeightMillis: 600, ReliabilityWeightMillis: 300, LatencyWeightMillis: 100}
|
|
value, err := BuildTaskEvaluation(profile, "evaluation-1", "binding-1", 2, "creator-advisory", []capability.TaskCaseObservation{
|
|
{CaseID: "claim", Result: capability.EvaluationSupported, QualityScoreMillis: 900, LatencyMicros: 400_000, EvidenceArtifactRef: "artifact-case-1", VerificationEvidenceRef: "verify-case-1", VerifierIdentity: "verifier-1"},
|
|
{CaseID: "uncertainty", Result: capability.EvaluationSupported, QualityScoreMillis: 800, LatencyMicros: 600_000, EvidenceArtifactRef: "artifact-case-2", VerificationEvidenceRef: "verify-case-2", VerifierIdentity: "verifier-1"},
|
|
}, artifact.ArtifactID("artifact-summary"), 10)
|
|
if err != nil || value.Result != capability.EvaluationSupported || value.OverallScoreMillis == 0 || value.ProfileID != profile.ID {
|
|
t.Fatalf("evaluation=%+v err=%v", value, err)
|
|
}
|
|
}
|
|
|
|
func TestBuildTaskEvaluationRejectsSupportedWithoutIndependentVerifier(t *testing.T) {
|
|
profile := capability.TaskEvaluationProfile{ID: "quality-v1", Revision: 1, CapabilityID: "creator-advisory", Stage: capability.TaskEvaluationGeneral, BenchmarkVersion: "quality-v1", RequiredCaseIDs: []string{"case"}, QualityWeightMillis: 700, ReliabilityWeightMillis: 200, LatencyWeightMillis: 100}
|
|
_, err := BuildTaskEvaluation(profile, "evaluation-unverified", "binding-1", 1, profile.CapabilityID, []capability.TaskCaseObservation{{CaseID: "case", Result: capability.EvaluationSupported, QualityScoreMillis: 900, EvidenceArtifactRef: "artifact-case"}}, artifact.ArtifactID("artifact-summary"), 1)
|
|
if err == nil {
|
|
t.Fatal("supported observation without verifier evidence was accepted")
|
|
}
|
|
}
|
|
|
|
func TestBuildTaskEvaluationDoesNotPromoteUnknown(t *testing.T) {
|
|
profile := capability.TaskEvaluationProfile{ID: "coding-v1", Revision: 1, CapabilityID: "creator-advisory", Stage: capability.TaskEvaluationGeneral, BenchmarkVersion: "coding-v1", RequiredCaseIDs: []string{"compile"}, QualityWeightMillis: 700, ReliabilityWeightMillis: 200, LatencyWeightMillis: 100}
|
|
value, err := BuildTaskEvaluation(profile, "evaluation-2", "binding-1", 2, "creator-advisory", []capability.TaskCaseObservation{{CaseID: "compile", Result: capability.EvaluationUnknown, QualityScoreMillis: 0, LatencyMicros: 0, EvidenceArtifactRef: "artifact-case"}}, artifact.ArtifactID("artifact-summary"), 11)
|
|
if err != nil || value.Result != capability.EvaluationUnknown {
|
|
t.Fatalf("evaluation=%+v err=%v", value, err)
|
|
}
|
|
}
|
|
|
|
func TestBuildAdditionalProfileRequiresUnresolvedEvidenceAndPreservesHistory(t *testing.T) {
|
|
parent := capability.TaskEvaluationProfile{ID: "general-v1", Revision: 1, CapabilityID: "creator-advisory", Stage: capability.TaskEvaluationGeneral, BenchmarkVersion: "general-v1", RequiredCaseIDs: []string{"general"}, QualityWeightMillis: 700, ReliabilityWeightMillis: 200, LatencyWeightMillis: 100}
|
|
profile, err := BuildAdditionalTaskEvaluationProfile(parent, capability.TaskEvaluationExpansionRequest{ParentProfileID: parent.ID, CapabilityID: parent.CapabilityID, TriggerEvidenceRef: "experience-unsolved-1", NewCaseIDs: []string{"edge-case"}, BenchmarkVersion: "additional-v1"}, "additional-v1", 1)
|
|
if err != nil || profile.Stage != capability.TaskEvaluationAdditional || profile.ParentProfileID != parent.ID || profile.RequiredCaseIDs[0] != "edge-case" {
|
|
t.Fatalf("profile=%+v err=%v", profile, err)
|
|
}
|
|
}
|
|
|
|
func TestAutomaticExpansionWaitsForGeneralGraphAndUnresolvedCandidates(t *testing.T) {
|
|
parent := capability.TaskEvaluationProfile{ID: "general-v1", Revision: 1, CapabilityID: "creator-advisory", Stage: capability.TaskEvaluationGeneral, BenchmarkVersion: "general-v1", RequiredCaseIDs: []string{"general"}, QualityWeightMillis: 700, ReliabilityWeightMillis: 200, LatencyWeightMillis: 100}
|
|
request := capability.TaskEvaluationExpansionRequest{ParentProfileID: parent.ID, CapabilityID: parent.CapabilityID, TriggerEvidenceRef: "outcome-unsolved", NewCaseIDs: []string{"edge-case"}, BenchmarkVersion: "additional-v1"}
|
|
value, expanded, err := AutomaticTaskEvaluationExpansion(parent, "COMPLETED", []capability.TaskEvaluation{{ID: "evaluation-1", ProfileID: parent.ID, BindingID: "binding-1", BindingRevision: 1, CapabilityID: parent.CapabilityID, Result: capability.EvaluationUnknown, OverallScoreMillis: 0, CriticalFailureMillis: 1000, BenchmarkVersion: parent.BenchmarkVersion, ArtifactID: "artifact-1", ObservedAtUnixMillis: 1}}, request, "additional-v1", 1)
|
|
if err != nil || !expanded || value.Stage != capability.TaskEvaluationAdditional || value.ParentProfileID != parent.ID {
|
|
t.Fatalf("expected bounded expansion, profile=%+v expanded=%t err=%v", value, expanded, err)
|
|
}
|
|
}
|
|
|
|
func TestAutomaticExpansionDoesNotRunWhenAQualifiedCandidateExists(t *testing.T) {
|
|
parent := capability.TaskEvaluationProfile{ID: "general-v1", Revision: 1, CapabilityID: "creator-advisory", Stage: capability.TaskEvaluationGeneral, BenchmarkVersion: "general-v1", RequiredCaseIDs: []string{"general"}, QualityWeightMillis: 700, ReliabilityWeightMillis: 200, LatencyWeightMillis: 100}
|
|
request := capability.TaskEvaluationExpansionRequest{ParentProfileID: parent.ID, CapabilityID: parent.CapabilityID, TriggerEvidenceRef: "outcome-unsolved", NewCaseIDs: []string{"edge-case"}, BenchmarkVersion: "additional-v1"}
|
|
value, expanded, err := AutomaticTaskEvaluationExpansion(parent, "PARTIAL", []capability.TaskEvaluation{{ID: "evaluation-1", ProfileID: parent.ID, BindingID: "binding-1", BindingRevision: 1, CapabilityID: parent.CapabilityID, Result: capability.EvaluationSupported, OverallScoreMillis: 850, CriticalFailureMillis: 0, BenchmarkVersion: parent.BenchmarkVersion, ArtifactID: "artifact-1", ObservedAtUnixMillis: 1}}, request, "additional-v1", 1)
|
|
if err != nil || expanded || value.ID != "" {
|
|
t.Fatalf("qualified candidate must suppress expansion: profile=%+v expanded=%t err=%v", value, expanded, err)
|
|
}
|
|
}
|
|
|
|
func TestBuildAdditionalTaskEvaluationCasesFromUnresolvedEvidence(t *testing.T) {
|
|
cases, err := BuildAdditionalTaskEvaluationCases(UnresolvedTaskEvidence{ID: "outcome-1", OriginalPrompt: "compute answer", FailureSummary: "the answer was not verified", EvidenceArtifactRef: "artifact-outcome-1"})
|
|
if err != nil || len(cases) != 2 || cases[0].CaseID != "outcome-1:reproduce" || !strings.Contains(cases[1].Prompt, "not verified") {
|
|
t.Fatalf("unexpected generated cases: cases=%+v err=%v", cases, err)
|
|
}
|
|
}
|