feat: extend task verifier and unresolved case builder
This commit is contained in:
@@ -3,6 +3,7 @@ package domain
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"git.cin.su/m/Assistant/contracts/artifact"
|
||||
"git.cin.su/m/Assistant/contracts/capability"
|
||||
@@ -10,6 +11,28 @@ import (
|
||||
|
||||
var ErrInvalidTaskEvaluation = errors.New("invalid task evaluation")
|
||||
|
||||
type UnresolvedTaskEvidence struct {
|
||||
ID string
|
||||
OriginalPrompt string
|
||||
FailureSummary string
|
||||
EvidenceArtifactRef string
|
||||
}
|
||||
|
||||
// BuildAdditionalTaskEvaluationCases creates two bounded probes from one
|
||||
// unresolved outcome. It creates prompts only; quality still requires a
|
||||
// verification spec and independent verifier evidence.
|
||||
func BuildAdditionalTaskEvaluationCases(value UnresolvedTaskEvidence) ([]capability.TaskEvaluationCase, error) {
|
||||
if strings.TrimSpace(value.ID) == "" || strings.TrimSpace(value.OriginalPrompt) == "" || strings.TrimSpace(value.FailureSummary) == "" || strings.TrimSpace(value.EvidenceArtifactRef) == "" {
|
||||
return nil, ErrInvalidTaskEvaluation
|
||||
}
|
||||
base := strings.TrimSpace(value.OriginalPrompt)
|
||||
failure := strings.TrimSpace(value.FailureSummary)
|
||||
return []capability.TaskEvaluationCase{
|
||||
{CaseID: value.ID + ":reproduce", Prompt: base, EvidenceArtifactRef: value.EvidenceArtifactRef},
|
||||
{CaseID: value.ID + ":address-failure", Prompt: base + "\n\nAddress this observed unresolved condition without assuming it is true: " + failure, EvidenceArtifactRef: value.EvidenceArtifactRef},
|
||||
}, nil
|
||||
}
|
||||
|
||||
const automaticExpansionMinimumOverallScore int64 = 700
|
||||
|
||||
// AutomaticTaskEvaluationExpansion decides whether a completed general
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package domain
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.cin.su/m/Assistant/contracts/artifact"
|
||||
@@ -59,3 +60,10 @@ func TestAutomaticExpansionDoesNotRunWhenAQualifiedCandidateExists(t *testing.T)
|
||||
t.Fatalf("qualified candidate must suppress expansion: profile=%+v expanded=%t err=%v", value, expanded, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildAdditionalTaskEvaluationCasesFromUnresolvedEvidence(t *testing.T) {
|
||||
cases, err := BuildAdditionalTaskEvaluationCases(UnresolvedTaskEvidence{ID: "outcome-1", OriginalPrompt: "compute answer", FailureSummary: "the answer was not verified", EvidenceArtifactRef: "artifact-outcome-1"})
|
||||
if err != nil || len(cases) != 2 || cases[0].CaseID != "outcome-1:reproduce" || !strings.Contains(cases[1].Prompt, "not verified") {
|
||||
t.Fatalf("unexpected generated cases: cases=%+v err=%v", cases, err)
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user