feat(capability): require verifier evidence for quality #197
@@ -15,7 +15,7 @@ func TestTaskEvaluationExecutorInvokesAggregatesAndCallbacks(t *testing.T) {
|
||||
profile := capability.TaskEvaluationProfile{ID: "profile-1", Revision: 1, CapabilityID: "capability-1", Stage: capability.TaskEvaluationGeneral, BenchmarkVersion: "general-v1", RequiredCaseIDs: []string{"case-1"}, QualityWeightMillis: 600, ReliabilityWeightMillis: 300, LatencyWeightMillis: 100}
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.URL.Path == "/v1/task-evaluations/execute" {
|
||||
_ = json.NewEncoder(w).Encode(capability.TaskEvaluationExecutionResponse{ProfileID: profile.ID, BindingID: "binding-1", BindingRevision: 3, Observations: []capability.TaskCaseObservation{{CaseID: "case-1", Result: capability.EvaluationSupported, EvidenceArtifactRef: "artifact-1"}}, ObservedAtUnixMillis: 10})
|
||||
_ = json.NewEncoder(w).Encode(capability.TaskEvaluationExecutionResponse{ProfileID: profile.ID, BindingID: "binding-1", BindingRevision: 3, Observations: []capability.TaskCaseObservation{{CaseID: "case-1", Result: capability.EvaluationSupported, QualityScoreMillis: 800, EvidenceArtifactRef: "artifact-1", VerificationEvidenceRef: "verify-1", VerifierIdentity: "verifier-1"}}, ObservedAtUnixMillis: 10})
|
||||
return
|
||||
}
|
||||
if r.URL.Path == "/v1/work-graphs/task-evaluations/callback" {
|
||||
|
||||
@@ -80,6 +80,10 @@ type TaskCaseObservation struct {
|
||||
LatencyMicros int64
|
||||
CriticalFailure bool
|
||||
EvidenceArtifactRef string
|
||||
// VerificationEvidenceRef and VerifierIdentity are required whenever an
|
||||
// observation claims semantic support or a non-zero quality score.
|
||||
VerificationEvidenceRef string
|
||||
VerifierIdentity string
|
||||
}
|
||||
|
||||
// TaskEvaluation is durable evidence for one model/binding revision under one
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
{
|
||||
"task_file": "dev/tasks/DEV-000219-reachability-not-quality.json",
|
||||
"task_id": "DEV-000219",
|
||||
"task_file": "dev/tasks/DEV-000220-verifier-evidence-boundary.json",
|
||||
"task_id": "DEV-000220",
|
||||
"task_revision": 1
|
||||
}
|
||||
|
||||
@@ -0,0 +1,23 @@
|
||||
{
|
||||
"task_id": "DEV-000220",
|
||||
"task_revision": 1,
|
||||
"architecture_revision": "ADR-0018",
|
||||
"base_repo_revision": "66ed523248ea6cf0a41383e2206d9a70eeaafda8",
|
||||
"risk_tier": "E",
|
||||
"intent": "Require explicit independent verifier identity and evidence before task evaluation can claim semantic support or non-zero quality.",
|
||||
"requirement_ids": ["independent verifier evidence", "no false quality", "typed model qualification"],
|
||||
"allowed_change_scope": {"modules": ["capability", "agent-worker"], "paths": ["contracts/capability/task_evaluation.go", "modules/capability/domain/task_evaluation.go", "modules/capability/domain/task_evaluation_test.go", "cmd/agent-worker/task_evaluation_executor_test.go", "dev/ACTIVE_TASK.json", "dev/tasks/DEV-000220-verifier-evidence-boundary.json"]},
|
||||
"forbidden_change_scope": {"paths": ["architecture/**", "security/**", "database/**", "deploy/**", "cmd/cognitive-worker/**", "LLM-local/**"]},
|
||||
"allowed_architecture_delta": {"new_cross_module_dependencies": 0, "new_canonical_owners": 0, "new_privileges": 0, "new_third_party_dependencies": 0, "new_database_owner_roles": 0},
|
||||
"contracts": {"consume": ["TaskCaseObservation", "TaskEvaluation", "existing Work Graph callback"], "may_change": ["typed observation fields for verifier evidence"]},
|
||||
"semantic_owner": "capability owns qualification evidence; verifier remains independent of provider invocation",
|
||||
"canonical_state": ["existing capability task-evaluation evidence"],
|
||||
"state_transitions": ["reachability UNKNOWN -> separately verified SUPPORTED only with evidence"],
|
||||
"side_effects_and_privileges": ["no provider invocation or external mutation"],
|
||||
"persistence_and_migrations": ["existing task evaluation persistence is unchanged"],
|
||||
"failure_retry_restart": ["missing verifier fields fail closed; unknown reachability remains non-promotable"],
|
||||
"assumptions": ["a later verifier runtime will produce VerificationEvidenceRef and VerifierIdentity"],
|
||||
"acceptance_scenarios": ["supported without verifier is rejected", "unknown with zero quality remains valid", "verified supported observation aggregates normally", "no hardware/provider name bypasses the boundary"],
|
||||
"required_tests": ["go test", "go vet", "preflight", "check-diff", "check-architecture", "independent", "git diff --check", "CI", "production public health check"],
|
||||
"terminal_gap_states": ["BLOCKED_BY_MISSING_CAPABILITY", "BLOCKED_BY_SPEC_GAP"]
|
||||
}
|
||||
@@ -85,6 +85,12 @@ func BuildTaskEvaluation(profile capability.TaskEvaluationProfile, evaluationID
|
||||
if _, ok := seen[observation.CaseID]; ok || observation.EvidenceArtifactRef == "" || observation.QualityScoreMillis < 0 || observation.QualityScoreMillis > 1000 || observation.LatencyMicros < 0 {
|
||||
return capability.TaskEvaluation{}, ErrInvalidTaskEvaluation
|
||||
}
|
||||
if observation.Result == capability.EvaluationSupported && (observation.VerificationEvidenceRef == "" || observation.VerifierIdentity == "") {
|
||||
return capability.TaskEvaluation{}, ErrInvalidTaskEvaluation
|
||||
}
|
||||
if observation.Result == capability.EvaluationUnknown && observation.QualityScoreMillis != 0 {
|
||||
return capability.TaskEvaluation{}, ErrInvalidTaskEvaluation
|
||||
}
|
||||
seen[observation.CaseID] = struct{}{}
|
||||
qualityTotal += observation.QualityScoreMillis
|
||||
latencyTotal += observation.LatencyMicros
|
||||
|
||||
@@ -10,14 +10,22 @@ import (
|
||||
func TestBuildTaskEvaluationAggregatesOnlyProfileCases(t *testing.T) {
|
||||
profile := capability.TaskEvaluationProfile{ID: "research-v1", Revision: 1, CapabilityID: "creator-advisory", Stage: capability.TaskEvaluationGeneral, BenchmarkVersion: "research-v1", RequiredCaseIDs: []string{"claim", "uncertainty"}, QualityWeightMillis: 600, ReliabilityWeightMillis: 300, LatencyWeightMillis: 100}
|
||||
value, err := BuildTaskEvaluation(profile, "evaluation-1", "binding-1", 2, "creator-advisory", []capability.TaskCaseObservation{
|
||||
{CaseID: "claim", Result: capability.EvaluationSupported, QualityScoreMillis: 900, LatencyMicros: 400_000, EvidenceArtifactRef: "artifact-case-1"},
|
||||
{CaseID: "uncertainty", Result: capability.EvaluationSupported, QualityScoreMillis: 800, LatencyMicros: 600_000, EvidenceArtifactRef: "artifact-case-2"},
|
||||
{CaseID: "claim", Result: capability.EvaluationSupported, QualityScoreMillis: 900, LatencyMicros: 400_000, EvidenceArtifactRef: "artifact-case-1", VerificationEvidenceRef: "verify-case-1", VerifierIdentity: "verifier-1"},
|
||||
{CaseID: "uncertainty", Result: capability.EvaluationSupported, QualityScoreMillis: 800, LatencyMicros: 600_000, EvidenceArtifactRef: "artifact-case-2", VerificationEvidenceRef: "verify-case-2", VerifierIdentity: "verifier-1"},
|
||||
}, artifact.ArtifactID("artifact-summary"), 10)
|
||||
if err != nil || value.Result != capability.EvaluationSupported || value.OverallScoreMillis == 0 || value.ProfileID != profile.ID {
|
||||
t.Fatalf("evaluation=%+v err=%v", value, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildTaskEvaluationRejectsSupportedWithoutIndependentVerifier(t *testing.T) {
|
||||
profile := capability.TaskEvaluationProfile{ID: "quality-v1", Revision: 1, CapabilityID: "creator-advisory", Stage: capability.TaskEvaluationGeneral, BenchmarkVersion: "quality-v1", RequiredCaseIDs: []string{"case"}, QualityWeightMillis: 700, ReliabilityWeightMillis: 200, LatencyWeightMillis: 100}
|
||||
_, err := BuildTaskEvaluation(profile, "evaluation-unverified", "binding-1", 1, profile.CapabilityID, []capability.TaskCaseObservation{{CaseID: "case", Result: capability.EvaluationSupported, QualityScoreMillis: 900, EvidenceArtifactRef: "artifact-case"}}, artifact.ArtifactID("artifact-summary"), 1)
|
||||
if err == nil {
|
||||
t.Fatal("supported observation without verifier evidence was accepted")
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildTaskEvaluationDoesNotPromoteUnknown(t *testing.T) {
|
||||
profile := capability.TaskEvaluationProfile{ID: "coding-v1", Revision: 1, CapabilityID: "creator-advisory", Stage: capability.TaskEvaluationGeneral, BenchmarkVersion: "coding-v1", RequiredCaseIDs: []string{"compile"}, QualityWeightMillis: 700, ReliabilityWeightMillis: 200, LatencyWeightMillis: 100}
|
||||
value, err := BuildTaskEvaluation(profile, "evaluation-2", "binding-1", 2, "creator-advisory", []capability.TaskCaseObservation{{CaseID: "compile", Result: capability.EvaluationUnknown, QualityScoreMillis: 0, LatencyMicros: 0, EvidenceArtifactRef: "artifact-case"}}, artifact.ArtifactID("artifact-summary"), 11)
|
||||
|
||||
Reference in New Issue
Block a user