feat(capability): require verifier evidence for quality #197

Merged
m merged 1 commits from codex/wp-220-verifier-boundary into main 2026-08-23 11:10:03 +00:00
6 changed files with 46 additions and 5 deletions
@@ -15,7 +15,7 @@ func TestTaskEvaluationExecutorInvokesAggregatesAndCallbacks(t *testing.T) {
profile := capability.TaskEvaluationProfile{ID: "profile-1", Revision: 1, CapabilityID: "capability-1", Stage: capability.TaskEvaluationGeneral, BenchmarkVersion: "general-v1", RequiredCaseIDs: []string{"case-1"}, QualityWeightMillis: 600, ReliabilityWeightMillis: 300, LatencyWeightMillis: 100}
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path == "/v1/task-evaluations/execute" {
_ = json.NewEncoder(w).Encode(capability.TaskEvaluationExecutionResponse{ProfileID: profile.ID, BindingID: "binding-1", BindingRevision: 3, Observations: []capability.TaskCaseObservation{{CaseID: "case-1", Result: capability.EvaluationSupported, EvidenceArtifactRef: "artifact-1"}}, ObservedAtUnixMillis: 10})
_ = json.NewEncoder(w).Encode(capability.TaskEvaluationExecutionResponse{ProfileID: profile.ID, BindingID: "binding-1", BindingRevision: 3, Observations: []capability.TaskCaseObservation{{CaseID: "case-1", Result: capability.EvaluationSupported, QualityScoreMillis: 800, EvidenceArtifactRef: "artifact-1", VerificationEvidenceRef: "verify-1", VerifierIdentity: "verifier-1"}}, ObservedAtUnixMillis: 10})
return
}
if r.URL.Path == "/v1/work-graphs/task-evaluations/callback" {
+4
View File
@@ -80,6 +80,10 @@ type TaskCaseObservation struct {
LatencyMicros int64
CriticalFailure bool
EvidenceArtifactRef string
// VerificationEvidenceRef and VerifierIdentity are required whenever an
// observation claims semantic support or a non-zero quality score.
VerificationEvidenceRef string
VerifierIdentity string
}
// TaskEvaluation is durable evidence for one model/binding revision under one
+2 -2
View File
@@ -1,5 +1,5 @@
{
"task_file": "dev/tasks/DEV-000219-reachability-not-quality.json",
"task_id": "DEV-000219",
"task_file": "dev/tasks/DEV-000220-verifier-evidence-boundary.json",
"task_id": "DEV-000220",
"task_revision": 1
}
@@ -0,0 +1,23 @@
{
"task_id": "DEV-000220",
"task_revision": 1,
"architecture_revision": "ADR-0018",
"base_repo_revision": "66ed523248ea6cf0a41383e2206d9a70eeaafda8",
"risk_tier": "E",
"intent": "Require explicit independent verifier identity and evidence before task evaluation can claim semantic support or non-zero quality.",
"requirement_ids": ["independent verifier evidence", "no false quality", "typed model qualification"],
"allowed_change_scope": {"modules": ["capability", "agent-worker"], "paths": ["contracts/capability/task_evaluation.go", "modules/capability/domain/task_evaluation.go", "modules/capability/domain/task_evaluation_test.go", "cmd/agent-worker/task_evaluation_executor_test.go", "dev/ACTIVE_TASK.json", "dev/tasks/DEV-000220-verifier-evidence-boundary.json"]},
"forbidden_change_scope": {"paths": ["architecture/**", "security/**", "database/**", "deploy/**", "cmd/cognitive-worker/**", "LLM-local/**"]},
"allowed_architecture_delta": {"new_cross_module_dependencies": 0, "new_canonical_owners": 0, "new_privileges": 0, "new_third_party_dependencies": 0, "new_database_owner_roles": 0},
"contracts": {"consume": ["TaskCaseObservation", "TaskEvaluation", "existing Work Graph callback"], "may_change": ["typed observation fields for verifier evidence"]},
"semantic_owner": "capability owns qualification evidence; verifier remains independent of provider invocation",
"canonical_state": ["existing capability task-evaluation evidence"],
"state_transitions": ["reachability UNKNOWN -> separately verified SUPPORTED only with evidence"],
"side_effects_and_privileges": ["no provider invocation or external mutation"],
"persistence_and_migrations": ["existing task evaluation persistence is unchanged"],
"failure_retry_restart": ["missing verifier fields fail closed; unknown reachability remains non-promotable"],
"assumptions": ["a later verifier runtime will produce VerificationEvidenceRef and VerifierIdentity"],
"acceptance_scenarios": ["supported without verifier is rejected", "unknown with zero quality remains valid", "verified supported observation aggregates normally", "no hardware/provider name bypasses the boundary"],
"required_tests": ["go test", "go vet", "preflight", "check-diff", "check-architecture", "independent", "git diff --check", "CI", "production public health check"],
"terminal_gap_states": ["BLOCKED_BY_MISSING_CAPABILITY", "BLOCKED_BY_SPEC_GAP"]
}
@@ -85,6 +85,12 @@ func BuildTaskEvaluation(profile capability.TaskEvaluationProfile, evaluationID
if _, ok := seen[observation.CaseID]; ok || observation.EvidenceArtifactRef == "" || observation.QualityScoreMillis < 0 || observation.QualityScoreMillis > 1000 || observation.LatencyMicros < 0 {
return capability.TaskEvaluation{}, ErrInvalidTaskEvaluation
}
if observation.Result == capability.EvaluationSupported && (observation.VerificationEvidenceRef == "" || observation.VerifierIdentity == "") {
return capability.TaskEvaluation{}, ErrInvalidTaskEvaluation
}
if observation.Result == capability.EvaluationUnknown && observation.QualityScoreMillis != 0 {
return capability.TaskEvaluation{}, ErrInvalidTaskEvaluation
}
seen[observation.CaseID] = struct{}{}
qualityTotal += observation.QualityScoreMillis
latencyTotal += observation.LatencyMicros
@@ -10,14 +10,22 @@ import (
func TestBuildTaskEvaluationAggregatesOnlyProfileCases(t *testing.T) {
profile := capability.TaskEvaluationProfile{ID: "research-v1", Revision: 1, CapabilityID: "creator-advisory", Stage: capability.TaskEvaluationGeneral, BenchmarkVersion: "research-v1", RequiredCaseIDs: []string{"claim", "uncertainty"}, QualityWeightMillis: 600, ReliabilityWeightMillis: 300, LatencyWeightMillis: 100}
value, err := BuildTaskEvaluation(profile, "evaluation-1", "binding-1", 2, "creator-advisory", []capability.TaskCaseObservation{
{CaseID: "claim", Result: capability.EvaluationSupported, QualityScoreMillis: 900, LatencyMicros: 400_000, EvidenceArtifactRef: "artifact-case-1"},
{CaseID: "uncertainty", Result: capability.EvaluationSupported, QualityScoreMillis: 800, LatencyMicros: 600_000, EvidenceArtifactRef: "artifact-case-2"},
{CaseID: "claim", Result: capability.EvaluationSupported, QualityScoreMillis: 900, LatencyMicros: 400_000, EvidenceArtifactRef: "artifact-case-1", VerificationEvidenceRef: "verify-case-1", VerifierIdentity: "verifier-1"},
{CaseID: "uncertainty", Result: capability.EvaluationSupported, QualityScoreMillis: 800, LatencyMicros: 600_000, EvidenceArtifactRef: "artifact-case-2", VerificationEvidenceRef: "verify-case-2", VerifierIdentity: "verifier-1"},
}, artifact.ArtifactID("artifact-summary"), 10)
if err != nil || value.Result != capability.EvaluationSupported || value.OverallScoreMillis == 0 || value.ProfileID != profile.ID {
t.Fatalf("evaluation=%+v err=%v", value, err)
}
}
func TestBuildTaskEvaluationRejectsSupportedWithoutIndependentVerifier(t *testing.T) {
profile := capability.TaskEvaluationProfile{ID: "quality-v1", Revision: 1, CapabilityID: "creator-advisory", Stage: capability.TaskEvaluationGeneral, BenchmarkVersion: "quality-v1", RequiredCaseIDs: []string{"case"}, QualityWeightMillis: 700, ReliabilityWeightMillis: 200, LatencyWeightMillis: 100}
_, err := BuildTaskEvaluation(profile, "evaluation-unverified", "binding-1", 1, profile.CapabilityID, []capability.TaskCaseObservation{{CaseID: "case", Result: capability.EvaluationSupported, QualityScoreMillis: 900, EvidenceArtifactRef: "artifact-case"}}, artifact.ArtifactID("artifact-summary"), 1)
if err == nil {
t.Fatal("supported observation without verifier evidence was accepted")
}
}
func TestBuildTaskEvaluationDoesNotPromoteUnknown(t *testing.T) {
profile := capability.TaskEvaluationProfile{ID: "coding-v1", Revision: 1, CapabilityID: "creator-advisory", Stage: capability.TaskEvaluationGeneral, BenchmarkVersion: "coding-v1", RequiredCaseIDs: []string{"compile"}, QualityWeightMillis: 700, ReliabilityWeightMillis: 200, LatencyWeightMillis: 100}
value, err := BuildTaskEvaluation(profile, "evaluation-2", "binding-1", 2, "creator-advisory", []capability.TaskCaseObservation{{CaseID: "compile", Result: capability.EvaluationUnknown, QualityScoreMillis: 0, LatencyMicros: 0, EvidenceArtifactRef: "artifact-case"}}, artifact.ArtifactID("artifact-summary"), 11)