diff --git a/cmd/agent-worker/task_evaluation_executor_test.go b/cmd/agent-worker/task_evaluation_executor_test.go index b4c0d7b..48c4e81 100644 --- a/cmd/agent-worker/task_evaluation_executor_test.go +++ b/cmd/agent-worker/task_evaluation_executor_test.go @@ -15,7 +15,7 @@ func TestTaskEvaluationExecutorInvokesAggregatesAndCallbacks(t *testing.T) { profile := capability.TaskEvaluationProfile{ID: "profile-1", Revision: 1, CapabilityID: "capability-1", Stage: capability.TaskEvaluationGeneral, BenchmarkVersion: "general-v1", RequiredCaseIDs: []string{"case-1"}, QualityWeightMillis: 600, ReliabilityWeightMillis: 300, LatencyWeightMillis: 100} server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.URL.Path == "/v1/task-evaluations/execute" { - _ = json.NewEncoder(w).Encode(capability.TaskEvaluationExecutionResponse{ProfileID: profile.ID, BindingID: "binding-1", BindingRevision: 3, Observations: []capability.TaskCaseObservation{{CaseID: "case-1", Result: capability.EvaluationSupported, EvidenceArtifactRef: "artifact-1"}}, ObservedAtUnixMillis: 10}) + _ = json.NewEncoder(w).Encode(capability.TaskEvaluationExecutionResponse{ProfileID: profile.ID, BindingID: "binding-1", BindingRevision: 3, Observations: []capability.TaskCaseObservation{{CaseID: "case-1", Result: capability.EvaluationSupported, QualityScoreMillis: 800, EvidenceArtifactRef: "artifact-1", VerificationEvidenceRef: "verify-1", VerifierIdentity: "verifier-1"}}, ObservedAtUnixMillis: 10}) return } if r.URL.Path == "/v1/work-graphs/task-evaluations/callback" { diff --git a/contracts/capability/task_evaluation.go b/contracts/capability/task_evaluation.go index 049aeb2..5b51c81 100644 --- a/contracts/capability/task_evaluation.go +++ b/contracts/capability/task_evaluation.go @@ -80,6 +80,10 @@ type TaskCaseObservation struct { LatencyMicros int64 CriticalFailure bool EvidenceArtifactRef string + // VerificationEvidenceRef and VerifierIdentity are required whenever an + // observation claims semantic support or a non-zero quality score. + VerificationEvidenceRef string + VerifierIdentity string } // TaskEvaluation is durable evidence for one model/binding revision under one diff --git a/dev/ACTIVE_TASK.json b/dev/ACTIVE_TASK.json index aebbe5d..1d5c817 100644 --- a/dev/ACTIVE_TASK.json +++ b/dev/ACTIVE_TASK.json @@ -1,5 +1,5 @@ { - "task_file": "dev/tasks/DEV-000219-reachability-not-quality.json", - "task_id": "DEV-000219", + "task_file": "dev/tasks/DEV-000220-verifier-evidence-boundary.json", + "task_id": "DEV-000220", "task_revision": 1 } diff --git a/dev/tasks/DEV-000220-verifier-evidence-boundary.json b/dev/tasks/DEV-000220-verifier-evidence-boundary.json new file mode 100644 index 0000000..4eb0881 --- /dev/null +++ b/dev/tasks/DEV-000220-verifier-evidence-boundary.json @@ -0,0 +1,23 @@ +{ + "task_id": "DEV-000220", + "task_revision": 1, + "architecture_revision": "ADR-0018", + "base_repo_revision": "66ed523248ea6cf0a41383e2206d9a70eeaafda8", + "risk_tier": "E", + "intent": "Require explicit independent verifier identity and evidence before task evaluation can claim semantic support or non-zero quality.", + "requirement_ids": ["independent verifier evidence", "no false quality", "typed model qualification"], + "allowed_change_scope": {"modules": ["capability", "agent-worker"], "paths": ["contracts/capability/task_evaluation.go", "modules/capability/domain/task_evaluation.go", "modules/capability/domain/task_evaluation_test.go", "cmd/agent-worker/task_evaluation_executor_test.go", "dev/ACTIVE_TASK.json", "dev/tasks/DEV-000220-verifier-evidence-boundary.json"]}, + "forbidden_change_scope": {"paths": ["architecture/**", "security/**", "database/**", "deploy/**", "cmd/cognitive-worker/**", "LLM-local/**"]}, + "allowed_architecture_delta": {"new_cross_module_dependencies": 0, "new_canonical_owners": 0, "new_privileges": 0, "new_third_party_dependencies": 0, "new_database_owner_roles": 0}, + "contracts": {"consume": ["TaskCaseObservation", "TaskEvaluation", "existing Work Graph callback"], "may_change": ["typed observation fields for verifier evidence"]}, + "semantic_owner": "capability owns qualification evidence; verifier remains independent of provider invocation", + "canonical_state": ["existing capability task-evaluation evidence"], + "state_transitions": ["reachability UNKNOWN -> separately verified SUPPORTED only with evidence"], + "side_effects_and_privileges": ["no provider invocation or external mutation"], + "persistence_and_migrations": ["existing task evaluation persistence is unchanged"], + "failure_retry_restart": ["missing verifier fields fail closed; unknown reachability remains non-promotable"], + "assumptions": ["a later verifier runtime will produce VerificationEvidenceRef and VerifierIdentity"], + "acceptance_scenarios": ["supported without verifier is rejected", "unknown with zero quality remains valid", "verified supported observation aggregates normally", "no hardware/provider name bypasses the boundary"], + "required_tests": ["go test", "go vet", "preflight", "check-diff", "check-architecture", "independent", "git diff --check", "CI", "production public health check"], + "terminal_gap_states": ["BLOCKED_BY_MISSING_CAPABILITY", "BLOCKED_BY_SPEC_GAP"] +} diff --git a/modules/capability/domain/task_evaluation.go b/modules/capability/domain/task_evaluation.go index 4e8c4cd..5f33b1e 100644 --- a/modules/capability/domain/task_evaluation.go +++ b/modules/capability/domain/task_evaluation.go @@ -85,6 +85,12 @@ func BuildTaskEvaluation(profile capability.TaskEvaluationProfile, evaluationID if _, ok := seen[observation.CaseID]; ok || observation.EvidenceArtifactRef == "" || observation.QualityScoreMillis < 0 || observation.QualityScoreMillis > 1000 || observation.LatencyMicros < 0 { return capability.TaskEvaluation{}, ErrInvalidTaskEvaluation } + if observation.Result == capability.EvaluationSupported && (observation.VerificationEvidenceRef == "" || observation.VerifierIdentity == "") { + return capability.TaskEvaluation{}, ErrInvalidTaskEvaluation + } + if observation.Result == capability.EvaluationUnknown && observation.QualityScoreMillis != 0 { + return capability.TaskEvaluation{}, ErrInvalidTaskEvaluation + } seen[observation.CaseID] = struct{}{} qualityTotal += observation.QualityScoreMillis latencyTotal += observation.LatencyMicros diff --git a/modules/capability/domain/task_evaluation_test.go b/modules/capability/domain/task_evaluation_test.go index e17622a..3671f50 100644 --- a/modules/capability/domain/task_evaluation_test.go +++ b/modules/capability/domain/task_evaluation_test.go @@ -10,14 +10,22 @@ import ( func TestBuildTaskEvaluationAggregatesOnlyProfileCases(t *testing.T) { profile := capability.TaskEvaluationProfile{ID: "research-v1", Revision: 1, CapabilityID: "creator-advisory", Stage: capability.TaskEvaluationGeneral, BenchmarkVersion: "research-v1", RequiredCaseIDs: []string{"claim", "uncertainty"}, QualityWeightMillis: 600, ReliabilityWeightMillis: 300, LatencyWeightMillis: 100} value, err := BuildTaskEvaluation(profile, "evaluation-1", "binding-1", 2, "creator-advisory", []capability.TaskCaseObservation{ - {CaseID: "claim", Result: capability.EvaluationSupported, QualityScoreMillis: 900, LatencyMicros: 400_000, EvidenceArtifactRef: "artifact-case-1"}, - {CaseID: "uncertainty", Result: capability.EvaluationSupported, QualityScoreMillis: 800, LatencyMicros: 600_000, EvidenceArtifactRef: "artifact-case-2"}, + {CaseID: "claim", Result: capability.EvaluationSupported, QualityScoreMillis: 900, LatencyMicros: 400_000, EvidenceArtifactRef: "artifact-case-1", VerificationEvidenceRef: "verify-case-1", VerifierIdentity: "verifier-1"}, + {CaseID: "uncertainty", Result: capability.EvaluationSupported, QualityScoreMillis: 800, LatencyMicros: 600_000, EvidenceArtifactRef: "artifact-case-2", VerificationEvidenceRef: "verify-case-2", VerifierIdentity: "verifier-1"}, }, artifact.ArtifactID("artifact-summary"), 10) if err != nil || value.Result != capability.EvaluationSupported || value.OverallScoreMillis == 0 || value.ProfileID != profile.ID { t.Fatalf("evaluation=%+v err=%v", value, err) } } +func TestBuildTaskEvaluationRejectsSupportedWithoutIndependentVerifier(t *testing.T) { + profile := capability.TaskEvaluationProfile{ID: "quality-v1", Revision: 1, CapabilityID: "creator-advisory", Stage: capability.TaskEvaluationGeneral, BenchmarkVersion: "quality-v1", RequiredCaseIDs: []string{"case"}, QualityWeightMillis: 700, ReliabilityWeightMillis: 200, LatencyWeightMillis: 100} + _, err := BuildTaskEvaluation(profile, "evaluation-unverified", "binding-1", 1, profile.CapabilityID, []capability.TaskCaseObservation{{CaseID: "case", Result: capability.EvaluationSupported, QualityScoreMillis: 900, EvidenceArtifactRef: "artifact-case"}}, artifact.ArtifactID("artifact-summary"), 1) + if err == nil { + t.Fatal("supported observation without verifier evidence was accepted") + } +} + func TestBuildTaskEvaluationDoesNotPromoteUnknown(t *testing.T) { profile := capability.TaskEvaluationProfile{ID: "coding-v1", Revision: 1, CapabilityID: "creator-advisory", Stage: capability.TaskEvaluationGeneral, BenchmarkVersion: "coding-v1", RequiredCaseIDs: []string{"compile"}, QualityWeightMillis: 700, ReliabilityWeightMillis: 200, LatencyWeightMillis: 100} value, err := BuildTaskEvaluation(profile, "evaluation-2", "binding-1", 2, "creator-advisory", []capability.TaskCaseObservation{{CaseID: "compile", Result: capability.EvaluationUnknown, QualityScoreMillis: 0, LatencyMicros: 0, EvidenceArtifactRef: "artifact-case"}}, artifact.ArtifactID("artifact-summary"), 11)