feat(capability): require verifier evidence for quality

fix(cognitive): separate reachability from quality evidence (#196)

cmd/agent-worker/task_evaluation_executor_test.go

@@ -15,7 +15,7 @@ func TestTaskEvaluationExecutorInvokesAggregatesAndCallbacks(t *testing.T) {
 	profile := capability.TaskEvaluationProfile{ID: "profile-1", Revision: 1, CapabilityID: "capability-1", Stage: capability.TaskEvaluationGeneral, BenchmarkVersion: "general-v1", RequiredCaseIDs: []string{"case-1"}, QualityWeightMillis: 600, ReliabilityWeightMillis: 300, LatencyWeightMillis: 100}
 	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		if r.URL.Path == "/v1/task-evaluations/execute" {
-			_ = json.NewEncoder(w).Encode(capability.TaskEvaluationExecutionResponse{ProfileID: profile.ID, BindingID: "binding-1", BindingRevision: 3, Observations: []capability.TaskCaseObservation{{CaseID: "case-1", Result: capability.EvaluationSupported, EvidenceArtifactRef: "artifact-1"}}, ObservedAtUnixMillis: 10})
+			_ = json.NewEncoder(w).Encode(capability.TaskEvaluationExecutionResponse{ProfileID: profile.ID, BindingID: "binding-1", BindingRevision: 3, Observations: []capability.TaskCaseObservation{{CaseID: "case-1", Result: capability.EvaluationSupported, QualityScoreMillis: 800, EvidenceArtifactRef: "artifact-1", VerificationEvidenceRef: "verify-1", VerifierIdentity: "verifier-1"}}, ObservedAtUnixMillis: 10})
 			return
 		}
 		if r.URL.Path == "/v1/work-graphs/task-evaluations/callback" {

cmd/cognitive-worker/task_evaluation.go

@@ -68,9 +68,10 @@ func (g cognitiveGateway) taskEvaluationHandler(w http.ResponseWriter, r *http.R
 			response, invokeErr := g.invokeAdmitted(r.Context(), string(invocation.RequestID), provider, invocation, int(request.MaxOutputBytes))
 			observation := capability.TaskCaseObservation{CaseID: item.CaseID, LatencyMicros: time.Since(started).Microseconds(), EvidenceArtifactRef: item.EvidenceArtifactRef}
 			if invokeErr == nil && response.FinishReason == cognitive.FinishCompleted && response.AdvisoryText != "" {
-				observation.Result = capability.EvaluationSupported
-				// Invocation success proves reachability/support for this case,
-				// not semantic answer quality. A verifier may score quality later.
+				// Invocation success proves reachability only. Without an
+				// independent verifier and answer evidence, semantic support and
+				// quality must remain UNKNOWN rather than becoming false evidence.
+				observation.Result = capability.EvaluationUnknown
 				observation.QualityScoreMillis = 0
 			} else {
 				observation.Result = capability.EvaluationUnknown

cmd/cognitive-worker/task_evaluation_test.go

@@ -47,7 +47,12 @@ func TestTaskEvaluationHandlerInvokesEveryBoundedCase(t *testing.T) {
 	if err := json.Unmarshal(recorder.Body.Bytes(), &response); err != nil {
 		t.Fatal(err)
 	}
-	if len(response.Observations) != 2 || response.Observations[0].Result != capability.EvaluationSupported || response.Observations[1].Result != capability.EvaluationSupported {
+	if len(response.Observations) != 2 || response.Observations[0].Result != capability.EvaluationUnknown || response.Observations[1].Result != capability.EvaluationUnknown {
 		t.Fatalf("unexpected observations: %+v", response.Observations)
 	}
+	for _, observation := range response.Observations {
+		if observation.QualityScoreMillis != 0 || observation.CriticalFailure {
+			t.Fatalf("reachability-only evidence must not carry quality or failure claims: %+v", observation)
+		}
+	}
 }

contracts/capability/task_evaluation.go

@@ -80,6 +80,10 @@ type TaskCaseObservation struct {
 	LatencyMicros       int64
 	CriticalFailure     bool
 	EvidenceArtifactRef string
+	// VerificationEvidenceRef and VerifierIdentity are required whenever an
+	// observation claims semantic support or a non-zero quality score.
+	VerificationEvidenceRef string
+	VerifierIdentity        string
 }
 
 // TaskEvaluation is durable evidence for one model/binding revision under one

dev/ACTIVE_TASK.json

@@ -1,5 +1,5 @@
 {
-  "task_file": "dev/tasks/DEV-000217-current-status-evidence.json",
-  "task_id": "DEV-000217",
+  "task_file": "dev/tasks/DEV-000220-verifier-evidence-boundary.json",
+  "task_id": "DEV-000220",
   "task_revision": 1
 }

dev/tasks/DEV-000218-automatic-expansion-policy.json

@@ -0,0 +1,23 @@
+{
+  "task_id": "DEV-000218",
+  "task_revision": 1,
+  "architecture_revision": "ADR-0018",
+  "base_repo_revision": "ab840878fe7d21a0a6aeb69193a3d58ea23a56e0",
+  "risk_tier": "E",
+  "intent": "Add a conservative deterministic policy gate for creating additional model-evaluation profiles only after general evaluation evidence remains unresolved.",
+  "requirement_ids": ["general-first evaluation", "bounded additional evaluation", "no unnecessary fleet-wide tests"],
+  "allowed_change_scope": {"modules": ["capability"], "paths": ["modules/capability/domain/task_evaluation.go", "modules/capability/domain/task_evaluation_test.go", "dev/ACTIVE_TASK.json", "dev/tasks/DEV-000218-automatic-expansion-policy.json"]},
+  "forbidden_change_scope": {"paths": ["architecture/**", "security/**", "database/**", "deploy/**", "cmd/**", "contracts/**", "LLM-local/**"]},
+  "allowed_architecture_delta": {"new_cross_module_dependencies": 0, "new_canonical_owners": 0, "new_privileges": 0, "new_third_party_dependencies": 0, "new_database_owner_roles": 0},
+  "contracts": {"consume": ["TaskEvaluationProfile", "TaskEvaluation", "TaskEvaluationExpansionRequest"], "may_change": []},
+  "semantic_owner": "capability owns model qualification evidence and expansion policy",
+  "canonical_state": ["none; this packet creates no persistence or activation authority"],
+  "state_transitions": ["general COMPLETED/PARTIAL unresolved evidence -> bounded additional profile candidate", "qualified candidate -> no expansion"],
+  "side_effects_and_privileges": ["pure deterministic decision; no provider call, no model invocation, no mutation"],
+  "persistence_and_migrations": ["none"],
+  "failure_retry_restart": ["missing trigger evidence or additional cases fails closed; repeated decision is deterministic"],
+  "assumptions": ["a later task supplies task-specific additional cases from an unresolved outcome and schedules them through the existing Work Graph"],
+  "acceptance_scenarios": ["general graph not completed/partial does not expand", "unknown/unsupported/low-scoring candidates expand only with evidence and new cases", "one qualified candidate suppresses expansion", "no provider or hardware label changes the decision"],
+  "required_tests": ["go test", "go vet", "preflight", "check-diff", "check-architecture", "independent", "git diff --check", "CI", "production public health check"],
+  "terminal_gap_states": ["BLOCKED_BY_MISSING_CAPABILITY", "BLOCKED_BY_SPEC_GAP"]
+}

dev/tasks/DEV-000219-reachability-not-quality.json

@@ -0,0 +1,23 @@
+{
+  "task_id": "DEV-000219",
+  "task_revision": 1,
+  "architecture_revision": "ADR-0018",
+  "base_repo_revision": "7476f43ffa57d55f5a5a9bddc7f1c74e9b87ae0e",
+  "risk_tier": "E",
+  "intent": "Prevent task-evaluation invocation success from being recorded as semantic model quality or support without independent verifier evidence.",
+  "requirement_ids": ["truthful model evidence", "independent verification", "no false qualification"],
+  "allowed_change_scope": {"modules": ["cognitive-worker"], "paths": ["cmd/cognitive-worker/task_evaluation.go", "cmd/cognitive-worker/task_evaluation_test.go", "dev/ACTIVE_TASK.json", "dev/tasks/DEV-000219-reachability-not-quality.json"]},
+  "forbidden_change_scope": {"paths": ["architecture/**", "security/**", "database/**", "deploy/**", "contracts/**", "modules/**", "LLM-local/**"]},
+  "allowed_architecture_delta": {"new_cross_module_dependencies": 0, "new_canonical_owners": 0, "new_privileges": 0, "new_third_party_dependencies": 0, "new_database_owner_roles": 0},
+  "contracts": {"consume": ["TaskEvaluationExecutionRequest/Response", "TaskCaseObservation", "Cognitive Gateway"], "may_change": []},
+  "semantic_owner": "capability evidence remains owner; cognitive-worker only records invocation observation",
+  "canonical_state": ["existing task-evaluation evidence; no new persistence"],
+  "state_transitions": ["successful invocation -> UNKNOWN reachability observation", "independent verifier evidence -> future supported/quality observation"],
+  "side_effects_and_privileges": ["no additional provider calls or privileges"],
+  "persistence_and_migrations": ["none"],
+  "failure_retry_restart": ["unknown evidence remains retryable/eligible for a separately verified evaluation; no promotion follows invocation alone"],
+  "assumptions": ["a later verifier task will attach answer-quality evidence and score cases"],
+  "acceptance_scenarios": ["nonempty provider response is not SUPPORTED", "quality score remains zero", "provider invocation failure remains UNKNOWN with critical failure", "no model/hardware label can manufacture quality evidence"],
+  "required_tests": ["go test", "go vet", "preflight", "check-diff", "check-architecture", "independent", "git diff --check", "CI", "production public health check"],
+  "terminal_gap_states": ["BLOCKED_BY_MISSING_CAPABILITY", "BLOCKED_BY_SPEC_GAP"]
+}

dev/tasks/DEV-000220-verifier-evidence-boundary.json

@@ -0,0 +1,23 @@
+{
+  "task_id": "DEV-000220",
+  "task_revision": 1,
+  "architecture_revision": "ADR-0018",
+  "base_repo_revision": "66ed523248ea6cf0a41383e2206d9a70eeaafda8",
+  "risk_tier": "E",
+  "intent": "Require explicit independent verifier identity and evidence before task evaluation can claim semantic support or non-zero quality.",
+  "requirement_ids": ["independent verifier evidence", "no false quality", "typed model qualification"],
+  "allowed_change_scope": {"modules": ["capability", "agent-worker"], "paths": ["contracts/capability/task_evaluation.go", "modules/capability/domain/task_evaluation.go", "modules/capability/domain/task_evaluation_test.go", "cmd/agent-worker/task_evaluation_executor_test.go", "dev/ACTIVE_TASK.json", "dev/tasks/DEV-000220-verifier-evidence-boundary.json"]},
+  "forbidden_change_scope": {"paths": ["architecture/**", "security/**", "database/**", "deploy/**", "cmd/cognitive-worker/**", "LLM-local/**"]},
+  "allowed_architecture_delta": {"new_cross_module_dependencies": 0, "new_canonical_owners": 0, "new_privileges": 0, "new_third_party_dependencies": 0, "new_database_owner_roles": 0},
+  "contracts": {"consume": ["TaskCaseObservation", "TaskEvaluation", "existing Work Graph callback"], "may_change": ["typed observation fields for verifier evidence"]},
+  "semantic_owner": "capability owns qualification evidence; verifier remains independent of provider invocation",
+  "canonical_state": ["existing capability task-evaluation evidence"],
+  "state_transitions": ["reachability UNKNOWN -> separately verified SUPPORTED only with evidence"],
+  "side_effects_and_privileges": ["no provider invocation or external mutation"],
+  "persistence_and_migrations": ["existing task evaluation persistence is unchanged"],
+  "failure_retry_restart": ["missing verifier fields fail closed; unknown reachability remains non-promotable"],
+  "assumptions": ["a later verifier runtime will produce VerificationEvidenceRef and VerifierIdentity"],
+  "acceptance_scenarios": ["supported without verifier is rejected", "unknown with zero quality remains valid", "verified supported observation aggregates normally", "no hardware/provider name bypasses the boundary"],
+  "required_tests": ["go test", "go vet", "preflight", "check-diff", "check-architecture", "independent", "git diff --check", "CI", "production public health check"],
+  "terminal_gap_states": ["BLOCKED_BY_MISSING_CAPABILITY", "BLOCKED_BY_SPEC_GAP"]
+}

modules/capability/domain/task_evaluation.go

@@ -2,6 +2,7 @@ package domain
 
 import (
 	"errors"
+	"fmt"
 
 	"git.cin.su/m/Assistant/contracts/artifact"
 	"git.cin.su/m/Assistant/contracts/capability"
@@ -9,6 +10,35 @@ import (
 
 var ErrInvalidTaskEvaluation = errors.New("invalid task evaluation")
 
+const automaticExpansionMinimumOverallScore int64 = 700
+
+// AutomaticTaskEvaluationExpansion decides whether a completed general
+// evaluation has evidence that warrants a bounded additional profile. It is
+// deliberately conservative: one supported candidate above the quality
+// threshold suppresses expansion, while unknown/unsupported/low-scoring or
+// critically failing candidates keep the unresolved path explicit.
+func AutomaticTaskEvaluationExpansion(parent capability.TaskEvaluationProfile, parentGraphState string, evaluations []capability.TaskEvaluation, request capability.TaskEvaluationExpansionRequest, profileID capability.TaskEvaluationProfileID, revision int64) (capability.TaskEvaluationProfile, bool, error) {
+	if parentGraphState != "COMPLETED" && parentGraphState != "PARTIAL" {
+		return capability.TaskEvaluationProfile{}, false, nil
+	}
+	if len(evaluations) == 0 {
+		return capability.TaskEvaluationProfile{}, false, nil
+	}
+	for _, evaluation := range evaluations {
+		if evaluation.Result == capability.EvaluationSupported && evaluation.OverallScoreMillis >= automaticExpansionMinimumOverallScore && evaluation.CriticalFailureMillis == 0 {
+			return capability.TaskEvaluationProfile{}, false, nil
+		}
+	}
+	if request.TriggerEvidenceRef == "" || len(request.NewCaseIDs) == 0 {
+		return capability.TaskEvaluationProfile{}, false, fmt.Errorf("unresolved task evaluation requires trigger evidence and additional cases")
+	}
+	profile, err := BuildAdditionalTaskEvaluationProfile(parent, request, profileID, revision)
+	if err != nil {
+		return capability.TaskEvaluationProfile{}, false, err
+	}
+	return profile, true, nil
+}
+
 func BuildAdditionalTaskEvaluationProfile(parent capability.TaskEvaluationProfile, request capability.TaskEvaluationExpansionRequest, profileID capability.TaskEvaluationProfileID, revision int64) (capability.TaskEvaluationProfile, error) {
 	if err := parent.Validate(); err != nil || parent.Stage != capability.TaskEvaluationGeneral || request.ParentProfileID != parent.ID || request.CapabilityID != parent.CapabilityID || request.TriggerEvidenceRef == "" || len(request.NewCaseIDs) == 0 || profileID == "" || revision < 1 || request.BenchmarkVersion == "" {
 		return capability.TaskEvaluationProfile{}, ErrInvalidTaskEvaluation
@@ -55,6 +85,12 @@ func BuildTaskEvaluation(profile capability.TaskEvaluationProfile, evaluationID
 		if _, ok := seen[observation.CaseID]; ok || observation.EvidenceArtifactRef == "" || observation.QualityScoreMillis < 0 || observation.QualityScoreMillis > 1000 || observation.LatencyMicros < 0 {
 			return capability.TaskEvaluation{}, ErrInvalidTaskEvaluation
 		}
+		if observation.Result == capability.EvaluationSupported && (observation.VerificationEvidenceRef == "" || observation.VerifierIdentity == "") {
+			return capability.TaskEvaluation{}, ErrInvalidTaskEvaluation
+		}
+		if observation.Result == capability.EvaluationUnknown && observation.QualityScoreMillis != 0 {
+			return capability.TaskEvaluation{}, ErrInvalidTaskEvaluation
+		}
 		seen[observation.CaseID] = struct{}{}
 		qualityTotal += observation.QualityScoreMillis
 		latencyTotal += observation.LatencyMicros

modules/capability/domain/task_evaluation_test.go

@@ -10,14 +10,22 @@ import (
 func TestBuildTaskEvaluationAggregatesOnlyProfileCases(t *testing.T) {
 	profile := capability.TaskEvaluationProfile{ID: "research-v1", Revision: 1, CapabilityID: "creator-advisory", Stage: capability.TaskEvaluationGeneral, BenchmarkVersion: "research-v1", RequiredCaseIDs: []string{"claim", "uncertainty"}, QualityWeightMillis: 600, ReliabilityWeightMillis: 300, LatencyWeightMillis: 100}
 	value, err := BuildTaskEvaluation(profile, "evaluation-1", "binding-1", 2, "creator-advisory", []capability.TaskCaseObservation{
-		{CaseID: "claim", Result: capability.EvaluationSupported, QualityScoreMillis: 900, LatencyMicros: 400_000, EvidenceArtifactRef: "artifact-case-1"},
-		{CaseID: "uncertainty", Result: capability.EvaluationSupported, QualityScoreMillis: 800, LatencyMicros: 600_000, EvidenceArtifactRef: "artifact-case-2"},
+		{CaseID: "claim", Result: capability.EvaluationSupported, QualityScoreMillis: 900, LatencyMicros: 400_000, EvidenceArtifactRef: "artifact-case-1", VerificationEvidenceRef: "verify-case-1", VerifierIdentity: "verifier-1"},
+		{CaseID: "uncertainty", Result: capability.EvaluationSupported, QualityScoreMillis: 800, LatencyMicros: 600_000, EvidenceArtifactRef: "artifact-case-2", VerificationEvidenceRef: "verify-case-2", VerifierIdentity: "verifier-1"},
 	}, artifact.ArtifactID("artifact-summary"), 10)
 	if err != nil || value.Result != capability.EvaluationSupported || value.OverallScoreMillis == 0 || value.ProfileID != profile.ID {
 		t.Fatalf("evaluation=%+v err=%v", value, err)
 	}
 }
 
+func TestBuildTaskEvaluationRejectsSupportedWithoutIndependentVerifier(t *testing.T) {
+	profile := capability.TaskEvaluationProfile{ID: "quality-v1", Revision: 1, CapabilityID: "creator-advisory", Stage: capability.TaskEvaluationGeneral, BenchmarkVersion: "quality-v1", RequiredCaseIDs: []string{"case"}, QualityWeightMillis: 700, ReliabilityWeightMillis: 200, LatencyWeightMillis: 100}
+	_, err := BuildTaskEvaluation(profile, "evaluation-unverified", "binding-1", 1, profile.CapabilityID, []capability.TaskCaseObservation{{CaseID: "case", Result: capability.EvaluationSupported, QualityScoreMillis: 900, EvidenceArtifactRef: "artifact-case"}}, artifact.ArtifactID("artifact-summary"), 1)
+	if err == nil {
+		t.Fatal("supported observation without verifier evidence was accepted")
+	}
+}
+
 func TestBuildTaskEvaluationDoesNotPromoteUnknown(t *testing.T) {
 	profile := capability.TaskEvaluationProfile{ID: "coding-v1", Revision: 1, CapabilityID: "creator-advisory", Stage: capability.TaskEvaluationGeneral, BenchmarkVersion: "coding-v1", RequiredCaseIDs: []string{"compile"}, QualityWeightMillis: 700, ReliabilityWeightMillis: 200, LatencyWeightMillis: 100}
 	value, err := BuildTaskEvaluation(profile, "evaluation-2", "binding-1", 2, "creator-advisory", []capability.TaskCaseObservation{{CaseID: "compile", Result: capability.EvaluationUnknown, QualityScoreMillis: 0, LatencyMicros: 0, EvidenceArtifactRef: "artifact-case"}}, artifact.ArtifactID("artifact-summary"), 11)
@@ -33,3 +41,21 @@ func TestBuildAdditionalProfileRequiresUnresolvedEvidenceAndPreservesHistory(t *
 		t.Fatalf("profile=%+v err=%v", profile, err)
 	}
 }
+
+func TestAutomaticExpansionWaitsForGeneralGraphAndUnresolvedCandidates(t *testing.T) {
+	parent := capability.TaskEvaluationProfile{ID: "general-v1", Revision: 1, CapabilityID: "creator-advisory", Stage: capability.TaskEvaluationGeneral, BenchmarkVersion: "general-v1", RequiredCaseIDs: []string{"general"}, QualityWeightMillis: 700, ReliabilityWeightMillis: 200, LatencyWeightMillis: 100}
+	request := capability.TaskEvaluationExpansionRequest{ParentProfileID: parent.ID, CapabilityID: parent.CapabilityID, TriggerEvidenceRef: "outcome-unsolved", NewCaseIDs: []string{"edge-case"}, BenchmarkVersion: "additional-v1"}
+	value, expanded, err := AutomaticTaskEvaluationExpansion(parent, "COMPLETED", []capability.TaskEvaluation{{ID: "evaluation-1", ProfileID: parent.ID, BindingID: "binding-1", BindingRevision: 1, CapabilityID: parent.CapabilityID, Result: capability.EvaluationUnknown, OverallScoreMillis: 0, CriticalFailureMillis: 1000, BenchmarkVersion: parent.BenchmarkVersion, ArtifactID: "artifact-1", ObservedAtUnixMillis: 1}}, request, "additional-v1", 1)
+	if err != nil || !expanded || value.Stage != capability.TaskEvaluationAdditional || value.ParentProfileID != parent.ID {
+		t.Fatalf("expected bounded expansion, profile=%+v expanded=%t err=%v", value, expanded, err)
+	}
+}
+
+func TestAutomaticExpansionDoesNotRunWhenAQualifiedCandidateExists(t *testing.T) {
+	parent := capability.TaskEvaluationProfile{ID: "general-v1", Revision: 1, CapabilityID: "creator-advisory", Stage: capability.TaskEvaluationGeneral, BenchmarkVersion: "general-v1", RequiredCaseIDs: []string{"general"}, QualityWeightMillis: 700, ReliabilityWeightMillis: 200, LatencyWeightMillis: 100}
+	request := capability.TaskEvaluationExpansionRequest{ParentProfileID: parent.ID, CapabilityID: parent.CapabilityID, TriggerEvidenceRef: "outcome-unsolved", NewCaseIDs: []string{"edge-case"}, BenchmarkVersion: "additional-v1"}
+	value, expanded, err := AutomaticTaskEvaluationExpansion(parent, "PARTIAL", []capability.TaskEvaluation{{ID: "evaluation-1", ProfileID: parent.ID, BindingID: "binding-1", BindingRevision: 1, CapabilityID: parent.CapabilityID, Result: capability.EvaluationSupported, OverallScoreMillis: 850, CriticalFailureMillis: 0, BenchmarkVersion: parent.BenchmarkVersion, ArtifactID: "artifact-1", ObservedAtUnixMillis: 1}}, request, "additional-v1", 1)
+	if err != nil || expanded || value.ID != "" {
+		t.Fatalf("qualified candidate must suppress expansion: profile=%+v expanded=%t err=%v", value, expanded, err)
+	}
+}