fix(cognitive): separate reachability from quality evidence

feat(capability): gate additional model evaluations (#195)

cmd/cognitive-worker/task_evaluation.go

@@ -68,9 +68,10 @@ func (g cognitiveGateway) taskEvaluationHandler(w http.ResponseWriter, r *http.R
 			response, invokeErr := g.invokeAdmitted(r.Context(), string(invocation.RequestID), provider, invocation, int(request.MaxOutputBytes))
 			observation := capability.TaskCaseObservation{CaseID: item.CaseID, LatencyMicros: time.Since(started).Microseconds(), EvidenceArtifactRef: item.EvidenceArtifactRef}
 			if invokeErr == nil && response.FinishReason == cognitive.FinishCompleted && response.AdvisoryText != "" {
-				observation.Result = capability.EvaluationSupported
-				// Invocation success proves reachability/support for this case,
-				// not semantic answer quality. A verifier may score quality later.
+				// Invocation success proves reachability only. Without an
+				// independent verifier and answer evidence, semantic support and
+				// quality must remain UNKNOWN rather than becoming false evidence.
+				observation.Result = capability.EvaluationUnknown
 				observation.QualityScoreMillis = 0
 			} else {
 				observation.Result = capability.EvaluationUnknown

cmd/cognitive-worker/task_evaluation_test.go

@@ -47,7 +47,12 @@ func TestTaskEvaluationHandlerInvokesEveryBoundedCase(t *testing.T) {
 	if err := json.Unmarshal(recorder.Body.Bytes(), &response); err != nil {
 		t.Fatal(err)
 	}
-	if len(response.Observations) != 2 || response.Observations[0].Result != capability.EvaluationSupported || response.Observations[1].Result != capability.EvaluationSupported {
+	if len(response.Observations) != 2 || response.Observations[0].Result != capability.EvaluationUnknown || response.Observations[1].Result != capability.EvaluationUnknown {
 		t.Fatalf("unexpected observations: %+v", response.Observations)
 	}
+	for _, observation := range response.Observations {
+		if observation.QualityScoreMillis != 0 || observation.CriticalFailure {
+			t.Fatalf("reachability-only evidence must not carry quality or failure claims: %+v", observation)
+		}
+	}
 }

dev/ACTIVE_TASK.json

@@ -1,5 +1,5 @@
 {
-  "task_file": "dev/tasks/DEV-000217-current-status-evidence.json",
-  "task_id": "DEV-000217",
+  "task_file": "dev/tasks/DEV-000219-reachability-not-quality.json",
+  "task_id": "DEV-000219",
   "task_revision": 1
 }

dev/tasks/DEV-000218-automatic-expansion-policy.json

@@ -0,0 +1,23 @@
+{
+  "task_id": "DEV-000218",
+  "task_revision": 1,
+  "architecture_revision": "ADR-0018",
+  "base_repo_revision": "ab840878fe7d21a0a6aeb69193a3d58ea23a56e0",
+  "risk_tier": "E",
+  "intent": "Add a conservative deterministic policy gate for creating additional model-evaluation profiles only after general evaluation evidence remains unresolved.",
+  "requirement_ids": ["general-first evaluation", "bounded additional evaluation", "no unnecessary fleet-wide tests"],
+  "allowed_change_scope": {"modules": ["capability"], "paths": ["modules/capability/domain/task_evaluation.go", "modules/capability/domain/task_evaluation_test.go", "dev/ACTIVE_TASK.json", "dev/tasks/DEV-000218-automatic-expansion-policy.json"]},
+  "forbidden_change_scope": {"paths": ["architecture/**", "security/**", "database/**", "deploy/**", "cmd/**", "contracts/**", "LLM-local/**"]},
+  "allowed_architecture_delta": {"new_cross_module_dependencies": 0, "new_canonical_owners": 0, "new_privileges": 0, "new_third_party_dependencies": 0, "new_database_owner_roles": 0},
+  "contracts": {"consume": ["TaskEvaluationProfile", "TaskEvaluation", "TaskEvaluationExpansionRequest"], "may_change": []},
+  "semantic_owner": "capability owns model qualification evidence and expansion policy",
+  "canonical_state": ["none; this packet creates no persistence or activation authority"],
+  "state_transitions": ["general COMPLETED/PARTIAL unresolved evidence -> bounded additional profile candidate", "qualified candidate -> no expansion"],
+  "side_effects_and_privileges": ["pure deterministic decision; no provider call, no model invocation, no mutation"],
+  "persistence_and_migrations": ["none"],
+  "failure_retry_restart": ["missing trigger evidence or additional cases fails closed; repeated decision is deterministic"],
+  "assumptions": ["a later task supplies task-specific additional cases from an unresolved outcome and schedules them through the existing Work Graph"],
+  "acceptance_scenarios": ["general graph not completed/partial does not expand", "unknown/unsupported/low-scoring candidates expand only with evidence and new cases", "one qualified candidate suppresses expansion", "no provider or hardware label changes the decision"],
+  "required_tests": ["go test", "go vet", "preflight", "check-diff", "check-architecture", "independent", "git diff --check", "CI", "production public health check"],
+  "terminal_gap_states": ["BLOCKED_BY_MISSING_CAPABILITY", "BLOCKED_BY_SPEC_GAP"]
+}

dev/tasks/DEV-000219-reachability-not-quality.json

@@ -0,0 +1,23 @@
+{
+  "task_id": "DEV-000219",
+  "task_revision": 1,
+  "architecture_revision": "ADR-0018",
+  "base_repo_revision": "7476f43ffa57d55f5a5a9bddc7f1c74e9b87ae0e",
+  "risk_tier": "E",
+  "intent": "Prevent task-evaluation invocation success from being recorded as semantic model quality or support without independent verifier evidence.",
+  "requirement_ids": ["truthful model evidence", "independent verification", "no false qualification"],
+  "allowed_change_scope": {"modules": ["cognitive-worker"], "paths": ["cmd/cognitive-worker/task_evaluation.go", "cmd/cognitive-worker/task_evaluation_test.go", "dev/ACTIVE_TASK.json", "dev/tasks/DEV-000219-reachability-not-quality.json"]},
+  "forbidden_change_scope": {"paths": ["architecture/**", "security/**", "database/**", "deploy/**", "contracts/**", "modules/**", "LLM-local/**"]},
+  "allowed_architecture_delta": {"new_cross_module_dependencies": 0, "new_canonical_owners": 0, "new_privileges": 0, "new_third_party_dependencies": 0, "new_database_owner_roles": 0},
+  "contracts": {"consume": ["TaskEvaluationExecutionRequest/Response", "TaskCaseObservation", "Cognitive Gateway"], "may_change": []},
+  "semantic_owner": "capability evidence remains owner; cognitive-worker only records invocation observation",
+  "canonical_state": ["existing task-evaluation evidence; no new persistence"],
+  "state_transitions": ["successful invocation -> UNKNOWN reachability observation", "independent verifier evidence -> future supported/quality observation"],
+  "side_effects_and_privileges": ["no additional provider calls or privileges"],
+  "persistence_and_migrations": ["none"],
+  "failure_retry_restart": ["unknown evidence remains retryable/eligible for a separately verified evaluation; no promotion follows invocation alone"],
+  "assumptions": ["a later verifier task will attach answer-quality evidence and score cases"],
+  "acceptance_scenarios": ["nonempty provider response is not SUPPORTED", "quality score remains zero", "provider invocation failure remains UNKNOWN with critical failure", "no model/hardware label can manufacture quality evidence"],
+  "required_tests": ["go test", "go vet", "preflight", "check-diff", "check-architecture", "independent", "git diff --check", "CI", "production public health check"],
+  "terminal_gap_states": ["BLOCKED_BY_MISSING_CAPABILITY", "BLOCKED_BY_SPEC_GAP"]
+}

modules/capability/domain/task_evaluation.go

@@ -2,6 +2,7 @@ package domain
 
 import (
 	"errors"
+	"fmt"
 
 	"git.cin.su/m/Assistant/contracts/artifact"
 	"git.cin.su/m/Assistant/contracts/capability"
@@ -9,6 +10,35 @@ import (
 
 var ErrInvalidTaskEvaluation = errors.New("invalid task evaluation")
 
+const automaticExpansionMinimumOverallScore int64 = 700
+
+// AutomaticTaskEvaluationExpansion decides whether a completed general
+// evaluation has evidence that warrants a bounded additional profile. It is
+// deliberately conservative: one supported candidate above the quality
+// threshold suppresses expansion, while unknown/unsupported/low-scoring or
+// critically failing candidates keep the unresolved path explicit.
+func AutomaticTaskEvaluationExpansion(parent capability.TaskEvaluationProfile, parentGraphState string, evaluations []capability.TaskEvaluation, request capability.TaskEvaluationExpansionRequest, profileID capability.TaskEvaluationProfileID, revision int64) (capability.TaskEvaluationProfile, bool, error) {
+	if parentGraphState != "COMPLETED" && parentGraphState != "PARTIAL" {
+		return capability.TaskEvaluationProfile{}, false, nil
+	}
+	if len(evaluations) == 0 {
+		return capability.TaskEvaluationProfile{}, false, nil
+	}
+	for _, evaluation := range evaluations {
+		if evaluation.Result == capability.EvaluationSupported && evaluation.OverallScoreMillis >= automaticExpansionMinimumOverallScore && evaluation.CriticalFailureMillis == 0 {
+			return capability.TaskEvaluationProfile{}, false, nil
+		}
+	}
+	if request.TriggerEvidenceRef == "" || len(request.NewCaseIDs) == 0 {
+		return capability.TaskEvaluationProfile{}, false, fmt.Errorf("unresolved task evaluation requires trigger evidence and additional cases")
+	}
+	profile, err := BuildAdditionalTaskEvaluationProfile(parent, request, profileID, revision)
+	if err != nil {
+		return capability.TaskEvaluationProfile{}, false, err
+	}
+	return profile, true, nil
+}
+
 func BuildAdditionalTaskEvaluationProfile(parent capability.TaskEvaluationProfile, request capability.TaskEvaluationExpansionRequest, profileID capability.TaskEvaluationProfileID, revision int64) (capability.TaskEvaluationProfile, error) {
 	if err := parent.Validate(); err != nil || parent.Stage != capability.TaskEvaluationGeneral || request.ParentProfileID != parent.ID || request.CapabilityID != parent.CapabilityID || request.TriggerEvidenceRef == "" || len(request.NewCaseIDs) == 0 || profileID == "" || revision < 1 || request.BenchmarkVersion == "" {
 		return capability.TaskEvaluationProfile{}, ErrInvalidTaskEvaluation

modules/capability/domain/task_evaluation_test.go

@@ -33,3 +33,21 @@ func TestBuildAdditionalProfileRequiresUnresolvedEvidenceAndPreservesHistory(t *
 		t.Fatalf("profile=%+v err=%v", profile, err)
 	}
 }
+
+func TestAutomaticExpansionWaitsForGeneralGraphAndUnresolvedCandidates(t *testing.T) {
+	parent := capability.TaskEvaluationProfile{ID: "general-v1", Revision: 1, CapabilityID: "creator-advisory", Stage: capability.TaskEvaluationGeneral, BenchmarkVersion: "general-v1", RequiredCaseIDs: []string{"general"}, QualityWeightMillis: 700, ReliabilityWeightMillis: 200, LatencyWeightMillis: 100}
+	request := capability.TaskEvaluationExpansionRequest{ParentProfileID: parent.ID, CapabilityID: parent.CapabilityID, TriggerEvidenceRef: "outcome-unsolved", NewCaseIDs: []string{"edge-case"}, BenchmarkVersion: "additional-v1"}
+	value, expanded, err := AutomaticTaskEvaluationExpansion(parent, "COMPLETED", []capability.TaskEvaluation{{ID: "evaluation-1", ProfileID: parent.ID, BindingID: "binding-1", BindingRevision: 1, CapabilityID: parent.CapabilityID, Result: capability.EvaluationUnknown, OverallScoreMillis: 0, CriticalFailureMillis: 1000, BenchmarkVersion: parent.BenchmarkVersion, ArtifactID: "artifact-1", ObservedAtUnixMillis: 1}}, request, "additional-v1", 1)
+	if err != nil || !expanded || value.Stage != capability.TaskEvaluationAdditional || value.ParentProfileID != parent.ID {
+		t.Fatalf("expected bounded expansion, profile=%+v expanded=%t err=%v", value, expanded, err)
+	}
+}
+
+func TestAutomaticExpansionDoesNotRunWhenAQualifiedCandidateExists(t *testing.T) {
+	parent := capability.TaskEvaluationProfile{ID: "general-v1", Revision: 1, CapabilityID: "creator-advisory", Stage: capability.TaskEvaluationGeneral, BenchmarkVersion: "general-v1", RequiredCaseIDs: []string{"general"}, QualityWeightMillis: 700, ReliabilityWeightMillis: 200, LatencyWeightMillis: 100}
+	request := capability.TaskEvaluationExpansionRequest{ParentProfileID: parent.ID, CapabilityID: parent.CapabilityID, TriggerEvidenceRef: "outcome-unsolved", NewCaseIDs: []string{"edge-case"}, BenchmarkVersion: "additional-v1"}
+	value, expanded, err := AutomaticTaskEvaluationExpansion(parent, "PARTIAL", []capability.TaskEvaluation{{ID: "evaluation-1", ProfileID: parent.ID, BindingID: "binding-1", BindingRevision: 1, CapabilityID: parent.CapabilityID, Result: capability.EvaluationSupported, OverallScoreMillis: 850, CriticalFailureMillis: 0, BenchmarkVersion: parent.BenchmarkVersion, ArtifactID: "artifact-1", ObservedAtUnixMillis: 1}}, request, "additional-v1", 1)
+	if err != nil || expanded || value.ID != "" {
+		t.Fatalf("qualified candidate must suppress expansion: profile=%+v expanded=%t err=%v", value, expanded, err)
+	}
+}