From c5b5055ff1580a1fba62ff4c2d58dafd9db978e6 Mon Sep 17 00:00:00 2001 From: Mikhail Date: Sun, 23 Aug 2026 14:02:02 +0300 Subject: [PATCH] fix(cognitive): separate reachability from quality evidence --- cmd/cognitive-worker/task_evaluation.go | 7 +++--- cmd/cognitive-worker/task_evaluation_test.go | 7 +++++- dev/ACTIVE_TASK.json | 4 ++-- .../DEV-000219-reachability-not-quality.json | 23 +++++++++++++++++++ 4 files changed, 35 insertions(+), 6 deletions(-) create mode 100644 dev/tasks/DEV-000219-reachability-not-quality.json diff --git a/cmd/cognitive-worker/task_evaluation.go b/cmd/cognitive-worker/task_evaluation.go index e528e28..c9a581b 100644 --- a/cmd/cognitive-worker/task_evaluation.go +++ b/cmd/cognitive-worker/task_evaluation.go @@ -68,9 +68,10 @@ func (g cognitiveGateway) taskEvaluationHandler(w http.ResponseWriter, r *http.R response, invokeErr := g.invokeAdmitted(r.Context(), string(invocation.RequestID), provider, invocation, int(request.MaxOutputBytes)) observation := capability.TaskCaseObservation{CaseID: item.CaseID, LatencyMicros: time.Since(started).Microseconds(), EvidenceArtifactRef: item.EvidenceArtifactRef} if invokeErr == nil && response.FinishReason == cognitive.FinishCompleted && response.AdvisoryText != "" { - observation.Result = capability.EvaluationSupported - // Invocation success proves reachability/support for this case, - // not semantic answer quality. A verifier may score quality later. + // Invocation success proves reachability only. Without an + // independent verifier and answer evidence, semantic support and + // quality must remain UNKNOWN rather than becoming false evidence. + observation.Result = capability.EvaluationUnknown observation.QualityScoreMillis = 0 } else { observation.Result = capability.EvaluationUnknown diff --git a/cmd/cognitive-worker/task_evaluation_test.go b/cmd/cognitive-worker/task_evaluation_test.go index a1ffa5b..67da325 100644 --- a/cmd/cognitive-worker/task_evaluation_test.go +++ b/cmd/cognitive-worker/task_evaluation_test.go @@ -47,7 +47,12 @@ func TestTaskEvaluationHandlerInvokesEveryBoundedCase(t *testing.T) { if err := json.Unmarshal(recorder.Body.Bytes(), &response); err != nil { t.Fatal(err) } - if len(response.Observations) != 2 || response.Observations[0].Result != capability.EvaluationSupported || response.Observations[1].Result != capability.EvaluationSupported { + if len(response.Observations) != 2 || response.Observations[0].Result != capability.EvaluationUnknown || response.Observations[1].Result != capability.EvaluationUnknown { t.Fatalf("unexpected observations: %+v", response.Observations) } + for _, observation := range response.Observations { + if observation.QualityScoreMillis != 0 || observation.CriticalFailure { + t.Fatalf("reachability-only evidence must not carry quality or failure claims: %+v", observation) + } + } } diff --git a/dev/ACTIVE_TASK.json b/dev/ACTIVE_TASK.json index c54e604..aebbe5d 100644 --- a/dev/ACTIVE_TASK.json +++ b/dev/ACTIVE_TASK.json @@ -1,5 +1,5 @@ { - "task_file": "dev/tasks/DEV-000218-automatic-expansion-policy.json", - "task_id": "DEV-000218", + "task_file": "dev/tasks/DEV-000219-reachability-not-quality.json", + "task_id": "DEV-000219", "task_revision": 1 } diff --git a/dev/tasks/DEV-000219-reachability-not-quality.json b/dev/tasks/DEV-000219-reachability-not-quality.json new file mode 100644 index 0000000..b28b521 --- /dev/null +++ b/dev/tasks/DEV-000219-reachability-not-quality.json @@ -0,0 +1,23 @@ +{ + "task_id": "DEV-000219", + "task_revision": 1, + "architecture_revision": "ADR-0018", + "base_repo_revision": "7476f43ffa57d55f5a5a9bddc7f1c74e9b87ae0e", + "risk_tier": "E", + "intent": "Prevent task-evaluation invocation success from being recorded as semantic model quality or support without independent verifier evidence.", + "requirement_ids": ["truthful model evidence", "independent verification", "no false qualification"], + "allowed_change_scope": {"modules": ["cognitive-worker"], "paths": ["cmd/cognitive-worker/task_evaluation.go", "cmd/cognitive-worker/task_evaluation_test.go", "dev/ACTIVE_TASK.json", "dev/tasks/DEV-000219-reachability-not-quality.json"]}, + "forbidden_change_scope": {"paths": ["architecture/**", "security/**", "database/**", "deploy/**", "contracts/**", "modules/**", "LLM-local/**"]}, + "allowed_architecture_delta": {"new_cross_module_dependencies": 0, "new_canonical_owners": 0, "new_privileges": 0, "new_third_party_dependencies": 0, "new_database_owner_roles": 0}, + "contracts": {"consume": ["TaskEvaluationExecutionRequest/Response", "TaskCaseObservation", "Cognitive Gateway"], "may_change": []}, + "semantic_owner": "capability evidence remains owner; cognitive-worker only records invocation observation", + "canonical_state": ["existing task-evaluation evidence; no new persistence"], + "state_transitions": ["successful invocation -> UNKNOWN reachability observation", "independent verifier evidence -> future supported/quality observation"], + "side_effects_and_privileges": ["no additional provider calls or privileges"], + "persistence_and_migrations": ["none"], + "failure_retry_restart": ["unknown evidence remains retryable/eligible for a separately verified evaluation; no promotion follows invocation alone"], + "assumptions": ["a later verifier task will attach answer-quality evidence and score cases"], + "acceptance_scenarios": ["nonempty provider response is not SUPPORTED", "quality score remains zero", "provider invocation failure remains UNKNOWN with critical failure", "no model/hardware label can manufacture quality evidence"], + "required_tests": ["go test", "go vet", "preflight", "check-diff", "check-architecture", "independent", "git diff --check", "CI", "production public health check"], + "terminal_gap_states": ["BLOCKED_BY_MISSING_CAPABILITY", "BLOCKED_BY_SPEC_GAP"] +} -- 2.54.0