fix(cognitive): separate reachability from quality evidence
This commit is contained in:
@@ -68,9 +68,10 @@ func (g cognitiveGateway) taskEvaluationHandler(w http.ResponseWriter, r *http.R
|
||||
response, invokeErr := g.invokeAdmitted(r.Context(), string(invocation.RequestID), provider, invocation, int(request.MaxOutputBytes))
|
||||
observation := capability.TaskCaseObservation{CaseID: item.CaseID, LatencyMicros: time.Since(started).Microseconds(), EvidenceArtifactRef: item.EvidenceArtifactRef}
|
||||
if invokeErr == nil && response.FinishReason == cognitive.FinishCompleted && response.AdvisoryText != "" {
|
||||
observation.Result = capability.EvaluationSupported
|
||||
// Invocation success proves reachability/support for this case,
|
||||
// not semantic answer quality. A verifier may score quality later.
|
||||
// Invocation success proves reachability only. Without an
|
||||
// independent verifier and answer evidence, semantic support and
|
||||
// quality must remain UNKNOWN rather than becoming false evidence.
|
||||
observation.Result = capability.EvaluationUnknown
|
||||
observation.QualityScoreMillis = 0
|
||||
} else {
|
||||
observation.Result = capability.EvaluationUnknown
|
||||
|
||||
@@ -47,7 +47,12 @@ func TestTaskEvaluationHandlerInvokesEveryBoundedCase(t *testing.T) {
|
||||
if err := json.Unmarshal(recorder.Body.Bytes(), &response); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(response.Observations) != 2 || response.Observations[0].Result != capability.EvaluationSupported || response.Observations[1].Result != capability.EvaluationSupported {
|
||||
if len(response.Observations) != 2 || response.Observations[0].Result != capability.EvaluationUnknown || response.Observations[1].Result != capability.EvaluationUnknown {
|
||||
t.Fatalf("unexpected observations: %+v", response.Observations)
|
||||
}
|
||||
for _, observation := range response.Observations {
|
||||
if observation.QualityScoreMillis != 0 || observation.CriticalFailure {
|
||||
t.Fatalf("reachability-only evidence must not carry quality or failure claims: %+v", observation)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user