fix(cognitive): fail over task evaluation candidates
CI / test (push) Successful in 1m17s
CI / test (pull_request) Successful in 50s

This commit is contained in:
m
2026-08-23 20:49:12 +03:00
parent f90ed7af35
commit c04a154d1e
6 changed files with 235 additions and 33 deletions
@@ -50,6 +50,17 @@ func (e taskEvaluationExecutor) Execute(ctx context.Context, request work.AgentE
if len(response.Responses) == 0 || response.ObservedAtUnixMillis < 1 {
return work.AgentExecutionResult{}, fmt.Errorf("task evaluation returned no observations")
}
if len(response.Observations) == len(intent.TaskEvaluationCases) && taskEvaluationHasCriticalFailure(response.Observations) {
evaluation, err := capabilitydomain.BuildTaskEvaluation(*intent.TaskEvaluationProfile, capability.TaskEvaluationID("evaluation:"+intent.DispatchID), response.BindingID, response.BindingRevision, intent.TaskEvaluationProfile.CapabilityID, response.Observations, artifact.ArtifactID(intent.TaskEvaluationCases[0].EvidenceArtifactRef), response.ObservedAtUnixMillis)
if err != nil {
return work.AgentExecutionResult{}, err
}
callback := work.TaskEvaluationEvidenceCallback{GraphID: execution.GraphID, Revision: execution.GraphRevision, NodeID: execution.NodeID, Evaluation: evaluation, Succeeded: true}
if _, err := postJSON[json.RawMessage](ctx, e.client, e.callbackURL+"/v1/work-graphs/task-evaluations/callback", callback); err != nil {
return work.AgentExecutionResult{}, err
}
return work.AgentExecutionResult{State: work.AgentExecutionCompleted, Summary: "task evaluation recorded as UNKNOWN after bounded provider recovery", OutcomeRef: string(evaluation.ID), ArtifactRefs: []string{string(evaluation.ArtifactID)}, ObservedAtUnixMillis: time.Now().UnixMilli()}, nil
}
if len(intent.TaskEvaluationCases) == 0 {
return work.AgentExecutionResult{}, fmt.Errorf("task evaluation cases are missing")
}
@@ -80,6 +91,15 @@ func (e taskEvaluationExecutor) Execute(ctx context.Context, request work.AgentE
return work.AgentExecutionResult{State: work.AgentExecutionCompleted, Summary: "task evaluation executed, aggregated and accepted by Work Graph callback", OutcomeRef: string(evaluation.ID), ArtifactRefs: []string{string(evaluation.ArtifactID)}, ObservedAtUnixMillis: time.Now().UnixMilli()}, nil
}
func taskEvaluationHasCriticalFailure(observations []capability.TaskCaseObservation) bool {
for _, observation := range observations {
if observation.CriticalFailure {
return true
}
}
return false
}
func postJSON[T any](ctx context.Context, client *http.Client, url string, payload any) (T, error) {
var result T
body, err := json.Marshal(payload)
@@ -41,3 +41,32 @@ func TestTaskEvaluationExecutorInvokesAggregatesAndCallbacks(t *testing.T) {
t.Fatalf("result=%+v err=%v", result, err)
}
}
func TestTaskEvaluationExecutorPersistsUnknownAfterProviderFailure(t *testing.T) {
profile := capability.TaskEvaluationProfile{ID: "profile-unknown", Revision: 1, CapabilityID: "capability-1", Stage: capability.TaskEvaluationGeneral, BenchmarkVersion: "unknown-v1", RequiredCaseIDs: []string{"case-1"}, QualityWeightMillis: 600, ReliabilityWeightMillis: 300, LatencyWeightMillis: 100}
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path == "/v1/task-evaluations/execute" {
_ = json.NewEncoder(w).Encode(capability.TaskEvaluationExecutionResponse{ProfileID: profile.ID, BindingID: "binding-1", BindingRevision: 3, Observations: []capability.TaskCaseObservation{{CaseID: "case-1", Result: capability.EvaluationUnknown, CriticalFailure: true, EvidenceArtifactRef: "artifact-1"}}, Responses: []capability.TaskCaseResponse{{CaseID: "case-1", EvidenceArtifactRef: "artifact-1"}}, ObservedAtUnixMillis: 10})
return
}
if r.URL.Path == "/v1/work-graphs/task-evaluations/callback" {
var callback work.TaskEvaluationEvidenceCallback
if err := json.NewDecoder(r.Body).Decode(&callback); err != nil || callback.Evaluation.Result != capability.EvaluationUnknown || !callback.Succeeded {
http.Error(w, "invalid unknown callback", http.StatusBadRequest)
return
}
w.WriteHeader(http.StatusNoContent)
return
}
if r.URL.Path == "/v1/task-evaluations/verify" {
t.Errorf("verifier must not be called for a provider failure")
}
http.NotFound(w, r)
}))
defer server.Close()
executor := newTaskEvaluationExecutor(server.URL, server.URL, server.URL, server.Client())
result, err := executor.Execute(context.Background(), work.AgentExecutionRequest{Intent: work.AgentDispatchIntent{DispatchID: "dispatch-unknown", TaskEvaluationBindingID: "binding-1", TaskEvaluationProfile: &profile, TaskEvaluationCases: []capability.TaskEvaluationCase{{CaseID: "case-1", Prompt: "answer", EvidenceArtifactRef: "artifact-1"}}, TaskEvaluationExecution: &work.TaskEvaluationDispatchContext{GraphID: "graph-1", GraphRevision: 1, NodeID: "node-1", BindingRevision: 3, ActorID: "actor-1", PrincipalID: "principal-1", ScopeID: "scope-1"}}})
if err != nil || result.State != work.AgentExecutionCompleted || result.OutcomeRef == "" {
t.Fatalf("result=%+v err=%v", result, err)
}
}