feat(work): add independent task verifier runtime
CI / test (push) Successful in 1m6s
CI / test (pull_request) Successful in 1m7s

This commit is contained in:
m
2026-08-23 14:19:38 +03:00
parent 0aab10863c
commit 33b51a4445
11 changed files with 209 additions and 13 deletions
+1 -1
View File
@@ -52,7 +52,7 @@ func main() {
// The local evaluator keeps task-qualification work alive, but it is not
// an external Agent Execution Bridge and must never be reported as one.
localFallback = true
executor = newTaskEvaluationExecutor(envOr("COGNITIVE_WORKER_URL", "http://cognitive-worker:8083"), envOr("WORK_GRAPH_RUNTIME_URL", "http://work-graph-runtime:8088"), &http.Client{Timeout: *leaseTTL})
executor = newTaskEvaluationExecutor(envOr("COGNITIVE_WORKER_URL", "http://cognitive-worker:8083"), envOr("WORK_GRAPH_RUNTIME_URL", "http://work-graph-runtime:8088"), envOr("TASK_VERIFIER_URL", "http://task-verifier:8091"), &http.Client{Timeout: *leaseTTL})
fmt.Fprintln(os.Stderr, "protected agent execution bridge URL is not configured; task evaluations use the local durable evaluator, other work remains in fallback")
}
go func() {
+22 -5
View File
@@ -18,15 +18,16 @@ import (
type taskEvaluationExecutor struct {
cognitiveURL string
callbackURL string
verifierURL string
client *http.Client
fallback localFallbackExecutor
}
func newTaskEvaluationExecutor(cognitiveURL, callbackURL string, client *http.Client) taskEvaluationExecutor {
func newTaskEvaluationExecutor(cognitiveURL, callbackURL, verifierURL string, client *http.Client) taskEvaluationExecutor {
if client == nil {
client = http.DefaultClient
}
return taskEvaluationExecutor{cognitiveURL: strings.TrimRight(cognitiveURL, "/"), callbackURL: strings.TrimRight(callbackURL, "/"), client: client, fallback: newLocalFallbackExecutor()}
return taskEvaluationExecutor{cognitiveURL: strings.TrimRight(cognitiveURL, "/"), callbackURL: strings.TrimRight(callbackURL, "/"), verifierURL: strings.TrimRight(verifierURL, "/"), client: client, fallback: newLocalFallbackExecutor()}
}
func (e taskEvaluationExecutor) Execute(ctx context.Context, request work.AgentExecutionRequest) (work.AgentExecutionResult, error) {
@@ -38,7 +39,7 @@ func (e taskEvaluationExecutor) Execute(ctx context.Context, request work.AgentE
if execution.GraphID == "" || execution.NodeID == "" || execution.BindingRevision < 1 || execution.ActorID == "" || execution.PrincipalID == "" || execution.ScopeID == "" {
return e.fallback.Execute(ctx, request)
}
if e.cognitiveURL == "" || e.callbackURL == "" {
if e.cognitiveURL == "" || e.callbackURL == "" || e.verifierURL == "" {
return e.fallback.Execute(ctx, request)
}
payload := capability.TaskEvaluationExecutionRequest{Profile: *intent.TaskEvaluationProfile, CapabilityID: intent.TaskEvaluationProfile.CapabilityID, BindingID: intent.TaskEvaluationBindingID, BindingRevision: execution.BindingRevision, ActorID: execution.ActorID, PrincipalID: execution.PrincipalID, ScopeID: execution.ScopeID, Cases: intent.TaskEvaluationCases, MaxOutputBytes: 4096}
@@ -46,13 +47,29 @@ func (e taskEvaluationExecutor) Execute(ctx context.Context, request work.AgentE
if err != nil {
return work.AgentExecutionResult{}, err
}
if len(response.Observations) == 0 || response.ObservedAtUnixMillis < 1 {
if len(response.Responses) == 0 || response.ObservedAtUnixMillis < 1 {
return work.AgentExecutionResult{}, fmt.Errorf("task evaluation returned no observations")
}
if len(intent.TaskEvaluationCases) == 0 {
return work.AgentExecutionResult{}, fmt.Errorf("task evaluation cases are missing")
}
evaluation, err := capabilitydomain.BuildTaskEvaluation(*intent.TaskEvaluationProfile, capability.TaskEvaluationID("evaluation:"+intent.DispatchID), response.BindingID, response.BindingRevision, intent.TaskEvaluationProfile.CapabilityID, response.Observations, artifact.ArtifactID(intent.TaskEvaluationCases[0].EvidenceArtifactRef), response.ObservedAtUnixMillis)
caseByID := make(map[string]capability.TaskEvaluationCase, len(intent.TaskEvaluationCases))
for _, item := range intent.TaskEvaluationCases {
caseByID[item.CaseID] = item
}
observations := make([]capability.TaskCaseObservation, 0, len(response.Responses))
for _, item := range response.Responses {
caseValue, ok := caseByID[item.CaseID]
if !ok || caseValue.Verification.Mode == "" {
return e.fallback.Execute(ctx, request)
}
verified, verifyErr := postJSON[capability.TaskCaseVerificationResponse](ctx, e.client, e.verifierURL+"/v1/task-evaluations/verify", capability.TaskCaseVerificationRequest{CaseID: item.CaseID, ResponseText: item.ResponseText, LatencyMicros: item.LatencyMicros, EvidenceArtifactRef: item.EvidenceArtifactRef, Verification: caseValue.Verification})
if verifyErr != nil {
return e.fallback.Execute(ctx, request)
}
observations = append(observations, capability.TaskCaseObservation{CaseID: verified.CaseID, Result: verified.Result, QualityScoreMillis: verified.QualityScoreMillis, LatencyMicros: verified.LatencyMicros, CriticalFailure: verified.CriticalFailure, EvidenceArtifactRef: verified.EvidenceArtifactRef, VerificationEvidenceRef: verified.VerificationEvidenceRef, VerifierIdentity: verified.VerifierIdentity})
}
evaluation, err := capabilitydomain.BuildTaskEvaluation(*intent.TaskEvaluationProfile, capability.TaskEvaluationID("evaluation:"+intent.DispatchID), response.BindingID, response.BindingRevision, intent.TaskEvaluationProfile.CapabilityID, observations, artifact.ArtifactID(intent.TaskEvaluationCases[0].EvidenceArtifactRef), response.ObservedAtUnixMillis)
if err != nil {
return work.AgentExecutionResult{}, err
}
@@ -13,9 +13,14 @@ import (
func TestTaskEvaluationExecutorInvokesAggregatesAndCallbacks(t *testing.T) {
profile := capability.TaskEvaluationProfile{ID: "profile-1", Revision: 1, CapabilityID: "capability-1", Stage: capability.TaskEvaluationGeneral, BenchmarkVersion: "general-v1", RequiredCaseIDs: []string{"case-1"}, QualityWeightMillis: 600, ReliabilityWeightMillis: 300, LatencyWeightMillis: 100}
verification := capability.TaskCaseVerification{Mode: capability.TaskCaseVerificationExactText, ExpectedText: "answer", MinimumQualityMillis: 900}
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path == "/v1/task-evaluations/execute" {
_ = json.NewEncoder(w).Encode(capability.TaskEvaluationExecutionResponse{ProfileID: profile.ID, BindingID: "binding-1", BindingRevision: 3, Observations: []capability.TaskCaseObservation{{CaseID: "case-1", Result: capability.EvaluationSupported, QualityScoreMillis: 800, EvidenceArtifactRef: "artifact-1", VerificationEvidenceRef: "verify-1", VerifierIdentity: "verifier-1"}}, ObservedAtUnixMillis: 10})
_ = json.NewEncoder(w).Encode(capability.TaskEvaluationExecutionResponse{ProfileID: profile.ID, BindingID: "binding-1", BindingRevision: 3, Responses: []capability.TaskCaseResponse{{CaseID: "case-1", ResponseText: "answer", LatencyMicros: 2, EvidenceArtifactRef: "artifact-1"}}, ObservedAtUnixMillis: 10})
return
}
if r.URL.Path == "/v1/task-evaluations/verify" {
_ = json.NewEncoder(w).Encode(capability.TaskCaseVerificationResponse{CaseID: "case-1", Result: capability.EvaluationSupported, QualityScoreMillis: 900, LatencyMicros: 2, EvidenceArtifactRef: "artifact-1", VerificationEvidenceRef: "verify-1", VerifierIdentity: "verifier-1"})
return
}
if r.URL.Path == "/v1/work-graphs/task-evaluations/callback" {
@@ -30,8 +35,8 @@ func TestTaskEvaluationExecutorInvokesAggregatesAndCallbacks(t *testing.T) {
http.NotFound(w, r)
}))
defer server.Close()
executor := newTaskEvaluationExecutor(server.URL, server.URL, server.Client())
result, err := executor.Execute(context.Background(), work.AgentExecutionRequest{Intent: work.AgentDispatchIntent{DispatchID: "dispatch-1", TaskEvaluationBindingID: "binding-1", TaskEvaluationProfile: &profile, TaskEvaluationCases: []capability.TaskEvaluationCase{{CaseID: "case-1", Prompt: "answer", EvidenceArtifactRef: "artifact-1"}}, TaskEvaluationExecution: &work.TaskEvaluationDispatchContext{GraphID: "graph-1", GraphRevision: 1, NodeID: "node-1", BindingRevision: 3, ActorID: "actor-1", PrincipalID: "principal-1", ScopeID: "scope-1"}}})
executor := newTaskEvaluationExecutor(server.URL, server.URL, server.URL, server.Client())
result, err := executor.Execute(context.Background(), work.AgentExecutionRequest{Intent: work.AgentDispatchIntent{DispatchID: "dispatch-1", TaskEvaluationBindingID: "binding-1", TaskEvaluationProfile: &profile, TaskEvaluationCases: []capability.TaskEvaluationCase{{CaseID: "case-1", Prompt: "answer", EvidenceArtifactRef: "artifact-1", Verification: verification}}, TaskEvaluationExecution: &work.TaskEvaluationDispatchContext{GraphID: "graph-1", GraphRevision: 1, NodeID: "node-1", BindingRevision: 3, ActorID: "actor-1", PrincipalID: "principal-1", ScopeID: "scope-1"}}})
if err != nil || result.State != work.AgentExecutionCompleted || result.OutcomeRef == "" {
t.Fatalf("result=%+v err=%v", result, err)
}
+3 -1
View File
@@ -44,6 +44,7 @@ func (g cognitiveGateway) taskEvaluationHandler(w http.ResponseWriter, r *http.R
caseSet[id] = struct{}{}
}
observations := make([]capability.TaskCaseObservation, len(request.Cases))
responses := make([]capability.TaskCaseResponse, len(request.Cases))
var wg sync.WaitGroup
seenCases := make(map[string]struct{}, len(request.Cases))
for i, item := range request.Cases {
@@ -67,6 +68,7 @@ func (g cognitiveGateway) taskEvaluationHandler(w http.ResponseWriter, r *http.R
invocation := cognitive.AdvisoryRequest{Version: cognitive.CreatorAdvisoryV1, RequestID: cognitive.RequestID(fmt.Sprintf("task-evaluation:%s:%s", request.Profile.ID, item.CaseID)), ActorID: request.ActorID, PrincipalID: request.PrincipalID, ScopeID: request.ScopeID, Message: item.Prompt, MaxOutputBytes: int(request.MaxOutputBytes), ProviderBindingID: selected.Binding.ID, ProviderBindingRevision: selected.Binding.Revision, ProviderInstanceID: selected.Provider.ID, ModelDescriptorID: selected.Model.ID, ProviderModelID: selected.Model.ProviderModelID}
response, invokeErr := g.invokeAdmitted(r.Context(), string(invocation.RequestID), provider, invocation, int(request.MaxOutputBytes))
observation := capability.TaskCaseObservation{CaseID: item.CaseID, LatencyMicros: time.Since(started).Microseconds(), EvidenceArtifactRef: item.EvidenceArtifactRef}
responses[index] = capability.TaskCaseResponse{CaseID: item.CaseID, ResponseText: response.AdvisoryText, LatencyMicros: observation.LatencyMicros, EvidenceArtifactRef: item.EvidenceArtifactRef}
if invokeErr == nil && response.FinishReason == cognitive.FinishCompleted && response.AdvisoryText != "" {
// Invocation success proves reachability only. Without an
// independent verifier and answer evidence, semantic support and
@@ -85,7 +87,7 @@ func (g cognitiveGateway) taskEvaluationHandler(w http.ResponseWriter, r *http.R
return
}
wg.Wait()
writeTaskEvaluationJSON(w, capability.TaskEvaluationExecutionResponse{ProfileID: request.Profile.ID, BindingID: selected.Binding.ID, BindingRevision: selected.Binding.Revision, Observations: observations, ObservedAtUnixMillis: time.Now().UnixMilli()})
writeTaskEvaluationJSON(w, capability.TaskEvaluationExecutionResponse{ProfileID: request.Profile.ID, BindingID: selected.Binding.ID, BindingRevision: selected.Binding.Revision, Observations: observations, Responses: responses, ObservedAtUnixMillis: time.Now().UnixMilli()})
}
func writeTaskEvaluationJSON(w http.ResponseWriter, value any) {
+64
View File
@@ -0,0 +1,64 @@
package main
import (
"crypto/sha256"
"encoding/hex"
"encoding/json"
"fmt"
"net/http"
"os"
"strings"
"git.cin.su/m/Assistant/contracts/capability"
)
const verifierIdentity = "deterministic-task-verifier-v1"
func main() {
mux := http.NewServeMux()
mux.HandleFunc("GET /healthz", func(w http.ResponseWriter, _ *http.Request) {
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(`{"runtime":"task-verifier","status":"ok"}`))
})
mux.HandleFunc("POST /v1/task-evaluations/verify", verifyHandler)
addr := ":8091"
if value := strings.TrimSpace(os.Getenv("LISTEN_ADDR")); value != "" {
addr = value
}
if err := http.ListenAndServe(addr, mux); err != nil {
panic(err)
}
}
func verifyHandler(w http.ResponseWriter, r *http.Request) {
var request capability.TaskCaseVerificationRequest
decoder := json.NewDecoder(http.MaxBytesReader(w, r.Body, 64<<10))
decoder.DisallowUnknownFields()
if err := decoder.Decode(&request); err != nil || request.CaseID == "" || request.ResponseText == "" || request.EvidenceArtifactRef == "" || request.LatencyMicros < 0 || request.Verification.Mode != capability.TaskCaseVerificationExactText || request.Verification.ExpectedText == "" || request.Verification.MinimumQualityMillis < 0 || request.Verification.MinimumQualityMillis > 1000 {
http.Error(w, "invalid task verification request", http.StatusBadRequest)
return
}
matched := strings.TrimSpace(request.ResponseText) == strings.TrimSpace(request.Verification.ExpectedText)
quality := int64(0)
result := capability.EvaluationUnsupported
if matched {
quality = request.Verification.MinimumQualityMillis
if quality == 0 {
quality = 1000
}
result = capability.EvaluationSupported
}
evidence := verificationEvidence(request, matched)
writeJSON(w, capability.TaskCaseVerificationResponse{CaseID: request.CaseID, Result: result, QualityScoreMillis: quality, LatencyMicros: request.LatencyMicros, EvidenceArtifactRef: request.EvidenceArtifactRef, VerificationEvidenceRef: evidence, VerifierIdentity: verifierIdentity})
}
func verificationEvidence(request capability.TaskCaseVerificationRequest, matched bool) string {
payload := fmt.Sprintf("%s|%s|%s|%t", verifierIdentity, request.CaseID, request.EvidenceArtifactRef, matched)
digest := sha256.Sum256([]byte(payload))
return "verification:" + hex.EncodeToString(digest[:])
}
func writeJSON(w http.ResponseWriter, value any) {
w.Header().Set("Content-Type", "application/json")
_ = json.NewEncoder(w).Encode(value)
}
+31
View File
@@ -0,0 +1,31 @@
package main
import (
"bytes"
"encoding/json"
"net/http"
"net/http/httptest"
"testing"
"git.cin.su/m/Assistant/contracts/capability"
)
func TestVerifyExactTextProducesIndependentEvidence(t *testing.T) {
payload, _ := json.Marshal(capability.TaskCaseVerificationRequest{CaseID: "case-1", ResponseText: " answer ", EvidenceArtifactRef: "case-artifact", Verification: capability.TaskCaseVerification{Mode: capability.TaskCaseVerificationExactText, ExpectedText: "answer", MinimumQualityMillis: 900}})
recorder := httptest.NewRecorder()
verifyHandler(recorder, httptest.NewRequest(http.MethodPost, "/v1/task-evaluations/verify", bytes.NewReader(payload)))
var response capability.TaskCaseVerificationResponse
if recorder.Code != http.StatusOK || json.NewDecoder(recorder.Body).Decode(&response) != nil || response.Result != capability.EvaluationSupported || response.QualityScoreMillis != 900 || response.VerifierIdentity == "" || response.VerificationEvidenceRef == "" {
t.Fatalf("unexpected verification response: code=%d body=%s value=%+v", recorder.Code, recorder.Body.String(), response)
}
}
func TestVerifyMismatchIsUnsupportedWithEvidence(t *testing.T) {
payload, _ := json.Marshal(capability.TaskCaseVerificationRequest{CaseID: "case-1", ResponseText: "wrong", EvidenceArtifactRef: "case-artifact", Verification: capability.TaskCaseVerification{Mode: capability.TaskCaseVerificationExactText, ExpectedText: "answer", MinimumQualityMillis: 900}})
recorder := httptest.NewRecorder()
verifyHandler(recorder, httptest.NewRequest(http.MethodPost, "/v1/task-evaluations/verify", bytes.NewReader(payload)))
var response capability.TaskCaseVerificationResponse
if recorder.Code != http.StatusOK || json.NewDecoder(recorder.Body).Decode(&response) != nil || response.Result != capability.EvaluationUnsupported || response.QualityScoreMillis != 0 || response.VerificationEvidenceRef == "" {
t.Fatalf("unexpected mismatch response: code=%d body=%s value=%+v", recorder.Code, recorder.Body.String(), response)
}
}