feat(work): add independent task verifier runtime #198

Merged
m merged 1 commits from codex/wp-221-verifier-runtime into main 2026-08-23 11:21:11 +00:00
11 changed files with 209 additions and 13 deletions
+1 -1
View File
@@ -52,7 +52,7 @@ func main() {
// The local evaluator keeps task-qualification work alive, but it is not
// an external Agent Execution Bridge and must never be reported as one.
localFallback = true
executor = newTaskEvaluationExecutor(envOr("COGNITIVE_WORKER_URL", "http://cognitive-worker:8083"), envOr("WORK_GRAPH_RUNTIME_URL", "http://work-graph-runtime:8088"), &http.Client{Timeout: *leaseTTL})
executor = newTaskEvaluationExecutor(envOr("COGNITIVE_WORKER_URL", "http://cognitive-worker:8083"), envOr("WORK_GRAPH_RUNTIME_URL", "http://work-graph-runtime:8088"), envOr("TASK_VERIFIER_URL", "http://task-verifier:8091"), &http.Client{Timeout: *leaseTTL})
fmt.Fprintln(os.Stderr, "protected agent execution bridge URL is not configured; task evaluations use the local durable evaluator, other work remains in fallback")
}
go func() {
+22 -5
View File
@@ -18,15 +18,16 @@ import (
type taskEvaluationExecutor struct {
cognitiveURL string
callbackURL string
verifierURL string
client *http.Client
fallback localFallbackExecutor
}
func newTaskEvaluationExecutor(cognitiveURL, callbackURL string, client *http.Client) taskEvaluationExecutor {
func newTaskEvaluationExecutor(cognitiveURL, callbackURL, verifierURL string, client *http.Client) taskEvaluationExecutor {
if client == nil {
client = http.DefaultClient
}
return taskEvaluationExecutor{cognitiveURL: strings.TrimRight(cognitiveURL, "/"), callbackURL: strings.TrimRight(callbackURL, "/"), client: client, fallback: newLocalFallbackExecutor()}
return taskEvaluationExecutor{cognitiveURL: strings.TrimRight(cognitiveURL, "/"), callbackURL: strings.TrimRight(callbackURL, "/"), verifierURL: strings.TrimRight(verifierURL, "/"), client: client, fallback: newLocalFallbackExecutor()}
}
func (e taskEvaluationExecutor) Execute(ctx context.Context, request work.AgentExecutionRequest) (work.AgentExecutionResult, error) {
@@ -38,7 +39,7 @@ func (e taskEvaluationExecutor) Execute(ctx context.Context, request work.AgentE
if execution.GraphID == "" || execution.NodeID == "" || execution.BindingRevision < 1 || execution.ActorID == "" || execution.PrincipalID == "" || execution.ScopeID == "" {
return e.fallback.Execute(ctx, request)
}
if e.cognitiveURL == "" || e.callbackURL == "" {
if e.cognitiveURL == "" || e.callbackURL == "" || e.verifierURL == "" {
return e.fallback.Execute(ctx, request)
}
payload := capability.TaskEvaluationExecutionRequest{Profile: *intent.TaskEvaluationProfile, CapabilityID: intent.TaskEvaluationProfile.CapabilityID, BindingID: intent.TaskEvaluationBindingID, BindingRevision: execution.BindingRevision, ActorID: execution.ActorID, PrincipalID: execution.PrincipalID, ScopeID: execution.ScopeID, Cases: intent.TaskEvaluationCases, MaxOutputBytes: 4096}
@@ -46,13 +47,29 @@ func (e taskEvaluationExecutor) Execute(ctx context.Context, request work.AgentE
if err != nil {
return work.AgentExecutionResult{}, err
}
if len(response.Observations) == 0 || response.ObservedAtUnixMillis < 1 {
if len(response.Responses) == 0 || response.ObservedAtUnixMillis < 1 {
return work.AgentExecutionResult{}, fmt.Errorf("task evaluation returned no observations")
}
if len(intent.TaskEvaluationCases) == 0 {
return work.AgentExecutionResult{}, fmt.Errorf("task evaluation cases are missing")
}
evaluation, err := capabilitydomain.BuildTaskEvaluation(*intent.TaskEvaluationProfile, capability.TaskEvaluationID("evaluation:"+intent.DispatchID), response.BindingID, response.BindingRevision, intent.TaskEvaluationProfile.CapabilityID, response.Observations, artifact.ArtifactID(intent.TaskEvaluationCases[0].EvidenceArtifactRef), response.ObservedAtUnixMillis)
caseByID := make(map[string]capability.TaskEvaluationCase, len(intent.TaskEvaluationCases))
for _, item := range intent.TaskEvaluationCases {
caseByID[item.CaseID] = item
}
observations := make([]capability.TaskCaseObservation, 0, len(response.Responses))
for _, item := range response.Responses {
caseValue, ok := caseByID[item.CaseID]
if !ok || caseValue.Verification.Mode == "" {
return e.fallback.Execute(ctx, request)
}
verified, verifyErr := postJSON[capability.TaskCaseVerificationResponse](ctx, e.client, e.verifierURL+"/v1/task-evaluations/verify", capability.TaskCaseVerificationRequest{CaseID: item.CaseID, ResponseText: item.ResponseText, LatencyMicros: item.LatencyMicros, EvidenceArtifactRef: item.EvidenceArtifactRef, Verification: caseValue.Verification})
if verifyErr != nil {
return e.fallback.Execute(ctx, request)
}
observations = append(observations, capability.TaskCaseObservation{CaseID: verified.CaseID, Result: verified.Result, QualityScoreMillis: verified.QualityScoreMillis, LatencyMicros: verified.LatencyMicros, CriticalFailure: verified.CriticalFailure, EvidenceArtifactRef: verified.EvidenceArtifactRef, VerificationEvidenceRef: verified.VerificationEvidenceRef, VerifierIdentity: verified.VerifierIdentity})
}
evaluation, err := capabilitydomain.BuildTaskEvaluation(*intent.TaskEvaluationProfile, capability.TaskEvaluationID("evaluation:"+intent.DispatchID), response.BindingID, response.BindingRevision, intent.TaskEvaluationProfile.CapabilityID, observations, artifact.ArtifactID(intent.TaskEvaluationCases[0].EvidenceArtifactRef), response.ObservedAtUnixMillis)
if err != nil {
return work.AgentExecutionResult{}, err
}
@@ -13,9 +13,14 @@ import (
func TestTaskEvaluationExecutorInvokesAggregatesAndCallbacks(t *testing.T) {
profile := capability.TaskEvaluationProfile{ID: "profile-1", Revision: 1, CapabilityID: "capability-1", Stage: capability.TaskEvaluationGeneral, BenchmarkVersion: "general-v1", RequiredCaseIDs: []string{"case-1"}, QualityWeightMillis: 600, ReliabilityWeightMillis: 300, LatencyWeightMillis: 100}
verification := capability.TaskCaseVerification{Mode: capability.TaskCaseVerificationExactText, ExpectedText: "answer", MinimumQualityMillis: 900}
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path == "/v1/task-evaluations/execute" {
_ = json.NewEncoder(w).Encode(capability.TaskEvaluationExecutionResponse{ProfileID: profile.ID, BindingID: "binding-1", BindingRevision: 3, Observations: []capability.TaskCaseObservation{{CaseID: "case-1", Result: capability.EvaluationSupported, QualityScoreMillis: 800, EvidenceArtifactRef: "artifact-1", VerificationEvidenceRef: "verify-1", VerifierIdentity: "verifier-1"}}, ObservedAtUnixMillis: 10})
_ = json.NewEncoder(w).Encode(capability.TaskEvaluationExecutionResponse{ProfileID: profile.ID, BindingID: "binding-1", BindingRevision: 3, Responses: []capability.TaskCaseResponse{{CaseID: "case-1", ResponseText: "answer", LatencyMicros: 2, EvidenceArtifactRef: "artifact-1"}}, ObservedAtUnixMillis: 10})
return
}
if r.URL.Path == "/v1/task-evaluations/verify" {
_ = json.NewEncoder(w).Encode(capability.TaskCaseVerificationResponse{CaseID: "case-1", Result: capability.EvaluationSupported, QualityScoreMillis: 900, LatencyMicros: 2, EvidenceArtifactRef: "artifact-1", VerificationEvidenceRef: "verify-1", VerifierIdentity: "verifier-1"})
return
}
if r.URL.Path == "/v1/work-graphs/task-evaluations/callback" {
@@ -30,8 +35,8 @@ func TestTaskEvaluationExecutorInvokesAggregatesAndCallbacks(t *testing.T) {
http.NotFound(w, r)
}))
defer server.Close()
executor := newTaskEvaluationExecutor(server.URL, server.URL, server.Client())
result, err := executor.Execute(context.Background(), work.AgentExecutionRequest{Intent: work.AgentDispatchIntent{DispatchID: "dispatch-1", TaskEvaluationBindingID: "binding-1", TaskEvaluationProfile: &profile, TaskEvaluationCases: []capability.TaskEvaluationCase{{CaseID: "case-1", Prompt: "answer", EvidenceArtifactRef: "artifact-1"}}, TaskEvaluationExecution: &work.TaskEvaluationDispatchContext{GraphID: "graph-1", GraphRevision: 1, NodeID: "node-1", BindingRevision: 3, ActorID: "actor-1", PrincipalID: "principal-1", ScopeID: "scope-1"}}})
executor := newTaskEvaluationExecutor(server.URL, server.URL, server.URL, server.Client())
result, err := executor.Execute(context.Background(), work.AgentExecutionRequest{Intent: work.AgentDispatchIntent{DispatchID: "dispatch-1", TaskEvaluationBindingID: "binding-1", TaskEvaluationProfile: &profile, TaskEvaluationCases: []capability.TaskEvaluationCase{{CaseID: "case-1", Prompt: "answer", EvidenceArtifactRef: "artifact-1", Verification: verification}}, TaskEvaluationExecution: &work.TaskEvaluationDispatchContext{GraphID: "graph-1", GraphRevision: 1, NodeID: "node-1", BindingRevision: 3, ActorID: "actor-1", PrincipalID: "principal-1", ScopeID: "scope-1"}}})
if err != nil || result.State != work.AgentExecutionCompleted || result.OutcomeRef == "" {
t.Fatalf("result=%+v err=%v", result, err)
}
+3 -1
View File
@@ -44,6 +44,7 @@ func (g cognitiveGateway) taskEvaluationHandler(w http.ResponseWriter, r *http.R
caseSet[id] = struct{}{}
}
observations := make([]capability.TaskCaseObservation, len(request.Cases))
responses := make([]capability.TaskCaseResponse, len(request.Cases))
var wg sync.WaitGroup
seenCases := make(map[string]struct{}, len(request.Cases))
for i, item := range request.Cases {
@@ -67,6 +68,7 @@ func (g cognitiveGateway) taskEvaluationHandler(w http.ResponseWriter, r *http.R
invocation := cognitive.AdvisoryRequest{Version: cognitive.CreatorAdvisoryV1, RequestID: cognitive.RequestID(fmt.Sprintf("task-evaluation:%s:%s", request.Profile.ID, item.CaseID)), ActorID: request.ActorID, PrincipalID: request.PrincipalID, ScopeID: request.ScopeID, Message: item.Prompt, MaxOutputBytes: int(request.MaxOutputBytes), ProviderBindingID: selected.Binding.ID, ProviderBindingRevision: selected.Binding.Revision, ProviderInstanceID: selected.Provider.ID, ModelDescriptorID: selected.Model.ID, ProviderModelID: selected.Model.ProviderModelID}
response, invokeErr := g.invokeAdmitted(r.Context(), string(invocation.RequestID), provider, invocation, int(request.MaxOutputBytes))
observation := capability.TaskCaseObservation{CaseID: item.CaseID, LatencyMicros: time.Since(started).Microseconds(), EvidenceArtifactRef: item.EvidenceArtifactRef}
responses[index] = capability.TaskCaseResponse{CaseID: item.CaseID, ResponseText: response.AdvisoryText, LatencyMicros: observation.LatencyMicros, EvidenceArtifactRef: item.EvidenceArtifactRef}
if invokeErr == nil && response.FinishReason == cognitive.FinishCompleted && response.AdvisoryText != "" {
// Invocation success proves reachability only. Without an
// independent verifier and answer evidence, semantic support and
@@ -85,7 +87,7 @@ func (g cognitiveGateway) taskEvaluationHandler(w http.ResponseWriter, r *http.R
return
}
wg.Wait()
writeTaskEvaluationJSON(w, capability.TaskEvaluationExecutionResponse{ProfileID: request.Profile.ID, BindingID: selected.Binding.ID, BindingRevision: selected.Binding.Revision, Observations: observations, ObservedAtUnixMillis: time.Now().UnixMilli()})
writeTaskEvaluationJSON(w, capability.TaskEvaluationExecutionResponse{ProfileID: request.Profile.ID, BindingID: selected.Binding.ID, BindingRevision: selected.Binding.Revision, Observations: observations, Responses: responses, ObservedAtUnixMillis: time.Now().UnixMilli()})
}
func writeTaskEvaluationJSON(w http.ResponseWriter, value any) {
+64
View File
@@ -0,0 +1,64 @@
package main
import (
"crypto/sha256"
"encoding/hex"
"encoding/json"
"fmt"
"net/http"
"os"
"strings"
"git.cin.su/m/Assistant/contracts/capability"
)
const verifierIdentity = "deterministic-task-verifier-v1"
func main() {
mux := http.NewServeMux()
mux.HandleFunc("GET /healthz", func(w http.ResponseWriter, _ *http.Request) {
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(`{"runtime":"task-verifier","status":"ok"}`))
})
mux.HandleFunc("POST /v1/task-evaluations/verify", verifyHandler)
addr := ":8091"
if value := strings.TrimSpace(os.Getenv("LISTEN_ADDR")); value != "" {
addr = value
}
if err := http.ListenAndServe(addr, mux); err != nil {
panic(err)
}
}
func verifyHandler(w http.ResponseWriter, r *http.Request) {
var request capability.TaskCaseVerificationRequest
decoder := json.NewDecoder(http.MaxBytesReader(w, r.Body, 64<<10))
decoder.DisallowUnknownFields()
if err := decoder.Decode(&request); err != nil || request.CaseID == "" || request.ResponseText == "" || request.EvidenceArtifactRef == "" || request.LatencyMicros < 0 || request.Verification.Mode != capability.TaskCaseVerificationExactText || request.Verification.ExpectedText == "" || request.Verification.MinimumQualityMillis < 0 || request.Verification.MinimumQualityMillis > 1000 {
http.Error(w, "invalid task verification request", http.StatusBadRequest)
return
}
matched := strings.TrimSpace(request.ResponseText) == strings.TrimSpace(request.Verification.ExpectedText)
quality := int64(0)
result := capability.EvaluationUnsupported
if matched {
quality = request.Verification.MinimumQualityMillis
if quality == 0 {
quality = 1000
}
result = capability.EvaluationSupported
}
evidence := verificationEvidence(request, matched)
writeJSON(w, capability.TaskCaseVerificationResponse{CaseID: request.CaseID, Result: result, QualityScoreMillis: quality, LatencyMicros: request.LatencyMicros, EvidenceArtifactRef: request.EvidenceArtifactRef, VerificationEvidenceRef: evidence, VerifierIdentity: verifierIdentity})
}
func verificationEvidence(request capability.TaskCaseVerificationRequest, matched bool) string {
payload := fmt.Sprintf("%s|%s|%s|%t", verifierIdentity, request.CaseID, request.EvidenceArtifactRef, matched)
digest := sha256.Sum256([]byte(payload))
return "verification:" + hex.EncodeToString(digest[:])
}
func writeJSON(w http.ResponseWriter, value any) {
w.Header().Set("Content-Type", "application/json")
_ = json.NewEncoder(w).Encode(value)
}
+31
View File
@@ -0,0 +1,31 @@
package main
import (
"bytes"
"encoding/json"
"net/http"
"net/http/httptest"
"testing"
"git.cin.su/m/Assistant/contracts/capability"
)
func TestVerifyExactTextProducesIndependentEvidence(t *testing.T) {
payload, _ := json.Marshal(capability.TaskCaseVerificationRequest{CaseID: "case-1", ResponseText: " answer ", EvidenceArtifactRef: "case-artifact", Verification: capability.TaskCaseVerification{Mode: capability.TaskCaseVerificationExactText, ExpectedText: "answer", MinimumQualityMillis: 900}})
recorder := httptest.NewRecorder()
verifyHandler(recorder, httptest.NewRequest(http.MethodPost, "/v1/task-evaluations/verify", bytes.NewReader(payload)))
var response capability.TaskCaseVerificationResponse
if recorder.Code != http.StatusOK || json.NewDecoder(recorder.Body).Decode(&response) != nil || response.Result != capability.EvaluationSupported || response.QualityScoreMillis != 900 || response.VerifierIdentity == "" || response.VerificationEvidenceRef == "" {
t.Fatalf("unexpected verification response: code=%d body=%s value=%+v", recorder.Code, recorder.Body.String(), response)
}
}
func TestVerifyMismatchIsUnsupportedWithEvidence(t *testing.T) {
payload, _ := json.Marshal(capability.TaskCaseVerificationRequest{CaseID: "case-1", ResponseText: "wrong", EvidenceArtifactRef: "case-artifact", Verification: capability.TaskCaseVerification{Mode: capability.TaskCaseVerificationExactText, ExpectedText: "answer", MinimumQualityMillis: 900}})
recorder := httptest.NewRecorder()
verifyHandler(recorder, httptest.NewRequest(http.MethodPost, "/v1/task-evaluations/verify", bytes.NewReader(payload)))
var response capability.TaskCaseVerificationResponse
if recorder.Code != http.StatusOK || json.NewDecoder(recorder.Body).Decode(&response) != nil || response.Result != capability.EvaluationUnsupported || response.QualityScoreMillis != 0 || response.VerificationEvidenceRef == "" {
t.Fatalf("unexpected mismatch response: code=%d body=%s value=%+v", recorder.Code, recorder.Body.String(), response)
}
}
+12
View File
@@ -70,6 +70,18 @@ type TaskEvaluationExpansionRequest struct {
BenchmarkVersion string
}
type TaskCaseVerificationMode string
const TaskCaseVerificationExactText TaskCaseVerificationMode = "EXACT_TEXT"
// TaskCaseVerification is a bounded, task-owned rubric. The expected text is
// test data, not model output, and is never sent to the cognitive provider.
type TaskCaseVerification struct {
Mode TaskCaseVerificationMode
ExpectedText string
MinimumQualityMillis int64
}
// TaskCaseObservation is produced by the bounded evaluator and verifier. A
// score without a case result and evidence reference is not accepted as
// qualification evidence.
@@ -11,6 +11,33 @@ type TaskEvaluationCase struct {
CaseID string
Prompt string
EvidenceArtifactRef string
Verification TaskCaseVerification
}
type TaskCaseResponse struct {
CaseID string
ResponseText string
LatencyMicros int64
EvidenceArtifactRef string
}
type TaskCaseVerificationRequest struct {
CaseID string
ResponseText string
LatencyMicros int64
EvidenceArtifactRef string
Verification TaskCaseVerification
}
type TaskCaseVerificationResponse struct {
CaseID string
Result EvaluationResult
QualityScoreMillis int64
LatencyMicros int64
CriticalFailure bool
EvidenceArtifactRef string
VerificationEvidenceRef string
VerifierIdentity string
}
type TaskEvaluationExecutionRequest struct {
@@ -31,5 +58,6 @@ type TaskEvaluationExecutionResponse struct {
BindingID CapabilityBindingID
BindingRevision int64
Observations []TaskCaseObservation
Responses []TaskCaseResponse
ObservedAtUnixMillis int64
}
+15 -1
View File
@@ -118,10 +118,24 @@ services:
WORK_POSTGRES_CREDENTIAL_REF: credential://domovoy/work/db-password/v1
WORK_POSTGRES_PASSWORD_FILE: /run/secrets/work_db_password
SECRET_BACKEND_ROOT: /run/secrets
depends_on: { postgres: { condition: service_healthy } }
TASK_VERIFIER_URL: http://task-verifier:8091
depends_on:
postgres: { condition: service_healthy }
task-verifier: { condition: service_healthy }
volumes:
- assistant_core_secrets:/run/secrets:ro
task-verifier:
build: { context: .., dockerfile: deploy/images/runtime.Dockerfile, args: { TARGET: task-verifier, SOURCE_REVISION: "${ASSISTANT_SOURCE_REVISION:-unknown}" } }
restart: unless-stopped
environment:
LISTEN_ADDR: :8091
healthcheck:
test: ["CMD-SHELL", "wget -qO- http://localhost:8091/healthz >/dev/null"]
interval: 10s
timeout: 3s
retries: 10
work-graph-runtime:
build: { context: .., dockerfile: deploy/images/runtime.Dockerfile, args: { TARGET: work-graph-runtime, SOURCE_REVISION: "${ASSISTANT_SOURCE_REVISION:-unknown}" } }
restart: unless-stopped
+2 -2
View File
@@ -1,5 +1,5 @@
{
"task_file": "dev/tasks/DEV-000220-verifier-evidence-boundary.json",
"task_id": "DEV-000220",
"task_file": "dev/tasks/DEV-000221-task-verifier-runtime.json",
"task_id": "DEV-000221",
"task_revision": 1
}
@@ -0,0 +1,23 @@
{
"task_id": "DEV-000221",
"task_revision": 1,
"architecture_revision": "ADR-0018",
"base_repo_revision": "0aab10863cb23824688029c0bab6734f100b4610",
"risk_tier": "E",
"intent": "Provide an independent deterministic task-verifier runtime and wire model task evaluation through response evidence rather than provider self-assertion.",
"requirement_ids": ["independent verifier runtime", "typed answer evidence", "quality qualification boundary", "degraded verifier fallback"],
"allowed_change_scope": {"modules": ["capability", "cognitive-worker", "agent-worker", "task-verifier", "deployment"], "paths": ["contracts/capability/task_evaluation.go", "contracts/capability/task_evaluation_execution.go", "cmd/cognitive-worker/task_evaluation.go", "cmd/agent-worker/main.go", "cmd/agent-worker/task_evaluation_executor.go", "cmd/agent-worker/task_evaluation_executor_test.go", "cmd/task-verifier/**", "deploy/docker-compose.prod.yml", "dev/ACTIVE_TASK.json", "dev/tasks/DEV-000221-task-verifier-runtime.json"]},
"forbidden_change_scope": {"paths": ["architecture/**", "security/**", "database/**", "modules/**", "LLM-local/**"]},
"allowed_architecture_delta": {"new_cross_module_dependencies": 0, "new_canonical_owners": 0, "new_privileges": 1, "new_third_party_dependencies": 0, "new_database_owner_roles": 0},
"contracts": {"consume": ["TaskEvaluationCase", "TaskEvaluationExecutionRequest/Response", "TaskCaseObservation", "existing worker callback"], "may_change": ["typed verification spec, response envelope and verifier result" ]},
"semantic_owner": "capability owns evidence; task-verifier is a replaceable independent verifier adapter and does not own canonical state",
"canonical_state": ["existing capability task-evaluation evidence only; verifier has no database"],
"state_transitions": ["provider response -> bounded response evidence -> verifier result -> TaskCaseObservation -> TaskEvaluation"],
"side_effects_and_privileges": ["local network calls only; no secrets; no external mutation; no authority expansion"],
"persistence_and_migrations": ["none; verifier evidence is carried through existing callback and capability persistence"],
"failure_retry_restart": ["verifier unavailable returns local fallback/waiting; no quality or promotion claim is made"],
"assumptions": ["initial verifier mode is exact-text for bounded deterministic cases; future verifiers may add typed modes under separate task packets"],
"acceptance_scenarios": ["exact response produces supported evidence with verifier identity", "mismatch produces unsupported evidence", "cognitive worker never asserts support from invocation alone", "worker wires verifier result before callback", "verifier process has health endpoint and production compose dependency", "verifier loss does not produce false quality"],
"required_tests": ["go test", "go vet", "preflight", "check-diff", "check-architecture", "independent", "git diff --check", "CI", "production public health and verifier health check"],
"terminal_gap_states": ["BLOCKED_BY_MISSING_CAPABILITY", "BLOCKED_BY_SPEC_GAP"]
}