73 lines
5.0 KiB
Go
73 lines
5.0 KiB
Go
package main
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"testing"
|
|
|
|
"git.cin.su/m/Assistant/contracts/capability"
|
|
"git.cin.su/m/Assistant/contracts/work"
|
|
)
|
|
|
|
func TestTaskEvaluationExecutorInvokesAggregatesAndCallbacks(t *testing.T) {
|
|
profile := capability.TaskEvaluationProfile{ID: "profile-1", Revision: 1, CapabilityID: "capability-1", Stage: capability.TaskEvaluationGeneral, BenchmarkVersion: "general-v1", RequiredCaseIDs: []string{"case-1"}, QualityWeightMillis: 600, ReliabilityWeightMillis: 300, LatencyWeightMillis: 100}
|
|
verification := capability.TaskCaseVerification{Mode: capability.TaskCaseVerificationExactText, ExpectedText: "answer", MinimumQualityMillis: 900}
|
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
if r.URL.Path == "/v1/task-evaluations/execute" {
|
|
_ = json.NewEncoder(w).Encode(capability.TaskEvaluationExecutionResponse{ProfileID: profile.ID, BindingID: "binding-1", BindingRevision: 3, Responses: []capability.TaskCaseResponse{{CaseID: "case-1", ResponseText: "answer", LatencyMicros: 2, EvidenceArtifactRef: "artifact-1"}}, ObservedAtUnixMillis: 10})
|
|
return
|
|
}
|
|
if r.URL.Path == "/v1/task-evaluations/verify" {
|
|
_ = json.NewEncoder(w).Encode(capability.TaskCaseVerificationResponse{CaseID: "case-1", Result: capability.EvaluationSupported, QualityScoreMillis: 900, LatencyMicros: 2, EvidenceArtifactRef: "artifact-1", VerificationEvidenceRef: "verify-1", VerifierIdentity: "verifier-1"})
|
|
return
|
|
}
|
|
if r.URL.Path == "/v1/work-graphs/task-evaluations/callback" {
|
|
var callback work.TaskEvaluationEvidenceCallback
|
|
if err := json.NewDecoder(r.Body).Decode(&callback); err != nil || callback.GraphID != "graph-1" || callback.NodeID != "node-1" || callback.Evaluation.Result != capability.EvaluationSupported {
|
|
http.Error(w, "invalid callback", http.StatusBadRequest)
|
|
return
|
|
}
|
|
w.WriteHeader(http.StatusNoContent)
|
|
return
|
|
}
|
|
http.NotFound(w, r)
|
|
}))
|
|
defer server.Close()
|
|
executor := newTaskEvaluationExecutor(server.URL, server.URL, server.URL, server.Client())
|
|
result, err := executor.Execute(context.Background(), work.AgentExecutionRequest{Intent: work.AgentDispatchIntent{DispatchID: "dispatch-1", TaskEvaluationBindingID: "binding-1", TaskEvaluationProfile: &profile, TaskEvaluationCases: []capability.TaskEvaluationCase{{CaseID: "case-1", Prompt: "answer", EvidenceArtifactRef: "artifact-1", Verification: verification}}, TaskEvaluationExecution: &work.TaskEvaluationDispatchContext{GraphID: "graph-1", GraphRevision: 1, NodeID: "node-1", BindingRevision: 3, ActorID: "actor-1", PrincipalID: "principal-1", ScopeID: "scope-1"}}})
|
|
if err != nil || result.State != work.AgentExecutionCompleted || result.OutcomeRef == "" {
|
|
t.Fatalf("result=%+v err=%v", result, err)
|
|
}
|
|
}
|
|
|
|
func TestTaskEvaluationExecutorPersistsUnknownAfterProviderFailure(t *testing.T) {
|
|
profile := capability.TaskEvaluationProfile{ID: "profile-unknown", Revision: 1, CapabilityID: "capability-1", Stage: capability.TaskEvaluationGeneral, BenchmarkVersion: "unknown-v1", RequiredCaseIDs: []string{"case-1"}, QualityWeightMillis: 600, ReliabilityWeightMillis: 300, LatencyWeightMillis: 100}
|
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
if r.URL.Path == "/v1/task-evaluations/execute" {
|
|
_ = json.NewEncoder(w).Encode(capability.TaskEvaluationExecutionResponse{ProfileID: profile.ID, BindingID: "binding-1", BindingRevision: 3, Observations: []capability.TaskCaseObservation{{CaseID: "case-1", Result: capability.EvaluationUnknown, CriticalFailure: true, EvidenceArtifactRef: "artifact-1"}}, Responses: []capability.TaskCaseResponse{{CaseID: "case-1", EvidenceArtifactRef: "artifact-1"}}, ObservedAtUnixMillis: 10})
|
|
return
|
|
}
|
|
if r.URL.Path == "/v1/work-graphs/task-evaluations/callback" {
|
|
var callback work.TaskEvaluationEvidenceCallback
|
|
if err := json.NewDecoder(r.Body).Decode(&callback); err != nil || callback.Evaluation.Result != capability.EvaluationUnknown || !callback.Succeeded {
|
|
http.Error(w, "invalid unknown callback", http.StatusBadRequest)
|
|
return
|
|
}
|
|
w.WriteHeader(http.StatusNoContent)
|
|
return
|
|
}
|
|
if r.URL.Path == "/v1/task-evaluations/verify" {
|
|
t.Errorf("verifier must not be called for a provider failure")
|
|
}
|
|
http.NotFound(w, r)
|
|
}))
|
|
defer server.Close()
|
|
executor := newTaskEvaluationExecutor(server.URL, server.URL, server.URL, server.Client())
|
|
result, err := executor.Execute(context.Background(), work.AgentExecutionRequest{Intent: work.AgentDispatchIntent{DispatchID: "dispatch-unknown", TaskEvaluationBindingID: "binding-1", TaskEvaluationProfile: &profile, TaskEvaluationCases: []capability.TaskEvaluationCase{{CaseID: "case-1", Prompt: "answer", EvidenceArtifactRef: "artifact-1"}}, TaskEvaluationExecution: &work.TaskEvaluationDispatchContext{GraphID: "graph-1", GraphRevision: 1, NodeID: "node-1", BindingRevision: 3, ActorID: "actor-1", PrincipalID: "principal-1", ScopeID: "scope-1"}}})
|
|
if err != nil || result.State != work.AgentExecutionCompleted || result.OutcomeRef == "" {
|
|
t.Fatalf("result=%+v err=%v", result, err)
|
|
}
|
|
}
|