Compare commits
10
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
c53d7ded06 | ||
|
|
e080c34a77 | ||
|
|
306f53d484 | ||
|
|
c4411afaaf | ||
|
|
c508941a8a | ||
|
|
ee68e2207e | ||
|
|
3d9a5bccba | ||
|
|
d736500d98 | ||
|
|
33b51a4445 | ||
|
|
0aab10863c |
@@ -52,7 +52,7 @@ func main() {
|
||||
// The local evaluator keeps task-qualification work alive, but it is not
|
||||
// an external Agent Execution Bridge and must never be reported as one.
|
||||
localFallback = true
|
||||
executor = newTaskEvaluationExecutor(envOr("COGNITIVE_WORKER_URL", "http://cognitive-worker:8083"), envOr("WORK_GRAPH_RUNTIME_URL", "http://work-graph-runtime:8088"), &http.Client{Timeout: *leaseTTL})
|
||||
executor = newTaskEvaluationExecutor(envOr("COGNITIVE_WORKER_URL", "http://cognitive-worker:8083"), envOr("WORK_GRAPH_RUNTIME_URL", "http://work-graph-runtime:8088"), envOr("TASK_VERIFIER_URL", "http://task-verifier:8091"), &http.Client{Timeout: *leaseTTL})
|
||||
fmt.Fprintln(os.Stderr, "protected agent execution bridge URL is not configured; task evaluations use the local durable evaluator, other work remains in fallback")
|
||||
}
|
||||
go func() {
|
||||
|
||||
@@ -18,15 +18,16 @@ import (
|
||||
type taskEvaluationExecutor struct {
|
||||
cognitiveURL string
|
||||
callbackURL string
|
||||
verifierURL string
|
||||
client *http.Client
|
||||
fallback localFallbackExecutor
|
||||
}
|
||||
|
||||
func newTaskEvaluationExecutor(cognitiveURL, callbackURL string, client *http.Client) taskEvaluationExecutor {
|
||||
func newTaskEvaluationExecutor(cognitiveURL, callbackURL, verifierURL string, client *http.Client) taskEvaluationExecutor {
|
||||
if client == nil {
|
||||
client = http.DefaultClient
|
||||
}
|
||||
return taskEvaluationExecutor{cognitiveURL: strings.TrimRight(cognitiveURL, "/"), callbackURL: strings.TrimRight(callbackURL, "/"), client: client, fallback: newLocalFallbackExecutor()}
|
||||
return taskEvaluationExecutor{cognitiveURL: strings.TrimRight(cognitiveURL, "/"), callbackURL: strings.TrimRight(callbackURL, "/"), verifierURL: strings.TrimRight(verifierURL, "/"), client: client, fallback: newLocalFallbackExecutor()}
|
||||
}
|
||||
|
||||
func (e taskEvaluationExecutor) Execute(ctx context.Context, request work.AgentExecutionRequest) (work.AgentExecutionResult, error) {
|
||||
@@ -38,7 +39,7 @@ func (e taskEvaluationExecutor) Execute(ctx context.Context, request work.AgentE
|
||||
if execution.GraphID == "" || execution.NodeID == "" || execution.BindingRevision < 1 || execution.ActorID == "" || execution.PrincipalID == "" || execution.ScopeID == "" {
|
||||
return e.fallback.Execute(ctx, request)
|
||||
}
|
||||
if e.cognitiveURL == "" || e.callbackURL == "" {
|
||||
if e.cognitiveURL == "" || e.callbackURL == "" || e.verifierURL == "" {
|
||||
return e.fallback.Execute(ctx, request)
|
||||
}
|
||||
payload := capability.TaskEvaluationExecutionRequest{Profile: *intent.TaskEvaluationProfile, CapabilityID: intent.TaskEvaluationProfile.CapabilityID, BindingID: intent.TaskEvaluationBindingID, BindingRevision: execution.BindingRevision, ActorID: execution.ActorID, PrincipalID: execution.PrincipalID, ScopeID: execution.ScopeID, Cases: intent.TaskEvaluationCases, MaxOutputBytes: 4096}
|
||||
@@ -46,13 +47,29 @@ func (e taskEvaluationExecutor) Execute(ctx context.Context, request work.AgentE
|
||||
if err != nil {
|
||||
return work.AgentExecutionResult{}, err
|
||||
}
|
||||
if len(response.Observations) == 0 || response.ObservedAtUnixMillis < 1 {
|
||||
if len(response.Responses) == 0 || response.ObservedAtUnixMillis < 1 {
|
||||
return work.AgentExecutionResult{}, fmt.Errorf("task evaluation returned no observations")
|
||||
}
|
||||
if len(intent.TaskEvaluationCases) == 0 {
|
||||
return work.AgentExecutionResult{}, fmt.Errorf("task evaluation cases are missing")
|
||||
}
|
||||
evaluation, err := capabilitydomain.BuildTaskEvaluation(*intent.TaskEvaluationProfile, capability.TaskEvaluationID("evaluation:"+intent.DispatchID), response.BindingID, response.BindingRevision, intent.TaskEvaluationProfile.CapabilityID, response.Observations, artifact.ArtifactID(intent.TaskEvaluationCases[0].EvidenceArtifactRef), response.ObservedAtUnixMillis)
|
||||
caseByID := make(map[string]capability.TaskEvaluationCase, len(intent.TaskEvaluationCases))
|
||||
for _, item := range intent.TaskEvaluationCases {
|
||||
caseByID[item.CaseID] = item
|
||||
}
|
||||
observations := make([]capability.TaskCaseObservation, 0, len(response.Responses))
|
||||
for _, item := range response.Responses {
|
||||
caseValue, ok := caseByID[item.CaseID]
|
||||
if !ok || caseValue.Verification.Mode == "" {
|
||||
return e.fallback.Execute(ctx, request)
|
||||
}
|
||||
verified, verifyErr := postJSON[capability.TaskCaseVerificationResponse](ctx, e.client, e.verifierURL+"/v1/task-evaluations/verify", capability.TaskCaseVerificationRequest{CaseID: item.CaseID, ResponseText: item.ResponseText, LatencyMicros: item.LatencyMicros, EvidenceArtifactRef: item.EvidenceArtifactRef, Verification: caseValue.Verification})
|
||||
if verifyErr != nil {
|
||||
return e.fallback.Execute(ctx, request)
|
||||
}
|
||||
observations = append(observations, capability.TaskCaseObservation{CaseID: verified.CaseID, Result: verified.Result, QualityScoreMillis: verified.QualityScoreMillis, LatencyMicros: verified.LatencyMicros, CriticalFailure: verified.CriticalFailure, EvidenceArtifactRef: verified.EvidenceArtifactRef, VerificationEvidenceRef: verified.VerificationEvidenceRef, VerifierIdentity: verified.VerifierIdentity})
|
||||
}
|
||||
evaluation, err := capabilitydomain.BuildTaskEvaluation(*intent.TaskEvaluationProfile, capability.TaskEvaluationID("evaluation:"+intent.DispatchID), response.BindingID, response.BindingRevision, intent.TaskEvaluationProfile.CapabilityID, observations, artifact.ArtifactID(intent.TaskEvaluationCases[0].EvidenceArtifactRef), response.ObservedAtUnixMillis)
|
||||
if err != nil {
|
||||
return work.AgentExecutionResult{}, err
|
||||
}
|
||||
|
||||
@@ -13,9 +13,14 @@ import (
|
||||
|
||||
func TestTaskEvaluationExecutorInvokesAggregatesAndCallbacks(t *testing.T) {
|
||||
profile := capability.TaskEvaluationProfile{ID: "profile-1", Revision: 1, CapabilityID: "capability-1", Stage: capability.TaskEvaluationGeneral, BenchmarkVersion: "general-v1", RequiredCaseIDs: []string{"case-1"}, QualityWeightMillis: 600, ReliabilityWeightMillis: 300, LatencyWeightMillis: 100}
|
||||
verification := capability.TaskCaseVerification{Mode: capability.TaskCaseVerificationExactText, ExpectedText: "answer", MinimumQualityMillis: 900}
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.URL.Path == "/v1/task-evaluations/execute" {
|
||||
_ = json.NewEncoder(w).Encode(capability.TaskEvaluationExecutionResponse{ProfileID: profile.ID, BindingID: "binding-1", BindingRevision: 3, Observations: []capability.TaskCaseObservation{{CaseID: "case-1", Result: capability.EvaluationSupported, QualityScoreMillis: 800, EvidenceArtifactRef: "artifact-1", VerificationEvidenceRef: "verify-1", VerifierIdentity: "verifier-1"}}, ObservedAtUnixMillis: 10})
|
||||
_ = json.NewEncoder(w).Encode(capability.TaskEvaluationExecutionResponse{ProfileID: profile.ID, BindingID: "binding-1", BindingRevision: 3, Responses: []capability.TaskCaseResponse{{CaseID: "case-1", ResponseText: "answer", LatencyMicros: 2, EvidenceArtifactRef: "artifact-1"}}, ObservedAtUnixMillis: 10})
|
||||
return
|
||||
}
|
||||
if r.URL.Path == "/v1/task-evaluations/verify" {
|
||||
_ = json.NewEncoder(w).Encode(capability.TaskCaseVerificationResponse{CaseID: "case-1", Result: capability.EvaluationSupported, QualityScoreMillis: 900, LatencyMicros: 2, EvidenceArtifactRef: "artifact-1", VerificationEvidenceRef: "verify-1", VerifierIdentity: "verifier-1"})
|
||||
return
|
||||
}
|
||||
if r.URL.Path == "/v1/work-graphs/task-evaluations/callback" {
|
||||
@@ -30,8 +35,8 @@ func TestTaskEvaluationExecutorInvokesAggregatesAndCallbacks(t *testing.T) {
|
||||
http.NotFound(w, r)
|
||||
}))
|
||||
defer server.Close()
|
||||
executor := newTaskEvaluationExecutor(server.URL, server.URL, server.Client())
|
||||
result, err := executor.Execute(context.Background(), work.AgentExecutionRequest{Intent: work.AgentDispatchIntent{DispatchID: "dispatch-1", TaskEvaluationBindingID: "binding-1", TaskEvaluationProfile: &profile, TaskEvaluationCases: []capability.TaskEvaluationCase{{CaseID: "case-1", Prompt: "answer", EvidenceArtifactRef: "artifact-1"}}, TaskEvaluationExecution: &work.TaskEvaluationDispatchContext{GraphID: "graph-1", GraphRevision: 1, NodeID: "node-1", BindingRevision: 3, ActorID: "actor-1", PrincipalID: "principal-1", ScopeID: "scope-1"}}})
|
||||
executor := newTaskEvaluationExecutor(server.URL, server.URL, server.URL, server.Client())
|
||||
result, err := executor.Execute(context.Background(), work.AgentExecutionRequest{Intent: work.AgentDispatchIntent{DispatchID: "dispatch-1", TaskEvaluationBindingID: "binding-1", TaskEvaluationProfile: &profile, TaskEvaluationCases: []capability.TaskEvaluationCase{{CaseID: "case-1", Prompt: "answer", EvidenceArtifactRef: "artifact-1", Verification: verification}}, TaskEvaluationExecution: &work.TaskEvaluationDispatchContext{GraphID: "graph-1", GraphRevision: 1, NodeID: "node-1", BindingRevision: 3, ActorID: "actor-1", PrincipalID: "principal-1", ScopeID: "scope-1"}}})
|
||||
if err != nil || result.State != work.AgentExecutionCompleted || result.OutcomeRef == "" {
|
||||
t.Fatalf("result=%+v err=%v", result, err)
|
||||
}
|
||||
|
||||
@@ -44,6 +44,7 @@ func (g cognitiveGateway) taskEvaluationHandler(w http.ResponseWriter, r *http.R
|
||||
caseSet[id] = struct{}{}
|
||||
}
|
||||
observations := make([]capability.TaskCaseObservation, len(request.Cases))
|
||||
responses := make([]capability.TaskCaseResponse, len(request.Cases))
|
||||
var wg sync.WaitGroup
|
||||
seenCases := make(map[string]struct{}, len(request.Cases))
|
||||
for i, item := range request.Cases {
|
||||
@@ -67,6 +68,7 @@ func (g cognitiveGateway) taskEvaluationHandler(w http.ResponseWriter, r *http.R
|
||||
invocation := cognitive.AdvisoryRequest{Version: cognitive.CreatorAdvisoryV1, RequestID: cognitive.RequestID(fmt.Sprintf("task-evaluation:%s:%s", request.Profile.ID, item.CaseID)), ActorID: request.ActorID, PrincipalID: request.PrincipalID, ScopeID: request.ScopeID, Message: item.Prompt, MaxOutputBytes: int(request.MaxOutputBytes), ProviderBindingID: selected.Binding.ID, ProviderBindingRevision: selected.Binding.Revision, ProviderInstanceID: selected.Provider.ID, ModelDescriptorID: selected.Model.ID, ProviderModelID: selected.Model.ProviderModelID}
|
||||
response, invokeErr := g.invokeAdmitted(r.Context(), string(invocation.RequestID), provider, invocation, int(request.MaxOutputBytes))
|
||||
observation := capability.TaskCaseObservation{CaseID: item.CaseID, LatencyMicros: time.Since(started).Microseconds(), EvidenceArtifactRef: item.EvidenceArtifactRef}
|
||||
responses[index] = capability.TaskCaseResponse{CaseID: item.CaseID, ResponseText: response.AdvisoryText, LatencyMicros: observation.LatencyMicros, EvidenceArtifactRef: item.EvidenceArtifactRef}
|
||||
if invokeErr == nil && response.FinishReason == cognitive.FinishCompleted && response.AdvisoryText != "" {
|
||||
// Invocation success proves reachability only. Without an
|
||||
// independent verifier and answer evidence, semantic support and
|
||||
@@ -85,7 +87,7 @@ func (g cognitiveGateway) taskEvaluationHandler(w http.ResponseWriter, r *http.R
|
||||
return
|
||||
}
|
||||
wg.Wait()
|
||||
writeTaskEvaluationJSON(w, capability.TaskEvaluationExecutionResponse{ProfileID: request.Profile.ID, BindingID: selected.Binding.ID, BindingRevision: selected.Binding.Revision, Observations: observations, ObservedAtUnixMillis: time.Now().UnixMilli()})
|
||||
writeTaskEvaluationJSON(w, capability.TaskEvaluationExecutionResponse{ProfileID: request.Profile.ID, BindingID: selected.Binding.ID, BindingRevision: selected.Binding.Revision, Observations: observations, Responses: responses, ObservedAtUnixMillis: time.Now().UnixMilli()})
|
||||
}
|
||||
|
||||
func writeTaskEvaluationJSON(w http.ResponseWriter, value any) {
|
||||
|
||||
@@ -150,7 +150,7 @@ func (g *gateway) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
|
||||
return
|
||||
}
|
||||
if r.URL.Path == "/" || r.URL.Path == "/chat" || r.URL.Path == "/chat.html" {
|
||||
if r.URL.Path == "/" || r.URL.Path == "/login" || r.URL.Path == "/chat" || r.URL.Path == "/chat.html" {
|
||||
g.serveEmbeddedHTML(w, "chat.html")
|
||||
return
|
||||
}
|
||||
|
||||
@@ -354,6 +354,18 @@ func TestConsoleAndSecurityHeaders(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoginRouteServesChatShell(t *testing.T) {
|
||||
handler, err := newHandler("http://core-runtime:8080", "http://continuity-runtime:8081", "http://commit-runtime:8082", http.DefaultClient)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
response := httptest.NewRecorder()
|
||||
handler.ServeHTTP(response, httptest.NewRequest(http.MethodGet, "/login", nil))
|
||||
if response.Code != http.StatusOK || !strings.Contains(response.Body.String(), "id=\"username\"") || !strings.Contains(response.Body.String(), "id=\"password\"") {
|
||||
t.Fatalf("login response = %d", response.Code)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAdvisoryModeRejectsInvalidConfiguration(t *testing.T) {
|
||||
_, err := newHandlerWithAdvisoryMode("http://core.invalid", "http://continuity.invalid", "http://commit.invalid", http.DefaultClient, "browser-selected")
|
||||
if err == nil || !strings.Contains(err.Error(), "invalid advisory binding mode") {
|
||||
|
||||
@@ -0,0 +1,76 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
"git.cin.su/m/Assistant/contracts/capability"
|
||||
)
|
||||
|
||||
const verifierIdentity = "deterministic-task-verifier-v1"
|
||||
|
||||
func main() {
|
||||
mux := http.NewServeMux()
|
||||
mux.HandleFunc("GET /healthz", func(w http.ResponseWriter, _ *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{"runtime":"task-verifier","status":"ok"}`))
|
||||
})
|
||||
mux.HandleFunc("POST /v1/task-evaluations/verify", verifyHandler)
|
||||
addr := ":8091"
|
||||
if value := strings.TrimSpace(os.Getenv("LISTEN_ADDR")); value != "" {
|
||||
addr = value
|
||||
}
|
||||
if err := http.ListenAndServe(addr, mux); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
}
|
||||
|
||||
func verifyHandler(w http.ResponseWriter, r *http.Request) {
|
||||
var request capability.TaskCaseVerificationRequest
|
||||
decoder := json.NewDecoder(http.MaxBytesReader(w, r.Body, 64<<10))
|
||||
decoder.DisallowUnknownFields()
|
||||
if err := decoder.Decode(&request); err != nil || request.CaseID == "" || request.ResponseText == "" || request.EvidenceArtifactRef == "" || request.LatencyMicros < 0 || (request.Verification.Mode != capability.TaskCaseVerificationExactText && request.Verification.Mode != capability.TaskCaseVerificationJSONFieldEquals) || (request.Verification.Mode == capability.TaskCaseVerificationJSONFieldEquals && request.Verification.FieldPath == "") || request.Verification.ExpectedText == "" || request.Verification.MinimumQualityMillis < 0 || request.Verification.MinimumQualityMillis > 1000 {
|
||||
http.Error(w, "invalid task verification request", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
matched := matches(request)
|
||||
quality := int64(0)
|
||||
result := capability.EvaluationUnsupported
|
||||
if matched {
|
||||
quality = request.Verification.MinimumQualityMillis
|
||||
if quality == 0 {
|
||||
quality = 1000
|
||||
}
|
||||
result = capability.EvaluationSupported
|
||||
}
|
||||
evidence := verificationEvidence(request, matched)
|
||||
writeJSON(w, capability.TaskCaseVerificationResponse{CaseID: request.CaseID, Result: result, QualityScoreMillis: quality, LatencyMicros: request.LatencyMicros, EvidenceArtifactRef: request.EvidenceArtifactRef, VerificationEvidenceRef: evidence, VerifierIdentity: verifierIdentity})
|
||||
}
|
||||
|
||||
func verificationEvidence(request capability.TaskCaseVerificationRequest, matched bool) string {
|
||||
payload := fmt.Sprintf("%s|%s|%s|%s|%s|%t", verifierIdentity, request.CaseID, request.EvidenceArtifactRef, request.Verification.Mode, request.Verification.FieldPath, matched)
|
||||
digest := sha256.Sum256([]byte(payload))
|
||||
return "verification:" + hex.EncodeToString(digest[:])
|
||||
}
|
||||
|
||||
func matches(request capability.TaskCaseVerificationRequest) bool {
|
||||
if request.Verification.Mode == capability.TaskCaseVerificationExactText {
|
||||
return strings.TrimSpace(request.ResponseText) == strings.TrimSpace(request.Verification.ExpectedText)
|
||||
}
|
||||
var document map[string]json.RawMessage
|
||||
if json.Unmarshal([]byte(request.ResponseText), &document) != nil {
|
||||
return false
|
||||
}
|
||||
value, ok := document[request.Verification.FieldPath]
|
||||
return ok && strings.TrimSpace(string(value)) == strings.TrimSpace(request.Verification.ExpectedText)
|
||||
}
|
||||
|
||||
func writeJSON(w http.ResponseWriter, value any) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_ = json.NewEncoder(w).Encode(value)
|
||||
}
|
||||
@@ -0,0 +1,41 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
|
||||
"git.cin.su/m/Assistant/contracts/capability"
|
||||
)
|
||||
|
||||
func TestVerifyExactTextProducesIndependentEvidence(t *testing.T) {
|
||||
payload, _ := json.Marshal(capability.TaskCaseVerificationRequest{CaseID: "case-1", ResponseText: " answer ", EvidenceArtifactRef: "case-artifact", Verification: capability.TaskCaseVerification{Mode: capability.TaskCaseVerificationExactText, ExpectedText: "answer", MinimumQualityMillis: 900}})
|
||||
recorder := httptest.NewRecorder()
|
||||
verifyHandler(recorder, httptest.NewRequest(http.MethodPost, "/v1/task-evaluations/verify", bytes.NewReader(payload)))
|
||||
var response capability.TaskCaseVerificationResponse
|
||||
if recorder.Code != http.StatusOK || json.NewDecoder(recorder.Body).Decode(&response) != nil || response.Result != capability.EvaluationSupported || response.QualityScoreMillis != 900 || response.VerifierIdentity == "" || response.VerificationEvidenceRef == "" {
|
||||
t.Fatalf("unexpected verification response: code=%d body=%s value=%+v", recorder.Code, recorder.Body.String(), response)
|
||||
}
|
||||
}
|
||||
|
||||
func TestVerifyMismatchIsUnsupportedWithEvidence(t *testing.T) {
|
||||
payload, _ := json.Marshal(capability.TaskCaseVerificationRequest{CaseID: "case-1", ResponseText: "wrong", EvidenceArtifactRef: "case-artifact", Verification: capability.TaskCaseVerification{Mode: capability.TaskCaseVerificationExactText, ExpectedText: "answer", MinimumQualityMillis: 900}})
|
||||
recorder := httptest.NewRecorder()
|
||||
verifyHandler(recorder, httptest.NewRequest(http.MethodPost, "/v1/task-evaluations/verify", bytes.NewReader(payload)))
|
||||
var response capability.TaskCaseVerificationResponse
|
||||
if recorder.Code != http.StatusOK || json.NewDecoder(recorder.Body).Decode(&response) != nil || response.Result != capability.EvaluationUnsupported || response.QualityScoreMillis != 0 || response.VerificationEvidenceRef == "" {
|
||||
t.Fatalf("unexpected mismatch response: code=%d body=%s value=%+v", recorder.Code, recorder.Body.String(), response)
|
||||
}
|
||||
}
|
||||
|
||||
func TestVerifyJSONFieldEquals(t *testing.T) {
|
||||
payload, _ := json.Marshal(capability.TaskCaseVerificationRequest{CaseID: "json-1", ResponseText: `{"answer":42}`, EvidenceArtifactRef: "case-artifact", Verification: capability.TaskCaseVerification{Mode: capability.TaskCaseVerificationJSONFieldEquals, FieldPath: "answer", ExpectedText: "42", MinimumQualityMillis: 850}})
|
||||
recorder := httptest.NewRecorder()
|
||||
verifyHandler(recorder, httptest.NewRequest(http.MethodPost, "/v1/task-evaluations/verify", bytes.NewReader(payload)))
|
||||
var response capability.TaskCaseVerificationResponse
|
||||
if recorder.Code != http.StatusOK || json.NewDecoder(recorder.Body).Decode(&response) != nil || response.Result != capability.EvaluationSupported || response.QualityScoreMillis != 850 {
|
||||
t.Fatalf("unexpected JSON verification response: code=%d body=%s value=%+v", recorder.Code, recorder.Body.String(), response)
|
||||
}
|
||||
}
|
||||
@@ -70,6 +70,22 @@ type TaskEvaluationExpansionRequest struct {
|
||||
BenchmarkVersion string
|
||||
}
|
||||
|
||||
type TaskCaseVerificationMode string
|
||||
|
||||
const (
|
||||
TaskCaseVerificationExactText TaskCaseVerificationMode = "EXACT_TEXT"
|
||||
TaskCaseVerificationJSONFieldEquals TaskCaseVerificationMode = "JSON_FIELD_EQUALS"
|
||||
)
|
||||
|
||||
// TaskCaseVerification is a bounded, task-owned rubric. The expected text is
|
||||
// test data, not model output, and is never sent to the cognitive provider.
|
||||
type TaskCaseVerification struct {
|
||||
Mode TaskCaseVerificationMode
|
||||
FieldPath string
|
||||
ExpectedText string
|
||||
MinimumQualityMillis int64
|
||||
}
|
||||
|
||||
// TaskCaseObservation is produced by the bounded evaluator and verifier. A
|
||||
// score without a case result and evidence reference is not accepted as
|
||||
// qualification evidence.
|
||||
|
||||
@@ -11,6 +11,33 @@ type TaskEvaluationCase struct {
|
||||
CaseID string
|
||||
Prompt string
|
||||
EvidenceArtifactRef string
|
||||
Verification TaskCaseVerification
|
||||
}
|
||||
|
||||
type TaskCaseResponse struct {
|
||||
CaseID string
|
||||
ResponseText string
|
||||
LatencyMicros int64
|
||||
EvidenceArtifactRef string
|
||||
}
|
||||
|
||||
type TaskCaseVerificationRequest struct {
|
||||
CaseID string
|
||||
ResponseText string
|
||||
LatencyMicros int64
|
||||
EvidenceArtifactRef string
|
||||
Verification TaskCaseVerification
|
||||
}
|
||||
|
||||
type TaskCaseVerificationResponse struct {
|
||||
CaseID string
|
||||
Result EvaluationResult
|
||||
QualityScoreMillis int64
|
||||
LatencyMicros int64
|
||||
CriticalFailure bool
|
||||
EvidenceArtifactRef string
|
||||
VerificationEvidenceRef string
|
||||
VerifierIdentity string
|
||||
}
|
||||
|
||||
type TaskEvaluationExecutionRequest struct {
|
||||
@@ -31,5 +58,6 @@ type TaskEvaluationExecutionResponse struct {
|
||||
BindingID CapabilityBindingID
|
||||
BindingRevision int64
|
||||
Observations []TaskCaseObservation
|
||||
Responses []TaskCaseResponse
|
||||
ObservedAtUnixMillis int64
|
||||
}
|
||||
|
||||
@@ -118,10 +118,24 @@ services:
|
||||
WORK_POSTGRES_CREDENTIAL_REF: credential://domovoy/work/db-password/v1
|
||||
WORK_POSTGRES_PASSWORD_FILE: /run/secrets/work_db_password
|
||||
SECRET_BACKEND_ROOT: /run/secrets
|
||||
depends_on: { postgres: { condition: service_healthy } }
|
||||
TASK_VERIFIER_URL: http://task-verifier:8091
|
||||
depends_on:
|
||||
postgres: { condition: service_healthy }
|
||||
task-verifier: { condition: service_healthy }
|
||||
volumes:
|
||||
- assistant_core_secrets:/run/secrets:ro
|
||||
|
||||
task-verifier:
|
||||
build: { context: .., dockerfile: deploy/images/runtime.Dockerfile, args: { TARGET: task-verifier, SOURCE_REVISION: "${ASSISTANT_SOURCE_REVISION:-unknown}" } }
|
||||
restart: unless-stopped
|
||||
environment:
|
||||
LISTEN_ADDR: :8091
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "wget -qO- http://localhost:8091/healthz >/dev/null"]
|
||||
interval: 10s
|
||||
timeout: 3s
|
||||
retries: 10
|
||||
|
||||
work-graph-runtime:
|
||||
build: { context: .., dockerfile: deploy/images/runtime.Dockerfile, args: { TARGET: work-graph-runtime, SOURCE_REVISION: "${ASSISTANT_SOURCE_REVISION:-unknown}" } }
|
||||
restart: unless-stopped
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
{
|
||||
"task_file": "dev/tasks/DEV-000220-verifier-evidence-boundary.json",
|
||||
"task_id": "DEV-000220",
|
||||
"task_file": "dev/tasks/DEV-000225-login-route-fix.json",
|
||||
"task_id": "DEV-000225",
|
||||
"task_revision": 1
|
||||
}
|
||||
|
||||
@@ -0,0 +1,23 @@
|
||||
{
|
||||
"task_id": "DEV-000221",
|
||||
"task_revision": 1,
|
||||
"architecture_revision": "ADR-0018",
|
||||
"base_repo_revision": "0aab10863cb23824688029c0bab6734f100b4610",
|
||||
"risk_tier": "E",
|
||||
"intent": "Provide an independent deterministic task-verifier runtime and wire model task evaluation through response evidence rather than provider self-assertion.",
|
||||
"requirement_ids": ["independent verifier runtime", "typed answer evidence", "quality qualification boundary", "degraded verifier fallback"],
|
||||
"allowed_change_scope": {"modules": ["capability", "cognitive-worker", "agent-worker", "task-verifier", "deployment"], "paths": ["contracts/capability/task_evaluation.go", "contracts/capability/task_evaluation_execution.go", "cmd/cognitive-worker/task_evaluation.go", "cmd/agent-worker/main.go", "cmd/agent-worker/task_evaluation_executor.go", "cmd/agent-worker/task_evaluation_executor_test.go", "cmd/task-verifier/**", "deploy/docker-compose.prod.yml", "dev/ACTIVE_TASK.json", "dev/tasks/DEV-000221-task-verifier-runtime.json"]},
|
||||
"forbidden_change_scope": {"paths": ["architecture/**", "security/**", "database/**", "modules/**", "LLM-local/**"]},
|
||||
"allowed_architecture_delta": {"new_cross_module_dependencies": 0, "new_canonical_owners": 0, "new_privileges": 1, "new_third_party_dependencies": 0, "new_database_owner_roles": 0},
|
||||
"contracts": {"consume": ["TaskEvaluationCase", "TaskEvaluationExecutionRequest/Response", "TaskCaseObservation", "existing worker callback"], "may_change": ["typed verification spec, response envelope and verifier result" ]},
|
||||
"semantic_owner": "capability owns evidence; task-verifier is a replaceable independent verifier adapter and does not own canonical state",
|
||||
"canonical_state": ["existing capability task-evaluation evidence only; verifier has no database"],
|
||||
"state_transitions": ["provider response -> bounded response evidence -> verifier result -> TaskCaseObservation -> TaskEvaluation"],
|
||||
"side_effects_and_privileges": ["local network calls only; no secrets; no external mutation; no authority expansion"],
|
||||
"persistence_and_migrations": ["none; verifier evidence is carried through existing callback and capability persistence"],
|
||||
"failure_retry_restart": ["verifier unavailable returns local fallback/waiting; no quality or promotion claim is made"],
|
||||
"assumptions": ["initial verifier mode is exact-text for bounded deterministic cases; future verifiers may add typed modes under separate task packets"],
|
||||
"acceptance_scenarios": ["exact response produces supported evidence with verifier identity", "mismatch produces unsupported evidence", "cognitive worker never asserts support from invocation alone", "worker wires verifier result before callback", "verifier process has health endpoint and production compose dependency", "verifier loss does not produce false quality"],
|
||||
"required_tests": ["go test", "go vet", "preflight", "check-diff", "check-architecture", "independent", "git diff --check", "CI", "production public health and verifier health check"],
|
||||
"terminal_gap_states": ["BLOCKED_BY_MISSING_CAPABILITY", "BLOCKED_BY_SPEC_GAP"]
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
{
|
||||
"task_id": "DEV-000222",
|
||||
"task_revision": 1,
|
||||
"architecture_revision": "ADR-0018",
|
||||
"base_repo_revision": "d736500d9818a7748c3715c442edb695c5aed1100",
|
||||
"risk_tier": "E",
|
||||
"intent": "Extend the independent task verifier with a structured JSON field mode and add a bounded case builder driven by unresolved evidence.",
|
||||
"requirement_ids": ["task-aware verification", "additional cases during lifetime", "general-first bounded expansion"],
|
||||
"allowed_change_scope": {"modules": ["capability", "task-verifier"], "paths": ["contracts/capability/task_evaluation.go", "cmd/task-verifier/main.go", "cmd/task-verifier/main_test.go", "modules/capability/domain/task_evaluation.go", "modules/capability/domain/task_evaluation_test.go", "dev/ACTIVE_TASK.json", "dev/tasks/DEV-000222-verifier-modes-and-case-builder.json"]},
|
||||
"forbidden_change_scope": {"paths": ["architecture/**", "security/**", "database/**", "deploy/**", "cmd/core-runtime/**", "cmd/creator-web/**", "cmd/agent-worker/**", "cmd/cognitive-worker/**", "modules/identity/**", "LLM-local/**"]},
|
||||
"allowed_architecture_delta": {"new_cross_module_dependencies": 0, "new_canonical_owners": 0, "new_privileges": 0, "new_third_party_dependencies": 0, "new_database_owner_roles": 0},
|
||||
"contracts": {"consume": ["TaskCaseVerification", "TaskEvaluationCase", "unresolved outcome evidence"], "may_change": ["verification modes and bounded case construction" ]},
|
||||
"semantic_owner": "capability owns evaluation policy; verifier remains stateless adapter",
|
||||
"canonical_state": ["none; case builder returns candidates and does not persist or activate them"],
|
||||
"state_transitions": ["unresolved evidence -> two bounded additional case candidates", "structured response -> verifier evidence -> supported/unsupported"],
|
||||
"side_effects_and_privileges": ["no model calls, no external mutation, no secrets"],
|
||||
"persistence_and_migrations": ["none"],
|
||||
"failure_retry_restart": ["invalid unresolved evidence fails closed; verifier remains stateless and repeatable"],
|
||||
"assumptions": ["generated cases still require an explicit verification spec before quality can be claimed"],
|
||||
"acceptance_scenarios": ["JSON field match verifies with evidence", "JSON mismatch is unsupported", "unresolved evidence yields exactly two bounded probes", "generated cases do not bypass verifier or promotion gate"],
|
||||
"required_tests": ["go test", "go vet", "preflight", "check-diff", "check-architecture", "independent", "git diff --check", "CI", "production public health check"],
|
||||
"terminal_gap_states": ["BLOCKED_BY_MISSING_CAPABILITY", "BLOCKED_BY_SPEC_GAP"]
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
{
|
||||
"task_id": "DEV-000223",
|
||||
"task_revision": 1,
|
||||
"architecture_revision": "ADR-0018",
|
||||
"base_repo_revision": "ee68e2207eff5535c79cc594637a971cea03d71c",
|
||||
"risk_tier": "E",
|
||||
"intent": "Reconcile the status projection with the merged DEV-000222 verifier stage and preserve explicit remaining runtime and UI gaps.",
|
||||
"requirement_ids": ["current audit evidence", "task lineage accuracy", "negative-state transparency"],
|
||||
"allowed_change_scope": {"modules": ["operations-evidence"], "paths": ["docs/PROJECT_STATUS.md", "dev/ACTIVE_TASK.json", "dev/tasks/DEV-000223-status-reconciliation.json"]},
|
||||
"forbidden_change_scope": {"paths": ["architecture/**", "security/**", "cmd/**", "modules/**", "contracts/**", "database/**", "deploy/**", "LLM-local/**"]},
|
||||
"allowed_architecture_delta": {"new_cross_module_dependencies": 0, "new_canonical_owners": 0, "new_privileges": 0, "new_third_party_dependencies": 0, "new_database_owner_roles": 0},
|
||||
"contracts": {"consume": ["merged task lineage", "existing redacted live evidence"], "may_change": ["operations status projection"]},
|
||||
"semantic_owner": "operations-evidence projection",
|
||||
"canonical_state": ["none; docs are rebuildable projections"],
|
||||
"state_transitions": ["stale task pointer -> merged verifier stage pointer"],
|
||||
"side_effects_and_privileges": ["none; documentation only"],
|
||||
"persistence_and_migrations": ["none"],
|
||||
"failure_retry_restart": ["historical evidence remains immutable"],
|
||||
"assumptions": ["DEV-000222 main revision ee68e22 is the current deployed baseline"],
|
||||
"acceptance_scenarios": ["status names DEV-000222 as last completed", "remaining bridge and current authenticated UI gaps remain explicit", "no secret value is added"],
|
||||
"required_tests": ["go test", "go vet", "preflight", "check-diff", "check-architecture", "independent", "git diff --check", "CI", "production public health check"],
|
||||
"terminal_gap_states": ["BLOCKED_BY_MISSING_CAPABILITY", "BLOCKED_BY_SPEC_GAP"]
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
{
|
||||
"task_id": "DEV-000224",
|
||||
"task_revision": 1,
|
||||
"architecture_revision": "ADR-0018",
|
||||
"base_repo_revision": "c4411afaaf21886cba9b5ead586afb26d697a355",
|
||||
"risk_tier": "E",
|
||||
"intent": "Record the current production model-fleet readiness and dynamic tier selection without claiming quality or promotion.",
|
||||
"requirement_ids": ["model fleet/failover", "GPU is not intelligence priority", "truthful operational evidence"],
|
||||
"allowed_change_scope": {"modules": ["operations-evidence"], "paths": ["docs/PROJECT_STATUS.md", "docs/evidence/fleet-readiness-live-2026-08-23-v2.json", "dev/ACTIVE_TASK.json", "dev/tasks/DEV-000224-live-fleet-evidence.json"]},
|
||||
"forbidden_change_scope": {"paths": ["architecture/**", "security/**", "cmd/**", "modules/**", "contracts/**", "database/**", "deploy/**", "LLM-local/**"]},
|
||||
"allowed_architecture_delta": {"new_cross_module_dependencies": 0, "new_canonical_owners": 0, "new_privileges": 0, "new_third_party_dependencies": 0, "new_database_owner_roles": 0},
|
||||
"contracts": {"consume": ["live cognitive-worker fleet readiness response"], "may_change": ["redacted evidence and status projection"]},
|
||||
"semantic_owner": "operations-evidence projection",
|
||||
"canonical_state": ["none; evidence is a rebuildable projection"],
|
||||
"state_transitions": ["live fleet readiness -> redacted evidence record"],
|
||||
"side_effects_and_privileges": ["read-only production inspection"],
|
||||
"persistence_and_migrations": ["none"],
|
||||
"failure_retry_restart": ["unavailable providers remain explicit; no missing data is promoted to ready"],
|
||||
"assumptions": ["fleet endpoint response is authoritative for this observation window"],
|
||||
"acceptance_scenarios": ["FAST unavailable selects available MEDIUM", "HEAVY availability does not imply priority", "LIFEBOAT remains fallback", "secret values are absent"],
|
||||
"required_tests": ["go test", "go vet", "preflight", "check-diff", "check-architecture", "independent", "git diff --check", "CI", "production public health and fleet live check"],
|
||||
"terminal_gap_states": ["BLOCKED_BY_MISSING_CAPABILITY", "BLOCKED_BY_SPEC_GAP"]
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
{
|
||||
"task_id": "DEV-000225",
|
||||
"task_revision": 1,
|
||||
"architecture_revision": "ADR-0018",
|
||||
"base_repo_revision": "e080c34a772caa5d256ab9f4ae0d2568bb8a7fb9",
|
||||
"risk_tier": "E",
|
||||
"intent": "Serve the existing chat/login shell at the direct /login route so users can reach authentication without a 404.",
|
||||
"requirement_ids": ["UI login direct route", "authenticated acceptance entrypoint"],
|
||||
"allowed_change_scope": {"modules": ["creator-web"], "paths": ["cmd/creator-web/server.go", "cmd/creator-web/server_test.go", "dev/ACTIVE_TASK.json", "dev/tasks/DEV-000225-login-route-fix.json"]},
|
||||
"forbidden_change_scope": {"paths": ["architecture/**", "security/**", "database/**", "deploy/**", "contracts/**", "modules/**", "LLM-local/**"]},
|
||||
"allowed_architecture_delta": {"new_cross_module_dependencies": 0, "new_canonical_owners": 0, "new_privileges": 0, "new_third_party_dependencies": 0, "new_database_owner_roles": 0},
|
||||
"contracts": {"consume": ["existing embedded chat/login shell"], "may_change": ["HTTP route mapping only"]},
|
||||
"semantic_owner": "creator-web route projection",
|
||||
"canonical_state": ["none"],
|
||||
"state_transitions": ["GET /login -> existing chat shell"],
|
||||
"side_effects_and_privileges": ["none; GET only"],
|
||||
"persistence_and_migrations": ["none"],
|
||||
"failure_retry_restart": ["route remains deterministic and stateless"],
|
||||
"assumptions": ["authentication behavior remains unchanged and is handled by the existing chat shell"],
|
||||
"acceptance_scenarios": ["GET /login returns 200", "login inputs are present", "root route remains unchanged", "no credential values are added"],
|
||||
"required_tests": ["go test", "go vet", "preflight", "check-diff", "check-architecture", "independent", "git diff --check", "CI", "production public health and /login check"],
|
||||
"terminal_gap_states": ["BLOCKED_BY_SPEC_GAP"]
|
||||
}
|
||||
@@ -2,6 +2,8 @@
|
||||
|
||||
## Current audit delta — 2026-08-23
|
||||
|
||||
- `LAST_COMPLETED = DEV-000222`; independent task verification now supports exact text and structured top-level JSON-field checks, and unresolved outcomes can produce two bounded additional case candidates after the general evaluation stage. The task packet is merged to `main` and deployed; quality remains unclaimed unless verifier evidence is present.
|
||||
|
||||
- `CURRENT_PRODUCTION_STATUS = LIVE_HEALTHY`; public `/healthz` is healthy. The exact deployed revision is authoritative in the deployment log and merge record, not duplicated in this mutable status projection. Evidence: `docs/evidence/audit-runtime-status-2026-08-23-v1.json`.
|
||||
- `FAST_PROVIDER_STATUS = LIVE_SMOKE_SUPPORTED`: local CPU gateway authentication is configured through a protected file reference; four bounded local-only advisory cases completed through cognitive-worker. Evidence: `docs/evidence/wp-601-fast-gateway-live-smoke-2026-08-22-v1.json`.
|
||||
- `MEDIUM_PROVIDER_STATUS = LIVE_BOUNDED_SMOKE_SUPPORTED_UNPROMOTED`; four bounded local-provider cases passed with template thinking disabled. This is availability/response-shape evidence only; lifecycle qualification, activation and promotion remain unclaimed. Evidence: `docs/evidence/medium-live-bounded-smoke-2026-08-23-v1.json`.
|
||||
@@ -15,13 +17,14 @@
|
||||
- `UI_AUTHENTICATED_ACCEPTANCE = AUTHENTICATED_QUICK_SMOKE_PASS`; browser-driven smoke completed in the active production session: chat response received, admin login accepted, diagnostics 7/7 passed, operations fleet 4/4 READY, and action history loaded. Deep mutation, delegation-specific and long-running acceptance remain intentionally deferred.
|
||||
- `CURRENT_UI_SESSION_STATUS = NO_CURRENT_AUTHENTICATED_SESSION`; the historical quick-smoke evidence above is retained, but the current acceptance run has no signed-in browser session, so no new authenticated visual result is claimed. Public cabinet shell and unauthenticated protection were checked separately.
|
||||
- `AGENT_WORKER_STATUS_CORRECTION = LIVE_TRUTHFUL_LOCAL_FALLBACK`; after DEV-000216, absence of the protected bridge is reported as `executor_configured=false`, `mode=local-fallback`, `ready=false`; local task evaluation is not represented as external agent execution. Evidence: `docs/evidence/audit-runtime-status-2026-08-23-v2.json`.
|
||||
- `FLEET_READINESS_LIVE = MEDIUM_SELECTED_BY_AVAILABILITY`; the live cognitive worker observed FAST unavailable, MEDIUM available, HEAVY available and LIFEBOAT available; it selected MEDIUM while the configured preference remained FAST. This is routing evidence only, not a quality or promotion claim. Evidence: `docs/evidence/fleet-readiness-live-2026-08-23-v2.json`.
|
||||
|
||||
## Current governance checkpoint
|
||||
|
||||
- `COMPLETION_MATRIX = docs/evidence/audit-completion-matrix-2026-08-23-v3.json`; all seven objective areas are mapped to current repository/production proof, with external bridge, GPU promotion/quality and deep-acceptance gaps explicit.
|
||||
|
||||
- `ACTIVE_TASK = dev/tasks/DEV-000217-current-status-evidence.json revision 1`
|
||||
- `ACTIVE_TASK = dev/tasks/DEV-000224-live-fleet-evidence.json revision 1`
|
||||
- `LINEAGE_DECISION = docs/GOVERNANCE_LINEAGE_DECISION_2026-08-22.md`; historical packets remain provenance and are not current authority.
|
||||
- `REGISTRY_STATUS = YAML and JSON semantic mirrors agree`
|
||||
- `PRODUCTION_DOMAIN = docker.cin.su`; shared docker-test is not used for this project.
|
||||
- `NEXT_WORK = provision/qualify an independently operated Agent Execution bridge when available, complete deferred deep browser acceptance with a signed-in session, and add task-quality verification only when live work produces an unresolved case; retain explicit evidence for all remaining negative directions.`
|
||||
- `NEXT_WORK = qualify an independently operated Agent Execution bridge when available and complete deferred deep browser acceptance with a current signed-in session; retain local fallback and general-first task evaluation when either capability is unavailable.`
|
||||
|
||||
@@ -0,0 +1,17 @@
|
||||
{
|
||||
"observed_on": "2026-08-23",
|
||||
"source": "production cognitive-worker read-only endpoint on docker.cin.su",
|
||||
"endpoint": "/v1/fleet/readiness",
|
||||
"preferred_tier": "FAST",
|
||||
"selected_tier": "MEDIUM",
|
||||
"ready_tiers": ["MEDIUM", "HEAVY", "LIFEBOAT"],
|
||||
"fallback_tier": "LIFEBOAT",
|
||||
"members": [
|
||||
{"tier": "FAST", "ready": false, "reason_class": "provider connection refused"},
|
||||
{"tier": "MEDIUM", "ready": true, "reason_class": "available"},
|
||||
{"tier": "HEAVY", "ready": true, "reason_class": "available"},
|
||||
{"tier": "LIFEBOAT", "ready": true, "reason_class": "available"}
|
||||
],
|
||||
"interpretation": "Selection follows current readiness and preserves LIFEBOAT fallback; GPU/HEAVY availability does not by itself override the preferred tier or prove superior quality.",
|
||||
"secret_policy": "values omitted; only typed states and reason classes retained"
|
||||
}
|
||||
@@ -3,6 +3,7 @@ package domain
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"git.cin.su/m/Assistant/contracts/artifact"
|
||||
"git.cin.su/m/Assistant/contracts/capability"
|
||||
@@ -10,6 +11,28 @@ import (
|
||||
|
||||
var ErrInvalidTaskEvaluation = errors.New("invalid task evaluation")
|
||||
|
||||
type UnresolvedTaskEvidence struct {
|
||||
ID string
|
||||
OriginalPrompt string
|
||||
FailureSummary string
|
||||
EvidenceArtifactRef string
|
||||
}
|
||||
|
||||
// BuildAdditionalTaskEvaluationCases creates two bounded probes from one
|
||||
// unresolved outcome. It creates prompts only; quality still requires a
|
||||
// verification spec and independent verifier evidence.
|
||||
func BuildAdditionalTaskEvaluationCases(value UnresolvedTaskEvidence) ([]capability.TaskEvaluationCase, error) {
|
||||
if strings.TrimSpace(value.ID) == "" || strings.TrimSpace(value.OriginalPrompt) == "" || strings.TrimSpace(value.FailureSummary) == "" || strings.TrimSpace(value.EvidenceArtifactRef) == "" {
|
||||
return nil, ErrInvalidTaskEvaluation
|
||||
}
|
||||
base := strings.TrimSpace(value.OriginalPrompt)
|
||||
failure := strings.TrimSpace(value.FailureSummary)
|
||||
return []capability.TaskEvaluationCase{
|
||||
{CaseID: value.ID + ":reproduce", Prompt: base, EvidenceArtifactRef: value.EvidenceArtifactRef},
|
||||
{CaseID: value.ID + ":address-failure", Prompt: base + "\n\nAddress this observed unresolved condition without assuming it is true: " + failure, EvidenceArtifactRef: value.EvidenceArtifactRef},
|
||||
}, nil
|
||||
}
|
||||
|
||||
const automaticExpansionMinimumOverallScore int64 = 700
|
||||
|
||||
// AutomaticTaskEvaluationExpansion decides whether a completed general
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package domain
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.cin.su/m/Assistant/contracts/artifact"
|
||||
@@ -59,3 +60,10 @@ func TestAutomaticExpansionDoesNotRunWhenAQualifiedCandidateExists(t *testing.T)
|
||||
t.Fatalf("qualified candidate must suppress expansion: profile=%+v expanded=%t err=%v", value, expanded, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildAdditionalTaskEvaluationCasesFromUnresolvedEvidence(t *testing.T) {
|
||||
cases, err := BuildAdditionalTaskEvaluationCases(UnresolvedTaskEvidence{ID: "outcome-1", OriginalPrompt: "compute answer", FailureSummary: "the answer was not verified", EvidenceArtifactRef: "artifact-outcome-1"})
|
||||
if err != nil || len(cases) != 2 || cases[0].CaseID != "outcome-1:reproduce" || !strings.Contains(cases[1].Prompt, "not verified") {
|
||||
t.Fatalf("unexpected generated cases: cases=%+v err=%v", cases, err)
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user