feat(work): add independent task verifier runtime
This commit is contained in:
@@ -0,0 +1,64 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
"git.cin.su/m/Assistant/contracts/capability"
|
||||
)
|
||||
|
||||
const verifierIdentity = "deterministic-task-verifier-v1"
|
||||
|
||||
func main() {
|
||||
mux := http.NewServeMux()
|
||||
mux.HandleFunc("GET /healthz", func(w http.ResponseWriter, _ *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{"runtime":"task-verifier","status":"ok"}`))
|
||||
})
|
||||
mux.HandleFunc("POST /v1/task-evaluations/verify", verifyHandler)
|
||||
addr := ":8091"
|
||||
if value := strings.TrimSpace(os.Getenv("LISTEN_ADDR")); value != "" {
|
||||
addr = value
|
||||
}
|
||||
if err := http.ListenAndServe(addr, mux); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
}
|
||||
|
||||
func verifyHandler(w http.ResponseWriter, r *http.Request) {
|
||||
var request capability.TaskCaseVerificationRequest
|
||||
decoder := json.NewDecoder(http.MaxBytesReader(w, r.Body, 64<<10))
|
||||
decoder.DisallowUnknownFields()
|
||||
if err := decoder.Decode(&request); err != nil || request.CaseID == "" || request.ResponseText == "" || request.EvidenceArtifactRef == "" || request.LatencyMicros < 0 || request.Verification.Mode != capability.TaskCaseVerificationExactText || request.Verification.ExpectedText == "" || request.Verification.MinimumQualityMillis < 0 || request.Verification.MinimumQualityMillis > 1000 {
|
||||
http.Error(w, "invalid task verification request", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
matched := strings.TrimSpace(request.ResponseText) == strings.TrimSpace(request.Verification.ExpectedText)
|
||||
quality := int64(0)
|
||||
result := capability.EvaluationUnsupported
|
||||
if matched {
|
||||
quality = request.Verification.MinimumQualityMillis
|
||||
if quality == 0 {
|
||||
quality = 1000
|
||||
}
|
||||
result = capability.EvaluationSupported
|
||||
}
|
||||
evidence := verificationEvidence(request, matched)
|
||||
writeJSON(w, capability.TaskCaseVerificationResponse{CaseID: request.CaseID, Result: result, QualityScoreMillis: quality, LatencyMicros: request.LatencyMicros, EvidenceArtifactRef: request.EvidenceArtifactRef, VerificationEvidenceRef: evidence, VerifierIdentity: verifierIdentity})
|
||||
}
|
||||
|
||||
func verificationEvidence(request capability.TaskCaseVerificationRequest, matched bool) string {
|
||||
payload := fmt.Sprintf("%s|%s|%s|%t", verifierIdentity, request.CaseID, request.EvidenceArtifactRef, matched)
|
||||
digest := sha256.Sum256([]byte(payload))
|
||||
return "verification:" + hex.EncodeToString(digest[:])
|
||||
}
|
||||
|
||||
func writeJSON(w http.ResponseWriter, value any) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_ = json.NewEncoder(w).Encode(value)
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
|
||||
"git.cin.su/m/Assistant/contracts/capability"
|
||||
)
|
||||
|
||||
func TestVerifyExactTextProducesIndependentEvidence(t *testing.T) {
|
||||
payload, _ := json.Marshal(capability.TaskCaseVerificationRequest{CaseID: "case-1", ResponseText: " answer ", EvidenceArtifactRef: "case-artifact", Verification: capability.TaskCaseVerification{Mode: capability.TaskCaseVerificationExactText, ExpectedText: "answer", MinimumQualityMillis: 900}})
|
||||
recorder := httptest.NewRecorder()
|
||||
verifyHandler(recorder, httptest.NewRequest(http.MethodPost, "/v1/task-evaluations/verify", bytes.NewReader(payload)))
|
||||
var response capability.TaskCaseVerificationResponse
|
||||
if recorder.Code != http.StatusOK || json.NewDecoder(recorder.Body).Decode(&response) != nil || response.Result != capability.EvaluationSupported || response.QualityScoreMillis != 900 || response.VerifierIdentity == "" || response.VerificationEvidenceRef == "" {
|
||||
t.Fatalf("unexpected verification response: code=%d body=%s value=%+v", recorder.Code, recorder.Body.String(), response)
|
||||
}
|
||||
}
|
||||
|
||||
func TestVerifyMismatchIsUnsupportedWithEvidence(t *testing.T) {
|
||||
payload, _ := json.Marshal(capability.TaskCaseVerificationRequest{CaseID: "case-1", ResponseText: "wrong", EvidenceArtifactRef: "case-artifact", Verification: capability.TaskCaseVerification{Mode: capability.TaskCaseVerificationExactText, ExpectedText: "answer", MinimumQualityMillis: 900}})
|
||||
recorder := httptest.NewRecorder()
|
||||
verifyHandler(recorder, httptest.NewRequest(http.MethodPost, "/v1/task-evaluations/verify", bytes.NewReader(payload)))
|
||||
var response capability.TaskCaseVerificationResponse
|
||||
if recorder.Code != http.StatusOK || json.NewDecoder(recorder.Body).Decode(&response) != nil || response.Result != capability.EvaluationUnsupported || response.QualityScoreMillis != 0 || response.VerificationEvidenceRef == "" {
|
||||
t.Fatalf("unexpected mismatch response: code=%d body=%s value=%+v", recorder.Code, recorder.Body.String(), response)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user