Compare commits
2
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
3d9a5bccba | ||
|
|
d736500d98 |
@@ -34,11 +34,11 @@ func verifyHandler(w http.ResponseWriter, r *http.Request) {
|
|||||||
var request capability.TaskCaseVerificationRequest
|
var request capability.TaskCaseVerificationRequest
|
||||||
decoder := json.NewDecoder(http.MaxBytesReader(w, r.Body, 64<<10))
|
decoder := json.NewDecoder(http.MaxBytesReader(w, r.Body, 64<<10))
|
||||||
decoder.DisallowUnknownFields()
|
decoder.DisallowUnknownFields()
|
||||||
if err := decoder.Decode(&request); err != nil || request.CaseID == "" || request.ResponseText == "" || request.EvidenceArtifactRef == "" || request.LatencyMicros < 0 || request.Verification.Mode != capability.TaskCaseVerificationExactText || request.Verification.ExpectedText == "" || request.Verification.MinimumQualityMillis < 0 || request.Verification.MinimumQualityMillis > 1000 {
|
if err := decoder.Decode(&request); err != nil || request.CaseID == "" || request.ResponseText == "" || request.EvidenceArtifactRef == "" || request.LatencyMicros < 0 || (request.Verification.Mode != capability.TaskCaseVerificationExactText && request.Verification.Mode != capability.TaskCaseVerificationJSONFieldEquals) || (request.Verification.Mode == capability.TaskCaseVerificationJSONFieldEquals && request.Verification.FieldPath == "") || request.Verification.ExpectedText == "" || request.Verification.MinimumQualityMillis < 0 || request.Verification.MinimumQualityMillis > 1000 {
|
||||||
http.Error(w, "invalid task verification request", http.StatusBadRequest)
|
http.Error(w, "invalid task verification request", http.StatusBadRequest)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
matched := strings.TrimSpace(request.ResponseText) == strings.TrimSpace(request.Verification.ExpectedText)
|
matched := matches(request)
|
||||||
quality := int64(0)
|
quality := int64(0)
|
||||||
result := capability.EvaluationUnsupported
|
result := capability.EvaluationUnsupported
|
||||||
if matched {
|
if matched {
|
||||||
@@ -53,11 +53,23 @@ func verifyHandler(w http.ResponseWriter, r *http.Request) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func verificationEvidence(request capability.TaskCaseVerificationRequest, matched bool) string {
|
func verificationEvidence(request capability.TaskCaseVerificationRequest, matched bool) string {
|
||||||
payload := fmt.Sprintf("%s|%s|%s|%t", verifierIdentity, request.CaseID, request.EvidenceArtifactRef, matched)
|
payload := fmt.Sprintf("%s|%s|%s|%s|%s|%t", verifierIdentity, request.CaseID, request.EvidenceArtifactRef, request.Verification.Mode, request.Verification.FieldPath, matched)
|
||||||
digest := sha256.Sum256([]byte(payload))
|
digest := sha256.Sum256([]byte(payload))
|
||||||
return "verification:" + hex.EncodeToString(digest[:])
|
return "verification:" + hex.EncodeToString(digest[:])
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func matches(request capability.TaskCaseVerificationRequest) bool {
|
||||||
|
if request.Verification.Mode == capability.TaskCaseVerificationExactText {
|
||||||
|
return strings.TrimSpace(request.ResponseText) == strings.TrimSpace(request.Verification.ExpectedText)
|
||||||
|
}
|
||||||
|
var document map[string]json.RawMessage
|
||||||
|
if json.Unmarshal([]byte(request.ResponseText), &document) != nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
value, ok := document[request.Verification.FieldPath]
|
||||||
|
return ok && strings.TrimSpace(string(value)) == strings.TrimSpace(request.Verification.ExpectedText)
|
||||||
|
}
|
||||||
|
|
||||||
func writeJSON(w http.ResponseWriter, value any) {
|
func writeJSON(w http.ResponseWriter, value any) {
|
||||||
w.Header().Set("Content-Type", "application/json")
|
w.Header().Set("Content-Type", "application/json")
|
||||||
_ = json.NewEncoder(w).Encode(value)
|
_ = json.NewEncoder(w).Encode(value)
|
||||||
|
|||||||
@@ -29,3 +29,13 @@ func TestVerifyMismatchIsUnsupportedWithEvidence(t *testing.T) {
|
|||||||
t.Fatalf("unexpected mismatch response: code=%d body=%s value=%+v", recorder.Code, recorder.Body.String(), response)
|
t.Fatalf("unexpected mismatch response: code=%d body=%s value=%+v", recorder.Code, recorder.Body.String(), response)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestVerifyJSONFieldEquals(t *testing.T) {
|
||||||
|
payload, _ := json.Marshal(capability.TaskCaseVerificationRequest{CaseID: "json-1", ResponseText: `{"answer":42}`, EvidenceArtifactRef: "case-artifact", Verification: capability.TaskCaseVerification{Mode: capability.TaskCaseVerificationJSONFieldEquals, FieldPath: "answer", ExpectedText: "42", MinimumQualityMillis: 850}})
|
||||||
|
recorder := httptest.NewRecorder()
|
||||||
|
verifyHandler(recorder, httptest.NewRequest(http.MethodPost, "/v1/task-evaluations/verify", bytes.NewReader(payload)))
|
||||||
|
var response capability.TaskCaseVerificationResponse
|
||||||
|
if recorder.Code != http.StatusOK || json.NewDecoder(recorder.Body).Decode(&response) != nil || response.Result != capability.EvaluationSupported || response.QualityScoreMillis != 850 {
|
||||||
|
t.Fatalf("unexpected JSON verification response: code=%d body=%s value=%+v", recorder.Code, recorder.Body.String(), response)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -72,12 +72,16 @@ type TaskEvaluationExpansionRequest struct {
|
|||||||
|
|
||||||
type TaskCaseVerificationMode string
|
type TaskCaseVerificationMode string
|
||||||
|
|
||||||
const TaskCaseVerificationExactText TaskCaseVerificationMode = "EXACT_TEXT"
|
const (
|
||||||
|
TaskCaseVerificationExactText TaskCaseVerificationMode = "EXACT_TEXT"
|
||||||
|
TaskCaseVerificationJSONFieldEquals TaskCaseVerificationMode = "JSON_FIELD_EQUALS"
|
||||||
|
)
|
||||||
|
|
||||||
// TaskCaseVerification is a bounded, task-owned rubric. The expected text is
|
// TaskCaseVerification is a bounded, task-owned rubric. The expected text is
|
||||||
// test data, not model output, and is never sent to the cognitive provider.
|
// test data, not model output, and is never sent to the cognitive provider.
|
||||||
type TaskCaseVerification struct {
|
type TaskCaseVerification struct {
|
||||||
Mode TaskCaseVerificationMode
|
Mode TaskCaseVerificationMode
|
||||||
|
FieldPath string
|
||||||
ExpectedText string
|
ExpectedText string
|
||||||
MinimumQualityMillis int64
|
MinimumQualityMillis int64
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,5 +1,5 @@
|
|||||||
{
|
{
|
||||||
"task_file": "dev/tasks/DEV-000221-task-verifier-runtime.json",
|
"task_file": "dev/tasks/DEV-000222-verifier-modes-and-case-builder.json",
|
||||||
"task_id": "DEV-000221",
|
"task_id": "DEV-000222",
|
||||||
"task_revision": 1
|
"task_revision": 1
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,23 @@
|
|||||||
|
{
|
||||||
|
"task_id": "DEV-000222",
|
||||||
|
"task_revision": 1,
|
||||||
|
"architecture_revision": "ADR-0018",
|
||||||
|
"base_repo_revision": "d736500d9818a7748c3715c442edb695c5aed1100",
|
||||||
|
"risk_tier": "E",
|
||||||
|
"intent": "Extend the independent task verifier with a structured JSON field mode and add a bounded case builder driven by unresolved evidence.",
|
||||||
|
"requirement_ids": ["task-aware verification", "additional cases during lifetime", "general-first bounded expansion"],
|
||||||
|
"allowed_change_scope": {"modules": ["capability", "task-verifier"], "paths": ["contracts/capability/task_evaluation.go", "cmd/task-verifier/main.go", "cmd/task-verifier/main_test.go", "modules/capability/domain/task_evaluation.go", "modules/capability/domain/task_evaluation_test.go", "dev/ACTIVE_TASK.json", "dev/tasks/DEV-000222-verifier-modes-and-case-builder.json"]},
|
||||||
|
"forbidden_change_scope": {"paths": ["architecture/**", "security/**", "database/**", "deploy/**", "cmd/core-runtime/**", "cmd/creator-web/**", "cmd/agent-worker/**", "cmd/cognitive-worker/**", "modules/identity/**", "LLM-local/**"]},
|
||||||
|
"allowed_architecture_delta": {"new_cross_module_dependencies": 0, "new_canonical_owners": 0, "new_privileges": 0, "new_third_party_dependencies": 0, "new_database_owner_roles": 0},
|
||||||
|
"contracts": {"consume": ["TaskCaseVerification", "TaskEvaluationCase", "unresolved outcome evidence"], "may_change": ["verification modes and bounded case construction" ]},
|
||||||
|
"semantic_owner": "capability owns evaluation policy; verifier remains stateless adapter",
|
||||||
|
"canonical_state": ["none; case builder returns candidates and does not persist or activate them"],
|
||||||
|
"state_transitions": ["unresolved evidence -> two bounded additional case candidates", "structured response -> verifier evidence -> supported/unsupported"],
|
||||||
|
"side_effects_and_privileges": ["no model calls, no external mutation, no secrets"],
|
||||||
|
"persistence_and_migrations": ["none"],
|
||||||
|
"failure_retry_restart": ["invalid unresolved evidence fails closed; verifier remains stateless and repeatable"],
|
||||||
|
"assumptions": ["generated cases still require an explicit verification spec before quality can be claimed"],
|
||||||
|
"acceptance_scenarios": ["JSON field match verifies with evidence", "JSON mismatch is unsupported", "unresolved evidence yields exactly two bounded probes", "generated cases do not bypass verifier or promotion gate"],
|
||||||
|
"required_tests": ["go test", "go vet", "preflight", "check-diff", "check-architecture", "independent", "git diff --check", "CI", "production public health check"],
|
||||||
|
"terminal_gap_states": ["BLOCKED_BY_MISSING_CAPABILITY", "BLOCKED_BY_SPEC_GAP"]
|
||||||
|
}
|
||||||
@@ -3,6 +3,7 @@ package domain
|
|||||||
import (
|
import (
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"strings"
|
||||||
|
|
||||||
"git.cin.su/m/Assistant/contracts/artifact"
|
"git.cin.su/m/Assistant/contracts/artifact"
|
||||||
"git.cin.su/m/Assistant/contracts/capability"
|
"git.cin.su/m/Assistant/contracts/capability"
|
||||||
@@ -10,6 +11,28 @@ import (
|
|||||||
|
|
||||||
var ErrInvalidTaskEvaluation = errors.New("invalid task evaluation")
|
var ErrInvalidTaskEvaluation = errors.New("invalid task evaluation")
|
||||||
|
|
||||||
|
type UnresolvedTaskEvidence struct {
|
||||||
|
ID string
|
||||||
|
OriginalPrompt string
|
||||||
|
FailureSummary string
|
||||||
|
EvidenceArtifactRef string
|
||||||
|
}
|
||||||
|
|
||||||
|
// BuildAdditionalTaskEvaluationCases creates two bounded probes from one
|
||||||
|
// unresolved outcome. It creates prompts only; quality still requires a
|
||||||
|
// verification spec and independent verifier evidence.
|
||||||
|
func BuildAdditionalTaskEvaluationCases(value UnresolvedTaskEvidence) ([]capability.TaskEvaluationCase, error) {
|
||||||
|
if strings.TrimSpace(value.ID) == "" || strings.TrimSpace(value.OriginalPrompt) == "" || strings.TrimSpace(value.FailureSummary) == "" || strings.TrimSpace(value.EvidenceArtifactRef) == "" {
|
||||||
|
return nil, ErrInvalidTaskEvaluation
|
||||||
|
}
|
||||||
|
base := strings.TrimSpace(value.OriginalPrompt)
|
||||||
|
failure := strings.TrimSpace(value.FailureSummary)
|
||||||
|
return []capability.TaskEvaluationCase{
|
||||||
|
{CaseID: value.ID + ":reproduce", Prompt: base, EvidenceArtifactRef: value.EvidenceArtifactRef},
|
||||||
|
{CaseID: value.ID + ":address-failure", Prompt: base + "\n\nAddress this observed unresolved condition without assuming it is true: " + failure, EvidenceArtifactRef: value.EvidenceArtifactRef},
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
const automaticExpansionMinimumOverallScore int64 = 700
|
const automaticExpansionMinimumOverallScore int64 = 700
|
||||||
|
|
||||||
// AutomaticTaskEvaluationExpansion decides whether a completed general
|
// AutomaticTaskEvaluationExpansion decides whether a completed general
|
||||||
|
|||||||
@@ -1,6 +1,7 @@
|
|||||||
package domain
|
package domain
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"git.cin.su/m/Assistant/contracts/artifact"
|
"git.cin.su/m/Assistant/contracts/artifact"
|
||||||
@@ -59,3 +60,10 @@ func TestAutomaticExpansionDoesNotRunWhenAQualifiedCandidateExists(t *testing.T)
|
|||||||
t.Fatalf("qualified candidate must suppress expansion: profile=%+v expanded=%t err=%v", value, expanded, err)
|
t.Fatalf("qualified candidate must suppress expansion: profile=%+v expanded=%t err=%v", value, expanded, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestBuildAdditionalTaskEvaluationCasesFromUnresolvedEvidence(t *testing.T) {
|
||||||
|
cases, err := BuildAdditionalTaskEvaluationCases(UnresolvedTaskEvidence{ID: "outcome-1", OriginalPrompt: "compute answer", FailureSummary: "the answer was not verified", EvidenceArtifactRef: "artifact-outcome-1"})
|
||||||
|
if err != nil || len(cases) != 2 || cases[0].CaseID != "outcome-1:reproduce" || !strings.Contains(cases[1].Prompt, "not verified") {
|
||||||
|
t.Fatalf("unexpected generated cases: cases=%+v err=%v", cases, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user