From 3d9a5bccbaa1f17a3eb7cfd7b8fcccdce00f11fe Mon Sep 17 00:00:00 2001 From: Mikhail Date: Sun, 23 Aug 2026 14:30:45 +0300 Subject: [PATCH] feat: extend task verifier and unresolved case builder --- cmd/task-verifier/main.go | 18 ++++++++++++--- cmd/task-verifier/main_test.go | 10 ++++++++ contracts/capability/task_evaluation.go | 6 ++++- dev/ACTIVE_TASK.json | 4 ++-- ...00222-verifier-modes-and-case-builder.json | 23 +++++++++++++++++++ modules/capability/domain/task_evaluation.go | 23 +++++++++++++++++++ .../capability/domain/task_evaluation_test.go | 8 +++++++ 7 files changed, 86 insertions(+), 6 deletions(-) create mode 100644 dev/tasks/DEV-000222-verifier-modes-and-case-builder.json diff --git a/cmd/task-verifier/main.go b/cmd/task-verifier/main.go index 7e74b50..ddfb906 100644 --- a/cmd/task-verifier/main.go +++ b/cmd/task-verifier/main.go @@ -34,11 +34,11 @@ func verifyHandler(w http.ResponseWriter, r *http.Request) { var request capability.TaskCaseVerificationRequest decoder := json.NewDecoder(http.MaxBytesReader(w, r.Body, 64<<10)) decoder.DisallowUnknownFields() - if err := decoder.Decode(&request); err != nil || request.CaseID == "" || request.ResponseText == "" || request.EvidenceArtifactRef == "" || request.LatencyMicros < 0 || request.Verification.Mode != capability.TaskCaseVerificationExactText || request.Verification.ExpectedText == "" || request.Verification.MinimumQualityMillis < 0 || request.Verification.MinimumQualityMillis > 1000 { + if err := decoder.Decode(&request); err != nil || request.CaseID == "" || request.ResponseText == "" || request.EvidenceArtifactRef == "" || request.LatencyMicros < 0 || (request.Verification.Mode != capability.TaskCaseVerificationExactText && request.Verification.Mode != capability.TaskCaseVerificationJSONFieldEquals) || (request.Verification.Mode == capability.TaskCaseVerificationJSONFieldEquals && request.Verification.FieldPath == "") || request.Verification.ExpectedText == "" || request.Verification.MinimumQualityMillis < 0 || request.Verification.MinimumQualityMillis > 1000 { http.Error(w, "invalid task verification request", http.StatusBadRequest) return } - matched := strings.TrimSpace(request.ResponseText) == strings.TrimSpace(request.Verification.ExpectedText) + matched := matches(request) quality := int64(0) result := capability.EvaluationUnsupported if matched { @@ -53,11 +53,23 @@ func verifyHandler(w http.ResponseWriter, r *http.Request) { } func verificationEvidence(request capability.TaskCaseVerificationRequest, matched bool) string { - payload := fmt.Sprintf("%s|%s|%s|%t", verifierIdentity, request.CaseID, request.EvidenceArtifactRef, matched) + payload := fmt.Sprintf("%s|%s|%s|%s|%s|%t", verifierIdentity, request.CaseID, request.EvidenceArtifactRef, request.Verification.Mode, request.Verification.FieldPath, matched) digest := sha256.Sum256([]byte(payload)) return "verification:" + hex.EncodeToString(digest[:]) } +func matches(request capability.TaskCaseVerificationRequest) bool { + if request.Verification.Mode == capability.TaskCaseVerificationExactText { + return strings.TrimSpace(request.ResponseText) == strings.TrimSpace(request.Verification.ExpectedText) + } + var document map[string]json.RawMessage + if json.Unmarshal([]byte(request.ResponseText), &document) != nil { + return false + } + value, ok := document[request.Verification.FieldPath] + return ok && strings.TrimSpace(string(value)) == strings.TrimSpace(request.Verification.ExpectedText) +} + func writeJSON(w http.ResponseWriter, value any) { w.Header().Set("Content-Type", "application/json") _ = json.NewEncoder(w).Encode(value) diff --git a/cmd/task-verifier/main_test.go b/cmd/task-verifier/main_test.go index eb2d35f..0662415 100644 --- a/cmd/task-verifier/main_test.go +++ b/cmd/task-verifier/main_test.go @@ -29,3 +29,13 @@ func TestVerifyMismatchIsUnsupportedWithEvidence(t *testing.T) { t.Fatalf("unexpected mismatch response: code=%d body=%s value=%+v", recorder.Code, recorder.Body.String(), response) } } + +func TestVerifyJSONFieldEquals(t *testing.T) { + payload, _ := json.Marshal(capability.TaskCaseVerificationRequest{CaseID: "json-1", ResponseText: `{"answer":42}`, EvidenceArtifactRef: "case-artifact", Verification: capability.TaskCaseVerification{Mode: capability.TaskCaseVerificationJSONFieldEquals, FieldPath: "answer", ExpectedText: "42", MinimumQualityMillis: 850}}) + recorder := httptest.NewRecorder() + verifyHandler(recorder, httptest.NewRequest(http.MethodPost, "/v1/task-evaluations/verify", bytes.NewReader(payload))) + var response capability.TaskCaseVerificationResponse + if recorder.Code != http.StatusOK || json.NewDecoder(recorder.Body).Decode(&response) != nil || response.Result != capability.EvaluationSupported || response.QualityScoreMillis != 850 { + t.Fatalf("unexpected JSON verification response: code=%d body=%s value=%+v", recorder.Code, recorder.Body.String(), response) + } +} diff --git a/contracts/capability/task_evaluation.go b/contracts/capability/task_evaluation.go index 3c7493f..494b681 100644 --- a/contracts/capability/task_evaluation.go +++ b/contracts/capability/task_evaluation.go @@ -72,12 +72,16 @@ type TaskEvaluationExpansionRequest struct { type TaskCaseVerificationMode string -const TaskCaseVerificationExactText TaskCaseVerificationMode = "EXACT_TEXT" +const ( + TaskCaseVerificationExactText TaskCaseVerificationMode = "EXACT_TEXT" + TaskCaseVerificationJSONFieldEquals TaskCaseVerificationMode = "JSON_FIELD_EQUALS" +) // TaskCaseVerification is a bounded, task-owned rubric. The expected text is // test data, not model output, and is never sent to the cognitive provider. type TaskCaseVerification struct { Mode TaskCaseVerificationMode + FieldPath string ExpectedText string MinimumQualityMillis int64 } diff --git a/dev/ACTIVE_TASK.json b/dev/ACTIVE_TASK.json index e4d74ee..d2f889e 100644 --- a/dev/ACTIVE_TASK.json +++ b/dev/ACTIVE_TASK.json @@ -1,5 +1,5 @@ { - "task_file": "dev/tasks/DEV-000221-task-verifier-runtime.json", - "task_id": "DEV-000221", + "task_file": "dev/tasks/DEV-000222-verifier-modes-and-case-builder.json", + "task_id": "DEV-000222", "task_revision": 1 } diff --git a/dev/tasks/DEV-000222-verifier-modes-and-case-builder.json b/dev/tasks/DEV-000222-verifier-modes-and-case-builder.json new file mode 100644 index 0000000..037119a --- /dev/null +++ b/dev/tasks/DEV-000222-verifier-modes-and-case-builder.json @@ -0,0 +1,23 @@ +{ + "task_id": "DEV-000222", + "task_revision": 1, + "architecture_revision": "ADR-0018", + "base_repo_revision": "d736500d9818a7748c3715c442edb695c5aed1100", + "risk_tier": "E", + "intent": "Extend the independent task verifier with a structured JSON field mode and add a bounded case builder driven by unresolved evidence.", + "requirement_ids": ["task-aware verification", "additional cases during lifetime", "general-first bounded expansion"], + "allowed_change_scope": {"modules": ["capability", "task-verifier"], "paths": ["contracts/capability/task_evaluation.go", "cmd/task-verifier/main.go", "cmd/task-verifier/main_test.go", "modules/capability/domain/task_evaluation.go", "modules/capability/domain/task_evaluation_test.go", "dev/ACTIVE_TASK.json", "dev/tasks/DEV-000222-verifier-modes-and-case-builder.json"]}, + "forbidden_change_scope": {"paths": ["architecture/**", "security/**", "database/**", "deploy/**", "cmd/core-runtime/**", "cmd/creator-web/**", "cmd/agent-worker/**", "cmd/cognitive-worker/**", "modules/identity/**", "LLM-local/**"]}, + "allowed_architecture_delta": {"new_cross_module_dependencies": 0, "new_canonical_owners": 0, "new_privileges": 0, "new_third_party_dependencies": 0, "new_database_owner_roles": 0}, + "contracts": {"consume": ["TaskCaseVerification", "TaskEvaluationCase", "unresolved outcome evidence"], "may_change": ["verification modes and bounded case construction" ]}, + "semantic_owner": "capability owns evaluation policy; verifier remains stateless adapter", + "canonical_state": ["none; case builder returns candidates and does not persist or activate them"], + "state_transitions": ["unresolved evidence -> two bounded additional case candidates", "structured response -> verifier evidence -> supported/unsupported"], + "side_effects_and_privileges": ["no model calls, no external mutation, no secrets"], + "persistence_and_migrations": ["none"], + "failure_retry_restart": ["invalid unresolved evidence fails closed; verifier remains stateless and repeatable"], + "assumptions": ["generated cases still require an explicit verification spec before quality can be claimed"], + "acceptance_scenarios": ["JSON field match verifies with evidence", "JSON mismatch is unsupported", "unresolved evidence yields exactly two bounded probes", "generated cases do not bypass verifier or promotion gate"], + "required_tests": ["go test", "go vet", "preflight", "check-diff", "check-architecture", "independent", "git diff --check", "CI", "production public health check"], + "terminal_gap_states": ["BLOCKED_BY_MISSING_CAPABILITY", "BLOCKED_BY_SPEC_GAP"] +} diff --git a/modules/capability/domain/task_evaluation.go b/modules/capability/domain/task_evaluation.go index 5f33b1e..23d001a 100644 --- a/modules/capability/domain/task_evaluation.go +++ b/modules/capability/domain/task_evaluation.go @@ -3,6 +3,7 @@ package domain import ( "errors" "fmt" + "strings" "git.cin.su/m/Assistant/contracts/artifact" "git.cin.su/m/Assistant/contracts/capability" @@ -10,6 +11,28 @@ import ( var ErrInvalidTaskEvaluation = errors.New("invalid task evaluation") +type UnresolvedTaskEvidence struct { + ID string + OriginalPrompt string + FailureSummary string + EvidenceArtifactRef string +} + +// BuildAdditionalTaskEvaluationCases creates two bounded probes from one +// unresolved outcome. It creates prompts only; quality still requires a +// verification spec and independent verifier evidence. +func BuildAdditionalTaskEvaluationCases(value UnresolvedTaskEvidence) ([]capability.TaskEvaluationCase, error) { + if strings.TrimSpace(value.ID) == "" || strings.TrimSpace(value.OriginalPrompt) == "" || strings.TrimSpace(value.FailureSummary) == "" || strings.TrimSpace(value.EvidenceArtifactRef) == "" { + return nil, ErrInvalidTaskEvaluation + } + base := strings.TrimSpace(value.OriginalPrompt) + failure := strings.TrimSpace(value.FailureSummary) + return []capability.TaskEvaluationCase{ + {CaseID: value.ID + ":reproduce", Prompt: base, EvidenceArtifactRef: value.EvidenceArtifactRef}, + {CaseID: value.ID + ":address-failure", Prompt: base + "\n\nAddress this observed unresolved condition without assuming it is true: " + failure, EvidenceArtifactRef: value.EvidenceArtifactRef}, + }, nil +} + const automaticExpansionMinimumOverallScore int64 = 700 // AutomaticTaskEvaluationExpansion decides whether a completed general diff --git a/modules/capability/domain/task_evaluation_test.go b/modules/capability/domain/task_evaluation_test.go index 3671f50..313b554 100644 --- a/modules/capability/domain/task_evaluation_test.go +++ b/modules/capability/domain/task_evaluation_test.go @@ -1,6 +1,7 @@ package domain import ( + "strings" "testing" "git.cin.su/m/Assistant/contracts/artifact" @@ -59,3 +60,10 @@ func TestAutomaticExpansionDoesNotRunWhenAQualifiedCandidateExists(t *testing.T) t.Fatalf("qualified candidate must suppress expansion: profile=%+v expanded=%t err=%v", value, expanded, err) } } + +func TestBuildAdditionalTaskEvaluationCasesFromUnresolvedEvidence(t *testing.T) { + cases, err := BuildAdditionalTaskEvaluationCases(UnresolvedTaskEvidence{ID: "outcome-1", OriginalPrompt: "compute answer", FailureSummary: "the answer was not verified", EvidenceArtifactRef: "artifact-outcome-1"}) + if err != nil || len(cases) != 2 || cases[0].CaseID != "outcome-1:reproduce" || !strings.Contains(cases[1].Prompt, "not verified") { + t.Fatalf("unexpected generated cases: cases=%+v err=%v", cases, err) + } +} -- 2.54.0