feat(capability): gate additional model evaluations #195

Merged
m merged 1 commits from codex/wp-218-expansion-policy into main 2026-08-23 10:57:53 +00:00
4 changed files with 73 additions and 2 deletions
+2 -2
View File
@@ -1,5 +1,5 @@
{
"task_file": "dev/tasks/DEV-000217-current-status-evidence.json",
"task_id": "DEV-000217",
"task_file": "dev/tasks/DEV-000218-automatic-expansion-policy.json",
"task_id": "DEV-000218",
"task_revision": 1
}
@@ -0,0 +1,23 @@
{
"task_id": "DEV-000218",
"task_revision": 1,
"architecture_revision": "ADR-0018",
"base_repo_revision": "ab840878fe7d21a0a6aeb69193a3d58ea23a56e0",
"risk_tier": "E",
"intent": "Add a conservative deterministic policy gate for creating additional model-evaluation profiles only after general evaluation evidence remains unresolved.",
"requirement_ids": ["general-first evaluation", "bounded additional evaluation", "no unnecessary fleet-wide tests"],
"allowed_change_scope": {"modules": ["capability"], "paths": ["modules/capability/domain/task_evaluation.go", "modules/capability/domain/task_evaluation_test.go", "dev/ACTIVE_TASK.json", "dev/tasks/DEV-000218-automatic-expansion-policy.json"]},
"forbidden_change_scope": {"paths": ["architecture/**", "security/**", "database/**", "deploy/**", "cmd/**", "contracts/**", "LLM-local/**"]},
"allowed_architecture_delta": {"new_cross_module_dependencies": 0, "new_canonical_owners": 0, "new_privileges": 0, "new_third_party_dependencies": 0, "new_database_owner_roles": 0},
"contracts": {"consume": ["TaskEvaluationProfile", "TaskEvaluation", "TaskEvaluationExpansionRequest"], "may_change": []},
"semantic_owner": "capability owns model qualification evidence and expansion policy",
"canonical_state": ["none; this packet creates no persistence or activation authority"],
"state_transitions": ["general COMPLETED/PARTIAL unresolved evidence -> bounded additional profile candidate", "qualified candidate -> no expansion"],
"side_effects_and_privileges": ["pure deterministic decision; no provider call, no model invocation, no mutation"],
"persistence_and_migrations": ["none"],
"failure_retry_restart": ["missing trigger evidence or additional cases fails closed; repeated decision is deterministic"],
"assumptions": ["a later task supplies task-specific additional cases from an unresolved outcome and schedules them through the existing Work Graph"],
"acceptance_scenarios": ["general graph not completed/partial does not expand", "unknown/unsupported/low-scoring candidates expand only with evidence and new cases", "one qualified candidate suppresses expansion", "no provider or hardware label changes the decision"],
"required_tests": ["go test", "go vet", "preflight", "check-diff", "check-architecture", "independent", "git diff --check", "CI", "production public health check"],
"terminal_gap_states": ["BLOCKED_BY_MISSING_CAPABILITY", "BLOCKED_BY_SPEC_GAP"]
}
@@ -2,6 +2,7 @@ package domain
import (
"errors"
"fmt"
"git.cin.su/m/Assistant/contracts/artifact"
"git.cin.su/m/Assistant/contracts/capability"
@@ -9,6 +10,35 @@ import (
var ErrInvalidTaskEvaluation = errors.New("invalid task evaluation")
const automaticExpansionMinimumOverallScore int64 = 700
// AutomaticTaskEvaluationExpansion decides whether a completed general
// evaluation has evidence that warrants a bounded additional profile. It is
// deliberately conservative: one supported candidate above the quality
// threshold suppresses expansion, while unknown/unsupported/low-scoring or
// critically failing candidates keep the unresolved path explicit.
func AutomaticTaskEvaluationExpansion(parent capability.TaskEvaluationProfile, parentGraphState string, evaluations []capability.TaskEvaluation, request capability.TaskEvaluationExpansionRequest, profileID capability.TaskEvaluationProfileID, revision int64) (capability.TaskEvaluationProfile, bool, error) {
if parentGraphState != "COMPLETED" && parentGraphState != "PARTIAL" {
return capability.TaskEvaluationProfile{}, false, nil
}
if len(evaluations) == 0 {
return capability.TaskEvaluationProfile{}, false, nil
}
for _, evaluation := range evaluations {
if evaluation.Result == capability.EvaluationSupported && evaluation.OverallScoreMillis >= automaticExpansionMinimumOverallScore && evaluation.CriticalFailureMillis == 0 {
return capability.TaskEvaluationProfile{}, false, nil
}
}
if request.TriggerEvidenceRef == "" || len(request.NewCaseIDs) == 0 {
return capability.TaskEvaluationProfile{}, false, fmt.Errorf("unresolved task evaluation requires trigger evidence and additional cases")
}
profile, err := BuildAdditionalTaskEvaluationProfile(parent, request, profileID, revision)
if err != nil {
return capability.TaskEvaluationProfile{}, false, err
}
return profile, true, nil
}
func BuildAdditionalTaskEvaluationProfile(parent capability.TaskEvaluationProfile, request capability.TaskEvaluationExpansionRequest, profileID capability.TaskEvaluationProfileID, revision int64) (capability.TaskEvaluationProfile, error) {
if err := parent.Validate(); err != nil || parent.Stage != capability.TaskEvaluationGeneral || request.ParentProfileID != parent.ID || request.CapabilityID != parent.CapabilityID || request.TriggerEvidenceRef == "" || len(request.NewCaseIDs) == 0 || profileID == "" || revision < 1 || request.BenchmarkVersion == "" {
return capability.TaskEvaluationProfile{}, ErrInvalidTaskEvaluation
@@ -33,3 +33,21 @@ func TestBuildAdditionalProfileRequiresUnresolvedEvidenceAndPreservesHistory(t *
t.Fatalf("profile=%+v err=%v", profile, err)
}
}
func TestAutomaticExpansionWaitsForGeneralGraphAndUnresolvedCandidates(t *testing.T) {
parent := capability.TaskEvaluationProfile{ID: "general-v1", Revision: 1, CapabilityID: "creator-advisory", Stage: capability.TaskEvaluationGeneral, BenchmarkVersion: "general-v1", RequiredCaseIDs: []string{"general"}, QualityWeightMillis: 700, ReliabilityWeightMillis: 200, LatencyWeightMillis: 100}
request := capability.TaskEvaluationExpansionRequest{ParentProfileID: parent.ID, CapabilityID: parent.CapabilityID, TriggerEvidenceRef: "outcome-unsolved", NewCaseIDs: []string{"edge-case"}, BenchmarkVersion: "additional-v1"}
value, expanded, err := AutomaticTaskEvaluationExpansion(parent, "COMPLETED", []capability.TaskEvaluation{{ID: "evaluation-1", ProfileID: parent.ID, BindingID: "binding-1", BindingRevision: 1, CapabilityID: parent.CapabilityID, Result: capability.EvaluationUnknown, OverallScoreMillis: 0, CriticalFailureMillis: 1000, BenchmarkVersion: parent.BenchmarkVersion, ArtifactID: "artifact-1", ObservedAtUnixMillis: 1}}, request, "additional-v1", 1)
if err != nil || !expanded || value.Stage != capability.TaskEvaluationAdditional || value.ParentProfileID != parent.ID {
t.Fatalf("expected bounded expansion, profile=%+v expanded=%t err=%v", value, expanded, err)
}
}
func TestAutomaticExpansionDoesNotRunWhenAQualifiedCandidateExists(t *testing.T) {
parent := capability.TaskEvaluationProfile{ID: "general-v1", Revision: 1, CapabilityID: "creator-advisory", Stage: capability.TaskEvaluationGeneral, BenchmarkVersion: "general-v1", RequiredCaseIDs: []string{"general"}, QualityWeightMillis: 700, ReliabilityWeightMillis: 200, LatencyWeightMillis: 100}
request := capability.TaskEvaluationExpansionRequest{ParentProfileID: parent.ID, CapabilityID: parent.CapabilityID, TriggerEvidenceRef: "outcome-unsolved", NewCaseIDs: []string{"edge-case"}, BenchmarkVersion: "additional-v1"}
value, expanded, err := AutomaticTaskEvaluationExpansion(parent, "PARTIAL", []capability.TaskEvaluation{{ID: "evaluation-1", ProfileID: parent.ID, BindingID: "binding-1", BindingRevision: 1, CapabilityID: parent.CapabilityID, Result: capability.EvaluationSupported, OverallScoreMillis: 850, CriticalFailureMillis: 0, BenchmarkVersion: parent.BenchmarkVersion, ArtifactID: "artifact-1", ObservedAtUnixMillis: 1}}, request, "additional-v1", 1)
if err != nil || expanded || value.ID != "" {
t.Fatalf("qualified candidate must suppress expansion: profile=%+v expanded=%t err=%v", value, expanded, err)
}
}