From be29ed5776334c65f36c9a8fc6028c8f02408bc0 Mon Sep 17 00:00:00 2001 From: Mikhail Date: Sun, 23 Aug 2026 13:56:16 +0300 Subject: [PATCH] feat(capability): gate additional model evaluations --- dev/ACTIVE_TASK.json | 4 +-- ...DEV-000218-automatic-expansion-policy.json | 23 ++++++++++++++ modules/capability/domain/task_evaluation.go | 30 +++++++++++++++++++ .../capability/domain/task_evaluation_test.go | 18 +++++++++++ 4 files changed, 73 insertions(+), 2 deletions(-) create mode 100644 dev/tasks/DEV-000218-automatic-expansion-policy.json diff --git a/dev/ACTIVE_TASK.json b/dev/ACTIVE_TASK.json index 2dfbf4f..c54e604 100644 --- a/dev/ACTIVE_TASK.json +++ b/dev/ACTIVE_TASK.json @@ -1,5 +1,5 @@ { - "task_file": "dev/tasks/DEV-000217-current-status-evidence.json", - "task_id": "DEV-000217", + "task_file": "dev/tasks/DEV-000218-automatic-expansion-policy.json", + "task_id": "DEV-000218", "task_revision": 1 } diff --git a/dev/tasks/DEV-000218-automatic-expansion-policy.json b/dev/tasks/DEV-000218-automatic-expansion-policy.json new file mode 100644 index 0000000..6610910 --- /dev/null +++ b/dev/tasks/DEV-000218-automatic-expansion-policy.json @@ -0,0 +1,23 @@ +{ + "task_id": "DEV-000218", + "task_revision": 1, + "architecture_revision": "ADR-0018", + "base_repo_revision": "ab840878fe7d21a0a6aeb69193a3d58ea23a56e0", + "risk_tier": "E", + "intent": "Add a conservative deterministic policy gate for creating additional model-evaluation profiles only after general evaluation evidence remains unresolved.", + "requirement_ids": ["general-first evaluation", "bounded additional evaluation", "no unnecessary fleet-wide tests"], + "allowed_change_scope": {"modules": ["capability"], "paths": ["modules/capability/domain/task_evaluation.go", "modules/capability/domain/task_evaluation_test.go", "dev/ACTIVE_TASK.json", "dev/tasks/DEV-000218-automatic-expansion-policy.json"]}, + "forbidden_change_scope": {"paths": ["architecture/**", "security/**", "database/**", "deploy/**", "cmd/**", "contracts/**", "LLM-local/**"]}, + "allowed_architecture_delta": {"new_cross_module_dependencies": 0, "new_canonical_owners": 0, "new_privileges": 0, "new_third_party_dependencies": 0, "new_database_owner_roles": 0}, + "contracts": {"consume": ["TaskEvaluationProfile", "TaskEvaluation", "TaskEvaluationExpansionRequest"], "may_change": []}, + "semantic_owner": "capability owns model qualification evidence and expansion policy", + "canonical_state": ["none; this packet creates no persistence or activation authority"], + "state_transitions": ["general COMPLETED/PARTIAL unresolved evidence -> bounded additional profile candidate", "qualified candidate -> no expansion"], + "side_effects_and_privileges": ["pure deterministic decision; no provider call, no model invocation, no mutation"], + "persistence_and_migrations": ["none"], + "failure_retry_restart": ["missing trigger evidence or additional cases fails closed; repeated decision is deterministic"], + "assumptions": ["a later task supplies task-specific additional cases from an unresolved outcome and schedules them through the existing Work Graph"], + "acceptance_scenarios": ["general graph not completed/partial does not expand", "unknown/unsupported/low-scoring candidates expand only with evidence and new cases", "one qualified candidate suppresses expansion", "no provider or hardware label changes the decision"], + "required_tests": ["go test", "go vet", "preflight", "check-diff", "check-architecture", "independent", "git diff --check", "CI", "production public health check"], + "terminal_gap_states": ["BLOCKED_BY_MISSING_CAPABILITY", "BLOCKED_BY_SPEC_GAP"] +} diff --git a/modules/capability/domain/task_evaluation.go b/modules/capability/domain/task_evaluation.go index 218d2bb..4e8c4cd 100644 --- a/modules/capability/domain/task_evaluation.go +++ b/modules/capability/domain/task_evaluation.go @@ -2,6 +2,7 @@ package domain import ( "errors" + "fmt" "git.cin.su/m/Assistant/contracts/artifact" "git.cin.su/m/Assistant/contracts/capability" @@ -9,6 +10,35 @@ import ( var ErrInvalidTaskEvaluation = errors.New("invalid task evaluation") +const automaticExpansionMinimumOverallScore int64 = 700 + +// AutomaticTaskEvaluationExpansion decides whether a completed general +// evaluation has evidence that warrants a bounded additional profile. It is +// deliberately conservative: one supported candidate above the quality +// threshold suppresses expansion, while unknown/unsupported/low-scoring or +// critically failing candidates keep the unresolved path explicit. +func AutomaticTaskEvaluationExpansion(parent capability.TaskEvaluationProfile, parentGraphState string, evaluations []capability.TaskEvaluation, request capability.TaskEvaluationExpansionRequest, profileID capability.TaskEvaluationProfileID, revision int64) (capability.TaskEvaluationProfile, bool, error) { + if parentGraphState != "COMPLETED" && parentGraphState != "PARTIAL" { + return capability.TaskEvaluationProfile{}, false, nil + } + if len(evaluations) == 0 { + return capability.TaskEvaluationProfile{}, false, nil + } + for _, evaluation := range evaluations { + if evaluation.Result == capability.EvaluationSupported && evaluation.OverallScoreMillis >= automaticExpansionMinimumOverallScore && evaluation.CriticalFailureMillis == 0 { + return capability.TaskEvaluationProfile{}, false, nil + } + } + if request.TriggerEvidenceRef == "" || len(request.NewCaseIDs) == 0 { + return capability.TaskEvaluationProfile{}, false, fmt.Errorf("unresolved task evaluation requires trigger evidence and additional cases") + } + profile, err := BuildAdditionalTaskEvaluationProfile(parent, request, profileID, revision) + if err != nil { + return capability.TaskEvaluationProfile{}, false, err + } + return profile, true, nil +} + func BuildAdditionalTaskEvaluationProfile(parent capability.TaskEvaluationProfile, request capability.TaskEvaluationExpansionRequest, profileID capability.TaskEvaluationProfileID, revision int64) (capability.TaskEvaluationProfile, error) { if err := parent.Validate(); err != nil || parent.Stage != capability.TaskEvaluationGeneral || request.ParentProfileID != parent.ID || request.CapabilityID != parent.CapabilityID || request.TriggerEvidenceRef == "" || len(request.NewCaseIDs) == 0 || profileID == "" || revision < 1 || request.BenchmarkVersion == "" { return capability.TaskEvaluationProfile{}, ErrInvalidTaskEvaluation diff --git a/modules/capability/domain/task_evaluation_test.go b/modules/capability/domain/task_evaluation_test.go index f3ef3ab..e17622a 100644 --- a/modules/capability/domain/task_evaluation_test.go +++ b/modules/capability/domain/task_evaluation_test.go @@ -33,3 +33,21 @@ func TestBuildAdditionalProfileRequiresUnresolvedEvidenceAndPreservesHistory(t * t.Fatalf("profile=%+v err=%v", profile, err) } } + +func TestAutomaticExpansionWaitsForGeneralGraphAndUnresolvedCandidates(t *testing.T) { + parent := capability.TaskEvaluationProfile{ID: "general-v1", Revision: 1, CapabilityID: "creator-advisory", Stage: capability.TaskEvaluationGeneral, BenchmarkVersion: "general-v1", RequiredCaseIDs: []string{"general"}, QualityWeightMillis: 700, ReliabilityWeightMillis: 200, LatencyWeightMillis: 100} + request := capability.TaskEvaluationExpansionRequest{ParentProfileID: parent.ID, CapabilityID: parent.CapabilityID, TriggerEvidenceRef: "outcome-unsolved", NewCaseIDs: []string{"edge-case"}, BenchmarkVersion: "additional-v1"} + value, expanded, err := AutomaticTaskEvaluationExpansion(parent, "COMPLETED", []capability.TaskEvaluation{{ID: "evaluation-1", ProfileID: parent.ID, BindingID: "binding-1", BindingRevision: 1, CapabilityID: parent.CapabilityID, Result: capability.EvaluationUnknown, OverallScoreMillis: 0, CriticalFailureMillis: 1000, BenchmarkVersion: parent.BenchmarkVersion, ArtifactID: "artifact-1", ObservedAtUnixMillis: 1}}, request, "additional-v1", 1) + if err != nil || !expanded || value.Stage != capability.TaskEvaluationAdditional || value.ParentProfileID != parent.ID { + t.Fatalf("expected bounded expansion, profile=%+v expanded=%t err=%v", value, expanded, err) + } +} + +func TestAutomaticExpansionDoesNotRunWhenAQualifiedCandidateExists(t *testing.T) { + parent := capability.TaskEvaluationProfile{ID: "general-v1", Revision: 1, CapabilityID: "creator-advisory", Stage: capability.TaskEvaluationGeneral, BenchmarkVersion: "general-v1", RequiredCaseIDs: []string{"general"}, QualityWeightMillis: 700, ReliabilityWeightMillis: 200, LatencyWeightMillis: 100} + request := capability.TaskEvaluationExpansionRequest{ParentProfileID: parent.ID, CapabilityID: parent.CapabilityID, TriggerEvidenceRef: "outcome-unsolved", NewCaseIDs: []string{"edge-case"}, BenchmarkVersion: "additional-v1"} + value, expanded, err := AutomaticTaskEvaluationExpansion(parent, "PARTIAL", []capability.TaskEvaluation{{ID: "evaluation-1", ProfileID: parent.ID, BindingID: "binding-1", BindingRevision: 1, CapabilityID: parent.CapabilityID, Result: capability.EvaluationSupported, OverallScoreMillis: 850, CriticalFailureMillis: 0, BenchmarkVersion: parent.BenchmarkVersion, ArtifactID: "artifact-1", ObservedAtUnixMillis: 1}}, request, "additional-v1", 1) + if err != nil || expanded || value.ID != "" { + t.Fatalf("qualified candidate must suppress expansion: profile=%+v expanded=%t err=%v", value, expanded, err) + } +} -- 2.54.0