diff --git a/dev/ACTIVE_TASK.json b/dev/ACTIVE_TASK.json index f653bed..03b46d3 100644 --- a/dev/ACTIVE_TASK.json +++ b/dev/ACTIVE_TASK.json @@ -1,5 +1,5 @@ { - "task_file": "dev/tasks/DEV-000255-refresh-live-audit-status-projection.json", - "task_id": "DEV-000255", + "task_file": "dev/tasks/DEV-000256-model-rating-reachability-evidence.json", + "task_id": "DEV-000256", "task_revision": 1 } diff --git a/dev/tasks/DEV-000256-model-rating-reachability-evidence.json b/dev/tasks/DEV-000256-model-rating-reachability-evidence.json new file mode 100644 index 0000000..4980097 --- /dev/null +++ b/dev/tasks/DEV-000256-model-rating-reachability-evidence.json @@ -0,0 +1,23 @@ +{ + "task_id": "DEV-000256", + "task_revision": 1, + "architecture_revision": "ADR-0018", + "base_repo_revision": "3d97b348f74741b9aca2aa8d77f2e66b9f5760c2", + "risk_tier": "E", + "intent": "Expose observed qualification latency and evidence strength in provider-neutral model ratings without inferring quality, activation or authority.", + "requirement_ids": ["model fleet/failover", "provider-neutral rating", "quality must remain unrated without task evaluation"], + "allowed_change_scope": {"modules": ["capability"], "paths": ["modules/capability/domain/cognitive_registry.go", "modules/capability/domain/model_rating_test.go", "dev/ACTIVE_TASK.json", "dev/tasks/DEV-000256-model-rating-reachability-evidence.json"]}, + "forbidden_change_scope": {"paths": ["architecture/**", "security/**", "database/**", "deploy/**", "contracts/**", "LLM-local/**", "cmd/**", "adapters/**"]}, + "allowed_architecture_delta": {"new_cross_module_dependencies": 0, "new_canonical_owners": 0, "new_privileges": 0, "new_third_party_dependencies": 0, "new_database_owner_roles": 0}, + "contracts": {"consume": ["existing ModelQualificationObservation", "existing provider-neutral ModelRating"], "may_change": ["derived rating values for reachability-only evidence"]}, + "semantic_owner": "Capability rating derivation", + "canonical_state": ["none; ModelRating is derived from existing evidence"], + "state_transitions": ["none"], + "side_effects_and_privileges": ["pure deterministic derivation only"], + "persistence_and_migrations": ["none"], + "failure_retry_restart": ["missing or unavailable qualification remains UNRATED with no quality claim"], + "assumptions": ["reachability and latency are operational evidence, not task quality"], + "acceptance_scenarios": ["reachable qualification exposes latency/evidence scores", "quality category remains UNRATED without supported task evaluation", "provider kind and hardware do not affect rating", "unavailable qualification does not create positive rating"], + "required_tests": ["go test ./modules/capability/domain", "go vet ./modules/capability/domain", "independent evidence review", "go run ./cmd/devctl preflight -active dev/ACTIVE_TASK.json", "go run ./cmd/devctl check-diff -active dev/ACTIVE_TASK.json", "go run ./cmd/devctl check-architecture", "git diff --check", "CI", "production health smoke"], + "terminal_gap_states": ["BLOCKED_BY_MISSING_CAPABILITY", "BLOCKED_BY_SPEC_GAP"] +} diff --git a/modules/capability/domain/cognitive_registry.go b/modules/capability/domain/cognitive_registry.go index eacae20..b63aaf2 100644 --- a/modules/capability/domain/cognitive_registry.go +++ b/modules/capability/domain/cognitive_registry.go @@ -191,6 +191,9 @@ func RateCognitiveBinding(candidate capability.ResolvedCognitiveBinding) capabil return capability.ModelRating{TaskProfileID: evaluation.ProfileID, TaskScoreMillis: evaluation.OverallScoreMillis, Category: category, OverallScoreMillis: evaluation.OverallScoreMillis, QualityScoreMillis: evaluation.QualityScoreMillis, ReliabilityScoreMillis: evaluation.ReliabilityScoreMillis, LatencyScoreMillis: evaluation.LatencyScoreMillis, EvidenceScoreMillis: 1000} } if candidate.Evaluation == nil || candidate.Evaluation.Result != capability.EvaluationSupported || candidate.Evaluation.QualityScoreMillis == nil || candidate.Evaluation.CriticalFailureMillis == nil { + if candidate.Qualification != nil && candidate.Qualification.Status == capability.ModelQualificationReachable { + return capability.ModelRating{Category: capability.ModelRatingUnrated, LatencyScoreMillis: latencyScore(candidate.Qualification.LatencyMillis * 1000), EvidenceScoreMillis: 750} + } return capability.ModelRating{Category: capability.ModelRatingUnrated} } quality := clampMillis(*candidate.Evaluation.QualityScoreMillis) diff --git a/modules/capability/domain/model_rating_test.go b/modules/capability/domain/model_rating_test.go index 1392317..ad0d165 100644 --- a/modules/capability/domain/model_rating_test.go +++ b/modules/capability/domain/model_rating_test.go @@ -13,6 +13,15 @@ func TestRateCognitiveBindingRequiresSupportedEvidence(t *testing.T) { } } +func TestRateCognitiveBindingExposesReachabilityWithoutQualityClaim(t *testing.T) { + rating := RateCognitiveBinding(capability.ResolvedCognitiveBinding{Qualification: &capability.ModelQualificationObservation{ + Status: capability.ModelQualificationReachable, LatencyMillis: 600, + }}) + if rating.Category != capability.ModelRatingUnrated || rating.OverallScoreMillis != 0 || rating.LatencyScoreMillis != 800 || rating.EvidenceScoreMillis != 750 { + t.Fatalf("reachability rating made an invalid quality claim: %+v", rating) + } +} + func TestRateCognitiveBindingUsesEvidenceNotProviderName(t *testing.T) { quality, critical, latency := int64(900), int64(0), int64(400_000) base := capability.ResolvedCognitiveBinding{