feat(capability): expose reachability rating evidence #234

Merged
m merged 1 commits from codex/dev-000248-self-development-runtime-status into main 2026-08-23 18:57:41 +00:00
4 changed files with 37 additions and 2 deletions
+2 -2
View File
@@ -1,5 +1,5 @@
{
"task_file": "dev/tasks/DEV-000255-refresh-live-audit-status-projection.json",
"task_id": "DEV-000255",
"task_file": "dev/tasks/DEV-000256-model-rating-reachability-evidence.json",
"task_id": "DEV-000256",
"task_revision": 1
}
@@ -0,0 +1,23 @@
{
"task_id": "DEV-000256",
"task_revision": 1,
"architecture_revision": "ADR-0018",
"base_repo_revision": "3d97b348f74741b9aca2aa8d77f2e66b9f5760c2",
"risk_tier": "E",
"intent": "Expose observed qualification latency and evidence strength in provider-neutral model ratings without inferring quality, activation or authority.",
"requirement_ids": ["model fleet/failover", "provider-neutral rating", "quality must remain unrated without task evaluation"],
"allowed_change_scope": {"modules": ["capability"], "paths": ["modules/capability/domain/cognitive_registry.go", "modules/capability/domain/model_rating_test.go", "dev/ACTIVE_TASK.json", "dev/tasks/DEV-000256-model-rating-reachability-evidence.json"]},
"forbidden_change_scope": {"paths": ["architecture/**", "security/**", "database/**", "deploy/**", "contracts/**", "LLM-local/**", "cmd/**", "adapters/**"]},
"allowed_architecture_delta": {"new_cross_module_dependencies": 0, "new_canonical_owners": 0, "new_privileges": 0, "new_third_party_dependencies": 0, "new_database_owner_roles": 0},
"contracts": {"consume": ["existing ModelQualificationObservation", "existing provider-neutral ModelRating"], "may_change": ["derived rating values for reachability-only evidence"]},
"semantic_owner": "Capability rating derivation",
"canonical_state": ["none; ModelRating is derived from existing evidence"],
"state_transitions": ["none"],
"side_effects_and_privileges": ["pure deterministic derivation only"],
"persistence_and_migrations": ["none"],
"failure_retry_restart": ["missing or unavailable qualification remains UNRATED with no quality claim"],
"assumptions": ["reachability and latency are operational evidence, not task quality"],
"acceptance_scenarios": ["reachable qualification exposes latency/evidence scores", "quality category remains UNRATED without supported task evaluation", "provider kind and hardware do not affect rating", "unavailable qualification does not create positive rating"],
"required_tests": ["go test ./modules/capability/domain", "go vet ./modules/capability/domain", "independent evidence review", "go run ./cmd/devctl preflight -active dev/ACTIVE_TASK.json", "go run ./cmd/devctl check-diff -active dev/ACTIVE_TASK.json", "go run ./cmd/devctl check-architecture", "git diff --check", "CI", "production health smoke"],
"terminal_gap_states": ["BLOCKED_BY_MISSING_CAPABILITY", "BLOCKED_BY_SPEC_GAP"]
}
@@ -191,6 +191,9 @@ func RateCognitiveBinding(candidate capability.ResolvedCognitiveBinding) capabil
return capability.ModelRating{TaskProfileID: evaluation.ProfileID, TaskScoreMillis: evaluation.OverallScoreMillis, Category: category, OverallScoreMillis: evaluation.OverallScoreMillis, QualityScoreMillis: evaluation.QualityScoreMillis, ReliabilityScoreMillis: evaluation.ReliabilityScoreMillis, LatencyScoreMillis: evaluation.LatencyScoreMillis, EvidenceScoreMillis: 1000}
}
if candidate.Evaluation == nil || candidate.Evaluation.Result != capability.EvaluationSupported || candidate.Evaluation.QualityScoreMillis == nil || candidate.Evaluation.CriticalFailureMillis == nil {
if candidate.Qualification != nil && candidate.Qualification.Status == capability.ModelQualificationReachable {
return capability.ModelRating{Category: capability.ModelRatingUnrated, LatencyScoreMillis: latencyScore(candidate.Qualification.LatencyMillis * 1000), EvidenceScoreMillis: 750}
}
return capability.ModelRating{Category: capability.ModelRatingUnrated}
}
quality := clampMillis(*candidate.Evaluation.QualityScoreMillis)
@@ -13,6 +13,15 @@ func TestRateCognitiveBindingRequiresSupportedEvidence(t *testing.T) {
}
}
func TestRateCognitiveBindingExposesReachabilityWithoutQualityClaim(t *testing.T) {
rating := RateCognitiveBinding(capability.ResolvedCognitiveBinding{Qualification: &capability.ModelQualificationObservation{
Status: capability.ModelQualificationReachable, LatencyMillis: 600,
}})
if rating.Category != capability.ModelRatingUnrated || rating.OverallScoreMillis != 0 || rating.LatencyScoreMillis != 800 || rating.EvidenceScoreMillis != 750 {
t.Fatalf("reachability rating made an invalid quality claim: %+v", rating)
}
}
func TestRateCognitiveBindingUsesEvidenceNotProviderName(t *testing.T) {
quality, critical, latency := int64(900), int64(0), int64(400_000)
base := capability.ResolvedCognitiveBinding{