feat(capability): expose reachability rating evidence #234
@@ -1,5 +1,5 @@
|
||||
{
|
||||
"task_file": "dev/tasks/DEV-000255-refresh-live-audit-status-projection.json",
|
||||
"task_id": "DEV-000255",
|
||||
"task_file": "dev/tasks/DEV-000256-model-rating-reachability-evidence.json",
|
||||
"task_id": "DEV-000256",
|
||||
"task_revision": 1
|
||||
}
|
||||
|
||||
@@ -0,0 +1,23 @@
|
||||
{
|
||||
"task_id": "DEV-000256",
|
||||
"task_revision": 1,
|
||||
"architecture_revision": "ADR-0018",
|
||||
"base_repo_revision": "3d97b348f74741b9aca2aa8d77f2e66b9f5760c2",
|
||||
"risk_tier": "E",
|
||||
"intent": "Expose observed qualification latency and evidence strength in provider-neutral model ratings without inferring quality, activation or authority.",
|
||||
"requirement_ids": ["model fleet/failover", "provider-neutral rating", "quality must remain unrated without task evaluation"],
|
||||
"allowed_change_scope": {"modules": ["capability"], "paths": ["modules/capability/domain/cognitive_registry.go", "modules/capability/domain/model_rating_test.go", "dev/ACTIVE_TASK.json", "dev/tasks/DEV-000256-model-rating-reachability-evidence.json"]},
|
||||
"forbidden_change_scope": {"paths": ["architecture/**", "security/**", "database/**", "deploy/**", "contracts/**", "LLM-local/**", "cmd/**", "adapters/**"]},
|
||||
"allowed_architecture_delta": {"new_cross_module_dependencies": 0, "new_canonical_owners": 0, "new_privileges": 0, "new_third_party_dependencies": 0, "new_database_owner_roles": 0},
|
||||
"contracts": {"consume": ["existing ModelQualificationObservation", "existing provider-neutral ModelRating"], "may_change": ["derived rating values for reachability-only evidence"]},
|
||||
"semantic_owner": "Capability rating derivation",
|
||||
"canonical_state": ["none; ModelRating is derived from existing evidence"],
|
||||
"state_transitions": ["none"],
|
||||
"side_effects_and_privileges": ["pure deterministic derivation only"],
|
||||
"persistence_and_migrations": ["none"],
|
||||
"failure_retry_restart": ["missing or unavailable qualification remains UNRATED with no quality claim"],
|
||||
"assumptions": ["reachability and latency are operational evidence, not task quality"],
|
||||
"acceptance_scenarios": ["reachable qualification exposes latency/evidence scores", "quality category remains UNRATED without supported task evaluation", "provider kind and hardware do not affect rating", "unavailable qualification does not create positive rating"],
|
||||
"required_tests": ["go test ./modules/capability/domain", "go vet ./modules/capability/domain", "independent evidence review", "go run ./cmd/devctl preflight -active dev/ACTIVE_TASK.json", "go run ./cmd/devctl check-diff -active dev/ACTIVE_TASK.json", "go run ./cmd/devctl check-architecture", "git diff --check", "CI", "production health smoke"],
|
||||
"terminal_gap_states": ["BLOCKED_BY_MISSING_CAPABILITY", "BLOCKED_BY_SPEC_GAP"]
|
||||
}
|
||||
@@ -191,6 +191,9 @@ func RateCognitiveBinding(candidate capability.ResolvedCognitiveBinding) capabil
|
||||
return capability.ModelRating{TaskProfileID: evaluation.ProfileID, TaskScoreMillis: evaluation.OverallScoreMillis, Category: category, OverallScoreMillis: evaluation.OverallScoreMillis, QualityScoreMillis: evaluation.QualityScoreMillis, ReliabilityScoreMillis: evaluation.ReliabilityScoreMillis, LatencyScoreMillis: evaluation.LatencyScoreMillis, EvidenceScoreMillis: 1000}
|
||||
}
|
||||
if candidate.Evaluation == nil || candidate.Evaluation.Result != capability.EvaluationSupported || candidate.Evaluation.QualityScoreMillis == nil || candidate.Evaluation.CriticalFailureMillis == nil {
|
||||
if candidate.Qualification != nil && candidate.Qualification.Status == capability.ModelQualificationReachable {
|
||||
return capability.ModelRating{Category: capability.ModelRatingUnrated, LatencyScoreMillis: latencyScore(candidate.Qualification.LatencyMillis * 1000), EvidenceScoreMillis: 750}
|
||||
}
|
||||
return capability.ModelRating{Category: capability.ModelRatingUnrated}
|
||||
}
|
||||
quality := clampMillis(*candidate.Evaluation.QualityScoreMillis)
|
||||
|
||||
@@ -13,6 +13,15 @@ func TestRateCognitiveBindingRequiresSupportedEvidence(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRateCognitiveBindingExposesReachabilityWithoutQualityClaim(t *testing.T) {
|
||||
rating := RateCognitiveBinding(capability.ResolvedCognitiveBinding{Qualification: &capability.ModelQualificationObservation{
|
||||
Status: capability.ModelQualificationReachable, LatencyMillis: 600,
|
||||
}})
|
||||
if rating.Category != capability.ModelRatingUnrated || rating.OverallScoreMillis != 0 || rating.LatencyScoreMillis != 800 || rating.EvidenceScoreMillis != 750 {
|
||||
t.Fatalf("reachability rating made an invalid quality claim: %+v", rating)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRateCognitiveBindingUsesEvidenceNotProviderName(t *testing.T) {
|
||||
quality, critical, latency := int64(900), int64(0), int64(400_000)
|
||||
base := capability.ResolvedCognitiveBinding{
|
||||
|
||||
Reference in New Issue
Block a user