From c81e2269e157c1a41b567d79d93bbbe49dd4b134 Mon Sep 17 00:00:00 2001 From: Mikhail Date: Sun, 23 Aug 2026 19:28:13 +0300 Subject: [PATCH] feat(capability): consume qualification evidence in selection --- adapters/postgres/capability_repository.go | 8 +++++++ contracts/capability/cognitive_registry.go | 1 + dev/ACTIVE_TASK.json | 4 ++-- ...0243-qualification-selection-evidence.json | 23 +++++++++++++++++++ .../capability/domain/cognitive_registry.go | 14 +++++++++++ .../capability/domain/model_rating_test.go | 19 +++++++++++++++ 6 files changed, 67 insertions(+), 2 deletions(-) create mode 100644 dev/tasks/DEV-000243-qualification-selection-evidence.json diff --git a/adapters/postgres/capability_repository.go b/adapters/postgres/capability_repository.go index 0a3e80e..aa0b5c5 100644 --- a/adapters/postgres/capability_repository.go +++ b/adapters/postgres/capability_repository.go @@ -505,6 +505,14 @@ func (r *CapabilityRepository) loadCognitiveEvidence(ctx context.Context, value if err == nil { value.Performance = &performance } + var qualification capability.ModelQualificationObservation + err = r.pool.QueryRow(ctx, `SELECT observation_id,provider_instance_id,provider_model_id,status,finish_reason,latency_millis,output_observed,quality_assessed,category,observed_at_unix_millis FROM capability.cognitive_model_qualification_observations WHERE provider_instance_id=$1 AND provider_model_id=$2 ORDER BY observed_at_unix_millis DESC LIMIT 1`, value.Provider.ID, value.Model.ProviderModelID).Scan(&qualification.ID, &qualification.ProviderInstanceID, &qualification.ProviderModelID, &qualification.Status, &qualification.FinishReason, &qualification.LatencyMillis, &qualification.OutputObserved, &qualification.QualityAssessed, &qualification.Category, &qualification.ObservedAtUnixMillis) + if err != nil && !errors.Is(err, pgx.ErrNoRows) { + return err + } + if err == nil { + value.Qualification = &qualification + } if profileID != "" { var taskEvaluation capability.TaskEvaluation err = r.pool.QueryRow(ctx, `SELECT evaluation_id,task_profile_id,binding_id,binding_revision,capability_id,result,quality_score_millis,reliability_score_millis,latency_score_millis,overall_score_millis,critical_failure_millis,benchmark_version,artifact_id,observed_at_unix_millis FROM capability.cognitive_task_evaluations WHERE task_profile_id=$1 AND binding_id=$2 AND binding_revision=$3 ORDER BY observed_at_unix_millis DESC LIMIT 1`, profileID, value.Binding.ID, value.Binding.Revision).Scan(&taskEvaluation.ID, &taskEvaluation.ProfileID, &taskEvaluation.BindingID, &taskEvaluation.BindingRevision, &taskEvaluation.CapabilityID, &taskEvaluation.Result, &taskEvaluation.QualityScoreMillis, &taskEvaluation.ReliabilityScoreMillis, &taskEvaluation.LatencyScoreMillis, &taskEvaluation.OverallScoreMillis, &taskEvaluation.CriticalFailureMillis, &taskEvaluation.BenchmarkVersion, &taskEvaluation.ArtifactID, &taskEvaluation.ObservedAtUnixMillis) diff --git a/contracts/capability/cognitive_registry.go b/contracts/capability/cognitive_registry.go index aed96eb..7ad4500 100644 --- a/contracts/capability/cognitive_registry.go +++ b/contracts/capability/cognitive_registry.go @@ -172,6 +172,7 @@ type ResolvedCognitiveBinding struct { Evaluation *EvaluatedCapability Performance *PerformanceEvidence TaskEvaluation *TaskEvaluation + Qualification *ModelQualificationObservation Rating *ModelRating } diff --git a/dev/ACTIVE_TASK.json b/dev/ACTIVE_TASK.json index ab44644..4c5b143 100644 --- a/dev/ACTIVE_TASK.json +++ b/dev/ACTIVE_TASK.json @@ -1,5 +1,5 @@ { - "task_file": "dev/tasks/DEV-000242-durable-qualification-observations.json", - "task_id": "DEV-000242", + "task_file": "dev/tasks/DEV-000243-qualification-selection-evidence.json", + "task_id": "DEV-000243", "task_revision": 1 } diff --git a/dev/tasks/DEV-000243-qualification-selection-evidence.json b/dev/tasks/DEV-000243-qualification-selection-evidence.json new file mode 100644 index 0000000..d72e57f --- /dev/null +++ b/dev/tasks/DEV-000243-qualification-selection-evidence.json @@ -0,0 +1,23 @@ +{ + "task_id": "DEV-000243", + "task_revision": 1, + "architecture_revision": "ADR-0018", + "base_repo_revision": "47327cc6d853da9084c96e3bf48502266706ca57", + "risk_tier": "E", + "intent": "Make Capability selection consume the latest durable qualification observation for fresh unavailability exclusion and transparent latency rating, without granting quality or activation.", + "requirement_ids": ["qualification selection loop", "hardware-neutral routing", "quality separated from reachability"], + "allowed_change_scope": {"modules": ["capability", "adapters/postgres"], "paths": ["adapters/postgres/capability_repository.go", "contracts/capability/cognitive_registry.go", "modules/capability/domain/cognitive_registry.go", "modules/capability/domain/model_rating_test.go", "dev/ACTIVE_TASK.json", "dev/tasks/DEV-000243-qualification-selection-evidence.json"]}, + "forbidden_change_scope": {"paths": ["architecture/**", "security/**", "database/**", "deploy/**", "LLM-local/**", "modules/work/secrets/**", "contracts/work/**"]}, + "allowed_architecture_delta": {"new_cross_module_dependencies": 0, "new_canonical_owners": 0, "new_privileges": 0, "new_third_party_dependencies": 0, "new_database_owner_roles": 0}, + "contracts": {"consume": ["durable ModelQualificationObservation"], "may_change": ["resolved binding evidence projection"]}, + "semantic_owner": "Capability selection and rating", + "canonical_state": ["qualification observation remains evidence; no activation mutation"], + "state_transitions": ["fresh UNAVAILABLE observation -> candidate excluded", "stale UNAVAILABLE observation -> discovery state remains authoritative", "REACHABLE observation -> latency evidence only"], + "side_effects_and_privileges": ["read-only projection and deterministic selection"], + "persistence_and_migrations": ["uses migration 045 from DEV-000242"], + "failure_retry_restart": ["missing observation preserves existing behavior"], + "assumptions": ["provider/model observed timestamps define freshness boundary"], + "acceptance_scenarios": ["fresh unavailable candidate is not selected", "stale unavailable candidate is not permanently poisoned", "quality remains task-evaluation controlled"], + "required_tests": ["go test ./modules/capability/... ./adapters/postgres/...", "go vet ./modules/capability/... ./adapters/postgres/...", "go run ./cmd/devctl preflight -active dev/ACTIVE_TASK.json", "go run ./cmd/devctl check-diff -active dev/ACTIVE_TASK.json", "go run ./cmd/devctl check-architecture", "git diff --check", "independent", "CI", "production health smoke"], + "terminal_gap_states": ["BLOCKED_BY_MISSING_CAPABILITY", "BLOCKED_BY_SPEC_GAP"] +} diff --git a/modules/capability/domain/cognitive_registry.go b/modules/capability/domain/cognitive_registry.go index 7647bc3..eacae20 100644 --- a/modules/capability/domain/cognitive_registry.go +++ b/modules/capability/domain/cognitive_registry.go @@ -148,6 +148,9 @@ func SelectCognitiveBinding(requirement capability.CognitiveRequirement, candida if candidate.Evaluation != nil && candidate.Evaluation.Result == capability.EvaluationUnsupported { continue } + if candidate.Qualification != nil && candidate.Qualification.Status == capability.ModelQualificationUnavailable && candidate.Qualification.ObservedAtUnixMillis >= maxObservedAt(candidate.Provider.ObservedAtUnixMillis, candidate.Model.ObservedAtUnixMillis) { + continue + } if requirement.TaskProfileID != "" && (candidate.TaskEvaluation == nil || candidate.TaskEvaluation.ProfileID != requirement.TaskProfileID || candidate.TaskEvaluation.Result != capability.EvaluationSupported) { continue } @@ -195,10 +198,14 @@ func RateCognitiveBinding(candidate capability.ResolvedCognitiveBinding) capabil latency := int64(500) if candidate.Performance != nil && candidate.Performance.LatencyMicros != nil { latency = latencyScore(*candidate.Performance.LatencyMicros) + } else if candidate.Qualification != nil && candidate.Qualification.Status == capability.ModelQualificationReachable { + latency = latencyScore(candidate.Qualification.LatencyMillis * 1000) } evidence := int64(700) if candidate.Performance != nil { evidence = 1000 + } else if candidate.Qualification != nil && candidate.Qualification.Status == capability.ModelQualificationReachable { + evidence = 750 } overall := (quality*55 + reliability*25 + latency*15 + evidence*5) / 100 category := capability.ModelRatingBalanced @@ -210,6 +217,13 @@ func RateCognitiveBinding(candidate capability.ResolvedCognitiveBinding) capabil return capability.ModelRating{Category: category, OverallScoreMillis: overall, QualityScoreMillis: quality, ReliabilityScoreMillis: reliability, LatencyScoreMillis: latency, EvidenceScoreMillis: evidence} } +func maxObservedAt(left, right int64) int64 { + if left > right { + return left + } + return right +} + func clampMillis(value int64) int64 { if value < 0 { return 0 diff --git a/modules/capability/domain/model_rating_test.go b/modules/capability/domain/model_rating_test.go index 820c72b..1392317 100644 --- a/modules/capability/domain/model_rating_test.go +++ b/modules/capability/domain/model_rating_test.go @@ -38,3 +38,22 @@ func TestRateCognitiveBindingClassifiesSlowEvidenceAsBalanced(t *testing.T) { t.Fatalf("rating=%+v", rating) } } + +func TestSelectionExcludesFreshUnavailableQualificationButNotStaleEvidence(t *testing.T) { + quality, critical := int64(900), int64(0) + base := capability.ResolvedCognitiveBinding{ + Binding: capability.CognitiveBinding{ID: "binding-qualification", Revision: 1, CapabilityID: "creator-advisory", OwnerPrincipalID: "owner", ProviderInstanceID: "provider", ProviderRevision: 1, ModelDescriptorID: "model", ModelRevision: 1, DataPolicy: capability.DataPolicyLocalOnly, Qualification: capability.QualificationValidated, Activation: capability.ActivationActive, Health: capability.HealthAvailable, LimitPolicyRef: "limits"}, + Provider: capability.ProviderInstance{ID: "provider", Revision: 1, Kind: "local", EndpointRef: "endpoint", Enabled: true, Qualification: capability.QualificationValidated, Health: capability.HealthAvailable, ObservedAtUnixMillis: 20}, + Model: capability.ModelDescriptor{ID: "model", Revision: 1, ProviderInstanceID: "provider", ProviderModelID: "model", MaxContextBytes: 1000, Modalities: []capability.ModelModality{capability.ModalityText}, Capabilities: []capability.ModelCapability{capability.CapabilityAdvisory}, Qualification: capability.QualificationValidated, Health: capability.HealthAvailable, ObservedAtUnixMillis: 20}, + Evaluation: &capability.EvaluatedCapability{ID: "evaluation", BindingID: "binding-qualification", BindingRevision: 1, CapabilityID: "creator-advisory", Result: capability.EvaluationSupported, QualityScoreMillis: &quality, CriticalFailureMillis: &critical}, + Qualification: &capability.ModelQualificationObservation{ID: "observation", ProviderInstanceID: "provider", ProviderModelID: "model", Status: capability.ModelQualificationUnavailable, FinishReason: "TIMEOUT", ObservedAtUnixMillis: 20, Category: capability.ModelRatingUnrated}, + } + requirement := capability.CognitiveRequirement{CapabilityID: "creator-advisory", OwnerPrincipalID: "owner", DataPolicy: capability.DataPolicyLocalOnly, RequiredModalities: []capability.ModelModality{capability.ModalityText}, RequiredCapabilities: []capability.ModelCapability{capability.CapabilityAdvisory}, MaxOutputBytes: 10} + if _, err := SelectCognitiveBinding(requirement, []capability.ResolvedCognitiveBinding{base}); err == nil { + t.Fatal("fresh unavailable qualification should exclude candidate") + } + base.Qualification.ObservedAtUnixMillis = 19 + if _, err := SelectCognitiveBinding(requirement, []capability.ResolvedCognitiveBinding{base}); err != nil { + t.Fatalf("stale unavailable qualification should not exclude candidate: %v", err) + } +} -- 2.54.0