From d0ede463529e9ae7d0afae6caee102fe92875589 Mon Sep 17 00:00:00 2001 From: Mikhail Date: Sun, 23 Aug 2026 19:20:38 +0300 Subject: [PATCH] feat(capability): persist model qualification observations --- adapters/postgres/capability_repository.go | 5 ++ cmd/cognitive-worker/main.go | 2 +- cmd/cognitive-worker/model_qualification.go | 52 ++++++++++++++++++- .../model_qualification_test.go | 26 ++++++++++ cmd/core-runtime/main.go | 14 +++++ contracts/capability/model_qualification.go | 47 +++++++++++++++++ ...ility_model_qualification_observations.sql | 24 +++++++++ dev/ACTIVE_TASK.json | 4 +- ...42-durable-qualification-observations.json | 23 ++++++++ modules/capability/ports/repository.go | 1 + 10 files changed, 194 insertions(+), 4 deletions(-) create mode 100644 contracts/capability/model_qualification.go create mode 100644 database/migrations/045_capability_model_qualification_observations.sql create mode 100644 dev/tasks/DEV-000242-durable-qualification-observations.json diff --git a/adapters/postgres/capability_repository.go b/adapters/postgres/capability_repository.go index 8a7fb04..0a3e80e 100644 --- a/adapters/postgres/capability_repository.go +++ b/adapters/postgres/capability_repository.go @@ -633,3 +633,8 @@ func (r *CapabilityRepository) SaveTaskEvaluation(ctx context.Context, value cap _, err := r.pool.Exec(ctx, `INSERT INTO capability.cognitive_task_evaluations (evaluation_id,task_profile_id,binding_id,binding_revision,capability_id,result,quality_score_millis,reliability_score_millis,latency_score_millis,overall_score_millis,critical_failure_millis,benchmark_version,artifact_id,observed_at_unix_millis) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14) ON CONFLICT DO NOTHING`, value.ID, value.ProfileID, value.BindingID, value.BindingRevision, value.CapabilityID, value.Result, value.QualityScoreMillis, value.ReliabilityScoreMillis, value.LatencyScoreMillis, value.OverallScoreMillis, value.CriticalFailureMillis, value.BenchmarkVersion, value.ArtifactID, value.ObservedAtUnixMillis) return err } + +func (r *CapabilityRepository) SaveModelQualificationObservation(ctx context.Context, value capability.ModelQualificationObservation) error { + _, err := r.pool.Exec(ctx, `INSERT INTO capability.cognitive_model_qualification_observations (observation_id,provider_instance_id,provider_model_id,status,finish_reason,latency_millis,output_observed,quality_assessed,category,observed_at_unix_millis) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10) ON CONFLICT DO NOTHING`, value.ID, value.ProviderInstanceID, value.ProviderModelID, value.Status, value.FinishReason, value.LatencyMillis, value.OutputObserved, value.QualityAssessed, value.Category, value.ObservedAtUnixMillis) + return err +} diff --git a/cmd/cognitive-worker/main.go b/cmd/cognitive-worker/main.go index 82f2198..6ecc6e6 100644 --- a/cmd/cognitive-worker/main.go +++ b/cmd/cognitive-worker/main.go @@ -169,7 +169,7 @@ func main() { mux.HandleFunc("POST /v1/task-evaluations/execute", gateway.taskEvaluationHandler) discovery := discoveryService{resolver: resolver, providers: discoveries, owner: identity.PrincipalID(envOr("COGNITIVE_OWNER_PRINCIPAL_ID", "domovoy-creator-principal")), now: time.Now} mux.HandleFunc("POST /v1/provider-discovery/{provider}", discovery.handler) - mux.Handle("POST /v1/provider-discovery/{provider}/qualify", modelQualificationHandler(discoveries, string(envOr("COGNITIVE_OWNER_PRINCIPAL_ID", "domovoy-creator-principal")))) + mux.Handle("POST /v1/provider-discovery/{provider}/qualify", modelQualificationHandler(discoveries, string(envOr("COGNITIVE_OWNER_PRINCIPAL_ID", "domovoy-creator-principal")), httpQualificationObserver{resolver: resolver})) mux.HandleFunc("GET /v1/provider-discovery/status", discovery.statusHandler) refreshProviderDiscoveries(resolver, discovery) log.Printf("cognitive-worker listening on %s", addr) diff --git a/cmd/cognitive-worker/model_qualification.go b/cmd/cognitive-worker/model_qualification.go index 6d1580e..0f02f04 100644 --- a/cmd/cognitive-worker/model_qualification.go +++ b/cmd/cognitive-worker/model_qualification.go @@ -14,8 +14,22 @@ import ( "git.cin.su/m/Assistant/contracts/capability" "git.cin.su/m/Assistant/contracts/cognitive" "git.cin.su/m/Assistant/contracts/identity" + "git.cin.su/m/Assistant/platform/ids" ) +type qualificationObserver interface { + ObserveQualification(context.Context, capability.ModelQualificationObservation) error +} + +type httpQualificationObserver struct{ resolver *httpBindingResolver } + +func (o httpQualificationObserver) ObserveQualification(ctx context.Context, value capability.ModelQualificationObservation) error { + if o.resolver == nil { + return fmt.Errorf("qualification observer resolver is nil") + } + return o.resolver.post(ctx, "/v1/cognitive-registry/model-qualification-observations", value, nil) +} + type modelQualificationRequest struct { Prompt string `json:"prompt"` MaxOutputBytes int `json:"max_output_bytes"` @@ -39,7 +53,7 @@ type modelQualificationResponse struct { Results []modelQualificationResult `json:"results"` } -func modelQualificationHandler(providers map[string]providerRegistration, owner string) http.Handler { +func modelQualificationHandler(providers map[string]providerRegistration, owner string, observers ...qualificationObserver) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { name := r.PathValue("provider") registration, ok := providers[name] @@ -76,6 +90,12 @@ func modelQualificationHandler(providers map[string]providerRegistration, owner modelsFound = modelsFound[:16] } results := qualifyModels(r.Context(), registration, owner, request.Prompt, request.MaxOutputBytes, modelsFound) + if len(observers) > 0 && observers[0] != nil { + if err := persistQualificationResults(r.Context(), observers[0], registration, results); err != nil { + http.Error(w, "persist model qualification observation", http.StatusBadGateway) + return + } + } w.Header().Set("Content-Type", "application/json") _ = jsonEncode(w, modelQualificationResponse{Provider: registration.ProviderInstanceID, ObservedAtUnixMillis: time.Now().UnixMilli(), Results: results}) }) @@ -94,6 +114,7 @@ func qualifyModels(ctx context.Context, registration providerRegistration, owner result := modelQualificationResult{ProviderModelID: model.ProviderModelID, LatencyMillis: time.Since(started).Milliseconds(), QualityAssessed: false, Category: capability.ModelRatingUnrated} if err != nil { result.Status = "UNAVAILABLE" + result.FinishReason = cognitive.FinishProviderUnavailable result.Error = err.Error() results[index] = result return @@ -112,6 +133,35 @@ func qualifyModels(ctx context.Context, registration providerRegistration, owner return results } +func persistQualificationResults(ctx context.Context, observer qualificationObserver, registration providerRegistration, results []modelQualificationResult) error { + for _, result := range results { + observationID, err := ids.New() + if err != nil { + return err + } + status := capability.ModelQualificationRespondedWithoutRating + switch result.Status { + case "REACHABLE": + status = capability.ModelQualificationReachable + case "UNAVAILABLE": + status = capability.ModelQualificationUnavailable + } + observation := capability.ModelQualificationObservation{ + ID: capability.ModelQualificationObservationID(observationID), ProviderInstanceID: registration.ProviderInstanceID, ProviderModelID: result.ProviderModelID, + Status: status, FinishReason: string(result.FinishReason), LatencyMillis: result.LatencyMillis, + OutputObserved: result.OutputObserved, QualityAssessed: false, Category: capability.ModelRatingUnrated, + ObservedAtUnixMillis: time.Now().UnixMilli(), + } + if err := observation.Validate(); err != nil { + return err + } + if err := observer.ObserveQualification(ctx, observation); err != nil { + return err + } + } + return nil +} + func jsonEncode(w http.ResponseWriter, value any) error { return json.NewEncoder(w).Encode(value) } diff --git a/cmd/cognitive-worker/model_qualification_test.go b/cmd/cognitive-worker/model_qualification_test.go index 3043b88..b4e05e8 100644 --- a/cmd/cognitive-worker/model_qualification_test.go +++ b/cmd/cognitive-worker/model_qualification_test.go @@ -57,3 +57,29 @@ func TestModelQualificationRejectsUnboundedOutput(t *testing.T) { t.Fatalf("unbounded qualification status=%d", response.Code) } } + +type qualificationObserverStub struct { + observations []capability.ModelQualificationObservation +} + +func (s *qualificationObserverStub) ObserveQualification(_ context.Context, value capability.ModelQualificationObservation) error { + s.observations = append(s.observations, value) + return nil +} + +func TestModelQualificationPersistsReachabilityWithoutQuality(t *testing.T) { + provider := qualificationProvider{} + observer := &qualificationObserverStub{} + mux := http.NewServeMux() + mux.Handle("POST /v1/provider-discovery/{provider}/qualify", modelQualificationHandler(map[string]providerRegistration{"cpu": {ProviderInstanceID: "cpu", Provider: provider, Discovery: provider}}, "owner", observer)) + request := httptest.NewRequest(http.MethodPost, "/v1/provider-discovery/cpu/qualify", strings.NewReader(`{"prompt":"ready","max_output_bytes":128,"model_ids":["model-a"]}`)) + request.Header.Set("Content-Type", "application/json") + response := httptest.NewRecorder() + mux.ServeHTTP(response, request) + if response.Code != http.StatusOK || len(observer.observations) != 1 { + t.Fatalf("qualification persistence status=%d observations=%d body=%s", response.Code, len(observer.observations), response.Body.String()) + } + if err := observer.observations[0].Validate(); err != nil || observer.observations[0].Status != capability.ModelQualificationReachable { + t.Fatalf("invalid qualification observation: %+v err=%v", observer.observations[0], err) + } +} diff --git a/cmd/core-runtime/main.go b/cmd/core-runtime/main.go index 4517451..a293e3b 100644 --- a/cmd/core-runtime/main.go +++ b/cmd/core-runtime/main.go @@ -1058,6 +1058,20 @@ func main() { } w.WriteHeader(http.StatusCreated) }) + mux.HandleFunc("POST /v1/cognitive-registry/model-qualification-observations", func(w http.ResponseWriter, r *http.Request) { + var value capability.ModelQualificationObservation + decoder := json.NewDecoder(io.LimitReader(r.Body, 16<<10)) + decoder.DisallowUnknownFields() + if err := decoder.Decode(&value); err != nil || value.Validate() != nil { + http.Error(w, "invalid model qualification observation", http.StatusBadRequest) + return + } + if err := capabilityRepository.SaveModelQualificationObservation(r.Context(), value); err != nil { + http.Error(w, "save model qualification observation", http.StatusInternalServerError) + return + } + w.WriteHeader(http.StatusCreated) + }) mux.HandleFunc("POST /v1/creator/cognitive-canary", func(w http.ResponseWriter, r *http.Request) { session, ok := authenticateCreator(w, r) if !ok { diff --git a/contracts/capability/model_qualification.go b/contracts/capability/model_qualification.go new file mode 100644 index 0000000..274a60d --- /dev/null +++ b/contracts/capability/model_qualification.go @@ -0,0 +1,47 @@ +package capability + +import ( + "errors" + + "git.cin.su/m/Assistant/platform/ids" +) + +type ModelQualificationObservationID ids.ID +type ModelQualificationStatus string + +const ( + ModelQualificationReachable ModelQualificationStatus = "REACHABLE" + ModelQualificationRespondedWithoutRating ModelQualificationStatus = "RESPONDED_WITHOUT_QUALIFICATION" + ModelQualificationUnavailable ModelQualificationStatus = "UNAVAILABLE" +) + +// ModelQualificationObservation is operational evidence only. It records +// reachability and latency, never quality, activation, routing priority or +// provider credentials. +type ModelQualificationObservation struct { + ID ModelQualificationObservationID + ProviderInstanceID ProviderInstanceID + ProviderModelID ProviderModelID + Status ModelQualificationStatus + FinishReason string + LatencyMillis int64 + OutputObserved bool + QualityAssessed bool + Category ModelRatingCategory + ObservedAtUnixMillis int64 +} + +func (o ModelQualificationObservation) Validate() error { + if o.ID == "" || o.ProviderInstanceID == "" || o.ProviderModelID == "" || o.FinishReason == "" || o.LatencyMillis < 0 || o.ObservedAtUnixMillis < 1 { + return errors.New("model qualification observation identity and timing are incomplete") + } + switch o.Status { + case ModelQualificationReachable, ModelQualificationRespondedWithoutRating, ModelQualificationUnavailable: + default: + return errors.New("model qualification observation status is unsupported") + } + if o.Category != ModelRatingUnrated || o.QualityAssessed { + return errors.New("model qualification observation cannot assert quality") + } + return nil +} diff --git a/database/migrations/045_capability_model_qualification_observations.sql b/database/migrations/045_capability_model_qualification_observations.sql new file mode 100644 index 0000000..37d052f --- /dev/null +++ b/database/migrations/045_capability_model_qualification_observations.sql @@ -0,0 +1,24 @@ +BEGIN; + +SET ROLE capability_role; + +CREATE TABLE IF NOT EXISTS capability.cognitive_model_qualification_observations ( + observation_id TEXT PRIMARY KEY, + provider_instance_id TEXT NOT NULL, + provider_model_id TEXT NOT NULL, + status TEXT NOT NULL CHECK (status IN ('REACHABLE','RESPONDED_WITHOUT_QUALIFICATION','UNAVAILABLE')), + finish_reason TEXT NOT NULL, + latency_millis BIGINT NOT NULL CHECK (latency_millis >= 0), + output_observed BOOLEAN NOT NULL, + quality_assessed BOOLEAN NOT NULL DEFAULT FALSE CHECK (quality_assessed = FALSE), + category TEXT NOT NULL CHECK (category = 'UNRATED'), + observed_at_unix_millis BIGINT NOT NULL CHECK (observed_at_unix_millis > 0) +); + +CREATE INDEX IF NOT EXISTS cognitive_model_qualification_lookup + ON capability.cognitive_model_qualification_observations(provider_instance_id, provider_model_id, observed_at_unix_millis DESC); + +GRANT SELECT, INSERT ON capability.cognitive_model_qualification_observations TO capability_role; + +RESET ROLE; +COMMIT; diff --git a/dev/ACTIVE_TASK.json b/dev/ACTIVE_TASK.json index 28c591c..ab44644 100644 --- a/dev/ACTIVE_TASK.json +++ b/dev/ACTIVE_TASK.json @@ -1,5 +1,5 @@ { - "task_file": "dev/tasks/DEV-000241-fleet-qualification-evidence.json", - "task_id": "DEV-000241", + "task_file": "dev/tasks/DEV-000242-durable-qualification-observations.json", + "task_id": "DEV-000242", "task_revision": 1 } diff --git a/dev/tasks/DEV-000242-durable-qualification-observations.json b/dev/tasks/DEV-000242-durable-qualification-observations.json new file mode 100644 index 0000000..6af6e9b --- /dev/null +++ b/dev/tasks/DEV-000242-durable-qualification-observations.json @@ -0,0 +1,23 @@ +{ + "task_id": "DEV-000242", + "task_revision": 1, + "architecture_revision": "ADR-0018", + "base_repo_revision": "e4ba1cb5fabf930396b2aa9f95d3b4b205e6319d", + "risk_tier": "E", + "intent": "Persist provider-neutral model qualification observations through the Capability owner while keeping quality unrated until independent task evaluation.", + "requirement_ids": ["durable qualification evidence", "model fleet/failover", "quality separated from reachability"], + "allowed_change_scope": {"modules": ["capability", "core-runtime", "cognitive-worker", "database"], "paths": ["adapters/postgres/capability_repository.go", "cmd/cognitive-worker/main.go", "cmd/cognitive-worker/model_qualification.go", "cmd/cognitive-worker/model_qualification_test.go", "cmd/core-runtime/main.go", "contracts/capability/model_qualification.go", "database/migrations/045_capability_model_qualification_observations.sql", "modules/capability/ports/repository.go", "dev/ACTIVE_TASK.json", "dev/tasks/DEV-000242-durable-qualification-observations.json"]}, + "forbidden_change_scope": {"paths": ["architecture/**", "security/**", "deploy/**", "LLM-local/**", "modules/work/secrets/**", "contracts/work/**"]}, + "allowed_architecture_delta": {"new_cross_module_dependencies": 0, "new_canonical_owners": 0, "new_privileges": 0, "new_third_party_dependencies": 0, "new_database_owner_roles": 1}, + "contracts": {"consume": ["bounded qualification result", "Capability repository boundary"], "may_change": ["typed model qualification observation contract", "internal core persistence route"]}, + "semantic_owner": "Capability owns qualification evidence; cognitive-worker produces observations", + "canonical_state": ["cognitive_model_qualification_observations"], + "state_transitions": ["probe -> REACHABLE/RESPONDED_WITHOUT_QUALIFICATION/UNAVAILABLE; category remains UNRATED"], + "side_effects_and_privileges": ["bounded provider call and capability-owned INSERT; no promotion, activation or secret persistence"], + "persistence_and_migrations": ["migration 045 creates capability-owned observation table and index"], + "failure_retry_restart": ["observation persistence failure returns bounded gateway error; duplicate IDs are idempotent"], + "assumptions": ["independent task evaluation remains the only quality/promotion evidence"], + "acceptance_scenarios": ["qualification route persists one observation per probed model", "invalid quality/category is rejected", "prompt/output/error text is not persisted", "existing qualification tests remain valid"], + "required_tests": ["go test ./cmd/cognitive-worker/... ./cmd/core-runtime/... ./modules/capability/... ./adapters/postgres/...", "go vet ./cmd/cognitive-worker/... ./cmd/core-runtime/... ./modules/capability/... ./adapters/postgres/...", "go run ./cmd/devctl preflight -active dev/ACTIVE_TASK.json", "go run ./cmd/devctl check-diff -active dev/ACTIVE_TASK.json", "go run ./cmd/devctl check-architecture", "git diff --check", "independent", "CI", "production health smoke"], + "terminal_gap_states": ["BLOCKED_BY_MISSING_CAPABILITY", "BLOCKED_BY_SPEC_GAP", "REQUIRES_MIGRATION_DECISION"] +} diff --git a/modules/capability/ports/repository.go b/modules/capability/ports/repository.go index 4fb5860..1c1b64c 100644 --- a/modules/capability/ports/repository.go +++ b/modules/capability/ports/repository.go @@ -27,4 +27,5 @@ type Repository interface { SaveEvaluatedCapability(context.Context, capability.EvaluatedCapability) error SavePerformanceEvidence(context.Context, capability.PerformanceEvidence) error SaveTaskEvaluation(context.Context, capability.TaskEvaluation) error + SaveModelQualificationObservation(context.Context, capability.ModelQualificationObservation) error } -- 2.54.0