feat(cognitive): add bounded model qualification probe
CI / test (push) Successful in 58s
CI / test (pull_request) Successful in 45s

This commit is contained in:
m
2026-08-23 18:27:22 +03:00
parent 60e8292c1d
commit d4972afd11
6 changed files with 201 additions and 4 deletions
+1
View File
@@ -169,6 +169,7 @@ func main() {
mux.HandleFunc("POST /v1/task-evaluations/execute", gateway.taskEvaluationHandler)
discovery := discoveryService{resolver: resolver, providers: discoveries, owner: identity.PrincipalID(envOr("COGNITIVE_OWNER_PRINCIPAL_ID", "domovoy-creator-principal")), now: time.Now}
mux.HandleFunc("POST /v1/provider-discovery/{provider}", discovery.handler)
mux.Handle("POST /v1/provider-discovery/{provider}/qualify", modelQualificationHandler(discoveries, string(envOr("COGNITIVE_OWNER_PRINCIPAL_ID", "domovoy-creator-principal"))))
mux.HandleFunc("GET /v1/provider-discovery/status", discovery.statusHandler)
refreshProviderDiscoveries(resolver, discovery)
log.Printf("cognitive-worker listening on %s", addr)
+117
View File
@@ -0,0 +1,117 @@
package main
import (
"context"
"encoding/json"
"fmt"
"net/http"
"slices"
"sync"
"time"
"git.cin.su/m/Assistant/adapters/models"
"git.cin.su/m/Assistant/contracts/authority"
"git.cin.su/m/Assistant/contracts/capability"
"git.cin.su/m/Assistant/contracts/cognitive"
"git.cin.su/m/Assistant/contracts/identity"
)
type modelQualificationRequest struct {
Prompt string `json:"prompt"`
MaxOutputBytes int `json:"max_output_bytes"`
ModelIDs []capability.ProviderModelID `json:"model_ids,omitempty"`
}
type modelQualificationResult struct {
ProviderModelID capability.ProviderModelID `json:"provider_model_id"`
Status string `json:"status"`
FinishReason cognitive.FinishReason `json:"finish_reason,omitempty"`
LatencyMillis int64 `json:"latency_millis"`
OutputObserved bool `json:"output_observed"`
QualityAssessed bool `json:"quality_assessed"`
Category capability.ModelRatingCategory `json:"category"`
Error string `json:"error,omitempty"`
}
type modelQualificationResponse struct {
Provider capability.ProviderInstanceID `json:"provider"`
ObservedAtUnixMillis int64 `json:"observed_at_unix_millis"`
Results []modelQualificationResult `json:"results"`
}
func modelQualificationHandler(providers map[string]providerRegistration, owner string) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
name := r.PathValue("provider")
registration, ok := providers[name]
if !ok || registration.Discovery == nil || registration.Provider == nil {
http.Error(w, "provider qualification not found", http.StatusNotFound)
return
}
var request modelQualificationRequest
if !decodeJSON(w, r, 16<<10, &request) || request.Prompt == "" {
return
}
if request.MaxOutputBytes < 1 {
request.MaxOutputBytes = 512
}
if request.MaxOutputBytes > 4096 {
http.Error(w, "max_output_bytes exceeds qualification bound", http.StatusBadRequest)
return
}
modelsFound, probe := registration.Discovery.Discover(r.Context())
if probe.Health != capability.HealthAvailable || len(modelsFound) == 0 {
http.Error(w, "provider unavailable for qualification", http.StatusServiceUnavailable)
return
}
if len(request.ModelIDs) > 0 {
modelsFound = slices.DeleteFunc(modelsFound, func(model models.DiscoveredModel) bool {
return !slices.Contains(request.ModelIDs, model.ProviderModelID)
})
}
if len(modelsFound) == 0 {
http.Error(w, "requested model not discovered", http.StatusNotFound)
return
}
if len(modelsFound) > 16 {
modelsFound = modelsFound[:16]
}
results := qualifyModels(r.Context(), registration, owner, request.Prompt, request.MaxOutputBytes, modelsFound)
w.Header().Set("Content-Type", "application/json")
_ = jsonEncode(w, modelQualificationResponse{Provider: registration.ProviderInstanceID, ObservedAtUnixMillis: time.Now().UnixMilli(), Results: results})
})
}
func qualifyModels(ctx context.Context, registration providerRegistration, owner, prompt string, maxOutputBytes int, discovered []models.DiscoveredModel) []modelQualificationResult {
results := make([]modelQualificationResult, len(discovered))
var wg sync.WaitGroup
for index, model := range discovered {
wg.Add(1)
go func(index int, model models.DiscoveredModel) {
defer wg.Done()
started := time.Now()
request := cognitive.AdvisoryRequest{Version: cognitive.CreatorAdvisoryV1, RequestID: cognitive.RequestID(fmt.Sprintf("qualification:%s:%d", model.ProviderModelID, started.UnixNano())), ActorID: identity.ActorID("qualification-runtime"), PrincipalID: identity.PrincipalID(owner), ScopeID: authority.ScopeID("qualification-scope"), Message: prompt, MaxOutputBytes: maxOutputBytes, ProviderBindingID: "qualification-binding-v1", ProviderBindingRevision: 1, ProviderInstanceID: registration.ProviderInstanceID, ModelDescriptorID: capability.ModelDescriptorID("qualification-" + string(model.ProviderModelID)), ProviderModelID: model.ProviderModelID}
response, err := registration.Provider.Advise(ctx, request)
result := modelQualificationResult{ProviderModelID: model.ProviderModelID, LatencyMillis: time.Since(started).Milliseconds(), QualityAssessed: false, Category: capability.ModelRatingUnrated}
if err != nil {
result.Status = "UNAVAILABLE"
result.Error = err.Error()
results[index] = result
return
}
result.FinishReason = response.FinishReason
result.OutputObserved = response.AdvisoryText != ""
if response.FinishReason == cognitive.FinishCompleted && result.OutputObserved {
result.Status = "REACHABLE"
} else {
result.Status = "RESPONDED_WITHOUT_QUALIFICATION"
}
results[index] = result
}(index, model)
}
wg.Wait()
return results
}
func jsonEncode(w http.ResponseWriter, value any) error {
return json.NewEncoder(w).Encode(value)
}
@@ -0,0 +1,56 @@
package main
import (
"context"
"net/http"
"net/http/httptest"
"strings"
"testing"
"git.cin.su/m/Assistant/adapters/models"
"git.cin.su/m/Assistant/contracts/capability"
"git.cin.su/m/Assistant/contracts/cognitive"
)
type qualificationProvider struct{}
func (qualificationProvider) Probe(context.Context) models.ProviderProbe {
return models.ProviderProbe{Health: capability.HealthAvailable}
}
func (qualificationProvider) Discover(context.Context) ([]models.DiscoveredModel, models.ProviderProbe) {
return []models.DiscoveredModel{{ProviderModelID: "model-a"}, {ProviderModelID: "model-b"}}, models.ProviderProbe{Health: capability.HealthAvailable}
}
func (qualificationProvider) Advise(_ context.Context, request cognitive.AdvisoryRequest) (cognitive.AdvisoryResponse, error) {
return cognitive.AdvisoryResponse{Version: request.Version, RequestID: request.RequestID, AdvisoryText: "reachable", FinishReason: cognitive.FinishCompleted}, nil
}
func TestModelQualificationUsesDiscoveredModelsAndLeavesQualityUnrated(t *testing.T) {
provider := qualificationProvider{}
mux := http.NewServeMux()
mux.Handle("POST /v1/provider-discovery/{provider}/qualify", modelQualificationHandler(map[string]providerRegistration{"gpu": {ProviderInstanceID: "gpu", Provider: provider, Discovery: provider}}, "owner"))
server := httptest.NewServer(mux)
defer server.Close()
request := httptest.NewRequest(http.MethodPost, server.URL+"/v1/provider-discovery/gpu/qualify", strings.NewReader(`{"prompt":"ready","max_output_bytes":128}`))
request.Header.Set("Content-Type", "application/json")
response := httptest.NewRecorder()
mux.ServeHTTP(response, request)
if response.Code != http.StatusOK {
t.Fatalf("qualification status=%d body=%s", response.Code, response.Body.String())
}
if !strings.Contains(response.Body.String(), `"status":"REACHABLE"`) || !strings.Contains(response.Body.String(), `"category":"UNRATED"`) || strings.Contains(response.Body.String(), `"quality_assessed":true`) {
t.Fatalf("qualification overclaimed quality: %s", response.Body.String())
}
}
func TestModelQualificationRejectsUnboundedOutput(t *testing.T) {
provider := qualificationProvider{}
mux := http.NewServeMux()
mux.Handle("POST /v1/provider-discovery/{provider}/qualify", modelQualificationHandler(map[string]providerRegistration{"cpu": {ProviderInstanceID: "cpu", Provider: provider, Discovery: provider}}, "owner"))
request := httptest.NewRequest(http.MethodPost, "/v1/provider-discovery/cpu/qualify", strings.NewReader(`{"prompt":"ready","max_output_bytes":8192}`))
request.Header.Set("Content-Type", "application/json")
response := httptest.NewRecorder()
mux.ServeHTTP(response, request)
if response.Code != http.StatusBadRequest {
t.Fatalf("unbounded qualification status=%d", response.Code)
}
}