feat(cognitive): add bounded model qualification probe
This commit is contained in:
@@ -169,6 +169,7 @@ func main() {
|
||||
mux.HandleFunc("POST /v1/task-evaluations/execute", gateway.taskEvaluationHandler)
|
||||
discovery := discoveryService{resolver: resolver, providers: discoveries, owner: identity.PrincipalID(envOr("COGNITIVE_OWNER_PRINCIPAL_ID", "domovoy-creator-principal")), now: time.Now}
|
||||
mux.HandleFunc("POST /v1/provider-discovery/{provider}", discovery.handler)
|
||||
mux.Handle("POST /v1/provider-discovery/{provider}/qualify", modelQualificationHandler(discoveries, string(envOr("COGNITIVE_OWNER_PRINCIPAL_ID", "domovoy-creator-principal"))))
|
||||
mux.HandleFunc("GET /v1/provider-discovery/status", discovery.statusHandler)
|
||||
refreshProviderDiscoveries(resolver, discovery)
|
||||
log.Printf("cognitive-worker listening on %s", addr)
|
||||
|
||||
@@ -0,0 +1,117 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"slices"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"git.cin.su/m/Assistant/adapters/models"
|
||||
"git.cin.su/m/Assistant/contracts/authority"
|
||||
"git.cin.su/m/Assistant/contracts/capability"
|
||||
"git.cin.su/m/Assistant/contracts/cognitive"
|
||||
"git.cin.su/m/Assistant/contracts/identity"
|
||||
)
|
||||
|
||||
type modelQualificationRequest struct {
|
||||
Prompt string `json:"prompt"`
|
||||
MaxOutputBytes int `json:"max_output_bytes"`
|
||||
ModelIDs []capability.ProviderModelID `json:"model_ids,omitempty"`
|
||||
}
|
||||
|
||||
type modelQualificationResult struct {
|
||||
ProviderModelID capability.ProviderModelID `json:"provider_model_id"`
|
||||
Status string `json:"status"`
|
||||
FinishReason cognitive.FinishReason `json:"finish_reason,omitempty"`
|
||||
LatencyMillis int64 `json:"latency_millis"`
|
||||
OutputObserved bool `json:"output_observed"`
|
||||
QualityAssessed bool `json:"quality_assessed"`
|
||||
Category capability.ModelRatingCategory `json:"category"`
|
||||
Error string `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
type modelQualificationResponse struct {
|
||||
Provider capability.ProviderInstanceID `json:"provider"`
|
||||
ObservedAtUnixMillis int64 `json:"observed_at_unix_millis"`
|
||||
Results []modelQualificationResult `json:"results"`
|
||||
}
|
||||
|
||||
func modelQualificationHandler(providers map[string]providerRegistration, owner string) http.Handler {
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
name := r.PathValue("provider")
|
||||
registration, ok := providers[name]
|
||||
if !ok || registration.Discovery == nil || registration.Provider == nil {
|
||||
http.Error(w, "provider qualification not found", http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
var request modelQualificationRequest
|
||||
if !decodeJSON(w, r, 16<<10, &request) || request.Prompt == "" {
|
||||
return
|
||||
}
|
||||
if request.MaxOutputBytes < 1 {
|
||||
request.MaxOutputBytes = 512
|
||||
}
|
||||
if request.MaxOutputBytes > 4096 {
|
||||
http.Error(w, "max_output_bytes exceeds qualification bound", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
modelsFound, probe := registration.Discovery.Discover(r.Context())
|
||||
if probe.Health != capability.HealthAvailable || len(modelsFound) == 0 {
|
||||
http.Error(w, "provider unavailable for qualification", http.StatusServiceUnavailable)
|
||||
return
|
||||
}
|
||||
if len(request.ModelIDs) > 0 {
|
||||
modelsFound = slices.DeleteFunc(modelsFound, func(model models.DiscoveredModel) bool {
|
||||
return !slices.Contains(request.ModelIDs, model.ProviderModelID)
|
||||
})
|
||||
}
|
||||
if len(modelsFound) == 0 {
|
||||
http.Error(w, "requested model not discovered", http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
if len(modelsFound) > 16 {
|
||||
modelsFound = modelsFound[:16]
|
||||
}
|
||||
results := qualifyModels(r.Context(), registration, owner, request.Prompt, request.MaxOutputBytes, modelsFound)
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_ = jsonEncode(w, modelQualificationResponse{Provider: registration.ProviderInstanceID, ObservedAtUnixMillis: time.Now().UnixMilli(), Results: results})
|
||||
})
|
||||
}
|
||||
|
||||
func qualifyModels(ctx context.Context, registration providerRegistration, owner, prompt string, maxOutputBytes int, discovered []models.DiscoveredModel) []modelQualificationResult {
|
||||
results := make([]modelQualificationResult, len(discovered))
|
||||
var wg sync.WaitGroup
|
||||
for index, model := range discovered {
|
||||
wg.Add(1)
|
||||
go func(index int, model models.DiscoveredModel) {
|
||||
defer wg.Done()
|
||||
started := time.Now()
|
||||
request := cognitive.AdvisoryRequest{Version: cognitive.CreatorAdvisoryV1, RequestID: cognitive.RequestID(fmt.Sprintf("qualification:%s:%d", model.ProviderModelID, started.UnixNano())), ActorID: identity.ActorID("qualification-runtime"), PrincipalID: identity.PrincipalID(owner), ScopeID: authority.ScopeID("qualification-scope"), Message: prompt, MaxOutputBytes: maxOutputBytes, ProviderBindingID: "qualification-binding-v1", ProviderBindingRevision: 1, ProviderInstanceID: registration.ProviderInstanceID, ModelDescriptorID: capability.ModelDescriptorID("qualification-" + string(model.ProviderModelID)), ProviderModelID: model.ProviderModelID}
|
||||
response, err := registration.Provider.Advise(ctx, request)
|
||||
result := modelQualificationResult{ProviderModelID: model.ProviderModelID, LatencyMillis: time.Since(started).Milliseconds(), QualityAssessed: false, Category: capability.ModelRatingUnrated}
|
||||
if err != nil {
|
||||
result.Status = "UNAVAILABLE"
|
||||
result.Error = err.Error()
|
||||
results[index] = result
|
||||
return
|
||||
}
|
||||
result.FinishReason = response.FinishReason
|
||||
result.OutputObserved = response.AdvisoryText != ""
|
||||
if response.FinishReason == cognitive.FinishCompleted && result.OutputObserved {
|
||||
result.Status = "REACHABLE"
|
||||
} else {
|
||||
result.Status = "RESPONDED_WITHOUT_QUALIFICATION"
|
||||
}
|
||||
results[index] = result
|
||||
}(index, model)
|
||||
}
|
||||
wg.Wait()
|
||||
return results
|
||||
}
|
||||
|
||||
func jsonEncode(w http.ResponseWriter, value any) error {
|
||||
return json.NewEncoder(w).Encode(value)
|
||||
}
|
||||
@@ -0,0 +1,56 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.cin.su/m/Assistant/adapters/models"
|
||||
"git.cin.su/m/Assistant/contracts/capability"
|
||||
"git.cin.su/m/Assistant/contracts/cognitive"
|
||||
)
|
||||
|
||||
type qualificationProvider struct{}
|
||||
|
||||
func (qualificationProvider) Probe(context.Context) models.ProviderProbe {
|
||||
return models.ProviderProbe{Health: capability.HealthAvailable}
|
||||
}
|
||||
func (qualificationProvider) Discover(context.Context) ([]models.DiscoveredModel, models.ProviderProbe) {
|
||||
return []models.DiscoveredModel{{ProviderModelID: "model-a"}, {ProviderModelID: "model-b"}}, models.ProviderProbe{Health: capability.HealthAvailable}
|
||||
}
|
||||
func (qualificationProvider) Advise(_ context.Context, request cognitive.AdvisoryRequest) (cognitive.AdvisoryResponse, error) {
|
||||
return cognitive.AdvisoryResponse{Version: request.Version, RequestID: request.RequestID, AdvisoryText: "reachable", FinishReason: cognitive.FinishCompleted}, nil
|
||||
}
|
||||
|
||||
func TestModelQualificationUsesDiscoveredModelsAndLeavesQualityUnrated(t *testing.T) {
|
||||
provider := qualificationProvider{}
|
||||
mux := http.NewServeMux()
|
||||
mux.Handle("POST /v1/provider-discovery/{provider}/qualify", modelQualificationHandler(map[string]providerRegistration{"gpu": {ProviderInstanceID: "gpu", Provider: provider, Discovery: provider}}, "owner"))
|
||||
server := httptest.NewServer(mux)
|
||||
defer server.Close()
|
||||
request := httptest.NewRequest(http.MethodPost, server.URL+"/v1/provider-discovery/gpu/qualify", strings.NewReader(`{"prompt":"ready","max_output_bytes":128}`))
|
||||
request.Header.Set("Content-Type", "application/json")
|
||||
response := httptest.NewRecorder()
|
||||
mux.ServeHTTP(response, request)
|
||||
if response.Code != http.StatusOK {
|
||||
t.Fatalf("qualification status=%d body=%s", response.Code, response.Body.String())
|
||||
}
|
||||
if !strings.Contains(response.Body.String(), `"status":"REACHABLE"`) || !strings.Contains(response.Body.String(), `"category":"UNRATED"`) || strings.Contains(response.Body.String(), `"quality_assessed":true`) {
|
||||
t.Fatalf("qualification overclaimed quality: %s", response.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestModelQualificationRejectsUnboundedOutput(t *testing.T) {
|
||||
provider := qualificationProvider{}
|
||||
mux := http.NewServeMux()
|
||||
mux.Handle("POST /v1/provider-discovery/{provider}/qualify", modelQualificationHandler(map[string]providerRegistration{"cpu": {ProviderInstanceID: "cpu", Provider: provider, Discovery: provider}}, "owner"))
|
||||
request := httptest.NewRequest(http.MethodPost, "/v1/provider-discovery/cpu/qualify", strings.NewReader(`{"prompt":"ready","max_output_bytes":8192}`))
|
||||
request.Header.Set("Content-Type", "application/json")
|
||||
response := httptest.NewRecorder()
|
||||
mux.ServeHTTP(response, request)
|
||||
if response.Code != http.StatusBadRequest {
|
||||
t.Fatalf("unbounded qualification status=%d", response.Code)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user