feat(capability): persist model qualification observations
This commit is contained in:
@@ -169,7 +169,7 @@ func main() {
|
||||
mux.HandleFunc("POST /v1/task-evaluations/execute", gateway.taskEvaluationHandler)
|
||||
discovery := discoveryService{resolver: resolver, providers: discoveries, owner: identity.PrincipalID(envOr("COGNITIVE_OWNER_PRINCIPAL_ID", "domovoy-creator-principal")), now: time.Now}
|
||||
mux.HandleFunc("POST /v1/provider-discovery/{provider}", discovery.handler)
|
||||
mux.Handle("POST /v1/provider-discovery/{provider}/qualify", modelQualificationHandler(discoveries, string(envOr("COGNITIVE_OWNER_PRINCIPAL_ID", "domovoy-creator-principal"))))
|
||||
mux.Handle("POST /v1/provider-discovery/{provider}/qualify", modelQualificationHandler(discoveries, string(envOr("COGNITIVE_OWNER_PRINCIPAL_ID", "domovoy-creator-principal")), httpQualificationObserver{resolver: resolver}))
|
||||
mux.HandleFunc("GET /v1/provider-discovery/status", discovery.statusHandler)
|
||||
refreshProviderDiscoveries(resolver, discovery)
|
||||
log.Printf("cognitive-worker listening on %s", addr)
|
||||
|
||||
@@ -14,8 +14,22 @@ import (
|
||||
"git.cin.su/m/Assistant/contracts/capability"
|
||||
"git.cin.su/m/Assistant/contracts/cognitive"
|
||||
"git.cin.su/m/Assistant/contracts/identity"
|
||||
"git.cin.su/m/Assistant/platform/ids"
|
||||
)
|
||||
|
||||
type qualificationObserver interface {
|
||||
ObserveQualification(context.Context, capability.ModelQualificationObservation) error
|
||||
}
|
||||
|
||||
type httpQualificationObserver struct{ resolver *httpBindingResolver }
|
||||
|
||||
func (o httpQualificationObserver) ObserveQualification(ctx context.Context, value capability.ModelQualificationObservation) error {
|
||||
if o.resolver == nil {
|
||||
return fmt.Errorf("qualification observer resolver is nil")
|
||||
}
|
||||
return o.resolver.post(ctx, "/v1/cognitive-registry/model-qualification-observations", value, nil)
|
||||
}
|
||||
|
||||
type modelQualificationRequest struct {
|
||||
Prompt string `json:"prompt"`
|
||||
MaxOutputBytes int `json:"max_output_bytes"`
|
||||
@@ -39,7 +53,7 @@ type modelQualificationResponse struct {
|
||||
Results []modelQualificationResult `json:"results"`
|
||||
}
|
||||
|
||||
func modelQualificationHandler(providers map[string]providerRegistration, owner string) http.Handler {
|
||||
func modelQualificationHandler(providers map[string]providerRegistration, owner string, observers ...qualificationObserver) http.Handler {
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
name := r.PathValue("provider")
|
||||
registration, ok := providers[name]
|
||||
@@ -76,6 +90,12 @@ func modelQualificationHandler(providers map[string]providerRegistration, owner
|
||||
modelsFound = modelsFound[:16]
|
||||
}
|
||||
results := qualifyModels(r.Context(), registration, owner, request.Prompt, request.MaxOutputBytes, modelsFound)
|
||||
if len(observers) > 0 && observers[0] != nil {
|
||||
if err := persistQualificationResults(r.Context(), observers[0], registration, results); err != nil {
|
||||
http.Error(w, "persist model qualification observation", http.StatusBadGateway)
|
||||
return
|
||||
}
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_ = jsonEncode(w, modelQualificationResponse{Provider: registration.ProviderInstanceID, ObservedAtUnixMillis: time.Now().UnixMilli(), Results: results})
|
||||
})
|
||||
@@ -94,6 +114,7 @@ func qualifyModels(ctx context.Context, registration providerRegistration, owner
|
||||
result := modelQualificationResult{ProviderModelID: model.ProviderModelID, LatencyMillis: time.Since(started).Milliseconds(), QualityAssessed: false, Category: capability.ModelRatingUnrated}
|
||||
if err != nil {
|
||||
result.Status = "UNAVAILABLE"
|
||||
result.FinishReason = cognitive.FinishProviderUnavailable
|
||||
result.Error = err.Error()
|
||||
results[index] = result
|
||||
return
|
||||
@@ -112,6 +133,35 @@ func qualifyModels(ctx context.Context, registration providerRegistration, owner
|
||||
return results
|
||||
}
|
||||
|
||||
func persistQualificationResults(ctx context.Context, observer qualificationObserver, registration providerRegistration, results []modelQualificationResult) error {
|
||||
for _, result := range results {
|
||||
observationID, err := ids.New()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
status := capability.ModelQualificationRespondedWithoutRating
|
||||
switch result.Status {
|
||||
case "REACHABLE":
|
||||
status = capability.ModelQualificationReachable
|
||||
case "UNAVAILABLE":
|
||||
status = capability.ModelQualificationUnavailable
|
||||
}
|
||||
observation := capability.ModelQualificationObservation{
|
||||
ID: capability.ModelQualificationObservationID(observationID), ProviderInstanceID: registration.ProviderInstanceID, ProviderModelID: result.ProviderModelID,
|
||||
Status: status, FinishReason: string(result.FinishReason), LatencyMillis: result.LatencyMillis,
|
||||
OutputObserved: result.OutputObserved, QualityAssessed: false, Category: capability.ModelRatingUnrated,
|
||||
ObservedAtUnixMillis: time.Now().UnixMilli(),
|
||||
}
|
||||
if err := observation.Validate(); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := observer.ObserveQualification(ctx, observation); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func jsonEncode(w http.ResponseWriter, value any) error {
|
||||
return json.NewEncoder(w).Encode(value)
|
||||
}
|
||||
|
||||
@@ -57,3 +57,29 @@ func TestModelQualificationRejectsUnboundedOutput(t *testing.T) {
|
||||
t.Fatalf("unbounded qualification status=%d", response.Code)
|
||||
}
|
||||
}
|
||||
|
||||
type qualificationObserverStub struct {
|
||||
observations []capability.ModelQualificationObservation
|
||||
}
|
||||
|
||||
func (s *qualificationObserverStub) ObserveQualification(_ context.Context, value capability.ModelQualificationObservation) error {
|
||||
s.observations = append(s.observations, value)
|
||||
return nil
|
||||
}
|
||||
|
||||
func TestModelQualificationPersistsReachabilityWithoutQuality(t *testing.T) {
|
||||
provider := qualificationProvider{}
|
||||
observer := &qualificationObserverStub{}
|
||||
mux := http.NewServeMux()
|
||||
mux.Handle("POST /v1/provider-discovery/{provider}/qualify", modelQualificationHandler(map[string]providerRegistration{"cpu": {ProviderInstanceID: "cpu", Provider: provider, Discovery: provider}}, "owner", observer))
|
||||
request := httptest.NewRequest(http.MethodPost, "/v1/provider-discovery/cpu/qualify", strings.NewReader(`{"prompt":"ready","max_output_bytes":128,"model_ids":["model-a"]}`))
|
||||
request.Header.Set("Content-Type", "application/json")
|
||||
response := httptest.NewRecorder()
|
||||
mux.ServeHTTP(response, request)
|
||||
if response.Code != http.StatusOK || len(observer.observations) != 1 {
|
||||
t.Fatalf("qualification persistence status=%d observations=%d body=%s", response.Code, len(observer.observations), response.Body.String())
|
||||
}
|
||||
if err := observer.observations[0].Validate(); err != nil || observer.observations[0].Status != capability.ModelQualificationReachable {
|
||||
t.Fatalf("invalid qualification observation: %+v err=%v", observer.observations[0], err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1058,6 +1058,20 @@ func main() {
|
||||
}
|
||||
w.WriteHeader(http.StatusCreated)
|
||||
})
|
||||
mux.HandleFunc("POST /v1/cognitive-registry/model-qualification-observations", func(w http.ResponseWriter, r *http.Request) {
|
||||
var value capability.ModelQualificationObservation
|
||||
decoder := json.NewDecoder(io.LimitReader(r.Body, 16<<10))
|
||||
decoder.DisallowUnknownFields()
|
||||
if err := decoder.Decode(&value); err != nil || value.Validate() != nil {
|
||||
http.Error(w, "invalid model qualification observation", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
if err := capabilityRepository.SaveModelQualificationObservation(r.Context(), value); err != nil {
|
||||
http.Error(w, "save model qualification observation", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
w.WriteHeader(http.StatusCreated)
|
||||
})
|
||||
mux.HandleFunc("POST /v1/creator/cognitive-canary", func(w http.ResponseWriter, r *http.Request) {
|
||||
session, ok := authenticateCreator(w, r)
|
||||
if !ok {
|
||||
|
||||
Reference in New Issue
Block a user