feat(capability): persist model qualification observations
CI / test (push) Successful in 1m14s
CI / test (pull_request) Successful in 1m15s

This commit is contained in:
m
2026-08-23 19:21:04 +03:00
parent e4ba1cb5fa
commit d0ede46352
10 changed files with 194 additions and 4 deletions
+1 -1
View File
@@ -169,7 +169,7 @@ func main() {
mux.HandleFunc("POST /v1/task-evaluations/execute", gateway.taskEvaluationHandler)
discovery := discoveryService{resolver: resolver, providers: discoveries, owner: identity.PrincipalID(envOr("COGNITIVE_OWNER_PRINCIPAL_ID", "domovoy-creator-principal")), now: time.Now}
mux.HandleFunc("POST /v1/provider-discovery/{provider}", discovery.handler)
mux.Handle("POST /v1/provider-discovery/{provider}/qualify", modelQualificationHandler(discoveries, string(envOr("COGNITIVE_OWNER_PRINCIPAL_ID", "domovoy-creator-principal"))))
mux.Handle("POST /v1/provider-discovery/{provider}/qualify", modelQualificationHandler(discoveries, string(envOr("COGNITIVE_OWNER_PRINCIPAL_ID", "domovoy-creator-principal")), httpQualificationObserver{resolver: resolver}))
mux.HandleFunc("GET /v1/provider-discovery/status", discovery.statusHandler)
refreshProviderDiscoveries(resolver, discovery)
log.Printf("cognitive-worker listening on %s", addr)
+51 -1
View File
@@ -14,8 +14,22 @@ import (
"git.cin.su/m/Assistant/contracts/capability"
"git.cin.su/m/Assistant/contracts/cognitive"
"git.cin.su/m/Assistant/contracts/identity"
"git.cin.su/m/Assistant/platform/ids"
)
type qualificationObserver interface {
ObserveQualification(context.Context, capability.ModelQualificationObservation) error
}
type httpQualificationObserver struct{ resolver *httpBindingResolver }
func (o httpQualificationObserver) ObserveQualification(ctx context.Context, value capability.ModelQualificationObservation) error {
if o.resolver == nil {
return fmt.Errorf("qualification observer resolver is nil")
}
return o.resolver.post(ctx, "/v1/cognitive-registry/model-qualification-observations", value, nil)
}
type modelQualificationRequest struct {
Prompt string `json:"prompt"`
MaxOutputBytes int `json:"max_output_bytes"`
@@ -39,7 +53,7 @@ type modelQualificationResponse struct {
Results []modelQualificationResult `json:"results"`
}
func modelQualificationHandler(providers map[string]providerRegistration, owner string) http.Handler {
func modelQualificationHandler(providers map[string]providerRegistration, owner string, observers ...qualificationObserver) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
name := r.PathValue("provider")
registration, ok := providers[name]
@@ -76,6 +90,12 @@ func modelQualificationHandler(providers map[string]providerRegistration, owner
modelsFound = modelsFound[:16]
}
results := qualifyModels(r.Context(), registration, owner, request.Prompt, request.MaxOutputBytes, modelsFound)
if len(observers) > 0 && observers[0] != nil {
if err := persistQualificationResults(r.Context(), observers[0], registration, results); err != nil {
http.Error(w, "persist model qualification observation", http.StatusBadGateway)
return
}
}
w.Header().Set("Content-Type", "application/json")
_ = jsonEncode(w, modelQualificationResponse{Provider: registration.ProviderInstanceID, ObservedAtUnixMillis: time.Now().UnixMilli(), Results: results})
})
@@ -94,6 +114,7 @@ func qualifyModels(ctx context.Context, registration providerRegistration, owner
result := modelQualificationResult{ProviderModelID: model.ProviderModelID, LatencyMillis: time.Since(started).Milliseconds(), QualityAssessed: false, Category: capability.ModelRatingUnrated}
if err != nil {
result.Status = "UNAVAILABLE"
result.FinishReason = cognitive.FinishProviderUnavailable
result.Error = err.Error()
results[index] = result
return
@@ -112,6 +133,35 @@ func qualifyModels(ctx context.Context, registration providerRegistration, owner
return results
}
func persistQualificationResults(ctx context.Context, observer qualificationObserver, registration providerRegistration, results []modelQualificationResult) error {
for _, result := range results {
observationID, err := ids.New()
if err != nil {
return err
}
status := capability.ModelQualificationRespondedWithoutRating
switch result.Status {
case "REACHABLE":
status = capability.ModelQualificationReachable
case "UNAVAILABLE":
status = capability.ModelQualificationUnavailable
}
observation := capability.ModelQualificationObservation{
ID: capability.ModelQualificationObservationID(observationID), ProviderInstanceID: registration.ProviderInstanceID, ProviderModelID: result.ProviderModelID,
Status: status, FinishReason: string(result.FinishReason), LatencyMillis: result.LatencyMillis,
OutputObserved: result.OutputObserved, QualityAssessed: false, Category: capability.ModelRatingUnrated,
ObservedAtUnixMillis: time.Now().UnixMilli(),
}
if err := observation.Validate(); err != nil {
return err
}
if err := observer.ObserveQualification(ctx, observation); err != nil {
return err
}
}
return nil
}
func jsonEncode(w http.ResponseWriter, value any) error {
return json.NewEncoder(w).Encode(value)
}
@@ -57,3 +57,29 @@ func TestModelQualificationRejectsUnboundedOutput(t *testing.T) {
t.Fatalf("unbounded qualification status=%d", response.Code)
}
}
type qualificationObserverStub struct {
observations []capability.ModelQualificationObservation
}
func (s *qualificationObserverStub) ObserveQualification(_ context.Context, value capability.ModelQualificationObservation) error {
s.observations = append(s.observations, value)
return nil
}
func TestModelQualificationPersistsReachabilityWithoutQuality(t *testing.T) {
provider := qualificationProvider{}
observer := &qualificationObserverStub{}
mux := http.NewServeMux()
mux.Handle("POST /v1/provider-discovery/{provider}/qualify", modelQualificationHandler(map[string]providerRegistration{"cpu": {ProviderInstanceID: "cpu", Provider: provider, Discovery: provider}}, "owner", observer))
request := httptest.NewRequest(http.MethodPost, "/v1/provider-discovery/cpu/qualify", strings.NewReader(`{"prompt":"ready","max_output_bytes":128,"model_ids":["model-a"]}`))
request.Header.Set("Content-Type", "application/json")
response := httptest.NewRecorder()
mux.ServeHTTP(response, request)
if response.Code != http.StatusOK || len(observer.observations) != 1 {
t.Fatalf("qualification persistence status=%d observations=%d body=%s", response.Code, len(observer.observations), response.Body.String())
}
if err := observer.observations[0].Validate(); err != nil || observer.observations[0].Status != capability.ModelQualificationReachable {
t.Fatalf("invalid qualification observation: %+v err=%v", observer.observations[0], err)
}
}
+14
View File
@@ -1058,6 +1058,20 @@ func main() {
}
w.WriteHeader(http.StatusCreated)
})
mux.HandleFunc("POST /v1/cognitive-registry/model-qualification-observations", func(w http.ResponseWriter, r *http.Request) {
var value capability.ModelQualificationObservation
decoder := json.NewDecoder(io.LimitReader(r.Body, 16<<10))
decoder.DisallowUnknownFields()
if err := decoder.Decode(&value); err != nil || value.Validate() != nil {
http.Error(w, "invalid model qualification observation", http.StatusBadRequest)
return
}
if err := capabilityRepository.SaveModelQualificationObservation(r.Context(), value); err != nil {
http.Error(w, "save model qualification observation", http.StatusInternalServerError)
return
}
w.WriteHeader(http.StatusCreated)
})
mux.HandleFunc("POST /v1/creator/cognitive-canary", func(w http.ResponseWriter, r *http.Request) {
session, ok := authenticateCreator(w, r)
if !ok {