This commit is contained in:
@@ -4,8 +4,11 @@ import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log"
|
||||
"net/http"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
@@ -53,8 +56,83 @@ type modelQualificationResponse struct {
|
||||
Results []modelQualificationResult `json:"results"`
|
||||
}
|
||||
|
||||
const qualificationControlPlaneTimeout = 45 * time.Second
|
||||
|
||||
// qualificationCoordinator bounds duplicate control-plane probes within one
|
||||
// cognitive-worker process. It is deliberately not canonical state: the
|
||||
// Capability owner remains the sole owner of qualification evidence and
|
||||
// activation. A restart may repeat an incomplete probe, but cannot create a
|
||||
// second semantic writer or bypass independent evaluation.
|
||||
type qualificationCoordinator struct {
|
||||
mu sync.Mutex
|
||||
flights map[string]*qualificationFlight
|
||||
}
|
||||
|
||||
type qualificationFlight struct {
|
||||
done chan struct{}
|
||||
response modelQualificationResponse
|
||||
err error
|
||||
waiters int
|
||||
}
|
||||
|
||||
func (c *qualificationCoordinator) do(ctx context.Context, key string, operation func() (modelQualificationResponse, error)) (modelQualificationResponse, bool, error) {
|
||||
c.mu.Lock()
|
||||
if c.flights == nil {
|
||||
c.flights = make(map[string]*qualificationFlight)
|
||||
}
|
||||
if current, exists := c.flights[key]; exists {
|
||||
current.waiters++
|
||||
c.mu.Unlock()
|
||||
select {
|
||||
case <-current.done:
|
||||
return current.response, false, current.err
|
||||
case <-ctx.Done():
|
||||
return modelQualificationResponse{}, false, ctx.Err()
|
||||
}
|
||||
}
|
||||
current := &qualificationFlight{done: make(chan struct{})}
|
||||
c.flights[key] = current
|
||||
c.mu.Unlock()
|
||||
|
||||
current.response, current.err = operation()
|
||||
close(current.done)
|
||||
c.mu.Lock()
|
||||
delete(c.flights, key)
|
||||
c.mu.Unlock()
|
||||
return current.response, true, current.err
|
||||
}
|
||||
|
||||
type qualificationPhaseTrace struct {
|
||||
correlationID string
|
||||
started time.Time
|
||||
}
|
||||
|
||||
func newQualificationPhaseTrace() qualificationPhaseTrace {
|
||||
correlationID, err := ids.New()
|
||||
if err != nil {
|
||||
correlationID = ids.ID(fmt.Sprintf("qualification-%d", time.Now().UnixNano()))
|
||||
}
|
||||
return qualificationPhaseTrace{correlationID: string(correlationID), started: time.Now()}
|
||||
}
|
||||
|
||||
func (t qualificationPhaseTrace) phase(name string, fields ...string) {
|
||||
log.Printf("qualification_phase correlation_id=%s phase=%s elapsed_ms=%d %s", t.correlationID, name, time.Since(t.started).Milliseconds(), strings.Join(fields, " "))
|
||||
}
|
||||
|
||||
func qualificationFlightKey(provider string, modelIDs []capability.ProviderModelID) string {
|
||||
ids := make([]string, 0, len(modelIDs))
|
||||
for _, modelID := range modelIDs {
|
||||
ids = append(ids, string(modelID))
|
||||
}
|
||||
sort.Strings(ids)
|
||||
return provider + "|" + strings.Join(ids, ",")
|
||||
}
|
||||
|
||||
func modelQualificationHandler(providers map[string]providerRegistration, owner string, observers ...qualificationObserver) http.Handler {
|
||||
coordinator := &qualificationCoordinator{}
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
trace := newQualificationPhaseTrace()
|
||||
trace.phase("Q0", "event=request_accepted")
|
||||
name := r.PathValue("provider")
|
||||
registration, ok := providers[name]
|
||||
if !ok || registration.Discovery == nil || registration.Provider == nil {
|
||||
@@ -72,32 +150,65 @@ func modelQualificationHandler(providers map[string]providerRegistration, owner
|
||||
http.Error(w, "max_output_bytes exceeds qualification bound", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
modelsFound, probe := registration.Discovery.Discover(r.Context())
|
||||
if probe.Health != capability.HealthAvailable || len(modelsFound) == 0 {
|
||||
http.Error(w, "provider unavailable for qualification", http.StatusServiceUnavailable)
|
||||
return
|
||||
controlCtx, cancel := context.WithTimeout(r.Context(), qualificationControlPlaneTimeout)
|
||||
defer cancel()
|
||||
response, leader, err := coordinator.do(controlCtx, qualificationFlightKey(name, request.ModelIDs), func() (modelQualificationResponse, error) {
|
||||
trace.phase("Q1", "event=binding_candidate_loaded")
|
||||
modelsFound, probe := registration.Discovery.Discover(controlCtx)
|
||||
if probe.Health != capability.HealthAvailable || len(modelsFound) == 0 {
|
||||
return modelQualificationResponse{}, fmt.Errorf("provider unavailable for qualification")
|
||||
}
|
||||
if len(request.ModelIDs) > 0 {
|
||||
modelsFound = slices.DeleteFunc(modelsFound, func(model models.DiscoveredModel) bool {
|
||||
return !slices.Contains(request.ModelIDs, model.ProviderModelID)
|
||||
})
|
||||
}
|
||||
if len(modelsFound) == 0 {
|
||||
return modelQualificationResponse{}, fmt.Errorf("requested model not discovered")
|
||||
}
|
||||
if len(modelsFound) > 16 {
|
||||
modelsFound = modelsFound[:16]
|
||||
}
|
||||
trace.phase("Q2", fmt.Sprintf("event=provider_model_resolved model_count=%d", len(modelsFound)))
|
||||
trace.phase("Q3", "event=bounded_invocation_started")
|
||||
results := qualifyModels(controlCtx, registration, owner, request.Prompt, request.MaxOutputBytes, modelsFound)
|
||||
trace.phase("Q4", "event=bounded_invocation_completed")
|
||||
trace.phase("Q5", fmt.Sprintf("event=invocation_evidence_built result_count=%d", len(results)))
|
||||
if len(observers) > 0 && observers[0] != nil {
|
||||
trace.phase("Q6", "event=observation_persistence_started")
|
||||
if err := persistQualificationResults(controlCtx, observers[0], registration, results); err != nil {
|
||||
return modelQualificationResponse{}, fmt.Errorf("persist model qualification observation: %w", err)
|
||||
}
|
||||
trace.phase("Q7", "event=observation_persistence_completed")
|
||||
}
|
||||
// Q8-Q17 are intentionally outside this worker's authority. Capability
|
||||
// owns independent evaluation and governed promotion; this trace makes
|
||||
// that hand-off explicit instead of disguising it as activation.
|
||||
trace.phase("Q8", "event=independent_evaluation_not_started owner=capability")
|
||||
return modelQualificationResponse{Provider: registration.ProviderInstanceID, ObservedAtUnixMillis: time.Now().UnixMilli(), Results: results}, nil
|
||||
})
|
||||
if !leader {
|
||||
trace.phase("Q0", "event=joined_in_progress_qualification")
|
||||
}
|
||||
if len(request.ModelIDs) > 0 {
|
||||
modelsFound = slices.DeleteFunc(modelsFound, func(model models.DiscoveredModel) bool {
|
||||
return !slices.Contains(request.ModelIDs, model.ProviderModelID)
|
||||
})
|
||||
}
|
||||
if len(modelsFound) == 0 {
|
||||
http.Error(w, "requested model not discovered", http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
if len(modelsFound) > 16 {
|
||||
modelsFound = modelsFound[:16]
|
||||
}
|
||||
results := qualifyModels(r.Context(), registration, owner, request.Prompt, request.MaxOutputBytes, modelsFound)
|
||||
if len(observers) > 0 && observers[0] != nil {
|
||||
if err := persistQualificationResults(r.Context(), observers[0], registration, results); err != nil {
|
||||
http.Error(w, "persist model qualification observation", http.StatusBadGateway)
|
||||
if err != nil {
|
||||
if controlCtx.Err() != nil {
|
||||
http.Error(w, "qualification timeout", http.StatusGatewayTimeout)
|
||||
return
|
||||
}
|
||||
if strings.Contains(err.Error(), "requested model not discovered") {
|
||||
http.Error(w, "requested model not discovered", http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
if strings.Contains(err.Error(), "provider unavailable") {
|
||||
http.Error(w, "provider unavailable for qualification", http.StatusServiceUnavailable)
|
||||
return
|
||||
}
|
||||
http.Error(w, "persist model qualification observation", http.StatusBadGateway)
|
||||
return
|
||||
}
|
||||
trace.phase("Q17", "event=qualification_response_returned activation=not_attempted")
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_ = jsonEncode(w, modelQualificationResponse{Provider: registration.ProviderInstanceID, ObservedAtUnixMillis: time.Now().UnixMilli(), Results: results})
|
||||
_ = jsonEncode(w, response)
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.cin.su/m/Assistant/adapters/models"
|
||||
"git.cin.su/m/Assistant/contracts/capability"
|
||||
@@ -17,6 +18,7 @@ type qualificationProvider struct{}
|
||||
func (qualificationProvider) Probe(context.Context) models.ProviderProbe {
|
||||
return models.ProviderProbe{Health: capability.HealthAvailable}
|
||||
}
|
||||
|
||||
func (qualificationProvider) Discover(context.Context) ([]models.DiscoveredModel, models.ProviderProbe) {
|
||||
return []models.DiscoveredModel{{ProviderModelID: "model-a"}, {ProviderModelID: "model-b"}}, models.ProviderProbe{Health: capability.HealthAvailable}
|
||||
}
|
||||
@@ -83,3 +85,63 @@ func TestModelQualificationPersistsReachabilityWithoutQuality(t *testing.T) {
|
||||
t.Fatalf("invalid qualification observation: %+v err=%v", observer.observations[0], err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestModelQualificationDeduplicatesSameProviderModelFlight(t *testing.T) {
|
||||
coordinator := &qualificationCoordinator{}
|
||||
started := make(chan struct{})
|
||||
release := make(chan struct{})
|
||||
firstDone := make(chan struct{})
|
||||
secondDone := make(chan struct{})
|
||||
result := modelQualificationResponse{Provider: "medium"}
|
||||
go func() {
|
||||
_, _, err := coordinator.do(context.Background(), "medium|model-a", func() (modelQualificationResponse, error) {
|
||||
close(started)
|
||||
<-release
|
||||
return result, nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Errorf("leader qualification failed: %v", err)
|
||||
}
|
||||
close(firstDone)
|
||||
}()
|
||||
select {
|
||||
case <-started:
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("qualification invocation did not start")
|
||||
}
|
||||
go func() {
|
||||
actual, leader, err := coordinator.do(context.Background(), "medium|model-a", func() (modelQualificationResponse, error) {
|
||||
t.Error("duplicate qualification operation ran")
|
||||
return modelQualificationResponse{}, nil
|
||||
})
|
||||
if err != nil || leader || actual.Provider != result.Provider {
|
||||
t.Errorf("joined qualification response=%+v leader=%t err=%v", actual, leader, err)
|
||||
}
|
||||
close(secondDone)
|
||||
}()
|
||||
deadline := time.Now().Add(time.Second)
|
||||
for {
|
||||
coordinator.mu.Lock()
|
||||
flight := coordinator.flights["medium|model-a"]
|
||||
waiters := 0
|
||||
if flight != nil {
|
||||
waiters = flight.waiters
|
||||
}
|
||||
coordinator.mu.Unlock()
|
||||
if waiters == 1 {
|
||||
break
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatal("second qualification did not join the in-progress flight")
|
||||
}
|
||||
time.Sleep(time.Millisecond)
|
||||
}
|
||||
close(release)
|
||||
for _, done := range []chan struct{}{firstDone, secondDone} {
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("qualification flight did not complete")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user