fix(cognitive): trace and bound qualification flights
CI / test (push) Failing after 38s

This commit is contained in:
m
2026-08-31 10:25:16 +03:00
parent 9d94a86300
commit dd9bca3ace
4 changed files with 203 additions and 29 deletions
+132 -21
View File
@@ -4,8 +4,11 @@ import (
"context"
"encoding/json"
"fmt"
"log"
"net/http"
"slices"
"sort"
"strings"
"sync"
"time"
@@ -53,8 +56,83 @@ type modelQualificationResponse struct {
Results []modelQualificationResult `json:"results"`
}
const qualificationControlPlaneTimeout = 45 * time.Second
// qualificationCoordinator bounds duplicate control-plane probes within one
// cognitive-worker process. It is deliberately not canonical state: the
// Capability owner remains the sole owner of qualification evidence and
// activation. A restart may repeat an incomplete probe, but cannot create a
// second semantic writer or bypass independent evaluation.
type qualificationCoordinator struct {
mu sync.Mutex
flights map[string]*qualificationFlight
}
type qualificationFlight struct {
done chan struct{}
response modelQualificationResponse
err error
waiters int
}
func (c *qualificationCoordinator) do(ctx context.Context, key string, operation func() (modelQualificationResponse, error)) (modelQualificationResponse, bool, error) {
c.mu.Lock()
if c.flights == nil {
c.flights = make(map[string]*qualificationFlight)
}
if current, exists := c.flights[key]; exists {
current.waiters++
c.mu.Unlock()
select {
case <-current.done:
return current.response, false, current.err
case <-ctx.Done():
return modelQualificationResponse{}, false, ctx.Err()
}
}
current := &qualificationFlight{done: make(chan struct{})}
c.flights[key] = current
c.mu.Unlock()
current.response, current.err = operation()
close(current.done)
c.mu.Lock()
delete(c.flights, key)
c.mu.Unlock()
return current.response, true, current.err
}
type qualificationPhaseTrace struct {
correlationID string
started time.Time
}
func newQualificationPhaseTrace() qualificationPhaseTrace {
correlationID, err := ids.New()
if err != nil {
correlationID = ids.ID(fmt.Sprintf("qualification-%d", time.Now().UnixNano()))
}
return qualificationPhaseTrace{correlationID: string(correlationID), started: time.Now()}
}
func (t qualificationPhaseTrace) phase(name string, fields ...string) {
log.Printf("qualification_phase correlation_id=%s phase=%s elapsed_ms=%d %s", t.correlationID, name, time.Since(t.started).Milliseconds(), strings.Join(fields, " "))
}
func qualificationFlightKey(provider string, modelIDs []capability.ProviderModelID) string {
ids := make([]string, 0, len(modelIDs))
for _, modelID := range modelIDs {
ids = append(ids, string(modelID))
}
sort.Strings(ids)
return provider + "|" + strings.Join(ids, ",")
}
func modelQualificationHandler(providers map[string]providerRegistration, owner string, observers ...qualificationObserver) http.Handler {
coordinator := &qualificationCoordinator{}
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
trace := newQualificationPhaseTrace()
trace.phase("Q0", "event=request_accepted")
name := r.PathValue("provider")
registration, ok := providers[name]
if !ok || registration.Discovery == nil || registration.Provider == nil {
@@ -72,32 +150,65 @@ func modelQualificationHandler(providers map[string]providerRegistration, owner
http.Error(w, "max_output_bytes exceeds qualification bound", http.StatusBadRequest)
return
}
modelsFound, probe := registration.Discovery.Discover(r.Context())
if probe.Health != capability.HealthAvailable || len(modelsFound) == 0 {
http.Error(w, "provider unavailable for qualification", http.StatusServiceUnavailable)
return
controlCtx, cancel := context.WithTimeout(r.Context(), qualificationControlPlaneTimeout)
defer cancel()
response, leader, err := coordinator.do(controlCtx, qualificationFlightKey(name, request.ModelIDs), func() (modelQualificationResponse, error) {
trace.phase("Q1", "event=binding_candidate_loaded")
modelsFound, probe := registration.Discovery.Discover(controlCtx)
if probe.Health != capability.HealthAvailable || len(modelsFound) == 0 {
return modelQualificationResponse{}, fmt.Errorf("provider unavailable for qualification")
}
if len(request.ModelIDs) > 0 {
modelsFound = slices.DeleteFunc(modelsFound, func(model models.DiscoveredModel) bool {
return !slices.Contains(request.ModelIDs, model.ProviderModelID)
})
}
if len(modelsFound) == 0 {
return modelQualificationResponse{}, fmt.Errorf("requested model not discovered")
}
if len(modelsFound) > 16 {
modelsFound = modelsFound[:16]
}
trace.phase("Q2", fmt.Sprintf("event=provider_model_resolved model_count=%d", len(modelsFound)))
trace.phase("Q3", "event=bounded_invocation_started")
results := qualifyModels(controlCtx, registration, owner, request.Prompt, request.MaxOutputBytes, modelsFound)
trace.phase("Q4", "event=bounded_invocation_completed")
trace.phase("Q5", fmt.Sprintf("event=invocation_evidence_built result_count=%d", len(results)))
if len(observers) > 0 && observers[0] != nil {
trace.phase("Q6", "event=observation_persistence_started")
if err := persistQualificationResults(controlCtx, observers[0], registration, results); err != nil {
return modelQualificationResponse{}, fmt.Errorf("persist model qualification observation: %w", err)
}
trace.phase("Q7", "event=observation_persistence_completed")
}
// Q8-Q17 are intentionally outside this worker's authority. Capability
// owns independent evaluation and governed promotion; this trace makes
// that hand-off explicit instead of disguising it as activation.
trace.phase("Q8", "event=independent_evaluation_not_started owner=capability")
return modelQualificationResponse{Provider: registration.ProviderInstanceID, ObservedAtUnixMillis: time.Now().UnixMilli(), Results: results}, nil
})
if !leader {
trace.phase("Q0", "event=joined_in_progress_qualification")
}
if len(request.ModelIDs) > 0 {
modelsFound = slices.DeleteFunc(modelsFound, func(model models.DiscoveredModel) bool {
return !slices.Contains(request.ModelIDs, model.ProviderModelID)
})
}
if len(modelsFound) == 0 {
http.Error(w, "requested model not discovered", http.StatusNotFound)
return
}
if len(modelsFound) > 16 {
modelsFound = modelsFound[:16]
}
results := qualifyModels(r.Context(), registration, owner, request.Prompt, request.MaxOutputBytes, modelsFound)
if len(observers) > 0 && observers[0] != nil {
if err := persistQualificationResults(r.Context(), observers[0], registration, results); err != nil {
http.Error(w, "persist model qualification observation", http.StatusBadGateway)
if err != nil {
if controlCtx.Err() != nil {
http.Error(w, "qualification timeout", http.StatusGatewayTimeout)
return
}
if strings.Contains(err.Error(), "requested model not discovered") {
http.Error(w, "requested model not discovered", http.StatusNotFound)
return
}
if strings.Contains(err.Error(), "provider unavailable") {
http.Error(w, "provider unavailable for qualification", http.StatusServiceUnavailable)
return
}
http.Error(w, "persist model qualification observation", http.StatusBadGateway)
return
}
trace.phase("Q17", "event=qualification_response_returned activation=not_attempted")
w.Header().Set("Content-Type", "application/json")
_ = jsonEncode(w, modelQualificationResponse{Provider: registration.ProviderInstanceID, ObservedAtUnixMillis: time.Now().UnixMilli(), Results: results})
_ = jsonEncode(w, response)
})
}
@@ -6,6 +6,7 @@ import (
"net/http/httptest"
"strings"
"testing"
"time"
"git.cin.su/m/Assistant/adapters/models"
"git.cin.su/m/Assistant/contracts/capability"
@@ -17,6 +18,7 @@ type qualificationProvider struct{}
func (qualificationProvider) Probe(context.Context) models.ProviderProbe {
return models.ProviderProbe{Health: capability.HealthAvailable}
}
func (qualificationProvider) Discover(context.Context) ([]models.DiscoveredModel, models.ProviderProbe) {
return []models.DiscoveredModel{{ProviderModelID: "model-a"}, {ProviderModelID: "model-b"}}, models.ProviderProbe{Health: capability.HealthAvailable}
}
@@ -83,3 +85,63 @@ func TestModelQualificationPersistsReachabilityWithoutQuality(t *testing.T) {
t.Fatalf("invalid qualification observation: %+v err=%v", observer.observations[0], err)
}
}
func TestModelQualificationDeduplicatesSameProviderModelFlight(t *testing.T) {
coordinator := &qualificationCoordinator{}
started := make(chan struct{})
release := make(chan struct{})
firstDone := make(chan struct{})
secondDone := make(chan struct{})
result := modelQualificationResponse{Provider: "medium"}
go func() {
_, _, err := coordinator.do(context.Background(), "medium|model-a", func() (modelQualificationResponse, error) {
close(started)
<-release
return result, nil
})
if err != nil {
t.Errorf("leader qualification failed: %v", err)
}
close(firstDone)
}()
select {
case <-started:
case <-time.After(time.Second):
t.Fatal("qualification invocation did not start")
}
go func() {
actual, leader, err := coordinator.do(context.Background(), "medium|model-a", func() (modelQualificationResponse, error) {
t.Error("duplicate qualification operation ran")
return modelQualificationResponse{}, nil
})
if err != nil || leader || actual.Provider != result.Provider {
t.Errorf("joined qualification response=%+v leader=%t err=%v", actual, leader, err)
}
close(secondDone)
}()
deadline := time.Now().Add(time.Second)
for {
coordinator.mu.Lock()
flight := coordinator.flights["medium|model-a"]
waiters := 0
if flight != nil {
waiters = flight.waiters
}
coordinator.mu.Unlock()
if waiters == 1 {
break
}
if time.Now().After(deadline) {
t.Fatal("second qualification did not join the in-progress flight")
}
time.Sleep(time.Millisecond)
}
close(release)
for _, done := range []chan struct{}{firstDone, secondDone} {
select {
case <-done:
case <-time.After(time.Second):
t.Fatal("qualification flight did not complete")
}
}
}