fix: revalidate stale cognitive binding health
CI / test (push) Failing after 35s

This commit is contained in:
m
2026-08-30 15:56:34 +03:00
parent 2e824b3c1f
commit 185317ee6a
5 changed files with 148 additions and 20 deletions
+16 -3
View File
@@ -16,15 +16,17 @@ func (p *admissionTestProvider) Advise(context.Context, cognitive.AdvisoryReques
}
type admissionTestClient struct {
decision capability.AdmissionDecision
releases int
decision capability.AdmissionDecision
releases int
releaseContextErr error
}
func (c *admissionTestClient) Admit(context.Context, string, capability.AdmissionRequest) (capability.AdmissionDecision, capability.AdmissionLease, error) {
return c.decision, capability.AdmissionLease{ID: "lease-1", PolicyID: "policy-1"}, nil
}
func (c *admissionTestClient) Release(context.Context, capability.AdmissionLease) error {
func (c *admissionTestClient) Release(ctx context.Context, _ capability.AdmissionLease) error {
c.releases++
c.releaseContextErr = ctx.Err()
return nil
}
@@ -47,3 +49,14 @@ func TestInvokeAdmittedReleasesDurableLeaseAfterProvider(t *testing.T) {
t.Fatalf("admitted lease was not released: response=%#v err=%v calls=%d releases=%d", response, err, provider.calls, client.releases)
}
}
func TestInvokeAdmittedReleasesLeaseAfterCallerCancellation(t *testing.T) {
provider := &admissionTestProvider{}
client := &admissionTestClient{decision: capability.AdmissionDecision{State: capability.AdmissionAdmitted, Reason: capability.AdmissionReasonNone, PolicyID: "policy-1"}}
ctx, cancel := context.WithCancel(context.Background())
cancel()
_, _ = (&cognitiveGateway{admission: client}).invokeAdmitted(ctx, "request-1", provider, cognitive.AdvisoryRequest{Message: "hello"}, 100)
if client.releases != 1 || client.releaseContextErr != nil {
t.Fatalf("cancelled caller leaked durable lease: releases=%d release_context_err=%v", client.releases, client.releaseContextErr)
}
}
+16 -11
View File
@@ -1,7 +1,9 @@
package main
import (
"context"
"encoding/json"
"fmt"
"net/http"
"sort"
"time"
@@ -22,17 +24,7 @@ func (s discoveryService) handler(w http.ResponseWriter, r *http.Request) {
s.statusHandler(w, r)
return
}
registration, ok := s.providers[r.PathValue("provider")]
if !ok || registration.Discovery == nil {
http.Error(w, "provider discovery not found", http.StatusNotFound)
return
}
discovered, probe := registration.Discovery.Discover(r.Context())
snapshot := capability.CognitiveDiscoverySnapshot{ProviderInstanceID: registration.ProviderInstanceID, ProviderKind: registration.Kind, EndpointRef: registration.EndpointRef, CredentialRef: registration.CredentialRef, AuthMode: registration.AuthMode, BillingMode: registration.BillingMode, Enabled: true, OwnerPrincipalID: s.owner, CapabilityID: creatorAdvisoryCapabilityID, DataPolicy: registration.DataPolicy, LimitPolicyRef: "creator-advisory-local-v1", Health: probe.Health, FailureClass: capability.CognitiveFailureType(probe.Failure), ObservedAtUnixMillis: s.now().UnixMilli()}
for _, model := range discovered {
snapshot.Models = append(snapshot.Models, capability.DiscoveredModelSnapshot{ProviderModelID: model.ProviderModelID, Fingerprint: model.Fingerprint, MaxContextBytes: model.MaxContextBytes, Modalities: model.Modalities, Capabilities: model.Capabilities, Installed: capability.OperationalYes, Loaded: capability.OperationalNotObserved, Invokable: capability.OperationalNotObserved})
}
result, err := s.resolver.RegisterDiscovery(r.Context(), snapshot)
result, err := s.register(r.Context(), r.PathValue("provider"))
if err != nil {
http.Error(w, "provider discovery registration unavailable", http.StatusBadGateway)
return
@@ -41,6 +33,19 @@ func (s discoveryService) handler(w http.ResponseWriter, r *http.Request) {
_ = json.NewEncoder(w).Encode(result)
}
func (s discoveryService) register(ctx context.Context, provider string) (capability.CognitiveDiscoveryRegistration, error) {
registration, ok := s.providers[provider]
if !ok || registration.Discovery == nil {
return capability.CognitiveDiscoveryRegistration{}, fmt.Errorf("provider discovery not found")
}
discovered, probe := registration.Discovery.Discover(ctx)
snapshot := capability.CognitiveDiscoverySnapshot{ProviderInstanceID: registration.ProviderInstanceID, ProviderKind: registration.Kind, EndpointRef: registration.EndpointRef, CredentialRef: registration.CredentialRef, AuthMode: registration.AuthMode, BillingMode: registration.BillingMode, Enabled: true, OwnerPrincipalID: s.owner, CapabilityID: creatorAdvisoryCapabilityID, DataPolicy: registration.DataPolicy, LimitPolicyRef: "creator-advisory-local-v1", Health: probe.Health, FailureClass: capability.CognitiveFailureType(probe.Failure), ObservedAtUnixMillis: s.now().UnixMilli()}
for _, model := range discovered {
snapshot.Models = append(snapshot.Models, capability.DiscoveredModelSnapshot{ProviderModelID: model.ProviderModelID, Fingerprint: model.Fingerprint, MaxContextBytes: model.MaxContextBytes, Modalities: model.Modalities, Capabilities: model.Capabilities, Installed: capability.OperationalYes, Loaded: capability.OperationalNotObserved, Invokable: capability.OperationalNotObserved})
}
return s.resolver.RegisterDiscovery(ctx, snapshot)
}
type discoveryStatus struct {
Provider string `json:"provider"`
ProviderKind capability.ProviderKind `json:"provider_kind"`
+37 -3
View File
@@ -4,6 +4,7 @@ import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
@@ -20,6 +21,8 @@ import (
const creatorAdvisoryCapabilityID capability.CapabilityID = "creator-advisory"
var errNoInvocableCognitiveBinding = errors.New("no invocable cognitive binding")
type bindingResolver interface {
Resolve(context.Context, capability.CognitiveRequirement) (capability.ResolvedCognitiveBinding, error)
Observe(context.Context, capability.CognitiveUsageObservation) error
@@ -56,6 +59,7 @@ type cognitiveGateway struct {
allowCanary bool
canaryMaxOutputBytes int
admission admissionClient
revalidatePrimary func(context.Context) error
}
func (g cognitiveGateway) policyHandler(w http.ResponseWriter, r *http.Request) {
@@ -89,7 +93,7 @@ func (g cognitiveGateway) advisoryHandler(w http.ResponseWriter, r *http.Request
maxOutputBytes = g.canaryMaxOutputBytes
}
requirement := capability.CognitiveRequirement{CapabilityID: creatorAdvisoryCapabilityID, OwnerPrincipalID: request.PrincipalID, DataPolicy: request.DataPolicy, RequiredModalities: []capability.ModelModality{capability.ModalityText}, RequiredCapabilities: []capability.ModelCapability{capability.CapabilityAdvisory}, AllowCanary: g.allowCanary, InputBytes: int64(len(request.Message)), MaxOutputBytes: int64(maxOutputBytes), Objective: selectionObjectiveForMeaning(request.Meaning, request.Message)}
selected, err := g.resolver.Resolve(r.Context(), requirement)
selected, err := g.resolveWithRevalidation(r.Context(), requirement)
if err != nil {
writeGatewayResponse(w, g.invokeBuiltinAdvisory(r.Context(), request, maxOutputBytes))
return
@@ -145,11 +149,33 @@ func (g cognitiveGateway) invokeAdmitted(ctx context.Context, leaseID string, pr
return cognitive.AdvisoryResponse{Version: invocation.Version, RequestID: invocation.RequestID, ProviderBindingID: invocation.ProviderBindingID, ProviderBindingRevision: invocation.ProviderBindingRevision, FinishReason: cognitive.FinishRateLimited}, nil
}
lease = admittedLease
defer func() { _ = g.admission.Release(ctx, lease) }()
defer func() {
releaseCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second)
defer cancel()
_ = g.admission.Release(releaseCtx, lease)
}()
}
return provider.Advise(ctx, invocation)
}
// resolveWithRevalidation makes one bounded recovery attempt when Capability
// has an active/validated binding but its durable health projection rejected it
// as non-invocable. Discovery remains the canonical Capability write path; the
// retry observes the configured primary local provider and re-runs selection
// once. Provider invocation, admission, and fallback remain governed elsewhere.
func (g cognitiveGateway) resolveWithRevalidation(ctx context.Context, requirement capability.CognitiveRequirement) (capability.ResolvedCognitiveBinding, error) {
selected, err := g.resolver.Resolve(ctx, requirement)
if err == nil || !errors.Is(err, errNoInvocableCognitiveBinding) || g.revalidatePrimary == nil {
return selected, err
}
revalidationCtx, cancel := context.WithTimeout(ctx, 45*time.Second)
defer cancel()
if revalidationErr := g.revalidatePrimary(revalidationCtx); revalidationErr != nil {
return selected, err
}
return g.resolver.Resolve(ctx, requirement)
}
func selectionObjectiveForMessage(message string) capability.SelectionObjective {
lower := strings.ToLower(message)
if strings.Contains(lower, "дешев") || strings.Contains(lower, "бесплат") || strings.Contains(lower, "эконом") || strings.Contains(lower, "cost") || strings.Contains(lower, "cheap") {
@@ -374,6 +400,10 @@ type httpBindingResolver struct {
client *http.Client
}
type httpStatusError struct{ status int }
func (e httpStatusError) Error() string { return fmt.Sprintf("registry status %d", e.status) }
func newHTTPBindingResolver(raw string, client *http.Client) (*httpBindingResolver, error) {
base, err := url.Parse(raw)
if err != nil || base.Scheme == "" || base.Host == "" {
@@ -385,6 +415,10 @@ func newHTTPBindingResolver(raw string, client *http.Client) (*httpBindingResolv
func (r *httpBindingResolver) Resolve(ctx context.Context, value capability.CognitiveRequirement) (capability.ResolvedCognitiveBinding, error) {
var result capability.ResolvedCognitiveBinding
err := r.post(ctx, "/v1/cognitive-bindings/resolve", value, &result)
var statusErr httpStatusError
if errors.As(err, &statusErr) && statusErr.status == http.StatusNotFound {
return result, errNoInvocableCognitiveBinding
}
return result, err
}
@@ -437,7 +471,7 @@ func (r *httpBindingResolver) post(ctx context.Context, path string, value, dest
defer response.Body.Close()
if response.StatusCode < 200 || response.StatusCode >= 300 {
_, _ = io.Copy(io.Discard, io.LimitReader(response.Body, 4096))
return fmt.Errorf("registry status %d", response.StatusCode)
return httpStatusError{status: response.StatusCode}
}
if destination != nil {
return json.NewDecoder(io.LimitReader(response.Body, 64<<10)).Decode(destination)
@@ -22,6 +22,26 @@ type registryStub struct {
observation *capability.CognitiveUsageObservation
}
type revalidatingRegistry struct {
selected capability.ResolvedCognitiveBinding
calls int
alwaysNo bool
}
func (s *revalidatingRegistry) Resolve(context.Context, capability.CognitiveRequirement) (capability.ResolvedCognitiveBinding, error) {
s.calls++
if s.alwaysNo || s.calls == 1 {
return capability.ResolvedCognitiveBinding{}, errNoInvocableCognitiveBinding
}
return s.selected, nil
}
func (*revalidatingRegistry) Observe(context.Context, capability.CognitiveUsageObservation) error {
return nil
}
func (*revalidatingRegistry) RegisterDiscovery(context.Context, capability.CognitiveDiscoverySnapshot) (capability.CognitiveDiscoveryRegistration, error) {
return capability.CognitiveDiscoveryRegistration{}, nil
}
type failoverRegistry struct {
primary, fallback capability.ResolvedCognitiveBinding
observations []capability.CognitiveUsageObservation
@@ -88,6 +108,59 @@ func TestActiveRegistryBindingReachesProviderAdapter(t *testing.T) {
}
}
func TestStaleUnavailableBindingRevalidatesOnceBeforeInvocation(t *testing.T) {
selected := gatewaySelection()
selected.Binding.DataPolicy = capability.DataPolicyLocalOnly
registry := &revalidatingRegistry{selected: selected}
provider := &stubAdvisoryProvider{response: cognitive.AdvisoryResponse{Version: cognitive.CreatorAdvisoryV1, RequestID: "request-1", AdvisoryText: "ok", FinishReason: cognitive.FinishCompleted}}
revalidations := 0
gateway := cognitiveGateway{resolver: registry, providers: map[capability.ProviderKind][]providerRegistration{"provider-kind-1": {{EndpointRef: "endpoint-1", CredentialRef: "credential-1", Provider: provider}}}, now: time.Now, revalidatePrimary: func(context.Context) error { revalidations++; return nil }}
response := httptest.NewRecorder()
gateway.advisoryHandler(response, httptest.NewRequest(http.MethodPost, "/v1/advisories", strings.NewReader(gatewayRequest())))
if response.Code != http.StatusOK || !provider.called || revalidations != 1 || registry.calls != 2 {
t.Fatalf("status=%d provider_called=%t revalidations=%d resolves=%d", response.Code, provider.called, revalidations, registry.calls)
}
}
func TestRevalidationFailureDoesNotMakeBindingInvocable(t *testing.T) {
registry := &registryStub{err: errNoInvocableCognitiveBinding}
provider := &stubAdvisoryProvider{}
revalidations := 0
gateway := cognitiveGateway{resolver: registry, providers: map[capability.ProviderKind][]providerRegistration{"provider-kind-1": {{Provider: provider}}}, now: time.Now, revalidatePrimary: func(context.Context) error { revalidations++; return errors.New("provider unavailable") }}
gateway.advisoryHandler(httptest.NewRecorder(), httptest.NewRequest(http.MethodPost, "/v1/advisories", strings.NewReader(gatewayRequest())))
if provider.called || revalidations != 1 {
t.Fatalf("failed provider revalidation was treated as invocable: provider_called=%t revalidations=%d", provider.called, revalidations)
}
}
func TestRevalidationIsNotRetriedInALoop(t *testing.T) {
registry := &revalidatingRegistry{alwaysNo: true}
revalidations := 0
gateway := cognitiveGateway{resolver: registry, now: time.Now, revalidatePrimary: func(context.Context) error { revalidations++; return nil }}
gateway.advisoryHandler(httptest.NewRecorder(), httptest.NewRequest(http.MethodPost, "/v1/advisories", strings.NewReader(gatewayRequest())))
if revalidations != 1 || registry.calls != 2 {
t.Fatalf("revalidation loop: revalidations=%d resolves=%d", revalidations, registry.calls)
}
}
func TestHTTPResolverMarksNoInvocableBindingForSingleRevalidation(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path != "/v1/cognitive-bindings/resolve" {
t.Fatalf("path=%s", r.URL.Path)
}
http.Error(w, "no invocable cognitive binding", http.StatusNotFound)
}))
defer server.Close()
resolver, err := newHTTPBindingResolver(server.URL, server.Client())
if err != nil {
t.Fatal(err)
}
_, err = resolver.Resolve(context.Background(), capability.CognitiveRequirement{CapabilityID: creatorAdvisoryCapabilityID, OwnerPrincipalID: "principal-1", MaxOutputBytes: 1})
if !errors.Is(err, errNoInvocableCognitiveBinding) {
t.Fatalf("resolve error=%v, want no-invocable sentinel", err)
}
}
func TestProviderUsesStableInstanceWhenCredentialMetadataIsStale(t *testing.T) {
provider := &stubAdvisoryProvider{}
gateway := cognitiveGateway{providers: map[capability.ProviderKind][]providerRegistration{
+6 -3
View File
@@ -113,11 +113,15 @@ func main() {
kind := capability.ProviderKind(envOr("OPENAI_PROVIDER_KIND", "openai-responses-v1"))
providers[kind] = append(providers[kind], providerRegistration{ProviderInstanceID: "openai-bootstrap", Kind: kind, EndpointRef: capability.EndpointBindingRef(envOr("OPENAI_ENDPOINT_REF", "openai-responses-default")), CredentialRef: capability.CredentialRef(envOr("OPENAI_CREDENTIAL_REF", "openai-advisory-v1")), AuthMode: capability.AuthKey, BillingMode: capability.BillingPerInput, DataPolicy: capability.DataPolicyExternalMessageOnly, Provider: openAIProvider})
}
gateway := cognitiveGateway{resolver: resolver, providers: providers, builtinLocal: builtinLocal, externalRecovery: externalRecovery, now: time.Now, allowCanary: os.Getenv("COGNITIVE_ALLOW_CANARY") == "true", canaryMaxOutputBytes: 4096, admission: newHTTPAdmissionClient(envOr("CORE_RUNTIME_URL", "http://core-runtime:8080"), &http.Client{Timeout: 5 * time.Second})}
discovery := discoveryService{resolver: resolver, providers: discoveries, owner: identity.PrincipalID(envOr("COGNITIVE_OWNER_PRINCIPAL_ID", "domovoy-creator-principal")), now: time.Now}
gateway := cognitiveGateway{resolver: resolver, providers: providers, builtinLocal: builtinLocal, externalRecovery: externalRecovery, now: time.Now, allowCanary: os.Getenv("COGNITIVE_ALLOW_CANARY") == "true", canaryMaxOutputBytes: 4096, admission: newHTTPAdmissionClient(envOr("CORE_RUNTIME_URL", "http://core-runtime:8080"), &http.Client{Timeout: 5 * time.Second}), revalidatePrimary: func(ctx context.Context) error {
_, err := discovery.register(ctx, "medium")
return err
}}
mux.Handle("/healthz", health.Handler{Runtime: "cognitive-worker", SemanticGeneration: 1, ArchitectureGeneration: 1})
readinessClient := &http.Client{Timeout: 2 * time.Second}
cpuReadinessClient := &http.Client{Timeout: 10 * time.Second}
mediumReadinessTimeout := 120 * time.Second
mediumReadinessTimeout := 120 * time.Second
mediumReadinessClient := &http.Client{Timeout: mediumReadinessTimeout}
lifeboatURL := strings.TrimSuffix(strings.TrimRight(envOr("LOCAL_LIFEBOAT_PROVIDER_URL", "http://cognitive-lifeboat:8086/v1"), "/"), "/v1") + "/health"
fleetClient := &http.Client{Timeout: 2 * time.Second}
@@ -199,7 +203,6 @@ func main() {
mux.HandleFunc("POST /v1/multimodal/ocr", multimodalGateway.handler(capability.MultimodalRecognizeOCR))
mux.HandleFunc("POST /v1/multimodal/video-understand", multimodalGateway.handler(capability.MultimodalUnderstandVideo))
mux.HandleFunc("POST /v1/multimodal/generate-image", multimodalGateway.handler(capability.MultimodalGenerateImage))
discovery := discoveryService{resolver: resolver, providers: discoveries, owner: identity.PrincipalID(envOr("COGNITIVE_OWNER_PRINCIPAL_ID", "domovoy-creator-principal")), now: time.Now}
mux.HandleFunc("POST /v1/provider-discovery/{provider}", discovery.handler)
mux.Handle("POST /v1/provider-discovery/{provider}/qualify", modelQualificationHandler(discoveries, string(envOr("COGNITIVE_OWNER_PRINCIPAL_ID", "domovoy-creator-principal")), httpQualificationObserver{resolver: resolver}))
mux.HandleFunc("GET /v1/provider-discovery/status", discovery.statusHandler)