diff --git a/cmd/cognitive-worker/admission_test.go b/cmd/cognitive-worker/admission_test.go index f58a104..307b42e 100644 --- a/cmd/cognitive-worker/admission_test.go +++ b/cmd/cognitive-worker/admission_test.go @@ -16,15 +16,17 @@ func (p *admissionTestProvider) Advise(context.Context, cognitive.AdvisoryReques } type admissionTestClient struct { - decision capability.AdmissionDecision - releases int + decision capability.AdmissionDecision + releases int + releaseContextErr error } func (c *admissionTestClient) Admit(context.Context, string, capability.AdmissionRequest) (capability.AdmissionDecision, capability.AdmissionLease, error) { return c.decision, capability.AdmissionLease{ID: "lease-1", PolicyID: "policy-1"}, nil } -func (c *admissionTestClient) Release(context.Context, capability.AdmissionLease) error { +func (c *admissionTestClient) Release(ctx context.Context, _ capability.AdmissionLease) error { c.releases++ + c.releaseContextErr = ctx.Err() return nil } @@ -47,3 +49,14 @@ func TestInvokeAdmittedReleasesDurableLeaseAfterProvider(t *testing.T) { t.Fatalf("admitted lease was not released: response=%#v err=%v calls=%d releases=%d", response, err, provider.calls, client.releases) } } + +func TestInvokeAdmittedReleasesLeaseAfterCallerCancellation(t *testing.T) { + provider := &admissionTestProvider{} + client := &admissionTestClient{decision: capability.AdmissionDecision{State: capability.AdmissionAdmitted, Reason: capability.AdmissionReasonNone, PolicyID: "policy-1"}} + ctx, cancel := context.WithCancel(context.Background()) + cancel() + _, _ = (&cognitiveGateway{admission: client}).invokeAdmitted(ctx, "request-1", provider, cognitive.AdvisoryRequest{Message: "hello"}, 100) + if client.releases != 1 || client.releaseContextErr != nil { + t.Fatalf("cancelled caller leaked durable lease: releases=%d release_context_err=%v", client.releases, client.releaseContextErr) + } +} diff --git a/cmd/cognitive-worker/discovery.go b/cmd/cognitive-worker/discovery.go index c873e35..29ffafc 100644 --- a/cmd/cognitive-worker/discovery.go +++ b/cmd/cognitive-worker/discovery.go @@ -1,7 +1,9 @@ package main import ( + "context" "encoding/json" + "fmt" "net/http" "sort" "time" @@ -22,17 +24,7 @@ func (s discoveryService) handler(w http.ResponseWriter, r *http.Request) { s.statusHandler(w, r) return } - registration, ok := s.providers[r.PathValue("provider")] - if !ok || registration.Discovery == nil { - http.Error(w, "provider discovery not found", http.StatusNotFound) - return - } - discovered, probe := registration.Discovery.Discover(r.Context()) - snapshot := capability.CognitiveDiscoverySnapshot{ProviderInstanceID: registration.ProviderInstanceID, ProviderKind: registration.Kind, EndpointRef: registration.EndpointRef, CredentialRef: registration.CredentialRef, AuthMode: registration.AuthMode, BillingMode: registration.BillingMode, Enabled: true, OwnerPrincipalID: s.owner, CapabilityID: creatorAdvisoryCapabilityID, DataPolicy: registration.DataPolicy, LimitPolicyRef: "creator-advisory-local-v1", Health: probe.Health, FailureClass: capability.CognitiveFailureType(probe.Failure), ObservedAtUnixMillis: s.now().UnixMilli()} - for _, model := range discovered { - snapshot.Models = append(snapshot.Models, capability.DiscoveredModelSnapshot{ProviderModelID: model.ProviderModelID, Fingerprint: model.Fingerprint, MaxContextBytes: model.MaxContextBytes, Modalities: model.Modalities, Capabilities: model.Capabilities, Installed: capability.OperationalYes, Loaded: capability.OperationalNotObserved, Invokable: capability.OperationalNotObserved}) - } - result, err := s.resolver.RegisterDiscovery(r.Context(), snapshot) + result, err := s.register(r.Context(), r.PathValue("provider")) if err != nil { http.Error(w, "provider discovery registration unavailable", http.StatusBadGateway) return @@ -41,6 +33,19 @@ func (s discoveryService) handler(w http.ResponseWriter, r *http.Request) { _ = json.NewEncoder(w).Encode(result) } +func (s discoveryService) register(ctx context.Context, provider string) (capability.CognitiveDiscoveryRegistration, error) { + registration, ok := s.providers[provider] + if !ok || registration.Discovery == nil { + return capability.CognitiveDiscoveryRegistration{}, fmt.Errorf("provider discovery not found") + } + discovered, probe := registration.Discovery.Discover(ctx) + snapshot := capability.CognitiveDiscoverySnapshot{ProviderInstanceID: registration.ProviderInstanceID, ProviderKind: registration.Kind, EndpointRef: registration.EndpointRef, CredentialRef: registration.CredentialRef, AuthMode: registration.AuthMode, BillingMode: registration.BillingMode, Enabled: true, OwnerPrincipalID: s.owner, CapabilityID: creatorAdvisoryCapabilityID, DataPolicy: registration.DataPolicy, LimitPolicyRef: "creator-advisory-local-v1", Health: probe.Health, FailureClass: capability.CognitiveFailureType(probe.Failure), ObservedAtUnixMillis: s.now().UnixMilli()} + for _, model := range discovered { + snapshot.Models = append(snapshot.Models, capability.DiscoveredModelSnapshot{ProviderModelID: model.ProviderModelID, Fingerprint: model.Fingerprint, MaxContextBytes: model.MaxContextBytes, Modalities: model.Modalities, Capabilities: model.Capabilities, Installed: capability.OperationalYes, Loaded: capability.OperationalNotObserved, Invokable: capability.OperationalNotObserved}) + } + return s.resolver.RegisterDiscovery(ctx, snapshot) +} + type discoveryStatus struct { Provider string `json:"provider"` ProviderKind capability.ProviderKind `json:"provider_kind"` diff --git a/cmd/cognitive-worker/gateway.go b/cmd/cognitive-worker/gateway.go index ddf409b..8f8715d 100644 --- a/cmd/cognitive-worker/gateway.go +++ b/cmd/cognitive-worker/gateway.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "encoding/json" + "errors" "fmt" "io" "net/http" @@ -20,6 +21,8 @@ import ( const creatorAdvisoryCapabilityID capability.CapabilityID = "creator-advisory" +var errNoInvocableCognitiveBinding = errors.New("no invocable cognitive binding") + type bindingResolver interface { Resolve(context.Context, capability.CognitiveRequirement) (capability.ResolvedCognitiveBinding, error) Observe(context.Context, capability.CognitiveUsageObservation) error @@ -56,6 +59,7 @@ type cognitiveGateway struct { allowCanary bool canaryMaxOutputBytes int admission admissionClient + revalidatePrimary func(context.Context) error } func (g cognitiveGateway) policyHandler(w http.ResponseWriter, r *http.Request) { @@ -89,7 +93,7 @@ func (g cognitiveGateway) advisoryHandler(w http.ResponseWriter, r *http.Request maxOutputBytes = g.canaryMaxOutputBytes } requirement := capability.CognitiveRequirement{CapabilityID: creatorAdvisoryCapabilityID, OwnerPrincipalID: request.PrincipalID, DataPolicy: request.DataPolicy, RequiredModalities: []capability.ModelModality{capability.ModalityText}, RequiredCapabilities: []capability.ModelCapability{capability.CapabilityAdvisory}, AllowCanary: g.allowCanary, InputBytes: int64(len(request.Message)), MaxOutputBytes: int64(maxOutputBytes), Objective: selectionObjectiveForMeaning(request.Meaning, request.Message)} - selected, err := g.resolver.Resolve(r.Context(), requirement) + selected, err := g.resolveWithRevalidation(r.Context(), requirement) if err != nil { writeGatewayResponse(w, g.invokeBuiltinAdvisory(r.Context(), request, maxOutputBytes)) return @@ -145,11 +149,33 @@ func (g cognitiveGateway) invokeAdmitted(ctx context.Context, leaseID string, pr return cognitive.AdvisoryResponse{Version: invocation.Version, RequestID: invocation.RequestID, ProviderBindingID: invocation.ProviderBindingID, ProviderBindingRevision: invocation.ProviderBindingRevision, FinishReason: cognitive.FinishRateLimited}, nil } lease = admittedLease - defer func() { _ = g.admission.Release(ctx, lease) }() + defer func() { + releaseCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second) + defer cancel() + _ = g.admission.Release(releaseCtx, lease) + }() } return provider.Advise(ctx, invocation) } +// resolveWithRevalidation makes one bounded recovery attempt when Capability +// has an active/validated binding but its durable health projection rejected it +// as non-invocable. Discovery remains the canonical Capability write path; the +// retry observes the configured primary local provider and re-runs selection +// once. Provider invocation, admission, and fallback remain governed elsewhere. +func (g cognitiveGateway) resolveWithRevalidation(ctx context.Context, requirement capability.CognitiveRequirement) (capability.ResolvedCognitiveBinding, error) { + selected, err := g.resolver.Resolve(ctx, requirement) + if err == nil || !errors.Is(err, errNoInvocableCognitiveBinding) || g.revalidatePrimary == nil { + return selected, err + } + revalidationCtx, cancel := context.WithTimeout(ctx, 45*time.Second) + defer cancel() + if revalidationErr := g.revalidatePrimary(revalidationCtx); revalidationErr != nil { + return selected, err + } + return g.resolver.Resolve(ctx, requirement) +} + func selectionObjectiveForMessage(message string) capability.SelectionObjective { lower := strings.ToLower(message) if strings.Contains(lower, "дешев") || strings.Contains(lower, "бесплат") || strings.Contains(lower, "эконом") || strings.Contains(lower, "cost") || strings.Contains(lower, "cheap") { @@ -374,6 +400,10 @@ type httpBindingResolver struct { client *http.Client } +type httpStatusError struct{ status int } + +func (e httpStatusError) Error() string { return fmt.Sprintf("registry status %d", e.status) } + func newHTTPBindingResolver(raw string, client *http.Client) (*httpBindingResolver, error) { base, err := url.Parse(raw) if err != nil || base.Scheme == "" || base.Host == "" { @@ -385,6 +415,10 @@ func newHTTPBindingResolver(raw string, client *http.Client) (*httpBindingResolv func (r *httpBindingResolver) Resolve(ctx context.Context, value capability.CognitiveRequirement) (capability.ResolvedCognitiveBinding, error) { var result capability.ResolvedCognitiveBinding err := r.post(ctx, "/v1/cognitive-bindings/resolve", value, &result) + var statusErr httpStatusError + if errors.As(err, &statusErr) && statusErr.status == http.StatusNotFound { + return result, errNoInvocableCognitiveBinding + } return result, err } @@ -437,7 +471,7 @@ func (r *httpBindingResolver) post(ctx context.Context, path string, value, dest defer response.Body.Close() if response.StatusCode < 200 || response.StatusCode >= 300 { _, _ = io.Copy(io.Discard, io.LimitReader(response.Body, 4096)) - return fmt.Errorf("registry status %d", response.StatusCode) + return httpStatusError{status: response.StatusCode} } if destination != nil { return json.NewDecoder(io.LimitReader(response.Body, 64<<10)).Decode(destination) diff --git a/cmd/cognitive-worker/gateway_registry_test.go b/cmd/cognitive-worker/gateway_registry_test.go index 911883e..a87c6f2 100644 --- a/cmd/cognitive-worker/gateway_registry_test.go +++ b/cmd/cognitive-worker/gateway_registry_test.go @@ -22,6 +22,26 @@ type registryStub struct { observation *capability.CognitiveUsageObservation } +type revalidatingRegistry struct { + selected capability.ResolvedCognitiveBinding + calls int + alwaysNo bool +} + +func (s *revalidatingRegistry) Resolve(context.Context, capability.CognitiveRequirement) (capability.ResolvedCognitiveBinding, error) { + s.calls++ + if s.alwaysNo || s.calls == 1 { + return capability.ResolvedCognitiveBinding{}, errNoInvocableCognitiveBinding + } + return s.selected, nil +} +func (*revalidatingRegistry) Observe(context.Context, capability.CognitiveUsageObservation) error { + return nil +} +func (*revalidatingRegistry) RegisterDiscovery(context.Context, capability.CognitiveDiscoverySnapshot) (capability.CognitiveDiscoveryRegistration, error) { + return capability.CognitiveDiscoveryRegistration{}, nil +} + type failoverRegistry struct { primary, fallback capability.ResolvedCognitiveBinding observations []capability.CognitiveUsageObservation @@ -88,6 +108,59 @@ func TestActiveRegistryBindingReachesProviderAdapter(t *testing.T) { } } +func TestStaleUnavailableBindingRevalidatesOnceBeforeInvocation(t *testing.T) { + selected := gatewaySelection() + selected.Binding.DataPolicy = capability.DataPolicyLocalOnly + registry := &revalidatingRegistry{selected: selected} + provider := &stubAdvisoryProvider{response: cognitive.AdvisoryResponse{Version: cognitive.CreatorAdvisoryV1, RequestID: "request-1", AdvisoryText: "ok", FinishReason: cognitive.FinishCompleted}} + revalidations := 0 + gateway := cognitiveGateway{resolver: registry, providers: map[capability.ProviderKind][]providerRegistration{"provider-kind-1": {{EndpointRef: "endpoint-1", CredentialRef: "credential-1", Provider: provider}}}, now: time.Now, revalidatePrimary: func(context.Context) error { revalidations++; return nil }} + response := httptest.NewRecorder() + gateway.advisoryHandler(response, httptest.NewRequest(http.MethodPost, "/v1/advisories", strings.NewReader(gatewayRequest()))) + if response.Code != http.StatusOK || !provider.called || revalidations != 1 || registry.calls != 2 { + t.Fatalf("status=%d provider_called=%t revalidations=%d resolves=%d", response.Code, provider.called, revalidations, registry.calls) + } +} + +func TestRevalidationFailureDoesNotMakeBindingInvocable(t *testing.T) { + registry := ®istryStub{err: errNoInvocableCognitiveBinding} + provider := &stubAdvisoryProvider{} + revalidations := 0 + gateway := cognitiveGateway{resolver: registry, providers: map[capability.ProviderKind][]providerRegistration{"provider-kind-1": {{Provider: provider}}}, now: time.Now, revalidatePrimary: func(context.Context) error { revalidations++; return errors.New("provider unavailable") }} + gateway.advisoryHandler(httptest.NewRecorder(), httptest.NewRequest(http.MethodPost, "/v1/advisories", strings.NewReader(gatewayRequest()))) + if provider.called || revalidations != 1 { + t.Fatalf("failed provider revalidation was treated as invocable: provider_called=%t revalidations=%d", provider.called, revalidations) + } +} + +func TestRevalidationIsNotRetriedInALoop(t *testing.T) { + registry := &revalidatingRegistry{alwaysNo: true} + revalidations := 0 + gateway := cognitiveGateway{resolver: registry, now: time.Now, revalidatePrimary: func(context.Context) error { revalidations++; return nil }} + gateway.advisoryHandler(httptest.NewRecorder(), httptest.NewRequest(http.MethodPost, "/v1/advisories", strings.NewReader(gatewayRequest()))) + if revalidations != 1 || registry.calls != 2 { + t.Fatalf("revalidation loop: revalidations=%d resolves=%d", revalidations, registry.calls) + } +} + +func TestHTTPResolverMarksNoInvocableBindingForSingleRevalidation(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/v1/cognitive-bindings/resolve" { + t.Fatalf("path=%s", r.URL.Path) + } + http.Error(w, "no invocable cognitive binding", http.StatusNotFound) + })) + defer server.Close() + resolver, err := newHTTPBindingResolver(server.URL, server.Client()) + if err != nil { + t.Fatal(err) + } + _, err = resolver.Resolve(context.Background(), capability.CognitiveRequirement{CapabilityID: creatorAdvisoryCapabilityID, OwnerPrincipalID: "principal-1", MaxOutputBytes: 1}) + if !errors.Is(err, errNoInvocableCognitiveBinding) { + t.Fatalf("resolve error=%v, want no-invocable sentinel", err) + } +} + func TestProviderUsesStableInstanceWhenCredentialMetadataIsStale(t *testing.T) { provider := &stubAdvisoryProvider{} gateway := cognitiveGateway{providers: map[capability.ProviderKind][]providerRegistration{ diff --git a/cmd/cognitive-worker/main.go b/cmd/cognitive-worker/main.go index 4ddd25b..feb63ed 100644 --- a/cmd/cognitive-worker/main.go +++ b/cmd/cognitive-worker/main.go @@ -113,11 +113,15 @@ func main() { kind := capability.ProviderKind(envOr("OPENAI_PROVIDER_KIND", "openai-responses-v1")) providers[kind] = append(providers[kind], providerRegistration{ProviderInstanceID: "openai-bootstrap", Kind: kind, EndpointRef: capability.EndpointBindingRef(envOr("OPENAI_ENDPOINT_REF", "openai-responses-default")), CredentialRef: capability.CredentialRef(envOr("OPENAI_CREDENTIAL_REF", "openai-advisory-v1")), AuthMode: capability.AuthKey, BillingMode: capability.BillingPerInput, DataPolicy: capability.DataPolicyExternalMessageOnly, Provider: openAIProvider}) } - gateway := cognitiveGateway{resolver: resolver, providers: providers, builtinLocal: builtinLocal, externalRecovery: externalRecovery, now: time.Now, allowCanary: os.Getenv("COGNITIVE_ALLOW_CANARY") == "true", canaryMaxOutputBytes: 4096, admission: newHTTPAdmissionClient(envOr("CORE_RUNTIME_URL", "http://core-runtime:8080"), &http.Client{Timeout: 5 * time.Second})} + discovery := discoveryService{resolver: resolver, providers: discoveries, owner: identity.PrincipalID(envOr("COGNITIVE_OWNER_PRINCIPAL_ID", "domovoy-creator-principal")), now: time.Now} + gateway := cognitiveGateway{resolver: resolver, providers: providers, builtinLocal: builtinLocal, externalRecovery: externalRecovery, now: time.Now, allowCanary: os.Getenv("COGNITIVE_ALLOW_CANARY") == "true", canaryMaxOutputBytes: 4096, admission: newHTTPAdmissionClient(envOr("CORE_RUNTIME_URL", "http://core-runtime:8080"), &http.Client{Timeout: 5 * time.Second}), revalidatePrimary: func(ctx context.Context) error { + _, err := discovery.register(ctx, "medium") + return err + }} mux.Handle("/healthz", health.Handler{Runtime: "cognitive-worker", SemanticGeneration: 1, ArchitectureGeneration: 1}) readinessClient := &http.Client{Timeout: 2 * time.Second} cpuReadinessClient := &http.Client{Timeout: 10 * time.Second} - mediumReadinessTimeout := 120 * time.Second + mediumReadinessTimeout := 120 * time.Second mediumReadinessClient := &http.Client{Timeout: mediumReadinessTimeout} lifeboatURL := strings.TrimSuffix(strings.TrimRight(envOr("LOCAL_LIFEBOAT_PROVIDER_URL", "http://cognitive-lifeboat:8086/v1"), "/"), "/v1") + "/health" fleetClient := &http.Client{Timeout: 2 * time.Second} @@ -199,7 +203,6 @@ func main() { mux.HandleFunc("POST /v1/multimodal/ocr", multimodalGateway.handler(capability.MultimodalRecognizeOCR)) mux.HandleFunc("POST /v1/multimodal/video-understand", multimodalGateway.handler(capability.MultimodalUnderstandVideo)) mux.HandleFunc("POST /v1/multimodal/generate-image", multimodalGateway.handler(capability.MultimodalGenerateImage)) - discovery := discoveryService{resolver: resolver, providers: discoveries, owner: identity.PrincipalID(envOr("COGNITIVE_OWNER_PRINCIPAL_ID", "domovoy-creator-principal")), now: time.Now} mux.HandleFunc("POST /v1/provider-discovery/{provider}", discovery.handler) mux.Handle("POST /v1/provider-discovery/{provider}/qualify", modelQualificationHandler(discoveries, string(envOr("COGNITIVE_OWNER_PRINCIPAL_ID", "domovoy-creator-principal")), httpQualificationObserver{resolver: resolver})) mux.HandleFunc("GET /v1/provider-discovery/status", discovery.statusHandler)