feat: keep Domovoy cognitive recovery active
CI / test (push) Failing after 15s

This commit is contained in:
m
2026-08-22 01:29:45 +03:00
parent 9f717d720e
commit 4f06720e3f
9 changed files with 159 additions and 13 deletions
+1 -1
View File
@@ -23,7 +23,7 @@ func (s discoveryService) handler(w http.ResponseWriter, r *http.Request) {
return
}
discovered, probe := registration.Discovery.Discover(r.Context())
snapshot := capability.CognitiveDiscoverySnapshot{ProviderInstanceID: registration.ProviderInstanceID, ProviderKind: registration.Kind, EndpointRef: registration.EndpointRef, CredentialRef: registration.CredentialRef, AuthMode: registration.AuthMode, BillingMode: registration.BillingMode, Enabled: true, OwnerPrincipalID: s.owner, CapabilityID: creatorAdvisoryCapabilityID, DataPolicy: capability.DataPolicyLocalOnly, LimitPolicyRef: "creator-advisory-local-v1", Health: probe.Health, FailureClass: capability.CognitiveFailureType(probe.Failure), ObservedAtUnixMillis: s.now().UnixMilli()}
snapshot := capability.CognitiveDiscoverySnapshot{ProviderInstanceID: registration.ProviderInstanceID, ProviderKind: registration.Kind, EndpointRef: registration.EndpointRef, CredentialRef: registration.CredentialRef, AuthMode: registration.AuthMode, BillingMode: registration.BillingMode, Enabled: true, OwnerPrincipalID: s.owner, CapabilityID: creatorAdvisoryCapabilityID, DataPolicy: registration.DataPolicy, LimitPolicyRef: "creator-advisory-local-v1", Health: probe.Health, FailureClass: capability.CognitiveFailureType(probe.Failure), ObservedAtUnixMillis: s.now().UnixMilli()}
for _, model := range discovered {
snapshot.Models = append(snapshot.Models, capability.DiscoveredModelSnapshot{ProviderModelID: model.ProviderModelID, Fingerprint: model.Fingerprint, MaxContextBytes: model.MaxContextBytes, Modalities: model.Modalities, Capabilities: model.Capabilities, Installed: capability.OperationalYes, Loaded: capability.OperationalNotObserved, Invokable: capability.OperationalNotObserved})
}
+43 -3
View File
@@ -32,6 +32,7 @@ type providerRegistration struct {
CredentialRef capability.CredentialRef
AuthMode capability.AuthMode
BillingMode capability.BillingMode
DataPolicy capability.DataPolicy
Provider advisoryProvider
Discovery models.LocalProvider
}
@@ -39,6 +40,8 @@ type providerRegistration struct {
type cognitiveGateway struct {
resolver bindingResolver
providers map[capability.ProviderKind][]providerRegistration
builtinLocal *providerRegistration
externalRecovery []providerRegistration
now func() time.Time
allowCanary bool
canaryMaxOutputBytes int
@@ -77,7 +80,7 @@ func (g cognitiveGateway) advisoryHandler(w http.ResponseWriter, r *http.Request
requirement := capability.CognitiveRequirement{CapabilityID: creatorAdvisoryCapabilityID, OwnerPrincipalID: request.PrincipalID, DataPolicy: request.DataPolicy, RequiredModalities: []capability.ModelModality{capability.ModalityText}, RequiredCapabilities: []capability.ModelCapability{capability.CapabilityAdvisory}, AllowCanary: g.allowCanary, InputBytes: int64(len(request.Message)), MaxOutputBytes: int64(maxOutputBytes), Objective: selectionObjectiveForMeaning(request.Meaning, request.Message)}
selected, err := g.resolver.Resolve(r.Context(), requirement)
if err != nil {
writeGatewayResponse(w, builtinAdvisory(request))
writeGatewayResponse(w, g.invokeBuiltinAdvisory(r.Context(), request, maxOutputBytes))
return
}
if selected.Binding.DataPolicy != capability.DataPolicyLocalOnly && !request.EgressAcknowledged {
@@ -86,7 +89,7 @@ func (g cognitiveGateway) advisoryHandler(w http.ResponseWriter, r *http.Request
}
provider := g.provider(selected)
if provider == nil {
writeGatewayResponse(w, builtinAdvisory(request))
writeGatewayResponse(w, g.invokeBuiltinAdvisory(r.Context(), request, maxOutputBytes))
return
}
invocation := cognitive.AdvisoryRequest{Version: request.Version, RequestID: request.RequestID, ActorID: request.ActorID, PrincipalID: request.PrincipalID, ScopeID: request.ScopeID, Message: request.Message, Context: request.Context, Locale: request.Locale, MaxOutputBytes: maxOutputBytes, ProviderBindingID: selected.Binding.ID, ProviderBindingRevision: selected.Binding.Revision, ProviderInstanceID: selected.Provider.ID, ModelDescriptorID: selected.Model.ID, ProviderModelID: selected.Model.ProviderModelID}
@@ -114,7 +117,7 @@ func (g cognitiveGateway) advisoryHandler(w http.ResponseWriter, r *http.Request
}
}
if response.FinishReason != cognitive.FinishCompleted && fallbackEligible(response.FinishReason) {
writeGatewayResponse(w, builtinAdvisory(request))
writeGatewayResponse(w, g.invokeBuiltinAdvisory(r.Context(), request, maxOutputBytes))
return
}
writeGatewayResponse(w, cognitive.GatewayAdvisoryResponse{Version: response.Version, RequestID: response.RequestID, AdvisoryText: response.AdvisoryText, ReferencedExcerptIDs: response.ReferencedExcerptIDs, FinishReason: response.FinishReason, Usage: response.Usage})
@@ -164,6 +167,43 @@ func builtinAdvisory(request cognitive.GatewayAdvisoryRequest) cognitive.Gateway
return cognitive.GatewayAdvisoryResponse{Version: request.Version, RequestID: request.RequestID, AdvisoryText: text, FinishReason: cognitive.FinishCompleted}
}
func (g cognitiveGateway) invokeBuiltinAdvisory(ctx context.Context, request cognitive.GatewayAdvisoryRequest, maxOutputBytes int) cognitive.GatewayAdvisoryResponse {
if g.builtinLocal != nil && g.builtinLocal.Provider != nil {
invocation := cognitive.AdvisoryRequest{Version: request.Version, RequestID: request.RequestID, ActorID: request.ActorID, PrincipalID: request.PrincipalID, ScopeID: request.ScopeID, Message: request.Message, Context: request.Context, Locale: request.Locale, MaxOutputBytes: maxOutputBytes, ProviderBindingID: "builtin-local-binding-v1", ProviderBindingRevision: 1, ProviderInstanceID: "builtin-local-lifeboat-v1", ModelDescriptorID: "builtin-local-model-v1", ProviderModelID: "domovoy-lifeboat"}
response, err := g.builtinLocal.Provider.Advise(ctx, invocation)
if err == nil && response.FinishReason == cognitive.FinishCompleted && strings.TrimSpace(response.AdvisoryText) != "" {
return cognitive.GatewayAdvisoryResponse{Version: request.Version, RequestID: request.RequestID, AdvisoryText: response.AdvisoryText, ReferencedExcerptIDs: response.ReferencedExcerptIDs, FinishReason: response.FinishReason, Usage: response.Usage}
}
}
if request.EgressAcknowledged {
if response, ok := g.recoverExternalAdvisory(ctx, request, maxOutputBytes); ok {
return response
}
}
return builtinAdvisory(request)
}
func (g cognitiveGateway) recoverExternalAdvisory(ctx context.Context, request cognitive.GatewayAdvisoryRequest, maxOutputBytes int) (cognitive.GatewayAdvisoryResponse, bool) {
for _, candidate := range g.externalRecovery {
if candidate.Provider == nil || candidate.Discovery == nil {
continue
}
probeCtx, cancel := context.WithTimeout(ctx, 8*time.Second)
modelsFound, probe := candidate.Discovery.Discover(probeCtx)
cancel()
if probe.Health != capability.HealthAvailable || len(modelsFound) == 0 {
continue
}
model := modelsFound[0]
invocation := cognitive.AdvisoryRequest{Version: request.Version, RequestID: request.RequestID, ActorID: request.ActorID, PrincipalID: request.PrincipalID, ScopeID: request.ScopeID, Message: request.Message, Context: request.Context, Locale: request.Locale, MaxOutputBytes: maxOutputBytes, ProviderBindingID: "recovery-external-binding-v1", ProviderBindingRevision: 1, ProviderInstanceID: capability.ProviderInstanceID("recovery-" + string(candidate.ProviderInstanceID)), ModelDescriptorID: capability.ModelDescriptorID("recovery-model-" + string(model.ProviderModelID)), ProviderModelID: model.ProviderModelID}
response, err := candidate.Provider.Advise(ctx, invocation)
if err == nil && response.FinishReason == cognitive.FinishCompleted && strings.TrimSpace(response.AdvisoryText) != "" {
return cognitive.GatewayAdvisoryResponse{Version: request.Version, RequestID: request.RequestID, AdvisoryText: response.AdvisoryText, ReferencedExcerptIDs: response.ReferencedExcerptIDs, FinishReason: response.FinishReason, Usage: response.Usage}, true
}
}
return cognitive.GatewayAdvisoryResponse{}, false
}
func (g cognitiveGateway) observe(ctx context.Context, selected capability.ResolvedCognitiveBinding, response cognitive.AdvisoryResponse, started time.Time) {
usage := response.Usage
observationID, idErr := ids.New()
@@ -168,6 +168,19 @@ func TestNoActiveBindingUsesBuiltInProviderWithoutInvocation(t *testing.T) {
}
}
func TestNoActiveBindingInvokesBuiltInLocalProvider(t *testing.T) {
registry := &registryStub{err: errors.New("no active binding")}
provider := &stubAdvisoryProvider{response: cognitive.AdvisoryResponse{Version: cognitive.CreatorAdvisoryV1, RequestID: "request-1", AdvisoryText: "локальный ответ", FinishReason: cognitive.FinishCompleted}}
builtin := providerRegistration{ProviderInstanceID: "local-lifeboat", Kind: "openai-compatible-chat-v1", EndpointRef: "local-lifeboat-v1", Provider: provider}
gateway := cognitiveGateway{resolver: registry, builtinLocal: &builtin, now: time.Now}
request := httptest.NewRequest(http.MethodPost, "/v1/advisories", strings.NewReader(gatewayRequest()))
response := httptest.NewRecorder()
gateway.advisoryHandler(response, request)
if response.Code != http.StatusOK || !provider.called || !strings.Contains(response.Body.String(), "локальный ответ") {
t.Fatalf("status=%d provider_called=%t body=%s", response.Code, provider.called, response.Body.String())
}
}
func TestProviderLossBetweenRoutingAndInvocationFallsBackOnce(t *testing.T) {
primary := gatewaySelection()
primary.Binding.ID = "gpu-binding"
+19 -2
View File
@@ -1,6 +1,7 @@
package main
import (
"context"
"encoding/json"
"fmt"
"net/http"
@@ -36,12 +37,12 @@ func (g cognitiveGateway) interpretationHandler(w http.ResponseWriter, r *http.R
}
selected, err := g.resolver.Resolve(r.Context(), requirement)
if err != nil {
writeInterpretationResponse(w, cognitive.GatewayInterpretationResponse{Version: request.Version, RequestID: request.RequestID, FinishReason: cognitive.FinishProviderUnavailable})
writeInterpretationResponse(w, g.interpretWithBuiltin(r.Context(), request, maxOutputBytes))
return
}
provider := g.provider(selected)
if provider == nil {
writeInterpretationResponse(w, cognitive.GatewayInterpretationResponse{Version: request.Version, RequestID: request.RequestID, FinishReason: cognitive.FinishProviderUnavailable})
writeInterpretationResponse(w, g.interpretWithBuiltin(r.Context(), request, maxOutputBytes))
return
}
invocation := cognitive.AdvisoryRequest{
@@ -77,6 +78,22 @@ func (g cognitiveGateway) interpretationHandler(w http.ResponseWriter, r *http.R
writeInterpretationResponse(w, cognitive.GatewayInterpretationResponse{Version: request.Version, RequestID: request.RequestID, Candidate: candidate, FinishReason: cognitive.FinishCompleted, Usage: response.Usage})
}
func (g cognitiveGateway) interpretWithBuiltin(ctx context.Context, request cognitive.GatewayInterpretationRequest, maxOutputBytes int) cognitive.GatewayInterpretationResponse {
if g.builtinLocal == nil || g.builtinLocal.Provider == nil {
return cognitive.GatewayInterpretationResponse{Version: request.Version, RequestID: request.RequestID, FinishReason: cognitive.FinishProviderUnavailable}
}
invocation := cognitive.AdvisoryRequest{Version: request.Version, RequestID: request.RequestID, ActorID: request.ActorID, PrincipalID: request.PrincipalID, ScopeID: request.ScopeID, Message: interpretationPrompt(request), Locale: request.Locale, MaxOutputBytes: maxOutputBytes, ProviderBindingID: "builtin-local-binding-v1", ProviderBindingRevision: 1, ProviderInstanceID: "builtin-local-lifeboat-v1", ModelDescriptorID: "builtin-local-model-v1", ProviderModelID: "domovoy-lifeboat"}
response, err := g.builtinLocal.Provider.Advise(ctx, invocation)
if err != nil || response.FinishReason != cognitive.FinishCompleted {
return cognitive.GatewayInterpretationResponse{Version: request.Version, RequestID: request.RequestID, FinishReason: cognitive.FinishProviderUnavailable}
}
candidate, ok := parseInterpretationCandidate(response.AdvisoryText)
if !ok {
return cognitive.GatewayInterpretationResponse{Version: request.Version, RequestID: request.RequestID, FinishReason: cognitive.FinishInvalidStructuredOutput, Usage: response.Usage}
}
return cognitive.GatewayInterpretationResponse{Version: request.Version, RequestID: request.RequestID, Candidate: candidate, FinishReason: cognitive.FinishCompleted, Usage: response.Usage}
}
func selectionObjectiveForInterpretation(request cognitive.GatewayInterpretationRequest) capability.SelectionObjective {
if request.RequestedOutcome != "" || request.DeterministicApplicability == cognitive.InterpretationNeedsBroader || request.DeterministicApplicability == cognitive.InterpretationMixedIntent || request.DeterministicApplicability == cognitive.InterpretationLowConfidence {
return capability.SelectionQuality
+10 -5
View File
@@ -40,31 +40,36 @@ func main() {
}
providers := map[capability.ProviderKind][]providerRegistration{}
discoveries := map[string]providerRegistration{}
var builtinLocal *providerRegistration
externalRecovery := []providerRegistration{}
cpuAdapter, err := models.NewOpenAICompatibleLocalAdapter(envOr("LOCAL_CPU_PROVIDER_URL", "http://docker.cin.su:18082/v1"), nil)
if err != nil {
log.Fatal(err)
}
cpu := providerRegistration{ProviderInstanceID: "local-cpu-primary", Kind: "openai-compatible-chat-v1", EndpointRef: "local-cpu-primary-v1", AuthMode: capability.AuthNone, BillingMode: capability.BillingFree, Provider: cpuAdapter, Discovery: cpuAdapter}
cpu := providerRegistration{ProviderInstanceID: "local-cpu-primary", Kind: "openai-compatible-chat-v1", EndpointRef: "local-cpu-primary-v1", AuthMode: capability.AuthNone, BillingMode: capability.BillingFree, DataPolicy: capability.DataPolicyLocalOnly, Provider: cpuAdapter, Discovery: cpuAdapter}
providers[cpu.Kind] = append(providers[cpu.Kind], cpu)
discoveries["cpu"] = cpu
if lifeboat, enabled, lifeboatErr := newLifeboatProvider(os.Getenv("LOCAL_LIFEBOAT_PROVIDER_URL")); lifeboatErr != nil {
log.Fatal(lifeboatErr)
} else if enabled {
lifeboat.DataPolicy = capability.DataPolicyLocalOnly
providers[lifeboat.Kind] = append(providers[lifeboat.Kind], lifeboat)
discoveries["lifeboat"] = lifeboat
builtinLocal = &lifeboat
}
gpuAdapter, err := models.NewOllamaAdapter(envOr("LOCAL_GPU_PROVIDER_URL", "http://192.168.220.91:11434"), nil)
if err != nil {
log.Fatal(err)
}
gpu := providerRegistration{ProviderInstanceID: "local-gpu-opportunistic", Kind: "ollama-v1", EndpointRef: "local-gpu-opportunistic-v1", AuthMode: capability.AuthNone, BillingMode: capability.BillingFree, Provider: gpuAdapter, Discovery: gpuAdapter}
gpu := providerRegistration{ProviderInstanceID: "local-gpu-opportunistic", Kind: "ollama-v1", EndpointRef: "local-gpu-opportunistic-v1", AuthMode: capability.AuthNone, BillingMode: capability.BillingFree, DataPolicy: capability.DataPolicyExternalMessageOnly, Provider: gpuAdapter, Discovery: gpuAdapter}
providers[gpu.Kind] = append(providers[gpu.Kind], gpu)
discoveries["gpu"] = gpu
externalRecovery = append(externalRecovery, gpu)
if openAIProvider != nil {
kind := capability.ProviderKind(envOr("OPENAI_PROVIDER_KIND", "openai-responses-v1"))
providers[kind] = append(providers[kind], providerRegistration{ProviderInstanceID: "openai-bootstrap", Kind: kind, EndpointRef: capability.EndpointBindingRef(envOr("OPENAI_ENDPOINT_REF", "openai-responses-default")), CredentialRef: capability.CredentialRef(envOr("OPENAI_CREDENTIAL_REF", "openai-advisory-v1")), AuthMode: capability.AuthKey, BillingMode: capability.BillingPerInput, Provider: openAIProvider})
providers[kind] = append(providers[kind], providerRegistration{ProviderInstanceID: "openai-bootstrap", Kind: kind, EndpointRef: capability.EndpointBindingRef(envOr("OPENAI_ENDPOINT_REF", "openai-responses-default")), CredentialRef: capability.CredentialRef(envOr("OPENAI_CREDENTIAL_REF", "openai-advisory-v1")), AuthMode: capability.AuthKey, BillingMode: capability.BillingPerInput, DataPolicy: capability.DataPolicyExternalMessageOnly, Provider: openAIProvider})
}
gateway := cognitiveGateway{resolver: resolver, providers: providers, now: time.Now, allowCanary: os.Getenv("COGNITIVE_ALLOW_CANARY") == "true", canaryMaxOutputBytes: 4096}
gateway := cognitiveGateway{resolver: resolver, providers: providers, builtinLocal: builtinLocal, externalRecovery: externalRecovery, now: time.Now, allowCanary: os.Getenv("COGNITIVE_ALLOW_CANARY") == "true", canaryMaxOutputBytes: 4096}
mux.Handle("/healthz", health.Handler{Runtime: "cognitive-worker", SemanticGeneration: 1, ArchitectureGeneration: 1})
mux.HandleFunc("POST /v1/proposals/deterministic", deterministicProposal)
mux.HandleFunc("POST /v1/advisory-policy", gateway.policyHandler)
@@ -99,7 +104,7 @@ func refreshProviderDiscoveries(resolver bindingResolver, service discoveryServi
Enabled: true,
OwnerPrincipalID: service.owner,
CapabilityID: creatorAdvisoryCapabilityID,
DataPolicy: capability.DataPolicyLocalOnly,
DataPolicy: registration.DataPolicy,
LimitPolicyRef: "creator-advisory-local-v1",
Health: probe.Health,
FailureClass: capability.CognitiveFailureType(probe.Failure),