This commit is contained in:
@@ -23,7 +23,7 @@ func (s discoveryService) handler(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
discovered, probe := registration.Discovery.Discover(r.Context())
|
||||
snapshot := capability.CognitiveDiscoverySnapshot{ProviderInstanceID: registration.ProviderInstanceID, ProviderKind: registration.Kind, EndpointRef: registration.EndpointRef, CredentialRef: registration.CredentialRef, AuthMode: registration.AuthMode, BillingMode: registration.BillingMode, Enabled: true, OwnerPrincipalID: s.owner, CapabilityID: creatorAdvisoryCapabilityID, DataPolicy: capability.DataPolicyLocalOnly, LimitPolicyRef: "creator-advisory-local-v1", Health: probe.Health, FailureClass: capability.CognitiveFailureType(probe.Failure), ObservedAtUnixMillis: s.now().UnixMilli()}
|
||||
snapshot := capability.CognitiveDiscoverySnapshot{ProviderInstanceID: registration.ProviderInstanceID, ProviderKind: registration.Kind, EndpointRef: registration.EndpointRef, CredentialRef: registration.CredentialRef, AuthMode: registration.AuthMode, BillingMode: registration.BillingMode, Enabled: true, OwnerPrincipalID: s.owner, CapabilityID: creatorAdvisoryCapabilityID, DataPolicy: registration.DataPolicy, LimitPolicyRef: "creator-advisory-local-v1", Health: probe.Health, FailureClass: capability.CognitiveFailureType(probe.Failure), ObservedAtUnixMillis: s.now().UnixMilli()}
|
||||
for _, model := range discovered {
|
||||
snapshot.Models = append(snapshot.Models, capability.DiscoveredModelSnapshot{ProviderModelID: model.ProviderModelID, Fingerprint: model.Fingerprint, MaxContextBytes: model.MaxContextBytes, Modalities: model.Modalities, Capabilities: model.Capabilities, Installed: capability.OperationalYes, Loaded: capability.OperationalNotObserved, Invokable: capability.OperationalNotObserved})
|
||||
}
|
||||
|
||||
@@ -32,6 +32,7 @@ type providerRegistration struct {
|
||||
CredentialRef capability.CredentialRef
|
||||
AuthMode capability.AuthMode
|
||||
BillingMode capability.BillingMode
|
||||
DataPolicy capability.DataPolicy
|
||||
Provider advisoryProvider
|
||||
Discovery models.LocalProvider
|
||||
}
|
||||
@@ -39,6 +40,8 @@ type providerRegistration struct {
|
||||
type cognitiveGateway struct {
|
||||
resolver bindingResolver
|
||||
providers map[capability.ProviderKind][]providerRegistration
|
||||
builtinLocal *providerRegistration
|
||||
externalRecovery []providerRegistration
|
||||
now func() time.Time
|
||||
allowCanary bool
|
||||
canaryMaxOutputBytes int
|
||||
@@ -77,7 +80,7 @@ func (g cognitiveGateway) advisoryHandler(w http.ResponseWriter, r *http.Request
|
||||
requirement := capability.CognitiveRequirement{CapabilityID: creatorAdvisoryCapabilityID, OwnerPrincipalID: request.PrincipalID, DataPolicy: request.DataPolicy, RequiredModalities: []capability.ModelModality{capability.ModalityText}, RequiredCapabilities: []capability.ModelCapability{capability.CapabilityAdvisory}, AllowCanary: g.allowCanary, InputBytes: int64(len(request.Message)), MaxOutputBytes: int64(maxOutputBytes), Objective: selectionObjectiveForMeaning(request.Meaning, request.Message)}
|
||||
selected, err := g.resolver.Resolve(r.Context(), requirement)
|
||||
if err != nil {
|
||||
writeGatewayResponse(w, builtinAdvisory(request))
|
||||
writeGatewayResponse(w, g.invokeBuiltinAdvisory(r.Context(), request, maxOutputBytes))
|
||||
return
|
||||
}
|
||||
if selected.Binding.DataPolicy != capability.DataPolicyLocalOnly && !request.EgressAcknowledged {
|
||||
@@ -86,7 +89,7 @@ func (g cognitiveGateway) advisoryHandler(w http.ResponseWriter, r *http.Request
|
||||
}
|
||||
provider := g.provider(selected)
|
||||
if provider == nil {
|
||||
writeGatewayResponse(w, builtinAdvisory(request))
|
||||
writeGatewayResponse(w, g.invokeBuiltinAdvisory(r.Context(), request, maxOutputBytes))
|
||||
return
|
||||
}
|
||||
invocation := cognitive.AdvisoryRequest{Version: request.Version, RequestID: request.RequestID, ActorID: request.ActorID, PrincipalID: request.PrincipalID, ScopeID: request.ScopeID, Message: request.Message, Context: request.Context, Locale: request.Locale, MaxOutputBytes: maxOutputBytes, ProviderBindingID: selected.Binding.ID, ProviderBindingRevision: selected.Binding.Revision, ProviderInstanceID: selected.Provider.ID, ModelDescriptorID: selected.Model.ID, ProviderModelID: selected.Model.ProviderModelID}
|
||||
@@ -114,7 +117,7 @@ func (g cognitiveGateway) advisoryHandler(w http.ResponseWriter, r *http.Request
|
||||
}
|
||||
}
|
||||
if response.FinishReason != cognitive.FinishCompleted && fallbackEligible(response.FinishReason) {
|
||||
writeGatewayResponse(w, builtinAdvisory(request))
|
||||
writeGatewayResponse(w, g.invokeBuiltinAdvisory(r.Context(), request, maxOutputBytes))
|
||||
return
|
||||
}
|
||||
writeGatewayResponse(w, cognitive.GatewayAdvisoryResponse{Version: response.Version, RequestID: response.RequestID, AdvisoryText: response.AdvisoryText, ReferencedExcerptIDs: response.ReferencedExcerptIDs, FinishReason: response.FinishReason, Usage: response.Usage})
|
||||
@@ -164,6 +167,43 @@ func builtinAdvisory(request cognitive.GatewayAdvisoryRequest) cognitive.Gateway
|
||||
return cognitive.GatewayAdvisoryResponse{Version: request.Version, RequestID: request.RequestID, AdvisoryText: text, FinishReason: cognitive.FinishCompleted}
|
||||
}
|
||||
|
||||
func (g cognitiveGateway) invokeBuiltinAdvisory(ctx context.Context, request cognitive.GatewayAdvisoryRequest, maxOutputBytes int) cognitive.GatewayAdvisoryResponse {
|
||||
if g.builtinLocal != nil && g.builtinLocal.Provider != nil {
|
||||
invocation := cognitive.AdvisoryRequest{Version: request.Version, RequestID: request.RequestID, ActorID: request.ActorID, PrincipalID: request.PrincipalID, ScopeID: request.ScopeID, Message: request.Message, Context: request.Context, Locale: request.Locale, MaxOutputBytes: maxOutputBytes, ProviderBindingID: "builtin-local-binding-v1", ProviderBindingRevision: 1, ProviderInstanceID: "builtin-local-lifeboat-v1", ModelDescriptorID: "builtin-local-model-v1", ProviderModelID: "domovoy-lifeboat"}
|
||||
response, err := g.builtinLocal.Provider.Advise(ctx, invocation)
|
||||
if err == nil && response.FinishReason == cognitive.FinishCompleted && strings.TrimSpace(response.AdvisoryText) != "" {
|
||||
return cognitive.GatewayAdvisoryResponse{Version: request.Version, RequestID: request.RequestID, AdvisoryText: response.AdvisoryText, ReferencedExcerptIDs: response.ReferencedExcerptIDs, FinishReason: response.FinishReason, Usage: response.Usage}
|
||||
}
|
||||
}
|
||||
if request.EgressAcknowledged {
|
||||
if response, ok := g.recoverExternalAdvisory(ctx, request, maxOutputBytes); ok {
|
||||
return response
|
||||
}
|
||||
}
|
||||
return builtinAdvisory(request)
|
||||
}
|
||||
|
||||
func (g cognitiveGateway) recoverExternalAdvisory(ctx context.Context, request cognitive.GatewayAdvisoryRequest, maxOutputBytes int) (cognitive.GatewayAdvisoryResponse, bool) {
|
||||
for _, candidate := range g.externalRecovery {
|
||||
if candidate.Provider == nil || candidate.Discovery == nil {
|
||||
continue
|
||||
}
|
||||
probeCtx, cancel := context.WithTimeout(ctx, 8*time.Second)
|
||||
modelsFound, probe := candidate.Discovery.Discover(probeCtx)
|
||||
cancel()
|
||||
if probe.Health != capability.HealthAvailable || len(modelsFound) == 0 {
|
||||
continue
|
||||
}
|
||||
model := modelsFound[0]
|
||||
invocation := cognitive.AdvisoryRequest{Version: request.Version, RequestID: request.RequestID, ActorID: request.ActorID, PrincipalID: request.PrincipalID, ScopeID: request.ScopeID, Message: request.Message, Context: request.Context, Locale: request.Locale, MaxOutputBytes: maxOutputBytes, ProviderBindingID: "recovery-external-binding-v1", ProviderBindingRevision: 1, ProviderInstanceID: capability.ProviderInstanceID("recovery-" + string(candidate.ProviderInstanceID)), ModelDescriptorID: capability.ModelDescriptorID("recovery-model-" + string(model.ProviderModelID)), ProviderModelID: model.ProviderModelID}
|
||||
response, err := candidate.Provider.Advise(ctx, invocation)
|
||||
if err == nil && response.FinishReason == cognitive.FinishCompleted && strings.TrimSpace(response.AdvisoryText) != "" {
|
||||
return cognitive.GatewayAdvisoryResponse{Version: request.Version, RequestID: request.RequestID, AdvisoryText: response.AdvisoryText, ReferencedExcerptIDs: response.ReferencedExcerptIDs, FinishReason: response.FinishReason, Usage: response.Usage}, true
|
||||
}
|
||||
}
|
||||
return cognitive.GatewayAdvisoryResponse{}, false
|
||||
}
|
||||
|
||||
func (g cognitiveGateway) observe(ctx context.Context, selected capability.ResolvedCognitiveBinding, response cognitive.AdvisoryResponse, started time.Time) {
|
||||
usage := response.Usage
|
||||
observationID, idErr := ids.New()
|
||||
|
||||
@@ -168,6 +168,19 @@ func TestNoActiveBindingUsesBuiltInProviderWithoutInvocation(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestNoActiveBindingInvokesBuiltInLocalProvider(t *testing.T) {
|
||||
registry := ®istryStub{err: errors.New("no active binding")}
|
||||
provider := &stubAdvisoryProvider{response: cognitive.AdvisoryResponse{Version: cognitive.CreatorAdvisoryV1, RequestID: "request-1", AdvisoryText: "локальный ответ", FinishReason: cognitive.FinishCompleted}}
|
||||
builtin := providerRegistration{ProviderInstanceID: "local-lifeboat", Kind: "openai-compatible-chat-v1", EndpointRef: "local-lifeboat-v1", Provider: provider}
|
||||
gateway := cognitiveGateway{resolver: registry, builtinLocal: &builtin, now: time.Now}
|
||||
request := httptest.NewRequest(http.MethodPost, "/v1/advisories", strings.NewReader(gatewayRequest()))
|
||||
response := httptest.NewRecorder()
|
||||
gateway.advisoryHandler(response, request)
|
||||
if response.Code != http.StatusOK || !provider.called || !strings.Contains(response.Body.String(), "локальный ответ") {
|
||||
t.Fatalf("status=%d provider_called=%t body=%s", response.Code, provider.called, response.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestProviderLossBetweenRoutingAndInvocationFallsBackOnce(t *testing.T) {
|
||||
primary := gatewaySelection()
|
||||
primary.Binding.ID = "gpu-binding"
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
@@ -36,12 +37,12 @@ func (g cognitiveGateway) interpretationHandler(w http.ResponseWriter, r *http.R
|
||||
}
|
||||
selected, err := g.resolver.Resolve(r.Context(), requirement)
|
||||
if err != nil {
|
||||
writeInterpretationResponse(w, cognitive.GatewayInterpretationResponse{Version: request.Version, RequestID: request.RequestID, FinishReason: cognitive.FinishProviderUnavailable})
|
||||
writeInterpretationResponse(w, g.interpretWithBuiltin(r.Context(), request, maxOutputBytes))
|
||||
return
|
||||
}
|
||||
provider := g.provider(selected)
|
||||
if provider == nil {
|
||||
writeInterpretationResponse(w, cognitive.GatewayInterpretationResponse{Version: request.Version, RequestID: request.RequestID, FinishReason: cognitive.FinishProviderUnavailable})
|
||||
writeInterpretationResponse(w, g.interpretWithBuiltin(r.Context(), request, maxOutputBytes))
|
||||
return
|
||||
}
|
||||
invocation := cognitive.AdvisoryRequest{
|
||||
@@ -77,6 +78,22 @@ func (g cognitiveGateway) interpretationHandler(w http.ResponseWriter, r *http.R
|
||||
writeInterpretationResponse(w, cognitive.GatewayInterpretationResponse{Version: request.Version, RequestID: request.RequestID, Candidate: candidate, FinishReason: cognitive.FinishCompleted, Usage: response.Usage})
|
||||
}
|
||||
|
||||
func (g cognitiveGateway) interpretWithBuiltin(ctx context.Context, request cognitive.GatewayInterpretationRequest, maxOutputBytes int) cognitive.GatewayInterpretationResponse {
|
||||
if g.builtinLocal == nil || g.builtinLocal.Provider == nil {
|
||||
return cognitive.GatewayInterpretationResponse{Version: request.Version, RequestID: request.RequestID, FinishReason: cognitive.FinishProviderUnavailable}
|
||||
}
|
||||
invocation := cognitive.AdvisoryRequest{Version: request.Version, RequestID: request.RequestID, ActorID: request.ActorID, PrincipalID: request.PrincipalID, ScopeID: request.ScopeID, Message: interpretationPrompt(request), Locale: request.Locale, MaxOutputBytes: maxOutputBytes, ProviderBindingID: "builtin-local-binding-v1", ProviderBindingRevision: 1, ProviderInstanceID: "builtin-local-lifeboat-v1", ModelDescriptorID: "builtin-local-model-v1", ProviderModelID: "domovoy-lifeboat"}
|
||||
response, err := g.builtinLocal.Provider.Advise(ctx, invocation)
|
||||
if err != nil || response.FinishReason != cognitive.FinishCompleted {
|
||||
return cognitive.GatewayInterpretationResponse{Version: request.Version, RequestID: request.RequestID, FinishReason: cognitive.FinishProviderUnavailable}
|
||||
}
|
||||
candidate, ok := parseInterpretationCandidate(response.AdvisoryText)
|
||||
if !ok {
|
||||
return cognitive.GatewayInterpretationResponse{Version: request.Version, RequestID: request.RequestID, FinishReason: cognitive.FinishInvalidStructuredOutput, Usage: response.Usage}
|
||||
}
|
||||
return cognitive.GatewayInterpretationResponse{Version: request.Version, RequestID: request.RequestID, Candidate: candidate, FinishReason: cognitive.FinishCompleted, Usage: response.Usage}
|
||||
}
|
||||
|
||||
func selectionObjectiveForInterpretation(request cognitive.GatewayInterpretationRequest) capability.SelectionObjective {
|
||||
if request.RequestedOutcome != "" || request.DeterministicApplicability == cognitive.InterpretationNeedsBroader || request.DeterministicApplicability == cognitive.InterpretationMixedIntent || request.DeterministicApplicability == cognitive.InterpretationLowConfidence {
|
||||
return capability.SelectionQuality
|
||||
|
||||
@@ -40,31 +40,36 @@ func main() {
|
||||
}
|
||||
providers := map[capability.ProviderKind][]providerRegistration{}
|
||||
discoveries := map[string]providerRegistration{}
|
||||
var builtinLocal *providerRegistration
|
||||
externalRecovery := []providerRegistration{}
|
||||
cpuAdapter, err := models.NewOpenAICompatibleLocalAdapter(envOr("LOCAL_CPU_PROVIDER_URL", "http://docker.cin.su:18082/v1"), nil)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
cpu := providerRegistration{ProviderInstanceID: "local-cpu-primary", Kind: "openai-compatible-chat-v1", EndpointRef: "local-cpu-primary-v1", AuthMode: capability.AuthNone, BillingMode: capability.BillingFree, Provider: cpuAdapter, Discovery: cpuAdapter}
|
||||
cpu := providerRegistration{ProviderInstanceID: "local-cpu-primary", Kind: "openai-compatible-chat-v1", EndpointRef: "local-cpu-primary-v1", AuthMode: capability.AuthNone, BillingMode: capability.BillingFree, DataPolicy: capability.DataPolicyLocalOnly, Provider: cpuAdapter, Discovery: cpuAdapter}
|
||||
providers[cpu.Kind] = append(providers[cpu.Kind], cpu)
|
||||
discoveries["cpu"] = cpu
|
||||
if lifeboat, enabled, lifeboatErr := newLifeboatProvider(os.Getenv("LOCAL_LIFEBOAT_PROVIDER_URL")); lifeboatErr != nil {
|
||||
log.Fatal(lifeboatErr)
|
||||
} else if enabled {
|
||||
lifeboat.DataPolicy = capability.DataPolicyLocalOnly
|
||||
providers[lifeboat.Kind] = append(providers[lifeboat.Kind], lifeboat)
|
||||
discoveries["lifeboat"] = lifeboat
|
||||
builtinLocal = &lifeboat
|
||||
}
|
||||
gpuAdapter, err := models.NewOllamaAdapter(envOr("LOCAL_GPU_PROVIDER_URL", "http://192.168.220.91:11434"), nil)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
gpu := providerRegistration{ProviderInstanceID: "local-gpu-opportunistic", Kind: "ollama-v1", EndpointRef: "local-gpu-opportunistic-v1", AuthMode: capability.AuthNone, BillingMode: capability.BillingFree, Provider: gpuAdapter, Discovery: gpuAdapter}
|
||||
gpu := providerRegistration{ProviderInstanceID: "local-gpu-opportunistic", Kind: "ollama-v1", EndpointRef: "local-gpu-opportunistic-v1", AuthMode: capability.AuthNone, BillingMode: capability.BillingFree, DataPolicy: capability.DataPolicyExternalMessageOnly, Provider: gpuAdapter, Discovery: gpuAdapter}
|
||||
providers[gpu.Kind] = append(providers[gpu.Kind], gpu)
|
||||
discoveries["gpu"] = gpu
|
||||
externalRecovery = append(externalRecovery, gpu)
|
||||
if openAIProvider != nil {
|
||||
kind := capability.ProviderKind(envOr("OPENAI_PROVIDER_KIND", "openai-responses-v1"))
|
||||
providers[kind] = append(providers[kind], providerRegistration{ProviderInstanceID: "openai-bootstrap", Kind: kind, EndpointRef: capability.EndpointBindingRef(envOr("OPENAI_ENDPOINT_REF", "openai-responses-default")), CredentialRef: capability.CredentialRef(envOr("OPENAI_CREDENTIAL_REF", "openai-advisory-v1")), AuthMode: capability.AuthKey, BillingMode: capability.BillingPerInput, Provider: openAIProvider})
|
||||
providers[kind] = append(providers[kind], providerRegistration{ProviderInstanceID: "openai-bootstrap", Kind: kind, EndpointRef: capability.EndpointBindingRef(envOr("OPENAI_ENDPOINT_REF", "openai-responses-default")), CredentialRef: capability.CredentialRef(envOr("OPENAI_CREDENTIAL_REF", "openai-advisory-v1")), AuthMode: capability.AuthKey, BillingMode: capability.BillingPerInput, DataPolicy: capability.DataPolicyExternalMessageOnly, Provider: openAIProvider})
|
||||
}
|
||||
gateway := cognitiveGateway{resolver: resolver, providers: providers, now: time.Now, allowCanary: os.Getenv("COGNITIVE_ALLOW_CANARY") == "true", canaryMaxOutputBytes: 4096}
|
||||
gateway := cognitiveGateway{resolver: resolver, providers: providers, builtinLocal: builtinLocal, externalRecovery: externalRecovery, now: time.Now, allowCanary: os.Getenv("COGNITIVE_ALLOW_CANARY") == "true", canaryMaxOutputBytes: 4096}
|
||||
mux.Handle("/healthz", health.Handler{Runtime: "cognitive-worker", SemanticGeneration: 1, ArchitectureGeneration: 1})
|
||||
mux.HandleFunc("POST /v1/proposals/deterministic", deterministicProposal)
|
||||
mux.HandleFunc("POST /v1/advisory-policy", gateway.policyHandler)
|
||||
@@ -99,7 +104,7 @@ func refreshProviderDiscoveries(resolver bindingResolver, service discoveryServi
|
||||
Enabled: true,
|
||||
OwnerPrincipalID: service.owner,
|
||||
CapabilityID: creatorAdvisoryCapabilityID,
|
||||
DataPolicy: capability.DataPolicyLocalOnly,
|
||||
DataPolicy: registration.DataPolicy,
|
||||
LimitPolicyRef: "creator-advisory-local-v1",
|
||||
Health: probe.Health,
|
||||
FailureClass: capability.CognitiveFailureType(probe.Failure),
|
||||
|
||||
Reference in New Issue
Block a user