feat(cognitive): enforce durable admission leases
This commit is contained in:
@@ -0,0 +1,68 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"strings"
|
||||
|
||||
"git.cin.su/m/Assistant/contracts/capability"
|
||||
)
|
||||
|
||||
type httpAdmissionClient struct {
|
||||
baseURL string
|
||||
client *http.Client
|
||||
}
|
||||
|
||||
func newHTTPAdmissionClient(baseURL string, client *http.Client) *httpAdmissionClient {
|
||||
return &httpAdmissionClient{baseURL: strings.TrimRight(baseURL, "/"), client: client}
|
||||
}
|
||||
|
||||
func (c *httpAdmissionClient) Admit(ctx context.Context, leaseID string, request capability.AdmissionRequest) (capability.AdmissionDecision, capability.AdmissionLease, error) {
|
||||
payload, err := json.Marshal(struct {
|
||||
LeaseID string `json:"lease_id"`
|
||||
Request capability.AdmissionRequest `json:"request"`
|
||||
}{leaseID, request})
|
||||
if err != nil {
|
||||
return capability.AdmissionDecision{}, capability.AdmissionLease{}, err
|
||||
}
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodPost, c.baseURL+"/v1/internal/cognitive/admission", bytes.NewReader(payload))
|
||||
if err != nil {
|
||||
return capability.AdmissionDecision{}, capability.AdmissionLease{}, err
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
resp, err := c.client.Do(req)
|
||||
if err != nil {
|
||||
return capability.AdmissionDecision{}, capability.AdmissionLease{}, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return capability.AdmissionDecision{}, capability.AdmissionLease{}, fmt.Errorf("admission status %d", resp.StatusCode)
|
||||
}
|
||||
var result struct {
|
||||
Decision capability.AdmissionDecision `json:"decision"`
|
||||
Lease capability.AdmissionLease `json:"lease"`
|
||||
}
|
||||
if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
|
||||
return capability.AdmissionDecision{}, capability.AdmissionLease{}, err
|
||||
}
|
||||
return result.Decision, result.Lease, nil
|
||||
}
|
||||
|
||||
func (c *httpAdmissionClient) Release(ctx context.Context, lease capability.AdmissionLease) error {
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodPost, c.baseURL+"/v1/internal/cognitive/admission/"+lease.ID+"/release", nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
resp, err := c.client.Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusNoContent {
|
||||
return fmt.Errorf("admission release status %d", resp.StatusCode)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,49 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"git.cin.su/m/Assistant/contracts/capability"
|
||||
"git.cin.su/m/Assistant/contracts/cognitive"
|
||||
)
|
||||
|
||||
type admissionTestProvider struct{ calls int }
|
||||
|
||||
func (p *admissionTestProvider) Advise(context.Context, cognitive.AdvisoryRequest) (cognitive.AdvisoryResponse, error) {
|
||||
p.calls++
|
||||
return cognitive.AdvisoryResponse{FinishReason: cognitive.FinishCompleted}, nil
|
||||
}
|
||||
|
||||
type admissionTestClient struct {
|
||||
decision capability.AdmissionDecision
|
||||
releases int
|
||||
}
|
||||
|
||||
func (c *admissionTestClient) Admit(context.Context, string, capability.AdmissionRequest) (capability.AdmissionDecision, capability.AdmissionLease, error) {
|
||||
return c.decision, capability.AdmissionLease{ID: "lease-1", PolicyID: "policy-1"}, nil
|
||||
}
|
||||
func (c *admissionTestClient) Release(context.Context, capability.AdmissionLease) error {
|
||||
c.releases++
|
||||
return nil
|
||||
}
|
||||
|
||||
func TestInvokeAdmittedDoesNotCallProviderWhenRejected(t *testing.T) {
|
||||
provider := &admissionTestProvider{}
|
||||
client := &admissionTestClient{decision: capability.AdmissionDecision{State: capability.AdmissionRejected, Reason: capability.AdmissionReasonConcurrency, PolicyID: "policy-1"}}
|
||||
gateway := cognitiveGateway{admission: client}
|
||||
response, err := gateway.invokeAdmitted(context.Background(), "request-1", provider, cognitive.AdvisoryRequest{Message: "hello"}, 100)
|
||||
if err != nil || response.FinishReason != cognitive.FinishRateLimited || provider.calls != 0 || client.releases != 0 {
|
||||
t.Fatalf("rejected admission leaked provider call: response=%#v err=%v calls=%d releases=%d", response, err, provider.calls, client.releases)
|
||||
}
|
||||
}
|
||||
|
||||
func TestInvokeAdmittedReleasesDurableLeaseAfterProvider(t *testing.T) {
|
||||
provider := &admissionTestProvider{}
|
||||
client := &admissionTestClient{decision: capability.AdmissionDecision{State: capability.AdmissionAdmitted, Reason: capability.AdmissionReasonNone, PolicyID: "policy-1"}}
|
||||
gateway := cognitiveGateway{admission: client}
|
||||
response, err := gateway.invokeAdmitted(context.Background(), "request-1", provider, cognitive.AdvisoryRequest{Message: "hello"}, 100)
|
||||
if err != nil || response.FinishReason != cognitive.FinishCompleted || provider.calls != 1 || client.releases != 1 {
|
||||
t.Fatalf("admitted lease was not released: response=%#v err=%v calls=%d releases=%d", response, err, provider.calls, client.releases)
|
||||
}
|
||||
}
|
||||
@@ -26,6 +26,11 @@ type bindingResolver interface {
|
||||
RegisterDiscovery(context.Context, capability.CognitiveDiscoverySnapshot) (capability.CognitiveDiscoveryRegistration, error)
|
||||
}
|
||||
|
||||
type admissionClient interface {
|
||||
Admit(context.Context, string, capability.AdmissionRequest) (capability.AdmissionDecision, capability.AdmissionLease, error)
|
||||
Release(context.Context, capability.AdmissionLease) error
|
||||
}
|
||||
|
||||
type providerRegistration struct {
|
||||
ProviderInstanceID capability.ProviderInstanceID
|
||||
Kind capability.ProviderKind
|
||||
@@ -46,6 +51,7 @@ type cognitiveGateway struct {
|
||||
now func() time.Time
|
||||
allowCanary bool
|
||||
canaryMaxOutputBytes int
|
||||
admission admissionClient
|
||||
}
|
||||
|
||||
func (g cognitiveGateway) policyHandler(w http.ResponseWriter, r *http.Request) {
|
||||
@@ -95,9 +101,9 @@ func (g cognitiveGateway) advisoryHandler(w http.ResponseWriter, r *http.Request
|
||||
}
|
||||
invocation := cognitive.AdvisoryRequest{Version: request.Version, RequestID: request.RequestID, ActorID: request.ActorID, PrincipalID: request.PrincipalID, ScopeID: request.ScopeID, Message: request.Message, Context: request.Context, Locale: request.Locale, MaxOutputBytes: maxOutputBytes, ProviderBindingID: selected.Binding.ID, ProviderBindingRevision: selected.Binding.Revision, ProviderInstanceID: selected.Provider.ID, ModelDescriptorID: selected.Model.ID, ProviderModelID: selected.Model.ProviderModelID}
|
||||
started := g.now()
|
||||
response, invokeErr := provider.Advise(r.Context(), invocation)
|
||||
response, invokeErr := g.invokeAdmitted(r.Context(), string(request.RequestID), provider, invocation, maxOutputBytes)
|
||||
if invokeErr != nil {
|
||||
response = cognitive.AdvisoryResponse{Version: request.Version, RequestID: request.RequestID, ProviderBindingID: selected.Binding.ID, ProviderBindingRevision: selected.Binding.Revision, FinishReason: cognitive.FinishInvalidProviderResponse}
|
||||
response = cognitive.AdvisoryResponse{Version: request.Version, RequestID: request.RequestID, ProviderBindingID: selected.Binding.ID, ProviderBindingRevision: selected.Binding.Revision, FinishReason: cognitive.FinishProviderUnavailable}
|
||||
}
|
||||
g.observe(r.Context(), selected, response, started)
|
||||
if fallbackEligible(response.FinishReason) {
|
||||
@@ -109,9 +115,9 @@ func (g cognitiveGateway) advisoryHandler(w http.ResponseWriter, r *http.Request
|
||||
fallbackRequest.ProviderBindingID, fallbackRequest.ProviderBindingRevision = fallback.Binding.ID, fallback.Binding.Revision
|
||||
fallbackRequest.ProviderInstanceID, fallbackRequest.ModelDescriptorID, fallbackRequest.ProviderModelID = fallback.Provider.ID, fallback.Model.ID, fallback.Model.ProviderModelID
|
||||
fallbackStarted := g.now()
|
||||
fallbackResponse, fallbackErr := fallbackProvider.Advise(r.Context(), fallbackRequest)
|
||||
fallbackResponse, fallbackErr := g.invokeAdmitted(r.Context(), string(request.RequestID)+":fallback", fallbackProvider, fallbackRequest, maxOutputBytes)
|
||||
if fallbackErr != nil {
|
||||
fallbackResponse = cognitive.AdvisoryResponse{Version: request.Version, RequestID: request.RequestID, ProviderBindingID: fallback.Binding.ID, ProviderBindingRevision: fallback.Binding.Revision, FinishReason: cognitive.FinishInvalidProviderResponse}
|
||||
fallbackResponse = cognitive.AdvisoryResponse{Version: request.Version, RequestID: request.RequestID, ProviderBindingID: fallback.Binding.ID, ProviderBindingRevision: fallback.Binding.Revision, FinishReason: cognitive.FinishProviderUnavailable}
|
||||
}
|
||||
g.observe(r.Context(), fallback, fallbackResponse, fallbackStarted)
|
||||
response = fallbackResponse
|
||||
@@ -124,6 +130,22 @@ func (g cognitiveGateway) advisoryHandler(w http.ResponseWriter, r *http.Request
|
||||
writeGatewayResponse(w, cognitive.GatewayAdvisoryResponse{Version: response.Version, RequestID: response.RequestID, AdvisoryText: response.AdvisoryText, ReferencedExcerptIDs: response.ReferencedExcerptIDs, FinishReason: response.FinishReason, Usage: response.Usage})
|
||||
}
|
||||
|
||||
func (g cognitiveGateway) invokeAdmitted(ctx context.Context, leaseID string, provider advisoryProvider, invocation cognitive.AdvisoryRequest, maxOutputBytes int) (cognitive.AdvisoryResponse, error) {
|
||||
var lease capability.AdmissionLease
|
||||
if g.admission != nil {
|
||||
decision, admittedLease, err := g.admission.Admit(ctx, leaseID, capability.AdmissionRequest{InputBytes: len(invocation.Message), MaxOutputBytes: maxOutputBytes, DeadlineMillis: 0})
|
||||
if err != nil {
|
||||
return cognitive.AdvisoryResponse{}, err
|
||||
}
|
||||
if decision.State != capability.AdmissionAdmitted {
|
||||
return cognitive.AdvisoryResponse{Version: invocation.Version, RequestID: invocation.RequestID, ProviderBindingID: invocation.ProviderBindingID, ProviderBindingRevision: invocation.ProviderBindingRevision, FinishReason: cognitive.FinishRateLimited}, nil
|
||||
}
|
||||
lease = admittedLease
|
||||
defer func() { _ = g.admission.Release(ctx, lease) }()
|
||||
}
|
||||
return provider.Advise(ctx, invocation)
|
||||
}
|
||||
|
||||
func selectionObjectiveForMessage(message string) capability.SelectionObjective {
|
||||
lower := strings.ToLower(message)
|
||||
if strings.Contains(lower, "дешев") || strings.Contains(lower, "бесплат") || strings.Contains(lower, "эконом") || strings.Contains(lower, "cost") || strings.Contains(lower, "cheap") {
|
||||
|
||||
@@ -70,7 +70,7 @@ func main() {
|
||||
kind := capability.ProviderKind(envOr("OPENAI_PROVIDER_KIND", "openai-responses-v1"))
|
||||
providers[kind] = append(providers[kind], providerRegistration{ProviderInstanceID: "openai-bootstrap", Kind: kind, EndpointRef: capability.EndpointBindingRef(envOr("OPENAI_ENDPOINT_REF", "openai-responses-default")), CredentialRef: capability.CredentialRef(envOr("OPENAI_CREDENTIAL_REF", "openai-advisory-v1")), AuthMode: capability.AuthKey, BillingMode: capability.BillingPerInput, DataPolicy: capability.DataPolicyExternalMessageOnly, Provider: openAIProvider})
|
||||
}
|
||||
gateway := cognitiveGateway{resolver: resolver, providers: providers, builtinLocal: builtinLocal, externalRecovery: externalRecovery, now: time.Now, allowCanary: os.Getenv("COGNITIVE_ALLOW_CANARY") == "true", canaryMaxOutputBytes: 4096}
|
||||
gateway := cognitiveGateway{resolver: resolver, providers: providers, builtinLocal: builtinLocal, externalRecovery: externalRecovery, now: time.Now, allowCanary: os.Getenv("COGNITIVE_ALLOW_CANARY") == "true", canaryMaxOutputBytes: 4096, admission: newHTTPAdmissionClient(envOr("CORE_RUNTIME_URL", "http://core-runtime:8080"), &http.Client{Timeout: 5 * time.Second})}
|
||||
mux.Handle("/healthz", health.Handler{Runtime: "cognitive-worker", SemanticGeneration: 1, ArchitectureGeneration: 1})
|
||||
readinessClient := &http.Client{Timeout: 2 * time.Second}
|
||||
lifeboatURL := strings.TrimSuffix(strings.TrimRight(envOr("LOCAL_LIFEBOAT_PROVIDER_URL", "http://cognitive-lifeboat:8086/v1"), "/"), "/v1") + "/health"
|
||||
|
||||
@@ -141,6 +141,34 @@ func main() {
|
||||
health.HTTPProbe(readinessClient, strings.TrimRight(envOr("CONTINUITY_RUNTIME_URL", "http://continuity-runtime:8081"), "/")+"/healthz"),
|
||||
health.HTTPProbe(readinessClient, strings.TrimRight(envOr("COGNITIVE_RUNTIME_URL", "http://cognitive-worker:8083"), "/")+"/healthz"),
|
||||
}})
|
||||
admissionPolicy := capability.AdmissionPolicy{ID: envOr("COGNITIVE_ADMISSION_POLICY_ID", "creator-advisory-local-v1"), MaxConcurrent: envInt("COGNITIVE_ADMISSION_MAX_CONCURRENT", 1), MaxAcceptedPerHour: envInt("COGNITIVE_ADMISSION_MAX_ACCEPTED_PER_HOUR", 60), MaxInputBytes: envInt("COGNITIVE_ADMISSION_MAX_INPUT_BYTES", 80<<10), MaxOutputBytes: envInt("COGNITIVE_ADMISSION_MAX_OUTPUT_BYTES", 16<<10)}
|
||||
mux.HandleFunc("POST /v1/internal/cognitive/admission", func(w http.ResponseWriter, r *http.Request) {
|
||||
var request struct {
|
||||
LeaseID string `json:"lease_id"`
|
||||
Request capability.AdmissionRequest `json:"request"`
|
||||
}
|
||||
if err := json.NewDecoder(http.MaxBytesReader(w, r.Body, 16<<10)).Decode(&request); err != nil || request.LeaseID == "" {
|
||||
http.Error(w, "invalid admission request", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
decision, lease, err := capabilityRepository.AdmitCognitive(r.Context(), request.LeaseID, admissionPolicy, request.Request, time.Now().UnixMilli())
|
||||
if err != nil {
|
||||
http.Error(w, "admission unavailable", http.StatusServiceUnavailable)
|
||||
return
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_ = json.NewEncoder(w).Encode(struct {
|
||||
Decision capability.AdmissionDecision `json:"decision"`
|
||||
Lease capability.AdmissionLease `json:"lease"`
|
||||
}{decision, lease})
|
||||
})
|
||||
mux.HandleFunc("POST /v1/internal/cognitive/admission/{lease}/release", func(w http.ResponseWriter, r *http.Request) {
|
||||
if err := capabilityRepository.ReleaseCognitiveAdmission(r.Context(), capability.AdmissionLease{ID: r.PathValue("lease")}, time.Now().UnixMilli()); err != nil {
|
||||
http.Error(w, "admission release unavailable", http.StatusServiceUnavailable)
|
||||
return
|
||||
}
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
})
|
||||
mux.HandleFunc("POST /v1/creator/sessions", creatorSessionHandler(bootstrapToken, creatorUsername, creatorPassword, sessionKey, time.Now))
|
||||
mux.HandleFunc("GET /v1/creator/sessions/current", func(w http.ResponseWriter, r *http.Request) {
|
||||
session, ok := authenticateCreator(w, r)
|
||||
@@ -1159,6 +1187,14 @@ func envOr(name, fallback string) string {
|
||||
return fallback
|
||||
}
|
||||
|
||||
func envInt(name string, fallback int) int {
|
||||
value, err := strconv.Atoi(os.Getenv(name))
|
||||
if err != nil || value < 1 {
|
||||
return fallback
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
func humanCognitiveModelInventory(candidates []capability.ResolvedCognitiveBinding, inventoryErr error) string {
|
||||
if inventoryErr != nil {
|
||||
return "Я не смогла прочитать реестр подключённых моделей. Встроенный канал Домового остаётся доступен; настройки можно проверить повторно после восстановления реестра. Ключи и пароли не выводятся в чат."
|
||||
|
||||
Reference in New Issue
Block a user