feat: connect protected OpenAI-compatible GPU provider
This commit is contained in:
@@ -17,6 +17,7 @@ import (
|
||||
"log"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
@@ -92,14 +93,20 @@ func main() {
|
||||
discoveries["lifeboat"] = lifeboat
|
||||
builtinLocal = &lifeboat
|
||||
}
|
||||
gpuAdapter, err := models.NewOllamaAdapter(envOr("LOCAL_GPU_PROVIDER_URL", "http://192.168.220.91:11434"), nil)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
gpuURL, gpuKey, gpuConfigured, gpuErr := loadProtectedGPUConfiguration()
|
||||
if gpuErr != nil {
|
||||
log.Printf("optional GPU provider disabled: %v", gpuErr)
|
||||
} else if gpuConfigured {
|
||||
gpuAdapter, adapterErr := models.NewOpenAICompatibleLocalAdapterWithAPIKey(gpuURL, gpuKey, nil)
|
||||
if adapterErr != nil {
|
||||
log.Printf("optional GPU provider disabled: %v", adapterErr)
|
||||
} else {
|
||||
gpu := providerRegistration{ProviderInstanceID: "local-gpu-opportunistic", Kind: "openai-compatible-chat-v1", EndpointRef: "local-gpu-opportunistic-v1", CredentialRef: "credential://domovoy/gpu/provider-api-key/v1", AuthMode: capability.AuthKey, BillingMode: capability.BillingFree, DataPolicy: capability.DataPolicyExternalMessageOnly, Provider: gpuAdapter, Discovery: gpuAdapter}
|
||||
providers[gpu.Kind] = append(providers[gpu.Kind], gpu)
|
||||
discoveries["gpu"] = gpu
|
||||
externalRecovery = append(externalRecovery, gpu)
|
||||
}
|
||||
}
|
||||
gpu := providerRegistration{ProviderInstanceID: "local-gpu-opportunistic", Kind: "ollama-v1", EndpointRef: "local-gpu-opportunistic-v1", AuthMode: capability.AuthNone, BillingMode: capability.BillingFree, DataPolicy: capability.DataPolicyExternalMessageOnly, Provider: gpuAdapter, Discovery: gpuAdapter}
|
||||
providers[gpu.Kind] = append(providers[gpu.Kind], gpu)
|
||||
discoveries["gpu"] = gpu
|
||||
externalRecovery = append(externalRecovery, gpu)
|
||||
if openAIProvider != nil {
|
||||
kind := capability.ProviderKind(envOr("OPENAI_PROVIDER_KIND", "openai-responses-v1"))
|
||||
providers[kind] = append(providers[kind], providerRegistration{ProviderInstanceID: "openai-bootstrap", Kind: kind, EndpointRef: capability.EndpointBindingRef(envOr("OPENAI_ENDPOINT_REF", "openai-responses-default")), CredentialRef: capability.CredentialRef(envOr("OPENAI_CREDENTIAL_REF", "openai-advisory-v1")), AuthMode: capability.AuthKey, BillingMode: capability.BillingPerInput, DataPolicy: capability.DataPolicyExternalMessageOnly, Provider: openAIProvider})
|
||||
@@ -137,17 +144,24 @@ func main() {
|
||||
if mediumURL != "" {
|
||||
readinessProbes = append(readinessProbes, mediumHealthProbe)
|
||||
}
|
||||
readinessProbes = append(readinessProbes,
|
||||
health.HTTPProbe(readinessClient, lifeboatURL),
|
||||
health.OptionalHTTPProbe(readinessClient, strings.TrimRight(envOr("LOCAL_GPU_PROVIDER_URL", "http://192.168.220.91:11434"), "/")+"/api/tags"),
|
||||
)
|
||||
mux.Handle("/readyz", health.ReadinessHandler{Runtime: "cognitive-worker", Probes: readinessProbes})
|
||||
mux.Handle("/v1/fleet/readiness", fleetReadinessHandler(fleetClient, preferredFleet, []fleetEndpoint{
|
||||
readinessProbes = append(readinessProbes, health.HTTPProbe(readinessClient, lifeboatURL))
|
||||
if gpuConfigured {
|
||||
gpuHeaders := map[string]string{"Authorization": "Bearer " + gpuKey}
|
||||
gpuProbe := health.OptionalHTTPProbeWithHeaders(readinessClient, strings.TrimRight(gpuURL, "/")+"/models", gpuHeaders)
|
||||
gpuProbe.Timeout = 10 * time.Second
|
||||
readinessProbes = append(readinessProbes, gpuProbe)
|
||||
}
|
||||
|
||||
fleetEndpoints := []fleetEndpoint{
|
||||
{ProviderID: "local-cpu-primary", ModelID: "local-cpu-model", Tier: capability.FleetTierFast, URL: cpuEndpoint, Headers: cpuFleetHeaders, Client: &http.Client{Timeout: 10 * time.Second}},
|
||||
{ProviderID: "local-medium-primary", ModelID: "local-medium-model", Tier: capability.FleetTierMedium, URL: mediumEndpoint, Headers: mediumHeaders, Client: mediumReadinessClient},
|
||||
{ProviderID: "local-gpu-opportunistic", ModelID: "gpu-opportunistic", Tier: capability.FleetTierHeavy, URL: strings.TrimRight(envOr("LOCAL_GPU_PROVIDER_URL", "http://192.168.220.91:11434"), "/") + "/api/tags"},
|
||||
{ProviderID: "cognitive-lifeboat", ModelID: "qwen3-8b-lifeboat", Tier: capability.FleetTierLifeboat, URL: lifeboatURL},
|
||||
}))
|
||||
}
|
||||
if gpuConfigured {
|
||||
fleetEndpoints = append(fleetEndpoints, fleetEndpoint{ProviderID: "local-gpu-opportunistic", ModelID: "gpu-opportunistic", Tier: capability.FleetTierHeavy, URL: strings.TrimRight(gpuURL, "/") + "/models", Headers: map[string]string{"Authorization": "Bearer " + gpuKey}})
|
||||
}
|
||||
mux.Handle("/readyz", health.ReadinessHandler{Runtime: "cognitive-worker", Probes: readinessProbes})
|
||||
mux.Handle("/v1/fleet/readiness", fleetReadinessHandler(fleetClient, preferredFleet, fleetEndpoints))
|
||||
mux.HandleFunc("POST /v1/proposals/deterministic", deterministicProposal)
|
||||
mux.HandleFunc("POST /v1/advisory-policy", gateway.policyHandler)
|
||||
mux.HandleFunc("POST /v1/advisories", gateway.advisoryHandler)
|
||||
@@ -244,6 +258,46 @@ func loadLocalCPUProviderAPIKey() (string, error) {
|
||||
return loadProviderAPIKey("LOCAL_CPU_PROVIDER_API_KEY", "LOCAL_CPU_PROVIDER_API_KEY_FILE", "LOCAL_CPU_PROVIDER_CREDENTIAL_REF", "credential://domovoy/local-cpu/provider-api-key/v1")
|
||||
}
|
||||
|
||||
const (
|
||||
gpuProviderURLFile = "/run/secrets/llm_gpu_gateway_url"
|
||||
gpuProviderKeyFile = "/run/secrets/llm_gpu_gateway_api_key"
|
||||
)
|
||||
|
||||
func loadProtectedGPUConfiguration() (string, string, bool, error) {
|
||||
urlValue, err := readProtectedRuntimeValue(gpuProviderURLFile)
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
return "", "", false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return "", "", false, err
|
||||
}
|
||||
key, err := readProtectedRuntimeValue(gpuProviderKeyFile)
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
return "", "", false, fmt.Errorf("GPU provider key is not configured")
|
||||
}
|
||||
if err != nil {
|
||||
return "", "", false, err
|
||||
}
|
||||
return urlValue, key, true, nil
|
||||
}
|
||||
|
||||
func readProtectedRuntimeValue(path string) (string, error) {
|
||||
const root = "/run/secrets"
|
||||
clean := filepath.Clean(path)
|
||||
if clean != root && !strings.HasPrefix(clean, root+string(os.PathSeparator)) {
|
||||
return "", fmt.Errorf("protected runtime path outside secret root")
|
||||
}
|
||||
payload, err := os.ReadFile(clean)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
value := strings.TrimSpace(string(payload))
|
||||
if value == "" || strings.ContainsAny(value, "\r\n") {
|
||||
return "", fmt.Errorf("protected runtime value is empty or multiline")
|
||||
}
|
||||
return value, nil
|
||||
}
|
||||
|
||||
func deterministicAdvisory(w http.ResponseWriter, r *http.Request) {
|
||||
request, ok := decodeAdvisoryRequest(w, r)
|
||||
if !ok {
|
||||
|
||||
@@ -68,3 +68,9 @@ func TestLoadProviderAPIKeyRejectsMediumDirectEnvironmentSecret(t *testing.T) {
|
||||
t.Fatal("expected direct medium environment secret to be rejected")
|
||||
}
|
||||
}
|
||||
|
||||
func TestReadProtectedRuntimeValueRejectsPathOutsideSecretRoot(t *testing.T) {
|
||||
if _, err := readProtectedRuntimeValue("/tmp/not-a-domovoy-secret"); err == nil {
|
||||
t.Fatal("expected protected path rejection")
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user