feat: add configurable medium model fleet slot
This commit is contained in:
@@ -45,7 +45,7 @@ func main() {
|
||||
discoveries := map[string]providerRegistration{}
|
||||
var builtinLocal *providerRegistration
|
||||
externalRecovery := []providerRegistration{}
|
||||
cpuAPIKey, err := loadLocalCPUProviderAPIKey()
|
||||
cpuAPIKey, err := loadProviderAPIKey("LOCAL_CPU_PROVIDER_API_KEY", "LOCAL_CPU_PROVIDER_API_KEY_FILE", "LOCAL_CPU_PROVIDER_CREDENTIAL_REF", "credential://domovoy/local-cpu/provider-api-key/v1")
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
@@ -62,6 +62,27 @@ func main() {
|
||||
cpu := providerRegistration{ProviderInstanceID: "local-cpu-primary", Kind: "openai-compatible-chat-v1", EndpointRef: "local-cpu-primary-v1", CredentialRef: cpuCredentialRef, AuthMode: cpuAuthMode, BillingMode: capability.BillingFree, DataPolicy: capability.DataPolicyLocalOnly, Provider: cpuAdapter, Discovery: cpuAdapter}
|
||||
providers[cpu.Kind] = append(providers[cpu.Kind], cpu)
|
||||
discoveries["cpu"] = cpu
|
||||
mediumURL := strings.TrimSpace(os.Getenv("LOCAL_MEDIUM_PROVIDER_URL"))
|
||||
mediumAPIKey := ""
|
||||
if mediumURL != "" {
|
||||
mediumAPIKey, err = loadProviderAPIKey("LOCAL_MEDIUM_PROVIDER_API_KEY", "LOCAL_MEDIUM_PROVIDER_API_KEY_FILE", "LOCAL_MEDIUM_PROVIDER_CREDENTIAL_REF", "credential://domovoy/local-medium/provider-api-key/v1")
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
mediumAdapter, adapterErr := models.NewOpenAICompatibleLocalAdapterWithAPIKey(strings.TrimRight(mediumURL, "/"), mediumAPIKey, nil)
|
||||
if adapterErr != nil {
|
||||
log.Fatal(adapterErr)
|
||||
}
|
||||
mediumAuthMode := capability.AuthNone
|
||||
mediumCredentialRef := capability.CredentialRef("")
|
||||
if mediumAPIKey != "" {
|
||||
mediumAuthMode = capability.AuthKey
|
||||
mediumCredentialRef = capability.CredentialRef(envOr("LOCAL_MEDIUM_PROVIDER_CREDENTIAL_REF", "credential://domovoy/local-medium/provider-api-key/v1"))
|
||||
}
|
||||
medium := providerRegistration{ProviderInstanceID: "local-medium-primary", Kind: "openai-compatible-chat-v1", EndpointRef: "local-medium-primary-v1", CredentialRef: mediumCredentialRef, AuthMode: mediumAuthMode, BillingMode: capability.BillingFree, DataPolicy: capability.DataPolicyLocalOnly, Provider: mediumAdapter, Discovery: mediumAdapter}
|
||||
providers[medium.Kind] = append(providers[medium.Kind], medium)
|
||||
discoveries["medium"] = medium
|
||||
}
|
||||
if lifeboat, enabled, lifeboatErr := newLifeboatProvider(os.Getenv("LOCAL_LIFEBOAT_PROVIDER_URL")); lifeboatErr != nil {
|
||||
log.Fatal(lifeboatErr)
|
||||
} else if enabled {
|
||||
@@ -86,6 +107,7 @@ func main() {
|
||||
mux.Handle("/healthz", health.Handler{Runtime: "cognitive-worker", SemanticGeneration: 1, ArchitectureGeneration: 1})
|
||||
readinessClient := &http.Client{Timeout: 2 * time.Second}
|
||||
cpuReadinessClient := &http.Client{Timeout: 10 * time.Second}
|
||||
mediumReadinessClient := &http.Client{Timeout: 10 * time.Second}
|
||||
lifeboatURL := strings.TrimSuffix(strings.TrimRight(envOr("LOCAL_LIFEBOAT_PROVIDER_URL", "http://cognitive-lifeboat:8086/v1"), "/"), "/v1") + "/health"
|
||||
fleetClient := &http.Client{Timeout: 2 * time.Second}
|
||||
preferredFleet := capability.FleetTier(envOr("COGNITIVE_PREFERRED_FLEET_TIER", string(capability.FleetTierHeavy)))
|
||||
@@ -96,16 +118,28 @@ func main() {
|
||||
cpuEndpoint := strings.TrimRight(envOr("LOCAL_CPU_PROVIDER_URL", "http://docker.cin.su:18082/v1"), "/") + "/models"
|
||||
cpuHealthProbe := health.OptionalHTTPProbeWithHeaders(cpuReadinessClient, cpuEndpoint, cpuFleetHeaders)
|
||||
cpuHealthProbe.Timeout = 10 * time.Second
|
||||
mediumEndpoint := ""
|
||||
mediumHeaders := map[string]string{}
|
||||
mediumHealthProbe := health.Probe{}
|
||||
if mediumURL != "" {
|
||||
mediumEndpoint = strings.TrimRight(mediumURL, "/") + "/models"
|
||||
if mediumAPIKey != "" {
|
||||
mediumHeaders["Authorization"] = "Bearer " + mediumAPIKey
|
||||
}
|
||||
mediumHealthProbe = health.OptionalHTTPProbeWithHeaders(mediumReadinessClient, mediumEndpoint, mediumHeaders)
|
||||
mediumHealthProbe.Timeout = 10 * time.Second
|
||||
}
|
||||
readinessProbes := []health.Probe{
|
||||
health.HTTPProbe(readinessClient, strings.TrimRight(envOr("CORE_RUNTIME_URL", "http://core-runtime:8080"), "/")+"/healthz"),
|
||||
cpuHealthProbe,
|
||||
mediumHealthProbe,
|
||||
health.HTTPProbe(readinessClient, lifeboatURL),
|
||||
health.OptionalHTTPProbe(readinessClient, strings.TrimRight(envOr("LOCAL_GPU_PROVIDER_URL", "http://192.168.220.91:11434"), "/")+"/api/tags"),
|
||||
}
|
||||
mux.Handle("/readyz", health.ReadinessHandler{Runtime: "cognitive-worker", Probes: readinessProbes})
|
||||
mux.Handle("/v1/fleet/readiness", fleetReadinessHandler(fleetClient, preferredFleet, []fleetEndpoint{
|
||||
{ProviderID: "local-cpu-primary", ModelID: "local-cpu-model", Tier: capability.FleetTierFast, URL: cpuEndpoint, Headers: cpuFleetHeaders, Client: &http.Client{Timeout: 10 * time.Second}},
|
||||
{ProviderID: "local-medium-unconfigured", ModelID: "medium-unconfigured", Tier: capability.FleetTierMedium},
|
||||
{ProviderID: "local-medium-primary", ModelID: "local-medium-model", Tier: capability.FleetTierMedium, URL: mediumEndpoint, Headers: mediumHeaders, Client: mediumReadinessClient},
|
||||
{ProviderID: "local-gpu-opportunistic", ModelID: "gpu-opportunistic", Tier: capability.FleetTierHeavy, URL: strings.TrimRight(envOr("LOCAL_GPU_PROVIDER_URL", "http://192.168.220.91:11434"), "/") + "/api/tags"},
|
||||
{ProviderID: "cognitive-lifeboat", ModelID: "qwen3-8b-lifeboat", Tier: capability.FleetTierLifeboat, URL: lifeboatURL},
|
||||
}))
|
||||
@@ -166,17 +200,20 @@ func refreshProviderDiscoveries(resolver bindingResolver, service discoveryServi
|
||||
}
|
||||
}
|
||||
|
||||
func loadLocalCPUProviderAPIKey() (string, error) {
|
||||
if _, set := os.LookupEnv("LOCAL_CPU_PROVIDER_API_KEY"); set {
|
||||
return "", fmt.Errorf("LOCAL_CPU_PROVIDER_API_KEY direct environment secret is forbidden; use LOCAL_CPU_PROVIDER_API_KEY_FILE")
|
||||
func loadProviderAPIKey(directEnvName, fileEnvName, refEnvName, defaultRef string) (string, error) {
|
||||
if _, set := os.LookupEnv(directEnvName); set {
|
||||
return "", fmt.Errorf("%s direct environment secret is forbidden; use %s", directEnvName, fileEnvName)
|
||||
}
|
||||
fileName := strings.TrimSpace(os.Getenv("LOCAL_CPU_PROVIDER_API_KEY_FILE"))
|
||||
fileName := strings.TrimSpace(os.Getenv(fileEnvName))
|
||||
if fileName == "" {
|
||||
return "", nil
|
||||
}
|
||||
ref := strings.TrimSpace(os.Getenv("LOCAL_CPU_PROVIDER_CREDENTIAL_REF"))
|
||||
ref := strings.TrimSpace(os.Getenv(refEnvName))
|
||||
if ref == "" {
|
||||
return "", fmt.Errorf("LOCAL_CPU_PROVIDER_CREDENTIAL_REF is required when LOCAL_CPU_PROVIDER_API_KEY_FILE is configured")
|
||||
ref = defaultRef
|
||||
}
|
||||
if ref == "" {
|
||||
return "", fmt.Errorf("%s is required when %s is configured", refEnvName, fileEnvName)
|
||||
}
|
||||
resolver, err := platformsecrets.NewFileResolver(envOr("SECRET_BACKEND_ROOT", "/run/secrets"))
|
||||
if err != nil {
|
||||
@@ -185,6 +222,10 @@ func loadLocalCPUProviderAPIKey() (string, error) {
|
||||
return resolver.ResolveFile(secretcontracts.SecretFileRef{CredentialRef: secretcontracts.CredentialRef(ref), Purpose: secretcontracts.SecretPurposeProviderAPIKey, Consumer: secretcontracts.RuntimeConsumerCognitive, Path: fileName})
|
||||
}
|
||||
|
||||
func loadLocalCPUProviderAPIKey() (string, error) {
|
||||
return loadProviderAPIKey("LOCAL_CPU_PROVIDER_API_KEY", "LOCAL_CPU_PROVIDER_API_KEY_FILE", "LOCAL_CPU_PROVIDER_CREDENTIAL_REF", "credential://domovoy/local-cpu/provider-api-key/v1")
|
||||
}
|
||||
|
||||
func deterministicAdvisory(w http.ResponseWriter, r *http.Request) {
|
||||
request, ok := decodeAdvisoryRequest(w, r)
|
||||
if !ok {
|
||||
|
||||
@@ -43,3 +43,28 @@ func TestLoadLocalCPUProviderAPIKeyRejectsDirectEnvironmentSecret(t *testing.T)
|
||||
t.Fatal("expected direct environment secret to be rejected")
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadProviderAPIKeySupportsMediumCredentialRef(t *testing.T) {
|
||||
t.Setenv("LOCAL_MEDIUM_PROVIDER_API_KEY", "")
|
||||
_ = os.Unsetenv("LOCAL_MEDIUM_PROVIDER_API_KEY")
|
||||
root := t.TempDir()
|
||||
path := filepath.Join(root, "medium_provider_api_key")
|
||||
if err := os.WriteFile(path, []byte("medium-key\n"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Setenv("LOCAL_MEDIUM_PROVIDER_API_KEY_FILE", path)
|
||||
t.Setenv("LOCAL_MEDIUM_PROVIDER_CREDENTIAL_REF", "credential://domovoy/local-medium/provider-api-key/v1")
|
||||
t.Setenv("SECRET_BACKEND_ROOT", root)
|
||||
key, err := loadProviderAPIKey("LOCAL_MEDIUM_PROVIDER_API_KEY", "LOCAL_MEDIUM_PROVIDER_API_KEY_FILE", "LOCAL_MEDIUM_PROVIDER_CREDENTIAL_REF", "credential://domovoy/local-medium/provider-api-key/v1")
|
||||
if err != nil || key != "medium-key" {
|
||||
t.Fatalf("unexpected medium key resolution result: err=%v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadProviderAPIKeyRejectsMediumDirectEnvironmentSecret(t *testing.T) {
|
||||
t.Setenv("LOCAL_MEDIUM_PROVIDER_API_KEY", "must-not-be-used")
|
||||
t.Setenv("LOCAL_MEDIUM_PROVIDER_API_KEY_FILE", "")
|
||||
if _, err := loadProviderAPIKey("LOCAL_MEDIUM_PROVIDER_API_KEY", "LOCAL_MEDIUM_PROVIDER_API_KEY_FILE", "LOCAL_MEDIUM_PROVIDER_CREDENTIAL_REF", "credential://domovoy/local-medium/provider-api-key/v1"); err == nil {
|
||||
t.Fatal("expected direct medium environment secret to be rejected")
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user