Add ComfyUI image generation adapter
This commit is contained in:
@@ -171,15 +171,20 @@ func main() {
|
||||
mux.HandleFunc("POST /v1/advisories", gateway.advisoryHandler)
|
||||
mux.HandleFunc("POST /v1/interpretations", gateway.interpretationHandler)
|
||||
mux.HandleFunc("POST /v1/task-evaluations/execute", gateway.taskEvaluationHandler)
|
||||
imageGenerator := multimodal.ImageGenerator(multimodal.ImageGenerationClient{BaseURL: os.Getenv("LOCAL_IMAGE_PROVIDER_URL"), Client: &http.Client{Timeout: 300 * time.Second}})
|
||||
if strings.EqualFold(strings.TrimSpace(os.Getenv("LOCAL_IMAGE_PROVIDER_KIND")), "comfyui") {
|
||||
imageGenerator = multimodal.ComfyImageGenerationClient{BaseURL: os.Getenv("LOCAL_IMAGE_PROVIDER_URL"), Client: &http.Client{Timeout: 300 * time.Second}, PollTimeout: 15 * time.Minute}
|
||||
}
|
||||
multimodalGateway := multimodalGateway{
|
||||
speech: multimodal.SpeechClient{BaseURL: envOr("LOCAL_SPEECH_PROVIDER_URL", "http://llm-local-speech:8080"), Client: &http.Client{Timeout: 120 * time.Second}},
|
||||
vision: multimodal.VisionClient{BaseURL: envOr("LOCAL_VISION_PROVIDER_URL", "http://llm-local-vision:8080/v1"), Client: &http.Client{Timeout: 180 * time.Second}},
|
||||
imageGenerator: multimodal.ImageGenerationClient{BaseURL: os.Getenv("LOCAL_IMAGE_PROVIDER_URL"), Client: &http.Client{Timeout: 300 * time.Second}},
|
||||
video: multimodal.VideoUnderstandingClient{BaseURL: os.Getenv("LOCAL_VIDEO_PROVIDER_URL"), Client: &http.Client{Timeout: 300 * time.Second}},
|
||||
model: envOr("LOCAL_VISION_MODEL", "/models/gemma-4-26B-A4B-it-UD-Q4_K_M.gguf"),
|
||||
imageModel: os.Getenv("LOCAL_IMAGE_MODEL"),
|
||||
videoModel: os.Getenv("LOCAL_VIDEO_MODEL"),
|
||||
now: time.Now,
|
||||
speech: multimodal.SpeechClient{BaseURL: envOr("LOCAL_SPEECH_PROVIDER_URL", "http://llm-local-speech:8080"), Client: &http.Client{Timeout: 120 * time.Second}},
|
||||
vision: multimodal.VisionClient{BaseURL: envOr("LOCAL_VISION_PROVIDER_URL", "http://llm-local-vision:8080/v1"), Client: &http.Client{Timeout: 180 * time.Second}},
|
||||
imageGenerator: imageGenerator,
|
||||
imageProviderURL: os.Getenv("LOCAL_IMAGE_PROVIDER_URL"),
|
||||
video: multimodal.VideoUnderstandingClient{BaseURL: os.Getenv("LOCAL_VIDEO_PROVIDER_URL"), Client: &http.Client{Timeout: 300 * time.Second}},
|
||||
model: envOr("LOCAL_VISION_MODEL", "/models/gemma-4-26B-A4B-it-UD-Q4_K_M.gguf"),
|
||||
imageModel: os.Getenv("LOCAL_IMAGE_MODEL"),
|
||||
videoModel: os.Getenv("LOCAL_VIDEO_MODEL"),
|
||||
now: time.Now,
|
||||
}
|
||||
mux.HandleFunc("POST /v1/multimodal/transcribe", multimodalGateway.handler(capability.MultimodalTranscribe))
|
||||
mux.HandleFunc("POST /v1/multimodal/understand", multimodalGateway.handler(capability.MultimodalUnderstand))
|
||||
|
||||
@@ -15,14 +15,15 @@ import (
|
||||
const maxMultimodalUploadBytes = 12 << 20
|
||||
|
||||
type multimodalGateway struct {
|
||||
speech multimodal.SpeechClient
|
||||
vision multimodal.VisionClient
|
||||
imageGenerator multimodal.ImageGenerationClient
|
||||
video multimodal.VideoUnderstandingClient
|
||||
model string
|
||||
imageModel string
|
||||
videoModel string
|
||||
now func() time.Time
|
||||
speech multimodal.SpeechClient
|
||||
vision multimodal.VisionClient
|
||||
imageGenerator multimodal.ImageGenerator
|
||||
imageProviderURL string
|
||||
video multimodal.VideoUnderstandingClient
|
||||
model string
|
||||
imageModel string
|
||||
videoModel string
|
||||
now func() time.Time
|
||||
}
|
||||
|
||||
func (g multimodalGateway) handler(operation capability.MultimodalOperation) http.HandlerFunc {
|
||||
@@ -33,7 +34,7 @@ func (g multimodalGateway) handler(operation capability.MultimodalOperation) htt
|
||||
return
|
||||
}
|
||||
if operation == capability.MultimodalGenerateImage {
|
||||
if strings.TrimSpace(g.imageGenerator.BaseURL) == "" {
|
||||
if strings.TrimSpace(g.imageProviderURL) == "" {
|
||||
http.Error(w, "local image generation capability unavailable", http.StatusBadGateway)
|
||||
return
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user