This commit is contained in:
@@ -172,14 +172,20 @@ func main() {
|
||||
mux.HandleFunc("POST /v1/interpretations", gateway.interpretationHandler)
|
||||
mux.HandleFunc("POST /v1/task-evaluations/execute", gateway.taskEvaluationHandler)
|
||||
multimodalGateway := multimodalGateway{
|
||||
speech: multimodal.SpeechClient{BaseURL: envOr("LOCAL_SPEECH_PROVIDER_URL", "http://llm-local-speech:8080"), Client: &http.Client{Timeout: 120 * time.Second}},
|
||||
vision: multimodal.VisionClient{BaseURL: envOr("LOCAL_VISION_PROVIDER_URL", "http://llm-local-vision:8080/v1"), Client: &http.Client{Timeout: 180 * time.Second}},
|
||||
model: envOr("LOCAL_VISION_MODEL", "/models/gemma-4-26B-A4B-it-UD-Q4_K_M.gguf"),
|
||||
now: time.Now,
|
||||
speech: multimodal.SpeechClient{BaseURL: envOr("LOCAL_SPEECH_PROVIDER_URL", "http://llm-local-speech:8080"), Client: &http.Client{Timeout: 120 * time.Second}},
|
||||
vision: multimodal.VisionClient{BaseURL: envOr("LOCAL_VISION_PROVIDER_URL", "http://llm-local-vision:8080/v1"), Client: &http.Client{Timeout: 180 * time.Second}},
|
||||
imageGenerator: multimodal.ImageGenerationClient{BaseURL: os.Getenv("LOCAL_IMAGE_PROVIDER_URL"), Client: &http.Client{Timeout: 300 * time.Second}},
|
||||
video: multimodal.VideoUnderstandingClient{BaseURL: os.Getenv("LOCAL_VIDEO_PROVIDER_URL"), Client: &http.Client{Timeout: 300 * time.Second}},
|
||||
model: envOr("LOCAL_VISION_MODEL", "/models/gemma-4-26B-A4B-it-UD-Q4_K_M.gguf"),
|
||||
imageModel: os.Getenv("LOCAL_IMAGE_MODEL"),
|
||||
videoModel: os.Getenv("LOCAL_VIDEO_MODEL"),
|
||||
now: time.Now,
|
||||
}
|
||||
mux.HandleFunc("POST /v1/multimodal/transcribe", multimodalGateway.handler(capability.MultimodalTranscribe))
|
||||
mux.HandleFunc("POST /v1/multimodal/understand", multimodalGateway.handler(capability.MultimodalUnderstand))
|
||||
mux.HandleFunc("POST /v1/multimodal/ocr", multimodalGateway.handler(capability.MultimodalRecognizeOCR))
|
||||
mux.HandleFunc("POST /v1/multimodal/video-understand", multimodalGateway.handler(capability.MultimodalUnderstandVideo))
|
||||
mux.HandleFunc("POST /v1/multimodal/generate-image", multimodalGateway.handler(capability.MultimodalGenerateImage))
|
||||
discovery := discoveryService{resolver: resolver, providers: discoveries, owner: identity.PrincipalID(envOr("COGNITIVE_OWNER_PRINCIPAL_ID", "domovoy-creator-principal")), now: time.Now}
|
||||
mux.HandleFunc("POST /v1/provider-discovery/{provider}", discovery.handler)
|
||||
mux.Handle("POST /v1/provider-discovery/{provider}/qualify", modelQualificationHandler(discoveries, string(envOr("COGNITIVE_OWNER_PRINCIPAL_ID", "domovoy-creator-principal")), httpQualificationObserver{resolver: resolver}))
|
||||
|
||||
@@ -15,10 +15,14 @@ import (
|
||||
const maxMultimodalUploadBytes = 12 << 20
|
||||
|
||||
type multimodalGateway struct {
|
||||
speech multimodal.SpeechClient
|
||||
vision multimodal.VisionClient
|
||||
model string
|
||||
now func() time.Time
|
||||
speech multimodal.SpeechClient
|
||||
vision multimodal.VisionClient
|
||||
imageGenerator multimodal.ImageGenerationClient
|
||||
video multimodal.VideoUnderstandingClient
|
||||
model string
|
||||
imageModel string
|
||||
videoModel string
|
||||
now func() time.Time
|
||||
}
|
||||
|
||||
func (g multimodalGateway) handler(operation capability.MultimodalOperation) http.HandlerFunc {
|
||||
@@ -28,6 +32,22 @@ func (g multimodalGateway) handler(operation capability.MultimodalOperation) htt
|
||||
http.Error(w, "media upload is too large or invalid", http.StatusRequestEntityTooLarge)
|
||||
return
|
||||
}
|
||||
if operation == capability.MultimodalGenerateImage {
|
||||
if strings.TrimSpace(g.imageGenerator.BaseURL) == "" {
|
||||
http.Error(w, "local image generation capability unavailable", http.StatusBadGateway)
|
||||
return
|
||||
}
|
||||
image, err := g.imageGenerator.Generate(r.Context(), g.imageModel, r.FormValue("prompt"), r.FormValue("size"))
|
||||
if err != nil || strings.TrimSpace(image.DataBase64) == "" {
|
||||
logMultimodalFailure(operation, err)
|
||||
http.Error(w, "local image generation capability unavailable", http.StatusBadGateway)
|
||||
return
|
||||
}
|
||||
result := capability.MultimodalResponse{Operation: operation, Provider: "local-image-generator", ObservedAt: g.now().UnixMilli(), ContentType: image.ContentType, DataBase64: image.DataBase64}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_ = json.NewEncoder(w).Encode(result)
|
||||
return
|
||||
}
|
||||
file, header, err := r.FormFile("file")
|
||||
if err != nil {
|
||||
http.Error(w, "media file is required", http.StatusBadRequest)
|
||||
@@ -46,6 +66,12 @@ func (g multimodalGateway) handler(operation capability.MultimodalOperation) htt
|
||||
var text string
|
||||
if operation == capability.MultimodalTranscribe {
|
||||
text, err = g.speech.Transcribe(r.Context(), header.Filename, contentType, data)
|
||||
} else if operation == capability.MultimodalUnderstandVideo {
|
||||
if strings.TrimSpace(g.video.BaseURL) == "" {
|
||||
http.Error(w, "local video understanding capability unavailable", http.StatusBadGateway)
|
||||
return
|
||||
}
|
||||
text, err = g.video.Describe(r.Context(), g.videoModel, header.Filename, contentType, data, r.FormValue("instruction"))
|
||||
} else {
|
||||
instruction := "Опиши, что изображено на фотографии, коротко и по существу. Не выдумывай нечитаемые детали."
|
||||
if operation == capability.MultimodalRecognizeOCR {
|
||||
|
||||
+39
-23
@@ -688,17 +688,6 @@ func (g *gateway) handleMultimodal(w http.ResponseWriter, r *http.Request) {
|
||||
http.Error(w, "media upload is too large or invalid", http.StatusRequestEntityTooLarge)
|
||||
return
|
||||
}
|
||||
file, header, err := r.FormFile("file")
|
||||
if err != nil {
|
||||
http.Error(w, "media file is required", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
defer file.Close()
|
||||
data, err := io.ReadAll(io.LimitReader(file, maxMediaBytes+1))
|
||||
if err != nil || len(data) == 0 || len(data) > maxMediaBytes {
|
||||
http.Error(w, "media file is invalid", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
path := ""
|
||||
switch r.URL.Path {
|
||||
case "/api/multimodal/transcribe":
|
||||
@@ -707,25 +696,52 @@ func (g *gateway) handleMultimodal(w http.ResponseWriter, r *http.Request) {
|
||||
path = "/v1/multimodal/understand"
|
||||
case "/api/multimodal/ocr":
|
||||
path = "/v1/multimodal/ocr"
|
||||
case "/api/multimodal/video-understand":
|
||||
path = "/v1/multimodal/video-understand"
|
||||
case "/api/multimodal/generate-image":
|
||||
path = "/v1/multimodal/generate-image"
|
||||
default:
|
||||
http.Error(w, "multimodal route not found", http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
body := &bytes.Buffer{}
|
||||
writer := multipart.NewWriter(body)
|
||||
mediaHeader := make(textproto.MIMEHeader)
|
||||
mediaHeader.Set("Content-Disposition", fmt.Sprintf(`form-data; name="file"; filename="%s"`, strings.ReplaceAll(header.Filename, `"`, "")))
|
||||
mediaType := header.Header.Get("Content-Type")
|
||||
if mediaType == "" {
|
||||
mediaType = "application/octet-stream"
|
||||
if r.URL.Path == "/api/multimodal/generate-image" {
|
||||
_ = writer.WriteField("prompt", r.FormValue("prompt"))
|
||||
_ = writer.WriteField("size", r.FormValue("size"))
|
||||
} else {
|
||||
if r.URL.Path == "/api/multimodal/video-understand" {
|
||||
_ = writer.WriteField("instruction", r.FormValue("instruction"))
|
||||
}
|
||||
file, header, err := r.FormFile("file")
|
||||
if err != nil {
|
||||
http.Error(w, "media file is required", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
defer file.Close()
|
||||
data, err := io.ReadAll(io.LimitReader(file, maxMediaBytes+1))
|
||||
if err != nil || len(data) == 0 || len(data) > maxMediaBytes {
|
||||
http.Error(w, "media file is invalid", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
mediaHeader := make(textproto.MIMEHeader)
|
||||
mediaHeader.Set("Content-Disposition", fmt.Sprintf(`form-data; name="file"; filename="%s"`, strings.ReplaceAll(header.Filename, `"`, "")))
|
||||
mediaType := header.Header.Get("Content-Type")
|
||||
if mediaType == "" {
|
||||
mediaType = "application/octet-stream"
|
||||
}
|
||||
mediaHeader.Set("Content-Type", mediaType)
|
||||
part, err := writer.CreatePart(mediaHeader)
|
||||
if err != nil {
|
||||
http.Error(w, "media upload unavailable", http.StatusServiceUnavailable)
|
||||
return
|
||||
}
|
||||
if _, err := part.Write(data); err != nil {
|
||||
http.Error(w, "media upload unavailable", http.StatusServiceUnavailable)
|
||||
return
|
||||
}
|
||||
}
|
||||
mediaHeader.Set("Content-Type", mediaType)
|
||||
part, err := writer.CreatePart(mediaHeader)
|
||||
if err != nil {
|
||||
http.Error(w, "media upload unavailable", http.StatusServiceUnavailable)
|
||||
return
|
||||
}
|
||||
if _, err := part.Write(data); err != nil || writer.Close() != nil {
|
||||
if err := writer.Close(); err != nil {
|
||||
http.Error(w, "media upload unavailable", http.StatusServiceUnavailable)
|
||||
return
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user