Add replaceable image and video media routes
CI / test (push) Successful in 1m12s

This commit is contained in:
m
2026-08-25 16:12:33 +03:00
parent cd8ba740de
commit 8260e69dfe
5 changed files with 257 additions and 31 deletions
+10 -4
View File
@@ -172,14 +172,20 @@ func main() {
mux.HandleFunc("POST /v1/interpretations", gateway.interpretationHandler)
mux.HandleFunc("POST /v1/task-evaluations/execute", gateway.taskEvaluationHandler)
multimodalGateway := multimodalGateway{
speech: multimodal.SpeechClient{BaseURL: envOr("LOCAL_SPEECH_PROVIDER_URL", "http://llm-local-speech:8080"), Client: &http.Client{Timeout: 120 * time.Second}},
vision: multimodal.VisionClient{BaseURL: envOr("LOCAL_VISION_PROVIDER_URL", "http://llm-local-vision:8080/v1"), Client: &http.Client{Timeout: 180 * time.Second}},
model: envOr("LOCAL_VISION_MODEL", "/models/gemma-4-26B-A4B-it-UD-Q4_K_M.gguf"),
now: time.Now,
speech: multimodal.SpeechClient{BaseURL: envOr("LOCAL_SPEECH_PROVIDER_URL", "http://llm-local-speech:8080"), Client: &http.Client{Timeout: 120 * time.Second}},
vision: multimodal.VisionClient{BaseURL: envOr("LOCAL_VISION_PROVIDER_URL", "http://llm-local-vision:8080/v1"), Client: &http.Client{Timeout: 180 * time.Second}},
imageGenerator: multimodal.ImageGenerationClient{BaseURL: os.Getenv("LOCAL_IMAGE_PROVIDER_URL"), Client: &http.Client{Timeout: 300 * time.Second}},
video: multimodal.VideoUnderstandingClient{BaseURL: os.Getenv("LOCAL_VIDEO_PROVIDER_URL"), Client: &http.Client{Timeout: 300 * time.Second}},
model: envOr("LOCAL_VISION_MODEL", "/models/gemma-4-26B-A4B-it-UD-Q4_K_M.gguf"),
imageModel: os.Getenv("LOCAL_IMAGE_MODEL"),
videoModel: os.Getenv("LOCAL_VIDEO_MODEL"),
now: time.Now,
}
mux.HandleFunc("POST /v1/multimodal/transcribe", multimodalGateway.handler(capability.MultimodalTranscribe))
mux.HandleFunc("POST /v1/multimodal/understand", multimodalGateway.handler(capability.MultimodalUnderstand))
mux.HandleFunc("POST /v1/multimodal/ocr", multimodalGateway.handler(capability.MultimodalRecognizeOCR))
mux.HandleFunc("POST /v1/multimodal/video-understand", multimodalGateway.handler(capability.MultimodalUnderstandVideo))
mux.HandleFunc("POST /v1/multimodal/generate-image", multimodalGateway.handler(capability.MultimodalGenerateImage))
discovery := discoveryService{resolver: resolver, providers: discoveries, owner: identity.PrincipalID(envOr("COGNITIVE_OWNER_PRINCIPAL_ID", "domovoy-creator-principal")), now: time.Now}
mux.HandleFunc("POST /v1/provider-discovery/{provider}", discovery.handler)
mux.Handle("POST /v1/provider-discovery/{provider}/qualify", modelQualificationHandler(discoveries, string(envOr("COGNITIVE_OWNER_PRINCIPAL_ID", "domovoy-creator-principal")), httpQualificationObserver{resolver: resolver}))
+30 -4
View File
@@ -15,10 +15,14 @@ import (
const maxMultimodalUploadBytes = 12 << 20
type multimodalGateway struct {
speech multimodal.SpeechClient
vision multimodal.VisionClient
model string
now func() time.Time
speech multimodal.SpeechClient
vision multimodal.VisionClient
imageGenerator multimodal.ImageGenerationClient
video multimodal.VideoUnderstandingClient
model string
imageModel string
videoModel string
now func() time.Time
}
func (g multimodalGateway) handler(operation capability.MultimodalOperation) http.HandlerFunc {
@@ -28,6 +32,22 @@ func (g multimodalGateway) handler(operation capability.MultimodalOperation) htt
http.Error(w, "media upload is too large or invalid", http.StatusRequestEntityTooLarge)
return
}
if operation == capability.MultimodalGenerateImage {
if strings.TrimSpace(g.imageGenerator.BaseURL) == "" {
http.Error(w, "local image generation capability unavailable", http.StatusBadGateway)
return
}
image, err := g.imageGenerator.Generate(r.Context(), g.imageModel, r.FormValue("prompt"), r.FormValue("size"))
if err != nil || strings.TrimSpace(image.DataBase64) == "" {
logMultimodalFailure(operation, err)
http.Error(w, "local image generation capability unavailable", http.StatusBadGateway)
return
}
result := capability.MultimodalResponse{Operation: operation, Provider: "local-image-generator", ObservedAt: g.now().UnixMilli(), ContentType: image.ContentType, DataBase64: image.DataBase64}
w.Header().Set("Content-Type", "application/json")
_ = json.NewEncoder(w).Encode(result)
return
}
file, header, err := r.FormFile("file")
if err != nil {
http.Error(w, "media file is required", http.StatusBadRequest)
@@ -46,6 +66,12 @@ func (g multimodalGateway) handler(operation capability.MultimodalOperation) htt
var text string
if operation == capability.MultimodalTranscribe {
text, err = g.speech.Transcribe(r.Context(), header.Filename, contentType, data)
} else if operation == capability.MultimodalUnderstandVideo {
if strings.TrimSpace(g.video.BaseURL) == "" {
http.Error(w, "local video understanding capability unavailable", http.StatusBadGateway)
return
}
text, err = g.video.Describe(r.Context(), g.videoModel, header.Filename, contentType, data, r.FormValue("instruction"))
} else {
instruction := "Опиши, что изображено на фотографии, коротко и по существу. Не выдумывай нечитаемые детали."
if operation == capability.MultimodalRecognizeOCR {
+39 -23
View File
@@ -688,17 +688,6 @@ func (g *gateway) handleMultimodal(w http.ResponseWriter, r *http.Request) {
http.Error(w, "media upload is too large or invalid", http.StatusRequestEntityTooLarge)
return
}
file, header, err := r.FormFile("file")
if err != nil {
http.Error(w, "media file is required", http.StatusBadRequest)
return
}
defer file.Close()
data, err := io.ReadAll(io.LimitReader(file, maxMediaBytes+1))
if err != nil || len(data) == 0 || len(data) > maxMediaBytes {
http.Error(w, "media file is invalid", http.StatusBadRequest)
return
}
path := ""
switch r.URL.Path {
case "/api/multimodal/transcribe":
@@ -707,25 +696,52 @@ func (g *gateway) handleMultimodal(w http.ResponseWriter, r *http.Request) {
path = "/v1/multimodal/understand"
case "/api/multimodal/ocr":
path = "/v1/multimodal/ocr"
case "/api/multimodal/video-understand":
path = "/v1/multimodal/video-understand"
case "/api/multimodal/generate-image":
path = "/v1/multimodal/generate-image"
default:
http.Error(w, "multimodal route not found", http.StatusNotFound)
return
}
body := &bytes.Buffer{}
writer := multipart.NewWriter(body)
mediaHeader := make(textproto.MIMEHeader)
mediaHeader.Set("Content-Disposition", fmt.Sprintf(`form-data; name="file"; filename="%s"`, strings.ReplaceAll(header.Filename, `"`, "")))
mediaType := header.Header.Get("Content-Type")
if mediaType == "" {
mediaType = "application/octet-stream"
if r.URL.Path == "/api/multimodal/generate-image" {
_ = writer.WriteField("prompt", r.FormValue("prompt"))
_ = writer.WriteField("size", r.FormValue("size"))
} else {
if r.URL.Path == "/api/multimodal/video-understand" {
_ = writer.WriteField("instruction", r.FormValue("instruction"))
}
file, header, err := r.FormFile("file")
if err != nil {
http.Error(w, "media file is required", http.StatusBadRequest)
return
}
defer file.Close()
data, err := io.ReadAll(io.LimitReader(file, maxMediaBytes+1))
if err != nil || len(data) == 0 || len(data) > maxMediaBytes {
http.Error(w, "media file is invalid", http.StatusBadRequest)
return
}
mediaHeader := make(textproto.MIMEHeader)
mediaHeader.Set("Content-Disposition", fmt.Sprintf(`form-data; name="file"; filename="%s"`, strings.ReplaceAll(header.Filename, `"`, "")))
mediaType := header.Header.Get("Content-Type")
if mediaType == "" {
mediaType = "application/octet-stream"
}
mediaHeader.Set("Content-Type", mediaType)
part, err := writer.CreatePart(mediaHeader)
if err != nil {
http.Error(w, "media upload unavailable", http.StatusServiceUnavailable)
return
}
if _, err := part.Write(data); err != nil {
http.Error(w, "media upload unavailable", http.StatusServiceUnavailable)
return
}
}
mediaHeader.Set("Content-Type", mediaType)
part, err := writer.CreatePart(mediaHeader)
if err != nil {
http.Error(w, "media upload unavailable", http.StatusServiceUnavailable)
return
}
if _, err := part.Write(data); err != nil || writer.Close() != nil {
if err := writer.Close(); err != nil {
http.Error(w, "media upload unavailable", http.StatusServiceUnavailable)
return
}