103 lines
4.3 KiB
Go
103 lines
4.3 KiB
Go
package main
|
|
|
|
import (
|
|
"encoding/json"
|
|
"io"
|
|
"log"
|
|
"net/http"
|
|
"strings"
|
|
"time"
|
|
|
|
"git.cin.su/m/Assistant/adapters/multimodal"
|
|
"git.cin.su/m/Assistant/contracts/capability"
|
|
)
|
|
|
|
const maxMultimodalUploadBytes = 12 << 20
|
|
|
|
type multimodalGateway struct {
|
|
speech multimodal.SpeechClient
|
|
vision multimodal.VisionClient
|
|
imageGenerator multimodal.ImageGenerator
|
|
imageProviderURL string
|
|
video multimodal.VideoUnderstandingClient
|
|
model string
|
|
imageModel string
|
|
videoModel string
|
|
now func() time.Time
|
|
}
|
|
|
|
func (g multimodalGateway) handler(operation capability.MultimodalOperation) http.HandlerFunc {
|
|
return func(w http.ResponseWriter, r *http.Request) {
|
|
r.Body = http.MaxBytesReader(w, r.Body, maxMultimodalUploadBytes)
|
|
if err := r.ParseMultipartForm(maxMultimodalUploadBytes); err != nil {
|
|
http.Error(w, "media upload is too large or invalid", http.StatusRequestEntityTooLarge)
|
|
return
|
|
}
|
|
if operation == capability.MultimodalGenerateImage {
|
|
if strings.TrimSpace(g.imageProviderURL) == "" {
|
|
http.Error(w, "local image generation capability unavailable", http.StatusBadGateway)
|
|
return
|
|
}
|
|
image, err := g.imageGenerator.Generate(r.Context(), g.imageModel, r.FormValue("prompt"), r.FormValue("size"))
|
|
if err != nil || strings.TrimSpace(image.DataBase64) == "" {
|
|
logMultimodalFailure(operation, err)
|
|
http.Error(w, "local image generation capability unavailable", http.StatusBadGateway)
|
|
return
|
|
}
|
|
result := capability.MultimodalResponse{Operation: operation, Provider: "local-image-generator", ObservedAt: g.now().UnixMilli(), ContentType: image.ContentType, DataBase64: image.DataBase64}
|
|
w.Header().Set("Content-Type", "application/json")
|
|
_ = json.NewEncoder(w).Encode(result)
|
|
return
|
|
}
|
|
file, header, err := r.FormFile("file")
|
|
if err != nil {
|
|
http.Error(w, "media file is required", http.StatusBadRequest)
|
|
return
|
|
}
|
|
defer file.Close()
|
|
data, err := io.ReadAll(io.LimitReader(file, maxMultimodalUploadBytes+1))
|
|
if err != nil || len(data) == 0 || len(data) > maxMultimodalUploadBytes {
|
|
http.Error(w, "media file is invalid", http.StatusBadRequest)
|
|
return
|
|
}
|
|
contentType := header.Header.Get("Content-Type")
|
|
if contentType == "" {
|
|
contentType = "application/octet-stream"
|
|
}
|
|
var text string
|
|
if operation == capability.MultimodalTranscribe {
|
|
text, err = g.speech.Transcribe(r.Context(), header.Filename, contentType, data)
|
|
} else if operation == capability.MultimodalUnderstandVideo {
|
|
if strings.TrimSpace(g.video.BaseURL) == "" {
|
|
http.Error(w, "local video understanding capability unavailable", http.StatusBadGateway)
|
|
return
|
|
}
|
|
text, err = g.video.Describe(r.Context(), g.videoModel, header.Filename, contentType, data, r.FormValue("instruction"))
|
|
} else {
|
|
instruction := "Опиши, что изображено на фотографии, коротко и по существу. Не выдумывай нечитаемые детали."
|
|
if operation == capability.MultimodalRecognizeOCR {
|
|
instruction = "Точно распознай весь читаемый текст на изображении. Сохрани строки и символы; нечитаемые места обозначь как [неразборчиво]. Не добавляй пояснений."
|
|
}
|
|
text, err = g.vision.Describe(r.Context(), g.model, contentType, data, instruction)
|
|
}
|
|
if err != nil {
|
|
logMultimodalFailure(operation, err)
|
|
http.Error(w, "local media capability unavailable", http.StatusBadGateway)
|
|
return
|
|
}
|
|
result := capability.MultimodalResponse{Operation: operation, Text: strings.TrimSpace(text), ObservedAt: g.now().UnixMilli(), ConfidenceNote: "Результат получен локальным специализированным сервисом; нечитаемые или неоднозначные фрагменты требуют проверки."}
|
|
if operation == capability.MultimodalTranscribe {
|
|
result.Provider = "local-whisper"
|
|
} else {
|
|
result.Provider = "local-vision"
|
|
}
|
|
w.Header().Set("Content-Type", "application/json")
|
|
_ = json.NewEncoder(w).Encode(result)
|
|
}
|
|
}
|
|
|
|
func logMultimodalFailure(operation capability.MultimodalOperation, err error) {
|
|
// Keep provider details in server diagnostics; never return them to chat.
|
|
log.Printf("multimodal operation %s failed: %v", operation, err)
|
|
}
|