package main import ( "encoding/json" "fmt" "io" "log" "net/http" "strings" "time" "git.cin.su/m/Assistant/adapters/multimodal" "git.cin.su/m/Assistant/contracts/capability" ) const maxMultimodalUploadBytes = 12 << 20 type multimodalGateway struct { speech multimodal.SpeechClient vision multimodal.VisionClient imageGenerator multimodal.ImageGenerator imageProviderURL string video multimodal.VideoUnderstandingClient model string imageModel string videoModel string now func() time.Time } func (g multimodalGateway) handler(operation capability.MultimodalOperation) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { r.Body = http.MaxBytesReader(w, r.Body, maxMultimodalUploadBytes) if err := r.ParseMultipartForm(maxMultimodalUploadBytes); err != nil { http.Error(w, "media upload is too large or invalid", http.StatusRequestEntityTooLarge) return } if operation == capability.MultimodalGenerateImage { if strings.TrimSpace(g.imageProviderURL) == "" { http.Error(w, "local image generation capability unavailable", http.StatusBadGateway) return } prompt := r.FormValue("prompt") image, err := g.imageGenerator.Generate(r.Context(), g.imageModel, prompt, imageRequestSize(prompt, r.FormValue("size"))) if err != nil || strings.TrimSpace(image.DataBase64) == "" { logMultimodalFailure(operation, err) http.Error(w, "local image generation capability unavailable", http.StatusBadGateway) return } result := capability.MultimodalResponse{Operation: operation, Provider: "local-image-generator", ObservedAt: g.now().UnixMilli(), ContentType: image.ContentType, DataBase64: image.DataBase64} w.Header().Set("Content-Type", "application/json") _ = json.NewEncoder(w).Encode(result) return } file, header, err := r.FormFile("file") if err != nil { http.Error(w, "media file is required", http.StatusBadRequest) return } defer file.Close() data, err := io.ReadAll(io.LimitReader(file, maxMultimodalUploadBytes+1)) if err != nil || len(data) == 0 || len(data) > maxMultimodalUploadBytes { http.Error(w, "media file is invalid", http.StatusBadRequest) return } contentType := header.Header.Get("Content-Type") if contentType == "" { contentType = "application/octet-stream" } var text string if operation == capability.MultimodalTranscribe { text, err = g.speech.Transcribe(r.Context(), header.Filename, contentType, data) } else if operation == capability.MultimodalUnderstandVideo { if strings.TrimSpace(g.video.BaseURL) == "" { http.Error(w, "local video understanding capability unavailable", http.StatusBadGateway) return } text, err = g.video.Describe(r.Context(), g.videoModel, header.Filename, contentType, data, r.FormValue("instruction")) } else { instruction := "Опиши, что изображено на фотографии, коротко и по существу. Не выдумывай нечитаемые детали." if operation == capability.MultimodalRecognizeOCR { instruction = "Точно распознай весь читаемый текст на изображении. Сохрани строки и символы; нечитаемые места обозначь как [неразборчиво]. Не добавляй пояснений." } text, err = g.vision.Describe(r.Context(), g.model, contentType, data, instruction) } if err != nil { logMultimodalFailure(operation, err) http.Error(w, "local media capability unavailable", http.StatusBadGateway) return } result := capability.MultimodalResponse{Operation: operation, Text: strings.TrimSpace(text), ObservedAt: g.now().UnixMilli(), ConfidenceNote: "Результат получен локальным специализированным сервисом; нечитаемые или неоднозначные фрагменты требуют проверки."} if operation == capability.MultimodalTranscribe { result.Provider = "local-whisper" } else { result.Provider = "local-vision" } w.Header().Set("Content-Type", "application/json") _ = json.NewEncoder(w).Encode(result) } } // imageRequestSize is a transport-level normalization for the local image // capability. An explicit, valid size always wins. When the caller only // expresses a conventional output-quality target, the capability translates it // to a supported provider size without exposing provider workflow details to // the interaction core. This policy is deliberately local and replaceable // with the image capability implementation. func imageRequestSize(prompt, requested string) string { if isImageSize(requested) { return strings.TrimSpace(requested) } words := strings.Fields(strings.ToLower(prompt)) for index, word := range words { switch word { case "4k", "uhd": return "2048x2048" case "hd": return "1024x1024" case "ultra": if index+1 < len(words) && words[index+1] == "hd" { return "2048x2048" } } } return "" } func isImageSize(value string) bool { var width, height int if _, err := fmt.Sscanf(strings.TrimSpace(value), "%dx%d", &width, &height); err != nil { return false } return width >= 256 && height >= 256 && width <= 2048 && height <= 2048 } func logMultimodalFailure(operation capability.MultimodalOperation, err error) { // Keep provider details in server diagnostics; never return them to chat. log.Printf("multimodal operation %s failed: %v", operation, err) }