Assistant/cmd/cognitive-worker/multimodal.go

package main

import (
	"encoding/json"
	"fmt"
	"io"
	"log"
	"net/http"
	"strings"
	"time"

	"git.cin.su/m/Assistant/adapters/multimodal"
	"git.cin.su/m/Assistant/contracts/capability"
)

const maxMultimodalUploadBytes = 12 << 20

type multimodalGateway struct {
	speech           multimodal.SpeechClient
	vision           multimodal.VisionClient
	imageGenerator   multimodal.ImageGenerator
	imageProviderURL string
	video            multimodal.VideoUnderstandingClient
	model            string
	imageModel       string
	videoModel       string
	now              func() time.Time
}

func (g multimodalGateway) handler(operation capability.MultimodalOperation) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		r.Body = http.MaxBytesReader(w, r.Body, maxMultimodalUploadBytes)
		if err := r.ParseMultipartForm(maxMultimodalUploadBytes); err != nil {
			http.Error(w, "media upload is too large or invalid", http.StatusRequestEntityTooLarge)
			return
		}
		if operation == capability.MultimodalGenerateImage {
			if strings.TrimSpace(g.imageProviderURL) == "" {
				http.Error(w, "local image generation capability unavailable", http.StatusBadGateway)
				return
			}
			prompt := r.FormValue("prompt")
			image, err := g.imageGenerator.Generate(r.Context(), g.imageModel, prompt, imageRequestSize(prompt, r.FormValue("size")))
			if err != nil || strings.TrimSpace(image.DataBase64) == "" {
				logMultimodalFailure(operation, err)
				http.Error(w, "local image generation capability unavailable", http.StatusBadGateway)
				return
			}
			result := capability.MultimodalResponse{Operation: operation, Provider: "local-image-generator", ObservedAt: g.now().UnixMilli(), ContentType: image.ContentType, DataBase64: image.DataBase64}
			w.Header().Set("Content-Type", "application/json")
			_ = json.NewEncoder(w).Encode(result)
			return
		}
		file, header, err := r.FormFile("file")
		if err != nil {
			http.Error(w, "media file is required", http.StatusBadRequest)
			return
		}
		defer file.Close()
		data, err := io.ReadAll(io.LimitReader(file, maxMultimodalUploadBytes+1))
		if err != nil || len(data) == 0 || len(data) > maxMultimodalUploadBytes {
			http.Error(w, "media file is invalid", http.StatusBadRequest)
			return
		}
		contentType := header.Header.Get("Content-Type")
		if contentType == "" {
			contentType = "application/octet-stream"
		}
		var text string
		if operation == capability.MultimodalTranscribe {
			text, err = g.speech.Transcribe(r.Context(), header.Filename, contentType, data)
		} else if operation == capability.MultimodalUnderstandVideo {
			if strings.TrimSpace(g.video.BaseURL) == "" {
				http.Error(w, "local video understanding capability unavailable", http.StatusBadGateway)
				return
			}
			text, err = g.video.Describe(r.Context(), g.videoModel, header.Filename, contentType, data, r.FormValue("instruction"))
		} else {
			instruction := "Опиши, что изображено на фотографии, коротко и по существу. Не выдумывай нечитаемые детали."
			if operation == capability.MultimodalRecognizeOCR {
				instruction = "Точно распознай весь читаемый текст на изображении. Сохрани строки и символы; нечитаемые места обозначь как [неразборчиво]. Не добавляй пояснений."
			}
			text, err = g.vision.Describe(r.Context(), g.model, contentType, data, instruction)
		}
		if err != nil {
			logMultimodalFailure(operation, err)
			http.Error(w, "local media capability unavailable", http.StatusBadGateway)
			return
		}
		result := capability.MultimodalResponse{Operation: operation, Text: strings.TrimSpace(text), ObservedAt: g.now().UnixMilli(), ConfidenceNote: "Результат получен локальным специализированным сервисом; нечитаемые или неоднозначные фрагменты требуют проверки."}
		if operation == capability.MultimodalTranscribe {
			result.Provider = "local-whisper"
		} else {
			result.Provider = "local-vision"
		}
		w.Header().Set("Content-Type", "application/json")
		_ = json.NewEncoder(w).Encode(result)
	}
}

// imageRequestSize is a transport-level normalization for the local image
// capability.  An explicit, valid size always wins.  When the caller only
// expresses a conventional output-quality target, the capability translates it
// to a supported provider size without exposing provider workflow details to
// the interaction core.  This policy is deliberately local and replaceable
// with the image capability implementation.
func imageRequestSize(prompt, requested string) string {
	if isImageSize(requested) {
		return strings.TrimSpace(requested)
	}
	words := strings.Fields(strings.ToLower(prompt))
	for index, word := range words {
		switch word {
		case "4k", "uhd":
			return "2048x2048"
		case "hd":
			return "1024x1024"
		case "ultra":
			if index+1 < len(words) && words[index+1] == "hd" {
				return "2048x2048"
			}
		}
	}
	return ""
}

func isImageSize(value string) bool {
	var width, height int
	if _, err := fmt.Sscanf(strings.TrimSpace(value), "%dx%d", &width, &height); err != nil {
		return false
	}
	return width >= 256 && height >= 256 && width <= 2048 && height <= 2048
}

func logMultimodalFailure(operation capability.MultimodalOperation, err error) {
	// Keep provider details in server diagnostics; never return them to chat.
	log.Printf("multimodal operation %s failed: %v", operation, err)
}