Assistant/adapters/multimodal/client.go

package multimodal

import (
	"bytes"
	"context"
	"encoding/base64"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"mime/multipart"
	"net/http"
	"net/textproto"
	"strings"
)

var ErrEmptyResult = errors.New("multimodal provider returned an empty result")

type SpeechClient struct {
	BaseURL string
	Client  *http.Client
}

type VisionClient struct {
	BaseURL string
	Client  *http.Client
}

func (c SpeechClient) Transcribe(ctx context.Context, filename, contentType string, data []byte) (string, error) {
	if len(data) == 0 {
		return "", errors.New("audio payload is empty")
	}
	body := &bytes.Buffer{}
	writer := multipart.NewWriter(body)
	header := make(textproto.MIMEHeader)
	header.Set("Content-Disposition", fmt.Sprintf(`form-data; name="file"; filename="%s"`, strings.ReplaceAll(filename, `"`, "")))
	header.Set("Content-Type", contentType)
	part, err := writer.CreatePart(header)
	if err != nil {
		return "", err
	}
	if _, err := part.Write(data); err != nil {
		return "", err
	}
	if err := writer.WriteField("response_format", "json"); err != nil {
		return "", err
	}
	if err := writer.Close(); err != nil {
		return "", err
	}
	request, err := http.NewRequestWithContext(ctx, http.MethodPost, strings.TrimRight(c.BaseURL, "/")+"/inference", body)
	if err != nil {
		return "", err
	}
	request.Header.Set("Content-Type", writer.FormDataContentType())
	response, err := c.client().Do(request)
	if err != nil {
		return "", err
	}
	defer response.Body.Close()
	if response.StatusCode != http.StatusOK {
		return "", fmt.Errorf("speech provider status %d", response.StatusCode)
	}
	var result struct {
		Text string `json:"text"`
	}
	if err := json.NewDecoder(io.LimitReader(response.Body, 1<<20)).Decode(&result); err != nil {
		return "", err
	}
	if strings.TrimSpace(result.Text) == "" {
		return "", ErrEmptyResult
	}
	return strings.TrimSpace(result.Text), nil
}

func (c VisionClient) Describe(ctx context.Context, model, contentType string, data []byte, instruction string) (string, error) {
	if len(data) == 0 {
		return "", errors.New("image payload is empty")
	}
	payload := struct {
		Model    string `json:"model"`
		Messages []struct {
			Role    string `json:"role"`
			Content []struct {
				Type     string `json:"type"`
				Text     string `json:"text,omitempty"`
				ImageURL *struct {
					URL string `json:"url"`
				} `json:"image_url,omitempty"`
			} `json:"content"`
		} `json:"messages"`
		MaxTokens int `json:"max_tokens"`
	}{}
	payload.Model = model
	payload.MaxTokens = 1024
	payload.Messages = append(payload.Messages, struct {
		Role    string `json:"role"`
		Content []struct {
			Type     string `json:"type"`
			Text     string `json:"text,omitempty"`
			ImageURL *struct {
				URL string `json:"url"`
			} `json:"image_url,omitempty"`
		} `json:"content"`
	}{Role: "user", Content: []struct {
		Type     string `json:"type"`
		Text     string `json:"text,omitempty"`
		ImageURL *struct {
			URL string `json:"url"`
		} `json:"image_url,omitempty"`
	}{
		{Type: "text", Text: instruction},
		{Type: "image_url", ImageURL: &struct {
			URL string `json:"url"`
		}{URL: "data:" + contentType + ";base64," + encodeBase64(data)},
		}}})
	encoded, err := json.Marshal(payload)
	if err != nil {
		return "", err
	}
	request, err := http.NewRequestWithContext(ctx, http.MethodPost, strings.TrimRight(c.BaseURL, "/")+"/chat/completions", bytes.NewReader(encoded))
	if err != nil {
		return "", err
	}
	request.Header.Set("Content-Type", "application/json")
	response, err := c.client().Do(request)
	if err != nil {
		return "", err
	}
	defer response.Body.Close()
	if response.StatusCode != http.StatusOK {
		return "", fmt.Errorf("vision provider status %d", response.StatusCode)
	}
	var result struct {
		Choices []struct {
			Message struct {
				Content          string `json:"content"`
				ReasoningContent string `json:"reasoning_content"`
			} `json:"message"`
		} `json:"choices"`
	}
	if err := json.NewDecoder(io.LimitReader(response.Body, 2<<20)).Decode(&result); err != nil {
		return "", err
	}
	if len(result.Choices) == 0 {
		return "", ErrEmptyResult
	}
	text := strings.TrimSpace(result.Choices[0].Message.Content)
	if text == "" {
		text = strings.TrimSpace(result.Choices[0].Message.ReasoningContent)
	}
	if text == "" {
		return "", ErrEmptyResult
	}
	return text, nil
}

func (c SpeechClient) client() *http.Client {
	if c.Client != nil {
		return c.Client
	}
	return http.DefaultClient
}

func (c VisionClient) client() *http.Client {
	if c.Client != nil {
		return c.Client
	}
	return http.DefaultClient
}

func encodeBase64(data []byte) string {
	return base64.StdEncoding.EncodeToString(data)
}