Files
Assistant/adapters/multimodal/client.go
T
2026-08-25 16:12:33 +03:00

299 lines
8.6 KiB
Go

package multimodal
import (
"bytes"
"context"
"encoding/base64"
"encoding/json"
"errors"
"fmt"
"io"
"mime/multipart"
"net/http"
"net/textproto"
"strings"
)
var ErrEmptyResult = errors.New("multimodal provider returned an empty result")
type SpeechClient struct {
BaseURL string
Client *http.Client
}
type VisionClient struct {
BaseURL string
Client *http.Client
}
type ImageGenerationClient struct {
BaseURL string
Client *http.Client
}
type VideoUnderstandingClient struct {
BaseURL string
Client *http.Client
}
type GeneratedImage struct {
DataBase64 string
SourceURL string
ContentType string
}
func (c SpeechClient) Transcribe(ctx context.Context, filename, contentType string, data []byte) (string, error) {
if len(data) == 0 {
return "", errors.New("audio payload is empty")
}
body := &bytes.Buffer{}
writer := multipart.NewWriter(body)
header := make(textproto.MIMEHeader)
header.Set("Content-Disposition", fmt.Sprintf(`form-data; name="file"; filename="%s"`, strings.ReplaceAll(filename, `"`, "")))
header.Set("Content-Type", contentType)
part, err := writer.CreatePart(header)
if err != nil {
return "", err
}
if _, err := part.Write(data); err != nil {
return "", err
}
if err := writer.WriteField("response_format", "json"); err != nil {
return "", err
}
if err := writer.Close(); err != nil {
return "", err
}
request, err := http.NewRequestWithContext(ctx, http.MethodPost, strings.TrimRight(c.BaseURL, "/")+"/inference", body)
if err != nil {
return "", err
}
request.Header.Set("Content-Type", writer.FormDataContentType())
response, err := c.client().Do(request)
if err != nil {
return "", err
}
defer response.Body.Close()
if response.StatusCode != http.StatusOK {
return "", fmt.Errorf("speech provider status %d", response.StatusCode)
}
var result struct {
Text string `json:"text"`
}
if err := json.NewDecoder(io.LimitReader(response.Body, 1<<20)).Decode(&result); err != nil {
return "", err
}
if strings.TrimSpace(result.Text) == "" {
return "", ErrEmptyResult
}
return strings.TrimSpace(result.Text), nil
}
func (c VisionClient) Describe(ctx context.Context, model, contentType string, data []byte, instruction string) (string, error) {
if len(data) == 0 {
return "", errors.New("image payload is empty")
}
payload := struct {
Model string `json:"model"`
Messages []struct {
Role string `json:"role"`
Content []struct {
Type string `json:"type"`
Text string `json:"text,omitempty"`
ImageURL *struct {
URL string `json:"url"`
} `json:"image_url,omitempty"`
} `json:"content"`
} `json:"messages"`
MaxTokens int `json:"max_tokens"`
}{}
payload.Model = model
payload.MaxTokens = 1024
payload.Messages = append(payload.Messages, struct {
Role string `json:"role"`
Content []struct {
Type string `json:"type"`
Text string `json:"text,omitempty"`
ImageURL *struct {
URL string `json:"url"`
} `json:"image_url,omitempty"`
} `json:"content"`
}{Role: "user", Content: []struct {
Type string `json:"type"`
Text string `json:"text,omitempty"`
ImageURL *struct {
URL string `json:"url"`
} `json:"image_url,omitempty"`
}{
{Type: "text", Text: instruction},
{Type: "image_url", ImageURL: &struct {
URL string `json:"url"`
}{URL: "data:" + contentType + ";base64," + encodeBase64(data)},
}}})
encoded, err := json.Marshal(payload)
if err != nil {
return "", err
}
request, err := http.NewRequestWithContext(ctx, http.MethodPost, strings.TrimRight(c.BaseURL, "/")+"/chat/completions", bytes.NewReader(encoded))
if err != nil {
return "", err
}
request.Header.Set("Content-Type", "application/json")
response, err := c.client().Do(request)
if err != nil {
return "", err
}
defer response.Body.Close()
if response.StatusCode != http.StatusOK {
return "", fmt.Errorf("vision provider status %d", response.StatusCode)
}
var result struct {
Choices []struct {
Message struct {
Content string `json:"content"`
ReasoningContent string `json:"reasoning_content"`
} `json:"message"`
} `json:"choices"`
}
if err := json.NewDecoder(io.LimitReader(response.Body, 2<<20)).Decode(&result); err != nil {
return "", err
}
if len(result.Choices) == 0 {
return "", ErrEmptyResult
}
text := strings.TrimSpace(result.Choices[0].Message.Content)
if text == "" {
text = strings.TrimSpace(result.Choices[0].Message.ReasoningContent)
}
if text == "" {
return "", ErrEmptyResult
}
return text, nil
}
// Generate uses an OpenAI-compatible image-generation skill. The provider is
// selected outside the core by the capability binding; this adapter contains
// transport only and never decides what the user meant.
func (c ImageGenerationClient) Generate(ctx context.Context, model, prompt, size string) (GeneratedImage, error) {
if strings.TrimSpace(prompt) == "" {
return GeneratedImage{}, errors.New("image prompt is empty")
}
payload := struct {
Model string `json:"model"`
Prompt string `json:"prompt"`
Size string `json:"size,omitempty"`
ResponseFormat string `json:"response_format,omitempty"`
}{Model: model, Prompt: prompt, Size: size, ResponseFormat: "b64_json"}
encoded, err := json.Marshal(payload)
if err != nil {
return GeneratedImage{}, err
}
request, err := http.NewRequestWithContext(ctx, http.MethodPost, strings.TrimRight(c.BaseURL, "/")+"/images/generations", bytes.NewReader(encoded))
if err != nil {
return GeneratedImage{}, err
}
request.Header.Set("Content-Type", "application/json")
response, err := c.client().Do(request)
if err != nil {
return GeneratedImage{}, err
}
defer response.Body.Close()
if response.StatusCode < 200 || response.StatusCode > 299 {
return GeneratedImage{}, fmt.Errorf("image provider status %d", response.StatusCode)
}
var result struct {
Data []struct {
Base64 string `json:"b64_json"`
URL string `json:"url"`
} `json:"data"`
}
if err := json.NewDecoder(io.LimitReader(response.Body, 32<<20)).Decode(&result); err != nil {
return GeneratedImage{}, err
}
if len(result.Data) == 0 || (strings.TrimSpace(result.Data[0].Base64) == "" && strings.TrimSpace(result.Data[0].URL) == "") {
return GeneratedImage{}, ErrEmptyResult
}
return GeneratedImage{DataBase64: result.Data[0].Base64, SourceURL: result.Data[0].URL, ContentType: "image/png"}, nil
}
// Describe accepts a video skill endpoint that performs its own bounded frame
// sampling. Frame extraction is provider-owned so the core does not acquire a
// media-processing dependency or hide a policy in orchestration code.
func (c VideoUnderstandingClient) Describe(ctx context.Context, model, filename, contentType string, data []byte, instruction string) (string, error) {
if len(data) == 0 {
return "", errors.New("video payload is empty")
}
body := &bytes.Buffer{}
writer := multipart.NewWriter(body)
header := make(textproto.MIMEHeader)
header.Set("Content-Disposition", fmt.Sprintf(`form-data; name="file"; filename="%s"`, strings.ReplaceAll(filename, `"`, "")))
header.Set("Content-Type", contentType)
part, err := writer.CreatePart(header)
if err != nil {
return "", err
}
if _, err := part.Write(data); err != nil {
return "", err
}
_ = writer.WriteField("model", model)
_ = writer.WriteField("instruction", instruction)
if err := writer.Close(); err != nil {
return "", err
}
request, err := http.NewRequestWithContext(ctx, http.MethodPost, strings.TrimRight(c.BaseURL, "/")+"/understand", body)
if err != nil {
return "", err
}
request.Header.Set("Content-Type", writer.FormDataContentType())
response, err := c.client().Do(request)
if err != nil {
return "", err
}
defer response.Body.Close()
if response.StatusCode < 200 || response.StatusCode > 299 {
return "", fmt.Errorf("video provider status %d", response.StatusCode)
}
var result struct {
Text string `json:"text"`
}
if err := json.NewDecoder(io.LimitReader(response.Body, 2<<20)).Decode(&result); err != nil {
return "", err
}
if strings.TrimSpace(result.Text) == "" {
return "", ErrEmptyResult
}
return strings.TrimSpace(result.Text), nil
}
func (c SpeechClient) client() *http.Client {
if c.Client != nil {
return c.Client
}
return http.DefaultClient
}
func (c VisionClient) client() *http.Client {
if c.Client != nil {
return c.Client
}
return http.DefaultClient
}
func (c ImageGenerationClient) client() *http.Client {
if c.Client != nil {
return c.Client
}
return http.DefaultClient
}
func (c VideoUnderstandingClient) client() *http.Client {
if c.Client != nil {
return c.Client
}
return http.DefaultClient
}
func encodeBase64(data []byte) string {
return base64.StdEncoding.EncodeToString(data)
}