package multimodal import ( "bytes" "context" "encoding/base64" "encoding/json" "errors" "fmt" "io" "mime/multipart" "net/http" "net/textproto" "strings" ) var ErrEmptyResult = errors.New("multimodal provider returned an empty result") type SpeechClient struct { BaseURL string Client *http.Client } type VisionClient struct { BaseURL string Client *http.Client } func (c SpeechClient) Transcribe(ctx context.Context, filename, contentType string, data []byte) (string, error) { if len(data) == 0 { return "", errors.New("audio payload is empty") } body := &bytes.Buffer{} writer := multipart.NewWriter(body) header := make(textproto.MIMEHeader) header.Set("Content-Disposition", fmt.Sprintf(`form-data; name="file"; filename="%s"`, strings.ReplaceAll(filename, `"`, ""))) header.Set("Content-Type", contentType) part, err := writer.CreatePart(header) if err != nil { return "", err } if _, err := part.Write(data); err != nil { return "", err } if err := writer.WriteField("response_format", "json"); err != nil { return "", err } if err := writer.Close(); err != nil { return "", err } request, err := http.NewRequestWithContext(ctx, http.MethodPost, strings.TrimRight(c.BaseURL, "/")+"/inference", body) if err != nil { return "", err } request.Header.Set("Content-Type", writer.FormDataContentType()) response, err := c.client().Do(request) if err != nil { return "", err } defer response.Body.Close() if response.StatusCode != http.StatusOK { return "", fmt.Errorf("speech provider status %d", response.StatusCode) } var result struct { Text string `json:"text"` } if err := json.NewDecoder(io.LimitReader(response.Body, 1<<20)).Decode(&result); err != nil { return "", err } if strings.TrimSpace(result.Text) == "" { return "", ErrEmptyResult } return strings.TrimSpace(result.Text), nil } func (c VisionClient) Describe(ctx context.Context, model, contentType string, data []byte, instruction string) (string, error) { if len(data) == 0 { return "", errors.New("image payload is empty") } payload := struct { Model string `json:"model"` Messages []struct { Role string `json:"role"` Content []struct { Type string `json:"type"` Text string `json:"text,omitempty"` ImageURL *struct { URL string `json:"url"` } `json:"image_url,omitempty"` } `json:"content"` } `json:"messages"` MaxTokens int `json:"max_tokens"` }{} payload.Model = model payload.MaxTokens = 1024 payload.Messages = append(payload.Messages, struct { Role string `json:"role"` Content []struct { Type string `json:"type"` Text string `json:"text,omitempty"` ImageURL *struct { URL string `json:"url"` } `json:"image_url,omitempty"` } `json:"content"` }{Role: "user", Content: []struct { Type string `json:"type"` Text string `json:"text,omitempty"` ImageURL *struct { URL string `json:"url"` } `json:"image_url,omitempty"` }{ {Type: "text", Text: instruction}, {Type: "image_url", ImageURL: &struct { URL string `json:"url"` }{URL: "data:" + contentType + ";base64," + encodeBase64(data)}, }}}) encoded, err := json.Marshal(payload) if err != nil { return "", err } request, err := http.NewRequestWithContext(ctx, http.MethodPost, strings.TrimRight(c.BaseURL, "/")+"/chat/completions", bytes.NewReader(encoded)) if err != nil { return "", err } request.Header.Set("Content-Type", "application/json") response, err := c.client().Do(request) if err != nil { return "", err } defer response.Body.Close() if response.StatusCode != http.StatusOK { return "", fmt.Errorf("vision provider status %d", response.StatusCode) } var result struct { Choices []struct { Message struct { Content string `json:"content"` } `json:"message"` } `json:"choices"` } if err := json.NewDecoder(io.LimitReader(response.Body, 2<<20)).Decode(&result); err != nil { return "", err } if len(result.Choices) == 0 || strings.TrimSpace(result.Choices[0].Message.Content) == "" { return "", ErrEmptyResult } return strings.TrimSpace(result.Choices[0].Message.Content), nil } func (c SpeechClient) client() *http.Client { if c.Client != nil { return c.Client } return http.DefaultClient } func (c VisionClient) client() *http.Client { if c.Client != nil { return c.Client } return http.DefaultClient } func encodeBase64(data []byte) string { return base64.StdEncoding.EncodeToString(data) }