Add local speech and vision chat skills
This commit is contained in:
@@ -0,0 +1,166 @@
|
||||
package multimodal
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"mime/multipart"
|
||||
"net/http"
|
||||
"net/textproto"
|
||||
"strings"
|
||||
)
|
||||
|
||||
var ErrEmptyResult = errors.New("multimodal provider returned an empty result")
|
||||
|
||||
type SpeechClient struct {
|
||||
BaseURL string
|
||||
Client *http.Client
|
||||
}
|
||||
|
||||
type VisionClient struct {
|
||||
BaseURL string
|
||||
Client *http.Client
|
||||
}
|
||||
|
||||
func (c SpeechClient) Transcribe(ctx context.Context, filename, contentType string, data []byte) (string, error) {
|
||||
if len(data) == 0 {
|
||||
return "", errors.New("audio payload is empty")
|
||||
}
|
||||
body := &bytes.Buffer{}
|
||||
writer := multipart.NewWriter(body)
|
||||
header := make(textproto.MIMEHeader)
|
||||
header.Set("Content-Disposition", fmt.Sprintf(`form-data; name="file"; filename="%s"`, strings.ReplaceAll(filename, `"`, "")))
|
||||
header.Set("Content-Type", contentType)
|
||||
part, err := writer.CreatePart(header)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if _, err := part.Write(data); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if err := writer.WriteField("response_format", "json"); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if err := writer.Close(); err != nil {
|
||||
return "", err
|
||||
}
|
||||
request, err := http.NewRequestWithContext(ctx, http.MethodPost, strings.TrimRight(c.BaseURL, "/")+"/inference", body)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
request.Header.Set("Content-Type", writer.FormDataContentType())
|
||||
response, err := c.client().Do(request)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer response.Body.Close()
|
||||
if response.StatusCode != http.StatusOK {
|
||||
return "", fmt.Errorf("speech provider status %d", response.StatusCode)
|
||||
}
|
||||
var result struct {
|
||||
Text string `json:"text"`
|
||||
}
|
||||
if err := json.NewDecoder(io.LimitReader(response.Body, 1<<20)).Decode(&result); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if strings.TrimSpace(result.Text) == "" {
|
||||
return "", ErrEmptyResult
|
||||
}
|
||||
return strings.TrimSpace(result.Text), nil
|
||||
}
|
||||
|
||||
func (c VisionClient) Describe(ctx context.Context, model, contentType string, data []byte, instruction string) (string, error) {
|
||||
if len(data) == 0 {
|
||||
return "", errors.New("image payload is empty")
|
||||
}
|
||||
payload := struct {
|
||||
Model string `json:"model"`
|
||||
Messages []struct {
|
||||
Role string `json:"role"`
|
||||
Content []struct {
|
||||
Type string `json:"type"`
|
||||
Text string `json:"text,omitempty"`
|
||||
ImageURL *struct {
|
||||
URL string `json:"url"`
|
||||
} `json:"image_url,omitempty"`
|
||||
} `json:"content"`
|
||||
} `json:"messages"`
|
||||
MaxTokens int `json:"max_tokens"`
|
||||
}{}
|
||||
payload.Model = model
|
||||
payload.MaxTokens = 1024
|
||||
payload.Messages = append(payload.Messages, struct {
|
||||
Role string `json:"role"`
|
||||
Content []struct {
|
||||
Type string `json:"type"`
|
||||
Text string `json:"text,omitempty"`
|
||||
ImageURL *struct {
|
||||
URL string `json:"url"`
|
||||
} `json:"image_url,omitempty"`
|
||||
} `json:"content"`
|
||||
}{Role: "user", Content: []struct {
|
||||
Type string `json:"type"`
|
||||
Text string `json:"text,omitempty"`
|
||||
ImageURL *struct {
|
||||
URL string `json:"url"`
|
||||
} `json:"image_url,omitempty"`
|
||||
}{
|
||||
{Type: "text", Text: instruction},
|
||||
{Type: "image_url", ImageURL: &struct {
|
||||
URL string `json:"url"`
|
||||
}{URL: "data:" + contentType + ";base64," + encodeBase64(data)},
|
||||
}}})
|
||||
encoded, err := json.Marshal(payload)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
request, err := http.NewRequestWithContext(ctx, http.MethodPost, strings.TrimRight(c.BaseURL, "/")+"/chat/completions", bytes.NewReader(encoded))
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
request.Header.Set("Content-Type", "application/json")
|
||||
response, err := c.client().Do(request)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer response.Body.Close()
|
||||
if response.StatusCode != http.StatusOK {
|
||||
return "", fmt.Errorf("vision provider status %d", response.StatusCode)
|
||||
}
|
||||
var result struct {
|
||||
Choices []struct {
|
||||
Message struct {
|
||||
Content string `json:"content"`
|
||||
} `json:"message"`
|
||||
} `json:"choices"`
|
||||
}
|
||||
if err := json.NewDecoder(io.LimitReader(response.Body, 2<<20)).Decode(&result); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if len(result.Choices) == 0 || strings.TrimSpace(result.Choices[0].Message.Content) == "" {
|
||||
return "", ErrEmptyResult
|
||||
}
|
||||
return strings.TrimSpace(result.Choices[0].Message.Content), nil
|
||||
}
|
||||
|
||||
func (c SpeechClient) client() *http.Client {
|
||||
if c.Client != nil {
|
||||
return c.Client
|
||||
}
|
||||
return http.DefaultClient
|
||||
}
|
||||
|
||||
func (c VisionClient) client() *http.Client {
|
||||
if c.Client != nil {
|
||||
return c.Client
|
||||
}
|
||||
return http.DefaultClient
|
||||
}
|
||||
|
||||
func encodeBase64(data []byte) string {
|
||||
return base64.StdEncoding.EncodeToString(data)
|
||||
}
|
||||
@@ -0,0 +1,65 @@
|
||||
package multimodal
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestSpeechClientSendsMultipartAndReturnsTranscript(t *testing.T) {
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.URL.Path != "/inference" || r.Method != http.MethodPost {
|
||||
t.Fatalf("unexpected speech request: %s %s", r.Method, r.URL.Path)
|
||||
}
|
||||
if err := r.ParseMultipartForm(1 << 20); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
file, _, err := r.FormFile("file")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer file.Close()
|
||||
data, _ := io.ReadAll(file)
|
||||
if string(data) != "audio" {
|
||||
t.Fatalf("audio payload = %q", data)
|
||||
}
|
||||
_ = json.NewEncoder(w).Encode(map[string]string{"text": "распознанная фраза"})
|
||||
}))
|
||||
defer server.Close()
|
||||
text, err := (SpeechClient{BaseURL: server.URL, Client: server.Client()}).Transcribe(context.Background(), "voice.wav", "audio/wav", []byte("audio"))
|
||||
if err != nil || text != "распознанная фраза" {
|
||||
t.Fatalf("transcript=%q err=%v", text, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestVisionClientSendsImageDataAndInstruction(t *testing.T) {
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
var payload struct {
|
||||
Messages []struct {
|
||||
Content []struct {
|
||||
Type string `json:"type"`
|
||||
Text string `json:"text"`
|
||||
ImageURL *struct {
|
||||
URL string `json:"url"`
|
||||
} `json:"image_url"`
|
||||
} `json:"content"`
|
||||
} `json:"messages"`
|
||||
}
|
||||
if err := json.NewDecoder(r.Body).Decode(&payload); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(payload.Messages) != 1 || len(payload.Messages[0].Content) != 2 || !strings.Contains(payload.Messages[0].Content[0].Text, "Опиши") || payload.Messages[0].Content[1].ImageURL == nil || !strings.HasPrefix(payload.Messages[0].Content[1].ImageURL.URL, "data:image/png;base64,") {
|
||||
t.Fatalf("unexpected vision payload: %+v", payload)
|
||||
}
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{"choices": []any{map[string]any{"message": map[string]string{"content": "на изображении круг"}}}})
|
||||
}))
|
||||
defer server.Close()
|
||||
text, err := (VisionClient{BaseURL: server.URL + "/v1", Client: server.Client()}).Describe(context.Background(), "vision-model", "image/png", []byte{0, 1, 2}, "Опиши изображение")
|
||||
if err != nil || text != "на изображении круг" {
|
||||
t.Fatalf("description=%q err=%v", text, err)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user