Add local speech and vision chat skills
This commit is contained in:
@@ -40,6 +40,11 @@ func TestChatPageServedWithoutInternalIDs(t *testing.T) {
|
||||
if !strings.Contains(body, "версия 0123456789ab") || !strings.Contains(body, "обновлено 22.08.2026 03:04 UTC") || !strings.Contains(body, "datetime=\"2026-08-22T03:04:05Z\"") {
|
||||
t.Fatalf("/chat does not render build metadata: %q", body)
|
||||
}
|
||||
for _, required := range []string{"media-tools", "audio-input", "image-input", "ocr-input"} {
|
||||
if !strings.Contains(body, required) {
|
||||
t.Fatalf("/chat is missing multimodal control %q", required)
|
||||
}
|
||||
}
|
||||
for _, path := range []string{"/", "/chat.html"} {
|
||||
alternate := httptest.NewRecorder()
|
||||
handler.ServeHTTP(alternate, httptest.NewRequest(http.MethodGet, path, nil))
|
||||
@@ -60,6 +65,11 @@ func TestChatScriptExplainsSessionAndRuntimeFailures(t *testing.T) {
|
||||
t.Fatalf("chat script is missing recovery message %q", required)
|
||||
}
|
||||
}
|
||||
for _, required := range []string{"uploadMedia", "/api/multimodal/transcribe", "/api/multimodal/understand", "/api/multimodal/ocr"} {
|
||||
if !strings.Contains(script, required) {
|
||||
t.Fatalf("chat script is missing multimodal route %q", required)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestPublicSurfacesSeparateChatAndConsole(t *testing.T) {
|
||||
|
||||
@@ -0,0 +1,57 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"mime/multipart"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
|
||||
"git.cin.su/m/Assistant/contracts/capability"
|
||||
"git.cin.su/m/Assistant/contracts/identity"
|
||||
)
|
||||
|
||||
func TestMultimodalUploadRequiresSessionAndProxiesLocalResult(t *testing.T) {
|
||||
core := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.URL.Path != "/v1/creator/sessions/current" || r.Header.Get("Authorization") != "Bearer session" {
|
||||
t.Fatalf("unexpected session request: %s %q", r.URL.Path, r.Header.Get("Authorization"))
|
||||
}
|
||||
_ = json.NewEncoder(w).Encode(identity.CreatorSession{ActorID: "actor", PrincipalID: "principal"})
|
||||
}))
|
||||
defer core.Close()
|
||||
cognitive := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.URL.Path != "/v1/multimodal/transcribe" || r.Method != http.MethodPost {
|
||||
t.Fatalf("unexpected multimodal request: %s %s", r.Method, r.URL.Path)
|
||||
}
|
||||
_ = json.NewEncoder(w).Encode(capability.MultimodalResponse{Operation: capability.MultimodalTranscribe, Text: "локальный текст", Provider: "local-whisper", ObservedAt: 1})
|
||||
}))
|
||||
defer cognitive.Close()
|
||||
handler, err := newHandler(core.URL, "http://continuity.invalid", "http://commit.invalid", core.Client(), cognitive.URL)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
denied := httptest.NewRecorder()
|
||||
handler.ServeHTTP(denied, httptest.NewRequest(http.MethodPost, "/api/multimodal/transcribe", bytes.NewReader(nil)))
|
||||
if denied.Code != http.StatusUnauthorized {
|
||||
t.Fatalf("unauthenticated upload status=%d", denied.Code)
|
||||
}
|
||||
body := &bytes.Buffer{}
|
||||
writer := multipart.NewWriter(body)
|
||||
part, err := writer.CreateFormFile("file", "voice.wav")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, _ = part.Write([]byte("audio"))
|
||||
if err := writer.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
request := httptest.NewRequest(http.MethodPost, "/api/multimodal/transcribe", body)
|
||||
request.Header.Set("Authorization", "Bearer session")
|
||||
request.Header.Set("Content-Type", writer.FormDataContentType())
|
||||
response := httptest.NewRecorder()
|
||||
handler.ServeHTTP(response, request)
|
||||
if response.Code != http.StatusOK || !bytes.Contains(response.Body.Bytes(), []byte("локальный текст")) {
|
||||
t.Fatalf("status=%d body=%s", response.Code, response.Body.String())
|
||||
}
|
||||
}
|
||||
@@ -7,7 +7,9 @@ import (
|
||||
"fmt"
|
||||
"io"
|
||||
"io/fs"
|
||||
"mime/multipart"
|
||||
"net/http"
|
||||
"net/textproto"
|
||||
"net/url"
|
||||
"os"
|
||||
"strings"
|
||||
@@ -135,6 +137,10 @@ func (g *gateway) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
g.handleChatMessage(w, r)
|
||||
return
|
||||
}
|
||||
if r.Method == http.MethodPost && strings.HasPrefix(r.URL.Path, "/api/multimodal/") {
|
||||
g.handleMultimodal(w, r)
|
||||
return
|
||||
}
|
||||
if r.Method == http.MethodGet && r.URL.Path == "/api/fleet/readiness" {
|
||||
g.handleFleetReadiness(w, r)
|
||||
return
|
||||
@@ -650,6 +656,88 @@ func (g *gateway) postTrustedJSON(r *http.Request, upstream *url.URL, path, auth
|
||||
return response.StatusCode, nil
|
||||
}
|
||||
|
||||
func (g *gateway) handleMultimodal(w http.ResponseWriter, r *http.Request) {
|
||||
authorization := r.Header.Get("Authorization")
|
||||
if authorization == "" {
|
||||
http.Error(w, "creator session denied", http.StatusUnauthorized)
|
||||
return
|
||||
}
|
||||
var session identity.CreatorSession
|
||||
status, err := g.getTrustedJSON(r, g.core, "/v1/creator/sessions/current", "", authorization, &session)
|
||||
if err != nil || status != http.StatusOK || session.ActorID == "" || session.PrincipalID == "" {
|
||||
http.Error(w, "creator session denied", http.StatusUnauthorized)
|
||||
return
|
||||
}
|
||||
const maxMediaBytes = 12 << 20
|
||||
r.Body = http.MaxBytesReader(w, r.Body, maxMediaBytes)
|
||||
if err := r.ParseMultipartForm(maxMediaBytes); err != nil {
|
||||
http.Error(w, "media upload is too large or invalid", http.StatusRequestEntityTooLarge)
|
||||
return
|
||||
}
|
||||
file, header, err := r.FormFile("file")
|
||||
if err != nil {
|
||||
http.Error(w, "media file is required", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
defer file.Close()
|
||||
data, err := io.ReadAll(io.LimitReader(file, maxMediaBytes+1))
|
||||
if err != nil || len(data) == 0 || len(data) > maxMediaBytes {
|
||||
http.Error(w, "media file is invalid", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
path := ""
|
||||
switch r.URL.Path {
|
||||
case "/api/multimodal/transcribe":
|
||||
path = "/v1/multimodal/transcribe"
|
||||
case "/api/multimodal/understand":
|
||||
path = "/v1/multimodal/understand"
|
||||
case "/api/multimodal/ocr":
|
||||
path = "/v1/multimodal/ocr"
|
||||
default:
|
||||
http.Error(w, "multimodal route not found", http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
body := &bytes.Buffer{}
|
||||
writer := multipart.NewWriter(body)
|
||||
mediaHeader := make(textproto.MIMEHeader)
|
||||
mediaHeader.Set("Content-Disposition", fmt.Sprintf(`form-data; name="file"; filename="%s"`, strings.ReplaceAll(header.Filename, `"`, "")))
|
||||
mediaType := header.Header.Get("Content-Type")
|
||||
if mediaType == "" {
|
||||
mediaType = "application/octet-stream"
|
||||
}
|
||||
mediaHeader.Set("Content-Type", mediaType)
|
||||
part, err := writer.CreatePart(mediaHeader)
|
||||
if err != nil {
|
||||
http.Error(w, "media upload unavailable", http.StatusServiceUnavailable)
|
||||
return
|
||||
}
|
||||
if _, err := part.Write(data); err != nil || writer.Close() != nil {
|
||||
http.Error(w, "media upload unavailable", http.StatusServiceUnavailable)
|
||||
return
|
||||
}
|
||||
target := *g.cognitive
|
||||
target.Path = path
|
||||
request, err := http.NewRequestWithContext(r.Context(), http.MethodPost, target.String(), body)
|
||||
if err != nil {
|
||||
http.Error(w, "media capability unavailable", http.StatusBadGateway)
|
||||
return
|
||||
}
|
||||
request.Header.Set("Content-Type", writer.FormDataContentType())
|
||||
response, err := g.client.Do(request)
|
||||
if err != nil {
|
||||
http.Error(w, "media capability unavailable", http.StatusBadGateway)
|
||||
return
|
||||
}
|
||||
defer response.Body.Close()
|
||||
if response.StatusCode != http.StatusOK {
|
||||
_, _ = io.Copy(io.Discard, io.LimitReader(response.Body, maxResponseBytes))
|
||||
http.Error(w, "media capability unavailable", http.StatusBadGateway)
|
||||
return
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = io.Copy(w, io.LimitReader(response.Body, maxResponseBytes))
|
||||
}
|
||||
|
||||
func decodeStrictJSON(reader io.Reader, limit int64, destination any) error {
|
||||
decoder := json.NewDecoder(io.LimitReader(reader, limit+1))
|
||||
decoder.DisallowUnknownFields()
|
||||
|
||||
@@ -75,6 +75,10 @@ textarea { resize: vertical; min-height: 84px; max-height: 240px; padding: 14px;
|
||||
.composer { width: min(100%, 920px); margin: 0 auto; padding: 16px 18px 10px; display: grid; grid-template-columns: minmax(0, 1fr) 130px; gap: 12px; background: linear-gradient(transparent, var(--base) 20%); }
|
||||
.composer button { height: 48px; align-self: end; }
|
||||
.composer button:disabled { cursor: wait; filter: saturate(.3); }
|
||||
.media-tools { width: min(100%, 920px); margin: 0 auto; padding: 0 18px 10px; display: flex; flex-wrap: wrap; align-items: center; gap: 8px; color: var(--muted); font: 10px/1 var(--mono); text-transform: uppercase; letter-spacing: .06em; }
|
||||
.media-tools label { display: inline-flex; align-items: center; gap: 6px; min-height: 30px; padding: 0 8px; border: 1px solid var(--line); color: var(--ink); cursor: pointer; }
|
||||
.media-tools input { max-width: 150px; font: 10px var(--mono); }
|
||||
.media-tools span { color: var(--signal); }
|
||||
.conversation-starters { width: min(100%, 920px); margin: -4px auto 0; padding: 0 18px 18px; display: flex; flex-wrap: wrap; align-items: center; gap: 8px; color: var(--muted); font: 10px/1 var(--mono); letter-spacing: .06em; text-transform: uppercase; }
|
||||
.conversation-starters button { min-height: 32px; padding: 0 10px; border: 1px solid var(--line); color: var(--ink); background: #101814; font: 10px/1 var(--mono); }
|
||||
.conversation-starters button:hover, .conversation-starters button:focus-visible { border-color: var(--signal); color: var(--signal); outline: none; }
|
||||
|
||||
@@ -132,6 +132,16 @@ async function api(path, options = {}) {
|
||||
return response.json();
|
||||
}
|
||||
|
||||
async function uploadMedia(path, file) {
|
||||
const body = new FormData();
|
||||
body.append("file", file, file.name);
|
||||
const headers = {};
|
||||
if (token) headers.Authorization = `Bearer ${token}`;
|
||||
const response = await fetch(path, {method: "POST", headers, body});
|
||||
if (!response.ok) throw new ChatApiError(response.status, await response.text());
|
||||
return response.json();
|
||||
}
|
||||
|
||||
function expireSession(message = "") {
|
||||
token = "";
|
||||
conversationID = "";
|
||||
@@ -320,6 +330,34 @@ chatForm.addEventListener("submit", async (event) => {
|
||||
await sendChatMessage(text);
|
||||
});
|
||||
|
||||
async function processMedia(input, path, label) {
|
||||
const file = input?.files?.[0];
|
||||
if (!file || !mediaStatus) return;
|
||||
mediaStatus.textContent = `${label} обрабатывается...`;
|
||||
input.value = "";
|
||||
try {
|
||||
const result = await uploadMedia(path, file);
|
||||
const text = String(result.text || "").trim();
|
||||
if (!text) throw new Error("empty media result");
|
||||
mediaStatus.textContent = "Готово";
|
||||
await sendChatMessage(`Пользователь передал ${label.toLowerCase()}. Обработай содержание и ответь по смыслу:\n${text}`);
|
||||
} catch (error) {
|
||||
if (error instanceof ChatApiError && (error.status === 401 || error.status === 403)) {
|
||||
expireSession("Сессия устарела. Войдите снова, чтобы продолжить разговор.");
|
||||
} else {
|
||||
mediaStatus.textContent = `${label} сейчас недоступно.`;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const mediaStatus = document.querySelector("#media-status");
|
||||
const audioInput = document.querySelector("#audio-input");
|
||||
const imageInput = document.querySelector("#image-input");
|
||||
const ocrInput = document.querySelector("#ocr-input");
|
||||
if (audioInput) audioInput.addEventListener("change", () => processMedia(audioInput, "/api/multimodal/transcribe", "Распознанное аудио"));
|
||||
if (imageInput) imageInput.addEventListener("change", () => processMedia(imageInput, "/api/multimodal/understand", "Описание изображения"));
|
||||
if (ocrInput) ocrInput.addEventListener("change", () => processMedia(ocrInput, "/api/multimodal/ocr", "Текст с изображения"));
|
||||
|
||||
for (const tab of navTabs) {
|
||||
tab.addEventListener("click", () => activateTab(tab.dataset.tab));
|
||||
}
|
||||
|
||||
@@ -60,6 +60,12 @@
|
||||
<textarea id="message" rows="3" maxlength="16384" placeholder="Напишите Домовому обычное сообщение..." required></textarea>
|
||||
<button id="send" type="submit">Отправить</button>
|
||||
</form>
|
||||
<div class="media-tools" aria-label="Мультимодальные сообщения">
|
||||
<label>Аудио <input id="audio-input" type="file" accept="audio/*"></label>
|
||||
<label>Фото <input id="image-input" type="file" accept="image/*"></label>
|
||||
<label>Текст с фото <input id="ocr-input" type="file" accept="image/*"></label>
|
||||
<span id="media-status" role="status" aria-live="polite"></span>
|
||||
</div>
|
||||
<div class="conversation-starters" aria-label="Частые вопросы">
|
||||
<span>Можно начать так:</span>
|
||||
<button type="button" data-starter="Проверь локальный канал мышления">Проверь канал</button>
|
||||
|
||||
Reference in New Issue
Block a user