Add local speech and vision chat skills
CI / test (push) Successful in 1m9s
CI / test (pull_request) Successful in 1m7s

This commit is contained in:
m
2026-08-25 12:26:41 +03:00
parent df017a349a
commit 78e7e3fedc
14 changed files with 602 additions and 5 deletions
+10
View File
@@ -40,6 +40,11 @@ func TestChatPageServedWithoutInternalIDs(t *testing.T) {
if !strings.Contains(body, "версия 0123456789ab") || !strings.Contains(body, "обновлено 22.08.2026 03:04 UTC") || !strings.Contains(body, "datetime=\"2026-08-22T03:04:05Z\"") {
t.Fatalf("/chat does not render build metadata: %q", body)
}
for _, required := range []string{"media-tools", "audio-input", "image-input", "ocr-input"} {
if !strings.Contains(body, required) {
t.Fatalf("/chat is missing multimodal control %q", required)
}
}
for _, path := range []string{"/", "/chat.html"} {
alternate := httptest.NewRecorder()
handler.ServeHTTP(alternate, httptest.NewRequest(http.MethodGet, path, nil))
@@ -60,6 +65,11 @@ func TestChatScriptExplainsSessionAndRuntimeFailures(t *testing.T) {
t.Fatalf("chat script is missing recovery message %q", required)
}
}
for _, required := range []string{"uploadMedia", "/api/multimodal/transcribe", "/api/multimodal/understand", "/api/multimodal/ocr"} {
if !strings.Contains(script, required) {
t.Fatalf("chat script is missing multimodal route %q", required)
}
}
}
func TestPublicSurfacesSeparateChatAndConsole(t *testing.T) {
+57
View File
@@ -0,0 +1,57 @@
package main
import (
"bytes"
"encoding/json"
"mime/multipart"
"net/http"
"net/http/httptest"
"testing"
"git.cin.su/m/Assistant/contracts/capability"
"git.cin.su/m/Assistant/contracts/identity"
)
func TestMultimodalUploadRequiresSessionAndProxiesLocalResult(t *testing.T) {
core := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path != "/v1/creator/sessions/current" || r.Header.Get("Authorization") != "Bearer session" {
t.Fatalf("unexpected session request: %s %q", r.URL.Path, r.Header.Get("Authorization"))
}
_ = json.NewEncoder(w).Encode(identity.CreatorSession{ActorID: "actor", PrincipalID: "principal"})
}))
defer core.Close()
cognitive := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path != "/v1/multimodal/transcribe" || r.Method != http.MethodPost {
t.Fatalf("unexpected multimodal request: %s %s", r.Method, r.URL.Path)
}
_ = json.NewEncoder(w).Encode(capability.MultimodalResponse{Operation: capability.MultimodalTranscribe, Text: "локальный текст", Provider: "local-whisper", ObservedAt: 1})
}))
defer cognitive.Close()
handler, err := newHandler(core.URL, "http://continuity.invalid", "http://commit.invalid", core.Client(), cognitive.URL)
if err != nil {
t.Fatal(err)
}
denied := httptest.NewRecorder()
handler.ServeHTTP(denied, httptest.NewRequest(http.MethodPost, "/api/multimodal/transcribe", bytes.NewReader(nil)))
if denied.Code != http.StatusUnauthorized {
t.Fatalf("unauthenticated upload status=%d", denied.Code)
}
body := &bytes.Buffer{}
writer := multipart.NewWriter(body)
part, err := writer.CreateFormFile("file", "voice.wav")
if err != nil {
t.Fatal(err)
}
_, _ = part.Write([]byte("audio"))
if err := writer.Close(); err != nil {
t.Fatal(err)
}
request := httptest.NewRequest(http.MethodPost, "/api/multimodal/transcribe", body)
request.Header.Set("Authorization", "Bearer session")
request.Header.Set("Content-Type", writer.FormDataContentType())
response := httptest.NewRecorder()
handler.ServeHTTP(response, request)
if response.Code != http.StatusOK || !bytes.Contains(response.Body.Bytes(), []byte("локальный текст")) {
t.Fatalf("status=%d body=%s", response.Code, response.Body.String())
}
}
+88
View File
@@ -7,7 +7,9 @@ import (
"fmt"
"io"
"io/fs"
"mime/multipart"
"net/http"
"net/textproto"
"net/url"
"os"
"strings"
@@ -135,6 +137,10 @@ func (g *gateway) ServeHTTP(w http.ResponseWriter, r *http.Request) {
g.handleChatMessage(w, r)
return
}
if r.Method == http.MethodPost && strings.HasPrefix(r.URL.Path, "/api/multimodal/") {
g.handleMultimodal(w, r)
return
}
if r.Method == http.MethodGet && r.URL.Path == "/api/fleet/readiness" {
g.handleFleetReadiness(w, r)
return
@@ -650,6 +656,88 @@ func (g *gateway) postTrustedJSON(r *http.Request, upstream *url.URL, path, auth
return response.StatusCode, nil
}
func (g *gateway) handleMultimodal(w http.ResponseWriter, r *http.Request) {
authorization := r.Header.Get("Authorization")
if authorization == "" {
http.Error(w, "creator session denied", http.StatusUnauthorized)
return
}
var session identity.CreatorSession
status, err := g.getTrustedJSON(r, g.core, "/v1/creator/sessions/current", "", authorization, &session)
if err != nil || status != http.StatusOK || session.ActorID == "" || session.PrincipalID == "" {
http.Error(w, "creator session denied", http.StatusUnauthorized)
return
}
const maxMediaBytes = 12 << 20
r.Body = http.MaxBytesReader(w, r.Body, maxMediaBytes)
if err := r.ParseMultipartForm(maxMediaBytes); err != nil {
http.Error(w, "media upload is too large or invalid", http.StatusRequestEntityTooLarge)
return
}
file, header, err := r.FormFile("file")
if err != nil {
http.Error(w, "media file is required", http.StatusBadRequest)
return
}
defer file.Close()
data, err := io.ReadAll(io.LimitReader(file, maxMediaBytes+1))
if err != nil || len(data) == 0 || len(data) > maxMediaBytes {
http.Error(w, "media file is invalid", http.StatusBadRequest)
return
}
path := ""
switch r.URL.Path {
case "/api/multimodal/transcribe":
path = "/v1/multimodal/transcribe"
case "/api/multimodal/understand":
path = "/v1/multimodal/understand"
case "/api/multimodal/ocr":
path = "/v1/multimodal/ocr"
default:
http.Error(w, "multimodal route not found", http.StatusNotFound)
return
}
body := &bytes.Buffer{}
writer := multipart.NewWriter(body)
mediaHeader := make(textproto.MIMEHeader)
mediaHeader.Set("Content-Disposition", fmt.Sprintf(`form-data; name="file"; filename="%s"`, strings.ReplaceAll(header.Filename, `"`, "")))
mediaType := header.Header.Get("Content-Type")
if mediaType == "" {
mediaType = "application/octet-stream"
}
mediaHeader.Set("Content-Type", mediaType)
part, err := writer.CreatePart(mediaHeader)
if err != nil {
http.Error(w, "media upload unavailable", http.StatusServiceUnavailable)
return
}
if _, err := part.Write(data); err != nil || writer.Close() != nil {
http.Error(w, "media upload unavailable", http.StatusServiceUnavailable)
return
}
target := *g.cognitive
target.Path = path
request, err := http.NewRequestWithContext(r.Context(), http.MethodPost, target.String(), body)
if err != nil {
http.Error(w, "media capability unavailable", http.StatusBadGateway)
return
}
request.Header.Set("Content-Type", writer.FormDataContentType())
response, err := g.client.Do(request)
if err != nil {
http.Error(w, "media capability unavailable", http.StatusBadGateway)
return
}
defer response.Body.Close()
if response.StatusCode != http.StatusOK {
_, _ = io.Copy(io.Discard, io.LimitReader(response.Body, maxResponseBytes))
http.Error(w, "media capability unavailable", http.StatusBadGateway)
return
}
w.Header().Set("Content-Type", "application/json")
_, _ = io.Copy(w, io.LimitReader(response.Body, maxResponseBytes))
}
func decodeStrictJSON(reader io.Reader, limit int64, destination any) error {
decoder := json.NewDecoder(io.LimitReader(reader, limit+1))
decoder.DisallowUnknownFields()
+4
View File
@@ -75,6 +75,10 @@ textarea { resize: vertical; min-height: 84px; max-height: 240px; padding: 14px;
.composer { width: min(100%, 920px); margin: 0 auto; padding: 16px 18px 10px; display: grid; grid-template-columns: minmax(0, 1fr) 130px; gap: 12px; background: linear-gradient(transparent, var(--base) 20%); }
.composer button { height: 48px; align-self: end; }
.composer button:disabled { cursor: wait; filter: saturate(.3); }
.media-tools { width: min(100%, 920px); margin: 0 auto; padding: 0 18px 10px; display: flex; flex-wrap: wrap; align-items: center; gap: 8px; color: var(--muted); font: 10px/1 var(--mono); text-transform: uppercase; letter-spacing: .06em; }
.media-tools label { display: inline-flex; align-items: center; gap: 6px; min-height: 30px; padding: 0 8px; border: 1px solid var(--line); color: var(--ink); cursor: pointer; }
.media-tools input { max-width: 150px; font: 10px var(--mono); }
.media-tools span { color: var(--signal); }
.conversation-starters { width: min(100%, 920px); margin: -4px auto 0; padding: 0 18px 18px; display: flex; flex-wrap: wrap; align-items: center; gap: 8px; color: var(--muted); font: 10px/1 var(--mono); letter-spacing: .06em; text-transform: uppercase; }
.conversation-starters button { min-height: 32px; padding: 0 10px; border: 1px solid var(--line); color: var(--ink); background: #101814; font: 10px/1 var(--mono); }
.conversation-starters button:hover, .conversation-starters button:focus-visible { border-color: var(--signal); color: var(--signal); outline: none; }
+38
View File
@@ -132,6 +132,16 @@ async function api(path, options = {}) {
return response.json();
}
async function uploadMedia(path, file) {
const body = new FormData();
body.append("file", file, file.name);
const headers = {};
if (token) headers.Authorization = `Bearer ${token}`;
const response = await fetch(path, {method: "POST", headers, body});
if (!response.ok) throw new ChatApiError(response.status, await response.text());
return response.json();
}
function expireSession(message = "") {
token = "";
conversationID = "";
@@ -320,6 +330,34 @@ chatForm.addEventListener("submit", async (event) => {
await sendChatMessage(text);
});
async function processMedia(input, path, label) {
const file = input?.files?.[0];
if (!file || !mediaStatus) return;
mediaStatus.textContent = `${label} обрабатывается...`;
input.value = "";
try {
const result = await uploadMedia(path, file);
const text = String(result.text || "").trim();
if (!text) throw new Error("empty media result");
mediaStatus.textContent = "Готово";
await sendChatMessage(`Пользователь передал ${label.toLowerCase()}. Обработай содержание и ответь по смыслу:\n${text}`);
} catch (error) {
if (error instanceof ChatApiError && (error.status === 401 || error.status === 403)) {
expireSession("Сессия устарела. Войдите снова, чтобы продолжить разговор.");
} else {
mediaStatus.textContent = `${label} сейчас недоступно.`;
}
}
}
const mediaStatus = document.querySelector("#media-status");
const audioInput = document.querySelector("#audio-input");
const imageInput = document.querySelector("#image-input");
const ocrInput = document.querySelector("#ocr-input");
if (audioInput) audioInput.addEventListener("change", () => processMedia(audioInput, "/api/multimodal/transcribe", "Распознанное аудио"));
if (imageInput) imageInput.addEventListener("change", () => processMedia(imageInput, "/api/multimodal/understand", "Описание изображения"));
if (ocrInput) ocrInput.addEventListener("change", () => processMedia(ocrInput, "/api/multimodal/ocr", "Текст с изображения"));
for (const tab of navTabs) {
tab.addEventListener("click", () => activateTab(tab.dataset.tab));
}
+6
View File
@@ -60,6 +60,12 @@
<textarea id="message" rows="3" maxlength="16384" placeholder="Напишите Домовому обычное сообщение..." required></textarea>
<button id="send" type="submit">Отправить</button>
</form>
<div class="media-tools" aria-label="Мультимодальные сообщения">
<label>Аудио <input id="audio-input" type="file" accept="audio/*"></label>
<label>Фото <input id="image-input" type="file" accept="image/*"></label>
<label>Текст с фото <input id="ocr-input" type="file" accept="image/*"></label>
<span id="media-status" role="status" aria-live="polite"></span>
</div>
<div class="conversation-starters" aria-label="Частые вопросы">
<span>Можно начать так:</span>
<button type="button" data-starter="Проверь локальный канал мышления">Проверь канал</button>