Add local speech and vision chat skills
CI / test (push) Successful in 1m9s
CI / test (pull_request) Successful in 1m7s

This commit is contained in:
m
2026-08-25 12:26:41 +03:00
parent df017a349a
commit 78e7e3fedc
14 changed files with 602 additions and 5 deletions
+10
View File
@@ -7,6 +7,7 @@ import (
"errors"
"fmt"
"git.cin.su/m/Assistant/adapters/models"
"git.cin.su/m/Assistant/adapters/multimodal"
"git.cin.su/m/Assistant/contracts/capability"
"git.cin.su/m/Assistant/contracts/cognitive"
"git.cin.su/m/Assistant/contracts/identity"
@@ -170,6 +171,15 @@ func main() {
mux.HandleFunc("POST /v1/advisories", gateway.advisoryHandler)
mux.HandleFunc("POST /v1/interpretations", gateway.interpretationHandler)
mux.HandleFunc("POST /v1/task-evaluations/execute", gateway.taskEvaluationHandler)
multimodalGateway := multimodalGateway{
speech: multimodal.SpeechClient{BaseURL: envOr("LOCAL_SPEECH_PROVIDER_URL", "http://llm-local-speech:8080"), Client: &http.Client{Timeout: 120 * time.Second}},
vision: multimodal.VisionClient{BaseURL: envOr("LOCAL_VISION_PROVIDER_URL", "http://llm-local-vision:8080/v1"), Client: &http.Client{Timeout: 180 * time.Second}},
model: envOr("LOCAL_VISION_MODEL", "/models/gemma-4-26B-A4B-it-UD-Q4_K_M.gguf"),
now: time.Now,
}
mux.HandleFunc("POST /v1/multimodal/transcribe", multimodalGateway.handler(capability.MultimodalTranscribe))
mux.HandleFunc("POST /v1/multimodal/understand", multimodalGateway.handler(capability.MultimodalUnderstand))
mux.HandleFunc("POST /v1/multimodal/ocr", multimodalGateway.handler(capability.MultimodalRecognizeOCR))
discovery := discoveryService{resolver: resolver, providers: discoveries, owner: identity.PrincipalID(envOr("COGNITIVE_OWNER_PRINCIPAL_ID", "domovoy-creator-principal")), now: time.Now}
mux.HandleFunc("POST /v1/provider-discovery/{provider}", discovery.handler)
mux.Handle("POST /v1/provider-discovery/{provider}/qualify", modelQualificationHandler(discoveries, string(envOr("COGNITIVE_OWNER_PRINCIPAL_ID", "domovoy-creator-principal")), httpQualificationObserver{resolver: resolver}))
+75
View File
@@ -0,0 +1,75 @@
package main
import (
"encoding/json"
"io"
"log"
"net/http"
"strings"
"time"
"git.cin.su/m/Assistant/adapters/multimodal"
"git.cin.su/m/Assistant/contracts/capability"
)
const maxMultimodalUploadBytes = 12 << 20
type multimodalGateway struct {
speech multimodal.SpeechClient
vision multimodal.VisionClient
model string
now func() time.Time
}
func (g multimodalGateway) handler(operation capability.MultimodalOperation) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
r.Body = http.MaxBytesReader(w, r.Body, maxMultimodalUploadBytes)
if err := r.ParseMultipartForm(maxMultimodalUploadBytes); err != nil {
http.Error(w, "media upload is too large or invalid", http.StatusRequestEntityTooLarge)
return
}
file, header, err := r.FormFile("file")
if err != nil {
http.Error(w, "media file is required", http.StatusBadRequest)
return
}
defer file.Close()
data, err := io.ReadAll(io.LimitReader(file, maxMultimodalUploadBytes+1))
if err != nil || len(data) == 0 || len(data) > maxMultimodalUploadBytes {
http.Error(w, "media file is invalid", http.StatusBadRequest)
return
}
contentType := header.Header.Get("Content-Type")
if contentType == "" {
contentType = "application/octet-stream"
}
var text string
if operation == capability.MultimodalTranscribe {
text, err = g.speech.Transcribe(r.Context(), header.Filename, contentType, data)
} else {
instruction := "Опиши, что изображено на фотографии, коротко и по существу. Не выдумывай нечитаемые детали."
if operation == capability.MultimodalRecognizeOCR {
instruction = "Точно распознай весь читаемый текст на изображении. Сохрани строки и символы; нечитаемые места обозначь как [неразборчиво]. Не добавляй пояснений."
}
text, err = g.vision.Describe(r.Context(), g.model, contentType, data, instruction)
}
if err != nil {
logMultimodalFailure(operation, err)
http.Error(w, "local media capability unavailable", http.StatusBadGateway)
return
}
result := capability.MultimodalResponse{Operation: operation, Text: strings.TrimSpace(text), ObservedAt: g.now().UnixMilli(), ConfidenceNote: "Результат получен локальным специализированным сервисом; нечитаемые или неоднозначные фрагменты требуют проверки."}
if operation == capability.MultimodalTranscribe {
result.Provider = "local-whisper"
} else {
result.Provider = "local-vision"
}
w.Header().Set("Content-Type", "application/json")
_ = json.NewEncoder(w).Encode(result)
}
}
func logMultimodalFailure(operation capability.MultimodalOperation, err error) {
// Keep provider details in server diagnostics; never return them to chat.
log.Printf("multimodal operation %s failed: %v", operation, err)
}
+49
View File
@@ -0,0 +1,49 @@
package main
import (
"bytes"
"context"
"encoding/json"
"mime/multipart"
"net/http"
"net/http/httptest"
"testing"
"time"
"git.cin.su/m/Assistant/adapters/multimodal"
"git.cin.su/m/Assistant/contracts/capability"
)
func TestMultimodalTranscriptionHandlerReturnsLocalResult(t *testing.T) {
upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path != "/inference" || r.Method != http.MethodPost {
t.Fatalf("unexpected upstream request: %s %s", r.Method, r.URL.Path)
}
_ = json.NewEncoder(w).Encode(map[string]string{"text": "добрый день"})
}))
defer upstream.Close()
server := multimodalGateway{speech: multimodal.SpeechClient{BaseURL: upstream.URL, Client: upstream.Client()}, now: time.Now}
request := multipartRequest(t, "/v1/multimodal/transcribe", "voice.wav", []byte("audio"))
response := httptest.NewRecorder()
server.handler(capability.MultimodalTranscribe)(response, request)
if response.Code != http.StatusOK || !bytes.Contains(response.Body.Bytes(), []byte("добрый день")) {
t.Fatalf("status=%d body=%s", response.Code, response.Body.String())
}
}
func multipartRequest(t *testing.T, path, filename string, data []byte) *http.Request {
t.Helper()
body := &bytes.Buffer{}
writer := multipart.NewWriter(body)
part, err := writer.CreateFormFile("file", filename)
if err != nil {
t.Fatal(err)
}
if _, err := part.Write(data); err != nil || writer.Close() != nil {
t.Fatal("failed to build multipart request")
}
request := httptest.NewRequest(http.MethodPost, path, body)
request = request.WithContext(context.Background())
request.Header.Set("Content-Type", writer.FormDataContentType())
return request
}
+10
View File
@@ -40,6 +40,11 @@ func TestChatPageServedWithoutInternalIDs(t *testing.T) {
if !strings.Contains(body, "версия 0123456789ab") || !strings.Contains(body, "обновлено 22.08.2026 03:04 UTC") || !strings.Contains(body, "datetime=\"2026-08-22T03:04:05Z\"") {
t.Fatalf("/chat does not render build metadata: %q", body)
}
for _, required := range []string{"media-tools", "audio-input", "image-input", "ocr-input"} {
if !strings.Contains(body, required) {
t.Fatalf("/chat is missing multimodal control %q", required)
}
}
for _, path := range []string{"/", "/chat.html"} {
alternate := httptest.NewRecorder()
handler.ServeHTTP(alternate, httptest.NewRequest(http.MethodGet, path, nil))
@@ -60,6 +65,11 @@ func TestChatScriptExplainsSessionAndRuntimeFailures(t *testing.T) {
t.Fatalf("chat script is missing recovery message %q", required)
}
}
for _, required := range []string{"uploadMedia", "/api/multimodal/transcribe", "/api/multimodal/understand", "/api/multimodal/ocr"} {
if !strings.Contains(script, required) {
t.Fatalf("chat script is missing multimodal route %q", required)
}
}
}
func TestPublicSurfacesSeparateChatAndConsole(t *testing.T) {
+57
View File
@@ -0,0 +1,57 @@
package main
import (
"bytes"
"encoding/json"
"mime/multipart"
"net/http"
"net/http/httptest"
"testing"
"git.cin.su/m/Assistant/contracts/capability"
"git.cin.su/m/Assistant/contracts/identity"
)
func TestMultimodalUploadRequiresSessionAndProxiesLocalResult(t *testing.T) {
core := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path != "/v1/creator/sessions/current" || r.Header.Get("Authorization") != "Bearer session" {
t.Fatalf("unexpected session request: %s %q", r.URL.Path, r.Header.Get("Authorization"))
}
_ = json.NewEncoder(w).Encode(identity.CreatorSession{ActorID: "actor", PrincipalID: "principal"})
}))
defer core.Close()
cognitive := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path != "/v1/multimodal/transcribe" || r.Method != http.MethodPost {
t.Fatalf("unexpected multimodal request: %s %s", r.Method, r.URL.Path)
}
_ = json.NewEncoder(w).Encode(capability.MultimodalResponse{Operation: capability.MultimodalTranscribe, Text: "локальный текст", Provider: "local-whisper", ObservedAt: 1})
}))
defer cognitive.Close()
handler, err := newHandler(core.URL, "http://continuity.invalid", "http://commit.invalid", core.Client(), cognitive.URL)
if err != nil {
t.Fatal(err)
}
denied := httptest.NewRecorder()
handler.ServeHTTP(denied, httptest.NewRequest(http.MethodPost, "/api/multimodal/transcribe", bytes.NewReader(nil)))
if denied.Code != http.StatusUnauthorized {
t.Fatalf("unauthenticated upload status=%d", denied.Code)
}
body := &bytes.Buffer{}
writer := multipart.NewWriter(body)
part, err := writer.CreateFormFile("file", "voice.wav")
if err != nil {
t.Fatal(err)
}
_, _ = part.Write([]byte("audio"))
if err := writer.Close(); err != nil {
t.Fatal(err)
}
request := httptest.NewRequest(http.MethodPost, "/api/multimodal/transcribe", body)
request.Header.Set("Authorization", "Bearer session")
request.Header.Set("Content-Type", writer.FormDataContentType())
response := httptest.NewRecorder()
handler.ServeHTTP(response, request)
if response.Code != http.StatusOK || !bytes.Contains(response.Body.Bytes(), []byte("локальный текст")) {
t.Fatalf("status=%d body=%s", response.Code, response.Body.String())
}
}
+88
View File
@@ -7,7 +7,9 @@ import (
"fmt"
"io"
"io/fs"
"mime/multipart"
"net/http"
"net/textproto"
"net/url"
"os"
"strings"
@@ -135,6 +137,10 @@ func (g *gateway) ServeHTTP(w http.ResponseWriter, r *http.Request) {
g.handleChatMessage(w, r)
return
}
if r.Method == http.MethodPost && strings.HasPrefix(r.URL.Path, "/api/multimodal/") {
g.handleMultimodal(w, r)
return
}
if r.Method == http.MethodGet && r.URL.Path == "/api/fleet/readiness" {
g.handleFleetReadiness(w, r)
return
@@ -650,6 +656,88 @@ func (g *gateway) postTrustedJSON(r *http.Request, upstream *url.URL, path, auth
return response.StatusCode, nil
}
func (g *gateway) handleMultimodal(w http.ResponseWriter, r *http.Request) {
authorization := r.Header.Get("Authorization")
if authorization == "" {
http.Error(w, "creator session denied", http.StatusUnauthorized)
return
}
var session identity.CreatorSession
status, err := g.getTrustedJSON(r, g.core, "/v1/creator/sessions/current", "", authorization, &session)
if err != nil || status != http.StatusOK || session.ActorID == "" || session.PrincipalID == "" {
http.Error(w, "creator session denied", http.StatusUnauthorized)
return
}
const maxMediaBytes = 12 << 20
r.Body = http.MaxBytesReader(w, r.Body, maxMediaBytes)
if err := r.ParseMultipartForm(maxMediaBytes); err != nil {
http.Error(w, "media upload is too large or invalid", http.StatusRequestEntityTooLarge)
return
}
file, header, err := r.FormFile("file")
if err != nil {
http.Error(w, "media file is required", http.StatusBadRequest)
return
}
defer file.Close()
data, err := io.ReadAll(io.LimitReader(file, maxMediaBytes+1))
if err != nil || len(data) == 0 || len(data) > maxMediaBytes {
http.Error(w, "media file is invalid", http.StatusBadRequest)
return
}
path := ""
switch r.URL.Path {
case "/api/multimodal/transcribe":
path = "/v1/multimodal/transcribe"
case "/api/multimodal/understand":
path = "/v1/multimodal/understand"
case "/api/multimodal/ocr":
path = "/v1/multimodal/ocr"
default:
http.Error(w, "multimodal route not found", http.StatusNotFound)
return
}
body := &bytes.Buffer{}
writer := multipart.NewWriter(body)
mediaHeader := make(textproto.MIMEHeader)
mediaHeader.Set("Content-Disposition", fmt.Sprintf(`form-data; name="file"; filename="%s"`, strings.ReplaceAll(header.Filename, `"`, "")))
mediaType := header.Header.Get("Content-Type")
if mediaType == "" {
mediaType = "application/octet-stream"
}
mediaHeader.Set("Content-Type", mediaType)
part, err := writer.CreatePart(mediaHeader)
if err != nil {
http.Error(w, "media upload unavailable", http.StatusServiceUnavailable)
return
}
if _, err := part.Write(data); err != nil || writer.Close() != nil {
http.Error(w, "media upload unavailable", http.StatusServiceUnavailable)
return
}
target := *g.cognitive
target.Path = path
request, err := http.NewRequestWithContext(r.Context(), http.MethodPost, target.String(), body)
if err != nil {
http.Error(w, "media capability unavailable", http.StatusBadGateway)
return
}
request.Header.Set("Content-Type", writer.FormDataContentType())
response, err := g.client.Do(request)
if err != nil {
http.Error(w, "media capability unavailable", http.StatusBadGateway)
return
}
defer response.Body.Close()
if response.StatusCode != http.StatusOK {
_, _ = io.Copy(io.Discard, io.LimitReader(response.Body, maxResponseBytes))
http.Error(w, "media capability unavailable", http.StatusBadGateway)
return
}
w.Header().Set("Content-Type", "application/json")
_, _ = io.Copy(w, io.LimitReader(response.Body, maxResponseBytes))
}
func decodeStrictJSON(reader io.Reader, limit int64, destination any) error {
decoder := json.NewDecoder(io.LimitReader(reader, limit+1))
decoder.DisallowUnknownFields()
+4
View File
@@ -75,6 +75,10 @@ textarea { resize: vertical; min-height: 84px; max-height: 240px; padding: 14px;
.composer { width: min(100%, 920px); margin: 0 auto; padding: 16px 18px 10px; display: grid; grid-template-columns: minmax(0, 1fr) 130px; gap: 12px; background: linear-gradient(transparent, var(--base) 20%); }
.composer button { height: 48px; align-self: end; }
.composer button:disabled { cursor: wait; filter: saturate(.3); }
.media-tools { width: min(100%, 920px); margin: 0 auto; padding: 0 18px 10px; display: flex; flex-wrap: wrap; align-items: center; gap: 8px; color: var(--muted); font: 10px/1 var(--mono); text-transform: uppercase; letter-spacing: .06em; }
.media-tools label { display: inline-flex; align-items: center; gap: 6px; min-height: 30px; padding: 0 8px; border: 1px solid var(--line); color: var(--ink); cursor: pointer; }
.media-tools input { max-width: 150px; font: 10px var(--mono); }
.media-tools span { color: var(--signal); }
.conversation-starters { width: min(100%, 920px); margin: -4px auto 0; padding: 0 18px 18px; display: flex; flex-wrap: wrap; align-items: center; gap: 8px; color: var(--muted); font: 10px/1 var(--mono); letter-spacing: .06em; text-transform: uppercase; }
.conversation-starters button { min-height: 32px; padding: 0 10px; border: 1px solid var(--line); color: var(--ink); background: #101814; font: 10px/1 var(--mono); }
.conversation-starters button:hover, .conversation-starters button:focus-visible { border-color: var(--signal); color: var(--signal); outline: none; }
+38
View File
@@ -132,6 +132,16 @@ async function api(path, options = {}) {
return response.json();
}
async function uploadMedia(path, file) {
const body = new FormData();
body.append("file", file, file.name);
const headers = {};
if (token) headers.Authorization = `Bearer ${token}`;
const response = await fetch(path, {method: "POST", headers, body});
if (!response.ok) throw new ChatApiError(response.status, await response.text());
return response.json();
}
function expireSession(message = "") {
token = "";
conversationID = "";
@@ -320,6 +330,34 @@ chatForm.addEventListener("submit", async (event) => {
await sendChatMessage(text);
});
async function processMedia(input, path, label) {
const file = input?.files?.[0];
if (!file || !mediaStatus) return;
mediaStatus.textContent = `${label} обрабатывается...`;
input.value = "";
try {
const result = await uploadMedia(path, file);
const text = String(result.text || "").trim();
if (!text) throw new Error("empty media result");
mediaStatus.textContent = "Готово";
await sendChatMessage(`Пользователь передал ${label.toLowerCase()}. Обработай содержание и ответь по смыслу:\n${text}`);
} catch (error) {
if (error instanceof ChatApiError && (error.status === 401 || error.status === 403)) {
expireSession("Сессия устарела. Войдите снова, чтобы продолжить разговор.");
} else {
mediaStatus.textContent = `${label} сейчас недоступно.`;
}
}
}
const mediaStatus = document.querySelector("#media-status");
const audioInput = document.querySelector("#audio-input");
const imageInput = document.querySelector("#image-input");
const ocrInput = document.querySelector("#ocr-input");
if (audioInput) audioInput.addEventListener("change", () => processMedia(audioInput, "/api/multimodal/transcribe", "Распознанное аудио"));
if (imageInput) imageInput.addEventListener("change", () => processMedia(imageInput, "/api/multimodal/understand", "Описание изображения"));
if (ocrInput) ocrInput.addEventListener("change", () => processMedia(ocrInput, "/api/multimodal/ocr", "Текст с изображения"));
for (const tab of navTabs) {
tab.addEventListener("click", () => activateTab(tab.dataset.tab));
}
+6
View File
@@ -60,6 +60,12 @@
<textarea id="message" rows="3" maxlength="16384" placeholder="Напишите Домовому обычное сообщение..." required></textarea>
<button id="send" type="submit">Отправить</button>
</form>
<div class="media-tools" aria-label="Мультимодальные сообщения">
<label>Аудио <input id="audio-input" type="file" accept="audio/*"></label>
<label>Фото <input id="image-input" type="file" accept="image/*"></label>
<label>Текст с фото <input id="ocr-input" type="file" accept="image/*"></label>
<span id="media-status" role="status" aria-live="polite"></span>
</div>
<div class="conversation-starters" aria-label="Частые вопросы">
<span>Можно начать так:</span>
<button type="button" data-starter="Проверь локальный канал мышления">Проверь канал</button>