adapters/multimodal/client.go

@@ -0,0 +1,166 @@
+package multimodal
+
+import (
+	"bytes"
+	"context"
+	"encoding/base64"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"mime/multipart"
+	"net/http"
+	"net/textproto"
+	"strings"
+)
+
+var ErrEmptyResult = errors.New("multimodal provider returned an empty result")
+
+type SpeechClient struct {
+	BaseURL string
+	Client  *http.Client
+}
+
+type VisionClient struct {
+	BaseURL string
+	Client  *http.Client
+}
+
+func (c SpeechClient) Transcribe(ctx context.Context, filename, contentType string, data []byte) (string, error) {
+	if len(data) == 0 {
+		return "", errors.New("audio payload is empty")
+	}
+	body := &bytes.Buffer{}
+	writer := multipart.NewWriter(body)
+	header := make(textproto.MIMEHeader)
+	header.Set("Content-Disposition", fmt.Sprintf(`form-data; name="file"; filename="%s"`, strings.ReplaceAll(filename, `"`, "")))
+	header.Set("Content-Type", contentType)
+	part, err := writer.CreatePart(header)
+	if err != nil {
+		return "", err
+	}
+	if _, err := part.Write(data); err != nil {
+		return "", err
+	}
+	if err := writer.WriteField("response_format", "json"); err != nil {
+		return "", err
+	}
+	if err := writer.Close(); err != nil {
+		return "", err
+	}
+	request, err := http.NewRequestWithContext(ctx, http.MethodPost, strings.TrimRight(c.BaseURL, "/")+"/inference", body)
+	if err != nil {
+		return "", err
+	}
+	request.Header.Set("Content-Type", writer.FormDataContentType())
+	response, err := c.client().Do(request)
+	if err != nil {
+		return "", err
+	}
+	defer response.Body.Close()
+	if response.StatusCode != http.StatusOK {
+		return "", fmt.Errorf("speech provider status %d", response.StatusCode)
+	}
+	var result struct {
+		Text string `json:"text"`
+	}
+	if err := json.NewDecoder(io.LimitReader(response.Body, 1<<20)).Decode(&result); err != nil {
+		return "", err
+	}
+	if strings.TrimSpace(result.Text) == "" {
+		return "", ErrEmptyResult
+	}
+	return strings.TrimSpace(result.Text), nil
+}
+
+func (c VisionClient) Describe(ctx context.Context, model, contentType string, data []byte, instruction string) (string, error) {
+	if len(data) == 0 {
+		return "", errors.New("image payload is empty")
+	}
+	payload := struct {
+		Model    string `json:"model"`
+		Messages []struct {
+			Role    string `json:"role"`
+			Content []struct {
+				Type     string `json:"type"`
+				Text     string `json:"text,omitempty"`
+				ImageURL *struct {
+					URL string `json:"url"`
+				} `json:"image_url,omitempty"`
+			} `json:"content"`
+		} `json:"messages"`
+		MaxTokens int `json:"max_tokens"`
+	}{}
+	payload.Model = model
+	payload.MaxTokens = 1024
+	payload.Messages = append(payload.Messages, struct {
+		Role    string `json:"role"`
+		Content []struct {
+			Type     string `json:"type"`
+			Text     string `json:"text,omitempty"`
+			ImageURL *struct {
+				URL string `json:"url"`
+			} `json:"image_url,omitempty"`
+		} `json:"content"`
+	}{Role: "user", Content: []struct {
+		Type     string `json:"type"`
+		Text     string `json:"text,omitempty"`
+		ImageURL *struct {
+			URL string `json:"url"`
+		} `json:"image_url,omitempty"`
+	}{
+		{Type: "text", Text: instruction},
+		{Type: "image_url", ImageURL: &struct {
+			URL string `json:"url"`
+		}{URL: "data:" + contentType + ";base64," + encodeBase64(data)},
+		}}})
+	encoded, err := json.Marshal(payload)
+	if err != nil {
+		return "", err
+	}
+	request, err := http.NewRequestWithContext(ctx, http.MethodPost, strings.TrimRight(c.BaseURL, "/")+"/chat/completions", bytes.NewReader(encoded))
+	if err != nil {
+		return "", err
+	}
+	request.Header.Set("Content-Type", "application/json")
+	response, err := c.client().Do(request)
+	if err != nil {
+		return "", err
+	}
+	defer response.Body.Close()
+	if response.StatusCode != http.StatusOK {
+		return "", fmt.Errorf("vision provider status %d", response.StatusCode)
+	}
+	var result struct {
+		Choices []struct {
+			Message struct {
+				Content string `json:"content"`
+			} `json:"message"`
+		} `json:"choices"`
+	}
+	if err := json.NewDecoder(io.LimitReader(response.Body, 2<<20)).Decode(&result); err != nil {
+		return "", err
+	}
+	if len(result.Choices) == 0 || strings.TrimSpace(result.Choices[0].Message.Content) == "" {
+		return "", ErrEmptyResult
+	}
+	return strings.TrimSpace(result.Choices[0].Message.Content), nil
+}
+
+func (c SpeechClient) client() *http.Client {
+	if c.Client != nil {
+		return c.Client
+	}
+	return http.DefaultClient
+}
+
+func (c VisionClient) client() *http.Client {
+	if c.Client != nil {
+		return c.Client
+	}
+	return http.DefaultClient
+}
+
+func encodeBase64(data []byte) string {
+	return base64.StdEncoding.EncodeToString(data)
+}

adapters/multimodal/client_test.go

@@ -0,0 +1,65 @@
+package multimodal
+
+import (
+	"context"
+	"encoding/json"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+)
+
+func TestSpeechClientSendsMultipartAndReturnsTranscript(t *testing.T) {
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Path != "/inference" || r.Method != http.MethodPost {
+			t.Fatalf("unexpected speech request: %s %s", r.Method, r.URL.Path)
+		}
+		if err := r.ParseMultipartForm(1 << 20); err != nil {
+			t.Fatal(err)
+		}
+		file, _, err := r.FormFile("file")
+		if err != nil {
+			t.Fatal(err)
+		}
+		defer file.Close()
+		data, _ := io.ReadAll(file)
+		if string(data) != "audio" {
+			t.Fatalf("audio payload = %q", data)
+		}
+		_ = json.NewEncoder(w).Encode(map[string]string{"text": "распознанная фраза"})
+	}))
+	defer server.Close()
+	text, err := (SpeechClient{BaseURL: server.URL, Client: server.Client()}).Transcribe(context.Background(), "voice.wav", "audio/wav", []byte("audio"))
+	if err != nil || text != "распознанная фраза" {
+		t.Fatalf("transcript=%q err=%v", text, err)
+	}
+}
+
+func TestVisionClientSendsImageDataAndInstruction(t *testing.T) {
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		var payload struct {
+			Messages []struct {
+				Content []struct {
+					Type     string `json:"type"`
+					Text     string `json:"text"`
+					ImageURL *struct {
+						URL string `json:"url"`
+					} `json:"image_url"`
+				} `json:"content"`
+			} `json:"messages"`
+		}
+		if err := json.NewDecoder(r.Body).Decode(&payload); err != nil {
+			t.Fatal(err)
+		}
+		if len(payload.Messages) != 1 || len(payload.Messages[0].Content) != 2 || !strings.Contains(payload.Messages[0].Content[0].Text, "Опиши") || payload.Messages[0].Content[1].ImageURL == nil || !strings.HasPrefix(payload.Messages[0].Content[1].ImageURL.URL, "data:image/png;base64,") {
+			t.Fatalf("unexpected vision payload: %+v", payload)
+		}
+		_ = json.NewEncoder(w).Encode(map[string]any{"choices": []any{map[string]any{"message": map[string]string{"content": "на изображении круг"}}}})
+	}))
+	defer server.Close()
+	text, err := (VisionClient{BaseURL: server.URL + "/v1", Client: server.Client()}).Describe(context.Background(), "vision-model", "image/png", []byte{0, 1, 2}, "Опиши изображение")
+	if err != nil || text != "на изображении круг" {
+		t.Fatalf("description=%q err=%v", text, err)
+	}
+}

cmd/cognitive-worker/main.go

@@ -7,6 +7,7 @@ import (
 	"errors"
 	"fmt"
 	"git.cin.su/m/Assistant/adapters/models"
+	"git.cin.su/m/Assistant/adapters/multimodal"
 	"git.cin.su/m/Assistant/contracts/capability"
 	"git.cin.su/m/Assistant/contracts/cognitive"
 	"git.cin.su/m/Assistant/contracts/identity"
@@ -170,6 +171,15 @@ func main() {
 	mux.HandleFunc("POST /v1/advisories", gateway.advisoryHandler)
 	mux.HandleFunc("POST /v1/interpretations", gateway.interpretationHandler)
 	mux.HandleFunc("POST /v1/task-evaluations/execute", gateway.taskEvaluationHandler)
+	multimodalGateway := multimodalGateway{
+		speech: multimodal.SpeechClient{BaseURL: envOr("LOCAL_SPEECH_PROVIDER_URL", "http://llm-local-speech:8080"), Client: &http.Client{Timeout: 120 * time.Second}},
+		vision: multimodal.VisionClient{BaseURL: envOr("LOCAL_VISION_PROVIDER_URL", "http://llm-local-vision:8080/v1"), Client: &http.Client{Timeout: 180 * time.Second}},
+		model:  envOr("LOCAL_VISION_MODEL", "/models/gemma-4-26B-A4B-it-UD-Q4_K_M.gguf"),
+		now:    time.Now,
+	}
+	mux.HandleFunc("POST /v1/multimodal/transcribe", multimodalGateway.handler(capability.MultimodalTranscribe))
+	mux.HandleFunc("POST /v1/multimodal/understand", multimodalGateway.handler(capability.MultimodalUnderstand))
+	mux.HandleFunc("POST /v1/multimodal/ocr", multimodalGateway.handler(capability.MultimodalRecognizeOCR))
 	discovery := discoveryService{resolver: resolver, providers: discoveries, owner: identity.PrincipalID(envOr("COGNITIVE_OWNER_PRINCIPAL_ID", "domovoy-creator-principal")), now: time.Now}
 	mux.HandleFunc("POST /v1/provider-discovery/{provider}", discovery.handler)
 	mux.Handle("POST /v1/provider-discovery/{provider}/qualify", modelQualificationHandler(discoveries, string(envOr("COGNITIVE_OWNER_PRINCIPAL_ID", "domovoy-creator-principal")), httpQualificationObserver{resolver: resolver}))

cmd/cognitive-worker/multimodal.go

@@ -0,0 +1,75 @@
+package main
+
+import (
+	"encoding/json"
+	"io"
+	"log"
+	"net/http"
+	"strings"
+	"time"
+
+	"git.cin.su/m/Assistant/adapters/multimodal"
+	"git.cin.su/m/Assistant/contracts/capability"
+)
+
+const maxMultimodalUploadBytes = 12 << 20
+
+type multimodalGateway struct {
+	speech multimodal.SpeechClient
+	vision multimodal.VisionClient
+	model  string
+	now    func() time.Time
+}
+
+func (g multimodalGateway) handler(operation capability.MultimodalOperation) http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		r.Body = http.MaxBytesReader(w, r.Body, maxMultimodalUploadBytes)
+		if err := r.ParseMultipartForm(maxMultimodalUploadBytes); err != nil {
+			http.Error(w, "media upload is too large or invalid", http.StatusRequestEntityTooLarge)
+			return
+		}
+		file, header, err := r.FormFile("file")
+		if err != nil {
+			http.Error(w, "media file is required", http.StatusBadRequest)
+			return
+		}
+		defer file.Close()
+		data, err := io.ReadAll(io.LimitReader(file, maxMultimodalUploadBytes+1))
+		if err != nil || len(data) == 0 || len(data) > maxMultimodalUploadBytes {
+			http.Error(w, "media file is invalid", http.StatusBadRequest)
+			return
+		}
+		contentType := header.Header.Get("Content-Type")
+		if contentType == "" {
+			contentType = "application/octet-stream"
+		}
+		var text string
+		if operation == capability.MultimodalTranscribe {
+			text, err = g.speech.Transcribe(r.Context(), header.Filename, contentType, data)
+		} else {
+			instruction := "Опиши, что изображено на фотографии, коротко и по существу. Не выдумывай нечитаемые детали."
+			if operation == capability.MultimodalRecognizeOCR {
+				instruction = "Точно распознай весь читаемый текст на изображении. Сохрани строки и символы; нечитаемые места обозначь как [неразборчиво]. Не добавляй пояснений."
+			}
+			text, err = g.vision.Describe(r.Context(), g.model, contentType, data, instruction)
+		}
+		if err != nil {
+			logMultimodalFailure(operation, err)
+			http.Error(w, "local media capability unavailable", http.StatusBadGateway)
+			return
+		}
+		result := capability.MultimodalResponse{Operation: operation, Text: strings.TrimSpace(text), ObservedAt: g.now().UnixMilli(), ConfidenceNote: "Результат получен локальным специализированным сервисом; нечитаемые или неоднозначные фрагменты требуют проверки."}
+		if operation == capability.MultimodalTranscribe {
+			result.Provider = "local-whisper"
+		} else {
+			result.Provider = "local-vision"
+		}
+		w.Header().Set("Content-Type", "application/json")
+		_ = json.NewEncoder(w).Encode(result)
+	}
+}
+
+func logMultimodalFailure(operation capability.MultimodalOperation, err error) {
+	// Keep provider details in server diagnostics; never return them to chat.
+	log.Printf("multimodal operation %s failed: %v", operation, err)
+}

cmd/cognitive-worker/multimodal_test.go

@@ -0,0 +1,49 @@
+package main
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"mime/multipart"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+	"time"
+
+	"git.cin.su/m/Assistant/adapters/multimodal"
+	"git.cin.su/m/Assistant/contracts/capability"
+)
+
+func TestMultimodalTranscriptionHandlerReturnsLocalResult(t *testing.T) {
+	upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Path != "/inference" || r.Method != http.MethodPost {
+			t.Fatalf("unexpected upstream request: %s %s", r.Method, r.URL.Path)
+		}
+		_ = json.NewEncoder(w).Encode(map[string]string{"text": "добрый день"})
+	}))
+	defer upstream.Close()
+	server := multimodalGateway{speech: multimodal.SpeechClient{BaseURL: upstream.URL, Client: upstream.Client()}, now: time.Now}
+	request := multipartRequest(t, "/v1/multimodal/transcribe", "voice.wav", []byte("audio"))
+	response := httptest.NewRecorder()
+	server.handler(capability.MultimodalTranscribe)(response, request)
+	if response.Code != http.StatusOK || !bytes.Contains(response.Body.Bytes(), []byte("добрый день")) {
+		t.Fatalf("status=%d body=%s", response.Code, response.Body.String())
+	}
+}
+
+func multipartRequest(t *testing.T, path, filename string, data []byte) *http.Request {
+	t.Helper()
+	body := &bytes.Buffer{}
+	writer := multipart.NewWriter(body)
+	part, err := writer.CreateFormFile("file", filename)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, err := part.Write(data); err != nil || writer.Close() != nil {
+		t.Fatal("failed to build multipart request")
+	}
+	request := httptest.NewRequest(http.MethodPost, path, body)
+	request = request.WithContext(context.Background())
+	request.Header.Set("Content-Type", writer.FormDataContentType())
+	return request
+}

cmd/creator-web/chat_test.go

@@ -40,6 +40,11 @@ func TestChatPageServedWithoutInternalIDs(t *testing.T) {
 	if !strings.Contains(body, "версия 0123456789ab") || !strings.Contains(body, "обновлено 22.08.2026 03:04 UTC") || !strings.Contains(body, "datetime=\"2026-08-22T03:04:05Z\"") {
 		t.Fatalf("/chat does not render build metadata: %q", body)
 	}
+	for _, required := range []string{"media-tools", "audio-input", "image-input", "ocr-input"} {
+		if !strings.Contains(body, required) {
+			t.Fatalf("/chat is missing multimodal control %q", required)
+		}
+	}
 	for _, path := range []string{"/", "/chat.html"} {
 		alternate := httptest.NewRecorder()
 		handler.ServeHTTP(alternate, httptest.NewRequest(http.MethodGet, path, nil))
@@ -60,6 +65,11 @@ func TestChatScriptExplainsSessionAndRuntimeFailures(t *testing.T) {
 			t.Fatalf("chat script is missing recovery message %q", required)
 		}
 	}
+	for _, required := range []string{"uploadMedia", "/api/multimodal/transcribe", "/api/multimodal/understand", "/api/multimodal/ocr"} {
+		if !strings.Contains(script, required) {
+			t.Fatalf("chat script is missing multimodal route %q", required)
+		}
+	}
 }
 
 func TestPublicSurfacesSeparateChatAndConsole(t *testing.T) {

cmd/creator-web/multimodal_test.go

@@ -0,0 +1,57 @@
+package main
+
+import (
+	"bytes"
+	"encoding/json"
+	"mime/multipart"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+
+	"git.cin.su/m/Assistant/contracts/capability"
+	"git.cin.su/m/Assistant/contracts/identity"
+)
+
+func TestMultimodalUploadRequiresSessionAndProxiesLocalResult(t *testing.T) {
+	core := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Path != "/v1/creator/sessions/current" || r.Header.Get("Authorization") != "Bearer session" {
+			t.Fatalf("unexpected session request: %s %q", r.URL.Path, r.Header.Get("Authorization"))
+		}
+		_ = json.NewEncoder(w).Encode(identity.CreatorSession{ActorID: "actor", PrincipalID: "principal"})
+	}))
+	defer core.Close()
+	cognitive := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Path != "/v1/multimodal/transcribe" || r.Method != http.MethodPost {
+			t.Fatalf("unexpected multimodal request: %s %s", r.Method, r.URL.Path)
+		}
+		_ = json.NewEncoder(w).Encode(capability.MultimodalResponse{Operation: capability.MultimodalTranscribe, Text: "локальный текст", Provider: "local-whisper", ObservedAt: 1})
+	}))
+	defer cognitive.Close()
+	handler, err := newHandler(core.URL, "http://continuity.invalid", "http://commit.invalid", core.Client(), cognitive.URL)
+	if err != nil {
+		t.Fatal(err)
+	}
+	denied := httptest.NewRecorder()
+	handler.ServeHTTP(denied, httptest.NewRequest(http.MethodPost, "/api/multimodal/transcribe", bytes.NewReader(nil)))
+	if denied.Code != http.StatusUnauthorized {
+		t.Fatalf("unauthenticated upload status=%d", denied.Code)
+	}
+	body := &bytes.Buffer{}
+	writer := multipart.NewWriter(body)
+	part, err := writer.CreateFormFile("file", "voice.wav")
+	if err != nil {
+		t.Fatal(err)
+	}
+	_, _ = part.Write([]byte("audio"))
+	if err := writer.Close(); err != nil {
+		t.Fatal(err)
+	}
+	request := httptest.NewRequest(http.MethodPost, "/api/multimodal/transcribe", body)
+	request.Header.Set("Authorization", "Bearer session")
+	request.Header.Set("Content-Type", writer.FormDataContentType())
+	response := httptest.NewRecorder()
+	handler.ServeHTTP(response, request)
+	if response.Code != http.StatusOK || !bytes.Contains(response.Body.Bytes(), []byte("локальный текст")) {
+		t.Fatalf("status=%d body=%s", response.Code, response.Body.String())
+	}
+}

cmd/creator-web/server.go

@@ -7,7 +7,9 @@ import (
 	"fmt"
 	"io"
 	"io/fs"
+	"mime/multipart"
 	"net/http"
+	"net/textproto"
 	"net/url"
 	"os"
 	"strings"
@@ -135,6 +137,10 @@ func (g *gateway) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 			g.handleChatMessage(w, r)
 			return
 		}
+		if r.Method == http.MethodPost && strings.HasPrefix(r.URL.Path, "/api/multimodal/") {
+			g.handleMultimodal(w, r)
+			return
+		}
 		if r.Method == http.MethodGet && r.URL.Path == "/api/fleet/readiness" {
 			g.handleFleetReadiness(w, r)
 			return
@@ -650,6 +656,88 @@ func (g *gateway) postTrustedJSON(r *http.Request, upstream *url.URL, path, auth
 	return response.StatusCode, nil
 }
 
+func (g *gateway) handleMultimodal(w http.ResponseWriter, r *http.Request) {
+	authorization := r.Header.Get("Authorization")
+	if authorization == "" {
+		http.Error(w, "creator session denied", http.StatusUnauthorized)
+		return
+	}
+	var session identity.CreatorSession
+	status, err := g.getTrustedJSON(r, g.core, "/v1/creator/sessions/current", "", authorization, &session)
+	if err != nil || status != http.StatusOK || session.ActorID == "" || session.PrincipalID == "" {
+		http.Error(w, "creator session denied", http.StatusUnauthorized)
+		return
+	}
+	const maxMediaBytes = 12 << 20
+	r.Body = http.MaxBytesReader(w, r.Body, maxMediaBytes)
+	if err := r.ParseMultipartForm(maxMediaBytes); err != nil {
+		http.Error(w, "media upload is too large or invalid", http.StatusRequestEntityTooLarge)
+		return
+	}
+	file, header, err := r.FormFile("file")
+	if err != nil {
+		http.Error(w, "media file is required", http.StatusBadRequest)
+		return
+	}
+	defer file.Close()
+	data, err := io.ReadAll(io.LimitReader(file, maxMediaBytes+1))
+	if err != nil || len(data) == 0 || len(data) > maxMediaBytes {
+		http.Error(w, "media file is invalid", http.StatusBadRequest)
+		return
+	}
+	path := ""
+	switch r.URL.Path {
+	case "/api/multimodal/transcribe":
+		path = "/v1/multimodal/transcribe"
+	case "/api/multimodal/understand":
+		path = "/v1/multimodal/understand"
+	case "/api/multimodal/ocr":
+		path = "/v1/multimodal/ocr"
+	default:
+		http.Error(w, "multimodal route not found", http.StatusNotFound)
+		return
+	}
+	body := &bytes.Buffer{}
+	writer := multipart.NewWriter(body)
+	mediaHeader := make(textproto.MIMEHeader)
+	mediaHeader.Set("Content-Disposition", fmt.Sprintf(`form-data; name="file"; filename="%s"`, strings.ReplaceAll(header.Filename, `"`, "")))
+	mediaType := header.Header.Get("Content-Type")
+	if mediaType == "" {
+		mediaType = "application/octet-stream"
+	}
+	mediaHeader.Set("Content-Type", mediaType)
+	part, err := writer.CreatePart(mediaHeader)
+	if err != nil {
+		http.Error(w, "media upload unavailable", http.StatusServiceUnavailable)
+		return
+	}
+	if _, err := part.Write(data); err != nil || writer.Close() != nil {
+		http.Error(w, "media upload unavailable", http.StatusServiceUnavailable)
+		return
+	}
+	target := *g.cognitive
+	target.Path = path
+	request, err := http.NewRequestWithContext(r.Context(), http.MethodPost, target.String(), body)
+	if err != nil {
+		http.Error(w, "media capability unavailable", http.StatusBadGateway)
+		return
+	}
+	request.Header.Set("Content-Type", writer.FormDataContentType())
+	response, err := g.client.Do(request)
+	if err != nil {
+		http.Error(w, "media capability unavailable", http.StatusBadGateway)
+		return
+	}
+	defer response.Body.Close()
+	if response.StatusCode != http.StatusOK {
+		_, _ = io.Copy(io.Discard, io.LimitReader(response.Body, maxResponseBytes))
+		http.Error(w, "media capability unavailable", http.StatusBadGateway)
+		return
+	}
+	w.Header().Set("Content-Type", "application/json")
+	_, _ = io.Copy(w, io.LimitReader(response.Body, maxResponseBytes))
+}
+
 func decodeStrictJSON(reader io.Reader, limit int64, destination any) error {
 	decoder := json.NewDecoder(io.LimitReader(reader, limit+1))
 	decoder.DisallowUnknownFields()

cmd/creator-web/static/assets/chat.css

@@ -75,6 +75,10 @@ textarea { resize: vertical; min-height: 84px; max-height: 240px; padding: 14px;
 .composer { width: min(100%, 920px); margin: 0 auto; padding: 16px 18px 10px; display: grid; grid-template-columns: minmax(0, 1fr) 130px; gap: 12px; background: linear-gradient(transparent, var(--base) 20%); }
 .composer button { height: 48px; align-self: end; }
 .composer button:disabled { cursor: wait; filter: saturate(.3); }
+.media-tools { width: min(100%, 920px); margin: 0 auto; padding: 0 18px 10px; display: flex; flex-wrap: wrap; align-items: center; gap: 8px; color: var(--muted); font: 10px/1 var(--mono); text-transform: uppercase; letter-spacing: .06em; }
+.media-tools label { display: inline-flex; align-items: center; gap: 6px; min-height: 30px; padding: 0 8px; border: 1px solid var(--line); color: var(--ink); cursor: pointer; }
+.media-tools input { max-width: 150px; font: 10px var(--mono); }
+.media-tools span { color: var(--signal); }
 .conversation-starters { width: min(100%, 920px); margin: -4px auto 0; padding: 0 18px 18px; display: flex; flex-wrap: wrap; align-items: center; gap: 8px; color: var(--muted); font: 10px/1 var(--mono); letter-spacing: .06em; text-transform: uppercase; }
 .conversation-starters button { min-height: 32px; padding: 0 10px; border: 1px solid var(--line); color: var(--ink); background: #101814; font: 10px/1 var(--mono); }
 .conversation-starters button:hover, .conversation-starters button:focus-visible { border-color: var(--signal); color: var(--signal); outline: none; }

cmd/creator-web/static/assets/chat.js

@@ -132,6 +132,16 @@ async function api(path, options = {}) {
   return response.json();
 }
 
+async function uploadMedia(path, file) {
+  const body = new FormData();
+  body.append("file", file, file.name);
+  const headers = {};
+  if (token) headers.Authorization = `Bearer ${token}`;
+  const response = await fetch(path, {method: "POST", headers, body});
+  if (!response.ok) throw new ChatApiError(response.status, await response.text());
+  return response.json();
+}
+
 function expireSession(message = "") {
   token = "";
   conversationID = "";
@@ -320,6 +330,34 @@ chatForm.addEventListener("submit", async (event) => {
   await sendChatMessage(text);
 });
 
+async function processMedia(input, path, label) {
+  const file = input?.files?.[0];
+  if (!file || !mediaStatus) return;
+  mediaStatus.textContent = `${label} обрабатывается...`;
+  input.value = "";
+  try {
+    const result = await uploadMedia(path, file);
+    const text = String(result.text || "").trim();
+    if (!text) throw new Error("empty media result");
+    mediaStatus.textContent = "Готово";
+    await sendChatMessage(`Пользователь передал ${label.toLowerCase()}. Обработай содержание и ответь по смыслу:\n${text}`);
+  } catch (error) {
+    if (error instanceof ChatApiError && (error.status === 401 || error.status === 403)) {
+      expireSession("Сессия устарела. Войдите снова, чтобы продолжить разговор.");
+    } else {
+      mediaStatus.textContent = `${label} сейчас недоступно.`;
+    }
+  }
+}
+
+const mediaStatus = document.querySelector("#media-status");
+const audioInput = document.querySelector("#audio-input");
+const imageInput = document.querySelector("#image-input");
+const ocrInput = document.querySelector("#ocr-input");
+if (audioInput) audioInput.addEventListener("change", () => processMedia(audioInput, "/api/multimodal/transcribe", "Распознанное аудио"));
+if (imageInput) imageInput.addEventListener("change", () => processMedia(imageInput, "/api/multimodal/understand", "Описание изображения"));
+if (ocrInput) ocrInput.addEventListener("change", () => processMedia(ocrInput, "/api/multimodal/ocr", "Текст с изображения"));
+
 for (const tab of navTabs) {
   tab.addEventListener("click", () => activateTab(tab.dataset.tab));
 }

cmd/creator-web/static/chat.html

@@ -60,6 +60,12 @@
               <textarea id="message" rows="3" maxlength="16384" placeholder="Напишите Домовому обычное сообщение..." required></textarea>
               <button id="send" type="submit">Отправить</button>
             </form>
+            <div class="media-tools" aria-label="Мультимодальные сообщения">
+              <label>Аудио <input id="audio-input" type="file" accept="audio/*"></label>
+              <label>Фото <input id="image-input" type="file" accept="image/*"></label>
+              <label>Текст с фото <input id="ocr-input" type="file" accept="image/*"></label>
+              <span id="media-status" role="status" aria-live="polite"></span>
+            </div>
             <div class="conversation-starters" aria-label="Частые вопросы">
               <span>Можно начать так:</span>
               <button type="button" data-starter="Проверь локальный канал мышления">Проверь канал</button>

contracts/capability/multimodal.go

@@ -0,0 +1,20 @@
+package capability
+
+// MultimodalOperation identifies one bounded local media skill. It is kept
+// separate from general cognition so health, routing and failure evidence do
+// not collapse into a generic model response.
+type MultimodalOperation string
+
+const (
+	MultimodalTranscribe   MultimodalOperation = "TRANSCRIBE_AUDIO"
+	MultimodalUnderstand   MultimodalOperation = "UNDERSTAND_IMAGE"
+	MultimodalRecognizeOCR MultimodalOperation = "RECOGNIZE_OCR"
+)
+
+type MultimodalResponse struct {
+	Operation      MultimodalOperation `json:"operation"`
+	Text           string              `json:"text"`
+	Provider       string              `json:"provider,omitempty"`
+	ObservedAt     int64               `json:"observed_at_unix_millis"`
+	ConfidenceNote string              `json:"confidence_note,omitempty"`
+}

contracts/capability/types.go

@@ -29,6 +29,9 @@ const (
 	CapabilityKindCognitiveReasoning CapabilityKind = "COGNITIVE_REASONING"
 	CapabilityKindScheduleReminder   CapabilityKind = "SCHEDULE_REMINDER"
 	CapabilityKindVerifyFact         CapabilityKind = "VERIFY_FACT"
+	CapabilityKindSpeechToText       CapabilityKind = "SPEECH_TO_TEXT"
+	CapabilityKindImageUnderstanding CapabilityKind = "IMAGE_UNDERSTANDING"
+	CapabilityKindOCR                CapabilityKind = "OCR"
 )
 
 // Binding is Capability-owned correctness state. SecretRef is an opaque

dev/tasks/DEV-000295-household-scenario-audit.json

@@ -12,9 +12,12 @@
     "REQ-HOUSEHOLD-004"
   ],
   "allowed_change_scope": {
-    "modules": ["interaction", "capability", "epistemics", "situation", "work", "continuity"],
+    "modules": ["interaction", "capability", "epistemics", "situation", "work", "continuity", "cognitive-worker", "creator-web", "multimodal-adapters"],
     "paths": [
       "cmd/core-runtime/**",
+      "cmd/cognitive-worker/**",
+      "cmd/creator-web/**",
+      "adapters/multimodal/**",
       "modules/interaction/**",
       "modules/capability/**",
       "modules/epistemics/**",
@@ -34,7 +37,7 @@
       "security/**",
       "adapters/models/**",
       "docker-compose*.yml",
-      "deploy/**",
+      "deploy/images/**",
       "migrations/**",
       "go.mod",
       "go.sum"
@@ -56,12 +59,12 @@
       "contracts.interaction@v1",
       "contracts.work@v1"
     ],
-    "may_change": []
+    "may_change": ["multimodal capability kinds and internal speech/image/OCR transport contracts; no canonical persistence contract"]
   },
-  "semantic_owner": "interaction owns meaning classification; capability owns capability health and selection; existing semantic owners retain canonical state",
+  "semantic_owner": "interaction owns meaning classification; capability owns capability health and selection; cognitive-worker/adapters own provider transport; creator-web owns browser transport; existing semantic owners retain canonical state",
   "canonical_state": ["existing Interpretation, CapabilityResolution, Situation, Goal/Task and ProspectiveItem only"],
   "state_transitions": ["request -> meaning -> bounded capability route -> result/clarification/degraded answer -> verification evidence"],
-  "side_effects_and_privileges": ["Scenario tests are read-only/fake by default; real reminders, websites, messages and code changes remain behind existing Commit/Executor and authority boundaries."],
+  "side_effects_and_privileges": ["Scenario tests are read-only/fake by default; multimodal requests send user-provided media only to configured local speech/vision services; real reminders, websites, messages and code changes remain behind existing Commit/Executor and authority boundaries."],
   "persistence_and_migrations": ["No new tables, migrations or events; verify existing durable memory, situation, goal/task and continuity paths."],
   "failure_retry_restart": ["Each scenario distinguishes unavailable, partial and unknown outcome; retries must not duplicate external effects; restart restores context and commitments from canonical state."],
   "acceptance_scenarios": [
@@ -88,6 +91,9 @@
     "unit",
     "semantic-routing",
     "scenario-contract",
+    "multimodal-adapter",
+    "multimodal-proxy",
+    "multimodal-production-smoke",
     "degraded-mode",
     "restart-recovery",
     "architecture",