diff --git a/adapters/multimodal/client.go b/adapters/multimodal/client.go new file mode 100644 index 0000000..4335f71 --- /dev/null +++ b/adapters/multimodal/client.go @@ -0,0 +1,166 @@ +package multimodal + +import ( + "bytes" + "context" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "io" + "mime/multipart" + "net/http" + "net/textproto" + "strings" +) + +var ErrEmptyResult = errors.New("multimodal provider returned an empty result") + +type SpeechClient struct { + BaseURL string + Client *http.Client +} + +type VisionClient struct { + BaseURL string + Client *http.Client +} + +func (c SpeechClient) Transcribe(ctx context.Context, filename, contentType string, data []byte) (string, error) { + if len(data) == 0 { + return "", errors.New("audio payload is empty") + } + body := &bytes.Buffer{} + writer := multipart.NewWriter(body) + header := make(textproto.MIMEHeader) + header.Set("Content-Disposition", fmt.Sprintf(`form-data; name="file"; filename="%s"`, strings.ReplaceAll(filename, `"`, ""))) + header.Set("Content-Type", contentType) + part, err := writer.CreatePart(header) + if err != nil { + return "", err + } + if _, err := part.Write(data); err != nil { + return "", err + } + if err := writer.WriteField("response_format", "json"); err != nil { + return "", err + } + if err := writer.Close(); err != nil { + return "", err + } + request, err := http.NewRequestWithContext(ctx, http.MethodPost, strings.TrimRight(c.BaseURL, "/")+"/inference", body) + if err != nil { + return "", err + } + request.Header.Set("Content-Type", writer.FormDataContentType()) + response, err := c.client().Do(request) + if err != nil { + return "", err + } + defer response.Body.Close() + if response.StatusCode != http.StatusOK { + return "", fmt.Errorf("speech provider status %d", response.StatusCode) + } + var result struct { + Text string `json:"text"` + } + if err := json.NewDecoder(io.LimitReader(response.Body, 1<<20)).Decode(&result); err != nil { + return "", err + } + if strings.TrimSpace(result.Text) == "" { + return "", ErrEmptyResult + } + return strings.TrimSpace(result.Text), nil +} + +func (c VisionClient) Describe(ctx context.Context, model, contentType string, data []byte, instruction string) (string, error) { + if len(data) == 0 { + return "", errors.New("image payload is empty") + } + payload := struct { + Model string `json:"model"` + Messages []struct { + Role string `json:"role"` + Content []struct { + Type string `json:"type"` + Text string `json:"text,omitempty"` + ImageURL *struct { + URL string `json:"url"` + } `json:"image_url,omitempty"` + } `json:"content"` + } `json:"messages"` + MaxTokens int `json:"max_tokens"` + }{} + payload.Model = model + payload.MaxTokens = 1024 + payload.Messages = append(payload.Messages, struct { + Role string `json:"role"` + Content []struct { + Type string `json:"type"` + Text string `json:"text,omitempty"` + ImageURL *struct { + URL string `json:"url"` + } `json:"image_url,omitempty"` + } `json:"content"` + }{Role: "user", Content: []struct { + Type string `json:"type"` + Text string `json:"text,omitempty"` + ImageURL *struct { + URL string `json:"url"` + } `json:"image_url,omitempty"` + }{ + {Type: "text", Text: instruction}, + {Type: "image_url", ImageURL: &struct { + URL string `json:"url"` + }{URL: "data:" + contentType + ";base64," + encodeBase64(data)}, + }}}) + encoded, err := json.Marshal(payload) + if err != nil { + return "", err + } + request, err := http.NewRequestWithContext(ctx, http.MethodPost, strings.TrimRight(c.BaseURL, "/")+"/chat/completions", bytes.NewReader(encoded)) + if err != nil { + return "", err + } + request.Header.Set("Content-Type", "application/json") + response, err := c.client().Do(request) + if err != nil { + return "", err + } + defer response.Body.Close() + if response.StatusCode != http.StatusOK { + return "", fmt.Errorf("vision provider status %d", response.StatusCode) + } + var result struct { + Choices []struct { + Message struct { + Content string `json:"content"` + } `json:"message"` + } `json:"choices"` + } + if err := json.NewDecoder(io.LimitReader(response.Body, 2<<20)).Decode(&result); err != nil { + return "", err + } + if len(result.Choices) == 0 || strings.TrimSpace(result.Choices[0].Message.Content) == "" { + return "", ErrEmptyResult + } + return strings.TrimSpace(result.Choices[0].Message.Content), nil +} + +func (c SpeechClient) client() *http.Client { + if c.Client != nil { + return c.Client + } + return http.DefaultClient +} + +func (c VisionClient) client() *http.Client { + if c.Client != nil { + return c.Client + } + return http.DefaultClient +} + +func encodeBase64(data []byte) string { + return base64.StdEncoding.EncodeToString(data) +} diff --git a/adapters/multimodal/client_test.go b/adapters/multimodal/client_test.go new file mode 100644 index 0000000..a176d24 --- /dev/null +++ b/adapters/multimodal/client_test.go @@ -0,0 +1,65 @@ +package multimodal + +import ( + "context" + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" +) + +func TestSpeechClientSendsMultipartAndReturnsTranscript(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/inference" || r.Method != http.MethodPost { + t.Fatalf("unexpected speech request: %s %s", r.Method, r.URL.Path) + } + if err := r.ParseMultipartForm(1 << 20); err != nil { + t.Fatal(err) + } + file, _, err := r.FormFile("file") + if err != nil { + t.Fatal(err) + } + defer file.Close() + data, _ := io.ReadAll(file) + if string(data) != "audio" { + t.Fatalf("audio payload = %q", data) + } + _ = json.NewEncoder(w).Encode(map[string]string{"text": "распознанная фраза"}) + })) + defer server.Close() + text, err := (SpeechClient{BaseURL: server.URL, Client: server.Client()}).Transcribe(context.Background(), "voice.wav", "audio/wav", []byte("audio")) + if err != nil || text != "распознанная фраза" { + t.Fatalf("transcript=%q err=%v", text, err) + } +} + +func TestVisionClientSendsImageDataAndInstruction(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var payload struct { + Messages []struct { + Content []struct { + Type string `json:"type"` + Text string `json:"text"` + ImageURL *struct { + URL string `json:"url"` + } `json:"image_url"` + } `json:"content"` + } `json:"messages"` + } + if err := json.NewDecoder(r.Body).Decode(&payload); err != nil { + t.Fatal(err) + } + if len(payload.Messages) != 1 || len(payload.Messages[0].Content) != 2 || !strings.Contains(payload.Messages[0].Content[0].Text, "Опиши") || payload.Messages[0].Content[1].ImageURL == nil || !strings.HasPrefix(payload.Messages[0].Content[1].ImageURL.URL, "data:image/png;base64,") { + t.Fatalf("unexpected vision payload: %+v", payload) + } + _ = json.NewEncoder(w).Encode(map[string]any{"choices": []any{map[string]any{"message": map[string]string{"content": "на изображении круг"}}}}) + })) + defer server.Close() + text, err := (VisionClient{BaseURL: server.URL + "/v1", Client: server.Client()}).Describe(context.Background(), "vision-model", "image/png", []byte{0, 1, 2}, "Опиши изображение") + if err != nil || text != "на изображении круг" { + t.Fatalf("description=%q err=%v", text, err) + } +} diff --git a/cmd/cognitive-worker/main.go b/cmd/cognitive-worker/main.go index 9b679b5..02ea091 100644 --- a/cmd/cognitive-worker/main.go +++ b/cmd/cognitive-worker/main.go @@ -7,6 +7,7 @@ import ( "errors" "fmt" "git.cin.su/m/Assistant/adapters/models" + "git.cin.su/m/Assistant/adapters/multimodal" "git.cin.su/m/Assistant/contracts/capability" "git.cin.su/m/Assistant/contracts/cognitive" "git.cin.su/m/Assistant/contracts/identity" @@ -170,6 +171,15 @@ func main() { mux.HandleFunc("POST /v1/advisories", gateway.advisoryHandler) mux.HandleFunc("POST /v1/interpretations", gateway.interpretationHandler) mux.HandleFunc("POST /v1/task-evaluations/execute", gateway.taskEvaluationHandler) + multimodalGateway := multimodalGateway{ + speech: multimodal.SpeechClient{BaseURL: envOr("LOCAL_SPEECH_PROVIDER_URL", "http://llm-local-speech:8080"), Client: &http.Client{Timeout: 120 * time.Second}}, + vision: multimodal.VisionClient{BaseURL: envOr("LOCAL_VISION_PROVIDER_URL", "http://llm-local-vision:8080/v1"), Client: &http.Client{Timeout: 180 * time.Second}}, + model: envOr("LOCAL_VISION_MODEL", "/models/gemma-4-26B-A4B-it-UD-Q4_K_M.gguf"), + now: time.Now, + } + mux.HandleFunc("POST /v1/multimodal/transcribe", multimodalGateway.handler(capability.MultimodalTranscribe)) + mux.HandleFunc("POST /v1/multimodal/understand", multimodalGateway.handler(capability.MultimodalUnderstand)) + mux.HandleFunc("POST /v1/multimodal/ocr", multimodalGateway.handler(capability.MultimodalRecognizeOCR)) discovery := discoveryService{resolver: resolver, providers: discoveries, owner: identity.PrincipalID(envOr("COGNITIVE_OWNER_PRINCIPAL_ID", "domovoy-creator-principal")), now: time.Now} mux.HandleFunc("POST /v1/provider-discovery/{provider}", discovery.handler) mux.Handle("POST /v1/provider-discovery/{provider}/qualify", modelQualificationHandler(discoveries, string(envOr("COGNITIVE_OWNER_PRINCIPAL_ID", "domovoy-creator-principal")), httpQualificationObserver{resolver: resolver})) diff --git a/cmd/cognitive-worker/multimodal.go b/cmd/cognitive-worker/multimodal.go new file mode 100644 index 0000000..8feb2e2 --- /dev/null +++ b/cmd/cognitive-worker/multimodal.go @@ -0,0 +1,75 @@ +package main + +import ( + "encoding/json" + "io" + "log" + "net/http" + "strings" + "time" + + "git.cin.su/m/Assistant/adapters/multimodal" + "git.cin.su/m/Assistant/contracts/capability" +) + +const maxMultimodalUploadBytes = 12 << 20 + +type multimodalGateway struct { + speech multimodal.SpeechClient + vision multimodal.VisionClient + model string + now func() time.Time +} + +func (g multimodalGateway) handler(operation capability.MultimodalOperation) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + r.Body = http.MaxBytesReader(w, r.Body, maxMultimodalUploadBytes) + if err := r.ParseMultipartForm(maxMultimodalUploadBytes); err != nil { + http.Error(w, "media upload is too large or invalid", http.StatusRequestEntityTooLarge) + return + } + file, header, err := r.FormFile("file") + if err != nil { + http.Error(w, "media file is required", http.StatusBadRequest) + return + } + defer file.Close() + data, err := io.ReadAll(io.LimitReader(file, maxMultimodalUploadBytes+1)) + if err != nil || len(data) == 0 || len(data) > maxMultimodalUploadBytes { + http.Error(w, "media file is invalid", http.StatusBadRequest) + return + } + contentType := header.Header.Get("Content-Type") + if contentType == "" { + contentType = "application/octet-stream" + } + var text string + if operation == capability.MultimodalTranscribe { + text, err = g.speech.Transcribe(r.Context(), header.Filename, contentType, data) + } else { + instruction := "Опиши, что изображено на фотографии, коротко и по существу. Не выдумывай нечитаемые детали." + if operation == capability.MultimodalRecognizeOCR { + instruction = "Точно распознай весь читаемый текст на изображении. Сохрани строки и символы; нечитаемые места обозначь как [неразборчиво]. Не добавляй пояснений." + } + text, err = g.vision.Describe(r.Context(), g.model, contentType, data, instruction) + } + if err != nil { + logMultimodalFailure(operation, err) + http.Error(w, "local media capability unavailable", http.StatusBadGateway) + return + } + result := capability.MultimodalResponse{Operation: operation, Text: strings.TrimSpace(text), ObservedAt: g.now().UnixMilli(), ConfidenceNote: "Результат получен локальным специализированным сервисом; нечитаемые или неоднозначные фрагменты требуют проверки."} + if operation == capability.MultimodalTranscribe { + result.Provider = "local-whisper" + } else { + result.Provider = "local-vision" + } + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(result) + } +} + +func logMultimodalFailure(operation capability.MultimodalOperation, err error) { + // Keep provider details in server diagnostics; never return them to chat. + log.Printf("multimodal operation %s failed: %v", operation, err) +} diff --git a/cmd/cognitive-worker/multimodal_test.go b/cmd/cognitive-worker/multimodal_test.go new file mode 100644 index 0000000..bc0ae07 --- /dev/null +++ b/cmd/cognitive-worker/multimodal_test.go @@ -0,0 +1,49 @@ +package main + +import ( + "bytes" + "context" + "encoding/json" + "mime/multipart" + "net/http" + "net/http/httptest" + "testing" + "time" + + "git.cin.su/m/Assistant/adapters/multimodal" + "git.cin.su/m/Assistant/contracts/capability" +) + +func TestMultimodalTranscriptionHandlerReturnsLocalResult(t *testing.T) { + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/inference" || r.Method != http.MethodPost { + t.Fatalf("unexpected upstream request: %s %s", r.Method, r.URL.Path) + } + _ = json.NewEncoder(w).Encode(map[string]string{"text": "добрый день"}) + })) + defer upstream.Close() + server := multimodalGateway{speech: multimodal.SpeechClient{BaseURL: upstream.URL, Client: upstream.Client()}, now: time.Now} + request := multipartRequest(t, "/v1/multimodal/transcribe", "voice.wav", []byte("audio")) + response := httptest.NewRecorder() + server.handler(capability.MultimodalTranscribe)(response, request) + if response.Code != http.StatusOK || !bytes.Contains(response.Body.Bytes(), []byte("добрый день")) { + t.Fatalf("status=%d body=%s", response.Code, response.Body.String()) + } +} + +func multipartRequest(t *testing.T, path, filename string, data []byte) *http.Request { + t.Helper() + body := &bytes.Buffer{} + writer := multipart.NewWriter(body) + part, err := writer.CreateFormFile("file", filename) + if err != nil { + t.Fatal(err) + } + if _, err := part.Write(data); err != nil || writer.Close() != nil { + t.Fatal("failed to build multipart request") + } + request := httptest.NewRequest(http.MethodPost, path, body) + request = request.WithContext(context.Background()) + request.Header.Set("Content-Type", writer.FormDataContentType()) + return request +} diff --git a/cmd/creator-web/chat_test.go b/cmd/creator-web/chat_test.go index ab68d5a..07c2e79 100644 --- a/cmd/creator-web/chat_test.go +++ b/cmd/creator-web/chat_test.go @@ -40,6 +40,11 @@ func TestChatPageServedWithoutInternalIDs(t *testing.T) { if !strings.Contains(body, "версия 0123456789ab") || !strings.Contains(body, "обновлено 22.08.2026 03:04 UTC") || !strings.Contains(body, "datetime=\"2026-08-22T03:04:05Z\"") { t.Fatalf("/chat does not render build metadata: %q", body) } + for _, required := range []string{"media-tools", "audio-input", "image-input", "ocr-input"} { + if !strings.Contains(body, required) { + t.Fatalf("/chat is missing multimodal control %q", required) + } + } for _, path := range []string{"/", "/chat.html"} { alternate := httptest.NewRecorder() handler.ServeHTTP(alternate, httptest.NewRequest(http.MethodGet, path, nil)) @@ -60,6 +65,11 @@ func TestChatScriptExplainsSessionAndRuntimeFailures(t *testing.T) { t.Fatalf("chat script is missing recovery message %q", required) } } + for _, required := range []string{"uploadMedia", "/api/multimodal/transcribe", "/api/multimodal/understand", "/api/multimodal/ocr"} { + if !strings.Contains(script, required) { + t.Fatalf("chat script is missing multimodal route %q", required) + } + } } func TestPublicSurfacesSeparateChatAndConsole(t *testing.T) { diff --git a/cmd/creator-web/multimodal_test.go b/cmd/creator-web/multimodal_test.go new file mode 100644 index 0000000..7ef8412 --- /dev/null +++ b/cmd/creator-web/multimodal_test.go @@ -0,0 +1,57 @@ +package main + +import ( + "bytes" + "encoding/json" + "mime/multipart" + "net/http" + "net/http/httptest" + "testing" + + "git.cin.su/m/Assistant/contracts/capability" + "git.cin.su/m/Assistant/contracts/identity" +) + +func TestMultimodalUploadRequiresSessionAndProxiesLocalResult(t *testing.T) { + core := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/v1/creator/sessions/current" || r.Header.Get("Authorization") != "Bearer session" { + t.Fatalf("unexpected session request: %s %q", r.URL.Path, r.Header.Get("Authorization")) + } + _ = json.NewEncoder(w).Encode(identity.CreatorSession{ActorID: "actor", PrincipalID: "principal"}) + })) + defer core.Close() + cognitive := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/v1/multimodal/transcribe" || r.Method != http.MethodPost { + t.Fatalf("unexpected multimodal request: %s %s", r.Method, r.URL.Path) + } + _ = json.NewEncoder(w).Encode(capability.MultimodalResponse{Operation: capability.MultimodalTranscribe, Text: "локальный текст", Provider: "local-whisper", ObservedAt: 1}) + })) + defer cognitive.Close() + handler, err := newHandler(core.URL, "http://continuity.invalid", "http://commit.invalid", core.Client(), cognitive.URL) + if err != nil { + t.Fatal(err) + } + denied := httptest.NewRecorder() + handler.ServeHTTP(denied, httptest.NewRequest(http.MethodPost, "/api/multimodal/transcribe", bytes.NewReader(nil))) + if denied.Code != http.StatusUnauthorized { + t.Fatalf("unauthenticated upload status=%d", denied.Code) + } + body := &bytes.Buffer{} + writer := multipart.NewWriter(body) + part, err := writer.CreateFormFile("file", "voice.wav") + if err != nil { + t.Fatal(err) + } + _, _ = part.Write([]byte("audio")) + if err := writer.Close(); err != nil { + t.Fatal(err) + } + request := httptest.NewRequest(http.MethodPost, "/api/multimodal/transcribe", body) + request.Header.Set("Authorization", "Bearer session") + request.Header.Set("Content-Type", writer.FormDataContentType()) + response := httptest.NewRecorder() + handler.ServeHTTP(response, request) + if response.Code != http.StatusOK || !bytes.Contains(response.Body.Bytes(), []byte("локальный текст")) { + t.Fatalf("status=%d body=%s", response.Code, response.Body.String()) + } +} diff --git a/cmd/creator-web/server.go b/cmd/creator-web/server.go index 1c6483f..65ae1d4 100644 --- a/cmd/creator-web/server.go +++ b/cmd/creator-web/server.go @@ -7,7 +7,9 @@ import ( "fmt" "io" "io/fs" + "mime/multipart" "net/http" + "net/textproto" "net/url" "os" "strings" @@ -135,6 +137,10 @@ func (g *gateway) ServeHTTP(w http.ResponseWriter, r *http.Request) { g.handleChatMessage(w, r) return } + if r.Method == http.MethodPost && strings.HasPrefix(r.URL.Path, "/api/multimodal/") { + g.handleMultimodal(w, r) + return + } if r.Method == http.MethodGet && r.URL.Path == "/api/fleet/readiness" { g.handleFleetReadiness(w, r) return @@ -650,6 +656,88 @@ func (g *gateway) postTrustedJSON(r *http.Request, upstream *url.URL, path, auth return response.StatusCode, nil } +func (g *gateway) handleMultimodal(w http.ResponseWriter, r *http.Request) { + authorization := r.Header.Get("Authorization") + if authorization == "" { + http.Error(w, "creator session denied", http.StatusUnauthorized) + return + } + var session identity.CreatorSession + status, err := g.getTrustedJSON(r, g.core, "/v1/creator/sessions/current", "", authorization, &session) + if err != nil || status != http.StatusOK || session.ActorID == "" || session.PrincipalID == "" { + http.Error(w, "creator session denied", http.StatusUnauthorized) + return + } + const maxMediaBytes = 12 << 20 + r.Body = http.MaxBytesReader(w, r.Body, maxMediaBytes) + if err := r.ParseMultipartForm(maxMediaBytes); err != nil { + http.Error(w, "media upload is too large or invalid", http.StatusRequestEntityTooLarge) + return + } + file, header, err := r.FormFile("file") + if err != nil { + http.Error(w, "media file is required", http.StatusBadRequest) + return + } + defer file.Close() + data, err := io.ReadAll(io.LimitReader(file, maxMediaBytes+1)) + if err != nil || len(data) == 0 || len(data) > maxMediaBytes { + http.Error(w, "media file is invalid", http.StatusBadRequest) + return + } + path := "" + switch r.URL.Path { + case "/api/multimodal/transcribe": + path = "/v1/multimodal/transcribe" + case "/api/multimodal/understand": + path = "/v1/multimodal/understand" + case "/api/multimodal/ocr": + path = "/v1/multimodal/ocr" + default: + http.Error(w, "multimodal route not found", http.StatusNotFound) + return + } + body := &bytes.Buffer{} + writer := multipart.NewWriter(body) + mediaHeader := make(textproto.MIMEHeader) + mediaHeader.Set("Content-Disposition", fmt.Sprintf(`form-data; name="file"; filename="%s"`, strings.ReplaceAll(header.Filename, `"`, ""))) + mediaType := header.Header.Get("Content-Type") + if mediaType == "" { + mediaType = "application/octet-stream" + } + mediaHeader.Set("Content-Type", mediaType) + part, err := writer.CreatePart(mediaHeader) + if err != nil { + http.Error(w, "media upload unavailable", http.StatusServiceUnavailable) + return + } + if _, err := part.Write(data); err != nil || writer.Close() != nil { + http.Error(w, "media upload unavailable", http.StatusServiceUnavailable) + return + } + target := *g.cognitive + target.Path = path + request, err := http.NewRequestWithContext(r.Context(), http.MethodPost, target.String(), body) + if err != nil { + http.Error(w, "media capability unavailable", http.StatusBadGateway) + return + } + request.Header.Set("Content-Type", writer.FormDataContentType()) + response, err := g.client.Do(request) + if err != nil { + http.Error(w, "media capability unavailable", http.StatusBadGateway) + return + } + defer response.Body.Close() + if response.StatusCode != http.StatusOK { + _, _ = io.Copy(io.Discard, io.LimitReader(response.Body, maxResponseBytes)) + http.Error(w, "media capability unavailable", http.StatusBadGateway) + return + } + w.Header().Set("Content-Type", "application/json") + _, _ = io.Copy(w, io.LimitReader(response.Body, maxResponseBytes)) +} + func decodeStrictJSON(reader io.Reader, limit int64, destination any) error { decoder := json.NewDecoder(io.LimitReader(reader, limit+1)) decoder.DisallowUnknownFields() diff --git a/cmd/creator-web/static/assets/chat.css b/cmd/creator-web/static/assets/chat.css index 525649d..763b669 100644 --- a/cmd/creator-web/static/assets/chat.css +++ b/cmd/creator-web/static/assets/chat.css @@ -75,6 +75,10 @@ textarea { resize: vertical; min-height: 84px; max-height: 240px; padding: 14px; .composer { width: min(100%, 920px); margin: 0 auto; padding: 16px 18px 10px; display: grid; grid-template-columns: minmax(0, 1fr) 130px; gap: 12px; background: linear-gradient(transparent, var(--base) 20%); } .composer button { height: 48px; align-self: end; } .composer button:disabled { cursor: wait; filter: saturate(.3); } +.media-tools { width: min(100%, 920px); margin: 0 auto; padding: 0 18px 10px; display: flex; flex-wrap: wrap; align-items: center; gap: 8px; color: var(--muted); font: 10px/1 var(--mono); text-transform: uppercase; letter-spacing: .06em; } +.media-tools label { display: inline-flex; align-items: center; gap: 6px; min-height: 30px; padding: 0 8px; border: 1px solid var(--line); color: var(--ink); cursor: pointer; } +.media-tools input { max-width: 150px; font: 10px var(--mono); } +.media-tools span { color: var(--signal); } .conversation-starters { width: min(100%, 920px); margin: -4px auto 0; padding: 0 18px 18px; display: flex; flex-wrap: wrap; align-items: center; gap: 8px; color: var(--muted); font: 10px/1 var(--mono); letter-spacing: .06em; text-transform: uppercase; } .conversation-starters button { min-height: 32px; padding: 0 10px; border: 1px solid var(--line); color: var(--ink); background: #101814; font: 10px/1 var(--mono); } .conversation-starters button:hover, .conversation-starters button:focus-visible { border-color: var(--signal); color: var(--signal); outline: none; } diff --git a/cmd/creator-web/static/assets/chat.js b/cmd/creator-web/static/assets/chat.js index a397cca..b5f6c9e 100644 --- a/cmd/creator-web/static/assets/chat.js +++ b/cmd/creator-web/static/assets/chat.js @@ -132,6 +132,16 @@ async function api(path, options = {}) { return response.json(); } +async function uploadMedia(path, file) { + const body = new FormData(); + body.append("file", file, file.name); + const headers = {}; + if (token) headers.Authorization = `Bearer ${token}`; + const response = await fetch(path, {method: "POST", headers, body}); + if (!response.ok) throw new ChatApiError(response.status, await response.text()); + return response.json(); +} + function expireSession(message = "") { token = ""; conversationID = ""; @@ -320,6 +330,34 @@ chatForm.addEventListener("submit", async (event) => { await sendChatMessage(text); }); +async function processMedia(input, path, label) { + const file = input?.files?.[0]; + if (!file || !mediaStatus) return; + mediaStatus.textContent = `${label} обрабатывается...`; + input.value = ""; + try { + const result = await uploadMedia(path, file); + const text = String(result.text || "").trim(); + if (!text) throw new Error("empty media result"); + mediaStatus.textContent = "Готово"; + await sendChatMessage(`Пользователь передал ${label.toLowerCase()}. Обработай содержание и ответь по смыслу:\n${text}`); + } catch (error) { + if (error instanceof ChatApiError && (error.status === 401 || error.status === 403)) { + expireSession("Сессия устарела. Войдите снова, чтобы продолжить разговор."); + } else { + mediaStatus.textContent = `${label} сейчас недоступно.`; + } + } +} + +const mediaStatus = document.querySelector("#media-status"); +const audioInput = document.querySelector("#audio-input"); +const imageInput = document.querySelector("#image-input"); +const ocrInput = document.querySelector("#ocr-input"); +if (audioInput) audioInput.addEventListener("change", () => processMedia(audioInput, "/api/multimodal/transcribe", "Распознанное аудио")); +if (imageInput) imageInput.addEventListener("change", () => processMedia(imageInput, "/api/multimodal/understand", "Описание изображения")); +if (ocrInput) ocrInput.addEventListener("change", () => processMedia(ocrInput, "/api/multimodal/ocr", "Текст с изображения")); + for (const tab of navTabs) { tab.addEventListener("click", () => activateTab(tab.dataset.tab)); } diff --git a/cmd/creator-web/static/chat.html b/cmd/creator-web/static/chat.html index d1b6c5d..e3b1017 100644 --- a/cmd/creator-web/static/chat.html +++ b/cmd/creator-web/static/chat.html @@ -60,6 +60,12 @@ +
+ + + + +
Можно начать так: diff --git a/contracts/capability/multimodal.go b/contracts/capability/multimodal.go new file mode 100644 index 0000000..c242192 --- /dev/null +++ b/contracts/capability/multimodal.go @@ -0,0 +1,20 @@ +package capability + +// MultimodalOperation identifies one bounded local media skill. It is kept +// separate from general cognition so health, routing and failure evidence do +// not collapse into a generic model response. +type MultimodalOperation string + +const ( + MultimodalTranscribe MultimodalOperation = "TRANSCRIBE_AUDIO" + MultimodalUnderstand MultimodalOperation = "UNDERSTAND_IMAGE" + MultimodalRecognizeOCR MultimodalOperation = "RECOGNIZE_OCR" +) + +type MultimodalResponse struct { + Operation MultimodalOperation `json:"operation"` + Text string `json:"text"` + Provider string `json:"provider,omitempty"` + ObservedAt int64 `json:"observed_at_unix_millis"` + ConfidenceNote string `json:"confidence_note,omitempty"` +} diff --git a/contracts/capability/types.go b/contracts/capability/types.go index 1dc0cff..3e13aba 100644 --- a/contracts/capability/types.go +++ b/contracts/capability/types.go @@ -29,6 +29,9 @@ const ( CapabilityKindCognitiveReasoning CapabilityKind = "COGNITIVE_REASONING" CapabilityKindScheduleReminder CapabilityKind = "SCHEDULE_REMINDER" CapabilityKindVerifyFact CapabilityKind = "VERIFY_FACT" + CapabilityKindSpeechToText CapabilityKind = "SPEECH_TO_TEXT" + CapabilityKindImageUnderstanding CapabilityKind = "IMAGE_UNDERSTANDING" + CapabilityKindOCR CapabilityKind = "OCR" ) // Binding is Capability-owned correctness state. SecretRef is an opaque diff --git a/dev/tasks/DEV-000295-household-scenario-audit.json b/dev/tasks/DEV-000295-household-scenario-audit.json index 992a160..4c0cf26 100644 --- a/dev/tasks/DEV-000295-household-scenario-audit.json +++ b/dev/tasks/DEV-000295-household-scenario-audit.json @@ -12,9 +12,12 @@ "REQ-HOUSEHOLD-004" ], "allowed_change_scope": { - "modules": ["interaction", "capability", "epistemics", "situation", "work", "continuity"], + "modules": ["interaction", "capability", "epistemics", "situation", "work", "continuity", "cognitive-worker", "creator-web", "multimodal-adapters"], "paths": [ "cmd/core-runtime/**", + "cmd/cognitive-worker/**", + "cmd/creator-web/**", + "adapters/multimodal/**", "modules/interaction/**", "modules/capability/**", "modules/epistemics/**", @@ -34,7 +37,7 @@ "security/**", "adapters/models/**", "docker-compose*.yml", - "deploy/**", + "deploy/images/**", "migrations/**", "go.mod", "go.sum" @@ -56,12 +59,12 @@ "contracts.interaction@v1", "contracts.work@v1" ], - "may_change": [] + "may_change": ["multimodal capability kinds and internal speech/image/OCR transport contracts; no canonical persistence contract"] }, - "semantic_owner": "interaction owns meaning classification; capability owns capability health and selection; existing semantic owners retain canonical state", + "semantic_owner": "interaction owns meaning classification; capability owns capability health and selection; cognitive-worker/adapters own provider transport; creator-web owns browser transport; existing semantic owners retain canonical state", "canonical_state": ["existing Interpretation, CapabilityResolution, Situation, Goal/Task and ProspectiveItem only"], "state_transitions": ["request -> meaning -> bounded capability route -> result/clarification/degraded answer -> verification evidence"], - "side_effects_and_privileges": ["Scenario tests are read-only/fake by default; real reminders, websites, messages and code changes remain behind existing Commit/Executor and authority boundaries."], + "side_effects_and_privileges": ["Scenario tests are read-only/fake by default; multimodal requests send user-provided media only to configured local speech/vision services; real reminders, websites, messages and code changes remain behind existing Commit/Executor and authority boundaries."], "persistence_and_migrations": ["No new tables, migrations or events; verify existing durable memory, situation, goal/task and continuity paths."], "failure_retry_restart": ["Each scenario distinguishes unavailable, partial and unknown outcome; retries must not duplicate external effects; restart restores context and commitments from canonical state."], "acceptance_scenarios": [ @@ -88,6 +91,9 @@ "unit", "semantic-routing", "scenario-contract", + "multimodal-adapter", + "multimodal-proxy", + "multimodal-production-smoke", "degraded-mode", "restart-recovery", "architecture",