From 8260e69dfe21190966ee809436d41cddb2498d7f Mon Sep 17 00:00:00 2001 From: Mikhail Date: Tue, 25 Aug 2026 16:12:31 +0300 Subject: [PATCH 1/2] Add replaceable image and video media routes --- adapters/multimodal/client.go | 124 +++++++++++++++++++++++++++++ adapters/multimodal/client_test.go | 54 +++++++++++++ cmd/cognitive-worker/main.go | 14 +++- cmd/cognitive-worker/multimodal.go | 34 +++++++- cmd/creator-web/server.go | 62 +++++++++------ 5 files changed, 257 insertions(+), 31 deletions(-) diff --git a/adapters/multimodal/client.go b/adapters/multimodal/client.go index 51385ab..18dce42 100644 --- a/adapters/multimodal/client.go +++ b/adapters/multimodal/client.go @@ -26,6 +26,22 @@ type VisionClient struct { Client *http.Client } +type ImageGenerationClient struct { + BaseURL string + Client *http.Client +} + +type VideoUnderstandingClient struct { + BaseURL string + Client *http.Client +} + +type GeneratedImage struct { + DataBase64 string + SourceURL string + ContentType string +} + func (c SpeechClient) Transcribe(ctx context.Context, filename, contentType string, data []byte) (string, error) { if len(data) == 0 { return "", errors.New("audio payload is empty") @@ -155,6 +171,100 @@ func (c VisionClient) Describe(ctx context.Context, model, contentType string, d return text, nil } +// Generate uses an OpenAI-compatible image-generation skill. The provider is +// selected outside the core by the capability binding; this adapter contains +// transport only and never decides what the user meant. +func (c ImageGenerationClient) Generate(ctx context.Context, model, prompt, size string) (GeneratedImage, error) { + if strings.TrimSpace(prompt) == "" { + return GeneratedImage{}, errors.New("image prompt is empty") + } + payload := struct { + Model string `json:"model"` + Prompt string `json:"prompt"` + Size string `json:"size,omitempty"` + ResponseFormat string `json:"response_format,omitempty"` + }{Model: model, Prompt: prompt, Size: size, ResponseFormat: "b64_json"} + encoded, err := json.Marshal(payload) + if err != nil { + return GeneratedImage{}, err + } + request, err := http.NewRequestWithContext(ctx, http.MethodPost, strings.TrimRight(c.BaseURL, "/")+"/images/generations", bytes.NewReader(encoded)) + if err != nil { + return GeneratedImage{}, err + } + request.Header.Set("Content-Type", "application/json") + response, err := c.client().Do(request) + if err != nil { + return GeneratedImage{}, err + } + defer response.Body.Close() + if response.StatusCode < 200 || response.StatusCode > 299 { + return GeneratedImage{}, fmt.Errorf("image provider status %d", response.StatusCode) + } + var result struct { + Data []struct { + Base64 string `json:"b64_json"` + URL string `json:"url"` + } `json:"data"` + } + if err := json.NewDecoder(io.LimitReader(response.Body, 32<<20)).Decode(&result); err != nil { + return GeneratedImage{}, err + } + if len(result.Data) == 0 || (strings.TrimSpace(result.Data[0].Base64) == "" && strings.TrimSpace(result.Data[0].URL) == "") { + return GeneratedImage{}, ErrEmptyResult + } + return GeneratedImage{DataBase64: result.Data[0].Base64, SourceURL: result.Data[0].URL, ContentType: "image/png"}, nil +} + +// Describe accepts a video skill endpoint that performs its own bounded frame +// sampling. Frame extraction is provider-owned so the core does not acquire a +// media-processing dependency or hide a policy in orchestration code. +func (c VideoUnderstandingClient) Describe(ctx context.Context, model, filename, contentType string, data []byte, instruction string) (string, error) { + if len(data) == 0 { + return "", errors.New("video payload is empty") + } + body := &bytes.Buffer{} + writer := multipart.NewWriter(body) + header := make(textproto.MIMEHeader) + header.Set("Content-Disposition", fmt.Sprintf(`form-data; name="file"; filename="%s"`, strings.ReplaceAll(filename, `"`, ""))) + header.Set("Content-Type", contentType) + part, err := writer.CreatePart(header) + if err != nil { + return "", err + } + if _, err := part.Write(data); err != nil { + return "", err + } + _ = writer.WriteField("model", model) + _ = writer.WriteField("instruction", instruction) + if err := writer.Close(); err != nil { + return "", err + } + request, err := http.NewRequestWithContext(ctx, http.MethodPost, strings.TrimRight(c.BaseURL, "/")+"/understand", body) + if err != nil { + return "", err + } + request.Header.Set("Content-Type", writer.FormDataContentType()) + response, err := c.client().Do(request) + if err != nil { + return "", err + } + defer response.Body.Close() + if response.StatusCode < 200 || response.StatusCode > 299 { + return "", fmt.Errorf("video provider status %d", response.StatusCode) + } + var result struct { + Text string `json:"text"` + } + if err := json.NewDecoder(io.LimitReader(response.Body, 2<<20)).Decode(&result); err != nil { + return "", err + } + if strings.TrimSpace(result.Text) == "" { + return "", ErrEmptyResult + } + return strings.TrimSpace(result.Text), nil +} + func (c SpeechClient) client() *http.Client { if c.Client != nil { return c.Client @@ -169,6 +279,20 @@ func (c VisionClient) client() *http.Client { return http.DefaultClient } +func (c ImageGenerationClient) client() *http.Client { + if c.Client != nil { + return c.Client + } + return http.DefaultClient +} + +func (c VideoUnderstandingClient) client() *http.Client { + if c.Client != nil { + return c.Client + } + return http.DefaultClient +} + func encodeBase64(data []byte) string { return base64.StdEncoding.EncodeToString(data) } diff --git a/adapters/multimodal/client_test.go b/adapters/multimodal/client_test.go index 7b2d93a..4068320 100644 --- a/adapters/multimodal/client_test.go +++ b/adapters/multimodal/client_test.go @@ -76,3 +76,57 @@ func TestVisionClientUsesReasoningContentWhenProviderOmitsContent(t *testing.T) t.Fatalf("description=%q err=%v", text, err) } } + +func TestImageGenerationClientSendsPromptAndReturnsBase64(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/images/generations" || r.Method != http.MethodPost { + t.Fatalf("unexpected image request: %s %s", r.Method, r.URL.Path) + } + var payload struct { + Model string `json:"model"` + Prompt string `json:"prompt"` + Size string `json:"size"` + } + if err := json.NewDecoder(r.Body).Decode(&payload); err != nil { + t.Fatal(err) + } + if payload.Model != "image-model" || payload.Prompt != "green bird" || payload.Size != "1024x1024" { + t.Fatalf("unexpected image payload: %+v", payload) + } + _ = json.NewEncoder(w).Encode(map[string]any{"data": []any{map[string]string{"b64_json": "aW1hZ2U="}}}) + })) + defer server.Close() + image, err := (ImageGenerationClient{BaseURL: server.URL, Client: server.Client()}).Generate(context.Background(), "image-model", "green bird", "1024x1024") + if err != nil || image.DataBase64 != "aW1hZ2U=" || image.ContentType != "image/png" { + t.Fatalf("image=%+v err=%v", image, err) + } +} + +func TestVideoUnderstandingClientSendsMultipartAndReturnsDescription(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/understand" || r.Method != http.MethodPost { + t.Fatalf("unexpected video request: %s %s", r.Method, r.URL.Path) + } + if err := r.ParseMultipartForm(1 << 20); err != nil { + t.Fatal(err) + } + if r.FormValue("model") != "video-model" || r.FormValue("instruction") != "describe movement" { + t.Fatalf("unexpected video fields: model=%q instruction=%q", r.FormValue("model"), r.FormValue("instruction")) + } + file, _, err := r.FormFile("file") + if err != nil { + t.Fatal(err) + } + defer file.Close() + data, _ := io.ReadAll(file) + if string(data) != "video" { + t.Fatalf("video payload=%q", data) + } + _ = json.NewEncoder(w).Encode(map[string]string{"text": "человек идёт"}) + })) + defer server.Close() + text, err := (VideoUnderstandingClient{BaseURL: server.URL, Client: server.Client()}).Describe(context.Background(), "video-model", "clip.mp4", "video/mp4", []byte("video"), "describe movement") + if err != nil || text != "человек идёт" { + t.Fatalf("description=%q err=%v", text, err) + } +} diff --git a/cmd/cognitive-worker/main.go b/cmd/cognitive-worker/main.go index 02ea091..74f0557 100644 --- a/cmd/cognitive-worker/main.go +++ b/cmd/cognitive-worker/main.go @@ -172,14 +172,20 @@ func main() { mux.HandleFunc("POST /v1/interpretations", gateway.interpretationHandler) mux.HandleFunc("POST /v1/task-evaluations/execute", gateway.taskEvaluationHandler) multimodalGateway := multimodalGateway{ - speech: multimodal.SpeechClient{BaseURL: envOr("LOCAL_SPEECH_PROVIDER_URL", "http://llm-local-speech:8080"), Client: &http.Client{Timeout: 120 * time.Second}}, - vision: multimodal.VisionClient{BaseURL: envOr("LOCAL_VISION_PROVIDER_URL", "http://llm-local-vision:8080/v1"), Client: &http.Client{Timeout: 180 * time.Second}}, - model: envOr("LOCAL_VISION_MODEL", "/models/gemma-4-26B-A4B-it-UD-Q4_K_M.gguf"), - now: time.Now, + speech: multimodal.SpeechClient{BaseURL: envOr("LOCAL_SPEECH_PROVIDER_URL", "http://llm-local-speech:8080"), Client: &http.Client{Timeout: 120 * time.Second}}, + vision: multimodal.VisionClient{BaseURL: envOr("LOCAL_VISION_PROVIDER_URL", "http://llm-local-vision:8080/v1"), Client: &http.Client{Timeout: 180 * time.Second}}, + imageGenerator: multimodal.ImageGenerationClient{BaseURL: os.Getenv("LOCAL_IMAGE_PROVIDER_URL"), Client: &http.Client{Timeout: 300 * time.Second}}, + video: multimodal.VideoUnderstandingClient{BaseURL: os.Getenv("LOCAL_VIDEO_PROVIDER_URL"), Client: &http.Client{Timeout: 300 * time.Second}}, + model: envOr("LOCAL_VISION_MODEL", "/models/gemma-4-26B-A4B-it-UD-Q4_K_M.gguf"), + imageModel: os.Getenv("LOCAL_IMAGE_MODEL"), + videoModel: os.Getenv("LOCAL_VIDEO_MODEL"), + now: time.Now, } mux.HandleFunc("POST /v1/multimodal/transcribe", multimodalGateway.handler(capability.MultimodalTranscribe)) mux.HandleFunc("POST /v1/multimodal/understand", multimodalGateway.handler(capability.MultimodalUnderstand)) mux.HandleFunc("POST /v1/multimodal/ocr", multimodalGateway.handler(capability.MultimodalRecognizeOCR)) + mux.HandleFunc("POST /v1/multimodal/video-understand", multimodalGateway.handler(capability.MultimodalUnderstandVideo)) + mux.HandleFunc("POST /v1/multimodal/generate-image", multimodalGateway.handler(capability.MultimodalGenerateImage)) discovery := discoveryService{resolver: resolver, providers: discoveries, owner: identity.PrincipalID(envOr("COGNITIVE_OWNER_PRINCIPAL_ID", "domovoy-creator-principal")), now: time.Now} mux.HandleFunc("POST /v1/provider-discovery/{provider}", discovery.handler) mux.Handle("POST /v1/provider-discovery/{provider}/qualify", modelQualificationHandler(discoveries, string(envOr("COGNITIVE_OWNER_PRINCIPAL_ID", "domovoy-creator-principal")), httpQualificationObserver{resolver: resolver})) diff --git a/cmd/cognitive-worker/multimodal.go b/cmd/cognitive-worker/multimodal.go index 8feb2e2..aea8baf 100644 --- a/cmd/cognitive-worker/multimodal.go +++ b/cmd/cognitive-worker/multimodal.go @@ -15,10 +15,14 @@ import ( const maxMultimodalUploadBytes = 12 << 20 type multimodalGateway struct { - speech multimodal.SpeechClient - vision multimodal.VisionClient - model string - now func() time.Time + speech multimodal.SpeechClient + vision multimodal.VisionClient + imageGenerator multimodal.ImageGenerationClient + video multimodal.VideoUnderstandingClient + model string + imageModel string + videoModel string + now func() time.Time } func (g multimodalGateway) handler(operation capability.MultimodalOperation) http.HandlerFunc { @@ -28,6 +32,22 @@ func (g multimodalGateway) handler(operation capability.MultimodalOperation) htt http.Error(w, "media upload is too large or invalid", http.StatusRequestEntityTooLarge) return } + if operation == capability.MultimodalGenerateImage { + if strings.TrimSpace(g.imageGenerator.BaseURL) == "" { + http.Error(w, "local image generation capability unavailable", http.StatusBadGateway) + return + } + image, err := g.imageGenerator.Generate(r.Context(), g.imageModel, r.FormValue("prompt"), r.FormValue("size")) + if err != nil || strings.TrimSpace(image.DataBase64) == "" { + logMultimodalFailure(operation, err) + http.Error(w, "local image generation capability unavailable", http.StatusBadGateway) + return + } + result := capability.MultimodalResponse{Operation: operation, Provider: "local-image-generator", ObservedAt: g.now().UnixMilli(), ContentType: image.ContentType, DataBase64: image.DataBase64} + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(result) + return + } file, header, err := r.FormFile("file") if err != nil { http.Error(w, "media file is required", http.StatusBadRequest) @@ -46,6 +66,12 @@ func (g multimodalGateway) handler(operation capability.MultimodalOperation) htt var text string if operation == capability.MultimodalTranscribe { text, err = g.speech.Transcribe(r.Context(), header.Filename, contentType, data) + } else if operation == capability.MultimodalUnderstandVideo { + if strings.TrimSpace(g.video.BaseURL) == "" { + http.Error(w, "local video understanding capability unavailable", http.StatusBadGateway) + return + } + text, err = g.video.Describe(r.Context(), g.videoModel, header.Filename, contentType, data, r.FormValue("instruction")) } else { instruction := "Опиши, что изображено на фотографии, коротко и по существу. Не выдумывай нечитаемые детали." if operation == capability.MultimodalRecognizeOCR { diff --git a/cmd/creator-web/server.go b/cmd/creator-web/server.go index 6435c62..2a48312 100644 --- a/cmd/creator-web/server.go +++ b/cmd/creator-web/server.go @@ -688,17 +688,6 @@ func (g *gateway) handleMultimodal(w http.ResponseWriter, r *http.Request) { http.Error(w, "media upload is too large or invalid", http.StatusRequestEntityTooLarge) return } - file, header, err := r.FormFile("file") - if err != nil { - http.Error(w, "media file is required", http.StatusBadRequest) - return - } - defer file.Close() - data, err := io.ReadAll(io.LimitReader(file, maxMediaBytes+1)) - if err != nil || len(data) == 0 || len(data) > maxMediaBytes { - http.Error(w, "media file is invalid", http.StatusBadRequest) - return - } path := "" switch r.URL.Path { case "/api/multimodal/transcribe": @@ -707,25 +696,52 @@ func (g *gateway) handleMultimodal(w http.ResponseWriter, r *http.Request) { path = "/v1/multimodal/understand" case "/api/multimodal/ocr": path = "/v1/multimodal/ocr" + case "/api/multimodal/video-understand": + path = "/v1/multimodal/video-understand" + case "/api/multimodal/generate-image": + path = "/v1/multimodal/generate-image" default: http.Error(w, "multimodal route not found", http.StatusNotFound) return } body := &bytes.Buffer{} writer := multipart.NewWriter(body) - mediaHeader := make(textproto.MIMEHeader) - mediaHeader.Set("Content-Disposition", fmt.Sprintf(`form-data; name="file"; filename="%s"`, strings.ReplaceAll(header.Filename, `"`, ""))) - mediaType := header.Header.Get("Content-Type") - if mediaType == "" { - mediaType = "application/octet-stream" + if r.URL.Path == "/api/multimodal/generate-image" { + _ = writer.WriteField("prompt", r.FormValue("prompt")) + _ = writer.WriteField("size", r.FormValue("size")) + } else { + if r.URL.Path == "/api/multimodal/video-understand" { + _ = writer.WriteField("instruction", r.FormValue("instruction")) + } + file, header, err := r.FormFile("file") + if err != nil { + http.Error(w, "media file is required", http.StatusBadRequest) + return + } + defer file.Close() + data, err := io.ReadAll(io.LimitReader(file, maxMediaBytes+1)) + if err != nil || len(data) == 0 || len(data) > maxMediaBytes { + http.Error(w, "media file is invalid", http.StatusBadRequest) + return + } + mediaHeader := make(textproto.MIMEHeader) + mediaHeader.Set("Content-Disposition", fmt.Sprintf(`form-data; name="file"; filename="%s"`, strings.ReplaceAll(header.Filename, `"`, ""))) + mediaType := header.Header.Get("Content-Type") + if mediaType == "" { + mediaType = "application/octet-stream" + } + mediaHeader.Set("Content-Type", mediaType) + part, err := writer.CreatePart(mediaHeader) + if err != nil { + http.Error(w, "media upload unavailable", http.StatusServiceUnavailable) + return + } + if _, err := part.Write(data); err != nil { + http.Error(w, "media upload unavailable", http.StatusServiceUnavailable) + return + } } - mediaHeader.Set("Content-Type", mediaType) - part, err := writer.CreatePart(mediaHeader) - if err != nil { - http.Error(w, "media upload unavailable", http.StatusServiceUnavailable) - return - } - if _, err := part.Write(data); err != nil || writer.Close() != nil { + if err := writer.Close(); err != nil { http.Error(w, "media upload unavailable", http.StatusServiceUnavailable) return } -- 2.54.0 From cdcbd50fe70eef614d32d902995455d74ce9ceab Mon Sep 17 00:00:00 2001 From: Mikhail Date: Tue, 25 Aug 2026 16:14:28 +0300 Subject: [PATCH 2/2] Test degraded media capability routing --- cmd/cognitive-worker/multimodal_test.go | 26 +++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/cmd/cognitive-worker/multimodal_test.go b/cmd/cognitive-worker/multimodal_test.go index bc0ae07..a1207fd 100644 --- a/cmd/cognitive-worker/multimodal_test.go +++ b/cmd/cognitive-worker/multimodal_test.go @@ -31,6 +31,32 @@ func TestMultimodalTranscriptionHandlerReturnsLocalResult(t *testing.T) { } } +func TestMultimodalImageGenerationFailsClosedWhenSkillIsUnbound(t *testing.T) { + server := multimodalGateway{now: time.Now} + body := &bytes.Buffer{} + writer := multipart.NewWriter(body) + if err := writer.WriteField("prompt", "green bird"); err != nil || writer.Close() != nil { + t.Fatal("failed to build generation request") + } + request := httptest.NewRequest(http.MethodPost, "/v1/multimodal/generate-image", body) + request.Header.Set("Content-Type", writer.FormDataContentType()) + response := httptest.NewRecorder() + server.handler(capability.MultimodalGenerateImage)(response, request) + if response.Code != http.StatusBadGateway || !bytes.Contains(response.Body.Bytes(), []byte("unavailable")) { + t.Fatalf("status=%d body=%s", response.Code, response.Body.String()) + } +} + +func TestMultimodalVideoUnderstandingFailsClosedWhenSkillIsUnbound(t *testing.T) { + server := multimodalGateway{now: time.Now} + request := multipartRequest(t, "/v1/multimodal/video-understand", "clip.mp4", []byte("video")) + response := httptest.NewRecorder() + server.handler(capability.MultimodalUnderstandVideo)(response, request) + if response.Code != http.StatusBadGateway || !bytes.Contains(response.Body.Bytes(), []byte("unavailable")) { + t.Fatalf("status=%d body=%s", response.Code, response.Body.String()) + } +} + func multipartRequest(t *testing.T, path, filename string, data []byte) *http.Request { t.Helper() body := &bytes.Buffer{} -- 2.54.0