adapters/multimodal/client.go

@@ -26,6 +26,22 @@ type VisionClient struct {
 	Client  *http.Client
 }
 
+type ImageGenerationClient struct {
+	BaseURL string
+	Client  *http.Client
+}
+
+type VideoUnderstandingClient struct {
+	BaseURL string
+	Client  *http.Client
+}
+
+type GeneratedImage struct {
+	DataBase64  string
+	SourceURL   string
+	ContentType string
+}
+
 func (c SpeechClient) Transcribe(ctx context.Context, filename, contentType string, data []byte) (string, error) {
 	if len(data) == 0 {
 		return "", errors.New("audio payload is empty")
@@ -155,6 +171,100 @@ func (c VisionClient) Describe(ctx context.Context, model, contentType string, d
 	return text, nil
 }
 
+// Generate uses an OpenAI-compatible image-generation skill. The provider is
+// selected outside the core by the capability binding; this adapter contains
+// transport only and never decides what the user meant.
+func (c ImageGenerationClient) Generate(ctx context.Context, model, prompt, size string) (GeneratedImage, error) {
+	if strings.TrimSpace(prompt) == "" {
+		return GeneratedImage{}, errors.New("image prompt is empty")
+	}
+	payload := struct {
+		Model          string `json:"model"`
+		Prompt         string `json:"prompt"`
+		Size           string `json:"size,omitempty"`
+		ResponseFormat string `json:"response_format,omitempty"`
+	}{Model: model, Prompt: prompt, Size: size, ResponseFormat: "b64_json"}
+	encoded, err := json.Marshal(payload)
+	if err != nil {
+		return GeneratedImage{}, err
+	}
+	request, err := http.NewRequestWithContext(ctx, http.MethodPost, strings.TrimRight(c.BaseURL, "/")+"/images/generations", bytes.NewReader(encoded))
+	if err != nil {
+		return GeneratedImage{}, err
+	}
+	request.Header.Set("Content-Type", "application/json")
+	response, err := c.client().Do(request)
+	if err != nil {
+		return GeneratedImage{}, err
+	}
+	defer response.Body.Close()
+	if response.StatusCode < 200 || response.StatusCode > 299 {
+		return GeneratedImage{}, fmt.Errorf("image provider status %d", response.StatusCode)
+	}
+	var result struct {
+		Data []struct {
+			Base64 string `json:"b64_json"`
+			URL    string `json:"url"`
+		} `json:"data"`
+	}
+	if err := json.NewDecoder(io.LimitReader(response.Body, 32<<20)).Decode(&result); err != nil {
+		return GeneratedImage{}, err
+	}
+	if len(result.Data) == 0 || (strings.TrimSpace(result.Data[0].Base64) == "" && strings.TrimSpace(result.Data[0].URL) == "") {
+		return GeneratedImage{}, ErrEmptyResult
+	}
+	return GeneratedImage{DataBase64: result.Data[0].Base64, SourceURL: result.Data[0].URL, ContentType: "image/png"}, nil
+}
+
+// Describe accepts a video skill endpoint that performs its own bounded frame
+// sampling. Frame extraction is provider-owned so the core does not acquire a
+// media-processing dependency or hide a policy in orchestration code.
+func (c VideoUnderstandingClient) Describe(ctx context.Context, model, filename, contentType string, data []byte, instruction string) (string, error) {
+	if len(data) == 0 {
+		return "", errors.New("video payload is empty")
+	}
+	body := &bytes.Buffer{}
+	writer := multipart.NewWriter(body)
+	header := make(textproto.MIMEHeader)
+	header.Set("Content-Disposition", fmt.Sprintf(`form-data; name="file"; filename="%s"`, strings.ReplaceAll(filename, `"`, "")))
+	header.Set("Content-Type", contentType)
+	part, err := writer.CreatePart(header)
+	if err != nil {
+		return "", err
+	}
+	if _, err := part.Write(data); err != nil {
+		return "", err
+	}
+	_ = writer.WriteField("model", model)
+	_ = writer.WriteField("instruction", instruction)
+	if err := writer.Close(); err != nil {
+		return "", err
+	}
+	request, err := http.NewRequestWithContext(ctx, http.MethodPost, strings.TrimRight(c.BaseURL, "/")+"/understand", body)
+	if err != nil {
+		return "", err
+	}
+	request.Header.Set("Content-Type", writer.FormDataContentType())
+	response, err := c.client().Do(request)
+	if err != nil {
+		return "", err
+	}
+	defer response.Body.Close()
+	if response.StatusCode < 200 || response.StatusCode > 299 {
+		return "", fmt.Errorf("video provider status %d", response.StatusCode)
+	}
+	var result struct {
+		Text string `json:"text"`
+	}
+	if err := json.NewDecoder(io.LimitReader(response.Body, 2<<20)).Decode(&result); err != nil {
+		return "", err
+	}
+	if strings.TrimSpace(result.Text) == "" {
+		return "", ErrEmptyResult
+	}
+	return strings.TrimSpace(result.Text), nil
+}
+
 func (c SpeechClient) client() *http.Client {
 	if c.Client != nil {
 		return c.Client
@@ -169,6 +279,20 @@ func (c VisionClient) client() *http.Client {
 	return http.DefaultClient
 }
 
+func (c ImageGenerationClient) client() *http.Client {
+	if c.Client != nil {
+		return c.Client
+	}
+	return http.DefaultClient
+}
+
+func (c VideoUnderstandingClient) client() *http.Client {
+	if c.Client != nil {
+		return c.Client
+	}
+	return http.DefaultClient
+}
+
 func encodeBase64(data []byte) string {
 	return base64.StdEncoding.EncodeToString(data)
 }

adapters/multimodal/client_test.go

@@ -76,3 +76,57 @@ func TestVisionClientUsesReasoningContentWhenProviderOmitsContent(t *testing.T)
 		t.Fatalf("description=%q err=%v", text, err)
 	}
 }
+
+func TestImageGenerationClientSendsPromptAndReturnsBase64(t *testing.T) {
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Path != "/images/generations" || r.Method != http.MethodPost {
+			t.Fatalf("unexpected image request: %s %s", r.Method, r.URL.Path)
+		}
+		var payload struct {
+			Model  string `json:"model"`
+			Prompt string `json:"prompt"`
+			Size   string `json:"size"`
+		}
+		if err := json.NewDecoder(r.Body).Decode(&payload); err != nil {
+			t.Fatal(err)
+		}
+		if payload.Model != "image-model" || payload.Prompt != "green bird" || payload.Size != "1024x1024" {
+			t.Fatalf("unexpected image payload: %+v", payload)
+		}
+		_ = json.NewEncoder(w).Encode(map[string]any{"data": []any{map[string]string{"b64_json": "aW1hZ2U="}}})
+	}))
+	defer server.Close()
+	image, err := (ImageGenerationClient{BaseURL: server.URL, Client: server.Client()}).Generate(context.Background(), "image-model", "green bird", "1024x1024")
+	if err != nil || image.DataBase64 != "aW1hZ2U=" || image.ContentType != "image/png" {
+		t.Fatalf("image=%+v err=%v", image, err)
+	}
+}
+
+func TestVideoUnderstandingClientSendsMultipartAndReturnsDescription(t *testing.T) {
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Path != "/understand" || r.Method != http.MethodPost {
+			t.Fatalf("unexpected video request: %s %s", r.Method, r.URL.Path)
+		}
+		if err := r.ParseMultipartForm(1 << 20); err != nil {
+			t.Fatal(err)
+		}
+		if r.FormValue("model") != "video-model" || r.FormValue("instruction") != "describe movement" {
+			t.Fatalf("unexpected video fields: model=%q instruction=%q", r.FormValue("model"), r.FormValue("instruction"))
+		}
+		file, _, err := r.FormFile("file")
+		if err != nil {
+			t.Fatal(err)
+		}
+		defer file.Close()
+		data, _ := io.ReadAll(file)
+		if string(data) != "video" {
+			t.Fatalf("video payload=%q", data)
+		}
+		_ = json.NewEncoder(w).Encode(map[string]string{"text": "человек идёт"})
+	}))
+	defer server.Close()
+	text, err := (VideoUnderstandingClient{BaseURL: server.URL, Client: server.Client()}).Describe(context.Background(), "video-model", "clip.mp4", "video/mp4", []byte("video"), "describe movement")
+	if err != nil || text != "человек идёт" {
+		t.Fatalf("description=%q err=%v", text, err)
+	}
+}

cmd/cognitive-worker/main.go

@@ -172,14 +172,20 @@ func main() {
 	mux.HandleFunc("POST /v1/interpretations", gateway.interpretationHandler)
 	mux.HandleFunc("POST /v1/task-evaluations/execute", gateway.taskEvaluationHandler)
 	multimodalGateway := multimodalGateway{
-		speech: multimodal.SpeechClient{BaseURL: envOr("LOCAL_SPEECH_PROVIDER_URL", "http://llm-local-speech:8080"), Client: &http.Client{Timeout: 120 * time.Second}},
-		vision: multimodal.VisionClient{BaseURL: envOr("LOCAL_VISION_PROVIDER_URL", "http://llm-local-vision:8080/v1"), Client: &http.Client{Timeout: 180 * time.Second}},
-		model:  envOr("LOCAL_VISION_MODEL", "/models/gemma-4-26B-A4B-it-UD-Q4_K_M.gguf"),
-		now:    time.Now,
+		speech:         multimodal.SpeechClient{BaseURL: envOr("LOCAL_SPEECH_PROVIDER_URL", "http://llm-local-speech:8080"), Client: &http.Client{Timeout: 120 * time.Second}},
+		vision:         multimodal.VisionClient{BaseURL: envOr("LOCAL_VISION_PROVIDER_URL", "http://llm-local-vision:8080/v1"), Client: &http.Client{Timeout: 180 * time.Second}},
+		imageGenerator: multimodal.ImageGenerationClient{BaseURL: os.Getenv("LOCAL_IMAGE_PROVIDER_URL"), Client: &http.Client{Timeout: 300 * time.Second}},
+		video:          multimodal.VideoUnderstandingClient{BaseURL: os.Getenv("LOCAL_VIDEO_PROVIDER_URL"), Client: &http.Client{Timeout: 300 * time.Second}},
+		model:          envOr("LOCAL_VISION_MODEL", "/models/gemma-4-26B-A4B-it-UD-Q4_K_M.gguf"),
+		imageModel:     os.Getenv("LOCAL_IMAGE_MODEL"),
+		videoModel:     os.Getenv("LOCAL_VIDEO_MODEL"),
+		now:            time.Now,
 	}
 	mux.HandleFunc("POST /v1/multimodal/transcribe", multimodalGateway.handler(capability.MultimodalTranscribe))
 	mux.HandleFunc("POST /v1/multimodal/understand", multimodalGateway.handler(capability.MultimodalUnderstand))
 	mux.HandleFunc("POST /v1/multimodal/ocr", multimodalGateway.handler(capability.MultimodalRecognizeOCR))
+	mux.HandleFunc("POST /v1/multimodal/video-understand", multimodalGateway.handler(capability.MultimodalUnderstandVideo))
+	mux.HandleFunc("POST /v1/multimodal/generate-image", multimodalGateway.handler(capability.MultimodalGenerateImage))
 	discovery := discoveryService{resolver: resolver, providers: discoveries, owner: identity.PrincipalID(envOr("COGNITIVE_OWNER_PRINCIPAL_ID", "domovoy-creator-principal")), now: time.Now}
 	mux.HandleFunc("POST /v1/provider-discovery/{provider}", discovery.handler)
 	mux.Handle("POST /v1/provider-discovery/{provider}/qualify", modelQualificationHandler(discoveries, string(envOr("COGNITIVE_OWNER_PRINCIPAL_ID", "domovoy-creator-principal")), httpQualificationObserver{resolver: resolver}))

cmd/cognitive-worker/multimodal.go

@@ -15,10 +15,14 @@ import (
 const maxMultimodalUploadBytes = 12 << 20
 
 type multimodalGateway struct {
-	speech multimodal.SpeechClient
-	vision multimodal.VisionClient
-	model  string
-	now    func() time.Time
+	speech         multimodal.SpeechClient
+	vision         multimodal.VisionClient
+	imageGenerator multimodal.ImageGenerationClient
+	video          multimodal.VideoUnderstandingClient
+	model          string
+	imageModel     string
+	videoModel     string
+	now            func() time.Time
 }
 
 func (g multimodalGateway) handler(operation capability.MultimodalOperation) http.HandlerFunc {
@@ -28,6 +32,22 @@ func (g multimodalGateway) handler(operation capability.MultimodalOperation) htt
 			http.Error(w, "media upload is too large or invalid", http.StatusRequestEntityTooLarge)
 			return
 		}
+		if operation == capability.MultimodalGenerateImage {
+			if strings.TrimSpace(g.imageGenerator.BaseURL) == "" {
+				http.Error(w, "local image generation capability unavailable", http.StatusBadGateway)
+				return
+			}
+			image, err := g.imageGenerator.Generate(r.Context(), g.imageModel, r.FormValue("prompt"), r.FormValue("size"))
+			if err != nil || strings.TrimSpace(image.DataBase64) == "" {
+				logMultimodalFailure(operation, err)
+				http.Error(w, "local image generation capability unavailable", http.StatusBadGateway)
+				return
+			}
+			result := capability.MultimodalResponse{Operation: operation, Provider: "local-image-generator", ObservedAt: g.now().UnixMilli(), ContentType: image.ContentType, DataBase64: image.DataBase64}
+			w.Header().Set("Content-Type", "application/json")
+			_ = json.NewEncoder(w).Encode(result)
+			return
+		}
 		file, header, err := r.FormFile("file")
 		if err != nil {
 			http.Error(w, "media file is required", http.StatusBadRequest)
@@ -46,6 +66,12 @@ func (g multimodalGateway) handler(operation capability.MultimodalOperation) htt
 		var text string
 		if operation == capability.MultimodalTranscribe {
 			text, err = g.speech.Transcribe(r.Context(), header.Filename, contentType, data)
+		} else if operation == capability.MultimodalUnderstandVideo {
+			if strings.TrimSpace(g.video.BaseURL) == "" {
+				http.Error(w, "local video understanding capability unavailable", http.StatusBadGateway)
+				return
+			}
+			text, err = g.video.Describe(r.Context(), g.videoModel, header.Filename, contentType, data, r.FormValue("instruction"))
 		} else {
 			instruction := "Опиши, что изображено на фотографии, коротко и по существу. Не выдумывай нечитаемые детали."
 			if operation == capability.MultimodalRecognizeOCR {

cmd/cognitive-worker/multimodal_test.go

@@ -31,6 +31,32 @@ func TestMultimodalTranscriptionHandlerReturnsLocalResult(t *testing.T) {
 	}
 }
 
+func TestMultimodalImageGenerationFailsClosedWhenSkillIsUnbound(t *testing.T) {
+	server := multimodalGateway{now: time.Now}
+	body := &bytes.Buffer{}
+	writer := multipart.NewWriter(body)
+	if err := writer.WriteField("prompt", "green bird"); err != nil || writer.Close() != nil {
+		t.Fatal("failed to build generation request")
+	}
+	request := httptest.NewRequest(http.MethodPost, "/v1/multimodal/generate-image", body)
+	request.Header.Set("Content-Type", writer.FormDataContentType())
+	response := httptest.NewRecorder()
+	server.handler(capability.MultimodalGenerateImage)(response, request)
+	if response.Code != http.StatusBadGateway || !bytes.Contains(response.Body.Bytes(), []byte("unavailable")) {
+		t.Fatalf("status=%d body=%s", response.Code, response.Body.String())
+	}
+}
+
+func TestMultimodalVideoUnderstandingFailsClosedWhenSkillIsUnbound(t *testing.T) {
+	server := multimodalGateway{now: time.Now}
+	request := multipartRequest(t, "/v1/multimodal/video-understand", "clip.mp4", []byte("video"))
+	response := httptest.NewRecorder()
+	server.handler(capability.MultimodalUnderstandVideo)(response, request)
+	if response.Code != http.StatusBadGateway || !bytes.Contains(response.Body.Bytes(), []byte("unavailable")) {
+		t.Fatalf("status=%d body=%s", response.Code, response.Body.String())
+	}
+}
+
 func multipartRequest(t *testing.T, path, filename string, data []byte) *http.Request {
 	t.Helper()
 	body := &bytes.Buffer{}

cmd/creator-web/server.go

@@ -688,17 +688,6 @@ func (g *gateway) handleMultimodal(w http.ResponseWriter, r *http.Request) {
 		http.Error(w, "media upload is too large or invalid", http.StatusRequestEntityTooLarge)
 		return
 	}
-	file, header, err := r.FormFile("file")
-	if err != nil {
-		http.Error(w, "media file is required", http.StatusBadRequest)
-		return
-	}
-	defer file.Close()
-	data, err := io.ReadAll(io.LimitReader(file, maxMediaBytes+1))
-	if err != nil || len(data) == 0 || len(data) > maxMediaBytes {
-		http.Error(w, "media file is invalid", http.StatusBadRequest)
-		return
-	}
 	path := ""
 	switch r.URL.Path {
 	case "/api/multimodal/transcribe":
@@ -707,25 +696,52 @@ func (g *gateway) handleMultimodal(w http.ResponseWriter, r *http.Request) {
 		path = "/v1/multimodal/understand"
 	case "/api/multimodal/ocr":
 		path = "/v1/multimodal/ocr"
+	case "/api/multimodal/video-understand":
+		path = "/v1/multimodal/video-understand"
+	case "/api/multimodal/generate-image":
+		path = "/v1/multimodal/generate-image"
 	default:
 		http.Error(w, "multimodal route not found", http.StatusNotFound)
 		return
 	}
 	body := &bytes.Buffer{}
 	writer := multipart.NewWriter(body)
-	mediaHeader := make(textproto.MIMEHeader)
-	mediaHeader.Set("Content-Disposition", fmt.Sprintf(`form-data; name="file"; filename="%s"`, strings.ReplaceAll(header.Filename, `"`, "")))
-	mediaType := header.Header.Get("Content-Type")
-	if mediaType == "" {
-		mediaType = "application/octet-stream"
+	if r.URL.Path == "/api/multimodal/generate-image" {
+		_ = writer.WriteField("prompt", r.FormValue("prompt"))
+		_ = writer.WriteField("size", r.FormValue("size"))
+	} else {
+		if r.URL.Path == "/api/multimodal/video-understand" {
+			_ = writer.WriteField("instruction", r.FormValue("instruction"))
+		}
+		file, header, err := r.FormFile("file")
+		if err != nil {
+			http.Error(w, "media file is required", http.StatusBadRequest)
+			return
+		}
+		defer file.Close()
+		data, err := io.ReadAll(io.LimitReader(file, maxMediaBytes+1))
+		if err != nil || len(data) == 0 || len(data) > maxMediaBytes {
+			http.Error(w, "media file is invalid", http.StatusBadRequest)
+			return
+		}
+		mediaHeader := make(textproto.MIMEHeader)
+		mediaHeader.Set("Content-Disposition", fmt.Sprintf(`form-data; name="file"; filename="%s"`, strings.ReplaceAll(header.Filename, `"`, "")))
+		mediaType := header.Header.Get("Content-Type")
+		if mediaType == "" {
+			mediaType = "application/octet-stream"
+		}
+		mediaHeader.Set("Content-Type", mediaType)
+		part, err := writer.CreatePart(mediaHeader)
+		if err != nil {
+			http.Error(w, "media upload unavailable", http.StatusServiceUnavailable)
+			return
+		}
+		if _, err := part.Write(data); err != nil {
+			http.Error(w, "media upload unavailable", http.StatusServiceUnavailable)
+			return
+		}
 	}
-	mediaHeader.Set("Content-Type", mediaType)
-	part, err := writer.CreatePart(mediaHeader)
-	if err != nil {
-		http.Error(w, "media upload unavailable", http.StatusServiceUnavailable)
-		return
-	}
-	if _, err := part.Write(data); err != nil || writer.Close() != nil {
+	if err := writer.Close(); err != nil {
 		http.Error(w, "media upload unavailable", http.StatusServiceUnavailable)
 		return
 	}