diff --git a/cmd/core-runtime/chat_orchestrator.go b/cmd/core-runtime/chat_orchestrator.go index a2e45ea..8ee76ce 100644 --- a/cmd/core-runtime/chat_orchestrator.go +++ b/cmd/core-runtime/chat_orchestrator.go @@ -12,6 +12,7 @@ type chatRouteKind string const ( chatRouteResearch chatRouteKind = "RESEARCH" chatRouteConversation chatRouteKind = "CONVERSATION" + chatRouteMedia chatRouteKind = "MEDIA" chatRouteAction chatRouteKind = "ACTION" chatRouteAgent chatRouteKind = "AGENT" ) @@ -42,6 +43,8 @@ func buildChatRoutePlan(result interactiondomain.SemanticResult) []chatRoute { switch capability.CapabilityKind(hint) { case capability.CapabilityKindWebResearch: add(chatRouteResearch, capability.CapabilityKindWebResearch, "truth-research-v1", 10) + case capability.CapabilityKindImageGeneration: + add(chatRouteMedia, capability.CapabilityKindImageGeneration, "local-image-generation-v1", 5) case capability.CapabilityKindCognitiveReasoning, capability.CapabilityKindVerifyFact: add(chatRouteConversation, capability.CapabilityKind(hint), "local-reasoning-v1", 20) case capability.CapabilityKindScheduleReminder: diff --git a/cmd/core-runtime/cognitive_interpretation.go b/cmd/core-runtime/cognitive_interpretation.go index 56b807b..beed5a0 100644 --- a/cmd/core-runtime/cognitive_interpretation.go +++ b/cmd/core-runtime/cognitive_interpretation.go @@ -6,6 +6,8 @@ import ( "encoding/json" "fmt" "hash/fnv" + "io" + "mime/multipart" "net/http" "net/url" "strings" @@ -23,6 +25,7 @@ import ( type cognitiveInterpretationClient struct { baseURL *url.URL client *http.Client + media *http.Client } // ReviewResponse asks the cognitive gateway to turn a locally assembled draft @@ -83,7 +86,7 @@ func newCognitiveInterpretationClient(rawURL string, client *http.Client) (cogni if client == nil { client = http.DefaultClient } - return cognitiveInterpretationClient{baseURL: base, client: client}, nil + return cognitiveInterpretationClient{baseURL: base, client: client, media: &http.Client{Timeout: 15 * time.Minute}}, nil } func (c cognitiveInterpretationClient) Interpret(ctx context.Context, request cognitive.GatewayInterpretationRequest) (cognitive.GatewayInterpretationResponse, error) { @@ -113,6 +116,49 @@ func (c cognitiveInterpretationClient) Interpret(ctx context.Context, request co return result, nil } +func (c cognitiveInterpretationClient) GenerateImage(ctx context.Context, prompt, size string) (capability.MultimodalResponse, error) { + var body bytes.Buffer + writer := multipart.NewWriter(&body) + if err := writer.WriteField("prompt", prompt); err != nil { + return capability.MultimodalResponse{}, err + } + if strings.TrimSpace(size) != "" { + if err := writer.WriteField("size", size); err != nil { + return capability.MultimodalResponse{}, err + } + } + if err := writer.Close(); err != nil { + return capability.MultimodalResponse{}, err + } + target := *c.baseURL + target.Path = "/v1/multimodal/generate-image" + request, err := http.NewRequestWithContext(ctx, http.MethodPost, target.String(), &body) + if err != nil { + return capability.MultimodalResponse{}, err + } + request.Header.Set("Content-Type", writer.FormDataContentType()) + client := c.media + if client == nil { + client = c.client + } + response, err := client.Do(request) + if err != nil { + return capability.MultimodalResponse{}, err + } + defer response.Body.Close() + if response.StatusCode != http.StatusOK { + return capability.MultimodalResponse{}, fmt.Errorf("image generation status %d", response.StatusCode) + } + var result capability.MultimodalResponse + if err := json.NewDecoder(io.LimitReader(response.Body, 32<<20)).Decode(&result); err != nil { + return capability.MultimodalResponse{}, err + } + if strings.TrimSpace(result.DataBase64) == "" { + return capability.MultimodalResponse{}, fmt.Errorf("image generation returned empty payload") + } + return result, nil +} + func shouldRequestBroaderInterpretation(result interactiondomain.SemanticResult) bool { return shouldRequestSemanticInterpretation(result) } diff --git a/cmd/core-runtime/cognitive_interpretation_test.go b/cmd/core-runtime/cognitive_interpretation_test.go index 1abfd5f..169756e 100644 --- a/cmd/core-runtime/cognitive_interpretation_test.go +++ b/cmd/core-runtime/cognitive_interpretation_test.go @@ -5,9 +5,11 @@ import ( "encoding/json" "net/http" "net/http/httptest" + "strings" "testing" "time" + "git.cin.su/m/Assistant/contracts/capability" "git.cin.su/m/Assistant/contracts/cognitive" "git.cin.su/m/Assistant/contracts/identity" "git.cin.su/m/Assistant/contracts/interaction" @@ -40,6 +42,30 @@ func TestReviewResponseUsesCognitiveGatewayAndReturnsOnlyDraftAnswer(t *testing. } } +func TestGenerateImageUsesDedicatedCognitiveMediaRoute(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost || r.URL.Path != "/v1/multimodal/generate-image" { + t.Fatalf("unexpected image route: %s %s", r.Method, r.URL.Path) + } + if err := r.ParseMultipartForm(1 << 20); err != nil { + t.Fatalf("parse image request: %v", err) + } + if !strings.Contains(r.FormValue("prompt"), "собаку") { + t.Fatalf("prompt was not preserved: %q", r.FormValue("prompt")) + } + _ = json.NewEncoder(w).Encode(capability.MultimodalResponse{Operation: capability.MultimodalGenerateImage, ContentType: "image/png", DataBase64: "cG5n"}) + })) + defer server.Close() + client, err := newCognitiveInterpretationClient(server.URL, server.Client()) + if err != nil { + t.Fatal(err) + } + image, err := client.GenerateImage(context.Background(), "нарисуй собаку", "") + if err != nil || image.DataBase64 != "cG5n" || image.ContentType != "image/png" { + t.Fatalf("image=%+v err=%v", image, err) + } +} + func TestReviewPreservesResearchEvidence(t *testing.T) { draft := "Нашёл вариант. Источник: Aviasales (https://www.aviasales.ru/search)" if reviewPreservesResearchEvidence("Нашёл вариант на Авиасейлс.", draft) { diff --git a/cmd/core-runtime/household_scenario_test.go b/cmd/core-runtime/household_scenario_test.go index 6b839a1..036a20a 100644 --- a/cmd/core-runtime/household_scenario_test.go +++ b/cmd/core-runtime/household_scenario_test.go @@ -68,6 +68,10 @@ func TestHouseholdBearDrawingUsesCreativeLocalRoute(t *testing.T) { if decomposition.Intent != interactiondomain.TaskIntentCreate || decomposition.Object != "IMAGE" || decomposition.Target != "бурого медведя" || decomposition.NeedsResearch { t.Fatalf("creative request was not split into local execution atoms: %+v", decomposition) } + routes := buildChatRoutePlan(interactiondomain.SemanticResult{NeedsCognition: candidate.NeedsCognition, CapabilityHints: candidate.CapabilityHints}) + if !hasChatRoute(routes, chatRouteMedia) { + t.Fatalf("creative image request did not select media route: candidate=%+v routes=%+v", candidate, routes) + } for _, hint := range candidate.CapabilityHints { if capability.CapabilityKind(hint) == capability.CapabilityKindWebResearch { t.Fatalf("bear drawing selected web research: %+v", candidate) diff --git a/cmd/core-runtime/main.go b/cmd/core-runtime/main.go index 3c918d0..3bf0976 100644 --- a/cmd/core-runtime/main.go +++ b/cmd/core-runtime/main.go @@ -819,6 +819,23 @@ func main() { var delegationID work.DelegationID var delegationState work.DelegationState var reminder *continuity.ProspectiveItem + var generatedImage capability.MultimodalResponse + mediaHandled := false + if hasChatRoute(routePlan, chatRouteMedia) && semantic.Interpretation.Clarification != interaction.ClarificationRequired { + imageCtx, cancelImage := context.WithTimeout(r.Context(), 15*time.Minute) + generatedImage, err = cognitiveClient.GenerateImage(imageCtx, effectiveTaskMessage, "") + cancelImage() + if err == nil && strings.TrimSpace(generatedImage.DataBase64) != "" { + planningReply = "Готово." + semantic.Reply = planningReply + needsCognition = false + mediaHandled = true + } else { + planningReply = "Сейчас не удалось создать изображение." + semantic.Reply = planningReply + needsCognition = false + } + } if helpPath.Kind == work.HelpPathDirectCapability && len(helpPath.SuggestedCapabilityKinds) > 0 { capabilityKind = helpPath.SuggestedCapabilityKinds[0] started := time.Now() @@ -903,7 +920,7 @@ func main() { ConfidenceMillis: semantic.Interpretation.ConfidenceMillis, }) cancelReview() - stableSemanticReply := identityReply != "" || creativeReply != "" || (creativeFallbackReply != "" && !cognitiveAvailable) || semantic.InformationRequestKind != "" || locationReply != "" || deterministicSpeechAct == interaction.SpeechActConversational || deterministicSpeechAct == interaction.SpeechActConfirmation || deterministicSpeechAct == interaction.SpeechActRejection + stableSemanticReply := mediaHandled || identityReply != "" || creativeReply != "" || (creativeFallbackReply != "" && !cognitiveAvailable) || semantic.InformationRequestKind != "" || locationReply != "" || deterministicSpeechAct == interaction.SpeechActConversational || deterministicSpeechAct == interaction.SpeechActConfirmation || deterministicSpeechAct == interaction.SpeechActRejection if reviewErr != nil { log.Printf("cognitive response review unavailable: %v", reviewErr) } else if !stableSemanticReply && (!researchBacked || reviewPreservesResearchEvidence(reviewedReply, planningReply)) { @@ -912,7 +929,7 @@ func main() { log.Printf("cognitive response review proposed stable semantic rewrite; preserving semantic reply") } } - if identityReply == "" && creativeReply == "" && creativeFallbackReply == "" { + if !mediaHandled && identityReply == "" && creativeReply == "" && creativeFallbackReply == "" { stableAct := interactiondomain.ClassifySpeechAct(utterance.Content, utterance.CorrectsUtteranceID) travelClarification := "" if !creativeRequest { @@ -967,6 +984,8 @@ func main() { DelegationID: string(delegationID), DelegationState: string(delegationState), Reminder: reminder, + GeneratedImageContentType: generatedImage.ContentType, + GeneratedImageBase64: generatedImage.DataBase64, } w.Header().Set("Content-Type", "application/json") w.Header().Set("Cache-Control", "no-store") diff --git a/cmd/creator-web/main.go b/cmd/creator-web/main.go index 5cc14a6..48529f1 100644 --- a/cmd/creator-web/main.go +++ b/cmd/creator-web/main.go @@ -20,7 +20,7 @@ func main() { // A cold local model may need more than one short HTTP round-trip to // load on CPU. Keep the browser gateway aligned with the cognition // deadline instead of turning that wait into a false unavailable reply. - &http.Client{Timeout: 90 * time.Second}, + &http.Client{Timeout: 15 * time.Minute}, envOr("COGNITIVE_RUNTIME_URL", "http://cognitive-worker:8083"), envOr("AGENT_WORKER_URL", "http://agent-worker:8090"), ) diff --git a/cmd/creator-web/server.go b/cmd/creator-web/server.go index 2a48312..cfa9a72 100644 --- a/cmd/creator-web/server.go +++ b/cmd/creator-web/server.go @@ -220,6 +220,8 @@ type browserChatResponse struct { DelegationID string `json:"delegation_id"` DelegationState string `json:"delegation_state"` ReminderScheduled bool `json:"reminder_scheduled"` + GeneratedImageContentType string `json:"generated_image_content_type,omitempty"` + GeneratedImageBase64 string `json:"generated_image_base64,omitempty"` } func (g *gateway) handleAdvisoryPolicy(w http.ResponseWriter, r *http.Request) { @@ -513,6 +515,8 @@ func (g *gateway) handleChatMessage(w http.ResponseWriter, r *http.Request) { DelegationID: saved.DelegationID, DelegationState: saved.DelegationState, ReminderScheduled: saved.Reminder != nil, + GeneratedImageContentType: saved.GeneratedImageContentType, + GeneratedImageBase64: saved.GeneratedImageBase64, }) } diff --git a/cmd/creator-web/static/assets/chat.css b/cmd/creator-web/static/assets/chat.css index 763b669..07625df 100644 --- a/cmd/creator-web/static/assets/chat.css +++ b/cmd/creator-web/static/assets/chat.css @@ -56,6 +56,7 @@ textarea { resize: vertical; min-height: 84px; max-height: 240px; padding: 14px; .chat-header > .operator-link { flex: 0 0 auto; } .chat-layout { min-height: 0; height: 100%; display: grid; grid-template-columns: minmax(220px, 280px) minmax(0, 1fr) minmax(250px, 320px); } .chat-side { min-width: 0; min-height: 0; padding: 28px 20px; overflow-y: auto; background: #0e1512; } +.generated-image { display: block; max-width: min(100%, 640px); height: auto; margin-top: 12px; border: 1px solid var(--line); } .chat-left { border-right: 1px solid var(--line); } .chat-right { border-left: 1px solid var(--line); } .side-intro h1, .side-intro h2 { margin: 12px 0; font: 400 clamp(28px, 2.5vw, 40px)/.95 var(--serif); letter-spacing: -.03em; } diff --git a/cmd/creator-web/static/assets/chat.js b/cmd/creator-web/static/assets/chat.js index b5f6c9e..0298a26 100644 --- a/cmd/creator-web/static/assets/chat.js +++ b/cmd/creator-web/static/assets/chat.js @@ -103,7 +103,7 @@ function showLogin() { loginView.hidden = false; } -function appendMessage(kind, text, meta = "") { +function appendMessage(kind, text, meta = "", media = null) { const node = document.createElement("div"); node.className = `message ${kind}`; node.textContent = text; @@ -112,6 +112,13 @@ function appendMessage(kind, text, meta = "") { small.textContent = meta; node.appendChild(small); } + if (media?.data && media?.contentType) { + const image = document.createElement("img"); + image.className = "generated-image"; + image.alt = text || "Сгенерированное изображение"; + image.src = `data:${media.contentType};base64,${media.data}`; + node.appendChild(image); + } messages.appendChild(node); messages.scrollTop = messages.scrollHeight; localHistory.push({kind, text, meta, at: new Date()}); @@ -277,7 +284,7 @@ async function sendChatMessage(text, options = {}) { conversationID = result.conversation_id; sessionStorage.setItem("domovoy.chat.conversation", conversationID); updateActiveFocus(result.active_focus, result.project_provenance); - appendMessage("assistant", result.reply_text || "Сообщение записано.", humanMeta(result)); + appendMessage("assistant", result.reply_text || "Сообщение записано.", humanMeta(result), result.generated_image_base64 ? {contentType: result.generated_image_content_type || "image/png", data: result.generated_image_base64} : null); if (/что сейчас требует моего внимания/i.test(clean)) { await loadToday(); activateTab("today-panel"); diff --git a/contracts/interaction/types.go b/contracts/interaction/types.go index 49d387c..a9d01fe 100644 --- a/contracts/interaction/types.go +++ b/contracts/interaction/types.go @@ -187,4 +187,6 @@ type ChatMessageResponse struct { DelegationID string DelegationState string Reminder *continuity.ProspectiveItem + GeneratedImageContentType string + GeneratedImageBase64 string } diff --git a/modules/interaction/domain/chat.go b/modules/interaction/domain/chat.go index f75289f..0051039 100644 --- a/modules/interaction/domain/chat.go +++ b/modules/interaction/domain/chat.go @@ -496,7 +496,11 @@ func capabilityHintsFor(primary, secondary interaction.SpeechAct, tokens []strin add("WEB_RESEARCH") } if isCreativeRequestTokens(tokens) { - add("COGNITIVE_REASONING") + if isImageCreationRequestTokens(tokens) { + add("IMAGE_GENERATION") + } else { + add("COGNITIVE_REASONING") + } } if containsToken(tokens, "напомни") || containsToken(tokens, "напомин") { add("SCHEDULE_REMINDER") @@ -539,6 +543,10 @@ func capabilityHintsFor(primary, secondary interaction.SpeechAct, tokens []strin return values } +func isImageCreationRequestTokens(tokens []string) bool { + return containsStemInTokens(tokens, "нарис", "рису", "изобраз") +} + func isCreativeRequestTokens(tokens []string) bool { if containsStemInTokens(tokens, "нарис", "рису", "изобраз") { return true