Route creative image requests to media generation #362

Merged
m merged 2 commits from codex/household-scenario-audit into main 2026-08-25 14:42:40 +00:00
11 changed files with 127 additions and 7 deletions
+3
View File
@@ -12,6 +12,7 @@ type chatRouteKind string
const (
chatRouteResearch chatRouteKind = "RESEARCH"
chatRouteConversation chatRouteKind = "CONVERSATION"
chatRouteMedia chatRouteKind = "MEDIA"
chatRouteAction chatRouteKind = "ACTION"
chatRouteAgent chatRouteKind = "AGENT"
)
@@ -42,6 +43,8 @@ func buildChatRoutePlan(result interactiondomain.SemanticResult) []chatRoute {
switch capability.CapabilityKind(hint) {
case capability.CapabilityKindWebResearch:
add(chatRouteResearch, capability.CapabilityKindWebResearch, "truth-research-v1", 10)
case capability.CapabilityKindImageGeneration:
add(chatRouteMedia, capability.CapabilityKindImageGeneration, "local-image-generation-v1", 5)
case capability.CapabilityKindCognitiveReasoning, capability.CapabilityKindVerifyFact:
add(chatRouteConversation, capability.CapabilityKind(hint), "local-reasoning-v1", 20)
case capability.CapabilityKindScheduleReminder:
+47 -1
View File
@@ -6,6 +6,8 @@ import (
"encoding/json"
"fmt"
"hash/fnv"
"io"
"mime/multipart"
"net/http"
"net/url"
"strings"
@@ -23,6 +25,7 @@ import (
type cognitiveInterpretationClient struct {
baseURL *url.URL
client *http.Client
media *http.Client
}
// ReviewResponse asks the cognitive gateway to turn a locally assembled draft
@@ -83,7 +86,7 @@ func newCognitiveInterpretationClient(rawURL string, client *http.Client) (cogni
if client == nil {
client = http.DefaultClient
}
return cognitiveInterpretationClient{baseURL: base, client: client}, nil
return cognitiveInterpretationClient{baseURL: base, client: client, media: &http.Client{Timeout: 15 * time.Minute}}, nil
}
func (c cognitiveInterpretationClient) Interpret(ctx context.Context, request cognitive.GatewayInterpretationRequest) (cognitive.GatewayInterpretationResponse, error) {
@@ -113,6 +116,49 @@ func (c cognitiveInterpretationClient) Interpret(ctx context.Context, request co
return result, nil
}
func (c cognitiveInterpretationClient) GenerateImage(ctx context.Context, prompt, size string) (capability.MultimodalResponse, error) {
var body bytes.Buffer
writer := multipart.NewWriter(&body)
if err := writer.WriteField("prompt", prompt); err != nil {
return capability.MultimodalResponse{}, err
}
if strings.TrimSpace(size) != "" {
if err := writer.WriteField("size", size); err != nil {
return capability.MultimodalResponse{}, err
}
}
if err := writer.Close(); err != nil {
return capability.MultimodalResponse{}, err
}
target := *c.baseURL
target.Path = "/v1/multimodal/generate-image"
request, err := http.NewRequestWithContext(ctx, http.MethodPost, target.String(), &body)
if err != nil {
return capability.MultimodalResponse{}, err
}
request.Header.Set("Content-Type", writer.FormDataContentType())
client := c.media
if client == nil {
client = c.client
}
response, err := client.Do(request)
if err != nil {
return capability.MultimodalResponse{}, err
}
defer response.Body.Close()
if response.StatusCode != http.StatusOK {
return capability.MultimodalResponse{}, fmt.Errorf("image generation status %d", response.StatusCode)
}
var result capability.MultimodalResponse
if err := json.NewDecoder(io.LimitReader(response.Body, 32<<20)).Decode(&result); err != nil {
return capability.MultimodalResponse{}, err
}
if strings.TrimSpace(result.DataBase64) == "" {
return capability.MultimodalResponse{}, fmt.Errorf("image generation returned empty payload")
}
return result, nil
}
func shouldRequestBroaderInterpretation(result interactiondomain.SemanticResult) bool {
return shouldRequestSemanticInterpretation(result)
}
@@ -5,9 +5,11 @@ import (
"encoding/json"
"net/http"
"net/http/httptest"
"strings"
"testing"
"time"
"git.cin.su/m/Assistant/contracts/capability"
"git.cin.su/m/Assistant/contracts/cognitive"
"git.cin.su/m/Assistant/contracts/identity"
"git.cin.su/m/Assistant/contracts/interaction"
@@ -40,6 +42,30 @@ func TestReviewResponseUsesCognitiveGatewayAndReturnsOnlyDraftAnswer(t *testing.
}
}
func TestGenerateImageUsesDedicatedCognitiveMediaRoute(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodPost || r.URL.Path != "/v1/multimodal/generate-image" {
t.Fatalf("unexpected image route: %s %s", r.Method, r.URL.Path)
}
if err := r.ParseMultipartForm(1 << 20); err != nil {
t.Fatalf("parse image request: %v", err)
}
if !strings.Contains(r.FormValue("prompt"), "собаку") {
t.Fatalf("prompt was not preserved: %q", r.FormValue("prompt"))
}
_ = json.NewEncoder(w).Encode(capability.MultimodalResponse{Operation: capability.MultimodalGenerateImage, ContentType: "image/png", DataBase64: "cG5n"})
}))
defer server.Close()
client, err := newCognitiveInterpretationClient(server.URL, server.Client())
if err != nil {
t.Fatal(err)
}
image, err := client.GenerateImage(context.Background(), "нарисуй собаку", "")
if err != nil || image.DataBase64 != "cG5n" || image.ContentType != "image/png" {
t.Fatalf("image=%+v err=%v", image, err)
}
}
func TestReviewPreservesResearchEvidence(t *testing.T) {
draft := "Нашёл вариант. Источник: Aviasales (https://www.aviasales.ru/search)"
if reviewPreservesResearchEvidence("Нашёл вариант на Авиасейлс.", draft) {
@@ -68,6 +68,10 @@ func TestHouseholdBearDrawingUsesCreativeLocalRoute(t *testing.T) {
if decomposition.Intent != interactiondomain.TaskIntentCreate || decomposition.Object != "IMAGE" || decomposition.Target != "бурого медведя" || decomposition.NeedsResearch {
t.Fatalf("creative request was not split into local execution atoms: %+v", decomposition)
}
routes := buildChatRoutePlan(interactiondomain.SemanticResult{NeedsCognition: candidate.NeedsCognition, CapabilityHints: candidate.CapabilityHints})
if !hasChatRoute(routes, chatRouteMedia) {
t.Fatalf("creative image request did not select media route: candidate=%+v routes=%+v", candidate, routes)
}
for _, hint := range candidate.CapabilityHints {
if capability.CapabilityKind(hint) == capability.CapabilityKindWebResearch {
t.Fatalf("bear drawing selected web research: %+v", candidate)
+21 -2
View File
@@ -819,6 +819,23 @@ func main() {
var delegationID work.DelegationID
var delegationState work.DelegationState
var reminder *continuity.ProspectiveItem
var generatedImage capability.MultimodalResponse
mediaHandled := false
if hasChatRoute(routePlan, chatRouteMedia) && semantic.Interpretation.Clarification != interaction.ClarificationRequired {
imageCtx, cancelImage := context.WithTimeout(r.Context(), 15*time.Minute)
generatedImage, err = cognitiveClient.GenerateImage(imageCtx, effectiveTaskMessage, "")
cancelImage()
if err == nil && strings.TrimSpace(generatedImage.DataBase64) != "" {
planningReply = "Готово."
semantic.Reply = planningReply
needsCognition = false
mediaHandled = true
} else {
planningReply = "Сейчас не удалось создать изображение."
semantic.Reply = planningReply
needsCognition = false
}
}
if helpPath.Kind == work.HelpPathDirectCapability && len(helpPath.SuggestedCapabilityKinds) > 0 {
capabilityKind = helpPath.SuggestedCapabilityKinds[0]
started := time.Now()
@@ -903,7 +920,7 @@ func main() {
ConfidenceMillis: semantic.Interpretation.ConfidenceMillis,
})
cancelReview()
stableSemanticReply := identityReply != "" || creativeReply != "" || (creativeFallbackReply != "" && !cognitiveAvailable) || semantic.InformationRequestKind != "" || locationReply != "" || deterministicSpeechAct == interaction.SpeechActConversational || deterministicSpeechAct == interaction.SpeechActConfirmation || deterministicSpeechAct == interaction.SpeechActRejection
stableSemanticReply := mediaHandled || identityReply != "" || creativeReply != "" || (creativeFallbackReply != "" && !cognitiveAvailable) || semantic.InformationRequestKind != "" || locationReply != "" || deterministicSpeechAct == interaction.SpeechActConversational || deterministicSpeechAct == interaction.SpeechActConfirmation || deterministicSpeechAct == interaction.SpeechActRejection
if reviewErr != nil {
log.Printf("cognitive response review unavailable: %v", reviewErr)
} else if !stableSemanticReply && (!researchBacked || reviewPreservesResearchEvidence(reviewedReply, planningReply)) {
@@ -912,7 +929,7 @@ func main() {
log.Printf("cognitive response review proposed stable semantic rewrite; preserving semantic reply")
}
}
if identityReply == "" && creativeReply == "" && creativeFallbackReply == "" {
if !mediaHandled && identityReply == "" && creativeReply == "" && creativeFallbackReply == "" {
stableAct := interactiondomain.ClassifySpeechAct(utterance.Content, utterance.CorrectsUtteranceID)
travelClarification := ""
if !creativeRequest {
@@ -967,6 +984,8 @@ func main() {
DelegationID: string(delegationID),
DelegationState: string(delegationState),
Reminder: reminder,
GeneratedImageContentType: generatedImage.ContentType,
GeneratedImageBase64: generatedImage.DataBase64,
}
w.Header().Set("Content-Type", "application/json")
w.Header().Set("Cache-Control", "no-store")
+1 -1
View File
@@ -20,7 +20,7 @@ func main() {
// A cold local model may need more than one short HTTP round-trip to
// load on CPU. Keep the browser gateway aligned with the cognition
// deadline instead of turning that wait into a false unavailable reply.
&http.Client{Timeout: 90 * time.Second},
&http.Client{Timeout: 15 * time.Minute},
envOr("COGNITIVE_RUNTIME_URL", "http://cognitive-worker:8083"),
envOr("AGENT_WORKER_URL", "http://agent-worker:8090"),
)
+4
View File
@@ -220,6 +220,8 @@ type browserChatResponse struct {
DelegationID string `json:"delegation_id"`
DelegationState string `json:"delegation_state"`
ReminderScheduled bool `json:"reminder_scheduled"`
GeneratedImageContentType string `json:"generated_image_content_type,omitempty"`
GeneratedImageBase64 string `json:"generated_image_base64,omitempty"`
}
func (g *gateway) handleAdvisoryPolicy(w http.ResponseWriter, r *http.Request) {
@@ -513,6 +515,8 @@ func (g *gateway) handleChatMessage(w http.ResponseWriter, r *http.Request) {
DelegationID: saved.DelegationID,
DelegationState: saved.DelegationState,
ReminderScheduled: saved.Reminder != nil,
GeneratedImageContentType: saved.GeneratedImageContentType,
GeneratedImageBase64: saved.GeneratedImageBase64,
})
}
+1
View File
@@ -56,6 +56,7 @@ textarea { resize: vertical; min-height: 84px; max-height: 240px; padding: 14px;
.chat-header > .operator-link { flex: 0 0 auto; }
.chat-layout { min-height: 0; height: 100%; display: grid; grid-template-columns: minmax(220px, 280px) minmax(0, 1fr) minmax(250px, 320px); }
.chat-side { min-width: 0; min-height: 0; padding: 28px 20px; overflow-y: auto; background: #0e1512; }
.generated-image { display: block; max-width: min(100%, 640px); height: auto; margin-top: 12px; border: 1px solid var(--line); }
.chat-left { border-right: 1px solid var(--line); }
.chat-right { border-left: 1px solid var(--line); }
.side-intro h1, .side-intro h2 { margin: 12px 0; font: 400 clamp(28px, 2.5vw, 40px)/.95 var(--serif); letter-spacing: -.03em; }
+9 -2
View File
@@ -103,7 +103,7 @@ function showLogin() {
loginView.hidden = false;
}
function appendMessage(kind, text, meta = "") {
function appendMessage(kind, text, meta = "", media = null) {
const node = document.createElement("div");
node.className = `message ${kind}`;
node.textContent = text;
@@ -112,6 +112,13 @@ function appendMessage(kind, text, meta = "") {
small.textContent = meta;
node.appendChild(small);
}
if (media?.data && media?.contentType) {
const image = document.createElement("img");
image.className = "generated-image";
image.alt = text || "Сгенерированное изображение";
image.src = `data:${media.contentType};base64,${media.data}`;
node.appendChild(image);
}
messages.appendChild(node);
messages.scrollTop = messages.scrollHeight;
localHistory.push({kind, text, meta, at: new Date()});
@@ -277,7 +284,7 @@ async function sendChatMessage(text, options = {}) {
conversationID = result.conversation_id;
sessionStorage.setItem("domovoy.chat.conversation", conversationID);
updateActiveFocus(result.active_focus, result.project_provenance);
appendMessage("assistant", result.reply_text || "Сообщение записано.", humanMeta(result));
appendMessage("assistant", result.reply_text || "Сообщение записано.", humanMeta(result), result.generated_image_base64 ? {contentType: result.generated_image_content_type || "image/png", data: result.generated_image_base64} : null);
if (/что сейчас требует моего внимания/i.test(clean)) {
await loadToday();
activateTab("today-panel");
+2
View File
@@ -187,4 +187,6 @@ type ChatMessageResponse struct {
DelegationID string
DelegationState string
Reminder *continuity.ProspectiveItem
GeneratedImageContentType string
GeneratedImageBase64 string
}
+9 -1
View File
@@ -496,7 +496,11 @@ func capabilityHintsFor(primary, secondary interaction.SpeechAct, tokens []strin
add("WEB_RESEARCH")
}
if isCreativeRequestTokens(tokens) {
add("COGNITIVE_REASONING")
if isImageCreationRequestTokens(tokens) {
add("IMAGE_GENERATION")
} else {
add("COGNITIVE_REASONING")
}
}
if containsToken(tokens, "напомни") || containsToken(tokens, "напомин") {
add("SCHEDULE_REMINDER")
@@ -539,6 +543,10 @@ func capabilityHintsFor(primary, secondary interaction.SpeechAct, tokens []strin
return values
}
func isImageCreationRequestTokens(tokens []string) bool {
return containsStemInTokens(tokens, "нарис", "рису", "изобраз")
}
func isCreativeRequestTokens(tokens []string) bool {
if containsStemInTokens(tokens, "нарис", "рису", "изобраз") {
return true