Route creative image requests to media generation #362
@@ -12,6 +12,7 @@ type chatRouteKind string
|
||||
const (
|
||||
chatRouteResearch chatRouteKind = "RESEARCH"
|
||||
chatRouteConversation chatRouteKind = "CONVERSATION"
|
||||
chatRouteMedia chatRouteKind = "MEDIA"
|
||||
chatRouteAction chatRouteKind = "ACTION"
|
||||
chatRouteAgent chatRouteKind = "AGENT"
|
||||
)
|
||||
@@ -42,6 +43,8 @@ func buildChatRoutePlan(result interactiondomain.SemanticResult) []chatRoute {
|
||||
switch capability.CapabilityKind(hint) {
|
||||
case capability.CapabilityKindWebResearch:
|
||||
add(chatRouteResearch, capability.CapabilityKindWebResearch, "truth-research-v1", 10)
|
||||
case capability.CapabilityKindImageGeneration:
|
||||
add(chatRouteMedia, capability.CapabilityKindImageGeneration, "local-image-generation-v1", 5)
|
||||
case capability.CapabilityKindCognitiveReasoning, capability.CapabilityKindVerifyFact:
|
||||
add(chatRouteConversation, capability.CapabilityKind(hint), "local-reasoning-v1", 20)
|
||||
case capability.CapabilityKindScheduleReminder:
|
||||
|
||||
@@ -6,6 +6,8 @@ import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"hash/fnv"
|
||||
"io"
|
||||
"mime/multipart"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strings"
|
||||
@@ -23,6 +25,7 @@ import (
|
||||
type cognitiveInterpretationClient struct {
|
||||
baseURL *url.URL
|
||||
client *http.Client
|
||||
media *http.Client
|
||||
}
|
||||
|
||||
// ReviewResponse asks the cognitive gateway to turn a locally assembled draft
|
||||
@@ -83,7 +86,7 @@ func newCognitiveInterpretationClient(rawURL string, client *http.Client) (cogni
|
||||
if client == nil {
|
||||
client = http.DefaultClient
|
||||
}
|
||||
return cognitiveInterpretationClient{baseURL: base, client: client}, nil
|
||||
return cognitiveInterpretationClient{baseURL: base, client: client, media: &http.Client{Timeout: 15 * time.Minute}}, nil
|
||||
}
|
||||
|
||||
func (c cognitiveInterpretationClient) Interpret(ctx context.Context, request cognitive.GatewayInterpretationRequest) (cognitive.GatewayInterpretationResponse, error) {
|
||||
@@ -113,6 +116,49 @@ func (c cognitiveInterpretationClient) Interpret(ctx context.Context, request co
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (c cognitiveInterpretationClient) GenerateImage(ctx context.Context, prompt, size string) (capability.MultimodalResponse, error) {
|
||||
var body bytes.Buffer
|
||||
writer := multipart.NewWriter(&body)
|
||||
if err := writer.WriteField("prompt", prompt); err != nil {
|
||||
return capability.MultimodalResponse{}, err
|
||||
}
|
||||
if strings.TrimSpace(size) != "" {
|
||||
if err := writer.WriteField("size", size); err != nil {
|
||||
return capability.MultimodalResponse{}, err
|
||||
}
|
||||
}
|
||||
if err := writer.Close(); err != nil {
|
||||
return capability.MultimodalResponse{}, err
|
||||
}
|
||||
target := *c.baseURL
|
||||
target.Path = "/v1/multimodal/generate-image"
|
||||
request, err := http.NewRequestWithContext(ctx, http.MethodPost, target.String(), &body)
|
||||
if err != nil {
|
||||
return capability.MultimodalResponse{}, err
|
||||
}
|
||||
request.Header.Set("Content-Type", writer.FormDataContentType())
|
||||
client := c.media
|
||||
if client == nil {
|
||||
client = c.client
|
||||
}
|
||||
response, err := client.Do(request)
|
||||
if err != nil {
|
||||
return capability.MultimodalResponse{}, err
|
||||
}
|
||||
defer response.Body.Close()
|
||||
if response.StatusCode != http.StatusOK {
|
||||
return capability.MultimodalResponse{}, fmt.Errorf("image generation status %d", response.StatusCode)
|
||||
}
|
||||
var result capability.MultimodalResponse
|
||||
if err := json.NewDecoder(io.LimitReader(response.Body, 32<<20)).Decode(&result); err != nil {
|
||||
return capability.MultimodalResponse{}, err
|
||||
}
|
||||
if strings.TrimSpace(result.DataBase64) == "" {
|
||||
return capability.MultimodalResponse{}, fmt.Errorf("image generation returned empty payload")
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func shouldRequestBroaderInterpretation(result interactiondomain.SemanticResult) bool {
|
||||
return shouldRequestSemanticInterpretation(result)
|
||||
}
|
||||
|
||||
@@ -5,9 +5,11 @@ import (
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.cin.su/m/Assistant/contracts/capability"
|
||||
"git.cin.su/m/Assistant/contracts/cognitive"
|
||||
"git.cin.su/m/Assistant/contracts/identity"
|
||||
"git.cin.su/m/Assistant/contracts/interaction"
|
||||
@@ -40,6 +42,30 @@ func TestReviewResponseUsesCognitiveGatewayAndReturnsOnlyDraftAnswer(t *testing.
|
||||
}
|
||||
}
|
||||
|
||||
func TestGenerateImageUsesDedicatedCognitiveMediaRoute(t *testing.T) {
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method != http.MethodPost || r.URL.Path != "/v1/multimodal/generate-image" {
|
||||
t.Fatalf("unexpected image route: %s %s", r.Method, r.URL.Path)
|
||||
}
|
||||
if err := r.ParseMultipartForm(1 << 20); err != nil {
|
||||
t.Fatalf("parse image request: %v", err)
|
||||
}
|
||||
if !strings.Contains(r.FormValue("prompt"), "собаку") {
|
||||
t.Fatalf("prompt was not preserved: %q", r.FormValue("prompt"))
|
||||
}
|
||||
_ = json.NewEncoder(w).Encode(capability.MultimodalResponse{Operation: capability.MultimodalGenerateImage, ContentType: "image/png", DataBase64: "cG5n"})
|
||||
}))
|
||||
defer server.Close()
|
||||
client, err := newCognitiveInterpretationClient(server.URL, server.Client())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
image, err := client.GenerateImage(context.Background(), "нарисуй собаку", "")
|
||||
if err != nil || image.DataBase64 != "cG5n" || image.ContentType != "image/png" {
|
||||
t.Fatalf("image=%+v err=%v", image, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestReviewPreservesResearchEvidence(t *testing.T) {
|
||||
draft := "Нашёл вариант. Источник: Aviasales (https://www.aviasales.ru/search)"
|
||||
if reviewPreservesResearchEvidence("Нашёл вариант на Авиасейлс.", draft) {
|
||||
|
||||
@@ -68,6 +68,10 @@ func TestHouseholdBearDrawingUsesCreativeLocalRoute(t *testing.T) {
|
||||
if decomposition.Intent != interactiondomain.TaskIntentCreate || decomposition.Object != "IMAGE" || decomposition.Target != "бурого медведя" || decomposition.NeedsResearch {
|
||||
t.Fatalf("creative request was not split into local execution atoms: %+v", decomposition)
|
||||
}
|
||||
routes := buildChatRoutePlan(interactiondomain.SemanticResult{NeedsCognition: candidate.NeedsCognition, CapabilityHints: candidate.CapabilityHints})
|
||||
if !hasChatRoute(routes, chatRouteMedia) {
|
||||
t.Fatalf("creative image request did not select media route: candidate=%+v routes=%+v", candidate, routes)
|
||||
}
|
||||
for _, hint := range candidate.CapabilityHints {
|
||||
if capability.CapabilityKind(hint) == capability.CapabilityKindWebResearch {
|
||||
t.Fatalf("bear drawing selected web research: %+v", candidate)
|
||||
|
||||
@@ -819,6 +819,23 @@ func main() {
|
||||
var delegationID work.DelegationID
|
||||
var delegationState work.DelegationState
|
||||
var reminder *continuity.ProspectiveItem
|
||||
var generatedImage capability.MultimodalResponse
|
||||
mediaHandled := false
|
||||
if hasChatRoute(routePlan, chatRouteMedia) && semantic.Interpretation.Clarification != interaction.ClarificationRequired {
|
||||
imageCtx, cancelImage := context.WithTimeout(r.Context(), 15*time.Minute)
|
||||
generatedImage, err = cognitiveClient.GenerateImage(imageCtx, effectiveTaskMessage, "")
|
||||
cancelImage()
|
||||
if err == nil && strings.TrimSpace(generatedImage.DataBase64) != "" {
|
||||
planningReply = "Готово."
|
||||
semantic.Reply = planningReply
|
||||
needsCognition = false
|
||||
mediaHandled = true
|
||||
} else {
|
||||
planningReply = "Сейчас не удалось создать изображение."
|
||||
semantic.Reply = planningReply
|
||||
needsCognition = false
|
||||
}
|
||||
}
|
||||
if helpPath.Kind == work.HelpPathDirectCapability && len(helpPath.SuggestedCapabilityKinds) > 0 {
|
||||
capabilityKind = helpPath.SuggestedCapabilityKinds[0]
|
||||
started := time.Now()
|
||||
@@ -903,7 +920,7 @@ func main() {
|
||||
ConfidenceMillis: semantic.Interpretation.ConfidenceMillis,
|
||||
})
|
||||
cancelReview()
|
||||
stableSemanticReply := identityReply != "" || creativeReply != "" || (creativeFallbackReply != "" && !cognitiveAvailable) || semantic.InformationRequestKind != "" || locationReply != "" || deterministicSpeechAct == interaction.SpeechActConversational || deterministicSpeechAct == interaction.SpeechActConfirmation || deterministicSpeechAct == interaction.SpeechActRejection
|
||||
stableSemanticReply := mediaHandled || identityReply != "" || creativeReply != "" || (creativeFallbackReply != "" && !cognitiveAvailable) || semantic.InformationRequestKind != "" || locationReply != "" || deterministicSpeechAct == interaction.SpeechActConversational || deterministicSpeechAct == interaction.SpeechActConfirmation || deterministicSpeechAct == interaction.SpeechActRejection
|
||||
if reviewErr != nil {
|
||||
log.Printf("cognitive response review unavailable: %v", reviewErr)
|
||||
} else if !stableSemanticReply && (!researchBacked || reviewPreservesResearchEvidence(reviewedReply, planningReply)) {
|
||||
@@ -912,7 +929,7 @@ func main() {
|
||||
log.Printf("cognitive response review proposed stable semantic rewrite; preserving semantic reply")
|
||||
}
|
||||
}
|
||||
if identityReply == "" && creativeReply == "" && creativeFallbackReply == "" {
|
||||
if !mediaHandled && identityReply == "" && creativeReply == "" && creativeFallbackReply == "" {
|
||||
stableAct := interactiondomain.ClassifySpeechAct(utterance.Content, utterance.CorrectsUtteranceID)
|
||||
travelClarification := ""
|
||||
if !creativeRequest {
|
||||
@@ -967,6 +984,8 @@ func main() {
|
||||
DelegationID: string(delegationID),
|
||||
DelegationState: string(delegationState),
|
||||
Reminder: reminder,
|
||||
GeneratedImageContentType: generatedImage.ContentType,
|
||||
GeneratedImageBase64: generatedImage.DataBase64,
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.Header().Set("Cache-Control", "no-store")
|
||||
|
||||
@@ -20,7 +20,7 @@ func main() {
|
||||
// A cold local model may need more than one short HTTP round-trip to
|
||||
// load on CPU. Keep the browser gateway aligned with the cognition
|
||||
// deadline instead of turning that wait into a false unavailable reply.
|
||||
&http.Client{Timeout: 90 * time.Second},
|
||||
&http.Client{Timeout: 15 * time.Minute},
|
||||
envOr("COGNITIVE_RUNTIME_URL", "http://cognitive-worker:8083"),
|
||||
envOr("AGENT_WORKER_URL", "http://agent-worker:8090"),
|
||||
)
|
||||
|
||||
@@ -220,6 +220,8 @@ type browserChatResponse struct {
|
||||
DelegationID string `json:"delegation_id"`
|
||||
DelegationState string `json:"delegation_state"`
|
||||
ReminderScheduled bool `json:"reminder_scheduled"`
|
||||
GeneratedImageContentType string `json:"generated_image_content_type,omitempty"`
|
||||
GeneratedImageBase64 string `json:"generated_image_base64,omitempty"`
|
||||
}
|
||||
|
||||
func (g *gateway) handleAdvisoryPolicy(w http.ResponseWriter, r *http.Request) {
|
||||
@@ -513,6 +515,8 @@ func (g *gateway) handleChatMessage(w http.ResponseWriter, r *http.Request) {
|
||||
DelegationID: saved.DelegationID,
|
||||
DelegationState: saved.DelegationState,
|
||||
ReminderScheduled: saved.Reminder != nil,
|
||||
GeneratedImageContentType: saved.GeneratedImageContentType,
|
||||
GeneratedImageBase64: saved.GeneratedImageBase64,
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
@@ -56,6 +56,7 @@ textarea { resize: vertical; min-height: 84px; max-height: 240px; padding: 14px;
|
||||
.chat-header > .operator-link { flex: 0 0 auto; }
|
||||
.chat-layout { min-height: 0; height: 100%; display: grid; grid-template-columns: minmax(220px, 280px) minmax(0, 1fr) minmax(250px, 320px); }
|
||||
.chat-side { min-width: 0; min-height: 0; padding: 28px 20px; overflow-y: auto; background: #0e1512; }
|
||||
.generated-image { display: block; max-width: min(100%, 640px); height: auto; margin-top: 12px; border: 1px solid var(--line); }
|
||||
.chat-left { border-right: 1px solid var(--line); }
|
||||
.chat-right { border-left: 1px solid var(--line); }
|
||||
.side-intro h1, .side-intro h2 { margin: 12px 0; font: 400 clamp(28px, 2.5vw, 40px)/.95 var(--serif); letter-spacing: -.03em; }
|
||||
|
||||
@@ -103,7 +103,7 @@ function showLogin() {
|
||||
loginView.hidden = false;
|
||||
}
|
||||
|
||||
function appendMessage(kind, text, meta = "") {
|
||||
function appendMessage(kind, text, meta = "", media = null) {
|
||||
const node = document.createElement("div");
|
||||
node.className = `message ${kind}`;
|
||||
node.textContent = text;
|
||||
@@ -112,6 +112,13 @@ function appendMessage(kind, text, meta = "") {
|
||||
small.textContent = meta;
|
||||
node.appendChild(small);
|
||||
}
|
||||
if (media?.data && media?.contentType) {
|
||||
const image = document.createElement("img");
|
||||
image.className = "generated-image";
|
||||
image.alt = text || "Сгенерированное изображение";
|
||||
image.src = `data:${media.contentType};base64,${media.data}`;
|
||||
node.appendChild(image);
|
||||
}
|
||||
messages.appendChild(node);
|
||||
messages.scrollTop = messages.scrollHeight;
|
||||
localHistory.push({kind, text, meta, at: new Date()});
|
||||
@@ -277,7 +284,7 @@ async function sendChatMessage(text, options = {}) {
|
||||
conversationID = result.conversation_id;
|
||||
sessionStorage.setItem("domovoy.chat.conversation", conversationID);
|
||||
updateActiveFocus(result.active_focus, result.project_provenance);
|
||||
appendMessage("assistant", result.reply_text || "Сообщение записано.", humanMeta(result));
|
||||
appendMessage("assistant", result.reply_text || "Сообщение записано.", humanMeta(result), result.generated_image_base64 ? {contentType: result.generated_image_content_type || "image/png", data: result.generated_image_base64} : null);
|
||||
if (/что сейчас требует моего внимания/i.test(clean)) {
|
||||
await loadToday();
|
||||
activateTab("today-panel");
|
||||
|
||||
@@ -187,4 +187,6 @@ type ChatMessageResponse struct {
|
||||
DelegationID string
|
||||
DelegationState string
|
||||
Reminder *continuity.ProspectiveItem
|
||||
GeneratedImageContentType string
|
||||
GeneratedImageBase64 string
|
||||
}
|
||||
|
||||
@@ -496,7 +496,11 @@ func capabilityHintsFor(primary, secondary interaction.SpeechAct, tokens []strin
|
||||
add("WEB_RESEARCH")
|
||||
}
|
||||
if isCreativeRequestTokens(tokens) {
|
||||
add("COGNITIVE_REASONING")
|
||||
if isImageCreationRequestTokens(tokens) {
|
||||
add("IMAGE_GENERATION")
|
||||
} else {
|
||||
add("COGNITIVE_REASONING")
|
||||
}
|
||||
}
|
||||
if containsToken(tokens, "напомни") || containsToken(tokens, "напомин") {
|
||||
add("SCHEDULE_REMINDER")
|
||||
@@ -539,6 +543,10 @@ func capabilityHintsFor(primary, secondary interaction.SpeechAct, tokens []strin
|
||||
return values
|
||||
}
|
||||
|
||||
func isImageCreationRequestTokens(tokens []string) bool {
|
||||
return containsStemInTokens(tokens, "нарис", "рису", "изобраз")
|
||||
}
|
||||
|
||||
func isCreativeRequestTokens(tokens []string) bool {
|
||||
if containsStemInTokens(tokens, "нарис", "рису", "изобраз") {
|
||||
return true
|
||||
|
||||
Reference in New Issue
Block a user