Route creative image requests to media generation
CI / test (push) Failing after 1m8s
CI / test (pull_request) Failing after 1m1s

This commit is contained in:
m
2026-08-25 17:34:05 +03:00
parent 6cb125001f
commit 14c5d9e412
11 changed files with 127 additions and 7 deletions
+3
View File
@@ -12,6 +12,7 @@ type chatRouteKind string
const (
chatRouteResearch chatRouteKind = "RESEARCH"
chatRouteConversation chatRouteKind = "CONVERSATION"
chatRouteMedia chatRouteKind = "MEDIA"
chatRouteAction chatRouteKind = "ACTION"
chatRouteAgent chatRouteKind = "AGENT"
)
@@ -42,6 +43,8 @@ func buildChatRoutePlan(result interactiondomain.SemanticResult) []chatRoute {
switch capability.CapabilityKind(hint) {
case capability.CapabilityKindWebResearch:
add(chatRouteResearch, capability.CapabilityKindWebResearch, "truth-research-v1", 10)
case capability.CapabilityKindImageGeneration:
add(chatRouteMedia, capability.CapabilityKindImageGeneration, "local-image-generation-v1", 5)
case capability.CapabilityKindCognitiveReasoning, capability.CapabilityKindVerifyFact:
add(chatRouteConversation, capability.CapabilityKind(hint), "local-reasoning-v1", 20)
case capability.CapabilityKindScheduleReminder:
+47 -1
View File
@@ -6,6 +6,8 @@ import (
"encoding/json"
"fmt"
"hash/fnv"
"io"
"mime/multipart"
"net/http"
"net/url"
"strings"
@@ -23,6 +25,7 @@ import (
type cognitiveInterpretationClient struct {
baseURL *url.URL
client *http.Client
media *http.Client
}
// ReviewResponse asks the cognitive gateway to turn a locally assembled draft
@@ -83,7 +86,7 @@ func newCognitiveInterpretationClient(rawURL string, client *http.Client) (cogni
if client == nil {
client = http.DefaultClient
}
return cognitiveInterpretationClient{baseURL: base, client: client}, nil
return cognitiveInterpretationClient{baseURL: base, client: client, media: &http.Client{Timeout: 15 * time.Minute}}, nil
}
func (c cognitiveInterpretationClient) Interpret(ctx context.Context, request cognitive.GatewayInterpretationRequest) (cognitive.GatewayInterpretationResponse, error) {
@@ -113,6 +116,49 @@ func (c cognitiveInterpretationClient) Interpret(ctx context.Context, request co
return result, nil
}
func (c cognitiveInterpretationClient) GenerateImage(ctx context.Context, prompt, size string) (capability.MultimodalResponse, error) {
var body bytes.Buffer
writer := multipart.NewWriter(&body)
if err := writer.WriteField("prompt", prompt); err != nil {
return capability.MultimodalResponse{}, err
}
if strings.TrimSpace(size) != "" {
if err := writer.WriteField("size", size); err != nil {
return capability.MultimodalResponse{}, err
}
}
if err := writer.Close(); err != nil {
return capability.MultimodalResponse{}, err
}
target := *c.baseURL
target.Path = "/v1/multimodal/generate-image"
request, err := http.NewRequestWithContext(ctx, http.MethodPost, target.String(), &body)
if err != nil {
return capability.MultimodalResponse{}, err
}
request.Header.Set("Content-Type", writer.FormDataContentType())
client := c.media
if client == nil {
client = c.client
}
response, err := client.Do(request)
if err != nil {
return capability.MultimodalResponse{}, err
}
defer response.Body.Close()
if response.StatusCode != http.StatusOK {
return capability.MultimodalResponse{}, fmt.Errorf("image generation status %d", response.StatusCode)
}
var result capability.MultimodalResponse
if err := json.NewDecoder(io.LimitReader(response.Body, 32<<20)).Decode(&result); err != nil {
return capability.MultimodalResponse{}, err
}
if strings.TrimSpace(result.DataBase64) == "" {
return capability.MultimodalResponse{}, fmt.Errorf("image generation returned empty payload")
}
return result, nil
}
func shouldRequestBroaderInterpretation(result interactiondomain.SemanticResult) bool {
return shouldRequestSemanticInterpretation(result)
}
@@ -5,9 +5,11 @@ import (
"encoding/json"
"net/http"
"net/http/httptest"
"strings"
"testing"
"time"
"git.cin.su/m/Assistant/contracts/capability"
"git.cin.su/m/Assistant/contracts/cognitive"
"git.cin.su/m/Assistant/contracts/identity"
"git.cin.su/m/Assistant/contracts/interaction"
@@ -40,6 +42,30 @@ func TestReviewResponseUsesCognitiveGatewayAndReturnsOnlyDraftAnswer(t *testing.
}
}
func TestGenerateImageUsesDedicatedCognitiveMediaRoute(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodPost || r.URL.Path != "/v1/multimodal/generate-image" {
t.Fatalf("unexpected image route: %s %s", r.Method, r.URL.Path)
}
if err := r.ParseMultipartForm(1 << 20); err != nil {
t.Fatalf("parse image request: %v", err)
}
if !strings.Contains(r.FormValue("prompt"), "собаку") {
t.Fatalf("prompt was not preserved: %q", r.FormValue("prompt"))
}
_ = json.NewEncoder(w).Encode(capability.MultimodalResponse{Operation: capability.MultimodalGenerateImage, ContentType: "image/png", DataBase64: "cG5n"})
}))
defer server.Close()
client, err := newCognitiveInterpretationClient(server.URL, server.Client())
if err != nil {
t.Fatal(err)
}
image, err := client.GenerateImage(context.Background(), "нарисуй собаку", "")
if err != nil || image.DataBase64 != "cG5n" || image.ContentType != "image/png" {
t.Fatalf("image=%+v err=%v", image, err)
}
}
func TestReviewPreservesResearchEvidence(t *testing.T) {
draft := "Нашёл вариант. Источник: Aviasales (https://www.aviasales.ru/search)"
if reviewPreservesResearchEvidence("Нашёл вариант на Авиасейлс.", draft) {
@@ -68,6 +68,10 @@ func TestHouseholdBearDrawingUsesCreativeLocalRoute(t *testing.T) {
if decomposition.Intent != interactiondomain.TaskIntentCreate || decomposition.Object != "IMAGE" || decomposition.Target != "бурого медведя" || decomposition.NeedsResearch {
t.Fatalf("creative request was not split into local execution atoms: %+v", decomposition)
}
routes := buildChatRoutePlan(interactiondomain.SemanticResult{NeedsCognition: candidate.NeedsCognition, CapabilityHints: candidate.CapabilityHints})
if !hasChatRoute(routes, chatRouteMedia) {
t.Fatalf("creative image request did not select media route: candidate=%+v routes=%+v", candidate, routes)
}
for _, hint := range candidate.CapabilityHints {
if capability.CapabilityKind(hint) == capability.CapabilityKindWebResearch {
t.Fatalf("bear drawing selected web research: %+v", candidate)
+21 -2
View File
@@ -819,6 +819,23 @@ func main() {
var delegationID work.DelegationID
var delegationState work.DelegationState
var reminder *continuity.ProspectiveItem
var generatedImage capability.MultimodalResponse
mediaHandled := false
if hasChatRoute(routePlan, chatRouteMedia) && semantic.Interpretation.Clarification != interaction.ClarificationRequired {
imageCtx, cancelImage := context.WithTimeout(r.Context(), 15*time.Minute)
generatedImage, err = cognitiveClient.GenerateImage(imageCtx, effectiveTaskMessage, "")
cancelImage()
if err == nil && strings.TrimSpace(generatedImage.DataBase64) != "" {
planningReply = "Готово."
semantic.Reply = planningReply
needsCognition = false
mediaHandled = true
} else {
planningReply = "Сейчас не удалось создать изображение."
semantic.Reply = planningReply
needsCognition = false
}
}
if helpPath.Kind == work.HelpPathDirectCapability && len(helpPath.SuggestedCapabilityKinds) > 0 {
capabilityKind = helpPath.SuggestedCapabilityKinds[0]
started := time.Now()
@@ -903,7 +920,7 @@ func main() {
ConfidenceMillis: semantic.Interpretation.ConfidenceMillis,
})
cancelReview()
stableSemanticReply := identityReply != "" || creativeReply != "" || (creativeFallbackReply != "" && !cognitiveAvailable) || semantic.InformationRequestKind != "" || locationReply != "" || deterministicSpeechAct == interaction.SpeechActConversational || deterministicSpeechAct == interaction.SpeechActConfirmation || deterministicSpeechAct == interaction.SpeechActRejection
stableSemanticReply := mediaHandled || identityReply != "" || creativeReply != "" || (creativeFallbackReply != "" && !cognitiveAvailable) || semantic.InformationRequestKind != "" || locationReply != "" || deterministicSpeechAct == interaction.SpeechActConversational || deterministicSpeechAct == interaction.SpeechActConfirmation || deterministicSpeechAct == interaction.SpeechActRejection
if reviewErr != nil {
log.Printf("cognitive response review unavailable: %v", reviewErr)
} else if !stableSemanticReply && (!researchBacked || reviewPreservesResearchEvidence(reviewedReply, planningReply)) {
@@ -912,7 +929,7 @@ func main() {
log.Printf("cognitive response review proposed stable semantic rewrite; preserving semantic reply")
}
}
if identityReply == "" && creativeReply == "" && creativeFallbackReply == "" {
if !mediaHandled && identityReply == "" && creativeReply == "" && creativeFallbackReply == "" {
stableAct := interactiondomain.ClassifySpeechAct(utterance.Content, utterance.CorrectsUtteranceID)
travelClarification := ""
if !creativeRequest {
@@ -967,6 +984,8 @@ func main() {
DelegationID: string(delegationID),
DelegationState: string(delegationState),
Reminder: reminder,
GeneratedImageContentType: generatedImage.ContentType,
GeneratedImageBase64: generatedImage.DataBase64,
}
w.Header().Set("Content-Type", "application/json")
w.Header().Set("Cache-Control", "no-store")