Route creative image requests to media generation
This commit is contained in:
@@ -12,6 +12,7 @@ type chatRouteKind string
|
||||
const (
|
||||
chatRouteResearch chatRouteKind = "RESEARCH"
|
||||
chatRouteConversation chatRouteKind = "CONVERSATION"
|
||||
chatRouteMedia chatRouteKind = "MEDIA"
|
||||
chatRouteAction chatRouteKind = "ACTION"
|
||||
chatRouteAgent chatRouteKind = "AGENT"
|
||||
)
|
||||
@@ -42,6 +43,8 @@ func buildChatRoutePlan(result interactiondomain.SemanticResult) []chatRoute {
|
||||
switch capability.CapabilityKind(hint) {
|
||||
case capability.CapabilityKindWebResearch:
|
||||
add(chatRouteResearch, capability.CapabilityKindWebResearch, "truth-research-v1", 10)
|
||||
case capability.CapabilityKindImageGeneration:
|
||||
add(chatRouteMedia, capability.CapabilityKindImageGeneration, "local-image-generation-v1", 5)
|
||||
case capability.CapabilityKindCognitiveReasoning, capability.CapabilityKindVerifyFact:
|
||||
add(chatRouteConversation, capability.CapabilityKind(hint), "local-reasoning-v1", 20)
|
||||
case capability.CapabilityKindScheduleReminder:
|
||||
|
||||
@@ -6,6 +6,8 @@ import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"hash/fnv"
|
||||
"io"
|
||||
"mime/multipart"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strings"
|
||||
@@ -23,6 +25,7 @@ import (
|
||||
type cognitiveInterpretationClient struct {
|
||||
baseURL *url.URL
|
||||
client *http.Client
|
||||
media *http.Client
|
||||
}
|
||||
|
||||
// ReviewResponse asks the cognitive gateway to turn a locally assembled draft
|
||||
@@ -83,7 +86,7 @@ func newCognitiveInterpretationClient(rawURL string, client *http.Client) (cogni
|
||||
if client == nil {
|
||||
client = http.DefaultClient
|
||||
}
|
||||
return cognitiveInterpretationClient{baseURL: base, client: client}, nil
|
||||
return cognitiveInterpretationClient{baseURL: base, client: client, media: &http.Client{Timeout: 15 * time.Minute}}, nil
|
||||
}
|
||||
|
||||
func (c cognitiveInterpretationClient) Interpret(ctx context.Context, request cognitive.GatewayInterpretationRequest) (cognitive.GatewayInterpretationResponse, error) {
|
||||
@@ -113,6 +116,49 @@ func (c cognitiveInterpretationClient) Interpret(ctx context.Context, request co
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (c cognitiveInterpretationClient) GenerateImage(ctx context.Context, prompt, size string) (capability.MultimodalResponse, error) {
|
||||
var body bytes.Buffer
|
||||
writer := multipart.NewWriter(&body)
|
||||
if err := writer.WriteField("prompt", prompt); err != nil {
|
||||
return capability.MultimodalResponse{}, err
|
||||
}
|
||||
if strings.TrimSpace(size) != "" {
|
||||
if err := writer.WriteField("size", size); err != nil {
|
||||
return capability.MultimodalResponse{}, err
|
||||
}
|
||||
}
|
||||
if err := writer.Close(); err != nil {
|
||||
return capability.MultimodalResponse{}, err
|
||||
}
|
||||
target := *c.baseURL
|
||||
target.Path = "/v1/multimodal/generate-image"
|
||||
request, err := http.NewRequestWithContext(ctx, http.MethodPost, target.String(), &body)
|
||||
if err != nil {
|
||||
return capability.MultimodalResponse{}, err
|
||||
}
|
||||
request.Header.Set("Content-Type", writer.FormDataContentType())
|
||||
client := c.media
|
||||
if client == nil {
|
||||
client = c.client
|
||||
}
|
||||
response, err := client.Do(request)
|
||||
if err != nil {
|
||||
return capability.MultimodalResponse{}, err
|
||||
}
|
||||
defer response.Body.Close()
|
||||
if response.StatusCode != http.StatusOK {
|
||||
return capability.MultimodalResponse{}, fmt.Errorf("image generation status %d", response.StatusCode)
|
||||
}
|
||||
var result capability.MultimodalResponse
|
||||
if err := json.NewDecoder(io.LimitReader(response.Body, 32<<20)).Decode(&result); err != nil {
|
||||
return capability.MultimodalResponse{}, err
|
||||
}
|
||||
if strings.TrimSpace(result.DataBase64) == "" {
|
||||
return capability.MultimodalResponse{}, fmt.Errorf("image generation returned empty payload")
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func shouldRequestBroaderInterpretation(result interactiondomain.SemanticResult) bool {
|
||||
return shouldRequestSemanticInterpretation(result)
|
||||
}
|
||||
|
||||
@@ -5,9 +5,11 @@ import (
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.cin.su/m/Assistant/contracts/capability"
|
||||
"git.cin.su/m/Assistant/contracts/cognitive"
|
||||
"git.cin.su/m/Assistant/contracts/identity"
|
||||
"git.cin.su/m/Assistant/contracts/interaction"
|
||||
@@ -40,6 +42,30 @@ func TestReviewResponseUsesCognitiveGatewayAndReturnsOnlyDraftAnswer(t *testing.
|
||||
}
|
||||
}
|
||||
|
||||
func TestGenerateImageUsesDedicatedCognitiveMediaRoute(t *testing.T) {
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method != http.MethodPost || r.URL.Path != "/v1/multimodal/generate-image" {
|
||||
t.Fatalf("unexpected image route: %s %s", r.Method, r.URL.Path)
|
||||
}
|
||||
if err := r.ParseMultipartForm(1 << 20); err != nil {
|
||||
t.Fatalf("parse image request: %v", err)
|
||||
}
|
||||
if !strings.Contains(r.FormValue("prompt"), "собаку") {
|
||||
t.Fatalf("prompt was not preserved: %q", r.FormValue("prompt"))
|
||||
}
|
||||
_ = json.NewEncoder(w).Encode(capability.MultimodalResponse{Operation: capability.MultimodalGenerateImage, ContentType: "image/png", DataBase64: "cG5n"})
|
||||
}))
|
||||
defer server.Close()
|
||||
client, err := newCognitiveInterpretationClient(server.URL, server.Client())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
image, err := client.GenerateImage(context.Background(), "нарисуй собаку", "")
|
||||
if err != nil || image.DataBase64 != "cG5n" || image.ContentType != "image/png" {
|
||||
t.Fatalf("image=%+v err=%v", image, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestReviewPreservesResearchEvidence(t *testing.T) {
|
||||
draft := "Нашёл вариант. Источник: Aviasales (https://www.aviasales.ru/search)"
|
||||
if reviewPreservesResearchEvidence("Нашёл вариант на Авиасейлс.", draft) {
|
||||
|
||||
@@ -68,6 +68,10 @@ func TestHouseholdBearDrawingUsesCreativeLocalRoute(t *testing.T) {
|
||||
if decomposition.Intent != interactiondomain.TaskIntentCreate || decomposition.Object != "IMAGE" || decomposition.Target != "бурого медведя" || decomposition.NeedsResearch {
|
||||
t.Fatalf("creative request was not split into local execution atoms: %+v", decomposition)
|
||||
}
|
||||
routes := buildChatRoutePlan(interactiondomain.SemanticResult{NeedsCognition: candidate.NeedsCognition, CapabilityHints: candidate.CapabilityHints})
|
||||
if !hasChatRoute(routes, chatRouteMedia) {
|
||||
t.Fatalf("creative image request did not select media route: candidate=%+v routes=%+v", candidate, routes)
|
||||
}
|
||||
for _, hint := range candidate.CapabilityHints {
|
||||
if capability.CapabilityKind(hint) == capability.CapabilityKindWebResearch {
|
||||
t.Fatalf("bear drawing selected web research: %+v", candidate)
|
||||
|
||||
@@ -819,6 +819,23 @@ func main() {
|
||||
var delegationID work.DelegationID
|
||||
var delegationState work.DelegationState
|
||||
var reminder *continuity.ProspectiveItem
|
||||
var generatedImage capability.MultimodalResponse
|
||||
mediaHandled := false
|
||||
if hasChatRoute(routePlan, chatRouteMedia) && semantic.Interpretation.Clarification != interaction.ClarificationRequired {
|
||||
imageCtx, cancelImage := context.WithTimeout(r.Context(), 15*time.Minute)
|
||||
generatedImage, err = cognitiveClient.GenerateImage(imageCtx, effectiveTaskMessage, "")
|
||||
cancelImage()
|
||||
if err == nil && strings.TrimSpace(generatedImage.DataBase64) != "" {
|
||||
planningReply = "Готово."
|
||||
semantic.Reply = planningReply
|
||||
needsCognition = false
|
||||
mediaHandled = true
|
||||
} else {
|
||||
planningReply = "Сейчас не удалось создать изображение."
|
||||
semantic.Reply = planningReply
|
||||
needsCognition = false
|
||||
}
|
||||
}
|
||||
if helpPath.Kind == work.HelpPathDirectCapability && len(helpPath.SuggestedCapabilityKinds) > 0 {
|
||||
capabilityKind = helpPath.SuggestedCapabilityKinds[0]
|
||||
started := time.Now()
|
||||
@@ -903,7 +920,7 @@ func main() {
|
||||
ConfidenceMillis: semantic.Interpretation.ConfidenceMillis,
|
||||
})
|
||||
cancelReview()
|
||||
stableSemanticReply := identityReply != "" || creativeReply != "" || (creativeFallbackReply != "" && !cognitiveAvailable) || semantic.InformationRequestKind != "" || locationReply != "" || deterministicSpeechAct == interaction.SpeechActConversational || deterministicSpeechAct == interaction.SpeechActConfirmation || deterministicSpeechAct == interaction.SpeechActRejection
|
||||
stableSemanticReply := mediaHandled || identityReply != "" || creativeReply != "" || (creativeFallbackReply != "" && !cognitiveAvailable) || semantic.InformationRequestKind != "" || locationReply != "" || deterministicSpeechAct == interaction.SpeechActConversational || deterministicSpeechAct == interaction.SpeechActConfirmation || deterministicSpeechAct == interaction.SpeechActRejection
|
||||
if reviewErr != nil {
|
||||
log.Printf("cognitive response review unavailable: %v", reviewErr)
|
||||
} else if !stableSemanticReply && (!researchBacked || reviewPreservesResearchEvidence(reviewedReply, planningReply)) {
|
||||
@@ -912,7 +929,7 @@ func main() {
|
||||
log.Printf("cognitive response review proposed stable semantic rewrite; preserving semantic reply")
|
||||
}
|
||||
}
|
||||
if identityReply == "" && creativeReply == "" && creativeFallbackReply == "" {
|
||||
if !mediaHandled && identityReply == "" && creativeReply == "" && creativeFallbackReply == "" {
|
||||
stableAct := interactiondomain.ClassifySpeechAct(utterance.Content, utterance.CorrectsUtteranceID)
|
||||
travelClarification := ""
|
||||
if !creativeRequest {
|
||||
@@ -967,6 +984,8 @@ func main() {
|
||||
DelegationID: string(delegationID),
|
||||
DelegationState: string(delegationState),
|
||||
Reminder: reminder,
|
||||
GeneratedImageContentType: generatedImage.ContentType,
|
||||
GeneratedImageBase64: generatedImage.DataBase64,
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.Header().Set("Cache-Control", "no-store")
|
||||
|
||||
Reference in New Issue
Block a user