280 lines
10 KiB
Go
280 lines
10 KiB
Go
package main
|
||
|
||
import (
|
||
"bytes"
|
||
"context"
|
||
"encoding/json"
|
||
"io"
|
||
"net/http"
|
||
"net/http/httptest"
|
||
"os"
|
||
"testing"
|
||
"time"
|
||
|
||
"git.cin.su/m/Assistant/adapters/models"
|
||
"git.cin.su/m/Assistant/contracts/authority"
|
||
"git.cin.su/m/Assistant/contracts/capability"
|
||
"git.cin.su/m/Assistant/contracts/cognitive"
|
||
"git.cin.su/m/Assistant/contracts/identity"
|
||
"git.cin.su/m/Assistant/contracts/interaction"
|
||
"git.cin.su/m/Assistant/platform/ids"
|
||
"git.cin.su/m/Assistant/contracts/situation"
|
||
interactiondomain "git.cin.su/m/Assistant/modules/interaction/domain"
|
||
)
|
||
|
||
func TestLiveInterpretationTrace(t *testing.T) {
|
||
if os.Getenv("ASSISTANT_LIVE_SEMANTIC_TRACE") != "1" {
|
||
t.Skip("set ASSISTANT_LIVE_SEMANTIC_TRACE=1 for live semantic/gateway/raw trace")
|
||
}
|
||
ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second)
|
||
defer cancel()
|
||
|
||
rawBase := envOr("LOCAL_CPU_PROVIDER_URL", "http://docker.cin.su:18082/v1")
|
||
rawModelID, err := firstRawModel(ctx, rawBase)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
cpuAdapter, err := models.NewOpenAICompatibleLocalAdapter(rawBase, nil)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
discovered, probe := cpuAdapter.Discover(ctx)
|
||
if probe.Health != capability.HealthAvailable || len(discovered) == 0 {
|
||
t.Fatalf("cpu discovery unavailable: probe=%+v models=%+v", probe, discovered)
|
||
}
|
||
selected := capability.ResolvedCognitiveBinding{
|
||
Binding: capability.CognitiveBinding{
|
||
ID: "trace-binding",
|
||
Revision: 1,
|
||
CapabilityID: "creator-advisory",
|
||
OwnerPrincipalID: "domovoy-creator-principal",
|
||
DataPolicy: capability.DataPolicyLocalOnly,
|
||
},
|
||
Provider: capability.ProviderInstance{
|
||
ID: "local-cpu-primary",
|
||
Revision: 1,
|
||
Kind: "openai-compatible-chat-v1",
|
||
EndpointRef: "local-cpu-primary-v1",
|
||
},
|
||
Model: capability.ModelDescriptor{
|
||
ID: capability.ModelDescriptorID("trace-model"),
|
||
Revision: 1,
|
||
ProviderModelID: discovered[0].ProviderModelID,
|
||
},
|
||
}
|
||
gateway := cognitiveGateway{
|
||
resolver: traceResolver{selected: selected},
|
||
providers: map[capability.ProviderKind][]providerRegistration{
|
||
"openai-compatible-chat-v1": {{
|
||
ProviderInstanceID: "local-cpu-primary",
|
||
Kind: "openai-compatible-chat-v1",
|
||
EndpointRef: "local-cpu-primary-v1",
|
||
Provider: cpuAdapter,
|
||
}},
|
||
},
|
||
now: time.Now,
|
||
allowCanary: true,
|
||
}
|
||
for _, message := range []string{
|
||
"расскажи как это работает",
|
||
"Расскажи, как это работает?",
|
||
"можешь объяснить почему база растёт",
|
||
"почему postgres использует столько памяти",
|
||
"разберись почему сервис упал",
|
||
"посмотри что произошло",
|
||
"исправь ошибку если найдёшь",
|
||
"разберись почему падает и исправь если причина очевидна",
|
||
"нет не UPO а UPO2",
|
||
"нет не UPO а UPO2 и обнови конфиг",
|
||
"postgres сейчас работает",
|
||
"основная база называется UPO",
|
||
"привет",
|
||
"спасибо",
|
||
"ты кто",
|
||
"ты кто?",
|
||
"как тебя зовут",
|
||
"кто ты вообще",
|
||
"что ты думаешь",
|
||
"расскажи подробнее",
|
||
} {
|
||
conversation := interaction.Conversation{ID: "trace-conversation", ActorID: "trace-actor", PrincipalID: "domovoy-creator-principal", Channel: interaction.ChannelWebChat, State: interaction.ConversationOpen}
|
||
utterance, _, _, err := interactiondomain.NewHumanUtterance(conversation, interaction.ChatMessageRequest{Message: message}, time.Now())
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
semantic, err := interactiondomain.InterpretUtterance(utterance, []situation.ProjectDescriptor{{ID: "assistant", DisplayName: "Assistant"}}, nil, time.Now())
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
rawResponse, err := rawCompletion(ctx, rawBase, rawModelID, message)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
request := cognitive.GatewayInterpretationRequest{
|
||
Version: cognitive.CreatorAdvisoryV1,
|
||
RequestID: "trace-request",
|
||
ActorID: identity.ActorID("trace-actor"),
|
||
PrincipalID: identity.PrincipalID("domovoy-creator-principal"),
|
||
ScopeID: authority.ScopeID("domovoy-global-scope"),
|
||
Message: message,
|
||
Locale: "ru",
|
||
MaxOutputBytes: 4096,
|
||
DataPolicy: capability.DataPolicyLocalOnly,
|
||
DeterministicSpeechAct: string(semantic.Interpretation.SpeechAct),
|
||
DeterministicSecondary: string(semantic.Interpretation.SecondarySpeechAct),
|
||
DeterministicConfidence: semantic.Interpretation.ConfidenceMillis,
|
||
DeterministicApplicability: cognitive.InterpretationApplicability(semantic.Interpretation.Applicability),
|
||
RequestedOutcome: semantic.Interpretation.RequestedOutcome,
|
||
}
|
||
payload, _ := json.Marshal(request)
|
||
recorder := httptest.NewRecorder()
|
||
httpRequest := httptest.NewRequest(http.MethodPost, "/v1/interpretations", bytes.NewReader(payload))
|
||
gateway.interpretationHandler(recorder, httpRequest.WithContext(ctx))
|
||
if recorder.Code != http.StatusOK {
|
||
t.Fatalf("message=%q gateway status=%d body=%s", message, recorder.Code, recorder.Body.String())
|
||
}
|
||
var gatewayResponse cognitive.GatewayInterpretationResponse
|
||
if err := json.Unmarshal(recorder.Body.Bytes(), &gatewayResponse); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if gatewayResponse.FinishReason != cognitive.FinishCompleted {
|
||
t.Fatalf("message=%q gateway finish=%s body=%s", message, gatewayResponse.FinishReason, recorder.Body.String())
|
||
}
|
||
if rawResponse == "" {
|
||
t.Fatalf("message=%q raw response is empty", message)
|
||
}
|
||
cognitiveCandidate, err := cognitiveCandidateFromGateway(utterance.ID, gatewayResponse.Candidate, time.Now())
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
selected := interactiondomain.ApplyCognitiveCandidate(semantic, utterance.Content, cognitiveCandidate, nil)
|
||
cognitionUsed := selected.Interpretation.Source == string(interaction.CandidateSourceCognitive)
|
||
clarificationReason := string(selected.Interpretation.Clarification)
|
||
t.Logf("TRACE message=%q", message)
|
||
deterministic := semantic.Candidates[0]
|
||
t.Logf(" deterministic candidate: %+v", deterministic)
|
||
t.Logf(" raw response=%q", truncate(rawResponse, 180))
|
||
t.Logf(" cognitive candidate: %+v", cognitiveCandidate)
|
||
t.Logf(" selected interpretation: act=%s secondary=%s selection_source=%s needs_cognition=%t",
|
||
selected.Interpretation.SpeechAct,
|
||
selected.Interpretation.SecondarySpeechAct,
|
||
selected.Interpretation.Source,
|
||
selected.Interpretation.NeedsCognition,
|
||
)
|
||
t.Logf(" cognition used: %t", cognitionUsed)
|
||
t.Logf(" clarification reason: %s", clarificationReason)
|
||
t.Logf(" reply=%q", selected.Reply)
|
||
}
|
||
}
|
||
|
||
type traceResolver struct {
|
||
selected capability.ResolvedCognitiveBinding
|
||
}
|
||
|
||
func (r traceResolver) Resolve(context.Context, capability.CognitiveRequirement) (capability.ResolvedCognitiveBinding, error) {
|
||
return r.selected, nil
|
||
}
|
||
|
||
func (r traceResolver) Observe(context.Context, capability.CognitiveUsageObservation) error {
|
||
return nil
|
||
}
|
||
|
||
func (r traceResolver) RegisterDiscovery(context.Context, capability.CognitiveDiscoverySnapshot) (capability.CognitiveDiscoveryRegistration, error) {
|
||
return capability.CognitiveDiscoveryRegistration{}, nil
|
||
}
|
||
|
||
func firstRawModel(ctx context.Context, base string) (string, error) {
|
||
request, err := http.NewRequestWithContext(ctx, http.MethodGet, base+"/models", nil)
|
||
if err != nil {
|
||
return "", err
|
||
}
|
||
response, err := http.DefaultClient.Do(request)
|
||
if err != nil {
|
||
return "", err
|
||
}
|
||
defer response.Body.Close()
|
||
var payload struct {
|
||
Data []struct {
|
||
ID string `json:"id"`
|
||
} `json:"data"`
|
||
}
|
||
if err := json.NewDecoder(io.LimitReader(response.Body, 64<<10)).Decode(&payload); err != nil {
|
||
return "", err
|
||
}
|
||
if len(payload.Data) == 0 || payload.Data[0].ID == "" {
|
||
return "", io.EOF
|
||
}
|
||
return payload.Data[0].ID, nil
|
||
}
|
||
|
||
func rawCompletion(ctx context.Context, base, model, message string) (string, error) {
|
||
payload, err := json.Marshal(map[string]any{
|
||
"model": model,
|
||
"max_tokens": 128,
|
||
"messages": []map[string]string{
|
||
{"role": "user", "content": message},
|
||
},
|
||
"stream": false,
|
||
})
|
||
if err != nil {
|
||
return "", err
|
||
}
|
||
request, err := http.NewRequestWithContext(ctx, http.MethodPost, base+"/chat/completions", bytes.NewReader(payload))
|
||
if err != nil {
|
||
return "", err
|
||
}
|
||
request.Header.Set("Content-Type", "application/json")
|
||
response, err := http.DefaultClient.Do(request)
|
||
if err != nil {
|
||
return "", err
|
||
}
|
||
defer response.Body.Close()
|
||
var result struct {
|
||
Choices []struct {
|
||
Message struct {
|
||
Content string `json:"content"`
|
||
} `json:"message"`
|
||
} `json:"choices"`
|
||
}
|
||
if err := json.NewDecoder(io.LimitReader(response.Body, 1<<20)).Decode(&result); err != nil {
|
||
return "", err
|
||
}
|
||
if len(result.Choices) == 0 {
|
||
return "", io.EOF
|
||
}
|
||
return result.Choices[0].Message.Content, nil
|
||
}
|
||
|
||
func truncate(value string, max int) string {
|
||
if len(value) <= max {
|
||
return value
|
||
}
|
||
return value[:max] + "..."
|
||
}
|
||
|
||
func cognitiveCandidateFromGateway(utteranceID interaction.HumanUtteranceID, candidate cognitive.GatewayInterpretationCandidate, now interface{ UnixMilli() int64 }) (interaction.SemanticCandidate, error) {
|
||
id, err := ids.New()
|
||
if err != nil {
|
||
return interaction.SemanticCandidate{}, err
|
||
}
|
||
capabilityHints := make([]string, 0, len(candidate.CapabilityHints))
|
||
for _, hint := range candidate.CapabilityHints {
|
||
capabilityHints = append(capabilityHints, string(hint))
|
||
}
|
||
return interaction.SemanticCandidate{
|
||
ID: interaction.SemanticCandidateID(id),
|
||
UtteranceID: utteranceID,
|
||
Source: interaction.CandidateSourceCognitive,
|
||
SelectionState: interaction.CandidateSelected,
|
||
PrimarySpeechAct: interaction.SpeechAct(candidate.PrimarySpeechAct),
|
||
SecondarySpeechAct: interaction.SpeechAct(candidate.SecondarySpeechAct),
|
||
ConfidenceMillis: candidate.ConfidenceMillis,
|
||
Applicability: interaction.SemanticApplicability(candidate.Applicability),
|
||
RequestedOutcome: candidate.RequestedOutcome,
|
||
Clarification: interaction.ClarificationState(candidate.Clarification),
|
||
NeedsCognition: candidate.NeedsCognition,
|
||
SelectionReason: candidate.SelectionReason,
|
||
CapabilityHints: capabilityHints,
|
||
RecordedAtUnixMillis: now.UnixMilli(),
|
||
}, nil
|
||
} |