Compare commits
2
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
43556e1603 | ||
|
|
dd00dabf32 |
@@ -0,0 +1,18 @@
|
|||||||
|
package multimodal
|
||||||
|
|
||||||
|
import (
|
||||||
|
"crypto/sha256"
|
||||||
|
"encoding/hex"
|
||||||
|
"git.cin.su/m/Assistant/contracts/artifact"
|
||||||
|
"git.cin.su/m/Assistant/contracts/capability"
|
||||||
|
"git.cin.su/m/Assistant/contracts/interaction"
|
||||||
|
)
|
||||||
|
|
||||||
|
func NormalizeIngress(id string, modality interaction.IngressModality, sourceRef, deviceRef, locationRef, principalID, contentType string, data []byte, response capability.MultimodalResponse, at int64) (interaction.IngressEnvelope, artifact.MediaArtifact) {
|
||||||
|
hash := sha256.Sum256(data)
|
||||||
|
artifactValue := artifact.MediaArtifact{ID: artifact.ArtifactID("artifact:media:" + hex.EncodeToString(hash[:8])), ContentType: contentType, SizeBytes: int64(len(data)), ContentDigest: hex.EncodeToString(hash[:]), SourceRef: sourceRef, RecordedAtUnixMillis: at}
|
||||||
|
if len(data) == 0 {
|
||||||
|
return interaction.IngressEnvelope{ID: id, Modality: modality, SourceRef: sourceRef, DeviceRef: deviceRef, LocationRef: locationRef, PrincipalID: principalID, Degraded: true, RecordedAtUnixMillis: at}, artifact.MediaArtifact{}
|
||||||
|
}
|
||||||
|
return interaction.IngressEnvelope{ID: id, Modality: modality, SourceRef: sourceRef, DeviceRef: deviceRef, LocationRef: locationRef, PrincipalID: principalID, ArtifactRef: string(artifactValue.ID), ObservationRef: response.EvidenceRef, ProviderRef: response.Provider, ModelRef: response.Model, ModelVersion: response.ModelVersion, ConfidenceMillis: response.ConfidenceMillis, RecordedAtUnixMillis: at}, artifactValue
|
||||||
|
}
|
||||||
@@ -0,0 +1,22 @@
|
|||||||
|
package multimodal_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"git.cin.su/m/Assistant/adapters/multimodal"
|
||||||
|
"git.cin.su/m/Assistant/contracts/capability"
|
||||||
|
"git.cin.su/m/Assistant/contracts/interaction"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestIngressNormalizesMediaArtifactAndObservationProvenance(t *testing.T) {
|
||||||
|
e, a := multimodal.NormalizeIngress("ingress-1", interaction.IngressImage, "camera:street", "device:cam", "world:street", "principal", "image/png", []byte("image"), capability.MultimodalResponse{Provider: "local", Model: "vision", ModelVersion: "1", EvidenceRef: "observation-1", ConfidenceMillis: 800}, 1)
|
||||||
|
if !e.Valid() || !a.Valid() || e.ArtifactRef == "" || e.ObservationRef != "observation-1" || e.DeviceRef == "" || e.LocationRef == "" {
|
||||||
|
t.Fatalf("envelope=%+v artifact=%+v", e, a)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestUnavailableIngressDegradesWithoutFabricatingObservation(t *testing.T) {
|
||||||
|
e, a := multimodal.NormalizeIngress("ingress-1", interaction.IngressAudio, "mic", "device:mic", "", "principal", "audio/wav", nil, capability.MultimodalResponse{}, 1)
|
||||||
|
if !e.Degraded || e.ObservationRef != "" || a.Valid() {
|
||||||
|
t.Fatalf("envelope=%+v artifact=%+v", e, a)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -533,7 +533,7 @@ func main() {
|
|||||||
log.Printf("interaction principal history loaded: %d utterances", len(recentUserHistory))
|
log.Printf("interaction principal history loaded: %d utterances", len(recentUserHistory))
|
||||||
}
|
}
|
||||||
conversationHistory = append(conversationHistory, recentUserHistory...)
|
conversationHistory = append(conversationHistory, recentUserHistory...)
|
||||||
gen2Context := interactiondomain.BuildGen2Context(conversationHistory)
|
gen2Context := interactiondomain.BuildScopedGen2Context(conversationHistory, string(session.PrincipalID))
|
||||||
if !creativeUserRequest && locationReply == "" {
|
if !creativeUserRequest && locationReply == "" {
|
||||||
if continuation := interactiondomain.ContinueLocalSearch(effectiveTaskMessage, conversationHistory); continuation != "" {
|
if continuation := interactiondomain.ContinueLocalSearch(effectiveTaskMessage, conversationHistory); continuation != "" {
|
||||||
continuationMessage = continuation
|
continuationMessage = continuation
|
||||||
|
|||||||
@@ -5,6 +5,18 @@ import "git.cin.su/m/Assistant/platform/ids"
|
|||||||
import "errors"
|
import "errors"
|
||||||
|
|
||||||
type ArtifactID ids.ID
|
type ArtifactID ids.ID
|
||||||
|
type MediaArtifact struct {
|
||||||
|
ID ArtifactID
|
||||||
|
ContentType string
|
||||||
|
SizeBytes int64
|
||||||
|
ContentDigest string
|
||||||
|
SourceRef string
|
||||||
|
RecordedAtUnixMillis int64
|
||||||
|
}
|
||||||
|
|
||||||
|
func (a MediaArtifact) Valid() bool {
|
||||||
|
return a.ID != "" && a.ContentType != "" && a.SizeBytes > 0 && a.ContentDigest != "" && a.SourceRef != "" && a.RecordedAtUnixMillis > 0
|
||||||
|
}
|
||||||
|
|
||||||
type AgentArtifactAttestation struct {
|
type AgentArtifactAttestation struct {
|
||||||
ArtifactID ArtifactID
|
ArtifactID ArtifactID
|
||||||
|
|||||||
@@ -15,11 +15,15 @@ const (
|
|||||||
)
|
)
|
||||||
|
|
||||||
type MultimodalResponse struct {
|
type MultimodalResponse struct {
|
||||||
Operation MultimodalOperation `json:"operation"`
|
Operation MultimodalOperation `json:"operation"`
|
||||||
Text string `json:"text"`
|
Text string `json:"text"`
|
||||||
Provider string `json:"provider,omitempty"`
|
Provider string `json:"provider,omitempty"`
|
||||||
ObservedAt int64 `json:"observed_at_unix_millis"`
|
ObservedAt int64 `json:"observed_at_unix_millis"`
|
||||||
ConfidenceNote string `json:"confidence_note,omitempty"`
|
ConfidenceNote string `json:"confidence_note,omitempty"`
|
||||||
ContentType string `json:"content_type,omitempty"`
|
ContentType string `json:"content_type,omitempty"`
|
||||||
DataBase64 string `json:"data_base64,omitempty"`
|
DataBase64 string `json:"data_base64,omitempty"`
|
||||||
|
Model string `json:"model,omitempty"`
|
||||||
|
ModelVersion string `json:"model_version,omitempty"`
|
||||||
|
ConfidenceMillis int64 `json:"confidence_millis,omitempty"`
|
||||||
|
EvidenceRef string `json:"evidence_ref,omitempty"`
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -33,6 +33,36 @@ type SpeakerAttribution struct {
|
|||||||
Authenticated bool
|
Authenticated bool
|
||||||
ObservationRef string
|
ObservationRef string
|
||||||
}
|
}
|
||||||
|
type IngressModality string
|
||||||
|
|
||||||
|
const (
|
||||||
|
IngressText IngressModality = "TEXT"
|
||||||
|
IngressAudio IngressModality = "AUDIO"
|
||||||
|
IngressImage IngressModality = "IMAGE"
|
||||||
|
IngressVideo IngressModality = "VIDEO"
|
||||||
|
IngressDevice IngressModality = "DEVICE"
|
||||||
|
)
|
||||||
|
|
||||||
|
type IngressEnvelope struct {
|
||||||
|
ID string
|
||||||
|
Modality IngressModality
|
||||||
|
SourceRef string
|
||||||
|
DeviceRef string
|
||||||
|
LocationRef string
|
||||||
|
PrincipalID string
|
||||||
|
ArtifactRef string
|
||||||
|
ObservationRef string
|
||||||
|
ProviderRef string
|
||||||
|
ModelRef string
|
||||||
|
ModelVersion string
|
||||||
|
ConfidenceMillis int64
|
||||||
|
Degraded bool
|
||||||
|
RecordedAtUnixMillis int64
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e IngressEnvelope) Valid() bool {
|
||||||
|
return e.ID != "" && e.Modality != "" && e.SourceRef != "" && e.ConfidenceMillis >= 0 && e.ConfidenceMillis <= 1000 && e.RecordedAtUnixMillis > 0 && (!e.Degraded || e.ObservationRef == "")
|
||||||
|
}
|
||||||
|
|
||||||
const (
|
const (
|
||||||
ChannelWebChat Channel = "WEB_CHAT"
|
ChannelWebChat Channel = "WEB_CHAT"
|
||||||
|
|||||||
@@ -1 +1 @@
|
|||||||
{"task_file":"dev/tasks/GEN2-024-multi-person-realms-identity-boundaries-and-contextual-source-trust.json","task_id":"GEN2-024","task_revision":1}
|
{"task_file":"dev/tasks/GEN2-026-migrate-household-behavior-to-gen2-context-memory-skills-with-local-degr.json","task_id":"GEN2-026","task_revision":1}
|
||||||
|
|||||||
@@ -11,6 +11,23 @@ type ContextProjection struct {
|
|||||||
// BuildGen2Context is a read-side projection over canonical utterance history.
|
// BuildGen2Context is a read-side projection over canonical utterance history.
|
||||||
// PersonalMemory remains a compatibility fallback and is never written here.
|
// PersonalMemory remains a compatibility fallback and is never written here.
|
||||||
func BuildGen2Context(history []interaction.HumanUtterance) ContextProjection {
|
func BuildGen2Context(history []interaction.HumanUtterance) ContextProjection {
|
||||||
|
return BuildScopedGen2Context(history, "")
|
||||||
|
}
|
||||||
|
|
||||||
|
// BuildScopedGen2Context is the Gen2 retrieval boundary for household
|
||||||
|
// context. The caller supplies already-authorized history; this defensive
|
||||||
|
// filter prevents accidental cross-principal projection when a mixed source
|
||||||
|
// is passed during degraded operation.
|
||||||
|
func BuildScopedGen2Context(history []interaction.HumanUtterance, principalID string) ContextProjection {
|
||||||
|
if principalID != "" {
|
||||||
|
scoped := make([]interaction.HumanUtterance, 0, len(history))
|
||||||
|
for _, item := range history {
|
||||||
|
if item.PrincipalID == "" || string(item.PrincipalID) == principalID {
|
||||||
|
scoped = append(scoped, item)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
history = scoped
|
||||||
|
}
|
||||||
text := CompilePersonalMemory(history).ContextText()
|
text := CompilePersonalMemory(history).ContextText()
|
||||||
if text != "" {
|
if text != "" {
|
||||||
return ContextProjection{Text: text, Source: "retrieval:interaction-utterances"}
|
return ContextProjection{Text: text, Source: "retrieval:interaction-utterances"}
|
||||||
|
|||||||
@@ -3,6 +3,7 @@ package domain_test
|
|||||||
import (
|
import (
|
||||||
"git.cin.su/m/Assistant/contracts/interaction"
|
"git.cin.su/m/Assistant/contracts/interaction"
|
||||||
"git.cin.su/m/Assistant/modules/interaction/domain"
|
"git.cin.su/m/Assistant/modules/interaction/domain"
|
||||||
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -16,3 +17,13 @@ func TestGen2ContextUsesRetrievalProjectionAndKeepsV1Fallback(t *testing.T) {
|
|||||||
t.Fatalf("fallback missing: %#v", empty)
|
t.Fatalf("fallback missing: %#v", empty)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestGen2ContextDoesNotProjectAnotherPrincipal(t *testing.T) {
|
||||||
|
got := domain.BuildScopedGen2Context([]interaction.HumanUtterance{
|
||||||
|
{ID: "mine", PrincipalID: "principal-1", Content: "Я сейчас в Москве", ReceivedAtUnixMillis: 2},
|
||||||
|
{ID: "other", PrincipalID: "principal-2", Content: "Я сейчас в Казани", ReceivedAtUnixMillis: 3},
|
||||||
|
}, "principal-1")
|
||||||
|
if got.Text == "" || strings.Contains(got.Text, "Казани") || !strings.Contains(got.Text, "Москве") {
|
||||||
|
t.Fatalf("scoped context=%#v", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user