diff --git a/adapters/multimodal/ingress.go b/adapters/multimodal/ingress.go new file mode 100644 index 0000000..79cb7b0 --- /dev/null +++ b/adapters/multimodal/ingress.go @@ -0,0 +1,18 @@ +package multimodal + +import ( + "crypto/sha256" + "encoding/hex" + "git.cin.su/m/Assistant/contracts/artifact" + "git.cin.su/m/Assistant/contracts/capability" + "git.cin.su/m/Assistant/contracts/interaction" +) + +func NormalizeIngress(id string, modality interaction.IngressModality, sourceRef, deviceRef, locationRef, principalID, contentType string, data []byte, response capability.MultimodalResponse, at int64) (interaction.IngressEnvelope, artifact.MediaArtifact) { + hash := sha256.Sum256(data) + artifactValue := artifact.MediaArtifact{ID: artifact.ArtifactID("artifact:media:" + hex.EncodeToString(hash[:8])), ContentType: contentType, SizeBytes: int64(len(data)), ContentDigest: hex.EncodeToString(hash[:]), SourceRef: sourceRef, RecordedAtUnixMillis: at} + if len(data) == 0 { + return interaction.IngressEnvelope{ID: id, Modality: modality, SourceRef: sourceRef, DeviceRef: deviceRef, LocationRef: locationRef, PrincipalID: principalID, Degraded: true, RecordedAtUnixMillis: at}, artifact.MediaArtifact{} + } + return interaction.IngressEnvelope{ID: id, Modality: modality, SourceRef: sourceRef, DeviceRef: deviceRef, LocationRef: locationRef, PrincipalID: principalID, ArtifactRef: string(artifactValue.ID), ObservationRef: response.EvidenceRef, ProviderRef: response.Provider, ModelRef: response.Model, ModelVersion: response.ModelVersion, ConfidenceMillis: response.ConfidenceMillis, RecordedAtUnixMillis: at}, artifactValue +} diff --git a/adapters/multimodal/ingress_test.go b/adapters/multimodal/ingress_test.go new file mode 100644 index 0000000..74978cf --- /dev/null +++ b/adapters/multimodal/ingress_test.go @@ -0,0 +1,22 @@ +package multimodal_test + +import ( + "git.cin.su/m/Assistant/adapters/multimodal" + "git.cin.su/m/Assistant/contracts/capability" + "git.cin.su/m/Assistant/contracts/interaction" + "testing" +) + +func TestIngressNormalizesMediaArtifactAndObservationProvenance(t *testing.T) { + e, a := multimodal.NormalizeIngress("ingress-1", interaction.IngressImage, "camera:street", "device:cam", "world:street", "principal", "image/png", []byte("image"), capability.MultimodalResponse{Provider: "local", Model: "vision", ModelVersion: "1", EvidenceRef: "observation-1", ConfidenceMillis: 800}, 1) + if !e.Valid() || !a.Valid() || e.ArtifactRef == "" || e.ObservationRef != "observation-1" || e.DeviceRef == "" || e.LocationRef == "" { + t.Fatalf("envelope=%+v artifact=%+v", e, a) + } +} + +func TestUnavailableIngressDegradesWithoutFabricatingObservation(t *testing.T) { + e, a := multimodal.NormalizeIngress("ingress-1", interaction.IngressAudio, "mic", "device:mic", "", "principal", "audio/wav", nil, capability.MultimodalResponse{}, 1) + if !e.Degraded || e.ObservationRef != "" || a.Valid() { + t.Fatalf("envelope=%+v artifact=%+v", e, a) + } +} diff --git a/contracts/artifact/types.go b/contracts/artifact/types.go index 825216f..a742bdb 100644 --- a/contracts/artifact/types.go +++ b/contracts/artifact/types.go @@ -5,6 +5,18 @@ import "git.cin.su/m/Assistant/platform/ids" import "errors" type ArtifactID ids.ID +type MediaArtifact struct { + ID ArtifactID + ContentType string + SizeBytes int64 + ContentDigest string + SourceRef string + RecordedAtUnixMillis int64 +} + +func (a MediaArtifact) Valid() bool { + return a.ID != "" && a.ContentType != "" && a.SizeBytes > 0 && a.ContentDigest != "" && a.SourceRef != "" && a.RecordedAtUnixMillis > 0 +} type AgentArtifactAttestation struct { ArtifactID ArtifactID diff --git a/contracts/capability/multimodal.go b/contracts/capability/multimodal.go index c12946e..834549a 100644 --- a/contracts/capability/multimodal.go +++ b/contracts/capability/multimodal.go @@ -15,11 +15,15 @@ const ( ) type MultimodalResponse struct { - Operation MultimodalOperation `json:"operation"` - Text string `json:"text"` - Provider string `json:"provider,omitempty"` - ObservedAt int64 `json:"observed_at_unix_millis"` - ConfidenceNote string `json:"confidence_note,omitempty"` - ContentType string `json:"content_type,omitempty"` - DataBase64 string `json:"data_base64,omitempty"` + Operation MultimodalOperation `json:"operation"` + Text string `json:"text"` + Provider string `json:"provider,omitempty"` + ObservedAt int64 `json:"observed_at_unix_millis"` + ConfidenceNote string `json:"confidence_note,omitempty"` + ContentType string `json:"content_type,omitempty"` + DataBase64 string `json:"data_base64,omitempty"` + Model string `json:"model,omitempty"` + ModelVersion string `json:"model_version,omitempty"` + ConfidenceMillis int64 `json:"confidence_millis,omitempty"` + EvidenceRef string `json:"evidence_ref,omitempty"` } diff --git a/contracts/interaction/types.go b/contracts/interaction/types.go index e743dc6..a086c58 100644 --- a/contracts/interaction/types.go +++ b/contracts/interaction/types.go @@ -33,6 +33,36 @@ type SpeakerAttribution struct { Authenticated bool ObservationRef string } +type IngressModality string + +const ( + IngressText IngressModality = "TEXT" + IngressAudio IngressModality = "AUDIO" + IngressImage IngressModality = "IMAGE" + IngressVideo IngressModality = "VIDEO" + IngressDevice IngressModality = "DEVICE" +) + +type IngressEnvelope struct { + ID string + Modality IngressModality + SourceRef string + DeviceRef string + LocationRef string + PrincipalID string + ArtifactRef string + ObservationRef string + ProviderRef string + ModelRef string + ModelVersion string + ConfidenceMillis int64 + Degraded bool + RecordedAtUnixMillis int64 +} + +func (e IngressEnvelope) Valid() bool { + return e.ID != "" && e.Modality != "" && e.SourceRef != "" && e.ConfidenceMillis >= 0 && e.ConfidenceMillis <= 1000 && e.RecordedAtUnixMillis > 0 && (!e.Degraded || e.ObservationRef == "") +} const ( ChannelWebChat Channel = "WEB_CHAT" diff --git a/dev/ACTIVE_TASK.json b/dev/ACTIVE_TASK.json index 919a88f..461db90 100644 --- a/dev/ACTIVE_TASK.json +++ b/dev/ACTIVE_TASK.json @@ -1 +1 @@ -{"task_file":"dev/tasks/GEN2-024-multi-person-realms-identity-boundaries-and-contextual-source-trust.json","task_id":"GEN2-024","task_revision":1} +{"task_file":"dev/tasks/GEN2-025-multimodal-and-ambient-ingress-readiness-without-implementing-the-whole-.json","task_id":"GEN2-025","task_revision":1}