171 lines
12 KiB
Go
171 lines
12 KiB
Go
package main
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"testing"
|
|
"time"
|
|
|
|
"git.cin.su/m/Assistant/contracts/authority"
|
|
"git.cin.su/m/Assistant/contracts/capability"
|
|
"git.cin.su/m/Assistant/contracts/cognitive"
|
|
"git.cin.su/m/Assistant/contracts/identity"
|
|
)
|
|
|
|
type candidateRegistryStub struct {
|
|
registryStub
|
|
selected capability.ResolvedCognitiveBinding
|
|
}
|
|
|
|
func (s *candidateRegistryStub) ResolveCandidate(context.Context, capability.CapabilityBindingID, int64) (capability.ResolvedCognitiveBinding, error) {
|
|
return s.selected, nil
|
|
}
|
|
|
|
func TestTaskEvaluationHandlerInvokesEveryBoundedCase(t *testing.T) {
|
|
selected := gatewaySelection()
|
|
registry := &candidateRegistryStub{selected: selected}
|
|
provider := &stubAdvisoryProvider{response: cognitive.AdvisoryResponse{Version: cognitive.CreatorAdvisoryV1, FinishReason: cognitive.FinishCompleted, AdvisoryText: "observed response"}}
|
|
gateway := cognitiveGateway{
|
|
resolver: registry,
|
|
providers: map[capability.ProviderKind][]providerRegistration{
|
|
selected.Provider.Kind: {{ProviderInstanceID: selected.Provider.ID, Kind: selected.Provider.Kind, EndpointRef: selected.Provider.EndpointRef, CredentialRef: selected.Provider.CredentialRef, Provider: provider}},
|
|
},
|
|
now: func() time.Time { return time.UnixMilli(1000) },
|
|
}
|
|
profile := capability.TaskEvaluationProfile{ID: "profile-general", Revision: 1, CapabilityID: selected.Binding.CapabilityID, Stage: capability.TaskEvaluationGeneral, BenchmarkVersion: "general-v1", RequiredCaseIDs: []string{"case-a", "case-b"}, QualityWeightMillis: 600, ReliabilityWeightMillis: 300, LatencyWeightMillis: 100}
|
|
request := capability.TaskEvaluationExecutionRequest{Profile: profile, CapabilityID: selected.Binding.CapabilityID, BindingID: selected.Binding.ID, BindingRevision: selected.Binding.Revision, ActorID: identity.ActorID("actor-1"), PrincipalID: selected.Binding.OwnerPrincipalID, ScopeID: authority.ScopeID("scope-1"), MaxOutputBytes: 512, EgressAcknowledged: true, Cases: []capability.TaskEvaluationCase{{CaseID: "case-a", Prompt: "one", EvidenceArtifactRef: "artifact:a"}, {CaseID: "case-b", Prompt: "two", EvidenceArtifactRef: "artifact:b"}}}
|
|
payload, _ := json.Marshal(request)
|
|
recorder := httptest.NewRecorder()
|
|
gateway.taskEvaluationHandler(recorder, httptest.NewRequest(http.MethodPost, "/v1/task-evaluations/execute", bytes.NewReader(payload)))
|
|
if recorder.Code != http.StatusOK {
|
|
t.Fatalf("status=%d body=%s", recorder.Code, recorder.Body.String())
|
|
}
|
|
var response capability.TaskEvaluationExecutionResponse
|
|
if err := json.Unmarshal(recorder.Body.Bytes(), &response); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(response.Observations) != 2 || response.Observations[0].Result != capability.EvaluationUnknown || response.Observations[1].Result != capability.EvaluationUnknown {
|
|
t.Fatalf("unexpected observations: %+v", response.Observations)
|
|
}
|
|
for _, observation := range response.Observations {
|
|
if observation.QualityScoreMillis != 0 || observation.CriticalFailure {
|
|
t.Fatalf("reachability-only evidence must not carry quality or failure claims: %+v", observation)
|
|
}
|
|
}
|
|
}
|
|
|
|
type localeCheckingProvider struct{ stubAdvisoryProvider }
|
|
|
|
func (p *localeCheckingProvider) Advise(ctx context.Context, request cognitive.AdvisoryRequest) (cognitive.AdvisoryResponse, error) {
|
|
if request.Locale == "" {
|
|
return cognitive.AdvisoryResponse{}, fmt.Errorf("task evaluation omitted required locale")
|
|
}
|
|
return p.stubAdvisoryProvider.Advise(ctx, request)
|
|
}
|
|
|
|
func TestTaskEvaluationSuppliesLocaleToProviderAdapter(t *testing.T) {
|
|
selected := gatewaySelection()
|
|
provider := &localeCheckingProvider{stubAdvisoryProvider{response: cognitive.AdvisoryResponse{Version: cognitive.CreatorAdvisoryV1, FinishReason: cognitive.FinishCompleted, AdvisoryText: "observed response"}}}
|
|
registry := &candidateRegistryStub{selected: selected}
|
|
gateway := cognitiveGateway{resolver: registry, providers: map[capability.ProviderKind][]providerRegistration{selected.Provider.Kind: {{ProviderInstanceID: selected.Provider.ID, Kind: selected.Provider.Kind, EndpointRef: selected.Provider.EndpointRef, CredentialRef: selected.Provider.CredentialRef, Provider: provider}}}, now: time.Now}
|
|
profile := capability.TaskEvaluationProfile{ID: "profile-locale", Revision: 1, CapabilityID: selected.Binding.CapabilityID, Stage: capability.TaskEvaluationGeneral, BenchmarkVersion: "locale-v1", RequiredCaseIDs: []string{"case-locale"}, QualityWeightMillis: 600, ReliabilityWeightMillis: 300, LatencyWeightMillis: 100}
|
|
request := capability.TaskEvaluationExecutionRequest{Profile: profile, CapabilityID: selected.Binding.CapabilityID, BindingID: selected.Binding.ID, BindingRevision: selected.Binding.Revision, ActorID: identity.ActorID("actor-1"), PrincipalID: selected.Binding.OwnerPrincipalID, ScopeID: authority.ScopeID("scope-1"), MaxOutputBytes: 512, EgressAcknowledged: true, Cases: []capability.TaskEvaluationCase{{CaseID: "case-locale", Prompt: "one", EvidenceArtifactRef: "artifact:locale"}}}
|
|
payload, _ := json.Marshal(request)
|
|
recorder := httptest.NewRecorder()
|
|
gateway.taskEvaluationHandler(recorder, httptest.NewRequest(http.MethodPost, "/v1/task-evaluations/execute", bytes.NewReader(payload)))
|
|
if recorder.Code != http.StatusOK {
|
|
t.Fatalf("status=%d body=%s", recorder.Code, recorder.Body.String())
|
|
}
|
|
}
|
|
|
|
type taskEvaluationFailoverRegistry struct {
|
|
primary, fallback capability.ResolvedCognitiveBinding
|
|
resolveCalls int
|
|
}
|
|
|
|
func (s *taskEvaluationFailoverRegistry) ResolveCandidate(context.Context, capability.CapabilityBindingID, int64) (capability.ResolvedCognitiveBinding, error) {
|
|
return s.primary, nil
|
|
}
|
|
|
|
func (s *taskEvaluationFailoverRegistry) Resolve(_ context.Context, requirement capability.CognitiveRequirement) (capability.ResolvedCognitiveBinding, error) {
|
|
s.resolveCalls++
|
|
if len(requirement.ExcludedBindingIDs) == 0 {
|
|
return s.primary, nil
|
|
}
|
|
return s.fallback, nil
|
|
}
|
|
|
|
func (s *taskEvaluationFailoverRegistry) Observe(context.Context, capability.CognitiveUsageObservation) error {
|
|
return nil
|
|
}
|
|
|
|
func (s *taskEvaluationFailoverRegistry) RegisterDiscovery(context.Context, capability.CognitiveDiscoverySnapshot) (capability.CognitiveDiscoveryRegistration, error) {
|
|
return capability.CognitiveDiscoveryRegistration{}, nil
|
|
}
|
|
|
|
func TestTaskEvaluationExcludesUnavailableCandidateAndUsesFallback(t *testing.T) {
|
|
primary := gatewaySelection()
|
|
fallback := primary
|
|
fallback.Binding.ID = "binding-fallback"
|
|
fallback.Binding.Revision = 7
|
|
fallback.Binding.ProviderInstanceID = "provider-fallback"
|
|
fallback.Provider.ID = "provider-fallback"
|
|
fallback.Provider.Kind = "provider-fallback-kind"
|
|
fallback.Provider.EndpointRef = "fallback-endpoint"
|
|
fallback.Provider.CredentialRef = "fallback-credential"
|
|
fallback.Model.ProviderInstanceID = "provider-fallback"
|
|
registry := &taskEvaluationFailoverRegistry{primary: primary, fallback: fallback}
|
|
primaryProvider := &stubAdvisoryProvider{response: cognitive.AdvisoryResponse{Version: cognitive.CreatorAdvisoryV1, FinishReason: cognitive.FinishProviderUnavailable}}
|
|
fallbackProvider := &stubAdvisoryProvider{response: cognitive.AdvisoryResponse{Version: cognitive.CreatorAdvisoryV1, FinishReason: cognitive.FinishCompleted, AdvisoryText: "fallback response"}}
|
|
gateway := cognitiveGateway{resolver: registry, providers: map[capability.ProviderKind][]providerRegistration{
|
|
primary.Provider.Kind: {{ProviderInstanceID: primary.Provider.ID, Kind: primary.Provider.Kind, EndpointRef: primary.Provider.EndpointRef, CredentialRef: primary.Provider.CredentialRef, Provider: primaryProvider}},
|
|
fallback.Provider.Kind: {{ProviderInstanceID: fallback.Provider.ID, Kind: fallback.Provider.Kind, EndpointRef: fallback.Provider.EndpointRef, CredentialRef: fallback.Provider.CredentialRef, Provider: fallbackProvider}},
|
|
}, now: func() time.Time { return time.UnixMilli(1000) }}
|
|
profile := capability.TaskEvaluationProfile{ID: "profile-failover", Revision: 1, CapabilityID: primary.Binding.CapabilityID, Stage: capability.TaskEvaluationGeneral, BenchmarkVersion: "failover-v1", RequiredCaseIDs: []string{"case-a"}, QualityWeightMillis: 600, ReliabilityWeightMillis: 300, LatencyWeightMillis: 100}
|
|
request := capability.TaskEvaluationExecutionRequest{Profile: profile, CapabilityID: profile.CapabilityID, BindingID: primary.Binding.ID, BindingRevision: primary.Binding.Revision, ActorID: identity.ActorID("actor-1"), PrincipalID: primary.Binding.OwnerPrincipalID, ScopeID: authority.ScopeID("scope-1"), MaxOutputBytes: 512, EgressAcknowledged: true, Cases: []capability.TaskEvaluationCase{{CaseID: "case-a", Prompt: "one", EvidenceArtifactRef: "artifact:a"}}}
|
|
payload, _ := json.Marshal(request)
|
|
recorder := httptest.NewRecorder()
|
|
gateway.taskEvaluationHandler(recorder, httptest.NewRequest(http.MethodPost, "/v1/task-evaluations/execute", bytes.NewReader(payload)))
|
|
if recorder.Code != http.StatusOK {
|
|
t.Fatalf("status=%d body=%s", recorder.Code, recorder.Body.String())
|
|
}
|
|
var response capability.TaskEvaluationExecutionResponse
|
|
if err := json.Unmarshal(recorder.Body.Bytes(), &response); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if response.BindingID != fallback.Binding.ID || !primaryProvider.called || !fallbackProvider.called || registry.resolveCalls != 1 {
|
|
t.Fatalf("failover binding=%s primary_called=%t fallback_called=%t resolve_calls=%d", response.BindingID, primaryProvider.called, fallbackProvider.called, registry.resolveCalls)
|
|
}
|
|
if response.Observations[0].CriticalFailure || response.Responses[0].ResponseText != "fallback response" {
|
|
t.Fatalf("fallback response was not usable: %+v %+v", response.Observations, response.Responses)
|
|
}
|
|
}
|
|
|
|
func TestTaskEvaluationUnavailableResponseIsMarkedCriticalUnknown(t *testing.T) {
|
|
selected := gatewaySelection()
|
|
registry := &taskEvaluationFailoverRegistry{primary: selected, fallback: selected}
|
|
provider := &stubAdvisoryProvider{response: cognitive.AdvisoryResponse{Version: cognitive.CreatorAdvisoryV1, FinishReason: cognitive.FinishTimeout}}
|
|
gateway := cognitiveGateway{resolver: registry, providers: map[capability.ProviderKind][]providerRegistration{
|
|
selected.Provider.Kind: {{ProviderInstanceID: selected.Provider.ID, Kind: selected.Provider.Kind, EndpointRef: selected.Provider.EndpointRef, CredentialRef: selected.Provider.CredentialRef, Provider: provider}},
|
|
}, now: func() time.Time { return time.UnixMilli(1000) }}
|
|
profile := capability.TaskEvaluationProfile{ID: "profile-unknown", Revision: 1, CapabilityID: selected.Binding.CapabilityID, Stage: capability.TaskEvaluationGeneral, BenchmarkVersion: "unknown-v1", RequiredCaseIDs: []string{"case-a"}, QualityWeightMillis: 600, ReliabilityWeightMillis: 300, LatencyWeightMillis: 100}
|
|
request := capability.TaskEvaluationExecutionRequest{Profile: profile, CapabilityID: profile.CapabilityID, BindingID: selected.Binding.ID, BindingRevision: selected.Binding.Revision, ActorID: identity.ActorID("actor-1"), PrincipalID: selected.Binding.OwnerPrincipalID, ScopeID: authority.ScopeID("scope-1"), MaxOutputBytes: 512, EgressAcknowledged: true, Cases: []capability.TaskEvaluationCase{{CaseID: "case-a", Prompt: "one", EvidenceArtifactRef: "artifact:a"}}}
|
|
payload, _ := json.Marshal(request)
|
|
recorder := httptest.NewRecorder()
|
|
gateway.taskEvaluationHandler(recorder, httptest.NewRequest(http.MethodPost, "/v1/task-evaluations/execute", bytes.NewReader(payload)))
|
|
if recorder.Code != http.StatusOK {
|
|
t.Fatalf("status=%d body=%s", recorder.Code, recorder.Body.String())
|
|
}
|
|
var response capability.TaskEvaluationExecutionResponse
|
|
if err := json.Unmarshal(recorder.Body.Bytes(), &response); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(response.Observations) != 1 || response.Observations[0].Result != capability.EvaluationUnknown || !response.Observations[0].CriticalFailure {
|
|
t.Fatalf("unavailable response was overclaimed: %+v", response.Observations)
|
|
}
|
|
}
|