package main import ( "bytes" "context" "encoding/json" "fmt" "net/http" "net/http/httptest" "testing" "time" "git.cin.su/m/Assistant/contracts/authority" "git.cin.su/m/Assistant/contracts/capability" "git.cin.su/m/Assistant/contracts/cognitive" "git.cin.su/m/Assistant/contracts/identity" ) type candidateRegistryStub struct { registryStub selected capability.ResolvedCognitiveBinding } func (s *candidateRegistryStub) ResolveCandidate(context.Context, capability.CapabilityBindingID, int64) (capability.ResolvedCognitiveBinding, error) { return s.selected, nil } func TestTaskEvaluationHandlerInvokesEveryBoundedCase(t *testing.T) { selected := gatewaySelection() registry := &candidateRegistryStub{selected: selected} provider := &stubAdvisoryProvider{response: cognitive.AdvisoryResponse{Version: cognitive.CreatorAdvisoryV1, FinishReason: cognitive.FinishCompleted, AdvisoryText: "observed response"}} gateway := cognitiveGateway{ resolver: registry, providers: map[capability.ProviderKind][]providerRegistration{ selected.Provider.Kind: {{ProviderInstanceID: selected.Provider.ID, Kind: selected.Provider.Kind, EndpointRef: selected.Provider.EndpointRef, CredentialRef: selected.Provider.CredentialRef, Provider: provider}}, }, now: func() time.Time { return time.UnixMilli(1000) }, } profile := capability.TaskEvaluationProfile{ID: "profile-general", Revision: 1, CapabilityID: selected.Binding.CapabilityID, Stage: capability.TaskEvaluationGeneral, BenchmarkVersion: "general-v1", RequiredCaseIDs: []string{"case-a", "case-b"}, QualityWeightMillis: 600, ReliabilityWeightMillis: 300, LatencyWeightMillis: 100} request := capability.TaskEvaluationExecutionRequest{Profile: profile, CapabilityID: selected.Binding.CapabilityID, BindingID: selected.Binding.ID, BindingRevision: selected.Binding.Revision, ActorID: identity.ActorID("actor-1"), PrincipalID: selected.Binding.OwnerPrincipalID, ScopeID: authority.ScopeID("scope-1"), MaxOutputBytes: 512, EgressAcknowledged: true, Cases: []capability.TaskEvaluationCase{{CaseID: "case-a", Prompt: "one", EvidenceArtifactRef: "artifact:a"}, {CaseID: "case-b", Prompt: "two", EvidenceArtifactRef: "artifact:b"}}} payload, _ := json.Marshal(request) recorder := httptest.NewRecorder() gateway.taskEvaluationHandler(recorder, httptest.NewRequest(http.MethodPost, "/v1/task-evaluations/execute", bytes.NewReader(payload))) if recorder.Code != http.StatusOK { t.Fatalf("status=%d body=%s", recorder.Code, recorder.Body.String()) } var response capability.TaskEvaluationExecutionResponse if err := json.Unmarshal(recorder.Body.Bytes(), &response); err != nil { t.Fatal(err) } if len(response.Observations) != 2 || response.Observations[0].Result != capability.EvaluationUnknown || response.Observations[1].Result != capability.EvaluationUnknown { t.Fatalf("unexpected observations: %+v", response.Observations) } for _, observation := range response.Observations { if observation.QualityScoreMillis != 0 || observation.CriticalFailure { t.Fatalf("reachability-only evidence must not carry quality or failure claims: %+v", observation) } } } type localeCheckingProvider struct{ stubAdvisoryProvider } func (p *localeCheckingProvider) Advise(ctx context.Context, request cognitive.AdvisoryRequest) (cognitive.AdvisoryResponse, error) { if request.Locale == "" { return cognitive.AdvisoryResponse{}, fmt.Errorf("task evaluation omitted required locale") } return p.stubAdvisoryProvider.Advise(ctx, request) } func TestTaskEvaluationSuppliesLocaleToProviderAdapter(t *testing.T) { selected := gatewaySelection() provider := &localeCheckingProvider{stubAdvisoryProvider{response: cognitive.AdvisoryResponse{Version: cognitive.CreatorAdvisoryV1, FinishReason: cognitive.FinishCompleted, AdvisoryText: "observed response"}}} registry := &candidateRegistryStub{selected: selected} gateway := cognitiveGateway{resolver: registry, providers: map[capability.ProviderKind][]providerRegistration{selected.Provider.Kind: {{ProviderInstanceID: selected.Provider.ID, Kind: selected.Provider.Kind, EndpointRef: selected.Provider.EndpointRef, CredentialRef: selected.Provider.CredentialRef, Provider: provider}}}, now: time.Now} profile := capability.TaskEvaluationProfile{ID: "profile-locale", Revision: 1, CapabilityID: selected.Binding.CapabilityID, Stage: capability.TaskEvaluationGeneral, BenchmarkVersion: "locale-v1", RequiredCaseIDs: []string{"case-locale"}, QualityWeightMillis: 600, ReliabilityWeightMillis: 300, LatencyWeightMillis: 100} request := capability.TaskEvaluationExecutionRequest{Profile: profile, CapabilityID: selected.Binding.CapabilityID, BindingID: selected.Binding.ID, BindingRevision: selected.Binding.Revision, ActorID: identity.ActorID("actor-1"), PrincipalID: selected.Binding.OwnerPrincipalID, ScopeID: authority.ScopeID("scope-1"), MaxOutputBytes: 512, EgressAcknowledged: true, Cases: []capability.TaskEvaluationCase{{CaseID: "case-locale", Prompt: "one", EvidenceArtifactRef: "artifact:locale"}}} payload, _ := json.Marshal(request) recorder := httptest.NewRecorder() gateway.taskEvaluationHandler(recorder, httptest.NewRequest(http.MethodPost, "/v1/task-evaluations/execute", bytes.NewReader(payload))) if recorder.Code != http.StatusOK { t.Fatalf("status=%d body=%s", recorder.Code, recorder.Body.String()) } } type taskEvaluationFailoverRegistry struct { primary, fallback capability.ResolvedCognitiveBinding resolveCalls int } func (s *taskEvaluationFailoverRegistry) ResolveCandidate(context.Context, capability.CapabilityBindingID, int64) (capability.ResolvedCognitiveBinding, error) { return s.primary, nil } func (s *taskEvaluationFailoverRegistry) Resolve(_ context.Context, requirement capability.CognitiveRequirement) (capability.ResolvedCognitiveBinding, error) { s.resolveCalls++ if len(requirement.ExcludedBindingIDs) == 0 { return s.primary, nil } return s.fallback, nil } func (s *taskEvaluationFailoverRegistry) Observe(context.Context, capability.CognitiveUsageObservation) error { return nil } func (s *taskEvaluationFailoverRegistry) RegisterDiscovery(context.Context, capability.CognitiveDiscoverySnapshot) (capability.CognitiveDiscoveryRegistration, error) { return capability.CognitiveDiscoveryRegistration{}, nil } func TestTaskEvaluationExcludesUnavailableCandidateAndUsesFallback(t *testing.T) { primary := gatewaySelection() fallback := primary fallback.Binding.ID = "binding-fallback" fallback.Binding.Revision = 7 fallback.Binding.ProviderInstanceID = "provider-fallback" fallback.Provider.ID = "provider-fallback" fallback.Provider.Kind = "provider-fallback-kind" fallback.Provider.EndpointRef = "fallback-endpoint" fallback.Provider.CredentialRef = "fallback-credential" fallback.Model.ProviderInstanceID = "provider-fallback" registry := &taskEvaluationFailoverRegistry{primary: primary, fallback: fallback} primaryProvider := &stubAdvisoryProvider{response: cognitive.AdvisoryResponse{Version: cognitive.CreatorAdvisoryV1, FinishReason: cognitive.FinishProviderUnavailable}} fallbackProvider := &stubAdvisoryProvider{response: cognitive.AdvisoryResponse{Version: cognitive.CreatorAdvisoryV1, FinishReason: cognitive.FinishCompleted, AdvisoryText: "fallback response"}} gateway := cognitiveGateway{resolver: registry, providers: map[capability.ProviderKind][]providerRegistration{ primary.Provider.Kind: {{ProviderInstanceID: primary.Provider.ID, Kind: primary.Provider.Kind, EndpointRef: primary.Provider.EndpointRef, CredentialRef: primary.Provider.CredentialRef, Provider: primaryProvider}}, fallback.Provider.Kind: {{ProviderInstanceID: fallback.Provider.ID, Kind: fallback.Provider.Kind, EndpointRef: fallback.Provider.EndpointRef, CredentialRef: fallback.Provider.CredentialRef, Provider: fallbackProvider}}, }, now: func() time.Time { return time.UnixMilli(1000) }} profile := capability.TaskEvaluationProfile{ID: "profile-failover", Revision: 1, CapabilityID: primary.Binding.CapabilityID, Stage: capability.TaskEvaluationGeneral, BenchmarkVersion: "failover-v1", RequiredCaseIDs: []string{"case-a"}, QualityWeightMillis: 600, ReliabilityWeightMillis: 300, LatencyWeightMillis: 100} request := capability.TaskEvaluationExecutionRequest{Profile: profile, CapabilityID: profile.CapabilityID, BindingID: primary.Binding.ID, BindingRevision: primary.Binding.Revision, ActorID: identity.ActorID("actor-1"), PrincipalID: primary.Binding.OwnerPrincipalID, ScopeID: authority.ScopeID("scope-1"), MaxOutputBytes: 512, EgressAcknowledged: true, Cases: []capability.TaskEvaluationCase{{CaseID: "case-a", Prompt: "one", EvidenceArtifactRef: "artifact:a"}}} payload, _ := json.Marshal(request) recorder := httptest.NewRecorder() gateway.taskEvaluationHandler(recorder, httptest.NewRequest(http.MethodPost, "/v1/task-evaluations/execute", bytes.NewReader(payload))) if recorder.Code != http.StatusOK { t.Fatalf("status=%d body=%s", recorder.Code, recorder.Body.String()) } var response capability.TaskEvaluationExecutionResponse if err := json.Unmarshal(recorder.Body.Bytes(), &response); err != nil { t.Fatal(err) } if response.BindingID != fallback.Binding.ID || !primaryProvider.called || !fallbackProvider.called || registry.resolveCalls != 1 { t.Fatalf("failover binding=%s primary_called=%t fallback_called=%t resolve_calls=%d", response.BindingID, primaryProvider.called, fallbackProvider.called, registry.resolveCalls) } if response.Observations[0].CriticalFailure || response.Responses[0].ResponseText != "fallback response" { t.Fatalf("fallback response was not usable: %+v %+v", response.Observations, response.Responses) } } func TestTaskEvaluationUnavailableResponseIsMarkedCriticalUnknown(t *testing.T) { selected := gatewaySelection() registry := &taskEvaluationFailoverRegistry{primary: selected, fallback: selected} provider := &stubAdvisoryProvider{response: cognitive.AdvisoryResponse{Version: cognitive.CreatorAdvisoryV1, FinishReason: cognitive.FinishTimeout}} gateway := cognitiveGateway{resolver: registry, providers: map[capability.ProviderKind][]providerRegistration{ selected.Provider.Kind: {{ProviderInstanceID: selected.Provider.ID, Kind: selected.Provider.Kind, EndpointRef: selected.Provider.EndpointRef, CredentialRef: selected.Provider.CredentialRef, Provider: provider}}, }, now: func() time.Time { return time.UnixMilli(1000) }} profile := capability.TaskEvaluationProfile{ID: "profile-unknown", Revision: 1, CapabilityID: selected.Binding.CapabilityID, Stage: capability.TaskEvaluationGeneral, BenchmarkVersion: "unknown-v1", RequiredCaseIDs: []string{"case-a"}, QualityWeightMillis: 600, ReliabilityWeightMillis: 300, LatencyWeightMillis: 100} request := capability.TaskEvaluationExecutionRequest{Profile: profile, CapabilityID: profile.CapabilityID, BindingID: selected.Binding.ID, BindingRevision: selected.Binding.Revision, ActorID: identity.ActorID("actor-1"), PrincipalID: selected.Binding.OwnerPrincipalID, ScopeID: authority.ScopeID("scope-1"), MaxOutputBytes: 512, EgressAcknowledged: true, Cases: []capability.TaskEvaluationCase{{CaseID: "case-a", Prompt: "one", EvidenceArtifactRef: "artifact:a"}}} payload, _ := json.Marshal(request) recorder := httptest.NewRecorder() gateway.taskEvaluationHandler(recorder, httptest.NewRequest(http.MethodPost, "/v1/task-evaluations/execute", bytes.NewReader(payload))) if recorder.Code != http.StatusOK { t.Fatalf("status=%d body=%s", recorder.Code, recorder.Body.String()) } var response capability.TaskEvaluationExecutionResponse if err := json.Unmarshal(recorder.Body.Bytes(), &response); err != nil { t.Fatal(err) } if len(response.Observations) != 1 || response.Observations[0].Result != capability.EvaluationUnknown || !response.Observations[0].CriticalFailure { t.Fatalf("unavailable response was overclaimed: %+v", response.Observations) } }