fix(cognitive): fail over task evaluation candidates
This commit is contained in:
@@ -56,3 +56,90 @@ func TestTaskEvaluationHandlerInvokesEveryBoundedCase(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type taskEvaluationFailoverRegistry struct {
|
||||
primary, fallback capability.ResolvedCognitiveBinding
|
||||
resolveCalls int
|
||||
}
|
||||
|
||||
func (s *taskEvaluationFailoverRegistry) ResolveCandidate(context.Context, capability.CapabilityBindingID, int64) (capability.ResolvedCognitiveBinding, error) {
|
||||
return s.primary, nil
|
||||
}
|
||||
|
||||
func (s *taskEvaluationFailoverRegistry) Resolve(_ context.Context, requirement capability.CognitiveRequirement) (capability.ResolvedCognitiveBinding, error) {
|
||||
s.resolveCalls++
|
||||
if len(requirement.ExcludedBindingIDs) == 0 {
|
||||
return s.primary, nil
|
||||
}
|
||||
return s.fallback, nil
|
||||
}
|
||||
|
||||
func (s *taskEvaluationFailoverRegistry) Observe(context.Context, capability.CognitiveUsageObservation) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *taskEvaluationFailoverRegistry) RegisterDiscovery(context.Context, capability.CognitiveDiscoverySnapshot) (capability.CognitiveDiscoveryRegistration, error) {
|
||||
return capability.CognitiveDiscoveryRegistration{}, nil
|
||||
}
|
||||
|
||||
func TestTaskEvaluationExcludesUnavailableCandidateAndUsesFallback(t *testing.T) {
|
||||
primary := gatewaySelection()
|
||||
fallback := primary
|
||||
fallback.Binding.ID = "binding-fallback"
|
||||
fallback.Binding.Revision = 7
|
||||
fallback.Binding.ProviderInstanceID = "provider-fallback"
|
||||
fallback.Provider.ID = "provider-fallback"
|
||||
fallback.Provider.Kind = "provider-fallback-kind"
|
||||
fallback.Provider.EndpointRef = "fallback-endpoint"
|
||||
fallback.Provider.CredentialRef = "fallback-credential"
|
||||
fallback.Model.ProviderInstanceID = "provider-fallback"
|
||||
registry := &taskEvaluationFailoverRegistry{primary: primary, fallback: fallback}
|
||||
primaryProvider := &stubAdvisoryProvider{response: cognitive.AdvisoryResponse{Version: cognitive.CreatorAdvisoryV1, FinishReason: cognitive.FinishProviderUnavailable}}
|
||||
fallbackProvider := &stubAdvisoryProvider{response: cognitive.AdvisoryResponse{Version: cognitive.CreatorAdvisoryV1, FinishReason: cognitive.FinishCompleted, AdvisoryText: "fallback response"}}
|
||||
gateway := cognitiveGateway{resolver: registry, providers: map[capability.ProviderKind][]providerRegistration{
|
||||
primary.Provider.Kind: {{ProviderInstanceID: primary.Provider.ID, Kind: primary.Provider.Kind, EndpointRef: primary.Provider.EndpointRef, CredentialRef: primary.Provider.CredentialRef, Provider: primaryProvider}},
|
||||
fallback.Provider.Kind: {{ProviderInstanceID: fallback.Provider.ID, Kind: fallback.Provider.Kind, EndpointRef: fallback.Provider.EndpointRef, CredentialRef: fallback.Provider.CredentialRef, Provider: fallbackProvider}},
|
||||
}, now: func() time.Time { return time.UnixMilli(1000) }}
|
||||
profile := capability.TaskEvaluationProfile{ID: "profile-failover", Revision: 1, CapabilityID: primary.Binding.CapabilityID, Stage: capability.TaskEvaluationGeneral, BenchmarkVersion: "failover-v1", RequiredCaseIDs: []string{"case-a"}, QualityWeightMillis: 600, ReliabilityWeightMillis: 300, LatencyWeightMillis: 100}
|
||||
request := capability.TaskEvaluationExecutionRequest{Profile: profile, CapabilityID: profile.CapabilityID, BindingID: primary.Binding.ID, BindingRevision: primary.Binding.Revision, ActorID: identity.ActorID("actor-1"), PrincipalID: primary.Binding.OwnerPrincipalID, ScopeID: authority.ScopeID("scope-1"), MaxOutputBytes: 512, EgressAcknowledged: true, Cases: []capability.TaskEvaluationCase{{CaseID: "case-a", Prompt: "one", EvidenceArtifactRef: "artifact:a"}}}
|
||||
payload, _ := json.Marshal(request)
|
||||
recorder := httptest.NewRecorder()
|
||||
gateway.taskEvaluationHandler(recorder, httptest.NewRequest(http.MethodPost, "/v1/task-evaluations/execute", bytes.NewReader(payload)))
|
||||
if recorder.Code != http.StatusOK {
|
||||
t.Fatalf("status=%d body=%s", recorder.Code, recorder.Body.String())
|
||||
}
|
||||
var response capability.TaskEvaluationExecutionResponse
|
||||
if err := json.Unmarshal(recorder.Body.Bytes(), &response); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if response.BindingID != fallback.Binding.ID || !primaryProvider.called || !fallbackProvider.called || registry.resolveCalls != 1 {
|
||||
t.Fatalf("failover binding=%s primary_called=%t fallback_called=%t resolve_calls=%d", response.BindingID, primaryProvider.called, fallbackProvider.called, registry.resolveCalls)
|
||||
}
|
||||
if response.Observations[0].CriticalFailure || response.Responses[0].ResponseText != "fallback response" {
|
||||
t.Fatalf("fallback response was not usable: %+v %+v", response.Observations, response.Responses)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTaskEvaluationUnavailableResponseIsMarkedCriticalUnknown(t *testing.T) {
|
||||
selected := gatewaySelection()
|
||||
registry := &taskEvaluationFailoverRegistry{primary: selected, fallback: selected}
|
||||
provider := &stubAdvisoryProvider{response: cognitive.AdvisoryResponse{Version: cognitive.CreatorAdvisoryV1, FinishReason: cognitive.FinishTimeout}}
|
||||
gateway := cognitiveGateway{resolver: registry, providers: map[capability.ProviderKind][]providerRegistration{
|
||||
selected.Provider.Kind: {{ProviderInstanceID: selected.Provider.ID, Kind: selected.Provider.Kind, EndpointRef: selected.Provider.EndpointRef, CredentialRef: selected.Provider.CredentialRef, Provider: provider}},
|
||||
}, now: func() time.Time { return time.UnixMilli(1000) }}
|
||||
profile := capability.TaskEvaluationProfile{ID: "profile-unknown", Revision: 1, CapabilityID: selected.Binding.CapabilityID, Stage: capability.TaskEvaluationGeneral, BenchmarkVersion: "unknown-v1", RequiredCaseIDs: []string{"case-a"}, QualityWeightMillis: 600, ReliabilityWeightMillis: 300, LatencyWeightMillis: 100}
|
||||
request := capability.TaskEvaluationExecutionRequest{Profile: profile, CapabilityID: profile.CapabilityID, BindingID: selected.Binding.ID, BindingRevision: selected.Binding.Revision, ActorID: identity.ActorID("actor-1"), PrincipalID: selected.Binding.OwnerPrincipalID, ScopeID: authority.ScopeID("scope-1"), MaxOutputBytes: 512, EgressAcknowledged: true, Cases: []capability.TaskEvaluationCase{{CaseID: "case-a", Prompt: "one", EvidenceArtifactRef: "artifact:a"}}}
|
||||
payload, _ := json.Marshal(request)
|
||||
recorder := httptest.NewRecorder()
|
||||
gateway.taskEvaluationHandler(recorder, httptest.NewRequest(http.MethodPost, "/v1/task-evaluations/execute", bytes.NewReader(payload)))
|
||||
if recorder.Code != http.StatusOK {
|
||||
t.Fatalf("status=%d body=%s", recorder.Code, recorder.Body.String())
|
||||
}
|
||||
var response capability.TaskEvaluationExecutionResponse
|
||||
if err := json.Unmarshal(recorder.Body.Bytes(), &response); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(response.Observations) != 1 || response.Observations[0].Result != capability.EvaluationUnknown || !response.Observations[0].CriticalFailure {
|
||||
t.Fatalf("unavailable response was overclaimed: %+v", response.Observations)
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user