fix(cognitive): fail over task evaluation candidates #227
@@ -50,6 +50,17 @@ func (e taskEvaluationExecutor) Execute(ctx context.Context, request work.AgentE
|
||||
if len(response.Responses) == 0 || response.ObservedAtUnixMillis < 1 {
|
||||
return work.AgentExecutionResult{}, fmt.Errorf("task evaluation returned no observations")
|
||||
}
|
||||
if len(response.Observations) == len(intent.TaskEvaluationCases) && taskEvaluationHasCriticalFailure(response.Observations) {
|
||||
evaluation, err := capabilitydomain.BuildTaskEvaluation(*intent.TaskEvaluationProfile, capability.TaskEvaluationID("evaluation:"+intent.DispatchID), response.BindingID, response.BindingRevision, intent.TaskEvaluationProfile.CapabilityID, response.Observations, artifact.ArtifactID(intent.TaskEvaluationCases[0].EvidenceArtifactRef), response.ObservedAtUnixMillis)
|
||||
if err != nil {
|
||||
return work.AgentExecutionResult{}, err
|
||||
}
|
||||
callback := work.TaskEvaluationEvidenceCallback{GraphID: execution.GraphID, Revision: execution.GraphRevision, NodeID: execution.NodeID, Evaluation: evaluation, Succeeded: true}
|
||||
if _, err := postJSON[json.RawMessage](ctx, e.client, e.callbackURL+"/v1/work-graphs/task-evaluations/callback", callback); err != nil {
|
||||
return work.AgentExecutionResult{}, err
|
||||
}
|
||||
return work.AgentExecutionResult{State: work.AgentExecutionCompleted, Summary: "task evaluation recorded as UNKNOWN after bounded provider recovery", OutcomeRef: string(evaluation.ID), ArtifactRefs: []string{string(evaluation.ArtifactID)}, ObservedAtUnixMillis: time.Now().UnixMilli()}, nil
|
||||
}
|
||||
if len(intent.TaskEvaluationCases) == 0 {
|
||||
return work.AgentExecutionResult{}, fmt.Errorf("task evaluation cases are missing")
|
||||
}
|
||||
@@ -80,6 +91,15 @@ func (e taskEvaluationExecutor) Execute(ctx context.Context, request work.AgentE
|
||||
return work.AgentExecutionResult{State: work.AgentExecutionCompleted, Summary: "task evaluation executed, aggregated and accepted by Work Graph callback", OutcomeRef: string(evaluation.ID), ArtifactRefs: []string{string(evaluation.ArtifactID)}, ObservedAtUnixMillis: time.Now().UnixMilli()}, nil
|
||||
}
|
||||
|
||||
func taskEvaluationHasCriticalFailure(observations []capability.TaskCaseObservation) bool {
|
||||
for _, observation := range observations {
|
||||
if observation.CriticalFailure {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func postJSON[T any](ctx context.Context, client *http.Client, url string, payload any) (T, error) {
|
||||
var result T
|
||||
body, err := json.Marshal(payload)
|
||||
|
||||
@@ -41,3 +41,32 @@ func TestTaskEvaluationExecutorInvokesAggregatesAndCallbacks(t *testing.T) {
|
||||
t.Fatalf("result=%+v err=%v", result, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTaskEvaluationExecutorPersistsUnknownAfterProviderFailure(t *testing.T) {
|
||||
profile := capability.TaskEvaluationProfile{ID: "profile-unknown", Revision: 1, CapabilityID: "capability-1", Stage: capability.TaskEvaluationGeneral, BenchmarkVersion: "unknown-v1", RequiredCaseIDs: []string{"case-1"}, QualityWeightMillis: 600, ReliabilityWeightMillis: 300, LatencyWeightMillis: 100}
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.URL.Path == "/v1/task-evaluations/execute" {
|
||||
_ = json.NewEncoder(w).Encode(capability.TaskEvaluationExecutionResponse{ProfileID: profile.ID, BindingID: "binding-1", BindingRevision: 3, Observations: []capability.TaskCaseObservation{{CaseID: "case-1", Result: capability.EvaluationUnknown, CriticalFailure: true, EvidenceArtifactRef: "artifact-1"}}, Responses: []capability.TaskCaseResponse{{CaseID: "case-1", EvidenceArtifactRef: "artifact-1"}}, ObservedAtUnixMillis: 10})
|
||||
return
|
||||
}
|
||||
if r.URL.Path == "/v1/work-graphs/task-evaluations/callback" {
|
||||
var callback work.TaskEvaluationEvidenceCallback
|
||||
if err := json.NewDecoder(r.Body).Decode(&callback); err != nil || callback.Evaluation.Result != capability.EvaluationUnknown || !callback.Succeeded {
|
||||
http.Error(w, "invalid unknown callback", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
return
|
||||
}
|
||||
if r.URL.Path == "/v1/task-evaluations/verify" {
|
||||
t.Errorf("verifier must not be called for a provider failure")
|
||||
}
|
||||
http.NotFound(w, r)
|
||||
}))
|
||||
defer server.Close()
|
||||
executor := newTaskEvaluationExecutor(server.URL, server.URL, server.URL, server.Client())
|
||||
result, err := executor.Execute(context.Background(), work.AgentExecutionRequest{Intent: work.AgentDispatchIntent{DispatchID: "dispatch-unknown", TaskEvaluationBindingID: "binding-1", TaskEvaluationProfile: &profile, TaskEvaluationCases: []capability.TaskEvaluationCase{{CaseID: "case-1", Prompt: "answer", EvidenceArtifactRef: "artifact-1"}}, TaskEvaluationExecution: &work.TaskEvaluationDispatchContext{GraphID: "graph-1", GraphRevision: 1, NodeID: "node-1", BindingRevision: 3, ActorID: "actor-1", PrincipalID: "principal-1", ScopeID: "scope-1"}}})
|
||||
if err != nil || result.State != work.AgentExecutionCompleted || result.OutcomeRef == "" {
|
||||
t.Fatalf("result=%+v err=%v", result, err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,9 +1,11 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
@@ -34,20 +36,13 @@ func (g cognitiveGateway) taskEvaluationHandler(w http.ResponseWriter, r *http.R
|
||||
http.Error(w, "external egress acknowledgement required", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
provider := g.provider(selected)
|
||||
if provider == nil {
|
||||
http.Error(w, "cognitive provider unavailable", http.StatusServiceUnavailable)
|
||||
return
|
||||
}
|
||||
|
||||
caseSet := make(map[string]struct{}, len(request.Profile.RequiredCaseIDs))
|
||||
seenCases := make(map[string]struct{}, len(request.Cases))
|
||||
for _, id := range request.Profile.RequiredCaseIDs {
|
||||
caseSet[id] = struct{}{}
|
||||
}
|
||||
observations := make([]capability.TaskCaseObservation, len(request.Cases))
|
||||
responses := make([]capability.TaskCaseResponse, len(request.Cases))
|
||||
var wg sync.WaitGroup
|
||||
seenCases := make(map[string]struct{}, len(request.Cases))
|
||||
for i, item := range request.Cases {
|
||||
for _, item := range request.Cases {
|
||||
if item.CaseID == "" || item.Prompt == "" || item.EvidenceArtifactRef == "" {
|
||||
http.Error(w, "invalid task evaluation case", http.StatusBadRequest)
|
||||
return
|
||||
@@ -61,35 +56,83 @@ func (g cognitiveGateway) taskEvaluationHandler(w http.ResponseWriter, r *http.R
|
||||
return
|
||||
}
|
||||
seenCases[item.CaseID] = struct{}{}
|
||||
wg.Add(1)
|
||||
go func(index int, item capability.TaskEvaluationCase) {
|
||||
defer wg.Done()
|
||||
started := time.Now()
|
||||
invocation := cognitive.AdvisoryRequest{Version: cognitive.CreatorAdvisoryV1, RequestID: cognitive.RequestID(fmt.Sprintf("task-evaluation:%s:%s", request.Profile.ID, item.CaseID)), ActorID: request.ActorID, PrincipalID: request.PrincipalID, ScopeID: request.ScopeID, Message: item.Prompt, MaxOutputBytes: int(request.MaxOutputBytes), ProviderBindingID: selected.Binding.ID, ProviderBindingRevision: selected.Binding.Revision, ProviderInstanceID: selected.Provider.ID, ModelDescriptorID: selected.Model.ID, ProviderModelID: selected.Model.ProviderModelID}
|
||||
response, invokeErr := g.invokeAdmitted(r.Context(), string(invocation.RequestID), provider, invocation, int(request.MaxOutputBytes))
|
||||
observation := capability.TaskCaseObservation{CaseID: item.CaseID, LatencyMicros: time.Since(started).Microseconds(), EvidenceArtifactRef: item.EvidenceArtifactRef}
|
||||
responses[index] = capability.TaskCaseResponse{CaseID: item.CaseID, ResponseText: response.AdvisoryText, LatencyMicros: observation.LatencyMicros, EvidenceArtifactRef: item.EvidenceArtifactRef}
|
||||
if invokeErr == nil && response.FinishReason == cognitive.FinishCompleted && response.AdvisoryText != "" {
|
||||
// Invocation success proves reachability only. Without an
|
||||
// independent verifier and answer evidence, semantic support and
|
||||
// quality must remain UNKNOWN rather than becoming false evidence.
|
||||
observation.Result = capability.EvaluationUnknown
|
||||
observation.QualityScoreMillis = 0
|
||||
} else {
|
||||
observation.Result = capability.EvaluationUnknown
|
||||
observation.CriticalFailure = invokeErr != nil
|
||||
}
|
||||
observations[index] = observation
|
||||
}(i, item)
|
||||
}
|
||||
if len(seenCases) != len(caseSet) {
|
||||
http.Error(w, "task evaluation case set is incomplete", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
wg.Wait()
|
||||
|
||||
// One bounded fan-out wave uses one binding snapshot. If that candidate is
|
||||
// unavailable, only this binding is excluded and the existing resolver
|
||||
// chooses the next eligible candidate. Provider output remains untrusted
|
||||
// until the independent verifier accepts it downstream.
|
||||
requirement := capability.CognitiveRequirement{CapabilityID: request.CapabilityID, OwnerPrincipalID: request.PrincipalID, DataPolicy: selected.Binding.DataPolicy, RequiredModalities: []capability.ModelModality{capability.ModalityText}, RequiredCapabilities: []capability.ModelCapability{capability.CapabilityAdvisory}, AllowCanary: g.allowCanary, MaxOutputBytes: request.MaxOutputBytes}
|
||||
var observations []capability.TaskCaseObservation
|
||||
var responses []capability.TaskCaseResponse
|
||||
for attempt := 0; attempt < 3; attempt++ {
|
||||
provider := g.provider(selected)
|
||||
if provider == nil {
|
||||
observations, responses = taskEvaluationUnavailable(request.Cases)
|
||||
} else {
|
||||
observations, responses = g.executeTaskEvaluationCases(r.Context(), request, selected, provider)
|
||||
}
|
||||
if taskEvaluationResponsesUsable(responses) {
|
||||
break
|
||||
}
|
||||
requirement.ExcludedBindingIDs = append(requirement.ExcludedBindingIDs, selected.Binding.ID)
|
||||
fallback, resolveErr := g.resolver.Resolve(r.Context(), requirement)
|
||||
fallbackProvider := g.provider(fallback)
|
||||
if resolveErr != nil || fallbackProvider == nil || fallback.Binding.ID == selected.Binding.ID || (fallback.Binding.DataPolicy != capability.DataPolicyLocalOnly && !request.EgressAcknowledged) {
|
||||
break
|
||||
}
|
||||
selected = fallback
|
||||
}
|
||||
|
||||
writeTaskEvaluationJSON(w, capability.TaskEvaluationExecutionResponse{ProfileID: request.Profile.ID, BindingID: selected.Binding.ID, BindingRevision: selected.Binding.Revision, Observations: observations, Responses: responses, ObservedAtUnixMillis: time.Now().UnixMilli()})
|
||||
}
|
||||
|
||||
func (g cognitiveGateway) executeTaskEvaluationCases(ctx context.Context, request capability.TaskEvaluationExecutionRequest, selected capability.ResolvedCognitiveBinding, provider advisoryProvider) ([]capability.TaskCaseObservation, []capability.TaskCaseResponse) {
|
||||
observations := make([]capability.TaskCaseObservation, len(request.Cases))
|
||||
responses := make([]capability.TaskCaseResponse, len(request.Cases))
|
||||
var wg sync.WaitGroup
|
||||
for i, item := range request.Cases {
|
||||
wg.Add(1)
|
||||
go func(index int, item capability.TaskEvaluationCase) {
|
||||
defer wg.Done()
|
||||
started := time.Now()
|
||||
invocation := cognitive.AdvisoryRequest{Version: cognitive.CreatorAdvisoryV1, RequestID: cognitive.RequestID(fmt.Sprintf("task-evaluation:%s:%s", request.Profile.ID, item.CaseID)), ActorID: request.ActorID, PrincipalID: request.PrincipalID, ScopeID: request.ScopeID, Message: item.Prompt, MaxOutputBytes: int(request.MaxOutputBytes), ProviderBindingID: selected.Binding.ID, ProviderBindingRevision: selected.Binding.Revision, ProviderInstanceID: selected.Provider.ID, ModelDescriptorID: selected.Model.ID, ProviderModelID: selected.Model.ProviderModelID}
|
||||
response, invokeErr := g.invokeAdmitted(ctx, string(invocation.RequestID), provider, invocation, int(request.MaxOutputBytes))
|
||||
latency := time.Since(started).Microseconds()
|
||||
observations[index] = capability.TaskCaseObservation{CaseID: item.CaseID, Result: capability.EvaluationUnknown, CriticalFailure: invokeErr != nil || response.FinishReason != cognitive.FinishCompleted || strings.TrimSpace(response.AdvisoryText) == "", EvidenceArtifactRef: item.EvidenceArtifactRef, LatencyMicros: latency}
|
||||
responses[index] = capability.TaskCaseResponse{CaseID: item.CaseID, ResponseText: response.AdvisoryText, LatencyMicros: latency, EvidenceArtifactRef: item.EvidenceArtifactRef}
|
||||
}(i, item)
|
||||
}
|
||||
wg.Wait()
|
||||
return observations, responses
|
||||
}
|
||||
|
||||
func taskEvaluationUnavailable(cases []capability.TaskEvaluationCase) ([]capability.TaskCaseObservation, []capability.TaskCaseResponse) {
|
||||
observations := make([]capability.TaskCaseObservation, len(cases))
|
||||
responses := make([]capability.TaskCaseResponse, len(cases))
|
||||
for i, item := range cases {
|
||||
observations[i] = capability.TaskCaseObservation{CaseID: item.CaseID, Result: capability.EvaluationUnknown, CriticalFailure: true, EvidenceArtifactRef: item.EvidenceArtifactRef}
|
||||
responses[i] = capability.TaskCaseResponse{CaseID: item.CaseID, EvidenceArtifactRef: item.EvidenceArtifactRef}
|
||||
}
|
||||
return observations, responses
|
||||
}
|
||||
|
||||
func taskEvaluationResponsesUsable(responses []capability.TaskCaseResponse) bool {
|
||||
if len(responses) == 0 {
|
||||
return false
|
||||
}
|
||||
for _, response := range responses {
|
||||
if strings.TrimSpace(response.ResponseText) == "" {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func writeTaskEvaluationJSON(w http.ResponseWriter, value any) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_ = json.NewEncoder(w).Encode(value)
|
||||
|
||||
@@ -56,3 +56,90 @@ func TestTaskEvaluationHandlerInvokesEveryBoundedCase(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type taskEvaluationFailoverRegistry struct {
|
||||
primary, fallback capability.ResolvedCognitiveBinding
|
||||
resolveCalls int
|
||||
}
|
||||
|
||||
func (s *taskEvaluationFailoverRegistry) ResolveCandidate(context.Context, capability.CapabilityBindingID, int64) (capability.ResolvedCognitiveBinding, error) {
|
||||
return s.primary, nil
|
||||
}
|
||||
|
||||
func (s *taskEvaluationFailoverRegistry) Resolve(_ context.Context, requirement capability.CognitiveRequirement) (capability.ResolvedCognitiveBinding, error) {
|
||||
s.resolveCalls++
|
||||
if len(requirement.ExcludedBindingIDs) == 0 {
|
||||
return s.primary, nil
|
||||
}
|
||||
return s.fallback, nil
|
||||
}
|
||||
|
||||
func (s *taskEvaluationFailoverRegistry) Observe(context.Context, capability.CognitiveUsageObservation) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *taskEvaluationFailoverRegistry) RegisterDiscovery(context.Context, capability.CognitiveDiscoverySnapshot) (capability.CognitiveDiscoveryRegistration, error) {
|
||||
return capability.CognitiveDiscoveryRegistration{}, nil
|
||||
}
|
||||
|
||||
func TestTaskEvaluationExcludesUnavailableCandidateAndUsesFallback(t *testing.T) {
|
||||
primary := gatewaySelection()
|
||||
fallback := primary
|
||||
fallback.Binding.ID = "binding-fallback"
|
||||
fallback.Binding.Revision = 7
|
||||
fallback.Binding.ProviderInstanceID = "provider-fallback"
|
||||
fallback.Provider.ID = "provider-fallback"
|
||||
fallback.Provider.Kind = "provider-fallback-kind"
|
||||
fallback.Provider.EndpointRef = "fallback-endpoint"
|
||||
fallback.Provider.CredentialRef = "fallback-credential"
|
||||
fallback.Model.ProviderInstanceID = "provider-fallback"
|
||||
registry := &taskEvaluationFailoverRegistry{primary: primary, fallback: fallback}
|
||||
primaryProvider := &stubAdvisoryProvider{response: cognitive.AdvisoryResponse{Version: cognitive.CreatorAdvisoryV1, FinishReason: cognitive.FinishProviderUnavailable}}
|
||||
fallbackProvider := &stubAdvisoryProvider{response: cognitive.AdvisoryResponse{Version: cognitive.CreatorAdvisoryV1, FinishReason: cognitive.FinishCompleted, AdvisoryText: "fallback response"}}
|
||||
gateway := cognitiveGateway{resolver: registry, providers: map[capability.ProviderKind][]providerRegistration{
|
||||
primary.Provider.Kind: {{ProviderInstanceID: primary.Provider.ID, Kind: primary.Provider.Kind, EndpointRef: primary.Provider.EndpointRef, CredentialRef: primary.Provider.CredentialRef, Provider: primaryProvider}},
|
||||
fallback.Provider.Kind: {{ProviderInstanceID: fallback.Provider.ID, Kind: fallback.Provider.Kind, EndpointRef: fallback.Provider.EndpointRef, CredentialRef: fallback.Provider.CredentialRef, Provider: fallbackProvider}},
|
||||
}, now: func() time.Time { return time.UnixMilli(1000) }}
|
||||
profile := capability.TaskEvaluationProfile{ID: "profile-failover", Revision: 1, CapabilityID: primary.Binding.CapabilityID, Stage: capability.TaskEvaluationGeneral, BenchmarkVersion: "failover-v1", RequiredCaseIDs: []string{"case-a"}, QualityWeightMillis: 600, ReliabilityWeightMillis: 300, LatencyWeightMillis: 100}
|
||||
request := capability.TaskEvaluationExecutionRequest{Profile: profile, CapabilityID: profile.CapabilityID, BindingID: primary.Binding.ID, BindingRevision: primary.Binding.Revision, ActorID: identity.ActorID("actor-1"), PrincipalID: primary.Binding.OwnerPrincipalID, ScopeID: authority.ScopeID("scope-1"), MaxOutputBytes: 512, EgressAcknowledged: true, Cases: []capability.TaskEvaluationCase{{CaseID: "case-a", Prompt: "one", EvidenceArtifactRef: "artifact:a"}}}
|
||||
payload, _ := json.Marshal(request)
|
||||
recorder := httptest.NewRecorder()
|
||||
gateway.taskEvaluationHandler(recorder, httptest.NewRequest(http.MethodPost, "/v1/task-evaluations/execute", bytes.NewReader(payload)))
|
||||
if recorder.Code != http.StatusOK {
|
||||
t.Fatalf("status=%d body=%s", recorder.Code, recorder.Body.String())
|
||||
}
|
||||
var response capability.TaskEvaluationExecutionResponse
|
||||
if err := json.Unmarshal(recorder.Body.Bytes(), &response); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if response.BindingID != fallback.Binding.ID || !primaryProvider.called || !fallbackProvider.called || registry.resolveCalls != 1 {
|
||||
t.Fatalf("failover binding=%s primary_called=%t fallback_called=%t resolve_calls=%d", response.BindingID, primaryProvider.called, fallbackProvider.called, registry.resolveCalls)
|
||||
}
|
||||
if response.Observations[0].CriticalFailure || response.Responses[0].ResponseText != "fallback response" {
|
||||
t.Fatalf("fallback response was not usable: %+v %+v", response.Observations, response.Responses)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTaskEvaluationUnavailableResponseIsMarkedCriticalUnknown(t *testing.T) {
|
||||
selected := gatewaySelection()
|
||||
registry := &taskEvaluationFailoverRegistry{primary: selected, fallback: selected}
|
||||
provider := &stubAdvisoryProvider{response: cognitive.AdvisoryResponse{Version: cognitive.CreatorAdvisoryV1, FinishReason: cognitive.FinishTimeout}}
|
||||
gateway := cognitiveGateway{resolver: registry, providers: map[capability.ProviderKind][]providerRegistration{
|
||||
selected.Provider.Kind: {{ProviderInstanceID: selected.Provider.ID, Kind: selected.Provider.Kind, EndpointRef: selected.Provider.EndpointRef, CredentialRef: selected.Provider.CredentialRef, Provider: provider}},
|
||||
}, now: func() time.Time { return time.UnixMilli(1000) }}
|
||||
profile := capability.TaskEvaluationProfile{ID: "profile-unknown", Revision: 1, CapabilityID: selected.Binding.CapabilityID, Stage: capability.TaskEvaluationGeneral, BenchmarkVersion: "unknown-v1", RequiredCaseIDs: []string{"case-a"}, QualityWeightMillis: 600, ReliabilityWeightMillis: 300, LatencyWeightMillis: 100}
|
||||
request := capability.TaskEvaluationExecutionRequest{Profile: profile, CapabilityID: profile.CapabilityID, BindingID: selected.Binding.ID, BindingRevision: selected.Binding.Revision, ActorID: identity.ActorID("actor-1"), PrincipalID: selected.Binding.OwnerPrincipalID, ScopeID: authority.ScopeID("scope-1"), MaxOutputBytes: 512, EgressAcknowledged: true, Cases: []capability.TaskEvaluationCase{{CaseID: "case-a", Prompt: "one", EvidenceArtifactRef: "artifact:a"}}}
|
||||
payload, _ := json.Marshal(request)
|
||||
recorder := httptest.NewRecorder()
|
||||
gateway.taskEvaluationHandler(recorder, httptest.NewRequest(http.MethodPost, "/v1/task-evaluations/execute", bytes.NewReader(payload)))
|
||||
if recorder.Code != http.StatusOK {
|
||||
t.Fatalf("status=%d body=%s", recorder.Code, recorder.Body.String())
|
||||
}
|
||||
var response capability.TaskEvaluationExecutionResponse
|
||||
if err := json.Unmarshal(recorder.Body.Bytes(), &response); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(response.Observations) != 1 || response.Observations[0].Result != capability.EvaluationUnknown || !response.Observations[0].CriticalFailure {
|
||||
t.Fatalf("unavailable response was overclaimed: %+v", response.Observations)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
{
|
||||
"task_file": "dev/tasks/DEV-000248-live-task-evaluation-scheduling.json",
|
||||
"task_id": "DEV-000248",
|
||||
"task_file": "dev/tasks/DEV-000249-task-evaluation-model-failover.json",
|
||||
"task_id": "DEV-000249",
|
||||
"task_revision": 1
|
||||
}
|
||||
|
||||
@@ -0,0 +1,23 @@
|
||||
{
|
||||
"task_id": "DEV-000249",
|
||||
"task_revision": 1,
|
||||
"architecture_revision": "ADR-0018",
|
||||
"base_repo_revision": "f90ed7af35f9f19c72039812272640ae1d63635b",
|
||||
"risk_tier": "E",
|
||||
"intent": "Make task-evaluation execution use the existing cognitive model resolution and local fallback path without recording unsupported output as qualification evidence.",
|
||||
"requirement_ids": ["model fleet/failover", "live task evaluation", "local recovery"],
|
||||
"allowed_change_scope": {"modules": ["capability", "work"], "paths": ["cmd/cognitive-worker/**", "cmd/agent-worker/**", "modules/capability/domain/**", "dev/ACTIVE_TASK.json", "dev/tasks/DEV-000249-task-evaluation-model-failover.json"]},
|
||||
"forbidden_change_scope": {"paths": ["architecture/**", "security/**", "database/**", "deploy/**", "contracts/**", "LLM-local/**", "modules/work/**", "modules/work/secrets/**"]},
|
||||
"allowed_architecture_delta": {"new_cross_module_dependencies": 0, "new_canonical_owners": 0, "new_privileges": 0, "new_third_party_dependencies": 0, "new_database_owner_roles": 0},
|
||||
"contracts": {"consume": ["existing CognitiveRequirement", "existing ResolvedCognitiveBinding", "existing TaskEvaluationExecutionRequest/Response"], "may_change": []},
|
||||
"semantic_owner": "capability cognitive execution adapter",
|
||||
"canonical_state": ["existing cognitive usage observations and task evaluation records only"],
|
||||
"state_transitions": ["candidate invocation -> bounded fallback candidate -> verified evaluation or UNKNOWN"],
|
||||
"side_effects_and_privileges": ["existing cognitive provider calls and capability observations; no new privileges"],
|
||||
"persistence_and_migrations": ["none"],
|
||||
"failure_retry_restart": ["provider failure excludes the failed binding for the current bounded execution; all candidates unavailable returns UNKNOWN without qualification"],
|
||||
"assumptions": ["fallback resolution remains governed by the existing capability resolver and data-policy checks"],
|
||||
"acceptance_scenarios": ["task evaluation invokes active candidate", "provider-unavailable candidate is excluded and local candidate is attempted", "empty/non-completed provider response never becomes supported evidence", "all candidates unavailable produces durable UNKNOWN evaluation and terminal graph callback"],
|
||||
"required_tests": ["go test ./cmd/cognitive-worker ./modules/capability/domain", "go vet ./cmd/cognitive-worker ./modules/capability/domain", "go run ./cmd/devctl preflight -active dev/ACTIVE_TASK.json", "go run ./cmd/devctl check-diff -active dev/ACTIVE_TASK.json", "go run ./cmd/devctl check-architecture", "git diff --check", "independent", "CI", "production health smoke", "live task-evaluation canary"],
|
||||
"terminal_gap_states": ["BLOCKED_BY_MISSING_CAPABILITY", "BLOCKED_BY_SPEC_GAP"]
|
||||
}
|
||||
Reference in New Issue
Block a user