feat(cognitive): execute bounded task evaluation cases
This commit is contained in:
@@ -26,6 +26,10 @@ type bindingResolver interface {
|
||||
RegisterDiscovery(context.Context, capability.CognitiveDiscoverySnapshot) (capability.CognitiveDiscoveryRegistration, error)
|
||||
}
|
||||
|
||||
type candidateBindingResolver interface {
|
||||
ResolveCandidate(context.Context, capability.CapabilityBindingID, int64) (capability.ResolvedCognitiveBinding, error)
|
||||
}
|
||||
|
||||
type admissionClient interface {
|
||||
Admit(context.Context, string, capability.AdmissionRequest) (capability.AdmissionDecision, capability.AdmissionLease, error)
|
||||
Release(context.Context, capability.AdmissionLease) error
|
||||
@@ -341,6 +345,26 @@ func (r *httpBindingResolver) Resolve(ctx context.Context, value capability.Cogn
|
||||
return result, err
|
||||
}
|
||||
|
||||
func (r *httpBindingResolver) ResolveCandidate(ctx context.Context, bindingID capability.CapabilityBindingID, revision int64) (capability.ResolvedCognitiveBinding, error) {
|
||||
var result capability.ResolvedCognitiveBinding
|
||||
target := *r.base
|
||||
target.Path = "/v1/cognitive-bindings/" + url.PathEscape(string(bindingID)) + "/revisions/" + fmt.Sprint(revision) + "/snapshot"
|
||||
request, err := http.NewRequestWithContext(ctx, http.MethodGet, target.String(), nil)
|
||||
if err != nil {
|
||||
return result, err
|
||||
}
|
||||
response, err := r.client.Do(request)
|
||||
if err != nil {
|
||||
return result, err
|
||||
}
|
||||
defer response.Body.Close()
|
||||
if response.StatusCode < 200 || response.StatusCode >= 300 {
|
||||
return result, fmt.Errorf("candidate status %d", response.StatusCode)
|
||||
}
|
||||
err = json.NewDecoder(io.LimitReader(response.Body, 64<<10)).Decode(&result)
|
||||
return result, err
|
||||
}
|
||||
|
||||
func (r *httpBindingResolver) Observe(ctx context.Context, value capability.CognitiveUsageObservation) error {
|
||||
return r.post(ctx, "/v1/cognitive-usage-observations", value, nil)
|
||||
}
|
||||
|
||||
@@ -166,6 +166,7 @@ func main() {
|
||||
mux.HandleFunc("POST /v1/advisory-policy", gateway.policyHandler)
|
||||
mux.HandleFunc("POST /v1/advisories", gateway.advisoryHandler)
|
||||
mux.HandleFunc("POST /v1/interpretations", gateway.interpretationHandler)
|
||||
mux.HandleFunc("POST /v1/task-evaluations/execute", gateway.taskEvaluationHandler)
|
||||
discovery := discoveryService{resolver: resolver, providers: discoveries, owner: identity.PrincipalID(envOr("COGNITIVE_OWNER_PRINCIPAL_ID", "domovoy-creator-principal")), now: time.Now}
|
||||
mux.HandleFunc("POST /v1/provider-discovery/{provider}", discovery.handler)
|
||||
refreshProviderDiscoveries(resolver, discovery)
|
||||
|
||||
@@ -0,0 +1,93 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"git.cin.su/m/Assistant/contracts/capability"
|
||||
"git.cin.su/m/Assistant/contracts/cognitive"
|
||||
)
|
||||
|
||||
func (g cognitiveGateway) taskEvaluationHandler(w http.ResponseWriter, r *http.Request) {
|
||||
var request capability.TaskEvaluationExecutionRequest
|
||||
if !decodeJSON(w, r, 256<<10, &request) {
|
||||
return
|
||||
}
|
||||
if err := request.Profile.Validate(); err != nil || request.CapabilityID == "" || request.CapabilityID != request.Profile.CapabilityID || request.BindingID == "" || request.BindingRevision < 1 || request.ActorID == "" || request.PrincipalID == "" || request.ScopeID == "" || request.MaxOutputBytes < 1 || len(request.Cases) != len(request.Profile.RequiredCaseIDs) {
|
||||
http.Error(w, "invalid task evaluation execution request", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
resolver, ok := g.resolver.(candidateBindingResolver)
|
||||
if !ok {
|
||||
http.Error(w, "exact candidate resolver unavailable", http.StatusServiceUnavailable)
|
||||
return
|
||||
}
|
||||
selected, err := resolver.ResolveCandidate(r.Context(), request.BindingID, request.BindingRevision)
|
||||
if err != nil || selected.Binding.ID != request.BindingID || selected.Binding.Revision != request.BindingRevision || selected.Binding.CapabilityID != request.CapabilityID || selected.Binding.OwnerPrincipalID != request.PrincipalID {
|
||||
http.Error(w, "cognitive binding snapshot unavailable", http.StatusConflict)
|
||||
return
|
||||
}
|
||||
if selected.Binding.DataPolicy != capability.DataPolicyLocalOnly && !request.EgressAcknowledged {
|
||||
http.Error(w, "external egress acknowledgement required", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
provider := g.provider(selected)
|
||||
if provider == nil {
|
||||
http.Error(w, "cognitive provider unavailable", http.StatusServiceUnavailable)
|
||||
return
|
||||
}
|
||||
caseSet := make(map[string]struct{}, len(request.Profile.RequiredCaseIDs))
|
||||
for _, id := range request.Profile.RequiredCaseIDs {
|
||||
caseSet[id] = struct{}{}
|
||||
}
|
||||
observations := make([]capability.TaskCaseObservation, len(request.Cases))
|
||||
var wg sync.WaitGroup
|
||||
seenCases := make(map[string]struct{}, len(request.Cases))
|
||||
for i, item := range request.Cases {
|
||||
if item.CaseID == "" || item.Prompt == "" || item.EvidenceArtifactRef == "" {
|
||||
http.Error(w, "invalid task evaluation case", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
if _, exists := caseSet[item.CaseID]; !exists {
|
||||
http.Error(w, "task evaluation case is not in profile", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
if _, exists := seenCases[item.CaseID]; exists {
|
||||
http.Error(w, "duplicate task evaluation case", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
seenCases[item.CaseID] = struct{}{}
|
||||
wg.Add(1)
|
||||
go func(index int, item capability.TaskEvaluationCase) {
|
||||
defer wg.Done()
|
||||
started := time.Now()
|
||||
invocation := cognitive.AdvisoryRequest{Version: cognitive.CreatorAdvisoryV1, RequestID: cognitive.RequestID(fmt.Sprintf("task-evaluation:%s:%s", request.Profile.ID, item.CaseID)), ActorID: request.ActorID, PrincipalID: request.PrincipalID, ScopeID: request.ScopeID, Message: item.Prompt, MaxOutputBytes: int(request.MaxOutputBytes), ProviderBindingID: selected.Binding.ID, ProviderBindingRevision: selected.Binding.Revision, ProviderInstanceID: selected.Provider.ID, ModelDescriptorID: selected.Model.ID, ProviderModelID: selected.Model.ProviderModelID}
|
||||
response, invokeErr := g.invokeAdmitted(r.Context(), string(invocation.RequestID), provider, invocation, int(request.MaxOutputBytes))
|
||||
observation := capability.TaskCaseObservation{CaseID: item.CaseID, LatencyMicros: time.Since(started).Microseconds(), EvidenceArtifactRef: item.EvidenceArtifactRef}
|
||||
if invokeErr == nil && response.FinishReason == cognitive.FinishCompleted && response.AdvisoryText != "" {
|
||||
observation.Result = capability.EvaluationSupported
|
||||
// Invocation success proves reachability/support for this case,
|
||||
// not semantic answer quality. A verifier may score quality later.
|
||||
observation.QualityScoreMillis = 0
|
||||
} else {
|
||||
observation.Result = capability.EvaluationUnknown
|
||||
observation.CriticalFailure = invokeErr != nil
|
||||
}
|
||||
observations[index] = observation
|
||||
}(i, item)
|
||||
}
|
||||
if len(seenCases) != len(caseSet) {
|
||||
http.Error(w, "task evaluation case set is incomplete", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
wg.Wait()
|
||||
writeTaskEvaluationJSON(w, capability.TaskEvaluationExecutionResponse{ProfileID: request.Profile.ID, BindingID: selected.Binding.ID, BindingRevision: selected.Binding.Revision, Observations: observations, ObservedAtUnixMillis: time.Now().UnixMilli()})
|
||||
}
|
||||
|
||||
func writeTaskEvaluationJSON(w http.ResponseWriter, value any) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_ = json.NewEncoder(w).Encode(value)
|
||||
}
|
||||
@@ -0,0 +1,53 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.cin.su/m/Assistant/contracts/authority"
|
||||
"git.cin.su/m/Assistant/contracts/capability"
|
||||
"git.cin.su/m/Assistant/contracts/cognitive"
|
||||
"git.cin.su/m/Assistant/contracts/identity"
|
||||
)
|
||||
|
||||
type candidateRegistryStub struct {
|
||||
registryStub
|
||||
selected capability.ResolvedCognitiveBinding
|
||||
}
|
||||
|
||||
func (s *candidateRegistryStub) ResolveCandidate(context.Context, capability.CapabilityBindingID, int64) (capability.ResolvedCognitiveBinding, error) {
|
||||
return s.selected, nil
|
||||
}
|
||||
|
||||
func TestTaskEvaluationHandlerInvokesEveryBoundedCase(t *testing.T) {
|
||||
selected := gatewaySelection()
|
||||
registry := &candidateRegistryStub{selected: selected}
|
||||
provider := &stubAdvisoryProvider{response: cognitive.AdvisoryResponse{Version: cognitive.CreatorAdvisoryV1, FinishReason: cognitive.FinishCompleted, AdvisoryText: "observed response"}}
|
||||
gateway := cognitiveGateway{
|
||||
resolver: registry,
|
||||
providers: map[capability.ProviderKind][]providerRegistration{
|
||||
selected.Provider.Kind: {{ProviderInstanceID: selected.Provider.ID, Kind: selected.Provider.Kind, EndpointRef: selected.Provider.EndpointRef, CredentialRef: selected.Provider.CredentialRef, Provider: provider}},
|
||||
},
|
||||
now: func() time.Time { return time.UnixMilli(1000) },
|
||||
}
|
||||
profile := capability.TaskEvaluationProfile{ID: "profile-general", Revision: 1, CapabilityID: selected.Binding.CapabilityID, Stage: capability.TaskEvaluationGeneral, BenchmarkVersion: "general-v1", RequiredCaseIDs: []string{"case-a", "case-b"}, QualityWeightMillis: 600, ReliabilityWeightMillis: 300, LatencyWeightMillis: 100}
|
||||
request := capability.TaskEvaluationExecutionRequest{Profile: profile, CapabilityID: selected.Binding.CapabilityID, BindingID: selected.Binding.ID, BindingRevision: selected.Binding.Revision, ActorID: identity.ActorID("actor-1"), PrincipalID: selected.Binding.OwnerPrincipalID, ScopeID: authority.ScopeID("scope-1"), MaxOutputBytes: 512, EgressAcknowledged: true, Cases: []capability.TaskEvaluationCase{{CaseID: "case-a", Prompt: "one", EvidenceArtifactRef: "artifact:a"}, {CaseID: "case-b", Prompt: "two", EvidenceArtifactRef: "artifact:b"}}}
|
||||
payload, _ := json.Marshal(request)
|
||||
recorder := httptest.NewRecorder()
|
||||
gateway.taskEvaluationHandler(recorder, httptest.NewRequest(http.MethodPost, "/v1/task-evaluations/execute", bytes.NewReader(payload)))
|
||||
if recorder.Code != http.StatusOK {
|
||||
t.Fatalf("status=%d body=%s", recorder.Code, recorder.Body.String())
|
||||
}
|
||||
var response capability.TaskEvaluationExecutionResponse
|
||||
if err := json.Unmarshal(recorder.Body.Bytes(), &response); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(response.Observations) != 2 || response.Observations[0].Result != capability.EvaluationSupported || response.Observations[1].Result != capability.EvaluationSupported {
|
||||
t.Fatalf("unexpected observations: %+v", response.Observations)
|
||||
}
|
||||
}
|
||||
@@ -834,6 +834,20 @@ func main() {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_ = json.NewEncoder(w).Encode(binding)
|
||||
})
|
||||
mux.HandleFunc("GET /v1/cognitive-bindings/{id}/revisions/{revision}/snapshot", func(w http.ResponseWriter, r *http.Request) {
|
||||
revision, err := strconv.ParseInt(r.PathValue("revision"), 10, 64)
|
||||
if err != nil || revision < 1 {
|
||||
http.Error(w, "invalid cognitive binding revision", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
snapshot, err := capabilityRepository.LoadCognitiveCandidate(r.Context(), capability.CapabilityBindingID(r.PathValue("id")), revision)
|
||||
if err != nil {
|
||||
http.Error(w, "cognitive binding snapshot not found", http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_ = json.NewEncoder(w).Encode(snapshot)
|
||||
})
|
||||
mux.HandleFunc("POST /v1/cognitive-bindings/resolve", func(w http.ResponseWriter, r *http.Request) {
|
||||
var requirement capability.CognitiveRequirement
|
||||
decoder := json.NewDecoder(io.LimitReader(r.Body, 16*1024+1))
|
||||
|
||||
Reference in New Issue
Block a user