Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
feea909d69 | ||
|
|
b78a026ee1 |
@@ -430,6 +430,14 @@ func (r *CapabilityRepository) SaveCognitiveBinding(ctx context.Context, value c
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (r *CapabilityRepository) LoadCognitiveCandidates(ctx context.Context, capabilityID capability.CapabilityID, owner identity.PrincipalID) ([]capability.ResolvedCognitiveBinding, error) {
|
func (r *CapabilityRepository) LoadCognitiveCandidates(ctx context.Context, capabilityID capability.CapabilityID, owner identity.PrincipalID) ([]capability.ResolvedCognitiveBinding, error) {
|
||||||
|
return r.loadCognitiveCandidates(ctx, capabilityID, owner, "")
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *CapabilityRepository) LoadCognitiveCandidatesForProfile(ctx context.Context, capabilityID capability.CapabilityID, owner identity.PrincipalID, profileID capability.TaskEvaluationProfileID) ([]capability.ResolvedCognitiveBinding, error) {
|
||||||
|
return r.loadCognitiveCandidates(ctx, capabilityID, owner, profileID)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *CapabilityRepository) loadCognitiveCandidates(ctx context.Context, capabilityID capability.CapabilityID, owner identity.PrincipalID, profileID capability.TaskEvaluationProfileID) ([]capability.ResolvedCognitiveBinding, error) {
|
||||||
rows, err := r.pool.Query(ctx, `SELECT
|
rows, err := r.pool.Query(ctx, `SELECT
|
||||||
b.binding_id,b.revision,b.capability_id,b.owner_principal_id,b.provider_instance_id,b.provider_revision,b.model_descriptor_id,b.model_revision,b.data_policy,b.qualification,b.activation,b.health,b.limit_policy_ref,
|
b.binding_id,b.revision,b.capability_id,b.owner_principal_id,b.provider_instance_id,b.provider_revision,b.model_descriptor_id,b.model_revision,b.data_policy,b.qualification,b.activation,b.health,b.limit_policy_ref,
|
||||||
p.provider_instance_id,p.revision,p.provider_kind,p.endpoint_ref,COALESCE(p.credential_ref,''),p.enabled,p.qualification,COALESCE(ph.health,p.health),GREATEST(p.observed_at_unix_millis,COALESCE(ph.observed_at_unix_millis,0)),p.billing_mode,p.auth_mode,p.input_cost_micros,p.output_cost_micros,p.request_cost_micros,
|
p.provider_instance_id,p.revision,p.provider_kind,p.endpoint_ref,COALESCE(p.credential_ref,''),p.enabled,p.qualification,COALESCE(ph.health,p.health),GREATEST(p.observed_at_unix_millis,COALESCE(ph.observed_at_unix_millis,0)),p.billing_mode,p.auth_mode,p.input_cost_micros,p.output_cost_micros,p.request_cost_micros,
|
||||||
@@ -454,7 +462,7 @@ func (r *CapabilityRepository) LoadCognitiveCandidates(ctx context.Context, capa
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
if err = r.loadCognitiveEvidence(ctx, &value); err != nil {
|
if err = r.loadCognitiveEvidence(ctx, &value, profileID); err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
result = append(result, value)
|
result = append(result, value)
|
||||||
@@ -462,7 +470,7 @@ func (r *CapabilityRepository) LoadCognitiveCandidates(ctx context.Context, capa
|
|||||||
return result, rows.Err()
|
return result, rows.Err()
|
||||||
}
|
}
|
||||||
|
|
||||||
func (r *CapabilityRepository) loadCognitiveEvidence(ctx context.Context, value *capability.ResolvedCognitiveBinding) error {
|
func (r *CapabilityRepository) loadCognitiveEvidence(ctx context.Context, value *capability.ResolvedCognitiveBinding, profileID capability.TaskEvaluationProfileID) error {
|
||||||
var evaluation capability.EvaluatedCapability
|
var evaluation capability.EvaluatedCapability
|
||||||
err := r.pool.QueryRow(ctx, `SELECT evaluation_id,binding_id,binding_revision,capability_id,result,quality_score_millis,critical_failure_millis,benchmark_version,artifact_id,observed_at_unix_millis FROM capability.cognitive_evaluated_capabilities WHERE binding_id=$1 AND binding_revision=$2 ORDER BY observed_at_unix_millis DESC LIMIT 1`, value.Binding.ID, value.Binding.Revision).Scan(&evaluation.ID, &evaluation.BindingID, &evaluation.BindingRevision, &evaluation.CapabilityID, &evaluation.Result, &evaluation.QualityScoreMillis, &evaluation.CriticalFailureMillis, &evaluation.BenchmarkVersion, &evaluation.ArtifactID, &evaluation.ObservedAtUnixMillis)
|
err := r.pool.QueryRow(ctx, `SELECT evaluation_id,binding_id,binding_revision,capability_id,result,quality_score_millis,critical_failure_millis,benchmark_version,artifact_id,observed_at_unix_millis FROM capability.cognitive_evaluated_capabilities WHERE binding_id=$1 AND binding_revision=$2 ORDER BY observed_at_unix_millis DESC LIMIT 1`, value.Binding.ID, value.Binding.Revision).Scan(&evaluation.ID, &evaluation.BindingID, &evaluation.BindingRevision, &evaluation.CapabilityID, &evaluation.Result, &evaluation.QualityScoreMillis, &evaluation.CriticalFailureMillis, &evaluation.BenchmarkVersion, &evaluation.ArtifactID, &evaluation.ObservedAtUnixMillis)
|
||||||
if err != nil && !errors.Is(err, pgx.ErrNoRows) {
|
if err != nil && !errors.Is(err, pgx.ErrNoRows) {
|
||||||
@@ -479,6 +487,16 @@ func (r *CapabilityRepository) loadCognitiveEvidence(ctx context.Context, value
|
|||||||
if err == nil {
|
if err == nil {
|
||||||
value.Performance = &performance
|
value.Performance = &performance
|
||||||
}
|
}
|
||||||
|
if profileID != "" {
|
||||||
|
var taskEvaluation capability.TaskEvaluation
|
||||||
|
err = r.pool.QueryRow(ctx, `SELECT evaluation_id,task_profile_id,binding_id,binding_revision,capability_id,result,quality_score_millis,reliability_score_millis,latency_score_millis,overall_score_millis,critical_failure_millis,benchmark_version,artifact_id,observed_at_unix_millis FROM capability.cognitive_task_evaluations WHERE task_profile_id=$1 AND binding_id=$2 AND binding_revision=$3 ORDER BY observed_at_unix_millis DESC LIMIT 1`, profileID, value.Binding.ID, value.Binding.Revision).Scan(&taskEvaluation.ID, &taskEvaluation.ProfileID, &taskEvaluation.BindingID, &taskEvaluation.BindingRevision, &taskEvaluation.CapabilityID, &taskEvaluation.Result, &taskEvaluation.QualityScoreMillis, &taskEvaluation.ReliabilityScoreMillis, &taskEvaluation.LatencyScoreMillis, &taskEvaluation.OverallScoreMillis, &taskEvaluation.CriticalFailureMillis, &taskEvaluation.BenchmarkVersion, &taskEvaluation.ArtifactID, &taskEvaluation.ObservedAtUnixMillis)
|
||||||
|
if err != nil && !errors.Is(err, pgx.ErrNoRows) {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err == nil {
|
||||||
|
value.TaskEvaluation = &taskEvaluation
|
||||||
|
}
|
||||||
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -592,3 +610,8 @@ func (r *CapabilityRepository) SavePerformanceEvidence(ctx context.Context, valu
|
|||||||
_, err := r.pool.Exec(ctx, `INSERT INTO capability.cognitive_performance_evidence (evidence_id,binding_id,binding_revision,model_descriptor_id,model_revision,hardware_profile,runtime,quantization,context_units,thinking_enabled,generation_tps_millis,prompt_tps_millis,ttft_micros,latency_micros,memory_bytes,reliability_millis,benchmark_version,artifact_id,observed_at_unix_millis) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17,$18,$19) ON CONFLICT DO NOTHING`, value.ID, value.BindingID, value.BindingRevision, value.ModelDescriptorID, value.ModelRevision, value.HardwareProfile, value.Runtime, value.Quantization, value.ContextUnits, value.ThinkingEnabled, value.GenerationTPSMillis, value.PromptTPSMillis, value.TTFTMicros, value.LatencyMicros, value.MemoryBytes, value.ReliabilityMillis, value.BenchmarkVersion, value.ArtifactID, value.ObservedAtUnixMillis)
|
_, err := r.pool.Exec(ctx, `INSERT INTO capability.cognitive_performance_evidence (evidence_id,binding_id,binding_revision,model_descriptor_id,model_revision,hardware_profile,runtime,quantization,context_units,thinking_enabled,generation_tps_millis,prompt_tps_millis,ttft_micros,latency_micros,memory_bytes,reliability_millis,benchmark_version,artifact_id,observed_at_unix_millis) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17,$18,$19) ON CONFLICT DO NOTHING`, value.ID, value.BindingID, value.BindingRevision, value.ModelDescriptorID, value.ModelRevision, value.HardwareProfile, value.Runtime, value.Quantization, value.ContextUnits, value.ThinkingEnabled, value.GenerationTPSMillis, value.PromptTPSMillis, value.TTFTMicros, value.LatencyMicros, value.MemoryBytes, value.ReliabilityMillis, value.BenchmarkVersion, value.ArtifactID, value.ObservedAtUnixMillis)
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (r *CapabilityRepository) SaveTaskEvaluation(ctx context.Context, value capability.TaskEvaluation) error {
|
||||||
|
_, err := r.pool.Exec(ctx, `INSERT INTO capability.cognitive_task_evaluations (evaluation_id,task_profile_id,binding_id,binding_revision,capability_id,result,quality_score_millis,reliability_score_millis,latency_score_millis,overall_score_millis,critical_failure_millis,benchmark_version,artifact_id,observed_at_unix_millis) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14) ON CONFLICT DO NOTHING`, value.ID, value.ProfileID, value.BindingID, value.BindingRevision, value.CapabilityID, value.Result, value.QualityScoreMillis, value.ReliabilityScoreMillis, value.LatencyScoreMillis, value.OverallScoreMillis, value.CriticalFailureMillis, value.BenchmarkVersion, value.ArtifactID, value.ObservedAtUnixMillis)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|||||||
@@ -842,7 +842,13 @@ func main() {
|
|||||||
http.Error(w, "invalid cognitive requirement", http.StatusBadRequest)
|
http.Error(w, "invalid cognitive requirement", http.StatusBadRequest)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
candidates, err := capabilityRepository.LoadCognitiveCandidates(r.Context(), requirement.CapabilityID, requirement.OwnerPrincipalID)
|
var candidates []capability.ResolvedCognitiveBinding
|
||||||
|
var err error
|
||||||
|
if requirement.TaskProfileID != "" {
|
||||||
|
candidates, err = capabilityRepository.LoadCognitiveCandidatesForProfile(r.Context(), requirement.CapabilityID, requirement.OwnerPrincipalID, requirement.TaskProfileID)
|
||||||
|
} else {
|
||||||
|
candidates, err = capabilityRepository.LoadCognitiveCandidates(r.Context(), requirement.CapabilityID, requirement.OwnerPrincipalID)
|
||||||
|
}
|
||||||
if err != nil {
|
if err != nil {
|
||||||
http.Error(w, "cognitive registry unavailable", http.StatusServiceUnavailable)
|
http.Error(w, "cognitive registry unavailable", http.StatusServiceUnavailable)
|
||||||
return
|
return
|
||||||
@@ -891,6 +897,20 @@ func main() {
|
|||||||
}
|
}
|
||||||
w.WriteHeader(http.StatusCreated)
|
w.WriteHeader(http.StatusCreated)
|
||||||
})
|
})
|
||||||
|
mux.HandleFunc("POST /v1/cognitive-registry/task-evaluations", func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
var value capability.TaskEvaluation
|
||||||
|
decoder := json.NewDecoder(io.LimitReader(r.Body, 64<<10))
|
||||||
|
decoder.DisallowUnknownFields()
|
||||||
|
if err := decoder.Decode(&value); err != nil || capabilitydomain.ValidateTaskEvaluation(value) != nil {
|
||||||
|
http.Error(w, "invalid task evaluation", http.StatusBadRequest)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if err := capabilityRepository.SaveTaskEvaluation(r.Context(), value); err != nil {
|
||||||
|
http.Error(w, "save task evaluation", http.StatusInternalServerError)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
w.WriteHeader(http.StatusCreated)
|
||||||
|
})
|
||||||
mux.HandleFunc("POST /v1/creator/cognitive-canary", func(w http.ResponseWriter, r *http.Request) {
|
mux.HandleFunc("POST /v1/creator/cognitive-canary", func(w http.ResponseWriter, r *http.Request) {
|
||||||
session, ok := authenticateCreator(w, r)
|
session, ok := authenticateCreator(w, r)
|
||||||
if !ok {
|
if !ok {
|
||||||
|
|||||||
@@ -160,28 +160,32 @@ type CognitiveRequirement struct {
|
|||||||
ExcludedBindingIDs []CapabilityBindingID
|
ExcludedBindingIDs []CapabilityBindingID
|
||||||
Objective SelectionObjective
|
Objective SelectionObjective
|
||||||
MaxCostMicros *int64
|
MaxCostMicros *int64
|
||||||
|
TaskProfileID TaskEvaluationProfileID
|
||||||
}
|
}
|
||||||
|
|
||||||
// ResolvedCognitiveBinding is a provider-neutral invocation snapshot selected
|
// ResolvedCognitiveBinding is a provider-neutral invocation snapshot selected
|
||||||
// by Capability. It contains references and declared metadata, never secrets.
|
// by Capability. It contains references and declared metadata, never secrets.
|
||||||
type ResolvedCognitiveBinding struct {
|
type ResolvedCognitiveBinding struct {
|
||||||
Binding CognitiveBinding
|
Binding CognitiveBinding
|
||||||
Provider ProviderInstance
|
Provider ProviderInstance
|
||||||
Model ModelDescriptor
|
Model ModelDescriptor
|
||||||
Evaluation *EvaluatedCapability
|
Evaluation *EvaluatedCapability
|
||||||
Performance *PerformanceEvidence
|
Performance *PerformanceEvidence
|
||||||
Rating *ModelRating
|
TaskEvaluation *TaskEvaluation
|
||||||
|
Rating *ModelRating
|
||||||
}
|
}
|
||||||
|
|
||||||
// ModelRating is a derived, provider-neutral explanation of why an eligible
|
// ModelRating is a derived, provider-neutral explanation of why an eligible
|
||||||
// model is routed. It never grants qualification or activation authority.
|
// model is routed. It never grants qualification or activation authority.
|
||||||
type ModelRating struct {
|
type ModelRating struct {
|
||||||
Category ModelRatingCategory `json:"category"`
|
TaskProfileID TaskEvaluationProfileID `json:"task_profile_id,omitempty"`
|
||||||
OverallScoreMillis int64 `json:"overall_score_millis"`
|
TaskScoreMillis int64 `json:"task_score_millis,omitempty"`
|
||||||
QualityScoreMillis int64 `json:"quality_score_millis"`
|
Category ModelRatingCategory `json:"category"`
|
||||||
ReliabilityScoreMillis int64 `json:"reliability_score_millis"`
|
OverallScoreMillis int64 `json:"overall_score_millis"`
|
||||||
LatencyScoreMillis int64 `json:"latency_score_millis"`
|
QualityScoreMillis int64 `json:"quality_score_millis"`
|
||||||
EvidenceScoreMillis int64 `json:"evidence_score_millis"`
|
ReliabilityScoreMillis int64 `json:"reliability_score_millis"`
|
||||||
|
LatencyScoreMillis int64 `json:"latency_score_millis"`
|
||||||
|
EvidenceScoreMillis int64 `json:"evidence_score_millis"`
|
||||||
}
|
}
|
||||||
|
|
||||||
type CognitiveResultClass string
|
type CognitiveResultClass string
|
||||||
|
|||||||
@@ -0,0 +1,102 @@
|
|||||||
|
package capability
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
|
||||||
|
"git.cin.su/m/Assistant/contracts/artifact"
|
||||||
|
"git.cin.su/m/Assistant/platform/ids"
|
||||||
|
)
|
||||||
|
|
||||||
|
type TaskEvaluationProfileID ids.ID
|
||||||
|
type TaskEvaluationID ids.ID
|
||||||
|
type TaskEvaluationProfileStage string
|
||||||
|
|
||||||
|
const (
|
||||||
|
TaskEvaluationGeneral TaskEvaluationProfileStage = "GENERAL"
|
||||||
|
TaskEvaluationAdditional TaskEvaluationProfileStage = "ADDITIONAL"
|
||||||
|
)
|
||||||
|
|
||||||
|
// TaskEvaluationProfile is the extensible, provider-neutral description of a
|
||||||
|
// task-specific qualification run. Case payloads live in Artifact; the
|
||||||
|
// profile stores only stable case references and scoring policy.
|
||||||
|
type TaskEvaluationProfile struct {
|
||||||
|
ID TaskEvaluationProfileID
|
||||||
|
Revision int64
|
||||||
|
CapabilityID CapabilityID
|
||||||
|
Stage TaskEvaluationProfileStage
|
||||||
|
ParentProfileID TaskEvaluationProfileID
|
||||||
|
TriggerEvidenceRef string
|
||||||
|
BenchmarkVersion string
|
||||||
|
RequiredCaseIDs []string
|
||||||
|
QualityWeightMillis int64
|
||||||
|
ReliabilityWeightMillis int64
|
||||||
|
LatencyWeightMillis int64
|
||||||
|
}
|
||||||
|
|
||||||
|
func (p TaskEvaluationProfile) Validate() error {
|
||||||
|
if p.ID == "" || p.Revision < 1 || p.CapabilityID == "" || (p.Stage != TaskEvaluationGeneral && p.Stage != TaskEvaluationAdditional) || p.BenchmarkVersion == "" || len(p.RequiredCaseIDs) == 0 {
|
||||||
|
return errors.New("task evaluation profile identity and cases are incomplete")
|
||||||
|
}
|
||||||
|
if p.Stage == TaskEvaluationGeneral && (p.ParentProfileID != "" || p.TriggerEvidenceRef != "") {
|
||||||
|
return errors.New("general task evaluation profile cannot have an expansion parent")
|
||||||
|
}
|
||||||
|
if p.Stage == TaskEvaluationAdditional && (p.ParentProfileID == "" || p.TriggerEvidenceRef == "") {
|
||||||
|
return errors.New("additional task evaluation profile requires parent and trigger evidence")
|
||||||
|
}
|
||||||
|
if p.QualityWeightMillis < 0 || p.ReliabilityWeightMillis < 0 || p.LatencyWeightMillis < 0 || p.QualityWeightMillis+p.ReliabilityWeightMillis+p.LatencyWeightMillis != 1000 {
|
||||||
|
return errors.New("task evaluation profile weights must be non-negative and sum to 1000")
|
||||||
|
}
|
||||||
|
seen := make(map[string]struct{}, len(p.RequiredCaseIDs))
|
||||||
|
for _, caseID := range p.RequiredCaseIDs {
|
||||||
|
if caseID == "" {
|
||||||
|
return errors.New("task evaluation profile contains an empty case")
|
||||||
|
}
|
||||||
|
if _, ok := seen[caseID]; ok {
|
||||||
|
return errors.New("task evaluation profile contains duplicate cases")
|
||||||
|
}
|
||||||
|
seen[caseID] = struct{}{}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// TaskEvaluationExpansionRequest is emitted after a task remains unresolved
|
||||||
|
// with the current best eligible candidate. It creates only a bounded new
|
||||||
|
// profile; it never silently rewrites the general benchmark.
|
||||||
|
type TaskEvaluationExpansionRequest struct {
|
||||||
|
ParentProfileID TaskEvaluationProfileID
|
||||||
|
CapabilityID CapabilityID
|
||||||
|
TriggerEvidenceRef string
|
||||||
|
NewCaseIDs []string
|
||||||
|
BenchmarkVersion string
|
||||||
|
}
|
||||||
|
|
||||||
|
// TaskCaseObservation is produced by the bounded evaluator and verifier. A
|
||||||
|
// score without a case result and evidence reference is not accepted as
|
||||||
|
// qualification evidence.
|
||||||
|
type TaskCaseObservation struct {
|
||||||
|
CaseID string
|
||||||
|
Result EvaluationResult
|
||||||
|
QualityScoreMillis int64
|
||||||
|
LatencyMicros int64
|
||||||
|
CriticalFailure bool
|
||||||
|
EvidenceArtifactRef string
|
||||||
|
}
|
||||||
|
|
||||||
|
// TaskEvaluation is durable evidence for one model/binding revision under one
|
||||||
|
// task profile. It is not activation authority.
|
||||||
|
type TaskEvaluation struct {
|
||||||
|
ID TaskEvaluationID
|
||||||
|
ProfileID TaskEvaluationProfileID
|
||||||
|
BindingID CapabilityBindingID
|
||||||
|
BindingRevision int64
|
||||||
|
CapabilityID CapabilityID
|
||||||
|
Result EvaluationResult
|
||||||
|
QualityScoreMillis int64
|
||||||
|
ReliabilityScoreMillis int64
|
||||||
|
LatencyScoreMillis int64
|
||||||
|
OverallScoreMillis int64
|
||||||
|
CriticalFailureMillis int64
|
||||||
|
BenchmarkVersion string
|
||||||
|
ArtifactID artifact.ArtifactID
|
||||||
|
ObservedAtUnixMillis int64
|
||||||
|
}
|
||||||
@@ -0,0 +1,35 @@
|
|||||||
|
BEGIN;
|
||||||
|
SET ROLE capability_role;
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS capability.cognitive_task_evaluations (
|
||||||
|
evaluation_id TEXT PRIMARY KEY,
|
||||||
|
task_profile_id TEXT NOT NULL,
|
||||||
|
binding_id TEXT NOT NULL,
|
||||||
|
binding_revision BIGINT NOT NULL,
|
||||||
|
capability_id TEXT NOT NULL,
|
||||||
|
result TEXT NOT NULL,
|
||||||
|
quality_score_millis BIGINT NOT NULL,
|
||||||
|
reliability_score_millis BIGINT NOT NULL,
|
||||||
|
latency_score_millis BIGINT NOT NULL,
|
||||||
|
overall_score_millis BIGINT NOT NULL,
|
||||||
|
critical_failure_millis BIGINT NOT NULL,
|
||||||
|
benchmark_version TEXT NOT NULL,
|
||||||
|
artifact_id TEXT NOT NULL,
|
||||||
|
observed_at_unix_millis BIGINT NOT NULL,
|
||||||
|
recorded_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||||
|
CHECK (binding_revision > 0),
|
||||||
|
CHECK (result IN ('UNKNOWN','SUPPORTED','UNSUPPORTED')),
|
||||||
|
CHECK (quality_score_millis BETWEEN 0 AND 1000),
|
||||||
|
CHECK (reliability_score_millis BETWEEN 0 AND 1000),
|
||||||
|
CHECK (latency_score_millis BETWEEN 0 AND 1000),
|
||||||
|
CHECK (overall_score_millis BETWEEN 0 AND 1000),
|
||||||
|
CHECK (critical_failure_millis BETWEEN 0 AND 1000),
|
||||||
|
CHECK (observed_at_unix_millis > 0)
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE INDEX IF NOT EXISTS cognitive_task_evaluations_lookup
|
||||||
|
ON capability.cognitive_task_evaluations(task_profile_id, binding_id, binding_revision, observed_at_unix_millis DESC);
|
||||||
|
|
||||||
|
GRANT SELECT, INSERT ON capability.cognitive_task_evaluations TO capability_role;
|
||||||
|
RESET ROLE;
|
||||||
|
COMMIT;
|
||||||
@@ -1,5 +1,5 @@
|
|||||||
{
|
{
|
||||||
"task_file": "dev/tasks/DEV-000207-gpu-invocation-qualification.json",
|
"task_file": "dev/tasks/DEV-000208-task-aware-model-evaluation.json",
|
||||||
"task_id": "DEV-000207",
|
"task_id": "DEV-000208",
|
||||||
"task_revision": 1
|
"task_revision": 1
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,23 @@
|
|||||||
|
{
|
||||||
|
"task_id": "DEV-000208",
|
||||||
|
"task_revision": 1,
|
||||||
|
"architecture_revision": "ADR-0018",
|
||||||
|
"base_repo_revision": "b78a026ee169047feb8468f0799fa1b9f4e27562",
|
||||||
|
"risk_tier": "E",
|
||||||
|
"intent": "Make Domovoy's model qualification and routing task-aware through parallel durable specialist work: typed evaluation profiles/evidence, durable task scores, and explainable selection without hardware/provider heuristics.",
|
||||||
|
"requirement_ids": ["task-specific model evaluation", "parallel specialist evaluation", "no main execution thread", "model fleet/failover", "hardware-neutral routing", "explainable model rating"],
|
||||||
|
"allowed_change_scope": {"modules": ["capability", "core-runtime"], "paths": ["contracts/capability/**", "modules/capability/**", "adapters/postgres/capability_repository.go", "cmd/core-runtime/main.go", "database/migrations/039_capability_task_evaluations.sql", "tests/model_qualification/**", "dev/ACTIVE_TASK.json", "dev/tasks/DEV-000208-task-aware-model-evaluation.json"]},
|
||||||
|
"forbidden_change_scope": {"paths": ["architecture/**", "security/**", "deploy/**", ".gitea/**", "contracts/authority/**", "modules/identity/**"]},
|
||||||
|
"allowed_architecture_delta": {"new_cross_module_dependencies": 0, "new_canonical_owners": 0, "new_privileges": 0, "new_third_party_dependencies": 0, "new_database_owner_roles": 1},
|
||||||
|
"contracts": {"consume": ["CognitiveRequirement", "ResolvedCognitiveBinding", "existing capability evidence", "existing capability owner repository", "Work Graph durable fan-out/fan-in"], "may_change": ["task profile/evaluation contracts", "task-aware selection request/response serialization"]},
|
||||||
|
"semantic_owner": "capability",
|
||||||
|
"canonical_state": ["capability task evaluation evidence keyed by task profile and binding revision"],
|
||||||
|
"state_transitions": ["GENERAL profile -> first bounded evaluation", "unresolved task evidence -> ADDITIONAL profile with parent and trigger evidence", "profile result UNKNOWN -> SUPPORTED/UNSUPPORTED; evidence is append-only by binding revision", "task-aware requirement -> eligible candidates with explainable task rating", "evaluation request -> durable fan-out -> specialist evidence -> fan-in aggregate"],
|
||||||
|
"side_effects_and_privileges": ["core capability database role writes only; no provider calls and no secret access in this task"],
|
||||||
|
"persistence_and_migrations": ["039 adds capability.cognitive_task_evaluations owned by capability"],
|
||||||
|
"failure_retry_restart": ["missing, stale or unknown task evidence excludes a task-aware candidate; no score defaults to success", "one specialist failure does not block unrelated tasks; stale leases are recoverable and fan-in preserves partial/unknown state"],
|
||||||
|
"assumptions": ["a later Work Graph evaluation runner will produce per-case evidence; this task establishes the durable contract and routing gate", "the chat/request path never waits synchronously for the full evaluation graph"],
|
||||||
|
"acceptance_scenarios": ["general profile runs before additional profiles", "an unresolved task creates only a bounded additional profile linked to failure evidence", "task-aware requirement rejects a model without current supported task evidence", "task scores are included in explainable rating", "provider kind, endpoint and hardware do not affect score", "unknown/failed task evidence never silently becomes supported", "existing requirements without a task profile retain current behavior"],
|
||||||
|
"required_tests": ["go test", "go vet", "preflight", "check-diff", "check-architecture", "independent", "git diff --check", "migration/schema checks", "CI", "production public health check"],
|
||||||
|
"terminal_gap_states": ["BLOCKED_BY_SPEC_GAP", "BLOCKED_BY_ARCHITECTURE_GAP", "BLOCKED_BY_MISSING_CAPABILITY"]
|
||||||
|
}
|
||||||
@@ -148,6 +148,9 @@ func SelectCognitiveBinding(requirement capability.CognitiveRequirement, candida
|
|||||||
if candidate.Evaluation != nil && candidate.Evaluation.Result == capability.EvaluationUnsupported {
|
if candidate.Evaluation != nil && candidate.Evaluation.Result == capability.EvaluationUnsupported {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
if requirement.TaskProfileID != "" && (candidate.TaskEvaluation == nil || candidate.TaskEvaluation.ProfileID != requirement.TaskProfileID || candidate.TaskEvaluation.Result != capability.EvaluationSupported) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
rating := RateCognitiveBinding(candidate)
|
rating := RateCognitiveBinding(candidate)
|
||||||
candidate.Rating = &rating
|
candidate.Rating = &rating
|
||||||
if requirement.MaxCostMicros != nil {
|
if requirement.MaxCostMicros != nil {
|
||||||
@@ -171,6 +174,19 @@ func SelectCognitiveBinding(requirement capability.CognitiveRequirement, candida
|
|||||||
// evidence. Provider kind, endpoint name and hardware are intentionally not
|
// evidence. Provider kind, endpoint name and hardware are intentionally not
|
||||||
// inputs to the score.
|
// inputs to the score.
|
||||||
func RateCognitiveBinding(candidate capability.ResolvedCognitiveBinding) capability.ModelRating {
|
func RateCognitiveBinding(candidate capability.ResolvedCognitiveBinding) capability.ModelRating {
|
||||||
|
if candidate.TaskEvaluation != nil {
|
||||||
|
evaluation := candidate.TaskEvaluation
|
||||||
|
if evaluation.Result != capability.EvaluationSupported {
|
||||||
|
return capability.ModelRating{TaskProfileID: evaluation.ProfileID, Category: capability.ModelRatingUnrated}
|
||||||
|
}
|
||||||
|
category := capability.ModelRatingBalanced
|
||||||
|
if evaluation.OverallScoreMillis >= 850 && evaluation.QualityScoreMillis >= 800 {
|
||||||
|
category = capability.ModelRatingQuality
|
||||||
|
} else if evaluation.LatencyScoreMillis >= 800 && evaluation.OverallScoreMillis >= 700 {
|
||||||
|
category = capability.ModelRatingFast
|
||||||
|
}
|
||||||
|
return capability.ModelRating{TaskProfileID: evaluation.ProfileID, TaskScoreMillis: evaluation.OverallScoreMillis, Category: category, OverallScoreMillis: evaluation.OverallScoreMillis, QualityScoreMillis: evaluation.QualityScoreMillis, ReliabilityScoreMillis: evaluation.ReliabilityScoreMillis, LatencyScoreMillis: evaluation.LatencyScoreMillis, EvidenceScoreMillis: 1000}
|
||||||
|
}
|
||||||
if candidate.Evaluation == nil || candidate.Evaluation.Result != capability.EvaluationSupported || candidate.Evaluation.QualityScoreMillis == nil || candidate.Evaluation.CriticalFailureMillis == nil {
|
if candidate.Evaluation == nil || candidate.Evaluation.Result != capability.EvaluationSupported || candidate.Evaluation.QualityScoreMillis == nil || candidate.Evaluation.CriticalFailureMillis == nil {
|
||||||
return capability.ModelRating{Category: capability.ModelRatingUnrated}
|
return capability.ModelRating{Category: capability.ModelRatingUnrated}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,106 @@
|
|||||||
|
package domain
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
|
||||||
|
"git.cin.su/m/Assistant/contracts/artifact"
|
||||||
|
"git.cin.su/m/Assistant/contracts/capability"
|
||||||
|
)
|
||||||
|
|
||||||
|
var ErrInvalidTaskEvaluation = errors.New("invalid task evaluation")
|
||||||
|
|
||||||
|
func BuildAdditionalTaskEvaluationProfile(parent capability.TaskEvaluationProfile, request capability.TaskEvaluationExpansionRequest, profileID capability.TaskEvaluationProfileID, revision int64) (capability.TaskEvaluationProfile, error) {
|
||||||
|
if err := parent.Validate(); err != nil || parent.Stage != capability.TaskEvaluationGeneral || request.ParentProfileID != parent.ID || request.CapabilityID != parent.CapabilityID || request.TriggerEvidenceRef == "" || len(request.NewCaseIDs) == 0 || profileID == "" || revision < 1 || request.BenchmarkVersion == "" {
|
||||||
|
return capability.TaskEvaluationProfile{}, ErrInvalidTaskEvaluation
|
||||||
|
}
|
||||||
|
known := make(map[string]struct{}, len(parent.RequiredCaseIDs))
|
||||||
|
for _, caseID := range parent.RequiredCaseIDs {
|
||||||
|
known[caseID] = struct{}{}
|
||||||
|
}
|
||||||
|
seen := make(map[string]struct{}, len(request.NewCaseIDs))
|
||||||
|
for _, caseID := range request.NewCaseIDs {
|
||||||
|
if caseID == "" {
|
||||||
|
return capability.TaskEvaluationProfile{}, ErrInvalidTaskEvaluation
|
||||||
|
}
|
||||||
|
if _, exists := known[caseID]; exists {
|
||||||
|
return capability.TaskEvaluationProfile{}, ErrInvalidTaskEvaluation
|
||||||
|
}
|
||||||
|
if _, exists := seen[caseID]; exists {
|
||||||
|
return capability.TaskEvaluationProfile{}, ErrInvalidTaskEvaluation
|
||||||
|
}
|
||||||
|
seen[caseID] = struct{}{}
|
||||||
|
}
|
||||||
|
return capability.TaskEvaluationProfile{ID: profileID, Revision: revision, CapabilityID: parent.CapabilityID, Stage: capability.TaskEvaluationAdditional, ParentProfileID: parent.ID, TriggerEvidenceRef: request.TriggerEvidenceRef, BenchmarkVersion: request.BenchmarkVersion, RequiredCaseIDs: append([]string(nil), request.NewCaseIDs...), QualityWeightMillis: parent.QualityWeightMillis, ReliabilityWeightMillis: parent.ReliabilityWeightMillis, LatencyWeightMillis: parent.LatencyWeightMillis}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// BuildTaskEvaluation aggregates independently observed case results. It is
|
||||||
|
// intentionally deterministic: no provider name, hardware label or caller
|
||||||
|
// supplied aggregate score can influence the result.
|
||||||
|
func BuildTaskEvaluation(profile capability.TaskEvaluationProfile, evaluationID capability.TaskEvaluationID, bindingID capability.CapabilityBindingID, bindingRevision int64, capabilityID capability.CapabilityID, observations []capability.TaskCaseObservation, artifactID artifact.ArtifactID, observedAtUnixMillis int64) (capability.TaskEvaluation, error) {
|
||||||
|
if err := profile.Validate(); err != nil || evaluationID == "" || bindingID == "" || bindingRevision < 1 || capabilityID == "" || capabilityID != profile.CapabilityID || artifactID == "" || observedAtUnixMillis < 1 || len(observations) != len(profile.RequiredCaseIDs) {
|
||||||
|
return capability.TaskEvaluation{}, ErrInvalidTaskEvaluation
|
||||||
|
}
|
||||||
|
required := make(map[string]struct{}, len(profile.RequiredCaseIDs))
|
||||||
|
for _, caseID := range profile.RequiredCaseIDs {
|
||||||
|
required[caseID] = struct{}{}
|
||||||
|
}
|
||||||
|
seen := make(map[string]struct{}, len(observations))
|
||||||
|
qualityTotal, latencyTotal, passed, critical := int64(0), int64(0), int64(0), int64(0)
|
||||||
|
unknown := false
|
||||||
|
unsupported := false
|
||||||
|
for _, observation := range observations {
|
||||||
|
if _, ok := required[observation.CaseID]; !ok {
|
||||||
|
return capability.TaskEvaluation{}, ErrInvalidTaskEvaluation
|
||||||
|
}
|
||||||
|
if _, ok := seen[observation.CaseID]; ok || observation.EvidenceArtifactRef == "" || observation.QualityScoreMillis < 0 || observation.QualityScoreMillis > 1000 || observation.LatencyMicros < 0 {
|
||||||
|
return capability.TaskEvaluation{}, ErrInvalidTaskEvaluation
|
||||||
|
}
|
||||||
|
seen[observation.CaseID] = struct{}{}
|
||||||
|
qualityTotal += observation.QualityScoreMillis
|
||||||
|
latencyTotal += observation.LatencyMicros
|
||||||
|
if observation.CriticalFailure {
|
||||||
|
critical++
|
||||||
|
}
|
||||||
|
switch observation.Result {
|
||||||
|
case capability.EvaluationSupported:
|
||||||
|
passed++
|
||||||
|
case capability.EvaluationUnsupported:
|
||||||
|
unsupported = true
|
||||||
|
case capability.EvaluationUnknown:
|
||||||
|
unknown = true
|
||||||
|
default:
|
||||||
|
return capability.TaskEvaluation{}, ErrInvalidTaskEvaluation
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(seen) != len(required) {
|
||||||
|
return capability.TaskEvaluation{}, ErrInvalidTaskEvaluation
|
||||||
|
}
|
||||||
|
count := int64(len(observations))
|
||||||
|
quality := qualityTotal / count
|
||||||
|
reliability := passed * 1000 / count
|
||||||
|
latency := latencyScore(latencyTotal / count)
|
||||||
|
criticalMillis := critical * 1000 / count
|
||||||
|
overall := (quality*profile.QualityWeightMillis + reliability*profile.ReliabilityWeightMillis + latency*profile.LatencyWeightMillis) / 1000
|
||||||
|
result := capability.EvaluationSupported
|
||||||
|
if unknown {
|
||||||
|
result = capability.EvaluationUnknown
|
||||||
|
} else if unsupported {
|
||||||
|
result = capability.EvaluationUnsupported
|
||||||
|
}
|
||||||
|
return capability.TaskEvaluation{ID: evaluationID, ProfileID: profile.ID, BindingID: bindingID, BindingRevision: bindingRevision, CapabilityID: capabilityID, Result: result, QualityScoreMillis: quality, ReliabilityScoreMillis: reliability, LatencyScoreMillis: latency, OverallScoreMillis: overall, CriticalFailureMillis: criticalMillis, BenchmarkVersion: profile.BenchmarkVersion, ArtifactID: artifactID, ObservedAtUnixMillis: observedAtUnixMillis}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func ValidateTaskEvaluation(value capability.TaskEvaluation) error {
|
||||||
|
if value.ID == "" || value.ProfileID == "" || value.BindingID == "" || value.BindingRevision < 1 || value.CapabilityID == "" || value.BenchmarkVersion == "" || value.ArtifactID == "" || value.ObservedAtUnixMillis < 1 {
|
||||||
|
return ErrInvalidTaskEvaluation
|
||||||
|
}
|
||||||
|
if value.Result != capability.EvaluationUnknown && value.Result != capability.EvaluationSupported && value.Result != capability.EvaluationUnsupported {
|
||||||
|
return ErrInvalidTaskEvaluation
|
||||||
|
}
|
||||||
|
for _, score := range []int64{value.QualityScoreMillis, value.ReliabilityScoreMillis, value.LatencyScoreMillis, value.OverallScoreMillis, value.CriticalFailureMillis} {
|
||||||
|
if score < 0 || score > 1000 {
|
||||||
|
return ErrInvalidTaskEvaluation
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,35 @@
|
|||||||
|
package domain
|
||||||
|
|
||||||
|
import (
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"git.cin.su/m/Assistant/contracts/artifact"
|
||||||
|
"git.cin.su/m/Assistant/contracts/capability"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestBuildTaskEvaluationAggregatesOnlyProfileCases(t *testing.T) {
|
||||||
|
profile := capability.TaskEvaluationProfile{ID: "research-v1", Revision: 1, CapabilityID: "creator-advisory", Stage: capability.TaskEvaluationGeneral, BenchmarkVersion: "research-v1", RequiredCaseIDs: []string{"claim", "uncertainty"}, QualityWeightMillis: 600, ReliabilityWeightMillis: 300, LatencyWeightMillis: 100}
|
||||||
|
value, err := BuildTaskEvaluation(profile, "evaluation-1", "binding-1", 2, "creator-advisory", []capability.TaskCaseObservation{
|
||||||
|
{CaseID: "claim", Result: capability.EvaluationSupported, QualityScoreMillis: 900, LatencyMicros: 400_000, EvidenceArtifactRef: "artifact-case-1"},
|
||||||
|
{CaseID: "uncertainty", Result: capability.EvaluationSupported, QualityScoreMillis: 800, LatencyMicros: 600_000, EvidenceArtifactRef: "artifact-case-2"},
|
||||||
|
}, artifact.ArtifactID("artifact-summary"), 10)
|
||||||
|
if err != nil || value.Result != capability.EvaluationSupported || value.OverallScoreMillis == 0 || value.ProfileID != profile.ID {
|
||||||
|
t.Fatalf("evaluation=%+v err=%v", value, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBuildTaskEvaluationDoesNotPromoteUnknown(t *testing.T) {
|
||||||
|
profile := capability.TaskEvaluationProfile{ID: "coding-v1", Revision: 1, CapabilityID: "creator-advisory", Stage: capability.TaskEvaluationGeneral, BenchmarkVersion: "coding-v1", RequiredCaseIDs: []string{"compile"}, QualityWeightMillis: 700, ReliabilityWeightMillis: 200, LatencyWeightMillis: 100}
|
||||||
|
value, err := BuildTaskEvaluation(profile, "evaluation-2", "binding-1", 2, "creator-advisory", []capability.TaskCaseObservation{{CaseID: "compile", Result: capability.EvaluationUnknown, QualityScoreMillis: 0, LatencyMicros: 0, EvidenceArtifactRef: "artifact-case"}}, artifact.ArtifactID("artifact-summary"), 11)
|
||||||
|
if err != nil || value.Result != capability.EvaluationUnknown {
|
||||||
|
t.Fatalf("evaluation=%+v err=%v", value, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBuildAdditionalProfileRequiresUnresolvedEvidenceAndPreservesHistory(t *testing.T) {
|
||||||
|
parent := capability.TaskEvaluationProfile{ID: "general-v1", Revision: 1, CapabilityID: "creator-advisory", Stage: capability.TaskEvaluationGeneral, BenchmarkVersion: "general-v1", RequiredCaseIDs: []string{"general"}, QualityWeightMillis: 700, ReliabilityWeightMillis: 200, LatencyWeightMillis: 100}
|
||||||
|
profile, err := BuildAdditionalTaskEvaluationProfile(parent, capability.TaskEvaluationExpansionRequest{ParentProfileID: parent.ID, CapabilityID: parent.CapabilityID, TriggerEvidenceRef: "experience-unsolved-1", NewCaseIDs: []string{"edge-case"}, BenchmarkVersion: "additional-v1"}, "additional-v1", 1)
|
||||||
|
if err != nil || profile.Stage != capability.TaskEvaluationAdditional || profile.ParentProfileID != parent.ID || profile.RequiredCaseIDs[0] != "edge-case" {
|
||||||
|
t.Fatalf("profile=%+v err=%v", profile, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -18,6 +18,7 @@ type Repository interface {
|
|||||||
SaveModelDescriptor(context.Context, capability.ModelDescriptor) error
|
SaveModelDescriptor(context.Context, capability.ModelDescriptor) error
|
||||||
SaveCognitiveBinding(context.Context, capability.CognitiveBinding) error
|
SaveCognitiveBinding(context.Context, capability.CognitiveBinding) error
|
||||||
LoadCognitiveCandidates(context.Context, capability.CapabilityID, identity.PrincipalID) ([]capability.ResolvedCognitiveBinding, error)
|
LoadCognitiveCandidates(context.Context, capability.CapabilityID, identity.PrincipalID) ([]capability.ResolvedCognitiveBinding, error)
|
||||||
|
LoadCognitiveCandidatesForProfile(context.Context, capability.CapabilityID, identity.PrincipalID, capability.TaskEvaluationProfileID) ([]capability.ResolvedCognitiveBinding, error)
|
||||||
LoadCognitiveCandidate(context.Context, capability.CapabilityBindingID, int64) (capability.ResolvedCognitiveBinding, error)
|
LoadCognitiveCandidate(context.Context, capability.CapabilityBindingID, int64) (capability.ResolvedCognitiveBinding, error)
|
||||||
SaveCognitiveCanaryPromotion(context.Context, capability.CognitiveCanaryPromotion) error
|
SaveCognitiveCanaryPromotion(context.Context, capability.CognitiveCanaryPromotion) error
|
||||||
SaveCognitiveActivationPromotion(context.Context, capability.CognitiveActivationPromotion) error
|
SaveCognitiveActivationPromotion(context.Context, capability.CognitiveActivationPromotion) error
|
||||||
@@ -25,4 +26,5 @@ type Repository interface {
|
|||||||
SaveCognitiveDiscovery(context.Context, capability.CognitiveDiscoveryRegistration) error
|
SaveCognitiveDiscovery(context.Context, capability.CognitiveDiscoveryRegistration) error
|
||||||
SaveEvaluatedCapability(context.Context, capability.EvaluatedCapability) error
|
SaveEvaluatedCapability(context.Context, capability.EvaluatedCapability) error
|
||||||
SavePerformanceEvidence(context.Context, capability.PerformanceEvidence) error
|
SavePerformanceEvidence(context.Context, capability.PerformanceEvidence) error
|
||||||
|
SaveTaskEvaluation(context.Context, capability.TaskEvaluation) error
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user