Assistant/contracts/capability/task_evaluation.go

package capability

import (
	"errors"

	"git.cin.su/m/Assistant/contracts/artifact"
	"git.cin.su/m/Assistant/platform/ids"
)

type TaskEvaluationProfileID ids.ID
type TaskEvaluationID ids.ID
type TaskEvaluationProfileStage string

const (
	TaskEvaluationGeneral    TaskEvaluationProfileStage = "GENERAL"
	TaskEvaluationAdditional TaskEvaluationProfileStage = "ADDITIONAL"
)

// TaskEvaluationProfile is the extensible, provider-neutral description of a
// task-specific qualification run. Case payloads live in Artifact; the
// profile stores only stable case references and scoring policy.
type TaskEvaluationProfile struct {
	ID                      TaskEvaluationProfileID
	Revision                int64
	CapabilityID            CapabilityID
	Stage                   TaskEvaluationProfileStage
	ParentProfileID         TaskEvaluationProfileID
	TriggerEvidenceRef      string
	BenchmarkVersion        string
	RequiredCaseIDs         []string
	QualityWeightMillis     int64
	ReliabilityWeightMillis int64
	LatencyWeightMillis     int64
}

func (p TaskEvaluationProfile) Validate() error {
	if p.ID == "" || p.Revision < 1 || p.CapabilityID == "" || (p.Stage != TaskEvaluationGeneral && p.Stage != TaskEvaluationAdditional) || p.BenchmarkVersion == "" || len(p.RequiredCaseIDs) == 0 {
		return errors.New("task evaluation profile identity and cases are incomplete")
	}
	if p.Stage == TaskEvaluationGeneral && (p.ParentProfileID != "" || p.TriggerEvidenceRef != "") {
		return errors.New("general task evaluation profile cannot have an expansion parent")
	}
	if p.Stage == TaskEvaluationAdditional && (p.ParentProfileID == "" || p.TriggerEvidenceRef == "") {
		return errors.New("additional task evaluation profile requires parent and trigger evidence")
	}
	if p.QualityWeightMillis < 0 || p.ReliabilityWeightMillis < 0 || p.LatencyWeightMillis < 0 || p.QualityWeightMillis+p.ReliabilityWeightMillis+p.LatencyWeightMillis != 1000 {
		return errors.New("task evaluation profile weights must be non-negative and sum to 1000")
	}
	seen := make(map[string]struct{}, len(p.RequiredCaseIDs))
	for _, caseID := range p.RequiredCaseIDs {
		if caseID == "" {
			return errors.New("task evaluation profile contains an empty case")
		}
		if _, ok := seen[caseID]; ok {
			return errors.New("task evaluation profile contains duplicate cases")
		}
		seen[caseID] = struct{}{}
	}
	return nil
}

// TaskEvaluationExpansionRequest is emitted after a task remains unresolved
// with the current best eligible candidate. It creates only a bounded new
// profile; it never silently rewrites the general benchmark.
type TaskEvaluationExpansionRequest struct {
	ParentProfileID    TaskEvaluationProfileID
	CapabilityID       CapabilityID
	TriggerEvidenceRef string
	NewCaseIDs         []string
	BenchmarkVersion   string
}

type TaskCaseVerificationMode string

const (
	TaskCaseVerificationExactText       TaskCaseVerificationMode = "EXACT_TEXT"
	TaskCaseVerificationJSONFieldEquals TaskCaseVerificationMode = "JSON_FIELD_EQUALS"
)

// TaskCaseVerification is a bounded, task-owned rubric. The expected text is
// test data, not model output, and is never sent to the cognitive provider.
type TaskCaseVerification struct {
	Mode                 TaskCaseVerificationMode
	FieldPath            string
	ExpectedText         string
	MinimumQualityMillis int64
}

// TaskCaseObservation is produced by the bounded evaluator and verifier. A
// score without a case result and evidence reference is not accepted as
// qualification evidence.
type TaskCaseObservation struct {
	CaseID              string
	Result              EvaluationResult
	QualityScoreMillis  int64
	LatencyMicros       int64
	CriticalFailure     bool
	EvidenceArtifactRef string
	// VerificationEvidenceRef and VerifierIdentity are required whenever an
	// observation claims semantic support or a non-zero quality score.
	VerificationEvidenceRef string
	VerifierIdentity        string
}

// TaskEvaluation is durable evidence for one model/binding revision under one
// task profile. It is not activation authority.
type TaskEvaluation struct {
	ID                     TaskEvaluationID
	ProfileID              TaskEvaluationProfileID
	BindingID              CapabilityBindingID
	BindingRevision        int64
	CapabilityID           CapabilityID
	Result                 EvaluationResult
	QualityScoreMillis     int64
	ReliabilityScoreMillis int64
	LatencyScoreMillis     int64
	OverallScoreMillis     int64
	CriticalFailureMillis  int64
	BenchmarkVersion       string
	ArtifactID             artifact.ArtifactID
	ObservedAtUnixMillis   int64
}