Assistant/modules/capability/domain/task_evaluation.go

package domain

import (
	"errors"
	"fmt"
	"strings"

	"git.cin.su/m/Assistant/contracts/artifact"
	"git.cin.su/m/Assistant/contracts/capability"
)

var ErrInvalidTaskEvaluation = errors.New("invalid task evaluation")

type UnresolvedTaskEvidence struct {
	ID                  string
	OriginalPrompt      string
	FailureSummary      string
	EvidenceArtifactRef string
}

// BuildAdditionalTaskEvaluationCases creates two bounded probes from one
// unresolved outcome. It creates prompts only; quality still requires a
// verification spec and independent verifier evidence.
func BuildAdditionalTaskEvaluationCases(value UnresolvedTaskEvidence) ([]capability.TaskEvaluationCase, error) {
	if strings.TrimSpace(value.ID) == "" || strings.TrimSpace(value.OriginalPrompt) == "" || strings.TrimSpace(value.FailureSummary) == "" || strings.TrimSpace(value.EvidenceArtifactRef) == "" {
		return nil, ErrInvalidTaskEvaluation
	}
	base := strings.TrimSpace(value.OriginalPrompt)
	failure := strings.TrimSpace(value.FailureSummary)
	return []capability.TaskEvaluationCase{
		{CaseID: value.ID + ":reproduce", Prompt: base, EvidenceArtifactRef: value.EvidenceArtifactRef},
		{CaseID: value.ID + ":address-failure", Prompt: base + "\n\nAddress this observed unresolved condition without assuming it is true: " + failure, EvidenceArtifactRef: value.EvidenceArtifactRef},
	}, nil
}

const automaticExpansionMinimumOverallScore int64 = 700

// AutomaticTaskEvaluationExpansion decides whether a completed general
// evaluation has evidence that warrants a bounded additional profile. It is
// deliberately conservative: one supported candidate above the quality
// threshold suppresses expansion, while unknown/unsupported/low-scoring or
// critically failing candidates keep the unresolved path explicit.
func AutomaticTaskEvaluationExpansion(parent capability.TaskEvaluationProfile, parentGraphState string, evaluations []capability.TaskEvaluation, request capability.TaskEvaluationExpansionRequest, profileID capability.TaskEvaluationProfileID, revision int64) (capability.TaskEvaluationProfile, bool, error) {
	if parentGraphState != "COMPLETED" && parentGraphState != "PARTIAL" {
		return capability.TaskEvaluationProfile{}, false, nil
	}
	if len(evaluations) == 0 {
		return capability.TaskEvaluationProfile{}, false, nil
	}
	for _, evaluation := range evaluations {
		if evaluation.Result == capability.EvaluationSupported && evaluation.OverallScoreMillis >= automaticExpansionMinimumOverallScore && evaluation.CriticalFailureMillis == 0 {
			return capability.TaskEvaluationProfile{}, false, nil
		}
	}
	if request.TriggerEvidenceRef == "" || len(request.NewCaseIDs) == 0 {
		return capability.TaskEvaluationProfile{}, false, fmt.Errorf("unresolved task evaluation requires trigger evidence and additional cases")
	}
	profile, err := BuildAdditionalTaskEvaluationProfile(parent, request, profileID, revision)
	if err != nil {
		return capability.TaskEvaluationProfile{}, false, err
	}
	return profile, true, nil
}

func BuildAdditionalTaskEvaluationProfile(parent capability.TaskEvaluationProfile, request capability.TaskEvaluationExpansionRequest, profileID capability.TaskEvaluationProfileID, revision int64) (capability.TaskEvaluationProfile, error) {
	if err := parent.Validate(); err != nil || parent.Stage != capability.TaskEvaluationGeneral || request.ParentProfileID != parent.ID || request.CapabilityID != parent.CapabilityID || request.TriggerEvidenceRef == "" || len(request.NewCaseIDs) == 0 || profileID == "" || revision < 1 || request.BenchmarkVersion == "" {
		return capability.TaskEvaluationProfile{}, ErrInvalidTaskEvaluation
	}
	known := make(map[string]struct{}, len(parent.RequiredCaseIDs))
	for _, caseID := range parent.RequiredCaseIDs {
		known[caseID] = struct{}{}
	}
	seen := make(map[string]struct{}, len(request.NewCaseIDs))
	for _, caseID := range request.NewCaseIDs {
		if caseID == "" {
			return capability.TaskEvaluationProfile{}, ErrInvalidTaskEvaluation
		}
		if _, exists := known[caseID]; exists {
			return capability.TaskEvaluationProfile{}, ErrInvalidTaskEvaluation
		}
		if _, exists := seen[caseID]; exists {
			return capability.TaskEvaluationProfile{}, ErrInvalidTaskEvaluation
		}
		seen[caseID] = struct{}{}
	}
	return capability.TaskEvaluationProfile{ID: profileID, Revision: revision, CapabilityID: parent.CapabilityID, Stage: capability.TaskEvaluationAdditional, ParentProfileID: parent.ID, TriggerEvidenceRef: request.TriggerEvidenceRef, BenchmarkVersion: request.BenchmarkVersion, RequiredCaseIDs: append([]string(nil), request.NewCaseIDs...), QualityWeightMillis: parent.QualityWeightMillis, ReliabilityWeightMillis: parent.ReliabilityWeightMillis, LatencyWeightMillis: parent.LatencyWeightMillis}, nil
}

// BuildTaskEvaluation aggregates independently observed case results. It is
// intentionally deterministic: no provider name, hardware label or caller
// supplied aggregate score can influence the result.
func BuildTaskEvaluation(profile capability.TaskEvaluationProfile, evaluationID capability.TaskEvaluationID, bindingID capability.CapabilityBindingID, bindingRevision int64, capabilityID capability.CapabilityID, observations []capability.TaskCaseObservation, artifactID artifact.ArtifactID, observedAtUnixMillis int64) (capability.TaskEvaluation, error) {
	if err := profile.Validate(); err != nil || evaluationID == "" || bindingID == "" || bindingRevision < 1 || capabilityID == "" || capabilityID != profile.CapabilityID || artifactID == "" || observedAtUnixMillis < 1 || len(observations) != len(profile.RequiredCaseIDs) {
		return capability.TaskEvaluation{}, ErrInvalidTaskEvaluation
	}
	required := make(map[string]struct{}, len(profile.RequiredCaseIDs))
	for _, caseID := range profile.RequiredCaseIDs {
		required[caseID] = struct{}{}
	}
	seen := make(map[string]struct{}, len(observations))
	qualityTotal, latencyTotal, passed, critical := int64(0), int64(0), int64(0), int64(0)
	unknown := false
	unsupported := false
	for _, observation := range observations {
		if _, ok := required[observation.CaseID]; !ok {
			return capability.TaskEvaluation{}, ErrInvalidTaskEvaluation
		}
		if _, ok := seen[observation.CaseID]; ok || observation.EvidenceArtifactRef == "" || observation.QualityScoreMillis < 0 || observation.QualityScoreMillis > 1000 || observation.LatencyMicros < 0 {
			return capability.TaskEvaluation{}, ErrInvalidTaskEvaluation
		}
		if observation.Result == capability.EvaluationSupported && (observation.VerificationEvidenceRef == "" || observation.VerifierIdentity == "") {
			return capability.TaskEvaluation{}, ErrInvalidTaskEvaluation
		}
		if observation.Result == capability.EvaluationUnknown && observation.QualityScoreMillis != 0 {
			return capability.TaskEvaluation{}, ErrInvalidTaskEvaluation
		}
		seen[observation.CaseID] = struct{}{}
		qualityTotal += observation.QualityScoreMillis
		latencyTotal += observation.LatencyMicros
		if observation.CriticalFailure {
			critical++
		}
		switch observation.Result {
		case capability.EvaluationSupported:
			passed++
		case capability.EvaluationUnsupported:
			unsupported = true
		case capability.EvaluationUnknown:
			unknown = true
		default:
			return capability.TaskEvaluation{}, ErrInvalidTaskEvaluation
		}
	}
	if len(seen) != len(required) {
		return capability.TaskEvaluation{}, ErrInvalidTaskEvaluation
	}
	count := int64(len(observations))
	quality := qualityTotal / count
	reliability := passed * 1000 / count
	latency := latencyScore(latencyTotal / count)
	criticalMillis := critical * 1000 / count
	overall := (quality*profile.QualityWeightMillis + reliability*profile.ReliabilityWeightMillis + latency*profile.LatencyWeightMillis) / 1000
	result := capability.EvaluationSupported
	if unknown {
		result = capability.EvaluationUnknown
	} else if unsupported {
		result = capability.EvaluationUnsupported
	}
	return capability.TaskEvaluation{ID: evaluationID, ProfileID: profile.ID, BindingID: bindingID, BindingRevision: bindingRevision, CapabilityID: capabilityID, Result: result, QualityScoreMillis: quality, ReliabilityScoreMillis: reliability, LatencyScoreMillis: latency, OverallScoreMillis: overall, CriticalFailureMillis: criticalMillis, BenchmarkVersion: profile.BenchmarkVersion, ArtifactID: artifactID, ObservedAtUnixMillis: observedAtUnixMillis}, nil
}

func ValidateTaskEvaluation(value capability.TaskEvaluation) error {
	if value.ID == "" || value.ProfileID == "" || value.BindingID == "" || value.BindingRevision < 1 || value.CapabilityID == "" || value.BenchmarkVersion == "" || value.ArtifactID == "" || value.ObservedAtUnixMillis < 1 {
		return ErrInvalidTaskEvaluation
	}
	if value.Result != capability.EvaluationUnknown && value.Result != capability.EvaluationSupported && value.Result != capability.EvaluationUnsupported {
		return ErrInvalidTaskEvaluation
	}
	for _, score := range []int64{value.QualityScoreMillis, value.ReliabilityScoreMillis, value.LatencyScoreMillis, value.OverallScoreMillis, value.CriticalFailureMillis} {
		if score < 0 || score > 1000 {
			return ErrInvalidTaskEvaluation
		}
	}
	return nil
}