package domain import ( "errors" "fmt" "strings" "git.cin.su/m/Assistant/contracts/artifact" "git.cin.su/m/Assistant/contracts/capability" ) var ErrInvalidTaskEvaluation = errors.New("invalid task evaluation") type UnresolvedTaskEvidence struct { ID string OriginalPrompt string FailureSummary string EvidenceArtifactRef string } // BuildAdditionalTaskEvaluationCases creates two bounded probes from one // unresolved outcome. It creates prompts only; quality still requires a // verification spec and independent verifier evidence. func BuildAdditionalTaskEvaluationCases(value UnresolvedTaskEvidence) ([]capability.TaskEvaluationCase, error) { if strings.TrimSpace(value.ID) == "" || strings.TrimSpace(value.OriginalPrompt) == "" || strings.TrimSpace(value.FailureSummary) == "" || strings.TrimSpace(value.EvidenceArtifactRef) == "" { return nil, ErrInvalidTaskEvaluation } base := strings.TrimSpace(value.OriginalPrompt) failure := strings.TrimSpace(value.FailureSummary) return []capability.TaskEvaluationCase{ {CaseID: value.ID + ":reproduce", Prompt: base, EvidenceArtifactRef: value.EvidenceArtifactRef}, {CaseID: value.ID + ":address-failure", Prompt: base + "\n\nAddress this observed unresolved condition without assuming it is true: " + failure, EvidenceArtifactRef: value.EvidenceArtifactRef}, }, nil } const automaticExpansionMinimumOverallScore int64 = 700 // AutomaticTaskEvaluationExpansion decides whether a completed general // evaluation has evidence that warrants a bounded additional profile. It is // deliberately conservative: one supported candidate above the quality // threshold suppresses expansion, while unknown/unsupported/low-scoring or // critically failing candidates keep the unresolved path explicit. func AutomaticTaskEvaluationExpansion(parent capability.TaskEvaluationProfile, parentGraphState string, evaluations []capability.TaskEvaluation, request capability.TaskEvaluationExpansionRequest, profileID capability.TaskEvaluationProfileID, revision int64) (capability.TaskEvaluationProfile, bool, error) { if parentGraphState != "COMPLETED" && parentGraphState != "PARTIAL" { return capability.TaskEvaluationProfile{}, false, nil } if len(evaluations) == 0 { return capability.TaskEvaluationProfile{}, false, nil } for _, evaluation := range evaluations { if evaluation.Result == capability.EvaluationSupported && evaluation.OverallScoreMillis >= automaticExpansionMinimumOverallScore && evaluation.CriticalFailureMillis == 0 { return capability.TaskEvaluationProfile{}, false, nil } } if request.TriggerEvidenceRef == "" || len(request.NewCaseIDs) == 0 { return capability.TaskEvaluationProfile{}, false, fmt.Errorf("unresolved task evaluation requires trigger evidence and additional cases") } profile, err := BuildAdditionalTaskEvaluationProfile(parent, request, profileID, revision) if err != nil { return capability.TaskEvaluationProfile{}, false, err } return profile, true, nil } func BuildAdditionalTaskEvaluationProfile(parent capability.TaskEvaluationProfile, request capability.TaskEvaluationExpansionRequest, profileID capability.TaskEvaluationProfileID, revision int64) (capability.TaskEvaluationProfile, error) { if err := parent.Validate(); err != nil || parent.Stage != capability.TaskEvaluationGeneral || request.ParentProfileID != parent.ID || request.CapabilityID != parent.CapabilityID || request.TriggerEvidenceRef == "" || len(request.NewCaseIDs) == 0 || profileID == "" || revision < 1 || request.BenchmarkVersion == "" { return capability.TaskEvaluationProfile{}, ErrInvalidTaskEvaluation } known := make(map[string]struct{}, len(parent.RequiredCaseIDs)) for _, caseID := range parent.RequiredCaseIDs { known[caseID] = struct{}{} } seen := make(map[string]struct{}, len(request.NewCaseIDs)) for _, caseID := range request.NewCaseIDs { if caseID == "" { return capability.TaskEvaluationProfile{}, ErrInvalidTaskEvaluation } if _, exists := known[caseID]; exists { return capability.TaskEvaluationProfile{}, ErrInvalidTaskEvaluation } if _, exists := seen[caseID]; exists { return capability.TaskEvaluationProfile{}, ErrInvalidTaskEvaluation } seen[caseID] = struct{}{} } return capability.TaskEvaluationProfile{ID: profileID, Revision: revision, CapabilityID: parent.CapabilityID, Stage: capability.TaskEvaluationAdditional, ParentProfileID: parent.ID, TriggerEvidenceRef: request.TriggerEvidenceRef, BenchmarkVersion: request.BenchmarkVersion, RequiredCaseIDs: append([]string(nil), request.NewCaseIDs...), QualityWeightMillis: parent.QualityWeightMillis, ReliabilityWeightMillis: parent.ReliabilityWeightMillis, LatencyWeightMillis: parent.LatencyWeightMillis}, nil } // BuildTaskEvaluation aggregates independently observed case results. It is // intentionally deterministic: no provider name, hardware label or caller // supplied aggregate score can influence the result. func BuildTaskEvaluation(profile capability.TaskEvaluationProfile, evaluationID capability.TaskEvaluationID, bindingID capability.CapabilityBindingID, bindingRevision int64, capabilityID capability.CapabilityID, observations []capability.TaskCaseObservation, artifactID artifact.ArtifactID, observedAtUnixMillis int64) (capability.TaskEvaluation, error) { if err := profile.Validate(); err != nil || evaluationID == "" || bindingID == "" || bindingRevision < 1 || capabilityID == "" || capabilityID != profile.CapabilityID || artifactID == "" || observedAtUnixMillis < 1 || len(observations) != len(profile.RequiredCaseIDs) { return capability.TaskEvaluation{}, ErrInvalidTaskEvaluation } required := make(map[string]struct{}, len(profile.RequiredCaseIDs)) for _, caseID := range profile.RequiredCaseIDs { required[caseID] = struct{}{} } seen := make(map[string]struct{}, len(observations)) qualityTotal, latencyTotal, passed, critical := int64(0), int64(0), int64(0), int64(0) unknown := false unsupported := false for _, observation := range observations { if _, ok := required[observation.CaseID]; !ok { return capability.TaskEvaluation{}, ErrInvalidTaskEvaluation } if _, ok := seen[observation.CaseID]; ok || observation.EvidenceArtifactRef == "" || observation.QualityScoreMillis < 0 || observation.QualityScoreMillis > 1000 || observation.LatencyMicros < 0 { return capability.TaskEvaluation{}, ErrInvalidTaskEvaluation } if observation.Result == capability.EvaluationSupported && (observation.VerificationEvidenceRef == "" || observation.VerifierIdentity == "") { return capability.TaskEvaluation{}, ErrInvalidTaskEvaluation } if observation.Result == capability.EvaluationUnknown && observation.QualityScoreMillis != 0 { return capability.TaskEvaluation{}, ErrInvalidTaskEvaluation } seen[observation.CaseID] = struct{}{} qualityTotal += observation.QualityScoreMillis latencyTotal += observation.LatencyMicros if observation.CriticalFailure { critical++ } switch observation.Result { case capability.EvaluationSupported: passed++ case capability.EvaluationUnsupported: unsupported = true case capability.EvaluationUnknown: unknown = true default: return capability.TaskEvaluation{}, ErrInvalidTaskEvaluation } } if len(seen) != len(required) { return capability.TaskEvaluation{}, ErrInvalidTaskEvaluation } count := int64(len(observations)) quality := qualityTotal / count reliability := passed * 1000 / count latency := latencyScore(latencyTotal / count) criticalMillis := critical * 1000 / count overall := (quality*profile.QualityWeightMillis + reliability*profile.ReliabilityWeightMillis + latency*profile.LatencyWeightMillis) / 1000 result := capability.EvaluationSupported if unknown { result = capability.EvaluationUnknown } else if unsupported { result = capability.EvaluationUnsupported } return capability.TaskEvaluation{ID: evaluationID, ProfileID: profile.ID, BindingID: bindingID, BindingRevision: bindingRevision, CapabilityID: capabilityID, Result: result, QualityScoreMillis: quality, ReliabilityScoreMillis: reliability, LatencyScoreMillis: latency, OverallScoreMillis: overall, CriticalFailureMillis: criticalMillis, BenchmarkVersion: profile.BenchmarkVersion, ArtifactID: artifactID, ObservedAtUnixMillis: observedAtUnixMillis}, nil } func ValidateTaskEvaluation(value capability.TaskEvaluation) error { if value.ID == "" || value.ProfileID == "" || value.BindingID == "" || value.BindingRevision < 1 || value.CapabilityID == "" || value.BenchmarkVersion == "" || value.ArtifactID == "" || value.ObservedAtUnixMillis < 1 { return ErrInvalidTaskEvaluation } if value.Result != capability.EvaluationUnknown && value.Result != capability.EvaluationSupported && value.Result != capability.EvaluationUnsupported { return ErrInvalidTaskEvaluation } for _, score := range []int64{value.QualityScoreMillis, value.ReliabilityScoreMillis, value.LatencyScoreMillis, value.OverallScoreMillis, value.CriticalFailureMillis} { if score < 0 || score > 1000 { return ErrInvalidTaskEvaluation } } return nil }