|
|
|
@@ -56,32 +56,32 @@ type CognitiveCaseResult struct {
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
type CognitiveArtifact struct {
|
|
|
|
|
ArtifactID string `json:"artifact_id"`
|
|
|
|
|
ArtifactVersion string `json:"artifact_version"`
|
|
|
|
|
ObservedAt string `json:"observed_at"`
|
|
|
|
|
SuiteID string `json:"suite_id"`
|
|
|
|
|
SuiteVersion string `json:"suite_version"`
|
|
|
|
|
ProviderInstanceID string `json:"provider_instance_id"`
|
|
|
|
|
ProviderModelID string `json:"provider_model_id"`
|
|
|
|
|
ModelDescriptorID string `json:"model_descriptor_id"`
|
|
|
|
|
BindingID string `json:"binding_id"`
|
|
|
|
|
BindingRevision int64 `json:"binding_revision"`
|
|
|
|
|
TotalCases int `json:"total_cases"`
|
|
|
|
|
DevelopmentCases int `json:"development_cases"`
|
|
|
|
|
HoldoutCases int `json:"holdout_cases"`
|
|
|
|
|
PassedCases int `json:"passed_cases"`
|
|
|
|
|
UnsupportedCertaintyFailures int `json:"unsupported_certainty_failures"`
|
|
|
|
|
AntiSycophancyFailures int `json:"anti_sycophancy_failures"`
|
|
|
|
|
UncertaintyFailures int `json:"uncertainty_failures"`
|
|
|
|
|
CapabilityFailures int `json:"capability_failures"`
|
|
|
|
|
ConsequenceFailures int `json:"consequence_failures"`
|
|
|
|
|
FabricatedObservationFailures int `json:"fabricated_observation_failures"`
|
|
|
|
|
FabricatedActionFailures int `json:"fabricated_action_failures"`
|
|
|
|
|
RelevanceFailures int `json:"relevance_failures"`
|
|
|
|
|
IntentPreservationFailures int `json:"intent_preservation_failures"`
|
|
|
|
|
HumanReviewStatus string `json:"human_review_status"`
|
|
|
|
|
EvaluatorIdentity string `json:"evaluator_identity"`
|
|
|
|
|
Cases []CognitiveCaseResult `json:"cases"`
|
|
|
|
|
ArtifactID string `json:"artifact_id"`
|
|
|
|
|
ArtifactVersion string `json:"artifact_version"`
|
|
|
|
|
ObservedAt string `json:"observed_at"`
|
|
|
|
|
SuiteID string `json:"suite_id"`
|
|
|
|
|
SuiteVersion string `json:"suite_version"`
|
|
|
|
|
ProviderInstanceID string `json:"provider_instance_id"`
|
|
|
|
|
ProviderModelID string `json:"provider_model_id"`
|
|
|
|
|
ModelDescriptorID string `json:"model_descriptor_id"`
|
|
|
|
|
BindingID string `json:"binding_id"`
|
|
|
|
|
BindingRevision int64 `json:"binding_revision"`
|
|
|
|
|
TotalCases int `json:"total_cases"`
|
|
|
|
|
DevelopmentCases int `json:"development_cases"`
|
|
|
|
|
HoldoutCases int `json:"holdout_cases"`
|
|
|
|
|
PassedCases int `json:"passed_cases"`
|
|
|
|
|
UnsupportedCertaintyFailures int `json:"unsupported_certainty_failures"`
|
|
|
|
|
AntiSycophancyFailures int `json:"anti_sycophancy_failures"`
|
|
|
|
|
UncertaintyFailures int `json:"uncertainty_failures"`
|
|
|
|
|
CapabilityFailures int `json:"capability_failures"`
|
|
|
|
|
ConsequenceFailures int `json:"consequence_failures"`
|
|
|
|
|
FabricatedObservationFailures int `json:"fabricated_observation_failures"`
|
|
|
|
|
FabricatedActionFailures int `json:"fabricated_action_failures"`
|
|
|
|
|
RelevanceFailures int `json:"relevance_failures"`
|
|
|
|
|
IntentPreservationFailures int `json:"intent_preservation_failures"`
|
|
|
|
|
HumanReviewStatus string `json:"human_review_status"`
|
|
|
|
|
EvaluatorIdentity string `json:"evaluator_identity"`
|
|
|
|
|
Cases []CognitiveCaseResult `json:"cases"`
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
func LoadCognitiveSuite(path string) (CognitiveSuite, error) {
|
|
|
|
@@ -236,11 +236,11 @@ func observeCognitiveResponse(response string) CognitiveObserved {
|
|
|
|
|
}
|
|
|
|
|
sort.Strings(signals)
|
|
|
|
|
return CognitiveObserved{
|
|
|
|
|
Response: response,
|
|
|
|
|
Signals: signals,
|
|
|
|
|
UnsupportedCertaintyRisk: unsupportedCertainty,
|
|
|
|
|
FabricatedObservationRisk: fabricatedObservation,
|
|
|
|
|
FabricatedActionRisk: fabricatedAction,
|
|
|
|
|
Response: response,
|
|
|
|
|
Signals: signals,
|
|
|
|
|
UnsupportedCertaintyRisk: unsupportedCertainty,
|
|
|
|
|
FabricatedObservationRisk: fabricatedObservation,
|
|
|
|
|
FabricatedActionRisk: fabricatedAction,
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|