69 lines
3.4 KiB
Go
69 lines
3.4 KiB
Go
package realityevaluation_test
|
|
|
|
import (
|
|
"encoding/json"
|
|
"os"
|
|
"testing"
|
|
)
|
|
|
|
func TestRealityEvaluationArtifactIsCompleteAndReplayable(t *testing.T) {
|
|
payload, err := os.ReadFile("../../docs/evidence/reality-evaluation-baseline-2026-08-17-v1.json")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var artifact struct {
|
|
ArtifactID string `json:"artifact_id"`
|
|
ArtifactVersion string `json:"artifact_version"`
|
|
SuiteID string `json:"suite_id"`
|
|
SuiteVersion string `json:"suite_version"`
|
|
TotalCases int `json:"total_cases"`
|
|
CategoryDistribution map[string]int `json:"category_distribution"`
|
|
SolvedAutonomously int `json:"solved_autonomously"`
|
|
CorrectHumanIntervention int `json:"correct_human_intervention"`
|
|
GenuineCapabilityGaps int `json:"genuine_capability_gaps"`
|
|
FailedWithAvailableCapability int `json:"failed_with_available_capability"`
|
|
TruthfulnessFailures int `json:"truthfulness_failures"`
|
|
FalseExecutionClaims int `json:"false_execution_claims"`
|
|
ConsequenceReasoningFailures int `json:"consequence_reasoning_failures"`
|
|
EvaluatorIndependence string `json:"evaluator_independence"`
|
|
HumanReviewStatus string `json:"human_review_status"`
|
|
ReplayStatus string `json:"replay_status"`
|
|
Cases []struct {
|
|
CaseID string `json:"case_id"`
|
|
Category string `json:"category"`
|
|
Passed bool `json:"passed"`
|
|
ActualBucket string `json:"actual_bucket"`
|
|
} `json:"cases"`
|
|
WorstFailures []struct {
|
|
CaseID string `json:"case_id"`
|
|
} `json:"worst_failures"`
|
|
}
|
|
if err := json.Unmarshal(payload, &artifact); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if artifact.ArtifactID != "reality-evaluation-baseline-2026-08-17-v1" || artifact.ArtifactVersion != "reality-evaluation-baseline-v1" || artifact.SuiteID != "reality-evaluation-suite-v1" || artifact.SuiteVersion != "reality-evaluation-suite-v1" {
|
|
t.Fatalf("unexpected artifact identity: %+v", artifact)
|
|
}
|
|
if artifact.TotalCases != 38 || len(artifact.Cases) != 38 {
|
|
t.Fatalf("unexpected total coverage: total=%d cases=%d", artifact.TotalCases, len(artifact.Cases))
|
|
}
|
|
if artifact.SolvedAutonomously != 25 || artifact.CorrectHumanIntervention != 4 || artifact.GenuineCapabilityGaps != 5 || artifact.FailedWithAvailableCapability != 4 {
|
|
t.Fatalf("unexpected outcome counts: %+v", artifact)
|
|
}
|
|
if artifact.TruthfulnessFailures != 0 || artifact.FalseExecutionClaims != 0 || artifact.ConsequenceReasoningFailures != 1 {
|
|
t.Fatalf("unexpected critical failure counts: %+v", artifact)
|
|
}
|
|
if artifact.EvaluatorIndependence != "OBJECTIVE_DETERMINISTIC_CHECKS_ONLY" || artifact.HumanReviewStatus != "REPORT_READY_FOR_OPERATOR_REVIEW" || artifact.ReplayStatus != "REPLAYABLE_VERSIONED_CORPUS" {
|
|
t.Fatalf("unexpected meta statuses: %+v", artifact)
|
|
}
|
|
if len(artifact.WorstFailures) == 0 {
|
|
t.Fatalf("expected captured failures")
|
|
}
|
|
requiredCategories := []string{"A", "B", "C", "D", "E", "F", "G", "H", "I", "J", "K", "L", "M", "N", "O", "P", "Q", "R"}
|
|
for _, category := range requiredCategories {
|
|
if artifact.CategoryDistribution[category] == 0 {
|
|
t.Fatalf("missing category %s in artifact", category)
|
|
}
|
|
}
|
|
}
|