76 lines
2.5 KiB
Go
76 lines
2.5 KiB
Go
package realityevaluation
|
|
|
|
import (
|
|
"testing"
|
|
"time"
|
|
)
|
|
|
|
func TestSuiteLoadsAndSpansRealityCategories(t *testing.T) {
|
|
suite, err := LoadSuite("suite_v1.json")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if suite.SuiteID != SuiteVersionV1 || len(suite.Cases) < 30 || len(suite.Cases) > 50 {
|
|
t.Fatalf("unexpected suite identity/size: id=%s cases=%d", suite.SuiteID, len(suite.Cases))
|
|
}
|
|
required := []string{"A", "B", "C", "D", "E", "F", "G", "H", "I", "J", "K", "L", "M", "N", "O", "P", "Q", "R"}
|
|
seen := map[string]bool{}
|
|
for _, item := range suite.Cases {
|
|
seen[item.Category] = true
|
|
}
|
|
for _, category := range required {
|
|
if !seen[category] {
|
|
t.Fatalf("missing category %s", category)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestRealityEvaluationBaselineProducesInspectableSignals(t *testing.T) {
|
|
suite, err := LoadSuite("suite_v1.json")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
artifact, err := RunSuite(suite, time.Date(2026, time.August, 17, 12, 0, 0, 0, time.UTC))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if artifact.TotalCases != len(suite.Cases) {
|
|
t.Fatalf("total cases mismatch: %d vs %d", artifact.TotalCases, len(suite.Cases))
|
|
}
|
|
if artifact.EvaluatorIndependence != "OBJECTIVE_DETERMINISTIC_CHECKS_ONLY" || artifact.ReplayStatus == "" || artifact.HumanReviewStatus == "" {
|
|
t.Fatalf("missing meta signals: %+v", artifact)
|
|
}
|
|
if artifact.SolvedAutonomously == 0 || artifact.GenuineCapabilityGaps == 0 {
|
|
t.Fatalf("expected multiple baseline buckets, got %+v", artifact)
|
|
}
|
|
if artifact.FailedWithAvailableCapability == 0 && artifact.ConsequenceReasoningFailures == 0 && artifact.IntentFailures == 0 {
|
|
if artifact.CorrectHumanIntervention == 0 || artifact.GenuineCapabilityGaps == 0 {
|
|
t.Fatalf("expected inspectable routing signals even with zero current failures: %+v", artifact)
|
|
}
|
|
}
|
|
if artifact.FailedWithAvailableCapability > 0 && len(artifact.WorstFailures) == 0 {
|
|
t.Fatalf("expected worst failures to be captured when failures remain")
|
|
}
|
|
}
|
|
|
|
func TestGeneralizationSuiteLoadsWithHoldoutCoverage(t *testing.T) {
|
|
suite, err := LoadSuite("suite_v2_generalization.json")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if suite.SuiteID != SuiteVersionV2 || len(suite.Cases) < 45 {
|
|
t.Fatalf("unexpected generalization suite: id=%s cases=%d", suite.SuiteID, len(suite.Cases))
|
|
}
|
|
devCases, holdoutCases := 0, 0
|
|
for _, item := range suite.Cases {
|
|
if item.Partition == "HOLDOUT" {
|
|
holdoutCases++
|
|
} else {
|
|
devCases++
|
|
}
|
|
}
|
|
if devCases == 0 || holdoutCases == 0 {
|
|
t.Fatalf("expected both development and holdout partitions: dev=%d holdout=%d", devCases, holdoutCases)
|
|
}
|
|
}
|