129 lines
5.3 KiB
Go
129 lines
5.3 KiB
Go
package realityevaluation_test
|
|
|
|
import (
|
|
"encoding/json"
|
|
"os"
|
|
"testing"
|
|
)
|
|
|
|
func TestGeneralizationArtifactExistsAndTracksHoldout(t *testing.T) {
|
|
payload, err := os.ReadFile("../../docs/evidence/reality-evaluation-generalization-2026-08-17-v1.json")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var artifact struct {
|
|
ArtifactID string `json:"artifact_id"`
|
|
SuiteID string `json:"suite_id"`
|
|
TotalCases int `json:"total_cases"`
|
|
DevelopmentCases int `json:"development_cases"`
|
|
HoldoutCases int `json:"holdout_cases"`
|
|
FailedAvailable int `json:"failed_with_available_capability"`
|
|
ConsequenceFailure int `json:"consequence_reasoning_failures"`
|
|
}
|
|
if err := json.Unmarshal(payload, &artifact); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if artifact.ArtifactID != "reality-evaluation-generalization-2026-08-17-v1" || artifact.SuiteID != "reality-evaluation-suite-v2" {
|
|
t.Fatalf("unexpected artifact identity: %+v", artifact)
|
|
}
|
|
if artifact.TotalCases < 45 || artifact.DevelopmentCases == 0 || artifact.HoldoutCases == 0 {
|
|
t.Fatalf("unexpected case distribution: %+v", artifact)
|
|
}
|
|
if artifact.FailedAvailable == 0 || artifact.ConsequenceFailure == 0 {
|
|
t.Fatalf("expected non-trivial baseline failures: %+v", artifact)
|
|
}
|
|
}
|
|
|
|
func TestCognitiveBaselineArtifactExists(t *testing.T) {
|
|
payload, err := os.ReadFile("../../docs/evidence/cognitive-generalization-baseline-2026-08-17-v1.json")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var artifact struct {
|
|
ArtifactID string `json:"artifact_id"`
|
|
SuiteID string `json:"suite_id"`
|
|
ProviderInstanceID string `json:"provider_instance_id"`
|
|
TotalCases int `json:"total_cases"`
|
|
PassedCases int `json:"passed_cases"`
|
|
UncertaintyFailures int `json:"uncertainty_failures"`
|
|
CapabilityFailures int `json:"capability_failures"`
|
|
FabricatedObservationFails int `json:"fabricated_observation_failures"`
|
|
}
|
|
if err := json.Unmarshal(payload, &artifact); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if artifact.ArtifactID != "cognitive-generalization-baseline-2026-08-17-v1" || artifact.SuiteID != "cognitive-generalization-suite-v1" {
|
|
t.Fatalf("unexpected artifact identity: %+v", artifact)
|
|
}
|
|
if artifact.ProviderInstanceID != "local-cpu-primary" || artifact.TotalCases < 10 || artifact.PassedCases == 0 {
|
|
t.Fatalf("unexpected cognitive baseline coverage: %+v", artifact)
|
|
}
|
|
if artifact.UncertaintyFailures == 0 || artifact.CapabilityFailures == 0 || artifact.FabricatedObservationFails == 0 {
|
|
t.Fatalf("expected visible cognitive baseline failures: %+v", artifact)
|
|
}
|
|
}
|
|
|
|
func TestProductionAuditArtifactExists(t *testing.T) {
|
|
payload, err := os.ReadFile("../../docs/evidence/production-cognitive-path-audit-2026-08-17-v1.json")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var artifact struct {
|
|
ArtifactID string `json:"artifact_id"`
|
|
RealHumanSequence struct {
|
|
Turn3 struct {
|
|
SpeechAct string `json:"speech_act"`
|
|
} `json:"turn_3"`
|
|
} `json:"real_human_sequence"`
|
|
HistoricalRoutingEvidence struct {
|
|
OllamaCount int `json:"ollama_gpu_invocation_count"`
|
|
} `json:"historical_routing_evidence"`
|
|
DeterministicBoundaryAssessment struct {
|
|
WrongDeterministicInterpretationCanPreventCognitionEntirely bool `json:"wrong_deterministic_interpretation_can_prevent_cognition_entirely"`
|
|
} `json:"deterministic_boundary_assessment"`
|
|
}
|
|
if err := json.Unmarshal(payload, &artifact); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if artifact.ArtifactID != "production-cognitive-path-audit-2026-08-17-v1" {
|
|
t.Fatalf("unexpected production audit identity: %+v", artifact)
|
|
}
|
|
if artifact.RealHumanSequence.Turn3.SpeechAct != "ASSERTION" || artifact.HistoricalRoutingEvidence.OllamaCount != 0 || !artifact.DeterministicBoundaryAssessment.WrongDeterministicInterpretationCanPreventCognitionEntirely {
|
|
t.Fatalf("unexpected audit facts: %+v", artifact)
|
|
}
|
|
}
|
|
|
|
func TestSemanticDeferralMetamorphicEvidenceExists(t *testing.T) {
|
|
payload, err := os.ReadFile("../../docs/evidence/semantic-deferral-metamorphic-evidence-2026-08-17-v1.json")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var artifact struct {
|
|
ArtifactID string `json:"artifact_id"`
|
|
TaskID string `json:"task_id"`
|
|
RawModelEvidence []struct {
|
|
Utterance string `json:"utterance"`
|
|
Result string `json:"result"`
|
|
} `json:"raw_model_evidence"`
|
|
DomovoyBoundaryEvidence struct {
|
|
BeforeFix []struct {
|
|
Utterance string `json:"utterance"`
|
|
SemanticPath string `json:"semantic_path"`
|
|
CognitionRequested bool `json:"cognition_requested"`
|
|
} `json:"before_fix"`
|
|
} `json:"domovoy_boundary_evidence"`
|
|
}
|
|
if err := json.Unmarshal(payload, &artifact); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if artifact.ArtifactID != "semantic-deferral-metamorphic-evidence-2026-08-17-v1" || artifact.TaskID != "DEV-000072" {
|
|
t.Fatalf("unexpected semantic deferral evidence identity: %+v", artifact)
|
|
}
|
|
if len(artifact.RawModelEvidence) < 4 {
|
|
t.Fatalf("expected CPU and GPU raw evidence, got %+v", artifact.RawModelEvidence)
|
|
}
|
|
if len(artifact.DomovoyBoundaryEvidence.BeforeFix) < 2 || artifact.DomovoyBoundaryEvidence.BeforeFix[0].SemanticPath != "ASSERTION" || artifact.DomovoyBoundaryEvidence.BeforeFix[0].CognitionRequested {
|
|
t.Fatalf("expected pre-fix Domovoy boundary evidence, got %+v", artifact.DomovoyBoundaryEvidence)
|
|
}
|
|
}
|