75 lines
2.5 KiB
Go
75 lines
2.5 KiB
Go
package cognitive_benchmark_test
|
|
|
|
import (
|
|
"encoding/json"
|
|
"os"
|
|
"testing"
|
|
)
|
|
|
|
func TestQualificationArtifactIsRevisionBoundAndComplete(t *testing.T) {
|
|
payload, err := os.ReadFile("../../docs/evidence/local-cognitive-qualification-2026-08-17-v1.json")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var artifact struct {
|
|
ArtifactID string `json:"artifact_id"`
|
|
BenchmarkVersion string `json:"benchmark_version"`
|
|
Corpus []struct {
|
|
ID string `json:"id"`
|
|
PromptSHA256 string `json:"prompt_sha256"`
|
|
Critical bool `json:"critical"`
|
|
} `json:"corpus"`
|
|
Results []struct {
|
|
ModelDescriptorID string `json:"model_descriptor_id"`
|
|
ModelRevision int64 `json:"model_revision"`
|
|
BindingID string `json:"binding_id"`
|
|
BindingRevision int64 `json:"binding_revision"`
|
|
EvaluationResult string `json:"evaluation_result"`
|
|
QualityScoreMillis int64 `json:"quality_score_millis"`
|
|
CriticalFailureMillis int64 `json:"critical_failure_millis"`
|
|
} `json:"results"`
|
|
}
|
|
if err := json.Unmarshal(payload, &artifact); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if artifact.ArtifactID != "local-cognitive-qualification-2026-08-17-v1" || artifact.BenchmarkVersion != "creator-advisory-synthetic-v1" || len(artifact.Corpus) != 4 || len(artifact.Results) != 11 {
|
|
t.Fatalf("unexpected artifact identity or coverage: %+v", artifact)
|
|
}
|
|
critical := 0
|
|
for _, item := range artifact.Corpus {
|
|
if item.ID == "" || len(item.PromptSHA256) != 64 {
|
|
t.Fatalf("invalid corpus record: %+v", item)
|
|
}
|
|
if item.Critical {
|
|
critical++
|
|
}
|
|
}
|
|
if critical != 1 {
|
|
t.Fatalf("critical case count = %d", critical)
|
|
}
|
|
seenBindings := make(map[string]struct{}, len(artifact.Results))
|
|
supported := 0
|
|
for _, result := range artifact.Results {
|
|
if result.ModelDescriptorID == "" || result.ModelRevision < 1 || result.BindingID == "" || result.BindingRevision < 1 {
|
|
t.Fatalf("result is not revision-bound: %+v", result)
|
|
}
|
|
if _, exists := seenBindings[result.BindingID]; exists {
|
|
t.Fatalf("duplicate binding %s", result.BindingID)
|
|
}
|
|
seenBindings[result.BindingID] = struct{}{}
|
|
switch result.EvaluationResult {
|
|
case "SUPPORTED":
|
|
supported++
|
|
if result.QualityScoreMillis < 750 || result.CriticalFailureMillis != 0 {
|
|
t.Fatalf("unsupported score marked supported: %+v", result)
|
|
}
|
|
case "UNSUPPORTED":
|
|
default:
|
|
t.Fatalf("unknown evaluation result %q", result.EvaluationResult)
|
|
}
|
|
}
|
|
if supported != 4 {
|
|
t.Fatalf("supported result count = %d", supported)
|
|
}
|
|
}
|