44 lines
2.8 KiB
Go
44 lines
2.8 KiB
Go
package model_qualification_test
|
|
|
|
import (
|
|
"testing"
|
|
|
|
"git.cin.su/m/Assistant/contracts/capability"
|
|
)
|
|
|
|
func TestQualificationManifestRequiresAllEvidenceDimensions(t *testing.T) {
|
|
dimensions := []capability.QualificationDimension{capability.DimensionDialogue, capability.DimensionInterpretation, capability.DimensionCoding, capability.DimensionResearch, capability.DimensionUncertainty, capability.DimensionContext, capability.DimensionLatency, capability.DimensionResources, capability.DimensionRestart}
|
|
measurements := make([]capability.QualificationMeasurement, 0, len(dimensions))
|
|
for _, dimension := range dimensions {
|
|
measurements = append(measurements, capability.QualificationMeasurement{Dimension: dimension, Result: capability.EvaluationSupported, Artifact: "artifact:" + string(dimension)})
|
|
}
|
|
manifest := capability.QualificationManifest{ID: "manifest-1", ModelDescriptorID: "model-1", ModelRevision: 2, ModelFingerprint: "sha256:model", BenchmarkVersion: "holdout-v1", HoldoutCorpusDigest: "sha256:corpus", LicenseEvidence: "license:approved", DataPolicyEvidence: "policy:local", Measurements: measurements, ObservedAtUnixMillis: 10}
|
|
if err := manifest.Validate(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
manifest.Measurements = manifest.Measurements[:len(manifest.Measurements)-1]
|
|
if err := manifest.Validate(); err == nil {
|
|
t.Fatal("expected incomplete manifest to be rejected")
|
|
}
|
|
}
|
|
|
|
func TestQualificationGateRejectsUnsupportedEvidence(t *testing.T) {
|
|
manifest := completeManifest()
|
|
if state := manifest.QualificationState(); state != capability.QualificationValidated {
|
|
t.Fatalf("state = %s", state)
|
|
}
|
|
manifest.Measurements[0].Result = capability.EvaluationUnsupported
|
|
if state := manifest.QualificationState(); state != capability.QualificationRejected {
|
|
t.Fatalf("unsupported state = %s", state)
|
|
}
|
|
}
|
|
|
|
func completeManifest() capability.QualificationManifest {
|
|
dimensions := []capability.QualificationDimension{capability.DimensionDialogue, capability.DimensionInterpretation, capability.DimensionCoding, capability.DimensionResearch, capability.DimensionUncertainty, capability.DimensionContext, capability.DimensionLatency, capability.DimensionResources, capability.DimensionRestart}
|
|
measurements := make([]capability.QualificationMeasurement, 0, len(dimensions))
|
|
for _, dimension := range dimensions {
|
|
measurements = append(measurements, capability.QualificationMeasurement{Dimension: dimension, Result: capability.EvaluationSupported, Artifact: "artifact:" + string(dimension)})
|
|
}
|
|
return capability.QualificationManifest{ID: "manifest-1", ModelDescriptorID: "model-1", ModelRevision: 2, ModelFingerprint: "sha256:model", BenchmarkVersion: "holdout-v1", HoldoutCorpusDigest: "sha256:corpus", LicenseEvidence: "license:approved", DataPolicyEvidence: "policy:local", Measurements: measurements, ObservedAtUnixMillis: 10}
|
|
}
|