Add model qualification gate
This commit is contained in:
@@ -21,3 +21,23 @@ func TestQualificationManifestRequiresAllEvidenceDimensions(t *testing.T) {
|
||||
t.Fatal("expected incomplete manifest to be rejected")
|
||||
}
|
||||
}
|
||||
|
||||
func TestQualificationGateRejectsUnsupportedEvidence(t *testing.T) {
|
||||
manifest := completeManifest()
|
||||
if state := manifest.QualificationState(); state != capability.QualificationValidated {
|
||||
t.Fatalf("state = %s", state)
|
||||
}
|
||||
manifest.Measurements[0].Result = capability.EvaluationUnsupported
|
||||
if state := manifest.QualificationState(); state != capability.QualificationRejected {
|
||||
t.Fatalf("unsupported state = %s", state)
|
||||
}
|
||||
}
|
||||
|
||||
func completeManifest() capability.QualificationManifest {
|
||||
dimensions := []capability.QualificationDimension{capability.DimensionDialogue, capability.DimensionInterpretation, capability.DimensionCoding, capability.DimensionResearch, capability.DimensionUncertainty, capability.DimensionContext, capability.DimensionLatency, capability.DimensionResources, capability.DimensionRestart}
|
||||
measurements := make([]capability.QualificationMeasurement, 0, len(dimensions))
|
||||
for _, dimension := range dimensions {
|
||||
measurements = append(measurements, capability.QualificationMeasurement{Dimension: dimension, Result: capability.EvaluationSupported, Artifact: "artifact:" + string(dimension)})
|
||||
}
|
||||
return capability.QualificationManifest{ID: "manifest-1", ModelDescriptorID: "model-1", ModelRevision: 2, ModelFingerprint: "sha256:model", BenchmarkVersion: "holdout-v1", HoldoutCorpusDigest: "sha256:corpus", LicenseEvidence: "license:approved", DataPolicyEvidence: "policy:local", Measurements: measurements, ObservedAtUnixMillis: 10}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user