73 lines
2.7 KiB
Go
73 lines
2.7 KiB
Go
package capability
|
|
|
|
import "errors"
|
|
|
|
type QualificationManifestID string
|
|
type QualificationDimension string
|
|
|
|
const (
|
|
DimensionDialogue QualificationDimension = "DIALOGUE"
|
|
DimensionInterpretation QualificationDimension = "INTERPRETATION"
|
|
DimensionCoding QualificationDimension = "CODING"
|
|
DimensionResearch QualificationDimension = "RESEARCH"
|
|
DimensionUncertainty QualificationDimension = "UNCERTAINTY"
|
|
DimensionContext QualificationDimension = "CONTEXT"
|
|
DimensionLatency QualificationDimension = "LATENCY"
|
|
DimensionResources QualificationDimension = "RESOURCES"
|
|
DimensionRestart QualificationDimension = "RESTART"
|
|
)
|
|
|
|
type QualificationMeasurement struct {
|
|
Dimension QualificationDimension
|
|
Result EvaluationResult
|
|
Artifact string
|
|
}
|
|
|
|
type QualificationManifest struct {
|
|
ID QualificationManifestID
|
|
ModelDescriptorID ModelDescriptorID
|
|
ModelRevision int64
|
|
ModelFingerprint ModelFingerprint
|
|
BenchmarkVersion string
|
|
HoldoutCorpusDigest string
|
|
LicenseEvidence string
|
|
DataPolicyEvidence string
|
|
Measurements []QualificationMeasurement
|
|
ObservedAtUnixMillis int64
|
|
}
|
|
|
|
func (m QualificationManifest) Validate() error {
|
|
if m.ID == "" || m.ModelDescriptorID == "" || m.ModelRevision < 1 || m.ModelFingerprint == "" || m.BenchmarkVersion == "" || m.HoldoutCorpusDigest == "" || m.LicenseEvidence == "" || m.DataPolicyEvidence == "" || m.ObservedAtUnixMillis < 1 {
|
|
return errors.New("qualification manifest identity and evidence are incomplete")
|
|
}
|
|
required := map[QualificationDimension]bool{DimensionDialogue: false, DimensionInterpretation: false, DimensionCoding: false, DimensionResearch: false, DimensionUncertainty: false, DimensionContext: false, DimensionLatency: false, DimensionResources: false, DimensionRestart: false}
|
|
for _, measurement := range m.Measurements {
|
|
if measurement.Dimension == "" || measurement.Artifact == "" || measurement.Result == EvaluationUnknown {
|
|
return errors.New("qualification measurement is incomplete")
|
|
}
|
|
if _, ok := required[measurement.Dimension]; ok {
|
|
required[measurement.Dimension] = true
|
|
}
|
|
}
|
|
for _, present := range required {
|
|
if !present {
|
|
return errors.New("qualification manifest lacks required measurement")
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// QualificationState projects evidence into a qualification result. It does
|
|
// not activate a binding or mutate the model registry.
|
|
func (m QualificationManifest) QualificationState() QualificationState {
|
|
if m.Validate() != nil {
|
|
return QualificationRejected
|
|
}
|
|
for _, measurement := range m.Measurements {
|
|
if measurement.Result != EvaluationSupported {
|
|
return QualificationRejected
|
|
}
|
|
}
|
|
return QualificationValidated
|
|
}
|