Add model qualification manifest contract
This commit is contained in:
@@ -0,0 +1,58 @@
|
||||
package capability
|
||||
|
||||
import "errors"
|
||||
|
||||
type QualificationManifestID string
|
||||
type QualificationDimension string
|
||||
|
||||
const (
|
||||
DimensionDialogue QualificationDimension = "DIALOGUE"
|
||||
DimensionInterpretation QualificationDimension = "INTERPRETATION"
|
||||
DimensionCoding QualificationDimension = "CODING"
|
||||
DimensionResearch QualificationDimension = "RESEARCH"
|
||||
DimensionUncertainty QualificationDimension = "UNCERTAINTY"
|
||||
DimensionContext QualificationDimension = "CONTEXT"
|
||||
DimensionLatency QualificationDimension = "LATENCY"
|
||||
DimensionResources QualificationDimension = "RESOURCES"
|
||||
DimensionRestart QualificationDimension = "RESTART"
|
||||
)
|
||||
|
||||
type QualificationMeasurement struct {
|
||||
Dimension QualificationDimension
|
||||
Result EvaluationResult
|
||||
Artifact string
|
||||
}
|
||||
|
||||
type QualificationManifest struct {
|
||||
ID QualificationManifestID
|
||||
ModelDescriptorID ModelDescriptorID
|
||||
ModelRevision int64
|
||||
ModelFingerprint ModelFingerprint
|
||||
BenchmarkVersion string
|
||||
HoldoutCorpusDigest string
|
||||
LicenseEvidence string
|
||||
DataPolicyEvidence string
|
||||
Measurements []QualificationMeasurement
|
||||
ObservedAtUnixMillis int64
|
||||
}
|
||||
|
||||
func (m QualificationManifest) Validate() error {
|
||||
if m.ID == "" || m.ModelDescriptorID == "" || m.ModelRevision < 1 || m.ModelFingerprint == "" || m.BenchmarkVersion == "" || m.HoldoutCorpusDigest == "" || m.LicenseEvidence == "" || m.DataPolicyEvidence == "" || m.ObservedAtUnixMillis < 1 {
|
||||
return errors.New("qualification manifest identity and evidence are incomplete")
|
||||
}
|
||||
required := map[QualificationDimension]bool{DimensionDialogue: false, DimensionInterpretation: false, DimensionCoding: false, DimensionResearch: false, DimensionUncertainty: false, DimensionContext: false, DimensionLatency: false, DimensionResources: false, DimensionRestart: false}
|
||||
for _, measurement := range m.Measurements {
|
||||
if measurement.Dimension == "" || measurement.Artifact == "" || measurement.Result == EvaluationUnknown {
|
||||
return errors.New("qualification measurement is incomplete")
|
||||
}
|
||||
if _, ok := required[measurement.Dimension]; ok {
|
||||
required[measurement.Dimension] = true
|
||||
}
|
||||
}
|
||||
for _, present := range required {
|
||||
if !present {
|
||||
return errors.New("qualification manifest lacks required measurement")
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -1,5 +1,5 @@
|
||||
{
|
||||
"task_file": "dev/tasks/DEV-000111-hardware-model-inventory-evidence.json",
|
||||
"task_id": "DEV-000111",
|
||||
"task_file": "dev/tasks/DEV-000112-model-qualification-manifest.json",
|
||||
"task_id": "DEV-000112",
|
||||
"task_revision": 1
|
||||
}
|
||||
|
||||
@@ -0,0 +1,23 @@
|
||||
{
|
||||
"task_id": "DEV-000112",
|
||||
"task_revision": 1,
|
||||
"architecture_revision": "ADR-0018",
|
||||
"base_repo_revision": "93bc4a0",
|
||||
"risk_tier": "E",
|
||||
"intent": "Add an immutable model qualification manifest contract and validation for holdout, latency, resource, license and restart evidence without activating a model.",
|
||||
"requirement_ids": ["WP-201", "Qualification evidence manifest", "No activation from model name alone"],
|
||||
"allowed_change_scope": {"modules": ["capability-model-qualification"], "paths": ["contracts/capability/**", "tests/model_qualification/**", "dev/ACTIVE_TASK.json", "dev/tasks/DEV-000112-model-qualification-manifest.json"]},
|
||||
"forbidden_change_scope": {"paths": ["architecture/**", "database/**", "migrations/**", "deploy/**", "security/**", "modules/**", "cmd/**", "runtime/**", "adapters/**", ".gitea/**"]},
|
||||
"allowed_architecture_delta": {"new_cross_module_dependencies": 0, "new_canonical_owners": 0, "new_privileges": 0, "new_third_party_dependencies": 0, "new_database_owner_roles": 0},
|
||||
"contracts": {"consume": ["Existing Capability model descriptor/evaluation types", "WP-201 audit acceptance"], "may_change": ["Capability qualification evidence manifest contract"]},
|
||||
"semantic_owner": "capability-model-qualification",
|
||||
"canonical_state": [],
|
||||
"state_transitions": ["candidate model -> measured -> evidence complete/incomplete -> qualification decision pending"],
|
||||
"side_effects_and_privileges": ["Pure validation; no provider call, model activation, deployment, credential access or external mutation"],
|
||||
"persistence_and_migrations": ["No schema or migration changes"],
|
||||
"failure_retry_restart": ["Incomplete or stale manifest remains unqualified; rerun creates a new manifest revision"],
|
||||
"assumptions": ["A model digest and provider revision identify the tested artifact", "License/data policy evidence is explicit", "Qualification is separate from activation"],
|
||||
"acceptance_scenarios": ["Manifest binds model revision and digest", "Holdout corpus and all required dimensions are recorded", "Latency/concurrency/resource/restart evidence is present", "License and data policy are present", "Incomplete manifest cannot claim validated"],
|
||||
"required_tests": ["go test ./...", "go vet ./...", "go run ./cmd/devctl preflight -active dev/ACTIVE_TASK.json", "go run ./cmd/devctl check-diff -active dev/ACTIVE_TASK.json", "go run ./cmd/devctl check-architecture", "go run ./cmd/devctl/independent", "git diff --check", "model qualification manifest tests"],
|
||||
"terminal_gap_states": ["BLOCKED_BY_MISSING_CAPABILITY", "BLOCKED_BY_SPEC_GAP", "BLOCKED_BY_CONFLICT"]
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
package model_qualification_test
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"git.cin.su/m/Assistant/contracts/capability"
|
||||
)
|
||||
|
||||
func TestQualificationManifestRequiresAllEvidenceDimensions(t *testing.T) {
|
||||
dimensions := []capability.QualificationDimension{capability.DimensionDialogue, capability.DimensionInterpretation, capability.DimensionCoding, capability.DimensionResearch, capability.DimensionUncertainty, capability.DimensionContext, capability.DimensionLatency, capability.DimensionResources, capability.DimensionRestart}
|
||||
measurements := make([]capability.QualificationMeasurement, 0, len(dimensions))
|
||||
for _, dimension := range dimensions {
|
||||
measurements = append(measurements, capability.QualificationMeasurement{Dimension: dimension, Result: capability.EvaluationSupported, Artifact: "artifact:" + string(dimension)})
|
||||
}
|
||||
manifest := capability.QualificationManifest{ID: "manifest-1", ModelDescriptorID: "model-1", ModelRevision: 2, ModelFingerprint: "sha256:model", BenchmarkVersion: "holdout-v1", HoldoutCorpusDigest: "sha256:corpus", LicenseEvidence: "license:approved", DataPolicyEvidence: "policy:local", Measurements: measurements, ObservedAtUnixMillis: 10}
|
||||
if err := manifest.Validate(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
manifest.Measurements = manifest.Measurements[:len(manifest.Measurements)-1]
|
||||
if err := manifest.Validate(); err == nil {
|
||||
t.Fatal("expected incomplete manifest to be rejected")
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user