Add model qualification gate
This commit is contained in:
@@ -56,3 +56,17 @@ func (m QualificationManifest) Validate() error {
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// QualificationState projects evidence into a qualification result. It does
|
||||
// not activate a binding or mutate the model registry.
|
||||
func (m QualificationManifest) QualificationState() QualificationState {
|
||||
if m.Validate() != nil {
|
||||
return QualificationRejected
|
||||
}
|
||||
for _, measurement := range m.Measurements {
|
||||
if measurement.Result != EvaluationSupported {
|
||||
return QualificationRejected
|
||||
}
|
||||
}
|
||||
return QualificationValidated
|
||||
}
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
{
|
||||
"task_file": "dev/tasks/DEV-000112-model-qualification-manifest.json",
|
||||
"task_id": "DEV-000112",
|
||||
"task_file": "dev/tasks/DEV-000113-model-qualification-gate.json",
|
||||
"task_id": "DEV-000113",
|
||||
"task_revision": 1
|
||||
}
|
||||
|
||||
@@ -0,0 +1,23 @@
|
||||
{
|
||||
"task_id": "DEV-000113",
|
||||
"task_revision": 1,
|
||||
"architecture_revision": "ADR-0018",
|
||||
"base_repo_revision": "1f01b70",
|
||||
"risk_tier": "E",
|
||||
"intent": "Add a deterministic qualification gate that maps complete supported evidence to VALIDATED and incomplete/unsupported evidence to REJECTED without changing activation.",
|
||||
"requirement_ids": ["WP-201", "Qualification decision", "GPU/resource evidence cannot be inferred"],
|
||||
"allowed_change_scope": {"modules": ["capability-model-qualification"], "paths": ["contracts/capability/**", "tests/model_qualification/**", "dev/ACTIVE_TASK.json", "dev/tasks/DEV-000113-model-qualification-gate.json"]},
|
||||
"forbidden_change_scope": {"paths": ["architecture/**", "database/**", "migrations/**", "deploy/**", "security/**", "modules/**", "cmd/**", "runtime/**", "adapters/**", ".gitea/**"]},
|
||||
"allowed_architecture_delta": {"new_cross_module_dependencies": 0, "new_canonical_owners": 0, "new_privileges": 0, "new_third_party_dependencies": 0, "new_database_owner_roles": 0},
|
||||
"contracts": {"consume": ["QualificationManifest from DEV-000112", "Existing QualificationState and EvaluationResult values"], "may_change": ["Qualification gate projection"]},
|
||||
"semantic_owner": "capability-model-qualification",
|
||||
"canonical_state": [],
|
||||
"state_transitions": ["complete supported manifest -> VALIDATED", "incomplete/unsupported/unknown manifest -> REJECTED"],
|
||||
"side_effects_and_privileges": ["Pure validation; no binding activation, provider call, deployment, credential access or external mutation"],
|
||||
"persistence_and_migrations": ["No schema or migration changes"],
|
||||
"failure_retry_restart": ["Gate is deterministic; a new manifest revision is required after new measurements"],
|
||||
"assumptions": ["Qualification and activation remain separate lifecycle decisions", "All required dimensions are mandatory", "Model names never substitute for evidence"],
|
||||
"acceptance_scenarios": ["All supported measurements produce VALIDATED", "Unsupported resource/restart result produces REJECTED", "Unknown measurement produces REJECTED", "Gate does not mutate binding activation"],
|
||||
"required_tests": ["go test ./...", "go vet ./...", "go run ./cmd/devctl preflight -active dev/ACTIVE_TASK.json", "go run ./cmd/devctl check-diff -active dev/ACTIVE_TASK.json", "go run ./cmd/devctl check-architecture", "go run ./cmd/devctl/independent", "git diff --check", "qualification gate tests"],
|
||||
"terminal_gap_states": ["BLOCKED_BY_MISSING_CAPABILITY", "BLOCKED_BY_SPEC_GAP", "BLOCKED_BY_CONFLICT"]
|
||||
}
|
||||
@@ -21,3 +21,23 @@ func TestQualificationManifestRequiresAllEvidenceDimensions(t *testing.T) {
|
||||
t.Fatal("expected incomplete manifest to be rejected")
|
||||
}
|
||||
}
|
||||
|
||||
func TestQualificationGateRejectsUnsupportedEvidence(t *testing.T) {
|
||||
manifest := completeManifest()
|
||||
if state := manifest.QualificationState(); state != capability.QualificationValidated {
|
||||
t.Fatalf("state = %s", state)
|
||||
}
|
||||
manifest.Measurements[0].Result = capability.EvaluationUnsupported
|
||||
if state := manifest.QualificationState(); state != capability.QualificationRejected {
|
||||
t.Fatalf("unsupported state = %s", state)
|
||||
}
|
||||
}
|
||||
|
||||
func completeManifest() capability.QualificationManifest {
|
||||
dimensions := []capability.QualificationDimension{capability.DimensionDialogue, capability.DimensionInterpretation, capability.DimensionCoding, capability.DimensionResearch, capability.DimensionUncertainty, capability.DimensionContext, capability.DimensionLatency, capability.DimensionResources, capability.DimensionRestart}
|
||||
measurements := make([]capability.QualificationMeasurement, 0, len(dimensions))
|
||||
for _, dimension := range dimensions {
|
||||
measurements = append(measurements, capability.QualificationMeasurement{Dimension: dimension, Result: capability.EvaluationSupported, Artifact: "artifact:" + string(dimension)})
|
||||
}
|
||||
return capability.QualificationManifest{ID: "manifest-1", ModelDescriptorID: "model-1", ModelRevision: 2, ModelFingerprint: "sha256:model", BenchmarkVersion: "holdout-v1", HoldoutCorpusDigest: "sha256:corpus", LicenseEvidence: "license:approved", DataPolicyEvidence: "policy:local", Measurements: measurements, ObservedAtUnixMillis: 10}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user