Add model qualification manifest contract

This commit is contained in:
m
2026-08-22 11:51:19 +03:00
parent 93bc4a0fdf
commit 1f01b7013c
4 changed files with 106 additions and 2 deletions
+58
View File
@@ -0,0 +1,58 @@
package capability
import "errors"
type QualificationManifestID string
type QualificationDimension string
const (
DimensionDialogue QualificationDimension = "DIALOGUE"
DimensionInterpretation QualificationDimension = "INTERPRETATION"
DimensionCoding QualificationDimension = "CODING"
DimensionResearch QualificationDimension = "RESEARCH"
DimensionUncertainty QualificationDimension = "UNCERTAINTY"
DimensionContext QualificationDimension = "CONTEXT"
DimensionLatency QualificationDimension = "LATENCY"
DimensionResources QualificationDimension = "RESOURCES"
DimensionRestart QualificationDimension = "RESTART"
)
type QualificationMeasurement struct {
Dimension QualificationDimension
Result EvaluationResult
Artifact string
}
type QualificationManifest struct {
ID QualificationManifestID
ModelDescriptorID ModelDescriptorID
ModelRevision int64
ModelFingerprint ModelFingerprint
BenchmarkVersion string
HoldoutCorpusDigest string
LicenseEvidence string
DataPolicyEvidence string
Measurements []QualificationMeasurement
ObservedAtUnixMillis int64
}
func (m QualificationManifest) Validate() error {
if m.ID == "" || m.ModelDescriptorID == "" || m.ModelRevision < 1 || m.ModelFingerprint == "" || m.BenchmarkVersion == "" || m.HoldoutCorpusDigest == "" || m.LicenseEvidence == "" || m.DataPolicyEvidence == "" || m.ObservedAtUnixMillis < 1 {
return errors.New("qualification manifest identity and evidence are incomplete")
}
required := map[QualificationDimension]bool{DimensionDialogue: false, DimensionInterpretation: false, DimensionCoding: false, DimensionResearch: false, DimensionUncertainty: false, DimensionContext: false, DimensionLatency: false, DimensionResources: false, DimensionRestart: false}
for _, measurement := range m.Measurements {
if measurement.Dimension == "" || measurement.Artifact == "" || measurement.Result == EvaluationUnknown {
return errors.New("qualification measurement is incomplete")
}
if _, ok := required[measurement.Dimension]; ok {
required[measurement.Dimension] = true
}
}
for _, present := range required {
if !present {
return errors.New("qualification manifest lacks required measurement")
}
}
return nil
}
+2 -2
View File
@@ -1,5 +1,5 @@
{
"task_file": "dev/tasks/DEV-000111-hardware-model-inventory-evidence.json",
"task_id": "DEV-000111",
"task_file": "dev/tasks/DEV-000112-model-qualification-manifest.json",
"task_id": "DEV-000112",
"task_revision": 1
}
@@ -0,0 +1,23 @@
{
"task_id": "DEV-000112",
"task_revision": 1,
"architecture_revision": "ADR-0018",
"base_repo_revision": "93bc4a0",
"risk_tier": "E",
"intent": "Add an immutable model qualification manifest contract and validation for holdout, latency, resource, license and restart evidence without activating a model.",
"requirement_ids": ["WP-201", "Qualification evidence manifest", "No activation from model name alone"],
"allowed_change_scope": {"modules": ["capability-model-qualification"], "paths": ["contracts/capability/**", "tests/model_qualification/**", "dev/ACTIVE_TASK.json", "dev/tasks/DEV-000112-model-qualification-manifest.json"]},
"forbidden_change_scope": {"paths": ["architecture/**", "database/**", "migrations/**", "deploy/**", "security/**", "modules/**", "cmd/**", "runtime/**", "adapters/**", ".gitea/**"]},
"allowed_architecture_delta": {"new_cross_module_dependencies": 0, "new_canonical_owners": 0, "new_privileges": 0, "new_third_party_dependencies": 0, "new_database_owner_roles": 0},
"contracts": {"consume": ["Existing Capability model descriptor/evaluation types", "WP-201 audit acceptance"], "may_change": ["Capability qualification evidence manifest contract"]},
"semantic_owner": "capability-model-qualification",
"canonical_state": [],
"state_transitions": ["candidate model -> measured -> evidence complete/incomplete -> qualification decision pending"],
"side_effects_and_privileges": ["Pure validation; no provider call, model activation, deployment, credential access or external mutation"],
"persistence_and_migrations": ["No schema or migration changes"],
"failure_retry_restart": ["Incomplete or stale manifest remains unqualified; rerun creates a new manifest revision"],
"assumptions": ["A model digest and provider revision identify the tested artifact", "License/data policy evidence is explicit", "Qualification is separate from activation"],
"acceptance_scenarios": ["Manifest binds model revision and digest", "Holdout corpus and all required dimensions are recorded", "Latency/concurrency/resource/restart evidence is present", "License and data policy are present", "Incomplete manifest cannot claim validated"],
"required_tests": ["go test ./...", "go vet ./...", "go run ./cmd/devctl preflight -active dev/ACTIVE_TASK.json", "go run ./cmd/devctl check-diff -active dev/ACTIVE_TASK.json", "go run ./cmd/devctl check-architecture", "go run ./cmd/devctl/independent", "git diff --check", "model qualification manifest tests"],
"terminal_gap_states": ["BLOCKED_BY_MISSING_CAPABILITY", "BLOCKED_BY_SPEC_GAP", "BLOCKED_BY_CONFLICT"]
}
@@ -0,0 +1,23 @@
package model_qualification_test
import (
"testing"
"git.cin.su/m/Assistant/contracts/capability"
)
func TestQualificationManifestRequiresAllEvidenceDimensions(t *testing.T) {
dimensions := []capability.QualificationDimension{capability.DimensionDialogue, capability.DimensionInterpretation, capability.DimensionCoding, capability.DimensionResearch, capability.DimensionUncertainty, capability.DimensionContext, capability.DimensionLatency, capability.DimensionResources, capability.DimensionRestart}
measurements := make([]capability.QualificationMeasurement, 0, len(dimensions))
for _, dimension := range dimensions {
measurements = append(measurements, capability.QualificationMeasurement{Dimension: dimension, Result: capability.EvaluationSupported, Artifact: "artifact:" + string(dimension)})
}
manifest := capability.QualificationManifest{ID: "manifest-1", ModelDescriptorID: "model-1", ModelRevision: 2, ModelFingerprint: "sha256:model", BenchmarkVersion: "holdout-v1", HoldoutCorpusDigest: "sha256:corpus", LicenseEvidence: "license:approved", DataPolicyEvidence: "policy:local", Measurements: measurements, ObservedAtUnixMillis: 10}
if err := manifest.Validate(); err != nil {
t.Fatal(err)
}
manifest.Measurements = manifest.Measurements[:len(manifest.Measurements)-1]
if err := manifest.Validate(); err == nil {
t.Fatal("expected incomplete manifest to be rejected")
}
}