diff --git a/contracts/capability/qualification.go b/contracts/capability/qualification.go new file mode 100644 index 0000000..895a7ed --- /dev/null +++ b/contracts/capability/qualification.go @@ -0,0 +1,58 @@ +package capability + +import "errors" + +type QualificationManifestID string +type QualificationDimension string + +const ( + DimensionDialogue QualificationDimension = "DIALOGUE" + DimensionInterpretation QualificationDimension = "INTERPRETATION" + DimensionCoding QualificationDimension = "CODING" + DimensionResearch QualificationDimension = "RESEARCH" + DimensionUncertainty QualificationDimension = "UNCERTAINTY" + DimensionContext QualificationDimension = "CONTEXT" + DimensionLatency QualificationDimension = "LATENCY" + DimensionResources QualificationDimension = "RESOURCES" + DimensionRestart QualificationDimension = "RESTART" +) + +type QualificationMeasurement struct { + Dimension QualificationDimension + Result EvaluationResult + Artifact string +} + +type QualificationManifest struct { + ID QualificationManifestID + ModelDescriptorID ModelDescriptorID + ModelRevision int64 + ModelFingerprint ModelFingerprint + BenchmarkVersion string + HoldoutCorpusDigest string + LicenseEvidence string + DataPolicyEvidence string + Measurements []QualificationMeasurement + ObservedAtUnixMillis int64 +} + +func (m QualificationManifest) Validate() error { + if m.ID == "" || m.ModelDescriptorID == "" || m.ModelRevision < 1 || m.ModelFingerprint == "" || m.BenchmarkVersion == "" || m.HoldoutCorpusDigest == "" || m.LicenseEvidence == "" || m.DataPolicyEvidence == "" || m.ObservedAtUnixMillis < 1 { + return errors.New("qualification manifest identity and evidence are incomplete") + } + required := map[QualificationDimension]bool{DimensionDialogue: false, DimensionInterpretation: false, DimensionCoding: false, DimensionResearch: false, DimensionUncertainty: false, DimensionContext: false, DimensionLatency: false, DimensionResources: false, DimensionRestart: false} + for _, measurement := range m.Measurements { + if measurement.Dimension == "" || measurement.Artifact == "" || measurement.Result == EvaluationUnknown { + return errors.New("qualification measurement is incomplete") + } + if _, ok := required[measurement.Dimension]; ok { + required[measurement.Dimension] = true + } + } + for _, present := range required { + if !present { + return errors.New("qualification manifest lacks required measurement") + } + } + return nil +} diff --git a/dev/ACTIVE_TASK.json b/dev/ACTIVE_TASK.json index 95bb6c5..9c98be5 100644 --- a/dev/ACTIVE_TASK.json +++ b/dev/ACTIVE_TASK.json @@ -1,5 +1,5 @@ { - "task_file": "dev/tasks/DEV-000111-hardware-model-inventory-evidence.json", - "task_id": "DEV-000111", + "task_file": "dev/tasks/DEV-000112-model-qualification-manifest.json", + "task_id": "DEV-000112", "task_revision": 1 } diff --git a/dev/tasks/DEV-000112-model-qualification-manifest.json b/dev/tasks/DEV-000112-model-qualification-manifest.json new file mode 100644 index 0000000..c100786 --- /dev/null +++ b/dev/tasks/DEV-000112-model-qualification-manifest.json @@ -0,0 +1,23 @@ +{ + "task_id": "DEV-000112", + "task_revision": 1, + "architecture_revision": "ADR-0018", + "base_repo_revision": "93bc4a0", + "risk_tier": "E", + "intent": "Add an immutable model qualification manifest contract and validation for holdout, latency, resource, license and restart evidence without activating a model.", + "requirement_ids": ["WP-201", "Qualification evidence manifest", "No activation from model name alone"], + "allowed_change_scope": {"modules": ["capability-model-qualification"], "paths": ["contracts/capability/**", "tests/model_qualification/**", "dev/ACTIVE_TASK.json", "dev/tasks/DEV-000112-model-qualification-manifest.json"]}, + "forbidden_change_scope": {"paths": ["architecture/**", "database/**", "migrations/**", "deploy/**", "security/**", "modules/**", "cmd/**", "runtime/**", "adapters/**", ".gitea/**"]}, + "allowed_architecture_delta": {"new_cross_module_dependencies": 0, "new_canonical_owners": 0, "new_privileges": 0, "new_third_party_dependencies": 0, "new_database_owner_roles": 0}, + "contracts": {"consume": ["Existing Capability model descriptor/evaluation types", "WP-201 audit acceptance"], "may_change": ["Capability qualification evidence manifest contract"]}, + "semantic_owner": "capability-model-qualification", + "canonical_state": [], + "state_transitions": ["candidate model -> measured -> evidence complete/incomplete -> qualification decision pending"], + "side_effects_and_privileges": ["Pure validation; no provider call, model activation, deployment, credential access or external mutation"], + "persistence_and_migrations": ["No schema or migration changes"], + "failure_retry_restart": ["Incomplete or stale manifest remains unqualified; rerun creates a new manifest revision"], + "assumptions": ["A model digest and provider revision identify the tested artifact", "License/data policy evidence is explicit", "Qualification is separate from activation"], + "acceptance_scenarios": ["Manifest binds model revision and digest", "Holdout corpus and all required dimensions are recorded", "Latency/concurrency/resource/restart evidence is present", "License and data policy are present", "Incomplete manifest cannot claim validated"], + "required_tests": ["go test ./...", "go vet ./...", "go run ./cmd/devctl preflight -active dev/ACTIVE_TASK.json", "go run ./cmd/devctl check-diff -active dev/ACTIVE_TASK.json", "go run ./cmd/devctl check-architecture", "go run ./cmd/devctl/independent", "git diff --check", "model qualification manifest tests"], + "terminal_gap_states": ["BLOCKED_BY_MISSING_CAPABILITY", "BLOCKED_BY_SPEC_GAP", "BLOCKED_BY_CONFLICT"] +} diff --git a/tests/model_qualification/qualification_test.go b/tests/model_qualification/qualification_test.go new file mode 100644 index 0000000..6263dea --- /dev/null +++ b/tests/model_qualification/qualification_test.go @@ -0,0 +1,23 @@ +package model_qualification_test + +import ( + "testing" + + "git.cin.su/m/Assistant/contracts/capability" +) + +func TestQualificationManifestRequiresAllEvidenceDimensions(t *testing.T) { + dimensions := []capability.QualificationDimension{capability.DimensionDialogue, capability.DimensionInterpretation, capability.DimensionCoding, capability.DimensionResearch, capability.DimensionUncertainty, capability.DimensionContext, capability.DimensionLatency, capability.DimensionResources, capability.DimensionRestart} + measurements := make([]capability.QualificationMeasurement, 0, len(dimensions)) + for _, dimension := range dimensions { + measurements = append(measurements, capability.QualificationMeasurement{Dimension: dimension, Result: capability.EvaluationSupported, Artifact: "artifact:" + string(dimension)}) + } + manifest := capability.QualificationManifest{ID: "manifest-1", ModelDescriptorID: "model-1", ModelRevision: 2, ModelFingerprint: "sha256:model", BenchmarkVersion: "holdout-v1", HoldoutCorpusDigest: "sha256:corpus", LicenseEvidence: "license:approved", DataPolicyEvidence: "policy:local", Measurements: measurements, ObservedAtUnixMillis: 10} + if err := manifest.Validate(); err != nil { + t.Fatal(err) + } + manifest.Measurements = manifest.Measurements[:len(manifest.Measurements)-1] + if err := manifest.Validate(); err == nil { + t.Fatal("expected incomplete manifest to be rejected") + } +}