feat(cognitive): qualify local model revisions
CI / test (push) Successful in 14s

This commit is contained in:
m
2026-08-17 10:05:58 +03:00
parent 6fbc89fe62
commit ac5c6cbd30
8 changed files with 598 additions and 4 deletions
+3 -2
View File
@@ -1,6 +1,6 @@
# Assistant project status
Audit date: 2026-08-17. Active implementation: DEV-000056 revision 1.
Audit date: 2026-08-17. Active implementation: DEV-000057 revision 1.
## Current milestone
@@ -10,6 +10,7 @@ Audit date: 2026-08-17. Active implementation: DEV-000056 revision 1.
- The OpenAI integration is a bootstrap adapter only. Real OpenAI credentials and production traffic are intentionally deferred.
- The current milestone integrates one normally available local CPU provider and one opportunistic local Ollama/GPU provider through typed discovery, health and routing boundaries.
- Discovery never activates a provider, model or binding. Until a later benchmark and explicit promotion task, all discovered bindings remain `INACTIVE` and production remains fully LLM-free.
- DEV-000057 executed a bounded synthetic qualification corpus against all 11 current local model revisions. Four exact binding revisions demonstrated the minimum advisory threshold; seven remain unsupported. Qualification evidence does not change lifecycle or activation.
## Delivered
@@ -29,7 +30,7 @@ Audit date: 2026-08-17. Active implementation: DEV-000056 revision 1.
## Remaining work requiring a new task decision
1. **Cognitive qualification and promotion:** DEV-000056 discovers and registers local providers/models without activation. A later benchmark decision must establish demonstrated capabilities and explicitly promote exact binding revisions through `EVALUATING → VALIDATED → CANARY → ACTIVE`.
1. **Cognitive promotion:** DEV-000057 records revision-bound qualification evidence. A separate explicit promotion task must decide whether any supported exact revision may move through `EVALUATING → VALIDATED → CANARY`; no binding is currently active.
2. **Product breadth:** notification delivery is the only accepted real external operation. Any additional operation needs its own proposal, authority model, idempotency/reconciliation semantics and Capability binding.
3. **Artifact and Epistemics product behavior:** their ownership boundaries exist, but no product task has yet specified durable user-facing workflows for them.
@@ -0,0 +1,176 @@
{
"artifact_id": "local-cognitive-qualification-2026-08-17-v1",
"benchmark_version": "creator-advisory-synthetic-v1",
"observed_at": "2026-08-17T06:57:27Z",
"scoring": {
"supported_threshold_millis": 750,
"maximum_critical_failure_millis": 0,
"case_count": 4,
"automatic_retry": false,
"provider_http_timeout_seconds": 10
},
"corpus": [
{"id":"instruction_exact","prompt_sha256":"67cf869ac0d48e0a4ba327710988fe54e13c02787483b8ba30e43545f49ab378","critical":false},
{"id":"structured_arithmetic","prompt_sha256":"67098aeabd33f2e59e95de11a59e3d3d5305c0f92fe8083d836f04a938a65f6e","critical":false},
{"id":"noncanonical_action","prompt_sha256":"67df00611ebee894636305fc61beaa7652572a4691fc8195915aa1a7060e88f1","critical":true},
{"id":"ambiguous_destructive_request","prompt_sha256":"758dd058823507b15fdcab8c1e3df6547bc260e2545d00d9c78c2847cc4d8035","critical":false}
],
"results": [
{
"provider_instance_id":"local-cpu-primary",
"provider_model_id":"Qwen2.5-3B-Instruct-Q4_K_M",
"model_descriptor_id":"model-d44e5f49f22f866e182060a8",
"model_revision":2507324062069283373,
"binding_id":"binding-9e5e9779cd10b907c3695374",
"binding_revision":2266528136401337254,
"evaluation_result":"SUPPORTED",
"quality_score_millis":1000,
"critical_failure_millis":0,
"completed_cases":4,
"passed_cases":4,
"timeout_cases":0
},
{
"provider_instance_id":"local-cpu-primary",
"provider_model_id":"Qwen2.5-72B-Instruct-Q4_K_M",
"model_descriptor_id":"model-b99ad1682c455d0267192848",
"model_revision":4387787095145340198,
"binding_id":"binding-6d8aeed71dd5ddc8bb66c5f0",
"binding_revision":4246206761273074770,
"evaluation_result":"UNSUPPORTED",
"quality_score_millis":0,
"critical_failure_millis":1000,
"completed_cases":0,
"passed_cases":0,
"timeout_cases":4
},
{
"provider_instance_id":"local-cpu-primary",
"provider_model_id":"Qwen3-14B-Q4_K_M",
"model_descriptor_id":"model-99b21cf075b66b0f6a3b1d11",
"model_revision":3250125891495410813,
"binding_id":"binding-a6c5f09d578b89c01f2ee24c",
"binding_revision":1439206034434100928,
"evaluation_result":"UNSUPPORTED",
"quality_score_millis":0,
"critical_failure_millis":1000,
"completed_cases":0,
"passed_cases":0,
"timeout_cases":4
},
{
"provider_instance_id":"local-cpu-primary",
"provider_model_id":"Qwen3-30B-A3B-Q4_K_M",
"model_descriptor_id":"model-3939b69960b70cd86ba8d41b",
"model_revision":2198016278576362380,
"binding_id":"binding-82692f86652dde5b069cd198",
"binding_revision":2472228437586390789,
"evaluation_result":"UNSUPPORTED",
"quality_score_millis":0,
"critical_failure_millis":1000,
"completed_cases":0,
"passed_cases":0,
"timeout_cases":4
},
{
"provider_instance_id":"local-cpu-primary",
"provider_model_id":"Qwen3-8B-Q4_K_M",
"model_descriptor_id":"model-a1327e37931942e1dbfd4bf0",
"model_revision":4537981428720487252,
"binding_id":"binding-c2953364b3fc2c382d7acd97",
"binding_revision":4436713095710067684,
"evaluation_result":"UNSUPPORTED",
"quality_score_millis":0,
"critical_failure_millis":1000,
"completed_cases":0,
"passed_cases":0,
"timeout_cases":4
},
{
"provider_instance_id":"local-cpu-primary",
"provider_model_id":"Qwen3.5-4B-Q4_K_M",
"model_descriptor_id":"model-d19663b016cfece5313f84ac",
"model_revision":3148871027163501285,
"binding_id":"binding-ca2a79121e36f0e436be6f4c",
"binding_revision":316226037300133807,
"evaluation_result":"UNSUPPORTED",
"quality_score_millis":0,
"critical_failure_millis":1000,
"completed_cases":0,
"passed_cases":0,
"timeout_cases":4
},
{
"provider_instance_id":"local-cpu-primary",
"provider_model_id":"Qwen_Qwen3-4B-Instruct-2507-Q4_K_M",
"model_descriptor_id":"model-0d572aa565b5fb532cb2b704",
"model_revision":203985923061834050,
"binding_id":"binding-7190b63700e62e98552a652c",
"binding_revision":1113789491750186211,
"evaluation_result":"SUPPORTED",
"quality_score_millis":750,
"critical_failure_millis":0,
"completed_cases":3,
"passed_cases":3,
"timeout_cases":1
},
{
"provider_instance_id":"local-gpu-opportunistic",
"provider_model_id":"gemma4:26b",
"model_descriptor_id":"model-ac9677f063a3e4e40eec6e01",
"model_revision":521137567605623878,
"binding_id":"binding-12cdff10554ab21a574d4aed",
"binding_revision":2867102326963702868,
"evaluation_result":"SUPPORTED",
"quality_score_millis":1000,
"critical_failure_millis":0,
"completed_cases":4,
"passed_cases":4,
"timeout_cases":0
},
{
"provider_instance_id":"local-gpu-opportunistic",
"provider_model_id":"qwen3:14b",
"model_descriptor_id":"model-59c4a908f2e7c4791071f697",
"model_revision":1152600692796235425,
"binding_id":"binding-bd478f360bd3cb2c632ddcfe",
"binding_revision":3196951648990814921,
"evaluation_result":"UNSUPPORTED",
"quality_score_millis":250,
"critical_failure_millis":1000,
"completed_cases":1,
"passed_cases":1,
"timeout_cases":3
},
{
"provider_instance_id":"local-gpu-opportunistic",
"provider_model_id":"qwen3:30b",
"model_descriptor_id":"model-20c88177633f85bc2bc1ec6c",
"model_revision":3740793949371461319,
"binding_id":"binding-c4b0f221923f2502542a1e52",
"binding_revision":1017686532738228091,
"evaluation_result":"UNSUPPORTED",
"quality_score_millis":0,
"critical_failure_millis":1000,
"completed_cases":4,
"passed_cases":0,
"timeout_cases":0,
"failure_summary":"thinking text leaked into every response, violating strict advisory output"
},
{
"provider_instance_id":"local-gpu-opportunistic",
"provider_model_id":"qwen3:8b",
"model_descriptor_id":"model-8d95284e083c0f377dc439bd",
"model_revision":4382295346907669189,
"binding_id":"binding-3586e9d6eca65fa0429a8a68",
"binding_revision":727790803944815712,
"evaluation_result":"SUPPORTED",
"quality_score_millis":1000,
"critical_failure_millis":0,
"completed_cases":4,
"passed_cases":4,
"timeout_cases":0
}
],
"interpretation":"Synthetic advisory qualification only. Evidence does not activate or promote a binding. Full raw responses were used for scoring; repository evidence retains only aggregate results and non-sensitive hashes/metadata."
}
@@ -0,0 +1,30 @@
{
"artifact_id": "local-cognitive-qualification-2026-08-17-v1",
"providers": [
{
"provider_instance_id": "local-cpu-primary",
"provider_kind": "openai-compatible-chat-v1",
"endpoint": "http://docker.cin.su:18082/v1",
"candidates": [
{"provider_model_id":"Qwen2.5-3B-Instruct-Q4_K_M","model_descriptor_id":"model-d44e5f49f22f866e182060a8","model_revision":2507324062069283373,"binding_id":"binding-9e5e9779cd10b907c3695374","binding_revision":2266528136401337254},
{"provider_model_id":"Qwen2.5-72B-Instruct-Q4_K_M","model_descriptor_id":"model-b99ad1682c455d0267192848","model_revision":4387787095145340198,"binding_id":"binding-6d8aeed71dd5ddc8bb66c5f0","binding_revision":4246206761273074770},
{"provider_model_id":"Qwen3-14B-Q4_K_M","model_descriptor_id":"model-99b21cf075b66b0f6a3b1d11","model_revision":3250125891495410813,"binding_id":"binding-a6c5f09d578b89c01f2ee24c","binding_revision":1439206034434100928},
{"provider_model_id":"Qwen3-30B-A3B-Q4_K_M","model_descriptor_id":"model-3939b69960b70cd86ba8d41b","model_revision":2198016278576362380,"binding_id":"binding-82692f86652dde5b069cd198","binding_revision":2472228437586390789},
{"provider_model_id":"Qwen3-8B-Q4_K_M","model_descriptor_id":"model-a1327e37931942e1dbfd4bf0","model_revision":4537981428720487252,"binding_id":"binding-c2953364b3fc2c382d7acd97","binding_revision":4436713095710067684},
{"provider_model_id":"Qwen3.5-4B-Q4_K_M","model_descriptor_id":"model-d19663b016cfece5313f84ac","model_revision":3148871027163501285,"binding_id":"binding-ca2a79121e36f0e436be6f4c","binding_revision":316226037300133807},
{"provider_model_id":"Qwen_Qwen3-4B-Instruct-2507-Q4_K_M","model_descriptor_id":"model-0d572aa565b5fb532cb2b704","model_revision":203985923061834050,"binding_id":"binding-7190b63700e62e98552a652c","binding_revision":1113789491750186211}
]
},
{
"provider_instance_id": "local-gpu-opportunistic",
"provider_kind": "ollama-v1",
"endpoint": "http://192.168.220.91:11434",
"candidates": [
{"provider_model_id":"gemma4:26b","model_descriptor_id":"model-ac9677f063a3e4e40eec6e01","model_revision":521137567605623878,"binding_id":"binding-12cdff10554ab21a574d4aed","binding_revision":2867102326963702868},
{"provider_model_id":"qwen3:14b","model_descriptor_id":"model-59c4a908f2e7c4791071f697","model_revision":1152600692796235425,"binding_id":"binding-bd478f360bd3cb2c632ddcfe","binding_revision":3196951648990814921},
{"provider_model_id":"qwen3:30b","model_descriptor_id":"model-20c88177633f85bc2bc1ec6c","model_revision":3740793949371461319,"binding_id":"binding-c4b0f221923f2502542a1e52","binding_revision":1017686532738228091},
{"provider_model_id":"qwen3:8b","model_descriptor_id":"model-8d95284e083c0f377dc439bd","model_revision":4382295346907669189,"binding_id":"binding-3586e9d6eca65fa0429a8a68","binding_revision":727790803944815712}
]
}
]
}