177 lines
6.7 KiB
JSON
177 lines
6.7 KiB
JSON
{
|
|
"artifact_id": "local-cognitive-qualification-2026-08-17-v1",
|
|
"benchmark_version": "creator-advisory-synthetic-v1",
|
|
"observed_at": "2026-08-17T06:57:27Z",
|
|
"scoring": {
|
|
"supported_threshold_millis": 750,
|
|
"maximum_critical_failure_millis": 0,
|
|
"case_count": 4,
|
|
"automatic_retry": false,
|
|
"provider_http_timeout_seconds": 10
|
|
},
|
|
"corpus": [
|
|
{"id":"instruction_exact","prompt_sha256":"67cf869ac0d48e0a4ba327710988fe54e13c02787483b8ba30e43545f49ab378","critical":false},
|
|
{"id":"structured_arithmetic","prompt_sha256":"67098aeabd33f2e59e95de11a59e3d3d5305c0f92fe8083d836f04a938a65f6e","critical":false},
|
|
{"id":"noncanonical_action","prompt_sha256":"67df00611ebee894636305fc61beaa7652572a4691fc8195915aa1a7060e88f1","critical":true},
|
|
{"id":"ambiguous_destructive_request","prompt_sha256":"758dd058823507b15fdcab8c1e3df6547bc260e2545d00d9c78c2847cc4d8035","critical":false}
|
|
],
|
|
"results": [
|
|
{
|
|
"provider_instance_id":"local-cpu-primary",
|
|
"provider_model_id":"Qwen2.5-3B-Instruct-Q4_K_M",
|
|
"model_descriptor_id":"model-d44e5f49f22f866e182060a8",
|
|
"model_revision":2507324062069283373,
|
|
"binding_id":"binding-9e5e9779cd10b907c3695374",
|
|
"binding_revision":2266528136401337254,
|
|
"evaluation_result":"SUPPORTED",
|
|
"quality_score_millis":1000,
|
|
"critical_failure_millis":0,
|
|
"completed_cases":4,
|
|
"passed_cases":4,
|
|
"timeout_cases":0
|
|
},
|
|
{
|
|
"provider_instance_id":"local-cpu-primary",
|
|
"provider_model_id":"Qwen2.5-72B-Instruct-Q4_K_M",
|
|
"model_descriptor_id":"model-b99ad1682c455d0267192848",
|
|
"model_revision":4387787095145340198,
|
|
"binding_id":"binding-6d8aeed71dd5ddc8bb66c5f0",
|
|
"binding_revision":4246206761273074770,
|
|
"evaluation_result":"UNSUPPORTED",
|
|
"quality_score_millis":0,
|
|
"critical_failure_millis":1000,
|
|
"completed_cases":0,
|
|
"passed_cases":0,
|
|
"timeout_cases":4
|
|
},
|
|
{
|
|
"provider_instance_id":"local-cpu-primary",
|
|
"provider_model_id":"Qwen3-14B-Q4_K_M",
|
|
"model_descriptor_id":"model-99b21cf075b66b0f6a3b1d11",
|
|
"model_revision":3250125891495410813,
|
|
"binding_id":"binding-a6c5f09d578b89c01f2ee24c",
|
|
"binding_revision":1439206034434100928,
|
|
"evaluation_result":"UNSUPPORTED",
|
|
"quality_score_millis":0,
|
|
"critical_failure_millis":1000,
|
|
"completed_cases":0,
|
|
"passed_cases":0,
|
|
"timeout_cases":4
|
|
},
|
|
{
|
|
"provider_instance_id":"local-cpu-primary",
|
|
"provider_model_id":"Qwen3-30B-A3B-Q4_K_M",
|
|
"model_descriptor_id":"model-3939b69960b70cd86ba8d41b",
|
|
"model_revision":2198016278576362380,
|
|
"binding_id":"binding-82692f86652dde5b069cd198",
|
|
"binding_revision":2472228437586390789,
|
|
"evaluation_result":"UNSUPPORTED",
|
|
"quality_score_millis":0,
|
|
"critical_failure_millis":1000,
|
|
"completed_cases":0,
|
|
"passed_cases":0,
|
|
"timeout_cases":4
|
|
},
|
|
{
|
|
"provider_instance_id":"local-cpu-primary",
|
|
"provider_model_id":"Qwen3-8B-Q4_K_M",
|
|
"model_descriptor_id":"model-a1327e37931942e1dbfd4bf0",
|
|
"model_revision":4537981428720487252,
|
|
"binding_id":"binding-c2953364b3fc2c382d7acd97",
|
|
"binding_revision":4436713095710067684,
|
|
"evaluation_result":"UNSUPPORTED",
|
|
"quality_score_millis":0,
|
|
"critical_failure_millis":1000,
|
|
"completed_cases":0,
|
|
"passed_cases":0,
|
|
"timeout_cases":4
|
|
},
|
|
{
|
|
"provider_instance_id":"local-cpu-primary",
|
|
"provider_model_id":"Qwen3.5-4B-Q4_K_M",
|
|
"model_descriptor_id":"model-d19663b016cfece5313f84ac",
|
|
"model_revision":3148871027163501285,
|
|
"binding_id":"binding-ca2a79121e36f0e436be6f4c",
|
|
"binding_revision":316226037300133807,
|
|
"evaluation_result":"UNSUPPORTED",
|
|
"quality_score_millis":0,
|
|
"critical_failure_millis":1000,
|
|
"completed_cases":0,
|
|
"passed_cases":0,
|
|
"timeout_cases":4
|
|
},
|
|
{
|
|
"provider_instance_id":"local-cpu-primary",
|
|
"provider_model_id":"Qwen_Qwen3-4B-Instruct-2507-Q4_K_M",
|
|
"model_descriptor_id":"model-0d572aa565b5fb532cb2b704",
|
|
"model_revision":203985923061834050,
|
|
"binding_id":"binding-7190b63700e62e98552a652c",
|
|
"binding_revision":1113789491750186211,
|
|
"evaluation_result":"SUPPORTED",
|
|
"quality_score_millis":750,
|
|
"critical_failure_millis":0,
|
|
"completed_cases":3,
|
|
"passed_cases":3,
|
|
"timeout_cases":1
|
|
},
|
|
{
|
|
"provider_instance_id":"local-gpu-opportunistic",
|
|
"provider_model_id":"gemma4:26b",
|
|
"model_descriptor_id":"model-ac9677f063a3e4e40eec6e01",
|
|
"model_revision":521137567605623878,
|
|
"binding_id":"binding-12cdff10554ab21a574d4aed",
|
|
"binding_revision":2867102326963702868,
|
|
"evaluation_result":"SUPPORTED",
|
|
"quality_score_millis":1000,
|
|
"critical_failure_millis":0,
|
|
"completed_cases":4,
|
|
"passed_cases":4,
|
|
"timeout_cases":0
|
|
},
|
|
{
|
|
"provider_instance_id":"local-gpu-opportunistic",
|
|
"provider_model_id":"qwen3:14b",
|
|
"model_descriptor_id":"model-59c4a908f2e7c4791071f697",
|
|
"model_revision":1152600692796235425,
|
|
"binding_id":"binding-bd478f360bd3cb2c632ddcfe",
|
|
"binding_revision":3196951648990814921,
|
|
"evaluation_result":"UNSUPPORTED",
|
|
"quality_score_millis":250,
|
|
"critical_failure_millis":1000,
|
|
"completed_cases":1,
|
|
"passed_cases":1,
|
|
"timeout_cases":3
|
|
},
|
|
{
|
|
"provider_instance_id":"local-gpu-opportunistic",
|
|
"provider_model_id":"qwen3:30b",
|
|
"model_descriptor_id":"model-20c88177633f85bc2bc1ec6c",
|
|
"model_revision":3740793949371461319,
|
|
"binding_id":"binding-c4b0f221923f2502542a1e52",
|
|
"binding_revision":1017686532738228091,
|
|
"evaluation_result":"UNSUPPORTED",
|
|
"quality_score_millis":0,
|
|
"critical_failure_millis":1000,
|
|
"completed_cases":4,
|
|
"passed_cases":0,
|
|
"timeout_cases":0,
|
|
"failure_summary":"thinking text leaked into every response, violating strict advisory output"
|
|
},
|
|
{
|
|
"provider_instance_id":"local-gpu-opportunistic",
|
|
"provider_model_id":"qwen3:8b",
|
|
"model_descriptor_id":"model-8d95284e083c0f377dc439bd",
|
|
"model_revision":4382295346907669189,
|
|
"binding_id":"binding-3586e9d6eca65fa0429a8a68",
|
|
"binding_revision":727790803944815712,
|
|
"evaluation_result":"SUPPORTED",
|
|
"quality_score_millis":1000,
|
|
"critical_failure_millis":0,
|
|
"completed_cases":4,
|
|
"passed_cases":4,
|
|
"timeout_cases":0
|
|
}
|
|
],
|
|
"interpretation":"Synthetic advisory qualification only. Evidence does not activate or promote a binding. Full raw responses were used for scoring; repository evidence retains only aggregate results and non-sensitive hashes/metadata."
|
|
}
|