feat(cognitive): add bounded model qualification probe #212
@@ -169,6 +169,7 @@ func main() {
|
||||
mux.HandleFunc("POST /v1/task-evaluations/execute", gateway.taskEvaluationHandler)
|
||||
discovery := discoveryService{resolver: resolver, providers: discoveries, owner: identity.PrincipalID(envOr("COGNITIVE_OWNER_PRINCIPAL_ID", "domovoy-creator-principal")), now: time.Now}
|
||||
mux.HandleFunc("POST /v1/provider-discovery/{provider}", discovery.handler)
|
||||
mux.Handle("POST /v1/provider-discovery/{provider}/qualify", modelQualificationHandler(discoveries, string(envOr("COGNITIVE_OWNER_PRINCIPAL_ID", "domovoy-creator-principal"))))
|
||||
mux.HandleFunc("GET /v1/provider-discovery/status", discovery.statusHandler)
|
||||
refreshProviderDiscoveries(resolver, discovery)
|
||||
log.Printf("cognitive-worker listening on %s", addr)
|
||||
|
||||
@@ -0,0 +1,117 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"slices"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"git.cin.su/m/Assistant/adapters/models"
|
||||
"git.cin.su/m/Assistant/contracts/authority"
|
||||
"git.cin.su/m/Assistant/contracts/capability"
|
||||
"git.cin.su/m/Assistant/contracts/cognitive"
|
||||
"git.cin.su/m/Assistant/contracts/identity"
|
||||
)
|
||||
|
||||
type modelQualificationRequest struct {
|
||||
Prompt string `json:"prompt"`
|
||||
MaxOutputBytes int `json:"max_output_bytes"`
|
||||
ModelIDs []capability.ProviderModelID `json:"model_ids,omitempty"`
|
||||
}
|
||||
|
||||
type modelQualificationResult struct {
|
||||
ProviderModelID capability.ProviderModelID `json:"provider_model_id"`
|
||||
Status string `json:"status"`
|
||||
FinishReason cognitive.FinishReason `json:"finish_reason,omitempty"`
|
||||
LatencyMillis int64 `json:"latency_millis"`
|
||||
OutputObserved bool `json:"output_observed"`
|
||||
QualityAssessed bool `json:"quality_assessed"`
|
||||
Category capability.ModelRatingCategory `json:"category"`
|
||||
Error string `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
type modelQualificationResponse struct {
|
||||
Provider capability.ProviderInstanceID `json:"provider"`
|
||||
ObservedAtUnixMillis int64 `json:"observed_at_unix_millis"`
|
||||
Results []modelQualificationResult `json:"results"`
|
||||
}
|
||||
|
||||
func modelQualificationHandler(providers map[string]providerRegistration, owner string) http.Handler {
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
name := r.PathValue("provider")
|
||||
registration, ok := providers[name]
|
||||
if !ok || registration.Discovery == nil || registration.Provider == nil {
|
||||
http.Error(w, "provider qualification not found", http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
var request modelQualificationRequest
|
||||
if !decodeJSON(w, r, 16<<10, &request) || request.Prompt == "" {
|
||||
return
|
||||
}
|
||||
if request.MaxOutputBytes < 1 {
|
||||
request.MaxOutputBytes = 512
|
||||
}
|
||||
if request.MaxOutputBytes > 4096 {
|
||||
http.Error(w, "max_output_bytes exceeds qualification bound", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
modelsFound, probe := registration.Discovery.Discover(r.Context())
|
||||
if probe.Health != capability.HealthAvailable || len(modelsFound) == 0 {
|
||||
http.Error(w, "provider unavailable for qualification", http.StatusServiceUnavailable)
|
||||
return
|
||||
}
|
||||
if len(request.ModelIDs) > 0 {
|
||||
modelsFound = slices.DeleteFunc(modelsFound, func(model models.DiscoveredModel) bool {
|
||||
return !slices.Contains(request.ModelIDs, model.ProviderModelID)
|
||||
})
|
||||
}
|
||||
if len(modelsFound) == 0 {
|
||||
http.Error(w, "requested model not discovered", http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
if len(modelsFound) > 16 {
|
||||
modelsFound = modelsFound[:16]
|
||||
}
|
||||
results := qualifyModels(r.Context(), registration, owner, request.Prompt, request.MaxOutputBytes, modelsFound)
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_ = jsonEncode(w, modelQualificationResponse{Provider: registration.ProviderInstanceID, ObservedAtUnixMillis: time.Now().UnixMilli(), Results: results})
|
||||
})
|
||||
}
|
||||
|
||||
func qualifyModels(ctx context.Context, registration providerRegistration, owner, prompt string, maxOutputBytes int, discovered []models.DiscoveredModel) []modelQualificationResult {
|
||||
results := make([]modelQualificationResult, len(discovered))
|
||||
var wg sync.WaitGroup
|
||||
for index, model := range discovered {
|
||||
wg.Add(1)
|
||||
go func(index int, model models.DiscoveredModel) {
|
||||
defer wg.Done()
|
||||
started := time.Now()
|
||||
request := cognitive.AdvisoryRequest{Version: cognitive.CreatorAdvisoryV1, RequestID: cognitive.RequestID(fmt.Sprintf("qualification:%s:%d", model.ProviderModelID, started.UnixNano())), ActorID: identity.ActorID("qualification-runtime"), PrincipalID: identity.PrincipalID(owner), ScopeID: authority.ScopeID("qualification-scope"), Message: prompt, MaxOutputBytes: maxOutputBytes, ProviderBindingID: "qualification-binding-v1", ProviderBindingRevision: 1, ProviderInstanceID: registration.ProviderInstanceID, ModelDescriptorID: capability.ModelDescriptorID("qualification-" + string(model.ProviderModelID)), ProviderModelID: model.ProviderModelID}
|
||||
response, err := registration.Provider.Advise(ctx, request)
|
||||
result := modelQualificationResult{ProviderModelID: model.ProviderModelID, LatencyMillis: time.Since(started).Milliseconds(), QualityAssessed: false, Category: capability.ModelRatingUnrated}
|
||||
if err != nil {
|
||||
result.Status = "UNAVAILABLE"
|
||||
result.Error = err.Error()
|
||||
results[index] = result
|
||||
return
|
||||
}
|
||||
result.FinishReason = response.FinishReason
|
||||
result.OutputObserved = response.AdvisoryText != ""
|
||||
if response.FinishReason == cognitive.FinishCompleted && result.OutputObserved {
|
||||
result.Status = "REACHABLE"
|
||||
} else {
|
||||
result.Status = "RESPONDED_WITHOUT_QUALIFICATION"
|
||||
}
|
||||
results[index] = result
|
||||
}(index, model)
|
||||
}
|
||||
wg.Wait()
|
||||
return results
|
||||
}
|
||||
|
||||
func jsonEncode(w http.ResponseWriter, value any) error {
|
||||
return json.NewEncoder(w).Encode(value)
|
||||
}
|
||||
@@ -0,0 +1,56 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.cin.su/m/Assistant/adapters/models"
|
||||
"git.cin.su/m/Assistant/contracts/capability"
|
||||
"git.cin.su/m/Assistant/contracts/cognitive"
|
||||
)
|
||||
|
||||
type qualificationProvider struct{}
|
||||
|
||||
func (qualificationProvider) Probe(context.Context) models.ProviderProbe {
|
||||
return models.ProviderProbe{Health: capability.HealthAvailable}
|
||||
}
|
||||
func (qualificationProvider) Discover(context.Context) ([]models.DiscoveredModel, models.ProviderProbe) {
|
||||
return []models.DiscoveredModel{{ProviderModelID: "model-a"}, {ProviderModelID: "model-b"}}, models.ProviderProbe{Health: capability.HealthAvailable}
|
||||
}
|
||||
func (qualificationProvider) Advise(_ context.Context, request cognitive.AdvisoryRequest) (cognitive.AdvisoryResponse, error) {
|
||||
return cognitive.AdvisoryResponse{Version: request.Version, RequestID: request.RequestID, AdvisoryText: "reachable", FinishReason: cognitive.FinishCompleted}, nil
|
||||
}
|
||||
|
||||
func TestModelQualificationUsesDiscoveredModelsAndLeavesQualityUnrated(t *testing.T) {
|
||||
provider := qualificationProvider{}
|
||||
mux := http.NewServeMux()
|
||||
mux.Handle("POST /v1/provider-discovery/{provider}/qualify", modelQualificationHandler(map[string]providerRegistration{"gpu": {ProviderInstanceID: "gpu", Provider: provider, Discovery: provider}}, "owner"))
|
||||
server := httptest.NewServer(mux)
|
||||
defer server.Close()
|
||||
request := httptest.NewRequest(http.MethodPost, server.URL+"/v1/provider-discovery/gpu/qualify", strings.NewReader(`{"prompt":"ready","max_output_bytes":128}`))
|
||||
request.Header.Set("Content-Type", "application/json")
|
||||
response := httptest.NewRecorder()
|
||||
mux.ServeHTTP(response, request)
|
||||
if response.Code != http.StatusOK {
|
||||
t.Fatalf("qualification status=%d body=%s", response.Code, response.Body.String())
|
||||
}
|
||||
if !strings.Contains(response.Body.String(), `"status":"REACHABLE"`) || !strings.Contains(response.Body.String(), `"category":"UNRATED"`) || strings.Contains(response.Body.String(), `"quality_assessed":true`) {
|
||||
t.Fatalf("qualification overclaimed quality: %s", response.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestModelQualificationRejectsUnboundedOutput(t *testing.T) {
|
||||
provider := qualificationProvider{}
|
||||
mux := http.NewServeMux()
|
||||
mux.Handle("POST /v1/provider-discovery/{provider}/qualify", modelQualificationHandler(map[string]providerRegistration{"cpu": {ProviderInstanceID: "cpu", Provider: provider, Discovery: provider}}, "owner"))
|
||||
request := httptest.NewRequest(http.MethodPost, "/v1/provider-discovery/cpu/qualify", strings.NewReader(`{"prompt":"ready","max_output_bytes":8192}`))
|
||||
request.Header.Set("Content-Type", "application/json")
|
||||
response := httptest.NewRecorder()
|
||||
mux.ServeHTTP(response, request)
|
||||
if response.Code != http.StatusBadRequest {
|
||||
t.Fatalf("unbounded qualification status=%d", response.Code)
|
||||
}
|
||||
}
|
||||
@@ -1,5 +1,5 @@
|
||||
{
|
||||
"task_file": "dev/tasks/DEV-000234-orchestration-snapshot-replan.json",
|
||||
"task_id": "DEV-000234",
|
||||
"task_file": "dev/tasks/DEV-000235-model-qualification-probe.json",
|
||||
"task_id": "DEV-000235",
|
||||
"task_revision": 1
|
||||
}
|
||||
|
||||
@@ -0,0 +1,23 @@
|
||||
{
|
||||
"task_id": "DEV-000235",
|
||||
"task_revision": 1,
|
||||
"architecture_revision": "ADR-0018",
|
||||
"base_repo_revision": "60e8292c1df947b887079413b3b35ce8b4212086",
|
||||
"risk_tier": "E",
|
||||
"intent": "Add a bounded provider-neutral model qualification probe that discovers actual model IDs, invokes them in parallel, records reachability and latency, and never overclaims quality or hardware superiority.",
|
||||
"requirement_ids": ["model fleet/failover", "dynamic model discovery", "quality separated from reachability", "parallel qualification"],
|
||||
"allowed_change_scope": {"modules": ["cognitive-worker", "capability-observation"], "paths": ["cmd/cognitive-worker/main.go", "cmd/cognitive-worker/model_qualification.go", "cmd/cognitive-worker/model_qualification_test.go", "dev/ACTIVE_TASK.json", "dev/tasks/DEV-000235-model-qualification-probe.json", "docs/PROJECT_STATUS.md"]},
|
||||
"forbidden_change_scope": {"paths": ["architecture/**", "security/**", "database/**", "deploy/**", "LLM-local/**", "modules/work/secrets/**"]},
|
||||
"allowed_architecture_delta": {"new_cross_module_dependencies": 0, "new_canonical_owners": 0, "new_privileges": 0, "new_third_party_dependencies": 0, "new_database_owner_roles": 0},
|
||||
"contracts": {"consume": ["OpenAI-compatible LocalProvider discovery/advisory adapter", "Capability model rating categories"], "may_change": ["internal bounded provider qualification route"]},
|
||||
"semantic_owner": "Capability cognitive runtime observation",
|
||||
"canonical_state": ["none; probe response is an operational observation and does not promote a binding"],
|
||||
"state_transitions": ["DISCOVERED model -> bounded probe -> REACHABLE/UNAVAILABLE/RESPONDED_WITHOUT_QUALIFICATION", "quality remains UNRATED until independent task tests"],
|
||||
"side_effects_and_privileges": ["read-only provider model discovery and bounded chat invocation; no external mutation, secret exposure, activation, promotion or routing priority"],
|
||||
"persistence_and_migrations": ["none; existing discovery/observation and promotion flows remain unchanged"],
|
||||
"failure_retry_restart": ["each probe has provider adapter timeout; one model failure does not cancel sibling probes; rerun is bounded and idempotent in effect"],
|
||||
"assumptions": ["provider credentials are resolved inside cognitive-worker protected runtime", "additional task-specific tests can be added later without changing the primary probe"],
|
||||
"acceptance_scenarios": ["actual discovered model IDs are used instead of static placeholders", "models are probed in parallel", "successful reachability reports latency", "quality is explicitly UNRATED", "GPU presence does not alter category or promotion", "output bound and model selection are constrained"],
|
||||
"required_tests": ["go test ./cmd/cognitive-worker/... ./modules/capability/domain ./adapters/models/...", "go vet ./cmd/cognitive-worker/... ./modules/capability/domain ./adapters/models/...", "preflight", "check-diff", "check-architecture", "independent", "git diff --check", "CI", "production health smoke"],
|
||||
"terminal_gap_states": ["BLOCKED_BY_MISSING_CAPABILITY", "BLOCKED_BY_SPEC_GAP"]
|
||||
}
|
||||
@@ -24,8 +24,8 @@
|
||||
|
||||
- `COMPLETION_MATRIX = docs/evidence/audit-completion-matrix-2026-08-23-v3.json`; all seven objective areas are mapped to current repository/production proof, with external bridge, GPU promotion/quality and deep-acceptance gaps explicit.
|
||||
|
||||
- `ACTIVE_TASK = dev/tasks/DEV-000234-orchestration-snapshot-replan.json revision 1`; orchestration waves now persist their GoalID/search snapshot in immutable Work Graph revisions, and verified graph completion returns the next durable search decision. Dispatch-intent discovery/materialization remains an explicit planner hand-off rather than fabricated authority.
|
||||
- `ACTIVE_TASK = dev/tasks/DEV-000235-model-qualification-probe.json revision 1`; cognitive-worker now exposes a bounded provider-neutral qualification probe that discovers actual model IDs, invokes them in parallel, reports reachability/latency and keeps quality `UNRATED` until independent task tests. Hardware presence does not grant promotion or priority.
|
||||
- `LINEAGE_DECISION = docs/GOVERNANCE_LINEAGE_DECISION_2026-08-22.md`; historical packets remain provenance and are not current authority.
|
||||
- `REGISTRY_STATUS = YAML and JSON semantic mirrors agree`
|
||||
- `PRODUCTION_DOMAIN = docker.cin.su`; shared docker-test is not used for this project.
|
||||
- `NEXT_WORK = connect the returned replan decision to capability/agent-intent discovery and automatic next-wave materialization; then add bounded stale-wave recovery and continue remaining audit gaps.`
|
||||
- `NEXT_WORK = use qualification evidence in the provider-neutral Capability rating/selection loop, then continue authenticated UI acceptance, reflection recovery and governed self-development gaps.`
|
||||
|
||||
Reference in New Issue
Block a user