Compare commits

...
Author SHA1 Message Date
m 7978b495d1 feat(reflection): expose worker runtime status
CI / test (push) Successful in 48s
CI / test (pull_request) Successful in 46s
2026-08-23 19:42:07 +03:00
m d73d4fc118 Merge pull request 'feat(capability): consume qualification evidence in selection' (#220) from codex/dev-000243-qualification-selection-evidence into main
CI / test (push) Successful in 1m8s
feat(capability): consume qualification evidence in selection
2026-08-23 16:30:23 +00:00
m c81e2269e1 feat(capability): consume qualification evidence in selection
CI / test (push) Successful in 1m16s
CI / test (pull_request) Successful in 1m26s
2026-08-23 19:28:13 +03:00
m 47327cc6d8 Merge pull request 'feat(capability): persist model qualification observations' (#219) from codex/dev-000242-durable-qualification-observations into main
CI / test (push) Successful in 1m16s
feat(capability): persist model qualification observations
2026-08-23 16:22:58 +00:00
m d0ede46352 feat(capability): persist model qualification observations
CI / test (push) Successful in 1m14s
CI / test (pull_request) Successful in 1m15s
2026-08-23 19:21:04 +03:00
m e4ba1cb5fa Merge pull request 'docs(fleet): record medium and gpu qualification' (#218) from codex/dev-000241-fleet-qualification-evidence into main
CI / test (push) Successful in 1m15s
docs(fleet): record medium and gpu qualification
2026-08-23 16:13:58 +00:00
18 changed files with 411 additions and 4 deletions
@@ -505,6 +505,14 @@ func (r *CapabilityRepository) loadCognitiveEvidence(ctx context.Context, value
if err == nil {
value.Performance = &performance
}
var qualification capability.ModelQualificationObservation
err = r.pool.QueryRow(ctx, `SELECT observation_id,provider_instance_id,provider_model_id,status,finish_reason,latency_millis,output_observed,quality_assessed,category,observed_at_unix_millis FROM capability.cognitive_model_qualification_observations WHERE provider_instance_id=$1 AND provider_model_id=$2 ORDER BY observed_at_unix_millis DESC LIMIT 1`, value.Provider.ID, value.Model.ProviderModelID).Scan(&qualification.ID, &qualification.ProviderInstanceID, &qualification.ProviderModelID, &qualification.Status, &qualification.FinishReason, &qualification.LatencyMillis, &qualification.OutputObserved, &qualification.QualityAssessed, &qualification.Category, &qualification.ObservedAtUnixMillis)
if err != nil && !errors.Is(err, pgx.ErrNoRows) {
return err
}
if err == nil {
value.Qualification = &qualification
}
if profileID != "" {
var taskEvaluation capability.TaskEvaluation
err = r.pool.QueryRow(ctx, `SELECT evaluation_id,task_profile_id,binding_id,binding_revision,capability_id,result,quality_score_millis,reliability_score_millis,latency_score_millis,overall_score_millis,critical_failure_millis,benchmark_version,artifact_id,observed_at_unix_millis FROM capability.cognitive_task_evaluations WHERE task_profile_id=$1 AND binding_id=$2 AND binding_revision=$3 ORDER BY observed_at_unix_millis DESC LIMIT 1`, profileID, value.Binding.ID, value.Binding.Revision).Scan(&taskEvaluation.ID, &taskEvaluation.ProfileID, &taskEvaluation.BindingID, &taskEvaluation.BindingRevision, &taskEvaluation.CapabilityID, &taskEvaluation.Result, &taskEvaluation.QualityScoreMillis, &taskEvaluation.ReliabilityScoreMillis, &taskEvaluation.LatencyScoreMillis, &taskEvaluation.OverallScoreMillis, &taskEvaluation.CriticalFailureMillis, &taskEvaluation.BenchmarkVersion, &taskEvaluation.ArtifactID, &taskEvaluation.ObservedAtUnixMillis)
@@ -633,3 +641,8 @@ func (r *CapabilityRepository) SaveTaskEvaluation(ctx context.Context, value cap
_, err := r.pool.Exec(ctx, `INSERT INTO capability.cognitive_task_evaluations (evaluation_id,task_profile_id,binding_id,binding_revision,capability_id,result,quality_score_millis,reliability_score_millis,latency_score_millis,overall_score_millis,critical_failure_millis,benchmark_version,artifact_id,observed_at_unix_millis) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14) ON CONFLICT DO NOTHING`, value.ID, value.ProfileID, value.BindingID, value.BindingRevision, value.CapabilityID, value.Result, value.QualityScoreMillis, value.ReliabilityScoreMillis, value.LatencyScoreMillis, value.OverallScoreMillis, value.CriticalFailureMillis, value.BenchmarkVersion, value.ArtifactID, value.ObservedAtUnixMillis)
return err
}
func (r *CapabilityRepository) SaveModelQualificationObservation(ctx context.Context, value capability.ModelQualificationObservation) error {
_, err := r.pool.Exec(ctx, `INSERT INTO capability.cognitive_model_qualification_observations (observation_id,provider_instance_id,provider_model_id,status,finish_reason,latency_millis,output_observed,quality_assessed,category,observed_at_unix_millis) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10) ON CONFLICT DO NOTHING`, value.ID, value.ProviderInstanceID, value.ProviderModelID, value.Status, value.FinishReason, value.LatencyMillis, value.OutputObserved, value.QualityAssessed, value.Category, value.ObservedAtUnixMillis)
return err
}
+1 -1
View File
@@ -169,7 +169,7 @@ func main() {
mux.HandleFunc("POST /v1/task-evaluations/execute", gateway.taskEvaluationHandler)
discovery := discoveryService{resolver: resolver, providers: discoveries, owner: identity.PrincipalID(envOr("COGNITIVE_OWNER_PRINCIPAL_ID", "domovoy-creator-principal")), now: time.Now}
mux.HandleFunc("POST /v1/provider-discovery/{provider}", discovery.handler)
mux.Handle("POST /v1/provider-discovery/{provider}/qualify", modelQualificationHandler(discoveries, string(envOr("COGNITIVE_OWNER_PRINCIPAL_ID", "domovoy-creator-principal"))))
mux.Handle("POST /v1/provider-discovery/{provider}/qualify", modelQualificationHandler(discoveries, string(envOr("COGNITIVE_OWNER_PRINCIPAL_ID", "domovoy-creator-principal")), httpQualificationObserver{resolver: resolver}))
mux.HandleFunc("GET /v1/provider-discovery/status", discovery.statusHandler)
refreshProviderDiscoveries(resolver, discovery)
log.Printf("cognitive-worker listening on %s", addr)
+51 -1
View File
@@ -14,8 +14,22 @@ import (
"git.cin.su/m/Assistant/contracts/capability"
"git.cin.su/m/Assistant/contracts/cognitive"
"git.cin.su/m/Assistant/contracts/identity"
"git.cin.su/m/Assistant/platform/ids"
)
type qualificationObserver interface {
ObserveQualification(context.Context, capability.ModelQualificationObservation) error
}
type httpQualificationObserver struct{ resolver *httpBindingResolver }
func (o httpQualificationObserver) ObserveQualification(ctx context.Context, value capability.ModelQualificationObservation) error {
if o.resolver == nil {
return fmt.Errorf("qualification observer resolver is nil")
}
return o.resolver.post(ctx, "/v1/cognitive-registry/model-qualification-observations", value, nil)
}
type modelQualificationRequest struct {
Prompt string `json:"prompt"`
MaxOutputBytes int `json:"max_output_bytes"`
@@ -39,7 +53,7 @@ type modelQualificationResponse struct {
Results []modelQualificationResult `json:"results"`
}
func modelQualificationHandler(providers map[string]providerRegistration, owner string) http.Handler {
func modelQualificationHandler(providers map[string]providerRegistration, owner string, observers ...qualificationObserver) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
name := r.PathValue("provider")
registration, ok := providers[name]
@@ -76,6 +90,12 @@ func modelQualificationHandler(providers map[string]providerRegistration, owner
modelsFound = modelsFound[:16]
}
results := qualifyModels(r.Context(), registration, owner, request.Prompt, request.MaxOutputBytes, modelsFound)
if len(observers) > 0 && observers[0] != nil {
if err := persistQualificationResults(r.Context(), observers[0], registration, results); err != nil {
http.Error(w, "persist model qualification observation", http.StatusBadGateway)
return
}
}
w.Header().Set("Content-Type", "application/json")
_ = jsonEncode(w, modelQualificationResponse{Provider: registration.ProviderInstanceID, ObservedAtUnixMillis: time.Now().UnixMilli(), Results: results})
})
@@ -94,6 +114,7 @@ func qualifyModels(ctx context.Context, registration providerRegistration, owner
result := modelQualificationResult{ProviderModelID: model.ProviderModelID, LatencyMillis: time.Since(started).Milliseconds(), QualityAssessed: false, Category: capability.ModelRatingUnrated}
if err != nil {
result.Status = "UNAVAILABLE"
result.FinishReason = cognitive.FinishProviderUnavailable
result.Error = err.Error()
results[index] = result
return
@@ -112,6 +133,35 @@ func qualifyModels(ctx context.Context, registration providerRegistration, owner
return results
}
func persistQualificationResults(ctx context.Context, observer qualificationObserver, registration providerRegistration, results []modelQualificationResult) error {
for _, result := range results {
observationID, err := ids.New()
if err != nil {
return err
}
status := capability.ModelQualificationRespondedWithoutRating
switch result.Status {
case "REACHABLE":
status = capability.ModelQualificationReachable
case "UNAVAILABLE":
status = capability.ModelQualificationUnavailable
}
observation := capability.ModelQualificationObservation{
ID: capability.ModelQualificationObservationID(observationID), ProviderInstanceID: registration.ProviderInstanceID, ProviderModelID: result.ProviderModelID,
Status: status, FinishReason: string(result.FinishReason), LatencyMillis: result.LatencyMillis,
OutputObserved: result.OutputObserved, QualityAssessed: false, Category: capability.ModelRatingUnrated,
ObservedAtUnixMillis: time.Now().UnixMilli(),
}
if err := observation.Validate(); err != nil {
return err
}
if err := observer.ObserveQualification(ctx, observation); err != nil {
return err
}
}
return nil
}
func jsonEncode(w http.ResponseWriter, value any) error {
return json.NewEncoder(w).Encode(value)
}
@@ -57,3 +57,29 @@ func TestModelQualificationRejectsUnboundedOutput(t *testing.T) {
t.Fatalf("unbounded qualification status=%d", response.Code)
}
}
type qualificationObserverStub struct {
observations []capability.ModelQualificationObservation
}
func (s *qualificationObserverStub) ObserveQualification(_ context.Context, value capability.ModelQualificationObservation) error {
s.observations = append(s.observations, value)
return nil
}
func TestModelQualificationPersistsReachabilityWithoutQuality(t *testing.T) {
provider := qualificationProvider{}
observer := &qualificationObserverStub{}
mux := http.NewServeMux()
mux.Handle("POST /v1/provider-discovery/{provider}/qualify", modelQualificationHandler(map[string]providerRegistration{"cpu": {ProviderInstanceID: "cpu", Provider: provider, Discovery: provider}}, "owner", observer))
request := httptest.NewRequest(http.MethodPost, "/v1/provider-discovery/cpu/qualify", strings.NewReader(`{"prompt":"ready","max_output_bytes":128,"model_ids":["model-a"]}`))
request.Header.Set("Content-Type", "application/json")
response := httptest.NewRecorder()
mux.ServeHTTP(response, request)
if response.Code != http.StatusOK || len(observer.observations) != 1 {
t.Fatalf("qualification persistence status=%d observations=%d body=%s", response.Code, len(observer.observations), response.Body.String())
}
if err := observer.observations[0].Validate(); err != nil || observer.observations[0].Status != capability.ModelQualificationReachable {
t.Fatalf("invalid qualification observation: %+v err=%v", observer.observations[0], err)
}
}
+14
View File
@@ -1058,6 +1058,20 @@ func main() {
}
w.WriteHeader(http.StatusCreated)
})
mux.HandleFunc("POST /v1/cognitive-registry/model-qualification-observations", func(w http.ResponseWriter, r *http.Request) {
var value capability.ModelQualificationObservation
decoder := json.NewDecoder(io.LimitReader(r.Body, 16<<10))
decoder.DisallowUnknownFields()
if err := decoder.Decode(&value); err != nil || value.Validate() != nil {
http.Error(w, "invalid model qualification observation", http.StatusBadRequest)
return
}
if err := capabilityRepository.SaveModelQualificationObservation(r.Context(), value); err != nil {
http.Error(w, "save model qualification observation", http.StatusInternalServerError)
return
}
w.WriteHeader(http.StatusCreated)
})
mux.HandleFunc("POST /v1/creator/cognitive-canary", func(w http.ResponseWriter, r *http.Request) {
session, ok := authenticateCreator(w, r)
if !ok {
+17
View File
@@ -4,6 +4,8 @@ import (
"context"
"flag"
"fmt"
"log"
"net/http"
"os"
"time"
@@ -29,9 +31,21 @@ func main() {
panic(err)
}
defer repository.Close()
status := newWorkerStatus(*workerID, *once)
listenAddr := os.Getenv("REFLECTION_LISTEN_ADDR")
if listenAddr == "" {
listenAddr = ":8092"
}
go func() {
if err := http.ListenAndServe(listenAddr, status.handler()); err != nil {
log.Printf("reflection status server stopped: %v", err)
}
}()
for {
now := time.Now().UnixMilli()
status.cycleStarted(time.Now())
if _, err := repository.RecoverReflectionClaims(ctx, now); err != nil {
status.cycleFailed(time.Now(), err)
panic(err)
}
candidate, claimErr := repository.ClaimReflectionCandidate(ctx, *workerID, now, lease.Milliseconds())
@@ -39,16 +53,19 @@ func main() {
// This worker only closes the durable evaluation loop. Any cognitive
// proposal remains a candidate and must be admitted by a separate governed flow.
if err := repository.ResolveReflectionCandidate(ctx, candidate.ID, *workerID, time.Now().UnixMilli()); err != nil {
status.cycleFailed(time.Now(), err)
panic(err)
}
fmt.Printf("resolved reflection candidate %s\n", candidate.ID)
} else if *once {
fmt.Println("no reflection candidate")
}
status.cycleFinished(time.Now(), map[bool]string{true: "candidate-resolved", false: "idle"}[claimErr == nil])
if *once {
return
}
if err := repository.WaitForReflectionCandidate(ctx, *pollInterval); err != nil {
status.cycleFailed(time.Now(), err)
panic(err)
}
}
+74
View File
@@ -0,0 +1,74 @@
package main
import (
"encoding/json"
"net/http"
"sync"
"time"
"git.cin.su/m/Assistant/runtime/health"
)
type workerStatus struct {
mu sync.RWMutex
workerID string
once bool
startedAt time.Time
lastCycleAt time.Time
lastCompleted time.Time
lastResult string
lastError string
}
func newWorkerStatus(workerID string, once bool) *workerStatus {
return &workerStatus{workerID: workerID, once: once, startedAt: time.Now().UTC()}
}
func (s *workerStatus) cycleStarted(at time.Time) {
s.mu.Lock()
defer s.mu.Unlock()
s.lastCycleAt = at.UTC()
s.lastResult = "running"
s.lastError = ""
}
func (s *workerStatus) cycleFinished(at time.Time, result string) {
s.mu.Lock()
defer s.mu.Unlock()
s.lastCompleted = at.UTC()
s.lastResult = result
s.lastError = ""
}
func (s *workerStatus) cycleFailed(at time.Time, err error) {
s.mu.Lock()
defer s.mu.Unlock()
s.lastCompleted = at.UTC()
s.lastResult = "failed"
s.lastError = err.Error()
}
func (s *workerStatus) handler() http.Handler {
mux := http.NewServeMux()
mux.Handle("/healthz", health.Handler{Runtime: "reflection-worker", SemanticGeneration: 1, ArchitectureGeneration: 1})
mux.HandleFunc("/status", func(w http.ResponseWriter, _ *http.Request) {
s.mu.RLock()
response := struct {
Runtime string `json:"runtime"`
WorkerID string `json:"worker_id"`
Mode string `json:"mode"`
StartedAt time.Time `json:"started_at"`
LastCycleAt time.Time `json:"last_cycle_at,omitempty"`
LastCompleted time.Time `json:"last_completed_at,omitempty"`
LastResult string `json:"last_result"`
LastError string `json:"last_error,omitempty"`
}{Runtime: "reflection-worker", WorkerID: s.workerID, Mode: "durable-reflection-loop", StartedAt: s.startedAt, LastCycleAt: s.lastCycleAt, LastCompleted: s.lastCompleted, LastResult: s.lastResult, LastError: s.lastError}
if s.once {
response.Mode = "durable-reflection-once"
}
s.mu.RUnlock()
w.Header().Set("Content-Type", "application/json")
_ = json.NewEncoder(w).Encode(response)
})
return mux
}
+35
View File
@@ -0,0 +1,35 @@
package main
import (
"net/http"
"net/http/httptest"
"strings"
"testing"
"time"
)
func TestWorkerStatusReportsCycle(t *testing.T) {
status := newWorkerStatus("test-worker", false)
status.cycleStarted(time.Date(2026, 8, 23, 10, 0, 0, 0, time.UTC))
status.cycleFinished(time.Date(2026, 8, 23, 10, 0, 1, 0, time.UTC), "idle")
recorder := httptest.NewRecorder()
status.handler().ServeHTTP(recorder, httptest.NewRequest(http.MethodGet, "/status", nil))
if recorder.Code != http.StatusOK {
t.Fatalf("status code = %d, want 200", recorder.Code)
}
body := recorder.Body.String()
for _, expected := range []string{"\"runtime\":\"reflection-worker\"", "\"worker_id\":\"test-worker\"", "\"last_result\":\"idle\""} {
if !strings.Contains(body, expected) {
t.Fatalf("status body missing %q: %s", expected, body)
}
}
}
func TestWorkerHealthIsAvailable(t *testing.T) {
recorder := httptest.NewRecorder()
newWorkerStatus("test-worker", true).handler().ServeHTTP(recorder, httptest.NewRequest(http.MethodGet, "/healthz", nil))
if recorder.Code != http.StatusOK {
t.Fatalf("health status code = %d, want 200", recorder.Code)
}
}
@@ -172,6 +172,7 @@ type ResolvedCognitiveBinding struct {
Evaluation *EvaluatedCapability
Performance *PerformanceEvidence
TaskEvaluation *TaskEvaluation
Qualification *ModelQualificationObservation
Rating *ModelRating
}
@@ -0,0 +1,47 @@
package capability
import (
"errors"
"git.cin.su/m/Assistant/platform/ids"
)
type ModelQualificationObservationID ids.ID
type ModelQualificationStatus string
const (
ModelQualificationReachable ModelQualificationStatus = "REACHABLE"
ModelQualificationRespondedWithoutRating ModelQualificationStatus = "RESPONDED_WITHOUT_QUALIFICATION"
ModelQualificationUnavailable ModelQualificationStatus = "UNAVAILABLE"
)
// ModelQualificationObservation is operational evidence only. It records
// reachability and latency, never quality, activation, routing priority or
// provider credentials.
type ModelQualificationObservation struct {
ID ModelQualificationObservationID
ProviderInstanceID ProviderInstanceID
ProviderModelID ProviderModelID
Status ModelQualificationStatus
FinishReason string
LatencyMillis int64
OutputObserved bool
QualityAssessed bool
Category ModelRatingCategory
ObservedAtUnixMillis int64
}
func (o ModelQualificationObservation) Validate() error {
if o.ID == "" || o.ProviderInstanceID == "" || o.ProviderModelID == "" || o.FinishReason == "" || o.LatencyMillis < 0 || o.ObservedAtUnixMillis < 1 {
return errors.New("model qualification observation identity and timing are incomplete")
}
switch o.Status {
case ModelQualificationReachable, ModelQualificationRespondedWithoutRating, ModelQualificationUnavailable:
default:
return errors.New("model qualification observation status is unsupported")
}
if o.Category != ModelRatingUnrated || o.QualityAssessed {
return errors.New("model qualification observation cannot assert quality")
}
return nil
}
@@ -0,0 +1,24 @@
BEGIN;
SET ROLE capability_role;
CREATE TABLE IF NOT EXISTS capability.cognitive_model_qualification_observations (
observation_id TEXT PRIMARY KEY,
provider_instance_id TEXT NOT NULL,
provider_model_id TEXT NOT NULL,
status TEXT NOT NULL CHECK (status IN ('REACHABLE','RESPONDED_WITHOUT_QUALIFICATION','UNAVAILABLE')),
finish_reason TEXT NOT NULL,
latency_millis BIGINT NOT NULL CHECK (latency_millis >= 0),
output_observed BOOLEAN NOT NULL,
quality_assessed BOOLEAN NOT NULL DEFAULT FALSE CHECK (quality_assessed = FALSE),
category TEXT NOT NULL CHECK (category = 'UNRATED'),
observed_at_unix_millis BIGINT NOT NULL CHECK (observed_at_unix_millis > 0)
);
CREATE INDEX IF NOT EXISTS cognitive_model_qualification_lookup
ON capability.cognitive_model_qualification_observations(provider_instance_id, provider_model_id, observed_at_unix_millis DESC);
GRANT SELECT, INSERT ON capability.cognitive_model_qualification_observations TO capability_role;
RESET ROLE;
COMMIT;
+2 -2
View File
@@ -1,5 +1,5 @@
{
"task_file": "dev/tasks/DEV-000241-fleet-qualification-evidence.json",
"task_id": "DEV-000241",
"task_file": "dev/tasks/DEV-000244-reflection-worker-observability.json",
"task_id": "DEV-000244",
"task_revision": 1
}
@@ -0,0 +1,23 @@
{
"task_id": "DEV-000242",
"task_revision": 1,
"architecture_revision": "ADR-0018",
"base_repo_revision": "e4ba1cb5fabf930396b2aa9f95d3b4b205e6319d",
"risk_tier": "E",
"intent": "Persist provider-neutral model qualification observations through the Capability owner while keeping quality unrated until independent task evaluation.",
"requirement_ids": ["durable qualification evidence", "model fleet/failover", "quality separated from reachability"],
"allowed_change_scope": {"modules": ["capability", "core-runtime", "cognitive-worker", "database"], "paths": ["adapters/postgres/capability_repository.go", "cmd/cognitive-worker/main.go", "cmd/cognitive-worker/model_qualification.go", "cmd/cognitive-worker/model_qualification_test.go", "cmd/core-runtime/main.go", "contracts/capability/model_qualification.go", "database/migrations/045_capability_model_qualification_observations.sql", "modules/capability/ports/repository.go", "dev/ACTIVE_TASK.json", "dev/tasks/DEV-000242-durable-qualification-observations.json"]},
"forbidden_change_scope": {"paths": ["architecture/**", "security/**", "deploy/**", "LLM-local/**", "modules/work/secrets/**", "contracts/work/**"]},
"allowed_architecture_delta": {"new_cross_module_dependencies": 0, "new_canonical_owners": 0, "new_privileges": 0, "new_third_party_dependencies": 0, "new_database_owner_roles": 1},
"contracts": {"consume": ["bounded qualification result", "Capability repository boundary"], "may_change": ["typed model qualification observation contract", "internal core persistence route"]},
"semantic_owner": "Capability owns qualification evidence; cognitive-worker produces observations",
"canonical_state": ["cognitive_model_qualification_observations"],
"state_transitions": ["probe -> REACHABLE/RESPONDED_WITHOUT_QUALIFICATION/UNAVAILABLE; category remains UNRATED"],
"side_effects_and_privileges": ["bounded provider call and capability-owned INSERT; no promotion, activation or secret persistence"],
"persistence_and_migrations": ["migration 045 creates capability-owned observation table and index"],
"failure_retry_restart": ["observation persistence failure returns bounded gateway error; duplicate IDs are idempotent"],
"assumptions": ["independent task evaluation remains the only quality/promotion evidence"],
"acceptance_scenarios": ["qualification route persists one observation per probed model", "invalid quality/category is rejected", "prompt/output/error text is not persisted", "existing qualification tests remain valid"],
"required_tests": ["go test ./cmd/cognitive-worker/... ./cmd/core-runtime/... ./modules/capability/... ./adapters/postgres/...", "go vet ./cmd/cognitive-worker/... ./cmd/core-runtime/... ./modules/capability/... ./adapters/postgres/...", "go run ./cmd/devctl preflight -active dev/ACTIVE_TASK.json", "go run ./cmd/devctl check-diff -active dev/ACTIVE_TASK.json", "go run ./cmd/devctl check-architecture", "git diff --check", "independent", "CI", "production health smoke"],
"terminal_gap_states": ["BLOCKED_BY_MISSING_CAPABILITY", "BLOCKED_BY_SPEC_GAP", "REQUIRES_MIGRATION_DECISION"]
}
@@ -0,0 +1,23 @@
{
"task_id": "DEV-000243",
"task_revision": 1,
"architecture_revision": "ADR-0018",
"base_repo_revision": "47327cc6d853da9084c96e3bf48502266706ca57",
"risk_tier": "E",
"intent": "Make Capability selection consume the latest durable qualification observation for fresh unavailability exclusion and transparent latency rating, without granting quality or activation.",
"requirement_ids": ["qualification selection loop", "hardware-neutral routing", "quality separated from reachability"],
"allowed_change_scope": {"modules": ["capability", "adapters/postgres"], "paths": ["adapters/postgres/capability_repository.go", "contracts/capability/cognitive_registry.go", "modules/capability/domain/cognitive_registry.go", "modules/capability/domain/model_rating_test.go", "dev/ACTIVE_TASK.json", "dev/tasks/DEV-000243-qualification-selection-evidence.json"]},
"forbidden_change_scope": {"paths": ["architecture/**", "security/**", "database/**", "deploy/**", "LLM-local/**", "modules/work/secrets/**", "contracts/work/**"]},
"allowed_architecture_delta": {"new_cross_module_dependencies": 0, "new_canonical_owners": 0, "new_privileges": 0, "new_third_party_dependencies": 0, "new_database_owner_roles": 0},
"contracts": {"consume": ["durable ModelQualificationObservation"], "may_change": ["resolved binding evidence projection"]},
"semantic_owner": "Capability selection and rating",
"canonical_state": ["qualification observation remains evidence; no activation mutation"],
"state_transitions": ["fresh UNAVAILABLE observation -> candidate excluded", "stale UNAVAILABLE observation -> discovery state remains authoritative", "REACHABLE observation -> latency evidence only"],
"side_effects_and_privileges": ["read-only projection and deterministic selection"],
"persistence_and_migrations": ["uses migration 045 from DEV-000242"],
"failure_retry_restart": ["missing observation preserves existing behavior"],
"assumptions": ["provider/model observed timestamps define freshness boundary"],
"acceptance_scenarios": ["fresh unavailable candidate is not selected", "stale unavailable candidate is not permanently poisoned", "quality remains task-evaluation controlled"],
"required_tests": ["go test ./modules/capability/... ./adapters/postgres/...", "go vet ./modules/capability/... ./adapters/postgres/...", "go run ./cmd/devctl preflight -active dev/ACTIVE_TASK.json", "go run ./cmd/devctl check-diff -active dev/ACTIVE_TASK.json", "go run ./cmd/devctl check-architecture", "git diff --check", "independent", "CI", "production health smoke"],
"terminal_gap_states": ["BLOCKED_BY_MISSING_CAPABILITY", "BLOCKED_BY_SPEC_GAP"]
}
@@ -0,0 +1,26 @@
{
"task_id": "DEV-000244",
"task_revision": 1,
"architecture_revision": "ADR-0018",
"base_repo_revision": "c81e2269e157c1a41b567d79d93bbbe49dd4b134",
"risk_tier": "E",
"intent": "Expose bounded health and status evidence for the durable reflection worker without changing reflection semantics or authority.",
"requirement_ids": ["learning reflection loop", "runtime observability", "fast operational verification"],
"allowed_change_scope": {
"modules": ["epistemics"],
"paths": ["cmd/reflection-worker/**", "dev/ACTIVE_TASK.json", "dev/tasks/DEV-000244-reflection-worker-observability.json"]
},
"forbidden_change_scope": {"paths": ["architecture/**", "security/**", "database/**", "LLM-local/**", "modules/work/secrets/**", "contracts/**"]},
"allowed_architecture_delta": {"new_cross_module_dependencies": 0, "new_canonical_owners": 0, "new_privileges": 0, "new_third_party_dependencies": 0, "new_database_owner_roles": 0},
"contracts": {"consume": ["existing reflection repository port", "runtime health conventions"], "may_change": ["worker operational HTTP surface"]},
"semantic_owner": "Epistemics reflection worker runtime",
"canonical_state": ["none; status is derived operational evidence"],
"state_transitions": ["cycle start/complete/failure are reported without changing candidate lifecycle rules"],
"side_effects_and_privileges": ["read-only health/status endpoint; existing repository writes unchanged"],
"persistence_and_migrations": ["none"],
"failure_retry_restart": ["status resets on process restart; durable candidate state remains authoritative"],
"assumptions": ["internal production network is sufficient for worker status checks"],
"acceptance_scenarios": ["health endpoint returns 200", "status exposes loop mode and latest cycle", "reflection loop still resolves candidates"],
"required_tests": ["go test ./cmd/reflection-worker", "go vet ./cmd/reflection-worker", "go run ./cmd/devctl preflight -active dev/ACTIVE_TASK.json", "go run ./cmd/devctl check-diff -active dev/ACTIVE_TASK.json", "go run ./cmd/devctl check-architecture", "git diff --check", "independent", "CI", "production health smoke"],
"terminal_gap_states": ["BLOCKED_BY_MISSING_CAPABILITY", "BLOCKED_BY_SPEC_GAP"]
}
@@ -148,6 +148,9 @@ func SelectCognitiveBinding(requirement capability.CognitiveRequirement, candida
if candidate.Evaluation != nil && candidate.Evaluation.Result == capability.EvaluationUnsupported {
continue
}
if candidate.Qualification != nil && candidate.Qualification.Status == capability.ModelQualificationUnavailable && candidate.Qualification.ObservedAtUnixMillis >= maxObservedAt(candidate.Provider.ObservedAtUnixMillis, candidate.Model.ObservedAtUnixMillis) {
continue
}
if requirement.TaskProfileID != "" && (candidate.TaskEvaluation == nil || candidate.TaskEvaluation.ProfileID != requirement.TaskProfileID || candidate.TaskEvaluation.Result != capability.EvaluationSupported) {
continue
}
@@ -195,10 +198,14 @@ func RateCognitiveBinding(candidate capability.ResolvedCognitiveBinding) capabil
latency := int64(500)
if candidate.Performance != nil && candidate.Performance.LatencyMicros != nil {
latency = latencyScore(*candidate.Performance.LatencyMicros)
} else if candidate.Qualification != nil && candidate.Qualification.Status == capability.ModelQualificationReachable {
latency = latencyScore(candidate.Qualification.LatencyMillis * 1000)
}
evidence := int64(700)
if candidate.Performance != nil {
evidence = 1000
} else if candidate.Qualification != nil && candidate.Qualification.Status == capability.ModelQualificationReachable {
evidence = 750
}
overall := (quality*55 + reliability*25 + latency*15 + evidence*5) / 100
category := capability.ModelRatingBalanced
@@ -210,6 +217,13 @@ func RateCognitiveBinding(candidate capability.ResolvedCognitiveBinding) capabil
return capability.ModelRating{Category: category, OverallScoreMillis: overall, QualityScoreMillis: quality, ReliabilityScoreMillis: reliability, LatencyScoreMillis: latency, EvidenceScoreMillis: evidence}
}
func maxObservedAt(left, right int64) int64 {
if left > right {
return left
}
return right
}
func clampMillis(value int64) int64 {
if value < 0 {
return 0
@@ -38,3 +38,22 @@ func TestRateCognitiveBindingClassifiesSlowEvidenceAsBalanced(t *testing.T) {
t.Fatalf("rating=%+v", rating)
}
}
func TestSelectionExcludesFreshUnavailableQualificationButNotStaleEvidence(t *testing.T) {
quality, critical := int64(900), int64(0)
base := capability.ResolvedCognitiveBinding{
Binding: capability.CognitiveBinding{ID: "binding-qualification", Revision: 1, CapabilityID: "creator-advisory", OwnerPrincipalID: "owner", ProviderInstanceID: "provider", ProviderRevision: 1, ModelDescriptorID: "model", ModelRevision: 1, DataPolicy: capability.DataPolicyLocalOnly, Qualification: capability.QualificationValidated, Activation: capability.ActivationActive, Health: capability.HealthAvailable, LimitPolicyRef: "limits"},
Provider: capability.ProviderInstance{ID: "provider", Revision: 1, Kind: "local", EndpointRef: "endpoint", Enabled: true, Qualification: capability.QualificationValidated, Health: capability.HealthAvailable, ObservedAtUnixMillis: 20},
Model: capability.ModelDescriptor{ID: "model", Revision: 1, ProviderInstanceID: "provider", ProviderModelID: "model", MaxContextBytes: 1000, Modalities: []capability.ModelModality{capability.ModalityText}, Capabilities: []capability.ModelCapability{capability.CapabilityAdvisory}, Qualification: capability.QualificationValidated, Health: capability.HealthAvailable, ObservedAtUnixMillis: 20},
Evaluation: &capability.EvaluatedCapability{ID: "evaluation", BindingID: "binding-qualification", BindingRevision: 1, CapabilityID: "creator-advisory", Result: capability.EvaluationSupported, QualityScoreMillis: &quality, CriticalFailureMillis: &critical},
Qualification: &capability.ModelQualificationObservation{ID: "observation", ProviderInstanceID: "provider", ProviderModelID: "model", Status: capability.ModelQualificationUnavailable, FinishReason: "TIMEOUT", ObservedAtUnixMillis: 20, Category: capability.ModelRatingUnrated},
}
requirement := capability.CognitiveRequirement{CapabilityID: "creator-advisory", OwnerPrincipalID: "owner", DataPolicy: capability.DataPolicyLocalOnly, RequiredModalities: []capability.ModelModality{capability.ModalityText}, RequiredCapabilities: []capability.ModelCapability{capability.CapabilityAdvisory}, MaxOutputBytes: 10}
if _, err := SelectCognitiveBinding(requirement, []capability.ResolvedCognitiveBinding{base}); err == nil {
t.Fatal("fresh unavailable qualification should exclude candidate")
}
base.Qualification.ObservedAtUnixMillis = 19
if _, err := SelectCognitiveBinding(requirement, []capability.ResolvedCognitiveBinding{base}); err != nil {
t.Fatalf("stale unavailable qualification should not exclude candidate: %v", err)
}
}
+1
View File
@@ -27,4 +27,5 @@ type Repository interface {
SaveEvaluatedCapability(context.Context, capability.EvaluatedCapability) error
SavePerformanceEvidence(context.Context, capability.PerformanceEvidence) error
SaveTaskEvaluation(context.Context, capability.TaskEvaluation) error
SaveModelQualificationObservation(context.Context, capability.ModelQualificationObservation) error
}