Compare commits
12
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
c81e2269e1 | ||
|
|
47327cc6d8 | ||
|
|
d0ede46352 | ||
|
|
e4ba1cb5fa | ||
|
|
90104d2553 | ||
|
|
ca0d002073 | ||
|
|
20db36f52e | ||
|
|
383d4c6e4f | ||
|
|
f3ddb45fe5 | ||
|
|
6061eb5c13 | ||
|
|
0e46a64702 | ||
|
|
0609561dfe |
@@ -505,6 +505,14 @@ func (r *CapabilityRepository) loadCognitiveEvidence(ctx context.Context, value
|
||||
if err == nil {
|
||||
value.Performance = &performance
|
||||
}
|
||||
var qualification capability.ModelQualificationObservation
|
||||
err = r.pool.QueryRow(ctx, `SELECT observation_id,provider_instance_id,provider_model_id,status,finish_reason,latency_millis,output_observed,quality_assessed,category,observed_at_unix_millis FROM capability.cognitive_model_qualification_observations WHERE provider_instance_id=$1 AND provider_model_id=$2 ORDER BY observed_at_unix_millis DESC LIMIT 1`, value.Provider.ID, value.Model.ProviderModelID).Scan(&qualification.ID, &qualification.ProviderInstanceID, &qualification.ProviderModelID, &qualification.Status, &qualification.FinishReason, &qualification.LatencyMillis, &qualification.OutputObserved, &qualification.QualityAssessed, &qualification.Category, &qualification.ObservedAtUnixMillis)
|
||||
if err != nil && !errors.Is(err, pgx.ErrNoRows) {
|
||||
return err
|
||||
}
|
||||
if err == nil {
|
||||
value.Qualification = &qualification
|
||||
}
|
||||
if profileID != "" {
|
||||
var taskEvaluation capability.TaskEvaluation
|
||||
err = r.pool.QueryRow(ctx, `SELECT evaluation_id,task_profile_id,binding_id,binding_revision,capability_id,result,quality_score_millis,reliability_score_millis,latency_score_millis,overall_score_millis,critical_failure_millis,benchmark_version,artifact_id,observed_at_unix_millis FROM capability.cognitive_task_evaluations WHERE task_profile_id=$1 AND binding_id=$2 AND binding_revision=$3 ORDER BY observed_at_unix_millis DESC LIMIT 1`, profileID, value.Binding.ID, value.Binding.Revision).Scan(&taskEvaluation.ID, &taskEvaluation.ProfileID, &taskEvaluation.BindingID, &taskEvaluation.BindingRevision, &taskEvaluation.CapabilityID, &taskEvaluation.Result, &taskEvaluation.QualityScoreMillis, &taskEvaluation.ReliabilityScoreMillis, &taskEvaluation.LatencyScoreMillis, &taskEvaluation.OverallScoreMillis, &taskEvaluation.CriticalFailureMillis, &taskEvaluation.BenchmarkVersion, &taskEvaluation.ArtifactID, &taskEvaluation.ObservedAtUnixMillis)
|
||||
@@ -633,3 +641,8 @@ func (r *CapabilityRepository) SaveTaskEvaluation(ctx context.Context, value cap
|
||||
_, err := r.pool.Exec(ctx, `INSERT INTO capability.cognitive_task_evaluations (evaluation_id,task_profile_id,binding_id,binding_revision,capability_id,result,quality_score_millis,reliability_score_millis,latency_score_millis,overall_score_millis,critical_failure_millis,benchmark_version,artifact_id,observed_at_unix_millis) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14) ON CONFLICT DO NOTHING`, value.ID, value.ProfileID, value.BindingID, value.BindingRevision, value.CapabilityID, value.Result, value.QualityScoreMillis, value.ReliabilityScoreMillis, value.LatencyScoreMillis, value.OverallScoreMillis, value.CriticalFailureMillis, value.BenchmarkVersion, value.ArtifactID, value.ObservedAtUnixMillis)
|
||||
return err
|
||||
}
|
||||
|
||||
func (r *CapabilityRepository) SaveModelQualificationObservation(ctx context.Context, value capability.ModelQualificationObservation) error {
|
||||
_, err := r.pool.Exec(ctx, `INSERT INTO capability.cognitive_model_qualification_observations (observation_id,provider_instance_id,provider_model_id,status,finish_reason,latency_millis,output_observed,quality_assessed,category,observed_at_unix_millis) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10) ON CONFLICT DO NOTHING`, value.ID, value.ProviderInstanceID, value.ProviderModelID, value.Status, value.FinishReason, value.LatencyMillis, value.OutputObserved, value.QualityAssessed, value.Category, value.ObservedAtUnixMillis)
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -298,12 +298,21 @@ func fallbackEligible(reason cognitive.FinishReason) bool {
|
||||
}
|
||||
|
||||
func (g cognitiveGateway) provider(selected capability.ResolvedCognitiveBinding) advisoryProvider {
|
||||
var byStableID advisoryProvider
|
||||
for _, registration := range g.providers[selected.Provider.Kind] {
|
||||
if registration.EndpointRef == selected.Provider.EndpointRef && registration.CredentialRef == selected.Provider.CredentialRef {
|
||||
return registration.Provider
|
||||
}
|
||||
// Capability revisions can legitimately retain an older credential
|
||||
// reference while the protected runtime has rotated or reloaded the
|
||||
// current credential. ProviderInstanceID is the stable owner identity;
|
||||
// use it after exact metadata matching so stale non-secret metadata does
|
||||
// not force an unnecessary builtin fallback.
|
||||
if byStableID == nil && registration.ProviderInstanceID == selected.Provider.ID && registration.Kind == selected.Provider.Kind {
|
||||
byStableID = registration.Provider
|
||||
}
|
||||
}
|
||||
return nil
|
||||
return byStableID
|
||||
}
|
||||
|
||||
func writeGatewayResponse(w http.ResponseWriter, response cognitive.GatewayAdvisoryResponse) {
|
||||
|
||||
@@ -88,6 +88,19 @@ func TestActiveRegistryBindingReachesProviderAdapter(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestProviderUsesStableInstanceWhenCredentialMetadataIsStale(t *testing.T) {
|
||||
provider := &stubAdvisoryProvider{}
|
||||
gateway := cognitiveGateway{providers: map[capability.ProviderKind][]providerRegistration{
|
||||
"provider-kind-1": {{ProviderInstanceID: "provider-1", Kind: "provider-kind-1", EndpointRef: "current-endpoint", CredentialRef: "current-credential", Provider: provider}},
|
||||
}}
|
||||
selected := gatewaySelection()
|
||||
selected.Provider.EndpointRef = "old-endpoint"
|
||||
selected.Provider.CredentialRef = ""
|
||||
if got := gateway.provider(selected); got != provider {
|
||||
t.Fatalf("stable provider identity was not used for stale metadata")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCanaryFlagIsForwardedToCapabilityResolver(t *testing.T) {
|
||||
registry := ®istryStub{err: errors.New("stop after resolve")}
|
||||
gateway := cognitiveGateway{resolver: registry, providers: map[capability.ProviderKind][]providerRegistration{}, now: time.Now, allowCanary: true}
|
||||
|
||||
@@ -169,7 +169,7 @@ func main() {
|
||||
mux.HandleFunc("POST /v1/task-evaluations/execute", gateway.taskEvaluationHandler)
|
||||
discovery := discoveryService{resolver: resolver, providers: discoveries, owner: identity.PrincipalID(envOr("COGNITIVE_OWNER_PRINCIPAL_ID", "domovoy-creator-principal")), now: time.Now}
|
||||
mux.HandleFunc("POST /v1/provider-discovery/{provider}", discovery.handler)
|
||||
mux.Handle("POST /v1/provider-discovery/{provider}/qualify", modelQualificationHandler(discoveries, string(envOr("COGNITIVE_OWNER_PRINCIPAL_ID", "domovoy-creator-principal"))))
|
||||
mux.Handle("POST /v1/provider-discovery/{provider}/qualify", modelQualificationHandler(discoveries, string(envOr("COGNITIVE_OWNER_PRINCIPAL_ID", "domovoy-creator-principal")), httpQualificationObserver{resolver: resolver}))
|
||||
mux.HandleFunc("GET /v1/provider-discovery/status", discovery.statusHandler)
|
||||
refreshProviderDiscoveries(resolver, discovery)
|
||||
log.Printf("cognitive-worker listening on %s", addr)
|
||||
|
||||
@@ -14,8 +14,22 @@ import (
|
||||
"git.cin.su/m/Assistant/contracts/capability"
|
||||
"git.cin.su/m/Assistant/contracts/cognitive"
|
||||
"git.cin.su/m/Assistant/contracts/identity"
|
||||
"git.cin.su/m/Assistant/platform/ids"
|
||||
)
|
||||
|
||||
type qualificationObserver interface {
|
||||
ObserveQualification(context.Context, capability.ModelQualificationObservation) error
|
||||
}
|
||||
|
||||
type httpQualificationObserver struct{ resolver *httpBindingResolver }
|
||||
|
||||
func (o httpQualificationObserver) ObserveQualification(ctx context.Context, value capability.ModelQualificationObservation) error {
|
||||
if o.resolver == nil {
|
||||
return fmt.Errorf("qualification observer resolver is nil")
|
||||
}
|
||||
return o.resolver.post(ctx, "/v1/cognitive-registry/model-qualification-observations", value, nil)
|
||||
}
|
||||
|
||||
type modelQualificationRequest struct {
|
||||
Prompt string `json:"prompt"`
|
||||
MaxOutputBytes int `json:"max_output_bytes"`
|
||||
@@ -39,7 +53,7 @@ type modelQualificationResponse struct {
|
||||
Results []modelQualificationResult `json:"results"`
|
||||
}
|
||||
|
||||
func modelQualificationHandler(providers map[string]providerRegistration, owner string) http.Handler {
|
||||
func modelQualificationHandler(providers map[string]providerRegistration, owner string, observers ...qualificationObserver) http.Handler {
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
name := r.PathValue("provider")
|
||||
registration, ok := providers[name]
|
||||
@@ -76,6 +90,12 @@ func modelQualificationHandler(providers map[string]providerRegistration, owner
|
||||
modelsFound = modelsFound[:16]
|
||||
}
|
||||
results := qualifyModels(r.Context(), registration, owner, request.Prompt, request.MaxOutputBytes, modelsFound)
|
||||
if len(observers) > 0 && observers[0] != nil {
|
||||
if err := persistQualificationResults(r.Context(), observers[0], registration, results); err != nil {
|
||||
http.Error(w, "persist model qualification observation", http.StatusBadGateway)
|
||||
return
|
||||
}
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_ = jsonEncode(w, modelQualificationResponse{Provider: registration.ProviderInstanceID, ObservedAtUnixMillis: time.Now().UnixMilli(), Results: results})
|
||||
})
|
||||
@@ -94,6 +114,7 @@ func qualifyModels(ctx context.Context, registration providerRegistration, owner
|
||||
result := modelQualificationResult{ProviderModelID: model.ProviderModelID, LatencyMillis: time.Since(started).Milliseconds(), QualityAssessed: false, Category: capability.ModelRatingUnrated}
|
||||
if err != nil {
|
||||
result.Status = "UNAVAILABLE"
|
||||
result.FinishReason = cognitive.FinishProviderUnavailable
|
||||
result.Error = err.Error()
|
||||
results[index] = result
|
||||
return
|
||||
@@ -112,6 +133,35 @@ func qualifyModels(ctx context.Context, registration providerRegistration, owner
|
||||
return results
|
||||
}
|
||||
|
||||
func persistQualificationResults(ctx context.Context, observer qualificationObserver, registration providerRegistration, results []modelQualificationResult) error {
|
||||
for _, result := range results {
|
||||
observationID, err := ids.New()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
status := capability.ModelQualificationRespondedWithoutRating
|
||||
switch result.Status {
|
||||
case "REACHABLE":
|
||||
status = capability.ModelQualificationReachable
|
||||
case "UNAVAILABLE":
|
||||
status = capability.ModelQualificationUnavailable
|
||||
}
|
||||
observation := capability.ModelQualificationObservation{
|
||||
ID: capability.ModelQualificationObservationID(observationID), ProviderInstanceID: registration.ProviderInstanceID, ProviderModelID: result.ProviderModelID,
|
||||
Status: status, FinishReason: string(result.FinishReason), LatencyMillis: result.LatencyMillis,
|
||||
OutputObserved: result.OutputObserved, QualityAssessed: false, Category: capability.ModelRatingUnrated,
|
||||
ObservedAtUnixMillis: time.Now().UnixMilli(),
|
||||
}
|
||||
if err := observation.Validate(); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := observer.ObserveQualification(ctx, observation); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func jsonEncode(w http.ResponseWriter, value any) error {
|
||||
return json.NewEncoder(w).Encode(value)
|
||||
}
|
||||
|
||||
@@ -57,3 +57,29 @@ func TestModelQualificationRejectsUnboundedOutput(t *testing.T) {
|
||||
t.Fatalf("unbounded qualification status=%d", response.Code)
|
||||
}
|
||||
}
|
||||
|
||||
type qualificationObserverStub struct {
|
||||
observations []capability.ModelQualificationObservation
|
||||
}
|
||||
|
||||
func (s *qualificationObserverStub) ObserveQualification(_ context.Context, value capability.ModelQualificationObservation) error {
|
||||
s.observations = append(s.observations, value)
|
||||
return nil
|
||||
}
|
||||
|
||||
func TestModelQualificationPersistsReachabilityWithoutQuality(t *testing.T) {
|
||||
provider := qualificationProvider{}
|
||||
observer := &qualificationObserverStub{}
|
||||
mux := http.NewServeMux()
|
||||
mux.Handle("POST /v1/provider-discovery/{provider}/qualify", modelQualificationHandler(map[string]providerRegistration{"cpu": {ProviderInstanceID: "cpu", Provider: provider, Discovery: provider}}, "owner", observer))
|
||||
request := httptest.NewRequest(http.MethodPost, "/v1/provider-discovery/cpu/qualify", strings.NewReader(`{"prompt":"ready","max_output_bytes":128,"model_ids":["model-a"]}`))
|
||||
request.Header.Set("Content-Type", "application/json")
|
||||
response := httptest.NewRecorder()
|
||||
mux.ServeHTTP(response, request)
|
||||
if response.Code != http.StatusOK || len(observer.observations) != 1 {
|
||||
t.Fatalf("qualification persistence status=%d observations=%d body=%s", response.Code, len(observer.observations), response.Body.String())
|
||||
}
|
||||
if err := observer.observations[0].Validate(); err != nil || observer.observations[0].Status != capability.ModelQualificationReachable {
|
||||
t.Fatalf("invalid qualification observation: %+v err=%v", observer.observations[0], err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1058,6 +1058,20 @@ func main() {
|
||||
}
|
||||
w.WriteHeader(http.StatusCreated)
|
||||
})
|
||||
mux.HandleFunc("POST /v1/cognitive-registry/model-qualification-observations", func(w http.ResponseWriter, r *http.Request) {
|
||||
var value capability.ModelQualificationObservation
|
||||
decoder := json.NewDecoder(io.LimitReader(r.Body, 16<<10))
|
||||
decoder.DisallowUnknownFields()
|
||||
if err := decoder.Decode(&value); err != nil || value.Validate() != nil {
|
||||
http.Error(w, "invalid model qualification observation", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
if err := capabilityRepository.SaveModelQualificationObservation(r.Context(), value); err != nil {
|
||||
http.Error(w, "save model qualification observation", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
w.WriteHeader(http.StatusCreated)
|
||||
})
|
||||
mux.HandleFunc("POST /v1/creator/cognitive-canary", func(w http.ResponseWriter, r *http.Request) {
|
||||
session, ok := authenticateCreator(w, r)
|
||||
if !ok {
|
||||
|
||||
@@ -172,6 +172,7 @@ type ResolvedCognitiveBinding struct {
|
||||
Evaluation *EvaluatedCapability
|
||||
Performance *PerformanceEvidence
|
||||
TaskEvaluation *TaskEvaluation
|
||||
Qualification *ModelQualificationObservation
|
||||
Rating *ModelRating
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,47 @@
|
||||
package capability
|
||||
|
||||
import (
|
||||
"errors"
|
||||
|
||||
"git.cin.su/m/Assistant/platform/ids"
|
||||
)
|
||||
|
||||
type ModelQualificationObservationID ids.ID
|
||||
type ModelQualificationStatus string
|
||||
|
||||
const (
|
||||
ModelQualificationReachable ModelQualificationStatus = "REACHABLE"
|
||||
ModelQualificationRespondedWithoutRating ModelQualificationStatus = "RESPONDED_WITHOUT_QUALIFICATION"
|
||||
ModelQualificationUnavailable ModelQualificationStatus = "UNAVAILABLE"
|
||||
)
|
||||
|
||||
// ModelQualificationObservation is operational evidence only. It records
|
||||
// reachability and latency, never quality, activation, routing priority or
|
||||
// provider credentials.
|
||||
type ModelQualificationObservation struct {
|
||||
ID ModelQualificationObservationID
|
||||
ProviderInstanceID ProviderInstanceID
|
||||
ProviderModelID ProviderModelID
|
||||
Status ModelQualificationStatus
|
||||
FinishReason string
|
||||
LatencyMillis int64
|
||||
OutputObserved bool
|
||||
QualityAssessed bool
|
||||
Category ModelRatingCategory
|
||||
ObservedAtUnixMillis int64
|
||||
}
|
||||
|
||||
func (o ModelQualificationObservation) Validate() error {
|
||||
if o.ID == "" || o.ProviderInstanceID == "" || o.ProviderModelID == "" || o.FinishReason == "" || o.LatencyMillis < 0 || o.ObservedAtUnixMillis < 1 {
|
||||
return errors.New("model qualification observation identity and timing are incomplete")
|
||||
}
|
||||
switch o.Status {
|
||||
case ModelQualificationReachable, ModelQualificationRespondedWithoutRating, ModelQualificationUnavailable:
|
||||
default:
|
||||
return errors.New("model qualification observation status is unsupported")
|
||||
}
|
||||
if o.Category != ModelRatingUnrated || o.QualityAssessed {
|
||||
return errors.New("model qualification observation cannot assert quality")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
BEGIN;
|
||||
|
||||
SET ROLE capability_role;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS capability.cognitive_model_qualification_observations (
|
||||
observation_id TEXT PRIMARY KEY,
|
||||
provider_instance_id TEXT NOT NULL,
|
||||
provider_model_id TEXT NOT NULL,
|
||||
status TEXT NOT NULL CHECK (status IN ('REACHABLE','RESPONDED_WITHOUT_QUALIFICATION','UNAVAILABLE')),
|
||||
finish_reason TEXT NOT NULL,
|
||||
latency_millis BIGINT NOT NULL CHECK (latency_millis >= 0),
|
||||
output_observed BOOLEAN NOT NULL,
|
||||
quality_assessed BOOLEAN NOT NULL DEFAULT FALSE CHECK (quality_assessed = FALSE),
|
||||
category TEXT NOT NULL CHECK (category = 'UNRATED'),
|
||||
observed_at_unix_millis BIGINT NOT NULL CHECK (observed_at_unix_millis > 0)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS cognitive_model_qualification_lookup
|
||||
ON capability.cognitive_model_qualification_observations(provider_instance_id, provider_model_id, observed_at_unix_millis DESC);
|
||||
|
||||
GRANT SELECT, INSERT ON capability.cognitive_model_qualification_observations TO capability_role;
|
||||
|
||||
RESET ROLE;
|
||||
COMMIT;
|
||||
@@ -1,5 +1,5 @@
|
||||
{
|
||||
"task_file": "dev/tasks/DEV-000237-ui-authenticated-section-smoke.json",
|
||||
"task_id": "DEV-000237",
|
||||
"task_file": "dev/tasks/DEV-000243-qualification-selection-evidence.json",
|
||||
"task_id": "DEV-000243",
|
||||
"task_revision": 1
|
||||
}
|
||||
|
||||
@@ -0,0 +1,23 @@
|
||||
{
|
||||
"task_id": "DEV-000238",
|
||||
"task_revision": 1,
|
||||
"architecture_revision": "ADR-0018",
|
||||
"base_repo_revision": "0609561dfefb6dce302f56d50bb56bd5a222fb64",
|
||||
"risk_tier": "E",
|
||||
"intent": "Record a bounded live proof that the advisory gateway keeps a local lifeboat path when the normal model route cannot be used.",
|
||||
"requirement_ids": ["Agent Execution local recovery", "model fleet fallback", "false completion prevention"],
|
||||
"allowed_change_scope": {"modules": ["evidence"], "paths": ["docs/evidence/local-fallback-gateway-live-2026-08-23-v1.json", "dev/ACTIVE_TASK.json", "dev/tasks/DEV-000238-local-fallback-evidence.json"]},
|
||||
"forbidden_change_scope": {"paths": ["architecture/**", "security/**", "database/**", "deploy/**", "LLM-local/**", "modules/work/secrets/**", "contracts/**"]},
|
||||
"allowed_architecture_delta": {"new_cross_module_dependencies": 0, "new_canonical_owners": 0, "new_privileges": 0, "new_third_party_dependencies": 0, "new_database_owner_roles": 0},
|
||||
"contracts": {"consume": ["creator advisory gateway", "durable cognitive usage observation"], "may_change": []},
|
||||
"semantic_owner": "Evidence only",
|
||||
"canonical_state": ["no runtime state mutation; read-only evidence capture"],
|
||||
"state_transitions": ["normal route unavailable or not eligible -> builtin local lifeboat response"],
|
||||
"side_effects_and_privileges": ["bounded local advisory request; no external egress and no secret exposure"],
|
||||
"persistence_and_migrations": ["existing cognitive usage observation only"],
|
||||
"failure_retry_restart": ["live request is bounded and idempotent in effect"],
|
||||
"assumptions": ["the production database observation is the authoritative route attribution"],
|
||||
"acceptance_scenarios": ["local response is COMPLETED", "usage evidence identifies builtin-local-lifeboat-v1", "no claim of external bridge or multi-model failover"],
|
||||
"required_tests": ["go test ./cmd/core-runtime/...", "go vet ./cmd/core-runtime/...", "go run ./cmd/devctl preflight -active dev/ACTIVE_TASK.json", "go run ./cmd/devctl check-diff -active dev/ACTIVE_TASK.json", "go run ./cmd/devctl check-architecture", "git diff --check", "independent", "CI", "production health smoke"],
|
||||
"terminal_gap_states": ["BLOCKED_BY_MISSING_CAPABILITY", "BLOCKED_BY_SPEC_GAP"]
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
{
|
||||
"task_id": "DEV-000239",
|
||||
"task_revision": 1,
|
||||
"architecture_revision": "ADR-0018",
|
||||
"base_repo_revision": "6061eb5c13e46cec1e60fc104b022394f0d94dee",
|
||||
"risk_tier": "E",
|
||||
"intent": "Resolve a registered provider by its stable instance identity when a persisted Capability revision contains stale non-secret endpoint credential metadata, while preserving exact metadata matching first.",
|
||||
"requirement_ids": ["model fleet/failover", "credential rotation compatibility", "local fallback attribution"],
|
||||
"allowed_change_scope": {"modules": ["cognitive-worker"], "paths": ["cmd/cognitive-worker/gateway.go", "cmd/cognitive-worker/gateway_registry_test.go", "dev/ACTIVE_TASK.json", "dev/tasks/DEV-000239-stable-provider-resolution.json"]},
|
||||
"forbidden_change_scope": {"paths": ["architecture/**", "security/**", "database/**", "deploy/**", "LLM-local/**", "modules/work/secrets/**", "contracts/**"]},
|
||||
"allowed_architecture_delta": {"new_cross_module_dependencies": 0, "new_canonical_owners": 0, "new_privileges": 0, "new_third_party_dependencies": 0, "new_database_owner_roles": 0},
|
||||
"contracts": {"consume": ["Capability ResolvedCognitiveBinding", "protected provider registration"], "may_change": ["internal provider lookup fallback order"]},
|
||||
"semantic_owner": "Cognitive worker provider adapter boundary",
|
||||
"canonical_state": ["no new durable state"],
|
||||
"state_transitions": ["exact provider metadata match -> stable provider identity match -> no provider"],
|
||||
"side_effects_and_privileges": ["provider lookup only; secret values remain in protected runtime"],
|
||||
"persistence_and_migrations": ["none"],
|
||||
"failure_retry_restart": ["missing provider remains explicit and uses existing builtin fallback"],
|
||||
"assumptions": ["ProviderInstanceID is the stable Capability-owned provider identity"],
|
||||
"acceptance_scenarios": ["stale credential metadata still reaches current registered adapter", "exact metadata match retains priority", "no provider remains nil"],
|
||||
"required_tests": ["go test ./cmd/cognitive-worker/...", "go vet ./cmd/cognitive-worker/...", "go run ./cmd/devctl preflight -active dev/ACTIVE_TASK.json", "go run ./cmd/devctl check-diff -active dev/ACTIVE_TASK.json", "go run ./cmd/devctl check-architecture", "git diff --check", "independent", "CI", "production health smoke"],
|
||||
"terminal_gap_states": ["BLOCKED_BY_MISSING_CAPABILITY", "BLOCKED_BY_SPEC_GAP"]
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
{
|
||||
"task_id": "DEV-000240",
|
||||
"task_revision": 1,
|
||||
"architecture_revision": "ADR-0018",
|
||||
"base_repo_revision": "383d4c6e4fd53bb0c34ed358e0d1be6a416da6cc",
|
||||
"risk_tier": "E",
|
||||
"intent": "Record live durable evidence of CPU model failure followed by successful local Lifeboat failover after stable provider resolution was deployed.",
|
||||
"requirement_ids": ["model fleet/failover", "durable usage evidence", "false completion prevention"],
|
||||
"allowed_change_scope": {"modules": ["evidence"], "paths": ["docs/evidence/cpu-lifeboat-failover-2026-08-23-v1.json", "dev/ACTIVE_TASK.json", "dev/tasks/DEV-000240-cpu-lifeboat-failover-evidence.json"]},
|
||||
"forbidden_change_scope": {"paths": ["architecture/**", "security/**", "database/**", "deploy/**", "LLM-local/**", "modules/work/secrets/**", "contracts/**"]},
|
||||
"allowed_architecture_delta": {"new_cross_module_dependencies": 0, "new_canonical_owners": 0, "new_privileges": 0, "new_third_party_dependencies": 0, "new_database_owner_roles": 0},
|
||||
"contracts": {"consume": ["cognitive advisory gateway", "cognitive usage observations"], "may_change": []},
|
||||
"semantic_owner": "Evidence only",
|
||||
"canonical_state": ["no runtime state mutation; evidence captures existing durable observations"],
|
||||
"state_transitions": ["CPU MODEL_UNAVAILABLE -> builtin local Lifeboat COMPLETED"],
|
||||
"side_effects_and_privileges": ["bounded LOCAL_ONLY advisory request; no external egress"],
|
||||
"persistence_and_migrations": ["existing usage observation persistence only"],
|
||||
"failure_retry_restart": ["the fallback is selected within the bounded gateway request"],
|
||||
"assumptions": ["provider_instance_id and result_class in capability usage observations are authoritative route attribution"],
|
||||
"acceptance_scenarios": ["primary failure is durably observed", "fallback completion is durably observed", "HTTP gateway returns COMPLETED", "public health remains OK"],
|
||||
"required_tests": ["go test ./cmd/cognitive-worker/...", "go vet ./cmd/cognitive-worker/...", "go run ./cmd/devctl preflight -active dev/ACTIVE_TASK.json", "go run ./cmd/devctl check-diff -active dev/ACTIVE_TASK.json", "go run ./cmd/devctl check-architecture", "git diff --check", "independent", "CI", "production health smoke"],
|
||||
"terminal_gap_states": ["BLOCKED_BY_MISSING_CAPABILITY", "BLOCKED_BY_SPEC_GAP"]
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
{
|
||||
"task_id": "DEV-000241",
|
||||
"task_revision": 1,
|
||||
"architecture_revision": "ADR-0018",
|
||||
"base_repo_revision": "ca0d0020739d8f70e2a3ac04820e32794072ab30",
|
||||
"risk_tier": "E",
|
||||
"intent": "Record bounded medium and GPU qualification observations without treating hardware presence or reachability as quality or promotion evidence.",
|
||||
"requirement_ids": ["model fleet inventory", "provider-neutral qualification", "hardware-neutral routing"],
|
||||
"allowed_change_scope": {"modules": ["evidence"], "paths": ["docs/evidence/fleet-qualification-2026-08-23-v1.json", "dev/ACTIVE_TASK.json", "dev/tasks/DEV-000241-fleet-qualification-evidence.json"]},
|
||||
"forbidden_change_scope": {"paths": ["architecture/**", "security/**", "database/**", "deploy/**", "LLM-local/**", "modules/work/secrets/**", "contracts/**"]},
|
||||
"allowed_architecture_delta": {"new_cross_module_dependencies": 0, "new_canonical_owners": 0, "new_privileges": 0, "new_third_party_dependencies": 0, "new_database_owner_roles": 0},
|
||||
"contracts": {"consume": ["provider discovery status", "bounded qualification response"], "may_change": []},
|
||||
"semantic_owner": "Evidence only",
|
||||
"canonical_state": ["no runtime state mutation"],
|
||||
"state_transitions": ["discovered model -> bounded reachability observation -> UNRATED result"],
|
||||
"side_effects_and_privileges": ["bounded provider invocation; no promotion, activation or routing mutation"],
|
||||
"persistence_and_migrations": ["evidence artifact only"],
|
||||
"failure_retry_restart": ["one model timeout does not cancel or misclassify other tier evidence"],
|
||||
"assumptions": ["provider discovery IDs are the exact IDs returned by the live worker"],
|
||||
"acceptance_scenarios": ["medium reachability records latency", "GPU timeout is explicit", "both results remain UNRATED", "GPU does not receive an implicit priority"],
|
||||
"required_tests": ["go test ./cmd/cognitive-worker/...", "go vet ./cmd/cognitive-worker/...", "go run ./cmd/devctl preflight -active dev/ACTIVE_TASK.json", "go run ./cmd/devctl check-diff -active dev/ACTIVE_TASK.json", "go run ./cmd/devctl check-architecture", "git diff --check", "independent", "CI", "production health smoke"],
|
||||
"terminal_gap_states": ["BLOCKED_BY_MISSING_CAPABILITY", "BLOCKED_BY_SPEC_GAP"]
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
{
|
||||
"task_id": "DEV-000242",
|
||||
"task_revision": 1,
|
||||
"architecture_revision": "ADR-0018",
|
||||
"base_repo_revision": "e4ba1cb5fabf930396b2aa9f95d3b4b205e6319d",
|
||||
"risk_tier": "E",
|
||||
"intent": "Persist provider-neutral model qualification observations through the Capability owner while keeping quality unrated until independent task evaluation.",
|
||||
"requirement_ids": ["durable qualification evidence", "model fleet/failover", "quality separated from reachability"],
|
||||
"allowed_change_scope": {"modules": ["capability", "core-runtime", "cognitive-worker", "database"], "paths": ["adapters/postgres/capability_repository.go", "cmd/cognitive-worker/main.go", "cmd/cognitive-worker/model_qualification.go", "cmd/cognitive-worker/model_qualification_test.go", "cmd/core-runtime/main.go", "contracts/capability/model_qualification.go", "database/migrations/045_capability_model_qualification_observations.sql", "modules/capability/ports/repository.go", "dev/ACTIVE_TASK.json", "dev/tasks/DEV-000242-durable-qualification-observations.json"]},
|
||||
"forbidden_change_scope": {"paths": ["architecture/**", "security/**", "deploy/**", "LLM-local/**", "modules/work/secrets/**", "contracts/work/**"]},
|
||||
"allowed_architecture_delta": {"new_cross_module_dependencies": 0, "new_canonical_owners": 0, "new_privileges": 0, "new_third_party_dependencies": 0, "new_database_owner_roles": 1},
|
||||
"contracts": {"consume": ["bounded qualification result", "Capability repository boundary"], "may_change": ["typed model qualification observation contract", "internal core persistence route"]},
|
||||
"semantic_owner": "Capability owns qualification evidence; cognitive-worker produces observations",
|
||||
"canonical_state": ["cognitive_model_qualification_observations"],
|
||||
"state_transitions": ["probe -> REACHABLE/RESPONDED_WITHOUT_QUALIFICATION/UNAVAILABLE; category remains UNRATED"],
|
||||
"side_effects_and_privileges": ["bounded provider call and capability-owned INSERT; no promotion, activation or secret persistence"],
|
||||
"persistence_and_migrations": ["migration 045 creates capability-owned observation table and index"],
|
||||
"failure_retry_restart": ["observation persistence failure returns bounded gateway error; duplicate IDs are idempotent"],
|
||||
"assumptions": ["independent task evaluation remains the only quality/promotion evidence"],
|
||||
"acceptance_scenarios": ["qualification route persists one observation per probed model", "invalid quality/category is rejected", "prompt/output/error text is not persisted", "existing qualification tests remain valid"],
|
||||
"required_tests": ["go test ./cmd/cognitive-worker/... ./cmd/core-runtime/... ./modules/capability/... ./adapters/postgres/...", "go vet ./cmd/cognitive-worker/... ./cmd/core-runtime/... ./modules/capability/... ./adapters/postgres/...", "go run ./cmd/devctl preflight -active dev/ACTIVE_TASK.json", "go run ./cmd/devctl check-diff -active dev/ACTIVE_TASK.json", "go run ./cmd/devctl check-architecture", "git diff --check", "independent", "CI", "production health smoke"],
|
||||
"terminal_gap_states": ["BLOCKED_BY_MISSING_CAPABILITY", "BLOCKED_BY_SPEC_GAP", "REQUIRES_MIGRATION_DECISION"]
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
{
|
||||
"task_id": "DEV-000243",
|
||||
"task_revision": 1,
|
||||
"architecture_revision": "ADR-0018",
|
||||
"base_repo_revision": "47327cc6d853da9084c96e3bf48502266706ca57",
|
||||
"risk_tier": "E",
|
||||
"intent": "Make Capability selection consume the latest durable qualification observation for fresh unavailability exclusion and transparent latency rating, without granting quality or activation.",
|
||||
"requirement_ids": ["qualification selection loop", "hardware-neutral routing", "quality separated from reachability"],
|
||||
"allowed_change_scope": {"modules": ["capability", "adapters/postgres"], "paths": ["adapters/postgres/capability_repository.go", "contracts/capability/cognitive_registry.go", "modules/capability/domain/cognitive_registry.go", "modules/capability/domain/model_rating_test.go", "dev/ACTIVE_TASK.json", "dev/tasks/DEV-000243-qualification-selection-evidence.json"]},
|
||||
"forbidden_change_scope": {"paths": ["architecture/**", "security/**", "database/**", "deploy/**", "LLM-local/**", "modules/work/secrets/**", "contracts/work/**"]},
|
||||
"allowed_architecture_delta": {"new_cross_module_dependencies": 0, "new_canonical_owners": 0, "new_privileges": 0, "new_third_party_dependencies": 0, "new_database_owner_roles": 0},
|
||||
"contracts": {"consume": ["durable ModelQualificationObservation"], "may_change": ["resolved binding evidence projection"]},
|
||||
"semantic_owner": "Capability selection and rating",
|
||||
"canonical_state": ["qualification observation remains evidence; no activation mutation"],
|
||||
"state_transitions": ["fresh UNAVAILABLE observation -> candidate excluded", "stale UNAVAILABLE observation -> discovery state remains authoritative", "REACHABLE observation -> latency evidence only"],
|
||||
"side_effects_and_privileges": ["read-only projection and deterministic selection"],
|
||||
"persistence_and_migrations": ["uses migration 045 from DEV-000242"],
|
||||
"failure_retry_restart": ["missing observation preserves existing behavior"],
|
||||
"assumptions": ["provider/model observed timestamps define freshness boundary"],
|
||||
"acceptance_scenarios": ["fresh unavailable candidate is not selected", "stale unavailable candidate is not permanently poisoned", "quality remains task-evaluation controlled"],
|
||||
"required_tests": ["go test ./modules/capability/... ./adapters/postgres/...", "go vet ./modules/capability/... ./adapters/postgres/...", "go run ./cmd/devctl preflight -active dev/ACTIVE_TASK.json", "go run ./cmd/devctl check-diff -active dev/ACTIVE_TASK.json", "go run ./cmd/devctl check-architecture", "git diff --check", "independent", "CI", "production health smoke"],
|
||||
"terminal_gap_states": ["BLOCKED_BY_MISSING_CAPABILITY", "BLOCKED_BY_SPEC_GAP"]
|
||||
}
|
||||
@@ -0,0 +1,15 @@
|
||||
{
|
||||
"evidence_id": "cpu-lifeboat-failover-2026-08-23-v1",
|
||||
"observed_at": "2026-08-23",
|
||||
"production_revision": "383d4c6e4fd53bb0c34ed358e0d1be6a416da6cc",
|
||||
"request": {"surface": "internal cognitive-worker advisory gateway", "data_policy": "LOCAL_ONLY", "max_output_bytes": 128, "prompt_class": "bounded readiness response"},
|
||||
"http_result": {"status": 200, "finish_reason": "COMPLETED", "output_observed": true},
|
||||
"durable_route_observations": [
|
||||
{"provider_instance_id": "local-cpu-primary", "result_class": "MODEL_UNAVAILABLE", "failure_type": "MODEL_UNAVAILABLE", "latency_millis": 580},
|
||||
{"provider_instance_id": "builtin-local-lifeboat-v1", "result_class": "COMPLETED", "failure_type": null, "latency_millis": 455}
|
||||
],
|
||||
"public_health": {"url": "https://assistant.cin.su/healthz", "status": 200, "runtime": "creator-web"},
|
||||
"status": "CPU_TO_LIFEBOAT_FAILOVER_LIVE_PASS",
|
||||
"secrets_exposed": false,
|
||||
"remaining": ["medium/GPU validated multi-model failover", "quality-based promotion", "long-running failover and restart chaos"]
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
{
|
||||
"evidence_id": "fleet-qualification-2026-08-23-v1",
|
||||
"observed_at": "2026-08-23",
|
||||
"production_revision": "ca0d0020739d8f70e2a3ac04820e32794072ab30",
|
||||
"discovery": {"cpu": "AVAILABLE", "medium": "AVAILABLE", "gpu": "AVAILABLE", "lifeboat": "AVAILABLE"},
|
||||
"probes": [
|
||||
{"provider": "local-medium-primary", "provider_model_id": "Qwen3.6-27B-UD-Q4_K_XL", "status": "REACHABLE", "finish_reason": "COMPLETED", "latency_millis": 8541, "output_observed": true, "quality_assessed": false, "category": "UNRATED"},
|
||||
{"provider": "local-gpu-opportunistic", "provider_model_id": "Qwen3.8-27B-UD-Q4_K_XL", "status": "RESPONDED_WITHOUT_QUALIFICATION", "finish_reason": "TIMEOUT", "latency_millis": 10006, "output_observed": false, "quality_assessed": false, "category": "UNRATED"}
|
||||
],
|
||||
"status": "FLEET_REACHABILITY_OBSERVED_QUALITY_UNRATED",
|
||||
"interpretation": "The medium provider is reachable for this bounded probe; the selected GPU model timed out. Hardware presence does not grant priority or promotion.",
|
||||
"remaining": ["durable qualification result persistence", "independent task-quality evaluation for medium/GPU", "GPU retry/failover under sustained load"],
|
||||
"secrets_exposed": false
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
{
|
||||
"evidence_id": "local-fallback-gateway-live-2026-08-23-v1",
|
||||
"observed_at": "2026-08-23",
|
||||
"production_revision": "0609561dfefb6dce302f56d50bb56bd5a222fb64",
|
||||
"request": {"surface": "internal cognitive-worker advisory gateway", "data_policy": "LOCAL_ONLY", "max_output_bytes": 128, "prompt_class": "bounded readiness response"},
|
||||
"http_result": {"status": 200, "finish_reason": "COMPLETED", "output_observed": true, "output_content": "готов"},
|
||||
"durable_usage_observation": {"provider_instance_id": "builtin-local-lifeboat-v1", "model_descriptor_id": "builtin-local-model-v1", "result_class": "COMPLETED", "failure_type": null, "latency_millis": 665},
|
||||
"status": "LOCAL_FALLBACK_LIVE_PASS",
|
||||
"proves": ["local lifeboat path can answer when normal route is not eligible", "completion is durably attributed to the local fallback"],
|
||||
"does_not_prove": ["external Agent Execution bridge", "multi-model failover between validated provider bindings", "GPU quality or promotion", "long-running recovery"],
|
||||
"secrets_exposed": false
|
||||
}
|
||||
@@ -148,6 +148,9 @@ func SelectCognitiveBinding(requirement capability.CognitiveRequirement, candida
|
||||
if candidate.Evaluation != nil && candidate.Evaluation.Result == capability.EvaluationUnsupported {
|
||||
continue
|
||||
}
|
||||
if candidate.Qualification != nil && candidate.Qualification.Status == capability.ModelQualificationUnavailable && candidate.Qualification.ObservedAtUnixMillis >= maxObservedAt(candidate.Provider.ObservedAtUnixMillis, candidate.Model.ObservedAtUnixMillis) {
|
||||
continue
|
||||
}
|
||||
if requirement.TaskProfileID != "" && (candidate.TaskEvaluation == nil || candidate.TaskEvaluation.ProfileID != requirement.TaskProfileID || candidate.TaskEvaluation.Result != capability.EvaluationSupported) {
|
||||
continue
|
||||
}
|
||||
@@ -195,10 +198,14 @@ func RateCognitiveBinding(candidate capability.ResolvedCognitiveBinding) capabil
|
||||
latency := int64(500)
|
||||
if candidate.Performance != nil && candidate.Performance.LatencyMicros != nil {
|
||||
latency = latencyScore(*candidate.Performance.LatencyMicros)
|
||||
} else if candidate.Qualification != nil && candidate.Qualification.Status == capability.ModelQualificationReachable {
|
||||
latency = latencyScore(candidate.Qualification.LatencyMillis * 1000)
|
||||
}
|
||||
evidence := int64(700)
|
||||
if candidate.Performance != nil {
|
||||
evidence = 1000
|
||||
} else if candidate.Qualification != nil && candidate.Qualification.Status == capability.ModelQualificationReachable {
|
||||
evidence = 750
|
||||
}
|
||||
overall := (quality*55 + reliability*25 + latency*15 + evidence*5) / 100
|
||||
category := capability.ModelRatingBalanced
|
||||
@@ -210,6 +217,13 @@ func RateCognitiveBinding(candidate capability.ResolvedCognitiveBinding) capabil
|
||||
return capability.ModelRating{Category: category, OverallScoreMillis: overall, QualityScoreMillis: quality, ReliabilityScoreMillis: reliability, LatencyScoreMillis: latency, EvidenceScoreMillis: evidence}
|
||||
}
|
||||
|
||||
func maxObservedAt(left, right int64) int64 {
|
||||
if left > right {
|
||||
return left
|
||||
}
|
||||
return right
|
||||
}
|
||||
|
||||
func clampMillis(value int64) int64 {
|
||||
if value < 0 {
|
||||
return 0
|
||||
|
||||
@@ -38,3 +38,22 @@ func TestRateCognitiveBindingClassifiesSlowEvidenceAsBalanced(t *testing.T) {
|
||||
t.Fatalf("rating=%+v", rating)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSelectionExcludesFreshUnavailableQualificationButNotStaleEvidence(t *testing.T) {
|
||||
quality, critical := int64(900), int64(0)
|
||||
base := capability.ResolvedCognitiveBinding{
|
||||
Binding: capability.CognitiveBinding{ID: "binding-qualification", Revision: 1, CapabilityID: "creator-advisory", OwnerPrincipalID: "owner", ProviderInstanceID: "provider", ProviderRevision: 1, ModelDescriptorID: "model", ModelRevision: 1, DataPolicy: capability.DataPolicyLocalOnly, Qualification: capability.QualificationValidated, Activation: capability.ActivationActive, Health: capability.HealthAvailable, LimitPolicyRef: "limits"},
|
||||
Provider: capability.ProviderInstance{ID: "provider", Revision: 1, Kind: "local", EndpointRef: "endpoint", Enabled: true, Qualification: capability.QualificationValidated, Health: capability.HealthAvailable, ObservedAtUnixMillis: 20},
|
||||
Model: capability.ModelDescriptor{ID: "model", Revision: 1, ProviderInstanceID: "provider", ProviderModelID: "model", MaxContextBytes: 1000, Modalities: []capability.ModelModality{capability.ModalityText}, Capabilities: []capability.ModelCapability{capability.CapabilityAdvisory}, Qualification: capability.QualificationValidated, Health: capability.HealthAvailable, ObservedAtUnixMillis: 20},
|
||||
Evaluation: &capability.EvaluatedCapability{ID: "evaluation", BindingID: "binding-qualification", BindingRevision: 1, CapabilityID: "creator-advisory", Result: capability.EvaluationSupported, QualityScoreMillis: &quality, CriticalFailureMillis: &critical},
|
||||
Qualification: &capability.ModelQualificationObservation{ID: "observation", ProviderInstanceID: "provider", ProviderModelID: "model", Status: capability.ModelQualificationUnavailable, FinishReason: "TIMEOUT", ObservedAtUnixMillis: 20, Category: capability.ModelRatingUnrated},
|
||||
}
|
||||
requirement := capability.CognitiveRequirement{CapabilityID: "creator-advisory", OwnerPrincipalID: "owner", DataPolicy: capability.DataPolicyLocalOnly, RequiredModalities: []capability.ModelModality{capability.ModalityText}, RequiredCapabilities: []capability.ModelCapability{capability.CapabilityAdvisory}, MaxOutputBytes: 10}
|
||||
if _, err := SelectCognitiveBinding(requirement, []capability.ResolvedCognitiveBinding{base}); err == nil {
|
||||
t.Fatal("fresh unavailable qualification should exclude candidate")
|
||||
}
|
||||
base.Qualification.ObservedAtUnixMillis = 19
|
||||
if _, err := SelectCognitiveBinding(requirement, []capability.ResolvedCognitiveBinding{base}); err != nil {
|
||||
t.Fatalf("stale unavailable qualification should not exclude candidate: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -27,4 +27,5 @@ type Repository interface {
|
||||
SaveEvaluatedCapability(context.Context, capability.EvaluatedCapability) error
|
||||
SavePerformanceEvidence(context.Context, capability.PerformanceEvidence) error
|
||||
SaveTaskEvaluation(context.Context, capability.TaskEvaluation) error
|
||||
SaveModelQualificationObservation(context.Context, capability.ModelQualificationObservation) error
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user