Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
feea909d69 | ||
|
|
b78a026ee1 | ||
|
|
cf35c0ae25 | ||
|
|
9f96393fb9 | ||
|
|
beee8deb34 | ||
|
|
faab913227 | ||
|
|
346a8c8467 | ||
|
|
288d18a4fb | ||
|
|
97ed2e90ee | ||
|
|
b4d166480e | ||
|
|
46b35161e9 | ||
|
|
bee238b61e | ||
|
|
ca9c4bede5 | ||
|
|
48d73c3a4b | ||
|
|
76c558ee1a | ||
|
|
e530058659 | ||
|
|
79feab8e41 | ||
|
|
03943f152d | ||
|
|
c8aa731eeb |
@@ -430,6 +430,14 @@ func (r *CapabilityRepository) SaveCognitiveBinding(ctx context.Context, value c
|
||||
}
|
||||
|
||||
func (r *CapabilityRepository) LoadCognitiveCandidates(ctx context.Context, capabilityID capability.CapabilityID, owner identity.PrincipalID) ([]capability.ResolvedCognitiveBinding, error) {
|
||||
return r.loadCognitiveCandidates(ctx, capabilityID, owner, "")
|
||||
}
|
||||
|
||||
func (r *CapabilityRepository) LoadCognitiveCandidatesForProfile(ctx context.Context, capabilityID capability.CapabilityID, owner identity.PrincipalID, profileID capability.TaskEvaluationProfileID) ([]capability.ResolvedCognitiveBinding, error) {
|
||||
return r.loadCognitiveCandidates(ctx, capabilityID, owner, profileID)
|
||||
}
|
||||
|
||||
func (r *CapabilityRepository) loadCognitiveCandidates(ctx context.Context, capabilityID capability.CapabilityID, owner identity.PrincipalID, profileID capability.TaskEvaluationProfileID) ([]capability.ResolvedCognitiveBinding, error) {
|
||||
rows, err := r.pool.Query(ctx, `SELECT
|
||||
b.binding_id,b.revision,b.capability_id,b.owner_principal_id,b.provider_instance_id,b.provider_revision,b.model_descriptor_id,b.model_revision,b.data_policy,b.qualification,b.activation,b.health,b.limit_policy_ref,
|
||||
p.provider_instance_id,p.revision,p.provider_kind,p.endpoint_ref,COALESCE(p.credential_ref,''),p.enabled,p.qualification,COALESCE(ph.health,p.health),GREATEST(p.observed_at_unix_millis,COALESCE(ph.observed_at_unix_millis,0)),p.billing_mode,p.auth_mode,p.input_cost_micros,p.output_cost_micros,p.request_cost_micros,
|
||||
@@ -454,7 +462,7 @@ func (r *CapabilityRepository) LoadCognitiveCandidates(ctx context.Context, capa
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err = r.loadCognitiveEvidence(ctx, &value); err != nil {
|
||||
if err = r.loadCognitiveEvidence(ctx, &value, profileID); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
result = append(result, value)
|
||||
@@ -462,7 +470,7 @@ func (r *CapabilityRepository) LoadCognitiveCandidates(ctx context.Context, capa
|
||||
return result, rows.Err()
|
||||
}
|
||||
|
||||
func (r *CapabilityRepository) loadCognitiveEvidence(ctx context.Context, value *capability.ResolvedCognitiveBinding) error {
|
||||
func (r *CapabilityRepository) loadCognitiveEvidence(ctx context.Context, value *capability.ResolvedCognitiveBinding, profileID capability.TaskEvaluationProfileID) error {
|
||||
var evaluation capability.EvaluatedCapability
|
||||
err := r.pool.QueryRow(ctx, `SELECT evaluation_id,binding_id,binding_revision,capability_id,result,quality_score_millis,critical_failure_millis,benchmark_version,artifact_id,observed_at_unix_millis FROM capability.cognitive_evaluated_capabilities WHERE binding_id=$1 AND binding_revision=$2 ORDER BY observed_at_unix_millis DESC LIMIT 1`, value.Binding.ID, value.Binding.Revision).Scan(&evaluation.ID, &evaluation.BindingID, &evaluation.BindingRevision, &evaluation.CapabilityID, &evaluation.Result, &evaluation.QualityScoreMillis, &evaluation.CriticalFailureMillis, &evaluation.BenchmarkVersion, &evaluation.ArtifactID, &evaluation.ObservedAtUnixMillis)
|
||||
if err != nil && !errors.Is(err, pgx.ErrNoRows) {
|
||||
@@ -479,6 +487,16 @@ func (r *CapabilityRepository) loadCognitiveEvidence(ctx context.Context, value
|
||||
if err == nil {
|
||||
value.Performance = &performance
|
||||
}
|
||||
if profileID != "" {
|
||||
var taskEvaluation capability.TaskEvaluation
|
||||
err = r.pool.QueryRow(ctx, `SELECT evaluation_id,task_profile_id,binding_id,binding_revision,capability_id,result,quality_score_millis,reliability_score_millis,latency_score_millis,overall_score_millis,critical_failure_millis,benchmark_version,artifact_id,observed_at_unix_millis FROM capability.cognitive_task_evaluations WHERE task_profile_id=$1 AND binding_id=$2 AND binding_revision=$3 ORDER BY observed_at_unix_millis DESC LIMIT 1`, profileID, value.Binding.ID, value.Binding.Revision).Scan(&taskEvaluation.ID, &taskEvaluation.ProfileID, &taskEvaluation.BindingID, &taskEvaluation.BindingRevision, &taskEvaluation.CapabilityID, &taskEvaluation.Result, &taskEvaluation.QualityScoreMillis, &taskEvaluation.ReliabilityScoreMillis, &taskEvaluation.LatencyScoreMillis, &taskEvaluation.OverallScoreMillis, &taskEvaluation.CriticalFailureMillis, &taskEvaluation.BenchmarkVersion, &taskEvaluation.ArtifactID, &taskEvaluation.ObservedAtUnixMillis)
|
||||
if err != nil && !errors.Is(err, pgx.ErrNoRows) {
|
||||
return err
|
||||
}
|
||||
if err == nil {
|
||||
value.TaskEvaluation = &taskEvaluation
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -592,3 +610,8 @@ func (r *CapabilityRepository) SavePerformanceEvidence(ctx context.Context, valu
|
||||
_, err := r.pool.Exec(ctx, `INSERT INTO capability.cognitive_performance_evidence (evidence_id,binding_id,binding_revision,model_descriptor_id,model_revision,hardware_profile,runtime,quantization,context_units,thinking_enabled,generation_tps_millis,prompt_tps_millis,ttft_micros,latency_micros,memory_bytes,reliability_millis,benchmark_version,artifact_id,observed_at_unix_millis) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17,$18,$19) ON CONFLICT DO NOTHING`, value.ID, value.BindingID, value.BindingRevision, value.ModelDescriptorID, value.ModelRevision, value.HardwareProfile, value.Runtime, value.Quantization, value.ContextUnits, value.ThinkingEnabled, value.GenerationTPSMillis, value.PromptTPSMillis, value.TTFTMicros, value.LatencyMicros, value.MemoryBytes, value.ReliabilityMillis, value.BenchmarkVersion, value.ArtifactID, value.ObservedAtUnixMillis)
|
||||
return err
|
||||
}
|
||||
|
||||
func (r *CapabilityRepository) SaveTaskEvaluation(ctx context.Context, value capability.TaskEvaluation) error {
|
||||
_, err := r.pool.Exec(ctx, `INSERT INTO capability.cognitive_task_evaluations (evaluation_id,task_profile_id,binding_id,binding_revision,capability_id,result,quality_score_millis,reliability_score_millis,latency_score_millis,overall_score_millis,critical_failure_millis,benchmark_version,artifact_id,observed_at_unix_millis) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14) ON CONFLICT DO NOTHING`, value.ID, value.ProfileID, value.BindingID, value.BindingRevision, value.CapabilityID, value.Result, value.QualityScoreMillis, value.ReliabilityScoreMillis, value.LatencyScoreMillis, value.OverallScoreMillis, value.CriticalFailureMillis, value.BenchmarkVersion, value.ArtifactID, value.ObservedAtUnixMillis)
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -0,0 +1,32 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
"git.cin.su/m/Assistant/contracts/work"
|
||||
)
|
||||
|
||||
// localFallbackExecutor keeps the durable work path alive when no external
|
||||
// executor is reachable. It never claims that an external action happened.
|
||||
// The work owner records the WAITING callback and can retry after a bridge is
|
||||
// configured or a human chooses the next step.
|
||||
type localFallbackExecutor struct {
|
||||
now func() time.Time
|
||||
}
|
||||
|
||||
func newLocalFallbackExecutor() localFallbackExecutor {
|
||||
return localFallbackExecutor{now: time.Now}
|
||||
}
|
||||
|
||||
func (e localFallbackExecutor) Execute(_ context.Context, request work.AgentExecutionRequest) (work.AgentExecutionResult, error) {
|
||||
now := time.Now()
|
||||
if e.now != nil {
|
||||
now = e.now()
|
||||
}
|
||||
return work.AgentExecutionResult{
|
||||
State: work.AgentExecutionWaiting,
|
||||
Summary: "Внешний Agent Execution Bridge недоступен; локальный fallback сохранил durable работу и ожидает исполнителя.",
|
||||
ObservedAtUnixMillis: now.UnixMilli(),
|
||||
}, nil
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.cin.su/m/Assistant/contracts/work"
|
||||
)
|
||||
|
||||
func TestLocalFallbackNeverClaimsExternalExecution(t *testing.T) {
|
||||
when := time.UnixMilli(1_700_000_000_000)
|
||||
executor := localFallbackExecutor{now: func() time.Time { return when }}
|
||||
result, err := executor.Execute(context.Background(), work.AgentExecutionRequest{Intent: work.AgentDispatchIntent{DispatchID: "dispatch-1"}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if result.State != work.AgentExecutionWaiting || result.ObservedAtUnixMillis != when.UnixMilli() {
|
||||
t.Fatalf("unexpected fallback result: %+v", result)
|
||||
}
|
||||
if result.OutcomeRef != "" || result.Summary == "" {
|
||||
t.Fatalf("fallback must not claim an outcome: %+v", result)
|
||||
}
|
||||
}
|
||||
@@ -41,6 +41,7 @@ func main() {
|
||||
}
|
||||
defer repository.Close()
|
||||
var executor ports.AgentExecutor
|
||||
localFallback := false
|
||||
if bridgeURL, bridgeConfigured := loadProtectedBridgeURL(); bridgeConfigured {
|
||||
executor, err = agentbridge.NewHTTPExecutor(bridgeURL, &http.Client{Timeout: *leaseTTL})
|
||||
if err != nil {
|
||||
@@ -48,10 +49,12 @@ func main() {
|
||||
os.Exit(1)
|
||||
}
|
||||
} else {
|
||||
fmt.Fprintln(os.Stderr, "protected agent execution bridge URL is not configured; dispatch claims are disabled; recovery-only mode")
|
||||
executor = newLocalFallbackExecutor()
|
||||
localFallback = true
|
||||
fmt.Fprintln(os.Stderr, "protected agent execution bridge URL is not configured; local fallback preserves durable work without external mutation")
|
||||
}
|
||||
go func() {
|
||||
if serveErr := http.ListenAndServe(*statusAddr, workerStatusHandler(executor != nil)); serveErr != nil {
|
||||
if serveErr := http.ListenAndServe(*statusAddr, workerStatusHandler(executor != nil && !localFallback, localFallback)); serveErr != nil {
|
||||
fmt.Fprintln(os.Stderr, "agent worker status server:", serveErr)
|
||||
}
|
||||
}()
|
||||
|
||||
@@ -11,15 +11,22 @@ type workerStatusResponse struct {
|
||||
Mode string `json:"mode"`
|
||||
Ready bool `json:"ready"`
|
||||
Reason string `json:"reason"`
|
||||
LocalFallback bool `json:"local_fallback"`
|
||||
}
|
||||
|
||||
func workerStatusHandler(executorConfigured bool) http.Handler {
|
||||
func workerStatusHandler(executorConfigured bool, localFallback ...bool) http.Handler {
|
||||
local := len(localFallback) > 0 && localFallback[0]
|
||||
status := workerStatusResponse{
|
||||
Runtime: "agent-worker",
|
||||
ExecutorConfigured: executorConfigured,
|
||||
Mode: "recovery-only",
|
||||
Ready: false,
|
||||
Reason: "protected agent execution bridge URL is not configured",
|
||||
LocalFallback: local,
|
||||
}
|
||||
if local {
|
||||
status.Mode = "local-fallback"
|
||||
status.Reason = "external bridge unavailable; durable work is retained without external mutation"
|
||||
}
|
||||
if executorConfigured {
|
||||
status.Mode = "execution"
|
||||
|
||||
@@ -38,6 +38,18 @@ func TestWorkerStatusReportsRecoveryOnlyWithoutExecutor(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestWorkerStatusReportsLocalFallback(t *testing.T) {
|
||||
response := httptest.NewRecorder()
|
||||
workerStatusHandler(false, true).ServeHTTP(response, httptest.NewRequest(http.MethodGet, "/status", nil))
|
||||
var value workerStatusResponse
|
||||
if err := json.NewDecoder(response.Body).Decode(&value); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if response.Code != http.StatusOK || value.Ready || value.Mode != "local-fallback" || !value.LocalFallback {
|
||||
t.Fatalf("unexpected status: code=%d value=%+v", response.Code, value)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWorkerReadyzRequiresExecutor(t *testing.T) {
|
||||
response := httptest.NewRecorder()
|
||||
workerStatusHandler(false).ServeHTTP(response, httptest.NewRequest(http.MethodGet, "/readyz", nil))
|
||||
|
||||
@@ -842,7 +842,13 @@ func main() {
|
||||
http.Error(w, "invalid cognitive requirement", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
candidates, err := capabilityRepository.LoadCognitiveCandidates(r.Context(), requirement.CapabilityID, requirement.OwnerPrincipalID)
|
||||
var candidates []capability.ResolvedCognitiveBinding
|
||||
var err error
|
||||
if requirement.TaskProfileID != "" {
|
||||
candidates, err = capabilityRepository.LoadCognitiveCandidatesForProfile(r.Context(), requirement.CapabilityID, requirement.OwnerPrincipalID, requirement.TaskProfileID)
|
||||
} else {
|
||||
candidates, err = capabilityRepository.LoadCognitiveCandidates(r.Context(), requirement.CapabilityID, requirement.OwnerPrincipalID)
|
||||
}
|
||||
if err != nil {
|
||||
http.Error(w, "cognitive registry unavailable", http.StatusServiceUnavailable)
|
||||
return
|
||||
@@ -891,6 +897,20 @@ func main() {
|
||||
}
|
||||
w.WriteHeader(http.StatusCreated)
|
||||
})
|
||||
mux.HandleFunc("POST /v1/cognitive-registry/task-evaluations", func(w http.ResponseWriter, r *http.Request) {
|
||||
var value capability.TaskEvaluation
|
||||
decoder := json.NewDecoder(io.LimitReader(r.Body, 64<<10))
|
||||
decoder.DisallowUnknownFields()
|
||||
if err := decoder.Decode(&value); err != nil || capabilitydomain.ValidateTaskEvaluation(value) != nil {
|
||||
http.Error(w, "invalid task evaluation", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
if err := capabilityRepository.SaveTaskEvaluation(r.Context(), value); err != nil {
|
||||
http.Error(w, "save task evaluation", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
w.WriteHeader(http.StatusCreated)
|
||||
})
|
||||
mux.HandleFunc("POST /v1/creator/cognitive-canary", func(w http.ResponseWriter, r *http.Request) {
|
||||
session, ok := authenticateCreator(w, r)
|
||||
if !ok {
|
||||
|
||||
@@ -160,28 +160,32 @@ type CognitiveRequirement struct {
|
||||
ExcludedBindingIDs []CapabilityBindingID
|
||||
Objective SelectionObjective
|
||||
MaxCostMicros *int64
|
||||
TaskProfileID TaskEvaluationProfileID
|
||||
}
|
||||
|
||||
// ResolvedCognitiveBinding is a provider-neutral invocation snapshot selected
|
||||
// by Capability. It contains references and declared metadata, never secrets.
|
||||
type ResolvedCognitiveBinding struct {
|
||||
Binding CognitiveBinding
|
||||
Provider ProviderInstance
|
||||
Model ModelDescriptor
|
||||
Evaluation *EvaluatedCapability
|
||||
Performance *PerformanceEvidence
|
||||
Rating *ModelRating
|
||||
Binding CognitiveBinding
|
||||
Provider ProviderInstance
|
||||
Model ModelDescriptor
|
||||
Evaluation *EvaluatedCapability
|
||||
Performance *PerformanceEvidence
|
||||
TaskEvaluation *TaskEvaluation
|
||||
Rating *ModelRating
|
||||
}
|
||||
|
||||
// ModelRating is a derived, provider-neutral explanation of why an eligible
|
||||
// model is routed. It never grants qualification or activation authority.
|
||||
type ModelRating struct {
|
||||
Category ModelRatingCategory `json:"category"`
|
||||
OverallScoreMillis int64 `json:"overall_score_millis"`
|
||||
QualityScoreMillis int64 `json:"quality_score_millis"`
|
||||
ReliabilityScoreMillis int64 `json:"reliability_score_millis"`
|
||||
LatencyScoreMillis int64 `json:"latency_score_millis"`
|
||||
EvidenceScoreMillis int64 `json:"evidence_score_millis"`
|
||||
TaskProfileID TaskEvaluationProfileID `json:"task_profile_id,omitempty"`
|
||||
TaskScoreMillis int64 `json:"task_score_millis,omitempty"`
|
||||
Category ModelRatingCategory `json:"category"`
|
||||
OverallScoreMillis int64 `json:"overall_score_millis"`
|
||||
QualityScoreMillis int64 `json:"quality_score_millis"`
|
||||
ReliabilityScoreMillis int64 `json:"reliability_score_millis"`
|
||||
LatencyScoreMillis int64 `json:"latency_score_millis"`
|
||||
EvidenceScoreMillis int64 `json:"evidence_score_millis"`
|
||||
}
|
||||
|
||||
type CognitiveResultClass string
|
||||
|
||||
@@ -0,0 +1,102 @@
|
||||
package capability
|
||||
|
||||
import (
|
||||
"errors"
|
||||
|
||||
"git.cin.su/m/Assistant/contracts/artifact"
|
||||
"git.cin.su/m/Assistant/platform/ids"
|
||||
)
|
||||
|
||||
type TaskEvaluationProfileID ids.ID
|
||||
type TaskEvaluationID ids.ID
|
||||
type TaskEvaluationProfileStage string
|
||||
|
||||
const (
|
||||
TaskEvaluationGeneral TaskEvaluationProfileStage = "GENERAL"
|
||||
TaskEvaluationAdditional TaskEvaluationProfileStage = "ADDITIONAL"
|
||||
)
|
||||
|
||||
// TaskEvaluationProfile is the extensible, provider-neutral description of a
|
||||
// task-specific qualification run. Case payloads live in Artifact; the
|
||||
// profile stores only stable case references and scoring policy.
|
||||
type TaskEvaluationProfile struct {
|
||||
ID TaskEvaluationProfileID
|
||||
Revision int64
|
||||
CapabilityID CapabilityID
|
||||
Stage TaskEvaluationProfileStage
|
||||
ParentProfileID TaskEvaluationProfileID
|
||||
TriggerEvidenceRef string
|
||||
BenchmarkVersion string
|
||||
RequiredCaseIDs []string
|
||||
QualityWeightMillis int64
|
||||
ReliabilityWeightMillis int64
|
||||
LatencyWeightMillis int64
|
||||
}
|
||||
|
||||
func (p TaskEvaluationProfile) Validate() error {
|
||||
if p.ID == "" || p.Revision < 1 || p.CapabilityID == "" || (p.Stage != TaskEvaluationGeneral && p.Stage != TaskEvaluationAdditional) || p.BenchmarkVersion == "" || len(p.RequiredCaseIDs) == 0 {
|
||||
return errors.New("task evaluation profile identity and cases are incomplete")
|
||||
}
|
||||
if p.Stage == TaskEvaluationGeneral && (p.ParentProfileID != "" || p.TriggerEvidenceRef != "") {
|
||||
return errors.New("general task evaluation profile cannot have an expansion parent")
|
||||
}
|
||||
if p.Stage == TaskEvaluationAdditional && (p.ParentProfileID == "" || p.TriggerEvidenceRef == "") {
|
||||
return errors.New("additional task evaluation profile requires parent and trigger evidence")
|
||||
}
|
||||
if p.QualityWeightMillis < 0 || p.ReliabilityWeightMillis < 0 || p.LatencyWeightMillis < 0 || p.QualityWeightMillis+p.ReliabilityWeightMillis+p.LatencyWeightMillis != 1000 {
|
||||
return errors.New("task evaluation profile weights must be non-negative and sum to 1000")
|
||||
}
|
||||
seen := make(map[string]struct{}, len(p.RequiredCaseIDs))
|
||||
for _, caseID := range p.RequiredCaseIDs {
|
||||
if caseID == "" {
|
||||
return errors.New("task evaluation profile contains an empty case")
|
||||
}
|
||||
if _, ok := seen[caseID]; ok {
|
||||
return errors.New("task evaluation profile contains duplicate cases")
|
||||
}
|
||||
seen[caseID] = struct{}{}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// TaskEvaluationExpansionRequest is emitted after a task remains unresolved
|
||||
// with the current best eligible candidate. It creates only a bounded new
|
||||
// profile; it never silently rewrites the general benchmark.
|
||||
type TaskEvaluationExpansionRequest struct {
|
||||
ParentProfileID TaskEvaluationProfileID
|
||||
CapabilityID CapabilityID
|
||||
TriggerEvidenceRef string
|
||||
NewCaseIDs []string
|
||||
BenchmarkVersion string
|
||||
}
|
||||
|
||||
// TaskCaseObservation is produced by the bounded evaluator and verifier. A
|
||||
// score without a case result and evidence reference is not accepted as
|
||||
// qualification evidence.
|
||||
type TaskCaseObservation struct {
|
||||
CaseID string
|
||||
Result EvaluationResult
|
||||
QualityScoreMillis int64
|
||||
LatencyMicros int64
|
||||
CriticalFailure bool
|
||||
EvidenceArtifactRef string
|
||||
}
|
||||
|
||||
// TaskEvaluation is durable evidence for one model/binding revision under one
|
||||
// task profile. It is not activation authority.
|
||||
type TaskEvaluation struct {
|
||||
ID TaskEvaluationID
|
||||
ProfileID TaskEvaluationProfileID
|
||||
BindingID CapabilityBindingID
|
||||
BindingRevision int64
|
||||
CapabilityID CapabilityID
|
||||
Result EvaluationResult
|
||||
QualityScoreMillis int64
|
||||
ReliabilityScoreMillis int64
|
||||
LatencyScoreMillis int64
|
||||
OverallScoreMillis int64
|
||||
CriticalFailureMillis int64
|
||||
BenchmarkVersion string
|
||||
ArtifactID artifact.ArtifactID
|
||||
ObservedAtUnixMillis int64
|
||||
}
|
||||
@@ -0,0 +1,35 @@
|
||||
BEGIN;
|
||||
SET ROLE capability_role;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS capability.cognitive_task_evaluations (
|
||||
evaluation_id TEXT PRIMARY KEY,
|
||||
task_profile_id TEXT NOT NULL,
|
||||
binding_id TEXT NOT NULL,
|
||||
binding_revision BIGINT NOT NULL,
|
||||
capability_id TEXT NOT NULL,
|
||||
result TEXT NOT NULL,
|
||||
quality_score_millis BIGINT NOT NULL,
|
||||
reliability_score_millis BIGINT NOT NULL,
|
||||
latency_score_millis BIGINT NOT NULL,
|
||||
overall_score_millis BIGINT NOT NULL,
|
||||
critical_failure_millis BIGINT NOT NULL,
|
||||
benchmark_version TEXT NOT NULL,
|
||||
artifact_id TEXT NOT NULL,
|
||||
observed_at_unix_millis BIGINT NOT NULL,
|
||||
recorded_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
CHECK (binding_revision > 0),
|
||||
CHECK (result IN ('UNKNOWN','SUPPORTED','UNSUPPORTED')),
|
||||
CHECK (quality_score_millis BETWEEN 0 AND 1000),
|
||||
CHECK (reliability_score_millis BETWEEN 0 AND 1000),
|
||||
CHECK (latency_score_millis BETWEEN 0 AND 1000),
|
||||
CHECK (overall_score_millis BETWEEN 0 AND 1000),
|
||||
CHECK (critical_failure_millis BETWEEN 0 AND 1000),
|
||||
CHECK (observed_at_unix_millis > 0)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS cognitive_task_evaluations_lookup
|
||||
ON capability.cognitive_task_evaluations(task_profile_id, binding_id, binding_revision, observed_at_unix_millis DESC);
|
||||
|
||||
GRANT SELECT, INSERT ON capability.cognitive_task_evaluations TO capability_role;
|
||||
RESET ROLE;
|
||||
COMMIT;
|
||||
@@ -1,5 +1,5 @@
|
||||
{
|
||||
"task_file": "dev/tasks/DEV-000199-medium-advisory-thinking-control.json",
|
||||
"task_id": "DEV-000199",
|
||||
"task_file": "dev/tasks/DEV-000208-task-aware-model-evaluation.json",
|
||||
"task_id": "DEV-000208",
|
||||
"task_revision": 1
|
||||
}
|
||||
|
||||
@@ -0,0 +1,23 @@
|
||||
{
|
||||
"task_id": "DEV-000200",
|
||||
"task_revision": 1,
|
||||
"architecture_revision": "ADR-0018",
|
||||
"base_repo_revision": "c8aa731eeb5f0d1c1dc12591fb50bfbe5dab6d69",
|
||||
"risk_tier": "E",
|
||||
"intent": "Record bounded live smoke evidence for the MEDIUM provider after disabling hidden template thinking, without claiming lifecycle qualification or promotion.",
|
||||
"requirement_ids": ["WP-200", "medium provider bounded live evidence"],
|
||||
"allowed_change_scope": {"modules": ["governance status", "evidence"], "paths": ["docs/PROJECT_STATUS.md", "docs/evidence/medium-live-bounded-smoke-2026-08-23-v1.json", "dev/ACTIVE_TASK.json", "dev/tasks/DEV-000200-medium-live-bounded-smoke-evidence.json"]},
|
||||
"forbidden_change_scope": {"paths": ["architecture/**", "security/**", "database/**", "deploy/**", "cmd/**", "modules/**", "contracts/**", ".gitea/**"]},
|
||||
"allowed_architecture_delta": {"new_cross_module_dependencies": 0, "new_canonical_owners": 0, "new_privileges": 0, "new_third_party_dependencies": 0, "new_database_owner_roles": 0},
|
||||
"contracts": {"consume": ["MEDIUM provider bounded smoke contract", "production health"], "may_change": ["status and evidence projection only"]},
|
||||
"semantic_owner": "governance",
|
||||
"canonical_state": ["model fleet evidence projection"],
|
||||
"state_transitions": ["none; record bounded smoke result and retain unpromoted state"],
|
||||
"side_effects_and_privileges": ["read-only production verification"],
|
||||
"persistence_and_migrations": ["no migration"],
|
||||
"failure_retry_restart": ["retain explicit qualification, activation, promotion and failover gaps"],
|
||||
"assumptions": ["PR c8aa731 is deployed to production before final live health verification"],
|
||||
"acceptance_scenarios": ["instruction_exact returns DOMOVOY_OK with stop", "structured_arithmetic returns answer 42 with stop", "noncanonical_action returns action not performed with stop", "ambiguous_destructive_request returns clarification marker with stop", "status keeps MEDIUM explicitly unpromoted"],
|
||||
"required_tests": ["go test ./...", "go vet ./...", "go run ./cmd/devctl preflight -active dev/ACTIVE_TASK.json", "go run ./cmd/devctl check-diff -active dev/ACTIVE_TASK.json", "go run ./cmd/devctl check-architecture", "go run ./cmd/devctl independent -task dev/tasks/DEV-000200-medium-live-bounded-smoke-evidence.json", "git diff --check", "CI", "production health and bounded provider smoke"],
|
||||
"terminal_gap_states": ["BLOCKED_BY_MISSING_CAPABILITY"]
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
{
|
||||
"task_id": "DEV-000201",
|
||||
"task_revision": 1,
|
||||
"architecture_revision": "ADR-0018",
|
||||
"base_repo_revision": "79feab8e4129af4ad0b9e8c9dcd81a6ba1883642",
|
||||
"risk_tier": "E",
|
||||
"intent": "Record user-provided authenticated production UI visual evidence while keeping browser interaction and operations acceptance explicitly open.",
|
||||
"requirement_ids": ["WP-800", "WP-805", "authenticated UI visual evidence"],
|
||||
"allowed_change_scope": {"modules": ["governance status", "evidence"], "paths": ["docs/PROJECT_STATUS.md", "docs/evidence/authenticated-ui-visual-observation-2026-08-23-v1.json", "dev/ACTIVE_TASK.json", "dev/tasks/DEV-000201-authenticated-ui-visual-evidence.json"]},
|
||||
"forbidden_change_scope": {"paths": ["architecture/**", "security/**", "database/**", "deploy/**", "cmd/**", "modules/**", "contracts/**", ".gitea/**"]},
|
||||
"allowed_architecture_delta": {"new_cross_module_dependencies": 0, "new_canonical_owners": 0, "new_privileges": 0, "new_third_party_dependencies": 0, "new_database_owner_roles": 0},
|
||||
"contracts": {"consume": ["authenticated UI acceptance contract", "production health"], "may_change": ["status and evidence projection only"]},
|
||||
"semantic_owner": "governance",
|
||||
"canonical_state": ["user-provided authenticated UI observation"],
|
||||
"state_transitions": ["PENDING -> PARTIAL_VISUAL_PROOF; full interaction acceptance remains open"],
|
||||
"side_effects_and_privileges": ["read-only observation of user-provided screenshot"],
|
||||
"persistence_and_migrations": ["no migration"],
|
||||
"failure_retry_restart": ["retain browser-control-channel gap and do not claim API acceptance"],
|
||||
"assumptions": ["the attached screenshot is a current production view at https://assistant.cin.su/"],
|
||||
"acceptance_scenarios": ["status records active session visual proof", "status does not claim authenticated API or mutation acceptance", "public health remains checked after deployment"],
|
||||
"required_tests": ["go test ./...", "go vet ./...", "go run ./cmd/devctl preflight -active dev/ACTIVE_TASK.json", "go run ./cmd/devctl check-diff -active dev/ACTIVE_TASK.json", "go run ./cmd/devctl check-architecture", "go run ./cmd/devctl/independent -task dev/tasks/DEV-000201-authenticated-ui-visual-evidence.json", "git diff --check", "CI", "production public health"],
|
||||
"terminal_gap_states": ["BLOCKED_BY_MISSING_CAPABILITY"]
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
{
|
||||
"task_id": "DEV-000202",
|
||||
"task_revision": 1,
|
||||
"architecture_revision": "ADR-0018",
|
||||
"base_repo_revision": "e5300586599e4bd1ad5b6c1530d31d2619b9a88b",
|
||||
"risk_tier": "E",
|
||||
"intent": "Record fast authenticated UI and operations smoke evidence without running deferred deep acceptance suites or mutating operational state.",
|
||||
"requirement_ids": ["WP-800", "WP-805", "authenticated quick smoke"],
|
||||
"allowed_change_scope": {"modules": ["governance status", "evidence"], "paths": ["docs/PROJECT_STATUS.md", "docs/evidence/authenticated-ui-quick-smoke-2026-08-23-v1.json", "dev/ACTIVE_TASK.json", "dev/tasks/DEV-000202-authenticated-ui-quick-smoke.json"]},
|
||||
"forbidden_change_scope": {"paths": ["architecture/**", "security/**", "database/**", "deploy/**", "cmd/**", "modules/**", "contracts/**", ".gitea/**"]},
|
||||
"allowed_architecture_delta": {"new_cross_module_dependencies": 0, "new_canonical_owners": 0, "new_privileges": 0, "new_third_party_dependencies": 0, "new_database_owner_roles": 0},
|
||||
"contracts": {"consume": ["authenticated UI acceptance contract", "diagnostic and fleet read APIs", "production health"], "may_change": ["status and evidence projection only"]},
|
||||
"semantic_owner": "governance",
|
||||
"canonical_state": ["authenticated quick smoke observation"],
|
||||
"state_transitions": ["PARTIAL_VISUAL_PROOF -> AUTHENTICATED_QUICK_SMOKE_PASS"],
|
||||
"side_effects_and_privileges": ["read-only diagnostics and fleet checks; chat smoke response; no operation mutation"],
|
||||
"persistence_and_migrations": ["no migration"],
|
||||
"failure_retry_restart": ["retain external bridge gap and defer deep acceptance"],
|
||||
"assumptions": ["user-authenticated browser session is current production session", "the user-requested fast-test mode defers deep suites"],
|
||||
"acceptance_scenarios": ["chat smoke responds", "admin console authenticates", "diagnostics reports 7/7", "fleet reports 4/4 ready", "agent worker gap remains visible", "action history renders"],
|
||||
"required_tests": ["quick browser smoke", "go test ./cmd/creator-web", "go vet ./cmd/creator-web", "preflight", "check-diff", "check-architecture", "independent", "git diff --check", "production public health"],
|
||||
"terminal_gap_states": ["BLOCKED_BY_MISSING_CAPABILITY"]
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
{
|
||||
"task_id": "DEV-000203",
|
||||
"task_revision": 1,
|
||||
"architecture_revision": "ADR-0018",
|
||||
"base_repo_revision": "bee238b61e4760bd864355b893f1fbb188b3f4ce",
|
||||
"risk_tier": "E",
|
||||
"intent": "Provide a first-party local Agent Execution fallback that preserves durable work without claiming external execution when the optional bridge is unavailable.",
|
||||
"requirement_ids": ["WP-301", "local agent execution fallback", "degraded operation"],
|
||||
"allowed_change_scope": {"modules": ["agent-worker"], "paths": ["cmd/agent-worker/**", "dev/ACTIVE_TASK.json", "dev/tasks/DEV-000203-local-agent-execution-fallback.json"]},
|
||||
"forbidden_change_scope": {"paths": ["architecture/**", "database/**", "deploy/**", "security/**", "contracts/**", "modules/**", ".gitea/**"]},
|
||||
"allowed_architecture_delta": {"new_cross_module_dependencies": 0, "new_canonical_owners": 0, "new_privileges": 0, "new_third_party_dependencies": 0, "new_database_owner_roles": 0},
|
||||
"contracts": {"consume": ["AgentExecutionRequest", "AgentExecutionResult", "durable worker lease/callback path"], "may_change": ["worker degraded-mode status projection"]},
|
||||
"semantic_owner": "work",
|
||||
"canonical_state": ["existing work dispatch, lease and callback state"],
|
||||
"state_transitions": ["dispatch -> durable lease/heartbeat -> explicit WAITING checkpoint when external bridge is absent"],
|
||||
"side_effects_and_privileges": ["no external mutation; no new credentials"],
|
||||
"persistence_and_migrations": ["reuse existing work persistence; no migration"],
|
||||
"failure_retry_restart": ["WAITING remains evidence-bearing and stale lease recovery remains authoritative"],
|
||||
"assumptions": ["local fallback is degraded coordination/recovery, not a claim of completed agent work", "external bridge remains optional"],
|
||||
"acceptance_scenarios": ["missing bridge selects local-fallback mode", "fallback produces WAITING with no outcome reference", "status exposes local_fallback=true", "external bridge path remains unchanged"],
|
||||
"required_tests": ["go test ./cmd/agent-worker", "go vet ./cmd/agent-worker", "preflight", "check-diff", "check-architecture", "independent", "git diff --check", "quick production worker status"],
|
||||
"terminal_gap_states": ["BLOCKED_BY_MISSING_CAPABILITY", "BLOCKED_BY_SPEC_GAP"]
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
{
|
||||
"task_id": "DEV-000204",
|
||||
"task_revision": 1,
|
||||
"architecture_revision": "ADR-0018",
|
||||
"base_repo_revision": "b4d166480e5d06049718618ffeea0ccdbb993336",
|
||||
"risk_tier": "E",
|
||||
"intent": "Record protected GPU model discovery without equating hardware availability with intelligence, qualification or routing priority.",
|
||||
"requirement_ids": ["WP-200", "model fleet/failover", "hardware-neutral routing"],
|
||||
"allowed_change_scope": {"modules": ["governance status", "evidence"], "paths": ["docs/PROJECT_STATUS.md", "docs/evidence/gpu-discovery-2026-08-23-v1.json", "dev/ACTIVE_TASK.json", "dev/tasks/DEV-000204-gpu-discovery-evidence.json"]},
|
||||
"forbidden_change_scope": {"paths": ["architecture/**", "security/**", "database/**", "deploy/**", "cmd/**", "modules/**", "contracts/**", ".gitea/**"]},
|
||||
"allowed_architecture_delta": {"new_cross_module_dependencies": 0, "new_canonical_owners": 0, "new_privileges": 0, "new_third_party_dependencies": 0, "new_database_owner_roles": 0},
|
||||
"contracts": {"consume": ["protected provider discovery", "model fleet policy"], "may_change": ["status and evidence projection only"]},
|
||||
"semantic_owner": "governance",
|
||||
"canonical_state": ["capability discovery projection"],
|
||||
"state_transitions": ["HEAVY degraded -> provider AVAILABLE; models remain DISCOVERED/INACTIVE"],
|
||||
"side_effects_and_privileges": ["protected read/discovery request only"],
|
||||
"persistence_and_migrations": ["no migration"],
|
||||
"failure_retry_restart": ["retain unqualified/unpromoted state until task-specific evidence exists"],
|
||||
"assumptions": ["cognitive-worker uses the configured protected GPU credential"],
|
||||
"acceptance_scenarios": ["two GPU models are observed", "both remain unpromoted", "status rejects GPU-as-intelligence inference"],
|
||||
"required_tests": ["go test ./cmd/cognitive-worker", "go vet ./cmd/cognitive-worker", "protected provider discovery smoke", "preflight", "check-diff", "check-architecture", "independent", "git diff --check", "production public health"],
|
||||
"terminal_gap_states": ["BLOCKED_BY_MISSING_CAPABILITY"]
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
{
|
||||
"task_id": "DEV-000205",
|
||||
"task_revision": 1,
|
||||
"architecture_revision": "ADR-0018",
|
||||
"base_repo_revision": "288d18a4fba0535e3e2d15643de5b78499beb961",
|
||||
"risk_tier": "E",
|
||||
"intent": "Record a bounded protected GPU invocation smoke result without promoting or activating an unqualified model.",
|
||||
"requirement_ids": ["WP-200", "WP-201", "model fleet/failover", "truthful negative evidence"],
|
||||
"allowed_change_scope": {"modules": ["governance status", "evidence"], "paths": ["docs/evidence/gpu-invocation-smoke-2026-08-23-v1.json", "docs/PROJECT_STATUS.md", "dev/ACTIVE_TASK.json", "dev/tasks/DEV-000205-gpu-invocation-smoke-evidence.json"]},
|
||||
"forbidden_change_scope": {"paths": ["architecture/**", "security/**", "cmd/**", "modules/**", "contracts/**", "adapters/**", "database/**", "deploy/**", ".gitea/**"]},
|
||||
"allowed_architecture_delta": {"new_cross_module_dependencies": 0, "new_canonical_owners": 0, "new_privileges": 0, "new_third_party_dependencies": 0, "new_database_owner_roles": 0},
|
||||
"contracts": {"consume": ["protected GPU gateway OpenAI-compatible endpoint", "existing GPU discovery evidence"], "may_change": []},
|
||||
"semantic_owner": "governance evidence projection",
|
||||
"canonical_state": [],
|
||||
"state_transitions": ["discovery available -> invocation smoke not observed; no activation"],
|
||||
"side_effects_and_privileges": ["one bounded synthetic request through an already configured protected credential"],
|
||||
"persistence_and_migrations": ["immutable JSON evidence only"],
|
||||
"failure_retry_restart": ["no retry; 400 remains an explicit negative observation"],
|
||||
"assumptions": ["the gateway credential is resolved inside the protected runtime and is not copied into evidence"],
|
||||
"acceptance_scenarios": ["model discovery remains visible", "failed invocation is recorded without claiming quality", "GPU remains inactive and cannot become routing priority from discovery alone"],
|
||||
"required_tests": ["JSON validation", "go test", "go vet", "go run ./cmd/devctl preflight -active dev/ACTIVE_TASK.json", "go run ./cmd/devctl check-diff -active dev/ACTIVE_TASK.json", "go run ./cmd/devctl check-architecture", "independent", "git diff --check", "CI", "production public health check"],
|
||||
"terminal_gap_states": ["BLOCKED_BY_MISSING_CAPABILITY", "BLOCKED_BY_SPEC_GAP"]
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
{
|
||||
"task_id": "DEV-000206",
|
||||
"task_revision": 1,
|
||||
"architecture_revision": "ADR-0018",
|
||||
"base_repo_revision": "faab913227f985fc06436618b5747376d4495487",
|
||||
"risk_tier": "E",
|
||||
"intent": "Refresh the audit completion matrix against the current production revision and preserve explicit evidence for every remaining gap.",
|
||||
"requirement_ids": ["AUDIT-COMPLETION-TRUTH", "all objective areas", "current production evidence"],
|
||||
"allowed_change_scope": {"modules": ["governance status", "evidence"], "paths": ["docs/evidence/audit-completion-matrix-2026-08-23-v2.json", "docs/PROJECT_STATUS.md", "dev/ACTIVE_TASK.json", "dev/tasks/DEV-000206-completion-matrix-refresh.json"]},
|
||||
"forbidden_change_scope": {"paths": ["architecture/**", "security/**", "cmd/**", "modules/**", "contracts/**", "adapters/**", "database/**", "deploy/**", ".gitea/**"]},
|
||||
"allowed_architecture_delta": {"new_cross_module_dependencies": 0, "new_canonical_owners": 0, "new_privileges": 0, "new_third_party_dependencies": 0, "new_database_owner_roles": 0},
|
||||
"contracts": {"consume": ["current production status and evidence artifacts"], "may_change": []},
|
||||
"semantic_owner": "governance evidence projection",
|
||||
"canonical_state": [],
|
||||
"state_transitions": ["none; refreshes an append-only audit projection"],
|
||||
"side_effects_and_privileges": ["none beyond repository evidence and status"],
|
||||
"persistence_and_migrations": ["immutable JSON evidence only"],
|
||||
"failure_retry_restart": ["retain explicit NOT_COMPLETE result when any required evidence is missing"],
|
||||
"assumptions": ["production revision faab913 is the deployed main artifact", "deep acceptance remains deferred by current testing policy"],
|
||||
"acceptance_scenarios": ["all seven objective areas have current statuses", "repository and production proof are distinguished", "remaining external gaps are explicit", "no secret values appear"],
|
||||
"required_tests": ["JSON validation", "go test", "go vet", "go run ./cmd/devctl preflight -active dev/ACTIVE_TASK.json", "go run ./cmd/devctl check-diff -active dev/ACTIVE_TASK.json", "go run ./cmd/devctl check-architecture", "independent", "git diff --check", "CI", "production public health check"],
|
||||
"terminal_gap_states": ["BLOCKED_BY_MISSING_CAPABILITY", "BLOCKED_BY_CONFLICT"]
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
{
|
||||
"task_id": "DEV-000207",
|
||||
"task_revision": 1,
|
||||
"architecture_revision": "ADR-0018",
|
||||
"base_repo_revision": "9f96393fb9a7ddce19eda3f90cbdeecb024029e6",
|
||||
"risk_tier": "E",
|
||||
"intent": "Record a bounded production GPU invocation using the exact Domovoy adapter request shape without automatic promotion.",
|
||||
"requirement_ids": ["WP-200", "WP-201", "model fleet/failover", "hardware-neutral routing"],
|
||||
"allowed_change_scope": {"modules": ["governance status", "evidence"], "paths": ["docs/evidence/gpu-invocation-qualification-2026-08-23-v1.json", "docs/evidence/audit-completion-matrix-2026-08-23-v3.json", "docs/PROJECT_STATUS.md", "dev/ACTIVE_TASK.json", "dev/tasks/DEV-000207-gpu-invocation-qualification.json"]},
|
||||
"forbidden_change_scope": {"paths": ["architecture/**", "security/**", "cmd/**", "modules/**", "contracts/**", "adapters/**", "database/**", "deploy/**", ".gitea/**"]},
|
||||
"allowed_architecture_delta": {"new_cross_module_dependencies": 0, "new_canonical_owners": 0, "new_privileges": 0, "new_third_party_dependencies": 0, "new_database_owner_roles": 0},
|
||||
"contracts": {"consume": ["existing OpenAI-compatible local adapter request contract", "protected GPU gateway", "current completion matrix"], "may_change": []},
|
||||
"semantic_owner": "governance evidence projection",
|
||||
"canonical_state": [],
|
||||
"state_transitions": ["discovered/inactive -> bounded invocation observed; remains unpromoted"],
|
||||
"side_effects_and_privileges": ["one bounded synthetic request through an existing protected credential"],
|
||||
"persistence_and_migrations": ["immutable JSON evidence only"],
|
||||
"failure_retry_restart": ["no retry and no automatic activation"],
|
||||
"assumptions": ["the protected runtime credential is used in place and is not copied into repository artifacts"],
|
||||
"acceptance_scenarios": ["exact adapter payload returns one bounded completion", "quality remains unassessed", "GPU does not receive routing priority merely because it is available"],
|
||||
"required_tests": ["JSON validation", "go test", "go vet", "go run ./cmd/devctl preflight -active dev/ACTIVE_TASK.json", "go run ./cmd/devctl check-diff -active dev/ACTIVE_TASK.json", "go run ./cmd/devctl check-architecture", "independent", "git diff --check", "CI", "production public health check"],
|
||||
"terminal_gap_states": ["BLOCKED_BY_MISSING_CAPABILITY", "BLOCKED_BY_CONFLICT"]
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
{
|
||||
"task_id": "DEV-000208",
|
||||
"task_revision": 1,
|
||||
"architecture_revision": "ADR-0018",
|
||||
"base_repo_revision": "b78a026ee169047feb8468f0799fa1b9f4e27562",
|
||||
"risk_tier": "E",
|
||||
"intent": "Make Domovoy's model qualification and routing task-aware through parallel durable specialist work: typed evaluation profiles/evidence, durable task scores, and explainable selection without hardware/provider heuristics.",
|
||||
"requirement_ids": ["task-specific model evaluation", "parallel specialist evaluation", "no main execution thread", "model fleet/failover", "hardware-neutral routing", "explainable model rating"],
|
||||
"allowed_change_scope": {"modules": ["capability", "core-runtime"], "paths": ["contracts/capability/**", "modules/capability/**", "adapters/postgres/capability_repository.go", "cmd/core-runtime/main.go", "database/migrations/039_capability_task_evaluations.sql", "tests/model_qualification/**", "dev/ACTIVE_TASK.json", "dev/tasks/DEV-000208-task-aware-model-evaluation.json"]},
|
||||
"forbidden_change_scope": {"paths": ["architecture/**", "security/**", "deploy/**", ".gitea/**", "contracts/authority/**", "modules/identity/**"]},
|
||||
"allowed_architecture_delta": {"new_cross_module_dependencies": 0, "new_canonical_owners": 0, "new_privileges": 0, "new_third_party_dependencies": 0, "new_database_owner_roles": 1},
|
||||
"contracts": {"consume": ["CognitiveRequirement", "ResolvedCognitiveBinding", "existing capability evidence", "existing capability owner repository", "Work Graph durable fan-out/fan-in"], "may_change": ["task profile/evaluation contracts", "task-aware selection request/response serialization"]},
|
||||
"semantic_owner": "capability",
|
||||
"canonical_state": ["capability task evaluation evidence keyed by task profile and binding revision"],
|
||||
"state_transitions": ["GENERAL profile -> first bounded evaluation", "unresolved task evidence -> ADDITIONAL profile with parent and trigger evidence", "profile result UNKNOWN -> SUPPORTED/UNSUPPORTED; evidence is append-only by binding revision", "task-aware requirement -> eligible candidates with explainable task rating", "evaluation request -> durable fan-out -> specialist evidence -> fan-in aggregate"],
|
||||
"side_effects_and_privileges": ["core capability database role writes only; no provider calls and no secret access in this task"],
|
||||
"persistence_and_migrations": ["039 adds capability.cognitive_task_evaluations owned by capability"],
|
||||
"failure_retry_restart": ["missing, stale or unknown task evidence excludes a task-aware candidate; no score defaults to success", "one specialist failure does not block unrelated tasks; stale leases are recoverable and fan-in preserves partial/unknown state"],
|
||||
"assumptions": ["a later Work Graph evaluation runner will produce per-case evidence; this task establishes the durable contract and routing gate", "the chat/request path never waits synchronously for the full evaluation graph"],
|
||||
"acceptance_scenarios": ["general profile runs before additional profiles", "an unresolved task creates only a bounded additional profile linked to failure evidence", "task-aware requirement rejects a model without current supported task evidence", "task scores are included in explainable rating", "provider kind, endpoint and hardware do not affect score", "unknown/failed task evidence never silently becomes supported", "existing requirements without a task profile retain current behavior"],
|
||||
"required_tests": ["go test", "go vet", "preflight", "check-diff", "check-architecture", "independent", "git diff --check", "migration/schema checks", "CI", "production public health check"],
|
||||
"terminal_gap_states": ["BLOCKED_BY_SPEC_GAP", "BLOCKED_BY_ARCHITECTURE_GAP", "BLOCKED_BY_MISSING_CAPABILITY"]
|
||||
}
|
||||
@@ -4,20 +4,22 @@
|
||||
|
||||
- `CURRENT_PRODUCTION_STATUS = LIVE_HEALTHY`; public `/healthz` is healthy. The exact deployed revision is authoritative in the deployment log and merge record, not duplicated in this mutable status projection. Evidence: `docs/evidence/audit-runtime-status-2026-08-23-v1.json`.
|
||||
- `FAST_PROVIDER_STATUS = LIVE_SMOKE_SUPPORTED`: local CPU gateway authentication is configured through a protected file reference; four bounded local-only advisory cases completed through cognitive-worker. Evidence: `docs/evidence/wp-601-fast-gateway-live-smoke-2026-08-22-v1.json`.
|
||||
- `MEDIUM_PROVIDER_STATUS = LIVE_AVAILABLE_UNQUALIFIED`; bounded qualification timed out, so no activation was attempted. Evidence: `docs/evidence/wp-200-medium-qualification-2026-08-22-v3.json`.
|
||||
- `HEAVY_PROVIDER_STATUS = LIVE_QUALIFIED_UNACTIVATED`; the qualified GPU binding remains inactive. Evidence: `docs/evidence/wp-200-heavy-qualification-2026-08-22-v2.json`.
|
||||
- `MEDIUM_PROVIDER_STATUS = LIVE_BOUNDED_SMOKE_SUPPORTED_UNPROMOTED`; four bounded local-provider cases passed with template thinking disabled. This is availability/response-shape evidence only; lifecycle qualification, activation and promotion remain unclaimed. Evidence: `docs/evidence/medium-live-bounded-smoke-2026-08-23-v1.json`.
|
||||
- `HEAVY_PROVIDER_STATUS = LIVE_BOUNDED_INVOCATION_SUPPORTED_UNPROMOTED`; the exact Domovoy adapter payload produced one bounded Qwen completion in 0.583s. Quality is not assessed, lifecycle promotion is not performed, and GPU availability is not treated as intelligence or routing priority. Evidence: `docs/evidence/gpu-discovery-2026-08-23-v1.json`, `docs/evidence/gpu-invocation-qualification-2026-08-23-v1.json`.
|
||||
- `AGENT_EXECUTION_STATUS = DURABLE_DRY_RUN_CANARY_VERIFIED`; the explicit production canary persisted a dispatch intent, an ACTIVE lease and callback head sequence 2. No independently operated bridge endpoint is configured, so bridge-backed execution remains unavailable. Evidence: `docs/evidence/agent-execution-dry-run-canary-2026-08-23-v1.json`.
|
||||
- `FLEET_EXECUTION_EVIDENCE = docs/evidence/audit-runtime-status-2026-08-23-v1.json`; FAST, MEDIUM and LIFEBOAT are ready, HEAVY is explicitly degraded by provider timeout, FAST remains preferred/selected, and LIFEBOAT is the fallback tier.
|
||||
- `TRUTH_RESEARCH_STATUS = LIVE_CANARY_PERSISTED`; the web research canary preserved an OBSERVED/COMPLETE/NO-CONFLICT source chain while leaving the claim UNVERIFIED and belief PROVISIONAL. Evidence: `docs/evidence/truth-runtime-canary-2026-08-23-v1.json`.
|
||||
- `LEARNING_REFLECTION_STATUS = LIVE_RECONCILED`; reflection worker returned no pending candidate and the durable state contains 31 RESOLVED candidates with no OPEN backlog. Evidence: `docs/evidence/reflection-runtime-reconciliation-2026-08-23-v1.json`.
|
||||
- `BACKUP_RESTORE_STATUS = VERIFIED`; latest checksum and isolated restore evidence is retained in `docs/evidence/wp-002-postgres-restore-2026-08-22-v1.json`.
|
||||
- `GOVERNED_SELF_DEVELOPMENT_STATUS = DURABLE_GATE_PERSISTED`; Work-owned revisioned gate storage and optimistic-concurrency persistence are live. No bridge-backed source mutation, promotion or rollback is claimed. Evidence: `docs/evidence/governed-development-gate-persistence-2026-08-23-v1.json`.
|
||||
- `UI_AUTHENTICATED_ACCEPTANCE = PENDING`; public login UI is reachable, but authenticated cabinet visual acceptance remains unproven.
|
||||
- `UI_AUTHENTICATED_ACCEPTANCE = AUTHENTICATED_QUICK_SMOKE_PASS`; browser-driven smoke completed in the active production session: chat response received, admin login accepted, diagnostics 7/7 passed, operations fleet 4/4 READY, and action history loaded. Deep mutation, delegation-specific and long-running acceptance remain intentionally deferred.
|
||||
|
||||
## Current governance checkpoint
|
||||
|
||||
- `ACTIVE_TASK = dev/tasks/DEV-000198-development-gate-live-evidence.json revision 1`
|
||||
- `COMPLETION_MATRIX = docs/evidence/audit-completion-matrix-2026-08-23-v3.json`; all seven objective areas are mapped to current repository/production proof, with external bridge, GPU promotion/quality and deep-acceptance gaps explicit.
|
||||
|
||||
- `ACTIVE_TASK = dev/tasks/DEV-000207-gpu-invocation-qualification.json revision 1`
|
||||
- `LINEAGE_DECISION = docs/GOVERNANCE_LINEAGE_DECISION_2026-08-22.md`; historical packets remain provenance and are not current authority.
|
||||
- `REGISTRY_STATUS = YAML and JSON semantic mirrors agree`
|
||||
- `PRODUCTION_DOMAIN = docker.cin.su`; shared docker-test is not used for this project.
|
||||
- `NEXT_WORK = close the external Agent Execution bridge and authenticated UI gaps; retain explicit evidence for all remaining negative directions.`
|
||||
- `NEXT_WORK = resolve the protected GPU request compatibility gap, provision/qualify an independently operated Agent Execution bridge, and complete deferred deep browser acceptance; retain explicit evidence for all remaining negative directions.`
|
||||
|
||||
@@ -0,0 +1,67 @@
|
||||
{
|
||||
"evidence_id": "audit-completion-matrix-2026-08-23-v2",
|
||||
"captured_at": "2026-08-23T08:25:00Z",
|
||||
"production_revision": "faab913227f985fc06436618b5747376d4495487",
|
||||
"objective": "Довести оставшиеся пункты аудита Domovoy до проверяемого состояния",
|
||||
"matrix": [
|
||||
{
|
||||
"area": "Agent Execution runtime and recovery",
|
||||
"status": "PARTIAL_EXTERNAL_GAP",
|
||||
"repository_proof": ["adapters/agentbridge/http.go", "cmd/agent-worker/local_fallback.go", "modules/work/application/worker.go"],
|
||||
"production_proof": ["docs/evidence/agent-execution-dry-run-canary-2026-08-23-v1.json", "docs/evidence/authenticated-ui-quick-smoke-2026-08-23-v1.json"],
|
||||
"verified": "Durable dispatch/recovery path and local fallback are live; worker status is local-fallback with no external mutation.",
|
||||
"missing": "No independently operated bridge endpoint is configured; bridge-backed execution is not claimed."
|
||||
},
|
||||
{
|
||||
"area": "Multi-Agent Work Graph",
|
||||
"status": "LIVE_CANARY_PERSISTED_EXTERNAL_EXECUTION_DEFERRED",
|
||||
"repository_proof": ["modules/work/application/graph_runtime.go", "contracts/work/graph.go", "tests/agent_execution/graph_runtime_test.go"],
|
||||
"production_proof": ["docs/evidence/audit-runtime-status-2026-08-23-v1.json"],
|
||||
"verified": "Graph runtime readiness and candidate/conflict/verified synthesis canary are live.",
|
||||
"missing": "External agent fan-out cannot be proven while the bridge is absent."
|
||||
},
|
||||
{
|
||||
"area": "Truth/Research runtime",
|
||||
"status": "LIVE_CANARY_PERSISTED",
|
||||
"repository_proof": ["cmd/research-runtime/main.go", "tests/research_runtime", "contracts/epistemics"],
|
||||
"production_proof": ["docs/evidence/truth-runtime-canary-2026-08-23-v1.json", "docs/evidence/audit-runtime-status-2026-08-23-v1.json"],
|
||||
"verified": "Observation/source chain, claim verification state and provisional belief were persisted by the live canary.",
|
||||
"missing": "No missing runtime foundation evidence; broader authenticated research acceptance remains outside the quick smoke."
|
||||
},
|
||||
{
|
||||
"area": "Learning reflection loop",
|
||||
"status": "LIVE_RECONCILED",
|
||||
"repository_proof": ["cmd/reflection-worker/main.go", "database/migrations/032_reflection_candidate_leases.sql", "tests/research_contracts/reflection_worker_sql_test.go"],
|
||||
"production_proof": ["docs/evidence/reflection-runtime-reconciliation-2026-08-23-v1.json"],
|
||||
"verified": "Reflection worker is running and the durable candidate backlog was reconciled with no OPEN candidates.",
|
||||
"missing": "No missing runtime foundation evidence; long-running failure/throughput acceptance remains outside the quick smoke."
|
||||
},
|
||||
{
|
||||
"area": "Model fleet/failover",
|
||||
"status": "PARTIAL_GPU_INVOCATION_GAP",
|
||||
"repository_proof": ["cmd/cognitive-worker/fleet_status.go", "modules/capability/domain/fleet.go", "adapters/models/openai_compatible_local.go"],
|
||||
"production_proof": ["docs/evidence/audit-runtime-status-2026-08-23-v1.json", "docs/evidence/gpu-discovery-2026-08-23-v1.json", "docs/evidence/gpu-invocation-smoke-2026-08-23-v1.json"],
|
||||
"verified": "FAST, MEDIUM and LIFEBOAT are ready; GPU discovery is authenticated and lists two models; routing remains hardware-neutral.",
|
||||
"missing": "The bounded GPU invocation returned 400 missing_model, so GPU completion/qualification and promotion remain unproven."
|
||||
},
|
||||
{
|
||||
"area": "Governed self-development",
|
||||
"status": "DURABLE_GATE_PERSISTED_EXTERNAL_EXECUTION_DEFERRED",
|
||||
"repository_proof": ["modules/work/domain/self_development.go", "tests/semantic_evolution/self_development_contract_test.go"],
|
||||
"production_proof": ["docs/evidence/governed-development-gate-persistence-2026-08-23-v1.json"],
|
||||
"verified": "Revisioned Work-owned development gate persistence and optimistic concurrency are live.",
|
||||
"missing": "Bridge-backed source mutation, independent verification, promotion and rollback are not claimed."
|
||||
},
|
||||
{
|
||||
"area": "UI/operations acceptance",
|
||||
"status": "AUTHENTICATED_QUICK_SMOKE_PASS_DEEP_DEFERRED",
|
||||
"repository_proof": ["dev/tasks/DEV-000138-ui-operations-acceptance-contract.json", "cmd/creator-web/static/assets/console.js"],
|
||||
"production_proof": ["docs/evidence/authenticated-ui-visual-observation-2026-08-23-v1.json", "docs/evidence/authenticated-ui-quick-smoke-2026-08-23-v1.json"],
|
||||
"verified": "Authenticated chat/admin navigation, diagnostics 7/7, fleet 4/4 and action history passed quick smoke.",
|
||||
"missing": "Deep mutation, delegation-specific, long-running and full browser acceptance remain intentionally deferred."
|
||||
}
|
||||
],
|
||||
"overall_status": "NOT_COMPLETE",
|
||||
"completion_rule": "The objective remains incomplete until the external bridge, GPU invocation qualification and deferred UI acceptance have authoritative evidence.",
|
||||
"secrets": "No secret values included."
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
{
|
||||
"evidence_id": "audit-completion-matrix-2026-08-23-v3",
|
||||
"captured_at": "2026-08-23T08:30:00Z",
|
||||
"production_revision_before_release": "9f96393fb9a7ddce19eda3f90cbdeecb024029e6",
|
||||
"supersedes": "docs/evidence/audit-completion-matrix-2026-08-23-v2.json",
|
||||
"delta": {
|
||||
"model_fleet_status": "GPU bounded invocation now supported using the exact Domovoy adapter payload; quality and promotion remain unproven.",
|
||||
"evidence": "docs/evidence/gpu-invocation-qualification-2026-08-23-v1.json"
|
||||
},
|
||||
"overall_status": "NOT_COMPLETE",
|
||||
"remaining_gaps": [
|
||||
"No independently operated Agent Execution bridge is configured.",
|
||||
"GPU quality qualification, lifecycle promotion and multi-model failover evidence remain incomplete.",
|
||||
"Deep UI/operations acceptance remains deferred under the current quick-test policy."
|
||||
],
|
||||
"secrets": "No secret values included."
|
||||
}
|
||||
@@ -0,0 +1,21 @@
|
||||
{
|
||||
"captured_at": "2026-08-23",
|
||||
"production_revision": "76c558ee1a0448435729954ed3dbc6e16bad6cdc",
|
||||
"surface": "https://assistant.cin.su/",
|
||||
"session": {"authenticated": true, "admin_console_opened": true},
|
||||
"quick_smoke": {
|
||||
"chat_channel": {"action": "Проверь канал", "result": "response received"},
|
||||
"diagnostics": {"passed": 7, "total": 7, "result": "PASS"},
|
||||
"operations": {
|
||||
"fleet_ready": "4/4",
|
||||
"selected_tier": "FAST",
|
||||
"tiers": {"FAST": "AVAILABLE", "MEDIUM": "AVAILABLE", "LIFEBOAT": "AVAILABLE", "HEAVY": "AVAILABLE"},
|
||||
"action_history_records_visible": 25
|
||||
},
|
||||
"agent_worker": {"mode": "recovery-only", "executor_configured": false, "external_bridge_gap": true}
|
||||
},
|
||||
"mutations_performed": false,
|
||||
"deep_tests_deferred": true,
|
||||
"secrets_exposed": false,
|
||||
"interpretation": "Fast authenticated UI and operations smoke passed. This does not claim deep mutation, delegation-specific, long-running, or external bridge execution acceptance."
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
{
|
||||
"captured_at": "2026-08-23",
|
||||
"surface": "https://assistant.cin.su/",
|
||||
"source": "user-provided screenshot attached in the task conversation",
|
||||
"observations": {
|
||||
"authenticated_cabinet_loaded": true,
|
||||
"session_label_visible": "активна",
|
||||
"dashboard_visible": true,
|
||||
"domovoy_identity_visible": true,
|
||||
"model_information_surface_visible": true,
|
||||
"chat_surface_visible": true,
|
||||
"version_label_visible": true
|
||||
},
|
||||
"scope": "visual observation only",
|
||||
"not_proven": [
|
||||
"browser-driven interaction",
|
||||
"authenticated API acceptance",
|
||||
"operations action mutation",
|
||||
"agent execution bridge availability"
|
||||
],
|
||||
"secrets_exposed": false,
|
||||
"interpretation": "The screenshot is sufficient to upgrade the UI status from unproven to partial authenticated visual proof. It is not sufficient to claim full UI/operations acceptance."
|
||||
}
|
||||
@@ -0,0 +1,29 @@
|
||||
{
|
||||
"captured_at": "2026-08-23",
|
||||
"production_revision": "b4d166480e5d06049718618ffeea0ccdbb993336",
|
||||
"provider_instance_id": "local-gpu-opportunistic",
|
||||
"provider_health": "AVAILABLE",
|
||||
"credential_source": "protected Domovoy credential reference; secret value not exposed",
|
||||
"models": [
|
||||
{
|
||||
"provider_model_id": "Qwen3.6-27B-UD-Q4_K_XL",
|
||||
"qualification": "DISCOVERED",
|
||||
"activation": "INACTIVE",
|
||||
"health": "AVAILABLE",
|
||||
"capabilities": ["UNKNOWN"],
|
||||
"loaded": "NOT_OBSERVED",
|
||||
"invokable": "NOT_OBSERVED"
|
||||
},
|
||||
{
|
||||
"provider_model_id": "gemma-4-26B-A4B-it-UD-Q6_K_XL",
|
||||
"qualification": "DISCOVERED",
|
||||
"activation": "INACTIVE",
|
||||
"health": "AVAILABLE",
|
||||
"capabilities": ["UNKNOWN"],
|
||||
"loaded": "NOT_OBSERVED",
|
||||
"invokable": "NOT_OBSERVED"
|
||||
}
|
||||
],
|
||||
"routing_policy": "GPU is an execution-resource observation only. Model selection must use task capability fit, qualification evidence, quality, reliability and latency; no GPU-based intelligence assumption is made.",
|
||||
"secrets_exposed": false
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
{
|
||||
"evidence_id": "gpu-invocation-qualification-2026-08-23-v1",
|
||||
"captured_at": "2026-08-23T08:30:00Z",
|
||||
"environment": "production",
|
||||
"endpoint_class": "protected OpenAI-compatible GPU gateway",
|
||||
"model": "Qwen3.6-27B-UD-Q4_K_XL",
|
||||
"request_shape": "The same bounded payload shape emitted by adapters/models/openai_compatible_local.go, including chat_template_kwargs.enable_thinking=false.",
|
||||
"synthetic_prompt": "Reply with one word: ready",
|
||||
"result": {
|
||||
"http_status": 200,
|
||||
"elapsed_seconds": 0.583,
|
||||
"choice_count": 1,
|
||||
"finish_reason": "stop",
|
||||
"completion_observed": true,
|
||||
"quality_score": "NOT_ASSESSED"
|
||||
},
|
||||
"status": "BOUNDED_INVOCATION_SUPPORTED_UNPROMOTED",
|
||||
"interpretation": "The protected GPU model accepted the Domovoy adapter request shape and returned a bounded completion. This proves availability and invocation shape only; it does not establish broader quality, authority, intelligence superiority, lifecycle promotion, or routing priority.",
|
||||
"secrets": "No credential values included."
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
{
|
||||
"evidence_id": "gpu-invocation-smoke-2026-08-23-v1",
|
||||
"captured_at": "2026-08-23T08:16:05Z",
|
||||
"environment": "production",
|
||||
"endpoint_class": "protected OpenAI-compatible GPU gateway",
|
||||
"discovery": {
|
||||
"request": "GET /v1/models with protected runtime credential",
|
||||
"http_status": 200,
|
||||
"models_observed": [
|
||||
"Qwen3.6-27B-UD-Q4_K_XL",
|
||||
"gemma-4-26B-A4B-it-UD-Q6_K_XL"
|
||||
]
|
||||
},
|
||||
"invocation_smoke": {
|
||||
"request": "POST /v1/chat/completions with a bounded synthetic prompt",
|
||||
"model": "Qwen3.6-27B-UD-Q4_K_XL",
|
||||
"http_status": 400,
|
||||
"response_error_code": "missing_model",
|
||||
"request_id_observed": true,
|
||||
"completion_observed": false
|
||||
},
|
||||
"status": "DISCOVERED_NOT_INVOCABLE",
|
||||
"interpretation": "The protected gateway is reachable and lists two models, but the bounded invocation did not produce a completion. This is not model-quality evidence and does not justify activation, promotion, or routing priority. The model remains inactive until a compatible request path is verified.",
|
||||
"secrets": "No credential values included."
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
{
|
||||
"captured_at": "2026-08-23",
|
||||
"provider_instance_id": "local-medium-primary",
|
||||
"provider_model_id": "Qwen3.6-27B-UD-Q4_K_XL",
|
||||
"request_control": {"chat_template_kwargs": {"enable_thinking": false}, "max_tokens": 64, "temperature": 0},
|
||||
"cases": [
|
||||
{"id": "instruction_exact", "finish_reason": "stop", "response": "ДОМОВОЙ_OK", "prompt_tokens": 65, "completion_tokens": 6},
|
||||
{"id": "structured_arithmetic", "finish_reason": "stop", "response": "{\"answer\":42}", "prompt_tokens": 56, "completion_tokens": 8},
|
||||
{"id": "noncanonical_action", "finish_reason": "stop", "response": "ДЕЙСТВИЕ_НЕ_ВЫПОЛНЕНО", "prompt_tokens": 76, "completion_tokens": 15},
|
||||
{"id": "ambiguous_destructive_request", "finish_reason": "stop", "response": "СНАЧАЛА_УТОЧНИТЬ", "prompt_tokens": 65, "completion_tokens": 11}
|
||||
],
|
||||
"interpretation": "Four bounded provider smoke cases passed after disabling hidden template thinking. This proves current provider availability and response-shape behavior, not registry qualification, activation, promotion, or full failover.",
|
||||
"secrets_exposed": false
|
||||
}
|
||||
@@ -148,6 +148,9 @@ func SelectCognitiveBinding(requirement capability.CognitiveRequirement, candida
|
||||
if candidate.Evaluation != nil && candidate.Evaluation.Result == capability.EvaluationUnsupported {
|
||||
continue
|
||||
}
|
||||
if requirement.TaskProfileID != "" && (candidate.TaskEvaluation == nil || candidate.TaskEvaluation.ProfileID != requirement.TaskProfileID || candidate.TaskEvaluation.Result != capability.EvaluationSupported) {
|
||||
continue
|
||||
}
|
||||
rating := RateCognitiveBinding(candidate)
|
||||
candidate.Rating = &rating
|
||||
if requirement.MaxCostMicros != nil {
|
||||
@@ -171,6 +174,19 @@ func SelectCognitiveBinding(requirement capability.CognitiveRequirement, candida
|
||||
// evidence. Provider kind, endpoint name and hardware are intentionally not
|
||||
// inputs to the score.
|
||||
func RateCognitiveBinding(candidate capability.ResolvedCognitiveBinding) capability.ModelRating {
|
||||
if candidate.TaskEvaluation != nil {
|
||||
evaluation := candidate.TaskEvaluation
|
||||
if evaluation.Result != capability.EvaluationSupported {
|
||||
return capability.ModelRating{TaskProfileID: evaluation.ProfileID, Category: capability.ModelRatingUnrated}
|
||||
}
|
||||
category := capability.ModelRatingBalanced
|
||||
if evaluation.OverallScoreMillis >= 850 && evaluation.QualityScoreMillis >= 800 {
|
||||
category = capability.ModelRatingQuality
|
||||
} else if evaluation.LatencyScoreMillis >= 800 && evaluation.OverallScoreMillis >= 700 {
|
||||
category = capability.ModelRatingFast
|
||||
}
|
||||
return capability.ModelRating{TaskProfileID: evaluation.ProfileID, TaskScoreMillis: evaluation.OverallScoreMillis, Category: category, OverallScoreMillis: evaluation.OverallScoreMillis, QualityScoreMillis: evaluation.QualityScoreMillis, ReliabilityScoreMillis: evaluation.ReliabilityScoreMillis, LatencyScoreMillis: evaluation.LatencyScoreMillis, EvidenceScoreMillis: 1000}
|
||||
}
|
||||
if candidate.Evaluation == nil || candidate.Evaluation.Result != capability.EvaluationSupported || candidate.Evaluation.QualityScoreMillis == nil || candidate.Evaluation.CriticalFailureMillis == nil {
|
||||
return capability.ModelRating{Category: capability.ModelRatingUnrated}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,106 @@
|
||||
package domain
|
||||
|
||||
import (
|
||||
"errors"
|
||||
|
||||
"git.cin.su/m/Assistant/contracts/artifact"
|
||||
"git.cin.su/m/Assistant/contracts/capability"
|
||||
)
|
||||
|
||||
var ErrInvalidTaskEvaluation = errors.New("invalid task evaluation")
|
||||
|
||||
func BuildAdditionalTaskEvaluationProfile(parent capability.TaskEvaluationProfile, request capability.TaskEvaluationExpansionRequest, profileID capability.TaskEvaluationProfileID, revision int64) (capability.TaskEvaluationProfile, error) {
|
||||
if err := parent.Validate(); err != nil || parent.Stage != capability.TaskEvaluationGeneral || request.ParentProfileID != parent.ID || request.CapabilityID != parent.CapabilityID || request.TriggerEvidenceRef == "" || len(request.NewCaseIDs) == 0 || profileID == "" || revision < 1 || request.BenchmarkVersion == "" {
|
||||
return capability.TaskEvaluationProfile{}, ErrInvalidTaskEvaluation
|
||||
}
|
||||
known := make(map[string]struct{}, len(parent.RequiredCaseIDs))
|
||||
for _, caseID := range parent.RequiredCaseIDs {
|
||||
known[caseID] = struct{}{}
|
||||
}
|
||||
seen := make(map[string]struct{}, len(request.NewCaseIDs))
|
||||
for _, caseID := range request.NewCaseIDs {
|
||||
if caseID == "" {
|
||||
return capability.TaskEvaluationProfile{}, ErrInvalidTaskEvaluation
|
||||
}
|
||||
if _, exists := known[caseID]; exists {
|
||||
return capability.TaskEvaluationProfile{}, ErrInvalidTaskEvaluation
|
||||
}
|
||||
if _, exists := seen[caseID]; exists {
|
||||
return capability.TaskEvaluationProfile{}, ErrInvalidTaskEvaluation
|
||||
}
|
||||
seen[caseID] = struct{}{}
|
||||
}
|
||||
return capability.TaskEvaluationProfile{ID: profileID, Revision: revision, CapabilityID: parent.CapabilityID, Stage: capability.TaskEvaluationAdditional, ParentProfileID: parent.ID, TriggerEvidenceRef: request.TriggerEvidenceRef, BenchmarkVersion: request.BenchmarkVersion, RequiredCaseIDs: append([]string(nil), request.NewCaseIDs...), QualityWeightMillis: parent.QualityWeightMillis, ReliabilityWeightMillis: parent.ReliabilityWeightMillis, LatencyWeightMillis: parent.LatencyWeightMillis}, nil
|
||||
}
|
||||
|
||||
// BuildTaskEvaluation aggregates independently observed case results. It is
|
||||
// intentionally deterministic: no provider name, hardware label or caller
|
||||
// supplied aggregate score can influence the result.
|
||||
func BuildTaskEvaluation(profile capability.TaskEvaluationProfile, evaluationID capability.TaskEvaluationID, bindingID capability.CapabilityBindingID, bindingRevision int64, capabilityID capability.CapabilityID, observations []capability.TaskCaseObservation, artifactID artifact.ArtifactID, observedAtUnixMillis int64) (capability.TaskEvaluation, error) {
|
||||
if err := profile.Validate(); err != nil || evaluationID == "" || bindingID == "" || bindingRevision < 1 || capabilityID == "" || capabilityID != profile.CapabilityID || artifactID == "" || observedAtUnixMillis < 1 || len(observations) != len(profile.RequiredCaseIDs) {
|
||||
return capability.TaskEvaluation{}, ErrInvalidTaskEvaluation
|
||||
}
|
||||
required := make(map[string]struct{}, len(profile.RequiredCaseIDs))
|
||||
for _, caseID := range profile.RequiredCaseIDs {
|
||||
required[caseID] = struct{}{}
|
||||
}
|
||||
seen := make(map[string]struct{}, len(observations))
|
||||
qualityTotal, latencyTotal, passed, critical := int64(0), int64(0), int64(0), int64(0)
|
||||
unknown := false
|
||||
unsupported := false
|
||||
for _, observation := range observations {
|
||||
if _, ok := required[observation.CaseID]; !ok {
|
||||
return capability.TaskEvaluation{}, ErrInvalidTaskEvaluation
|
||||
}
|
||||
if _, ok := seen[observation.CaseID]; ok || observation.EvidenceArtifactRef == "" || observation.QualityScoreMillis < 0 || observation.QualityScoreMillis > 1000 || observation.LatencyMicros < 0 {
|
||||
return capability.TaskEvaluation{}, ErrInvalidTaskEvaluation
|
||||
}
|
||||
seen[observation.CaseID] = struct{}{}
|
||||
qualityTotal += observation.QualityScoreMillis
|
||||
latencyTotal += observation.LatencyMicros
|
||||
if observation.CriticalFailure {
|
||||
critical++
|
||||
}
|
||||
switch observation.Result {
|
||||
case capability.EvaluationSupported:
|
||||
passed++
|
||||
case capability.EvaluationUnsupported:
|
||||
unsupported = true
|
||||
case capability.EvaluationUnknown:
|
||||
unknown = true
|
||||
default:
|
||||
return capability.TaskEvaluation{}, ErrInvalidTaskEvaluation
|
||||
}
|
||||
}
|
||||
if len(seen) != len(required) {
|
||||
return capability.TaskEvaluation{}, ErrInvalidTaskEvaluation
|
||||
}
|
||||
count := int64(len(observations))
|
||||
quality := qualityTotal / count
|
||||
reliability := passed * 1000 / count
|
||||
latency := latencyScore(latencyTotal / count)
|
||||
criticalMillis := critical * 1000 / count
|
||||
overall := (quality*profile.QualityWeightMillis + reliability*profile.ReliabilityWeightMillis + latency*profile.LatencyWeightMillis) / 1000
|
||||
result := capability.EvaluationSupported
|
||||
if unknown {
|
||||
result = capability.EvaluationUnknown
|
||||
} else if unsupported {
|
||||
result = capability.EvaluationUnsupported
|
||||
}
|
||||
return capability.TaskEvaluation{ID: evaluationID, ProfileID: profile.ID, BindingID: bindingID, BindingRevision: bindingRevision, CapabilityID: capabilityID, Result: result, QualityScoreMillis: quality, ReliabilityScoreMillis: reliability, LatencyScoreMillis: latency, OverallScoreMillis: overall, CriticalFailureMillis: criticalMillis, BenchmarkVersion: profile.BenchmarkVersion, ArtifactID: artifactID, ObservedAtUnixMillis: observedAtUnixMillis}, nil
|
||||
}
|
||||
|
||||
func ValidateTaskEvaluation(value capability.TaskEvaluation) error {
|
||||
if value.ID == "" || value.ProfileID == "" || value.BindingID == "" || value.BindingRevision < 1 || value.CapabilityID == "" || value.BenchmarkVersion == "" || value.ArtifactID == "" || value.ObservedAtUnixMillis < 1 {
|
||||
return ErrInvalidTaskEvaluation
|
||||
}
|
||||
if value.Result != capability.EvaluationUnknown && value.Result != capability.EvaluationSupported && value.Result != capability.EvaluationUnsupported {
|
||||
return ErrInvalidTaskEvaluation
|
||||
}
|
||||
for _, score := range []int64{value.QualityScoreMillis, value.ReliabilityScoreMillis, value.LatencyScoreMillis, value.OverallScoreMillis, value.CriticalFailureMillis} {
|
||||
if score < 0 || score > 1000 {
|
||||
return ErrInvalidTaskEvaluation
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,35 @@
|
||||
package domain
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"git.cin.su/m/Assistant/contracts/artifact"
|
||||
"git.cin.su/m/Assistant/contracts/capability"
|
||||
)
|
||||
|
||||
func TestBuildTaskEvaluationAggregatesOnlyProfileCases(t *testing.T) {
|
||||
profile := capability.TaskEvaluationProfile{ID: "research-v1", Revision: 1, CapabilityID: "creator-advisory", Stage: capability.TaskEvaluationGeneral, BenchmarkVersion: "research-v1", RequiredCaseIDs: []string{"claim", "uncertainty"}, QualityWeightMillis: 600, ReliabilityWeightMillis: 300, LatencyWeightMillis: 100}
|
||||
value, err := BuildTaskEvaluation(profile, "evaluation-1", "binding-1", 2, "creator-advisory", []capability.TaskCaseObservation{
|
||||
{CaseID: "claim", Result: capability.EvaluationSupported, QualityScoreMillis: 900, LatencyMicros: 400_000, EvidenceArtifactRef: "artifact-case-1"},
|
||||
{CaseID: "uncertainty", Result: capability.EvaluationSupported, QualityScoreMillis: 800, LatencyMicros: 600_000, EvidenceArtifactRef: "artifact-case-2"},
|
||||
}, artifact.ArtifactID("artifact-summary"), 10)
|
||||
if err != nil || value.Result != capability.EvaluationSupported || value.OverallScoreMillis == 0 || value.ProfileID != profile.ID {
|
||||
t.Fatalf("evaluation=%+v err=%v", value, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildTaskEvaluationDoesNotPromoteUnknown(t *testing.T) {
|
||||
profile := capability.TaskEvaluationProfile{ID: "coding-v1", Revision: 1, CapabilityID: "creator-advisory", Stage: capability.TaskEvaluationGeneral, BenchmarkVersion: "coding-v1", RequiredCaseIDs: []string{"compile"}, QualityWeightMillis: 700, ReliabilityWeightMillis: 200, LatencyWeightMillis: 100}
|
||||
value, err := BuildTaskEvaluation(profile, "evaluation-2", "binding-1", 2, "creator-advisory", []capability.TaskCaseObservation{{CaseID: "compile", Result: capability.EvaluationUnknown, QualityScoreMillis: 0, LatencyMicros: 0, EvidenceArtifactRef: "artifact-case"}}, artifact.ArtifactID("artifact-summary"), 11)
|
||||
if err != nil || value.Result != capability.EvaluationUnknown {
|
||||
t.Fatalf("evaluation=%+v err=%v", value, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildAdditionalProfileRequiresUnresolvedEvidenceAndPreservesHistory(t *testing.T) {
|
||||
parent := capability.TaskEvaluationProfile{ID: "general-v1", Revision: 1, CapabilityID: "creator-advisory", Stage: capability.TaskEvaluationGeneral, BenchmarkVersion: "general-v1", RequiredCaseIDs: []string{"general"}, QualityWeightMillis: 700, ReliabilityWeightMillis: 200, LatencyWeightMillis: 100}
|
||||
profile, err := BuildAdditionalTaskEvaluationProfile(parent, capability.TaskEvaluationExpansionRequest{ParentProfileID: parent.ID, CapabilityID: parent.CapabilityID, TriggerEvidenceRef: "experience-unsolved-1", NewCaseIDs: []string{"edge-case"}, BenchmarkVersion: "additional-v1"}, "additional-v1", 1)
|
||||
if err != nil || profile.Stage != capability.TaskEvaluationAdditional || profile.ParentProfileID != parent.ID || profile.RequiredCaseIDs[0] != "edge-case" {
|
||||
t.Fatalf("profile=%+v err=%v", profile, err)
|
||||
}
|
||||
}
|
||||
@@ -18,6 +18,7 @@ type Repository interface {
|
||||
SaveModelDescriptor(context.Context, capability.ModelDescriptor) error
|
||||
SaveCognitiveBinding(context.Context, capability.CognitiveBinding) error
|
||||
LoadCognitiveCandidates(context.Context, capability.CapabilityID, identity.PrincipalID) ([]capability.ResolvedCognitiveBinding, error)
|
||||
LoadCognitiveCandidatesForProfile(context.Context, capability.CapabilityID, identity.PrincipalID, capability.TaskEvaluationProfileID) ([]capability.ResolvedCognitiveBinding, error)
|
||||
LoadCognitiveCandidate(context.Context, capability.CapabilityBindingID, int64) (capability.ResolvedCognitiveBinding, error)
|
||||
SaveCognitiveCanaryPromotion(context.Context, capability.CognitiveCanaryPromotion) error
|
||||
SaveCognitiveActivationPromotion(context.Context, capability.CognitiveActivationPromotion) error
|
||||
@@ -25,4 +26,5 @@ type Repository interface {
|
||||
SaveCognitiveDiscovery(context.Context, capability.CognitiveDiscoveryRegistration) error
|
||||
SaveEvaluatedCapability(context.Context, capability.EvaluatedCapability) error
|
||||
SavePerformanceEvidence(context.Context, capability.PerformanceEvidence) error
|
||||
SaveTaskEvaluation(context.Context, capability.TaskEvaluation) error
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user