Preserve learning evidence integrity #91

Merged
m merged 2 commits from codex/creator-chat-model-status-v1 into main 2026-08-21 23:45:04 +00:00
8 changed files with 144 additions and 14 deletions
+1 -5
View File
@@ -145,7 +145,6 @@ func (r learningRecorder) recordCorrection(ctx context.Context, input correction
OriginalIntentID: epistemics.OriginalIntentRef(input.Intent.ID),
SourceUtteranceID: epistemics.SourceUtteranceRef(input.Utterance.ID),
HelpMethod: epistemics.HelpMethodClarification,
SkillRef: epistemics.SkillRef(reminderResponseSkillID),
ExpectedOutcome: epistemics.OutcomeSuccess,
ActualOutcome: epistemics.OutcomeUserCorrected,
OutcomeDetail: "human correction indicates prior interpretation or target resolution may be wrong",
@@ -154,10 +153,7 @@ func (r learningRecorder) recordCorrection(ctx context.Context, input correction
ConfidenceMillis: 1000,
RecordedAtUnixMillis: input.At.UnixMilli(),
}
saved, ok := r.saveExperienceAndReflection(ctx, experience)
if ok {
r.updateReminderSkill(ctx, saved, capability.UsageFailed, true)
}
r.saveExperienceAndReflection(ctx, experience)
}
func (r learningRecorder) recordFastPathDecline(ctx context.Context, input fastPathDeclineLearningInput) {
+57
View File
@@ -0,0 +1,57 @@
package main
import (
"context"
"testing"
"time"
"git.cin.su/m/Assistant/contracts/capability"
"git.cin.su/m/Assistant/contracts/epistemics"
"git.cin.su/m/Assistant/contracts/interaction"
)
type learningEpistemicsStub struct {
experiences []epistemics.ExperienceRecord
}
func (s *learningEpistemicsStub) SaveSystemExpectation(context.Context, epistemics.SystemExpectation) error {
return nil
}
func (s *learningEpistemicsStub) SaveExperienceRecord(_ context.Context, value epistemics.ExperienceRecord) error {
s.experiences = append(s.experiences, value)
return nil
}
func (s *learningEpistemicsStub) SaveReflectionCandidate(context.Context, epistemics.ReflectionCandidate) error {
return nil
}
type learningCapabilityStub struct {
loadCalls int
saveCalls int
}
func (s *learningCapabilityStub) LoadSkillProfile(context.Context, capability.SkillProfileID) (capability.SkillProfile, error) {
s.loadCalls++
return capability.SkillProfile{}, nil
}
func (s *learningCapabilityStub) SaveSkillProfile(context.Context, capability.SkillProfile) error {
s.saveCalls++
return nil
}
func TestGenericCorrectionDoesNotDegradeReminderSkill(t *testing.T) {
epistemicsStore := &learningEpistemicsStub{}
capabilityStore := &learningCapabilityStub{}
recorder := learningRecorder{epistemics: epistemicsStore, capability: capabilityStore}
recorder.recordCorrection(context.Background(), correctionLearningInput{
Intent: interaction.OriginalHumanIntent{ID: "intent-1"},
Utterance: interaction.HumanUtterance{ID: "utterance-1"},
At: time.UnixMilli(1000),
})
if capabilityStore.loadCalls != 0 || capabilityStore.saveCalls != 0 {
t.Fatalf("generic correction must not mutate reminder skill: load=%d save=%d", capabilityStore.loadCalls, capabilityStore.saveCalls)
}
if len(epistemicsStore.experiences) != 1 || epistemicsStore.experiences[0].SkillRef != "" {
t.Fatalf("generic correction must remain evidence without a reminder skill reference: %#v", epistemicsStore.experiences)
}
}
+2 -2
View File
@@ -1,5 +1,5 @@
{
"task_file": "dev/tasks/DEV-000092-cognitive-model-inventory-fast-path.json",
"task_id": "DEV-000092",
"task_file": "dev/tasks/DEV-000093-learning-evidence-integrity.json",
"task_id": "DEV-000093",
"task_revision": 1
}
@@ -0,0 +1,35 @@
{
"task_id": "DEV-000093",
"task_revision": 1,
"architecture_revision": "DEV-000073-baseline",
"base_repo_revision": "cf66cd472cb791125347922c9e74ded534c9a3a7",
"risk_tier": "E",
"intent": "Repair learning-evidence integrity so reflection candidates preserve normalized source evidence, corrections do not degrade unrelated skills, and unknown or skipped outcomes remain distinct from verified failures.",
"requirement_ids": ["Reflection references valid experience", "Skill evidence stays capability-specific", "Unknown is not failure"],
"allowed_change_scope": {
"modules": ["core-runtime", "epistemics", "capability", "development-governance"],
"paths": [
"cmd/core-runtime/learning.go",
"cmd/core-runtime/learning_test.go",
"modules/epistemics/domain/experience.go",
"modules/epistemics/domain/experience_test.go",
"modules/capability/domain/skill.go",
"modules/capability/domain/skill_test.go",
"dev/ACTIVE_TASK.json",
"dev/tasks/DEV-000093-learning-evidence-integrity.json"
]
},
"forbidden_change_scope": {"paths": ["architecture/**", "security/**", "database/**", "deploy/**"]},
"allowed_architecture_delta": {"new_cross_module_dependencies": 0, "new_canonical_owners": 0, "new_privileges": 0, "new_third_party_dependencies": 0, "new_database_owner_roles": 0},
"contracts": {"consume": ["existing epistemics experience/reflection contract", "existing capability skill contract"], "may_change": []},
"semantic_owner": "epistemics",
"canonical_state": ["epistemics ExperienceRecord and ReflectionCandidate", "capability SkillProfile"],
"state_transitions": ["failure/correction/low-confidence evidence -> reflection candidate requiring cognition", "unknown/skipped skill observation -> retained as non-failure evidence"],
"side_effects_and_privileges": ["No external mutation", "No new privilege", "No model/provider invocation"],
"persistence_and_migrations": ["No schema migration"],
"failure_retry_restart": ["Learning write failure remains non-authoritative and cannot alter external state", "No retry is inferred as a success"],
"assumptions": ["A generic user correction is not evidence against the reminder-resolution skill unless it is explicitly linked to that skill"],
"acceptance_scenarios": ["Reflection accepts an otherwise valid experience without a caller-supplied ID", "Correction outside reminder flow does not mutate reminder skill", "Unknown and skipped outcomes do not populate known-failure evidence"],
"required_tests": ["go test ./cmd/core-runtime", "go test ./modules/epistemics/domain", "go test ./modules/capability/domain", "go test ./...", "go vet ./...", "devctl preflight", "devctl check-diff", "git diff --check"],
"terminal_gap_states": ["BLOCKED_BY_ARCHITECTURE_GAP", "BLOCKED_BY_SPEC_GAP", "BLOCKED_BY_CONFLICT"]
}
+4 -4
View File
@@ -42,13 +42,13 @@ func ObserveSkillOutcome(profile capability.SkillProfile, observation capability
}
next.LastValidatedUnixMillis = observation.ObservedAtUnixMillis
next.ConfidenceMillis = clampConfidence(next.ConfidenceMillis + 25)
case capability.UsageFailed, capability.UsageUnknown:
case capability.UsageFailed:
next.KnownFailureExperienceRefs = appendEvidenceRef(next.KnownFailureExperienceRefs, observation.ExperienceRef)
next.ObservedFailureCount++
next.ConfidenceMillis = clampConfidence(next.ConfidenceMillis - 150)
case capability.UsageSkipped:
next.KnownFailureExperienceRefs = appendEvidenceRef(next.KnownFailureExperienceRefs, observation.ExperienceRef)
next.ConfidenceMillis = clampConfidence(next.ConfidenceMillis - 25)
case capability.UsageUnknown, capability.UsageSkipped:
// UNKNOWN and SKIPPED are evidence of incomplete observation or a safe
// non-application, not evidence that the skill itself failed.
default:
return capability.SkillProfile{}, ErrInvalidSkillProfile
}
+14
View File
@@ -44,6 +44,20 @@ func TestValidatedSkillCanApplyFastPathWhenFreshAndConfident(t *testing.T) {
}
}
func TestUnknownAndSkippedOutcomesDoNotBecomeSkillFailures(t *testing.T) {
profile := skillProfile()
for _, outcome := range []capability.UsageOutcomeClass{capability.UsageUnknown, capability.UsageSkipped} {
next, err := ObserveSkillOutcome(profile, capability.SkillOutcomeObservation{ExperienceRef: capability.SkillEvidenceRef("experience:" + string(outcome)), Outcome: outcome, ObservedAtUnixMillis: profile.RecordedAtUnixMillis + 1})
if err != nil {
t.Fatal(err)
}
if next.ObservedFailureCount != profile.ObservedFailureCount || len(next.KnownFailureExperienceRefs) != len(profile.KnownFailureExperienceRefs) || next.ConfidenceMillis != profile.ConfidenceMillis {
t.Fatalf("outcome %q was incorrectly recorded as failure: %#v", outcome, next)
}
profile = next
}
}
func skillProfile() capability.SkillProfile {
return capability.SkillProfile{
ID: "skill-obvious-reminder-response-v1",
+13 -2
View File
@@ -54,9 +54,11 @@ func NewExperienceRecord(value epistemics.ExperienceRecord) (epistemics.Experien
}
func ReflectionForExperience(value epistemics.ExperienceRecord) (epistemics.ReflectionCandidate, bool, error) {
if _, err := NewExperienceRecord(value); err != nil {
normalized, err := NewExperienceRecord(value)
if err != nil {
return epistemics.ReflectionCandidate{}, false, err
}
value = normalized
trigger, ok := reflectionTrigger(value)
if !ok {
return epistemics.ReflectionCandidate{}, false, nil
@@ -74,7 +76,7 @@ func ReflectionForExperience(value epistemics.ExperienceRecord) (epistemics.Refl
SourceUtteranceID: value.SourceUtteranceID,
Summary: reflectionSummary(trigger, value),
EvidenceRefs: reflectionEvidence(value),
RequiresCognition: trigger == epistemics.ReflectionContradiction || trigger == epistemics.ReflectionRepeatedDegradation,
RequiresCognition: reflectionNeedsCognition(trigger),
RecordedAtUnixMillis: value.RecordedAtUnixMillis,
Revision: 1,
}
@@ -84,6 +86,15 @@ func ReflectionForExperience(value epistemics.ExperienceRecord) (epistemics.Refl
return candidate, true, nil
}
func reflectionNeedsCognition(trigger epistemics.ReflectionTriggerKind) bool {
switch trigger {
case epistemics.ReflectionUserCorrection, epistemics.ReflectionUnexpectedOutcome, epistemics.ReflectionFailure, epistemics.ReflectionLowConfidence, epistemics.ReflectionContradiction, epistemics.ReflectionRepeatedDegradation:
return true
default:
return false
}
}
func ValidateReflectionCandidate(value epistemics.ReflectionCandidate) error {
if value.ID == "" || value.Trigger == "" || value.State == "" || value.SourceExperienceID == "" ||
value.OriginalIntentID == "" || value.SourceUtteranceID == "" || strings.TrimSpace(value.Summary) == "" ||
+18 -1
View File
@@ -50,11 +50,28 @@ func TestUserCorrectionCreatesReflectionCandidate(t *testing.T) {
if err != nil || !ok {
t.Fatalf("expected reflection candidate, ok=%v err=%v", ok, err)
}
if candidate.Trigger != epistemics.ReflectionUserCorrection || candidate.State != epistemics.ReflectionCandidateOpen || len(candidate.EvidenceRefs) == 0 {
if candidate.Trigger != epistemics.ReflectionUserCorrection || candidate.State != epistemics.ReflectionCandidateOpen || !candidate.RequiresCognition || len(candidate.EvidenceRefs) == 0 {
t.Fatalf("unexpected reflection candidate: %#v", candidate)
}
}
func TestReflectionNormalizesExperienceIDBeforeReferencingIt(t *testing.T) {
candidate, ok, err := ReflectionForExperience(epistemics.ExperienceRecord{
SourceKind: epistemics.ExperienceSourceInteraction,
OriginalIntentID: "intent-1",
SourceUtteranceID: "utterance-3",
HelpMethod: epistemics.HelpMethodClarification,
ExpectedOutcome: epistemics.OutcomeSuccess,
ActualOutcome: epistemics.OutcomeFailed,
OutcomeDetail: "deterministic route failed",
ConfidenceMillis: 800,
RecordedAtUnixMillis: 1200,
})
if err != nil || !ok || candidate.SourceExperienceID == "" || !candidate.RequiresCognition {
t.Fatalf("reflection did not preserve normalized source experience: candidate=%#v ok=%v err=%v", candidate, ok, err)
}
}
func TestFastPathDeclineReturnsToThinkingInsteadOfGuessing(t *testing.T) {
experience, err := FastPathDeclineExperience("intent-1", "utterance-1", "skill:obvious-reminder-response-v1", string(epistemics.OutcomeOutOfDistribution), 1200)
if err != nil {