Preserve learning evidence integrity #91
@@ -145,7 +145,6 @@ func (r learningRecorder) recordCorrection(ctx context.Context, input correction
|
||||
OriginalIntentID: epistemics.OriginalIntentRef(input.Intent.ID),
|
||||
SourceUtteranceID: epistemics.SourceUtteranceRef(input.Utterance.ID),
|
||||
HelpMethod: epistemics.HelpMethodClarification,
|
||||
SkillRef: epistemics.SkillRef(reminderResponseSkillID),
|
||||
ExpectedOutcome: epistemics.OutcomeSuccess,
|
||||
ActualOutcome: epistemics.OutcomeUserCorrected,
|
||||
OutcomeDetail: "human correction indicates prior interpretation or target resolution may be wrong",
|
||||
@@ -154,10 +153,7 @@ func (r learningRecorder) recordCorrection(ctx context.Context, input correction
|
||||
ConfidenceMillis: 1000,
|
||||
RecordedAtUnixMillis: input.At.UnixMilli(),
|
||||
}
|
||||
saved, ok := r.saveExperienceAndReflection(ctx, experience)
|
||||
if ok {
|
||||
r.updateReminderSkill(ctx, saved, capability.UsageFailed, true)
|
||||
}
|
||||
r.saveExperienceAndReflection(ctx, experience)
|
||||
}
|
||||
|
||||
func (r learningRecorder) recordFastPathDecline(ctx context.Context, input fastPathDeclineLearningInput) {
|
||||
|
||||
@@ -0,0 +1,57 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.cin.su/m/Assistant/contracts/capability"
|
||||
"git.cin.su/m/Assistant/contracts/epistemics"
|
||||
"git.cin.su/m/Assistant/contracts/interaction"
|
||||
)
|
||||
|
||||
type learningEpistemicsStub struct {
|
||||
experiences []epistemics.ExperienceRecord
|
||||
}
|
||||
|
||||
func (s *learningEpistemicsStub) SaveSystemExpectation(context.Context, epistemics.SystemExpectation) error {
|
||||
return nil
|
||||
}
|
||||
func (s *learningEpistemicsStub) SaveExperienceRecord(_ context.Context, value epistemics.ExperienceRecord) error {
|
||||
s.experiences = append(s.experiences, value)
|
||||
return nil
|
||||
}
|
||||
func (s *learningEpistemicsStub) SaveReflectionCandidate(context.Context, epistemics.ReflectionCandidate) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
type learningCapabilityStub struct {
|
||||
loadCalls int
|
||||
saveCalls int
|
||||
}
|
||||
|
||||
func (s *learningCapabilityStub) LoadSkillProfile(context.Context, capability.SkillProfileID) (capability.SkillProfile, error) {
|
||||
s.loadCalls++
|
||||
return capability.SkillProfile{}, nil
|
||||
}
|
||||
func (s *learningCapabilityStub) SaveSkillProfile(context.Context, capability.SkillProfile) error {
|
||||
s.saveCalls++
|
||||
return nil
|
||||
}
|
||||
|
||||
func TestGenericCorrectionDoesNotDegradeReminderSkill(t *testing.T) {
|
||||
epistemicsStore := &learningEpistemicsStub{}
|
||||
capabilityStore := &learningCapabilityStub{}
|
||||
recorder := learningRecorder{epistemics: epistemicsStore, capability: capabilityStore}
|
||||
recorder.recordCorrection(context.Background(), correctionLearningInput{
|
||||
Intent: interaction.OriginalHumanIntent{ID: "intent-1"},
|
||||
Utterance: interaction.HumanUtterance{ID: "utterance-1"},
|
||||
At: time.UnixMilli(1000),
|
||||
})
|
||||
if capabilityStore.loadCalls != 0 || capabilityStore.saveCalls != 0 {
|
||||
t.Fatalf("generic correction must not mutate reminder skill: load=%d save=%d", capabilityStore.loadCalls, capabilityStore.saveCalls)
|
||||
}
|
||||
if len(epistemicsStore.experiences) != 1 || epistemicsStore.experiences[0].SkillRef != "" {
|
||||
t.Fatalf("generic correction must remain evidence without a reminder skill reference: %#v", epistemicsStore.experiences)
|
||||
}
|
||||
}
|
||||
@@ -1,5 +1,5 @@
|
||||
{
|
||||
"task_file": "dev/tasks/DEV-000092-cognitive-model-inventory-fast-path.json",
|
||||
"task_id": "DEV-000092",
|
||||
"task_file": "dev/tasks/DEV-000093-learning-evidence-integrity.json",
|
||||
"task_id": "DEV-000093",
|
||||
"task_revision": 1
|
||||
}
|
||||
|
||||
@@ -0,0 +1,35 @@
|
||||
{
|
||||
"task_id": "DEV-000093",
|
||||
"task_revision": 1,
|
||||
"architecture_revision": "DEV-000073-baseline",
|
||||
"base_repo_revision": "cf66cd472cb791125347922c9e74ded534c9a3a7",
|
||||
"risk_tier": "E",
|
||||
"intent": "Repair learning-evidence integrity so reflection candidates preserve normalized source evidence, corrections do not degrade unrelated skills, and unknown or skipped outcomes remain distinct from verified failures.",
|
||||
"requirement_ids": ["Reflection references valid experience", "Skill evidence stays capability-specific", "Unknown is not failure"],
|
||||
"allowed_change_scope": {
|
||||
"modules": ["core-runtime", "epistemics", "capability", "development-governance"],
|
||||
"paths": [
|
||||
"cmd/core-runtime/learning.go",
|
||||
"cmd/core-runtime/learning_test.go",
|
||||
"modules/epistemics/domain/experience.go",
|
||||
"modules/epistemics/domain/experience_test.go",
|
||||
"modules/capability/domain/skill.go",
|
||||
"modules/capability/domain/skill_test.go",
|
||||
"dev/ACTIVE_TASK.json",
|
||||
"dev/tasks/DEV-000093-learning-evidence-integrity.json"
|
||||
]
|
||||
},
|
||||
"forbidden_change_scope": {"paths": ["architecture/**", "security/**", "database/**", "deploy/**"]},
|
||||
"allowed_architecture_delta": {"new_cross_module_dependencies": 0, "new_canonical_owners": 0, "new_privileges": 0, "new_third_party_dependencies": 0, "new_database_owner_roles": 0},
|
||||
"contracts": {"consume": ["existing epistemics experience/reflection contract", "existing capability skill contract"], "may_change": []},
|
||||
"semantic_owner": "epistemics",
|
||||
"canonical_state": ["epistemics ExperienceRecord and ReflectionCandidate", "capability SkillProfile"],
|
||||
"state_transitions": ["failure/correction/low-confidence evidence -> reflection candidate requiring cognition", "unknown/skipped skill observation -> retained as non-failure evidence"],
|
||||
"side_effects_and_privileges": ["No external mutation", "No new privilege", "No model/provider invocation"],
|
||||
"persistence_and_migrations": ["No schema migration"],
|
||||
"failure_retry_restart": ["Learning write failure remains non-authoritative and cannot alter external state", "No retry is inferred as a success"],
|
||||
"assumptions": ["A generic user correction is not evidence against the reminder-resolution skill unless it is explicitly linked to that skill"],
|
||||
"acceptance_scenarios": ["Reflection accepts an otherwise valid experience without a caller-supplied ID", "Correction outside reminder flow does not mutate reminder skill", "Unknown and skipped outcomes do not populate known-failure evidence"],
|
||||
"required_tests": ["go test ./cmd/core-runtime", "go test ./modules/epistemics/domain", "go test ./modules/capability/domain", "go test ./...", "go vet ./...", "devctl preflight", "devctl check-diff", "git diff --check"],
|
||||
"terminal_gap_states": ["BLOCKED_BY_ARCHITECTURE_GAP", "BLOCKED_BY_SPEC_GAP", "BLOCKED_BY_CONFLICT"]
|
||||
}
|
||||
@@ -42,13 +42,13 @@ func ObserveSkillOutcome(profile capability.SkillProfile, observation capability
|
||||
}
|
||||
next.LastValidatedUnixMillis = observation.ObservedAtUnixMillis
|
||||
next.ConfidenceMillis = clampConfidence(next.ConfidenceMillis + 25)
|
||||
case capability.UsageFailed, capability.UsageUnknown:
|
||||
case capability.UsageFailed:
|
||||
next.KnownFailureExperienceRefs = appendEvidenceRef(next.KnownFailureExperienceRefs, observation.ExperienceRef)
|
||||
next.ObservedFailureCount++
|
||||
next.ConfidenceMillis = clampConfidence(next.ConfidenceMillis - 150)
|
||||
case capability.UsageSkipped:
|
||||
next.KnownFailureExperienceRefs = appendEvidenceRef(next.KnownFailureExperienceRefs, observation.ExperienceRef)
|
||||
next.ConfidenceMillis = clampConfidence(next.ConfidenceMillis - 25)
|
||||
case capability.UsageUnknown, capability.UsageSkipped:
|
||||
// UNKNOWN and SKIPPED are evidence of incomplete observation or a safe
|
||||
// non-application, not evidence that the skill itself failed.
|
||||
default:
|
||||
return capability.SkillProfile{}, ErrInvalidSkillProfile
|
||||
}
|
||||
|
||||
@@ -44,6 +44,20 @@ func TestValidatedSkillCanApplyFastPathWhenFreshAndConfident(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestUnknownAndSkippedOutcomesDoNotBecomeSkillFailures(t *testing.T) {
|
||||
profile := skillProfile()
|
||||
for _, outcome := range []capability.UsageOutcomeClass{capability.UsageUnknown, capability.UsageSkipped} {
|
||||
next, err := ObserveSkillOutcome(profile, capability.SkillOutcomeObservation{ExperienceRef: capability.SkillEvidenceRef("experience:" + string(outcome)), Outcome: outcome, ObservedAtUnixMillis: profile.RecordedAtUnixMillis + 1})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if next.ObservedFailureCount != profile.ObservedFailureCount || len(next.KnownFailureExperienceRefs) != len(profile.KnownFailureExperienceRefs) || next.ConfidenceMillis != profile.ConfidenceMillis {
|
||||
t.Fatalf("outcome %q was incorrectly recorded as failure: %#v", outcome, next)
|
||||
}
|
||||
profile = next
|
||||
}
|
||||
}
|
||||
|
||||
func skillProfile() capability.SkillProfile {
|
||||
return capability.SkillProfile{
|
||||
ID: "skill-obvious-reminder-response-v1",
|
||||
|
||||
@@ -54,9 +54,11 @@ func NewExperienceRecord(value epistemics.ExperienceRecord) (epistemics.Experien
|
||||
}
|
||||
|
||||
func ReflectionForExperience(value epistemics.ExperienceRecord) (epistemics.ReflectionCandidate, bool, error) {
|
||||
if _, err := NewExperienceRecord(value); err != nil {
|
||||
normalized, err := NewExperienceRecord(value)
|
||||
if err != nil {
|
||||
return epistemics.ReflectionCandidate{}, false, err
|
||||
}
|
||||
value = normalized
|
||||
trigger, ok := reflectionTrigger(value)
|
||||
if !ok {
|
||||
return epistemics.ReflectionCandidate{}, false, nil
|
||||
@@ -74,7 +76,7 @@ func ReflectionForExperience(value epistemics.ExperienceRecord) (epistemics.Refl
|
||||
SourceUtteranceID: value.SourceUtteranceID,
|
||||
Summary: reflectionSummary(trigger, value),
|
||||
EvidenceRefs: reflectionEvidence(value),
|
||||
RequiresCognition: trigger == epistemics.ReflectionContradiction || trigger == epistemics.ReflectionRepeatedDegradation,
|
||||
RequiresCognition: reflectionNeedsCognition(trigger),
|
||||
RecordedAtUnixMillis: value.RecordedAtUnixMillis,
|
||||
Revision: 1,
|
||||
}
|
||||
@@ -84,6 +86,15 @@ func ReflectionForExperience(value epistemics.ExperienceRecord) (epistemics.Refl
|
||||
return candidate, true, nil
|
||||
}
|
||||
|
||||
func reflectionNeedsCognition(trigger epistemics.ReflectionTriggerKind) bool {
|
||||
switch trigger {
|
||||
case epistemics.ReflectionUserCorrection, epistemics.ReflectionUnexpectedOutcome, epistemics.ReflectionFailure, epistemics.ReflectionLowConfidence, epistemics.ReflectionContradiction, epistemics.ReflectionRepeatedDegradation:
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func ValidateReflectionCandidate(value epistemics.ReflectionCandidate) error {
|
||||
if value.ID == "" || value.Trigger == "" || value.State == "" || value.SourceExperienceID == "" ||
|
||||
value.OriginalIntentID == "" || value.SourceUtteranceID == "" || strings.TrimSpace(value.Summary) == "" ||
|
||||
|
||||
@@ -50,11 +50,28 @@ func TestUserCorrectionCreatesReflectionCandidate(t *testing.T) {
|
||||
if err != nil || !ok {
|
||||
t.Fatalf("expected reflection candidate, ok=%v err=%v", ok, err)
|
||||
}
|
||||
if candidate.Trigger != epistemics.ReflectionUserCorrection || candidate.State != epistemics.ReflectionCandidateOpen || len(candidate.EvidenceRefs) == 0 {
|
||||
if candidate.Trigger != epistemics.ReflectionUserCorrection || candidate.State != epistemics.ReflectionCandidateOpen || !candidate.RequiresCognition || len(candidate.EvidenceRefs) == 0 {
|
||||
t.Fatalf("unexpected reflection candidate: %#v", candidate)
|
||||
}
|
||||
}
|
||||
|
||||
func TestReflectionNormalizesExperienceIDBeforeReferencingIt(t *testing.T) {
|
||||
candidate, ok, err := ReflectionForExperience(epistemics.ExperienceRecord{
|
||||
SourceKind: epistemics.ExperienceSourceInteraction,
|
||||
OriginalIntentID: "intent-1",
|
||||
SourceUtteranceID: "utterance-3",
|
||||
HelpMethod: epistemics.HelpMethodClarification,
|
||||
ExpectedOutcome: epistemics.OutcomeSuccess,
|
||||
ActualOutcome: epistemics.OutcomeFailed,
|
||||
OutcomeDetail: "deterministic route failed",
|
||||
ConfidenceMillis: 800,
|
||||
RecordedAtUnixMillis: 1200,
|
||||
})
|
||||
if err != nil || !ok || candidate.SourceExperienceID == "" || !candidate.RequiresCognition {
|
||||
t.Fatalf("reflection did not preserve normalized source experience: candidate=%#v ok=%v err=%v", candidate, ok, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFastPathDeclineReturnsToThinkingInsteadOfGuessing(t *testing.T) {
|
||||
experience, err := FastPathDeclineExperience("intent-1", "utterance-1", "skill:obvious-reminder-response-v1", string(epistemics.OutcomeOutOfDistribution), 1200)
|
||||
if err != nil {
|
||||
|
||||
Reference in New Issue
Block a user