fix: preserve learning evidence integrity
This commit is contained in:
@@ -42,13 +42,13 @@ func ObserveSkillOutcome(profile capability.SkillProfile, observation capability
|
||||
}
|
||||
next.LastValidatedUnixMillis = observation.ObservedAtUnixMillis
|
||||
next.ConfidenceMillis = clampConfidence(next.ConfidenceMillis + 25)
|
||||
case capability.UsageFailed, capability.UsageUnknown:
|
||||
case capability.UsageFailed:
|
||||
next.KnownFailureExperienceRefs = appendEvidenceRef(next.KnownFailureExperienceRefs, observation.ExperienceRef)
|
||||
next.ObservedFailureCount++
|
||||
next.ConfidenceMillis = clampConfidence(next.ConfidenceMillis - 150)
|
||||
case capability.UsageSkipped:
|
||||
next.KnownFailureExperienceRefs = appendEvidenceRef(next.KnownFailureExperienceRefs, observation.ExperienceRef)
|
||||
next.ConfidenceMillis = clampConfidence(next.ConfidenceMillis - 25)
|
||||
case capability.UsageUnknown, capability.UsageSkipped:
|
||||
// UNKNOWN and SKIPPED are evidence of incomplete observation or a safe
|
||||
// non-application, not evidence that the skill itself failed.
|
||||
default:
|
||||
return capability.SkillProfile{}, ErrInvalidSkillProfile
|
||||
}
|
||||
|
||||
@@ -44,6 +44,20 @@ func TestValidatedSkillCanApplyFastPathWhenFreshAndConfident(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestUnknownAndSkippedOutcomesDoNotBecomeSkillFailures(t *testing.T) {
|
||||
profile := skillProfile()
|
||||
for _, outcome := range []capability.UsageOutcomeClass{capability.UsageUnknown, capability.UsageSkipped} {
|
||||
next, err := ObserveSkillOutcome(profile, capability.SkillOutcomeObservation{ExperienceRef: capability.SkillEvidenceRef("experience:" + string(outcome)), Outcome: outcome, ObservedAtUnixMillis: profile.RecordedAtUnixMillis + 1})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if next.ObservedFailureCount != profile.ObservedFailureCount || len(next.KnownFailureExperienceRefs) != len(profile.KnownFailureExperienceRefs) || next.ConfidenceMillis != profile.ConfidenceMillis {
|
||||
t.Fatalf("outcome %q was incorrectly recorded as failure: %#v", outcome, next)
|
||||
}
|
||||
profile = next
|
||||
}
|
||||
}
|
||||
|
||||
func skillProfile() capability.SkillProfile {
|
||||
return capability.SkillProfile{
|
||||
ID: "skill-obvious-reminder-response-v1",
|
||||
|
||||
@@ -54,9 +54,11 @@ func NewExperienceRecord(value epistemics.ExperienceRecord) (epistemics.Experien
|
||||
}
|
||||
|
||||
func ReflectionForExperience(value epistemics.ExperienceRecord) (epistemics.ReflectionCandidate, bool, error) {
|
||||
if _, err := NewExperienceRecord(value); err != nil {
|
||||
normalized, err := NewExperienceRecord(value)
|
||||
if err != nil {
|
||||
return epistemics.ReflectionCandidate{}, false, err
|
||||
}
|
||||
value = normalized
|
||||
trigger, ok := reflectionTrigger(value)
|
||||
if !ok {
|
||||
return epistemics.ReflectionCandidate{}, false, nil
|
||||
@@ -74,7 +76,7 @@ func ReflectionForExperience(value epistemics.ExperienceRecord) (epistemics.Refl
|
||||
SourceUtteranceID: value.SourceUtteranceID,
|
||||
Summary: reflectionSummary(trigger, value),
|
||||
EvidenceRefs: reflectionEvidence(value),
|
||||
RequiresCognition: trigger == epistemics.ReflectionContradiction || trigger == epistemics.ReflectionRepeatedDegradation,
|
||||
RequiresCognition: reflectionNeedsCognition(trigger),
|
||||
RecordedAtUnixMillis: value.RecordedAtUnixMillis,
|
||||
Revision: 1,
|
||||
}
|
||||
@@ -84,6 +86,15 @@ func ReflectionForExperience(value epistemics.ExperienceRecord) (epistemics.Refl
|
||||
return candidate, true, nil
|
||||
}
|
||||
|
||||
func reflectionNeedsCognition(trigger epistemics.ReflectionTriggerKind) bool {
|
||||
switch trigger {
|
||||
case epistemics.ReflectionUserCorrection, epistemics.ReflectionUnexpectedOutcome, epistemics.ReflectionFailure, epistemics.ReflectionLowConfidence, epistemics.ReflectionContradiction, epistemics.ReflectionRepeatedDegradation:
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func ValidateReflectionCandidate(value epistemics.ReflectionCandidate) error {
|
||||
if value.ID == "" || value.Trigger == "" || value.State == "" || value.SourceExperienceID == "" ||
|
||||
value.OriginalIntentID == "" || value.SourceUtteranceID == "" || strings.TrimSpace(value.Summary) == "" ||
|
||||
|
||||
@@ -50,11 +50,28 @@ func TestUserCorrectionCreatesReflectionCandidate(t *testing.T) {
|
||||
if err != nil || !ok {
|
||||
t.Fatalf("expected reflection candidate, ok=%v err=%v", ok, err)
|
||||
}
|
||||
if candidate.Trigger != epistemics.ReflectionUserCorrection || candidate.State != epistemics.ReflectionCandidateOpen || len(candidate.EvidenceRefs) == 0 {
|
||||
if candidate.Trigger != epistemics.ReflectionUserCorrection || candidate.State != epistemics.ReflectionCandidateOpen || !candidate.RequiresCognition || len(candidate.EvidenceRefs) == 0 {
|
||||
t.Fatalf("unexpected reflection candidate: %#v", candidate)
|
||||
}
|
||||
}
|
||||
|
||||
func TestReflectionNormalizesExperienceIDBeforeReferencingIt(t *testing.T) {
|
||||
candidate, ok, err := ReflectionForExperience(epistemics.ExperienceRecord{
|
||||
SourceKind: epistemics.ExperienceSourceInteraction,
|
||||
OriginalIntentID: "intent-1",
|
||||
SourceUtteranceID: "utterance-3",
|
||||
HelpMethod: epistemics.HelpMethodClarification,
|
||||
ExpectedOutcome: epistemics.OutcomeSuccess,
|
||||
ActualOutcome: epistemics.OutcomeFailed,
|
||||
OutcomeDetail: "deterministic route failed",
|
||||
ConfidenceMillis: 800,
|
||||
RecordedAtUnixMillis: 1200,
|
||||
})
|
||||
if err != nil || !ok || candidate.SourceExperienceID == "" || !candidate.RequiresCognition {
|
||||
t.Fatalf("reflection did not preserve normalized source experience: candidate=%#v ok=%v err=%v", candidate, ok, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFastPathDeclineReturnsToThinkingInsteadOfGuessing(t *testing.T) {
|
||||
experience, err := FastPathDeclineExperience("intent-1", "utterance-1", "skill:obvious-reminder-response-v1", string(epistemics.OutcomeOutOfDistribution), 1200)
|
||||
if err != nil {
|
||||
|
||||
Reference in New Issue
Block a user