feat(epistemics): evaluate read-only research evidence #102
@@ -16,19 +16,24 @@ type FreshnessState string
|
||||
type CoverageState string
|
||||
type ConflictState string
|
||||
type EvidenceKind string
|
||||
type ResearchFindingState string
|
||||
|
||||
const (
|
||||
FreshnessCurrent FreshnessState = "CURRENT"
|
||||
FreshnessStale FreshnessState = "STALE"
|
||||
FreshnessUnknown FreshnessState = "UNKNOWN"
|
||||
CoverageComplete CoverageState = "COMPLETE"
|
||||
CoveragePartial CoverageState = "PARTIAL"
|
||||
CoverageUnknown CoverageState = "UNKNOWN"
|
||||
ConflictNone ConflictState = "NONE"
|
||||
ConflictPresent ConflictState = "PRESENT"
|
||||
ConflictUnknown ConflictState = "UNKNOWN"
|
||||
EvidenceObserved EvidenceKind = "OBSERVED"
|
||||
EvidenceAssumed EvidenceKind = "ASSUMED"
|
||||
FreshnessCurrent FreshnessState = "CURRENT"
|
||||
FreshnessStale FreshnessState = "STALE"
|
||||
FreshnessUnknown FreshnessState = "UNKNOWN"
|
||||
CoverageComplete CoverageState = "COMPLETE"
|
||||
CoveragePartial CoverageState = "PARTIAL"
|
||||
CoverageUnknown CoverageState = "UNKNOWN"
|
||||
ConflictNone ConflictState = "NONE"
|
||||
ConflictPresent ConflictState = "PRESENT"
|
||||
ConflictUnknown ConflictState = "UNKNOWN"
|
||||
EvidenceObserved EvidenceKind = "OBSERVED"
|
||||
EvidenceAssumed EvidenceKind = "ASSUMED"
|
||||
FindingSupported ResearchFindingState = "SUPPORTED"
|
||||
FindingStale ResearchFindingState = "STALE"
|
||||
FindingConflicted ResearchFindingState = "CONFLICTED"
|
||||
FindingUnknown ResearchFindingState = "UNKNOWN"
|
||||
)
|
||||
|
||||
type ResearchPlan struct {
|
||||
@@ -72,6 +77,35 @@ type Citation struct {
|
||||
ResolvedAtUnixMillis int64
|
||||
}
|
||||
|
||||
// ResearchCandidate is read-only evidence returned by a connector. The
|
||||
// connector supplies the immutable snapshot and exact excerpt; the evaluator
|
||||
// never fetches, rewrites or invents either one.
|
||||
type ResearchCandidate struct {
|
||||
PropositionKey string
|
||||
Statement string
|
||||
Source Source
|
||||
Snapshot SourceSnapshot
|
||||
Excerpt SourceExcerpt
|
||||
Evidence EvidenceKind
|
||||
Coverage CoverageState
|
||||
Conflict ConflictState
|
||||
}
|
||||
|
||||
type ResearchFinding struct {
|
||||
PropositionKey string
|
||||
Statement string
|
||||
Claim Claim
|
||||
Citations []Citation
|
||||
State ResearchFindingState
|
||||
Reason string
|
||||
}
|
||||
|
||||
type ResearchResult struct {
|
||||
PlanID ResearchPlanID
|
||||
Findings []ResearchFinding
|
||||
EvaluatedAtUnixMillis int64
|
||||
}
|
||||
|
||||
// ResolveCitation creates evidence only when all identifiers form one exact
|
||||
// source chain. It deliberately does not fetch, infer or rewrite source text.
|
||||
func ResolveCitation(source Source, snapshot SourceSnapshot, excerpt SourceExcerpt, evidence EvidenceKind, coverage CoverageState, conflict ConflictState, resolvedAtUnixMillis int64) (Citation, error) {
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
{
|
||||
"task_file": "dev/tasks/DEV-000139-production-worker-runtime.json",
|
||||
"task_id": "DEV-000139",
|
||||
"task_revision": 2
|
||||
"task_file": "dev/tasks/DEV-000140-research-evaluation-runtime.json",
|
||||
"task_id": "DEV-000140",
|
||||
"task_revision": 1
|
||||
}
|
||||
|
||||
@@ -0,0 +1,23 @@
|
||||
{
|
||||
"task_id": "DEV-000140",
|
||||
"task_revision": 1,
|
||||
"architecture_revision": "ADR-0018",
|
||||
"base_repo_revision": "07a8de9",
|
||||
"risk_tier": "E",
|
||||
"intent": "Evaluate read-only research evidence into explicit supported, stale, conflicted or unknown findings without persistence or authority mutation.",
|
||||
"requirement_ids": ["WP-102", "research evaluation", "citation provenance", "freshness", "conflict visibility"],
|
||||
"allowed_change_scope": {"modules": ["epistemics-research-runtime"], "paths": ["contracts/epistemics/research.go", "modules/epistemics/domain/research.go", "modules/epistemics/application/research.go", "tests/research_runtime/**", "dev/ACTIVE_TASK.json", "dev/tasks/DEV-000140-research-evaluation-runtime.json"]},
|
||||
"forbidden_change_scope": {"paths": ["architecture/**", "database/**", "deploy/**", "security/**", "cmd/**", ".gitea/**"]},
|
||||
"allowed_architecture_delta": {"new_cross_module_dependencies": 0, "new_canonical_owners": 0, "new_privileges": 0, "new_third_party_dependencies": 0, "new_database_owner_roles": 0},
|
||||
"contracts": {"consume": ["ADR-0019 typed source, snapshot, excerpt and citation contracts"], "may_change": ["typed research finding and read-only connector boundary"]},
|
||||
"semantic_owner": "epistemics",
|
||||
"canonical_state": [],
|
||||
"state_transitions": ["connector evidence -> supported/stale/conflicted/unknown result", "invalid source chain -> rejected"],
|
||||
"side_effects_and_privileges": ["read-only connector boundary; no external mutation; no claim persistence"],
|
||||
"persistence_and_migrations": ["none"],
|
||||
"failure_retry_restart": ["evaluation is pure and repeatable; connector failures remain errors"],
|
||||
"assumptions": ["connectors provide immutable exact excerpts and source snapshots", "proposition keys are connector-supplied typed grouping keys"],
|
||||
"acceptance_scenarios": ["no evidence remains UNKNOWN", "stale evidence is marked STALE", "incompatible statements remain CONFLICTED", "invalid citation chain is rejected", "connector boundary cannot mutate canonical state"],
|
||||
"required_tests": ["go test ./...", "go vet ./...", "preflight", "check-diff", "check-architecture", "independent", "git diff --check", "research runtime tests"],
|
||||
"terminal_gap_states": ["BLOCKED_BY_SPEC_GAP", "BLOCKED_BY_ARCHITECTURE_GAP", "BLOCKED_BY_CONFLICT"]
|
||||
}
|
||||
@@ -0,0 +1,27 @@
|
||||
package application
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"git.cin.su/m/Assistant/contracts/epistemics"
|
||||
domain "git.cin.su/m/Assistant/modules/epistemics/domain"
|
||||
)
|
||||
|
||||
// ReadOnlyConnector is the only integration point for web/repository sources.
|
||||
// Implementations return immutable evidence; they cannot persist claims.
|
||||
type ReadOnlyConnector interface {
|
||||
Collect(context.Context, epistemics.ResearchPlan) ([]epistemics.ResearchCandidate, error)
|
||||
}
|
||||
|
||||
type ResearchService struct{ Connector ReadOnlyConnector }
|
||||
|
||||
func (s ResearchService) Evaluate(ctx context.Context, plan epistemics.ResearchPlan, nowUnixMillis int64) (epistemics.ResearchResult, error) {
|
||||
if s.Connector == nil {
|
||||
return epistemics.ResearchResult{}, domain.ErrNoResearchEvidence
|
||||
}
|
||||
candidates, err := s.Connector.Collect(ctx, plan)
|
||||
if err != nil {
|
||||
return epistemics.ResearchResult{}, err
|
||||
}
|
||||
return domain.EvaluateResearch(plan, candidates, nowUnixMillis)
|
||||
}
|
||||
@@ -0,0 +1,110 @@
|
||||
package domain
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"strings"
|
||||
|
||||
"git.cin.su/m/Assistant/contracts/epistemics"
|
||||
)
|
||||
|
||||
var (
|
||||
ErrInvalidResearchPlan = errors.New("invalid research plan")
|
||||
ErrInvalidResearchCandidate = errors.New("invalid research candidate")
|
||||
ErrNoResearchEvidence = errors.New("no research evidence")
|
||||
)
|
||||
|
||||
// EvaluateResearch turns connector evidence into an explainable, non-canonical
|
||||
// result. It is deliberately pure: it does not call a provider, persist a
|
||||
// claim, or promote an assessment into authority.
|
||||
func EvaluateResearch(plan epistemics.ResearchPlan, candidates []epistemics.ResearchCandidate, evaluatedAtUnixMillis int64) (epistemics.ResearchResult, error) {
|
||||
if err := plan.Validate(); err != nil || evaluatedAtUnixMillis < 1 {
|
||||
return epistemics.ResearchResult{}, ErrInvalidResearchPlan
|
||||
}
|
||||
result := epistemics.ResearchResult{PlanID: plan.ID, EvaluatedAtUnixMillis: evaluatedAtUnixMillis}
|
||||
if len(candidates) == 0 {
|
||||
result.Findings = []epistemics.ResearchFinding{{State: epistemics.FindingUnknown, Reason: "no connector evidence was returned"}}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
groups := make(map[string][]epistemics.ResearchCandidate)
|
||||
order := make([]string, 0, len(candidates))
|
||||
for _, candidate := range candidates {
|
||||
if strings.TrimSpace(candidate.PropositionKey) == "" || strings.TrimSpace(candidate.Statement) == "" || candidate.Evidence == "" {
|
||||
return epistemics.ResearchResult{}, ErrInvalidResearchCandidate
|
||||
}
|
||||
if _, exists := groups[candidate.PropositionKey]; !exists {
|
||||
order = append(order, candidate.PropositionKey)
|
||||
}
|
||||
groups[candidate.PropositionKey] = append(groups[candidate.PropositionKey], candidate)
|
||||
}
|
||||
|
||||
for _, key := range order {
|
||||
finding, err := evaluateGroup(key, groups[key], evaluatedAtUnixMillis)
|
||||
if err != nil {
|
||||
return epistemics.ResearchResult{}, err
|
||||
}
|
||||
result.Findings = append(result.Findings, finding)
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func evaluateGroup(key string, candidates []epistemics.ResearchCandidate, evaluatedAt int64) (epistemics.ResearchFinding, error) {
|
||||
if len(candidates) == 0 {
|
||||
return epistemics.ResearchFinding{}, ErrNoResearchEvidence
|
||||
}
|
||||
first := candidates[0]
|
||||
firstBundle, err := compileResearchCandidate(first)
|
||||
if err != nil {
|
||||
return epistemics.ResearchFinding{}, ErrInvalidResearchCandidate
|
||||
}
|
||||
finding := epistemics.ResearchFinding{PropositionKey: key, Statement: first.Statement, Claim: firstBundle.Claim, Citations: []epistemics.Citation{firstBundle.Citation}, State: epistemics.FindingSupported}
|
||||
statements := map[string]struct{}{normalize(first.Statement): {}}
|
||||
if first.Snapshot.Freshness == epistemics.FreshnessUnknown || first.Evidence == epistemics.EvidenceAssumed || first.Coverage != epistemics.CoverageComplete {
|
||||
finding.State, finding.Reason = epistemics.FindingUnknown, "evidence is assumed, unknown or incomplete"
|
||||
} else if first.Snapshot.Freshness == epistemics.FreshnessStale {
|
||||
finding.State, finding.Reason = epistemics.FindingStale, "source snapshot is stale"
|
||||
}
|
||||
for _, candidate := range candidates[1:] {
|
||||
bundle, resolveErr := compileResearchCandidate(candidate)
|
||||
if resolveErr != nil {
|
||||
return epistemics.ResearchFinding{}, ErrInvalidResearchCandidate
|
||||
}
|
||||
finding.Citations = append(finding.Citations, bundle.Citation)
|
||||
statements[normalize(candidate.Statement)] = struct{}{}
|
||||
if candidate.Snapshot.Freshness == epistemics.FreshnessUnknown || candidate.Evidence == epistemics.EvidenceAssumed || candidate.Coverage != epistemics.CoverageComplete {
|
||||
finding.State, finding.Reason = epistemics.FindingUnknown, "at least one source has assumed, unknown or incomplete evidence"
|
||||
} else if candidate.Snapshot.Freshness == epistemics.FreshnessStale {
|
||||
finding.State, finding.Reason = epistemics.FindingStale, "at least one source snapshot is stale"
|
||||
}
|
||||
}
|
||||
if len(statements) > 1 {
|
||||
finding.State, finding.Reason = epistemics.FindingConflicted, "connector evidence contains incompatible statements"
|
||||
}
|
||||
for _, citation := range finding.Citations {
|
||||
if citation.Conflict == epistemics.ConflictPresent {
|
||||
finding.State, finding.Reason = epistemics.FindingConflicted, "connector marked evidence as conflicting"
|
||||
}
|
||||
}
|
||||
if len(finding.Citations) == 0 || evaluatedAt < 1 {
|
||||
return epistemics.ResearchFinding{}, ErrInvalidResearchCandidate
|
||||
}
|
||||
return finding, nil
|
||||
}
|
||||
|
||||
func normalize(value string) string { return strings.ToLower(strings.Join(strings.Fields(value), " ")) }
|
||||
|
||||
// candidateConflict keeps unknown source freshness explicit at the candidate
|
||||
// boundary. A connector may mark evidence as conflicting without changing the
|
||||
// source; group-level incompatible statements are detected separately.
|
||||
func compileResearchCandidate(candidate epistemics.ResearchCandidate) (epistemics.TruthBundle, error) {
|
||||
if candidate.Coverage == "" || candidate.Conflict == "" {
|
||||
return epistemics.TruthBundle{}, ErrInvalidResearchCandidate
|
||||
}
|
||||
citation, err := epistemics.ResolveCitation(candidate.Source, candidate.Snapshot, candidate.Excerpt, candidate.Evidence, candidate.Coverage, candidate.Conflict, candidate.Snapshot.CapturedAtUnixMillis)
|
||||
if err != nil {
|
||||
return epistemics.TruthBundle{}, err
|
||||
}
|
||||
observation := epistemics.Observation{ID: epistemics.ObservationID("observation:" + string(candidate.Excerpt.ID)), SourceSnapshot: candidate.Snapshot.ID, Excerpt: candidate.Excerpt.ID, Text: candidate.Excerpt.Text, ObservedAt: candidate.Snapshot.CapturedAtUnixMillis}
|
||||
claim := epistemics.Claim{ID: epistemics.ClaimID("claim:" + string(candidate.Excerpt.ID)), Statement: candidate.Statement, ObservationID: observation.ID, CitationID: citation.ID, Verification: epistemics.VerificationUnverified}
|
||||
return epistemics.TruthBundle{Observation: observation, Claim: claim, Belief: epistemics.Belief{ID: epistemics.BeliefID("belief:" + string(candidate.Excerpt.ID)), ClaimID: claim.ID, State: epistemics.BeliefProvisional}, Citation: citation}, nil
|
||||
}
|
||||
@@ -0,0 +1,66 @@
|
||||
package research_runtime_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"git.cin.su/m/Assistant/contracts/epistemics"
|
||||
app "git.cin.su/m/Assistant/modules/epistemics/application"
|
||||
domain "git.cin.su/m/Assistant/modules/epistemics/domain"
|
||||
)
|
||||
|
||||
type connector struct {
|
||||
candidates []epistemics.ResearchCandidate
|
||||
}
|
||||
|
||||
func (c connector) Collect(context.Context, epistemics.ResearchPlan) ([]epistemics.ResearchCandidate, error) {
|
||||
return c.candidates, nil
|
||||
}
|
||||
|
||||
func candidate(statement string, freshness epistemics.FreshnessState, source, snapshot, excerpt string) epistemics.ResearchCandidate {
|
||||
return epistemics.ResearchCandidate{PropositionKey: "domovoy", Statement: statement, Evidence: epistemics.EvidenceObserved,
|
||||
Coverage: epistemics.CoverageComplete, Conflict: epistemics.ConflictNone,
|
||||
Source: epistemics.Source{ID: epistemics.SourceID(source), Locator: "https://example.test/" + source},
|
||||
Snapshot: epistemics.SourceSnapshot{ID: epistemics.SourceSnapshotID(snapshot), SourceID: epistemics.SourceID(source), ContentDigest: "sha256:" + snapshot, CapturedAtUnixMillis: 10, Freshness: freshness},
|
||||
Excerpt: epistemics.SourceExcerpt{ID: epistemics.SourceExcerptID(excerpt), SnapshotID: epistemics.SourceSnapshotID(snapshot), Text: statement, Start: 0, End: int64(len(statement))}}
|
||||
}
|
||||
|
||||
func TestResearchServiceKeepsUnknownStaleAndConflictExplicit(t *testing.T) {
|
||||
plan := epistemics.ResearchPlan{ID: "plan-1", Question: "question", SourceClasses: []string{"repository"}, FreshnessPolicy: "current"}
|
||||
unknown, err := (app.ResearchService{Connector: connector{}}).Evaluate(context.Background(), plan, 100)
|
||||
if err != nil || len(unknown.Findings) != 1 || unknown.Findings[0].State != epistemics.FindingUnknown {
|
||||
t.Fatalf("unknown evidence was not explicit: %#v %v", unknown, err)
|
||||
}
|
||||
result, err := (app.ResearchService{Connector: connector{candidates: []epistemics.ResearchCandidate{
|
||||
candidate("one answer", epistemics.FreshnessStale, "s1", "ss1", "e1"),
|
||||
candidate("another answer", epistemics.FreshnessCurrent, "s2", "ss2", "e2"),
|
||||
}}}).Evaluate(context.Background(), plan, 100)
|
||||
if err != nil || len(result.Findings) != 1 || result.Findings[0].State != epistemics.FindingConflicted || len(result.Findings[0].Citations) != 2 {
|
||||
t.Fatalf("conflict was not retained: %#v %v", result, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEvaluateResearchRejectsFabricatedEvidenceChain(t *testing.T) {
|
||||
value := candidate("answer", epistemics.FreshnessCurrent, "s1", "ss1", "e1")
|
||||
value.Excerpt.SnapshotID = "other-snapshot"
|
||||
plan := epistemics.ResearchPlan{ID: "plan-1", Question: "question", SourceClasses: []string{"web"}, FreshnessPolicy: "current"}
|
||||
if _, err := domain.EvaluateResearch(plan, []epistemics.ResearchCandidate{value}, 100); err != domain.ErrInvalidResearchCandidate {
|
||||
t.Fatalf("fabricated evidence was accepted: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResearchServiceKeepsAssumptionAndUnknownFreshnessVisible(t *testing.T) {
|
||||
plan := epistemics.ResearchPlan{ID: "plan-1", Question: "question", SourceClasses: []string{"web"}, FreshnessPolicy: "current"}
|
||||
value := candidate("assumed answer", epistemics.FreshnessCurrent, "s1", "ss1", "e1")
|
||||
value.Evidence = epistemics.EvidenceAssumed
|
||||
result, err := (app.ResearchService{Connector: connector{candidates: []epistemics.ResearchCandidate{value}}}).Evaluate(context.Background(), plan, 100)
|
||||
if err != nil || result.Findings[0].State != epistemics.FindingUnknown {
|
||||
t.Fatalf("assumption was promoted: %#v %v", result, err)
|
||||
}
|
||||
value = candidate("unknown answer", epistemics.FreshnessUnknown, "s2", "ss2", "e2")
|
||||
value.Conflict = epistemics.ConflictUnknown
|
||||
result, err = (app.ResearchService{Connector: connector{candidates: []epistemics.ResearchCandidate{value}}}).Evaluate(context.Background(), plan, 100)
|
||||
if err != nil || result.Findings[0].State != epistemics.FindingUnknown {
|
||||
t.Fatalf("unknown freshness was promoted: %#v %v", result, err)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user