Assistant/dev/tasks/DEV-000070-reality-evaluation-harness.json

{
  "task_id": "DEV-000070",
  "task_revision": 1,
  "architecture_revision": "DEV-000069-baseline",
  "base_repo_revision": "ade274011f6619d2e5265fca467ebbf20d8dec56",
  "risk_tier": "E",
  "intent": "Build a replayable reality evaluation harness and baseline report that measure what current Domovoy actually does well or badly across heterogeneous assistance scenarios without patching product behavior to improve the baseline.",
  "requirement_ids": [
    "DEV-000069 baseline complete",
    "Measurement before improvement",
    "No new canonical owner for evaluation",
    "Truthfulness versus user assertion distinction",
    "Human intervention measurement",
    "False claim measurement",
    "Replayable baseline and candidate comparison foundation",
    "Operator review surface"
  ],
  "allowed_change_scope": {
    "modules": [
      "interaction",
      "situation",
      "work",
      "capability",
      "continuity",
      "epistemics",
      "artifact"
    ],
    "paths": [
      "contracts/epistemics/**",
      "contracts/capability/**",
      "contracts/work/**",
      "contracts/interaction/**",
      "contracts/situation/**",
      "contracts/continuity/**",
      "modules/epistemics/**",
      "modules/capability/**",
      "modules/work/**",
      "modules/interaction/**",
      "modules/situation/**",
      "adapters/postgres/**",
      "cmd/core-runtime/**",
      "cmd/creator-web/**",
      "cmd/cognitive-benchmark/**",
      "tests/**",
      "docs/PROJECT_STATUS.md",
      "docs/evidence/**",
      "dev/ACTIVE_TASK.json",
      "dev/tasks/DEV-000070-reality-evaluation-harness.json"
    ]
  },
  "forbidden_change_scope": {
    "paths": [
      "architecture/ARCHITECTURE_CONSTITUTION.md",
      "architecture/architecture-registry.yaml",
      "architecture/architecture-registry.json",
      "architecture/adrs/**",
      "modules/authority/**",
      "modules/commit/**",
      "cmd/commit-runtime/**",
      "security/**",
      "ci/**",
      ".gitea/**",
      "deploy/**",
      "database/migrations/**"
    ]
  },
  "allowed_architecture_delta": {
    "new_cross_module_dependencies": 0,
    "new_canonical_owners": 0,
    "new_privileges": 0,
    "new_third_party_dependencies": 0,
    "new_provider_adapters": 0,
    "new_database_owner_roles": 0,
    "new_durable_canonical_types": 0
  },
  "contracts": {
    "consume": [
      "Interaction Conversation / HumanUtterance / OriginalHumanIntent / UtteranceInterpretation",
      "Situation ActiveProjectFocus / ProjectRegistry",
      "Work Goal / WorkPlan / Delegation / AgentRun / AgentResult / DelegationVerification",
      "Capability CapabilityResolution / CapabilitySelection / CapabilityGap / CapabilityUsageObservation",
      "Continuity ProspectiveItem / AttentionItem / HumanResponse",
      "Epistemics ExperienceRecord / ReflectionCandidate / SystemExpectation",
      "Artifact refs for immutable evaluation evidence"
    ],
    "may_change": [
      "contracts/epistemics@v1",
      "contracts/work@v1",
      "contracts/capability@v1"
    ]
  },
  "semantic_owner": "epistemics",
  "canonical_state": [
    "Existing ExperienceRecord evidence",
    "Existing CapabilityUsageObservation evidence",
    "Existing Work / Delegation / Verification evidence"
  ],
  "state_transitions": [
    "evaluation case replay -> ordinary Domovoy runtime behavior -> existing canonical evidence observed",
    "baseline report classifies outcome dimensions without mutating product behavior to make a case pass",
    "operator review annotates findings without becoming normal human-chat state"
  ],
  "side_effects_and_privileges": [
    "Evaluation must not create uncontrolled real-world side effects",
    "Measurement tooling may use isolated or simulated capability availability where real side effects would be unsafe",
    "No self-improvement, automatic self-modification, or learning-loop expansion is authorized",
    "Do not redesign /chat",
    "Do not optimize specifically for Codex",
    "Docker-test remains prohibited"
  ],
  "persistence_and_migrations": [
    "Prefer test/evaluation tooling and artifact evidence over new canonical persistence",
    "No new canonical owner",
    "No schema migration unless a genuine existing-owner evidence gap makes it unavoidable",
    "No cross-owner FK or cross-owner SQL join dependency"
  ],
  "failure_retry_restart": [
    "Baseline failures are evidence and must be recorded rather than patched away during this milestone",
    "Unknown / clarification / cannot verify are allowed correct outcomes",
    "Evaluation harness defects may be fixed, but product behavior must not be changed merely to improve baseline numbers"
  ],
  "assumptions": [
    "DEV-000069 production baseline is active",
    "Existing ExperienceRecord / Verification / Capability evidence is sufficient for an initial reality harness",
    "Operator-facing review may be implemented through existing Creator/Operator surface rather than a new owner"
  ],
  "acceptance_scenarios": [
    "Versioned heterogeneous evaluation suite of 30-50 cases exists",
    "Cases cover conversation, factual assertions, correction, ambiguity, reminders, context switching, capability gaps, delegation, failures, blockers, unsupported requests, consequence sensitivity and multi-turn continuation",
    "Baseline run records failures without patching product behavior",
    "Expectations are semantic/property-based rather than exact-string-based",
    "Objective evidence is used where possible and evaluator identity is recorded where semantic judgment is needed",
    "Human intervention, false claims, truthfulness and consequence awareness are measured separately",
    "Operator can review cases and findings outside normal /chat"
  ],
  "required_tests": [
    "go test ./...",
    "go vet ./...",
    "go run ./cmd/devctl preflight -active dev/ACTIVE_TASK.json",
    "go run ./cmd/devctl check-architecture",
    "go run ./cmd/devctl check-diff -active dev/ACTIVE_TASK.json",
    "tests/cognitive_advisory/cognitive_worker_env_policy_test.ps1",
    "tests/owner_acl/verify.ps1 -DockerContext docker-prod -PostgresContainer assistant-postgres-1",
    "evaluation harness unit/replay tests",
    "baseline suite execution",
    "production health"
  ],
  "terminal_gap_states": [
    "BLOCKED_BY_ARCHITECTURE_GAP",
    "BLOCKED_BY_SPEC_GAP",
    "BLOCKED_BY_MISSING_CAPABILITY",
    "BLOCKED_BY_CONFLICT"
  ]
}