DEV-000070 add reality evaluation baseline harness
This commit is contained in:
@@ -1,5 +1,5 @@
|
||||
{
|
||||
"task_file": "dev/tasks/DEV-000069-general-agent-delegation-codex-bridge.json",
|
||||
"task_id": "DEV-000069",
|
||||
"task_file": "dev/tasks/DEV-000070-reality-evaluation-harness.json",
|
||||
"task_id": "DEV-000070",
|
||||
"task_revision": 1
|
||||
}
|
||||
|
||||
@@ -0,0 +1,154 @@
|
||||
{
|
||||
"task_id": "DEV-000070",
|
||||
"task_revision": 1,
|
||||
"architecture_revision": "DEV-000069-baseline",
|
||||
"base_repo_revision": "ade274011f6619d2e5265fca467ebbf20d8dec56",
|
||||
"risk_tier": "E",
|
||||
"intent": "Build a replayable reality evaluation harness and baseline report that measure what current Domovoy actually does well or badly across heterogeneous assistance scenarios without patching product behavior to improve the baseline.",
|
||||
"requirement_ids": [
|
||||
"DEV-000069 baseline complete",
|
||||
"Measurement before improvement",
|
||||
"No new canonical owner for evaluation",
|
||||
"Truthfulness versus user assertion distinction",
|
||||
"Human intervention measurement",
|
||||
"False claim measurement",
|
||||
"Replayable baseline and candidate comparison foundation",
|
||||
"Operator review surface"
|
||||
],
|
||||
"allowed_change_scope": {
|
||||
"modules": [
|
||||
"interaction",
|
||||
"situation",
|
||||
"work",
|
||||
"capability",
|
||||
"continuity",
|
||||
"epistemics",
|
||||
"artifact"
|
||||
],
|
||||
"paths": [
|
||||
"contracts/epistemics/**",
|
||||
"contracts/capability/**",
|
||||
"contracts/work/**",
|
||||
"contracts/interaction/**",
|
||||
"contracts/situation/**",
|
||||
"contracts/continuity/**",
|
||||
"modules/epistemics/**",
|
||||
"modules/capability/**",
|
||||
"modules/work/**",
|
||||
"modules/interaction/**",
|
||||
"modules/situation/**",
|
||||
"adapters/postgres/**",
|
||||
"cmd/core-runtime/**",
|
||||
"cmd/creator-web/**",
|
||||
"cmd/cognitive-benchmark/**",
|
||||
"tests/**",
|
||||
"docs/PROJECT_STATUS.md",
|
||||
"docs/evidence/**",
|
||||
"dev/ACTIVE_TASK.json",
|
||||
"dev/tasks/DEV-000070-reality-evaluation-harness.json"
|
||||
]
|
||||
},
|
||||
"forbidden_change_scope": {
|
||||
"paths": [
|
||||
"architecture/ARCHITECTURE_CONSTITUTION.md",
|
||||
"architecture/architecture-registry.yaml",
|
||||
"architecture/architecture-registry.json",
|
||||
"architecture/adrs/**",
|
||||
"modules/authority/**",
|
||||
"modules/commit/**",
|
||||
"cmd/commit-runtime/**",
|
||||
"security/**",
|
||||
"ci/**",
|
||||
".gitea/**",
|
||||
"deploy/**",
|
||||
"database/migrations/**"
|
||||
]
|
||||
},
|
||||
"allowed_architecture_delta": {
|
||||
"new_cross_module_dependencies": 0,
|
||||
"new_canonical_owners": 0,
|
||||
"new_privileges": 0,
|
||||
"new_third_party_dependencies": 0,
|
||||
"new_provider_adapters": 0,
|
||||
"new_database_owner_roles": 0,
|
||||
"new_durable_canonical_types": 0
|
||||
},
|
||||
"contracts": {
|
||||
"consume": [
|
||||
"Interaction Conversation / HumanUtterance / OriginalHumanIntent / UtteranceInterpretation",
|
||||
"Situation ActiveProjectFocus / ProjectRegistry",
|
||||
"Work Goal / WorkPlan / Delegation / AgentRun / AgentResult / DelegationVerification",
|
||||
"Capability CapabilityResolution / CapabilitySelection / CapabilityGap / CapabilityUsageObservation",
|
||||
"Continuity ProspectiveItem / AttentionItem / HumanResponse",
|
||||
"Epistemics ExperienceRecord / ReflectionCandidate / SystemExpectation",
|
||||
"Artifact refs for immutable evaluation evidence"
|
||||
],
|
||||
"may_change": [
|
||||
"contracts/epistemics@v1",
|
||||
"contracts/work@v1",
|
||||
"contracts/capability@v1"
|
||||
]
|
||||
},
|
||||
"semantic_owner": "epistemics",
|
||||
"canonical_state": [
|
||||
"Existing ExperienceRecord evidence",
|
||||
"Existing CapabilityUsageObservation evidence",
|
||||
"Existing Work / Delegation / Verification evidence"
|
||||
],
|
||||
"state_transitions": [
|
||||
"evaluation case replay -> ordinary Domovoy runtime behavior -> existing canonical evidence observed",
|
||||
"baseline report classifies outcome dimensions without mutating product behavior to make a case pass",
|
||||
"operator review annotates findings without becoming normal human-chat state"
|
||||
],
|
||||
"side_effects_and_privileges": [
|
||||
"Evaluation must not create uncontrolled real-world side effects",
|
||||
"Measurement tooling may use isolated or simulated capability availability where real side effects would be unsafe",
|
||||
"No self-improvement, automatic self-modification, or learning-loop expansion is authorized",
|
||||
"Do not redesign /chat",
|
||||
"Do not optimize specifically for Codex",
|
||||
"Docker-test remains prohibited"
|
||||
],
|
||||
"persistence_and_migrations": [
|
||||
"Prefer test/evaluation tooling and artifact evidence over new canonical persistence",
|
||||
"No new canonical owner",
|
||||
"No schema migration unless a genuine existing-owner evidence gap makes it unavoidable",
|
||||
"No cross-owner FK or cross-owner SQL join dependency"
|
||||
],
|
||||
"failure_retry_restart": [
|
||||
"Baseline failures are evidence and must be recorded rather than patched away during this milestone",
|
||||
"Unknown / clarification / cannot verify are allowed correct outcomes",
|
||||
"Evaluation harness defects may be fixed, but product behavior must not be changed merely to improve baseline numbers"
|
||||
],
|
||||
"assumptions": [
|
||||
"DEV-000069 production baseline is active",
|
||||
"Existing ExperienceRecord / Verification / Capability evidence is sufficient for an initial reality harness",
|
||||
"Operator-facing review may be implemented through existing Creator/Operator surface rather than a new owner"
|
||||
],
|
||||
"acceptance_scenarios": [
|
||||
"Versioned heterogeneous evaluation suite of 30-50 cases exists",
|
||||
"Cases cover conversation, factual assertions, correction, ambiguity, reminders, context switching, capability gaps, delegation, failures, blockers, unsupported requests, consequence sensitivity and multi-turn continuation",
|
||||
"Baseline run records failures without patching product behavior",
|
||||
"Expectations are semantic/property-based rather than exact-string-based",
|
||||
"Objective evidence is used where possible and evaluator identity is recorded where semantic judgment is needed",
|
||||
"Human intervention, false claims, truthfulness and consequence awareness are measured separately",
|
||||
"Operator can review cases and findings outside normal /chat"
|
||||
],
|
||||
"required_tests": [
|
||||
"go test ./...",
|
||||
"go vet ./...",
|
||||
"go run ./cmd/devctl preflight -active dev/ACTIVE_TASK.json",
|
||||
"go run ./cmd/devctl check-architecture",
|
||||
"go run ./cmd/devctl check-diff -active dev/ACTIVE_TASK.json",
|
||||
"tests/cognitive_advisory/cognitive_worker_env_policy_test.ps1",
|
||||
"tests/owner_acl/verify.ps1 -DockerContext docker-prod -PostgresContainer assistant-postgres-1",
|
||||
"evaluation harness unit/replay tests",
|
||||
"baseline suite execution",
|
||||
"production health"
|
||||
],
|
||||
"terminal_gap_states": [
|
||||
"BLOCKED_BY_ARCHITECTURE_GAP",
|
||||
"BLOCKED_BY_SPEC_GAP",
|
||||
"BLOCKED_BY_MISSING_CAPABILITY",
|
||||
"BLOCKED_BY_CONFLICT"
|
||||
]
|
||||
}
|
||||
Reference in New Issue
Block a user