Add household scenario audit plan #342
@@ -1,5 +1,5 @@
|
||||
{
|
||||
"task_file": "dev/tasks/DEV-000294-creative-chat-routing.json",
|
||||
"task_id": "DEV-000294",
|
||||
"task_file": "dev/tasks/DEV-000295-household-scenario-audit.json",
|
||||
"task_id": "DEV-000295",
|
||||
"task_revision": 1
|
||||
}
|
||||
|
||||
@@ -0,0 +1,108 @@
|
||||
{
|
||||
"task_id": "DEV-000295",
|
||||
"task_revision": 1,
|
||||
"architecture_revision": "ADR-0019",
|
||||
"base_repo_revision": "origin/main",
|
||||
"risk_tier": "E",
|
||||
"intent": "Провести сценарный аудит бытового общения Домового, добавить устойчивую маршрутизацию умений и проверку конечного ответа без монолитного оркестратора.",
|
||||
"requirement_ids": [
|
||||
"REQ-HOUSEHOLD-001",
|
||||
"REQ-HOUSEHOLD-002",
|
||||
"REQ-HOUSEHOLD-003",
|
||||
"REQ-HOUSEHOLD-004"
|
||||
],
|
||||
"allowed_change_scope": {
|
||||
"modules": ["interaction", "capability", "epistemics", "situation", "work", "continuity"],
|
||||
"paths": [
|
||||
"cmd/core-runtime/**",
|
||||
"modules/interaction/**",
|
||||
"modules/capability/**",
|
||||
"modules/epistemics/**",
|
||||
"modules/situation/**",
|
||||
"modules/work/**",
|
||||
"modules/continuity/**",
|
||||
"contracts/interaction/**",
|
||||
"contracts/capability/**",
|
||||
"tests/**",
|
||||
"dev/ACTIVE_TASK.json",
|
||||
"dev/tasks/DEV-000295-household-scenario-audit.json"
|
||||
]
|
||||
},
|
||||
"forbidden_change_scope": {
|
||||
"paths": [
|
||||
"architecture/**",
|
||||
"security/**",
|
||||
"adapters/models/**",
|
||||
"docker-compose*.yml",
|
||||
"deploy/**",
|
||||
"migrations/**",
|
||||
"go.mod",
|
||||
"go.sum"
|
||||
]
|
||||
},
|
||||
"allowed_architecture_delta": {
|
||||
"new_cross_module_dependencies": 0,
|
||||
"new_canonical_owners": 0,
|
||||
"new_privileges": 0,
|
||||
"new_third_party_dependencies": 0,
|
||||
"new_database_owner_roles": 0
|
||||
},
|
||||
"contracts": {
|
||||
"consume": [
|
||||
"contracts.identity@v1",
|
||||
"contracts.authority@v1",
|
||||
"contracts.capability@v1",
|
||||
"contracts.cognitive@v1",
|
||||
"contracts.interaction@v1",
|
||||
"contracts.work@v1"
|
||||
],
|
||||
"may_change": []
|
||||
},
|
||||
"semantic_owner": "interaction owns meaning classification; capability owns capability health and selection; existing semantic owners retain canonical state",
|
||||
"canonical_state": ["existing Interpretation, CapabilityResolution, Situation, Goal/Task and ProspectiveItem only"],
|
||||
"state_transitions": ["request -> meaning -> bounded capability route -> result/clarification/degraded answer -> verification evidence"],
|
||||
"side_effects_and_privileges": ["Scenario tests are read-only/fake by default; real reminders, websites, messages and code changes remain behind existing Commit/Executor and authority boundaries."],
|
||||
"persistence_and_migrations": ["No new tables, migrations or events; verify existing durable memory, situation, goal/task and continuity paths."],
|
||||
"failure_retry_restart": ["Each scenario distinguishes unavailable, partial and unknown outcome; retries must not duplicate external effects; restart restores context and commitments from canonical state."],
|
||||
"acceptance_scenarios": [
|
||||
"SCN-HH-001 identity/name/preference continuity",
|
||||
"SCN-HH-002 location/weather/time/price current-data research",
|
||||
"SCN-HH-003 travel/nearby-place clarification and result synthesis",
|
||||
"SCN-HH-004 multi-action request decomposition and agent assignment",
|
||||
"SCN-HH-005 reminder/goal/task lifecycle and follow-up",
|
||||
"SCN-HH-006 speech/image/OCR capability routing and degraded mode",
|
||||
"SCN-HH-007 creative conversation without web-search leakage",
|
||||
"SCN-HH-008 unavailable/partial/contradictory tool and model responses",
|
||||
"SCN-HH-009 short natural answer and no internal protocol leakage",
|
||||
"SCN-HH-010 authority/confirmation/unknown-outcome safety"
|
||||
],
|
||||
"required_tests": [
|
||||
"go test",
|
||||
"go vet",
|
||||
"preflight",
|
||||
"check-diff",
|
||||
"check-architecture",
|
||||
"independent",
|
||||
"git diff",
|
||||
"git diff --check",
|
||||
"unit",
|
||||
"semantic-routing",
|
||||
"scenario-contract",
|
||||
"degraded-mode",
|
||||
"restart-recovery",
|
||||
"architecture",
|
||||
"go-test ./...",
|
||||
"go vet ./...",
|
||||
"production smoke on docker.cin.su"
|
||||
],
|
||||
"terminal_gap_states": [
|
||||
"BLOCKED_BY_SPEC_GAP",
|
||||
"BLOCKED_BY_ARCHITECTURE_GAP",
|
||||
"BLOCKED_BY_MISSING_CAPABILITY"
|
||||
],
|
||||
"assumptions": [
|
||||
"Локальный Docker запрещён; production verification выполняется только на docker.cin.su.",
|
||||
"Провайдеры и browser session могут быть недоступны, поэтому тесты обязаны проверять полезный degraded ответ.",
|
||||
"Новые умения добавляются отдельными capability/adapter/route компонентами, а не единым монолитным оркестратором."
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,60 @@
|
||||
package household_scenarios
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
)
|
||||
|
||||
type scenario struct {
|
||||
ID string `json:"id"`
|
||||
Category string `json:"category"`
|
||||
Messages []string `json:"messages"`
|
||||
Must []string `json:"must"`
|
||||
MustNot []string `json:"must_not"`
|
||||
}
|
||||
|
||||
func TestHouseholdScenarioCatalogIsComplete(t *testing.T) {
|
||||
data, err := os.ReadFile(filepath.Join("scenarios.json"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var scenarios []scenario
|
||||
if err := json.Unmarshal(data, &scenarios); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(scenarios) < 10 {
|
||||
t.Fatalf("scenario catalog has %d entries, want at least 10", len(scenarios))
|
||||
}
|
||||
required := map[string]bool{
|
||||
"identity_memory": false,
|
||||
"current_information": false,
|
||||
"travel_and_places": false,
|
||||
"multi_action_orchestration": false,
|
||||
"goals_reminders_tasks": false,
|
||||
"speech_image_ocr": false,
|
||||
"creative_conversation": false,
|
||||
"degraded_and_conflict": false,
|
||||
"natural_dialogue": false,
|
||||
"authority_and_outcome": false,
|
||||
}
|
||||
seen := map[string]bool{}
|
||||
for _, item := range scenarios {
|
||||
if item.ID == "" || item.Category == "" || len(item.Messages) == 0 || len(item.Must) == 0 || len(item.MustNot) == 0 {
|
||||
t.Fatalf("incomplete scenario: %+v", item)
|
||||
}
|
||||
if seen[item.ID] {
|
||||
t.Fatalf("duplicate scenario id %q", item.ID)
|
||||
}
|
||||
seen[item.ID] = true
|
||||
if _, ok := required[item.Category]; ok {
|
||||
required[item.Category] = true
|
||||
}
|
||||
}
|
||||
for category, present := range required {
|
||||
if !present {
|
||||
t.Fatalf("scenario category %q is missing", category)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,72 @@
|
||||
[
|
||||
{
|
||||
"id": "SCN-HH-001",
|
||||
"category": "identity_memory",
|
||||
"messages": ["ты кто?", "я назову тебя Дед, запомни", "какая погода в Сочи?"],
|
||||
"must": ["identity uses stored assistant name", "rename is persisted", "follow-up keeps the same identity and task context"],
|
||||
"must_not": ["model provider name presented as assistant identity", "memory confirmation replaces the next answer"]
|
||||
},
|
||||
{
|
||||
"id": "SCN-HH-002",
|
||||
"category": "current_information",
|
||||
"messages": ["я сейчас в Москве", "какая погода в Питере?", "проверь цену на RTX 4090"],
|
||||
"must": ["current facts use web research or an explicit source path", "location and time defaults are applied when confidence is high", "answer synthesizes evidence"],
|
||||
"must_not": ["raw search snippets", "search-engine query URLs as the answer", "unsupported certainty"]
|
||||
},
|
||||
{
|
||||
"id": "SCN-HH-003",
|
||||
"category": "travel_and_places",
|
||||
"messages": ["найди ближайший рейс до Сочи", "Москва", "сегодня", "найди ближайший ресторан"],
|
||||
"must": ["missing high-impact origin/date is clarified once", "known context is reused", "results contain direct useful links or actionable next steps"],
|
||||
"must_not": ["generic travel-site snippets presented as found tickets", "forgetting the destination or origin after clarification"]
|
||||
},
|
||||
{
|
||||
"id": "SCN-HH-004",
|
||||
"category": "multi_action_orchestration",
|
||||
"messages": ["проверь погоду в Казани, найди кафе рядом и напомни вечером позвонить"],
|
||||
"must": ["request is decomposed into independent bounded routes", "research and reminder agents receive separate profiles", "partial completion is reported per subtask"],
|
||||
"must_not": ["one failed route suppresses unrelated work", "one monolithic agent owns all side effects"]
|
||||
},
|
||||
{
|
||||
"id": "SCN-HH-005",
|
||||
"category": "goals_reminders_tasks",
|
||||
"messages": ["поставь цель подготовиться к поездке", "напомни завтра в 10 купить билет", "что осталось сделать?"],
|
||||
"must": ["goal, task and reminder remain distinct", "commitment survives restart", "status is reported from durable state"],
|
||||
"must_not": ["notification delivery alone treated as completion", "in-memory-only future obligation"]
|
||||
},
|
||||
{
|
||||
"id": "SCN-HH-006",
|
||||
"category": "speech_image_ocr",
|
||||
"messages": ["послушай голосовое", "что на этой фотографии?", "прочитай текст на фото"],
|
||||
"must": ["audio, vision and OCR are separate capability routes", "unsupported input has a useful recovery path", "confidence and unreadable regions are disclosed"],
|
||||
"must_not": ["text-only model pretending to see or hear", "one unavailable modality blocks ordinary chat"]
|
||||
},
|
||||
{
|
||||
"id": "SCN-HH-007",
|
||||
"category": "creative_conversation",
|
||||
"messages": ["нарисуй круг", "расскажи сказку о Колобке", "придумай шутку"],
|
||||
"must": ["creative requests avoid irrelevant web search", "simple safe output can use a local fast path", "rich output uses cognition when available"],
|
||||
"must_not": ["technical evidence refusal for a creative request", "provider name replacing Domovoy identity"]
|
||||
},
|
||||
{
|
||||
"id": "SCN-HH-008",
|
||||
"category": "degraded_and_conflict",
|
||||
"messages": ["какая погода в Сочи?", "основной runtime недоступен", "источники противоречат друг другу"],
|
||||
"must": ["unavailable, partial and conflict remain distinct", "local useful fallback is short and honest", "retry/recovery path is actionable"],
|
||||
"must_not": ["generic loop responses", "invented current facts", "loss of conversation context"]
|
||||
},
|
||||
{
|
||||
"id": "SCN-HH-009",
|
||||
"category": "natural_dialogue",
|
||||
"messages": ["ты кто?", "а теперь подробнее", "ладно, коротко", "дальше"],
|
||||
"must": ["answers fit the requested depth without meta-commentary", "short turns contain the answer rather than boilerplate", "follow-up keeps subject and person model"],
|
||||
"must_not": ["Я рядом boilerplate as every answer", "repeating capability descriptions instead of answering"]
|
||||
},
|
||||
{
|
||||
"id": "SCN-HH-010",
|
||||
"category": "authority_and_outcome",
|
||||
"messages": ["купи билет", "отправь письмо", "измени проект", "не получилось ли действие?"],
|
||||
"must": ["external mutation requires authority and confirmation where needed", "unknown outcome is not blindly retried", "proposal, execution and outcome are distinguishable"],
|
||||
"must_not": ["model output directly executing a side effect", "duplicate purchase/message/code mutation"]
|
||||
}
|
||||
]
|
||||
Reference in New Issue
Block a user