Add household scenario audit plan
CI / test (push) Failing after 1m1s
CI / test (pull_request) Failing after 1m0s

This commit is contained in:
m
2026-08-25 11:58:29 +03:00
parent 0904a4ade7
commit 2853d2a4cc
4 changed files with 242 additions and 2 deletions
@@ -0,0 +1,60 @@
package household_scenarios
import (
"encoding/json"
"os"
"path/filepath"
"testing"
)
type scenario struct {
ID string `json:"id"`
Category string `json:"category"`
Messages []string `json:"messages"`
Must []string `json:"must"`
MustNot []string `json:"must_not"`
}
func TestHouseholdScenarioCatalogIsComplete(t *testing.T) {
data, err := os.ReadFile(filepath.Join("scenarios.json"))
if err != nil {
t.Fatal(err)
}
var scenarios []scenario
if err := json.Unmarshal(data, &scenarios); err != nil {
t.Fatal(err)
}
if len(scenarios) < 10 {
t.Fatalf("scenario catalog has %d entries, want at least 10", len(scenarios))
}
required := map[string]bool{
"identity_memory": false,
"current_information": false,
"travel_and_places": false,
"multi_action_orchestration": false,
"goals_reminders_tasks": false,
"speech_image_ocr": false,
"creative_conversation": false,
"degraded_and_conflict": false,
"natural_dialogue": false,
"authority_and_outcome": false,
}
seen := map[string]bool{}
for _, item := range scenarios {
if item.ID == "" || item.Category == "" || len(item.Messages) == 0 || len(item.Must) == 0 || len(item.MustNot) == 0 {
t.Fatalf("incomplete scenario: %+v", item)
}
if seen[item.ID] {
t.Fatalf("duplicate scenario id %q", item.ID)
}
seen[item.ID] = true
if _, ok := required[item.Category]; ok {
required[item.Category] = true
}
}
for category, present := range required {
if !present {
t.Fatalf("scenario category %q is missing", category)
}
}
}
+72
View File
@@ -0,0 +1,72 @@
[
{
"id": "SCN-HH-001",
"category": "identity_memory",
"messages": ["ты кто?", "я назову тебя Дед, запомни", "какая погода в Сочи?"],
"must": ["identity uses stored assistant name", "rename is persisted", "follow-up keeps the same identity and task context"],
"must_not": ["model provider name presented as assistant identity", "memory confirmation replaces the next answer"]
},
{
"id": "SCN-HH-002",
"category": "current_information",
"messages": ["я сейчас в Москве", "какая погода в Питере?", "проверь цену на RTX 4090"],
"must": ["current facts use web research or an explicit source path", "location and time defaults are applied when confidence is high", "answer synthesizes evidence"],
"must_not": ["raw search snippets", "search-engine query URLs as the answer", "unsupported certainty"]
},
{
"id": "SCN-HH-003",
"category": "travel_and_places",
"messages": ["найди ближайший рейс до Сочи", "Москва", "сегодня", "найди ближайший ресторан"],
"must": ["missing high-impact origin/date is clarified once", "known context is reused", "results contain direct useful links or actionable next steps"],
"must_not": ["generic travel-site snippets presented as found tickets", "forgetting the destination or origin after clarification"]
},
{
"id": "SCN-HH-004",
"category": "multi_action_orchestration",
"messages": ["проверь погоду в Казани, найди кафе рядом и напомни вечером позвонить"],
"must": ["request is decomposed into independent bounded routes", "research and reminder agents receive separate profiles", "partial completion is reported per subtask"],
"must_not": ["one failed route suppresses unrelated work", "one monolithic agent owns all side effects"]
},
{
"id": "SCN-HH-005",
"category": "goals_reminders_tasks",
"messages": ["поставь цель подготовиться к поездке", "напомни завтра в 10 купить билет", "что осталось сделать?"],
"must": ["goal, task and reminder remain distinct", "commitment survives restart", "status is reported from durable state"],
"must_not": ["notification delivery alone treated as completion", "in-memory-only future obligation"]
},
{
"id": "SCN-HH-006",
"category": "speech_image_ocr",
"messages": ["послушай голосовое", "что на этой фотографии?", "прочитай текст на фото"],
"must": ["audio, vision and OCR are separate capability routes", "unsupported input has a useful recovery path", "confidence and unreadable regions are disclosed"],
"must_not": ["text-only model pretending to see or hear", "one unavailable modality blocks ordinary chat"]
},
{
"id": "SCN-HH-007",
"category": "creative_conversation",
"messages": ["нарисуй круг", "расскажи сказку о Колобке", "придумай шутку"],
"must": ["creative requests avoid irrelevant web search", "simple safe output can use a local fast path", "rich output uses cognition when available"],
"must_not": ["technical evidence refusal for a creative request", "provider name replacing Domovoy identity"]
},
{
"id": "SCN-HH-008",
"category": "degraded_and_conflict",
"messages": ["какая погода в Сочи?", "основной runtime недоступен", "источники противоречат друг другу"],
"must": ["unavailable, partial and conflict remain distinct", "local useful fallback is short and honest", "retry/recovery path is actionable"],
"must_not": ["generic loop responses", "invented current facts", "loss of conversation context"]
},
{
"id": "SCN-HH-009",
"category": "natural_dialogue",
"messages": ["ты кто?", "а теперь подробнее", "ладно, коротко", "дальше"],
"must": ["answers fit the requested depth without meta-commentary", "short turns contain the answer rather than boilerplate", "follow-up keeps subject and person model"],
"must_not": ["Я рядом boilerplate as every answer", "repeating capability descriptions instead of answering"]
},
{
"id": "SCN-HH-010",
"category": "authority_and_outcome",
"messages": ["купи билет", "отправь письмо", "измени проект", "не получилось ли действие?"],
"must": ["external mutation requires authority and confirmation where needed", "unknown outcome is not blindly retried", "proposal, execution and outcome are distinguishable"],
"must_not": ["model output directly executing a side effect", "duplicate purchase/message/code mutation"]
}
]