From 775fa03ed4faa20e8923b8f82890ed092b924687 Mon Sep 17 00:00:00 2001 From: Mikhail Date: Sat, 22 Aug 2026 22:49:23 +0300 Subject: [PATCH] feat: bound benchmark advisory output --- cmd/cognitive-benchmark/main.go | 13 ++++++----- dev/ACTIVE_TASK.json | 4 ++-- ...0-bounded-medium-qualification-output.json | 23 +++++++++++++++++++ 3 files changed, 32 insertions(+), 8 deletions(-) create mode 100644 dev/tasks/DEV-000180-bounded-medium-qualification-output.json diff --git a/cmd/cognitive-benchmark/main.go b/cmd/cognitive-benchmark/main.go index fe87b25..e973b65 100644 --- a/cmd/cognitive-benchmark/main.go +++ b/cmd/cognitive-benchmark/main.go @@ -98,8 +98,9 @@ func main() { manifestPath := flag.String("manifest", "", "benchmark manifest JSON") outputPath := flag.String("output", "", "output artifact JSON") timeout := flag.Duration("case-timeout", 30*time.Second, "deadline for each case") + maxOutputBytes := flag.Int64("max-output-bytes", 4096, "maximum advisory output bytes per case") flag.Parse() - if *manifestPath == "" || *outputPath == "" || *timeout <= 0 { + if *manifestPath == "" || *outputPath == "" || *timeout <= 0 || *maxOutputBytes < 1 { fatal(errors.New("manifest, output and positive case-timeout are required")) } input, err := os.ReadFile(*manifestPath) @@ -110,7 +111,7 @@ func main() { if err := json.Unmarshal(input, &plan); err != nil { fatal(err) } - artifact, err := run(context.Background(), plan, *timeout, time.Now().UTC()) + artifact, err := run(context.Background(), plan, *timeout, *maxOutputBytes, time.Now().UTC()) if err != nil { fatal(err) } @@ -125,7 +126,7 @@ func main() { fmt.Printf("artifact=%s models=%d output=%s\n", artifact.ArtifactID, len(artifact.Results), *outputPath) } -func run(ctx context.Context, plan manifest, timeout time.Duration, observed time.Time) (benchmarkArtifact, error) { +func run(ctx context.Context, plan manifest, timeout time.Duration, maxOutputBytes int64, observed time.Time) (benchmarkArtifact, error) { if plan.ArtifactID == "" || len(plan.Providers) == 0 { return benchmarkArtifact{}, errors.New("invalid benchmark manifest") } @@ -142,7 +143,7 @@ func run(ctx context.Context, plan manifest, timeout time.Duration, observed tim return provider.Candidates[i].ProviderModelID < provider.Candidates[j].ProviderModelID }) for _, candidate := range provider.Candidates { - result, err := runModel(ctx, adapter, provider.ProviderInstanceID, candidate, timeout) + result, err := runModel(ctx, adapter, provider.ProviderInstanceID, candidate, timeout, maxOutputBytes) if err != nil { return benchmarkArtifact{}, err } @@ -169,7 +170,7 @@ func newProvider(candidate providerCandidate, timeout time.Duration) (models.Loc } } -func runModel(ctx context.Context, adapter models.LocalProvider, providerID string, candidate modelCandidate, timeout time.Duration) (modelResult, error) { +func runModel(ctx context.Context, adapter models.LocalProvider, providerID string, candidate modelCandidate, timeout time.Duration, maxOutputBytes int64) (modelResult, error) { if providerID == "" || candidate.ProviderModelID == "" || candidate.ModelDescriptorID == "" || candidate.ModelRevision < 1 || candidate.BindingID == "" || candidate.BindingRevision < 1 { return modelResult{}, errors.New("invalid benchmark candidate") } @@ -178,7 +179,7 @@ func runModel(ctx context.Context, adapter models.LocalProvider, providerID stri for index, item := range corpus { caseCtx, cancel := context.WithTimeout(ctx, timeout) started := time.Now() - response, err := adapter.Advise(caseCtx, cognitive.AdvisoryRequest{Version: cognitive.CreatorAdvisoryV1, RequestID: cognitive.RequestID(fmt.Sprintf("bench-%s-%d", digest(candidate.BindingID)[:12], index)), ActorID: "benchmark-actor", PrincipalID: "domovoy-creator-principal", ScopeID: "benchmark-public-synthetic", Message: item.prompt, Locale: "ru-RU", MaxOutputBytes: 4096, ProviderBindingID: capability.CapabilityBindingID(candidate.BindingID), ProviderBindingRevision: candidate.BindingRevision, ProviderInstanceID: capability.ProviderInstanceID(providerID), ModelDescriptorID: capability.ModelDescriptorID(candidate.ModelDescriptorID), ProviderModelID: capability.ProviderModelID(candidate.ProviderModelID)}) + response, err := adapter.Advise(caseCtx, cognitive.AdvisoryRequest{Version: cognitive.CreatorAdvisoryV1, RequestID: cognitive.RequestID(fmt.Sprintf("bench-%s-%d", digest(candidate.BindingID)[:12], index)), ActorID: "benchmark-actor", PrincipalID: "domovoy-creator-principal", ScopeID: "benchmark-public-synthetic", Message: item.prompt, Locale: "ru-RU", MaxOutputBytes: int(maxOutputBytes), ProviderBindingID: capability.CapabilityBindingID(candidate.BindingID), ProviderBindingRevision: candidate.BindingRevision, ProviderInstanceID: capability.ProviderInstanceID(providerID), ModelDescriptorID: capability.ModelDescriptorID(candidate.ModelDescriptorID), ProviderModelID: capability.ProviderModelID(candidate.ProviderModelID)}) cancel() caseResult := caseResult{CaseID: item.id, LatencyMillis: time.Since(started).Milliseconds()} if err != nil { diff --git a/dev/ACTIVE_TASK.json b/dev/ACTIVE_TASK.json index 7ee1897..05244b3 100644 --- a/dev/ACTIVE_TASK.json +++ b/dev/ACTIVE_TASK.json @@ -1,5 +1,5 @@ { - "task_file": "dev/tasks/DEV-000179-runtime-status-consolidation.json", - "task_id": "DEV-000179", + "task_file": "dev/tasks/DEV-000180-bounded-medium-qualification-output.json", + "task_id": "DEV-000180", "task_revision": 1 } diff --git a/dev/tasks/DEV-000180-bounded-medium-qualification-output.json b/dev/tasks/DEV-000180-bounded-medium-qualification-output.json new file mode 100644 index 0000000..87f95d0 --- /dev/null +++ b/dev/tasks/DEV-000180-bounded-medium-qualification-output.json @@ -0,0 +1,23 @@ +{ + "task_id": "DEV-000180", + "task_revision": 1, + "architecture_revision": "ADR-0018", + "base_repo_revision": "2ed2302526f6915c9ddef852ed17a8c47f716cc2", + "risk_tier": "E", + "intent": "Allow the existing synthetic qualification benchmark to bound maximum output bytes per case for slow local providers.", + "requirement_ids": ["WP-200", "AUDIT-COMPLETION-TRUTH"], + "allowed_change_scope": {"modules": ["cognitive benchmark"], "paths": ["cmd/cognitive-benchmark/main.go", "dev/ACTIVE_TASK.json", "dev/tasks/DEV-000180-bounded-medium-qualification-output.json"]}, + "forbidden_change_scope": {"paths": ["architecture/**", "security/**", "cmd/cognitive-worker/**", "modules/**", "contracts/**", "adapters/**", "database/**", "deploy/**", ".gitea/**"]}, + "allowed_architecture_delta": {"new_cross_module_dependencies": 0, "new_canonical_owners": 0, "new_privileges": 0, "new_third_party_dependencies": 0, "new_database_owner_roles": 0}, + "contracts": {"consume": ["existing cognitive advisory request contract"], "may_change": ["benchmark CLI parameter only"]}, + "semantic_owner": "cognitive benchmark", + "canonical_state": [], + "state_transitions": ["fixed benchmark output cap -> explicit bounded qualification run"], + "side_effects_and_privileges": ["none"], + "persistence_and_migrations": ["none"], + "failure_retry_restart": ["non-positive output cap rejected; default preserves previous 4096-byte behavior"], + "assumptions": ["bounded output is valid only for synthetic exact-answer corpus and does not alter production provider configuration"], + "acceptance_scenarios": ["default behavior remains unchanged", "positive output cap is forwarded to advisory request", "invalid cap fails closed", "no provider boundary is bypassed"], + "required_tests": ["go test ./...", "go vet ./...", "go run ./cmd/devctl preflight -active dev/ACTIVE_TASK.json", "go run ./cmd/devctl check-diff -active dev/ACTIVE_TASK.json", "go run ./cmd/devctl check-architecture -active dev/ACTIVE_TASK.json", "go run ./cmd/devctl/independent", "git diff --check", "CI"], + "terminal_gap_states": ["BLOCKED_BY_CONFLICT"] +} -- 2.54.0