m/Assistant
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

feat: bound benchmark advisory output
CI / test (push) Successful in 35s
Details
CI / test (pull_request) Successful in 34s
Details

Browse Source
This commit is contained in:
m
2026-08-22 22:50:20 +03:00
parent 2ed2302526
commit 775fa03ed4
3 changed files with 32 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmd/cognitive-benchmark/main.go
+7 -6
View File
@@ -98,8 +98,9 @@ func main() {
manifestPath := flag.String("manifest", "", "benchmark manifest JSON")
outputPath := flag.String("output", "", "output artifact JSON")
timeout := flag.Duration("case-timeout", 30*time.Second, "deadline for each case")
maxOutputBytes := flag.Int64("max-output-bytes", 4096, "maximum advisory output bytes per case")
flag.Parse()
if *manifestPath == "" || *outputPath == "" || *timeout <= 0 {
if *manifestPath == "" || *outputPath == "" || *timeout <= 0 || *maxOutputBytes < 1 {
fatal(errors.New("manifest, output and positive case-timeout are required"))
}
input, err := os.ReadFile(*manifestPath)
@@ -110,7 +111,7 @@ func main() {
if err := json.Unmarshal(input, &plan); err != nil {
fatal(err)
}
artifact, err := run(context.Background(), plan, *timeout, time.Now().UTC())
artifact, err := run(context.Background(), plan, *timeout, *maxOutputBytes, time.Now().UTC())
if err != nil {
fatal(err)
}
@@ -125,7 +126,7 @@ func main() {
fmt.Printf("artifact=%s models=%d output=%s\n", artifact.ArtifactID, len(artifact.Results), *outputPath)
}
func run(ctx context.Context, plan manifest, timeout time.Duration, observed time.Time) (benchmarkArtifact, error) {
func run(ctx context.Context, plan manifest, timeout time.Duration, maxOutputBytes int64, observed time.Time) (benchmarkArtifact, error) {
if plan.ArtifactID == "" || len(plan.Providers) == 0 {
return benchmarkArtifact{}, errors.New("invalid benchmark manifest")
}
@@ -142,7 +143,7 @@ func run(ctx context.Context, plan manifest, timeout time.Duration, observed tim
return provider.Candidates[i].ProviderModelID < provider.Candidates[j].ProviderModelID
})
for _, candidate := range provider.Candidates {
result, err := runModel(ctx, adapter, provider.ProviderInstanceID, candidate, timeout)
result, err := runModel(ctx, adapter, provider.ProviderInstanceID, candidate, timeout, maxOutputBytes)
if err != nil {
return benchmarkArtifact{}, err
}
@@ -169,7 +170,7 @@ func newProvider(candidate providerCandidate, timeout time.Duration) (models.Loc
}
}
func runModel(ctx context.Context, adapter models.LocalProvider, providerID string, candidate modelCandidate, timeout time.Duration) (modelResult, error) {
func runModel(ctx context.Context, adapter models.LocalProvider, providerID string, candidate modelCandidate, timeout time.Duration, maxOutputBytes int64) (modelResult, error) {
if providerID == "" || candidate.ProviderModelID == "" || candidate.ModelDescriptorID == "" || candidate.ModelRevision < 1 || candidate.BindingID == "" || candidate.BindingRevision < 1 {
return modelResult{}, errors.New("invalid benchmark candidate")
}
@@ -178,7 +179,7 @@ func runModel(ctx context.Context, adapter models.LocalProvider, providerID stri
for index, item := range corpus {
caseCtx, cancel := context.WithTimeout(ctx, timeout)
started := time.Now()
response, err := adapter.Advise(caseCtx, cognitive.AdvisoryRequest{Version: cognitive.CreatorAdvisoryV1, RequestID: cognitive.RequestID(fmt.Sprintf("bench-%s-%d", digest(candidate.BindingID)[:12], index)), ActorID: "benchmark-actor", PrincipalID: "domovoy-creator-principal", ScopeID: "benchmark-public-synthetic", Message: item.prompt, Locale: "ru-RU", MaxOutputBytes: 4096, ProviderBindingID: capability.CapabilityBindingID(candidate.BindingID), ProviderBindingRevision: candidate.BindingRevision, ProviderInstanceID: capability.ProviderInstanceID(providerID), ModelDescriptorID: capability.ModelDescriptorID(candidate.ModelDescriptorID), ProviderModelID: capability.ProviderModelID(candidate.ProviderModelID)})
response, err := adapter.Advise(caseCtx, cognitive.AdvisoryRequest{Version: cognitive.CreatorAdvisoryV1, RequestID: cognitive.RequestID(fmt.Sprintf("bench-%s-%d", digest(candidate.BindingID)[:12], index)), ActorID: "benchmark-actor", PrincipalID: "domovoy-creator-principal", ScopeID: "benchmark-public-synthetic", Message: item.prompt, Locale: "ru-RU", MaxOutputBytes: int(maxOutputBytes), ProviderBindingID: capability.CapabilityBindingID(candidate.BindingID), ProviderBindingRevision: candidate.BindingRevision, ProviderInstanceID: capability.ProviderInstanceID(providerID), ModelDescriptorID: capability.ModelDescriptorID(candidate.ModelDescriptorID), ProviderModelID: capability.ProviderModelID(candidate.ProviderModelID)})
cancel()
caseResult := caseResult{CaseID: item.id, LatencyMillis: time.Since(started).Milliseconds()}
if err != nil {
dev/ACTIVE_TASK.json
+2 -2
View File
@@ -1,5 +1,5 @@
{
"task_file": "dev/tasks/DEV-000179-runtime-status-consolidation.json",
"task_id": "DEV-000179",
"task_file": "dev/tasks/DEV-000180-bounded-medium-qualification-output.json",
"task_id": "DEV-000180",
"task_revision": 1
}
dev/tasks/DEV-000180-bounded-medium-qualification-output.json
+23
View File
@@ -0,0 +1,23 @@
{
"task_id": "DEV-000180",
"task_revision": 1,
"architecture_revision": "ADR-0018",
"base_repo_revision": "2ed2302526f6915c9ddef852ed17a8c47f716cc2",
"risk_tier": "E",
"intent": "Allow the existing synthetic qualification benchmark to bound maximum output bytes per case for slow local providers.",
"requirement_ids": ["WP-200", "AUDIT-COMPLETION-TRUTH"],
"allowed_change_scope": {"modules": ["cognitive benchmark"], "paths": ["cmd/cognitive-benchmark/main.go", "dev/ACTIVE_TASK.json", "dev/tasks/DEV-000180-bounded-medium-qualification-output.json"]},
"forbidden_change_scope": {"paths": ["architecture/**", "security/**", "cmd/cognitive-worker/**", "modules/**", "contracts/**", "adapters/**", "database/**", "deploy/**", ".gitea/**"]},
"allowed_architecture_delta": {"new_cross_module_dependencies": 0, "new_canonical_owners": 0, "new_privileges": 0, "new_third_party_dependencies": 0, "new_database_owner_roles": 0},
"contracts": {"consume": ["existing cognitive advisory request contract"], "may_change": ["benchmark CLI parameter only"]},
"semantic_owner": "cognitive benchmark",
"canonical_state": [],
"state_transitions": ["fixed benchmark output cap -> explicit bounded qualification run"],
"side_effects_and_privileges": ["none"],
"persistence_and_migrations": ["none"],
"failure_retry_restart": ["non-positive output cap rejected; default preserves previous 4096-byte behavior"],
"assumptions": ["bounded output is valid only for synthetic exact-answer corpus and does not alter production provider configuration"],
"acceptance_scenarios": ["default behavior remains unchanged", "positive output cap is forwarded to advisory request", "invalid cap fails closed", "no provider boundary is bypassed"],
"required_tests": ["go test ./...", "go vet ./...", "go run ./cmd/devctl preflight -active dev/ACTIVE_TASK.json", "go run ./cmd/devctl check-diff -active dev/ACTIVE_TASK.json", "go run ./cmd/devctl check-architecture -active dev/ACTIVE_TASK.json", "go run ./cmd/devctl/independent", "git diff --check", "CI"],
"terminal_gap_states": ["BLOCKED_BY_CONFLICT"]
}