feat: bound benchmark advisory output
CI / test (push) Successful in 35s
CI / test (pull_request) Successful in 34s

This commit is contained in:
m
2026-08-22 22:50:20 +03:00
parent 2ed2302526
commit 775fa03ed4
3 changed files with 32 additions and 8 deletions
+7 -6
View File
@@ -98,8 +98,9 @@ func main() {
manifestPath := flag.String("manifest", "", "benchmark manifest JSON")
outputPath := flag.String("output", "", "output artifact JSON")
timeout := flag.Duration("case-timeout", 30*time.Second, "deadline for each case")
maxOutputBytes := flag.Int64("max-output-bytes", 4096, "maximum advisory output bytes per case")
flag.Parse()
if *manifestPath == "" || *outputPath == "" || *timeout <= 0 {
if *manifestPath == "" || *outputPath == "" || *timeout <= 0 || *maxOutputBytes < 1 {
fatal(errors.New("manifest, output and positive case-timeout are required"))
}
input, err := os.ReadFile(*manifestPath)
@@ -110,7 +111,7 @@ func main() {
if err := json.Unmarshal(input, &plan); err != nil {
fatal(err)
}
artifact, err := run(context.Background(), plan, *timeout, time.Now().UTC())
artifact, err := run(context.Background(), plan, *timeout, *maxOutputBytes, time.Now().UTC())
if err != nil {
fatal(err)
}
@@ -125,7 +126,7 @@ func main() {
fmt.Printf("artifact=%s models=%d output=%s\n", artifact.ArtifactID, len(artifact.Results), *outputPath)
}
func run(ctx context.Context, plan manifest, timeout time.Duration, observed time.Time) (benchmarkArtifact, error) {
func run(ctx context.Context, plan manifest, timeout time.Duration, maxOutputBytes int64, observed time.Time) (benchmarkArtifact, error) {
if plan.ArtifactID == "" || len(plan.Providers) == 0 {
return benchmarkArtifact{}, errors.New("invalid benchmark manifest")
}
@@ -142,7 +143,7 @@ func run(ctx context.Context, plan manifest, timeout time.Duration, observed tim
return provider.Candidates[i].ProviderModelID < provider.Candidates[j].ProviderModelID
})
for _, candidate := range provider.Candidates {
result, err := runModel(ctx, adapter, provider.ProviderInstanceID, candidate, timeout)
result, err := runModel(ctx, adapter, provider.ProviderInstanceID, candidate, timeout, maxOutputBytes)
if err != nil {
return benchmarkArtifact{}, err
}
@@ -169,7 +170,7 @@ func newProvider(candidate providerCandidate, timeout time.Duration) (models.Loc
}
}
func runModel(ctx context.Context, adapter models.LocalProvider, providerID string, candidate modelCandidate, timeout time.Duration) (modelResult, error) {
func runModel(ctx context.Context, adapter models.LocalProvider, providerID string, candidate modelCandidate, timeout time.Duration, maxOutputBytes int64) (modelResult, error) {
if providerID == "" || candidate.ProviderModelID == "" || candidate.ModelDescriptorID == "" || candidate.ModelRevision < 1 || candidate.BindingID == "" || candidate.BindingRevision < 1 {
return modelResult{}, errors.New("invalid benchmark candidate")
}
@@ -178,7 +179,7 @@ func runModel(ctx context.Context, adapter models.LocalProvider, providerID stri
for index, item := range corpus {
caseCtx, cancel := context.WithTimeout(ctx, timeout)
started := time.Now()
response, err := adapter.Advise(caseCtx, cognitive.AdvisoryRequest{Version: cognitive.CreatorAdvisoryV1, RequestID: cognitive.RequestID(fmt.Sprintf("bench-%s-%d", digest(candidate.BindingID)[:12], index)), ActorID: "benchmark-actor", PrincipalID: "domovoy-creator-principal", ScopeID: "benchmark-public-synthetic", Message: item.prompt, Locale: "ru-RU", MaxOutputBytes: 4096, ProviderBindingID: capability.CapabilityBindingID(candidate.BindingID), ProviderBindingRevision: candidate.BindingRevision, ProviderInstanceID: capability.ProviderInstanceID(providerID), ModelDescriptorID: capability.ModelDescriptorID(candidate.ModelDescriptorID), ProviderModelID: capability.ProviderModelID(candidate.ProviderModelID)})
response, err := adapter.Advise(caseCtx, cognitive.AdvisoryRequest{Version: cognitive.CreatorAdvisoryV1, RequestID: cognitive.RequestID(fmt.Sprintf("bench-%s-%d", digest(candidate.BindingID)[:12], index)), ActorID: "benchmark-actor", PrincipalID: "domovoy-creator-principal", ScopeID: "benchmark-public-synthetic", Message: item.prompt, Locale: "ru-RU", MaxOutputBytes: int(maxOutputBytes), ProviderBindingID: capability.CapabilityBindingID(candidate.BindingID), ProviderBindingRevision: candidate.BindingRevision, ProviderInstanceID: capability.ProviderInstanceID(providerID), ModelDescriptorID: capability.ModelDescriptorID(candidate.ModelDescriptorID), ProviderModelID: capability.ProviderModelID(candidate.ProviderModelID)})
cancel()
caseResult := caseResult{CaseID: item.id, LatencyMillis: time.Since(started).Milliseconds()}
if err != nil {