Assistant/cmd/cognitive-benchmark/cognitiveeval/main.go

package main

import (
	"context"
	"encoding/json"
	"errors"
	"flag"
	"fmt"
	"os"
	"time"

	"git.cin.su/m/Assistant/adapters/models"
	"git.cin.su/m/Assistant/contracts/capability"
	"git.cin.su/m/Assistant/contracts/cognitive"
	realityevaluation "git.cin.su/m/Assistant/tests/reality_evaluation"
)

type manifest struct {
	ArtifactID string              `json:"artifact_id"`
	Providers  []providerCandidate `json:"providers"`
}

type providerCandidate struct {
	ProviderInstanceID string           `json:"provider_instance_id"`
	ProviderKind       string           `json:"provider_kind"`
	Endpoint           string           `json:"endpoint"`
	Candidates         []modelCandidate `json:"candidates"`
}

type modelCandidate struct {
	ProviderModelID   string `json:"provider_model_id"`
	ModelDescriptorID string `json:"model_descriptor_id"`
	ModelRevision     int64  `json:"model_revision"`
	BindingID         string `json:"binding_id"`
	BindingRevision   int64  `json:"binding_revision"`
}

func main() {
	manifestPath := flag.String("manifest", "", "benchmark manifest JSON")
	suitePath := flag.String("suite", "", "cognitive evaluation suite JSON")
	outputPath := flag.String("output", "", "output artifact JSON")
	observedAt := flag.String("observed-at", "2026-08-17T12:00:00Z", "RFC3339 timestamp for reproducible artifact generation")
	timeout := flag.Duration("case-timeout", 30*time.Second, "deadline for each case")
	flag.Parse()
	if *manifestPath == "" || *suitePath == "" || *outputPath == "" || *timeout <= 0 {
		fatal(errors.New("manifest, suite, output and positive case-timeout are required"))
	}
	at, err := time.Parse(time.RFC3339, *observedAt)
	if err != nil {
		fatal(err)
	}
	input, err := os.ReadFile(*manifestPath)
	if err != nil {
		fatal(err)
	}
	var plan manifest
	if err := json.Unmarshal(input, &plan); err != nil {
		fatal(err)
	}
	if len(plan.Providers) == 0 || len(plan.Providers[0].Candidates) == 0 {
		fatal(errors.New("manifest has no candidates"))
	}
	suite, err := realityevaluation.LoadCognitiveSuite(*suitePath)
	if err != nil {
		fatal(err)
	}
	provider := plan.Providers[0]
	candidate := provider.Candidates[0]
	adapter, err := newProvider(provider)
	if err != nil {
		fatal(err)
	}
	responses := map[string]string{}
	for index, item := range suite.Cases {
		caseCtx, cancel := context.WithTimeout(context.Background(), *timeout)
		response, err := adapter.Advise(caseCtx, cognitive.AdvisoryRequest{
			Version:                 cognitive.CreatorAdvisoryV1,
			RequestID:               cognitive.RequestID(fmt.Sprintf("cognitive-eval-%d", index)),
			ActorID:                 "benchmark-actor",
			PrincipalID:             "domovoy-creator-principal",
			ScopeID:                 "benchmark-public-synthetic",
			Message:                 item.Prompt,
			Locale:                  "ru-RU",
			MaxOutputBytes:          4096,
			ProviderBindingID:       capability.CapabilityBindingID(candidate.BindingID),
			ProviderBindingRevision: candidate.BindingRevision,
			ProviderInstanceID:      capability.ProviderInstanceID(provider.ProviderInstanceID),
			ModelDescriptorID:       capability.ModelDescriptorID(candidate.ModelDescriptorID),
			ProviderModelID:         capability.ProviderModelID(candidate.ProviderModelID),
		})
		cancel()
		if err != nil {
			responses[item.ID] = "ADAPTER_ERROR: " + err.Error()
			continue
		}
		responses[item.ID] = response.AdvisoryText
	}
	artifact, err := realityevaluation.EvaluateCognitiveSuite(
		suite,
		provider.ProviderInstanceID,
		candidate.ProviderModelID,
		candidate.ModelDescriptorID,
		candidate.BindingID,
		candidate.BindingRevision,
		responses,
		at,
	)
	if err != nil {
		fatal(err)
	}
	payload, err := json.MarshalIndent(artifact, "", "  ")
	if err != nil {
		fatal(err)
	}
	payload = append(payload, '\n')
	if err := os.WriteFile(*outputPath, payload, 0o600); err != nil {
		fatal(err)
	}
	fmt.Printf("artifact=%s total_cases=%d output=%s\n", artifact.ArtifactID, artifact.TotalCases, *outputPath)
}

func newProvider(candidate providerCandidate) (models.LocalProvider, error) {
	switch candidate.ProviderKind {
	case "openai-compatible-chat-v1":
		return models.NewOpenAICompatibleLocalAdapter(candidate.Endpoint, nil)
	case "ollama-v1":
		return models.NewOllamaAdapter(candidate.Endpoint, nil)
	default:
		return nil, fmt.Errorf("unsupported provider kind %q", candidate.ProviderKind)
	}
}

func fatal(err error) {
	fmt.Fprintln(os.Stderr, err)
	os.Exit(1)
}