package main import ( "context" "encoding/json" "errors" "flag" "fmt" "os" "time" "git.cin.su/m/Assistant/adapters/models" "git.cin.su/m/Assistant/contracts/capability" "git.cin.su/m/Assistant/contracts/cognitive" realityevaluation "git.cin.su/m/Assistant/tests/reality_evaluation" ) type manifest struct { ArtifactID string `json:"artifact_id"` Providers []providerCandidate `json:"providers"` } type providerCandidate struct { ProviderInstanceID string `json:"provider_instance_id"` ProviderKind string `json:"provider_kind"` Endpoint string `json:"endpoint"` Candidates []modelCandidate `json:"candidates"` } type modelCandidate struct { ProviderModelID string `json:"provider_model_id"` ModelDescriptorID string `json:"model_descriptor_id"` ModelRevision int64 `json:"model_revision"` BindingID string `json:"binding_id"` BindingRevision int64 `json:"binding_revision"` } func main() { manifestPath := flag.String("manifest", "", "benchmark manifest JSON") suitePath := flag.String("suite", "", "cognitive evaluation suite JSON") outputPath := flag.String("output", "", "output artifact JSON") observedAt := flag.String("observed-at", "2026-08-17T12:00:00Z", "RFC3339 timestamp for reproducible artifact generation") timeout := flag.Duration("case-timeout", 30*time.Second, "deadline for each case") flag.Parse() if *manifestPath == "" || *suitePath == "" || *outputPath == "" || *timeout <= 0 { fatal(errors.New("manifest, suite, output and positive case-timeout are required")) } at, err := time.Parse(time.RFC3339, *observedAt) if err != nil { fatal(err) } input, err := os.ReadFile(*manifestPath) if err != nil { fatal(err) } var plan manifest if err := json.Unmarshal(input, &plan); err != nil { fatal(err) } if len(plan.Providers) == 0 || len(plan.Providers[0].Candidates) == 0 { fatal(errors.New("manifest has no candidates")) } suite, err := realityevaluation.LoadCognitiveSuite(*suitePath) if err != nil { fatal(err) } provider := plan.Providers[0] candidate := provider.Candidates[0] adapter, err := newProvider(provider) if err != nil { fatal(err) } responses := map[string]string{} for index, item := range suite.Cases { caseCtx, cancel := context.WithTimeout(context.Background(), *timeout) response, err := adapter.Advise(caseCtx, cognitive.AdvisoryRequest{ Version: cognitive.CreatorAdvisoryV1, RequestID: cognitive.RequestID(fmt.Sprintf("cognitive-eval-%d", index)), ActorID: "benchmark-actor", PrincipalID: "domovoy-creator-principal", ScopeID: "benchmark-public-synthetic", Message: item.Prompt, Locale: "ru-RU", MaxOutputBytes: 4096, ProviderBindingID: capability.CapabilityBindingID(candidate.BindingID), ProviderBindingRevision: candidate.BindingRevision, ProviderInstanceID: capability.ProviderInstanceID(provider.ProviderInstanceID), ModelDescriptorID: capability.ModelDescriptorID(candidate.ModelDescriptorID), ProviderModelID: capability.ProviderModelID(candidate.ProviderModelID), }) cancel() if err != nil { responses[item.ID] = "ADAPTER_ERROR: " + err.Error() continue } responses[item.ID] = response.AdvisoryText } artifact, err := realityevaluation.EvaluateCognitiveSuite( suite, provider.ProviderInstanceID, candidate.ProviderModelID, candidate.ModelDescriptorID, candidate.BindingID, candidate.BindingRevision, responses, at, ) if err != nil { fatal(err) } payload, err := json.MarshalIndent(artifact, "", " ") if err != nil { fatal(err) } payload = append(payload, '\n') if err := os.WriteFile(*outputPath, payload, 0o600); err != nil { fatal(err) } fmt.Printf("artifact=%s total_cases=%d output=%s\n", artifact.ArtifactID, artifact.TotalCases, *outputPath) } func newProvider(candidate providerCandidate) (models.LocalProvider, error) { switch candidate.ProviderKind { case "openai-compatible-chat-v1": return models.NewOpenAICompatibleLocalAdapter(candidate.Endpoint, nil) case "ollama-v1": return models.NewOllamaAdapter(candidate.Endpoint, nil) default: return nil, fmt.Errorf("unsupported provider kind %q", candidate.ProviderKind) } } func fatal(err error) { fmt.Fprintln(os.Stderr, err) os.Exit(1) }