Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
 m
|
775fa03ed4 | ||
|
m
|
2ed2302526 | ||
|
m
|
46538ba1e9 | ||
|
m
|
5960844fcb |
@@ -98,8 +98,9 @@ func main() {
|
||||
manifestPath := flag.String("manifest", "", "benchmark manifest JSON")
|
||||
outputPath := flag.String("output", "", "output artifact JSON")
|
||||
timeout := flag.Duration("case-timeout", 30*time.Second, "deadline for each case")
|
||||
maxOutputBytes := flag.Int64("max-output-bytes", 4096, "maximum advisory output bytes per case")
|
||||
flag.Parse()
|
||||
if *manifestPath == "" || *outputPath == "" || *timeout <= 0 {
|
||||
if *manifestPath == "" || *outputPath == "" || *timeout <= 0 || *maxOutputBytes < 1 {
|
||||
fatal(errors.New("manifest, output and positive case-timeout are required"))
|
||||
}
|
||||
input, err := os.ReadFile(*manifestPath)
|
||||
@@ -110,7 +111,7 @@ func main() {
|
||||
if err := json.Unmarshal(input, &plan); err != nil {
|
||||
fatal(err)
|
||||
}
|
||||
artifact, err := run(context.Background(), plan, *timeout, time.Now().UTC())
|
||||
artifact, err := run(context.Background(), plan, *timeout, *maxOutputBytes, time.Now().UTC())
|
||||
if err != nil {
|
||||
fatal(err)
|
||||
}
|
||||
@@ -125,7 +126,7 @@ func main() {
|
||||
fmt.Printf("artifact=%s models=%d output=%s\n", artifact.ArtifactID, len(artifact.Results), *outputPath)
|
||||
}
|
||||
|
||||
func run(ctx context.Context, plan manifest, timeout time.Duration, observed time.Time) (benchmarkArtifact, error) {
|
||||
func run(ctx context.Context, plan manifest, timeout time.Duration, maxOutputBytes int64, observed time.Time) (benchmarkArtifact, error) {
|
||||
if plan.ArtifactID == "" || len(plan.Providers) == 0 {
|
||||
return benchmarkArtifact{}, errors.New("invalid benchmark manifest")
|
||||
}
|
||||
@@ -142,7 +143,7 @@ func run(ctx context.Context, plan manifest, timeout time.Duration, observed tim
|
||||
return provider.Candidates[i].ProviderModelID < provider.Candidates[j].ProviderModelID
|
||||
})
|
||||
for _, candidate := range provider.Candidates {
|
||||
result, err := runModel(ctx, adapter, provider.ProviderInstanceID, candidate, timeout)
|
||||
result, err := runModel(ctx, adapter, provider.ProviderInstanceID, candidate, timeout, maxOutputBytes)
|
||||
if err != nil {
|
||||
return benchmarkArtifact{}, err
|
||||
}
|
||||
@@ -169,7 +170,7 @@ func newProvider(candidate providerCandidate, timeout time.Duration) (models.Loc
|
||||
}
|
||||
}
|
||||
|
||||
func runModel(ctx context.Context, adapter models.LocalProvider, providerID string, candidate modelCandidate, timeout time.Duration) (modelResult, error) {
|
||||
func runModel(ctx context.Context, adapter models.LocalProvider, providerID string, candidate modelCandidate, timeout time.Duration, maxOutputBytes int64) (modelResult, error) {
|
||||
if providerID == "" || candidate.ProviderModelID == "" || candidate.ModelDescriptorID == "" || candidate.ModelRevision < 1 || candidate.BindingID == "" || candidate.BindingRevision < 1 {
|
||||
return modelResult{}, errors.New("invalid benchmark candidate")
|
||||
}
|
||||
@@ -178,7 +179,7 @@ func runModel(ctx context.Context, adapter models.LocalProvider, providerID stri
|
||||
for index, item := range corpus {
|
||||
caseCtx, cancel := context.WithTimeout(ctx, timeout)
|
||||
started := time.Now()
|
||||
response, err := adapter.Advise(caseCtx, cognitive.AdvisoryRequest{Version: cognitive.CreatorAdvisoryV1, RequestID: cognitive.RequestID(fmt.Sprintf("bench-%s-%d", digest(candidate.BindingID)[:12], index)), ActorID: "benchmark-actor", PrincipalID: "domovoy-creator-principal", ScopeID: "benchmark-public-synthetic", Message: item.prompt, Locale: "ru-RU", MaxOutputBytes: 4096, ProviderBindingID: capability.CapabilityBindingID(candidate.BindingID), ProviderBindingRevision: candidate.BindingRevision, ProviderInstanceID: capability.ProviderInstanceID(providerID), ModelDescriptorID: capability.ModelDescriptorID(candidate.ModelDescriptorID), ProviderModelID: capability.ProviderModelID(candidate.ProviderModelID)})
|
||||
response, err := adapter.Advise(caseCtx, cognitive.AdvisoryRequest{Version: cognitive.CreatorAdvisoryV1, RequestID: cognitive.RequestID(fmt.Sprintf("bench-%s-%d", digest(candidate.BindingID)[:12], index)), ActorID: "benchmark-actor", PrincipalID: "domovoy-creator-principal", ScopeID: "benchmark-public-synthetic", Message: item.prompt, Locale: "ru-RU", MaxOutputBytes: int(maxOutputBytes), ProviderBindingID: capability.CapabilityBindingID(candidate.BindingID), ProviderBindingRevision: candidate.BindingRevision, ProviderInstanceID: capability.ProviderInstanceID(providerID), ModelDescriptorID: capability.ModelDescriptorID(candidate.ModelDescriptorID), ProviderModelID: capability.ProviderModelID(candidate.ProviderModelID)})
|
||||
cancel()
|
||||
caseResult := caseResult{CaseID: item.id, LatencyMillis: time.Since(started).Milliseconds()}
|
||||
if err != nil {
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
{
|
||||
"task_file": "dev/tasks/DEV-000178-heavy-qualification-refresh.json",
|
||||
"task_id": "DEV-000178",
|
||||
"task_file": "dev/tasks/DEV-000180-bounded-medium-qualification-output.json",
|
||||
"task_id": "DEV-000180",
|
||||
"task_revision": 1
|
||||
}
|
||||
|
||||
@@ -0,0 +1,23 @@
|
||||
{
|
||||
"task_id": "DEV-000179",
|
||||
"task_revision": 1,
|
||||
"architecture_revision": "ADR-0018",
|
||||
"base_repo_revision": "5960844fcb7b9bfba9c24e18a5d15b569d0d3a08",
|
||||
"risk_tier": "E",
|
||||
"intent": "Synchronize audit projections with current production runtime evidence after the Heavy qualification deployment.",
|
||||
"requirement_ids": ["AUDIT-COMPLETION-TRUTH", "WP-301", "WP-400", "WP-102", "WP-600"],
|
||||
"allowed_change_scope": {"modules": [], "paths": ["docs/evidence/audit-runtime-status-2026-08-22-v4.json", "docs/evidence/audit-completion-matrix-2026-08-22-v1.json", "docs/PROJECT_STATUS.md", "dev/ACTIVE_TASK.json", "dev/tasks/DEV-000179-runtime-status-consolidation.json"]},
|
||||
"forbidden_change_scope": {"paths": ["architecture/**", "security/**", "cmd/**", "modules/**", "contracts/**", "adapters/**", "database/**", "deploy/**", ".gitea/**"]},
|
||||
"allowed_architecture_delta": {"new_cross_module_dependencies": 0, "new_canonical_owners": 0, "new_privileges": 0, "new_third_party_dependencies": 0, "new_database_owner_roles": 0},
|
||||
"contracts": {"consume": ["live health/readiness/status endpoints"], "may_change": []},
|
||||
"semantic_owner": "none; audit projection",
|
||||
"canonical_state": [],
|
||||
"state_transitions": ["stale audit projection -> current production runtime evidence"],
|
||||
"side_effects_and_privileges": ["read-only live probes"],
|
||||
"persistence_and_migrations": ["immutable JSON evidence only"],
|
||||
"failure_retry_restart": ["not applicable"],
|
||||
"assumptions": ["production revision is read from creator-web runtime environment", "runtime readiness is not treated as capability qualification"],
|
||||
"acceptance_scenarios": ["public health is current", "research/work-graph readiness is current", "agent recovery-only state is explicit", "fleet tiers and selected tier are current", "no secrets appear"],
|
||||
"required_tests": ["JSON validation", "go test ./...", "go vet ./...", "go run ./cmd/devctl preflight -active dev/ACTIVE_TASK.json", "go run ./cmd/devctl check-diff -active dev/ACTIVE_TASK.json", "go run ./cmd/devctl check-architecture -active dev/ACTIVE_TASK.json", "go run ./cmd/devctl/independent", "git diff --check", "CI"],
|
||||
"terminal_gap_states": ["BLOCKED_BY_MISSING_CAPABILITY", "BLOCKED_BY_CONFLICT"]
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
{
|
||||
"task_id": "DEV-000180",
|
||||
"task_revision": 1,
|
||||
"architecture_revision": "ADR-0018",
|
||||
"base_repo_revision": "2ed2302526f6915c9ddef852ed17a8c47f716cc2",
|
||||
"risk_tier": "E",
|
||||
"intent": "Allow the existing synthetic qualification benchmark to bound maximum output bytes per case for slow local providers.",
|
||||
"requirement_ids": ["WP-200", "AUDIT-COMPLETION-TRUTH"],
|
||||
"allowed_change_scope": {"modules": ["cognitive benchmark"], "paths": ["cmd/cognitive-benchmark/main.go", "dev/ACTIVE_TASK.json", "dev/tasks/DEV-000180-bounded-medium-qualification-output.json"]},
|
||||
"forbidden_change_scope": {"paths": ["architecture/**", "security/**", "cmd/cognitive-worker/**", "modules/**", "contracts/**", "adapters/**", "database/**", "deploy/**", ".gitea/**"]},
|
||||
"allowed_architecture_delta": {"new_cross_module_dependencies": 0, "new_canonical_owners": 0, "new_privileges": 0, "new_third_party_dependencies": 0, "new_database_owner_roles": 0},
|
||||
"contracts": {"consume": ["existing cognitive advisory request contract"], "may_change": ["benchmark CLI parameter only"]},
|
||||
"semantic_owner": "cognitive benchmark",
|
||||
"canonical_state": [],
|
||||
"state_transitions": ["fixed benchmark output cap -> explicit bounded qualification run"],
|
||||
"side_effects_and_privileges": ["none"],
|
||||
"persistence_and_migrations": ["none"],
|
||||
"failure_retry_restart": ["non-positive output cap rejected; default preserves previous 4096-byte behavior"],
|
||||
"assumptions": ["bounded output is valid only for synthetic exact-answer corpus and does not alter production provider configuration"],
|
||||
"acceptance_scenarios": ["default behavior remains unchanged", "positive output cap is forwarded to advisory request", "invalid cap fails closed", "no provider boundary is bypassed"],
|
||||
"required_tests": ["go test ./...", "go vet ./...", "go run ./cmd/devctl preflight -active dev/ACTIVE_TASK.json", "go run ./cmd/devctl check-diff -active dev/ACTIVE_TASK.json", "go run ./cmd/devctl check-architecture -active dev/ACTIVE_TASK.json", "go run ./cmd/devctl/independent", "git diff --check", "CI"],
|
||||
"terminal_gap_states": ["BLOCKED_BY_CONFLICT"]
|
||||
}
|
||||
@@ -2,12 +2,12 @@
|
||||
|
||||
## Current audit delta — 2026-08-22
|
||||
|
||||
- `CURRENT_PRODUCTION_REVISION = 64642f4baed28f9051efe6a269769c5838b547e4`; public `/healthz` is healthy.
|
||||
- `CURRENT_PRODUCTION_REVISION = 5960844fcb7b9bfba9c24e18a5d15b569d0d3a08`; public `/healthz` is healthy. Current runtime evidence: `docs/evidence/audit-runtime-status-2026-08-22-v4.json`.
|
||||
- `FAST_PROVIDER_STATUS = LIVE_SMOKE_SUPPORTED`: local CPU gateway authentication is configured through a protected file reference; four bounded local-only advisory cases completed through cognitive-worker. Evidence: `docs/evidence/wp-601-fast-gateway-live-smoke-2026-08-22-v1.json`.
|
||||
- `MEDIUM_PROVIDER_STATUS = LIVE_AVAILABLE_UNQUALIFIED`; after the adapter deadline was extended, qualification still timed out on all four cases at the full 45-second case deadline. Evidence: `docs/evidence/wp-200-medium-qualification-2026-08-22-v2.json`; no activation was attempted.
|
||||
- `HEAVY_PROVIDER_STATUS = LIVE_QUALIFIED_UNACTIVATED`; discovery refreshed the stale GPU registry and current `gemma4:26b` passed all four bounded cases (`SUPPORTED`, 1000‰ quality, 0‰ critical failures). Evidence: `docs/evidence/wp-200-heavy-qualification-2026-08-22-v2.json`; binding remains INACTIVE and no activation was attempted.
|
||||
- `AGENT_EXECUTION_STATUS = RECOVERY_ONLY`; optional `AGENT_EXECUTION_BRIDGE_URL` is wired into production compose but remains unset, and no Codex/bridge runtime is deployed. Durable recovery and status endpoints remain active.
|
||||
- `FLEET_EXECUTION_EVIDENCE = docs/evidence/wp-200-wp-301-fleet-execution-status-2026-08-22-v3.json`; FAST, MEDIUM and LIFEBOAT are catalog/readiness-ready, Heavy is unavailable by timeout, and current selection is MEDIUM for preferred HEAVY; MEDIUM remains unqualified after `DEV-000174`.
|
||||
- `FLEET_EXECUTION_EVIDENCE = docs/evidence/audit-runtime-status-2026-08-22-v4.json`; FAST, MEDIUM, HEAVY and LIFEBOAT are currently readiness-ready, selected tier is HEAVY, Heavy gemma4:26b is qualified but inactive, and Medium remains unqualified.
|
||||
- `BACKUP_RESTORE_EVIDENCE = docs/evidence/wp-002-postgres-restore-2026-08-22-v1.json`; latest backup checksum and isolated restore verified with 11 schemas and 60 tables.
|
||||
- `UI_AUTHENTICATED_ACCEPTANCE = PENDING`; public login UI is reachable, but no authenticated browser session was available for cabinet visual acceptance.
|
||||
- `SECURITY_NOTE = one production executor secret was rotated after an accidental diagnostic disclosure; values are not retained in repository evidence.`
|
||||
|
||||
@@ -1,13 +1,13 @@
|
||||
{
|
||||
"evidence_id": "audit-completion-matrix-2026-08-22-v1",
|
||||
"captured_at": "2026-08-22",
|
||||
"production_revision": "64642f4baed28f9051efe6a269769c5838b547e4",
|
||||
"production_revision": "5960844fcb7b9bfba9c24e18a5d15b569d0d3a08",
|
||||
"objective": "Dovести оставшиеся пункты аудита Domovoy до проверяемого состояния",
|
||||
"matrix": [
|
||||
{"area":"Agent Execution runtime and recovery","status":"PARTIAL_EXTERNAL_GAP","repository_proof":["dev/tasks/DEV-000145-agent-execution-bridge-runtime.json","adapters/agentbridge/http.go","cmd/agent-worker/main.go"],"production_proof":["docs/evidence/wp-200-wp-301-fleet-execution-status-2026-08-22-v1.json"],"missing":"independently operated bridge endpoint is not configured; live worker is recovery-only"},
|
||||
{"area":"Multi-Agent Work Graph","status":"REPOSITORY_AND_RUNTIME_FOUNDATION_VERIFIED","repository_proof":["dev/tasks/DEV-000131-work-graph-harness.json","dev/tasks/DEV-000146-work-graph-runtime.json","tests/agent_execution/graph_runtime_test.go"],"production_proof":["docs/PROJECT_STATUS.md"],"missing":"no claim of external agent execution fan-out is made while bridge is absent"},
|
||||
{"area":"Truth/Research runtime","status":"REPOSITORY_AND_RUNTIME_FOUNDATION_VERIFIED","repository_proof":["dev/tasks/DEV-000134-truth-bundle-compiler.json","dev/tasks/DEV-000143-research-runtime-orchestration.json","tests/research_runtime"],"production_proof":["docs/PROJECT_STATUS.md"],"missing":"authenticated end-to-end research acceptance is not claimed here"},
|
||||
{"area":"Learning reflection loop","status":"REPOSITORY_AND_RUNTIME_FOUNDATION_VERIFIED","repository_proof":["dev/tasks/DEV-000135-reflection-worker.json","cmd/reflection-worker/main.go","tests/research_contracts/reflection_worker_sql_test.go"],"production_proof":["docs/PROJECT_STATUS.md"],"missing":"no claim of fully closed historical candidate backlog"},
|
||||
{"area":"Agent Execution runtime and recovery","status":"PARTIAL_EXTERNAL_GAP","repository_proof":["dev/tasks/DEV-000145-agent-execution-bridge-runtime.json","adapters/agentbridge/http.go","cmd/agent-worker/main.go"],"production_proof":["docs/evidence/audit-runtime-status-2026-08-22-v4.json"],"missing":"independently operated bridge endpoint is not configured; live worker is recovery-only"},
|
||||
{"area":"Multi-Agent Work Graph","status":"REPOSITORY_AND_RUNTIME_FOUNDATION_VERIFIED","repository_proof":["dev/tasks/DEV-000131-work-graph-harness.json","dev/tasks/DEV-000146-work-graph-runtime.json","tests/agent_execution/graph_runtime_test.go"],"production_proof":["docs/evidence/audit-runtime-status-2026-08-22-v4.json"],"missing":"work-graph runtime is ready, but no claim of external agent execution fan-out is made while bridge is absent"},
|
||||
{"area":"Truth/Research runtime","status":"REPOSITORY_AND_RUNTIME_FOUNDATION_VERIFIED","repository_proof":["dev/tasks/DEV-000134-truth-bundle-compiler.json","dev/tasks/DEV-000143-research-runtime-orchestration.json","tests/research_runtime"],"production_proof":["docs/evidence/audit-runtime-status-2026-08-22-v4.json"],"missing":"research runtime readiness is live, but authenticated end-to-end research acceptance is not claimed here"},
|
||||
{"area":"Learning reflection loop","status":"REPOSITORY_AND_RUNTIME_FOUNDATION_VERIFIED","repository_proof":["dev/tasks/DEV-000135-reflection-worker.json","cmd/reflection-worker/main.go","tests/research_contracts/reflection_worker_sql_test.go"],"production_proof":["docs/evidence/audit-runtime-status-2026-08-22-v4.json"],"missing":"reflection worker process is running, but no claim of fully closed historical candidate backlog"},
|
||||
{"area":"Model fleet/failover","status":"PARTIAL_EXTERNAL_GAP","repository_proof":["dev/tasks/DEV-000144-fleet-readiness-runtime.json","dev/tasks/DEV-000168-configurable-medium-fleet-slot.json","dev/tasks/DEV-000169-optional-medium-readiness.json","dev/tasks/DEV-000171-connect-existing-medium-runtime.json","dev/tasks/DEV-000172-fleet-failover-priority.json","dev/tasks/DEV-000174-medium-qualification-evidence.json","dev/tasks/DEV-000176-medium-qualification-retest.json","dev/tasks/DEV-000177-heavy-qualification-evidence.json","dev/tasks/DEV-000178-heavy-qualification-refresh.json","cmd/cognitive-worker/fleet_status.go","modules/capability/domain/fleet.go"],"production_proof":["docs/evidence/wp-200-wp-301-fleet-execution-status-2026-08-22-v3.json","docs/evidence/wp-200-medium-qualification-2026-08-22-v2.json","docs/evidence/wp-200-heavy-qualification-2026-08-22-v2.json"],"missing":"Heavy gemma4:26b is qualified but remains INACTIVE pending authenticated lifecycle promotion; Medium still timed out at 45 seconds, so multi-provider active invocation and full fleet promotion evidence remain unavailable"},
|
||||
{"area":"Governed self-development","status":"REPOSITORY_FOUNDATION_VERIFIED","repository_proof":["dev/tasks/DEV-000137-governed-self-development-gate.json","tests/semantic_evolution"],"production_proof":["docs/PROJECT_STATUS.md"],"missing":"real bridge-backed self-development execution is not claimed"},
|
||||
{"area":"UI/operations acceptance","status":"PARTIAL_EXTERNAL_GAP","repository_proof":["dev/tasks/DEV-000138-ui-operations-acceptance-contract.json","tests/creator_web/cabinet_acceptance_contract_test.go","dev/tasks/DEV-000150-agent-operations-ui.json","dev/tasks/DEV-000151-agent-worker-status-ui.json"],"production_proof":["docs/PROJECT_STATUS.md"],"missing":"authenticated browser visual acceptance is pending because no signed-in session is available"}
|
||||
|
||||
@@ -0,0 +1,15 @@
|
||||
{
|
||||
"evidence_id": "audit-runtime-status-2026-08-22-v4",
|
||||
"observed_at": "2026-08-22T19:42:00Z",
|
||||
"production_revision": "5960844fcb7b9bfba9c24e18a5d15b569d0d3a08",
|
||||
"public_health": {"endpoint": "https://assistant.cin.su/healthz", "status": 200, "runtime": "creator-web", "state": "ok"},
|
||||
"runtime_readiness": {
|
||||
"research": {"endpoint": "http://research-runtime:8087/readyz", "status": "ready", "checks": ["epistemics-postgres"]},
|
||||
"work_graph": {"endpoint": "http://work-graph-runtime:8088/readyz", "status": "ready", "checks": ["work-postgres"]},
|
||||
"reflection_worker": {"container": "assistant-reflection-worker-1", "state": "running", "restart_count": 0, "http_readiness": "not_exposed"},
|
||||
"agent_worker": {"endpoint": "http://agent-worker:8090/status", "executor_configured": false, "mode": "recovery-only", "ready": false, "reason": "AGENT_EXECUTION_BRIDGE_URL is not configured"}
|
||||
},
|
||||
"fleet_readiness": {"ready_tiers": ["FAST", "MEDIUM", "HEAVY", "LIFEBOAT"], "preferred_tier": "HEAVY", "selected_tier": "HEAVY", "fallback_tier": "LIFEBOAT"},
|
||||
"interpretation": "Runtime readiness and durable worker process state are live evidence only; readiness does not imply model qualification, bridge execution, lifecycle activation or authenticated UI acceptance.",
|
||||
"secrets": "No secret values included."
|
||||
}
|
||||
Reference in New Issue
Block a user