Compare commits

...
Author SHA1 Message Date
m 6e0d1a2279 docs: record live medium failover evidence
CI / test (push) Successful in 35s
CI / test (pull_request) Successful in 34s
2026-08-22 21:42:01 +03:00
m 64642f4bae Merge pull request 'fix: prefer available fleet tier before lifeboat' (#145) from codex/fleet-failover-priority into main
CI / test (push) Successful in 44s
2026-08-22 18:36:44 +00:00
m 2f9e9bdc9f fix: prefer available fleet tier before lifeboat
CI / test (push) Successful in 33s
CI / test (pull_request) Successful in 33s
2026-08-22 21:35:16 +03:00
m 2c7a85779b Merge pull request 'feat: connect existing medium model runtime' (#144) from codex/connect-medium-runtime into main
CI / test (push) Successful in 42s
2026-08-22 18:30:30 +00:00
m b4c3027e6d feat: connect existing medium model runtime
CI / test (push) Successful in 35s
CI / test (pull_request) Successful in 33s
2026-08-22 21:29:02 +03:00
m 503fe84074 Merge pull request 'docs: refresh fleet audit evidence' (#143) from codex/refresh-fleet-evidence into main
CI / test (push) Successful in 42s
2026-08-22 18:21:37 +00:00
11 changed files with 163 additions and 12 deletions
+2 -2
View File
@@ -9,7 +9,7 @@ import (
"git.cin.su/m/Assistant/contracts/capability"
)
func TestFleetReadinessSelectsLifeboatWhenPrimaryTiersUnavailable(t *testing.T) {
func TestFleetReadinessSelectsBestAvailablePrimaryTierBeforeLifeboat(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusOK) }))
defer server.Close()
handler := fleetReadinessHandler(server.Client(), capability.FleetTierHeavy, []fleetEndpoint{
@@ -21,7 +21,7 @@ func TestFleetReadinessSelectsLifeboatWhenPrimaryTiersUnavailable(t *testing.T)
recorder := httptest.NewRecorder()
request := httptest.NewRequest(http.MethodGet, "/v1/fleet/readiness", nil)
handler.ServeHTTP(recorder, request)
if recorder.Code != http.StatusOK || !strings.Contains(recorder.Body.String(), `"selected_tier":"LIFEBOAT"`) || !strings.Contains(recorder.Body.String(), `"tier":"MEDIUM"`) {
if recorder.Code != http.StatusOK || !strings.Contains(recorder.Body.String(), `"selected_tier":"FAST"`) || !strings.Contains(recorder.Body.String(), `"tier":"MEDIUM"`) {
t.Fatalf("status=%d body=%s", recorder.Code, recorder.Body.String())
}
}
+7 -1
View File
@@ -214,7 +214,7 @@ services:
LOCAL_CPU_PROVIDER_URL: http://docker.cin.su:18082/v1
LOCAL_CPU_PROVIDER_API_KEY_FILE: ${LOCAL_CPU_PROVIDER_API_KEY_FILE:-}
LOCAL_CPU_PROVIDER_CREDENTIAL_REF: ${LOCAL_CPU_PROVIDER_CREDENTIAL_REF:-credential://domovoy/local-cpu/provider-api-key/v1}
LOCAL_MEDIUM_PROVIDER_URL: ${LOCAL_MEDIUM_PROVIDER_URL:-}
LOCAL_MEDIUM_PROVIDER_URL: ${LOCAL_MEDIUM_PROVIDER_URL:-http://llm-local-api:8080/v1}
LOCAL_MEDIUM_PROVIDER_API_KEY_FILE: ${LOCAL_MEDIUM_PROVIDER_API_KEY_FILE:-}
LOCAL_MEDIUM_PROVIDER_CREDENTIAL_REF: ${LOCAL_MEDIUM_PROVIDER_CREDENTIAL_REF:-credential://domovoy/local-medium/provider-api-key/v1}
LOCAL_LIFEBOAT_PROVIDER_URL: http://cognitive-lifeboat:8086/v1
@@ -224,6 +224,7 @@ services:
depends_on: [core-runtime, cognitive-lifeboat]
volumes:
- assistant_core_secrets:/run/secrets:ro
networks: [default, llm-local]
cognitive-lifeboat:
image: ghcr.io/ggml-org/llama.cpp:server
@@ -290,3 +291,8 @@ volumes:
external: true
assistant_research_sources:
name: assistant_research_sources
networks:
llm-local:
external: true
name: llm-local_default
+2 -2
View File
@@ -1,5 +1,5 @@
{
"task_file": "dev/tasks/DEV-000170-refresh-fleet-audit-evidence.json",
"task_id": "DEV-000170",
"task_file": "dev/tasks/DEV-000173-refresh-medium-failover-evidence.json",
"task_id": "DEV-000173",
"task_revision": 1
}
@@ -0,0 +1,23 @@
{
"task_id": "DEV-000171",
"task_revision": 1,
"architecture_revision": "ADR-0018",
"base_repo_revision": "503fe840741ac99499e06bd93df977acf1baad41",
"risk_tier": "E",
"intent": "Connect cognitive-worker to the already-operated llm-local CPU model runtime as the configured MEDIUM fleet member through an explicit external Docker network.",
"requirement_ids": ["WP-200", "WP-301", "Model fleet/failover"],
"allowed_change_scope": {"modules": ["deployment-topology", "cognitive-worker"], "paths": ["deploy/docker-compose.prod.yml", "dev/ACTIVE_TASK.json", "dev/tasks/DEV-000171-connect-existing-medium-runtime.json"]},
"forbidden_change_scope": {"paths": ["architecture/**", "security/**", "database/**", "contracts/**", "adapters/**", ".gitea/**"]},
"allowed_architecture_delta": {"new_cross_module_dependencies": 0, "new_canonical_owners": 0, "new_privileges": 1, "new_third_party_dependencies": 0, "new_database_owner_roles": 0},
"contracts": {"consume": ["existing OpenAI-compatible local provider adapter", "existing MEDIUM fleet readiness contract", "existing llm-local Docker network"], "may_change": []},
"semantic_owner": "deployment topology/cognitive-worker registration",
"canonical_state": [],
"state_transitions": ["MEDIUM unconfigured -> live provider discovery/readiness", "preferred HEAVY unavailable -> highest available fallback selection"],
"side_effects_and_privileges": ["one explicit read-only network attachment from cognitive-worker to existing llm-local runtime; no secret or database privilege added"],
"persistence_and_migrations": ["none"],
"failure_retry_restart": ["if external llm-local network is absent deployment fails closed; provider timeout remains bounded; Lifeboat remains fallback"],
"assumptions": ["production llm-local_default network and llm-local-api are independently operated and present", "llm-local-api exposes OpenAI-compatible /v1/models and chat routes on port 8080"],
"acceptance_scenarios": ["compose config validates with external network declaration", "cognitive-worker resolves llm-local-api", "MEDIUM fleet member is AVAILABLE live", "runtime /readyz remains 200", "Lifeboat fallback remains available", "no secret value is added"],
"required_tests": ["docker compose config", "go test ./...", "go vet ./...", "preflight", "check-diff", "check-architecture", "independent", "git diff --check", "CI", "production /readyz and fleet readiness"],
"terminal_gap_states": ["BLOCKED_BY_MISSING_CAPABILITY", "BLOCKED_BY_CONFLICT"]
}
@@ -0,0 +1,23 @@
{
"task_id": "DEV-000172",
"task_revision": 1,
"architecture_revision": "ADR-0018",
"base_repo_revision": "2c7a85779b530e3147fbf47b71742759cc010d58",
"risk_tier": "E",
"intent": "Make fleet failover choose the best available non-Lifeboat tier below the preferred tier before using Lifeboat.",
"requirement_ids": ["WP-301", "Model fleet/failover"],
"allowed_change_scope": {"modules": ["capability", "cognitive-worker"], "paths": ["modules/capability/domain/fleet.go", "modules/capability/domain/fleet_test.go", "cmd/cognitive-worker/fleet_status_test.go", "dev/ACTIVE_TASK.json", "dev/tasks/DEV-000172-fleet-failover-priority.json"]},
"forbidden_change_scope": {"paths": ["architecture/**", "security/**", "database/**", "deploy/**", "contracts/**", "adapters/**", ".gitea/**"]},
"allowed_architecture_delta": {"new_cross_module_dependencies": 0, "new_canonical_owners": 0, "new_privileges": 0, "new_third_party_dependencies": 0, "new_database_owner_roles": 0},
"contracts": {"consume": ["existing FleetReadiness contract", "existing FleetTier selection contract"], "may_change": []},
"semantic_owner": "capability",
"canonical_state": [],
"state_transitions": ["preferred unavailable -> next lower ready tier -> Lifeboat fallback"],
"side_effects_and_privileges": ["none"],
"persistence_and_migrations": ["none"],
"failure_retry_restart": ["selection remains deterministic from observed readiness; invalid preferred tier fails closed"],
"assumptions": ["HEAVY > MEDIUM > FAST is the quality/capability fallback order; LIFEBOAT is last-resort recovery"],
"acceptance_scenarios": ["HEAVY unavailable and MEDIUM ready selects MEDIUM", "HEAVY and MEDIUM unavailable and FAST ready selects FAST", "all primary tiers unavailable selects Lifeboat", "invalid preferred tier is rejected"],
"required_tests": ["go test ./...", "go vet ./...", "preflight", "check-diff", "check-architecture", "independent", "git diff --check", "CI", "production fleet readiness"],
"terminal_gap_states": ["BLOCKED_BY_MISSING_CAPABILITY", "BLOCKED_BY_CONFLICT"]
}
@@ -0,0 +1,23 @@
{
"task_id": "DEV-000173",
"task_revision": 1,
"architecture_revision": "ADR-0018",
"base_repo_revision": "64642f4baed28f9051efe6a269769c5838b547e4",
"risk_tier": "E",
"intent": "Refresh audit and production evidence after live MEDIUM runtime availability and deterministic fleet failover priority are deployed.",
"requirement_ids": ["AUDIT-COMPLETION-TRUTH", "WP-200", "WP-301"],
"allowed_change_scope": {"modules": [], "paths": ["docs/evidence/audit-completion-matrix-2026-08-22-v1.json", "docs/evidence/wp-200-wp-301-fleet-execution-status-2026-08-22-v3.json", "docs/PROJECT_STATUS.md", "dev/ACTIVE_TASK.json", "dev/tasks/DEV-000173-refresh-medium-failover-evidence.json"]},
"forbidden_change_scope": {"paths": ["architecture/**", "security/**", "cmd/**", "modules/**", "contracts/**", "adapters/**", "database/**", "deploy/**", ".gitea/**"]},
"allowed_architecture_delta": {"new_cross_module_dependencies": 0, "new_canonical_owners": 0, "new_privileges": 0, "new_third_party_dependencies": 0, "new_database_owner_roles": 0},
"contracts": {"consume": ["live fleet readiness evidence", "audit completion matrix"], "may_change": []},
"semantic_owner": "none; audit evidence projection",
"canonical_state": [],
"state_transitions": ["stale evidence -> current MEDIUM/failover evidence"],
"side_effects_and_privileges": ["none"],
"persistence_and_migrations": ["immutable JSON evidence artifacts only"],
"failure_retry_restart": ["not applicable"],
"assumptions": ["live probes were captured after production deployment of 64642f4b", "availability is not interpreted as qualification or activation"],
"acceptance_scenarios": ["production revision is current", "MEDIUM model catalog and readiness are recorded", "preferred HEAVY falls back to MEDIUM", "remaining bridge/GPU/browser gaps remain explicit", "no secret value appears"],
"required_tests": ["JSON validation", "go test ./...", "go vet ./...", "preflight", "check-diff", "check-architecture", "independent", "git diff --check", "CI"],
"terminal_gap_states": ["BLOCKED_BY_MISSING_CAPABILITY", "BLOCKED_BY_CONFLICT"]
}
+3 -3
View File
@@ -2,12 +2,12 @@
## Current audit delta — 2026-08-22
- `CURRENT_PRODUCTION_REVISION = 51427a246e1ccc115a6c2640790d9220beca95e9`; public `/healthz` is healthy.
- `CURRENT_PRODUCTION_REVISION = 64642f4baed28f9051efe6a269769c5838b547e4`; public `/healthz` is healthy.
- `FAST_PROVIDER_STATUS = LIVE_SMOKE_SUPPORTED`: local CPU gateway authentication is configured through a protected file reference; four bounded local-only advisory cases completed through cognitive-worker. Evidence: `docs/evidence/wp-601-fast-gateway-live-smoke-2026-08-22-v1.json`.
- `MEDIUM_PROVIDER_STATUS = CONFIGURABLE_NOT_CONFIGURED`; the optional OpenAI-compatible MEDIUM slot, protected CredentialRef file path, discovery and readiness wiring are deployed; no medium runtime is supplied in production.
- `MEDIUM_PROVIDER_STATUS = LIVE_AVAILABLE`; existing llm-local Qwen3.6-27B CPU runtime is reachable through the explicit external Docker network and reports a loaded model; qualification/activation remains separate.
- `HEAVY_PROVIDER_STATUS = UNAVAILABLE`; GPU endpoint `192.168.220.91:11434` timed out during live discovery.
- `AGENT_EXECUTION_STATUS = RECOVERY_ONLY`; optional `AGENT_EXECUTION_BRIDGE_URL` is wired into production compose but remains unset, and no Codex/bridge runtime is deployed. Durable recovery and status endpoints remain active.
- `FLEET_EXECUTION_EVIDENCE = docs/evidence/wp-200-wp-301-fleet-execution-status-2026-08-22-v2.json`; FAST and LIFEBOAT are ready, MEDIUM is NOT_CONFIGURED, Heavy is unavailable by timeout, and current selection is LIFEBOAT for preferred HEAVY.
- `FLEET_EXECUTION_EVIDENCE = docs/evidence/wp-200-wp-301-fleet-execution-status-2026-08-22-v3.json`; FAST, MEDIUM and LIFEBOAT are ready, Heavy is unavailable by timeout, and current selection is MEDIUM for preferred HEAVY.
- `BACKUP_RESTORE_EVIDENCE = docs/evidence/wp-002-postgres-restore-2026-08-22-v1.json`; latest backup checksum and isolated restore verified with 11 schemas and 60 tables.
- `UI_AUTHENTICATED_ACCEPTANCE = PENDING`; public login UI is reachable, but no authenticated browser session was available for cabinet visual acceptance.
- `SECURITY_NOTE = one production executor secret was rotated after an accidental diagnostic disclosure; values are not retained in repository evidence.`
@@ -1,14 +1,14 @@
{
"evidence_id": "audit-completion-matrix-2026-08-22-v1",
"captured_at": "2026-08-22",
"production_revision": "51427a246e1ccc115a6c2640790d9220beca95e9",
"production_revision": "64642f4baed28f9051efe6a269769c5838b547e4",
"objective": "Dovести оставшиеся пункты аудита Domovoy до проверяемого состояния",
"matrix": [
{"area":"Agent Execution runtime and recovery","status":"PARTIAL_EXTERNAL_GAP","repository_proof":["dev/tasks/DEV-000145-agent-execution-bridge-runtime.json","adapters/agentbridge/http.go","cmd/agent-worker/main.go"],"production_proof":["docs/evidence/wp-200-wp-301-fleet-execution-status-2026-08-22-v1.json"],"missing":"independently operated bridge endpoint is not configured; live worker is recovery-only"},
{"area":"Multi-Agent Work Graph","status":"REPOSITORY_AND_RUNTIME_FOUNDATION_VERIFIED","repository_proof":["dev/tasks/DEV-000131-work-graph-harness.json","dev/tasks/DEV-000146-work-graph-runtime.json","tests/agent_execution/graph_runtime_test.go"],"production_proof":["docs/PROJECT_STATUS.md"],"missing":"no claim of external agent execution fan-out is made while bridge is absent"},
{"area":"Truth/Research runtime","status":"REPOSITORY_AND_RUNTIME_FOUNDATION_VERIFIED","repository_proof":["dev/tasks/DEV-000134-truth-bundle-compiler.json","dev/tasks/DEV-000143-research-runtime-orchestration.json","tests/research_runtime"],"production_proof":["docs/PROJECT_STATUS.md"],"missing":"authenticated end-to-end research acceptance is not claimed here"},
{"area":"Learning reflection loop","status":"REPOSITORY_AND_RUNTIME_FOUNDATION_VERIFIED","repository_proof":["dev/tasks/DEV-000135-reflection-worker.json","cmd/reflection-worker/main.go","tests/research_contracts/reflection_worker_sql_test.go"],"production_proof":["docs/PROJECT_STATUS.md"],"missing":"no claim of fully closed historical candidate backlog"},
{"area":"Model fleet/failover","status":"PARTIAL_EXTERNAL_GAP","repository_proof":["dev/tasks/DEV-000144-fleet-readiness-runtime.json","dev/tasks/DEV-000168-configurable-medium-fleet-slot.json","dev/tasks/DEV-000169-optional-medium-readiness.json","cmd/cognitive-worker/fleet_status.go"],"production_proof":["docs/evidence/wp-200-wp-301-fleet-execution-status-2026-08-22-v2.json"],"missing":"Medium runtime is not configured; Heavy GPU probe times out; FAST and Lifeboat are ready and Lifeboat is selected fallback"},
{"area":"Model fleet/failover","status":"PARTIAL_EXTERNAL_GAP","repository_proof":["dev/tasks/DEV-000144-fleet-readiness-runtime.json","dev/tasks/DEV-000168-configurable-medium-fleet-slot.json","dev/tasks/DEV-000169-optional-medium-readiness.json","dev/tasks/DEV-000171-connect-existing-medium-runtime.json","dev/tasks/DEV-000172-fleet-failover-priority.json","cmd/cognitive-worker/fleet_status.go","modules/capability/domain/fleet.go"],"production_proof":["docs/evidence/wp-200-wp-301-fleet-execution-status-2026-08-22-v3.json"],"missing":"Heavy GPU runtime remains unavailable by timeout; Medium availability is proven but model qualification/activation and multi-provider invocation evidence remain separate"},
{"area":"Governed self-development","status":"REPOSITORY_FOUNDATION_VERIFIED","repository_proof":["dev/tasks/DEV-000137-governed-self-development-gate.json","tests/semantic_evolution"],"production_proof":["docs/PROJECT_STATUS.md"],"missing":"real bridge-backed self-development execution is not claimed"},
{"area":"UI/operations acceptance","status":"PARTIAL_EXTERNAL_GAP","repository_proof":["dev/tasks/DEV-000138-ui-operations-acceptance-contract.json","tests/creator_web/cabinet_acceptance_contract_test.go","dev/tasks/DEV-000150-agent-operations-ui.json","dev/tasks/DEV-000151-agent-worker-status-ui.json"],"production_proof":["docs/PROJECT_STATUS.md"],"missing":"authenticated browser visual acceptance is pending because no signed-in session is available"}
],
@@ -0,0 +1,26 @@
{
"evidence_id": "wp-200-wp-301-fleet-execution-status-2026-08-22-v3",
"captured_at": "2026-08-22",
"production_revision": "64642f4baed28f9051efe6a269769c5838b547e4",
"source": "live internal status probes on docker.cin.su after DEV-000171 and DEV-000172 deployment",
"fleet_readiness": {
"preferred_tier": "HEAVY",
"selected_tier": "MEDIUM",
"ready_tiers": ["FAST", "MEDIUM", "LIFEBOAT"],
"members": [
{"provider_instance_id": "local-cpu-primary", "tier": "FAST", "ready": true, "reason": "AVAILABLE"},
{"provider_instance_id": "local-medium-primary", "tier": "MEDIUM", "ready": true, "reason": "AVAILABLE", "provider_model_observed": "Qwen3.6-27B-UD-Q4_K_XL", "runtime": "llama.cpp CPU"},
{"provider_instance_id": "local-gpu-opportunistic", "tier": "HEAVY", "ready": false, "reason": "context deadline exceeded while awaiting headers"},
{"provider_instance_id": "cognitive-lifeboat", "tier": "LIFEBOAT", "ready": true, "reason": "AVAILABLE"}
]
},
"runtime_readiness": {"status": "ready", "cpu": "READY/AVAILABLE", "medium": "READY/AVAILABLE", "lifeboat": "READY/AVAILABLE", "gpu": "DEGRADED/DEPENDENCY_UNAVAILABLE"},
"failover_policy": "HEAVY -> MEDIUM -> FAST -> LIFEBOAT",
"agent_execution": {"runtime": "agent-worker", "executor_configured": false, "mode": "recovery-only", "ready": false, "reason": "AGENT_EXECUTION_BRIDGE_URL is not configured"},
"interpretation": [
"The existing Qwen3.6-27B runtime is available as MEDIUM through an explicit external network attachment.",
"When preferred HEAVY is unavailable, fleet selection now chooses MEDIUM before FAST or Lifeboat.",
"Availability evidence does not claim model qualification, binding activation or authenticated browser acceptance.",
"This artifact contains no secret values."
]
}
+22 -2
View File
@@ -37,8 +37,13 @@ func SelectFleetTier(readiness capability.FleetReadiness, preferred capability.F
if readiness.ObservedAtUnixMillis < 1 || len(readiness.Members) == 0 || !validFleetTier(readiness.FallbackTier) {
return "", ErrInvalidFleetReadiness
}
if slices.Contains(readiness.ReadyTiers, preferred) {
return preferred, nil
if !validFleetTier(preferred) {
return "", ErrInvalidFleetReadiness
}
for _, tier := range fleetFailoverOrder(preferred) {
if slices.Contains(readiness.ReadyTiers, tier) {
return tier, nil
}
}
if slices.Contains(readiness.ReadyTiers, readiness.FallbackTier) {
return readiness.FallbackTier, nil
@@ -46,6 +51,21 @@ func SelectFleetTier(readiness capability.FleetReadiness, preferred capability.F
return "", ErrInvalidFleetReadiness
}
func fleetFailoverOrder(preferred capability.FleetTier) []capability.FleetTier {
switch preferred {
case capability.FleetTierHeavy:
return []capability.FleetTier{capability.FleetTierHeavy, capability.FleetTierMedium, capability.FleetTierFast}
case capability.FleetTierMedium:
return []capability.FleetTier{capability.FleetTierMedium, capability.FleetTierFast}
case capability.FleetTierFast:
return []capability.FleetTier{capability.FleetTierFast}
case capability.FleetTierLifeboat:
return []capability.FleetTier{capability.FleetTierLifeboat}
default:
return nil
}
}
func validFleetTier(value capability.FleetTier) bool {
return value == capability.FleetTierFast || value == capability.FleetTierMedium || value == capability.FleetTierHeavy || value == capability.FleetTierLifeboat
}
+30
View File
@@ -16,3 +16,33 @@ func TestFleetReadinessFallsBackToLifeboatWhenPrimaryTiersUnavailable(t *testing
t.Fatalf("selected=%q err=%v", selected, err)
}
}
func TestFleetReadinessFallsBackFromHeavyToMediumBeforeLifeboat(t *testing.T) {
readiness, err := BuildFleetReadiness(100, []capability.FleetMember{
{ProviderInstanceID: "medium", ModelDescriptorID: "qwen-medium", Tier: capability.FleetTierMedium, Ready: true, Reason: "available"},
{ProviderInstanceID: "heavy", ModelDescriptorID: "gpu", Tier: capability.FleetTierHeavy, Ready: false, Reason: "timeout"},
{ProviderInstanceID: "lifeboat", ModelDescriptorID: "qwen-lifeboat", Tier: capability.FleetTierLifeboat, Ready: true, Reason: "available"},
})
if err != nil {
t.Fatal(err)
}
selected, err := SelectFleetTier(readiness, capability.FleetTierHeavy)
if err != nil || selected != capability.FleetTierMedium {
t.Fatalf("selected=%q err=%v", selected, err)
}
}
func TestFleetReadinessFallsBackFromHeavyToFastBeforeLifeboat(t *testing.T) {
readiness, err := BuildFleetReadiness(100, []capability.FleetMember{
{ProviderInstanceID: "fast", ModelDescriptorID: "qwen-fast", Tier: capability.FleetTierFast, Ready: true, Reason: "available"},
{ProviderInstanceID: "heavy", ModelDescriptorID: "gpu", Tier: capability.FleetTierHeavy, Ready: false, Reason: "timeout"},
{ProviderInstanceID: "lifeboat", ModelDescriptorID: "qwen-lifeboat", Tier: capability.FleetTierLifeboat, Ready: true, Reason: "available"},
})
if err != nil {
t.Fatal(err)
}
selected, err := SelectFleetTier(readiness, capability.FleetTierHeavy)
if err != nil || selected != capability.FleetTierFast {
t.Fatalf("selected=%q err=%v", selected, err)
}
}