Compare commits
15
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
b30eb38eca | ||
|
|
2083096aed | ||
|
|
671ecb4968 | ||
|
|
af9d5809e9 | ||
|
|
990928618d | ||
|
|
9fac8284f5 | ||
|
|
7d9a5edee5 | ||
|
|
32d3f318a5 | ||
|
|
3a563a29ee | ||
|
|
734db5d779 | ||
|
|
36fc8ec8c6 | ||
|
|
b8c001be4c | ||
|
|
eeeba6b7c0 | ||
|
|
fce3ec569c | ||
|
|
a925d787dc |
+207
-5
@@ -1,10 +1,212 @@
|
||||
package main
|
||||
|
||||
import "net/http"
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httputil"
|
||||
"net/url"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.cin.su/m/Assistant/runtime/supervisor"
|
||||
)
|
||||
|
||||
type server struct {
|
||||
lifeboat *supervisor.PersistentLifeboat
|
||||
stableUpstream string
|
||||
candidate string
|
||||
client *http.Client
|
||||
}
|
||||
|
||||
func main() {
|
||||
_ = http.ListenAndServe(":8088", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.WriteHeader(http.StatusOK)
|
||||
_, _ = w.Write([]byte("lifeboat-ready\n"))
|
||||
}))
|
||||
statePath := envOr("SUPERVISOR_STATE_PATH", "/var/lib/supervisor/release-state.json")
|
||||
stable := strings.TrimRight(envOr("SUPERVISOR_STABLE_UPSTREAM", "http://creator-web:8084"), "/")
|
||||
candidate := strings.TrimRight(envOr("SUPERVISOR_CANDIDATE_UPSTREAM", "http://creator-web-candidate:8084"), "/")
|
||||
store, err := supervisor.NewDurableStore(statePath)
|
||||
if err != nil {
|
||||
fatal(err)
|
||||
}
|
||||
lifeboat := supervisor.NewPersistentLifeboat(store)
|
||||
if _, err := lifeboat.Snapshot(); errors.Is(err, os.ErrNotExist) {
|
||||
baseline := supervisor.Release{ID: envOr("SUPERVISOR_BASELINE_RELEASE", "bootstrap"), Version: envOr("SUPERVISOR_BASELINE_RELEASE", "bootstrap")}
|
||||
if ok, startErr := lifeboat.EstablishBaseline(baseline); startErr != nil || !ok {
|
||||
fatal(fmt.Errorf("initialize baseline: %v", startErr))
|
||||
}
|
||||
if ok, observeErr := lifeboat.Record(supervisor.Observation{ReleaseID: baseline.ID, Ready: true, Reason: "supervisor baseline"}); observeErr != nil || !ok {
|
||||
fatal(fmt.Errorf("initialize baseline observation: %v", observeErr))
|
||||
}
|
||||
if ok, trafficErr := lifeboat.SetTraffic(supervisor.TrafficStable, stable, candidate); trafficErr != nil || !ok {
|
||||
fatal(fmt.Errorf("initialize traffic: %v", trafficErr))
|
||||
}
|
||||
} else if err != nil {
|
||||
fatal(err)
|
||||
}
|
||||
|
||||
s := &server{lifeboat: lifeboat, stableUpstream: stable, candidate: candidate, client: &http.Client{Timeout: 5 * time.Second}}
|
||||
public := http.NewServeMux()
|
||||
public.HandleFunc("GET /healthz", s.health)
|
||||
public.HandleFunc("GET /readyz", s.ready)
|
||||
public.HandleFunc("/", s.proxy)
|
||||
control := http.NewServeMux()
|
||||
control.HandleFunc("GET /v1/state", s.state)
|
||||
control.HandleFunc("POST /v1/releases/start", s.start)
|
||||
control.HandleFunc("POST /v1/releases/baseline", s.baseline)
|
||||
control.HandleFunc("POST /v1/releases/observe", s.observe)
|
||||
control.HandleFunc("POST /v1/releases/promote", s.promote)
|
||||
control.HandleFunc("POST /v1/releases/rollback", s.rollback)
|
||||
control.HandleFunc("POST /v1/traffic/canary", s.canary)
|
||||
control.HandleFunc("POST /v1/traffic/promote", s.promoteTraffic)
|
||||
control.HandleFunc("POST /v1/traffic/rollback", s.rollbackTraffic)
|
||||
|
||||
go func() {
|
||||
if err := http.ListenAndServe(envOr("SUPERVISOR_CONTROL_ADDR", ":8095"), control); err != nil {
|
||||
fatal(err)
|
||||
}
|
||||
}()
|
||||
if err := http.ListenAndServe(envOr("LISTEN_ADDR", ":8080"), public); err != nil {
|
||||
fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *server) health(w http.ResponseWriter, _ *http.Request) {
|
||||
writeJSON(w, map[string]string{"runtime": "supervisor", "status": "ok"})
|
||||
}
|
||||
|
||||
func (s *server) ready(w http.ResponseWriter, r *http.Request) {
|
||||
state, err := s.lifeboat.Snapshot()
|
||||
if err != nil || state.StableUpstream == "" {
|
||||
http.Error(w, "supervisor state unavailable", http.StatusServiceUnavailable)
|
||||
return
|
||||
}
|
||||
request, err := http.NewRequestWithContext(r.Context(), http.MethodGet, state.StableUpstream+"/healthz", nil)
|
||||
if err != nil {
|
||||
http.Error(w, "stable upstream unavailable", http.StatusServiceUnavailable)
|
||||
return
|
||||
}
|
||||
response, err := s.client.Do(request)
|
||||
if err != nil || response.StatusCode/100 != 2 {
|
||||
if response != nil {
|
||||
response.Body.Close()
|
||||
}
|
||||
http.Error(w, "stable upstream unavailable", http.StatusServiceUnavailable)
|
||||
return
|
||||
}
|
||||
response.Body.Close()
|
||||
writeJSON(w, map[string]string{"runtime": "supervisor", "status": "ready"})
|
||||
}
|
||||
|
||||
func (s *server) proxy(w http.ResponseWriter, r *http.Request) {
|
||||
state, err := s.lifeboat.Snapshot()
|
||||
if err != nil {
|
||||
http.Error(w, "supervisor state unavailable", http.StatusServiceUnavailable)
|
||||
return
|
||||
}
|
||||
target := state.StableUpstream
|
||||
selected := "stable"
|
||||
if state.TrafficMode == supervisor.TrafficCandidate || (state.TrafficMode == supervisor.TrafficCanaryHeader && strings.EqualFold(r.Header.Get("X-Assistant-Canary"), "true")) {
|
||||
target = state.CandidateUpstream
|
||||
selected = "candidate"
|
||||
}
|
||||
parsed, err := url.Parse(target)
|
||||
if err != nil || parsed.Scheme == "" || parsed.Host == "" {
|
||||
http.Error(w, "invalid upstream", http.StatusServiceUnavailable)
|
||||
return
|
||||
}
|
||||
proxy := httputil.NewSingleHostReverseProxy(parsed)
|
||||
proxy.ModifyResponse = func(response *http.Response) error {
|
||||
response.Header.Set("X-Assistant-Traffic-Target", selected)
|
||||
return nil
|
||||
}
|
||||
proxy.ErrorHandler = func(writer http.ResponseWriter, _ *http.Request, _ error) {
|
||||
http.Error(writer, "upstream unavailable", http.StatusBadGateway)
|
||||
}
|
||||
proxy.ServeHTTP(w, r)
|
||||
}
|
||||
|
||||
func (s *server) state(w http.ResponseWriter, _ *http.Request) {
|
||||
state, err := s.lifeboat.Snapshot()
|
||||
if err != nil {
|
||||
http.Error(w, "state unavailable", 503)
|
||||
return
|
||||
}
|
||||
writeJSON(w, state)
|
||||
}
|
||||
func (s *server) start(w http.ResponseWriter, r *http.Request) {
|
||||
var release supervisor.Release
|
||||
if !decode(w, r, &release) {
|
||||
return
|
||||
}
|
||||
ok, err := s.lifeboat.Start(release)
|
||||
respond(w, ok, err)
|
||||
}
|
||||
func (s *server) baseline(w http.ResponseWriter, r *http.Request) {
|
||||
var release supervisor.Release
|
||||
if !decode(w, r, &release) {
|
||||
return
|
||||
}
|
||||
ok, err := s.lifeboat.EstablishBaseline(release)
|
||||
respond(w, ok, err)
|
||||
}
|
||||
func (s *server) observe(w http.ResponseWriter, r *http.Request) {
|
||||
var observation supervisor.Observation
|
||||
if !decode(w, r, &observation) {
|
||||
return
|
||||
}
|
||||
ok, err := s.lifeboat.Record(observation)
|
||||
respond(w, ok, err)
|
||||
}
|
||||
func (s *server) promote(w http.ResponseWriter, _ *http.Request) {
|
||||
ok, err := s.lifeboat.Promote()
|
||||
respond(w, ok, err)
|
||||
}
|
||||
func (s *server) rollback(w http.ResponseWriter, _ *http.Request) {
|
||||
ok, err := s.lifeboat.Rollback()
|
||||
respond(w, ok, err)
|
||||
}
|
||||
func (s *server) canary(w http.ResponseWriter, _ *http.Request) {
|
||||
ok, err := s.lifeboat.SetTraffic(supervisor.TrafficCanaryHeader, s.stableUpstream, s.candidate)
|
||||
respond(w, ok, err)
|
||||
}
|
||||
func (s *server) promoteTraffic(w http.ResponseWriter, _ *http.Request) {
|
||||
ok, err := s.lifeboat.Promote()
|
||||
respond(w, ok, err)
|
||||
}
|
||||
func (s *server) rollbackTraffic(w http.ResponseWriter, _ *http.Request) {
|
||||
ok, err := s.lifeboat.RollbackTraffic()
|
||||
respond(w, ok, err)
|
||||
}
|
||||
|
||||
func decode(w http.ResponseWriter, r *http.Request, value any) bool {
|
||||
decoder := json.NewDecoder(http.MaxBytesReader(w, r.Body, 16<<10))
|
||||
decoder.DisallowUnknownFields()
|
||||
if err := decoder.Decode(value); err != nil {
|
||||
http.Error(w, "invalid control request", 400)
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
func respond(w http.ResponseWriter, ok bool, err error) {
|
||||
if err != nil {
|
||||
http.Error(w, "control state unavailable", 503)
|
||||
return
|
||||
}
|
||||
if !ok {
|
||||
http.Error(w, "control transition rejected", 409)
|
||||
return
|
||||
}
|
||||
writeJSON(w, map[string]bool{"ok": true})
|
||||
}
|
||||
func writeJSON(w http.ResponseWriter, value any) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_ = json.NewEncoder(w).Encode(value)
|
||||
}
|
||||
func envOr(name, fallback string) string {
|
||||
if value := strings.TrimSpace(os.Getenv(name)); value != "" {
|
||||
return value
|
||||
}
|
||||
return fallback
|
||||
}
|
||||
func fatal(err error) { fmt.Fprintln(os.Stderr, err); os.Exit(1) }
|
||||
|
||||
@@ -0,0 +1,58 @@
|
||||
-- Gen2 owner-role provisioning for the dedicated bootstrap/migration principal.
|
||||
-- This is intentionally separate from feature migrations and may be run repeatedly.
|
||||
BEGIN;
|
||||
|
||||
DO $$
|
||||
BEGIN
|
||||
IF NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'world_role') THEN
|
||||
CREATE ROLE world_role LOGIN;
|
||||
END IF;
|
||||
IF NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'configuration_role') THEN
|
||||
CREATE ROLE configuration_role LOGIN;
|
||||
END IF;
|
||||
IF NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'self_role') THEN
|
||||
CREATE ROLE self_role LOGIN;
|
||||
END IF;
|
||||
END
|
||||
$$;
|
||||
|
||||
DO $$
|
||||
DECLARE
|
||||
role_name TEXT;
|
||||
can_login BOOLEAN;
|
||||
BEGIN
|
||||
FOREACH role_name IN ARRAY ARRAY['world_role', 'configuration_role', 'self_role'] LOOP
|
||||
SELECT rolcanlogin INTO can_login FROM pg_roles WHERE rolname = role_name;
|
||||
IF can_login IS DISTINCT FROM TRUE THEN
|
||||
RAISE EXCEPTION 'Gen2 owner role % exists but is not a V1-compatible LOGIN role', role_name;
|
||||
END IF;
|
||||
END LOOP;
|
||||
END
|
||||
$$;
|
||||
|
||||
CREATE SCHEMA IF NOT EXISTS world AUTHORIZATION world_role;
|
||||
CREATE SCHEMA IF NOT EXISTS configuration AUTHORIZATION configuration_role;
|
||||
CREATE SCHEMA IF NOT EXISTS self AUTHORIZATION self_role;
|
||||
|
||||
REVOKE ALL ON SCHEMA world, configuration, self FROM PUBLIC;
|
||||
|
||||
DO $$
|
||||
DECLARE
|
||||
schema_name TEXT;
|
||||
expected_owner TEXT;
|
||||
actual_owner TEXT;
|
||||
BEGIN
|
||||
FOREACH schema_name IN ARRAY ARRAY['world', 'configuration', 'self'] LOOP
|
||||
expected_owner := schema_name || '_role';
|
||||
SELECT pg_get_userbyid(nspowner)
|
||||
INTO actual_owner
|
||||
FROM pg_namespace
|
||||
WHERE nspname = schema_name;
|
||||
IF actual_owner IS DISTINCT FROM expected_owner THEN
|
||||
RAISE EXCEPTION 'Gen2 schema % is owned by %, expected %', schema_name, actual_owner, expected_owner;
|
||||
END IF;
|
||||
END LOOP;
|
||||
END
|
||||
$$;
|
||||
|
||||
COMMIT;
|
||||
@@ -25,6 +25,16 @@ psql_exec() {
|
||||
docker exec -i "$POSTGRES_CONTAINER" psql -v ON_ERROR_STOP=1 -U "$POSTGRES_SUPERUSER" -d "$DATABASE" "$@"
|
||||
}
|
||||
|
||||
# Owner roles and schemas are provisioned by the dedicated bootstrap boundary,
|
||||
# before feature migrations run. This keeps role creation out of migrations and
|
||||
# makes existing PostgreSQL volumes converge idempotently as well.
|
||||
GEN2_BOOTSTRAP="$REPO_DIR/database/bootstrap/002_gen2_roles_and_schemas.sql"
|
||||
if [[ ! -f "$GEN2_BOOTSTRAP" ]]; then
|
||||
echo "migration runner: Gen2 bootstrap is missing" >&2
|
||||
exit 1
|
||||
fi
|
||||
psql_exec < "$GEN2_BOOTSTRAP"
|
||||
|
||||
psql_exec -c "CREATE TABLE IF NOT EXISTS public.assistant_migration_history (migration_id TEXT PRIMARY KEY, applied_at_unix_millis BIGINT NOT NULL CHECK (applied_at_unix_millis > 0));"
|
||||
|
||||
if [[ -n "${MIGRATION_BASELINE:-}" ]]; then
|
||||
|
||||
@@ -277,7 +277,7 @@ services:
|
||||
- --port
|
||||
- "8086"
|
||||
- --model
|
||||
- /models/Qwen3-8B-Q4_K_M.gguf
|
||||
- /models/Qwen3.8-27B-UD-Q2_K_XL.gguf
|
||||
- --alias
|
||||
- domovoy-lifeboat
|
||||
- --jinja
|
||||
@@ -299,6 +299,29 @@ services:
|
||||
timeout: 3s
|
||||
retries: 10
|
||||
|
||||
supervisor:
|
||||
build: { context: .., dockerfile: deploy/images/runtime.Dockerfile, args: { TARGET: supervisor, SOURCE_REVISION: "${ASSISTANT_SOURCE_REVISION:-unknown}" } }
|
||||
restart: unless-stopped
|
||||
environment:
|
||||
LISTEN_ADDR: ":8080"
|
||||
SUPERVISOR_CONTROL_ADDR: ":8095"
|
||||
SUPERVISOR_STATE_PATH: /var/lib/supervisor/release-state.json
|
||||
SUPERVISOR_STABLE_UPSTREAM: http://creator-web:8084
|
||||
SUPERVISOR_CANDIDATE_UPSTREAM: http://creator-web-candidate:8084
|
||||
SUPERVISOR_BASELINE_RELEASE: ${ASSISTANT_SOURCE_REVISION:-unknown}
|
||||
ports:
|
||||
- "${CREATOR_WEB_PORT:-18080}:8080"
|
||||
- "127.0.0.1:${SUPERVISOR_CONTROL_PORT:-18085}:8095"
|
||||
depends_on:
|
||||
creator-web: { condition: service_healthy }
|
||||
volumes:
|
||||
- assistant_supervisor_state:/var/lib/supervisor
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "wget -qO- http://localhost:8080/healthz >/dev/null && wget -qO- http://localhost:8080/readyz >/dev/null"]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 20
|
||||
|
||||
creator-web:
|
||||
build: { context: .., dockerfile: deploy/images/runtime.Dockerfile, args: { TARGET: creator-web, SOURCE_REVISION: "${ASSISTANT_SOURCE_REVISION:-unknown}" } }
|
||||
restart: unless-stopped
|
||||
@@ -310,7 +333,6 @@ services:
|
||||
COGNITIVE_RUNTIME_URL: http://cognitive-worker:8083
|
||||
AGENT_WORKER_URL: http://agent-worker:8090
|
||||
ASSISTANT_BUILD_REVISION: ${ASSISTANT_SOURCE_REVISION:-unknown}
|
||||
ports: ["${CREATOR_WEB_PORT:-18080}:8084"]
|
||||
depends_on: [core-runtime, continuity-runtime, commit-runtime, cognitive-worker]
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "wget -qO- http://localhost:8084/healthz >/dev/null"]
|
||||
@@ -318,6 +340,12 @@ services:
|
||||
timeout: 3s
|
||||
retries: 10
|
||||
|
||||
creator-web-candidate:
|
||||
profiles: ["assistant-canary"]
|
||||
extends:
|
||||
service: creator-web
|
||||
restart: "no"
|
||||
|
||||
volumes:
|
||||
assistant_pg:
|
||||
assistant_nats:
|
||||
@@ -333,6 +361,7 @@ volumes:
|
||||
external: true
|
||||
assistant_research_sources:
|
||||
name: assistant_research_sources
|
||||
assistant_supervisor_state:
|
||||
|
||||
networks:
|
||||
llm-local:
|
||||
|
||||
@@ -4,5 +4,6 @@ LABEL org.opencontainers.image.revision=$SOURCE_REVISION \
|
||||
org.opencontainers.image.source="https://git.cin.su/m/Assistant.git"
|
||||
|
||||
COPY database/bootstrap/001_roles_and_schemas.sql /docker-entrypoint-initdb.d/001_roles_and_schemas.sql
|
||||
COPY database/bootstrap/002_gen2_roles_and_schemas.sql /docker-entrypoint-initdb.d/002_gen2_roles_and_schemas.sql
|
||||
COPY deploy/images/002_owner_passwords.sh /docker-entrypoint-initdb.d/002_owner_passwords.sh
|
||||
RUN chmod 0555 /docker-entrypoint-initdb.d/002_owner_passwords.sh
|
||||
|
||||
@@ -11,6 +11,7 @@ ARG SOURCE_REVISION=unknown
|
||||
LABEL org.opencontainers.image.revision=$SOURCE_REVISION \
|
||||
org.opencontainers.image.source="https://git.cin.su/m/Assistant.git"
|
||||
RUN addgroup -S assistant && adduser -S -G assistant assistant
|
||||
RUN mkdir -p /var/lib/supervisor && chown assistant:assistant /var/lib/supervisor
|
||||
USER assistant
|
||||
COPY --from=build /out/runtime /usr/local/bin/assistant-runtime
|
||||
ENTRYPOINT ["/usr/local/bin/assistant-runtime"]
|
||||
|
||||
@@ -0,0 +1,43 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
BASE_DIR="${REPO_DIR:-$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)}"
|
||||
ENV_FILE="${ENV_FILE:-/home/m/assistant-deploy/production-compose.env}"
|
||||
COMPOSE_FILE="$BASE_DIR/deploy/docker-compose.prod.yml"
|
||||
SUPERVISOR_CONTROL_URL="${SUPERVISOR_CONTROL_URL:-http://127.0.0.1:${SUPERVISOR_CONTROL_PORT:-18085}}"
|
||||
REVISION="${1:?candidate revision is required}"
|
||||
|
||||
set -a
|
||||
. "$ENV_FILE"
|
||||
set +a
|
||||
|
||||
compose() { docker compose --env-file "$ENV_FILE" -f "$COMPOSE_FILE" "$@"; }
|
||||
control() { curl -fsS --max-time 10 -X POST "$SUPERVISOR_CONTROL_URL$1" "${@:2}" >/dev/null; }
|
||||
fail_rollback() {
|
||||
control /v1/traffic/rollback || true
|
||||
control /v1/releases/rollback || true
|
||||
echo "production canary failed; stable traffic rollback requested" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
compose --profile assistant-canary build --build-arg "SOURCE_REVISION=$REVISION" creator-web-candidate
|
||||
compose --profile assistant-canary up -d --no-build creator-web-candidate
|
||||
for attempt in $(seq 1 30); do
|
||||
status="$(docker inspect -f '{{if .State.Health}}{{.State.Health.Status}}{{else}}unknown{{end}}' assistant-creator-web-candidate-1 2>/dev/null || true)"
|
||||
[[ "$status" == healthy ]] && break
|
||||
[[ "$attempt" == 30 ]] && fail_rollback
|
||||
sleep 2
|
||||
done
|
||||
|
||||
control /v1/releases/baseline -H 'Content-Type: application/json' --data "{\"id\":\"stable-$REVISION\",\"version\":\"$REVISION\"}" || fail_rollback
|
||||
control /v1/releases/start -H 'Content-Type: application/json' --data "{\"id\":\"candidate-$REVISION\",\"version\":\"$REVISION\"}" || fail_rollback
|
||||
control /v1/releases/observe -H 'Content-Type: application/json' --data "{\"release_id\":\"candidate-$REVISION\",\"ready\":true,\"reason\":\"candidate healthcheck passed\",\"observed_at_unix_millis\":$(date +%s%3N)}" || fail_rollback
|
||||
control /v1/traffic/canary || fail_rollback
|
||||
|
||||
headers="$(mktemp)"
|
||||
trap 'rm -f "$headers"' EXIT
|
||||
if ! curl -fsS --max-time 10 -D "$headers" -H 'X-Assistant-Canary: true' "https://assistant.cin.su/login" -o /dev/null; then fail_rollback; fi
|
||||
grep -qi '^X-Assistant-Traffic-Target: candidate' "$headers" || fail_rollback
|
||||
|
||||
control /v1/releases/promote || fail_rollback
|
||||
echo "production canary promoted revision=$REVISION"
|
||||
@@ -1 +1 @@
|
||||
{"task_file":"dev/tasks/GEN2-026-migrate-household-behavior-to-gen2-context-memory-skills-with-local-degr.json","task_id":"GEN2-026","task_revision":1}
|
||||
{"task_file":"dev/tasks/GEN2-027-production-canary-and-controlled-gen2-traffic-transition.json","task_id":"GEN2-027","task_revision":1}
|
||||
|
||||
@@ -0,0 +1,42 @@
|
||||
{
|
||||
"task_id": "GEN2-DB-001",
|
||||
"task_revision": 1,
|
||||
"architecture_revision": "ARCH-GEN2-000",
|
||||
"base_repo_revision": "gen2-integration",
|
||||
"risk_tier": "E",
|
||||
"intent": "Исправить provisioning Gen2 owner roles/schemas до применения migrations 046-050 без ослабления V1 DB isolation.",
|
||||
"requirement_ids": ["REQ-GEN2-DB-001-ROLES", "REQ-GEN2-DB-001-OWNERSHIP", "REQ-GEN2-DB-001-IDEMPOTENCY", "REQ-GEN2-DB-001-RECOVERY"],
|
||||
"allowed_change_scope": {
|
||||
"modules": ["database bootstrap", "deployment migration runner"],
|
||||
"paths": [
|
||||
"database/bootstrap/002_gen2_roles_and_schemas.sql",
|
||||
"deploy/apply-migrations.sh",
|
||||
"deploy/images/postgres.Dockerfile",
|
||||
"dev/tasks/GEN2-DB-001-provision-gen2-owner-roles.json",
|
||||
"dev/ACTIVE_TASK.json",
|
||||
"dev/tasks/GEN2-DB-001-provision-gen2-owner-roles.json"
|
||||
],
|
||||
},
|
||||
"forbidden_change_scope": {
|
||||
"paths": [
|
||||
"GEN2-028",
|
||||
"production-canary-before-provisioning",
|
||||
"feature-migration-role-creation",
|
||||
"cross-owner-grants",
|
||||
"new canonical owner",
|
||||
"production canary before provisioning"
|
||||
]
|
||||
},
|
||||
"allowed_architecture_delta": {"new_cross_module_dependencies": 0, "new_canonical_owners": 0, "new_privileges": 0, "new_third_party_dependencies": 0, "new_database_owner_roles": 0},
|
||||
"contracts": {"consume": ["V1 database ownership/ACL", "GEN2 canonical owner registry", "GEN2 execution contract"], "may_change": ["Bootstrap provisioning only; feature migration semantics are unchanged."]},
|
||||
"semantic_owner": "Dedicated PostgreSQL bootstrap/migration principal; semantic ownership remains World, Configuration, SELF and existing V1 owners.",
|
||||
"canonical_state": ["world schema owned by world_role", "configuration schema owned by configuration_role", "self schema owned by self_role", "no cross-owner canonical writes"],
|
||||
"state_transitions": ["missing Gen2 roles/schemas -> idempotently provisioned -> migrations 046-050 apply -> restart/recovery preserves schema and migration state"],
|
||||
"side_effects_and_privileges": ["Create only the three declared LOGIN owner roles when absent.", "Revoke PUBLIC schema privileges and fail closed on an unexpected schema owner.", "Do not invent or print credentials; do not grant cross-owner access."],
|
||||
"persistence_and_migrations": ["Provisioning runs in the dedicated bootstrap boundary before feature migrations.", "Migrations 046-050 remain unchanged and are applied by the existing migration history runner."],
|
||||
"failure_retry_restart": ["Provisioning is safe to run repeatedly.", "Unexpected existing role/schema ownership fails closed and requires an architecture decision.", "PostgreSQL restart/recovery must retain owner roles, schemas, ACL and migration history."],
|
||||
"assumptions": ["This is a user-authorized remediation task for the previously stopped GEN2-027 migration decision.", "No Gen2 runtime consumer currently requires a new database password; credentials are not invented by this task.", "GEN2-027 resumes only after provisioning and recovery evidence; GEN2-028 is not started."],
|
||||
"acceptance_scenarios": ["GEN2-DB-001-SCN-01: all roles required by migrations 046-050 are identified", "GEN2-DB-001-SCN-02: provisioning succeeds twice without duplicate objects", "GEN2-DB-001-SCN-03: migrations 046-050 apply in the controlled deployment path", "GEN2-DB-001-SCN-04: schema owners and PUBLIC ACL are confirmed", "GEN2-DB-001-SCN-05: PostgreSQL restart/recovery preserves the result", "GEN2-DB-001-EXIT: migration blocker is removed without weakening V1 isolation"],
|
||||
"required_tests": ["FAST governance: go run ./cmd/devctl preflight -task dev/tasks/GEN2-DB-001-provision-gen2-owner-roles.json", "FAST scope: go run ./cmd/devctl check-diff -active dev/ACTIVE_TASK.json", "FAST whitespace: git diff --check gen2-integration --", "TARGETED: provisioning repeated twice, migrations 046-050, ACL/ownership verification and PostgreSQL restart/recovery"],
|
||||
"terminal_gap_states": ["BLOCKED_BY_SPEC_GAP", "BLOCKED_BY_ARCHITECTURE_GAP", "BLOCKED_BY_MISSING_CAPABILITY", "BLOCKED_BY_CONFLICT", "REQUIRES_MIGRATION_DECISION"]
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
{
|
||||
"task_id": "GEN2-DEPLOY-001",
|
||||
"task_revision": 1,
|
||||
"architecture_revision": "ARCH-GEN2-000",
|
||||
"base_repo_revision": "gen2-integration",
|
||||
"risk_tier": "E",
|
||||
"intent": "Развернуть независимый Supervisor control plane с durable release state, controlled traffic switching и доказуемым rollback перед GEN2-027.",
|
||||
"requirement_ids": ["REQ-GEN2-DEPLOY-001-SUPERVISOR", "REQ-GEN2-DEPLOY-001-TRAFFIC", "REQ-GEN2-DEPLOY-001-ROLLBACK", "REQ-GEN2-DEPLOY-001-LIFEBOAT"],
|
||||
"allowed_change_scope": {
|
||||
"modules": ["deployment control", "runtime supervisor", "runtime health"],
|
||||
"paths": [
|
||||
"deploy/**",
|
||||
"runtime/supervisor/**",
|
||||
"runtime/health/**",
|
||||
"cmd/supervisor/**",
|
||||
"dev/ACTIVE_TASK.json",
|
||||
"dev/tasks/GEN2-DEPLOY-001-supervisor-and-production-rollback.json",
|
||||
"docs/evidence/**"
|
||||
]
|
||||
},
|
||||
"forbidden_change_scope": {
|
||||
"paths": ["GEN2-028", "database/**", "security/**", "architecture/**", "Authority/Commit bypass", "cognitive-lifeboat as Supervisor"]
|
||||
},
|
||||
"allowed_architecture_delta": {"new_cross_module_dependencies": 0, "new_canonical_owners": 0, "new_privileges": 1, "new_third_party_dependencies": 0, "new_database_owner_roles": 0},
|
||||
"contracts": {"consume": ["GEN2-027 production canary contract", "runtime/supervisor durable state", "existing deployment update controller"], "may_change": ["deployment topology, Supervisor HTTP control contract and traffic routing only"]},
|
||||
"semantic_owner": "Operations/Supervisor; not a semantic owner and not a cognitive provider.",
|
||||
"canonical_state": ["durable release/current/last-known-good evidence owned by Supervisor control plane", "traffic target is Supervisor-owned operational state"],
|
||||
"state_transitions": ["baseline -> candidate -> observed healthy -> header canary -> promoted OR independent rollback to last-known-good"],
|
||||
"side_effects_and_privileges": ["Control API is bound only to the production-internal loopback port.", "Supervisor has no database, Authority or Commit credentials.", "Candidate traffic is explicit and rollback remains available."],
|
||||
"persistence_and_migrations": ["No database changes; durable state is stored in a dedicated named volume outside candidate containers."],
|
||||
"failure_retry_restart": ["Supervisor restart reloads release and traffic state from its volume.", "Candidate failure must not erase last-known-good or block stable traffic rollback.", "Lifeboat model failure remains a separately reported degraded provider condition."],
|
||||
"assumptions": ["GEN2-027 resumes only after this task's production evidence; GEN2-028 is not started.", "The existing public ingress forwards to the configured creator web host port."],
|
||||
"acceptance_scenarios": ["GEN2-DEPLOY-001-SCN-01: Supervisor is a separate healthy production container", "GEN2-DEPLOY-001-SCN-02: release state survives Supervisor restart", "GEN2-DEPLOY-001-SCN-03: canary header reaches candidate while default traffic stays stable", "GEN2-DEPLOY-001-SCN-04: candidate failure rolls traffic back to last-known-good", "GEN2-DEPLOY-001-SCN-05: cognitive lifeboat diagnosis and degraded role are explicit", "GEN2-DEPLOY-001-EXIT: GEN2-027 rollback prerequisites are proven"],
|
||||
"required_tests": ["FAST: go test ./runtime/supervisor/... ./runtime/health/... ./cmd/supervisor/...", "FAST: go vet ./runtime/supervisor/... ./runtime/health/... ./cmd/supervisor/...", "FAST governance: preflight", "FAST scope: check-diff", "FAST independent: independent", "FAST whitespace: git diff --check", "TARGETED recovery: Supervisor restart, canary and rollback on docker.cin.su"],
|
||||
"terminal_gap_states": ["BLOCKED_BY_SPEC_GAP", "BLOCKED_BY_ARCHITECTURE_GAP", "BLOCKED_BY_MISSING_CAPABILITY", "BLOCKED_BY_CONFLICT", "REQUIRES_MIGRATION_DECISION"]
|
||||
}
|
||||
@@ -0,0 +1,44 @@
|
||||
{
|
||||
"evidence_id": "gen2-deploy-001-production-recovery-2026-08-29",
|
||||
"task_id": "GEN2-DEPLOY-001",
|
||||
"production_revision_before": "36fc8ec",
|
||||
"production_revision_after": "2083096aedaec73191e0bc9e2566f596e1810d82",
|
||||
"topology": {
|
||||
"supervisor": "assistant-supervisor-1",
|
||||
"public_traffic": "Supervisor reverse proxy on creator web host port",
|
||||
"control_api": "Supervisor loopback-only host control port",
|
||||
"durable_state": "assistant_assistant_supervisor_state:/var/lib/supervisor/release-state.json",
|
||||
"candidate": "assistant-creator-web-candidate-1"
|
||||
},
|
||||
"canary": {
|
||||
"candidate_health": "healthy",
|
||||
"header_route": "X-Assistant-Canary=true -> candidate",
|
||||
"header_route_http": 200,
|
||||
"promoted": true
|
||||
},
|
||||
"rollback": {
|
||||
"drill": "candidate container stopped, Supervisor traffic rollback and release rollback invoked",
|
||||
"stable_http": 200,
|
||||
"stable_target_header": "stable",
|
||||
"supervisor_restart": "healthy; traffic_mode=STABLE and stable release state restored"
|
||||
},
|
||||
"postgres_recovery": {
|
||||
"restart": "healthy",
|
||||
"public_healthz": 200,
|
||||
"public_readyz": 200
|
||||
},
|
||||
"cognitive_lifeboat": {
|
||||
"previous_status": "restart loop/unhealthy",
|
||||
"cause": "configured Qwen3-8B-Q4_K_M.gguf was absent from llm-local-models volume",
|
||||
"remediation": "compose model reference changed to existing Qwen3.8-27B-UD-Q2_K_XL.gguf",
|
||||
"current_status": "healthy",
|
||||
"role": "local degraded cognitive provider; not Supervisor"
|
||||
},
|
||||
"smoke": {
|
||||
"public_healthz": 200,
|
||||
"public_readyz": 200,
|
||||
"public_login": 200,
|
||||
"authenticated_household": "UNVERIFIED: browser session was unauthenticated; no credentials were read or entered"
|
||||
},
|
||||
"interpretation": "Supervisor independence, durable release state, controlled header canary, stable rollback and restart recovery are production-observed. Authenticated household acceptance remains an explicit gap."
|
||||
}
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
)
|
||||
|
||||
// DurableStore is supervisor-owned external control-plane state. It is not a
|
||||
@@ -63,12 +64,17 @@ func (s *DurableStore) Load() (Lifeboat, error) {
|
||||
}
|
||||
|
||||
// PersistentLifeboat reloads before every transition and commits after it.
|
||||
type PersistentLifeboat struct{ store *DurableStore }
|
||||
type PersistentLifeboat struct {
|
||||
store *DurableStore
|
||||
mu sync.Mutex
|
||||
}
|
||||
|
||||
func NewPersistentLifeboat(store *DurableStore) *PersistentLifeboat {
|
||||
return &PersistentLifeboat{store: store}
|
||||
}
|
||||
func (p *PersistentLifeboat) transition(fn func(*Lifeboat) bool) (bool, error) {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
value, err := p.store.Load()
|
||||
if err != nil && !os.IsNotExist(err) {
|
||||
return false, err
|
||||
@@ -85,10 +91,40 @@ func (p *PersistentLifeboat) transition(fn func(*Lifeboat) bool) (bool, error) {
|
||||
func (p *PersistentLifeboat) Start(candidate Release) (bool, error) {
|
||||
return p.transition(func(value *Lifeboat) bool { return value.Start(candidate) })
|
||||
}
|
||||
|
||||
func (p *PersistentLifeboat) EstablishBaseline(release Release) (bool, error) {
|
||||
return p.transition(func(value *Lifeboat) bool { return value.EstablishBaseline(release) })
|
||||
}
|
||||
func (p *PersistentLifeboat) Record(observation Observation) (bool, error) {
|
||||
return p.transition(func(value *Lifeboat) bool { return value.Record(observation) })
|
||||
}
|
||||
func (p *PersistentLifeboat) Rollback() (bool, error) {
|
||||
return p.transition(func(value *Lifeboat) bool { return value.Rollback() })
|
||||
}
|
||||
func (p *PersistentLifeboat) Snapshot() (Lifeboat, error) { return p.store.Load() }
|
||||
func (p *PersistentLifeboat) SetTraffic(mode TrafficMode, stableUpstream, candidateUpstream string) (bool, error) {
|
||||
return p.transition(func(value *Lifeboat) bool { return value.SetTraffic(mode, stableUpstream, candidateUpstream) })
|
||||
}
|
||||
|
||||
func (p *PersistentLifeboat) Promote() (bool, error) {
|
||||
return p.transition(func(value *Lifeboat) bool {
|
||||
if value.Current.State != ReleaseHealthy || value.Current.ID == "" {
|
||||
return false
|
||||
}
|
||||
return value.SetTraffic(TrafficCandidate, value.StableUpstream, value.CandidateUpstream)
|
||||
})
|
||||
}
|
||||
|
||||
func (p *PersistentLifeboat) RollbackTraffic() (bool, error) {
|
||||
return p.transition(func(value *Lifeboat) bool {
|
||||
if !value.Rollback() {
|
||||
return false
|
||||
}
|
||||
return value.SetTraffic(TrafficStable, value.StableUpstream, value.CandidateUpstream)
|
||||
})
|
||||
}
|
||||
|
||||
func (p *PersistentLifeboat) Snapshot() (Lifeboat, error) {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
return p.store.Load()
|
||||
}
|
||||
|
||||
@@ -9,22 +9,34 @@ const (
|
||||
ReleaseRolledBack ReleaseState = "ROLLED_BACK"
|
||||
)
|
||||
|
||||
type TrafficMode string
|
||||
|
||||
const (
|
||||
TrafficStable TrafficMode = "STABLE"
|
||||
TrafficCanaryHeader TrafficMode = "CANARY_HEADER"
|
||||
TrafficCandidate TrafficMode = "CANDIDATE"
|
||||
)
|
||||
|
||||
type Release struct {
|
||||
ID string
|
||||
Version string
|
||||
State ReleaseState
|
||||
LastKnownGood bool
|
||||
ID string `json:"id"`
|
||||
Version string `json:"version"`
|
||||
State ReleaseState `json:"state"`
|
||||
LastKnownGood bool `json:"last_known_good"`
|
||||
}
|
||||
type Observation struct {
|
||||
ReleaseID string
|
||||
Ready bool
|
||||
Reason string
|
||||
ObservedAtUnixMillis int64
|
||||
ReleaseID string `json:"release_id"`
|
||||
Ready bool `json:"ready"`
|
||||
Reason string `json:"reason"`
|
||||
ObservedAtUnixMillis int64 `json:"observed_at_unix_millis"`
|
||||
}
|
||||
type Lifeboat struct {
|
||||
Current Release
|
||||
LastKnownGood Release
|
||||
Observations []Observation
|
||||
Current Release `json:"current"`
|
||||
LastKnownGood Release `json:"last_known_good"`
|
||||
RollbackTarget Release `json:"rollback_target"`
|
||||
Observations []Observation `json:"observations"`
|
||||
TrafficMode TrafficMode `json:"traffic_mode"`
|
||||
StableUpstream string `json:"stable_upstream"`
|
||||
CandidateUpstream string `json:"candidate_upstream"`
|
||||
}
|
||||
|
||||
func (l *Lifeboat) Start(candidate Release) bool {
|
||||
@@ -32,9 +44,25 @@ func (l *Lifeboat) Start(candidate Release) bool {
|
||||
return false
|
||||
}
|
||||
candidate.State = ReleaseCandidate
|
||||
if l.LastKnownGood.ID != "" {
|
||||
l.RollbackTarget = l.LastKnownGood
|
||||
}
|
||||
l.Current = candidate
|
||||
return true
|
||||
}
|
||||
|
||||
func (l *Lifeboat) EstablishBaseline(release Release) bool {
|
||||
if release.ID == "" || release.Version == "" {
|
||||
return false
|
||||
}
|
||||
release.State = ReleaseHealthy
|
||||
release.LastKnownGood = true
|
||||
l.Current = release
|
||||
l.LastKnownGood = release
|
||||
l.RollbackTarget = Release{}
|
||||
l.TrafficMode = TrafficStable
|
||||
return true
|
||||
}
|
||||
func (l *Lifeboat) Record(o Observation) bool {
|
||||
if o.ReleaseID == "" {
|
||||
return false
|
||||
@@ -53,10 +81,30 @@ func (l *Lifeboat) Record(o Observation) bool {
|
||||
return true
|
||||
}
|
||||
func (l *Lifeboat) Rollback() bool {
|
||||
if l.LastKnownGood.ID == "" {
|
||||
target := l.RollbackTarget
|
||||
if target.ID == "" {
|
||||
target = l.LastKnownGood
|
||||
}
|
||||
if target.ID == "" {
|
||||
return false
|
||||
}
|
||||
l.Current = l.LastKnownGood
|
||||
l.Current = target
|
||||
l.Current.State = ReleaseRolledBack
|
||||
l.Current.LastKnownGood = true
|
||||
l.LastKnownGood = l.Current
|
||||
l.RollbackTarget = Release{}
|
||||
return true
|
||||
}
|
||||
|
||||
func (l *Lifeboat) SetTraffic(mode TrafficMode, stableUpstream, candidateUpstream string) bool {
|
||||
if mode != TrafficStable && mode != TrafficCanaryHeader && mode != TrafficCandidate {
|
||||
return false
|
||||
}
|
||||
if stableUpstream == "" || (mode != TrafficStable && candidateUpstream == "") {
|
||||
return false
|
||||
}
|
||||
l.TrafficMode = mode
|
||||
l.StableUpstream = stableUpstream
|
||||
l.CandidateUpstream = candidateUpstream
|
||||
return true
|
||||
}
|
||||
|
||||
@@ -47,3 +47,38 @@ func TestPersistentLifeboatRestoresLastKnownGoodAfterCandidateFailure(t *testing
|
||||
t.Fatalf("state=%+v", state)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPersistentSupervisorRestoresTrafficModeAfterRestart(t *testing.T) {
|
||||
store, err := supervisor.NewDurableStore(filepath.Join(t.TempDir(), "control", "release-state.json"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
first := supervisor.NewPersistentLifeboat(store)
|
||||
if ok, err := first.Start(supervisor.Release{ID: "candidate", Version: "2"}); err != nil || !ok {
|
||||
t.Fatalf("start: %v %v", ok, err)
|
||||
}
|
||||
if ok, err := first.Record(supervisor.Observation{ReleaseID: "candidate", Ready: true, ObservedAtUnixMillis: 1}); err != nil || !ok {
|
||||
t.Fatalf("observe: %v %v", ok, err)
|
||||
}
|
||||
if ok, err := first.SetTraffic(supervisor.TrafficCanaryHeader, "http://stable:8084", "http://candidate:8084"); err != nil || !ok {
|
||||
t.Fatalf("canary: %v %v", ok, err)
|
||||
}
|
||||
second := supervisor.NewPersistentLifeboat(store)
|
||||
state, err := second.Snapshot()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if state.TrafficMode != supervisor.TrafficCanaryHeader || state.CandidateUpstream != "http://candidate:8084" || state.LastKnownGood.ID != "candidate" {
|
||||
t.Fatalf("state=%+v", state)
|
||||
}
|
||||
if ok, err := second.RollbackTraffic(); err != nil || !ok {
|
||||
t.Fatalf("rollback traffic: %v %v", ok, err)
|
||||
}
|
||||
state, err = second.Snapshot()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if state.TrafficMode != supervisor.TrafficStable || state.Current.State != supervisor.ReleaseRolledBack {
|
||||
t.Fatalf("rollback state=%+v", state)
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user