This commit is contained in:
Executable
+66
@@ -0,0 +1,66 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
|
||||
umask 077
|
||||
|
||||
container="${ASSISTANT_POSTGRES_CONTAINER:-assistant-postgres-1}"
|
||||
backup_root="${ASSISTANT_BACKUP_DIR:-/home/m/assistant-backups/postgres}"
|
||||
timestamp="$(date -u +%Y%m%dT%H%M%SZ)"
|
||||
partial_dir="${backup_root}/.${timestamp}.partial"
|
||||
backup_dir="${backup_root}/${timestamp}"
|
||||
|
||||
cleanup() {
|
||||
if [ -d "$partial_dir" ]; then
|
||||
rm -f -- "$partial_dir/assistant.dump" "$partial_dir/globals.sql" \
|
||||
"$partial_dir/database.name" "$partial_dir/owner.name" "$partial_dir/SHA256SUMS"
|
||||
rmdir -- "$partial_dir" 2>/dev/null || true
|
||||
fi
|
||||
}
|
||||
trap cleanup EXIT HUP INT TERM
|
||||
|
||||
if ! docker inspect "$container" >/dev/null 2>&1; then
|
||||
echo "PostgreSQL container not found: $container" >&2
|
||||
exit 1
|
||||
fi
|
||||
if [ "$(docker inspect -f '{{.State.Running}}' "$container")" != "true" ]; then
|
||||
echo "PostgreSQL container is not running: $container" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
mkdir -p -- "$backup_root"
|
||||
chmod 700 -- "$backup_root"
|
||||
if [ -e "$partial_dir" ] || [ -e "$backup_dir" ]; then
|
||||
echo "Backup destination already exists for $timestamp" >&2
|
||||
exit 1
|
||||
fi
|
||||
mkdir -- "$partial_dir"
|
||||
|
||||
database="$(docker exec "$container" sh -c 'printf %s "$POSTGRES_DB"')"
|
||||
owner="$(docker exec "$container" sh -c 'printf %s "$POSTGRES_USER"')"
|
||||
[ -n "$database" ] && [ -n "$owner" ] || {
|
||||
echo "Database metadata is empty" >&2
|
||||
exit 1
|
||||
}
|
||||
case "$database:$owner" in
|
||||
*[!A-Za-z0-9_:.-]*) echo "Unsafe database metadata" >&2; exit 1 ;;
|
||||
esac
|
||||
|
||||
printf '%s\n' "$database" >"$partial_dir/database.name"
|
||||
printf '%s\n' "$owner" >"$partial_dir/owner.name"
|
||||
docker exec "$container" sh -c 'exec pg_dump --format=custom --compress=9 --no-password -U "$POSTGRES_USER" -d "$POSTGRES_DB"' >"$partial_dir/assistant.dump"
|
||||
docker exec "$container" sh -c 'exec pg_dumpall --globals-only --no-role-passwords --no-password -U "$POSTGRES_USER"' >"$partial_dir/globals.sql"
|
||||
|
||||
if [ ! -s "$partial_dir/assistant.dump" ] || [ ! -s "$partial_dir/globals.sql" ]; then
|
||||
echo "Backup output is incomplete" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
(
|
||||
cd "$partial_dir"
|
||||
sha256sum assistant.dump globals.sql database.name owner.name >SHA256SUMS
|
||||
sha256sum -c SHA256SUMS >/dev/null
|
||||
)
|
||||
chmod 600 "$partial_dir"/*
|
||||
mv -- "$partial_dir" "$backup_dir"
|
||||
trap - EXIT HUP INT TERM
|
||||
printf '%s\n' "$backup_dir"
|
||||
@@ -0,0 +1,2 @@
|
||||
# Domovoy Assistant PostgreSQL backup: 00:15 UTC is 03:15 Europe/Moscow.
|
||||
15 0 * * * umask 077; { /home/m/assistant-ops/backup-postgres.sh && /home/m/assistant-ops/verify-latest-postgres-backup.sh; } >>/home/m/assistant-backups/postgres/backup.log 2>&1
|
||||
Executable
+30
@@ -0,0 +1,30 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
|
||||
schedule_file="${1:-/home/m/assistant-ops/assistant-postgres-backup.crontab}"
|
||||
marker="/home/m/assistant-ops/backup-postgres.sh"
|
||||
current="$(mktemp)"
|
||||
updated="$(mktemp)"
|
||||
|
||||
cleanup() {
|
||||
rm -f -- "$current" "$updated"
|
||||
}
|
||||
trap cleanup EXIT HUP INT TERM
|
||||
|
||||
if [ ! -f "$schedule_file" ]; then
|
||||
echo "Cron schedule file not found: $schedule_file" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
crontab -l >"$current" 2>/dev/null || true
|
||||
grep -vF "$marker" "$current" >"$updated" || true
|
||||
cat "$schedule_file" >>"$updated"
|
||||
crontab "$updated"
|
||||
|
||||
matches="$(crontab -l | grep -Fc "$marker")"
|
||||
if [ "$matches" -ne 1 ]; then
|
||||
echo "Expected one backup cron entry, found $matches" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "PostgreSQL backup cron entry installed"
|
||||
Executable
+13
@@ -0,0 +1,13 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
|
||||
backup_root="${ASSISTANT_BACKUP_DIR:-/home/m/assistant-backups/postgres}"
|
||||
verify_script="${ASSISTANT_RESTORE_VERIFY_SCRIPT:-/home/m/assistant-ops/verify-postgres-restore.sh}"
|
||||
|
||||
latest="$(find "$backup_root" -mindepth 1 -maxdepth 1 -type d ! -name '.*' -printf '%f\n' | sort | tail -n 1)"
|
||||
if [ -z "$latest" ]; then
|
||||
echo "No completed PostgreSQL backup found in $backup_root" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
exec "$verify_script" "$backup_root/$latest"
|
||||
Executable
+72
@@ -0,0 +1,72 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
|
||||
if [ "$#" -ne 1 ]; then
|
||||
echo "usage: $0 <backup-directory>" >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
backup_dir="$1"
|
||||
image="${ASSISTANT_RESTORE_IMAGE:-postgres:18.6-alpine}"
|
||||
container="assistant-postgres-restore-check-$(date -u +%Y%m%dT%H%M%SZ)-$$"
|
||||
password="$(od -An -N24 -tx1 /dev/urandom | tr -d ' \n')"
|
||||
created=false
|
||||
|
||||
cleanup() {
|
||||
if [ "$created" = true ]; then
|
||||
docker rm -f "$container" >/dev/null 2>&1 || true
|
||||
fi
|
||||
}
|
||||
trap cleanup EXIT HUP INT TERM
|
||||
|
||||
for file in assistant.dump globals.sql database.name owner.name SHA256SUMS; do
|
||||
if [ ! -f "$backup_dir/$file" ]; then
|
||||
echo "Backup file is missing: $file" >&2
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
(
|
||||
cd "$backup_dir"
|
||||
sha256sum -c SHA256SUMS
|
||||
)
|
||||
|
||||
database="$(sed -n '1p' "$backup_dir/database.name")"
|
||||
owner="$(sed -n '1p' "$backup_dir/owner.name")"
|
||||
[ -n "$database" ] && [ -n "$owner" ] || {
|
||||
echo "Database metadata is empty" >&2
|
||||
exit 1
|
||||
}
|
||||
case "$database:$owner" in
|
||||
*[!A-Za-z0-9_:.-]*) echo "Unsafe database metadata" >&2; exit 1 ;;
|
||||
esac
|
||||
|
||||
docker run -d --name "$container" \
|
||||
--tmpfs /var/lib/postgresql:rw,noexec,nosuid,size=1g \
|
||||
-e POSTGRES_PASSWORD="$password" \
|
||||
"$image" >/dev/null
|
||||
created=true
|
||||
|
||||
attempt=0
|
||||
until docker exec "$container" pg_isready -U postgres -d postgres >/dev/null 2>&1; do
|
||||
attempt=$((attempt + 1))
|
||||
if [ "$attempt" -ge 60 ]; then
|
||||
docker logs "$container" >&2
|
||||
echo "Restore-check PostgreSQL did not become ready" >&2
|
||||
exit 1
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
|
||||
docker exec -i "$container" psql -v ON_ERROR_STOP=1 -U postgres -d postgres <"$backup_dir/globals.sql" >/dev/null
|
||||
docker exec "$container" psql -v ON_ERROR_STOP=1 -U postgres -d postgres \
|
||||
-c "CREATE DATABASE \"$database\" OWNER \"$owner\"" >/dev/null
|
||||
docker exec -i "$container" pg_restore --exit-on-error --no-password -U postgres -d "$database" <"$backup_dir/assistant.dump" >/dev/null
|
||||
|
||||
schema_count="$(docker exec "$container" psql -U postgres -d "$database" -Atqc "select count(*) from information_schema.schemata where schema_name not in ('pg_catalog','information_schema') and schema_name not like 'pg_toast%'")"
|
||||
table_count="$(docker exec "$container" psql -U postgres -d "$database" -Atqc "select count(*) from information_schema.tables where table_schema not in ('pg_catalog','information_schema')")"
|
||||
if [ "$schema_count" -lt 1 ] || [ "$table_count" -lt 1 ]; then
|
||||
echo "Restored database lacks expected user objects" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
printf 'restore verified: database=%s schemas=%s tables=%s\n' "$database" "$schema_count" "$table_count"
|
||||
@@ -1,5 +1,5 @@
|
||||
{
|
||||
"task_file": "dev/tasks/DEV-000034-production-host-only-policy.json",
|
||||
"task_id": "DEV-000034",
|
||||
"task_file": "dev/tasks/DEV-000035-postgres-backup-recovery.json",
|
||||
"task_id": "DEV-000035",
|
||||
"task_revision": 1
|
||||
}
|
||||
|
||||
@@ -0,0 +1,78 @@
|
||||
{
|
||||
"task_id": "DEV-000035",
|
||||
"task_revision": 1,
|
||||
"architecture_revision": "ADR-0004",
|
||||
"base_repo_revision": "6303dfe6eb2215ae79ab63a2b92b86d7fafd312d",
|
||||
"risk_tier": "E",
|
||||
"intent": "Add production-only PostgreSQL backups with integrity manifests, scheduled execution, and isolated restore verification without touching canonical production data.",
|
||||
"requirement_ids": ["CONSTITUTION-1.3.36", "INV-OWN-001", "USER-DOCKER-PROD-ONLY"],
|
||||
"allowed_change_scope": {
|
||||
"modules": [],
|
||||
"paths": [
|
||||
"deploy/backup-postgres.sh",
|
||||
"deploy/verify-postgres-restore.sh",
|
||||
"deploy/verify-latest-postgres-backup.sh",
|
||||
"deploy/install-postgres-backup-cron.sh",
|
||||
"deploy/cron/assistant-postgres-backup.crontab",
|
||||
"docs/DEPLOYMENT.md",
|
||||
"dev/ACTIVE_TASK.json",
|
||||
"dev/tasks/DEV-000035-postgres-backup-recovery.json"
|
||||
]
|
||||
},
|
||||
"forbidden_change_scope": {
|
||||
"paths": [
|
||||
"architecture/**",
|
||||
"ci/**",
|
||||
".gitea/**",
|
||||
"cmd/**",
|
||||
"contracts/**",
|
||||
"modules/**",
|
||||
"database/**",
|
||||
"security/**"
|
||||
]
|
||||
},
|
||||
"allowed_architecture_delta": {
|
||||
"new_cross_module_dependencies": 0,
|
||||
"new_canonical_owners": 0,
|
||||
"new_privileges": 0
|
||||
},
|
||||
"contracts": {
|
||||
"consume": ["PostgreSQL logical backup format"],
|
||||
"may_change": []
|
||||
},
|
||||
"semantic_owner": "none; production recovery mechanism only",
|
||||
"canonical_state": ["all PostgreSQL owner schemas are captured without mutation"],
|
||||
"state_transitions": ["none"],
|
||||
"side_effects_and_privileges": ["reads the production PostgreSQL cluster through its existing local superuser context", "writes mode-0600 backup artifacts on docker.cin.su", "creates and removes an isolated temporary restore-check container"],
|
||||
"persistence_and_migrations": ["no migration", "daily logical dump plus role globals and SHA-256 manifest under /home/m/assistant-backups/postgres", "user crontab schedule survives host reboot without a persistent login session"],
|
||||
"failure_retry_restart": ["backup is written to a partial directory and atomically renamed only after all checksums succeed", "a failed backup leaves no object named as complete", "restore verification always removes its isolated temporary container", "no automatic deletion or retention policy is introduced"],
|
||||
"assumptions": [
|
||||
"The production database container remains assistant-postgres-1 unless explicitly overridden",
|
||||
"The production Docker host is docker.cin.su and docker-test is prohibited",
|
||||
"The host has Docker, sha256sum, and cron"
|
||||
],
|
||||
"acceptance_scenarios": [
|
||||
"a backup contains the application database dump, cluster roles, database and owner metadata, and verified hashes",
|
||||
"backup artifacts and directories are private to the production SSH user",
|
||||
"restore verification succeeds in a temporary PostgreSQL 18.6 container without mounting or writing the production volume",
|
||||
"the production user crontab schedules one daily backup at 03:15 Europe/Moscow and persists across reboot",
|
||||
"no backup retention deletion is performed",
|
||||
"exact diff validation observes only declared paths"
|
||||
],
|
||||
"required_tests": [
|
||||
"shell syntax",
|
||||
"architecture preflight",
|
||||
"exact diff scope",
|
||||
"live production backup",
|
||||
"SHA-256 verification",
|
||||
"isolated live restore",
|
||||
"cron schedule installation",
|
||||
"production application health",
|
||||
"CI"
|
||||
],
|
||||
"terminal_gap_states": [
|
||||
"BLOCKED_BY_MISSING_CAPABILITY",
|
||||
"BLOCKED_BY_CONFLICT",
|
||||
"REQUIRES_MIGRATION_DECISION"
|
||||
]
|
||||
}
|
||||
@@ -17,3 +17,11 @@ The initial stack definition is `deploy/docker-compose.prod.yml`. Portainer may
|
||||
Credentials/API deployment automation is deliberately not embedded. Use `PORTAINER_API_KEY` from an external secret source or deploy through Portainer UI/API after authenticating.
|
||||
|
||||
The single Creator login is configured with external `CREATOR_USERNAME` and `CREATOR_PASSWORD` values. They issue the same bounded Creator session and never select Actor, Principal, Scope or Authority. Keep `CREATOR_BOOTSTRAP_TOKEN` as a separate recovery/automation credential; none of these values belongs in Git, compose defaults for production, image layers or logs.
|
||||
|
||||
## PostgreSQL recovery
|
||||
|
||||
Production backups run only on `docker.cin.su`; this project must not use the shared `docker-test` host. `deploy/backup-postgres.sh` creates an atomic, mode-0600 logical backup under `/home/m/assistant-backups/postgres`. Each completed directory contains the application database, cluster roles without role passwords, database/owner metadata, and `SHA256SUMS`. Partial output is never promoted to a completed backup directory, and no automatic retention deletion is performed.
|
||||
|
||||
`deploy/verify-postgres-restore.sh <backup-directory>` verifies all hashes and restores into an isolated PostgreSQL 18.6 container backed by tmpfs. It never mounts or writes `assistant_assistant_pg`, and removes its temporary container on exit. A backup is operationally accepted only after this restore check passes.
|
||||
|
||||
The production user crontab installs the single entry from `deploy/cron/assistant-postgres-backup.crontab`. It runs the backup and restore check daily at 00:15 UTC (03:15 Europe/Moscow), survives host reboot without a persistent login session, and writes a private operational log beside the backups.
|
||||
|
||||
Reference in New Issue
Block a user