ops: add verified PostgreSQL backups
CI / test (push) Successful in 10s

This commit is contained in:
m
2026-08-16 17:53:40 +03:00
parent 6303dfe6eb
commit e9594306e5
8 changed files with 271 additions and 2 deletions
+66
View File
@@ -0,0 +1,66 @@
#!/bin/sh
set -eu
umask 077
container="${ASSISTANT_POSTGRES_CONTAINER:-assistant-postgres-1}"
backup_root="${ASSISTANT_BACKUP_DIR:-/home/m/assistant-backups/postgres}"
timestamp="$(date -u +%Y%m%dT%H%M%SZ)"
partial_dir="${backup_root}/.${timestamp}.partial"
backup_dir="${backup_root}/${timestamp}"
cleanup() {
if [ -d "$partial_dir" ]; then
rm -f -- "$partial_dir/assistant.dump" "$partial_dir/globals.sql" \
"$partial_dir/database.name" "$partial_dir/owner.name" "$partial_dir/SHA256SUMS"
rmdir -- "$partial_dir" 2>/dev/null || true
fi
}
trap cleanup EXIT HUP INT TERM
if ! docker inspect "$container" >/dev/null 2>&1; then
echo "PostgreSQL container not found: $container" >&2
exit 1
fi
if [ "$(docker inspect -f '{{.State.Running}}' "$container")" != "true" ]; then
echo "PostgreSQL container is not running: $container" >&2
exit 1
fi
mkdir -p -- "$backup_root"
chmod 700 -- "$backup_root"
if [ -e "$partial_dir" ] || [ -e "$backup_dir" ]; then
echo "Backup destination already exists for $timestamp" >&2
exit 1
fi
mkdir -- "$partial_dir"
database="$(docker exec "$container" sh -c 'printf %s "$POSTGRES_DB"')"
owner="$(docker exec "$container" sh -c 'printf %s "$POSTGRES_USER"')"
[ -n "$database" ] && [ -n "$owner" ] || {
echo "Database metadata is empty" >&2
exit 1
}
case "$database:$owner" in
*[!A-Za-z0-9_:.-]*) echo "Unsafe database metadata" >&2; exit 1 ;;
esac
printf '%s\n' "$database" >"$partial_dir/database.name"
printf '%s\n' "$owner" >"$partial_dir/owner.name"
docker exec "$container" sh -c 'exec pg_dump --format=custom --compress=9 --no-password -U "$POSTGRES_USER" -d "$POSTGRES_DB"' >"$partial_dir/assistant.dump"
docker exec "$container" sh -c 'exec pg_dumpall --globals-only --no-role-passwords --no-password -U "$POSTGRES_USER"' >"$partial_dir/globals.sql"
if [ ! -s "$partial_dir/assistant.dump" ] || [ ! -s "$partial_dir/globals.sql" ]; then
echo "Backup output is incomplete" >&2
exit 1
fi
(
cd "$partial_dir"
sha256sum assistant.dump globals.sql database.name owner.name >SHA256SUMS
sha256sum -c SHA256SUMS >/dev/null
)
chmod 600 "$partial_dir"/*
mv -- "$partial_dir" "$backup_dir"
trap - EXIT HUP INT TERM
printf '%s\n' "$backup_dir"
@@ -0,0 +1,2 @@
# Domovoy Assistant PostgreSQL backup: 00:15 UTC is 03:15 Europe/Moscow.
15 0 * * * umask 077; { /home/m/assistant-ops/backup-postgres.sh && /home/m/assistant-ops/verify-latest-postgres-backup.sh; } >>/home/m/assistant-backups/postgres/backup.log 2>&1
+30
View File
@@ -0,0 +1,30 @@
#!/bin/sh
set -eu
schedule_file="${1:-/home/m/assistant-ops/assistant-postgres-backup.crontab}"
marker="/home/m/assistant-ops/backup-postgres.sh"
current="$(mktemp)"
updated="$(mktemp)"
cleanup() {
rm -f -- "$current" "$updated"
}
trap cleanup EXIT HUP INT TERM
if [ ! -f "$schedule_file" ]; then
echo "Cron schedule file not found: $schedule_file" >&2
exit 1
fi
crontab -l >"$current" 2>/dev/null || true
grep -vF "$marker" "$current" >"$updated" || true
cat "$schedule_file" >>"$updated"
crontab "$updated"
matches="$(crontab -l | grep -Fc "$marker")"
if [ "$matches" -ne 1 ]; then
echo "Expected one backup cron entry, found $matches" >&2
exit 1
fi
echo "PostgreSQL backup cron entry installed"
+13
View File
@@ -0,0 +1,13 @@
#!/bin/sh
set -eu
backup_root="${ASSISTANT_BACKUP_DIR:-/home/m/assistant-backups/postgres}"
verify_script="${ASSISTANT_RESTORE_VERIFY_SCRIPT:-/home/m/assistant-ops/verify-postgres-restore.sh}"
latest="$(find "$backup_root" -mindepth 1 -maxdepth 1 -type d ! -name '.*' -printf '%f\n' | sort | tail -n 1)"
if [ -z "$latest" ]; then
echo "No completed PostgreSQL backup found in $backup_root" >&2
exit 1
fi
exec "$verify_script" "$backup_root/$latest"
+72
View File
@@ -0,0 +1,72 @@
#!/bin/sh
set -eu
if [ "$#" -ne 1 ]; then
echo "usage: $0 <backup-directory>" >&2
exit 2
fi
backup_dir="$1"
image="${ASSISTANT_RESTORE_IMAGE:-postgres:18.6-alpine}"
container="assistant-postgres-restore-check-$(date -u +%Y%m%dT%H%M%SZ)-$$"
password="$(od -An -N24 -tx1 /dev/urandom | tr -d ' \n')"
created=false
cleanup() {
if [ "$created" = true ]; then
docker rm -f "$container" >/dev/null 2>&1 || true
fi
}
trap cleanup EXIT HUP INT TERM
for file in assistant.dump globals.sql database.name owner.name SHA256SUMS; do
if [ ! -f "$backup_dir/$file" ]; then
echo "Backup file is missing: $file" >&2
exit 1
fi
done
(
cd "$backup_dir"
sha256sum -c SHA256SUMS
)
database="$(sed -n '1p' "$backup_dir/database.name")"
owner="$(sed -n '1p' "$backup_dir/owner.name")"
[ -n "$database" ] && [ -n "$owner" ] || {
echo "Database metadata is empty" >&2
exit 1
}
case "$database:$owner" in
*[!A-Za-z0-9_:.-]*) echo "Unsafe database metadata" >&2; exit 1 ;;
esac
docker run -d --name "$container" \
--tmpfs /var/lib/postgresql:rw,noexec,nosuid,size=1g \
-e POSTGRES_PASSWORD="$password" \
"$image" >/dev/null
created=true
attempt=0
until docker exec "$container" pg_isready -U postgres -d postgres >/dev/null 2>&1; do
attempt=$((attempt + 1))
if [ "$attempt" -ge 60 ]; then
docker logs "$container" >&2
echo "Restore-check PostgreSQL did not become ready" >&2
exit 1
fi
sleep 1
done
docker exec -i "$container" psql -v ON_ERROR_STOP=1 -U postgres -d postgres <"$backup_dir/globals.sql" >/dev/null
docker exec "$container" psql -v ON_ERROR_STOP=1 -U postgres -d postgres \
-c "CREATE DATABASE \"$database\" OWNER \"$owner\"" >/dev/null
docker exec -i "$container" pg_restore --exit-on-error --no-password -U postgres -d "$database" <"$backup_dir/assistant.dump" >/dev/null
schema_count="$(docker exec "$container" psql -U postgres -d "$database" -Atqc "select count(*) from information_schema.schemata where schema_name not in ('pg_catalog','information_schema') and schema_name not like 'pg_toast%'")"
table_count="$(docker exec "$container" psql -U postgres -d "$database" -Atqc "select count(*) from information_schema.tables where table_schema not in ('pg_catalog','information_schema')")"
if [ "$schema_count" -lt 1 ] || [ "$table_count" -lt 1 ]; then
echo "Restored database lacks expected user objects" >&2
exit 1
fi
printf 'restore verified: database=%s schemas=%s tables=%s\n' "$database" "$schema_count" "$table_count"