Add PD to Serenity backup automation scaffolding

2026-05-15 03:08:03 +00:00
parent 295a22bc1f
commit 5f52b2e6cf
6 changed files with 250 additions and 0 deletions
--- a/automation/bin/pd_backup_lib.sh
+++ b/automation/bin/pd_backup_lib.sh
@@ -0,0 +1,73 @@
+#!/usr/bin/bash
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+AUTOMATION_DIR="$(cd "$SCRIPT_DIR/.." && pwd)"
+ENV_FILE="${ENV_FILE:-$AUTOMATION_DIR/.env}"
+
+if [[ ! -f "$ENV_FILE" ]]; then
+  echo "Missing env file: $ENV_FILE" >&2
+  echo "Copy automation/.env.example to automation/.env and fill in real values first." >&2
+  exit 1
+fi
+
+set -a
+# shellcheck disable=SC1090
+source "$ENV_FILE"
+set +a
+
+required_vars=(
+  SERENITY_BACKUP_HOST
+  SERENITY_BACKUP_ROOT
+  PD_APPDATA_ROOT
+  PD_DATABASES_ROOT
+  PD_TANK_DOCKER_ROOT
+  PD_DB_DUMP_ROOT
+  POSTGRES_CONTAINER
+)
+
+for var in "${required_vars[@]}"; do
+  if [[ -z "${!var:-}" ]]; then
+    echo "Required variable $var is empty in $ENV_FILE" >&2
+    exit 1
+  fi
+done
+
+mkdir -p "$PD_DB_DUMP_ROOT"
+
+DOCKER_BIN="${DOCKER_BIN:-/usr/bin/docker}"
+SUDO_BIN="${SUDO_BIN:-/usr/bin/sudo}"
+USE_SUDO_FOR_DOCKER="${USE_SUDO_FOR_DOCKER:-true}"
+BACKUP_KNOWN_HOSTS="${BACKUP_KNOWN_HOSTS:-${HOME:-/root}/.ssh/known_hosts}"
+
+if [[ ! -x "$DOCKER_BIN" ]]; then
+  echo "Docker binary not executable: $DOCKER_BIN" >&2
+  exit 1
+fi
+
+docker_cmd() {
+  if [[ "${EUID:-$(id -u)}" == "0" ]]; then
+    "$DOCKER_BIN" "$@"
+  elif [[ "$USE_SUDO_FOR_DOCKER" == "true" ]]; then
+    "$SUDO_BIN" -n "$DOCKER_BIN" "$@"
+  else
+    "$DOCKER_BIN" "$@"
+  fi
+}
+
+timestamp_utc() {
+  date -u +"%Y%m%dT%H%M%SZ"
+}
+
+ssh_args() {
+  local opts="-o BatchMode=yes -o StrictHostKeyChecking=accept-new -o UserKnownHostsFile=$BACKUP_KNOWN_HOSTS"
+  if [[ -n "${BACKUP_SSH_KEY:-}" ]]; then
+    printf '%s' "-i $BACKUP_SSH_KEY $opts"
+  else
+    printf '%s' "$opts"
+  fi
+}
+
+ssh_cmd() {
+  printf 'ssh %s' "$(ssh_args)"
+}
--- a/automation/bin/pd_backup_postgres.sh
+++ b/automation/bin/pd_backup_postgres.sh
@@ -0,0 +1,22 @@
+#!/usr/bin/bash
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
+# shellcheck disable=SC1091
+source "$SCRIPT_DIR/pd_backup_lib.sh"
+
+stamp="$(timestamp_utc)"
+dbs="${POSTGRES_DB_LIST:-}"
+
+if [[ -z "$dbs" ]]; then
+  echo "POSTGRES_DB_LIST is empty; nothing to dump." >&2
+  exit 1
+fi
+
+for db in $dbs; do
+  out="$PD_DB_DUMP_ROOT/${db}_${stamp}.sql.gz"
+  echo "Dumping Postgres database '$db' to $out"
+  docker_cmd exec "$POSTGRES_CONTAINER" pg_dump -U postgres "$db" | gzip -9 > "$out"
+done
+
+echo "Postgres dumps complete: $PD_DB_DUMP_ROOT"
--- a/automation/bin/pd_backup_sync_to_serenity.sh
+++ b/automation/bin/pd_backup_sync_to_serenity.sh
@@ -0,0 +1,51 @@
+#!/usr/bin/bash
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
+# shellcheck disable=SC1091
+source "$SCRIPT_DIR/pd_backup_lib.sh"
+
+ssh_opts="$(ssh_args)"
+remote_root="$SERENITY_BACKUP_HOST:$SERENITY_BACKUP_ROOT"
+
+# shellcheck disable=SC2086
+ssh $ssh_opts "$SERENITY_BACKUP_HOST" mkdir -p \
+  "$SERENITY_BACKUP_ROOT/appdata" \
+  "$SERENITY_BACKUP_ROOT/databases" \
+  "$SERENITY_BACKUP_ROOT/tank-docker" \
+  "$SERENITY_BACKUP_ROOT/db-dumps"
+
+rsync_cmd=(
+  rsync -a --delete
+  --exclude '*.sqlite-journal'
+  --exclude '*.db-journal'
+  --exclude '*.sqlite-wal'
+  --exclude '*.db-wal'
+  --exclude '*.tmp'
+)
+rsync_cmd+=( -e "$(ssh_cmd)" )
+
+run_rsync() {
+  local label="$1"
+  shift
+  echo "$label"
+  set +e
+  "${rsync_cmd[@]}" "$@"
+  local rc=$?
+  set -e
+  if [[ $rc -eq 24 ]]; then
+    echo "rsync completed with vanished-file warning (code 24); treating as acceptable for live appdata."
+    return 0
+  fi
+  return $rc
+}
+
+run_rsync "Syncing appdata to Serenity" "$PD_APPDATA_ROOT/" "$remote_root/appdata/"
+
+run_rsync "Syncing database volume roots to Serenity" "$PD_DATABASES_ROOT/" "$remote_root/databases/"
+
+run_rsync "Syncing selected tank docker data to Serenity" "$PD_TANK_DOCKER_ROOT/" "$remote_root/tank-docker/"
+
+run_rsync "Syncing DB dumps to Serenity" "$PD_DB_DUMP_ROOT/" "$remote_root/db-dumps/"
+
+echo "Serenity sync complete: $remote_root"
--- a/automation/bin/run_pd_backups.sh
+++ b/automation/bin/run_pd_backups.sh
@@ -0,0 +1,9 @@
+#!/usr/bin/bash
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
+
+"$SCRIPT_DIR/pd_backup_postgres.sh"
+"$SCRIPT_DIR/pd_backup_sync_to_serenity.sh"
+
+echo "PD backup run complete."
--- a/docs/operations/BACKUP_POLICY.md
+++ b/docs/operations/BACKUP_POLICY.md
@@ -26,6 +26,16 @@ Back up at minimum:

 Recommended dump locations: `/mnt/tank/docker/backups/db-dumps/`

+Repo-side scaffolding now lives under `automation/`:
+
+- `automation/bin/pd_backup_postgres.sh`
+- `automation/bin/pd_backup_sync_to_serenity.sh`
+- `automation/bin/run_pd_backups.sh`
+
+These are staged helper scripts only until they are deployed with a real `.env`, SSH trust to Serenity, and a live scheduler on PD.
+
+Recommended live deployment model on PD: a plain **root cron** job invoking `/usr/bin/bash bin/run_pd_backups.sh` from `/mnt/docker-ssd/docker/compose/automation`.
+
 ## .env File Backup Options
 1. **Private Gitea repo** — short term, push all .env files to a private repo on your local Gitea
 2. **Encrypted restic** — include .env files in restic backup with encryption, push offsite (Backblaze B2)
--- a/docs/operations/PD_BACKUP_DEPLOYMENT.md
+++ b/docs/operations/PD_BACKUP_DEPLOYMENT.md
@@ -0,0 +1,85 @@
+# PD Backup Deployment
+
+Deploy the PD → Serenity backup runner from the real compose path on PD:
+
+```bash
+/mnt/docker-ssd/docker/compose/automation
+```
+
+## Why this model
+
+- PD is the compose/data host, so backups should originate there.
+- TrueNAS SCALE is happier with plain cron + shell than extra appliance-fighting service glue.
+- Root cron on PD avoids the common `sudo`/Docker socket permission mess.
+- Backups should still run even if n8n is down.
+
+## Prerequisites
+
+1. Repo content present on PD at `/mnt/docker-ssd/docker/compose/automation`
+2. Writable dump directory:
+   ```bash
+   mkdir -p /mnt/tank/docker/backups/db-dumps
+   ```
+3. SSH trust from PD to Serenity for the selected account
+4. Real `.env` created from `.env.example`
+5. Preferred: install the cron job in root's crontab on PD
+6. If using a non-root PD account instead, `sudo -n /usr/bin/docker` must work
+
+## Suggested `.env` values to review
+
+```dotenv
+SERENITY_BACKUP_HOST=root@10.5.1.5
+SERENITY_BACKUP_ROOT=/mnt/user/backups/plausible-deniability
+PD_APPDATA_ROOT=/mnt/docker-ssd/docker/appdata
+PD_DATABASES_ROOT=/mnt/docker-ssd/docker/databases
+PD_TANK_DOCKER_ROOT=/mnt/tank/docker
+PD_DB_DUMP_ROOT=/mnt/tank/docker/backups/db-dumps
+POSTGRES_CONTAINER=shared-postgres
+POSTGRES_DB_LIST="n8n paperless"
+BACKUP_SSH_KEY=/home/truenas_admin/.ssh/serenity_backup_ed25519
+DOCKER_BIN=/usr/bin/docker
+SUDO_BIN=/usr/bin/sudo
+USE_SUDO_FOR_DOCKER=true
+```
+
+## Manual first run
+
+```bash
+cd /mnt/docker-ssd/docker/compose/automation
+cp .env.example .env
+chmod 600 .env
+mkdir -p /mnt/tank/docker/backups/db-dumps
+/usr/bin/bash bin/run_pd_backups.sh
+```
+
+Confirm:
+
+- fresh `*.sql.gz` files appear in `/mnt/tank/docker/backups/db-dumps`
+- appdata/database/tank-docker content lands under the chosen Serenity backup root
+- no sudo prompt appeared during `docker exec`
+
+## Cron install
+
+Preferred: install in **root's crontab** on PD.
+
+```cron
+# BEGIN PD BACKUPS
+15 2 * * * cd /mnt/docker-ssd/docker/compose/automation && /usr/bin/bash bin/run_pd_backups.sh >> /mnt/tank/docker/backups/pd-backups.log 2>&1
+# END PD BACKUPS
+```
+
+## Post-deploy checks
+
+```bash
+tail -100 /mnt/tank/docker/backups/pd-backups.log
+ls -lh /mnt/tank/docker/backups/db-dumps
+ssh root@10.5.1.5 'find /mnt/user/backups/plausible-deniability -maxdepth 2 -type d | sort | head -40'
+```
+
+## Important limitations
+
+- Do **not** use `systemctl restart docker` on PD.
+- Keep source/destination paths under `/mnt/...`.
+- `rsync --delete` is intentional; use a dedicated destination root, not a shared miscellaneous folder.
+- This scaffolding does not yet back up `.env` files here because those are already handled by the separate encrypted/private-repo process.
+- If you insist on a non-root cron user on PD, you will need passwordless Docker access (`sudo -n /usr/bin/docker ...`) and write access to the chosen dump directory.