chore: sync homelab ops, identity, and monitoring docs

automation/.env.example

@@ -35,3 +35,10 @@ BACKUP_KNOWN_HOSTS=/home/truenas_admin/.ssh/known_hosts
 DOCKER_BIN=/usr/bin/docker
 SUDO_BIN=/usr/bin/sudo
 USE_SUDO_FOR_DOCKER=true
+
+# Restore verification
+RESTORE_VERIFY_WORK_ROOT=/mnt/tank/docker/backups/restore-verify
+RESTORE_VERIFY_POSTGRES_IMAGE=postgres:17-alpine
+RESTORE_VERIFY_CONFIG_SOURCE=tank-docker/appdata/grafana
+RESTORE_VERIFY_REQUIRED_PATHS="grafana.db dashboards"
+RESTORE_VERIFY_METRICS_DIR=/mnt/tank/docker/metrics/node-exporter

automation/README.md

@@ -33,6 +33,10 @@ automation/bin/karakeep_api.py /api/health --raw-path --pretty
 - `bin/pd_backup_postgres.sh` — creates gzipped `pg_dump` exports for the configured Postgres DB list (quote space-separated DB names in `.env`, e.g. `POSTGRES_DB_LIST="n8n paperless"`)
 - `bin/pd_backup_sync_to_serenity.sh` — `rsync`s configured backup/config roots to Serenity
 - `bin/run_pd_backups.sh` — wrapper that runs both steps in order
+- `bin/pd_restore_verify_postgres.sh` — restores the latest dump for each configured database into a throwaway Postgres container and fails on SQL errors
+- `bin/pd_restore_verify_appdata.sh` — stages one appdata/config tree back from Serenity and verifies required files exist
+- `bin/run_pd_restore_verification.sh` — wrapper that runs both restore checks in order
+  - writes Prometheus textfile metrics for run status, duration, and last successful verification timestamp
 
 These scripts are the live PD backup runner model.
 
@@ -49,6 +53,7 @@ Current documented live state:
 - the PD → Serenity backup flow is the intended deployed model
 - root cron on PD is the preferred scheduler
 - first-run verification and the deployed status are tracked in `docs/operations/PD_BACKUP_DEPLOYMENT.md` and `docs/planning/TODO.md`
+- quarterly restore verification is now expected via `bin/run_pd_restore_verification.sh`
 
 ## PD / TrueNAS deployment recommendation
 
@@ -93,6 +98,34 @@ Install in **root's crontab on PD**:
 # END PD BACKUPS
 ```
 
+## Recommended quarterly restore verification
+
+Run from PD after a backup has completed:
+
+```bash
+cd /mnt/docker-ssd/docker/compose/automation
+/usr/bin/bash bin/run_pd_restore_verification.sh
+```
+
+This validates:
+
+- latest configured Postgres dumps can be restored cleanly
+- a real config/appdata sample can be pulled back from Serenity
+- expected files still exist in the restored copy
+
+If node-exporter is configured with the textfile collector, the wrapper also publishes:
+
+- `pd_restore_verification_success`
+- `pd_restore_verification_last_run_timestamp_seconds`
+- `pd_restore_verification_last_success_timestamp_seconds`
+- `pd_restore_verification_duration_seconds`
+
+Recommended root cron entry on PD:
+
+```cron
+30 3 1 */3 * cd /mnt/docker-ssd/docker/compose/automation && /usr/bin/bash bin/run_pd_restore_verification.sh >> /mnt/tank/docker/backups/pd-restore-verify.log 2>&1
+```
+
 Recommended first-run checklist:
 
 ```bash

automation/bin/pd_backup_lib.sh

@@ -71,3 +71,8 @@ ssh_args() {
 ssh_cmd() {
   printf 'ssh %s' "$(ssh_args)"
 }
+
+latest_dump_for_db() {
+  local db="$1"
+  find "$PD_DB_DUMP_ROOT" -maxdepth 1 -type f -name "${db}_*.sql.gz" | sort | tail -n 1
+}

automation/bin/pd_restore_verify_appdata.sh

@@ -0,0 +1,34 @@
+#!/usr/bin/bash
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
+# shellcheck disable=SC1091
+source "$SCRIPT_DIR/pd_backup_lib.sh"
+
+verify_root="${RESTORE_VERIFY_WORK_ROOT:-/mnt/tank/docker/backups/restore-verify}"
+remote_source="${RESTORE_VERIFY_CONFIG_SOURCE:-tank-docker/appdata/grafana}"
+required_paths="${RESTORE_VERIFY_REQUIRED_PATHS:-grafana.db dashboards}"
+stage_dir="$verify_root/appdata-stage"
+
+mkdir -p "$verify_root"
+rm -rf "$stage_dir"
+mkdir -p "$stage_dir"
+
+remote_path="$SERENITY_BACKUP_ROOT/$remote_source/"
+
+echo "Staging config restore sample from $SERENITY_BACKUP_HOST:$remote_path"
+rsync -a -e "$(ssh_cmd)" "$SERENITY_BACKUP_HOST:$remote_path" "$stage_dir/"
+
+for rel in $required_paths; do
+  if [[ ! -e "$stage_dir/$rel" ]]; then
+    echo "Missing expected restored path: $rel" >&2
+    exit 1
+  fi
+done
+
+if [[ -f "$stage_dir/grafana.db" ]] && command -v sqlite3 >/dev/null 2>&1; then
+  sqlite3 "$stage_dir/grafana.db" 'PRAGMA integrity_check;' | grep -qx 'ok'
+  echo "Verified SQLite integrity for grafana.db"
+fi
+
+echo "Config restore verification passed for $remote_source."

automation/bin/pd_restore_verify_postgres.sh

@@ -0,0 +1,69 @@
+#!/usr/bin/bash
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
+# shellcheck disable=SC1091
+source "$SCRIPT_DIR/pd_backup_lib.sh"
+
+verify_root="${RESTORE_VERIFY_WORK_ROOT:-/mnt/tank/docker/backups/restore-verify}"
+dbs="${RESTORE_VERIFY_DB_LIST:-${POSTGRES_DB_LIST:-}}"
+
+if [[ -z "$dbs" ]]; then
+  echo "RESTORE_VERIFY_DB_LIST and POSTGRES_DB_LIST are both empty." >&2
+  exit 1
+fi
+
+mkdir -p "$verify_root"
+stamp="$(timestamp_utc)"
+container="pd-restore-verify-$stamp"
+password="verify-$stamp"
+
+if [[ -n "${RESTORE_VERIFY_POSTGRES_IMAGE:-}" ]]; then
+  verify_image="$RESTORE_VERIFY_POSTGRES_IMAGE"
+else
+  source_major="$(docker_cmd exec "$POSTGRES_CONTAINER" postgres -V | awk '{print $3}' | cut -d. -f1)"
+  verify_image="postgres:${source_major}-alpine"
+fi
+
+cleanup() {
+  docker_cmd rm -f "$container" >/dev/null 2>&1 || true
+}
+trap cleanup EXIT
+
+echo "Starting temporary Postgres verifier container $container"
+docker_cmd run -d --rm \
+  --name "$container" \
+  -e POSTGRES_PASSWORD="$password" \
+  "$verify_image" >/dev/null
+
+echo "Waiting for temporary Postgres to accept connections"
+for _ in {1..30}; do
+  if docker_cmd exec "$container" pg_isready -U postgres >/dev/null 2>&1; then
+    break
+  fi
+  sleep 2
+done
+
+docker_cmd exec "$container" pg_isready -U postgres >/dev/null
+
+for db in $dbs; do
+  dump_file="$(latest_dump_for_db "$db")"
+  if [[ -z "$dump_file" ]]; then
+    echo "No dump found for database '$db' under $PD_DB_DUMP_ROOT" >&2
+    exit 1
+  fi
+
+  echo "Restoring $dump_file into temporary database '$db'"
+  while IFS= read -r role_name; do
+    [[ -z "$role_name" ]] && continue
+    docker_cmd exec "$container" createuser -U postgres "$role_name" >/dev/null 2>&1 || true
+  done < <(gunzip -c "$dump_file" | sed -n "s/.*OWNER TO \\([^;]*\\);/\\1/p" | sort -u)
+
+  docker_cmd exec "$container" createdb -U postgres "$db"
+  gunzip -c "$dump_file" | docker_cmd exec -i "$container" psql -v ON_ERROR_STOP=1 -U postgres -d "$db" >/dev/null
+
+  table_count="$(docker_cmd exec "$container" psql -At -U postgres -d "$db" -c "SELECT count(*) FROM information_schema.tables WHERE table_schema NOT IN ('pg_catalog','information_schema');")"
+  echo "Verified database '$db' with $table_count non-system tables."
+done
+
+echo "Postgres restore verification passed."

automation/bin/run_pd_restore_verification.sh

@@ -0,0 +1,55 @@
+#!/usr/bin/bash
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
+# shellcheck disable=SC1091
+source "$SCRIPT_DIR/pd_backup_lib.sh"
+
+metrics_dir="${RESTORE_VERIFY_METRICS_DIR:-/mnt/tank/docker/metrics/node-exporter}"
+metrics_file="$metrics_dir/pd_restore_verification.prom"
+start_epoch="$(date +%s)"
+status=1
+
+mkdir -p "$metrics_dir"
+
+previous_success_epoch() {
+  if [[ -f "$metrics_file" ]]; then
+    awk '/^pd_restore_verification_last_success_timestamp_seconds /{print $2}' "$metrics_file" | tail -n 1
+  else
+    echo 0
+  fi
+}
+
+write_metrics() {
+  local end_epoch duration last_success tmp
+  end_epoch="$(date +%s)"
+  duration="$((end_epoch - start_epoch))"
+  last_success="$(previous_success_epoch)"
+  if [[ "$status" -eq 0 ]]; then
+    last_success="$end_epoch"
+  fi
+  tmp="${metrics_file}.tmp"
+  cat > "$tmp" <<EOF
+# HELP pd_restore_verification_success Last PD restore verification result (1=success, 0=failure).
+# TYPE pd_restore_verification_success gauge
+pd_restore_verification_success $(( status == 0 ? 1 : 0 ))
+# HELP pd_restore_verification_last_run_timestamp_seconds Unix timestamp of the last PD restore verification run.
+# TYPE pd_restore_verification_last_run_timestamp_seconds gauge
+pd_restore_verification_last_run_timestamp_seconds $end_epoch
+# HELP pd_restore_verification_last_success_timestamp_seconds Unix timestamp of the last successful PD restore verification run.
+# TYPE pd_restore_verification_last_success_timestamp_seconds gauge
+pd_restore_verification_last_success_timestamp_seconds $last_success
+# HELP pd_restore_verification_duration_seconds Duration of the last PD restore verification run in seconds.
+# TYPE pd_restore_verification_duration_seconds gauge
+pd_restore_verification_duration_seconds $duration
+EOF
+  mv "$tmp" "$metrics_file"
+}
+
+trap write_metrics EXIT
+
+"$SCRIPT_DIR/pd_restore_verify_postgres.sh"
+"$SCRIPT_DIR/pd_restore_verify_appdata.sh"
+
+status=0
+echo "PD restore verification run complete."