chore: sync homelab ops, identity, and monitoring docs

automation/bin/pd_backup_lib.sh

@@ -71,3 +71,8 @@ ssh_args() {
 ssh_cmd() {
   printf 'ssh %s' "$(ssh_args)"
 }
+
+latest_dump_for_db() {
+  local db="$1"
+  find "$PD_DB_DUMP_ROOT" -maxdepth 1 -type f -name "${db}_*.sql.gz" | sort | tail -n 1
+}

automation/bin/pd_restore_verify_appdata.sh

@@ -0,0 +1,34 @@
+#!/usr/bin/bash
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
+# shellcheck disable=SC1091
+source "$SCRIPT_DIR/pd_backup_lib.sh"
+
+verify_root="${RESTORE_VERIFY_WORK_ROOT:-/mnt/tank/docker/backups/restore-verify}"
+remote_source="${RESTORE_VERIFY_CONFIG_SOURCE:-tank-docker/appdata/grafana}"
+required_paths="${RESTORE_VERIFY_REQUIRED_PATHS:-grafana.db dashboards}"
+stage_dir="$verify_root/appdata-stage"
+
+mkdir -p "$verify_root"
+rm -rf "$stage_dir"
+mkdir -p "$stage_dir"
+
+remote_path="$SERENITY_BACKUP_ROOT/$remote_source/"
+
+echo "Staging config restore sample from $SERENITY_BACKUP_HOST:$remote_path"
+rsync -a -e "$(ssh_cmd)" "$SERENITY_BACKUP_HOST:$remote_path" "$stage_dir/"
+
+for rel in $required_paths; do
+  if [[ ! -e "$stage_dir/$rel" ]]; then
+    echo "Missing expected restored path: $rel" >&2
+    exit 1
+  fi
+done
+
+if [[ -f "$stage_dir/grafana.db" ]] && command -v sqlite3 >/dev/null 2>&1; then
+  sqlite3 "$stage_dir/grafana.db" 'PRAGMA integrity_check;' | grep -qx 'ok'
+  echo "Verified SQLite integrity for grafana.db"
+fi
+
+echo "Config restore verification passed for $remote_source."

automation/bin/pd_restore_verify_postgres.sh

@@ -0,0 +1,69 @@
+#!/usr/bin/bash
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
+# shellcheck disable=SC1091
+source "$SCRIPT_DIR/pd_backup_lib.sh"
+
+verify_root="${RESTORE_VERIFY_WORK_ROOT:-/mnt/tank/docker/backups/restore-verify}"
+dbs="${RESTORE_VERIFY_DB_LIST:-${POSTGRES_DB_LIST:-}}"
+
+if [[ -z "$dbs" ]]; then
+  echo "RESTORE_VERIFY_DB_LIST and POSTGRES_DB_LIST are both empty." >&2
+  exit 1
+fi
+
+mkdir -p "$verify_root"
+stamp="$(timestamp_utc)"
+container="pd-restore-verify-$stamp"
+password="verify-$stamp"
+
+if [[ -n "${RESTORE_VERIFY_POSTGRES_IMAGE:-}" ]]; then
+  verify_image="$RESTORE_VERIFY_POSTGRES_IMAGE"
+else
+  source_major="$(docker_cmd exec "$POSTGRES_CONTAINER" postgres -V | awk '{print $3}' | cut -d. -f1)"
+  verify_image="postgres:${source_major}-alpine"
+fi
+
+cleanup() {
+  docker_cmd rm -f "$container" >/dev/null 2>&1 || true
+}
+trap cleanup EXIT
+
+echo "Starting temporary Postgres verifier container $container"
+docker_cmd run -d --rm \
+  --name "$container" \
+  -e POSTGRES_PASSWORD="$password" \
+  "$verify_image" >/dev/null
+
+echo "Waiting for temporary Postgres to accept connections"
+for _ in {1..30}; do
+  if docker_cmd exec "$container" pg_isready -U postgres >/dev/null 2>&1; then
+    break
+  fi
+  sleep 2
+done
+
+docker_cmd exec "$container" pg_isready -U postgres >/dev/null
+
+for db in $dbs; do
+  dump_file="$(latest_dump_for_db "$db")"
+  if [[ -z "$dump_file" ]]; then
+    echo "No dump found for database '$db' under $PD_DB_DUMP_ROOT" >&2
+    exit 1
+  fi
+
+  echo "Restoring $dump_file into temporary database '$db'"
+  while IFS= read -r role_name; do
+    [[ -z "$role_name" ]] && continue
+    docker_cmd exec "$container" createuser -U postgres "$role_name" >/dev/null 2>&1 || true
+  done < <(gunzip -c "$dump_file" | sed -n "s/.*OWNER TO \\([^;]*\\);/\\1/p" | sort -u)
+
+  docker_cmd exec "$container" createdb -U postgres "$db"
+  gunzip -c "$dump_file" | docker_cmd exec -i "$container" psql -v ON_ERROR_STOP=1 -U postgres -d "$db" >/dev/null
+
+  table_count="$(docker_cmd exec "$container" psql -At -U postgres -d "$db" -c "SELECT count(*) FROM information_schema.tables WHERE table_schema NOT IN ('pg_catalog','information_schema');")"
+  echo "Verified database '$db' with $table_count non-system tables."
+done
+
+echo "Postgres restore verification passed."

automation/bin/run_pd_restore_verification.sh

@@ -0,0 +1,55 @@
+#!/usr/bin/bash
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
+# shellcheck disable=SC1091
+source "$SCRIPT_DIR/pd_backup_lib.sh"
+
+metrics_dir="${RESTORE_VERIFY_METRICS_DIR:-/mnt/tank/docker/metrics/node-exporter}"
+metrics_file="$metrics_dir/pd_restore_verification.prom"
+start_epoch="$(date +%s)"
+status=1
+
+mkdir -p "$metrics_dir"
+
+previous_success_epoch() {
+  if [[ -f "$metrics_file" ]]; then
+    awk '/^pd_restore_verification_last_success_timestamp_seconds /{print $2}' "$metrics_file" | tail -n 1
+  else
+    echo 0
+  fi
+}
+
+write_metrics() {
+  local end_epoch duration last_success tmp
+  end_epoch="$(date +%s)"
+  duration="$((end_epoch - start_epoch))"
+  last_success="$(previous_success_epoch)"
+  if [[ "$status" -eq 0 ]]; then
+    last_success="$end_epoch"
+  fi
+  tmp="${metrics_file}.tmp"
+  cat > "$tmp" <<EOF
+# HELP pd_restore_verification_success Last PD restore verification result (1=success, 0=failure).
+# TYPE pd_restore_verification_success gauge
+pd_restore_verification_success $(( status == 0 ? 1 : 0 ))
+# HELP pd_restore_verification_last_run_timestamp_seconds Unix timestamp of the last PD restore verification run.
+# TYPE pd_restore_verification_last_run_timestamp_seconds gauge
+pd_restore_verification_last_run_timestamp_seconds $end_epoch
+# HELP pd_restore_verification_last_success_timestamp_seconds Unix timestamp of the last successful PD restore verification run.
+# TYPE pd_restore_verification_last_success_timestamp_seconds gauge
+pd_restore_verification_last_success_timestamp_seconds $last_success
+# HELP pd_restore_verification_duration_seconds Duration of the last PD restore verification run in seconds.
+# TYPE pd_restore_verification_duration_seconds gauge
+pd_restore_verification_duration_seconds $duration
+EOF
+  mv "$tmp" "$metrics_file"
+}
+
+trap write_metrics EXIT
+
+"$SCRIPT_DIR/pd_restore_verify_postgres.sh"
+"$SCRIPT_DIR/pd_restore_verify_appdata.sh"
+
+status=0
+echo "PD restore verification run complete."