chore: sync homelab ops, identity, and monitoring docs

automation/bin/run_pd_restore_verification.sh

@@ -0,0 +1,55 @@
+#!/usr/bin/bash
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
+# shellcheck disable=SC1091
+source "$SCRIPT_DIR/pd_backup_lib.sh"
+
+metrics_dir="${RESTORE_VERIFY_METRICS_DIR:-/mnt/tank/docker/metrics/node-exporter}"
+metrics_file="$metrics_dir/pd_restore_verification.prom"
+start_epoch="$(date +%s)"
+status=1
+
+mkdir -p "$metrics_dir"
+
+previous_success_epoch() {
+  if [[ -f "$metrics_file" ]]; then
+    awk '/^pd_restore_verification_last_success_timestamp_seconds /{print $2}' "$metrics_file" | tail -n 1
+  else
+    echo 0
+  fi
+}
+
+write_metrics() {
+  local end_epoch duration last_success tmp
+  end_epoch="$(date +%s)"
+  duration="$((end_epoch - start_epoch))"
+  last_success="$(previous_success_epoch)"
+  if [[ "$status" -eq 0 ]]; then
+    last_success="$end_epoch"
+  fi
+  tmp="${metrics_file}.tmp"
+  cat > "$tmp" <<EOF
+# HELP pd_restore_verification_success Last PD restore verification result (1=success, 0=failure).
+# TYPE pd_restore_verification_success gauge
+pd_restore_verification_success $(( status == 0 ? 1 : 0 ))
+# HELP pd_restore_verification_last_run_timestamp_seconds Unix timestamp of the last PD restore verification run.
+# TYPE pd_restore_verification_last_run_timestamp_seconds gauge
+pd_restore_verification_last_run_timestamp_seconds $end_epoch
+# HELP pd_restore_verification_last_success_timestamp_seconds Unix timestamp of the last successful PD restore verification run.
+# TYPE pd_restore_verification_last_success_timestamp_seconds gauge
+pd_restore_verification_last_success_timestamp_seconds $last_success
+# HELP pd_restore_verification_duration_seconds Duration of the last PD restore verification run in seconds.
+# TYPE pd_restore_verification_duration_seconds gauge
+pd_restore_verification_duration_seconds $duration
+EOF
+  mv "$tmp" "$metrics_file"
+}
+
+trap write_metrics EXIT
+
+"$SCRIPT_DIR/pd_restore_verify_postgres.sh"
+"$SCRIPT_DIR/pd_restore_verify_appdata.sh"
+
+status=0
+echo "PD restore verification run complete."