Add Technitium backup sync automation

technitium-pilot/bin/sync_backup_nodes.sh

@@ -0,0 +1,192 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+STACK_DIR="$(cd "$SCRIPT_DIR/.." && pwd)"
+
+PD_CONFIG_ROOT="${PD_CONFIG_ROOT:-/mnt/tank/docker/appdata/technitium-pilot/config}"
+LOG_ROOT="${LOG_ROOT:-/mnt/tank/docker/appdata/technitium-pilot/logs}"
+DIG_BIN="${DIG_BIN:-/usr/bin/dig}"
+RSYNC_BIN="${RSYNC_BIN:-/usr/bin/rsync}"
+SSH_BIN="${SSH_BIN:-/usr/bin/ssh}"
+DATE_BIN="${DATE_BIN:-/usr/bin/date}"
+MKDIR_BIN="${MKDIR_BIN:-/usr/bin/mkdir}"
+
+SSH_OPTS=(-o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new)
+RSYNC_EXCLUDES=(--exclude cache.bin --exclude 'stats/' --exclude 'blocklists/')
+
+NOMAD_HOST="${NOMAD_HOST:-fizzlepoof@10.5.30.7}"
+NOMAD_KEY="${NOMAD_KEY:-/home/truenas_admin/.ssh/nomad_technitium_sync_ed25519}"
+NOMAD_STACK_ROOT="${NOMAD_STACK_ROOT:-/opt/technitium-nomad}"
+NOMAD_DNS_IP="${NOMAD_DNS_IP:-10.5.30.9}"
+
+SERENITY_HOST="${SERENITY_HOST:-root@10.5.30.5}"
+SERENITY_KEY="${SERENITY_KEY:-/home/truenas_admin/.ssh/serenity_backup_ed25519}"
+SERENITY_STACK_ROOT="${SERENITY_STACK_ROOT:-/mnt/user/appdata/technitium-serenity}"
+SERENITY_DNS_IP="${SERENITY_DNS_IP:-10.5.30.10}"
+
+DRY_RUN=false
+ONLY_TARGET="all"
+SKIP_RESTART=false
+
+usage() {
+  cat <<'EOF'
+Usage: sync_backup_nodes.sh [--dry-run] [--target nomad|serenity|all] [--skip-restart]
+
+Push the live PD Technitium config to the Nomad and Serenity backup Technitium nodes,
+then recycle their compose stacks and verify both private and public DNS answers.
+EOF
+}
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --dry-run)
+      DRY_RUN=true
+      ;;
+    --target)
+      ONLY_TARGET="${2:-}"
+      shift
+      ;;
+    --skip-restart)
+      SKIP_RESTART=true
+      ;;
+    -h|--help)
+      usage
+      exit 0
+      ;;
+    *)
+      echo "Unknown argument: $1" >&2
+      usage >&2
+      exit 1
+      ;;
+  esac
+  shift
+done
+
+if [[ ! -d "$PD_CONFIG_ROOT" ]]; then
+  echo "PD config root missing: $PD_CONFIG_ROOT" >&2
+  exit 1
+fi
+
+if [[ ! -x "$RSYNC_BIN" ]]; then
+  echo "rsync missing at $RSYNC_BIN" >&2
+  exit 1
+fi
+
+if [[ ! -x "$SSH_BIN" ]]; then
+  echo "ssh missing at $SSH_BIN" >&2
+  exit 1
+fi
+
+if [[ ! -x "$DIG_BIN" ]]; then
+  echo "dig missing at $DIG_BIN" >&2
+  exit 1
+fi
+
+$MKDIR_BIN -p "$LOG_ROOT"
+STAMP="$($DATE_BIN -u +%Y-%m-%dT%H:%M:%SZ)"
+
+log() {
+  printf '[%s] %s\n' "$($DATE_BIN -u +%Y-%m-%dT%H:%M:%SZ)" "$*"
+}
+
+remote_prepare_nomad() {
+  "$SSH_BIN" -i "$NOMAD_KEY" "${SSH_OPTS[@]}" "$NOMAD_HOST" \
+    "sudo -n mkdir -p '$NOMAD_STACK_ROOT/data/config' '$NOMAD_STACK_ROOT/data/logs'"
+}
+
+remote_prepare_serenity() {
+  "$SSH_BIN" -i "$SERENITY_KEY" "${SSH_OPTS[@]}" "$SERENITY_HOST" \
+    "mkdir -p '$SERENITY_STACK_ROOT/data/config' '$SERENITY_STACK_ROOT/data/logs'"
+}
+
+remote_restart_nomad() {
+  "$SSH_BIN" -i "$NOMAD_KEY" "${SSH_OPTS[@]}" "$NOMAD_HOST" \
+    "printf '%s\n' '$STAMP' | sudo -n tee '$NOMAD_STACK_ROOT/data/last_pd_sync_utc.txt' >/dev/null && \
+     cd '$NOMAD_STACK_ROOT' && \
+     sudo -n docker compose --env-file .env down && \
+     sudo -n docker compose --env-file .env up -d"
+}
+
+remote_restart_serenity() {
+  "$SSH_BIN" -i "$SERENITY_KEY" "${SSH_OPTS[@]}" "$SERENITY_HOST" \
+    "printf '%s\n' '$STAMP' > '$SERENITY_STACK_ROOT/data/last_pd_sync_utc.txt' && \
+     cd '$SERENITY_STACK_ROOT' && \
+     docker compose --env-file .env down && \
+     docker compose --env-file .env up -d"
+}
+
+verify_dns() {
+  local label="$1"
+  local dns_ip="$2"
+  local private_tmp="/tmp/technitium-sync-${label}-private.txt"
+  local public_tmp="/tmp/technitium-sync-${label}-public.txt"
+  local attempt
+  log "Verifying $label at $dns_ip"
+  for attempt in {1..10}; do
+    if "$DIG_BIN" +time=2 +tries=1 @"$dns_ip" dns.home.paccoco.com A >"$private_tmp" 2>/dev/null && \
+       "$DIG_BIN" +time=2 +tries=1 @"$dns_ip" cloudflare.com A >"$public_tmp" 2>/dev/null && \
+       grep -q 'status: NOERROR' "$private_tmp" && \
+       grep -q 'status: NOERROR' "$public_tmp" && \
+       grep -q '10.5.30.8' "$private_tmp"; then
+      return 0
+    fi
+    sleep 2
+  done
+  echo "Verification failed for $label ($dns_ip) after retries" >&2
+  cat "$private_tmp" 2>/dev/null >&2 || true
+  echo '---' >&2
+  cat "$public_tmp" 2>/dev/null >&2 || true
+  return 1
+}
+
+sync_nomad() {
+  log "Preparing Nomad target"
+  remote_prepare_nomad
+  local rsync_cmd=("$RSYNC_BIN" -a --delete "${RSYNC_EXCLUDES[@]}" -e "$SSH_BIN -i $NOMAD_KEY ${SSH_OPTS[*]}" --rsync-path="sudo -n $RSYNC_BIN" "$PD_CONFIG_ROOT/" "$NOMAD_HOST:$NOMAD_STACK_ROOT/data/config/")
+  if $DRY_RUN; then
+    rsync_cmd=("$RSYNC_BIN" -an --delete "${RSYNC_EXCLUDES[@]}" -e "$SSH_BIN -i $NOMAD_KEY ${SSH_OPTS[*]}" --rsync-path="sudo -n $RSYNC_BIN" "$PD_CONFIG_ROOT/" "$NOMAD_HOST:$NOMAD_STACK_ROOT/data/config/")
+  fi
+  log "Syncing Nomad config"
+  "${rsync_cmd[@]}"
+  if ! $DRY_RUN && ! $SKIP_RESTART; then
+    log "Recycling Nomad Technitium stack"
+    remote_restart_nomad
+    verify_dns nomad "$NOMAD_DNS_IP"
+  fi
+}
+
+sync_serenity() {
+  log "Preparing Serenity target"
+  remote_prepare_serenity
+  local rsync_cmd=("$RSYNC_BIN" -a --delete "${RSYNC_EXCLUDES[@]}" -e "$SSH_BIN -i $SERENITY_KEY ${SSH_OPTS[*]}" "$PD_CONFIG_ROOT/" "$SERENITY_HOST:$SERENITY_STACK_ROOT/data/config/")
+  if $DRY_RUN; then
+    rsync_cmd=("$RSYNC_BIN" -an --delete "${RSYNC_EXCLUDES[@]}" -e "$SSH_BIN -i $SERENITY_KEY ${SSH_OPTS[*]}" "$PD_CONFIG_ROOT/" "$SERENITY_HOST:$SERENITY_STACK_ROOT/data/config/")
+  fi
+  log "Syncing Serenity config"
+  "${rsync_cmd[@]}"
+  if ! $DRY_RUN && ! $SKIP_RESTART; then
+    log "Recycling Serenity Technitium stack"
+    remote_restart_serenity
+    verify_dns serenity "$SERENITY_DNS_IP"
+  fi
+}
+
+case "$ONLY_TARGET" in
+  all)
+    sync_nomad
+    sync_serenity
+    ;;
+  nomad)
+    sync_nomad
+    ;;
+  serenity)
+    sync_serenity
+    ;;
+  *)
+    echo "Invalid --target value: $ONLY_TARGET" >&2
+    exit 1
+    ;;
+esac
+
+log "Technitium backup sync finished successfully"