truenas-stacks/technitium-pilot/bin/sync_backup_nodes.sh

#!/usr/bin/env bash
set -euo pipefail

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
STACK_DIR="$(cd "$SCRIPT_DIR/.." && pwd)"

PD_CONFIG_ROOT="${PD_CONFIG_ROOT:-/mnt/tank/docker/appdata/technitium-pilot/config}"
LOG_ROOT="${LOG_ROOT:-/mnt/tank/docker/appdata/technitium-pilot/logs}"
DIG_BIN="${DIG_BIN:-/usr/bin/dig}"
RSYNC_BIN="${RSYNC_BIN:-/usr/bin/rsync}"
SSH_BIN="${SSH_BIN:-/usr/bin/ssh}"
DATE_BIN="${DATE_BIN:-/usr/bin/date}"
MKDIR_BIN="${MKDIR_BIN:-/usr/bin/mkdir}"

SSH_OPTS=(-o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new)
RSYNC_EXCLUDES=(--exclude cache.bin --exclude 'stats/' --exclude 'blocklists/')

NOMAD_HOST="${NOMAD_HOST:-fizzlepoof@10.5.30.7}"
NOMAD_KEY="${NOMAD_KEY:-/home/truenas_admin/.ssh/nomad_technitium_sync_ed25519}"
NOMAD_STACK_ROOT="${NOMAD_STACK_ROOT:-/opt/technitium-nomad}"
NOMAD_DNS_IP="${NOMAD_DNS_IP:-10.5.30.9}"

SERENITY_HOST="${SERENITY_HOST:-root@10.5.30.5}"
SERENITY_KEY="${SERENITY_KEY:-/home/truenas_admin/.ssh/serenity_backup_ed25519}"
SERENITY_STACK_ROOT="${SERENITY_STACK_ROOT:-/mnt/user/appdata/technitium-serenity}"
SERENITY_DNS_IP="${SERENITY_DNS_IP:-10.5.30.10}"

DRY_RUN=false
ONLY_TARGET="all"
SKIP_RESTART=false

usage() {
  cat <<'EOF'
Usage: sync_backup_nodes.sh [--dry-run] [--target nomad|serenity|all] [--skip-restart]

Push the live PD Technitium config to the Nomad and Serenity backup Technitium nodes,
then recycle their compose stacks and verify both private and public DNS answers.
EOF
}

while [[ $# -gt 0 ]]; do
  case "$1" in
    --dry-run)
      DRY_RUN=true
      ;;
    --target)
      ONLY_TARGET="${2:-}"
      shift
      ;;
    --skip-restart)
      SKIP_RESTART=true
      ;;
    -h|--help)
      usage
      exit 0
      ;;
    *)
      echo "Unknown argument: $1" >&2
      usage >&2
      exit 1
      ;;
  esac
  shift
done

if [[ ! -d "$PD_CONFIG_ROOT" ]]; then
  echo "PD config root missing: $PD_CONFIG_ROOT" >&2
  exit 1
fi

if [[ ! -x "$RSYNC_BIN" ]]; then
  echo "rsync missing at $RSYNC_BIN" >&2
  exit 1
fi

if [[ ! -x "$SSH_BIN" ]]; then
  echo "ssh missing at $SSH_BIN" >&2
  exit 1
fi

if [[ ! -x "$DIG_BIN" ]]; then
  echo "dig missing at $DIG_BIN" >&2
  exit 1
fi

$MKDIR_BIN -p "$LOG_ROOT"
STAMP="$($DATE_BIN -u +%Y-%m-%dT%H:%M:%SZ)"

log() {
  printf '[%s] %s\n' "$($DATE_BIN -u +%Y-%m-%dT%H:%M:%SZ)" "$*"
}

remote_prepare_nomad() {
  "$SSH_BIN" -i "$NOMAD_KEY" "${SSH_OPTS[@]}" "$NOMAD_HOST" \
    "sudo -n mkdir -p '$NOMAD_STACK_ROOT/data/config' '$NOMAD_STACK_ROOT/data/logs'"
}

remote_prepare_serenity() {
  "$SSH_BIN" -i "$SERENITY_KEY" "${SSH_OPTS[@]}" "$SERENITY_HOST" \
    "mkdir -p '$SERENITY_STACK_ROOT/data/config' '$SERENITY_STACK_ROOT/data/logs'"
}

remote_restart_nomad() {
  "$SSH_BIN" -i "$NOMAD_KEY" "${SSH_OPTS[@]}" "$NOMAD_HOST" \
    "printf '%s\n' '$STAMP' | sudo -n tee '$NOMAD_STACK_ROOT/data/last_pd_sync_utc.txt' >/dev/null && \
     cd '$NOMAD_STACK_ROOT' && \
     sudo -n docker compose --env-file .env down && \
     sudo -n docker compose --env-file .env up -d"
}

remote_restart_serenity() {
  "$SSH_BIN" -i "$SERENITY_KEY" "${SSH_OPTS[@]}" "$SERENITY_HOST" \
    "printf '%s\n' '$STAMP' > '$SERENITY_STACK_ROOT/data/last_pd_sync_utc.txt' && \
     cd '$SERENITY_STACK_ROOT' && \
     docker compose --env-file .env down && \
     docker compose --env-file .env up -d"
}

verify_dns() {
  local label="$1"
  local dns_ip="$2"
  local private_tmp="/tmp/technitium-sync-${label}-private.txt"
  local public_tmp="/tmp/technitium-sync-${label}-public.txt"
  local attempt
  log "Verifying $label at $dns_ip"
  for attempt in {1..10}; do
    if "$DIG_BIN" +time=2 +tries=1 @"$dns_ip" dns.home.paccoco.com A >"$private_tmp" 2>/dev/null && \
       "$DIG_BIN" +time=2 +tries=1 @"$dns_ip" cloudflare.com A >"$public_tmp" 2>/dev/null && \
       grep -q 'status: NOERROR' "$private_tmp" && \
       grep -q 'status: NOERROR' "$public_tmp" && \
       grep -q '10.5.30.8' "$private_tmp"; then
      return 0
    fi
    sleep 2
  done
  echo "Verification failed for $label ($dns_ip) after retries" >&2
  cat "$private_tmp" 2>/dev/null >&2 || true
  echo '---' >&2
  cat "$public_tmp" 2>/dev/null >&2 || true
  return 1
}

sync_nomad() {
  log "Preparing Nomad target"
  remote_prepare_nomad
  local rsync_cmd=("$RSYNC_BIN" -a --delete "${RSYNC_EXCLUDES[@]}" -e "$SSH_BIN -i $NOMAD_KEY ${SSH_OPTS[*]}" --rsync-path="sudo -n $RSYNC_BIN" "$PD_CONFIG_ROOT/" "$NOMAD_HOST:$NOMAD_STACK_ROOT/data/config/")
  if $DRY_RUN; then
    rsync_cmd=("$RSYNC_BIN" -an --delete "${RSYNC_EXCLUDES[@]}" -e "$SSH_BIN -i $NOMAD_KEY ${SSH_OPTS[*]}" --rsync-path="sudo -n $RSYNC_BIN" "$PD_CONFIG_ROOT/" "$NOMAD_HOST:$NOMAD_STACK_ROOT/data/config/")
  fi
  log "Syncing Nomad config"
  "${rsync_cmd[@]}"
  if ! $DRY_RUN && ! $SKIP_RESTART; then
    log "Recycling Nomad Technitium stack"
    remote_restart_nomad
    verify_dns nomad "$NOMAD_DNS_IP"
  fi
}

sync_serenity() {
  log "Preparing Serenity target"
  remote_prepare_serenity
  local rsync_cmd=("$RSYNC_BIN" -a --delete "${RSYNC_EXCLUDES[@]}" -e "$SSH_BIN -i $SERENITY_KEY ${SSH_OPTS[*]}" "$PD_CONFIG_ROOT/" "$SERENITY_HOST:$SERENITY_STACK_ROOT/data/config/")
  if $DRY_RUN; then
    rsync_cmd=("$RSYNC_BIN" -an --delete "${RSYNC_EXCLUDES[@]}" -e "$SSH_BIN -i $SERENITY_KEY ${SSH_OPTS[*]}" "$PD_CONFIG_ROOT/" "$SERENITY_HOST:$SERENITY_STACK_ROOT/data/config/")
  fi
  log "Syncing Serenity config"
  "${rsync_cmd[@]}"
  if ! $DRY_RUN && ! $SKIP_RESTART; then
    log "Recycling Serenity Technitium stack"
    remote_restart_serenity
    verify_dns serenity "$SERENITY_DNS_IP"
  fi
}

case "$ONLY_TARGET" in
  all)
    sync_nomad
    sync_serenity
    ;;
  nomad)
    sync_nomad
    ;;
  serenity)
    sync_serenity
    ;;
  *)
    echo "Invalid --target value: $ONLY_TARGET" >&2
    exit 1
    ;;
esac

log "Technitium backup sync finished successfully"