#!/usr/bin/env bash set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" STACK_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" PD_CONFIG_ROOT="${PD_CONFIG_ROOT:-/mnt/tank/docker/appdata/technitium-pilot/config}" LOG_ROOT="${LOG_ROOT:-/mnt/tank/docker/appdata/technitium-pilot/logs}" DIG_BIN="${DIG_BIN:-/usr/bin/dig}" RSYNC_BIN="${RSYNC_BIN:-/usr/bin/rsync}" SSH_BIN="${SSH_BIN:-/usr/bin/ssh}" DATE_BIN="${DATE_BIN:-/usr/bin/date}" MKDIR_BIN="${MKDIR_BIN:-/usr/bin/mkdir}" SSH_OPTS=(-o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new) RSYNC_EXCLUDES=(--exclude cache.bin --exclude 'stats/' --exclude 'blocklists/') NOMAD_HOST="${NOMAD_HOST:-fizzlepoof@10.5.30.7}" NOMAD_KEY="${NOMAD_KEY:-/home/truenas_admin/.ssh/nomad_technitium_sync_ed25519}" NOMAD_STACK_ROOT="${NOMAD_STACK_ROOT:-/opt/technitium-nomad}" NOMAD_DNS_IP="${NOMAD_DNS_IP:-10.5.30.9}" SERENITY_HOST="${SERENITY_HOST:-root@10.5.30.5}" SERENITY_KEY="${SERENITY_KEY:-/home/truenas_admin/.ssh/serenity_backup_ed25519}" SERENITY_STACK_ROOT="${SERENITY_STACK_ROOT:-/mnt/user/appdata/technitium-serenity}" SERENITY_DNS_IP="${SERENITY_DNS_IP:-10.5.30.10}" DRY_RUN=false ONLY_TARGET="all" SKIP_RESTART=false usage() { cat <<'EOF' Usage: sync_backup_nodes.sh [--dry-run] [--target nomad|serenity|all] [--skip-restart] Push the live PD Technitium config to the Nomad and Serenity backup Technitium nodes, then recycle their compose stacks and verify both private and public DNS answers. EOF } while [[ $# -gt 0 ]]; do case "$1" in --dry-run) DRY_RUN=true ;; --target) ONLY_TARGET="${2:-}" shift ;; --skip-restart) SKIP_RESTART=true ;; -h|--help) usage exit 0 ;; *) echo "Unknown argument: $1" >&2 usage >&2 exit 1 ;; esac shift done if [[ ! -d "$PD_CONFIG_ROOT" ]]; then echo "PD config root missing: $PD_CONFIG_ROOT" >&2 exit 1 fi if [[ ! -x "$RSYNC_BIN" ]]; then echo "rsync missing at $RSYNC_BIN" >&2 exit 1 fi if [[ ! -x "$SSH_BIN" ]]; then echo "ssh missing at $SSH_BIN" >&2 exit 1 fi if [[ ! -x "$DIG_BIN" ]]; then echo "dig missing at $DIG_BIN" >&2 exit 1 fi $MKDIR_BIN -p "$LOG_ROOT" STAMP="$($DATE_BIN -u +%Y-%m-%dT%H:%M:%SZ)" log() { printf '[%s] %s\n' "$($DATE_BIN -u +%Y-%m-%dT%H:%M:%SZ)" "$*" } remote_prepare_nomad() { "$SSH_BIN" -i "$NOMAD_KEY" "${SSH_OPTS[@]}" "$NOMAD_HOST" \ "sudo -n mkdir -p '$NOMAD_STACK_ROOT/data/config' '$NOMAD_STACK_ROOT/data/logs'" } remote_prepare_serenity() { "$SSH_BIN" -i "$SERENITY_KEY" "${SSH_OPTS[@]}" "$SERENITY_HOST" \ "mkdir -p '$SERENITY_STACK_ROOT/data/config' '$SERENITY_STACK_ROOT/data/logs'" } remote_restart_nomad() { "$SSH_BIN" -i "$NOMAD_KEY" "${SSH_OPTS[@]}" "$NOMAD_HOST" \ "printf '%s\n' '$STAMP' | sudo -n tee '$NOMAD_STACK_ROOT/data/last_pd_sync_utc.txt' >/dev/null && \ cd '$NOMAD_STACK_ROOT' && \ sudo -n docker compose --env-file .env down && \ sudo -n docker compose --env-file .env up -d" } remote_restart_serenity() { "$SSH_BIN" -i "$SERENITY_KEY" "${SSH_OPTS[@]}" "$SERENITY_HOST" \ "printf '%s\n' '$STAMP' > '$SERENITY_STACK_ROOT/data/last_pd_sync_utc.txt' && \ cd '$SERENITY_STACK_ROOT' && \ docker compose --env-file .env down && \ docker compose --env-file .env up -d" } verify_dns() { local label="$1" local dns_ip="$2" local private_tmp="/tmp/technitium-sync-${label}-private.txt" local public_tmp="/tmp/technitium-sync-${label}-public.txt" local attempt log "Verifying $label at $dns_ip" for attempt in {1..10}; do if "$DIG_BIN" +time=2 +tries=1 @"$dns_ip" dns.home.paccoco.com A >"$private_tmp" 2>/dev/null && \ "$DIG_BIN" +time=2 +tries=1 @"$dns_ip" cloudflare.com A >"$public_tmp" 2>/dev/null && \ grep -q 'status: NOERROR' "$private_tmp" && \ grep -q 'status: NOERROR' "$public_tmp" && \ grep -q '10.5.30.8' "$private_tmp"; then return 0 fi sleep 2 done echo "Verification failed for $label ($dns_ip) after retries" >&2 cat "$private_tmp" 2>/dev/null >&2 || true echo '---' >&2 cat "$public_tmp" 2>/dev/null >&2 || true return 1 } sync_nomad() { log "Preparing Nomad target" remote_prepare_nomad local rsync_cmd=("$RSYNC_BIN" -a --delete "${RSYNC_EXCLUDES[@]}" -e "$SSH_BIN -i $NOMAD_KEY ${SSH_OPTS[*]}" --rsync-path="sudo -n $RSYNC_BIN" "$PD_CONFIG_ROOT/" "$NOMAD_HOST:$NOMAD_STACK_ROOT/data/config/") if $DRY_RUN; then rsync_cmd=("$RSYNC_BIN" -an --delete "${RSYNC_EXCLUDES[@]}" -e "$SSH_BIN -i $NOMAD_KEY ${SSH_OPTS[*]}" --rsync-path="sudo -n $RSYNC_BIN" "$PD_CONFIG_ROOT/" "$NOMAD_HOST:$NOMAD_STACK_ROOT/data/config/") fi log "Syncing Nomad config" "${rsync_cmd[@]}" if ! $DRY_RUN && ! $SKIP_RESTART; then log "Recycling Nomad Technitium stack" remote_restart_nomad verify_dns nomad "$NOMAD_DNS_IP" fi } sync_serenity() { log "Preparing Serenity target" remote_prepare_serenity local rsync_cmd=("$RSYNC_BIN" -a --delete "${RSYNC_EXCLUDES[@]}" -e "$SSH_BIN -i $SERENITY_KEY ${SSH_OPTS[*]}" "$PD_CONFIG_ROOT/" "$SERENITY_HOST:$SERENITY_STACK_ROOT/data/config/") if $DRY_RUN; then rsync_cmd=("$RSYNC_BIN" -an --delete "${RSYNC_EXCLUDES[@]}" -e "$SSH_BIN -i $SERENITY_KEY ${SSH_OPTS[*]}" "$PD_CONFIG_ROOT/" "$SERENITY_HOST:$SERENITY_STACK_ROOT/data/config/") fi log "Syncing Serenity config" "${rsync_cmd[@]}" if ! $DRY_RUN && ! $SKIP_RESTART; then log "Recycling Serenity Technitium stack" remote_restart_serenity verify_dns serenity "$SERENITY_DNS_IP" fi } case "$ONLY_TARGET" in all) sync_nomad sync_serenity ;; nomad) sync_nomad ;; serenity) sync_serenity ;; *) echo "Invalid --target value: $ONLY_TARGET" >&2 exit 1 ;; esac log "Technitium backup sync finished successfully"