Move Pi-hole backup node from Serenity to NOMAD

pihole/.env.example

@@ -19,7 +19,12 @@ KEEPALIVED_NOPREEMPT=true
 
 # Nebula Sync (PD primary -> replicas)
 NEBULA_SYNC_PRIMARY=http://127.0.0.1:8953|CHANGE_ME
-NEBULA_SYNC_REPLICAS=http://10.5.1.5:8954|CHANGE_ME
+NEBULA_SYNC_REPLICAS=http://10.5.1.16:8954|CHANGE_ME
 NEBULA_SYNC_FULL_SYNC=false
 NEBULA_SYNC_RUN_GRAVITY=true
 NEBULA_SYNC_CRON=0 3 * * *
+NEBULA_SYNC_GRAVITY_GROUP=true
+NEBULA_SYNC_GRAVITY_AD_LIST=true
+NEBULA_SYNC_GRAVITY_AD_LIST_BY_GROUP=true
+NEBULA_SYNC_GRAVITY_DOMAIN_LIST=true
+NEBULA_SYNC_GRAVITY_DOMAIN_LIST_BY_GROUP=true

pihole/README.md

@@ -13,7 +13,7 @@ This stack intentionally covers the **PD primary** plus PD-hosted sync:
 - Keepalived
 - Nebula Sync
 
-Serenity and RPi4 replicas still need their own deployment steps.
+NOMAD and RPi4 replicas still need their own deployment steps.
 
 ## Files
 
@@ -46,4 +46,5 @@ If that validates cleanly, bring it up:
 - `FTLCONF_dns_listeningMode=LOCAL` is set because TrueNAS binds loopback on port 53.
 - Pi-hole NTP is explicitly disabled here because PD host `chronyd` already owns UDP 123.
 - Nebula Sync runs on PD and uses `NEBULA_SYNC_PRIMARY` / `NEBULA_SYNC_REPLICAS` from `.env`.
+- Nebula Sync is intentionally set to selective mode with gravity sync toggles enabled for groups, adlists, and domain lists so replica-specific runtime/network settings are not clobbered.
 - `keepalived.conf` is generated and should stay local/untracked.

pihole/docker-compose.yaml

@@ -75,3 +75,8 @@ services:
       FULL_SYNC: ${NEBULA_SYNC_FULL_SYNC}
       RUN_GRAVITY: ${NEBULA_SYNC_RUN_GRAVITY}
       CRON: ${NEBULA_SYNC_CRON}
+      SYNC_GRAVITY_GROUP: ${NEBULA_SYNC_GRAVITY_GROUP}
+      SYNC_GRAVITY_AD_LIST: ${NEBULA_SYNC_GRAVITY_AD_LIST}
+      SYNC_GRAVITY_AD_LIST_BY_GROUP: ${NEBULA_SYNC_GRAVITY_AD_LIST_BY_GROUP}
+      SYNC_GRAVITY_DOMAIN_LIST: ${NEBULA_SYNC_GRAVITY_DOMAIN_LIST}
+      SYNC_GRAVITY_DOMAIN_LIST_BY_GROUP: ${NEBULA_SYNC_GRAVITY_DOMAIN_LIST_BY_GROUP}

pihole/nomad/.env.example

@@ -0,0 +1,12 @@
+TZ=America/Chicago
+APPDATA_PIHOLE_ROOT=/mnt/hdd-2/docker/appdata/pihole-nomad
+APPDATA_UNBOUND_ROOT=/mnt/hdd-2/docker/appdata/unbound-nomad
+NOMAD_PIHOLE_WEB_PORT=8954
+PIHOLE_PASSWORD=CHANGE_ME
+LAN_INTERFACE=enp5s0
+VIP_CIDR=10.5.1.53/24
+KEEPALIVED_ROUTER_ID=53
+KEEPALIVED_STATE=BACKUP
+KEEPALIVED_PRIORITY=90
+KEEPALIVED_PASSWORD=CHANGE01
+KEEPALIVED_NOPREEMPT=true

pihole/nomad/README.md

@@ -0,0 +1,16 @@
+# NOMAD Pi-hole / Unbound / Keepalived Replica
+
+- **Repo stack path:** `/home/fizzlepoof/repos/truenas-stacks/pihole/nomad`
+- **Suggested live stack path:** `/mnt/hdd-1/docker/compose/pihole-nomad`
+- **Node role:** NOMAD backup / BACKUP1
+- **VIP target:** `10.5.1.53/24`
+- **LAN interface:** `enp5s0`
+- **NOMAD LAN IP:** `10.5.1.16`
+
+Bring-up:
+
+```bash
+cd /mnt/hdd-1/docker/compose/pihole-nomad
+cp .env.example .env
+bash bin/prepare_nomad.sh --up
+```

pihole/nomad/bin/prepare_nomad.sh

@@ -0,0 +1,34 @@
+#!/usr/bin/env bash
+set -euo pipefail
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+STACK_DIR="$(cd "$SCRIPT_DIR/.." && pwd)"
+ENV_FILE="${ENV_FILE:-$STACK_DIR/.env}"
+DO_UP="${1:-}"
+
+if [[ ! -f "$ENV_FILE" ]]; then
+  echo "Missing $ENV_FILE — copy .env.example to .env and fill in real values first." >&2
+  exit 1
+fi
+
+set -a
+# shellcheck disable=SC1090
+source "$ENV_FILE"
+set +a
+
+mkdir -p \
+  "$APPDATA_PIHOLE_ROOT/etc-pihole" \
+  "$APPDATA_PIHOLE_ROOT/etc-dnsmasq.d" \
+  "$APPDATA_UNBOUND_ROOT/var"
+UNBOUND_TARGET="$APPDATA_UNBOUND_ROOT/unbound.conf"
+UNBOUND_TMP="$APPDATA_UNBOUND_ROOT/.unbound.conf.tmp.$$"
+cp "$STACK_DIR/unbound.conf" "$UNBOUND_TMP"
+mv -f "$UNBOUND_TMP" "$UNBOUND_TARGET"
+if [[ ! -s "$APPDATA_UNBOUND_ROOT/var/root.hints" ]]; then
+  curl -fsSL -o "$APPDATA_UNBOUND_ROOT/var/root.hints" https://www.internic.net/domain/named.cache
+fi
+ENV_FILE="$ENV_FILE" python3 "$STACK_DIR/bin/render_keepalived_conf.py"
+docker compose --env-file "$ENV_FILE" -f "$STACK_DIR/docker-compose.yaml" config >/dev/null
+if [[ "$DO_UP" == "--up" ]]; then
+  docker compose --env-file "$ENV_FILE" -f "$STACK_DIR/docker-compose.yaml" up -d
+  docker compose --env-file "$ENV_FILE" -f "$STACK_DIR/docker-compose.yaml" ps
+fi

pihole/nomad/bin/render_keepalived_conf.py

@@ -0,0 +1,73 @@
+#!/usr/bin/env python3
+from __future__ import annotations
+import os
+import re
+import sys
+from pathlib import Path
+
+STACK_DIR = Path(__file__).resolve().parent.parent
+ENV_FILE = Path(os.environ.get('ENV_FILE', STACK_DIR / '.env'))
+TEMPLATE = STACK_DIR / 'keepalived.conf.template'
+OUTPUT = STACK_DIR / 'keepalived.conf'
+
+
+def load_env(path: Path) -> dict[str, str]:
+    env = dict(os.environ)
+    if path.exists():
+        for raw in path.read_text(encoding='utf-8').splitlines():
+            line = raw.strip()
+            if not line or line.startswith('#') or '=' not in line:
+                continue
+            k, v = line.split('=', 1)
+            env.setdefault(k.strip(), v.strip().strip('"').strip("'"))
+    return env
+
+
+def main() -> int:
+    env = load_env(ENV_FILE)
+    required = [
+        'LAN_INTERFACE', 'VIP_CIDR', 'KEEPALIVED_ROUTER_ID', 'KEEPALIVED_STATE',
+        'KEEPALIVED_PRIORITY', 'KEEPALIVED_PASSWORD'
+    ]
+    missing = [k for k in required if not env.get(k)]
+    if missing:
+        print(f'Missing required vars in {ENV_FILE}: {", ".join(missing)}', file=sys.stderr)
+        return 1
+
+    web_port = env.get('PIHOLE_WEB_PORT') or env.get('PD_PIHOLE_WEB_PORT') or env.get('SERENITY_PIHOLE_WEB_PORT') or env.get('RPI4_PIHOLE_WEB_PORT')
+    if not web_port:
+        print('Missing Pi-hole web port in .env (set PIHOLE_WEB_PORT or node-specific port variable).', file=sys.stderr)
+        return 1
+
+    keepalived_password = env['KEEPALIVED_PASSWORD']
+    if len(keepalived_password) > 8:
+        print('KEEPALIVED_PASSWORD must be 8 characters or fewer for VRRP PASS auth.', file=sys.stderr)
+        return 1
+
+    try:
+        web_port_hex = f"{int(web_port):04X}"
+    except ValueError:
+        print('Pi-hole web port must be an integer.', file=sys.stderr)
+        return 1
+
+    router_id_name = f"pihole_{re.sub(r'[^a-z0-9]+', '_', env['KEEPALIVED_STATE'].lower())}"
+    nopreempt_line = '    nopreempt' if env.get('KEEPALIVED_NOPREEMPT', 'true').lower() in {'1','true','yes','on'} else ''
+
+    rendered = TEMPLATE.read_text(encoding='utf-8')
+    replacements = {
+        'ROUTER_ID_NAME': router_id_name,
+        'PIHOLE_WEB_PORT_HEX': web_port_hex,
+        'NOPREEMPT_LINE': nopreempt_line,
+    }
+    replacements.update({k: env[k] for k in required})
+    replacements['PIHOLE_WEB_PORT'] = web_port
+    for key, value in replacements.items():
+        rendered = rendered.replace('${' + key + '}', value)
+
+    OUTPUT.write_text(rendered.rstrip() + '\n', encoding='utf-8')
+    print(f'Rendered {OUTPUT}')
+    return 0
+
+
+if __name__ == '__main__':
+    raise SystemExit(main())

pihole/nomad/docker-compose.yaml

@@ -0,0 +1,62 @@
+services:
+  unbound:
+    image: mvance/unbound:latest
+    container_name: unbound-pihole-nomad
+    network_mode: host
+    restart: unless-stopped
+    environment:
+      TZ: ${TZ}
+      UNBOUND_PORT: "5335"
+    volumes:
+      - ${APPDATA_UNBOUND_ROOT}:/opt/unbound/etc/unbound/
+    healthcheck:
+      test: ["CMD-SHELL", "hex=$$(printf '%04X' \"$$UNBOUND_PORT\"); grep -qi \":$$hex\" /proc/net/tcp /proc/net/tcp6 || exit 1"]
+      interval: 30s
+      timeout: 10s
+      retries: 6
+      start_period: 20s
+
+  pihole:
+    image: pihole/pihole:latest
+    container_name: pihole-nomad
+    network_mode: host
+    restart: unless-stopped
+    depends_on:
+      unbound:
+        condition: service_healthy
+    environment:
+      TZ: ${TZ}
+      PIHOLE_WEB_PORT: ${NOMAD_PIHOLE_WEB_PORT}
+      FTLCONF_webserver_api_password: ${PIHOLE_PASSWORD}
+      FTLCONF_dns_listeningMode: "LOCAL"
+      FTLCONF_dns_upstreams: "127.0.0.1#5335"
+      FTLCONF_webserver_port: ${NOMAD_PIHOLE_WEB_PORT}
+      FTLCONF_ntp_ipv4_active: "false"
+      FTLCONF_ntp_ipv6_active: "false"
+      FTLCONF_ntp_sync_active: "false"
+    volumes:
+      - ${APPDATA_PIHOLE_ROOT}/etc-pihole:/etc/pihole
+      - ${APPDATA_PIHOLE_ROOT}/etc-dnsmasq.d:/etc/dnsmasq.d
+    cap_add:
+      - SYS_NICE
+    healthcheck:
+      test: ["CMD-SHELL", "hex=$$(printf '%04X' \"$$PIHOLE_WEB_PORT\"); grep -qi \":$$hex\" /proc/net/tcp /proc/net/tcp6 || exit 1"]
+      interval: 30s
+      timeout: 10s
+      retries: 6
+      start_period: 40s
+
+  keepalived:
+    image: osixia/keepalived:2.0.20
+    container_name: keepalived-pihole-nomad
+    network_mode: host
+    restart: unless-stopped
+    depends_on:
+      pihole:
+        condition: service_healthy
+    cap_add:
+      - NET_ADMIN
+      - NET_BROADCAST
+      - NET_RAW
+    volumes:
+      - ./keepalived.conf:/usr/local/etc/keepalived/keepalived.conf:ro

pihole/nomad/keepalived.conf

@@ -0,0 +1,32 @@
+global_defs {
+    router_id pihole_backup
+}
+
+vrrp_script chk_pihole_web {
+    script "grep -qi ':22FA' /proc/net/tcp /proc/net/tcp6"
+    interval 2
+    fall 2
+    rise 2
+}
+
+vrrp_instance VI_DNS {
+    state BACKUP
+    interface br0
+    virtual_router_id 53
+    priority 90
+    advert_int 1
+    nopreempt
+
+    authentication {
+        auth_type PASS
+        auth_pass CHANGE01
+    }
+
+    virtual_ipaddress {
+        10.5.1.53/24
+    }
+
+    track_script {
+        chk_pihole_web
+    }
+}

pihole/nomad/keepalived.conf.template

@@ -0,0 +1,32 @@
+global_defs {
+    router_id ${ROUTER_ID_NAME}
+}
+
+vrrp_script chk_pihole_web {
+    script "grep -qi ':${PIHOLE_WEB_PORT_HEX}' /proc/net/tcp /proc/net/tcp6"
+    interval 2
+    fall 2
+    rise 2
+}
+
+vrrp_instance VI_DNS {
+    state ${KEEPALIVED_STATE}
+    interface ${LAN_INTERFACE}
+    virtual_router_id ${KEEPALIVED_ROUTER_ID}
+    priority ${KEEPALIVED_PRIORITY}
+    advert_int 1
+${NOPREEMPT_LINE}
+
+    authentication {
+        auth_type PASS
+        auth_pass ${KEEPALIVED_PASSWORD}
+    }
+
+    virtual_ipaddress {
+        ${VIP_CIDR}
+    }
+
+    track_script {
+        chk_pihole_web
+    }
+}

pihole/nomad/unbound.conf

@@ -0,0 +1,27 @@
+server:
+    verbosity: 1
+    interface: 127.0.0.1
+    port: 5335
+    do-ip4: yes
+    do-udp: yes
+    do-tcp: yes
+    do-ip6: no
+
+    auto-trust-anchor-file: "/opt/unbound/etc/unbound/var/root.key"
+    val-permissive-mode: no
+
+    num-threads: 2
+    cache-min-ttl: 300
+    cache-max-ttl: 86400
+    prefetch: yes
+    prefetch-key: yes
+    minimal-responses: yes
+
+    qname-minimisation: yes
+    hide-identity: yes
+    hide-version: yes
+
+    access-control: 127.0.0.1/32 allow
+    access-control: 0.0.0.0/0 refuse
+
+    root-hints: "/opt/unbound/etc/unbound/var/root.hints"