Move Pi-hole backup node from Serenity to NOMAD

docs/planning/DEPLOY_PIHOLE.md

@@ -16,8 +16,8 @@
  │           10.5.1.53/24                 │
  └──────┬──────────────┬──────────────────┘
         │              │                  │
-   PD (MASTER)   Serenity (BACKUP1)  RPi4 (BACKUP2)
+   PD (MASTER)   N.O.M.A.D. (BACKUP1) RPi4 (BACKUP2)
-   10.5.1.6       10.5.1.5           10.5.1.X
+   10.5.1.6       10.5.1.16           10.5.1.X
    priority 100   priority 90        priority 80
    Pi-hole :53    Pi-hole :53        Pi-hole :53
    Unbound :5335  Unbound :5335      Unbound :5335
@@ -225,11 +225,11 @@ sudo docker logs unbound-pihole-primary --tail 20
 
 ---
 
-## Node 2: Serenity (BACKUP1)
+## Node 2: N.O.M.A.D. (BACKUP1)
 
-Serenity runs Unraid. Deploy via Docker Compose in Unraid's **User Scripts** or the **Compose Manager** plugin.
+N.O.M.A.D. runs Ubuntu bare metal. Deploy via Docker Compose on the host.
 
-**Appdata:** `/mnt/user/appdata/`  
+**Appdata:** `/mnt/hdd-2/docker/appdata/`  
 **Web UI port:** `8954`
 
 ### docker-compose.yaml
@@ -244,7 +244,7 @@ services:
     network_mode: host
     restart: unless-stopped
     volumes:
-      - /mnt/user/appdata/unbound-pihole:/opt/unbound/etc/unbound/
+      - /mnt/hdd-2/docker/appdata/unbound-nomad:/opt/unbound/etc/unbound/
     healthcheck:
       test: ["CMD-SHELL", "drill @127.0.0.1 -p 5335 cloudflare.com || exit 1"]
       interval: 30s
@@ -253,7 +253,7 @@ services:
 
   pihole:
     image: pihole/pihole:latest
-    container_name: pihole-secondary
+    container_name: pihole-nomad
     network_mode: host
     restart: unless-stopped
     depends_on:
@@ -265,8 +265,11 @@ services:
       FTLCONF_dns_listeningMode: "LOCAL"
       FTLCONF_dns_upstreams: "127.0.0.1#5335"
       FTLCONF_webserver_port: "8954"
+      FTLCONF_ntp_ipv4_active: "false"
+      FTLCONF_ntp_ipv6_active: "false"
+      FTLCONF_ntp_sync_active: "false"
     volumes:
-      - /mnt/user/appdata/pihole-secondary/etc-pihole:/etc/pihole
+      - /mnt/hdd-2/docker/appdata/pihole-nomad/etc-pihole:/etc/pihole
     cap_add:
       - SYS_NICE
 
@@ -283,16 +286,16 @@ services:
       - ./keepalived.conf:/container/environment/01-custom/keepalived.conf
 ```
 
-### keepalived.conf (Serenity — BACKUP1)
+### keepalived.conf (N.O.M.A.D. — BACKUP1)
 
 ```
 global_defs {
-    router_id pihole_serenity
+    router_id pihole_nomad
 }
 
 vrrp_instance VI_DNS {
     state BACKUP
-    interface eth0
+    interface enp5s0
     virtual_router_id 53
     priority 90
     advert_int 1
@@ -414,7 +417,7 @@ sudo systemctl start keepalived
     restart: unless-stopped
     environment:
       PRIMARY: "http://127.0.0.1:8953|${PIHOLE_PASSWORD}"
-      REPLICAS: "http://10.5.1.5:8954|${PIHOLE_PASSWORD_SERENITY},http://10.5.1.X:8955|${PIHOLE_PASSWORD_RPI}"
+      REPLICAS: "http://10.5.1.16:8954|${PIHOLE_PASSWORD_NOMAD},http://10.5.1.X:8955|${PIHOLE_PASSWORD_RPI}"
       RUN_GRAVITY: "true"
       FULL_SYNC: "false"
       CRON: "0 3 * * *"
@@ -426,7 +429,7 @@ sudo systemctl start keepalived
 > - `REPLICAS` — comma-separated list of secondary instances
 > - `CRON` — syncs daily at 3 AM; force a manual sync with `docker exec nebula-sync nebula-sync run`
 > - Each Pi-hole instance can (and should) have a **different** password — nebula-sync handles auth per-instance
-> - Add `PIHOLE_PASSWORD_SERENITY` and `PIHOLE_PASSWORD_RPI` to your `.env` file
+> - Add `PIHOLE_PASSWORD_NOMAD` and `PIHOLE_PASSWORD_RPI` to your `.env` file
 
 ### What Nebula Sync copies
 - Gravity database (blocklists, allowlists, group assignments)
@@ -469,7 +472,7 @@ nslookup doubleclick.net 10.5.1.53   # Should return 0.0.0.0 (blocked)
 # On PD, bring down keepalived
 sudo docker stop keepalived
 # Wait ~3 seconds, then check from another machine:
-nslookup google.com 10.5.1.53   # Should still work — Serenity took over VIP
+nslookup google.com 10.5.1.53   # Should still work — N.O.M.A.D. took over VIP
 # Restore
 sudo docker start keepalived
 ```
@@ -491,7 +494,6 @@ Add in Pi-hole (primary) → Local DNS → DNS Records:
 | Domain | IP |
 |--------|----|
 | pd.lan | 10.5.1.6 |
-| serenity.lan | 10.5.1.5 |
 | nomad.lan | 10.5.1.16 |
 | rocinante.lan | 10.5.1.112 |
 | rpi4.lan | 10.5.1.X |
@@ -503,7 +505,7 @@ Nebula Sync will propagate these to all replicas.
 | Subdomain | Target |
 |-----------|--------|
 | `pihole.paccoco.com` | `http://10.5.1.6:8953` (primary) |
-| `pihole2.paccoco.com` | `http://10.5.1.5:8954` (Serenity) |
+| `pihole2.paccoco.com` | `http://10.5.1.16:8954` (N.O.M.A.D.) |
 
 Enable Pangolin auth — Pi-hole admin should not be publicly accessible.
 
@@ -515,7 +517,7 @@ Enable Pangolin auth — Pi-hole admin should not be publicly accessible.
 |------|------|------|--------|
 | PD | `/mnt/tank/docker/appdata/pihole-primary` | tank | Low write frequency |
 | PD | `/mnt/tank/docker/appdata/unbound-primary` | tank | Config + root hints only |
-| Serenity | `/mnt/user/appdata/pihole-secondary` | Unraid array | Low write frequency |
+| N.O.M.A.D. | `/mnt/hdd-2/docker/appdata/pihole-nomad` | Local HDD | Docker-managed local appdata |
 | RPi4 | `~/pihole/` | SD card / SSD | Dedicated Pi, no other concerns |
 
 ---

docs/planning/TODO.md

@@ -5,7 +5,7 @@
 - [x] Regenerate N.O.M.A.D. Newt secret (was exposed in chat) — completed 2026-05-15
 
 ## Infrastructure
-- [ ] Deploy Pi-hole 3-node HA cluster (PD + Serenity + RPi4) — see docs/planning/DEPLOY_PIHOLE.md
+- [ ] Deploy Pi-hole 3-node HA cluster (PD + N.O.M.A.D. + RPi4) — see docs/planning/DEPLOY_PIHOLE.md
   - PD primary stack scaffolded repo-side under `pihole/` on 2026-05-15
   - RPi4 IP still TBD — fill in before deploying
 - [x] Deploy Kima-Hub — completed 2026-05-06; docs/planning/DEPLOY_KIMA_HUB.md now serves as deployment/repair reference

pihole/.env.example

@@ -19,7 +19,12 @@ KEEPALIVED_NOPREEMPT=true
 
 # Nebula Sync (PD primary -> replicas)
 NEBULA_SYNC_PRIMARY=http://127.0.0.1:8953|CHANGE_ME
-NEBULA_SYNC_REPLICAS=http://10.5.1.5:8954|CHANGE_ME
+NEBULA_SYNC_REPLICAS=http://10.5.1.16:8954|CHANGE_ME
 NEBULA_SYNC_FULL_SYNC=false
 NEBULA_SYNC_RUN_GRAVITY=true
 NEBULA_SYNC_CRON=0 3 * * *
+NEBULA_SYNC_GRAVITY_GROUP=true
+NEBULA_SYNC_GRAVITY_AD_LIST=true
+NEBULA_SYNC_GRAVITY_AD_LIST_BY_GROUP=true
+NEBULA_SYNC_GRAVITY_DOMAIN_LIST=true
+NEBULA_SYNC_GRAVITY_DOMAIN_LIST_BY_GROUP=true

pihole/README.md

@@ -13,7 +13,7 @@ This stack intentionally covers the **PD primary** plus PD-hosted sync:
 - Keepalived
 - Nebula Sync
 
-Serenity and RPi4 replicas still need their own deployment steps.
+NOMAD and RPi4 replicas still need their own deployment steps.
 
 ## Files
 
@@ -46,4 +46,5 @@ If that validates cleanly, bring it up:
 - `FTLCONF_dns_listeningMode=LOCAL` is set because TrueNAS binds loopback on port 53.
 - Pi-hole NTP is explicitly disabled here because PD host `chronyd` already owns UDP 123.
 - Nebula Sync runs on PD and uses `NEBULA_SYNC_PRIMARY` / `NEBULA_SYNC_REPLICAS` from `.env`.
+- Nebula Sync is intentionally set to selective mode with gravity sync toggles enabled for groups, adlists, and domain lists so replica-specific runtime/network settings are not clobbered.
 - `keepalived.conf` is generated and should stay local/untracked.

pihole/docker-compose.yaml

@@ -75,3 +75,8 @@ services:
       FULL_SYNC: ${NEBULA_SYNC_FULL_SYNC}
       RUN_GRAVITY: ${NEBULA_SYNC_RUN_GRAVITY}
       CRON: ${NEBULA_SYNC_CRON}
+      SYNC_GRAVITY_GROUP: ${NEBULA_SYNC_GRAVITY_GROUP}
+      SYNC_GRAVITY_AD_LIST: ${NEBULA_SYNC_GRAVITY_AD_LIST}
+      SYNC_GRAVITY_AD_LIST_BY_GROUP: ${NEBULA_SYNC_GRAVITY_AD_LIST_BY_GROUP}
+      SYNC_GRAVITY_DOMAIN_LIST: ${NEBULA_SYNC_GRAVITY_DOMAIN_LIST}
+      SYNC_GRAVITY_DOMAIN_LIST_BY_GROUP: ${NEBULA_SYNC_GRAVITY_DOMAIN_LIST_BY_GROUP}

pihole/nomad/.env.example

@@ -0,0 +1,12 @@
+TZ=America/Chicago
+APPDATA_PIHOLE_ROOT=/mnt/hdd-2/docker/appdata/pihole-nomad
+APPDATA_UNBOUND_ROOT=/mnt/hdd-2/docker/appdata/unbound-nomad
+NOMAD_PIHOLE_WEB_PORT=8954
+PIHOLE_PASSWORD=CHANGE_ME
+LAN_INTERFACE=enp5s0
+VIP_CIDR=10.5.1.53/24
+KEEPALIVED_ROUTER_ID=53
+KEEPALIVED_STATE=BACKUP
+KEEPALIVED_PRIORITY=90
+KEEPALIVED_PASSWORD=CHANGE01
+KEEPALIVED_NOPREEMPT=true

pihole/nomad/README.md

@@ -0,0 +1,16 @@
+# NOMAD Pi-hole / Unbound / Keepalived Replica
+
+- **Repo stack path:** `/home/fizzlepoof/repos/truenas-stacks/pihole/nomad`
+- **Suggested live stack path:** `/mnt/hdd-1/docker/compose/pihole-nomad`
+- **Node role:** NOMAD backup / BACKUP1
+- **VIP target:** `10.5.1.53/24`
+- **LAN interface:** `enp5s0`
+- **NOMAD LAN IP:** `10.5.1.16`
+
+Bring-up:
+
+```bash
+cd /mnt/hdd-1/docker/compose/pihole-nomad
+cp .env.example .env
+bash bin/prepare_nomad.sh --up
+```

pihole/nomad/bin/prepare_nomad.sh

@@ -0,0 +1,34 @@
+#!/usr/bin/env bash
+set -euo pipefail
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+STACK_DIR="$(cd "$SCRIPT_DIR/.." && pwd)"
+ENV_FILE="${ENV_FILE:-$STACK_DIR/.env}"
+DO_UP="${1:-}"
+
+if [[ ! -f "$ENV_FILE" ]]; then
+  echo "Missing $ENV_FILE — copy .env.example to .env and fill in real values first." >&2
+  exit 1
+fi
+
+set -a
+# shellcheck disable=SC1090
+source "$ENV_FILE"
+set +a
+
+mkdir -p \
+  "$APPDATA_PIHOLE_ROOT/etc-pihole" \
+  "$APPDATA_PIHOLE_ROOT/etc-dnsmasq.d" \
+  "$APPDATA_UNBOUND_ROOT/var"
+UNBOUND_TARGET="$APPDATA_UNBOUND_ROOT/unbound.conf"
+UNBOUND_TMP="$APPDATA_UNBOUND_ROOT/.unbound.conf.tmp.$$"
+cp "$STACK_DIR/unbound.conf" "$UNBOUND_TMP"
+mv -f "$UNBOUND_TMP" "$UNBOUND_TARGET"
+if [[ ! -s "$APPDATA_UNBOUND_ROOT/var/root.hints" ]]; then
+  curl -fsSL -o "$APPDATA_UNBOUND_ROOT/var/root.hints" https://www.internic.net/domain/named.cache
+fi
+ENV_FILE="$ENV_FILE" python3 "$STACK_DIR/bin/render_keepalived_conf.py"
+docker compose --env-file "$ENV_FILE" -f "$STACK_DIR/docker-compose.yaml" config >/dev/null
+if [[ "$DO_UP" == "--up" ]]; then
+  docker compose --env-file "$ENV_FILE" -f "$STACK_DIR/docker-compose.yaml" up -d
+  docker compose --env-file "$ENV_FILE" -f "$STACK_DIR/docker-compose.yaml" ps
+fi

pihole/nomad/bin/render_keepalived_conf.py

@@ -0,0 +1,73 @@
+#!/usr/bin/env python3
+from __future__ import annotations
+import os
+import re
+import sys
+from pathlib import Path
+
+STACK_DIR = Path(__file__).resolve().parent.parent
+ENV_FILE = Path(os.environ.get('ENV_FILE', STACK_DIR / '.env'))
+TEMPLATE = STACK_DIR / 'keepalived.conf.template'
+OUTPUT = STACK_DIR / 'keepalived.conf'
+
+
+def load_env(path: Path) -> dict[str, str]:
+    env = dict(os.environ)
+    if path.exists():
+        for raw in path.read_text(encoding='utf-8').splitlines():
+            line = raw.strip()
+            if not line or line.startswith('#') or '=' not in line:
+                continue
+            k, v = line.split('=', 1)
+            env.setdefault(k.strip(), v.strip().strip('"').strip("'"))
+    return env
+
+
+def main() -> int:
+    env = load_env(ENV_FILE)
+    required = [
+        'LAN_INTERFACE', 'VIP_CIDR', 'KEEPALIVED_ROUTER_ID', 'KEEPALIVED_STATE',
+        'KEEPALIVED_PRIORITY', 'KEEPALIVED_PASSWORD'
+    ]
+    missing = [k for k in required if not env.get(k)]
+    if missing:
+        print(f'Missing required vars in {ENV_FILE}: {", ".join(missing)}', file=sys.stderr)
+        return 1
+
+    web_port = env.get('PIHOLE_WEB_PORT') or env.get('PD_PIHOLE_WEB_PORT') or env.get('SERENITY_PIHOLE_WEB_PORT') or env.get('RPI4_PIHOLE_WEB_PORT')
+    if not web_port:
+        print('Missing Pi-hole web port in .env (set PIHOLE_WEB_PORT or node-specific port variable).', file=sys.stderr)
+        return 1
+
+    keepalived_password = env['KEEPALIVED_PASSWORD']
+    if len(keepalived_password) > 8:
+        print('KEEPALIVED_PASSWORD must be 8 characters or fewer for VRRP PASS auth.', file=sys.stderr)
+        return 1
+
+    try:
+        web_port_hex = f"{int(web_port):04X}"
+    except ValueError:
+        print('Pi-hole web port must be an integer.', file=sys.stderr)
+        return 1
+
+    router_id_name = f"pihole_{re.sub(r'[^a-z0-9]+', '_', env['KEEPALIVED_STATE'].lower())}"
+    nopreempt_line = '    nopreempt' if env.get('KEEPALIVED_NOPREEMPT', 'true').lower() in {'1','true','yes','on'} else ''
+
+    rendered = TEMPLATE.read_text(encoding='utf-8')
+    replacements = {
+        'ROUTER_ID_NAME': router_id_name,
+        'PIHOLE_WEB_PORT_HEX': web_port_hex,
+        'NOPREEMPT_LINE': nopreempt_line,
+    }
+    replacements.update({k: env[k] for k in required})
+    replacements['PIHOLE_WEB_PORT'] = web_port
+    for key, value in replacements.items():
+        rendered = rendered.replace('${' + key + '}', value)
+
+    OUTPUT.write_text(rendered.rstrip() + '\n', encoding='utf-8')
+    print(f'Rendered {OUTPUT}')
+    return 0
+
+
+if __name__ == '__main__':
+    raise SystemExit(main())

pihole/nomad/docker-compose.yaml

@@ -0,0 +1,62 @@
+services:
+  unbound:
+    image: mvance/unbound:latest
+    container_name: unbound-pihole-nomad
+    network_mode: host
+    restart: unless-stopped
+    environment:
+      TZ: ${TZ}
+      UNBOUND_PORT: "5335"
+    volumes:
+      - ${APPDATA_UNBOUND_ROOT}:/opt/unbound/etc/unbound/
+    healthcheck:
+      test: ["CMD-SHELL", "hex=$$(printf '%04X' \"$$UNBOUND_PORT\"); grep -qi \":$$hex\" /proc/net/tcp /proc/net/tcp6 || exit 1"]
+      interval: 30s
+      timeout: 10s
+      retries: 6
+      start_period: 20s
+
+  pihole:
+    image: pihole/pihole:latest
+    container_name: pihole-nomad
+    network_mode: host
+    restart: unless-stopped
+    depends_on:
+      unbound:
+        condition: service_healthy
+    environment:
+      TZ: ${TZ}
+      PIHOLE_WEB_PORT: ${NOMAD_PIHOLE_WEB_PORT}
+      FTLCONF_webserver_api_password: ${PIHOLE_PASSWORD}
+      FTLCONF_dns_listeningMode: "LOCAL"
+      FTLCONF_dns_upstreams: "127.0.0.1#5335"
+      FTLCONF_webserver_port: ${NOMAD_PIHOLE_WEB_PORT}
+      FTLCONF_ntp_ipv4_active: "false"
+      FTLCONF_ntp_ipv6_active: "false"
+      FTLCONF_ntp_sync_active: "false"
+    volumes:
+      - ${APPDATA_PIHOLE_ROOT}/etc-pihole:/etc/pihole
+      - ${APPDATA_PIHOLE_ROOT}/etc-dnsmasq.d:/etc/dnsmasq.d
+    cap_add:
+      - SYS_NICE
+    healthcheck:
+      test: ["CMD-SHELL", "hex=$$(printf '%04X' \"$$PIHOLE_WEB_PORT\"); grep -qi \":$$hex\" /proc/net/tcp /proc/net/tcp6 || exit 1"]
+      interval: 30s
+      timeout: 10s
+      retries: 6
+      start_period: 40s
+
+  keepalived:
+    image: osixia/keepalived:2.0.20
+    container_name: keepalived-pihole-nomad
+    network_mode: host
+    restart: unless-stopped
+    depends_on:
+      pihole:
+        condition: service_healthy
+    cap_add:
+      - NET_ADMIN
+      - NET_BROADCAST
+      - NET_RAW
+    volumes:
+      - ./keepalived.conf:/usr/local/etc/keepalived/keepalived.conf:ro

pihole/nomad/keepalived.conf

@@ -0,0 +1,32 @@
+global_defs {
+    router_id pihole_backup
+}
+
+vrrp_script chk_pihole_web {
+    script "grep -qi ':22FA' /proc/net/tcp /proc/net/tcp6"
+    interval 2
+    fall 2
+    rise 2
+}
+
+vrrp_instance VI_DNS {
+    state BACKUP
+    interface br0
+    virtual_router_id 53
+    priority 90
+    advert_int 1
+    nopreempt
+
+    authentication {
+        auth_type PASS
+        auth_pass CHANGE01
+    }
+
+    virtual_ipaddress {
+        10.5.1.53/24
+    }
+
+    track_script {
+        chk_pihole_web
+    }
+}

pihole/nomad/keepalived.conf.template

@@ -0,0 +1,32 @@
+global_defs {
+    router_id ${ROUTER_ID_NAME}
+}
+
+vrrp_script chk_pihole_web {
+    script "grep -qi ':${PIHOLE_WEB_PORT_HEX}' /proc/net/tcp /proc/net/tcp6"
+    interval 2
+    fall 2
+    rise 2
+}
+
+vrrp_instance VI_DNS {
+    state ${KEEPALIVED_STATE}
+    interface ${LAN_INTERFACE}
+    virtual_router_id ${KEEPALIVED_ROUTER_ID}
+    priority ${KEEPALIVED_PRIORITY}
+    advert_int 1
+${NOPREEMPT_LINE}
+
+    authentication {
+        auth_type PASS
+        auth_pass ${KEEPALIVED_PASSWORD}
+    }
+
+    virtual_ipaddress {
+        ${VIP_CIDR}
+    }
+
+    track_script {
+        chk_pihole_web
+    }
+}

pihole/nomad/unbound.conf

@@ -0,0 +1,27 @@
+server:
+    verbosity: 1
+    interface: 127.0.0.1
+    port: 5335
+    do-ip4: yes
+    do-udp: yes
+    do-tcp: yes
+    do-ip6: no
+
+    auto-trust-anchor-file: "/opt/unbound/etc/unbound/var/root.key"
+    val-permissive-mode: no
+
+    num-threads: 2
+    cache-min-ttl: 300
+    cache-max-ttl: 86400
+    prefetch: yes
+    prefetch-key: yes
+    minimal-responses: yes
+
+    qname-minimisation: yes
+    hide-identity: yes
+    hide-version: yes
+
+    access-control: 127.0.0.1/32 allow
+    access-control: 0.0.0.0/0 refuse
+
+    root-hints: "/opt/unbound/etc/unbound/var/root.hints"