truenas-stacks/monitoring/deploy.sh

#!/usr/bin/env bash
set -euo pipefail

# ============================================================
# Phase 6 — Grafana + Prometheus Deployment on PD
# Run this script on PlausibleDeniability as root or with sudo
# ============================================================

echo "=== Phase 6: Grafana + Prometheus ==="
echo ""

# ---- 1. Scaffold directories ----
echo "[1/6] Scaffolding directories..."

mkdir -p /mnt/docker-ssd/docker/compose/monitoring/provisioning/datasources
mkdir -p /mnt/docker-ssd/docker/compose/monitoring/provisioning/dashboards

mkdir -p /mnt/tank/docker/appdata/prometheus
chown 65534:65534 /mnt/tank/docker/appdata/prometheus

mkdir -p /mnt/tank/docker/appdata/grafana
chown 472:472 /mnt/tank/docker/appdata/grafana

echo "      Done."

# ---- 2. Write docker-compose.yaml ----
echo "[2/6] Writing docker-compose.yaml..."

cat > /mnt/docker-ssd/docker/compose/monitoring/docker-compose.yaml << 'COMPOSE'
name: monitoring

services:
  prometheus:
    image: prom/prometheus:v3.4.0
    container_name: prometheus
    restart: unless-stopped
    ports:
      - "9090:9090"
    command:
      - '--config.file=/etc/prometheus/prometheus.yml'
      - '--storage.tsdb.path=/prometheus'
      - '--storage.tsdb.retention.time=90d'
      - '--web.enable-lifecycle'
    volumes:
      - /mnt/tank/docker/appdata/prometheus:/prometheus
      - /mnt/docker-ssd/docker/compose/monitoring/prometheus.yml:/etc/prometheus/prometheus.yml:ro
    networks:
      - default

  grafana:
    image: grafana/grafana:13.0.1
    container_name: grafana
    restart: unless-stopped
    ports:
      - "3000:3000"
    environment:
      TZ: ${TZ}
      GF_SECURITY_ADMIN_USER: ${GF_ADMIN_USER}
      GF_SECURITY_ADMIN_PASSWORD: ${GF_ADMIN_PASS}
      GF_SERVER_ROOT_URL: https://${GF_HOST}/
    volumes:
      - /mnt/tank/docker/appdata/grafana:/var/lib/grafana
      - /mnt/docker-ssd/docker/compose/monitoring/provisioning:/etc/grafana/provisioning:ro
    networks:
      - pangolin
      - default

  node-exporter:
    image: prom/node-exporter:v1.9.0
    container_name: node-exporter
    restart: unless-stopped
    ports:
      - "9100:9100"
    command:
      - '--path.procfs=/host/proc'
      - '--path.sysfs=/host/sys'
      - '--path.rootfs=/rootfs'
      - '--collector.filesystem.mount-points-exclude=^/(sys|proc|dev|host|etc)($$|/)'
    volumes:
      - /proc:/host/proc:ro
      - /sys:/host/sys:ro
      - /:/rootfs:ro
    networks:
      - default

networks:
  pangolin:
    external: true
COMPOSE

echo "      Done."

# ---- 3. Write prometheus.yml ----
echo "[3/6] Writing prometheus.yml..."

cat > /mnt/docker-ssd/docker/compose/monitoring/prometheus.yml << 'PROMCFG'
global:
  scrape_interval: 15s
  evaluation_interval: 15s

scrape_configs:
  # ------- PlausibleDeniability (local) -------
  - job_name: 'pd-node'
    static_configs:
      - targets: ['node-exporter:9100']
        labels:
          host: 'plausible-deniability'

  - job_name: 'pd-netdata'
    metrics_path: /api/v1/allmetrics
    params:
      format: [prometheus]
    static_configs:
      - targets: ['10.5.1.6:19999']
        labels:
          host: 'plausible-deniability'

  # ------- Serenity -------
  - job_name: 'serenity-netdata'
    metrics_path: /api/v1/allmetrics
    params:
      format: [prometheus]
    static_configs:
      - targets: ['10.5.1.5:19999']
        labels:
          host: 'serenity'

  # ------- N.O.M.A.D. -------
  # Uncomment when Netdata or node-exporter is available on N.O.M.A.D.
  # - job_name: 'nomad-netdata'
  #   metrics_path: /api/v1/allmetrics
  #   params:
  #     format: [prometheus]
  #   static_configs:
  #     - targets: ['10.5.1.16:19999']
  #       labels:
  #         host: 'nomad'
PROMCFG

echo "      Done."

# ---- 4. Write Grafana provisioning ----
echo "[4/6] Writing Grafana provisioning configs..."

cat > /mnt/docker-ssd/docker/compose/monitoring/provisioning/datasources/prometheus.yml << 'DATASRC'
apiVersion: 1

datasources:
  - name: Prometheus
    type: prometheus
    access: proxy
    url: http://prometheus:9090
    isDefault: true
    editable: true
DATASRC

cat > /mnt/docker-ssd/docker/compose/monitoring/provisioning/dashboards/dashboards.yml << 'DASHCFG'
apiVersion: 1

providers:
  - name: 'default'
    orgId: 1
    folder: ''
    type: file
    disableDeletion: false
    editable: true
    options:
      path: /var/lib/grafana/dashboards
      foldersFromFilesStructure: false
DASHCFG

echo "      Done."

# ---- 5. Generate .env ----
echo "[5/6] Generating .env..."

GF_PASS=$(openssl rand -hex 16)

cat > /mnt/docker-ssd/docker/compose/monitoring/.env << ENV
TZ=America/New_York

# Grafana
GF_ADMIN_USER=admin
GF_ADMIN_PASS=${GF_PASS}
GF_HOST=grafana.paccoco.com
ENV

echo "      Done."
echo ""
echo "      Grafana admin password: ${GF_PASS}"
echo "      (saved in /mnt/docker-ssd/docker/compose/monitoring/.env)"
echo ""

# ---- 6. Validate and deploy ----
echo "[6/6] Validating and deploying..."

cd /mnt/docker-ssd/docker/compose/monitoring
docker compose --env-file .env config > /dev/null 2>&1
echo "      Compose config valid."

docker compose --env-file .env up -d
echo ""
echo "=== Deployment complete ==="
echo ""
echo "Waiting 10 seconds for containers to start..."
sleep 10

# ---- Post-deploy checks ----
echo ""
echo "=== Post-Deploy Verification ==="
echo ""

echo "--- Container status ---"
docker ps --filter name=prometheus --filter name=grafana --filter name=node-exporter --format "table {{.Names}}\t{{.Status}}\t{{.Ports}}"
echo ""

echo "--- Prometheus targets ---"
curl -s http://10.5.1.6:9090/api/v1/targets 2>/dev/null | python3 -m json.tool 2>/dev/null | head -30 || echo "Prometheus not yet responding (may need a moment)"
echo ""

echo "--- Grafana health ---"
HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" http://10.5.1.6:3000/ 2>/dev/null || echo "000")
echo "Grafana HTTP status: ${HTTP_CODE}"
echo ""

echo "--- Node exporter ---"
curl -s http://10.5.1.6:9100/metrics 2>/dev/null | head -5 || echo "Node exporter not yet responding"
echo ""

echo "=== All done! ==="
echo ""
echo "Next steps:"
echo "  1. Add Pangolin resource: grafana.paccoco.com -> http://grafana:3000"
echo "  2. Log in at https://grafana.paccoco.com with admin / ${GF_PASS}"
echo "  3. Import dashboards: Node Exporter Full (ID 1860), Docker Stats (ID 893)"
echo "  4. Set up Grafana -> n8n alert webhook at https://n8n.paccoco.com/webhook/grafana-alert"