chore: sync homelab ops, identity, and monitoring docs
This commit is contained in:
Fizzlepoof
251
monitoring/deploy.sh
Normal file → Executable file
251
monitoring/deploy.sh
Normal file → Executable file
@@ -1,236 +1,41 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
# ============================================================
|
||||
# Phase 6 — Grafana + Prometheus Deployment on PD
|
||||
# Run this script on PlausibleDeniability as root or with sudo
|
||||
# ============================================================
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
cd "$SCRIPT_DIR"
|
||||
|
||||
echo "=== Phase 6: Grafana + Prometheus ==="
|
||||
echo ""
|
||||
|
||||
# ---- 1. Scaffold directories ----
|
||||
echo "[1/6] Scaffolding directories..."
|
||||
|
||||
mkdir -p /mnt/docker-ssd/docker/compose/monitoring/provisioning/datasources
|
||||
mkdir -p /mnt/docker-ssd/docker/compose/monitoring/provisioning/dashboards
|
||||
echo "=== Monitoring stack deploy ==="
|
||||
|
||||
mkdir -p /mnt/tank/docker/appdata/prometheus
|
||||
chown 65534:65534 /mnt/tank/docker/appdata/prometheus
|
||||
|
||||
mkdir -p /mnt/tank/docker/appdata/grafana
|
||||
mkdir -p /mnt/tank/docker/appdata/grafana/dashboards
|
||||
mkdir -p /mnt/tank/docker/appdata/loki
|
||||
mkdir -p /mnt/tank/docker/appdata/promtail
|
||||
|
||||
chown 65534:65534 /mnt/tank/docker/appdata/prometheus
|
||||
chown 472:472 /mnt/tank/docker/appdata/grafana
|
||||
chown 472:472 /mnt/tank/docker/appdata/grafana/dashboards
|
||||
|
||||
echo " Done."
|
||||
if [[ ! -f .env ]]; then
|
||||
cp .env.example .env
|
||||
gf_pass="$(openssl rand -hex 16)"
|
||||
sed -i "s/^GF_ADMIN_PASS=CHANGE_ME$/GF_ADMIN_PASS=$gf_pass/" .env
|
||||
chmod 600 .env
|
||||
echo "Created .env with a random Grafana admin password."
|
||||
fi
|
||||
|
||||
# ---- 2. Write docker-compose.yaml ----
|
||||
echo "[2/6] Writing docker-compose.yaml..."
|
||||
|
||||
cat > /mnt/docker-ssd/docker/compose/monitoring/docker-compose.yaml << 'COMPOSE'
|
||||
name: monitoring
|
||||
|
||||
services:
|
||||
prometheus:
|
||||
image: prom/prometheus:v3.4.0
|
||||
container_name: prometheus
|
||||
restart: unless-stopped
|
||||
ports:
|
||||
- "9090:9090"
|
||||
command:
|
||||
- '--config.file=/etc/prometheus/prometheus.yml'
|
||||
- '--storage.tsdb.path=/prometheus'
|
||||
- '--storage.tsdb.retention.time=90d'
|
||||
- '--web.enable-lifecycle'
|
||||
volumes:
|
||||
- /mnt/tank/docker/appdata/prometheus:/prometheus
|
||||
- /mnt/docker-ssd/docker/compose/monitoring/prometheus.yml:/etc/prometheus/prometheus.yml:ro
|
||||
networks:
|
||||
- default
|
||||
|
||||
grafana:
|
||||
image: grafana/grafana:13.0.1
|
||||
container_name: grafana
|
||||
restart: unless-stopped
|
||||
ports:
|
||||
- "3000:3000"
|
||||
environment:
|
||||
TZ: ${TZ}
|
||||
GF_SECURITY_ADMIN_USER: ${GF_ADMIN_USER}
|
||||
GF_SECURITY_ADMIN_PASSWORD: ${GF_ADMIN_PASS}
|
||||
GF_SERVER_ROOT_URL: https://${GF_HOST}/
|
||||
volumes:
|
||||
- /mnt/tank/docker/appdata/grafana:/var/lib/grafana
|
||||
- /mnt/docker-ssd/docker/compose/monitoring/provisioning:/etc/grafana/provisioning:ro
|
||||
networks:
|
||||
- pangolin
|
||||
- default
|
||||
|
||||
node-exporter:
|
||||
image: prom/node-exporter:v1.9.0
|
||||
container_name: node-exporter
|
||||
restart: unless-stopped
|
||||
ports:
|
||||
- "9100:9100"
|
||||
command:
|
||||
- '--path.procfs=/host/proc'
|
||||
- '--path.sysfs=/host/sys'
|
||||
- '--path.rootfs=/rootfs'
|
||||
- '--collector.filesystem.mount-points-exclude=^/(sys|proc|dev|host|etc)($$|/)'
|
||||
volumes:
|
||||
- /proc:/host/proc:ro
|
||||
- /sys:/host/sys:ro
|
||||
- /:/rootfs:ro
|
||||
networks:
|
||||
- default
|
||||
|
||||
networks:
|
||||
pangolin:
|
||||
external: true
|
||||
COMPOSE
|
||||
|
||||
echo " Done."
|
||||
|
||||
# ---- 3. Write prometheus.yml ----
|
||||
echo "[3/6] Writing prometheus.yml..."
|
||||
|
||||
cat > /mnt/docker-ssd/docker/compose/monitoring/prometheus.yml << 'PROMCFG'
|
||||
global:
|
||||
scrape_interval: 15s
|
||||
evaluation_interval: 15s
|
||||
|
||||
scrape_configs:
|
||||
# ------- PlausibleDeniability (local) -------
|
||||
- job_name: 'pd-node'
|
||||
static_configs:
|
||||
- targets: ['node-exporter:9100']
|
||||
labels:
|
||||
host: 'plausible-deniability'
|
||||
|
||||
- job_name: 'pd-netdata'
|
||||
metrics_path: /api/v1/allmetrics
|
||||
params:
|
||||
format: [prometheus]
|
||||
static_configs:
|
||||
- targets: ['10.5.1.6:19999']
|
||||
labels:
|
||||
host: 'plausible-deniability'
|
||||
|
||||
# ------- Serenity -------
|
||||
- job_name: 'serenity-netdata'
|
||||
metrics_path: /api/v1/allmetrics
|
||||
params:
|
||||
format: [prometheus]
|
||||
static_configs:
|
||||
- targets: ['10.5.1.5:19999']
|
||||
labels:
|
||||
host: 'serenity'
|
||||
|
||||
# ------- N.O.M.A.D. -------
|
||||
# Uncomment when Netdata or node-exporter is available on N.O.M.A.D.
|
||||
# - job_name: 'nomad-netdata'
|
||||
# metrics_path: /api/v1/allmetrics
|
||||
# params:
|
||||
# format: [prometheus]
|
||||
# static_configs:
|
||||
# - targets: ['10.5.1.16:19999']
|
||||
# labels:
|
||||
# host: 'nomad'
|
||||
PROMCFG
|
||||
|
||||
echo " Done."
|
||||
|
||||
# ---- 4. Write Grafana provisioning ----
|
||||
echo "[4/6] Writing Grafana provisioning configs..."
|
||||
|
||||
cat > /mnt/docker-ssd/docker/compose/monitoring/provisioning/datasources/prometheus.yml << 'DATASRC'
|
||||
apiVersion: 1
|
||||
|
||||
datasources:
|
||||
- name: Prometheus
|
||||
type: prometheus
|
||||
access: proxy
|
||||
url: http://prometheus:9090
|
||||
isDefault: true
|
||||
editable: true
|
||||
DATASRC
|
||||
|
||||
cat > /mnt/docker-ssd/docker/compose/monitoring/provisioning/dashboards/dashboards.yml << 'DASHCFG'
|
||||
apiVersion: 1
|
||||
|
||||
providers:
|
||||
- name: 'default'
|
||||
orgId: 1
|
||||
folder: ''
|
||||
type: file
|
||||
disableDeletion: false
|
||||
editable: true
|
||||
options:
|
||||
path: /var/lib/grafana/dashboards
|
||||
foldersFromFilesStructure: false
|
||||
DASHCFG
|
||||
|
||||
echo " Done."
|
||||
|
||||
# ---- 5. Generate .env ----
|
||||
echo "[5/6] Generating .env..."
|
||||
|
||||
GF_PASS=$(openssl rand -hex 16)
|
||||
|
||||
cat > /mnt/docker-ssd/docker/compose/monitoring/.env << ENV
|
||||
TZ=America/New_York
|
||||
|
||||
# Grafana
|
||||
GF_ADMIN_USER=admin
|
||||
GF_ADMIN_PASS=${GF_PASS}
|
||||
GF_HOST=grafana.paccoco.com
|
||||
ENV
|
||||
|
||||
echo " Done."
|
||||
echo ""
|
||||
echo " Grafana admin password: ${GF_PASS}"
|
||||
echo " (saved in /mnt/docker-ssd/docker/compose/monitoring/.env)"
|
||||
echo ""
|
||||
|
||||
# ---- 6. Validate and deploy ----
|
||||
echo "[6/6] Validating and deploying..."
|
||||
|
||||
cd /mnt/docker-ssd/docker/compose/monitoring
|
||||
docker compose --env-file .env config > /dev/null 2>&1
|
||||
echo " Compose config valid."
|
||||
if compgen -G "$SCRIPT_DIR/dashboards/*.json" >/dev/null; then
|
||||
cp "$SCRIPT_DIR"/dashboards/*.json /mnt/tank/docker/appdata/grafana/dashboards/
|
||||
chown 472:472 /mnt/tank/docker/appdata/grafana/dashboards/*.json
|
||||
fi
|
||||
|
||||
docker compose --env-file .env config >/dev/null
|
||||
docker compose --env-file .env up -d
|
||||
echo ""
|
||||
echo "=== Deployment complete ==="
|
||||
echo ""
|
||||
echo "Waiting 10 seconds for containers to start..."
|
||||
sleep 10
|
||||
|
||||
# ---- Post-deploy checks ----
|
||||
echo ""
|
||||
echo "=== Post-Deploy Verification ==="
|
||||
echo ""
|
||||
|
||||
echo "--- Container status ---"
|
||||
docker ps --filter name=prometheus --filter name=grafana --filter name=node-exporter --format "table {{.Names}}\t{{.Status}}\t{{.Ports}}"
|
||||
echo ""
|
||||
|
||||
echo "--- Prometheus targets ---"
|
||||
curl -s http://10.5.1.6:9090/api/v1/targets 2>/dev/null | python3 -m json.tool 2>/dev/null | head -30 || echo "Prometheus not yet responding (may need a moment)"
|
||||
echo ""
|
||||
|
||||
echo "--- Grafana health ---"
|
||||
HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" http://10.5.1.6:3000/ 2>/dev/null || echo "000")
|
||||
echo "Grafana HTTP status: ${HTTP_CODE}"
|
||||
echo ""
|
||||
|
||||
echo "--- Node exporter ---"
|
||||
curl -s http://10.5.1.6:9100/metrics 2>/dev/null | head -5 || echo "Node exporter not yet responding"
|
||||
echo ""
|
||||
|
||||
echo "=== All done! ==="
|
||||
echo ""
|
||||
echo "Next steps:"
|
||||
echo " 1. Add Pangolin resource: grafana.paccoco.com -> http://grafana:3000"
|
||||
echo " 2. Log in at https://grafana.paccoco.com with admin / ${GF_PASS}"
|
||||
echo " 3. Import dashboards: Node Exporter Full (ID 1860), Docker Stats (ID 893)"
|
||||
echo " 4. Set up Grafana -> n8n alert webhook at https://n8n.paccoco.com/webhook/grafana-alert"
|
||||
echo
|
||||
docker ps --filter name=prometheus --filter name=grafana --filter name=loki --filter name=promtail --filter name=cadvisor --format "table {{.Names}}\t{{.Status}}\t{{.Ports}}"
|
||||
echo
|
||||
curl -fsS http://127.0.0.1:9090/-/ready >/dev/null && echo "Prometheus ready"
|
||||
[[ "$(docker inspect -f '{{.State.Running}}' loki 2>/dev/null)" == "true" ]] && echo "Loki running"
|
||||
curl -fsS http://127.0.0.1:9100/metrics >/dev/null && echo "node-exporter ready"
|
||||
docker exec cadvisor wget -qO- http://127.0.0.1:8080/metrics >/dev/null 2>&1 && echo "cAdvisor ready" || true
|
||||
|
||||
Reference in New Issue
Block a user