chore: sync homelab ops, identity, and monitoring docs

HOMELAB_BUILDOUT_PLAN.md

@@ -1239,7 +1239,7 @@ networks:
 
 > **Image notes:**
 > - `prom/prometheus:v3.4.0` — Prometheus v3 is the current major line. **Do not use `:latest`** — it still resolves to 2.x due to a known tagging issue. Always use an explicit `v3.x.y` tag.
-> - `grafana/grafana:13.0.1` — Grafana 13 is the current stable.
+> - `grafana/grafana:latest` — use the moving stable image by default here; only pin Grafana if a breakage or compatibility reason is documented.
 > - `prom/node-exporter:v1.9.0` — check [releases](https://github.com/prometheus/node_exporter/releases).
 
 ### 6.5 — .env.example

automation/.env.example

@@ -35,3 +35,10 @@ BACKUP_KNOWN_HOSTS=/home/truenas_admin/.ssh/known_hosts
 DOCKER_BIN=/usr/bin/docker
 SUDO_BIN=/usr/bin/sudo
 USE_SUDO_FOR_DOCKER=true
+
+# Restore verification
+RESTORE_VERIFY_WORK_ROOT=/mnt/tank/docker/backups/restore-verify
+RESTORE_VERIFY_POSTGRES_IMAGE=postgres:17-alpine
+RESTORE_VERIFY_CONFIG_SOURCE=tank-docker/appdata/grafana
+RESTORE_VERIFY_REQUIRED_PATHS="grafana.db dashboards"
+RESTORE_VERIFY_METRICS_DIR=/mnt/tank/docker/metrics/node-exporter

automation/README.md

@@ -33,6 +33,10 @@ automation/bin/karakeep_api.py /api/health --raw-path --pretty
 - `bin/pd_backup_postgres.sh` — creates gzipped `pg_dump` exports for the configured Postgres DB list (quote space-separated DB names in `.env`, e.g. `POSTGRES_DB_LIST="n8n paperless"`)
 - `bin/pd_backup_sync_to_serenity.sh` — `rsync`s configured backup/config roots to Serenity
 - `bin/run_pd_backups.sh` — wrapper that runs both steps in order
+- `bin/pd_restore_verify_postgres.sh` — restores the latest dump for each configured database into a throwaway Postgres container and fails on SQL errors
+- `bin/pd_restore_verify_appdata.sh` — stages one appdata/config tree back from Serenity and verifies required files exist
+- `bin/run_pd_restore_verification.sh` — wrapper that runs both restore checks in order
+  - writes Prometheus textfile metrics for run status, duration, and last successful verification timestamp
 
 These scripts are the live PD backup runner model.
 
@@ -49,6 +53,7 @@ Current documented live state:
 - the PD → Serenity backup flow is the intended deployed model
 - root cron on PD is the preferred scheduler
 - first-run verification and the deployed status are tracked in `docs/operations/PD_BACKUP_DEPLOYMENT.md` and `docs/planning/TODO.md`
+- quarterly restore verification is now expected via `bin/run_pd_restore_verification.sh`
 
 ## PD / TrueNAS deployment recommendation
 
@@ -93,6 +98,34 @@ Install in **root's crontab on PD**:
 # END PD BACKUPS
 ```
 
+## Recommended quarterly restore verification
+
+Run from PD after a backup has completed:
+
+```bash
+cd /mnt/docker-ssd/docker/compose/automation
+/usr/bin/bash bin/run_pd_restore_verification.sh
+```
+
+This validates:
+
+- latest configured Postgres dumps can be restored cleanly
+- a real config/appdata sample can be pulled back from Serenity
+- expected files still exist in the restored copy
+
+If node-exporter is configured with the textfile collector, the wrapper also publishes:
+
+- `pd_restore_verification_success`
+- `pd_restore_verification_last_run_timestamp_seconds`
+- `pd_restore_verification_last_success_timestamp_seconds`
+- `pd_restore_verification_duration_seconds`
+
+Recommended root cron entry on PD:
+
+```cron
+30 3 1 */3 * cd /mnt/docker-ssd/docker/compose/automation && /usr/bin/bash bin/run_pd_restore_verification.sh >> /mnt/tank/docker/backups/pd-restore-verify.log 2>&1
+```
+
 Recommended first-run checklist:
 
 ```bash

automation/bin/pd_backup_lib.sh

@@ -71,3 +71,8 @@ ssh_args() {
 ssh_cmd() {
   printf 'ssh %s' "$(ssh_args)"
 }
+
+latest_dump_for_db() {
+  local db="$1"
+  find "$PD_DB_DUMP_ROOT" -maxdepth 1 -type f -name "${db}_*.sql.gz" | sort | tail -n 1
+}

automation/bin/pd_restore_verify_appdata.sh

@@ -0,0 +1,34 @@
+#!/usr/bin/bash
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
+# shellcheck disable=SC1091
+source "$SCRIPT_DIR/pd_backup_lib.sh"
+
+verify_root="${RESTORE_VERIFY_WORK_ROOT:-/mnt/tank/docker/backups/restore-verify}"
+remote_source="${RESTORE_VERIFY_CONFIG_SOURCE:-tank-docker/appdata/grafana}"
+required_paths="${RESTORE_VERIFY_REQUIRED_PATHS:-grafana.db dashboards}"
+stage_dir="$verify_root/appdata-stage"
+
+mkdir -p "$verify_root"
+rm -rf "$stage_dir"
+mkdir -p "$stage_dir"
+
+remote_path="$SERENITY_BACKUP_ROOT/$remote_source/"
+
+echo "Staging config restore sample from $SERENITY_BACKUP_HOST:$remote_path"
+rsync -a -e "$(ssh_cmd)" "$SERENITY_BACKUP_HOST:$remote_path" "$stage_dir/"
+
+for rel in $required_paths; do
+  if [[ ! -e "$stage_dir/$rel" ]]; then
+    echo "Missing expected restored path: $rel" >&2
+    exit 1
+  fi
+done
+
+if [[ -f "$stage_dir/grafana.db" ]] && command -v sqlite3 >/dev/null 2>&1; then
+  sqlite3 "$stage_dir/grafana.db" 'PRAGMA integrity_check;' | grep -qx 'ok'
+  echo "Verified SQLite integrity for grafana.db"
+fi
+
+echo "Config restore verification passed for $remote_source."

automation/bin/pd_restore_verify_postgres.sh

@@ -0,0 +1,69 @@
+#!/usr/bin/bash
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
+# shellcheck disable=SC1091
+source "$SCRIPT_DIR/pd_backup_lib.sh"
+
+verify_root="${RESTORE_VERIFY_WORK_ROOT:-/mnt/tank/docker/backups/restore-verify}"
+dbs="${RESTORE_VERIFY_DB_LIST:-${POSTGRES_DB_LIST:-}}"
+
+if [[ -z "$dbs" ]]; then
+  echo "RESTORE_VERIFY_DB_LIST and POSTGRES_DB_LIST are both empty." >&2
+  exit 1
+fi
+
+mkdir -p "$verify_root"
+stamp="$(timestamp_utc)"
+container="pd-restore-verify-$stamp"
+password="verify-$stamp"
+
+if [[ -n "${RESTORE_VERIFY_POSTGRES_IMAGE:-}" ]]; then
+  verify_image="$RESTORE_VERIFY_POSTGRES_IMAGE"
+else
+  source_major="$(docker_cmd exec "$POSTGRES_CONTAINER" postgres -V | awk '{print $3}' | cut -d. -f1)"
+  verify_image="postgres:${source_major}-alpine"
+fi
+
+cleanup() {
+  docker_cmd rm -f "$container" >/dev/null 2>&1 || true
+}
+trap cleanup EXIT
+
+echo "Starting temporary Postgres verifier container $container"
+docker_cmd run -d --rm \
+  --name "$container" \
+  -e POSTGRES_PASSWORD="$password" \
+  "$verify_image" >/dev/null
+
+echo "Waiting for temporary Postgres to accept connections"
+for _ in {1..30}; do
+  if docker_cmd exec "$container" pg_isready -U postgres >/dev/null 2>&1; then
+    break
+  fi
+  sleep 2
+done
+
+docker_cmd exec "$container" pg_isready -U postgres >/dev/null
+
+for db in $dbs; do
+  dump_file="$(latest_dump_for_db "$db")"
+  if [[ -z "$dump_file" ]]; then
+    echo "No dump found for database '$db' under $PD_DB_DUMP_ROOT" >&2
+    exit 1
+  fi
+
+  echo "Restoring $dump_file into temporary database '$db'"
+  while IFS= read -r role_name; do
+    [[ -z "$role_name" ]] && continue
+    docker_cmd exec "$container" createuser -U postgres "$role_name" >/dev/null 2>&1 || true
+  done < <(gunzip -c "$dump_file" | sed -n "s/.*OWNER TO \\([^;]*\\);/\\1/p" | sort -u)
+
+  docker_cmd exec "$container" createdb -U postgres "$db"
+  gunzip -c "$dump_file" | docker_cmd exec -i "$container" psql -v ON_ERROR_STOP=1 -U postgres -d "$db" >/dev/null
+
+  table_count="$(docker_cmd exec "$container" psql -At -U postgres -d "$db" -c "SELECT count(*) FROM information_schema.tables WHERE table_schema NOT IN ('pg_catalog','information_schema');")"
+  echo "Verified database '$db' with $table_count non-system tables."
+done
+
+echo "Postgres restore verification passed."

automation/bin/run_pd_restore_verification.sh

@@ -0,0 +1,55 @@
+#!/usr/bin/bash
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
+# shellcheck disable=SC1091
+source "$SCRIPT_DIR/pd_backup_lib.sh"
+
+metrics_dir="${RESTORE_VERIFY_METRICS_DIR:-/mnt/tank/docker/metrics/node-exporter}"
+metrics_file="$metrics_dir/pd_restore_verification.prom"
+start_epoch="$(date +%s)"
+status=1
+
+mkdir -p "$metrics_dir"
+
+previous_success_epoch() {
+  if [[ -f "$metrics_file" ]]; then
+    awk '/^pd_restore_verification_last_success_timestamp_seconds /{print $2}' "$metrics_file" | tail -n 1
+  else
+    echo 0
+  fi
+}
+
+write_metrics() {
+  local end_epoch duration last_success tmp
+  end_epoch="$(date +%s)"
+  duration="$((end_epoch - start_epoch))"
+  last_success="$(previous_success_epoch)"
+  if [[ "$status" -eq 0 ]]; then
+    last_success="$end_epoch"
+  fi
+  tmp="${metrics_file}.tmp"
+  cat > "$tmp" <<EOF
+# HELP pd_restore_verification_success Last PD restore verification result (1=success, 0=failure).
+# TYPE pd_restore_verification_success gauge
+pd_restore_verification_success $(( status == 0 ? 1 : 0 ))
+# HELP pd_restore_verification_last_run_timestamp_seconds Unix timestamp of the last PD restore verification run.
+# TYPE pd_restore_verification_last_run_timestamp_seconds gauge
+pd_restore_verification_last_run_timestamp_seconds $end_epoch
+# HELP pd_restore_verification_last_success_timestamp_seconds Unix timestamp of the last successful PD restore verification run.
+# TYPE pd_restore_verification_last_success_timestamp_seconds gauge
+pd_restore_verification_last_success_timestamp_seconds $last_success
+# HELP pd_restore_verification_duration_seconds Duration of the last PD restore verification run in seconds.
+# TYPE pd_restore_verification_duration_seconds gauge
+pd_restore_verification_duration_seconds $duration
+EOF
+  mv "$tmp" "$metrics_file"
+}
+
+trap write_metrics EXIT
+
+"$SCRIPT_DIR/pd_restore_verify_postgres.sh"
+"$SCRIPT_DIR/pd_restore_verify_appdata.sh"
+
+status=0
+echo "PD restore verification run complete."

dev/docker-compose.yaml

@@ -30,6 +30,7 @@ services:
       GITEA__database__NAME: ${GITEA_DB_NAME}
       GITEA__database__USER: ${GITEA_DB_USER}
       GITEA__database__PASSWD: ${GITEA_DB_PASS}
+      GITEA__service__DISABLE_REGISTRATION: true
     volumes:
       - /mnt/docker-ssd/docker/appdata/gitea:/data
     healthcheck:

docs/architecture/ARCHITECTURE_OVERVIEW.md

@@ -8,7 +8,7 @@ Full homelab stack as of 2026-05-09. All 6 expansion phases are complete and dep
 |--------|----|----|------|
 | **PlausibleDeniability (PD)** | TrueNAS Scale 25.10.2.1 | 10.5.1.6 | Primary Docker host — all compose stacks |
 | **Serenity** | Unraid 7.2.4 | 10.5.1.5 | NAS, ARR stack, CPU reranker |
-| **N.O.M.A.D.** | Ubuntu 25.10 | 10.5.1.16 | Light Ollama models, game servers |
+| **N.O.M.A.D.** | Ubuntu 25.10 | 10.5.1.16 | Offline knowledge, game servers, and standalone local services |
 | **Rocinante** | (bare metal) | 10.5.1.112 | Heavy Ollama models (RTX 4090) |
 
 ## Network
@@ -28,10 +28,10 @@ Full homelab stack as of 2026-05-09. All 6 expansion phases are complete and dep
 - **Dockhand** (3230) — Docker management UI
 
 ### AI Layer
-- **LiteLLM** (4000) — unified LLM proxy; routes to Ollama on N.O.M.A.D. (light) and Rocinante (heavy)
+- **LiteLLM** (4000) — unified LLM proxy; routes to Ollama on PD (light) and Rocinante (heavy)
 - **OpenWebUI** (8282) — chat UI with RAG, STT, and web search
 - **Qdrant** (6333/6334) — vector database for RAG
-- **Whisper** (8786) — CUDA speech-to-text on PD's RTX 2080 Ti
+- **Whisper** (8786 / 8787) — CPU speech-to-text on N.O.M.A.D. plus CUDA speech-to-text on Rocinante
 - **SearXNG** (8888) — private web search backend
 - **Reranker** (Serenity:9787) — TEI bge-reranker-v2-m3, CPU-only
 
@@ -57,7 +57,7 @@ Full homelab stack as of 2026-05-09. All 6 expansion phases are complete and dep
 ## Key Architectural Decisions
 
 1. **LiteLLM as unified proxy** — all AI clients target LiteLLM; model routing is transparent to clients
-2. **Reranker on Serenity (CPU)** — frees RTX 2080 Ti on PD exclusively for Whisper CUDA; CPU latency acceptable for RAG reranking
+2. **Reranker on Serenity (CPU)** — keeps reranking off the primary inference GPUs; CPU latency is acceptable for RAG reranking
 3. **External Docker networks** — `ai-services` bridges all AI containers across compose files; `ix-databases_shared-databases` provides shared DB access
 4. **NFS for Immich** — photo library lives on Serenity's large array; PD mounts on demand (not at boot)
 5. **TrueNAS manages Docker** — never use `systemctl restart docker` on PD; use `docker compose` or `docker restart` for individual containers

docs/architecture/SERVICES_DIRECTORY.md

@@ -2,7 +2,7 @@
 
 All homelab services, their hosts, ports, and current status.
 
-*Last updated: 2026-05-10*
+*Last updated: 2026-05-20*
 
 ## Shared Databases (PD)
 
@@ -24,6 +24,7 @@ All homelab services, their hosts, ports, and current status.
 | Shlink | 8087 | http://pd:8087 | ✅ Active |
 | Shlink Web Client | 8088 | http://pd:8088 | ✅ Active |
 | RackPeek | 8283 | http://pd:8283 | ✅ Active |
+| Authelia | 9091 | http://pd:9091 / https://auth.paccoco.com | ✅ Active internally on PD; public Pangolin route still pending |
 
 ## AI Stack
 
@@ -36,7 +37,7 @@ All homelab services, their hosts, ports, and current status.
 | Whisper (faster-whisper, CUDA large-v3) | Rocinante (10.5.1.112) | 8787 | ✅ Active |
 | SearXNG | PD | 8888 | ✅ Active |
 | OpenClaw | PD | 18789 | ✅ Active |
-| Ollama — light tier | N.O.M.A.D. (10.5.1.16) | 11434 | ✅ Active |
+| Ollama — light tier | PD (10.5.1.6) | 11434 | ✅ Active |
 | Ollama — heavy tier | Rocinante (10.5.1.112) | 11434 | ✅ Active |
 | Reranker (TEI) | Serenity (10.5.1.5) | 9787 | ✅ Active |
 
@@ -76,8 +77,17 @@ All homelab services, their hosts, ports, and current status.
 |---------|------|--------|
 | Grafana | 3000 | ✅ Active |
 | Prometheus | 9090 | ✅ Active |
+| Loki | internal | ✅ Active |
+| Promtail | internal | ✅ Active |
+| cAdvisor | internal | ✅ Active |
 | MeshMonitor | 8081 | ✅ Active |
 
+## Radio / Mesh
+
+| Service | Host | Port | Status |
+|---------|------|------|--------|
+| MeshCore to MQTT relay | N.O.M.A.D. (10.5.1.16) | outbound MQTT only | ✅ Running as Docker container `mctomqtt` (`mctomqtt:latest`); config bind-mounted from `/etc/mctomqtt` |
+
 ## Mapping / Geo (PD)
 
 | Service | Port | Status |

docs/operations/BACKUP_POLICY.md

@@ -45,6 +45,10 @@ Recommended live deployment model on PD: a plain **root cron** job invoking `/us
 - Test restores quarterly
 - Test at least one app DB restore and one full-config restore
 - Verify that restored stacks can start with current compose files
+- Repo-staged verification runner: automation/bin/run_pd_restore_verification.sh
+- Current default verification targets:
+  - latest dumps from POSTGRES_DB_LIST
+  - Serenity-backed tank-docker/appdata/grafana sample restore with required files present
 
 ## Operational Notes
 - Media libraries on Unraid are large and may not need the same backup frequency as configs/databases

docs/operations/GITEA_SSO_PREP.md

@@ -0,0 +1,64 @@
+# Gitea SSO Prep
+
+This is the staged cutover plan for moving the live PD Gitea instance to Authelia-backed sign-in without flipping it live yet.
+
+## Current live shape
+
+- stack path: `dev/`
+- public URL: `https://gitea.paccoco.com`
+- internal HTTP: `http://gitea:3000` on the `pangolin` network
+- shared Postgres backend already in use
+- Gitea version observed live on PD: `1.25.3`
+
+## Safe hardening before SSO
+
+- disable public self-registration
+- keep normal local admin login working until Authelia sign-in is proven
+- do not change `ROOT_URL` or SSH settings during the auth cutover
+
+## Authelia side
+
+Add a dedicated OIDC client for Gitea in the live Authelia `configuration.yml` when ready to test.
+
+Recommended values:
+
+- `client_id`: `gitea`
+- `client_name`: `Gitea`
+- `authorization_policy`: `one_factor`
+- `scopes`: `openid`, `profile`, `email`, `groups`
+- `grant_types`: `authorization_code`
+- `response_types`: `code`
+- `token_endpoint_auth_method`: `client_secret_basic`
+
+Redirect URI guidance:
+
+- use the callback URL shown by the Gitea authentication-source form
+- for standard Gitea OIDC sources this is typically under `/user/oauth2/<source-name>/callback`
+- validate the exact callback path against the live Gitea UI before committing the Authelia client entry
+
+## Gitea side
+
+In Gitea:
+
+1. Sign in as a local admin.
+2. Go to `Site Administration` -> `Authentication Sources`.
+3. Add a new source of type `OpenID Connect`.
+4. Use Authelia discovery if the form supports it, otherwise enter the manual endpoints from `https://auth.paccoco.com/.well-known/openid-configuration`.
+
+Recommended source values:
+
+- name: `Authelia`
+- scopes: `openid profile email groups`
+- required claim mapping priority: email first, then username/login
+- keep the source enabled only after confirming the callback URL matches the Authelia client entry
+
+## Rollout notes
+
+- keep local admin auth available for the first pass
+- test with one admin account first
+- after successful sign-in, decide whether to keep local auth as break-glass only
+- do not disable Pangolin auth in front of Gitea until Authelia login is actually working end-to-end
+
+## Related weak spot
+
+The live Gitea app.ini observed on PD had public self-registration enabled. That should stay off for this homelab unless there is a deliberate reason to reopen it.

docs/operations/IDENTITY_AND_SSO.md

@@ -0,0 +1,60 @@
+# Identity and SSO
+
+Authelia is the staged identity provider for the homelab.
+
+## Why Authelia
+
+- centralizes auth for the growing pile of web apps
+- works with shared Postgres/Redis already in the lab
+- supports OIDC for apps that speak it properly
+- avoids inventing separate local auth for every exposed service
+
+## Current scope
+
+- stack path: identity/
+- host target: PlausibleDeniability
+- public entrypoint: auth.paccoco.com
+- status: repo-staged, not assumed live until deployed on PD
+
+## Initial bootstrap flow
+
+1. Copy identity/.env.example to identity/.env and generate fresh secrets.
+2. Copy identity/authelia/configuration.yml.example to identity/authelia/configuration.yml.
+3. Render concrete secrets and passwords into identity/authelia/configuration.yml on PD; do not rely on runtime env interpolation inside the YAML.
+4. Copy identity/authelia/users_database.example.yml to identity/authelia/users_database.yml.
+5. Generate an Argon2 password hash with the Authelia image and replace the sample hash.
+6. Create the authelia Postgres database and user in the shared database stack.
+7. Set `AUTHELIA_SESSION_REDIS_PASSWORD` from the live shared Redis stack on PD.
+8. Add a Pangolin resource for auth.paccoco.com pointing at container authelia:9091.
+9. Validate with docker compose --env-file .env config and docker exec authelia authelia config validate --config /config/configuration.yml.
+
+## Rollout order
+
+This repo stage is the control-plane bootstrap, not the full client cutover.
+
+After Authelia is healthy on PD, wire OIDC or forward-auth clients one at a time.
+
+Recommended first targets:
+
+- Grafana
+- Gitea
+- any new public app before it gains yet another standalone login
+
+## Grafana first-cutover notes
+
+- Use Authelia OIDC as a Generic OAuth provider in Grafana.
+- Keep Grafana's local admin login enabled for the first rollout.
+- After OIDC is verified, disable Pangolin auth in front of Grafana to avoid double-auth.
+
+## Gitea prep notes
+
+- The live Gitea stack is the `dev/` compose on PD.
+- Disable public self-registration before or during the SSO prep pass.
+- Keep local admin login working for the first OIDC test.
+- Stage the exact Gitea auth-source values in `docs/operations/GITEA_SSO_PREP.md` before creating the live source.
+
+## Notes
+
+- File-backed users are intentional for the initial rollout; this is enough for the current household scale.
+- If identity sprawl grows later, migrate the authentication backend to LDAP instead of bolting extra auth stores onto apps ad hoc.
+- OIDC signing keys and per-app client secrets should be generated during deployment on PD, not committed to the repo.

docs/operations/PD_BACKUP_DEPLOYMENT.md

@@ -68,6 +68,12 @@ Preferred: install in **root's crontab** on PD.
 # END PD BACKUPS
 ```
 
+Quarterly restore verification:
+
+```cron
+30 3 1 */3 * cd /mnt/docker-ssd/docker/compose/automation && /usr/bin/bash bin/run_pd_restore_verification.sh >> /mnt/tank/docker/backups/pd-restore-verify.log 2>&1
+```
+
 ## Post-deploy checks
 
 ```bash
@@ -76,6 +82,22 @@ ls -lh /mnt/tank/docker/backups/db-dumps
 ssh root@10.5.1.5 'find /mnt/user/backups/plausible-deniability -maxdepth 2 -type d | sort | head -40'
 ```
 
+## Restore verification
+
+After the backup path is stable, run the staged restore verifier from PD:
+
+```bash
+cd /mnt/docker-ssd/docker/compose/automation
+/usr/bin/bash bin/run_pd_restore_verification.sh
+```
+
+What it proves:
+
+- the latest configured Postgres dumps can restore into a temporary container
+- a real appdata/config sample can be staged back from Serenity
+- expected files still exist inside the restored sample
+- the latest run state can be exported to Prometheus via node-exporter's textfile collector
+
 ## Important limitations
 
 - Do **not** use `systemctl restart docker` on PD.

docs/reference/MESHTASTIC.md

@@ -3,6 +3,8 @@
 ## Overview
 Self-hosted Meshtastic monitoring and management stack running on PlausibleDeniability.
 
+Related NOMAD-side bridge note: a separate MeshCore-to-MQTT relay is running on N.O.M.A.D. as Docker container `mctomqtt` (`mctomqtt:latest`), with config bind-mounted from `/etc/mctomqtt`. The package also includes a systemd unit under `/opt/mctomqtt`, but that is not the active runtime on this host.
+
 ## Stack Location
 `/mnt/docker-ssd/docker/compose/meshtastic/`
 

docs/servers/NOMAD.md

@@ -199,6 +199,46 @@ These are expected to be managed by Wings/Pelican rather than a local compose fi
   - Topic thumbs write back to `doris-digest/data/feedback.json`.
   - Paperless acknowledge/dismiss state writes to `/opt/doris-dashboard/data/paperless_review_state.json` on the live host.
 
+### MeshCore to MQTT relay
+- Install date: 2026-05-18
+- Install root: `/opt/mctomqtt`
+- Config root: `/etc/mctomqtt`
+- Service account: `mctomqtt:mctomqtt`
+- Installer metadata: `.version_info` reports `installer_version: 1.3.0.0-preview`, repo `Cisien/meshcoretomqtt`, branch `main`, commit `10f7c0e`
+- Detected install type marker: `/opt/mctomqtt/.install_type` contains `docker`
+- Runtime model:
+  - Active runtime is a standalone Docker container named `mctomqtt`
+  - Image: `mctomqtt:latest`
+  - Command: `python3 /opt/mctomqtt/mctomqtt.py`
+  - User: `mctomqtt`
+  - Restart policy: `unless-stopped`
+  - Network mode: `bridge`
+  - Config bind mount: `/etc/mctomqtt:/etc/mctomqtt:ro`
+  - Device passthrough: the Heltec serial device is mapped directly into the container at the same `/dev/serial/by-id/... ` path
+- Systemd note:
+  - The packaged repo also includes `/opt/mctomqtt/mctomqtt.service`, but it is not the active runtime on NOMAD
+  - No installed unit was present in `/etc/systemd/system` or `/lib/systemd/system`
+  - The bundled unit expects `/opt/mctomqtt/venv/bin/python3`, but no `venv/` directory exists under `/opt/mctomqtt`
+  - Treat the systemd unit as an unused package artifact for this host unless the deployment model changes later
+- Config layering:
+  - Base reference: `/etc/mctomqtt/config.toml`
+  - Preset overlay: `/etc/mctomqtt/config.d/10-meshmapper.toml`
+  - Local overrides: `/etc/mctomqtt/config.d/99-user.toml`
+- Current local overrides observed:
+  - IATA region code set to `BNA`
+  - Serial device pinned to `/dev/serial/by-id/usb-Espressif_Systems_heltec_wifi_lora_32_v4__16_MB_FLASH__2_MB_PSRAM__441BF670B684-if00`
+  - Enabled broker targets include MeshMapper, LetsMesh US, and NashMe
+  - Credentials/secrets are stored inline in `99-user.toml`; do not mirror them into git-backed docs
+- Container health observed during inspection:
+  - Logs show the relay loading all three config layers, opening the Heltec serial device, and connecting successfully to MeshMapper, NashMe, and LetsMesh US after restart
+  - The current container had been up about 5 minutes during inspection with `MQTT: 3/3`, zero failures, and live RX air-time reported
+  - An earlier start logged one DNS resolution failure for a broker label shown as `custom-2`, but the current running container recovered and all three named brokers connected cleanly
+- Container logs / inspection:
+  - Use `docker logs mctomqtt` for runtime output
+  - Use `docker inspect mctomqtt` for bind mounts, device mappings, and restart policy
+- Documentation linkage:
+  - Related mesh stack notes live in [MESHTASTIC.md](../reference/MESHTASTIC.md)
+
 ### Live deployment path rule
 - On NOMAD, live service roots belong under `/opt/<service>`.
 - That includes host-run services **and** standalone Docker Compose deployments that are not part of the core Project N.O.M.A.D. stack.

docs/servers/PLAUSIBLEDENABILITY.md

@@ -31,18 +31,24 @@ Primary Docker host running all production workloads.
 RTX 2080 Ti used by:
 - Plex (hardware transcoding)
 - immich-ml (CUDA face recognition)
-- Ollama light-tier inference (qwen2.5:14b and smaller)
+- Ollama light-tier inference for the main AI stack and local Honcho backend
 
 > **Note:** Whisper moved to N.O.M.A.D. (10.5.1.16:8786) as CPU inference — avoids VRAM conflict with Ollama. See [NOMAD.md](NOMAD.md).
 
 Recommended models for 11GB VRAM:
 | Model | VRAM | Best for |
 |-------|------|---------|
-| `qwen2.5:14b` | ~8GB | General use, coding, reasoning |
+| `qwen2.5:7b` | ~5GB | Cheap/default reasoning, summaries, derivation |
+| `qwen2.5:14b` | ~8GB | Heavier reasoning / high-max paths |
 | `qwen2.5-coder:7b` | ~5GB | Code tasks |
 | `deepseek-r1:8b` | ~5GB | Math, logic |
 | `phi4` | ~8GB | Analytical tasks |
 
+Current light-tier notes:
+- PD Ollama is the active light-tier endpoint for LiteLLM on `http://10.5.1.6:11434`
+- Local Honcho currently points at PD Ollama over LAN (`http://10.5.1.6:11434/v1`) instead of using a separate local model host
+- Current Honcho routing split keeps embeddings on `nomic-embed-text:latest`, cheap/default paths on `qwen2.5:7b`, and deeper high/max paths on `qwen2.5:14b`
+
 ## Remote Access
 - Pangolin VPS + Newt for external services
 - SSH from Windows 11

home/doris-dashboard/.gitignore

@@ -11,3 +11,4 @@ data/todos.json
 data/weather.json
 data/paperless_review.json
 data/paperless_review_state.json
+data/hermes_usage.json

home/doris-dashboard/bin/generate_dashboard.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python3
 from __future__ import annotations
 
-import html, json, math, os, re, shutil, subprocess, urllib.error, urllib.request
+import html, json, math, os, re, shutil, sqlite3, subprocess, time, urllib.error, urllib.request, yaml
 from collections import Counter
 from datetime import datetime, timezone
 from pathlib import Path
@@ -15,6 +15,9 @@ PUBLIC=ROOT/'public'; DATA=ROOT/'data'; LOGS=ROOT/'logs'
 PAPERLESS_STATE_PATH=DATA/'paperless_review_state.json'
 LAT=36.53; LON=-87.36
 LOCAL_TZ=ZoneInfo('America/Chicago')
+HERMES_HOME=Path.home()/'.hermes'
+HERMES_STATE_DB=HERMES_HOME/'state.db'
+HERMES_CONFIG=HERMES_HOME/'config.yaml'
 
 LABELS={
  'astronomy':('🔭','Astronomy'),'homelab':('🏠','Homelab'),'smart_home':('🏡','Smart Home'),
@@ -50,6 +53,37 @@ def kmh_to_mph(k:float|int|None)->str:
     if k is None: return '—'
     return f"{round(k*0.621371)} mph"
 
+def compact_number(value:int|float|None)->str:
+    if value in (None,''): return '—'
+    try: n=float(value)
+    except Exception: return str(value)
+    abs_n=abs(n)
+    if abs_n >= 1_000_000_000: return f"{n/1_000_000_000:.1f}B"
+    if abs_n >= 1_000_000: return f"{n/1_000_000:.1f}M"
+    if abs_n >= 1_000: return f"{n/1_000:.1f}K"
+    if n.is_integer(): return f"{int(n):,}"
+    return f"{n:,.1f}"
+
+def total_tokens_value(row:dict[str,Any] | sqlite3.Row)->int:
+    keys=('input_tokens','output_tokens','cache_read_tokens','cache_write_tokens','reasoning_tokens')
+    total=0
+    for key in keys:
+        try: total += int((row[key] if isinstance(row, sqlite3.Row) else row.get(key,0)) or 0)
+        except Exception: pass
+    return total
+
+def read_hermes_model_config()->dict[str,str]:
+    default='unknown'; provider='unknown'
+    try:
+        data=yaml.safe_load(HERMES_CONFIG.read_text(encoding='utf-8')) or {}
+    except Exception:
+        return {'default_model':default,'provider':provider}
+    model_cfg=data.get('model') if isinstance(data,dict) else {}
+    if isinstance(model_cfg,dict):
+        default=str(model_cfg.get('default') or default)
+        provider=str(model_cfg.get('provider') or provider)
+    return {'default_model':default or 'unknown','provider':provider or 'unknown'}
+
 def moon_phase(dt:datetime)->dict[str,str]:
     # Known new moon: 2000-01-06 18:14 UTC; synodic month 29.53058867 days.
     known=datetime(2000,1,6,18,14,tzinfo=timezone.utc)
@@ -227,6 +261,51 @@ def homelab_pulse(candidates:list[dict[str,Any]])->list[dict[str,str]]:
       {'label':'Workspace disk','value':disk,'state':'ok' if usage.free/usage.total>.15 else 'warn'},
     ]
 
+
+def system_snapshot()->list[dict[str,str]]:
+    uptime_text='unknown'
+    try:
+        uptime_seconds=float(Path('/proc/uptime').read_text().split()[0])
+        hours=int(uptime_seconds//3600)
+        minutes=int((uptime_seconds%3600)//60)
+        uptime_text=(f'{hours}h {minutes}m' if hours else f'{minutes}m')
+    except Exception:
+        pass
+    load_text='unknown'
+    load_state='ok'
+    try:
+        load1,load5,load15=os.getloadavg()
+        load_text=f'{load1:.2f} / {load5:.2f} / {load15:.2f}'
+        cpu_count=os.cpu_count() or 1
+        if load1 > cpu_count*1.25:
+            load_state='warn'
+    except Exception:
+        load_state='warn'
+    mem_text='unknown'
+    mem_state='ok'
+    try:
+        parts=Path('/proc/meminfo').read_text().splitlines()
+        raw={line.split(':',1)[0]: int(line.split(':',1)[1].strip().split()[0]) for line in parts if ':' in line}
+        total=raw.get('MemTotal',0)
+        avail=raw.get('MemAvailable',0)
+        used=max(total-avail,0)
+        pct=(used/total*100) if total else 0
+        mem_text=f'{pct:.0f}% used ({used/1024/1024:.1f} / {total/1024/1024:.1f} GiB)'
+        if pct >= 85:
+            mem_state='warn'
+    except Exception:
+        mem_state='warn'
+    root_usage=shutil.disk_usage('/')
+    root_pct=root_usage.used/root_usage.total*100 if root_usage.total else 0
+    root_text=f'{root_pct:.0f}% used ({root_usage.free/1024**3:.1f} GiB free)'
+    return [
+      {'label':'Host','value':run(['hostname']) or 'nomad','state':'ok'},
+      {'label':'Uptime','value':uptime_text,'state':'ok'},
+      {'label':'Load avg','value':load_text,'state':load_state},
+      {'label':'Memory','value':mem_text,'state':mem_state},
+      {'label':'Root disk','value':root_text,'state':'warn' if root_pct >= 85 else 'ok'},
+    ]
+
 def score_item(item:dict[str,Any],prefs:dict[str,Any])->float:
     score=float(item.get('priority',1)); text=f"{item.get('title','')} {item.get('summary','')}".lower()
     if item.get('category') in prefs.get('boost_categories',[]): score+=2
@@ -313,7 +392,7 @@ def why(i:dict[str,Any])->str:
     return f'Interesting signal from {src}.'
 
 def card(i:dict[str,Any], compact:bool=False)->str:
-    summary=(i.get('summary') or '').strip()[:170 if compact else 300]
+    summary=(i.get('summary') or '').strip()[:120 if compact else 260]
     when=(i.get('published_at','')[:16].replace('T',' '))
     cls='compact-card' if compact else 'card'
     why_html='' if compact else f"<div class='why'>Why: {esc(why(i))}</div>"
@@ -371,6 +450,21 @@ def load_paperless_review()->list[dict[str,Any]]:
 def load_paperless_state()->dict[str,Any]:
     return load_json(PAPERLESS_STATE_PATH,{}) or {}
 
+def paperless_focus_summary(items:list[dict[str,Any]]|None=None,state:dict[str,Any]|None=None)->dict[str,Any]:
+    items=items if items is not None else load_paperless_review()
+    state=state if state is not None else load_paperless_state()
+    active=[]; flagged=[]
+    for item in items:
+        status=((state.get(str(item.get('id'))) or {}).get('status') or '').lower()
+        if status in {'dismissed','good_to_go'}:
+            continue
+        if status in {'acknowledged','needs_attention'}:
+            flagged.append(item)
+        else:
+            active.append(item)
+    headline=(flagged[0] if flagged else (active[0] if active else {})).get('title') or 'No documents waiting'
+    return {'active_count':len(active),'flagged_count':len(flagged),'headline':headline}
+
 def paperless_review_section()->str:
     items=load_paperless_review()
     state=load_paperless_state()
@@ -408,7 +502,8 @@ def paperless_review_section()->str:
     flagged_cards=''.join(render_card(item,True) for item in needs_attention[:8])
     count=len(active)
     flag_count=len(needs_attention)
-    body=f"<div class='paperless-grid'>{active_cards or '<p class=\'empty\'>No active documents waiting.</p>'}</div>"
+    empty_active="<p class='empty'>No active documents waiting.</p>"
+    body=f"<div class='paperless-grid'>{active_cards or empty_active}</div>"
     if overflow_active:
         body += f"<details class='paperless-more'><summary>Show {esc(str(len(overflow_active)))} more active items</summary><div class='paperless-grid overflow-grid'>{''.join(render_card(item,False) for item in overflow_active[:8])}</div></details>"
     if flag_count:
@@ -427,7 +522,10 @@ def list_panel(title:str,items:list[dict[str,Any]],empty:str)->str:
             headline=x.get('title') if x.get('time') else ''
             assignee=x.get('assignee') or ''
             detail=x.get('body') or x.get('summary') or x.get('value') or ''
-            rows.append(f"<li class='panel-item'><div class='panel-item-top'><strong>{esc(kicker)}</strong>{f'<span class=\'muted\'>{esc(assignee)}</span>' if assignee else ''}</div>{f'<div class=\'panel-item-title\'>{esc(headline)}</div>' if headline else ''}{f'<div class=\'panel-item-body\'>{esc(detail)}</div>' if detail else ''}</li>")
+            assignee_html=f"<span class='muted'>{esc(assignee)}</span>" if assignee else ''
+            headline_html=f"<div class='panel-item-title'>{esc(headline)}</div>" if headline else ''
+            detail_html=f"<div class='panel-item-body'>{esc(detail)}</div>" if detail else ''
+            rows.append(f"<li class='panel-item'><div class='panel-item-top'><strong>{esc(kicker)}</strong>{assignee_html}</div>{headline_html}{detail_html}</li>")
         body=''.join(rows)
     return f"<section class='panel'><h2>{esc(title)}</h2><ul>{body}</ul></section>"
 
@@ -449,7 +547,10 @@ def donetick_panel(items:list[dict[str,Any]],empty:str)->str:
         due_style=''
         if state=='overdue': due_style=' style="color:#ff6b6b;"'
         elif x.get('priority') in PRIORITY_COLORS: due_style=f' style="color:{esc(PRIORITY_COLORS[x.get("priority")])};"'
-        rows.append(f"<article class='task-card {state}'><div class='task-top'><div class='task-state-dot' aria-hidden='true'></div><div class='task-due'{due_style}>{esc(x.get('due_title') or x.get('time') or 'No due date')}</div>{f'<span class=\'task-assignee\'>{esc(x.get("assignee") or "")}</span>' if x.get('assignee') and x.get('assignee')!='Unassigned' else ''}</div><div class='task-name'>{esc(x.get('title') or 'Untitled task')}</div>{f'<div class=\'task-chips\'>{chips}</div>' if chips else ''}{f'<div class=\'task-subtitle\'>{esc(subtitle)}</div>' if subtitle else ''}</article>")
+        assignee_html=f"<span class='task-assignee'>{esc(x.get('assignee') or '')}</span>" if x.get('assignee') and x.get('assignee')!='Unassigned' else ''
+        chips_html=f"<div class='task-chips'>{chips}</div>" if chips else ''
+        subtitle_html=f"<div class='task-subtitle'>{esc(subtitle)}</div>" if subtitle else ''
+        rows.append(f"<article class='task-card {state}'><div class='task-top'><div class='task-state-dot' aria-hidden='true'></div><div class='task-due'{due_style}>{esc(x.get('due_title') or x.get('time') or 'No due date')}</div>{assignee_html}</div><div class='task-name'>{esc(x.get('title') or 'Untitled task')}</div>{chips_html}{subtitle_html}</article>")
     return f"<section class='panel donetick-panel'><h2>Donetick</h2><div class='task-grid'>{''.join(rows)}</div></section>"
 
 def feedback_bootstrap()->str:
@@ -584,10 +685,232 @@ def forecast_panel(weather:dict[str,Any])->str:
         return ""
     return collapsible_section('3-day forecast','Rain, sunrise, and sunset over the next three days.',f"<div class='forecast-grid'>{''.join(cards)}</div>",'forecast',open_by_default=False,extra_class='panel forecast-panel')
 
+def fetch_hermes_usage(days:int=30)->dict[str,Any]:
+    config=read_hermes_model_config()
+    stats={
+        'ok':False,
+        'period_days':days,
+        'default_model':config.get('default_model','unknown'),
+        'provider':config.get('provider','unknown'),
+        'sessions':0,
+        'messages':0,
+        'tool_calls':0,
+        'user_messages':0,
+        'assistant_messages':0,
+        'total_tokens':0,
+        'input_tokens':0,
+        'output_tokens':0,
+        'cache_read_tokens':0,
+        'cache_write_tokens':0,
+        'reasoning_tokens':0,
+        'models':[],
+        'platforms':[],
+        'top_tools':[],
+        'recent_sessions':[],
+        'generated_at':datetime.now(LOCAL_TZ).isoformat(),
+    }
+    if not HERMES_STATE_DB.exists():
+        stats['error']='Hermes state DB not found'
+        return stats
+    cutoff=time.time()-(days*86400)
+    try:
+        conn=sqlite3.connect(f"file:{HERMES_STATE_DB}?mode=ro", uri=True)
+        conn.row_factory=sqlite3.Row
+        cur=conn.cursor()
+        overview=cur.execute("""
+            SELECT
+              count(*) AS sessions,
+              sum(coalesce(input_tokens,0)) AS input_tokens,
+              sum(coalesce(output_tokens,0)) AS output_tokens,
+              sum(coalesce(cache_read_tokens,0)) AS cache_read_tokens,
+              sum(coalesce(cache_write_tokens,0)) AS cache_write_tokens,
+              sum(coalesce(reasoning_tokens,0)) AS reasoning_tokens,
+              sum(coalesce(api_call_count,0)) AS api_calls
+            FROM sessions
+            WHERE started_at >= ?
+        """,(cutoff,)).fetchone()
+        stats.update({
+            'sessions':int((overview['sessions'] or 0) if overview else 0),
+            'input_tokens':int((overview['input_tokens'] or 0) if overview else 0),
+            'output_tokens':int((overview['output_tokens'] or 0) if overview else 0),
+            'cache_read_tokens':int((overview['cache_read_tokens'] or 0) if overview else 0),
+            'cache_write_tokens':int((overview['cache_write_tokens'] or 0) if overview else 0),
+            'reasoning_tokens':int((overview['reasoning_tokens'] or 0) if overview else 0),
+        })
+        stats['total_tokens']=stats['input_tokens']+stats['output_tokens']+stats['cache_read_tokens']+stats['cache_write_tokens']+stats['reasoning_tokens']
+        row=cur.execute("SELECT count(*) AS c FROM messages WHERE role != 'session_meta' AND timestamp >= ?",(cutoff,)).fetchone()
+        stats['messages']=int((row['c'] or 0) if row else 0)
+        row=cur.execute("SELECT count(*) AS c FROM messages WHERE role='tool' AND timestamp >= ?",(cutoff,)).fetchone()
+        stats['tool_calls']=int((row['c'] or 0) if row else 0)
+        row=cur.execute("SELECT count(*) AS c FROM messages WHERE role='user' AND timestamp >= ?",(cutoff,)).fetchone()
+        stats['user_messages']=int((row['c'] or 0) if row else 0)
+        row=cur.execute("SELECT count(*) AS c FROM messages WHERE role='assistant' AND timestamp >= ?",(cutoff,)).fetchone()
+        stats['assistant_messages']=int((row['c'] or 0) if row else 0)
+        stats['models']=[dict(r) for r in cur.execute("""
+            SELECT model, count(*) AS sessions,
+                   sum(coalesce(input_tokens,0)+coalesce(output_tokens,0)+coalesce(cache_read_tokens,0)+coalesce(cache_write_tokens,0)+coalesce(reasoning_tokens,0)) AS total_tokens
+            FROM sessions
+            WHERE started_at >= ?
+            GROUP BY model
+            ORDER BY total_tokens DESC, sessions DESC
+            LIMIT 5
+        """,(cutoff,))]
+        stats['platforms']=[dict(r) for r in cur.execute("""
+            SELECT source, count(*) AS sessions,
+                   sum(coalesce(input_tokens,0)+coalesce(output_tokens,0)+coalesce(cache_read_tokens,0)+coalesce(cache_write_tokens,0)+coalesce(reasoning_tokens,0)) AS total_tokens
+            FROM sessions
+            WHERE started_at >= ?
+            GROUP BY source
+            ORDER BY total_tokens DESC, sessions DESC
+            LIMIT 5
+        """,(cutoff,))]
+        stats['top_tools']=[dict(r) for r in cur.execute("""
+            SELECT tool_name, count(*) AS calls
+            FROM messages
+            WHERE role='tool' AND tool_name IS NOT NULL AND tool_name != '' AND timestamp >= ?
+            GROUP BY tool_name
+            ORDER BY calls DESC, tool_name ASC
+            LIMIT 8
+        """,(cutoff,))]
+        stats['recent_sessions']=[dict(r) for r in cur.execute("""
+            SELECT source, model, started_at,
+                   (coalesce(input_tokens,0)+coalesce(output_tokens,0)+coalesce(cache_read_tokens,0)+coalesce(cache_write_tokens,0)+coalesce(reasoning_tokens,0)) AS total_tokens,
+                   coalesce(api_call_count,0) AS api_calls
+            FROM sessions
+            WHERE started_at >= ?
+            ORDER BY started_at DESC
+            LIMIT 5
+        """,(cutoff,))]
+        for item in stats['recent_sessions']:
+            started=item.get('started_at')
+            try: item['started_local']=datetime.fromtimestamp(float(started), LOCAL_TZ).strftime('%a %-I:%M %p')
+            except Exception: item['started_local']='recent'
+        stats['ok']=True
+        conn.close()
+        return stats
+    except Exception:
+        stats['error']='Hermes telemetry is temporarily unavailable.'
+        return stats
+
+def focus_card(title:str,kicker:str,headline:str,body:str,tone:str='')->str:
+    return f"<article class='focus-card {esc(tone)}'><span class='focus-kicker'>{esc(kicker)}</span><h3>{esc(title)}</h3><strong>{esc(headline)}</strong><p>{esc(body)}</p></article>"
+
+
+def build_focus_cards(local_items:list[dict[str,Any]], todos:list[dict[str,Any]], pulse:list[dict[str,str]], paperless:dict[str,Any])->str:
+    urgent_todos=[item for item in todos if (item.get('state') or '') in {'overdue','today'} and (item.get('source') or '') != 'Dashboard']
+    warn_items=[item for item in pulse if item.get('state')=='warn']
+    local_story=local_items[0] if local_items else None
+    if urgent_todos:
+        chores_headline=f"{len(urgent_todos)} urgent / due today"
+        first_urgent=urgent_todos[0]
+        chores_body=(first_urgent.get('title') or 'You have work waiting') + (f" • {first_urgent.get('due_title')}" if first_urgent.get('due_title') else '')
+    else:
+        chores_headline='No urgent chores'
+        chores_body='Nothing due soon enough to nag you here.'
+    cards=[
+        focus_card(
+            'Local watch',
+            'Useful nearby',
+            (local_story.get('title') if local_story else 'Local lane quiet'),
+            (f"{local_story.get('source','Local feed')} • {label(local_story.get('category','local'))}" if local_story else 'Nothing local survived filtering hard enough to earn the homepage.'),
+            'local' if local_story else ''
+        ),
+        focus_card(
+            'Chores with heat',
+            'Donetick',
+            chores_headline,
+            chores_body,
+            'warn' if urgent_todos else 'ok'
+        ),
+        focus_card(
+            'Paperless follow-up',
+            'Queue check',
+            (f"{paperless.get('active_count',0)} active · {paperless.get('flagged_count',0)} flagged" if paperless.get('active_count',0) or paperless.get('flagged_count',0) else 'No documents waiting'),
+            str(paperless.get('headline') or 'No documents waiting'),
+            'warn' if paperless.get('flagged_count',0) else ('ok' if not paperless.get('active_count',0) else '')
+        ),
+        focus_card(
+            'Homelab pulse',
+            'NOMAD checks',
+            ('Stable' if not warn_items else f"{len(warn_items)} warning{'s' if len(warn_items)!=1 else ''}"),
+            ('; '.join(f"{item.get('label')}: {item.get('value')}" for item in warn_items[:2]) if warn_items else 'Dashboard service, cron, digest freshness, and disk all look fine.'),
+            'warn' if warn_items else 'ok'
+        ),
+    ]
+    return "<section class='focus-grid'>" + ''.join(cards) + "</section>"
+
+
+def build_hero_summary(local_items:list[dict[str,Any]], todos:list[dict[str,Any]], pulse:list[dict[str,str]], paperless:dict[str,Any], hermes:dict[str,Any])->str:
+    urgent_todos=len([item for item in todos if (item.get('state') or '') in {'overdue','today'} and (item.get('source') or '') != 'Dashboard'])
+    warn_count=len([item for item in pulse if item.get('state')=='warn'])
+    local_status='local lane has signal' if local_items else 'local lane is quiet'
+    paperless_text=f"{paperless.get('active_count',0)} Paperless item{'s' if paperless.get('active_count',0)!=1 else ''}"
+    hermes_model=(hermes.get('models') or [{}])[0].get('model') or hermes.get('default_model') or 'your main model'
+    return f"{urgent_todos} urgent chore{'s' if urgent_todos!=1 else ''}, {paperless_text}, {warn_count} homelab warning{'s' if warn_count!=1 else ''}, and Hermes has mostly been running on {hermes_model} while the {local_status}."
+
+
+def list_rows(rows:list[str])->str:
+    return ''.join(f"<li>{row}</li>" for row in rows) if rows else "<li class='empty'>Nothing to show.</li>"
+
+
+def build_system_section(snapshot:list[dict[str,str]])->str:
+    metrics=''.join(metric_card(item.get('label','Metric'), item.get('value','—'), 'live host state', item.get('state','')) for item in snapshot)
+    subtitle='Live NOMAD host state: uptime, load, memory pressure, and root disk usage.'
+    return collapsible_section('System snapshot',subtitle,f"<section class='metrics metrics-compact'>{metrics}</section>",'system-snapshot',open_by_default=True)
+
+
+def build_hermes_section(hermes:dict[str,Any])->str:
+    if not hermes.get('ok'):
+        body="<section class='summary-grid'><article class='summary-card'><h3>Hermes telemetry unavailable</h3><p>Local Hermes usage stats could not be read right now.</p></article></section>"
+        return collapsible_section('Hermes usage',f"Could not read the last {hermes.get('period_days',30)} days of local Hermes state.",body,'hermes-usage',open_by_default=True)
+    metrics=''.join([
+        metric_card('Default model',str(hermes.get('default_model','unknown')),str(hermes.get('provider','unknown')),'space'),
+        metric_card(f"{hermes.get('period_days',30)}-day tokens",compact_number(hermes.get('total_tokens',0)),'all session rows','space'),
+        metric_card('Sessions / messages',f"{compact_number(hermes.get('sessions',0))} / {compact_number(hermes.get('messages',0))}",'state.db window','space'),
+        metric_card('Tool calls',compact_number(hermes.get('tool_calls',0)),f"{compact_number(hermes.get('user_messages',0))} user · {compact_number(hermes.get('assistant_messages',0))} assistant",'space'),
+    ])
+    model_rows=[]
+    for item in hermes.get('models',[])[:5]:
+        model_rows.append(f"{esc(item.get('model') or 'unknown')} · {compact_number(item.get('sessions',0))} sessions · {compact_number(item.get('total_tokens',0))} tokens")
+    platform_rows=[]
+    for item in hermes.get('platforms',[])[:5]:
+        platform_rows.append(f"{esc((item.get('source') or 'unknown').title())} · {compact_number(item.get('sessions',0))} sessions · {compact_number(item.get('total_tokens',0))} tokens")
+    tool_rows=[]
+    for item in hermes.get('top_tools',[])[:8]:
+        tool_rows.append(f"{esc(item.get('tool_name') or 'tool')} · {compact_number(item.get('calls',0))} calls")
+    recent_rows=[]
+    for item in hermes.get('recent_sessions',[])[:5]:
+        source=(item.get('source') or 'session').title()
+        model=item.get('model') or 'unknown'
+        recent_rows.append(f"{esc(source)} · {esc(item.get('started_local') or 'recent')} · {esc(model)} · {compact_number(item.get('total_tokens',0))} tokens")
+    detail_html=f"""
+    <section class='summary-grid'>
+      <article class='summary-card'>
+        <h3>Model mix</h3>
+        <ul>{list_rows(model_rows)}</ul>
+      </article>
+      <article class='summary-card'>
+        <h3>Platform split</h3>
+        <ul>{list_rows(platform_rows)}</ul>
+      </article>
+      <article class='summary-card'>
+        <h3>Top tools</h3>
+        <ul>{list_rows(tool_rows)}</ul>
+      </article>
+      <article class='summary-card'>
+        <h3>Recent sessions</h3>
+        <ul>{list_rows(recent_rows)}</ul>
+      </article>
+    </section>
+    """
+    body=f"<section class='metrics hermes-metrics'>{metrics}</section>{detail_html}"
+    subtitle=f"Live read from ~/.hermes/state.db over the last {hermes.get('period_days',30)} days."
+    return collapsible_section('Hermes usage',subtitle,body,'hermes-usage',open_by_default=True)
+
 def render()->str:
     candidates=load_json(DIGEST_ROOT/'data/candidates.json',[]); prefs=load_json(DIGEST_ROOT/'data/preferences.json',{})
     donetick_todos=fetch_donetick_todos(); todos=inject_dashboard_todos(donetick_todos or load_json(DATA/'todos.json',[])); calendar=load_json(DATA/'calendar.json',[]); notes=load_json(DATA/'notes.json',[])
-    selected=select_items(candidates,prefs); top_briefing=selected[:6]; top_urls={i.get('url','') for i in top_briefing if i.get('url')}; groups={'space':[],'homelab':[],'local':[],'signals':[]}
+    selected=select_items(candidates,prefs); top_briefing=selected[:4]; top_urls={i.get('url','') for i in top_briefing if i.get('url')}; groups={'space':[],'homelab':[],'local':[],'signals':[]}
     exploration=select_exploration_items(candidates,prefs,{i.get('url','') for i in selected if i.get('url')},4)
     for i in selected:
         if i.get('url','') in top_urls: continue
@@ -596,7 +919,8 @@ def render()->str:
         elif cat in {'homelab','smart_home','radio'}: groups['homelab'].append(i)
         elif cat=='local': groups['local'].append(i)
         else: groups['signals'].append(i)
-    weather=fetch_weather(); moon=moon_phase(datetime.now(timezone.utc)); pulse=homelab_pulse(candidates)
+    weather=fetch_weather(); moon=moon_phase(datetime.now(timezone.utc)); pulse=homelab_pulse(candidates); hermes=fetch_hermes_usage(30); system=system_snapshot()
+    paperless=paperless_focus_summary()
     now=datetime.now(LOCAL_TZ); counts=Counter(i.get('category','other') for i in candidates)
     if weather['ok']:
         cur=weather['current']; daily=weather.get('daily',{})
@@ -613,31 +937,45 @@ def render()->str:
       metric_card('Sky note','Space feeds live','Moon phase is computed locally','space')])
     pulse_metrics=''.join(metric_card(x['label'],x['value'],x['state'],x['state']) for x in pulse)
     forecast_html=forecast_panel(weather)
+    hero_summary=build_hero_summary(groups['local'],todos,pulse,paperless,hermes)
+    focus_html=build_focus_cards(groups['local'],todos,pulse,paperless)
+    hermes_html=build_hermes_section(hermes)
+    system_html=build_system_section(system)
+    hero_tokens=compact_number(hermes.get('total_tokens',0)) if hermes.get('ok') else '—'
+    primary_html=''.join([
+        section('Top briefing','Compact scan of what is worth your time.',top_briefing,'Nothing worth bothering you with.',compact=True),
+        section('Homelab / Smart Home / Radio','Infrastructure, self-hosting, smart home, and ham radio.',groups['homelab'][:4],'No homelab signal right now.',compact=True),
+        section('Sky / Space','SpaceX, NASA, planetary science, and astronomy.',groups['space'][:4],'No sky signal right now.',compact=True),
+        section('Local','Clarksville / Middle Tennessee items that seem useful.',groups['local'][:3],'No local signal right now.',compact=True),
+        section('New to You','Intentional oddballs outside your usual top lanes, so you can see if any new categories deserve a slot.',exploration[:4],'No worthwhile experimental picks right now.',compact=True),
+        section('Other signals','Everything else that survived filtering.',groups['signals'][:3],'No extra signals.',compact=True),
+    ])
+    secondary_html=''.join([
+        collapsible_section('Focus now','What actually deserves attention before you go wandering through feeds.',focus_html,'focus-now',open_by_default=True),
+        system_html,
+        hermes_html,
+        collapsible_section('Agenda','Calendar, Donetick, and pinned notes.',f"<section class='today'>{list_panel('Calendar',calendar,'Calendar integration not wired yet.')}{donetick_panel(todos,'No open Donetick tasks found; JSON placeholder active.')}{list_panel('Notes',notes,'No notes.')}</section>",'today-overview',open_by_default=False),
+        collapsible_section('Environment','Weather, sky, and outside-the-window context.',f"<section class='metrics metrics-compact'>{weather_metrics}{moon_metrics}</section>",'at-a-glance',open_by_default=False),
+        forecast_html,
+        paperless_review_section(),
+        collapsible_section('Homelab pulse','Safe local checks on NOMAD only.',f"<div class='metrics metrics-compact'>{pulse_metrics}</div>",'homelab-pulse',open_by_default=False),
+    ])
     return f"""<!doctype html><html lang='en'><head><meta charset='utf-8'><meta name='viewport' content='width=device-width,initial-scale=1'><meta http-equiv='refresh' content='1800'><title>Doris Dashboard</title><link rel='stylesheet' href='style.css'></head><body><main>
-<header class='hero'><div><p class='eyebrow'>PC Doris Thatcher</p><h1>Doris Dashboard</h1><p class='sub'>Morning briefing, sky watch, homelab pulse, and local signal — refreshed on NOMAD.</p><div class='hero-actions'><button type='button' class='reset-btn' data-reset-topic-prefs>Reset hidden topics</button><span class='muted'>Hidden topics: <strong data-hidden-topics-count>0</strong></span></div></div><div class='status'><div><strong>{esc(now.strftime('%a %b %-d, %-I:%M %p'))}</strong><span>Last generated Central · auto-refreshes at :05 / :35</span></div><div><strong>{len(candidates)}</strong><span>Candidates</span></div><div><strong>{len(set(i.get('source','') for i in candidates))}</strong><span>Sources</span></div></div></header>
+<header class='hero'><div><p class='eyebrow'>PC Doris Thatcher</p><h1>Doris Dashboard</h1><p class='sub'>{esc(hero_summary)}</p><div class='hero-actions'><button type='button' class='reset-btn' data-reset-topic-prefs>Reset hidden topics</button><span class='muted'>Hidden topics: <strong data-hidden-topics-count>0</strong></span></div></div><div class='status'><div><strong>{esc(now.strftime('%a %b %-d, %-I:%M %p'))}</strong><span>Central time · refreshes every 30m</span></div><div><strong>{len(candidates)}</strong><span>Candidates</span></div><div><strong>{len(set(i.get('source','') for i in candidates))}</strong><span>Sources</span></div><div><strong>{esc(hero_tokens)}</strong><span>30-day Hermes tokens</span></div></div></header>
-{collapsible_section('At a glance','Weather and sky cards.',f"<section class='metrics'>{weather_metrics}{moon_metrics}</section>",'at-a-glance',open_by_default=False)}
+<div class='dashboard-shell'><section class='primary-column'>{primary_html}</section><aside class='secondary-column'>{secondary_html}</aside></div>
-{collapsible_section('Today','Calendar, Donetick, and notes.',f"<section class='today'>{list_panel('Today',calendar,'Calendar integration not wired yet.')}{donetick_panel(todos,'No open Donetick tasks found; JSON placeholder active.')}{list_panel('Notes',notes,'No notes.')}</section>",'today-overview',open_by_default=True)}
+<footer><p>Category mix: {esc(', '.join(f'{k}: {v}' for k,v in counts.most_common(10)))}</p><p>Local-only on NOMAD unless John says otherwise. Weather from Open-Meteo; moon phase calculated locally. Hermes stats come from ~/.hermes/state.db.</p></footer></main>{feedback_bootstrap()}</body></html>"""
-{forecast_html}
-{paperless_review_section()}
-{collapsible_section('Homelab pulse','Safe local checks on NOMAD only.',f"<div class='metrics'>{pulse_metrics}</div>",'homelab-pulse',open_by_default=False)}
-{section('Top briefing','Compact scan of what is worth your time.',top_briefing,'Nothing worth bothering you with.',compact=True)}
-{section('Sky / Space','SpaceX, NASA, planetary science, and astronomy.',groups['space'][:5],'No sky signal right now.',compact=True)}
-{section('Homelab / Smart Home / Radio','Infrastructure, self-hosting, smart home, and ham radio.',groups['homelab'][:4],'No homelab signal right now.',compact=True)}
-{section('Local','Clarksville / Middle Tennessee items that seem useful.',groups['local'][:3],'No local signal right now.',compact=True)}
-{section('New to You','Intentional oddballs outside your usual top lanes, so you can see if any new categories deserve a slot.',exploration[:4],'No worthwhile experimental picks right now.',compact=True)}
-{section('Other signals','Everything else that survived filtering.',groups['signals'][:3],'No extra signals.',compact=True)}
-<footer><p>Category mix: {esc(', '.join(f'{k}: {v}' for k,v in counts.most_common(10)))}</p><p>Local-only on NOMAD unless John says otherwise. Weather from Open-Meteo; moon phase calculated locally.</p></footer></main>{feedback_bootstrap()}</body></html>"""
 
 def main()->int:
     PUBLIC.mkdir(parents=True,exist_ok=True); LOGS.mkdir(parents=True,exist_ok=True); DATA.mkdir(parents=True,exist_ok=True)
     candidates=load_json(DIGEST_ROOT/'data/candidates.json',[]); prefs=load_json(DIGEST_ROOT/'data/preferences.json',{})
-    weather=fetch_weather(); moon=moon_phase(datetime.now(timezone.utc)); pulse=homelab_pulse(candidates); articles=select_items(candidates,prefs,20); todos=inject_dashboard_todos(fetch_donetick_todos())
+    weather=fetch_weather(); moon=moon_phase(datetime.now(timezone.utc)); pulse=homelab_pulse(candidates); articles=select_items(candidates,prefs,20); todos=inject_dashboard_todos(fetch_donetick_todos()); hermes=fetch_hermes_usage(30)
     generated={'generated_at':datetime.now(timezone.utc).isoformat(),'generated_at_local':datetime.now(LOCAL_TZ).isoformat(),'timezone':'America/Chicago','candidate_count':len(candidates),'source_count':len(set(i.get('source','') for i in candidates))}
     (DATA/'dashboard.json').write_text(json.dumps(generated,indent=2)+'\n')
     (DATA/'weather.json').write_text(json.dumps(weather,indent=2)+'\n')
     (DATA/'sky.json').write_text(json.dumps({'moon':moon},indent=2)+'\n')
     (DATA/'homelab.json').write_text(json.dumps(pulse,indent=2)+'\n')
     (DATA/'articles.json').write_text(json.dumps(articles,indent=2)+'\n')
+    (DATA/'hermes_usage.json').write_text(json.dumps(hermes,indent=2)+'\n')
     if todos: (DATA/'todos.json').write_text(json.dumps(todos,indent=2)+'\n')
     (PUBLIC/'index.html').write_text(render(),encoding='utf-8')
     return 0

home/doris-dashboard/public/style.css

@@ -1,13 +1,482 @@
-:root{color-scheme:dark;--bg:#0c111b;--panel:#151b27;--panel2:#202939;--text:#eef4ff;--muted:#9cabbe;--line:#2d394b;--accent:#8fd3ff;--space:#9f8cff;--home:#4fd1a5;--local:#ffd166;--warn:#ff6b6b;--ok:#5ee787}*{box-sizing:border-box}body{margin:0;font-family:Inter,ui-sans-serif,system-ui,-apple-system,BlinkMacSystemFont,"Segoe UI",sans-serif;background:radial-gradient(circle at top left,#213455,#0c111b 42rem);color:var(--text)}a{color:inherit}main{max-width:1220px;margin:0 auto;padding:18px}.hero{display:grid;grid-template-columns:1fr auto;gap:18px;align-items:end;margin-bottom:18px}.eyebrow{color:var(--accent);text-transform:uppercase;letter-spacing:.16em;font-size:.78rem;margin:0 0 8px}h1{font-size:clamp(2.2rem,6vw,4.5rem);line-height:.94;margin:0}.sub{color:var(--muted);font-size:1rem;max-width:760px}.hero-actions{display:flex;gap:12px;align-items:center;margin-top:10px;flex-wrap:wrap}.reset-btn{background:var(--panel2);border:1px solid var(--line);border-radius:999px;color:var(--text);padding:8px 12px;font:inherit;cursor:pointer}.reset-btn:hover{border-color:var(--accent)}.status,.metrics{display:grid;grid-template-columns:repeat(5,minmax(0,1fr));gap:10px}.status div,.metric,.panel,.card{background:rgba(21,27,39,.9);border:1px solid var(--line);border-radius:20px;padding:14px;box-shadow:0 10px 30px rgba(0,0,0,.18);backdrop-filter:blur(6px)}.status strong,.metric strong{display:block;font-size:1.1rem}.status span,.metric span,.metric em,.meta,footer,.empty{color:var(--muted)}.metric{min-height:92px;border-top:4px solid var(--accent)}.metric strong{font-size:1.35rem;margin:6px 0}.metric em{font-style:normal;font-size:.84rem}.metric.weather{border-top-color:var(--local)}.metric.space{border-top-color:var(--space)}.metric.ok{border-top-color:var(--ok)}.metric.warn{border-top-color:var(--warn)}.today{display:grid;grid-template-columns:1fr 1.3fr 1fr;gap:12px;margin:0}.panel h2,section h2{margin:0 0 8px}ul{padding-left:18px;margin:0}.panel ul{list-style:none;padding-left:0;display:grid;gap:8px}.panel li{margin:0}section{margin:18px 0}.grid{display:grid;grid-template-columns:repeat(3,minmax(0,1fr));gap:14px}.card{min-height:245px;display:flex;flex-direction:column;gap:10px;border-top:4px solid var(--accent)}.card.astronomy{border-top-color:var(--space)}.card.homelab,.card.smart_home,.card.radio{border-top-color:var(--home)}.card.local{border-top-color:var(--local)}.card.security{border-top-color:var(--warn)}.meta{display:flex;flex-wrap:wrap;gap:6px;font-size:.74rem}.meta span{padding:3px 8px;border-radius:999px;background:var(--panel2)}h3{font-size:1rem;line-height:1.2;margin:0}h3 a{text-decoration:none}h3 a:hover{text-decoration:underline}p{line-height:1.35}.why{margin-top:auto;color:#c8d5ea;background:rgba(143,211,255,.08);padding:10px;border-radius:12px;font-size:.88rem}footer{border-top:1px solid var(--line);padding:18px 0 26px}.collapsible{margin:16px 0;border:1px solid var(--line);border-radius:20px;background:rgba(21,27,39,.9);box-shadow:0 10px 30px rgba(0,0,0,.18);backdrop-filter:blur(6px);overflow:hidden}.collapsible>summary{list-style:none;cursor:pointer;padding:14px 16px;display:flex;justify-content:space-between;gap:12px;align-items:center}.collapsible>summary::-webkit-details-marker{display:none}.collapsible-heading h2{margin:0}.collapsible-heading p{margin:4px 0 0;color:var(--muted)}.collapsible-body{padding:0 14px 14px}.collapse-indicator{color:var(--muted);transition:transform .15s ease}.collapsible[open] .collapse-indicator{transform:rotate(180deg)}@media(max-width:1050px){.status,.metrics,.today,.grid,.forecast-grid,.news-list,.task-grid{grid-template-columns:1fr 1fr}.hero{grid-template-columns:1fr}}@media(max-width:650px){main{padding:14px}.status,.metrics,.today,.grid,.forecast-grid,.news-list,.task-grid{grid-template-columns:1fr}.card{min-height:auto}}
+:root{
+  color-scheme:dark;
+  --bg:#08090a;
+  --bg-accent:#121826;
+  --panel:#11151d;
+  --panel-soft:#171d27;
+  --panel-raise:#1d2430;
+  --text:#f3f6fb;
+  --muted:#94a3b8;
+  --line:rgba(255,255,255,.08);
+  --line-soft:rgba(255,255,255,.05);
+  --accent:#7c8cff;
+  --accent-soft:rgba(124,140,255,.16);
+  --space:#9f8cff;
+  --home:#4fd1a5;
+  --local:#ffd166;
+  --warn:#ff7b72;
+  --ok:#5ee787;
+  --shadow:0 14px 34px rgba(0,0,0,.26);
+}
 
-/* Compact article layout: keep the number of stories, reduce scroll debt. */
+*{box-sizing:border-box}
-.news-list{display:grid;grid-template-columns:repeat(2,minmax(0,1fr));gap:10px}.compact-card{background:rgba(21,27,39,.9);border:1px solid var(--line);border-left:5px solid var(--accent);border-radius:16px;padding:10px 12px;box-shadow:0 6px 18px rgba(0,0,0,.14)}.compact-card.astronomy{border-left-color:var(--space)}.compact-card.homelab,.compact-card.smart_home,.compact-card.radio{border-left-color:var(--home)}.compact-card.local{border-left-color:var(--local)}.compact-card.security{border-left-color:var(--warn)}.compact-card .meta{margin-bottom:4px}.compact-card h3{font-size:.98rem;line-height:1.18;margin:0 0 3px}.compact-card p{margin:0;color:#c8d5ea;font-size:.88rem;line-height:1.25;display:-webkit-box;-webkit-line-clamp:3;-webkit-box-orient:vertical;overflow:hidden}.compact-card .meta span{padding:2px 7px}.compact-card:nth-child(n+7){display:none}section header{display:flex;justify-content:space-between;gap:12px;align-items:end}section header p{margin:0}.feedback{display:flex;gap:6px;justify-content:flex-end;margin-top:8px}.feedback-btn{background:transparent;border:1px solid var(--line);border-radius:999px;color:var(--text);padding:2px 8px;font:inherit;cursor:pointer}.feedback-btn:hover{border-color:var(--accent);transform:translateY(-1px)}.topic-liked{box-shadow:0 0 0 1px rgba(94,231,135,.45),0 6px 18px rgba(0,0,0,.14)}@media(max-width:650px){.compact-card p{display:-webkit-box;-webkit-line-clamp:2;-webkit-box-orient:vertical;overflow:hidden}section header{display:block}}
+html{scroll-behavior:smooth}
-.feedback-btn[aria-pressed='true']{font-weight:700}.feedback-btn.active-up{border-color:rgba(94,231,135,.65);background:rgba(94,231,135,.12);color:#c8ffd9}.feedback-btn.active-down{border-color:rgba(255,107,107,.6);background:rgba(255,107,107,.12);color:#ffd9d9}.dashboard-toast{position:fixed;right:16px;bottom:16px;z-index:1000;max-width:min(420px,calc(100vw - 24px));background:rgba(12,17,27,.96);border:1px solid var(--line);border-left:4px solid var(--accent);border-radius:16px;padding:12px 14px;color:var(--text);box-shadow:0 14px 34px rgba(0,0,0,.35)}.dashboard-toast[hidden]{display:none}.toast-action{margin-left:10px;background:transparent;border:0;color:var(--accent);font:inherit;font-weight:700;cursor:pointer;padding:0}
+body{
-.muted{color:var(--muted);font-size:.86rem;margin:0 .25rem}.panel-item{background:rgba(32,41,57,.55);border:1px solid var(--line);border-radius:14px;padding:10px 12px}.panel-item-top{display:flex;justify-content:space-between;gap:10px;align-items:baseline}.panel-item-title{margin-top:2px;font-weight:600;color:#dbe7f7}.panel-item-body{margin-top:4px;color:#c8d5ea;font-size:.9rem;line-height:1.25;display:-webkit-box;-webkit-line-clamp:2;-webkit-box-orient:vertical;overflow:hidden}.panel .empty{padding:12px 0}.donetick-panel{padding:12px 14px}.task-grid{display:grid;grid-template-columns:repeat(2,minmax(0,1fr));gap:10px}.task-card{background:rgba(32,41,57,.58);border:1px solid var(--line);border-radius:16px;padding:10px 12px;min-width:0}.task-top{display:flex;justify-content:space-between;gap:8px;align-items:center;margin-bottom:4px}.task-due{font-size:.8rem;font-weight:800;letter-spacing:.02em;text-transform:none;color:#dbe7f7}.task-assignee{font-size:.72rem;padding:2px 8px;border-radius:999px;background:var(--panel2);color:var(--muted)}.task-name{font-size:1rem;font-weight:700;line-height:1.08;color:var(--text);margin-bottom:6px}.task-subtitle{font-size:.78rem;line-height:1.16;color:var(--muted)}.task-chips{display:flex;flex-wrap:wrap;gap:5px;margin-bottom:5px}.task-chip{display:inline-block;padding:3px 8px;border-radius:999px;color:#fff;font-size:.72rem;font-weight:700;line-height:1.05}.task-card.overdue{border-color:rgba(255,107,107,.55)}.task-card.today{border-color:rgba(255,209,102,.5)}.task-card.tomorrow{border-color:rgba(142,202,230,.45)}.task-card.nodue{opacity:.92}
+  margin:0;
-.donetick-panel .task-grid{gap:8px}.task-card{padding:9px 10px;border-radius:14px}.task-top{gap:6px;margin-bottom:3px}.task-state-dot{width:9px;height:9px;border-radius:999px;background:var(--muted);flex:0 0 auto;box-shadow:0 0 0 2px rgba(255,255,255,.03)}.task-card.overdue .task-state-dot{background:var(--warn)}.task-card.today .task-state-dot{background:var(--local)}.task-card.tomorrow .task-state-dot,.task-card.upcoming .task-state-dot{background:#8ecae6}.task-card.nodue .task-state-dot{background:#6c7a90}.task-due{font-size:.76rem;line-height:1.1}.task-name{font-size:.95rem;line-height:1.15;margin-bottom:4px;display:-webkit-box;-webkit-line-clamp:2;-webkit-box-orient:vertical;overflow:hidden}.task-subtitle{font-size:.74rem;line-height:1.1;opacity:.9;display:-webkit-box;-webkit-line-clamp:1;-webkit-box-orient:vertical;overflow:hidden}.task-chips{gap:4px;margin-bottom:4px}.task-chip{padding:2px 7px;font-size:.68rem}
+  font-family:Inter,ui-sans-serif,system-ui,-apple-system,BlinkMacSystemFont,"Segoe UI",sans-serif;
+  background:
+    radial-gradient(circle at top left,rgba(124,140,255,.16),transparent 28rem),
+    radial-gradient(circle at top right,rgba(79,209,165,.08),transparent 22rem),
+    var(--bg);
+  color:var(--text);
+}
 
-/* Forecast */
+a{color:inherit}
-.forecast-panel{padding:14px 16px}.forecast-grid{display:grid;grid-template-columns:repeat(3,minmax(0,1fr));gap:12px}.forecast-card{background:rgba(32,41,57,.65);border:1px solid var(--line);border-radius:16px;padding:12px 14px;display:flex;flex-direction:column;gap:6px}.forecast-day{color:var(--accent);font-size:.82rem;text-transform:uppercase;letter-spacing:.08em}.forecast-card strong{font-size:1rem}.forecast-card em{font-style:normal;color:var(--muted);font-size:.88rem}
+main{max-width:1380px;margin:0 auto;padding:18px}
 
-/* Paperless review queue */
+.hero,
-.badge{display:inline-block;margin-left:.35rem;padding:2px 8px;border-radius:999px;background:var(--panel2);color:var(--accent);font-size:.85rem}.paperless-grid{display:grid;grid-template-columns:1fr;gap:10px}.paperless-card{background:rgba(21,27,39,.9);border:1px solid var(--line);border-left:5px solid var(--accent);border-radius:16px;padding:10px 12px}.paperless-card.warn{border-left-color:var(--warn)}.paperless-card.info{border-left-color:var(--local)}.paperless-card.ok{border-left-color:var(--ok)}.paperless-card.needs-attention{box-shadow:0 0 0 1px rgba(255,209,102,.25),0 10px 24px rgba(0,0,0,.22)}.paperless-head{display:flex;justify-content:space-between;gap:10px;align-items:flex-start}.paperless-card h4{margin:.35rem 0;font-size:.96rem;line-height:1.2}.paperless-card .summary{margin:.2rem 0;color:#d7e3f5;font-size:.9rem;line-height:1.25;display:-webkit-box;-webkit-line-clamp:2;-webkit-box-orient:vertical;overflow:hidden}.paperless-meta,.paperless-reason{margin:.2rem 0;color:var(--muted);font-size:.8rem;line-height:1.2}.paperless-card .link{display:inline-block;color:var(--accent);text-decoration:none}.paperless-card .link:hover{text-decoration:underline}.paperless-actions{display:flex;gap:6px;flex-wrap:wrap;align-items:center;justify-content:flex-end}.paperless-btn{background:var(--panel2);border:1px solid var(--line);border-radius:999px;color:var(--text);padding:5px 9px;font:inherit;cursor:pointer}.paperless-btn.good{border-color:rgba(94,231,135,.45);color:#c8ffd9}.paperless-btn:hover{border-color:var(--accent)}.paperless-acked,.paperless-more{margin-top:12px;border-top:1px solid var(--line);padding-top:12px}.paperless-acked summary,.paperless-more summary{cursor:pointer;color:var(--muted)}.urgency{font-weight:700}.urgency.warn{color:var(--warn)}.urgency.info{color:var(--local)}.urgency.ok{color:var(--ok)}@media(max-width:720px){.paperless-head{display:block}.paperless-actions{justify-content:flex-start;margin-top:8px}}
+.status div,
+.metric,
+.panel,
+.card,
+.compact-card,
+.focus-card,
+.summary-card,
+.paperless-card,
+.forecast-card,
+.panel-item,
+.task-card,
+.collapsible{
+  background:rgba(17,21,29,.92);
+  border:1px solid var(--line);
+  box-shadow:var(--shadow);
+}
+
+.hero{
+  display:grid;
+  grid-template-columns:minmax(0,1.25fr) 430px;
+  gap:14px;
+  align-items:stretch;
+  border-radius:24px;
+  padding:16px 18px;
+  margin-bottom:16px;
+}
+
+.eyebrow{
+  margin:0 0 6px;
+  color:#b7c0ff;
+  text-transform:uppercase;
+  letter-spacing:.18em;
+  font-size:.72rem;
+}
+
+h1{
+  margin:0;
+  font-size:clamp(2rem,4.6vw,3.5rem);
+  line-height:.94;
+  letter-spacing:-.04em;
+}
+
+.sub{
+  margin:10px 0 0;
+  color:var(--muted);
+  font-size:.97rem;
+  line-height:1.45;
+  max-width:70ch;
+}
+
+.hero-actions{
+  display:flex;
+  gap:10px;
+  align-items:center;
+  flex-wrap:wrap;
+  margin-top:12px;
+}
+
+.reset-btn,
+.feedback-btn,
+.paperless-btn{
+  appearance:none;
+  background:rgba(255,255,255,.03);
+  border:1px solid var(--line);
+  border-radius:999px;
+  color:var(--text);
+  cursor:pointer;
+  font:inherit;
+}
+
+.reset-btn{padding:7px 11px}
+.reset-btn:hover,
+.feedback-btn:hover,
+.paperless-btn:hover{border-color:rgba(124,140,255,.55)}
+
+.status,
+.metrics{
+  display:grid;
+  grid-template-columns:repeat(4,minmax(0,1fr));
+  gap:10px;
+}
+
+.metrics.metrics-compact{grid-template-columns:repeat(2,minmax(0,1fr))}
+
+.status div,
+.metric{
+  border-radius:18px;
+  padding:11px 12px;
+  min-width:0;
+}
+
+.status strong,
+.metric strong{display:block}
+.status strong{font-size:1rem}
+.metric{
+  min-height:78px;
+  border-top:3px solid var(--accent);
+}
+.metric strong{
+  margin:6px 0 4px;
+  font-size:1.12rem;
+  line-height:1.08;
+}
+.metric span,
+.metric em,
+.status span,
+.meta,
+.empty,
+footer,
+.muted{
+  color:var(--muted);
+}
+.metric em{font-style:normal;font-size:.78rem}
+.metric.weather{border-top-color:var(--local)}
+.metric.space{border-top-color:var(--space)}
+.metric.ok{border-top-color:var(--ok)}
+.metric.warn{border-top-color:var(--warn)}
+
+.dashboard-shell{
+  display:grid;
+  grid-template-columns:minmax(0,1.5fr) minmax(320px,.9fr);
+  gap:14px;
+  align-items:start;
+}
+
+.primary-column,
+.secondary-column{
+  display:flex;
+  flex-direction:column;
+  gap:12px;
+  min-width:0;
+}
+
+.collapsible{
+  border-radius:20px;
+  overflow:hidden;
+}
+
+.collapsible > summary{
+  list-style:none;
+  cursor:pointer;
+  display:flex;
+  align-items:flex-start;
+  justify-content:space-between;
+  gap:12px;
+  padding:12px 14px;
+}
+
+.collapsible > summary::-webkit-details-marker{display:none}
+.collapsible-heading{min-width:0}
+.collapsible-heading h2,
+.panel h2,
+section h2{
+  margin:0;
+  font-size:1.02rem;
+  line-height:1.15;
+  letter-spacing:-.02em;
+}
+
+.collapsible-heading p,
+section header p{
+  margin:4px 0 0;
+  color:var(--muted);
+  font-size:.83rem;
+  line-height:1.32;
+}
+
+.collapse-indicator{
+  color:var(--muted);
+  transition:transform .16s ease;
+  transform-origin:center;
+}
+.details[open] .collapse-indicator,
+.collapsible[open] .collapse-indicator{transform:rotate(180deg)}
+.collapsible-body{padding:0 14px 14px}
+
+.focus-grid,
+.summary-grid,
+.today,
+.news-list,
+.grid,
+.task-grid,
+.forecast-grid{
+  display:grid;
+  gap:10px;
+}
+
+.focus-grid,
+.summary-grid,
+.forecast-grid{grid-template-columns:repeat(2,minmax(0,1fr))}
+.today{grid-template-columns:1fr}
+.news-list{grid-template-columns:repeat(2,minmax(0,1fr))}
+.grid{grid-template-columns:repeat(2,minmax(0,1fr))}
+.task-grid{grid-template-columns:1fr}
+
+.focus-card,
+.summary-card,
+.panel,
+.compact-card,
+.paperless-card,
+.forecast-card,
+.panel-item,
+.task-card,
+.card{
+  border-radius:16px;
+}
+
+.focus-card,
+.summary-card{padding:12px}
+.focus-card{border-top:3px solid var(--accent)}
+.focus-card.local{border-top-color:var(--local)}
+.focus-card.ok{border-top-color:var(--ok)}
+.focus-card.warn{border-top-color:var(--warn)}
+.focus-kicker{
+  display:inline-block;
+  margin-bottom:8px;
+  font-size:.72rem;
+  letter-spacing:.12em;
+  text-transform:uppercase;
+  color:#b7c0ff;
+}
+.focus-card h3,
+.summary-card h3{margin:0 0 5px;font-size:.93rem}
+.focus-card strong{display:block;font-size:1rem;line-height:1.2;margin-bottom:5px}
+.focus-card p,
+.summary-card p,
+.summary-card li{margin:0;color:#d5deeb;font-size:.88rem;line-height:1.34}
+.summary-card ul{margin:0;padding-left:18px}
+.summary-card li+li{margin-top:6px}
+.hermes-metrics{margin-bottom:10px}
+
+.panel{padding:12px}
+.panel ul{list-style:none;padding-left:0;margin:0;display:grid;gap:8px}
+.panel-item{
+  padding:9px 10px;
+  background:rgba(255,255,255,.03);
+  border-color:var(--line-soft);
+}
+.panel-item-top{display:flex;justify-content:space-between;gap:8px;align-items:baseline}
+.panel-item-top strong{font-size:.82rem}
+.panel-item-title{margin-top:3px;font-weight:600;color:#dce5f6;font-size:.93rem;line-height:1.2}
+.panel-item-body{
+  margin-top:4px;
+  color:#c7d1e2;
+  font-size:.84rem;
+  line-height:1.28;
+  display:-webkit-box;
+  -webkit-line-clamp:2;
+  -webkit-box-orient:vertical;
+  overflow:hidden;
+}
+.panel .empty{padding:6px 2px}
+
+.donetick-panel{padding:12px}
+.task-card{
+  padding:9px 10px;
+  background:rgba(255,255,255,.035);
+  border-color:var(--line-soft);
+  min-width:0;
+}
+.task-top{display:flex;justify-content:space-between;gap:6px;align-items:center;margin-bottom:3px}
+.task-state-dot{width:8px;height:8px;border-radius:999px;background:var(--muted);flex:0 0 auto;box-shadow:0 0 0 2px rgba(255,255,255,.03)}
+.task-card.overdue .task-state-dot{background:var(--warn)}
+.task-card.today .task-state-dot{background:var(--local)}
+.task-card.tomorrow .task-state-dot,
+.task-card.upcoming .task-state-dot{background:#8ecae6}
+.task-card.nodue .task-state-dot{background:#6c7a90}
+.task-due{font-size:.74rem;font-weight:700;line-height:1.1;color:#dde6f6}
+.task-assignee{font-size:.68rem;padding:2px 7px;border-radius:999px;background:var(--panel-raise);color:var(--muted)}
+.task-name{
+  font-size:.93rem;
+  font-weight:700;
+  line-height:1.14;
+  color:var(--text);
+  margin-bottom:4px;
+  display:-webkit-box;
+  -webkit-line-clamp:2;
+  -webkit-box-orient:vertical;
+  overflow:hidden;
+}
+.task-subtitle{
+  font-size:.73rem;
+  line-height:1.14;
+  color:var(--muted);
+  display:-webkit-box;
+  -webkit-line-clamp:1;
+  -webkit-box-orient:vertical;
+  overflow:hidden;
+}
+.task-chips{display:flex;flex-wrap:wrap;gap:4px;margin-bottom:4px}
+.task-chip{display:inline-block;padding:2px 7px;border-radius:999px;color:#fff;font-size:.67rem;font-weight:700;line-height:1.05}
+.task-card.overdue{border-color:rgba(255,123,114,.45)}
+.task-card.today{border-color:rgba(255,209,102,.42)}
+.task-card.tomorrow{border-color:rgba(142,202,230,.35)}
+
+.compact-card,
+.card{
+  padding:10px 11px;
+  border-left:3px solid var(--accent);
+}
+.compact-card.astronomy,.card.astronomy{border-left-color:var(--space)}
+.compact-card.homelab,.compact-card.smart_home,.compact-card.radio,
+.card.homelab,.card.smart_home,.card.radio{border-left-color:var(--home)}
+.compact-card.local,.card.local{border-left-color:var(--local)}
+.compact-card.security,.card.security{border-left-color:var(--warn)}
+.compact-card .meta,
+.card .meta{
+  margin-bottom:5px;
+  display:flex;
+  flex-wrap:wrap;
+  gap:6px;
+  font-size:.72rem;
+}
+.compact-card .meta span,
+.card .meta span{
+  display:inline-flex;
+  align-items:center;
+  padding:2px 7px;
+  border-radius:999px;
+  background:rgba(255,255,255,.035);
+  border:1px solid var(--line-soft);
+}
+.compact-card h3,
+.card h3{
+  margin:0 0 4px;
+  font-size:.94rem;
+  line-height:1.2;
+}
+.compact-card p,
+.card p{
+  margin:0;
+  color:#c7d1e2;
+  font-size:.83rem;
+  line-height:1.28;
+  display:-webkit-box;
+  -webkit-line-clamp:2;
+  -webkit-box-orient:vertical;
+  overflow:hidden;
+}
+.compact-card:nth-child(n+7){display:none}
+.feedback{display:flex;gap:6px;justify-content:flex-end;margin-top:7px}
+.feedback-btn{padding:2px 8px;background:transparent}
+.feedback-btn[aria-pressed='true']{font-weight:700}
+.feedback-btn.active-up{border-color:rgba(94,231,135,.65);background:rgba(94,231,135,.12);color:#c8ffd9}
+.feedback-btn.active-down{border-color:rgba(255,107,107,.6);background:rgba(255,107,107,.12);color:#ffd9d9}
+.topic-liked{box-shadow:0 0 0 1px rgba(94,231,135,.45),var(--shadow)}
+
+.forecast-panel{padding:0}
+.forecast-card{padding:11px 12px;background:rgba(255,255,255,.035)}
+.forecast-day{color:#b7c0ff;font-size:.75rem;text-transform:uppercase;letter-spacing:.08em}
+.forecast-card strong{font-size:.95rem}
+.forecast-card em{font-style:normal;color:var(--muted);font-size:.82rem}
+
+.badge{
+  display:inline-block;
+  margin-left:.35rem;
+  padding:2px 7px;
+  border-radius:999px;
+  background:var(--panel-raise);
+  color:#b7c0ff;
+  font-size:.78rem;
+}
+.paperless-grid{display:grid;grid-template-columns:1fr;gap:8px}
+.paperless-card{padding:10px 11px;border-left:3px solid var(--accent)}
+.paperless-card.warn{border-left-color:var(--warn)}
+.paperless-card.info{border-left-color:var(--local)}
+.paperless-card.ok{border-left-color:var(--ok)}
+.paperless-card.needs-attention{box-shadow:0 0 0 1px rgba(255,209,102,.25),var(--shadow)}
+.paperless-head{display:flex;justify-content:space-between;gap:10px;align-items:flex-start}
+.paperless-card h4{margin:.3rem 0;font-size:.92rem;line-height:1.18}
+.paperless-card .summary{
+  margin:.2rem 0;
+  color:#d7e3f5;
+  font-size:.84rem;
+  line-height:1.26;
+  display:-webkit-box;
+  -webkit-line-clamp:2;
+  -webkit-box-orient:vertical;
+  overflow:hidden;
+}
+.paperless-meta,
+.paperless-reason{margin:.2rem 0;color:var(--muted);font-size:.76rem;line-height:1.18}
+.paperless-card .link{display:inline-block;color:#b7c0ff;text-decoration:none}
+.paperless-card .link:hover{text-decoration:underline}
+.paperless-actions{display:flex;gap:6px;flex-wrap:wrap;align-items:center;justify-content:flex-end}
+.paperless-btn{padding:5px 8px}
+.paperless-btn.good{border-color:rgba(94,231,135,.45);color:#c8ffd9}
+.paperless-acked,
+.paperless-more{margin-top:10px;border-top:1px solid var(--line-soft);padding-top:10px}
+.paperless-acked summary,
+.paperless-more summary{cursor:pointer;color:var(--muted);font-size:.84rem}
+.urgency{font-weight:700}
+.urgency.warn{color:var(--warn)}
+.urgency.info{color:var(--local)}
+.urgency.ok{color:var(--ok)}
+
+section header{display:flex;justify-content:space-between;gap:12px;align-items:end}
+section header p{margin:0}
+ul{margin:0}
+
+footer{
+  margin:14px 0 6px;
+  padding:0 2px;
+  font-size:.8rem;
+  line-height:1.4;
+}
+footer p{margin:.25rem 0}
+
+.dashboard-toast{
+  position:fixed;
+  right:16px;
+  bottom:16px;
+  z-index:1000;
+  max-width:min(420px,calc(100vw - 24px));
+  background:rgba(8,9,10,.97);
+  border:1px solid var(--line);
+  border-left:3px solid var(--accent);
+  border-radius:14px;
+  padding:12px 14px;
+  color:var(--text);
+  box-shadow:0 16px 40px rgba(0,0,0,.36);
+}
+.dashboard-toast[hidden]{display:none}
+.toast-action{margin-left:10px;background:transparent;border:0;color:#b7c0ff;font:inherit;font-weight:700;cursor:pointer;padding:0}
+
+@media (max-width:1180px){
+  .dashboard-shell{grid-template-columns:1fr}
+  .secondary-column{order:-1}
+  .focus-grid,.summary-grid,.forecast-grid,.metrics.metrics-compact{grid-template-columns:repeat(2,minmax(0,1fr))}
+}
+
+@media (max-width:920px){
+  .hero{grid-template-columns:1fr}
+  .status,.metrics,.metrics.metrics-compact,.focus-grid,.summary-grid,.news-list,.grid,.forecast-grid{grid-template-columns:1fr}
+}
+
+@media (max-width:720px){
+  main{padding:12px}
+  .hero{padding:14px}
+  .collapsible > summary{padding:11px 12px}
+  .collapsible-body{padding:0 12px 12px}
+  .paperless-head{display:block}
+  .paperless-actions{justify-content:flex-start;margin-top:8px}
+  section header{display:block}
+}

identity/.env.example

@@ -0,0 +1,10 @@
+TZ=America/Chicago
+
+AUTHELIA_HOST=auth.paccoco.com
+AUTHELIA_STORAGE_POSTGRES_DB=authelia
+AUTHELIA_STORAGE_POSTGRES_USER=authelia
+AUTHELIA_STORAGE_POSTGRES_PASSWORD=CHANGE_ME
+AUTHELIA_STORAGE_ENCRYPTION_KEY=CHANGE_ME
+AUTHELIA_SESSION_SECRET=CHANGE_ME
+AUTHELIA_SESSION_REDIS_PASSWORD=CHANGE_ME
+AUTHELIA_IDENTITY_VALIDATION_RESET_PASSWORD_JWT_SECRET=CHANGE_ME

identity/README.md

@@ -0,0 +1,65 @@
+# Identity / SSO
+
+This stack introduces Authelia as the homelab identity plane.
+
+Current design choices:
+
+- Authelia only, no LDAP yet
+- shared Postgres for durable storage
+- shared Redis for session storage
+- file-backed users database for initial bootstrap
+- Pangolin/Newt exposure at auth.paccoco.com
+
+Why this shape:
+
+- it gets a real SSO control plane online without dragging in extra moving parts
+- it matches the homelab preference for shared DB infrastructure over SQLite
+- file users are fine at this size and can be migrated to LDAP later if needed
+
+Files:
+
+- docker-compose.yaml
+- .env.example
+- authelia/configuration.yml.example
+- authelia/users_database.example.yml
+
+Bootstrap:
+
+1. Copy this directory to PD at /mnt/docker-ssd/docker/compose/identity
+2. Copy .env.example to .env and replace every CHANGE_ME
+3. Copy authelia/configuration.yml.example to authelia/configuration.yml
+4. Render concrete secrets into `authelia/configuration.yml`; do not leave template placeholders or `${...}` references in the live file.
+5. Copy authelia/users_database.example.yml to authelia/users_database.yml
+6. Create an Authelia password hash:
+
+   docker run --rm authelia/authelia:latest authelia crypto hash generate argon2 --password 'CHANGE_ME'
+
+7. Replace the sample hash in users_database.yml
+8. Create the authelia database/user in shared Postgres
+9. Set `AUTHELIA_SESSION_REDIS_PASSWORD` from the live shared Redis stack on PD.
+10. Create a Pangolin resource:
+   Domain: auth.paccoco.com
+   Scheme: http
+   Host: authelia
+   Port: 9091
+11. Validate and start:
+
+   cd /mnt/docker-ssd/docker/compose/identity
+   docker compose --env-file .env config
+   docker compose --env-file .env up -d
+   docker exec authelia authelia config validate --config /config/configuration.yml
+
+This repo stage gives you the identity control plane itself.
+
+OIDC client rollout is intentionally a second pass because it needs:
+
+- a real signing-key strategy
+- per-app client IDs and secrets
+- app-specific callback URLs verified against live domains
+
+First client recommendation:
+
+- Grafana via Generic OAuth against Authelia OIDC
+- Keep Grafana local-admin login enabled for the first cutover so recovery is easy if the public auth route or claims mapping is wrong
+
+Do not try to wire every app at once. Get Authelia healthy first, then cut services over one at a time.

identity/authelia/configuration.yml.example

@@ -0,0 +1,103 @@
+# Render concrete values into configuration.yml on PD.
+# Do not rely on runtime env interpolation for this file.
+theme: auto
+default_2fa_method: totp
+
+server:
+  address: tcp://0.0.0.0:9091
+
+log:
+  level: info
+
+totp:
+  issuer: paccoco.com
+
+authentication_backend:
+  file:
+    path: /config/users_database.yml
+    watch: false
+    password:
+      algorithm: argon2
+
+access_control:
+  default_policy: deny
+  rules:
+    - domain:
+        - '*.paccoco.com'
+      policy: one_factor
+
+session:
+  secret: 'CHANGE_ME_SESSION_SECRET'
+  name: authelia_session
+  same_site: lax
+  expiration: 8h
+  inactivity: 1h
+  remember_me: 30d
+  redis:
+    host: shared-redis
+    port: 6379
+    password: 'CHANGE_ME_REDIS_PASSWORD'
+  cookies:
+    - domain: paccoco.com
+      authelia_url: https://auth.paccoco.com
+      default_redirection_url: https://grafana.paccoco.com
+
+storage:
+  encryption_key: 'CHANGE_ME_STORAGE_ENCRYPTION_KEY'
+  postgres:
+    address: tcp://shared-postgres:5432
+    database: 'authelia'
+    schema: public
+    username: 'authelia'
+    password: 'CHANGE_ME_POSTGRES_PASSWORD'
+
+identity_validation:
+  reset_password:
+    jwt_secret: 'CHANGE_ME_RESET_PASSWORD_JWT_SECRET'
+
+notifier:
+  filesystem:
+    filename: /config/data/notification.txt
+
+identity_providers:
+  oidc:
+    hmac_secret: 'CHANGE_ME_OIDC_HMAC_SECRET'
+    jwks:
+      - key_id: 'grafana'
+        algorithm: 'RS256'
+        use: 'sig'
+        key: |
+          -----BEGIN PRIVATE KEY-----
+          CHANGE_ME_OIDC_PRIVATE_KEY
+          -----END PRIVATE KEY-----
+    claims_policies:
+      grafana:
+        id_token:
+          - email
+          - email_verified
+          - groups
+          - preferred_username
+          - name
+    clients:
+      - client_id: 'grafana'
+        client_name: 'Grafana'
+        client_secret: 'CHANGE_ME_GRAFANA_CLIENT_SECRET'
+        public: false
+        authorization_policy: 'one_factor'
+        require_pkce: true
+        pkce_challenge_method: 'S256'
+        redirect_uris:
+          - 'https://grafana.paccoco.com/login/generic_oauth'
+        scopes:
+          - openid
+          - profile
+          - email
+          - groups
+        response_types:
+          - code
+        grant_types:
+          - authorization_code
+        access_token_signed_response_alg: 'none'
+        userinfo_signed_response_alg: 'none'
+        token_endpoint_auth_method: 'client_secret_basic'
+        claims_policy: 'grafana'

identity/authelia/users_database.example.yml

@@ -0,0 +1,8 @@
+users:
+  john:
+    disabled: false
+    displayname: John
+    email: john@paccoco.com
+    password: "$argon2id$v=19$m=65536,t=3,p=4$REPLACE$ME"
+    groups:
+      - admins

identity/docker-compose.yaml

@@ -0,0 +1,30 @@
+name: identity
+
+services:
+  authelia:
+    image: authelia/authelia:latest
+    container_name: authelia
+    restart: unless-stopped
+    ports:
+      - "9091:9091"
+    environment:
+      TZ: ${TZ}
+    volumes:
+      - /mnt/docker-ssd/docker/compose/identity/authelia/configuration.yml:/config/configuration.yml:ro
+      - /mnt/docker-ssd/docker/compose/identity/authelia/users_database.yml:/config/users_database.yml:ro
+      - /mnt/tank/docker/appdata/authelia:/config/data
+    healthcheck:
+      test: ["CMD-SHELL", "wget -qO- http://127.0.0.1:9091/api/health >/dev/null 2>&1 || exit 1"]
+      interval: 30s
+      timeout: 10s
+      retries: 5
+      start_period: 45s
+    networks:
+      - ix-databases_shared-databases
+      - pangolin
+
+networks:
+  ix-databases_shared-databases:
+    external: true
+  pangolin:
+    external: true

infrastructure/hawser-nomad/.env.example

@@ -0,0 +1,16 @@
+TZ=America/Chicago
+
+# Hawser listener for Dockhand Standard mode
+HAWSER_PORT=2376
+HAWSER_BIND_ADDRESS=0.0.0.0
+HAWSER_TOKEN=CHANGE_ME
+HAWSER_AGENT_NAME=nomad
+HAWSER_LOG_LEVEL=info
+
+# Keep the stack path identical inside and outside the container so Dockhand
+# can deploy stacks that use relative bind mounts.
+STACKS_DIR=/opt/hawser-nomad/stacks
+
+# Set to any non-empty value to skip disk-usage collection if NOMAD's mounts
+# make df collection too noisy or slow.
+SKIP_DF_COLLECTION=

infrastructure/hawser-nomad/README.md

@@ -0,0 +1,21 @@
+# NOMAD Hawser Agent
+
+- **Repo stack path:** `/home/fizzlepoof/repos/truenas-stacks/infrastructure/hawser-nomad`
+- **Suggested live stack path:** `/opt/hawser-nomad`
+- **Dockhand environment:** existing `NOMAD` Hawser Standard environment on PD
+- **Why Standard mode:** NOMAD is on the LAN with a stable IP, so Dockhand can connect directly without the extra Edge-mode WebSocket plumbing
+
+Bring-up:
+
+```bash
+cd /opt/hawser-nomad
+cp .env.example .env
+bash bin/prepare_nomad.sh --up
+```
+
+Notes:
+
+- Use the Hawser token already configured in Dockhand's `NOMAD` environment.
+- Hawser listens on port `2376` by default, matching the existing Dockhand environment entry.
+- `STACKS_DIR` is bind-mounted to the same host path inside the container so Dockhand can deploy stacks that use relative bind mounts.
+- This is a standalone NOMAD deployment under `/opt`, not part of the core Project N.O.M.A.D. managed stack.

infrastructure/hawser-nomad/bin/prepare_nomad.sh

@@ -0,0 +1,25 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+STACK_DIR="$(cd "$SCRIPT_DIR/.." && pwd)"
+ENV_FILE="${ENV_FILE:-$STACK_DIR/.env}"
+DO_UP="${1:-}"
+
+if [[ ! -f "$ENV_FILE" ]]; then
+  echo "Missing $ENV_FILE - copy .env.example to .env and fill in real values first." >&2
+  exit 1
+fi
+
+set -a
+# shellcheck disable=SC1090
+source "$ENV_FILE"
+set +a
+
+mkdir -p "$STACKS_DIR"
+docker compose --env-file "$ENV_FILE" -f "$STACK_DIR/docker-compose.yaml" config >/dev/null
+
+if [[ "$DO_UP" == "--up" ]]; then
+  docker compose --env-file "$ENV_FILE" -f "$STACK_DIR/docker-compose.yaml" up -d
+  docker compose --env-file "$ENV_FILE" -f "$STACK_DIR/docker-compose.yaml" ps
+fi

infrastructure/hawser-nomad/docker-compose.yaml

@@ -0,0 +1,19 @@
+services:
+  hawser:
+    image: ghcr.io/finsys/hawser:latest
+    container_name: hawser-nomad
+    restart: unless-stopped
+    ports:
+      - "${HAWSER_BIND_ADDRESS:-0.0.0.0}:${HAWSER_PORT:-2376}:2376"
+    environment:
+      TZ: ${TZ}
+      PORT: "2376"
+      BIND_ADDRESS: ${HAWSER_BIND_ADDRESS}
+      TOKEN: ${HAWSER_TOKEN}
+      AGENT_NAME: ${HAWSER_AGENT_NAME}
+      LOG_LEVEL: ${HAWSER_LOG_LEVEL}
+      STACKS_DIR: ${STACKS_DIR}
+      SKIP_DF_COLLECTION: ${SKIP_DF_COLLECTION}
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock
+      - ${STACKS_DIR}:${STACKS_DIR}

monitoring/.env.example

@@ -4,3 +4,21 @@ TZ=America/New_York
 GF_ADMIN_USER=admin
 GF_ADMIN_PASS=CHANGE_ME
 GF_HOST=grafana.paccoco.com
+LOKI_HOST=loki.paccoco.com
+AUTHELIA_HOST=auth.paccoco.com
+GF_AUTH_GENERIC_OAUTH_ENABLED=true
+GF_AUTH_GENERIC_OAUTH_NAME=Authelia
+GF_AUTH_GENERIC_OAUTH_ICON=signin
+GF_AUTH_GENERIC_OAUTH_CLIENT_ID=grafana
+GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET=CHANGE_ME
+GF_AUTH_GENERIC_OAUTH_SCOPES=openid profile email groups
+GF_AUTH_GENERIC_OAUTH_EMPTY_SCOPES=false
+GF_AUTH_GENERIC_OAUTH_USE_PKCE=true
+GF_AUTH_GENERIC_OAUTH_ALLOW_SIGN_UP=true
+GF_AUTH_GENERIC_OAUTH_AUTO_LOGIN=false
+GF_AUTH_GENERIC_OAUTH_LOGIN_ATTRIBUTE_PATH=preferred_username
+GF_AUTH_GENERIC_OAUTH_NAME_ATTRIBUTE_PATH=name
+GF_AUTH_GENERIC_OAUTH_EMAIL_ATTRIBUTE_PATH=email
+GF_AUTH_GENERIC_OAUTH_ROLE_ATTRIBUTE_PATH=contains(groups[*], 'admins') && 'Admin' || 'Viewer'
+GF_AUTH_GENERIC_OAUTH_ALLOW_ASSIGN_GRAFANA_ADMIN=true
+GF_AUTH_DISABLE_LOGIN_FORM=false

monitoring/DEPLOY.md

@@ -1,118 +1,105 @@
-# Phase 6 — Grafana + Prometheus Deployment on PD
+# Monitoring Deployment on PD
 
-> Adapted from HOMELAB_BUILDOUT_PLAN.md Phase 6, changed host from N.O.M.A.D. to PlausibleDeniability.
+This stack now includes:
 
-## 1. Scaffold Directories
+- Prometheus
+- Grafana
+- Loki
+- Promtail
+- node-exporter
+- cAdvisor
 
-SSH into PD (or run locally):
+Source-of-truth path:
+
+- /mnt/docker-ssd/docker/compose/monitoring
+
+## 1. Stage repo files on PD
+
+Make sure the monitoring directory from this repo is present on PD at:
+
+- /mnt/docker-ssd/docker/compose/monitoring
+
+## 2. Prepare runtime directories
+
+Create the required appdata paths on PD:
 
 ```bash
-# Stack directory
+mkdir -p /mnt/tank/docker/appdata/prometheus
-sudo mkdir -p /opt/monitoring/provisioning/datasources
+mkdir -p /mnt/tank/docker/appdata/grafana/dashboards
-sudo mkdir -p /opt/monitoring/provisioning/dashboards
+mkdir -p /mnt/tank/docker/appdata/loki
+mkdir -p /mnt/tank/docker/appdata/promtail
 
-# Prometheus TSDB on tank (sequential writes, grows with retention)
+chown 65534:65534 /mnt/tank/docker/appdata/prometheus
-sudo mkdir -p /mnt/tank/docker/appdata/prometheus
+chown 472:472 /mnt/tank/docker/appdata/grafana
-sudo chown 65534:65534 /mnt/tank/docker/appdata/prometheus
+chown 472:472 /mnt/tank/docker/appdata/grafana/dashboards
-
-# Grafana data on tank
-sudo mkdir -p /mnt/tank/docker/appdata/grafana
-sudo chown 472:472 /mnt/tank/docker/appdata/grafana
 ```
 
-## 2. Copy Files to PD
+## 3. Create the live env file
-
-Copy everything from this `monitoring/` folder to `/opt/monitoring/` on PD:
 
 ```bash
-# From your local machine (adjust source path as needed)
+cd /mnt/docker-ssd/docker/compose/monitoring
-scp -r monitoring/* pd:/opt/monitoring/
-
-# Or if working directly on PD, copy files into place:
-# docker-compose.yaml   → /opt/monitoring/docker-compose.yaml
-# prometheus.yml        → /opt/monitoring/prometheus.yml
-# provisioning/         → /opt/monitoring/provisioning/
-# .env.example          → /opt/monitoring/.env.example
-```
-
-## 3. Create .env
-
-```bash
-cd /opt/monitoring
 cp .env.example .env
-nano .env
+chmod 600 .env
-# Set GF_ADMIN_PASS=$(openssl rand -hex 16)
 ```
 
-## 4. Pangolin Configuration
+Set a real Grafana password before deploy.
+Set a real Grafana OIDC client secret before enabling Authelia login.
 
-In Pangolin dashboard, create a new resource using PD's existing Newt:
+## 4. Pangolin exposure
 
-| Field  | Value                    |
+Create a Pangolin resource for Grafana:
-|--------|--------------------------|
-| Domain | `grafana.paccoco.com`    |
-| Scheme | `http`                   |
-| Host   | `grafana`                |
-| Port   | `3000`                   |
 
-Since Grafana joins the `pangolin` network, PD's Newt can reach it by container name.
+- Domain: grafana.paccoco.com
+- Scheme: http
+- Host: grafana
+- Port: 3000
 
-## 5. Validate & Deploy
+If Grafana is moving to Authelia OIDC, Pangolin auth in front of the Grafana resource should be disabled after OIDC is verified. Otherwise users get double-auth prompts for no good reason.
+
+Loki stays internal-only on PD. Do not publish its host port unless you intentionally move or retire any service already using `3100`.
+
+## 5. Validate and deploy
 
 ```bash
-cd /opt/monitoring
+cd /mnt/docker-ssd/docker/compose/monitoring
 docker compose --env-file .env config
 docker compose --env-file .env up -d
 ```
 
-## 6. Post-Deploy Verification
+Or run the helper:
 
 ```bash
-# All three containers running?
+cd /mnt/docker-ssd/docker/compose/monitoring
-docker ps --filter name=prometheus --filter name=grafana --filter name=node-exporter
+./deploy.sh
-
-# Prometheus scraping targets?
-curl -s http://10.5.1.6:9090/api/v1/targets | python3 -m json.tool | head -40
-
-# Grafana UI accessible?
-curl -s -o /dev/null -w "%{http_code}" http://10.5.1.6:3000/
-# Expected: 200 or 302
-
-# Node exporter metrics flowing?
-curl -s http://10.5.1.6:9100/metrics | head -10
 ```
 
-> Remember: use `10.5.1.6` not `localhost` for health checks on TrueNAS Scale.
+## 6. Post-deploy verification
 
-## 7. Recommended Dashboard Imports
+```bash
+docker ps --filter name=prometheus --filter name=grafana --filter name=loki --filter name=promtail --filter name=node-exporter --filter name=cadvisor
 
-In Grafana → Dashboards → New → Import → Enter ID:
+curl -fsS http://10.5.1.6:9090/-/ready
+curl -fsS http://10.5.1.6:9100/metrics | head -10
+curl -fsS http://10.5.1.6:9090/api/v1/targets | python3 -m json.tool | head -60
+sudo docker inspect -f '{{.State.Status}}' loki
+```
 
-| Dashboard              | ID   | Purpose                           |
+In Grafana, confirm:
-|------------------------|------|-----------------------------------|
-| Node Exporter Full     | 1860 | PD system metrics                 |
-| Docker Container Stats | 893  | Container resource usage          |
-| Netdata via Prometheus | search | PD and Serenity system metrics  |
 
-## 8. Grafana → n8n Alert Webhook
+- Prometheus datasource is healthy
+- Loki datasource is healthy
+- Explore can query labels from Loki
+- login page shows the `Authelia` OAuth option
 
-In Grafana → Alerting → Contact Points, create:
+## 7. What this adds
 
-| Field       | Value                                            |
+- host metrics from node-exporter
-|-------------|--------------------------------------------------|
+- PD container metrics from cAdvisor
-| Name        | `n8n-alerts`                                     |
+- PD Docker logs into Loki via Promtail
-| Type        | Webhook                                          |
+- Grafana Explore for metrics and logs in one place
-| URL         | `https://n8n.paccoco.com/webhook/grafana-alert`  |
-| HTTP Method | POST                                             |
-
-Suggested alert rules:
-- ZFS pool utilization > 85%
-- Container memory > 90% of limit
-- Host CPU sustained > 90% for 5 minutes
-- Disk I/O latency spikes
 
 ## Notes
 
-- **Storage:** Prometheus TSDB and Grafana data are on tank (plenty of headroom). 90-day retention is configured. Prometheus writes sequentially so tank performs fine here.
+- Promtail is intentionally used instead of Alloy here because it is simpler to validate and recover on this host.
-- **N.O.M.A.D. scraping:** The N.O.M.A.D. target in prometheus.yml is commented out. Uncomment once Netdata or node-exporter is running on N.O.M.A.D. at 10.5.1.16.
+- cAdvisor currently covers PD containers only. If you want NOMAD container metrics too, deploy a separate exporter on NOMAD instead of pretending PD can see across hosts.
-- **Image versions:** Prometheus v3.4.0, Grafana 13.0.1, node-exporter v1.9.0 — verify these are still current before deploying.
+- Prometheus remains explicitly pinned to the v3 line because latest has been sloppy about major-version intent.

monitoring/alerts.yml

@@ -0,0 +1,56 @@
+groups:
+  - name: homelab-core
+    rules:
+      - alert: ScrapeTargetDown
+        expr: up == 0
+        for: 5m
+        labels:
+          severity: warning
+        annotations:
+          summary: "Prometheus target down: {{ $labels.job }}"
+          description: "Target {{ $labels.instance }} for job {{ $labels.job }} has been down for more than 5 minutes."
+
+      - alert: PDMemoryPressureHigh
+        expr: 100 * (1 - (node_memory_MemAvailable_bytes{job="pd-node"} / node_memory_MemTotal_bytes{job="pd-node"})) > 90
+        for: 10m
+        labels:
+          severity: warning
+        annotations:
+          summary: "PD memory usage is high"
+          description: "PlausibleDeniability memory usage has been above 90% for more than 10 minutes."
+
+      - alert: PDFilesystemUsageHigh
+        expr: 100 * (1 - (node_filesystem_avail_bytes{job="pd-node", mountpoint="/", fstype!~"tmpfs|ramfs|overlay|squashfs|nfs4"} / node_filesystem_size_bytes{job="pd-node", mountpoint="/", fstype!~"tmpfs|ramfs|overlay|squashfs|nfs4"})) > 85
+        for: 15m
+        labels:
+          severity: warning
+        annotations:
+          summary: "PD root filesystem is filling up"
+          description: "PlausibleDeniability root filesystem has been above 85% for more than 15 minutes."
+
+      - alert: PDContainerRecentlyRestarted
+        expr: changes(container_start_time_seconds{job="pd-cadvisor", name!=""}[30m]) > 0
+        for: 5m
+        labels:
+          severity: info
+        annotations:
+          summary: "PD container restarted recently"
+          description: "Container {{ $labels.name }} restarted within the last 30 minutes."
+
+      - alert: PDRestoreVerificationFailed
+        expr: pd_restore_verification_success == 0
+        for: 15m
+        labels:
+          severity: warning
+        annotations:
+          summary: "PD restore verification failed"
+          description: "The last scheduled restore verification on PlausibleDeniability failed."
+
+      - alert: PDRestoreVerificationStale
+        expr: time() - pd_restore_verification_last_success_timestamp_seconds > 8640000
+        for: 1h
+        labels:
+          severity: warning
+        annotations:
+          summary: "PD restore verification is stale"
+          description: "No successful PD restore verification has been recorded in the last 100 days."

monitoring/dashboards/homelab-overview.json

@@ -0,0 +1,714 @@
+{
+  "annotations": {
+    "list": [
+      {
+        "builtIn": 1,
+        "datasource": {
+          "type": "grafana",
+          "uid": "-- Grafana --"
+        },
+        "enable": true,
+        "hide": true,
+        "iconColor": "rgba(0, 211, 255, 1)",
+        "name": "Annotations & Alerts",
+        "type": "dashboard"
+      }
+    ]
+  },
+  "editable": true,
+  "fiscalYearStartMonth": 0,
+  "graphTooltip": 1,
+  "id": null,
+  "links": [],
+  "panels": [
+    {
+      "datasource": "Prometheus",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "red",
+                "value": null
+              },
+              {
+                "color": "green",
+                "value": 1
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 4,
+        "w": 4,
+        "x": 0,
+        "y": 0
+      },
+      "id": 1,
+      "options": {
+        "colorMode": "background",
+        "graphMode": "none",
+        "justifyMode": "center",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "value_and_name"
+      },
+      "pluginVersion": "13.0.1",
+      "targets": [
+        {
+          "editorMode": "code",
+          "expr": "max(up{host=\"plausible-deniability\"})",
+          "instant": true,
+          "legendFormat": "PD",
+          "range": false,
+          "refId": "A"
+        }
+      ],
+      "title": "PD Up",
+      "type": "stat"
+    },
+    {
+      "datasource": "Prometheus",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "red",
+                "value": null
+              },
+              {
+                "color": "green",
+                "value": 1
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 4,
+        "w": 4,
+        "x": 4,
+        "y": 0
+      },
+      "id": 2,
+      "options": {
+        "colorMode": "background",
+        "graphMode": "none",
+        "justifyMode": "center",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "value_and_name"
+      },
+      "pluginVersion": "13.0.1",
+      "targets": [
+        {
+          "editorMode": "code",
+          "expr": "max(up{host=\"nomad\"})",
+          "instant": true,
+          "legendFormat": "NOMAD",
+          "range": false,
+          "refId": "A"
+        }
+      ],
+      "title": "NOMAD Up",
+      "type": "stat"
+    },
+    {
+      "datasource": "Prometheus",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "red",
+                "value": null
+              },
+              {
+                "color": "green",
+                "value": 1
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 4,
+        "w": 4,
+        "x": 8,
+        "y": 0
+      },
+      "id": 3,
+      "options": {
+        "colorMode": "background",
+        "graphMode": "none",
+        "justifyMode": "center",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "value_and_name"
+      },
+      "pluginVersion": "13.0.1",
+      "targets": [
+        {
+          "editorMode": "code",
+          "expr": "max(up{host=\"serenity\"})",
+          "instant": true,
+          "legendFormat": "Serenity",
+          "range": false,
+          "refId": "A"
+        }
+      ],
+      "title": "Serenity Up",
+      "type": "stat"
+    },
+    {
+      "datasource": "Prometheus",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "orange",
+                "value": 1
+              },
+              {
+                "color": "red",
+                "value": 2
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 4,
+        "w": 4,
+        "x": 12,
+        "y": 0
+      },
+      "id": 4,
+      "options": {
+        "colorMode": "background",
+        "graphMode": "none",
+        "justifyMode": "center",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "value_and_name"
+      },
+      "pluginVersion": "13.0.1",
+      "targets": [
+        {
+          "editorMode": "code",
+          "expr": "count(up) - sum(up)",
+          "instant": true,
+          "legendFormat": "Down",
+          "range": false,
+          "refId": "A"
+        }
+      ],
+      "title": "Targets Down",
+      "type": "stat"
+    },
+    {
+      "datasource": "Prometheus",
+      "fieldConfig": {
+        "defaults": {
+          "unit": "ops"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 4,
+        "w": 4,
+        "x": 16,
+        "y": 0
+      },
+      "id": 5,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "center",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "value_and_name"
+      },
+      "pluginVersion": "13.0.1",
+      "targets": [
+        {
+          "editorMode": "code",
+          "expr": "rate(prometheus_tsdb_head_samples_appended_total[5m])",
+          "instant": true,
+          "legendFormat": "samples/s",
+          "range": false,
+          "refId": "A"
+        }
+      ],
+      "title": "Samples / Sec",
+      "type": "stat"
+    },
+    {
+      "datasource": "Prometheus",
+      "fieldConfig": {
+        "defaults": {
+          "unit": "short"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 4,
+        "w": 4,
+        "x": 20,
+        "y": 0
+      },
+      "id": 6,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "center",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "value_and_name"
+      },
+      "pluginVersion": "13.0.1",
+      "targets": [
+        {
+          "editorMode": "code",
+          "expr": "prometheus_tsdb_head_series",
+          "instant": true,
+          "legendFormat": "series",
+          "range": false,
+          "refId": "A"
+        }
+      ],
+      "title": "Head Series",
+      "type": "stat"
+    },
+    {
+      "datasource": "Prometheus",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "drawStyle": "line",
+            "fillOpacity": 15,
+            "lineInterpolation": "smooth",
+            "lineWidth": 2,
+            "showPoints": "never"
+          },
+          "unit": "percent"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 4
+      },
+      "id": 7,
+      "options": {
+        "legend": {
+          "displayMode": "list",
+          "placement": "bottom"
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "pluginVersion": "13.0.1",
+      "targets": [
+        {
+          "editorMode": "code",
+          "expr": "100 - (avg by (host) (rate(node_cpu_seconds_total{host=~\"plausible-deniability|nomad\", mode=\"idle\"}[5m])) * 100)",
+          "legendFormat": "{{host}}",
+          "range": true,
+          "refId": "A"
+        },
+        {
+          "editorMode": "code",
+          "expr": "100 - avg by (host) (netdata_system_cpu_percentage_average{host=\"serenity\", dimension=\"idle\"})",
+          "legendFormat": "{{host}}",
+          "range": true,
+          "refId": "B"
+        }
+      ],
+      "title": "CPU Usage %",
+      "type": "timeseries"
+    },
+    {
+      "datasource": "Prometheus",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisPlacement": "auto",
+            "drawStyle": "line",
+            "fillOpacity": 15,
+            "lineInterpolation": "smooth",
+            "lineWidth": 2,
+            "showPoints": "never"
+          },
+          "unit": "percent"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 4
+      },
+      "id": 8,
+      "options": {
+        "legend": {
+          "displayMode": "list",
+          "placement": "bottom"
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "pluginVersion": "13.0.1",
+      "targets": [
+        {
+          "editorMode": "code",
+          "expr": "100 * (1 - (node_memory_MemAvailable_bytes{host=~\"plausible-deniability|nomad\"} / node_memory_MemTotal_bytes{host=~\"plausible-deniability|nomad\"}))",
+          "legendFormat": "{{host}}",
+          "range": true,
+          "refId": "A"
+        },
+        {
+          "editorMode": "code",
+          "expr": "100 * (netdata_system_ram_MiB_average{host=\"serenity\", dimension=\"used\"} / (netdata_system_ram_MiB_average{host=\"serenity\", dimension=\"used\"} + netdata_system_ram_MiB_average{host=\"serenity\", dimension=\"free\"} + netdata_system_ram_MiB_average{host=\"serenity\", dimension=\"cached\"} + netdata_system_ram_MiB_average{host=\"serenity\", dimension=\"buffers\"}))",
+          "legendFormat": "serenity",
+          "range": true,
+          "refId": "B"
+        }
+      ],
+      "title": "Memory Used %",
+      "type": "timeseries"
+    },
+    {
+      "datasource": "Prometheus",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisPlacement": "auto",
+            "drawStyle": "line",
+            "fillOpacity": 15,
+            "lineInterpolation": "smooth",
+            "lineWidth": 2,
+            "showPoints": "never"
+          },
+          "unit": "percent"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 12
+      },
+      "id": 9,
+      "options": {
+        "legend": {
+          "displayMode": "list",
+          "placement": "bottom"
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "pluginVersion": "13.0.1",
+      "targets": [
+        {
+          "editorMode": "code",
+          "expr": "100 * (1 - (node_filesystem_avail_bytes{host=~\"plausible-deniability|nomad\", mountpoint=\"/\", fstype!~\"tmpfs|ramfs|overlay|squashfs|nfs4\"} / node_filesystem_size_bytes{host=~\"plausible-deniability|nomad\", mountpoint=\"/\", fstype!~\"tmpfs|ramfs|overlay|squashfs|nfs4\"}))",
+          "legendFormat": "{{host}}",
+          "range": true,
+          "refId": "A"
+        },
+        {
+          "editorMode": "code",
+          "expr": "100 * (sum by (host) (netdata_disk_space_GiB_average{host=\"serenity\", mount_point=\"/\", dimension=\"used\"}) / (sum by (host) (netdata_disk_space_GiB_average{host=\"serenity\", mount_point=\"/\", dimension=\"used\"}) + sum by (host) (netdata_disk_space_GiB_average{host=\"serenity\", mount_point=\"/\", dimension=\"avail\"})))",
+          "legendFormat": "{{host}}",
+          "range": true,
+          "refId": "B"
+        }
+      ],
+      "title": "Root Filesystem Used %",
+      "type": "timeseries"
+    },
+    {
+      "datasource": "Prometheus",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisPlacement": "auto",
+            "drawStyle": "line",
+            "fillOpacity": 10,
+            "lineInterpolation": "smooth",
+            "lineWidth": 2,
+            "showPoints": "never"
+          },
+          "unit": "short"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 12
+      },
+      "id": 10,
+      "options": {
+        "legend": {
+          "displayMode": "list",
+          "placement": "bottom"
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "pluginVersion": "13.0.1",
+      "targets": [
+        {
+          "editorMode": "code",
+          "expr": "node_load1{host=~\"plausible-deniability|nomad\"}",
+          "legendFormat": "{{host}}",
+          "range": true,
+          "refId": "A"
+        },
+        {
+          "editorMode": "code",
+          "expr": "netdata_system_load_load_average{host=\"serenity\", dimension=\"load1\"}",
+          "legendFormat": "{{host}}",
+          "range": true,
+          "refId": "B"
+        }
+      ],
+      "title": "Load 1m",
+      "type": "timeseries"
+    },
+    {
+      "datasource": "Prometheus",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisPlacement": "auto",
+            "drawStyle": "line",
+            "fillOpacity": 10,
+            "lineInterpolation": "smooth",
+            "lineWidth": 2,
+            "showPoints": "never"
+          },
+          "unit": "Bps"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 20
+      },
+      "id": 11,
+      "options": {
+        "legend": {
+          "displayMode": "list",
+          "placement": "bottom"
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "pluginVersion": "13.0.1",
+      "targets": [
+        {
+          "editorMode": "code",
+          "expr": "sum by (host) (rate(node_network_receive_bytes_total{host=~\"plausible-deniability|nomad\", device!~\"lo|veth.*|br-.*|docker.*\"}[5m]))",
+          "legendFormat": "{{host}}",
+          "range": true,
+          "refId": "A"
+        },
+        {
+          "editorMode": "code",
+          "expr": "sum by (host) (netdata_net_net_kilobits_persec_average{host=\"serenity\", interface_type=\"real\", dimension=\"received\"}) * 125",
+          "legendFormat": "{{host}}",
+          "range": true,
+          "refId": "B"
+        }
+      ],
+      "title": "Network Receive",
+      "type": "timeseries"
+    },
+    {
+      "datasource": "Prometheus",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "max": 100,
+          "min": 0,
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "orange",
+                "value": 75
+              },
+              {
+                "color": "red",
+                "value": 90
+              }
+            ]
+          },
+          "unit": "percent"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 20
+      },
+      "id": 12,
+      "options": {
+        "displayMode": "gradient",
+        "minVizHeight": 16,
+        "minVizWidth": 8,
+        "orientation": "horizontal",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "showUnfilled": true
+      },
+      "pluginVersion": "13.0.1",
+      "targets": [
+        {
+          "editorMode": "code",
+          "expr": "topk(12, 100 * (1 - node_filesystem_avail_bytes{host=~\"plausible-deniability|nomad\", fstype!~\"tmpfs|ramfs|overlay|squashfs|nfs4\", mountpoint!~\"/var/lib/docker/.+\"} / node_filesystem_size_bytes{host=~\"plausible-deniability|nomad\", fstype!~\"tmpfs|ramfs|overlay|squashfs|nfs4\", mountpoint!~\"/var/lib/docker/.+\"}))",
+          "instant": true,
+          "legendFormat": "{{host}} {{mountpoint}}",
+          "range": false,
+          "refId": "A"
+        }
+      ],
+      "title": "Top Filesystem Usage (PD + NOMAD)",
+      "type": "bargauge"
+    }
+  ],
+  "refresh": "30s",
+  "schemaVersion": 41,
+  "tags": [
+    "homelab",
+    "prometheus",
+    "overview"
+  ],
+  "templating": {
+    "list": []
+  },
+  "time": {
+    "from": "now-24h",
+    "to": "now"
+  },
+  "timepicker": {},
+  "timezone": "",
+  "title": "Homelab Overview",
+  "uid": "homelab-overview",
+  "version": 1,
+  "weekStart": ""
+}

monitoring/dashboards/monitoring-stack.json

@@ -0,0 +1,562 @@
+{
+  "annotations": {
+    "list": [
+      {
+        "builtIn": 1,
+        "datasource": {
+          "type": "grafana",
+          "uid": "-- Grafana --"
+        },
+        "enable": true,
+        "hide": true,
+        "iconColor": "rgba(0, 211, 255, 1)",
+        "name": "Annotations & Alerts",
+        "type": "dashboard"
+      }
+    ]
+  },
+  "editable": true,
+  "fiscalYearStartMonth": 0,
+  "graphTooltip": 1,
+  "id": null,
+  "links": [],
+  "panels": [
+    {
+      "datasource": "Prometheus",
+      "fieldConfig": {
+        "defaults": {
+          "unit": "short"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 4,
+        "w": 4,
+        "x": 0,
+        "y": 0
+      },
+      "id": 1,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "center",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "value_and_name"
+      },
+      "targets": [
+        {
+          "editorMode": "code",
+          "expr": "sum(up)",
+          "instant": true,
+          "range": false,
+          "refId": "A"
+        }
+      ],
+      "title": "Targets Up",
+      "type": "stat"
+    },
+    {
+      "datasource": "Prometheus",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "orange",
+                "value": 1
+              },
+              {
+                "color": "red",
+                "value": 2
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 4,
+        "w": 4,
+        "x": 4,
+        "y": 0
+      },
+      "id": 2,
+      "options": {
+        "colorMode": "background",
+        "graphMode": "none",
+        "justifyMode": "center",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "value_and_name"
+      },
+      "targets": [
+        {
+          "editorMode": "code",
+          "expr": "count(up) - sum(up)",
+          "instant": true,
+          "range": false,
+          "refId": "A"
+        }
+      ],
+      "title": "Targets Down",
+      "type": "stat"
+    },
+    {
+      "datasource": "Prometheus",
+      "fieldConfig": {
+        "defaults": {
+          "unit": "short"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 4,
+        "w": 4,
+        "x": 8,
+        "y": 0
+      },
+      "id": 3,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "none",
+        "justifyMode": "center",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "value_and_name"
+      },
+      "targets": [
+        {
+          "editorMode": "code",
+          "expr": "sum(prometheus_target_scrape_pool_targets)",
+          "instant": true,
+          "range": false,
+          "refId": "A"
+        }
+      ],
+      "title": "Scrape Targets",
+      "type": "stat"
+    },
+    {
+      "datasource": "Prometheus",
+      "fieldConfig": {
+        "defaults": {
+          "unit": "ops"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 4,
+        "w": 4,
+        "x": 12,
+        "y": 0
+      },
+      "id": 4,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "center",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "value_and_name"
+      },
+      "targets": [
+        {
+          "editorMode": "code",
+          "expr": "rate(prometheus_tsdb_head_samples_appended_total[5m])",
+          "instant": true,
+          "range": false,
+          "refId": "A"
+        }
+      ],
+      "title": "Samples / Sec",
+      "type": "stat"
+    },
+    {
+      "datasource": "Prometheus",
+      "fieldConfig": {
+        "defaults": {
+          "unit": "short"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 4,
+        "w": 4,
+        "x": 16,
+        "y": 0
+      },
+      "id": 5,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "center",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "value_and_name"
+      },
+      "targets": [
+        {
+          "editorMode": "code",
+          "expr": "prometheus_tsdb_head_series",
+          "instant": true,
+          "range": false,
+          "refId": "A"
+        }
+      ],
+      "title": "Head Series",
+      "type": "stat"
+    },
+    {
+      "datasource": "Prometheus",
+      "fieldConfig": {
+        "defaults": {
+          "unit": "bytes"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 4,
+        "w": 4,
+        "x": 20,
+        "y": 0
+      },
+      "id": 6,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "center",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "value_and_name"
+      },
+      "targets": [
+        {
+          "editorMode": "code",
+          "expr": "prometheus_tsdb_storage_blocks_bytes",
+          "instant": true,
+          "range": false,
+          "refId": "A"
+        }
+      ],
+      "title": "TSDB Blocks",
+      "type": "stat"
+    },
+    {
+      "datasource": "Prometheus",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisPlacement": "auto",
+            "drawStyle": "line",
+            "fillOpacity": 10,
+            "lineInterpolation": "smooth",
+            "lineWidth": 2,
+            "showPoints": "never"
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 4
+      },
+      "id": 7,
+      "options": {
+        "legend": {
+          "displayMode": "list",
+          "placement": "bottom"
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "editorMode": "code",
+          "expr": "up",
+          "legendFormat": "{{job}} {{host}}",
+          "range": true,
+          "refId": "A"
+        }
+      ],
+      "title": "Target Health",
+      "type": "timeseries"
+    },
+    {
+      "datasource": "Prometheus",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisPlacement": "auto",
+            "drawStyle": "line",
+            "fillOpacity": 10,
+            "lineInterpolation": "smooth",
+            "lineWidth": 2,
+            "showPoints": "never"
+          },
+          "unit": "s"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 4
+      },
+      "id": 8,
+      "options": {
+        "legend": {
+          "displayMode": "list",
+          "placement": "bottom"
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "editorMode": "code",
+          "expr": "scrape_duration_seconds{job=~\"pd-node|nomad-node|serenity-netdata\"}",
+          "legendFormat": "{{job}} {{host}}",
+          "range": true,
+          "refId": "A"
+        }
+      ],
+      "title": "Scrape Duration",
+      "type": "timeseries"
+    },
+    {
+      "datasource": "Prometheus",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisPlacement": "auto",
+            "drawStyle": "line",
+            "fillOpacity": 10,
+            "lineInterpolation": "smooth",
+            "lineWidth": 2,
+            "showPoints": "never"
+          },
+          "unit": "short"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 12
+      },
+      "id": 9,
+      "options": {
+        "legend": {
+          "displayMode": "list",
+          "placement": "bottom"
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "editorMode": "code",
+          "expr": "scrape_samples_post_metric_relabeling{job=~\"pd-node|nomad-node|serenity-netdata\"}",
+          "legendFormat": "{{job}} {{host}}",
+          "range": true,
+          "refId": "A"
+        }
+      ],
+      "title": "Samples Per Scrape",
+      "type": "timeseries"
+    },
+    {
+      "datasource": "Prometheus",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisPlacement": "auto",
+            "drawStyle": "line",
+            "fillOpacity": 10,
+            "lineInterpolation": "smooth",
+            "lineWidth": 2,
+            "showPoints": "never"
+          },
+          "unit": "ops"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 12
+      },
+      "id": 10,
+      "options": {
+        "legend": {
+          "displayMode": "list",
+          "placement": "bottom"
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "editorMode": "code",
+          "expr": "rate(prometheus_tsdb_head_samples_appended_total[5m])",
+          "legendFormat": "samples/s",
+          "range": true,
+          "refId": "A"
+        },
+        {
+          "editorMode": "code",
+          "expr": "rate(prometheus_http_requests_total[5m])",
+          "legendFormat": "http req/s",
+          "range": true,
+          "refId": "B"
+        }
+      ],
+      "title": "Prometheus Throughput",
+      "type": "timeseries"
+    },
+    {
+      "datasource": "Prometheus",
+      "fieldConfig": {
+        "defaults": {
+          "custom": {
+            "align": "auto",
+            "cellOptions": {
+              "type": "auto"
+            },
+            "inspect": false
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 24,
+        "x": 0,
+        "y": 20
+      },
+      "id": 11,
+      "options": {
+        "cellHeight": "sm",
+        "footer": {
+          "show": false
+        },
+        "showHeader": true
+      },
+      "targets": [
+        {
+          "editorMode": "code",
+          "expr": "up",
+          "format": "table",
+          "instant": true,
+          "range": false,
+          "refId": "A"
+        }
+      ],
+      "title": "Current Target States",
+      "transformations": [
+        {
+          "id": "organize",
+          "options": {
+            "excludeByName": {
+              "Time": true
+            },
+            "indexByName": {
+              "Value": 0,
+              "host": 1,
+              "instance": 2,
+              "job": 3
+            },
+            "renameByName": {
+              "Value": "up"
+            }
+          }
+        }
+      ],
+      "type": "table"
+    }
+  ],
+  "refresh": "30s",
+  "schemaVersion": 41,
+  "tags": [
+    "homelab",
+    "prometheus",
+    "monitoring"
+  ],
+  "templating": {
+    "list": []
+  },
+  "time": {
+    "from": "now-24h",
+    "to": "now"
+  },
+  "timepicker": {},
+  "timezone": "",
+  "title": "Monitoring Stack",
+  "uid": "monitoring-stack",
+  "version": 1,
+  "weekStart": ""
+}

monitoring/dashboards/pd-containers.json

@@ -0,0 +1,512 @@
+{
+  "annotations": {
+    "list": [
+      {
+        "builtIn": 1,
+        "datasource": {
+          "type": "grafana",
+          "uid": "-- Grafana --"
+        },
+        "enable": true,
+        "hide": true,
+        "iconColor": "rgba(0, 211, 255, 1)",
+        "name": "Annotations & Alerts",
+        "type": "dashboard"
+      }
+    ]
+  },
+  "editable": true,
+  "fiscalYearStartMonth": 0,
+  "graphTooltip": 1,
+  "id": null,
+  "links": [],
+  "panels": [
+    {
+      "datasource": "Prometheus",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "red",
+                "value": null
+              },
+              {
+                "color": "green",
+                "value": 1
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 4,
+        "w": 6,
+        "x": 0,
+        "y": 0
+      },
+      "id": 1,
+      "options": {
+        "colorMode": "background",
+        "graphMode": "none",
+        "justifyMode": "center",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "value_and_name"
+      },
+      "targets": [
+        {
+          "editorMode": "code",
+          "expr": "count(container_start_time_seconds{job=\"pd-cadvisor\",name!=\"\"})",
+          "instant": true,
+          "range": false,
+          "refId": "A"
+        }
+      ],
+      "title": "Tracked Containers",
+      "type": "stat"
+    },
+    {
+      "datasource": "Prometheus",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "orange",
+                "value": 1
+              },
+              {
+                "color": "red",
+                "value": 5
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 4,
+        "w": 6,
+        "x": 6,
+        "y": 0
+      },
+      "id": 2,
+      "options": {
+        "colorMode": "background",
+        "graphMode": "none",
+        "justifyMode": "center",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "value_and_name"
+      },
+      "targets": [
+        {
+          "editorMode": "code",
+          "expr": "sum(changes(container_start_time_seconds{job=\"pd-cadvisor\",name!=\"\"}[24h]) > 0)",
+          "instant": true,
+          "range": false,
+          "refId": "A"
+        }
+      ],
+      "title": "Containers Restarted (24h)",
+      "type": "stat"
+    },
+    {
+      "datasource": "Prometheus",
+      "fieldConfig": {
+        "defaults": {
+          "unit": "bytes",
+          "color": {
+            "mode": "thresholds"
+          },
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "orange",
+                "value": 1073741824
+              },
+              {
+                "color": "red",
+                "value": 2147483648
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 4,
+        "w": 6,
+        "x": 12,
+        "y": 0
+      },
+      "id": 3,
+      "options": {
+        "colorMode": "background",
+        "graphMode": "none",
+        "justifyMode": "center",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "value_and_name"
+      },
+      "targets": [
+        {
+          "editorMode": "code",
+          "expr": "max(container_memory_working_set_bytes{job=\"pd-cadvisor\",name!=\"\"})",
+          "instant": true,
+          "range": false,
+          "refId": "A"
+        }
+      ],
+      "title": "Top Container RAM",
+      "type": "stat"
+    },
+    {
+      "datasource": "Prometheus",
+      "fieldConfig": {
+        "defaults": {
+          "unit": "percentunit",
+          "decimals": 2,
+          "color": {
+            "mode": "thresholds"
+          },
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "orange",
+                "value": 0.7
+              },
+              {
+                "color": "red",
+                "value": 1.5
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 4,
+        "w": 6,
+        "x": 18,
+        "y": 0
+      },
+      "id": 4,
+      "options": {
+        "colorMode": "background",
+        "graphMode": "none",
+        "justifyMode": "center",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "value_and_name"
+      },
+      "targets": [
+        {
+          "editorMode": "code",
+          "expr": "max(rate(container_cpu_usage_seconds_total{job=\"pd-cadvisor\",name!=\"\"}[5m]))",
+          "instant": true,
+          "range": false,
+          "refId": "A"
+        }
+      ],
+      "title": "Top Container CPU/sec",
+      "type": "stat"
+    },
+    {
+      "datasource": "Prometheus",
+      "fieldConfig": {
+        "defaults": {
+          "unit": "bytes",
+          "color": {
+            "mode": "palette-classic"
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 9,
+        "w": 12,
+        "x": 0,
+        "y": 4
+      },
+      "id": 5,
+      "options": {
+        "displayMode": "gradient",
+        "orientation": "horizontal",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "showUnfilled": true,
+        "sizing": "auto"
+      },
+      "targets": [
+        {
+          "editorMode": "code",
+          "expr": "topk(12, container_memory_working_set_bytes{job=\"pd-cadvisor\",name!=\"\"})",
+          "legendFormat": "{{name}}",
+          "refId": "A"
+        }
+      ],
+      "title": "Top Container RAM (PD)",
+      "type": "bargauge"
+    },
+    {
+      "datasource": "Prometheus",
+      "fieldConfig": {
+        "defaults": {
+          "unit": "percentunit",
+          "decimals": 3,
+          "color": {
+            "mode": "palette-classic"
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 9,
+        "w": 12,
+        "x": 12,
+        "y": 4
+      },
+      "id": 6,
+      "options": {
+        "displayMode": "gradient",
+        "orientation": "horizontal",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "showUnfilled": true,
+        "sizing": "auto"
+      },
+      "targets": [
+        {
+          "editorMode": "code",
+          "expr": "topk(12, rate(container_cpu_usage_seconds_total{job=\"pd-cadvisor\",name!=\"\"}[5m]))",
+          "legendFormat": "{{name}}",
+          "refId": "A"
+        }
+      ],
+      "title": "Top Container CPU/sec (PD)",
+      "type": "bargauge"
+    },
+    {
+      "datasource": "Prometheus",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 15,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "lineInterpolation": "linear",
+            "lineWidth": 2,
+            "pointSize": 4,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "never",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "unit": "bytes"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 9,
+        "w": 12,
+        "x": 0,
+        "y": 13
+      },
+      "id": 7,
+      "options": {
+        "legend": {
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "editorMode": "code",
+          "expr": "topk(8, container_memory_working_set_bytes{job=\"pd-cadvisor\",name!=\"\"})",
+          "legendFormat": "{{name}}",
+          "refId": "A"
+        }
+      ],
+      "title": "Top RAM Consumers Over Time (PD)",
+      "type": "timeseries"
+    },
+    {
+      "datasource": "Prometheus",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 9,
+        "w": 12,
+        "x": 12,
+        "y": 13
+      },
+      "id": 8,
+      "options": {
+        "showHeader": true,
+        "sortBy": [
+          {
+            "desc": true,
+            "displayName": "Value"
+          }
+        ]
+      },
+      "targets": [
+        {
+          "editorMode": "code",
+          "expr": "sort_desc(container_start_time_seconds{job=\"pd-cadvisor\",name!=\"\"})",
+          "format": "table",
+          "instant": true,
+          "legendFormat": "{{name}}",
+          "refId": "A"
+        }
+      ],
+      "title": "Container Start Times (PD)",
+      "transformations": [
+        {
+          "id": "organize",
+          "options": {
+            "excludeByName": {
+              "Time": true,
+              "__name__": true,
+              "container_label_com_docker_compose_config_hash": true,
+              "container_label_com_docker_compose_container_number": true,
+              "container_label_com_docker_compose_image": true,
+              "container_label_com_docker_compose_oneoff": true,
+              "container_label_com_docker_compose_project_config_files": true,
+              "container_label_com_docker_compose_project_environment_file": true,
+              "container_label_com_docker_compose_project_working_dir": true,
+              "container_label_com_docker_compose_replace": true,
+              "container_label_com_docker_compose_version": true,
+              "id": true,
+              "image": true,
+              "instance": true,
+              "job": true
+            },
+            "indexByName": {
+              "name": 0,
+              "container_label_com_docker_compose_project": 1,
+              "container_label_com_docker_compose_service": 2,
+              "Value": 3
+            },
+            "renameByName": {
+              "name": "container",
+              "container_label_com_docker_compose_project": "compose_project",
+              "container_label_com_docker_compose_service": "compose_service",
+              "Value": "started_at_epoch"
+            }
+          }
+        }
+      ],
+      "type": "table"
+    }
+  ],
+  "refresh": "30s",
+  "schemaVersion": 39,
+  "style": "dark",
+  "tags": [
+    "monitoring",
+    "docker",
+    "pd"
+  ],
+  "templating": {
+    "list": []
+  },
+  "time": {
+    "from": "now-24h",
+    "to": "now"
+  },
+  "timepicker": {},
+  "timezone": "",
+  "title": "PD Containers",
+  "uid": "pd-containers",
+  "version": 1,
+  "weekStart": ""
+}

monitoring/dashboards/serenity-containers.json

@@ -0,0 +1,550 @@
+{
+  "annotations": {
+    "list": [
+      {
+        "builtIn": 1,
+        "datasource": {
+          "type": "grafana",
+          "uid": "-- Grafana --"
+        },
+        "enable": true,
+        "hide": true,
+        "iconColor": "rgba(0, 211, 255, 1)",
+        "name": "Annotations & Alerts",
+        "type": "dashboard"
+      }
+    ]
+  },
+  "editable": true,
+  "fiscalYearStartMonth": 0,
+  "graphTooltip": 1,
+  "id": null,
+  "links": [],
+  "panels": [
+    {
+      "datasource": "Prometheus",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "red",
+                "value": null
+              },
+              {
+                "color": "green",
+                "value": 1
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 4,
+        "w": 6,
+        "x": 0,
+        "y": 0
+      },
+      "id": 1,
+      "options": {
+        "colorMode": "background",
+        "graphMode": "none",
+        "justifyMode": "center",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "value_and_name"
+      },
+      "targets": [
+        {
+          "editorMode": "code",
+          "expr": "count(netdata_docker_container_state_state_average{host=\"serenity\", dimension=\"running\"} == 1)",
+          "instant": true,
+          "range": false,
+          "refId": "A"
+        }
+      ],
+      "title": "Running Containers",
+      "type": "stat"
+    },
+    {
+      "datasource": "Prometheus",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 1
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 4,
+        "w": 6,
+        "x": 6,
+        "y": 0
+      },
+      "id": 2,
+      "options": {
+        "colorMode": "background",
+        "graphMode": "none",
+        "justifyMode": "center",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "value_and_name"
+      },
+      "targets": [
+        {
+          "editorMode": "code",
+          "expr": "count(netdata_docker_container_state_state_average{host=\"serenity\", dimension=\"exited\"} == 1)",
+          "instant": true,
+          "range": false,
+          "refId": "A"
+        }
+      ],
+      "title": "Exited Containers",
+      "type": "stat"
+    },
+    {
+      "datasource": "Prometheus",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "orange",
+                "value": 1
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 4,
+        "w": 6,
+        "x": 12,
+        "y": 0
+      },
+      "id": 3,
+      "options": {
+        "colorMode": "background",
+        "graphMode": "none",
+        "justifyMode": "center",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "value_and_name"
+      },
+      "targets": [
+        {
+          "editorMode": "code",
+          "expr": "count(netdata_docker_container_state_state_average{host=\"serenity\", dimension=\"created\"} == 1)",
+          "instant": true,
+          "range": false,
+          "refId": "A"
+        }
+      ],
+      "title": "Created Containers",
+      "type": "stat"
+    },
+    {
+      "datasource": "Prometheus",
+      "fieldConfig": {
+        "defaults": {
+          "unit": "short"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 4,
+        "w": 6,
+        "x": 18,
+        "y": 0
+      },
+      "id": 4,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "center",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "value_and_name"
+      },
+      "targets": [
+        {
+          "editorMode": "code",
+          "expr": "count(netdata_cgroup_mem_usage_MiB_average{host=\"serenity\", dimension=\"ram\"})",
+          "instant": true,
+          "range": false,
+          "refId": "A"
+        }
+      ],
+      "title": "Tracked Cgroups",
+      "type": "stat"
+    },
+    {
+      "datasource": "Prometheus",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 7,
+        "w": 8,
+        "x": 0,
+        "y": 4
+      },
+      "id": 5,
+      "options": {
+        "displayLabels": [
+          "name",
+          "value"
+        ],
+        "legend": {
+          "displayMode": "list",
+          "placement": "bottom"
+        },
+        "pieType": "pie",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "tooltip": {
+          "mode": "single"
+        }
+      },
+      "targets": [
+        {
+          "editorMode": "code",
+          "expr": "count by (dimension) (netdata_docker_container_state_state_average{host=\"serenity\"} == 1)",
+          "legendFormat": "{{dimension}}",
+          "range": true,
+          "refId": "A"
+        }
+      ],
+      "title": "Container States",
+      "type": "piechart"
+    },
+    {
+      "datasource": "Prometheus",
+      "fieldConfig": {
+        "defaults": {
+          "max": 16384,
+          "min": 0,
+          "unit": "decmbytes"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 7,
+        "w": 8,
+        "x": 8,
+        "y": 4
+      },
+      "id": 6,
+      "options": {
+        "displayMode": "gradient",
+        "minVizHeight": 16,
+        "minVizWidth": 8,
+        "orientation": "horizontal",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "showUnfilled": true
+      },
+      "targets": [
+        {
+          "editorMode": "code",
+          "expr": "topk(12, max by (cgroup_name) (netdata_cgroup_mem_usage_MiB_average{host=\"serenity\", dimension=\"ram\"}))",
+          "instant": true,
+          "legendFormat": "{{cgroup_name}}",
+          "range": false,
+          "refId": "A"
+        }
+      ],
+      "title": "Top Container RAM (MiB)",
+      "type": "bargauge"
+    },
+    {
+      "datasource": "Prometheus",
+      "fieldConfig": {
+        "defaults": {
+          "max": 100,
+          "min": 0,
+          "unit": "percent"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 7,
+        "w": 8,
+        "x": 16,
+        "y": 4
+      },
+      "id": 7,
+      "options": {
+        "displayMode": "gradient",
+        "minVizHeight": 16,
+        "minVizWidth": 8,
+        "orientation": "horizontal",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "showUnfilled": true
+      },
+      "targets": [
+        {
+          "editorMode": "code",
+          "expr": "topk(12, max by (cgroup_name) (netdata_cgroup_cpu_percentage_average{host=\"serenity\"}))",
+          "instant": true,
+          "legendFormat": "{{cgroup_name}}",
+          "range": false,
+          "refId": "A"
+        }
+      ],
+      "title": "Top Container CPU %",
+      "type": "bargauge"
+    },
+    {
+      "datasource": "Prometheus",
+      "fieldConfig": {
+        "defaults": {
+          "custom": {
+            "align": "auto",
+            "cellOptions": {
+              "type": "auto"
+            }
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 9,
+        "w": 24,
+        "x": 0,
+        "y": 11
+      },
+      "id": 8,
+      "options": {
+        "cellHeight": "sm",
+        "footer": {
+          "show": false
+        },
+        "showHeader": true
+      },
+      "targets": [
+        {
+          "editorMode": "code",
+          "expr": "max by (container_name, image, dimension) (netdata_docker_container_state_state_average{host=\"serenity\", dimension!=\"running\"} == 1)",
+          "format": "table",
+          "instant": true,
+          "range": false,
+          "refId": "A"
+        }
+      ],
+      "title": "Non-Running Containers",
+      "transformations": [
+        {
+          "id": "organize",
+          "options": {
+            "excludeByName": {
+              "Time": true
+            },
+            "indexByName": {
+              "Value": 0,
+              "container_name": 1,
+              "dimension": 2,
+              "image": 3
+            },
+            "renameByName": {
+              "Value": "state_active"
+            }
+          }
+        }
+      ],
+      "type": "table"
+    },
+    {
+      "datasource": "Prometheus",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisPlacement": "auto",
+            "drawStyle": "line",
+            "fillOpacity": 10,
+            "lineInterpolation": "smooth",
+            "lineWidth": 2,
+            "showPoints": "never"
+          },
+          "unit": "short"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 20
+      },
+      "id": 9,
+      "options": {
+        "legend": {
+          "displayMode": "list",
+          "placement": "bottom"
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "editorMode": "code",
+          "expr": "count(netdata_docker_container_state_state_average{host=\"serenity\", dimension=\"running\"} == 1)",
+          "legendFormat": "running",
+          "range": true,
+          "refId": "A"
+        },
+        {
+          "editorMode": "code",
+          "expr": "count(netdata_docker_container_state_state_average{host=\"serenity\", dimension!=\"running\"} == 1)",
+          "legendFormat": "not running",
+          "range": true,
+          "refId": "B"
+        }
+      ],
+      "title": "Container Count Over Time",
+      "type": "timeseries"
+    },
+    {
+      "datasource": "Prometheus",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisPlacement": "auto",
+            "drawStyle": "line",
+            "fillOpacity": 10,
+            "lineInterpolation": "smooth",
+            "lineWidth": 2,
+            "showPoints": "never"
+          },
+          "unit": "decmbytes"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 20
+      },
+      "id": 10,
+      "options": {
+        "legend": {
+          "displayMode": "list",
+          "placement": "bottom"
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "editorMode": "code",
+          "expr": "topk(8, max by (cgroup_name) (netdata_cgroup_mem_usage_MiB_average{host=\"serenity\", dimension=\"ram\"}))",
+          "legendFormat": "{{cgroup_name}}",
+          "range": true,
+          "refId": "A"
+        }
+      ],
+      "title": "Top RAM Consumers Over Time",
+      "type": "timeseries"
+    }
+  ],
+  "refresh": "30s",
+  "schemaVersion": 41,
+  "tags": [
+    "serenity",
+    "containers",
+    "netdata"
+  ],
+  "templating": {
+    "list": []
+  },
+  "time": {
+    "from": "now-24h",
+    "to": "now"
+  },
+  "timepicker": {},
+  "timezone": "",
+  "title": "Serenity Containers",
+  "uid": "serenity-containers",
+  "version": 1,
+  "weekStart": ""
+}

monitoring/dashboards/serenity-host-health.json

@@ -0,0 +1,997 @@
+{
+  "annotations": {
+    "list": [
+      {
+        "builtIn": 1,
+        "datasource": {
+          "type": "grafana",
+          "uid": "-- Grafana --"
+        },
+        "enable": true,
+        "hide": true,
+        "iconColor": "rgba(0, 211, 255, 1)",
+        "name": "Annotations & Alerts",
+        "type": "dashboard"
+      }
+    ]
+  },
+  "editable": true,
+  "fiscalYearStartMonth": 0,
+  "graphTooltip": 0,
+  "links": [],
+  "liveNow": false,
+  "panels": [
+    {
+      "type": "stat",
+      "title": "Serenity Up",
+      "gridPos": {
+        "h": 4,
+        "w": 4,
+        "x": 0,
+        "y": 0
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "red",
+                "value": null
+              },
+              {
+                "color": "green",
+                "value": 1
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "options": {
+        "colorMode": "background",
+        "graphMode": "none",
+        "justifyMode": "center",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "value_and_name"
+      },
+      "targets": [
+        {
+          "editorMode": "code",
+          "expr": "max(up{host=\"serenity\"})",
+          "instant": true,
+          "legendFormat": "serenity",
+          "range": false,
+          "refId": "A"
+        }
+      ]
+    },
+    {
+      "type": "stat",
+      "title": "CPU Busy %",
+      "gridPos": {
+        "h": 4,
+        "w": 4,
+        "x": 4,
+        "y": 0
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "percent",
+          "color": {
+            "mode": "thresholds"
+          },
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "orange",
+                "value": 70
+              },
+              {
+                "color": "red",
+                "value": 90
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "options": {
+        "colorMode": "background",
+        "graphMode": "none",
+        "justifyMode": "center",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "value_and_name"
+      },
+      "targets": [
+        {
+          "editorMode": "code",
+          "expr": "sum(netdata_system_cpu_percentage_average{host=\"serenity\",dimension!~\"idle|iowait|steal|guest|guest_nice\"})",
+          "instant": true,
+          "legendFormat": "CPU Busy",
+          "range": false,
+          "refId": "A"
+        }
+      ]
+    },
+    {
+      "type": "stat",
+      "title": "Memory Used %",
+      "gridPos": {
+        "h": 4,
+        "w": 4,
+        "x": 8,
+        "y": 0
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "percent",
+          "color": {
+            "mode": "thresholds"
+          },
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "orange",
+                "value": 75
+              },
+              {
+                "color": "red",
+                "value": 90
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "options": {
+        "colorMode": "background",
+        "graphMode": "none",
+        "justifyMode": "center",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "value_and_name"
+      },
+      "targets": [
+        {
+          "editorMode": "code",
+          "expr": "100 * (1 - (netdata_mem_available_MiB_average{host=\"serenity\",dimension=\"avail\"} / scalar(sum(netdata_system_ram_MiB_average{host=\"serenity\",dimension=~\"free|used|cached|buffers\"}))))",
+          "instant": true,
+          "legendFormat": "Memory",
+          "range": false,
+          "refId": "A"
+        }
+      ]
+    },
+    {
+      "type": "stat",
+      "title": "Load 1m",
+      "gridPos": {
+        "h": 4,
+        "w": 4,
+        "x": 12,
+        "y": 0
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "orange",
+                "value": 4
+              },
+              {
+                "color": "red",
+                "value": 8
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "options": {
+        "colorMode": "background",
+        "graphMode": "none",
+        "justifyMode": "center",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "value_and_name"
+      },
+      "targets": [
+        {
+          "editorMode": "code",
+          "expr": "netdata_system_load_load_average{host=\"serenity\",dimension=\"load1\"}",
+          "instant": true,
+          "legendFormat": "load1",
+          "range": false,
+          "refId": "A"
+        }
+      ]
+    },
+    {
+      "type": "stat",
+      "title": "Root Used %",
+      "gridPos": {
+        "h": 4,
+        "w": 4,
+        "x": 16,
+        "y": 0
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "percent",
+          "color": {
+            "mode": "thresholds"
+          },
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "orange",
+                "value": 75
+              },
+              {
+                "color": "red",
+                "value": 90
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "options": {
+        "colorMode": "background",
+        "graphMode": "none",
+        "justifyMode": "center",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "value_and_name"
+      },
+      "targets": [
+        {
+          "editorMode": "code",
+          "expr": "100 * sum(netdata_disk_space_GiB_average{host=\"serenity\",mount_point=\"/\",dimension=\"used\"}) / sum(netdata_disk_space_GiB_average{host=\"serenity\",mount_point=\"/\",dimension=~\"used|avail\"})",
+          "instant": true,
+          "legendFormat": "/",
+          "range": false,
+          "refId": "A"
+        }
+      ]
+    },
+    {
+      "type": "stat",
+      "title": "Uptime",
+      "gridPos": {
+        "h": 4,
+        "w": 4,
+        "x": 20,
+        "y": 0
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "s",
+          "color": {
+            "mode": "thresholds"
+          },
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "options": {
+        "colorMode": "background",
+        "graphMode": "none",
+        "justifyMode": "center",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "value_and_name"
+      },
+      "targets": [
+        {
+          "editorMode": "code",
+          "expr": "netdata_system_uptime_seconds_average{host=\"serenity\",dimension=\"uptime\"}",
+          "instant": true,
+          "legendFormat": "uptime",
+          "range": false,
+          "refId": "A"
+        }
+      ]
+    },
+    {
+      "type": "timeseries",
+      "title": "CPU Busy Over Time",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 4
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "percent",
+          "min": 0,
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 12,
+            "gradientMode": "opacity",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "lineInterpolation": "linear",
+            "lineWidth": 2,
+            "pointSize": 3,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "never",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          }
+        },
+        "overrides": []
+      },
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "editorMode": "code",
+          "expr": "sum(netdata_system_cpu_percentage_average{host=\"serenity\",dimension!~\"idle|iowait|steal|guest|guest_nice\"})",
+          "legendFormat": "CPU Busy",
+          "range": true,
+          "refId": "A"
+        },
+        {
+          "editorMode": "code",
+          "expr": "sum(netdata_system_cpu_percentage_average{host=\"serenity\",dimension=\"iowait\"})",
+          "legendFormat": "IO Wait",
+          "range": true,
+          "refId": "B"
+        }
+      ]
+    },
+    {
+      "type": "timeseries",
+      "title": "Load Averages",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 4
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 12,
+            "gradientMode": "opacity",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "lineInterpolation": "linear",
+            "lineWidth": 2,
+            "pointSize": 3,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "never",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          }
+        },
+        "overrides": []
+      },
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "editorMode": "code",
+          "expr": "netdata_system_load_load_average{host=\"serenity\",dimension=\"load1\"}",
+          "legendFormat": "load1",
+          "range": true,
+          "refId": "A"
+        },
+        {
+          "editorMode": "code",
+          "expr": "netdata_system_load_load_average{host=\"serenity\",dimension=\"load5\"}",
+          "legendFormat": "load5",
+          "range": true,
+          "refId": "B"
+        },
+        {
+          "editorMode": "code",
+          "expr": "netdata_system_load_load_average{host=\"serenity\",dimension=\"load15\"}",
+          "legendFormat": "load15",
+          "range": true,
+          "refId": "C"
+        }
+      ]
+    },
+    {
+      "type": "timeseries",
+      "title": "Memory Breakdown",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 12
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "decgbytes",
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 12,
+            "gradientMode": "opacity",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "lineInterpolation": "linear",
+            "lineWidth": 2,
+            "pointSize": 3,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "never",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          }
+        },
+        "overrides": []
+      },
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "editorMode": "code",
+          "expr": "netdata_system_ram_MiB_average{host=\"serenity\",dimension=\"used\"} * 1048576",
+          "legendFormat": "used",
+          "range": true,
+          "refId": "A"
+        },
+        {
+          "editorMode": "code",
+          "expr": "netdata_system_ram_MiB_average{host=\"serenity\",dimension=\"cached\"} * 1048576",
+          "legendFormat": "cached",
+          "range": true,
+          "refId": "B"
+        },
+        {
+          "editorMode": "code",
+          "expr": "netdata_system_ram_MiB_average{host=\"serenity\",dimension=\"free\"} * 1048576",
+          "legendFormat": "free",
+          "range": true,
+          "refId": "C"
+        },
+        {
+          "editorMode": "code",
+          "expr": "netdata_mem_available_MiB_average{host=\"serenity\",dimension=\"avail\"} * 1048576",
+          "legendFormat": "available",
+          "range": true,
+          "refId": "D"
+        }
+      ]
+    },
+    {
+      "type": "timeseries",
+      "title": "Network Throughput",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 12
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "binbps",
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 12,
+            "gradientMode": "opacity",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "lineInterpolation": "linear",
+            "lineWidth": 2,
+            "pointSize": 3,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "never",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          }
+        },
+        "overrides": []
+      },
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "editorMode": "code",
+          "expr": "sum by (device) (netdata_net_net_kilobits_persec_average{host=\"serenity\",interface_type=\"real\",dimension=\"received\"} * 1024)",
+          "legendFormat": "{{device}} rx",
+          "range": true,
+          "refId": "A"
+        },
+        {
+          "editorMode": "code",
+          "expr": "sum by (device) (-1 * netdata_net_net_kilobits_persec_average{host=\"serenity\",interface_type=\"real\",dimension=\"sent\"} * 1024)",
+          "legendFormat": "{{device}} tx",
+          "range": true,
+          "refId": "B"
+        }
+      ]
+    },
+    {
+      "type": "bargauge",
+      "title": "Filesystem Used %",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 20
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "percent",
+          "min": 0,
+          "max": 100,
+          "color": {
+            "mode": "thresholds"
+          },
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "orange",
+                "value": 75
+              },
+              {
+                "color": "red",
+                "value": 90
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "options": {
+        "displayMode": "gradient",
+        "orientation": "horizontal",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "showUnfilled": true,
+        "text": {}
+      },
+      "targets": [
+        {
+          "editorMode": "code",
+          "expr": "100 * (netdata_disk_space_GiB_average{host=\"serenity\",dimension=\"used\",filesystem!~\"tmpfs|proc|sysfs|overlay|squashfs|fusectl|cgroup2?|shm\"} / (netdata_disk_space_GiB_average{host=\"serenity\",dimension=\"used\",filesystem!~\"tmpfs|proc|sysfs|overlay|squashfs|fusectl|cgroup2?|shm\"} + netdata_disk_space_GiB_average{host=\"serenity\",dimension=\"avail\",filesystem!~\"tmpfs|proc|sysfs|overlay|squashfs|fusectl|cgroup2?|shm\"}))",
+          "legendFormat": "{{mount_point}}",
+          "range": true,
+          "refId": "A"
+        }
+      ]
+    },
+    {
+      "type": "timeseries",
+      "title": "Disk Utilization",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 20
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "percent",
+          "min": 0,
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 12,
+            "gradientMode": "opacity",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "lineInterpolation": "linear",
+            "lineWidth": 2,
+            "pointSize": 3,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "never",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          }
+        },
+        "overrides": []
+      },
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "editorMode": "code",
+          "expr": "netdata_disk_util___of_time_working_average{host=\"serenity\",device_type=\"physical\"}",
+          "legendFormat": "{{device}}",
+          "range": true,
+          "refId": "A"
+        }
+      ]
+    },
+    {
+      "type": "table",
+      "title": "Top Disk Busy Devices",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 28
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "percent",
+          "color": {
+            "mode": "thresholds"
+          },
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "orange",
+                "value": 75
+              },
+              {
+                "color": "red",
+                "value": 90
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "options": {
+        "cellHeight": "sm",
+        "footer": {
+          "enablePagination": true,
+          "fields": "",
+          "reducer": [
+            "sum"
+          ],
+          "show": false
+        },
+        "showHeader": true,
+        "sortBy": [
+          {
+            "desc": true,
+            "displayName": "Value"
+          }
+        ]
+      },
+      "targets": [
+        {
+          "editorMode": "code",
+          "expr": "topk(10, netdata_disk_util___of_time_working_average{host=\"serenity\",device_type=\"physical\"})",
+          "format": "table",
+          "instant": true,
+          "refId": "A"
+        }
+      ]
+    },
+    {
+      "type": "timeseries",
+      "title": "Disk IO Throughput",
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 28
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "Bps",
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 12,
+            "gradientMode": "opacity",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "lineInterpolation": "linear",
+            "lineWidth": 2,
+            "pointSize": 3,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "never",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          }
+        },
+        "overrides": []
+      },
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "editorMode": "code",
+          "expr": "sum by (device) (netdata_disk_io_KiB_persec_average{host=\"serenity\",dimension=\"reads\",device_type=\"physical\"} * 1024)",
+          "legendFormat": "{{device}} read",
+          "range": true,
+          "refId": "A"
+        },
+        {
+          "editorMode": "code",
+          "expr": "sum by (device) (-1 * netdata_disk_io_KiB_persec_average{host=\"serenity\",dimension=\"writes\",device_type=\"physical\"} * 1024)",
+          "legendFormat": "{{device}} write",
+          "range": true,
+          "refId": "B"
+        }
+      ]
+    }
+  ],
+  "refresh": "30s",
+  "schemaVersion": 41,
+  "tags": [
+    "homelab",
+    "serenity",
+    "netdata"
+  ],
+  "templating": {
+    "list": []
+  },
+  "time": {
+    "from": "now-24h",
+    "to": "now"
+  },
+  "timepicker": {},
+  "timezone": "browser",
+  "title": "Serenity Host Health",
+  "uid": "serenity-host-health",
+  "version": 1,
+  "weekStart": ""
+}

monitoring/deploy.sh

@@ -1,236 +1,41 @@
 #!/usr/bin/env bash
 set -euo pipefail
 
-# ============================================================
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-# Phase 6 — Grafana + Prometheus Deployment on PD
+cd "$SCRIPT_DIR"
-# Run this script on PlausibleDeniability as root or with sudo
-# ============================================================
 
-echo "=== Phase 6: Grafana + Prometheus ==="
+echo "=== Monitoring stack deploy ==="
-echo ""
-
-# ---- 1. Scaffold directories ----
-echo "[1/6] Scaffolding directories..."
-
-mkdir -p /mnt/docker-ssd/docker/compose/monitoring/provisioning/datasources
-mkdir -p /mnt/docker-ssd/docker/compose/monitoring/provisioning/dashboards
 
 mkdir -p /mnt/tank/docker/appdata/prometheus
-chown 65534:65534 /mnt/tank/docker/appdata/prometheus
-
 mkdir -p /mnt/tank/docker/appdata/grafana
+mkdir -p /mnt/tank/docker/appdata/grafana/dashboards
+mkdir -p /mnt/tank/docker/appdata/loki
+mkdir -p /mnt/tank/docker/appdata/promtail
+
+chown 65534:65534 /mnt/tank/docker/appdata/prometheus
 chown 472:472 /mnt/tank/docker/appdata/grafana
+chown 472:472 /mnt/tank/docker/appdata/grafana/dashboards
 
-echo "      Done."
+if [[ ! -f .env ]]; then
+  cp .env.example .env
+  gf_pass="$(openssl rand -hex 16)"
+  sed -i "s/^GF_ADMIN_PASS=CHANGE_ME$/GF_ADMIN_PASS=$gf_pass/" .env
+  chmod 600 .env
+  echo "Created .env with a random Grafana admin password."
+fi
 
-# ---- 2. Write docker-compose.yaml ----
+if compgen -G "$SCRIPT_DIR/dashboards/*.json" >/dev/null; then
-echo "[2/6] Writing docker-compose.yaml..."
+  cp "$SCRIPT_DIR"/dashboards/*.json /mnt/tank/docker/appdata/grafana/dashboards/
-
+  chown 472:472 /mnt/tank/docker/appdata/grafana/dashboards/*.json
-cat > /mnt/docker-ssd/docker/compose/monitoring/docker-compose.yaml << 'COMPOSE'
+fi
-name: monitoring
-
-services:
-  prometheus:
-    image: prom/prometheus:v3.4.0
-    container_name: prometheus
-    restart: unless-stopped
-    ports:
-      - "9090:9090"
-    command:
-      - '--config.file=/etc/prometheus/prometheus.yml'
-      - '--storage.tsdb.path=/prometheus'
-      - '--storage.tsdb.retention.time=90d'
-      - '--web.enable-lifecycle'
-    volumes:
-      - /mnt/tank/docker/appdata/prometheus:/prometheus
-      - /mnt/docker-ssd/docker/compose/monitoring/prometheus.yml:/etc/prometheus/prometheus.yml:ro
-    networks:
-      - default
-
-  grafana:
-    image: grafana/grafana:13.0.1
-    container_name: grafana
-    restart: unless-stopped
-    ports:
-      - "3000:3000"
-    environment:
-      TZ: ${TZ}
-      GF_SECURITY_ADMIN_USER: ${GF_ADMIN_USER}
-      GF_SECURITY_ADMIN_PASSWORD: ${GF_ADMIN_PASS}
-      GF_SERVER_ROOT_URL: https://${GF_HOST}/
-    volumes:
-      - /mnt/tank/docker/appdata/grafana:/var/lib/grafana
-      - /mnt/docker-ssd/docker/compose/monitoring/provisioning:/etc/grafana/provisioning:ro
-    networks:
-      - pangolin
-      - default
-
-  node-exporter:
-    image: prom/node-exporter:v1.9.0
-    container_name: node-exporter
-    restart: unless-stopped
-    ports:
-      - "9100:9100"
-    command:
-      - '--path.procfs=/host/proc'
-      - '--path.sysfs=/host/sys'
-      - '--path.rootfs=/rootfs'
-      - '--collector.filesystem.mount-points-exclude=^/(sys|proc|dev|host|etc)($$|/)'
-    volumes:
-      - /proc:/host/proc:ro
-      - /sys:/host/sys:ro
-      - /:/rootfs:ro
-    networks:
-      - default
-
-networks:
-  pangolin:
-    external: true
-COMPOSE
-
-echo "      Done."
-
-# ---- 3. Write prometheus.yml ----
-echo "[3/6] Writing prometheus.yml..."
-
-cat > /mnt/docker-ssd/docker/compose/monitoring/prometheus.yml << 'PROMCFG'
-global:
-  scrape_interval: 15s
-  evaluation_interval: 15s
-
-scrape_configs:
-  # ------- PlausibleDeniability (local) -------
-  - job_name: 'pd-node'
-    static_configs:
-      - targets: ['node-exporter:9100']
-        labels:
-          host: 'plausible-deniability'
-
-  - job_name: 'pd-netdata'
-    metrics_path: /api/v1/allmetrics
-    params:
-      format: [prometheus]
-    static_configs:
-      - targets: ['10.5.1.6:19999']
-        labels:
-          host: 'plausible-deniability'
-
-  # ------- Serenity -------
-  - job_name: 'serenity-netdata'
-    metrics_path: /api/v1/allmetrics
-    params:
-      format: [prometheus]
-    static_configs:
-      - targets: ['10.5.1.5:19999']
-        labels:
-          host: 'serenity'
-
-  # ------- N.O.M.A.D. -------
-  # Uncomment when Netdata or node-exporter is available on N.O.M.A.D.
-  # - job_name: 'nomad-netdata'
-  #   metrics_path: /api/v1/allmetrics
-  #   params:
-  #     format: [prometheus]
-  #   static_configs:
-  #     - targets: ['10.5.1.16:19999']
-  #       labels:
-  #         host: 'nomad'
-PROMCFG
-
-echo "      Done."
-
-# ---- 4. Write Grafana provisioning ----
-echo "[4/6] Writing Grafana provisioning configs..."
-
-cat > /mnt/docker-ssd/docker/compose/monitoring/provisioning/datasources/prometheus.yml << 'DATASRC'
-apiVersion: 1
-
-datasources:
-  - name: Prometheus
-    type: prometheus
-    access: proxy
-    url: http://prometheus:9090
-    isDefault: true
-    editable: true
-DATASRC
-
-cat > /mnt/docker-ssd/docker/compose/monitoring/provisioning/dashboards/dashboards.yml << 'DASHCFG'
-apiVersion: 1
-
-providers:
-  - name: 'default'
-    orgId: 1
-    folder: ''
-    type: file
-    disableDeletion: false
-    editable: true
-    options:
-      path: /var/lib/grafana/dashboards
-      foldersFromFilesStructure: false
-DASHCFG
-
-echo "      Done."
-
-# ---- 5. Generate .env ----
-echo "[5/6] Generating .env..."
-
-GF_PASS=$(openssl rand -hex 16)
-
-cat > /mnt/docker-ssd/docker/compose/monitoring/.env << ENV
-TZ=America/New_York
-
-# Grafana
-GF_ADMIN_USER=admin
-GF_ADMIN_PASS=${GF_PASS}
-GF_HOST=grafana.paccoco.com
-ENV
-
-echo "      Done."
-echo ""
-echo "      Grafana admin password: ${GF_PASS}"
-echo "      (saved in /mnt/docker-ssd/docker/compose/monitoring/.env)"
-echo ""
-
-# ---- 6. Validate and deploy ----
-echo "[6/6] Validating and deploying..."
-
-cd /mnt/docker-ssd/docker/compose/monitoring
-docker compose --env-file .env config > /dev/null 2>&1
-echo "      Compose config valid."
 
+docker compose --env-file .env config >/dev/null
 docker compose --env-file .env up -d
-echo ""
-echo "=== Deployment complete ==="
-echo ""
-echo "Waiting 10 seconds for containers to start..."
-sleep 10
 
-# ---- Post-deploy checks ----
+echo
-echo ""
+docker ps --filter name=prometheus --filter name=grafana --filter name=loki --filter name=promtail --filter name=cadvisor --format "table {{.Names}}\t{{.Status}}\t{{.Ports}}"
-echo "=== Post-Deploy Verification ==="
+echo
-echo ""
+curl -fsS http://127.0.0.1:9090/-/ready >/dev/null && echo "Prometheus ready"
-
+[[ "$(docker inspect -f '{{.State.Running}}' loki 2>/dev/null)" == "true" ]] && echo "Loki running"
-echo "--- Container status ---"
+curl -fsS http://127.0.0.1:9100/metrics >/dev/null && echo "node-exporter ready"
-docker ps --filter name=prometheus --filter name=grafana --filter name=node-exporter --format "table {{.Names}}\t{{.Status}}\t{{.Ports}}"
+docker exec cadvisor wget -qO- http://127.0.0.1:8080/metrics >/dev/null 2>&1 && echo "cAdvisor ready" || true
-echo ""
-
-echo "--- Prometheus targets ---"
-curl -s http://10.5.1.6:9090/api/v1/targets 2>/dev/null | python3 -m json.tool 2>/dev/null | head -30 || echo "Prometheus not yet responding (may need a moment)"
-echo ""
-
-echo "--- Grafana health ---"
-HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" http://10.5.1.6:3000/ 2>/dev/null || echo "000")
-echo "Grafana HTTP status: ${HTTP_CODE}"
-echo ""
-
-echo "--- Node exporter ---"
-curl -s http://10.5.1.6:9100/metrics 2>/dev/null | head -5 || echo "Node exporter not yet responding"
-echo ""
-
-echo "=== All done! ==="
-echo ""
-echo "Next steps:"
-echo "  1. Add Pangolin resource: grafana.paccoco.com -> http://grafana:3000"
-echo "  2. Log in at https://grafana.paccoco.com with admin / ${GF_PASS}"
-echo "  3. Import dashboards: Node Exporter Full (ID 1860), Docker Stats (ID 893)"
-echo "  4. Set up Grafana -> n8n alert webhook at https://n8n.paccoco.com/webhook/grafana-alert"

monitoring/docker-compose.yaml

@@ -15,11 +15,24 @@ services:
     volumes:
       - /mnt/tank/docker/appdata/prometheus:/prometheus
       - /mnt/docker-ssd/docker/compose/monitoring/prometheus.yml:/etc/prometheus/prometheus.yml:ro
+      - /mnt/docker-ssd/docker/compose/monitoring/alerts.yml:/etc/prometheus/alerts.yml:ro
+    networks:
+      - default
+
+  loki:
+    image: grafana/loki:latest
+    container_name: loki
+    restart: unless-stopped
+    command:
+      - '-config.file=/etc/loki/config.yml'
+    volumes:
+      - /mnt/tank/docker/appdata/loki:/loki
+      - /mnt/docker-ssd/docker/compose/monitoring/loki-config.yml:/etc/loki/config.yml:ro
     networks:
       - default
 
   grafana:
-    image: grafana/grafana:13.0.1
+    image: grafana/grafana:latest
     container_name: grafana
     restart: unless-stopped
     ports:
@@ -29,13 +42,46 @@ services:
       GF_SECURITY_ADMIN_USER: ${GF_ADMIN_USER}
       GF_SECURITY_ADMIN_PASSWORD: ${GF_ADMIN_PASS}
       GF_SERVER_ROOT_URL: https://${GF_HOST}/
+      GF_AUTH_GENERIC_OAUTH_ENABLED: ${GF_AUTH_GENERIC_OAUTH_ENABLED}
+      GF_AUTH_GENERIC_OAUTH_NAME: ${GF_AUTH_GENERIC_OAUTH_NAME}
+      GF_AUTH_GENERIC_OAUTH_ICON: ${GF_AUTH_GENERIC_OAUTH_ICON}
+      GF_AUTH_GENERIC_OAUTH_CLIENT_ID: ${GF_AUTH_GENERIC_OAUTH_CLIENT_ID}
+      GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET: ${GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET}
+      GF_AUTH_GENERIC_OAUTH_SCOPES: ${GF_AUTH_GENERIC_OAUTH_SCOPES}
+      GF_AUTH_GENERIC_OAUTH_EMPTY_SCOPES: ${GF_AUTH_GENERIC_OAUTH_EMPTY_SCOPES}
+      GF_AUTH_GENERIC_OAUTH_AUTH_URL: https://${AUTHELIA_HOST}/api/oidc/authorization
+      GF_AUTH_GENERIC_OAUTH_TOKEN_URL: https://${AUTHELIA_HOST}/api/oidc/token
+      GF_AUTH_GENERIC_OAUTH_API_URL: https://${AUTHELIA_HOST}/api/oidc/userinfo
+      GF_AUTH_GENERIC_OAUTH_USE_PKCE: ${GF_AUTH_GENERIC_OAUTH_USE_PKCE}
+      GF_AUTH_GENERIC_OAUTH_ALLOW_SIGN_UP: ${GF_AUTH_GENERIC_OAUTH_ALLOW_SIGN_UP}
+      GF_AUTH_GENERIC_OAUTH_AUTO_LOGIN: ${GF_AUTH_GENERIC_OAUTH_AUTO_LOGIN}
+      GF_AUTH_GENERIC_OAUTH_LOGIN_ATTRIBUTE_PATH: ${GF_AUTH_GENERIC_OAUTH_LOGIN_ATTRIBUTE_PATH}
+      GF_AUTH_GENERIC_OAUTH_NAME_ATTRIBUTE_PATH: ${GF_AUTH_GENERIC_OAUTH_NAME_ATTRIBUTE_PATH}
+      GF_AUTH_GENERIC_OAUTH_EMAIL_ATTRIBUTE_PATH: ${GF_AUTH_GENERIC_OAUTH_EMAIL_ATTRIBUTE_PATH}
+      GF_AUTH_GENERIC_OAUTH_ROLE_ATTRIBUTE_PATH: ${GF_AUTH_GENERIC_OAUTH_ROLE_ATTRIBUTE_PATH}
+      GF_AUTH_GENERIC_OAUTH_ALLOW_ASSIGN_GRAFANA_ADMIN: ${GF_AUTH_GENERIC_OAUTH_ALLOW_ASSIGN_GRAFANA_ADMIN}
+      GF_AUTH_DISABLE_LOGIN_FORM: ${GF_AUTH_DISABLE_LOGIN_FORM}
     volumes:
       - /mnt/tank/docker/appdata/grafana:/var/lib/grafana
+      - /mnt/docker-ssd/docker/compose/monitoring/dashboards:/var/lib/grafana/dashboards:ro
       - /mnt/docker-ssd/docker/compose/monitoring/provisioning:/etc/grafana/provisioning:ro
     networks:
       - pangolin
       - default
 
+  promtail:
+    image: grafana/promtail:latest
+    container_name: promtail
+    restart: unless-stopped
+    command:
+      - '-config.file=/etc/promtail/config.yml'
+    volumes:
+      - /mnt/tank/docker/appdata/promtail:/var/lib/promtail
+      - /mnt/docker-ssd/docker/compose/monitoring/promtail-config.yml:/etc/promtail/config.yml:ro
+      - /var/run/docker.sock:/var/run/docker.sock:ro
+    networks:
+      - default
+
   node-exporter:
     image: prom/node-exporter:v1.9.0
     container_name: node-exporter
@@ -47,10 +93,29 @@ services:
       - '--path.sysfs=/host/sys'
       - '--path.rootfs=/rootfs'
       - '--collector.filesystem.mount-points-exclude=^/(sys|proc|dev|host|etc)($$|/)'
+      - '--collector.textfile.directory=/var/lib/node_exporter/textfile_collector'
     volumes:
       - /proc:/host/proc:ro
       - /sys:/host/sys:ro
       - /:/rootfs:ro
+      - /mnt/tank/docker/metrics/node-exporter:/var/lib/node_exporter/textfile_collector
+    networks:
+      - default
+
+  cadvisor:
+    image: gcr.io/cadvisor/cadvisor:latest
+    container_name: cadvisor
+    restart: unless-stopped
+    privileged: true
+    command:
+      - '--docker_only=true'
+      - '--housekeeping_interval=30s'
+    volumes:
+      - /:/rootfs:ro
+      - /var/run:/var/run:rw
+      - /sys:/sys:ro
+      - /var/lib/docker:/var/lib/docker:ro
+      - /dev/disk:/dev/disk:ro
     networks:
       - default
 

monitoring/loki-config.yml

@@ -0,0 +1,38 @@
+auth_enabled: false
+
+server:
+  http_listen_port: 3100
+
+common:
+  path_prefix: /loki
+  replication_factor: 1
+  ring:
+    kvstore:
+      store: inmemory
+  storage:
+    filesystem:
+      chunks_directory: /loki/chunks
+      rules_directory: /loki/rules
+
+schema_config:
+  configs:
+    - from: 2024-01-01
+      store: tsdb
+      object_store: filesystem
+      schema: v13
+      index:
+        prefix: index_
+        period: 24h
+
+limits_config:
+  retention_period: 30d
+  allow_structured_metadata: true
+  volume_enabled: true
+
+compactor:
+  working_directory: /loki/compactor
+  retention_enabled: true
+  delete_request_store: filesystem
+
+analytics:
+  reporting_enabled: false

monitoring/prometheus.yml

@@ -2,6 +2,9 @@ global:
   scrape_interval: 15s
   evaluation_interval: 15s
 
+rule_files:
+  - /etc/prometheus/alerts.yml
+
 scrape_configs:
   # ------- PlausibleDeniability (local via node-exporter) -------
   - job_name: 'pd-node'
@@ -10,6 +13,12 @@ scrape_configs:
         labels:
           host: 'plausible-deniability'
 
+  - job_name: 'pd-cadvisor'
+    static_configs:
+      - targets: ['cadvisor:8080']
+        labels:
+          host: 'plausible-deniability'
+
   # ------- Serenity (via Netdata Prometheus exporter) -------
   - job_name: 'serenity-netdata'
     metrics_path: /api/v1/allmetrics

monitoring/promtail-config.yml

@@ -0,0 +1,27 @@
+server:
+  http_listen_port: 9080
+  grpc_listen_port: 0
+
+positions:
+  filename: /var/lib/promtail/positions.yml
+
+clients:
+  - url: http://loki:3100/loki/api/v1/push
+
+scrape_configs:
+  - job_name: docker
+    docker_sd_configs:
+      - host: unix:///var/run/docker.sock
+        refresh_interval: 15s
+    relabel_configs:
+      - source_labels: ['__meta_docker_container_name']
+        regex: '/(.*)'
+        target_label: container
+      - source_labels: ['__meta_docker_container_label_com_docker_compose_project']
+        target_label: compose_project
+      - source_labels: ['__meta_docker_container_label_com_docker_compose_service']
+        target_label: compose_service
+      - source_labels: ['__meta_docker_container_log_stream']
+        target_label: stream
+      - replacement: plausible-deniability
+        target_label: host

monitoring/provisioning/datasources/prometheus.yml

@@ -7,3 +7,8 @@ datasources:
     url: http://prometheus:9090
     isDefault: true
     editable: true
+  - name: Loki
+    type: loki
+    access: proxy
+    url: http://loki:3100
+    editable: true