diff --git a/HOMELAB_BUILDOUT_PLAN.md b/HOMELAB_BUILDOUT_PLAN.md index a3e77f5..f80c202 100644 --- a/HOMELAB_BUILDOUT_PLAN.md +++ b/HOMELAB_BUILDOUT_PLAN.md @@ -1239,7 +1239,7 @@ networks: > **Image notes:** > - `prom/prometheus:v3.4.0` — Prometheus v3 is the current major line. **Do not use `:latest`** — it still resolves to 2.x due to a known tagging issue. Always use an explicit `v3.x.y` tag. -> - `grafana/grafana:13.0.1` — Grafana 13 is the current stable. +> - `grafana/grafana:latest` — use the moving stable image by default here; only pin Grafana if a breakage or compatibility reason is documented. > - `prom/node-exporter:v1.9.0` — check [releases](https://github.com/prometheus/node_exporter/releases). ### 6.5 — .env.example diff --git a/automation/.env.example b/automation/.env.example index 9cb5401..df591c5 100644 --- a/automation/.env.example +++ b/automation/.env.example @@ -35,3 +35,10 @@ BACKUP_KNOWN_HOSTS=/home/truenas_admin/.ssh/known_hosts DOCKER_BIN=/usr/bin/docker SUDO_BIN=/usr/bin/sudo USE_SUDO_FOR_DOCKER=true + +# Restore verification +RESTORE_VERIFY_WORK_ROOT=/mnt/tank/docker/backups/restore-verify +RESTORE_VERIFY_POSTGRES_IMAGE=postgres:17-alpine +RESTORE_VERIFY_CONFIG_SOURCE=tank-docker/appdata/grafana +RESTORE_VERIFY_REQUIRED_PATHS="grafana.db dashboards" +RESTORE_VERIFY_METRICS_DIR=/mnt/tank/docker/metrics/node-exporter diff --git a/automation/README.md b/automation/README.md index 5a906eb..85f41f1 100644 --- a/automation/README.md +++ b/automation/README.md @@ -33,6 +33,10 @@ automation/bin/karakeep_api.py /api/health --raw-path --pretty - `bin/pd_backup_postgres.sh` — creates gzipped `pg_dump` exports for the configured Postgres DB list (quote space-separated DB names in `.env`, e.g. `POSTGRES_DB_LIST="n8n paperless"`) - `bin/pd_backup_sync_to_serenity.sh` — `rsync`s configured backup/config roots to Serenity - `bin/run_pd_backups.sh` — wrapper that runs both steps in order +- `bin/pd_restore_verify_postgres.sh` — restores the latest dump for each configured database into a throwaway Postgres container and fails on SQL errors +- `bin/pd_restore_verify_appdata.sh` — stages one appdata/config tree back from Serenity and verifies required files exist +- `bin/run_pd_restore_verification.sh` — wrapper that runs both restore checks in order + - writes Prometheus textfile metrics for run status, duration, and last successful verification timestamp These scripts are the live PD backup runner model. @@ -49,6 +53,7 @@ Current documented live state: - the PD → Serenity backup flow is the intended deployed model - root cron on PD is the preferred scheduler - first-run verification and the deployed status are tracked in `docs/operations/PD_BACKUP_DEPLOYMENT.md` and `docs/planning/TODO.md` +- quarterly restore verification is now expected via `bin/run_pd_restore_verification.sh` ## PD / TrueNAS deployment recommendation @@ -93,6 +98,34 @@ Install in **root's crontab on PD**: # END PD BACKUPS ``` +## Recommended quarterly restore verification + +Run from PD after a backup has completed: + +```bash +cd /mnt/docker-ssd/docker/compose/automation +/usr/bin/bash bin/run_pd_restore_verification.sh +``` + +This validates: + +- latest configured Postgres dumps can be restored cleanly +- a real config/appdata sample can be pulled back from Serenity +- expected files still exist in the restored copy + +If node-exporter is configured with the textfile collector, the wrapper also publishes: + +- `pd_restore_verification_success` +- `pd_restore_verification_last_run_timestamp_seconds` +- `pd_restore_verification_last_success_timestamp_seconds` +- `pd_restore_verification_duration_seconds` + +Recommended root cron entry on PD: + +```cron +30 3 1 */3 * cd /mnt/docker-ssd/docker/compose/automation && /usr/bin/bash bin/run_pd_restore_verification.sh >> /mnt/tank/docker/backups/pd-restore-verify.log 2>&1 +``` + Recommended first-run checklist: ```bash diff --git a/automation/bin/pd_backup_lib.sh b/automation/bin/pd_backup_lib.sh index cc3192c..cfa0c7d 100755 --- a/automation/bin/pd_backup_lib.sh +++ b/automation/bin/pd_backup_lib.sh @@ -71,3 +71,8 @@ ssh_args() { ssh_cmd() { printf 'ssh %s' "$(ssh_args)" } + +latest_dump_for_db() { + local db="$1" + find "$PD_DB_DUMP_ROOT" -maxdepth 1 -type f -name "${db}_*.sql.gz" | sort | tail -n 1 +} diff --git a/automation/bin/pd_restore_verify_appdata.sh b/automation/bin/pd_restore_verify_appdata.sh new file mode 100755 index 0000000..c8d42db --- /dev/null +++ b/automation/bin/pd_restore_verify_appdata.sh @@ -0,0 +1,34 @@ +#!/usr/bin/bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +# shellcheck disable=SC1091 +source "$SCRIPT_DIR/pd_backup_lib.sh" + +verify_root="${RESTORE_VERIFY_WORK_ROOT:-/mnt/tank/docker/backups/restore-verify}" +remote_source="${RESTORE_VERIFY_CONFIG_SOURCE:-tank-docker/appdata/grafana}" +required_paths="${RESTORE_VERIFY_REQUIRED_PATHS:-grafana.db dashboards}" +stage_dir="$verify_root/appdata-stage" + +mkdir -p "$verify_root" +rm -rf "$stage_dir" +mkdir -p "$stage_dir" + +remote_path="$SERENITY_BACKUP_ROOT/$remote_source/" + +echo "Staging config restore sample from $SERENITY_BACKUP_HOST:$remote_path" +rsync -a -e "$(ssh_cmd)" "$SERENITY_BACKUP_HOST:$remote_path" "$stage_dir/" + +for rel in $required_paths; do + if [[ ! -e "$stage_dir/$rel" ]]; then + echo "Missing expected restored path: $rel" >&2 + exit 1 + fi +done + +if [[ -f "$stage_dir/grafana.db" ]] && command -v sqlite3 >/dev/null 2>&1; then + sqlite3 "$stage_dir/grafana.db" 'PRAGMA integrity_check;' | grep -qx 'ok' + echo "Verified SQLite integrity for grafana.db" +fi + +echo "Config restore verification passed for $remote_source." diff --git a/automation/bin/pd_restore_verify_postgres.sh b/automation/bin/pd_restore_verify_postgres.sh new file mode 100755 index 0000000..2193a85 --- /dev/null +++ b/automation/bin/pd_restore_verify_postgres.sh @@ -0,0 +1,69 @@ +#!/usr/bin/bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +# shellcheck disable=SC1091 +source "$SCRIPT_DIR/pd_backup_lib.sh" + +verify_root="${RESTORE_VERIFY_WORK_ROOT:-/mnt/tank/docker/backups/restore-verify}" +dbs="${RESTORE_VERIFY_DB_LIST:-${POSTGRES_DB_LIST:-}}" + +if [[ -z "$dbs" ]]; then + echo "RESTORE_VERIFY_DB_LIST and POSTGRES_DB_LIST are both empty." >&2 + exit 1 +fi + +mkdir -p "$verify_root" +stamp="$(timestamp_utc)" +container="pd-restore-verify-$stamp" +password="verify-$stamp" + +if [[ -n "${RESTORE_VERIFY_POSTGRES_IMAGE:-}" ]]; then + verify_image="$RESTORE_VERIFY_POSTGRES_IMAGE" +else + source_major="$(docker_cmd exec "$POSTGRES_CONTAINER" postgres -V | awk '{print $3}' | cut -d. -f1)" + verify_image="postgres:${source_major}-alpine" +fi + +cleanup() { + docker_cmd rm -f "$container" >/dev/null 2>&1 || true +} +trap cleanup EXIT + +echo "Starting temporary Postgres verifier container $container" +docker_cmd run -d --rm \ + --name "$container" \ + -e POSTGRES_PASSWORD="$password" \ + "$verify_image" >/dev/null + +echo "Waiting for temporary Postgres to accept connections" +for _ in {1..30}; do + if docker_cmd exec "$container" pg_isready -U postgres >/dev/null 2>&1; then + break + fi + sleep 2 +done + +docker_cmd exec "$container" pg_isready -U postgres >/dev/null + +for db in $dbs; do + dump_file="$(latest_dump_for_db "$db")" + if [[ -z "$dump_file" ]]; then + echo "No dump found for database '$db' under $PD_DB_DUMP_ROOT" >&2 + exit 1 + fi + + echo "Restoring $dump_file into temporary database '$db'" + while IFS= read -r role_name; do + [[ -z "$role_name" ]] && continue + docker_cmd exec "$container" createuser -U postgres "$role_name" >/dev/null 2>&1 || true + done < <(gunzip -c "$dump_file" | sed -n "s/.*OWNER TO \\([^;]*\\);/\\1/p" | sort -u) + + docker_cmd exec "$container" createdb -U postgres "$db" + gunzip -c "$dump_file" | docker_cmd exec -i "$container" psql -v ON_ERROR_STOP=1 -U postgres -d "$db" >/dev/null + + table_count="$(docker_cmd exec "$container" psql -At -U postgres -d "$db" -c "SELECT count(*) FROM information_schema.tables WHERE table_schema NOT IN ('pg_catalog','information_schema');")" + echo "Verified database '$db' with $table_count non-system tables." +done + +echo "Postgres restore verification passed." diff --git a/automation/bin/run_pd_restore_verification.sh b/automation/bin/run_pd_restore_verification.sh new file mode 100755 index 0000000..3ccaab2 --- /dev/null +++ b/automation/bin/run_pd_restore_verification.sh @@ -0,0 +1,55 @@ +#!/usr/bin/bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +# shellcheck disable=SC1091 +source "$SCRIPT_DIR/pd_backup_lib.sh" + +metrics_dir="${RESTORE_VERIFY_METRICS_DIR:-/mnt/tank/docker/metrics/node-exporter}" +metrics_file="$metrics_dir/pd_restore_verification.prom" +start_epoch="$(date +%s)" +status=1 + +mkdir -p "$metrics_dir" + +previous_success_epoch() { + if [[ -f "$metrics_file" ]]; then + awk '/^pd_restore_verification_last_success_timestamp_seconds /{print $2}' "$metrics_file" | tail -n 1 + else + echo 0 + fi +} + +write_metrics() { + local end_epoch duration last_success tmp + end_epoch="$(date +%s)" + duration="$((end_epoch - start_epoch))" + last_success="$(previous_success_epoch)" + if [[ "$status" -eq 0 ]]; then + last_success="$end_epoch" + fi + tmp="${metrics_file}.tmp" + cat > "$tmp" </callback` +- validate the exact callback path against the live Gitea UI before committing the Authelia client entry + +## Gitea side + +In Gitea: + +1. Sign in as a local admin. +2. Go to `Site Administration` -> `Authentication Sources`. +3. Add a new source of type `OpenID Connect`. +4. Use Authelia discovery if the form supports it, otherwise enter the manual endpoints from `https://auth.paccoco.com/.well-known/openid-configuration`. + +Recommended source values: + +- name: `Authelia` +- scopes: `openid profile email groups` +- required claim mapping priority: email first, then username/login +- keep the source enabled only after confirming the callback URL matches the Authelia client entry + +## Rollout notes + +- keep local admin auth available for the first pass +- test with one admin account first +- after successful sign-in, decide whether to keep local auth as break-glass only +- do not disable Pangolin auth in front of Gitea until Authelia login is actually working end-to-end + +## Related weak spot + +The live Gitea app.ini observed on PD had public self-registration enabled. That should stay off for this homelab unless there is a deliberate reason to reopen it. diff --git a/docs/operations/IDENTITY_AND_SSO.md b/docs/operations/IDENTITY_AND_SSO.md new file mode 100644 index 0000000..cce1222 --- /dev/null +++ b/docs/operations/IDENTITY_AND_SSO.md @@ -0,0 +1,60 @@ +# Identity and SSO + +Authelia is the staged identity provider for the homelab. + +## Why Authelia + +- centralizes auth for the growing pile of web apps +- works with shared Postgres/Redis already in the lab +- supports OIDC for apps that speak it properly +- avoids inventing separate local auth for every exposed service + +## Current scope + +- stack path: identity/ +- host target: PlausibleDeniability +- public entrypoint: auth.paccoco.com +- status: repo-staged, not assumed live until deployed on PD + +## Initial bootstrap flow + +1. Copy identity/.env.example to identity/.env and generate fresh secrets. +2. Copy identity/authelia/configuration.yml.example to identity/authelia/configuration.yml. +3. Render concrete secrets and passwords into identity/authelia/configuration.yml on PD; do not rely on runtime env interpolation inside the YAML. +4. Copy identity/authelia/users_database.example.yml to identity/authelia/users_database.yml. +5. Generate an Argon2 password hash with the Authelia image and replace the sample hash. +6. Create the authelia Postgres database and user in the shared database stack. +7. Set `AUTHELIA_SESSION_REDIS_PASSWORD` from the live shared Redis stack on PD. +8. Add a Pangolin resource for auth.paccoco.com pointing at container authelia:9091. +9. Validate with docker compose --env-file .env config and docker exec authelia authelia config validate --config /config/configuration.yml. + +## Rollout order + +This repo stage is the control-plane bootstrap, not the full client cutover. + +After Authelia is healthy on PD, wire OIDC or forward-auth clients one at a time. + +Recommended first targets: + +- Grafana +- Gitea +- any new public app before it gains yet another standalone login + +## Grafana first-cutover notes + +- Use Authelia OIDC as a Generic OAuth provider in Grafana. +- Keep Grafana's local admin login enabled for the first rollout. +- After OIDC is verified, disable Pangolin auth in front of Grafana to avoid double-auth. + +## Gitea prep notes + +- The live Gitea stack is the `dev/` compose on PD. +- Disable public self-registration before or during the SSO prep pass. +- Keep local admin login working for the first OIDC test. +- Stage the exact Gitea auth-source values in `docs/operations/GITEA_SSO_PREP.md` before creating the live source. + +## Notes + +- File-backed users are intentional for the initial rollout; this is enough for the current household scale. +- If identity sprawl grows later, migrate the authentication backend to LDAP instead of bolting extra auth stores onto apps ad hoc. +- OIDC signing keys and per-app client secrets should be generated during deployment on PD, not committed to the repo. diff --git a/docs/operations/PD_BACKUP_DEPLOYMENT.md b/docs/operations/PD_BACKUP_DEPLOYMENT.md index 6a982bd..0d27737 100644 --- a/docs/operations/PD_BACKUP_DEPLOYMENT.md +++ b/docs/operations/PD_BACKUP_DEPLOYMENT.md @@ -68,6 +68,12 @@ Preferred: install in **root's crontab** on PD. # END PD BACKUPS ``` +Quarterly restore verification: + +```cron +30 3 1 */3 * cd /mnt/docker-ssd/docker/compose/automation && /usr/bin/bash bin/run_pd_restore_verification.sh >> /mnt/tank/docker/backups/pd-restore-verify.log 2>&1 +``` + ## Post-deploy checks ```bash @@ -76,6 +82,22 @@ ls -lh /mnt/tank/docker/backups/db-dumps ssh root@10.5.1.5 'find /mnt/user/backups/plausible-deniability -maxdepth 2 -type d | sort | head -40' ``` +## Restore verification + +After the backup path is stable, run the staged restore verifier from PD: + +```bash +cd /mnt/docker-ssd/docker/compose/automation +/usr/bin/bash bin/run_pd_restore_verification.sh +``` + +What it proves: + +- the latest configured Postgres dumps can restore into a temporary container +- a real appdata/config sample can be staged back from Serenity +- expected files still exist inside the restored sample +- the latest run state can be exported to Prometheus via node-exporter's textfile collector + ## Important limitations - Do **not** use `systemctl restart docker` on PD. diff --git a/docs/reference/MESHTASTIC.md b/docs/reference/MESHTASTIC.md index 8cc783e..ed11fe6 100644 --- a/docs/reference/MESHTASTIC.md +++ b/docs/reference/MESHTASTIC.md @@ -3,6 +3,8 @@ ## Overview Self-hosted Meshtastic monitoring and management stack running on PlausibleDeniability. +Related NOMAD-side bridge note: a separate MeshCore-to-MQTT relay is running on N.O.M.A.D. as Docker container `mctomqtt` (`mctomqtt:latest`), with config bind-mounted from `/etc/mctomqtt`. The package also includes a systemd unit under `/opt/mctomqtt`, but that is not the active runtime on this host. + ## Stack Location `/mnt/docker-ssd/docker/compose/meshtastic/` diff --git a/docs/servers/NOMAD.md b/docs/servers/NOMAD.md index e75d6fd..2d8a370 100644 --- a/docs/servers/NOMAD.md +++ b/docs/servers/NOMAD.md @@ -199,6 +199,46 @@ These are expected to be managed by Wings/Pelican rather than a local compose fi - Topic thumbs write back to `doris-digest/data/feedback.json`. - Paperless acknowledge/dismiss state writes to `/opt/doris-dashboard/data/paperless_review_state.json` on the live host. +### MeshCore to MQTT relay +- Install date: 2026-05-18 +- Install root: `/opt/mctomqtt` +- Config root: `/etc/mctomqtt` +- Service account: `mctomqtt:mctomqtt` +- Installer metadata: `.version_info` reports `installer_version: 1.3.0.0-preview`, repo `Cisien/meshcoretomqtt`, branch `main`, commit `10f7c0e` +- Detected install type marker: `/opt/mctomqtt/.install_type` contains `docker` +- Runtime model: + - Active runtime is a standalone Docker container named `mctomqtt` + - Image: `mctomqtt:latest` + - Command: `python3 /opt/mctomqtt/mctomqtt.py` + - User: `mctomqtt` + - Restart policy: `unless-stopped` + - Network mode: `bridge` + - Config bind mount: `/etc/mctomqtt:/etc/mctomqtt:ro` + - Device passthrough: the Heltec serial device is mapped directly into the container at the same `/dev/serial/by-id/... ` path +- Systemd note: + - The packaged repo also includes `/opt/mctomqtt/mctomqtt.service`, but it is not the active runtime on NOMAD + - No installed unit was present in `/etc/systemd/system` or `/lib/systemd/system` + - The bundled unit expects `/opt/mctomqtt/venv/bin/python3`, but no `venv/` directory exists under `/opt/mctomqtt` + - Treat the systemd unit as an unused package artifact for this host unless the deployment model changes later +- Config layering: + - Base reference: `/etc/mctomqtt/config.toml` + - Preset overlay: `/etc/mctomqtt/config.d/10-meshmapper.toml` + - Local overrides: `/etc/mctomqtt/config.d/99-user.toml` +- Current local overrides observed: + - IATA region code set to `BNA` + - Serial device pinned to `/dev/serial/by-id/usb-Espressif_Systems_heltec_wifi_lora_32_v4__16_MB_FLASH__2_MB_PSRAM__441BF670B684-if00` + - Enabled broker targets include MeshMapper, LetsMesh US, and NashMe + - Credentials/secrets are stored inline in `99-user.toml`; do not mirror them into git-backed docs +- Container health observed during inspection: + - Logs show the relay loading all three config layers, opening the Heltec serial device, and connecting successfully to MeshMapper, NashMe, and LetsMesh US after restart + - The current container had been up about 5 minutes during inspection with `MQTT: 3/3`, zero failures, and live RX air-time reported + - An earlier start logged one DNS resolution failure for a broker label shown as `custom-2`, but the current running container recovered and all three named brokers connected cleanly +- Container logs / inspection: + - Use `docker logs mctomqtt` for runtime output + - Use `docker inspect mctomqtt` for bind mounts, device mappings, and restart policy +- Documentation linkage: + - Related mesh stack notes live in [MESHTASTIC.md](../reference/MESHTASTIC.md) + ### Live deployment path rule - On NOMAD, live service roots belong under `/opt/`. - That includes host-run services **and** standalone Docker Compose deployments that are not part of the core Project N.O.M.A.D. stack. diff --git a/docs/servers/PLAUSIBLEDENABILITY.md b/docs/servers/PLAUSIBLEDENABILITY.md index d600e8a..3a26a36 100644 --- a/docs/servers/PLAUSIBLEDENABILITY.md +++ b/docs/servers/PLAUSIBLEDENABILITY.md @@ -31,18 +31,24 @@ Primary Docker host running all production workloads. RTX 2080 Ti used by: - Plex (hardware transcoding) - immich-ml (CUDA face recognition) -- Ollama light-tier inference (qwen2.5:14b and smaller) +- Ollama light-tier inference for the main AI stack and local Honcho backend > **Note:** Whisper moved to N.O.M.A.D. (10.5.1.16:8786) as CPU inference — avoids VRAM conflict with Ollama. See [NOMAD.md](NOMAD.md). Recommended models for 11GB VRAM: | Model | VRAM | Best for | |-------|------|---------| -| `qwen2.5:14b` | ~8GB | General use, coding, reasoning | +| `qwen2.5:7b` | ~5GB | Cheap/default reasoning, summaries, derivation | +| `qwen2.5:14b` | ~8GB | Heavier reasoning / high-max paths | | `qwen2.5-coder:7b` | ~5GB | Code tasks | | `deepseek-r1:8b` | ~5GB | Math, logic | | `phi4` | ~8GB | Analytical tasks | +Current light-tier notes: +- PD Ollama is the active light-tier endpoint for LiteLLM on `http://10.5.1.6:11434` +- Local Honcho currently points at PD Ollama over LAN (`http://10.5.1.6:11434/v1`) instead of using a separate local model host +- Current Honcho routing split keeps embeddings on `nomic-embed-text:latest`, cheap/default paths on `qwen2.5:7b`, and deeper high/max paths on `qwen2.5:14b` + ## Remote Access - Pangolin VPS + Newt for external services - SSH from Windows 11 diff --git a/home/doris-dashboard/.gitignore b/home/doris-dashboard/.gitignore index b838a2e..0fde4f2 100644 --- a/home/doris-dashboard/.gitignore +++ b/home/doris-dashboard/.gitignore @@ -11,3 +11,4 @@ data/todos.json data/weather.json data/paperless_review.json data/paperless_review_state.json +data/hermes_usage.json diff --git a/home/doris-dashboard/bin/generate_dashboard.py b/home/doris-dashboard/bin/generate_dashboard.py index 87c4f81..dc5de62 100755 --- a/home/doris-dashboard/bin/generate_dashboard.py +++ b/home/doris-dashboard/bin/generate_dashboard.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 from __future__ import annotations -import html, json, math, os, re, shutil, subprocess, urllib.error, urllib.request +import html, json, math, os, re, shutil, sqlite3, subprocess, time, urllib.error, urllib.request, yaml from collections import Counter from datetime import datetime, timezone from pathlib import Path @@ -15,6 +15,9 @@ PUBLIC=ROOT/'public'; DATA=ROOT/'data'; LOGS=ROOT/'logs' PAPERLESS_STATE_PATH=DATA/'paperless_review_state.json' LAT=36.53; LON=-87.36 LOCAL_TZ=ZoneInfo('America/Chicago') +HERMES_HOME=Path.home()/'.hermes' +HERMES_STATE_DB=HERMES_HOME/'state.db' +HERMES_CONFIG=HERMES_HOME/'config.yaml' LABELS={ 'astronomy':('🔭','Astronomy'),'homelab':('🏠','Homelab'),'smart_home':('🏡','Smart Home'), @@ -50,6 +53,37 @@ def kmh_to_mph(k:float|int|None)->str: if k is None: return '—' return f"{round(k*0.621371)} mph" +def compact_number(value:int|float|None)->str: + if value in (None,''): return '—' + try: n=float(value) + except Exception: return str(value) + abs_n=abs(n) + if abs_n >= 1_000_000_000: return f"{n/1_000_000_000:.1f}B" + if abs_n >= 1_000_000: return f"{n/1_000_000:.1f}M" + if abs_n >= 1_000: return f"{n/1_000:.1f}K" + if n.is_integer(): return f"{int(n):,}" + return f"{n:,.1f}" + +def total_tokens_value(row:dict[str,Any] | sqlite3.Row)->int: + keys=('input_tokens','output_tokens','cache_read_tokens','cache_write_tokens','reasoning_tokens') + total=0 + for key in keys: + try: total += int((row[key] if isinstance(row, sqlite3.Row) else row.get(key,0)) or 0) + except Exception: pass + return total + +def read_hermes_model_config()->dict[str,str]: + default='unknown'; provider='unknown' + try: + data=yaml.safe_load(HERMES_CONFIG.read_text(encoding='utf-8')) or {} + except Exception: + return {'default_model':default,'provider':provider} + model_cfg=data.get('model') if isinstance(data,dict) else {} + if isinstance(model_cfg,dict): + default=str(model_cfg.get('default') or default) + provider=str(model_cfg.get('provider') or provider) + return {'default_model':default or 'unknown','provider':provider or 'unknown'} + def moon_phase(dt:datetime)->dict[str,str]: # Known new moon: 2000-01-06 18:14 UTC; synodic month 29.53058867 days. known=datetime(2000,1,6,18,14,tzinfo=timezone.utc) @@ -227,6 +261,51 @@ def homelab_pulse(candidates:list[dict[str,Any]])->list[dict[str,str]]: {'label':'Workspace disk','value':disk,'state':'ok' if usage.free/usage.total>.15 else 'warn'}, ] + +def system_snapshot()->list[dict[str,str]]: + uptime_text='unknown' + try: + uptime_seconds=float(Path('/proc/uptime').read_text().split()[0]) + hours=int(uptime_seconds//3600) + minutes=int((uptime_seconds%3600)//60) + uptime_text=(f'{hours}h {minutes}m' if hours else f'{minutes}m') + except Exception: + pass + load_text='unknown' + load_state='ok' + try: + load1,load5,load15=os.getloadavg() + load_text=f'{load1:.2f} / {load5:.2f} / {load15:.2f}' + cpu_count=os.cpu_count() or 1 + if load1 > cpu_count*1.25: + load_state='warn' + except Exception: + load_state='warn' + mem_text='unknown' + mem_state='ok' + try: + parts=Path('/proc/meminfo').read_text().splitlines() + raw={line.split(':',1)[0]: int(line.split(':',1)[1].strip().split()[0]) for line in parts if ':' in line} + total=raw.get('MemTotal',0) + avail=raw.get('MemAvailable',0) + used=max(total-avail,0) + pct=(used/total*100) if total else 0 + mem_text=f'{pct:.0f}% used ({used/1024/1024:.1f} / {total/1024/1024:.1f} GiB)' + if pct >= 85: + mem_state='warn' + except Exception: + mem_state='warn' + root_usage=shutil.disk_usage('/') + root_pct=root_usage.used/root_usage.total*100 if root_usage.total else 0 + root_text=f'{root_pct:.0f}% used ({root_usage.free/1024**3:.1f} GiB free)' + return [ + {'label':'Host','value':run(['hostname']) or 'nomad','state':'ok'}, + {'label':'Uptime','value':uptime_text,'state':'ok'}, + {'label':'Load avg','value':load_text,'state':load_state}, + {'label':'Memory','value':mem_text,'state':mem_state}, + {'label':'Root disk','value':root_text,'state':'warn' if root_pct >= 85 else 'ok'}, + ] + def score_item(item:dict[str,Any],prefs:dict[str,Any])->float: score=float(item.get('priority',1)); text=f"{item.get('title','')} {item.get('summary','')}".lower() if item.get('category') in prefs.get('boost_categories',[]): score+=2 @@ -313,7 +392,7 @@ def why(i:dict[str,Any])->str: return f'Interesting signal from {src}.' def card(i:dict[str,Any], compact:bool=False)->str: - summary=(i.get('summary') or '').strip()[:170 if compact else 300] + summary=(i.get('summary') or '').strip()[:120 if compact else 260] when=(i.get('published_at','')[:16].replace('T',' ')) cls='compact-card' if compact else 'card' why_html='' if compact else f"
Why: {esc(why(i))}
" @@ -371,6 +450,21 @@ def load_paperless_review()->list[dict[str,Any]]: def load_paperless_state()->dict[str,Any]: return load_json(PAPERLESS_STATE_PATH,{}) or {} +def paperless_focus_summary(items:list[dict[str,Any]]|None=None,state:dict[str,Any]|None=None)->dict[str,Any]: + items=items if items is not None else load_paperless_review() + state=state if state is not None else load_paperless_state() + active=[]; flagged=[] + for item in items: + status=((state.get(str(item.get('id'))) or {}).get('status') or '').lower() + if status in {'dismissed','good_to_go'}: + continue + if status in {'acknowledged','needs_attention'}: + flagged.append(item) + else: + active.append(item) + headline=(flagged[0] if flagged else (active[0] if active else {})).get('title') or 'No documents waiting' + return {'active_count':len(active),'flagged_count':len(flagged),'headline':headline} + def paperless_review_section()->str: items=load_paperless_review() state=load_paperless_state() @@ -408,7 +502,8 @@ def paperless_review_section()->str: flagged_cards=''.join(render_card(item,True) for item in needs_attention[:8]) count=len(active) flag_count=len(needs_attention) - body=f"
{active_cards or '

No active documents waiting.

'}
" + empty_active="

No active documents waiting.

" + body=f"
{active_cards or empty_active}
" if overflow_active: body += f"
Show {esc(str(len(overflow_active)))} more active items
{''.join(render_card(item,False) for item in overflow_active[:8])}
" if flag_count: @@ -427,7 +522,10 @@ def list_panel(title:str,items:list[dict[str,Any]],empty:str)->str: headline=x.get('title') if x.get('time') else '' assignee=x.get('assignee') or '' detail=x.get('body') or x.get('summary') or x.get('value') or '' - rows.append(f"
  • {esc(kicker)}{f'{esc(assignee)}' if assignee else ''}
    {f'
    {esc(headline)}
    ' if headline else ''}{f'
    {esc(detail)}
    ' if detail else ''}
    • ") + assignee_html=f"{esc(assignee)}" if assignee else '' + headline_html=f"
    {esc(headline)}
    " if headline else '' + detail_html=f"
    {esc(detail)}
    " if detail else '' + rows.append(f"
  • {esc(kicker)}{assignee_html}
    {headline_html}{detail_html}
    • ") body=''.join(rows) return f"

    {esc(title)}

      {body}
    " @@ -449,7 +547,10 @@ def donetick_panel(items:list[dict[str,Any]],empty:str)->str: due_style='' if state=='overdue': due_style=' style="color:#ff6b6b;"' elif x.get('priority') in PRIORITY_COLORS: due_style=f' style="color:{esc(PRIORITY_COLORS[x.get("priority")])};"' - rows.append(f"
    {esc(x.get('due_title') or x.get('time') or 'No due date')}
    {f'{esc(x.get("assignee") or "")}' if x.get('assignee') and x.get('assignee')!='Unassigned' else ''}
    {esc(x.get('title') or 'Untitled task')}
    {f'
    {chips}
    ' if chips else ''}{f'
    {esc(subtitle)}
    ' if subtitle else ''}
    ") + assignee_html=f"{esc(x.get('assignee') or '')}" if x.get('assignee') and x.get('assignee')!='Unassigned' else '' + chips_html=f"
    {chips}
    " if chips else '' + subtitle_html=f"
    {esc(subtitle)}
    " if subtitle else '' + rows.append(f"
    {esc(x.get('due_title') or x.get('time') or 'No due date')}
    {assignee_html}
    {esc(x.get('title') or 'Untitled task')}
    {chips_html}{subtitle_html}
    ") return f"

    Donetick

    {''.join(rows)}
    " def feedback_bootstrap()->str: @@ -584,10 +685,232 @@ def forecast_panel(weather:dict[str,Any])->str: return "" return collapsible_section('3-day forecast','Rain, sunrise, and sunset over the next three days.',f"
    {''.join(cards)}
    ",'forecast',open_by_default=False,extra_class='panel forecast-panel') +def fetch_hermes_usage(days:int=30)->dict[str,Any]: + config=read_hermes_model_config() + stats={ + 'ok':False, + 'period_days':days, + 'default_model':config.get('default_model','unknown'), + 'provider':config.get('provider','unknown'), + 'sessions':0, + 'messages':0, + 'tool_calls':0, + 'user_messages':0, + 'assistant_messages':0, + 'total_tokens':0, + 'input_tokens':0, + 'output_tokens':0, + 'cache_read_tokens':0, + 'cache_write_tokens':0, + 'reasoning_tokens':0, + 'models':[], + 'platforms':[], + 'top_tools':[], + 'recent_sessions':[], + 'generated_at':datetime.now(LOCAL_TZ).isoformat(), + } + if not HERMES_STATE_DB.exists(): + stats['error']='Hermes state DB not found' + return stats + cutoff=time.time()-(days*86400) + try: + conn=sqlite3.connect(f"file:{HERMES_STATE_DB}?mode=ro", uri=True) + conn.row_factory=sqlite3.Row + cur=conn.cursor() + overview=cur.execute(""" + SELECT + count(*) AS sessions, + sum(coalesce(input_tokens,0)) AS input_tokens, + sum(coalesce(output_tokens,0)) AS output_tokens, + sum(coalesce(cache_read_tokens,0)) AS cache_read_tokens, + sum(coalesce(cache_write_tokens,0)) AS cache_write_tokens, + sum(coalesce(reasoning_tokens,0)) AS reasoning_tokens, + sum(coalesce(api_call_count,0)) AS api_calls + FROM sessions + WHERE started_at >= ? + """,(cutoff,)).fetchone() + stats.update({ + 'sessions':int((overview['sessions'] or 0) if overview else 0), + 'input_tokens':int((overview['input_tokens'] or 0) if overview else 0), + 'output_tokens':int((overview['output_tokens'] or 0) if overview else 0), + 'cache_read_tokens':int((overview['cache_read_tokens'] or 0) if overview else 0), + 'cache_write_tokens':int((overview['cache_write_tokens'] or 0) if overview else 0), + 'reasoning_tokens':int((overview['reasoning_tokens'] or 0) if overview else 0), + }) + stats['total_tokens']=stats['input_tokens']+stats['output_tokens']+stats['cache_read_tokens']+stats['cache_write_tokens']+stats['reasoning_tokens'] + row=cur.execute("SELECT count(*) AS c FROM messages WHERE role != 'session_meta' AND timestamp >= ?",(cutoff,)).fetchone() + stats['messages']=int((row['c'] or 0) if row else 0) + row=cur.execute("SELECT count(*) AS c FROM messages WHERE role='tool' AND timestamp >= ?",(cutoff,)).fetchone() + stats['tool_calls']=int((row['c'] or 0) if row else 0) + row=cur.execute("SELECT count(*) AS c FROM messages WHERE role='user' AND timestamp >= ?",(cutoff,)).fetchone() + stats['user_messages']=int((row['c'] or 0) if row else 0) + row=cur.execute("SELECT count(*) AS c FROM messages WHERE role='assistant' AND timestamp >= ?",(cutoff,)).fetchone() + stats['assistant_messages']=int((row['c'] or 0) if row else 0) + stats['models']=[dict(r) for r in cur.execute(""" + SELECT model, count(*) AS sessions, + sum(coalesce(input_tokens,0)+coalesce(output_tokens,0)+coalesce(cache_read_tokens,0)+coalesce(cache_write_tokens,0)+coalesce(reasoning_tokens,0)) AS total_tokens + FROM sessions + WHERE started_at >= ? + GROUP BY model + ORDER BY total_tokens DESC, sessions DESC + LIMIT 5 + """,(cutoff,))] + stats['platforms']=[dict(r) for r in cur.execute(""" + SELECT source, count(*) AS sessions, + sum(coalesce(input_tokens,0)+coalesce(output_tokens,0)+coalesce(cache_read_tokens,0)+coalesce(cache_write_tokens,0)+coalesce(reasoning_tokens,0)) AS total_tokens + FROM sessions + WHERE started_at >= ? + GROUP BY source + ORDER BY total_tokens DESC, sessions DESC + LIMIT 5 + """,(cutoff,))] + stats['top_tools']=[dict(r) for r in cur.execute(""" + SELECT tool_name, count(*) AS calls + FROM messages + WHERE role='tool' AND tool_name IS NOT NULL AND tool_name != '' AND timestamp >= ? + GROUP BY tool_name + ORDER BY calls DESC, tool_name ASC + LIMIT 8 + """,(cutoff,))] + stats['recent_sessions']=[dict(r) for r in cur.execute(""" + SELECT source, model, started_at, + (coalesce(input_tokens,0)+coalesce(output_tokens,0)+coalesce(cache_read_tokens,0)+coalesce(cache_write_tokens,0)+coalesce(reasoning_tokens,0)) AS total_tokens, + coalesce(api_call_count,0) AS api_calls + FROM sessions + WHERE started_at >= ? + ORDER BY started_at DESC + LIMIT 5 + """,(cutoff,))] + for item in stats['recent_sessions']: + started=item.get('started_at') + try: item['started_local']=datetime.fromtimestamp(float(started), LOCAL_TZ).strftime('%a %-I:%M %p') + except Exception: item['started_local']='recent' + stats['ok']=True + conn.close() + return stats + except Exception: + stats['error']='Hermes telemetry is temporarily unavailable.' + return stats + +def focus_card(title:str,kicker:str,headline:str,body:str,tone:str='')->str: + return f"
    {esc(kicker)}

    {esc(title)}

    {esc(headline)}

    {esc(body)}

    " + + +def build_focus_cards(local_items:list[dict[str,Any]], todos:list[dict[str,Any]], pulse:list[dict[str,str]], paperless:dict[str,Any])->str: + urgent_todos=[item for item in todos if (item.get('state') or '') in {'overdue','today'} and (item.get('source') or '') != 'Dashboard'] + warn_items=[item for item in pulse if item.get('state')=='warn'] + local_story=local_items[0] if local_items else None + if urgent_todos: + chores_headline=f"{len(urgent_todos)} urgent / due today" + first_urgent=urgent_todos[0] + chores_body=(first_urgent.get('title') or 'You have work waiting') + (f" • {first_urgent.get('due_title')}" if first_urgent.get('due_title') else '') + else: + chores_headline='No urgent chores' + chores_body='Nothing due soon enough to nag you here.' + cards=[ + focus_card( + 'Local watch', + 'Useful nearby', + (local_story.get('title') if local_story else 'Local lane quiet'), + (f"{local_story.get('source','Local feed')} • {label(local_story.get('category','local'))}" if local_story else 'Nothing local survived filtering hard enough to earn the homepage.'), + 'local' if local_story else '' + ), + focus_card( + 'Chores with heat', + 'Donetick', + chores_headline, + chores_body, + 'warn' if urgent_todos else 'ok' + ), + focus_card( + 'Paperless follow-up', + 'Queue check', + (f"{paperless.get('active_count',0)} active · {paperless.get('flagged_count',0)} flagged" if paperless.get('active_count',0) or paperless.get('flagged_count',0) else 'No documents waiting'), + str(paperless.get('headline') or 'No documents waiting'), + 'warn' if paperless.get('flagged_count',0) else ('ok' if not paperless.get('active_count',0) else '') + ), + focus_card( + 'Homelab pulse', + 'NOMAD checks', + ('Stable' if not warn_items else f"{len(warn_items)} warning{'s' if len(warn_items)!=1 else ''}"), + ('; '.join(f"{item.get('label')}: {item.get('value')}" for item in warn_items[:2]) if warn_items else 'Dashboard service, cron, digest freshness, and disk all look fine.'), + 'warn' if warn_items else 'ok' + ), + ] + return "
    " + ''.join(cards) + "
    " + + +def build_hero_summary(local_items:list[dict[str,Any]], todos:list[dict[str,Any]], pulse:list[dict[str,str]], paperless:dict[str,Any], hermes:dict[str,Any])->str: + urgent_todos=len([item for item in todos if (item.get('state') or '') in {'overdue','today'} and (item.get('source') or '') != 'Dashboard']) + warn_count=len([item for item in pulse if item.get('state')=='warn']) + local_status='local lane has signal' if local_items else 'local lane is quiet' + paperless_text=f"{paperless.get('active_count',0)} Paperless item{'s' if paperless.get('active_count',0)!=1 else ''}" + hermes_model=(hermes.get('models') or [{}])[0].get('model') or hermes.get('default_model') or 'your main model' + return f"{urgent_todos} urgent chore{'s' if urgent_todos!=1 else ''}, {paperless_text}, {warn_count} homelab warning{'s' if warn_count!=1 else ''}, and Hermes has mostly been running on {hermes_model} while the {local_status}." + + +def list_rows(rows:list[str])->str: + return ''.join(f"
  • {row}
    • " for row in rows) if rows else "
  • Nothing to show.
    • " + + +def build_system_section(snapshot:list[dict[str,str]])->str: + metrics=''.join(metric_card(item.get('label','Metric'), item.get('value','—'), 'live host state', item.get('state','')) for item in snapshot) + subtitle='Live NOMAD host state: uptime, load, memory pressure, and root disk usage.' + return collapsible_section('System snapshot',subtitle,f"
    {metrics}
    ",'system-snapshot',open_by_default=True) + + +def build_hermes_section(hermes:dict[str,Any])->str: + if not hermes.get('ok'): + body="

    Hermes telemetry unavailable

    Local Hermes usage stats could not be read right now.

    " + return collapsible_section('Hermes usage',f"Could not read the last {hermes.get('period_days',30)} days of local Hermes state.",body,'hermes-usage',open_by_default=True) + metrics=''.join([ + metric_card('Default model',str(hermes.get('default_model','unknown')),str(hermes.get('provider','unknown')),'space'), + metric_card(f"{hermes.get('period_days',30)}-day tokens",compact_number(hermes.get('total_tokens',0)),'all session rows','space'), + metric_card('Sessions / messages',f"{compact_number(hermes.get('sessions',0))} / {compact_number(hermes.get('messages',0))}",'state.db window','space'), + metric_card('Tool calls',compact_number(hermes.get('tool_calls',0)),f"{compact_number(hermes.get('user_messages',0))} user · {compact_number(hermes.get('assistant_messages',0))} assistant",'space'), + ]) + model_rows=[] + for item in hermes.get('models',[])[:5]: + model_rows.append(f"{esc(item.get('model') or 'unknown')} · {compact_number(item.get('sessions',0))} sessions · {compact_number(item.get('total_tokens',0))} tokens") + platform_rows=[] + for item in hermes.get('platforms',[])[:5]: + platform_rows.append(f"{esc((item.get('source') or 'unknown').title())} · {compact_number(item.get('sessions',0))} sessions · {compact_number(item.get('total_tokens',0))} tokens") + tool_rows=[] + for item in hermes.get('top_tools',[])[:8]: + tool_rows.append(f"{esc(item.get('tool_name') or 'tool')} · {compact_number(item.get('calls',0))} calls") + recent_rows=[] + for item in hermes.get('recent_sessions',[])[:5]: + source=(item.get('source') or 'session').title() + model=item.get('model') or 'unknown' + recent_rows.append(f"{esc(source)} · {esc(item.get('started_local') or 'recent')} · {esc(model)} · {compact_number(item.get('total_tokens',0))} tokens") + detail_html=f""" +
    +
    +

    Model mix

    +
      {list_rows(model_rows)}
    +
    +
    +

    Platform split

    +
      {list_rows(platform_rows)}
    +
    +
    +

    Top tools

    +
      {list_rows(tool_rows)}
    +
    +
    +

    Recent sessions

    +
      {list_rows(recent_rows)}
    +
    +
    + """ + body=f"
    {metrics}
    {detail_html}" + subtitle=f"Live read from ~/.hermes/state.db over the last {hermes.get('period_days',30)} days." + return collapsible_section('Hermes usage',subtitle,body,'hermes-usage',open_by_default=True) + def render()->str: candidates=load_json(DIGEST_ROOT/'data/candidates.json',[]); prefs=load_json(DIGEST_ROOT/'data/preferences.json',{}) donetick_todos=fetch_donetick_todos(); todos=inject_dashboard_todos(donetick_todos or load_json(DATA/'todos.json',[])); calendar=load_json(DATA/'calendar.json',[]); notes=load_json(DATA/'notes.json',[]) - selected=select_items(candidates,prefs); top_briefing=selected[:6]; top_urls={i.get('url','') for i in top_briefing if i.get('url')}; groups={'space':[],'homelab':[],'local':[],'signals':[]} + selected=select_items(candidates,prefs); top_briefing=selected[:4]; top_urls={i.get('url','') for i in top_briefing if i.get('url')}; groups={'space':[],'homelab':[],'local':[],'signals':[]} exploration=select_exploration_items(candidates,prefs,{i.get('url','') for i in selected if i.get('url')},4) for i in selected: if i.get('url','') in top_urls: continue @@ -596,7 +919,8 @@ def render()->str: elif cat in {'homelab','smart_home','radio'}: groups['homelab'].append(i) elif cat=='local': groups['local'].append(i) else: groups['signals'].append(i) - weather=fetch_weather(); moon=moon_phase(datetime.now(timezone.utc)); pulse=homelab_pulse(candidates) + weather=fetch_weather(); moon=moon_phase(datetime.now(timezone.utc)); pulse=homelab_pulse(candidates); hermes=fetch_hermes_usage(30); system=system_snapshot() + paperless=paperless_focus_summary() now=datetime.now(LOCAL_TZ); counts=Counter(i.get('category','other') for i in candidates) if weather['ok']: cur=weather['current']; daily=weather.get('daily',{}) @@ -613,31 +937,45 @@ def render()->str: metric_card('Sky note','Space feeds live','Moon phase is computed locally','space')]) pulse_metrics=''.join(metric_card(x['label'],x['value'],x['state'],x['state']) for x in pulse) forecast_html=forecast_panel(weather) + hero_summary=build_hero_summary(groups['local'],todos,pulse,paperless,hermes) + focus_html=build_focus_cards(groups['local'],todos,pulse,paperless) + hermes_html=build_hermes_section(hermes) + system_html=build_system_section(system) + hero_tokens=compact_number(hermes.get('total_tokens',0)) if hermes.get('ok') else '—' + primary_html=''.join([ + section('Top briefing','Compact scan of what is worth your time.',top_briefing,'Nothing worth bothering you with.',compact=True), + section('Homelab / Smart Home / Radio','Infrastructure, self-hosting, smart home, and ham radio.',groups['homelab'][:4],'No homelab signal right now.',compact=True), + section('Sky / Space','SpaceX, NASA, planetary science, and astronomy.',groups['space'][:4],'No sky signal right now.',compact=True), + section('Local','Clarksville / Middle Tennessee items that seem useful.',groups['local'][:3],'No local signal right now.',compact=True), + section('New to You','Intentional oddballs outside your usual top lanes, so you can see if any new categories deserve a slot.',exploration[:4],'No worthwhile experimental picks right now.',compact=True), + section('Other signals','Everything else that survived filtering.',groups['signals'][:3],'No extra signals.',compact=True), + ]) + secondary_html=''.join([ + collapsible_section('Focus now','What actually deserves attention before you go wandering through feeds.',focus_html,'focus-now',open_by_default=True), + system_html, + hermes_html, + collapsible_section('Agenda','Calendar, Donetick, and pinned notes.',f"
    {list_panel('Calendar',calendar,'Calendar integration not wired yet.')}{donetick_panel(todos,'No open Donetick tasks found; JSON placeholder active.')}{list_panel('Notes',notes,'No notes.')}
    ",'today-overview',open_by_default=False), + collapsible_section('Environment','Weather, sky, and outside-the-window context.',f"
    {weather_metrics}{moon_metrics}
    ",'at-a-glance',open_by_default=False), + forecast_html, + paperless_review_section(), + collapsible_section('Homelab pulse','Safe local checks on NOMAD only.',f"
    {pulse_metrics}
    ",'homelab-pulse',open_by_default=False), + ]) return f"""Doris Dashboard
    -

    PC Doris Thatcher

    Doris Dashboard

    Morning briefing, sky watch, homelab pulse, and local signal — refreshed on NOMAD.

    Hidden topics: 0
    {esc(now.strftime('%a %b %-d, %-I:%M %p'))}Last generated Central · auto-refreshes at :05 / :35
    {len(candidates)}Candidates
    {len(set(i.get('source','') for i in candidates))}Sources
    -{collapsible_section('At a glance','Weather and sky cards.',f"
    {weather_metrics}{moon_metrics}
    ",'at-a-glance',open_by_default=False)} -{collapsible_section('Today','Calendar, Donetick, and notes.',f"
    {list_panel('Today',calendar,'Calendar integration not wired yet.')}{donetick_panel(todos,'No open Donetick tasks found; JSON placeholder active.')}{list_panel('Notes',notes,'No notes.')}
    ",'today-overview',open_by_default=True)} -{forecast_html} -{paperless_review_section()} -{collapsible_section('Homelab pulse','Safe local checks on NOMAD only.',f"
    {pulse_metrics}
    ",'homelab-pulse',open_by_default=False)} -{section('Top briefing','Compact scan of what is worth your time.',top_briefing,'Nothing worth bothering you with.',compact=True)} -{section('Sky / Space','SpaceX, NASA, planetary science, and astronomy.',groups['space'][:5],'No sky signal right now.',compact=True)} -{section('Homelab / Smart Home / Radio','Infrastructure, self-hosting, smart home, and ham radio.',groups['homelab'][:4],'No homelab signal right now.',compact=True)} -{section('Local','Clarksville / Middle Tennessee items that seem useful.',groups['local'][:3],'No local signal right now.',compact=True)} -{section('New to You','Intentional oddballs outside your usual top lanes, so you can see if any new categories deserve a slot.',exploration[:4],'No worthwhile experimental picks right now.',compact=True)} -{section('Other signals','Everything else that survived filtering.',groups['signals'][:3],'No extra signals.',compact=True)} -

    Category mix: {esc(', '.join(f'{k}: {v}' for k,v in counts.most_common(10)))}

    Local-only on NOMAD unless John says otherwise. Weather from Open-Meteo; moon phase calculated locally.

    {feedback_bootstrap()}""" +

    PC Doris Thatcher

    Doris Dashboard

    {esc(hero_summary)}

    Hidden topics: 0
    {esc(now.strftime('%a %b %-d, %-I:%M %p'))}Central time · refreshes every 30m
    {len(candidates)}Candidates
    {len(set(i.get('source','') for i in candidates))}Sources
    {esc(hero_tokens)}30-day Hermes tokens
    +
    {primary_html}
    +{feedback_bootstrap()}""" def main()->int: PUBLIC.mkdir(parents=True,exist_ok=True); LOGS.mkdir(parents=True,exist_ok=True); DATA.mkdir(parents=True,exist_ok=True) candidates=load_json(DIGEST_ROOT/'data/candidates.json',[]); prefs=load_json(DIGEST_ROOT/'data/preferences.json',{}) - weather=fetch_weather(); moon=moon_phase(datetime.now(timezone.utc)); pulse=homelab_pulse(candidates); articles=select_items(candidates,prefs,20); todos=inject_dashboard_todos(fetch_donetick_todos()) + weather=fetch_weather(); moon=moon_phase(datetime.now(timezone.utc)); pulse=homelab_pulse(candidates); articles=select_items(candidates,prefs,20); todos=inject_dashboard_todos(fetch_donetick_todos()); hermes=fetch_hermes_usage(30) generated={'generated_at':datetime.now(timezone.utc).isoformat(),'generated_at_local':datetime.now(LOCAL_TZ).isoformat(),'timezone':'America/Chicago','candidate_count':len(candidates),'source_count':len(set(i.get('source','') for i in candidates))} (DATA/'dashboard.json').write_text(json.dumps(generated,indent=2)+'\n') (DATA/'weather.json').write_text(json.dumps(weather,indent=2)+'\n') (DATA/'sky.json').write_text(json.dumps({'moon':moon},indent=2)+'\n') (DATA/'homelab.json').write_text(json.dumps(pulse,indent=2)+'\n') (DATA/'articles.json').write_text(json.dumps(articles,indent=2)+'\n') + (DATA/'hermes_usage.json').write_text(json.dumps(hermes,indent=2)+'\n') if todos: (DATA/'todos.json').write_text(json.dumps(todos,indent=2)+'\n') (PUBLIC/'index.html').write_text(render(),encoding='utf-8') return 0 diff --git a/home/doris-dashboard/public/style.css b/home/doris-dashboard/public/style.css index e5f2cbf..36cc10f 100644 --- a/home/doris-dashboard/public/style.css +++ b/home/doris-dashboard/public/style.css @@ -1,13 +1,482 @@ -:root{color-scheme:dark;--bg:#0c111b;--panel:#151b27;--panel2:#202939;--text:#eef4ff;--muted:#9cabbe;--line:#2d394b;--accent:#8fd3ff;--space:#9f8cff;--home:#4fd1a5;--local:#ffd166;--warn:#ff6b6b;--ok:#5ee787}*{box-sizing:border-box}body{margin:0;font-family:Inter,ui-sans-serif,system-ui,-apple-system,BlinkMacSystemFont,"Segoe UI",sans-serif;background:radial-gradient(circle at top left,#213455,#0c111b 42rem);color:var(--text)}a{color:inherit}main{max-width:1220px;margin:0 auto;padding:18px}.hero{display:grid;grid-template-columns:1fr auto;gap:18px;align-items:end;margin-bottom:18px}.eyebrow{color:var(--accent);text-transform:uppercase;letter-spacing:.16em;font-size:.78rem;margin:0 0 8px}h1{font-size:clamp(2.2rem,6vw,4.5rem);line-height:.94;margin:0}.sub{color:var(--muted);font-size:1rem;max-width:760px}.hero-actions{display:flex;gap:12px;align-items:center;margin-top:10px;flex-wrap:wrap}.reset-btn{background:var(--panel2);border:1px solid var(--line);border-radius:999px;color:var(--text);padding:8px 12px;font:inherit;cursor:pointer}.reset-btn:hover{border-color:var(--accent)}.status,.metrics{display:grid;grid-template-columns:repeat(5,minmax(0,1fr));gap:10px}.status div,.metric,.panel,.card{background:rgba(21,27,39,.9);border:1px solid var(--line);border-radius:20px;padding:14px;box-shadow:0 10px 30px rgba(0,0,0,.18);backdrop-filter:blur(6px)}.status strong,.metric strong{display:block;font-size:1.1rem}.status span,.metric span,.metric em,.meta,footer,.empty{color:var(--muted)}.metric{min-height:92px;border-top:4px solid var(--accent)}.metric strong{font-size:1.35rem;margin:6px 0}.metric em{font-style:normal;font-size:.84rem}.metric.weather{border-top-color:var(--local)}.metric.space{border-top-color:var(--space)}.metric.ok{border-top-color:var(--ok)}.metric.warn{border-top-color:var(--warn)}.today{display:grid;grid-template-columns:1fr 1.3fr 1fr;gap:12px;margin:0}.panel h2,section h2{margin:0 0 8px}ul{padding-left:18px;margin:0}.panel ul{list-style:none;padding-left:0;display:grid;gap:8px}.panel li{margin:0}section{margin:18px 0}.grid{display:grid;grid-template-columns:repeat(3,minmax(0,1fr));gap:14px}.card{min-height:245px;display:flex;flex-direction:column;gap:10px;border-top:4px solid var(--accent)}.card.astronomy{border-top-color:var(--space)}.card.homelab,.card.smart_home,.card.radio{border-top-color:var(--home)}.card.local{border-top-color:var(--local)}.card.security{border-top-color:var(--warn)}.meta{display:flex;flex-wrap:wrap;gap:6px;font-size:.74rem}.meta span{padding:3px 8px;border-radius:999px;background:var(--panel2)}h3{font-size:1rem;line-height:1.2;margin:0}h3 a{text-decoration:none}h3 a:hover{text-decoration:underline}p{line-height:1.35}.why{margin-top:auto;color:#c8d5ea;background:rgba(143,211,255,.08);padding:10px;border-radius:12px;font-size:.88rem}footer{border-top:1px solid var(--line);padding:18px 0 26px}.collapsible{margin:16px 0;border:1px solid var(--line);border-radius:20px;background:rgba(21,27,39,.9);box-shadow:0 10px 30px rgba(0,0,0,.18);backdrop-filter:blur(6px);overflow:hidden}.collapsible>summary{list-style:none;cursor:pointer;padding:14px 16px;display:flex;justify-content:space-between;gap:12px;align-items:center}.collapsible>summary::-webkit-details-marker{display:none}.collapsible-heading h2{margin:0}.collapsible-heading p{margin:4px 0 0;color:var(--muted)}.collapsible-body{padding:0 14px 14px}.collapse-indicator{color:var(--muted);transition:transform .15s ease}.collapsible[open] .collapse-indicator{transform:rotate(180deg)}@media(max-width:1050px){.status,.metrics,.today,.grid,.forecast-grid,.news-list,.task-grid{grid-template-columns:1fr 1fr}.hero{grid-template-columns:1fr}}@media(max-width:650px){main{padding:14px}.status,.metrics,.today,.grid,.forecast-grid,.news-list,.task-grid{grid-template-columns:1fr}.card{min-height:auto}} +:root{ + color-scheme:dark; + --bg:#08090a; + --bg-accent:#121826; + --panel:#11151d; + --panel-soft:#171d27; + --panel-raise:#1d2430; + --text:#f3f6fb; + --muted:#94a3b8; + --line:rgba(255,255,255,.08); + --line-soft:rgba(255,255,255,.05); + --accent:#7c8cff; + --accent-soft:rgba(124,140,255,.16); + --space:#9f8cff; + --home:#4fd1a5; + --local:#ffd166; + --warn:#ff7b72; + --ok:#5ee787; + --shadow:0 14px 34px rgba(0,0,0,.26); +} -/* Compact article layout: keep the number of stories, reduce scroll debt. */ -.news-list{display:grid;grid-template-columns:repeat(2,minmax(0,1fr));gap:10px}.compact-card{background:rgba(21,27,39,.9);border:1px solid var(--line);border-left:5px solid var(--accent);border-radius:16px;padding:10px 12px;box-shadow:0 6px 18px rgba(0,0,0,.14)}.compact-card.astronomy{border-left-color:var(--space)}.compact-card.homelab,.compact-card.smart_home,.compact-card.radio{border-left-color:var(--home)}.compact-card.local{border-left-color:var(--local)}.compact-card.security{border-left-color:var(--warn)}.compact-card .meta{margin-bottom:4px}.compact-card h3{font-size:.98rem;line-height:1.18;margin:0 0 3px}.compact-card p{margin:0;color:#c8d5ea;font-size:.88rem;line-height:1.25;display:-webkit-box;-webkit-line-clamp:3;-webkit-box-orient:vertical;overflow:hidden}.compact-card .meta span{padding:2px 7px}.compact-card:nth-child(n+7){display:none}section header{display:flex;justify-content:space-between;gap:12px;align-items:end}section header p{margin:0}.feedback{display:flex;gap:6px;justify-content:flex-end;margin-top:8px}.feedback-btn{background:transparent;border:1px solid var(--line);border-radius:999px;color:var(--text);padding:2px 8px;font:inherit;cursor:pointer}.feedback-btn:hover{border-color:var(--accent);transform:translateY(-1px)}.topic-liked{box-shadow:0 0 0 1px rgba(94,231,135,.45),0 6px 18px rgba(0,0,0,.14)}@media(max-width:650px){.compact-card p{display:-webkit-box;-webkit-line-clamp:2;-webkit-box-orient:vertical;overflow:hidden}section header{display:block}} -.feedback-btn[aria-pressed='true']{font-weight:700}.feedback-btn.active-up{border-color:rgba(94,231,135,.65);background:rgba(94,231,135,.12);color:#c8ffd9}.feedback-btn.active-down{border-color:rgba(255,107,107,.6);background:rgba(255,107,107,.12);color:#ffd9d9}.dashboard-toast{position:fixed;right:16px;bottom:16px;z-index:1000;max-width:min(420px,calc(100vw - 24px));background:rgba(12,17,27,.96);border:1px solid var(--line);border-left:4px solid var(--accent);border-radius:16px;padding:12px 14px;color:var(--text);box-shadow:0 14px 34px rgba(0,0,0,.35)}.dashboard-toast[hidden]{display:none}.toast-action{margin-left:10px;background:transparent;border:0;color:var(--accent);font:inherit;font-weight:700;cursor:pointer;padding:0} -.muted{color:var(--muted);font-size:.86rem;margin:0 .25rem}.panel-item{background:rgba(32,41,57,.55);border:1px solid var(--line);border-radius:14px;padding:10px 12px}.panel-item-top{display:flex;justify-content:space-between;gap:10px;align-items:baseline}.panel-item-title{margin-top:2px;font-weight:600;color:#dbe7f7}.panel-item-body{margin-top:4px;color:#c8d5ea;font-size:.9rem;line-height:1.25;display:-webkit-box;-webkit-line-clamp:2;-webkit-box-orient:vertical;overflow:hidden}.panel .empty{padding:12px 0}.donetick-panel{padding:12px 14px}.task-grid{display:grid;grid-template-columns:repeat(2,minmax(0,1fr));gap:10px}.task-card{background:rgba(32,41,57,.58);border:1px solid var(--line);border-radius:16px;padding:10px 12px;min-width:0}.task-top{display:flex;justify-content:space-between;gap:8px;align-items:center;margin-bottom:4px}.task-due{font-size:.8rem;font-weight:800;letter-spacing:.02em;text-transform:none;color:#dbe7f7}.task-assignee{font-size:.72rem;padding:2px 8px;border-radius:999px;background:var(--panel2);color:var(--muted)}.task-name{font-size:1rem;font-weight:700;line-height:1.08;color:var(--text);margin-bottom:6px}.task-subtitle{font-size:.78rem;line-height:1.16;color:var(--muted)}.task-chips{display:flex;flex-wrap:wrap;gap:5px;margin-bottom:5px}.task-chip{display:inline-block;padding:3px 8px;border-radius:999px;color:#fff;font-size:.72rem;font-weight:700;line-height:1.05}.task-card.overdue{border-color:rgba(255,107,107,.55)}.task-card.today{border-color:rgba(255,209,102,.5)}.task-card.tomorrow{border-color:rgba(142,202,230,.45)}.task-card.nodue{opacity:.92} -.donetick-panel .task-grid{gap:8px}.task-card{padding:9px 10px;border-radius:14px}.task-top{gap:6px;margin-bottom:3px}.task-state-dot{width:9px;height:9px;border-radius:999px;background:var(--muted);flex:0 0 auto;box-shadow:0 0 0 2px rgba(255,255,255,.03)}.task-card.overdue .task-state-dot{background:var(--warn)}.task-card.today .task-state-dot{background:var(--local)}.task-card.tomorrow .task-state-dot,.task-card.upcoming .task-state-dot{background:#8ecae6}.task-card.nodue .task-state-dot{background:#6c7a90}.task-due{font-size:.76rem;line-height:1.1}.task-name{font-size:.95rem;line-height:1.15;margin-bottom:4px;display:-webkit-box;-webkit-line-clamp:2;-webkit-box-orient:vertical;overflow:hidden}.task-subtitle{font-size:.74rem;line-height:1.1;opacity:.9;display:-webkit-box;-webkit-line-clamp:1;-webkit-box-orient:vertical;overflow:hidden}.task-chips{gap:4px;margin-bottom:4px}.task-chip{padding:2px 7px;font-size:.68rem} +*{box-sizing:border-box} +html{scroll-behavior:smooth} +body{ + margin:0; + font-family:Inter,ui-sans-serif,system-ui,-apple-system,BlinkMacSystemFont,"Segoe UI",sans-serif; + background: + radial-gradient(circle at top left,rgba(124,140,255,.16),transparent 28rem), + radial-gradient(circle at top right,rgba(79,209,165,.08),transparent 22rem), + var(--bg); + color:var(--text); +} -/* Forecast */ -.forecast-panel{padding:14px 16px}.forecast-grid{display:grid;grid-template-columns:repeat(3,minmax(0,1fr));gap:12px}.forecast-card{background:rgba(32,41,57,.65);border:1px solid var(--line);border-radius:16px;padding:12px 14px;display:flex;flex-direction:column;gap:6px}.forecast-day{color:var(--accent);font-size:.82rem;text-transform:uppercase;letter-spacing:.08em}.forecast-card strong{font-size:1rem}.forecast-card em{font-style:normal;color:var(--muted);font-size:.88rem} +a{color:inherit} +main{max-width:1380px;margin:0 auto;padding:18px} -/* Paperless review queue */ -.badge{display:inline-block;margin-left:.35rem;padding:2px 8px;border-radius:999px;background:var(--panel2);color:var(--accent);font-size:.85rem}.paperless-grid{display:grid;grid-template-columns:1fr;gap:10px}.paperless-card{background:rgba(21,27,39,.9);border:1px solid var(--line);border-left:5px solid var(--accent);border-radius:16px;padding:10px 12px}.paperless-card.warn{border-left-color:var(--warn)}.paperless-card.info{border-left-color:var(--local)}.paperless-card.ok{border-left-color:var(--ok)}.paperless-card.needs-attention{box-shadow:0 0 0 1px rgba(255,209,102,.25),0 10px 24px rgba(0,0,0,.22)}.paperless-head{display:flex;justify-content:space-between;gap:10px;align-items:flex-start}.paperless-card h4{margin:.35rem 0;font-size:.96rem;line-height:1.2}.paperless-card .summary{margin:.2rem 0;color:#d7e3f5;font-size:.9rem;line-height:1.25;display:-webkit-box;-webkit-line-clamp:2;-webkit-box-orient:vertical;overflow:hidden}.paperless-meta,.paperless-reason{margin:.2rem 0;color:var(--muted);font-size:.8rem;line-height:1.2}.paperless-card .link{display:inline-block;color:var(--accent);text-decoration:none}.paperless-card .link:hover{text-decoration:underline}.paperless-actions{display:flex;gap:6px;flex-wrap:wrap;align-items:center;justify-content:flex-end}.paperless-btn{background:var(--panel2);border:1px solid var(--line);border-radius:999px;color:var(--text);padding:5px 9px;font:inherit;cursor:pointer}.paperless-btn.good{border-color:rgba(94,231,135,.45);color:#c8ffd9}.paperless-btn:hover{border-color:var(--accent)}.paperless-acked,.paperless-more{margin-top:12px;border-top:1px solid var(--line);padding-top:12px}.paperless-acked summary,.paperless-more summary{cursor:pointer;color:var(--muted)}.urgency{font-weight:700}.urgency.warn{color:var(--warn)}.urgency.info{color:var(--local)}.urgency.ok{color:var(--ok)}@media(max-width:720px){.paperless-head{display:block}.paperless-actions{justify-content:flex-start;margin-top:8px}} +.hero, +.status div, +.metric, +.panel, +.card, +.compact-card, +.focus-card, +.summary-card, +.paperless-card, +.forecast-card, +.panel-item, +.task-card, +.collapsible{ + background:rgba(17,21,29,.92); + border:1px solid var(--line); + box-shadow:var(--shadow); +} + +.hero{ + display:grid; + grid-template-columns:minmax(0,1.25fr) 430px; + gap:14px; + align-items:stretch; + border-radius:24px; + padding:16px 18px; + margin-bottom:16px; +} + +.eyebrow{ + margin:0 0 6px; + color:#b7c0ff; + text-transform:uppercase; + letter-spacing:.18em; + font-size:.72rem; +} + +h1{ + margin:0; + font-size:clamp(2rem,4.6vw,3.5rem); + line-height:.94; + letter-spacing:-.04em; +} + +.sub{ + margin:10px 0 0; + color:var(--muted); + font-size:.97rem; + line-height:1.45; + max-width:70ch; +} + +.hero-actions{ + display:flex; + gap:10px; + align-items:center; + flex-wrap:wrap; + margin-top:12px; +} + +.reset-btn, +.feedback-btn, +.paperless-btn{ + appearance:none; + background:rgba(255,255,255,.03); + border:1px solid var(--line); + border-radius:999px; + color:var(--text); + cursor:pointer; + font:inherit; +} + +.reset-btn{padding:7px 11px} +.reset-btn:hover, +.feedback-btn:hover, +.paperless-btn:hover{border-color:rgba(124,140,255,.55)} + +.status, +.metrics{ + display:grid; + grid-template-columns:repeat(4,minmax(0,1fr)); + gap:10px; +} + +.metrics.metrics-compact{grid-template-columns:repeat(2,minmax(0,1fr))} + +.status div, +.metric{ + border-radius:18px; + padding:11px 12px; + min-width:0; +} + +.status strong, +.metric strong{display:block} +.status strong{font-size:1rem} +.metric{ + min-height:78px; + border-top:3px solid var(--accent); +} +.metric strong{ + margin:6px 0 4px; + font-size:1.12rem; + line-height:1.08; +} +.metric span, +.metric em, +.status span, +.meta, +.empty, +footer, +.muted{ + color:var(--muted); +} +.metric em{font-style:normal;font-size:.78rem} +.metric.weather{border-top-color:var(--local)} +.metric.space{border-top-color:var(--space)} +.metric.ok{border-top-color:var(--ok)} +.metric.warn{border-top-color:var(--warn)} + +.dashboard-shell{ + display:grid; + grid-template-columns:minmax(0,1.5fr) minmax(320px,.9fr); + gap:14px; + align-items:start; +} + +.primary-column, +.secondary-column{ + display:flex; + flex-direction:column; + gap:12px; + min-width:0; +} + +.collapsible{ + border-radius:20px; + overflow:hidden; +} + +.collapsible > summary{ + list-style:none; + cursor:pointer; + display:flex; + align-items:flex-start; + justify-content:space-between; + gap:12px; + padding:12px 14px; +} + +.collapsible > summary::-webkit-details-marker{display:none} +.collapsible-heading{min-width:0} +.collapsible-heading h2, +.panel h2, +section h2{ + margin:0; + font-size:1.02rem; + line-height:1.15; + letter-spacing:-.02em; +} + +.collapsible-heading p, +section header p{ + margin:4px 0 0; + color:var(--muted); + font-size:.83rem; + line-height:1.32; +} + +.collapse-indicator{ + color:var(--muted); + transition:transform .16s ease; + transform-origin:center; +} +.details[open] .collapse-indicator, +.collapsible[open] .collapse-indicator{transform:rotate(180deg)} +.collapsible-body{padding:0 14px 14px} + +.focus-grid, +.summary-grid, +.today, +.news-list, +.grid, +.task-grid, +.forecast-grid{ + display:grid; + gap:10px; +} + +.focus-grid, +.summary-grid, +.forecast-grid{grid-template-columns:repeat(2,minmax(0,1fr))} +.today{grid-template-columns:1fr} +.news-list{grid-template-columns:repeat(2,minmax(0,1fr))} +.grid{grid-template-columns:repeat(2,minmax(0,1fr))} +.task-grid{grid-template-columns:1fr} + +.focus-card, +.summary-card, +.panel, +.compact-card, +.paperless-card, +.forecast-card, +.panel-item, +.task-card, +.card{ + border-radius:16px; +} + +.focus-card, +.summary-card{padding:12px} +.focus-card{border-top:3px solid var(--accent)} +.focus-card.local{border-top-color:var(--local)} +.focus-card.ok{border-top-color:var(--ok)} +.focus-card.warn{border-top-color:var(--warn)} +.focus-kicker{ + display:inline-block; + margin-bottom:8px; + font-size:.72rem; + letter-spacing:.12em; + text-transform:uppercase; + color:#b7c0ff; +} +.focus-card h3, +.summary-card h3{margin:0 0 5px;font-size:.93rem} +.focus-card strong{display:block;font-size:1rem;line-height:1.2;margin-bottom:5px} +.focus-card p, +.summary-card p, +.summary-card li{margin:0;color:#d5deeb;font-size:.88rem;line-height:1.34} +.summary-card ul{margin:0;padding-left:18px} +.summary-card li+li{margin-top:6px} +.hermes-metrics{margin-bottom:10px} + +.panel{padding:12px} +.panel ul{list-style:none;padding-left:0;margin:0;display:grid;gap:8px} +.panel-item{ + padding:9px 10px; + background:rgba(255,255,255,.03); + border-color:var(--line-soft); +} +.panel-item-top{display:flex;justify-content:space-between;gap:8px;align-items:baseline} +.panel-item-top strong{font-size:.82rem} +.panel-item-title{margin-top:3px;font-weight:600;color:#dce5f6;font-size:.93rem;line-height:1.2} +.panel-item-body{ + margin-top:4px; + color:#c7d1e2; + font-size:.84rem; + line-height:1.28; + display:-webkit-box; + -webkit-line-clamp:2; + -webkit-box-orient:vertical; + overflow:hidden; +} +.panel .empty{padding:6px 2px} + +.donetick-panel{padding:12px} +.task-card{ + padding:9px 10px; + background:rgba(255,255,255,.035); + border-color:var(--line-soft); + min-width:0; +} +.task-top{display:flex;justify-content:space-between;gap:6px;align-items:center;margin-bottom:3px} +.task-state-dot{width:8px;height:8px;border-radius:999px;background:var(--muted);flex:0 0 auto;box-shadow:0 0 0 2px rgba(255,255,255,.03)} +.task-card.overdue .task-state-dot{background:var(--warn)} +.task-card.today .task-state-dot{background:var(--local)} +.task-card.tomorrow .task-state-dot, +.task-card.upcoming .task-state-dot{background:#8ecae6} +.task-card.nodue .task-state-dot{background:#6c7a90} +.task-due{font-size:.74rem;font-weight:700;line-height:1.1;color:#dde6f6} +.task-assignee{font-size:.68rem;padding:2px 7px;border-radius:999px;background:var(--panel-raise);color:var(--muted)} +.task-name{ + font-size:.93rem; + font-weight:700; + line-height:1.14; + color:var(--text); + margin-bottom:4px; + display:-webkit-box; + -webkit-line-clamp:2; + -webkit-box-orient:vertical; + overflow:hidden; +} +.task-subtitle{ + font-size:.73rem; + line-height:1.14; + color:var(--muted); + display:-webkit-box; + -webkit-line-clamp:1; + -webkit-box-orient:vertical; + overflow:hidden; +} +.task-chips{display:flex;flex-wrap:wrap;gap:4px;margin-bottom:4px} +.task-chip{display:inline-block;padding:2px 7px;border-radius:999px;color:#fff;font-size:.67rem;font-weight:700;line-height:1.05} +.task-card.overdue{border-color:rgba(255,123,114,.45)} +.task-card.today{border-color:rgba(255,209,102,.42)} +.task-card.tomorrow{border-color:rgba(142,202,230,.35)} + +.compact-card, +.card{ + padding:10px 11px; + border-left:3px solid var(--accent); +} +.compact-card.astronomy,.card.astronomy{border-left-color:var(--space)} +.compact-card.homelab,.compact-card.smart_home,.compact-card.radio, +.card.homelab,.card.smart_home,.card.radio{border-left-color:var(--home)} +.compact-card.local,.card.local{border-left-color:var(--local)} +.compact-card.security,.card.security{border-left-color:var(--warn)} +.compact-card .meta, +.card .meta{ + margin-bottom:5px; + display:flex; + flex-wrap:wrap; + gap:6px; + font-size:.72rem; +} +.compact-card .meta span, +.card .meta span{ + display:inline-flex; + align-items:center; + padding:2px 7px; + border-radius:999px; + background:rgba(255,255,255,.035); + border:1px solid var(--line-soft); +} +.compact-card h3, +.card h3{ + margin:0 0 4px; + font-size:.94rem; + line-height:1.2; +} +.compact-card p, +.card p{ + margin:0; + color:#c7d1e2; + font-size:.83rem; + line-height:1.28; + display:-webkit-box; + -webkit-line-clamp:2; + -webkit-box-orient:vertical; + overflow:hidden; +} +.compact-card:nth-child(n+7){display:none} +.feedback{display:flex;gap:6px;justify-content:flex-end;margin-top:7px} +.feedback-btn{padding:2px 8px;background:transparent} +.feedback-btn[aria-pressed='true']{font-weight:700} +.feedback-btn.active-up{border-color:rgba(94,231,135,.65);background:rgba(94,231,135,.12);color:#c8ffd9} +.feedback-btn.active-down{border-color:rgba(255,107,107,.6);background:rgba(255,107,107,.12);color:#ffd9d9} +.topic-liked{box-shadow:0 0 0 1px rgba(94,231,135,.45),var(--shadow)} + +.forecast-panel{padding:0} +.forecast-card{padding:11px 12px;background:rgba(255,255,255,.035)} +.forecast-day{color:#b7c0ff;font-size:.75rem;text-transform:uppercase;letter-spacing:.08em} +.forecast-card strong{font-size:.95rem} +.forecast-card em{font-style:normal;color:var(--muted);font-size:.82rem} + +.badge{ + display:inline-block; + margin-left:.35rem; + padding:2px 7px; + border-radius:999px; + background:var(--panel-raise); + color:#b7c0ff; + font-size:.78rem; +} +.paperless-grid{display:grid;grid-template-columns:1fr;gap:8px} +.paperless-card{padding:10px 11px;border-left:3px solid var(--accent)} +.paperless-card.warn{border-left-color:var(--warn)} +.paperless-card.info{border-left-color:var(--local)} +.paperless-card.ok{border-left-color:var(--ok)} +.paperless-card.needs-attention{box-shadow:0 0 0 1px rgba(255,209,102,.25),var(--shadow)} +.paperless-head{display:flex;justify-content:space-between;gap:10px;align-items:flex-start} +.paperless-card h4{margin:.3rem 0;font-size:.92rem;line-height:1.18} +.paperless-card .summary{ + margin:.2rem 0; + color:#d7e3f5; + font-size:.84rem; + line-height:1.26; + display:-webkit-box; + -webkit-line-clamp:2; + -webkit-box-orient:vertical; + overflow:hidden; +} +.paperless-meta, +.paperless-reason{margin:.2rem 0;color:var(--muted);font-size:.76rem;line-height:1.18} +.paperless-card .link{display:inline-block;color:#b7c0ff;text-decoration:none} +.paperless-card .link:hover{text-decoration:underline} +.paperless-actions{display:flex;gap:6px;flex-wrap:wrap;align-items:center;justify-content:flex-end} +.paperless-btn{padding:5px 8px} +.paperless-btn.good{border-color:rgba(94,231,135,.45);color:#c8ffd9} +.paperless-acked, +.paperless-more{margin-top:10px;border-top:1px solid var(--line-soft);padding-top:10px} +.paperless-acked summary, +.paperless-more summary{cursor:pointer;color:var(--muted);font-size:.84rem} +.urgency{font-weight:700} +.urgency.warn{color:var(--warn)} +.urgency.info{color:var(--local)} +.urgency.ok{color:var(--ok)} + +section header{display:flex;justify-content:space-between;gap:12px;align-items:end} +section header p{margin:0} +ul{margin:0} + +footer{ + margin:14px 0 6px; + padding:0 2px; + font-size:.8rem; + line-height:1.4; +} +footer p{margin:.25rem 0} + +.dashboard-toast{ + position:fixed; + right:16px; + bottom:16px; + z-index:1000; + max-width:min(420px,calc(100vw - 24px)); + background:rgba(8,9,10,.97); + border:1px solid var(--line); + border-left:3px solid var(--accent); + border-radius:14px; + padding:12px 14px; + color:var(--text); + box-shadow:0 16px 40px rgba(0,0,0,.36); +} +.dashboard-toast[hidden]{display:none} +.toast-action{margin-left:10px;background:transparent;border:0;color:#b7c0ff;font:inherit;font-weight:700;cursor:pointer;padding:0} + +@media (max-width:1180px){ + .dashboard-shell{grid-template-columns:1fr} + .secondary-column{order:-1} + .focus-grid,.summary-grid,.forecast-grid,.metrics.metrics-compact{grid-template-columns:repeat(2,minmax(0,1fr))} +} + +@media (max-width:920px){ + .hero{grid-template-columns:1fr} + .status,.metrics,.metrics.metrics-compact,.focus-grid,.summary-grid,.news-list,.grid,.forecast-grid{grid-template-columns:1fr} +} + +@media (max-width:720px){ + main{padding:12px} + .hero{padding:14px} + .collapsible > summary{padding:11px 12px} + .collapsible-body{padding:0 12px 12px} + .paperless-head{display:block} + .paperless-actions{justify-content:flex-start;margin-top:8px} + section header{display:block} +} diff --git a/identity/.env.example b/identity/.env.example new file mode 100644 index 0000000..7e4838a --- /dev/null +++ b/identity/.env.example @@ -0,0 +1,10 @@ +TZ=America/Chicago + +AUTHELIA_HOST=auth.paccoco.com +AUTHELIA_STORAGE_POSTGRES_DB=authelia +AUTHELIA_STORAGE_POSTGRES_USER=authelia +AUTHELIA_STORAGE_POSTGRES_PASSWORD=CHANGE_ME +AUTHELIA_STORAGE_ENCRYPTION_KEY=CHANGE_ME +AUTHELIA_SESSION_SECRET=CHANGE_ME +AUTHELIA_SESSION_REDIS_PASSWORD=CHANGE_ME +AUTHELIA_IDENTITY_VALIDATION_RESET_PASSWORD_JWT_SECRET=CHANGE_ME diff --git a/identity/README.md b/identity/README.md new file mode 100644 index 0000000..dde2e59 --- /dev/null +++ b/identity/README.md @@ -0,0 +1,65 @@ +# Identity / SSO + +This stack introduces Authelia as the homelab identity plane. + +Current design choices: + +- Authelia only, no LDAP yet +- shared Postgres for durable storage +- shared Redis for session storage +- file-backed users database for initial bootstrap +- Pangolin/Newt exposure at auth.paccoco.com + +Why this shape: + +- it gets a real SSO control plane online without dragging in extra moving parts +- it matches the homelab preference for shared DB infrastructure over SQLite +- file users are fine at this size and can be migrated to LDAP later if needed + +Files: + +- docker-compose.yaml +- .env.example +- authelia/configuration.yml.example +- authelia/users_database.example.yml + +Bootstrap: + +1. Copy this directory to PD at /mnt/docker-ssd/docker/compose/identity +2. Copy .env.example to .env and replace every CHANGE_ME +3. Copy authelia/configuration.yml.example to authelia/configuration.yml +4. Render concrete secrets into `authelia/configuration.yml`; do not leave template placeholders or `${...}` references in the live file. +5. Copy authelia/users_database.example.yml to authelia/users_database.yml +6. Create an Authelia password hash: + + docker run --rm authelia/authelia:latest authelia crypto hash generate argon2 --password 'CHANGE_ME' + +7. Replace the sample hash in users_database.yml +8. Create the authelia database/user in shared Postgres +9. Set `AUTHELIA_SESSION_REDIS_PASSWORD` from the live shared Redis stack on PD. +10. Create a Pangolin resource: + Domain: auth.paccoco.com + Scheme: http + Host: authelia + Port: 9091 +11. Validate and start: + + cd /mnt/docker-ssd/docker/compose/identity + docker compose --env-file .env config + docker compose --env-file .env up -d + docker exec authelia authelia config validate --config /config/configuration.yml + +This repo stage gives you the identity control plane itself. + +OIDC client rollout is intentionally a second pass because it needs: + +- a real signing-key strategy +- per-app client IDs and secrets +- app-specific callback URLs verified against live domains + +First client recommendation: + +- Grafana via Generic OAuth against Authelia OIDC +- Keep Grafana local-admin login enabled for the first cutover so recovery is easy if the public auth route or claims mapping is wrong + +Do not try to wire every app at once. Get Authelia healthy first, then cut services over one at a time. diff --git a/identity/authelia/configuration.yml.example b/identity/authelia/configuration.yml.example new file mode 100644 index 0000000..eb30238 --- /dev/null +++ b/identity/authelia/configuration.yml.example @@ -0,0 +1,103 @@ +# Render concrete values into configuration.yml on PD. +# Do not rely on runtime env interpolation for this file. +theme: auto +default_2fa_method: totp + +server: + address: tcp://0.0.0.0:9091 + +log: + level: info + +totp: + issuer: paccoco.com + +authentication_backend: + file: + path: /config/users_database.yml + watch: false + password: + algorithm: argon2 + +access_control: + default_policy: deny + rules: + - domain: + - '*.paccoco.com' + policy: one_factor + +session: + secret: 'CHANGE_ME_SESSION_SECRET' + name: authelia_session + same_site: lax + expiration: 8h + inactivity: 1h + remember_me: 30d + redis: + host: shared-redis + port: 6379 + password: 'CHANGE_ME_REDIS_PASSWORD' + cookies: + - domain: paccoco.com + authelia_url: https://auth.paccoco.com + default_redirection_url: https://grafana.paccoco.com + +storage: + encryption_key: 'CHANGE_ME_STORAGE_ENCRYPTION_KEY' + postgres: + address: tcp://shared-postgres:5432 + database: 'authelia' + schema: public + username: 'authelia' + password: 'CHANGE_ME_POSTGRES_PASSWORD' + +identity_validation: + reset_password: + jwt_secret: 'CHANGE_ME_RESET_PASSWORD_JWT_SECRET' + +notifier: + filesystem: + filename: /config/data/notification.txt + +identity_providers: + oidc: + hmac_secret: 'CHANGE_ME_OIDC_HMAC_SECRET' + jwks: + - key_id: 'grafana' + algorithm: 'RS256' + use: 'sig' + key: | + -----BEGIN PRIVATE KEY----- + CHANGE_ME_OIDC_PRIVATE_KEY + -----END PRIVATE KEY----- + claims_policies: + grafana: + id_token: + - email + - email_verified + - groups + - preferred_username + - name + clients: + - client_id: 'grafana' + client_name: 'Grafana' + client_secret: 'CHANGE_ME_GRAFANA_CLIENT_SECRET' + public: false + authorization_policy: 'one_factor' + require_pkce: true + pkce_challenge_method: 'S256' + redirect_uris: + - 'https://grafana.paccoco.com/login/generic_oauth' + scopes: + - openid + - profile + - email + - groups + response_types: + - code + grant_types: + - authorization_code + access_token_signed_response_alg: 'none' + userinfo_signed_response_alg: 'none' + token_endpoint_auth_method: 'client_secret_basic' + claims_policy: 'grafana' diff --git a/identity/authelia/users_database.example.yml b/identity/authelia/users_database.example.yml new file mode 100644 index 0000000..5817c7b --- /dev/null +++ b/identity/authelia/users_database.example.yml @@ -0,0 +1,8 @@ +users: + john: + disabled: false + displayname: John + email: john@paccoco.com + password: "$argon2id$v=19$m=65536,t=3,p=4$REPLACE$ME" + groups: + - admins diff --git a/identity/docker-compose.yaml b/identity/docker-compose.yaml new file mode 100644 index 0000000..8d244f9 --- /dev/null +++ b/identity/docker-compose.yaml @@ -0,0 +1,30 @@ +name: identity + +services: + authelia: + image: authelia/authelia:latest + container_name: authelia + restart: unless-stopped + ports: + - "9091:9091" + environment: + TZ: ${TZ} + volumes: + - /mnt/docker-ssd/docker/compose/identity/authelia/configuration.yml:/config/configuration.yml:ro + - /mnt/docker-ssd/docker/compose/identity/authelia/users_database.yml:/config/users_database.yml:ro + - /mnt/tank/docker/appdata/authelia:/config/data + healthcheck: + test: ["CMD-SHELL", "wget -qO- http://127.0.0.1:9091/api/health >/dev/null 2>&1 || exit 1"] + interval: 30s + timeout: 10s + retries: 5 + start_period: 45s + networks: + - ix-databases_shared-databases + - pangolin + +networks: + ix-databases_shared-databases: + external: true + pangolin: + external: true diff --git a/infrastructure/hawser-nomad/.env.example b/infrastructure/hawser-nomad/.env.example new file mode 100644 index 0000000..c83d509 --- /dev/null +++ b/infrastructure/hawser-nomad/.env.example @@ -0,0 +1,16 @@ +TZ=America/Chicago + +# Hawser listener for Dockhand Standard mode +HAWSER_PORT=2376 +HAWSER_BIND_ADDRESS=0.0.0.0 +HAWSER_TOKEN=CHANGE_ME +HAWSER_AGENT_NAME=nomad +HAWSER_LOG_LEVEL=info + +# Keep the stack path identical inside and outside the container so Dockhand +# can deploy stacks that use relative bind mounts. +STACKS_DIR=/opt/hawser-nomad/stacks + +# Set to any non-empty value to skip disk-usage collection if NOMAD's mounts +# make df collection too noisy or slow. +SKIP_DF_COLLECTION= diff --git a/infrastructure/hawser-nomad/README.md b/infrastructure/hawser-nomad/README.md new file mode 100644 index 0000000..f6a6ae1 --- /dev/null +++ b/infrastructure/hawser-nomad/README.md @@ -0,0 +1,21 @@ +# NOMAD Hawser Agent + +- **Repo stack path:** `/home/fizzlepoof/repos/truenas-stacks/infrastructure/hawser-nomad` +- **Suggested live stack path:** `/opt/hawser-nomad` +- **Dockhand environment:** existing `NOMAD` Hawser Standard environment on PD +- **Why Standard mode:** NOMAD is on the LAN with a stable IP, so Dockhand can connect directly without the extra Edge-mode WebSocket plumbing + +Bring-up: + +```bash +cd /opt/hawser-nomad +cp .env.example .env +bash bin/prepare_nomad.sh --up +``` + +Notes: + +- Use the Hawser token already configured in Dockhand's `NOMAD` environment. +- Hawser listens on port `2376` by default, matching the existing Dockhand environment entry. +- `STACKS_DIR` is bind-mounted to the same host path inside the container so Dockhand can deploy stacks that use relative bind mounts. +- This is a standalone NOMAD deployment under `/opt`, not part of the core Project N.O.M.A.D. managed stack. diff --git a/infrastructure/hawser-nomad/bin/prepare_nomad.sh b/infrastructure/hawser-nomad/bin/prepare_nomad.sh new file mode 100755 index 0000000..2d8077f --- /dev/null +++ b/infrastructure/hawser-nomad/bin/prepare_nomad.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +STACK_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" +ENV_FILE="${ENV_FILE:-$STACK_DIR/.env}" +DO_UP="${1:-}" + +if [[ ! -f "$ENV_FILE" ]]; then + echo "Missing $ENV_FILE - copy .env.example to .env and fill in real values first." >&2 + exit 1 +fi + +set -a +# shellcheck disable=SC1090 +source "$ENV_FILE" +set +a + +mkdir -p "$STACKS_DIR" +docker compose --env-file "$ENV_FILE" -f "$STACK_DIR/docker-compose.yaml" config >/dev/null + +if [[ "$DO_UP" == "--up" ]]; then + docker compose --env-file "$ENV_FILE" -f "$STACK_DIR/docker-compose.yaml" up -d + docker compose --env-file "$ENV_FILE" -f "$STACK_DIR/docker-compose.yaml" ps +fi diff --git a/infrastructure/hawser-nomad/docker-compose.yaml b/infrastructure/hawser-nomad/docker-compose.yaml new file mode 100644 index 0000000..6d69859 --- /dev/null +++ b/infrastructure/hawser-nomad/docker-compose.yaml @@ -0,0 +1,19 @@ +services: + hawser: + image: ghcr.io/finsys/hawser:latest + container_name: hawser-nomad + restart: unless-stopped + ports: + - "${HAWSER_BIND_ADDRESS:-0.0.0.0}:${HAWSER_PORT:-2376}:2376" + environment: + TZ: ${TZ} + PORT: "2376" + BIND_ADDRESS: ${HAWSER_BIND_ADDRESS} + TOKEN: ${HAWSER_TOKEN} + AGENT_NAME: ${HAWSER_AGENT_NAME} + LOG_LEVEL: ${HAWSER_LOG_LEVEL} + STACKS_DIR: ${STACKS_DIR} + SKIP_DF_COLLECTION: ${SKIP_DF_COLLECTION} + volumes: + - /var/run/docker.sock:/var/run/docker.sock + - ${STACKS_DIR}:${STACKS_DIR} diff --git a/monitoring/.env.example b/monitoring/.env.example index 247522c..bb044b1 100644 --- a/monitoring/.env.example +++ b/monitoring/.env.example @@ -4,3 +4,21 @@ TZ=America/New_York GF_ADMIN_USER=admin GF_ADMIN_PASS=CHANGE_ME GF_HOST=grafana.paccoco.com +LOKI_HOST=loki.paccoco.com +AUTHELIA_HOST=auth.paccoco.com +GF_AUTH_GENERIC_OAUTH_ENABLED=true +GF_AUTH_GENERIC_OAUTH_NAME=Authelia +GF_AUTH_GENERIC_OAUTH_ICON=signin +GF_AUTH_GENERIC_OAUTH_CLIENT_ID=grafana +GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET=CHANGE_ME +GF_AUTH_GENERIC_OAUTH_SCOPES=openid profile email groups +GF_AUTH_GENERIC_OAUTH_EMPTY_SCOPES=false +GF_AUTH_GENERIC_OAUTH_USE_PKCE=true +GF_AUTH_GENERIC_OAUTH_ALLOW_SIGN_UP=true +GF_AUTH_GENERIC_OAUTH_AUTO_LOGIN=false +GF_AUTH_GENERIC_OAUTH_LOGIN_ATTRIBUTE_PATH=preferred_username +GF_AUTH_GENERIC_OAUTH_NAME_ATTRIBUTE_PATH=name +GF_AUTH_GENERIC_OAUTH_EMAIL_ATTRIBUTE_PATH=email +GF_AUTH_GENERIC_OAUTH_ROLE_ATTRIBUTE_PATH=contains(groups[*], 'admins') && 'Admin' || 'Viewer' +GF_AUTH_GENERIC_OAUTH_ALLOW_ASSIGN_GRAFANA_ADMIN=true +GF_AUTH_DISABLE_LOGIN_FORM=false diff --git a/monitoring/DEPLOY.md b/monitoring/DEPLOY.md index 9ca3191..77d66f6 100644 --- a/monitoring/DEPLOY.md +++ b/monitoring/DEPLOY.md @@ -1,118 +1,105 @@ -# Phase 6 — Grafana + Prometheus Deployment on PD +# Monitoring Deployment on PD -> Adapted from HOMELAB_BUILDOUT_PLAN.md Phase 6, changed host from N.O.M.A.D. to PlausibleDeniability. +This stack now includes: -## 1. Scaffold Directories +- Prometheus +- Grafana +- Loki +- Promtail +- node-exporter +- cAdvisor -SSH into PD (or run locally): +Source-of-truth path: + +- /mnt/docker-ssd/docker/compose/monitoring + +## 1. Stage repo files on PD + +Make sure the monitoring directory from this repo is present on PD at: + +- /mnt/docker-ssd/docker/compose/monitoring + +## 2. Prepare runtime directories + +Create the required appdata paths on PD: ```bash -# Stack directory -sudo mkdir -p /opt/monitoring/provisioning/datasources -sudo mkdir -p /opt/monitoring/provisioning/dashboards +mkdir -p /mnt/tank/docker/appdata/prometheus +mkdir -p /mnt/tank/docker/appdata/grafana/dashboards +mkdir -p /mnt/tank/docker/appdata/loki +mkdir -p /mnt/tank/docker/appdata/promtail -# Prometheus TSDB on tank (sequential writes, grows with retention) -sudo mkdir -p /mnt/tank/docker/appdata/prometheus -sudo chown 65534:65534 /mnt/tank/docker/appdata/prometheus - -# Grafana data on tank -sudo mkdir -p /mnt/tank/docker/appdata/grafana -sudo chown 472:472 /mnt/tank/docker/appdata/grafana +chown 65534:65534 /mnt/tank/docker/appdata/prometheus +chown 472:472 /mnt/tank/docker/appdata/grafana +chown 472:472 /mnt/tank/docker/appdata/grafana/dashboards ``` -## 2. Copy Files to PD - -Copy everything from this `monitoring/` folder to `/opt/monitoring/` on PD: +## 3. Create the live env file ```bash -# From your local machine (adjust source path as needed) -scp -r monitoring/* pd:/opt/monitoring/ - -# Or if working directly on PD, copy files into place: -# docker-compose.yaml → /opt/monitoring/docker-compose.yaml -# prometheus.yml → /opt/monitoring/prometheus.yml -# provisioning/ → /opt/monitoring/provisioning/ -# .env.example → /opt/monitoring/.env.example -``` - -## 3. Create .env - -```bash -cd /opt/monitoring +cd /mnt/docker-ssd/docker/compose/monitoring cp .env.example .env -nano .env -# Set GF_ADMIN_PASS=$(openssl rand -hex 16) +chmod 600 .env ``` -## 4. Pangolin Configuration +Set a real Grafana password before deploy. +Set a real Grafana OIDC client secret before enabling Authelia login. -In Pangolin dashboard, create a new resource using PD's existing Newt: +## 4. Pangolin exposure -| Field | Value | -|--------|--------------------------| -| Domain | `grafana.paccoco.com` | -| Scheme | `http` | -| Host | `grafana` | -| Port | `3000` | +Create a Pangolin resource for Grafana: -Since Grafana joins the `pangolin` network, PD's Newt can reach it by container name. +- Domain: grafana.paccoco.com +- Scheme: http +- Host: grafana +- Port: 3000 -## 5. Validate & Deploy +If Grafana is moving to Authelia OIDC, Pangolin auth in front of the Grafana resource should be disabled after OIDC is verified. Otherwise users get double-auth prompts for no good reason. + +Loki stays internal-only on PD. Do not publish its host port unless you intentionally move or retire any service already using `3100`. + +## 5. Validate and deploy ```bash -cd /opt/monitoring +cd /mnt/docker-ssd/docker/compose/monitoring docker compose --env-file .env config docker compose --env-file .env up -d ``` -## 6. Post-Deploy Verification +Or run the helper: ```bash -# All three containers running? -docker ps --filter name=prometheus --filter name=grafana --filter name=node-exporter - -# Prometheus scraping targets? -curl -s http://10.5.1.6:9090/api/v1/targets | python3 -m json.tool | head -40 - -# Grafana UI accessible? -curl -s -o /dev/null -w "%{http_code}" http://10.5.1.6:3000/ -# Expected: 200 or 302 - -# Node exporter metrics flowing? -curl -s http://10.5.1.6:9100/metrics | head -10 +cd /mnt/docker-ssd/docker/compose/monitoring +./deploy.sh ``` -> Remember: use `10.5.1.6` not `localhost` for health checks on TrueNAS Scale. +## 6. Post-deploy verification -## 7. Recommended Dashboard Imports +```bash +docker ps --filter name=prometheus --filter name=grafana --filter name=loki --filter name=promtail --filter name=node-exporter --filter name=cadvisor -In Grafana → Dashboards → New → Import → Enter ID: +curl -fsS http://10.5.1.6:9090/-/ready +curl -fsS http://10.5.1.6:9100/metrics | head -10 +curl -fsS http://10.5.1.6:9090/api/v1/targets | python3 -m json.tool | head -60 +sudo docker inspect -f '{{.State.Status}}' loki +``` -| Dashboard | ID | Purpose | -|------------------------|------|-----------------------------------| -| Node Exporter Full | 1860 | PD system metrics | -| Docker Container Stats | 893 | Container resource usage | -| Netdata via Prometheus | search | PD and Serenity system metrics | +In Grafana, confirm: -## 8. Grafana → n8n Alert Webhook +- Prometheus datasource is healthy +- Loki datasource is healthy +- Explore can query labels from Loki +- login page shows the `Authelia` OAuth option -In Grafana → Alerting → Contact Points, create: +## 7. What this adds -| Field | Value | -|-------------|--------------------------------------------------| -| Name | `n8n-alerts` | -| Type | Webhook | -| URL | `https://n8n.paccoco.com/webhook/grafana-alert` | -| HTTP Method | POST | - -Suggested alert rules: -- ZFS pool utilization > 85% -- Container memory > 90% of limit -- Host CPU sustained > 90% for 5 minutes -- Disk I/O latency spikes +- host metrics from node-exporter +- PD container metrics from cAdvisor +- PD Docker logs into Loki via Promtail +- Grafana Explore for metrics and logs in one place ## Notes -- **Storage:** Prometheus TSDB and Grafana data are on tank (plenty of headroom). 90-day retention is configured. Prometheus writes sequentially so tank performs fine here. -- **N.O.M.A.D. scraping:** The N.O.M.A.D. target in prometheus.yml is commented out. Uncomment once Netdata or node-exporter is running on N.O.M.A.D. at 10.5.1.16. -- **Image versions:** Prometheus v3.4.0, Grafana 13.0.1, node-exporter v1.9.0 — verify these are still current before deploying. +- Promtail is intentionally used instead of Alloy here because it is simpler to validate and recover on this host. +- cAdvisor currently covers PD containers only. If you want NOMAD container metrics too, deploy a separate exporter on NOMAD instead of pretending PD can see across hosts. +- Prometheus remains explicitly pinned to the v3 line because latest has been sloppy about major-version intent. diff --git a/monitoring/alerts.yml b/monitoring/alerts.yml new file mode 100644 index 0000000..ea4bf31 --- /dev/null +++ b/monitoring/alerts.yml @@ -0,0 +1,56 @@ +groups: + - name: homelab-core + rules: + - alert: ScrapeTargetDown + expr: up == 0 + for: 5m + labels: + severity: warning + annotations: + summary: "Prometheus target down: {{ $labels.job }}" + description: "Target {{ $labels.instance }} for job {{ $labels.job }} has been down for more than 5 minutes." + + - alert: PDMemoryPressureHigh + expr: 100 * (1 - (node_memory_MemAvailable_bytes{job="pd-node"} / node_memory_MemTotal_bytes{job="pd-node"})) > 90 + for: 10m + labels: + severity: warning + annotations: + summary: "PD memory usage is high" + description: "PlausibleDeniability memory usage has been above 90% for more than 10 minutes." + + - alert: PDFilesystemUsageHigh + expr: 100 * (1 - (node_filesystem_avail_bytes{job="pd-node", mountpoint="/", fstype!~"tmpfs|ramfs|overlay|squashfs|nfs4"} / node_filesystem_size_bytes{job="pd-node", mountpoint="/", fstype!~"tmpfs|ramfs|overlay|squashfs|nfs4"})) > 85 + for: 15m + labels: + severity: warning + annotations: + summary: "PD root filesystem is filling up" + description: "PlausibleDeniability root filesystem has been above 85% for more than 15 minutes." + + - alert: PDContainerRecentlyRestarted + expr: changes(container_start_time_seconds{job="pd-cadvisor", name!=""}[30m]) > 0 + for: 5m + labels: + severity: info + annotations: + summary: "PD container restarted recently" + description: "Container {{ $labels.name }} restarted within the last 30 minutes." + + - alert: PDRestoreVerificationFailed + expr: pd_restore_verification_success == 0 + for: 15m + labels: + severity: warning + annotations: + summary: "PD restore verification failed" + description: "The last scheduled restore verification on PlausibleDeniability failed." + + - alert: PDRestoreVerificationStale + expr: time() - pd_restore_verification_last_success_timestamp_seconds > 8640000 + for: 1h + labels: + severity: warning + annotations: + summary: "PD restore verification is stale" + description: "No successful PD restore verification has been recorded in the last 100 days." diff --git a/monitoring/dashboards/homelab-overview.json b/monitoring/dashboards/homelab-overview.json new file mode 100644 index 0000000..52f0c89 --- /dev/null +++ b/monitoring/dashboards/homelab-overview.json @@ -0,0 +1,714 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": null, + "links": [], + "panels": [ + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 0, + "y": 0 + }, + "id": 1, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value_and_name" + }, + "pluginVersion": "13.0.1", + "targets": [ + { + "editorMode": "code", + "expr": "max(up{host=\"plausible-deniability\"})", + "instant": true, + "legendFormat": "PD", + "range": false, + "refId": "A" + } + ], + "title": "PD Up", + "type": "stat" + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 4, + "y": 0 + }, + "id": 2, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value_and_name" + }, + "pluginVersion": "13.0.1", + "targets": [ + { + "editorMode": "code", + "expr": "max(up{host=\"nomad\"})", + "instant": true, + "legendFormat": "NOMAD", + "range": false, + "refId": "A" + } + ], + "title": "NOMAD Up", + "type": "stat" + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 8, + "y": 0 + }, + "id": 3, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value_and_name" + }, + "pluginVersion": "13.0.1", + "targets": [ + { + "editorMode": "code", + "expr": "max(up{host=\"serenity\"})", + "instant": true, + "legendFormat": "Serenity", + "range": false, + "refId": "A" + } + ], + "title": "Serenity Up", + "type": "stat" + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 1 + }, + { + "color": "red", + "value": 2 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 12, + "y": 0 + }, + "id": 4, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value_and_name" + }, + "pluginVersion": "13.0.1", + "targets": [ + { + "editorMode": "code", + "expr": "count(up) - sum(up)", + "instant": true, + "legendFormat": "Down", + "range": false, + "refId": "A" + } + ], + "title": "Targets Down", + "type": "stat" + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 16, + "y": 0 + }, + "id": 5, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value_and_name" + }, + "pluginVersion": "13.0.1", + "targets": [ + { + "editorMode": "code", + "expr": "rate(prometheus_tsdb_head_samples_appended_total[5m])", + "instant": true, + "legendFormat": "samples/s", + "range": false, + "refId": "A" + } + ], + "title": "Samples / Sec", + "type": "stat" + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 20, + "y": 0 + }, + "id": 6, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value_and_name" + }, + "pluginVersion": "13.0.1", + "targets": [ + { + "editorMode": "code", + "expr": "prometheus_tsdb_head_series", + "instant": true, + "legendFormat": "series", + "range": false, + "refId": "A" + } + ], + "title": "Head Series", + "type": "stat" + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "drawStyle": "line", + "fillOpacity": 15, + "lineInterpolation": "smooth", + "lineWidth": 2, + "showPoints": "never" + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 4 + }, + "id": 7, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "13.0.1", + "targets": [ + { + "editorMode": "code", + "expr": "100 - (avg by (host) (rate(node_cpu_seconds_total{host=~\"plausible-deniability|nomad\", mode=\"idle\"}[5m])) * 100)", + "legendFormat": "{{host}}", + "range": true, + "refId": "A" + }, + { + "editorMode": "code", + "expr": "100 - avg by (host) (netdata_system_cpu_percentage_average{host=\"serenity\", dimension=\"idle\"})", + "legendFormat": "{{host}}", + "range": true, + "refId": "B" + } + ], + "title": "CPU Usage %", + "type": "timeseries" + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisPlacement": "auto", + "drawStyle": "line", + "fillOpacity": 15, + "lineInterpolation": "smooth", + "lineWidth": 2, + "showPoints": "never" + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 4 + }, + "id": 8, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "13.0.1", + "targets": [ + { + "editorMode": "code", + "expr": "100 * (1 - (node_memory_MemAvailable_bytes{host=~\"plausible-deniability|nomad\"} / node_memory_MemTotal_bytes{host=~\"plausible-deniability|nomad\"}))", + "legendFormat": "{{host}}", + "range": true, + "refId": "A" + }, + { + "editorMode": "code", + "expr": "100 * (netdata_system_ram_MiB_average{host=\"serenity\", dimension=\"used\"} / (netdata_system_ram_MiB_average{host=\"serenity\", dimension=\"used\"} + netdata_system_ram_MiB_average{host=\"serenity\", dimension=\"free\"} + netdata_system_ram_MiB_average{host=\"serenity\", dimension=\"cached\"} + netdata_system_ram_MiB_average{host=\"serenity\", dimension=\"buffers\"}))", + "legendFormat": "serenity", + "range": true, + "refId": "B" + } + ], + "title": "Memory Used %", + "type": "timeseries" + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisPlacement": "auto", + "drawStyle": "line", + "fillOpacity": 15, + "lineInterpolation": "smooth", + "lineWidth": 2, + "showPoints": "never" + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 12 + }, + "id": 9, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "13.0.1", + "targets": [ + { + "editorMode": "code", + "expr": "100 * (1 - (node_filesystem_avail_bytes{host=~\"plausible-deniability|nomad\", mountpoint=\"/\", fstype!~\"tmpfs|ramfs|overlay|squashfs|nfs4\"} / node_filesystem_size_bytes{host=~\"plausible-deniability|nomad\", mountpoint=\"/\", fstype!~\"tmpfs|ramfs|overlay|squashfs|nfs4\"}))", + "legendFormat": "{{host}}", + "range": true, + "refId": "A" + }, + { + "editorMode": "code", + "expr": "100 * (sum by (host) (netdata_disk_space_GiB_average{host=\"serenity\", mount_point=\"/\", dimension=\"used\"}) / (sum by (host) (netdata_disk_space_GiB_average{host=\"serenity\", mount_point=\"/\", dimension=\"used\"}) + sum by (host) (netdata_disk_space_GiB_average{host=\"serenity\", mount_point=\"/\", dimension=\"avail\"})))", + "legendFormat": "{{host}}", + "range": true, + "refId": "B" + } + ], + "title": "Root Filesystem Used %", + "type": "timeseries" + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisPlacement": "auto", + "drawStyle": "line", + "fillOpacity": 10, + "lineInterpolation": "smooth", + "lineWidth": 2, + "showPoints": "never" + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 12 + }, + "id": 10, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "13.0.1", + "targets": [ + { + "editorMode": "code", + "expr": "node_load1{host=~\"plausible-deniability|nomad\"}", + "legendFormat": "{{host}}", + "range": true, + "refId": "A" + }, + { + "editorMode": "code", + "expr": "netdata_system_load_load_average{host=\"serenity\", dimension=\"load1\"}", + "legendFormat": "{{host}}", + "range": true, + "refId": "B" + } + ], + "title": "Load 1m", + "type": "timeseries" + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisPlacement": "auto", + "drawStyle": "line", + "fillOpacity": 10, + "lineInterpolation": "smooth", + "lineWidth": 2, + "showPoints": "never" + }, + "unit": "Bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 20 + }, + "id": 11, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "13.0.1", + "targets": [ + { + "editorMode": "code", + "expr": "sum by (host) (rate(node_network_receive_bytes_total{host=~\"plausible-deniability|nomad\", device!~\"lo|veth.*|br-.*|docker.*\"}[5m]))", + "legendFormat": "{{host}}", + "range": true, + "refId": "A" + }, + { + "editorMode": "code", + "expr": "sum by (host) (netdata_net_net_kilobits_persec_average{host=\"serenity\", interface_type=\"real\", dimension=\"received\"}) * 125", + "legendFormat": "{{host}}", + "range": true, + "refId": "B" + } + ], + "title": "Network Receive", + "type": "timeseries" + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 75 + }, + { + "color": "red", + "value": 90 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 20 + }, + "id": 12, + "options": { + "displayMode": "gradient", + "minVizHeight": 16, + "minVizWidth": 8, + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": true + }, + "pluginVersion": "13.0.1", + "targets": [ + { + "editorMode": "code", + "expr": "topk(12, 100 * (1 - node_filesystem_avail_bytes{host=~\"plausible-deniability|nomad\", fstype!~\"tmpfs|ramfs|overlay|squashfs|nfs4\", mountpoint!~\"/var/lib/docker/.+\"} / node_filesystem_size_bytes{host=~\"plausible-deniability|nomad\", fstype!~\"tmpfs|ramfs|overlay|squashfs|nfs4\", mountpoint!~\"/var/lib/docker/.+\"}))", + "instant": true, + "legendFormat": "{{host}} {{mountpoint}}", + "range": false, + "refId": "A" + } + ], + "title": "Top Filesystem Usage (PD + NOMAD)", + "type": "bargauge" + } + ], + "refresh": "30s", + "schemaVersion": 41, + "tags": [ + "homelab", + "prometheus", + "overview" + ], + "templating": { + "list": [] + }, + "time": { + "from": "now-24h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Homelab Overview", + "uid": "homelab-overview", + "version": 1, + "weekStart": "" +} diff --git a/monitoring/dashboards/linux-host-drilldown.json b/monitoring/dashboards/linux-host-drilldown.json new file mode 100644 index 0000000..ef58357 --- /dev/null +++ b/monitoring/dashboards/linux-host-drilldown.json @@ -0,0 +1,1087 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "links": [], + "liveNow": false, + "panels": [ + { + "type": "stat", + "title": "Target Up", + "gridPos": { + "h": 4, + "w": 4, + "x": 0, + "y": 0 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value_and_name" + }, + "targets": [ + { + "editorMode": "code", + "expr": "max(up{host=\"$host\"})", + "instant": true, + "legendFormat": "$host", + "range": false, + "refId": "A" + } + ] + }, + { + "type": "stat", + "title": "CPU Busy %", + "gridPos": { + "h": 4, + "w": 4, + "x": 4, + "y": 0 + }, + "fieldConfig": { + "defaults": { + "unit": "percent", + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 70 + }, + { + "color": "red", + "value": 90 + } + ] + } + }, + "overrides": [] + }, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value_and_name" + }, + "targets": [ + { + "editorMode": "code", + "expr": "100 - (avg(rate(node_cpu_seconds_total{host=\"$host\", mode=\"idle\"}[5m])) * 100)", + "instant": true, + "legendFormat": "CPU Busy", + "range": false, + "refId": "A" + } + ] + }, + { + "type": "stat", + "title": "Memory Used %", + "gridPos": { + "h": 4, + "w": 4, + "x": 8, + "y": 0 + }, + "fieldConfig": { + "defaults": { + "unit": "percent", + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 75 + }, + { + "color": "red", + "value": 90 + } + ] + } + }, + "overrides": [] + }, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value_and_name" + }, + "targets": [ + { + "editorMode": "code", + "expr": "100 * (1 - (node_memory_MemAvailable_bytes{host=\"$host\"} / node_memory_MemTotal_bytes{host=\"$host\"}))", + "instant": true, + "legendFormat": "Memory", + "range": false, + "refId": "A" + } + ] + }, + { + "type": "stat", + "title": "Root Used %", + "gridPos": { + "h": 4, + "w": 4, + "x": 12, + "y": 0 + }, + "fieldConfig": { + "defaults": { + "unit": "percent", + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 75 + }, + { + "color": "red", + "value": 90 + } + ] + } + }, + "overrides": [] + }, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value_and_name" + }, + "targets": [ + { + "editorMode": "code", + "expr": "100 * (1 - (node_filesystem_avail_bytes{host=\"$host\", mountpoint=\"/\", fstype!~\"tmpfs|ramfs|overlay|squashfs|nfs4\"} / node_filesystem_size_bytes{host=\"$host\", mountpoint=\"/\", fstype!~\"tmpfs|ramfs|overlay|squashfs|nfs4\"}))", + "instant": true, + "legendFormat": "/", + "range": false, + "refId": "A" + } + ] + }, + { + "type": "stat", + "title": "Load 1m", + "gridPos": { + "h": 4, + "w": 4, + "x": 16, + "y": 0 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 4 + }, + { + "color": "red", + "value": 8 + } + ] + } + }, + "overrides": [] + }, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value_and_name" + }, + "targets": [ + { + "editorMode": "code", + "expr": "node_load1{host=\"$host\"}", + "instant": true, + "legendFormat": "load1", + "range": false, + "refId": "A" + } + ] + }, + { + "type": "stat", + "title": "Uptime", + "gridPos": { + "h": 4, + "w": 4, + "x": 20, + "y": 0 + }, + "fieldConfig": { + "defaults": { + "unit": "s", + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value_and_name" + }, + "targets": [ + { + "editorMode": "code", + "expr": "node_time_seconds{host=\"$host\"} - node_boot_time_seconds{host=\"$host\"}", + "instant": true, + "legendFormat": "uptime", + "range": false, + "refId": "A" + } + ] + }, + { + "type": "timeseries", + "title": "CPU Busy Over Time", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 4 + }, + "fieldConfig": { + "defaults": { + "unit": "percent", + "min": 0, + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 12, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + } + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "editorMode": "code", + "expr": "100 - (avg(rate(node_cpu_seconds_total{host=\"$host\", mode=\"idle\"}[5m])) * 100)", + "legendFormat": "CPU Busy", + "range": true, + "refId": "A" + }, + { + "editorMode": "code", + "expr": "avg(rate(node_cpu_seconds_total{host=\"$host\", mode=\"iowait\"}[5m])) * 100", + "legendFormat": "IO Wait", + "range": true, + "refId": "B" + } + ] + }, + { + "type": "timeseries", + "title": "Memory Pressure", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 4 + }, + "fieldConfig": { + "defaults": { + "unit": "percent", + "min": 0, + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 12, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + } + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "editorMode": "code", + "expr": "100 * (1 - (node_memory_MemAvailable_bytes{host=\"$host\"} / node_memory_MemTotal_bytes{host=\"$host\"}))", + "legendFormat": "Used %", + "range": true, + "refId": "A" + } + ] + }, + { + "type": "timeseries", + "title": "Load Averages", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 12 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 12, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + } + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "editorMode": "code", + "expr": "node_load1{host=\"$host\"}", + "legendFormat": "load1", + "range": true, + "refId": "A" + }, + { + "editorMode": "code", + "expr": "node_load5{host=\"$host\"}", + "legendFormat": "load5", + "range": true, + "refId": "B" + }, + { + "editorMode": "code", + "expr": "node_load15{host=\"$host\"}", + "legendFormat": "load15", + "range": true, + "refId": "C" + } + ] + }, + { + "type": "timeseries", + "title": "Network Throughput", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 12 + }, + "fieldConfig": { + "defaults": { + "unit": "Bps", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 12, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + } + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "editorMode": "code", + "expr": "sum by (device) (rate(node_network_receive_bytes_total{host=\"$host\", device=~\"$device\"}[5m]))", + "legendFormat": "{{device}} rx", + "range": true, + "refId": "A" + }, + { + "editorMode": "code", + "expr": "-sum by (device) (rate(node_network_transmit_bytes_total{host=\"$host\", device=~\"$device\"}[5m]))", + "legendFormat": "{{device}} tx", + "range": true, + "refId": "B" + } + ] + }, + { + "type": "bargauge", + "title": "Filesystem Used %", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 20 + }, + "fieldConfig": { + "defaults": { + "unit": "percent", + "min": 0, + "max": 100, + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 75 + }, + { + "color": "red", + "value": 90 + } + ] + } + }, + "overrides": [] + }, + "options": { + "displayMode": "gradient", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": true, + "text": {} + }, + "targets": [ + { + "editorMode": "code", + "expr": "100 * (1 - node_filesystem_avail_bytes{host=\"$host\", mountpoint=~\"$mountpoint\", fstype!~\"tmpfs|ramfs|overlay|squashfs|nfs4\"} / node_filesystem_size_bytes{host=\"$host\", mountpoint=~\"$mountpoint\", fstype!~\"tmpfs|ramfs|overlay|squashfs|nfs4\"})", + "legendFormat": "{{mountpoint}}", + "range": true, + "refId": "A" + } + ] + }, + { + "type": "timeseries", + "title": "Disk Throughput", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 20 + }, + "fieldConfig": { + "defaults": { + "unit": "Bps", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 12, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + } + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "editorMode": "code", + "expr": "sum by (device) (rate(node_disk_read_bytes_total{host=\"$host\", device!~\"loop.*|ram.*|fd.*\"}[5m]))", + "legendFormat": "{{device}} read", + "range": true, + "refId": "A" + }, + { + "editorMode": "code", + "expr": "-sum by (device) (rate(node_disk_written_bytes_total{host=\"$host\", device!~\"loop.*|ram.*|fd.*\"}[5m]))", + "legendFormat": "{{device}} write", + "range": true, + "refId": "B" + } + ] + }, + { + "type": "table", + "title": "Top Filesystems", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 28 + }, + "fieldConfig": { + "defaults": { + "unit": "percent", + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 75 + }, + { + "color": "red", + "value": 90 + } + ] + } + }, + "overrides": [] + }, + "options": { + "cellHeight": "sm", + "footer": { + "enablePagination": true, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [ + { + "desc": true, + "displayName": "Value" + } + ] + }, + "targets": [ + { + "editorMode": "code", + "expr": "topk(12, 100 * (1 - node_filesystem_avail_bytes{host=\"$host\", fstype!~\"tmpfs|ramfs|overlay|squashfs|nfs4\", mountpoint!~\"/var/lib/docker/.+|/tmp/.+\"} / node_filesystem_size_bytes{host=\"$host\", fstype!~\"tmpfs|ramfs|overlay|squashfs|nfs4\", mountpoint!~\"/var/lib/docker/.+|/tmp/.+\"}))", + "format": "table", + "instant": true, + "refId": "A" + } + ] + }, + { + "type": "timeseries", + "title": "Memory Breakdown", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 28 + }, + "fieldConfig": { + "defaults": { + "unit": "bytes", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 12, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + } + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "editorMode": "code", + "expr": "node_memory_MemTotal_bytes{host=\"$host\"} - node_memory_MemAvailable_bytes{host=\"$host\"}", + "legendFormat": "used", + "range": true, + "refId": "A" + }, + { + "editorMode": "code", + "expr": "node_memory_Cached_bytes{host=\"$host\"}", + "legendFormat": "cached", + "range": true, + "refId": "B" + }, + { + "editorMode": "code", + "expr": "node_memory_Buffers_bytes{host=\"$host\"}", + "legendFormat": "buffers", + "range": true, + "refId": "C" + }, + { + "editorMode": "code", + "expr": "node_memory_MemAvailable_bytes{host=\"$host\"}", + "legendFormat": "available", + "range": true, + "refId": "D" + } + ] + } + ], + "refresh": "30s", + "schemaVersion": 41, + "tags": [ + "homelab", + "linux", + "node-exporter" + ], + "templating": { + "list": [ + { + "current": { + "selected": true, + "text": "nomad", + "value": "nomad" + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values(up{job=~\"pd-node|nomad-node\"}, host)", + "hide": 0, + "includeAll": false, + "label": "Host", + "multi": false, + "name": "host", + "options": [], + "query": { + "query": "label_values(up{job=~\"pd-node|nomad-node\"}, host)", + "refId": "Prometheus-host-Variable-Query" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "current": { + "selected": false, + "text": [ + "/" + ], + "value": [ + "/" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values(node_filesystem_size_bytes{host=\"$host\",fstype!~\"tmpfs|ramfs|overlay|squashfs|nfs4\",mountpoint!~\"/var/lib/docker/.+|/tmp/.+\"}, mountpoint)", + "hide": 0, + "includeAll": true, + "label": "Mountpoint", + "multi": true, + "name": "mountpoint", + "options": [], + "query": { + "query": "label_values(node_filesystem_size_bytes{host=\"$host\",fstype!~\"tmpfs|ramfs|overlay|squashfs|nfs4\",mountpoint!~\"/var/lib/docker/.+|/tmp/.+\"}, mountpoint)", + "refId": "Prometheus-mountpoint-Variable-Query" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "current": { + "selected": false, + "text": [ + "enp5s0" + ], + "value": [ + "enp5s0" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values(node_network_receive_bytes_total{host=\"$host\",device!~\"lo|veth.*|br-.*|docker.*\"}, device)", + "hide": 0, + "includeAll": true, + "label": "Interface", + "multi": true, + "name": "device", + "options": [], + "query": { + "query": "label_values(node_network_receive_bytes_total{host=\"$host\",device!~\"lo|veth.*|br-.*|docker.*\"}, device)", + "refId": "Prometheus-device-Variable-Query" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + } + ] + }, + "time": { + "from": "now-24h", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "Linux Host Drilldown", + "uid": "linux-host-drilldown", + "version": 1, + "weekStart": "" +} diff --git a/monitoring/dashboards/monitoring-stack.json b/monitoring/dashboards/monitoring-stack.json new file mode 100644 index 0000000..0829eab --- /dev/null +++ b/monitoring/dashboards/monitoring-stack.json @@ -0,0 +1,562 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": null, + "links": [], + "panels": [ + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 0, + "y": 0 + }, + "id": 1, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value_and_name" + }, + "targets": [ + { + "editorMode": "code", + "expr": "sum(up)", + "instant": true, + "range": false, + "refId": "A" + } + ], + "title": "Targets Up", + "type": "stat" + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 1 + }, + { + "color": "red", + "value": 2 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 4, + "y": 0 + }, + "id": 2, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "center", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value_and_name" + }, + "targets": [ + { + "editorMode": "code", + "expr": "count(up) - sum(up)", + "instant": true, + "range": false, + "refId": "A" + } + ], + "title": "Targets Down", + "type": "stat" + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 8, + "y": 0 + }, + "id": 3, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "center", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value_and_name" + }, + "targets": [ + { + "editorMode": "code", + "expr": "sum(prometheus_target_scrape_pool_targets)", + "instant": true, + "range": false, + "refId": "A" + } + ], + "title": "Scrape Targets", + "type": "stat" + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 12, + "y": 0 + }, + "id": 4, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value_and_name" + }, + "targets": [ + { + "editorMode": "code", + "expr": "rate(prometheus_tsdb_head_samples_appended_total[5m])", + "instant": true, + "range": false, + "refId": "A" + } + ], + "title": "Samples / Sec", + "type": "stat" + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 16, + "y": 0 + }, + "id": 5, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value_and_name" + }, + "targets": [ + { + "editorMode": "code", + "expr": "prometheus_tsdb_head_series", + "instant": true, + "range": false, + "refId": "A" + } + ], + "title": "Head Series", + "type": "stat" + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 20, + "y": 0 + }, + "id": 6, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value_and_name" + }, + "targets": [ + { + "editorMode": "code", + "expr": "prometheus_tsdb_storage_blocks_bytes", + "instant": true, + "range": false, + "refId": "A" + } + ], + "title": "TSDB Blocks", + "type": "stat" + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisPlacement": "auto", + "drawStyle": "line", + "fillOpacity": 10, + "lineInterpolation": "smooth", + "lineWidth": 2, + "showPoints": "never" + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 4 + }, + "id": 7, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "editorMode": "code", + "expr": "up", + "legendFormat": "{{job}} {{host}}", + "range": true, + "refId": "A" + } + ], + "title": "Target Health", + "type": "timeseries" + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisPlacement": "auto", + "drawStyle": "line", + "fillOpacity": 10, + "lineInterpolation": "smooth", + "lineWidth": 2, + "showPoints": "never" + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 4 + }, + "id": 8, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "editorMode": "code", + "expr": "scrape_duration_seconds{job=~\"pd-node|nomad-node|serenity-netdata\"}", + "legendFormat": "{{job}} {{host}}", + "range": true, + "refId": "A" + } + ], + "title": "Scrape Duration", + "type": "timeseries" + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisPlacement": "auto", + "drawStyle": "line", + "fillOpacity": 10, + "lineInterpolation": "smooth", + "lineWidth": 2, + "showPoints": "never" + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 12 + }, + "id": 9, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "editorMode": "code", + "expr": "scrape_samples_post_metric_relabeling{job=~\"pd-node|nomad-node|serenity-netdata\"}", + "legendFormat": "{{job}} {{host}}", + "range": true, + "refId": "A" + } + ], + "title": "Samples Per Scrape", + "type": "timeseries" + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisPlacement": "auto", + "drawStyle": "line", + "fillOpacity": 10, + "lineInterpolation": "smooth", + "lineWidth": 2, + "showPoints": "never" + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 12 + }, + "id": 10, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "editorMode": "code", + "expr": "rate(prometheus_tsdb_head_samples_appended_total[5m])", + "legendFormat": "samples/s", + "range": true, + "refId": "A" + }, + { + "editorMode": "code", + "expr": "rate(prometheus_http_requests_total[5m])", + "legendFormat": "http req/s", + "range": true, + "refId": "B" + } + ], + "title": "Prometheus Throughput", + "type": "timeseries" + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 20 + }, + "id": 11, + "options": { + "cellHeight": "sm", + "footer": { + "show": false + }, + "showHeader": true + }, + "targets": [ + { + "editorMode": "code", + "expr": "up", + "format": "table", + "instant": true, + "range": false, + "refId": "A" + } + ], + "title": "Current Target States", + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true + }, + "indexByName": { + "Value": 0, + "host": 1, + "instance": 2, + "job": 3 + }, + "renameByName": { + "Value": "up" + } + } + } + ], + "type": "table" + } + ], + "refresh": "30s", + "schemaVersion": 41, + "tags": [ + "homelab", + "prometheus", + "monitoring" + ], + "templating": { + "list": [] + }, + "time": { + "from": "now-24h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Monitoring Stack", + "uid": "monitoring-stack", + "version": 1, + "weekStart": "" +} diff --git a/monitoring/dashboards/pd-containers.json b/monitoring/dashboards/pd-containers.json new file mode 100644 index 0000000..f3da1cf --- /dev/null +++ b/monitoring/dashboards/pd-containers.json @@ -0,0 +1,512 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": null, + "links": [], + "panels": [ + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 0 + }, + "id": 1, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "center", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value_and_name" + }, + "targets": [ + { + "editorMode": "code", + "expr": "count(container_start_time_seconds{job=\"pd-cadvisor\",name!=\"\"})", + "instant": true, + "range": false, + "refId": "A" + } + ], + "title": "Tracked Containers", + "type": "stat" + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 1 + }, + { + "color": "red", + "value": 5 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 6, + "y": 0 + }, + "id": 2, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "center", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value_and_name" + }, + "targets": [ + { + "editorMode": "code", + "expr": "sum(changes(container_start_time_seconds{job=\"pd-cadvisor\",name!=\"\"}[24h]) > 0)", + "instant": true, + "range": false, + "refId": "A" + } + ], + "title": "Containers Restarted (24h)", + "type": "stat" + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "unit": "bytes", + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 1073741824 + }, + { + "color": "red", + "value": 2147483648 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 12, + "y": 0 + }, + "id": 3, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "center", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value_and_name" + }, + "targets": [ + { + "editorMode": "code", + "expr": "max(container_memory_working_set_bytes{job=\"pd-cadvisor\",name!=\"\"})", + "instant": true, + "range": false, + "refId": "A" + } + ], + "title": "Top Container RAM", + "type": "stat" + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "decimals": 2, + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 0.7 + }, + { + "color": "red", + "value": 1.5 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 18, + "y": 0 + }, + "id": 4, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "center", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value_and_name" + }, + "targets": [ + { + "editorMode": "code", + "expr": "max(rate(container_cpu_usage_seconds_total{job=\"pd-cadvisor\",name!=\"\"}[5m]))", + "instant": true, + "range": false, + "refId": "A" + } + ], + "title": "Top Container CPU/sec", + "type": "stat" + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "unit": "bytes", + "color": { + "mode": "palette-classic" + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 4 + }, + "id": 5, + "options": { + "displayMode": "gradient", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": true, + "sizing": "auto" + }, + "targets": [ + { + "editorMode": "code", + "expr": "topk(12, container_memory_working_set_bytes{job=\"pd-cadvisor\",name!=\"\"})", + "legendFormat": "{{name}}", + "refId": "A" + } + ], + "title": "Top Container RAM (PD)", + "type": "bargauge" + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "decimals": 3, + "color": { + "mode": "palette-classic" + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 4 + }, + "id": 6, + "options": { + "displayMode": "gradient", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": true, + "sizing": "auto" + }, + "targets": [ + { + "editorMode": "code", + "expr": "topk(12, rate(container_cpu_usage_seconds_total{job=\"pd-cadvisor\",name!=\"\"}[5m]))", + "legendFormat": "{{name}}", + "refId": "A" + } + ], + "title": "Top Container CPU/sec (PD)", + "type": "bargauge" + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 15, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 4, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 13 + }, + "id": 7, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "editorMode": "code", + "expr": "topk(8, container_memory_working_set_bytes{job=\"pd-cadvisor\",name!=\"\"})", + "legendFormat": "{{name}}", + "refId": "A" + } + ], + "title": "Top RAM Consumers Over Time (PD)", + "type": "timeseries" + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 13 + }, + "id": 8, + "options": { + "showHeader": true, + "sortBy": [ + { + "desc": true, + "displayName": "Value" + } + ] + }, + "targets": [ + { + "editorMode": "code", + "expr": "sort_desc(container_start_time_seconds{job=\"pd-cadvisor\",name!=\"\"})", + "format": "table", + "instant": true, + "legendFormat": "{{name}}", + "refId": "A" + } + ], + "title": "Container Start Times (PD)", + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true, + "__name__": true, + "container_label_com_docker_compose_config_hash": true, + "container_label_com_docker_compose_container_number": true, + "container_label_com_docker_compose_image": true, + "container_label_com_docker_compose_oneoff": true, + "container_label_com_docker_compose_project_config_files": true, + "container_label_com_docker_compose_project_environment_file": true, + "container_label_com_docker_compose_project_working_dir": true, + "container_label_com_docker_compose_replace": true, + "container_label_com_docker_compose_version": true, + "id": true, + "image": true, + "instance": true, + "job": true + }, + "indexByName": { + "name": 0, + "container_label_com_docker_compose_project": 1, + "container_label_com_docker_compose_service": 2, + "Value": 3 + }, + "renameByName": { + "name": "container", + "container_label_com_docker_compose_project": "compose_project", + "container_label_com_docker_compose_service": "compose_service", + "Value": "started_at_epoch" + } + } + } + ], + "type": "table" + } + ], + "refresh": "30s", + "schemaVersion": 39, + "style": "dark", + "tags": [ + "monitoring", + "docker", + "pd" + ], + "templating": { + "list": [] + }, + "time": { + "from": "now-24h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "PD Containers", + "uid": "pd-containers", + "version": 1, + "weekStart": "" +} diff --git a/monitoring/dashboards/serenity-containers.json b/monitoring/dashboards/serenity-containers.json new file mode 100644 index 0000000..4f56f52 --- /dev/null +++ b/monitoring/dashboards/serenity-containers.json @@ -0,0 +1,550 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": null, + "links": [], + "panels": [ + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 0 + }, + "id": 1, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "center", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value_and_name" + }, + "targets": [ + { + "editorMode": "code", + "expr": "count(netdata_docker_container_state_state_average{host=\"serenity\", dimension=\"running\"} == 1)", + "instant": true, + "range": false, + "refId": "A" + } + ], + "title": "Running Containers", + "type": "stat" + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 6, + "y": 0 + }, + "id": 2, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "center", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value_and_name" + }, + "targets": [ + { + "editorMode": "code", + "expr": "count(netdata_docker_container_state_state_average{host=\"serenity\", dimension=\"exited\"} == 1)", + "instant": true, + "range": false, + "refId": "A" + } + ], + "title": "Exited Containers", + "type": "stat" + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 12, + "y": 0 + }, + "id": 3, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "center", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value_and_name" + }, + "targets": [ + { + "editorMode": "code", + "expr": "count(netdata_docker_container_state_state_average{host=\"serenity\", dimension=\"created\"} == 1)", + "instant": true, + "range": false, + "refId": "A" + } + ], + "title": "Created Containers", + "type": "stat" + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 18, + "y": 0 + }, + "id": 4, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value_and_name" + }, + "targets": [ + { + "editorMode": "code", + "expr": "count(netdata_cgroup_mem_usage_MiB_average{host=\"serenity\", dimension=\"ram\"})", + "instant": true, + "range": false, + "refId": "A" + } + ], + "title": "Tracked Cgroups", + "type": "stat" + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 4 + }, + "id": 5, + "options": { + "displayLabels": [ + "name", + "value" + ], + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "pieType": "pie", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "editorMode": "code", + "expr": "count by (dimension) (netdata_docker_container_state_state_average{host=\"serenity\"} == 1)", + "legendFormat": "{{dimension}}", + "range": true, + "refId": "A" + } + ], + "title": "Container States", + "type": "piechart" + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "max": 16384, + "min": 0, + "unit": "decmbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 4 + }, + "id": 6, + "options": { + "displayMode": "gradient", + "minVizHeight": 16, + "minVizWidth": 8, + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": true + }, + "targets": [ + { + "editorMode": "code", + "expr": "topk(12, max by (cgroup_name) (netdata_cgroup_mem_usage_MiB_average{host=\"serenity\", dimension=\"ram\"}))", + "instant": true, + "legendFormat": "{{cgroup_name}}", + "range": false, + "refId": "A" + } + ], + "title": "Top Container RAM (MiB)", + "type": "bargauge" + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "max": 100, + "min": 0, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 4 + }, + "id": 7, + "options": { + "displayMode": "gradient", + "minVizHeight": 16, + "minVizWidth": 8, + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": true + }, + "targets": [ + { + "editorMode": "code", + "expr": "topk(12, max by (cgroup_name) (netdata_cgroup_cpu_percentage_average{host=\"serenity\"}))", + "instant": true, + "legendFormat": "{{cgroup_name}}", + "range": false, + "refId": "A" + } + ], + "title": "Top Container CPU %", + "type": "bargauge" + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 11 + }, + "id": 8, + "options": { + "cellHeight": "sm", + "footer": { + "show": false + }, + "showHeader": true + }, + "targets": [ + { + "editorMode": "code", + "expr": "max by (container_name, image, dimension) (netdata_docker_container_state_state_average{host=\"serenity\", dimension!=\"running\"} == 1)", + "format": "table", + "instant": true, + "range": false, + "refId": "A" + } + ], + "title": "Non-Running Containers", + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true + }, + "indexByName": { + "Value": 0, + "container_name": 1, + "dimension": 2, + "image": 3 + }, + "renameByName": { + "Value": "state_active" + } + } + } + ], + "type": "table" + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisPlacement": "auto", + "drawStyle": "line", + "fillOpacity": 10, + "lineInterpolation": "smooth", + "lineWidth": 2, + "showPoints": "never" + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 20 + }, + "id": 9, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "editorMode": "code", + "expr": "count(netdata_docker_container_state_state_average{host=\"serenity\", dimension=\"running\"} == 1)", + "legendFormat": "running", + "range": true, + "refId": "A" + }, + { + "editorMode": "code", + "expr": "count(netdata_docker_container_state_state_average{host=\"serenity\", dimension!=\"running\"} == 1)", + "legendFormat": "not running", + "range": true, + "refId": "B" + } + ], + "title": "Container Count Over Time", + "type": "timeseries" + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisPlacement": "auto", + "drawStyle": "line", + "fillOpacity": 10, + "lineInterpolation": "smooth", + "lineWidth": 2, + "showPoints": "never" + }, + "unit": "decmbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 20 + }, + "id": 10, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "editorMode": "code", + "expr": "topk(8, max by (cgroup_name) (netdata_cgroup_mem_usage_MiB_average{host=\"serenity\", dimension=\"ram\"}))", + "legendFormat": "{{cgroup_name}}", + "range": true, + "refId": "A" + } + ], + "title": "Top RAM Consumers Over Time", + "type": "timeseries" + } + ], + "refresh": "30s", + "schemaVersion": 41, + "tags": [ + "serenity", + "containers", + "netdata" + ], + "templating": { + "list": [] + }, + "time": { + "from": "now-24h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Serenity Containers", + "uid": "serenity-containers", + "version": 1, + "weekStart": "" +} diff --git a/monitoring/dashboards/serenity-host-health.json b/monitoring/dashboards/serenity-host-health.json new file mode 100644 index 0000000..9c1e904 --- /dev/null +++ b/monitoring/dashboards/serenity-host-health.json @@ -0,0 +1,997 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "links": [], + "liveNow": false, + "panels": [ + { + "type": "stat", + "title": "Serenity Up", + "gridPos": { + "h": 4, + "w": 4, + "x": 0, + "y": 0 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value_and_name" + }, + "targets": [ + { + "editorMode": "code", + "expr": "max(up{host=\"serenity\"})", + "instant": true, + "legendFormat": "serenity", + "range": false, + "refId": "A" + } + ] + }, + { + "type": "stat", + "title": "CPU Busy %", + "gridPos": { + "h": 4, + "w": 4, + "x": 4, + "y": 0 + }, + "fieldConfig": { + "defaults": { + "unit": "percent", + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 70 + }, + { + "color": "red", + "value": 90 + } + ] + } + }, + "overrides": [] + }, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value_and_name" + }, + "targets": [ + { + "editorMode": "code", + "expr": "sum(netdata_system_cpu_percentage_average{host=\"serenity\",dimension!~\"idle|iowait|steal|guest|guest_nice\"})", + "instant": true, + "legendFormat": "CPU Busy", + "range": false, + "refId": "A" + } + ] + }, + { + "type": "stat", + "title": "Memory Used %", + "gridPos": { + "h": 4, + "w": 4, + "x": 8, + "y": 0 + }, + "fieldConfig": { + "defaults": { + "unit": "percent", + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 75 + }, + { + "color": "red", + "value": 90 + } + ] + } + }, + "overrides": [] + }, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value_and_name" + }, + "targets": [ + { + "editorMode": "code", + "expr": "100 * (1 - (netdata_mem_available_MiB_average{host=\"serenity\",dimension=\"avail\"} / scalar(sum(netdata_system_ram_MiB_average{host=\"serenity\",dimension=~\"free|used|cached|buffers\"}))))", + "instant": true, + "legendFormat": "Memory", + "range": false, + "refId": "A" + } + ] + }, + { + "type": "stat", + "title": "Load 1m", + "gridPos": { + "h": 4, + "w": 4, + "x": 12, + "y": 0 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 4 + }, + { + "color": "red", + "value": 8 + } + ] + } + }, + "overrides": [] + }, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value_and_name" + }, + "targets": [ + { + "editorMode": "code", + "expr": "netdata_system_load_load_average{host=\"serenity\",dimension=\"load1\"}", + "instant": true, + "legendFormat": "load1", + "range": false, + "refId": "A" + } + ] + }, + { + "type": "stat", + "title": "Root Used %", + "gridPos": { + "h": 4, + "w": 4, + "x": 16, + "y": 0 + }, + "fieldConfig": { + "defaults": { + "unit": "percent", + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 75 + }, + { + "color": "red", + "value": 90 + } + ] + } + }, + "overrides": [] + }, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value_and_name" + }, + "targets": [ + { + "editorMode": "code", + "expr": "100 * sum(netdata_disk_space_GiB_average{host=\"serenity\",mount_point=\"/\",dimension=\"used\"}) / sum(netdata_disk_space_GiB_average{host=\"serenity\",mount_point=\"/\",dimension=~\"used|avail\"})", + "instant": true, + "legendFormat": "/", + "range": false, + "refId": "A" + } + ] + }, + { + "type": "stat", + "title": "Uptime", + "gridPos": { + "h": 4, + "w": 4, + "x": 20, + "y": 0 + }, + "fieldConfig": { + "defaults": { + "unit": "s", + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value_and_name" + }, + "targets": [ + { + "editorMode": "code", + "expr": "netdata_system_uptime_seconds_average{host=\"serenity\",dimension=\"uptime\"}", + "instant": true, + "legendFormat": "uptime", + "range": false, + "refId": "A" + } + ] + }, + { + "type": "timeseries", + "title": "CPU Busy Over Time", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 4 + }, + "fieldConfig": { + "defaults": { + "unit": "percent", + "min": 0, + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 12, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + } + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "editorMode": "code", + "expr": "sum(netdata_system_cpu_percentage_average{host=\"serenity\",dimension!~\"idle|iowait|steal|guest|guest_nice\"})", + "legendFormat": "CPU Busy", + "range": true, + "refId": "A" + }, + { + "editorMode": "code", + "expr": "sum(netdata_system_cpu_percentage_average{host=\"serenity\",dimension=\"iowait\"})", + "legendFormat": "IO Wait", + "range": true, + "refId": "B" + } + ] + }, + { + "type": "timeseries", + "title": "Load Averages", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 4 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 12, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + } + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "editorMode": "code", + "expr": "netdata_system_load_load_average{host=\"serenity\",dimension=\"load1\"}", + "legendFormat": "load1", + "range": true, + "refId": "A" + }, + { + "editorMode": "code", + "expr": "netdata_system_load_load_average{host=\"serenity\",dimension=\"load5\"}", + "legendFormat": "load5", + "range": true, + "refId": "B" + }, + { + "editorMode": "code", + "expr": "netdata_system_load_load_average{host=\"serenity\",dimension=\"load15\"}", + "legendFormat": "load15", + "range": true, + "refId": "C" + } + ] + }, + { + "type": "timeseries", + "title": "Memory Breakdown", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 12 + }, + "fieldConfig": { + "defaults": { + "unit": "decgbytes", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 12, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + } + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "editorMode": "code", + "expr": "netdata_system_ram_MiB_average{host=\"serenity\",dimension=\"used\"} * 1048576", + "legendFormat": "used", + "range": true, + "refId": "A" + }, + { + "editorMode": "code", + "expr": "netdata_system_ram_MiB_average{host=\"serenity\",dimension=\"cached\"} * 1048576", + "legendFormat": "cached", + "range": true, + "refId": "B" + }, + { + "editorMode": "code", + "expr": "netdata_system_ram_MiB_average{host=\"serenity\",dimension=\"free\"} * 1048576", + "legendFormat": "free", + "range": true, + "refId": "C" + }, + { + "editorMode": "code", + "expr": "netdata_mem_available_MiB_average{host=\"serenity\",dimension=\"avail\"} * 1048576", + "legendFormat": "available", + "range": true, + "refId": "D" + } + ] + }, + { + "type": "timeseries", + "title": "Network Throughput", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 12 + }, + "fieldConfig": { + "defaults": { + "unit": "binbps", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 12, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + } + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "editorMode": "code", + "expr": "sum by (device) (netdata_net_net_kilobits_persec_average{host=\"serenity\",interface_type=\"real\",dimension=\"received\"} * 1024)", + "legendFormat": "{{device}} rx", + "range": true, + "refId": "A" + }, + { + "editorMode": "code", + "expr": "sum by (device) (-1 * netdata_net_net_kilobits_persec_average{host=\"serenity\",interface_type=\"real\",dimension=\"sent\"} * 1024)", + "legendFormat": "{{device}} tx", + "range": true, + "refId": "B" + } + ] + }, + { + "type": "bargauge", + "title": "Filesystem Used %", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 20 + }, + "fieldConfig": { + "defaults": { + "unit": "percent", + "min": 0, + "max": 100, + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 75 + }, + { + "color": "red", + "value": 90 + } + ] + } + }, + "overrides": [] + }, + "options": { + "displayMode": "gradient", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": true, + "text": {} + }, + "targets": [ + { + "editorMode": "code", + "expr": "100 * (netdata_disk_space_GiB_average{host=\"serenity\",dimension=\"used\",filesystem!~\"tmpfs|proc|sysfs|overlay|squashfs|fusectl|cgroup2?|shm\"} / (netdata_disk_space_GiB_average{host=\"serenity\",dimension=\"used\",filesystem!~\"tmpfs|proc|sysfs|overlay|squashfs|fusectl|cgroup2?|shm\"} + netdata_disk_space_GiB_average{host=\"serenity\",dimension=\"avail\",filesystem!~\"tmpfs|proc|sysfs|overlay|squashfs|fusectl|cgroup2?|shm\"}))", + "legendFormat": "{{mount_point}}", + "range": true, + "refId": "A" + } + ] + }, + { + "type": "timeseries", + "title": "Disk Utilization", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 20 + }, + "fieldConfig": { + "defaults": { + "unit": "percent", + "min": 0, + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 12, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + } + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "editorMode": "code", + "expr": "netdata_disk_util___of_time_working_average{host=\"serenity\",device_type=\"physical\"}", + "legendFormat": "{{device}}", + "range": true, + "refId": "A" + } + ] + }, + { + "type": "table", + "title": "Top Disk Busy Devices", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 28 + }, + "fieldConfig": { + "defaults": { + "unit": "percent", + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 75 + }, + { + "color": "red", + "value": 90 + } + ] + } + }, + "overrides": [] + }, + "options": { + "cellHeight": "sm", + "footer": { + "enablePagination": true, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [ + { + "desc": true, + "displayName": "Value" + } + ] + }, + "targets": [ + { + "editorMode": "code", + "expr": "topk(10, netdata_disk_util___of_time_working_average{host=\"serenity\",device_type=\"physical\"})", + "format": "table", + "instant": true, + "refId": "A" + } + ] + }, + { + "type": "timeseries", + "title": "Disk IO Throughput", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 28 + }, + "fieldConfig": { + "defaults": { + "unit": "Bps", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 12, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + } + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "editorMode": "code", + "expr": "sum by (device) (netdata_disk_io_KiB_persec_average{host=\"serenity\",dimension=\"reads\",device_type=\"physical\"} * 1024)", + "legendFormat": "{{device}} read", + "range": true, + "refId": "A" + }, + { + "editorMode": "code", + "expr": "sum by (device) (-1 * netdata_disk_io_KiB_persec_average{host=\"serenity\",dimension=\"writes\",device_type=\"physical\"} * 1024)", + "legendFormat": "{{device}} write", + "range": true, + "refId": "B" + } + ] + } + ], + "refresh": "30s", + "schemaVersion": 41, + "tags": [ + "homelab", + "serenity", + "netdata" + ], + "templating": { + "list": [] + }, + "time": { + "from": "now-24h", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "Serenity Host Health", + "uid": "serenity-host-health", + "version": 1, + "weekStart": "" +} diff --git a/monitoring/deploy.sh b/monitoring/deploy.sh old mode 100644 new mode 100755 index 03197b9..ac25c6a --- a/monitoring/deploy.sh +++ b/monitoring/deploy.sh @@ -1,236 +1,41 @@ #!/usr/bin/env bash set -euo pipefail -# ============================================================ -# Phase 6 — Grafana + Prometheus Deployment on PD -# Run this script on PlausibleDeniability as root or with sudo -# ============================================================ +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +cd "$SCRIPT_DIR" -echo "=== Phase 6: Grafana + Prometheus ===" -echo "" - -# ---- 1. Scaffold directories ---- -echo "[1/6] Scaffolding directories..." - -mkdir -p /mnt/docker-ssd/docker/compose/monitoring/provisioning/datasources -mkdir -p /mnt/docker-ssd/docker/compose/monitoring/provisioning/dashboards +echo "=== Monitoring stack deploy ===" mkdir -p /mnt/tank/docker/appdata/prometheus -chown 65534:65534 /mnt/tank/docker/appdata/prometheus - mkdir -p /mnt/tank/docker/appdata/grafana +mkdir -p /mnt/tank/docker/appdata/grafana/dashboards +mkdir -p /mnt/tank/docker/appdata/loki +mkdir -p /mnt/tank/docker/appdata/promtail + +chown 65534:65534 /mnt/tank/docker/appdata/prometheus chown 472:472 /mnt/tank/docker/appdata/grafana +chown 472:472 /mnt/tank/docker/appdata/grafana/dashboards -echo " Done." +if [[ ! -f .env ]]; then + cp .env.example .env + gf_pass="$(openssl rand -hex 16)" + sed -i "s/^GF_ADMIN_PASS=CHANGE_ME$/GF_ADMIN_PASS=$gf_pass/" .env + chmod 600 .env + echo "Created .env with a random Grafana admin password." +fi -# ---- 2. Write docker-compose.yaml ---- -echo "[2/6] Writing docker-compose.yaml..." - -cat > /mnt/docker-ssd/docker/compose/monitoring/docker-compose.yaml << 'COMPOSE' -name: monitoring - -services: - prometheus: - image: prom/prometheus:v3.4.0 - container_name: prometheus - restart: unless-stopped - ports: - - "9090:9090" - command: - - '--config.file=/etc/prometheus/prometheus.yml' - - '--storage.tsdb.path=/prometheus' - - '--storage.tsdb.retention.time=90d' - - '--web.enable-lifecycle' - volumes: - - /mnt/tank/docker/appdata/prometheus:/prometheus - - /mnt/docker-ssd/docker/compose/monitoring/prometheus.yml:/etc/prometheus/prometheus.yml:ro - networks: - - default - - grafana: - image: grafana/grafana:13.0.1 - container_name: grafana - restart: unless-stopped - ports: - - "3000:3000" - environment: - TZ: ${TZ} - GF_SECURITY_ADMIN_USER: ${GF_ADMIN_USER} - GF_SECURITY_ADMIN_PASSWORD: ${GF_ADMIN_PASS} - GF_SERVER_ROOT_URL: https://${GF_HOST}/ - volumes: - - /mnt/tank/docker/appdata/grafana:/var/lib/grafana - - /mnt/docker-ssd/docker/compose/monitoring/provisioning:/etc/grafana/provisioning:ro - networks: - - pangolin - - default - - node-exporter: - image: prom/node-exporter:v1.9.0 - container_name: node-exporter - restart: unless-stopped - ports: - - "9100:9100" - command: - - '--path.procfs=/host/proc' - - '--path.sysfs=/host/sys' - - '--path.rootfs=/rootfs' - - '--collector.filesystem.mount-points-exclude=^/(sys|proc|dev|host|etc)($$|/)' - volumes: - - /proc:/host/proc:ro - - /sys:/host/sys:ro - - /:/rootfs:ro - networks: - - default - -networks: - pangolin: - external: true -COMPOSE - -echo " Done." - -# ---- 3. Write prometheus.yml ---- -echo "[3/6] Writing prometheus.yml..." - -cat > /mnt/docker-ssd/docker/compose/monitoring/prometheus.yml << 'PROMCFG' -global: - scrape_interval: 15s - evaluation_interval: 15s - -scrape_configs: - # ------- PlausibleDeniability (local) ------- - - job_name: 'pd-node' - static_configs: - - targets: ['node-exporter:9100'] - labels: - host: 'plausible-deniability' - - - job_name: 'pd-netdata' - metrics_path: /api/v1/allmetrics - params: - format: [prometheus] - static_configs: - - targets: ['10.5.1.6:19999'] - labels: - host: 'plausible-deniability' - - # ------- Serenity ------- - - job_name: 'serenity-netdata' - metrics_path: /api/v1/allmetrics - params: - format: [prometheus] - static_configs: - - targets: ['10.5.1.5:19999'] - labels: - host: 'serenity' - - # ------- N.O.M.A.D. ------- - # Uncomment when Netdata or node-exporter is available on N.O.M.A.D. - # - job_name: 'nomad-netdata' - # metrics_path: /api/v1/allmetrics - # params: - # format: [prometheus] - # static_configs: - # - targets: ['10.5.1.16:19999'] - # labels: - # host: 'nomad' -PROMCFG - -echo " Done." - -# ---- 4. Write Grafana provisioning ---- -echo "[4/6] Writing Grafana provisioning configs..." - -cat > /mnt/docker-ssd/docker/compose/monitoring/provisioning/datasources/prometheus.yml << 'DATASRC' -apiVersion: 1 - -datasources: - - name: Prometheus - type: prometheus - access: proxy - url: http://prometheus:9090 - isDefault: true - editable: true -DATASRC - -cat > /mnt/docker-ssd/docker/compose/monitoring/provisioning/dashboards/dashboards.yml << 'DASHCFG' -apiVersion: 1 - -providers: - - name: 'default' - orgId: 1 - folder: '' - type: file - disableDeletion: false - editable: true - options: - path: /var/lib/grafana/dashboards - foldersFromFilesStructure: false -DASHCFG - -echo " Done." - -# ---- 5. Generate .env ---- -echo "[5/6] Generating .env..." - -GF_PASS=$(openssl rand -hex 16) - -cat > /mnt/docker-ssd/docker/compose/monitoring/.env << ENV -TZ=America/New_York - -# Grafana -GF_ADMIN_USER=admin -GF_ADMIN_PASS=${GF_PASS} -GF_HOST=grafana.paccoco.com -ENV - -echo " Done." -echo "" -echo " Grafana admin password: ${GF_PASS}" -echo " (saved in /mnt/docker-ssd/docker/compose/monitoring/.env)" -echo "" - -# ---- 6. Validate and deploy ---- -echo "[6/6] Validating and deploying..." - -cd /mnt/docker-ssd/docker/compose/monitoring -docker compose --env-file .env config > /dev/null 2>&1 -echo " Compose config valid." +if compgen -G "$SCRIPT_DIR/dashboards/*.json" >/dev/null; then + cp "$SCRIPT_DIR"/dashboards/*.json /mnt/tank/docker/appdata/grafana/dashboards/ + chown 472:472 /mnt/tank/docker/appdata/grafana/dashboards/*.json +fi +docker compose --env-file .env config >/dev/null docker compose --env-file .env up -d -echo "" -echo "=== Deployment complete ===" -echo "" -echo "Waiting 10 seconds for containers to start..." -sleep 10 -# ---- Post-deploy checks ---- -echo "" -echo "=== Post-Deploy Verification ===" -echo "" - -echo "--- Container status ---" -docker ps --filter name=prometheus --filter name=grafana --filter name=node-exporter --format "table {{.Names}}\t{{.Status}}\t{{.Ports}}" -echo "" - -echo "--- Prometheus targets ---" -curl -s http://10.5.1.6:9090/api/v1/targets 2>/dev/null | python3 -m json.tool 2>/dev/null | head -30 || echo "Prometheus not yet responding (may need a moment)" -echo "" - -echo "--- Grafana health ---" -HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" http://10.5.1.6:3000/ 2>/dev/null || echo "000") -echo "Grafana HTTP status: ${HTTP_CODE}" -echo "" - -echo "--- Node exporter ---" -curl -s http://10.5.1.6:9100/metrics 2>/dev/null | head -5 || echo "Node exporter not yet responding" -echo "" - -echo "=== All done! ===" -echo "" -echo "Next steps:" -echo " 1. Add Pangolin resource: grafana.paccoco.com -> http://grafana:3000" -echo " 2. Log in at https://grafana.paccoco.com with admin / ${GF_PASS}" -echo " 3. Import dashboards: Node Exporter Full (ID 1860), Docker Stats (ID 893)" -echo " 4. Set up Grafana -> n8n alert webhook at https://n8n.paccoco.com/webhook/grafana-alert" +echo +docker ps --filter name=prometheus --filter name=grafana --filter name=loki --filter name=promtail --filter name=cadvisor --format "table {{.Names}}\t{{.Status}}\t{{.Ports}}" +echo +curl -fsS http://127.0.0.1:9090/-/ready >/dev/null && echo "Prometheus ready" +[[ "$(docker inspect -f '{{.State.Running}}' loki 2>/dev/null)" == "true" ]] && echo "Loki running" +curl -fsS http://127.0.0.1:9100/metrics >/dev/null && echo "node-exporter ready" +docker exec cadvisor wget -qO- http://127.0.0.1:8080/metrics >/dev/null 2>&1 && echo "cAdvisor ready" || true diff --git a/monitoring/docker-compose.yaml b/monitoring/docker-compose.yaml index 55e0b2d..00841a4 100644 --- a/monitoring/docker-compose.yaml +++ b/monitoring/docker-compose.yaml @@ -15,11 +15,24 @@ services: volumes: - /mnt/tank/docker/appdata/prometheus:/prometheus - /mnt/docker-ssd/docker/compose/monitoring/prometheus.yml:/etc/prometheus/prometheus.yml:ro + - /mnt/docker-ssd/docker/compose/monitoring/alerts.yml:/etc/prometheus/alerts.yml:ro + networks: + - default + + loki: + image: grafana/loki:latest + container_name: loki + restart: unless-stopped + command: + - '-config.file=/etc/loki/config.yml' + volumes: + - /mnt/tank/docker/appdata/loki:/loki + - /mnt/docker-ssd/docker/compose/monitoring/loki-config.yml:/etc/loki/config.yml:ro networks: - default grafana: - image: grafana/grafana:13.0.1 + image: grafana/grafana:latest container_name: grafana restart: unless-stopped ports: @@ -29,13 +42,46 @@ services: GF_SECURITY_ADMIN_USER: ${GF_ADMIN_USER} GF_SECURITY_ADMIN_PASSWORD: ${GF_ADMIN_PASS} GF_SERVER_ROOT_URL: https://${GF_HOST}/ + GF_AUTH_GENERIC_OAUTH_ENABLED: ${GF_AUTH_GENERIC_OAUTH_ENABLED} + GF_AUTH_GENERIC_OAUTH_NAME: ${GF_AUTH_GENERIC_OAUTH_NAME} + GF_AUTH_GENERIC_OAUTH_ICON: ${GF_AUTH_GENERIC_OAUTH_ICON} + GF_AUTH_GENERIC_OAUTH_CLIENT_ID: ${GF_AUTH_GENERIC_OAUTH_CLIENT_ID} + GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET: ${GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET} + GF_AUTH_GENERIC_OAUTH_SCOPES: ${GF_AUTH_GENERIC_OAUTH_SCOPES} + GF_AUTH_GENERIC_OAUTH_EMPTY_SCOPES: ${GF_AUTH_GENERIC_OAUTH_EMPTY_SCOPES} + GF_AUTH_GENERIC_OAUTH_AUTH_URL: https://${AUTHELIA_HOST}/api/oidc/authorization + GF_AUTH_GENERIC_OAUTH_TOKEN_URL: https://${AUTHELIA_HOST}/api/oidc/token + GF_AUTH_GENERIC_OAUTH_API_URL: https://${AUTHELIA_HOST}/api/oidc/userinfo + GF_AUTH_GENERIC_OAUTH_USE_PKCE: ${GF_AUTH_GENERIC_OAUTH_USE_PKCE} + GF_AUTH_GENERIC_OAUTH_ALLOW_SIGN_UP: ${GF_AUTH_GENERIC_OAUTH_ALLOW_SIGN_UP} + GF_AUTH_GENERIC_OAUTH_AUTO_LOGIN: ${GF_AUTH_GENERIC_OAUTH_AUTO_LOGIN} + GF_AUTH_GENERIC_OAUTH_LOGIN_ATTRIBUTE_PATH: ${GF_AUTH_GENERIC_OAUTH_LOGIN_ATTRIBUTE_PATH} + GF_AUTH_GENERIC_OAUTH_NAME_ATTRIBUTE_PATH: ${GF_AUTH_GENERIC_OAUTH_NAME_ATTRIBUTE_PATH} + GF_AUTH_GENERIC_OAUTH_EMAIL_ATTRIBUTE_PATH: ${GF_AUTH_GENERIC_OAUTH_EMAIL_ATTRIBUTE_PATH} + GF_AUTH_GENERIC_OAUTH_ROLE_ATTRIBUTE_PATH: ${GF_AUTH_GENERIC_OAUTH_ROLE_ATTRIBUTE_PATH} + GF_AUTH_GENERIC_OAUTH_ALLOW_ASSIGN_GRAFANA_ADMIN: ${GF_AUTH_GENERIC_OAUTH_ALLOW_ASSIGN_GRAFANA_ADMIN} + GF_AUTH_DISABLE_LOGIN_FORM: ${GF_AUTH_DISABLE_LOGIN_FORM} volumes: - /mnt/tank/docker/appdata/grafana:/var/lib/grafana + - /mnt/docker-ssd/docker/compose/monitoring/dashboards:/var/lib/grafana/dashboards:ro - /mnt/docker-ssd/docker/compose/monitoring/provisioning:/etc/grafana/provisioning:ro networks: - pangolin - default + promtail: + image: grafana/promtail:latest + container_name: promtail + restart: unless-stopped + command: + - '-config.file=/etc/promtail/config.yml' + volumes: + - /mnt/tank/docker/appdata/promtail:/var/lib/promtail + - /mnt/docker-ssd/docker/compose/monitoring/promtail-config.yml:/etc/promtail/config.yml:ro + - /var/run/docker.sock:/var/run/docker.sock:ro + networks: + - default + node-exporter: image: prom/node-exporter:v1.9.0 container_name: node-exporter @@ -47,10 +93,29 @@ services: - '--path.sysfs=/host/sys' - '--path.rootfs=/rootfs' - '--collector.filesystem.mount-points-exclude=^/(sys|proc|dev|host|etc)($$|/)' + - '--collector.textfile.directory=/var/lib/node_exporter/textfile_collector' volumes: - /proc:/host/proc:ro - /sys:/host/sys:ro - /:/rootfs:ro + - /mnt/tank/docker/metrics/node-exporter:/var/lib/node_exporter/textfile_collector + networks: + - default + + cadvisor: + image: gcr.io/cadvisor/cadvisor:latest + container_name: cadvisor + restart: unless-stopped + privileged: true + command: + - '--docker_only=true' + - '--housekeeping_interval=30s' + volumes: + - /:/rootfs:ro + - /var/run:/var/run:rw + - /sys:/sys:ro + - /var/lib/docker:/var/lib/docker:ro + - /dev/disk:/dev/disk:ro networks: - default diff --git a/monitoring/loki-config.yml b/monitoring/loki-config.yml new file mode 100644 index 0000000..efa25d2 --- /dev/null +++ b/monitoring/loki-config.yml @@ -0,0 +1,38 @@ +auth_enabled: false + +server: + http_listen_port: 3100 + +common: + path_prefix: /loki + replication_factor: 1 + ring: + kvstore: + store: inmemory + storage: + filesystem: + chunks_directory: /loki/chunks + rules_directory: /loki/rules + +schema_config: + configs: + - from: 2024-01-01 + store: tsdb + object_store: filesystem + schema: v13 + index: + prefix: index_ + period: 24h + +limits_config: + retention_period: 30d + allow_structured_metadata: true + volume_enabled: true + +compactor: + working_directory: /loki/compactor + retention_enabled: true + delete_request_store: filesystem + +analytics: + reporting_enabled: false diff --git a/monitoring/prometheus.yml b/monitoring/prometheus.yml index 85a6141..0e2c619 100644 --- a/monitoring/prometheus.yml +++ b/monitoring/prometheus.yml @@ -2,6 +2,9 @@ global: scrape_interval: 15s evaluation_interval: 15s +rule_files: + - /etc/prometheus/alerts.yml + scrape_configs: # ------- PlausibleDeniability (local via node-exporter) ------- - job_name: 'pd-node' @@ -10,6 +13,12 @@ scrape_configs: labels: host: 'plausible-deniability' + - job_name: 'pd-cadvisor' + static_configs: + - targets: ['cadvisor:8080'] + labels: + host: 'plausible-deniability' + # ------- Serenity (via Netdata Prometheus exporter) ------- - job_name: 'serenity-netdata' metrics_path: /api/v1/allmetrics diff --git a/monitoring/promtail-config.yml b/monitoring/promtail-config.yml new file mode 100644 index 0000000..1e91e0e --- /dev/null +++ b/monitoring/promtail-config.yml @@ -0,0 +1,27 @@ +server: + http_listen_port: 9080 + grpc_listen_port: 0 + +positions: + filename: /var/lib/promtail/positions.yml + +clients: + - url: http://loki:3100/loki/api/v1/push + +scrape_configs: + - job_name: docker + docker_sd_configs: + - host: unix:///var/run/docker.sock + refresh_interval: 15s + relabel_configs: + - source_labels: ['__meta_docker_container_name'] + regex: '/(.*)' + target_label: container + - source_labels: ['__meta_docker_container_label_com_docker_compose_project'] + target_label: compose_project + - source_labels: ['__meta_docker_container_label_com_docker_compose_service'] + target_label: compose_service + - source_labels: ['__meta_docker_container_log_stream'] + target_label: stream + - replacement: plausible-deniability + target_label: host diff --git a/monitoring/provisioning/datasources/prometheus.yml b/monitoring/provisioning/datasources/prometheus.yml index 1a57b69..991dc75 100644 --- a/monitoring/provisioning/datasources/prometheus.yml +++ b/monitoring/provisioning/datasources/prometheus.yml @@ -7,3 +7,8 @@ datasources: url: http://prometheus:9090 isDefault: true editable: true + - name: Loki + type: loki + access: proxy + url: http://loki:3100 + editable: true