From 7ec40ef43785011525840419ad475ba0959863e1 Mon Sep 17 00:00:00 2001 From: Paccoco <99041736+Paccoco@users.noreply.github.com> Date: Sat, 9 May 2026 13:20:11 -0500 Subject: [PATCH] Add n8n workflows 01-08 (with upgrades), 14-16; planning docs; .gitignore --- .gitignore | 29 +- DEPLOYMENT_HANDOFF.md | 65 + HOMELAB_BUILDOUT_PLAN.md | 2078 +++++++++++++++++ docs/planning/DEPLOY_KIMA_HUB.md | 91 + docs/planning/DEPLOY_PIHOLE.md | 527 +++++ docs/planning/N8N_WORKFLOW_IDEAS.md | 186 ++ docs/planning/TODO.md | 16 +- docs/troubleshooting/KNOWN_QUIRKS.md | 9 + meshtastic/scripts/write-test | 0 n8n-workflows/01-grafana-alert-ai-gotify.json | 363 +++ n8n-workflows/02-rag-ingest-and-query.json | 394 ++++ n8n-workflows/03-paperless-ai-processing.json | 348 +++ n8n-workflows/04-whisper-transcription.json | 589 +++++ n8n-workflows/05-paperless-to-rag.json | 357 +++ n8n-workflows/06-web-scrape-summarize.json | 588 +++++ n8n-workflows/07-git-commit-summarizer.json | 210 ++ n8n-workflows/08-class-recording-rag.json | 532 +++++ .../14-paperless-inbox-reminder.json | 292 +++ n8n-workflows/15-n8n-health-monitor.json | 383 +++ n8n-workflows/16-nfs-watchdog.json | 449 ++++ n8n-workflows/README.md | 242 ++ n8n-workflows/class-watcher.sh | 121 + 22 files changed, 7859 insertions(+), 10 deletions(-) create mode 100644 DEPLOYMENT_HANDOFF.md create mode 100644 HOMELAB_BUILDOUT_PLAN.md create mode 100644 docs/planning/DEPLOY_KIMA_HUB.md create mode 100644 docs/planning/DEPLOY_PIHOLE.md create mode 100644 docs/planning/N8N_WORKFLOW_IDEAS.md create mode 100644 meshtastic/scripts/write-test create mode 100644 n8n-workflows/01-grafana-alert-ai-gotify.json create mode 100644 n8n-workflows/02-rag-ingest-and-query.json create mode 100644 n8n-workflows/03-paperless-ai-processing.json create mode 100644 n8n-workflows/04-whisper-transcription.json create mode 100644 n8n-workflows/05-paperless-to-rag.json create mode 100644 n8n-workflows/06-web-scrape-summarize.json create mode 100644 n8n-workflows/07-git-commit-summarizer.json create mode 100644 n8n-workflows/08-class-recording-rag.json create mode 100644 n8n-workflows/14-paperless-inbox-reminder.json create mode 100644 n8n-workflows/15-n8n-health-monitor.json create mode 100644 n8n-workflows/16-nfs-watchdog.json create mode 100644 n8n-workflows/README.md create mode 100644 n8n-workflows/class-watcher.sh diff --git a/.gitignore b/.gitignore index 94fa287..7de923d 100644 --- a/.gitignore +++ b/.gitignore @@ -1,16 +1,27 @@ -# Never commit real secrets +# Environment files .env +.env.* +!.env.example -# Postgres init SQL contains hardcoded passwords (Option A decision) -# Copy from databases/initdb/01-create-databases.sql.example and fill in real values -databases/initdb/01-create-databases.sql +# Backup files +*.bak +*.bak.* +documents/*.bak* -# OS noise -.DS_Store -Thumbs.db +# Python +__pycache__/ +*.pyc +*.pyo -# Editor noise +# Temp scripts +update-homelab-docs.py + +# Editor .vscode/ .idea/ *.swp -*.swo +*~ + +# OS +.DS_Store +Thumbs.db diff --git a/DEPLOYMENT_HANDOFF.md b/DEPLOYMENT_HANDOFF.md new file mode 100644 index 0000000..704d67f --- /dev/null +++ b/DEPLOYMENT_HANDOFF.md @@ -0,0 +1,65 @@ +# Deployment Handoff — Resume Here + +## Where We Left Off + +Phase 2 (AI stack) is fully deployed and verified on PlausibleDeniability (10.5.1.6). We were about to verify Gotify (Phase 1) and then deploy n8n (Phase 3). + +## What's Done + +### Phase 2 — AI Stack (COMPLETE) +All 6 containers running and verified on PD (10.5.1.6): +- **Ollama:** port 11434 — qwen2.5:14b loaded +- **OpenWebUI:** port 8282 (→8080) +- **Qdrant:** port 6333 (HTTP), 6334 (gRPC) +- **LiteLLM:** port 4000 — auth working, inference verified (medium tier → qwen2.5:14b) +- **Reranker (TEI):** port 8787 (→80) — bge-reranker-v2-m3 via safetensors +- **Whisper:** port 8786 (→8000) + +### Key Fixes Applied (reference for future troubleshooting) +1. **TrueNAS Scale Docker quirk:** `curl localhost:PORT` returns 000. Always use `curl 10.5.1.6:PORT`. +2. **Reranker image:** Use `cpu-latest` not `cpu-1.5` (hf-hub bug). +3. **LiteLLM config:** Remove `database_url` (newer versions try Prisma/PostgreSQL). Set `master_key: "os.environ/LITELLM_MASTER_KEY"` to read from env var. + +### Phase 1 — Gotify (DEPLOYED, NOT VERIFIED) +Gotify was deployed previously but hasn't been verified this session. + +## What's Next (in order) + +### 1. Verify Gotify +Run these on PD and confirm output: +```bash +docker ps --filter name=gotify --format "table {{.Names}}\t{{.Image}}\t{{.Status}}\t{{.Ports}}" +docker logs gotify --tail 20 +docker inspect gotify --format '{{json .Mounts}}' | python3 -m json.tool +docker inspect gotify --format '{{json .NetworkSettings.Networks}}' | python3 -m json.tool +curl -sw "\n%{http_code}" http://10.5.1.6:8484/health +ls -la /mnt/docker-ssd/docker/compose/automation/ +cat /mnt/docker-ssd/docker/compose/automation/docker-compose.yaml +``` + +### 2. Deploy n8n (Phase 3) +Per HOMELAB_BUILDOUT_PLAN.md Phase 3: +- Create postgres user/db for n8n on shared-postgres +- Scaffold `/mnt/tank/docker/appdata/n8n` (chown 1000:1000) +- Update automation compose to add n8n alongside Gotify +- Generate secrets for .env (N8N_DB_PASS, N8N_ENCRYPTION_KEY) +- Deploy and verify (DB migration, network connectivity, UI access) + +### 3. Pending Config (do whenever) +- OpenWebUI → add LiteLLM as OpenAI connection (`http://litellm:4000/v1`) +- OpenWebUI → optionally set RAG vector DB to Qdrant (`http://qdrant:6333`) + +## Network Reference +| Network | Purpose | +|---------|---------| +| ai-services | Cross-stack: AI containers ↔ n8n, future services | +| ix-databases_shared-databases | Shared postgres/redis access | +| pangolin | Reverse proxy / external exposure | + +## Machine IPs +- **PlausibleDeniability (PD):** 10.5.1.6 — TrueNAS Scale, RTX 2080 Ti +- **ROCINANTE:** 10.5.1.112 — RTX 4090 +- **N.O.M.A.D.:** 10.5.1.16 — Ubuntu, GTX 1080 + +## LiteLLM .env Secrets (on PD at /mnt/docker-ssd/docker/compose/ai/.env) +Already configured — do not regenerate. Master key is in the .env file as LITELLM_MASTER_KEY. diff --git a/HOMELAB_BUILDOUT_PLAN.md b/HOMELAB_BUILDOUT_PLAN.md new file mode 100644 index 0000000..a3e77f5 --- /dev/null +++ b/HOMELAB_BUILDOUT_PLAN.md @@ -0,0 +1,2078 @@ +# Homelab Expansion — Full Buildout Plan + +> Generated: 2026-05-04 +> Based on: HOMELAB_EXPANSION_PLAN.md, ARCHITECTURE_OVERVIEW.md, STACK_STANDARDS.md + +This document contains everything needed to deploy all six expansion phases — docker-compose files, .env templates, directory scaffolding commands, database init SQL, Pangolin configuration, and validation steps. Each phase follows the established stack standards and deployment checklist. + +--- + +## Table of Contents + +1. [Prerequisites & Conventions](#prerequisites--conventions) +2. [Phase 1 — Gotify (Notifications)](#phase-1--gotify-notifications) +3. [Phase 2 — Qdrant (Vector Database)](#phase-2--qdrant-vector-database) +4. [Phase 3 — n8n (Workflow Automation)](#phase-3--n8n-workflow-automation) +5. [Phase 4 — Paperless-NGX (Document Intelligence)](#phase-4--paperless-ngx-document-intelligence) +6. [Phase 5 — Home Assistant (Home Automation)](#phase-5--home-assistant-home-automation) +7. [Phase 6 — Grafana + Prometheus (Observability)](#phase-6--grafana--prometheus-observability) +8. [Phase 7 — LiteLLM (AI Gateway)](#phase-7--litellm-ai-gateway) +9. [Phase 8 — Reranker (RAG Quality)](#phase-8--reranker-rag-quality) +10. [Phase 9 — faster-whisper (Speech-to-Text)](#phase-9--faster-whisper-speech-to-text) +11. [Additional Tools Setup](#additional-tools-setup) +12. [Deployment Order Summary](#deployment-order-summary) +13. [Post-Deployment Validation Master Checklist](#post-deployment-validation-master-checklist) + +--- + +## Prerequisites & Conventions + +### Deployment Standards (recap) + +All stacks on PlausibleDeniability follow the same pattern: + +- **Repo root:** `/mnt/docker-ssd/docker/compose` +- **Validate before deploy:** `docker compose --env-file .env config` +- **Deploy:** `docker compose --env-file .env up -d` +- **Teardown:** `docker compose --env-file .env down` + +### Storage Tiers + +| Tier | Mount | Use For | +|------|-------|---------| +| SSD | `/mnt/docker-ssd/docker/appdata/` | Write-heavy, SQLite, GPU/model, databases | +| Tank | `/mnt/tank/docker/appdata/` | General appdata, configs, uploads | +| Unraid | `/mnt/unraid/data/media/` | Media libraries only | + +### Networks + +| Network | Created By | Purpose | +|---------|-----------|---------| +| `ix-databases_shared-databases` | databases stack | Access to shared-postgres, shared-mariadb, shared-redis | +| `pangolin` | newt (infrastructure stack) | Reverse proxy / external exposure | + +### Secret Generation Commands + +```bash +# Database passwords +openssl rand -hex 24 + +# JWT / encryption keys +openssl rand -hex 32 + +# Paperless secret key +python3 -c "import secrets; print(secrets.token_urlsafe(50))" +``` + +### Deployment Order + +``` +databases (already running) → infrastructure (already running) + → Phase 1: Gotify + → Phase 2: Qdrant (creates the ai-services network) + → Phase 3: n8n (depends on Gotify + Qdrant — joins ai-services as external) + → Phase 4: Paperless-NGX (depends on n8n for automation hooks) + → Phase 5: Home Assistant (depends on n8n for heavy automation) + → Phase 6: Grafana + Prometheus (on N.O.M.A.D., independent but benefits from all above) +``` + +> **Important:** The `ai` stack (Phase 2) must be deployed before the `automation` stack (Phase 3) because n8n declares `ai-services` as an external network. If the `ai` stack isn't up, `docker compose up` for `automation` will fail with a missing network error. + +--- + +## Phase 1 — Gotify (Notifications) + +**Host:** PlausibleDeniability +**Stack directory:** `/mnt/docker-ssd/docker/compose/automation/` +**Why first:** Every subsequent phase sends notifications through Gotify. It's the output bus. + +### 1.1 — Scaffold Directories + +```bash +# Appdata on SSD (SQLite backend — must not be on NFS) +sudo mkdir -p /mnt/docker-ssd/docker/appdata/gotify + +# Stack directory (may already exist if automation/ is planned) +sudo mkdir -p /mnt/docker-ssd/docker/compose/automation +``` + +### 1.2 — Database Init + +None required — Gotify uses an embedded SQLite database stored in its data volume. + +### 1.3 — docker-compose.yaml + +Add to `/mnt/docker-ssd/docker/compose/automation/docker-compose.yaml`: + +```yaml +name: automation + +services: + gotify: + image: gotify/server:2.6.1 + container_name: gotify + restart: unless-stopped + ports: + - "8484:80" + environment: + TZ: ${TZ} + GOTIFY_DEFAULTUSER_PASS: ${GOTIFY_ADMIN_PASS} + GOTIFY_SERVER_PORT: 80 + GOTIFY_DATABASE_DIALECT: sqlite3 + GOTIFY_DATABASE_CONNECTION: data/gotify.db + volumes: + - /mnt/docker-ssd/docker/appdata/gotify:/app/data + networks: + - pangolin + - default + +networks: + pangolin: + external: true +``` + +> **Image note:** `gotify/server:2.6.1` is the latest stable as of May 2026. Check [Docker Hub](https://hub.docker.com/r/gotify/server/tags) before deploying — pin to the exact version. + +### 1.4 — .env.example + +```bash +# /mnt/docker-ssd/docker/compose/automation/.env.example + +TZ=America/New_York +GOTIFY_ADMIN_PASS=CHANGE_ME +``` + +### 1.5 — .env (create from example) + +```bash +cd /mnt/docker-ssd/docker/compose/automation +cp .env.example .env +# Edit .env with real values: +# GOTIFY_ADMIN_PASS=$(openssl rand -hex 16) +nano .env +``` + +### 1.6 — Pangolin Configuration + +In your Pangolin dashboard, create a new resource: + +| Field | Value | +|-------|-------| +| Domain | `gotify.paccoco.com` | +| Scheme | `http` | +| Host | `gotify` | +| Port | `80` | +| Network | Docker service name resolution via `pangolin` network | + +### 1.7 — Validate & Deploy + +```bash +cd /mnt/docker-ssd/docker/compose/automation +docker compose --env-file .env config +docker compose --env-file .env up -d +``` + +### 1.8 — Post-Deploy Verification + +```bash +# Container running? +docker ps --filter name=gotify + +# Clean startup logs? +docker logs gotify --tail 20 + +# Mounts correct? +docker inspect gotify --format '{{json .Mounts}}' | python3 -m json.tool + +# Quick health check +curl -s http://localhost:8484/health + +# Test notification via API +curl -s "http://localhost:8484/message?token=YOUR_APP_TOKEN" \ + -F "title=Homelab" \ + -F "message=Gotify is online" \ + -F "priority=5" +``` + +### 1.9 — First-Run Setup + +1. Navigate to `https://gotify.paccoco.com` +2. Log in with the admin password from `.env` +3. **Change the default admin password** in the UI +4. Create application tokens for each notification source: + - `n8n-workflows` — for all n8n automations + - `infrastructure-alerts` — for Uptime Kuma, Grafana, etc. + - `media-notifications` — for Sonarr/Radarr/Tautulli hooks + - `home-assistant` — for HA automations +5. Create client tokens for each receiving device (phone, desktop) +6. Install the Gotify Android app and configure with your client token + +--- + +## Phase 2 — AI Stack (Vector DB, LLM, Embeddings, Reranker, STT) + +**Host:** PlausibleDeniability +**Stack directory:** `/mnt/docker-ssd/docker/compose/ai/` +**Status:** DEPLOYED AND VERIFIED (2026-05-05) + +> **DEPLOYED NOTE:** This phase was deployed as a complete AI stack with 6 services: Ollama (11434), OpenWebUI (8282), Qdrant (6333/6334), LiteLLM (4000), Reranker/TEI (8787), and faster-whisper (8786). The compose below shows the original Qdrant-only plan — see the actual running compose on PD at `/mnt/docker-ssd/docker/compose/ai/docker-compose.yaml` and the `project_ai_stack_deployed.md` memory for port mappings and fixes. Key lesson: always use `10.5.1.6` not `localhost` for health checks on TrueNAS Scale. + +### 2.1 — Scaffold Directories + +```bash +# Appdata on SSD (write-heavy vector storage) +sudo mkdir -p /mnt/docker-ssd/docker/appdata/qdrant/storage +sudo mkdir -p /mnt/docker-ssd/docker/appdata/qdrant/snapshots + +# Stack directory +sudo mkdir -p /mnt/docker-ssd/docker/compose/ai +``` + +### 2.2 — docker-compose.yaml + +Create or update `/mnt/docker-ssd/docker/compose/ai/docker-compose.yaml`: + +```yaml +name: ai + +services: + qdrant: + image: qdrant/qdrant:v1.14.0 + container_name: qdrant + restart: unless-stopped + ports: + - "6333:6333" # REST API + - "6334:6334" # gRPC + environment: + TZ: ${TZ} + QDRANT__SERVICE__GRPC_PORT: 6334 + QDRANT__STORAGE__STORAGE_PATH: /qdrant/storage + QDRANT__STORAGE__SNAPSHOTS_PATH: /qdrant/snapshots + volumes: + - /mnt/docker-ssd/docker/appdata/qdrant/storage:/qdrant/storage + - /mnt/docker-ssd/docker/appdata/qdrant/snapshots:/qdrant/snapshots + networks: + - ai-services + - default + + # ------------------------------------------------------- + # Ollama and OpenWebUI go here when deployed. + # They share this stack and the default + ai-services networks so + # OpenWebUI can reach Qdrant at http://qdrant:6333 + # ------------------------------------------------------- + + # ollama: + # image: ollama/ollama:latest + # container_name: ollama + # ... + + # openwebui: + # image: ghcr.io/open-webui/open-webui:main + # container_name: openwebui + # ... + +networks: + ai-services: + name: ai-services +``` + +> **Image note:** `qdrant/qdrant:v1.14.0` is the latest stable as of May 2026. Check [GitHub releases](https://github.com/qdrant/qdrant/releases) before deploying. + +> **Cross-stack connectivity:** The `ai-services` network is defined here with an explicit `name:` so other stacks (like `automation/n8n`) can declare it as `external: true` and reach Qdrant by service name (`http://qdrant:6333`). This follows the same pattern as `ix-databases_shared-databases`. + +### 2.3 — .env.example + +```bash +# /mnt/docker-ssd/docker/compose/ai/.env.example + +TZ=America/New_York +``` + +### 2.4 — Validate & Deploy + +```bash +cd /mnt/docker-ssd/docker/compose/ai +cp .env.example .env +nano .env # Set timezone + +docker compose --env-file .env config +docker compose --env-file .env up -d +``` + +### 2.5 — Post-Deploy Verification + +```bash +# Container running? +docker ps --filter name=qdrant + +# Clean startup? +docker logs qdrant --tail 20 + +# REST API responding? +curl -s http://localhost:6333/healthz +# Expected: {"title":"qdrant - vectorass engine","version":"1.14.0","commit":"..."} + +# Create a test collection +curl -X PUT http://localhost:6333/collections/test_collection \ + -H "Content-Type: application/json" \ + -d '{ + "vectors": { + "size": 384, + "distance": "Cosine" + } + }' + +# Verify it exists +curl -s http://localhost:6333/collections | python3 -m json.tool + +# Clean up test +curl -X DELETE http://localhost:6333/collections/test_collection +``` + +### 2.6 — OpenWebUI Integration (when deployed) + +When Ollama and OpenWebUI are brought online in this same stack, configure OpenWebUI's RAG settings: + +1. Go to OpenWebUI → Admin → Settings → Documents +2. Set the vector database to Qdrant +3. Endpoint: `http://qdrant:6333` (Docker DNS within the `ai` stack network) +4. Collection name: `openwebui_docs` (or your preference) + +### 2.7 — Collections to Create (for n8n in Phase 3) + +These collections will be created programmatically by n8n workflows, but for reference: + +| Collection | Vector Size | Content | +|-----------|-------------|---------| +| `homelab_docs` | 384 (nomic-embed-text) | Homelab markdown documentation | +| `gitea_commits` | 384 | Gitea commit messages + diffs | +| `media_metadata` | 384 | Plex/Tautulli metadata | +| `obsidian_notes` | 384 | Personal notes from Obsidian vault | + +> **Vector size note:** 384 is the dimension for `nomic-embed-text` via Ollama. If you use a different embedding model, adjust accordingly. + +--- + +## Phase 3 — n8n (Workflow Automation) + +**Host:** PlausibleDeniability +**Stack directory:** `/mnt/docker-ssd/docker/compose/automation/` (same stack as Gotify) +**Why third:** n8n is the orchestration backbone — it ties Gotify, Qdrant, Ollama, and all triggers together. + +### 3.1 — Scaffold Directories + +```bash +# Config on tank (workflow definitions, credentials store) +# NOTE: n8n also writes execution logs here. If execution logging becomes +# heavy (many workflows running frequently), consider moving to SSD. +# For typical homelab usage (~20 workflows), tank is fine. +sudo mkdir -p /mnt/tank/docker/appdata/n8n + +# Set ownership — n8n runs as UID 1000 (node user) +sudo chown 1000:1000 /mnt/tank/docker/appdata/n8n +``` + +### 3.2 — Database Init + +n8n uses the existing shared-postgres. Create its database and user: + +```bash +docker exec -i shared-postgres psql -U postgres <<'SQL' +CREATE USER n8n WITH PASSWORD 'REPLACE_WITH_GENERATED_PASSWORD'; +CREATE DATABASE n8n OWNER n8n; +GRANT ALL PRIVILEGES ON DATABASE n8n TO n8n; +SQL +``` + +Generate the password first: +```bash +openssl rand -hex 24 +``` + +### 3.3 — docker-compose.yaml + +Update `/mnt/docker-ssd/docker/compose/automation/docker-compose.yaml` to add n8n alongside Gotify: + +```yaml +name: automation + +services: + gotify: + image: gotify/server:2.6.1 + container_name: gotify + restart: unless-stopped + ports: + - "8484:80" + environment: + TZ: ${TZ} + GOTIFY_DEFAULTUSER_PASS: ${GOTIFY_ADMIN_PASS} + GOTIFY_SERVER_PORT: 80 + GOTIFY_DATABASE_DIALECT: sqlite3 + GOTIFY_DATABASE_CONNECTION: data/gotify.db + volumes: + - /mnt/docker-ssd/docker/appdata/gotify:/app/data + networks: + - pangolin + - default + + n8n: + image: docker.n8n.io/n8nio/n8n:1.88.0 + container_name: n8n + restart: unless-stopped + ports: + - "5678:5678" + environment: + TZ: ${TZ} + # Database + DB_TYPE: postgresdb + DB_POSTGRESDB_HOST: shared-postgres + DB_POSTGRESDB_PORT: 5432 + DB_POSTGRESDB_DATABASE: ${N8N_DB_NAME} + DB_POSTGRESDB_USER: ${N8N_DB_USER} + DB_POSTGRESDB_PASSWORD: ${N8N_DB_PASS} + # Security + N8N_ENCRYPTION_KEY: ${N8N_ENCRYPTION_KEY} + # Webhook / Reverse Proxy + N8N_HOST: ${N8N_HOST} + N8N_PROTOCOL: https + N8N_PORT: 5678 + WEBHOOK_URL: https://${N8N_HOST}/ + N8N_PROXY_HOPS: 1 + # General + GENERIC_TIMEZONE: ${TZ} + N8N_DIAGNOSTICS_ENABLED: false + N8N_PERSONALIZATION_ENABLED: false + volumes: + - /mnt/tank/docker/appdata/n8n:/home/node/.n8n + networks: + - pangolin + - ix-databases_shared-databases + - ai-services + - default + depends_on: + - gotify + +networks: + pangolin: + external: true + ix-databases_shared-databases: + external: true + ai-services: + external: true +``` + +> **Image note:** `docker.n8n.io/n8nio/n8n:1.88.0` — n8n uses calendar-ish versioning. Check [n8n releases](https://github.com/n8n-io/n8n/releases) for the latest stable. The `docker.n8n.io` registry is preferred for production. + +> **Cross-stack access:** n8n joins `ai-services` (created by the `ai` stack) so it can reach Qdrant at `http://qdrant:6333` by Docker DNS. It also joins `ix-databases_shared-databases` for Postgres access — same pattern. + +### 3.4 — .env.example (updated for both services) + +```bash +# /mnt/docker-ssd/docker/compose/automation/.env.example + +TZ=America/New_York + +# Gotify +GOTIFY_ADMIN_PASS=CHANGE_ME + +# n8n — Database +N8N_DB_NAME=n8n +N8N_DB_USER=n8n +N8N_DB_PASS=CHANGE_ME + +# n8n — Security +N8N_ENCRYPTION_KEY=CHANGE_ME + +# n8n — Hostname +N8N_HOST=n8n.paccoco.com +``` + +### 3.5 — Pangolin Configuration + +| Field | Value | +|-------|-------| +| Domain | `n8n.paccoco.com` | +| Scheme | `http` | +| Host | `n8n` | +| Port | `5678` | +| Headers | Forward `X-Forwarded-Proto: https` | + +### 3.6 — Validate & Deploy + +```bash +cd /mnt/docker-ssd/docker/compose/automation + +# Update .env with real passwords +nano .env + +docker compose --env-file .env config +docker compose --env-file .env up -d +``` + +### 3.7 — Post-Deploy Verification + +```bash +# Both containers running? +docker ps --filter name=gotify --filter name=n8n + +# n8n logs clean? +docker logs n8n --tail 30 + +# n8n on correct networks? +docker inspect n8n --format '{{json .NetworkSettings.Networks}}' | python3 -m json.tool + +# n8n UI accessible? +curl -s -o /dev/null -w "%{http_code}" http://localhost:5678/ +# Expected: 200 + +# Postgres connection working? (check logs for DB migration messages) +docker logs n8n 2>&1 | grep -i "migrat" +``` + +### 3.8 — First-Run Setup + +1. Navigate to `https://n8n.paccoco.com` +2. Create your admin account +3. Install community nodes you'll need: + - `n8n-nodes-gotify` (if available) or use HTTP Request node + - Ollama nodes (built-in as of n8n 1.x) + +### 3.9 — Workflow Blueprints + +Below are starter workflow descriptions for each planned automation. These are meant to be built in the n8n UI — the structure and node types are described so you can wire them up. + +#### Media Pipeline Workflows + +**Workflow: Sonarr/Radarr Download Notification** +``` +Trigger: Webhook node (POST from Sonarr/Radarr on download/import) +Step 1: Extract series/movie name, quality, file path from webhook body +Step 2: HTTP Request → TMDB API to fetch poster image URL +Step 3: HTTP Request → Gotify REST API (POST /message) + - Title: "New Download: {title}" + - Message: "{quality} — {episodeTitle or year}" + - Priority: 5 + - Extras: attach poster URL as markdown image +Also: HTTP Request → Discord webhook (formatted embed with poster) +``` + +Configure Sonarr/Radarr webhooks: +- Sonarr: Settings → Connect → Webhook → URL: `https://n8n.paccoco.com/webhook/sonarr` +- Radarr: Settings → Connect → Webhook → URL: `https://n8n.paccoco.com/webhook/radarr` + +**Workflow: Tautulli Play Logging** +``` +Trigger: Webhook node (POST from Tautulli on playback start) +Step 1: Extract user, media title, player, quality from payload +Step 2: Postgres node → INSERT into watch_history table +Step 3: (Optional) Gotify notification for specific users/media +``` + +Tautulli config: Settings → Notification Agents → Webhook → URL: `https://n8n.paccoco.com/webhook/tautulli-play` + +**Workflow: Weekly Watch Digest** +``` +Trigger: Cron node (every Sunday at 10:00 AM) +Step 1: Postgres node → SELECT watch history for past 7 days +Step 2: Format data as structured text +Step 3: HTTP Request → Ollama API (POST to PD's qwen2.5:14b) + - Prompt: "Summarize this week's viewing in a fun digest: {data}" +Step 4: HTTP Request → Gotify (send digest) +``` + +#### Infrastructure Monitoring Workflows + +**Workflow: Uptime Kuma Enhanced Alerts** +``` +Trigger: Webhook node (from Uptime Kuma notification) +Step 1: Extract monitor name, status, response time +Step 2: HTTP Request → Netdata API for related metrics context +Step 3: HTTP Request → Gotify + - Title: "🔴 {monitor} DOWN" or "🟢 {monitor} UP" + - Message: include Netdata context (CPU, mem, disk at time of alert) + - Priority: 8 (high for down, 3 for recovery) +``` + +**Workflow: ZFS Pool Utilization Alert** +``` +Trigger: Cron node (every 6 hours) +Step 1: SSH node → Serenity: `zpool list -Hp malcolm` +Step 2: Parse capacity percentage +Step 3: IF capacity > 85% → Gotify alert (priority 8) +Step 4: IF capacity > 90% → Gotify alert (priority 10) + Discord webhook +``` + +**Workflow: Grafana Alert Remediation** +``` +Trigger: Webhook node (from Grafana alerting) +Step 1: Parse alert labels (container, host, metric) +Step 2: Switch node → route by alert type: + - High CPU container → SSH → docker restart {container} + - Disk full → SSH → pause qBittorrent, notify via Gotify + - Memory pressure → Gotify alert only (manual intervention) +Step 3: Log action taken to Postgres +``` + +#### Homelab Ops Workflows + +**Workflow: Gitea Commit Summary** +``` +Trigger: Webhook node (Gitea webhook on push to truenas-stacks) +Step 1: Extract commit messages, author, files changed +Step 2: HTTP Request → Ollama API + - Prompt: "Summarize this commit in one sentence: {commit_message}" +Step 3: HTTP Request → Gotify + - Title: "Commit to truenas-stacks" + - Message: Ollama-generated summary +``` + +Gitea config: Repository → Settings → Webhooks → URL: `https://n8n.paccoco.com/webhook/gitea-push` + +**Workflow: qBittorrent Auto-Rescan** +``` +Trigger: Webhook or polling (qBittorrent API for completed+moved torrents) +Step 1: Determine if file is in Sonarr or Radarr path +Step 2: HTTP Request → Sonarr API (POST /command → RescanSeries) + OR HTTP Request → Radarr API (POST /command → RescanMovie) +Step 3: Gotify notification confirming rescan triggered +``` + +#### AI Pipeline Workflows + +**Workflow: URL Digest Pipeline** +``` +Trigger: Webhook node (POST with URL in body) +Step 1: HTTP Request → fetch page content +Step 2: Code node → extract text, chunk into ~500 token segments +Step 3: HTTP Request → Ollama API → summarize each chunk +Step 4: Code node → combine summaries into digest +Step 5: Postgres node → store digest with metadata +Step 6: Return digest in webhook response +``` + +**Workflow: Multi-Model Query Router (simplified by LiteLLM — see Phase 7)** +``` +Trigger: Webhook node (POST with query + complexity hint) +Step 1: HTTP Request → LiteLLM (POST http://litellm:4000/v1/chat/completions) + - model: complexity parameter ("light", "medium", or "heavy") + - LiteLLM handles routing to the correct Ollama instance +Step 2: Return response via webhook +``` +> With LiteLLM deployed, the Switch node and three separate Ollama endpoints +> are replaced by a single HTTP Request node. The routing logic lives in +> LiteLLM's config.yaml instead of n8n workflow logic. + +**Workflow: Qdrant Index Updater** +``` +Trigger: Webhook node (from Gitea push to any watched repo) +Step 1: HTTP Request → Gitea API → fetch changed files content +Step 2: Code node → chunk text into embedding-sized segments +Step 3: HTTP Request → Ollama API (embed endpoint with nomic-embed-text) +Step 4: HTTP Request → Qdrant API (PUT /collections/homelab_docs/points) +Step 5: Gotify notification: "{n} documents re-indexed" +``` + +#### Home / Business Workflows + +**Workflow: KitchenOwl Grocery Notification** +``` +Trigger: Polling (KitchenOwl API) or webhook if supported +Step 1: Fetch current shopping list items +Step 2: Format as clean text list +Step 3: HTTP Request → Gotify → phone push notification +``` + +**Workflow: Donetick Task Reminder** +``` +Trigger: Cron node (daily at 9:00 AM) +Step 1: HTTP Request → Donetick API → fetch tasks due today/overdue +Step 2: Format task list +Step 3: HTTP Request → Gotify (priority 5) +``` + +**Workflow: Long and Low Crafts Order Pipeline** +``` +Trigger: Webhook (Etsy webhook or email trigger via IMAP node) +Step 1: Parse order details (item, quantity, customer, shipping) +Step 2: HTTP Request → Donetick API → create fulfillment task +Step 3: HTTP Request → Gotify DM notification + - Title: "New L&L Order" + - Message: "{item} x{qty} — ship by {date}" + - Priority: 7 +``` + +--- + +## Phase 4 — Paperless-NGX (Document Intelligence) + +**Host:** PlausibleDeniability +**Stack directory:** `/mnt/docker-ssd/docker/compose/documents/` (new stack) +**Why fourth:** Depends on n8n for automation hooks and Gotify for notifications. + +### 4.1 — Scaffold Directories + +```bash +# Index/data on SSD (search index is write-heavy) +sudo mkdir -p /mnt/docker-ssd/docker/appdata/paperless/data + +# Documents (media) on tank (bulk storage, read-heavy) +sudo mkdir -p /mnt/tank/docker/appdata/paperless/media + +# Consume folder on tank (drop zone for new documents) +sudo mkdir -p /mnt/tank/docker/appdata/paperless/consume + +# Export folder on tank +sudo mkdir -p /mnt/tank/docker/appdata/paperless/export + +# Stack directory +sudo mkdir -p /mnt/docker-ssd/docker/compose/documents +``` + +### 4.2 — Database Init + +Paperless uses the existing shared-postgres and shared-redis: + +```bash +docker exec -i shared-postgres psql -U postgres <<'SQL' +CREATE USER paperless WITH PASSWORD 'REPLACE_WITH_GENERATED_PASSWORD'; +CREATE DATABASE paperless OWNER paperless; +GRANT ALL PRIVILEGES ON DATABASE paperless TO paperless; +SQL +``` + +### 4.3 — docker-compose.yaml + +Create `/mnt/docker-ssd/docker/compose/documents/docker-compose.yaml`: + +```yaml +name: documents + +services: + paperless: + image: ghcr.io/paperless-ngx/paperless-ngx:2.16 + container_name: paperless + restart: unless-stopped + ports: + - "8000:8000" + environment: + TZ: ${TZ} + # Database + PAPERLESS_DBENGINE: postgresql + PAPERLESS_DBHOST: shared-postgres + PAPERLESS_DBPORT: 5432 + PAPERLESS_DBNAME: ${PAPERLESS_DB_NAME} + PAPERLESS_DBUSER: ${PAPERLESS_DB_USER} + PAPERLESS_DBPASS: ${PAPERLESS_DB_PASS} + # Redis (using shared-redis) + PAPERLESS_REDIS: redis://shared-redis:6379 + # Security + PAPERLESS_SECRET_KEY: ${PAPERLESS_SECRET_KEY} + PAPERLESS_URL: https://${PAPERLESS_HOST} + PAPERLESS_ADMIN_USER: ${PAPERLESS_ADMIN_USER} + PAPERLESS_ADMIN_PASSWORD: ${PAPERLESS_ADMIN_PASS} + # OCR + PAPERLESS_OCR_LANGUAGE: eng + PAPERLESS_OCR_MODE: skip + # Tika/Gotenberg for Office documents + PAPERLESS_TIKA_ENABLED: 1 + PAPERLESS_TIKA_ENDPOINT: http://tika:9998 + PAPERLESS_TIKA_GOTENBERG_ENDPOINT: http://gotenberg:3000 + # Performance + PAPERLESS_TASK_WORKERS: 2 + PAPERLESS_THREADS_PER_WORKER: 2 + volumes: + - /mnt/docker-ssd/docker/appdata/paperless/data:/usr/src/paperless/data + - /mnt/tank/docker/appdata/paperless/media:/usr/src/paperless/media + - /mnt/tank/docker/appdata/paperless/consume:/usr/src/paperless/consume + - /mnt/tank/docker/appdata/paperless/export:/usr/src/paperless/export + networks: + - pangolin + - ix-databases_shared-databases + - default + depends_on: + - gotenberg + - tika + + gotenberg: + image: gotenberg/gotenberg:8.17 + container_name: paperless-gotenberg + restart: unless-stopped + command: + - "gotenberg" + - "--chromium-disable-javascript=true" + - "--chromium-allow-list=file:///tmp/.*" + networks: + - default + + tika: + image: apache/tika:3.1 + container_name: paperless-tika + restart: unless-stopped + networks: + - default + +networks: + pangolin: + external: true + ix-databases_shared-databases: + external: true +``` + +> **Image notes:** +> - `ghcr.io/paperless-ngx/paperless-ngx:2.16` — check [releases](https://github.com/paperless-ngx/paperless-ngx/releases) for latest. +> - `gotenberg/gotenberg:8.17` — pin to a specific 8.x tag. +> - `apache/tika:3.1` — needed for Office doc support (.docx, .xlsx, .odt). +> - Gotenberg and Tika are only needed if you ingest Office documents. For PDF-only, you can omit them and set `PAPERLESS_TIKA_ENABLED: 0`. + +### 4.4 — .env.example + +```bash +# /mnt/docker-ssd/docker/compose/documents/.env.example + +TZ=America/New_York + +# Paperless — Database +PAPERLESS_DB_NAME=paperless +PAPERLESS_DB_USER=paperless +PAPERLESS_DB_PASS=CHANGE_ME + +# Paperless — Security +PAPERLESS_SECRET_KEY=CHANGE_ME +PAPERLESS_ADMIN_USER=admin +PAPERLESS_ADMIN_PASS=CHANGE_ME + +# Paperless — Hostname +PAPERLESS_HOST=paperless.paccoco.com +``` + +### 4.5 — Pangolin Configuration + +| Field | Value | +|-------|-------| +| Domain | `paperless.paccoco.com` | +| Scheme | `http` | +| Host | `paperless` | +| Port | `8000` | + +### 4.6 — Validate & Deploy + +```bash +cd /mnt/docker-ssd/docker/compose/documents +cp .env.example .env +nano .env # Fill in real values + +docker compose --env-file .env config +docker compose --env-file .env up -d +``` + +### 4.7 — Post-Deploy Verification + +```bash +# All three containers running? +docker ps --filter name=paperless --filter name=paperless-gotenberg --filter name=paperless-tika + +# Paperless logs clean? (watch for DB migration output) +docker logs paperless --tail 30 + +# Network connectivity to shared-postgres and shared-redis? +docker exec paperless python3 -c "import psycopg2; print('postgres OK')" 2>/dev/null || echo "Check DB connection" + +# UI accessible? +curl -s -o /dev/null -w "%{http_code}" http://localhost:8000/ +# Expected: 200 or 302 (redirect to login) +``` + +### 4.8 — n8n Integration Workflows + +**Workflow: Email Attachment → Paperless** +``` +Trigger: IMAP node (poll email inbox every 5 minutes) +Step 1: Filter for emails with PDF/document attachments +Step 2: Download attachment binary +Step 3: HTTP Request → Paperless API (POST /api/documents/post_document/) + - Multipart form with document file + - Optional: set correspondent, document type, tags +Step 4: Wait node (30 seconds for OCR processing) +Step 5: HTTP Request → Paperless API (GET document details) +Step 6: IF tagged as "action-required" → + HTTP Request → Donetick API → create task +Step 7: HTTP Request → Gotify → "New document ingested: {title}" +``` + +**Workflow: Ollama Document Summarizer** +``` +Trigger: Paperless webhook (on document consumed) or n8n polling +Step 1: HTTP Request → Paperless API → fetch document text +Step 2: HTTP Request → Ollama API (PD's qwen2.5:14b) + - Prompt: "Extract key data from this document: dates, amounts, parties, action items. {text}" +Step 3: HTTP Request → Paperless API → update document notes with summary +Step 4: HTTP Request → Gotify → "Document summarized: {title}" +``` + +--- + +## Phase 5 — Home Assistant (Home Automation) + +**Host:** PlausibleDeniability (preferred — same network as most services) +**Stack directory:** `/mnt/docker-ssd/docker/compose/homeassistant/` (standalone stack — host networking isolates it) +**Why fifth:** Benefits from n8n being operational. n8n handles complex automation logic while HA handles device control. + +### 5.1 — Scaffold Directories + +```bash +# Config on SSD (SQLite database, write-heavy) +sudo mkdir -p /mnt/docker-ssd/docker/appdata/homeassistant +``` + +### 5.2 — docker-compose.yaml + +Create `/mnt/docker-ssd/docker/compose/homeassistant/docker-compose.yaml`: + +```yaml +name: homeassistant + +services: + homeassistant: + image: ghcr.io/home-assistant/home-assistant:2026.5 + container_name: homeassistant + restart: unless-stopped + privileged: true + network_mode: host + environment: + TZ: ${TZ} + volumes: + - /mnt/docker-ssd/docker/appdata/homeassistant:/config + - /etc/localtime:/etc/localtime:ro + - /run/dbus:/run/dbus:ro + # devices: + # - /dev/ttyUSB0:/dev/ttyUSB0 # Uncomment for USB Zigbee/Z-Wave sticks +``` + +> **Image note:** `ghcr.io/home-assistant/home-assistant:2026.5` — use the latest stable monthly release. Check [HA releases](https://www.home-assistant.io/blog/categories/release-notes/). Pin to a specific minor like `2026.5.1` once you confirm it's stable. + +> **Why host networking:** Home Assistant requires `network_mode: host` for mDNS/SSDP device discovery. This means it does NOT join `pangolin` — you'll access it directly by IP and port, or configure Pangolin to proxy to `http://10.5.1.X:8123`. + +> **Why standalone stack:** Host networking is incompatible with other services in the same compose that use bridge networking. HA must be in its own compose file. + +### 5.3 — .env.example + +```bash +# /mnt/docker-ssd/docker/compose/homeassistant/.env.example + +TZ=America/New_York +``` + +### 5.4 — Pangolin Configuration + +Since HA uses host networking, Pangolin needs to reach it by the host's IP: + +| Field | Value | +|-------|-------| +| Domain | `ha.paccoco.com` | +| Scheme | `http` | +| Host | `10.5.1.X` (PlausibleDeniability's LAN IP) | +| Port | `8123` | + +> Find PD's IP with: `hostname -I | awk '{print $1}'` + +### 5.5 — Validate & Deploy + +```bash +cd /mnt/docker-ssd/docker/compose/homeassistant +cp .env.example .env +nano .env + +docker compose --env-file .env config +docker compose --env-file .env up -d +``` + +### 5.6 — Post-Deploy Verification + +```bash +# Container running? +docker ps --filter name=homeassistant + +# Clean startup? +docker logs homeassistant --tail 30 + +# Web UI accessible? +curl -s -o /dev/null -w "%{http_code}" http://localhost:8123/ +# Expected: 200 +``` + +### 5.7 — First-Run Setup & Integrations + +1. Navigate to `http://PD_IP:8123` (or `https://ha.paccoco.com`) +2. Complete the onboarding wizard +3. Add integrations: + +**Smart Plug Power Monitoring:** +- Install TP-Link Kasa / Tapo / Shelly integration (depends on your plug brand) +- Add plugs for Serenity, PD, N.O.M.A.D. +- Create energy dashboard for power consumption tracking + +**UPS Monitoring:** +- Install NUT (Network UPS Tools) integration if you add a UPS +- Note: PD and N.O.M.A.D. currently have no UPS — this is a known gap +- When UPS is added, create automations for graceful shutdown via n8n + +**Presence Detection:** +- Mobile app integration for phone-based presence +- Or use router/UniFi integration for network-based detection + +### 5.8 — n8n ↔ Home Assistant Integration + +In n8n, use the Home Assistant nodes (built-in): + +1. In HA: Profile → Long-Lived Access Tokens → Create Token +2. In n8n: Settings → Credentials → Home Assistant API + - Host: `http://10.5.1.X:8123` + - Access Token: (paste from step 1) + +**Workflow: Presence → Server Sleep/Wake** +``` +Trigger: HA event node (person.fizzlepoof state change) +Step 1: IF state = "not_home" for > 30 min AND no active Plex streams: + SSH → N.O.M.A.D.: systemctl suspend +Step 2: IF state = "home": + Wake-on-LAN → N.O.M.A.D. MAC address +Step 3: Gotify notification: "Server {action}: N.O.M.A.D." +``` + +**Workflow: Physical Device → n8n Trigger** +``` +Trigger: HA webhook or event node (smart plug power draw spike) +Step 1: IF washing machine plug power < 5W for 5 min after being > 100W: + Gotify notification: "Laundry is done!" +``` + +--- + +## Phase 6 — Grafana + Prometheus (Observability) + +**Host:** N.O.M.A.D. (Ubuntu 25.10 — dedicated HDDs have headroom) +**Stack directory:** `/opt/monitoring/` (outside the N.O.M.A.D. project directory) +**Why last:** Observability benefits from all other services being online — more to monitor. + +### 6.1 — Scaffold Directories + +SSH into N.O.M.A.D.: + +```bash +ssh nomad@10.5.1.16 + +# Stack directory +sudo mkdir -p /opt/monitoring + +# Prometheus data on hdd-2 (has more headroom) +sudo mkdir -p /mnt/hdd-2/prometheus-data +sudo chown 65534:65534 /mnt/hdd-2/prometheus-data # nobody user (Prometheus default) + +# Grafana data on hdd-2 +sudo mkdir -p /mnt/hdd-2/grafana-data +sudo chown 472:472 /mnt/hdd-2/grafana-data # grafana user + +# Config directories +sudo mkdir -p /opt/monitoring/provisioning/datasources +sudo mkdir -p /opt/monitoring/provisioning/dashboards +``` + +### 6.2 — prometheus.yml + +Create `/opt/monitoring/prometheus.yml`: + +```yaml +global: + scrape_interval: 15s + evaluation_interval: 15s + +scrape_configs: + # ------- Local (N.O.M.A.D.) ------- + - job_name: 'nomad-node' + static_configs: + - targets: ['node-exporter:9100'] + labels: + host: 'nomad' + + # ------- PlausibleDeniability ------- + - job_name: 'pd-netdata' + metrics_path: /api/v1/allmetrics + params: + format: [prometheus] + static_configs: + - targets: ['10.5.1.X:19999'] # Replace with PD's IP + labels: + host: 'plausible-deniability' + + # ------- Serenity ------- + - job_name: 'serenity-netdata' + metrics_path: /api/v1/allmetrics + params: + format: [prometheus] + static_configs: + - targets: ['10.5.1.5:19999'] + labels: + host: 'serenity' + + # ------- Gotify ------- + # Gotify exposes /health but no Prometheus endpoint natively. + # Use blackbox exporter or just rely on Uptime Kuma. + + # ------- n8n ------- + # n8n doesn't expose Prometheus metrics by default. + # Monitor via container resource metrics from Netdata. + + # -------- Add more targets as needed -------- + # When node-exporter is installed on PD and Serenity: + # - job_name: 'pd-node' + # static_configs: + # - targets: ['10.5.1.X:9100'] + # labels: + # host: 'plausible-deniability' + # + # - job_name: 'serenity-node' + # static_configs: + # - targets: ['10.5.1.5:9100'] + # labels: + # host: 'serenity' +``` + +> **Netdata as a Prometheus target:** Both PD and Serenity already run Netdata. Netdata has a built-in Prometheus exporter at `/api/v1/allmetrics?format=prometheus`. This gives you CPU, memory, disk, network, and ZFS metrics without installing node-exporter on those hosts. + +### 6.3 — Grafana Provisioning + +Create `/opt/monitoring/provisioning/datasources/prometheus.yml`: + +```yaml +apiVersion: 1 + +datasources: + - name: Prometheus + type: prometheus + access: proxy + url: http://prometheus:9090 + isDefault: true + editable: true +``` + +Create `/opt/monitoring/provisioning/dashboards/dashboards.yml`: + +```yaml +apiVersion: 1 + +providers: + - name: 'default' + orgId: 1 + folder: '' + type: file + disableDeletion: false + editable: true + options: + path: /var/lib/grafana/dashboards + foldersFromFilesStructure: false +``` + +### 6.4 — docker-compose.yaml + +Create `/opt/monitoring/docker-compose.yaml`: + +```yaml +name: monitoring + +services: + prometheus: + image: prom/prometheus:v3.4.0 + container_name: prometheus + restart: unless-stopped + ports: + - "9090:9090" + command: + - '--config.file=/etc/prometheus/prometheus.yml' + - '--storage.tsdb.path=/prometheus' + - '--storage.tsdb.retention.time=90d' + - '--web.enable-lifecycle' + volumes: + - /opt/monitoring/prometheus.yml:/etc/prometheus/prometheus.yml:ro + - /mnt/hdd-2/prometheus-data:/prometheus + networks: + - monitoring + + grafana: + image: grafana/grafana:13.0.1 + container_name: grafana + restart: unless-stopped + ports: + - "3000:3000" + environment: + TZ: ${TZ} + GF_SECURITY_ADMIN_USER: ${GF_ADMIN_USER} + GF_SECURITY_ADMIN_PASSWORD: ${GF_ADMIN_PASS} + GF_SERVER_ROOT_URL: https://${GF_HOST}/ + volumes: + - /mnt/hdd-2/grafana-data:/var/lib/grafana + - /opt/monitoring/provisioning:/etc/grafana/provisioning:ro + networks: + - monitoring + + node-exporter: + image: prom/node-exporter:v1.9.0 + container_name: node-exporter + restart: unless-stopped + ports: + - "9100:9100" + command: + - '--path.procfs=/host/proc' + - '--path.sysfs=/host/sys' + - '--path.rootfs=/rootfs' + - '--collector.filesystem.mount-points-exclude=^/(sys|proc|dev|host|etc)($$|/)' + volumes: + - /proc:/host/proc:ro + - /sys:/host/sys:ro + - /:/rootfs:ro + networks: + - monitoring + +networks: + monitoring: + driver: bridge +``` + +> **Image notes:** +> - `prom/prometheus:v3.4.0` — Prometheus v3 is the current major line. **Do not use `:latest`** — it still resolves to 2.x due to a known tagging issue. Always use an explicit `v3.x.y` tag. +> - `grafana/grafana:13.0.1` — Grafana 13 is the current stable. +> - `prom/node-exporter:v1.9.0` — check [releases](https://github.com/prometheus/node_exporter/releases). + +### 6.5 — .env.example + +```bash +# /opt/monitoring/.env.example + +TZ=America/New_York + +# Grafana +GF_ADMIN_USER=admin +GF_ADMIN_PASS=CHANGE_ME +GF_HOST=grafana.paccoco.com +``` + +### 6.6 — Pangolin Configuration + +Grafana runs on N.O.M.A.D., not PD where the main Newt agent lives. You have two options: + +**Option A — Route via N.O.M.A.D.'s existing Newt (recommended if already connected)** + +N.O.M.A.D. already has a Newt container from the Project N.O.M.A.D. setup. If it's connected to your Pangolin VPS, just add a new resource in the Pangolin dashboard pointing to `http://grafana:3000` or `http://10.5.1.16:3000`. + +**Option B — Add a dedicated Newt to the monitoring stack** + +If N.O.M.A.D.'s existing Newt is not connected to Pangolin (or is a separate Pangolin instance), add Newt to the monitoring compose: + +```yaml + newt: + image: ghcr.io/fosrl/newt:latest + container_name: monitoring-newt + restart: unless-stopped + environment: + PANGOLIN_ENDPOINT: ${PANGOLIN_ENDPOINT} + NEWT_ID: ${NEWT_ID} + NEWT_SECRET: ${NEWT_SECRET} + networks: + - monitoring +``` + +Then in Pangolin dashboard: + +| Field | Value | +|-------|-------| +| Domain | `grafana.paccoco.com` | +| Scheme | `http` | +| Host | `grafana` (Docker DNS via shared network) | +| Port | `3000` | + +**Option C — Direct IP routing (simplest, no Newt needed)** + +If Pangolin's Newt on PD can reach N.O.M.A.D. by LAN IP (they're on the same subnet): + +| Field | Value | +|-------|-------| +| Domain | `grafana.paccoco.com` | +| Scheme | `http` | +| Host | `10.5.1.16` | +| Port | `3000` | + +> **Decision point:** Check if N.O.M.A.D.'s existing Newt is connected to your Pangolin instance before deploying. Run `docker ps --filter name=newt` on N.O.M.A.D. to verify. If it's running and connected, Option A is zero-effort. If not, Option C is the simplest fallback. + +### 6.7 — Validate & Deploy + +```bash +ssh nomad@10.5.1.16 + +cd /opt/monitoring +cp .env.example .env +nano .env # Fill in real values + +docker compose --env-file .env config +docker compose --env-file .env up -d +``` + +### 6.8 — Post-Deploy Verification + +```bash +# All three containers running? +docker ps --filter name=prometheus --filter name=grafana --filter name=node-exporter + +# Prometheus scraping targets? +curl -s http://localhost:9090/api/v1/targets | python3 -m json.tool | head -40 + +# Grafana UI accessible? +curl -s -o /dev/null -w "%{http_code}" http://localhost:3000/ +# Expected: 200 or 302 + +# Node exporter metrics flowing? +curl -s http://localhost:9100/metrics | head -10 +``` + +### 6.9 — Recommended Dashboards + +Import these from [Grafana Dashboard Library](https://grafana.com/grafana/dashboards/): + +| Dashboard | ID | Purpose | +|-----------|----|---------| +| Node Exporter Full | 1860 | System metrics for N.O.M.A.D. | +| Docker Container Stats | 893 | Container resource usage | +| Netdata via Prometheus | (search) | PD and Serenity system metrics | + +To import: Grafana → Dashboards → New → Import → Enter dashboard ID. + +### 6.10 — Metrics Targets Summary + +| Target | Host | Method | Endpoint | +|--------|------|--------|----------| +| N.O.M.A.D. system | localhost | node-exporter | node-exporter:9100 | +| PD system | 10.5.1.X | Netdata Prometheus | 10.5.1.X:19999/api/v1/allmetrics | +| Serenity system | 10.5.1.5 | Netdata Prometheus | 10.5.1.5:19999/api/v1/allmetrics | +| ZFS pools | via Netdata | Netdata exports ZFS metrics | Included in Netdata scrape | +| Container stats | via Netdata | Netdata exports cgroup metrics | Included in Netdata scrape | +| Plex streams | Tautulli | n8n polling → Postgres | Via n8n workflow (Phase 3) | +| qBit stats | qBittorrent API | n8n polling → Postgres | Via n8n workflow (Phase 3) | +| Tailscale latency | Tailscale API | n8n polling → Postgres | Via n8n workflow (Phase 3) | + +### 6.11 — n8n Integration (Grafana → n8n alert webhook) + +In Grafana → Alerting → Contact Points, create a webhook contact point: + +| Field | Value | +|-------|-------| +| Name | `n8n-alerts` | +| Type | Webhook | +| URL | `https://n8n.paccoco.com/webhook/grafana-alert` | +| HTTP Method | POST | + +Then create alert rules for: +- ZFS pool utilization > 85% +- Container memory > 90% of limit +- Host CPU sustained > 90% for 5 minutes +- Disk I/O latency spikes + +These fire into the "Grafana Alert Remediation" n8n workflow (see Phase 3). + +--- + +## Phase 7 — LiteLLM (AI Gateway) + +**Host:** PlausibleDeniability +**Stack directory:** `/mnt/docker-ssd/docker/compose/ai/` (same stack as Qdrant) +**Why:** Replaces manual multi-model routing with a unified OpenAI-compatible API. Every app (OpenWebUI, Continue.dev, n8n, Paperless summarization) points at one endpoint instead of juggling three Ollama IPs. + +### 7.1 — What LiteLLM Does + +LiteLLM sits in front of your three Ollama instances and presents a single OpenAI-compatible API at `http://litellm:4000`. It handles: + +- **Model routing** — requests for `qwen3:32b` go to ROCINANTE, `qwen2.5:14b` goes to PD, `phi4` goes to N.O.M.A.D. +- **Failover** — if ROCINANTE is offline, LiteLLM can fall back to PD automatically +- **Load balancing** — distribute requests across instances running the same model +- **Usage tracking** — logs token counts, latency, and costs per model/user via its built-in database + +### 7.2 — Scaffold Directories + +```bash +# Config on SSD (SQLite DB for usage tracking) +sudo mkdir -p /mnt/docker-ssd/docker/appdata/litellm + +# LiteLLM config file +sudo mkdir -p /mnt/docker-ssd/docker/compose/ai/litellm +``` + +### 7.3 — LiteLLM Config + +Create `/mnt/docker-ssd/docker/compose/ai/litellm/config.yaml`: + +```yaml +model_list: + # ---- ROCINANTE (RTX 4090, 24GB) — heavy reasoning ---- + - model_name: "heavy" + litellm_params: + model: "ollama/qwen3:32b" + api_base: "http://10.5.1.ROCINANTE:11434" + timeout: 300 + stream_timeout: 300 + model_info: + description: "Heavy reasoning, long context, complex code" + + - model_name: "heavy" + litellm_params: + model: "ollama/deepseek-r1:32b" + api_base: "http://10.5.1.ROCINANTE:11434" + timeout: 300 + stream_timeout: 300 + model_info: + description: "Deep reasoning fallback on ROCINANTE" + + # ---- PlausibleDeniability (RTX 2080 Ti, 11GB) — general ---- + - model_name: "medium" + litellm_params: + model: "ollama/qwen2.5:14b" + api_base: "http://host.docker.internal:11434" + timeout: 120 + stream_timeout: 120 + model_info: + description: "General homelab assistant, RAG queries" + + # ---- N.O.M.A.D. (GTX 1080, 8GB) — lightweight ---- + - model_name: "light" + litellm_params: + model: "ollama/phi4" + api_base: "http://10.5.1.16:11434" + timeout: 60 + stream_timeout: 60 + model_info: + description: "Fast local inference, lightweight tasks" + + - model_name: "light" + litellm_params: + model: "ollama/llama3.2:3b" + api_base: "http://10.5.1.16:11434" + timeout: 60 + stream_timeout: 60 + model_info: + description: "Ultra-light fallback on N.O.M.A.D." + + # ---- Embeddings ---- + - model_name: "embed" + litellm_params: + model: "ollama/nomic-embed-text" + api_base: "http://host.docker.internal:11434" + model_info: + description: "Text embeddings for RAG pipeline" + + # ---- Direct model access (bypass routing) ---- + # These let you request a specific model by its full name + - model_name: "ollama/qwen3:32b" + litellm_params: + model: "ollama/qwen3:32b" + api_base: "http://10.5.1.ROCINANTE:11434" + + - model_name: "ollama/qwen2.5:14b" + litellm_params: + model: "ollama/qwen2.5:14b" + api_base: "http://host.docker.internal:11434" + + - model_name: "ollama/phi4" + litellm_params: + model: "ollama/phi4" + api_base: "http://10.5.1.16:11434" + +litellm_settings: + drop_params: true + set_verbose: false + request_timeout: 300 + num_retries: 2 + retry_after: 5 + allowed_fails: 3 + cooldown_time: 60 + +general_settings: + master_key: "os.environ/LITELLM_MASTER_KEY" +``` + +> **DEPLOYED NOTE (2026-05-05):** Do NOT add `database_url` to general_settings — newer LiteLLM versions include Prisma ORM that requires PostgreSQL, and adding any database_url causes a crash loop. Without it, LiteLLM runs in config-only mode which is fine for homelab use. The `master_key` uses `os.environ/LITELLM_MASTER_KEY` syntax to read from the container's environment variable (set via .env → docker compose). + +> **Routing explained:** Multiple entries with the same `model_name` (like "heavy") enable load balancing and failover within that tier. When you request model `heavy`, LiteLLM picks the healthiest deployment. The direct-access entries (like `ollama/qwen3:32b`) let you bypass the tier system and target a specific model when needed. + +> **Replace IPs:** Update `10.5.1.ROCINANTE` with ROCINANTE's actual LAN or Tailscale IP. PD uses `host.docker.internal` since LiteLLM runs on PD alongside Ollama. + +### 7.4 — docker-compose.yaml (updated ai stack) + +Update `/mnt/docker-ssd/docker/compose/ai/docker-compose.yaml` to add LiteLLM: + +```yaml +name: ai + +services: + qdrant: + image: qdrant/qdrant:v1.14.0 + container_name: qdrant + restart: unless-stopped + ports: + - "6333:6333" # REST API + - "6334:6334" # gRPC + environment: + TZ: ${TZ} + QDRANT__SERVICE__GRPC_PORT: 6334 + QDRANT__STORAGE__STORAGE_PATH: /qdrant/storage + QDRANT__STORAGE__SNAPSHOTS_PATH: /qdrant/snapshots + volumes: + - /mnt/docker-ssd/docker/appdata/qdrant/storage:/qdrant/storage + - /mnt/docker-ssd/docker/appdata/qdrant/snapshots:/qdrant/snapshots + networks: + - ai-services + - default + + litellm: + image: ghcr.io/berriai/litellm:main-latest + container_name: litellm + restart: unless-stopped + ports: + - "4000:4000" + environment: + TZ: ${TZ} + LITELLM_MASTER_KEY: ${LITELLM_MASTER_KEY} + volumes: + - /mnt/docker-ssd/docker/compose/ai/litellm/config.yaml:/app/config.yaml:ro + - /mnt/docker-ssd/docker/appdata/litellm:/app/data + command: ["--config", "/app/config.yaml", "--port", "4000"] + extra_hosts: + - "host.docker.internal:host-gateway" + networks: + - ai-services + - default + + reranker: + image: ghcr.io/huggingface/text-embeddings-inference:cpu-latest + container_name: reranker + restart: unless-stopped + ports: + - "8787:80" + environment: + MODEL_ID: ${RERANKER_MODEL} + volumes: + - /mnt/docker-ssd/docker/appdata/reranker:/data + networks: + - ai-services + - default + + whisper: + image: fedirz/faster-whisper-server:latest-cuda + container_name: whisper + restart: unless-stopped + ports: + - "8786:8000" + environment: + TZ: ${TZ} + WHISPER__MODEL: ${WHISPER_MODEL} + WHISPER__DEVICE: cuda + WHISPER__COMPUTE_TYPE: float16 + volumes: + - /mnt/docker-ssd/docker/appdata/whisper:/root/.cache/huggingface + deploy: + resources: + reservations: + devices: + - driver: nvidia + count: 1 + capabilities: [gpu] + networks: + - ai-services + - default + + # ------------------------------------------------------- + # Ollama and OpenWebUI go here when deployed. + # They share this stack and the default + ai-services networks. + # OpenWebUI → Qdrant at http://qdrant:6333 + # OpenWebUI → LiteLLM at http://litellm:4000 (or direct Ollama) + # ------------------------------------------------------- + + # ollama: + # image: ollama/ollama:latest + # container_name: ollama + # ... + + # openwebui: + # image: ghcr.io/open-webui/open-webui:main + # container_name: openwebui + # ... + +networks: + ai-services: + name: ai-services +``` + +> **GPU sharing note:** On PD, the RTX 2080 Ti is shared between Plex (hardware transcoding), Immich ML, and Ollama. faster-whisper will also need the GPU when processing audio. These workloads are bursty (not constant), so time-sharing the GPU is viable — but be aware that a large Whisper transcription job will temporarily impact Ollama inference latency. If this becomes an issue, consider running Whisper on N.O.M.A.D.'s GTX 1080 instead and change `whisper`'s Ollama-style routing accordingly. + +> **Reranker on CPU:** The reranker uses the `cpu-latest` image variant intentionally — do NOT pin to `cpu-1.5` as it has an hf-hub compatibility bug. Reranking is a lightweight operation (scoring ~50 chunks takes <1s on CPU) and doesn't justify competing for GPU VRAM. The model loads via safetensors fallback since no ONNX files exist for bge-reranker-v2-m3. Max batch size is 4 on CPU. If you find latency is an issue, a GPU variant exists (`ghcr.io/huggingface/text-embeddings-inference:turing-latest` for your 2080 Ti). + +### 7.5 — .env.example (updated ai stack) + +```bash +# /mnt/docker-ssd/docker/compose/ai/.env.example + +TZ=America/New_York + +# LiteLLM +LITELLM_MASTER_KEY=sk-CHANGE_ME + +# Reranker +RERANKER_MODEL=BAAI/bge-reranker-v2-m3 + +# Whisper +WHISPER_MODEL=Systran/faster-distil-whisper-large-v3 +``` + +### 7.6 — Validate & Deploy + +```bash +cd /mnt/docker-ssd/docker/compose/ai +cp .env.example .env +nano .env # Set LITELLM_MASTER_KEY=$(openssl rand -hex 32) + +docker compose --env-file .env config +docker compose --env-file .env up -d +``` + +### 7.7 — Post-Deploy Verification + +```bash +# All containers running? +docker ps --filter name=qdrant --filter name=litellm --filter name=reranker --filter name=whisper + +# LiteLLM health? +curl -s http://localhost:4000/health + +# LiteLLM can see all models? +curl -s http://localhost:4000/v1/models \ + -H "Authorization: Bearer YOUR_MASTER_KEY" | python3 -m json.tool + +# Test a chat completion through LiteLLM +curl -s http://localhost:4000/v1/chat/completions \ + -H "Authorization: Bearer YOUR_MASTER_KEY" \ + -H "Content-Type: application/json" \ + -d '{ + "model": "medium", + "messages": [{"role": "user", "content": "Hello, which model are you?"}] + }' | python3 -m json.tool + +# Test embeddings through LiteLLM +curl -s http://localhost:4000/v1/embeddings \ + -H "Authorization: Bearer YOUR_MASTER_KEY" \ + -H "Content-Type: application/json" \ + -d '{ + "model": "embed", + "input": "test embedding" + }' | python3 -m json.tool +``` + +### 7.8 — Integration Updates + +With LiteLLM deployed, all apps should point at `http://litellm:4000` (within the `ai-services` network) or `http://PD_IP:4000` (from external hosts) instead of direct Ollama endpoints. + +**OpenWebUI:** Settings → Connections → add OpenAI-compatible endpoint: +- URL: `http://litellm:4000/v1` +- API Key: your `LITELLM_MASTER_KEY` +- This gives OpenWebUI access to all models across all three machines through one connection + +**Continue.dev:** Update `~/.continue/config.json`: +```json +{ + "models": [ + { + "title": "Heavy (ROCINANTE)", + "provider": "openai", + "model": "heavy", + "apiBase": "http://PD_TAILSCALE_IP:4000/v1", + "apiKey": "YOUR_MASTER_KEY" + }, + { + "title": "Medium (PD)", + "provider": "openai", + "model": "medium", + "apiBase": "http://PD_TAILSCALE_IP:4000/v1", + "apiKey": "YOUR_MASTER_KEY" + } + ], + "tabAutocompleteModel": { + "title": "Light (N.O.M.A.D.)", + "provider": "openai", + "model": "light", + "apiBase": "http://PD_TAILSCALE_IP:4000/v1", + "apiKey": "YOUR_MASTER_KEY" + } +} +``` + +> **Key change:** Continue.dev now uses `provider: "openai"` instead of `provider: "ollama"` and points at LiteLLM. One IP to remember, and if you add or move models between machines, you only update `config.yaml` — not every app. + +**n8n workflows:** All HTTP Request nodes that call Ollama directly should be updated: +- Old: `http://10.5.1.16:11434/api/generate` (N.O.M.A.D.) +- New: `http://litellm:4000/v1/chat/completions` with `model: "light"` +- n8n is on the `ai-services` network, so it reaches LiteLLM by Docker DNS + +**n8n Multi-Model Query Router workflow:** This workflow is now **simplified dramatically** — instead of a Switch node routing to three different Ollama IPs, it becomes a single HTTP Request node that passes the model tier name (`light`, `medium`, `heavy`) to LiteLLM and lets the gateway handle routing. The Switch node logic can be removed entirely. + +--- + +## Phase 8 — Reranker (RAG Quality) + +**Host:** PlausibleDeniability (deployed as part of the `ai` stack in Phase 7) +**Service:** `reranker` container (already in the compose above) +**Why:** Dramatically improves RAG answer quality by filtering out noisy retrieval results before they reach the LLM. + +### 8.1 — How Reranking Works + +Without a reranker, your RAG pipeline does: +``` +Query → embed → Qdrant top-10 by vector similarity → all 10 chunks go to LLM +``` + +The problem: vector similarity often returns "close but irrelevant" chunks. The LLM gets noisy context and hallucinates. + +With a reranker: +``` +Query → embed → Qdrant top-50 by vector similarity → reranker scores each (query, chunk) pair → top-5 by relevance go to LLM +``` + +The reranker is a cross-encoder that reads the full query AND each chunk together, producing a much more accurate relevance score than vector distance alone. It over-retrieves cheaply from Qdrant, then precisely filters. + +### 8.2 — Scaffold Directories + +```bash +# Model cache on SSD (reranker model is ~1.1GB, downloaded on first start) +sudo mkdir -p /mnt/docker-ssd/docker/appdata/reranker +``` + +### 8.3 — Post-Deploy Verification + +```bash +# Container running? +docker ps --filter name=reranker + +# Health check? +curl -s http://localhost:8787/health + +# Test reranking +curl -s http://localhost:8787/rerank \ + -H "Content-Type: application/json" \ + -d '{ + "query": "How do I restart a Docker container?", + "texts": [ + "Use docker restart to restart a running container.", + "Docker was founded in 2013 by Solomon Hykes.", + "The docker compose down command stops and removes containers.", + "Kubernetes pods can be restarted by deleting them." + ] + }' | python3 -m json.tool +# Expected: the first text scores highest, second scores lowest +``` + +### 8.4 — n8n RAG Pipeline Integration + +Update the "Qdrant Index Updater" and any RAG query workflows to include a reranking step. + +**Updated RAG Query Workflow (for OpenWebUI or any n8n-based query):** +``` +Trigger: Webhook node (POST with query) +Step 1: HTTP Request → LiteLLM /v1/embeddings + - model: "embed" + - input: query text +Step 2: HTTP Request → Qdrant API (POST /collections/{name}/points/search) + - vector: embedding from step 1 + - limit: 50 (over-retrieve) +Step 3: HTTP Request → Reranker (POST http://reranker:80/rerank) + - query: original query text + - texts: array of 50 chunk texts from Qdrant results +Step 4: Code node → take top 5 by reranker score, format as context +Step 5: HTTP Request → LiteLLM /v1/chat/completions + - model: "medium" (or "heavy" for complex queries) + - messages: system prompt with top-5 context + user query +Step 6: Return response via webhook +``` + +> **Key difference from the original plan:** Step 2 now retrieves 50 results instead of 10, and Step 3 (reranking) filters down to the best 5. This "over-retrieve then rerank" pattern is the standard approach for production RAG systems. + +--- + +## Phase 9 — faster-whisper (Speech-to-Text) + +**Host:** PlausibleDeniability (deployed as part of the `ai` stack in Phase 7) +**Service:** `whisper` container (already in the compose above) +**Why:** Replaces shelved Scriberr with an OpenAI-compatible STT API. No SQLite dependency, no ZFS/ACL issues. + +### 9.1 — Scaffold Directories + +```bash +# Model cache on SSD (Whisper models are 1-3GB) +sudo mkdir -p /mnt/docker-ssd/docker/appdata/whisper +``` + +### 9.2 — Post-Deploy Verification + +```bash +# Container running? +docker ps --filter name=whisper + +# Health check? +curl -s http://localhost:8786/health + +# Test transcription with a sample audio file +curl -s http://localhost:8786/v1/audio/transcriptions \ + -F "file=@/path/to/test-audio.wav" \ + -F "model=Systran/faster-distil-whisper-large-v3" \ + | python3 -m json.tool +``` + +### 9.3 — n8n Integration Workflows + +**Workflow: Voice Note → Text → Summary** +``` +Trigger: Webhook node (POST with audio file in body) +Step 1: HTTP Request → Whisper (POST http://whisper:8000/v1/audio/transcriptions) + - Multipart form with audio file + - response_format: "json" +Step 2: HTTP Request → LiteLLM /v1/chat/completions + - model: "medium" + - Prompt: "Summarize this voice note concisely: {transcript}" +Step 3: HTTP Request → Gotify → push summary to phone +Step 4: (Optional) Postgres node → log transcript and summary +``` + +**Workflow: Audio File → Paperless Document** +``` +Trigger: Webhook or filesystem watcher +Step 1: HTTP Request → Whisper → get transcript +Step 2: Code node → format transcript as text document +Step 3: HTTP Request → Paperless API (POST /api/documents/post_document/) + - Upload transcript as .txt + - Tag: "transcription" +Step 4: Gotify notification: "Audio transcribed and filed: {title}" +``` + +**Home Assistant Voice Integration:** +If you want HA voice commands, Whisper can serve as the STT backend: +1. In Home Assistant → Settings → Voice Assistants +2. Add speech-to-text provider: "Whisper" at `http://PD_IP:8786` +3. Pair with Piper TTS (future addition) for full voice assistant loop + +### 9.4 — GPU vs CPU Considerations + +The compose file above uses the GPU variant with CUDA. If GPU contention with Ollama becomes an issue: + +**Option A — CPU fallback on PD:** +Change the image from `latest-cuda` to `fedirz/faster-whisper-server:latest-cpu` and remove the `deploy.resources` block and the CUDA environment variables (`WHISPER__DEVICE`, `WHISPER__COMPUTE_TYPE`). Transcription will be slower (~4x real-time instead of ~20x) but won't compete for VRAM. + +**Option B — Move to N.O.M.A.D.:** +Run Whisper on N.O.M.A.D.'s GTX 1080 which is less contested. Add it to N.O.M.A.D.'s compose or run standalone. N.O.M.A.D.'s 8GB VRAM handles Whisper models comfortably since phi4 doesn't fill it. + +--- + +## Additional Tools Setup + +### Continue.dev (Local AI Code Completion) + +1. Install the Continue extension in VS Code +2. Create/edit `~/.continue/config.json`: + +**With LiteLLM (recommended — see Phase 7):** +```json +{ + "models": [ + { + "title": "Heavy (ROCINANTE)", + "provider": "openai", + "model": "heavy", + "apiBase": "http://PD_TAILSCALE_IP:4000/v1", + "apiKey": "YOUR_LITELLM_MASTER_KEY" + }, + { + "title": "Medium (PD)", + "provider": "openai", + "model": "medium", + "apiBase": "http://PD_TAILSCALE_IP:4000/v1", + "apiKey": "YOUR_LITELLM_MASTER_KEY" + } + ], + "tabAutocompleteModel": { + "title": "Light (N.O.M.A.D.)", + "provider": "openai", + "model": "light", + "apiBase": "http://PD_TAILSCALE_IP:4000/v1", + "apiKey": "YOUR_LITELLM_MASTER_KEY" + } +} +``` + +> One IP, one API key, all three machines. If you add or move models, update LiteLLM's `config.yaml` — not every app. + +**Without LiteLLM (direct Ollama, if Phase 7 is not yet deployed):** +```json +{ + "models": [ + { + "title": "PD - qwen2.5:14b", + "provider": "ollama", + "model": "qwen2.5:14b", + "apiBase": "http://PD_TAILSCALE_IP:11434" + }, + { + "title": "ROCINANTE - qwen3:32b", + "provider": "ollama", + "model": "qwen3:32b", + "apiBase": "http://ROCINANTE_TAILSCALE_IP:11434" + } + ], + "tabAutocompleteModel": { + "title": "N.O.M.A.D. - phi4", + "provider": "ollama", + "model": "phi4", + "apiBase": "http://NOMAD_TAILSCALE_IP:11434" + } +} +``` + +### Obsidian + Gitea Sync + +1. In Obsidian, install the "Obsidian Git" community plugin +2. Initialize your vault as a git repo: + ```bash + cd /path/to/obsidian/vault + git init + git remote add origin http://PD_IP:3000/fizzlepoof/obsidian-vault.git + ``` +3. In Obsidian Git settings: + - Auto backup interval: 10 minutes + - Pull on startup: enabled +4. Create the repo in Gitea first: `http://PD_IP:3000` → New Repository → `obsidian-vault` +5. Add a Gitea webhook to trigger the n8n "Qdrant Index Updater" workflow (see Phase 3) + +### Homepage Ollama Widget + +Add to your Homepage configuration (`/mnt/tank/docker/appdata/homepage/services.yaml`): + +```yaml +- AI: + - Ollama (PD): + icon: ollama.svg + href: http://PD_IP:11434 + widget: + type: ollama + url: http://PD_IP:11434 + - Ollama (ROCINANTE): + icon: ollama.svg + href: http://ROCINANTE_IP:11434 + widget: + type: ollama + url: http://ROCINANTE_IP:11434 +``` + +--- + +## Deployment Order Summary + +``` +Week 1: Phase 1 — Gotify + └─ Deploy, create app tokens, install phone app + └─ Test: send manual notification via API + +Week 1: Phase 2 — Qdrant + └─ Deploy, verify REST API + └─ Create initial collections (empty, ready for n8n) + +Week 2: Phase 3 — n8n + └─ Deploy, create admin account + └─ Build workflows incrementally: + Day 1: Gitea commit → Gotify (simplest, proves the pipeline) + Day 2: Sonarr/Radarr → TMDB → Gotify + Discord + Day 3: Tautulli play logging + weekly digest + Day 4: Uptime Kuma enhanced alerts + Day 5: ZFS pool monitoring + Day 6: Multi-model query router + Day 7: Qdrant index updater + +Week 3: Phase 4 — Paperless-NGX + └─ Deploy, ingest test documents + └─ Build n8n workflow: email → Paperless → Ollama summary + └─ Set up document tags and correspondents for L&L Crafts + +Week 4: Phase 5 — Home Assistant + └─ Deploy, onboard, add integrations + └─ Connect to n8n via long-lived access token + └─ Set up smart plug monitoring + +Week 4: Phase 6 — Grafana + Prometheus (on N.O.M.A.D.) + └─ Deploy, verify scrape targets + └─ Import dashboards + └─ Set up Grafana → n8n alert webhook + └─ Build alert rules + +Week 5: Phase 7/8/9 — AI Stack Expansion (all deploy together in ai stack) + └─ LiteLLM: deploy, verify model routing across all 3 machines + └─ Reranker: deploy, test scoring with sample chunks + └─ faster-whisper: deploy, test transcription + └─ Update OpenWebUI, Continue.dev, and n8n to use LiteLLM endpoint + └─ Update RAG workflows to include reranking step + └─ Build voice note → transcription → summary workflow +``` + +--- + +## Post-Deployment Validation Master Checklist + +Run through this after all six phases are deployed: + +### Infrastructure Health + +- [ ] `docker ps` on PD shows: gotify, n8n, qdrant, litellm, reranker, whisper, paperless, paperless-gotenberg, paperless-tika, homeassistant — all healthy +- [ ] `docker ps` on N.O.M.A.D. shows: prometheus, grafana, node-exporter — all healthy +- [ ] All services accessible via Pangolin subdomains (gotify, n8n, paperless, ha, grafana) + +### Network Connectivity + +- [ ] n8n can reach shared-postgres (test: check n8n logs for successful DB migration) +- [ ] n8n can reach Gotify (test: trigger a workflow that sends a notification) +- [ ] n8n can reach Qdrant at `http://qdrant:6333` via `ai-services` network (test: query collections from n8n HTTP node) +- [ ] n8n can reach LiteLLM at `http://litellm:4000` via `ai-services` network (test: send a chat completion) +- [ ] LiteLLM can route to all 3 Ollama instances (test: request model "light", "medium", "heavy" and verify each responds) +- [ ] Reranker responds at `http://reranker:80/rerank` (test: POST sample texts) +- [ ] Whisper responds at `http://whisper:8000/v1/audio/transcriptions` (test: transcribe a sample .wav) +- [ ] Paperless can reach shared-postgres and shared-redis +- [ ] Prometheus can scrape PD and Serenity Netdata endpoints +- [ ] Grafana alert webhook reaches n8n + +### Data Flow End-to-End + +- [ ] Sonarr webhook → n8n → Gotify push on phone +- [ ] Gitea commit → n8n → Qdrant indexing → Gotify notification +- [ ] RAG query → Qdrant top-50 → reranker top-5 → LiteLLM → accurate answer +- [ ] Audio file → Whisper transcription → n8n → Gotify summary +- [ ] Document dropped in consume folder → Paperless OCR → n8n summary → Gotify +- [ ] Grafana alert fires → n8n webhook → remediation action or Gotify alert +- [ ] HA presence change → n8n → server wake/sleep + +### Backup Considerations + +- [ ] Gotify SQLite DB is on SSD — include `/mnt/docker-ssd/docker/appdata/gotify` in backup plan +- [ ] Qdrant storage on SSD — include `/mnt/docker-ssd/docker/appdata/qdrant` in backup plan +- [ ] n8n config/workflows — include `/mnt/tank/docker/appdata/n8n` in backup plan +- [ ] Paperless media — include `/mnt/tank/docker/appdata/paperless/media` in backup plan +- [ ] HA config — include `/mnt/docker-ssd/docker/appdata/homeassistant` in backup plan +- [ ] LiteLLM DB + config — include `/mnt/docker-ssd/docker/appdata/litellm` and `compose/ai/litellm/config.yaml` in backup plan +- [ ] Grafana/Prometheus data — include `/mnt/hdd-2/grafana-data` and `/mnt/hdd-2/prometheus-data` in N.O.M.A.D. backup plan +- [ ] All new databases (n8n, paperless) are in shared-postgres — ensure pg_dump covers them + +### Storage Capacity Check + +- [ ] `docker-ssd` (500GB Crucial MX500) still has headroom after adding Gotify, Qdrant, HA, Paperless data +- [ ] `tank` mirror has room for n8n config + Paperless media + consume/export +- [ ] N.O.M.A.D. hdd-2 has room for Prometheus TSDB (90-day retention) + Grafana data + +--- + +> **Remember:** Everything self-hosted. Nothing touches the cloud. diff --git a/docs/planning/DEPLOY_KIMA_HUB.md b/docs/planning/DEPLOY_KIMA_HUB.md new file mode 100644 index 0000000..0a06cdb --- /dev/null +++ b/docs/planning/DEPLOY_KIMA_HUB.md @@ -0,0 +1,91 @@ +# Kima-Hub Deployment Plan + +**Host:** PlausibleDeniability (PD) +**Port:** 3333 +**Stack dir:** `/mnt/docker-ssd/docker/compose/media-extra/` +**Status:** Pending + +## What It Is + +Kima-Hub is a self-hosted music streaming platform — think personal Spotify. Point it at your music library and it handles cataloging, artist discovery, AI-powered playlists, podcast subscriptions, and seamless integration with Lidarr and Audiobookshelf. + +## Pre-Deploy Checklist + +- [ ] Confirm music library path on Unraid NFS (`/mnt/unraid/data/media/music` or similar) +- [ ] Create appdata directory: `sudo mkdir -p /mnt/tank/docker/appdata/kima-hub` +- [ ] Create stack directory: `sudo mkdir -p /mnt/docker-ssd/docker/compose/media-extra` + +## docker-compose.yaml + +```yaml +services: + kima-hub: + image: chevron7locked/kima:latest + container_name: kima-hub + restart: unless-stopped + ports: + - "3333:3030" + volumes: + - /mnt/tank/docker/appdata/kima-hub:/data + - /mnt/unraid/data/media/music:/music:ro + environment: + - TZ=${TZ} + healthcheck: + test: ["CMD-SHELL", "cat /proc/net/tcp6 | grep -q 0BD6 || exit 1"] + interval: 30s + timeout: 10s + retries: 3 + networks: + - pangolin + +networks: + pangolin: + external: true +``` + +> **Note:** Internal port is `3030`, mapped to `3333` on the host. Hex `0BD6` = port 3030 for healthcheck. +> **Music path:** Adjust `/mnt/unraid/data/media/music` to match your actual Unraid music library path. +> **Read-only mount:** Music library is mounted `:ro` — Kima-Hub should not need write access to source files. + +## .env.example + +```env +TZ=America/New_York +``` + +## Deploy + +```bash +cd /mnt/docker-ssd/docker/compose/media-extra +sudo docker compose --env-file .env config # validate +sudo docker compose --env-file .env up -d +sudo docker logs kima-hub --tail 30 +``` + +## Post-Deploy + +1. Open `http://10.5.1.6:3333` and create your account +2. Point Kima at your music directory (`/music`) +3. In Lidarr: Settings → Connect → add Kima-Hub webhook +4. In Audiobookshelf: verify Kima can reach it at `http://10.5.1.6:13358` +5. Add Pangolin resource: `kima.paccoco.com` → port 3333 + +## Pangolin Resource + +Add in Pangolin dashboard: +- **Subdomain:** `kima.paccoco.com` +- **Target:** `http://10.5.1.6:3333` +- **Auth:** Enable (personal use only) + +## Storage Decision + +| Path | Tier | Reason | +|------|------|--------| +| `/mnt/tank/docker/appdata/kima-hub` | tank | Config, DB, playlists — not write-heavy | +| `/mnt/unraid/data/media/music` | Unraid NFS (read-only) | Source library stays on Serenity | + +## Notes + +- NFS mount must be up before starting (`sudo bash /mnt/tank/docker/scripts/mount-unraid-nfs.sh`) +- If you transcode on the fly, CPU load on PD will spike; Kima-Hub is CPU-bound for transcoding +- Kima-Hub has no GPU acceleration — pure CPU transcoding diff --git a/docs/planning/DEPLOY_PIHOLE.md b/docs/planning/DEPLOY_PIHOLE.md new file mode 100644 index 0000000..d7a690b --- /dev/null +++ b/docs/planning/DEPLOY_PIHOLE.md @@ -0,0 +1,527 @@ +# Pi-hole Deployment Plan + +**Status:** Pending +**Architecture:** 3-node HA cluster with Keepalived VIP, Unbound recursive DNS, Nebula Sync + +--- + +## Architecture Overview + +``` +[All LAN clients] + ↓ DNS: 10.5.1.53 (Virtual IP — floats to whichever node is MASTER) + ↓ + ┌─────┴──────────────────────────────────┐ + │ Keepalived VIP │ + │ 10.5.1.53/24 │ + └──────┬──────────────┬──────────────────┘ + │ │ │ + PD (MASTER) Serenity (BACKUP1) RPi4 (BACKUP2) + 10.5.1.6 10.5.1.5 10.5.1.X + priority 100 priority 90 priority 80 + Pi-hole :53 Pi-hole :53 Pi-hole :53 + Unbound :5335 Unbound :5335 Unbound :5335 + │ │ │ + └──────────────┴──────────────────┘ + ↓ + [Root DNS servers] + (recursive — no third-party upstream) +``` + +- **Keepalived** floats the VIP `10.5.1.53` to whichever node is alive with the highest priority +- **Unbound** on each node acts as the local recursive resolver (replaces Cloudflare/Quad9) +- **Pi-hole** on each node forwards upstream queries to its local Unbound (`127.0.0.1#5335`) +- **Nebula Sync** (running on PD) syncs all Pi-hole configs from primary → replicas daily + +All nodes use `network_mode: host` for Pi-hole, Unbound, and Keepalived — this is required for Keepalived to manipulate the VIP and for Pi-hole to bind cleanly to port 53. + +--- + +## Pre-Deploy: Network Planning + +1. **Pick a free static IP for the VIP** — `10.5.1.53` is recommended (memorable for DNS). Reserve it in UniFi so DHCP never assigns it. +2. **Find the LAN interface name** on each node: + ```bash + # PD / RPi4: + ip link show + # Look for the interface with 10.5.1.x address — likely eth0, eno1, enp3s0, etc. + ``` +3. **Note RPi4 IP** — fill in `10.5.1.X` throughout this doc once known. Reserve it as static in UniFi. +4. **Verify port 53 is free on PD:** + ```bash + sudo ss -tulnp | grep ':53' + # Should show only 127.0.0.1:53 (TrueNAS loopback resolver). + # If it shows 0.0.0.0:53, see Gotchas section. + ``` + +--- + +## Unbound Config (shared across all nodes) + +Each node runs Unbound as a Docker container on the host network, listening on `127.0.0.1:5335`. + +**Save this as `unbound.conf` in each node's appdata path** (see per-node sections below): + +```ini +server: + verbosity: 1 + interface: 127.0.0.1 + port: 5335 + do-ip4: yes + do-udp: yes + do-tcp: yes + do-ip6: no + + # DNSSEC + auto-trust-anchor-file: "/opt/unbound/etc/unbound/var/root.key" + val-permissive-mode: no + + # Performance + num-threads: 2 + cache-min-ttl: 300 + cache-max-ttl: 86400 + prefetch: yes + prefetch-key: yes + minimal-responses: yes + + # Privacy + qname-minimisation: yes + hide-identity: yes + hide-version: yes + + # Access control + access-control: 127.0.0.1/32 allow + access-control: 0.0.0.0/0 refuse + + # Root hints — download updated copy periodically + root-hints: "/opt/unbound/etc/unbound/var/root.hints" +``` + +**Download root hints** (run once on each node, into the appdata path): +```bash +curl -o /path/to/appdata/unbound/var/root.hints https://www.internic.net/domain/named.cache +mkdir -p /path/to/appdata/unbound/var +# Unbound will auto-manage root.key for DNSSEC +``` + +--- + +## Node 1: PlausibleDeniability (MASTER) + +**Stack dir:** `/mnt/docker-ssd/docker/compose/pihole/` +**Web UI port:** `8953` + +### Pre-Deploy + +```bash +sudo mkdir -p /mnt/tank/docker/appdata/pihole-primary/etc-pihole +sudo mkdir -p /mnt/tank/docker/appdata/unbound-primary/var +sudo mkdir -p /mnt/docker-ssd/docker/compose/pihole +# Download root hints +sudo curl -o /mnt/tank/docker/appdata/unbound-primary/var/root.hints https://www.internic.net/domain/named.cache +``` + +### docker-compose.yaml + +```yaml +services: + unbound: + image: mvance/unbound:latest + container_name: unbound + network_mode: host + restart: unless-stopped + volumes: + - /mnt/tank/docker/appdata/unbound-primary:/opt/unbound/etc/unbound/ + healthcheck: + test: ["CMD-SHELL", "drill @127.0.0.1 -p 5335 cloudflare.com || exit 1"] + interval: 30s + timeout: 10s + retries: 3 + + pihole: + image: pihole/pihole:latest + container_name: pihole + network_mode: host + restart: unless-stopped + depends_on: + unbound: + condition: service_healthy + environment: + TZ: ${TZ} + FTLCONF_webserver_api_password: ${PIHOLE_PASSWORD} + FTLCONF_dns_listeningMode: "LOCAL" + FTLCONF_dns_upstreams: "127.0.0.1#5335" + FTLCONF_webserver_port: "8953" + volumes: + - /mnt/tank/docker/appdata/pihole-primary/etc-pihole:/etc/pihole + cap_add: + - SYS_NICE + + keepalived: + image: osixia/keepalived:2.0.20 + container_name: keepalived + network_mode: host + restart: unless-stopped + cap_add: + - NET_ADMIN + - NET_BROADCAST + - NET_RAW + volumes: + - ./keepalived.conf:/container/environment/01-custom/keepalived.conf + environment: + KEEPALIVED_VIRTUAL_IPS: "#PYTHON2BASH:['10.5.1.53/24']" +``` + +### keepalived.conf (PD — MASTER) + +``` +global_defs { + router_id pihole_pd +} + +vrrp_instance VI_DNS { + state MASTER + interface ${LAN_INTERFACE} + virtual_router_id 53 + priority 100 + advert_int 1 + nopreempt + + authentication { + auth_type PASS + auth_pass ${KEEPALIVED_PASSWORD} + } + + virtual_ipaddress { + 10.5.1.53/24 + } +} +``` +> Replace `${LAN_INTERFACE}` with your actual interface (e.g. `eno1`). Replace `${KEEPALIVED_PASSWORD}` with a shared secret used on all three nodes. + +### .env.example + +```env +TZ=America/New_York +PIHOLE_PASSWORD=CHANGE_ME +KEEPALIVED_PASSWORD=CHANGE_ME +LAN_INTERFACE=eno1 +``` + +### Deploy + +```bash +cd /mnt/docker-ssd/docker/compose/pihole +# Copy unbound.conf to appdata +sudo cp unbound.conf /mnt/tank/docker/appdata/unbound-primary/ +sudo docker compose --env-file .env config +sudo docker compose --env-file .env up -d +sudo docker logs pihole --tail 20 +sudo docker logs unbound --tail 20 +``` + +--- + +## Node 2: Serenity (BACKUP1) + +Serenity runs Unraid. Deploy via Docker Compose in Unraid's **User Scripts** or the **Compose Manager** plugin. + +**Appdata:** `/mnt/user/appdata/` +**Web UI port:** `8954` + +### docker-compose.yaml + +Same structure as PD but with different ports/paths and priority: + +```yaml +services: + unbound: + image: mvance/unbound:latest + container_name: unbound-pihole + network_mode: host + restart: unless-stopped + volumes: + - /mnt/user/appdata/unbound-pihole:/opt/unbound/etc/unbound/ + healthcheck: + test: ["CMD-SHELL", "drill @127.0.0.1 -p 5335 cloudflare.com || exit 1"] + interval: 30s + timeout: 10s + retries: 3 + + pihole: + image: pihole/pihole:latest + container_name: pihole-secondary + network_mode: host + restart: unless-stopped + depends_on: + unbound-pihole: + condition: service_healthy + environment: + TZ: ${TZ} + FTLCONF_webserver_api_password: ${PIHOLE_PASSWORD} + FTLCONF_dns_listeningMode: "LOCAL" + FTLCONF_dns_upstreams: "127.0.0.1#5335" + FTLCONF_webserver_port: "8954" + volumes: + - /mnt/user/appdata/pihole-secondary/etc-pihole:/etc/pihole + cap_add: + - SYS_NICE + + keepalived: + image: osixia/keepalived:2.0.20 + container_name: keepalived-pihole + network_mode: host + restart: unless-stopped + cap_add: + - NET_ADMIN + - NET_BROADCAST + - NET_RAW + volumes: + - ./keepalived.conf:/container/environment/01-custom/keepalived.conf +``` + +### keepalived.conf (Serenity — BACKUP1) + +``` +global_defs { + router_id pihole_serenity +} + +vrrp_instance VI_DNS { + state BACKUP + interface eth0 + virtual_router_id 53 + priority 90 + advert_int 1 + + authentication { + auth_type PASS + auth_pass SAME_PASSWORD_AS_PRIMARY + } + + virtual_ipaddress { + 10.5.1.53/24 + } +} +``` + +--- + +## Node 3: Raspberry Pi 4 (BACKUP2) + +### Install Docker (if not already installed) + +```bash +curl -fsSL https://get.docker.com | sh +sudo usermod -aG docker $USER +``` + +### Pre-Deploy + +```bash +mkdir -p ~/pihole/etc-pihole +mkdir -p ~/pihole/unbound/var +curl -o ~/pihole/unbound/var/root.hints https://www.internic.net/domain/named.cache +sudo apt install keepalived -y +``` + +### docker-compose.yaml + +```yaml +services: + unbound: + image: mvance/unbound:latest + container_name: unbound + network_mode: host + restart: unless-stopped + volumes: + - ~/pihole/unbound:/opt/unbound/etc/unbound/ + healthcheck: + test: ["CMD-SHELL", "drill @127.0.0.1 -p 5335 cloudflare.com || exit 1"] + interval: 30s + timeout: 10s + retries: 3 + + pihole: + image: pihole/pihole:latest + container_name: pihole + network_mode: host + restart: unless-stopped + depends_on: + unbound: + condition: service_healthy + environment: + TZ: ${TZ} + FTLCONF_webserver_api_password: ${PIHOLE_PASSWORD} + FTLCONF_dns_listeningMode: "LOCAL" + FTLCONF_dns_upstreams: "127.0.0.1#5335" + FTLCONF_webserver_port: "8955" + volumes: + - ~/pihole/etc-pihole:/etc/pihole + cap_add: + - SYS_NICE +``` + +### keepalived (native on RPi4) + +```bash +sudo nano /etc/keepalived/keepalived.conf +``` + +``` +global_defs { + router_id pihole_rpi4 +} + +vrrp_instance VI_DNS { + state BACKUP + interface eth0 + virtual_router_id 53 + priority 80 + advert_int 1 + + authentication { + auth_type PASS + auth_pass SAME_PASSWORD_AS_PRIMARY + } + + virtual_ipaddress { + 10.5.1.53/24 + } +} +``` + +```bash +sudo systemctl enable keepalived +sudo systemctl start keepalived +``` + +--- + +## Nebula Sync (on PD) + +[Nebula Sync](https://github.com/lovelaze/nebula-sync) syncs Pi-hole v6 configuration (gravity, blocklists, allowlists, custom DNS) from the primary to both replicas. + +**Add to PD's pihole docker-compose.yaml:** + +```yaml + nebula-sync: + image: lovelaze/nebula-sync:latest + container_name: nebula-sync + restart: unless-stopped + environment: + PRIMARY: "http://127.0.0.1:8953|${PIHOLE_PASSWORD}" + REPLICAS: "http://10.5.1.5:8954|${PIHOLE_PASSWORD_SERENITY},http://10.5.1.X:8955|${PIHOLE_PASSWORD_RPI}" + RUN_GRAVITY: "true" + FULL_SYNC: "false" + CRON: "0 3 * * *" + networks: + - default +``` + +> - `PRIMARY` — your PD Pi-hole (note: uses `127.0.0.1` since nebula-sync is on the same host; port 8953 is the web UI, also the API port in Pi-hole v6) +> - `REPLICAS` — comma-separated list of secondary instances +> - `CRON` — syncs daily at 3 AM; force a manual sync with `docker exec nebula-sync nebula-sync run` +> - Each Pi-hole instance can (and should) have a **different** password — nebula-sync handles auth per-instance +> - Add `PIHOLE_PASSWORD_SERENITY` and `PIHOLE_PASSWORD_RPI` to your `.env` file + +### What Nebula Sync copies +- Gravity database (blocklists, allowlists, group assignments) +- Custom DNS/CNAME records +- Client groups +- Does **not** sync per-instance settings (web password, interface binding — these stay local) + +--- + +## Router Configuration (UniFi) + +Point **only the VIP** as DNS: + +1. UniFi Network → Settings → Networks → **LAN** → DHCP Name Server: **Manual** +2. **DNS Server 1:** `10.5.1.53` (VIP — floats between nodes) +3. **DNS Server 2:** *(leave blank or set a public fallback like `1.1.1.1` only if you want internet DNS as last resort)* +4. Save and renew DHCP leases + +> **Why single VIP instead of listing all three IPs?** +> Clients round-robin across all listed DNS servers — so some queries bypass Pi-hole entirely. With Keepalived the VIP always points to exactly one live Pi-hole node, ensuring 100% of queries are filtered. + +--- + +## Post-Deploy + +### Verify VIP is working + +```bash +# Check which node holds the VIP +ip addr show | grep 10.5.1.53 # Run on each node — only MASTER shows it + +# Test DNS through VIP +nslookup google.com 10.5.1.53 +nslookup doubleclick.net 10.5.1.53 # Should return 0.0.0.0 (blocked) +``` + +### Test failover + +```bash +# On PD, bring down keepalived +sudo docker stop keepalived +# Wait ~3 seconds, then check from another machine: +nslookup google.com 10.5.1.53 # Should still work — Serenity took over VIP +# Restore +sudo docker start keepalived +``` + +### Add blocklists (on primary — Nebula Sync propagates to replicas) + +| List | URL | +|------|-----| +| Steven Black (unified) | `https://raw.githubusercontent.com/StevenBlack/hosts/master/hosts` | +| OISD Basic | `https://basic.oisd.nl/` | +| Hagezi Pro | `https://raw.githubusercontent.com/hagezi/dns-blocklists/main/adblock/pro.txt` | + +After adding: **Tools → Update Gravity** on PD, then run Nebula Sync to push to replicas. + +### Local DNS records + +Add in Pi-hole (primary) → Local DNS → DNS Records: + +| Domain | IP | +|--------|----| +| pd.lan | 10.5.1.6 | +| serenity.lan | 10.5.1.5 | +| nomad.lan | 10.5.1.16 | +| rocinante.lan | 10.5.1.112 | +| rpi4.lan | 10.5.1.X | + +Nebula Sync will propagate these to all replicas. + +### Pangolin for admin UIs (optional) + +| Subdomain | Target | +|-----------|--------| +| `pihole.paccoco.com` | `http://10.5.1.6:8953` (primary) | +| `pihole2.paccoco.com` | `http://10.5.1.5:8954` (Serenity) | + +Enable Pangolin auth — Pi-hole admin should not be publicly accessible. + +--- + +## Storage Decisions + +| Node | Path | Tier | Reason | +|------|------|------|--------| +| PD | `/mnt/tank/docker/appdata/pihole-primary` | tank | Low write frequency | +| PD | `/mnt/tank/docker/appdata/unbound-primary` | tank | Config + root hints only | +| Serenity | `/mnt/user/appdata/pihole-secondary` | Unraid array | Low write frequency | +| RPi4 | `~/pihole/` | SD card / SSD | Dedicated Pi, no other concerns | + +--- + +## Known Gotchas + +- **Port 53 conflict on PD:** TrueNAS binds to `127.0.0.1:53`. With `network_mode: host`, Pi-hole will try to bind to all interfaces. Set `FTLCONF_dns_listeningMode: LOCAL` so it only listens on the actual LAN interface, not loopback. +- **keepalived interface name:** Must match the actual NIC name on each host. Get it with `ip link show`. +- **`virtual_router_id` must be unique** per VRRP group on your LAN. `53` is a good mnemonic. Confirm no other keepalived instance on your LAN uses the same ID. +- **Pi-hole v6 API port:** In v6, the web UI and API both run on the `FTLCONF_webserver_port`. Nebula Sync and the web browser use the same port. +- **nopreempt on MASTER:** The `nopreempt` option in PD's keepalived.conf prevents PD from immediately reclaiming the VIP when it comes back up after a failure. Remove it if you want PD to always retake MASTER on recovery. +- **Unbound cold start:** Unbound takes a few seconds to start and validate DNSSEC. The `depends_on: condition: service_healthy` in the Pi-hole compose ensures Pi-hole waits for Unbound before starting. diff --git a/docs/planning/N8N_WORKFLOW_IDEAS.md b/docs/planning/N8N_WORKFLOW_IDEAS.md new file mode 100644 index 0000000..076cf05 --- /dev/null +++ b/docs/planning/N8N_WORKFLOW_IDEAS.md @@ -0,0 +1,186 @@ +# n8n Workflow Ideas & Upgrade Proposals + +Generated: 2026-05-09 + +--- + +## Upgrades to Existing Workflows + +### 01 — Grafana Alert → AI → Gotify +**Current state:** Webhook → parse → LiteLLM summarize → Gotify +**Suggested upgrades:** +- **Alert deduplication:** Store last-seen alert fingerprint in Postgres. If the same alert fires again within 30 min, suppress the Gotify push (you're already awake, you've seen it). +- **Recovery notifications:** Grafana sends both `firing` and `resolved` states. Add a branch so resolved alerts get a `✅ RESOLVED` message with a distinct priority (1 = low). +- **Auto-acknowledge low-priority alerts:** If priority < 3 and it's between midnight and 7am, skip Gotify and log to Postgres only. + +--- + +### 04 — Whisper Audio Transcription +**Current state:** Returns transcript inline, no Gotify notification +**Suggested upgrades:** +- **Add completion notification:** When a long transcription finishes, push to Gotify (Whisper Transcriptions app) so you know it's done without polling. +- **Save transcript to Paperless:** After transcription, POST the text to Paperless-NGX as a new document with auto-tagging (`transcription`, `audio`, date). +- **Save to Qdrant:** After transcription + summary, auto-ingest into RAG so all your audio content is searchable. + +--- + +### 05 — Paperless → RAG Ingest +**Current state:** Webhook → fetch → chunk → embed → Qdrant → Gotify +**Suggested upgrades:** +- **Retry on failure:** If Qdrant upsert fails, retry up to 3 times with exponential backoff before erroring out. +- **Collection routing:** Add metadata tagging so documents from different Paperless tags go into different Qdrant collections (e.g., `knowledge_base` vs `receipts` vs `medical`). + +--- + +### 08 — Class Recording → Transcribe → RAG +**Current state:** Webhook → Whisper → AI notes → RAG → Gotify +**Suggested upgrades:** +- **Export notes to Paperless:** After extracting key concepts, create a Paperless document for the class notes (title = class + lecture, tags = class name, `lecture-notes`). +- **Weekly class digest:** Scheduled workflow that queries RAG for all chunks from the past week, grouped by class, and sends a consolidated study summary to Gotify every Sunday night. +- **Quiz generation:** Add an optional webhook endpoint that takes a class name + RAG query and generates 5 practice questions from the lecture material via LiteLLM. + +--- + +## New Workflow Proposals + +### 09 — Immich → AI Event Tagger +**Trigger:** Immich post-upload webhook (or scheduled poll via Immich API) +**What it does:** +1. Fetch newly uploaded photos from Immich API +2. Send image to a vision-capable model (gemma3:27b on Rocinante via LiteLLM) +3. AI generates descriptive tags, detects events (birthday, trip, holiday), extracts visible text +4. Write tags back to Immich via API +5. Notify via Gotify (Immich app) + +**Why:** Immich's built-in ML does faces, not semantic tagging. This adds searchable event context automatically. + +--- + +### 10 — Home Assistant → Morning Briefing +**Trigger:** Schedule (e.g. 7:00 AM daily) +**What it does:** +1. Fetch current weather from Home Assistant or weather API +2. Query Grafana/Prometheus for overnight anomalies (high CPU, disk warnings) +3. Check Pi-hole stats (queries blocked, top blocked domain) +4. Check Paperless for any unread/untagged documents +5. Summarize via LiteLLM into a 5-bullet morning briefing +6. Push to Gotify (Morning Briefing app) and optionally to Home Assistant notification + +**Why:** Single daily digest so you know what happened overnight without checking 4 different dashboards. + +--- + +### 11 — Uptime Kuma → Smart Alert Escalation +**Trigger:** Uptime Kuma webhook on status change +**What it does:** +1. Receive down/recovery event from Uptime Kuma +2. Check Prometheus to confirm actual service health (avoids false positives) +3. If confirmed down: check how long it's been down + - < 2 min: log only (transient blip) + - 2–10 min: Gotify push (medium priority) + - > 10 min: Gotify push (high priority) + attempt auto-restart via `docker restart` API call to PD +4. On recovery: send resolution notification with total downtime + +**Why:** Uptime Kuma already sends webhooks but has no intelligence. This adds confirmation, suppression of transient blips, and auto-remediation for stuck containers. + +--- + +### 12 — Pi-hole Stats Digest +**Trigger:** Schedule (daily at 8 AM) +**What it does:** +1. Query Pi-hole API on PD (`http://10.5.1.6:8953/api/stats/summary`) +2. Query secondary Pi-hole on Serenity +3. LiteLLM summarizes notable blocked domains and compares primary vs secondary query load +4. Push digest to Gotify (Pi-hole app) with 24h block count, top blocked domains, and any unusual spikes + +**Why:** Passive DNS visibility — know if something on your network is phoning home unexpectedly. + +--- + +### 13 — Lidarr → Kima-Hub New Music Notifier +**Trigger:** Lidarr webhook on album download complete +**What it does:** +1. Receive download event from Lidarr +2. Fetch album art and metadata from MusicBrainz or Lidarr API +3. Format a rich notification: artist, album, year, genre +4. Push to Gotify (Music app) with album art link +5. Optionally trigger Kima-Hub library rescan via API + +**Why:** Know when new music lands without watching Lidarr activity logs. + +--- + +### 14 — Paperless Inbox Reminder +**Trigger:** Schedule (every Monday 9 AM) +**What it does:** +1. Query Paperless API for documents with no tags or in the inbox +2. If count > 0: push reminder to Gotify listing the oldest unprocessed documents +3. If inbox is clear: send a ✅ confirmation + +**Why:** It's easy to forget to process scanned documents. This ensures nothing sits in the inbox untagged for weeks. + +--- + +### 15 — n8n Self-Health Monitor +**Trigger:** Schedule (every 15 min) +**What it does:** +1. Check n8n's own execution history via API — look for failed executions in the past hour +2. If failures found: fetch the error message from n8n API +3. Push a "Workflow Failed" notification to Gotify (n8n Health app) with the workflow name and error summary +4. Skip if the failure is the same as the last-notified one (dedup via Postgres) + +**Why:** n8n doesn't natively alert you when a workflow silently fails. This closes that blind spot. + +--- + +### 16 — NFS Mount Watchdog + Auto-Heal +**Trigger:** Schedule (every 5 min) +**What it does:** +1. SSH into PD (10.5.1.6) and probe each expected NFS mount path with `mountpoint -q`: + - `/mnt/unraid/data/media` (Plex, Audiobookshelf) + - `/mnt/unraid/data/photos` (Immich) + - Any other mounts defined in the mount script +2. If all mounts are healthy: silent exit +3. If any mount is missing: + a. Run the NFS mount script: `sudo bash /mnt/tank/docker/scripts/mount-unraid-nfs.sh` + b. Wait 10 seconds, re-probe to confirm mounts came up + c. If mounts confirmed: restart only the affected containers: + - Media mount missing → `sudo docker restart plex audiobookshelf` + - Photos mount missing → `sudo docker restart immich-server immich-microservices immich-machine-learning` + d. If mounts still missing after script: escalate to Gotify (high priority — NFS down, manual intervention needed) +4. Push result to Gotify (Infra app): + - Auto-healed: priority 5, list which mounts were remounted and which containers restarted + - Escalation: priority 9, list which paths are still unavailable + +**Implementation notes:** +- Uses n8n SSH node connecting to `10.5.1.6` with an SSH credential (key-based auth from n8n container) +- SSH node runs a single compound command: `mountpoint -q /mnt/unraid/data/media && echo MEDIA_OK || echo MEDIA_MISSING` etc. +- Parse output in a Code node to determine which mounts failed +- Use separate SSH nodes for: (1) running the mount script, (2) restarting containers — keeps the flow readable +- `continueOnFail: true` on the mount script SSH node so a TrueNAS-unreachable scenario still escalates instead of dying silently +- Add a Postgres dedup table (`nfs_heal_log`) so repeated failures within 30 min only send one Gotify push, but every successful auto-heal always notifies + +**Why:** TrueNAS reboots (updates, power blips) drop NFS mounts and crash-loop dependent containers. This closes the gap between "Serenity rebooted at 3 AM" and "you notice Immich is broken at noon." + +--- + +## Suggested Gotify App / Channel Structure + +Based on all workflows above, here's the full recommended channel layout: + +| App Name | Workflows | +|----------|-----------| +| Grafana Alerts | 01 | +| Paperless | 03 | +| Paperless RAG | 05 | +| RSS Feed | 06 | +| Git Commits | 07 | +| Class Recordings | 08 | +| Immich | 09 (new) | +| Morning Briefing | 10 (new) | +| Uptime / Infra | 11, 16 (new) | +| Pi-hole | 12 (new) | +| Music | 13 (new) | +| Reminders | 14 (new) | +| n8n Health | 15 (new) | +| Whisper | 04 (after upgrade) | diff --git a/docs/planning/TODO.md b/docs/planning/TODO.md index 8dbabcc..5c5c198 100644 --- a/docs/planning/TODO.md +++ b/docs/planning/TODO.md @@ -5,7 +5,9 @@ - [ ] Regenerate N.O.M.A.D. Newt secret (was exposed in chat) ## Infrastructure -- [ ] Deploy Pi-hole ×2 (primary on PD + backup on Serenity or Pi4) +- [ ] Deploy Pi-hole 3-node HA cluster (PD + Serenity + RPi4) — see docs/planning/DEPLOY_PIHOLE.md + - RPi4 IP still TBD — fill in before deploying +- [ ] Deploy Kima-Hub — see docs/planning/DEPLOY_KIMA_HUB.md - [ ] Set up off-site backup for vital data (appdata, databases, config repo, photos) - [ ] Remove dead Unraid-Cloudflared-Tunnel container from Serenity - [ ] Serenity malcolm pool capacity planning (heavily utilized) @@ -32,6 +34,14 @@ - [ ] Document meshtastic stack fully (see docs/reference/MESHTASTIC.md) - [ ] Document N.O.M.A.D. containers (start_nomad.sh services) +## n8n Workflows +- [ ] Create Gotify apps and n8n credentials: Reminders, n8n Health, Uptime / Infra +- [ ] Add env vars to n8n: `PAPERLESS_API_TOKEN`, `PAPERLESS_TAG_TRANSCRIPTION`, `PAPERLESS_TAG_LECTURE_NOTES`, `N8N_API_KEY` +- [ ] Set up `PD SSH` credential in n8n for workflow 16 (see n8n-workflows/README.md) +- [ ] Add sudoers NOPASSWD rule for n8n watchdog (see n8n-workflows/README.md) +- [ ] Import and activate upgraded workflows: 01, 04, 08 +- [ ] Import and activate new workflows: 14, 15 (16 already active) + ## Completed - [x] Deploy Gotify (Phase 1) — 2026-05-05 - [x] Deploy AI stack — LiteLLM, OpenWebUI, Qdrant, Whisper, SearXNG (Phase 2) — 2026-05-05 @@ -43,3 +53,7 @@ - [x] Fill in port/URL columns in SERVICES_DIRECTORY.md — 2026-05-09 - [x] Create ROCINANTE.md server doc — 2026-05-09 - [x] Update all docs to reflect deployed state — 2026-05-09 +- [x] Write n8n workflow upgrade + new workflow plans (01, 04, 08, 14, 15, 16) — 2026-05-09 +- [x] Write Pi-hole 3-node HA deployment plan — 2026-05-09 +- [x] Write Kima-Hub deployment plan — 2026-05-09 +- [x] Create Postgres dedup tables (grafana_alert_dedup, n8n_health_dedup) on PD — 2026-05-09 diff --git a/docs/troubleshooting/KNOWN_QUIRKS.md b/docs/troubleshooting/KNOWN_QUIRKS.md index c305788..24ac51b 100644 --- a/docs/troubleshooting/KNOWN_QUIRKS.md +++ b/docs/troubleshooting/KNOWN_QUIRKS.md @@ -4,6 +4,15 @@ Per-service gotchas that aren't bugs but will bite you if you forget them. ## PlausibleDeniability +### n8n workflow 16 (NFS Watchdog) +- Runs `sudo docker restart` and `sudo bash .../mount-unraid-nfs.sh` over SSH from n8n container +- Requires a NOPASSWD sudoers rule or SSH commands will hang waiting for a password: + ``` + truenas_admin ALL=(ALL) NOPASSWD: /usr/bin/docker, /bin/bash /mnt/tank/docker/scripts/mount-unraid-nfs.sh + ``` + Add via `sudo visudo -f /etc/sudoers.d/n8n-watchdog` +- SSH credential in n8n must use key-based auth (see n8n-workflows/README.md for keygen steps) + ### immich-ml - Healthcheck has no curl/wget — uses `/proc/net/tcp6` pattern - Cache goes to `/mnt/docker-ssd/docker/appdata/immich-ml` (separate from immich-server's appdata) diff --git a/meshtastic/scripts/write-test b/meshtastic/scripts/write-test new file mode 100644 index 0000000..e69de29 diff --git a/n8n-workflows/01-grafana-alert-ai-gotify.json b/n8n-workflows/01-grafana-alert-ai-gotify.json new file mode 100644 index 0000000..1704138 --- /dev/null +++ b/n8n-workflows/01-grafana-alert-ai-gotify.json @@ -0,0 +1,363 @@ +{ + "updatedAt": "2026-05-07T03:32:35.930Z", + "createdAt": "2026-05-07T03:32:32.026Z", + "id": "GXtkp4xKka90XrXH", + "name": "Grafana Alert \u2192 AI Summary \u2192 Gotify", + "description": null, + "active": true, + "isArchived": false, + "nodes": [ + { + "parameters": { + "httpMethod": "POST", + "path": "grafana-alert", + "options": {} + }, + "id": "a6e7dfb1-8e9b-4048-b028-4383c30a7e0d", + "name": "Grafana Webhook", + "type": "n8n-nodes-base.webhook", + "typeVersion": 2, + "position": [ + 0, + 0 + ], + "webhookId": "0492788e-db4e-4d15-9572-9d6c3efc80c3" + }, + { + "parameters": { + "jsCode": "// Parse Grafana alert payload into a clean summary for the LLM\nconst alerts = $input.first().json.body?.alerts || $input.first().json.alerts || [];\n\nconst parsed = alerts.map(a => ({\n status: a.status || 'unknown',\n alertName: a.labels?.alertname || 'Unnamed Alert',\n host: a.labels?.host || a.labels?.instance || 'unknown host',\n severity: a.labels?.severity || 'warning',\n summary: a.annotations?.summary || '',\n description: a.annotations?.description || '',\n value: a.valueString || '',\n startsAt: a.startsAt || '',\n endsAt: a.endsAt || ''\n}));\n\nconst alertText = parsed.map(a => \n `[${a.status.toUpperCase()}] ${a.alertName} on ${a.host}\\nSeverity: ${a.severity}\\nSummary: ${a.summary}\\nDescription: ${a.description}\\nValue: ${a.value}\\nStarted: ${a.startsAt}`\n).join('\\n---\\n');\n\nreturn [{\n json: {\n alertText,\n alertCount: parsed.length,\n hasResolved: parsed.some(a => a.status === 'resolved'),\n hasFiring: parsed.some(a => a.status === 'firing'),\n parsed\n }\n}];" + }, + "id": "ea216058-69db-42cc-88d0-8f67e1b8951b", + "name": "Parse Alert Payload", + "type": "n8n-nodes-base.code", + "typeVersion": 2, + "position": [ + 224, + 0 + ] + }, + { + "parameters": { + "method": "POST", + "url": "http://10.5.1.6:4000/v1/chat/completions", + "sendHeaders": true, + "headerParameters": { + "parameters": [ + { + "name": "Content-Type", + "value": "application/json" + }, + { + "name": "Authorization", + "value": "Bearer sk-REPLACE_WITH_LITELLM_KEY" + } + ] + }, + "sendBody": true, + "specifyBody": "json", + "jsonBody": "={{\n {\n \"model\": \"medium\",\n \"messages\": [\n {\n \"role\": \"system\",\n \"content\": \"You are a concise homelab monitoring assistant. Summarize infrastructure alerts in 2-3 sentences. Include: what happened, which host, severity, and a suggested action. Keep it brief \u2014 this goes to a phone notification.\"\n },\n {\n \"role\": \"user\",\n \"content\": \"Summarize this Grafana alert:\\n\\n\" + $json.alertText\n }\n ],\n \"max_tokens\": 200,\n \"temperature\": 0.3\n }\n}}", + "options": {} + }, + "id": "8ef1aa9d-d5b1-41ea-abc6-703b6491ea33", + "name": "LiteLLM Summarize", + "type": "n8n-nodes-base.httpRequest", + "typeVersion": 4.2, + "position": [ + 1792, + 0 + ] + }, + { + "parameters": { + "jsCode": "const response = $input.first().json;\nconst summary = response.choices?.[0]?.message?.content || 'Could not generate summary';\nconst alertData = $('Parse Alert Payload').first().json;\n\nconst title = alertData.hasFiring \n ? `\ud83d\udd25 Alert Firing (${alertData.alertCount})` \n : `\u2705 Alert Resolved (${alertData.alertCount})`;\n\nconst priority = alertData.hasFiring ? 8 : 2;\n\nreturn [{\n json: {\n title,\n message: summary,\n priority\n }\n}];" + }, + "id": "2e969bca-8458-4477-a358-664fd1c50c4f", + "name": "Format Notification", + "type": "n8n-nodes-base.code", + "typeVersion": 2, + "position": [ + 2016, + 0 + ] + }, + { + "parameters": { + "message": "={{ $json.message }}", + "additionalFields": { + "priority": "={{ $json.priority }}", + "title": "={{ $json.title }}" + }, + "options": { + "contentType": "text/plain" + } + }, + "id": "65c2b53a-6e03-4460-9aa6-5d967d244de0", + "name": "Push to Gotify", + "type": "n8n-nodes-base.gotify", + "typeVersion": 1, + "position": [ + 2224, + 0 + ], + "credentials": { + "gotifyApi": { + "id": "DXdYZDfVZecDTaNU", + "name": "Gotify account" + } + } + }, + { + "parameters": { + "jsCode": "const parsed = $('Parse Alert Payload').first().json;\nconst now = new Date();\nconst hour = now.getHours();\nconst fingerprint = (parsed.parsed || [])\n .map(a => `${a.alertName}:${a.host}`)\n .sort().join('|') || 'unknown';\nreturn [{ json: { ...parsed, fingerprint, isNightTime: hour >= 0 && hour < 7 } }];\n" + }, + "id": "df26d9ca-c72c-4dee-9327-14a0f5fcf3a0", + "name": "Build Fingerprint", + "type": "n8n-nodes-base.code", + "typeVersion": 2, + "position": [ + 448, + 0 + ] + }, + { + "parameters": { + "operation": "executeQuery", + "query": "SELECT fingerprint, last_seen FROM grafana_alert_dedup WHERE fingerprint = '{{ $json.fingerprint }}' AND last_seen > NOW() - INTERVAL '30 minutes' LIMIT 1;", + "options": {} + }, + "id": "3fd78a53-7885-4739-a4b7-b441a27e993e", + "name": "Lookup Dedup", + "type": "n8n-nodes-base.postgres", + "typeVersion": 2.5, + "position": [ + 672, + 0 + ], + "credentials": { + "postgres": { + "id": "n9svoXemqSZoNNUB", + "name": "Postgres account" + } + }, + "continueOnFail": true + }, + { + "parameters": { + "jsCode": "const alertData = $('Build Fingerprint').first().json;\nconst rows = $input.all().filter(r => r.json && r.json.fingerprint);\nconst alreadySeen = rows.length > 0;\nconst isLowPriority = !alertData.hasFiring;\nconst suppressNight = alertData.isNightTime && isLowPriority;\nconst suppress = alreadySeen || suppressNight;\nreturn [{ json: { ...alertData, suppress, suppressReason: alreadySeen ? 'dedup' : suppressNight ? 'night_suppress' : null } }];\n" + }, + "id": "c34b4b0e-1a47-4624-b1c4-9ef6b5cdeb6c", + "name": "Check Should Notify", + "type": "n8n-nodes-base.code", + "typeVersion": 2, + "position": [ + 896, + 0 + ] + }, + { + "parameters": { + "conditions": { + "options": { + "caseSensitive": true, + "leftValue": "", + "typeValidation": "strict" + }, + "conditions": [ + { + "id": "edee2fbd-0b2b-43bf-b542-9d9297d39d66", + "leftValue": "={{ $json.suppress }}", + "rightValue": false, + "operator": { + "type": "boolean", + "operation": "equal" + } + } + ], + "combinator": "and" + } + }, + "id": "29929dc8-17f6-4cc8-90c2-c501eaba8be5", + "name": "Should Notify?", + "type": "n8n-nodes-base.if", + "typeVersion": 2.2, + "position": [ + 1120, + 0 + ] + }, + { + "parameters": { + "operation": "executeQuery", + "query": "INSERT INTO grafana_alert_dedup (fingerprint, last_seen, alert_name, host) VALUES ('{{ $json.fingerprint }}', NOW(), '{{ ($json.parsed && $json.parsed[0]) ? $json.parsed[0].alertName : \"unknown\" }}', '{{ ($json.parsed && $json.parsed[0]) ? $json.parsed[0].host : \"unknown\" }}') ON CONFLICT (fingerprint) DO UPDATE SET last_seen = NOW();", + "options": {} + }, + "id": "25a14a6a-db17-49ca-a6d2-6ed2f97d2889", + "name": "Upsert Dedup Record", + "type": "n8n-nodes-base.postgres", + "typeVersion": 2.5, + "position": [ + 1344, + 0 + ], + "credentials": { + "postgres": { + "id": "n9svoXemqSZoNNUB", + "name": "Postgres account" + } + }, + "continueOnFail": true + } + ], + "connections": { + "Grafana Webhook": { + "main": [ + [ + { + "node": "Parse Alert Payload", + "type": "main", + "index": 0 + } + ] + ] + }, + "Parse Alert Payload": { + "main": [ + [ + { + "node": "Build Fingerprint", + "type": "main", + "index": 0 + } + ] + ] + }, + "Build Fingerprint": { + "main": [ + [ + { + "node": "Lookup Dedup", + "type": "main", + "index": 0 + } + ] + ] + }, + "Lookup Dedup": { + "main": [ + [ + { + "node": "Check Should Notify", + "type": "main", + "index": 0 + } + ] + ] + }, + "Check Should Notify": { + "main": [ + [ + { + "node": "Should Notify?", + "type": "main", + "index": 0 + } + ] + ] + }, + "Should Notify?": { + "main": [ + [ + { + "node": "Upsert Dedup Record", + "type": "main", + "index": 0 + } + ], + [] + ] + }, + "Upsert Dedup Record": { + "main": [ + [ + { + "node": "LiteLLM Summarize", + "type": "main", + "index": 0 + } + ] + ] + }, + "LiteLLM Summarize": { + "main": [ + [ + { + "node": "Format Notification", + "type": "main", + "index": 0 + } + ] + ] + }, + "Format Notification": { + "main": [ + [ + { + "node": "Push to Gotify", + "type": "main", + "index": 0 + } + ] + ] + } + }, + "settings": { + "executionOrder": "v1", + "binaryMode": "separate" + }, + "staticData": null, + "meta": { + "templateCredsSetupCompleted": true + }, + "pinData": {}, + "versionId": "ba3a4eab-60b8-4b1c-ac9f-07a0fdcec9c0", + "activeVersionId": "ba3a4eab-60b8-4b1c-ac9f-07a0fdcec9c0", + "versionCounter": 10, + "triggerCount": 1, + "tags": [ + { + "updatedAt": "2026-05-06T22:38:51.678Z", + "createdAt": "2026-05-06T22:38:51.678Z", + "id": "S9FxzVfHLsHmyzyt", + "name": "ai" + }, + { + "updatedAt": "2026-05-06T22:38:51.660Z", + "createdAt": "2026-05-06T22:38:51.660Z", + "id": "0ETpkL5jJ5wwgt8k", + "name": "monitoring" + } + ], + "shared": [ + { + "updatedAt": "2026-05-07T03:32:32.026Z", + "createdAt": "2026-05-07T03:32:32.026Z", + "role": "workflow:owner", + "workflowId": "GXtkp4xKka90XrXH", + "projectId": "dxCRnBdX5uJizCGa", + "project": { + "updatedAt": "2026-05-06T01:10:37.484Z", + "createdAt": "2026-05-06T01:08:07.721Z", + "id": "dxCRnBdX5uJizCGa", + "name": "Wilfred Fizzlepoof ", + "type": "personal", + "icon": null, + "description": null, + "creatorId": "47b2227f-2fc2-4a08-bd35-ea157b92df0d" + } + } + ], + "versionMetadata": { + "name": "Version ba3a4eab", + "description": "" + } +} \ No newline at end of file diff --git a/n8n-workflows/02-rag-ingest-and-query.json b/n8n-workflows/02-rag-ingest-and-query.json new file mode 100644 index 0000000..fabb8f9 --- /dev/null +++ b/n8n-workflows/02-rag-ingest-and-query.json @@ -0,0 +1,394 @@ +{ + "updatedAt": "2026-05-06T23:25:33.293Z", + "createdAt": "2026-05-06T23:25:29.005Z", + "id": "gYyOggC4J98keLbj", + "name": "RAG Pipeline — Ingest & Query", + "description": null, + "active": true, + "isArchived": false, + "nodes": [ + { + "parameters": { + "httpMethod": "POST", + "path": "rag/ingest", + "responseMode": "lastNode", + "responseData": "allEntries", + "options": {} + }, + "id": "31ce8413-d15c-490f-92d1-956ef214e9a3", + "name": "Ingest Webhook", + "type": "n8n-nodes-base.webhook", + "typeVersion": 2, + "position": [ + 0, + 0 + ], + "webhookId": "411ad4e3-eb9f-49e2-97c4-2f30e01dd609" + }, + { + "parameters": { + "jsCode": "// Split incoming text into chunks for embedding\nconst input = $input.first().json.body || $input.first().json;\nconst text = input.text || input.content || '';\nconst source = input.source || 'manual';\nconst metadata = input.metadata || {};\n\nconst CHUNK_SIZE = 500;\nconst OVERLAP = 50;\n\nconst chunks = [];\nconst sentences = text.split(/(?<=[.!?])\\s+/);\nlet current = '';\n\nfor (const sentence of sentences) {\n if ((current + ' ' + sentence).length > CHUNK_SIZE && current.length > 0) {\n chunks.push(current.trim());\n // Keep overlap\n const words = current.split(' ');\n current = words.slice(-OVERLAP).join(' ') + ' ' + sentence;\n } else {\n current = current ? current + ' ' + sentence : sentence;\n }\n}\nif (current.trim()) chunks.push(current.trim());\n\nreturn chunks.map((chunk, i) => ({\n json: {\n chunk,\n chunkIndex: i,\n totalChunks: chunks.length,\n source,\n metadata\n }\n}));" + }, + "id": "5d5aa0b1-c84b-48f6-9b5e-618cb05e5e61", + "name": "Chunk Text", + "type": "n8n-nodes-base.code", + "typeVersion": 2, + "position": [ + 224, + 0 + ] + }, + { + "parameters": { + "method": "POST", + "url": "http://10.5.1.6:11434/v1/embeddings", + "sendHeaders": true, + "headerParameters": { + "parameters": [ + { + "name": "Content-Type", + "value": "application/json" + } + ] + }, + "sendBody": true, + "specifyBody": "json", + "jsonBody": "={\n \"model\": \"nomic-embed-text\",\n \"input\": \"{{ $json.chunk }}\"\n}", + "options": { + "batching": { + "batch": { + "batchSize": 5, + "batchInterval": 500 + } + } + } + }, + "id": "89c692c6-b987-409c-b5c1-3533097d06e3", + "name": "Get Embedding", + "type": "n8n-nodes-base.httpRequest", + "typeVersion": 4.2, + "position": [ + 448, + 0 + ] + }, + { + "parameters": { + "jsCode": "const embeddingResponse = $input.first().json;\nconst chunkData = $('Chunk Text').item;\nconst embedding = embeddingResponse.data?.[0]?.embedding || [];\n\nconst pointId = 'xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx'.replace(/[xy]/g, c => { const r = Math.random() * 16 | 0; return (c === 'x' ? r : (r & 0x3 | 0x8)).toString(16); });\n\nreturn [{\n json: {\n id: pointId,\n vector: embedding,\n payload: {\n text: chunkData.json.chunk,\n source: chunkData.json.source,\n chunkIndex: chunkData.json.chunkIndex,\n totalChunks: chunkData.json.totalChunks,\n metadata: chunkData.json.metadata,\n ingested_at: new Date().toISOString()\n }\n }\n}];" + }, + "id": "fec59066-958f-4b96-8224-6f4e48ad5abb", + "name": "Prepare Qdrant Point", + "type": "n8n-nodes-base.code", + "typeVersion": 2, + "position": [ + 672, + 0 + ] + }, + { + "parameters": { + "method": "PUT", + "url": "http://10.5.1.6:6333/collections/knowledge_base/points", + "sendHeaders": true, + "headerParameters": { + "parameters": [ + { + "name": "Content-Type", + "value": "application/json" + } + ] + }, + "sendBody": true, + "specifyBody": "json", + "jsonBody": "={\n \"points\": [\n {\n \"id\": \"{{ $json.id }}\",\n \"vector\": {{ JSON.stringify($json.vector) }},\n \"payload\": {{ JSON.stringify($json.payload) }}\n }\n ]\n}", + "options": {} + }, + "id": "9404c8fe-46ae-4d8a-bf35-35ecba4939c0", + "name": "Upsert to Qdrant", + "type": "n8n-nodes-base.httpRequest", + "typeVersion": 4.2, + "position": [ + 880, + 0 + ] + }, + { + "parameters": { + "httpMethod": "POST", + "path": "rag/query", + "responseMode": "lastNode", + "responseData": "allEntries", + "options": {} + }, + "id": "1a5c921c-0351-4a5e-a3e4-0aab71313af1", + "name": "Query Webhook", + "type": "n8n-nodes-base.webhook", + "typeVersion": 2, + "position": [ + 0, + 304 + ], + "webhookId": "aad4fe04-74b0-4136-80e9-380a7e4db1c2" + }, + { + "parameters": { + "method": "POST", + "url": "http://10.5.1.6:11434/v1/embeddings", + "sendHeaders": true, + "headerParameters": { + "parameters": [ + { + "name": "Content-Type", + "value": "application/json" + } + ] + }, + "sendBody": true, + "specifyBody": "json", + "jsonBody": "={\n \"model\": \"nomic-embed-text\",\n \"input\": \"{{ $json.body.query || $json.query }}\"\n}", + "options": {} + }, + "id": "0a294ba5-a890-4dfa-8ae7-ba25cd0a4534", + "name": "Embed Query", + "type": "n8n-nodes-base.httpRequest", + "typeVersion": 4.2, + "position": [ + 224, + 304 + ] + }, + { + "parameters": { + "method": "POST", + "url": "http://10.5.1.6:6333/collections/knowledge_base/points/search", + "sendHeaders": true, + "headerParameters": { + "parameters": [ + { + "name": "Content-Type", + "value": "application/json" + } + ] + }, + "sendBody": true, + "specifyBody": "json", + "jsonBody": "={\n \"vector\": {{ JSON.stringify($json.data[0].embedding) }},\n \"limit\": 5,\n \"with_payload\": true\n}", + "options": {} + }, + "id": "2cb68891-c57e-4006-b3ac-c290aec28220", + "name": "Search Qdrant", + "type": "n8n-nodes-base.httpRequest", + "typeVersion": 4.2, + "position": [ + 448, + 304 + ] + }, + { + "parameters": { + "jsCode": "const searchResults = $input.first().json.result || [];\nconst query = $('Query Webhook').first().json.body?.query || $('Query Webhook').first().json.query || '';\n\nconst context = searchResults.map((r, i) => \n `[${i+1}] (score: ${r.score.toFixed(3)}) ${r.payload.text}`\n).join('\\n\\n');\n\nreturn [{\n json: {\n query,\n context,\n sources: searchResults.map(r => ({\n source: r.payload.source,\n score: r.score,\n text_preview: r.payload.text.substring(0, 100) + '...'\n }))\n }\n}];" + }, + "id": "d32ee814-81d0-4ea6-8f28-59632d09b1b9", + "name": "Build Context", + "type": "n8n-nodes-base.code", + "typeVersion": 2, + "position": [ + 672, + 304 + ] + }, + { + "parameters": { + "method": "POST", + "url": "http://10.5.1.6:4000/v1/chat/completions", + "sendHeaders": true, + "headerParameters": { + "parameters": [ + { + "name": "Content-Type", + "value": "application/json" + }, + { + "name": "Authorization", + "value": "Bearer sk-REPLACE_WITH_LITELLM_KEY" + } + ] + }, + "sendBody": true, + "specifyBody": "json", + "jsonBody": "={\n \"model\": \"medium\",\n \"messages\": [\n {\n \"role\": \"system\",\n \"content\": \"You are a helpful knowledge base assistant. Answer the user's question using ONLY the provided context. If the context doesn't contain enough information, say so. Cite which source numbers you used.\"\n },\n {\n \"role\": \"user\",\n \"content\": \"Context:\\n{{ $json.context }}\\n\\nQuestion: {{ $json.query }}\"\n }\n ],\n \"max_tokens\": 500,\n \"temperature\": 0.2\n}", + "options": {} + }, + "id": "e1360454-8457-40ce-b179-670c182da59e", + "name": "LiteLLM Answer", + "type": "n8n-nodes-base.httpRequest", + "typeVersion": 4.2, + "position": [ + 880, + 304 + ] + }, + { + "parameters": { + "jsCode": "const response = $input.first().json;\nconst contextData = $('Build Context').first().json;\n\nreturn [{\n json: {\n answer: response.choices?.[0]?.message?.content || 'No answer generated',\n sources: contextData.sources,\n query: contextData.query\n }\n}];" + }, + "id": "fe16d3a1-0d11-442f-b941-56fb74804601", + "name": "Format Response", + "type": "n8n-nodes-base.code", + "typeVersion": 2, + "position": [ + 1104, + 304 + ] + } + ], + "connections": { + "Ingest Webhook": { + "main": [ + [ + { + "node": "Chunk Text", + "type": "main", + "index": 0 + } + ] + ] + }, + "Chunk Text": { + "main": [ + [ + { + "node": "Get Embedding", + "type": "main", + "index": 0 + } + ] + ] + }, + "Get Embedding": { + "main": [ + [ + { + "node": "Prepare Qdrant Point", + "type": "main", + "index": 0 + } + ] + ] + }, + "Prepare Qdrant Point": { + "main": [ + [ + { + "node": "Upsert to Qdrant", + "type": "main", + "index": 0 + } + ] + ] + }, + "Query Webhook": { + "main": [ + [ + { + "node": "Embed Query", + "type": "main", + "index": 0 + } + ] + ] + }, + "Embed Query": { + "main": [ + [ + { + "node": "Search Qdrant", + "type": "main", + "index": 0 + } + ] + ] + }, + "Search Qdrant": { + "main": [ + [ + { + "node": "Build Context", + "type": "main", + "index": 0 + } + ] + ] + }, + "Build Context": { + "main": [ + [ + { + "node": "LiteLLM Answer", + "type": "main", + "index": 0 + } + ] + ] + }, + "LiteLLM Answer": { + "main": [ + [ + { + "node": "Format Response", + "type": "main", + "index": 0 + } + ] + ] + } + }, + "settings": { + "executionOrder": "v1", + "binaryMode": "separate" + }, + "staticData": null, + "meta": null, + "pinData": {}, + "versionId": "832248e5-106e-4fc4-8ebd-a5d0034e2b14", + "activeVersionId": "832248e5-106e-4fc4-8ebd-a5d0034e2b14", + "versionCounter": 14, + "triggerCount": 2, + "tags": [ + { + "updatedAt": "2026-05-06T22:38:51.678Z", + "createdAt": "2026-05-06T22:38:51.678Z", + "id": "S9FxzVfHLsHmyzyt", + "name": "ai" + }, + { + "updatedAt": "2026-05-06T22:39:33.284Z", + "createdAt": "2026-05-06T22:39:33.284Z", + "id": "7uuEyQrIlcuruZBk", + "name": "rag" + } + ], + "shared": [ + { + "updatedAt": "2026-05-06T23:25:29.005Z", + "createdAt": "2026-05-06T23:25:29.005Z", + "role": "workflow:owner", + "workflowId": "gYyOggC4J98keLbj", + "projectId": "dxCRnBdX5uJizCGa", + "project": { + "updatedAt": "2026-05-06T01:10:37.484Z", + "createdAt": "2026-05-06T01:08:07.721Z", + "id": "dxCRnBdX5uJizCGa", + "name": "Wilfred Fizzlepoof ", + "type": "personal", + "icon": null, + "description": null, + "creatorId": "47b2227f-2fc2-4a08-bd35-ea157b92df0d" + } + } + ], + "versionMetadata": { + "name": "Version 832248e5", + "description": "" + } +} \ No newline at end of file diff --git a/n8n-workflows/03-paperless-ai-processing.json b/n8n-workflows/03-paperless-ai-processing.json new file mode 100644 index 0000000..285a1b4 --- /dev/null +++ b/n8n-workflows/03-paperless-ai-processing.json @@ -0,0 +1,348 @@ +{ + "updatedAt": "2026-05-07T20:24:47.896Z", + "createdAt": "2026-05-07T20:24:40.153Z", + "id": "dQ1d5deV4mF0s5eP", + "name": "Paperless-NGX → AI Processing → Native Gotify + Apply Tags", + "description": null, + "active": true, + "isArchived": false, + "nodes": [ + { + "parameters": { + "httpMethod": "POST", + "path": "paperless/new-document", + "options": {} + }, + "id": "3d8bc021-e27f-4f7b-b716-a202d66551be", + "name": "Paperless Webhook", + "type": "n8n-nodes-base.webhook", + "typeVersion": 2, + "position": [ + 0, + 0 + ], + "webhookId": "fd1eaa3a-fbf1-4a17-94fd-bb483df602b7" + }, + { + "parameters": { + "jsCode": "// Extract document ID from whatever Paperless sends\nconst data = $input.first().json;\nconst body = data.body || data;\n\n// Try multiple possible field names and formats\nlet docId = body.document_id || body.id || body.doc_id || body.pk;\n\n// If it's a string like '{document_id}' (template not resolved), fall back to latest doc\nif (!docId || typeof docId === 'string' && docId.includes('{')) {\n docId = null;\n}\n\nreturn [{ json: { docId } }];" + }, + "id": "48277e9e-a835-4e12-8608-63e905abe74d", + "name": "Extract Document ID", + "type": "n8n-nodes-base.code", + "typeVersion": 2, + "position": [ + 224, + 0 + ] + }, + { + "parameters": { + "url": "=https://paperless.paccoco.com/api/documents/{{ $json.docId ? $json.docId + '/' : '?ordering=-added&page_size=1' }}", + "sendHeaders": true, + "headerParameters": { + "parameters": [ + { + "name": "Authorization", + "value": "Token 4fb34ba35b4ab76a0715e8c56faf00a31de4fb5d" + } + ] + }, + "options": {} + }, + "id": "17ed7e6f-ccaa-43d3-9031-9f3d2bb12dec", + "name": "Fetch Document", + "type": "n8n-nodes-base.httpRequest", + "typeVersion": 4.2, + "position": [ + 448, + 0 + ] + }, + { + "parameters": { + "jsCode": "const response = $input.first().json;\n\n// Handle both single doc response and list response\nconst doc = response.results ? response.results[0] : response;\n\nif (!doc || !doc.id) {\n throw new Error('No document found in response');\n}\n\n// Content is already in the metadata from Paperless OCR\nconst content = (doc.content || '').substring(0, 4000);\n\nreturn [{\n json: {\n documentId: doc.id,\n title: doc.title || 'Untitled',\n originalFilename: doc.original_file_name || '',\n correspondent: doc.correspondent_name || null,\n content: content,\n created: doc.created || new Date().toISOString()\n }\n}];" + }, + "id": "3a026b3b-141b-48b4-a9ae-03998914f45c", + "name": "Prepare Content", + "type": "n8n-nodes-base.code", + "typeVersion": 2, + "position": [ + 672, + 0 + ] + }, + { + "parameters": { + "method": "POST", + "url": "http://10.5.1.6:4000/v1/chat/completions", + "sendHeaders": true, + "headerParameters": { + "parameters": [ + { + "name": "Content-Type", + "value": "application/json" + }, + { + "name": "Authorization", + "value": "Bearer sk-REPLACE_WITH_LITELLM_KEY" + } + ] + }, + "sendBody": true, + "specifyBody": "json", + "jsonBody": "={\n \"model\": \"medium\",\n \"messages\": [\n {\n \"role\": \"system\",\n \"content\": \"You are a document classification assistant. Analyze the document and return ONLY valid JSON. Do not include markdown. Do not include code fences.\\n\\nReturn this exact JSON structure:\\n{\\n \\\"summary\\\": \\\"short 1-2 sentence summary with no sensitive identifiers\\\",\\n \\\"doc_type\\\": \\\"invoice | receipt | letter | contract | manual | report | medical | tax | military | other\\\",\\n \\\"suggested_tags\\\": [\\\"tag1\\\", \\\"tag2\\\", \\\"tag3\\\"]\\n}\\n\\nIf the document is sensitive, summarize only generally and say sensitive personal identifiers were present and redacted. If the document is not sensitive, provide a normal 1-2 sentence summary. Never mention or summarize sensitive field categories such as SSNs, dates of birth, addresses, phone numbers, financial amounts, ID numbers, service numbers, or exact dates.\"\n },\n {\n \"role\": \"user\",\n \"content\": \"{{ JSON.stringify('Document title: ' + ($json.title || 'Untitled') + '\\nFilename: ' + ($json.originalFilename || '') + '\\nCorrespondent: ' + ($json.correspondent || 'Unknown') + '\\n\\nContent:\\n' + ($json.content || '')).slice(1, -1) }}\"\n }\n ],\n \"max_tokens\": 350,\n \"temperature\": 0.1,\n \"response_format\": {\n \"type\": \"json_object\"\n }\n}", + "options": {} + }, + "id": "51374897-d90e-49a2-889e-0476817dd9d8", + "name": "AI Classify & Extract", + "type": "n8n-nodes-base.httpRequest", + "typeVersion": 4.2, + "position": [ + 880, + 0 + ] + }, + { + "parameters": { + "jsCode": "const response = $input.first().json;\nconst docData = $('Prepare Content').first().json;\n\nconst raw = response.choices?.[0]?.message?.content || '{}';\nlet analysis;\n\ntry {\n analysis = JSON.parse(raw);\n} catch (e) {\n analysis = {\n summary: 'Document processed, but AI response could not be parsed cleanly. Sensitive details were not included.',\n doc_type: 'other',\n suggested_tags: []\n };\n}\n\nfunction cleanText(value) {\n return String(value || '')\n .replace(/\\b\\d{3}[- ]?\\d{2}[- ]?\\d{4}\\b/g, '[REDACTED_SSN]')\n .replace(/\\b\\d{8}\\b/g, '[REDACTED_DATE]')\n .replace(/\\b\\d{1,5}\\s+[A-Za-z0-9 .'-]+\\s+(Road|RD|Street|St|Avenue|Ave|Drive|Dr|Lane|Ln|Court|Ct|Branch|Blvd|Boulevard)\\b[^,\\n]*/gi, '[REDACTED_ADDRESS]')\n .replace(/\\b\\d{5}(?:-\\d{4})?\\b/g, '[REDACTED_ZIP]');\n}\n\nconst tags = Array.isArray(analysis.suggested_tags)\n ? analysis.suggested_tags\n .map(t => cleanText(t).trim().toLowerCase())\n .filter(Boolean)\n .map(t => t.replace(/[^a-z0-9 _-]/gi, '').trim())\n .filter(Boolean)\n .slice(0, 5)\n : [];\n\nreturn [\n {\n json: {\n documentId: docData.documentId,\n title: cleanText(docData.title || 'Untitled'),\n summary: cleanText(analysis.summary || 'Document processed.'),\n doc_type: cleanText(analysis.doc_type || 'other').toLowerCase(),\n suggested_tags: tags\n }\n }\n];" + }, + "id": "1f37f096-132b-43b4-bcb4-f5ed094bc9e5", + "name": "Parse Analysis", + "type": "n8n-nodes-base.code", + "typeVersion": 2, + "position": [ + 1104, + 0 + ] + }, + { + "parameters": { + "method": "POST", + "url": "=https://paperless.paccoco.com/api/documents/{{ $json.documentId }}/notes/", + "sendHeaders": true, + "headerParameters": { + "parameters": [ + { + "name": "Authorization", + "value": "Token 4fb34ba35b4ab76a0715e8c56faf00a31de4fb5d" + }, + { + "name": "Content-Type", + "value": "application/json" + } + ] + }, + "sendBody": true, + "specifyBody": "json", + "jsonBody": "={\n \"note\": \"{{ JSON.stringify('**AI Analysis**\\nType: ' + ($json.doc_type || 'other') + '\\nSummary: ' + ($json.summary || '') + '\\nSuggested tags: ' + (($json.suggested_tags || []).join(', '))).slice(1, -1) }}\"\n}", + "options": {} + }, + "id": "5597c8c1-436e-4373-bb54-0180be4e7bca", + "name": "Add Note to Document", + "type": "n8n-nodes-base.httpRequest", + "typeVersion": 4.2, + "position": [ + 1328, + 0 + ] + }, + { + "parameters": { + "jsCode": "const data = $input.first().json;\n\nconst tags = Array.isArray(data.suggested_tags)\n ? data.suggested_tags.join(', ')\n : 'None';\n\nconst message = [\n data.title || 'Untitled',\n '',\n `Type: ${data.doc_type || 'other'}`,\n '',\n `Summary: ${data.summary || 'No summary returned'}`,\n '',\n `Suggested tags: ${tags || 'None'}`\n].join('\\n');\n\nreturn [{\n json: {\n title: '📄 New Document Processed',\n message,\n priority: 4\n }\n}];" + }, + "id": "c3319471-e3f1-4cd3-aae1-c794f081b1e3", + "name": "Format Gotify Document Message", + "type": "n8n-nodes-base.code", + "typeVersion": 2, + "position": [ + 1328, + 208 + ] + }, + { + "parameters": { + "message": "={{ $json.message }}", + "additionalFields": { + "priority": "={{ $json.priority }}", + "title": "={{ $json.title }}" + }, + "options": { + "contentType": "text/plain" + } + }, + "id": "193f19dc-2ad6-41af-bd29-a0f5dc0ef1a0", + "name": "Send Gotify Message", + "type": "n8n-nodes-base.gotify", + "typeVersion": 1, + "position": [ + 1552, + 208 + ], + "credentials": { + "gotifyApi": { + "id": "ajvRjvj0QldLQYmo", + "name": "Paperless" + } + } + }, + { + "parameters": { + "jsCode": "const data = $input.first().json;\n\nconst PAPERLESS_BASE_URL = 'https://paperless.paccoco.com';\nconst PAPERLESS_TOKEN = '4fb34ba35b4ab76a0715e8c56faf00a31de4fb5d';\n\nconst headers = {\n Authorization: `Token ${PAPERLESS_TOKEN}`,\n 'Content-Type': 'application/json',\n};\n\nfunction normalizeTagName(tag) {\n return String(tag || '')\n .trim()\n .toLowerCase()\n .replace(/[^a-z0-9 _-]/gi, '')\n .replace(/\\s+/g, ' ')\n .slice(0, 64);\n}\n\nconst suggestedTags = Array.isArray(data.suggested_tags)\n ? [...new Set(data.suggested_tags.map(normalizeTagName).filter(Boolean))].slice(0, 5)\n : [];\n\nif (!data.documentId) {\n throw new Error('No documentId found; cannot apply tags.');\n}\n\nif (suggestedTags.length === 0) {\n return [{\n json: {\n ...data,\n tag_apply_status: 'skipped_no_suggested_tags',\n applied_tag_names: [],\n applied_tag_ids: []\n }\n }];\n}\n\nasync function request(method, url, body) {\n const options = {\n method,\n url,\n headers,\n json: true,\n };\n\n if (body !== undefined) {\n options.body = body;\n }\n\n return await this.helpers.httpRequest(options);\n}\n\n// Fetch all existing tags. Handles pagination.\nconst existingTags = [];\nlet tagsUrl = `${PAPERLESS_BASE_URL}/api/tags/?page_size=100`;\n\nwhile (tagsUrl) {\n const page = await request.call(this, 'GET', tagsUrl);\n const results = Array.isArray(page.results) ? page.results : (Array.isArray(page) ? page : []);\n existingTags.push(...results);\n\n if (page.next) {\n tagsUrl = page.next.startsWith('http') ? page.next : `${PAPERLESS_BASE_URL}${page.next}`;\n } else {\n tagsUrl = null;\n }\n}\n\nconst tagsByName = new Map(\n existingTags.map(tag => [normalizeTagName(tag.name), tag])\n);\n\nconst appliedTags = [];\n\nfor (const tagName of suggestedTags) {\n let tag = tagsByName.get(tagName);\n\n if (!tag) {\n tag = await request.call(this, 'POST', `${PAPERLESS_BASE_URL}/api/tags/`, {\n name: tagName,\n });\n tagsByName.set(tagName, tag);\n }\n\n if (tag?.id) {\n appliedTags.push(tag);\n }\n}\n\n// Fetch current document so we merge tags instead of overwriting existing ones.\nconst doc = await request.call(this, 'GET', `${PAPERLESS_BASE_URL}/api/documents/${data.documentId}/`);\nconst currentTagIds = Array.isArray(doc.tags) ? doc.tags : [];\nconst appliedTagIds = appliedTags.map(tag => tag.id);\nconst mergedTagIds = [...new Set([...currentTagIds, ...appliedTagIds])];\n\nawait request.call(this, 'PATCH', `${PAPERLESS_BASE_URL}/api/documents/${data.documentId}/`, {\n tags: mergedTagIds,\n});\n\nreturn [{\n json: {\n ...data,\n tag_apply_status: 'updated',\n applied_tag_names: appliedTags.map(tag => tag.name),\n applied_tag_ids: appliedTagIds,\n final_tag_ids: mergedTagIds,\n }\n}];" + }, + "id": "b0d5423b-dc83-4adf-98c6-d95afa3e5c1b", + "name": "Apply Suggested Tags to Paperless", + "type": "n8n-nodes-base.code", + "typeVersion": 2, + "position": [ + 1328, + 400 + ] + } + ], + "connections": { + "Paperless Webhook": { + "main": [ + [ + { + "node": "Extract Document ID", + "type": "main", + "index": 0 + } + ] + ] + }, + "Extract Document ID": { + "main": [ + [ + { + "node": "Fetch Document", + "type": "main", + "index": 0 + } + ] + ] + }, + "Fetch Document": { + "main": [ + [ + { + "node": "Prepare Content", + "type": "main", + "index": 0 + } + ] + ] + }, + "Prepare Content": { + "main": [ + [ + { + "node": "AI Classify & Extract", + "type": "main", + "index": 0 + } + ] + ] + }, + "AI Classify & Extract": { + "main": [ + [ + { + "node": "Parse Analysis", + "type": "main", + "index": 0 + } + ] + ] + }, + "Parse Analysis": { + "main": [ + [ + { + "node": "Add Note to Document", + "type": "main", + "index": 0 + }, + { + "node": "Format Gotify Document Message", + "type": "main", + "index": 0 + }, + { + "node": "Apply Suggested Tags to Paperless", + "type": "main", + "index": 0 + } + ] + ] + }, + "Format Gotify Document Message": { + "main": [ + [ + { + "node": "Send Gotify Message", + "type": "main", + "index": 0 + } + ] + ] + } + }, + "settings": { + "executionOrder": "v1", + "binaryMode": "separate" + }, + "staticData": null, + "meta": { + "templateCredsSetupCompleted": true + }, + "pinData": {}, + "versionId": "199d2609-e3e7-46aa-b144-df038eb10a0d", + "activeVersionId": "199d2609-e3e7-46aa-b144-df038eb10a0d", + "versionCounter": 11, + "triggerCount": 1, + "tags": [ + { + "updatedAt": "2026-05-06T22:39:46.433Z", + "createdAt": "2026-05-06T22:39:46.433Z", + "id": "dZUptsMy60GbLHft", + "name": "documents" + }, + { + "updatedAt": "2026-05-06T22:38:51.678Z", + "createdAt": "2026-05-06T22:38:51.678Z", + "id": "S9FxzVfHLsHmyzyt", + "name": "ai" + }, + { + "updatedAt": "2026-05-06T22:39:46.476Z", + "createdAt": "2026-05-06T22:39:46.476Z", + "id": "NxHjuO4MMd5kdPR7", + "name": "paperless" + } + ], + "shared": [ + { + "updatedAt": "2026-05-07T20:24:40.153Z", + "createdAt": "2026-05-07T20:24:40.153Z", + "role": "workflow:owner", + "workflowId": "dQ1d5deV4mF0s5eP", + "projectId": "dxCRnBdX5uJizCGa", + "project": { + "updatedAt": "2026-05-06T01:10:37.484Z", + "createdAt": "2026-05-06T01:08:07.721Z", + "id": "dxCRnBdX5uJizCGa", + "name": "Wilfred Fizzlepoof ", + "type": "personal", + "icon": null, + "description": null, + "creatorId": "47b2227f-2fc2-4a08-bd35-ea157b92df0d" + } + } + ], + "versionMetadata": { + "name": "Version 199d2609", + "description": "" + } +} \ No newline at end of file diff --git a/n8n-workflows/04-whisper-transcription.json b/n8n-workflows/04-whisper-transcription.json new file mode 100644 index 0000000..749adee --- /dev/null +++ b/n8n-workflows/04-whisper-transcription.json @@ -0,0 +1,589 @@ +{ + "updatedAt": "2026-05-06T23:30:36.237Z", + "createdAt": "2026-05-06T23:30:36.237Z", + "id": "QEVnqyN4h5kcpVuJ", + "name": "Whisper Audio Transcription", + "description": null, + "active": true, + "isArchived": false, + "nodes": [ + { + "parameters": { + "httpMethod": "POST", + "path": "transcribe", + "responseMode": "lastNode", + "responseData": "allEntries", + "options": { + "rawBody": true + } + }, + "id": "91945dd9-42f8-4343-b131-9e46272f04bf", + "name": "Transcription Webhook", + "type": "n8n-nodes-base.webhook", + "typeVersion": 2, + "position": [ + 0, + 0 + ], + "webhookId": "f50eec94-1e56-4189-88ee-51b701c1d889" + }, + { + "parameters": { + "method": "POST", + "url": "http://10.5.1.6:8786/v1/audio/transcriptions", + "sendHeaders": true, + "headerParameters": { + "parameters": [ + { + "name": "Accept", + "value": "application/json" + } + ] + }, + "sendBody": true, + "contentType": "multipart-form-data", + "bodyParameters": { + "parameters": [ + { + "parameterType": "formBinaryData", + "name": "file", + "inputDataFieldName": "data" + }, + { + "name": "model", + "value": "Systran/faster-whisper-large-v3" + }, + { + "name": "language", + "value": "en" + }, + { + "name": "response_format", + "value": "verbose_json" + } + ] + }, + "options": {} + }, + "id": "fc726196-f62d-4ff4-9be6-a84b9dbfff9e", + "name": "Whisper Transcribe", + "type": "n8n-nodes-base.httpRequest", + "typeVersion": 4.2, + "position": [ + 224, + 0 + ] + }, + { + "parameters": { + "jsCode": "const result = $input.first().json;\n\nconst segments = result.segments || [];\nconst fullText = result.text || segments.map(s => s.text).join(' ');\nconst duration = result.duration || segments[segments.length - 1]?.end || 0;\n\n// Format timestamps\nconst timestamped = segments.map(s => {\n const start = new Date(s.start * 1000).toISOString().substr(11, 8);\n return `[${start}] ${s.text.trim()}`;\n}).join('\\n');\n\nreturn [{\n json: {\n text: fullText.trim(),\n timestamped_text: timestamped,\n duration_seconds: Math.round(duration),\n segment_count: segments.length,\n language: result.language || 'en'\n }\n}];" + }, + "id": "1b6f017f-849d-456f-9e4b-a0bced1d8e9f", + "name": "Format Transcript", + "type": "n8n-nodes-base.code", + "typeVersion": 2, + "position": [ + 448, + 0 + ] + }, + { + "parameters": { + "httpMethod": "POST", + "path": "transcribe/summarize", + "responseMode": "lastNode", + "responseData": "allEntries", + "options": { + "rawBody": true + } + }, + "id": "c2f8a2dd-cbbc-4b62-ae24-0f4a95bdf862", + "name": "Transcribe + Summarize Webhook", + "type": "n8n-nodes-base.webhook", + "typeVersion": 2, + "position": [ + 0, + 304 + ], + "webhookId": "69b4b2e8-9486-4aa3-b9db-fb8e6bdae98e" + }, + { + "parameters": { + "method": "POST", + "url": "http://10.5.1.6:8786/v1/audio/transcriptions", + "sendHeaders": true, + "headerParameters": { + "parameters": [ + { + "name": "Accept", + "value": "application/json" + } + ] + }, + "sendBody": true, + "contentType": "multipart-form-data", + "bodyParameters": { + "parameters": [ + { + "parameterType": "formBinaryData", + "name": "file", + "inputDataFieldName": "data" + }, + { + "name": "model", + "value": "Systran/faster-whisper-large-v3" + }, + { + "name": "language", + "value": "en" + } + ] + }, + "options": {} + }, + "id": "ee25bf40-99ef-472b-928d-6ef63516094c", + "name": "Whisper Transcribe 2", + "type": "n8n-nodes-base.httpRequest", + "typeVersion": 4.2, + "position": [ + 224, + 304 + ] + }, + { + "parameters": { + "method": "POST", + "url": "http://10.5.1.6:4000/v1/chat/completions", + "sendHeaders": true, + "headerParameters": { + "parameters": [ + { + "name": "Content-Type", + "value": "application/json" + }, + { + "name": "Authorization", + "value": "Bearer sk-REPLACE_WITH_LITELLM_KEY" + } + ] + }, + "sendBody": true, + "specifyBody": "json", + "jsonBody": "={\n \"model\": \"medium\",\n \"messages\": [\n {\n \"role\": \"system\",\n \"content\": \"You are a meeting/audio note summarizer. Given a transcription, provide:\\n1. A brief summary (2-3 sentences)\\n2. Key points as a bullet list\\n3. Any action items mentioned\\n\\nRespond in JSON with keys: summary, key_points (array), action_items (array)\"\n },\n {\n \"role\": \"user\",\n \"content\": \"Summarize this transcription:\\n\\n{{ $json.text }}\"\n }\n ],\n \"max_tokens\": 500,\n \"temperature\": 0.2,\n \"response_format\": { \"type\": \"json_object\" }\n}", + "options": {} + }, + "id": "eda24425-dc26-4b4b-858e-9ae0d5e1a5e4", + "name": "AI Summarize Audio", + "type": "n8n-nodes-base.httpRequest", + "typeVersion": 4.2, + "position": [ + 448, + 304 + ] + }, + { + "parameters": { + "jsCode": "const llmResponse = $input.first().json;\nconst transcript = $('Whisper Transcribe 2').first().json;\nlet analysis;\n\ntry {\n analysis = JSON.parse(llmResponse.choices?.[0]?.message?.content || '{}');\n} catch(e) {\n analysis = { summary: llmResponse.choices?.[0]?.message?.content || 'Summary failed', key_points: [], action_items: [] };\n}\n\nreturn [{\n json: {\n transcript: transcript.text || '',\n duration_seconds: Math.round(transcript.duration || 0),\n language: transcript.language || 'en',\n summary: analysis.summary,\n key_points: analysis.key_points || [],\n action_items: analysis.action_items || []\n }\n}];" + }, + "id": "cf41669b-8bf7-433b-b390-10765954b0a9", + "name": "Format Summary Response", + "type": "n8n-nodes-base.code", + "typeVersion": 2, + "position": [ + 672, + 304 + ] + }, + { + "parameters": { + "jsCode": "const t = $input.first().json;\nconst mins = Math.floor(t.duration_seconds / 60);\nconst secs = t.duration_seconds % 60;\nreturn [{ json: {\n title: '\ud83c\udf99\ufe0f Transcription Complete',\n message: `Duration: ${mins}m ${secs}s \u00b7 ${t.segment_count} segments \u00b7 Language: ${t.language}`,\n priority: 3\n}}];\n" + }, + "id": "5c692355-2a46-4c66-860a-c02a21c73597", + "name": "Format Transcription Notify", + "type": "n8n-nodes-base.code", + "typeVersion": 2, + "position": [ + 672, + 0 + ] + }, + { + "parameters": { + "message": "={{ $json.message }}", + "additionalFields": { + "priority": "={{ $json.priority }}", + "title": "={{ $json.title }}" + }, + "options": { + "contentType": "text/plain" + } + }, + "id": "0e96240d-50e2-49f7-be62-4f57451e0543", + "name": "Notify Transcription Done", + "type": "n8n-nodes-base.gotify", + "typeVersion": 1, + "position": [ + 896, + 0 + ], + "credentials": { + "gotifyApi": { + "id": "SETUP_REQUIRED", + "name": "Whisper" + } + } + }, + { + "parameters": { + "jsCode": "const t = $('Format Transcript').first().json;\nconst title = `Transcript ${new Date().toLocaleDateString('en-US')}`;\nconst content = `Transcript\\nDate: ${new Date().toISOString()}\\nDuration: ${t.duration_seconds}s\\nLanguage: ${t.language}\\n\\n${t.timestamped_text || t.text}`;\nreturn [{ binary: { document: { data: Buffer.from(content,'utf-8').toString('base64'), mimeType:'text/plain', fileName: `${title}.txt` }}, json: { title } }];\n" + }, + "id": "da9f4d8d-8c5b-413a-badd-71287dbe0a3c", + "name": "Prepare Transcript Doc", + "type": "n8n-nodes-base.code", + "typeVersion": 2, + "position": [ + 1120, + 0 + ] + }, + { + "parameters": { + "method": "POST", + "url": "https://paperless.paccoco.com/api/documents/post_document/", + "options": {}, + "sendHeaders": true, + "headerParameters": { + "parameters": [ + { + "name": "Authorization", + "value": "Bearer ${PAPERLESS_API_TOKEN}" + } + ] + }, + "sendBody": true, + "contentType": "multipart-form-data", + "bodyParameters": { + "parameters": [ + { + "name": "document", + "value": "data", + "parameterType": "formBinaryData" + }, + { + "name": "title", + "value": "={{ $json.title }}" + }, + { + "name": "tags[]", + "value": "${PAPERLESS_TAG_TRANSCRIPTION}" + } + ] + } + }, + "id": "7c84da5a-41e2-4b38-9e58-19acfd73f81d", + "name": "Save Transcript to Paperless", + "type": "n8n-nodes-base.httpRequest", + "typeVersion": 4.2, + "position": [ + 1344, + 0 + ], + "continueOnFail": true + }, + { + "parameters": { + "jsCode": "const s = $input.first().json;\nconst mins = Math.floor(s.duration_seconds / 60);\nconst points = (s.key_points || []).slice(0,3).map((p,i) => `${i+1}. ${p}`).join('\\n');\nreturn [{ json: {\n title: '\ud83c\udf99\ufe0f Transcription + Summary Complete',\n message: `${mins}m audio\\n\\nSummary: ${s.summary}\\n\\nKey points:\\n${points}`,\n priority: 4\n}}];\n" + }, + "id": "e55363e4-ff9b-4df4-b40a-fd23227c23e8", + "name": "Format Summary Notify", + "type": "n8n-nodes-base.code", + "typeVersion": 2, + "position": [ + 896, + 304 + ] + }, + { + "parameters": { + "message": "={{ $json.message }}", + "additionalFields": { + "priority": "={{ $json.priority }}", + "title": "={{ $json.title }}" + }, + "options": { + "contentType": "text/plain" + } + }, + "id": "b5defea9-35a0-4a22-8769-ed9496a022f6", + "name": "Notify Summary Done", + "type": "n8n-nodes-base.gotify", + "typeVersion": 1, + "position": [ + 1120, + 304 + ], + "credentials": { + "gotifyApi": { + "id": "SETUP_REQUIRED", + "name": "Whisper" + } + } + }, + { + "parameters": { + "jsCode": "const s = $('Format Summary Response').first().json;\nconst title = `Audio Summary ${new Date().toLocaleDateString('en-US')}`;\nconst points = (s.key_points || []).map((p,i) => `${i+1}. ${p}`).join('\\n');\nconst actions = (s.action_items || []).map((a,i) => `${i+1}. ${a}`).join('\\n');\nconst content = [\n `Audio Transcription & Summary`, `Date: ${new Date().toISOString()}`,\n `Duration: ${Math.floor(s.duration_seconds/60)}m ${s.duration_seconds%60}s`,\n `Language: ${s.language}`, '', `Summary`, `=======`, s.summary,\n '', `Key Points`, `==========`, points,\n ...(actions ? ['', `Action Items`, `============`, actions] : []),\n '', `Full Transcript`, `==============`, s.transcript\n].join('\\n');\nreturn [{ binary: { document: { data: Buffer.from(content,'utf-8').toString('base64'), mimeType:'text/plain', fileName:`${title}.txt` }}, json: { title } }];\n" + }, + "id": "1f31f608-7985-4a9a-afcf-6cf4e44150ba", + "name": "Prepare Summary Doc", + "type": "n8n-nodes-base.code", + "typeVersion": 2, + "position": [ + 1344, + 304 + ] + }, + { + "parameters": { + "method": "POST", + "url": "https://paperless.paccoco.com/api/documents/post_document/", + "options": {}, + "sendHeaders": true, + "headerParameters": { + "parameters": [ + { + "name": "Authorization", + "value": "Bearer ${PAPERLESS_API_TOKEN}" + } + ] + }, + "sendBody": true, + "contentType": "multipart-form-data", + "bodyParameters": { + "parameters": [ + { + "name": "document", + "value": "data", + "parameterType": "formBinaryData" + }, + { + "name": "title", + "value": "={{ $json.title }}" + }, + { + "name": "tags[]", + "value": "${PAPERLESS_TAG_TRANSCRIPTION}" + } + ] + } + }, + "id": "e9eaee2f-d08e-4a76-af53-ed824ebfe68f", + "name": "Save Summary to Paperless", + "type": "n8n-nodes-base.httpRequest", + "typeVersion": 4.2, + "position": [ + 1568, + 304 + ], + "continueOnFail": true + } + ], + "connections": { + "Transcription Webhook": { + "main": [ + [ + { + "node": "Whisper Transcribe", + "type": "main", + "index": 0 + } + ] + ] + }, + "Whisper Transcribe": { + "main": [ + [ + { + "node": "Format Transcript", + "type": "main", + "index": 0 + } + ] + ] + }, + "Transcribe + Summarize Webhook": { + "main": [ + [ + { + "node": "Whisper Transcribe 2", + "type": "main", + "index": 0 + } + ] + ] + }, + "Whisper Transcribe 2": { + "main": [ + [ + { + "node": "AI Summarize Audio", + "type": "main", + "index": 0 + } + ] + ] + }, + "AI Summarize Audio": { + "main": [ + [ + { + "node": "Format Summary Response", + "type": "main", + "index": 0 + } + ] + ] + }, + "Format Transcript": { + "main": [ + [ + { + "node": "Format Transcription Notify", + "type": "main", + "index": 0 + } + ] + ] + }, + "Format Transcription Notify": { + "main": [ + [ + { + "node": "Notify Transcription Done", + "type": "main", + "index": 0 + } + ] + ] + }, + "Notify Transcription Done": { + "main": [ + [ + { + "node": "Prepare Transcript Doc", + "type": "main", + "index": 0 + } + ] + ] + }, + "Prepare Transcript Doc": { + "main": [ + [ + { + "node": "Save Transcript to Paperless", + "type": "main", + "index": 0 + } + ] + ] + }, + "Format Summary Response": { + "main": [ + [ + { + "node": "Format Summary Notify", + "type": "main", + "index": 0 + } + ] + ] + }, + "Format Summary Notify": { + "main": [ + [ + { + "node": "Notify Summary Done", + "type": "main", + "index": 0 + } + ] + ] + }, + "Notify Summary Done": { + "main": [ + [ + { + "node": "Prepare Summary Doc", + "type": "main", + "index": 0 + } + ] + ] + }, + "Prepare Summary Doc": { + "main": [ + [ + { + "node": "Save Summary to Paperless", + "type": "main", + "index": 0 + } + ] + ] + } + }, + "settings": { + "executionOrder": "v1", + "binaryMode": "separate" + }, + "staticData": null, + "meta": null, + "pinData": {}, + "versionId": "20275daf-b924-4311-9a7f-116a99911764", + "activeVersionId": "20275daf-b924-4311-9a7f-116a99911764", + "versionCounter": 12, + "triggerCount": 2, + "tags": [ + { + "updatedAt": "2026-05-06T22:39:59.719Z", + "createdAt": "2026-05-06T22:39:59.719Z", + "id": "GLLrEIR8I7eA4ydv", + "name": "transcription" + }, + { + "updatedAt": "2026-05-06T22:39:59.716Z", + "createdAt": "2026-05-06T22:39:59.716Z", + "id": "tuPxDoKFqpVrrQoX", + "name": "whisper" + }, + { + "updatedAt": "2026-05-06T22:38:51.678Z", + "createdAt": "2026-05-06T22:38:51.678Z", + "id": "S9FxzVfHLsHmyzyt", + "name": "ai" + } + ], + "shared": [ + { + "updatedAt": "2026-05-06T23:30:36.237Z", + "createdAt": "2026-05-06T23:30:36.237Z", + "role": "workflow:owner", + "workflowId": "QEVnqyN4h5kcpVuJ", + "projectId": "dxCRnBdX5uJizCGa", + "project": { + "updatedAt": "2026-05-06T01:10:37.484Z", + "createdAt": "2026-05-06T01:08:07.721Z", + "id": "dxCRnBdX5uJizCGa", + "name": "Wilfred Fizzlepoof ", + "type": "personal", + "icon": null, + "description": null, + "creatorId": "47b2227f-2fc2-4a08-bd35-ea157b92df0d" + } + } + ], + "versionMetadata": { + "name": "Version 20275daf", + "description": "" + } +} \ No newline at end of file diff --git a/n8n-workflows/05-paperless-to-rag.json b/n8n-workflows/05-paperless-to-rag.json new file mode 100644 index 0000000..2894749 --- /dev/null +++ b/n8n-workflows/05-paperless-to-rag.json @@ -0,0 +1,357 @@ +{ + "updatedAt": "2026-05-07T00:48:49.355Z", + "createdAt": "2026-05-07T00:48:49.355Z", + "id": "4YKnRQ9rOiQCeYEU", + "name": "Paperless \u2192 RAG Ingest", + "description": null, + "active": true, + "isArchived": false, + "nodes": [ + { + "parameters": { + "httpMethod": "POST", + "path": "paperless/rag-ingest", + "options": {} + }, + "id": "a86d7025-bbf0-4c1b-a59c-ad254ddc913e", + "name": "Paperless RAG Webhook", + "type": "n8n-nodes-base.webhook", + "typeVersion": 2, + "position": [ + 0, + 0 + ], + "webhookId": "f2eccecb-9935-425b-be6a-f57fe4473f34" + }, + { + "parameters": { + "jsCode": "const data = $input.first().json;\nconst body = data.body || data;\n\nlet docId = body.document_id || body.id || body.doc_id || body.pk;\n\nif (!docId || typeof docId === 'string' && docId.includes('{')) {\n docId = null;\n}\n\nreturn [{ json: { docId } }];" + }, + "id": "8f6af849-bdd9-466e-af81-84d057fd7ca1", + "name": "Extract Document ID", + "type": "n8n-nodes-base.code", + "typeVersion": 2, + "position": [ + 224, + 0 + ] + }, + { + "parameters": { + "url": "=https://paperless.paccoco.com/api/documents/{{ $json.docId ? $json.docId + '/' : '?ordering=-added&page_size=1' }}", + "sendHeaders": true, + "headerParameters": { + "parameters": [ + { + "name": "Authorization", + "value": "Token 4fb34ba35b4ab76a0715e8c56faf00a31de4fb5d" + } + ] + }, + "options": {} + }, + "id": "0b53d951-40cd-4ef0-ab81-3dea73f8b1d7", + "name": "Fetch Document", + "type": "n8n-nodes-base.httpRequest", + "typeVersion": 4.2, + "position": [ + 448, + 0 + ] + }, + { + "parameters": { + "jsCode": "const response = $input.first().json;\nconst doc = response.results ? response.results[0] : response;\n\nif (!doc || !doc.id) {\n throw new Error('No document found in response');\n}\n\nconst content = doc.content || '';\nconst title = doc.title || 'Untitled';\nconst correspondent = doc.correspondent_name || 'Unknown';\nconst created = doc.created || new Date().toISOString();\nconst docType = doc.document_type_name || 'unknown';\nconst tags = (doc.tags || []).join(', ');\n\n// Build rich text for RAG with metadata context\nconst ragText = `Document: ${title}\\nCorrespondent: ${correspondent}\\nType: ${docType}\\nTags: ${tags}\\nDate: ${created}\\n\\n${content}`;\n\nreturn [{\n json: {\n documentId: doc.id,\n title,\n ragText,\n source: `paperless-doc-${doc.id}`,\n metadata: {\n paperless_id: doc.id,\n title,\n correspondent,\n document_type: docType,\n created\n }\n }\n}];" + }, + "id": "e3d506bc-1677-4825-b3d3-f6187156da22", + "name": "Prepare RAG Text", + "type": "n8n-nodes-base.code", + "typeVersion": 2, + "position": [ + 672, + 0 + ] + }, + { + "parameters": { + "jsCode": "// Split text into chunks for embedding\nconst input = $input.first().json;\nconst text = input.ragText || '';\nconst source = input.source;\nconst metadata = input.metadata;\n\nconst CHUNK_SIZE = 500;\nconst OVERLAP = 50;\n\nconst chunks = [];\nconst sentences = text.split(/(?<=[.!?])\\s+/);\nlet current = '';\n\nfor (const sentence of sentences) {\n if ((current + ' ' + sentence).length > CHUNK_SIZE && current.length > 0) {\n chunks.push(current.trim());\n const words = current.split(' ');\n current = words.slice(-OVERLAP).join(' ') + ' ' + sentence;\n } else {\n current = current ? current + ' ' + sentence : sentence;\n }\n}\nif (current.trim()) chunks.push(current.trim());\n\nreturn chunks.map((chunk, i) => ({\n json: {\n chunk,\n chunkIndex: i,\n totalChunks: chunks.length,\n source,\n metadata\n }\n}));" + }, + "id": "9a523501-3adb-41a3-b963-b6bb5adb25fe", + "name": "Chunk Text", + "type": "n8n-nodes-base.code", + "typeVersion": 2, + "position": [ + 880, + 0 + ] + }, + { + "parameters": { + "method": "POST", + "url": "http://10.5.1.6:11434/v1/embeddings", + "sendHeaders": true, + "headerParameters": { + "parameters": [ + { + "name": "Content-Type", + "value": "application/json" + } + ] + }, + "sendBody": true, + "specifyBody": "json", + "jsonBody": "={\n \"model\": \"nomic-embed-text\",\n \"input\": \"{{ $json.chunk }}\"\n}", + "options": { + "batching": { + "batch": { + "batchSize": 5, + "batchInterval": 500 + } + } + } + }, + "id": "0b631ea4-b11f-4533-960f-f5161ac798f3", + "name": "Get Embedding", + "type": "n8n-nodes-base.httpRequest", + "typeVersion": 4.2, + "position": [ + 1104, + 0 + ] + }, + { + "parameters": { + "jsCode": "const embeddingResponse = $input.first().json;\nconst chunkData = $('Chunk Text').item;\nconst embedding = embeddingResponse.data?.[0]?.embedding || [];\n\nconst pointId = 'xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx'.replace(/[xy]/g, c => {\n const r = Math.random() * 16 | 0;\n return (c === 'x' ? r : (r & 0x3 | 0x8)).toString(16);\n});\n\nreturn [{\n json: {\n id: pointId,\n vector: embedding,\n payload: {\n text: chunkData.json.chunk,\n source: chunkData.json.source,\n chunkIndex: chunkData.json.chunkIndex,\n totalChunks: chunkData.json.totalChunks,\n metadata: chunkData.json.metadata,\n ingested_at: new Date().toISOString()\n }\n }\n}];" + }, + "id": "ad9dc677-e3a2-464e-a82e-2aa4d8b9b042", + "name": "Prepare Qdrant Point", + "type": "n8n-nodes-base.code", + "typeVersion": 2, + "position": [ + 1328, + 0 + ] + }, + { + "parameters": { + "method": "PUT", + "url": "http://10.5.1.6:6333/collections/knowledge_base/points", + "sendHeaders": true, + "headerParameters": { + "parameters": [ + { + "name": "Content-Type", + "value": "application/json" + } + ] + }, + "sendBody": true, + "specifyBody": "json", + "jsonBody": "={\n \"points\": [\n {\n \"id\": \"{{ $json.id }}\",\n \"vector\": {{ JSON.stringify($json.vector) }},\n \"payload\": {{ JSON.stringify($json.payload) }}\n }\n ]\n}", + "options": {} + }, + "id": "e977b536-d470-4da0-a970-794a858e8aa9", + "name": "Upsert to Qdrant", + "type": "n8n-nodes-base.httpRequest", + "typeVersion": 4.2, + "position": [ + 1552, + 0 + ] + }, + { + "parameters": { + "jsCode": "const ragText = $('Prepare RAG Text').first().json;\nconst chunks = $('Chunk Text').first().json;\nreturn [{\n json: {\n title: '\ud83d\udcda Document Added to RAG',\n message: `${ragText.title} has been chunked and ingested into the knowledge base (${chunks.totalChunks || '?'} chunks)`,\n priority: 3\n }\n}];" + }, + "id": "fmt05-0001-0000-0000-000000000005", + "name": "Format Gotify Message", + "type": "n8n-nodes-base.code", + "typeVersion": 2, + "position": [ + 1520, + 0 + ] + }, + { + "parameters": { + "message": "={{ $json.message }}", + "additionalFields": { + "priority": "={{ $json.priority }}", + "title": "={{ $json.title }}" + }, + "options": { + "contentType": "text/plain" + } + }, + "id": "ccdee208-b8f8-4ad4-a20c-94c4735d2e3d", + "name": "Notify via Gotify", + "type": "n8n-nodes-base.gotify", + "typeVersion": 1, + "position": [ + 1760, + 0 + ], + "credentials": { + "gotifyApi": { + "id": "SETUP_REQUIRED", + "name": "Paperless RAG" + } + } + } + ], + "connections": { + "Paperless RAG Webhook": { + "main": [ + [ + { + "node": "Extract Document ID", + "type": "main", + "index": 0 + } + ] + ] + }, + "Extract Document ID": { + "main": [ + [ + { + "node": "Fetch Document", + "type": "main", + "index": 0 + } + ] + ] + }, + "Fetch Document": { + "main": [ + [ + { + "node": "Prepare RAG Text", + "type": "main", + "index": 0 + } + ] + ] + }, + "Prepare RAG Text": { + "main": [ + [ + { + "node": "Chunk Text", + "type": "main", + "index": 0 + } + ] + ] + }, + "Chunk Text": { + "main": [ + [ + { + "node": "Get Embedding", + "type": "main", + "index": 0 + } + ] + ] + }, + "Get Embedding": { + "main": [ + [ + { + "node": "Prepare Qdrant Point", + "type": "main", + "index": 0 + } + ] + ] + }, + "Prepare Qdrant Point": { + "main": [ + [ + { + "node": "Upsert to Qdrant", + "type": "main", + "index": 0 + } + ] + ] + }, + "Upsert to Qdrant": { + "main": [ + [ + { + "node": "Format Gotify Message", + "type": "main", + "index": 0 + } + ] + ] + }, + "Format Gotify Message": { + "main": [ + [ + { + "node": "Notify via Gotify", + "type": "main", + "index": 0 + } + ] + ] + } + }, + "settings": { + "executionOrder": "v1", + "binaryMode": "separate" + }, + "staticData": null, + "meta": null, + "pinData": {}, + "versionId": "3c20b423-8708-4540-8bf9-8db4880335a5", + "activeVersionId": "3c20b423-8708-4540-8bf9-8db4880335a5", + "versionCounter": 12, + "triggerCount": 1, + "tags": [ + { + "updatedAt": "2026-05-06T22:39:46.476Z", + "createdAt": "2026-05-06T22:39:46.476Z", + "id": "NxHjuO4MMd5kdPR7", + "name": "paperless" + }, + { + "updatedAt": "2026-05-06T22:39:33.284Z", + "createdAt": "2026-05-06T22:39:33.284Z", + "id": "7uuEyQrIlcuruZBk", + "name": "rag" + }, + { + "updatedAt": "2026-05-06T22:38:51.678Z", + "createdAt": "2026-05-06T22:38:51.678Z", + "id": "S9FxzVfHLsHmyzyt", + "name": "ai" + } + ], + "shared": [ + { + "updatedAt": "2026-05-07T00:48:49.355Z", + "createdAt": "2026-05-07T00:48:49.355Z", + "role": "workflow:owner", + "workflowId": "4YKnRQ9rOiQCeYEU", + "projectId": "dxCRnBdX5uJizCGa", + "project": { + "updatedAt": "2026-05-06T01:10:37.484Z", + "createdAt": "2026-05-06T01:08:07.721Z", + "id": "dxCRnBdX5uJizCGa", + "name": "Wilfred Fizzlepoof ", + "type": "personal", + "icon": null, + "description": null, + "creatorId": "47b2227f-2fc2-4a08-bd35-ea157b92df0d" + } + } + ], + "versionMetadata": { + "name": "Version 3c20b423", + "description": "" + } +} \ No newline at end of file diff --git a/n8n-workflows/06-web-scrape-summarize.json b/n8n-workflows/06-web-scrape-summarize.json new file mode 100644 index 0000000..b6e32ae --- /dev/null +++ b/n8n-workflows/06-web-scrape-summarize.json @@ -0,0 +1,588 @@ +{ + "updatedAt": "2026-05-07T18:46:25.970Z", + "createdAt": "2026-05-07T04:01:33.583Z", + "id": "40wcTHzFz6GSFlJX", + "name": "Web Scrape AI Summarize Gotify - Deduped v8 Native Gotify Priority Tuned", + "description": null, + "active": true, + "isArchived": false, + "nodes": [ + { + "parameters": { + "rule": { + "interval": [ + { + "field": "hours", + "hoursInterval": 6 + } + ] + } + }, + "id": "ca37c1d7-3290-407f-a3d7-90f3ce036c05", + "name": "Every 6 Hours", + "type": "n8n-nodes-base.scheduleTrigger", + "typeVersion": 1.2, + "position": [ + 0, + 0 + ] + }, + { + "parameters": { + "httpMethod": "POST", + "path": "scrape/summarize", + "responseMode": "lastNode", + "responseData": "allEntries", + "options": {} + }, + "id": "57fe2624-5844-4bd9-bd63-297876ab5372", + "name": "Manual Trigger Webhook", + "type": "n8n-nodes-base.webhook", + "typeVersion": 2, + "position": [ + 0, + 224 + ], + "webhookId": "31dca7cf-5733-4501-90e1-1cb31c6331b2" + }, + { + "parameters": { + "jsCode": "// Define RSS feeds to monitor.\n// Each feed carries category/tags/basePriority into parsing, dedupe, scoring, AI, and Gotify.\n// basePriority is added to the keyword score before mapping to Gotify priority.\n\nconst feeds = [\n // Existing/core\n {\n name: 'Self-Hosted Show',\n url: 'https://selfhosted.show/rss',\n category: 'podcast',\n tags: ['selfhosted', 'homelab', 'linux'],\n priority: 'normal',\n basePriority: 1\n },\n {\n name: 'r/selfhosted',\n url: 'https://www.reddit.com/r/selfhosted/.rss',\n category: 'reddit',\n tags: ['selfhosted', 'docker', 'apps'],\n priority: 'normal',\n basePriority: 1\n },\n {\n name: 'r/homelab',\n url: 'https://www.reddit.com/r/homelab/.rss',\n category: 'reddit',\n tags: ['homelab', 'networking', 'hardware'],\n priority: 'normal',\n basePriority: 1\n },\n {\n name: 'Hacker News',\n url: 'https://news.ycombinator.com/rss',\n category: 'tech-news',\n tags: ['programming', 'ai', 'security'],\n priority: 'normal',\n basePriority: 1\n },\n\n // Homelab / infrastructure\n {\n name: 'ServeTheHome',\n url: 'https://www.servethehome.com/feed/',\n category: 'hardware',\n tags: ['server', 'storage', 'networking', 'homelab'],\n priority: 'normal',\n basePriority: 2\n },\n {\n name: 'Cloudflare Blog',\n url: 'https://blog.cloudflare.com/rss/',\n category: 'infrastructure',\n tags: ['networking', 'security', 'dns', 'cloudflare'],\n priority: 'normal',\n basePriority: 2\n },\n {\n name: 'Phoronix',\n url: 'https://www.phoronix.com/rss.php',\n category: 'linux',\n tags: ['linux', 'kernel', 'drivers', 'hardware'],\n priority: 'normal',\n basePriority: 2\n },\n {\n name: 'Jeff Geerling',\n url: 'https://www.jeffgeerling.com/blog.xml',\n category: 'homelab',\n tags: ['homelab', 'raspberry-pi', 'ansible', 'linux'],\n priority: 'normal',\n basePriority: 2\n },\n {\n name: 'LinuxServer.io Blog',\n url: 'https://www.linuxserver.io/blog/rss.xml',\n category: 'containers',\n tags: ['docker', 'containers', 'selfhosted'],\n priority: 'normal',\n basePriority: 2\n },\n {\n name: 'TrueNAS Blog',\n url: 'https://www.truenas.com/blog/feed/',\n category: 'storage',\n tags: ['truenas', 'zfs', 'nas', 'storage'],\n priority: 'high',\n basePriority: 4\n },\n {\n name: 'Proxmox News',\n url: 'https://www.proxmox.com/en/feeds/news',\n category: 'virtualization',\n tags: ['proxmox', 'virtualization', 'linux', 'backup'],\n priority: 'high',\n basePriority: 4\n },\n {\n name: 'Tailscale Changelog',\n url: 'https://tailscale.com/changelog/rss.xml',\n category: 'networking',\n tags: ['tailscale', 'vpn', 'networking', 'security'],\n priority: 'normal',\n basePriority: 3\n },\n {\n name: 'K3s Blog',\n url: 'https://k3s.io/blog/index.xml',\n category: 'kubernetes',\n tags: ['k3s', 'kubernetes', 'containers'],\n priority: 'normal',\n basePriority: 2\n },\n\n // Security\n {\n name: 'The Hacker News',\n url: 'https://feeds.feedburner.com/TheHackersNews',\n category: 'security',\n tags: ['security', 'vulnerabilities', 'breaches'],\n priority: 'high',\n basePriority: 5\n },\n {\n name: 'Krebs on Security',\n url: 'https://krebsonsecurity.com/feed/',\n category: 'security',\n tags: ['security', 'breaches', 'fraud'],\n priority: 'high',\n basePriority: 5\n },\n {\n name: 'BleepingComputer',\n url: 'https://www.bleepingcomputer.com/feed/',\n category: 'security',\n tags: ['security', 'windows', 'vulnerabilities', 'ransomware'],\n priority: 'high',\n basePriority: 5\n },\n {\n name: 'CISA Advisories',\n url: 'https://www.cisa.gov/cybersecurity-advisories/all.xml',\n category: 'security',\n tags: ['cisa', 'cve', 'security', 'advisory'],\n priority: 'high',\n basePriority: 6\n },\n {\n name: 'NVD CVE Feed',\n url: 'https://nvd.nist.gov/feeds/xml/cve/misc/nvd-rss.xml',\n category: 'security',\n tags: ['cve', 'vulnerability', 'security'],\n priority: 'high',\n basePriority: 6\n },\n {\n name: 'Dark Reading',\n url: 'https://www.darkreading.com/rss.xml',\n category: 'security',\n tags: ['security', 'enterprise', 'vulnerabilities'],\n priority: 'high',\n basePriority: 5\n },\n {\n name: 'Schneier on Security',\n url: 'https://www.schneier.com/feed/atom/',\n category: 'security-analysis',\n tags: ['security', 'privacy', 'policy'],\n priority: 'normal',\n basePriority: 3\n },\n\n // AI / LLM\n {\n name: 'Simon Willison',\n url: 'https://simonwillison.net/atom/everything/',\n category: 'ai-dev',\n tags: ['ai', 'llm', 'programming', 'python'],\n priority: 'normal',\n basePriority: 2\n },\n {\n name: 'Hugging Face Blog',\n url: 'https://huggingface.co/blog/feed.xml',\n category: 'ai',\n tags: ['ai', 'ml', 'models', 'llm'],\n priority: 'normal',\n basePriority: 2\n },\n {\n name: 'Latent Space',\n url: 'https://www.latent.space/feed',\n category: 'ai',\n tags: ['ai', 'llm', 'engineering'],\n priority: 'normal',\n basePriority: 2\n },\n {\n name: 'OpenAI News',\n url: 'https://openai.com/news/rss.xml',\n category: 'ai',\n tags: ['openai', 'ai', 'llm'],\n priority: 'normal',\n basePriority: 2\n },\n {\n name: 'Anthropic News',\n url: 'https://www.anthropic.com/news/rss.xml',\n category: 'ai',\n tags: ['anthropic', 'ai', 'llm'],\n priority: 'normal',\n basePriority: 2\n },\n {\n name: 'Ollama Blog',\n url: 'https://ollama.com/blog/rss.xml',\n category: 'local-ai',\n tags: ['ollama', 'local-ai', 'llm', 'homelab'],\n priority: 'normal',\n basePriority: 3\n },\n\n // Docker / DevOps\n {\n name: 'Docker Blog',\n url: 'https://www.docker.com/blog/feed/',\n category: 'containers',\n tags: ['docker', 'containers', 'devops'],\n priority: 'normal',\n basePriority: 2\n },\n {\n name: 'Kubernetes Blog',\n url: 'https://kubernetes.io/feed.xml',\n category: 'kubernetes',\n tags: ['kubernetes', 'containers', 'devops'],\n priority: 'normal',\n basePriority: 2\n },\n {\n name: 'GitHub Blog',\n url: 'https://github.blog/feed/',\n category: 'devtools',\n tags: ['github', 'programming', 'security', 'devops'],\n priority: 'normal',\n basePriority: 1\n },\n {\n name: 'The New Stack',\n url: 'https://thenewstack.io/feed/',\n category: 'devops',\n tags: ['cloud-native', 'devops', 'kubernetes'],\n priority: 'normal',\n basePriority: 1\n },\n\n // Reddit/community. These are noisy, so base priority is low and keyword scoring decides importance.\n {\n name: 'r/DataHoarder',\n url: 'https://www.reddit.com/r/DataHoarder/.rss',\n category: 'reddit',\n tags: ['storage', 'backup', 'datahoarder'],\n priority: 'normal',\n basePriority: 1\n },\n {\n name: 'r/LocalLLaMA',\n url: 'https://www.reddit.com/r/LocalLLaMA/.rss',\n category: 'reddit',\n tags: ['local-ai', 'llm', 'ollama'],\n priority: 'normal',\n basePriority: 1\n },\n {\n name: 'r/sysadmin',\n url: 'https://www.reddit.com/r/sysadmin/.rss',\n category: 'reddit',\n tags: ['sysadmin', 'security', 'infrastructure'],\n priority: 'normal',\n basePriority: 1\n }\n];\n\nreturn feeds.map(feed => ({\n json: {\n feedName: feed.name,\n name: feed.name,\n url: feed.url,\n category: feed.category,\n tags: feed.tags,\n priority: feed.priority,\n basePriority: feed.basePriority || 0\n }\n}));" + }, + "id": "1057fff2-07c6-48ed-97c9-353a36b5daf1", + "name": "Define RSS Feeds", + "type": "n8n-nodes-base.code", + "typeVersion": 2, + "position": [ + 272, + 128 + ] + }, + { + "parameters": { + "url": "={{ $json.url }}", + "sendHeaders": true, + "headerParameters": { + "parameters": [ + { + "name": "User-Agent", + "value": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:126.0) Gecko/20100101 Firefox/126.0 n8n-rss-reader/1.0" + }, + { + "name": "Accept", + "value": "application/rss+xml, application/atom+xml, application/xml, text/xml, */*" + } + ] + }, + "options": { + "response": { + "response": { + "responseFormat": "text" + } + }, + "timeout": 30000 + } + }, + "id": "4f5f0d61-10c8-42c5-b332-4df59baae50e", + "name": "Fetch RSS Feed", + "type": "n8n-nodes-base.httpRequest", + "typeVersion": 4.2, + "position": [ + 528, + 128 + ], + "continueOnFail": true + }, + { + "parameters": { + "jsCode": "const inputs = $input.all();\nconst sourceItems = $items('Define RSS Feeds');\n\nfunction decodeEntities(value = '') {\n const entities = {\n '&': '&',\n '"': '\"',\n ''': \"'\",\n ''': \"'\",\n '<': '<',\n '>': '>'\n };\n\n return String(value)\n .replace(/&(amp|quot|#39|apos|lt|gt);/g, m => entities[m] || m)\n .replace(/&#x([0-9a-fA-F]+);/g, (_, hex) => String.fromCharCode(parseInt(hex, 16)))\n .replace(/&#([0-9]+);/g, (_, num) => String.fromCharCode(parseInt(num, 10)));\n}\n\nfunction cleanText(value = '') {\n return decodeEntities(value)\n .replace(//g, '')\n .replace(//gi, '')\n .replace(//gi, '')\n .replace(/<[^>]*>/g, '')\n .replace(/\\s+/g, ' ')\n .trim();\n}\n\nfunction getTag(block, tag) {\n const escaped = tag.replace(/[.*+?^${}()|[\\]\\\\]/g, '\\\\$&');\n const regex = new RegExp(`<${escaped}[^>]*>([\\\\s\\\\S]*?)<\\\\/${escaped}>`, 'i');\n return cleanText((regex.exec(block) || [])[1] || '');\n}\n\nfunction getRawTag(block, tag) {\n const escaped = tag.replace(/[.*+?^${}()|[\\]\\\\]/g, '\\\\$&');\n const regex = new RegExp(`<${escaped}[^>]*>([\\\\s\\\\S]*?)<\\\\/${escaped}>`, 'i');\n return (regex.exec(block) || [])[1] || '';\n}\n\nfunction getAtomLink(block) {\n const href = /]+href=[\"']([^\"']+)[\"']/i.exec(block);\n return href ? href[1] : getTag(block, 'link');\n}\n\nfunction getFetchError(input) {\n if (!input) return '';\n if (typeof input.error === 'string') return input.error;\n if (input.error?.message) return input.error.message;\n if (input.error?.description) return input.error.description;\n if (input.errorMessage) return input.errorMessage;\n if (input.message && input.name) return `${input.name}: ${input.message}`;\n return '';\n}\n\nfunction parseFeed(xml) {\n const items = [];\n let match;\n\n // RSS format\n const rssRegex = /]*>([\\s\\S]*?)<\\/item>/gi;\n while ((match = rssRegex.exec(xml)) !== null && items.length < 2) {\n const block = match[1];\n const descRaw = getRawTag(block, 'description') || getRawTag(block, 'content:encoded') || '';\n\n items.push({\n title: getTag(block, 'title') || 'No title',\n description: cleanText(descRaw).substring(0, 280),\n link: getTag(block, 'link') || getTag(block, 'guid'),\n pubDate: getTag(block, 'pubDate') || getTag(block, 'dc:date')\n });\n }\n\n // Atom format\n if (items.length === 0) {\n const atomRegex = /]*>([\\s\\S]*?)<\\/entry>/gi;\n while ((match = atomRegex.exec(xml)) !== null && items.length < 2) {\n const block = match[1];\n const descRaw = getRawTag(block, 'summary') || getRawTag(block, 'content') || '';\n\n items.push({\n title: getTag(block, 'title') || 'No title',\n description: cleanText(descRaw).substring(0, 280),\n link: getAtomLink(block),\n pubDate: getTag(block, 'updated') || getTag(block, 'published')\n });\n }\n }\n\n return items;\n}\n\nreturn inputs.map((item, index) => {\n const input = item.json || {};\n\n // HTTP Request replaces the JSON item with the response body, so recover feed metadata\n // from the Define RSS Feeds node by item index.\n const source = sourceItems[index]?.json || {};\n\n const feedName = source.feedName || source.name || input.feedName || input.name || `Feed ${index + 1}`;\n const feedUrl = source.url || input.url || '';\n const category = source.category || input.category || '';\n const tags = source.tags || input.tags || [];\n const priority = source.priority || input.priority || 'normal';\n const basePriority = source.basePriority ?? input.basePriority ?? 0;\n\n const fetchError = getFetchError(input);\n if (fetchError) {\n return {\n json: {\n feedName,\n feedUrl,\n category,\n tags,\n priority,\n basePriority,\n itemCount: 0,\n skipped: true,\n fetchError,\n error: fetchError,\n digest: `Skipped ${feedName}: RSS fetch failed. ${fetchError}`,\n items: []\n }\n };\n }\n\n const xml =\n typeof input === 'string'\n ? input\n : input.data || input.body || input.response || input.xml || '';\n\n if (!xml || typeof xml !== 'string' || xml.length < 50) {\n return {\n json: {\n feedName,\n feedUrl,\n category,\n tags,\n priority,\n basePriority,\n itemCount: 0,\n skipped: true,\n error: `No usable XML received. Input keys: ${Object.keys(input).join(', ')}`,\n digest: `Skipped ${feedName}: no usable XML received.`,\n items: []\n }\n };\n }\n\n if (/^\\s* {\n return [\n `${i + 1}. ${feedItem.title || 'No title'}`,\n feedItem.description ? feedItem.description : '',\n feedItem.pubDate ? `Date: ${feedItem.pubDate}` : '',\n feedItem.link ? `Link: ${feedItem.link}` : ''\n ].filter(Boolean).join('\\n');\n }).join('\\n\\n');\n\n return {\n json: {\n feedName,\n feedUrl,\n category,\n tags,\n priority,\n basePriority,\n itemCount: items.length,\n skipped: items.length === 0,\n error: items.length === 0 ? `No feed items parsed. XML starts with: ${xml.substring(0, 300)}` : '',\n digest: digest || `No feed items found for ${feedName}.`,\n items\n }\n };\n});" + }, + "id": "63c99767-847b-447c-a361-766cd66b390d", + "name": "Parse RSS", + "type": "n8n-nodes-base.code", + "typeVersion": 2, + "position": [ + 784, + 128 + ] + }, + { + "parameters": { + "jsCode": "// Flatten feed items and create stable item hashes for deduplication.\n// Uses a local FNV-style hash so we do not need crypto/require in the n8n sandbox.\nfunction stableHash(input) {\n let h1 = 0x811c9dc5;\n let h2 = 0x01000193;\n\n for (let i = 0; i < input.length; i++) {\n const c = input.charCodeAt(i);\n h1 ^= c;\n h1 = Math.imul(h1, 0x01000193);\n h2 ^= c;\n h2 = Math.imul(h2, 0x811c9dc5);\n }\n\n const a = (h1 >>> 0).toString(16).padStart(8, '0');\n const b = (h2 >>> 0).toString(16).padStart(8, '0');\n return `${a}${b}`;\n}\n\nconst output = [];\n\nfor (const item of $input.all()) {\n const feed = item.json;\n\n // Do not send failed/empty feeds into Postgres. They can be reported separately later,\n // but they should not create Unknown Feed rows or null hashes.\n if (feed.fetchError || feed.skipped || !Array.isArray(feed.items) || feed.items.length === 0) {\n continue;\n }\n\n for (const rssItem of feed.items) {\n const rawKey = [\n feed.feedName || '',\n rssItem.title || '',\n rssItem.link || '',\n rssItem.pubDate || ''\n ].join('|');\n\n output.push({\n json: {\n feedName: feed.feedName || 'Unknown Feed',\n category: feed.category || '',\n tags: feed.tags || [],\n priority: feed.priority || 'normal',\n basePriority: feed.basePriority || 0,\n title: rssItem.title || '',\n description: rssItem.description || '',\n link: rssItem.link || '',\n pubDate: rssItem.pubDate || '',\n itemHash: stableHash(rawKey)\n }\n });\n }\n}\n\nreturn output;" + }, + "id": "0de4d305-b8c3-4d7b-a5fc-09202a7c5155", + "name": "Prepare Deduplication", + "type": "n8n-nodes-base.code", + "typeVersion": 2, + "position": [ + 1040, + 128 + ] + }, + { + "parameters": { + "operation": "executeQuery", + "query": "INSERT INTO rss_seen_items (\n item_hash,\n feed_name,\n title,\n link,\n pub_date\n)\nVALUES (\n '{{ $json.itemHash }}',\n '{{ ($json.feedName || '').replace(/'/g, \"''\") }}',\n '{{ ($json.title || '').replace(/'/g, \"''\") }}',\n '{{ ($json.link || '').replace(/'/g, \"''\") }}',\n '{{ ($json.pubDate || '').replace(/'/g, \"''\") }}'\n)\nON CONFLICT (item_hash) DO NOTHING\nRETURNING\n item_hash AS \"itemHash\",\n feed_name AS \"feedName\",\n title,\n link,\n pub_date AS \"pubDate\",\n '{{ ($json.description || '').replace(/'/g, \"''\") }}' AS description,\n '{{ ($json.category || '').replace(/'/g, \"''\") }}' AS category,\n '{{ JSON.stringify($json.tags || []).replace(/'/g, \"''\") }}' AS tags_json,\n '{{ ($json.priority || \"normal\").replace(/'/g, \"''\") }}' AS priority,\n {{ Number($json.basePriority || 0) }} AS \"basePriority\";", + "options": {} + }, + "id": "887e275c-3b16-44db-aed7-339e76aa2e0d", + "name": "Insert New Seen Items", + "type": "n8n-nodes-base.postgres", + "typeVersion": 2.5, + "position": [ + 1312, + 128 + ], + "credentials": { + "postgres": { + "id": "n9svoXemqSZoNNUB", + "name": "Postgres account" + } + }, + "continueOnFail": true + }, + { + "parameters": { + "jsCode": "// Group rows returned by INSERT ... RETURNING, then filter/score BEFORE AI.\n// This keeps the LLM workload small and prevents Gotify spam.\n// v8 scoring split:\n// - relevanceScore decides whether an item is worth summarizing.\n// - urgencyScore decides Gotify priority.\n// Normal interest terms like docker/container/ai should not create emergency notifications.\n\nconst grouped = {};\n\nfunction normalizeTags(value) {\n if (Array.isArray(value)) return value;\n try { return JSON.parse(value || '[]'); } catch (e) { return []; }\n}\n\nfunction escapeRegExp(s) {\n return String(s).replace(/[.*+?^${}()|[\\\\\\]\\\\]/g, '\\\\$&');\n}\n\nfunction hasTerm(text, term) {\n const t = String(term).toLowerCase();\n const pattern = t.includes(' ')\n ? `(^|[^a-z0-9])${escapeRegExp(t)}([^a-z0-9]|$)`\n : `\\\\b${escapeRegExp(t)}\\\\b`;\n return new RegExp(pattern, 'i').test(text);\n}\n\nconst scoringRules = [\n // urgent security: these should raise Gotify priority\n { term: 'cve', relevance: 10, urgency: 8 },\n { term: 'zero-day', relevance: 12, urgency: 12 },\n { term: '0day', relevance: 12, urgency: 12 },\n { term: 'actively exploited', relevance: 12, urgency: 12 },\n { term: 'authentication bypass', relevance: 10, urgency: 9 },\n { term: 'remote code execution', relevance: 10, urgency: 9 },\n { term: 'rce', relevance: 10, urgency: 9 },\n { term: 'critical vulnerability', relevance: 10, urgency: 9 },\n { term: 'exploit', relevance: 8, urgency: 7 },\n { term: 'privilege escalation', relevance: 8, urgency: 7 },\n { term: 'ransomware', relevance: 9, urgency: 8 },\n { term: 'data breach', relevance: 8, urgency: 7 },\n { term: 'breach', relevance: 6, urgency: 5 },\n { term: 'vulnerability', relevance: 6, urgency: 4 },\n { term: 'security advisory', relevance: 7, urgency: 5 },\n\n // urgent infrastructure/data-safety terms\n { term: 'data loss', relevance: 8, urgency: 8 },\n { term: 'corruption', relevance: 7, urgency: 7 },\n { term: 'outage', relevance: 6, urgency: 6 },\n { term: 'backup failure', relevance: 7, urgency: 6 },\n { term: 'restore failure', relevance: 7, urgency: 6 },\n\n // relevant homelab infrastructure, but not automatically urgent\n { term: 'zfs', relevance: 6, urgency: 1 },\n { term: 'truenas', relevance: 6, urgency: 1 },\n { term: 'backup', relevance: 5, urgency: 1 },\n { term: 'restore', relevance: 5, urgency: 1 },\n { term: 'proxmox', relevance: 5, urgency: 1 },\n { term: 'postgres', relevance: 5, urgency: 1 },\n { term: 'postgresql', relevance: 5, urgency: 1 },\n { term: 'tailscale', relevance: 4, urgency: 0 },\n { term: 'cloudflare', relevance: 3, urgency: 0 },\n { term: 'kubernetes', relevance: 4, urgency: 0 },\n { term: 'k3s', relevance: 4, urgency: 0 },\n { term: 'linux kernel', relevance: 4, urgency: 1 },\n { term: 'nfs', relevance: 4, urgency: 1 },\n { term: 'vlan', relevance: 3, urgency: 0 },\n { term: 'firewall', relevance: 4, urgency: 1 },\n { term: 'vpn', relevance: 3, urgency: 0 },\n\n // Docker/dev/AI interest terms. These make an item relevant, not urgent.\n { term: 'docker', relevance: 3, urgency: 0 },\n { term: 'container', relevance: 2, urgency: 0 },\n { term: 'containers', relevance: 2, urgency: 0 },\n { term: 'ollama', relevance: 4, urgency: 0 },\n { term: 'local llm', relevance: 4, urgency: 0 },\n { term: 'llm', relevance: 3, urgency: 0 },\n { term: 'openwebui', relevance: 4, urgency: 0 },\n { term: 'machine learning', relevance: 2, urgency: 0 },\n { term: 'python', relevance: 2, urgency: 0 },\n { term: 'github', relevance: 2, urgency: 0 },\n { term: 'authentication', relevance: 4, urgency: 1 },\n { term: 'oauth', relevance: 4, urgency: 1 },\n { term: 'ai', relevance: 1, urgency: 0 }\n];\n\nconst categoryBonus = {\n security: { relevance: 4, urgency: 2 },\n 'security-analysis': { relevance: 3, urgency: 1 },\n storage: { relevance: 2, urgency: 0 },\n virtualization: { relevance: 2, urgency: 0 },\n networking: { relevance: 2, urgency: 0 },\n containers: { relevance: 2, urgency: 0 },\n kubernetes: { relevance: 2, urgency: 0 },\n 'local-ai': { relevance: 2, urgency: 0 },\n ai: { relevance: 1, urgency: 0 },\n 'ai-dev': { relevance: 1, urgency: 0 },\n homelab: { relevance: 2, urgency: 0 },\n infrastructure: { relevance: 2, urgency: 0 },\n hardware: { relevance: 1, urgency: 0 },\n reddit: { relevance: 0, urgency: 0 }\n};\n\nfunction scoreItem(feed, rssItem) {\n const text = `${rssItem.title || ''}\\n${rssItem.description || ''}\\n${rssItem.link || ''}`.toLowerCase();\n const bonus = categoryBonus[feed.category] || { relevance: 0, urgency: 0 };\n\n let relevanceScore = Number(feed.basePriority || 0) + Number(bonus.relevance || 0);\n let urgencyScore = Number(bonus.urgency || 0);\n\n const matches = [];\n const urgentMatches = [];\n const weightedMatches = [];\n\n for (const rule of scoringRules) {\n if (hasTerm(text, rule.term)) {\n relevanceScore += Number(rule.relevance || 0);\n urgencyScore += Number(rule.urgency || 0);\n matches.push(rule.term);\n weightedMatches.push({ term: rule.term, relevance: rule.relevance || 0, urgency: rule.urgency || 0 });\n if (Number(rule.urgency || 0) >= 4) urgentMatches.push(rule.term);\n }\n }\n\n // High-priority/security feeds are inherently worth summarizing,\n // but they only become urgent when the actual item content has urgent terms.\n if (feed.priority === 'high' || feed.category === 'security') {\n relevanceScore += 3;\n matches.push('feed-priority-high');\n weightedMatches.push({ term: 'feed-priority-high', relevance: 3, urgency: 0 });\n }\n\n return { relevanceScore, urgencyScore, matches, urgentMatches, weightedMatches };\n}\n\nfor (const item of $input.all()) {\n const data = item.json || {};\n\n if (data.error) continue;\n if (!data.itemHash && !data.feedName && !data.feed_name && !data.title && !data.link) continue;\n\n const feedName = data.feedName || data.feed_name;\n if (!feedName) continue;\n\n if (!grouped[feedName]) {\n grouped[feedName] = {\n feedName,\n category: data.category || '',\n tags: normalizeTags(data.tags_json || data.tags),\n priority: data.priority || 'normal',\n basePriority: Number(data.basePriority || data.base_priority || 0),\n items: []\n };\n }\n\n grouped[feedName].items.push({\n title: data.title || '',\n description: data.description || '',\n link: data.link || '',\n pubDate: data.pubDate || data.pub_date || ''\n });\n}\n\nlet feeds = Object.values(grouped);\n\n// Filter and cap per feed before AI.\nfeeds = feeds.map(feed => {\n const scoredItems = feed.items.map(rssItem => {\n const scored = scoreItem(feed, rssItem);\n return {\n ...rssItem,\n itemScore: scored.relevanceScore,\n itemUrgencyScore: scored.urgencyScore,\n itemMatches: scored.matches,\n itemUrgentMatches: scored.urgentMatches,\n weightedMatches: scored.weightedMatches\n };\n });\n\n const alwaysKeep = feed.priority === 'high' || feed.category === 'security' || feed.category === 'security-analysis';\n const kept = scoredItems\n .filter(x => alwaysKeep || x.itemScore >= 3)\n .sort((a, b) => (b.itemUrgencyScore - a.itemUrgencyScore) || (b.itemScore - a.itemScore))\n .slice(0, 2);\n\n const relevanceScore = kept.reduce((sum, x) => sum + Number(x.itemScore || 0), 0);\n const urgencyScore = kept.reduce((sum, x) => sum + Number(x.itemUrgencyScore || 0), 0);\n const relevanceMatches = [...new Set(kept.flatMap(x => x.itemMatches || []))];\n const urgentMatches = [...new Set(kept.flatMap(x => x.itemUrgentMatches || []))];\n const weightedMatches = kept.flatMap(x => x.weightedMatches || []);\n\n return {\n ...feed,\n items: kept,\n itemCount: kept.length,\n relevanceScore,\n urgencyScore,\n relevanceMatches,\n highPriorityMatches: urgentMatches,\n weightedMatches\n };\n}).filter(feed => feed.items.length > 0);\n\nif (feeds.length === 0) {\n return [{\n json: {\n noNewItems: true,\n sent: false,\n message: 'No new relevant RSS items found. Deduplication and keyword filtering removed everything.',\n itemCount: 0,\n relevanceScore: 0,\n urgencyScore: 0,\n gotifyPriority: 0,\n highPriorityMatches: [],\n relevanceMatches: []\n }\n }];\n}\n\nreturn feeds.map(feed => {\n const digest = feed.items.map((rssItem, i) => {\n return `${i + 1}. ${rssItem.title || 'No title'}\nRelevance score: ${rssItem.itemScore || 0}\nUrgency score: ${rssItem.itemUrgencyScore || 0}\nMatches: ${(rssItem.itemMatches || []).join(', ') || 'none'}\nUrgent matches: ${(rssItem.itemUrgentMatches || []).join(', ') || 'none'}\n${rssItem.description || ''}\n${rssItem.link || ''}\n${rssItem.pubDate || ''}`;\n }).join('\\n\\n');\n\n return {\n json: {\n ...feed,\n noNewItems: false,\n digest\n }\n };\n});\n" + }, + "id": "091f4634-f130-4b91-8cb5-dd0d1ee85626", + "name": "Group New Items by Feed", + "type": "n8n-nodes-base.code", + "typeVersion": 2, + "position": [ + 1568, + 128 + ] + }, + { + "parameters": { + "jsCode": "const output = [];\n\nfor (const item of $input.all()) {\n const data = item.json;\n\n const body = {\n model: 'medium',\n messages: [\n {\n role: 'system',\n content: \"You are a news digest assistant for a homelab enthusiast and cybersecurity student.\\n\\nSummarize only the provided NEW and pre-filtered RSS items into a concise plain-text Gotify message.\\n\\nFormatting rules:\\n- Plain text only.\\n- Do not use Markdown.\\n- Do not use # headings.\\n- Do not use bold markers like **text**.\\n- Do not use markdown links like [title](url).\\n- Do not use tables.\\n- Use an uppercase title line.\\n- Use numbered entries like 1), 2).\\n- Put the item title on the numbered line.\\n- Put 1 short summary line under each title.\\n- Put raw URLs on their own line.\\n- Keep it easy to read on a phone notification screen.\\n- Keep the whole digest under 150 words.\\n\\nPrioritize self-hosting, Docker, AI/LLM, Linux, networking, cybersecurity, programming, TrueNAS, ZFS, storage, backups, virtualization, and homelab operations.\"\n },\n {\n role: 'user',\n content: `Feed: ${data.feedName}\nCategory: ${data.category || 'uncategorized'}\nTags: ${(data.tags || []).join(', ')}\nFeed relevance score: ${data.relevanceScore || 0}\nNew relevant item count: ${data.itemCount}\nMatched keywords: ${(data.highPriorityMatches || []).join(', ') || 'none'}\n\nLatest NEW relevant items:\n${data.digest}`\n }\n ],\n max_tokens: 280,\n temperature: 0.2\n };\n\n output.push({ json: { ...data, aiBody: body } });\n}\n\nreturn output;" + }, + "id": "38bcb515-60bd-48f8-a993-4f371a336598", + "name": "Prepare AI Request", + "type": "n8n-nodes-base.code", + "typeVersion": 2, + "position": [ + 1904, + 128 + ] + }, + { + "parameters": { + "method": "POST", + "url": "http://10.5.1.6:4000/v1/chat/completions", + "sendHeaders": true, + "headerParameters": { + "parameters": [ + { + "name": "Content-Type", + "value": "application/json" + }, + { + "name": "Authorization", + "value": "Bearer sk-REPLACE_WITH_LITELLM_KEY" + } + ] + }, + "sendBody": true, + "specifyBody": "json", + "jsonBody": "={{ JSON.stringify($json.aiBody) }}", + "options": { + "response": { + "response": { + "responseFormat": "json" + } + }, + "timeout": 120000 + } + }, + "id": "3395fab9-604a-45fb-88f5-8de90b782940", + "name": "AI Summarize Feed", + "type": "n8n-nodes-base.httpRequest", + "typeVersion": 4.2, + "position": [ + 2080, + 128 + ], + "continueOnFail": true + }, + { + "parameters": { + "jsCode": "const responses = $input.all();\nconst sourceItems = $items('Prepare AI Request');\n\nfunction stringifyError(value) {\n if (!value) return '';\n if (typeof value === 'string') return value;\n try { return JSON.stringify(value, null, 2); } catch (e) { return String(value); }\n}\n\nfunction plainTextForGotify(text) {\n if (!text) return '';\n\n return String(text)\n // Markdown headings -> plain uppercase-ish text without #\n .replace(/^\\s{0,3}#{1,6}\\s*/gm, '')\n // Bold/italic/code markers\n .replace(/\\*\\*([^*]+)\\*\\*/g, '$1')\n .replace(/__([^_]+)__/g, '$1')\n .replace(/`([^`]+)`/g, '$1')\n // Markdown links: [label](url) -> label\\nurl\n .replace(/\\[([^\\]]+)\\]\\((https?:\\/\\/[^\\s)]+)\\)/g, '$1\\n$2')\n // Markdown bullets -> simple indented dash for readability\n .replace(/^\\s*[-*+]\\s+/gm, ' - ')\n // Clean excessive blank lines\n .replace(/\\n{3,}/g, '\\n\\n')\n .trim();\n}\n\nreturn responses.map((item, index) => {\n const response = item.json || {};\n const source = sourceItems[index]?.json || {};\n\n let summary = '';\n let error = '';\n\n if (response.error || response.errorMessage) {\n error = response.errorMessage || stringifyError(response.error) || stringifyError(response);\n summary = `AI summary failed for ${source.feedName || `Feed ${index + 1}`}: ${error}`;\n } else if (response.choices?.[0]?.message?.content) {\n summary = response.choices[0].message.content;\n } else if (response.message || response.detail) {\n error = stringifyError(response.message || response.detail || response);\n summary = `AI summary failed for ${source.feedName || `Feed ${index + 1}`}: ${error}`;\n } else {\n error = stringifyError(response);\n summary = `AI summary unavailable for ${source.feedName || `Feed ${index + 1}`}. Raw response: ${error}`;\n }\n\n summary = plainTextForGotify(summary);\n\n if (summary.length > 3500) {\n summary = summary.substring(0, 3500) + '\\n\\n[truncated]';\n }\n\n return {\n json: {\n ...source,\n summary,\n error\n }\n };\n});" + }, + "id": "53ca1624-4fee-4025-883c-d38be407978e", + "name": "Format AI Summary", + "type": "n8n-nodes-base.code", + "typeVersion": 2, + "position": [ + 2352, + 128 + ] + }, + { + "parameters": { + "jsCode": "const output = [];\n\nfunction unique(arr) {\n return [...new Set((arr || []).filter(Boolean))];\n}\n\nfunction scoreToPriority(urgencyScore, relevanceScore) {\n const urgency = Number(urgencyScore || 0);\n const relevance = Number(relevanceScore || 0);\n\n // Only true urgent/security/data-safety matches should become loud alerts.\n if (urgency >= 18) return 8;\n if (urgency >= 12) return 7;\n if (urgency >= 7) return 5;\n\n // Interesting but non-urgent news stays normal/quiet.\n if (relevance >= 3) return 3;\n return 2;\n}\n\nfunction prefixForPriority(priority) {\n if (priority >= 7) return '🚨';\n if (priority >= 5) return '⚠️';\n if (priority <= 2) return '💤';\n return '📰';\n}\n\nfor (const item of $input.all()) {\n const data = item.json;\n\n const relevanceScore = Number(data.relevanceScore || 0);\n const urgencyScore = Number(data.urgencyScore || 0);\n const gotifyPriority = scoreToPriority(urgencyScore, relevanceScore);\n const titlePrefix = prefixForPriority(gotifyPriority);\n const urgentMatches = unique(data.highPriorityMatches || []);\n const relevanceMatches = unique(data.relevanceMatches || []);\n\n let message = data.summary || 'No summary returned.';\n\n if (gotifyPriority >= 5) {\n message = `${message}\\n\\nUrgency score: ${urgencyScore}\\nRelevance score: ${relevanceScore}\\nUrgent matched terms: ${urgentMatches.join(', ') || 'none'}`;\n }\n\n if (message.length > 3000) {\n message = message.substring(0, 3000) + '\\n\\n[truncated]';\n }\n\n output.push({\n json: {\n ...data,\n relevanceScore,\n urgencyScore,\n gotifyPriority,\n highPriorityMatches: urgentMatches,\n relevanceMatches,\n gotifyBody: {\n title: `${titlePrefix} ${data.feedName} Digest`,\n message,\n priority: gotifyPriority,\n extras: {\n 'client::display': {\n contentType: 'text/plain'\n }\n }\n }\n }\n });\n}\n\nreturn output;\n" + }, + "id": "4513e852-d740-41ab-b4ce-6547cdbeead9", + "name": "Prepare Gotify Push", + "type": "n8n-nodes-base.code", + "typeVersion": 2, + "position": [ + 2608, + 128 + ] + }, + { + "parameters": { + "message": "={{ $json.gotifyBody.message }}", + "additionalFields": { + "priority": "={{ $json.gotifyBody.priority }}", + "title": "={{ $json.gotifyBody.title }}" + }, + "options": { + "contentType": "text/plain" + } + }, + "id": "4946e30e-9332-4ee2-915c-cd641bdefb0f", + "name": "Create Gotify Message", + "type": "n8n-nodes-base.gotify", + "typeVersion": 1, + "position": [ + 2864, + 128 + ], + "credentials": { + "gotifyApi": { + "id": "ExUx3oqYQyhkqZcp", + "name": "Rss Feed" + } + }, + "continueOnFail": true + }, + { + "parameters": { + "jsCode": "const responses = $input.all();\nconst sourceItems = $items('Prepare Gotify Push');\n\nconst output = responses.map((item, index) => {\n const response = item.json || {};\n const source = sourceItems[index]?.json || {};\n const error = response.errorMessage || response.error || source.error || '';\n\n return {\n json: {\n feedName: source.feedName || `Feed ${index + 1}`,\n itemCount: source.itemCount || 0,\n sent: !error,\n gotifyResponse: error ? null : response,\n gotifyPriority: source.gotifyPriority,\n relevanceScore: source.relevanceScore || 0,\n urgencyScore: source.urgencyScore || 0,\n highPriorityMatches: source.highPriorityMatches || [],\n relevanceMatches: source.relevanceMatches || [],\n error: typeof error === 'string' ? error : JSON.stringify(error),\n summary: source.summary || ''\n }\n };\n});\n\nif (output.length === 0) {\n return [{\n json: {\n sent: false,\n message: 'No new RSS items found. Deduplication filtered everything out.'\n }\n }];\n}\n\nreturn output;\n" + }, + "id": "c1da7901-98c0-459f-84ee-98ed30675bc5", + "name": "Final Response", + "type": "n8n-nodes-base.code", + "typeVersion": 2, + "position": [ + 3120, + 128 + ] + }, + { + "parameters": { + "conditions": { + "options": { + "caseSensitive": true, + "leftValue": "", + "typeValidation": "strict", + "version": 2 + }, + "conditions": [ + { + "id": "ba2925fb-2bad-4971-a5ac-d610b91a9d40", + "leftValue": "={{ $json.noNewItems }}", + "rightValue": true, + "operator": { + "type": "boolean", + "operation": "false", + "singleValue": true + } + } + ], + "combinator": "and" + }, + "options": {} + }, + "id": "3fe3c568-d143-425e-9e8d-d463c38a8f41", + "name": "Has New Items?", + "type": "n8n-nodes-base.if", + "typeVersion": 2.2, + "position": [ + 1728, + 144 + ] + }, + { + "parameters": { + "jsCode": "return [{\n json: {\n sent: false,\n itemCount: 0,\n message: $json.message || 'No new RSS items found. Deduplication filtered everything out.',\n highPriorityMatches: [],\n error: ''\n }\n}];" + }, + "id": "0b8f121c-4339-49a9-bbc9-622fafe60403", + "name": "No New Items Response", + "type": "n8n-nodes-base.code", + "typeVersion": 2, + "position": [ + 1888, + 336 + ] + }, + { + "parameters": {}, + "type": "n8n-nodes-base.manualTrigger", + "typeVersion": 1, + "position": [ + 144, + 496 + ], + "id": "9cfd7a45-912a-4895-8a04-23411a893a09", + "name": "When clicking ‘Execute workflow’" + } + ], + "connections": { + "Every 6 Hours": { + "main": [ + [ + { + "node": "Define RSS Feeds", + "type": "main", + "index": 0 + } + ] + ] + }, + "Manual Trigger Webhook": { + "main": [ + [ + { + "node": "Define RSS Feeds", + "type": "main", + "index": 0 + } + ] + ] + }, + "Define RSS Feeds": { + "main": [ + [ + { + "node": "Fetch RSS Feed", + "type": "main", + "index": 0 + } + ] + ] + }, + "Fetch RSS Feed": { + "main": [ + [ + { + "node": "Parse RSS", + "type": "main", + "index": 0 + } + ] + ] + }, + "Parse RSS": { + "main": [ + [ + { + "node": "Prepare Deduplication", + "type": "main", + "index": 0 + } + ] + ] + }, + "Prepare Deduplication": { + "main": [ + [ + { + "node": "Insert New Seen Items", + "type": "main", + "index": 0 + } + ] + ] + }, + "Insert New Seen Items": { + "main": [ + [ + { + "node": "Group New Items by Feed", + "type": "main", + "index": 0 + } + ] + ] + }, + "Group New Items by Feed": { + "main": [ + [ + { + "node": "Has New Items?", + "type": "main", + "index": 0 + } + ] + ] + }, + "Prepare AI Request": { + "main": [ + [ + { + "node": "AI Summarize Feed", + "type": "main", + "index": 0 + } + ] + ] + }, + "AI Summarize Feed": { + "main": [ + [ + { + "node": "Format AI Summary", + "type": "main", + "index": 0 + } + ] + ] + }, + "Format AI Summary": { + "main": [ + [ + { + "node": "Prepare Gotify Push", + "type": "main", + "index": 0 + } + ] + ] + }, + "Prepare Gotify Push": { + "main": [ + [ + { + "node": "Create Gotify Message", + "type": "main", + "index": 0 + } + ] + ] + }, + "Has New Items?": { + "main": [ + [ + { + "node": "Prepare AI Request", + "type": "main", + "index": 0 + } + ], + [ + { + "node": "No New Items Response", + "type": "main", + "index": 0 + } + ] + ] + }, + "Create Gotify Message": { + "main": [ + [ + { + "node": "Final Response", + "type": "main", + "index": 0 + } + ] + ] + }, + "When clicking ‘Execute workflow’": { + "main": [ + [ + { + "node": "Define RSS Feeds", + "type": "main", + "index": 0 + } + ] + ] + } + }, + "settings": { + "executionOrder": "v1", + "binaryMode": "separate" + }, + "staticData": { + "node:Every 6 Hours": { + "recurrenceRules": [ + 12 + ] + } + }, + "meta": { + "templateCredsSetupCompleted": true + }, + "pinData": {}, + "versionId": "7476f4df-daf7-4b00-8f20-8397a10a7bcf", + "activeVersionId": "7476f4df-daf7-4b00-8f20-8397a10a7bcf", + "versionCounter": 30, + "triggerCount": 2, + "tags": [ + { + "updatedAt": "2026-05-07T02:27:25.820Z", + "createdAt": "2026-05-07T02:27:25.820Z", + "id": "Vx2eghtcMKzEcaTy", + "name": "dedupe" + }, + { + "updatedAt": "2026-05-07T00:49:03.692Z", + "createdAt": "2026-05-07T00:49:03.692Z", + "id": "Uj6bEYJrGolubkKM", + "name": "rss" + }, + { + "updatedAt": "2026-05-06T22:38:51.678Z", + "createdAt": "2026-05-06T22:38:51.678Z", + "id": "S9FxzVfHLsHmyzyt", + "name": "ai" + }, + { + "updatedAt": "2026-05-07T04:01:31.904Z", + "createdAt": "2026-05-07T04:01:31.904Z", + "id": "RnvhD9aHKN7v7iOi", + "name": "priority-tuned" + }, + { + "updatedAt": "2026-05-07T02:27:25.824Z", + "createdAt": "2026-05-07T02:27:25.824Z", + "id": "MfaTegnw8p0SsGGm", + "name": "postgres" + }, + { + "updatedAt": "2026-05-07T00:49:03.703Z", + "createdAt": "2026-05-07T00:49:03.703Z", + "id": "3XoednfZ00DRBn07", + "name": "news" + } + ], + "shared": [ + { + "updatedAt": "2026-05-07T04:01:33.583Z", + "createdAt": "2026-05-07T04:01:33.583Z", + "role": "workflow:owner", + "workflowId": "40wcTHzFz6GSFlJX", + "projectId": "dxCRnBdX5uJizCGa", + "project": { + "updatedAt": "2026-05-06T01:10:37.484Z", + "createdAt": "2026-05-06T01:08:07.721Z", + "id": "dxCRnBdX5uJizCGa", + "name": "Wilfred Fizzlepoof ", + "type": "personal", + "icon": null, + "description": null, + "creatorId": "47b2227f-2fc2-4a08-bd35-ea157b92df0d" + } + } + ], + "versionMetadata": { + "name": "Version 7476f4df", + "description": "" + } +} \ No newline at end of file diff --git a/n8n-workflows/07-git-commit-summarizer.json b/n8n-workflows/07-git-commit-summarizer.json new file mode 100644 index 0000000..c25387f --- /dev/null +++ b/n8n-workflows/07-git-commit-summarizer.json @@ -0,0 +1,210 @@ +{ + "updatedAt": "2026-05-07T00:49:43.226Z", + "createdAt": "2026-05-07T00:49:43.226Z", + "id": "CFYWbBRx9RPf4HjS", + "name": "Git Commit \u2192 AI Summary \u2192 Gotify", + "description": null, + "active": true, + "isArchived": false, + "nodes": [ + { + "parameters": { + "httpMethod": "POST", + "path": "git/push", + "options": {} + }, + "id": "fe377ac9-adcf-427d-a096-679f5469f8d8", + "name": "Git Push Webhook", + "type": "n8n-nodes-base.webhook", + "typeVersion": 2, + "position": [ + 0, + 0 + ], + "webhookId": "d1d31942-6fe8-48a2-aab7-7eefaf800420" + }, + { + "parameters": { + "jsCode": "const data = $input.first().json;\nconst body = data.body || data;\n\n// Works with Gitea, Forgejo, GitHub, and GitLab webhook payloads\nconst repo = body.repository?.full_name || body.repository?.name || body.project?.path_with_namespace || 'unknown-repo';\nconst branch = (body.ref || '').replace('refs/heads/', '');\nconst pusher = body.pusher?.name || body.pusher?.login || body.user_name || body.sender?.login || 'unknown';\n\n// Extract commits\nconst commits = (body.commits || []).map(c => ({\n id: (c.id || c.sha || '').substring(0, 7),\n message: (c.message || '').split('\\n')[0],\n author: c.author?.name || c.author?.username || 'unknown',\n added: (c.added || []).length,\n modified: (c.modified || []).length,\n removed: (c.removed || []).length\n}));\n\nconst commitSummary = commits.map(c =>\n `${c.id}: ${c.message} (by ${c.author}) [+${c.added} ~${c.modified} -${c.removed}]`\n).join('\\n');\n\nreturn [{\n json: {\n repo,\n branch,\n pusher,\n commitCount: commits.length,\n commitSummary,\n compareUrl: body.compare_url || body.compare || ''\n }\n}];" + }, + "id": "8dda4f14-9563-490f-89b3-aef80f64ba0b", + "name": "Parse Git Payload", + "type": "n8n-nodes-base.code", + "typeVersion": 2, + "position": [ + 224, + 0 + ] + }, + { + "parameters": { + "method": "POST", + "url": "http://10.5.1.6:4000/v1/chat/completions", + "sendHeaders": true, + "headerParameters": { + "parameters": [ + { + "name": "Content-Type", + "value": "application/json" + }, + { + "name": "Authorization", + "value": "Bearer sk-REPLACE_WITH_LITELLM_KEY" + } + ] + }, + "sendBody": true, + "specifyBody": "json", + "jsonBody": "={\n \"model\": \"light\",\n \"messages\": [\n {\n \"role\": \"system\",\n \"content\": \"You are a git commit summarizer. Given a list of commits pushed to a repo, write a 1-2 sentence summary of what changed. Be concise and technical. Focus on the purpose of the changes, not just listing files.\"\n },\n {\n \"role\": \"user\",\n \"content\": \"Repository: {{ $json.repo }}\\nBranch: {{ $json.branch }}\\nPushed by: {{ $json.pusher }}\\n\\nCommits ({{ $json.commitCount }}):\\n{{ $json.commitSummary }}\"\n }\n ],\n \"max_tokens\": 150,\n \"temperature\": 0.2\n}", + "options": {} + }, + "id": "c6b2b71b-2612-4ef7-8561-8adccbfa9be8", + "name": "AI Summarize Commits", + "type": "n8n-nodes-base.httpRequest", + "typeVersion": 4.2, + "position": [ + 448, + 0 + ] + }, + { + "parameters": { + "jsCode": "const response = $input.first().json;\nconst gitData = $('Parse Git Payload').first().json;\nconst summary = response.choices?.[0]?.message?.content || 'Could not summarize';\n\nconst title = `\ud83d\udd00 ${gitData.repo}:${gitData.branch} (${gitData.commitCount} commit${gitData.commitCount !== 1 ? 's' : ''})`;\n\nreturn [{\n json: {\n title,\n message: `${summary}\\n\\nPushed by: ${gitData.pusher}`,\n priority: 3\n }\n}];" + }, + "id": "94f5c120-0ae6-4a23-9556-d4f564f9f967", + "name": "Format Notification", + "type": "n8n-nodes-base.code", + "typeVersion": 2, + "position": [ + 672, + 0 + ] + }, + { + "parameters": { + "message": "={{ $json.message }}", + "additionalFields": { + "priority": "={{ $json.priority }}", + "title": "={{ $json.title }}" + }, + "options": { + "contentType": "text/plain" + } + }, + "id": "4f731683-574c-4a39-84fe-025280193ba1", + "name": "Push to Gotify", + "type": "n8n-nodes-base.gotify", + "typeVersion": 1, + "position": [ + 880, + 0 + ], + "credentials": { + "gotifyApi": { + "id": "SETUP_REQUIRED", + "name": "Git Commits" + } + } + } + ], + "connections": { + "Git Push Webhook": { + "main": [ + [ + { + "node": "Parse Git Payload", + "type": "main", + "index": 0 + } + ] + ] + }, + "Parse Git Payload": { + "main": [ + [ + { + "node": "AI Summarize Commits", + "type": "main", + "index": 0 + } + ] + ] + }, + "AI Summarize Commits": { + "main": [ + [ + { + "node": "Format Notification", + "type": "main", + "index": 0 + } + ] + ] + }, + "Format Notification": { + "main": [ + [ + { + "node": "Push to Gotify", + "type": "main", + "index": 0 + } + ] + ] + } + }, + "settings": { + "executionOrder": "v1", + "binaryMode": "separate" + }, + "staticData": null, + "meta": null, + "pinData": {}, + "versionId": "61fca126-b5dd-4425-b0ee-c621d3ed58af", + "activeVersionId": "61fca126-b5dd-4425-b0ee-c621d3ed58af", + "versionCounter": 12, + "triggerCount": 1, + "tags": [ + { + "updatedAt": "2026-05-07T00:49:41.640Z", + "createdAt": "2026-05-07T00:49:41.640Z", + "id": "36CVfQVO41QQyOmh", + "name": "notifications" + }, + { + "updatedAt": "2026-05-07T00:49:41.638Z", + "createdAt": "2026-05-07T00:49:41.638Z", + "id": "2oumYE52ygvhrieZ", + "name": "git" + }, + { + "updatedAt": "2026-05-06T22:38:51.678Z", + "createdAt": "2026-05-06T22:38:51.678Z", + "id": "S9FxzVfHLsHmyzyt", + "name": "ai" + } + ], + "shared": [ + { + "updatedAt": "2026-05-07T00:49:43.226Z", + "createdAt": "2026-05-07T00:49:43.226Z", + "role": "workflow:owner", + "workflowId": "CFYWbBRx9RPf4HjS", + "projectId": "dxCRnBdX5uJizCGa", + "project": { + "updatedAt": "2026-05-06T01:10:37.484Z", + "createdAt": "2026-05-06T01:08:07.721Z", + "id": "dxCRnBdX5uJizCGa", + "name": "Wilfred Fizzlepoof ", + "type": "personal", + "icon": null, + "description": null, + "creatorId": "47b2227f-2fc2-4a08-bd35-ea157b92df0d" + } + } + ], + "versionMetadata": { + "name": "Version 61fca126", + "description": "" + } +} \ No newline at end of file diff --git a/n8n-workflows/08-class-recording-rag.json b/n8n-workflows/08-class-recording-rag.json new file mode 100644 index 0000000..9b3c6f6 --- /dev/null +++ b/n8n-workflows/08-class-recording-rag.json @@ -0,0 +1,532 @@ +{ + "updatedAt": "2026-05-07T01:16:50.340Z", + "createdAt": "2026-05-07T01:16:50.340Z", + "id": "91I278E72FAaQenD", + "name": "Class Recording \u2192 Transcribe \u2192 RAG", + "description": null, + "active": true, + "isArchived": false, + "nodes": [ + { + "parameters": { + "httpMethod": "POST", + "path": "class/upload", + "responseMode": "lastNode", + "responseData": "allEntries", + "options": { + "rawBody": true + } + }, + "id": "e475f17b-8560-4376-852e-a613ba6b300e", + "name": "Class Upload Webhook", + "type": "n8n-nodes-base.webhook", + "typeVersion": 2, + "position": [ + 0, + 0 + ], + "webhookId": "d0ee8226-db56-477e-8a3f-572a9129aea9" + }, + { + "parameters": { + "jsCode": "const data = $input.first().json;\nconst body = data.body || data;\n\n// Class name comes from:\n// 1. Form field 'class_name' in multipart upload\n// 2. Query parameter ?class=CS101\n// 3. Header X-Class-Name\nconst className = body.class_name || data.headers?.['x-class-name'] || data.query?.class || 'unclassified';\n\n// Lecture title from:\n// 1. Form field 'lecture_title'\n// 2. Query param ?title=...\n// 3. Header X-Lecture-Title\n// 4. Form field 'filename' (original filename)\nconst rawTitle = body.lecture_title || data.headers?.['x-lecture-title'] || data.query?.title || body.filename || 'Untitled Lecture';\nconst lectureTitle = rawTitle.replace(/\\.wav$/i, '').replace(/[-_]/g, ' ');\n\nconst classSlug = className.toLowerCase().replace(/[^a-z0-9]+/g, '-').replace(/^-|-$/g, '');\n\nreturn [{\n json: {\n className,\n classSlug,\n lectureTitle,\n uploadedAt: new Date().toISOString()\n }\n}];" + }, + "id": "24e4a726-ab50-4767-a144-1751616c781d", + "name": "Extract Class Info", + "type": "n8n-nodes-base.code", + "typeVersion": 2, + "position": [ + 224, + 0 + ] + }, + { + "parameters": { + "method": "POST", + "url": "http://10.5.1.6:8786/v1/audio/transcriptions", + "sendHeaders": true, + "headerParameters": { + "parameters": [ + { + "name": "Accept", + "value": "application/json" + } + ] + }, + "sendBody": true, + "contentType": "multipart-form-data", + "bodyParameters": { + "parameters": [ + { + "parameterType": "formBinaryData", + "name": "file", + "inputDataFieldName": "data" + }, + { + "name": "model", + "value": "Systran/faster-whisper-large-v3" + }, + { + "name": "language", + "value": "en" + }, + { + "name": "response_format", + "value": "verbose_json" + } + ] + }, + "options": { + "timeout": 600000 + } + }, + "id": "fcd1062d-8f60-4214-85ee-f53b914abcbb", + "name": "Whisper Transcribe", + "type": "n8n-nodes-base.httpRequest", + "typeVersion": 4.2, + "position": [ + 448, + 0 + ] + }, + { + "parameters": { + "jsCode": "const result = $input.first().json;\nconst classInfo = $('Extract Class Info').first().json;\n\nconst segments = result.segments || [];\nconst fullText = result.text || segments.map(s => s.text).join(' ');\nconst duration = result.duration || segments[segments.length - 1]?.end || 0;\n\nconst timestamped = segments.map(s => {\n const mins = Math.floor(s.start / 60);\n const secs = Math.floor(s.start % 60);\n return `[${mins}:${String(secs).padStart(2, '0')}] ${s.text.trim()}`;\n}).join('\\n');\n\nreturn [{\n json: {\n className: classInfo.className,\n classSlug: classInfo.classSlug,\n lectureTitle: classInfo.lectureTitle,\n fullText: fullText.trim(),\n timestampedText: timestamped,\n durationMinutes: Math.round(duration / 60),\n segmentCount: segments.length\n }\n}];" + }, + "id": "ecf09ad2-327e-4a82-90f3-95da5590e0ac", + "name": "Format Transcript", + "type": "n8n-nodes-base.code", + "typeVersion": 2, + "position": [ + 672, + 0 + ] + }, + { + "parameters": { + "method": "POST", + "url": "http://10.5.1.6:4000/v1/chat/completions", + "sendHeaders": true, + "headerParameters": { + "parameters": [ + { + "name": "Content-Type", + "value": "application/json" + }, + { + "name": "Authorization", + "value": "Bearer sk-REPLACE_WITH_LITELLM_KEY" + } + ] + }, + "sendBody": true, + "specifyBody": "json", + "jsonBody": "={\n \"model\": \"medium\",\n \"messages\": [\n {\n \"role\": \"system\",\n \"content\": \"You are an academic note-taking assistant. Given a class lecture transcription, provide:\\n1. A concise summary (3-5 sentences)\\n2. Key concepts covered (as a list)\\n3. Important definitions or formulas\\n4. Any assignments or deadlines mentioned\\n\\nRespond in JSON with keys: summary, key_concepts (array), definitions (array of {term, definition}), assignments (array)\"\n },\n {\n \"role\": \"user\",\n \"content\": \"Class: {{ $json.className }}\\nLecture: {{ $json.lectureTitle }}\\n\\nTranscription (first 3000 chars):\\n{{ $json.fullText.substring(0, 3000) }}\"\n }\n ],\n \"max_tokens\": 600,\n \"temperature\": 0.2,\n \"response_format\": { \"type\": \"json_object\" }\n}", + "options": {} + }, + "id": "e7407d06-f7f4-4105-a817-055d4703d59d", + "name": "AI Extract Notes", + "type": "n8n-nodes-base.httpRequest", + "typeVersion": 4.2, + "position": [ + 880, + 0 + ] + }, + { + "parameters": { + "jsCode": "const llmResponse = $input.first().json;\nconst transcriptData = $('Format Transcript').first().json;\n\nlet notes;\ntry {\n notes = JSON.parse(llmResponse.choices?.[0]?.message?.content || '{}');\n} catch(e) {\n notes = { summary: 'Could not parse notes', key_concepts: [], definitions: [], assignments: [] };\n}\n\nconst ragText = [\n `Class: ${transcriptData.className}`,\n `Lecture: ${transcriptData.lectureTitle}`,\n `Duration: ${transcriptData.durationMinutes} minutes`,\n `Summary: ${notes.summary}`,\n `Key Concepts: ${(notes.key_concepts || []).join(', ')}`,\n '',\n 'Full Transcript:',\n transcriptData.fullText\n].join('\\n');\n\nreturn [{\n json: {\n className: transcriptData.className,\n classSlug: transcriptData.classSlug,\n lectureTitle: transcriptData.lectureTitle,\n durationMinutes: transcriptData.durationMinutes,\n notes,\n ragText,\n source: `class-${transcriptData.classSlug}-${Date.now()}`,\n metadata: {\n class_name: transcriptData.className,\n class_slug: transcriptData.classSlug,\n lecture_title: transcriptData.lectureTitle,\n duration_minutes: transcriptData.durationMinutes,\n type: 'class_recording',\n recorded_at: new Date().toISOString()\n }\n }\n}];" + }, + "id": "71b0807d-03ba-45b2-b75b-b6fbeee5687e", + "name": "Prepare for RAG", + "type": "n8n-nodes-base.code", + "typeVersion": 2, + "position": [ + 1104, + 0 + ] + }, + { + "parameters": { + "jsCode": "const input = $input.first().json;\nconst text = input.ragText || '';\nconst source = input.source;\nconst metadata = input.metadata;\n\nconst CHUNK_SIZE = 500;\nconst OVERLAP = 50;\n\nconst chunks = [];\nconst sentences = text.split(/(?<=[.!?])\\s+/);\nlet current = '';\n\nfor (const sentence of sentences) {\n if ((current + ' ' + sentence).length > CHUNK_SIZE && current.length > 0) {\n chunks.push(current.trim());\n const words = current.split(' ');\n current = words.slice(-OVERLAP).join(' ') + ' ' + sentence;\n } else {\n current = current ? current + ' ' + sentence : sentence;\n }\n}\nif (current.trim()) chunks.push(current.trim());\n\nreturn chunks.map((chunk, i) => ({\n json: {\n chunk,\n chunkIndex: i,\n totalChunks: chunks.length,\n source,\n metadata\n }\n}));" + }, + "id": "9e4bbe70-72b0-4955-848f-0d1b62ae94a4", + "name": "Chunk Text", + "type": "n8n-nodes-base.code", + "typeVersion": 2, + "position": [ + 1328, + 0 + ] + }, + { + "parameters": { + "method": "POST", + "url": "http://10.5.1.6:11434/v1/embeddings", + "sendHeaders": true, + "headerParameters": { + "parameters": [ + { + "name": "Content-Type", + "value": "application/json" + } + ] + }, + "sendBody": true, + "specifyBody": "json", + "jsonBody": "={\n \"model\": \"nomic-embed-text\",\n \"input\": \"{{ $json.chunk }}\"\n}", + "options": { + "batching": { + "batch": { + "batchSize": 5, + "batchInterval": 500 + } + } + } + }, + "id": "4ae1ab11-c76f-4478-84e0-7bcdfb20b83b", + "name": "Get Embedding", + "type": "n8n-nodes-base.httpRequest", + "typeVersion": 4.2, + "position": [ + 1552, + 0 + ] + }, + { + "parameters": { + "jsCode": "const embeddingResponse = $input.first().json;\nconst chunkData = $('Chunk Text').item;\nconst embedding = embeddingResponse.data?.[0]?.embedding || [];\n\nconst pointId = 'xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx'.replace(/[xy]/g, c => {\n const r = Math.random() * 16 | 0;\n return (c === 'x' ? r : (r & 0x3 | 0x8)).toString(16);\n});\n\nreturn [{\n json: {\n id: pointId,\n vector: embedding,\n payload: {\n text: chunkData.json.chunk,\n source: chunkData.json.source,\n chunkIndex: chunkData.json.chunkIndex,\n totalChunks: chunkData.json.totalChunks,\n metadata: chunkData.json.metadata,\n ingested_at: new Date().toISOString()\n }\n }\n}];" + }, + "id": "d4232b02-23fc-4cbd-ac2d-2fdbfbe9583a", + "name": "Prepare Qdrant Point", + "type": "n8n-nodes-base.code", + "typeVersion": 2, + "position": [ + 1760, + 0 + ] + }, + { + "parameters": { + "method": "PUT", + "url": "http://10.5.1.6:6333/collections/knowledge_base/points", + "sendHeaders": true, + "headerParameters": { + "parameters": [ + { + "name": "Content-Type", + "value": "application/json" + } + ] + }, + "sendBody": true, + "specifyBody": "json", + "jsonBody": "={\n \"points\": [\n {\n \"id\": \"{{ $json.id }}\",\n \"vector\": {{ JSON.stringify($json.vector) }},\n \"payload\": {{ JSON.stringify($json.payload) }}\n }\n ]\n}", + "options": {} + }, + "id": "845a2aa0-2546-431d-96e8-b36875d1fe7d", + "name": "Upsert to Qdrant", + "type": "n8n-nodes-base.httpRequest", + "typeVersion": 4.2, + "position": [ + 1984, + 0 + ] + }, + { + "parameters": { + "jsCode": "const rag = $('Prepare for RAG').first().json;\nreturn [{\n json: {\n title: `\ud83c\udf93 ${rag.className} Lecture Processed`,\n message: `${rag.lectureTitle} (${rag.durationMinutes}min)\\nSummary: ${rag.notes.summary}\\nKey concepts: ${(rag.notes.key_concepts || []).slice(0, 5).join(', ')}`,\n priority: 5\n }\n}];" + }, + "id": "fmt08-0001-0000-0000-000000000008", + "name": "Format Gotify Message", + "type": "n8n-nodes-base.code", + "typeVersion": 2, + "position": [ + 1968, + 0 + ] + }, + { + "parameters": { + "message": "={{ $json.message }}", + "additionalFields": { + "priority": "={{ $json.priority }}", + "title": "={{ $json.title }}" + }, + "options": { + "contentType": "text/plain" + } + }, + "id": "5b31b660-c414-40d1-82a5-d68511203133", + "name": "Notify via Gotify", + "type": "n8n-nodes-base.gotify", + "typeVersion": 1, + "position": [ + 2208, + 0 + ], + "credentials": { + "gotifyApi": { + "id": "SETUP_REQUIRED", + "name": "Class Recordings" + } + } + }, + { + "parameters": { + "jsCode": "const rag = $('Prepare for RAG').first().json;\nconst notes = rag.notes || {};\nconst concepts = (notes.key_concepts || []).map((c,i) => `${i+1}. ${c}`).join('\\n');\nconst defs = Object.entries(notes.definitions || {}).map(([k,v]) => `${k}: ${v}`).join('\\n');\nconst assignments = (notes.assignments || []).map((a,i) => `${i+1}. ${a}`).join('\\n');\nconst content = [\n `Lecture Notes: ${rag.lectureTitle}`,\n `Class: ${rag.className}`,\n `Duration: ${rag.durationMinutes} minutes`,\n `Date: ${new Date().toISOString()}`, '',\n `Summary`, `=======`, notes.summary || '', '',\n `Key Concepts`, `=============`, concepts,\n ...(defs ? ['', `Definitions`, `===========`, defs] : []),\n ...(assignments ? ['', `Assignments`, `===========`, assignments] : []),\n].join('\\n');\nconst title = `${rag.className} \u2014 ${rag.lectureTitle}`;\nreturn [{ binary: { document: { data: Buffer.from(content,'utf-8').toString('base64'), mimeType:'text/plain', fileName:`${title}.txt` }}, json: { title, className: rag.className } }];\n" + }, + "id": "95aea3a7-3b15-4196-9e4a-dc8f6ed8d49e", + "name": "Prepare Notes Doc", + "type": "n8n-nodes-base.code", + "typeVersion": 2, + "position": [ + 2208, + -200 + ] + }, + { + "parameters": { + "method": "POST", + "url": "https://paperless.paccoco.com/api/documents/post_document/", + "options": {}, + "sendHeaders": true, + "headerParameters": { + "parameters": [ + { + "name": "Authorization", + "value": "Bearer ${PAPERLESS_API_TOKEN}" + } + ] + }, + "sendBody": true, + "contentType": "multipart-form-data", + "bodyParameters": { + "parameters": [ + { + "name": "document", + "value": "data", + "parameterType": "formBinaryData" + }, + { + "name": "title", + "value": "={{ $json.title }}" + }, + { + "name": "tags[]", + "value": "${PAPERLESS_TAG_LECTURE_NOTES}" + } + ] + } + }, + "id": "c902a153-ab26-4518-af2a-585f0b72fe6c", + "name": "Save Notes to Paperless", + "type": "n8n-nodes-base.httpRequest", + "typeVersion": 4.2, + "position": [ + 2432, + -200 + ], + "continueOnFail": true + } + ], + "connections": { + "Class Upload Webhook": { + "main": [ + [ + { + "node": "Extract Class Info", + "type": "main", + "index": 0 + } + ] + ] + }, + "Extract Class Info": { + "main": [ + [ + { + "node": "Whisper Transcribe", + "type": "main", + "index": 0 + } + ] + ] + }, + "Whisper Transcribe": { + "main": [ + [ + { + "node": "Format Transcript", + "type": "main", + "index": 0 + } + ] + ] + }, + "Format Transcript": { + "main": [ + [ + { + "node": "AI Extract Notes", + "type": "main", + "index": 0 + } + ] + ] + }, + "AI Extract Notes": { + "main": [ + [ + { + "node": "Prepare for RAG", + "type": "main", + "index": 0 + } + ] + ] + }, + "Prepare for RAG": { + "main": [ + [ + { + "node": "Chunk Text", + "type": "main", + "index": 0 + } + ] + ] + }, + "Chunk Text": { + "main": [ + [ + { + "node": "Get Embedding", + "type": "main", + "index": 0 + } + ] + ] + }, + "Get Embedding": { + "main": [ + [ + { + "node": "Prepare Qdrant Point", + "type": "main", + "index": 0 + } + ] + ] + }, + "Prepare Qdrant Point": { + "main": [ + [ + { + "node": "Upsert to Qdrant", + "type": "main", + "index": 0 + } + ] + ] + }, + "Upsert to Qdrant": { + "main": [ + [ + { + "node": "Format Gotify Message", + "type": "main", + "index": 0 + }, + { + "node": "Prepare Notes Doc", + "type": "main", + "index": 0 + } + ] + ] + }, + "Format Gotify Message": { + "main": [ + [ + { + "node": "Notify via Gotify", + "type": "main", + "index": 0 + } + ] + ] + }, + "Prepare Notes Doc": { + "main": [ + [ + { + "node": "Save Notes to Paperless", + "type": "main", + "index": 0 + } + ] + ] + } + }, + "settings": { + "executionOrder": "v1", + "binaryMode": "separate" + }, + "staticData": null, + "meta": null, + "pinData": {}, + "versionId": "d64293a7-5537-4b01-a11b-6b838d6cacbf", + "activeVersionId": "d64293a7-5537-4b01-a11b-6b838d6cacbf", + "versionCounter": 11, + "triggerCount": 1, + "tags": [ + { + "updatedAt": "2026-05-07T00:49:56.081Z", + "createdAt": "2026-05-07T00:49:56.081Z", + "id": "Q9lCYtCOuy9XmeyL", + "name": "school" + }, + { + "updatedAt": "2026-05-06T22:39:59.716Z", + "createdAt": "2026-05-06T22:39:59.716Z", + "id": "tuPxDoKFqpVrrQoX", + "name": "whisper" + }, + { + "updatedAt": "2026-05-06T22:39:33.284Z", + "createdAt": "2026-05-06T22:39:33.284Z", + "id": "7uuEyQrIlcuruZBk", + "name": "rag" + }, + { + "updatedAt": "2026-05-06T22:38:51.678Z", + "createdAt": "2026-05-06T22:38:51.678Z", + "id": "S9FxzVfHLsHmyzyt", + "name": "ai" + } + ], + "shared": [ + { + "updatedAt": "2026-05-07T01:16:50.340Z", + "createdAt": "2026-05-07T01:16:50.340Z", + "role": "workflow:owner", + "workflowId": "91I278E72FAaQenD", + "projectId": "dxCRnBdX5uJizCGa", + "project": { + "updatedAt": "2026-05-06T01:10:37.484Z", + "createdAt": "2026-05-06T01:08:07.721Z", + "id": "dxCRnBdX5uJizCGa", + "name": "Wilfred Fizzlepoof ", + "type": "personal", + "icon": null, + "description": null, + "creatorId": "47b2227f-2fc2-4a08-bd35-ea157b92df0d" + } + } + ], + "versionMetadata": { + "name": "Version d64293a7", + "description": "" + } +} \ No newline at end of file diff --git a/n8n-workflows/14-paperless-inbox-reminder.json b/n8n-workflows/14-paperless-inbox-reminder.json new file mode 100644 index 0000000..3860059 --- /dev/null +++ b/n8n-workflows/14-paperless-inbox-reminder.json @@ -0,0 +1,292 @@ +{ + "name": "Paperless Inbox Reminder", + "active": true, + "isArchived": false, + "nodes": [ + { + "parameters": { + "rule": { + "interval": [ + { + "field": "weeks", + "weeksInterval": 1, + "triggerAtDay": [ + 1 + ], + "triggerAtHour": 9, + "triggerAtMinute": 0 + } + ] + } + }, + "id": "41404371-29ed-4660-9989-a5cbd890555b", + "name": "Every Monday 9 AM", + "type": "n8n-nodes-base.scheduleTrigger", + "typeVersion": 1.2, + "position": [ + 0, + 0 + ] + }, + { + "parameters": { + "method": "GET", + "url": "https://paperless.paccoco.com/api/documents/", + "sendHeaders": true, + "headerParameters": { + "parameters": [ + { + "name": "Authorization", + "value": "Bearer ={{ $env.PAPERLESS_API_TOKEN }}" + } + ] + }, + "sendQuery": true, + "queryParameters": { + "parameters": [ + { + "name": "is_tagged", + "value": "0" + }, + { + "name": "page_size", + "value": "25" + }, + { + "name": "ordering", + "value": "created" + }, + { + "name": "fields", + "value": "id,title,created,document_type,tags" + } + ] + }, + "options": {} + }, + "id": "fdcd5728-6d27-445f-961b-18f204297469", + "name": "Fetch Inbox Docs", + "type": "n8n-nodes-base.httpRequest", + "typeVersion": 4.2, + "position": [ + 224, + 0 + ] + }, + { + "parameters": { + "jsCode": "const data = $input.first().json;\nconst docs = data.results || [];\nconst total = data.count || 0;\n\n// Oldest 5 for the reminder body\nconst oldest = docs.slice(0, 5).map(d => {\n const date = d.created ? d.created.substring(0, 10) : 'unknown date';\n return ` \u2022 ${d.title || 'Untitled'} (added ${date})`;\n});\n\nreturn [{\n json: {\n total,\n hasItems: total > 0,\n oldestList: oldest.join('\\n'),\n remaining: Math.max(0, total - 5)\n }\n}];" + }, + "id": "8fbe6be0-1ed3-4126-8f21-c3f4efb161c6", + "name": "Parse Response", + "type": "n8n-nodes-base.code", + "typeVersion": 2, + "position": [ + 448, + 0 + ] + }, + { + "parameters": { + "conditions": { + "options": { + "caseSensitive": true, + "leftValue": "", + "typeValidation": "strict" + }, + "conditions": [ + { + "id": "efdd4f81-1976-4d26-a565-d110bfeba1df", + "leftValue": "={{ $json.hasItems }}", + "rightValue": true, + "operator": { + "type": "boolean", + "operation": "equal" + } + } + ], + "combinator": "and" + } + }, + "id": "e24344a1-e857-462c-b96a-842169e83a6d", + "name": "Has Inbox Items?", + "type": "n8n-nodes-base.if", + "typeVersion": 2.2, + "position": [ + 672, + 0 + ] + }, + { + "parameters": { + "jsCode": "const d = $input.first().json;\nconst extra = d.remaining > 0 ? `\\n \u2026 and ${d.remaining} more` : '';\nconst body = `${d.total} untagged document${d.total !== 1 ? 's' : ''} in Paperless inbox:\\n${d.oldestList}${extra}\\n\\nhttps://paperless.paccoco.com`;\nreturn [{ json: { title: '\ud83d\udce5 Paperless Inbox Needs Attention', message: body, priority: 5 } }];" + }, + "id": "897af9c9-4429-445f-8160-a03a14a318a3", + "name": "Format Reminder", + "type": "n8n-nodes-base.code", + "typeVersion": 2, + "position": [ + 896, + -120 + ] + }, + { + "parameters": { + "jsCode": "return [{ json: { title: '\u2705 Paperless Inbox Clear', message: 'No untagged documents \u2014 inbox is clean.', priority: 1 } }];" + }, + "id": "bd8fe88e-7876-45dd-8344-983674fba79d", + "name": "Format Clear", + "type": "n8n-nodes-base.code", + "typeVersion": 2, + "position": [ + 896, + 120 + ] + }, + { + "parameters": { + "message": "={{ $json.message }}", + "additionalFields": { + "priority": "={{ $json.priority }}", + "title": "={{ $json.title }}" + }, + "options": { + "contentType": "text/plain" + } + }, + "id": "74860e7f-42e3-448f-9e46-5cfb6f12cdc2", + "name": "Push to Gotify (Reminder)", + "type": "n8n-nodes-base.gotify", + "typeVersion": 1, + "position": [ + 1120, + -120 + ], + "credentials": { + "gotifyApi": { + "id": "SETUP_REQUIRED", + "name": "Reminders" + } + } + }, + { + "parameters": { + "message": "={{ $json.message }}", + "additionalFields": { + "priority": "={{ $json.priority }}", + "title": "={{ $json.title }}" + }, + "options": { + "contentType": "text/plain" + } + }, + "id": "b1dbd39b-87c3-43ec-9cec-f63eabbd0fa5", + "name": "Push to Gotify (Clear)", + "type": "n8n-nodes-base.gotify", + "typeVersion": 1, + "position": [ + 1120, + 120 + ], + "credentials": { + "gotifyApi": { + "id": "SETUP_REQUIRED", + "name": "Reminders" + } + } + } + ], + "connections": { + "Every Monday 9 AM": { + "main": [ + [ + { + "node": "Fetch Inbox Docs", + "type": "main", + "index": 0 + } + ] + ] + }, + "Fetch Inbox Docs": { + "main": [ + [ + { + "node": "Parse Response", + "type": "main", + "index": 0 + } + ] + ] + }, + "Parse Response": { + "main": [ + [ + { + "node": "Has Inbox Items?", + "type": "main", + "index": 0 + } + ] + ] + }, + "Has Inbox Items?": { + "main": [ + [ + { + "node": "Format Reminder", + "type": "main", + "index": 0 + } + ], + [ + { + "node": "Format Clear", + "type": "main", + "index": 0 + } + ] + ] + }, + "Format Reminder": { + "main": [ + [ + { + "node": "Push to Gotify (Reminder)", + "type": "main", + "index": 0 + } + ] + ] + }, + "Format Clear": { + "main": [ + [ + { + "node": "Push to Gotify (Clear)", + "type": "main", + "index": 0 + } + ] + ] + } + }, + "settings": { + "executionOrder": "v1", + "binaryMode": "separate" + }, + "staticData": null, + "meta": null, + "pinData": {}, + "tags": [ + { + "name": "reminders" + }, + { + "name": "paperless" + }, + { + "name": "scheduled" + } + ] +} \ No newline at end of file diff --git a/n8n-workflows/15-n8n-health-monitor.json b/n8n-workflows/15-n8n-health-monitor.json new file mode 100644 index 0000000..b48083d --- /dev/null +++ b/n8n-workflows/15-n8n-health-monitor.json @@ -0,0 +1,383 @@ +{ + "name": "n8n Self-Health Monitor", + "active": true, + "isArchived": false, + "nodes": [ + { + "parameters": { + "rule": { + "interval": [ + { + "field": "minutes", + "minutesInterval": 15 + } + ] + } + }, + "id": "dabcb0ae-d513-410e-b779-4570df5c4a35", + "name": "Every 15 Minutes", + "type": "n8n-nodes-base.scheduleTrigger", + "typeVersion": 1.2, + "position": [ + 0, + 0 + ] + }, + { + "parameters": { + "method": "GET", + "url": "http://10.5.1.6:5678/api/v1/executions", + "sendHeaders": true, + "headerParameters": { + "parameters": [ + { + "name": "X-N8N-API-KEY", + "value": "={{ $env.N8N_API_KEY }}" + } + ] + }, + "sendQuery": true, + "queryParameters": { + "parameters": [ + { + "name": "status", + "value": "error" + }, + { + "name": "includeData", + "value": "false" + }, + { + "name": "limit", + "value": "50" + } + ] + }, + "options": {} + }, + "id": "806d75e9-b95a-499a-b997-a7f7dc046775", + "name": "Fetch Failed Executions", + "type": "n8n-nodes-base.httpRequest", + "typeVersion": 4.2, + "position": [ + 224, + 0 + ] + }, + { + "parameters": { + "jsCode": "const data = $input.first().json;\nconst executions = data.data || [];\nconst oneHourAgo = Date.now() - (60 * 60 * 1000);\n\n// Filter to failures in the past hour\nconst recent = executions.filter(e => {\n const ts = new Date(e.startedAt || e.stoppedAt || 0).getTime();\n return ts >= oneHourAgo;\n});\n\n// Group by workflow name + error message for dedup key\nconst failures = recent.map(e => ({\n executionId: e.id,\n workflowId: e.workflowId,\n workflowName: e.workflowData?.name || `Workflow ${e.workflowId}`,\n startedAt: e.startedAt,\n stoppedAt: e.stoppedAt,\n errorMessage: e.data?.resultData?.error?.message || 'Unknown error'\n}));\n\n// Deduplicate by workflowName (report once per workflow per check)\nconst seen = new Set();\nconst unique = failures.filter(f => {\n if (seen.has(f.workflowName)) return false;\n seen.add(f.workflowName);\n return true;\n});\n\nreturn [{ json: { hasFailures: unique.length > 0, count: unique.length, failures: unique } }];" + }, + "id": "f5d47910-5047-4c4f-aa84-a6907aa38551", + "name": "Filter Recent Failures", + "type": "n8n-nodes-base.code", + "typeVersion": 2, + "position": [ + 448, + 0 + ] + }, + { + "parameters": { + "conditions": { + "options": { + "caseSensitive": true, + "leftValue": "", + "typeValidation": "strict" + }, + "conditions": [ + { + "id": "a76d10a7-1313-467e-9648-471b296229a3", + "leftValue": "={{ $json.hasFailures }}", + "rightValue": true, + "operator": { + "type": "boolean", + "operation": "equal" + } + } + ], + "combinator": "and" + } + }, + "id": "7c298433-ab89-435e-8955-73da6b986f46", + "name": "Has Failures?", + "type": "n8n-nodes-base.if", + "typeVersion": 2.2, + "position": [ + 672, + 0 + ] + }, + { + "parameters": { + "jsCode": "const d = $input.first().json;\n// Fingerprint = sorted workflow names joined \u2014 stable across retriggers\nconst fingerprint = (d.failures || [])\n .map(f => f.workflowName)\n .sort()\n .join('|') || 'unknown';\nreturn [{ json: { ...d, fingerprint } }];" + }, + "id": "d3609d45-ba9f-41d3-a9bd-8c558a719f10", + "name": "Build Fingerprint", + "type": "n8n-nodes-base.code", + "typeVersion": 2, + "position": [ + 896, + -120 + ] + }, + { + "parameters": { + "operation": "executeQuery", + "query": "SELECT fingerprint, last_seen FROM n8n_health_dedup WHERE fingerprint = '{{ $json.fingerprint }}' AND last_seen > NOW() - INTERVAL '1 hour' LIMIT 1;", + "options": {} + }, + "id": "15a3dd71-bc5b-4396-9534-63518e097438", + "name": "Lookup Dedup", + "type": "n8n-nodes-base.postgres", + "typeVersion": 2.5, + "position": [ + 1120, + -120 + ], + "credentials": { + "postgres": { + "id": "n9svoXemqSZoNNUB", + "name": "Postgres account" + } + }, + "continueOnFail": true + }, + { + "parameters": { + "jsCode": "const alertData = $('Build Fingerprint').first().json;\nconst rows = $input.all().filter(r => r.json && r.json.fingerprint);\nconst alreadyNotified = rows.length > 0;\nreturn [{ json: { ...alertData, alreadyNotified, shouldNotify: !alreadyNotified } }];" + }, + "id": "a92590e3-7044-4ba3-9b39-f6a9457c5fbe", + "name": "Check Already Notified", + "type": "n8n-nodes-base.code", + "typeVersion": 2, + "position": [ + 1344, + -120 + ] + }, + { + "parameters": { + "conditions": { + "options": { + "caseSensitive": true, + "leftValue": "", + "typeValidation": "strict" + }, + "conditions": [ + { + "id": "b1e77af5-6875-47dd-97af-8e13958a7217", + "leftValue": "={{ $json.shouldNotify }}", + "rightValue": true, + "operator": { + "type": "boolean", + "operation": "equal" + } + } + ], + "combinator": "and" + } + }, + "id": "39ac0eaf-73ce-4fe1-9c4b-d09ac920291d", + "name": "Should Notify?", + "type": "n8n-nodes-base.if", + "typeVersion": 2.2, + "position": [ + 1568, + -120 + ] + }, + { + "parameters": { + "operation": "executeQuery", + "query": "INSERT INTO n8n_health_dedup (fingerprint, last_seen, workflow_names) VALUES ('{{ $json.fingerprint }}', NOW(), '{{ $json.failures.map(f => f.workflowName).join(\", \") }}') ON CONFLICT (fingerprint) DO UPDATE SET last_seen = NOW(), workflow_names = EXCLUDED.workflow_names;", + "options": {} + }, + "id": "0c3be155-0c3a-4354-b863-73a3f5bbf598", + "name": "Upsert Dedup Record", + "type": "n8n-nodes-base.postgres", + "typeVersion": 2.5, + "position": [ + 1792, + -120 + ], + "credentials": { + "postgres": { + "id": "n9svoXemqSZoNNUB", + "name": "Postgres account" + } + } + }, + { + "parameters": { + "jsCode": "const d = $('Build Fingerprint').first().json;\nconst lines = (d.failures || []).map(f =>\n ` \u2022 ${f.workflowName}\\n ${f.errorMessage.substring(0, 120)}`\n);\nconst body = `${d.count} workflow${d.count !== 1 ? 's' : ''} failed in the past hour:\\n\\n${lines.join('\\n')}\\n\\nhttps://n8n.paccoco.com`;\nreturn [{ json: { title: `\u26a0\ufe0f n8n: ${d.count} Workflow Failure${d.count !== 1 ? 's' : ''}`, message: body, priority: 7 } }];" + }, + "id": "1c6b42bf-a38e-47b6-82ac-50c43ee9e583", + "name": "Format Notification", + "type": "n8n-nodes-base.code", + "typeVersion": 2, + "position": [ + 2016, + -120 + ] + }, + { + "parameters": { + "message": "={{ $json.message }}", + "additionalFields": { + "priority": "={{ $json.priority }}", + "title": "={{ $json.title }}" + }, + "options": { + "contentType": "text/plain" + } + }, + "id": "09435c2f-9044-432a-8ca8-dd3ce3262c76", + "name": "Push to Gotify", + "type": "n8n-nodes-base.gotify", + "typeVersion": 1, + "position": [ + 2240, + -120 + ], + "credentials": { + "gotifyApi": { + "id": "SETUP_REQUIRED", + "name": "n8n Health" + } + } + } + ], + "connections": { + "Every 15 Minutes": { + "main": [ + [ + { + "node": "Fetch Failed Executions", + "type": "main", + "index": 0 + } + ] + ] + }, + "Fetch Failed Executions": { + "main": [ + [ + { + "node": "Filter Recent Failures", + "type": "main", + "index": 0 + } + ] + ] + }, + "Filter Recent Failures": { + "main": [ + [ + { + "node": "Has Failures?", + "type": "main", + "index": 0 + } + ] + ] + }, + "Has Failures?": { + "main": [ + [ + { + "node": "Build Fingerprint", + "type": "main", + "index": 0 + } + ], + [] + ] + }, + "Build Fingerprint": { + "main": [ + [ + { + "node": "Lookup Dedup", + "type": "main", + "index": 0 + } + ] + ] + }, + "Lookup Dedup": { + "main": [ + [ + { + "node": "Check Already Notified", + "type": "main", + "index": 0 + } + ] + ] + }, + "Check Already Notified": { + "main": [ + [ + { + "node": "Should Notify?", + "type": "main", + "index": 0 + } + ] + ] + }, + "Should Notify?": { + "main": [ + [ + { + "node": "Upsert Dedup Record", + "type": "main", + "index": 0 + } + ], + [] + ] + }, + "Upsert Dedup Record": { + "main": [ + [ + { + "node": "Format Notification", + "type": "main", + "index": 0 + } + ] + ] + }, + "Format Notification": { + "main": [ + [ + { + "node": "Push to Gotify", + "type": "main", + "index": 0 + } + ] + ] + } + }, + "settings": { + "executionOrder": "v1", + "binaryMode": "separate" + }, + "staticData": null, + "meta": null, + "pinData": {}, + "tags": [ + { + "name": "monitoring" + }, + { + "name": "scheduled" + }, + { + "name": "n8n" + } + ] +} \ No newline at end of file diff --git a/n8n-workflows/16-nfs-watchdog.json b/n8n-workflows/16-nfs-watchdog.json new file mode 100644 index 0000000..eac62eb --- /dev/null +++ b/n8n-workflows/16-nfs-watchdog.json @@ -0,0 +1,449 @@ +{ + "name": "NFS Mount Watchdog + Auto-Heal", + "active": true, + "isArchived": false, + "nodes": [ + { + "parameters": { + "rule": { + "interval": [ + { + "field": "minutes", + "minutesInterval": 5 + } + ] + } + }, + "id": "8c466a67-a0fa-42dc-8ed8-0918834fc3a7", + "name": "Every 5 Minutes", + "type": "n8n-nodes-base.scheduleTrigger", + "typeVersion": 1.2, + "position": [ + 0, + 0 + ] + }, + { + "parameters": { + "command": "mountpoint -q /mnt/unraid/data/media && echo MEDIA_OK || echo MEDIA_MISSING; mountpoint -q /mnt/unraid/data/photos && echo PHOTOS_OK || echo PHOTOS_MISSING" + }, + "id": "f5b2fee9-d67a-4da7-80dd-f72e6078aed9", + "name": "Probe Mounts", + "type": "n8n-nodes-base.ssh", + "typeVersion": 1, + "position": [ + 224, + 0 + ], + "credentials": { + "sshPrivateKey": { + "id": "SETUP_REQUIRED", + "name": "PD SSH" + } + }, + "continueOnFail": true + }, + { + "parameters": { + "jsCode": "\nconst out = ($input.first().json.stdout || '') + ($input.first().json.stderr || '');\nconst mediaOk = out.includes('MEDIA_OK');\nconst photosOk = out.includes('PHOTOS_OK');\nconst missing = [];\nif (!mediaOk) missing.push({ mount: '/mnt/unraid/data/media', label: 'media',\n containers: ['plex','audiobookshelf'] });\nif (!photosOk) missing.push({ mount: '/mnt/unraid/data/photos', label: 'photos',\n containers: ['immich-server','immich-microservices','immich-machine-learning'] });\nreturn [{ json: { mediaOk, photosOk, anyMissing: missing.length > 0, missing } }];\n" + }, + "id": "25bb3239-49cb-4570-b706-4022b9aa751c", + "name": "Parse Mount Status", + "type": "n8n-nodes-base.code", + "typeVersion": 2, + "position": [ + 448, + 0 + ] + }, + { + "parameters": { + "conditions": { + "options": { + "caseSensitive": true, + "leftValue": "", + "typeValidation": "strict" + }, + "conditions": [ + { + "id": "442964fa-a29c-4aa2-98d1-9e25e927c496", + "leftValue": "={{ $json.anyMissing }}", + "rightValue": true, + "operator": { + "type": "boolean", + "operation": "equal" + } + } + ], + "combinator": "and" + } + }, + "id": "68b74832-85a7-4193-9382-a0c22a666e66", + "name": "Any Missing?", + "type": "n8n-nodes-base.if", + "typeVersion": 2.2, + "position": [ + 672, + 0 + ] + }, + { + "parameters": { + "command": "sudo bash /mnt/tank/docker/scripts/mount-unraid-nfs.sh 2>&1" + }, + "id": "b18c7537-0c2f-4122-bfe5-6469d6be6853", + "name": "Run Mount Script", + "type": "n8n-nodes-base.ssh", + "typeVersion": 1, + "position": [ + 896, + -120 + ], + "credentials": { + "sshPrivateKey": { + "id": "SETUP_REQUIRED", + "name": "PD SSH" + } + }, + "continueOnFail": true + }, + { + "parameters": { + "resume": "timeInterval", + "unit": "seconds", + "value": 10 + }, + "id": "2e927f83-9ddc-4816-8ee6-76d21ce21a5e", + "name": "Wait 10s", + "type": "n8n-nodes-base.wait", + "typeVersion": 1.1, + "position": [ + 1120, + -120 + ], + "webhookId": "4e89e8d0-7f08-4378-a2b7-63756665b039" + }, + { + "parameters": { + "command": "mountpoint -q /mnt/unraid/data/media && echo MEDIA_OK || echo MEDIA_MISSING; mountpoint -q /mnt/unraid/data/photos && echo PHOTOS_OK || echo PHOTOS_MISSING" + }, + "id": "d15c0c90-e94f-404a-a787-75f1b1b5befa", + "name": "Re-probe Mounts", + "type": "n8n-nodes-base.ssh", + "typeVersion": 1, + "position": [ + 1344, + -120 + ], + "credentials": { + "sshPrivateKey": { + "id": "SETUP_REQUIRED", + "name": "PD SSH" + } + }, + "continueOnFail": true + }, + { + "parameters": { + "jsCode": "\nconst out = ($input.first().json.stdout || '') + ($input.first().json.stderr || '');\nconst mediaOk = out.includes('MEDIA_OK');\nconst photosOk = out.includes('PHOTOS_OK');\n\n// What was originally missing (from step 3)\nconst originalMissing = $('Parse Mount Status').first().json.missing || [];\n\n// Which of the originally-missing mounts are now up?\nconst healed = originalMissing.filter(m =>\n (m.label === 'media' && mediaOk) ||\n (m.label === 'photos' && photosOk)\n);\nconst stillDown = originalMissing.filter(m =>\n (m.label === 'media' && !mediaOk) ||\n (m.label === 'photos' && !photosOk)\n);\n\n// Build flat container list to restart (only for healed mounts)\nconst containersToRestart = [...new Set(healed.flatMap(m => m.containers))];\nconst restartCmd = containersToRestart.length > 0\n ? `sudo docker restart ${containersToRestart.join(' ')}`\n : 'echo NO_RESTART_NEEDED';\n\nreturn [{ json: {\n allHealed: stillDown.length === 0,\n healed, stillDown,\n containersToRestart,\n restartCmd,\n originalMissing\n}}];\n" + }, + "id": "48de38c2-df08-406f-ad61-809ca6e306a1", + "name": "Parse Heal Result", + "type": "n8n-nodes-base.code", + "typeVersion": 2, + "position": [ + 1568, + -120 + ] + }, + { + "parameters": { + "conditions": { + "options": { + "caseSensitive": true, + "leftValue": "", + "typeValidation": "strict" + }, + "conditions": [ + { + "id": "566dda6e-2234-406a-ad87-c886c7c5ae08", + "leftValue": "={{ $json.allHealed }}", + "rightValue": true, + "operator": { + "type": "boolean", + "operation": "equal" + } + } + ], + "combinator": "and" + } + }, + "id": "29a1b544-97b3-433f-9821-2bf5a8af95d2", + "name": "Healed?", + "type": "n8n-nodes-base.if", + "typeVersion": 2.2, + "position": [ + 1792, + -120 + ] + }, + { + "parameters": { + "command": "={{ $json.restartCmd }}" + }, + "id": "8d791509-9e18-44c1-bf58-ed3027927710", + "name": "Restart Containers", + "type": "n8n-nodes-base.ssh", + "typeVersion": 1, + "position": [ + 2016, + -240 + ], + "credentials": { + "sshPrivateKey": { + "id": "SETUP_REQUIRED", + "name": "PD SSH" + } + }, + "continueOnFail": true + }, + { + "parameters": { + "jsCode": "\nconst d = $('Parse Heal Result').first().json;\nconst mountList = d.healed.map(m => ` \u2022 ${m.mount}`).join('\\n');\nconst containers = d.containersToRestart.join(', ') || 'none';\nconst body = `NFS mounts were missing and have been automatically remounted.\\n\\nRemounted:\\n${mountList}\\n\\nContainers restarted: ${containers}`;\nreturn [{ json: { title: '\ud83d\udd17 NFS Auto-Healed', message: body, priority: 5 } }];\n" + }, + "id": "75c8976f-d622-43cd-a332-72c32f0c7c60", + "name": "Format Healed Notify", + "type": "n8n-nodes-base.code", + "typeVersion": 2, + "position": [ + 2240, + -240 + ] + }, + { + "parameters": { + "message": "={{ $json.message }}", + "additionalFields": { + "priority": "={{ $json.priority }}", + "title": "={{ $json.title }}" + }, + "options": { + "contentType": "text/plain" + } + }, + "id": "38705cd2-775b-4ef9-b470-4a7709f132e5", + "name": "Gotify (Healed)", + "type": "n8n-nodes-base.gotify", + "typeVersion": 1, + "position": [ + 2464, + -240 + ], + "credentials": { + "gotifyApi": { + "id": "SETUP_REQUIRED", + "name": "Uptime / Infra" + } + } + }, + { + "parameters": { + "jsCode": "\nconst d = $('Parse Heal Result').first().json;\nconst downList = d.stillDown.map(m => ` \u2022 ${m.mount}`).join('\\n');\nconst healedList = d.healed.length > 0\n ? `\\nPartially healed:\\n${d.healed.map(m => ' \u2022 ' + m.mount).join('\\n')}`\n : '';\nconst body = `NFS mount script ran but the following paths are still unavailable. Manual intervention required.\\n\\nStill down:\\n${downList}${healedList}\\n\\nCheck Serenity (TrueNAS) is up and NFS service is running.`;\nreturn [{ json: { title: '\ud83d\udea8 NFS Down \u2014 Manual Fix Needed', message: body, priority: 9 } }];\n" + }, + "id": "074b984f-7fc2-45bc-b97f-f5db4d8d589b", + "name": "Format Escalation", + "type": "n8n-nodes-base.code", + "typeVersion": 2, + "position": [ + 2016, + -20 + ] + }, + { + "parameters": { + "message": "={{ $json.message }}", + "additionalFields": { + "priority": "={{ $json.priority }}", + "title": "={{ $json.title }}" + }, + "options": { + "contentType": "text/plain" + } + }, + "id": "f929dd77-6072-4ac9-9dc4-11246110efed", + "name": "Gotify (Escalation)", + "type": "n8n-nodes-base.gotify", + "typeVersion": 1, + "position": [ + 2240, + -20 + ], + "credentials": { + "gotifyApi": { + "id": "SETUP_REQUIRED", + "name": "Uptime / Infra" + } + } + } + ], + "connections": { + "Every 5 Minutes": { + "main": [ + [ + { + "node": "Probe Mounts", + "type": "main", + "index": 0 + } + ] + ] + }, + "Probe Mounts": { + "main": [ + [ + { + "node": "Parse Mount Status", + "type": "main", + "index": 0 + } + ] + ] + }, + "Parse Mount Status": { + "main": [ + [ + { + "node": "Any Missing?", + "type": "main", + "index": 0 + } + ] + ] + }, + "Any Missing?": { + "main": [ + [ + { + "node": "Run Mount Script", + "type": "main", + "index": 0 + } + ], + [] + ] + }, + "Run Mount Script": { + "main": [ + [ + { + "node": "Wait 10s", + "type": "main", + "index": 0 + } + ] + ] + }, + "Wait 10s": { + "main": [ + [ + { + "node": "Re-probe Mounts", + "type": "main", + "index": 0 + } + ] + ] + }, + "Re-probe Mounts": { + "main": [ + [ + { + "node": "Parse Heal Result", + "type": "main", + "index": 0 + } + ] + ] + }, + "Parse Heal Result": { + "main": [ + [ + { + "node": "Healed?", + "type": "main", + "index": 0 + } + ] + ] + }, + "Healed?": { + "main": [ + [ + { + "node": "Restart Containers", + "type": "main", + "index": 0 + } + ], + [ + { + "node": "Format Escalation", + "type": "main", + "index": 0 + } + ] + ] + }, + "Restart Containers": { + "main": [ + [ + { + "node": "Format Healed Notify", + "type": "main", + "index": 0 + } + ] + ] + }, + "Format Healed Notify": { + "main": [ + [ + { + "node": "Gotify (Healed)", + "type": "main", + "index": 0 + } + ] + ] + }, + "Format Escalation": { + "main": [ + [ + { + "node": "Gotify (Escalation)", + "type": "main", + "index": 0 + } + ] + ] + } + }, + "settings": { + "executionOrder": "v1", + "binaryMode": "separate" + }, + "staticData": null, + "meta": null, + "pinData": {}, + "tags": [ + { + "name": "monitoring" + }, + { + "name": "infrastructure" + }, + { + "name": "scheduled" + } + ] +} \ No newline at end of file diff --git a/n8n-workflows/README.md b/n8n-workflows/README.md new file mode 100644 index 0000000..bfca771 --- /dev/null +++ b/n8n-workflows/README.md @@ -0,0 +1,242 @@ +# n8n AI Workflows for Homelab + +## Workflows Included + +| # | Workflow | Trigger | What It Does | +|---|---------|---------|--------------| +| 01 | Grafana Alert → AI → Gotify | `POST /webhook/grafana-alert` | Receives Grafana alerts, AI-summarizes, deduplicates within 30 min, suppresses low-priority alerts overnight — pushes to Gotify | +| 02 | RAG Pipeline | `POST /webhook/rag/ingest` + `POST /webhook/rag/query` | Ingest docs into Qdrant, query them with AI-powered answers | +| 03 | Paperless AI Processing | `POST /webhook/paperless/new-document` | Auto-classifies & summarizes new Paperless docs, applies AI-suggested tags back to Paperless, notifies via Gotify | +| 04 | Whisper Transcription | `POST /webhook/transcribe` + `POST /webhook/transcribe/summarize` | Transcribe audio → text, optionally with AI summary; notifies via Gotify when done and saves transcript to Paperless | +| 05 | Paperless → RAG | `POST /webhook/paperless/rag-ingest` | Auto-ingests Paperless documents into Qdrant for RAG search; notifies via Gotify | +| 06 | RSS Digest | Scheduled (every 6h) + `POST /webhook/scrape/summarize` | Fetches RSS feeds, deduplicates via Postgres, AI-summarizes per-feed, pushes to Gotify | +| 07 | Git Commit Summarizer | `POST /webhook/git/push` | Receives git push webhooks, AI-summarizes changes, notifies via Gotify | +| 08 | Class Recording → RAG | Watches `/mnt/data/class-recordings/` | Auto-transcribes new .wav files, extracts key notes, ingests into RAG by class, saves notes to Paperless, notifies via Gotify | +| 14 | Paperless Inbox Reminder | Scheduled (Monday 9 AM) | Queries Paperless for untagged documents; sends Gotify reminder with oldest items listed, or a ✅ clear if inbox is empty | +| 15 | n8n Self-Health Monitor | Scheduled (every 15 min) | Checks n8n execution history for failures in the past hour; deduplicates via Postgres; pushes Gotify alert with workflow name and error summary | +| 16 | NFS Mount Watchdog | Scheduled (every 5 min) | Probes NFS mount points on PD; if missing: runs mount script, re-probes, restarts affected containers (Plex, Audiobookshelf, Immich); notifies via Gotify on heal or escalation | + +## How to Import + +1. Open n8n at https://n8n.paccoco.com +2. Go to **Workflows** → click the **⋮** menu → **Import from File** +3. Select each `.json` file from this folder +4. Activate the workflow (toggle in top-right) + +## Gotify Channel Map + +Each workflow uses a dedicated Gotify app credential. Create these apps in Gotify and add matching credentials in n8n → Credentials: + +| Gotify App | Used by | +|------------|---------| +| Gotify account | 01 (Grafana Alerts) | +| Paperless | 03 | +| Whisper | 04 | +| Paperless RAG | 05 | +| RSS Feed | 06 | +| Git Commits | 07 | +| Class Recordings | 08 | +| Reminders | 14 | +| n8n Health | 15 | +| Uptime / Infra | 16 | + +## One-Time Setup + +### Postgres Dedup Tables (required for workflows 01, 15) + +```sql +-- Run via: sudo docker exec -it shared-postgres psql -U postgres -d n8n +CREATE TABLE IF NOT EXISTS grafana_alert_dedup ( + fingerprint TEXT PRIMARY KEY, + last_seen TIMESTAMPTZ DEFAULT NOW(), + alert_name TEXT, + host TEXT +); +CREATE TABLE IF NOT EXISTS n8n_health_dedup ( + fingerprint TEXT PRIMARY KEY, + last_seen TIMESTAMPTZ DEFAULT NOW(), + workflow_names TEXT +); +``` + +### n8n Environment Variables + +Add these to n8n's `.env` and run `sudo docker compose up -d` to apply: + +| Variable | Used by | How to get | +|----------|---------|------------| +| `PAPERLESS_API_TOKEN` | 04, 08 | Paperless → Settings → API token | +| `PAPERLESS_TAG_TRANSCRIPTION` | 04 | Tag ID from Paperless API | +| `PAPERLESS_TAG_LECTURE_NOTES` | 08 | Tag ID from Paperless API | +| `N8N_API_KEY` | 15 | n8n → Settings → API → Create API Key | + +### SSH Credential (required for workflow 16) + +Workflow 16 SSHes into PD to probe mounts and restart containers. Set up key-based auth: + +```bash +# Run on PD — generate a dedicated key for n8n +ssh-keygen -t ed25519 -f ~/.ssh/n8n_watchdog -N "" +cat ~/.ssh/n8n_watchdog.pub >> ~/.ssh/authorized_keys +cat ~/.ssh/n8n_watchdog # paste this into n8n → Credentials → SSH (Private Key) +``` + +In n8n: Credentials → New → SSH (Private Key) → host `10.5.1.6`, user `truenas_admin`, paste private key. Name it `PD SSH`. + +Workflow 16 also runs `sudo docker restart` and the mount script over SSH. Add a NOPASSWD sudoers rule so it doesn't hang: + +```bash +sudo visudo -f /etc/sudoers.d/n8n-watchdog +# Add: +# truenas_admin ALL=(ALL) NOPASSWD: /usr/bin/docker, /bin/bash /mnt/tank/docker/scripts/mount-unraid-nfs.sh +``` + +### Qdrant Collection (required for workflows 02, 05, 08) + +Create the `knowledge_base` collection before using the RAG workflows: + +```bash +curl -X PUT http://10.5.1.6:6333/collections/knowledge_base \ + -H 'Content-Type: application/json' \ + -d '{ + "vectors": { + "size": 768, + "distance": "Cosine" + } + }' +``` + +(768 dimensions matches `nomic-embed-text`. If you use a different embedding model, adjust the size.) + +### Pull nomic-embed-text into Ollama (required for workflows 02, 05, 08) + +```bash +curl http://10.5.1.6:11434/api/pull -d '{"name": "nomic-embed-text"}' +``` + +### Paperless Webhook (required for workflows 03 and 05) + +In Paperless-NGX, set up post-consumption webhooks to POST to: +``` +https://n8n.paccoco.com/webhook/paperless/new-document +https://n8n.paccoco.com/webhook/paperless/rag-ingest +``` +with body: `{"document_id": }` + +You can trigger both from the same Paperless event — workflow 03 classifies/tags the document, workflow 05 ingests it into RAG. + +### Grafana Webhook (already configured) + +Your Grafana contact point is already set to `https://n8n.paccoco.com/webhook/grafana-alert` — workflow 01 will work immediately once activated. + +### Git Webhook (required for workflow 07) + +In your Gitea/Forgejo/GitHub repo settings, add a webhook: +- URL: `https://n8n.paccoco.com/webhook/git/push` +- Content type: `application/json` +- Events: Push only + +Works with Gitea, Forgejo, GitHub, and GitLab webhook payloads. + +## Usage Examples + +### Ingest a document into RAG: +```bash +curl -X POST https://n8n.paccoco.com/webhook/rag/ingest \ + -H 'Content-Type: application/json' \ + -d '{ + "text": "Your document content here...", + "source": "my-notes", + "metadata": {"topic": "homelab"} + }' +``` + +### Query your knowledge base: +```bash +curl -X POST https://n8n.paccoco.com/webhook/rag/query \ + -H 'Content-Type: application/json' \ + -d '{"query": "How do I configure Qdrant?"}' +``` + +### Transcribe audio: +```bash +curl -X POST https://n8n.paccoco.com/webhook/transcribe \ + -F "data=@recording.mp3" +``` + +### Transcribe + AI summarize: +```bash +curl -X POST https://n8n.paccoco.com/webhook/transcribe/summarize \ + -F "data=@meeting.mp3" +``` + +### Add a class recording (just drop a file): +``` +/mnt/data/class-recordings/ +├── CS101-Java/ +│ ├── lecture-01-intro.wav +│ ├── lecture-02-variables.wav +│ └── lecture-03-loops.wav +├── MATH201/ +│ └── lecture-01-derivatives.wav +└── ENG102/ + └── essay-workshop.wav +``` + +The workflow detects the class from the subfolder name and the lecture title from the filename. +Just drop a `.wav` file into the right class folder and it auto-processes. + +### Query RAG for class-specific content: +```bash +curl -X POST https://n8n.paccoco.com/webhook/rag/query \ + -H 'Content-Type: application/json' \ + -d '{"query": "What did my CS101 lecture say about inheritance in Java?"}' +``` + +The RAG system stores class name metadata with each chunk, so when you mention a class name in your query it will prioritize results from that class. + +### Class Recordings Setup (Workflow 08) + +Requires n8n volume mount to the NFS share. Add to n8n's docker-compose: +```yaml +volumes: + - /mnt/tank/docker/appdata/n8n:/home/node/.n8n + - /mnt/data/class-recordings:/data/class-recordings +``` + +Create class subfolders: +```bash +mkdir -p /mnt/data/class-recordings/CS101-Java +mkdir -p /mnt/data/class-recordings/MATH201 +``` + +Then restart n8n and activate the workflow. Any `.wav` dropped into a subfolder triggers automatic transcription + RAG ingest. + +### Trigger RSS digest manually: +```bash +curl -X POST https://n8n.paccoco.com/webhook/scrape/summarize +``` + +### Test git webhook: +```bash +curl -X POST https://n8n.paccoco.com/webhook/git/push \ + -H 'Content-Type: application/json' \ + -d '{ + "ref": "refs/heads/main", + "pusher": {"name": "john"}, + "repository": {"full_name": "john/my-project"}, + "commits": [ + {"id": "abc1234", "message": "Fix null pointer in UserService", "author": {"name": "john"}, "added": [], "modified": ["src/UserService.java"], "removed": []} + ] + }' +``` + +## Customizing RSS Feeds (Workflow 06) + +Edit the "Define RSS Feeds" Code node to add/remove feeds. Default feeds: +- Self-Hosted Show +- r/selfhosted +- r/homelab +- Hacker News + +Add any RSS feed URL you want monitored. diff --git a/n8n-workflows/class-watcher.sh b/n8n-workflows/class-watcher.sh new file mode 100644 index 0000000..d41efc9 --- /dev/null +++ b/n8n-workflows/class-watcher.sh @@ -0,0 +1,121 @@ +#!/bin/bash +# class-watcher.sh — Watches /mnt/data/class-recordings for new .wav files +# and POSTs them to the n8n class/upload webhook. +# Saves the transcript + notes response as a .md file alongside the recording. +# +# Install: +# 1. Copy to PD: scp class-watcher.sh truenas_admin@10.5.1.6:/tmp/class-watcher.sh +# 2. sudo mv /tmp/class-watcher.sh /mnt/data/class-watcher.sh +# 3. sudo chmod +x /mnt/data/class-watcher.sh +# 4. sudo crontab -e +# * * * * * /mnt/data/class-watcher.sh >> /var/log/class-watcher.log 2>&1 +# +# Folder structure after processing: +# /mnt/data/class-recordings/ +# └── CS101-Java/ +# ├── new-lecture.wav ← waiting to be processed +# └── done/ +# ├── lecture-01-intro.wav ← already processed +# └── lecture-01-intro.md ← transcript + notes + +WATCH_DIR="/mnt/data/class-recordings" +WEBHOOK_URL="http://10.5.1.6:5678/webhook/class/upload" + +# Find all .wav files directly inside class subfolders (not in done/) +find "$WATCH_DIR" -mindepth 2 -maxdepth 2 -name '*.wav' -type f | while read -r filepath; do + # Skip files in 'done' directories + if echo "$filepath" | grep -q '/done/'; then + continue + fi + + # Extract class name from parent folder + class_dir=$(dirname "$filepath") + class_name=$(basename "$class_dir") + file_name=$(basename "$filepath") + lecture_title="${file_name%.wav}" + + echo "[$(date)] Processing: $filepath (class: $class_name)" + + # Create done folder + mkdir -p "$class_dir/done" + + # POST the file to n8n webhook and capture the response + response=$(curl -s -X POST "$WEBHOOK_URL" \ + -H "X-Class-Name: $class_name" \ + -H "X-Lecture-Title: $lecture_title" \ + -F "data=@$filepath" \ + -F "class_name=$class_name" \ + -F "lecture_title=$lecture_title" \ + -F "filename=$file_name" \ + -w "\n%{http_code}" 2>/dev/null) + + http_code=$(echo "$response" | tail -1) + body=$(echo "$response" | sed '$d') + + echo "[$(date)] HTTP $http_code" + + # Save the transcript + notes as markdown + md_file="$class_dir/done/${lecture_title}.md" + + # Try to parse JSON response and format as markdown + if command -v python3 &>/dev/null; then + echo "$body" | python3 -c " +import json, sys +try: + data = json.load(sys.stdin) + # Handle array response from n8n + if isinstance(data, list): + data = data[0] if data else {} + cn = data.get('className', '$class_name') + lt = data.get('lectureTitle', '$lecture_title') + dm = data.get('durationMinutes', '?') + notes = data.get('notes', {}) + summary = notes.get('summary', 'No summary available') + concepts = notes.get('key_concepts', []) + defs = notes.get('definitions', []) + assignments = notes.get('assignments', []) + transcript = data.get('timestampedText', data.get('fullText', 'No transcript')) + + print(f'# {cn} — {lt}') + print(f'**Duration:** {dm} minutes') + print() + print('## Summary') + print(summary) + print() + if concepts: + print('## Key Concepts') + for c in concepts: + print(f'- {c}') + print() + if defs: + print('## Definitions') + for d in defs: + if isinstance(d, dict): + print(f'- **{d.get(\"term\", \"\")}**: {d.get(\"definition\", \"\")}') + else: + print(f'- {d}') + print() + if assignments: + print('## Assignments') + for a in assignments: + print(f'- {a}') + print() + print('## Full Transcript') + print(transcript) +except Exception as e: + print(f'# {\"$class_name\"} — {\"$lecture_title\"}') + print() + print('## Raw Response') + print(sys.stdin.read() if not data else json.dumps(data, indent=2)) +" > "$md_file" 2>/dev/null + else + # Fallback: save raw JSON + echo "$body" > "$md_file" + fi + + echo "[$(date)] Saved notes to: $md_file" + + # Move wav to done folder + mv "$filepath" "$class_dir/done/$file_name" + echo "[$(date)] Moved to: $class_dir/done/$file_name" +done