From f3f1c346ac5f0bf546f15c7094fa063133758f51 Mon Sep 17 00:00:00 2001 From: fizzlepoof Date: Sun, 10 May 2026 09:56:29 -0500 Subject: [PATCH] Local homelab changes --- .../01-create-uptime-kuma-db.sh | 0 meshtastic/scripts/PirateWeatherADV.py | 0 meshtastic/scripts/mm_wx.py | 0 meshtastic/scripts/upgrade-watchdog.sh | 0 monitoring/.env.example | 6 + monitoring/DEPLOY.md | 118 ++++ monitoring/deploy.sh | 236 ++++++++ .../01-grafana-alert-ai-gotify-v1.1.json | 363 ++++++++++++ .../01-grafana-alert-ai-gotify-v1.2.json | 387 +++++++++++++ .../01-grafana-alert-ai-gotify-v1.3.json | 387 +++++++++++++ .../04-whisper-transcription-v1.1.json | 406 +++++++++++++ .../08-class-recording-rag-v1.2.json | 538 ++++++++++++++++++ .../09-class-transcript-ingest-v1.0.json | 262 +++++++++ .../09-class-transcript-ingest-v1.1.json | 423 ++++++++++++++ .../14-paperless-inbox-reminder-v1.1.json | 293 ++++++++++ .../14-paperless-inbox-reminder-v1.2.json | 0 n8n-workflows/15-n8n-health-monitor-v1.1.json | 384 +++++++++++++ n8n-workflows/15-n8n-health-monitor-v1.2.json | 0 n8n-workflows/16-nfs-watchdog-v1.1.json | 450 +++++++++++++++ n8n-workflows/TESTING-STATUS.md | 0 rocinante/.env.example | 2 + rocinante/docker-compose.yaml | 42 ++ rocinante/watcher/Dockerfile | 12 + rocinante/watcher/watch.py | 360 ++++++++++++ 24 files changed, 4669 insertions(+) mode change 100755 => 100644 databases/initdb-mariadb/01-create-uptime-kuma-db.sh mode change 100755 => 100644 meshtastic/scripts/PirateWeatherADV.py mode change 100755 => 100644 meshtastic/scripts/mm_wx.py mode change 100755 => 100644 meshtastic/scripts/upgrade-watchdog.sh create mode 100644 monitoring/.env.example create mode 100644 monitoring/DEPLOY.md create mode 100644 monitoring/deploy.sh create mode 100644 n8n-workflows/01-grafana-alert-ai-gotify-v1.1.json create mode 100644 n8n-workflows/01-grafana-alert-ai-gotify-v1.2.json create mode 100644 n8n-workflows/01-grafana-alert-ai-gotify-v1.3.json create mode 100644 n8n-workflows/04-whisper-transcription-v1.1.json create mode 100644 n8n-workflows/08-class-recording-rag-v1.2.json create mode 100644 n8n-workflows/09-class-transcript-ingest-v1.0.json create mode 100644 n8n-workflows/09-class-transcript-ingest-v1.1.json create mode 100644 n8n-workflows/14-paperless-inbox-reminder-v1.1.json mode change 100755 => 100644 n8n-workflows/14-paperless-inbox-reminder-v1.2.json create mode 100644 n8n-workflows/15-n8n-health-monitor-v1.1.json mode change 100755 => 100644 n8n-workflows/15-n8n-health-monitor-v1.2.json create mode 100644 n8n-workflows/16-nfs-watchdog-v1.1.json mode change 100755 => 100644 n8n-workflows/TESTING-STATUS.md create mode 100644 rocinante/.env.example create mode 100644 rocinante/docker-compose.yaml create mode 100644 rocinante/watcher/Dockerfile create mode 100644 rocinante/watcher/watch.py diff --git a/databases/initdb-mariadb/01-create-uptime-kuma-db.sh b/databases/initdb-mariadb/01-create-uptime-kuma-db.sh old mode 100755 new mode 100644 diff --git a/meshtastic/scripts/PirateWeatherADV.py b/meshtastic/scripts/PirateWeatherADV.py old mode 100755 new mode 100644 diff --git a/meshtastic/scripts/mm_wx.py b/meshtastic/scripts/mm_wx.py old mode 100755 new mode 100644 diff --git a/meshtastic/scripts/upgrade-watchdog.sh b/meshtastic/scripts/upgrade-watchdog.sh old mode 100755 new mode 100644 diff --git a/monitoring/.env.example b/monitoring/.env.example new file mode 100644 index 0000000..247522c --- /dev/null +++ b/monitoring/.env.example @@ -0,0 +1,6 @@ +TZ=America/New_York + +# Grafana +GF_ADMIN_USER=admin +GF_ADMIN_PASS=CHANGE_ME +GF_HOST=grafana.paccoco.com diff --git a/monitoring/DEPLOY.md b/monitoring/DEPLOY.md new file mode 100644 index 0000000..9ca3191 --- /dev/null +++ b/monitoring/DEPLOY.md @@ -0,0 +1,118 @@ +# Phase 6 — Grafana + Prometheus Deployment on PD + +> Adapted from HOMELAB_BUILDOUT_PLAN.md Phase 6, changed host from N.O.M.A.D. to PlausibleDeniability. + +## 1. Scaffold Directories + +SSH into PD (or run locally): + +```bash +# Stack directory +sudo mkdir -p /opt/monitoring/provisioning/datasources +sudo mkdir -p /opt/monitoring/provisioning/dashboards + +# Prometheus TSDB on tank (sequential writes, grows with retention) +sudo mkdir -p /mnt/tank/docker/appdata/prometheus +sudo chown 65534:65534 /mnt/tank/docker/appdata/prometheus + +# Grafana data on tank +sudo mkdir -p /mnt/tank/docker/appdata/grafana +sudo chown 472:472 /mnt/tank/docker/appdata/grafana +``` + +## 2. Copy Files to PD + +Copy everything from this `monitoring/` folder to `/opt/monitoring/` on PD: + +```bash +# From your local machine (adjust source path as needed) +scp -r monitoring/* pd:/opt/monitoring/ + +# Or if working directly on PD, copy files into place: +# docker-compose.yaml → /opt/monitoring/docker-compose.yaml +# prometheus.yml → /opt/monitoring/prometheus.yml +# provisioning/ → /opt/monitoring/provisioning/ +# .env.example → /opt/monitoring/.env.example +``` + +## 3. Create .env + +```bash +cd /opt/monitoring +cp .env.example .env +nano .env +# Set GF_ADMIN_PASS=$(openssl rand -hex 16) +``` + +## 4. Pangolin Configuration + +In Pangolin dashboard, create a new resource using PD's existing Newt: + +| Field | Value | +|--------|--------------------------| +| Domain | `grafana.paccoco.com` | +| Scheme | `http` | +| Host | `grafana` | +| Port | `3000` | + +Since Grafana joins the `pangolin` network, PD's Newt can reach it by container name. + +## 5. Validate & Deploy + +```bash +cd /opt/monitoring +docker compose --env-file .env config +docker compose --env-file .env up -d +``` + +## 6. Post-Deploy Verification + +```bash +# All three containers running? +docker ps --filter name=prometheus --filter name=grafana --filter name=node-exporter + +# Prometheus scraping targets? +curl -s http://10.5.1.6:9090/api/v1/targets | python3 -m json.tool | head -40 + +# Grafana UI accessible? +curl -s -o /dev/null -w "%{http_code}" http://10.5.1.6:3000/ +# Expected: 200 or 302 + +# Node exporter metrics flowing? +curl -s http://10.5.1.6:9100/metrics | head -10 +``` + +> Remember: use `10.5.1.6` not `localhost` for health checks on TrueNAS Scale. + +## 7. Recommended Dashboard Imports + +In Grafana → Dashboards → New → Import → Enter ID: + +| Dashboard | ID | Purpose | +|------------------------|------|-----------------------------------| +| Node Exporter Full | 1860 | PD system metrics | +| Docker Container Stats | 893 | Container resource usage | +| Netdata via Prometheus | search | PD and Serenity system metrics | + +## 8. Grafana → n8n Alert Webhook + +In Grafana → Alerting → Contact Points, create: + +| Field | Value | +|-------------|--------------------------------------------------| +| Name | `n8n-alerts` | +| Type | Webhook | +| URL | `https://n8n.paccoco.com/webhook/grafana-alert` | +| HTTP Method | POST | + +Suggested alert rules: +- ZFS pool utilization > 85% +- Container memory > 90% of limit +- Host CPU sustained > 90% for 5 minutes +- Disk I/O latency spikes + +## Notes + +- **Storage:** Prometheus TSDB and Grafana data are on tank (plenty of headroom). 90-day retention is configured. Prometheus writes sequentially so tank performs fine here. +- **N.O.M.A.D. scraping:** The N.O.M.A.D. target in prometheus.yml is commented out. Uncomment once Netdata or node-exporter is running on N.O.M.A.D. at 10.5.1.16. +- **Image versions:** Prometheus v3.4.0, Grafana 13.0.1, node-exporter v1.9.0 — verify these are still current before deploying. diff --git a/monitoring/deploy.sh b/monitoring/deploy.sh new file mode 100644 index 0000000..03197b9 --- /dev/null +++ b/monitoring/deploy.sh @@ -0,0 +1,236 @@ +#!/usr/bin/env bash +set -euo pipefail + +# ============================================================ +# Phase 6 — Grafana + Prometheus Deployment on PD +# Run this script on PlausibleDeniability as root or with sudo +# ============================================================ + +echo "=== Phase 6: Grafana + Prometheus ===" +echo "" + +# ---- 1. Scaffold directories ---- +echo "[1/6] Scaffolding directories..." + +mkdir -p /mnt/docker-ssd/docker/compose/monitoring/provisioning/datasources +mkdir -p /mnt/docker-ssd/docker/compose/monitoring/provisioning/dashboards + +mkdir -p /mnt/tank/docker/appdata/prometheus +chown 65534:65534 /mnt/tank/docker/appdata/prometheus + +mkdir -p /mnt/tank/docker/appdata/grafana +chown 472:472 /mnt/tank/docker/appdata/grafana + +echo " Done." + +# ---- 2. Write docker-compose.yaml ---- +echo "[2/6] Writing docker-compose.yaml..." + +cat > /mnt/docker-ssd/docker/compose/monitoring/docker-compose.yaml << 'COMPOSE' +name: monitoring + +services: + prometheus: + image: prom/prometheus:v3.4.0 + container_name: prometheus + restart: unless-stopped + ports: + - "9090:9090" + command: + - '--config.file=/etc/prometheus/prometheus.yml' + - '--storage.tsdb.path=/prometheus' + - '--storage.tsdb.retention.time=90d' + - '--web.enable-lifecycle' + volumes: + - /mnt/tank/docker/appdata/prometheus:/prometheus + - /mnt/docker-ssd/docker/compose/monitoring/prometheus.yml:/etc/prometheus/prometheus.yml:ro + networks: + - default + + grafana: + image: grafana/grafana:13.0.1 + container_name: grafana + restart: unless-stopped + ports: + - "3000:3000" + environment: + TZ: ${TZ} + GF_SECURITY_ADMIN_USER: ${GF_ADMIN_USER} + GF_SECURITY_ADMIN_PASSWORD: ${GF_ADMIN_PASS} + GF_SERVER_ROOT_URL: https://${GF_HOST}/ + volumes: + - /mnt/tank/docker/appdata/grafana:/var/lib/grafana + - /mnt/docker-ssd/docker/compose/monitoring/provisioning:/etc/grafana/provisioning:ro + networks: + - pangolin + - default + + node-exporter: + image: prom/node-exporter:v1.9.0 + container_name: node-exporter + restart: unless-stopped + ports: + - "9100:9100" + command: + - '--path.procfs=/host/proc' + - '--path.sysfs=/host/sys' + - '--path.rootfs=/rootfs' + - '--collector.filesystem.mount-points-exclude=^/(sys|proc|dev|host|etc)($$|/)' + volumes: + - /proc:/host/proc:ro + - /sys:/host/sys:ro + - /:/rootfs:ro + networks: + - default + +networks: + pangolin: + external: true +COMPOSE + +echo " Done." + +# ---- 3. Write prometheus.yml ---- +echo "[3/6] Writing prometheus.yml..." + +cat > /mnt/docker-ssd/docker/compose/monitoring/prometheus.yml << 'PROMCFG' +global: + scrape_interval: 15s + evaluation_interval: 15s + +scrape_configs: + # ------- PlausibleDeniability (local) ------- + - job_name: 'pd-node' + static_configs: + - targets: ['node-exporter:9100'] + labels: + host: 'plausible-deniability' + + - job_name: 'pd-netdata' + metrics_path: /api/v1/allmetrics + params: + format: [prometheus] + static_configs: + - targets: ['10.5.1.6:19999'] + labels: + host: 'plausible-deniability' + + # ------- Serenity ------- + - job_name: 'serenity-netdata' + metrics_path: /api/v1/allmetrics + params: + format: [prometheus] + static_configs: + - targets: ['10.5.1.5:19999'] + labels: + host: 'serenity' + + # ------- N.O.M.A.D. ------- + # Uncomment when Netdata or node-exporter is available on N.O.M.A.D. + # - job_name: 'nomad-netdata' + # metrics_path: /api/v1/allmetrics + # params: + # format: [prometheus] + # static_configs: + # - targets: ['10.5.1.16:19999'] + # labels: + # host: 'nomad' +PROMCFG + +echo " Done." + +# ---- 4. Write Grafana provisioning ---- +echo "[4/6] Writing Grafana provisioning configs..." + +cat > /mnt/docker-ssd/docker/compose/monitoring/provisioning/datasources/prometheus.yml << 'DATASRC' +apiVersion: 1 + +datasources: + - name: Prometheus + type: prometheus + access: proxy + url: http://prometheus:9090 + isDefault: true + editable: true +DATASRC + +cat > /mnt/docker-ssd/docker/compose/monitoring/provisioning/dashboards/dashboards.yml << 'DASHCFG' +apiVersion: 1 + +providers: + - name: 'default' + orgId: 1 + folder: '' + type: file + disableDeletion: false + editable: true + options: + path: /var/lib/grafana/dashboards + foldersFromFilesStructure: false +DASHCFG + +echo " Done." + +# ---- 5. Generate .env ---- +echo "[5/6] Generating .env..." + +GF_PASS=$(openssl rand -hex 16) + +cat > /mnt/docker-ssd/docker/compose/monitoring/.env << ENV +TZ=America/New_York + +# Grafana +GF_ADMIN_USER=admin +GF_ADMIN_PASS=${GF_PASS} +GF_HOST=grafana.paccoco.com +ENV + +echo " Done." +echo "" +echo " Grafana admin password: ${GF_PASS}" +echo " (saved in /mnt/docker-ssd/docker/compose/monitoring/.env)" +echo "" + +# ---- 6. Validate and deploy ---- +echo "[6/6] Validating and deploying..." + +cd /mnt/docker-ssd/docker/compose/monitoring +docker compose --env-file .env config > /dev/null 2>&1 +echo " Compose config valid." + +docker compose --env-file .env up -d +echo "" +echo "=== Deployment complete ===" +echo "" +echo "Waiting 10 seconds for containers to start..." +sleep 10 + +# ---- Post-deploy checks ---- +echo "" +echo "=== Post-Deploy Verification ===" +echo "" + +echo "--- Container status ---" +docker ps --filter name=prometheus --filter name=grafana --filter name=node-exporter --format "table {{.Names}}\t{{.Status}}\t{{.Ports}}" +echo "" + +echo "--- Prometheus targets ---" +curl -s http://10.5.1.6:9090/api/v1/targets 2>/dev/null | python3 -m json.tool 2>/dev/null | head -30 || echo "Prometheus not yet responding (may need a moment)" +echo "" + +echo "--- Grafana health ---" +HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" http://10.5.1.6:3000/ 2>/dev/null || echo "000") +echo "Grafana HTTP status: ${HTTP_CODE}" +echo "" + +echo "--- Node exporter ---" +curl -s http://10.5.1.6:9100/metrics 2>/dev/null | head -5 || echo "Node exporter not yet responding" +echo "" + +echo "=== All done! ===" +echo "" +echo "Next steps:" +echo " 1. Add Pangolin resource: grafana.paccoco.com -> http://grafana:3000" +echo " 2. Log in at https://grafana.paccoco.com with admin / ${GF_PASS}" +echo " 3. Import dashboards: Node Exporter Full (ID 1860), Docker Stats (ID 893)" +echo " 4. Set up Grafana -> n8n alert webhook at https://n8n.paccoco.com/webhook/grafana-alert" diff --git a/n8n-workflows/01-grafana-alert-ai-gotify-v1.1.json b/n8n-workflows/01-grafana-alert-ai-gotify-v1.1.json new file mode 100644 index 0000000..6656e2c --- /dev/null +++ b/n8n-workflows/01-grafana-alert-ai-gotify-v1.1.json @@ -0,0 +1,363 @@ +{ + "updatedAt": "2026-05-07T03:32:35.930Z", + "createdAt": "2026-05-07T03:32:32.026Z", + "id": "GXtkp4xKka90XrXH", + "name": "Grafana Alert \u2192 AI \u2192 Gotify v1.1", + "description": null, + "active": true, + "isArchived": false, + "nodes": [ + { + "parameters": { + "httpMethod": "POST", + "path": "grafana-alert", + "options": {} + }, + "id": "a6e7dfb1-8e9b-4048-b028-4383c30a7e0d", + "name": "Grafana Webhook", + "type": "n8n-nodes-base.webhook", + "typeVersion": 2, + "position": [ + 0, + 0 + ], + "webhookId": "0492788e-db4e-4d15-9572-9d6c3efc80c3" + }, + { + "parameters": { + "jsCode": "// Parse Grafana alert payload into a clean summary for the LLM\nconst alerts = $input.first().json.body?.alerts || $input.first().json.alerts || [];\n\nconst parsed = alerts.map(a => ({\n status: a.status || 'unknown',\n alertName: a.labels?.alertname || 'Unnamed Alert',\n host: a.labels?.host || a.labels?.instance || 'unknown host',\n severity: a.labels?.severity || 'warning',\n summary: a.annotations?.summary || '',\n description: a.annotations?.description || '',\n value: a.valueString || '',\n startsAt: a.startsAt || '',\n endsAt: a.endsAt || ''\n}));\n\nconst alertText = parsed.map(a => \n `[${a.status.toUpperCase()}] ${a.alertName} on ${a.host}\\nSeverity: ${a.severity}\\nSummary: ${a.summary}\\nDescription: ${a.description}\\nValue: ${a.value}\\nStarted: ${a.startsAt}`\n).join('\\n---\\n');\n\nreturn [{\n json: {\n alertText,\n alertCount: parsed.length,\n hasResolved: parsed.some(a => a.status === 'resolved'),\n hasFiring: parsed.some(a => a.status === 'firing'),\n parsed\n }\n}];" + }, + "id": "ea216058-69db-42cc-88d0-8f67e1b8951b", + "name": "Parse Alert Payload", + "type": "n8n-nodes-base.code", + "typeVersion": 2, + "position": [ + 224, + 0 + ] + }, + { + "parameters": { + "method": "POST", + "url": "http://10.5.1.6:4000/v1/chat/completions", + "sendHeaders": true, + "headerParameters": { + "parameters": [ + { + "name": "Content-Type", + "value": "application/json" + }, + { + "name": "Authorization", + "value": "Bearer sk-a95bd142d43d175b6476ebc862e81aa1f6ef34b1153f839698d07541d2f55308" + } + ] + }, + "sendBody": true, + "specifyBody": "json", + "jsonBody": "={{\n {\n \"model\": \"medium\",\n \"messages\": [\n {\n \"role\": \"system\",\n \"content\": \"You are a concise homelab monitoring assistant. Summarize infrastructure alerts in 2-3 sentences. Include: what happened, which host, severity, and a suggested action. Keep it brief \u2014 this goes to a phone notification.\"\n },\n {\n \"role\": \"user\",\n \"content\": \"Summarize this Grafana alert:\\n\\n\" + $json.alertText\n }\n ],\n \"max_tokens\": 200,\n \"temperature\": 0.3\n }\n}}", + "options": {} + }, + "id": "8ef1aa9d-d5b1-41ea-abc6-703b6491ea33", + "name": "LiteLLM Summarize", + "type": "n8n-nodes-base.httpRequest", + "typeVersion": 4.2, + "position": [ + 1792, + 0 + ] + }, + { + "parameters": { + "jsCode": "const response = $input.first().json;\nconst summary = response.choices?.[0]?.message?.content || 'Could not generate summary';\nconst alertData = $('Parse Alert Payload').first().json;\n\nconst title = alertData.hasFiring \n ? `\ud83d\udd25 Alert Firing (${alertData.alertCount})` \n : `\u2705 Alert Resolved (${alertData.alertCount})`;\n\nconst priority = alertData.hasFiring ? 8 : 2;\n\nreturn [{\n json: {\n title,\n message: summary,\n priority\n }\n}];" + }, + "id": "2e969bca-8458-4477-a358-664fd1c50c4f", + "name": "Format Notification", + "type": "n8n-nodes-base.code", + "typeVersion": 2, + "position": [ + 2016, + 0 + ] + }, + { + "parameters": { + "message": "={{ $json.message }}", + "additionalFields": { + "priority": "={{ $json.priority }}", + "title": "={{ $json.title }}" + }, + "options": { + "contentType": "text/plain" + } + }, + "id": "65c2b53a-6e03-4460-9aa6-5d967d244de0", + "name": "Push to Gotify", + "type": "n8n-nodes-base.gotify", + "typeVersion": 1, + "position": [ + 2224, + 0 + ], + "credentials": { + "gotifyApi": { + "id": "DXdYZDfVZecDTaNU", + "name": "Gotify account" + } + } + }, + { + "parameters": { + "jsCode": "const parsed = $('Parse Alert Payload').first().json;\nconst now = new Date();\nconst hour = now.getHours();\nconst fingerprint = (parsed.parsed || [])\n .map(a => `${a.alertName}:${a.host}`)\n .sort().join('|') || 'unknown';\nreturn [{ json: { ...parsed, fingerprint, isNightTime: hour >= 0 && hour < 7 } }];\n" + }, + "id": "df26d9ca-c72c-4dee-9327-14a0f5fcf3a0", + "name": "Build Fingerprint", + "type": "n8n-nodes-base.code", + "typeVersion": 2, + "position": [ + 448, + 0 + ] + }, + { + "parameters": { + "operation": "executeQuery", + "query": "SELECT fingerprint, last_seen FROM grafana_alert_dedup WHERE fingerprint = '{{ $json.fingerprint }}' AND last_seen > NOW() - INTERVAL '30 minutes' LIMIT 1;", + "options": {} + }, + "id": "3fd78a53-7885-4739-a4b7-b441a27e993e", + "name": "Lookup Dedup", + "type": "n8n-nodes-base.postgres", + "typeVersion": 2.5, + "position": [ + 672, + 0 + ], + "credentials": { + "postgres": { + "id": "n9svoXemqSZoNNUB", + "name": "Postgres account" + } + }, + "continueOnFail": true + }, + { + "parameters": { + "jsCode": "const alertData = $('Build Fingerprint').first().json;\nconst rows = $input.all().filter(r => r.json && r.json.fingerprint);\nconst alreadySeen = rows.length > 0;\nconst isLowPriority = !alertData.hasFiring;\nconst suppressNight = alertData.isNightTime && isLowPriority;\nconst suppress = alreadySeen || suppressNight;\nreturn [{ json: { ...alertData, suppress, shouldNotify: suppress ? 'no' : 'yes', suppressReason: alreadySeen ? 'dedup' : suppressNight ? 'night_suppress' : null } }];\n" + }, + "id": "c34b4b0e-1a47-4624-b1c4-9ef6b5cdeb6c", + "name": "Check Should Notify", + "type": "n8n-nodes-base.code", + "typeVersion": 2, + "position": [ + 896, + 0 + ] + }, + { + "parameters": { + "conditions": { + "options": { + "caseSensitive": true, + "leftValue": "", + "typeValidation": "strict" + }, + "conditions": [ + { + "id": "cc41d77c-b659-4c6f-8e71-96a13dced242", + "leftValue": "={{ $json.shouldNotify }}", + "rightValue": "yes", + "operator": { + "type": "string", + "operation": "equals" + } + } + ], + "combinator": "and" + } + }, + "id": "29929dc8-17f6-4cc8-90c2-c501eaba8be5", + "name": "Should Notify?", + "type": "n8n-nodes-base.if", + "typeVersion": 2.2, + "position": [ + 1120, + 0 + ] + }, + { + "parameters": { + "operation": "executeQuery", + "query": "INSERT INTO grafana_alert_dedup (fingerprint, last_seen, alert_name, host) VALUES ('{{ $json.fingerprint }}', NOW(), '{{ ($json.parsed && $json.parsed[0]) ? $json.parsed[0].alertName : \"unknown\" }}', '{{ ($json.parsed && $json.parsed[0]) ? $json.parsed[0].host : \"unknown\" }}') ON CONFLICT (fingerprint) DO UPDATE SET last_seen = NOW();", + "options": {} + }, + "id": "25a14a6a-db17-49ca-a6d2-6ed2f97d2889", + "name": "Upsert Dedup Record", + "type": "n8n-nodes-base.postgres", + "typeVersion": 2.5, + "position": [ + 1344, + 0 + ], + "credentials": { + "postgres": { + "id": "n9svoXemqSZoNNUB", + "name": "Postgres account" + } + }, + "continueOnFail": true + } + ], + "connections": { + "Grafana Webhook": { + "main": [ + [ + { + "node": "Parse Alert Payload", + "type": "main", + "index": 0 + } + ] + ] + }, + "Parse Alert Payload": { + "main": [ + [ + { + "node": "Build Fingerprint", + "type": "main", + "index": 0 + } + ] + ] + }, + "Build Fingerprint": { + "main": [ + [ + { + "node": "Lookup Dedup", + "type": "main", + "index": 0 + } + ] + ] + }, + "Lookup Dedup": { + "main": [ + [ + { + "node": "Check Should Notify", + "type": "main", + "index": 0 + } + ] + ] + }, + "Check Should Notify": { + "main": [ + [ + { + "node": "Should Notify?", + "type": "main", + "index": 0 + } + ] + ] + }, + "Should Notify?": { + "main": [ + [ + { + "node": "Upsert Dedup Record", + "type": "main", + "index": 0 + } + ], + [] + ] + }, + "Upsert Dedup Record": { + "main": [ + [ + { + "node": "LiteLLM Summarize", + "type": "main", + "index": 0 + } + ] + ] + }, + "LiteLLM Summarize": { + "main": [ + [ + { + "node": "Format Notification", + "type": "main", + "index": 0 + } + ] + ] + }, + "Format Notification": { + "main": [ + [ + { + "node": "Push to Gotify", + "type": "main", + "index": 0 + } + ] + ] + } + }, + "settings": { + "executionOrder": "v1", + "binaryMode": "separate" + }, + "staticData": null, + "meta": { + "templateCredsSetupCompleted": true + }, + "pinData": {}, + "versionId": "ba3a4eab-60b8-4b1c-ac9f-07a0fdcec9c0", + "activeVersionId": "ba3a4eab-60b8-4b1c-ac9f-07a0fdcec9c0", + "versionCounter": 10, + "triggerCount": 1, + "tags": [ + { + "updatedAt": "2026-05-06T22:38:51.678Z", + "createdAt": "2026-05-06T22:38:51.678Z", + "id": "S9FxzVfHLsHmyzyt", + "name": "ai" + }, + { + "updatedAt": "2026-05-06T22:38:51.660Z", + "createdAt": "2026-05-06T22:38:51.660Z", + "id": "0ETpkL5jJ5wwgt8k", + "name": "monitoring" + } + ], + "shared": [ + { + "updatedAt": "2026-05-07T03:32:32.026Z", + "createdAt": "2026-05-07T03:32:32.026Z", + "role": "workflow:owner", + "workflowId": "GXtkp4xKka90XrXH", + "projectId": "dxCRnBdX5uJizCGa", + "project": { + "updatedAt": "2026-05-06T01:10:37.484Z", + "createdAt": "2026-05-06T01:08:07.721Z", + "id": "dxCRnBdX5uJizCGa", + "name": "Wilfred Fizzlepoof ", + "type": "personal", + "icon": null, + "description": null, + "creatorId": "47b2227f-2fc2-4a08-bd35-ea157b92df0d" + } + } + ], + "versionMetadata": { + "name": "Version ba3a4eab", + "description": "" + } +} \ No newline at end of file diff --git a/n8n-workflows/01-grafana-alert-ai-gotify-v1.2.json b/n8n-workflows/01-grafana-alert-ai-gotify-v1.2.json new file mode 100644 index 0000000..db508f7 --- /dev/null +++ b/n8n-workflows/01-grafana-alert-ai-gotify-v1.2.json @@ -0,0 +1,387 @@ +{ + "updatedAt": "2026-05-07T03:32:35.930Z", + "createdAt": "2026-05-07T03:32:32.026Z", + "id": "GXtkp4xKka90XrXH", + "name": "Grafana Alert \u2192 AI \u2192 Gotify v1.2", + "description": null, + "active": true, + "isArchived": false, + "nodes": [ + { + "parameters": { + "httpMethod": "POST", + "path": "grafana-alert", + "options": {} + }, + "id": "a6e7dfb1-8e9b-4048-b028-4383c30a7e0d", + "name": "Grafana Webhook", + "type": "n8n-nodes-base.webhook", + "typeVersion": 2, + "position": [ + 0, + 0 + ], + "webhookId": "0492788e-db4e-4d15-9572-9d6c3efc80c3" + }, + { + "parameters": { + "jsCode": "// Parse Grafana alert payload into a clean summary for the LLM\nconst alerts = $input.first().json.body?.alerts || $input.first().json.alerts || [];\n\nconst parsed = alerts.map(a => ({\n status: a.status || 'unknown',\n alertName: a.labels?.alertname || 'Unnamed Alert',\n host: a.labels?.host || a.labels?.instance || 'unknown host',\n severity: a.labels?.severity || 'warning',\n summary: a.annotations?.summary || '',\n description: a.annotations?.description || '',\n value: a.valueString || '',\n startsAt: a.startsAt || '',\n endsAt: a.endsAt || ''\n}));\n\nconst alertText = parsed.map(a => \n `[${a.status.toUpperCase()}] ${a.alertName} on ${a.host}\\nSeverity: ${a.severity}\\nSummary: ${a.summary}\\nDescription: ${a.description}\\nValue: ${a.value}\\nStarted: ${a.startsAt}`\n).join('\\n---\\n');\n\nreturn [{\n json: {\n alertText,\n alertCount: parsed.length,\n hasResolved: parsed.some(a => a.status === 'resolved'),\n hasFiring: parsed.some(a => a.status === 'firing'),\n parsed\n }\n}];" + }, + "id": "ea216058-69db-42cc-88d0-8f67e1b8951b", + "name": "Parse Alert Payload", + "type": "n8n-nodes-base.code", + "typeVersion": 2, + "position": [ + 224, + 0 + ] + }, + { + "parameters": { + "method": "POST", + "url": "http://10.5.1.6:4000/v1/chat/completions", + "sendHeaders": true, + "headerParameters": { + "parameters": [ + { + "name": "Content-Type", + "value": "application/json" + }, + { + "name": "Authorization", + "value": "Bearer sk-REPLACE_WITH_LITELLM_KEY" + } + ] + }, + "sendBody": true, + "specifyBody": "json", + "jsonBody": "={{\n {\n \"model\": \"medium\",\n \"messages\": [\n {\n \"role\": \"system\",\n \"content\": \"You are a concise homelab monitoring assistant. Summarize infrastructure alerts in 2-3 sentences. Include: what happened, which host, severity, and a suggested action. Keep it brief \u2014 this goes to a phone notification.\"\n },\n {\n \"role\": \"user\",\n \"content\": \"Summarize this Grafana alert:\\n\\n\" + $('Parse Alert Payload').first().json.alertText\n }\n ],\n \"max_tokens\": 200,\n \"temperature\": 0.3\n }\n}}", + "options": {} + }, + "id": "8ef1aa9d-d5b1-41ea-abc6-703b6491ea33", + "name": "LiteLLM Summarize", + "type": "n8n-nodes-base.httpRequest", + "typeVersion": 4.2, + "position": [ + 1792, + 0 + ] + }, + { + "parameters": { + "jsCode": "const response = $input.first().json;\nconst summary = response.choices?.[0]?.message?.content || 'Could not generate summary';\nconst alertData = $('Parse Alert Payload').first().json;\n\nconst title = alertData.hasFiring \n ? `\ud83d\udd25 Alert Firing (${alertData.alertCount})` \n : `\u2705 Alert Resolved (${alertData.alertCount})`;\n\nconst priority = alertData.hasFiring ? 8 : 2;\n\nreturn [{\n json: {\n title,\n message: summary,\n priority\n }\n}];" + }, + "id": "2e969bca-8458-4477-a358-664fd1c50c4f", + "name": "Format Notification", + "type": "n8n-nodes-base.code", + "typeVersion": 2, + "position": [ + 2016, + 0 + ] + }, + { + "parameters": { + "message": "={{ $json.message }}", + "additionalFields": { + "priority": "={{ $json.priority }}", + "title": "={{ $json.title }}" + }, + "options": { + "contentType": "text/plain" + } + }, + "id": "65c2b53a-6e03-4460-9aa6-5d967d244de0", + "name": "Push to Gotify", + "type": "n8n-nodes-base.gotify", + "typeVersion": 1, + "position": [ + 2224, + 0 + ], + "credentials": { + "gotifyApi": { + "id": "DXdYZDfVZecDTaNU", + "name": "Gotify account" + } + } + }, + { + "parameters": { + "jsCode": "const parsed = $('Parse Alert Payload').first().json;\nconst now = new Date();\nconst hour = now.getHours();\nconst fingerprint = (parsed.parsed || [])\n .map(a => `${a.alertName}:${a.host}`)\n .sort().join('|') || 'unknown';\nreturn [{ json: { ...parsed, fingerprint, isNightTime: hour >= 0 && hour < 7 } }];\n" + }, + "id": "df26d9ca-c72c-4dee-9327-14a0f5fcf3a0", + "name": "Build Fingerprint", + "type": "n8n-nodes-base.code", + "typeVersion": 2, + "position": [ + 448, + 0 + ] + }, + { + "parameters": { + "operation": "executeQuery", + "query": "SELECT fingerprint, last_seen FROM grafana_alert_dedup WHERE fingerprint = '{{ $json.fingerprint }}' AND last_seen > NOW() - INTERVAL '30 minutes' LIMIT 1;", + "options": {} + }, + "id": "3fd78a53-7885-4739-a4b7-b441a27e993e", + "name": "Lookup Dedup", + "type": "n8n-nodes-base.postgres", + "typeVersion": 2.5, + "position": [ + 672, + 0 + ], + "credentials": { + "postgres": { + "id": "n9svoXemqSZoNNUB", + "name": "Postgres account" + } + }, + "continueOnFail": true + }, + { + "parameters": { + "jsCode": "const alertData = $('Build Fingerprint').first().json;\nconst rows = $input.all().filter(r => r.json && r.json.fingerprint);\nconst alreadySeen = rows.length > 0;\nconst isLowPriority = !alertData.hasFiring;\nconst suppressNight = alertData.isNightTime && isLowPriority;\nconst suppress = alreadySeen || suppressNight;\nreturn [{ json: { ...alertData, suppress, shouldNotify: suppress ? 'no' : 'yes', suppressReason: alreadySeen ? 'dedup' : suppressNight ? 'night_suppress' : null } }];\n" + }, + "id": "c34b4b0e-1a47-4624-b1c4-9ef6b5cdeb6c", + "name": "Check Should Notify", + "type": "n8n-nodes-base.code", + "typeVersion": 2, + "position": [ + 896, + 0 + ] + }, + { + "parameters": { + "conditions": { + "options": { + "caseSensitive": true, + "leftValue": "", + "typeValidation": "strict" + }, + "conditions": [ + { + "id": "cc41d77c-b659-4c6f-8e71-96a13dced242", + "leftValue": "={{ $json.shouldNotify }}", + "rightValue": "yes", + "operator": { + "type": "string", + "operation": "equals" + } + } + ], + "combinator": "and" + } + }, + "id": "29929dc8-17f6-4cc8-90c2-c501eaba8be5", + "name": "Should Notify?", + "type": "n8n-nodes-base.if", + "typeVersion": 2.2, + "position": [ + 1120, + 0 + ] + }, + { + "parameters": { + "operation": "executeQuery", + "query": "INSERT INTO grafana_alert_dedup (fingerprint, last_seen, alert_name, host) VALUES ('{{ $json.fingerprint }}', NOW(), '{{ $json.dedupAlertName }}', '{{ $json.dedupHost }}') ON CONFLICT (fingerprint) DO UPDATE SET last_seen = NOW();", + "options": {} + }, + "id": "25a14a6a-db17-49ca-a6d2-6ed2f97d2889", + "name": "Upsert Dedup Record", + "type": "n8n-nodes-base.postgres", + "typeVersion": 2.5, + "position": [ + 1568, + 0 + ], + "credentials": { + "postgres": { + "id": "n9svoXemqSZoNNUB", + "name": "Postgres account" + } + }, + "continueOnFail": true + }, + { + "parameters": { + "jsCode": "const d = $input.first().json;\nconst first = (d.parsed || [])[0] || {};\nreturn [{ json: {\n fingerprint: d.fingerprint,\n dedupAlertName: first.alertName || 'unknown',\n dedupHost: first.host || 'unknown'\n} }];" + }, + "id": "54705c76-512d-4479-889f-dc0a3c24d1b3", + "name": "Prepare Dedup Data", + "type": "n8n-nodes-base.code", + "typeVersion": 2, + "position": [ + 1344, + 0 + ] + } + ], + "connections": { + "Grafana Webhook": { + "main": [ + [ + { + "node": "Parse Alert Payload", + "type": "main", + "index": 0 + } + ] + ] + }, + "Parse Alert Payload": { + "main": [ + [ + { + "node": "Build Fingerprint", + "type": "main", + "index": 0 + } + ] + ] + }, + "Build Fingerprint": { + "main": [ + [ + { + "node": "Lookup Dedup", + "type": "main", + "index": 0 + } + ] + ] + }, + "Lookup Dedup": { + "main": [ + [ + { + "node": "Check Should Notify", + "type": "main", + "index": 0 + } + ] + ] + }, + "Check Should Notify": { + "main": [ + [ + { + "node": "Should Notify?", + "type": "main", + "index": 0 + } + ] + ] + }, + "Should Notify?": { + "main": [ + [ + { + "node": "Prepare Dedup Data", + "type": "main", + "index": 0 + } + ], + [] + ] + }, + "Upsert Dedup Record": { + "main": [ + [ + { + "node": "LiteLLM Summarize", + "type": "main", + "index": 0 + } + ] + ] + }, + "LiteLLM Summarize": { + "main": [ + [ + { + "node": "Format Notification", + "type": "main", + "index": 0 + } + ] + ] + }, + "Format Notification": { + "main": [ + [ + { + "node": "Push to Gotify", + "type": "main", + "index": 0 + } + ] + ] + }, + "Prepare Dedup Data": { + "main": [ + [ + { + "node": "Upsert Dedup Record", + "type": "main", + "index": 0 + } + ] + ] + } + }, + "settings": { + "executionOrder": "v1", + "binaryMode": "separate" + }, + "staticData": null, + "meta": { + "templateCredsSetupCompleted": true + }, + "pinData": {}, + "versionId": "ba3a4eab-60b8-4b1c-ac9f-07a0fdcec9c0", + "activeVersionId": "ba3a4eab-60b8-4b1c-ac9f-07a0fdcec9c0", + "versionCounter": 10, + "triggerCount": 1, + "tags": [ + { + "updatedAt": "2026-05-06T22:38:51.678Z", + "createdAt": "2026-05-06T22:38:51.678Z", + "id": "S9FxzVfHLsHmyzyt", + "name": "ai" + }, + { + "updatedAt": "2026-05-06T22:38:51.660Z", + "createdAt": "2026-05-06T22:38:51.660Z", + "id": "0ETpkL5jJ5wwgt8k", + "name": "monitoring" + } + ], + "shared": [ + { + "updatedAt": "2026-05-07T03:32:32.026Z", + "createdAt": "2026-05-07T03:32:32.026Z", + "role": "workflow:owner", + "workflowId": "GXtkp4xKka90XrXH", + "projectId": "dxCRnBdX5uJizCGa", + "project": { + "updatedAt": "2026-05-06T01:10:37.484Z", + "createdAt": "2026-05-06T01:08:07.721Z", + "id": "dxCRnBdX5uJizCGa", + "name": "Wilfred Fizzlepoof ", + "type": "personal", + "icon": null, + "description": null, + "creatorId": "47b2227f-2fc2-4a08-bd35-ea157b92df0d" + } + } + ], + "versionMetadata": { + "name": "Version ba3a4eab", + "description": "" + } +} \ No newline at end of file diff --git a/n8n-workflows/01-grafana-alert-ai-gotify-v1.3.json b/n8n-workflows/01-grafana-alert-ai-gotify-v1.3.json new file mode 100644 index 0000000..be8eb38 --- /dev/null +++ b/n8n-workflows/01-grafana-alert-ai-gotify-v1.3.json @@ -0,0 +1,387 @@ +{ + "updatedAt": "2026-05-07T03:32:35.930Z", + "createdAt": "2026-05-07T03:32:32.026Z", + "id": "GXtkp4xKka90XrXH", + "name": "Grafana Alert \u2192 AI \u2192 Gotify v1.3", + "description": null, + "active": true, + "isArchived": false, + "nodes": [ + { + "parameters": { + "httpMethod": "POST", + "path": "grafana-alert", + "options": {} + }, + "id": "a6e7dfb1-8e9b-4048-b028-4383c30a7e0d", + "name": "Grafana Webhook", + "type": "n8n-nodes-base.webhook", + "typeVersion": 2, + "position": [ + 0, + 0 + ], + "webhookId": "0492788e-db4e-4d15-9572-9d6c3efc80c3" + }, + { + "parameters": { + "jsCode": "// Parse Grafana alert payload into a clean summary for the LLM\nconst alerts = $input.first().json.body?.alerts || $input.first().json.alerts || [];\n\nconst parsed = alerts.map(a => ({\n status: a.status || 'unknown',\n alertName: a.labels?.alertname || 'Unnamed Alert',\n host: a.labels?.host || a.labels?.instance || 'unknown host',\n severity: a.labels?.severity || 'warning',\n summary: a.annotations?.summary || '',\n description: a.annotations?.description || '',\n value: a.valueString || '',\n startsAt: a.startsAt || '',\n endsAt: a.endsAt || ''\n}));\n\nconst alertText = parsed.map(a => \n `[${a.status.toUpperCase()}] ${a.alertName} on ${a.host}\\nSeverity: ${a.severity}\\nSummary: ${a.summary}\\nDescription: ${a.description}\\nValue: ${a.value}\\nStarted: ${a.startsAt}`\n).join('\\n---\\n');\n\nreturn [{\n json: {\n alertText,\n alertCount: parsed.length,\n hasResolved: parsed.some(a => a.status === 'resolved'),\n hasFiring: parsed.some(a => a.status === 'firing'),\n parsed\n }\n}];" + }, + "id": "ea216058-69db-42cc-88d0-8f67e1b8951b", + "name": "Parse Alert Payload", + "type": "n8n-nodes-base.code", + "typeVersion": 2, + "position": [ + 224, + 0 + ] + }, + { + "parameters": { + "method": "POST", + "url": "http://10.5.1.6:4000/v1/chat/completions", + "sendHeaders": true, + "headerParameters": { + "parameters": [ + { + "name": "Content-Type", + "value": "application/json" + }, + { + "name": "Authorization", + "value": "Bearer {{ $env.LITELLM_API_KEY }}" + } + ] + }, + "sendBody": true, + "specifyBody": "json", + "jsonBody": "={{\n {\n \"model\": \"medium\",\n \"messages\": [\n {\n \"role\": \"system\",\n \"content\": \"You are a concise homelab monitoring assistant. Summarize infrastructure alerts in 2-3 sentences. Include: what happened, which host, severity, and a suggested action. Keep it brief \u2014 this goes to a phone notification.\"\n },\n {\n \"role\": \"user\",\n \"content\": \"Summarize this Grafana alert:\\n\\n\" + $('Parse Alert Payload').first().json.alertText\n }\n ],\n \"max_tokens\": 200,\n \"temperature\": 0.3\n }\n}}", + "options": {} + }, + "id": "8ef1aa9d-d5b1-41ea-abc6-703b6491ea33", + "name": "LiteLLM Summarize", + "type": "n8n-nodes-base.httpRequest", + "typeVersion": 4.2, + "position": [ + 1792, + 0 + ] + }, + { + "parameters": { + "jsCode": "const response = $input.first().json;\nconst summary = response.choices?.[0]?.message?.content || 'Could not generate summary';\nconst alertData = $('Parse Alert Payload').first().json;\n\nconst title = alertData.hasFiring \n ? `\ud83d\udd25 Alert Firing (${alertData.alertCount})` \n : `\u2705 Alert Resolved (${alertData.alertCount})`;\n\nconst priority = alertData.hasFiring ? 8 : 2;\n\nreturn [{\n json: {\n title,\n message: summary,\n priority\n }\n}];" + }, + "id": "2e969bca-8458-4477-a358-664fd1c50c4f", + "name": "Format Notification", + "type": "n8n-nodes-base.code", + "typeVersion": 2, + "position": [ + 2016, + 0 + ] + }, + { + "parameters": { + "message": "={{ $json.message }}", + "additionalFields": { + "priority": "={{ $json.priority }}", + "title": "={{ $json.title }}" + }, + "options": { + "contentType": "text/plain" + } + }, + "id": "65c2b53a-6e03-4460-9aa6-5d967d244de0", + "name": "Push to Gotify", + "type": "n8n-nodes-base.gotify", + "typeVersion": 1, + "position": [ + 2224, + 0 + ], + "credentials": { + "gotifyApi": { + "id": "DXdYZDfVZecDTaNU", + "name": "Gotify account" + } + } + }, + { + "parameters": { + "jsCode": "const parsed = $('Parse Alert Payload').first().json;\nconst now = new Date();\nconst hour = now.getHours();\nconst fingerprint = (parsed.parsed || [])\n .map(a => `${a.alertName}:${a.host}`)\n .sort().join('|') || 'unknown';\nreturn [{ json: { ...parsed, fingerprint, isNightTime: hour >= 0 && hour < 7 } }];\n" + }, + "id": "df26d9ca-c72c-4dee-9327-14a0f5fcf3a0", + "name": "Build Fingerprint", + "type": "n8n-nodes-base.code", + "typeVersion": 2, + "position": [ + 448, + 0 + ] + }, + { + "parameters": { + "operation": "executeQuery", + "query": "={{ 'SELECT fingerprint, last_seen FROM grafana_alert_dedup WHERE fingerprint = \\'' + $json.fingerprint + '\\' AND last_seen > NOW() - INTERVAL \\'30 minutes\\' LIMIT 1;' }}", + "options": {} + }, + "id": "3fd78a53-7885-4739-a4b7-b441a27e993e", + "name": "Lookup Dedup", + "type": "n8n-nodes-base.postgres", + "typeVersion": 2.5, + "position": [ + 672, + 0 + ], + "credentials": { + "postgres": { + "id": "n9svoXemqSZoNNUB", + "name": "Postgres account" + } + }, + "continueOnFail": true + }, + { + "parameters": { + "jsCode": "const alertData = $('Build Fingerprint').first().json;\nconst rows = $input.all().filter(r => r.json && r.json.fingerprint);\nconst alreadySeen = rows.length > 0;\nconst isLowPriority = !alertData.hasFiring;\nconst suppressNight = alertData.isNightTime && isLowPriority;\nconst suppress = alreadySeen || suppressNight;\nreturn [{ json: { ...alertData, suppress, shouldNotify: suppress ? 'no' : 'yes', suppressReason: alreadySeen ? 'dedup' : suppressNight ? 'night_suppress' : null } }];\n" + }, + "id": "c34b4b0e-1a47-4624-b1c4-9ef6b5cdeb6c", + "name": "Check Should Notify", + "type": "n8n-nodes-base.code", + "typeVersion": 2, + "position": [ + 896, + 0 + ] + }, + { + "parameters": { + "conditions": { + "options": { + "caseSensitive": true, + "leftValue": "", + "typeValidation": "strict" + }, + "conditions": [ + { + "id": "cc41d77c-b659-4c6f-8e71-96a13dced242", + "leftValue": "={{ $json.shouldNotify }}", + "rightValue": "yes", + "operator": { + "type": "string", + "operation": "equals" + } + } + ], + "combinator": "and" + } + }, + "id": "29929dc8-17f6-4cc8-90c2-c501eaba8be5", + "name": "Should Notify?", + "type": "n8n-nodes-base.if", + "typeVersion": 2.2, + "position": [ + 1120, + 0 + ] + }, + { + "parameters": { + "operation": "executeQuery", + "query": "={{ 'INSERT INTO grafana_alert_dedup (fingerprint, last_seen, alert_name, host) VALUES (\\'' + $json.fingerprint + '\\', NOW(), \\'' + $json.dedupAlertName + '\\', \\'' + $json.dedupHost + '\\') ON CONFLICT (fingerprint) DO UPDATE SET last_seen = NOW();' }}", + "options": {} + }, + "id": "25a14a6a-db17-49ca-a6d2-6ed2f97d2889", + "name": "Upsert Dedup Record", + "type": "n8n-nodes-base.postgres", + "typeVersion": 2.5, + "position": [ + 1568, + 0 + ], + "credentials": { + "postgres": { + "id": "n9svoXemqSZoNNUB", + "name": "Postgres account" + } + }, + "continueOnFail": true + }, + { + "parameters": { + "jsCode": "const d = $input.first().json;\nconst first = (d.parsed || [])[0] || {};\nreturn [{ json: {\n fingerprint: d.fingerprint,\n dedupAlertName: first.alertName || 'unknown',\n dedupHost: first.host || 'unknown'\n} }];" + }, + "id": "54705c76-512d-4479-889f-dc0a3c24d1b3", + "name": "Prepare Dedup Data", + "type": "n8n-nodes-base.code", + "typeVersion": 2, + "position": [ + 1344, + 0 + ] + } + ], + "connections": { + "Grafana Webhook": { + "main": [ + [ + { + "node": "Parse Alert Payload", + "type": "main", + "index": 0 + } + ] + ] + }, + "Parse Alert Payload": { + "main": [ + [ + { + "node": "Build Fingerprint", + "type": "main", + "index": 0 + } + ] + ] + }, + "Build Fingerprint": { + "main": [ + [ + { + "node": "Lookup Dedup", + "type": "main", + "index": 0 + } + ] + ] + }, + "Lookup Dedup": { + "main": [ + [ + { + "node": "Check Should Notify", + "type": "main", + "index": 0 + } + ] + ] + }, + "Check Should Notify": { + "main": [ + [ + { + "node": "Should Notify?", + "type": "main", + "index": 0 + } + ] + ] + }, + "Should Notify?": { + "main": [ + [ + { + "node": "Prepare Dedup Data", + "type": "main", + "index": 0 + } + ], + [] + ] + }, + "Upsert Dedup Record": { + "main": [ + [ + { + "node": "LiteLLM Summarize", + "type": "main", + "index": 0 + } + ] + ] + }, + "LiteLLM Summarize": { + "main": [ + [ + { + "node": "Format Notification", + "type": "main", + "index": 0 + } + ] + ] + }, + "Format Notification": { + "main": [ + [ + { + "node": "Push to Gotify", + "type": "main", + "index": 0 + } + ] + ] + }, + "Prepare Dedup Data": { + "main": [ + [ + { + "node": "Upsert Dedup Record", + "type": "main", + "index": 0 + } + ] + ] + } + }, + "settings": { + "executionOrder": "v1", + "binaryMode": "separate" + }, + "staticData": null, + "meta": { + "templateCredsSetupCompleted": true + }, + "pinData": {}, + "versionId": "ba3a4eab-60b8-4b1c-ac9f-07a0fdcec9c0", + "activeVersionId": "ba3a4eab-60b8-4b1c-ac9f-07a0fdcec9c0", + "versionCounter": 10, + "triggerCount": 1, + "tags": [ + { + "updatedAt": "2026-05-06T22:38:51.678Z", + "createdAt": "2026-05-06T22:38:51.678Z", + "id": "S9FxzVfHLsHmyzyt", + "name": "ai" + }, + { + "updatedAt": "2026-05-06T22:38:51.660Z", + "createdAt": "2026-05-06T22:38:51.660Z", + "id": "0ETpkL5jJ5wwgt8k", + "name": "monitoring" + } + ], + "shared": [ + { + "updatedAt": "2026-05-07T03:32:32.026Z", + "createdAt": "2026-05-07T03:32:32.026Z", + "role": "workflow:owner", + "workflowId": "GXtkp4xKka90XrXH", + "projectId": "dxCRnBdX5uJizCGa", + "project": { + "updatedAt": "2026-05-06T01:10:37.484Z", + "createdAt": "2026-05-06T01:08:07.721Z", + "id": "dxCRnBdX5uJizCGa", + "name": "Wilfred Fizzlepoof ", + "type": "personal", + "icon": null, + "description": null, + "creatorId": "47b2227f-2fc2-4a08-bd35-ea157b92df0d" + } + } + ], + "versionMetadata": { + "name": "Version ba3a4eab", + "description": "" + } +} \ No newline at end of file diff --git a/n8n-workflows/04-whisper-transcription-v1.1.json b/n8n-workflows/04-whisper-transcription-v1.1.json new file mode 100644 index 0000000..0d3315b --- /dev/null +++ b/n8n-workflows/04-whisper-transcription-v1.1.json @@ -0,0 +1,406 @@ +{ + "name": "Whisper Transcription v1.1", + "active": true, + "isArchived": false, + "nodes": [ + { + "parameters": { + "httpMethod": "POST", + "path": "transcribe", + "responseMode": "lastNode", + "responseData": "allEntries", + "options": { + "rawBody": true + } + }, + "id": "91945dd9-42f8-4343-b131-9e46272f04bf", + "name": "Transcription Webhook", + "type": "n8n-nodes-base.webhook", + "typeVersion": 2, + "position": [0, 0], + "webhookId": "f50eec94-1e56-4189-88ee-51b701c1d889" + }, + { + "parameters": { + "method": "POST", + "url": "http://10.5.1.16:8786/v1/audio/transcriptions", + "sendHeaders": true, + "headerParameters": { + "parameters": [ + { + "name": "Accept", + "value": "application/json" + } + ] + }, + "sendBody": true, + "contentType": "multipart-form-data", + "bodyParameters": { + "parameters": [ + { + "parameterType": "formBinaryData", + "name": "file", + "inputDataFieldName": "data" + }, + { + "name": "model", + "value": "Systran/faster-distil-whisper-small.en" + }, + { + "name": "language", + "value": "en" + }, + { + "name": "response_format", + "value": "verbose_json" + } + ] + }, + "options": {} + }, + "id": "fc726196-f62d-4ff4-9be6-a84b9dbfff9e", + "name": "Whisper Transcribe", + "type": "n8n-nodes-base.httpRequest", + "typeVersion": 4.2, + "position": [224, 0] + }, + { + "parameters": { + "jsCode": "const result = $input.first().json;\n\nconst segments = result.segments || [];\nconst fullText = result.text || segments.map(s => s.text).join(' ');\nconst duration = result.duration || segments[segments.length - 1]?.end || 0;\n\n// Format timestamps\nconst timestamped = segments.map(s => {\n const start = new Date(s.start * 1000).toISOString().substr(11, 8);\n return `[${start}] ${s.text.trim()}`;\n}).join('\\n');\n\nreturn [{\n json: {\n text: fullText.trim(),\n timestamped_text: timestamped,\n duration_seconds: Math.round(duration),\n segment_count: segments.length,\n language: result.language || 'en'\n }\n}];" + }, + "id": "1b6f017f-849d-456f-9e4b-a0bced1d8e9f", + "name": "Format Transcript", + "type": "n8n-nodes-base.code", + "typeVersion": 2, + "position": [448, 0] + }, + { + "parameters": { + "httpMethod": "POST", + "path": "transcribe/summarize", + "responseMode": "lastNode", + "responseData": "allEntries", + "options": { + "rawBody": true + } + }, + "id": "c2f8a2dd-cbbc-4b62-ae24-0f4a95bdf862", + "name": "Transcribe + Summarize Webhook", + "type": "n8n-nodes-base.webhook", + "typeVersion": 2, + "position": [0, 304], + "webhookId": "69b4b2e8-9486-4aa3-b9db-fb8e6bdae98e" + }, + { + "parameters": { + "method": "POST", + "url": "http://10.5.1.16:8786/v1/audio/transcriptions", + "sendHeaders": true, + "headerParameters": { + "parameters": [ + { + "name": "Accept", + "value": "application/json" + } + ] + }, + "sendBody": true, + "contentType": "multipart-form-data", + "bodyParameters": { + "parameters": [ + { + "parameterType": "formBinaryData", + "name": "file", + "inputDataFieldName": "data" + }, + { + "name": "model", + "value": "Systran/faster-distil-whisper-small.en" + }, + { + "name": "language", + "value": "en" + } + ] + }, + "options": {} + }, + "id": "ee25bf40-99ef-472b-928d-6ef63516094c", + "name": "Whisper Transcribe 2", + "type": "n8n-nodes-base.httpRequest", + "typeVersion": 4.2, + "position": [224, 304] + }, + { + "parameters": { + "method": "POST", + "url": "http://10.5.1.6:4000/v1/chat/completions", + "sendHeaders": true, + "headerParameters": { + "parameters": [ + { + "name": "Content-Type", + "value": "application/json" + }, + { + "name": "Authorization", + "value": "=Bearer {{ $env.LITELLM_API_KEY }}" + } + ] + }, + "sendBody": true, + "specifyBody": "json", + "jsonBody": "={\n \"model\": \"medium\",\n \"messages\": [\n {\n \"role\": \"system\",\n \"content\": \"You are a meeting/audio note summarizer. Given a transcription, provide:\\n1. A brief summary (2-3 sentences)\\n2. Key points as a bullet list\\n3. Any action items mentioned\\n\\nRespond in JSON with keys: summary, key_points (array), action_items (array)\"\n },\n {\n \"role\": \"user\",\n \"content\": \"Summarize this transcription:\\n\\n{{ $json.text }}\"\n }\n ],\n \"max_tokens\": 500,\n \"temperature\": 0.2,\n \"response_format\": { \"type\": \"json_object\" }\n}", + "options": {} + }, + "id": "eda24425-dc26-4b4b-858e-9ae0d5e1a5e4", + "name": "AI Summarize Audio", + "type": "n8n-nodes-base.httpRequest", + "typeVersion": 4.2, + "position": [448, 304] + }, + { + "parameters": { + "jsCode": "const llmResponse = $input.first().json;\nconst transcript = $('Whisper Transcribe 2').first().json;\nlet analysis;\n\ntry {\n analysis = JSON.parse(llmResponse.choices?.[0]?.message?.content || '{}');\n} catch(e) {\n analysis = { summary: llmResponse.choices?.[0]?.message?.content || 'Summary failed', key_points: [], action_items: [] };\n}\n\nreturn [{\n json: {\n transcript: transcript.text || '',\n duration_seconds: Math.round(transcript.duration || 0),\n language: transcript.language || 'en',\n summary: analysis.summary,\n key_points: analysis.key_points || [],\n action_items: analysis.action_items || []\n }\n}];" + }, + "id": "cf41669b-8bf7-433b-b390-10765954b0a9", + "name": "Format Summary Response", + "type": "n8n-nodes-base.code", + "typeVersion": 2, + "position": [672, 304] + }, + { + "parameters": { + "jsCode": "const t = $input.first().json;\nconst mins = Math.floor(t.duration_seconds / 60);\nconst secs = t.duration_seconds % 60;\nreturn [{ json: {\n title: '🎙️ Transcription Complete',\n message: `Duration: ${mins}m ${secs}s · ${t.segment_count} segments · Language: ${t.language}`,\n priority: 3\n}}];\n" + }, + "id": "5c692355-2a46-4c66-860a-c02a21c73597", + "name": "Format Transcription Notify", + "type": "n8n-nodes-base.code", + "typeVersion": 2, + "position": [672, 0] + }, + { + "parameters": { + "message": "={{ $json.message }}", + "additionalFields": { + "priority": "={{ $json.priority }}", + "title": "={{ $json.title }}" + }, + "options": { + "contentType": "text/plain" + } + }, + "id": "0e96240d-50e2-49f7-be62-4f57451e0543", + "name": "Notify Transcription Done", + "type": "n8n-nodes-base.gotify", + "typeVersion": 1, + "position": [896, 0], + "credentials": { + "gotifyApi": { + "id": "SETUP_REQUIRED", + "name": "Whisper" + } + } + }, + { + "parameters": { + "jsCode": "const t = $('Format Transcript').first().json;\nconst title = `Transcript ${new Date().toLocaleDateString('en-US')}`;\nconst content = `Transcript\\nDate: ${new Date().toISOString()}\\nDuration: ${t.duration_seconds}s\\nLanguage: ${t.language}\\n\\n${t.timestamped_text || t.text}`;\nreturn [{ binary: { document: { data: Buffer.from(content,'utf-8').toString('base64'), mimeType:'text/plain', fileName: `${title}.txt` }}, json: { title } }];\n" + }, + "id": "da9f4d8d-8c5b-413a-badd-71287dbe0a3c", + "name": "Prepare Transcript Doc", + "type": "n8n-nodes-base.code", + "typeVersion": 2, + "position": [1120, 0] + }, + { + "parameters": { + "method": "POST", + "url": "https://paperless.paccoco.com/api/documents/post_document/", + "sendHeaders": true, + "headerParameters": { + "parameters": [ + { + "name": "Authorization", + "value": "=Token {{ $env.PAPERLESS_API_TOKEN }}" + } + ] + }, + "sendBody": true, + "contentType": "multipart-form-data", + "bodyParameters": { + "parameters": [ + { + "parameterType": "formBinaryData", + "name": "document", + "inputDataFieldName": "document" + }, + { + "name": "title", + "value": "={{ $json.title }}" + }, + { + "name": "tags[]", + "value": "={{ $env.PAPERLESS_TAG_TRANSCRIPTION }}" + } + ] + }, + "options": { + "response": { + "response": { + "responseFormat": "text" + } + } + } + }, + "id": "7c84da5a-41e2-4b38-9e58-19acfd73f81d", + "name": "Save Transcript to Paperless", + "type": "n8n-nodes-base.httpRequest", + "typeVersion": 4.2, + "position": [1344, 0], + "continueOnFail": true + }, + { + "parameters": { + "jsCode": "const s = $input.first().json;\nconst mins = Math.floor(s.duration_seconds / 60);\nconst points = (s.key_points || []).slice(0,3).map((p,i) => `${i+1}. ${p}`).join('\\n');\nreturn [{ json: {\n title: '🎙️ Transcription + Summary Complete',\n message: `${mins}m audio\\n\\nSummary: ${s.summary}\\n\\nKey points:\\n${points}`,\n priority: 4\n}}];\n" + }, + "id": "e55363e4-ff9b-4df4-b40a-fd23227c23e8", + "name": "Format Summary Notify", + "type": "n8n-nodes-base.code", + "typeVersion": 2, + "position": [896, 304] + }, + { + "parameters": { + "message": "={{ $json.message }}", + "additionalFields": { + "priority": "={{ $json.priority }}", + "title": "={{ $json.title }}" + }, + "options": { + "contentType": "text/plain" + } + }, + "id": "b5defea9-35a0-4a22-8769-ed9496a022f6", + "name": "Notify Summary Done", + "type": "n8n-nodes-base.gotify", + "typeVersion": 1, + "position": [1120, 304], + "credentials": { + "gotifyApi": { + "id": "SETUP_REQUIRED", + "name": "Whisper" + } + } + }, + { + "parameters": { + "jsCode": "const s = $('Format Summary Response').first().json;\nconst title = `Audio Summary ${new Date().toLocaleDateString('en-US')}`;\nconst points = (s.key_points || []).map((p,i) => `${i+1}. ${p}`).join('\\n');\nconst actions = (s.action_items || []).map((a,i) => `${i+1}. ${a}`).join('\\n');\nconst content = [\n `Audio Transcription & Summary`, `Date: ${new Date().toISOString()}`,\n `Duration: ${Math.floor(s.duration_seconds/60)}m ${s.duration_seconds%60}s`,\n `Language: ${s.language}`, '', `Summary`, `=======`, s.summary,\n '', `Key Points`, `==========`, points,\n ...(actions ? ['', `Action Items`, `============`, actions] : []),\n '', `Full Transcript`, `==============`, s.transcript\n].join('\\n');\nreturn [{ binary: { document: { data: Buffer.from(content,'utf-8').toString('base64'), mimeType:'text/plain', fileName:`${title}.txt` }}, json: { title } }];\n" + }, + "id": "1f31f608-7985-4a9a-afcf-6cf4e44150ba", + "name": "Prepare Summary Doc", + "type": "n8n-nodes-base.code", + "typeVersion": 2, + "position": [1344, 304] + }, + { + "parameters": { + "method": "POST", + "url": "https://paperless.paccoco.com/api/documents/post_document/", + "sendHeaders": true, + "headerParameters": { + "parameters": [ + { + "name": "Authorization", + "value": "=Token {{ $env.PAPERLESS_API_TOKEN }}" + } + ] + }, + "sendBody": true, + "contentType": "multipart-form-data", + "bodyParameters": { + "parameters": [ + { + "parameterType": "formBinaryData", + "name": "document", + "inputDataFieldName": "document" + }, + { + "name": "title", + "value": "={{ $json.title }}" + }, + { + "name": "tags[]", + "value": "={{ $env.PAPERLESS_TAG_TRANSCRIPTION }}" + } + ] + }, + "options": { + "response": { + "response": { + "responseFormat": "text" + } + } + } + }, + "id": "e9eaee2f-d08e-4a76-af53-ed824ebfe68f", + "name": "Save Summary to Paperless", + "type": "n8n-nodes-base.httpRequest", + "typeVersion": 4.2, + "position": [1568, 304], + "continueOnFail": true + } + ], + "connections": { + "Transcription Webhook": { + "main": [[{ "node": "Whisper Transcribe", "type": "main", "index": 0 }]] + }, + "Whisper Transcribe": { + "main": [[{ "node": "Format Transcript", "type": "main", "index": 0 }]] + }, + "Transcribe + Summarize Webhook": { + "main": [[{ "node": "Whisper Transcribe 2", "type": "main", "index": 0 }]] + }, + "Whisper Transcribe 2": { + "main": [[{ "node": "AI Summarize Audio", "type": "main", "index": 0 }]] + }, + "AI Summarize Audio": { + "main": [[{ "node": "Format Summary Response", "type": "main", "index": 0 }]] + }, + "Format Transcript": { + "main": [[{ "node": "Format Transcription Notify", "type": "main", "index": 0 }]] + }, + "Format Transcription Notify": { + "main": [[{ "node": "Notify Transcription Done", "type": "main", "index": 0 }]] + }, + "Notify Transcription Done": { + "main": [[{ "node": "Prepare Transcript Doc", "type": "main", "index": 0 }]] + }, + "Prepare Transcript Doc": { + "main": [[{ "node": "Save Transcript to Paperless", "type": "main", "index": 0 }]] + }, + "Format Summary Response": { + "main": [[{ "node": "Format Summary Notify", "type": "main", "index": 0 }]] + }, + "Format Summary Notify": { + "main": [[{ "node": "Notify Summary Done", "type": "main", "index": 0 }]] + }, + "Notify Summary Done": { + "main": [[{ "node": "Prepare Summary Doc", "type": "main", "index": 0 }]] + }, + "Prepare Summary Doc": { + "main": [[{ "node": "Save Summary to Paperless", "type": "main", "index": 0 }]] + } + }, + "settings": { + "executionOrder": "v1", + "binaryMode": "separate" + }, + "staticData": null, + "meta": null, + "pinData": {}, + "tags": [ + { "name": "transcription" }, + { "name": "whisper" }, + { "name": "ai" } + ] +} diff --git a/n8n-workflows/08-class-recording-rag-v1.2.json b/n8n-workflows/08-class-recording-rag-v1.2.json new file mode 100644 index 0000000..ac40158 --- /dev/null +++ b/n8n-workflows/08-class-recording-rag-v1.2.json @@ -0,0 +1,538 @@ +{ + "updatedAt": "2026-05-07T01:16:50.340Z", + "createdAt": "2026-05-07T01:16:50.340Z", + "id": "91I278E72FAaQenD", + "name": "Class Recording \u2192 RAG v1.2", + "description": null, + "active": true, + "isArchived": false, + "nodes": [ + { + "parameters": { + "httpMethod": "POST", + "path": "class/upload", + "responseMode": "lastNode", + "responseData": "allEntries", + "options": { + "rawBody": true + } + }, + "id": "e475f17b-8560-4376-852e-a613ba6b300e", + "name": "Class Upload Webhook", + "type": "n8n-nodes-base.webhook", + "typeVersion": 2, + "position": [ + 0, + 0 + ], + "webhookId": "d0ee8226-db56-477e-8a3f-572a9129aea9" + }, + { + "parameters": { + "jsCode": "const data = $input.first().json;\nconst body = data.body || data;\n\n// Class name comes from:\n// 1. Form field 'class_name' in multipart upload\n// 2. Query parameter ?class=CS101\n// 3. Header X-Class-Name\nconst className = body.class_name || data.headers?.['x-class-name'] || data.query?.class || 'unclassified';\n\n// Lecture title from:\n// 1. Form field 'lecture_title'\n// 2. Query param ?title=...\n// 3. Header X-Lecture-Title\n// 4. Form field 'filename' (original filename)\nconst rawTitle = body.lecture_title || data.headers?.['x-lecture-title'] || data.query?.title || body.filename || 'Untitled Lecture';\nconst lectureTitle = rawTitle.replace(/\\.(wav|mp3|m4a|ogg)$/i, '').replace(/[-_]/g, ' ');\n\nconst classSlug = className.toLowerCase().replace(/[^a-z0-9]+/g, '-').replace(/^-|-$/g, '');\n\nreturn [{\n json: {\n className,\n classSlug,\n lectureTitle,\n uploadedAt: new Date().toISOString()\n },\n binary: $input.first().binary\n}];" + }, + "id": "24e4a726-ab50-4767-a144-1751616c781d", + "name": "Extract Class Info", + "type": "n8n-nodes-base.code", + "typeVersion": 2, + "position": [ + 224, + 0 + ] + }, + { + "parameters": { + "method": "POST", + "url": "http://10.5.1.16:8786/v1/audio/transcriptions", + "sendHeaders": true, + "headerParameters": { + "parameters": [ + { + "name": "Accept", + "value": "application/json" + } + ] + }, + "sendBody": true, + "contentType": "multipart-form-data", + "bodyParameters": { + "parameters": [ + { + "parameterType": "formBinaryData", + "name": "file", + "inputDataFieldName": "data" + }, + { + "name": "model", + "value": "Systran/faster-distil-whisper-small.en" + }, + { + "name": "language", + "value": "en" + }, + { + "name": "response_format", + "value": "verbose_json" + } + ] + }, + "options": { + "timeout": 3600000 + } + }, + "id": "fcd1062d-8f60-4214-85ee-f53b914abcbb", + "name": "Whisper Transcribe", + "type": "n8n-nodes-base.httpRequest", + "typeVersion": 4.2, + "position": [ + 448, + 0 + ] + }, + { + "parameters": { + "jsCode": "const result = $input.first().json;\nconst classInfo = $('Extract Class Info').first().json;\n\nconst segments = result.segments || [];\nconst fullText = result.text || segments.map(s => s.text).join(' ');\nconst duration = result.duration || segments[segments.length - 1]?.end || 0;\n\nconst timestamped = segments.map(s => {\n const mins = Math.floor(s.start / 60);\n const secs = Math.floor(s.start % 60);\n return `[${mins}:${String(secs).padStart(2, '0')}] ${s.text.trim()}`;\n}).join('\\n');\n\nreturn [{\n json: {\n className: classInfo.className,\n classSlug: classInfo.classSlug,\n lectureTitle: classInfo.lectureTitle,\n fullText: fullText.trim(),\n timestampedText: timestamped,\n durationMinutes: Math.round(duration / 60),\n segmentCount: segments.length\n }\n}];" + }, + "id": "ecf09ad2-327e-4a82-90f3-95da5590e0ac", + "name": "Format Transcript", + "type": "n8n-nodes-base.code", + "typeVersion": 2, + "position": [ + 672, + 0 + ] + }, + { + "parameters": { + "method": "POST", + "url": "http://10.5.1.6:4000/v1/chat/completions", + "sendHeaders": true, + "headerParameters": { + "parameters": [ + { + "name": "Content-Type", + "value": "application/json" + }, + { + "name": "Authorization", + "value": "=Bearer {{ $env.LITELLM_API_KEY }}" + } + ] + }, + "sendBody": true, + "specifyBody": "json", + "jsonBody": "={\n \"model\": \"medium\",\n \"messages\": [\n {\n \"role\": \"system\",\n \"content\": \"You are an academic note-taking assistant. Given a class lecture transcription, provide:\\n1. A concise summary (3-5 sentences)\\n2. Key concepts covered (as a list)\\n3. Important definitions or formulas\\n4. Any assignments or deadlines mentioned\\n\\nRespond in JSON with keys: summary, key_concepts (array), definitions (array of {term, definition}), assignments (array)\"\n },\n {\n \"role\": \"user\",\n \"content\": \"Class: {{ $json.className }}\\nLecture: {{ $json.lectureTitle }}\\n\\nTranscription (first 3000 chars):\\n{{ $json.fullText.substring(0, 3000) }}\"\n }\n ],\n \"max_tokens\": 600,\n \"temperature\": 0.2,\n \"response_format\": { \"type\": \"json_object\" }\n}", + "options": {} + }, + "id": "e7407d06-f7f4-4105-a817-055d4703d59d", + "name": "AI Extract Notes", + "type": "n8n-nodes-base.httpRequest", + "typeVersion": 4.2, + "position": [ + 880, + 0 + ] + }, + { + "parameters": { + "jsCode": "const llmResponse = $input.first().json;\nconst transcriptData = $('Format Transcript').first().json;\n\nlet notes;\ntry {\n notes = JSON.parse(llmResponse.choices?.[0]?.message?.content || '{}');\n} catch(e) {\n notes = { summary: 'Could not parse notes', key_concepts: [], definitions: [], assignments: [] };\n}\n\nconst ragText = [\n `Class: ${transcriptData.className}`,\n `Lecture: ${transcriptData.lectureTitle}`,\n `Duration: ${transcriptData.durationMinutes} minutes`,\n `Summary: ${notes.summary}`,\n `Key Concepts: ${(notes.key_concepts || []).join(', ')}`,\n '',\n 'Full Transcript:',\n transcriptData.fullText\n].join('\\n');\n\nreturn [{\n json: {\n className: transcriptData.className,\n classSlug: transcriptData.classSlug,\n lectureTitle: transcriptData.lectureTitle,\n durationMinutes: transcriptData.durationMinutes,\n fullText: transcriptData.fullText,\n notes,\n ragText,\n source: `class-${transcriptData.classSlug}-${Date.now()}`,\n metadata: {\n class_name: transcriptData.className,\n class_slug: transcriptData.classSlug,\n lecture_title: transcriptData.lectureTitle,\n duration_minutes: transcriptData.durationMinutes,\n type: 'class_recording',\n recorded_at: new Date().toISOString()\n }\n }\n}];" + }, + "id": "71b0807d-03ba-45b2-b75b-b6fbeee5687e", + "name": "Prepare for RAG", + "type": "n8n-nodes-base.code", + "typeVersion": 2, + "position": [ + 1104, + 0 + ] + }, + { + "parameters": { + "jsCode": "const input = $input.first().json;\nconst text = input.ragText || '';\nconst source = input.source;\nconst metadata = input.metadata;\n\nconst CHUNK_SIZE = 400;\nconst MAX_CHUNK = 600;\nconst OVERLAP = 50;\n\nconst chunks = [];\nconst sentences = text.split(/(?<=[.!?])\\s+/);\nlet current = '';\n\nfor (const sentence of sentences) {\n // If a single sentence exceeds MAX_CHUNK, split it by words first\n const parts = [];\n if (sentence.length > MAX_CHUNK) {\n const words = sentence.split(' ');\n let part = '';\n for (const word of words) {\n if ((part + ' ' + word).length > CHUNK_SIZE && part.length > 0) {\n parts.push(part.trim());\n part = word;\n } else {\n part = part ? part + ' ' + word : word;\n }\n }\n if (part.trim()) parts.push(part.trim());\n } else {\n parts.push(sentence);\n }\n\n for (const part of parts) {\n if ((current + ' ' + part).length > CHUNK_SIZE && current.length > 0) {\n chunks.push(current.trim());\n const words = current.split(' ');\n current = words.slice(-OVERLAP).join(' ') + ' ' + part;\n } else {\n current = current ? current + ' ' + part : part;\n }\n }\n}\nif (current.trim()) chunks.push(current.trim());\n\nreturn chunks.map((chunk, i) => ({\n json: {\n chunk,\n chunkIndex: i,\n totalChunks: chunks.length,\n source,\n metadata\n }\n}));" + }, + "id": "9e4bbe70-72b0-4955-848f-0d1b62ae94a4", + "name": "Chunk Text", + "type": "n8n-nodes-base.code", + "typeVersion": 2, + "position": [ + 1328, + 0 + ] + }, + { + "parameters": { + "method": "POST", + "url": "http://10.5.1.6:11434/v1/embeddings", + "sendHeaders": true, + "headerParameters": { + "parameters": [ + { + "name": "Content-Type", + "value": "application/json" + } + ] + }, + "sendBody": true, + "specifyBody": "json", + "jsonBody": "={\n \"model\": \"nomic-embed-text\",\n \"input\": {{ JSON.stringify($json.chunk) }}\n}", + "options": { + "batching": { + "batch": { + "batchSize": 5, + "batchInterval": 500 + } + } + } + }, + "id": "4ae1ab11-c76f-4478-84e0-7bcdfb20b83b", + "name": "Get Embedding", + "type": "n8n-nodes-base.httpRequest", + "typeVersion": 4.2, + "position": [ + 1552, + 0 + ] + }, + { + "parameters": { + "jsCode": "const embeddingResponse = $input.first().json;\nconst chunkData = $('Chunk Text').item;\nconst embedding = embeddingResponse.data?.[0]?.embedding || [];\n\nconst pointId = 'xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx'.replace(/[xy]/g, c => {\n const r = Math.random() * 16 | 0;\n return (c === 'x' ? r : (r & 0x3 | 0x8)).toString(16);\n});\n\nreturn [{\n json: {\n id: pointId,\n vector: embedding,\n payload: {\n text: chunkData.json.chunk,\n source: chunkData.json.source,\n chunkIndex: chunkData.json.chunkIndex,\n totalChunks: chunkData.json.totalChunks,\n metadata: chunkData.json.metadata,\n ingested_at: new Date().toISOString()\n }\n }\n}];" + }, + "id": "d4232b02-23fc-4cbd-ac2d-2fdbfbe9583a", + "name": "Prepare Qdrant Point", + "type": "n8n-nodes-base.code", + "typeVersion": 2, + "position": [ + 1760, + 0 + ] + }, + { + "parameters": { + "method": "PUT", + "url": "http://10.5.1.6:6333/collections/knowledge_base/points", + "sendHeaders": true, + "headerParameters": { + "parameters": [ + { + "name": "Content-Type", + "value": "application/json" + } + ] + }, + "sendBody": true, + "specifyBody": "json", + "jsonBody": "={\n \"points\": [\n {\n \"id\": \"{{ $json.id }}\",\n \"vector\": {{ JSON.stringify($json.vector) }},\n \"payload\": {{ JSON.stringify($json.payload) }}\n }\n ]\n}", + "options": {} + }, + "id": "845a2aa0-2546-431d-96e8-b36875d1fe7d", + "name": "Upsert to Qdrant", + "type": "n8n-nodes-base.httpRequest", + "typeVersion": 4.2, + "position": [ + 1984, + 0 + ] + }, + { + "parameters": { + "jsCode": "const rag = $('Prepare for RAG').first().json;\nreturn [{\n json: {\n title: `\ud83c\udf93 ${rag.className} Lecture Processed`,\n message: `${rag.lectureTitle} (${rag.durationMinutes}min)\\nSummary: ${rag.notes.summary}\\nKey concepts: ${(rag.notes.key_concepts || []).slice(0, 5).join(', ')}`,\n priority: 5\n }\n}];" + }, + "id": "fmt08-0001-0000-0000-000000000008", + "name": "Format Gotify Message", + "type": "n8n-nodes-base.code", + "typeVersion": 2, + "position": [ + 1968, + 0 + ] + }, + { + "parameters": { + "message": "={{ $json.message }}", + "additionalFields": { + "priority": "={{ $json.priority }}", + "title": "={{ $json.title }}" + }, + "options": { + "contentType": "text/plain" + } + }, + "id": "5b31b660-c414-40d1-82a5-d68511203133", + "name": "Notify via Gotify", + "type": "n8n-nodes-base.gotify", + "typeVersion": 1, + "position": [ + 2208, + 0 + ], + "credentials": { + "gotifyApi": { + "id": "SETUP_REQUIRED", + "name": "Class Recordings" + } + } + }, + { + "parameters": { + "jsCode": "const rag = $('Prepare for RAG').first().json;\nconst notes = rag.notes || {};\nconst concepts = (notes.key_concepts || []).map((c,i) => `${i+1}. ${c}`).join('\\n');\nconst defList = notes.definitions || [];\nconst defs = Array.isArray(defList)\n ? defList.map(d => `${d.term}: ${d.definition}`).join('\\n')\n : Object.entries(defList).map(([k,v]) => `${k}: ${v}`).join('\\n');\nconst assignments = (notes.assignments || []).map((a,i) => `${i+1}. ${a}`).join('\\n');\nconst content = [\n `Lecture Notes: ${rag.lectureTitle}`,\n `Class: ${rag.className}`,\n `Duration: ${rag.durationMinutes} minutes`,\n `Date: ${new Date().toISOString()}`, '',\n `Summary`, `=======`, notes.summary || '', '',\n `Key Concepts`, `=============`, concepts,\n ...(defs ? ['', `Definitions`, `===========`, defs] : []),\n ...(assignments ? ['', `Assignments`, `===========`, assignments] : []),\n '', `Full Transcript`, `================`, rag.fullText || '',\n].join('\\n');\nconst title = `${rag.className} \u2014 ${rag.lectureTitle}`;\nreturn [{ binary: { document: { data: Buffer.from(content,'utf-8').toString('base64'), mimeType:'text/plain', fileName:`${title}.txt` }}, json: { title, className: rag.className } }];\n" + }, + "id": "95aea3a7-3b15-4196-9e4a-dc8f6ed8d49e", + "name": "Prepare Notes Doc", + "type": "n8n-nodes-base.code", + "typeVersion": 2, + "position": [ + 2208, + -200 + ] + }, + { + "parameters": { + "method": "POST", + "url": "https://paperless.paccoco.com/api/documents/post_document/", + "sendHeaders": true, + "headerParameters": { + "parameters": [ + { + "name": "Authorization", + "value": "=Token {{ $env.PAPERLESS_API_TOKEN }}" + } + ] + }, + "sendBody": true, + "contentType": "multipart-form-data", + "bodyParameters": { + "parameters": [ + { + "parameterType": "formBinaryData", + "name": "document", + "inputDataFieldName": "document" + }, + { + "name": "title", + "value": "={{ $json.title }}" + }, + { + "name": "tags[]", + "value": "={{ $env.PAPERLESS_TAG_LECTURE_NOTES }}" + } + ] + }, + "options": { + "response": { + "response": { + "responseFormat": "text" + } + } + } + }, + "id": "c902a153-ab26-4518-af2a-585f0b72fe6c", + "name": "Save Notes to Paperless", + "type": "n8n-nodes-base.httpRequest", + "typeVersion": 4.2, + "position": [ + 2432, + -200 + ], + "continueOnFail": true + } + ], + "connections": { + "Class Upload Webhook": { + "main": [ + [ + { + "node": "Extract Class Info", + "type": "main", + "index": 0 + } + ] + ] + }, + "Extract Class Info": { + "main": [ + [ + { + "node": "Whisper Transcribe", + "type": "main", + "index": 0 + } + ] + ] + }, + "Whisper Transcribe": { + "main": [ + [ + { + "node": "Format Transcript", + "type": "main", + "index": 0 + } + ] + ] + }, + "Format Transcript": { + "main": [ + [ + { + "node": "AI Extract Notes", + "type": "main", + "index": 0 + } + ] + ] + }, + "AI Extract Notes": { + "main": [ + [ + { + "node": "Prepare for RAG", + "type": "main", + "index": 0 + } + ] + ] + }, + "Prepare for RAG": { + "main": [ + [ + { + "node": "Chunk Text", + "type": "main", + "index": 0 + } + ] + ] + }, + "Chunk Text": { + "main": [ + [ + { + "node": "Get Embedding", + "type": "main", + "index": 0 + } + ] + ] + }, + "Get Embedding": { + "main": [ + [ + { + "node": "Prepare Qdrant Point", + "type": "main", + "index": 0 + } + ] + ] + }, + "Prepare Qdrant Point": { + "main": [ + [ + { + "node": "Upsert to Qdrant", + "type": "main", + "index": 0 + } + ] + ] + }, + "Upsert to Qdrant": { + "main": [ + [ + { + "node": "Format Gotify Message", + "type": "main", + "index": 0 + }, + { + "node": "Prepare Notes Doc", + "type": "main", + "index": 0 + } + ] + ] + }, + "Format Gotify Message": { + "main": [ + [ + { + "node": "Notify via Gotify", + "type": "main", + "index": 0 + } + ] + ] + }, + "Prepare Notes Doc": { + "main": [ + [ + { + "node": "Save Notes to Paperless", + "type": "main", + "index": 0 + } + ] + ] + } + }, + "settings": { + "executionOrder": "v1", + "binaryMode": "separate" + }, + "staticData": null, + "meta": null, + "pinData": {}, + "versionId": "d64293a7-5537-4b01-a11b-6b838d6cacbf", + "activeVersionId": "d64293a7-5537-4b01-a11b-6b838d6cacbf", + "versionCounter": 11, + "triggerCount": 1, + "tags": [ + { + "updatedAt": "2026-05-07T00:49:56.081Z", + "createdAt": "2026-05-07T00:49:56.081Z", + "id": "Q9lCYtCOuy9XmeyL", + "name": "school" + }, + { + "updatedAt": "2026-05-06T22:39:59.716Z", + "createdAt": "2026-05-06T22:39:59.716Z", + "id": "tuPxDoKFqpVrrQoX", + "name": "whisper" + }, + { + "updatedAt": "2026-05-06T22:39:33.284Z", + "createdAt": "2026-05-06T22:39:33.284Z", + "id": "7uuEyQrIlcuruZBk", + "name": "rag" + }, + { + "updatedAt": "2026-05-06T22:38:51.678Z", + "createdAt": "2026-05-06T22:38:51.678Z", + "id": "S9FxzVfHLsHmyzyt", + "name": "ai" + } + ], + "shared": [ + { + "updatedAt": "2026-05-07T01:16:50.340Z", + "createdAt": "2026-05-07T01:16:50.340Z", + "role": "workflow:owner", + "workflowId": "91I278E72FAaQenD", + "projectId": "dxCRnBdX5uJizCGa", + "project": { + "updatedAt": "2026-05-06T01:10:37.638Z", + "createdAt": "2026-05-06T01:08:07.820Z", + "id": "dxCRnBdX5uJizCGa", + "name": "Wilfred Fizzlepoof ", + "type": "personal", + "icon": null, + "description": null, + "creatorId": "47b2227f-2fc2-4a08-bd35-ea157b92df0d" + } + } + ], + "versionMetadata": { + "name": "Version d64293a7", + "description": "" + } +} \ No newline at end of file diff --git a/n8n-workflows/09-class-transcript-ingest-v1.0.json b/n8n-workflows/09-class-transcript-ingest-v1.0.json new file mode 100644 index 0000000..7a94dc2 --- /dev/null +++ b/n8n-workflows/09-class-transcript-ingest-v1.0.json @@ -0,0 +1,262 @@ +{ + "name": "Class Transcript Ingest v1.0", + "nodes": [ + { + "parameters": { + "httpMethod": "POST", + "path": "class/ingest", + "responseMode": "responseNode", + "options": {} + }, + "id": "a1b2c3d4-0001-0001-0001-000000000001", + "name": "Class Ingest Webhook", + "type": "n8n-nodes-base.webhook", + "typeVersion": 2, + "position": [0, 0], + "webhookId": "class-ingest-rocinante-v1" + }, + { + "parameters": { + "respondWith": "json", + "responseBody": "={ \"status\": \"accepted\" }", + "options": {} + }, + "id": "a1b2c3d4-0001-0001-0001-000000000002", + "name": "Respond Accepted", + "type": "n8n-nodes-base.respondToWebhook", + "typeVersion": 1, + "position": [224, -200] + }, + { + "parameters": { + "jsCode": "const body = $input.first().json.body || $input.first().json;\n\nconst className = body.class_name || 'unclassified';\nconst lectureTitle = body.lecture_title || 'Untitled Lecture';\nconst fullText = body.transcript || '';\nconst timestampedText = body.timestamped_transcript || fullText;\nconst durationMinutes = Math.round((body.duration_seconds || 0) / 60);\nconst classSlug = className.toLowerCase().replace(/[^a-z0-9]+/g, '-').replace(/^-|-$/g, '');\n\nreturn [{\n json: {\n className,\n classSlug,\n lectureTitle,\n fullText,\n timestampedText,\n durationMinutes\n }\n}];" + }, + "id": "a1b2c3d4-0001-0001-0001-000000000003", + "name": "Extract Class Info", + "type": "n8n-nodes-base.code", + "typeVersion": 2, + "position": [224, 0] + }, + { + "parameters": { + "method": "POST", + "url": "http://10.5.1.6:4000/v1/chat/completions", + "sendHeaders": true, + "headerParameters": { + "parameters": [ + { "name": "Content-Type", "value": "application/json" }, + { "name": "Authorization", "value": "=Bearer {{ $env.LITELLM_API_KEY }}" } + ] + }, + "sendBody": true, + "specifyBody": "json", + "jsonBody": "={\n \"model\": \"medium\",\n \"messages\": [\n {\n \"role\": \"system\",\n \"content\": \"You are an academic note-taking assistant. Given a class lecture transcription, provide:\\n1. A concise summary (3-5 sentences)\\n2. Key concepts covered (as a list)\\n3. Important definitions or formulas\\n4. Any assignments or deadlines mentioned\\n\\nRespond in JSON with keys: summary, key_concepts (array), definitions (array of {term, definition}), assignments (array)\"\n },\n {\n \"role\": \"user\",\n \"content\": \"Class: {{ $json.className }}\\nLecture: {{ $json.lectureTitle }}\\n\\nTranscription (first 4000 chars):\\n{{ $json.fullText.substring(0, 4000) }}\"\n }\n ],\n \"max_tokens\": 600,\n \"temperature\": 0.2,\n \"response_format\": { \"type\": \"json_object\" }\n}", + "options": {} + }, + "id": "a1b2c3d4-0001-0001-0001-000000000004", + "name": "AI Extract Notes", + "type": "n8n-nodes-base.httpRequest", + "typeVersion": 4.2, + "position": [448, 0] + }, + { + "parameters": { + "jsCode": "const llmResponse = $input.first().json;\nconst classInfo = $('Extract Class Info').first().json;\n\nlet notes;\ntry {\n notes = JSON.parse(llmResponse.choices?.[0]?.message?.content || '{}');\n} catch(e) {\n notes = { summary: 'Could not parse notes', key_concepts: [], definitions: [], assignments: [] };\n}\n\nconst ragText = [\n `Class: ${classInfo.className}`,\n `Lecture: ${classInfo.lectureTitle}`,\n `Duration: ${classInfo.durationMinutes} minutes`,\n `Summary: ${notes.summary}`,\n `Key Concepts: ${(notes.key_concepts || []).join(', ')}`,\n '',\n 'Full Transcript:',\n classInfo.fullText\n].join('\\n');\n\nreturn [{\n json: {\n className: classInfo.className,\n classSlug: classInfo.classSlug,\n lectureTitle: classInfo.lectureTitle,\n durationMinutes: classInfo.durationMinutes,\n fullText: classInfo.fullText,\n notes,\n ragText,\n source: `class-${classInfo.classSlug}-${Date.now()}`,\n metadata: {\n class_name: classInfo.className,\n class_slug: classInfo.classSlug,\n lecture_title: classInfo.lectureTitle,\n duration_minutes: classInfo.durationMinutes,\n type: 'class_recording',\n recorded_at: new Date().toISOString()\n }\n }\n}];" + }, + "id": "a1b2c3d4-0001-0001-0001-000000000005", + "name": "Prepare for RAG", + "type": "n8n-nodes-base.code", + "typeVersion": 2, + "position": [672, 0] + }, + { + "parameters": { + "jsCode": "const input = $input.first().json;\nconst text = input.ragText || '';\nconst source = input.source;\nconst metadata = input.metadata;\n\nconst CHUNK_SIZE = 400;\nconst MAX_CHUNK = 600;\nconst OVERLAP = 50;\n\nconst chunks = [];\nconst sentences = text.split(/(?<=[.!?])\\s+/);\nlet current = '';\n\nfor (const sentence of sentences) {\n // If a single sentence exceeds MAX_CHUNK, split it by words first\n const parts = [];\n if (sentence.length > MAX_CHUNK) {\n const words = sentence.split(' ');\n let part = '';\n for (const word of words) {\n if ((part + ' ' + word).length > CHUNK_SIZE && part.length > 0) {\n parts.push(part.trim());\n part = word;\n } else {\n part = part ? part + ' ' + word : word;\n }\n }\n if (part.trim()) parts.push(part.trim());\n } else {\n parts.push(sentence);\n }\n\n for (const part of parts) {\n if ((current + ' ' + part).length > CHUNK_SIZE && current.length > 0) {\n chunks.push(current.trim());\n const words = current.split(' ');\n current = words.slice(-OVERLAP).join(' ') + ' ' + part;\n } else {\n current = current ? current + ' ' + part : part;\n }\n }\n}\nif (current.trim()) chunks.push(current.trim());\n\nreturn chunks.map((chunk, i) => ({\n json: { chunk, chunkIndex: i, totalChunks: chunks.length, source, metadata }\n}));" + }, + "id": "a1b2c3d4-0001-0001-0001-000000000006", + "name": "Chunk Text", + "type": "n8n-nodes-base.code", + "typeVersion": 2, + "position": [896, 0] + }, + { + "parameters": { + "method": "POST", + "url": "http://10.5.1.6:11434/v1/embeddings", + "sendHeaders": true, + "headerParameters": { + "parameters": [ + { "name": "Content-Type", "value": "application/json" } + ] + }, + "sendBody": true, + "specifyBody": "json", + "jsonBody": "={\n \"model\": \"nomic-embed-text\",\n \"input\": {{ JSON.stringify($json.chunk) }}\n}", + "options": { + "batching": { + "batch": { "batchSize": 5, "batchInterval": 500 } + } + } + }, + "id": "a1b2c3d4-0001-0001-0001-000000000007", + "name": "Get Embedding", + "type": "n8n-nodes-base.httpRequest", + "typeVersion": 4.2, + "position": [1120, 0] + }, + { + "parameters": { + "jsCode": "const embeddingResponse = $input.first().json;\nconst chunkData = $('Chunk Text').item;\nconst embedding = embeddingResponse.data?.[0]?.embedding || [];\n\nconst pointId = 'xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx'.replace(/[xy]/g, c => {\n const r = Math.random() * 16 | 0;\n return (c === 'x' ? r : (r & 0x3 | 0x8)).toString(16);\n});\n\nreturn [{\n json: {\n id: pointId,\n vector: embedding,\n payload: {\n text: chunkData.json.chunk,\n source: chunkData.json.source,\n chunkIndex: chunkData.json.chunkIndex,\n totalChunks: chunkData.json.totalChunks,\n metadata: chunkData.json.metadata,\n ingested_at: new Date().toISOString()\n }\n }\n}];" + }, + "id": "a1b2c3d4-0001-0001-0001-000000000008", + "name": "Prepare Qdrant Point", + "type": "n8n-nodes-base.code", + "typeVersion": 2, + "position": [1344, 0] + }, + { + "parameters": { + "method": "PUT", + "url": "http://10.5.1.6:6333/collections/knowledge_base/points", + "sendHeaders": true, + "headerParameters": { + "parameters": [ + { "name": "Content-Type", "value": "application/json" } + ] + }, + "sendBody": true, + "specifyBody": "json", + "jsonBody": "={\n \"points\": [\n {\n \"id\": \"{{ $json.id }}\",\n \"vector\": {{ JSON.stringify($json.vector) }},\n \"payload\": {{ JSON.stringify($json.payload) }}\n }\n ]\n}", + "options": {} + }, + "id": "a1b2c3d4-0001-0001-0001-000000000009", + "name": "Upsert to Qdrant", + "type": "n8n-nodes-base.httpRequest", + "typeVersion": 4.2, + "position": [1568, 0] + }, + { + "parameters": { + "jsCode": "const rag = $('Prepare for RAG').first().json;\nreturn [{\n json: {\n title: `🎓 ${rag.className} Lecture Processed`,\n message: `${rag.lectureTitle} (${rag.durationMinutes}min)\\nSummary: ${rag.notes.summary}\\nKey concepts: ${(rag.notes.key_concepts || []).slice(0, 5).join(', ')}`,\n priority: 5\n }\n}];" + }, + "id": "a1b2c3d4-0001-0001-0001-000000000010", + "name": "Format Gotify Message", + "type": "n8n-nodes-base.code", + "typeVersion": 2, + "position": [1552, -200] + }, + { + "parameters": { + "message": "={{ $json.message }}", + "additionalFields": { + "priority": "={{ $json.priority }}", + "title": "={{ $json.title }}" + }, + "options": { "contentType": "text/plain" } + }, + "id": "a1b2c3d4-0001-0001-0001-000000000011", + "name": "Notify via Gotify", + "type": "n8n-nodes-base.gotify", + "typeVersion": 1, + "position": [1776, -200], + "credentials": { + "gotifyApi": { + "id": "SETUP_REQUIRED", + "name": "Class Recordings" + } + } + }, + { + "parameters": { + "jsCode": "const rag = $('Prepare for RAG').first().json;\nconst notes = rag.notes || {};\nconst concepts = (notes.key_concepts || []).map((c,i) => `${i+1}. ${c}`).join('\\n');\nconst defList = notes.definitions || [];\nconst defs = Array.isArray(defList)\n ? defList.map(d => `${d.term}: ${d.definition}`).join('\\n')\n : Object.entries(defList).map(([k,v]) => `${k}: ${v}`).join('\\n');\nconst assignments = (notes.assignments || []).map((a,i) => `${i+1}. ${a}`).join('\\n');\n\nconst content = [\n `Lecture Notes: ${rag.lectureTitle}`,\n `Class: ${rag.className}`,\n `Duration: ${rag.durationMinutes} minutes`,\n `Date: ${new Date().toISOString()}`, '',\n `Summary`, `=======`, notes.summary || '', '',\n `Key Concepts`, `=============`, concepts,\n ...(defs ? ['', `Definitions`, `===========`, defs] : []),\n ...(assignments ? ['', `Assignments`, `===========`, assignments] : []),\n '', `Full Transcript`, `================`, rag.fullText || '',\n].join('\\n');\n\nconst title = `${rag.className} — ${rag.lectureTitle}`;\nreturn [{\n binary: {\n document: {\n data: Buffer.from(content, 'utf-8').toString('base64'),\n mimeType: 'text/plain',\n fileName: `${title}.txt`\n }\n },\n json: { title, className: rag.className }\n}];" + }, + "id": "a1b2c3d4-0001-0001-0001-000000000012", + "name": "Prepare Notes Doc", + "type": "n8n-nodes-base.code", + "typeVersion": 2, + "position": [1776, 200] + }, + { + "parameters": { + "method": "POST", + "url": "https://paperless.paccoco.com/api/documents/post_document/", + "sendHeaders": true, + "headerParameters": { + "parameters": [ + { "name": "Authorization", "value": "=Token {{ $env.PAPERLESS_API_TOKEN }}" } + ] + }, + "sendBody": true, + "contentType": "multipart-form-data", + "bodyParameters": { + "parameters": [ + { "parameterType": "formBinaryData", "name": "document", "inputDataFieldName": "document" }, + { "name": "title", "value": "={{ $json.title }}" }, + { "name": "tags[]", "value": "={{ $env.PAPERLESS_TAG_LECTURE_NOTES }}" } + ] + }, + "options": { + "response": { + "response": { "responseFormat": "text" } + } + } + }, + "id": "a1b2c3d4-0001-0001-0001-000000000013", + "name": "Save Notes to Paperless", + "type": "n8n-nodes-base.httpRequest", + "typeVersion": 4.2, + "position": [2000, 200], + "continueOnFail": true + } + ], + "connections": { + "Class Ingest Webhook": { + "main": [[ + { "node": "Extract Class Info", "type": "main", "index": 0 }, + { "node": "Respond Accepted", "type": "main", "index": 0 } + ]] + }, + "Extract Class Info": { + "main": [[{ "node": "AI Extract Notes", "type": "main", "index": 0 }]] + }, + "AI Extract Notes": { + "main": [[{ "node": "Prepare for RAG", "type": "main", "index": 0 }]] + }, + "Prepare for RAG": { + "main": [[{ "node": "Chunk Text", "type": "main", "index": 0 }]] + }, + "Chunk Text": { + "main": [[{ "node": "Get Embedding", "type": "main", "index": 0 }]] + }, + "Get Embedding": { + "main": [[{ "node": "Prepare Qdrant Point", "type": "main", "index": 0 }]] + }, + "Prepare Qdrant Point": { + "main": [[{ "node": "Upsert to Qdrant", "type": "main", "index": 0 }]] + }, + "Upsert to Qdrant": { + "main": [[ + { "node": "Format Gotify Message", "type": "main", "index": 0 }, + { "node": "Prepare Notes Doc", "type": "main", "index": 0 } + ]] + }, + "Format Gotify Message": { + "main": [[{ "node": "Notify via Gotify", "type": "main", "index": 0 }]] + }, + "Prepare Notes Doc": { + "main": [[{ "node": "Save Notes to Paperless", "type": "main", "index": 0 }]] + } + }, + "settings": { + "executionOrder": "v1", + "binaryMode": "separate" + }, + "staticData": null, + "meta": null, + "pinData": {}, + "tags": [ + { "name": "school" }, + { "name": "rag" }, + { "name": "ai" } + ] +} diff --git a/n8n-workflows/09-class-transcript-ingest-v1.1.json b/n8n-workflows/09-class-transcript-ingest-v1.1.json new file mode 100644 index 0000000..a6aae51 --- /dev/null +++ b/n8n-workflows/09-class-transcript-ingest-v1.1.json @@ -0,0 +1,423 @@ +{ + "name": "Class Transcript Ingest v1.1", + "nodes": [ + { + "parameters": { + "httpMethod": "POST", + "path": "class/ingest", + "responseMode": "responseNode", + "options": {} + }, + "id": "a1b2c3d4-0001-0001-0001-000000000001", + "name": "Class Ingest Webhook", + "type": "n8n-nodes-base.webhook", + "typeVersion": 2, + "position": [ + 0, + 0 + ], + "webhookId": "class-ingest-rocinante-v1" + }, + { + "parameters": { + "respondWith": "json", + "responseBody": "={ \"status\": \"accepted\" }", + "options": {} + }, + "id": "a1b2c3d4-0001-0001-0001-000000000002", + "name": "Respond Accepted", + "type": "n8n-nodes-base.respondToWebhook", + "typeVersion": 1, + "position": [ + 224, + -200 + ] + }, + { + "parameters": { + "jsCode": "const body = $input.first().json.body || $input.first().json;\n\nconst className = body.class_name || 'unclassified';\nconst lectureTitle = body.lecture_title || 'Untitled Lecture';\nconst fullText = body.transcript || '';\nconst timestampedText = body.timestamped_transcript || fullText;\nconst durationMinutes = Math.round((body.duration_seconds || 0) / 60);\nconst classSlug = className.toLowerCase().replace(/[^a-z0-9]+/g, '-').replace(/^-|-$/g, '');\n\nreturn [{\n json: {\n className,\n classSlug,\n lectureTitle,\n fullText,\n timestampedText,\n durationMinutes\n }\n}];" + }, + "id": "a1b2c3d4-0001-0001-0001-000000000003", + "name": "Extract Class Info", + "type": "n8n-nodes-base.code", + "typeVersion": 2, + "position": [ + 224, + 0 + ] + }, + { + "parameters": { + "method": "POST", + "url": "http://10.5.1.6:4000/v1/chat/completions", + "sendHeaders": true, + "headerParameters": { + "parameters": [ + { + "name": "Content-Type", + "value": "application/json" + }, + { + "name": "Authorization", + "value": "=Bearer {{ $env.LITELLM_API_KEY }}" + } + ] + }, + "sendBody": true, + "specifyBody": "json", + "jsonBody": "={\n \"model\": \"medium\",\n \"messages\": [\n {\n \"role\": \"system\",\n \"content\": \"You are an academic note-taking assistant. Given a class lecture transcription, provide:\\n1. A concise summary (3-5 sentences)\\n2. Key concepts covered (as a list)\\n3. Important definitions or formulas\\n4. Any assignments or deadlines mentioned\\n\\nRespond in JSON with keys: summary, key_concepts (array), definitions (array of {term, definition}), assignments (array)\"\n },\n {\n \"role\": \"user\",\n \"content\": \"Class: {{ $json.className }}\\nLecture: {{ $json.lectureTitle }}\\n\\nTranscription (first 4000 chars):\\n{{ $json.fullText.substring(0, 4000) }}\"\n }\n ],\n \"max_tokens\": 600,\n \"temperature\": 0.2,\n \"response_format\": { \"type\": \"json_object\" }\n}", + "options": {} + }, + "id": "a1b2c3d4-0001-0001-0001-000000000004", + "name": "AI Extract Notes", + "type": "n8n-nodes-base.httpRequest", + "typeVersion": 4.2, + "position": [ + 448, + 0 + ] + }, + { + "parameters": { + "jsCode": "const llmResponse = $input.first().json;\nconst classInfo = $('Extract Class Info').first().json;\n\nlet notes;\ntry {\n notes = JSON.parse(llmResponse.choices?.[0]?.message?.content || '{}');\n} catch(e) {\n notes = { summary: 'Could not parse notes', key_concepts: [], definitions: [], assignments: [] };\n}\n\nconst ragText = [\n `Class: ${classInfo.className}`,\n `Lecture: ${classInfo.lectureTitle}`,\n `Duration: ${classInfo.durationMinutes} minutes`,\n `Summary: ${notes.summary}`,\n `Key Concepts: ${(notes.key_concepts || []).join(', ')}`,\n '',\n 'Full Transcript:',\n classInfo.fullText\n].join('\\n');\n\nreturn [{\n json: {\n className: classInfo.className,\n classSlug: classInfo.classSlug,\n lectureTitle: classInfo.lectureTitle,\n durationMinutes: classInfo.durationMinutes,\n fullText: classInfo.fullText,\n notes,\n ragText,\n source: `class-${classInfo.classSlug}-${Date.now()}`,\n metadata: {\n class_name: classInfo.className,\n class_slug: classInfo.classSlug,\n lecture_title: classInfo.lectureTitle,\n duration_minutes: classInfo.durationMinutes,\n type: 'class_recording',\n recorded_at: new Date().toISOString()\n }\n }\n}];" + }, + "id": "a1b2c3d4-0001-0001-0001-000000000005", + "name": "Prepare for RAG", + "type": "n8n-nodes-base.code", + "typeVersion": 2, + "position": [ + 672, + 0 + ] + }, + { + "parameters": { + "jsCode": "const input = $input.first().json;\nconst text = input.ragText || '';\nconst source = input.source;\nconst metadata = input.metadata;\n\nconst CHUNK_SIZE = 400;\nconst MAX_CHUNK = 600;\nconst OVERLAP = 50;\n\nconst chunks = [];\nconst sentences = text.split(/(?<=[.!?])\\s+/);\nlet current = '';\n\nfor (const sentence of sentences) {\n // If a single sentence exceeds MAX_CHUNK, split it by words first\n const parts = [];\n if (sentence.length > MAX_CHUNK) {\n const words = sentence.split(' ');\n let part = '';\n for (const word of words) {\n if ((part + ' ' + word).length > CHUNK_SIZE && part.length > 0) {\n parts.push(part.trim());\n part = word;\n } else {\n part = part ? part + ' ' + word : word;\n }\n }\n if (part.trim()) parts.push(part.trim());\n } else {\n parts.push(sentence);\n }\n\n for (const part of parts) {\n if ((current + ' ' + part).length > CHUNK_SIZE && current.length > 0) {\n chunks.push(current.trim());\n const words = current.split(' ');\n current = words.slice(-OVERLAP).join(' ') + ' ' + part;\n } else {\n current = current ? current + ' ' + part : part;\n }\n }\n}\nif (current.trim()) chunks.push(current.trim());\n\nreturn chunks.map((chunk, i) => ({\n json: { chunk, chunkIndex: i, totalChunks: chunks.length, source, metadata }\n}));" + }, + "id": "a1b2c3d4-0001-0001-0001-000000000006", + "name": "Chunk Text", + "type": "n8n-nodes-base.code", + "typeVersion": 2, + "position": [ + 896, + 0 + ] + }, + { + "parameters": { + "method": "POST", + "url": "http://10.5.1.6:11434/v1/embeddings", + "sendHeaders": true, + "headerParameters": { + "parameters": [ + { + "name": "Content-Type", + "value": "application/json" + } + ] + }, + "sendBody": true, + "specifyBody": "json", + "jsonBody": "={\n \"model\": \"nomic-embed-text\",\n \"input\": {{ JSON.stringify($json.chunk) }}\n}", + "options": { + "batching": { + "batch": { + "batchSize": 5, + "batchInterval": 500 + } + } + } + }, + "id": "a1b2c3d4-0001-0001-0001-000000000007", + "name": "Get Embedding", + "type": "n8n-nodes-base.httpRequest", + "typeVersion": 4.2, + "position": [ + 1120, + 0 + ] + }, + { + "parameters": { + "jsCode": "const embeddingResponse = $input.first().json;\nconst chunkData = $('Chunk Text').item;\nconst embedding = embeddingResponse.data?.[0]?.embedding || [];\n\nconst pointId = 'xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx'.replace(/[xy]/g, c => {\n const r = Math.random() * 16 | 0;\n return (c === 'x' ? r : (r & 0x3 | 0x8)).toString(16);\n});\n\nreturn [{\n json: {\n id: pointId,\n vector: embedding,\n payload: {\n text: chunkData.json.chunk,\n source: chunkData.json.source,\n chunkIndex: chunkData.json.chunkIndex,\n totalChunks: chunkData.json.totalChunks,\n metadata: chunkData.json.metadata,\n ingested_at: new Date().toISOString()\n }\n }\n}];" + }, + "id": "a1b2c3d4-0001-0001-0001-000000000008", + "name": "Prepare Qdrant Point", + "type": "n8n-nodes-base.code", + "typeVersion": 2, + "position": [ + 1344, + 0 + ] + }, + { + "parameters": { + "method": "PUT", + "url": "http://10.5.1.6:6333/collections/knowledge_base/points", + "sendHeaders": true, + "headerParameters": { + "parameters": [ + { + "name": "Content-Type", + "value": "application/json" + } + ] + }, + "sendBody": true, + "specifyBody": "json", + "jsonBody": "={\n \"points\": [\n {\n \"id\": \"{{ $json.id }}\",\n \"vector\": {{ JSON.stringify($json.vector) }},\n \"payload\": {{ JSON.stringify($json.payload) }}\n }\n ]\n}", + "options": {} + }, + "id": "a1b2c3d4-0001-0001-0001-000000000009", + "name": "Upsert to Qdrant", + "type": "n8n-nodes-base.httpRequest", + "typeVersion": 4.2, + "position": [ + 1568, + 0 + ] + }, + { + "parameters": { + "jsCode": "const rag = $('Prepare for RAG').first().json;\nreturn [{\n json: {\n title: `\ud83c\udf93 ${rag.className} Lecture Processed`,\n message: `${rag.lectureTitle} (${rag.durationMinutes}min)\\nSummary: ${rag.notes.summary}\\nKey concepts: ${(rag.notes.key_concepts || []).slice(0, 5).join(', ')}`,\n priority: 5\n }\n}];" + }, + "id": "a1b2c3d4-0001-0001-0001-000000000010", + "name": "Format Gotify Message", + "type": "n8n-nodes-base.code", + "typeVersion": 2, + "position": [ + 1552, + -200 + ] + }, + { + "parameters": { + "message": "={{ $json.message }}", + "additionalFields": { + "priority": "={{ $json.priority }}", + "title": "={{ $json.title }}" + }, + "options": { + "contentType": "text/plain" + } + }, + "id": "a1b2c3d4-0001-0001-0001-000000000011", + "name": "Notify via Gotify", + "type": "n8n-nodes-base.gotify", + "typeVersion": 1, + "position": [ + 1776, + -200 + ], + "credentials": { + "gotifyApi": { + "id": "SETUP_REQUIRED", + "name": "Class Recordings" + } + } + }, + { + "parameters": { + "jsCode": "const rag = $('Prepare for RAG').first().json;\nconst notes = rag.notes || {};\nconst concepts = (notes.key_concepts || []).map((c,i) => `${i+1}. ${c}`).join('\\n');\nconst defList = notes.definitions || [];\nconst defs = Array.isArray(defList)\n ? defList.map(d => `${d.term}: ${d.definition}`).join('\\n')\n : Object.entries(defList).map(([k,v]) => `${k}: ${v}`).join('\\n');\nconst assignments = (notes.assignments || []).map((a,i) => `${i+1}. ${a}`).join('\\n');\n\nconst content = [\n `Lecture Notes: ${rag.lectureTitle}`,\n `Class: ${rag.className}`,\n `Duration: ${rag.durationMinutes} minutes`,\n `Date: ${new Date().toISOString()}`, '',\n `Summary`, `=======`, notes.summary || '', '',\n `Key Concepts`, `=============`, concepts,\n ...(defs ? ['', `Definitions`, `===========`, defs] : []),\n ...(assignments ? ['', `Assignments`, `===========`, assignments] : []),\n '', `Full Transcript`, `================`, rag.fullText || '',\n].join('\\n');\n\nconst title = `${rag.className} \u2014 ${rag.lectureTitle}`;\nreturn [{\n binary: {\n document: {\n data: Buffer.from(content, 'utf-8').toString('base64'),\n mimeType: 'text/plain',\n fileName: `${title}.txt`\n }\n },\n json: { title, className: rag.className }\n}];" + }, + "id": "a1b2c3d4-0001-0001-0001-000000000012", + "name": "Prepare Notes Doc", + "type": "n8n-nodes-base.code", + "typeVersion": 2, + "position": [ + 1776, + 200 + ] + }, + { + "parameters": { + "method": "POST", + "url": "https://paperless.paccoco.com/api/documents/post_document/", + "sendHeaders": true, + "headerParameters": { + "parameters": [ + { + "name": "Authorization", + "value": "=Token {{ $env.PAPERLESS_API_TOKEN }}" + } + ] + }, + "sendBody": true, + "contentType": "multipart-form-data", + "bodyParameters": { + "parameters": [ + { + "parameterType": "formBinaryData", + "name": "document", + "inputDataFieldName": "document" + }, + { + "name": "title", + "value": "={{ $json.title }}" + }, + { + "name": "tags[]", + "value": "={{ $env.PAPERLESS_TAG_LECTURE_NOTES }}" + } + ] + }, + "options": { + "response": { + "response": { + "responseFormat": "text" + } + } + } + }, + "id": "a1b2c3d4-0001-0001-0001-000000000013", + "name": "Save Notes to Paperless", + "type": "n8n-nodes-base.httpRequest", + "typeVersion": 4.2, + "position": [ + 2000, + 200 + ], + "continueOnFail": true + } + ], + "connections": { + "Class Ingest Webhook": { + "main": [ + [ + { + "node": "Extract Class Info", + "type": "main", + "index": 0 + }, + { + "node": "Respond Accepted", + "type": "main", + "index": 0 + } + ] + ] + }, + "Extract Class Info": { + "main": [ + [ + { + "node": "AI Extract Notes", + "type": "main", + "index": 0 + } + ] + ] + }, + "AI Extract Notes": { + "main": [ + [ + { + "node": "Prepare for RAG", + "type": "main", + "index": 0 + } + ] + ] + }, + "Prepare for RAG": { + "main": [ + [ + { + "node": "Chunk Text", + "type": "main", + "index": 0 + } + ] + ] + }, + "Chunk Text": { + "main": [ + [ + { + "node": "Get Embedding", + "type": "main", + "index": 0 + } + ] + ] + }, + "Get Embedding": { + "main": [ + [ + { + "node": "Prepare Qdrant Point", + "type": "main", + "index": 0 + } + ] + ] + }, + "Prepare Qdrant Point": { + "main": [ + [ + { + "node": "Upsert to Qdrant", + "type": "main", + "index": 0 + } + ] + ] + }, + "Upsert to Qdrant": { + "main": [ + [ + { + "node": "Format Gotify Message", + "type": "main", + "index": 0 + }, + { + "node": "Prepare Notes Doc", + "type": "main", + "index": 0 + } + ] + ] + }, + "Format Gotify Message": { + "main": [ + [ + { + "node": "Notify via Gotify", + "type": "main", + "index": 0 + } + ] + ] + }, + "Prepare Notes Doc": { + "main": [ + [ + { + "node": "Save Notes to Paperless", + "type": "main", + "index": 0 + } + ] + ] + } + }, + "settings": { + "executionOrder": "v1", + "binaryMode": "separate" + }, + "staticData": null, + "meta": null, + "pinData": {}, + "tags": [ + { + "name": "school" + }, + { + "name": "rag" + }, + { + "name": "ai" + } + ] +} \ No newline at end of file diff --git a/n8n-workflows/14-paperless-inbox-reminder-v1.1.json b/n8n-workflows/14-paperless-inbox-reminder-v1.1.json new file mode 100644 index 0000000..da91189 --- /dev/null +++ b/n8n-workflows/14-paperless-inbox-reminder-v1.1.json @@ -0,0 +1,293 @@ +{ + "name": "Paperless Inbox Reminder v1.1", + "active": true, + "isArchived": false, + "nodes": [ + { + "parameters": { + "rule": { + "interval": [ + { + "field": "weeks", + "weeksInterval": 1, + "triggerAtDay": [ + 1 + ], + "triggerAtHour": 9, + "triggerAtMinute": 0 + } + ] + } + }, + "id": "41404371-29ed-4660-9989-a5cbd890555b", + "name": "Every Monday 9 AM", + "type": "n8n-nodes-base.scheduleTrigger", + "typeVersion": 1.2, + "position": [ + 0, + 0 + ] + }, + { + "parameters": { + "method": "GET", + "url": "https://paperless.paccoco.com/api/documents/", + "sendHeaders": true, + "headerParameters": { + "parameters": [ + { + "name": "Authorization", + "value": "Bearer ={{ $env.PAPERLESS_API_TOKEN }}" + } + ] + }, + "sendQuery": true, + "queryParameters": { + "parameters": [ + { + "name": "is_tagged", + "value": "0" + }, + { + "name": "page_size", + "value": "25" + }, + { + "name": "ordering", + "value": "created" + }, + { + "name": "fields", + "value": "id,title,created,document_type,tags" + } + ] + }, + "options": {} + }, + "id": "fdcd5728-6d27-445f-961b-18f204297469", + "name": "Fetch Inbox Docs", + "type": "n8n-nodes-base.httpRequest", + "typeVersion": 4.2, + "position": [ + 224, + 0 + ] + }, + { + "parameters": { + "jsCode": "const data = $input.first().json;\nconst docs = data.results || [];\nconst total = data.count || 0;\n\n// Oldest 5 for the reminder body\nconst oldest = docs.slice(0, 5).map(d => {\n const date = d.created ? d.created.substring(0, 10) : 'unknown date';\n return ` \u2022 ${d.title || 'Untitled'} (added ${date})`;\n});\n\nreturn [{\n json: {\n total,\n hasItems: total > 0,\n oldestList: oldest.join('\\n'),\n remaining: Math.max(0, total - 5)\n }\n}];" + }, + "id": "8fbe6be0-1ed3-4126-8f21-c3f4efb161c6", + "name": "Parse Response", + "type": "n8n-nodes-base.code", + "typeVersion": 2, + "position": [ + 448, + 0 + ] + }, + { + "parameters": { + "conditions": { + "options": { + "caseSensitive": true, + "leftValue": "", + "typeValidation": "strict" + }, + "conditions": [ + { + "id": "b206599d-0c79-4a32-9fb9-91e1c41dcff2", + "leftValue": "={{ $json.hasItemsStr }}", + "rightValue": "yes", + "operator": { + "type": "string", + "operation": "equals" + } + } + ], + "combinator": "and" + } + }, + "id": "e24344a1-e857-462c-b96a-842169e83a6d", + "name": "Has Inbox Items?", + "type": "n8n-nodes-base.if", + "typeVersion": 2.2, + "position": [ + 672, + 0 + ] + }, + { + "parameters": { + "jsCode": "const d = $input.first().json;\nconst extra = d.remaining > 0 ? `\\n \u2026 and ${d.remaining} more` : '';\nconst body = `${d.total} untagged document${d.total !== 1 ? 's' : ''} in Paperless inbox:\\n${d.oldestList}${extra}\\n\\nhttps://paperless.paccoco.com`;\nreturn [{ json: { title: '\ud83d\udce5 Paperless Inbox Needs Attention', message: body, priority: 5 } }];" + }, + "id": "897af9c9-4429-445f-8160-a03a14a318a3", + "name": "Format Reminder", + "type": "n8n-nodes-base.code", + "typeVersion": 2, + "position": [ + 896, + -120 + ] + }, + { + "parameters": { + "jsCode": "return [{ json: { title: '\u2705 Paperless Inbox Clear', message: 'No untagged documents \u2014 inbox is clean.', priority: 1 } }];" + }, + "id": "bd8fe88e-7876-45dd-8344-983674fba79d", + "name": "Format Clear", + "type": "n8n-nodes-base.code", + "typeVersion": 2, + "position": [ + 896, + 120 + ] + }, + { + "parameters": { + "message": "={{ $json.message }}", + "additionalFields": { + "priority": "={{ $json.priority }}", + "title": "={{ $json.title }}" + }, + "options": { + "contentType": "text/plain" + } + }, + "id": "74860e7f-42e3-448f-9e46-5cfb6f12cdc2", + "name": "Push to Gotify (Reminder)", + "type": "n8n-nodes-base.gotify", + "typeVersion": 1, + "position": [ + 1120, + -120 + ], + "credentials": { + "gotifyApi": { + "id": "SETUP_REQUIRED", + "name": "Reminders" + } + } + }, + { + "parameters": { + "message": "={{ $json.message }}", + "additionalFields": { + "priority": "={{ $json.priority }}", + "title": "={{ $json.title }}" + }, + "options": { + "contentType": "text/plain" + } + }, + "id": "b1dbd39b-87c3-43ec-9cec-f63eabbd0fa5", + "name": "Push to Gotify (Clear)", + "type": "n8n-nodes-base.gotify", + "typeVersion": 1, + "position": [ + 1120, + 120 + ], + "credentials": { + "gotifyApi": { + "id": "SETUP_REQUIRED", + "name": "Reminders" + } + } + } + ], + "connections": { + "Every Monday 9 AM": { + "main": [ + [ + { + "node": "Fetch Inbox Docs", + "type": "main", + "index": 0 + } + ] + ] + }, + "Fetch Inbox Docs": { + "main": [ + [ + { + "node": "Parse Response", + "type": "main", + "index": 0 + } + ] + ] + }, + "Parse Response": { + "main": [ + [ + { + "node": "Has Inbox Items?", + "type": "main", + "index": 0 + } + ] + ] + }, + "Has Inbox Items?": { + "main": [ + [ + { + "node": "Format Reminder", + "type": "main", + "index": 0 + } + ], + [ + { + "node": "Format Clear", + "type": "main", + "index": 0 + } + ] + ] + }, + "Format Reminder": { + "main": [ + [ + { + "node": "Push to Gotify (Reminder)", + "type": "main", + "index": 0 + } + ] + ] + }, + "Format Clear": { + "main": [ + [ + { + "node": "Push to Gotify (Clear)", + "type": "main", + "index": 0 + } + ] + ] + } + }, + "settings": { + "executionOrder": "v1", + "binaryMode": "separate" + }, + "staticData": null, + "meta": null, + "pinData": {}, + "tags": [ + { + "name": "reminders" + }, + { + "name": "paperless" + }, + { + "name": "scheduled" + } + ], + "id": "2b80579b-8d89-42d5-8b8b-9137f59ea206" +} \ No newline at end of file diff --git a/n8n-workflows/14-paperless-inbox-reminder-v1.2.json b/n8n-workflows/14-paperless-inbox-reminder-v1.2.json old mode 100755 new mode 100644 diff --git a/n8n-workflows/15-n8n-health-monitor-v1.1.json b/n8n-workflows/15-n8n-health-monitor-v1.1.json new file mode 100644 index 0000000..0dd5847 --- /dev/null +++ b/n8n-workflows/15-n8n-health-monitor-v1.1.json @@ -0,0 +1,384 @@ +{ + "name": "n8n Self-Health Monitor v1.1", + "active": true, + "isArchived": false, + "nodes": [ + { + "parameters": { + "rule": { + "interval": [ + { + "field": "minutes", + "minutesInterval": 15 + } + ] + } + }, + "id": "dabcb0ae-d513-410e-b779-4570df5c4a35", + "name": "Every 15 Minutes", + "type": "n8n-nodes-base.scheduleTrigger", + "typeVersion": 1.2, + "position": [ + 0, + 0 + ] + }, + { + "parameters": { + "method": "GET", + "url": "http://10.5.1.6:5678/api/v1/executions", + "sendHeaders": true, + "headerParameters": { + "parameters": [ + { + "name": "X-N8N-API-KEY", + "value": "={{ $env.N8N_API_KEY }}" + } + ] + }, + "sendQuery": true, + "queryParameters": { + "parameters": [ + { + "name": "status", + "value": "error" + }, + { + "name": "includeData", + "value": "false" + }, + { + "name": "limit", + "value": "50" + } + ] + }, + "options": {} + }, + "id": "806d75e9-b95a-499a-b997-a7f7dc046775", + "name": "Fetch Failed Executions", + "type": "n8n-nodes-base.httpRequest", + "typeVersion": 4.2, + "position": [ + 224, + 0 + ] + }, + { + "parameters": { + "jsCode": "const data = $input.first().json;\nconst executions = data.data || [];\nconst oneHourAgo = Date.now() - (60 * 60 * 1000);\n\n// Filter to failures in the past hour\nconst recent = executions.filter(e => {\n const ts = new Date(e.startedAt || e.stoppedAt || 0).getTime();\n return ts >= oneHourAgo;\n});\n\n// Group by workflow name + error message for dedup key\nconst failures = recent.map(e => ({\n executionId: e.id,\n workflowId: e.workflowId,\n workflowName: e.workflowData?.name || `Workflow ${e.workflowId}`,\n startedAt: e.startedAt,\n stoppedAt: e.stoppedAt,\n errorMessage: e.data?.resultData?.error?.message || 'Unknown error'\n}));\n\n// Deduplicate by workflowName (report once per workflow per check)\nconst seen = new Set();\nconst unique = failures.filter(f => {\n if (seen.has(f.workflowName)) return false;\n seen.add(f.workflowName);\n return true;\n});\n\nreturn [{ json: { hasFailures: unique.length > 0, hasFailuresStr: unique.length > 0 ? 'yes' : 'no', count: unique.length, failures: unique } }];" + }, + "id": "f5d47910-5047-4c4f-aa84-a6907aa38551", + "name": "Filter Recent Failures", + "type": "n8n-nodes-base.code", + "typeVersion": 2, + "position": [ + 448, + 0 + ] + }, + { + "parameters": { + "conditions": { + "options": { + "caseSensitive": true, + "leftValue": "", + "typeValidation": "strict" + }, + "conditions": [ + { + "id": "02ff82df-a31d-491b-9cd1-96874c8ae2fe", + "leftValue": "={{ $json.hasFailuresStr }}", + "rightValue": "yes", + "operator": { + "type": "string", + "operation": "equals" + } + } + ], + "combinator": "and" + } + }, + "id": "7c298433-ab89-435e-8955-73da6b986f46", + "name": "Has Failures?", + "type": "n8n-nodes-base.if", + "typeVersion": 2.2, + "position": [ + 672, + 0 + ] + }, + { + "parameters": { + "jsCode": "const d = $input.first().json;\n// Fingerprint = sorted workflow names joined \u2014 stable across retriggers\nconst fingerprint = (d.failures || [])\n .map(f => f.workflowName)\n .sort()\n .join('|') || 'unknown';\nreturn [{ json: { ...d, fingerprint } }];" + }, + "id": "d3609d45-ba9f-41d3-a9bd-8c558a719f10", + "name": "Build Fingerprint", + "type": "n8n-nodes-base.code", + "typeVersion": 2, + "position": [ + 896, + -120 + ] + }, + { + "parameters": { + "operation": "executeQuery", + "query": "SELECT fingerprint, last_seen FROM n8n_health_dedup WHERE fingerprint = '{{ $json.fingerprint }}' AND last_seen > NOW() - INTERVAL '1 hour' LIMIT 1;", + "options": {} + }, + "id": "15a3dd71-bc5b-4396-9534-63518e097438", + "name": "Lookup Dedup", + "type": "n8n-nodes-base.postgres", + "typeVersion": 2.5, + "position": [ + 1120, + -120 + ], + "credentials": { + "postgres": { + "id": "n9svoXemqSZoNNUB", + "name": "Postgres account" + } + }, + "continueOnFail": true + }, + { + "parameters": { + "jsCode": "const alertData = $('Build Fingerprint').first().json;\nconst rows = $input.all().filter(r => r.json && r.json.fingerprint);\nconst alreadyNotified = rows.length > 0;\nreturn [{ json: { ...alertData, alreadyNotified, shouldNotify: !alreadyNotified, shouldNotifyStr: !alreadyNotified ? 'yes' : 'no' } }];" + }, + "id": "a92590e3-7044-4ba3-9b39-f6a9457c5fbe", + "name": "Check Already Notified", + "type": "n8n-nodes-base.code", + "typeVersion": 2, + "position": [ + 1344, + -120 + ] + }, + { + "parameters": { + "conditions": { + "options": { + "caseSensitive": true, + "leftValue": "", + "typeValidation": "strict" + }, + "conditions": [ + { + "id": "aa5db0c3-75a7-4ea4-b29e-c12568ccf320", + "leftValue": "={{ $json.shouldNotifyStr }}", + "rightValue": "yes", + "operator": { + "type": "string", + "operation": "equals" + } + } + ], + "combinator": "and" + } + }, + "id": "39ac0eaf-73ce-4fe1-9c4b-d09ac920291d", + "name": "Should Notify?", + "type": "n8n-nodes-base.if", + "typeVersion": 2.2, + "position": [ + 1568, + -120 + ] + }, + { + "parameters": { + "operation": "executeQuery", + "query": "INSERT INTO n8n_health_dedup (fingerprint, last_seen, workflow_names) VALUES ('{{ $json.fingerprint }}', NOW(), '{{ $json.failures.map(f => f.workflowName).join(\", \") }}') ON CONFLICT (fingerprint) DO UPDATE SET last_seen = NOW(), workflow_names = EXCLUDED.workflow_names;", + "options": {} + }, + "id": "0c3be155-0c3a-4354-b863-73a3f5bbf598", + "name": "Upsert Dedup Record", + "type": "n8n-nodes-base.postgres", + "typeVersion": 2.5, + "position": [ + 1792, + -120 + ], + "credentials": { + "postgres": { + "id": "n9svoXemqSZoNNUB", + "name": "Postgres account" + } + } + }, + { + "parameters": { + "jsCode": "const d = $('Build Fingerprint').first().json;\nconst lines = (d.failures || []).map(f =>\n ` \u2022 ${f.workflowName}\\n ${f.errorMessage.substring(0, 120)}`\n);\nconst body = `${d.count} workflow${d.count !== 1 ? 's' : ''} failed in the past hour:\\n\\n${lines.join('\\n')}\\n\\nhttps://n8n.paccoco.com`;\nreturn [{ json: { title: `\u26a0\ufe0f n8n: ${d.count} Workflow Failure${d.count !== 1 ? 's' : ''}`, message: body, priority: 7 } }];" + }, + "id": "1c6b42bf-a38e-47b6-82ac-50c43ee9e583", + "name": "Format Notification", + "type": "n8n-nodes-base.code", + "typeVersion": 2, + "position": [ + 2016, + -120 + ] + }, + { + "parameters": { + "message": "={{ $json.message }}", + "additionalFields": { + "priority": "={{ $json.priority }}", + "title": "={{ $json.title }}" + }, + "options": { + "contentType": "text/plain" + } + }, + "id": "09435c2f-9044-432a-8ca8-dd3ce3262c76", + "name": "Push to Gotify", + "type": "n8n-nodes-base.gotify", + "typeVersion": 1, + "position": [ + 2240, + -120 + ], + "credentials": { + "gotifyApi": { + "id": "SETUP_REQUIRED", + "name": "n8n Health" + } + } + } + ], + "connections": { + "Every 15 Minutes": { + "main": [ + [ + { + "node": "Fetch Failed Executions", + "type": "main", + "index": 0 + } + ] + ] + }, + "Fetch Failed Executions": { + "main": [ + [ + { + "node": "Filter Recent Failures", + "type": "main", + "index": 0 + } + ] + ] + }, + "Filter Recent Failures": { + "main": [ + [ + { + "node": "Has Failures?", + "type": "main", + "index": 0 + } + ] + ] + }, + "Has Failures?": { + "main": [ + [ + { + "node": "Build Fingerprint", + "type": "main", + "index": 0 + } + ], + [] + ] + }, + "Build Fingerprint": { + "main": [ + [ + { + "node": "Lookup Dedup", + "type": "main", + "index": 0 + } + ] + ] + }, + "Lookup Dedup": { + "main": [ + [ + { + "node": "Check Already Notified", + "type": "main", + "index": 0 + } + ] + ] + }, + "Check Already Notified": { + "main": [ + [ + { + "node": "Should Notify?", + "type": "main", + "index": 0 + } + ] + ] + }, + "Should Notify?": { + "main": [ + [ + { + "node": "Upsert Dedup Record", + "type": "main", + "index": 0 + } + ], + [] + ] + }, + "Upsert Dedup Record": { + "main": [ + [ + { + "node": "Format Notification", + "type": "main", + "index": 0 + } + ] + ] + }, + "Format Notification": { + "main": [ + [ + { + "node": "Push to Gotify", + "type": "main", + "index": 0 + } + ] + ] + } + }, + "settings": { + "executionOrder": "v1", + "binaryMode": "separate" + }, + "staticData": null, + "meta": null, + "pinData": {}, + "tags": [ + { + "name": "monitoring" + }, + { + "name": "scheduled" + }, + { + "name": "n8n" + } + ], + "id": "febfda9b-0f4e-40b5-ab84-82ba9b7fcf29" +} \ No newline at end of file diff --git a/n8n-workflows/15-n8n-health-monitor-v1.2.json b/n8n-workflows/15-n8n-health-monitor-v1.2.json old mode 100755 new mode 100644 diff --git a/n8n-workflows/16-nfs-watchdog-v1.1.json b/n8n-workflows/16-nfs-watchdog-v1.1.json new file mode 100644 index 0000000..75a2246 --- /dev/null +++ b/n8n-workflows/16-nfs-watchdog-v1.1.json @@ -0,0 +1,450 @@ +{ + "name": "NFS Mount Watchdog + Auto-Heal v1.1", + "active": true, + "isArchived": false, + "nodes": [ + { + "parameters": { + "rule": { + "interval": [ + { + "field": "minutes", + "minutesInterval": 5 + } + ] + } + }, + "id": "8c466a67-a0fa-42dc-8ed8-0918834fc3a7", + "name": "Every 5 Minutes", + "type": "n8n-nodes-base.scheduleTrigger", + "typeVersion": 1.2, + "position": [ + 0, + 0 + ] + }, + { + "parameters": { + "command": "mountpoint -q /mnt/unraid/data/media && echo MEDIA_OK || echo MEDIA_MISSING; mountpoint -q /mnt/unraid/data/photos && echo PHOTOS_OK || echo PHOTOS_MISSING" + }, + "id": "f5b2fee9-d67a-4da7-80dd-f72e6078aed9", + "name": "Probe Mounts", + "type": "n8n-nodes-base.ssh", + "typeVersion": 1, + "position": [ + 224, + 0 + ], + "credentials": { + "sshPrivateKey": { + "id": "SETUP_REQUIRED", + "name": "PD SSH" + } + }, + "continueOnFail": true + }, + { + "parameters": { + "jsCode": "\nconst out = ($input.first().json.stdout || '') + ($input.first().json.stderr || '');\nconst mediaOk = out.includes('MEDIA_OK');\nconst photosOk = out.includes('PHOTOS_OK');\nconst missing = [];\nif (!mediaOk) missing.push({ mount: '/mnt/unraid/data/media', label: 'media',\n containers: ['plex','audiobookshelf'] });\nif (!photosOk) missing.push({ mount: '/mnt/unraid/data/photos', label: 'photos',\n containers: ['immich-server','immich-microservices','immich-machine-learning'] });\nreturn [{ json: { mediaOk, photosOk, anyMissing: missing.length > 0, anyMissingStr: missing.length > 0 ? 'yes' : 'no', missing } }];\n" + }, + "id": "25bb3239-49cb-4570-b706-4022b9aa751c", + "name": "Parse Mount Status", + "type": "n8n-nodes-base.code", + "typeVersion": 2, + "position": [ + 448, + 0 + ] + }, + { + "parameters": { + "conditions": { + "options": { + "caseSensitive": true, + "leftValue": "", + "typeValidation": "strict" + }, + "conditions": [ + { + "id": "70384f84-9d15-4e45-b295-67202a39a227", + "leftValue": "={{ $json.anyMissingStr }}", + "rightValue": "yes", + "operator": { + "type": "string", + "operation": "equals" + } + } + ], + "combinator": "and" + } + }, + "id": "68b74832-85a7-4193-9382-a0c22a666e66", + "name": "Any Missing?", + "type": "n8n-nodes-base.if", + "typeVersion": 2.2, + "position": [ + 672, + 0 + ] + }, + { + "parameters": { + "command": "sudo bash /mnt/tank/docker/scripts/mount-unraid-nfs.sh 2>&1" + }, + "id": "b18c7537-0c2f-4122-bfe5-6469d6be6853", + "name": "Run Mount Script", + "type": "n8n-nodes-base.ssh", + "typeVersion": 1, + "position": [ + 896, + -120 + ], + "credentials": { + "sshPrivateKey": { + "id": "SETUP_REQUIRED", + "name": "PD SSH" + } + }, + "continueOnFail": true + }, + { + "parameters": { + "resume": "timeInterval", + "unit": "seconds", + "value": 10 + }, + "id": "2e927f83-9ddc-4816-8ee6-76d21ce21a5e", + "name": "Wait 10s", + "type": "n8n-nodes-base.wait", + "typeVersion": 1.1, + "position": [ + 1120, + -120 + ], + "webhookId": "4e89e8d0-7f08-4378-a2b7-63756665b039" + }, + { + "parameters": { + "command": "mountpoint -q /mnt/unraid/data/media && echo MEDIA_OK || echo MEDIA_MISSING; mountpoint -q /mnt/unraid/data/photos && echo PHOTOS_OK || echo PHOTOS_MISSING" + }, + "id": "d15c0c90-e94f-404a-a787-75f1b1b5befa", + "name": "Re-probe Mounts", + "type": "n8n-nodes-base.ssh", + "typeVersion": 1, + "position": [ + 1344, + -120 + ], + "credentials": { + "sshPrivateKey": { + "id": "SETUP_REQUIRED", + "name": "PD SSH" + } + }, + "continueOnFail": true + }, + { + "parameters": { + "jsCode": "\nconst out = ($input.first().json.stdout || '') + ($input.first().json.stderr || '');\nconst mediaOk = out.includes('MEDIA_OK');\nconst photosOk = out.includes('PHOTOS_OK');\n\n// What was originally missing (from step 3)\nconst originalMissing = $('Parse Mount Status').first().json.missing || [];\n\n// Which of the originally-missing mounts are now up?\nconst healed = originalMissing.filter(m =>\n (m.label === 'media' && mediaOk) ||\n (m.label === 'photos' && photosOk)\n);\nconst stillDown = originalMissing.filter(m =>\n (m.label === 'media' && !mediaOk) ||\n (m.label === 'photos' && !photosOk)\n);\n\n// Build flat container list to restart (only for healed mounts)\nconst containersToRestart = [...new Set(healed.flatMap(m => m.containers))];\nconst restartCmd = containersToRestart.length > 0\n ? `sudo docker restart ${containersToRestart.join(' ')}`\n : 'echo NO_RESTART_NEEDED';\n\nreturn [{ json: {\n allHealed: stillDown.length === 0, allHealedStr: stillDown.length === 0 ? 'yes' : 'no',\n healed, stillDown,\n containersToRestart,\n restartCmd,\n originalMissing\n}}];\n" + }, + "id": "48de38c2-df08-406f-ad61-809ca6e306a1", + "name": "Parse Heal Result", + "type": "n8n-nodes-base.code", + "typeVersion": 2, + "position": [ + 1568, + -120 + ] + }, + { + "parameters": { + "conditions": { + "options": { + "caseSensitive": true, + "leftValue": "", + "typeValidation": "strict" + }, + "conditions": [ + { + "id": "32b2dfd5-9958-45d4-94fc-410ce4d9543d", + "leftValue": "={{ $json.allHealedStr }}", + "rightValue": "yes", + "operator": { + "type": "string", + "operation": "equals" + } + } + ], + "combinator": "and" + } + }, + "id": "29a1b544-97b3-433f-9821-2bf5a8af95d2", + "name": "Healed?", + "type": "n8n-nodes-base.if", + "typeVersion": 2.2, + "position": [ + 1792, + -120 + ] + }, + { + "parameters": { + "command": "={{ $json.restartCmd }}" + }, + "id": "8d791509-9e18-44c1-bf58-ed3027927710", + "name": "Restart Containers", + "type": "n8n-nodes-base.ssh", + "typeVersion": 1, + "position": [ + 2016, + -240 + ], + "credentials": { + "sshPrivateKey": { + "id": "SETUP_REQUIRED", + "name": "PD SSH" + } + }, + "continueOnFail": true + }, + { + "parameters": { + "jsCode": "\nconst d = $('Parse Heal Result').first().json;\nconst mountList = d.healed.map(m => ` \u2022 ${m.mount}`).join('\\n');\nconst containers = d.containersToRestart.join(', ') || 'none';\nconst body = `NFS mounts were missing and have been automatically remounted.\\n\\nRemounted:\\n${mountList}\\n\\nContainers restarted: ${containers}`;\nreturn [{ json: { title: '\ud83d\udd17 NFS Auto-Healed', message: body, priority: 5 } }];\n" + }, + "id": "75c8976f-d622-43cd-a332-72c32f0c7c60", + "name": "Format Healed Notify", + "type": "n8n-nodes-base.code", + "typeVersion": 2, + "position": [ + 2240, + -240 + ] + }, + { + "parameters": { + "message": "={{ $json.message }}", + "additionalFields": { + "priority": "={{ $json.priority }}", + "title": "={{ $json.title }}" + }, + "options": { + "contentType": "text/plain" + } + }, + "id": "38705cd2-775b-4ef9-b470-4a7709f132e5", + "name": "Gotify (Healed)", + "type": "n8n-nodes-base.gotify", + "typeVersion": 1, + "position": [ + 2464, + -240 + ], + "credentials": { + "gotifyApi": { + "id": "SETUP_REQUIRED", + "name": "Uptime / Infra" + } + } + }, + { + "parameters": { + "jsCode": "\nconst d = $('Parse Heal Result').first().json;\nconst downList = d.stillDown.map(m => ` \u2022 ${m.mount}`).join('\\n');\nconst healedList = d.healed.length > 0\n ? `\\nPartially healed:\\n${d.healed.map(m => ' \u2022 ' + m.mount).join('\\n')}`\n : '';\nconst body = `NFS mount script ran but the following paths are still unavailable. Manual intervention required.\\n\\nStill down:\\n${downList}${healedList}\\n\\nCheck Serenity (TrueNAS) is up and NFS service is running.`;\nreturn [{ json: { title: '\ud83d\udea8 NFS Down \u2014 Manual Fix Needed', message: body, priority: 9 } }];\n" + }, + "id": "074b984f-7fc2-45bc-b97f-f5db4d8d589b", + "name": "Format Escalation", + "type": "n8n-nodes-base.code", + "typeVersion": 2, + "position": [ + 2016, + -20 + ] + }, + { + "parameters": { + "message": "={{ $json.message }}", + "additionalFields": { + "priority": "={{ $json.priority }}", + "title": "={{ $json.title }}" + }, + "options": { + "contentType": "text/plain" + } + }, + "id": "f929dd77-6072-4ac9-9dc4-11246110efed", + "name": "Gotify (Escalation)", + "type": "n8n-nodes-base.gotify", + "typeVersion": 1, + "position": [ + 2240, + -20 + ], + "credentials": { + "gotifyApi": { + "id": "SETUP_REQUIRED", + "name": "Uptime / Infra" + } + } + } + ], + "connections": { + "Every 5 Minutes": { + "main": [ + [ + { + "node": "Probe Mounts", + "type": "main", + "index": 0 + } + ] + ] + }, + "Probe Mounts": { + "main": [ + [ + { + "node": "Parse Mount Status", + "type": "main", + "index": 0 + } + ] + ] + }, + "Parse Mount Status": { + "main": [ + [ + { + "node": "Any Missing?", + "type": "main", + "index": 0 + } + ] + ] + }, + "Any Missing?": { + "main": [ + [ + { + "node": "Run Mount Script", + "type": "main", + "index": 0 + } + ], + [] + ] + }, + "Run Mount Script": { + "main": [ + [ + { + "node": "Wait 10s", + "type": "main", + "index": 0 + } + ] + ] + }, + "Wait 10s": { + "main": [ + [ + { + "node": "Re-probe Mounts", + "type": "main", + "index": 0 + } + ] + ] + }, + "Re-probe Mounts": { + "main": [ + [ + { + "node": "Parse Heal Result", + "type": "main", + "index": 0 + } + ] + ] + }, + "Parse Heal Result": { + "main": [ + [ + { + "node": "Healed?", + "type": "main", + "index": 0 + } + ] + ] + }, + "Healed?": { + "main": [ + [ + { + "node": "Restart Containers", + "type": "main", + "index": 0 + } + ], + [ + { + "node": "Format Escalation", + "type": "main", + "index": 0 + } + ] + ] + }, + "Restart Containers": { + "main": [ + [ + { + "node": "Format Healed Notify", + "type": "main", + "index": 0 + } + ] + ] + }, + "Format Healed Notify": { + "main": [ + [ + { + "node": "Gotify (Healed)", + "type": "main", + "index": 0 + } + ] + ] + }, + "Format Escalation": { + "main": [ + [ + { + "node": "Gotify (Escalation)", + "type": "main", + "index": 0 + } + ] + ] + } + }, + "settings": { + "executionOrder": "v1", + "binaryMode": "separate" + }, + "staticData": null, + "meta": null, + "pinData": {}, + "tags": [ + { + "name": "monitoring" + }, + { + "name": "infrastructure" + }, + { + "name": "scheduled" + } + ], + "id": "8587d430-3d53-4559-b610-214e12164c14" +} \ No newline at end of file diff --git a/n8n-workflows/TESTING-STATUS.md b/n8n-workflows/TESTING-STATUS.md old mode 100755 new mode 100644 diff --git a/rocinante/.env.example b/rocinante/.env.example new file mode 100644 index 0000000..0a0f5da --- /dev/null +++ b/rocinante/.env.example @@ -0,0 +1,2 @@ +TZ=America/Chicago +N8N_WEBHOOK_URL=https://n8n.paccoco.com/webhook/class/ingest diff --git a/rocinante/docker-compose.yaml b/rocinante/docker-compose.yaml new file mode 100644 index 0000000..1608ce8 --- /dev/null +++ b/rocinante/docker-compose.yaml @@ -0,0 +1,42 @@ +name: rocinante + +services: + + whisper: + image: fedirz/faster-whisper-server:latest-cuda + container_name: rocinante-whisper + restart: unless-stopped + ports: + - "8787:8000" + environment: + WHISPER__MODEL: large-v3 + WHISPER__INFERENCE_DEVICE: cuda + WHISPER__VAD_FILTER: "false" + WHISPER__CONDITION_ON_PREVIOUS_TEXT: "false" + TZ: ${TZ} + deploy: + resources: + reservations: + devices: + - driver: nvidia + count: 1 + capabilities: [gpu] + volumes: + - whisper-cache:/root/.cache/huggingface + + watcher: + build: ./watcher + container_name: rocinante-watcher + restart: unless-stopped + environment: + WHISPER_URL: http://whisper:8000 + N8N_WEBHOOK_URL: ${N8N_WEBHOOK_URL} + WATCH_DIR: /recordings + TZ: ${TZ} + volumes: + - C:/Recordings:/recordings + depends_on: + - whisper + +volumes: + whisper-cache: diff --git a/rocinante/watcher/Dockerfile b/rocinante/watcher/Dockerfile new file mode 100644 index 0000000..80b9673 --- /dev/null +++ b/rocinante/watcher/Dockerfile @@ -0,0 +1,12 @@ +FROM python:3.12-slim + +RUN apt-get update && \ + apt-get install -y --no-install-recommends ffmpeg && \ + rm -rf /var/lib/apt/lists/* + +RUN pip install --no-cache-dir watchdog requests + +COPY watch.py /app/watch.py +WORKDIR /app + +CMD ["python", "-u", "watch.py"] diff --git a/rocinante/watcher/watch.py b/rocinante/watcher/watch.py new file mode 100644 index 0000000..b14cff9 --- /dev/null +++ b/rocinante/watcher/watch.py @@ -0,0 +1,360 @@ +#!/usr/bin/env python3 +""" +Class recording watcher for Rocinante. + +Watches C:/Recordings (mounted as /recordings) for new .wav files. +For each file detected: + 1. Waits for the file to finish being written + 2. Converts WAV -> MP3 via ffmpeg (deletes original WAV) + 3. Splits long recordings into 10-minute chunks + 4. Transcribes each chunk via local faster-whisper-server (large-v3, CUDA) + 5. Stitches results back together with corrected timestamps + 6. POSTs transcript + metadata to n8n /webhook/class/ingest + +Folder convention: subfolder name = class name + /recordings/HIST-2020/lecture-03-3.wav -> class=HIST-2020, title=lecture 03 3 +""" + +import os +import time +import subprocess +import logging +import requests +from pathlib import Path +from watchdog.observers.polling import PollingObserver as Observer +from watchdog.events import FileSystemEventHandler + +# Config from environment +WHISPER_URL = os.environ.get("WHISPER_URL", "http://whisper:8000") +N8N_WEBHOOK = os.environ.get("N8N_WEBHOOK_URL", "https://n8n.paccoco.com/webhook/class/ingest") +WATCH_DIR = os.environ.get("WATCH_DIR", "/recordings") +WHISPER_MODEL = os.environ.get("WHISPER_MODEL", "large-v3") +CHUNK_MINUTES = int(os.environ.get("CHUNK_MINUTES", "10")) + +logging.basicConfig( + level=logging.INFO, + format="%(asctime)s %(levelname)-8s %(message)s", + datefmt="%Y-%m-%d %H:%M:%S", +) +log = logging.getLogger("watcher") + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +def wait_for_file_stable(path: Path, interval: float = 2.0, required_stable: int = 3) -> None: + """Block until the file size stops changing (i.e. the copy is complete).""" + log.info(f"Waiting for {path.name} to finish writing...") + prev_size = -1 + stable_count = 0 + while stable_count < required_stable: + try: + size = path.stat().st_size + except FileNotFoundError: + time.sleep(interval) + continue + if size == prev_size and size > 0: + stable_count += 1 + else: + stable_count = 0 + prev_size = size + time.sleep(interval) + log.info(f"{path.name} is stable ({prev_size:,} bytes)") + + # Extra check: verify the file handle is released and readable + for attempt in range(60): + try: + with open(path, 'rb') as f: + f.read(1024) + break + except (PermissionError, OSError) as e: + log.info(f"File not yet readable ({e}), retrying... ({attempt + 1}/60)") + time.sleep(5) + else: + raise RuntimeError(f"File {path.name} is not readable after 60 attempts") + + +def convert_to_mp3(wav_path: Path) -> Path: + mp3_path = wav_path.with_suffix(".mp3") + log.info(f"Converting to MP3: {wav_path.name}") + result = subprocess.run( + [ + "ffmpeg", "-y", + "-i", str(wav_path), + "-codec:a", "libmp3lame", + "-qscale:a", "4", + str(mp3_path), + ], + capture_output=True, + text=True, + ) + if result.returncode != 0: + raise RuntimeError(f"ffmpeg failed:\n{result.stderr}") + wav_path.unlink() + log.info(f"Deleted original WAV: {wav_path.name}") + return mp3_path + + +def get_duration(mp3_path: Path) -> float: + """Return audio duration in seconds via ffprobe.""" + result = subprocess.run( + [ + "ffprobe", "-v", "quiet", + "-show_entries", "format=duration", + "-of", "csv=p=0", + str(mp3_path), + ], + capture_output=True, + text=True, + ) + return float(result.stdout.strip() or "0") + + +def split_into_chunks(mp3_path: Path, chunk_minutes: int = CHUNK_MINUTES) -> list: + """Split MP3 into chunks. Returns [mp3_path] if short enough.""" + duration = get_duration(mp3_path) + chunk_secs = chunk_minutes * 60 + + if duration <= chunk_secs: + log.info(f"Recording is {duration/60:.1f}min — no splitting needed") + return [mp3_path] + + total_chunks = int(duration / chunk_secs) + 1 + log.info(f"Recording is {duration/60:.1f}min — splitting into {total_chunks} x {chunk_minutes}min chunks") + + chunks = [] + start = 0.0 + i = 0 + while start < duration: + chunk_path = mp3_path.parent / f"{mp3_path.stem}_chunk{i:03d}.mp3" + subprocess.run( + [ + "ffmpeg", "-y", + "-i", str(mp3_path), + "-ss", str(start), + "-t", str(chunk_secs), + "-codec:a", "copy", + str(chunk_path), + ], + capture_output=True, + ) + chunks.append((chunk_path, start)) + start += chunk_secs + i += 1 + + return chunks + + +def transcribe_chunk(mp3_path: Path) -> dict: + with open(mp3_path, "rb") as f: + resp = requests.post( + f"{WHISPER_URL}/v1/audio/transcriptions", + files={"file": (mp3_path.name, f, "audio/mpeg")}, + data={ + "model": WHISPER_MODEL, + "language": "en", + "response_format": "verbose_json", + }, + timeout=3600, + ) + resp.raise_for_status() + return resp.json() + + +def transcribe(mp3_path: Path) -> dict: + """Transcribe an MP3, splitting into chunks if needed.""" + chunks = split_into_chunks(mp3_path) + + # Short recording — single chunk, no splitting + if len(chunks) == 1 and isinstance(chunks[0], Path): + log.info(f"Transcribing {mp3_path.name} with {WHISPER_MODEL}...") + result = transcribe_chunk(mp3_path) + result["segments"] = deduplicate_hallucinations(result.get("segments", [])) + log.info("Transcription complete") + return result + + # Long recording — transcribe each chunk and stitch + all_segments = [] + all_text = [] + + for i, (chunk_path, time_offset) in enumerate(chunks): + log.info(f"Transcribing chunk {i+1}/{len(chunks)}: {chunk_path.name} (offset {time_offset/60:.1f}min)") + try: + result = transcribe_chunk(chunk_path) + segments = result.get("segments", []) + for s in segments: + s["start"] = s.get("start", 0) + time_offset + s["end"] = s.get("end", 0) + time_offset + all_segments.extend(segments) + all_text.append(result.get("text", "").strip()) + except Exception as e: + log.error(f"Chunk {i+1} failed: {e}") + finally: + chunk_path.unlink(missing_ok=True) + + all_segments = deduplicate_hallucinations(all_segments) + total_duration = get_duration(mp3_path) + log.info(f"Transcription complete — {len(chunks)} chunks stitched") + return { + "text": " ".join(all_text), + "segments": all_segments, + "duration": total_duration, + } + + +def is_silence(text: str) -> bool: + """Return True if the segment is just silence/noise markers.""" + stripped = text.strip() + return not stripped or all(c in '.…♪ \t' for c in stripped) + + +def deduplicate_hallucinations(segments: list, max_repeats: int = 3) -> list: + """ + Remove Whisper phrase-loop hallucinations. + + When Whisper gets stuck it emits the same segment text hundreds of times + in a row (e.g. "I'm also a software engineer." × 200). This function + collapses any run longer than `max_repeats` down to at most `max_repeats` + occurrences and logs a warning so the truncation is visible in the logs. + """ + if not segments: + return segments + + result = [] + i = 0 + while i < len(segments): + text = segments[i].get("text", "").strip().lower() + j = i + 1 + while j < len(segments) and segments[j].get("text", "").strip().lower() == text: + j += 1 + count = j - i + if count > max_repeats: + log.warning( + f"Hallucination detected: '{segments[i].get('text', '').strip()[:60]}'" + f" × {count} — keeping {max_repeats}, dropping {count - max_repeats}" + ) + result.extend(segments[i : i + max_repeats]) + else: + result.extend(segments[i:j]) + i = j + + return result + + +def build_timestamped(segments: list) -> str: + lines = [] + for s in segments: + text = s.get('text', '').strip() + if is_silence(text): + continue + mins = int(s.get("start", 0) // 60) + secs = int(s.get("start", 0) % 60) + lines.append(f"[{mins}:{secs:02d}] {text}") + return "\n".join(lines) + + +def send_to_n8n(class_name: str, lecture_title: str, result: dict) -> None: + segments = result.get("segments", []) + # Filter silence from full text + meaningful = [s.get("text", "") for s in segments if not is_silence(s.get("text", ""))] + full_text = " ".join(meaningful).strip() or result.get("text", "").strip() + duration = result.get("duration") or (segments[-1].get("end", 0) if segments else 0) + + payload = { + "class_name": class_name, + "lecture_title": lecture_title, + "transcript": full_text, + "timestamped_transcript": build_timestamped(segments), + "duration_seconds": duration, + } + + log.info(f"Sending to n8n: {class_name} / {lecture_title} ({len(full_text)} chars)") + resp = requests.post(N8N_WEBHOOK, json=payload, timeout=300) + resp.raise_for_status() + log.info(f"n8n accepted — HTTP {resp.status_code}") + + +# --------------------------------------------------------------------------- +# Processing pipeline +# --------------------------------------------------------------------------- + +def process_wav(wav_path: Path) -> None: + # Must be inside a class subfolder, not the root recordings dir + if wav_path.parent == Path(WATCH_DIR): + log.warning(f"Skipping {wav_path.name} — drop files into a class subfolder, e.g. /recordings/HIST-2020/") + return + + class_name = wav_path.parent.name + lecture_title = wav_path.stem.replace("-", " ").replace("_", " ") + + log.info(f"--- Processing: {class_name} / {lecture_title} ---") + + wait_for_file_stable(wav_path) + mp3_path = convert_to_mp3(wav_path) + + result = transcribe(mp3_path) + send_to_n8n(class_name, lecture_title, result) + + log.info(f"--- Done: {class_name} / {lecture_title} ---") + + +# --------------------------------------------------------------------------- +# Watchdog handler +# --------------------------------------------------------------------------- + +class WavHandler(FileSystemEventHandler): + def on_created(self, event): + if event.is_directory: + return + path = Path(event.src_path) + if path.suffix.lower() == ".wav": + log.info(f"Detected new WAV: {path}") + try: + process_wav(path) + except Exception as exc: + log.error(f"Failed to process {path}: {exc}", exc_info=True) + + +# --------------------------------------------------------------------------- +# Entry point +# --------------------------------------------------------------------------- + +def scan_existing(watch_dir: Path) -> None: + """Process any .wav files already present at startup.""" + existing = list(watch_dir.rglob("*.wav")) + list(watch_dir.rglob("*.WAV")) + if not existing: + return + log.info(f"Found {len(existing)} existing WAV(s) at startup — processing...") + for wav in existing: + try: + process_wav(wav) + except Exception as exc: + log.error(f"Failed to process {wav}: {exc}", exc_info=True) + + +if __name__ == "__main__": + watch_dir = Path(WATCH_DIR) + watch_dir.mkdir(parents=True, exist_ok=True) + + log.info(f"Whisper URL : {WHISPER_URL}") + log.info(f"n8n webhook : {N8N_WEBHOOK}") + log.info(f"Watching : {watch_dir} (recursive)") + + # Process any WAVs already sitting in the folder + scan_existing(watch_dir) + + observer = Observer() + observer.schedule(WavHandler(), str(watch_dir), recursive=True) + observer.start() + log.info("Watcher started — drop a .wav into a class subfolder to begin") + + try: + while True: + time.sleep(1) + except KeyboardInterrupt: + pass + finally: + observer.stop() + observer.join() + log.info("Watcher stopped")