diff --git a/docs/troubleshooting/KNOWN_QUIRKS.md b/docs/troubleshooting/KNOWN_QUIRKS.md index 24ac51b..b728741 100644 --- a/docs/troubleshooting/KNOWN_QUIRKS.md +++ b/docs/troubleshooting/KNOWN_QUIRKS.md @@ -5,12 +5,13 @@ Per-service gotchas that aren't bugs but will bite you if you forget them. ## PlausibleDeniability ### n8n workflow 16 (NFS Watchdog) -- Runs `sudo docker restart` and `sudo bash .../mount-unraid-nfs.sh` over SSH from n8n container +- Runs `sudo docker restart` and `sudo /usr/bin/bash .../mount-unraid-nfs.sh` over SSH from n8n container - Requires a NOPASSWD sudoers rule or SSH commands will hang waiting for a password: ``` - truenas_admin ALL=(ALL) NOPASSWD: /usr/bin/docker, /bin/bash /mnt/tank/docker/scripts/mount-unraid-nfs.sh + truenas_admin ALL=(ALL) NOPASSWD: /usr/bin/docker, /usr/bin/bash /mnt/tank/docker/scripts/mount-unraid-nfs.sh ``` Add via `sudo visudo -f /etc/sudoers.d/n8n-watchdog` +- **Must use `/usr/bin/bash` not `/bin/bash`** — on TrueNAS SCALE, bash is at `/usr/bin/bash`. The sudoers rule and the workflow command must match exactly or sudo will prompt for a password. - SSH credential in n8n must use key-based auth (see n8n-workflows/README.md for keygen steps) ### immich-ml diff --git a/n8n-workflows/16-nfs-watchdog-v1.2.json b/n8n-workflows/16-nfs-watchdog-v1.2.json new file mode 100644 index 0000000..16aa457 --- /dev/null +++ b/n8n-workflows/16-nfs-watchdog-v1.2.json @@ -0,0 +1,303 @@ +{ + "name": "NFS Mount Watchdog + Auto-Heal v1.2", + "active": true, + "isArchived": false, + "nodes": [ + { + "parameters": { + "rule": { + "interval": [ + { + "field": "minutes", + "minutesInterval": 5 + } + ] + } + }, + "id": "8c466a67-a0fa-42dc-8ed8-0918834fc3a7", + "name": "Every 5 Minutes", + "type": "n8n-nodes-base.scheduleTrigger", + "typeVersion": 1.2, + "position": [0, 0] + }, + { + "parameters": { + "command": "mountpoint -q /mnt/unraid/data && echo DATA_OK || echo DATA_MISSING; mountpoint -q /mnt/unraid/immich && echo IMMICH_OK || echo IMMICH_MISSING" + }, + "id": "f5b2fee9-d67a-4da7-80dd-f72e6078aed9", + "name": "Probe Mounts", + "type": "n8n-nodes-base.ssh", + "typeVersion": 1, + "position": [224, 0], + "credentials": { + "sshPrivateKey": { + "id": "SETUP_REQUIRED", + "name": "PD SSH" + } + }, + "continueOnFail": true + }, + { + "parameters": { + "jsCode": "const out = ($input.first().json.stdout || '') + ($input.first().json.stderr || '');\nconst dataOk = out.includes('DATA_OK');\nconst immichOk = out.includes('IMMICH_OK');\nconst missing = [];\nif (!dataOk) missing.push({ mount: '/mnt/unraid/data', label: 'data',\n containers: ['ix-plex-plex-1', 'ix-audiobookshelf-audiobookshelf-1'] });\nif (!immichOk) missing.push({ mount: '/mnt/unraid/immich', label: 'immich',\n containers: ['ix-immich-immich-server-1', 'ix-immich-immich-machine-learning-1'] });\nreturn [{ json: { dataOk, immichOk, anyMissing: missing.length > 0, anyMissingStr: missing.length > 0 ? 'yes' : 'no', missing } }];\n" + }, + "id": "25bb3239-49cb-4570-b706-4022b9aa751c", + "name": "Parse Mount Status", + "type": "n8n-nodes-base.code", + "typeVersion": 2, + "position": [448, 0] + }, + { + "parameters": { + "conditions": { + "options": { + "caseSensitive": true, + "leftValue": "", + "typeValidation": "strict" + }, + "conditions": [ + { + "id": "70384f84-9d15-4e45-b295-67202a39a227", + "leftValue": "={{ $json.anyMissingStr }}", + "rightValue": "yes", + "operator": { + "type": "string", + "operation": "equals" + } + } + ], + "combinator": "and" + } + }, + "id": "68b74832-85a7-4193-9382-a0c22a666e66", + "name": "Any Missing?", + "type": "n8n-nodes-base.if", + "typeVersion": 2.2, + "position": [672, 0] + }, + { + "parameters": { + "command": "sudo /usr/bin/bash /mnt/tank/docker/scripts/mount-unraid-nfs.sh 2>&1" + }, + "id": "b18c7537-0c2f-4122-bfe5-6469d6be6853", + "name": "Run Mount Script", + "type": "n8n-nodes-base.ssh", + "typeVersion": 1, + "position": [896, -120], + "credentials": { + "sshPrivateKey": { + "id": "SETUP_REQUIRED", + "name": "PD SSH" + } + }, + "continueOnFail": true + }, + { + "parameters": { + "resume": "timeInterval", + "unit": "seconds", + "value": 10 + }, + "id": "2e927f83-9ddc-4816-8ee6-76d21ce21a5e", + "name": "Wait 10s", + "type": "n8n-nodes-base.wait", + "typeVersion": 1.1, + "position": [1120, -120], + "webhookId": "4e89e8d0-7f08-4378-a2b7-63756665b039" + }, + { + "parameters": { + "command": "mountpoint -q /mnt/unraid/data && echo DATA_OK || echo DATA_MISSING; mountpoint -q /mnt/unraid/immich && echo IMMICH_OK || echo IMMICH_MISSING" + }, + "id": "d15c0c90-e94f-404a-a787-75f1b1b5befa", + "name": "Re-probe Mounts", + "type": "n8n-nodes-base.ssh", + "typeVersion": 1, + "position": [1344, -120], + "credentials": { + "sshPrivateKey": { + "id": "SETUP_REQUIRED", + "name": "PD SSH" + } + }, + "continueOnFail": true + }, + { + "parameters": { + "jsCode": "const out = ($input.first().json.stdout || '') + ($input.first().json.stderr || '');\nconst dataOk = out.includes('DATA_OK');\nconst immichOk = out.includes('IMMICH_OK');\n\nconst originalMissing = $('Parse Mount Status').first().json.missing || [];\n\nconst healed = originalMissing.filter(m =>\n (m.label === 'data' && dataOk) ||\n (m.label === 'immich' && immichOk)\n);\nconst stillDown = originalMissing.filter(m =>\n (m.label === 'data' && !dataOk) ||\n (m.label === 'immich' && !immichOk)\n);\n\nconst containersToRestart = [...new Set(healed.flatMap(m => m.containers))];\nconst restartCmd = containersToRestart.length > 0\n ? `sudo docker restart ${containersToRestart.join(' ')}`\n : 'echo NO_RESTART_NEEDED';\n\nreturn [{ json: {\n allHealed: stillDown.length === 0,\n allHealedStr: stillDown.length === 0 ? 'yes' : 'no',\n healed,\n stillDown,\n containersToRestart,\n restartCmd,\n originalMissing\n} }];\n" + }, + "id": "48de38c2-df08-406f-ad61-809ca6e306a1", + "name": "Parse Heal Result", + "type": "n8n-nodes-base.code", + "typeVersion": 2, + "position": [1568, -120] + }, + { + "parameters": { + "conditions": { + "options": { + "caseSensitive": true, + "leftValue": "", + "typeValidation": "strict" + }, + "conditions": [ + { + "id": "32b2dfd5-9958-45d4-94fc-410ce4d9543d", + "leftValue": "={{ $json.allHealedStr }}", + "rightValue": "yes", + "operator": { + "type": "string", + "operation": "equals" + } + } + ], + "combinator": "and" + } + }, + "id": "29a1b544-97b3-433f-9821-2bf5a8af95d2", + "name": "Healed?", + "type": "n8n-nodes-base.if", + "typeVersion": 2.2, + "position": [1792, -120] + }, + { + "parameters": { + "command": "={{ $json.restartCmd }}" + }, + "id": "8d791509-9e18-44c1-bf58-ed3027927710", + "name": "Restart Containers", + "type": "n8n-nodes-base.ssh", + "typeVersion": 1, + "position": [2016, -240], + "credentials": { + "sshPrivateKey": { + "id": "SETUP_REQUIRED", + "name": "PD SSH" + } + }, + "continueOnFail": true + }, + { + "parameters": { + "jsCode": "const d = $('Parse Heal Result').first().json;\nconst mountList = d.healed.map(m => ` • ${m.mount}`).join('\\n');\nconst containers = d.containersToRestart.join(', ') || 'none';\nconst body = `NFS mounts were missing and have been automatically remounted.\\n\\nRemounted:\\n${mountList}\\n\\nContainers restarted: ${containers}`;\nreturn [{ json: { title: '🔗 NFS Auto-Healed', message: body, priority: 5 } }];\n" + }, + "id": "75c8976f-d622-43cd-a332-72c32f0c7c60", + "name": "Format Healed Notify", + "type": "n8n-nodes-base.code", + "typeVersion": 2, + "position": [2240, -240] + }, + { + "parameters": { + "message": "={{ $json.message }}", + "additionalFields": { + "priority": "={{ $json.priority }}", + "title": "={{ $json.title }}" + }, + "options": { + "contentType": "text/plain" + } + }, + "id": "38705cd2-775b-4ef9-b470-4a7709f132e5", + "name": "Gotify (Healed)", + "type": "n8n-nodes-base.gotify", + "typeVersion": 1, + "position": [2464, -240], + "credentials": { + "gotifyApi": { + "id": "SETUP_REQUIRED", + "name": "Uptime / Infra" + } + } + }, + { + "parameters": { + "jsCode": "const d = $('Parse Heal Result').first().json;\nconst downList = d.stillDown.map(m => ` • ${m.mount}`).join('\\n');\nconst healedList = d.healed.length > 0\n ? `\\nPartially healed:\\n${d.healed.map(m => ' • ' + m.mount).join('\\n')}`\n : '';\nconst body = `NFS mount script ran but the following paths are still unavailable. Manual intervention required.\\n\\nStill down:\\n${downList}${healedList}\\n\\nCheck Serenity (10.5.1.5) is up and NFS service is running.`;\nreturn [{ json: { title: '🚨 NFS Down — Manual Fix Needed', message: body, priority: 9 } }];\n" + }, + "id": "074b984f-7fc2-45bc-b97f-f5db4d8d589b", + "name": "Format Escalation", + "type": "n8n-nodes-base.code", + "typeVersion": 2, + "position": [2016, -20] + }, + { + "parameters": { + "message": "={{ $json.message }}", + "additionalFields": { + "priority": "={{ $json.priority }}", + "title": "={{ $json.title }}" + }, + "options": { + "contentType": "text/plain" + } + }, + "id": "f929dd77-6072-4ac9-9dc4-11246110efed", + "name": "Gotify (Escalation)", + "type": "n8n-nodes-base.gotify", + "typeVersion": 1, + "position": [2240, -20], + "credentials": { + "gotifyApi": { + "id": "SETUP_REQUIRED", + "name": "Uptime / Infra" + } + } + } + ], + "connections": { + "Every 5 Minutes": { + "main": [[{ "node": "Probe Mounts", "type": "main", "index": 0 }]] + }, + "Probe Mounts": { + "main": [[{ "node": "Parse Mount Status", "type": "main", "index": 0 }]] + }, + "Parse Mount Status": { + "main": [[{ "node": "Any Missing?", "type": "main", "index": 0 }]] + }, + "Any Missing?": { + "main": [ + [{ "node": "Run Mount Script", "type": "main", "index": 0 }], + [] + ] + }, + "Run Mount Script": { + "main": [[{ "node": "Wait 10s", "type": "main", "index": 0 }]] + }, + "Wait 10s": { + "main": [[{ "node": "Re-probe Mounts", "type": "main", "index": 0 }]] + }, + "Re-probe Mounts": { + "main": [[{ "node": "Parse Heal Result", "type": "main", "index": 0 }]] + }, + "Parse Heal Result": { + "main": [[{ "node": "Healed?", "type": "main", "index": 0 }]] + }, + "Healed?": { + "main": [ + [{ "node": "Restart Containers", "type": "main", "index": 0 }], + [{ "node": "Format Escalation", "type": "main", "index": 0 }] + ] + }, + "Restart Containers": { + "main": [[{ "node": "Format Healed Notify", "type": "main", "index": 0 }]] + }, + "Format Healed Notify": { + "main": [[{ "node": "Gotify (Healed)", "type": "main", "index": 0 }]] + }, + "Format Escalation": { + "main": [[{ "node": "Gotify (Escalation)", "type": "main", "index": 0 }]] + } + }, + "settings": { + "executionOrder": "v1", + "binaryMode": "separate" + }, + "staticData": null, + "meta": null, + "pinData": {}, + "tags": [ + { "name": "monitoring" }, + { "name": "infrastructure" }, + { "name": "scheduled" } + ] +}