From 5fd46b20caeacac3e2f44338e52a993d972278ed Mon Sep 17 00:00:00 2001 From: Fizzlepoof Date: Wed, 13 May 2026 03:38:31 +0000 Subject: [PATCH] feat: add paperless intake triage workflow --- n8n-workflows/17-paperless-intake-triage.json | 369 ++++++++++++++++++ n8n-workflows/README.md | 108 +++++ n8n-workflows/TESTING-STATUS.md | 19 + .../paperless-intake-document.example.json | 11 + ...erless-intake-expected-triage.example.json | 12 + .../paperless-intake-webhook.example.json | 3 + 6 files changed, 522 insertions(+) create mode 100644 n8n-workflows/17-paperless-intake-triage.json create mode 100644 n8n-workflows/fixtures/paperless-intake-document.example.json create mode 100644 n8n-workflows/fixtures/paperless-intake-expected-triage.example.json create mode 100644 n8n-workflows/fixtures/paperless-intake-webhook.example.json diff --git a/n8n-workflows/17-paperless-intake-triage.json b/n8n-workflows/17-paperless-intake-triage.json new file mode 100644 index 0000000..eb8df55 --- /dev/null +++ b/n8n-workflows/17-paperless-intake-triage.json @@ -0,0 +1,369 @@ +{ + "name": "Paperless Intake Triage \u2192 Doris Review Queue", + "active": false, + "isArchived": false, + "nodes": [ + { + "parameters": { + "httpMethod": "POST", + "path": "paperless/intake-triage", + "options": {} + }, + "id": "paperless-webhook", + "name": "Paperless Intake Webhook", + "type": "n8n-nodes-base.webhook", + "typeVersion": 2, + "position": [ + -980, + 0 + ] + }, + { + "parameters": { + "jsCode": "\nconst body = $json.body || $json;\nconst id = body.document_id || body.id || body.document;\nif (!id) throw new Error('Missing document_id/id in Paperless webhook payload');\nreturn [{ json: { document_id: String(id), received_at: new Date().toISOString(), webhook_payload_keys: Object.keys(body) } }];\n" + }, + "id": "extract-doc-id", + "name": "Extract Document ID", + "type": "n8n-nodes-base.code", + "typeVersion": 2, + "position": [ + -760, + 0 + ] + }, + { + "parameters": { + "url": "={{($env.PAPERLESS_BASE_URL || 'https://paperless.paccoco.com').replace(/\\/$/, '') + '/api/documents/' + $json.document_id + '/'}}", + "sendHeaders": true, + "headerParameters": { + "parameters": [ + { + "name": "Authorization", + "value": "={{'Token ' + $env.PAPERLESS_API_TOKEN}}" + }, + { + "name": "Accept", + "value": "application/json" + } + ] + }, + "options": { + "timeout": 30000 + } + }, + "id": "fetch-document", + "name": "Fetch Paperless Document", + "type": "n8n-nodes-base.httpRequest", + "typeVersion": 4.2, + "position": [ + -540, + 0 + ] + }, + { + "parameters": { + "jsCode": "\nconst doc = $json;\nconst truncate = (s, n=9000) => String(s || '').replace(/\\s+/g, ' ').trim().slice(0, n);\nconst archiveUrl = `${($env.PAPERLESS_BASE_URL || 'https://paperless.paccoco.com').replace(/\\/$/, '')}/documents/${doc.id}/details`;\nconst payload = {\n id: doc.id,\n title: doc.title || null,\n original_filename: doc.original_filename || doc.archive_filename || null,\n created: doc.created || null,\n added: doc.added || doc.created_date || null,\n correspondent: doc.correspondent_detail?.name || doc.correspondent || null,\n document_type: doc.document_type_detail?.name || doc.document_type || null,\n tags: (doc.tags_detail || doc.tags || []).map(t => typeof t === 'string' ? t : (t.name || t.id)).filter(Boolean),\n archive_url: archiveUrl,\n content: truncate(doc.content || doc.text || '')\n};\nconst schema = `Return STRICT JSON only with exactly these keys: summary, suggested_title, document_type, suggested_correspondent, suggested_tags, due_date, needs_review, urgency, reason, confidence. document_type must be one of bill, receipt, school, medical, tax, insurance, legal, manual, letter, other. due_date is YYYY-MM-DD or null. urgency is none, low, medium, high. confidence is high, medium, low.`;\nreturn [{ json: { document: payload, llm_messages: [\n { role: 'system', content: 'You triage Paperless-NGX documents for a human review-first dashboard. Never invent due dates. Be conservative. ' + schema },\n { role: 'user', content: JSON.stringify(payload) }\n] } }];\n" + }, + "id": "prepare-triage", + "name": "Prepare Compact Triage Payload", + "type": "n8n-nodes-base.code", + "typeVersion": 2, + "position": [ + -320, + 0 + ] + }, + { + "parameters": { + "method": "POST", + "url": "={{($env.LITELLM_BASE_URL || 'http://10.5.1.6:4000').replace(/\\/$/, '') + '/v1/chat/completions'}}", + "sendHeaders": true, + "headerParameters": { + "parameters": [ + { + "name": "Authorization", + "value": "={{'Bearer ' + $env.LITELLM_API_KEY}}" + }, + { + "name": "Content-Type", + "value": "application/json" + } + ] + }, + "sendBody": true, + "specifyBody": "json", + "jsonBody": "={{ JSON.stringify({ model: $env.PAPERLESS_TRIAGE_MODEL || 'gpt-4o-mini', temperature: 0.1, response_format: { type: 'json_object' }, messages: $json.llm_messages }) }}", + "options": { + "timeout": 60000 + } + }, + "id": "llm-triage", + "name": "LiteLLM Triage JSON", + "type": "n8n-nodes-base.httpRequest", + "typeVersion": 4.2, + "position": [ + -100, + 0 + ] + }, + { + "parameters": { + "jsCode": "\nfunction parseTriage(input) {\n let content = input.choices?.[0]?.message?.content ?? input.text ?? input;\n if (typeof content !== 'string') content = JSON.stringify(content);\n content = content.trim().replace(/^```json\\s*/i, '').replace(/```$/,'').trim();\n return JSON.parse(content);\n}\nconst sourceDoc = $('Prepare Compact Triage Payload').first().json.document;\nconst t = parseTriage($json);\nconst allowedTypes = new Set(['bill','receipt','school','medical','tax','insurance','legal','manual','letter','other']);\nconst out = {\n summary: String(t.summary || '').slice(0, 800),\n suggested_title: String(t.suggested_title || sourceDoc.title || sourceDoc.original_filename || `Document ${sourceDoc.id}`).slice(0, 200),\n document_type: allowedTypes.has(t.document_type) ? t.document_type : 'other',\n suggested_correspondent: t.suggested_correspondent ? String(t.suggested_correspondent).slice(0, 160) : null,\n suggested_tags: Array.isArray(t.suggested_tags) ? [...new Set(t.suggested_tags.map(x => String(x).trim().toLowerCase()).filter(Boolean))].slice(0, 12) : [],\n due_date: /^\\d{4}-\\d{2}-\\d{2}$/.test(String(t.due_date || '')) ? String(t.due_date) : null,\n needs_review: Boolean(t.needs_review),\n urgency: ['none','low','medium','high'].includes(t.urgency) ? t.urgency : 'medium',\n reason: String(t.reason || 'No reason supplied; review required.').slice(0, 500),\n confidence: ['high','medium','low'].includes(t.confidence) ? t.confidence : 'low'\n};\nconst riskTypes = new Set(['legal','medical','school','tax','insurance']);\nconst billSignals = /\b(bill|payment|invoice|amount due|pay by|past due|statement)\b/i.test(`${out.summary} ${out.suggested_title} ${sourceDoc.content || ''}`);\nconst forcedReview = riskTypes.has(out.document_type) || Boolean(out.due_date) || billSignals || out.confidence === 'low' || ['medium','high'].includes(out.urgency);\nout.needs_review = out.needs_review || forcedReview;\nif (forcedReview && !/review/i.test(out.reason)) out.reason = `${out.reason} Safety gate requires human review.`;\nconst applySafe = String($env.PAPERLESS_TRIAGE_APPLY_SAFE || 'false').toLowerCase() === 'true';\nconst eligible_for_safe_apply = !out.needs_review && out.confidence === 'high' && ['none','low'].includes(out.urgency) && !riskTypes.has(out.document_type) && !out.due_date;\nconst reviewItem = {\n id: sourceDoc.id,\n title: sourceDoc.title,\n original_filename: sourceDoc.original_filename,\n added: sourceDoc.added,\n archive_url: sourceDoc.archive_url,\n current: { correspondent: sourceDoc.correspondent, document_type: sourceDoc.document_type, tags: sourceDoc.tags },\n triage: out,\n safety: { forced_review: forcedReview, eligible_for_safe_apply, apply_safe_enabled: applySafe },\n queued_at: new Date().toISOString()\n};\nreturn [{ json: { document: sourceDoc, triage: out, review_item: reviewItem, should_apply_safe: applySafe && eligible_for_safe_apply, should_notify: out.urgency === 'high' } }];\n" + }, + "id": "parse-safety", + "name": "Parse JSON + Safety Gate", + "type": "n8n-nodes-base.code", + "typeVersion": 2, + "position": [ + 120, + 0 + ] + }, + { + "parameters": { + "jsCode": "\n// Requires n8n Code node filesystem access: NODE_FUNCTION_ALLOW_BUILTIN=fs,path\nconst fs = require('fs');\nconst path = require('path');\nconst queuePath = $env.PAPERLESS_TRIAGE_REVIEW_QUEUE_PATH || '/data/paperless_review.json';\nconst maxItems = Number($env.PAPERLESS_TRIAGE_REVIEW_MAX_ITEMS || 200);\nfs.mkdirSync(path.dirname(queuePath), { recursive: true });\nlet queue = [];\ntry { queue = JSON.parse(fs.readFileSync(queuePath, 'utf8')); } catch (_) { queue = []; }\nif (!Array.isArray(queue)) queue = [];\nconst item = $json.review_item;\nqueue = [item, ...queue.filter(x => String(x.id) !== String(item.id))].slice(0, maxItems);\nconst tmp = queuePath + '.tmp';\nfs.writeFileSync(tmp, JSON.stringify(queue, null, 2) + '\n', 'utf8');\nfs.renameSync(tmp, queuePath);\nreturn [{ json: { ...$json, queue_path: queuePath, queue_count: queue.length } }];\n" + }, + "id": "write-queue", + "name": "Write Doris Review Queue JSON", + "type": "n8n-nodes-base.code", + "typeVersion": 2, + "position": [ + 340, + 0 + ] + }, + { + "parameters": { + "conditions": { + "boolean": [ + { + "value1": "={{$json.should_apply_safe}}", + "value2": true + } + ] + } + }, + "id": "if-safe-apply", + "name": "Apply Safe Updates Enabled?", + "type": "n8n-nodes-base.if", + "typeVersion": 2.2, + "position": [ + 560, + -80 + ] + }, + { + "parameters": { + "method": "PATCH", + "url": "={{($env.PAPERLESS_BASE_URL || 'https://paperless.paccoco.com').replace(/\\/$/, '') + '/api/documents/' + $json.document.id + '/'}}", + "sendHeaders": true, + "headerParameters": { + "parameters": [ + { + "name": "Authorization", + "value": "={{'Token ' + $env.PAPERLESS_API_TOKEN}}" + }, + { + "name": "Content-Type", + "value": "application/json" + } + ] + }, + "sendBody": true, + "specifyBody": "json", + "jsonBody": "={{ JSON.stringify({ title: $json.triage.suggested_title }) }}", + "options": { + "timeout": 30000 + } + }, + "id": "safe-title-update", + "name": "Safe Title Update Only", + "type": "n8n-nodes-base.httpRequest", + "typeVersion": 4.2, + "position": [ + 780, + -160 + ] + }, + { + "parameters": { + "conditions": { + "boolean": [ + { + "value1": "={{$json.should_notify && !!$env.GOTIFY_URL && !!$env.GOTIFY_TOKEN}}", + "value2": true + } + ] + } + }, + "id": "if-urgent", + "name": "Urgent Notification Configured?", + "type": "n8n-nodes-base.if", + "typeVersion": 2.2, + "position": [ + 560, + 140 + ] + }, + { + "parameters": { + "method": "POST", + "url": "={{$env.GOTIFY_URL.replace(/\\/$/, '') + '/message'}}", + "sendHeaders": true, + "headerParameters": { + "parameters": [ + { + "name": "X-Gotify-Key", + "value": "={{$env.GOTIFY_TOKEN}}" + }, + { + "name": "Content-Type", + "value": "application/json" + } + ] + }, + "sendBody": true, + "specifyBody": "json", + "jsonBody": "={{ JSON.stringify({ title: 'Urgent Paperless document', message: `${$json.triage.suggested_title}\n${$json.triage.summary}\n${$json.document.archive_url}`, priority: 8 }) }}", + "options": { + "timeout": 15000 + } + }, + "id": "gotify-urgent", + "name": "Send Urgent Gotify", + "type": "n8n-nodes-base.httpRequest", + "typeVersion": 4.2, + "position": [ + 780, + 140 + ] + } + ], + "connections": { + "Paperless Intake Webhook": { + "main": [ + [ + { + "node": "Extract Document ID", + "type": "main", + "index": 0 + } + ] + ] + }, + "Extract Document ID": { + "main": [ + [ + { + "node": "Fetch Paperless Document", + "type": "main", + "index": 0 + } + ] + ] + }, + "Fetch Paperless Document": { + "main": [ + [ + { + "node": "Prepare Compact Triage Payload", + "type": "main", + "index": 0 + } + ] + ] + }, + "Prepare Compact Triage Payload": { + "main": [ + [ + { + "node": "LiteLLM Triage JSON", + "type": "main", + "index": 0 + } + ] + ] + }, + "LiteLLM Triage JSON": { + "main": [ + [ + { + "node": "Parse JSON + Safety Gate", + "type": "main", + "index": 0 + } + ] + ] + }, + "Parse JSON + Safety Gate": { + "main": [ + [ + { + "node": "Write Doris Review Queue JSON", + "type": "main", + "index": 0 + } + ] + ] + }, + "Write Doris Review Queue JSON": { + "main": [ + [ + { + "node": "Apply Safe Updates Enabled?", + "type": "main", + "index": 0 + }, + { + "node": "Urgent Notification Configured?", + "type": "main", + "index": 0 + } + ] + ] + }, + "Apply Safe Updates Enabled?": { + "main": [ + [ + { + "node": "Safe Title Update Only", + "type": "main", + "index": 0 + } + ], + [] + ] + }, + "Urgent Notification Configured?": { + "main": [ + [ + { + "node": "Send Urgent Gotify", + "type": "main", + "index": 0 + } + ], + [] + ] + } + }, + "settings": { + "executionOrder": "v1" + }, + "staticData": null, + "meta": { + "templateCredsSetupCompleted": false + }, + "pinData": {}, + "tags": [ + { + "name": "paperless" + }, + { + "name": "doris" + }, + { + "name": "review-first" + } + ] +} diff --git a/n8n-workflows/README.md b/n8n-workflows/README.md index f62cddb..2926c78 100644 --- a/n8n-workflows/README.md +++ b/n8n-workflows/README.md @@ -268,3 +268,111 @@ Edit the "Define RSS Feeds" Code node to add/remove feeds. Default feeds: - Hacker News Add any RSS feed URL you want monitored. + +--- + +## Workflow 17 — Paperless Intake Triage → Doris Review Queue + +Import `17-paperless-intake-triage.json` for a review-first Paperless intake workflow. It is intentionally conservative: it writes a sanitized local review queue for Doris Dashboard and does **not** delete documents. Automatic Paperless updates are disabled by default. + +### Trigger + +Configure a Paperless-NGX post-consumption webhook to POST to: + +```text +https://n8n.paccoco.com/webhook/paperless/intake-triage +``` + +Body: + +```json +{"document_id": 12345} +``` + +Fixture: `fixtures/paperless-intake-webhook.example.json`. + +### Environment variables + +Add these to `automation/.env` and expose them in the n8n `environment:` block: + +| Variable | Required | Default | Purpose | +|----------|----------|---------|---------| +| `PAPERLESS_BASE_URL` | yes | `https://paperless.paccoco.com` | Paperless API/UI base URL | +| `PAPERLESS_API_TOKEN` | yes | none | Paperless API token; used as `Token ...` | +| `LITELLM_BASE_URL` | yes | `http://10.5.1.6:4000` | LiteLLM OpenAI-compatible endpoint | +| `LITELLM_API_KEY` | yes | none | LiteLLM key | +| `PAPERLESS_TRIAGE_MODEL` | no | `gpt-4o-mini` | Model for strict JSON triage | +| `PAPERLESS_TRIAGE_REVIEW_QUEUE_PATH` | yes | `/data/paperless_review.json` | Sanitized JSON queue path mounted for Doris Dashboard | +| `PAPERLESS_TRIAGE_REVIEW_MAX_ITEMS` | no | `200` | Max queue entries retained | +| `PAPERLESS_TRIAGE_APPLY_SAFE` | no | `false` | Enables low-risk title-only update branch when `true` | +| `GOTIFY_URL` | no | none | Enables urgent-doc notification only when token also set | +| `GOTIFY_TOKEN` | no | none | Gotify app token for urgent docs | +| `NODE_FUNCTION_ALLOW_BUILTIN` | yes for queue file | none | Must include `fs,path` for the Code node that writes JSON | +| `N8N_BLOCK_ENV_ACCESS_IN_NODE` | yes | existing note | Must be `false` so Code nodes can read `$env` | + +Example environment additions: + +```yaml + N8N_BLOCK_ENV_ACCESS_IN_NODE: "false" + NODE_FUNCTION_ALLOW_BUILTIN: fs,path + PAPERLESS_BASE_URL: ${PAPERLESS_BASE_URL} + PAPERLESS_API_TOKEN: ${PAPERLESS_API_TOKEN} + LITELLM_BASE_URL: ${LITELLM_BASE_URL} + LITELLM_API_KEY: ${LITELLM_API_KEY} + PAPERLESS_TRIAGE_MODEL: ${PAPERLESS_TRIAGE_MODEL} + PAPERLESS_TRIAGE_REVIEW_QUEUE_PATH: /workspace/doris-dashboard/data/paperless_review.json + PAPERLESS_TRIAGE_APPLY_SAFE: "false" +``` + +Mount the Doris dashboard data directory read/write into the n8n container if it is not already mounted. The workflow writes only sanitized display data: document IDs, titles, filenames, archive URL, current metadata names/IDs, triage summary, review reason, urgency, confidence, and safety flags. It must not contain tokens. + +### Safety gates + +The LLM must return strict JSON: + +```json +{ + "summary": "plain English summary", + "suggested_title": "clean title", + "document_type": "bill|receipt|school|medical|tax|insurance|legal|manual|letter|other", + "suggested_correspondent": "string or null", + "suggested_tags": ["..."], + "due_date": "YYYY-MM-DD or null", + "needs_review": true, + "urgency": "none|low|medium|high", + "reason": "why review/why not", + "confidence": "high|medium|low" +} +``` + +The workflow forcibly marks `needs_review=true` for: + +- legal, medical, school, tax, or insurance documents +- any due date +- bill/payment/invoice language +- low confidence +- urgency medium or high + +Only high-confidence, low-risk docs with no due date and urgency `none|low` become eligible for automatic update, and even then the branch is disabled unless `PAPERLESS_TRIAGE_APPLY_SAFE=true`. The safe update branch currently updates **title only**. It does not delete, archive, or apply high-risk metadata. + +### Test payload and expected result + +- Webhook payload: `fixtures/paperless-intake-webhook.example.json` +- Example Paperless API document: `fixtures/paperless-intake-document.example.json` +- Expected triage JSON: `fixtures/paperless-intake-expected-triage.example.json` + +The fixture is a utility bill with a due date, so the safety gate must force human review. + +### Import/deploy steps + +1. Import `17-paperless-intake-triage.json` in n8n. +2. Add env vars and required volume mount. +3. Restart n8n using the normal automation compose procedure. +4. Keep `PAPERLESS_TRIAGE_APPLY_SAFE=false` for initial testing. +5. Use a known test document and POST `{"document_id": }` to `/webhook/paperless/intake-triage`. +6. Confirm `doris-dashboard/data/paperless_review.json` updates and contains no secrets. +7. Activate the Paperless webhook only after the test queue file is correct. + +### Rollback + +Deactivate the workflow and remove the Paperless webhook target. If needed, move `doris-dashboard/data/paperless_review.json` aside; it is display-only derived data. No Paperless deletion path exists in this workflow. diff --git a/n8n-workflows/TESTING-STATUS.md b/n8n-workflows/TESTING-STATUS.md index c85915a..2ff856f 100644 --- a/n8n-workflows/TESTING-STATUS.md +++ b/n8n-workflows/TESTING-STATUS.md @@ -86,3 +86,22 @@ _Last updated: 2026-05-10_ | 07 | Hardcoded LiteLLM API key (working but tech debt) | Low | | 15 | `includeData: false` means error messages show as "Unknown error" | Low | | 09 v1.0 | Still active in n8n alongside v1.1 — should be deactivated | Low | + +--- + +### 17 — Paperless Intake Triage → Doris Review Queue v1.0 +**What it does:** Paperless post-consumption webhook → fetch document → LiteLLM strict JSON triage → enforce review-first safety gate → write sanitized Doris Dashboard queue JSON → optional urgent Gotify notification. Safe title-only Paperless update branch exists but is disabled unless `PAPERLESS_TRIAGE_APPLY_SAFE=true`. + +**Local artifact checks completed:** +- [x] Workflow JSON validates with `python3 -m json.tool` +- [x] Fixture JSON files validate with `python3 -m json.tool` +- [x] Dashboard example queue JSON validates with `python3 -m json.tool` + +**Pre-test checklist:** +- [ ] Paperless webhook configured to POST `https://n8n.paccoco.com/webhook/paperless/intake-triage` +- [ ] `PAPERLESS_API_TOKEN` and `LITELLM_API_KEY` exposed to n8n through env vars; no hardcoded secrets +- [ ] `NODE_FUNCTION_ALLOW_BUILTIN=fs,path` set for queue file writer +- [ ] n8n has a writeable mount to `doris-dashboard/data/paperless_review.json` +- [ ] `PAPERLESS_TRIAGE_APPLY_SAFE=false` during initial validation + +**Test approach:** Upload a harmless test document to Paperless, POST `{"document_id": }` to the webhook, and confirm `doris-dashboard/data/paperless_review.json` contains a sanitized review item. Use a bill/due-date fixture to verify forced human review. diff --git a/n8n-workflows/fixtures/paperless-intake-document.example.json b/n8n-workflows/fixtures/paperless-intake-document.example.json new file mode 100644 index 0000000..69f6787 --- /dev/null +++ b/n8n-workflows/fixtures/paperless-intake-document.example.json @@ -0,0 +1,11 @@ +{ + "id": 12345, + "title": "scan_2026_05_12", + "original_filename": "utility_statement_may.pdf", + "created": "2026-05-12", + "added": "2026-05-13T02:14:00Z", + "correspondent_detail": {"name": "CDE Lightband"}, + "document_type_detail": {"name": "Statement"}, + "tags_detail": [{"name": "inbox"}], + "content": "CDE Lightband monthly utility statement. Amount due $142.17. Payment due by 2026-06-05." +} diff --git a/n8n-workflows/fixtures/paperless-intake-expected-triage.example.json b/n8n-workflows/fixtures/paperless-intake-expected-triage.example.json new file mode 100644 index 0000000..ca107b2 --- /dev/null +++ b/n8n-workflows/fixtures/paperless-intake-expected-triage.example.json @@ -0,0 +1,12 @@ +{ + "summary": "Monthly utility bill from CDE Lightband for $142.17 due on 2026-06-05.", + "suggested_title": "CDE Lightband Utility Bill - May 2026", + "document_type": "bill", + "suggested_correspondent": "CDE Lightband", + "suggested_tags": ["paperless-triage", "utility", "bill"], + "due_date": "2026-06-05", + "needs_review": true, + "urgency": "medium", + "reason": "Contains a payment due date; safety gate requires human review.", + "confidence": "high" +} diff --git a/n8n-workflows/fixtures/paperless-intake-webhook.example.json b/n8n-workflows/fixtures/paperless-intake-webhook.example.json new file mode 100644 index 0000000..543389c --- /dev/null +++ b/n8n-workflows/fixtures/paperless-intake-webhook.example.json @@ -0,0 +1,3 @@ +{ + "document_id": 12345 +}