Local homelab changes

n8n-workflows/09-class-transcript-ingest-v1.1.json

@@ -0,0 +1,423 @@
+{
+  "name": "Class Transcript Ingest v1.1",
+  "nodes": [
+    {
+      "parameters": {
+        "httpMethod": "POST",
+        "path": "class/ingest",
+        "responseMode": "responseNode",
+        "options": {}
+      },
+      "id": "a1b2c3d4-0001-0001-0001-000000000001",
+      "name": "Class Ingest Webhook",
+      "type": "n8n-nodes-base.webhook",
+      "typeVersion": 2,
+      "position": [
+        0,
+        0
+      ],
+      "webhookId": "class-ingest-rocinante-v1"
+    },
+    {
+      "parameters": {
+        "respondWith": "json",
+        "responseBody": "={ \"status\": \"accepted\" }",
+        "options": {}
+      },
+      "id": "a1b2c3d4-0001-0001-0001-000000000002",
+      "name": "Respond Accepted",
+      "type": "n8n-nodes-base.respondToWebhook",
+      "typeVersion": 1,
+      "position": [
+        224,
+        -200
+      ]
+    },
+    {
+      "parameters": {
+        "jsCode": "const body = $input.first().json.body || $input.first().json;\n\nconst className    = body.class_name    || 'unclassified';\nconst lectureTitle = body.lecture_title || 'Untitled Lecture';\nconst fullText     = body.transcript    || '';\nconst timestampedText = body.timestamped_transcript || fullText;\nconst durationMinutes = Math.round((body.duration_seconds || 0) / 60);\nconst classSlug = className.toLowerCase().replace(/[^a-z0-9]+/g, '-').replace(/^-|-$/g, '');\n\nreturn [{\n  json: {\n    className,\n    classSlug,\n    lectureTitle,\n    fullText,\n    timestampedText,\n    durationMinutes\n  }\n}];"
+      },
+      "id": "a1b2c3d4-0001-0001-0001-000000000003",
+      "name": "Extract Class Info",
+      "type": "n8n-nodes-base.code",
+      "typeVersion": 2,
+      "position": [
+        224,
+        0
+      ]
+    },
+    {
+      "parameters": {
+        "method": "POST",
+        "url": "http://10.5.1.6:4000/v1/chat/completions",
+        "sendHeaders": true,
+        "headerParameters": {
+          "parameters": [
+            {
+              "name": "Content-Type",
+              "value": "application/json"
+            },
+            {
+              "name": "Authorization",
+              "value": "=Bearer {{ $env.LITELLM_API_KEY }}"
+            }
+          ]
+        },
+        "sendBody": true,
+        "specifyBody": "json",
+        "jsonBody": "={\n  \"model\": \"medium\",\n  \"messages\": [\n    {\n      \"role\": \"system\",\n      \"content\": \"You are an academic note-taking assistant. Given a class lecture transcription, provide:\\n1. A concise summary (3-5 sentences)\\n2. Key concepts covered (as a list)\\n3. Important definitions or formulas\\n4. Any assignments or deadlines mentioned\\n\\nRespond in JSON with keys: summary, key_concepts (array), definitions (array of {term, definition}), assignments (array)\"\n    },\n    {\n      \"role\": \"user\",\n      \"content\": \"Class: {{ $json.className }}\\nLecture: {{ $json.lectureTitle }}\\n\\nTranscription (first 4000 chars):\\n{{ $json.fullText.substring(0, 4000) }}\"\n    }\n  ],\n  \"max_tokens\": 600,\n  \"temperature\": 0.2,\n  \"response_format\": { \"type\": \"json_object\" }\n}",
+        "options": {}
+      },
+      "id": "a1b2c3d4-0001-0001-0001-000000000004",
+      "name": "AI Extract Notes",
+      "type": "n8n-nodes-base.httpRequest",
+      "typeVersion": 4.2,
+      "position": [
+        448,
+        0
+      ]
+    },
+    {
+      "parameters": {
+        "jsCode": "const llmResponse   = $input.first().json;\nconst classInfo     = $('Extract Class Info').first().json;\n\nlet notes;\ntry {\n  notes = JSON.parse(llmResponse.choices?.[0]?.message?.content || '{}');\n} catch(e) {\n  notes = { summary: 'Could not parse notes', key_concepts: [], definitions: [], assignments: [] };\n}\n\nconst ragText = [\n  `Class: ${classInfo.className}`,\n  `Lecture: ${classInfo.lectureTitle}`,\n  `Duration: ${classInfo.durationMinutes} minutes`,\n  `Summary: ${notes.summary}`,\n  `Key Concepts: ${(notes.key_concepts || []).join(', ')}`,\n  '',\n  'Full Transcript:',\n  classInfo.fullText\n].join('\\n');\n\nreturn [{\n  json: {\n    className:       classInfo.className,\n    classSlug:       classInfo.classSlug,\n    lectureTitle:    classInfo.lectureTitle,\n    durationMinutes: classInfo.durationMinutes,\n    fullText:        classInfo.fullText,\n    notes,\n    ragText,\n    source: `class-${classInfo.classSlug}-${Date.now()}`,\n    metadata: {\n      class_name:       classInfo.className,\n      class_slug:       classInfo.classSlug,\n      lecture_title:    classInfo.lectureTitle,\n      duration_minutes: classInfo.durationMinutes,\n      type:             'class_recording',\n      recorded_at:      new Date().toISOString()\n    }\n  }\n}];"
+      },
+      "id": "a1b2c3d4-0001-0001-0001-000000000005",
+      "name": "Prepare for RAG",
+      "type": "n8n-nodes-base.code",
+      "typeVersion": 2,
+      "position": [
+        672,
+        0
+      ]
+    },
+    {
+      "parameters": {
+        "jsCode": "const input    = $input.first().json;\nconst text     = input.ragText || '';\nconst source   = input.source;\nconst metadata = input.metadata;\n\nconst CHUNK_SIZE = 400;\nconst MAX_CHUNK  = 600;\nconst OVERLAP    = 50;\n\nconst chunks = [];\nconst sentences = text.split(/(?<=[.!?])\\s+/);\nlet current = '';\n\nfor (const sentence of sentences) {\n  // If a single sentence exceeds MAX_CHUNK, split it by words first\n  const parts = [];\n  if (sentence.length > MAX_CHUNK) {\n    const words = sentence.split(' ');\n    let part = '';\n    for (const word of words) {\n      if ((part + ' ' + word).length > CHUNK_SIZE && part.length > 0) {\n        parts.push(part.trim());\n        part = word;\n      } else {\n        part = part ? part + ' ' + word : word;\n      }\n    }\n    if (part.trim()) parts.push(part.trim());\n  } else {\n    parts.push(sentence);\n  }\n\n  for (const part of parts) {\n    if ((current + ' ' + part).length > CHUNK_SIZE && current.length > 0) {\n      chunks.push(current.trim());\n      const words = current.split(' ');\n      current = words.slice(-OVERLAP).join(' ') + ' ' + part;\n    } else {\n      current = current ? current + ' ' + part : part;\n    }\n  }\n}\nif (current.trim()) chunks.push(current.trim());\n\nreturn chunks.map((chunk, i) => ({\n  json: { chunk, chunkIndex: i, totalChunks: chunks.length, source, metadata }\n}));"
+      },
+      "id": "a1b2c3d4-0001-0001-0001-000000000006",
+      "name": "Chunk Text",
+      "type": "n8n-nodes-base.code",
+      "typeVersion": 2,
+      "position": [
+        896,
+        0
+      ]
+    },
+    {
+      "parameters": {
+        "method": "POST",
+        "url": "http://10.5.1.6:11434/v1/embeddings",
+        "sendHeaders": true,
+        "headerParameters": {
+          "parameters": [
+            {
+              "name": "Content-Type",
+              "value": "application/json"
+            }
+          ]
+        },
+        "sendBody": true,
+        "specifyBody": "json",
+        "jsonBody": "={\n  \"model\": \"nomic-embed-text\",\n  \"input\": {{ JSON.stringify($json.chunk) }}\n}",
+        "options": {
+          "batching": {
+            "batch": {
+              "batchSize": 5,
+              "batchInterval": 500
+            }
+          }
+        }
+      },
+      "id": "a1b2c3d4-0001-0001-0001-000000000007",
+      "name": "Get Embedding",
+      "type": "n8n-nodes-base.httpRequest",
+      "typeVersion": 4.2,
+      "position": [
+        1120,
+        0
+      ]
+    },
+    {
+      "parameters": {
+        "jsCode": "const embeddingResponse = $input.first().json;\nconst chunkData         = $('Chunk Text').item;\nconst embedding         = embeddingResponse.data?.[0]?.embedding || [];\n\nconst pointId = 'xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx'.replace(/[xy]/g, c => {\n  const r = Math.random() * 16 | 0;\n  return (c === 'x' ? r : (r & 0x3 | 0x8)).toString(16);\n});\n\nreturn [{\n  json: {\n    id: pointId,\n    vector: embedding,\n    payload: {\n      text:        chunkData.json.chunk,\n      source:      chunkData.json.source,\n      chunkIndex:  chunkData.json.chunkIndex,\n      totalChunks: chunkData.json.totalChunks,\n      metadata:    chunkData.json.metadata,\n      ingested_at: new Date().toISOString()\n    }\n  }\n}];"
+      },
+      "id": "a1b2c3d4-0001-0001-0001-000000000008",
+      "name": "Prepare Qdrant Point",
+      "type": "n8n-nodes-base.code",
+      "typeVersion": 2,
+      "position": [
+        1344,
+        0
+      ]
+    },
+    {
+      "parameters": {
+        "method": "PUT",
+        "url": "http://10.5.1.6:6333/collections/knowledge_base/points",
+        "sendHeaders": true,
+        "headerParameters": {
+          "parameters": [
+            {
+              "name": "Content-Type",
+              "value": "application/json"
+            }
+          ]
+        },
+        "sendBody": true,
+        "specifyBody": "json",
+        "jsonBody": "={\n  \"points\": [\n    {\n      \"id\": \"{{ $json.id }}\",\n      \"vector\": {{ JSON.stringify($json.vector) }},\n      \"payload\": {{ JSON.stringify($json.payload) }}\n    }\n  ]\n}",
+        "options": {}
+      },
+      "id": "a1b2c3d4-0001-0001-0001-000000000009",
+      "name": "Upsert to Qdrant",
+      "type": "n8n-nodes-base.httpRequest",
+      "typeVersion": 4.2,
+      "position": [
+        1568,
+        0
+      ]
+    },
+    {
+      "parameters": {
+        "jsCode": "const rag = $('Prepare for RAG').first().json;\nreturn [{\n  json: {\n    title:    `\ud83c\udf93 ${rag.className} Lecture Processed`,\n    message:  `${rag.lectureTitle} (${rag.durationMinutes}min)\\nSummary: ${rag.notes.summary}\\nKey concepts: ${(rag.notes.key_concepts || []).slice(0, 5).join(', ')}`,\n    priority: 5\n  }\n}];"
+      },
+      "id": "a1b2c3d4-0001-0001-0001-000000000010",
+      "name": "Format Gotify Message",
+      "type": "n8n-nodes-base.code",
+      "typeVersion": 2,
+      "position": [
+        1552,
+        -200
+      ]
+    },
+    {
+      "parameters": {
+        "message": "={{ $json.message }}",
+        "additionalFields": {
+          "priority": "={{ $json.priority }}",
+          "title": "={{ $json.title }}"
+        },
+        "options": {
+          "contentType": "text/plain"
+        }
+      },
+      "id": "a1b2c3d4-0001-0001-0001-000000000011",
+      "name": "Notify via Gotify",
+      "type": "n8n-nodes-base.gotify",
+      "typeVersion": 1,
+      "position": [
+        1776,
+        -200
+      ],
+      "credentials": {
+        "gotifyApi": {
+          "id": "SETUP_REQUIRED",
+          "name": "Class Recordings"
+        }
+      }
+    },
+    {
+      "parameters": {
+        "jsCode": "const rag   = $('Prepare for RAG').first().json;\nconst notes = rag.notes || {};\nconst concepts = (notes.key_concepts || []).map((c,i) => `${i+1}. ${c}`).join('\\n');\nconst defList  = notes.definitions || [];\nconst defs     = Array.isArray(defList)\n  ? defList.map(d => `${d.term}: ${d.definition}`).join('\\n')\n  : Object.entries(defList).map(([k,v]) => `${k}: ${v}`).join('\\n');\nconst assignments = (notes.assignments || []).map((a,i) => `${i+1}. ${a}`).join('\\n');\n\nconst content = [\n  `Lecture Notes: ${rag.lectureTitle}`,\n  `Class: ${rag.className}`,\n  `Duration: ${rag.durationMinutes} minutes`,\n  `Date: ${new Date().toISOString()}`, '',\n  `Summary`, `=======`, notes.summary || '', '',\n  `Key Concepts`, `=============`, concepts,\n  ...(defs         ? ['', `Definitions`,  `===========`, defs]        : []),\n  ...(assignments  ? ['', `Assignments`,  `===========`, assignments] : []),\n  '', `Full Transcript`, `================`, rag.fullText || '',\n].join('\\n');\n\nconst title = `${rag.className} \u2014 ${rag.lectureTitle}`;\nreturn [{\n  binary: {\n    document: {\n      data:     Buffer.from(content, 'utf-8').toString('base64'),\n      mimeType: 'text/plain',\n      fileName: `${title}.txt`\n    }\n  },\n  json: { title, className: rag.className }\n}];"
+      },
+      "id": "a1b2c3d4-0001-0001-0001-000000000012",
+      "name": "Prepare Notes Doc",
+      "type": "n8n-nodes-base.code",
+      "typeVersion": 2,
+      "position": [
+        1776,
+        200
+      ]
+    },
+    {
+      "parameters": {
+        "method": "POST",
+        "url": "https://paperless.paccoco.com/api/documents/post_document/",
+        "sendHeaders": true,
+        "headerParameters": {
+          "parameters": [
+            {
+              "name": "Authorization",
+              "value": "=Token {{ $env.PAPERLESS_API_TOKEN }}"
+            }
+          ]
+        },
+        "sendBody": true,
+        "contentType": "multipart-form-data",
+        "bodyParameters": {
+          "parameters": [
+            {
+              "parameterType": "formBinaryData",
+              "name": "document",
+              "inputDataFieldName": "document"
+            },
+            {
+              "name": "title",
+              "value": "={{ $json.title }}"
+            },
+            {
+              "name": "tags[]",
+              "value": "={{ $env.PAPERLESS_TAG_LECTURE_NOTES }}"
+            }
+          ]
+        },
+        "options": {
+          "response": {
+            "response": {
+              "responseFormat": "text"
+            }
+          }
+        }
+      },
+      "id": "a1b2c3d4-0001-0001-0001-000000000013",
+      "name": "Save Notes to Paperless",
+      "type": "n8n-nodes-base.httpRequest",
+      "typeVersion": 4.2,
+      "position": [
+        2000,
+        200
+      ],
+      "continueOnFail": true
+    }
+  ],
+  "connections": {
+    "Class Ingest Webhook": {
+      "main": [
+        [
+          {
+            "node": "Extract Class Info",
+            "type": "main",
+            "index": 0
+          },
+          {
+            "node": "Respond Accepted",
+            "type": "main",
+            "index": 0
+          }
+        ]
+      ]
+    },
+    "Extract Class Info": {
+      "main": [
+        [
+          {
+            "node": "AI Extract Notes",
+            "type": "main",
+            "index": 0
+          }
+        ]
+      ]
+    },
+    "AI Extract Notes": {
+      "main": [
+        [
+          {
+            "node": "Prepare for RAG",
+            "type": "main",
+            "index": 0
+          }
+        ]
+      ]
+    },
+    "Prepare for RAG": {
+      "main": [
+        [
+          {
+            "node": "Chunk Text",
+            "type": "main",
+            "index": 0
+          }
+        ]
+      ]
+    },
+    "Chunk Text": {
+      "main": [
+        [
+          {
+            "node": "Get Embedding",
+            "type": "main",
+            "index": 0
+          }
+        ]
+      ]
+    },
+    "Get Embedding": {
+      "main": [
+        [
+          {
+            "node": "Prepare Qdrant Point",
+            "type": "main",
+            "index": 0
+          }
+        ]
+      ]
+    },
+    "Prepare Qdrant Point": {
+      "main": [
+        [
+          {
+            "node": "Upsert to Qdrant",
+            "type": "main",
+            "index": 0
+          }
+        ]
+      ]
+    },
+    "Upsert to Qdrant": {
+      "main": [
+        [
+          {
+            "node": "Format Gotify Message",
+            "type": "main",
+            "index": 0
+          },
+          {
+            "node": "Prepare Notes Doc",
+            "type": "main",
+            "index": 0
+          }
+        ]
+      ]
+    },
+    "Format Gotify Message": {
+      "main": [
+        [
+          {
+            "node": "Notify via Gotify",
+            "type": "main",
+            "index": 0
+          }
+        ]
+      ]
+    },
+    "Prepare Notes Doc": {
+      "main": [
+        [
+          {
+            "node": "Save Notes to Paperless",
+            "type": "main",
+            "index": 0
+          }
+        ]
+      ]
+    }
+  },
+  "settings": {
+    "executionOrder": "v1",
+    "binaryMode": "separate"
+  },
+  "staticData": null,
+  "meta": null,
+  "pinData": {},
+  "tags": [
+    {
+      "name": "school"
+    },
+    {
+      "name": "rag"
+    },
+    {
+      "name": "ai"
+    }
+  ]
+}