From 383cd41ae70387f921cacd53c03537984c81e863 Mon Sep 17 00:00:00 2001 From: Fizzlepoof Date: Fri, 15 May 2026 21:58:42 +0000 Subject: [PATCH] Add Doris Schoolhouse and clean pending homelab changes --- ai/litellm/config.yaml | 4 +- docs/planning/TODO.md | 15 +- .../doris-schoolhouse-technical-spec.md | 391 ++++++++++++++++++ home/doris-schoolhouse/.env.example | 23 ++ home/doris-schoolhouse/.gitignore | 6 + home/doris-schoolhouse/DEPLOYMENT.md | 68 +++ home/doris-schoolhouse/Dockerfile | 22 + home/doris-schoolhouse/README.md | 204 +++++++++ home/doris-schoolhouse/app/config.py | 24 ++ home/doris-schoolhouse/app/db/schema.sql | 92 +++++ home/doris-schoolhouse/app/main.py | 20 + home/doris-schoolhouse/app/models.py | 61 +++ home/doris-schoolhouse/app/routes/__init__.py | 0 .../app/routes/api_assignments.py | 148 +++++++ .../app/routes/api_dashboard.py | 31 ++ .../app/routes/api_paperless.py | 69 ++++ .../app/routes/api_recordings.py | 130 ++++++ home/doris-schoolhouse/app/routes/api_sync.py | 27 ++ home/doris-schoolhouse/app/routes/ui.py | 24 ++ .../app/scripts/apply_schema.py | 31 ++ .../app/scripts/bootstrap_state.py | 27 ++ .../app/scripts/process_recording.py | 44 ++ home/doris-schoolhouse/app/services/d2l.py | 139 +++++++ .../app/services/diagnostics.py | 23 ++ home/doris-schoolhouse/app/services/media.py | 57 +++ .../app/services/paperless.py | 109 +++++ .../app/services/paperless_client.py | 31 ++ .../app/services/repository.py | 388 +++++++++++++++++ home/doris-schoolhouse/app/services/store.py | 53 +++ .../app/services/webhooks.py | 52 +++ home/doris-schoolhouse/app/static/app.css | 60 +++ home/doris-schoolhouse/app/static/app.js | 127 ++++++ .../app/templates/assignment_upload.html | 23 ++ .../doris-schoolhouse/app/templates/base.html | 26 ++ .../app/templates/dashboard.html | 42 ++ .../app/templates/recording_upload.html | 19 + .../bin/apply-postgres-schema.sh | 24 ++ home/doris-schoolhouse/bin/bootstrap-local.sh | 8 + home/doris-schoolhouse/bin/check-env.sh | 22 + .../bin/process-recording.sh | 12 + home/doris-schoolhouse/bin/run-local.sh | 8 + .../bin/test-assignment-handoff.sh | 21 + .../bin/test-recording-handoff.sh | 19 + home/doris-schoolhouse/docker-compose.yaml | 30 ++ home/doris-schoolhouse/requirements.txt | 8 + .../08-class-recording-rag-v1.2.json | 10 +- n8n-workflows/08-class-recording-rag.json | 12 +- pihole/rpi4/.env.example | 5 + pihole/rpi4/README.md | 9 + pihole/rpi4/docker-compose.yaml | 38 ++ pihole/rpi4/keepalived.conf.example | 21 + pihole/rpi4/unbound.conf | 27 ++ pihole/serenity/.env.example | 12 + pihole/serenity/README.md | 15 + pihole/serenity/bin/prepare_serenity.sh | 31 ++ pihole/serenity/bin/render_keepalived_conf.py | 73 ++++ pihole/serenity/docker-compose.yaml | 59 +++ pihole/serenity/keepalived.conf | 32 ++ pihole/serenity/keepalived.conf.template | 32 ++ pihole/serenity/unbound.conf | 27 ++ 60 files changed, 3145 insertions(+), 20 deletions(-) create mode 100644 docs/planning/doris-schoolhouse-technical-spec.md create mode 100644 home/doris-schoolhouse/.env.example create mode 100644 home/doris-schoolhouse/.gitignore create mode 100644 home/doris-schoolhouse/DEPLOYMENT.md create mode 100644 home/doris-schoolhouse/Dockerfile create mode 100644 home/doris-schoolhouse/README.md create mode 100644 home/doris-schoolhouse/app/config.py create mode 100644 home/doris-schoolhouse/app/db/schema.sql create mode 100644 home/doris-schoolhouse/app/main.py create mode 100644 home/doris-schoolhouse/app/models.py create mode 100644 home/doris-schoolhouse/app/routes/__init__.py create mode 100644 home/doris-schoolhouse/app/routes/api_assignments.py create mode 100644 home/doris-schoolhouse/app/routes/api_dashboard.py create mode 100644 home/doris-schoolhouse/app/routes/api_paperless.py create mode 100644 home/doris-schoolhouse/app/routes/api_recordings.py create mode 100644 home/doris-schoolhouse/app/routes/api_sync.py create mode 100644 home/doris-schoolhouse/app/routes/ui.py create mode 100644 home/doris-schoolhouse/app/scripts/apply_schema.py create mode 100644 home/doris-schoolhouse/app/scripts/bootstrap_state.py create mode 100644 home/doris-schoolhouse/app/scripts/process_recording.py create mode 100644 home/doris-schoolhouse/app/services/d2l.py create mode 100644 home/doris-schoolhouse/app/services/diagnostics.py create mode 100644 home/doris-schoolhouse/app/services/media.py create mode 100644 home/doris-schoolhouse/app/services/paperless.py create mode 100644 home/doris-schoolhouse/app/services/paperless_client.py create mode 100644 home/doris-schoolhouse/app/services/repository.py create mode 100644 home/doris-schoolhouse/app/services/store.py create mode 100644 home/doris-schoolhouse/app/services/webhooks.py create mode 100644 home/doris-schoolhouse/app/static/app.css create mode 100644 home/doris-schoolhouse/app/static/app.js create mode 100644 home/doris-schoolhouse/app/templates/assignment_upload.html create mode 100644 home/doris-schoolhouse/app/templates/base.html create mode 100644 home/doris-schoolhouse/app/templates/dashboard.html create mode 100644 home/doris-schoolhouse/app/templates/recording_upload.html create mode 100755 home/doris-schoolhouse/bin/apply-postgres-schema.sh create mode 100755 home/doris-schoolhouse/bin/bootstrap-local.sh create mode 100755 home/doris-schoolhouse/bin/check-env.sh create mode 100755 home/doris-schoolhouse/bin/process-recording.sh create mode 100755 home/doris-schoolhouse/bin/run-local.sh create mode 100755 home/doris-schoolhouse/bin/test-assignment-handoff.sh create mode 100755 home/doris-schoolhouse/bin/test-recording-handoff.sh create mode 100644 home/doris-schoolhouse/docker-compose.yaml create mode 100644 home/doris-schoolhouse/requirements.txt create mode 100644 pihole/rpi4/.env.example create mode 100644 pihole/rpi4/README.md create mode 100644 pihole/rpi4/docker-compose.yaml create mode 100644 pihole/rpi4/keepalived.conf.example create mode 100644 pihole/rpi4/unbound.conf create mode 100644 pihole/serenity/.env.example create mode 100644 pihole/serenity/README.md create mode 100755 pihole/serenity/bin/prepare_serenity.sh create mode 100755 pihole/serenity/bin/render_keepalived_conf.py create mode 100644 pihole/serenity/docker-compose.yaml create mode 100644 pihole/serenity/keepalived.conf create mode 100644 pihole/serenity/keepalived.conf.template create mode 100644 pihole/serenity/unbound.conf diff --git a/ai/litellm/config.yaml b/ai/litellm/config.yaml index 7a90da6..8a03d77 100644 --- a/ai/litellm/config.yaml +++ b/ai/litellm/config.yaml @@ -30,7 +30,7 @@ model_list: - model_name: "medium" litellm_params: model: "ollama/qwen2.5:14b" - api_base: "http://host.docker.internal:11434" + api_base: "http://ollama:11434" timeout: 120 stream_timeout: 120 @@ -74,7 +74,7 @@ model_list: - model_name: "ollama/qwen2.5:14b" litellm_params: model: "ollama/qwen2.5:14b" - api_base: "http://host.docker.internal:11434" + api_base: "http://ollama:11434" - model_name: "ollama/deepseek-r1" litellm_params: diff --git a/docs/planning/TODO.md b/docs/planning/TODO.md index 96108eb..47b0827 100644 --- a/docs/planning/TODO.md +++ b/docs/planning/TODO.md @@ -5,9 +5,10 @@ - [x] Regenerate N.O.M.A.D. Newt secret (was exposed in chat) — completed 2026-05-15 ## Infrastructure -- [ ] Deploy Pi-hole 3-node HA cluster (PD + N.O.M.A.D. + RPi4) — see docs/planning/DEPLOY_PIHOLE.md - - PD primary stack scaffolded repo-side under `pihole/` on 2026-05-15 - - RPi4 IP still TBD — fill in before deploying +- [x] Deploy Pi-hole HA on PD + N.O.M.A.D. with VIP `10.5.1.53` — completed 2026-05-15; see docs/planning/DEPLOY_PIHOLE.md + - PD primary live at `/mnt/docker-ssd/docker/compose/pihole` + - NOMAD backup live at `/opt/pihole-nomad` + - RPi4 remains optional future BACKUP2 work if you want a third node later - [x] Deploy Kima-Hub — completed 2026-05-06; docs/planning/DEPLOY_KIMA_HUB.md now serves as deployment/repair reference - [ ] Set up off-site backup for vital data (appdata, databases, config repo, photos) - [ ] Remove dead Unraid-Cloudflared-Tunnel container from Serenity @@ -40,10 +41,10 @@ - Pelican/Wings-managed game containers stay and are documented as permanent game-server infrastructure on NOMAD. ## n8n Workflows -- [ ] Create Gotify apps and n8n credentials: Reminders, n8n Health, Uptime / Infra -- [ ] Add remaining env vars to n8n docker-compose + .env: `PAPERLESS_API_TOKEN`, `PAPERLESS_TAG_TRANSCRIPTION`, `PAPERLESS_TAG_LECTURE_NOTES`, `N8N_API_KEY` (see n8n-workflows/README.md for docker-compose.yaml changes required) -- [ ] Set up `PD SSH` credential in n8n for workflow 16 (see n8n-workflows/README.md) -- [ ] Add sudoers NOPASSWD rule for n8n watchdog (see n8n-workflows/README.md) +- [x] Create Gotify apps and n8n credentials: Reminders, n8n Health, Uptime / Infra — completed 2026-05-15 +- [x] Add remaining env vars to n8n docker-compose + .env: `PAPERLESS_API_TOKEN`, `PAPERLESS_TAG_TRANSCRIPTION`, `PAPERLESS_TAG_LECTURE_NOTES`, `N8N_API_KEY` — completed 2026-05-15 +- [x] Set up `PD SSH` credential in n8n for workflow 16 — completed 2026-05-15 +- [x] Add sudoers NOPASSWD rule for n8n watchdog — completed 2026-05-15 - [x] Import and activate upgraded workflows: 04, 08 (01 done — v1.4 active and tested) — completed 2026-05-15 - [x] Import and activate new workflows: 14, 15 (16 already active) — completed 2026-05-15 - [x] Configure class profiles + Paperless metadata mapping for current courses (ENGL-1010, HIST-2020, future classes) — completed 2026-05-15 diff --git a/docs/planning/doris-schoolhouse-technical-spec.md b/docs/planning/doris-schoolhouse-technical-spec.md new file mode 100644 index 0000000..4d464ce --- /dev/null +++ b/docs/planning/doris-schoolhouse-technical-spec.md @@ -0,0 +1,391 @@ +# Doris Schoolhouse — Technical Spec + +## Goal + +Build a desktop-first local web app on NOMAD for school intake and tracking. + +Primary jobs: + +- upload assignment turn-ins +- upload class recordings +- suggest metadata from D2L +- confirm metadata before Paperless submission +- transcribe and summarize recordings +- track assignment lifecycle through grading +- append grades and professor comments into Paperless notes + +## Deployment shape + +- **Source path:** `home/doris-schoolhouse` +- **Live NOMAD runtime target:** `/opt/doris-schoolhouse` +- **Access:** local + Pangolin-authenticated +- **Networks:** `pangolin`, `ix-databases_shared-databases` +- **DB:** shared Postgres +- **Archive/document system:** Paperless-NGX +- **Workflow helpers:** existing n8n workflows for school intake and class recording processing when appropriate + +## Source of truth + +| Domain | Source of truth | +|---|---| +| Course roster | D2L sync cache | +| Assignment roster / due dates | D2L sync cache with manual override support | +| Submission workflow state | Doris Schoolhouse DB | +| Uploaded files / transcripts / summaries | Paperless | +| Grades / professor comments | D2L on refresh, cached in Doris Schoolhouse DB | + +The app DB owns workflow state and linkage. Paperless owns archived artifacts. + +## Existing reusable assets + +### D2L + +Existing workspace assets already prove the login/scrape path: + +- `~/.openclaw/workspace/school/d2l_scraper_final.js` +- `~/.openclaw/workspace/school/d2l_school_watcher.js` +- `~/.openclaw/workspace/school/d2l_current_snapshot.json` +- `~/.openclaw/workspace/school/d2l_grades.json` + +Current confirmed D2L coverage: + +- courses +- grades +- grade items +- some material/date counts +- calendar/ICS-derived schedule + +Gap to close for Schoolhouse: + +- normalize D2L data into app DB tables +- stronger assignment/date extraction +- deterministic matching between D2L assignment rows and Paperless-backed submissions + +### Paperless / n8n + +Existing repo assets: + +- `n8n-workflows/08-class-recording-rag-v1.2.json` +- `n8n-workflows/18-school-paperless-intake-v1.2.json` +- `n8n-workflows/19-paperless-school-metadata-enrichment-v1.1.json` +- `docs/operations/SCHOOL_PAPERLESS_INTAKE.md` +- `docs/reference/SCHOOL_INTAKE_POSTGRES_SCHEMA.sql` + +Current confirmed behavior from docs/testing notes: + +- school uploads can already be pushed to Paperless with deterministic filename/title correlation +- class recording pipeline already has a webhook path and downstream Paperless/RAG behavior +- workflow 19 already does deterministic post-consumption enrichment + +## App architecture + +Use a server-rendered Python app for the first build: + +- **Backend:** FastAPI +- **Templates:** Jinja2 +- **Frontend:** vanilla JS + server-rendered forms/pages +- **DB access:** SQLAlchemy / psycopg +- **Background work:** app jobs can hand off long-running audio processing to existing n8n recording/transcription pipeline + +Why this split: + +- easier large-upload handling +- easy form-heavy UI +- less frontend build nonsense +- plays nicely with existing Python-heavy repo bits + +## Core pages + +### 1. Dashboard `/` + +Shows: + +- upcoming assignments +- unsubmitted assignments +- recent uploads pending confirmation +- newly graded work +- recording processing status +- sync health + +### 2. Assignment Upload `/assignments/upload` + +Form fields: + +- class +- assignment +- submitted date +- version +- notes +- file upload / drag-drop / paste + +Behavior: + +1. choose file +2. app suggests class + assignment from D2L cache +3. user confirms/edits metadata +4. upload goes to Paperless intake path +5. app stores linkage row +6. assignment state becomes `submitted` + +### 3. Recording Upload `/recordings/upload` + +Form fields: + +- class +- session date +- optional related assignment +- notes +- `.wav` upload + +Behavior: + +1. accept large WAV +2. persist upload job row +3. convert WAV → MP3 +4. delete WAV only after MP3 success +5. hand MP3 to transcription/summarization flow +6. send transcript + summary to Paperless +7. store resulting Paperless document links + +### 4. Classes `/classes` and `/classes/{id}` + +Shows: + +- assignments +- due dates +- grades +- comments +- linked submissions +- linked recordings + +### 5. D2L Sync `/admin/d2l` + +Shows: + +- last sync time +- sync status +- changed courses/assignments/grades +- unmatched rows requiring manual mapping + +## API surface + +### Health +- `GET /api/health` + +### D2L +- `POST /api/d2l/sync` +- `GET /api/d2l/courses` +- `GET /api/d2l/assignments?course_id=` +- `GET /api/d2l/grades?course_id=` + +### Assignments +- `GET /api/assignments` +- `POST /api/assignments/intake` +- `POST /api/assignments/{id}/confirm-paperless` +- `POST /api/assignments/{id}/status` + +### Recordings +- `GET /api/recordings` +- `POST /api/recordings/intake` +- `POST /api/recordings/{id}/reprocess` + +### Paperless linkage +- `POST /api/paperless/webhook/intake-enriched` +- `POST /api/paperless/webhook/recording-complete` +- `POST /api/paperless/assignments/{id}/append-grade-note` + +## Data model + +### `schoolhouse_course` + +- id +- d2l_course_id +- name +- code +- semester +- instructor_name +- active +- first_seen_at +- last_synced_at + +### `schoolhouse_assignment` + +- id +- course_id +- d2l_assignment_key +- title +- category +- due_at +- status (`assigned|in_progress|submitted|graded|late|missing`) +- grade_text +- grade_numeric +- professor_comments +- source (`d2l|manual|template`) +- created_at +- updated_at + +### `schoolhouse_submission` + +- id +- assignment_id +- version_label +- original_filename +- mime_type +- checksum_sha256 +- submitted_at +- suggested_metadata_json +- confirmed_metadata_json +- paperless_document_id +- paperless_title +- paperless_note_appended_at +- status (`draft|uploaded|confirmed|failed`) + +### `schoolhouse_recording` + +- id +- course_id +- assignment_id nullable +- session_date +- original_filename +- mp3_filename +- duration_seconds +- transcription_status (`queued|processing|ready|failed`) +- summary_status (`queued|processing|ready|failed`) +- transcript_paperless_document_id +- summary_paperless_document_id +- recording_paperless_document_id nullable +- notes + +### `schoolhouse_d2l_sync_run` + +- id +- started_at +- completed_at +- status +- raw_snapshot_path +- changes_summary_json +- error_text + +### `schoolhouse_assignment_mapping` + +- id +- course_id +- d2l_grade_item_name +- normalized_key +- assignment_id +- confidence +- mapping_source (`auto|manual`) + +This table is the bastard that keeps D2L grade items tied to internal assignments cleanly. + +## Matching rules + +### Assignment matching priority + +1. exact D2L assignment key when available +2. normalized title within course +3. filename heuristics +4. latest due date proximity +5. manual confirmation + +### Grade/comment update policy + +- D2L refresh updates cached grade/comment fields for mapped assignments +- if a submission has a Paperless document ID and grade/comment changed, append note once per distinct change hash +- never overwrite user notes in Paperless; append a dated Schoolhouse block + +## Recording pipeline contract + +### Input limits + +- accept WAV up to at least 1 GB +- support up to 90 minutes +- reject unsupported formats with clear message + +### Processing steps + +1. store upload metadata row +2. convert WAV → MP3 +3. delete WAV only after MP3 success +4. submit MP3 to recording workflow +5. receive transcript/summary callback +6. archive transcript + summary into Paperless + +## Secrets/config + +Required env: + +- `SCHOOLHOUSE_HOST` +- `SCHOOLHOUSE_PORT` +- `DATABASE_URL` +- `PAPERLESS_BASE_URL` +- `PAPERLESS_API_TOKEN` +- `N8N_BASE_URL` +- `SCHOOLHOUSE_N8N_ASSIGNMENT_WEBHOOK` +- `SCHOOLHOUSE_N8N_RECORDING_WEBHOOK` +- `D2L_SCRAPER_WORKDIR` +- `D2L_SCRAPER_COMMAND` +- `UPLOAD_TMP_DIR` +- `MAX_UPLOAD_MB` +- `SESSION_SECRET` + +Do not commit live values. Use `.env.example` only. + +## Repo / deployment plan + +Repo source path: + +- `home/doris-schoolhouse` + +Expected live NOMAD runtime path later: + +- `/opt/doris-schoolhouse` + +Expected live data path later: + +- `/opt/doris-schoolhouse/data` + +Expected compose validation before deploy: + +```bash +cd /opt/doris-schoolhouse +docker compose --env-file .env config +``` + +## Build phases + +### Phase 1 + +- app scaffold +- schema +- D2L import command/service +- assignment upload UI +- Paperless confirmation flow + +### Phase 2 + +- recording upload flow +- MP3 conversion +- recording job tracking +- transcript/summary callbacks + +### Phase 3 + +- grade sync +- Paperless note append logic +- dashboard views + +### Phase 4 + +- manual mappings UI +- recurring assignment templates +- mobile cleanup + +## Verification checklist + +- Python modules compile +- schema file present and readable +- compose file parses +- route map loads without import failure +- README explains runtime/deploy expectations +- no secrets committed diff --git a/home/doris-schoolhouse/.env.example b/home/doris-schoolhouse/.env.example new file mode 100644 index 0000000..ea86333 --- /dev/null +++ b/home/doris-schoolhouse/.env.example @@ -0,0 +1,23 @@ +TZ=America/Chicago + +SCHOOLHOUSE_HOST=0.0.0.0 +SCHOOLHOUSE_PORT=8091 +SCHOOLHOUSE_BASE_URL=https://schoolhouse.example.com +SESSION_SECRET=changeme + +DATABASE_URL=postgresql+psycopg://schoolhouse:changeme@shared-postgres:5432/schoolhouse + +PAPERLESS_BASE_URL=https://paperless.example.com +PAPERLESS_API_TOKEN=changeme + +N8N_BASE_URL=https://n8n.example.com +SCHOOLHOUSE_N8N_ASSIGNMENT_WEBHOOK=/webhook/school/intake/upload +SCHOOLHOUSE_N8N_RECORDING_WEBHOOK=/webhook/class/upload + +D2L_SCRAPER_WORKDIR=/workspace/.openclaw-workspace/school +D2L_SCRAPER_COMMAND=node d2l_scraper_final.js + +UPLOAD_TMP_DIR=/data/uploads +SCHOOLHOUSE_STATE_DIR=/data/state +SCHOOLHOUSE_STORAGE_BACKEND=auto +MAX_UPLOAD_MB=1536 diff --git a/home/doris-schoolhouse/.gitignore b/home/doris-schoolhouse/.gitignore new file mode 100644 index 0000000..ae7761d --- /dev/null +++ b/home/doris-schoolhouse/.gitignore @@ -0,0 +1,6 @@ +.env +__pycache__/ +*.pyc +/data/ +/uploads/ +.venv/ diff --git a/home/doris-schoolhouse/DEPLOYMENT.md b/home/doris-schoolhouse/DEPLOYMENT.md new file mode 100644 index 0000000..f23034f --- /dev/null +++ b/home/doris-schoolhouse/DEPLOYMENT.md @@ -0,0 +1,68 @@ +# Doris Schoolhouse Deployment Notes + +## Intended runtime shape on NOMAD + +Doris Schoolhouse should run as a standalone NOMAD service from **`/opt/doris-schoolhouse`**. + +- **Repo/source path:** `/home/fizzlepoof/repos/truenas-stacks/home/doris-schoolhouse` +- **Live runtime path:** `/opt/doris-schoolhouse` +- **Persistent app data:** `/opt/doris-schoolhouse/data` +- **Published app port:** `8091` +- **Workspace mount for D2L snapshot input:** `/home/fizzlepoof/.openclaw/workspace:/workspace/.openclaw-workspace:ro` + +## Important NOMAD networking note + +NOMAD does **not** currently have PD's Docker-only external networks (`pangolin`, `ix-databases_shared-databases`). + +For NOMAD, Schoolhouse should: +- publish `8091` on the host +- let Pangolin/Newt target the host port externally +- reach shared Postgres on PD over LAN TCP (`10.5.1.6:5432`) +- reach LiteLLM / Qdrant / Paperless / n8n over their existing LAN/public endpoints + +## Pre-deploy checklist + +1. Copy the repo source into `/opt/doris-schoolhouse`. +2. Create a real `.env` from `.env.nomad.example`. +3. Replace placeholder values for: + - `SESSION_SECRET` + - `DATABASE_URL` password + - `PAPERLESS_API_TOKEN` +4. Ensure the shared Postgres target database/user exist on PD. +5. Apply `app/db/schema.sql` against the `schoolhouse` database. +6. Ensure `/opt/doris-schoolhouse/data` exists and is writable by Docker. +7. Confirm Pangolin/Newt has a resource forwarding `schoolhouse.paccoco.com` to NOMAD host port `8091` if public access is wanted immediately. + +## Current live integration status + +### Working +- Assignment intake webhook (`/webhook/school/intake/upload`) succeeded in live smoke testing on 2026-05-15. +- Recording workflow (`/webhook/class/upload`) was repaired live on 2026-05-15 and now returns HTTP 200. +- Direct Whisper transcription endpoint on NOMAD succeeded in live smoke testing. +- LiteLLM `medium` on PD succeeded after its Ollama route was fixed. + +## Suggested deploy flow + +```bash +sudo mkdir -p /opt/doris-schoolhouse/data +sudo rsync -a --delete --exclude '.env' --exclude 'data/' /home/fizzlepoof/repos/truenas-stacks/home/doris-schoolhouse/ /opt/doris-schoolhouse/ +cd /opt/doris-schoolhouse +sudo cp .env.nomad.example .env +# edit .env +sudo docker compose --env-file .env config +sudo docker compose --env-file .env up -d --build +``` + +## Post-deploy verification + +```bash +curl -fsS http://127.0.0.1:8091/api/health +curl -fsS http://127.0.0.1:8091/api/health/details +curl -fsS http://127.0.0.1:8091/api/dashboard/summary +``` + +Then verify: +- D2L snapshot counts load +- assignment intake confirms into n8n/Paperless +- recording conversion works inside container +- recording downstream workflow returns success diff --git a/home/doris-schoolhouse/Dockerfile b/home/doris-schoolhouse/Dockerfile new file mode 100644 index 0000000..dd59b2d --- /dev/null +++ b/home/doris-schoolhouse/Dockerfile @@ -0,0 +1,22 @@ +FROM python:3.12-slim + +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 + +RUN apt-get update \ + && apt-get install -y --no-install-recommends ffmpeg curl \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /app + +COPY requirements.txt ./ +RUN pip install --no-cache-dir -r requirements.txt + +COPY app ./app +COPY .env.example ./ +COPY README.md ./ +COPY bin ./bin + +EXPOSE 8091 + +CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8091"] diff --git a/home/doris-schoolhouse/README.md b/home/doris-schoolhouse/README.md new file mode 100644 index 0000000..c753320 --- /dev/null +++ b/home/doris-schoolhouse/README.md @@ -0,0 +1,204 @@ +# Doris Schoolhouse + +Desktop-first local web app for school intake, recordings, D2L sync, and Paperless linkage. + +## Purpose + +This app gives John a fast local UI to: + +- upload assignment turn-ins +- upload class recordings +- confirm suggested metadata +- track assignment status through grading +- sync classes/assignments/grades/comments from D2L +- archive submissions/transcripts/summaries in Paperless + +## Source vs runtime + +- **Repo source:** `home/doris-schoolhouse` +- **Expected live NOMAD runtime:** `/opt/doris-schoolhouse` + +Do not run production from an agent workspace. + +## Stack + +- FastAPI +- Jinja2 templates +- shared Postgres +- Pangolin network +- existing Paperless and n8n school workflows +- container build now includes Python deps + `ffmpeg` for recording conversion + +## Current state + +This is an implementation scaffold with: + +## Current local persistence + +Until the full shared-Postgres integration is wired, the scaffold uses JSON state files for fast local iteration: + +- default state dir: `/data/state` +- default upload temp dir: `/data/uploads` + +For local host-side testing, override with env vars such as: + +```bash +export SCHOOLHOUSE_STATE_DIR=/tmp/doris-schoolhouse-state +export UPLOAD_TMP_DIR=/tmp/doris-schoolhouse-uploads +``` + +Production target remains shared Postgres plus Paperless/n8n integration. + +`SCHOOLHOUSE_STORAGE_BACKEND=auto` prefers Postgres when psycopg and a PostgreSQL DSN are available, and falls back to JSON state for local iteration. + + +- route stubs +- templates +- schema draft +- compose/env examples +- D2L / Paperless integration placeholders + +It is not deployed yet. + +## Layout + +```text +home/doris-schoolhouse/ +├── .env.example +├── docker-compose.yaml +├── requirements.txt +└── app/ + ├── main.py + ├── config.py + ├── models.py + ├── db/schema.sql + ├── routes/ + ├── services/ + ├── static/ + └── templates/ +``` + + + + +## Environment and integration helpers + +Useful helpers now included: + +```bash +bin/check-env.sh +bin/apply-postgres-schema.sh +bin/test-assignment-handoff.sh [assignment_name] +bin/test-recording-handoff.sh +``` + +These are safe prep helpers. The handoff scripts are dry-run previews; they print the exact curl commands rather than firing them automatically. + +## Helper scripts + +Local iteration helpers: + +```bash +bin/bootstrap-local.sh +bin/run-local.sh +bin/process-recording.sh +``` + +These default to the JSON backend and `/tmp` paths unless you override the env vars. + +## Bootstrap local state quickly + +For local iteration without live Postgres: + +```bash +cd /home/fizzlepoof/repos/truenas-stacks/home/doris-schoolhouse +export SCHOOLHOUSE_STORAGE_BACKEND=json +export SCHOOLHOUSE_STATE_DIR=/tmp/doris-schoolhouse-state +export UPLOAD_TMP_DIR=/tmp/doris-schoolhouse-uploads +PYTHONPATH=. python3 app/scripts/bootstrap_state.py +``` + +That seeds local state from the latest D2L snapshot. + + +## Recording worker helper + +There is now a local worker entrypoint for WAV → MP3 processing: + +```bash +cd /home/fizzlepoof/repos/truenas-stacks/home/doris-schoolhouse +export SCHOOLHOUSE_STORAGE_BACKEND=json +export SCHOOLHOUSE_STATE_DIR=/tmp/doris-schoolhouse-state +export UPLOAD_TMP_DIR=/tmp/doris-schoolhouse-uploads +PYTHONPATH=. python3 app/scripts/process_recording.py +``` + +If `ffmpeg` is not installed, the worker records a clean `blocked` result with the exact conversion command it wanted to run. + +## Local validation + +```bash +cd /home/fizzlepoof/repos/truenas-stacks/home/doris-schoolhouse +python3 -m py_compile app/main.py app/config.py app/models.py app/routes/*.py app/services/*.py +python3 - <<'PY' +import yaml +from pathlib import Path +print(yaml.safe_load(Path('docker-compose.yaml').read_text())['services'].keys()) +PY +``` + + +## Container build + +The source now includes a `Dockerfile` so the app can run with its own dependencies instead of relying on the host. + +That image includes: +- Python dependencies from `requirements.txt` +- `ffmpeg` +- `curl` for healthchecks + +So even if the host is missing `ffmpeg`, the containerized runtime can still do WAV → MP3 conversion. + +## Future deploy shape + +Expected live deploy flow on NOMAD: + +```bash +sudo mkdir -p /opt/doris-schoolhouse/data +sudo rsync -a --delete --exclude '.env' --exclude 'data/' /home/fizzlepoof/repos/truenas-stacks/home/doris-schoolhouse/ /opt/doris-schoolhouse/ +cd /opt/doris-schoolhouse +sudo cp .env.nomad.example .env +docker compose --env-file .env config +docker compose --env-file .env up -d --build +``` + +Not doing that yet. No live deployment should happen without explicit rollout work. + + +## Live integration notes + +Current verified live behavior: +- `https://n8n.paccoco.com/webhook/school/intake/upload` accepts Doris Schoolhouse multipart assignment uploads and returned a successful live smoke-test response on 2026-05-15. +- `https://n8n.paccoco.com/webhook/class/upload` now succeeds in live smoke testing after upstream workflow and LiteLLM fixes on 2026-05-15. +- Direct Whisper testing against `http://10.5.1.16:8786/v1/audio/transcriptions` succeeded with a tiny MP3. +- Direct LiteLLM testing against `http://10.5.1.6:4000/v1/chat/completions` now succeeds for model `medium` after the PD route fix. + +The app now captures webhook HTTP failures as structured responses instead of crashing blindly. + + + +## NOMAD-specific deployment notes + +On NOMAD, Schoolhouse should run as a standalone service from `/opt/doris-schoolhouse`. + +Unlike PD stacks, this host does **not** currently use the PD Docker external networks for Pangolin or shared databases. The practical deploy model here is: +- publish host port `8091` +- point Pangolin/Newt at that host port +- reach shared Postgres on PD over `10.5.1.6:5432` + +Use `.env.nomad.example` as the starting point for the live runtime. + +## Deployment references + +- `DEPLOYMENT.md` — clean NOMAD rollout notes +- `.env.nomad.example` — compose-oriented runtime example for NOMAD diff --git a/home/doris-schoolhouse/app/config.py b/home/doris-schoolhouse/app/config.py new file mode 100644 index 0000000..96d1abf --- /dev/null +++ b/home/doris-schoolhouse/app/config.py @@ -0,0 +1,24 @@ +from dataclasses import dataclass +import os + + +@dataclass +class Settings: + schoolhouse_host: str = os.getenv("SCHOOLHOUSE_HOST", "0.0.0.0") + schoolhouse_port: int = int(os.getenv("SCHOOLHOUSE_PORT", "8091")) + schoolhouse_base_url: str = os.getenv("SCHOOLHOUSE_BASE_URL", "http://localhost:8091") + database_url: str = os.getenv("DATABASE_URL", "postgresql+psycopg://schoolhouse:changeme@shared-postgres:5432/schoolhouse") + paperless_base_url: str = os.getenv("PAPERLESS_BASE_URL", "https://paperless.example.com") + paperless_api_token: str = os.getenv("PAPERLESS_API_TOKEN", "changeme") + n8n_base_url: str = os.getenv("N8N_BASE_URL", "https://n8n.example.com") + n8n_assignment_webhook: str = os.getenv("SCHOOLHOUSE_N8N_ASSIGNMENT_WEBHOOK", "/webhook/school/intake/upload") + n8n_recording_webhook: str = os.getenv("SCHOOLHOUSE_N8N_RECORDING_WEBHOOK", "/webhook/class/upload") + d2l_scraper_workdir: str = os.getenv("D2L_SCRAPER_WORKDIR", "/workspace/.openclaw-workspace/school") + d2l_scraper_command: str = os.getenv("D2L_SCRAPER_COMMAND", "node d2l_scraper_final.js") + upload_tmp_dir: str = os.getenv("UPLOAD_TMP_DIR", "/data/uploads") + state_dir: str = os.getenv("SCHOOLHOUSE_STATE_DIR", "/data/state") + storage_backend: str = os.getenv("SCHOOLHOUSE_STORAGE_BACKEND", "auto") + max_upload_mb: int = int(os.getenv("MAX_UPLOAD_MB", "1536")) + + +settings = Settings() diff --git a/home/doris-schoolhouse/app/db/schema.sql b/home/doris-schoolhouse/app/db/schema.sql new file mode 100644 index 0000000..62934d8 --- /dev/null +++ b/home/doris-schoolhouse/app/db/schema.sql @@ -0,0 +1,92 @@ +CREATE TABLE IF NOT EXISTS schoolhouse_course ( + id BIGSERIAL PRIMARY KEY, + d2l_course_id TEXT NOT NULL UNIQUE, + name TEXT NOT NULL, + code TEXT, + semester TEXT, + instructor_name TEXT, + active BOOLEAN NOT NULL DEFAULT TRUE, + first_seen_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + last_synced_at TIMESTAMPTZ +); + +CREATE TABLE IF NOT EXISTS schoolhouse_assignment ( + id BIGSERIAL PRIMARY KEY, + course_id BIGINT NOT NULL REFERENCES schoolhouse_course(id) ON DELETE CASCADE, + d2l_assignment_key TEXT, + title TEXT NOT NULL, + category TEXT, + due_at TIMESTAMPTZ, + status TEXT NOT NULL DEFAULT 'assigned', + grade_text TEXT, + grade_numeric DOUBLE PRECISION, + professor_comments TEXT, + source TEXT NOT NULL DEFAULT 'd2l', + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +CREATE TABLE IF NOT EXISTS schoolhouse_submission ( + id BIGSERIAL PRIMARY KEY, + assignment_id BIGINT NOT NULL REFERENCES schoolhouse_assignment(id) ON DELETE CASCADE, + version_label TEXT NOT NULL DEFAULT 'v1', + original_filename TEXT NOT NULL, + mime_type TEXT NOT NULL, + checksum_sha256 TEXT, + submitted_at TIMESTAMPTZ, + suggested_metadata_json JSONB NOT NULL DEFAULT '{}'::jsonb, + confirmed_metadata_json JSONB NOT NULL DEFAULT '{}'::jsonb, + paperless_document_id BIGINT, + paperless_title TEXT, + paperless_note_last_hash TEXT, + paperless_note_appended_at TIMESTAMPTZ, + status TEXT NOT NULL DEFAULT 'draft', + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +CREATE TABLE IF NOT EXISTS schoolhouse_recording ( + id BIGSERIAL PRIMARY KEY, + course_id BIGINT NOT NULL REFERENCES schoolhouse_course(id) ON DELETE CASCADE, + assignment_id BIGINT REFERENCES schoolhouse_assignment(id) ON DELETE SET NULL, + session_date DATE NOT NULL, + original_filename TEXT NOT NULL, + mp3_filename TEXT, + duration_seconds INTEGER, + transcription_status TEXT NOT NULL DEFAULT 'queued', + summary_status TEXT NOT NULL DEFAULT 'queued', + transcript_paperless_document_id BIGINT, + summary_paperless_document_id BIGINT, + recording_paperless_document_id BIGINT, + notes TEXT, + recording_metadata_json JSONB NOT NULL DEFAULT '{}'::jsonb, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +CREATE TABLE IF NOT EXISTS schoolhouse_d2l_sync_run ( + id BIGSERIAL PRIMARY KEY, + started_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + completed_at TIMESTAMPTZ, + status TEXT NOT NULL DEFAULT 'running', + raw_snapshot_path TEXT, + changes_summary_json JSONB NOT NULL DEFAULT '{}'::jsonb, + error_text TEXT +); + +CREATE TABLE IF NOT EXISTS schoolhouse_assignment_mapping ( + id BIGSERIAL PRIMARY KEY, + course_id BIGINT NOT NULL REFERENCES schoolhouse_course(id) ON DELETE CASCADE, + d2l_grade_item_name TEXT NOT NULL, + normalized_key TEXT NOT NULL, + assignment_id BIGINT NOT NULL REFERENCES schoolhouse_assignment(id) ON DELETE CASCADE, + confidence DOUBLE PRECISION, + mapping_source TEXT NOT NULL DEFAULT 'auto', + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + UNIQUE (course_id, d2l_grade_item_name) +); + +CREATE INDEX IF NOT EXISTS idx_schoolhouse_assignment_course_id ON schoolhouse_assignment(course_id); +CREATE INDEX IF NOT EXISTS idx_schoolhouse_submission_assignment_id ON schoolhouse_submission(assignment_id); +CREATE INDEX IF NOT EXISTS idx_schoolhouse_recording_course_id ON schoolhouse_recording(course_id); +CREATE INDEX IF NOT EXISTS idx_schoolhouse_recording_assignment_id ON schoolhouse_recording(assignment_id); diff --git a/home/doris-schoolhouse/app/main.py b/home/doris-schoolhouse/app/main.py new file mode 100644 index 0000000..29505b4 --- /dev/null +++ b/home/doris-schoolhouse/app/main.py @@ -0,0 +1,20 @@ +from fastapi import FastAPI +from fastapi.staticfiles import StaticFiles + +from app.routes.api_assignments import router as assignments_router +from app.routes.api_dashboard import router as dashboard_router +from app.routes.api_paperless import router as paperless_router +from app.routes.api_recordings import router as recordings_router +from app.routes.api_sync import router as sync_router +from app.routes.ui import router as ui_router + + +app = FastAPI(title="Doris Schoolhouse") +app.mount("/static", StaticFiles(directory="app/static"), name="static") + +app.include_router(ui_router) +app.include_router(dashboard_router) +app.include_router(assignments_router) +app.include_router(recordings_router) +app.include_router(sync_router) +app.include_router(paperless_router) diff --git a/home/doris-schoolhouse/app/models.py b/home/doris-schoolhouse/app/models.py new file mode 100644 index 0000000..ed389bd --- /dev/null +++ b/home/doris-schoolhouse/app/models.py @@ -0,0 +1,61 @@ +from dataclasses import dataclass, field +from datetime import date, datetime +from typing import Literal, Optional + + +AssignmentStatus = Literal["assigned", "in_progress", "submitted", "graded", "late", "missing"] +UploadStatus = Literal["draft", "uploaded", "confirmed", "failed"] +ProcessingStatus = Literal["queued", "processing", "ready", "failed"] + + +@dataclass +class Course: + d2l_course_id: str + name: str + id: Optional[int] = None + code: Optional[str] = None + semester: Optional[str] = None + active: bool = True + + +@dataclass +class Assignment: + course_id: int + title: str + id: Optional[int] = None + d2l_assignment_key: Optional[str] = None + category: Optional[str] = None + due_at: Optional[datetime] = None + status: AssignmentStatus = "assigned" + grade_text: Optional[str] = None + grade_numeric: Optional[float] = None + professor_comments: Optional[str] = None + source: Literal["d2l", "manual", "template"] = "d2l" + + +@dataclass +class Submission: + assignment_id: int + original_filename: str + mime_type: str + id: Optional[int] = None + version_label: str = "v1" + submitted_at: datetime = field(default_factory=datetime.utcnow) + paperless_document_id: Optional[int] = None + paperless_title: Optional[str] = None + status: UploadStatus = "draft" + + +@dataclass +class Recording: + course_id: int + session_date: date + original_filename: str + id: Optional[int] = None + assignment_id: Optional[int] = None + mp3_filename: Optional[str] = None + duration_seconds: Optional[int] = None + transcription_status: ProcessingStatus = "queued" + summary_status: ProcessingStatus = "queued" + transcript_paperless_document_id: Optional[int] = None + summary_paperless_document_id: Optional[int] = None diff --git a/home/doris-schoolhouse/app/routes/__init__.py b/home/doris-schoolhouse/app/routes/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/home/doris-schoolhouse/app/routes/api_assignments.py b/home/doris-schoolhouse/app/routes/api_assignments.py new file mode 100644 index 0000000..24c0966 --- /dev/null +++ b/home/doris-schoolhouse/app/routes/api_assignments.py @@ -0,0 +1,148 @@ +import hashlib +from pathlib import Path + +from fastapi import APIRouter, Form, HTTPException, UploadFile + +from app.config import settings +from app.services.d2l import D2LService +from app.services.paperless import PaperlessService +from app.services.repository import get_repository +from app.services.store import JsonStore +from app.services.webhooks import WebhookService + + +router = APIRouter(prefix="/api/assignments", tags=["assignments"]) +store = JsonStore() +repo = get_repository() +d2l = D2LService() +paperless = PaperlessService() +webhooks = WebhookService() + + +@router.get("") +async def list_assignments(): + courses = {item["id"]: item for item in d2l.get_courses()} + items = repo.list_assignments() + submissions = repo.list_submissions() + submission_counts = {} + for submission in submissions: + submission_counts[submission.get("assignment_id")] = submission_counts.get(submission.get("assignment_id"), 0) + 1 + for item in items: + item["submission_count"] = submission_counts.get(item.get("id"), 0) + item["course"] = courses.get(item.get("course_id"), {}) + return {"items": items, "backend": repo.backend} + + + + +@router.get('/submissions') +async def list_submissions(): + return {"items": repo.list_submissions(), "backend": repo.backend} + +@router.post("/intake") +async def intake_assignment( + file: UploadFile, + class_id: str = Form(...), + class_name: str = Form(...), + assignment_name: str = Form(...), + version: str = Form("v1"), + submission_kind: str = Form("assignment_turn_in"), + semester: str = Form(""), + course_code: str = Form(""), + paper_kind: str = Form(""), + source: str = Form("doris-schoolhouse"), + telegram_chat_id: str = Form(""), + telegram_message_id: str = Form(""), + notes: str = Form(""), +): + assignments = d2l.get_assignments(class_id) + matching = next((item for item in assignments if item.get("title") == assignment_name), None) + if not matching: + raise HTTPException(status_code=404, detail="Assignment not found in cached D2L data for selected class.") + + upload_root = Path(settings.upload_tmp_dir) + upload_root.mkdir(parents=True, exist_ok=True) + safe_name = Path(file.filename or "upload.bin").name + submission_id = store.new_id("submission") + stored_path = upload_root / f"{submission_id}-{safe_name}" + payload = await file.read() + stored_path.write_bytes(payload) + checksum = hashlib.sha256(payload).hexdigest() + + record = { + "id": submission_id, + "assignment_id": matching["id"], + "class_id": class_id, + "class_name": class_name, + "assignment_name": assignment_name, + "version_label": version, + "submission_kind": submission_kind, + "semester": semester or None, + "course_code": course_code or (course.get("code") if course else None), + "paper_kind": paper_kind or None, + "source": source or "doris-schoolhouse", + "telegram_chat_id": telegram_chat_id or None, + "telegram_message_id": telegram_message_id or None, + "notes": notes, + "original_filename": safe_name, + "stored_path": str(stored_path), + "mime_type": file.content_type or "application/octet-stream", + "file_size_bytes": len(payload), + "checksum_sha256": checksum, + "status": "uploaded-local", + "received_at": store.now(), + } + record["paperless_handoff"] = paperless.assignment_intake_payload(record) + created = repo.create_submission(record) + return {"status": "ok", "submission": created, "backend": repo.backend} + + +@router.post("/{submission_id}/confirm-paperless") +async def confirm_assignment_paperless(submission_id: str): + submissions = repo.list_submissions() + current = next((item for item in submissions if str(item.get("id")) == str(submission_id)), None) + if not current: + raise HTTPException(status_code=404, detail="Submission not found.") + handoff = paperless.assignment_intake_payload(current) + file_path = current.get("stored_path") or current.get("local_file") + if not file_path or not Path(file_path).exists(): + raise HTTPException(status_code=400, detail="Stored upload file is missing.") + fields = { + "class_name": handoff["class_name"], + "assignment_name": handoff["assignment_name"], + "submission_kind": handoff["submission_kind"], + "notes": handoff.get("notes", ""), + "source": handoff.get("source", "doris-schoolhouse"), + } + for optional_key in ("semester", "course_code", "paper_kind", "telegram_chat_id", "telegram_message_id"): + value = handoff.get(optional_key) + if value: + fields[optional_key] = value + response = webhooks.post_multipart(handoff["target_url"], fields, "document", file_path) + body = response.get("body", {}) if isinstance(response, dict) else {} + body_obj = body[0] if isinstance(body, list) and body and isinstance(body[0], dict) else (body if isinstance(body, dict) else {}) + ok = bool(response.get("ok", False)) if isinstance(response, dict) else False + updated = repo.update_submission(submission_id, { + "status": "paperless-queued" if ok else "paperless-error", + "confirmed_at": store.now(), + "webhook_response": response, + "paperless_document_id": body_obj.get("paperless_document_id") or body_obj.get("document_id"), + "paperless_title": body_obj.get("paperless_title") or body_obj.get("title") or handoff.get("assignment_name"), + }) + return {"status": "ok" if ok else "error", "submission": updated or current, "handoff": handoff, "webhook": response} + + +@router.post("/{assignment_id}/status") +async def update_assignment_status(assignment_id: str, status: str = Form(...)): + assignments = repo.list_assignments() + updated = None + for item in assignments: + if str(item.get("id")) == str(assignment_id): + item["status"] = status + item["updated_at"] = store.now() + updated = item + break + if not updated: + raise HTTPException(status_code=404, detail="Assignment not found.") + repo.replace_assignments(assignments) + return {"status": "ok", "assignment": updated} diff --git a/home/doris-schoolhouse/app/routes/api_dashboard.py b/home/doris-schoolhouse/app/routes/api_dashboard.py new file mode 100644 index 0000000..1dda37b --- /dev/null +++ b/home/doris-schoolhouse/app/routes/api_dashboard.py @@ -0,0 +1,31 @@ +from fastapi import APIRouter + +from app.services.d2l import D2LService +from app.services.repository import get_repository + + +router = APIRouter(prefix="/api/dashboard", tags=["dashboard"]) +repo = get_repository() +d2l = D2LService() + + +@router.get("/summary") +async def dashboard_summary(): + courses = d2l.get_courses() + assignments = repo.list_assignments() + submissions = repo.list_submissions() + recordings = repo.list_recordings() + sync_runs = repo.list_sync_runs() + return { + "backend": repo.backend, + "counts": { + "courses": len(courses), + "assignments": len(assignments), + "submissions": len(submissions), + "recordings": len(recordings), + }, + "latest_sync": sync_runs[-1] if sync_runs else None, + "recent_assignments": assignments[:8], + "recent_submissions": submissions[:8], + "recent_recordings": recordings[:8], + } diff --git a/home/doris-schoolhouse/app/routes/api_paperless.py b/home/doris-schoolhouse/app/routes/api_paperless.py new file mode 100644 index 0000000..0cc51dc --- /dev/null +++ b/home/doris-schoolhouse/app/routes/api_paperless.py @@ -0,0 +1,69 @@ +from fastapi import APIRouter, HTTPException + +from app.config import settings +from app.services.diagnostics import DiagnosticsService +from app.services.paperless import PaperlessService +from app.services.repository import get_repository +from app.services.store import JsonStore + + +router = APIRouter(prefix="/api", tags=["paperless", "health"]) +service = PaperlessService() +repo = get_repository() +store = JsonStore() +diagnostics = DiagnosticsService() + + +@router.get("/health") +async def health(): + return {"ok": True, "service": "doris-schoolhouse", "state_dir": settings.state_dir, "backend": repo.backend} + + +@router.get("/health/details") +async def health_details(): + return diagnostics.collect() + + +@router.post("/paperless/webhook/intake-enriched") +async def paperless_intake_enriched(submission_id: str, document_id: int, paperless_title: str | None = None): + updated = repo.update_submission( + submission_id, + { + "status": "paperless-linked", + "paperless_document_id": document_id, + "paperless_title": paperless_title, + "paperless_linked_at": store.now(), + }, + ) + if not updated: + raise HTTPException(status_code=404, detail="Submission not found.") + return {"status": "ok", "submission": updated} + + +@router.post("/paperless/webhook/recording-complete") +async def paperless_recording_complete(recording_id: str, transcript_document_id: int | None = None, summary_document_id: int | None = None, recording_document_id: int | None = None): + updated = repo.update_recording( + recording_id, + { + "status": "paperless-linked", + "transcript_paperless_document_id": transcript_document_id, + "summary_paperless_document_id": summary_document_id, + "recording_paperless_document_id": recording_document_id, + "transcription_status": "ready" if transcript_document_id else None, + "summary_status": "ready" if summary_document_id else None, + "paperless_linked_at": store.now(), + }, + ) + if not updated: + raise HTTPException(status_code=404, detail="Recording not found.") + return {"status": "ok", "recording": updated} + + +@router.post("/paperless/assignments/{assignment_id}/append-grade-note") +async def append_grade_note(assignment_id: str, execute: bool = False): + if not execute: + return service.describe_grade_note_plan(assignment_id) + try: + return service.append_grade_note(assignment_id) + except ValueError as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc diff --git a/home/doris-schoolhouse/app/routes/api_recordings.py b/home/doris-schoolhouse/app/routes/api_recordings.py new file mode 100644 index 0000000..f0beedf --- /dev/null +++ b/home/doris-schoolhouse/app/routes/api_recordings.py @@ -0,0 +1,130 @@ +import hashlib +from pathlib import Path + +from fastapi import APIRouter, Form, HTTPException, UploadFile + +from app.config import settings +from app.services.d2l import D2LService +from app.services.media import MediaService +from app.services.paperless import PaperlessService +from app.services.repository import get_repository +from app.services.store import JsonStore +from app.services.webhooks import WebhookService + + +router = APIRouter(prefix="/api/recordings", tags=["recordings"]) +store = JsonStore() +repo = get_repository() +d2l = D2LService() +paperless = PaperlessService() +media = MediaService() +webhooks = WebhookService() + + +@router.get("") +async def list_recordings(): + return {"items": repo.list_recordings(), "backend": repo.backend} + + +@router.post("/intake") +async def intake_recording( + file: UploadFile, + class_id: str = Form(...), + class_name: str = Form(...), + session_date: str = Form(...), + assignment_id: str | None = Form(None), + notes: str = Form(""), +): + course = next((item for item in d2l.get_courses() if str(item.get("d2l_course_id") or item.get("id")) == str(class_id)), None) + if not course: + raise HTTPException(status_code=404, detail="Class not found in cached D2L data.") + + upload_root = Path(settings.upload_tmp_dir) + upload_root.mkdir(parents=True, exist_ok=True) + recording_id = store.new_id("recording") + safe_name = Path(file.filename or "recording.wav").name + stored_path = upload_root / f"{recording_id}-{safe_name}" + payload = await file.read() + stored_path.write_bytes(payload) + checksum = hashlib.sha256(payload).hexdigest() + mp3_path = media.planned_mp3_path(str(stored_path)) + + record = { + "id": recording_id, + "class_id": str(course.get("d2l_course_id") or class_id), + "class_name": class_name, + "assignment_id": assignment_id, + "session_date": session_date, + "notes": notes, + "original_filename": safe_name, + "stored_path": str(stored_path), + "mp3_path": mp3_path, + "conversion_command": media.conversion_command(str(stored_path), mp3_path), + "delete_wav_after_success": True, + "mime_type": file.content_type or "audio/wav", + "file_size_bytes": len(payload), + "checksum_sha256": checksum, + "status": "uploaded-local", + "conversion_status": "queued", + "transcription_status": "queued", + "summary_status": "queued", + "received_at": store.now(), + } + record["paperless_handoff"] = paperless.recording_intake_payload(record) + created = repo.create_recording(record) + return {"status": "ok", "recording": created, "backend": repo.backend} + + +@router.post("/{recording_id}/reprocess") +async def reprocess_recording(recording_id: str): + recordings = repo.list_recordings() + current = next((item for item in recordings if str(item.get("id")) == str(recording_id)), None) + if not current: + raise HTTPException(status_code=404, detail="Recording not found.") + handoff = paperless.recording_intake_payload(current) + file_path = current.get("mp3_path") or current.get("stored_path") or current.get("local_file") + if not file_path or not Path(file_path).exists(): + raise HTTPException(status_code=400, detail="Local recording file is missing.") + fields = { + "class_name": handoff["class_name"], + "lecture_title": handoff["lecture_title"], + "notes": handoff.get("notes", ""), + "source": handoff.get("source", "doris-schoolhouse"), + } + response = webhooks.post_multipart(handoff["target_url"], fields, "data", file_path) + body = response.get("body", {}) if isinstance(response, dict) else {} + body_obj = body[0] if isinstance(body, list) and body and isinstance(body[0], dict) else (body if isinstance(body, dict) else {}) + ok = bool(response.get("ok", False)) if isinstance(response, dict) else False + updated = repo.update_recording(recording_id, { + "status": "requeued" if ok else "requeue-error", + "requeued_at": store.now(), + "webhook_response": response, + "transcription_status": "processing" if ok else "error", + "summary_status": "processing" if ok else "error", + "recording_paperless_document_id": body_obj.get("recording_paperless_document_id") or body_obj.get("document_id"), + }) + return {"status": "ok" if ok else "error", "recording": updated or current, "handoff": handoff, "webhook": response} + + +@router.post("/{recording_id}/convert") +async def convert_recording(recording_id: str): + recordings = repo.list_recordings() + current = next((item for item in recordings if str(item.get("id")) == str(recording_id)), None) + if not current: + raise HTTPException(status_code=404, detail="Recording not found.") + wav_path = current.get("stored_path") + if not wav_path or not Path(wav_path).exists(): + raise HTTPException(status_code=400, detail="Source WAV file is missing.") + mp3_path = current.get("mp3_path") or media.planned_mp3_path(wav_path) + result = media.convert_wav_to_mp3(wav_path, mp3_path) + updates = { + "conversion_status": result["status"], + "conversion_result": result, + "updated_at": store.now(), + } + if result["status"] == "ok": + updates["mp3_path"] = mp3_path + updates["delete_source_result"] = media.delete_source_if_requested(wav_path, bool(current.get("delete_wav_after_success"))) + updates["status"] = "converted" + updated = repo.update_recording(recording_id, updates) + return {"status": result["status"], "recording": updated or current, "conversion": result} diff --git a/home/doris-schoolhouse/app/routes/api_sync.py b/home/doris-schoolhouse/app/routes/api_sync.py new file mode 100644 index 0000000..d5e53b7 --- /dev/null +++ b/home/doris-schoolhouse/app/routes/api_sync.py @@ -0,0 +1,27 @@ +from fastapi import APIRouter + +from app.services.d2l import D2LService + + +router = APIRouter(prefix="/api/d2l", tags=["d2l"]) +service = D2LService() + + +@router.get("/courses") +async def get_courses(): + return {"items": service.get_courses()} + + +@router.get("/assignments") +async def get_course_assignments(course_id: str): + return {"items": service.get_assignments(course_id), "course_id": course_id} + + +@router.get("/grades") +async def get_course_grades(course_id: str): + return service.get_grades(course_id) + + +@router.post("/sync") +async def sync_d2l(): + return service.sync_to_store() diff --git a/home/doris-schoolhouse/app/routes/ui.py b/home/doris-schoolhouse/app/routes/ui.py new file mode 100644 index 0000000..7d7e1bb --- /dev/null +++ b/home/doris-schoolhouse/app/routes/ui.py @@ -0,0 +1,24 @@ +from pathlib import Path + +from fastapi import APIRouter, Request +from fastapi.responses import HTMLResponse +from fastapi.templating import Jinja2Templates + + +router = APIRouter() +templates = Jinja2Templates(directory=str(Path(__file__).resolve().parent.parent / "templates")) + + +@router.get("/", response_class=HTMLResponse) +async def dashboard(request: Request): + return templates.TemplateResponse("dashboard.html", {"request": request, "title": "Doris Schoolhouse"}) + + +@router.get("/assignments/upload", response_class=HTMLResponse) +async def assignment_upload(request: Request): + return templates.TemplateResponse("assignment_upload.html", {"request": request, "title": "Assignment Upload"}) + + +@router.get("/recordings/upload", response_class=HTMLResponse) +async def recording_upload(request: Request): + return templates.TemplateResponse("recording_upload.html", {"request": request, "title": "Recording Upload"}) diff --git a/home/doris-schoolhouse/app/scripts/apply_schema.py b/home/doris-schoolhouse/app/scripts/apply_schema.py new file mode 100644 index 0000000..728b74a --- /dev/null +++ b/home/doris-schoolhouse/app/scripts/apply_schema.py @@ -0,0 +1,31 @@ +import os +import sys +from pathlib import Path + + +def main() -> int: + schema_path = Path(__file__).resolve().parent.parent / 'db' / 'schema.sql' + if not schema_path.exists(): + print(f'missing schema file: {schema_path}', file=sys.stderr) + return 1 + dsn = os.getenv('DATABASE_URL', '') + if not dsn: + print('DATABASE_URL is not set', file=sys.stderr) + return 1 + try: + import psycopg + except Exception: + print('psycopg is not installed; cannot apply schema directly', file=sys.stderr) + return 1 + dsn = dsn.replace('postgresql+psycopg://', 'postgresql://') + sql = schema_path.read_text() + with psycopg.connect(dsn) as conn: + with conn.cursor() as cur: + cur.execute(sql) + conn.commit() + print(f'Applied schema from {schema_path}') + return 0 + + +if __name__ == '__main__': + raise SystemExit(main()) diff --git a/home/doris-schoolhouse/app/scripts/bootstrap_state.py b/home/doris-schoolhouse/app/scripts/bootstrap_state.py new file mode 100644 index 0000000..88c4c1b --- /dev/null +++ b/home/doris-schoolhouse/app/scripts/bootstrap_state.py @@ -0,0 +1,27 @@ +from pathlib import Path + +from app.config import settings +from app.services.d2l import D2LService +from app.services.store import JsonStore + + +def main() -> None: + Path(settings.upload_tmp_dir).mkdir(parents=True, exist_ok=True) + Path(settings.state_dir).mkdir(parents=True, exist_ok=True) + store = JsonStore() + for name, default in { + 'courses': [], + 'assignments': [], + 'submissions': [], + 'recordings': [], + 'sync_runs': [], + }.items(): + path = Path(settings.state_dir) / f'{name}.json' + if not path.exists(): + store.save(name, default) + result = D2LService().sync_to_store() + print(result) + + +if __name__ == '__main__': + main() diff --git a/home/doris-schoolhouse/app/scripts/process_recording.py b/home/doris-schoolhouse/app/scripts/process_recording.py new file mode 100644 index 0000000..4ce9ecf --- /dev/null +++ b/home/doris-schoolhouse/app/scripts/process_recording.py @@ -0,0 +1,44 @@ +import argparse +import json + +from app.services.media import MediaService +from app.services.repository import get_repository +from app.services.store import JsonStore + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument('recording_id') + parser.add_argument('--convert-only', action='store_true') + args = parser.parse_args() + + repo = get_repository() + store = JsonStore() + media = MediaService() + + recordings = repo.list_recordings() + current = next((item for item in recordings if str(item.get('id')) == str(args.recording_id)), None) + if not current: + raise SystemExit(f'recording not found: {args.recording_id}') + + wav_path = current.get('stored_path') + mp3_path = current.get('mp3_path') or media.planned_mp3_path(wav_path) + result = media.convert_wav_to_mp3(wav_path, mp3_path) + + updates = { + 'conversion_status': result['status'], + 'conversion_result': result, + 'updated_at': store.now(), + } + if result['status'] == 'ok': + updates['mp3_path'] = mp3_path + delete_result = media.delete_source_if_requested(wav_path, bool(current.get('delete_wav_after_success'))) + updates['delete_source_result'] = delete_result + if not args.convert_only: + updates['status'] = 'converted' + repo.update_recording(str(current.get('id')), updates) + print(json.dumps({'recording_id': current.get('id'), 'result': result, 'updates': updates}, indent=2)) + + +if __name__ == '__main__': + main() diff --git a/home/doris-schoolhouse/app/services/d2l.py b/home/doris-schoolhouse/app/services/d2l.py new file mode 100644 index 0000000..070fb67 --- /dev/null +++ b/home/doris-schoolhouse/app/services/d2l.py @@ -0,0 +1,139 @@ +import json +from pathlib import Path +from typing import Any + +from app.config import settings +from app.services.repository import get_repository +from app.services.store import JsonStore + + +class D2LService: + def __init__(self) -> None: + self.store = JsonStore() + self.repo = get_repository() + + def _snapshot_path(self) -> Path: + candidate_dirs = [ + Path(settings.d2l_scraper_workdir), + Path('/home/fizzlepoof/.openclaw/workspace/school'), + ] + for workdir in candidate_dirs: + for name in ('d2l_current_snapshot.json', 'd2l_grades.json', 'd2l_scraper_output.json'): + candidate = workdir / name + if candidate.exists(): + return candidate + return candidate_dirs[0] / 'd2l_current_snapshot.json' + + def load_snapshot(self) -> dict[str, Any]: + path = self._snapshot_path() + if not path.exists(): + return {"courses": [], "source_path": str(path), "missing": True} + data = json.loads(path.read_text()) + if isinstance(data, dict): + data["source_path"] = str(path) + return data + return {"courses": [], "source_path": str(path), "missing": True} + + def _normalize_courses(self, snapshot: dict[str, Any]) -> list[dict[str, Any]]: + items = [] + for course in snapshot.get("courses", []): + items.append({ + "id": str(course.get("id")), + "d2l_course_id": str(course.get("id")), + "name": course.get("name"), + "code": course.get("code"), + "finalSummary": course.get("finalSummary"), + "gradeItemCount": course.get("gradeItemCount"), + "synced_at": self.store.now(), + }) + return items + + def _normalize_assignments(self, snapshot: dict[str, Any]) -> list[dict[str, Any]]: + items = [] + for course in snapshot.get("courses", []): + course_id = str(course.get("id")) + for grade_item in course.get("gradeItems", []): + if grade_item.get("isCategory"): + continue + title = grade_item.get("name") or "Untitled" + items.append({ + "id": f"{course_id}:{title.lower().strip()}", + "course_id": course_id, + "course_name": course.get("name"), + "title": title, + "points": grade_item.get("points"), + "grade": grade_item.get("grade"), + "comments": grade_item.get("comments"), + "source": "d2l-grade-item", + "synced_at": self.store.now(), + }) + return items + + def sync_to_store(self) -> dict[str, Any]: + snapshot = self.load_snapshot() + courses = self._normalize_courses(snapshot) + assignments = self._normalize_assignments(snapshot) + existing_assignments = {item.get('id'): item for item in self.repo.list_assignments()} + for item in assignments: + prior = existing_assignments.get(item['id']) or {} + item['status'] = prior.get('status', 'assigned') + item['updated_at'] = self.store.now() + item['created_at'] = prior.get('created_at', self.store.now()) + self.repo.replace_courses(courses) + self.repo.replace_assignments(assignments) + sync_runs = self.repo.list_sync_runs() + sync_runs.append({ + "id": self.store.new_id("sync"), + "snapshot_path": snapshot.get("source_path"), + "course_count": len(courses), + "assignment_count": len(assignments), + "completed_at": self.store.now(), + "status": "completed", + }) + self.repo.save_sync_runs(sync_runs) + return { + "status": "ok", + "snapshot_path": snapshot.get("source_path"), + "course_count": len(courses), + "assignment_count": len(assignments), + "backend": self.repo.backend, + } + + def get_courses(self) -> list[dict[str, Any]]: + items = self.repo.list_courses() + if items: + return items + self.sync_to_store() + return self.repo.list_courses() + + def get_assignments(self, course_id: str) -> list[dict[str, Any]]: + items = self.repo.list_assignments() + if not items: + self.sync_to_store() + items = self.repo.list_assignments() + return [item for item in items if str(item.get("course_id")) == str(course_id)] + + def get_grades(self, course_id: str) -> dict[str, Any]: + snapshot = self.load_snapshot() + for course in snapshot.get("courses", []): + if str(course.get("id")) == str(course_id): + return { + "course_id": str(course.get("id")), + "course_name": course.get("name"), + "final": course.get("final"), + "finalSummary": course.get("finalSummary"), + "gradeItems": course.get("gradeItems", []), + } + return {"course_id": course_id, "gradeItems": []} + + def describe_sync_plan(self) -> dict: + snapshot = self.load_snapshot() + return { + "status": "stub", + "backend": self.repo.backend, + "scraper_workdir": settings.d2l_scraper_workdir, + "scraper_command": settings.d2l_scraper_command, + "snapshot_path": snapshot.get("source_path"), + "course_count": len(snapshot.get("courses", [])), + "message": "Run existing D2L scraper, normalize courses/assignments/grades, then store into Schoolhouse persistence.", + } diff --git a/home/doris-schoolhouse/app/services/diagnostics.py b/home/doris-schoolhouse/app/services/diagnostics.py new file mode 100644 index 0000000..6e8e072 --- /dev/null +++ b/home/doris-schoolhouse/app/services/diagnostics.py @@ -0,0 +1,23 @@ +from pathlib import Path + +from app.config import settings +from app.services.media import MediaService +from app.services.repository import get_repository + + +class DiagnosticsService: + def collect(self) -> dict: + repo = get_repository() + media = MediaService() + return { + 'backend': repo.backend, + 'state_dir': settings.state_dir, + 'state_dir_exists': Path(settings.state_dir).exists(), + 'upload_tmp_dir': settings.upload_tmp_dir, + 'upload_tmp_dir_exists': Path(settings.upload_tmp_dir).exists(), + 'ffmpeg_path': media.ffmpeg_path(), + 'can_convert_audio': media.can_convert(), + 'n8n_base_url': settings.n8n_base_url, + 'paperless_base_url': settings.paperless_base_url, + 'd2l_scraper_workdir': settings.d2l_scraper_workdir, + } diff --git a/home/doris-schoolhouse/app/services/media.py b/home/doris-schoolhouse/app/services/media.py new file mode 100644 index 0000000..fe4e34b --- /dev/null +++ b/home/doris-schoolhouse/app/services/media.py @@ -0,0 +1,57 @@ +import shutil +import subprocess +from pathlib import Path + + +class MediaService: + def planned_mp3_path(self, wav_path: str) -> str: + path = Path(wav_path) + return str(path.with_suffix('.mp3')) + + def conversion_command(self, wav_path: str, mp3_path: str) -> list[str]: + return [ + 'ffmpeg', + '-y', + '-i', wav_path, + '-codec:a', 'libmp3lame', + '-q:a', '2', + mp3_path, + ] + + def ffmpeg_path(self) -> str | None: + return shutil.which('ffmpeg') + + def can_convert(self) -> bool: + return self.ffmpeg_path() is not None + + def convert_wav_to_mp3(self, wav_path: str, mp3_path: str) -> dict: + if not self.can_convert(): + return { + 'status': 'blocked', + 'reason': 'ffmpeg-not-installed', + 'command': self.conversion_command(wav_path, mp3_path), + } + cmd = self.conversion_command(wav_path, mp3_path) + result = subprocess.run(cmd, capture_output=True, text=True) + if result.returncode != 0: + return { + 'status': 'failed', + 'returncode': result.returncode, + 'stderr': result.stderr[-4000:], + 'stdout': result.stdout[-2000:], + 'command': cmd, + } + return { + 'status': 'ok', + 'mp3_path': mp3_path, + 'command': cmd, + } + + def delete_source_if_requested(self, wav_path: str, should_delete: bool) -> dict: + if not should_delete: + return {'status': 'skipped', 'reason': 'delete-not-requested'} + path = Path(wav_path) + if not path.exists(): + return {'status': 'skipped', 'reason': 'source-missing'} + path.unlink() + return {'status': 'ok', 'deleted': wav_path} diff --git a/home/doris-schoolhouse/app/services/paperless.py b/home/doris-schoolhouse/app/services/paperless.py new file mode 100644 index 0000000..67afb8e --- /dev/null +++ b/home/doris-schoolhouse/app/services/paperless.py @@ -0,0 +1,109 @@ +import hashlib +from urllib.parse import urljoin + +from app.config import settings +from app.services.paperless_client import PaperlessClient +from app.services.repository import get_repository +from app.services.store import JsonStore + + +class PaperlessService: + def __init__(self) -> None: + self.store = JsonStore() + self.repo = get_repository() + self.client = PaperlessClient() + + def assignment_intake_payload(self, submission: dict) -> dict: + return { + "target_url": urljoin(settings.n8n_base_url, settings.n8n_assignment_webhook), + "class_name": submission.get("class_name"), + "assignment_name": submission.get("assignment_name"), + "submission_kind": submission.get("submission_kind") or "assignment_turn_in", + "semester": submission.get("semester"), + "course_code": submission.get("course_code"), + "paper_kind": submission.get("paper_kind"), + "notes": submission.get("notes", ""), + "source": submission.get("source") or "doris-schoolhouse", + "telegram_chat_id": submission.get("telegram_chat_id"), + "telegram_message_id": submission.get("telegram_message_id"), + "local_file": submission.get("stored_path"), + "version": submission.get("version_label"), + } + + def recording_intake_payload(self, recording: dict) -> dict: + return { + "target_url": urljoin(settings.n8n_base_url, settings.n8n_recording_webhook), + "class_name": recording.get("class_name"), + "lecture_title": f"{recording.get('class_name')} {recording.get('session_date')}", + "notes": recording.get("notes", ""), + "local_file": recording.get("mp3_path") or recording.get("stored_path"), + "source": "doris-schoolhouse", + } + + def build_grade_note(self, assignment_id: str) -> str: + submissions = self.repo.list_submissions() + assignments = {item.get("id"): item for item in self.repo.list_assignments()} + assignment = assignments.get(str(assignment_id), {}) + linked = [s for s in submissions if str(s.get("assignment_id")) == str(assignment_id)] + note_lines = [ + "Grade sync from Doris Schoolhouse", + f"Assignment: {assignment.get('title', 'unknown')}", + ] + if assignment.get("grade"): + note_lines.append(f"Grade: {assignment['grade']}") + if assignment.get("points"): + note_lines.append(f"Points: {assignment['points']}") + if assignment.get("comments"): + note_lines.append(f"Professor comments: {assignment['comments']}") + if linked: + note_lines.append(f"Linked submissions: {len(linked)}") + return "\n".join(note_lines) + + def describe_grade_note_plan(self, assignment_id: str) -> dict: + submissions = self.repo.list_submissions() + linked = [s for s in submissions if str(s.get("assignment_id")) == str(assignment_id)] + return { + "status": "stub", + "backend": self.repo.backend, + "assignment_id": assignment_id, + "paperless_base_url": settings.paperless_base_url, + "linked_submission_count": len(linked), + "note_preview": self.build_grade_note(assignment_id), + "message": "Resolve linked Paperless submission document, hash latest grade/comment payload, and append a dated Schoolhouse note if changed.", + } + + def append_grade_note(self, assignment_id: str) -> dict: + submissions = self.repo.list_submissions() + linked = [s for s in submissions if str(s.get("assignment_id")) == str(assignment_id)] + if not linked: + raise ValueError("No linked submissions for assignment") + target = next((s for s in linked if s.get("paperless_document_id")), linked[0]) + document_id = target.get("paperless_document_id") + if not document_id: + raise ValueError("Linked submission has no Paperless document id yet") + note = self.build_grade_note(assignment_id) + note_hash = hashlib.sha256(note.encode('utf-8')).hexdigest() + if target.get("paperless_note_last_hash") == note_hash: + return { + "status": "skipped", + "reason": "note-unchanged", + "assignment_id": assignment_id, + "document_id": document_id, + "note_hash": note_hash, + } + response = self.client.add_note(document_id, note) + self.repo.update_submission( + str(target.get("id")), + { + "paperless_note_last_hash": note_hash, + "paperless_title": target.get("paperless_title"), + "status": target.get("status", "paperless-linked"), + }, + ) + return { + "status": "ok", + "assignment_id": assignment_id, + "document_id": document_id, + "note_hash": note_hash, + "paperless_response": response, + } diff --git a/home/doris-schoolhouse/app/services/paperless_client.py b/home/doris-schoolhouse/app/services/paperless_client.py new file mode 100644 index 0000000..5fca78e --- /dev/null +++ b/home/doris-schoolhouse/app/services/paperless_client.py @@ -0,0 +1,31 @@ +import json +from urllib import request +from urllib.parse import urljoin + +from app.config import settings + + +class PaperlessClient: + def __init__(self) -> None: + self.base_url = settings.paperless_base_url.rstrip('/') + '/' + self.token = settings.paperless_api_token + + def _req(self, method: str, path: str, payload: dict | None = None) -> dict: + url = urljoin(self.base_url, path.lstrip('/')) + body = None if payload is None else json.dumps(payload).encode('utf-8') + headers = { + 'Authorization': f'Token {self.token}', + 'Accept': 'application/json', + } + if body is not None: + headers['Content-Type'] = 'application/json' + req = request.Request(url, data=body, headers=headers, method=method) + with request.urlopen(req, timeout=120) as resp: + raw = resp.read().decode('utf-8', errors='replace') + return json.loads(raw) if raw else {} + + def get_document(self, document_id: int | str) -> dict: + return self._req('GET', f'/api/documents/{document_id}/') + + def add_note(self, document_id: int | str, note: str) -> dict: + return self._req('POST', f'/api/documents/{document_id}/notes/', {'note': note}) diff --git a/home/doris-schoolhouse/app/services/repository.py b/home/doris-schoolhouse/app/services/repository.py new file mode 100644 index 0000000..8efbcca --- /dev/null +++ b/home/doris-schoolhouse/app/services/repository.py @@ -0,0 +1,388 @@ +import json +from pathlib import Path +from typing import Any + +from app.config import settings +from app.services.store import JsonStore + + +class JsonRepository: + backend = "json" + + def __init__(self) -> None: + self.store = JsonStore() + + def list_courses(self) -> list[dict[str, Any]]: + return self.store.load("courses", []) + + def replace_courses(self, items: list[dict[str, Any]]) -> list[dict[str, Any]]: + return self.store.replace_collection("courses", items) + + def list_assignments(self) -> list[dict[str, Any]]: + return self.store.load("assignments", []) + + def replace_assignments(self, items: list[dict[str, Any]]) -> list[dict[str, Any]]: + return self.store.replace_collection("assignments", items) + + def list_sync_runs(self) -> list[dict[str, Any]]: + return self.store.load("sync_runs", []) + + def save_sync_runs(self, items: list[dict[str, Any]]) -> list[dict[str, Any]]: + return self.store.save("sync_runs", items) + + def list_submissions(self) -> list[dict[str, Any]]: + return self.store.load("submissions", []) + + def create_submission(self, item: dict[str, Any]) -> dict[str, Any]: + return self.store.append_collection("submissions", item) + + def update_submission(self, submission_id: str, updates: dict[str, Any]) -> dict[str, Any] | None: + return self.store.update_collection_item( + "submissions", + lambda item: str(item.get("id")) == str(submission_id), + lambda item: {**item, **updates}, + ) + + def list_recordings(self) -> list[dict[str, Any]]: + return self.store.load("recordings", []) + + def create_recording(self, item: dict[str, Any]) -> dict[str, Any]: + return self.store.append_collection("recordings", item) + + def update_recording(self, recording_id: str, updates: dict[str, Any]) -> dict[str, Any] | None: + return self.store.update_collection_item( + "recordings", + lambda item: str(item.get("id")) == str(recording_id), + lambda item: {**item, **updates}, + ) + + +class PostgresRepository: + backend = "postgres" + + def __init__(self) -> None: + import psycopg + from psycopg.rows import dict_row + + self._psycopg = psycopg + self._dict_row = dict_row + self.dsn = settings.database_url.replace("postgresql+psycopg://", "postgresql://") + + def _connect(self): + return self._psycopg.connect(self.dsn, row_factory=self._dict_row) + + def _ensure_course(self, cur, item: dict[str, Any]) -> dict[str, Any] | None: + course_id = item.get("class_id") or item.get("course_id") + if not course_id: + return None + cur.execute("SELECT id, d2l_course_id, name FROM schoolhouse_course WHERE d2l_course_id = %s", (course_id,)) + row = cur.fetchone() + if row: + return row + course_name = item.get("class_name") or item.get("course_name") or str(course_id) + cur.execute( + """ + INSERT INTO schoolhouse_course (d2l_course_id, name, last_synced_at) + VALUES (%s, %s, NOW()) + RETURNING id, d2l_course_id, name + """, + (course_id, course_name), + ) + return cur.fetchone() + + def _ensure_assignment(self, cur, item: dict[str, Any]) -> dict[str, Any] | None: + course = self._ensure_course(cur, item) + if not course: + return None + assignment_id = item.get("assignment_id") + assignment_name = item.get("assignment_name") or item.get("title") + if assignment_id: + cur.execute( + "SELECT id, title FROM schoolhouse_assignment WHERE course_id = %s AND d2l_assignment_key = %s LIMIT 1", + (course["id"], assignment_id), + ) + row = cur.fetchone() + if row: + return row + if assignment_name: + cur.execute( + "SELECT id, title FROM schoolhouse_assignment WHERE course_id = %s AND title = %s LIMIT 1", + (course["id"], assignment_name), + ) + row = cur.fetchone() + if row: + return row + cur.execute( + """ + INSERT INTO schoolhouse_assignment (course_id, d2l_assignment_key, title, source, status, created_at, updated_at) + VALUES (%s, %s, %s, %s, %s, NOW(), NOW()) + RETURNING id, title + """, + (course["id"], assignment_id, assignment_name, item.get("source", "schoolhouse-intake"), item.get("status", "assigned")), + ) + return cur.fetchone() + return None + + def list_courses(self) -> list[dict[str, Any]]: + with self._connect() as conn, conn.cursor() as cur: + cur.execute("SELECT id::text AS id, d2l_course_id, name, code, semester, last_synced_at FROM schoolhouse_course ORDER BY name") + return list(cur.fetchall()) + + def replace_courses(self, items: list[dict[str, Any]]) -> list[dict[str, Any]]: + with self._connect() as conn, conn.cursor() as cur: + for item in items: + cur.execute( + """ + INSERT INTO schoolhouse_course (d2l_course_id, name, code, semester, last_synced_at) + VALUES (%s, %s, %s, %s, NOW()) + ON CONFLICT (d2l_course_id) + DO UPDATE SET name = EXCLUDED.name, code = EXCLUDED.code, semester = EXCLUDED.semester, last_synced_at = NOW() + """, + (item.get("d2l_course_id"), item.get("name"), item.get("code"), item.get("semester")), + ) + conn.commit() + return items + + def list_assignments(self) -> list[dict[str, Any]]: + with self._connect() as conn, conn.cursor() as cur: + cur.execute( + """ + SELECT a.id::text AS id, + c.d2l_course_id AS course_id, + c.name AS course_name, + a.title, + a.grade_text AS grade, + a.professor_comments AS comments, + a.status, + a.updated_at + FROM schoolhouse_assignment a + JOIN schoolhouse_course c ON c.id = a.course_id + ORDER BY c.name, a.title + """ + ) + return list(cur.fetchall()) + + def replace_assignments(self, items: list[dict[str, Any]]) -> list[dict[str, Any]]: + with self._connect() as conn, conn.cursor() as cur: + for item in items: + cur.execute("SELECT id FROM schoolhouse_course WHERE d2l_course_id = %s", (item.get("course_id"),)) + row = cur.fetchone() + if not row: + continue + course_pk = row["id"] + cur.execute( + """ + INSERT INTO schoolhouse_assignment (course_id, d2l_assignment_key, title, grade_text, professor_comments, source, status, created_at, updated_at) + VALUES (%s, %s, %s, %s, %s, %s, %s, NOW(), NOW()) + ON CONFLICT DO NOTHING + """, + (course_pk, item.get("id"), item.get("title"), item.get("grade"), item.get("comments"), item.get("source", "d2l"), item.get("status", "assigned")), + ) + cur.execute( + """ + UPDATE schoolhouse_assignment + SET grade_text = %s, + professor_comments = %s, + status = COALESCE(status, %s), + updated_at = NOW() + WHERE course_id = %s AND title = %s + """, + (item.get("grade"), item.get("comments"), item.get("status", "assigned"), course_pk, item.get("title")), + ) + conn.commit() + return items + + def list_sync_runs(self) -> list[dict[str, Any]]: + with self._connect() as conn, conn.cursor() as cur: + cur.execute("SELECT id::text AS id, started_at, completed_at, status, raw_snapshot_path AS snapshot_path, changes_summary_json FROM schoolhouse_d2l_sync_run ORDER BY started_at DESC") + return list(cur.fetchall()) + + def save_sync_runs(self, items: list[dict[str, Any]]) -> list[dict[str, Any]]: + if not items: + return items + last = items[-1] + with self._connect() as conn, conn.cursor() as cur: + cur.execute( + "INSERT INTO schoolhouse_d2l_sync_run (completed_at, status, raw_snapshot_path, changes_summary_json) VALUES (NOW(), %s, %s, %s::jsonb)", + (last.get("status", "completed"), last.get("snapshot_path"), json.dumps(last)), + ) + conn.commit() + return items + + def list_submissions(self) -> list[dict[str, Any]]: + with self._connect() as conn, conn.cursor() as cur: + cur.execute( + """ + SELECT s.id::text AS id, + a.id::text AS assignment_id, + c.d2l_course_id AS class_id, + c.name AS class_name, + a.title AS assignment_name, + s.version_label, + s.original_filename, + s.mime_type, + s.status, + s.paperless_document_id, + s.paperless_title, + s.paperless_note_last_hash, + s.submitted_at, + s.confirmed_metadata_json, + s.confirmed_metadata_json->>'stored_path' AS stored_path, + s.confirmed_metadata_json->>'local_file' AS local_file + FROM schoolhouse_submission s + JOIN schoolhouse_assignment a ON a.id = s.assignment_id + JOIN schoolhouse_course c ON c.id = a.course_id + ORDER BY s.created_at DESC + """ + ) + return list(cur.fetchall()) + + def create_submission(self, item: dict[str, Any]) -> dict[str, Any]: + with self._connect() as conn, conn.cursor() as cur: + row = self._ensure_assignment(cur, item) + if not row: + raise ValueError("Assignment not found and could not be created in Postgres repository") + cur.execute( + """ + INSERT INTO schoolhouse_submission ( + assignment_id, version_label, original_filename, mime_type, checksum_sha256, + submitted_at, suggested_metadata_json, confirmed_metadata_json, status + ) VALUES (%s, %s, %s, %s, %s, NOW(), %s::jsonb, %s::jsonb, %s) + RETURNING id::text AS id + """, + ( + row["id"], item.get("version_label") or "v1", item.get("original_filename"), item.get("mime_type", "application/octet-stream"), item.get("checksum_sha256"), + json.dumps(item.get("paperless_handoff", {})), json.dumps(item), item.get("status", "uploaded-local"), + ), + ) + created = cur.fetchone() + conn.commit() + item["id"] = created["id"] + return item + + def update_submission(self, submission_id: str, updates: dict[str, Any]) -> dict[str, Any] | None: + with self._connect() as conn, conn.cursor() as cur: + cur.execute("SELECT confirmed_metadata_json FROM schoolhouse_submission WHERE id = %s::bigint", (submission_id,)) + current = cur.fetchone() + if not current: + return None + merged_metadata = {**(current.get("confirmed_metadata_json") or {}), **updates} + cur.execute( + """ + UPDATE schoolhouse_submission + SET status = COALESCE(%s, status), + paperless_document_id = COALESCE(%s, paperless_document_id), + paperless_title = COALESCE(%s, paperless_title), + paperless_note_last_hash = COALESCE(%s, paperless_note_last_hash), + paperless_note_appended_at = CASE WHEN %s::text IS NOT NULL THEN NOW() ELSE paperless_note_appended_at END, + confirmed_metadata_json = %s::jsonb, + updated_at = NOW() + WHERE id = %s::bigint + RETURNING id::text AS id, status, paperless_document_id, paperless_title, paperless_note_last_hash + """, + ( + updates.get("status"), updates.get("paperless_document_id"), updates.get("paperless_title"), updates.get("paperless_note_last_hash"), updates.get("paperless_note_last_hash"), + json.dumps(merged_metadata), submission_id, + ), + ) + row = cur.fetchone() + conn.commit() + return {**merged_metadata, **dict(row)} if row else None + + def list_recordings(self) -> list[dict[str, Any]]: + with self._connect() as conn, conn.cursor() as cur: + cur.execute( + """ + SELECT r.id::text AS id, + c.d2l_course_id AS class_id, + c.name AS class_name, + r.assignment_id::text AS assignment_id, + r.course_id::text AS course_id, + r.session_date::text AS session_date, + r.original_filename, + r.mp3_filename, + r.transcription_status, + r.summary_status, + r.recording_paperless_document_id, + r.recording_metadata_json, + r.recording_metadata_json->>'stored_path' AS stored_path, + r.recording_metadata_json->>'mp3_path' AS mp3_path, + r.recording_metadata_json->>'status' AS status, + r.recording_metadata_json->>'conversion_status' AS conversion_status, + r.recording_metadata_json->>'notes' AS notes, + r.recording_metadata_json->>'delete_wav_after_success' AS delete_wav_after_success + FROM schoolhouse_recording r + JOIN schoolhouse_course c ON c.id = r.course_id + ORDER BY r.created_at DESC + """ + ) + return list(cur.fetchall()) + + def create_recording(self, item: dict[str, Any]) -> dict[str, Any]: + with self._connect() as conn, conn.cursor() as cur: + row = self._ensure_course(cur, item) + if not row: + raise ValueError("Course not found and could not be created in Postgres repository") + cur.execute( + """ + INSERT INTO schoolhouse_recording ( + course_id, assignment_id, session_date, original_filename, mp3_filename, notes, + transcription_status, summary_status, recording_metadata_json + ) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s::jsonb) + RETURNING id::text AS id + """, + ( + row["id"], item.get("assignment_id") or None, item.get("session_date"), item.get("original_filename"), + Path(item.get("mp3_path")).name if item.get("mp3_path") else None, item.get("notes"), item.get("transcription_status"), + item.get("summary_status"), json.dumps(item), + ), + ) + created = cur.fetchone() + conn.commit() + item["id"] = created["id"] + return item + + def update_recording(self, recording_id: str, updates: dict[str, Any]) -> dict[str, Any] | None: + with self._connect() as conn, conn.cursor() as cur: + cur.execute("SELECT recording_metadata_json FROM schoolhouse_recording WHERE id = %s::bigint", (recording_id,)) + current = cur.fetchone() + if not current: + return None + merged_metadata = {**(current.get("recording_metadata_json") or {}), **updates} + cur.execute( + """ + UPDATE schoolhouse_recording + SET transcription_status = COALESCE(%s, transcription_status), + summary_status = COALESCE(%s, summary_status), + transcript_paperless_document_id = COALESCE(%s, transcript_paperless_document_id), + summary_paperless_document_id = COALESCE(%s, summary_paperless_document_id), + recording_paperless_document_id = COALESCE(%s, recording_paperless_document_id), + mp3_filename = COALESCE(%s, mp3_filename), + notes = COALESCE(%s, notes), + recording_metadata_json = %s::jsonb, + updated_at = NOW() + WHERE id = %s::bigint + RETURNING id::text AS id, transcription_status, summary_status, recording_paperless_document_id + """, + ( + updates.get("transcription_status"), updates.get("summary_status"), updates.get("transcript_paperless_document_id"), + updates.get("summary_paperless_document_id"), updates.get("recording_paperless_document_id"), + Path(updates.get("mp3_path")).name if updates.get("mp3_path") else None, + updates.get("notes"), json.dumps(merged_metadata), recording_id, + ), + ) + row = cur.fetchone() + conn.commit() + return {**merged_metadata, **dict(row)} if row else None + + +def get_repository(): + backend = getattr(settings, "storage_backend", "auto") + if backend == "json": + return JsonRepository() + if settings.database_url.startswith("postgresql"): + try: + return PostgresRepository() + except Exception: + return JsonRepository() + return JsonRepository() diff --git a/home/doris-schoolhouse/app/services/store.py b/home/doris-schoolhouse/app/services/store.py new file mode 100644 index 0000000..d59b352 --- /dev/null +++ b/home/doris-schoolhouse/app/services/store.py @@ -0,0 +1,53 @@ +import json +from datetime import datetime, timezone +from pathlib import Path +from typing import Any +from uuid import uuid4 + +from app.config import settings + + +class JsonStore: + def __init__(self) -> None: + self.root = Path(settings.state_dir) + self.root.mkdir(parents=True, exist_ok=True) + + def _path(self, name: str) -> Path: + return self.root / f"{name}.json" + + def load(self, name: str, default: Any) -> Any: + path = self._path(name) + if not path.exists(): + return default + return json.loads(path.read_text()) + + def save(self, name: str, value: Any) -> Any: + path = self._path(name) + path.write_text(json.dumps(value, indent=2, sort_keys=True)) + return value + + def replace_collection(self, name: str, items: list[dict[str, Any]]) -> list[dict[str, Any]]: + return self.save(name, items) + + def append_collection(self, name: str, item: dict[str, Any]) -> dict[str, Any]: + items = self.load(name, []) + items.append(item) + self.save(name, items) + return item + + def update_collection_item(self, name: str, predicate, updater): + items = self.load(name, []) + updated = None + for idx, item in enumerate(items): + if predicate(item): + items[idx] = updater(dict(item)) + updated = items[idx] + break + self.save(name, items) + return updated + + def now(self) -> str: + return datetime.now(timezone.utc).isoformat() + + def new_id(self, prefix: str) -> str: + return f"{prefix}-{uuid4().hex[:12]}" diff --git a/home/doris-schoolhouse/app/services/webhooks.py b/home/doris-schoolhouse/app/services/webhooks.py new file mode 100644 index 0000000..b36cad5 --- /dev/null +++ b/home/doris-schoolhouse/app/services/webhooks.py @@ -0,0 +1,52 @@ +import json +import mimetypes +import uuid +from pathlib import Path +from urllib import error, request + + +class WebhookService: + def _multipart_body(self, fields: dict[str, str], file_field: str, file_path: str) -> tuple[bytes, str]: + boundary = f"----DorisSchoolhouse{uuid.uuid4().hex}" + parts: list[bytes] = [] + for key, value in fields.items(): + parts.extend([ + f"--{boundary}\r\n".encode(), + f'Content-Disposition: form-data; name="{key}"\r\n\r\n'.encode(), + str(value).encode(), + b"\r\n", + ]) + path = Path(file_path) + mime = mimetypes.guess_type(path.name)[0] or "application/octet-stream" + parts.extend([ + f"--{boundary}\r\n".encode(), + f'Content-Disposition: form-data; name="{file_field}"; filename="{path.name}"\r\n'.encode(), + f"Content-Type: {mime}\r\n\r\n".encode(), + path.read_bytes(), + b"\r\n", + f"--{boundary}--\r\n".encode(), + ]) + return b"".join(parts), boundary + + def post_multipart(self, url: str, fields: dict[str, str], file_field: str, file_path: str) -> dict: + body, boundary = self._multipart_body(fields, file_field, file_path) + req = request.Request( + url, + data=body, + headers={"Content-Type": f"multipart/form-data; boundary={boundary}"}, + method="POST", + ) + + def parse_raw(raw: str): + try: + return json.loads(raw) + except Exception: + return {"raw": raw} + + try: + with request.urlopen(req, timeout=120) as response: + raw = response.read().decode("utf-8", errors="replace") + return {"ok": True, "status_code": response.status, "body": parse_raw(raw)} + except error.HTTPError as exc: + raw = exc.read().decode("utf-8", errors="replace") + return {"ok": False, "status_code": exc.code, "body": parse_raw(raw), "error": str(exc)} diff --git a/home/doris-schoolhouse/app/static/app.css b/home/doris-schoolhouse/app/static/app.css new file mode 100644 index 0000000..ea361bb --- /dev/null +++ b/home/doris-schoolhouse/app/static/app.css @@ -0,0 +1,60 @@ +:root { + color-scheme: dark; + --bg: #0f172a; + --panel: #111827; + --panel-2: #1f2937; + --line: #334155; + --text: #e5e7eb; + --muted: #94a3b8; + --accent: #38bdf8; +} + +* { box-sizing: border-box; } +body { + margin: 0; + font-family: Inter, system-ui, sans-serif; + background: linear-gradient(180deg, #020617, var(--bg)); + color: var(--text); +} + +.site-header, .page-shell { max-width: 1200px; margin: 0 auto; padding: 24px; } +.site-header { display: flex; justify-content: space-between; gap: 24px; align-items: end; } +.site-header p { color: var(--muted); margin: 6px 0 0; } +nav { display: flex; gap: 16px; flex-wrap: wrap; } +nav a { color: var(--accent); text-decoration: none; } + +.hero { display: grid; grid-template-columns: repeat(auto-fit, minmax(280px, 1fr)); gap: 20px; } +.card { + background: rgba(17, 24, 39, 0.92); + border: 1px solid var(--line); + border-radius: 16px; + padding: 20px; +} + +.form-card { max-width: 760px; } +.stack { display: grid; gap: 14px; } +label { display: grid; gap: 8px; color: var(--muted); } +input, textarea, button { + font: inherit; + border-radius: 10px; + border: 1px solid var(--line); + padding: 12px 14px; +} +input, textarea { background: var(--panel-2); color: var(--text); } +button { + background: var(--accent); + color: #082f49; + font-weight: 700; + cursor: pointer; +} + + +.muted { color: var(--muted); } +.stat-grid { display: grid; grid-template-columns: repeat(2, minmax(0, 1fr)); gap: 14px; margin: 12px 0 16px; } +.stat-grid div { background: var(--panel-2); border: 1px solid var(--line); border-radius: 12px; padding: 12px; display: grid; gap: 8px; } +.stat-grid span { font-size: 1.5rem; font-weight: 700; color: var(--text); } +.three-up { display: grid; grid-template-columns: repeat(auto-fit, minmax(280px, 1fr)); gap: 20px; margin-top: 20px; } +.list-block { display: grid; gap: 10px; } +.list-item { padding: 10px 12px; background: var(--panel-2); border: 1px solid var(--line); border-radius: 12px; } +.list-item strong { display: block; margin-bottom: 6px; } +.list-item small { color: var(--muted); display: block; } diff --git a/home/doris-schoolhouse/app/static/app.js b/home/doris-schoolhouse/app/static/app.js new file mode 100644 index 0000000..6f1ea44 --- /dev/null +++ b/home/doris-schoolhouse/app/static/app.js @@ -0,0 +1,127 @@ +async function fetchJson(url, options) { + const res = await fetch(url, options); + if (!res.ok) throw new Error(`Fetch failed: ${url}`); + return await res.json(); +} + +function option(value, label) { + const el = document.createElement('option'); + el.value = value; + el.textContent = label; + return el; +} + +function renderList(targetId, items, formatter) { + const target = document.getElementById(targetId); + if (!target) return; + if (!items.length) { + target.textContent = 'Nothing here yet.'; + target.classList.add('muted'); + return; + } + target.classList.remove('muted'); + target.innerHTML = ''; + for (const item of items) { + const wrapper = document.createElement('div'); + wrapper.className = 'list-item'; + wrapper.innerHTML = formatter(item); + target.appendChild(wrapper); + } +} + +async function populateCourses() { + const data = await fetchJson('/api/d2l/courses'); + const courseSelect = document.getElementById('class-select'); + const courseNameInput = document.getElementById('class-name-input'); + const recordingSelect = document.getElementById('recording-class-select'); + const recordingNameInput = document.getElementById('recording-class-name-input'); + const courses = data.items || []; + const byId = new Map(courses.map((course) => [String(course.d2l_course_id || course.id), course])); + + if (courseSelect) { + courseSelect.innerHTML = ''; + courseSelect.appendChild(option('', 'Choose a class')); + for (const course of courses) { + courseSelect.appendChild(option(course.d2l_course_id || course.id, `${course.name}${course.finalSummary ? ' — ' + course.finalSummary : ''}`)); + } + courseSelect.addEventListener('change', async (event) => { + const courseId = event.target.value; + const course = byId.get(String(courseId)); + if (courseNameInput) courseNameInput.value = course?.name || ''; + const assignmentSelect = document.getElementById('assignment-select'); + assignmentSelect.innerHTML = ''; + if (!courseId) { + assignmentSelect.appendChild(option('', 'Pick a class first')); + return; + } + assignmentSelect.appendChild(option('', 'Loading assignments…')); + const assignmentData = await fetchJson(`/api/d2l/assignments?course_id=${encodeURIComponent(courseId)}`); + assignmentSelect.innerHTML = ''; + assignmentSelect.appendChild(option('', 'Choose an assignment')); + for (const assignment of assignmentData.items || []) { + assignmentSelect.appendChild(option(assignment.title, assignment.title)); + } + }); + } + + if (recordingSelect) { + recordingSelect.innerHTML = ''; + recordingSelect.appendChild(option('', 'Choose a class')); + for (const course of courses) { + recordingSelect.appendChild(option(course.d2l_course_id || course.id, course.name)); + } + recordingSelect.addEventListener('change', (event) => { + const course = byId.get(String(event.target.value)); + if (recordingNameInput) recordingNameInput.value = course?.name || ''; + }); + } +} + +async function loadDashboard() { + const data = await fetchJson('/api/dashboard/summary'); + const counts = data.counts || {}; + const setText = (id, value) => { + const el = document.getElementById(id); + if (el) el.textContent = value; + }; + setText('stat-courses', counts.courses ?? '0'); + setText('stat-assignments', counts.assignments ?? '0'); + setText('stat-submissions', counts.submissions ?? '0'); + setText('stat-recordings', counts.recordings ?? '0'); + setText('backend-note', `Storage backend: ${data.backend}`); + if (data.latest_sync) { + setText('sync-status', `Last sync: ${data.latest_sync.completed_at || 'unknown'} · ${data.latest_sync.assignment_count || 0} assignment rows`); + } + renderList('recent-assignments', data.recent_assignments || [], (item) => `${item.title || 'Untitled'}${item.course_name || ''}Status: ${item.status || 'assigned'} · Grade: ${item.grade || '—'}`); + renderList('recent-submissions', data.recent_submissions || [], (item) => `${item.assignment_name || item.original_filename || 'Submission'}${item.class_name || ''}Status: ${item.status || 'unknown'} · File: ${item.original_filename || '—'}`); + renderList('recent-recordings', data.recent_recordings || [], (item) => `${item.original_filename || 'Recording'}${item.session_date || ''}Transcription: ${item.transcription_status || '—'} · Summary: ${item.summary_status || '—'}`); +} + +async function wireSyncButton() { + const button = document.getElementById('sync-button'); + if (!button) return; + button.addEventListener('click', async () => { + button.disabled = true; + button.textContent = 'Refreshing…'; + try { + const result = await fetchJson('/api/d2l/sync', { method: 'POST' }); + const el = document.getElementById('sync-status'); + if (el) el.textContent = `Last sync: just now · ${result.assignment_count} assignment rows · backend ${result.backend}`; + await loadDashboard(); + } catch (error) { + const el = document.getElementById('sync-status'); + if (el) el.textContent = `Sync failed: ${error}`; + } finally { + button.disabled = false; + button.textContent = 'Refresh from D2L'; + } + }); +} + +populateCourses().catch((error) => { + console.error('Failed to populate D2L selectors', error); +}); +loadDashboard().catch((error) => { + console.error('Failed to load dashboard summary', error); +}); +wireSyncButton(); diff --git a/home/doris-schoolhouse/app/templates/assignment_upload.html b/home/doris-schoolhouse/app/templates/assignment_upload.html new file mode 100644 index 0000000..eab9cdd --- /dev/null +++ b/home/doris-schoolhouse/app/templates/assignment_upload.html @@ -0,0 +1,23 @@ +{% extends 'base.html' %} +{% block content %} +
+

Assignment Upload

+
+ + + + + + +
+
+{% endblock %} diff --git a/home/doris-schoolhouse/app/templates/base.html b/home/doris-schoolhouse/app/templates/base.html new file mode 100644 index 0000000..ef2f689 --- /dev/null +++ b/home/doris-schoolhouse/app/templates/base.html @@ -0,0 +1,26 @@ + + + + + + {{ title or 'Doris Schoolhouse' }} + + + +
+
+

Doris Schoolhouse

+

School intake, recordings, D2L sync, and Paperless linkage.

+
+ +
+
+ {% block content %}{% endblock %} +
+ + + diff --git a/home/doris-schoolhouse/app/templates/dashboard.html b/home/doris-schoolhouse/app/templates/dashboard.html new file mode 100644 index 0000000..adb0ca1 --- /dev/null +++ b/home/doris-schoolhouse/app/templates/dashboard.html @@ -0,0 +1,42 @@ +{% extends 'base.html' %} +{% block content %} +
+
+

Overview

+
+
Courses—
+
Assignments—
+
Submissions—
+
Recordings—
+
+

Loading backend…

+ +

No sync run loaded yet.

+
+
+

What this app currently does

+
    +
  • Reads current D2L scrape data
    • +
  • Caches course and assignment rows
    • +
  • Stores assignment upload intake records
    • +
  • Stores recording upload intake records
    • +
  • Builds Paperless/n8n handoff payloads
    • +
+
+
+ +
+
+

Recent assignments

+
Loading…
+
+
+

Recent submissions

+
Loading…
+
+
+

Recent recordings

+
Loading…
+
+
+{% endblock %} diff --git a/home/doris-schoolhouse/app/templates/recording_upload.html b/home/doris-schoolhouse/app/templates/recording_upload.html new file mode 100644 index 0000000..09a8e04 --- /dev/null +++ b/home/doris-schoolhouse/app/templates/recording_upload.html @@ -0,0 +1,19 @@ +{% extends 'base.html' %} +{% block content %} +
+

Recording Upload

+
+ + + + + + +
+
+{% endblock %} diff --git a/home/doris-schoolhouse/bin/apply-postgres-schema.sh b/home/doris-schoolhouse/bin/apply-postgres-schema.sh new file mode 100755 index 0000000..43d5896 --- /dev/null +++ b/home/doris-schoolhouse/bin/apply-postgres-schema.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +set -euo pipefail +cd "$(dirname "$0")/.." +SCHEMA_FILE="app/db/schema.sql" +if [ ! -f "$SCHEMA_FILE" ]; then + echo "schema file missing: $SCHEMA_FILE" >&2 + exit 1 +fi + +if [ -n "${DATABASE_URL:-}" ] && command -v python3 >/dev/null 2>&1; then + export PYTHONPATH=. + python3 app/scripts/apply_schema.py + exit $? +fi + +cat >&2 <<'MSG' +Cannot apply schema automatically. +Requirements: +- DATABASE_URL must be set +- python3 must be available +- psycopg must be installed for direct apply +Alternative: run the SQL in app/db/schema.sql manually against the schoolhouse database. +MSG +exit 1 diff --git a/home/doris-schoolhouse/bin/bootstrap-local.sh b/home/doris-schoolhouse/bin/bootstrap-local.sh new file mode 100755 index 0000000..1cd49db --- /dev/null +++ b/home/doris-schoolhouse/bin/bootstrap-local.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash +set -euo pipefail +cd "$(dirname "$0")/.." +export SCHOOLHOUSE_STORAGE_BACKEND="${SCHOOLHOUSE_STORAGE_BACKEND:-json}" +export SCHOOLHOUSE_STATE_DIR="${SCHOOLHOUSE_STATE_DIR:-/tmp/doris-schoolhouse-state}" +export UPLOAD_TMP_DIR="${UPLOAD_TMP_DIR:-/tmp/doris-schoolhouse-uploads}" +export PYTHONPATH=. +python3 app/scripts/bootstrap_state.py diff --git a/home/doris-schoolhouse/bin/check-env.sh b/home/doris-schoolhouse/bin/check-env.sh new file mode 100755 index 0000000..d11d131 --- /dev/null +++ b/home/doris-schoolhouse/bin/check-env.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +set -euo pipefail + +have() { command -v "$1" >/dev/null 2>&1; } + +printf 'Doris Schoolhouse environment check\n' +printf '================================\n' + +for cmd in python3 ffmpeg psql docker curl; do + if have "$cmd"; then + printf '%-10s %s\n' "$cmd" "$(command -v "$cmd")" + else + printf '%-10s %s\n' "$cmd" "MISSING" + fi +done + +python3 - <<'PY' +import importlib.util +mods = ['fastapi', 'uvicorn', 'psycopg'] +for mod in mods: + print(f'{mod:10} ', 'installed' if importlib.util.find_spec(mod) else 'MISSING') +PY diff --git a/home/doris-schoolhouse/bin/process-recording.sh b/home/doris-schoolhouse/bin/process-recording.sh new file mode 100755 index 0000000..5ef7730 --- /dev/null +++ b/home/doris-schoolhouse/bin/process-recording.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +set -euo pipefail +if [ "$#" -lt 1 ]; then + echo "usage: $0 " >&2 + exit 1 +fi +cd "$(dirname "$0")/.." +export SCHOOLHOUSE_STORAGE_BACKEND="${SCHOOLHOUSE_STORAGE_BACKEND:-json}" +export SCHOOLHOUSE_STATE_DIR="${SCHOOLHOUSE_STATE_DIR:-/tmp/doris-schoolhouse-state}" +export UPLOAD_TMP_DIR="${UPLOAD_TMP_DIR:-/tmp/doris-schoolhouse-uploads}" +export PYTHONPATH=. +python3 app/scripts/process_recording.py "$1" diff --git a/home/doris-schoolhouse/bin/run-local.sh b/home/doris-schoolhouse/bin/run-local.sh new file mode 100755 index 0000000..d235c93 --- /dev/null +++ b/home/doris-schoolhouse/bin/run-local.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash +set -euo pipefail +cd "$(dirname "$0")/.." +export SCHOOLHOUSE_STORAGE_BACKEND="${SCHOOLHOUSE_STORAGE_BACKEND:-json}" +export SCHOOLHOUSE_STATE_DIR="${SCHOOLHOUSE_STATE_DIR:-/tmp/doris-schoolhouse-state}" +export UPLOAD_TMP_DIR="${UPLOAD_TMP_DIR:-/tmp/doris-schoolhouse-uploads}" +export PYTHONPATH=. +uvicorn app.main:app --host 0.0.0.0 --port "${SCHOOLHOUSE_PORT:-8091}" diff --git a/home/doris-schoolhouse/bin/test-assignment-handoff.sh b/home/doris-schoolhouse/bin/test-assignment-handoff.sh new file mode 100755 index 0000000..f918a58 --- /dev/null +++ b/home/doris-schoolhouse/bin/test-assignment-handoff.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash +set -euo pipefail +if [ "$#" -lt 2 ]; then + echo "usage: $0 [assignment_name]" >&2 + exit 1 +fi +FILE="$1" +CLASS_NAME="$2" +ASSIGNMENT_NAME="${3:-Sample Assignment}" +TARGET="${N8N_BASE_URL:-https://n8n.example.com}${SCHOOLHOUSE_N8N_ASSIGNMENT_WEBHOOK:-/webhook/school/intake/upload}" +cat < " >&2 + exit 1 +fi +FILE="$1" +CLASS_NAME="$2" +TARGET="${N8N_BASE_URL:-https://n8n.example.com}${SCHOOLHOUSE_N8N_RECORDING_WEBHOOK:-/webhook/class/upload}" +cat <&2 + exit 1 +fi + +set -a +# shellcheck disable=SC1090 +source "$ENV_FILE" +set +a + +mkdir -p \ + "$APPDATA_PIHOLE_ROOT/etc-pihole" \ + "$APPDATA_PIHOLE_ROOT/etc-dnsmasq.d" \ + "$APPDATA_UNBOUND_ROOT/var" +cp "$STACK_DIR/unbound.conf" "$APPDATA_UNBOUND_ROOT/unbound.conf" +if [[ ! -s "$APPDATA_UNBOUND_ROOT/var/root.hints" ]]; then + curl -fsSL -o "$APPDATA_UNBOUND_ROOT/var/root.hints" https://www.internic.net/domain/named.cache +fi +ENV_FILE="$ENV_FILE" python3 "$STACK_DIR/bin/render_keepalived_conf.py" +docker compose --env-file "$ENV_FILE" -f "$STACK_DIR/docker-compose.yaml" config >/dev/null +if [[ "$DO_UP" == "--up" ]]; then + docker compose --env-file "$ENV_FILE" -f "$STACK_DIR/docker-compose.yaml" up -d + docker compose --env-file "$ENV_FILE" -f "$STACK_DIR/docker-compose.yaml" ps +fi diff --git a/pihole/serenity/bin/render_keepalived_conf.py b/pihole/serenity/bin/render_keepalived_conf.py new file mode 100755 index 0000000..da0b93c --- /dev/null +++ b/pihole/serenity/bin/render_keepalived_conf.py @@ -0,0 +1,73 @@ +#!/usr/bin/env python3 +from __future__ import annotations +import os +import re +import sys +from pathlib import Path + +STACK_DIR = Path(__file__).resolve().parent.parent +ENV_FILE = Path(os.environ.get('ENV_FILE', STACK_DIR / '.env')) +TEMPLATE = STACK_DIR / 'keepalived.conf.template' +OUTPUT = STACK_DIR / 'keepalived.conf' + + +def load_env(path: Path) -> dict[str, str]: + env = dict(os.environ) + if path.exists(): + for raw in path.read_text(encoding='utf-8').splitlines(): + line = raw.strip() + if not line or line.startswith('#') or '=' not in line: + continue + k, v = line.split('=', 1) + env.setdefault(k.strip(), v.strip().strip('"').strip("'")) + return env + + +def main() -> int: + env = load_env(ENV_FILE) + required = [ + 'LAN_INTERFACE', 'VIP_CIDR', 'KEEPALIVED_ROUTER_ID', 'KEEPALIVED_STATE', + 'KEEPALIVED_PRIORITY', 'KEEPALIVED_PASSWORD' + ] + missing = [k for k in required if not env.get(k)] + if missing: + print(f'Missing required vars in {ENV_FILE}: {", ".join(missing)}', file=sys.stderr) + return 1 + + web_port = env.get('PIHOLE_WEB_PORT') or env.get('PD_PIHOLE_WEB_PORT') or env.get('SERENITY_PIHOLE_WEB_PORT') or env.get('RPI4_PIHOLE_WEB_PORT') + if not web_port: + print('Missing Pi-hole web port in .env (set PIHOLE_WEB_PORT or node-specific port variable).', file=sys.stderr) + return 1 + + keepalived_password = env['KEEPALIVED_PASSWORD'] + if len(keepalived_password) > 8: + print('KEEPALIVED_PASSWORD must be 8 characters or fewer for VRRP PASS auth.', file=sys.stderr) + return 1 + + try: + web_port_hex = f"{int(web_port):04X}" + except ValueError: + print('Pi-hole web port must be an integer.', file=sys.stderr) + return 1 + + router_id_name = f"pihole_{re.sub(r'[^a-z0-9]+', '_', env['KEEPALIVED_STATE'].lower())}" + nopreempt_line = ' nopreempt' if env.get('KEEPALIVED_NOPREEMPT', 'true').lower() in {'1','true','yes','on'} else '' + + rendered = TEMPLATE.read_text(encoding='utf-8') + replacements = { + 'ROUTER_ID_NAME': router_id_name, + 'PIHOLE_WEB_PORT_HEX': web_port_hex, + 'NOPREEMPT_LINE': nopreempt_line, + } + replacements.update({k: env[k] for k in required}) + replacements['PIHOLE_WEB_PORT'] = web_port + for key, value in replacements.items(): + rendered = rendered.replace('${' + key + '}', value) + + OUTPUT.write_text(rendered.rstrip() + '\n', encoding='utf-8') + print(f'Rendered {OUTPUT}') + return 0 + + +if __name__ == '__main__': + raise SystemExit(main()) diff --git a/pihole/serenity/docker-compose.yaml b/pihole/serenity/docker-compose.yaml new file mode 100644 index 0000000..2b60792 --- /dev/null +++ b/pihole/serenity/docker-compose.yaml @@ -0,0 +1,59 @@ +services: + unbound: + image: mvance/unbound:latest + container_name: unbound-pihole-serenity + network_mode: host + restart: unless-stopped + environment: + TZ: ${TZ} + UNBOUND_PORT: "5335" + volumes: + - ${APPDATA_UNBOUND_ROOT}:/opt/unbound/etc/unbound/ + healthcheck: + test: ["CMD-SHELL", "hex=$$(printf '%04X' \"$$UNBOUND_PORT\"); grep -qi \":$$hex\" /proc/net/tcp /proc/net/tcp6 || exit 1"] + interval: 30s + timeout: 10s + retries: 6 + start_period: 20s + + pihole: + image: pihole/pihole:latest + container_name: pihole-serenity + network_mode: host + restart: unless-stopped + depends_on: + unbound: + condition: service_healthy + environment: + TZ: ${TZ} + PIHOLE_WEB_PORT: ${SERENITY_PIHOLE_WEB_PORT} + FTLCONF_webserver_api_password: ${PIHOLE_PASSWORD} + FTLCONF_dns_listeningMode: "LOCAL" + FTLCONF_dns_upstreams: "127.0.0.1#5335" + FTLCONF_webserver_port: ${SERENITY_PIHOLE_WEB_PORT} + volumes: + - ${APPDATA_PIHOLE_ROOT}/etc-pihole:/etc/pihole + - ${APPDATA_PIHOLE_ROOT}/etc-dnsmasq.d:/etc/dnsmasq.d + cap_add: + - SYS_NICE + healthcheck: + test: ["CMD-SHELL", "hex=$$(printf '%04X' \"$$PIHOLE_WEB_PORT\"); grep -qi \":$$hex\" /proc/net/tcp /proc/net/tcp6 || exit 1"] + interval: 30s + timeout: 10s + retries: 6 + start_period: 40s + + keepalived: + image: osixia/keepalived:2.0.20 + container_name: keepalived-pihole-serenity + network_mode: host + restart: unless-stopped + depends_on: + pihole: + condition: service_healthy + cap_add: + - NET_ADMIN + - NET_BROADCAST + - NET_RAW + volumes: + - ./keepalived.conf:/usr/local/etc/keepalived/keepalived.conf:ro diff --git a/pihole/serenity/keepalived.conf b/pihole/serenity/keepalived.conf new file mode 100644 index 0000000..2e6049f --- /dev/null +++ b/pihole/serenity/keepalived.conf @@ -0,0 +1,32 @@ +global_defs { + router_id pihole_backup +} + +vrrp_script chk_pihole_web { + script "grep -qi ':22FA' /proc/net/tcp /proc/net/tcp6" + interval 2 + fall 2 + rise 2 +} + +vrrp_instance VI_DNS { + state BACKUP + interface br0 + virtual_router_id 53 + priority 90 + advert_int 1 + nopreempt + + authentication { + auth_type PASS + auth_pass CHANGE01 + } + + virtual_ipaddress { + 10.5.1.53/24 + } + + track_script { + chk_pihole_web + } +} diff --git a/pihole/serenity/keepalived.conf.template b/pihole/serenity/keepalived.conf.template new file mode 100644 index 0000000..a92361f --- /dev/null +++ b/pihole/serenity/keepalived.conf.template @@ -0,0 +1,32 @@ +global_defs { + router_id ${ROUTER_ID_NAME} +} + +vrrp_script chk_pihole_web { + script "grep -qi ':${PIHOLE_WEB_PORT_HEX}' /proc/net/tcp /proc/net/tcp6" + interval 2 + fall 2 + rise 2 +} + +vrrp_instance VI_DNS { + state ${KEEPALIVED_STATE} + interface ${LAN_INTERFACE} + virtual_router_id ${KEEPALIVED_ROUTER_ID} + priority ${KEEPALIVED_PRIORITY} + advert_int 1 +${NOPREEMPT_LINE} + + authentication { + auth_type PASS + auth_pass ${KEEPALIVED_PASSWORD} + } + + virtual_ipaddress { + ${VIP_CIDR} + } + + track_script { + chk_pihole_web + } +} diff --git a/pihole/serenity/unbound.conf b/pihole/serenity/unbound.conf new file mode 100644 index 0000000..05df211 --- /dev/null +++ b/pihole/serenity/unbound.conf @@ -0,0 +1,27 @@ +server: + verbosity: 1 + interface: 127.0.0.1 + port: 5335 + do-ip4: yes + do-udp: yes + do-tcp: yes + do-ip6: no + + auto-trust-anchor-file: "/opt/unbound/etc/unbound/var/root.key" + val-permissive-mode: no + + num-threads: 2 + cache-min-ttl: 300 + cache-max-ttl: 86400 + prefetch: yes + prefetch-key: yes + minimal-responses: yes + + qname-minimisation: yes + hide-identity: yes + hide-version: yes + + access-control: 127.0.0.1/32 allow + access-control: 0.0.0.0/0 refuse + + root-hints: "/opt/unbound/etc/unbound/var/root.hints"