From e5a100d1a0ad10ff54576e92c6fea98d8d52437c Mon Sep 17 00:00:00 2001 From: Fizzlepoof Date: Sat, 16 May 2026 19:13:10 +0000 Subject: [PATCH] Finish Schoolhouse programming submission flow --- automation/README.md | 12 +- home/doris-schoolhouse/DEPLOYMENT.md | 23 +++ home/doris-schoolhouse/README.md | 21 +- .../app/routes/api_assignments.py | 184 ++++++++++++++---- .../doris-schoolhouse/app/services/bundles.py | 172 ++++++++++++++++ home/doris-schoolhouse/app/services/d2l.py | 11 ++ .../app/services/paperless.py | 138 ++++++++++++- .../app/services/paperless_client.py | 94 +++++++++ .../app/services/repository.py | 17 ++ .../app/services/webhooks.py | 48 ++++- home/doris-schoolhouse/app/static/app.css | 6 + home/doris-schoolhouse/app/static/app.js | 6 +- .../app/templates/assignment_upload.html | 16 +- .../bin/backup-to-serenity.sh | 34 ++++ .../bin/test-assignment-handoff.sh | 45 +++-- 15 files changed, 753 insertions(+), 74 deletions(-) create mode 100644 home/doris-schoolhouse/app/services/bundles.py create mode 100755 home/doris-schoolhouse/bin/backup-to-serenity.sh diff --git a/automation/README.md b/automation/README.md index eefa3f8..5a906eb 100644 --- a/automation/README.md +++ b/automation/README.md @@ -34,13 +34,21 @@ automation/bin/karakeep_api.py /api/health --raw-path --pretty - `bin/pd_backup_sync_to_serenity.sh` — `rsync`s configured backup/config roots to Serenity - `bin/run_pd_backups.sh` — wrapper that runs both steps in order -These scripts are staged repo-side only. They still need: +These scripts are the live PD backup runner model. + +Repo-side deployment prerequisites remain: 1. real values in `.env` 2. a writable dump destination on PD 3. SSH trust / key path for Serenity (`root@10.5.1.5` by default) - recommended known_hosts path on PD: `/home/truenas_admin/.ssh/known_hosts` -4. a real scheduler (cron/systemd timer/n8n) on the live host +4. a real scheduler on the live host + +Current documented live state: + +- the PD → Serenity backup flow is the intended deployed model +- root cron on PD is the preferred scheduler +- first-run verification and the deployed status are tracked in `docs/operations/PD_BACKUP_DEPLOYMENT.md` and `docs/planning/TODO.md` ## PD / TrueNAS deployment recommendation diff --git a/home/doris-schoolhouse/DEPLOYMENT.md b/home/doris-schoolhouse/DEPLOYMENT.md index f23034f..7414ab9 100644 --- a/home/doris-schoolhouse/DEPLOYMENT.md +++ b/home/doris-schoolhouse/DEPLOYMENT.md @@ -66,3 +66,26 @@ Then verify: - assignment intake confirms into n8n/Paperless - recording conversion works inside container - recording downstream workflow returns success +- browser uploads return to `/assignments/upload` with a success notice instead of rendering raw JSON +- programming-class uploads create one `source_bundle` submission and immediately queue the Paperless receipt upload + +## Serenity backup on NOMAD + +If Schoolhouse submission bundles should survive a NOMAD disk problem, back up `/opt/doris-schoolhouse/data` to Serenity. + +Current live helper: + +```bash +/opt/doris-schoolhouse/bin/backup-to-serenity.sh +``` + +Behavior: +- mounts Serenity NFS export `10.5.1.5:/mnt/user/data` at `/mnt/serenity-data` when needed +- mirrors `/opt/doris-schoolhouse/data/` into `/mnt/serenity-data/backups/nomad/doris-schoolhouse/data/` +- writes `last_success_utc.txt` and `source_host.txt` markers beside the mirrored data + +Suggested cron on NOMAD: + +```cron +40 2 * * * /opt/doris-schoolhouse/bin/backup-to-serenity.sh >> /opt/doris-schoolhouse/logs/backup-to-serenity.log 2>&1 +``` diff --git a/home/doris-schoolhouse/README.md b/home/doris-schoolhouse/README.md index 70b0703..b5258a9 100644 --- a/home/doris-schoolhouse/README.md +++ b/home/doris-schoolhouse/README.md @@ -6,7 +6,7 @@ Desktop-first local web app for school intake, recordings, D2L sync, and Paperle This app gives John a fast local UI to: -- upload assignment turn-ins +- upload assignment turn-ins, with programming-class source files bundled into one archived submission artifact - upload class recordings - confirm suggested metadata - track assignment status through grading @@ -83,11 +83,28 @@ Useful helpers now included: ```bash bin/check-env.sh bin/apply-postgres-schema.sh -bin/test-assignment-handoff.sh [assignment_name] +bin/test-assignment-handoff.sh [more_files ...] [assignment_name] bin/test-recording-handoff.sh ``` These are safe prep helpers. The handoff scripts are dry-run previews; they print the exact curl commands rather than firing them automatically. +Assignment intake accepts either a legacy single `file` field or the newer multi-file `files` field. Programming-class uploads are normalized into one source-bundle zip per submission version, with a manifest and source-listing sidecar kept alongside the bundle locally. For Paperless archival, Schoolhouse uploads a generated PDF submission receipt/listing while keeping the real source bundle zip as the local canonical artifact. + +Browser submits now auto-confirm into Paperless before redirecting back to `/assignments/upload`. The success notice reports the resulting Paperless queue state, so a normal web upload should no longer stop at `uploaded-local` unless archival actually failed. + +## Serenity backup + +On NOMAD, the canonical local Schoolhouse artifacts live under `/opt/doris-schoolhouse/data`. The helper `bin/backup-to-serenity.sh` mounts Serenity's NFS export at `/mnt/serenity-data` if needed and mirrors that data tree into: + +`/mnt/serenity-data/backups/nomad/doris-schoolhouse/data` + +Recommended daily cron on NOMAD: + +```cron +40 2 * * * /opt/doris-schoolhouse/bin/backup-to-serenity.sh >> /opt/doris-schoolhouse/logs/backup-to-serenity.log 2>&1 +``` + +This keeps the real source bundles, receipt PDFs, manifests, and other Schoolhouse runtime data copied onto Serenity without depending on broken SSH trust from NOMAD. ## Helper scripts diff --git a/home/doris-schoolhouse/app/routes/api_assignments.py b/home/doris-schoolhouse/app/routes/api_assignments.py index 24c0966..60a137b 100644 --- a/home/doris-schoolhouse/app/routes/api_assignments.py +++ b/home/doris-schoolhouse/app/routes/api_assignments.py @@ -1,9 +1,12 @@ import hashlib from pathlib import Path +from typing import Any -from fastapi import APIRouter, Form, HTTPException, UploadFile +from fastapi import APIRouter, File, Form, HTTPException, Request, UploadFile +from fastapi.responses import RedirectResponse from app.config import settings +from app.services.bundles import SubmissionBundleService from app.services.d2l import D2LService from app.services.paperless import PaperlessService from app.services.repository import get_repository @@ -17,6 +20,74 @@ repo = get_repository() d2l = D2LService() paperless = PaperlessService() webhooks = WebhookService() +bundles = SubmissionBundleService() + + +def _confirm_submission_to_paperless(current: dict) -> dict: + handoff = paperless.assignment_intake_payload(current) + file_path = current.get("stored_path") or current.get("local_file") + if not file_path or not Path(file_path).exists(): + raise HTTPException(status_code=400, detail="Stored upload file is missing.") + fields = { + "class_name": handoff["class_name"], + "assignment_name": handoff["assignment_name"], + "submission_kind": handoff["submission_kind"], + "notes": handoff.get("notes", ""), + "source": handoff.get("source", "doris-schoolhouse"), + } + for optional_key in ("semester", "course_code", "paper_kind", "telegram_chat_id", "telegram_message_id"): + value = handoff.get(optional_key) + if value: + fields[optional_key] = value + artifact_kind = handoff.get("artifact_kind") or "document" + if artifact_kind == "source_bundle": + paperless_response = paperless.upload_source_bundle_receipt(current) + enrichment = paperless.enrich_direct_submission(current, paperless_response) + response = { + "ok": True, + "status_code": 200, + "body": paperless_response, + "mode": "direct-paperless", + "enrichment": enrichment, + } + task_id = paperless_response.get("task_id") or paperless_response.get("id") + body_obj = { + "paperless_document_id": enrichment.get("document_id"), + "paperless_title": enrichment.get("title") or paperless.build_submission_title(current), + "paperless_task_id": task_id, + "paperless_metadata": enrichment.get("metadata_payload"), + } + ok = True + else: + if handoff.get("artifact_kind"): + fields["artifact_kind"] = str(handoff["artifact_kind"]) + if handoff.get("file_count"): + fields["file_count"] = str(handoff["file_count"]) + response = webhooks.post_multipart(handoff["target_url"], fields, "document", file_path) + body = response.get("body", {}) if isinstance(response, dict) else {} + body_obj = body[0] if isinstance(body, list) and body and isinstance(body[0], dict) else (body if isinstance(body, dict) else {}) + ok = bool(response.get("ok", False)) if isinstance(response, dict) else False + updates = { + "status": "paperless-queued" if ok else "paperless-error", + "confirmed_at": store.now(), + "webhook_response": response, + "paperless_document_id": body_obj.get("paperless_document_id") or body_obj.get("document_id"), + "paperless_title": body_obj.get("paperless_title") or body_obj.get("title") or handoff.get("assignment_name"), + "paperless_task_id": body_obj.get("paperless_task_id"), + } + updated = repo.update_submission(str(current.get("id")), updates) or current + grade_note = None + if ok and updated.get("assignment_id"): + grade_note = paperless.append_grade_note_if_available(str(updated.get("assignment_id"))) + return { + "status": "ok" if ok else "error", + "submission": updated, + "submissions": [updated], + "handoff": handoff, + "webhook": [response], + "grade_note": grade_note, + "file_count": handoff.get("file_count") or 1, + } @router.get("") @@ -41,7 +112,9 @@ async def list_submissions(): @router.post("/intake") async def intake_assignment( - file: UploadFile, + request: Request, + files: list[UploadFile] | None = File(None), + file: UploadFile | None = File(None), class_id: str = Form(...), class_name: str = Form(...), assignment_name: str = Form(...), @@ -59,16 +132,43 @@ async def intake_assignment( matching = next((item for item in assignments if item.get("title") == assignment_name), None) if not matching: raise HTTPException(status_code=404, detail="Assignment not found in cached D2L data for selected class.") + uploads = list(files or []) + if file is not None: + uploads.append(file) + if not uploads: + raise HTTPException(status_code=400, detail="At least one upload is required.") upload_root = Path(settings.upload_tmp_dir) upload_root.mkdir(parents=True, exist_ok=True) - safe_name = Path(file.filename or "upload.bin").name - submission_id = store.new_id("submission") - stored_path = upload_root / f"{submission_id}-{safe_name}" - payload = await file.read() - stored_path.write_bytes(payload) - checksum = hashlib.sha256(payload).hexdigest() + course = next((item for item in d2l.get_courses() if str(item.get("d2l_course_id") or item.get("id")) == str(class_id)), None) + duplicate_names: dict[str, int] = {} + prepared_uploads: list[dict[str, Any]] = [] + for index, upload in enumerate(uploads, start=1): + safe_name = Path(upload.filename or f"upload-{index}.bin").name + duplicate_count = duplicate_names.get(safe_name, 0) + duplicate_names[safe_name] = duplicate_count + 1 + if duplicate_count: + stem = Path(safe_name).stem + suffix = Path(safe_name).suffix + safe_name = f"{stem}-{duplicate_count + 1}{suffix}" + payload = await upload.read() + prepared_uploads.append({ + "filename": safe_name, + "payload": payload, + "mime_type": upload.content_type or "application/octet-stream", + }) + + submission_id = store.new_id("submission") + bundle = bundles.build_bundle( + upload_root=upload_root, + submission_id=submission_id, + class_name=class_name, + assignment_name=assignment_name, + version_label=version, + uploads=prepared_uploads, + notes=notes, + ) record = { "id": submission_id, "assignment_id": matching["id"], @@ -77,6 +177,7 @@ async def intake_assignment( "assignment_name": assignment_name, "version_label": version, "submission_kind": submission_kind, + "artifact_kind": bundle["artifact_kind"], "semester": semester or None, "course_code": course_code or (course.get("code") if course else None), "paper_kind": paper_kind or None, @@ -84,17 +185,44 @@ async def intake_assignment( "telegram_chat_id": telegram_chat_id or None, "telegram_message_id": telegram_message_id or None, "notes": notes, - "original_filename": safe_name, - "stored_path": str(stored_path), - "mime_type": file.content_type or "application/octet-stream", - "file_size_bytes": len(payload), - "checksum_sha256": checksum, + "original_filename": bundle["bundle_filename"], + "stored_path": bundle["bundle_path"], + "stored_paths": [item["stored_path"] for item in bundle["file_manifest_json"]], + "bundle_path": bundle["bundle_path"], + "bundle_filename": bundle["bundle_filename"], + "source_listing_path": bundle["source_listing_path"], + "receipt_pdf_path": bundle["receipt_pdf_path"], + "file_manifest_json": bundle["file_manifest_json"], + "mime_type": bundle["bundle_mime_type"], + "file_size_bytes": bundle["bundle_size_bytes"], + "checksum_sha256": bundle["bundle_checksum_sha256"], "status": "uploaded-local", "received_at": store.now(), + "batch_id": None, + "file_count": bundle["file_count"], + "file_index": None, + "combined_source_checksum_sha256": bundle["combined_source_checksum_sha256"], } record["paperless_handoff"] = paperless.assignment_intake_payload(record) created = repo.create_submission(record) - return {"status": "ok", "submission": created, "backend": repo.backend} + accept = request.headers.get("accept", "") + auto_confirmed = None + if "text/html" in accept: + auto_confirmed = _confirm_submission_to_paperless(created) + created = auto_confirmed["submission"] + response_payload = { + "status": "ok", + "submission": created, + "submissions": [created], + "batch_id": None, + "file_count": bundle["file_count"], + "paperless": auto_confirmed, + "backend": repo.backend, + } + if "text/html" in accept: + redirect_url = "/assignments/upload?submitted=" + str(created["id"]) + "&file_count=" + str(bundle["file_count"]) + "&artifact_kind=" + str(created.get("artifact_kind", "document")) + "&paperless_status=" + str(created.get("status", "uploaded-local")) + return RedirectResponse(url=redirect_url, status_code=303) + return response_payload @router.post("/{submission_id}/confirm-paperless") @@ -103,33 +231,7 @@ async def confirm_assignment_paperless(submission_id: str): current = next((item for item in submissions if str(item.get("id")) == str(submission_id)), None) if not current: raise HTTPException(status_code=404, detail="Submission not found.") - handoff = paperless.assignment_intake_payload(current) - file_path = current.get("stored_path") or current.get("local_file") - if not file_path or not Path(file_path).exists(): - raise HTTPException(status_code=400, detail="Stored upload file is missing.") - fields = { - "class_name": handoff["class_name"], - "assignment_name": handoff["assignment_name"], - "submission_kind": handoff["submission_kind"], - "notes": handoff.get("notes", ""), - "source": handoff.get("source", "doris-schoolhouse"), - } - for optional_key in ("semester", "course_code", "paper_kind", "telegram_chat_id", "telegram_message_id"): - value = handoff.get(optional_key) - if value: - fields[optional_key] = value - response = webhooks.post_multipart(handoff["target_url"], fields, "document", file_path) - body = response.get("body", {}) if isinstance(response, dict) else {} - body_obj = body[0] if isinstance(body, list) and body and isinstance(body[0], dict) else (body if isinstance(body, dict) else {}) - ok = bool(response.get("ok", False)) if isinstance(response, dict) else False - updated = repo.update_submission(submission_id, { - "status": "paperless-queued" if ok else "paperless-error", - "confirmed_at": store.now(), - "webhook_response": response, - "paperless_document_id": body_obj.get("paperless_document_id") or body_obj.get("document_id"), - "paperless_title": body_obj.get("paperless_title") or body_obj.get("title") or handoff.get("assignment_name"), - }) - return {"status": "ok" if ok else "error", "submission": updated or current, "handoff": handoff, "webhook": response} + return _confirm_submission_to_paperless(current) @router.post("/{assignment_id}/status") diff --git a/home/doris-schoolhouse/app/services/bundles.py b/home/doris-schoolhouse/app/services/bundles.py new file mode 100644 index 0000000..74a1173 --- /dev/null +++ b/home/doris-schoolhouse/app/services/bundles.py @@ -0,0 +1,172 @@ +import hashlib +import json +from pathlib import Path +from zipfile import ZIP_DEFLATED, ZipFile + + +def slugify(value: str, fallback: str = "item") -> str: + cleaned = "".join(ch.lower() if ch.isalnum() else "-" for ch in (value or fallback)) + parts = [part for part in cleaned.split("-") if part] + return "-".join(parts)[:80] or fallback + + +class SubmissionBundleService: + def _wrap_pdf_text(self, text: str, width: int = 96) -> list[str]: + wrapped: list[str] = [] + for raw_line in text.splitlines(): + line = raw_line.rstrip() + if not line: + wrapped.append("") + continue + while len(line) > width: + wrapped.append(line[:width]) + line = line[width:] + wrapped.append(line) + return wrapped + + def _escape_pdf_text(self, value: str) -> str: + return value.replace("\\", "\\\\").replace("(", "\\(").replace(")", "\\)") + + def _write_text_pdf(self, path: Path, title: str, body: str) -> None: + lines = self._wrap_pdf_text(f"{title}\n\n{body}") + page_height = 792 + top = 756 + line_height = 12 + lines_per_page = 58 + pages = [lines[i:i + lines_per_page] for i in range(0, len(lines), lines_per_page)] or [[""]] + + objects: list[bytes] = [] + page_object_numbers: list[int] = [] + object_number = 4 + for page_lines in pages: + content_lines = ["BT", "/F1 10 Tf", f"72 {top} Td"] + first = True + for line in page_lines: + escaped = self._escape_pdf_text(line) + if first: + content_lines.append(f"({escaped}) Tj") + first = False + else: + content_lines.append(f"0 -{line_height} Td ({escaped}) Tj") + content_lines.append("ET") + content_bytes = "\n".join(content_lines).encode("utf-8") + page_object_numbers.append(object_number) + objects.append( + f"{object_number} 0 obj\n<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 {page_height}] /Contents {object_number + 1} 0 R /Resources << /Font << /F1 3 0 R >> >> >>\nendobj\n".encode("utf-8") + ) + objects.append( + f"{object_number + 1} 0 obj\n<< /Length {len(content_bytes)} >>\nstream\n".encode("utf-8") + content_bytes + b"\nendstream\nendobj\n" + ) + object_number += 2 + + pages_kids = " ".join(f"{num} 0 R" for num in page_object_numbers) + header = b"%PDF-1.4\n" + root_objects = [ + b"1 0 obj\n<< /Type /Catalog /Pages 2 0 R >>\nendobj\n", + f"2 0 obj\n<< /Type /Pages /Kids [{pages_kids}] /Count {len(page_object_numbers)} >>\nendobj\n".encode("utf-8"), + b"3 0 obj\n<< /Type /Font /Subtype /Type1 /BaseFont /Courier >>\nendobj\n", + ] + all_objects = root_objects + objects + offsets = [] + current = len(header) + for obj in all_objects: + offsets.append(current) + current += len(obj) + xref_start = current + xref_lines = [b"xref\n", f"0 {len(all_objects) + 1}\n".encode("utf-8"), b"0000000000 65535 f \n"] + for offset in offsets: + xref_lines.append(f"{offset:010d} 00000 n \n".encode("utf-8")) + trailer = f"trailer\n<< /Size {len(all_objects) + 1} /Root 1 0 R >>\nstartxref\n{xref_start}\n%%EOF\n".encode("utf-8") + pdf_bytes = header + b"".join(all_objects) + b"".join(xref_lines) + trailer + path.write_bytes(pdf_bytes) + + def build_bundle( + self, + *, + upload_root: Path, + submission_id: str, + class_name: str, + assignment_name: str, + version_label: str, + uploads: list[dict], + notes: str = "", + ) -> dict: + bundle_root = upload_root / "submission-bundles" / submission_id + source_root = bundle_root / "source" + source_root.mkdir(parents=True, exist_ok=True) + + manifest_files = [] + combined_hash = hashlib.sha256() + for item in uploads: + source_path = source_root / item["filename"] + source_path.write_bytes(item["payload"]) + combined_hash.update(item["payload"]) + manifest_files.append( + { + "filename": item["filename"], + "stored_path": str(source_path), + "mime_type": item["mime_type"], + "size_bytes": len(item["payload"]), + "checksum_sha256": hashlib.sha256(item["payload"]).hexdigest(), + } + ) + + source_listing_path = bundle_root / f"{slugify(assignment_name, 'assignment')}-{slugify(version_label, 'v1')}-source-listing.txt" + listing_parts = [] + for item in manifest_files: + listing_parts.append(f"===== {item['filename']} =====\n") + listing_parts.append(Path(item["stored_path"]).read_text(encoding="utf-8", errors="replace")) + listing_parts.append("\n\n") + source_listing_path.write_text("".join(listing_parts), encoding="utf-8") + + manifest_path = bundle_root / "manifest.json" + manifest = { + "class_name": class_name, + "assignment_name": assignment_name, + "version_label": version_label, + "notes": notes, + "file_count": len(manifest_files), + "files": manifest_files, + "source_listing_path": str(source_listing_path), + } + manifest_path.write_text(json.dumps(manifest, indent=2, sort_keys=True), encoding="utf-8") + + bundle_name = f"{slugify(class_name, 'class')}-{slugify(assignment_name, 'assignment')}-{slugify(version_label, 'v1')}.zip" + bundle_path = bundle_root / bundle_name + with ZipFile(bundle_path, "w", compression=ZIP_DEFLATED) as archive: + for item in manifest_files: + archive.write(item["stored_path"], arcname=item["filename"]) + archive.write(source_listing_path, arcname=source_listing_path.name) + archive.write(manifest_path, arcname="manifest.json") + + receipt_pdf_path = bundle_root / f"{slugify(assignment_name, 'assignment')}-{slugify(version_label, 'v1')}-submission-receipt.pdf" + receipt_body = "\n".join([ + f"Class: {class_name}", + f"Assignment: {assignment_name}", + f"Version: {version_label}", + f"Bundle file: {bundle_name}", + f"Source file count: {len(manifest_files)}", + "", + "Included files:", + *[f"- {item['filename']} ({item['size_bytes']} bytes)" for item in manifest_files], + "", + "Source listing:", + source_listing_path.read_text(encoding="utf-8", errors="replace"), + ]) + self._write_text_pdf(receipt_pdf_path, f"{class_name} — {assignment_name} — {version_label}", receipt_body) + + return { + "artifact_kind": "source_bundle", + "bundle_path": str(bundle_path), + "bundle_filename": bundle_name, + "bundle_mime_type": "application/zip", + "bundle_size_bytes": bundle_path.stat().st_size, + "bundle_checksum_sha256": hashlib.sha256(bundle_path.read_bytes()).hexdigest(), + "file_manifest_json": manifest_files, + "file_count": len(manifest_files), + "source_root": str(source_root), + "source_listing_path": str(source_listing_path), + "receipt_pdf_path": str(receipt_pdf_path), + "manifest_path": str(manifest_path), + "combined_source_checksum_sha256": combined_hash.hexdigest(), + } diff --git a/home/doris-schoolhouse/app/services/d2l.py b/home/doris-schoolhouse/app/services/d2l.py index 070fb67..d87a7b3 100644 --- a/home/doris-schoolhouse/app/services/d2l.py +++ b/home/doris-schoolhouse/app/services/d2l.py @@ -91,12 +91,23 @@ class D2LService: "status": "completed", }) self.repo.save_sync_runs(sync_runs) + grade_note_results = [] + try: + from app.services.paperless import PaperlessService + paperless = PaperlessService() + for item in assignments: + result = paperless.append_grade_note_if_available(str(item.get("id"))) + if result.get("status") != "skipped": + grade_note_results.append(result) + except Exception as exc: + grade_note_results.append({"status": "error", "reason": str(exc)}) return { "status": "ok", "snapshot_path": snapshot.get("source_path"), "course_count": len(courses), "assignment_count": len(assignments), "backend": self.repo.backend, + "grade_note_results": grade_note_results, } def get_courses(self) -> list[dict[str, Any]]: diff --git a/home/doris-schoolhouse/app/services/paperless.py b/home/doris-schoolhouse/app/services/paperless.py index 67afb8e..a0fe458 100644 --- a/home/doris-schoolhouse/app/services/paperless.py +++ b/home/doris-schoolhouse/app/services/paperless.py @@ -1,4 +1,5 @@ import hashlib +import re from urllib.parse import urljoin from app.config import settings @@ -14,6 +15,10 @@ class PaperlessService: self.client = PaperlessClient() def assignment_intake_payload(self, submission: dict) -> dict: + local_files = submission.get("stored_paths") or [] + if not local_files and submission.get("stored_path"): + local_files = [submission.get("stored_path")] + local_file = submission.get("receipt_pdf_path") or submission.get("bundle_path") or submission.get("stored_path") return { "target_url": urljoin(settings.n8n_base_url, settings.n8n_assignment_webhook), "class_name": submission.get("class_name"), @@ -26,8 +31,14 @@ class PaperlessService: "source": submission.get("source") or "doris-schoolhouse", "telegram_chat_id": submission.get("telegram_chat_id"), "telegram_message_id": submission.get("telegram_message_id"), - "local_file": submission.get("stored_path"), + "local_file": local_file, + "local_files": local_files, "version": submission.get("version_label"), + "batch_id": submission.get("batch_id"), + "file_count": submission.get("file_count"), + "artifact_kind": submission.get("artifact_kind") or "document", + "source_listing_path": submission.get("source_listing_path"), + "receipt_pdf_path": submission.get("receipt_pdf_path"), } def recording_intake_payload(self, recording: dict) -> dict: @@ -40,21 +51,122 @@ class PaperlessService: "source": "doris-schoolhouse", } + def upload_source_bundle_receipt(self, submission: dict) -> dict: + file_path = submission.get("receipt_pdf_path") or submission.get("bundle_path") or submission.get("stored_path") + title = self.build_submission_title(submission) + return self.client.upload_document(file_path, title) + + def _course_short_label(self, submission: dict) -> str: + course_code = str(submission.get("course_code") or "").strip() + match = re.search(r"([A-Z]{3,4})_([A-Z]{3,4})_(\d{4})", course_code) + if match: + return f"{match.group(2)}-{match.group(3)}" + class_name = str(submission.get("class_name") or "").strip() + match = re.search(r"([A-Za-z]+(?:\s+[A-Za-z]+)*)\s+(\d{4})", class_name) + if not match: + return class_name + words = match.group(1).split() + prefix = ''.join(word[0] for word in words[:4]).upper() + return f"{prefix}-{match.group(2)}" + + def _course_tag_name(self, submission: dict) -> str | None: + label = self._course_short_label(submission) + if not label or "-" not in label: + return None + return label.replace("-", " ").lower() + + def build_submission_title(self, submission: dict) -> str: + return " — ".join( + part for part in [ + self._course_short_label(submission) or submission.get("class_name"), + submission.get("assignment_name"), + submission.get("version_label"), + "source bundle" if submission.get("artifact_kind") == "source_bundle" else submission.get("submission_kind"), + ] if part + ) + + def build_direct_metadata_payload(self, submission: dict, current_document: dict | None = None) -> dict: + current_document = current_document or {} + known_tags = { + str(tag.get("name", "")).strip().casefold(): tag + for tag in self.client.list_tags() + } + merged_tags = { + int(tag_id) + for tag_id in (current_document.get("tags") or []) + if str(tag_id).isdigit() + } + for tag_name in ("school", "assignments", "spring-2026"): + tag = known_tags.get(tag_name) + if tag and tag.get("id"): + merged_tags.add(int(tag["id"])) + course_tag_name = self._course_tag_name(submission) + if course_tag_name: + tag = known_tags.get(course_tag_name) + if tag and tag.get("id"): + merged_tags.add(int(tag["id"])) + if submission.get("artifact_kind") == "source_bundle": + for tag_name in ("programming", "java"): + tag = known_tags.get(tag_name) + if tag and tag.get("id"): + merged_tags.add(int(tag["id"])) + payload = { + "title": self.build_submission_title(submission), + "tags": sorted(merged_tags), + } + for doc_type in self.client.list_document_types(): + if str(doc_type.get("name", "")).strip().casefold() == "school" and doc_type.get("id"): + payload["document_type"] = int(doc_type["id"]) + break + for correspondent in self.client.list_correspondents(): + if str(correspondent.get("name", "")).strip().casefold() == "austin peay state university" and correspondent.get("id"): + payload["correspondent"] = int(correspondent["id"]) + break + return payload + + def enrich_direct_submission(self, submission: dict, upload_response: dict | None = None) -> dict: + upload_response = upload_response or {} + task_id = upload_response.get("task_id") or upload_response.get("id") or submission.get("paperless_task_id") + if not task_id: + raise ValueError("Paperless upload did not return a task id") + task = self.client.wait_for_task(str(task_id)) + status = str(task.get("status") or "").upper() + document_id = task.get("related_document") + if status != "SUCCESS": + duplicate_match = re.search(r"#(\d+)\)", str(task.get("result") or "")) + if duplicate_match: + document_id = duplicate_match.group(1) + else: + raise ValueError(f"Paperless task did not complete successfully: {status or 'unknown'}") + if not document_id: + raise ValueError("Paperless task completed without a related document id") + current = self.client.get_document(document_id) + payload = self.build_direct_metadata_payload(submission, current) + updated = self.client.update_document(document_id, payload) + return { + "task": task, + "document": updated, + "document_id": updated.get("id") or current.get("id") or int(document_id), + "title": updated.get("title") or payload.get("title"), + "metadata_payload": payload, + } + def build_grade_note(self, assignment_id: str) -> str: submissions = self.repo.list_submissions() assignments = {item.get("id"): item for item in self.repo.list_assignments()} assignment = assignments.get(str(assignment_id), {}) linked = [s for s in submissions if str(s.get("assignment_id")) == str(assignment_id)] note_lines = [ - "Grade sync from Doris Schoolhouse", + "D2L grade note imported from Doris Schoolhouse.", f"Assignment: {assignment.get('title', 'unknown')}", ] if assignment.get("grade"): note_lines.append(f"Grade: {assignment['grade']}") - if assignment.get("points"): - note_lines.append(f"Points: {assignment['points']}") - if assignment.get("comments"): - note_lines.append(f"Professor comments: {assignment['comments']}") + comments = str(assignment.get("comments") or "").strip() + if comments: + note_lines.append(f"Comments: {comments}") + else: + note_lines.append("Comments: No professor comment captured in the D2L scrape.") if linked: note_lines.append(f"Linked submissions: {len(linked)}") return "\n".join(note_lines) @@ -107,3 +219,17 @@ class PaperlessService: "note_hash": note_hash, "paperless_response": response, } + + def append_grade_note_if_available(self, assignment_id: str) -> dict: + assignments = {item.get("id"): item for item in self.repo.list_assignments()} + assignment = assignments.get(str(assignment_id), {}) + if not assignment: + return {"status": "skipped", "reason": "assignment-not-found", "assignment_id": assignment_id} + if not (assignment.get("grade") or str(assignment.get("comments") or "").strip()): + return {"status": "skipped", "reason": "no-grade-or-comments", "assignment_id": assignment_id} + linked = [s for s in self.repo.list_submissions() if str(s.get("assignment_id")) == str(assignment_id)] + if not linked: + return {"status": "skipped", "reason": "no-linked-submissions", "assignment_id": assignment_id} + if not any(item.get("paperless_document_id") for item in linked): + return {"status": "skipped", "reason": "no-paperless-document-id", "assignment_id": assignment_id} + return self.append_grade_note(assignment_id) diff --git a/home/doris-schoolhouse/app/services/paperless_client.py b/home/doris-schoolhouse/app/services/paperless_client.py index 5fca78e..356770e 100644 --- a/home/doris-schoolhouse/app/services/paperless_client.py +++ b/home/doris-schoolhouse/app/services/paperless_client.py @@ -1,4 +1,7 @@ import json +import mimetypes +import time +from pathlib import Path from urllib import request from urllib.parse import urljoin @@ -27,5 +30,96 @@ class PaperlessClient: def get_document(self, document_id: int | str) -> dict: return self._req('GET', f'/api/documents/{document_id}/') + def update_document(self, document_id: int | str, payload: dict) -> dict: + return self._req('PATCH', f'/api/documents/{document_id}/', payload) + def add_note(self, document_id: int | str, note: str) -> dict: return self._req('POST', f'/api/documents/{document_id}/notes/', {'note': note}) + + def _list_endpoint(self, path: str) -> list[dict]: + data = self._req('GET', path) + if isinstance(data, list): + return data + if isinstance(data, dict): + results = data.get('results') + if isinstance(results, list): + return results + return [] + + def list_tags(self) -> list[dict]: + return self._list_endpoint('/api/tags/?page_size=200') + + def create_tag(self, name: str) -> dict: + return self._req('POST', '/api/tags/', {'name': name}) + + def get_or_create_tag(self, name: str) -> dict: + needle = name.strip().casefold() + for tag in self.list_tags(): + if str(tag.get('name', '')).strip().casefold() == needle: + return tag + return self.create_tag(name) + + def list_document_types(self) -> list[dict]: + return self._list_endpoint('/api/document_types/?page_size=200') + + def list_correspondents(self) -> list[dict]: + return self._list_endpoint('/api/correspondents/?page_size=200') + + def list_tasks(self, task_id: str | None = None) -> list[dict]: + path = '/api/tasks/?page_size=100' + if task_id: + path += f'&task_id={task_id}' + return self._list_endpoint(path) + + def wait_for_task(self, task_id: str, timeout_seconds: int = 90, poll_seconds: float = 1.0) -> dict: + deadline = time.time() + timeout_seconds + last = {} + while time.time() < deadline: + tasks = self.list_tasks(task_id=task_id) + if tasks: + last = tasks[0] + status = str(last.get('status') or '').upper() + if status in {'SUCCESS', 'FAILURE', 'CANCELLED'}: + return last + time.sleep(poll_seconds) + return last + + def upload_document(self, file_path: str, title: str) -> dict: + path = Path(file_path) + boundary = "----DorisSchoolhousePaperlessUpload" + mime = mimetypes.guess_type(path.name)[0] or "application/octet-stream" + parts: list[bytes] = [] + for key, value in {"title": title}.items(): + parts.extend([ + f"--{boundary}\r\n".encode(), + f'Content-Disposition: form-data; name="{key}"\r\n\r\n'.encode(), + str(value).encode(), + b"\r\n", + ]) + parts.extend([ + f"--{boundary}\r\n".encode(), + f'Content-Disposition: form-data; name="document"; filename="{path.name}"\r\n'.encode(), + f"Content-Type: {mime}\r\n\r\n".encode(), + path.read_bytes(), + b"\r\n", + f"--{boundary}--\r\n".encode(), + ]) + body = b"".join(parts) + req = request.Request( + urljoin(self.base_url, "/api/documents/post_document/"), + data=body, + headers={ + "Authorization": f"Token {self.token}", + "Accept": "application/json", + "Content-Type": f"multipart/form-data; boundary={boundary}", + }, + method="POST", + ) + with request.urlopen(req, timeout=120) as resp: + raw = resp.read().decode("utf-8", errors="replace") + if not raw: + return {} + parsed = json.loads(raw) + if isinstance(parsed, str): + return {"task_id": parsed} + return parsed diff --git a/home/doris-schoolhouse/app/services/repository.py b/home/doris-schoolhouse/app/services/repository.py index 8efbcca..66edded 100644 --- a/home/doris-schoolhouse/app/services/repository.py +++ b/home/doris-schoolhouse/app/services/repository.py @@ -227,6 +227,23 @@ class PostgresRepository: s.paperless_note_last_hash, s.submitted_at, s.confirmed_metadata_json, + s.confirmed_metadata_json->>'batch_id' AS batch_id, + s.confirmed_metadata_json->>'file_count' AS file_count, + s.confirmed_metadata_json->>'file_index' AS file_index, + s.confirmed_metadata_json->>'submission_kind' AS submission_kind, + s.confirmed_metadata_json->>'artifact_kind' AS artifact_kind, + s.confirmed_metadata_json->>'course_code' AS course_code, + s.confirmed_metadata_json->>'paper_kind' AS paper_kind, + s.confirmed_metadata_json->>'source' AS source, + s.confirmed_metadata_json->>'notes' AS notes, + s.confirmed_metadata_json->>'received_at' AS received_at, + s.confirmed_metadata_json->>'bundle_path' AS bundle_path, + s.confirmed_metadata_json->>'bundle_filename' AS bundle_filename, + s.confirmed_metadata_json->>'source_listing_path' AS source_listing_path, + s.confirmed_metadata_json->>'receipt_pdf_path' AS receipt_pdf_path, + s.confirmed_metadata_json->>'paperless_task_id' AS paperless_task_id, + s.confirmed_metadata_json->'file_manifest_json' AS file_manifest_json, + s.confirmed_metadata_json->'stored_paths' AS stored_paths, s.confirmed_metadata_json->>'stored_path' AS stored_path, s.confirmed_metadata_json->>'local_file' AS local_file FROM schoolhouse_submission s diff --git a/home/doris-schoolhouse/app/services/webhooks.py b/home/doris-schoolhouse/app/services/webhooks.py index b36cad5..cccf81d 100644 --- a/home/doris-schoolhouse/app/services/webhooks.py +++ b/home/doris-schoolhouse/app/services/webhooks.py @@ -6,8 +6,7 @@ from urllib import error, request class WebhookService: - def _multipart_body(self, fields: dict[str, str], file_field: str, file_path: str) -> tuple[bytes, str]: - boundary = f"----DorisSchoolhouse{uuid.uuid4().hex}" + def _multipart_parts_for_fields(self, boundary: str, fields: dict[str, str]) -> list[bytes]: parts: list[bytes] = [] for key, value in fields.items(): parts.extend([ @@ -16,16 +15,32 @@ class WebhookService: str(value).encode(), b"\r\n", ]) + return parts + + def _multipart_parts_for_file(self, boundary: str, file_field: str, file_path: str) -> list[bytes]: path = Path(file_path) mime = mimetypes.guess_type(path.name)[0] or "application/octet-stream" - parts.extend([ + return [ f"--{boundary}\r\n".encode(), f'Content-Disposition: form-data; name="{file_field}"; filename="{path.name}"\r\n'.encode(), f"Content-Type: {mime}\r\n\r\n".encode(), path.read_bytes(), b"\r\n", - f"--{boundary}--\r\n".encode(), - ]) + ] + + def _multipart_body(self, fields: dict[str, str], file_field: str, file_path: str) -> tuple[bytes, str]: + boundary = f"----DorisSchoolhouse{uuid.uuid4().hex}" + parts = self._multipart_parts_for_fields(boundary, fields) + parts.extend(self._multipart_parts_for_file(boundary, file_field, file_path)) + parts.append(f"--{boundary}--\r\n".encode()) + return b"".join(parts), boundary + + def _multipart_body_many(self, fields: dict[str, str], file_field: str, file_paths: list[str]) -> tuple[bytes, str]: + boundary = f"----DorisSchoolhouse{uuid.uuid4().hex}" + parts = self._multipart_parts_for_fields(boundary, fields) + for file_path in file_paths: + parts.extend(self._multipart_parts_for_file(boundary, file_field, file_path)) + parts.append(f"--{boundary}--\r\n".encode()) return b"".join(parts), boundary def post_multipart(self, url: str, fields: dict[str, str], file_field: str, file_path: str) -> dict: @@ -50,3 +65,26 @@ class WebhookService: except error.HTTPError as exc: raw = exc.read().decode("utf-8", errors="replace") return {"ok": False, "status_code": exc.code, "body": parse_raw(raw), "error": str(exc)} + + def post_multipart_many(self, url: str, fields: dict[str, str], file_field: str, file_paths: list[str]) -> dict: + body, boundary = self._multipart_body_many(fields, file_field, file_paths) + req = request.Request( + url, + data=body, + headers={"Content-Type": f"multipart/form-data; boundary={boundary}"}, + method="POST", + ) + + def parse_raw(raw: str): + try: + return json.loads(raw) + except Exception: + return {"raw": raw} + + try: + with request.urlopen(req, timeout=120) as response: + raw = response.read().decode("utf-8", errors="replace") + return {"ok": True, "status_code": response.status, "body": parse_raw(raw)} + except error.HTTPError as exc: + raw = exc.read().decode("utf-8", errors="replace") + return {"ok": False, "status_code": exc.code, "body": parse_raw(raw), "error": str(exc)} diff --git a/home/doris-schoolhouse/app/static/app.css b/home/doris-schoolhouse/app/static/app.css index ea361bb..7549217 100644 --- a/home/doris-schoolhouse/app/static/app.css +++ b/home/doris-schoolhouse/app/static/app.css @@ -31,6 +31,12 @@ nav a { color: var(--accent); text-decoration: none; } padding: 20px; } +.notice-card { margin-bottom: 20px; } +.success-card { + border-color: #14532d; + background: linear-gradient(180deg, rgba(20, 83, 45, 0.28), rgba(17, 24, 39, 0.92)); +} + .form-card { max-width: 760px; } .stack { display: grid; gap: 14px; } label { display: grid; gap: 8px; color: var(--muted); } diff --git a/home/doris-schoolhouse/app/static/app.js b/home/doris-schoolhouse/app/static/app.js index 6f1ea44..999445e 100644 --- a/home/doris-schoolhouse/app/static/app.js +++ b/home/doris-schoolhouse/app/static/app.js @@ -93,7 +93,11 @@ async function loadDashboard() { setText('sync-status', `Last sync: ${data.latest_sync.completed_at || 'unknown'} · ${data.latest_sync.assignment_count || 0} assignment rows`); } renderList('recent-assignments', data.recent_assignments || [], (item) => `${item.title || 'Untitled'}${item.course_name || ''}Status: ${item.status || 'assigned'} · Grade: ${item.grade || '—'}`); - renderList('recent-submissions', data.recent_submissions || [], (item) => `${item.assignment_name || item.original_filename || 'Submission'}${item.class_name || ''}Status: ${item.status || 'unknown'} · File: ${item.original_filename || '—'}`); + renderList('recent-submissions', data.recent_submissions || [], (item) => { + const count = Number(item.file_count || (item.file_manifest_json || []).length || 1); + const artifact = item.artifact_kind === 'source_bundle' ? 'Source bundle' : 'File'; + return `${item.assignment_name || item.original_filename || 'Submission'}${item.class_name || ''}Status: ${item.status || 'unknown'} · ${artifact}: ${item.original_filename || '—'} · ${count} file${count === 1 ? '' : 's'}`; + }); renderList('recent-recordings', data.recent_recordings || [], (item) => `${item.original_filename || 'Recording'}${item.session_date || ''}Transcription: ${item.transcription_status || '—'} · Summary: ${item.summary_status || '—'}`); } diff --git a/home/doris-schoolhouse/app/templates/assignment_upload.html b/home/doris-schoolhouse/app/templates/assignment_upload.html index eab9cdd..1896e31 100644 --- a/home/doris-schoolhouse/app/templates/assignment_upload.html +++ b/home/doris-schoolhouse/app/templates/assignment_upload.html @@ -1,5 +1,13 @@ {% extends 'base.html' %} {% block content %} +{% if request.query_params.get('submitted') %} +
+

Submission Created

+

Submission {{ request.query_params.get('submitted') }} was created successfully.

+

Artifact: {{ request.query_params.get('artifact_kind', 'document') }} · Files: {{ request.query_params.get('file_count', '1') }}

+

Paperless status: {{ request.query_params.get('paperless_status', 'uploaded-local') }}

+
+{% endif %}

Assignment Upload

@@ -15,9 +23,13 @@ +

Programming-class uploads are bundled into one zip artifact for archival. The source files stay grouped together as one submission.

- - + +
{% endblock %} diff --git a/home/doris-schoolhouse/bin/backup-to-serenity.sh b/home/doris-schoolhouse/bin/backup-to-serenity.sh new file mode 100755 index 0000000..f6f68ce --- /dev/null +++ b/home/doris-schoolhouse/bin/backup-to-serenity.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +set -euo pipefail + +SERENITY_NFS_EXPORT="${SERENITY_NFS_EXPORT:-10.5.1.5:/mnt/user/data}" +SERENITY_MOUNT_POINT="${SERENITY_MOUNT_POINT:-/mnt/serenity-data}" +SCHOOLHOUSE_DATA_ROOT="${SCHOOLHOUSE_DATA_ROOT:-/opt/doris-schoolhouse/data}" +SERENITY_BACKUP_SUBDIR="${SERENITY_BACKUP_SUBDIR:-backups/nomad/doris-schoolhouse}" +RSYNC_BIN="${RSYNC_BIN:-/usr/bin/rsync}" +SUDO_BIN="${SUDO_BIN:-/usr/bin/sudo}" +MOUNT_BIN="${MOUNT_BIN:-/usr/bin/mount}" +DATE_BIN="${DATE_BIN:-/usr/bin/date}" +HOSTNAME_BIN="${HOSTNAME_BIN:-/usr/bin/hostname}" + +if [[ ! -d "$SCHOOLHOUSE_DATA_ROOT" ]]; then + echo "Schoolhouse data root missing: $SCHOOLHOUSE_DATA_ROOT" >&2 + exit 1 +fi + +"$SUDO_BIN" mkdir -p "$SERENITY_MOUNT_POINT" +if ! mountpoint -q "$SERENITY_MOUNT_POINT"; then + "$SUDO_BIN" "$MOUNT_BIN" -t nfs -o rw,nfsvers=4 "$SERENITY_NFS_EXPORT" "$SERENITY_MOUNT_POINT" +fi + +dest_root="$SERENITY_MOUNT_POINT/$SERENITY_BACKUP_SUBDIR" +mkdir -p "$dest_root/data" + +"$RSYNC_BIN" -rlt --delete --omit-dir-times --no-perms --no-owner --no-group "$SCHOOLHOUSE_DATA_ROOT/" "$dest_root/data/" + +stamp="$("$DATE_BIN" -u +%Y-%m-%dT%H:%M:%SZ)" +host_name="$("$HOSTNAME_BIN")" +printf '%s\n' "$stamp" > "$dest_root/last_success_utc.txt" +printf '%s\n' "$host_name" > "$dest_root/source_host.txt" + +echo "Schoolhouse backup synced to $dest_root at $stamp" diff --git a/home/doris-schoolhouse/bin/test-assignment-handoff.sh b/home/doris-schoolhouse/bin/test-assignment-handoff.sh index f918a58..0b10e66 100755 --- a/home/doris-schoolhouse/bin/test-assignment-handoff.sh +++ b/home/doris-schoolhouse/bin/test-assignment-handoff.sh @@ -1,21 +1,36 @@ #!/usr/bin/env bash set -euo pipefail if [ "$#" -lt 2 ]; then - echo "usage: $0 [assignment_name]" >&2 + echo "usage: $0 [more_files ...] [assignment_name]" >&2 exit 1 fi -FILE="$1" -CLASS_NAME="$2" -ASSIGNMENT_NAME="${3:-Sample Assignment}" + +ARGS=("$@") +COUNT=$# +if [ "$COUNT" -eq 2 ]; then + FILES=("${ARGS[0]}") + CLASS_NAME="${ARGS[1]}" + ASSIGNMENT_NAME="Sample Assignment" +elif [ "$COUNT" -eq 3 ]; then + FILES=("${ARGS[0]}") + CLASS_NAME="${ARGS[1]}" + ASSIGNMENT_NAME="${ARGS[2]}" +else + FILES=("${ARGS[@]:0:$COUNT-2}") + CLASS_NAME="${ARGS[$COUNT-2]}" + ASSIGNMENT_NAME="${ARGS[$COUNT-1]}" +fi + TARGET="${N8N_BASE_URL:-https://n8n.example.com}${SCHOOLHOUSE_N8N_ASSIGNMENT_WEBHOOK:-/webhook/school/intake/upload}" -cat <