Add Doris Schoolhouse and clean pending homelab changes

home/doris-schoolhouse/app/services/d2l.py

@@ -0,0 +1,139 @@
+import json
+from pathlib import Path
+from typing import Any
+
+from app.config import settings
+from app.services.repository import get_repository
+from app.services.store import JsonStore
+
+
+class D2LService:
+    def __init__(self) -> None:
+        self.store = JsonStore()
+        self.repo = get_repository()
+
+    def _snapshot_path(self) -> Path:
+        candidate_dirs = [
+            Path(settings.d2l_scraper_workdir),
+            Path('/home/fizzlepoof/.openclaw/workspace/school'),
+        ]
+        for workdir in candidate_dirs:
+            for name in ('d2l_current_snapshot.json', 'd2l_grades.json', 'd2l_scraper_output.json'):
+                candidate = workdir / name
+                if candidate.exists():
+                    return candidate
+        return candidate_dirs[0] / 'd2l_current_snapshot.json'
+
+    def load_snapshot(self) -> dict[str, Any]:
+        path = self._snapshot_path()
+        if not path.exists():
+            return {"courses": [], "source_path": str(path), "missing": True}
+        data = json.loads(path.read_text())
+        if isinstance(data, dict):
+            data["source_path"] = str(path)
+            return data
+        return {"courses": [], "source_path": str(path), "missing": True}
+
+    def _normalize_courses(self, snapshot: dict[str, Any]) -> list[dict[str, Any]]:
+        items = []
+        for course in snapshot.get("courses", []):
+            items.append({
+                "id": str(course.get("id")),
+                "d2l_course_id": str(course.get("id")),
+                "name": course.get("name"),
+                "code": course.get("code"),
+                "finalSummary": course.get("finalSummary"),
+                "gradeItemCount": course.get("gradeItemCount"),
+                "synced_at": self.store.now(),
+            })
+        return items
+
+    def _normalize_assignments(self, snapshot: dict[str, Any]) -> list[dict[str, Any]]:
+        items = []
+        for course in snapshot.get("courses", []):
+            course_id = str(course.get("id"))
+            for grade_item in course.get("gradeItems", []):
+                if grade_item.get("isCategory"):
+                    continue
+                title = grade_item.get("name") or "Untitled"
+                items.append({
+                    "id": f"{course_id}:{title.lower().strip()}",
+                    "course_id": course_id,
+                    "course_name": course.get("name"),
+                    "title": title,
+                    "points": grade_item.get("points"),
+                    "grade": grade_item.get("grade"),
+                    "comments": grade_item.get("comments"),
+                    "source": "d2l-grade-item",
+                    "synced_at": self.store.now(),
+                })
+        return items
+
+    def sync_to_store(self) -> dict[str, Any]:
+        snapshot = self.load_snapshot()
+        courses = self._normalize_courses(snapshot)
+        assignments = self._normalize_assignments(snapshot)
+        existing_assignments = {item.get('id'): item for item in self.repo.list_assignments()}
+        for item in assignments:
+            prior = existing_assignments.get(item['id']) or {}
+            item['status'] = prior.get('status', 'assigned')
+            item['updated_at'] = self.store.now()
+            item['created_at'] = prior.get('created_at', self.store.now())
+        self.repo.replace_courses(courses)
+        self.repo.replace_assignments(assignments)
+        sync_runs = self.repo.list_sync_runs()
+        sync_runs.append({
+            "id": self.store.new_id("sync"),
+            "snapshot_path": snapshot.get("source_path"),
+            "course_count": len(courses),
+            "assignment_count": len(assignments),
+            "completed_at": self.store.now(),
+            "status": "completed",
+        })
+        self.repo.save_sync_runs(sync_runs)
+        return {
+            "status": "ok",
+            "snapshot_path": snapshot.get("source_path"),
+            "course_count": len(courses),
+            "assignment_count": len(assignments),
+            "backend": self.repo.backend,
+        }
+
+    def get_courses(self) -> list[dict[str, Any]]:
+        items = self.repo.list_courses()
+        if items:
+            return items
+        self.sync_to_store()
+        return self.repo.list_courses()
+
+    def get_assignments(self, course_id: str) -> list[dict[str, Any]]:
+        items = self.repo.list_assignments()
+        if not items:
+            self.sync_to_store()
+            items = self.repo.list_assignments()
+        return [item for item in items if str(item.get("course_id")) == str(course_id)]
+
+    def get_grades(self, course_id: str) -> dict[str, Any]:
+        snapshot = self.load_snapshot()
+        for course in snapshot.get("courses", []):
+            if str(course.get("id")) == str(course_id):
+                return {
+                    "course_id": str(course.get("id")),
+                    "course_name": course.get("name"),
+                    "final": course.get("final"),
+                    "finalSummary": course.get("finalSummary"),
+                    "gradeItems": course.get("gradeItems", []),
+                }
+        return {"course_id": course_id, "gradeItems": []}
+
+    def describe_sync_plan(self) -> dict:
+        snapshot = self.load_snapshot()
+        return {
+            "status": "stub",
+            "backend": self.repo.backend,
+            "scraper_workdir": settings.d2l_scraper_workdir,
+            "scraper_command": settings.d2l_scraper_command,
+            "snapshot_path": snapshot.get("source_path"),
+            "course_count": len(snapshot.get("courses", [])),
+            "message": "Run existing D2L scraper, normalize courses/assignments/grades, then store into Schoolhouse persistence.",
+        }

home/doris-schoolhouse/app/services/diagnostics.py

@@ -0,0 +1,23 @@
+from pathlib import Path
+
+from app.config import settings
+from app.services.media import MediaService
+from app.services.repository import get_repository
+
+
+class DiagnosticsService:
+    def collect(self) -> dict:
+        repo = get_repository()
+        media = MediaService()
+        return {
+            'backend': repo.backend,
+            'state_dir': settings.state_dir,
+            'state_dir_exists': Path(settings.state_dir).exists(),
+            'upload_tmp_dir': settings.upload_tmp_dir,
+            'upload_tmp_dir_exists': Path(settings.upload_tmp_dir).exists(),
+            'ffmpeg_path': media.ffmpeg_path(),
+            'can_convert_audio': media.can_convert(),
+            'n8n_base_url': settings.n8n_base_url,
+            'paperless_base_url': settings.paperless_base_url,
+            'd2l_scraper_workdir': settings.d2l_scraper_workdir,
+        }

home/doris-schoolhouse/app/services/media.py

@@ -0,0 +1,57 @@
+import shutil
+import subprocess
+from pathlib import Path
+
+
+class MediaService:
+    def planned_mp3_path(self, wav_path: str) -> str:
+        path = Path(wav_path)
+        return str(path.with_suffix('.mp3'))
+
+    def conversion_command(self, wav_path: str, mp3_path: str) -> list[str]:
+        return [
+            'ffmpeg',
+            '-y',
+            '-i', wav_path,
+            '-codec:a', 'libmp3lame',
+            '-q:a', '2',
+            mp3_path,
+        ]
+
+    def ffmpeg_path(self) -> str | None:
+        return shutil.which('ffmpeg')
+
+    def can_convert(self) -> bool:
+        return self.ffmpeg_path() is not None
+
+    def convert_wav_to_mp3(self, wav_path: str, mp3_path: str) -> dict:
+        if not self.can_convert():
+            return {
+                'status': 'blocked',
+                'reason': 'ffmpeg-not-installed',
+                'command': self.conversion_command(wav_path, mp3_path),
+            }
+        cmd = self.conversion_command(wav_path, mp3_path)
+        result = subprocess.run(cmd, capture_output=True, text=True)
+        if result.returncode != 0:
+            return {
+                'status': 'failed',
+                'returncode': result.returncode,
+                'stderr': result.stderr[-4000:],
+                'stdout': result.stdout[-2000:],
+                'command': cmd,
+            }
+        return {
+            'status': 'ok',
+            'mp3_path': mp3_path,
+            'command': cmd,
+        }
+
+    def delete_source_if_requested(self, wav_path: str, should_delete: bool) -> dict:
+        if not should_delete:
+            return {'status': 'skipped', 'reason': 'delete-not-requested'}
+        path = Path(wav_path)
+        if not path.exists():
+            return {'status': 'skipped', 'reason': 'source-missing'}
+        path.unlink()
+        return {'status': 'ok', 'deleted': wav_path}

home/doris-schoolhouse/app/services/paperless.py

@@ -0,0 +1,109 @@
+import hashlib
+from urllib.parse import urljoin
+
+from app.config import settings
+from app.services.paperless_client import PaperlessClient
+from app.services.repository import get_repository
+from app.services.store import JsonStore
+
+
+class PaperlessService:
+    def __init__(self) -> None:
+        self.store = JsonStore()
+        self.repo = get_repository()
+        self.client = PaperlessClient()
+
+    def assignment_intake_payload(self, submission: dict) -> dict:
+        return {
+            "target_url": urljoin(settings.n8n_base_url, settings.n8n_assignment_webhook),
+            "class_name": submission.get("class_name"),
+            "assignment_name": submission.get("assignment_name"),
+            "submission_kind": submission.get("submission_kind") or "assignment_turn_in",
+            "semester": submission.get("semester"),
+            "course_code": submission.get("course_code"),
+            "paper_kind": submission.get("paper_kind"),
+            "notes": submission.get("notes", ""),
+            "source": submission.get("source") or "doris-schoolhouse",
+            "telegram_chat_id": submission.get("telegram_chat_id"),
+            "telegram_message_id": submission.get("telegram_message_id"),
+            "local_file": submission.get("stored_path"),
+            "version": submission.get("version_label"),
+        }
+
+    def recording_intake_payload(self, recording: dict) -> dict:
+        return {
+            "target_url": urljoin(settings.n8n_base_url, settings.n8n_recording_webhook),
+            "class_name": recording.get("class_name"),
+            "lecture_title": f"{recording.get('class_name')} {recording.get('session_date')}",
+            "notes": recording.get("notes", ""),
+            "local_file": recording.get("mp3_path") or recording.get("stored_path"),
+            "source": "doris-schoolhouse",
+        }
+
+    def build_grade_note(self, assignment_id: str) -> str:
+        submissions = self.repo.list_submissions()
+        assignments = {item.get("id"): item for item in self.repo.list_assignments()}
+        assignment = assignments.get(str(assignment_id), {})
+        linked = [s for s in submissions if str(s.get("assignment_id")) == str(assignment_id)]
+        note_lines = [
+            "Grade sync from Doris Schoolhouse",
+            f"Assignment: {assignment.get('title', 'unknown')}",
+        ]
+        if assignment.get("grade"):
+            note_lines.append(f"Grade: {assignment['grade']}")
+        if assignment.get("points"):
+            note_lines.append(f"Points: {assignment['points']}")
+        if assignment.get("comments"):
+            note_lines.append(f"Professor comments: {assignment['comments']}")
+        if linked:
+            note_lines.append(f"Linked submissions: {len(linked)}")
+        return "\n".join(note_lines)
+
+    def describe_grade_note_plan(self, assignment_id: str) -> dict:
+        submissions = self.repo.list_submissions()
+        linked = [s for s in submissions if str(s.get("assignment_id")) == str(assignment_id)]
+        return {
+            "status": "stub",
+            "backend": self.repo.backend,
+            "assignment_id": assignment_id,
+            "paperless_base_url": settings.paperless_base_url,
+            "linked_submission_count": len(linked),
+            "note_preview": self.build_grade_note(assignment_id),
+            "message": "Resolve linked Paperless submission document, hash latest grade/comment payload, and append a dated Schoolhouse note if changed.",
+        }
+
+    def append_grade_note(self, assignment_id: str) -> dict:
+        submissions = self.repo.list_submissions()
+        linked = [s for s in submissions if str(s.get("assignment_id")) == str(assignment_id)]
+        if not linked:
+            raise ValueError("No linked submissions for assignment")
+        target = next((s for s in linked if s.get("paperless_document_id")), linked[0])
+        document_id = target.get("paperless_document_id")
+        if not document_id:
+            raise ValueError("Linked submission has no Paperless document id yet")
+        note = self.build_grade_note(assignment_id)
+        note_hash = hashlib.sha256(note.encode('utf-8')).hexdigest()
+        if target.get("paperless_note_last_hash") == note_hash:
+            return {
+                "status": "skipped",
+                "reason": "note-unchanged",
+                "assignment_id": assignment_id,
+                "document_id": document_id,
+                "note_hash": note_hash,
+            }
+        response = self.client.add_note(document_id, note)
+        self.repo.update_submission(
+            str(target.get("id")),
+            {
+                "paperless_note_last_hash": note_hash,
+                "paperless_title": target.get("paperless_title"),
+                "status": target.get("status", "paperless-linked"),
+            },
+        )
+        return {
+            "status": "ok",
+            "assignment_id": assignment_id,
+            "document_id": document_id,
+            "note_hash": note_hash,
+            "paperless_response": response,
+        }

home/doris-schoolhouse/app/services/paperless_client.py

@@ -0,0 +1,31 @@
+import json
+from urllib import request
+from urllib.parse import urljoin
+
+from app.config import settings
+
+
+class PaperlessClient:
+    def __init__(self) -> None:
+        self.base_url = settings.paperless_base_url.rstrip('/') + '/'
+        self.token = settings.paperless_api_token
+
+    def _req(self, method: str, path: str, payload: dict | None = None) -> dict:
+        url = urljoin(self.base_url, path.lstrip('/'))
+        body = None if payload is None else json.dumps(payload).encode('utf-8')
+        headers = {
+            'Authorization': f'Token {self.token}',
+            'Accept': 'application/json',
+        }
+        if body is not None:
+            headers['Content-Type'] = 'application/json'
+        req = request.Request(url, data=body, headers=headers, method=method)
+        with request.urlopen(req, timeout=120) as resp:
+            raw = resp.read().decode('utf-8', errors='replace')
+            return json.loads(raw) if raw else {}
+
+    def get_document(self, document_id: int | str) -> dict:
+        return self._req('GET', f'/api/documents/{document_id}/')
+
+    def add_note(self, document_id: int | str, note: str) -> dict:
+        return self._req('POST', f'/api/documents/{document_id}/notes/', {'note': note})

home/doris-schoolhouse/app/services/repository.py

@@ -0,0 +1,388 @@
+import json
+from pathlib import Path
+from typing import Any
+
+from app.config import settings
+from app.services.store import JsonStore
+
+
+class JsonRepository:
+    backend = "json"
+
+    def __init__(self) -> None:
+        self.store = JsonStore()
+
+    def list_courses(self) -> list[dict[str, Any]]:
+        return self.store.load("courses", [])
+
+    def replace_courses(self, items: list[dict[str, Any]]) -> list[dict[str, Any]]:
+        return self.store.replace_collection("courses", items)
+
+    def list_assignments(self) -> list[dict[str, Any]]:
+        return self.store.load("assignments", [])
+
+    def replace_assignments(self, items: list[dict[str, Any]]) -> list[dict[str, Any]]:
+        return self.store.replace_collection("assignments", items)
+
+    def list_sync_runs(self) -> list[dict[str, Any]]:
+        return self.store.load("sync_runs", [])
+
+    def save_sync_runs(self, items: list[dict[str, Any]]) -> list[dict[str, Any]]:
+        return self.store.save("sync_runs", items)
+
+    def list_submissions(self) -> list[dict[str, Any]]:
+        return self.store.load("submissions", [])
+
+    def create_submission(self, item: dict[str, Any]) -> dict[str, Any]:
+        return self.store.append_collection("submissions", item)
+
+    def update_submission(self, submission_id: str, updates: dict[str, Any]) -> dict[str, Any] | None:
+        return self.store.update_collection_item(
+            "submissions",
+            lambda item: str(item.get("id")) == str(submission_id),
+            lambda item: {**item, **updates},
+        )
+
+    def list_recordings(self) -> list[dict[str, Any]]:
+        return self.store.load("recordings", [])
+
+    def create_recording(self, item: dict[str, Any]) -> dict[str, Any]:
+        return self.store.append_collection("recordings", item)
+
+    def update_recording(self, recording_id: str, updates: dict[str, Any]) -> dict[str, Any] | None:
+        return self.store.update_collection_item(
+            "recordings",
+            lambda item: str(item.get("id")) == str(recording_id),
+            lambda item: {**item, **updates},
+        )
+
+
+class PostgresRepository:
+    backend = "postgres"
+
+    def __init__(self) -> None:
+        import psycopg
+        from psycopg.rows import dict_row
+
+        self._psycopg = psycopg
+        self._dict_row = dict_row
+        self.dsn = settings.database_url.replace("postgresql+psycopg://", "postgresql://")
+
+    def _connect(self):
+        return self._psycopg.connect(self.dsn, row_factory=self._dict_row)
+
+    def _ensure_course(self, cur, item: dict[str, Any]) -> dict[str, Any] | None:
+        course_id = item.get("class_id") or item.get("course_id")
+        if not course_id:
+            return None
+        cur.execute("SELECT id, d2l_course_id, name FROM schoolhouse_course WHERE d2l_course_id = %s", (course_id,))
+        row = cur.fetchone()
+        if row:
+            return row
+        course_name = item.get("class_name") or item.get("course_name") or str(course_id)
+        cur.execute(
+            """
+            INSERT INTO schoolhouse_course (d2l_course_id, name, last_synced_at)
+            VALUES (%s, %s, NOW())
+            RETURNING id, d2l_course_id, name
+            """,
+            (course_id, course_name),
+        )
+        return cur.fetchone()
+
+    def _ensure_assignment(self, cur, item: dict[str, Any]) -> dict[str, Any] | None:
+        course = self._ensure_course(cur, item)
+        if not course:
+            return None
+        assignment_id = item.get("assignment_id")
+        assignment_name = item.get("assignment_name") or item.get("title")
+        if assignment_id:
+            cur.execute(
+                "SELECT id, title FROM schoolhouse_assignment WHERE course_id = %s AND d2l_assignment_key = %s LIMIT 1",
+                (course["id"], assignment_id),
+            )
+            row = cur.fetchone()
+            if row:
+                return row
+        if assignment_name:
+            cur.execute(
+                "SELECT id, title FROM schoolhouse_assignment WHERE course_id = %s AND title = %s LIMIT 1",
+                (course["id"], assignment_name),
+            )
+            row = cur.fetchone()
+            if row:
+                return row
+            cur.execute(
+                """
+                INSERT INTO schoolhouse_assignment (course_id, d2l_assignment_key, title, source, status, created_at, updated_at)
+                VALUES (%s, %s, %s, %s, %s, NOW(), NOW())
+                RETURNING id, title
+                """,
+                (course["id"], assignment_id, assignment_name, item.get("source", "schoolhouse-intake"), item.get("status", "assigned")),
+            )
+            return cur.fetchone()
+        return None
+
+    def list_courses(self) -> list[dict[str, Any]]:
+        with self._connect() as conn, conn.cursor() as cur:
+            cur.execute("SELECT id::text AS id, d2l_course_id, name, code, semester, last_synced_at FROM schoolhouse_course ORDER BY name")
+            return list(cur.fetchall())
+
+    def replace_courses(self, items: list[dict[str, Any]]) -> list[dict[str, Any]]:
+        with self._connect() as conn, conn.cursor() as cur:
+            for item in items:
+                cur.execute(
+                    """
+                    INSERT INTO schoolhouse_course (d2l_course_id, name, code, semester, last_synced_at)
+                    VALUES (%s, %s, %s, %s, NOW())
+                    ON CONFLICT (d2l_course_id)
+                    DO UPDATE SET name = EXCLUDED.name, code = EXCLUDED.code, semester = EXCLUDED.semester, last_synced_at = NOW()
+                    """,
+                    (item.get("d2l_course_id"), item.get("name"), item.get("code"), item.get("semester")),
+                )
+            conn.commit()
+        return items
+
+    def list_assignments(self) -> list[dict[str, Any]]:
+        with self._connect() as conn, conn.cursor() as cur:
+            cur.execute(
+                """
+                SELECT a.id::text AS id,
+                       c.d2l_course_id AS course_id,
+                       c.name AS course_name,
+                       a.title,
+                       a.grade_text AS grade,
+                       a.professor_comments AS comments,
+                       a.status,
+                       a.updated_at
+                  FROM schoolhouse_assignment a
+                  JOIN schoolhouse_course c ON c.id = a.course_id
+                 ORDER BY c.name, a.title
+                """
+            )
+            return list(cur.fetchall())
+
+    def replace_assignments(self, items: list[dict[str, Any]]) -> list[dict[str, Any]]:
+        with self._connect() as conn, conn.cursor() as cur:
+            for item in items:
+                cur.execute("SELECT id FROM schoolhouse_course WHERE d2l_course_id = %s", (item.get("course_id"),))
+                row = cur.fetchone()
+                if not row:
+                    continue
+                course_pk = row["id"]
+                cur.execute(
+                    """
+                    INSERT INTO schoolhouse_assignment (course_id, d2l_assignment_key, title, grade_text, professor_comments, source, status, created_at, updated_at)
+                    VALUES (%s, %s, %s, %s, %s, %s, %s, NOW(), NOW())
+                    ON CONFLICT DO NOTHING
+                    """,
+                    (course_pk, item.get("id"), item.get("title"), item.get("grade"), item.get("comments"), item.get("source", "d2l"), item.get("status", "assigned")),
+                )
+                cur.execute(
+                    """
+                    UPDATE schoolhouse_assignment
+                       SET grade_text = %s,
+                           professor_comments = %s,
+                           status = COALESCE(status, %s),
+                           updated_at = NOW()
+                     WHERE course_id = %s AND title = %s
+                    """,
+                    (item.get("grade"), item.get("comments"), item.get("status", "assigned"), course_pk, item.get("title")),
+                )
+            conn.commit()
+        return items
+
+    def list_sync_runs(self) -> list[dict[str, Any]]:
+        with self._connect() as conn, conn.cursor() as cur:
+            cur.execute("SELECT id::text AS id, started_at, completed_at, status, raw_snapshot_path AS snapshot_path, changes_summary_json FROM schoolhouse_d2l_sync_run ORDER BY started_at DESC")
+            return list(cur.fetchall())
+
+    def save_sync_runs(self, items: list[dict[str, Any]]) -> list[dict[str, Any]]:
+        if not items:
+            return items
+        last = items[-1]
+        with self._connect() as conn, conn.cursor() as cur:
+            cur.execute(
+                "INSERT INTO schoolhouse_d2l_sync_run (completed_at, status, raw_snapshot_path, changes_summary_json) VALUES (NOW(), %s, %s, %s::jsonb)",
+                (last.get("status", "completed"), last.get("snapshot_path"), json.dumps(last)),
+            )
+            conn.commit()
+        return items
+
+    def list_submissions(self) -> list[dict[str, Any]]:
+        with self._connect() as conn, conn.cursor() as cur:
+            cur.execute(
+                """
+                SELECT s.id::text AS id,
+                       a.id::text AS assignment_id,
+                       c.d2l_course_id AS class_id,
+                       c.name AS class_name,
+                       a.title AS assignment_name,
+                       s.version_label,
+                       s.original_filename,
+                       s.mime_type,
+                       s.status,
+                       s.paperless_document_id,
+                       s.paperless_title,
+                       s.paperless_note_last_hash,
+                       s.submitted_at,
+                       s.confirmed_metadata_json,
+                       s.confirmed_metadata_json->>'stored_path' AS stored_path,
+                       s.confirmed_metadata_json->>'local_file' AS local_file
+                  FROM schoolhouse_submission s
+                  JOIN schoolhouse_assignment a ON a.id = s.assignment_id
+                  JOIN schoolhouse_course c ON c.id = a.course_id
+                 ORDER BY s.created_at DESC
+                """
+            )
+            return list(cur.fetchall())
+
+    def create_submission(self, item: dict[str, Any]) -> dict[str, Any]:
+        with self._connect() as conn, conn.cursor() as cur:
+            row = self._ensure_assignment(cur, item)
+            if not row:
+                raise ValueError("Assignment not found and could not be created in Postgres repository")
+            cur.execute(
+                """
+                INSERT INTO schoolhouse_submission (
+                  assignment_id, version_label, original_filename, mime_type, checksum_sha256,
+                  submitted_at, suggested_metadata_json, confirmed_metadata_json, status
+                ) VALUES (%s, %s, %s, %s, %s, NOW(), %s::jsonb, %s::jsonb, %s)
+                RETURNING id::text AS id
+                """,
+                (
+                    row["id"], item.get("version_label") or "v1", item.get("original_filename"), item.get("mime_type", "application/octet-stream"), item.get("checksum_sha256"),
+                    json.dumps(item.get("paperless_handoff", {})), json.dumps(item), item.get("status", "uploaded-local"),
+                ),
+            )
+            created = cur.fetchone()
+            conn.commit()
+            item["id"] = created["id"]
+            return item
+
+    def update_submission(self, submission_id: str, updates: dict[str, Any]) -> dict[str, Any] | None:
+        with self._connect() as conn, conn.cursor() as cur:
+            cur.execute("SELECT confirmed_metadata_json FROM schoolhouse_submission WHERE id = %s::bigint", (submission_id,))
+            current = cur.fetchone()
+            if not current:
+                return None
+            merged_metadata = {**(current.get("confirmed_metadata_json") or {}), **updates}
+            cur.execute(
+                """
+                UPDATE schoolhouse_submission
+                   SET status = COALESCE(%s, status),
+                       paperless_document_id = COALESCE(%s, paperless_document_id),
+                       paperless_title = COALESCE(%s, paperless_title),
+                       paperless_note_last_hash = COALESCE(%s, paperless_note_last_hash),
+                       paperless_note_appended_at = CASE WHEN %s::text IS NOT NULL THEN NOW() ELSE paperless_note_appended_at END,
+                       confirmed_metadata_json = %s::jsonb,
+                       updated_at = NOW()
+                 WHERE id = %s::bigint
+             RETURNING id::text AS id, status, paperless_document_id, paperless_title, paperless_note_last_hash
+                """,
+                (
+                    updates.get("status"), updates.get("paperless_document_id"), updates.get("paperless_title"), updates.get("paperless_note_last_hash"), updates.get("paperless_note_last_hash"),
+                    json.dumps(merged_metadata), submission_id,
+                ),
+            )
+            row = cur.fetchone()
+            conn.commit()
+            return {**merged_metadata, **dict(row)} if row else None
+
+    def list_recordings(self) -> list[dict[str, Any]]:
+        with self._connect() as conn, conn.cursor() as cur:
+            cur.execute(
+                """
+                SELECT r.id::text AS id,
+                       c.d2l_course_id AS class_id,
+                       c.name AS class_name,
+                       r.assignment_id::text AS assignment_id,
+                       r.course_id::text AS course_id,
+                       r.session_date::text AS session_date,
+                       r.original_filename,
+                       r.mp3_filename,
+                       r.transcription_status,
+                       r.summary_status,
+                       r.recording_paperless_document_id,
+                       r.recording_metadata_json,
+                       r.recording_metadata_json->>'stored_path' AS stored_path,
+                       r.recording_metadata_json->>'mp3_path' AS mp3_path,
+                       r.recording_metadata_json->>'status' AS status,
+                       r.recording_metadata_json->>'conversion_status' AS conversion_status,
+                       r.recording_metadata_json->>'notes' AS notes,
+                       r.recording_metadata_json->>'delete_wav_after_success' AS delete_wav_after_success
+                  FROM schoolhouse_recording r
+                  JOIN schoolhouse_course c ON c.id = r.course_id
+                 ORDER BY r.created_at DESC
+                """
+            )
+            return list(cur.fetchall())
+
+    def create_recording(self, item: dict[str, Any]) -> dict[str, Any]:
+        with self._connect() as conn, conn.cursor() as cur:
+            row = self._ensure_course(cur, item)
+            if not row:
+                raise ValueError("Course not found and could not be created in Postgres repository")
+            cur.execute(
+                """
+                INSERT INTO schoolhouse_recording (
+                  course_id, assignment_id, session_date, original_filename, mp3_filename, notes,
+                  transcription_status, summary_status, recording_metadata_json
+                ) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s::jsonb)
+                RETURNING id::text AS id
+                """,
+                (
+                    row["id"], item.get("assignment_id") or None, item.get("session_date"), item.get("original_filename"),
+                    Path(item.get("mp3_path")).name if item.get("mp3_path") else None, item.get("notes"), item.get("transcription_status"),
+                    item.get("summary_status"), json.dumps(item),
+                ),
+            )
+            created = cur.fetchone()
+            conn.commit()
+            item["id"] = created["id"]
+            return item
+
+    def update_recording(self, recording_id: str, updates: dict[str, Any]) -> dict[str, Any] | None:
+        with self._connect() as conn, conn.cursor() as cur:
+            cur.execute("SELECT recording_metadata_json FROM schoolhouse_recording WHERE id = %s::bigint", (recording_id,))
+            current = cur.fetchone()
+            if not current:
+                return None
+            merged_metadata = {**(current.get("recording_metadata_json") or {}), **updates}
+            cur.execute(
+                """
+                UPDATE schoolhouse_recording
+                   SET transcription_status = COALESCE(%s, transcription_status),
+                       summary_status = COALESCE(%s, summary_status),
+                       transcript_paperless_document_id = COALESCE(%s, transcript_paperless_document_id),
+                       summary_paperless_document_id = COALESCE(%s, summary_paperless_document_id),
+                       recording_paperless_document_id = COALESCE(%s, recording_paperless_document_id),
+                       mp3_filename = COALESCE(%s, mp3_filename),
+                       notes = COALESCE(%s, notes),
+                       recording_metadata_json = %s::jsonb,
+                       updated_at = NOW()
+                 WHERE id = %s::bigint
+             RETURNING id::text AS id, transcription_status, summary_status, recording_paperless_document_id
+                """,
+                (
+                    updates.get("transcription_status"), updates.get("summary_status"), updates.get("transcript_paperless_document_id"),
+                    updates.get("summary_paperless_document_id"), updates.get("recording_paperless_document_id"),
+                    Path(updates.get("mp3_path")).name if updates.get("mp3_path") else None,
+                    updates.get("notes"), json.dumps(merged_metadata), recording_id,
+                ),
+            )
+            row = cur.fetchone()
+            conn.commit()
+            return {**merged_metadata, **dict(row)} if row else None
+
+
+def get_repository():
+    backend = getattr(settings, "storage_backend", "auto")
+    if backend == "json":
+        return JsonRepository()
+    if settings.database_url.startswith("postgresql"):
+        try:
+            return PostgresRepository()
+        except Exception:
+            return JsonRepository()
+    return JsonRepository()

home/doris-schoolhouse/app/services/store.py

@@ -0,0 +1,53 @@
+import json
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any
+from uuid import uuid4
+
+from app.config import settings
+
+
+class JsonStore:
+    def __init__(self) -> None:
+        self.root = Path(settings.state_dir)
+        self.root.mkdir(parents=True, exist_ok=True)
+
+    def _path(self, name: str) -> Path:
+        return self.root / f"{name}.json"
+
+    def load(self, name: str, default: Any) -> Any:
+        path = self._path(name)
+        if not path.exists():
+            return default
+        return json.loads(path.read_text())
+
+    def save(self, name: str, value: Any) -> Any:
+        path = self._path(name)
+        path.write_text(json.dumps(value, indent=2, sort_keys=True))
+        return value
+
+    def replace_collection(self, name: str, items: list[dict[str, Any]]) -> list[dict[str, Any]]:
+        return self.save(name, items)
+
+    def append_collection(self, name: str, item: dict[str, Any]) -> dict[str, Any]:
+        items = self.load(name, [])
+        items.append(item)
+        self.save(name, items)
+        return item
+
+    def update_collection_item(self, name: str, predicate, updater):
+        items = self.load(name, [])
+        updated = None
+        for idx, item in enumerate(items):
+            if predicate(item):
+                items[idx] = updater(dict(item))
+                updated = items[idx]
+                break
+        self.save(name, items)
+        return updated
+
+    def now(self) -> str:
+        return datetime.now(timezone.utc).isoformat()
+
+    def new_id(self, prefix: str) -> str:
+        return f"{prefix}-{uuid4().hex[:12]}"

home/doris-schoolhouse/app/services/webhooks.py

@@ -0,0 +1,52 @@
+import json
+import mimetypes
+import uuid
+from pathlib import Path
+from urllib import error, request
+
+
+class WebhookService:
+    def _multipart_body(self, fields: dict[str, str], file_field: str, file_path: str) -> tuple[bytes, str]:
+        boundary = f"----DorisSchoolhouse{uuid.uuid4().hex}"
+        parts: list[bytes] = []
+        for key, value in fields.items():
+            parts.extend([
+                f"--{boundary}\r\n".encode(),
+                f'Content-Disposition: form-data; name="{key}"\r\n\r\n'.encode(),
+                str(value).encode(),
+                b"\r\n",
+            ])
+        path = Path(file_path)
+        mime = mimetypes.guess_type(path.name)[0] or "application/octet-stream"
+        parts.extend([
+            f"--{boundary}\r\n".encode(),
+            f'Content-Disposition: form-data; name="{file_field}"; filename="{path.name}"\r\n'.encode(),
+            f"Content-Type: {mime}\r\n\r\n".encode(),
+            path.read_bytes(),
+            b"\r\n",
+            f"--{boundary}--\r\n".encode(),
+        ])
+        return b"".join(parts), boundary
+
+    def post_multipart(self, url: str, fields: dict[str, str], file_field: str, file_path: str) -> dict:
+        body, boundary = self._multipart_body(fields, file_field, file_path)
+        req = request.Request(
+            url,
+            data=body,
+            headers={"Content-Type": f"multipart/form-data; boundary={boundary}"},
+            method="POST",
+        )
+
+        def parse_raw(raw: str):
+            try:
+                return json.loads(raw)
+            except Exception:
+                return {"raw": raw}
+
+        try:
+            with request.urlopen(req, timeout=120) as response:
+                raw = response.read().decode("utf-8", errors="replace")
+                return {"ok": True, "status_code": response.status, "body": parse_raw(raw)}
+        except error.HTTPError as exc:
+            raw = exc.read().decode("utf-8", errors="replace")
+            return {"ok": False, "status_code": exc.code, "body": parse_raw(raw), "error": str(exc)}