Add Doris Schoolhouse and clean pending homelab changes

This commit is contained in:
Fizzlepoof
2026-05-15 21:58:42 +00:00
parent 0e7dd21c66
commit 5c889f108e
60 changed files with 3145 additions and 20 deletions

View File

@@ -0,0 +1,24 @@
from dataclasses import dataclass
import os
@dataclass
class Settings:
schoolhouse_host: str = os.getenv("SCHOOLHOUSE_HOST", "0.0.0.0")
schoolhouse_port: int = int(os.getenv("SCHOOLHOUSE_PORT", "8091"))
schoolhouse_base_url: str = os.getenv("SCHOOLHOUSE_BASE_URL", "http://localhost:8091")
database_url: str = os.getenv("DATABASE_URL", "postgresql+psycopg://schoolhouse:changeme@shared-postgres:5432/schoolhouse")
paperless_base_url: str = os.getenv("PAPERLESS_BASE_URL", "https://paperless.example.com")
paperless_api_token: str = os.getenv("PAPERLESS_API_TOKEN", "changeme")
n8n_base_url: str = os.getenv("N8N_BASE_URL", "https://n8n.example.com")
n8n_assignment_webhook: str = os.getenv("SCHOOLHOUSE_N8N_ASSIGNMENT_WEBHOOK", "/webhook/school/intake/upload")
n8n_recording_webhook: str = os.getenv("SCHOOLHOUSE_N8N_RECORDING_WEBHOOK", "/webhook/class/upload")
d2l_scraper_workdir: str = os.getenv("D2L_SCRAPER_WORKDIR", "/workspace/.openclaw-workspace/school")
d2l_scraper_command: str = os.getenv("D2L_SCRAPER_COMMAND", "node d2l_scraper_final.js")
upload_tmp_dir: str = os.getenv("UPLOAD_TMP_DIR", "/data/uploads")
state_dir: str = os.getenv("SCHOOLHOUSE_STATE_DIR", "/data/state")
storage_backend: str = os.getenv("SCHOOLHOUSE_STORAGE_BACKEND", "auto")
max_upload_mb: int = int(os.getenv("MAX_UPLOAD_MB", "1536"))
settings = Settings()

View File

@@ -0,0 +1,92 @@
CREATE TABLE IF NOT EXISTS schoolhouse_course (
id BIGSERIAL PRIMARY KEY,
d2l_course_id TEXT NOT NULL UNIQUE,
name TEXT NOT NULL,
code TEXT,
semester TEXT,
instructor_name TEXT,
active BOOLEAN NOT NULL DEFAULT TRUE,
first_seen_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
last_synced_at TIMESTAMPTZ
);
CREATE TABLE IF NOT EXISTS schoolhouse_assignment (
id BIGSERIAL PRIMARY KEY,
course_id BIGINT NOT NULL REFERENCES schoolhouse_course(id) ON DELETE CASCADE,
d2l_assignment_key TEXT,
title TEXT NOT NULL,
category TEXT,
due_at TIMESTAMPTZ,
status TEXT NOT NULL DEFAULT 'assigned',
grade_text TEXT,
grade_numeric DOUBLE PRECISION,
professor_comments TEXT,
source TEXT NOT NULL DEFAULT 'd2l',
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
);
CREATE TABLE IF NOT EXISTS schoolhouse_submission (
id BIGSERIAL PRIMARY KEY,
assignment_id BIGINT NOT NULL REFERENCES schoolhouse_assignment(id) ON DELETE CASCADE,
version_label TEXT NOT NULL DEFAULT 'v1',
original_filename TEXT NOT NULL,
mime_type TEXT NOT NULL,
checksum_sha256 TEXT,
submitted_at TIMESTAMPTZ,
suggested_metadata_json JSONB NOT NULL DEFAULT '{}'::jsonb,
confirmed_metadata_json JSONB NOT NULL DEFAULT '{}'::jsonb,
paperless_document_id BIGINT,
paperless_title TEXT,
paperless_note_last_hash TEXT,
paperless_note_appended_at TIMESTAMPTZ,
status TEXT NOT NULL DEFAULT 'draft',
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
);
CREATE TABLE IF NOT EXISTS schoolhouse_recording (
id BIGSERIAL PRIMARY KEY,
course_id BIGINT NOT NULL REFERENCES schoolhouse_course(id) ON DELETE CASCADE,
assignment_id BIGINT REFERENCES schoolhouse_assignment(id) ON DELETE SET NULL,
session_date DATE NOT NULL,
original_filename TEXT NOT NULL,
mp3_filename TEXT,
duration_seconds INTEGER,
transcription_status TEXT NOT NULL DEFAULT 'queued',
summary_status TEXT NOT NULL DEFAULT 'queued',
transcript_paperless_document_id BIGINT,
summary_paperless_document_id BIGINT,
recording_paperless_document_id BIGINT,
notes TEXT,
recording_metadata_json JSONB NOT NULL DEFAULT '{}'::jsonb,
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
);
CREATE TABLE IF NOT EXISTS schoolhouse_d2l_sync_run (
id BIGSERIAL PRIMARY KEY,
started_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
completed_at TIMESTAMPTZ,
status TEXT NOT NULL DEFAULT 'running',
raw_snapshot_path TEXT,
changes_summary_json JSONB NOT NULL DEFAULT '{}'::jsonb,
error_text TEXT
);
CREATE TABLE IF NOT EXISTS schoolhouse_assignment_mapping (
id BIGSERIAL PRIMARY KEY,
course_id BIGINT NOT NULL REFERENCES schoolhouse_course(id) ON DELETE CASCADE,
d2l_grade_item_name TEXT NOT NULL,
normalized_key TEXT NOT NULL,
assignment_id BIGINT NOT NULL REFERENCES schoolhouse_assignment(id) ON DELETE CASCADE,
confidence DOUBLE PRECISION,
mapping_source TEXT NOT NULL DEFAULT 'auto',
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
UNIQUE (course_id, d2l_grade_item_name)
);
CREATE INDEX IF NOT EXISTS idx_schoolhouse_assignment_course_id ON schoolhouse_assignment(course_id);
CREATE INDEX IF NOT EXISTS idx_schoolhouse_submission_assignment_id ON schoolhouse_submission(assignment_id);
CREATE INDEX IF NOT EXISTS idx_schoolhouse_recording_course_id ON schoolhouse_recording(course_id);
CREATE INDEX IF NOT EXISTS idx_schoolhouse_recording_assignment_id ON schoolhouse_recording(assignment_id);

View File

@@ -0,0 +1,20 @@
from fastapi import FastAPI
from fastapi.staticfiles import StaticFiles
from app.routes.api_assignments import router as assignments_router
from app.routes.api_dashboard import router as dashboard_router
from app.routes.api_paperless import router as paperless_router
from app.routes.api_recordings import router as recordings_router
from app.routes.api_sync import router as sync_router
from app.routes.ui import router as ui_router
app = FastAPI(title="Doris Schoolhouse")
app.mount("/static", StaticFiles(directory="app/static"), name="static")
app.include_router(ui_router)
app.include_router(dashboard_router)
app.include_router(assignments_router)
app.include_router(recordings_router)
app.include_router(sync_router)
app.include_router(paperless_router)

View File

@@ -0,0 +1,61 @@
from dataclasses import dataclass, field
from datetime import date, datetime
from typing import Literal, Optional
AssignmentStatus = Literal["assigned", "in_progress", "submitted", "graded", "late", "missing"]
UploadStatus = Literal["draft", "uploaded", "confirmed", "failed"]
ProcessingStatus = Literal["queued", "processing", "ready", "failed"]
@dataclass
class Course:
d2l_course_id: str
name: str
id: Optional[int] = None
code: Optional[str] = None
semester: Optional[str] = None
active: bool = True
@dataclass
class Assignment:
course_id: int
title: str
id: Optional[int] = None
d2l_assignment_key: Optional[str] = None
category: Optional[str] = None
due_at: Optional[datetime] = None
status: AssignmentStatus = "assigned"
grade_text: Optional[str] = None
grade_numeric: Optional[float] = None
professor_comments: Optional[str] = None
source: Literal["d2l", "manual", "template"] = "d2l"
@dataclass
class Submission:
assignment_id: int
original_filename: str
mime_type: str
id: Optional[int] = None
version_label: str = "v1"
submitted_at: datetime = field(default_factory=datetime.utcnow)
paperless_document_id: Optional[int] = None
paperless_title: Optional[str] = None
status: UploadStatus = "draft"
@dataclass
class Recording:
course_id: int
session_date: date
original_filename: str
id: Optional[int] = None
assignment_id: Optional[int] = None
mp3_filename: Optional[str] = None
duration_seconds: Optional[int] = None
transcription_status: ProcessingStatus = "queued"
summary_status: ProcessingStatus = "queued"
transcript_paperless_document_id: Optional[int] = None
summary_paperless_document_id: Optional[int] = None

View File

@@ -0,0 +1,148 @@
import hashlib
from pathlib import Path
from fastapi import APIRouter, Form, HTTPException, UploadFile
from app.config import settings
from app.services.d2l import D2LService
from app.services.paperless import PaperlessService
from app.services.repository import get_repository
from app.services.store import JsonStore
from app.services.webhooks import WebhookService
router = APIRouter(prefix="/api/assignments", tags=["assignments"])
store = JsonStore()
repo = get_repository()
d2l = D2LService()
paperless = PaperlessService()
webhooks = WebhookService()
@router.get("")
async def list_assignments():
courses = {item["id"]: item for item in d2l.get_courses()}
items = repo.list_assignments()
submissions = repo.list_submissions()
submission_counts = {}
for submission in submissions:
submission_counts[submission.get("assignment_id")] = submission_counts.get(submission.get("assignment_id"), 0) + 1
for item in items:
item["submission_count"] = submission_counts.get(item.get("id"), 0)
item["course"] = courses.get(item.get("course_id"), {})
return {"items": items, "backend": repo.backend}
@router.get('/submissions')
async def list_submissions():
return {"items": repo.list_submissions(), "backend": repo.backend}
@router.post("/intake")
async def intake_assignment(
file: UploadFile,
class_id: str = Form(...),
class_name: str = Form(...),
assignment_name: str = Form(...),
version: str = Form("v1"),
submission_kind: str = Form("assignment_turn_in"),
semester: str = Form(""),
course_code: str = Form(""),
paper_kind: str = Form(""),
source: str = Form("doris-schoolhouse"),
telegram_chat_id: str = Form(""),
telegram_message_id: str = Form(""),
notes: str = Form(""),
):
assignments = d2l.get_assignments(class_id)
matching = next((item for item in assignments if item.get("title") == assignment_name), None)
if not matching:
raise HTTPException(status_code=404, detail="Assignment not found in cached D2L data for selected class.")
upload_root = Path(settings.upload_tmp_dir)
upload_root.mkdir(parents=True, exist_ok=True)
safe_name = Path(file.filename or "upload.bin").name
submission_id = store.new_id("submission")
stored_path = upload_root / f"{submission_id}-{safe_name}"
payload = await file.read()
stored_path.write_bytes(payload)
checksum = hashlib.sha256(payload).hexdigest()
record = {
"id": submission_id,
"assignment_id": matching["id"],
"class_id": class_id,
"class_name": class_name,
"assignment_name": assignment_name,
"version_label": version,
"submission_kind": submission_kind,
"semester": semester or None,
"course_code": course_code or (course.get("code") if course else None),
"paper_kind": paper_kind or None,
"source": source or "doris-schoolhouse",
"telegram_chat_id": telegram_chat_id or None,
"telegram_message_id": telegram_message_id or None,
"notes": notes,
"original_filename": safe_name,
"stored_path": str(stored_path),
"mime_type": file.content_type or "application/octet-stream",
"file_size_bytes": len(payload),
"checksum_sha256": checksum,
"status": "uploaded-local",
"received_at": store.now(),
}
record["paperless_handoff"] = paperless.assignment_intake_payload(record)
created = repo.create_submission(record)
return {"status": "ok", "submission": created, "backend": repo.backend}
@router.post("/{submission_id}/confirm-paperless")
async def confirm_assignment_paperless(submission_id: str):
submissions = repo.list_submissions()
current = next((item for item in submissions if str(item.get("id")) == str(submission_id)), None)
if not current:
raise HTTPException(status_code=404, detail="Submission not found.")
handoff = paperless.assignment_intake_payload(current)
file_path = current.get("stored_path") or current.get("local_file")
if not file_path or not Path(file_path).exists():
raise HTTPException(status_code=400, detail="Stored upload file is missing.")
fields = {
"class_name": handoff["class_name"],
"assignment_name": handoff["assignment_name"],
"submission_kind": handoff["submission_kind"],
"notes": handoff.get("notes", ""),
"source": handoff.get("source", "doris-schoolhouse"),
}
for optional_key in ("semester", "course_code", "paper_kind", "telegram_chat_id", "telegram_message_id"):
value = handoff.get(optional_key)
if value:
fields[optional_key] = value
response = webhooks.post_multipart(handoff["target_url"], fields, "document", file_path)
body = response.get("body", {}) if isinstance(response, dict) else {}
body_obj = body[0] if isinstance(body, list) and body and isinstance(body[0], dict) else (body if isinstance(body, dict) else {})
ok = bool(response.get("ok", False)) if isinstance(response, dict) else False
updated = repo.update_submission(submission_id, {
"status": "paperless-queued" if ok else "paperless-error",
"confirmed_at": store.now(),
"webhook_response": response,
"paperless_document_id": body_obj.get("paperless_document_id") or body_obj.get("document_id"),
"paperless_title": body_obj.get("paperless_title") or body_obj.get("title") or handoff.get("assignment_name"),
})
return {"status": "ok" if ok else "error", "submission": updated or current, "handoff": handoff, "webhook": response}
@router.post("/{assignment_id}/status")
async def update_assignment_status(assignment_id: str, status: str = Form(...)):
assignments = repo.list_assignments()
updated = None
for item in assignments:
if str(item.get("id")) == str(assignment_id):
item["status"] = status
item["updated_at"] = store.now()
updated = item
break
if not updated:
raise HTTPException(status_code=404, detail="Assignment not found.")
repo.replace_assignments(assignments)
return {"status": "ok", "assignment": updated}

View File

@@ -0,0 +1,31 @@
from fastapi import APIRouter
from app.services.d2l import D2LService
from app.services.repository import get_repository
router = APIRouter(prefix="/api/dashboard", tags=["dashboard"])
repo = get_repository()
d2l = D2LService()
@router.get("/summary")
async def dashboard_summary():
courses = d2l.get_courses()
assignments = repo.list_assignments()
submissions = repo.list_submissions()
recordings = repo.list_recordings()
sync_runs = repo.list_sync_runs()
return {
"backend": repo.backend,
"counts": {
"courses": len(courses),
"assignments": len(assignments),
"submissions": len(submissions),
"recordings": len(recordings),
},
"latest_sync": sync_runs[-1] if sync_runs else None,
"recent_assignments": assignments[:8],
"recent_submissions": submissions[:8],
"recent_recordings": recordings[:8],
}

View File

@@ -0,0 +1,69 @@
from fastapi import APIRouter, HTTPException
from app.config import settings
from app.services.diagnostics import DiagnosticsService
from app.services.paperless import PaperlessService
from app.services.repository import get_repository
from app.services.store import JsonStore
router = APIRouter(prefix="/api", tags=["paperless", "health"])
service = PaperlessService()
repo = get_repository()
store = JsonStore()
diagnostics = DiagnosticsService()
@router.get("/health")
async def health():
return {"ok": True, "service": "doris-schoolhouse", "state_dir": settings.state_dir, "backend": repo.backend}
@router.get("/health/details")
async def health_details():
return diagnostics.collect()
@router.post("/paperless/webhook/intake-enriched")
async def paperless_intake_enriched(submission_id: str, document_id: int, paperless_title: str | None = None):
updated = repo.update_submission(
submission_id,
{
"status": "paperless-linked",
"paperless_document_id": document_id,
"paperless_title": paperless_title,
"paperless_linked_at": store.now(),
},
)
if not updated:
raise HTTPException(status_code=404, detail="Submission not found.")
return {"status": "ok", "submission": updated}
@router.post("/paperless/webhook/recording-complete")
async def paperless_recording_complete(recording_id: str, transcript_document_id: int | None = None, summary_document_id: int | None = None, recording_document_id: int | None = None):
updated = repo.update_recording(
recording_id,
{
"status": "paperless-linked",
"transcript_paperless_document_id": transcript_document_id,
"summary_paperless_document_id": summary_document_id,
"recording_paperless_document_id": recording_document_id,
"transcription_status": "ready" if transcript_document_id else None,
"summary_status": "ready" if summary_document_id else None,
"paperless_linked_at": store.now(),
},
)
if not updated:
raise HTTPException(status_code=404, detail="Recording not found.")
return {"status": "ok", "recording": updated}
@router.post("/paperless/assignments/{assignment_id}/append-grade-note")
async def append_grade_note(assignment_id: str, execute: bool = False):
if not execute:
return service.describe_grade_note_plan(assignment_id)
try:
return service.append_grade_note(assignment_id)
except ValueError as exc:
raise HTTPException(status_code=400, detail=str(exc)) from exc

View File

@@ -0,0 +1,130 @@
import hashlib
from pathlib import Path
from fastapi import APIRouter, Form, HTTPException, UploadFile
from app.config import settings
from app.services.d2l import D2LService
from app.services.media import MediaService
from app.services.paperless import PaperlessService
from app.services.repository import get_repository
from app.services.store import JsonStore
from app.services.webhooks import WebhookService
router = APIRouter(prefix="/api/recordings", tags=["recordings"])
store = JsonStore()
repo = get_repository()
d2l = D2LService()
paperless = PaperlessService()
media = MediaService()
webhooks = WebhookService()
@router.get("")
async def list_recordings():
return {"items": repo.list_recordings(), "backend": repo.backend}
@router.post("/intake")
async def intake_recording(
file: UploadFile,
class_id: str = Form(...),
class_name: str = Form(...),
session_date: str = Form(...),
assignment_id: str | None = Form(None),
notes: str = Form(""),
):
course = next((item for item in d2l.get_courses() if str(item.get("d2l_course_id") or item.get("id")) == str(class_id)), None)
if not course:
raise HTTPException(status_code=404, detail="Class not found in cached D2L data.")
upload_root = Path(settings.upload_tmp_dir)
upload_root.mkdir(parents=True, exist_ok=True)
recording_id = store.new_id("recording")
safe_name = Path(file.filename or "recording.wav").name
stored_path = upload_root / f"{recording_id}-{safe_name}"
payload = await file.read()
stored_path.write_bytes(payload)
checksum = hashlib.sha256(payload).hexdigest()
mp3_path = media.planned_mp3_path(str(stored_path))
record = {
"id": recording_id,
"class_id": str(course.get("d2l_course_id") or class_id),
"class_name": class_name,
"assignment_id": assignment_id,
"session_date": session_date,
"notes": notes,
"original_filename": safe_name,
"stored_path": str(stored_path),
"mp3_path": mp3_path,
"conversion_command": media.conversion_command(str(stored_path), mp3_path),
"delete_wav_after_success": True,
"mime_type": file.content_type or "audio/wav",
"file_size_bytes": len(payload),
"checksum_sha256": checksum,
"status": "uploaded-local",
"conversion_status": "queued",
"transcription_status": "queued",
"summary_status": "queued",
"received_at": store.now(),
}
record["paperless_handoff"] = paperless.recording_intake_payload(record)
created = repo.create_recording(record)
return {"status": "ok", "recording": created, "backend": repo.backend}
@router.post("/{recording_id}/reprocess")
async def reprocess_recording(recording_id: str):
recordings = repo.list_recordings()
current = next((item for item in recordings if str(item.get("id")) == str(recording_id)), None)
if not current:
raise HTTPException(status_code=404, detail="Recording not found.")
handoff = paperless.recording_intake_payload(current)
file_path = current.get("mp3_path") or current.get("stored_path") or current.get("local_file")
if not file_path or not Path(file_path).exists():
raise HTTPException(status_code=400, detail="Local recording file is missing.")
fields = {
"class_name": handoff["class_name"],
"lecture_title": handoff["lecture_title"],
"notes": handoff.get("notes", ""),
"source": handoff.get("source", "doris-schoolhouse"),
}
response = webhooks.post_multipart(handoff["target_url"], fields, "data", file_path)
body = response.get("body", {}) if isinstance(response, dict) else {}
body_obj = body[0] if isinstance(body, list) and body and isinstance(body[0], dict) else (body if isinstance(body, dict) else {})
ok = bool(response.get("ok", False)) if isinstance(response, dict) else False
updated = repo.update_recording(recording_id, {
"status": "requeued" if ok else "requeue-error",
"requeued_at": store.now(),
"webhook_response": response,
"transcription_status": "processing" if ok else "error",
"summary_status": "processing" if ok else "error",
"recording_paperless_document_id": body_obj.get("recording_paperless_document_id") or body_obj.get("document_id"),
})
return {"status": "ok" if ok else "error", "recording": updated or current, "handoff": handoff, "webhook": response}
@router.post("/{recording_id}/convert")
async def convert_recording(recording_id: str):
recordings = repo.list_recordings()
current = next((item for item in recordings if str(item.get("id")) == str(recording_id)), None)
if not current:
raise HTTPException(status_code=404, detail="Recording not found.")
wav_path = current.get("stored_path")
if not wav_path or not Path(wav_path).exists():
raise HTTPException(status_code=400, detail="Source WAV file is missing.")
mp3_path = current.get("mp3_path") or media.planned_mp3_path(wav_path)
result = media.convert_wav_to_mp3(wav_path, mp3_path)
updates = {
"conversion_status": result["status"],
"conversion_result": result,
"updated_at": store.now(),
}
if result["status"] == "ok":
updates["mp3_path"] = mp3_path
updates["delete_source_result"] = media.delete_source_if_requested(wav_path, bool(current.get("delete_wav_after_success")))
updates["status"] = "converted"
updated = repo.update_recording(recording_id, updates)
return {"status": result["status"], "recording": updated or current, "conversion": result}

View File

@@ -0,0 +1,27 @@
from fastapi import APIRouter
from app.services.d2l import D2LService
router = APIRouter(prefix="/api/d2l", tags=["d2l"])
service = D2LService()
@router.get("/courses")
async def get_courses():
return {"items": service.get_courses()}
@router.get("/assignments")
async def get_course_assignments(course_id: str):
return {"items": service.get_assignments(course_id), "course_id": course_id}
@router.get("/grades")
async def get_course_grades(course_id: str):
return service.get_grades(course_id)
@router.post("/sync")
async def sync_d2l():
return service.sync_to_store()

View File

@@ -0,0 +1,24 @@
from pathlib import Path
from fastapi import APIRouter, Request
from fastapi.responses import HTMLResponse
from fastapi.templating import Jinja2Templates
router = APIRouter()
templates = Jinja2Templates(directory=str(Path(__file__).resolve().parent.parent / "templates"))
@router.get("/", response_class=HTMLResponse)
async def dashboard(request: Request):
return templates.TemplateResponse("dashboard.html", {"request": request, "title": "Doris Schoolhouse"})
@router.get("/assignments/upload", response_class=HTMLResponse)
async def assignment_upload(request: Request):
return templates.TemplateResponse("assignment_upload.html", {"request": request, "title": "Assignment Upload"})
@router.get("/recordings/upload", response_class=HTMLResponse)
async def recording_upload(request: Request):
return templates.TemplateResponse("recording_upload.html", {"request": request, "title": "Recording Upload"})

View File

@@ -0,0 +1,31 @@
import os
import sys
from pathlib import Path
def main() -> int:
schema_path = Path(__file__).resolve().parent.parent / 'db' / 'schema.sql'
if not schema_path.exists():
print(f'missing schema file: {schema_path}', file=sys.stderr)
return 1
dsn = os.getenv('DATABASE_URL', '')
if not dsn:
print('DATABASE_URL is not set', file=sys.stderr)
return 1
try:
import psycopg
except Exception:
print('psycopg is not installed; cannot apply schema directly', file=sys.stderr)
return 1
dsn = dsn.replace('postgresql+psycopg://', 'postgresql://')
sql = schema_path.read_text()
with psycopg.connect(dsn) as conn:
with conn.cursor() as cur:
cur.execute(sql)
conn.commit()
print(f'Applied schema from {schema_path}')
return 0
if __name__ == '__main__':
raise SystemExit(main())

View File

@@ -0,0 +1,27 @@
from pathlib import Path
from app.config import settings
from app.services.d2l import D2LService
from app.services.store import JsonStore
def main() -> None:
Path(settings.upload_tmp_dir).mkdir(parents=True, exist_ok=True)
Path(settings.state_dir).mkdir(parents=True, exist_ok=True)
store = JsonStore()
for name, default in {
'courses': [],
'assignments': [],
'submissions': [],
'recordings': [],
'sync_runs': [],
}.items():
path = Path(settings.state_dir) / f'{name}.json'
if not path.exists():
store.save(name, default)
result = D2LService().sync_to_store()
print(result)
if __name__ == '__main__':
main()

View File

@@ -0,0 +1,44 @@
import argparse
import json
from app.services.media import MediaService
from app.services.repository import get_repository
from app.services.store import JsonStore
def main() -> None:
parser = argparse.ArgumentParser()
parser.add_argument('recording_id')
parser.add_argument('--convert-only', action='store_true')
args = parser.parse_args()
repo = get_repository()
store = JsonStore()
media = MediaService()
recordings = repo.list_recordings()
current = next((item for item in recordings if str(item.get('id')) == str(args.recording_id)), None)
if not current:
raise SystemExit(f'recording not found: {args.recording_id}')
wav_path = current.get('stored_path')
mp3_path = current.get('mp3_path') or media.planned_mp3_path(wav_path)
result = media.convert_wav_to_mp3(wav_path, mp3_path)
updates = {
'conversion_status': result['status'],
'conversion_result': result,
'updated_at': store.now(),
}
if result['status'] == 'ok':
updates['mp3_path'] = mp3_path
delete_result = media.delete_source_if_requested(wav_path, bool(current.get('delete_wav_after_success')))
updates['delete_source_result'] = delete_result
if not args.convert_only:
updates['status'] = 'converted'
repo.update_recording(str(current.get('id')), updates)
print(json.dumps({'recording_id': current.get('id'), 'result': result, 'updates': updates}, indent=2))
if __name__ == '__main__':
main()

View File

@@ -0,0 +1,139 @@
import json
from pathlib import Path
from typing import Any
from app.config import settings
from app.services.repository import get_repository
from app.services.store import JsonStore
class D2LService:
def __init__(self) -> None:
self.store = JsonStore()
self.repo = get_repository()
def _snapshot_path(self) -> Path:
candidate_dirs = [
Path(settings.d2l_scraper_workdir),
Path('/home/fizzlepoof/.openclaw/workspace/school'),
]
for workdir in candidate_dirs:
for name in ('d2l_current_snapshot.json', 'd2l_grades.json', 'd2l_scraper_output.json'):
candidate = workdir / name
if candidate.exists():
return candidate
return candidate_dirs[0] / 'd2l_current_snapshot.json'
def load_snapshot(self) -> dict[str, Any]:
path = self._snapshot_path()
if not path.exists():
return {"courses": [], "source_path": str(path), "missing": True}
data = json.loads(path.read_text())
if isinstance(data, dict):
data["source_path"] = str(path)
return data
return {"courses": [], "source_path": str(path), "missing": True}
def _normalize_courses(self, snapshot: dict[str, Any]) -> list[dict[str, Any]]:
items = []
for course in snapshot.get("courses", []):
items.append({
"id": str(course.get("id")),
"d2l_course_id": str(course.get("id")),
"name": course.get("name"),
"code": course.get("code"),
"finalSummary": course.get("finalSummary"),
"gradeItemCount": course.get("gradeItemCount"),
"synced_at": self.store.now(),
})
return items
def _normalize_assignments(self, snapshot: dict[str, Any]) -> list[dict[str, Any]]:
items = []
for course in snapshot.get("courses", []):
course_id = str(course.get("id"))
for grade_item in course.get("gradeItems", []):
if grade_item.get("isCategory"):
continue
title = grade_item.get("name") or "Untitled"
items.append({
"id": f"{course_id}:{title.lower().strip()}",
"course_id": course_id,
"course_name": course.get("name"),
"title": title,
"points": grade_item.get("points"),
"grade": grade_item.get("grade"),
"comments": grade_item.get("comments"),
"source": "d2l-grade-item",
"synced_at": self.store.now(),
})
return items
def sync_to_store(self) -> dict[str, Any]:
snapshot = self.load_snapshot()
courses = self._normalize_courses(snapshot)
assignments = self._normalize_assignments(snapshot)
existing_assignments = {item.get('id'): item for item in self.repo.list_assignments()}
for item in assignments:
prior = existing_assignments.get(item['id']) or {}
item['status'] = prior.get('status', 'assigned')
item['updated_at'] = self.store.now()
item['created_at'] = prior.get('created_at', self.store.now())
self.repo.replace_courses(courses)
self.repo.replace_assignments(assignments)
sync_runs = self.repo.list_sync_runs()
sync_runs.append({
"id": self.store.new_id("sync"),
"snapshot_path": snapshot.get("source_path"),
"course_count": len(courses),
"assignment_count": len(assignments),
"completed_at": self.store.now(),
"status": "completed",
})
self.repo.save_sync_runs(sync_runs)
return {
"status": "ok",
"snapshot_path": snapshot.get("source_path"),
"course_count": len(courses),
"assignment_count": len(assignments),
"backend": self.repo.backend,
}
def get_courses(self) -> list[dict[str, Any]]:
items = self.repo.list_courses()
if items:
return items
self.sync_to_store()
return self.repo.list_courses()
def get_assignments(self, course_id: str) -> list[dict[str, Any]]:
items = self.repo.list_assignments()
if not items:
self.sync_to_store()
items = self.repo.list_assignments()
return [item for item in items if str(item.get("course_id")) == str(course_id)]
def get_grades(self, course_id: str) -> dict[str, Any]:
snapshot = self.load_snapshot()
for course in snapshot.get("courses", []):
if str(course.get("id")) == str(course_id):
return {
"course_id": str(course.get("id")),
"course_name": course.get("name"),
"final": course.get("final"),
"finalSummary": course.get("finalSummary"),
"gradeItems": course.get("gradeItems", []),
}
return {"course_id": course_id, "gradeItems": []}
def describe_sync_plan(self) -> dict:
snapshot = self.load_snapshot()
return {
"status": "stub",
"backend": self.repo.backend,
"scraper_workdir": settings.d2l_scraper_workdir,
"scraper_command": settings.d2l_scraper_command,
"snapshot_path": snapshot.get("source_path"),
"course_count": len(snapshot.get("courses", [])),
"message": "Run existing D2L scraper, normalize courses/assignments/grades, then store into Schoolhouse persistence.",
}

View File

@@ -0,0 +1,23 @@
from pathlib import Path
from app.config import settings
from app.services.media import MediaService
from app.services.repository import get_repository
class DiagnosticsService:
def collect(self) -> dict:
repo = get_repository()
media = MediaService()
return {
'backend': repo.backend,
'state_dir': settings.state_dir,
'state_dir_exists': Path(settings.state_dir).exists(),
'upload_tmp_dir': settings.upload_tmp_dir,
'upload_tmp_dir_exists': Path(settings.upload_tmp_dir).exists(),
'ffmpeg_path': media.ffmpeg_path(),
'can_convert_audio': media.can_convert(),
'n8n_base_url': settings.n8n_base_url,
'paperless_base_url': settings.paperless_base_url,
'd2l_scraper_workdir': settings.d2l_scraper_workdir,
}

View File

@@ -0,0 +1,57 @@
import shutil
import subprocess
from pathlib import Path
class MediaService:
def planned_mp3_path(self, wav_path: str) -> str:
path = Path(wav_path)
return str(path.with_suffix('.mp3'))
def conversion_command(self, wav_path: str, mp3_path: str) -> list[str]:
return [
'ffmpeg',
'-y',
'-i', wav_path,
'-codec:a', 'libmp3lame',
'-q:a', '2',
mp3_path,
]
def ffmpeg_path(self) -> str | None:
return shutil.which('ffmpeg')
def can_convert(self) -> bool:
return self.ffmpeg_path() is not None
def convert_wav_to_mp3(self, wav_path: str, mp3_path: str) -> dict:
if not self.can_convert():
return {
'status': 'blocked',
'reason': 'ffmpeg-not-installed',
'command': self.conversion_command(wav_path, mp3_path),
}
cmd = self.conversion_command(wav_path, mp3_path)
result = subprocess.run(cmd, capture_output=True, text=True)
if result.returncode != 0:
return {
'status': 'failed',
'returncode': result.returncode,
'stderr': result.stderr[-4000:],
'stdout': result.stdout[-2000:],
'command': cmd,
}
return {
'status': 'ok',
'mp3_path': mp3_path,
'command': cmd,
}
def delete_source_if_requested(self, wav_path: str, should_delete: bool) -> dict:
if not should_delete:
return {'status': 'skipped', 'reason': 'delete-not-requested'}
path = Path(wav_path)
if not path.exists():
return {'status': 'skipped', 'reason': 'source-missing'}
path.unlink()
return {'status': 'ok', 'deleted': wav_path}

View File

@@ -0,0 +1,109 @@
import hashlib
from urllib.parse import urljoin
from app.config import settings
from app.services.paperless_client import PaperlessClient
from app.services.repository import get_repository
from app.services.store import JsonStore
class PaperlessService:
def __init__(self) -> None:
self.store = JsonStore()
self.repo = get_repository()
self.client = PaperlessClient()
def assignment_intake_payload(self, submission: dict) -> dict:
return {
"target_url": urljoin(settings.n8n_base_url, settings.n8n_assignment_webhook),
"class_name": submission.get("class_name"),
"assignment_name": submission.get("assignment_name"),
"submission_kind": submission.get("submission_kind") or "assignment_turn_in",
"semester": submission.get("semester"),
"course_code": submission.get("course_code"),
"paper_kind": submission.get("paper_kind"),
"notes": submission.get("notes", ""),
"source": submission.get("source") or "doris-schoolhouse",
"telegram_chat_id": submission.get("telegram_chat_id"),
"telegram_message_id": submission.get("telegram_message_id"),
"local_file": submission.get("stored_path"),
"version": submission.get("version_label"),
}
def recording_intake_payload(self, recording: dict) -> dict:
return {
"target_url": urljoin(settings.n8n_base_url, settings.n8n_recording_webhook),
"class_name": recording.get("class_name"),
"lecture_title": f"{recording.get('class_name')} {recording.get('session_date')}",
"notes": recording.get("notes", ""),
"local_file": recording.get("mp3_path") or recording.get("stored_path"),
"source": "doris-schoolhouse",
}
def build_grade_note(self, assignment_id: str) -> str:
submissions = self.repo.list_submissions()
assignments = {item.get("id"): item for item in self.repo.list_assignments()}
assignment = assignments.get(str(assignment_id), {})
linked = [s for s in submissions if str(s.get("assignment_id")) == str(assignment_id)]
note_lines = [
"Grade sync from Doris Schoolhouse",
f"Assignment: {assignment.get('title', 'unknown')}",
]
if assignment.get("grade"):
note_lines.append(f"Grade: {assignment['grade']}")
if assignment.get("points"):
note_lines.append(f"Points: {assignment['points']}")
if assignment.get("comments"):
note_lines.append(f"Professor comments: {assignment['comments']}")
if linked:
note_lines.append(f"Linked submissions: {len(linked)}")
return "\n".join(note_lines)
def describe_grade_note_plan(self, assignment_id: str) -> dict:
submissions = self.repo.list_submissions()
linked = [s for s in submissions if str(s.get("assignment_id")) == str(assignment_id)]
return {
"status": "stub",
"backend": self.repo.backend,
"assignment_id": assignment_id,
"paperless_base_url": settings.paperless_base_url,
"linked_submission_count": len(linked),
"note_preview": self.build_grade_note(assignment_id),
"message": "Resolve linked Paperless submission document, hash latest grade/comment payload, and append a dated Schoolhouse note if changed.",
}
def append_grade_note(self, assignment_id: str) -> dict:
submissions = self.repo.list_submissions()
linked = [s for s in submissions if str(s.get("assignment_id")) == str(assignment_id)]
if not linked:
raise ValueError("No linked submissions for assignment")
target = next((s for s in linked if s.get("paperless_document_id")), linked[0])
document_id = target.get("paperless_document_id")
if not document_id:
raise ValueError("Linked submission has no Paperless document id yet")
note = self.build_grade_note(assignment_id)
note_hash = hashlib.sha256(note.encode('utf-8')).hexdigest()
if target.get("paperless_note_last_hash") == note_hash:
return {
"status": "skipped",
"reason": "note-unchanged",
"assignment_id": assignment_id,
"document_id": document_id,
"note_hash": note_hash,
}
response = self.client.add_note(document_id, note)
self.repo.update_submission(
str(target.get("id")),
{
"paperless_note_last_hash": note_hash,
"paperless_title": target.get("paperless_title"),
"status": target.get("status", "paperless-linked"),
},
)
return {
"status": "ok",
"assignment_id": assignment_id,
"document_id": document_id,
"note_hash": note_hash,
"paperless_response": response,
}

View File

@@ -0,0 +1,31 @@
import json
from urllib import request
from urllib.parse import urljoin
from app.config import settings
class PaperlessClient:
def __init__(self) -> None:
self.base_url = settings.paperless_base_url.rstrip('/') + '/'
self.token = settings.paperless_api_token
def _req(self, method: str, path: str, payload: dict | None = None) -> dict:
url = urljoin(self.base_url, path.lstrip('/'))
body = None if payload is None else json.dumps(payload).encode('utf-8')
headers = {
'Authorization': f'Token {self.token}',
'Accept': 'application/json',
}
if body is not None:
headers['Content-Type'] = 'application/json'
req = request.Request(url, data=body, headers=headers, method=method)
with request.urlopen(req, timeout=120) as resp:
raw = resp.read().decode('utf-8', errors='replace')
return json.loads(raw) if raw else {}
def get_document(self, document_id: int | str) -> dict:
return self._req('GET', f'/api/documents/{document_id}/')
def add_note(self, document_id: int | str, note: str) -> dict:
return self._req('POST', f'/api/documents/{document_id}/notes/', {'note': note})

View File

@@ -0,0 +1,388 @@
import json
from pathlib import Path
from typing import Any
from app.config import settings
from app.services.store import JsonStore
class JsonRepository:
backend = "json"
def __init__(self) -> None:
self.store = JsonStore()
def list_courses(self) -> list[dict[str, Any]]:
return self.store.load("courses", [])
def replace_courses(self, items: list[dict[str, Any]]) -> list[dict[str, Any]]:
return self.store.replace_collection("courses", items)
def list_assignments(self) -> list[dict[str, Any]]:
return self.store.load("assignments", [])
def replace_assignments(self, items: list[dict[str, Any]]) -> list[dict[str, Any]]:
return self.store.replace_collection("assignments", items)
def list_sync_runs(self) -> list[dict[str, Any]]:
return self.store.load("sync_runs", [])
def save_sync_runs(self, items: list[dict[str, Any]]) -> list[dict[str, Any]]:
return self.store.save("sync_runs", items)
def list_submissions(self) -> list[dict[str, Any]]:
return self.store.load("submissions", [])
def create_submission(self, item: dict[str, Any]) -> dict[str, Any]:
return self.store.append_collection("submissions", item)
def update_submission(self, submission_id: str, updates: dict[str, Any]) -> dict[str, Any] | None:
return self.store.update_collection_item(
"submissions",
lambda item: str(item.get("id")) == str(submission_id),
lambda item: {**item, **updates},
)
def list_recordings(self) -> list[dict[str, Any]]:
return self.store.load("recordings", [])
def create_recording(self, item: dict[str, Any]) -> dict[str, Any]:
return self.store.append_collection("recordings", item)
def update_recording(self, recording_id: str, updates: dict[str, Any]) -> dict[str, Any] | None:
return self.store.update_collection_item(
"recordings",
lambda item: str(item.get("id")) == str(recording_id),
lambda item: {**item, **updates},
)
class PostgresRepository:
backend = "postgres"
def __init__(self) -> None:
import psycopg
from psycopg.rows import dict_row
self._psycopg = psycopg
self._dict_row = dict_row
self.dsn = settings.database_url.replace("postgresql+psycopg://", "postgresql://")
def _connect(self):
return self._psycopg.connect(self.dsn, row_factory=self._dict_row)
def _ensure_course(self, cur, item: dict[str, Any]) -> dict[str, Any] | None:
course_id = item.get("class_id") or item.get("course_id")
if not course_id:
return None
cur.execute("SELECT id, d2l_course_id, name FROM schoolhouse_course WHERE d2l_course_id = %s", (course_id,))
row = cur.fetchone()
if row:
return row
course_name = item.get("class_name") or item.get("course_name") or str(course_id)
cur.execute(
"""
INSERT INTO schoolhouse_course (d2l_course_id, name, last_synced_at)
VALUES (%s, %s, NOW())
RETURNING id, d2l_course_id, name
""",
(course_id, course_name),
)
return cur.fetchone()
def _ensure_assignment(self, cur, item: dict[str, Any]) -> dict[str, Any] | None:
course = self._ensure_course(cur, item)
if not course:
return None
assignment_id = item.get("assignment_id")
assignment_name = item.get("assignment_name") or item.get("title")
if assignment_id:
cur.execute(
"SELECT id, title FROM schoolhouse_assignment WHERE course_id = %s AND d2l_assignment_key = %s LIMIT 1",
(course["id"], assignment_id),
)
row = cur.fetchone()
if row:
return row
if assignment_name:
cur.execute(
"SELECT id, title FROM schoolhouse_assignment WHERE course_id = %s AND title = %s LIMIT 1",
(course["id"], assignment_name),
)
row = cur.fetchone()
if row:
return row
cur.execute(
"""
INSERT INTO schoolhouse_assignment (course_id, d2l_assignment_key, title, source, status, created_at, updated_at)
VALUES (%s, %s, %s, %s, %s, NOW(), NOW())
RETURNING id, title
""",
(course["id"], assignment_id, assignment_name, item.get("source", "schoolhouse-intake"), item.get("status", "assigned")),
)
return cur.fetchone()
return None
def list_courses(self) -> list[dict[str, Any]]:
with self._connect() as conn, conn.cursor() as cur:
cur.execute("SELECT id::text AS id, d2l_course_id, name, code, semester, last_synced_at FROM schoolhouse_course ORDER BY name")
return list(cur.fetchall())
def replace_courses(self, items: list[dict[str, Any]]) -> list[dict[str, Any]]:
with self._connect() as conn, conn.cursor() as cur:
for item in items:
cur.execute(
"""
INSERT INTO schoolhouse_course (d2l_course_id, name, code, semester, last_synced_at)
VALUES (%s, %s, %s, %s, NOW())
ON CONFLICT (d2l_course_id)
DO UPDATE SET name = EXCLUDED.name, code = EXCLUDED.code, semester = EXCLUDED.semester, last_synced_at = NOW()
""",
(item.get("d2l_course_id"), item.get("name"), item.get("code"), item.get("semester")),
)
conn.commit()
return items
def list_assignments(self) -> list[dict[str, Any]]:
with self._connect() as conn, conn.cursor() as cur:
cur.execute(
"""
SELECT a.id::text AS id,
c.d2l_course_id AS course_id,
c.name AS course_name,
a.title,
a.grade_text AS grade,
a.professor_comments AS comments,
a.status,
a.updated_at
FROM schoolhouse_assignment a
JOIN schoolhouse_course c ON c.id = a.course_id
ORDER BY c.name, a.title
"""
)
return list(cur.fetchall())
def replace_assignments(self, items: list[dict[str, Any]]) -> list[dict[str, Any]]:
with self._connect() as conn, conn.cursor() as cur:
for item in items:
cur.execute("SELECT id FROM schoolhouse_course WHERE d2l_course_id = %s", (item.get("course_id"),))
row = cur.fetchone()
if not row:
continue
course_pk = row["id"]
cur.execute(
"""
INSERT INTO schoolhouse_assignment (course_id, d2l_assignment_key, title, grade_text, professor_comments, source, status, created_at, updated_at)
VALUES (%s, %s, %s, %s, %s, %s, %s, NOW(), NOW())
ON CONFLICT DO NOTHING
""",
(course_pk, item.get("id"), item.get("title"), item.get("grade"), item.get("comments"), item.get("source", "d2l"), item.get("status", "assigned")),
)
cur.execute(
"""
UPDATE schoolhouse_assignment
SET grade_text = %s,
professor_comments = %s,
status = COALESCE(status, %s),
updated_at = NOW()
WHERE course_id = %s AND title = %s
""",
(item.get("grade"), item.get("comments"), item.get("status", "assigned"), course_pk, item.get("title")),
)
conn.commit()
return items
def list_sync_runs(self) -> list[dict[str, Any]]:
with self._connect() as conn, conn.cursor() as cur:
cur.execute("SELECT id::text AS id, started_at, completed_at, status, raw_snapshot_path AS snapshot_path, changes_summary_json FROM schoolhouse_d2l_sync_run ORDER BY started_at DESC")
return list(cur.fetchall())
def save_sync_runs(self, items: list[dict[str, Any]]) -> list[dict[str, Any]]:
if not items:
return items
last = items[-1]
with self._connect() as conn, conn.cursor() as cur:
cur.execute(
"INSERT INTO schoolhouse_d2l_sync_run (completed_at, status, raw_snapshot_path, changes_summary_json) VALUES (NOW(), %s, %s, %s::jsonb)",
(last.get("status", "completed"), last.get("snapshot_path"), json.dumps(last)),
)
conn.commit()
return items
def list_submissions(self) -> list[dict[str, Any]]:
with self._connect() as conn, conn.cursor() as cur:
cur.execute(
"""
SELECT s.id::text AS id,
a.id::text AS assignment_id,
c.d2l_course_id AS class_id,
c.name AS class_name,
a.title AS assignment_name,
s.version_label,
s.original_filename,
s.mime_type,
s.status,
s.paperless_document_id,
s.paperless_title,
s.paperless_note_last_hash,
s.submitted_at,
s.confirmed_metadata_json,
s.confirmed_metadata_json->>'stored_path' AS stored_path,
s.confirmed_metadata_json->>'local_file' AS local_file
FROM schoolhouse_submission s
JOIN schoolhouse_assignment a ON a.id = s.assignment_id
JOIN schoolhouse_course c ON c.id = a.course_id
ORDER BY s.created_at DESC
"""
)
return list(cur.fetchall())
def create_submission(self, item: dict[str, Any]) -> dict[str, Any]:
with self._connect() as conn, conn.cursor() as cur:
row = self._ensure_assignment(cur, item)
if not row:
raise ValueError("Assignment not found and could not be created in Postgres repository")
cur.execute(
"""
INSERT INTO schoolhouse_submission (
assignment_id, version_label, original_filename, mime_type, checksum_sha256,
submitted_at, suggested_metadata_json, confirmed_metadata_json, status
) VALUES (%s, %s, %s, %s, %s, NOW(), %s::jsonb, %s::jsonb, %s)
RETURNING id::text AS id
""",
(
row["id"], item.get("version_label") or "v1", item.get("original_filename"), item.get("mime_type", "application/octet-stream"), item.get("checksum_sha256"),
json.dumps(item.get("paperless_handoff", {})), json.dumps(item), item.get("status", "uploaded-local"),
),
)
created = cur.fetchone()
conn.commit()
item["id"] = created["id"]
return item
def update_submission(self, submission_id: str, updates: dict[str, Any]) -> dict[str, Any] | None:
with self._connect() as conn, conn.cursor() as cur:
cur.execute("SELECT confirmed_metadata_json FROM schoolhouse_submission WHERE id = %s::bigint", (submission_id,))
current = cur.fetchone()
if not current:
return None
merged_metadata = {**(current.get("confirmed_metadata_json") or {}), **updates}
cur.execute(
"""
UPDATE schoolhouse_submission
SET status = COALESCE(%s, status),
paperless_document_id = COALESCE(%s, paperless_document_id),
paperless_title = COALESCE(%s, paperless_title),
paperless_note_last_hash = COALESCE(%s, paperless_note_last_hash),
paperless_note_appended_at = CASE WHEN %s::text IS NOT NULL THEN NOW() ELSE paperless_note_appended_at END,
confirmed_metadata_json = %s::jsonb,
updated_at = NOW()
WHERE id = %s::bigint
RETURNING id::text AS id, status, paperless_document_id, paperless_title, paperless_note_last_hash
""",
(
updates.get("status"), updates.get("paperless_document_id"), updates.get("paperless_title"), updates.get("paperless_note_last_hash"), updates.get("paperless_note_last_hash"),
json.dumps(merged_metadata), submission_id,
),
)
row = cur.fetchone()
conn.commit()
return {**merged_metadata, **dict(row)} if row else None
def list_recordings(self) -> list[dict[str, Any]]:
with self._connect() as conn, conn.cursor() as cur:
cur.execute(
"""
SELECT r.id::text AS id,
c.d2l_course_id AS class_id,
c.name AS class_name,
r.assignment_id::text AS assignment_id,
r.course_id::text AS course_id,
r.session_date::text AS session_date,
r.original_filename,
r.mp3_filename,
r.transcription_status,
r.summary_status,
r.recording_paperless_document_id,
r.recording_metadata_json,
r.recording_metadata_json->>'stored_path' AS stored_path,
r.recording_metadata_json->>'mp3_path' AS mp3_path,
r.recording_metadata_json->>'status' AS status,
r.recording_metadata_json->>'conversion_status' AS conversion_status,
r.recording_metadata_json->>'notes' AS notes,
r.recording_metadata_json->>'delete_wav_after_success' AS delete_wav_after_success
FROM schoolhouse_recording r
JOIN schoolhouse_course c ON c.id = r.course_id
ORDER BY r.created_at DESC
"""
)
return list(cur.fetchall())
def create_recording(self, item: dict[str, Any]) -> dict[str, Any]:
with self._connect() as conn, conn.cursor() as cur:
row = self._ensure_course(cur, item)
if not row:
raise ValueError("Course not found and could not be created in Postgres repository")
cur.execute(
"""
INSERT INTO schoolhouse_recording (
course_id, assignment_id, session_date, original_filename, mp3_filename, notes,
transcription_status, summary_status, recording_metadata_json
) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s::jsonb)
RETURNING id::text AS id
""",
(
row["id"], item.get("assignment_id") or None, item.get("session_date"), item.get("original_filename"),
Path(item.get("mp3_path")).name if item.get("mp3_path") else None, item.get("notes"), item.get("transcription_status"),
item.get("summary_status"), json.dumps(item),
),
)
created = cur.fetchone()
conn.commit()
item["id"] = created["id"]
return item
def update_recording(self, recording_id: str, updates: dict[str, Any]) -> dict[str, Any] | None:
with self._connect() as conn, conn.cursor() as cur:
cur.execute("SELECT recording_metadata_json FROM schoolhouse_recording WHERE id = %s::bigint", (recording_id,))
current = cur.fetchone()
if not current:
return None
merged_metadata = {**(current.get("recording_metadata_json") or {}), **updates}
cur.execute(
"""
UPDATE schoolhouse_recording
SET transcription_status = COALESCE(%s, transcription_status),
summary_status = COALESCE(%s, summary_status),
transcript_paperless_document_id = COALESCE(%s, transcript_paperless_document_id),
summary_paperless_document_id = COALESCE(%s, summary_paperless_document_id),
recording_paperless_document_id = COALESCE(%s, recording_paperless_document_id),
mp3_filename = COALESCE(%s, mp3_filename),
notes = COALESCE(%s, notes),
recording_metadata_json = %s::jsonb,
updated_at = NOW()
WHERE id = %s::bigint
RETURNING id::text AS id, transcription_status, summary_status, recording_paperless_document_id
""",
(
updates.get("transcription_status"), updates.get("summary_status"), updates.get("transcript_paperless_document_id"),
updates.get("summary_paperless_document_id"), updates.get("recording_paperless_document_id"),
Path(updates.get("mp3_path")).name if updates.get("mp3_path") else None,
updates.get("notes"), json.dumps(merged_metadata), recording_id,
),
)
row = cur.fetchone()
conn.commit()
return {**merged_metadata, **dict(row)} if row else None
def get_repository():
backend = getattr(settings, "storage_backend", "auto")
if backend == "json":
return JsonRepository()
if settings.database_url.startswith("postgresql"):
try:
return PostgresRepository()
except Exception:
return JsonRepository()
return JsonRepository()

View File

@@ -0,0 +1,53 @@
import json
from datetime import datetime, timezone
from pathlib import Path
from typing import Any
from uuid import uuid4
from app.config import settings
class JsonStore:
def __init__(self) -> None:
self.root = Path(settings.state_dir)
self.root.mkdir(parents=True, exist_ok=True)
def _path(self, name: str) -> Path:
return self.root / f"{name}.json"
def load(self, name: str, default: Any) -> Any:
path = self._path(name)
if not path.exists():
return default
return json.loads(path.read_text())
def save(self, name: str, value: Any) -> Any:
path = self._path(name)
path.write_text(json.dumps(value, indent=2, sort_keys=True))
return value
def replace_collection(self, name: str, items: list[dict[str, Any]]) -> list[dict[str, Any]]:
return self.save(name, items)
def append_collection(self, name: str, item: dict[str, Any]) -> dict[str, Any]:
items = self.load(name, [])
items.append(item)
self.save(name, items)
return item
def update_collection_item(self, name: str, predicate, updater):
items = self.load(name, [])
updated = None
for idx, item in enumerate(items):
if predicate(item):
items[idx] = updater(dict(item))
updated = items[idx]
break
self.save(name, items)
return updated
def now(self) -> str:
return datetime.now(timezone.utc).isoformat()
def new_id(self, prefix: str) -> str:
return f"{prefix}-{uuid4().hex[:12]}"

View File

@@ -0,0 +1,52 @@
import json
import mimetypes
import uuid
from pathlib import Path
from urllib import error, request
class WebhookService:
def _multipart_body(self, fields: dict[str, str], file_field: str, file_path: str) -> tuple[bytes, str]:
boundary = f"----DorisSchoolhouse{uuid.uuid4().hex}"
parts: list[bytes] = []
for key, value in fields.items():
parts.extend([
f"--{boundary}\r\n".encode(),
f'Content-Disposition: form-data; name="{key}"\r\n\r\n'.encode(),
str(value).encode(),
b"\r\n",
])
path = Path(file_path)
mime = mimetypes.guess_type(path.name)[0] or "application/octet-stream"
parts.extend([
f"--{boundary}\r\n".encode(),
f'Content-Disposition: form-data; name="{file_field}"; filename="{path.name}"\r\n'.encode(),
f"Content-Type: {mime}\r\n\r\n".encode(),
path.read_bytes(),
b"\r\n",
f"--{boundary}--\r\n".encode(),
])
return b"".join(parts), boundary
def post_multipart(self, url: str, fields: dict[str, str], file_field: str, file_path: str) -> dict:
body, boundary = self._multipart_body(fields, file_field, file_path)
req = request.Request(
url,
data=body,
headers={"Content-Type": f"multipart/form-data; boundary={boundary}"},
method="POST",
)
def parse_raw(raw: str):
try:
return json.loads(raw)
except Exception:
return {"raw": raw}
try:
with request.urlopen(req, timeout=120) as response:
raw = response.read().decode("utf-8", errors="replace")
return {"ok": True, "status_code": response.status, "body": parse_raw(raw)}
except error.HTTPError as exc:
raw = exc.read().decode("utf-8", errors="replace")
return {"ok": False, "status_code": exc.code, "body": parse_raw(raw), "error": str(exc)}

View File

@@ -0,0 +1,60 @@
:root {
color-scheme: dark;
--bg: #0f172a;
--panel: #111827;
--panel-2: #1f2937;
--line: #334155;
--text: #e5e7eb;
--muted: #94a3b8;
--accent: #38bdf8;
}
* { box-sizing: border-box; }
body {
margin: 0;
font-family: Inter, system-ui, sans-serif;
background: linear-gradient(180deg, #020617, var(--bg));
color: var(--text);
}
.site-header, .page-shell { max-width: 1200px; margin: 0 auto; padding: 24px; }
.site-header { display: flex; justify-content: space-between; gap: 24px; align-items: end; }
.site-header p { color: var(--muted); margin: 6px 0 0; }
nav { display: flex; gap: 16px; flex-wrap: wrap; }
nav a { color: var(--accent); text-decoration: none; }
.hero { display: grid; grid-template-columns: repeat(auto-fit, minmax(280px, 1fr)); gap: 20px; }
.card {
background: rgba(17, 24, 39, 0.92);
border: 1px solid var(--line);
border-radius: 16px;
padding: 20px;
}
.form-card { max-width: 760px; }
.stack { display: grid; gap: 14px; }
label { display: grid; gap: 8px; color: var(--muted); }
input, textarea, button {
font: inherit;
border-radius: 10px;
border: 1px solid var(--line);
padding: 12px 14px;
}
input, textarea { background: var(--panel-2); color: var(--text); }
button {
background: var(--accent);
color: #082f49;
font-weight: 700;
cursor: pointer;
}
.muted { color: var(--muted); }
.stat-grid { display: grid; grid-template-columns: repeat(2, minmax(0, 1fr)); gap: 14px; margin: 12px 0 16px; }
.stat-grid div { background: var(--panel-2); border: 1px solid var(--line); border-radius: 12px; padding: 12px; display: grid; gap: 8px; }
.stat-grid span { font-size: 1.5rem; font-weight: 700; color: var(--text); }
.three-up { display: grid; grid-template-columns: repeat(auto-fit, minmax(280px, 1fr)); gap: 20px; margin-top: 20px; }
.list-block { display: grid; gap: 10px; }
.list-item { padding: 10px 12px; background: var(--panel-2); border: 1px solid var(--line); border-radius: 12px; }
.list-item strong { display: block; margin-bottom: 6px; }
.list-item small { color: var(--muted); display: block; }

View File

@@ -0,0 +1,127 @@
async function fetchJson(url, options) {
const res = await fetch(url, options);
if (!res.ok) throw new Error(`Fetch failed: ${url}`);
return await res.json();
}
function option(value, label) {
const el = document.createElement('option');
el.value = value;
el.textContent = label;
return el;
}
function renderList(targetId, items, formatter) {
const target = document.getElementById(targetId);
if (!target) return;
if (!items.length) {
target.textContent = 'Nothing here yet.';
target.classList.add('muted');
return;
}
target.classList.remove('muted');
target.innerHTML = '';
for (const item of items) {
const wrapper = document.createElement('div');
wrapper.className = 'list-item';
wrapper.innerHTML = formatter(item);
target.appendChild(wrapper);
}
}
async function populateCourses() {
const data = await fetchJson('/api/d2l/courses');
const courseSelect = document.getElementById('class-select');
const courseNameInput = document.getElementById('class-name-input');
const recordingSelect = document.getElementById('recording-class-select');
const recordingNameInput = document.getElementById('recording-class-name-input');
const courses = data.items || [];
const byId = new Map(courses.map((course) => [String(course.d2l_course_id || course.id), course]));
if (courseSelect) {
courseSelect.innerHTML = '';
courseSelect.appendChild(option('', 'Choose a class'));
for (const course of courses) {
courseSelect.appendChild(option(course.d2l_course_id || course.id, `${course.name}${course.finalSummary ? ' — ' + course.finalSummary : ''}`));
}
courseSelect.addEventListener('change', async (event) => {
const courseId = event.target.value;
const course = byId.get(String(courseId));
if (courseNameInput) courseNameInput.value = course?.name || '';
const assignmentSelect = document.getElementById('assignment-select');
assignmentSelect.innerHTML = '';
if (!courseId) {
assignmentSelect.appendChild(option('', 'Pick a class first'));
return;
}
assignmentSelect.appendChild(option('', 'Loading assignments…'));
const assignmentData = await fetchJson(`/api/d2l/assignments?course_id=${encodeURIComponent(courseId)}`);
assignmentSelect.innerHTML = '';
assignmentSelect.appendChild(option('', 'Choose an assignment'));
for (const assignment of assignmentData.items || []) {
assignmentSelect.appendChild(option(assignment.title, assignment.title));
}
});
}
if (recordingSelect) {
recordingSelect.innerHTML = '';
recordingSelect.appendChild(option('', 'Choose a class'));
for (const course of courses) {
recordingSelect.appendChild(option(course.d2l_course_id || course.id, course.name));
}
recordingSelect.addEventListener('change', (event) => {
const course = byId.get(String(event.target.value));
if (recordingNameInput) recordingNameInput.value = course?.name || '';
});
}
}
async function loadDashboard() {
const data = await fetchJson('/api/dashboard/summary');
const counts = data.counts || {};
const setText = (id, value) => {
const el = document.getElementById(id);
if (el) el.textContent = value;
};
setText('stat-courses', counts.courses ?? '0');
setText('stat-assignments', counts.assignments ?? '0');
setText('stat-submissions', counts.submissions ?? '0');
setText('stat-recordings', counts.recordings ?? '0');
setText('backend-note', `Storage backend: ${data.backend}`);
if (data.latest_sync) {
setText('sync-status', `Last sync: ${data.latest_sync.completed_at || 'unknown'} · ${data.latest_sync.assignment_count || 0} assignment rows`);
}
renderList('recent-assignments', data.recent_assignments || [], (item) => `<strong>${item.title || 'Untitled'}</strong><small>${item.course_name || ''}</small><small>Status: ${item.status || 'assigned'} · Grade: ${item.grade || '—'}</small>`);
renderList('recent-submissions', data.recent_submissions || [], (item) => `<strong>${item.assignment_name || item.original_filename || 'Submission'}</strong><small>${item.class_name || ''}</small><small>Status: ${item.status || 'unknown'} · File: ${item.original_filename || '—'}</small>`);
renderList('recent-recordings', data.recent_recordings || [], (item) => `<strong>${item.original_filename || 'Recording'}</strong><small>${item.session_date || ''}</small><small>Transcription: ${item.transcription_status || '—'} · Summary: ${item.summary_status || '—'}</small>`);
}
async function wireSyncButton() {
const button = document.getElementById('sync-button');
if (!button) return;
button.addEventListener('click', async () => {
button.disabled = true;
button.textContent = 'Refreshing…';
try {
const result = await fetchJson('/api/d2l/sync', { method: 'POST' });
const el = document.getElementById('sync-status');
if (el) el.textContent = `Last sync: just now · ${result.assignment_count} assignment rows · backend ${result.backend}`;
await loadDashboard();
} catch (error) {
const el = document.getElementById('sync-status');
if (el) el.textContent = `Sync failed: ${error}`;
} finally {
button.disabled = false;
button.textContent = 'Refresh from D2L';
}
});
}
populateCourses().catch((error) => {
console.error('Failed to populate D2L selectors', error);
});
loadDashboard().catch((error) => {
console.error('Failed to load dashboard summary', error);
});
wireSyncButton();

View File

@@ -0,0 +1,23 @@
{% extends 'base.html' %}
{% block content %}
<section class="card form-card">
<h2>Assignment Upload</h2>
<form class="stack" method="post" enctype="multipart/form-data" action="/api/assignments/intake" id="assignment-upload-form">
<label>Class
<select name="class_id" id="class-select" required>
<option value="">Loading classes…</option>
</select>
<input type="hidden" name="class_name" id="class-name-input">
</label>
<label>Assignment
<select name="assignment_name" id="assignment-select" required>
<option value="">Pick a class first</option>
</select>
</label>
<label>Version <input name="version" value="v1"></label>
<label>Notes <textarea name="notes" rows="4" placeholder="Anything useful about this turn-in."></textarea></label>
<label>File <input type="file" name="file" required></label>
<button type="submit">Upload for metadata confirmation</button>
</form>
</section>
{% endblock %}

View File

@@ -0,0 +1,26 @@
<!doctype html>
<html lang="en">
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>{{ title or 'Doris Schoolhouse' }}</title>
<link rel="stylesheet" href="/static/app.css">
</head>
<body>
<header class="site-header">
<div>
<h1>Doris Schoolhouse</h1>
<p>School intake, recordings, D2L sync, and Paperless linkage.</p>
</div>
<nav>
<a href="/">Dashboard</a>
<a href="/assignments/upload">Assignment Upload</a>
<a href="/recordings/upload">Recording Upload</a>
</nav>
</header>
<main class="page-shell">
{% block content %}{% endblock %}
</main>
<script src="/static/app.js"></script>
</body>
</html>

View File

@@ -0,0 +1,42 @@
{% extends 'base.html' %}
{% block content %}
<section class="hero">
<div class="card">
<h2>Overview</h2>
<div class="stat-grid" id="summary-stats">
<div><strong>Courses</strong><span id="stat-courses"></span></div>
<div><strong>Assignments</strong><span id="stat-assignments"></span></div>
<div><strong>Submissions</strong><span id="stat-submissions"></span></div>
<div><strong>Recordings</strong><span id="stat-recordings"></span></div>
</div>
<p class="muted" id="backend-note">Loading backend…</p>
<button id="sync-button" type="button">Refresh from D2L</button>
<p class="muted" id="sync-status">No sync run loaded yet.</p>
</div>
<div class="card">
<h2>What this app currently does</h2>
<ul>
<li>Reads current D2L scrape data</li>
<li>Caches course and assignment rows</li>
<li>Stores assignment upload intake records</li>
<li>Stores recording upload intake records</li>
<li>Builds Paperless/n8n handoff payloads</li>
</ul>
</div>
</section>
<section class="three-up">
<div class="card">
<h2>Recent assignments</h2>
<div id="recent-assignments" class="list-block muted">Loading…</div>
</div>
<div class="card">
<h2>Recent submissions</h2>
<div id="recent-submissions" class="list-block muted">Loading…</div>
</div>
<div class="card">
<h2>Recent recordings</h2>
<div id="recent-recordings" class="list-block muted">Loading…</div>
</div>
</section>
{% endblock %}

View File

@@ -0,0 +1,19 @@
{% extends 'base.html' %}
{% block content %}
<section class="card form-card">
<h2>Recording Upload</h2>
<form class="stack" method="post" enctype="multipart/form-data" action="/api/recordings/intake">
<label>Class
<select name="class_id" id="recording-class-select" required>
<option value="">Loading classes…</option>
</select>
<input type="hidden" name="class_name" id="recording-class-name-input">
</label>
<label>Session date <input type="date" name="session_date" required></label>
<label>Related assignment ID (optional) <input type="text" name="assignment_id"></label>
<label>Notes <textarea name="notes" rows="4" placeholder="Lecture, lab, exam review, etc."></textarea></label>
<label>WAV file <input type="file" name="file" accept="audio/wav,.wav" required></label>
<button type="submit">Upload and queue conversion</button>
</form>
</section>
{% endblock %}