Finish Schoolhouse programming submission flow

This commit is contained in:
Fizzlepoof
2026-05-16 19:13:10 +00:00
parent d31de19227
commit e5a100d1a0
15 changed files with 753 additions and 74 deletions

View File

@@ -0,0 +1,172 @@
import hashlib
import json
from pathlib import Path
from zipfile import ZIP_DEFLATED, ZipFile
def slugify(value: str, fallback: str = "item") -> str:
cleaned = "".join(ch.lower() if ch.isalnum() else "-" for ch in (value or fallback))
parts = [part for part in cleaned.split("-") if part]
return "-".join(parts)[:80] or fallback
class SubmissionBundleService:
def _wrap_pdf_text(self, text: str, width: int = 96) -> list[str]:
wrapped: list[str] = []
for raw_line in text.splitlines():
line = raw_line.rstrip()
if not line:
wrapped.append("")
continue
while len(line) > width:
wrapped.append(line[:width])
line = line[width:]
wrapped.append(line)
return wrapped
def _escape_pdf_text(self, value: str) -> str:
return value.replace("\\", "\\\\").replace("(", "\\(").replace(")", "\\)")
def _write_text_pdf(self, path: Path, title: str, body: str) -> None:
lines = self._wrap_pdf_text(f"{title}\n\n{body}")
page_height = 792
top = 756
line_height = 12
lines_per_page = 58
pages = [lines[i:i + lines_per_page] for i in range(0, len(lines), lines_per_page)] or [[""]]
objects: list[bytes] = []
page_object_numbers: list[int] = []
object_number = 4
for page_lines in pages:
content_lines = ["BT", "/F1 10 Tf", f"72 {top} Td"]
first = True
for line in page_lines:
escaped = self._escape_pdf_text(line)
if first:
content_lines.append(f"({escaped}) Tj")
first = False
else:
content_lines.append(f"0 -{line_height} Td ({escaped}) Tj")
content_lines.append("ET")
content_bytes = "\n".join(content_lines).encode("utf-8")
page_object_numbers.append(object_number)
objects.append(
f"{object_number} 0 obj\n<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 {page_height}] /Contents {object_number + 1} 0 R /Resources << /Font << /F1 3 0 R >> >> >>\nendobj\n".encode("utf-8")
)
objects.append(
f"{object_number + 1} 0 obj\n<< /Length {len(content_bytes)} >>\nstream\n".encode("utf-8") + content_bytes + b"\nendstream\nendobj\n"
)
object_number += 2
pages_kids = " ".join(f"{num} 0 R" for num in page_object_numbers)
header = b"%PDF-1.4\n"
root_objects = [
b"1 0 obj\n<< /Type /Catalog /Pages 2 0 R >>\nendobj\n",
f"2 0 obj\n<< /Type /Pages /Kids [{pages_kids}] /Count {len(page_object_numbers)} >>\nendobj\n".encode("utf-8"),
b"3 0 obj\n<< /Type /Font /Subtype /Type1 /BaseFont /Courier >>\nendobj\n",
]
all_objects = root_objects + objects
offsets = []
current = len(header)
for obj in all_objects:
offsets.append(current)
current += len(obj)
xref_start = current
xref_lines = [b"xref\n", f"0 {len(all_objects) + 1}\n".encode("utf-8"), b"0000000000 65535 f \n"]
for offset in offsets:
xref_lines.append(f"{offset:010d} 00000 n \n".encode("utf-8"))
trailer = f"trailer\n<< /Size {len(all_objects) + 1} /Root 1 0 R >>\nstartxref\n{xref_start}\n%%EOF\n".encode("utf-8")
pdf_bytes = header + b"".join(all_objects) + b"".join(xref_lines) + trailer
path.write_bytes(pdf_bytes)
def build_bundle(
self,
*,
upload_root: Path,
submission_id: str,
class_name: str,
assignment_name: str,
version_label: str,
uploads: list[dict],
notes: str = "",
) -> dict:
bundle_root = upload_root / "submission-bundles" / submission_id
source_root = bundle_root / "source"
source_root.mkdir(parents=True, exist_ok=True)
manifest_files = []
combined_hash = hashlib.sha256()
for item in uploads:
source_path = source_root / item["filename"]
source_path.write_bytes(item["payload"])
combined_hash.update(item["payload"])
manifest_files.append(
{
"filename": item["filename"],
"stored_path": str(source_path),
"mime_type": item["mime_type"],
"size_bytes": len(item["payload"]),
"checksum_sha256": hashlib.sha256(item["payload"]).hexdigest(),
}
)
source_listing_path = bundle_root / f"{slugify(assignment_name, 'assignment')}-{slugify(version_label, 'v1')}-source-listing.txt"
listing_parts = []
for item in manifest_files:
listing_parts.append(f"===== {item['filename']} =====\n")
listing_parts.append(Path(item["stored_path"]).read_text(encoding="utf-8", errors="replace"))
listing_parts.append("\n\n")
source_listing_path.write_text("".join(listing_parts), encoding="utf-8")
manifest_path = bundle_root / "manifest.json"
manifest = {
"class_name": class_name,
"assignment_name": assignment_name,
"version_label": version_label,
"notes": notes,
"file_count": len(manifest_files),
"files": manifest_files,
"source_listing_path": str(source_listing_path),
}
manifest_path.write_text(json.dumps(manifest, indent=2, sort_keys=True), encoding="utf-8")
bundle_name = f"{slugify(class_name, 'class')}-{slugify(assignment_name, 'assignment')}-{slugify(version_label, 'v1')}.zip"
bundle_path = bundle_root / bundle_name
with ZipFile(bundle_path, "w", compression=ZIP_DEFLATED) as archive:
for item in manifest_files:
archive.write(item["stored_path"], arcname=item["filename"])
archive.write(source_listing_path, arcname=source_listing_path.name)
archive.write(manifest_path, arcname="manifest.json")
receipt_pdf_path = bundle_root / f"{slugify(assignment_name, 'assignment')}-{slugify(version_label, 'v1')}-submission-receipt.pdf"
receipt_body = "\n".join([
f"Class: {class_name}",
f"Assignment: {assignment_name}",
f"Version: {version_label}",
f"Bundle file: {bundle_name}",
f"Source file count: {len(manifest_files)}",
"",
"Included files:",
*[f"- {item['filename']} ({item['size_bytes']} bytes)" for item in manifest_files],
"",
"Source listing:",
source_listing_path.read_text(encoding="utf-8", errors="replace"),
])
self._write_text_pdf(receipt_pdf_path, f"{class_name}{assignment_name}{version_label}", receipt_body)
return {
"artifact_kind": "source_bundle",
"bundle_path": str(bundle_path),
"bundle_filename": bundle_name,
"bundle_mime_type": "application/zip",
"bundle_size_bytes": bundle_path.stat().st_size,
"bundle_checksum_sha256": hashlib.sha256(bundle_path.read_bytes()).hexdigest(),
"file_manifest_json": manifest_files,
"file_count": len(manifest_files),
"source_root": str(source_root),
"source_listing_path": str(source_listing_path),
"receipt_pdf_path": str(receipt_pdf_path),
"manifest_path": str(manifest_path),
"combined_source_checksum_sha256": combined_hash.hexdigest(),
}

View File

@@ -91,12 +91,23 @@ class D2LService:
"status": "completed",
})
self.repo.save_sync_runs(sync_runs)
grade_note_results = []
try:
from app.services.paperless import PaperlessService
paperless = PaperlessService()
for item in assignments:
result = paperless.append_grade_note_if_available(str(item.get("id")))
if result.get("status") != "skipped":
grade_note_results.append(result)
except Exception as exc:
grade_note_results.append({"status": "error", "reason": str(exc)})
return {
"status": "ok",
"snapshot_path": snapshot.get("source_path"),
"course_count": len(courses),
"assignment_count": len(assignments),
"backend": self.repo.backend,
"grade_note_results": grade_note_results,
}
def get_courses(self) -> list[dict[str, Any]]:

View File

@@ -1,4 +1,5 @@
import hashlib
import re
from urllib.parse import urljoin
from app.config import settings
@@ -14,6 +15,10 @@ class PaperlessService:
self.client = PaperlessClient()
def assignment_intake_payload(self, submission: dict) -> dict:
local_files = submission.get("stored_paths") or []
if not local_files and submission.get("stored_path"):
local_files = [submission.get("stored_path")]
local_file = submission.get("receipt_pdf_path") or submission.get("bundle_path") or submission.get("stored_path")
return {
"target_url": urljoin(settings.n8n_base_url, settings.n8n_assignment_webhook),
"class_name": submission.get("class_name"),
@@ -26,8 +31,14 @@ class PaperlessService:
"source": submission.get("source") or "doris-schoolhouse",
"telegram_chat_id": submission.get("telegram_chat_id"),
"telegram_message_id": submission.get("telegram_message_id"),
"local_file": submission.get("stored_path"),
"local_file": local_file,
"local_files": local_files,
"version": submission.get("version_label"),
"batch_id": submission.get("batch_id"),
"file_count": submission.get("file_count"),
"artifact_kind": submission.get("artifact_kind") or "document",
"source_listing_path": submission.get("source_listing_path"),
"receipt_pdf_path": submission.get("receipt_pdf_path"),
}
def recording_intake_payload(self, recording: dict) -> dict:
@@ -40,21 +51,122 @@ class PaperlessService:
"source": "doris-schoolhouse",
}
def upload_source_bundle_receipt(self, submission: dict) -> dict:
file_path = submission.get("receipt_pdf_path") or submission.get("bundle_path") or submission.get("stored_path")
title = self.build_submission_title(submission)
return self.client.upload_document(file_path, title)
def _course_short_label(self, submission: dict) -> str:
course_code = str(submission.get("course_code") or "").strip()
match = re.search(r"([A-Z]{3,4})_([A-Z]{3,4})_(\d{4})", course_code)
if match:
return f"{match.group(2)}-{match.group(3)}"
class_name = str(submission.get("class_name") or "").strip()
match = re.search(r"([A-Za-z]+(?:\s+[A-Za-z]+)*)\s+(\d{4})", class_name)
if not match:
return class_name
words = match.group(1).split()
prefix = ''.join(word[0] for word in words[:4]).upper()
return f"{prefix}-{match.group(2)}"
def _course_tag_name(self, submission: dict) -> str | None:
label = self._course_short_label(submission)
if not label or "-" not in label:
return None
return label.replace("-", " ").lower()
def build_submission_title(self, submission: dict) -> str:
return "".join(
part for part in [
self._course_short_label(submission) or submission.get("class_name"),
submission.get("assignment_name"),
submission.get("version_label"),
"source bundle" if submission.get("artifact_kind") == "source_bundle" else submission.get("submission_kind"),
] if part
)
def build_direct_metadata_payload(self, submission: dict, current_document: dict | None = None) -> dict:
current_document = current_document or {}
known_tags = {
str(tag.get("name", "")).strip().casefold(): tag
for tag in self.client.list_tags()
}
merged_tags = {
int(tag_id)
for tag_id in (current_document.get("tags") or [])
if str(tag_id).isdigit()
}
for tag_name in ("school", "assignments", "spring-2026"):
tag = known_tags.get(tag_name)
if tag and tag.get("id"):
merged_tags.add(int(tag["id"]))
course_tag_name = self._course_tag_name(submission)
if course_tag_name:
tag = known_tags.get(course_tag_name)
if tag and tag.get("id"):
merged_tags.add(int(tag["id"]))
if submission.get("artifact_kind") == "source_bundle":
for tag_name in ("programming", "java"):
tag = known_tags.get(tag_name)
if tag and tag.get("id"):
merged_tags.add(int(tag["id"]))
payload = {
"title": self.build_submission_title(submission),
"tags": sorted(merged_tags),
}
for doc_type in self.client.list_document_types():
if str(doc_type.get("name", "")).strip().casefold() == "school" and doc_type.get("id"):
payload["document_type"] = int(doc_type["id"])
break
for correspondent in self.client.list_correspondents():
if str(correspondent.get("name", "")).strip().casefold() == "austin peay state university" and correspondent.get("id"):
payload["correspondent"] = int(correspondent["id"])
break
return payload
def enrich_direct_submission(self, submission: dict, upload_response: dict | None = None) -> dict:
upload_response = upload_response or {}
task_id = upload_response.get("task_id") or upload_response.get("id") or submission.get("paperless_task_id")
if not task_id:
raise ValueError("Paperless upload did not return a task id")
task = self.client.wait_for_task(str(task_id))
status = str(task.get("status") or "").upper()
document_id = task.get("related_document")
if status != "SUCCESS":
duplicate_match = re.search(r"#(\d+)\)", str(task.get("result") or ""))
if duplicate_match:
document_id = duplicate_match.group(1)
else:
raise ValueError(f"Paperless task did not complete successfully: {status or 'unknown'}")
if not document_id:
raise ValueError("Paperless task completed without a related document id")
current = self.client.get_document(document_id)
payload = self.build_direct_metadata_payload(submission, current)
updated = self.client.update_document(document_id, payload)
return {
"task": task,
"document": updated,
"document_id": updated.get("id") or current.get("id") or int(document_id),
"title": updated.get("title") or payload.get("title"),
"metadata_payload": payload,
}
def build_grade_note(self, assignment_id: str) -> str:
submissions = self.repo.list_submissions()
assignments = {item.get("id"): item for item in self.repo.list_assignments()}
assignment = assignments.get(str(assignment_id), {})
linked = [s for s in submissions if str(s.get("assignment_id")) == str(assignment_id)]
note_lines = [
"Grade sync from Doris Schoolhouse",
"D2L grade note imported from Doris Schoolhouse.",
f"Assignment: {assignment.get('title', 'unknown')}",
]
if assignment.get("grade"):
note_lines.append(f"Grade: {assignment['grade']}")
if assignment.get("points"):
note_lines.append(f"Points: {assignment['points']}")
if assignment.get("comments"):
note_lines.append(f"Professor comments: {assignment['comments']}")
comments = str(assignment.get("comments") or "").strip()
if comments:
note_lines.append(f"Comments: {comments}")
else:
note_lines.append("Comments: No professor comment captured in the D2L scrape.")
if linked:
note_lines.append(f"Linked submissions: {len(linked)}")
return "\n".join(note_lines)
@@ -107,3 +219,17 @@ class PaperlessService:
"note_hash": note_hash,
"paperless_response": response,
}
def append_grade_note_if_available(self, assignment_id: str) -> dict:
assignments = {item.get("id"): item for item in self.repo.list_assignments()}
assignment = assignments.get(str(assignment_id), {})
if not assignment:
return {"status": "skipped", "reason": "assignment-not-found", "assignment_id": assignment_id}
if not (assignment.get("grade") or str(assignment.get("comments") or "").strip()):
return {"status": "skipped", "reason": "no-grade-or-comments", "assignment_id": assignment_id}
linked = [s for s in self.repo.list_submissions() if str(s.get("assignment_id")) == str(assignment_id)]
if not linked:
return {"status": "skipped", "reason": "no-linked-submissions", "assignment_id": assignment_id}
if not any(item.get("paperless_document_id") for item in linked):
return {"status": "skipped", "reason": "no-paperless-document-id", "assignment_id": assignment_id}
return self.append_grade_note(assignment_id)

View File

@@ -1,4 +1,7 @@
import json
import mimetypes
import time
from pathlib import Path
from urllib import request
from urllib.parse import urljoin
@@ -27,5 +30,96 @@ class PaperlessClient:
def get_document(self, document_id: int | str) -> dict:
return self._req('GET', f'/api/documents/{document_id}/')
def update_document(self, document_id: int | str, payload: dict) -> dict:
return self._req('PATCH', f'/api/documents/{document_id}/', payload)
def add_note(self, document_id: int | str, note: str) -> dict:
return self._req('POST', f'/api/documents/{document_id}/notes/', {'note': note})
def _list_endpoint(self, path: str) -> list[dict]:
data = self._req('GET', path)
if isinstance(data, list):
return data
if isinstance(data, dict):
results = data.get('results')
if isinstance(results, list):
return results
return []
def list_tags(self) -> list[dict]:
return self._list_endpoint('/api/tags/?page_size=200')
def create_tag(self, name: str) -> dict:
return self._req('POST', '/api/tags/', {'name': name})
def get_or_create_tag(self, name: str) -> dict:
needle = name.strip().casefold()
for tag in self.list_tags():
if str(tag.get('name', '')).strip().casefold() == needle:
return tag
return self.create_tag(name)
def list_document_types(self) -> list[dict]:
return self._list_endpoint('/api/document_types/?page_size=200')
def list_correspondents(self) -> list[dict]:
return self._list_endpoint('/api/correspondents/?page_size=200')
def list_tasks(self, task_id: str | None = None) -> list[dict]:
path = '/api/tasks/?page_size=100'
if task_id:
path += f'&task_id={task_id}'
return self._list_endpoint(path)
def wait_for_task(self, task_id: str, timeout_seconds: int = 90, poll_seconds: float = 1.0) -> dict:
deadline = time.time() + timeout_seconds
last = {}
while time.time() < deadline:
tasks = self.list_tasks(task_id=task_id)
if tasks:
last = tasks[0]
status = str(last.get('status') or '').upper()
if status in {'SUCCESS', 'FAILURE', 'CANCELLED'}:
return last
time.sleep(poll_seconds)
return last
def upload_document(self, file_path: str, title: str) -> dict:
path = Path(file_path)
boundary = "----DorisSchoolhousePaperlessUpload"
mime = mimetypes.guess_type(path.name)[0] or "application/octet-stream"
parts: list[bytes] = []
for key, value in {"title": title}.items():
parts.extend([
f"--{boundary}\r\n".encode(),
f'Content-Disposition: form-data; name="{key}"\r\n\r\n'.encode(),
str(value).encode(),
b"\r\n",
])
parts.extend([
f"--{boundary}\r\n".encode(),
f'Content-Disposition: form-data; name="document"; filename="{path.name}"\r\n'.encode(),
f"Content-Type: {mime}\r\n\r\n".encode(),
path.read_bytes(),
b"\r\n",
f"--{boundary}--\r\n".encode(),
])
body = b"".join(parts)
req = request.Request(
urljoin(self.base_url, "/api/documents/post_document/"),
data=body,
headers={
"Authorization": f"Token {self.token}",
"Accept": "application/json",
"Content-Type": f"multipart/form-data; boundary={boundary}",
},
method="POST",
)
with request.urlopen(req, timeout=120) as resp:
raw = resp.read().decode("utf-8", errors="replace")
if not raw:
return {}
parsed = json.loads(raw)
if isinstance(parsed, str):
return {"task_id": parsed}
return parsed

View File

@@ -227,6 +227,23 @@ class PostgresRepository:
s.paperless_note_last_hash,
s.submitted_at,
s.confirmed_metadata_json,
s.confirmed_metadata_json->>'batch_id' AS batch_id,
s.confirmed_metadata_json->>'file_count' AS file_count,
s.confirmed_metadata_json->>'file_index' AS file_index,
s.confirmed_metadata_json->>'submission_kind' AS submission_kind,
s.confirmed_metadata_json->>'artifact_kind' AS artifact_kind,
s.confirmed_metadata_json->>'course_code' AS course_code,
s.confirmed_metadata_json->>'paper_kind' AS paper_kind,
s.confirmed_metadata_json->>'source' AS source,
s.confirmed_metadata_json->>'notes' AS notes,
s.confirmed_metadata_json->>'received_at' AS received_at,
s.confirmed_metadata_json->>'bundle_path' AS bundle_path,
s.confirmed_metadata_json->>'bundle_filename' AS bundle_filename,
s.confirmed_metadata_json->>'source_listing_path' AS source_listing_path,
s.confirmed_metadata_json->>'receipt_pdf_path' AS receipt_pdf_path,
s.confirmed_metadata_json->>'paperless_task_id' AS paperless_task_id,
s.confirmed_metadata_json->'file_manifest_json' AS file_manifest_json,
s.confirmed_metadata_json->'stored_paths' AS stored_paths,
s.confirmed_metadata_json->>'stored_path' AS stored_path,
s.confirmed_metadata_json->>'local_file' AS local_file
FROM schoolhouse_submission s

View File

@@ -6,8 +6,7 @@ from urllib import error, request
class WebhookService:
def _multipart_body(self, fields: dict[str, str], file_field: str, file_path: str) -> tuple[bytes, str]:
boundary = f"----DorisSchoolhouse{uuid.uuid4().hex}"
def _multipart_parts_for_fields(self, boundary: str, fields: dict[str, str]) -> list[bytes]:
parts: list[bytes] = []
for key, value in fields.items():
parts.extend([
@@ -16,16 +15,32 @@ class WebhookService:
str(value).encode(),
b"\r\n",
])
return parts
def _multipart_parts_for_file(self, boundary: str, file_field: str, file_path: str) -> list[bytes]:
path = Path(file_path)
mime = mimetypes.guess_type(path.name)[0] or "application/octet-stream"
parts.extend([
return [
f"--{boundary}\r\n".encode(),
f'Content-Disposition: form-data; name="{file_field}"; filename="{path.name}"\r\n'.encode(),
f"Content-Type: {mime}\r\n\r\n".encode(),
path.read_bytes(),
b"\r\n",
f"--{boundary}--\r\n".encode(),
])
]
def _multipart_body(self, fields: dict[str, str], file_field: str, file_path: str) -> tuple[bytes, str]:
boundary = f"----DorisSchoolhouse{uuid.uuid4().hex}"
parts = self._multipart_parts_for_fields(boundary, fields)
parts.extend(self._multipart_parts_for_file(boundary, file_field, file_path))
parts.append(f"--{boundary}--\r\n".encode())
return b"".join(parts), boundary
def _multipart_body_many(self, fields: dict[str, str], file_field: str, file_paths: list[str]) -> tuple[bytes, str]:
boundary = f"----DorisSchoolhouse{uuid.uuid4().hex}"
parts = self._multipart_parts_for_fields(boundary, fields)
for file_path in file_paths:
parts.extend(self._multipart_parts_for_file(boundary, file_field, file_path))
parts.append(f"--{boundary}--\r\n".encode())
return b"".join(parts), boundary
def post_multipart(self, url: str, fields: dict[str, str], file_field: str, file_path: str) -> dict:
@@ -50,3 +65,26 @@ class WebhookService:
except error.HTTPError as exc:
raw = exc.read().decode("utf-8", errors="replace")
return {"ok": False, "status_code": exc.code, "body": parse_raw(raw), "error": str(exc)}
def post_multipart_many(self, url: str, fields: dict[str, str], file_field: str, file_paths: list[str]) -> dict:
body, boundary = self._multipart_body_many(fields, file_field, file_paths)
req = request.Request(
url,
data=body,
headers={"Content-Type": f"multipart/form-data; boundary={boundary}"},
method="POST",
)
def parse_raw(raw: str):
try:
return json.loads(raw)
except Exception:
return {"raw": raw}
try:
with request.urlopen(req, timeout=120) as response:
raw = response.read().decode("utf-8", errors="replace")
return {"ok": True, "status_code": response.status, "body": parse_raw(raw)}
except error.HTTPError as exc:
raw = exc.read().decode("utf-8", errors="replace")
return {"ok": False, "status_code": exc.code, "body": parse_raw(raw), "error": str(exc)}