Stabilize Firecrawl startup on PD

- catch Paperless handoff exceptions during assignment intake
- preserve local submission records when archive handoff fails
- redirect HTML uploads back to the form with readable status details
- show archive-pending messaging instead of a raw 500 page

home/doris-schoolhouse/app/routes/api_assignments.py

@@ -1,6 +1,7 @@
 import hashlib
 from pathlib import Path
 from typing import Any
+from urllib.parse import quote_plus
 
 from fastapi import APIRouter, File, Form, HTTPException, Request, UploadFile
 from fastapi.responses import RedirectResponse
@@ -40,33 +41,49 @@ def _confirm_submission_to_paperless(current: dict) -> dict:
         if value:
             fields[optional_key] = value
     artifact_kind = handoff.get("artifact_kind") or "document"
-    if artifact_kind == "source_bundle":
+    response: dict[str, Any]
-        paperless_response = paperless.upload_source_bundle_receipt(current)
+    body_obj: dict[str, Any]
-        enrichment = paperless.enrich_direct_submission(current, paperless_response)
+    ok = False
+    error_detail = None
+    try:
+        if artifact_kind == "source_bundle":
+            paperless_response = paperless.upload_source_bundle_receipt(current)
+            enrichment = paperless.enrich_direct_submission(current, paperless_response)
+            response = {
+                "ok": True,
+                "status_code": 200,
+                "body": paperless_response,
+                "mode": "direct-paperless",
+                "enrichment": enrichment,
+            }
+            task_id = paperless_response.get("task_id") or paperless_response.get("id")
+            body_obj = {
+                "paperless_document_id": enrichment.get("document_id"),
+                "paperless_title": enrichment.get("title") or paperless.build_submission_title(current),
+                "paperless_task_id": task_id,
+                "paperless_metadata": enrichment.get("metadata_payload"),
+            }
+            ok = True
+        else:
+            if handoff.get("artifact_kind"):
+                fields["artifact_kind"] = str(handoff["artifact_kind"])
+            if handoff.get("file_count"):
+                fields["file_count"] = str(handoff["file_count"])
+            response = webhooks.post_multipart(handoff["target_url"], fields, "document", file_path)
+            body = response.get("body", {}) if isinstance(response, dict) else {}
+            body_obj = body[0] if isinstance(body, list) and body and isinstance(body[0], dict) else (body if isinstance(body, dict) else {})
+            ok = bool(response.get("ok", False)) if isinstance(response, dict) else False
+        if not ok:
+            error_detail = str(response.get("error") or response.get("body") or "Paperless confirmation failed.")
+    except Exception as exc:
+        error_detail = f"{type(exc).__name__}: {exc}"
         response = {
-            "ok": True,
+            "ok": False,
-            "status_code": 200,
+            "status_code": 500,
-            "body": paperless_response,
+            "mode": "direct-paperless" if artifact_kind == "source_bundle" else "webhook",
-            "mode": "direct-paperless",
+            "error": error_detail,
-            "enrichment": enrichment,
         }
-        task_id = paperless_response.get("task_id") or paperless_response.get("id")
+        body_obj = {}
-        body_obj = {
-            "paperless_document_id": enrichment.get("document_id"),
-            "paperless_title": enrichment.get("title") or paperless.build_submission_title(current),
-            "paperless_task_id": task_id,
-            "paperless_metadata": enrichment.get("metadata_payload"),
-        }
-        ok = True
-    else:
-        if handoff.get("artifact_kind"):
-            fields["artifact_kind"] = str(handoff["artifact_kind"])
-        if handoff.get("file_count"):
-            fields["file_count"] = str(handoff["file_count"])
-        response = webhooks.post_multipart(handoff["target_url"], fields, "document", file_path)
-        body = response.get("body", {}) if isinstance(response, dict) else {}
-        body_obj = body[0] if isinstance(body, list) and body and isinstance(body[0], dict) else (body if isinstance(body, dict) else {})
-        ok = bool(response.get("ok", False)) if isinstance(response, dict) else False
     updates = {
         "status": "paperless-queued" if ok else "paperless-error",
         "confirmed_at": store.now(),
@@ -76,6 +93,7 @@ def _confirm_submission_to_paperless(current: dict) -> dict:
         "paperless_task_id": body_obj.get("paperless_task_id"),
     }
     updated = repo.update_submission(str(current.get("id")), updates) or current
+    updated["paperless_error_detail"] = error_detail
     grade_note = None
     if ok and updated.get("assignment_id"):
         grade_note = paperless.append_grade_note_if_available(str(updated.get("assignment_id")))
@@ -87,6 +105,7 @@ def _confirm_submission_to_paperless(current: dict) -> dict:
         "webhook": [response],
         "grade_note": grade_note,
         "file_count": handoff.get("file_count") or 1,
+        "error_detail": error_detail,
     }
 
 
@@ -220,7 +239,10 @@ async def intake_assignment(
         "backend": repo.backend,
     }
     if "text/html" in accept:
-        redirect_url = "/assignments/upload?submitted=" + str(created["id"]) + "&file_count=" + str(bundle["file_count"]) + "&artifact_kind=" + str(created.get("artifact_kind", "document")) + "&paperless_status=" + str(created.get("status", "uploaded-local"))
+        status_value = str(created.get("status", "uploaded-local"))
+        redirect_url = "/assignments/upload?submitted=" + str(created["id"]) + "&file_count=" + str(bundle["file_count"]) + "&artifact_kind=" + str(created.get("artifact_kind", "document")) + "&paperless_status=" + status_value
+        if auto_confirmed and auto_confirmed.get("error_detail"):
+            redirect_url += "&paperless_error=" + quote_plus(str(auto_confirmed["error_detail"]))
         return RedirectResponse(url=redirect_url, status_code=303)
     return response_payload
 

home/doris-schoolhouse/app/templates/assignment_upload.html

@@ -1,12 +1,19 @@
 {% extends 'base.html' %}
 {% block content %}
 {% if request.query_params.get('submitted') %}
-<section class="card form-card notice-card success-card evidence-board">
+{% set paperless_status = request.query_params.get('paperless_status', 'uploaded-local') %}
-  <div class="case-legend"><span class="section-label">Submission packet</span><span class="pill">Filed</span></div>
+<section class="card form-card notice-card {{ 'success-card' if paperless_status == 'paperless-queued' else '' }} evidence-board">
-  <h2>Submission Created</h2>
+  <div class="case-legend"><span class="section-label">Submission packet</span><span class="pill">{{ 'Filed' if paperless_status == 'paperless-queued' else 'Saved locally' }}</span></div>
+  <h2>{{ 'Submission Created' if paperless_status == 'paperless-queued' else 'Submission Saved, Archive Pending' }}</h2>
   <p>Submission {{ request.query_params.get('submitted') }} was created successfully.</p>
   <p class="muted">Artifact: {{ request.query_params.get('artifact_kind', 'document') }} · Files: {{ request.query_params.get('file_count', '1') }}</p>
-  <p class="muted">Paperless status: {{ request.query_params.get('paperless_status', 'uploaded-local') }}</p>
+  <p class="muted">Paperless status: {{ paperless_status }}</p>
+  {% if paperless_status != 'paperless-queued' %}
+  <p class="muted">Schoolhouse kept the submission locally, but the archive handoff failed this time.</p>
+  {% if request.query_params.get('paperless_error') %}
+  <p class="muted">Error: {{ request.query_params.get('paperless_error') }}</p>
+  {% endif %}
+  {% endif %}
 </section>
 {% endif %}
 <section class="card form-card evidence-board">

search/README.md

@@ -27,6 +27,12 @@ Basic verification after updates:
 - optional functional scrape smoke test:
   - POST `{"url":"https://example.com"}` to `http://127.0.0.1:3302/v1/scrape`
 
+Firecrawl stability note on PD:
+- The upstream `harness.js --start-docker` launcher fans out into API + queue worker + extract worker + NUQ workers.
+- On PD, the default fan-out has caused first-start flaps (`Port 3002 did not become available within 60000ms`) and steady `WORKER STALLED` / `Can't accept connection due to RAM/CPU load` messages.
+- Keep the PD compose defaults conservative unless the host is re-sized: `NUM_WORKERS_PER_QUEUE=1`, `CRAWL_CONCURRENT_REQUESTS=1`, `MAX_CONCURRENT_JOBS=1`, `BROWSER_POOL_SIZE=1`, `NUQ_WORKER_COUNT=1`, and `HARNESS_STARTUP_TIMEOUT_MS=180000`.
+- After changing those values, force-recreate `firecrawl-api` and confirm `curl -sS http://127.0.0.1:3302/` answers on the first start without an immediate harness timeout.
+
 Rollback-first notes:
 - Back up `docker-compose.yaml` before live edits under `/home/truenas_admin/doris-backups/`
 - Back up `settings.yml` before SearXNG config edits under `/home/truenas_admin/doris-backups/`

search/docker-compose.yaml

@@ -126,10 +126,11 @@ services:
       - POSTGRES_HOST=firecrawl-postgres
       - POSTGRES_PORT=5432
       - USE_DB_AUTHENTICATION=${USE_DB_AUTHENTICATION:-false}
-      - NUM_WORKERS_PER_QUEUE=${NUM_WORKERS_PER_QUEUE:-4}
+      - NUM_WORKERS_PER_QUEUE=${NUM_WORKERS_PER_QUEUE:-1}
-      - CRAWL_CONCURRENT_REQUESTS=${CRAWL_CONCURRENT_REQUESTS:-4}
+      - CRAWL_CONCURRENT_REQUESTS=${CRAWL_CONCURRENT_REQUESTS:-1}
-      - MAX_CONCURRENT_JOBS=${MAX_CONCURRENT_JOBS:-2}
+      - MAX_CONCURRENT_JOBS=${MAX_CONCURRENT_JOBS:-1}
-      - BROWSER_POOL_SIZE=${BROWSER_POOL_SIZE:-2}
+      - BROWSER_POOL_SIZE=${BROWSER_POOL_SIZE:-1}
+      - NUQ_WORKER_COUNT=${NUQ_WORKER_COUNT:-1}
       - BULL_AUTH_KEY=${BULL_AUTH_KEY}
       - TEST_API_KEY=${TEST_API_KEY}
       - LOGGING_LEVEL=${LOGGING_LEVEL:-info}
@@ -138,7 +139,7 @@ services:
       - SEARXNG_CATEGORIES=${SEARXNG_CATEGORIES:-}
       - ALLOW_LOCAL_WEBHOOKS=${ALLOW_LOCAL_WEBHOOKS:-false}
       - BLOCK_MEDIA=${BLOCK_MEDIA:-true}
-      - HARNESS_STARTUP_TIMEOUT_MS=60000
+      - HARNESS_STARTUP_TIMEOUT_MS=${HARNESS_STARTUP_TIMEOUT_MS:-180000}
       - NUQ_RABBITMQ_URL=amqp://firecrawl-rabbitmq:5672
     command: node dist/src/harness.js --start-docker