From 28b2edf2a3cb7615ceb4e55b7bad228871311545 Mon Sep 17 00:00:00 2001 From: Fizzlepoof Date: Sun, 7 Jun 2026 04:14:23 +0000 Subject: [PATCH] Stabilize Firecrawl startup on PD --- search/README.md | 6 ++++++ search/docker-compose.yaml | 11 ++++++----- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/search/README.md b/search/README.md index 9ecfe72..46ce71f 100644 --- a/search/README.md +++ b/search/README.md @@ -27,6 +27,12 @@ Basic verification after updates: - optional functional scrape smoke test: - POST `{"url":"https://example.com"}` to `http://127.0.0.1:3302/v1/scrape` +Firecrawl stability note on PD: +- The upstream `harness.js --start-docker` launcher fans out into API + queue worker + extract worker + NUQ workers. +- On PD, the default fan-out has caused first-start flaps (`Port 3002 did not become available within 60000ms`) and steady `WORKER STALLED` / `Can't accept connection due to RAM/CPU load` messages. +- Keep the PD compose defaults conservative unless the host is re-sized: `NUM_WORKERS_PER_QUEUE=1`, `CRAWL_CONCURRENT_REQUESTS=1`, `MAX_CONCURRENT_JOBS=1`, `BROWSER_POOL_SIZE=1`, `NUQ_WORKER_COUNT=1`, and `HARNESS_STARTUP_TIMEOUT_MS=180000`. +- After changing those values, force-recreate `firecrawl-api` and confirm `curl -sS http://127.0.0.1:3302/` answers on the first start without an immediate harness timeout. + Rollback-first notes: - Back up `docker-compose.yaml` before live edits under `/home/truenas_admin/doris-backups/` - Back up `settings.yml` before SearXNG config edits under `/home/truenas_admin/doris-backups/` diff --git a/search/docker-compose.yaml b/search/docker-compose.yaml index 63f9a37..514daef 100644 --- a/search/docker-compose.yaml +++ b/search/docker-compose.yaml @@ -126,10 +126,11 @@ services: - POSTGRES_HOST=firecrawl-postgres - POSTGRES_PORT=5432 - USE_DB_AUTHENTICATION=${USE_DB_AUTHENTICATION:-false} - - NUM_WORKERS_PER_QUEUE=${NUM_WORKERS_PER_QUEUE:-4} - - CRAWL_CONCURRENT_REQUESTS=${CRAWL_CONCURRENT_REQUESTS:-4} - - MAX_CONCURRENT_JOBS=${MAX_CONCURRENT_JOBS:-2} - - BROWSER_POOL_SIZE=${BROWSER_POOL_SIZE:-2} + - NUM_WORKERS_PER_QUEUE=${NUM_WORKERS_PER_QUEUE:-1} + - CRAWL_CONCURRENT_REQUESTS=${CRAWL_CONCURRENT_REQUESTS:-1} + - MAX_CONCURRENT_JOBS=${MAX_CONCURRENT_JOBS:-1} + - BROWSER_POOL_SIZE=${BROWSER_POOL_SIZE:-1} + - NUQ_WORKER_COUNT=${NUQ_WORKER_COUNT:-1} - BULL_AUTH_KEY=${BULL_AUTH_KEY} - TEST_API_KEY=${TEST_API_KEY} - LOGGING_LEVEL=${LOGGING_LEVEL:-info} @@ -138,7 +139,7 @@ services: - SEARXNG_CATEGORIES=${SEARXNG_CATEGORIES:-} - ALLOW_LOCAL_WEBHOOKS=${ALLOW_LOCAL_WEBHOOKS:-false} - BLOCK_MEDIA=${BLOCK_MEDIA:-true} - - HARNESS_STARTUP_TIMEOUT_MS=60000 + - HARNESS_STARTUP_TIMEOUT_MS=${HARNESS_STARTUP_TIMEOUT_MS:-180000} - NUQ_RABBITMQ_URL=amqp://firecrawl-rabbitmq:5672 command: node dist/src/harness.js --start-docker