Add Firecrawl stack behind Gluetun on PD

2026-05-22 02:22:44 +00:00
parent f88b90b4d8
commit 8003f65f4d
3 changed files with 201 additions and 1 deletions
--- a/search/.env.example
+++ b/search/.env.example
@@ -0,0 +1,37 @@
+# Firecrawl + Gluetun stack variables
+TZ=America/Chicago
+
+# Existing SearXNG public/base URL
+SEARXNG_BASE_URL=http://10.5.1.6:8888/
+
+# Firecrawl API is exposed from the gluetun network namespace on this host port.
+# 3002 is already occupied by Gitea on PD, so Firecrawl uses 3302 externally.
+FIRECRAWL_PORT=3302
+FIRECRAWL_INTERNAL_PORT=3002
+
+# Keep the Gluetun control UI localhost-only by default. Reverse proxy later if desired.
+GLUETUN_WEBUI_PORT=8390
+
+# Firecrawl runtime tuning
+USE_DB_AUTHENTICATION=false
+NUM_WORKERS_PER_QUEUE=4
+CRAWL_CONCURRENT_REQUESTS=4
+MAX_CONCURRENT_JOBS=2
+BROWSER_POOL_SIZE=2
+LOGGING_LEVEL=info
+BLOCK_MEDIA=true
+ALLOW_LOCAL_WEBHOOKS=false
+
+# Allow VPN-routed containers to still reach local RFC1918 services when needed.
+FIREWALL_OUTBOUND_SUBNETS=10.5.0.0/16,172.16.0.0/12,192.168.0.0/16
+
+# Firecrawl secrets — replace in the live .env only, never in git.
+POSTGRES_USER=firecrawl
+POSTGRES_PASSWORD=replace-me
+POSTGRES_DB=firecrawl
+BULL_AUTH_KEY=replace-me-with-a-long-random-string
+TEST_API_KEY=
+
+# Optional SearXNG tuning
+SEARXNG_ENGINES=
+SEARXNG_CATEGORIES=
--- a/search/.gitignore
+++ b/search/.gitignore
@@ -0,0 +1 @@
+secrets/
--- a/search/docker-compose.yaml
+++ b/search/docker-compose.yaml
@@ -8,7 +8,7 @@ services:
    volumes:
      - /mnt/tank/docker/appdata/searxng:/etc/searxng
    environment:
-      - SEARXNG_BASE_URL=http://localhost:8888/
+      - SEARXNG_BASE_URL=${SEARXNG_BASE_URL:-http://10.5.1.6:8888/}
    cap_drop:
      - ALL
    cap_add:
@@ -17,7 +17,169 @@ services:
      - SETUID
    networks:
      - pangolin
+      - search_internal
+
+  search-gluetun:
+    image: ghcr.io/qdm12/gluetun:latest
+    container_name: search-gluetun
+    restart: unless-stopped
+    cap_add:
+      - NET_ADMIN
+    devices:
+      - /dev/net/tun:/dev/net/tun
+    ports:
+      - "${FIRECRAWL_PORT:-3302}:${FIRECRAWL_INTERNAL_PORT:-3002}"
+    volumes:
+      - /mnt/docker-ssd/docker/appdata/firecrawl/gluetun:/gluetun
+    environment:
+      - TZ=${TZ:-America/Chicago}
+      - VPN_SERVICE_PROVIDER=custom
+      - VPN_TYPE=wireguard
+      - WIREGUARD_CONF_SECRETFILE=/run/secrets/wg0.conf
+      - HTTP_CONTROL_SERVER_ADDRESS=:8000
+      - HTTP_CONTROL_SERVER_LOG=on
+      - FIREWALL_OUTBOUND_SUBNETS=${FIREWALL_OUTBOUND_SUBNETS:-10.5.0.0/16,172.16.0.0/12,192.168.0.0/16}
+      - UPDATER_PERIOD=24h
+    secrets:
+      - wg0.conf
+    networks:
+      - search_internal
+
+  gluetun-webui:
+    image: scuzza/gluetun-webui:latest
+    container_name: gluetun-webui
+    restart: unless-stopped
+    depends_on:
+      - search-gluetun
+    ports:
+      - "127.0.0.1:${GLUETUN_WEBUI_PORT:-8390}:3000"
+    environment:
+      - GLUETUN_CONTROL_URL=http://search-gluetun:8000
+      - PORT=3000
+    networks:
+      - search_internal
+    read_only: true
+    tmpfs:
+      - /tmp
+    security_opt:
+      - no-new-privileges:true
+    cap_drop:
+      - ALL
+    healthcheck:
+      test: ["CMD", "wget", "-qO-", "http://localhost:3000/api/health"]
+      interval: 30s
+      timeout: 5s
+      start_period: 10s
+      retries: 3
+
+  firecrawl-playwright-service:
+    image: ghcr.io/firecrawl/playwright-service:latest
+    container_name: firecrawl-playwright-service
+    restart: unless-stopped
+    depends_on:
+      - search-gluetun
+    network_mode: "service:search-gluetun"
+    environment:
+      - PORT=3000
+      - ALLOW_LOCAL_WEBHOOKS=${ALLOW_LOCAL_WEBHOOKS:-false}
+      - BLOCK_MEDIA=${BLOCK_MEDIA:-true}
+      - MAX_CONCURRENT_PAGES=${CRAWL_CONCURRENT_REQUESTS:-4}
+    tmpfs:
+      - /tmp/.cache:noexec,nosuid,size=1g
+
+  firecrawl-api:
+    image: ghcr.io/firecrawl/firecrawl:latest
+    container_name: firecrawl-api
+    restart: unless-stopped
+    depends_on:
+      firecrawl-redis:
+        condition: service_started
+      firecrawl-playwright-service:
+        condition: service_started
+      firecrawl-rabbitmq:
+        condition: service_healthy
+      firecrawl-postgres:
+        condition: service_started
+      search-gluetun:
+        condition: service_started
+    network_mode: "service:search-gluetun"
+    environment:
+      - HOST=0.0.0.0
+      - PORT=${FIRECRAWL_INTERNAL_PORT:-3002}
+      - EXTRACT_WORKER_PORT=3004
+      - WORKER_PORT=3005
+      - ENV=local
+      - REDIS_URL=redis://firecrawl-redis:6379
+      - REDIS_RATE_LIMIT_URL=redis://firecrawl-redis:6379
+      - PLAYWRIGHT_MICROSERVICE_URL=http://127.0.0.1:3000/scrape
+      - POSTGRES_USER=${POSTGRES_USER:-firecrawl}
+      - POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
+      - POSTGRES_DB=${POSTGRES_DB:-firecrawl}
+      - POSTGRES_HOST=firecrawl-postgres
+      - POSTGRES_PORT=5432
+      - USE_DB_AUTHENTICATION=${USE_DB_AUTHENTICATION:-false}
+      - NUM_WORKERS_PER_QUEUE=${NUM_WORKERS_PER_QUEUE:-4}
+      - CRAWL_CONCURRENT_REQUESTS=${CRAWL_CONCURRENT_REQUESTS:-4}
+      - MAX_CONCURRENT_JOBS=${MAX_CONCURRENT_JOBS:-2}
+      - BROWSER_POOL_SIZE=${BROWSER_POOL_SIZE:-2}
+      - BULL_AUTH_KEY=${BULL_AUTH_KEY}
+      - TEST_API_KEY=${TEST_API_KEY}
+      - LOGGING_LEVEL=${LOGGING_LEVEL:-info}
+      - SEARXNG_ENDPOINT=http://searxng:8080
+      - SEARXNG_ENGINES=${SEARXNG_ENGINES:-}
+      - SEARXNG_CATEGORIES=${SEARXNG_CATEGORIES:-}
+      - ALLOW_LOCAL_WEBHOOKS=${ALLOW_LOCAL_WEBHOOKS:-false}
+      - BLOCK_MEDIA=${BLOCK_MEDIA:-true}
+      - HARNESS_STARTUP_TIMEOUT_MS=60000
+      - NUQ_RABBITMQ_URL=amqp://firecrawl-rabbitmq:5672
+    command: node dist/src/harness.js --start-docker
+
+  firecrawl-redis:
+    image: redis:alpine
+    container_name: firecrawl-redis
+    restart: unless-stopped
+    command: redis-server --bind 0.0.0.0 --appendonly yes
+    volumes:
+      - /mnt/docker-ssd/docker/appdata/firecrawl/redis:/data
+    networks:
+      - search_internal
+
+  firecrawl-rabbitmq:
+    image: rabbitmq:3-management
+    container_name: firecrawl-rabbitmq
+    restart: unless-stopped
+    volumes:
+      - /mnt/docker-ssd/docker/appdata/firecrawl/rabbitmq:/var/lib/rabbitmq
+    command: rabbitmq-server
+    healthcheck:
+      test: ["CMD", "rabbitmq-diagnostics", "-q", "check_running"]
+      interval: 5s
+      timeout: 5s
+      retries: 3
+      start_period: 5s
+    networks:
+      - search_internal
+
+  firecrawl-postgres:
+    image: ghcr.io/firecrawl/nuq-postgres:latest
+    container_name: firecrawl-postgres
+    restart: unless-stopped
+    environment:
+      - POSTGRES_USER=${POSTGRES_USER:-firecrawl}
+      - POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
+      - POSTGRES_DB=${POSTGRES_DB:-firecrawl}
+    volumes:
+      - /mnt/docker-ssd/docker/appdata/firecrawl/postgres:/var/lib/postgresql/data
+    networks:
+      - search_internal

 networks:
  pangolin:
    external: true
+  search_internal:
+    name: search_internal
+    driver: bridge
+
+secrets:
+  wg0.conf:
+    file: ./secrets/wg0.conf