From 8003f65f4d068acbb15f24803a9d8a42ec4de630 Mon Sep 17 00:00:00 2001 From: Fizzlepoof Date: Fri, 22 May 2026 02:22:44 +0000 Subject: [PATCH] Add Firecrawl stack behind Gluetun on PD --- search/.env.example | 37 +++++++++ search/.gitignore | 1 + search/docker-compose.yaml | 164 ++++++++++++++++++++++++++++++++++++- 3 files changed, 201 insertions(+), 1 deletion(-) create mode 100644 search/.env.example create mode 100644 search/.gitignore diff --git a/search/.env.example b/search/.env.example new file mode 100644 index 0000000..0561453 --- /dev/null +++ b/search/.env.example @@ -0,0 +1,37 @@ +# Firecrawl + Gluetun stack variables +TZ=America/Chicago + +# Existing SearXNG public/base URL +SEARXNG_BASE_URL=http://10.5.1.6:8888/ + +# Firecrawl API is exposed from the gluetun network namespace on this host port. +# 3002 is already occupied by Gitea on PD, so Firecrawl uses 3302 externally. +FIRECRAWL_PORT=3302 +FIRECRAWL_INTERNAL_PORT=3002 + +# Keep the Gluetun control UI localhost-only by default. Reverse proxy later if desired. +GLUETUN_WEBUI_PORT=8390 + +# Firecrawl runtime tuning +USE_DB_AUTHENTICATION=false +NUM_WORKERS_PER_QUEUE=4 +CRAWL_CONCURRENT_REQUESTS=4 +MAX_CONCURRENT_JOBS=2 +BROWSER_POOL_SIZE=2 +LOGGING_LEVEL=info +BLOCK_MEDIA=true +ALLOW_LOCAL_WEBHOOKS=false + +# Allow VPN-routed containers to still reach local RFC1918 services when needed. +FIREWALL_OUTBOUND_SUBNETS=10.5.0.0/16,172.16.0.0/12,192.168.0.0/16 + +# Firecrawl secrets — replace in the live .env only, never in git. +POSTGRES_USER=firecrawl +POSTGRES_PASSWORD=replace-me +POSTGRES_DB=firecrawl +BULL_AUTH_KEY=replace-me-with-a-long-random-string +TEST_API_KEY= + +# Optional SearXNG tuning +SEARXNG_ENGINES= +SEARXNG_CATEGORIES= diff --git a/search/.gitignore b/search/.gitignore new file mode 100644 index 0000000..4bd922a --- /dev/null +++ b/search/.gitignore @@ -0,0 +1 @@ +secrets/ diff --git a/search/docker-compose.yaml b/search/docker-compose.yaml index ac5cd3c..9c305ff 100644 --- a/search/docker-compose.yaml +++ b/search/docker-compose.yaml @@ -8,7 +8,7 @@ services: volumes: - /mnt/tank/docker/appdata/searxng:/etc/searxng environment: - - SEARXNG_BASE_URL=http://localhost:8888/ + - SEARXNG_BASE_URL=${SEARXNG_BASE_URL:-http://10.5.1.6:8888/} cap_drop: - ALL cap_add: @@ -17,7 +17,169 @@ services: - SETUID networks: - pangolin + - search_internal + + search-gluetun: + image: ghcr.io/qdm12/gluetun:latest + container_name: search-gluetun + restart: unless-stopped + cap_add: + - NET_ADMIN + devices: + - /dev/net/tun:/dev/net/tun + ports: + - "${FIRECRAWL_PORT:-3302}:${FIRECRAWL_INTERNAL_PORT:-3002}" + volumes: + - /mnt/docker-ssd/docker/appdata/firecrawl/gluetun:/gluetun + environment: + - TZ=${TZ:-America/Chicago} + - VPN_SERVICE_PROVIDER=custom + - VPN_TYPE=wireguard + - WIREGUARD_CONF_SECRETFILE=/run/secrets/wg0.conf + - HTTP_CONTROL_SERVER_ADDRESS=:8000 + - HTTP_CONTROL_SERVER_LOG=on + - FIREWALL_OUTBOUND_SUBNETS=${FIREWALL_OUTBOUND_SUBNETS:-10.5.0.0/16,172.16.0.0/12,192.168.0.0/16} + - UPDATER_PERIOD=24h + secrets: + - wg0.conf + networks: + - search_internal + + gluetun-webui: + image: scuzza/gluetun-webui:latest + container_name: gluetun-webui + restart: unless-stopped + depends_on: + - search-gluetun + ports: + - "127.0.0.1:${GLUETUN_WEBUI_PORT:-8390}:3000" + environment: + - GLUETUN_CONTROL_URL=http://search-gluetun:8000 + - PORT=3000 + networks: + - search_internal + read_only: true + tmpfs: + - /tmp + security_opt: + - no-new-privileges:true + cap_drop: + - ALL + healthcheck: + test: ["CMD", "wget", "-qO-", "http://localhost:3000/api/health"] + interval: 30s + timeout: 5s + start_period: 10s + retries: 3 + + firecrawl-playwright-service: + image: ghcr.io/firecrawl/playwright-service:latest + container_name: firecrawl-playwright-service + restart: unless-stopped + depends_on: + - search-gluetun + network_mode: "service:search-gluetun" + environment: + - PORT=3000 + - ALLOW_LOCAL_WEBHOOKS=${ALLOW_LOCAL_WEBHOOKS:-false} + - BLOCK_MEDIA=${BLOCK_MEDIA:-true} + - MAX_CONCURRENT_PAGES=${CRAWL_CONCURRENT_REQUESTS:-4} + tmpfs: + - /tmp/.cache:noexec,nosuid,size=1g + + firecrawl-api: + image: ghcr.io/firecrawl/firecrawl:latest + container_name: firecrawl-api + restart: unless-stopped + depends_on: + firecrawl-redis: + condition: service_started + firecrawl-playwright-service: + condition: service_started + firecrawl-rabbitmq: + condition: service_healthy + firecrawl-postgres: + condition: service_started + search-gluetun: + condition: service_started + network_mode: "service:search-gluetun" + environment: + - HOST=0.0.0.0 + - PORT=${FIRECRAWL_INTERNAL_PORT:-3002} + - EXTRACT_WORKER_PORT=3004 + - WORKER_PORT=3005 + - ENV=local + - REDIS_URL=redis://firecrawl-redis:6379 + - REDIS_RATE_LIMIT_URL=redis://firecrawl-redis:6379 + - PLAYWRIGHT_MICROSERVICE_URL=http://127.0.0.1:3000/scrape + - POSTGRES_USER=${POSTGRES_USER:-firecrawl} + - POSTGRES_PASSWORD=${POSTGRES_PASSWORD} + - POSTGRES_DB=${POSTGRES_DB:-firecrawl} + - POSTGRES_HOST=firecrawl-postgres + - POSTGRES_PORT=5432 + - USE_DB_AUTHENTICATION=${USE_DB_AUTHENTICATION:-false} + - NUM_WORKERS_PER_QUEUE=${NUM_WORKERS_PER_QUEUE:-4} + - CRAWL_CONCURRENT_REQUESTS=${CRAWL_CONCURRENT_REQUESTS:-4} + - MAX_CONCURRENT_JOBS=${MAX_CONCURRENT_JOBS:-2} + - BROWSER_POOL_SIZE=${BROWSER_POOL_SIZE:-2} + - BULL_AUTH_KEY=${BULL_AUTH_KEY} + - TEST_API_KEY=${TEST_API_KEY} + - LOGGING_LEVEL=${LOGGING_LEVEL:-info} + - SEARXNG_ENDPOINT=http://searxng:8080 + - SEARXNG_ENGINES=${SEARXNG_ENGINES:-} + - SEARXNG_CATEGORIES=${SEARXNG_CATEGORIES:-} + - ALLOW_LOCAL_WEBHOOKS=${ALLOW_LOCAL_WEBHOOKS:-false} + - BLOCK_MEDIA=${BLOCK_MEDIA:-true} + - HARNESS_STARTUP_TIMEOUT_MS=60000 + - NUQ_RABBITMQ_URL=amqp://firecrawl-rabbitmq:5672 + command: node dist/src/harness.js --start-docker + + firecrawl-redis: + image: redis:alpine + container_name: firecrawl-redis + restart: unless-stopped + command: redis-server --bind 0.0.0.0 --appendonly yes + volumes: + - /mnt/docker-ssd/docker/appdata/firecrawl/redis:/data + networks: + - search_internal + + firecrawl-rabbitmq: + image: rabbitmq:3-management + container_name: firecrawl-rabbitmq + restart: unless-stopped + volumes: + - /mnt/docker-ssd/docker/appdata/firecrawl/rabbitmq:/var/lib/rabbitmq + command: rabbitmq-server + healthcheck: + test: ["CMD", "rabbitmq-diagnostics", "-q", "check_running"] + interval: 5s + timeout: 5s + retries: 3 + start_period: 5s + networks: + - search_internal + + firecrawl-postgres: + image: ghcr.io/firecrawl/nuq-postgres:latest + container_name: firecrawl-postgres + restart: unless-stopped + environment: + - POSTGRES_USER=${POSTGRES_USER:-firecrawl} + - POSTGRES_PASSWORD=${POSTGRES_PASSWORD} + - POSTGRES_DB=${POSTGRES_DB:-firecrawl} + volumes: + - /mnt/docker-ssd/docker/appdata/firecrawl/postgres:/var/lib/postgresql/data + networks: + - search_internal networks: pangolin: external: true + search_internal: + name: search_internal + driver: bridge + +secrets: + wg0.conf: + file: ./secrets/wg0.conf