Add Firecrawl stack behind Gluetun on PD

search/docker-compose.yaml

@@ -8,7 +8,7 @@ services:
     volumes:
       - /mnt/tank/docker/appdata/searxng:/etc/searxng
     environment:
-      - SEARXNG_BASE_URL=http://localhost:8888/
+      - SEARXNG_BASE_URL=${SEARXNG_BASE_URL:-http://10.5.1.6:8888/}
     cap_drop:
       - ALL
     cap_add:
@@ -17,7 +17,169 @@ services:
       - SETUID
     networks:
       - pangolin
+      - search_internal
+
+  search-gluetun:
+    image: ghcr.io/qdm12/gluetun:latest
+    container_name: search-gluetun
+    restart: unless-stopped
+    cap_add:
+      - NET_ADMIN
+    devices:
+      - /dev/net/tun:/dev/net/tun
+    ports:
+      - "${FIRECRAWL_PORT:-3302}:${FIRECRAWL_INTERNAL_PORT:-3002}"
+    volumes:
+      - /mnt/docker-ssd/docker/appdata/firecrawl/gluetun:/gluetun
+    environment:
+      - TZ=${TZ:-America/Chicago}
+      - VPN_SERVICE_PROVIDER=custom
+      - VPN_TYPE=wireguard
+      - WIREGUARD_CONF_SECRETFILE=/run/secrets/wg0.conf
+      - HTTP_CONTROL_SERVER_ADDRESS=:8000
+      - HTTP_CONTROL_SERVER_LOG=on
+      - FIREWALL_OUTBOUND_SUBNETS=${FIREWALL_OUTBOUND_SUBNETS:-10.5.0.0/16,172.16.0.0/12,192.168.0.0/16}
+      - UPDATER_PERIOD=24h
+    secrets:
+      - wg0.conf
+    networks:
+      - search_internal
+
+  gluetun-webui:
+    image: scuzza/gluetun-webui:latest
+    container_name: gluetun-webui
+    restart: unless-stopped
+    depends_on:
+      - search-gluetun
+    ports:
+      - "127.0.0.1:${GLUETUN_WEBUI_PORT:-8390}:3000"
+    environment:
+      - GLUETUN_CONTROL_URL=http://search-gluetun:8000
+      - PORT=3000
+    networks:
+      - search_internal
+    read_only: true
+    tmpfs:
+      - /tmp
+    security_opt:
+      - no-new-privileges:true
+    cap_drop:
+      - ALL
+    healthcheck:
+      test: ["CMD", "wget", "-qO-", "http://localhost:3000/api/health"]
+      interval: 30s
+      timeout: 5s
+      start_period: 10s
+      retries: 3
+
+  firecrawl-playwright-service:
+    image: ghcr.io/firecrawl/playwright-service:latest
+    container_name: firecrawl-playwright-service
+    restart: unless-stopped
+    depends_on:
+      - search-gluetun
+    network_mode: "service:search-gluetun"
+    environment:
+      - PORT=3000
+      - ALLOW_LOCAL_WEBHOOKS=${ALLOW_LOCAL_WEBHOOKS:-false}
+      - BLOCK_MEDIA=${BLOCK_MEDIA:-true}
+      - MAX_CONCURRENT_PAGES=${CRAWL_CONCURRENT_REQUESTS:-4}
+    tmpfs:
+      - /tmp/.cache:noexec,nosuid,size=1g
+
+  firecrawl-api:
+    image: ghcr.io/firecrawl/firecrawl:latest
+    container_name: firecrawl-api
+    restart: unless-stopped
+    depends_on:
+      firecrawl-redis:
+        condition: service_started
+      firecrawl-playwright-service:
+        condition: service_started
+      firecrawl-rabbitmq:
+        condition: service_healthy
+      firecrawl-postgres:
+        condition: service_started
+      search-gluetun:
+        condition: service_started
+    network_mode: "service:search-gluetun"
+    environment:
+      - HOST=0.0.0.0
+      - PORT=${FIRECRAWL_INTERNAL_PORT:-3002}
+      - EXTRACT_WORKER_PORT=3004
+      - WORKER_PORT=3005
+      - ENV=local
+      - REDIS_URL=redis://firecrawl-redis:6379
+      - REDIS_RATE_LIMIT_URL=redis://firecrawl-redis:6379
+      - PLAYWRIGHT_MICROSERVICE_URL=http://127.0.0.1:3000/scrape
+      - POSTGRES_USER=${POSTGRES_USER:-firecrawl}
+      - POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
+      - POSTGRES_DB=${POSTGRES_DB:-firecrawl}
+      - POSTGRES_HOST=firecrawl-postgres
+      - POSTGRES_PORT=5432
+      - USE_DB_AUTHENTICATION=${USE_DB_AUTHENTICATION:-false}
+      - NUM_WORKERS_PER_QUEUE=${NUM_WORKERS_PER_QUEUE:-4}
+      - CRAWL_CONCURRENT_REQUESTS=${CRAWL_CONCURRENT_REQUESTS:-4}
+      - MAX_CONCURRENT_JOBS=${MAX_CONCURRENT_JOBS:-2}
+      - BROWSER_POOL_SIZE=${BROWSER_POOL_SIZE:-2}
+      - BULL_AUTH_KEY=${BULL_AUTH_KEY}
+      - TEST_API_KEY=${TEST_API_KEY}
+      - LOGGING_LEVEL=${LOGGING_LEVEL:-info}
+      - SEARXNG_ENDPOINT=http://searxng:8080
+      - SEARXNG_ENGINES=${SEARXNG_ENGINES:-}
+      - SEARXNG_CATEGORIES=${SEARXNG_CATEGORIES:-}
+      - ALLOW_LOCAL_WEBHOOKS=${ALLOW_LOCAL_WEBHOOKS:-false}
+      - BLOCK_MEDIA=${BLOCK_MEDIA:-true}
+      - HARNESS_STARTUP_TIMEOUT_MS=60000
+      - NUQ_RABBITMQ_URL=amqp://firecrawl-rabbitmq:5672
+    command: node dist/src/harness.js --start-docker
+
+  firecrawl-redis:
+    image: redis:alpine
+    container_name: firecrawl-redis
+    restart: unless-stopped
+    command: redis-server --bind 0.0.0.0 --appendonly yes
+    volumes:
+      - /mnt/docker-ssd/docker/appdata/firecrawl/redis:/data
+    networks:
+      - search_internal
+
+  firecrawl-rabbitmq:
+    image: rabbitmq:3-management
+    container_name: firecrawl-rabbitmq
+    restart: unless-stopped
+    volumes:
+      - /mnt/docker-ssd/docker/appdata/firecrawl/rabbitmq:/var/lib/rabbitmq
+    command: rabbitmq-server
+    healthcheck:
+      test: ["CMD", "rabbitmq-diagnostics", "-q", "check_running"]
+      interval: 5s
+      timeout: 5s
+      retries: 3
+      start_period: 5s
+    networks:
+      - search_internal
+
+  firecrawl-postgres:
+    image: ghcr.io/firecrawl/nuq-postgres:latest
+    container_name: firecrawl-postgres
+    restart: unless-stopped
+    environment:
+      - POSTGRES_USER=${POSTGRES_USER:-firecrawl}
+      - POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
+      - POSTGRES_DB=${POSTGRES_DB:-firecrawl}
+    volumes:
+      - /mnt/docker-ssd/docker/appdata/firecrawl/postgres:/var/lib/postgresql/data
+    networks:
+      - search_internal
 
 networks:
   pangolin:
     external: true
+  search_internal:
+    name: search_internal
+    driver: bridge
+
+secrets:
+  wg0.conf:
+    file: ./secrets/wg0.conf