diff --git a/ai/docker-compose.yaml b/ai/docker-compose.yaml index b73c92a..ac63a90 100644 --- a/ai/docker-compose.yaml +++ b/ai/docker-compose.yaml @@ -1,10 +1,13 @@ networks: ai-net: driver: bridge + ai-services: + name: ai-services ix-databases_shared-databases: external: true pangolin: external: true + services: ollama: container_name: ollama @@ -12,6 +15,7 @@ services: restart: unless-stopped networks: - ai-net + - ai-services ports: - "11434:11434" environment: @@ -27,12 +31,14 @@ services: - driver: nvidia count: all capabilities: [gpu] + openwebui: container_name: openwebui image: ghcr.io/open-webui/open-webui:main restart: unless-stopped networks: - ai-net + - ai-services - ix-databases_shared-databases - pangolin ports: @@ -44,6 +50,8 @@ services: DATABASE_URL: postgresql://openwebui:${OPENWEBUI_DB_PASS}@shared-postgres:5432/openwebui ENABLE_SIGNUP: "true" ENABLE_LOGIN_FORM: "true" + VECTOR_DB: qdrant + QDRANT_URI: http://qdrant:6333 volumes: - /mnt/docker-ssd/docker/appdata/openwebui:/app/backend/data depends_on: @@ -54,33 +62,78 @@ services: timeout: 10s retries: 3 start_period: 60s -# scriberr: -# container_name: scriberr -# image: ghcr.io/rishikanthc/scriberr-cuda:v1.2.0 -# restart: unless-stopped -# privileged: true -# networks: -# - ai-net -# - pangolin -# ports: -# - "8032:8080" -# environment: -# TZ: ${TZ} -# HF_TOKEN: ${HF_TOKEN} -# NVIDIA_VISIBLE_DEVICES: all -# NVIDIA_DRIVER_CAPABILITIES: compute,utility -# volumes: -# - /mnt/docker-ssd/docker/appdata/scriberr:/app/data -# deploy: -# resources: -# reservations: -# devices: -# - driver: nvidia -# count: all -# capabilities: [gpu] -# healthcheck: -# test: ["CMD-SHELL", "curl -sf http://127.0.0.1:8080/ || exit 1"] -# interval: 30s -# timeout: 10s -# retries: 3 -# start_period: 60s + + qdrant: + container_name: qdrant + image: qdrant/qdrant:v1.17.0 + restart: unless-stopped + networks: + - ai-net + - ai-services + ports: + - "6333:6333" + - "6334:6334" + environment: + TZ: ${TZ} + QDRANT__SERVICE__GRPC_PORT: 6334 + QDRANT__STORAGE__STORAGE_PATH: /qdrant/storage + QDRANT__STORAGE__SNAPSHOTS_PATH: /qdrant/snapshots + volumes: + - /mnt/docker-ssd/docker/appdata/qdrant/storage:/qdrant/storage + - /mnt/docker-ssd/docker/appdata/qdrant/snapshots:/qdrant/snapshots + + litellm: + container_name: litellm + image: ghcr.io/berriai/litellm:main-latest + restart: unless-stopped + networks: + - ai-net + - ai-services + ports: + - "4000:4000" + environment: + TZ: ${TZ} + LITELLM_MASTER_KEY: ${LITELLM_MASTER_KEY} + volumes: + - /mnt/docker-ssd/docker/compose/ai/litellm/config.yaml:/app/config.yaml:ro + - /mnt/docker-ssd/docker/appdata/litellm:/app/data + command: ["--config", "/app/config.yaml", "--port", "4000"] + extra_hosts: + - "host.docker.internal:host-gateway" + + reranker: + container_name: reranker + image: ghcr.io/huggingface/text-embeddings-inference:cpu-latest + restart: unless-stopped + networks: + - ai-net + - ai-services + ports: + - "8787:80" + command: ["--model-id", "BAAI/bge-reranker-v2-m3"] + volumes: + - /mnt/docker-ssd/docker/appdata/reranker:/data + + whisper: + container_name: whisper + image: fedirz/faster-whisper-server:latest-cuda + restart: unless-stopped + networks: + - ai-net + - ai-services + ports: + - "8786:8000" + environment: + TZ: ${TZ} + WHISPER__MODEL: ${WHISPER_MODEL} + WHISPER__DEVICE: cuda + WHISPER__COMPUTE_TYPE: float16 + volumes: + - /mnt/docker-ssd/docker/appdata/whisper:/root/.cache/huggingface + deploy: + resources: + reservations: + devices: + - driver: nvidia + count: 1 + capabilities: [gpu] diff --git a/ai/litellm/config.yaml b/ai/litellm/config.yaml new file mode 100644 index 0000000..c782fdc --- /dev/null +++ b/ai/litellm/config.yaml @@ -0,0 +1,99 @@ +model_list: + # ---- ROCINANTE (RTX 4090, 24GB) — heavy reasoning ---- + - model_name: "heavy" + litellm_params: + model: "ollama/qwen3.6:latest" + api_base: "http://10.5.1.112:11434" + timeout: 300 + stream_timeout: 300 + + - model_name: "heavy" + litellm_params: + model: "ollama/qwen2.5:32b" + api_base: "http://10.5.1.112:11434" + timeout: 300 + stream_timeout: 300 + + - model_name: "heavy" + litellm_params: + model: "ollama/gemma3:27b" + api_base: "http://10.5.1.112:11434" + timeout: 300 + stream_timeout: 300 + + # ---- PlausibleDeniability (RTX 2080 Ti, 11GB) — general ---- + - model_name: "medium" + litellm_params: + model: "ollama/qwen2.5:14b" + api_base: "http://host.docker.internal:11434" + timeout: 120 + stream_timeout: 120 + + # ---- N.O.M.A.D. (GTX 1080, 8GB) — lightweight ---- + - model_name: "light" + litellm_params: + model: "ollama/deepseek-r1" + api_base: "http://10.5.1.16:11434" + timeout: 60 + stream_timeout: 60 + + - model_name: "light" + litellm_params: + model: "ollama/qwen2.5:3b" + api_base: "http://10.5.1.16:11434" + timeout: 60 + stream_timeout: 60 + + # ---- Embeddings (N.O.M.A.D. has nomic-embed-text) ---- + - model_name: "embed" + litellm_params: + model: "ollama/nomic-embed-text:v1.5" + api_base: "http://10.5.1.16:11434" + + # ---- Direct model access (bypass tier routing) ---- + - model_name: "ollama/qwen3.6:latest" + litellm_params: + model: "ollama/qwen3.6:latest" + api_base: "http://10.5.1.112:11434" + + - model_name: "ollama/qwen2.5:32b" + litellm_params: + model: "ollama/qwen2.5:32b" + api_base: "http://10.5.1.112:11434" + + - model_name: "ollama/gemma3:27b" + litellm_params: + model: "ollama/gemma3:27b" + api_base: "http://10.5.1.112:11434" + + - model_name: "ollama/qwen2.5:14b" + litellm_params: + model: "ollama/qwen2.5:14b" + api_base: "http://host.docker.internal:11434" + + - model_name: "ollama/deepseek-r1" + litellm_params: + model: "ollama/deepseek-r1" + api_base: "http://10.5.1.16:11434" + + - model_name: "ollama/qwen2.5:3b" + litellm_params: + model: "ollama/qwen2.5:3b" + api_base: "http://10.5.1.16:11434" + + - model_name: "ollama/nomic-embed-text:v1.5" + litellm_params: + model: "ollama/nomic-embed-text:v1.5" + api_base: "http://10.5.1.16:11434" + +litellm_settings: + drop_params: true + set_verbose: false + request_timeout: 300 + num_retries: 2 + retry_after: 5 + allowed_fails: 3 + cooldown_time: 60 + +general_settings: + master_key: "os.environ/LITELLM_MASTER_KEY"