fizzlepoof/truenas-stacks
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

ai: fix QDRANT_URI env var, bump qdrant to v1.17.0 to match client

Browse Source
This commit is contained in:
Paccoco
2026-05-05 20:37:37 -05:00
parent 58c8fd661f
commit 62e0c6c4d7
2 changed files with 182 additions and 30 deletions
Download Patch File Download Diff File Expand all files Collapse all files

113
ai/docker-compose.yaml
View File

@@ -1,10 +1,13 @@
networks:
ai-net:
driver: bridge
ai-services:
name: ai-services
ix-databases_shared-databases:
external: true
pangolin:
external: true
services:
ollama:
container_name: ollama
@@ -12,6 +15,7 @@ services:
restart: unless-stopped
networks:
- ai-net
- ai-services
ports:
- "11434:11434"
environment:
@@ -27,12 +31,14 @@ services:
- driver: nvidia
count: all
capabilities: [gpu]
openwebui:
container_name: openwebui
image: ghcr.io/open-webui/open-webui:main
restart: unless-stopped
networks:
- ai-net
- ai-services
- ix-databases_shared-databases
- pangolin
ports:
@@ -44,6 +50,8 @@ services:
DATABASE_URL: postgresql://openwebui:${OPENWEBUI_DB_PASS}@shared-postgres:5432/openwebui
ENABLE_SIGNUP: "true"
ENABLE_LOGIN_FORM: "true"
VECTOR_DB: qdrant
QDRANT_URI: http://qdrant:6333
volumes:
- /mnt/docker-ssd/docker/appdata/openwebui:/app/backend/data
depends_on:
@@ -54,33 +62,78 @@ services:
timeout: 10s
retries: 3
start_period: 60s
# scriberr:
# container_name: scriberr
# image: ghcr.io/rishikanthc/scriberr-cuda:v1.2.0
# restart: unless-stopped
# privileged: true
# networks:
# - ai-net
# - pangolin
# ports:
# - "8032:8080"
# environment:
# TZ: ${TZ}
# HF_TOKEN: ${HF_TOKEN}
# NVIDIA_VISIBLE_DEVICES: all
# NVIDIA_DRIVER_CAPABILITIES: compute,utility
# volumes:
# - /mnt/docker-ssd/docker/appdata/scriberr:/app/data
# deploy:
# resources:
# reservations:
# devices:
# - driver: nvidia
# count: all
# capabilities: [gpu]
# healthcheck:
# test: ["CMD-SHELL", "curl -sf http://127.0.0.1:8080/ || exit 1"]
# interval: 30s
# timeout: 10s
# retries: 3
# start_period: 60s
qdrant:
container_name: qdrant
image: qdrant/qdrant:v1.17.0
restart: unless-stopped
networks:
- ai-net
- ai-services
ports:
- "6333:6333"
- "6334:6334"
environment:
TZ: ${TZ}
QDRANT__SERVICE__GRPC_PORT: 6334
QDRANT__STORAGE__STORAGE_PATH: /qdrant/storage
QDRANT__STORAGE__SNAPSHOTS_PATH: /qdrant/snapshots
volumes:
- /mnt/docker-ssd/docker/appdata/qdrant/storage:/qdrant/storage
- /mnt/docker-ssd/docker/appdata/qdrant/snapshots:/qdrant/snapshots
litellm:
container_name: litellm
image: ghcr.io/berriai/litellm:main-latest
restart: unless-stopped
networks:
- ai-net
- ai-services
ports:
- "4000:4000"
environment:
TZ: ${TZ}
LITELLM_MASTER_KEY: ${LITELLM_MASTER_KEY}
volumes:
- /mnt/docker-ssd/docker/compose/ai/litellm/config.yaml:/app/config.yaml:ro
- /mnt/docker-ssd/docker/appdata/litellm:/app/data
command: ["--config", "/app/config.yaml", "--port", "4000"]
extra_hosts:
- "host.docker.internal:host-gateway"
reranker:
container_name: reranker
image: ghcr.io/huggingface/text-embeddings-inference:cpu-latest
restart: unless-stopped
networks:
- ai-net
- ai-services
ports:
- "8787:80"
command: ["--model-id", "BAAI/bge-reranker-v2-m3"]
volumes:
- /mnt/docker-ssd/docker/appdata/reranker:/data
whisper:
container_name: whisper
image: fedirz/faster-whisper-server:latest-cuda
restart: unless-stopped
networks:
- ai-net
- ai-services
ports:
- "8786:8000"
environment:
TZ: ${TZ}
WHISPER__MODEL: ${WHISPER_MODEL}
WHISPER__DEVICE: cuda
WHISPER__COMPUTE_TYPE: float16
volumes:
- /mnt/docker-ssd/docker/appdata/whisper:/root/.cache/huggingface
deploy:
resources:
reservations:
devices:
- driver: nvidia
count: 1
capabilities: [gpu]

99
ai/litellm/config.yaml Normal file
View File

@@ -0,0 +1,99 @@
model_list:
# ---- ROCINANTE (RTX 4090, 24GB) — heavy reasoning ----
- model_name: "heavy"
litellm_params:
model: "ollama/qwen3.6:latest"
api_base: "http://10.5.1.112:11434"
timeout: 300
stream_timeout: 300
- model_name: "heavy"
litellm_params:
model: "ollama/qwen2.5:32b"
api_base: "http://10.5.1.112:11434"
timeout: 300
stream_timeout: 300
- model_name: "heavy"
litellm_params:
model: "ollama/gemma3:27b"
api_base: "http://10.5.1.112:11434"
timeout: 300
stream_timeout: 300
# ---- PlausibleDeniability (RTX 2080 Ti, 11GB) — general ----
- model_name: "medium"
litellm_params:
model: "ollama/qwen2.5:14b"
api_base: "http://host.docker.internal:11434"
timeout: 120
stream_timeout: 120
# ---- N.O.M.A.D. (GTX 1080, 8GB) — lightweight ----
- model_name: "light"
litellm_params:
model: "ollama/deepseek-r1"
api_base: "http://10.5.1.16:11434"
timeout: 60
stream_timeout: 60
- model_name: "light"
litellm_params:
model: "ollama/qwen2.5:3b"
api_base: "http://10.5.1.16:11434"
timeout: 60
stream_timeout: 60
# ---- Embeddings (N.O.M.A.D. has nomic-embed-text) ----
- model_name: "embed"
litellm_params:
model: "ollama/nomic-embed-text:v1.5"
api_base: "http://10.5.1.16:11434"
# ---- Direct model access (bypass tier routing) ----
- model_name: "ollama/qwen3.6:latest"
litellm_params:
model: "ollama/qwen3.6:latest"
api_base: "http://10.5.1.112:11434"
- model_name: "ollama/qwen2.5:32b"
litellm_params:
model: "ollama/qwen2.5:32b"
api_base: "http://10.5.1.112:11434"
- model_name: "ollama/gemma3:27b"
litellm_params:
model: "ollama/gemma3:27b"
api_base: "http://10.5.1.112:11434"
- model_name: "ollama/qwen2.5:14b"
litellm_params:
model: "ollama/qwen2.5:14b"
api_base: "http://host.docker.internal:11434"
- model_name: "ollama/deepseek-r1"
litellm_params:
model: "ollama/deepseek-r1"
api_base: "http://10.5.1.16:11434"
- model_name: "ollama/qwen2.5:3b"
litellm_params:
model: "ollama/qwen2.5:3b"
api_base: "http://10.5.1.16:11434"
- model_name: "ollama/nomic-embed-text:v1.5"
litellm_params:
model: "ollama/nomic-embed-text:v1.5"
api_base: "http://10.5.1.16:11434"
litellm_settings:
drop_params: true
set_verbose: false
request_timeout: 300
num_retries: 2
retry_after: 5
allowed_fails: 3
cooldown_time: 60
general_settings:
master_key: "os.environ/LITELLM_MASTER_KEY"