light-heart-labs--dreamserver/ods/docker-compose.amd.yml

# ODS — AMD GPU Overlay (Lemonade inference backend)
# Lemonade provides optimized local AI inference with Vulkan, ROCm, and NPU support.
# ComfyUI GPU config moved to extensions/services/comfyui/compose.amd.yaml
# Use with: docker compose -f docker-compose.base.yml -f docker-compose.amd.yml up -d

services:
  llama-server:
    # Custom llama.cpp ROCm build + Lemonade server
    # Service name stays "llama-server" for compose dependency compatibility
    build:
      context: ./extensions/services/llama-server
      dockerfile: Dockerfile.amd
      args:
        AMDGPU_TARGET: ${AMDGPU_TARGET:-gfx1151}
        LLAMA_CPP_REF: ${LLAMA_CPP_REF:-b8763}
        LEMONADE_SERVER_IMAGE: ${LEMONADE_SERVER_IMAGE:-ghcr.io/lemonade-sdk/lemonade-server:v10.2.0}
    image: ods-lemonade-server:latest
    # Wrapper entrypoint syncs ctx_size from LEMONADE_CTX_SIZE env into
    # Lemonade's cached config.json before launch. Without this, Lemonade
    # only reads the env on first-start and ignores subsequent bumps —
    # leaving e.g. Qwen3.6-35B-A3B (native 256k context) serving at 64k.
    # Bind-mounted from the source tree so updating the script doesn't
    # require an image rebuild.
    entrypoint: ["/bin/sh", "/opt/lemonade-entrypoint.sh"]
    command:
      - serve
      - --port
      - "8080"
      - --host
      - "0.0.0.0"
      - --no-tray
      - --llamacpp
      - auto
      - --llamacpp-args
      - "--metrics --host 0.0.0.0 -fa on -b 2048 -ub 256"
      - --extra-models-dir
      - /models
    expose:
      - "8001"
    volumes:
      - ./data/models:/models:ro,z
      - lemonade-cache:/root/.cache/huggingface
      - lemonade-llama:/opt/lemonade/llama
      - lemonade-recipe:/root/.cache/lemonade
      # Read-only bind of the wrapper entrypoint. Updating ctx-size sync
      # logic doesn't require rebuilding the image.
      - ./extensions/services/llama-server/lemonade-entrypoint.sh:/opt/lemonade-entrypoint.sh:ro,z
    devices:
      - /dev/dri:/dev/dri
      - /dev/kfd:/dev/kfd
    group_add:
      - "${VIDEO_GID:-44}"
      - "${RENDER_GID:-992}"
    environment:
      # Empty default — Phase 06 sets HSA_OVERRIDE_GFX_VERSION=11.5.1 only when
      # the detected gfx target is gfx1151 (Strix Halo, not in ROCm's native
      # support list). On gfx942/MI300X, gfx90a/MI250, gfx1100/RX 7900 and
      # other natively-supported parts, this stays empty and HSA reports the
      # real ISA. A non-empty default here would crash those hosts with
      # HSA_STATUS_ERROR_INVALID_ISA at model-load.
      - HSA_OVERRIDE_GFX_VERSION=${HSA_OVERRIDE_GFX_VERSION:-}
      - HSA_XNACK=${HSA_XNACK:-1}
      - ROCBLAS_USE_HIPBLASLT=${ROCBLAS_USE_HIPBLASLT:-1}
      # Lemonade's backend selector accepts auto|vulkan|cpu only; "rocm" is no
      # longer a valid value. Setting auto + LEMONADE_LLAMACPP_ROCM_BIN points
      # Lemonade at the custom ROCm-built llama-server binary while keeping the
      # field schema-valid.
      - LEMONADE_LLAMACPP_BACKEND=auto
      # Custom binary at /opt/llama-custom is gfx1151-only (Strix Halo patches +
      # AMDGPU_TARGET=gfx1151 at build time). Phase 06 sets this env var only
      # when the detected gfx target is gfx1151; on any other AMD architecture
      # it stays unset and Lemonade uses its bundled ROCm-aware binary.
      - LEMONADE_LLAMACPP_ROCM_BIN=${LEMONADE_LLAMACPP_ROCM_BIN:-}
      - LEMONADE_CTX_SIZE=${CTX_SIZE:-131072}
    healthcheck:
      test: ["CMD", "curl", "-sf", "http://127.0.0.1:8080/api/v1/health"]
      interval: 15s
      timeout: 10s
      retries: 10
      start_period: 300s  # Lemonade first boot builds llama-server binary (~3-5 min)
    deploy:
      resources:
        limits:
          cpus: '${LLAMA_CPU_LIMIT:-16.0}'
          memory: ${LLAMA_SERVER_MEMORY_LIMIT:-110G}
        reservations:
          cpus: '${LLAMA_CPU_RESERVATION:-4.0}'
          memory: 8G

  # Services route through LiteLLM (ODS_MODE=lemonade sets LLM_API_URL=http://litellm:4000).
  # LiteLLM handles the /api/v1 translation to Lemonade internally.
  # LiteLLM enforces auth via LITELLM_MASTER_KEY=${LITELLM_KEY}; clients must present LITELLM_KEY.

  open-webui:
    environment:
      - OPENAI_API_KEY=${LITELLM_KEY}
      - ENABLE_IMAGE_GENERATION=${ENABLE_IMAGE_GENERATION:-true}

  dashboard-api:
    environment:
      - GPU_BACKEND=amd
      - LLM_BACKEND=lemonade
      - LLM_API_BASE_PATH=/api/v1
      - ODS_TALK_HERMES_TIMEOUT=${ODS_TALK_HERMES_TIMEOUT:-900}
      - AMD_INFERENCE_RUNTIME=${AMD_INFERENCE_RUNTIME:-lemonade}
      - AMD_INFERENCE_BACKEND=${AMD_INFERENCE_BACKEND:-rocm}
      - AMD_INFERENCE_LOCATION=${AMD_INFERENCE_LOCATION:-container}
      - AMD_INFERENCE_PORT=${AMD_INFERENCE_PORT:-8080}
      - AMD_INFERENCE_SUPPORTED_BACKENDS=${AMD_INFERENCE_SUPPORTED_BACKENDS:-rocm}
      - AMD_INFERENCE_RUNTIME_MODE=${AMD_INFERENCE_RUNTIME_MODE:-linux-container}
      - AMD_INFERENCE_MANAGED=${AMD_INFERENCE_MANAGED:-true}
      - LLAMA_METRICS_PORT=8001
    volumes:
      - /sys/class/drm:/sys/class/drm:ro
      - /sys/class/hwmon:/sys/class/hwmon:ro

volumes:
  lemonade-cache:
  lemonade-llama:
  lemonade-recipe: