From 03916965522c1886b99b7b5713614a54a3371b85 Mon Sep 17 00:00:00 2001
From: wizardeur <57131776+wizardeur@users.noreply.github.com>
Date: Sat, 5 Jul 2025 02:59:23 +0200
Subject: [PATCH] Added support for ROCm 6.4 (#60)

* Added support for ROCm 6.4

* Update README.md with information about ROCm support

---------

Co-authored-by: Kamil Andrusz <kamil@woospeedup.com>
---
 Dockerfile.gpu-rocm         | 50 +++++++++++++++++++++
 README.md                   | 14 +++++-
 docker-compose-gpu-rocm.yml | 86 +++++++++++++++++++++++++++++++++++++
 3 files changed, 148 insertions(+), 2 deletions(-)
 create mode 100644 Dockerfile.gpu-rocm
 create mode 100644 docker-compose-gpu-rocm.yml

diff --git a/Dockerfile.gpu-rocm b/Dockerfile.gpu-rocm
new file mode 100644
index 0000000..c0742b8
--- /dev/null
+++ b/Dockerfile.gpu-rocm
@@ -0,0 +1,50 @@
+FROM rocm/dev-ubuntu-22.04:latest
+
+# Set non-interactive frontend
+ENV DEBIAN_FRONTEND=noninteractive
+
+# Install Python and other dependencies
+RUN apt-get update && apt-get install -y \
+    python3.10 \
+    python3-pip \
+    python3-venv \
+    libsndfile1 \
+    ffmpeg \
+    portaudio19-dev \
+    libjpeg-dev \
+    python3-dev \
+    && apt-get install python3-wheel python3-setuptools \
+    && apt-get clean && rm -rf /var/lib/apt/lists/*
+
+# Create non-root user and set up directories
+RUN useradd -m -u 1001 appuser && \
+    mkdir -p /app/outputs /app && \
+    chown -R appuser:appuser /app
+
+USER appuser
+WORKDIR /app
+
+# Copy dependency files
+COPY --chown=appuser:appuser requirements.txt ./requirements.txt
+
+# Create and activate virtual environment
+RUN python3 -m venv /app/venv
+ENV PATH="/app/venv/bin:$PATH"
+
+# Install PyTorch with ROCm support and other dependencies
+RUN pip3 install --no-cache-dir --pre torch torchvision torchaudio --index-url https://download.pytorch.org/whl/nightly/rocm6.4/ && \
+    pip3 install --no-cache-dir -r requirements.txt
+    
+# Copy project files
+COPY --chown=appuser:appuser . .
+
+# Set environment variables
+ENV PYTHONUNBUFFERED=1 \
+    PYTHONPATH=/app \
+    USE_GPU=true
+
+# Expose the port
+EXPOSE 5005
+
+# Run FastAPI server with uvicorn
+CMD ["uvicorn", "app:app", "--host", "0.0.0.0", "--port", "5005", "--workers", "1"]
diff --git a/README.md b/README.md
index bd1aeee..0cb9630 100644
--- a/README.md
+++ b/README.md
@@ -101,7 +101,7 @@ Orpheus-FastAPI/
 ### Prerequisites
 
 - Python 3.8-3.11 (Python 3.12 is not supported due to removal of pkgutil.ImpImporter)
-- CUDA-compatible GPU (recommended: RTX series for best performance)
+- CUDA-compatible or ROCm-compatible GPU (recommended: RTX series for best performance)
 - Using docker compose or separate LLM inference server running the Orpheus model (e.g., LM Studio or llama.cpp server)
 - For Docker GPU Support, ensure you're using an Nvidia GPU on either Linux or Windows with CUDA 12.4 or greater and NVIDIA Container Toolkit installed
 
@@ -123,11 +123,16 @@ ORPHEUS_MODEL_NAME=Orpheus-3b-French-FT-Q8_0.gguf  # Example for French
 
 Then start the services:
 
-For GPU support run
+For CUDA GPU support run
 ```bash
 docker compose -f docker-compose-gpu.yml up
 ```
 
+For ROCm GPU support run
+```bash
+docker compose -f docker-compose-gpu-rocm.yml up
+```
+
 For CPU support run:
 ```bash
 docker compose -f docker-compose-cpu.yml up
@@ -158,6 +163,11 @@ conda activate orpheus-tts
 ```bash
 pip3 install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cu124
 ```
+or
+Install PyTorch with ROCm support:
+```bash
+pip3 install --pre torch torchvision torchaudio --index-url https://download.pytorch.org/whl/nightly/rocm6.4/
+```
 
 4. Install other dependencies:
 ```bash
diff --git a/docker-compose-gpu-rocm.yml b/docker-compose-gpu-rocm.yml
new file mode 100644
index 0000000..a807304
--- /dev/null
+++ b/docker-compose-gpu-rocm.yml
@@ -0,0 +1,86 @@
+services:
+  orpheus-fastapi:
+    container_name: orpheus-fastapi
+    image: orpheus-tts-fastapi-server-orpheus-fastapi:latest
+    build:
+      context: .
+      dockerfile: Dockerfile.gpu-rocm
+    ports:
+      - "5005:5005"
+    env_file:
+      - .env
+    environment:
+      - ORPHEUS_API_URL=http://llama-cpp-server:5006/v1/completions
+    ipc: host
+    privileged: true
+    security_opt:
+      - seccomp=unconfined
+    cap_add:
+      - SYS_PTRACE
+      - CAP_SYS_ADMIN
+    devices:
+      - /dev/kfd
+      - /dev/dri
+      - /dev/mem
+    group_add:
+      - video
+      - render
+      # - 993 # Add numeric render/video group id(s) if your system has different group id(s) than the image - 44(video),109(render)
+    shm_size: 8g
+    restart: unless-stopped
+    depends_on:
+      llama-cpp-server:
+        condition: service_started
+
+  llama-cpp-server:
+    image: ghcr.io/ggml-org/llama.cpp:server-vulkan
+    ports:
+      - "5006:5006"
+    volumes:
+      - ./models:/models
+    env_file:
+      - .env
+    depends_on:
+      model-init:
+        condition: service_completed_successfully
+    cap_add:
+      - SYS_PTRACE
+      - CAP_SYS_ADMIN
+    security_opt:
+      - seccomp=unconfined
+    privileged: true
+    devices:
+      - /dev/kfd
+      - /dev/dri
+      - /dev/mem
+    group_add:
+      - video
+      - 993
+    ipc: host
+    shm_size: 8g
+    restart: unless-stopped
+    command: >
+      -m /models/${ORPHEUS_MODEL_NAME}
+      --port 5006 
+      --host 0.0.0.0 
+      --n-gpu-layers 29
+      --ctx-size ${ORPHEUS_MAX_TOKENS}
+      --n-predict ${ORPHEUS_MAX_TOKENS}
+      --rope-scaling linear
+
+  model-init:
+    image: curlimages/curl:latest
+    user: ${UID}:${GID}
+    volumes:
+      - ./models:/app/models
+    working_dir: /app
+    command: >
+      sh -c '
+      if [ ! -f /app/models/${ORPHEUS_MODEL_NAME} ]; then
+        echo "Downloading model file..."
+        wget -P /app/models https://huggingface.co/lex-au/${ORPHEUS_MODEL_NAME}/resolve/main/${ORPHEUS_MODEL_NAME}
+      else
+        echo "Model file already exists"
+      fi'
+    restart: "no"
+