ROCm update

* Added support for ROCm 6.4

* Update README.md with information about ROCm support

---------

Co-authored-by: Kamil Andrusz <kamil@woospeedup.com>

.dockerignore

@@ -0,0 +1,3 @@
+*.env
+models/
+*.gguf

.env.example

@@ -12,6 +12,7 @@ ORPHEUS_TOP_P=0.9
 # Repetition penalty is now hardcoded to 1.1 for stability (this is a model constraint) - this setting is no longer used
 # ORPHEUS_REPETITION_PENALTY=1.1
 ORPHEUS_SAMPLE_RATE=24000
+ORPHEUS_MODEL_NAME=Orpheus-3b-FT-Q8_0.gguf # Model name sent to inference server (Q2_K, Q4_K_M, or Q8_0 variants)
 
 # Web UI settings (keep in mind that the web UI is not secure and should not be exposed to the internet)
 ORPHEUS_PORT=5005

.gitignore

@@ -0,0 +1,6 @@
+.env
+__pycache__/
+.venv/
+venv/
+models/
+*.gguf

Dockerfile.cpu

@@ -0,0 +1,47 @@
+FROM ubuntu:22.04
+
+# Set non-interactive frontend
+ENV DEBIAN_FRONTEND=noninteractive
+
+# Install Python and other dependencies
+RUN apt-get update && apt-get install -y \
+    python3.10 \
+    python3-pip \
+    python3-venv \
+    libsndfile1 \
+    ffmpeg \
+    portaudio19-dev \
+ && apt-get clean && rm -rf /var/lib/apt/lists/*
+
+# Create non-root user and set up directories
+RUN useradd -m -u 1001 appuser && \
+    mkdir -p /app/outputs /app && \
+    chown -R appuser:appuser /app
+
+USER appuser
+WORKDIR /app
+
+# Copy dependency files
+COPY --chown=appuser:appuser requirements.txt ./requirements.txt
+
+# Create and activate virtual environment
+RUN python3 -m venv /app/venv
+ENV PATH="/app/venv/bin:$PATH"
+
+# Install CPU-only PyTorch and other dependencies
+RUN pip3 install --no-cache-dir torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cpu && \
+    pip3 install --no-cache-dir -r requirements.txt
+
+# Copy project files
+COPY --chown=appuser:appuser . .
+
+# Set environment variables
+ENV PYTHONUNBUFFERED=1 \
+    PYTHONPATH=/app \
+    USE_GPU=false
+
+# Expose the port
+EXPOSE 5005
+
+# Run FastAPI server with uvicorn
+CMD ["uvicorn", "app:app", "--host", "0.0.0.0", "--port", "5005", "--workers", "1"]

Dockerfile.gpu

@@ -0,0 +1,47 @@
+FROM ubuntu:22.04
+
+# Set non-interactive frontend
+ENV DEBIAN_FRONTEND=noninteractive
+
+# Install Python and other dependencies
+RUN apt-get update && apt-get install -y \
+    python3.10 \
+    python3-pip \
+    python3-venv \
+    libsndfile1 \
+    ffmpeg \
+    portaudio19-dev \
+ && apt-get clean && rm -rf /var/lib/apt/lists/*
+
+# Create non-root user and set up directories
+RUN useradd -m -u 1001 appuser && \
+    mkdir -p /app/outputs /app && \
+    chown -R appuser:appuser /app
+
+USER appuser
+WORKDIR /app
+
+# Copy dependency files
+COPY --chown=appuser:appuser requirements.txt ./requirements.txt
+
+# Create and activate virtual environment
+RUN python3 -m venv /app/venv
+ENV PATH="/app/venv/bin:$PATH"
+
+# Install PyTorch with CUDA support and other dependencies
+RUN pip3 install --no-cache-dir torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cu124 && \
+    pip3 install --no-cache-dir -r requirements.txt
+
+# Copy project files
+COPY --chown=appuser:appuser . .
+
+# Set environment variables
+ENV PYTHONUNBUFFERED=1 \
+    PYTHONPATH=/app \
+    USE_GPU=true
+
+# Expose the port
+EXPOSE 5005
+
+# Run FastAPI server with uvicorn
+CMD ["uvicorn", "app:app", "--host", "0.0.0.0", "--port", "5005", "--workers", "1"]

Dockerfile.gpu-rocm

@@ -0,0 +1,50 @@
+FROM rocm/dev-ubuntu-22.04:latest
+
+# Set non-interactive frontend
+ENV DEBIAN_FRONTEND=noninteractive
+
+# Install Python and other dependencies
+RUN apt-get update && apt-get install -y \
+    python3.10 \
+    python3-pip \
+    python3-venv \
+    libsndfile1 \
+    ffmpeg \
+    portaudio19-dev \
+    libjpeg-dev \
+    python3-dev \
+    && apt-get install python3-wheel python3-setuptools \
+    && apt-get clean && rm -rf /var/lib/apt/lists/*
+
+# Create non-root user and set up directories
+RUN useradd -m -u 1001 appuser && \
+    mkdir -p /app/outputs /app && \
+    chown -R appuser:appuser /app
+
+USER appuser
+WORKDIR /app
+
+# Copy dependency files
+COPY --chown=appuser:appuser requirements.txt ./requirements.txt
+
+# Create and activate virtual environment
+RUN python3 -m venv /app/venv
+ENV PATH="/app/venv/bin:$PATH"
+
+# Install PyTorch with ROCm support and other dependencies
+RUN pip3 install --no-cache-dir --pre torch torchvision torchaudio --index-url https://download.pytorch.org/whl/nightly/rocm6.4/ && \
+    pip3 install --no-cache-dir -r requirements.txt
+    
+# Copy project files
+COPY --chown=appuser:appuser . .
+
+# Set environment variables
+ENV PYTHONUNBUFFERED=1 \
+    PYTHONPATH=/app \
+    USE_GPU=true
+
+# Expose the port
+EXPOSE 5005
+
+# Run FastAPI server with uvicorn
+CMD ["uvicorn", "app:app", "--host", "0.0.0.0", "--port", "5005", "--workers", "1"]

README.md

@@ -4,19 +4,58 @@
 
 [![GitHub](https://img.shields.io/github/license/Lex-au/Orpheus-FastAPI)](https://github.com/Lex-au/Orpheus-FastAPI/blob/main/LICENSE.txt)
 
-High-performance Text-to-Speech server with OpenAI-compatible API, 8 voices, emotion tags, and modern web UI. Optimized for RTX GPUs.
+High-performance Text-to-Speech server with OpenAI-compatible API, multilingual support with 24 voices, emotion tags, and modern web UI. Optimized for RTX GPUs.
 
 ## Changelog
 
+**v1.3.1** (2025-07-05)
+- 🐳 ROCm Docker implementation contributed by [@wizardeur](https://github.com/wizardeur) – many thanks for your contribution ❤️
+
+**v1.3.0** (2025-04-18)
+- 🌐 Added comprehensive multilingual support with 16 new voice actors across 7 languages
+- 🗣️ New voice actors include:
+  - French: pierre, amelie, marie
+  - German: jana, thomas, max
+  - Korean: 유나, 준서
+  - Hindi: ऋतिका
+  - Mandarin: 长乐, 白芷
+  - Spanish: javi, sergio, maria
+  - Italian: pietro, giulia, carlo
+- 🔄 Enhanced UI with dynamic language selection and voice filtering
+- 🚀 Released language-specific optimized models:
+  - [Italian & Spanish Model](https://huggingface.co/lex-au/Orpheus-3b-Italian_Spanish-FT-Q8_0.gguf)
+  - [Korean Model](https://huggingface.co/lex-au/Orpheus-3b-Korean-FT-Q8_0.gguf)
+  - [French Model](https://huggingface.co/lex-au/Orpheus-3b-French-FT-Q8_0.gguf)
+  - [Hindi Model](https://huggingface.co/lex-au/Orpheus-3b-Hindi-FT-Q8_0.gguf)
+  - [Mandarin Model](https://huggingface.co/lex-au/Orpheus-3b-Chinese-FT-Q8_0.gguf)
+  - [German Model](https://huggingface.co/lex-au/Orpheus-3b-German-FT-Q8_0.gguf)
+- 🐳 Docker Compose users: To use a language-specific model, edit the `.env` file before installation and change `ORPHEUS_MODEL_NAME` to match the desired model repo ID (e.g., `Orpheus-3b-French-FT-Q8_0.gguf`)
+- An additional Docker Compose installation path is now available, specifically for CPU-bound scenarios. This contribution comes from [@alexjyong](https://github.com/alexjyong) - thank you!
+
+**v1.2.0** (2025-04-12)
+- ❤️ Added optional Docker Compose support with GPU-enabled `llama.cpp` server and Orpheus-FastAPI integration  
+- 🐳 Docker implementation contributed by [@richardr1126](https://github.com/richardr1126) – huge thanks for the clean setup and orchestration work!  
+- 🧱 Native install path remains unchanged for non-Docker users
+
 **v1.1.0** (2025-03-23)
 - ✨ Added long-form audio support with sentence-based batching and crossfade stitching
 - 🔊 Improved short audio quality with optimized token buffer handling
 - 🔄 Enhanced environment variable support with .env file loading (configurable via UI)
 - 🖥️ Added automatic hardware detection and optimization for different GPUs
 - 📊 Implemented detailed performance reporting for audio generation
+- ⚠️ Note: Python 3.12 is not supported due to removal of pkgutil.ImpImporter
 
 [GitHub Repository](https://github.com/Lex-au/Orpheus-FastAPI)
 
+## Model Collection
+
+🚀 **NEW:** Try the quantized models for improved performance!
+- **Q2_K**: Ultra-fast inference with 2-bit quantization
+- **Q4_K_M**: Balanced quality/speed with 4-bit quantization (mixed)
+- **Q8_0**: Original high-quality 8-bit model
+
+[Browse the Orpheus-FASTAPI Model Collection on HuggingFace](https://huggingface.co/collections/lex-au/orpheus-fastapi-67e125ae03fc96dae0517707)
+
 ## Voice Demos
 
 Listen to sample outputs with different voices and emotions:
@@ -34,7 +73,7 @@ Listen to sample outputs with different voices and emotions:
 - **OpenAI API Compatible**: Drop-in replacement for OpenAI's `/v1/audio/speech` endpoint
 - **Modern Web Interface**: Clean, responsive UI with waveform visualization
 - **High Performance**: Optimized for RTX GPUs with parallel processing
-- **Multiple Voices**: 8 different voice options with different characteristics
+- **Multilingual Support**: 24 different voices across 8 languages (English, French, German, Korean, Hindi, Mandarin, Spanish, Italian)
 - **Emotion Tags**: Support for laughter, sighs, and other emotional expressions
 - **Unlimited Audio Length**: Generate audio of any length through intelligent batching
 - **Smooth Transitions**: Crossfaded audio segments for seamless listening experience
@@ -47,6 +86,8 @@ Listen to sample outputs with different voices and emotions:
 ```
 Orpheus-FastAPI/
 ├── app.py                # FastAPI server and endpoints
+├── docker-compose.yml    # Docker compose configuration
+├── Dockerfile.gpu        # GPU-enabled Docker image
 ├── requirements.txt      # Dependencies
 ├── static/               # Static assets (favicon, etc.)
 ├── outputs/              # Generated audio files
@@ -62,11 +103,47 @@ Orpheus-FastAPI/
 
 ### Prerequisites
 
-- Python 3.8+
-- CUDA-compatible GPU (recommended: RTX series for best performance)
-- Separate LLM inference server running the Orpheus model (e.g., LM Studio or llama.cpp server)
+- Python 3.8-3.11 (Python 3.12 is not supported due to removal of pkgutil.ImpImporter)
+- CUDA-compatible or ROCm-compatible GPU (recommended: RTX series for best performance)
+- Using docker compose or separate LLM inference server running the Orpheus model (e.g., LM Studio or llama.cpp server)
+- For Docker GPU Support, ensure you're using an Nvidia GPU on either Linux or Windows with CUDA 12.4 or greater and NVIDIA Container Toolkit installed
 
-### Installation
+### 🐳 Docker compose
+
+The docker compose file orchestrates the Orpheus-FastAPI for audio and a llama.cpp inference server for the base model token generation. The GGUF model is downloaded with the model-init service.
+There are three versions, two for machines that have access to GPU support `docker-compose-gpu.yaml`, `docker-compose-gpu-rocm.yml`  and one for CPU support only: `docker-compose-cpu.yaml`
+
+```bash
+cp .env.example .env # Create your .env file from the example
+copy .env.example .env # For Windows CMD
+```
+
+For multilingual models, edit the `.env` file and change the model name:
+```
+# Change this line in .env to use a language-specific model
+ORPHEUS_MODEL_NAME=Orpheus-3b-French-FT-Q8_0.gguf  # Example for French
+```
+
+Then start the services:
+
+For CUDA GPU support run
+```bash
+docker compose -f docker-compose-gpu.yml up
+```
+
+For ROCm GPU support run
+```bash
+docker compose -f docker-compose-gpu-rocm.yml up
+```
+
+For CPU support run:
+```bash
+docker compose -f docker-compose-cpu.yml up
+```
+
+The system will automatically download the specified model from Hugging Face before starting the service.
+
+### FastAPI Service Native Installation
 
 1. Clone the repository:
 ```bash
@@ -89,6 +166,11 @@ conda activate orpheus-tts
 ```bash
 pip3 install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cu124
 ```
+or
+Install PyTorch with ROCm support:
+```bash
+pip3 install --pre torch torchvision torchaudio --index-url https://download.pytorch.org/whl/nightly/rocm6.4/
+```
 
 4. Install other dependencies:
 ```bash
@@ -164,6 +246,7 @@ curl -X POST http://localhost:5005/speak \
 
 ### Available Voices
 
+#### English
 - `tara`: Female, conversational, clear
 - `leah`: Female, warm, gentle
 - `jess`: Female, energetic, youthful
@@ -173,6 +256,37 @@ curl -X POST http://localhost:5005/speak \
 - `zac`: Male, enthusiastic, dynamic
 - `zoe`: Female, calm, soothing
 
+#### French
+- `pierre`: Male, sophisticated
+- `amelie`: Female, elegant
+- `marie`: Female, spirited
+
+#### German
+- `jana`: Female, clear
+- `thomas`: Male, authoritative
+- `max`: Male, energetic
+
+#### Korean
+- `유나`: Female, melodic
+- `준서`: Male, confident
+
+#### Hindi
+- `ऋतिका`: Female, expressive
+
+#### Mandarin
+- `长乐`: Female, gentle
+- `白芷`: Female, clear
+
+#### Spanish
+- `javi`: Male, warm
+- `sergio`: Male, professional
+- `maria`: Female, friendly
+
+#### Italian
+- `pietro`: Male, passionate
+- `giulia`: Female, expressive
+- `carlo`: Male, refined
+
 ### Emotion Tags
 
 You can insert emotion tags into your text to add expressiveness:
@@ -186,7 +300,7 @@ You can insert emotion tags into your text to add expressiveness:
 - `<yawn>`: Add a yawning sound
 - `<gasp>`: Add a gasping sound
 
-Example: "Well, that's interesting <laugh> I hadn't thought of that before."
+Example: `"Well, that's interesting <laugh> I hadn't thought of that before."`
 
 ## Technical Details
 
@@ -198,8 +312,6 @@ This server works as a frontend that connects to an external LLM inference serve
 - Token and audio caching
 - Optimised batch sizes
 
-For best performance, adjust the API_URL in `tts_engine/inference.py` to point to your LLM inference server endpoint.
-
 ### Hardware Detection and Optimization
 
 The system features intelligent hardware detection that automatically optimizes performance based on your hardware capabilities:
@@ -258,27 +370,34 @@ You can easily integrate this TTS solution with [OpenWebUI](https://github.com/o
 3. Change TTS from Web API to OpenAI
 4. Set APIBASE URL to your server address (e.g., `http://localhost:5005/v1`)
 5. API Key can be set to "not-needed"
-6. Set TTS Voice to one of the available voices: `tara`, `leah`, `jess`, `leo`, `dan`, `mia`, `zac`, or `zoe`
+6. Set TTS Voice to any of the available voices (e.g., `tara`, `pierre`, `jana`, `유나`, etc.)
 7. Set TTS Model to `tts-1`
 
 ### External Inference Server
 
-This application requires a separate LLM inference server running the Orpheus model. You can use:
+This application requires a separate LLM inference server running the Orpheus model. For easy setup, use Docker Compose, which automatically handles this for you. Alternatively, you can use:
 
 - [GPUStack](https://github.com/gpustack/gpustack) - GPU optimised LLM inference server (My pick) - supports LAN/WAN tensor split parallelisation
 - [LM Studio](https://lmstudio.ai/) - Load the GGUF model and start the local server
 - [llama.cpp server](https://github.com/ggerganov/llama.cpp) - Run with the appropriate model parameters
 - Any compatible OpenAI API-compatible server
 
-Download the quantised model from [lex-au/Orpheus-3b-FT-Q8_0.gguf](https://huggingface.co/lex-au/Orpheus-3b-FT-Q8_0.gguf) and load it in your inference server.
+**Quantized Model Options:**
+- **lex-au/Orpheus-3b-FT-Q2_K.gguf**: Fastest inference (~50% faster tokens/sec than Q8_0)
+- **lex-au/Orpheus-3b-FT-Q4_K_M.gguf**: Balanced quality/speed 
+- **lex-au/Orpheus-3b-FT-Q8_0.gguf**: Original high-quality model
+
+Choose based on your hardware and needs. Lower bit models (Q2_K, Q4_K_M) provide ~2x realtime performance on high-end GPUs.
+
+[Browse all models in the collection](https://huggingface.co/collections/lex-au/orpheus-fastapi-67e125ae03fc96dae0517707)
 
 The inference server should be configured to expose an API endpoint that this FastAPI application will connect to.
 
 ### Environment Variables
 
-You can configure the system using environment variables or a `.env` file:
+Configure in docker compose, if using docker. Not using docker; create a `.env` file:
 
-- `ORPHEUS_API_URL`: URL of the LLM inference API (tts_engine/inference.py)
+- `ORPHEUS_API_URL`: URL of the LLM inference API (default in Docker: http://llama-cpp-server:5006/v1/completions)
 - `ORPHEUS_API_TIMEOUT`: Timeout in seconds for API requests (default: 120)
 - `ORPHEUS_MAX_TOKENS`: Maximum tokens to generate (default: 8192)
 - `ORPHEUS_TEMPERATURE`: Temperature for generation (default: 0.6)
@@ -286,6 +405,7 @@ You can configure the system using environment variables or a `.env` file:
 - `ORPHEUS_SAMPLE_RATE`: Audio sample rate in Hz (default: 24000)
 - `ORPHEUS_PORT`: Web server port (default: 5005)
 - `ORPHEUS_HOST`: Web server host (default: 0.0.0.0)
+- `ORPHEUS_MODEL_NAME`: Model name for inference server
 
 The system now supports loading environment variables from a `.env` file in the project root, making it easier to configure without modifying system-wide environment settings. See `.env.example` for a template.
 
@@ -312,7 +432,7 @@ To add new voices, update the `AVAILABLE_VOICES` list in `tts_engine/inference.p
 When running the Orpheus model with llama.cpp, use these parameters to ensure optimal performance:
 
 ```bash
-./llama-server -m models/Orpheus-3b-FT-Q8_0.gguf \
+./llama-server -m models/Modelname.gguf \
   --ctx-size={{your ORPHEUS_MAX_TOKENS from .env}} \
   --n-predict={{your ORPHEUS_MAX_TOKENS from .env}} \
   --rope-scaling=linear

app.py

@@ -11,14 +11,30 @@ from dotenv import load_dotenv
 
 # Function to ensure .env file exists
 def ensure_env_file_exists():
-    """Create a default .env file if one doesn't exist"""
+    """Create a .env file from defaults and OS environment variables"""
     if not os.path.exists(".env") and os.path.exists(".env.example"):
         try:
-            # Copy .env.example to .env
+            # 1. Create default env dictionary from .env.example
+            default_env = {}
             with open(".env.example", "r") as example_file:
-                with open(".env", "w") as env_file:
-                    env_file.write(example_file.read())
-            print("✅ Created default configuration file at .env")
+                for line in example_file:
+                    line = line.strip()
+                    if line and not line.startswith("#") and "=" in line:
+                        key = line.split("=")[0].strip()
+                        default_env[key] = line.split("=", 1)[1].strip()
+
+            # 2. Override defaults with Docker environment variables if they exist
+            final_env = default_env.copy()
+            for key in default_env:
+                if key in os.environ:
+                    final_env[key] = os.environ[key]
+
+            # 3. Write dictionary to .env file in env format
+            with open(".env", "w") as env_file:
+                for key, value in final_env.items():
+                    env_file.write(f"{key}={value}\n")
+                    
+            print("✅ Created default .env file from .env.example and environment variables.")
         except Exception as e:
             print(f"⚠️ Error creating default .env file: {e}")
 
@@ -35,7 +51,7 @@ from fastapi.templating import Jinja2Templates
 from pydantic import BaseModel
 import json
 
-from tts_engine import generate_speech_from_api, AVAILABLE_VOICES, DEFAULT_VOICE
+from tts_engine import generate_speech_from_api, AVAILABLE_VOICES, DEFAULT_VOICE, VOICE_TO_LANGUAGE, AVAILABLE_LANGUAGES
 
 # Create FastAPI app
 app = FastAPI(
@@ -114,6 +130,18 @@ async def create_speech_api(request: SpeechRequest):
         filename=f"{request.voice}_{timestamp}.wav"
     )
 
+@app.get("/v1/audio/voices")
+async def list_voices():
+    """Return list of available voices"""
+    if not AVAILABLE_VOICES or len(AVAILABLE_VOICES) == 0:
+        raise HTTPException(status_code=404, detail="No voices available")
+    return JSONResponse(
+        content={
+            "status": "ok",
+            "voices": AVAILABLE_VOICES
+        }
+    )
+
 # Legacy API endpoint for compatibility
 @app.post("/speak")
 async def speak(request: Request):
@@ -161,7 +189,12 @@ async def root(request: Request):
     """Redirect to web UI"""
     return templates.TemplateResponse(
         "tts.html",
-        {"request": request, "voices": AVAILABLE_VOICES}
+        {
+            "request": request, 
+            "voices": AVAILABLE_VOICES,
+            "VOICE_TO_LANGUAGE": VOICE_TO_LANGUAGE,
+            "AVAILABLE_LANGUAGES": AVAILABLE_LANGUAGES
+        }
     )
 
 @app.get("/web/", response_class=HTMLResponse)
@@ -171,7 +204,13 @@ async def web_ui(request: Request):
     config = get_current_config()
     return templates.TemplateResponse(
         "tts.html",
-        {"request": request, "voices": AVAILABLE_VOICES, "config": config}
+        {
+            "request": request, 
+            "voices": AVAILABLE_VOICES, 
+            "config": config,
+            "VOICE_TO_LANGUAGE": VOICE_TO_LANGUAGE,
+            "AVAILABLE_LANGUAGES": AVAILABLE_LANGUAGES
+        }
     )
 
 @app.get("/get_config")
@@ -273,7 +312,9 @@ async def generate_from_web(
             {
                 "request": request,
                 "error": "Please enter some text.",
-                "voices": AVAILABLE_VOICES
+                "voices": AVAILABLE_VOICES,
+                "VOICE_TO_LANGUAGE": VOICE_TO_LANGUAGE,
+                "AVAILABLE_LANGUAGES": AVAILABLE_LANGUAGES
             }
         )
     
@@ -306,7 +347,9 @@ async def generate_from_web(
             "voice": voice,
             "output_file": output_path,
             "generation_time": generation_time,
-            "voices": AVAILABLE_VOICES
+            "voices": AVAILABLE_VOICES,
+            "VOICE_TO_LANGUAGE": VOICE_TO_LANGUAGE,
+            "AVAILABLE_LANGUAGES": AVAILABLE_LANGUAGES
         }
     )
 

docker-compose-cpu.yaml

@@ -0,0 +1,56 @@
+services:
+  orpheus-fastapi:
+    container_name: orpheus-fastapi
+    build:
+      context: .
+      dockerfile: Dockerfile.cpu  
+    ports:
+      - "5005:5005"
+    env_file:
+      - .env
+    environment:
+      - ORPHEUS_API_URL=http://llama-cpp-server:5006/v1/completions
+    restart: unless-stopped
+    depends_on:
+      llama-cpp-server:
+        condition: service_started
+
+  llama-cpp-server:
+    image: ghcr.io/ggml-org/llama.cpp:server
+    ports:
+      - "5006:5006"
+    volumes:
+      - ./models:/models
+    env_file:
+      - .env
+    depends_on:
+      model-init:
+        condition: service_completed_successfully
+    restart: unless-stopped
+    command: >
+      -m /models/${ORPHEUS_MODEL_NAME}
+      --host 0.0.0.0
+      --port 5006
+      --ctx-size ${ORPHEUS_MAX_TOKENS}
+      --n-predict ${ORPHEUS_MAX_TOKENS}
+      --threads ${LLAMA_CPU_THREADS:-6}
+      --threads-batch ${LLAMA_CPU_THREADS:-6}
+      --rope-scaling linear
+      --no-mmap
+      --no-slots
+      --no-webui
+  model-init:
+    image: curlimages/curl:latest
+    user: ${UID}:${GID}
+    volumes:
+      - ./models:/app/models
+    working_dir: /app
+    command: >
+      sh -c '
+      if [ ! -f /app/models/${ORPHEUS_MODEL_NAME} ]; then
+        echo "Downloading model file..."
+        wget -P /app/models https://huggingface.co/lex-au/${ORPHEUS_MODEL_NAME}/resolve/main/${ORPHEUS_MODEL_NAME}
+      else
+        echo "Model file already exists"
+      fi'
+    restart: "no"

docker-compose-gpu-rocm.yml

@@ -0,0 +1,86 @@
+services:
+  orpheus-fastapi:
+    container_name: orpheus-fastapi
+    image: orpheus-tts-fastapi-server-orpheus-fastapi:latest
+    build:
+      context: .
+      dockerfile: Dockerfile.gpu-rocm
+    ports:
+      - "5005:5005"
+    env_file:
+      - .env
+    environment:
+      - ORPHEUS_API_URL=http://llama-cpp-server:5006/v1/completions
+    ipc: host
+    privileged: true
+    security_opt:
+      - seccomp=unconfined
+    cap_add:
+      - SYS_PTRACE
+      - CAP_SYS_ADMIN
+    devices:
+      - /dev/kfd
+      - /dev/dri
+      - /dev/mem
+    group_add:
+      - video
+      - render
+      # - 993 # Add numeric render/video group id(s) if your system has different group id(s) than the image - 44(video),109(render)
+    shm_size: 8g
+    restart: unless-stopped
+    depends_on:
+      llama-cpp-server:
+        condition: service_started
+
+  llama-cpp-server:
+    image: ghcr.io/ggml-org/llama.cpp:server-vulkan
+    ports:
+      - "5006:5006"
+    volumes:
+      - ./models:/models
+    env_file:
+      - .env
+    depends_on:
+      model-init:
+        condition: service_completed_successfully
+    cap_add:
+      - SYS_PTRACE
+      - CAP_SYS_ADMIN
+    security_opt:
+      - seccomp=unconfined
+    privileged: true
+    devices:
+      - /dev/kfd
+      - /dev/dri
+      - /dev/mem
+    group_add:
+      - video
+      - 993
+    ipc: host
+    shm_size: 8g
+    restart: unless-stopped
+    command: >
+      -m /models/${ORPHEUS_MODEL_NAME}
+      --port 5006 
+      --host 0.0.0.0 
+      --n-gpu-layers 29
+      --ctx-size ${ORPHEUS_MAX_TOKENS}
+      --n-predict ${ORPHEUS_MAX_TOKENS}
+      --rope-scaling linear
+
+  model-init:
+    image: curlimages/curl:latest
+    user: ${UID}:${GID}
+    volumes:
+      - ./models:/app/models
+    working_dir: /app
+    command: >
+      sh -c '
+      if [ ! -f /app/models/${ORPHEUS_MODEL_NAME} ]; then
+        echo "Downloading model file..."
+        wget -P /app/models https://huggingface.co/lex-au/${ORPHEUS_MODEL_NAME}/resolve/main/${ORPHEUS_MODEL_NAME}
+      else
+        echo "Model file already exists"
+      fi'
+    restart: "no"
+    

docker-compose-gpu.yml

@@ -0,0 +1,68 @@
+services:
+  orpheus-fastapi:
+    container_name: orpheus-fastapi
+    build:
+      context: .
+      dockerfile: Dockerfile.gpu
+    ports:
+      - "5005:5005"
+    env_file:
+      - .env
+    environment:
+      - ORPHEUS_API_URL=http://llama-cpp-server:5006/v1/completions
+    deploy:
+      resources:
+        reservations:
+          devices:
+            - driver: nvidia
+              count: all
+              capabilities: [gpu]
+    restart: unless-stopped
+    depends_on:
+      llama-cpp-server:
+        condition: service_started
+
+  llama-cpp-server:
+    image: ghcr.io/ggml-org/llama.cpp:server-cuda
+    ports:
+      - "5006:5006"
+    volumes:
+      - ./models:/models
+    env_file:
+      - .env
+    depends_on:
+      model-init:
+        condition: service_completed_successfully
+    deploy:
+      resources:
+        reservations:
+          devices:
+            - driver: nvidia
+              count: all
+              capabilities: [gpu]
+    restart: unless-stopped
+    command: >
+      -m /models/${ORPHEUS_MODEL_NAME}
+      --port 5006 
+      --host 0.0.0.0 
+      --n-gpu-layers 29
+      --ctx-size ${ORPHEUS_MAX_TOKENS}
+      --n-predict ${ORPHEUS_MAX_TOKENS}
+      --rope-scaling linear
+
+  model-init:
+    image: curlimages/curl:latest
+    user: ${UID}:${GID}
+    volumes:
+      - ./models:/app/models
+    working_dir: /app
+    command: >
+      sh -c '
+      if [ ! -f /app/models/${ORPHEUS_MODEL_NAME} ]; then
+        echo "Downloading model file..."
+        wget -P /app/models https://huggingface.co/lex-au/${ORPHEUS_MODEL_NAME}/resolve/main/${ORPHEUS_MODEL_NAME}
+      else
+        echo "Model file already exists"
+      fi'
+    restart: "no"
+    

docs/index.html

@@ -64,11 +64,15 @@
                 <td>Default Test (MP3)</td>
                 <td><a href="DefaultTest.mp3" target="_blank">DefaultTest.mp3</a></td>
             </tr>
+            <tr>
+                <td>Default Test (MP3)</td>
+                <td><a href="TaraLongGeneration.mp3" target="_blank">TaraLongGeneration.mp3</a></td>
+            </tr>
             <tr>
                 <td>WebUI Screenshot (PNG)</td>
                 <td>
-                    <a href="Webui.png" target="_blank">Webui.png</a><br>
-                    <img src="Webui.png" alt="WebUI Preview">
+                    <a href="WebUI.png" target="_blank">Webui.png</a><br>
+                    <img src="WebUI.png" alt="WebUI Preview">
                 </td>
             </tr>
             <tr>

requirements.txt

@@ -8,7 +8,7 @@ python-multipart==0.0.6
 # API and Communication
 requests==2.31.0
 python-dotenv==1.0.0
-watchfiles=1.0.4
+watchfiles==1.0.4
 
 # Audio Processing
 numpy==1.24.0

templates/tts.html

@@ -155,25 +155,72 @@
                 </div>
               </div>
 
+              <!-- Language selection -->
+              <div class="mb-6">
+                <label class="block text-sm font-medium text-white mb-2">Language</label>
+                <div class="flex flex-wrap gap-3">
+                  {% for language in AVAILABLE_LANGUAGES %}
+                  <div class="language-option {% if language == 'english' %}active{% endif %}" data-language="{{ language }}">
+                    <input type="radio" name="language" value="{{ language }}" class="hidden" {% if language == 'english' %}checked{% endif %}>
+                    <span class="px-3 py-1 rounded-full bg-dark-700 hover:bg-dark-600 text-white text-sm cursor-pointer">{{ language|capitalize }}</span>
+                  </div>
+                  {% endfor %}
+                </div>
+              </div>
+
               <!-- Voice selection -->
               <div class="mb-6">
                 <label class="block text-sm font-medium text-white mb-2">Voice</label>
                 <div class="grid grid-cols-1 md:grid-cols-2 lg:grid-cols-4 gap-4">
                   {% for voice_option in voices %}
-                  <div class="voice-card {% if voice_option == DEFAULT_VOICE %}active{% endif %}" data-voice="{{ voice_option }}">
+                  <div class="voice-card {% if voice_option == DEFAULT_VOICE %}active{% endif %}" 
+                       data-voice="{{ voice_option }}" 
+                       data-language="{{ VOICE_TO_LANGUAGE[voice_option] }}"
+                       style="display: {% if VOICE_TO_LANGUAGE[voice_option] != 'english' %}none{% endif %}">
                     <input type="radio" name="voice" value="{{ voice_option }}" class="hidden" {% if voice_option == DEFAULT_VOICE %}checked{% endif %}>
                     <div class="flex items-center mb-2">
                       <span class="font-medium text-white">{{ voice_option|capitalize }}</span>
                     </div>
-    <div class="text-xs text-dark-300">
-                      {% if voice_option == "tara" %}Female, conversational, clear
-                      {% elif voice_option == "leah" %}Female, warm, gentle
-                      {% elif voice_option == "jess" %}Female, energetic, youthful
-                      {% elif voice_option == "leo" %}Male, authoritative, deep
-                      {% elif voice_option == "dan" %}Male, friendly, casual
-                      {% elif voice_option == "mia" %}Female, professional, articulate
-                      {% elif voice_option == "zac" %}Male, enthusiastic, dynamic
-                      {% elif voice_option == "zoe" %}Female, calm, soothing
+                    <div class="text-xs text-dark-300">
+                      {% if voice_option == "tara" %}Female, English, conversational, clear
+                      {% elif voice_option == "leah" %}Female, English, warm, gentle
+                      {% elif voice_option == "jess" %}Female, English, energetic, youthful
+                      {% elif voice_option == "leo" %}Male, English, authoritative, deep
+                      {% elif voice_option == "dan" %}Male, English, friendly, casual
+                      {% elif voice_option == "mia" %}Female, English, professional, articulate
+                      {% elif voice_option == "zac" %}Male, English, enthusiastic, dynamic
+                      {% elif voice_option == "zoe" %}Female, English, calm, soothing
+                      
+                      <!-- French voices -->
+                      {% elif voice_option == "pierre" %}Male, French, sophisticated
+                      {% elif voice_option == "amelie" %}Female, French, elegant
+                      {% elif voice_option == "marie" %}Female, French, spirited
+                      
+                      <!-- German voices -->
+                      {% elif voice_option == "jana" %}Female, German, clear
+                      {% elif voice_option == "thomas" %}Male, German, authoritative
+                      {% elif voice_option == "max" %}Male, German, energetic
+                      
+                      <!-- Korean voices -->
+                      {% elif voice_option == "유나" %}Female, Korean, melodic
+                      {% elif voice_option == "준서" %}Male, Korean, confident
+                      
+                      <!-- Hindi voice -->
+                      {% elif voice_option == "ऋतिका" %}Female, Hindi, expressive
+                      
+                      <!-- Mandarin voices -->
+                      {% elif voice_option == "长乐" %}Female, Mandarin, gentle
+                      {% elif voice_option == "白芷" %}Female, Mandarin, clear
+                      
+                      <!-- Spanish voices -->
+                      {% elif voice_option == "javi" %}Male, Spanish, warm
+                      {% elif voice_option == "sergio" %}Male, Spanish, professional
+                      {% elif voice_option == "maria" %}Female, Spanish, friendly
+                      
+                      <!-- Italian voices -->
+                      {% elif voice_option == "pietro" %}Male, Italian, passionate
+                      {% elif voice_option == "giulia" %}Female, Italian, expressive
+                      {% elif voice_option == "carlo" %}Male, Italian, refined
                       {% endif %}
                     </div>
                   </div>
@@ -404,20 +451,83 @@
       // Initialize char count
       charCount.textContent = textArea.value.length;
       
-      // Voice selection
+      // Language selection
+      const languageOptions = document.querySelectorAll('.language-option');
       const voiceCards = document.querySelectorAll('.voice-card');
       
-      voiceCards.forEach(card => {
-        card.addEventListener('click', function() {
-          // Unselect all cards
-          voiceCards.forEach(c => c.classList.remove('active'));
+      languageOptions.forEach(option => {
+        option.addEventListener('click', function() {
+          // Unselect all language options
+          languageOptions.forEach(o => o.classList.remove('active'));
           
-          // Select this card
+          // Select this language option
           this.classList.add('active');
           
           // Check the radio button
           const radio = this.querySelector('input[type="radio"]');
           radio.checked = true;
+          
+          // Get the selected language
+          const selectedLanguage = this.getAttribute('data-language');
+          
+          // Show/hide voice cards based on language
+          let firstVisibleCard = null;
+          
+          voiceCards.forEach(card => {
+            const cardLanguage = card.getAttribute('data-language');
+            if (cardLanguage === selectedLanguage) {
+              card.style.display = '';
+              if (!firstVisibleCard) {
+                firstVisibleCard = card;
+              }
+            } else {
+              card.style.display = 'none';
+              // If this hidden card was selected, unselect it
+              if (card.classList.contains('active')) {
+                card.classList.remove('active');
+                const cardRadio = card.querySelector('input[type="radio"]');
+                if (cardRadio) {
+                  cardRadio.checked = false;
+                }
+              }
+            }
+          });
+          
+          // If no voice card is selected for this language, select the first one
+          if (firstVisibleCard && document.querySelectorAll(`.voice-card[data-language="${selectedLanguage}"].active:not([style*="display: none"])`).length === 0) {
+            firstVisibleCard.classList.add('active');
+            const firstVisibleRadio = firstVisibleCard.querySelector('input[type="radio"]');
+            if (firstVisibleRadio) {
+              firstVisibleRadio.checked = true;
+            }
+          }
+        });
+      });
+      
+      // Voice card selection
+      voiceCards.forEach(card => {
+        card.addEventListener('click', function() {
+          // Only allow selection of cards that are visible (not display:none)
+          if (this.style.display !== 'none') {
+            // Unselect all cards with the same language
+            const cardLanguage = this.getAttribute('data-language');
+            document.querySelectorAll(`.voice-card[data-language="${cardLanguage}"]`).forEach(c => {
+              c.classList.remove('active');
+              const radio = c.querySelector('input[type="radio"]');
+              if (radio) {
+                radio.checked = false;
+              }
+            });
+            
+            // Select this card
+            this.classList.add('active');
+            
+            // Check the radio button
+            const radio = this.querySelector('input[type="radio"]');
+            if (radio) {
+              radio.checked = true;
+            }
+          }
         });
       });
       

tts_engine/__init__.py

@@ -11,5 +11,7 @@ from .inference import (
     generate_speech_from_api,
     AVAILABLE_VOICES,
     DEFAULT_VOICE,
+    VOICE_TO_LANGUAGE,
+    AVAILABLE_LANGUAGES,
     list_available_voices
 )

tts_engine/inference.py

@@ -136,14 +136,49 @@ if not IS_RELOADER:
 # Parallel processing settings
 NUM_WORKERS = 4 if HIGH_END_GPU else 2
 
-# Available voices based on the Orpheus-TTS repository
-AVAILABLE_VOICES = ["tara", "leah", "jess", "leo", "dan", "mia", "zac", "zoe"]
+# Define voices by language
+ENGLISH_VOICES = ["tara", "leah", "jess", "leo", "dan", "mia", "zac", "zoe"]
+FRENCH_VOICES = ["pierre", "amelie", "marie"]
+GERMAN_VOICES = ["jana", "thomas", "max"]
+KOREAN_VOICES = ["유나", "준서"]
+HINDI_VOICES = ["ऋतिका"]
+MANDARIN_VOICES = ["长乐", "白芷"]
+SPANISH_VOICES = ["javi", "sergio", "maria"]
+ITALIAN_VOICES = ["pietro", "giulia", "carlo"]
+
+# Combined list for API compatibility
+AVAILABLE_VOICES = (
+    ENGLISH_VOICES + 
+    FRENCH_VOICES + 
+    GERMAN_VOICES + 
+    KOREAN_VOICES + 
+    HINDI_VOICES + 
+    MANDARIN_VOICES + 
+    SPANISH_VOICES + 
+    ITALIAN_VOICES
+)
 DEFAULT_VOICE = "tara"  # Best voice according to documentation
 
+# Map voices to languages for the UI
+VOICE_TO_LANGUAGE = {}
+VOICE_TO_LANGUAGE.update({voice: "english" for voice in ENGLISH_VOICES})
+VOICE_TO_LANGUAGE.update({voice: "french" for voice in FRENCH_VOICES})
+VOICE_TO_LANGUAGE.update({voice: "german" for voice in GERMAN_VOICES})
+VOICE_TO_LANGUAGE.update({voice: "korean" for voice in KOREAN_VOICES})
+VOICE_TO_LANGUAGE.update({voice: "hindi" for voice in HINDI_VOICES})
+VOICE_TO_LANGUAGE.update({voice: "mandarin" for voice in MANDARIN_VOICES})
+VOICE_TO_LANGUAGE.update({voice: "spanish" for voice in SPANISH_VOICES})
+VOICE_TO_LANGUAGE.update({voice: "italian" for voice in ITALIAN_VOICES})
+
+# Languages list for the UI
+AVAILABLE_LANGUAGES = ["english", "french", "german", "korean", "hindi", "mandarin", "spanish", "italian"]
+
+# Import the unified token handling from speechpipe
+from .speechpipe import turn_token_into_id, CUSTOM_TOKEN_PREFIX
+
 # Special token IDs for Orpheus model
 START_TOKEN_ID = 128259
 END_TOKEN_IDS = [128009, 128260, 128261, 128257]
-CUSTOM_TOKEN_PREFIX = "<custom_token_"
 
 # Performance monitoring
 class PerformanceMonitor:
@@ -216,9 +251,8 @@ def generate_tokens_from_api(prompt: str, voice: str = DEFAULT_VOICE, temperatur
     elif torch.cuda.is_available():
         print("Using optimized parameters for GPU acceleration")
     
-    # Create the request payload
+    # Create the request payload (model field may not be required by some endpoints but included for compatibility)
     payload = {
-        "model": "orpheus-3b-0.1-ft-q4_k_m",  # Model name can be anything, endpoint will use loaded model
         "prompt": formatted_prompt,
         "max_tokens": max_tokens,
         "temperature": temperature,
@@ -227,6 +261,11 @@ def generate_tokens_from_api(prompt: str, voice: str = DEFAULT_VOICE, temperatur
         "stream": True  # Always stream for better performance
     }
     
+    # Add model field - this is ignored by many local inference servers for /v1/completions
+    # but included for compatibility with OpenAI API and some servers that may use it
+    model_name = os.environ.get("ORPHEUS_MODEL_NAME", "Orpheus-3b-FT-Q8_0.gguf")
+    payload["model"] = model_name
+    
     # Session for connection pooling and retry logic
     session = requests.Session()
     
@@ -273,12 +312,14 @@ def generate_tokens_from_api(prompt: str, voice: str = DEFAULT_VOICE, temperatur
                         try:
                             data = json.loads(data_str)
                             if 'choices' in data and len(data['choices']) > 0:
-                                token_text = data['choices'][0].get('text', '')
-                                token_counter += 1
-                                perf_monitor.add_tokens()
-                                
-                                if token_text:
-                                    yield token_text
+                                token_chunk = data['choices'][0].get('text', '')
+                                for token_text in token_chunk.split('>'):
+                                    token_text = f'{token_text}>'
+                                    token_counter += 1
+                                    perf_monitor.add_tokens()
+
+                                    if token_text:
+                                        yield token_text
                         except json.JSONDecodeError as e:
                             print(f"Error decoding JSON: {e}")
                             continue
@@ -311,44 +352,8 @@ def generate_tokens_from_api(prompt: str, voice: str = DEFAULT_VOICE, temperatur
                 print("Max retries reached. Token generation failed.")
                 return
 
-# Token ID cache to avoid repeated processing
-token_id_cache = {}
-MAX_CACHE_SIZE = 10000
-
-def turn_token_into_id(token_string: str, index: int) -> Optional[int]:
-    """Optimized token-to-ID conversion with caching."""
-    # Check cache first (significant speedup for repeated tokens)
-    cache_key = (token_string, index % 7)
-    if cache_key in token_id_cache:
-        return token_id_cache[cache_key]
-        
-    # Early rejection for obvious non-matches
-    if CUSTOM_TOKEN_PREFIX not in token_string:
-        return None
-        
-    # Process token
-    token_string = token_string.strip()
-    last_token_start = token_string.rfind(CUSTOM_TOKEN_PREFIX)
-    
-    if last_token_start == -1:
-        return None
-    
-    last_token = token_string[last_token_start:]
-    
-    if not (last_token.startswith(CUSTOM_TOKEN_PREFIX) and last_token.endswith(">")):
-        return None
-        
-    try:
-        number_str = last_token[14:-1]
-        token_id = int(number_str) - 10 - ((index % 7) * 4096)
-        
-        # Cache the result if it's valid
-        if len(token_id_cache) < MAX_CACHE_SIZE:
-            token_id_cache[cache_key] = token_id
-            
-        return token_id
-    except (ValueError, IndexError):
-        return None
+# The turn_token_into_id function is now imported from speechpipe.py
+# This eliminates duplicate code and ensures consistent behavior
 
 def convert_to_audio(multiframe: List[int], count: int) -> Optional[bytes]:
     """Convert token frames to audio with performance monitoring."""
@@ -363,13 +368,15 @@ def convert_to_audio(multiframe: List[int], count: int) -> Optional[bytes]:
     return result
 
 async def tokens_decoder(token_gen) -> Generator[bytes, None, None]:
-    """Simplified token decoder without complex ring buffer to ensure reliable output."""
+    """Simplified token decoder with early first-chunk processing for lower latency."""
     buffer = []
     count = 0
     
-    # Use conservative batch parameters to ensure output quality
-    min_frames = 28  # Default for reliability (4 chunks of 7)
-    process_every = 7  # Process every 7 tokens (standard for Orpheus)
+    # Use different thresholds for first chunk vs. subsequent chunks
+    first_chunk_processed = False
+    min_frames_first = 7  # Process after just 7 tokens for first chunk (ultra-low latency)
+    min_frames_subsequent = 28  # Default for reliability after first chunk (4 chunks of 7)
+    process_every = 7  # Process every 7 tokens (standard for Orpheus model)
     
     start_time = time.time()
     last_log_time = start_time
@@ -391,19 +398,32 @@ async def tokens_decoder(token_gen) -> Generator[bytes, None, None]:
                     print(f"Token processing rate: {token_count/elapsed:.1f} tokens/second")
                 last_log_time = current_time
             
-            # Process in standard batches for Orpheus model
-            if count % process_every == 0 and count >= min_frames:
-                # Use simple slice operation - reliable and correct
-                buffer_to_proc = buffer[-min_frames:]
-                
-                # Debug output to help diagnose issues
-                if count % 28 == 0:
-                    print(f"Processing buffer with {len(buffer_to_proc)} tokens, total collected: {len(buffer)}")
-                
-                # Process the tokens
-                audio_samples = convert_to_audio(buffer_to_proc, count)
-                if audio_samples is not None:
-                    yield audio_samples
+            # Different processing paths based on whether first chunk has been processed
+            if not first_chunk_processed:
+                # For first audio output, process as soon as we have enough tokens for one chunk
+                if count >= min_frames_first:
+                    buffer_to_proc = buffer[-min_frames_first:]
+                    
+                    # Process the first chunk for immediate audio feedback
+                    print(f"Processing first audio chunk with {len(buffer_to_proc)} tokens")
+                    audio_samples = convert_to_audio(buffer_to_proc, count)
+                    if audio_samples is not None:
+                        first_chunk_processed = True  # Mark first chunk as processed
+                        yield audio_samples
+            else:
+                # For subsequent chunks, use standard processing with larger batch
+                if count % process_every == 0 and count >= min_frames_subsequent:
+                    # Use simple slice operation - reliable and correct
+                    buffer_to_proc = buffer[-min_frames_subsequent:]
+                    
+                    # Debug output to help diagnose issues
+                    if count % 28 == 0:
+                        print(f"Processing buffer with {len(buffer_to_proc)} tokens, total collected: {len(buffer)}")
+                    
+                    # Process the tokens
+                    audio_samples = convert_to_audio(buffer_to_proc, count)
+                    if audio_samples is not None:
+                        yield audio_samples
 
 def tokens_decoder_sync(syn_token_gen, output_file=None):
     """Optimized synchronous wrapper with parallel processing and efficient file I/O."""

tts_engine/speechpipe.py

@@ -5,13 +5,25 @@ import asyncio
 import threading
 import queue
 import time
+import os
+import sys
+
+# Helper to detect if running in Uvicorn's reloader (same as in inference.py)
+def is_reloader_process():
+    """Check if the current process is a uvicorn reloader"""
+    return (sys.argv[0].endswith('_continuation.py') or 
+            os.environ.get('UVICORN_STARTED') == 'true')
+
+# Set a flag to avoid repeat messages
+IS_RELOADER = is_reloader_process()
 
 # Try to enable torch.compile if PyTorch 2.0+ is available
 TORCH_COMPILE_AVAILABLE = False
 try:
     if hasattr(torch, 'compile'):
         TORCH_COMPILE_AVAILABLE = True
-        print("PyTorch 2.0+ detected, torch.compile is available")
+        if not IS_RELOADER:
+            print("PyTorch 2.0+ detected, torch.compile is available")
 except:
     pass
 
@@ -20,7 +32,8 @@ CUDA_GRAPHS_AVAILABLE = False
 try:
     if torch.cuda.is_available() and hasattr(torch.cuda, 'make_graphed_callables'):
         CUDA_GRAPHS_AVAILABLE = True
-        print("CUDA graphs support is available")
+        if not IS_RELOADER:
+            print("CUDA graphs support is available")
 except:
     pass
 
@@ -28,18 +41,21 @@ model = SNAC.from_pretrained("hubertsiuzdak/snac_24khz").eval()
 
 # Check if CUDA is available and set device accordingly
 snac_device = "cuda" if torch.cuda.is_available() else "mps" if torch.backends.mps.is_available() else "cpu"
-print(f"Using device: {snac_device}")
+if not IS_RELOADER:
+    print(f"Using device: {snac_device}")
 model = model.to(snac_device)
 
 # Disable torch.compile as it requires Triton which isn't installed
 # We'll use regular PyTorch optimization techniques instead
-print("Using standard PyTorch optimizations (torch.compile disabled)")
+if not IS_RELOADER:
+    print("Using standard PyTorch optimizations (torch.compile disabled)")
 
 # Prepare CUDA streams for parallel processing if available
 cuda_stream = None
 if snac_device == "cuda":
     cuda_stream = torch.cuda.Stream()
-    print("Using CUDA stream for parallel processing")
+    if not IS_RELOADER:
+        print("Using CUDA stream for parallel processing")
 
 
 def convert_to_audio(multiframe, count):
@@ -117,41 +133,71 @@ def convert_to_audio(multiframe, count):
             
     return audio_bytes
 
+# Define the custom token prefix
+CUSTOM_TOKEN_PREFIX = "<custom_token_"
+
+# Use a single global cache for token processing
+token_id_cache = {}
+MAX_CACHE_SIZE = 10000  # Increased cache size for better performance
+
 def turn_token_into_id(token_string, index):
-    """Optimized token-to-id conversion with early returns and minimal string operations"""
-    token_string = token_string.strip()
+    """
+    Optimized token-to-ID conversion with caching.
+    This is the definitive implementation used by both inference.py and speechpipe.py.
     
-    # Early return for obvious mismatches
-    if "<custom_token_" not in token_string:
+    Args:
+        token_string: The token string to convert
+        index: Position index used for token offset calculation
+        
+    Returns:
+        int: Token ID if valid, None otherwise
+    """
+    # Check cache first (significant speedup for repeated tokens)
+    cache_key = (token_string, index % 7)
+    if cache_key in token_id_cache:
+        return token_id_cache[cache_key]
+        
+    # Early rejection for obvious non-matches
+    if CUSTOM_TOKEN_PREFIX not in token_string:
         return None
+        
+    # Process token
+    token_string = token_string.strip()
+    last_token_start = token_string.rfind(CUSTOM_TOKEN_PREFIX)
     
-    # Find the last token in the string
-    last_token_start = token_string.rfind("<custom_token_")
     if last_token_start == -1:
         return None
     
-    # Check if the token ends properly
-    if not token_string.endswith(">"):
+    last_token = token_string[last_token_start:]
+    
+    if not (last_token.startswith(CUSTOM_TOKEN_PREFIX) and last_token.endswith(">")):
         return None
         
     try:
-        # Extract and convert the number directly
-        number_str = token_string[last_token_start+14:-1]
-        return int(number_str) - 10 - ((index % 7) * 4096)
+        number_str = last_token[14:-1]
+        token_id = int(number_str) - 10 - ((index % 7) * 4096)
+        
+        # Cache the result if it's valid
+        if len(token_id_cache) < MAX_CACHE_SIZE:
+            token_id_cache[cache_key] = token_id
+            
+        return token_id
     except (ValueError, IndexError):
         return None
-# Cache for frequently processed tokens to avoid redundant computation
-token_cache = {}
-MAX_CACHE_SIZE = 1000  # Limit cache size to prevent memory bloat
 
 async def tokens_decoder(token_gen):
-    """Optimized token decoder with reliable end-of-buffer handling for complete audio generation"""
+    """Optimized token decoder with early first-chunk processing for lower latency"""
     buffer = []
     count = 0
-    # Use a smaller minimum frame requirement to allow more flexible processing
-    min_frames_required = 28  # Lower requirement (4 chunks of 7 tokens)
-    ideal_frames = 49  # Ideal standard frame size (7×7 window)
-    process_every_n = 7  # Process every 7 tokens (standard for Orpheus model)
+    
+    # Track if first chunk has been processed
+    first_chunk_processed = False
+    
+    # Use different thresholds for first chunk vs. subsequent chunks
+    min_frames_first = 7  # Just one chunk (7 tokens) for first audio - ultra-low latency
+    min_frames_subsequent = 28  # Standard minimum (4 chunks of 7 tokens) after first audio
+    ideal_frames = 49  # Ideal standard frame size (7×7 window) - unchanged
+    process_every_n = 7  # Process every 7 tokens (standard for Orpheus model) - unchanged
     
     start_time = time.time()
     token_count = 0
@@ -160,15 +206,8 @@ async def tokens_decoder(token_gen):
     async for token_sim in token_gen:
         token_count += 1
         
-        # Check cache first to avoid redundant computation
-        cache_key = (token_sim, count % 7)
-        if cache_key in token_cache:
-            token = token_cache[cache_key]
-        else:
-            token = turn_token_into_id(token_sim, count)
-            # Add to cache if valid token
-            if token is not None and len(token_cache) < MAX_CACHE_SIZE:
-                token_cache[cache_key] = token
+        # Use the unified turn_token_into_id which already handles caching
+        token = turn_token_into_id(token_sim, count)
         
         if token is not None and token > 0:
             buffer.append(token)
@@ -185,26 +224,37 @@ async def tokens_decoder(token_gen):
                 last_log_time = current_time
                 token_count = 0
             
-            # Process standard batches when we have enough tokens
-            if count % process_every_n == 0:
-                # Best case: we have enough for the ideal frame size
-                if len(buffer) >= ideal_frames:
-                    buffer_to_proc = buffer[-ideal_frames:]
-                # Fallback: we have enough for the minimum requirement
-                elif len(buffer) >= min_frames_required:
-                    buffer_to_proc = buffer[-min_frames_required:]
-                # For the first few frames, we may not have enough yet
-                else:
-                    continue
-                
-                # Debug output to help diagnose issues
-                if count % 28 == 0:
-                    print(f"Processing buffer with {len(buffer_to_proc)} tokens, total collected: {len(buffer)}")
-                
-                # Process the tokens
-                audio_samples = convert_to_audio(buffer_to_proc, count)
-                if audio_samples is not None:
-                    yield audio_samples
+            # Different processing logic based on whether first chunk has been processed
+            if not first_chunk_processed:
+                # Process first chunk as soon as possible for minimal latency
+                if count >= min_frames_first:
+                    buffer_to_proc = buffer[-min_frames_first:]
+                    
+                    # Process the first chunk of audio for immediate feedback
+                    print(f"Processing first audio chunk with {len(buffer_to_proc)} tokens for low latency")
+                    audio_samples = convert_to_audio(buffer_to_proc, count)
+                    if audio_samples is not None:
+                        first_chunk_processed = True  # Mark first chunk as processed
+                        yield audio_samples
+            else:
+                # For subsequent chunks, use original processing with proper batching
+                if count % process_every_n == 0:
+                    # Use same prioritization logic as before
+                    if len(buffer) >= ideal_frames:
+                        buffer_to_proc = buffer[-ideal_frames:]
+                    elif len(buffer) >= min_frames_subsequent:
+                        buffer_to_proc = buffer[-min_frames_subsequent:]
+                    else:
+                        continue
+                    
+                    # Debug output to help diagnose issues
+                    if count % 28 == 0:
+                        print(f"Processing buffer with {len(buffer_to_proc)} tokens, total collected: {len(buffer)}")
+                    
+                    # Process the tokens
+                    audio_samples = convert_to_audio(buffer_to_proc, count)
+                    if audio_samples is not None:
+                        yield audio_samples
     
     # CRITICAL: End-of-generation handling - process all remaining frames
     # Process remaining complete frames (ideal size)
@@ -215,8 +265,8 @@ async def tokens_decoder(token_gen):
             yield audio_samples
             
     # Process any additional complete frames (minimum size)
-    elif len(buffer) >= min_frames_required:
-        buffer_to_proc = buffer[-min_frames_required:]
+    elif len(buffer) >= min_frames_subsequent:
+        buffer_to_proc = buffer[-min_frames_subsequent:]
         audio_samples = convert_to_audio(buffer_to_proc, count)
         if audio_samples is not None:
             yield audio_samples
@@ -227,7 +277,7 @@ async def tokens_decoder(token_gen):
         # Pad to minimum frame requirement with copies of the final token
         # This is more continuous than using unrelated tokens from the beginning
         last_token = buffer[-1]
-        padding_needed = min_frames_required - len(buffer)
+        padding_needed = min_frames_subsequent - len(buffer)
         
         # Create a padding array of copies of the last token
         # This maintains continuity much better than circular buffering