.dockerignore

@@ -1,3 +0,0 @@
-*.env
-models/
-*.gguf

.env.example

@@ -12,7 +12,6 @@ ORPHEUS_TOP_P=0.9
 # Repetition penalty is now hardcoded to 1.1 for stability (this is a model constraint) - this setting is no longer used
 # ORPHEUS_REPETITION_PENALTY=1.1
 ORPHEUS_SAMPLE_RATE=24000
-ORPHEUS_MODEL_NAME=Orpheus-3b-FT-Q8_0.gguf # Model name sent to inference server (Q2_K, Q4_K_M, or Q8_0 variants)
 
 # Web UI settings (keep in mind that the web UI is not secure and should not be exposed to the internet)
 ORPHEUS_PORT=5005

.gitignore

@@ -1,6 +0,0 @@
-.env
-__pycache__/
-.venv/
-venv/
-models/
-*.gguf

Dockerfile.cpu

@@ -1,47 +0,0 @@
-FROM ubuntu:22.04
-
-# Set non-interactive frontend
-ENV DEBIAN_FRONTEND=noninteractive
-
-# Install Python and other dependencies
-RUN apt-get update && apt-get install -y \
-    python3.10 \
-    python3-pip \
-    python3-venv \
-    libsndfile1 \
-    ffmpeg \
-    portaudio19-dev \
- && apt-get clean && rm -rf /var/lib/apt/lists/*
-
-# Create non-root user and set up directories
-RUN useradd -m -u 1001 appuser && \
-    mkdir -p /app/outputs /app && \
-    chown -R appuser:appuser /app
-
-USER appuser
-WORKDIR /app
-
-# Copy dependency files
-COPY --chown=appuser:appuser requirements.txt ./requirements.txt
-
-# Create and activate virtual environment
-RUN python3 -m venv /app/venv
-ENV PATH="/app/venv/bin:$PATH"
-
-# Install CPU-only PyTorch and other dependencies
-RUN pip3 install --no-cache-dir torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cpu && \
-    pip3 install --no-cache-dir -r requirements.txt
-
-# Copy project files
-COPY --chown=appuser:appuser . .
-
-# Set environment variables
-ENV PYTHONUNBUFFERED=1 \
-    PYTHONPATH=/app \
-    USE_GPU=false
-
-# Expose the port
-EXPOSE 5005
-
-# Run FastAPI server with uvicorn
-CMD ["uvicorn", "app:app", "--host", "0.0.0.0", "--port", "5005", "--workers", "1"]

Dockerfile.gpu

@@ -1,47 +0,0 @@
-FROM ubuntu:22.04
-
-# Set non-interactive frontend
-ENV DEBIAN_FRONTEND=noninteractive
-
-# Install Python and other dependencies
-RUN apt-get update && apt-get install -y \
-    python3.10 \
-    python3-pip \
-    python3-venv \
-    libsndfile1 \
-    ffmpeg \
-    portaudio19-dev \
- && apt-get clean && rm -rf /var/lib/apt/lists/*
-
-# Create non-root user and set up directories
-RUN useradd -m -u 1001 appuser && \
-    mkdir -p /app/outputs /app && \
-    chown -R appuser:appuser /app
-
-USER appuser
-WORKDIR /app
-
-# Copy dependency files
-COPY --chown=appuser:appuser requirements.txt ./requirements.txt
-
-# Create and activate virtual environment
-RUN python3 -m venv /app/venv
-ENV PATH="/app/venv/bin:$PATH"
-
-# Install PyTorch with CUDA support and other dependencies
-RUN pip3 install --no-cache-dir torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cu124 && \
-    pip3 install --no-cache-dir -r requirements.txt
-
-# Copy project files
-COPY --chown=appuser:appuser . .
-
-# Set environment variables
-ENV PYTHONUNBUFFERED=1 \
-    PYTHONPATH=/app \
-    USE_GPU=true
-
-# Expose the port
-EXPOSE 5005
-
-# Run FastAPI server with uvicorn
-CMD ["uvicorn", "app:app", "--host", "0.0.0.0", "--port", "5005", "--workers", "1"]

Dockerfile.gpu-rocm

@@ -1,50 +0,0 @@
-FROM rocm/dev-ubuntu-22.04:latest
-
-# Set non-interactive frontend
-ENV DEBIAN_FRONTEND=noninteractive
-
-# Install Python and other dependencies
-RUN apt-get update && apt-get install -y \
-    python3.10 \
-    python3-pip \
-    python3-venv \
-    libsndfile1 \
-    ffmpeg \
-    portaudio19-dev \
-    libjpeg-dev \
-    python3-dev \
-    && apt-get install python3-wheel python3-setuptools \
-    && apt-get clean && rm -rf /var/lib/apt/lists/*
-
-# Create non-root user and set up directories
-RUN useradd -m -u 1001 appuser && \
-    mkdir -p /app/outputs /app && \
-    chown -R appuser:appuser /app
-
-USER appuser
-WORKDIR /app
-
-# Copy dependency files
-COPY --chown=appuser:appuser requirements.txt ./requirements.txt
-
-# Create and activate virtual environment
-RUN python3 -m venv /app/venv
-ENV PATH="/app/venv/bin:$PATH"
-
-# Install PyTorch with ROCm support and other dependencies
-RUN pip3 install --no-cache-dir --pre torch torchvision torchaudio --index-url https://download.pytorch.org/whl/nightly/rocm6.4/ && \
-    pip3 install --no-cache-dir -r requirements.txt
-    
-# Copy project files
-COPY --chown=appuser:appuser . .
-
-# Set environment variables
-ENV PYTHONUNBUFFERED=1 \
-    PYTHONPATH=/app \
-    USE_GPU=true
-
-# Expose the port
-EXPOSE 5005
-
-# Run FastAPI server with uvicorn
-CMD ["uvicorn", "app:app", "--host", "0.0.0.0", "--port", "5005", "--workers", "1"]

README.md

@@ -4,58 +4,19 @@
 
 [![GitHub](https://img.shields.io/github/license/Lex-au/Orpheus-FastAPI)](https://github.com/Lex-au/Orpheus-FastAPI/blob/main/LICENSE.txt)
 
-High-performance Text-to-Speech server with OpenAI-compatible API, multilingual support with 24 voices, emotion tags, and modern web UI. Optimized for RTX GPUs.
+High-performance Text-to-Speech server with OpenAI-compatible API, 8 voices, emotion tags, and modern web UI. Optimized for RTX GPUs.
 
 ## Changelog
 
-**v1.3.1** (2025-07-05)
-- 🐳 ROCm Docker implementation contributed by [@wizardeur](https://github.com/wizardeur) – many thanks for your contribution ❤️
-
-**v1.3.0** (2025-04-18)
-- 🌐 Added comprehensive multilingual support with 16 new voice actors across 7 languages
-- 🗣️ New voice actors include:
-  - French: pierre, amelie, marie
-  - German: jana, thomas, max
-  - Korean: 유나, 준서
-  - Hindi: ऋतिका
-  - Mandarin: 长乐, 白芷
-  - Spanish: javi, sergio, maria
-  - Italian: pietro, giulia, carlo
-- 🔄 Enhanced UI with dynamic language selection and voice filtering
-- 🚀 Released language-specific optimized models:
-  - [Italian & Spanish Model](https://huggingface.co/lex-au/Orpheus-3b-Italian_Spanish-FT-Q8_0.gguf)
-  - [Korean Model](https://huggingface.co/lex-au/Orpheus-3b-Korean-FT-Q8_0.gguf)
-  - [French Model](https://huggingface.co/lex-au/Orpheus-3b-French-FT-Q8_0.gguf)
-  - [Hindi Model](https://huggingface.co/lex-au/Orpheus-3b-Hindi-FT-Q8_0.gguf)
-  - [Mandarin Model](https://huggingface.co/lex-au/Orpheus-3b-Chinese-FT-Q8_0.gguf)
-  - [German Model](https://huggingface.co/lex-au/Orpheus-3b-German-FT-Q8_0.gguf)
-- 🐳 Docker Compose users: To use a language-specific model, edit the `.env` file before installation and change `ORPHEUS_MODEL_NAME` to match the desired model repo ID (e.g., `Orpheus-3b-French-FT-Q8_0.gguf`)
-- An additional Docker Compose installation path is now available, specifically for CPU-bound scenarios. This contribution comes from [@alexjyong](https://github.com/alexjyong) - thank you!
-
-**v1.2.0** (2025-04-12)
-- ❤️ Added optional Docker Compose support with GPU-enabled `llama.cpp` server and Orpheus-FastAPI integration  
-- 🐳 Docker implementation contributed by [@richardr1126](https://github.com/richardr1126) – huge thanks for the clean setup and orchestration work!  
-- 🧱 Native install path remains unchanged for non-Docker users
-
 **v1.1.0** (2025-03-23)
 - ✨ Added long-form audio support with sentence-based batching and crossfade stitching
 - 🔊 Improved short audio quality with optimized token buffer handling
 - 🔄 Enhanced environment variable support with .env file loading (configurable via UI)
 - 🖥️ Added automatic hardware detection and optimization for different GPUs
 - 📊 Implemented detailed performance reporting for audio generation
-- ⚠️ Note: Python 3.12 is not supported due to removal of pkgutil.ImpImporter
 
 [GitHub Repository](https://github.com/Lex-au/Orpheus-FastAPI)
 
-## Model Collection
-
-🚀 **NEW:** Try the quantized models for improved performance!
-- **Q2_K**: Ultra-fast inference with 2-bit quantization
-- **Q4_K_M**: Balanced quality/speed with 4-bit quantization (mixed)
-- **Q8_0**: Original high-quality 8-bit model
-
-[Browse the Orpheus-FASTAPI Model Collection on HuggingFace](https://huggingface.co/collections/lex-au/orpheus-fastapi-67e125ae03fc96dae0517707)
-
 ## Voice Demos
 
 Listen to sample outputs with different voices and emotions:
@@ -73,7 +34,7 @@ Listen to sample outputs with different voices and emotions:
 - **OpenAI API Compatible**: Drop-in replacement for OpenAI's `/v1/audio/speech` endpoint
 - **Modern Web Interface**: Clean, responsive UI with waveform visualization
 - **High Performance**: Optimized for RTX GPUs with parallel processing
-- **Multilingual Support**: 24 different voices across 8 languages (English, French, German, Korean, Hindi, Mandarin, Spanish, Italian)
+- **Multiple Voices**: 8 different voice options with different characteristics
 - **Emotion Tags**: Support for laughter, sighs, and other emotional expressions
 - **Unlimited Audio Length**: Generate audio of any length through intelligent batching
 - **Smooth Transitions**: Crossfaded audio segments for seamless listening experience
@@ -86,8 +47,6 @@ Listen to sample outputs with different voices and emotions:
 ```
 Orpheus-FastAPI/
 ├── app.py                # FastAPI server and endpoints
-├── docker-compose.yml    # Docker compose configuration
-├── Dockerfile.gpu        # GPU-enabled Docker image
 ├── requirements.txt      # Dependencies
 ├── static/               # Static assets (favicon, etc.)
 ├── outputs/              # Generated audio files
@@ -103,47 +62,11 @@ Orpheus-FastAPI/
 
 ### Prerequisites
 
-- Python 3.8-3.11 (Python 3.12 is not supported due to removal of pkgutil.ImpImporter)
+- Python 3.8+
-- CUDA-compatible or ROCm-compatible GPU (recommended: RTX series for best performance)
+- CUDA-compatible GPU (recommended: RTX series for best performance)
-- Using docker compose or separate LLM inference server running the Orpheus model (e.g., LM Studio or llama.cpp server)
+- Separate LLM inference server running the Orpheus model (e.g., LM Studio or llama.cpp server)
-- For Docker GPU Support, ensure you're using an Nvidia GPU on either Linux or Windows with CUDA 12.4 or greater and NVIDIA Container Toolkit installed
 
-### 🐳 Docker compose
+### Installation
-
-The docker compose file orchestrates the Orpheus-FastAPI for audio and a llama.cpp inference server for the base model token generation. The GGUF model is downloaded with the model-init service.
-There are three versions, two for machines that have access to GPU support `docker-compose-gpu.yaml`, `docker-compose-gpu-rocm.yml`  and one for CPU support only: `docker-compose-cpu.yaml`
-
-```bash
-cp .env.example .env # Create your .env file from the example
-copy .env.example .env # For Windows CMD
-```
-
-For multilingual models, edit the `.env` file and change the model name:
-```
-# Change this line in .env to use a language-specific model
-ORPHEUS_MODEL_NAME=Orpheus-3b-French-FT-Q8_0.gguf  # Example for French
-```
-
-Then start the services:
-
-For CUDA GPU support run
-```bash
-docker compose -f docker-compose-gpu.yml up
-```
-
-For ROCm GPU support run
-```bash
-docker compose -f docker-compose-gpu-rocm.yml up
-```
-
-For CPU support run:
-```bash
-docker compose -f docker-compose-cpu.yml up
-```
-
-The system will automatically download the specified model from Hugging Face before starting the service.
-
-### FastAPI Service Native Installation
 
 1. Clone the repository:
 ```bash
@@ -166,11 +89,6 @@ conda activate orpheus-tts
 ```bash
 pip3 install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cu124
 ```
-or
-Install PyTorch with ROCm support:
-```bash
-pip3 install --pre torch torchvision torchaudio --index-url https://download.pytorch.org/whl/nightly/rocm6.4/
-```
 
 4. Install other dependencies:
 ```bash
@@ -246,7 +164,6 @@ curl -X POST http://localhost:5005/speak \
 
 ### Available Voices
 
-#### English
 - `tara`: Female, conversational, clear
 - `leah`: Female, warm, gentle
 - `jess`: Female, energetic, youthful
@@ -256,37 +173,6 @@ curl -X POST http://localhost:5005/speak \
 - `zac`: Male, enthusiastic, dynamic
 - `zoe`: Female, calm, soothing
 
-#### French
-- `pierre`: Male, sophisticated
-- `amelie`: Female, elegant
-- `marie`: Female, spirited
-
-#### German
-- `jana`: Female, clear
-- `thomas`: Male, authoritative
-- `max`: Male, energetic
-
-#### Korean
-- `유나`: Female, melodic
-- `준서`: Male, confident
-
-#### Hindi
-- `ऋतिका`: Female, expressive
-
-#### Mandarin
-- `长乐`: Female, gentle
-- `白芷`: Female, clear
-
-#### Spanish
-- `javi`: Male, warm
-- `sergio`: Male, professional
-- `maria`: Female, friendly
-
-#### Italian
-- `pietro`: Male, passionate
-- `giulia`: Female, expressive
-- `carlo`: Male, refined
-
 ### Emotion Tags
 
 You can insert emotion tags into your text to add expressiveness:
@@ -300,7 +186,7 @@ You can insert emotion tags into your text to add expressiveness:
 - `<yawn>`: Add a yawning sound
 - `<gasp>`: Add a gasping sound
 
-Example: `"Well, that's interesting <laugh> I hadn't thought of that before."`
+Example: "Well, that's interesting <laugh> I hadn't thought of that before."
 
 ## Technical Details
 
@@ -312,6 +198,8 @@ This server works as a frontend that connects to an external LLM inference serve
 - Token and audio caching
 - Optimised batch sizes
 
+For best performance, adjust the API_URL in `tts_engine/inference.py` to point to your LLM inference server endpoint.
+
 ### Hardware Detection and Optimization
 
 The system features intelligent hardware detection that automatically optimizes performance based on your hardware capabilities:
@@ -370,34 +258,27 @@ You can easily integrate this TTS solution with [OpenWebUI](https://github.com/o
 3. Change TTS from Web API to OpenAI
 4. Set APIBASE URL to your server address (e.g., `http://localhost:5005/v1`)
 5. API Key can be set to "not-needed"
-6. Set TTS Voice to any of the available voices (e.g., `tara`, `pierre`, `jana`, `유나`, etc.)
+6. Set TTS Voice to one of the available voices: `tara`, `leah`, `jess`, `leo`, `dan`, `mia`, `zac`, or `zoe`
 7. Set TTS Model to `tts-1`
 
 ### External Inference Server
 
-This application requires a separate LLM inference server running the Orpheus model. For easy setup, use Docker Compose, which automatically handles this for you. Alternatively, you can use:
+This application requires a separate LLM inference server running the Orpheus model. You can use:
 
 - [GPUStack](https://github.com/gpustack/gpustack) - GPU optimised LLM inference server (My pick) - supports LAN/WAN tensor split parallelisation
 - [LM Studio](https://lmstudio.ai/) - Load the GGUF model and start the local server
 - [llama.cpp server](https://github.com/ggerganov/llama.cpp) - Run with the appropriate model parameters
 - Any compatible OpenAI API-compatible server
 
-**Quantized Model Options:**
+Download the quantised model from [lex-au/Orpheus-3b-FT-Q8_0.gguf](https://huggingface.co/lex-au/Orpheus-3b-FT-Q8_0.gguf) and load it in your inference server.
-- **lex-au/Orpheus-3b-FT-Q2_K.gguf**: Fastest inference (~50% faster tokens/sec than Q8_0)
-- **lex-au/Orpheus-3b-FT-Q4_K_M.gguf**: Balanced quality/speed 
-- **lex-au/Orpheus-3b-FT-Q8_0.gguf**: Original high-quality model
-
-Choose based on your hardware and needs. Lower bit models (Q2_K, Q4_K_M) provide ~2x realtime performance on high-end GPUs.
-
-[Browse all models in the collection](https://huggingface.co/collections/lex-au/orpheus-fastapi-67e125ae03fc96dae0517707)
 
 The inference server should be configured to expose an API endpoint that this FastAPI application will connect to.
 
 ### Environment Variables
 
-Configure in docker compose, if using docker. Not using docker; create a `.env` file:
+You can configure the system using environment variables or a `.env` file:
 
-- `ORPHEUS_API_URL`: URL of the LLM inference API (default in Docker: http://llama-cpp-server:5006/v1/completions)
+- `ORPHEUS_API_URL`: URL of the LLM inference API (tts_engine/inference.py)
 - `ORPHEUS_API_TIMEOUT`: Timeout in seconds for API requests (default: 120)
 - `ORPHEUS_MAX_TOKENS`: Maximum tokens to generate (default: 8192)
 - `ORPHEUS_TEMPERATURE`: Temperature for generation (default: 0.6)
@@ -405,7 +286,6 @@ Configure in docker compose, if using docker. Not using docker; create a `.env`
 - `ORPHEUS_SAMPLE_RATE`: Audio sample rate in Hz (default: 24000)
 - `ORPHEUS_PORT`: Web server port (default: 5005)
 - `ORPHEUS_HOST`: Web server host (default: 0.0.0.0)
-- `ORPHEUS_MODEL_NAME`: Model name for inference server
 
 The system now supports loading environment variables from a `.env` file in the project root, making it easier to configure without modifying system-wide environment settings. See `.env.example` for a template.
 
@@ -432,7 +312,7 @@ To add new voices, update the `AVAILABLE_VOICES` list in `tts_engine/inference.p
 When running the Orpheus model with llama.cpp, use these parameters to ensure optimal performance:
 
 ```bash
-./llama-server -m models/Modelname.gguf \
+./llama-server -m models/Orpheus-3b-FT-Q8_0.gguf \
   --ctx-size={{your ORPHEUS_MAX_TOKENS from .env}} \
   --n-predict={{your ORPHEUS_MAX_TOKENS from .env}} \
   --rope-scaling=linear

app.py

@@ -11,30 +11,14 @@ from dotenv import load_dotenv
 
 # Function to ensure .env file exists
 def ensure_env_file_exists():
-    """Create a .env file from defaults and OS environment variables"""
+    """Create a default .env file if one doesn't exist"""
     if not os.path.exists(".env") and os.path.exists(".env.example"):
         try:
-            # 1. Create default env dictionary from .env.example
+            # Copy .env.example to .env
-            default_env = {}
             with open(".env.example", "r") as example_file:
-                for line in example_file:
+                with open(".env", "w") as env_file:
-                    line = line.strip()
+                    env_file.write(example_file.read())
-                    if line and not line.startswith("#") and "=" in line:
+            print("✅ Created default configuration file at .env")
-                        key = line.split("=")[0].strip()
-                        default_env[key] = line.split("=", 1)[1].strip()
-
-            # 2. Override defaults with Docker environment variables if they exist
-            final_env = default_env.copy()
-            for key in default_env:
-                if key in os.environ:
-                    final_env[key] = os.environ[key]
-
-            # 3. Write dictionary to .env file in env format
-            with open(".env", "w") as env_file:
-                for key, value in final_env.items():
-                    env_file.write(f"{key}={value}\n")
-                    
-            print("✅ Created default .env file from .env.example and environment variables.")
         except Exception as e:
             print(f"⚠️ Error creating default .env file: {e}")
 
@@ -51,7 +35,7 @@ from fastapi.templating import Jinja2Templates
 from pydantic import BaseModel
 import json
 
-from tts_engine import generate_speech_from_api, AVAILABLE_VOICES, DEFAULT_VOICE, VOICE_TO_LANGUAGE, AVAILABLE_LANGUAGES
+from tts_engine import generate_speech_from_api, AVAILABLE_VOICES, DEFAULT_VOICE
 
 # Create FastAPI app
 app = FastAPI(
@@ -130,18 +114,6 @@ async def create_speech_api(request: SpeechRequest):
         filename=f"{request.voice}_{timestamp}.wav"
     )
 
-@app.get("/v1/audio/voices")
-async def list_voices():
-    """Return list of available voices"""
-    if not AVAILABLE_VOICES or len(AVAILABLE_VOICES) == 0:
-        raise HTTPException(status_code=404, detail="No voices available")
-    return JSONResponse(
-        content={
-            "status": "ok",
-            "voices": AVAILABLE_VOICES
-        }
-    )
-
 # Legacy API endpoint for compatibility
 @app.post("/speak")
 async def speak(request: Request):
@@ -189,12 +161,7 @@ async def root(request: Request):
     """Redirect to web UI"""
     return templates.TemplateResponse(
         "tts.html",
-        {
+        {"request": request, "voices": AVAILABLE_VOICES}
-            "request": request, 
-            "voices": AVAILABLE_VOICES,
-            "VOICE_TO_LANGUAGE": VOICE_TO_LANGUAGE,
-            "AVAILABLE_LANGUAGES": AVAILABLE_LANGUAGES
-        }
     )
 
 @app.get("/web/", response_class=HTMLResponse)
@@ -204,13 +171,7 @@ async def web_ui(request: Request):
     config = get_current_config()
     return templates.TemplateResponse(
         "tts.html",
-        {
+        {"request": request, "voices": AVAILABLE_VOICES, "config": config}
-            "request": request, 
-            "voices": AVAILABLE_VOICES, 
-            "config": config,
-            "VOICE_TO_LANGUAGE": VOICE_TO_LANGUAGE,
-            "AVAILABLE_LANGUAGES": AVAILABLE_LANGUAGES
-        }
     )
 
 @app.get("/get_config")
@@ -312,9 +273,7 @@ async def generate_from_web(
             {
                 "request": request,
                 "error": "Please enter some text.",
-                "voices": AVAILABLE_VOICES,
+                "voices": AVAILABLE_VOICES
-                "VOICE_TO_LANGUAGE": VOICE_TO_LANGUAGE,
-                "AVAILABLE_LANGUAGES": AVAILABLE_LANGUAGES
             }
         )
     
@@ -347,9 +306,7 @@ async def generate_from_web(
             "voice": voice,
             "output_file": output_path,
             "generation_time": generation_time,
-            "voices": AVAILABLE_VOICES,
+            "voices": AVAILABLE_VOICES
-            "VOICE_TO_LANGUAGE": VOICE_TO_LANGUAGE,
-            "AVAILABLE_LANGUAGES": AVAILABLE_LANGUAGES
         }
     )
 

docker-compose-cpu.yaml

@@ -1,56 +0,0 @@
-services:
-  orpheus-fastapi:
-    container_name: orpheus-fastapi
-    build:
-      context: .
-      dockerfile: Dockerfile.cpu  
-    ports:
-      - "5005:5005"
-    env_file:
-      - .env
-    environment:
-      - ORPHEUS_API_URL=http://llama-cpp-server:5006/v1/completions
-    restart: unless-stopped
-    depends_on:
-      llama-cpp-server:
-        condition: service_started
-
-  llama-cpp-server:
-    image: ghcr.io/ggml-org/llama.cpp:server
-    ports:
-      - "5006:5006"
-    volumes:
-      - ./models:/models
-    env_file:
-      - .env
-    depends_on:
-      model-init:
-        condition: service_completed_successfully
-    restart: unless-stopped
-    command: >
-      -m /models/${ORPHEUS_MODEL_NAME}
-      --host 0.0.0.0
-      --port 5006
-      --ctx-size ${ORPHEUS_MAX_TOKENS}
-      --n-predict ${ORPHEUS_MAX_TOKENS}
-      --threads ${LLAMA_CPU_THREADS:-6}
-      --threads-batch ${LLAMA_CPU_THREADS:-6}
-      --rope-scaling linear
-      --no-mmap
-      --no-slots
-      --no-webui
-  model-init:
-    image: curlimages/curl:latest
-    user: ${UID}:${GID}
-    volumes:
-      - ./models:/app/models
-    working_dir: /app
-    command: >
-      sh -c '
-      if [ ! -f /app/models/${ORPHEUS_MODEL_NAME} ]; then
-        echo "Downloading model file..."
-        wget -P /app/models https://huggingface.co/lex-au/${ORPHEUS_MODEL_NAME}/resolve/main/${ORPHEUS_MODEL_NAME}
-      else
-        echo "Model file already exists"
-      fi'
-    restart: "no"

docker-compose-gpu-rocm.yml

@@ -1,86 +0,0 @@
-services:
-  orpheus-fastapi:
-    container_name: orpheus-fastapi
-    image: orpheus-tts-fastapi-server-orpheus-fastapi:latest
-    build:
-      context: .
-      dockerfile: Dockerfile.gpu-rocm
-    ports:
-      - "5005:5005"
-    env_file:
-      - .env
-    environment:
-      - ORPHEUS_API_URL=http://llama-cpp-server:5006/v1/completions
-    ipc: host
-    privileged: true
-    security_opt:
-      - seccomp=unconfined
-    cap_add:
-      - SYS_PTRACE
-      - CAP_SYS_ADMIN
-    devices:
-      - /dev/kfd
-      - /dev/dri
-      - /dev/mem
-    group_add:
-      - video
-      - render
-      # - 993 # Add numeric render/video group id(s) if your system has different group id(s) than the image - 44(video),109(render)
-    shm_size: 8g
-    restart: unless-stopped
-    depends_on:
-      llama-cpp-server:
-        condition: service_started
-
-  llama-cpp-server:
-    image: ghcr.io/ggml-org/llama.cpp:server-vulkan
-    ports:
-      - "5006:5006"
-    volumes:
-      - ./models:/models
-    env_file:
-      - .env
-    depends_on:
-      model-init:
-        condition: service_completed_successfully
-    cap_add:
-      - SYS_PTRACE
-      - CAP_SYS_ADMIN
-    security_opt:
-      - seccomp=unconfined
-    privileged: true
-    devices:
-      - /dev/kfd
-      - /dev/dri
-      - /dev/mem
-    group_add:
-      - video
-      - 993
-    ipc: host
-    shm_size: 8g
-    restart: unless-stopped
-    command: >
-      -m /models/${ORPHEUS_MODEL_NAME}
-      --port 5006 
-      --host 0.0.0.0 
-      --n-gpu-layers 29
-      --ctx-size ${ORPHEUS_MAX_TOKENS}
-      --n-predict ${ORPHEUS_MAX_TOKENS}
-      --rope-scaling linear
-
-  model-init:
-    image: curlimages/curl:latest
-    user: ${UID}:${GID}
-    volumes:
-      - ./models:/app/models
-    working_dir: /app
-    command: >
-      sh -c '
-      if [ ! -f /app/models/${ORPHEUS_MODEL_NAME} ]; then
-        echo "Downloading model file..."
-        wget -P /app/models https://huggingface.co/lex-au/${ORPHEUS_MODEL_NAME}/resolve/main/${ORPHEUS_MODEL_NAME}
-      else
-        echo "Model file already exists"
-      fi'
-    restart: "no"
-    

docker-compose-gpu.yml

@@ -1,68 +0,0 @@
-services:
-  orpheus-fastapi:
-    container_name: orpheus-fastapi
-    build:
-      context: .
-      dockerfile: Dockerfile.gpu
-    ports:
-      - "5005:5005"
-    env_file:
-      - .env
-    environment:
-      - ORPHEUS_API_URL=http://llama-cpp-server:5006/v1/completions
-    deploy:
-      resources:
-        reservations:
-          devices:
-            - driver: nvidia
-              count: all
-              capabilities: [gpu]
-    restart: unless-stopped
-    depends_on:
-      llama-cpp-server:
-        condition: service_started
-
-  llama-cpp-server:
-    image: ghcr.io/ggml-org/llama.cpp:server-cuda
-    ports:
-      - "5006:5006"
-    volumes:
-      - ./models:/models
-    env_file:
-      - .env
-    depends_on:
-      model-init:
-        condition: service_completed_successfully
-    deploy:
-      resources:
-        reservations:
-          devices:
-            - driver: nvidia
-              count: all
-              capabilities: [gpu]
-    restart: unless-stopped
-    command: >
-      -m /models/${ORPHEUS_MODEL_NAME}
-      --port 5006 
-      --host 0.0.0.0 
-      --n-gpu-layers 29
-      --ctx-size ${ORPHEUS_MAX_TOKENS}
-      --n-predict ${ORPHEUS_MAX_TOKENS}
-      --rope-scaling linear
-
-  model-init:
-    image: curlimages/curl:latest
-    user: ${UID}:${GID}
-    volumes:
-      - ./models:/app/models
-    working_dir: /app
-    command: >
-      sh -c '
-      if [ ! -f /app/models/${ORPHEUS_MODEL_NAME} ]; then
-        echo "Downloading model file..."
-        wget -P /app/models https://huggingface.co/lex-au/${ORPHEUS_MODEL_NAME}/resolve/main/${ORPHEUS_MODEL_NAME}
-      else
-        echo "Model file already exists"
-      fi'
-    restart: "no"
-    

docs/index.html

@@ -64,15 +64,11 @@
                 <td>Default Test (MP3)</td>
                 <td><a href="DefaultTest.mp3" target="_blank">DefaultTest.mp3</a></td>
             </tr>
-            <tr>
-                <td>Default Test (MP3)</td>
-                <td><a href="TaraLongGeneration.mp3" target="_blank">TaraLongGeneration.mp3</a></td>
-            </tr>
             <tr>
                 <td>WebUI Screenshot (PNG)</td>
                 <td>
-                    <a href="WebUI.png" target="_blank">Webui.png</a><br>
+                    <a href="Webui.png" target="_blank">Webui.png</a><br>
-                    <img src="WebUI.png" alt="WebUI Preview">
+                    <img src="Webui.png" alt="WebUI Preview">
                 </td>
             </tr>
             <tr>

requirements.txt

@@ -8,7 +8,7 @@ python-multipart==0.0.6
 # API and Communication
 requests==2.31.0
 python-dotenv==1.0.0
-watchfiles==1.0.4
+watchfiles=1.0.4
 
 # Audio Processing
 numpy==1.24.0

templates/tts.html

@@ -155,72 +155,25 @@
                 </div>
               </div>
 
-              <!-- Language selection -->
-              <div class="mb-6">
-                <label class="block text-sm font-medium text-white mb-2">Language</label>
-                <div class="flex flex-wrap gap-3">
-                  {% for language in AVAILABLE_LANGUAGES %}
-                  <div class="language-option {% if language == 'english' %}active{% endif %}" data-language="{{ language }}">
-                    <input type="radio" name="language" value="{{ language }}" class="hidden" {% if language == 'english' %}checked{% endif %}>
-                    <span class="px-3 py-1 rounded-full bg-dark-700 hover:bg-dark-600 text-white text-sm cursor-pointer">{{ language|capitalize }}</span>
-                  </div>
-                  {% endfor %}
-                </div>
-              </div>
-
               <!-- Voice selection -->
               <div class="mb-6">
                 <label class="block text-sm font-medium text-white mb-2">Voice</label>
                 <div class="grid grid-cols-1 md:grid-cols-2 lg:grid-cols-4 gap-4">
                   {% for voice_option in voices %}
-                  <div class="voice-card {% if voice_option == DEFAULT_VOICE %}active{% endif %}" 
+                  <div class="voice-card {% if voice_option == DEFAULT_VOICE %}active{% endif %}" data-voice="{{ voice_option }}">
-                       data-voice="{{ voice_option }}" 
-                       data-language="{{ VOICE_TO_LANGUAGE[voice_option] }}"
-                       style="display: {% if VOICE_TO_LANGUAGE[voice_option] != 'english' %}none{% endif %}">
                     <input type="radio" name="voice" value="{{ voice_option }}" class="hidden" {% if voice_option == DEFAULT_VOICE %}checked{% endif %}>
                     <div class="flex items-center mb-2">
                       <span class="font-medium text-white">{{ voice_option|capitalize }}</span>
                     </div>
-                    <div class="text-xs text-dark-300">
+    <div class="text-xs text-dark-300">
-                      {% if voice_option == "tara" %}Female, English, conversational, clear
+                      {% if voice_option == "tara" %}Female, conversational, clear
-                      {% elif voice_option == "leah" %}Female, English, warm, gentle
+                      {% elif voice_option == "leah" %}Female, warm, gentle
-                      {% elif voice_option == "jess" %}Female, English, energetic, youthful
+                      {% elif voice_option == "jess" %}Female, energetic, youthful
-                      {% elif voice_option == "leo" %}Male, English, authoritative, deep
+                      {% elif voice_option == "leo" %}Male, authoritative, deep
-                      {% elif voice_option == "dan" %}Male, English, friendly, casual
+                      {% elif voice_option == "dan" %}Male, friendly, casual
-                      {% elif voice_option == "mia" %}Female, English, professional, articulate
+                      {% elif voice_option == "mia" %}Female, professional, articulate
-                      {% elif voice_option == "zac" %}Male, English, enthusiastic, dynamic
+                      {% elif voice_option == "zac" %}Male, enthusiastic, dynamic
-                      {% elif voice_option == "zoe" %}Female, English, calm, soothing
+                      {% elif voice_option == "zoe" %}Female, calm, soothing
-                      
-                      <!-- French voices -->
-                      {% elif voice_option == "pierre" %}Male, French, sophisticated
-                      {% elif voice_option == "amelie" %}Female, French, elegant
-                      {% elif voice_option == "marie" %}Female, French, spirited
-                      
-                      <!-- German voices -->
-                      {% elif voice_option == "jana" %}Female, German, clear
-                      {% elif voice_option == "thomas" %}Male, German, authoritative
-                      {% elif voice_option == "max" %}Male, German, energetic
-                      
-                      <!-- Korean voices -->
-                      {% elif voice_option == "유나" %}Female, Korean, melodic
-                      {% elif voice_option == "준서" %}Male, Korean, confident
-                      
-                      <!-- Hindi voice -->
-                      {% elif voice_option == "ऋतिका" %}Female, Hindi, expressive
-                      
-                      <!-- Mandarin voices -->
-                      {% elif voice_option == "长乐" %}Female, Mandarin, gentle
-                      {% elif voice_option == "白芷" %}Female, Mandarin, clear
-                      
-                      <!-- Spanish voices -->
-                      {% elif voice_option == "javi" %}Male, Spanish, warm
-                      {% elif voice_option == "sergio" %}Male, Spanish, professional
-                      {% elif voice_option == "maria" %}Female, Spanish, friendly
-                      
-                      <!-- Italian voices -->
-                      {% elif voice_option == "pietro" %}Male, Italian, passionate
-                      {% elif voice_option == "giulia" %}Female, Italian, expressive
-                      {% elif voice_option == "carlo" %}Male, Italian, refined
                       {% endif %}
                     </div>
                   </div>
@@ -451,83 +404,20 @@
       // Initialize char count
       charCount.textContent = textArea.value.length;
       
-      // Language selection
+      // Voice selection
-      const languageOptions = document.querySelectorAll('.language-option');
       const voiceCards = document.querySelectorAll('.voice-card');
       
-      languageOptions.forEach(option => {
+      voiceCards.forEach(card => {
-        option.addEventListener('click', function() {
+        card.addEventListener('click', function() {
-          // Unselect all language options
+          // Unselect all cards
-          languageOptions.forEach(o => o.classList.remove('active'));
+          voiceCards.forEach(c => c.classList.remove('active'));
           
-          // Select this language option
+          // Select this card
           this.classList.add('active');
           
           // Check the radio button
           const radio = this.querySelector('input[type="radio"]');
           radio.checked = true;
-          
-          // Get the selected language
-          const selectedLanguage = this.getAttribute('data-language');
-          
-          // Show/hide voice cards based on language
-          let firstVisibleCard = null;
-          
-          voiceCards.forEach(card => {
-            const cardLanguage = card.getAttribute('data-language');
-            if (cardLanguage === selectedLanguage) {
-              card.style.display = '';
-              if (!firstVisibleCard) {
-                firstVisibleCard = card;
-              }
-            } else {
-              card.style.display = 'none';
-              // If this hidden card was selected, unselect it
-              if (card.classList.contains('active')) {
-                card.classList.remove('active');
-                const cardRadio = card.querySelector('input[type="radio"]');
-                if (cardRadio) {
-                  cardRadio.checked = false;
-                }
-              }
-            }
-          });
-          
-          // If no voice card is selected for this language, select the first one
-          if (firstVisibleCard && document.querySelectorAll(`.voice-card[data-language="${selectedLanguage}"].active:not([style*="display: none"])`).length === 0) {
-            firstVisibleCard.classList.add('active');
-            const firstVisibleRadio = firstVisibleCard.querySelector('input[type="radio"]');
-            if (firstVisibleRadio) {
-              firstVisibleRadio.checked = true;
-            }
-          }
-        });
-      });
-      
-      // Voice card selection
-      voiceCards.forEach(card => {
-        card.addEventListener('click', function() {
-          // Only allow selection of cards that are visible (not display:none)
-          if (this.style.display !== 'none') {
-            // Unselect all cards with the same language
-            const cardLanguage = this.getAttribute('data-language');
-            document.querySelectorAll(`.voice-card[data-language="${cardLanguage}"]`).forEach(c => {
-              c.classList.remove('active');
-              const radio = c.querySelector('input[type="radio"]');
-              if (radio) {
-                radio.checked = false;
-              }
-            });
-            
-            // Select this card
-            this.classList.add('active');
-            
-            // Check the radio button
-            const radio = this.querySelector('input[type="radio"]');
-            if (radio) {
-              radio.checked = true;
-            }
-          }
         });
       });
       

tts_engine/__init__.py

@@ -11,7 +11,5 @@ from .inference import (
     generate_speech_from_api,
     AVAILABLE_VOICES,
     DEFAULT_VOICE,
-    VOICE_TO_LANGUAGE,
-    AVAILABLE_LANGUAGES,
     list_available_voices
 )

tts_engine/inference.py

@@ -136,49 +136,14 @@ if not IS_RELOADER:
 # Parallel processing settings
 NUM_WORKERS = 4 if HIGH_END_GPU else 2
 
-# Define voices by language
+# Available voices based on the Orpheus-TTS repository
-ENGLISH_VOICES = ["tara", "leah", "jess", "leo", "dan", "mia", "zac", "zoe"]
+AVAILABLE_VOICES = ["tara", "leah", "jess", "leo", "dan", "mia", "zac", "zoe"]
-FRENCH_VOICES = ["pierre", "amelie", "marie"]
-GERMAN_VOICES = ["jana", "thomas", "max"]
-KOREAN_VOICES = ["유나", "준서"]
-HINDI_VOICES = ["ऋतिका"]
-MANDARIN_VOICES = ["长乐", "白芷"]
-SPANISH_VOICES = ["javi", "sergio", "maria"]
-ITALIAN_VOICES = ["pietro", "giulia", "carlo"]
-
-# Combined list for API compatibility
-AVAILABLE_VOICES = (
-    ENGLISH_VOICES + 
-    FRENCH_VOICES + 
-    GERMAN_VOICES + 
-    KOREAN_VOICES + 
-    HINDI_VOICES + 
-    MANDARIN_VOICES + 
-    SPANISH_VOICES + 
-    ITALIAN_VOICES
-)
 DEFAULT_VOICE = "tara"  # Best voice according to documentation
 
-# Map voices to languages for the UI
-VOICE_TO_LANGUAGE = {}
-VOICE_TO_LANGUAGE.update({voice: "english" for voice in ENGLISH_VOICES})
-VOICE_TO_LANGUAGE.update({voice: "french" for voice in FRENCH_VOICES})
-VOICE_TO_LANGUAGE.update({voice: "german" for voice in GERMAN_VOICES})
-VOICE_TO_LANGUAGE.update({voice: "korean" for voice in KOREAN_VOICES})
-VOICE_TO_LANGUAGE.update({voice: "hindi" for voice in HINDI_VOICES})
-VOICE_TO_LANGUAGE.update({voice: "mandarin" for voice in MANDARIN_VOICES})
-VOICE_TO_LANGUAGE.update({voice: "spanish" for voice in SPANISH_VOICES})
-VOICE_TO_LANGUAGE.update({voice: "italian" for voice in ITALIAN_VOICES})
-
-# Languages list for the UI
-AVAILABLE_LANGUAGES = ["english", "french", "german", "korean", "hindi", "mandarin", "spanish", "italian"]
-
-# Import the unified token handling from speechpipe
-from .speechpipe import turn_token_into_id, CUSTOM_TOKEN_PREFIX
-
 # Special token IDs for Orpheus model
 START_TOKEN_ID = 128259
 END_TOKEN_IDS = [128009, 128260, 128261, 128257]
+CUSTOM_TOKEN_PREFIX = "<custom_token_"
 
 # Performance monitoring
 class PerformanceMonitor:
@@ -251,8 +216,9 @@ def generate_tokens_from_api(prompt: str, voice: str = DEFAULT_VOICE, temperatur
     elif torch.cuda.is_available():
         print("Using optimized parameters for GPU acceleration")
     
-    # Create the request payload (model field may not be required by some endpoints but included for compatibility)
+    # Create the request payload
     payload = {
+        "model": "orpheus-3b-0.1-ft-q4_k_m",  # Model name can be anything, endpoint will use loaded model
         "prompt": formatted_prompt,
         "max_tokens": max_tokens,
         "temperature": temperature,
@@ -261,11 +227,6 @@ def generate_tokens_from_api(prompt: str, voice: str = DEFAULT_VOICE, temperatur
         "stream": True  # Always stream for better performance
     }
     
-    # Add model field - this is ignored by many local inference servers for /v1/completions
-    # but included for compatibility with OpenAI API and some servers that may use it
-    model_name = os.environ.get("ORPHEUS_MODEL_NAME", "Orpheus-3b-FT-Q8_0.gguf")
-    payload["model"] = model_name
-    
     # Session for connection pooling and retry logic
     session = requests.Session()
     
@@ -312,14 +273,12 @@ def generate_tokens_from_api(prompt: str, voice: str = DEFAULT_VOICE, temperatur
                         try:
                             data = json.loads(data_str)
                             if 'choices' in data and len(data['choices']) > 0:
-                                token_chunk = data['choices'][0].get('text', '')
+                                token_text = data['choices'][0].get('text', '')
-                                for token_text in token_chunk.split('>'):
+                                token_counter += 1
-                                    token_text = f'{token_text}>'
+                                perf_monitor.add_tokens()
-                                    token_counter += 1
+                                
-                                    perf_monitor.add_tokens()
+                                if token_text:
-
+                                    yield token_text
-                                    if token_text:
-                                        yield token_text
                         except json.JSONDecodeError as e:
                             print(f"Error decoding JSON: {e}")
                             continue
@@ -352,8 +311,44 @@ def generate_tokens_from_api(prompt: str, voice: str = DEFAULT_VOICE, temperatur
                 print("Max retries reached. Token generation failed.")
                 return
 
-# The turn_token_into_id function is now imported from speechpipe.py
+# Token ID cache to avoid repeated processing
-# This eliminates duplicate code and ensures consistent behavior
+token_id_cache = {}
+MAX_CACHE_SIZE = 10000
+
+def turn_token_into_id(token_string: str, index: int) -> Optional[int]:
+    """Optimized token-to-ID conversion with caching."""
+    # Check cache first (significant speedup for repeated tokens)
+    cache_key = (token_string, index % 7)
+    if cache_key in token_id_cache:
+        return token_id_cache[cache_key]
+        
+    # Early rejection for obvious non-matches
+    if CUSTOM_TOKEN_PREFIX not in token_string:
+        return None
+        
+    # Process token
+    token_string = token_string.strip()
+    last_token_start = token_string.rfind(CUSTOM_TOKEN_PREFIX)
+    
+    if last_token_start == -1:
+        return None
+    
+    last_token = token_string[last_token_start:]
+    
+    if not (last_token.startswith(CUSTOM_TOKEN_PREFIX) and last_token.endswith(">")):
+        return None
+        
+    try:
+        number_str = last_token[14:-1]
+        token_id = int(number_str) - 10 - ((index % 7) * 4096)
+        
+        # Cache the result if it's valid
+        if len(token_id_cache) < MAX_CACHE_SIZE:
+            token_id_cache[cache_key] = token_id
+            
+        return token_id
+    except (ValueError, IndexError):
+        return None
 
 def convert_to_audio(multiframe: List[int], count: int) -> Optional[bytes]:
     """Convert token frames to audio with performance monitoring."""
@@ -368,15 +363,13 @@ def convert_to_audio(multiframe: List[int], count: int) -> Optional[bytes]:
     return result
 
 async def tokens_decoder(token_gen) -> Generator[bytes, None, None]:
-    """Simplified token decoder with early first-chunk processing for lower latency."""
+    """Simplified token decoder without complex ring buffer to ensure reliable output."""
     buffer = []
     count = 0
     
-    # Use different thresholds for first chunk vs. subsequent chunks
+    # Use conservative batch parameters to ensure output quality
-    first_chunk_processed = False
+    min_frames = 28  # Default for reliability (4 chunks of 7)
-    min_frames_first = 7  # Process after just 7 tokens for first chunk (ultra-low latency)
+    process_every = 7  # Process every 7 tokens (standard for Orpheus)
-    min_frames_subsequent = 28  # Default for reliability after first chunk (4 chunks of 7)
-    process_every = 7  # Process every 7 tokens (standard for Orpheus model)
     
     start_time = time.time()
     last_log_time = start_time
@@ -398,32 +391,19 @@ async def tokens_decoder(token_gen) -> Generator[bytes, None, None]:
                     print(f"Token processing rate: {token_count/elapsed:.1f} tokens/second")
                 last_log_time = current_time
             
-            # Different processing paths based on whether first chunk has been processed
+            # Process in standard batches for Orpheus model
-            if not first_chunk_processed:
+            if count % process_every == 0 and count >= min_frames:
-                # For first audio output, process as soon as we have enough tokens for one chunk
+                # Use simple slice operation - reliable and correct
-                if count >= min_frames_first:
+                buffer_to_proc = buffer[-min_frames:]
-                    buffer_to_proc = buffer[-min_frames_first:]
+                
-                    
+                # Debug output to help diagnose issues
-                    # Process the first chunk for immediate audio feedback
+                if count % 28 == 0:
-                    print(f"Processing first audio chunk with {len(buffer_to_proc)} tokens")
+                    print(f"Processing buffer with {len(buffer_to_proc)} tokens, total collected: {len(buffer)}")
-                    audio_samples = convert_to_audio(buffer_to_proc, count)
+                
-                    if audio_samples is not None:
+                # Process the tokens
-                        first_chunk_processed = True  # Mark first chunk as processed
+                audio_samples = convert_to_audio(buffer_to_proc, count)
-                        yield audio_samples
+                if audio_samples is not None:
-            else:
+                    yield audio_samples
-                # For subsequent chunks, use standard processing with larger batch
-                if count % process_every == 0 and count >= min_frames_subsequent:
-                    # Use simple slice operation - reliable and correct
-                    buffer_to_proc = buffer[-min_frames_subsequent:]
-                    
-                    # Debug output to help diagnose issues
-                    if count % 28 == 0:
-                        print(f"Processing buffer with {len(buffer_to_proc)} tokens, total collected: {len(buffer)}")
-                    
-                    # Process the tokens
-                    audio_samples = convert_to_audio(buffer_to_proc, count)
-                    if audio_samples is not None:
-                        yield audio_samples
 
 def tokens_decoder_sync(syn_token_gen, output_file=None):
     """Optimized synchronous wrapper with parallel processing and efficient file I/O."""

tts_engine/speechpipe.py

@@ -5,25 +5,13 @@ import asyncio
 import threading
 import queue
 import time
-import os
-import sys
-
-# Helper to detect if running in Uvicorn's reloader (same as in inference.py)
-def is_reloader_process():
-    """Check if the current process is a uvicorn reloader"""
-    return (sys.argv[0].endswith('_continuation.py') or 
-            os.environ.get('UVICORN_STARTED') == 'true')
-
-# Set a flag to avoid repeat messages
-IS_RELOADER = is_reloader_process()
 
 # Try to enable torch.compile if PyTorch 2.0+ is available
 TORCH_COMPILE_AVAILABLE = False
 try:
     if hasattr(torch, 'compile'):
         TORCH_COMPILE_AVAILABLE = True
-        if not IS_RELOADER:
+        print("PyTorch 2.0+ detected, torch.compile is available")
-            print("PyTorch 2.0+ detected, torch.compile is available")
 except:
     pass
 
@@ -32,8 +20,7 @@ CUDA_GRAPHS_AVAILABLE = False
 try:
     if torch.cuda.is_available() and hasattr(torch.cuda, 'make_graphed_callables'):
         CUDA_GRAPHS_AVAILABLE = True
-        if not IS_RELOADER:
+        print("CUDA graphs support is available")
-            print("CUDA graphs support is available")
 except:
     pass
 
@@ -41,21 +28,18 @@ model = SNAC.from_pretrained("hubertsiuzdak/snac_24khz").eval()
 
 # Check if CUDA is available and set device accordingly
 snac_device = "cuda" if torch.cuda.is_available() else "mps" if torch.backends.mps.is_available() else "cpu"
-if not IS_RELOADER:
+print(f"Using device: {snac_device}")
-    print(f"Using device: {snac_device}")
 model = model.to(snac_device)
 
 # Disable torch.compile as it requires Triton which isn't installed
 # We'll use regular PyTorch optimization techniques instead
-if not IS_RELOADER:
+print("Using standard PyTorch optimizations (torch.compile disabled)")
-    print("Using standard PyTorch optimizations (torch.compile disabled)")
 
 # Prepare CUDA streams for parallel processing if available
 cuda_stream = None
 if snac_device == "cuda":
     cuda_stream = torch.cuda.Stream()
-    if not IS_RELOADER:
+    print("Using CUDA stream for parallel processing")
-        print("Using CUDA stream for parallel processing")
 
 
 def convert_to_audio(multiframe, count):
@@ -133,71 +117,41 @@ def convert_to_audio(multiframe, count):
             
     return audio_bytes
 
-# Define the custom token prefix
-CUSTOM_TOKEN_PREFIX = "<custom_token_"
-
-# Use a single global cache for token processing
-token_id_cache = {}
-MAX_CACHE_SIZE = 10000  # Increased cache size for better performance
-
 def turn_token_into_id(token_string, index):
-    """
+    """Optimized token-to-id conversion with early returns and minimal string operations"""
-    Optimized token-to-ID conversion with caching.
-    This is the definitive implementation used by both inference.py and speechpipe.py.
-    
-    Args:
-        token_string: The token string to convert
-        index: Position index used for token offset calculation
-        
-    Returns:
-        int: Token ID if valid, None otherwise
-    """
-    # Check cache first (significant speedup for repeated tokens)
-    cache_key = (token_string, index % 7)
-    if cache_key in token_id_cache:
-        return token_id_cache[cache_key]
-        
-    # Early rejection for obvious non-matches
-    if CUSTOM_TOKEN_PREFIX not in token_string:
-        return None
-        
-    # Process token
     token_string = token_string.strip()
-    last_token_start = token_string.rfind(CUSTOM_TOKEN_PREFIX)
     
+    # Early return for obvious mismatches
+    if "<custom_token_" not in token_string:
+        return None
+    
+    # Find the last token in the string
+    last_token_start = token_string.rfind("<custom_token_")
     if last_token_start == -1:
         return None
     
-    last_token = token_string[last_token_start:]
+    # Check if the token ends properly
-    
+    if not token_string.endswith(">"):
-    if not (last_token.startswith(CUSTOM_TOKEN_PREFIX) and last_token.endswith(">")):
         return None
         
     try:
-        number_str = last_token[14:-1]
+        # Extract and convert the number directly
-        token_id = int(number_str) - 10 - ((index % 7) * 4096)
+        number_str = token_string[last_token_start+14:-1]
-        
+        return int(number_str) - 10 - ((index % 7) * 4096)
-        # Cache the result if it's valid
-        if len(token_id_cache) < MAX_CACHE_SIZE:
-            token_id_cache[cache_key] = token_id
-            
-        return token_id
     except (ValueError, IndexError):
         return None
+# Cache for frequently processed tokens to avoid redundant computation
+token_cache = {}
+MAX_CACHE_SIZE = 1000  # Limit cache size to prevent memory bloat
 
 async def tokens_decoder(token_gen):
-    """Optimized token decoder with early first-chunk processing for lower latency"""
+    """Optimized token decoder with reliable end-of-buffer handling for complete audio generation"""
     buffer = []
     count = 0
-    
+    # Use a smaller minimum frame requirement to allow more flexible processing
-    # Track if first chunk has been processed
+    min_frames_required = 28  # Lower requirement (4 chunks of 7 tokens)
-    first_chunk_processed = False
+    ideal_frames = 49  # Ideal standard frame size (7×7 window)
-    
+    process_every_n = 7  # Process every 7 tokens (standard for Orpheus model)
-    # Use different thresholds for first chunk vs. subsequent chunks
-    min_frames_first = 7  # Just one chunk (7 tokens) for first audio - ultra-low latency
-    min_frames_subsequent = 28  # Standard minimum (4 chunks of 7 tokens) after first audio
-    ideal_frames = 49  # Ideal standard frame size (7×7 window) - unchanged
-    process_every_n = 7  # Process every 7 tokens (standard for Orpheus model) - unchanged
     
     start_time = time.time()
     token_count = 0
@@ -206,8 +160,15 @@ async def tokens_decoder(token_gen):
     async for token_sim in token_gen:
         token_count += 1
         
-        # Use the unified turn_token_into_id which already handles caching
+        # Check cache first to avoid redundant computation
-        token = turn_token_into_id(token_sim, count)
+        cache_key = (token_sim, count % 7)
+        if cache_key in token_cache:
+            token = token_cache[cache_key]
+        else:
+            token = turn_token_into_id(token_sim, count)
+            # Add to cache if valid token
+            if token is not None and len(token_cache) < MAX_CACHE_SIZE:
+                token_cache[cache_key] = token
         
         if token is not None and token > 0:
             buffer.append(token)
@@ -224,37 +185,26 @@ async def tokens_decoder(token_gen):
                 last_log_time = current_time
                 token_count = 0
             
-            # Different processing logic based on whether first chunk has been processed
+            # Process standard batches when we have enough tokens
-            if not first_chunk_processed:
+            if count % process_every_n == 0:
-                # Process first chunk as soon as possible for minimal latency
+                # Best case: we have enough for the ideal frame size
-                if count >= min_frames_first:
+                if len(buffer) >= ideal_frames:
-                    buffer_to_proc = buffer[-min_frames_first:]
+                    buffer_to_proc = buffer[-ideal_frames:]
-                    
+                # Fallback: we have enough for the minimum requirement
-                    # Process the first chunk of audio for immediate feedback
+                elif len(buffer) >= min_frames_required:
-                    print(f"Processing first audio chunk with {len(buffer_to_proc)} tokens for low latency")
+                    buffer_to_proc = buffer[-min_frames_required:]
-                    audio_samples = convert_to_audio(buffer_to_proc, count)
+                # For the first few frames, we may not have enough yet
-                    if audio_samples is not None:
+                else:
-                        first_chunk_processed = True  # Mark first chunk as processed
+                    continue
-                        yield audio_samples
+                
-            else:
+                # Debug output to help diagnose issues
-                # For subsequent chunks, use original processing with proper batching
+                if count % 28 == 0:
-                if count % process_every_n == 0:
+                    print(f"Processing buffer with {len(buffer_to_proc)} tokens, total collected: {len(buffer)}")
-                    # Use same prioritization logic as before
+                
-                    if len(buffer) >= ideal_frames:
+                # Process the tokens
-                        buffer_to_proc = buffer[-ideal_frames:]
+                audio_samples = convert_to_audio(buffer_to_proc, count)
-                    elif len(buffer) >= min_frames_subsequent:
+                if audio_samples is not None:
-                        buffer_to_proc = buffer[-min_frames_subsequent:]
+                    yield audio_samples
-                    else:
-                        continue
-                    
-                    # Debug output to help diagnose issues
-                    if count % 28 == 0:
-                        print(f"Processing buffer with {len(buffer_to_proc)} tokens, total collected: {len(buffer)}")
-                    
-                    # Process the tokens
-                    audio_samples = convert_to_audio(buffer_to_proc, count)
-                    if audio_samples is not None:
-                        yield audio_samples
     
     # CRITICAL: End-of-generation handling - process all remaining frames
     # Process remaining complete frames (ideal size)
@@ -265,8 +215,8 @@ async def tokens_decoder(token_gen):
             yield audio_samples
             
     # Process any additional complete frames (minimum size)
-    elif len(buffer) >= min_frames_subsequent:
+    elif len(buffer) >= min_frames_required:
-        buffer_to_proc = buffer[-min_frames_subsequent:]
+        buffer_to_proc = buffer[-min_frames_required:]
         audio_samples = convert_to_audio(buffer_to_proc, count)
         if audio_samples is not None:
             yield audio_samples
@@ -277,7 +227,7 @@ async def tokens_decoder(token_gen):
         # Pad to minimum frame requirement with copies of the final token
         # This is more continuous than using unrelated tokens from the beginning
         last_token = buffer[-1]
-        padding_needed = min_frames_subsequent - len(buffer)
+        padding_needed = min_frames_required - len(buffer)
         
         # Create a padding array of copies of the last token
         # This maintains continuity much better than circular buffering