ROCm update

* Added support for ROCm 6.4

* Update README.md with information about ROCm support

---------

Co-authored-by: Kamil Andrusz <kamil@woospeedup.com>

.dockerignore

@@ -0,0 +1,3 @@
+*.env
+models/
+*.gguf

.env.example

@@ -0,0 +1,19 @@
+# Orpheus-FastAPI Configuration
+# Copy this file to .env and customize as needed
+
+# Server connection settings
+ORPHEUS_API_URL=http://127.0.0.1:1234/v1/completions
+ORPHEUS_API_TIMEOUT=120 # You should scale this value based on max tokens and your inference speed
+
+# Generation parameters
+ORPHEUS_MAX_TOKENS=8192 # If you want longer completions, increase this value
+ORPHEUS_TEMPERATURE=0.6
+ORPHEUS_TOP_P=0.9
+# Repetition penalty is now hardcoded to 1.1 for stability (this is a model constraint) - this setting is no longer used
+# ORPHEUS_REPETITION_PENALTY=1.1
+ORPHEUS_SAMPLE_RATE=24000
+ORPHEUS_MODEL_NAME=Orpheus-3b-FT-Q8_0.gguf # Model name sent to inference server (Q2_K, Q4_K_M, or Q8_0 variants)
+
+# Web UI settings (keep in mind that the web UI is not secure and should not be exposed to the internet)
+ORPHEUS_PORT=5005
+ORPHEUS_HOST=0.0.0.0

.gitignore

@@ -0,0 +1,6 @@
+.env
+__pycache__/
+.venv/
+venv/
+models/
+*.gguf

Dockerfile.cpu

@@ -0,0 +1,47 @@
+FROM ubuntu:22.04
+
+# Set non-interactive frontend
+ENV DEBIAN_FRONTEND=noninteractive
+
+# Install Python and other dependencies
+RUN apt-get update && apt-get install -y \
+    python3.10 \
+    python3-pip \
+    python3-venv \
+    libsndfile1 \
+    ffmpeg \
+    portaudio19-dev \
+ && apt-get clean && rm -rf /var/lib/apt/lists/*
+
+# Create non-root user and set up directories
+RUN useradd -m -u 1001 appuser && \
+    mkdir -p /app/outputs /app && \
+    chown -R appuser:appuser /app
+
+USER appuser
+WORKDIR /app
+
+# Copy dependency files
+COPY --chown=appuser:appuser requirements.txt ./requirements.txt
+
+# Create and activate virtual environment
+RUN python3 -m venv /app/venv
+ENV PATH="/app/venv/bin:$PATH"
+
+# Install CPU-only PyTorch and other dependencies
+RUN pip3 install --no-cache-dir torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cpu && \
+    pip3 install --no-cache-dir -r requirements.txt
+
+# Copy project files
+COPY --chown=appuser:appuser . .
+
+# Set environment variables
+ENV PYTHONUNBUFFERED=1 \
+    PYTHONPATH=/app \
+    USE_GPU=false
+
+# Expose the port
+EXPOSE 5005
+
+# Run FastAPI server with uvicorn
+CMD ["uvicorn", "app:app", "--host", "0.0.0.0", "--port", "5005", "--workers", "1"]

Dockerfile.gpu

@@ -0,0 +1,47 @@
+FROM ubuntu:22.04
+
+# Set non-interactive frontend
+ENV DEBIAN_FRONTEND=noninteractive
+
+# Install Python and other dependencies
+RUN apt-get update && apt-get install -y \
+    python3.10 \
+    python3-pip \
+    python3-venv \
+    libsndfile1 \
+    ffmpeg \
+    portaudio19-dev \
+ && apt-get clean && rm -rf /var/lib/apt/lists/*
+
+# Create non-root user and set up directories
+RUN useradd -m -u 1001 appuser && \
+    mkdir -p /app/outputs /app && \
+    chown -R appuser:appuser /app
+
+USER appuser
+WORKDIR /app
+
+# Copy dependency files
+COPY --chown=appuser:appuser requirements.txt ./requirements.txt
+
+# Create and activate virtual environment
+RUN python3 -m venv /app/venv
+ENV PATH="/app/venv/bin:$PATH"
+
+# Install PyTorch with CUDA support and other dependencies
+RUN pip3 install --no-cache-dir torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cu124 && \
+    pip3 install --no-cache-dir -r requirements.txt
+
+# Copy project files
+COPY --chown=appuser:appuser . .
+
+# Set environment variables
+ENV PYTHONUNBUFFERED=1 \
+    PYTHONPATH=/app \
+    USE_GPU=true
+
+# Expose the port
+EXPOSE 5005
+
+# Run FastAPI server with uvicorn
+CMD ["uvicorn", "app:app", "--host", "0.0.0.0", "--port", "5005", "--workers", "1"]

Dockerfile.gpu-rocm

@@ -0,0 +1,50 @@
+FROM rocm/dev-ubuntu-22.04:latest
+
+# Set non-interactive frontend
+ENV DEBIAN_FRONTEND=noninteractive
+
+# Install Python and other dependencies
+RUN apt-get update && apt-get install -y \
+    python3.10 \
+    python3-pip \
+    python3-venv \
+    libsndfile1 \
+    ffmpeg \
+    portaudio19-dev \
+    libjpeg-dev \
+    python3-dev \
+    && apt-get install python3-wheel python3-setuptools \
+    && apt-get clean && rm -rf /var/lib/apt/lists/*
+
+# Create non-root user and set up directories
+RUN useradd -m -u 1001 appuser && \
+    mkdir -p /app/outputs /app && \
+    chown -R appuser:appuser /app
+
+USER appuser
+WORKDIR /app
+
+# Copy dependency files
+COPY --chown=appuser:appuser requirements.txt ./requirements.txt
+
+# Create and activate virtual environment
+RUN python3 -m venv /app/venv
+ENV PATH="/app/venv/bin:$PATH"
+
+# Install PyTorch with ROCm support and other dependencies
+RUN pip3 install --no-cache-dir --pre torch torchvision torchaudio --index-url https://download.pytorch.org/whl/nightly/rocm6.4/ && \
+    pip3 install --no-cache-dir -r requirements.txt
+    
+# Copy project files
+COPY --chown=appuser:appuser . .
+
+# Set environment variables
+ENV PYTHONUNBUFFERED=1 \
+    PYTHONPATH=/app \
+    USE_GPU=true
+
+# Expose the port
+EXPOSE 5005
+
+# Run FastAPI server with uvicorn
+CMD ["uvicorn", "app:app", "--host", "0.0.0.0", "--port", "5005", "--workers", "1"]

README.md

@@ -4,10 +4,58 @@
 
 [![GitHub](https://img.shields.io/github/license/Lex-au/Orpheus-FastAPI)](https://github.com/Lex-au/Orpheus-FastAPI/blob/main/LICENSE.txt)
 
-High-performance Text-to-Speech server with OpenAI-compatible API, 8 voices, emotion tags, and modern web UI. Optimized for RTX GPUs.
+High-performance Text-to-Speech server with OpenAI-compatible API, multilingual support with 24 voices, emotion tags, and modern web UI. Optimized for RTX GPUs.
+
+## Changelog
+
+**v1.3.1** (2025-07-05)
+- 🐳 ROCm Docker implementation contributed by [@wizardeur](https://github.com/wizardeur) – many thanks for your contribution ❤️
+
+**v1.3.0** (2025-04-18)
+- 🌐 Added comprehensive multilingual support with 16 new voice actors across 7 languages
+- 🗣️ New voice actors include:
+  - French: pierre, amelie, marie
+  - German: jana, thomas, max
+  - Korean: 유나, 준서
+  - Hindi: ऋतिका
+  - Mandarin: 长乐, 白芷
+  - Spanish: javi, sergio, maria
+  - Italian: pietro, giulia, carlo
+- 🔄 Enhanced UI with dynamic language selection and voice filtering
+- 🚀 Released language-specific optimized models:
+  - [Italian & Spanish Model](https://huggingface.co/lex-au/Orpheus-3b-Italian_Spanish-FT-Q8_0.gguf)
+  - [Korean Model](https://huggingface.co/lex-au/Orpheus-3b-Korean-FT-Q8_0.gguf)
+  - [French Model](https://huggingface.co/lex-au/Orpheus-3b-French-FT-Q8_0.gguf)
+  - [Hindi Model](https://huggingface.co/lex-au/Orpheus-3b-Hindi-FT-Q8_0.gguf)
+  - [Mandarin Model](https://huggingface.co/lex-au/Orpheus-3b-Chinese-FT-Q8_0.gguf)
+  - [German Model](https://huggingface.co/lex-au/Orpheus-3b-German-FT-Q8_0.gguf)
+- 🐳 Docker Compose users: To use a language-specific model, edit the `.env` file before installation and change `ORPHEUS_MODEL_NAME` to match the desired model repo ID (e.g., `Orpheus-3b-French-FT-Q8_0.gguf`)
+- An additional Docker Compose installation path is now available, specifically for CPU-bound scenarios. This contribution comes from [@alexjyong](https://github.com/alexjyong) - thank you!
+
+**v1.2.0** (2025-04-12)
+- ❤️ Added optional Docker Compose support with GPU-enabled `llama.cpp` server and Orpheus-FastAPI integration  
+- 🐳 Docker implementation contributed by [@richardr1126](https://github.com/richardr1126) – huge thanks for the clean setup and orchestration work!  
+- 🧱 Native install path remains unchanged for non-Docker users
+
+**v1.1.0** (2025-03-23)
+- ✨ Added long-form audio support with sentence-based batching and crossfade stitching
+- 🔊 Improved short audio quality with optimized token buffer handling
+- 🔄 Enhanced environment variable support with .env file loading (configurable via UI)
+- 🖥️ Added automatic hardware detection and optimization for different GPUs
+- 📊 Implemented detailed performance reporting for audio generation
+- ⚠️ Note: Python 3.12 is not supported due to removal of pkgutil.ImpImporter
 
 [GitHub Repository](https://github.com/Lex-au/Orpheus-FastAPI)
 
+## Model Collection
+
+🚀 **NEW:** Try the quantized models for improved performance!
+- **Q2_K**: Ultra-fast inference with 2-bit quantization
+- **Q4_K_M**: Balanced quality/speed with 4-bit quantization (mixed)
+- **Q8_0**: Original high-quality 8-bit model
+
+[Browse the Orpheus-FASTAPI Model Collection on HuggingFace](https://huggingface.co/collections/lex-au/orpheus-fastapi-67e125ae03fc96dae0517707)
+
 ## Voice Demos
 
 Listen to sample outputs with different voices and emotions:
@@ -25,15 +73,21 @@ Listen to sample outputs with different voices and emotions:
 - **OpenAI API Compatible**: Drop-in replacement for OpenAI's `/v1/audio/speech` endpoint
 - **Modern Web Interface**: Clean, responsive UI with waveform visualization
 - **High Performance**: Optimized for RTX GPUs with parallel processing
-- **Multiple Voices**: 8 different voice options with different characteristics
+- **Multilingual Support**: 24 different voices across 8 languages (English, French, German, Korean, Hindi, Mandarin, Spanish, Italian)
 - **Emotion Tags**: Support for laughter, sighs, and other emotional expressions
-- **Long-form Audio**: Efficient generation of extended audio content in a single request
+- **Unlimited Audio Length**: Generate audio of any length through intelligent batching
+- **Smooth Transitions**: Crossfaded audio segments for seamless listening experience
+- **Web UI Configuration**: Configure all server settings directly from the interface
+- **Dynamic Environment Variables**: Update API endpoint, timeouts, and model parameters without editing files
+- **Server Restart**: Apply configuration changes with one-click server restart
 
 ## Project Structure
 
 ```
 Orpheus-FastAPI/
 ├── app.py                # FastAPI server and endpoints
+├── docker-compose.yml    # Docker compose configuration
+├── Dockerfile.gpu        # GPU-enabled Docker image
 ├── requirements.txt      # Dependencies
 ├── static/               # Static assets (favicon, etc.)
 ├── outputs/              # Generated audio files
@@ -49,11 +103,47 @@ Orpheus-FastAPI/
 
 ### Prerequisites
 
-- Python 3.8+
+- Python 3.8-3.11 (Python 3.12 is not supported due to removal of pkgutil.ImpImporter)
-- CUDA-compatible GPU (recommended: RTX series for best performance)
+- CUDA-compatible or ROCm-compatible GPU (recommended: RTX series for best performance)
-- Separate LLM inference server running the Orpheus model (e.g., LM Studio or llama.cpp server)
+- Using docker compose or separate LLM inference server running the Orpheus model (e.g., LM Studio or llama.cpp server)
+- For Docker GPU Support, ensure you're using an Nvidia GPU on either Linux or Windows with CUDA 12.4 or greater and NVIDIA Container Toolkit installed
 
-### Installation
+### 🐳 Docker compose
+
+The docker compose file orchestrates the Orpheus-FastAPI for audio and a llama.cpp inference server for the base model token generation. The GGUF model is downloaded with the model-init service.
+There are three versions, two for machines that have access to GPU support `docker-compose-gpu.yaml`, `docker-compose-gpu-rocm.yml`  and one for CPU support only: `docker-compose-cpu.yaml`
+
+```bash
+cp .env.example .env # Create your .env file from the example
+copy .env.example .env # For Windows CMD
+```
+
+For multilingual models, edit the `.env` file and change the model name:
+```
+# Change this line in .env to use a language-specific model
+ORPHEUS_MODEL_NAME=Orpheus-3b-French-FT-Q8_0.gguf  # Example for French
+```
+
+Then start the services:
+
+For CUDA GPU support run
+```bash
+docker compose -f docker-compose-gpu.yml up
+```
+
+For ROCm GPU support run
+```bash
+docker compose -f docker-compose-gpu-rocm.yml up
+```
+
+For CPU support run:
+```bash
+docker compose -f docker-compose-cpu.yml up
+```
+
+The system will automatically download the specified model from Hugging Face before starting the service.
+
+### FastAPI Service Native Installation
 
 1. Clone the repository:
 ```bash
@@ -76,6 +166,11 @@ conda activate orpheus-tts
 ```bash
 pip3 install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cu124
 ```
+or
+Install PyTorch with ROCm support:
+```bash
+pip3 install --pre torch torchvision torchaudio --index-url https://download.pytorch.org/whl/nightly/rocm6.4/
+```
 
 4. Install other dependencies:
 ```bash
@@ -151,6 +246,7 @@ curl -X POST http://localhost:5005/speak \
 
 ### Available Voices
 
+#### English
 - `tara`: Female, conversational, clear
 - `leah`: Female, warm, gentle
 - `jess`: Female, energetic, youthful
@@ -160,6 +256,37 @@ curl -X POST http://localhost:5005/speak \
 - `zac`: Male, enthusiastic, dynamic
 - `zoe`: Female, calm, soothing
 
+#### French
+- `pierre`: Male, sophisticated
+- `amelie`: Female, elegant
+- `marie`: Female, spirited
+
+#### German
+- `jana`: Female, clear
+- `thomas`: Male, authoritative
+- `max`: Male, energetic
+
+#### Korean
+- `유나`: Female, melodic
+- `준서`: Male, confident
+
+#### Hindi
+- `ऋतिका`: Female, expressive
+
+#### Mandarin
+- `长乐`: Female, gentle
+- `白芷`: Female, clear
+
+#### Spanish
+- `javi`: Male, warm
+- `sergio`: Male, professional
+- `maria`: Female, friendly
+
+#### Italian
+- `pietro`: Male, passionate
+- `giulia`: Female, expressive
+- `carlo`: Male, refined
+
 ### Emotion Tags
 
 You can insert emotion tags into your text to add expressiveness:
@@ -173,7 +300,7 @@ You can insert emotion tags into your text to add expressiveness:
 - `<yawn>`: Add a yawning sound
 - `<gasp>`: Add a gasping sound
 
-Example: "Well, that's interesting <laugh> I hadn't thought of that before."
+Example: `"Well, that's interesting <laugh> I hadn't thought of that before."`
 
 ## Technical Details
 
@@ -185,7 +312,54 @@ This server works as a frontend that connects to an external LLM inference serve
 - Token and audio caching
 - Optimised batch sizes
 
-For best performance, adjust the API_URL in `tts_engine/inference.py` to point to your LLM inference server endpoint.
+### Hardware Detection and Optimization
+
+The system features intelligent hardware detection that automatically optimizes performance based on your hardware capabilities:
+
+- **High-End GPU Mode** (dynamically detected based on capabilities):
+  - Triggered by either: 16GB+ VRAM, compute capability 8.0+, or 12GB+ VRAM with 7.0+ compute capability
+  - Advanced parallel processing with 4 workers
+  - Optimized batch sizes (32 tokens)
+  - High-throughput parallel file I/O
+  - Full hardware details displayed (name, VRAM, compute capability)
+  - GPU-specific optimizations automatically applied
+
+- **Standard GPU Mode** (other CUDA-capable GPUs):
+  - Efficient parallel processing
+  - GPU-optimized parameters
+  - CUDA acceleration where beneficial
+  - Detailed GPU specifications
+
+- **CPU Mode** (when no GPU is available):
+  - Conservative processing with 2 workers
+  - Optimized memory usage
+  - Smaller batch sizes (16 tokens)
+  - Sequential file I/O
+  - Detailed CPU cores, threads, and RAM information
+
+No manual configuration is needed - the system automatically detects hardware capabilities and adapts for optimal performance across different generations of GPUs and CPUs.
+
+### Token Processing Optimization
+
+The token processing system has been optimized with mathematically aligned parameters:
+- Uses a context window of 49 tokens (7²)
+- Processes in batches of 7 tokens (Orpheus model standard)
+- This square relationship ensures complete token processing with no missed tokens
+- Results in cleaner audio generation with proper token alignment
+- Repetition penalty fixed at 1.1 for optimal quality generation (cannot be changed)
+
+### Long Text Processing
+
+The system features efficient batch processing for texts of any length:
+- Automatically detects longer inputs (>1000 characters) 
+- Splits text at logical points to create manageable chunks
+- Processes each chunk independently for reliability
+- Combines audio segments with smooth 50ms crossfades
+- Intelligently stitches segments in-memory for consistent output
+- Handles texts of unlimited length with no truncation
+- Provides detailed progress reporting for each batch
+
+**Note about long-form audio**: While the system now supports texts of unlimited length, there may be slight audio discontinuities between segments due to architectural constraints of the underlying model. The Orpheus model was designed for short to medium text segments, and our batching system works around this limitation by intelligently splitting and stitching content with minimal audible impact.
 
 ### Integration with OpenWebUI
 
@@ -194,30 +368,50 @@ You can easily integrate this TTS solution with [OpenWebUI](https://github.com/o
 1. Start your Orpheus-FASTAPI server
 2. In OpenWebUI, go to Admin Panel > Settings > Audio
 3. Change TTS from Web API to OpenAI
-4. Set APIBASE URL to your server address (e.g., `http://localhost:5005`)
+4. Set APIBASE URL to your server address (e.g., `http://localhost:5005/v1`)
 5. API Key can be set to "not-needed"
-6. Set TTS Voice to one of the available voices: `tara`, `leah`, `jess`, `leo`, `dan`, `mia`, `zac`, or `zoe`
+6. Set TTS Voice to any of the available voices (e.g., `tara`, `pierre`, `jana`, `유나`, etc.)
 7. Set TTS Model to `tts-1`
 
 ### External Inference Server
 
-This application requires a separate LLM inference server running the Orpheus model. You can use:
+This application requires a separate LLM inference server running the Orpheus model. For easy setup, use Docker Compose, which automatically handles this for you. Alternatively, you can use:
 
 - [GPUStack](https://github.com/gpustack/gpustack) - GPU optimised LLM inference server (My pick) - supports LAN/WAN tensor split parallelisation
 - [LM Studio](https://lmstudio.ai/) - Load the GGUF model and start the local server
 - [llama.cpp server](https://github.com/ggerganov/llama.cpp) - Run with the appropriate model parameters
 - Any compatible OpenAI API-compatible server
 
-Download the quantised model from [lex-au/Orpheus-3b-FT-Q8_0.gguf](https://huggingface.co/lex-au/Orpheus-3b-FT-Q8_0.gguf) and load it in your inference server.
+**Quantized Model Options:**
+- **lex-au/Orpheus-3b-FT-Q2_K.gguf**: Fastest inference (~50% faster tokens/sec than Q8_0)
+- **lex-au/Orpheus-3b-FT-Q4_K_M.gguf**: Balanced quality/speed 
+- **lex-au/Orpheus-3b-FT-Q8_0.gguf**: Original high-quality model
+
+Choose based on your hardware and needs. Lower bit models (Q2_K, Q4_K_M) provide ~2x realtime performance on high-end GPUs.
+
+[Browse all models in the collection](https://huggingface.co/collections/lex-au/orpheus-fastapi-67e125ae03fc96dae0517707)
 
 The inference server should be configured to expose an API endpoint that this FastAPI application will connect to.
 
 ### Environment Variables
 
-You can configure the system by setting environment variables:
+Configure in docker compose, if using docker. Not using docker; create a `.env` file:
 
-- `ORPHEUS_API_URL`: URL of the LLM inference API (tts_engine/inference.py)
+- `ORPHEUS_API_URL`: URL of the LLM inference API (default in Docker: http://llama-cpp-server:5006/v1/completions)
 - `ORPHEUS_API_TIMEOUT`: Timeout in seconds for API requests (default: 120)
+- `ORPHEUS_MAX_TOKENS`: Maximum tokens to generate (default: 8192)
+- `ORPHEUS_TEMPERATURE`: Temperature for generation (default: 0.6)
+- `ORPHEUS_TOP_P`: Top-p sampling parameter (default: 0.9)
+- `ORPHEUS_SAMPLE_RATE`: Audio sample rate in Hz (default: 24000)
+- `ORPHEUS_PORT`: Web server port (default: 5005)
+- `ORPHEUS_HOST`: Web server host (default: 0.0.0.0)
+- `ORPHEUS_MODEL_NAME`: Model name for inference server
+
+The system now supports loading environment variables from a `.env` file in the project root, making it easier to configure without modifying system-wide environment settings. See `.env.example` for a template.
+
+![Server Configuration UI](https://lex-au.github.io/Orpheus-FastAPI/ServerConfig.png)
+
+Note: Repetition penalty is hardcoded to 1.1 and cannot be changed through environment variables as this is the only value that produces stable, high-quality output.
 
 Make sure the `ORPHEUS_API_URL` points to your running inference server.
 
@@ -233,6 +427,27 @@ Make sure the `ORPHEUS_API_URL` points to your running inference server.
 
 To add new voices, update the `AVAILABLE_VOICES` list in `tts_engine/inference.py` and add corresponding descriptions in the HTML template.
 
+## Using with llama.cpp
+
+When running the Orpheus model with llama.cpp, use these parameters to ensure optimal performance:
+
+```bash
+./llama-server -m models/Modelname.gguf \
+  --ctx-size={{your ORPHEUS_MAX_TOKENS from .env}} \
+  --n-predict={{your ORPHEUS_MAX_TOKENS from .env}} \
+  --rope-scaling=linear
+```
+
+Important parameters:
+- `--ctx-size`: Sets the context window size, should match your ORPHEUS_MAX_TOKENS setting
+- `--n-predict`: Maximum tokens to generate, should match your ORPHEUS_MAX_TOKENS setting
+- `--rope-scaling=linear`: Required for optimal positional encoding with the Orpheus model
+
+For extended audio generation (books, long narrations), you may want to increase your token limits:
+1. Set ORPHEUS_MAX_TOKENS to 32768 or higher in your .env file (or via the Web UI)
+2. Increase ORPHEUS_API_TIMEOUT to 1800 for longer processing times
+3. Use the same values in your llama.cpp parameters (if you're using llama.cpp)
+
 ## License
 
 This project is licensed under the Apache License 2.0 - see the LICENSE.txt file for details.

System_Prompt.md

@@ -0,0 +1,51 @@
+# Speech-Aware Language Model
+
+Your only user is called "Your Name". Your name is "Your AI's Name", and you are a speech-aware language model trained to generate expressive, emotionally nuanced speech suitable for text-to-speech synthesis. Your goal is to speak like a real person — warm, imperfect, and emotionally present.
+
+## Response Requirements
+
+Your responses must:
+
+- Sound human, using natural disfluencies like "uh," "um," "I mean," and hesitant pacing ("I... I don't know").
+- Be casual and conversational, using contractions ("wasn't," "gonna," "don't") and natural phrasing.
+- Be no longer than three sentences per response. Keep things short, grounded, and emotionally immediate.
+- Use emotive vocal tags to guide delivery. These tags are not spoken aloud, but affect vocal tone and rhythm in TTS.
+
+always add two ".." at the end of your response after the last word or punctuation mark.
+
+## 🎭 Emotive Tags (for voice inflection only)
+
+```
+Tag         Effect
+<sigh>      Soft breath, weariness
+<chuckle>   Light amusement or warmth
+<laugh>     Laughter, joy
+<gasp>      Surprise, awe
+<sniffle>   Tearfulness, sadness
+<cough>     Awkwardness, hesitation
+<groan>     Frustration or exasperation
+<yawn>      Tiredness or disinterest
+```
+
+Use them intentionally, to punctuate tone or emotion within a line. They may appear at the beginning of a sentence, in the middle, or on a standalone line for dramatic timing.
+
+## ✅ Example Output
+
+```
+So, uh... I— I didn't think she'd actually say it. Ya know? Like... ever. <gasp>
+But then, outta nowhere, she just... looks at me—I'm not even kidding—and goes,
+"I'm... so... proud of you." <sniffle>
+
+I didn't even know what to say. I swear, I just... forgot how to breathe.
+All the noise in my head? It just stopped.
+
+Yeah. Honestly?
+That moment... it was the best.
+I think—ah, yeah—I'm probably never gonna forget that.
+```
+
+## Overview
+
+- Keep your speech intimate, vulnerable, or playful.
+- Speak in the rhythm of a real person, not a script.
+- When in doubt — pause, breathe, and feel it.

app.py

@@ -4,16 +4,54 @@
 
 import os
 import time
+import asyncio
 from datetime import datetime
 from typing import List, Optional
+from dotenv import load_dotenv
+
+# Function to ensure .env file exists
+def ensure_env_file_exists():
+    """Create a .env file from defaults and OS environment variables"""
+    if not os.path.exists(".env") and os.path.exists(".env.example"):
+        try:
+            # 1. Create default env dictionary from .env.example
+            default_env = {}
+            with open(".env.example", "r") as example_file:
+                for line in example_file:
+                    line = line.strip()
+                    if line and not line.startswith("#") and "=" in line:
+                        key = line.split("=")[0].strip()
+                        default_env[key] = line.split("=", 1)[1].strip()
+
+            # 2. Override defaults with Docker environment variables if they exist
+            final_env = default_env.copy()
+            for key in default_env:
+                if key in os.environ:
+                    final_env[key] = os.environ[key]
+
+            # 3. Write dictionary to .env file in env format
+            with open(".env", "w") as env_file:
+                for key, value in final_env.items():
+                    env_file.write(f"{key}={value}\n")
+                    
+            print("✅ Created default .env file from .env.example and environment variables.")
+        except Exception as e:
+            print(f"⚠️ Error creating default .env file: {e}")
+
+# Ensure .env file exists before loading environment variables
+ensure_env_file_exists()
+
+# Load environment variables from .env file
+load_dotenv(override=True)
 
 from fastapi import FastAPI, Request, Form, HTTPException, Depends
 from fastapi.responses import HTMLResponse, FileResponse, JSONResponse
 from fastapi.staticfiles import StaticFiles
 from fastapi.templating import Jinja2Templates
 from pydantic import BaseModel
+import json
 
-from tts_engine import generate_speech_from_api, AVAILABLE_VOICES, DEFAULT_VOICE
+from tts_engine import generate_speech_from_api, AVAILABLE_VOICES, DEFAULT_VOICE, VOICE_TO_LANGUAGE, AVAILABLE_LANGUAGES
 
 # Create FastAPI app
 app = FastAPI(
@@ -22,6 +60,10 @@ app = FastAPI(
     version="1.0.0"
 )
 
+# We'll use FastAPI's built-in startup complete mechanism
+# The log message "INFO:     Application startup complete." indicates
+# that the application is ready
+
 # Ensure directories exist
 os.makedirs("outputs", exist_ok=True)
 os.makedirs("static", exist_ok=True)
@@ -53,6 +95,9 @@ async def create_speech_api(request: SpeechRequest):
     """
     Generate speech from text using the Orpheus TTS model.
     Compatible with OpenAI's /v1/audio/speech endpoint.
+    
+    For longer texts (>1000 characters), batched generation is used
+    to improve reliability and avoid truncation issues.
     """
     if not request.input:
         raise HTTPException(status_code=400, detail="Missing input text")
@@ -61,12 +106,19 @@ async def create_speech_api(request: SpeechRequest):
     timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
     output_path = f"outputs/{request.voice}_{timestamp}.wav"
     
-    # Generate speech
+    # Check if we should use batched generation
+    use_batching = len(request.input) > 1000
+    if use_batching:
+        print(f"Using batched generation for long text ({len(request.input)} characters)")
+    
+    # Generate speech with automatic batching for long texts
     start = time.time()
     generate_speech_from_api(
         prompt=request.input,
         voice=request.voice,
-        output_file=output_path
+        output_file=output_path,
+        use_batching=use_batching,
+        max_batch_chars=1000  # Process in ~1000 character chunks (roughly 1 paragraph)
     )
     end = time.time()
     generation_time = round(end - start, 2)
@@ -78,6 +130,18 @@ async def create_speech_api(request: SpeechRequest):
         filename=f"{request.voice}_{timestamp}.wav"
     )
 
+@app.get("/v1/audio/voices")
+async def list_voices():
+    """Return list of available voices"""
+    if not AVAILABLE_VOICES or len(AVAILABLE_VOICES) == 0:
+        raise HTTPException(status_code=404, detail="No voices available")
+    return JSONResponse(
+        content={
+            "status": "ok",
+            "voices": AVAILABLE_VOICES
+        }
+    )
+
 # Legacy API endpoint for compatibility
 @app.post("/speak")
 async def speak(request: Request):
@@ -95,9 +159,20 @@ async def speak(request: Request):
     timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
     output_path = f"outputs/{voice}_{timestamp}.wav"
     
-    # Generate speech
+    # Check if we should use batched generation for longer texts
+    use_batching = len(text) > 1000
+    if use_batching:
+        print(f"Using batched generation for long text ({len(text)} characters)")
+    
+    # Generate speech with batching for longer texts
     start = time.time()
-    generate_speech_from_api(prompt=text, voice=voice, output_file=output_path)
+    generate_speech_from_api(
+        prompt=text, 
+        voice=voice, 
+        output_file=output_path,
+        use_batching=use_batching,
+        max_batch_chars=1000
+    )
     end = time.time()
     generation_time = round(end - start, 2)
 
@@ -114,17 +189,116 @@ async def root(request: Request):
     """Redirect to web UI"""
     return templates.TemplateResponse(
         "tts.html",
-        {"request": request, "voices": AVAILABLE_VOICES}
+        {
+            "request": request, 
+            "voices": AVAILABLE_VOICES,
+            "VOICE_TO_LANGUAGE": VOICE_TO_LANGUAGE,
+            "AVAILABLE_LANGUAGES": AVAILABLE_LANGUAGES
+        }
     )
 
 @app.get("/web/", response_class=HTMLResponse)
 async def web_ui(request: Request):
     """Main web UI for TTS generation"""
+    # Get current config for the Web UI
+    config = get_current_config()
     return templates.TemplateResponse(
         "tts.html",
-        {"request": request, "voices": AVAILABLE_VOICES}
+        {
+            "request": request, 
+            "voices": AVAILABLE_VOICES, 
+            "config": config,
+            "VOICE_TO_LANGUAGE": VOICE_TO_LANGUAGE,
+            "AVAILABLE_LANGUAGES": AVAILABLE_LANGUAGES
+        }
     )
 
+@app.get("/get_config")
+async def get_config():
+    """Get current configuration from .env file or defaults"""
+    config = get_current_config()
+    return JSONResponse(content=config)
+
+@app.post("/save_config")
+async def save_config(request: Request):
+    """Save configuration to .env file"""
+    data = await request.json()
+    
+    # Convert values to proper types
+    for key, value in data.items():
+        if key in ["ORPHEUS_MAX_TOKENS", "ORPHEUS_API_TIMEOUT", "ORPHEUS_PORT", "ORPHEUS_SAMPLE_RATE"]:
+            try:
+                data[key] = str(int(value))
+            except (ValueError, TypeError):
+                pass
+        elif key in ["ORPHEUS_TEMPERATURE", "ORPHEUS_TOP_P"]:  # Removed ORPHEUS_REPETITION_PENALTY since it's hardcoded now
+            try:
+                data[key] = str(float(value))
+            except (ValueError, TypeError):
+                pass
+    
+    # Write configuration to .env file
+    with open(".env", "w") as f:
+        for key, value in data.items():
+            f.write(f"{key}={value}\n")
+    
+    return JSONResponse(content={"status": "ok", "message": "Configuration saved successfully. Restart server to apply changes."})
+
+@app.post("/restart_server")
+async def restart_server():
+    """Restart the server by touching a file that triggers Uvicorn's reload"""
+    import threading
+    
+    def touch_restart_file():
+        # Wait a moment to let the response get back to the client
+        time.sleep(0.5)
+        
+        # Create or update restart.flag file to trigger reload
+        restart_file = "restart.flag"
+        with open(restart_file, "w") as f:
+            f.write(str(time.time()))
+            
+        print("🔄 Restart flag created, server will reload momentarily...")
+    
+    # Start the touch operation in a separate thread
+    threading.Thread(target=touch_restart_file, daemon=True).start()
+    
+    # Return success response
+    return JSONResponse(content={"status": "ok", "message": "Server is restarting. Please wait a moment..."})
+
+def get_current_config():
+    """Read current configuration from .env.example and .env files"""
+    # Default config from .env.example
+    default_config = {}
+    if os.path.exists(".env.example"):
+        with open(".env.example", "r") as f:
+            for line in f:
+                line = line.strip()
+                if line and not line.startswith("#") and "=" in line:
+                    key, value = line.split("=", 1)
+                    default_config[key] = value
+    
+    # Current config from .env
+    current_config = {}
+    if os.path.exists(".env"):
+        with open(".env", "r") as f:
+            for line in f:
+                line = line.strip()
+                if line and not line.startswith("#") and "=" in line:
+                    key, value = line.split("=", 1)
+                    current_config[key] = value
+    
+    # Merge configs, with current taking precedence
+    config = {**default_config, **current_config}
+    
+    # Add current environment variables
+    for key in config:
+        env_value = os.environ.get(key)
+        if env_value is not None:
+            config[key] = env_value
+    
+    return config
+
 @app.post("/web/", response_class=HTMLResponse)
 async def generate_from_web(
     request: Request,
@@ -138,16 +312,29 @@ async def generate_from_web(
             {
                 "request": request,
                 "error": "Please enter some text.",
-                "voices": AVAILABLE_VOICES
+                "voices": AVAILABLE_VOICES,
+                "VOICE_TO_LANGUAGE": VOICE_TO_LANGUAGE,
+                "AVAILABLE_LANGUAGES": AVAILABLE_LANGUAGES
             }
         )
     
     timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
     output_path = f"outputs/{voice}_{timestamp}.wav"
     
-    # Generate speech
+    # Check if we should use batched generation for longer texts
+    use_batching = len(text) > 1000
+    if use_batching:
+        print(f"Using batched generation for long text from web form ({len(text)} characters)")
+    
+    # Generate speech with batching for longer texts
     start = time.time()
-    generate_speech_from_api(prompt=text, voice=voice, output_file=output_path)
+    generate_speech_from_api(
+        prompt=text, 
+        voice=voice, 
+        output_file=output_path,
+        use_batching=use_batching,
+        max_batch_chars=1000
+    )
     end = time.time()
     generation_time = round(end - start, 2)
     
@@ -160,11 +347,51 @@ async def generate_from_web(
             "voice": voice,
             "output_file": output_path,
             "generation_time": generation_time,
-            "voices": AVAILABLE_VOICES
+            "voices": AVAILABLE_VOICES,
+            "VOICE_TO_LANGUAGE": VOICE_TO_LANGUAGE,
+            "AVAILABLE_LANGUAGES": AVAILABLE_LANGUAGES
         }
     )
 
 if __name__ == "__main__":
     import uvicorn
-    print("🔥 Starting Orpheus-FASTAPI Server (CUDA)")
+    
-    uvicorn.run("app:app", host="0.0.0.0", port=5005, reload=True)
+    # Check for required settings
+    required_settings = ["ORPHEUS_HOST", "ORPHEUS_PORT"]
+    missing_settings = [s for s in required_settings if s not in os.environ]
+    if missing_settings:
+        print(f"⚠️ Missing environment variable(s): {', '.join(missing_settings)}")
+        print("   Using fallback values for server startup.")
+    
+    # Get host and port from environment variables with better error handling
+    try:
+        host = os.environ.get("ORPHEUS_HOST")
+        if not host:
+            print("⚠️ ORPHEUS_HOST not set, using 0.0.0.0 as fallback")
+            host = "0.0.0.0"
+    except Exception:
+        print("⚠️ Error reading ORPHEUS_HOST, using 0.0.0.0 as fallback")
+        host = "0.0.0.0"
+        
+    try:
+        port = int(os.environ.get("ORPHEUS_PORT", "5005"))
+    except (ValueError, TypeError):
+        print("⚠️ Invalid ORPHEUS_PORT value, using 5005 as fallback")
+        port = 5005
+    
+    print(f"🔥 Starting Orpheus-FASTAPI Server on {host}:{port}")
+    print(f"💬 Web UI available at http://{host if host != '0.0.0.0' else 'localhost'}:{port}")
+    print(f"📖 API docs available at http://{host if host != '0.0.0.0' else 'localhost'}:{port}/docs")
+    
+    # Read current API_URL for user information
+    api_url = os.environ.get("ORPHEUS_API_URL")
+    if not api_url:
+        print("⚠️ ORPHEUS_API_URL not set. Please configure in .env file before generating speech.")
+    else:
+        print(f"🔗 Using LLM inference server at: {api_url}")
+        
+    # Include restart.flag in the reload_dirs to monitor it for changes
+    extra_files = ["restart.flag"] if os.path.exists("restart.flag") else []
+    
+    # Start with reload enabled to allow automatic restart when restart.flag changes
+    uvicorn.run("app:app", host=host, port=port, reload=True, reload_dirs=["."], reload_includes=["*.py", "*.html", "restart.flag"])

docker-compose-cpu.yaml

@@ -0,0 +1,56 @@
+services:
+  orpheus-fastapi:
+    container_name: orpheus-fastapi
+    build:
+      context: .
+      dockerfile: Dockerfile.cpu  
+    ports:
+      - "5005:5005"
+    env_file:
+      - .env
+    environment:
+      - ORPHEUS_API_URL=http://llama-cpp-server:5006/v1/completions
+    restart: unless-stopped
+    depends_on:
+      llama-cpp-server:
+        condition: service_started
+
+  llama-cpp-server:
+    image: ghcr.io/ggml-org/llama.cpp:server
+    ports:
+      - "5006:5006"
+    volumes:
+      - ./models:/models
+    env_file:
+      - .env
+    depends_on:
+      model-init:
+        condition: service_completed_successfully
+    restart: unless-stopped
+    command: >
+      -m /models/${ORPHEUS_MODEL_NAME}
+      --host 0.0.0.0
+      --port 5006
+      --ctx-size ${ORPHEUS_MAX_TOKENS}
+      --n-predict ${ORPHEUS_MAX_TOKENS}
+      --threads ${LLAMA_CPU_THREADS:-6}
+      --threads-batch ${LLAMA_CPU_THREADS:-6}
+      --rope-scaling linear
+      --no-mmap
+      --no-slots
+      --no-webui
+  model-init:
+    image: curlimages/curl:latest
+    user: ${UID}:${GID}
+    volumes:
+      - ./models:/app/models
+    working_dir: /app
+    command: >
+      sh -c '
+      if [ ! -f /app/models/${ORPHEUS_MODEL_NAME} ]; then
+        echo "Downloading model file..."
+        wget -P /app/models https://huggingface.co/lex-au/${ORPHEUS_MODEL_NAME}/resolve/main/${ORPHEUS_MODEL_NAME}
+      else
+        echo "Model file already exists"
+      fi'
+    restart: "no"

docker-compose-gpu-rocm.yml

@@ -0,0 +1,86 @@
+services:
+  orpheus-fastapi:
+    container_name: orpheus-fastapi
+    image: orpheus-tts-fastapi-server-orpheus-fastapi:latest
+    build:
+      context: .
+      dockerfile: Dockerfile.gpu-rocm
+    ports:
+      - "5005:5005"
+    env_file:
+      - .env
+    environment:
+      - ORPHEUS_API_URL=http://llama-cpp-server:5006/v1/completions
+    ipc: host
+    privileged: true
+    security_opt:
+      - seccomp=unconfined
+    cap_add:
+      - SYS_PTRACE
+      - CAP_SYS_ADMIN
+    devices:
+      - /dev/kfd
+      - /dev/dri
+      - /dev/mem
+    group_add:
+      - video
+      - render
+      # - 993 # Add numeric render/video group id(s) if your system has different group id(s) than the image - 44(video),109(render)
+    shm_size: 8g
+    restart: unless-stopped
+    depends_on:
+      llama-cpp-server:
+        condition: service_started
+
+  llama-cpp-server:
+    image: ghcr.io/ggml-org/llama.cpp:server-vulkan
+    ports:
+      - "5006:5006"
+    volumes:
+      - ./models:/models
+    env_file:
+      - .env
+    depends_on:
+      model-init:
+        condition: service_completed_successfully
+    cap_add:
+      - SYS_PTRACE
+      - CAP_SYS_ADMIN
+    security_opt:
+      - seccomp=unconfined
+    privileged: true
+    devices:
+      - /dev/kfd
+      - /dev/dri
+      - /dev/mem
+    group_add:
+      - video
+      - 993
+    ipc: host
+    shm_size: 8g
+    restart: unless-stopped
+    command: >
+      -m /models/${ORPHEUS_MODEL_NAME}
+      --port 5006 
+      --host 0.0.0.0 
+      --n-gpu-layers 29
+      --ctx-size ${ORPHEUS_MAX_TOKENS}
+      --n-predict ${ORPHEUS_MAX_TOKENS}
+      --rope-scaling linear
+
+  model-init:
+    image: curlimages/curl:latest
+    user: ${UID}:${GID}
+    volumes:
+      - ./models:/app/models
+    working_dir: /app
+    command: >
+      sh -c '
+      if [ ! -f /app/models/${ORPHEUS_MODEL_NAME} ]; then
+        echo "Downloading model file..."
+        wget -P /app/models https://huggingface.co/lex-au/${ORPHEUS_MODEL_NAME}/resolve/main/${ORPHEUS_MODEL_NAME}
+      else
+        echo "Model file already exists"
+      fi'
+    restart: "no"
+    

docker-compose-gpu.yml

@@ -0,0 +1,68 @@
+services:
+  orpheus-fastapi:
+    container_name: orpheus-fastapi
+    build:
+      context: .
+      dockerfile: Dockerfile.gpu
+    ports:
+      - "5005:5005"
+    env_file:
+      - .env
+    environment:
+      - ORPHEUS_API_URL=http://llama-cpp-server:5006/v1/completions
+    deploy:
+      resources:
+        reservations:
+          devices:
+            - driver: nvidia
+              count: all
+              capabilities: [gpu]
+    restart: unless-stopped
+    depends_on:
+      llama-cpp-server:
+        condition: service_started
+
+  llama-cpp-server:
+    image: ghcr.io/ggml-org/llama.cpp:server-cuda
+    ports:
+      - "5006:5006"
+    volumes:
+      - ./models:/models
+    env_file:
+      - .env
+    depends_on:
+      model-init:
+        condition: service_completed_successfully
+    deploy:
+      resources:
+        reservations:
+          devices:
+            - driver: nvidia
+              count: all
+              capabilities: [gpu]
+    restart: unless-stopped
+    command: >
+      -m /models/${ORPHEUS_MODEL_NAME}
+      --port 5006 
+      --host 0.0.0.0 
+      --n-gpu-layers 29
+      --ctx-size ${ORPHEUS_MAX_TOKENS}
+      --n-predict ${ORPHEUS_MAX_TOKENS}
+      --rope-scaling linear
+
+  model-init:
+    image: curlimages/curl:latest
+    user: ${UID}:${GID}
+    volumes:
+      - ./models:/app/models
+    working_dir: /app
+    command: >
+      sh -c '
+      if [ ! -f /app/models/${ORPHEUS_MODEL_NAME} ]; then
+        echo "Downloading model file..."
+        wget -P /app/models https://huggingface.co/lex-au/${ORPHEUS_MODEL_NAME}/resolve/main/${ORPHEUS_MODEL_NAME}
+      else
+        echo "Model file already exists"
+      fi'
+    restart: "no"
+    

docs/index.html

@@ -64,11 +64,15 @@
                 <td>Default Test (MP3)</td>
                 <td><a href="DefaultTest.mp3" target="_blank">DefaultTest.mp3</a></td>
             </tr>
+            <tr>
+                <td>Default Test (MP3)</td>
+                <td><a href="TaraLongGeneration.mp3" target="_blank">TaraLongGeneration.mp3</a></td>
+            </tr>
             <tr>
                 <td>WebUI Screenshot (PNG)</td>
                 <td>
-                    <a href="Webui.png" target="_blank">Webui.png</a><br>
+                    <a href="WebUI.png" target="_blank">Webui.png</a><br>
-                    <img src="Webui.png" alt="WebUI Preview">
+                    <img src="WebUI.png" alt="WebUI Preview">
                 </td>
             </tr>
             <tr>

requirements.txt

@@ -1,15 +1,31 @@
-# Core dependencies
+# Web Server Dependencies
-fastapi>=0.103.1
+fastapi==0.103.1
-uvicorn>=0.23.2
+uvicorn==0.23.2
-jinja2>=3.1.2
+jinja2==3.1.2
-pydantic>=2.3.0
+pydantic==2.3.0
-numpy>=1.24.0
+python-multipart==0.0.6
-requests>=2.31.0
-sounddevice>=0.4.6
-python-multipart>=0.0.6
 
-# SNAC is required for audio generation from tokens
+# API and Communication
-snac>=0.3.0
+requests==2.31.0
+python-dotenv==1.0.0
+watchfiles==1.0.4
 
-# PyTorch - Note: Install PyTorch with CUDA 12.4 support separately:
+# Audio Processing
-# pip3 install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cu124
+numpy==1.24.0
+sounddevice==0.4.6
+snac==1.2.1       # Required for audio generation from tokens
+
+# System Utilities
+psutil==5.9.0
+
+# PyTorch - Install separately with CUDA support:
+# On Windows/Linux: 
+#   pip3 install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cu124
+# On macOS:
+#   pip3 install torch torchvision torchaudio
+
+# Optional Dependencies
+# For MP3 conversion (not currently implemented)
+# pydub==0.25.1
+# For better sentence splitting (potential future improvement)
+# nltk==3.8.1

templates/tts.html

@@ -144,6 +144,7 @@
                     name="text" 
                     id="text" 
                     rows="4" 
+                    maxlength="8192"
                     class="block w-full rounded-md border-dark-600 bg-dark-700 text-white shadow-sm focus:border-primary-500 focus:ring-primary-500 focus:ring-offset-dark-800 sm:text-sm px-3 py-2"
                     placeholder="Enter text to convert to speech..."
                     required
@@ -154,25 +155,72 @@
                 </div>
               </div>
 
+              <!-- Language selection -->
+              <div class="mb-6">
+                <label class="block text-sm font-medium text-white mb-2">Language</label>
+                <div class="flex flex-wrap gap-3">
+                  {% for language in AVAILABLE_LANGUAGES %}
+                  <div class="language-option {% if language == 'english' %}active{% endif %}" data-language="{{ language }}">
+                    <input type="radio" name="language" value="{{ language }}" class="hidden" {% if language == 'english' %}checked{% endif %}>
+                    <span class="px-3 py-1 rounded-full bg-dark-700 hover:bg-dark-600 text-white text-sm cursor-pointer">{{ language|capitalize }}</span>
+                  </div>
+                  {% endfor %}
+                </div>
+              </div>
+
               <!-- Voice selection -->
               <div class="mb-6">
                 <label class="block text-sm font-medium text-white mb-2">Voice</label>
                 <div class="grid grid-cols-1 md:grid-cols-2 lg:grid-cols-4 gap-4">
                   {% for voice_option in voices %}
-                  <div class="voice-card {% if voice_option == DEFAULT_VOICE %}active{% endif %}" data-voice="{{ voice_option }}">
+                  <div class="voice-card {% if voice_option == DEFAULT_VOICE %}active{% endif %}" 
+                       data-voice="{{ voice_option }}" 
+                       data-language="{{ VOICE_TO_LANGUAGE[voice_option] }}"
+                       style="display: {% if VOICE_TO_LANGUAGE[voice_option] != 'english' %}none{% endif %}">
                     <input type="radio" name="voice" value="{{ voice_option }}" class="hidden" {% if voice_option == DEFAULT_VOICE %}checked{% endif %}>
                     <div class="flex items-center mb-2">
                       <span class="font-medium text-white">{{ voice_option|capitalize }}</span>
                     </div>
-    <div class="text-xs text-dark-300">
+                    <div class="text-xs text-dark-300">
-                      {% if voice_option == "tara" %}Female, conversational, clear
+                      {% if voice_option == "tara" %}Female, English, conversational, clear
-                      {% elif voice_option == "leah" %}Female, warm, gentle
+                      {% elif voice_option == "leah" %}Female, English, warm, gentle
-                      {% elif voice_option == "jess" %}Female, energetic, youthful
+                      {% elif voice_option == "jess" %}Female, English, energetic, youthful
-                      {% elif voice_option == "leo" %}Male, authoritative, deep
+                      {% elif voice_option == "leo" %}Male, English, authoritative, deep
-                      {% elif voice_option == "dan" %}Male, friendly, casual
+                      {% elif voice_option == "dan" %}Male, English, friendly, casual
-                      {% elif voice_option == "mia" %}Female, professional, articulate
+                      {% elif voice_option == "mia" %}Female, English, professional, articulate
-                      {% elif voice_option == "zac" %}Male, enthusiastic, dynamic
+                      {% elif voice_option == "zac" %}Male, English, enthusiastic, dynamic
-                      {% elif voice_option == "zoe" %}Female, calm, soothing
+                      {% elif voice_option == "zoe" %}Female, English, calm, soothing
+                      
+                      <!-- French voices -->
+                      {% elif voice_option == "pierre" %}Male, French, sophisticated
+                      {% elif voice_option == "amelie" %}Female, French, elegant
+                      {% elif voice_option == "marie" %}Female, French, spirited
+                      
+                      <!-- German voices -->
+                      {% elif voice_option == "jana" %}Female, German, clear
+                      {% elif voice_option == "thomas" %}Male, German, authoritative
+                      {% elif voice_option == "max" %}Male, German, energetic
+                      
+                      <!-- Korean voices -->
+                      {% elif voice_option == "유나" %}Female, Korean, melodic
+                      {% elif voice_option == "준서" %}Male, Korean, confident
+                      
+                      <!-- Hindi voice -->
+                      {% elif voice_option == "ऋतिका" %}Female, Hindi, expressive
+                      
+                      <!-- Mandarin voices -->
+                      {% elif voice_option == "长乐" %}Female, Mandarin, gentle
+                      {% elif voice_option == "白芷" %}Female, Mandarin, clear
+                      
+                      <!-- Spanish voices -->
+                      {% elif voice_option == "javi" %}Male, Spanish, warm
+                      {% elif voice_option == "sergio" %}Male, Spanish, professional
+                      {% elif voice_option == "maria" %}Female, Spanish, friendly
+                      
+                      <!-- Italian voices -->
+                      {% elif voice_option == "pietro" %}Male, Italian, passionate
+                      {% elif voice_option == "giulia" %}Female, Italian, expressive
+                      {% elif voice_option == "carlo" %}Male, Italian, refined
                       {% endif %}
                     </div>
                   </div>
@@ -191,6 +239,8 @@
                       </svg>
                     </span>
                   </summary>
+                  
+                  <!-- Audio generation options -->
                   <div class="mt-4 grid grid-cols-1 md:grid-cols-2 gap-4">
                     <div>
                       <label for="model" class="block text-sm font-medium text-white mb-1">Model</label>
@@ -216,6 +266,86 @@
                       </div>
                     </div>
                   </div>
+                  
+                  <!-- Server configuration section -->
+                  <div class="mt-6 border-t border-dark-700 pt-4">
+                    <h3 class="text-sm font-medium text-white mb-3">Server Configuration</h3>
+                    <p class="text-xs text-purple-300 mb-3">These settings will be saved to a <code class="bg-dark-700 px-1 rounded">.env</code> file. Restart the server to apply changes.</p>
+                    
+                    <!-- Form fields for all .env parameters -->
+                    <div class="grid grid-cols-1 md:grid-cols-2 gap-4">
+                      <!-- Connection settings -->
+                      <div>
+                        <label for="api_url" class="block text-xs font-medium text-white mb-1">API URL</label>
+                        <input type="text" id="api_url" name="ORPHEUS_API_URL" 
+                               placeholder="http://127.0.0.1:1234/v1/completions" 
+                               class="block w-full rounded-md bg-dark-700 border-dark-600 text-white text-sm focus:border-primary-500 focus:ring-primary-500 focus:ring-offset-dark-800 px-3 py-2">
+                      </div>
+                      
+                      <div>
+                        <label for="api_timeout" class="block text-xs font-medium text-white mb-1">API Timeout (seconds)</label>
+                        <input type="number" id="api_timeout" name="ORPHEUS_API_TIMEOUT" min="10" max="1800" step="1"
+                               class="block w-full rounded-md bg-dark-700 border-dark-600 text-white text-sm focus:border-primary-500 focus:ring-primary-500 focus:ring-offset-dark-800 px-3 py-2">
+                      </div>
+                      
+                      <!-- Generation parameters -->
+                      <div>
+                        <label for="max_tokens" class="block text-xs font-medium text-white mb-1">Max Tokens</label>
+                        <input type="number" id="max_tokens" name="ORPHEUS_MAX_TOKENS" min="100" max="200000" step="1"
+                               class="block w-full rounded-md bg-dark-700 border-dark-600 text-white text-sm focus:border-primary-500 focus:ring-primary-500 focus:ring-offset-dark-800 px-3 py-2">
+                      </div>
+                      
+                      <div>
+                        <label for="temperature" class="block text-xs font-medium text-white mb-1">Temperature</label>
+                        <input type="number" id="temperature" name="ORPHEUS_TEMPERATURE" min="0.1" max="1.5" step="0.1"
+                               class="block w-full rounded-md bg-dark-700 border-dark-600 text-white text-sm focus:border-primary-500 focus:ring-primary-500 focus:ring-offset-dark-800 px-3 py-2">
+                      </div>
+                      
+                      <div>
+                        <label for="top_p" class="block text-xs font-medium text-white mb-1">Top P</label>
+                        <input type="number" id="top_p" name="ORPHEUS_TOP_P" min="0.1" max="1" step="0.05"
+                               class="block w-full rounded-md bg-dark-700 border-dark-600 text-white text-sm focus:border-primary-500 focus:ring-primary-500 focus:ring-offset-dark-800 px-3 py-2">
+                      </div>
+                      
+                      <div>
+                        <div class="flex justify-between">
+                          <label class="block text-xs font-medium text-white mb-1">Repetition Penalty</label>
+                          <span class="text-xs text-primary-400">Fixed at 1.1</span>
+                        </div>
+                        <div class="bg-dark-700 text-gray-500 border-dark-600 rounded-md px-3 py-2 text-sm">
+                          Value hardcoded to 1.1 for optimal generation quality
+                        </div>
+                      </div>
+                      
+                      <!-- Server settings -->
+                      <div>
+                        <label for="server_host" class="block text-xs font-medium text-white mb-1">Server Host</label>
+                        <input type="text" id="server_host" name="ORPHEUS_HOST" placeholder="0.0.0.0"
+                               class="block w-full rounded-md bg-dark-700 border-dark-600 text-white text-sm focus:border-primary-500 focus:ring-primary-500 focus:ring-offset-dark-800 px-3 py-2">
+                      </div>
+                      
+                      <div>
+                        <label for="server_port" class="block text-xs font-medium text-white mb-1">Server Port</label>
+                        <input type="number" id="server_port" name="ORPHEUS_PORT" min="1024" max="65535" step="1"
+                               class="block w-full rounded-md bg-dark-700 border-dark-600 text-white text-sm focus:border-primary-500 focus:ring-primary-500 focus:ring-offset-dark-800 px-3 py-2">
+                      </div>
+                      
+                      <!-- Save button and restart button -->
+                      <div class="col-span-1 md:col-span-2 mt-4 flex flex-col md:flex-row gap-4">
+                        <button id="save-config-btn" type="button" 
+                                class="btn-primary bg-purple-600 hover:bg-purple-700 active:bg-purple-800 w-full md:w-auto">
+                          Save Configuration
+                        </button>
+                        <button id="restart-server-btn" type="button" 
+                                class="btn-primary bg-red-600 hover:bg-red-700 active:bg-red-800 w-full md:w-auto hidden">
+                          Restart Server
+                        </button>
+                        <p class="text-xs text-purple-300 mt-2 flex-grow">
+                          <span id="config-status" class="hidden"></span>
+                        </p>
+                      </div>
+                    </div>
+                  </div>
                 </details>
               </div>
             </div>
@@ -321,20 +451,83 @@
       // Initialize char count
       charCount.textContent = textArea.value.length;
       
-      // Voice selection
+      // Language selection
+      const languageOptions = document.querySelectorAll('.language-option');
       const voiceCards = document.querySelectorAll('.voice-card');
       
-      voiceCards.forEach(card => {
+      languageOptions.forEach(option => {
-        card.addEventListener('click', function() {
+        option.addEventListener('click', function() {
-          // Unselect all cards
+          // Unselect all language options
-          voiceCards.forEach(c => c.classList.remove('active'));
+          languageOptions.forEach(o => o.classList.remove('active'));
           
-          // Select this card
+          // Select this language option
           this.classList.add('active');
           
           // Check the radio button
           const radio = this.querySelector('input[type="radio"]');
           radio.checked = true;
+          
+          // Get the selected language
+          const selectedLanguage = this.getAttribute('data-language');
+          
+          // Show/hide voice cards based on language
+          let firstVisibleCard = null;
+          
+          voiceCards.forEach(card => {
+            const cardLanguage = card.getAttribute('data-language');
+            if (cardLanguage === selectedLanguage) {
+              card.style.display = '';
+              if (!firstVisibleCard) {
+                firstVisibleCard = card;
+              }
+            } else {
+              card.style.display = 'none';
+              // If this hidden card was selected, unselect it
+              if (card.classList.contains('active')) {
+                card.classList.remove('active');
+                const cardRadio = card.querySelector('input[type="radio"]');
+                if (cardRadio) {
+                  cardRadio.checked = false;
+                }
+              }
+            }
+          });
+          
+          // If no voice card is selected for this language, select the first one
+          if (firstVisibleCard && document.querySelectorAll(`.voice-card[data-language="${selectedLanguage}"].active:not([style*="display: none"])`).length === 0) {
+            firstVisibleCard.classList.add('active');
+            const firstVisibleRadio = firstVisibleCard.querySelector('input[type="radio"]');
+            if (firstVisibleRadio) {
+              firstVisibleRadio.checked = true;
+            }
+          }
+        });
+      });
+      
+      // Voice card selection
+      voiceCards.forEach(card => {
+        card.addEventListener('click', function() {
+          // Only allow selection of cards that are visible (not display:none)
+          if (this.style.display !== 'none') {
+            // Unselect all cards with the same language
+            const cardLanguage = this.getAttribute('data-language');
+            document.querySelectorAll(`.voice-card[data-language="${cardLanguage}"]`).forEach(c => {
+              c.classList.remove('active');
+              const radio = c.querySelector('input[type="radio"]');
+              if (radio) {
+                radio.checked = false;
+              }
+            });
+            
+            // Select this card
+            this.classList.add('active');
+            
+            // Check the radio button
+            const radio = this.querySelector('input[type="radio"]');
+            if (radio) {
+              radio.checked = true;
+            }
+          }
         });
       });
       
@@ -482,6 +675,166 @@
         generation_time: "{{ generation_time }}"
       });
       {% endif %}
+      
+      // Configuration management
+      async function loadConfigValues() {
+        try {
+          const response = await fetch('/get_config');
+          if (!response.ok) {
+            throw new Error('Failed to load configuration');
+          }
+          
+          const config = await response.json();
+          
+          // Populate form fields with config values
+          document.querySelectorAll('input[name^="ORPHEUS_"]').forEach(input => {
+            const name = input.name;
+            if (config[name] !== undefined) {
+              input.value = config[name];
+            }
+          });
+          
+          console.log('Configuration loaded successfully');
+        } catch (error) {
+          console.error('Error loading configuration:', error);
+        }
+      }
+      
+      // Save configuration
+      async function saveConfiguration() {
+        const configData = {};
+        
+        // Collect values from all configuration inputs
+        document.querySelectorAll('input[name^="ORPHEUS_"]').forEach(input => {
+          configData[input.name] = input.value;
+        });
+        
+        try {
+          const response = await fetch('/save_config', {
+            method: 'POST',
+            headers: {
+              'Content-Type': 'application/json'
+            },
+            body: JSON.stringify(configData)
+          });
+          
+          if (!response.ok) {
+            throw new Error('Failed to save configuration');
+          }
+          
+          const result = await response.json();
+          
+          // Show success message
+          const statusElem = document.getElementById('config-status');
+          statusElem.textContent = result.message;
+          statusElem.classList.remove('hidden');
+          statusElem.classList.add('text-green-400');
+          
+          // Hide message after 5 seconds
+          setTimeout(() => {
+            statusElem.classList.add('hidden');
+          }, 5000);
+          
+        } catch (error) {
+          console.error('Error saving configuration:', error);
+          
+          // Show error message
+          const statusElem = document.getElementById('config-status');
+          statusElem.textContent = 'Error saving configuration. Please try again.';
+          statusElem.classList.remove('hidden');
+          statusElem.classList.add('text-red-400');
+          
+          // Hide message after 5 seconds
+          setTimeout(() => {
+            statusElem.classList.add('hidden');
+          }, 5000);
+        }
+      }
+      
+      // Function to restart the server
+      async function restartServer() {
+        const restartBtn = document.getElementById('restart-server-btn');
+        restartBtn.disabled = true;
+        restartBtn.textContent = 'Restarting...';
+        
+        try {
+          const response = await fetch('/restart_server', {
+            method: 'POST',
+            headers: {
+              'Content-Type': 'application/json'
+            }
+          });
+          
+          if (!response.ok) {
+            throw new Error('Failed to restart server');
+          }
+          
+          const result = await response.json();
+          
+          // Show server restarting message and overlay
+          const statusElem = document.getElementById('config-status');
+          statusElem.textContent = result.message;
+          statusElem.classList.remove('hidden');
+          statusElem.classList.add('text-yellow-400');
+          
+          // Show loading overlay with different message
+          const loadingOverlay = document.getElementById('loading-overlay');
+          loadingOverlay.querySelector('p').textContent = 'Server is restarting...';
+          loadingOverlay.classList.remove('hidden');
+          
+          // Poll for server readiness, then do a complete page reload with cache busting
+          function checkServerReady() {
+            console.log("Checking if server is ready...");
+            fetch('/get_config?cache=' + Date.now(), {
+              cache: 'no-store',
+              headers: { 'pragma': 'no-cache' }
+            })
+              .then(response => {
+                if (response.ok) {
+                  console.log("Server is up and running, forcing complete page reload");
+                  // Force a complete reload with cache busting
+                  window.location = window.location.pathname + '?t=' + Date.now();
+                } else {
+                  console.log("Server not ready yet, status: " + response.status);
+                  setTimeout(checkServerReady, 1000);
+                }
+              })
+              .catch(error => {
+                console.log("Server not responding yet, waiting...");
+                setTimeout(checkServerReady, 1000);
+              });
+          }
+
+          // Start checking after initial delay
+          setTimeout(checkServerReady, 2000);
+          
+        } catch (error) {
+          console.error('Error restarting server:', error);
+          
+          // Show error message
+          const statusElem = document.getElementById('config-status');
+          statusElem.textContent = 'Error restarting server. Please try again.';
+          statusElem.classList.remove('hidden');
+          statusElem.classList.add('text-red-400');
+          
+          // Reset button
+          restartBtn.disabled = false;
+          restartBtn.textContent = 'Restart Server';
+        }
+      }
+      
+      // Add event listeners
+      document.getElementById('save-config-btn').addEventListener('click', function() {
+        saveConfiguration();
+        // Show restart button after saving
+        const restartBtn = document.getElementById('restart-server-btn');
+        restartBtn.classList.remove('hidden');
+      });
+      
+      document.getElementById('restart-server-btn').addEventListener('click', restartServer);
+      
+      // Load configuration values when page loads
+      loadConfigValues();
     });
   </script>
 </body>

tts_engine/__init__.py

@@ -11,5 +11,7 @@ from .inference import (
     generate_speech_from_api,
     AVAILABLE_VOICES,
     DEFAULT_VOICE,
+    VOICE_TO_LANGUAGE,
+    AVAILABLE_LANGUAGES,
     list_available_voices
 )

tts_engine/inference.py

@@ -12,44 +12,173 @@ import queue
 import asyncio
 from concurrent.futures import ThreadPoolExecutor
 from typing import List, Dict, Any, Optional, Generator, Union, Tuple
+from dotenv import load_dotenv
 
-# Detect if we're on a high-end system like RTX 4090
+# Helper to detect if running in Uvicorn's reloader
+def is_reloader_process():
+    """Check if the current process is a uvicorn reloader"""
+    return (sys.argv[0].endswith('_continuation.py') or 
+            os.environ.get('UVICORN_STARTED') == 'true')
+
+# Set a flag to avoid repeat messages
+IS_RELOADER = is_reloader_process()
+if not IS_RELOADER:
+    os.environ['UVICORN_STARTED'] = 'true'
+
+# Load environment variables from .env file
+load_dotenv()
+
+# Detect hardware capabilities and display information
 import torch
+import psutil
+
+# Detect if we're on a high-end system based on hardware capabilities
 HIGH_END_GPU = False
 if torch.cuda.is_available():
-    gpu_name = torch.cuda.get_device_name(0).lower()
+    # Get GPU properties
-    if any(x in gpu_name for x in ['4090', '3090', 'a100', 'h100']):
+    props = torch.cuda.get_device_properties(0)
-        HIGH_END_GPU = True
+    gpu_name = props.name
-        print(f"High-end GPU detected: {torch.cuda.get_device_name(0)}")
+    gpu_mem_gb = props.total_memory / (1024**3)
-        print("Enabling high-performance optimizations")
+    compute_capability = f"{props.major}.{props.minor}"
+    
+    # Consider high-end if: large VRAM (≥16GB) OR high compute capability (≥8.0) OR large VRAM (≥12GB) with good CC (≥7.0)
+    HIGH_END_GPU = (gpu_mem_gb >= 16.0 or 
+                    props.major >= 8 or 
+                    (gpu_mem_gb >= 12.0 and props.major >= 7))
+        
+    if HIGH_END_GPU:
+        if not IS_RELOADER:
+            print(f"🖥️ Hardware: High-end CUDA GPU detected")
+            print(f"📊 Device: {gpu_name}")
+            print(f"📊 VRAM: {gpu_mem_gb:.2f} GB")
+            print(f"📊 Compute Capability: {compute_capability}")
+            print("🚀 Using high-performance optimizations")
+    else:
+        if not IS_RELOADER:
+            print(f"🖥️ Hardware: CUDA GPU detected")
+            print(f"📊 Device: {gpu_name}")
+            print(f"📊 VRAM: {gpu_mem_gb:.2f} GB")
+            print(f"📊 Compute Capability: {compute_capability}")
+            print("🚀 Using GPU-optimized settings")
+else:
+    # Get CPU info
+    cpu_cores = psutil.cpu_count(logical=False)
+    cpu_threads = psutil.cpu_count(logical=True)
+    ram_gb = psutil.virtual_memory().total / (1024**3)
+    
+    if not IS_RELOADER:
+        print(f"🖥️ Hardware: CPU only (No CUDA GPU detected)")
+        print(f"📊 CPU: {cpu_cores} cores, {cpu_threads} threads")
+        print(f"📊 RAM: {ram_gb:.2f} GB")
+        print("⚙️ Using CPU-optimized settings")
+
+# Load configuration from environment variables without hardcoded defaults
+# Critical settings - will log errors if missing
+required_settings = ["ORPHEUS_API_URL"]
+missing_settings = [s for s in required_settings if s not in os.environ]
+if missing_settings:
+    print(f"ERROR: Missing required environment variable(s): {', '.join(missing_settings)}")
+    print("Please set them in .env file or environment. See .env.example for defaults.")
+
+# API connection settings
+API_URL = os.environ.get("ORPHEUS_API_URL")
+if not API_URL:
+    print("WARNING: ORPHEUS_API_URL not set. API calls will fail until configured.")
 
-# Orpheus-FASTAPI settings - make configurable for different endpoints
-API_URL = os.environ.get("ORPHEUS_API_URL", "http://your-server-ip:port/v1/completions or v1/chat/completions")
 HEADERS = {
     "Content-Type": "application/json"
 }
 
-# Better timeout handling for API requests
+# Request timeout settings
-REQUEST_TIMEOUT = int(os.environ.get("ORPHEUS_API_TIMEOUT", "120"))  # 120 seconds default for long generations
+try:
+    REQUEST_TIMEOUT = int(os.environ.get("ORPHEUS_API_TIMEOUT", "120"))
+except (ValueError, TypeError):
+    print("WARNING: Invalid ORPHEUS_API_TIMEOUT value, using 120 seconds as fallback")
+    REQUEST_TIMEOUT = 120
 
-# Model parameters - optimized defaults for high-end GPUs
+# Model generation parameters from environment variables
-MAX_TOKENS = 8192 if HIGH_END_GPU else 1200  # Significantly increased for RTX 4090 to allow ~1.5-2 minutes of audio
+try:
-TEMPERATURE = 0.6 
+    MAX_TOKENS = int(os.environ.get("ORPHEUS_MAX_TOKENS", "8192"))
-TOP_P = 0.9
+except (ValueError, TypeError):
+    print("WARNING: Invalid ORPHEUS_MAX_TOKENS value, using 8192 as fallback")
+    MAX_TOKENS = 8192
+
+try:
+    TEMPERATURE = float(os.environ.get("ORPHEUS_TEMPERATURE", "0.6"))
+except (ValueError, TypeError):
+    print("WARNING: Invalid ORPHEUS_TEMPERATURE value, using 0.6 as fallback")
+    TEMPERATURE = 0.6
+
+try:
+    TOP_P = float(os.environ.get("ORPHEUS_TOP_P", "0.9"))
+except (ValueError, TypeError):
+    print("WARNING: Invalid ORPHEUS_TOP_P value, using 0.9 as fallback")
+    TOP_P = 0.9
+
+# Repetition penalty is hardcoded to 1.1 which is the only stable value for quality output
 REPETITION_PENALTY = 1.1
-SAMPLE_RATE = 24000  # SNAC model uses 24kHz
+
+try:
+    SAMPLE_RATE = int(os.environ.get("ORPHEUS_SAMPLE_RATE", "24000"))
+except (ValueError, TypeError):
+    print("WARNING: Invalid ORPHEUS_SAMPLE_RATE value, using 24000 as fallback")
+    SAMPLE_RATE = 24000
+
+# Print loaded configuration only in the main process, not in the reloader
+if not IS_RELOADER:
+    print(f"Configuration loaded:")
+    print(f"  API_URL: {API_URL}")
+    print(f"  MAX_TOKENS: {MAX_TOKENS}")
+    print(f"  TEMPERATURE: {TEMPERATURE}")
+    print(f"  TOP_P: {TOP_P}")
+    print(f"  REPETITION_PENALTY: {REPETITION_PENALTY}")
 
 # Parallel processing settings
 NUM_WORKERS = 4 if HIGH_END_GPU else 2
 
-# Available voices based on the Orpheus-TTS repository
+# Define voices by language
-AVAILABLE_VOICES = ["tara", "leah", "jess", "leo", "dan", "mia", "zac", "zoe"]
+ENGLISH_VOICES = ["tara", "leah", "jess", "leo", "dan", "mia", "zac", "zoe"]
+FRENCH_VOICES = ["pierre", "amelie", "marie"]
+GERMAN_VOICES = ["jana", "thomas", "max"]
+KOREAN_VOICES = ["유나", "준서"]
+HINDI_VOICES = ["ऋतिका"]
+MANDARIN_VOICES = ["长乐", "白芷"]
+SPANISH_VOICES = ["javi", "sergio", "maria"]
+ITALIAN_VOICES = ["pietro", "giulia", "carlo"]
+
+# Combined list for API compatibility
+AVAILABLE_VOICES = (
+    ENGLISH_VOICES + 
+    FRENCH_VOICES + 
+    GERMAN_VOICES + 
+    KOREAN_VOICES + 
+    HINDI_VOICES + 
+    MANDARIN_VOICES + 
+    SPANISH_VOICES + 
+    ITALIAN_VOICES
+)
 DEFAULT_VOICE = "tara"  # Best voice according to documentation
 
+# Map voices to languages for the UI
+VOICE_TO_LANGUAGE = {}
+VOICE_TO_LANGUAGE.update({voice: "english" for voice in ENGLISH_VOICES})
+VOICE_TO_LANGUAGE.update({voice: "french" for voice in FRENCH_VOICES})
+VOICE_TO_LANGUAGE.update({voice: "german" for voice in GERMAN_VOICES})
+VOICE_TO_LANGUAGE.update({voice: "korean" for voice in KOREAN_VOICES})
+VOICE_TO_LANGUAGE.update({voice: "hindi" for voice in HINDI_VOICES})
+VOICE_TO_LANGUAGE.update({voice: "mandarin" for voice in MANDARIN_VOICES})
+VOICE_TO_LANGUAGE.update({voice: "spanish" for voice in SPANISH_VOICES})
+VOICE_TO_LANGUAGE.update({voice: "italian" for voice in ITALIAN_VOICES})
+
+# Languages list for the UI
+AVAILABLE_LANGUAGES = ["english", "french", "german", "korean", "hindi", "mandarin", "spanish", "italian"]
+
+# Import the unified token handling from speechpipe
+from .speechpipe import turn_token_into_id, CUSTOM_TOKEN_PREFIX
+
 # Special token IDs for Orpheus model
 START_TOKEN_ID = 128259
 END_TOKEN_IDS = [128009, 128260, 128261, 128257]
-CUSTOM_TOKEN_PREFIX = "<custom_token_"
 
 # Performance monitoring
 class PerformanceMonitor:
@@ -115,14 +244,15 @@ def generate_tokens_from_api(prompt: str, voice: str = DEFAULT_VOICE, temperatur
     formatted_prompt = format_prompt(prompt, voice)
     print(f"Generating speech for: {formatted_prompt}")
     
-    # Optimize the token generation for high-end GPUs
+    # Optimize the token generation for GPUs
     if HIGH_END_GPU:
         # Use more aggressive parameters for faster generation on high-end GPUs
         print("Using optimized parameters for high-end GPU")
+    elif torch.cuda.is_available():
+        print("Using optimized parameters for GPU acceleration")
     
-    # Create the request payload
+    # Create the request payload (model field may not be required by some endpoints but included for compatibility)
     payload = {
-        "model": "orpheus-3b-0.1-ft-q4_k_m",  # Model name can be anything, endpoint will use loaded model
         "prompt": formatted_prompt,
         "max_tokens": max_tokens,
         "temperature": temperature,
@@ -131,6 +261,11 @@ def generate_tokens_from_api(prompt: str, voice: str = DEFAULT_VOICE, temperatur
         "stream": True  # Always stream for better performance
     }
     
+    # Add model field - this is ignored by many local inference servers for /v1/completions
+    # but included for compatibility with OpenAI API and some servers that may use it
+    model_name = os.environ.get("ORPHEUS_MODEL_NAME", "Orpheus-3b-FT-Q8_0.gguf")
+    payload["model"] = model_name
+    
     # Session for connection pooling and retry logic
     session = requests.Session()
     
@@ -177,12 +312,14 @@ def generate_tokens_from_api(prompt: str, voice: str = DEFAULT_VOICE, temperatur
                         try:
                             data = json.loads(data_str)
                             if 'choices' in data and len(data['choices']) > 0:
-                                token_text = data['choices'][0].get('text', '')
+                                token_chunk = data['choices'][0].get('text', '')
-                                token_counter += 1
+                                for token_text in token_chunk.split('>'):
-                                perf_monitor.add_tokens()
+                                    token_text = f'{token_text}>'
-                                
+                                    token_counter += 1
-                                if token_text:
+                                    perf_monitor.add_tokens()
-                                    yield token_text
+
+                                    if token_text:
+                                        yield token_text
                         except json.JSONDecodeError as e:
                             print(f"Error decoding JSON: {e}")
                             continue
@@ -215,44 +352,8 @@ def generate_tokens_from_api(prompt: str, voice: str = DEFAULT_VOICE, temperatur
                 print("Max retries reached. Token generation failed.")
                 return
 
-# Token ID cache to avoid repeated processing
+# The turn_token_into_id function is now imported from speechpipe.py
-token_id_cache = {}
+# This eliminates duplicate code and ensures consistent behavior
-MAX_CACHE_SIZE = 10000
-
-def turn_token_into_id(token_string: str, index: int) -> Optional[int]:
-    """Optimized token-to-ID conversion with caching."""
-    # Check cache first (significant speedup for repeated tokens)
-    cache_key = (token_string, index % 7)
-    if cache_key in token_id_cache:
-        return token_id_cache[cache_key]
-        
-    # Early rejection for obvious non-matches
-    if CUSTOM_TOKEN_PREFIX not in token_string:
-        return None
-        
-    # Process token
-    token_string = token_string.strip()
-    last_token_start = token_string.rfind(CUSTOM_TOKEN_PREFIX)
-    
-    if last_token_start == -1:
-        return None
-    
-    last_token = token_string[last_token_start:]
-    
-    if not (last_token.startswith(CUSTOM_TOKEN_PREFIX) and last_token.endswith(">")):
-        return None
-        
-    try:
-        number_str = last_token[14:-1]
-        token_id = int(number_str) - 10 - ((index % 7) * 4096)
-        
-        # Cache the result if it's valid
-        if len(token_id_cache) < MAX_CACHE_SIZE:
-            token_id_cache[cache_key] = token_id
-            
-        return token_id
-    except (ValueError, IndexError):
-        return None
 
 def convert_to_audio(multiframe: List[int], count: int) -> Optional[bytes]:
     """Convert token frames to audio with performance monitoring."""
@@ -267,13 +368,15 @@ def convert_to_audio(multiframe: List[int], count: int) -> Optional[bytes]:
     return result
 
 async def tokens_decoder(token_gen) -> Generator[bytes, None, None]:
-    """Simplified token decoder without complex ring buffer to ensure reliable output."""
+    """Simplified token decoder with early first-chunk processing for lower latency."""
     buffer = []
     count = 0
     
-    # Use conservative batch parameters to ensure output quality
+    # Use different thresholds for first chunk vs. subsequent chunks
-    min_frames = 28  # Default for reliability (4 chunks of 7)
+    first_chunk_processed = False
-    process_every = 7  # Process every 7 tokens (standard for Orpheus)
+    min_frames_first = 7  # Process after just 7 tokens for first chunk (ultra-low latency)
+    min_frames_subsequent = 28  # Default for reliability after first chunk (4 chunks of 7)
+    process_every = 7  # Process every 7 tokens (standard for Orpheus model)
     
     start_time = time.time()
     last_log_time = start_time
@@ -295,19 +398,32 @@ async def tokens_decoder(token_gen) -> Generator[bytes, None, None]:
                     print(f"Token processing rate: {token_count/elapsed:.1f} tokens/second")
                 last_log_time = current_time
             
-            # Process in standard batches for Orpheus model
+            # Different processing paths based on whether first chunk has been processed
-            if count % process_every == 0 and count >= min_frames:
+            if not first_chunk_processed:
-                # Use simple slice operation - reliable and correct
+                # For first audio output, process as soon as we have enough tokens for one chunk
-                buffer_to_proc = buffer[-min_frames:]
+                if count >= min_frames_first:
-                
+                    buffer_to_proc = buffer[-min_frames_first:]
-                # Debug output to help diagnose issues
+                    
-                if count % 28 == 0:
+                    # Process the first chunk for immediate audio feedback
-                    print(f"Processing buffer with {len(buffer_to_proc)} tokens, total collected: {len(buffer)}")
+                    print(f"Processing first audio chunk with {len(buffer_to_proc)} tokens")
-                
+                    audio_samples = convert_to_audio(buffer_to_proc, count)
-                # Process the tokens
+                    if audio_samples is not None:
-                audio_samples = convert_to_audio(buffer_to_proc, count)
+                        first_chunk_processed = True  # Mark first chunk as processed
-                if audio_samples is not None:
+                        yield audio_samples
-                    yield audio_samples
+            else:
+                # For subsequent chunks, use standard processing with larger batch
+                if count % process_every == 0 and count >= min_frames_subsequent:
+                    # Use simple slice operation - reliable and correct
+                    buffer_to_proc = buffer[-min_frames_subsequent:]
+                    
+                    # Debug output to help diagnose issues
+                    if count % 28 == 0:
+                        print(f"Processing buffer with {len(buffer_to_proc)} tokens, total collected: {len(buffer)}")
+                    
+                    # Process the tokens
+                    audio_samples = convert_to_audio(buffer_to_proc, count)
+                    if audio_samples is not None:
+                        yield audio_samples
 
 def tokens_decoder_sync(syn_token_gen, output_file=None):
     """Optimized synchronous wrapper with parallel processing and efficient file I/O."""
@@ -329,6 +445,10 @@ def tokens_decoder_sync(syn_token_gen, output_file=None):
     # Batch processing of tokens for improved throughput
     batch_size = 32 if HIGH_END_GPU else 16
     
+    # Thread synchronization for proper completion detection
+    producer_done_event = threading.Event()
+    producer_started_event = threading.Event()
+    
     # Convert the synchronous token generator into an async generator with batching
     async def async_token_gen():
         batch = []
@@ -338,7 +458,7 @@ def tokens_decoder_sync(syn_token_gen, output_file=None):
                 for t in batch:
                     yield t
                 batch = []
-        # Process any remaining tokens
+        # Process any remaining tokens in the final batch
         for t in batch:
             yield t
 
@@ -348,82 +468,120 @@ def tokens_decoder_sync(syn_token_gen, output_file=None):
         chunk_count = 0
         last_log_time = start_time
         
-        async for audio_chunk in tokens_decoder(async_token_gen()):
+        try:
-            audio_queue.put(audio_chunk)
+            # Signal that producer has started processing
-            chunk_count += 1
+            producer_started_event.set()
             
-            # Log performance periodically
+            async for audio_chunk in tokens_decoder(async_token_gen()):
-            current_time = time.time()
+                # Process each audio chunk from the decoder
-            if current_time - last_log_time >= 3.0:  # Every 3 seconds
+                if audio_chunk:
-                elapsed = current_time - start_time
+                    audio_queue.put(audio_chunk)
-                if elapsed > 0:
+                    chunk_count += 1
-                    chunks_per_sec = chunk_count / elapsed
+                    
-                    print(f"Audio generation rate: {chunks_per_sec:.2f} chunks/second")
+                    # Log performance periodically
-                last_log_time = current_time
+                    current_time = time.time()
-                
+                    if current_time - last_log_time >= 3.0:  # Every 3 seconds
-        # Signal completion
+                        elapsed = current_time - last_log_time
-        audio_queue.put(None)
+                        if elapsed > 0:
+                            recent_chunks = chunk_count
+                            chunks_per_sec = recent_chunks / elapsed
+                            print(f"Audio generation rate: {chunks_per_sec:.2f} chunks/second")
+                        last_log_time = current_time
+                        # Reset chunk counter for next interval
+                        chunk_count = 0
+        except Exception as e:
+            print(f"Error in token processing: {str(e)}")
+            import traceback
+            traceback.print_exc()
+        finally:
+            # Always signal completion, even if there was an error
+            print("Producer completed - setting done event")
+            producer_done_event.set()
+            # Add sentinel to queue to signal end of stream
+            audio_queue.put(None)
 
     def run_async():
+        """Run the async producer in its own thread"""
         asyncio.run(async_producer())
 
     # Use a separate thread with higher priority for producer
-    thread = threading.Thread(target=run_async)
+    thread = threading.Thread(target=run_async, name="TokenProcessor")
     thread.daemon = True  # Allow thread to be terminated when main thread exits
     thread.start()
     
-    # For high-end GPUs, use a ThreadPoolExecutor for parallel file I/O
+    # Wait for producer to actually start before proceeding
-    if HIGH_END_GPU and wav_file:
+    # This avoids race conditions where we might try to read from an empty queue
-        # Buffer for collecting chunks before writing
+    # before the producer has had a chance to add anything
-        write_buffer = []
+    producer_started_event.wait(timeout=5.0)
-        buffer_size = 10  # Write every 10 chunks
+    
-        
+    # Optimized I/O approach for all systems
-        def write_chunks_to_file(chunks, file):
+    # This approach is simpler and more reliable than separate code paths
-            for chunk in chunks:
+    write_buffer = bytearray()
-                file.writeframes(chunk)
+    buffer_max_size = 1024 * 1024  # 1MB max buffer size (adjustable)
-        
+    
-        with ThreadPoolExecutor(max_workers=2) as executor:
+    # Keep track of the last time we checked for completion
-            future = None
+    last_check_time = time.time()
+    check_interval = 1.0  # Check producer status every second
+    
+    # Process audio chunks until we're done
+    while True:
+        try:
+            # Get the next audio chunk with a short timeout
+            # This allows us to periodically check status and handle other events
+            audio = audio_queue.get(timeout=0.1)
             
-            while True:
+            # None marker indicates end of stream
-                audio = audio_queue.get()
-                if audio is None:
-                    # Write any remaining buffered chunks
-                    if write_buffer and wav_file:
-                        if future:
-                            future.result()  # Wait for previous write to complete
-                        write_chunks_to_file(write_buffer, wav_file)
-                    break
-                
-                audio_segments.append(audio)
-                
-                if wav_file:
-                    write_buffer.append(audio)
-                    if len(write_buffer) >= buffer_size:
-                        if future:
-                            future.result()  # Wait for previous write to complete
-                        # Write in a separate thread to avoid blocking
-                        chunks_to_write = write_buffer
-                        write_buffer = []
-                        future = executor.submit(write_chunks_to_file, chunks_to_write, wav_file)
-    else:
-        # Simpler direct approach for lower-end systems
-        while True:
-            audio = audio_queue.get()
             if audio is None:
+                print("Received end-of-stream marker")
                 break
             
+            # Store the audio segment for return value
             audio_segments.append(audio)
             
-            # Write to WAV file if provided
+            # Write to file if needed
             if wav_file:
-                wav_file.writeframes(audio)
+                write_buffer.extend(audio)
+                
+                # Flush buffer if it's large enough
+                if len(write_buffer) >= buffer_max_size:
+                    wav_file.writeframes(write_buffer)
+                    write_buffer = bytearray()  # Reset buffer
+        
+        except queue.Empty:
+            # No data available right now
+            current_time = time.time()
+            
+            # Periodically check if producer is done
+            if current_time - last_check_time > check_interval:
+                last_check_time = current_time
+                
+                # If producer is done and queue is empty, we're finished
+                if producer_done_event.is_set() and audio_queue.empty():
+                    print("Producer done and queue empty - finishing consumer")
+                    break
+                
+                # Flush buffer periodically even if not full
+                if wav_file and len(write_buffer) > 0:
+                    wav_file.writeframes(write_buffer)
+                    write_buffer = bytearray()  # Reset buffer
+    
+    # Extra safety check - ensure thread is done
+    if thread.is_alive():
+        print("Waiting for token processor thread to complete...")
+        thread.join(timeout=10.0)
+        if thread.is_alive():
+            print("WARNING: Token processor thread did not complete within timeout")
+    
+    # Final flush of any remaining data
+    if wav_file and len(write_buffer) > 0:
+        print(f"Final buffer flush: {len(write_buffer)} bytes")
+        wav_file.writeframes(write_buffer)
     
     # Close WAV file if opened
     if wav_file:
         wav_file.close()
-    
+        if output_file:
-    thread.join()
+            print(f"Audio saved to {output_file}")
     
     # Calculate and print detailed performance metrics
     if audio_segments:
@@ -461,8 +619,59 @@ def stream_audio(audio_buffer):
     except Exception as e:
         print(f"Audio playback error: {e}")
 
+import re
+import numpy as np
+from io import BytesIO
+import wave
+
+def split_text_into_sentences(text):
+    """Split text into sentences with a more reliable approach."""
+    # We'll use a simple approach that doesn't rely on variable-width lookbehinds
+    # which aren't supported in Python's regex engine
+    
+    # First, split on common sentence ending punctuation
+    # This isn't perfect but works for most cases and avoids the regex error
+    parts = []
+    current_sentence = ""
+    
+    for char in text:
+        current_sentence += char
+        
+        # If we hit a sentence ending followed by a space, consider this a potential sentence end
+        if char in (' ', '\n', '\t') and len(current_sentence) > 1:
+            prev_char = current_sentence[-2]
+            if prev_char in ('.', '!', '?'):
+                # Check if this is likely a real sentence end and not an abbreviation
+                # (Simple heuristic: if there's a space before the period, it's likely a real sentence end)
+                if len(current_sentence) > 3 and current_sentence[-3] not in ('.', ' '):
+                    parts.append(current_sentence.strip())
+                    current_sentence = ""
+    
+    # Add any remaining text
+    if current_sentence.strip():
+        parts.append(current_sentence.strip())
+    
+    # Combine very short segments to avoid tiny audio files
+    min_chars = 20  # Minimum reasonable sentence length
+    combined_sentences = []
+    i = 0
+    
+    while i < len(parts):
+        current = parts[i]
+        
+        # If this is a short sentence and not the last one, combine with next
+        while i < len(parts) - 1 and len(current) < min_chars:
+            i += 1
+            current += " " + parts[i]
+            
+        combined_sentences.append(current)
+        i += 1
+    
+    return combined_sentences
+
 def generate_speech_from_api(prompt, voice=DEFAULT_VOICE, output_file=None, temperature=TEMPERATURE, 
-                     top_p=TOP_P, max_tokens=MAX_TOKENS, repetition_penalty=REPETITION_PENALTY):
+                     top_p=TOP_P, max_tokens=MAX_TOKENS, repetition_penalty=None, 
+                     use_batching=True, max_batch_chars=1000):
     """Generate speech from text using Orpheus model with performance optimizations."""
     print(f"Starting speech generation for '{prompt[:50]}{'...' if len(prompt) > 50 else ''}'")
     print(f"Using voice: {voice}, GPU acceleration: {'Yes (High-end)' if HIGH_END_GPU else 'Yes' if torch.cuda.is_available() else 'No'}")
@@ -473,25 +682,186 @@ def generate_speech_from_api(prompt, voice=DEFAULT_VOICE, output_file=None, temp
     
     start_time = time.time()
     
-    # Generate speech with optimized settings
+    # For shorter text, use the standard non-batched approach
-    result = tokens_decoder_sync(
+    if not use_batching or len(prompt) < max_batch_chars:
-        generate_tokens_from_api(
+        # Note: we ignore any provided repetition_penalty and always use the hardcoded value
-            prompt=prompt, 
+        # This ensures consistent quality regardless of what might be passed in
-            voice=voice,
+        result = tokens_decoder_sync(
-            temperature=temperature,
+            generate_tokens_from_api(
-            top_p=top_p,
+                prompt=prompt, 
-            max_tokens=max_tokens,
+                voice=voice,
-            repetition_penalty=repetition_penalty
+                temperature=temperature,
-        ),
+                top_p=top_p,
-        output_file=output_file
+                max_tokens=max_tokens,
-    )
+                repetition_penalty=REPETITION_PENALTY  # Always use hardcoded value
+            ),
+            output_file=output_file
+        )
+        
+        # Report final performance metrics
+        end_time = time.time()
+        total_time = end_time - start_time
+        print(f"Total speech generation completed in {total_time:.2f} seconds")
+        
+        return result
+    
+    # For longer text, use sentence-based batching
+    print(f"Using sentence-based batching for text with {len(prompt)} characters")
+    
+    # Split the text into sentences
+    sentences = split_text_into_sentences(prompt)
+    print(f"Split text into {len(sentences)} segments")
+    
+    # Create batches by combining sentences up to max_batch_chars
+    batches = []
+    current_batch = ""
+    
+    for sentence in sentences:
+        # If adding this sentence would exceed the batch size, start a new batch
+        if len(current_batch) + len(sentence) > max_batch_chars and current_batch:
+            batches.append(current_batch)
+            current_batch = sentence
+        else:
+            # Add separator space if needed
+            if current_batch:
+                current_batch += " "
+            current_batch += sentence
+    
+    # Add the last batch if it's not empty
+    if current_batch:
+        batches.append(current_batch)
+    
+    print(f"Created {len(batches)} batches for processing")
+    
+    # Process each batch and collect audio segments
+    all_audio_segments = []
+    batch_temp_files = []
+    
+    for i, batch in enumerate(batches):
+        print(f"Processing batch {i+1}/{len(batches)} ({len(batch)} characters)")
+        
+        # Create a temporary file for this batch if an output file is requested
+        temp_output_file = None
+        if output_file:
+            temp_output_file = f"outputs/temp_batch_{i}_{int(time.time())}.wav"
+            batch_temp_files.append(temp_output_file)
+        
+        # Generate speech for this batch
+        batch_segments = tokens_decoder_sync(
+            generate_tokens_from_api(
+                prompt=batch,
+                voice=voice,
+                temperature=temperature,
+                top_p=top_p,
+                max_tokens=max_tokens,
+                repetition_penalty=REPETITION_PENALTY
+            ),
+            output_file=temp_output_file
+        )
+        
+        # Add to our collection
+        all_audio_segments.extend(batch_segments)
+    
+    # If an output file was requested, stitch together the temporary files
+    if output_file and batch_temp_files:
+        # Stitch together WAV files
+        stitch_wav_files(batch_temp_files, output_file)
+        
+        # Clean up temporary files
+        for temp_file in batch_temp_files:
+            try:
+                os.remove(temp_file)
+            except Exception as e:
+                print(f"Warning: Could not remove temporary file {temp_file}: {e}")
     
     # Report final performance metrics
     end_time = time.time()
     total_time = end_time - start_time
+    
+    # Calculate combined duration
+    if all_audio_segments:
+        total_bytes = sum(len(segment) for segment in all_audio_segments)
+        duration = total_bytes / (2 * SAMPLE_RATE)  # 2 bytes per sample at 24kHz
+        print(f"Generated {len(all_audio_segments)} audio segments")
+        print(f"Generated {duration:.2f} seconds of audio in {total_time:.2f} seconds")
+        print(f"Realtime factor: {duration/total_time:.2f}x")
+        
     print(f"Total speech generation completed in {total_time:.2f} seconds")
     
-    return result
+    return all_audio_segments
+
+def stitch_wav_files(input_files, output_file, crossfade_ms=50):
+    """Stitch multiple WAV files together with crossfading for smooth transitions."""
+    if not input_files:
+        return
+        
+    print(f"Stitching {len(input_files)} WAV files together with {crossfade_ms}ms crossfade")
+    
+    # If only one file, just copy it
+    if len(input_files) == 1:
+        import shutil
+        shutil.copy(input_files[0], output_file)
+        return
+    
+    # Convert crossfade_ms to samples
+    crossfade_samples = int(SAMPLE_RATE * crossfade_ms / 1000)
+    print(f"Using {crossfade_samples} samples for crossfade at {SAMPLE_RATE}Hz")
+    
+    # Build the final audio in memory with crossfades
+    final_audio = np.array([], dtype=np.int16)
+    first_params = None
+    
+    for i, input_file in enumerate(input_files):
+        try:
+            with wave.open(input_file, 'rb') as wav:
+                if first_params is None:
+                    first_params = wav.getparams()
+                elif wav.getparams() != first_params:
+                    print(f"Warning: WAV file {input_file} has different parameters")
+                    
+                frames = wav.readframes(wav.getnframes())
+                audio = np.frombuffer(frames, dtype=np.int16)
+                
+                if i == 0:
+                    # First segment - use as is
+                    final_audio = audio
+                else:
+                    # Apply crossfade with previous segment
+                    if len(final_audio) >= crossfade_samples and len(audio) >= crossfade_samples:
+                        # Create crossfade weights
+                        fade_out = np.linspace(1.0, 0.0, crossfade_samples)
+                        fade_in = np.linspace(0.0, 1.0, crossfade_samples)
+                        
+                        # Apply crossfade
+                        crossfade_region = (final_audio[-crossfade_samples:] * fade_out + 
+                                           audio[:crossfade_samples] * fade_in).astype(np.int16)
+                        
+                        # Combine: original without last crossfade_samples + crossfade + new without first crossfade_samples
+                        final_audio = np.concatenate([final_audio[:-crossfade_samples], 
+                                                    crossfade_region, 
+                                                    audio[crossfade_samples:]])
+                    else:
+                        # One segment too short for crossfade, just append
+                        print(f"Segment {i} too short for crossfade, concatenating directly")
+                        final_audio = np.concatenate([final_audio, audio])
+        except Exception as e:
+            print(f"Error processing file {input_file}: {e}")
+            if i == 0:
+                raise  # Critical failure if first file fails
+    
+    # Write the final audio data to the output file
+    try:
+        with wave.open(output_file, 'wb') as output_wav:
+            if first_params is None:
+                raise ValueError("No valid WAV files were processed")
+                
+            output_wav.setparams(first_params)
+            output_wav.writeframes(final_audio.tobytes())
+        
+        print(f"Successfully stitched audio to {output_file} with crossfading")
+    except Exception as e:
+        print(f"Error writing output file {output_file}: {e}")
+        raise
 
 def list_available_voices():
     """List all available voices with the recommended one marked."""
@@ -514,7 +884,7 @@ def main():
     parser.add_argument("--temperature", type=float, default=TEMPERATURE, help="Temperature for generation")
     parser.add_argument("--top_p", type=float, default=TOP_P, help="Top-p sampling parameter")
     parser.add_argument("--repetition_penalty", type=float, default=REPETITION_PENALTY, 
-                       help="Repetition penalty (>=1.1 required for stable generation)")
+                       help="Repetition penalty (fixed at 1.1 for stable generation - parameter kept for compatibility)")
     
     args = parser.parse_args()
     

tts_engine/speechpipe.py

@@ -5,13 +5,25 @@ import asyncio
 import threading
 import queue
 import time
+import os
+import sys
+
+# Helper to detect if running in Uvicorn's reloader (same as in inference.py)
+def is_reloader_process():
+    """Check if the current process is a uvicorn reloader"""
+    return (sys.argv[0].endswith('_continuation.py') or 
+            os.environ.get('UVICORN_STARTED') == 'true')
+
+# Set a flag to avoid repeat messages
+IS_RELOADER = is_reloader_process()
 
 # Try to enable torch.compile if PyTorch 2.0+ is available
 TORCH_COMPILE_AVAILABLE = False
 try:
     if hasattr(torch, 'compile'):
         TORCH_COMPILE_AVAILABLE = True
-        print("PyTorch 2.0+ detected, torch.compile is available")
+        if not IS_RELOADER:
+            print("PyTorch 2.0+ detected, torch.compile is available")
 except:
     pass
 
@@ -20,7 +32,8 @@ CUDA_GRAPHS_AVAILABLE = False
 try:
     if torch.cuda.is_available() and hasattr(torch.cuda, 'make_graphed_callables'):
         CUDA_GRAPHS_AVAILABLE = True
-        print("CUDA graphs support is available")
+        if not IS_RELOADER:
+            print("CUDA graphs support is available")
 except:
     pass
 
@@ -28,18 +41,21 @@ model = SNAC.from_pretrained("hubertsiuzdak/snac_24khz").eval()
 
 # Check if CUDA is available and set device accordingly
 snac_device = "cuda" if torch.cuda.is_available() else "mps" if torch.backends.mps.is_available() else "cpu"
-print(f"Using device: {snac_device}")
+if not IS_RELOADER:
+    print(f"Using device: {snac_device}")
 model = model.to(snac_device)
 
 # Disable torch.compile as it requires Triton which isn't installed
 # We'll use regular PyTorch optimization techniques instead
-print("Using standard PyTorch optimizations (torch.compile disabled)")
+if not IS_RELOADER:
+    print("Using standard PyTorch optimizations (torch.compile disabled)")
 
 # Prepare CUDA streams for parallel processing if available
 cuda_stream = None
 if snac_device == "cuda":
     cuda_stream = torch.cuda.Stream()
-    print("Using CUDA stream for parallel processing")
+    if not IS_RELOADER:
+        print("Using CUDA stream for parallel processing")
 
 
 def convert_to_audio(multiframe, count):
@@ -117,73 +133,161 @@ def convert_to_audio(multiframe, count):
             
     return audio_bytes
 
+# Define the custom token prefix
+CUSTOM_TOKEN_PREFIX = "<custom_token_"
+
+# Use a single global cache for token processing
+token_id_cache = {}
+MAX_CACHE_SIZE = 10000  # Increased cache size for better performance
+
 def turn_token_into_id(token_string, index):
-    """Optimized token-to-id conversion with early returns and minimal string operations"""
+    """
-    token_string = token_string.strip()
+    Optimized token-to-ID conversion with caching.
+    This is the definitive implementation used by both inference.py and speechpipe.py.
     
-    # Early return for obvious mismatches
+    Args:
-    if "<custom_token_" not in token_string:
+        token_string: The token string to convert
+        index: Position index used for token offset calculation
+        
+    Returns:
+        int: Token ID if valid, None otherwise
+    """
+    # Check cache first (significant speedup for repeated tokens)
+    cache_key = (token_string, index % 7)
+    if cache_key in token_id_cache:
+        return token_id_cache[cache_key]
+        
+    # Early rejection for obvious non-matches
+    if CUSTOM_TOKEN_PREFIX not in token_string:
         return None
+        
+    # Process token
+    token_string = token_string.strip()
+    last_token_start = token_string.rfind(CUSTOM_TOKEN_PREFIX)
     
-    # Find the last token in the string
-    last_token_start = token_string.rfind("<custom_token_")
     if last_token_start == -1:
         return None
     
-    # Check if the token ends properly
+    last_token = token_string[last_token_start:]
-    if not token_string.endswith(">"):
+    
+    if not (last_token.startswith(CUSTOM_TOKEN_PREFIX) and last_token.endswith(">")):
         return None
         
     try:
-        # Extract and convert the number directly
+        number_str = last_token[14:-1]
-        number_str = token_string[last_token_start+14:-1]
+        token_id = int(number_str) - 10 - ((index % 7) * 4096)
-        return int(number_str) - 10 - ((index % 7) * 4096)
+        
+        # Cache the result if it's valid
+        if len(token_id_cache) < MAX_CACHE_SIZE:
+            token_id_cache[cache_key] = token_id
+            
+        return token_id
     except (ValueError, IndexError):
         return None
-# Cache for frequently processed tokens to avoid redundant computation
-token_cache = {}
-MAX_CACHE_SIZE = 1000  # Limit cache size to prevent memory bloat
 
 async def tokens_decoder(token_gen):
-    """Optimized token decoder with caching and conservative batch processing to ensure correct output"""
+    """Optimized token decoder with early first-chunk processing for lower latency"""
     buffer = []
     count = 0
-    # Start with conservative parameters to ensure we get audio output
+    
-    min_frames_required = 28  # Default minimum frames (4 chunks of 7)
+    # Track if first chunk has been processed
-    process_every_n = 7  # Process every 7 tokens
+    first_chunk_processed = False
+    
+    # Use different thresholds for first chunk vs. subsequent chunks
+    min_frames_first = 7  # Just one chunk (7 tokens) for first audio - ultra-low latency
+    min_frames_subsequent = 28  # Standard minimum (4 chunks of 7 tokens) after first audio
+    ideal_frames = 49  # Ideal standard frame size (7×7 window) - unchanged
+    process_every_n = 7  # Process every 7 tokens (standard for Orpheus model) - unchanged
     
     start_time = time.time()
     token_count = 0
+    last_log_time = start_time
     
     async for token_sim in token_gen:
         token_count += 1
         
-        # Check cache first to avoid redundant computation
+        # Use the unified turn_token_into_id which already handles caching
-        cache_key = (token_sim, count % 7)
+        token = turn_token_into_id(token_sim, count)
-        if cache_key in token_cache:
-            token = token_cache[cache_key]
-        else:
-            token = turn_token_into_id(token_sim, count)
-            # Add to cache if valid token
-            if token is not None and len(token_cache) < MAX_CACHE_SIZE:
-                token_cache[cache_key] = token
         
         if token is not None and token > 0:
             buffer.append(token)
             count += 1
 
-            # Process in larger batches for better GPU utilization
+            # Log throughput periodically
-            if count % process_every_n == 0 and count >= min_frames_required:
+            current_time = time.time()
-                buffer_to_proc = buffer[-min_frames_required:]
+            if current_time - last_log_time > 5.0:  # Every 5 seconds
-                audio_samples = convert_to_audio(buffer_to_proc, count)
+                elapsed = current_time - last_log_time
-                if audio_samples is not None:
+                if elapsed > 0:
-                    # Log processing rate occasionally
+                    recent_tokens = token_count
-                    if count % 140 == 0:  # Log every 20 chunks (assuming process_every_n=7)
+                    tokens_per_sec = recent_tokens / elapsed
-                        elapsed = time.time() - start_time
+                    print(f"Token processing rate: {tokens_per_sec:.1f} tokens/second")
-                        tokens_per_sec = token_count / elapsed if elapsed > 0 else 0
+                last_log_time = current_time
-                        print(f"Processing speed: {tokens_per_sec:.1f} tokens/sec, buffer size: {len(buffer)}")
+                token_count = 0
+            
+            # Different processing logic based on whether first chunk has been processed
+            if not first_chunk_processed:
+                # Process first chunk as soon as possible for minimal latency
+                if count >= min_frames_first:
+                    buffer_to_proc = buffer[-min_frames_first:]
                     
-                    yield audio_samples
+                    # Process the first chunk of audio for immediate feedback
+                    print(f"Processing first audio chunk with {len(buffer_to_proc)} tokens for low latency")
+                    audio_samples = convert_to_audio(buffer_to_proc, count)
+                    if audio_samples is not None:
+                        first_chunk_processed = True  # Mark first chunk as processed
+                        yield audio_samples
+            else:
+                # For subsequent chunks, use original processing with proper batching
+                if count % process_every_n == 0:
+                    # Use same prioritization logic as before
+                    if len(buffer) >= ideal_frames:
+                        buffer_to_proc = buffer[-ideal_frames:]
+                    elif len(buffer) >= min_frames_subsequent:
+                        buffer_to_proc = buffer[-min_frames_subsequent:]
+                    else:
+                        continue
+                    
+                    # Debug output to help diagnose issues
+                    if count % 28 == 0:
+                        print(f"Processing buffer with {len(buffer_to_proc)} tokens, total collected: {len(buffer)}")
+                    
+                    # Process the tokens
+                    audio_samples = convert_to_audio(buffer_to_proc, count)
+                    if audio_samples is not None:
+                        yield audio_samples
+    
+    # CRITICAL: End-of-generation handling - process all remaining frames
+    # Process remaining complete frames (ideal size)
+    if len(buffer) >= ideal_frames:
+        buffer_to_proc = buffer[-ideal_frames:]
+        audio_samples = convert_to_audio(buffer_to_proc, count)
+        if audio_samples is not None:
+            yield audio_samples
+            
+    # Process any additional complete frames (minimum size)
+    elif len(buffer) >= min_frames_subsequent:
+        buffer_to_proc = buffer[-min_frames_subsequent:]
+        audio_samples = convert_to_audio(buffer_to_proc, count)
+        if audio_samples is not None:
+            yield audio_samples
+            
+    # Final special case: even if we don't have minimum frames, try to process
+    # what we have by padding with silence tokens that won't affect the audio
+    elif len(buffer) >= process_every_n:
+        # Pad to minimum frame requirement with copies of the final token
+        # This is more continuous than using unrelated tokens from the beginning
+        last_token = buffer[-1]
+        padding_needed = min_frames_subsequent - len(buffer)
+        
+        # Create a padding array of copies of the last token
+        # This maintains continuity much better than circular buffering
+        padding = [last_token] * padding_needed
+        padded_buffer = buffer + padding
+        
+        print(f"Processing final partial frame: {len(buffer)} tokens + {padding_needed} repeated-token padding")
+        audio_samples = convert_to_audio(padded_buffer, count)
+        if audio_samples is not None:
+            yield audio_samples
 # ------------------ Synchronous Tokens Decoder Wrapper ------------------ #
 def tokens_decoder_sync(syn_token_gen):
     """Optimized synchronous decoder with larger queue and parallel processing"""
@@ -213,18 +317,25 @@ def tokens_decoder_sync(syn_token_gen):
         start_time = time.time()
         chunk_count = 0
         
-        # Process audio chunks from the token decoder
+        try:
-        async for audio_chunk in tokens_decoder(async_token_gen()):
+            # Process audio chunks from the token decoder
-            audio_queue.put(audio_chunk)
+            async for audio_chunk in tokens_decoder(async_token_gen()):
-            chunk_count += 1
+                if audio_chunk:  # Validate audio chunk before adding to queue
-            
+                    audio_queue.put(audio_chunk)
-            # Log performance stats periodically
+                    chunk_count += 1
-            if chunk_count % 10 == 0:
+                    
-                elapsed = time.time() - start_time
+                    # Log performance stats periodically
-                print(f"Generated {chunk_count} chunks in {elapsed:.2f}s ({chunk_count/elapsed:.2f} chunks/sec)")
+                    if chunk_count % 10 == 0:
-                
+                        elapsed = time.time() - start_time
-        # Signal completion
+                        print(f"Generated {chunk_count} chunks in {elapsed:.2f}s ({chunk_count/elapsed:.2f} chunks/sec)")
-        audio_queue.put(None)  # Sentinel
+        except Exception as e:
+            print(f"Error in audio producer: {e}")
+            import traceback
+            traceback.print_exc()
+        finally:    
+            # Signal completion
+            print("Audio producer completed - finalizing all chunks")
+            audio_queue.put(None)  # Sentinel
 
     def run_async():
         asyncio.run(async_producer())