diff --git a/.gitattributes b/.gitattributes deleted file mode 100644 index dfe0770..0000000 --- a/.gitattributes +++ /dev/null @@ -1,2 +0,0 @@ -# Auto detect text files and perform LF normalization -* text=auto diff --git a/README.md b/README.md new file mode 100644 index 0000000..68c396d --- /dev/null +++ b/README.md @@ -0,0 +1,220 @@ +# Orpheus-FASTAPI + +[![GitHub](https://img.shields.io/github/license/Lex-au/Orpheus-FastAPI)](https://github.com/Lex-au/Orpheus-FastAPI/blob/main/LICENSE.txt) + +High-performance Text-to-Speech server with OpenAI-compatible API, 8 voices, emotion tags, and modern web UI. Optimized for RTX GPUs. + +[GitHub Repository](https://github.com/Lex-au/Orpheus-FastAPI) + +## Features + +- **OpenAI API Compatible**: Drop-in replacement for OpenAI's `/v1/audio/speech` endpoint +- **Modern Web Interface**: Clean, responsive UI with waveform visualization +- **High Performance**: Optimized for RTX GPUs with parallel processing +- **Multiple Voices**: 8 different voice options with different characteristics +- **Emotion Tags**: Support for laughter, sighs, and other emotional expressions +- **Long-form Audio**: Efficient generation of extended audio content in a single request + +## Project Structure + +``` +Orpheus-FastAPI/ +├── app.py # FastAPI server and endpoints +├── requirements.txt # Dependencies +├── static/ # Static assets (favicon, etc.) +├── outputs/ # Generated audio files +├── templates/ # HTML templates +│ └── tts.html # Web UI template +└── tts_engine/ # Core TTS functionality + ├── __init__.py # Package exports + ├── inference.py # Token generation and API handling + └── speechpipe.py # Audio conversion pipeline +``` + +## Setup + +### Prerequisites + +- Python 3.8+ +- CUDA-compatible GPU (recommended: RTX series for best performance) +- Separate LLM inference server running the Orpheus model (e.g., LM Studio or llama.cpp server) + +### Installation + +1. Clone the repository: +```bash +git clone https://github.com/Lex-au/Orpheus-FastAPI.git +cd Orpheus-FastAPI +``` + +2. Create a Python virtual environment: +```bash +# Using venv (Python's built-in virtual environment) +python -m venv venv +source venv/bin/activate # On Windows: venv\Scripts\activate + +# Or using conda +conda create -n orpheus-tts python=3.10 +conda activate orpheus-tts +``` + +3. Install PyTorch with CUDA support: +```bash +pip3 install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cu124 +``` + +4. Install other dependencies: +```bash +pip3 install -r requirements.txt +``` + +5. Set up the required directories: +```bash +# Create directories for outputs and static files +mkdir -p outputs static +``` + +### Starting the Server + +Run the FastAPI server: +```bash +python app.py +``` + +Or with specific host/port: +```bash +uvicorn app:app --host 0.0.0.0 --port 5005 --reload +``` + +Access: +- Web interface: http://localhost:5005/ (or http://127.0.0.1:5005/) +- API documentation: http://localhost:5005/docs (or http://127.0.0.1:5005/docs) + +## API Usage + +### OpenAI-Compatible Endpoint + +The server provides an OpenAI-compatible API endpoint at `/v1/audio/speech`: + +```bash +curl http://localhost:5005/v1/audio/speech \ + -H "Content-Type: application/json" \ + -d '{ + "model": "orpheus", + "input": "Hello world! This is a test of the Orpheus TTS system.", + "voice": "tara", + "response_format": "wav", + "speed": 1.0 + }' \ + --output speech.wav +``` + +### Parameters + +- `input` (required): The text to convert to speech +- `model` (optional): The model to use (default: "orpheus") +- `voice` (optional): Which voice to use (default: "tara") +- `response_format` (optional): Output format (currently only "wav" is supported) +- `speed` (optional): Speed factor (0.5 to 1.5, default: 1.0) + +### Legacy API + +Additionally, a simpler `/speak` endpoint is available: + +```bash +curl -X POST http://localhost:5005/speak \ + -H "Content-Type: application/json" \ + -d '{ + "text": "Hello world! This is a test.", + "voice": "tara" + }' \ + -o output.wav +``` + +### Available Voices + +- `tara`: Female, conversational, clear +- `leah`: Female, warm, gentle +- `jess`: Female, energetic, youthful +- `leo`: Male, authoritative, deep +- `dan`: Male, friendly, casual +- `mia`: Female, professional, articulate +- `zac`: Male, enthusiastic, dynamic +- `zoe`: Female, calm, soothing + +### Emotion Tags + +You can insert emotion tags into your text to add expressiveness: + +- ``: Add laughter +- ``: Add a sigh +- ``: Add a chuckle +- ``: Add a cough sound +- ``: Add a sniffle sound +- ``: Add a groan +- ``: Add a yawning sound +- ``: Add a gasping sound + +Example: "Well, that's interesting I hadn't thought of that before." + +## Technical Details + +This server works as a frontend that connects to an external LLM inference server. It sends text prompts to the inference server, which generates tokens that are then converted to audio using the SNAC model. The system has been optimised for RTX 4090 GPUs with: + +- Vectorised tensor operations +- Parallel processing with CUDA streams +- Efficient memory management +- Token and audio caching +- Optimised batch sizes + +For best performance, adjust the API_URL in `tts_engine/inference.py` to point to your LLM inference server endpoint. + +### Integration with OpenWebUI + +You can easily integrate this TTS solution with [OpenWebUI](https://github.com/open-webui/open-webui) to add high-quality voice capabilities to your chatbot: + +1. Start your Orpheus-FASTAPI server +2. In OpenWebUI, go to Admin Panel > Settings > Audio +3. Change TTS from Web API to OpenAI +4. Set APIBASE URL to your server address (e.g., `http://localhost:5005`) +5. API Key can be set to "not-needed" +6. Set TTS Voice to one of the available voices: `tara`, `leah`, `jess`, `leo`, `dan`, `mia`, `zac`, or `zoe` +7. Set TTS Model to `tts-1` + +### External Inference Server + +This application requires a separate LLM inference server running the Orpheus model. You can use: + +- [GPUStack](https://github.com/gpustack/gpustack) - GPU optimised LLM inference server (My pick) - supports LAN/WAN tensor split parallelisation +- [LM Studio](https://lmstudio.ai/) - Load the GGUF model and start the local server +- [llama.cpp server](https://github.com/ggerganov/llama.cpp) - Run with the appropriate model parameters +- Any compatible OpenAI API-compatible server + +Download the quantised model from [lex-au/Orpheus-3b-FT-Q8_0.gguf](https://huggingface.co/lex-au/Orpheus-3b-FT-Q8_0.gguf) and load it in your inference server. + +The inference server should be configured to expose an API endpoint that this FastAPI application will connect to. + +### Environment Variables + +You can configure the system by setting environment variables: + +- `ORPHEUS_API_URL`: URL of the LLM inference API (tts_engine/inference.py) +- `ORPHEUS_API_TIMEOUT`: Timeout in seconds for API requests (default: 120) + +Make sure the `ORPHEUS_API_URL` points to your running inference server. + +## Development + +### Project Components + +- **app.py**: FastAPI server that handles HTTP requests and serves the web UI +- **tts_engine/inference.py**: Handles token generation and API communication +- **tts_engine/speechpipe.py**: Converts token sequences to audio using the SNAC model + +### Adding New Voices + +To add new voices, update the `AVAILABLE_VOICES` list in `tts_engine/inference.py` and add corresponding descriptions in the HTML template. + +## License + +This project is licensed under the Apache License 2.0 - see the LICENSE.txt file for details. diff --git a/app.py b/app.py new file mode 100644 index 0000000..ff6b7bd --- /dev/null +++ b/app.py @@ -0,0 +1,166 @@ +import os +import time +from datetime import datetime +from typing import List, Optional + +from fastapi import FastAPI, Request, Form, HTTPException, Depends +from fastapi.responses import HTMLResponse, FileResponse, JSONResponse +from fastapi.staticfiles import StaticFiles +from fastapi.templating import Jinja2Templates +from pydantic import BaseModel + +from tts_engine import generate_speech_from_api, AVAILABLE_VOICES, DEFAULT_VOICE + +# Create FastAPI app +app = FastAPI( + title="Orpheus-FASTAPI", + description="High-performance Text-to-Speech server using Orpheus-FASTAPI", + version="1.0.0" +) + +# Ensure directories exist +os.makedirs("outputs", exist_ok=True) +os.makedirs("static", exist_ok=True) + +# Mount directories for serving files +app.mount("/outputs", StaticFiles(directory="outputs"), name="outputs") +app.mount("/static", StaticFiles(directory="static"), name="static") + +# Setup templates +templates = Jinja2Templates(directory="templates") + +# API models +class SpeechRequest(BaseModel): + input: str + model: str = "orpheus" + voice: str = DEFAULT_VOICE + response_format: str = "wav" + speed: float = 1.0 + +class APIResponse(BaseModel): + status: str + voice: str + output_file: str + generation_time: float + +# OpenAI-compatible API endpoint +@app.post("/v1/audio/speech") +async def create_speech_api(request: SpeechRequest): + """ + Generate speech from text using the Orpheus TTS model. + Compatible with OpenAI's /v1/audio/speech endpoint. + """ + if not request.input: + raise HTTPException(status_code=400, detail="Missing input text") + + # Generate unique filename + timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") + output_path = f"outputs/{request.voice}_{timestamp}.wav" + + # Generate speech + start = time.time() + generate_speech_from_api( + prompt=request.input, + voice=request.voice, + output_file=output_path + ) + end = time.time() + generation_time = round(end - start, 2) + + # Return audio file + return FileResponse( + path=output_path, + media_type="audio/wav", + filename=f"{request.voice}_{timestamp}.wav" + ) + +# Legacy API endpoint for compatibility +@app.post("/speak") +async def speak(request: Request): + """Legacy endpoint for compatibility with existing clients""" + data = await request.json() + text = data.get("text", "") + voice = data.get("voice", DEFAULT_VOICE) + + if not text: + return JSONResponse( + status_code=400, + content={"error": "Missing 'text'"} + ) + + timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") + output_path = f"outputs/{voice}_{timestamp}.wav" + + # Generate speech + start = time.time() + generate_speech_from_api(prompt=text, voice=voice, output_file=output_path) + end = time.time() + generation_time = round(end - start, 2) + + return JSONResponse(content={ + "status": "ok", + "voice": voice, + "output_file": output_path, + "generation_time": generation_time + }) + +# Web UI routes +@app.get("/", response_class=HTMLResponse) +async def root(request: Request): + """Redirect to web UI""" + return templates.TemplateResponse( + "tts.html", + {"request": request, "voices": AVAILABLE_VOICES} + ) + +@app.get("/web/", response_class=HTMLResponse) +async def web_ui(request: Request): + """Main web UI for TTS generation""" + return templates.TemplateResponse( + "tts.html", + {"request": request, "voices": AVAILABLE_VOICES} + ) + +@app.post("/web/", response_class=HTMLResponse) +async def generate_from_web( + request: Request, + text: str = Form(...), + voice: str = Form(DEFAULT_VOICE) +): + """Handle form submission from web UI""" + if not text: + return templates.TemplateResponse( + "tts.html", + { + "request": request, + "error": "Please enter some text.", + "voices": AVAILABLE_VOICES + } + ) + + timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") + output_path = f"outputs/{voice}_{timestamp}.wav" + + # Generate speech + start = time.time() + generate_speech_from_api(prompt=text, voice=voice, output_file=output_path) + end = time.time() + generation_time = round(end - start, 2) + + return templates.TemplateResponse( + "tts.html", + { + "request": request, + "success": True, + "text": text, + "voice": voice, + "output_file": output_path, + "generation_time": generation_time, + "voices": AVAILABLE_VOICES + } + ) + +if __name__ == "__main__": + import uvicorn + print("🔥 Starting Orpheus-FASTAPI Server (CUDA)") + uvicorn.run("app:app", host="0.0.0.0", port=5005, reload=True) diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..ed4ebef --- /dev/null +++ b/requirements.txt @@ -0,0 +1,15 @@ +# Core dependencies +fastapi>=0.103.1 +uvicorn>=0.23.2 +jinja2>=3.1.2 +pydantic>=2.3.0 +numpy>=1.24.0 +requests>=2.31.0 +sounddevice>=0.4.6 +python-multipart>=0.0.6 + +# SNAC is required for audio generation from tokens +snac>=0.3.0 + +# PyTorch - Note: Install PyTorch with CUDA 12.4 support separately: +# pip3 install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cu124 diff --git a/static/favicon.ico b/static/favicon.ico new file mode 100644 index 0000000..29d3da7 Binary files /dev/null and b/static/favicon.ico differ diff --git a/templates/tts.html b/templates/tts.html new file mode 100644 index 0000000..c9ca439 --- /dev/null +++ b/templates/tts.html @@ -0,0 +1,488 @@ + + + + + + Orpheus FASTAPI | Advanced Text-to-Speech + + + + + + + +
+ + + + +
+
+ + + {% if error %} +
+
+
+ + + +
+
+

{{ error }}

+
+
+
+ {% endif %} + + {% if success %} +
+
+
+ + + +
+
+

Audio generated successfully in {{ generation_time }}s!

+
+
+
+ {% endif %} + + +
+
+
+

Generate Speech

+ + +
+ +
+ +
+ 0 / 8192 characters +
+
+
+ + +
+ +
+ {% for voice_option in voices %} +
+ +
+ {{ voice_option|capitalize }} +
+
+ {% if voice_option == "tara" %}Female, conversational, clear + {% elif voice_option == "leah" %}Female, warm, gentle + {% elif voice_option == "jess" %}Female, energetic, youthful + {% elif voice_option == "leo" %}Male, authoritative, deep + {% elif voice_option == "dan" %}Male, friendly, casual + {% elif voice_option == "mia" %}Female, professional, articulate + {% elif voice_option == "zac" %}Male, enthusiastic, dynamic + {% elif voice_option == "zoe" %}Female, calm, soothing + {% endif %} +
+
+ {% endfor %} +
+
+ + +
+
+ + Advanced options + + + + + + +
+
+ +
+ +
+ + + +
+
+
+
+ + +
+ Slower + 1.0 + Faster +
+
+
+
+
+
+ +
+
+

Supports emotion tags: <laugh>, <sigh>, etc.

+
+ +
+
+
+ + +
+ + +
+

Tips & Tricks

+
+
+
    +
  • Use <laugh> to add laughter to the speech
    • +
  • Use <sigh> for a sighing sound
    • +
  • Other supported tags: <chuckle>, <cough>, <sniffle>, <groan>, <yawn>, <gasp>
    • +
  • For longer audio, the system can generate up to 2 minutes of speech in a single request
    • +
  • For API access, use the /v1/audio/speech endpoint (OpenAI compatible)
    • +
+
+
+
+
+
+ + + +
+ + + + + + + + + + + diff --git a/tts_engine/__init__.py b/tts_engine/__init__.py new file mode 100644 index 0000000..7252e9c --- /dev/null +++ b/tts_engine/__init__.py @@ -0,0 +1,15 @@ +""" +TTS Engine package for Orpheus text-to-speech system. + +This package contains the core components for audio generation: +- inference.py: Token generation and API handling +- speechpipe.py: Audio conversion pipeline +""" + +# Make key components available at package level +from .inference import ( + generate_speech_from_api, + AVAILABLE_VOICES, + DEFAULT_VOICE, + list_available_voices +) diff --git a/tts_engine/inference.py b/tts_engine/inference.py new file mode 100644 index 0000000..f746242 --- /dev/null +++ b/tts_engine/inference.py @@ -0,0 +1,561 @@ +import os +import sys +import requests +import json +import time +import wave +import numpy as np +import sounddevice as sd +import argparse +import threading +import queue +import asyncio +from concurrent.futures import ThreadPoolExecutor +from typing import List, Dict, Any, Optional, Generator, Union, Tuple + +# Detect if we're on a high-end system like RTX 4090 +import torch +HIGH_END_GPU = False +if torch.cuda.is_available(): + gpu_name = torch.cuda.get_device_name(0).lower() + if any(x in gpu_name for x in ['4090', '3090', 'a100', 'h100']): + HIGH_END_GPU = True + print(f"High-end GPU detected: {torch.cuda.get_device_name(0)}") + print("Enabling high-performance optimizations") + +# Orpheus-FASTAPI settings - make configurable for different endpoints +API_URL = os.environ.get("ORPHEUS_API_URL", "http://your-server-ip:port/v1/completions or v1/chat/completions") +HEADERS = { + "Content-Type": "application/json" +} + +# Better timeout handling for API requests +REQUEST_TIMEOUT = int(os.environ.get("ORPHEUS_API_TIMEOUT", "120")) # 120 seconds default for long generations + +# Model parameters - optimized defaults for high-end GPUs +MAX_TOKENS = 8192 if HIGH_END_GPU else 1200 # Significantly increased for RTX 4090 to allow ~1.5-2 minutes of audio +TEMPERATURE = 0.6 +TOP_P = 0.9 +REPETITION_PENALTY = 1.1 +SAMPLE_RATE = 24000 # SNAC model uses 24kHz + +# Parallel processing settings +NUM_WORKERS = 4 if HIGH_END_GPU else 2 + +# Available voices based on the Orpheus-TTS repository +AVAILABLE_VOICES = ["tara", "leah", "jess", "leo", "dan", "mia", "zac", "zoe"] +DEFAULT_VOICE = "tara" # Best voice according to documentation + +# Special token IDs for Orpheus model +START_TOKEN_ID = 128259 +END_TOKEN_IDS = [128009, 128260, 128261, 128257] +CUSTOM_TOKEN_PREFIX = " None: + self.token_count += count + self._check_report() + + def add_audio_chunk(self) -> None: + self.audio_chunks += 1 + self._check_report() + + def _check_report(self) -> None: + current_time = time.time() + if current_time - self.last_report_time >= self.report_interval: + self.report() + self.last_report_time = current_time + + def report(self) -> None: + elapsed = time.time() - self.start_time + if elapsed < 0.001: + return + + tokens_per_sec = self.token_count / elapsed + chunks_per_sec = self.audio_chunks / elapsed + + # Estimate audio duration based on audio chunks (each chunk is ~0.085s of audio) + est_duration = self.audio_chunks * 0.085 + + print(f"Progress: {tokens_per_sec:.1f} tokens/sec, est. {est_duration:.1f}s audio generated, {self.token_count} tokens, {self.audio_chunks} chunks in {elapsed:.1f}s") + +# Create global performance monitor +perf_monitor = PerformanceMonitor() + +def format_prompt(prompt: str, voice: str = DEFAULT_VOICE) -> str: + """Format prompt for Orpheus model with voice prefix and special tokens.""" + # Validate voice and provide fallback + if voice not in AVAILABLE_VOICES: + print(f"Warning: Voice '{voice}' not recognized. Using '{DEFAULT_VOICE}' instead.") + voice = DEFAULT_VOICE + + # Format similar to how engine_class.py does it with special tokens + formatted_prompt = f"{voice}: {prompt}" + + # Add special token markers for the Orpheus-FASTAPI + special_start = "<|audio|>" # Using the additional_special_token from config + special_end = "<|eot_id|>" # Using the eos_token from config + + return f"{special_start}{formatted_prompt}{special_end}" + +def generate_tokens_from_api(prompt: str, voice: str = DEFAULT_VOICE, temperature: float = TEMPERATURE, + top_p: float = TOP_P, max_tokens: int = MAX_TOKENS, + repetition_penalty: float = REPETITION_PENALTY) -> Generator[str, None, None]: + """Generate tokens from text using OpenAI-compatible API with optimized streaming and retry logic.""" + start_time = time.time() + formatted_prompt = format_prompt(prompt, voice) + print(f"Generating speech for: {formatted_prompt}") + + # Optimize the token generation for high-end GPUs + if HIGH_END_GPU: + # Use more aggressive parameters for faster generation on high-end GPUs + print("Using optimized parameters for high-end GPU") + + # Create the request payload + payload = { + "model": "orpheus-3b-0.1-ft-q4_k_m", # Model name can be anything, endpoint will use loaded model + "prompt": formatted_prompt, + "max_tokens": max_tokens, + "temperature": temperature, + "top_p": top_p, + "repeat_penalty": repetition_penalty, + "stream": True # Always stream for better performance + } + + # Session for connection pooling and retry logic + session = requests.Session() + + retry_count = 0 + max_retries = 3 + + while retry_count < max_retries: + try: + # Make the API request with streaming and timeout + response = session.post( + API_URL, + headers=HEADERS, + json=payload, + stream=True, + timeout=REQUEST_TIMEOUT + ) + + if response.status_code != 200: + print(f"Error: API request failed with status code {response.status_code}") + print(f"Error details: {response.text}") + # Retry on server errors (5xx) but not on client errors (4xx) + if response.status_code >= 500: + retry_count += 1 + wait_time = 2 ** retry_count # Exponential backoff + print(f"Retrying in {wait_time} seconds...") + time.sleep(wait_time) + continue + return + + # Process the streamed response with better buffering + buffer = "" + token_counter = 0 + + # Iterate through the response to get tokens + for line in response.iter_lines(): + if line: + line_str = line.decode('utf-8') + if line_str.startswith('data: '): + data_str = line_str[6:] # Remove the 'data: ' prefix + + if data_str.strip() == '[DONE]': + break + + try: + data = json.loads(data_str) + if 'choices' in data and len(data['choices']) > 0: + token_text = data['choices'][0].get('text', '') + token_counter += 1 + perf_monitor.add_tokens() + + if token_text: + yield token_text + except json.JSONDecodeError as e: + print(f"Error decoding JSON: {e}") + continue + + # Generation completed successfully + generation_time = time.time() - start_time + tokens_per_second = token_counter / generation_time if generation_time > 0 else 0 + print(f"Token generation complete: {token_counter} tokens in {generation_time:.2f}s ({tokens_per_second:.1f} tokens/sec)") + return + + except requests.exceptions.Timeout: + print(f"Request timed out after {REQUEST_TIMEOUT} seconds") + retry_count += 1 + if retry_count < max_retries: + wait_time = 2 ** retry_count + print(f"Retrying in {wait_time} seconds... (attempt {retry_count+1}/{max_retries})") + time.sleep(wait_time) + else: + print("Max retries reached. Token generation failed.") + return + + except requests.exceptions.ConnectionError: + print(f"Connection error to API at {API_URL}") + retry_count += 1 + if retry_count < max_retries: + wait_time = 2 ** retry_count + print(f"Retrying in {wait_time} seconds... (attempt {retry_count+1}/{max_retries})") + time.sleep(wait_time) + else: + print("Max retries reached. Token generation failed.") + return + +# Token ID cache to avoid repeated processing +token_id_cache = {} +MAX_CACHE_SIZE = 10000 + +def turn_token_into_id(token_string: str, index: int) -> Optional[int]: + """Optimized token-to-ID conversion with caching.""" + # Check cache first (significant speedup for repeated tokens) + cache_key = (token_string, index % 7) + if cache_key in token_id_cache: + return token_id_cache[cache_key] + + # Early rejection for obvious non-matches + if CUSTOM_TOKEN_PREFIX not in token_string: + return None + + # Process token + token_string = token_string.strip() + last_token_start = token_string.rfind(CUSTOM_TOKEN_PREFIX) + + if last_token_start == -1: + return None + + last_token = token_string[last_token_start:] + + if not (last_token.startswith(CUSTOM_TOKEN_PREFIX) and last_token.endswith(">")): + return None + + try: + number_str = last_token[14:-1] + token_id = int(number_str) - 10 - ((index % 7) * 4096) + + # Cache the result if it's valid + if len(token_id_cache) < MAX_CACHE_SIZE: + token_id_cache[cache_key] = token_id + + return token_id + except (ValueError, IndexError): + return None + +def convert_to_audio(multiframe: List[int], count: int) -> Optional[bytes]: + """Convert token frames to audio with performance monitoring.""" + # Import here to avoid circular imports + from .speechpipe import convert_to_audio as orpheus_convert_to_audio + start_time = time.time() + result = orpheus_convert_to_audio(multiframe, count) + + if result is not None: + perf_monitor.add_audio_chunk() + + return result + +async def tokens_decoder(token_gen) -> Generator[bytes, None, None]: + """Simplified token decoder without complex ring buffer to ensure reliable output.""" + buffer = [] + count = 0 + + # Use conservative batch parameters to ensure output quality + min_frames = 28 # Default for reliability (4 chunks of 7) + process_every = 7 # Process every 7 tokens (standard for Orpheus) + + start_time = time.time() + last_log_time = start_time + token_count = 0 + + async for token_text in token_gen: + token = turn_token_into_id(token_text, count) + if token is not None and token > 0: + # Add to buffer using simple append (reliable method) + buffer.append(token) + count += 1 + token_count += 1 + + # Log throughput periodically + current_time = time.time() + if current_time - last_log_time > 5.0: # Every 5 seconds + elapsed = current_time - start_time + if elapsed > 0: + print(f"Token processing rate: {token_count/elapsed:.1f} tokens/second") + last_log_time = current_time + + # Process in standard batches for Orpheus model + if count % process_every == 0 and count >= min_frames: + # Use simple slice operation - reliable and correct + buffer_to_proc = buffer[-min_frames:] + + # Debug output to help diagnose issues + if count % 28 == 0: + print(f"Processing buffer with {len(buffer_to_proc)} tokens, total collected: {len(buffer)}") + + # Process the tokens + audio_samples = convert_to_audio(buffer_to_proc, count) + if audio_samples is not None: + yield audio_samples + +def tokens_decoder_sync(syn_token_gen, output_file=None): + """Optimized synchronous wrapper with parallel processing and efficient file I/O.""" + # Use a larger queue for high-end systems + queue_size = 100 if HIGH_END_GPU else 50 + audio_queue = queue.Queue(maxsize=queue_size) + audio_segments = [] + + # If output_file is provided, prepare WAV file with buffered I/O + wav_file = None + if output_file: + # Create directory if it doesn't exist + os.makedirs(os.path.dirname(os.path.abspath(output_file)), exist_ok=True) + wav_file = wave.open(output_file, "wb") + wav_file.setnchannels(1) + wav_file.setsampwidth(2) + wav_file.setframerate(SAMPLE_RATE) + + # Batch processing of tokens for improved throughput + batch_size = 32 if HIGH_END_GPU else 16 + + # Convert the synchronous token generator into an async generator with batching + async def async_token_gen(): + batch = [] + for token in syn_token_gen: + batch.append(token) + if len(batch) >= batch_size: + for t in batch: + yield t + batch = [] + # Process any remaining tokens + for t in batch: + yield t + + async def async_producer(): + # Track performance with more granular metrics + start_time = time.time() + chunk_count = 0 + last_log_time = start_time + + async for audio_chunk in tokens_decoder(async_token_gen()): + audio_queue.put(audio_chunk) + chunk_count += 1 + + # Log performance periodically + current_time = time.time() + if current_time - last_log_time >= 3.0: # Every 3 seconds + elapsed = current_time - start_time + if elapsed > 0: + chunks_per_sec = chunk_count / elapsed + print(f"Audio generation rate: {chunks_per_sec:.2f} chunks/second") + last_log_time = current_time + + # Signal completion + audio_queue.put(None) + + def run_async(): + asyncio.run(async_producer()) + + # Use a separate thread with higher priority for producer + thread = threading.Thread(target=run_async) + thread.daemon = True # Allow thread to be terminated when main thread exits + thread.start() + + # For high-end GPUs, use a ThreadPoolExecutor for parallel file I/O + if HIGH_END_GPU and wav_file: + # Buffer for collecting chunks before writing + write_buffer = [] + buffer_size = 10 # Write every 10 chunks + + def write_chunks_to_file(chunks, file): + for chunk in chunks: + file.writeframes(chunk) + + with ThreadPoolExecutor(max_workers=2) as executor: + future = None + + while True: + audio = audio_queue.get() + if audio is None: + # Write any remaining buffered chunks + if write_buffer and wav_file: + if future: + future.result() # Wait for previous write to complete + write_chunks_to_file(write_buffer, wav_file) + break + + audio_segments.append(audio) + + if wav_file: + write_buffer.append(audio) + if len(write_buffer) >= buffer_size: + if future: + future.result() # Wait for previous write to complete + # Write in a separate thread to avoid blocking + chunks_to_write = write_buffer + write_buffer = [] + future = executor.submit(write_chunks_to_file, chunks_to_write, wav_file) + else: + # Simpler direct approach for lower-end systems + while True: + audio = audio_queue.get() + if audio is None: + break + + audio_segments.append(audio) + + # Write to WAV file if provided + if wav_file: + wav_file.writeframes(audio) + + # Close WAV file if opened + if wav_file: + wav_file.close() + + thread.join() + + # Calculate and print detailed performance metrics + if audio_segments: + total_bytes = sum(len(segment) for segment in audio_segments) + duration = total_bytes / (2 * SAMPLE_RATE) # 2 bytes per sample at 24kHz + total_time = time.time() - perf_monitor.start_time + realtime_factor = duration / total_time if total_time > 0 else 0 + + print(f"Generated {len(audio_segments)} audio segments") + print(f"Generated {duration:.2f} seconds of audio in {total_time:.2f} seconds") + print(f"Realtime factor: {realtime_factor:.2f}x") + + if realtime_factor < 1.0: + print("⚠️ Warning: Generation is slower than realtime") + else: + print(f"✓ Generation is {realtime_factor:.1f}x faster than realtime") + + return audio_segments + +def stream_audio(audio_buffer): + """Stream audio buffer to output device with error handling.""" + if audio_buffer is None or len(audio_buffer) == 0: + return + + try: + # Convert bytes to NumPy array (16-bit PCM) + audio_data = np.frombuffer(audio_buffer, dtype=np.int16) + + # Normalize to float in range [-1, 1] for playback + audio_float = audio_data.astype(np.float32) / 32767.0 + + # Play the audio with proper device selection and error handling + sd.play(audio_float, SAMPLE_RATE) + sd.wait() + except Exception as e: + print(f"Audio playback error: {e}") + +def generate_speech_from_api(prompt, voice=DEFAULT_VOICE, output_file=None, temperature=TEMPERATURE, + top_p=TOP_P, max_tokens=MAX_TOKENS, repetition_penalty=REPETITION_PENALTY): + """Generate speech from text using Orpheus model with performance optimizations.""" + print(f"Starting speech generation for '{prompt[:50]}{'...' if len(prompt) > 50 else ''}'") + print(f"Using voice: {voice}, GPU acceleration: {'Yes (High-end)' if HIGH_END_GPU else 'Yes' if torch.cuda.is_available() else 'No'}") + + # Reset performance monitor + global perf_monitor + perf_monitor = PerformanceMonitor() + + start_time = time.time() + + # Generate speech with optimized settings + result = tokens_decoder_sync( + generate_tokens_from_api( + prompt=prompt, + voice=voice, + temperature=temperature, + top_p=top_p, + max_tokens=max_tokens, + repetition_penalty=repetition_penalty + ), + output_file=output_file + ) + + # Report final performance metrics + end_time = time.time() + total_time = end_time - start_time + print(f"Total speech generation completed in {total_time:.2f} seconds") + + return result + +def list_available_voices(): + """List all available voices with the recommended one marked.""" + print("Available voices (in order of conversational realism):") + for i, voice in enumerate(AVAILABLE_VOICES): + marker = "★" if voice == DEFAULT_VOICE else " " + print(f"{marker} {voice}") + print(f"\nDefault voice: {DEFAULT_VOICE}") + + print("\nAvailable emotion tags:") + print(", , , , , , , ") + +def main(): + # Parse command line arguments + parser = argparse.ArgumentParser(description="Orpheus Text-to-Speech using Orpheus-FASTAPI") + parser.add_argument("--text", type=str, help="Text to convert to speech") + parser.add_argument("--voice", type=str, default=DEFAULT_VOICE, help=f"Voice to use (default: {DEFAULT_VOICE})") + parser.add_argument("--output", type=str, help="Output WAV file path") + parser.add_argument("--list-voices", action="store_true", help="List available voices") + parser.add_argument("--temperature", type=float, default=TEMPERATURE, help="Temperature for generation") + parser.add_argument("--top_p", type=float, default=TOP_P, help="Top-p sampling parameter") + parser.add_argument("--repetition_penalty", type=float, default=REPETITION_PENALTY, + help="Repetition penalty (>=1.1 required for stable generation)") + + args = parser.parse_args() + + if args.list_voices: + list_available_voices() + return + + # Use text from command line or prompt user + prompt = args.text + if not prompt: + if len(sys.argv) > 1 and sys.argv[1] not in ("--voice", "--output", "--temperature", "--top_p", "--repetition_penalty"): + prompt = " ".join([arg for arg in sys.argv[1:] if not arg.startswith("--")]) + else: + prompt = input("Enter text to synthesize: ") + if not prompt: + prompt = "Hello, I am Orpheus, an AI assistant with emotional speech capabilities." + + # Default output file if none provided + output_file = args.output + if not output_file: + # Create outputs directory if it doesn't exist + os.makedirs("outputs", exist_ok=True) + # Generate a filename based on the voice and a timestamp + timestamp = time.strftime("%Y%m%d_%H%M%S") + output_file = f"outputs/{args.voice}_{timestamp}.wav" + print(f"No output file specified. Saving to {output_file}") + + # Generate speech + start_time = time.time() + audio_segments = generate_speech_from_api( + prompt=prompt, + voice=args.voice, + temperature=args.temperature, + top_p=args.top_p, + repetition_penalty=args.repetition_penalty, + output_file=output_file + ) + end_time = time.time() + + print(f"Speech generation completed in {end_time - start_time:.2f} seconds") + print(f"Audio saved to {output_file}") + +if __name__ == "__main__": + main() diff --git a/tts_engine/speechpipe.py b/tts_engine/speechpipe.py new file mode 100644 index 0000000..3c1d6b2 --- /dev/null +++ b/tts_engine/speechpipe.py @@ -0,0 +1,257 @@ +from snac import SNAC +import numpy as np +import torch +import asyncio +import threading +import queue +import time + +# Try to enable torch.compile if PyTorch 2.0+ is available +TORCH_COMPILE_AVAILABLE = False +try: + if hasattr(torch, 'compile'): + TORCH_COMPILE_AVAILABLE = True + print("PyTorch 2.0+ detected, torch.compile is available") +except: + pass + +# Try to enable CUDA graphs if available +CUDA_GRAPHS_AVAILABLE = False +try: + if torch.cuda.is_available() and hasattr(torch.cuda, 'make_graphed_callables'): + CUDA_GRAPHS_AVAILABLE = True + print("CUDA graphs support is available") +except: + pass + +model = SNAC.from_pretrained("hubertsiuzdak/snac_24khz").eval() + +# Check if CUDA is available and set device accordingly +snac_device = "cuda" if torch.cuda.is_available() else "mps" if torch.backends.mps.is_available() else "cpu" +print(f"Using device: {snac_device}") +model = model.to(snac_device) + +# Disable torch.compile as it requires Triton which isn't installed +# We'll use regular PyTorch optimization techniques instead +print("Using standard PyTorch optimizations (torch.compile disabled)") + +# Prepare CUDA streams for parallel processing if available +cuda_stream = None +if snac_device == "cuda": + cuda_stream = torch.cuda.Stream() + print("Using CUDA stream for parallel processing") + + +def convert_to_audio(multiframe, count): + """ + Optimized version of convert_to_audio that eliminates inefficient tensor operations + and reduces CPU-GPU transfers for much faster inference on high-end GPUs. + """ + if len(multiframe) < 7: + return None + + num_frames = len(multiframe) // 7 + frame = multiframe[:num_frames*7] + + # Pre-allocate tensors instead of incrementally building them + codes_0 = torch.zeros(num_frames, dtype=torch.int32, device=snac_device) + codes_1 = torch.zeros(num_frames * 2, dtype=torch.int32, device=snac_device) + codes_2 = torch.zeros(num_frames * 4, dtype=torch.int32, device=snac_device) + + # Use vectorized operations where possible + frame_tensor = torch.tensor(frame, dtype=torch.int32, device=snac_device) + + # Direct indexing is much faster than concatenation in a loop + for j in range(num_frames): + idx = j * 7 + + # Code 0 - single value per frame + codes_0[j] = frame_tensor[idx] + + # Code 1 - two values per frame + codes_1[j*2] = frame_tensor[idx+1] + codes_1[j*2+1] = frame_tensor[idx+4] + + # Code 2 - four values per frame + codes_2[j*4] = frame_tensor[idx+2] + codes_2[j*4+1] = frame_tensor[idx+3] + codes_2[j*4+2] = frame_tensor[idx+5] + codes_2[j*4+3] = frame_tensor[idx+6] + + # Reshape codes into expected format + codes = [ + codes_0.unsqueeze(0), + codes_1.unsqueeze(0), + codes_2.unsqueeze(0) + ] + + # Check tokens are in valid range + if (torch.any(codes[0] < 0) or torch.any(codes[0] > 4096) or + torch.any(codes[1] < 0) or torch.any(codes[1] > 4096) or + torch.any(codes[2] < 0) or torch.any(codes[2] > 4096)): + return None + + # Use CUDA stream for parallel processing if available + stream_ctx = torch.cuda.stream(cuda_stream) if cuda_stream is not None else torch.no_grad() + + with stream_ctx, torch.inference_mode(): + # Decode the audio + audio_hat = model.decode(codes) + + # Extract the relevant slice and efficiently convert to bytes + # Keep data on GPU as long as possible + audio_slice = audio_hat[:, :, 2048:4096] + + # Process on GPU if possible, with minimal data transfer + if snac_device == "cuda": + # Scale directly on GPU + audio_int16_tensor = (audio_slice * 32767).to(torch.int16) + # Only transfer the final result to CPU + audio_bytes = audio_int16_tensor.cpu().numpy().tobytes() + else: + # For non-CUDA devices, fall back to the original approach + detached_audio = audio_slice.detach().cpu() + audio_np = detached_audio.numpy() + audio_int16 = (audio_np * 32767).astype(np.int16) + audio_bytes = audio_int16.tobytes() + + return audio_bytes + +def turn_token_into_id(token_string, index): + """Optimized token-to-id conversion with early returns and minimal string operations""" + token_string = token_string.strip() + + # Early return for obvious mismatches + if ""): + return None + + try: + # Extract and convert the number directly + number_str = token_string[last_token_start+14:-1] + return int(number_str) - 10 - ((index % 7) * 4096) + except (ValueError, IndexError): + return None +# Cache for frequently processed tokens to avoid redundant computation +token_cache = {} +MAX_CACHE_SIZE = 1000 # Limit cache size to prevent memory bloat + +async def tokens_decoder(token_gen): + """Optimized token decoder with caching and conservative batch processing to ensure correct output""" + buffer = [] + count = 0 + # Start with conservative parameters to ensure we get audio output + min_frames_required = 28 # Default minimum frames (4 chunks of 7) + process_every_n = 7 # Process every 7 tokens + + start_time = time.time() + token_count = 0 + + async for token_sim in token_gen: + token_count += 1 + + # Check cache first to avoid redundant computation + cache_key = (token_sim, count % 7) + if cache_key in token_cache: + token = token_cache[cache_key] + else: + token = turn_token_into_id(token_sim, count) + # Add to cache if valid token + if token is not None and len(token_cache) < MAX_CACHE_SIZE: + token_cache[cache_key] = token + + if token is not None and token > 0: + buffer.append(token) + count += 1 + + # Process in larger batches for better GPU utilization + if count % process_every_n == 0 and count >= min_frames_required: + buffer_to_proc = buffer[-min_frames_required:] + audio_samples = convert_to_audio(buffer_to_proc, count) + if audio_samples is not None: + # Log processing rate occasionally + if count % 140 == 0: # Log every 20 chunks (assuming process_every_n=7) + elapsed = time.time() - start_time + tokens_per_sec = token_count / elapsed if elapsed > 0 else 0 + print(f"Processing speed: {tokens_per_sec:.1f} tokens/sec, buffer size: {len(buffer)}") + + yield audio_samples +# ------------------ Synchronous Tokens Decoder Wrapper ------------------ # +def tokens_decoder_sync(syn_token_gen): + """Optimized synchronous decoder with larger queue and parallel processing""" + # Use a larger queue for RTX 4090 to maximize GPU utilization + max_queue_size = 32 if snac_device == "cuda" else 8 + audio_queue = queue.Queue(maxsize=max_queue_size) + + # Collect tokens in batches for higher throughput + batch_size = 16 if snac_device == "cuda" else 4 + + # Convert the synchronous token generator into an async generator with batching + async def async_token_gen(): + token_batch = [] + for token in syn_token_gen: + token_batch.append(token) + # Process in batches for efficiency + if len(token_batch) >= batch_size: + for t in token_batch: + yield t + token_batch = [] + # Process any remaining tokens + for t in token_batch: + yield t + + async def async_producer(): + # Start timer for performance logging + start_time = time.time() + chunk_count = 0 + + # Process audio chunks from the token decoder + async for audio_chunk in tokens_decoder(async_token_gen()): + audio_queue.put(audio_chunk) + chunk_count += 1 + + # Log performance stats periodically + if chunk_count % 10 == 0: + elapsed = time.time() - start_time + print(f"Generated {chunk_count} chunks in {elapsed:.2f}s ({chunk_count/elapsed:.2f} chunks/sec)") + + # Signal completion + audio_queue.put(None) # Sentinel + + def run_async(): + asyncio.run(async_producer()) + + # Use a higher priority thread for RTX 4090 to ensure it stays fed with work + thread = threading.Thread(target=run_async) + thread.daemon = True # Allow the thread to be terminated when the main thread exits + thread.start() + + # Use larger buffer for final audio assembly + buffer_size = 5 + audio_buffer = [] + + while True: + audio = audio_queue.get() + if audio is None: + break + + audio_buffer.append(audio) + # Yield buffered audio chunks for smoother playback + if len(audio_buffer) >= buffer_size: + for chunk in audio_buffer: + yield chunk + audio_buffer = [] + + # Yield any remaining audio in the buffer + for chunk in audio_buffer: + yield chunk + + thread.join()