diff --git a/app.py b/app.txt similarity index 92% rename from app.py rename to app.txt index 3bb7c61..b30bfbc 100644 --- a/app.py +++ b/app.txt @@ -8,6 +8,11 @@ import asyncio from datetime import datetime from typing import List, Optional from dotenv import load_dotenv +from fastapi.responses import StreamingResponse +from typing import Generator +from tts_engine.inference import generate_tokens_from_api +from tts_engine.inference import tokens_decoder +import aiohttp # ✅ async requests # Function to ensure .env file exists def ensure_env_file_exists(): @@ -92,44 +97,37 @@ class APIResponse(BaseModel): # OpenAI-compatible API endpoint @app.post("/v1/audio/speech") async def create_speech_api(request: SpeechRequest): - """ - Generate speech from text using the Orpheus TTS model. - Compatible with OpenAI's /v1/audio/speech endpoint. - - For longer texts (>1000 characters), batched generation is used - to improve reliability and avoid truncation issues. - """ if not request.input: raise HTTPException(status_code=400, detail="Missing input text") - - # Generate unique filename - timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") - output_path = f"outputs/{request.voice}_{timestamp}.wav" - - # Check if we should use batched generation + use_batching = len(request.input) > 1000 - if use_batching: - print(f"Using batched generation for long text ({len(request.input)} characters)") - - # Generate speech with automatic batching for long texts - start = time.time() - generate_speech_from_api( + + # Token generator + token_gen = generate_tokens_from_api( prompt=request.input, voice=request.voice, - output_file=output_path, - use_batching=use_batching, - max_batch_chars=1000 # Process in ~1000 character chunks (roughly 1 paragraph) - ) - end = time.time() - generation_time = round(end - start, 2) - - # Return audio file - return FileResponse( - path=output_path, - media_type="audio/wav", - filename=f"{request.voice}_{timestamp}.wav" + temperature=request.speed, # if you want speed control + max_tokens=8192, + top_p=0.9, + repetition_penalty=1.1 ) + # Async audio generator + async def audio_generator(): + async for audio_chunk in tokens_decoder(token_gen): + if audio_chunk: + yield audio_chunk + + return StreamingResponse( + audio_generator(), + media_type="audio/wav", + headers={ + "Content-Disposition": f'attachment; filename="{request.voice}_{int(time.time())}.wav"', + "Cache-Control": "no-cache", + "Connection": "keep-alive", + "Transfer-Encoding": "chunked" + } + ) @app.get("/v1/audio/voices") async def list_voices(): """Return list of available voices""" diff --git a/outputs/tara_20250414_161458.wav b/outputs/tara_20250414_161458.wav new file mode 100644 index 0000000..862467b Binary files /dev/null and b/outputs/tara_20250414_161458.wav differ diff --git a/outputs/tara_20250414_161509.wav b/outputs/tara_20250414_161509.wav new file mode 100644 index 0000000..b53c393 Binary files /dev/null and b/outputs/tara_20250414_161509.wav differ diff --git a/outputs/tara_20250414_161523.wav b/outputs/tara_20250414_161523.wav new file mode 100644 index 0000000..371b131 Binary files /dev/null and b/outputs/tara_20250414_161523.wav differ diff --git a/outputs/tara_20250414_161543.wav b/outputs/tara_20250414_161543.wav new file mode 100644 index 0000000..16413cd Binary files /dev/null and b/outputs/tara_20250414_161543.wav differ diff --git a/outputs/tara_20250414_161551.wav b/outputs/tara_20250414_161551.wav new file mode 100644 index 0000000..c687b57 Binary files /dev/null and b/outputs/tara_20250414_161551.wav differ diff --git a/outputs/tara_20250414_161608.wav b/outputs/tara_20250414_161608.wav new file mode 100644 index 0000000..74feeea Binary files /dev/null and b/outputs/tara_20250414_161608.wav differ diff --git a/outputs/tara_20250414_161630.wav b/outputs/tara_20250414_161630.wav new file mode 100644 index 0000000..806447e Binary files /dev/null and b/outputs/tara_20250414_161630.wav differ diff --git a/outputs/tara_20250414_161732.wav b/outputs/tara_20250414_161732.wav new file mode 100644 index 0000000..da92094 Binary files /dev/null and b/outputs/tara_20250414_161732.wav differ diff --git a/outputs/tara_20250414_162046.wav b/outputs/tara_20250414_162046.wav new file mode 100644 index 0000000..31cbbee Binary files /dev/null and b/outputs/tara_20250414_162046.wav differ diff --git a/outputs/tara_20250414_162352.wav b/outputs/tara_20250414_162352.wav new file mode 100644 index 0000000..be51ba9 Binary files /dev/null and b/outputs/tara_20250414_162352.wav differ diff --git a/outputs/tara_20250414_162421.wav b/outputs/tara_20250414_162421.wav new file mode 100644 index 0000000..ac12669 Binary files /dev/null and b/outputs/tara_20250414_162421.wav differ diff --git a/outputs/tara_20250414_162454.wav b/outputs/tara_20250414_162454.wav new file mode 100644 index 0000000..b7483bc Binary files /dev/null and b/outputs/tara_20250414_162454.wav differ diff --git a/outputs/tara_20250414_162531.wav b/outputs/tara_20250414_162531.wav new file mode 100644 index 0000000..b9f47f2 Binary files /dev/null and b/outputs/tara_20250414_162531.wav differ diff --git a/outputs/tara_20250414_162615.wav b/outputs/tara_20250414_162615.wav new file mode 100644 index 0000000..6b6b142 Binary files /dev/null and b/outputs/tara_20250414_162615.wav differ diff --git a/outputs/tara_20250414_162715.wav b/outputs/tara_20250414_162715.wav new file mode 100644 index 0000000..bc0f75a Binary files /dev/null and b/outputs/tara_20250414_162715.wav differ diff --git a/outputs/tara_20250414_162741.wav b/outputs/tara_20250414_162741.wav new file mode 100644 index 0000000..33f431c Binary files /dev/null and b/outputs/tara_20250414_162741.wav differ diff --git a/outputs/tara_20250414_162918.wav b/outputs/tara_20250414_162918.wav new file mode 100644 index 0000000..89df19f Binary files /dev/null and b/outputs/tara_20250414_162918.wav differ diff --git a/outputs/tara_20250414_163002.wav b/outputs/tara_20250414_163002.wav new file mode 100644 index 0000000..616df69 Binary files /dev/null and b/outputs/tara_20250414_163002.wav differ diff --git a/outputs/tara_20250414_163034.wav b/outputs/tara_20250414_163034.wav new file mode 100644 index 0000000..8d176db Binary files /dev/null and b/outputs/tara_20250414_163034.wav differ diff --git a/outputs/tara_20250414_163109.wav b/outputs/tara_20250414_163109.wav new file mode 100644 index 0000000..0c53793 Binary files /dev/null and b/outputs/tara_20250414_163109.wav differ diff --git a/outputs/tara_20250414_163230.wav b/outputs/tara_20250414_163230.wav new file mode 100644 index 0000000..1a4e1ee Binary files /dev/null and b/outputs/tara_20250414_163230.wav differ diff --git a/requirements.txt b/requirements.txt index 6618feb..ce07063 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,7 +3,7 @@ fastapi==0.103.1 uvicorn==0.23.2 jinja2==3.1.2 pydantic==2.3.0 -python-multipart==0.0.6 + # API and Communication requests==2.31.0 @@ -11,7 +11,7 @@ python-dotenv==1.0.0 watchfiles==1.0.4 # Audio Processing -numpy==1.24.0 +numpy==1.26.4 sounddevice==0.4.6 snac==1.2.1 # Required for audio generation from tokens diff --git a/tts_engine/inference.py b/tts_engine/inference.txt similarity index 88% rename from tts_engine/inference.py rename to tts_engine/inference.txt index 7e1e501..0fda8cb 100644 --- a/tts_engine/inference.py +++ b/tts_engine/inference.txt @@ -1,3 +1,4 @@ + import os import sys import requests @@ -13,6 +14,7 @@ import asyncio from concurrent.futures import ThreadPoolExecutor from typing import List, Dict, Any, Optional, Generator, Union, Tuple from dotenv import load_dotenv +import aiohttp # ✅ async requests # Helper to detect if running in Uvicorn's reloader def is_reloader_process(): @@ -203,124 +205,98 @@ def format_prompt(prompt: str, voice: str = DEFAULT_VOICE) -> str: return f"{special_start}{formatted_prompt}{special_end}" -def generate_tokens_from_api(prompt: str, voice: str = DEFAULT_VOICE, temperature: float = TEMPERATURE, - top_p: float = TOP_P, max_tokens: int = MAX_TOKENS, - repetition_penalty: float = REPETITION_PENALTY) -> Generator[str, None, None]: - """Generate tokens from text using OpenAI-compatible API with optimized streaming and retry logic.""" +async def generate_tokens_from_api(prompt: str, voice: str = DEFAULT_VOICE, temperature: float = TEMPERATURE, + top_p: float = TOP_P, max_tokens: int = MAX_TOKENS, + repetition_penalty: float = REPETITION_PENALTY): + """Generate tokens from text using OpenAI-compatible API with optimized streaming and retry logic (Async).""" start_time = time.time() formatted_prompt = format_prompt(prompt, voice) print(f"Generating speech for: {formatted_prompt}") - - # Optimize the token generation for GPUs - if HIGH_END_GPU: - # Use more aggressive parameters for faster generation on high-end GPUs - print("Using optimized parameters for high-end GPU") - elif torch.cuda.is_available(): - print("Using optimized parameters for GPU acceleration") - - # Create the request payload (model field may not be required by some endpoints but included for compatibility) + payload = { "prompt": formatted_prompt, "max_tokens": max_tokens, "temperature": temperature, "top_p": top_p, "repeat_penalty": repetition_penalty, - "stream": True # Always stream for better performance + "stream": True, + "model": os.environ.get("ORPHEUS_MODEL_NAME", "Orpheus-3b-FT-Q8_0.gguf") } - - # Add model field - this is ignored by many local inference servers for /v1/completions - # but included for compatibility with OpenAI API and some servers that may use it - model_name = os.environ.get("ORPHEUS_MODEL_NAME", "Orpheus-3b-FT-Q8_0.gguf") - payload["model"] = model_name - - # Session for connection pooling and retry logic - session = requests.Session() - + retry_count = 0 max_retries = 3 - + while retry_count < max_retries: try: - # Make the API request with streaming and timeout - response = session.post( - API_URL, - headers=HEADERS, - json=payload, - stream=True, - timeout=REQUEST_TIMEOUT - ) - - if response.status_code != 200: - print(f"Error: API request failed with status code {response.status_code}") - print(f"Error details: {response.text}") - # Retry on server errors (5xx) but not on client errors (4xx) - if response.status_code >= 500: - retry_count += 1 - wait_time = 2 ** retry_count # Exponential backoff - print(f"Retrying in {wait_time} seconds...") - time.sleep(wait_time) - continue - return - - # Process the streamed response with better buffering - buffer = "" - token_counter = 0 - - # Iterate through the response to get tokens - for line in response.iter_lines(): - if line: - line_str = line.decode('utf-8') - if line_str.startswith('data: '): - data_str = line_str[6:] # Remove the 'data: ' prefix - - if data_str.strip() == '[DONE]': - break - - try: - data = json.loads(data_str) - if 'choices' in data and len(data['choices']) > 0: - token_chunk = data['choices'][0].get('text', '') - for token_text in token_chunk.split('>'): - token_text = f'{token_text}>' - token_counter += 1 - perf_monitor.add_tokens() - - if token_text: - yield token_text - except json.JSONDecodeError as e: - print(f"Error decoding JSON: {e}") + # Use aiohttp for async HTTP requests + async with aiohttp.ClientSession() as session: + async with session.post(API_URL, headers=HEADERS, json=payload, timeout=REQUEST_TIMEOUT) as response: + if response.status != 200: + print(f"Error: API request failed with status code {response.status}") + text = await response.text() + print(f"Error details: {text}") + if response.status >= 500: + retry_count += 1 + wait_time = 2 ** retry_count + print(f"Retrying in {wait_time} seconds...") + await asyncio.sleep(wait_time) continue - - # Generation completed successfully - generation_time = time.time() - start_time - tokens_per_second = token_counter / generation_time if generation_time > 0 else 0 - print(f"Token generation complete: {token_counter} tokens in {generation_time:.2f}s ({tokens_per_second:.1f} tokens/sec)") - return - - except requests.exceptions.Timeout: + return + + token_counter = 0 + + # Process streaming response + async for line_bytes in response.content: + line = line_bytes.decode('utf-8').strip() + + if line.startswith('data: '): + data_str = line[6:].strip() + + if data_str == '[DONE]': + break + + try: + data = json.loads(data_str) + if 'choices' in data and len(data['choices']) > 0: + token_chunk = data['choices'][0].get('text', '') + for token_text in token_chunk.split('>'): + token_text = f'{token_text}>' + token_counter += 1 + perf_monitor.add_tokens() + + if token_text.strip(): + yield token_text + except json.JSONDecodeError as e: + print(f"Error decoding JSON: {e}") + continue + + generation_time = time.time() - start_time + tokens_per_second = token_counter / generation_time if generation_time > 0 else 0 + print(f"Token generation complete: {token_counter} tokens in {generation_time:.2f}s ({tokens_per_second:.1f} tokens/sec)") + return + + except asyncio.TimeoutError: print(f"Request timed out after {REQUEST_TIMEOUT} seconds") retry_count += 1 if retry_count < max_retries: wait_time = 2 ** retry_count - print(f"Retrying in {wait_time} seconds... (attempt {retry_count+1}/{max_retries})") - time.sleep(wait_time) - else: - print("Max retries reached. Token generation failed.") - return - - except requests.exceptions.ConnectionError: - print(f"Connection error to API at {API_URL}") - retry_count += 1 - if retry_count < max_retries: - wait_time = 2 ** retry_count - print(f"Retrying in {wait_time} seconds... (attempt {retry_count+1}/{max_retries})") - time.sleep(wait_time) + print(f"Retrying in {wait_time} seconds... (attempt {retry_count + 1}/{max_retries})") + await asyncio.sleep(wait_time) + else: + print("Max retries reached. Token generation failed.") + return + + except aiohttp.ClientConnectionError: + print(f"Connection error to API at {API_URL}") + retry_count += 1 + if retry_count < max_retries: + wait_time = 2 ** retry_count + print(f"Retrying in {wait_time} seconds... (attempt {retry_count + 1}/{max_retries})") + await asyncio.sleep(wait_time) else: print("Max retries reached. Token generation failed.") return -# The turn_token_into_id function is now imported from speechpipe.py -# This eliminates duplicate code and ensures consistent behavior def convert_to_audio(multiframe: List[int], count: int) -> Optional[bytes]: """Convert token frames to audio with performance monitoring.""" @@ -895,4 +871,4 @@ def main(): print(f"Audio saved to {output_file}") if __name__ == "__main__": - main() + main() \ No newline at end of file diff --git a/vosk-model-small-en-us-0.15/README b/vosk-model-small-en-us-0.15/README new file mode 100644 index 0000000..a7f7931 --- /dev/null +++ b/vosk-model-small-en-us-0.15/README @@ -0,0 +1,9 @@ +US English model for mobile Vosk applications + +Copyright 2020 Alpha Cephei Inc + +Accuracy: 10.38 (tedlium test) 9.85 (librispeech test-clean) +Speed: 0.11xRT (desktop) +Latency: 0.15s (right context) + + diff --git a/vosk-model-small-en-us-0.15/am/final.mdl b/vosk-model-small-en-us-0.15/am/final.mdl new file mode 100644 index 0000000..5596b31 Binary files /dev/null and b/vosk-model-small-en-us-0.15/am/final.mdl differ diff --git a/vosk-model-small-en-us-0.15/conf/mfcc.conf b/vosk-model-small-en-us-0.15/conf/mfcc.conf new file mode 100644 index 0000000..eaa40c5 --- /dev/null +++ b/vosk-model-small-en-us-0.15/conf/mfcc.conf @@ -0,0 +1,7 @@ +--sample-frequency=16000 +--use-energy=false +--num-mel-bins=40 +--num-ceps=40 +--low-freq=20 +--high-freq=7600 +--allow-downsample=true diff --git a/vosk-model-small-en-us-0.15/conf/model.conf b/vosk-model-small-en-us-0.15/conf/model.conf new file mode 100644 index 0000000..9d5b0da --- /dev/null +++ b/vosk-model-small-en-us-0.15/conf/model.conf @@ -0,0 +1,10 @@ +--min-active=200 +--max-active=3000 +--beam=10.0 +--lattice-beam=2.0 +--acoustic-scale=1.0 +--frame-subsampling-factor=3 +--endpoint.silence-phones=1:2:3:4:5:6:7:8:9:10 +--endpoint.rule2.min-trailing-silence=0.5 +--endpoint.rule3.min-trailing-silence=0.75 +--endpoint.rule4.min-trailing-silence=1.0 diff --git a/vosk-model-small-en-us-0.15/graph/Gr.fst b/vosk-model-small-en-us-0.15/graph/Gr.fst new file mode 100644 index 0000000..1f292e6 Binary files /dev/null and b/vosk-model-small-en-us-0.15/graph/Gr.fst differ diff --git a/vosk-model-small-en-us-0.15/graph/HCLr.fst b/vosk-model-small-en-us-0.15/graph/HCLr.fst new file mode 100644 index 0000000..9797b26 Binary files /dev/null and b/vosk-model-small-en-us-0.15/graph/HCLr.fst differ diff --git a/vosk-model-small-en-us-0.15/graph/disambig_tid.int b/vosk-model-small-en-us-0.15/graph/disambig_tid.int new file mode 100644 index 0000000..762fd5f --- /dev/null +++ b/vosk-model-small-en-us-0.15/graph/disambig_tid.int @@ -0,0 +1,17 @@ +10015 +10016 +10017 +10018 +10019 +10020 +10021 +10022 +10023 +10024 +10025 +10026 +10027 +10028 +10029 +10030 +10031 diff --git a/vosk-model-small-en-us-0.15/graph/phones/word_boundary.int b/vosk-model-small-en-us-0.15/graph/phones/word_boundary.int new file mode 100644 index 0000000..df23fd7 --- /dev/null +++ b/vosk-model-small-en-us-0.15/graph/phones/word_boundary.int @@ -0,0 +1,166 @@ +1 nonword +2 begin +3 end +4 internal +5 singleton +6 nonword +7 begin +8 end +9 internal +10 singleton +11 begin +12 end +13 internal +14 singleton +15 begin +16 end +17 internal +18 singleton +19 begin +20 end +21 internal +22 singleton +23 begin +24 end +25 internal +26 singleton +27 begin +28 end +29 internal +30 singleton +31 begin +32 end +33 internal +34 singleton +35 begin +36 end +37 internal +38 singleton +39 begin +40 end +41 internal +42 singleton +43 begin +44 end +45 internal +46 singleton +47 begin +48 end +49 internal +50 singleton +51 begin +52 end +53 internal +54 singleton +55 begin +56 end +57 internal +58 singleton +59 begin +60 end +61 internal +62 singleton +63 begin +64 end +65 internal +66 singleton +67 begin +68 end +69 internal +70 singleton +71 begin +72 end +73 internal +74 singleton +75 begin +76 end +77 internal +78 singleton +79 begin +80 end +81 internal +82 singleton +83 begin +84 end +85 internal +86 singleton +87 begin +88 end +89 internal +90 singleton +91 begin +92 end +93 internal +94 singleton +95 begin +96 end +97 internal +98 singleton +99 begin +100 end +101 internal +102 singleton +103 begin +104 end +105 internal +106 singleton +107 begin +108 end +109 internal +110 singleton +111 begin +112 end +113 internal +114 singleton +115 begin +116 end +117 internal +118 singleton +119 begin +120 end +121 internal +122 singleton +123 begin +124 end +125 internal +126 singleton +127 begin +128 end +129 internal +130 singleton +131 begin +132 end +133 internal +134 singleton +135 begin +136 end +137 internal +138 singleton +139 begin +140 end +141 internal +142 singleton +143 begin +144 end +145 internal +146 singleton +147 begin +148 end +149 internal +150 singleton +151 begin +152 end +153 internal +154 singleton +155 begin +156 end +157 internal +158 singleton +159 begin +160 end +161 internal +162 singleton +163 begin +164 end +165 internal +166 singleton diff --git a/vosk-model-small-en-us-0.15/ivector/final.dubm b/vosk-model-small-en-us-0.15/ivector/final.dubm new file mode 100644 index 0000000..db789eb Binary files /dev/null and b/vosk-model-small-en-us-0.15/ivector/final.dubm differ diff --git a/vosk-model-small-en-us-0.15/ivector/final.ie b/vosk-model-small-en-us-0.15/ivector/final.ie new file mode 100644 index 0000000..93737bf Binary files /dev/null and b/vosk-model-small-en-us-0.15/ivector/final.ie differ diff --git a/vosk-model-small-en-us-0.15/ivector/final.mat b/vosk-model-small-en-us-0.15/ivector/final.mat new file mode 100644 index 0000000..c3ec635 Binary files /dev/null and b/vosk-model-small-en-us-0.15/ivector/final.mat differ diff --git a/vosk-model-small-en-us-0.15/ivector/global_cmvn.stats b/vosk-model-small-en-us-0.15/ivector/global_cmvn.stats new file mode 100644 index 0000000..b9d92ef --- /dev/null +++ b/vosk-model-small-en-us-0.15/ivector/global_cmvn.stats @@ -0,0 +1,3 @@ + [ + 1.682383e+11 -1.1595e+10 -1.521733e+10 4.32034e+09 -2.257938e+10 -1.969666e+10 -2.559265e+10 -1.535687e+10 -1.276854e+10 -4.494483e+09 -1.209085e+10 -5.64008e+09 -1.134847e+10 -3.419512e+09 -1.079542e+10 -4.145463e+09 -6.637486e+09 -1.11318e+09 -3.479773e+09 -1.245932e+08 -1.386961e+09 6.560655e+07 -2.436518e+08 -4.032432e+07 4.620046e+08 -7.714964e+07 9.551484e+08 -4.119761e+08 8.208582e+08 -7.117156e+08 7.457703e+08 -4.3106e+08 1.202726e+09 2.904036e+08 1.231931e+09 3.629848e+08 6.366939e+08 -4.586172e+08 -5.267629e+08 -3.507819e+08 1.679838e+09 + 1.741141e+13 8.92488e+11 8.743834e+11 8.848896e+11 1.190313e+12 1.160279e+12 1.300066e+12 1.005678e+12 9.39335e+11 8.089614e+11 7.927041e+11 6.882427e+11 6.444235e+11 5.151451e+11 4.825723e+11 3.210106e+11 2.720254e+11 1.772539e+11 1.248102e+11 6.691599e+10 3.599804e+10 1.207574e+10 1.679301e+09 4.594778e+08 5.821614e+09 1.451758e+10 2.55803e+10 3.43277e+10 4.245286e+10 4.784859e+10 4.988591e+10 4.925451e+10 5.074584e+10 4.9557e+10 4.407876e+10 3.421443e+10 3.138606e+10 2.539716e+10 1.948134e+10 1.381167e+10 0 ] diff --git a/vosk-model-small-en-us-0.15/ivector/online_cmvn.conf b/vosk-model-small-en-us-0.15/ivector/online_cmvn.conf new file mode 100644 index 0000000..7748a4a --- /dev/null +++ b/vosk-model-small-en-us-0.15/ivector/online_cmvn.conf @@ -0,0 +1 @@ +# configuration file for apply-cmvn-online, used in the script ../local/run_online_decoding.sh diff --git a/vosk-model-small-en-us-0.15/ivector/splice.conf b/vosk-model-small-en-us-0.15/ivector/splice.conf new file mode 100644 index 0000000..960cd2e --- /dev/null +++ b/vosk-model-small-en-us-0.15/ivector/splice.conf @@ -0,0 +1,2 @@ +--left-context=3 +--right-context=3