Describe what you changed

app.txt

@@ -8,6 +8,11 @@ import asyncio
 from datetime import datetime
 from typing import List, Optional
 from dotenv import load_dotenv
+from fastapi.responses import StreamingResponse
+from typing import Generator
+from tts_engine.inference import generate_tokens_from_api
+from tts_engine.inference import tokens_decoder
+import aiohttp  # ✅ async requests
 
 # Function to ensure .env file exists
 def ensure_env_file_exists():
@@ -92,44 +97,37 @@ class APIResponse(BaseModel):
 # OpenAI-compatible API endpoint
 @app.post("/v1/audio/speech")
 async def create_speech_api(request: SpeechRequest):
-    """
-    Generate speech from text using the Orpheus TTS model.
-    Compatible with OpenAI's /v1/audio/speech endpoint.
-    
-    For longer texts (>1000 characters), batched generation is used
-    to improve reliability and avoid truncation issues.
-    """
     if not request.input:
         raise HTTPException(status_code=400, detail="Missing input text")
-    
+
-    # Generate unique filename
-    timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
-    output_path = f"outputs/{request.voice}_{timestamp}.wav"
-    
-    # Check if we should use batched generation
     use_batching = len(request.input) > 1000
-    if use_batching:
+
-        print(f"Using batched generation for long text ({len(request.input)} characters)")
+    # Token generator
-    
+    token_gen = generate_tokens_from_api(
-    # Generate speech with automatic batching for long texts
-    start = time.time()
-    generate_speech_from_api(
         prompt=request.input,
         voice=request.voice,
-        output_file=output_path,
+        temperature=request.speed,  # if you want speed control
-        use_batching=use_batching,
+        max_tokens=8192,
-        max_batch_chars=1000  # Process in ~1000 character chunks (roughly 1 paragraph)
+        top_p=0.9,
-    )
+        repetition_penalty=1.1
-    end = time.time()
-    generation_time = round(end - start, 2)
-    
-    # Return audio file
-    return FileResponse(
-        path=output_path,
-        media_type="audio/wav",
-        filename=f"{request.voice}_{timestamp}.wav"
     )
 
+    # Async audio generator
+    async def audio_generator():
+        async for audio_chunk in tokens_decoder(token_gen):
+            if audio_chunk:
+                yield audio_chunk
+
+    return StreamingResponse(
+        audio_generator(),
+        media_type="audio/wav",
+        headers={
+            "Content-Disposition": f'attachment; filename="{request.voice}_{int(time.time())}.wav"',
+            "Cache-Control": "no-cache",
+            "Connection": "keep-alive",
+            "Transfer-Encoding": "chunked"
+        }
+    )
 @app.get("/v1/audio/voices")
 async def list_voices():
     """Return list of available voices"""

requirements.txt

@@ -3,7 +3,7 @@ fastapi==0.103.1
 uvicorn==0.23.2
 jinja2==3.1.2
 pydantic==2.3.0
-python-multipart==0.0.6
+ 
 
 # API and Communication
 requests==2.31.0
@@ -11,7 +11,7 @@ python-dotenv==1.0.0
 watchfiles==1.0.4
 
 # Audio Processing
-numpy==1.24.0
+numpy==1.26.4
 sounddevice==0.4.6
 snac==1.2.1       # Required for audio generation from tokens
 

tts_engine/inference.txt

@@ -1,3 +1,4 @@
+
 import os
 import sys
 import requests
@@ -13,6 +14,7 @@ import asyncio
 from concurrent.futures import ThreadPoolExecutor
 from typing import List, Dict, Any, Optional, Generator, Union, Tuple
 from dotenv import load_dotenv
+import aiohttp  # ✅ async requests
 
 # Helper to detect if running in Uvicorn's reloader
 def is_reloader_process():
@@ -203,124 +205,98 @@ def format_prompt(prompt: str, voice: str = DEFAULT_VOICE) -> str:
     
     return f"{special_start}{formatted_prompt}{special_end}"
 
-def generate_tokens_from_api(prompt: str, voice: str = DEFAULT_VOICE, temperature: float = TEMPERATURE, 
+async def generate_tokens_from_api(prompt: str, voice: str = DEFAULT_VOICE, temperature: float = TEMPERATURE,
-                           top_p: float = TOP_P, max_tokens: int = MAX_TOKENS, 
+                                   top_p: float = TOP_P, max_tokens: int = MAX_TOKENS,
-                           repetition_penalty: float = REPETITION_PENALTY) -> Generator[str, None, None]:
+                                   repetition_penalty: float = REPETITION_PENALTY):
-    """Generate tokens from text using OpenAI-compatible API with optimized streaming and retry logic."""
+    """Generate tokens from text using OpenAI-compatible API with optimized streaming and retry logic (Async)."""
     start_time = time.time()
     formatted_prompt = format_prompt(prompt, voice)
     print(f"Generating speech for: {formatted_prompt}")
-    
+
-    # Optimize the token generation for GPUs
-    if HIGH_END_GPU:
-        # Use more aggressive parameters for faster generation on high-end GPUs
-        print("Using optimized parameters for high-end GPU")
-    elif torch.cuda.is_available():
-        print("Using optimized parameters for GPU acceleration")
-    
-    # Create the request payload (model field may not be required by some endpoints but included for compatibility)
     payload = {
         "prompt": formatted_prompt,
         "max_tokens": max_tokens,
         "temperature": temperature,
         "top_p": top_p,
         "repeat_penalty": repetition_penalty,
-        "stream": True  # Always stream for better performance
+        "stream": True,
+        "model": os.environ.get("ORPHEUS_MODEL_NAME", "Orpheus-3b-FT-Q8_0.gguf")
     }
-    
+
-    # Add model field - this is ignored by many local inference servers for /v1/completions
-    # but included for compatibility with OpenAI API and some servers that may use it
-    model_name = os.environ.get("ORPHEUS_MODEL_NAME", "Orpheus-3b-FT-Q8_0.gguf")
-    payload["model"] = model_name
-    
-    # Session for connection pooling and retry logic
-    session = requests.Session()
-    
     retry_count = 0
     max_retries = 3
-    
+
     while retry_count < max_retries:
         try:
-            # Make the API request with streaming and timeout
+            # Use aiohttp for async HTTP requests
-            response = session.post(
+            async with aiohttp.ClientSession() as session:
-                API_URL, 
+                async with session.post(API_URL, headers=HEADERS, json=payload, timeout=REQUEST_TIMEOUT) as response:
-                headers=HEADERS, 
+                    if response.status != 200:
-                json=payload, 
+                        print(f"Error: API request failed with status code {response.status}")
-                stream=True,
+                        text = await response.text()
-                timeout=REQUEST_TIMEOUT
+                        print(f"Error details: {text}")
-            )
+                        if response.status >= 500:
-            
+                            retry_count += 1
-            if response.status_code != 200:
+                            wait_time = 2 ** retry_count
-                print(f"Error: API request failed with status code {response.status_code}")
+                            print(f"Retrying in {wait_time} seconds...")
-                print(f"Error details: {response.text}")
+                            await asyncio.sleep(wait_time)
-                # Retry on server errors (5xx) but not on client errors (4xx)
-                if response.status_code >= 500:
-                    retry_count += 1
-                    wait_time = 2 ** retry_count  # Exponential backoff
-                    print(f"Retrying in {wait_time} seconds...")
-                    time.sleep(wait_time)
-                    continue
-                return
-            
-            # Process the streamed response with better buffering
-            buffer = ""
-            token_counter = 0
-            
-            # Iterate through the response to get tokens
-            for line in response.iter_lines():
-                if line:
-                    line_str = line.decode('utf-8')
-                    if line_str.startswith('data: '):
-                        data_str = line_str[6:]  # Remove the 'data: ' prefix
-                        
-                        if data_str.strip() == '[DONE]':
-                            break
-                            
-                        try:
-                            data = json.loads(data_str)
-                            if 'choices' in data and len(data['choices']) > 0:
-                                token_chunk = data['choices'][0].get('text', '')
-                                for token_text in token_chunk.split('>'):
-                                    token_text = f'{token_text}>'
-                                    token_counter += 1
-                                    perf_monitor.add_tokens()
-
-                                    if token_text:
-                                        yield token_text
-                        except json.JSONDecodeError as e:
-                            print(f"Error decoding JSON: {e}")
                             continue
-            
+                        return
-            # Generation completed successfully
+
-            generation_time = time.time() - start_time
+                    token_counter = 0
-            tokens_per_second = token_counter / generation_time if generation_time > 0 else 0
+
-            print(f"Token generation complete: {token_counter} tokens in {generation_time:.2f}s ({tokens_per_second:.1f} tokens/sec)")
+                    # Process streaming response
-            return
+                    async for line_bytes in response.content:
-            
+                        line = line_bytes.decode('utf-8').strip()
-        except requests.exceptions.Timeout:
+
+                        if line.startswith('data: '):
+                            data_str = line[6:].strip()
+
+                            if data_str == '[DONE]':
+                                break
+
+                            try:
+                                data = json.loads(data_str)
+                                if 'choices' in data and len(data['choices']) > 0:
+                                    token_chunk = data['choices'][0].get('text', '')
+                                    for token_text in token_chunk.split('>'):
+                                        token_text = f'{token_text}>'
+                                        token_counter += 1
+                                        perf_monitor.add_tokens()
+
+                                        if token_text.strip():
+                                            yield token_text
+                            except json.JSONDecodeError as e:
+                                print(f"Error decoding JSON: {e}")
+                                continue
+
+                    generation_time = time.time() - start_time
+                    tokens_per_second = token_counter / generation_time if generation_time > 0 else 0
+                    print(f"Token generation complete: {token_counter} tokens in {generation_time:.2f}s ({tokens_per_second:.1f} tokens/sec)")
+                    return
+
+        except asyncio.TimeoutError:
             print(f"Request timed out after {REQUEST_TIMEOUT} seconds")
             retry_count += 1
             if retry_count < max_retries:
                 wait_time = 2 ** retry_count
-                print(f"Retrying in {wait_time} seconds... (attempt {retry_count+1}/{max_retries})")
+                print(f"Retrying in {wait_time} seconds... (attempt {retry_count + 1}/{max_retries})")
-                time.sleep(wait_time)
+                await asyncio.sleep(wait_time)
-            else:
+            else:
-                print("Max retries reached. Token generation failed.")
+                print("Max retries reached. Token generation failed.")
-                return
+                return
-                
+
-        except requests.exceptions.ConnectionError:
+        except aiohttp.ClientConnectionError:
-            print(f"Connection error to API at {API_URL}")
+            print(f"Connection error to API at {API_URL}")
-            retry_count += 1
+            retry_count += 1
-            if retry_count < max_retries:
+            if retry_count < max_retries:
-                wait_time = 2 ** retry_count
+                wait_time = 2 ** retry_count
-                print(f"Retrying in {wait_time} seconds... (attempt {retry_count+1}/{max_retries})")
+                print(f"Retrying in {wait_time} seconds... (attempt {retry_count + 1}/{max_retries})")
-                time.sleep(wait_time)
+                await asyncio.sleep(wait_time)
             else:
                 print("Max retries reached. Token generation failed.")
                 return
 
-# The turn_token_into_id function is now imported from speechpipe.py
-# This eliminates duplicate code and ensures consistent behavior
 
 def convert_to_audio(multiframe: List[int], count: int) -> Optional[bytes]:
     """Convert token frames to audio with performance monitoring."""
@@ -895,4 +871,4 @@ def main():
     print(f"Audio saved to {output_file}")
 
 if __name__ == "__main__":
-    main()
+    main()

vosk-model-small-en-us-0.15/README

@@ -0,0 +1,9 @@
+US English model for mobile Vosk applications
+
+Copyright 2020 Alpha Cephei Inc
+
+Accuracy: 10.38 (tedlium test) 9.85 (librispeech test-clean)
+Speed: 0.11xRT (desktop)
+Latency: 0.15s (right context)
+
+

vosk-model-small-en-us-0.15/conf/mfcc.conf

@@ -0,0 +1,7 @@
+--sample-frequency=16000
+--use-energy=false
+--num-mel-bins=40
+--num-ceps=40
+--low-freq=20
+--high-freq=7600
+--allow-downsample=true

vosk-model-small-en-us-0.15/conf/model.conf

@@ -0,0 +1,10 @@
+--min-active=200
+--max-active=3000
+--beam=10.0
+--lattice-beam=2.0
+--acoustic-scale=1.0
+--frame-subsampling-factor=3
+--endpoint.silence-phones=1:2:3:4:5:6:7:8:9:10
+--endpoint.rule2.min-trailing-silence=0.5
+--endpoint.rule3.min-trailing-silence=0.75
+--endpoint.rule4.min-trailing-silence=1.0

vosk-model-small-en-us-0.15/graph/disambig_tid.int

@@ -0,0 +1,17 @@
+10015
+10016
+10017
+10018
+10019
+10020
+10021
+10022
+10023
+10024
+10025
+10026
+10027
+10028
+10029
+10030
+10031

vosk-model-small-en-us-0.15/graph/phones/word_boundary.int

@@ -0,0 +1,166 @@
+1 nonword
+2 begin
+3 end
+4 internal
+5 singleton
+6 nonword
+7 begin
+8 end
+9 internal
+10 singleton
+11 begin
+12 end
+13 internal
+14 singleton
+15 begin
+16 end
+17 internal
+18 singleton
+19 begin
+20 end
+21 internal
+22 singleton
+23 begin
+24 end
+25 internal
+26 singleton
+27 begin
+28 end
+29 internal
+30 singleton
+31 begin
+32 end
+33 internal
+34 singleton
+35 begin
+36 end
+37 internal
+38 singleton
+39 begin
+40 end
+41 internal
+42 singleton
+43 begin
+44 end
+45 internal
+46 singleton
+47 begin
+48 end
+49 internal
+50 singleton
+51 begin
+52 end
+53 internal
+54 singleton
+55 begin
+56 end
+57 internal
+58 singleton
+59 begin
+60 end
+61 internal
+62 singleton
+63 begin
+64 end
+65 internal
+66 singleton
+67 begin
+68 end
+69 internal
+70 singleton
+71 begin
+72 end
+73 internal
+74 singleton
+75 begin
+76 end
+77 internal
+78 singleton
+79 begin
+80 end
+81 internal
+82 singleton
+83 begin
+84 end
+85 internal
+86 singleton
+87 begin
+88 end
+89 internal
+90 singleton
+91 begin
+92 end
+93 internal
+94 singleton
+95 begin
+96 end
+97 internal
+98 singleton
+99 begin
+100 end
+101 internal
+102 singleton
+103 begin
+104 end
+105 internal
+106 singleton
+107 begin
+108 end
+109 internal
+110 singleton
+111 begin
+112 end
+113 internal
+114 singleton
+115 begin
+116 end
+117 internal
+118 singleton
+119 begin
+120 end
+121 internal
+122 singleton
+123 begin
+124 end
+125 internal
+126 singleton
+127 begin
+128 end
+129 internal
+130 singleton
+131 begin
+132 end
+133 internal
+134 singleton
+135 begin
+136 end
+137 internal
+138 singleton
+139 begin
+140 end
+141 internal
+142 singleton
+143 begin
+144 end
+145 internal
+146 singleton
+147 begin
+148 end
+149 internal
+150 singleton
+151 begin
+152 end
+153 internal
+154 singleton
+155 begin
+156 end
+157 internal
+158 singleton
+159 begin
+160 end
+161 internal
+162 singleton
+163 begin
+164 end
+165 internal
+166 singleton

vosk-model-small-en-us-0.15/ivector/global_cmvn.stats

@@ -0,0 +1,3 @@
+ [
+  1.682383e+11 -1.1595e+10 -1.521733e+10 4.32034e+09 -2.257938e+10 -1.969666e+10 -2.559265e+10 -1.535687e+10 -1.276854e+10 -4.494483e+09 -1.209085e+10 -5.64008e+09 -1.134847e+10 -3.419512e+09 -1.079542e+10 -4.145463e+09 -6.637486e+09 -1.11318e+09 -3.479773e+09 -1.245932e+08 -1.386961e+09 6.560655e+07 -2.436518e+08 -4.032432e+07 4.620046e+08 -7.714964e+07 9.551484e+08 -4.119761e+08 8.208582e+08 -7.117156e+08 7.457703e+08 -4.3106e+08 1.202726e+09 2.904036e+08 1.231931e+09 3.629848e+08 6.366939e+08 -4.586172e+08 -5.267629e+08 -3.507819e+08 1.679838e+09 
+  1.741141e+13 8.92488e+11 8.743834e+11 8.848896e+11 1.190313e+12 1.160279e+12 1.300066e+12 1.005678e+12 9.39335e+11 8.089614e+11 7.927041e+11 6.882427e+11 6.444235e+11 5.151451e+11 4.825723e+11 3.210106e+11 2.720254e+11 1.772539e+11 1.248102e+11 6.691599e+10 3.599804e+10 1.207574e+10 1.679301e+09 4.594778e+08 5.821614e+09 1.451758e+10 2.55803e+10 3.43277e+10 4.245286e+10 4.784859e+10 4.988591e+10 4.925451e+10 5.074584e+10 4.9557e+10 4.407876e+10 3.421443e+10 3.138606e+10 2.539716e+10 1.948134e+10 1.381167e+10 0 ]

vosk-model-small-en-us-0.15/ivector/online_cmvn.conf

@@ -0,0 +1 @@
+# configuration file for apply-cmvn-online, used in the script ../local/run_online_decoding.sh

vosk-model-small-en-us-0.15/ivector/splice.conf

@@ -0,0 +1,2 @@
+--left-context=3
+--right-context=3