danvics/orpheus-fastapi
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Add support for chunked streaming responses

Browse source 
Allows the inference engine to correctly yield a single token, even when the streaming response from the backend contains multiple tokens concatenated together.
This commit is contained in:
Bryce Shurts 2025-03-23 01:14:29 -05:00 committed by GitHub
parent db0ac922f0
commit f13fcef351
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
1 changed files with 8 additions and 6 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

14
tts_engine/inference.py
View file

@ -273,12 +273,14 @@ def generate_tokens_from_api(prompt: str, voice: str = DEFAULT_VOICE, temperatur
try:
data = json.loads(data_str)
if 'choices' in data and len(data['choices']) > 0:
token_text = data['choices'][0].get('text', '')
token_counter += 1
perf_monitor.add_tokens()
if token_text:
yield token_text
token_chunk = data['choices'][0].get('text', '')
for token_text in token_chunk.split('>'):
token_text = f'{token_text}>'
token_counter += 1
perf_monitor.add_tokens()
if token_text:
yield token_text
except json.JSONDecodeError as e:
print(f"Error decoding JSON: {e}")
continue