From f13fcef351ba6f1f0d5b64924998e70e7e0725a8 Mon Sep 17 00:00:00 2001 From: Bryce Shurts Date: Sun, 23 Mar 2025 01:14:29 -0500 Subject: [PATCH] Add support for chunked streaming responses Allows the inference engine to correctly yield a single token, even when the streaming response from the backend contains multiple tokens concatenated together. --- tts_engine/inference.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/tts_engine/inference.py b/tts_engine/inference.py index fabbce5..3ec4a54 100644 --- a/tts_engine/inference.py +++ b/tts_engine/inference.py @@ -273,12 +273,14 @@ def generate_tokens_from_api(prompt: str, voice: str = DEFAULT_VOICE, temperatur try: data = json.loads(data_str) if 'choices' in data and len(data['choices']) > 0: - token_text = data['choices'][0].get('text', '') - token_counter += 1 - perf_monitor.add_tokens() - - if token_text: - yield token_text + token_chunk = data['choices'][0].get('text', '') + for token_text in token_chunk.split('>'): + token_text = f'{token_text}>' + token_counter += 1 + perf_monitor.add_tokens() + + if token_text: + yield token_text except json.JSONDecodeError as e: print(f"Error decoding JSON: {e}") continue