From 01ba8d57baf4e471ee8cdfbb5a71a2e1c6d9a71f Mon Sep 17 00:00:00 2001
From: Richard Roberson <me@richardr.dev>
Date: Mon, 10 Feb 2025 19:58:14 -0700
Subject: [PATCH] Update sentence processing to using max sentence blocks of
 250 chars

---
 src/utils/nlp.ts | 35 ++++++++++++++++++++---------------
 1 file changed, 20 insertions(+), 15 deletions(-)

diff --git a/src/utils/nlp.ts b/src/utils/nlp.ts
index 3eb435f..cdd9f08 100644
--- a/src/utils/nlp.ts
+++ b/src/utils/nlp.ts
@@ -15,10 +15,7 @@ export const preprocessSentenceForAudio = (text: string): string => {
     .trim();
 };
 
-const isShortSentence = (sentence: string): boolean => {
-  const words = sentence.trim().split(/\s+/);
-  return words.length <= 5;
-};
+const MAX_BLOCK_LENGTH = 250; // Maximum characters per block
 
 export const splitIntoSentences = (text: string): string[] => {
   // Preprocess the text before splitting into sentences
@@ -26,20 +23,28 @@ export const splitIntoSentences = (text: string): string[] => {
   const doc = nlp(cleanedText);
   const rawSentences = doc.sentences().out('array') as string[];
   
-  // Combine short sentences with previous ones
-  const processedSentences: string[] = [];
-  
-  for (let i = 0; i < rawSentences.length; i++) {
-    const currentSentence = rawSentences[i].trim();
+  const blocks: string[] = [];
+  let currentBlock = '';
+
+  for (const sentence of rawSentences) {
+    const trimmedSentence = sentence.trim();
     
-    if (isShortSentence(currentSentence) && processedSentences.length > 0) {
-      // Combine with previous sentence
-      const lastIndex = processedSentences.length - 1;
-      processedSentences[lastIndex] = `${processedSentences[lastIndex]} ${currentSentence}`;
+    // If adding this sentence would exceed the limit, start a new block
+    if (currentBlock && (currentBlock.length + trimmedSentence.length + 1) > MAX_BLOCK_LENGTH) {
+      blocks.push(currentBlock.trim());
+      currentBlock = trimmedSentence;
     } else {
-      processedSentences.push(currentSentence);
+      // Add to current block with a space if not empty
+      currentBlock = currentBlock 
+        ? `${currentBlock} ${trimmedSentence}`
+        : trimmedSentence;
     }
   }
+
+  // Add the last block if not empty
+  if (currentBlock) {
+    blocks.push(currentBlock.trim());
+  }
   
-  return processedSentences;
+  return blocks;
 };
\ No newline at end of file