danvics/openreader
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Update sentence processing to using max sentence blocks of 250 chars

Browse source
This commit is contained in:
Richard Roberson 2025-02-10 19:58:14 -07:00
parent 039ac1fe61
commit 01ba8d57ba
1 changed files with 20 additions and 15 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

33
src/utils/nlp.ts
View file

@ -15,10 +15,7 @@ export const preprocessSentenceForAudio = (text: string): string => {
.trim(); .trim();
}; };
const isShortSentence = (sentence: string): boolean => { const MAX_BLOCK_LENGTH = 250; // Maximum characters per block
const words = sentence.trim().split(/\s+/);
return words.length <= 5;
};
export const splitIntoSentences = (text: string): string[] => { export const splitIntoSentences = (text: string): string[] => {
// Preprocess the text before splitting into sentences // Preprocess the text before splitting into sentences
@ -26,20 +23,28 @@ export const splitIntoSentences = (text: string): string[] => {
const doc = nlp(cleanedText); const doc = nlp(cleanedText);
const rawSentences = doc.sentences().out('array') as string[]; const rawSentences = doc.sentences().out('array') as string[];
// Combine short sentences with previous ones const blocks: string[] = [];
const processedSentences: string[] = []; let currentBlock = '';
for (let i = 0; i < rawSentences.length; i++) { for (const sentence of rawSentences) {
const currentSentence = rawSentences[i].trim(); const trimmedSentence = sentence.trim();
if (isShortSentence(currentSentence) && processedSentences.length > 0) { // If adding this sentence would exceed the limit, start a new block
// Combine with previous sentence if (currentBlock && (currentBlock.length + trimmedSentence.length + 1) > MAX_BLOCK_LENGTH) {
const lastIndex = processedSentences.length - 1; blocks.push(currentBlock.trim());
processedSentences[lastIndex] = `${processedSentences[lastIndex]} ${currentSentence}`; currentBlock = trimmedSentence;
} else { } else {
processedSentences.push(currentSentence); // Add to current block with a space if not empty
currentBlock = currentBlock
? `${currentBlock} ${trimmedSentence}`
: trimmedSentence;
} }
} }
return processedSentences; // Add the last block if not empty
if (currentBlock) {
blocks.push(currentBlock.trim());
}
return blocks;
}; };