danvics/openreader
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Update sentence processing to using max sentence blocks of 250 chars

Browse source
This commit is contained in:
Richard Roberson 2025-02-10 19:58:14 -07:00
parent 039ac1fe61
commit 01ba8d57ba
1 changed files with 20 additions and 15 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

35
src/utils/nlp.ts
View file

@ -15,10 +15,7 @@ export const preprocessSentenceForAudio = (text: string): string => {
.trim();
};
const isShortSentence = (sentence: string): boolean => {
const words = sentence.trim().split(/\s+/);
return words.length <= 5;
};
const MAX_BLOCK_LENGTH = 250; // Maximum characters per block
export const splitIntoSentences = (text: string): string[] => {
// Preprocess the text before splitting into sentences
@ -26,20 +23,28 @@ export const splitIntoSentences = (text: string): string[] => {
const doc = nlp(cleanedText);
const rawSentences = doc.sentences().out('array') as string[];
// Combine short sentences with previous ones
const processedSentences: string[] = [];
for (let i = 0; i < rawSentences.length; i++) {
const currentSentence = rawSentences[i].trim();
const blocks: string[] = [];
let currentBlock = '';
for (const sentence of rawSentences) {
const trimmedSentence = sentence.trim();
if (isShortSentence(currentSentence) && processedSentences.length > 0) {
// Combine with previous sentence
const lastIndex = processedSentences.length - 1;
processedSentences[lastIndex] = `${processedSentences[lastIndex]} ${currentSentence}`;
// If adding this sentence would exceed the limit, start a new block
if (currentBlock && (currentBlock.length + trimmedSentence.length + 1) > MAX_BLOCK_LENGTH) {
blocks.push(currentBlock.trim());
currentBlock = trimmedSentence;
} else {
processedSentences.push(currentSentence);
// Add to current block with a space if not empty
currentBlock = currentBlock
? `${currentBlock} ${trimmedSentence}`
: trimmedSentence;
}
}
// Add the last block if not empty
if (currentBlock) {
blocks.push(currentBlock.trim());
}
return processedSentences;
return blocks;
};