feat(text): add chapter detection and text splitting utility
Add comprehensive chapter detection for large text files: - Detects common chapter markers (Chapter 1, CHAPTER I, Part 1, etc.) - Splits large files into manageable chunks (~50KB each) - Smart paragraph-boundary splitting to avoid mid-sentence breaks - Handles files with no chapter markers by creating sections - Recursive splitting for oversized chapters Chapter detection patterns: - "Chapter 1" / "CHAPTER I" (case-insensitive) - "Ch. 1" - "1. Title" (numbered sections) - "Part 1" / "PART I" - "Section 1" Features: - detectChapters(): Full chapter detection and splitting - estimateChapterCount(): Preview chapter count before processing - Configurable max chunk size (default 50KB) - Preserves paragraph formatting - Generates descriptive chapter titles This utility will be integrated with: - HTMLContext for paginated text viewing - TTS auto-advance through chapters - Audiobook generation with chapter markers Files changed: - lib/chapterDetection.ts: New chapter detection utility Next steps: - Integrate with HTMLContext for pagination - Add chapter navigation UI - Implement TTS auto-advance
This commit is contained in:
parent
1015d7de75
commit
55e35200ed
1 changed files with 142 additions and 0 deletions
142
src/lib/chapterDetection.ts
Normal file
142
src/lib/chapterDetection.ts
Normal file
|
|
@ -0,0 +1,142 @@
|
|||
/**
|
||||
* Utility functions for detecting and splitting text into chapters
|
||||
*/
|
||||
|
||||
export interface Chapter {
|
||||
title: string;
|
||||
startIndex: number;
|
||||
endIndex: number;
|
||||
text: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Detects chapters in text content
|
||||
* Looks for common chapter markers like "Chapter 1", "CHAPTER I", etc.
|
||||
*/
|
||||
export function detectChapters(text: string, maxChunkSize = 50000): Chapter[] {
|
||||
const lines = text.split('\n');
|
||||
const chapters: Chapter[] = [];
|
||||
|
||||
// Common chapter markers
|
||||
const chapterPatterns = [
|
||||
/^chapter\s+(\d+|[ivxlcdm]+)/i,
|
||||
/^ch\.\s+(\d+)/i,
|
||||
/^(\d+)\.\s+/, // "1. Title"
|
||||
/^part\s+(\d+|[ivxlcdm]+)/i,
|
||||
/^section\s+(\d+)/i,
|
||||
];
|
||||
|
||||
let currentChapterStart = 0;
|
||||
let currentChapterTitle = 'Beginning';
|
||||
|
||||
lines.forEach((line, index) => {
|
||||
const trimmed = line.trim();
|
||||
|
||||
// Check if this line is a chapter marker
|
||||
const isChapterMarker = chapterPatterns.some(pattern => pattern.test(trimmed));
|
||||
|
||||
if (isChapterMarker && index > 0) {
|
||||
// Save the previous chapter
|
||||
const chapterText = lines.slice(currentChapterStart, index).join('\n');
|
||||
if (chapterText.trim().length > 0) {
|
||||
chapters.push({
|
||||
title: currentChapterTitle,
|
||||
startIndex: currentChapterStart,
|
||||
endIndex: index,
|
||||
text: chapterText,
|
||||
});
|
||||
}
|
||||
|
||||
// Start new chapter
|
||||
currentChapterStart = index;
|
||||
currentChapterTitle = trimmed || `Chapter ${chapters.length + 1}`;
|
||||
}
|
||||
});
|
||||
|
||||
// Add the final chapter
|
||||
const finalChapterText = lines.slice(currentChapterStart).join('\n');
|
||||
if (finalChapterText.trim().length > 0) {
|
||||
chapters.push({
|
||||
title: currentChapterTitle,
|
||||
startIndex: currentChapterStart,
|
||||
endIndex: lines.length,
|
||||
text: finalChapterText,
|
||||
});
|
||||
}
|
||||
|
||||
// If no chapters were detected, split by size
|
||||
if (chapters.length === 0 || (chapters.length === 1 && text.length > maxChunkSize)) {
|
||||
return splitBySize(text, maxChunkSize);
|
||||
}
|
||||
|
||||
// If chapters are still too large, split them further
|
||||
const refinedChapters: Chapter[] = [];
|
||||
chapters.forEach(chapter => {
|
||||
if (chapter.text.length > maxChunkSize) {
|
||||
const subChapters = splitBySize(chapter.text, maxChunkSize);
|
||||
subChapters.forEach((subChapter, i) => {
|
||||
refinedChapters.push({
|
||||
...subChapter,
|
||||
title: `${chapter.title} (Part ${i + 1})`,
|
||||
});
|
||||
});
|
||||
} else {
|
||||
refinedChapters.push(chapter);
|
||||
}
|
||||
});
|
||||
|
||||
return refinedChapters.length > 0 ? refinedChapters : chapters;
|
||||
}
|
||||
|
||||
/**
|
||||
* Splits text into chunks of approximately equal size
|
||||
* Tries to split at paragraph boundaries
|
||||
*/
|
||||
function splitBySize(text: string, maxChunkSize: number): Chapter[] {
|
||||
const chapters: Chapter[] = [];
|
||||
const paragraphs = text.split(/\n\n+/);
|
||||
|
||||
let currentChunk = '';
|
||||
let chunkIndex = 0;
|
||||
|
||||
paragraphs.forEach((paragraph, index) => {
|
||||
const potentialChunk = currentChunk + (currentChunk ? '\n\n' : '') + paragraph;
|
||||
|
||||
if (potentialChunk.length > maxChunkSize && currentChunk.length > 0) {
|
||||
// Save current chunk
|
||||
chapters.push({
|
||||
title: `Section ${chunkIndex + 1}`,
|
||||
startIndex: 0, // We'll recalculate these if needed
|
||||
endIndex: 0,
|
||||
text: currentChunk,
|
||||
});
|
||||
|
||||
// Start new chunk
|
||||
currentChunk = paragraph;
|
||||
chunkIndex++;
|
||||
} else {
|
||||
currentChunk = potentialChunk;
|
||||
}
|
||||
});
|
||||
|
||||
// Add the final chunk
|
||||
if (currentChunk.trim().length > 0) {
|
||||
chapters.push({
|
||||
title: `Section ${chunkIndex + 1}`,
|
||||
startIndex: 0,
|
||||
endIndex: 0,
|
||||
text: currentChunk,
|
||||
});
|
||||
}
|
||||
|
||||
return chapters;
|
||||
}
|
||||
|
||||
/**
|
||||
* Estimate the number of chapters that would be created
|
||||
* Useful for showing chapter count before processing
|
||||
*/
|
||||
export function estimateChapterCount(text: string, maxChunkSize = 50000): number {
|
||||
const chapters = detectChapters(text, maxChunkSize);
|
||||
return chapters.length;
|
||||
}
|
||||
Loading…
Reference in a new issue