feat(text): add chapter detection and text splitting utility

Add comprehensive chapter detection for large text files:
- Detects common chapter markers (Chapter 1, CHAPTER I, Part 1, etc.)
- Splits large files into manageable chunks (~50KB each)
- Smart paragraph-boundary splitting to avoid mid-sentence breaks
- Handles files with no chapter markers by creating sections
- Recursive splitting for oversized chapters

Chapter detection patterns:
- "Chapter 1" / "CHAPTER I" (case-insensitive)
- "Ch. 1"
- "1. Title" (numbered sections)
- "Part 1" / "PART I"
- "Section 1"

Features:
- detectChapters(): Full chapter detection and splitting
- estimateChapterCount(): Preview chapter count before processing
- Configurable max chunk size (default 50KB)
- Preserves paragraph formatting
- Generates descriptive chapter titles

This utility will be integrated with:
- HTMLContext for paginated text viewing
- TTS auto-advance through chapters
- Audiobook generation with chapter markers

Files changed:
- lib/chapterDetection.ts: New chapter detection utility

Next steps:
- Integrate with HTMLContext for pagination
- Add chapter navigation UI
- Implement TTS auto-advance
This commit is contained in:
Claude 2026-01-11 04:23:54 +00:00
parent 1015d7de75
commit 55e35200ed
No known key found for this signature in database

142
src/lib/chapterDetection.ts Normal file
View file

@ -0,0 +1,142 @@
/**
* Utility functions for detecting and splitting text into chapters
*/
export interface Chapter {
title: string;
startIndex: number;
endIndex: number;
text: string;
}
/**
* Detects chapters in text content
* Looks for common chapter markers like "Chapter 1", "CHAPTER I", etc.
*/
export function detectChapters(text: string, maxChunkSize = 50000): Chapter[] {
const lines = text.split('\n');
const chapters: Chapter[] = [];
// Common chapter markers
const chapterPatterns = [
/^chapter\s+(\d+|[ivxlcdm]+)/i,
/^ch\.\s+(\d+)/i,
/^(\d+)\.\s+/, // "1. Title"
/^part\s+(\d+|[ivxlcdm]+)/i,
/^section\s+(\d+)/i,
];
let currentChapterStart = 0;
let currentChapterTitle = 'Beginning';
lines.forEach((line, index) => {
const trimmed = line.trim();
// Check if this line is a chapter marker
const isChapterMarker = chapterPatterns.some(pattern => pattern.test(trimmed));
if (isChapterMarker && index > 0) {
// Save the previous chapter
const chapterText = lines.slice(currentChapterStart, index).join('\n');
if (chapterText.trim().length > 0) {
chapters.push({
title: currentChapterTitle,
startIndex: currentChapterStart,
endIndex: index,
text: chapterText,
});
}
// Start new chapter
currentChapterStart = index;
currentChapterTitle = trimmed || `Chapter ${chapters.length + 1}`;
}
});
// Add the final chapter
const finalChapterText = lines.slice(currentChapterStart).join('\n');
if (finalChapterText.trim().length > 0) {
chapters.push({
title: currentChapterTitle,
startIndex: currentChapterStart,
endIndex: lines.length,
text: finalChapterText,
});
}
// If no chapters were detected, split by size
if (chapters.length === 0 || (chapters.length === 1 && text.length > maxChunkSize)) {
return splitBySize(text, maxChunkSize);
}
// If chapters are still too large, split them further
const refinedChapters: Chapter[] = [];
chapters.forEach(chapter => {
if (chapter.text.length > maxChunkSize) {
const subChapters = splitBySize(chapter.text, maxChunkSize);
subChapters.forEach((subChapter, i) => {
refinedChapters.push({
...subChapter,
title: `${chapter.title} (Part ${i + 1})`,
});
});
} else {
refinedChapters.push(chapter);
}
});
return refinedChapters.length > 0 ? refinedChapters : chapters;
}
/**
* Splits text into chunks of approximately equal size
* Tries to split at paragraph boundaries
*/
function splitBySize(text: string, maxChunkSize: number): Chapter[] {
const chapters: Chapter[] = [];
const paragraphs = text.split(/\n\n+/);
let currentChunk = '';
let chunkIndex = 0;
paragraphs.forEach((paragraph, index) => {
const potentialChunk = currentChunk + (currentChunk ? '\n\n' : '') + paragraph;
if (potentialChunk.length > maxChunkSize && currentChunk.length > 0) {
// Save current chunk
chapters.push({
title: `Section ${chunkIndex + 1}`,
startIndex: 0, // We'll recalculate these if needed
endIndex: 0,
text: currentChunk,
});
// Start new chunk
currentChunk = paragraph;
chunkIndex++;
} else {
currentChunk = potentialChunk;
}
});
// Add the final chunk
if (currentChunk.trim().length > 0) {
chapters.push({
title: `Section ${chunkIndex + 1}`,
startIndex: 0,
endIndex: 0,
text: currentChunk,
});
}
return chapters;
}
/**
* Estimate the number of chapters that would be created
* Useful for showing chapter count before processing
*/
export function estimateChapterCount(text: string, maxChunkSize = 50000): number {
const chapters = detectChapters(text, maxChunkSize);
return chapters.length;
}