fix: address CodeRabbit review comments for PR #76
Security fixes: - Validate returnTo parameter in auth route to prevent open redirect attacks Performance fixes: - Add concurrency limit (3) for EPUB spine extraction to prevent memory exhaustion - Iteratively split oversized paragraphs in chunk splitting to prevent content loss - Add iteration limit (10) and shrink-check to summary combining loop Data integrity: - Remap summary rows when document IDs change during sync UX improvements: - Clear summary base URL when switching to non-custom providers - Add type="button" to SummarizeButton to prevent form submission - Add dev logging to PDFViewer error handler - Move document state detection from render to useEffect Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
parent
c46947b238
commit
567b29abf2
8 changed files with 186 additions and 29 deletions
|
|
@ -1,6 +1,23 @@
|
||||||
import { NextRequest, NextResponse } from 'next/server';
|
import { NextRequest, NextResponse } from 'next/server';
|
||||||
import { getAuthToken } from '@/lib/auth';
|
import { getAuthToken } from '@/lib/auth';
|
||||||
|
|
||||||
|
function isValidReturnTo(returnTo: string): boolean {
|
||||||
|
// Only allow relative paths starting with /
|
||||||
|
// Reject absolute URLs, protocol-relative URLs, and other schemes
|
||||||
|
if (!returnTo.startsWith('/')) return false;
|
||||||
|
if (returnTo.startsWith('//')) return false;
|
||||||
|
// Reject URLs with encoded characters that could bypass checks
|
||||||
|
if (returnTo.includes('%')) {
|
||||||
|
try {
|
||||||
|
const decoded = decodeURIComponent(returnTo);
|
||||||
|
if (decoded.startsWith('//') || decoded.includes('://')) return false;
|
||||||
|
} catch {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
export async function GET(request: NextRequest) {
|
export async function GET(request: NextRequest) {
|
||||||
const token = request.nextUrl.searchParams.get('token');
|
const token = request.nextUrl.searchParams.get('token');
|
||||||
const returnTo = request.nextUrl.searchParams.get('returnTo') || '/';
|
const returnTo = request.nextUrl.searchParams.get('returnTo') || '/';
|
||||||
|
|
@ -9,7 +26,9 @@ export async function GET(request: NextRequest) {
|
||||||
return NextResponse.json({ error: 'Invalid token' }, { status: 401 });
|
return NextResponse.json({ error: 'Invalid token' }, { status: 401 });
|
||||||
}
|
}
|
||||||
|
|
||||||
const response = NextResponse.redirect(new URL(returnTo, request.url));
|
// Validate returnTo to prevent open redirect attacks
|
||||||
|
const safeReturnTo = isValidReturnTo(returnTo) ? returnTo : '/';
|
||||||
|
const response = NextResponse.redirect(new URL(safeReturnTo, request.url));
|
||||||
response.cookies.set('auth_session', token, {
|
response.cookies.set('auth_session', token, {
|
||||||
httpOnly: true,
|
httpOnly: true,
|
||||||
secure: process.env.NODE_ENV === 'production',
|
secure: process.env.NODE_ENV === 'production',
|
||||||
|
|
|
||||||
|
|
@ -20,6 +20,7 @@ import { SummarizeButton } from '@/components/SummarizeButton';
|
||||||
import { SummarizeModal } from '@/components/SummarizeModal';
|
import { SummarizeModal } from '@/components/SummarizeModal';
|
||||||
import type { SummarizeMode } from '@/types/summary';
|
import type { SummarizeMode } from '@/types/summary';
|
||||||
import { resolveDocumentId } from '@/lib/dexie';
|
import { resolveDocumentId } from '@/lib/dexie';
|
||||||
|
import { processWithConcurrencyLimit } from '@/lib/concurrency';
|
||||||
|
|
||||||
const isDev = process.env.NEXT_PUBLIC_NODE_ENV !== 'production' || process.env.NODE_ENV == null;
|
const isDev = process.env.NEXT_PUBLIC_NODE_ENV !== 'production' || process.env.NODE_ENV == null;
|
||||||
|
|
||||||
|
|
@ -126,27 +127,38 @@ export default function EPUBPage() {
|
||||||
}
|
}
|
||||||
|
|
||||||
if (mode === 'whole_book') {
|
if (mode === 'whole_book') {
|
||||||
// Extract text from all spine sections
|
// Extract text from all spine sections with concurrency limit to prevent memory exhaustion
|
||||||
const promises: Promise<string>[] = [];
|
const spineItems: { href: string }[] = [];
|
||||||
const spine = book.spine;
|
const spine = book.spine;
|
||||||
|
|
||||||
spine.each((item: { href?: string }) => {
|
spine.each((item: { href?: string }) => {
|
||||||
const url = item.href || '';
|
const url = item.href || '';
|
||||||
if (!url) return;
|
if (url) {
|
||||||
|
spineItems.push({ href: url });
|
||||||
const promise = book.load(url)
|
}
|
||||||
.then((section) => (section as Document))
|
|
||||||
.then((section) => section.body?.textContent?.trim() || '')
|
|
||||||
.catch((err) => {
|
|
||||||
console.warn('Failed to extract text from section:', url, err);
|
|
||||||
return '';
|
|
||||||
});
|
|
||||||
|
|
||||||
promises.push(promise);
|
|
||||||
});
|
});
|
||||||
|
|
||||||
const textParts = await Promise.all(promises);
|
// Use concurrency limit of 3 to prevent memory spikes on large books
|
||||||
return textParts.filter(text => text).join('\n\n');
|
const results = await processWithConcurrencyLimit(
|
||||||
|
spineItems,
|
||||||
|
async (item) => {
|
||||||
|
try {
|
||||||
|
const section = await book.load(item.href) as Document;
|
||||||
|
return section.body?.textContent?.trim() || '';
|
||||||
|
} catch (err) {
|
||||||
|
console.warn('Failed to extract text from section:', item.href, err);
|
||||||
|
return '';
|
||||||
|
}
|
||||||
|
},
|
||||||
|
3 // Max concurrent extractions
|
||||||
|
);
|
||||||
|
|
||||||
|
const textParts = results
|
||||||
|
.filter((r): r is { status: 'fulfilled'; value: string } => r.status === 'fulfilled')
|
||||||
|
.map(r => r.value)
|
||||||
|
.filter(text => text);
|
||||||
|
|
||||||
|
return textParts.join('\n\n');
|
||||||
} else {
|
} else {
|
||||||
// Extract current page text
|
// Extract current page text
|
||||||
return extractPageText(book, rendition, false);
|
return extractPageText(book, rendition, false);
|
||||||
|
|
|
||||||
|
|
@ -62,15 +62,15 @@ export function PDFViewer({ zoomLevel }: PDFViewerProps) {
|
||||||
const lastDataRef = useRef<ArrayBuffer | undefined>(undefined);
|
const lastDataRef = useRef<ArrayBuffer | undefined>(undefined);
|
||||||
|
|
||||||
// Reset ready state when document data changes
|
// Reset ready state when document data changes
|
||||||
if (currDocData !== lastDataRef.current) {
|
useEffect(() => {
|
||||||
lastDataRef.current = currDocData;
|
if (currDocData !== lastDataRef.current) {
|
||||||
if (currDocData) {
|
lastDataRef.current = currDocData;
|
||||||
documentKeyRef.current += 1;
|
if (currDocData) {
|
||||||
}
|
documentKeyRef.current += 1;
|
||||||
if (isDocumentReady) {
|
}
|
||||||
setIsDocumentReady(false);
|
setIsDocumentReady(false);
|
||||||
}
|
}
|
||||||
}
|
}, [currDocData]);
|
||||||
|
|
||||||
// Create a Uint8Array copy to prevent "detached ArrayBuffer" errors
|
// Create a Uint8Array copy to prevent "detached ArrayBuffer" errors
|
||||||
const pdfFileData = useMemo(() => {
|
const pdfFileData = useMemo(() => {
|
||||||
|
|
@ -300,8 +300,11 @@ export function PDFViewer({ zoomLevel }: PDFViewerProps) {
|
||||||
onDocumentLoadSuccess(pdf);
|
onDocumentLoadSuccess(pdf);
|
||||||
setIsDocumentReady(true);
|
setIsDocumentReady(true);
|
||||||
}}
|
}}
|
||||||
onLoadError={() => {
|
onLoadError={(error) => {
|
||||||
// Ignore errors from destroyed documents during navigation/reload
|
// Log errors in development for debugging
|
||||||
|
if (process.env.NODE_ENV !== 'production') {
|
||||||
|
console.warn('PDFViewer load error (may be from destroyed document during navigation):', error);
|
||||||
|
}
|
||||||
}}
|
}}
|
||||||
onItemClick={(args: PDFOnLinkClickArgs) => {
|
onItemClick={(args: PDFOnLinkClickArgs) => {
|
||||||
if (args?.pageNumber) {
|
if (args?.pageNumber) {
|
||||||
|
|
|
||||||
|
|
@ -730,15 +730,19 @@ export function SettingsModal() {
|
||||||
switch (provider.id) {
|
switch (provider.id) {
|
||||||
case 'openai':
|
case 'openai':
|
||||||
setLocalSummaryModel('gpt-4o-mini');
|
setLocalSummaryModel('gpt-4o-mini');
|
||||||
|
setLocalSummaryBaseUrl(''); // Clear custom base URL
|
||||||
break;
|
break;
|
||||||
case 'anthropic':
|
case 'anthropic':
|
||||||
setLocalSummaryModel('claude-3-5-haiku-latest');
|
setLocalSummaryModel('claude-3-5-haiku-latest');
|
||||||
|
setLocalSummaryBaseUrl(''); // Clear custom base URL
|
||||||
break;
|
break;
|
||||||
case 'groq':
|
case 'groq':
|
||||||
setLocalSummaryModel('llama-3.3-70b-versatile');
|
setLocalSummaryModel('llama-3.3-70b-versatile');
|
||||||
|
setLocalSummaryBaseUrl(''); // Clear custom base URL
|
||||||
break;
|
break;
|
||||||
case 'openrouter':
|
case 'openrouter':
|
||||||
setLocalSummaryModel('google/gemini-2.0-flash-001');
|
setLocalSummaryModel('google/gemini-2.0-flash-001');
|
||||||
|
setLocalSummaryBaseUrl(''); // Clear custom base URL
|
||||||
break;
|
break;
|
||||||
case 'custom-openai':
|
case 'custom-openai':
|
||||||
setLocalSummaryModel('');
|
setLocalSummaryModel('');
|
||||||
|
|
|
||||||
|
|
@ -10,6 +10,7 @@ interface SummarizeButtonProps {
|
||||||
export function SummarizeButton({ onClick, disabled }: SummarizeButtonProps) {
|
export function SummarizeButton({ onClick, disabled }: SummarizeButtonProps) {
|
||||||
return (
|
return (
|
||||||
<button
|
<button
|
||||||
|
type="button"
|
||||||
onClick={onClick}
|
onClick={onClick}
|
||||||
disabled={disabled}
|
disabled={disabled}
|
||||||
className="inline-flex items-center py-1 px-2 rounded-md border border-offbase bg-base text-foreground text-xs hover:bg-offbase transition-all duration-200 ease-in-out hover:scale-[1.09] hover:text-accent disabled:opacity-50 disabled:cursor-not-allowed"
|
className="inline-flex items-center py-1 px-2 rounded-md border border-offbase bg-base text-foreground text-xs hover:bg-offbase transition-all duration-200 ease-in-out hover:scale-[1.09] hover:text-accent disabled:opacity-50 disabled:cursor-not-allowed"
|
||||||
|
|
|
||||||
50
src/lib/concurrency.ts
Normal file
50
src/lib/concurrency.ts
Normal file
|
|
@ -0,0 +1,50 @@
|
||||||
|
/**
|
||||||
|
* Utility for processing items with a concurrency limit
|
||||||
|
*
|
||||||
|
* This enables parallel processing of items (like TTS generation) while
|
||||||
|
* respecting rate limits and preventing resource exhaustion.
|
||||||
|
*/
|
||||||
|
|
||||||
|
export type ConcurrencyResult<R> =
|
||||||
|
| { status: 'fulfilled'; value: R }
|
||||||
|
| { status: 'rejected'; reason: Error };
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Process items in parallel with a maximum concurrency limit.
|
||||||
|
*
|
||||||
|
* @param items - Array of items to process
|
||||||
|
* @param processor - Async function to process each item
|
||||||
|
* @param maxConcurrent - Maximum number of concurrent processors
|
||||||
|
* @param signal - Optional AbortSignal for cancellation
|
||||||
|
* @returns Array of results in the same order as input items
|
||||||
|
*/
|
||||||
|
export async function processWithConcurrencyLimit<T, R>(
|
||||||
|
items: T[],
|
||||||
|
processor: (item: T, index: number) => Promise<R>,
|
||||||
|
maxConcurrent: number,
|
||||||
|
signal?: AbortSignal
|
||||||
|
): Promise<ConcurrencyResult<R>[]> {
|
||||||
|
const results: ConcurrencyResult<R>[] = new Array(items.length);
|
||||||
|
let currentIndex = 0;
|
||||||
|
|
||||||
|
async function worker(): Promise<void> {
|
||||||
|
while (currentIndex < items.length) {
|
||||||
|
if (signal?.aborted) break;
|
||||||
|
|
||||||
|
const index = currentIndex++;
|
||||||
|
if (index >= items.length) break;
|
||||||
|
|
||||||
|
try {
|
||||||
|
const result = await processor(items[index], index);
|
||||||
|
results[index] = { status: 'fulfilled', value: result };
|
||||||
|
} catch (error) {
|
||||||
|
results[index] = { status: 'rejected', reason: error as Error };
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const workerCount = Math.min(maxConcurrent, items.length);
|
||||||
|
await Promise.all(Array(workerCount).fill(null).map(() => worker()));
|
||||||
|
|
||||||
|
return results;
|
||||||
|
}
|
||||||
|
|
@ -329,6 +329,7 @@ async function applyDocumentIdMapping(oldId: string, newId: string): Promise<voi
|
||||||
db[LAST_LOCATION_TABLE],
|
db[LAST_LOCATION_TABLE],
|
||||||
db[APP_CONFIG_TABLE],
|
db[APP_CONFIG_TABLE],
|
||||||
db[DOCUMENT_ID_MAP_TABLE],
|
db[DOCUMENT_ID_MAP_TABLE],
|
||||||
|
db[SUMMARIES_TABLE],
|
||||||
],
|
],
|
||||||
async () => {
|
async () => {
|
||||||
await recordDocumentIdMapping(oldId, nextId);
|
await recordDocumentIdMapping(oldId, nextId);
|
||||||
|
|
@ -406,6 +407,24 @@ async function applyDocumentIdMapping(oldId: string, newId: string): Promise<voi
|
||||||
await db[APP_CONFIG_TABLE].update('singleton', { documentListState: mapped });
|
await db[APP_CONFIG_TABLE].update('singleton', { documentListState: mapped });
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Remap summaries to use the new document ID
|
||||||
|
const oldSummaries = await db[SUMMARIES_TABLE].where('docId').equals(oldId).toArray();
|
||||||
|
for (const summary of oldSummaries) {
|
||||||
|
const newSummaryId = `${nextId}-${summary.scope}-${summary.pageNumber ?? 'all'}`;
|
||||||
|
// Check if a summary with the new ID already exists
|
||||||
|
const existing = await db[SUMMARIES_TABLE].get(newSummaryId);
|
||||||
|
if (!existing) {
|
||||||
|
// Create new summary with updated docId and id
|
||||||
|
await db[SUMMARIES_TABLE].put({
|
||||||
|
...summary,
|
||||||
|
id: newSummaryId,
|
||||||
|
docId: nextId,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
// Delete the old summary
|
||||||
|
await db[SUMMARIES_TABLE].delete(summary.id);
|
||||||
|
}
|
||||||
},
|
},
|
||||||
);
|
);
|
||||||
});
|
});
|
||||||
|
|
|
||||||
|
|
@ -24,15 +24,48 @@ function splitTextIntoChunks(text: string, maxTokens: number): string[] {
|
||||||
const chunks: string[] = [];
|
const chunks: string[] = [];
|
||||||
let current = '';
|
let current = '';
|
||||||
|
|
||||||
|
// Helper to split oversized text into multiple chunks
|
||||||
|
function splitOversizedText(oversized: string): string[] {
|
||||||
|
const parts: string[] = [];
|
||||||
|
let remaining = oversized;
|
||||||
|
while (remaining.length > 0) {
|
||||||
|
// Try to split at sentence boundary within maxChars
|
||||||
|
let splitPoint = maxChars;
|
||||||
|
if (remaining.length > maxChars) {
|
||||||
|
const searchRange = remaining.slice(0, maxChars);
|
||||||
|
const lastSentence = Math.max(
|
||||||
|
searchRange.lastIndexOf('. '),
|
||||||
|
searchRange.lastIndexOf('! '),
|
||||||
|
searchRange.lastIndexOf('? ')
|
||||||
|
);
|
||||||
|
if (lastSentence > maxChars * 0.5) {
|
||||||
|
splitPoint = lastSentence + 2; // Include the punctuation and space
|
||||||
|
}
|
||||||
|
}
|
||||||
|
parts.push(remaining.slice(0, splitPoint).trim());
|
||||||
|
remaining = remaining.slice(splitPoint).trim();
|
||||||
|
}
|
||||||
|
return parts;
|
||||||
|
}
|
||||||
|
|
||||||
for (const para of text.split(/\n\n+/)) {
|
for (const para of text.split(/\n\n+/)) {
|
||||||
const trimmed = para.trim();
|
const trimmed = para.trim();
|
||||||
if (!trimmed) continue;
|
if (!trimmed) continue;
|
||||||
|
|
||||||
if ((current + '\n\n' + trimmed).length > maxChars) {
|
if ((current + '\n\n' + trimmed).length > maxChars) {
|
||||||
if (current) chunks.push(current.trim());
|
if (current) chunks.push(current.trim());
|
||||||
current = trimmed.length > maxChars
|
// Handle oversized paragraphs by splitting them iteratively
|
||||||
? trimmed.slice(0, maxChars) // Force split if paragraph too long
|
if (trimmed.length > maxChars) {
|
||||||
: trimmed;
|
const splitParts = splitOversizedText(trimmed);
|
||||||
|
// Add all but the last part as separate chunks
|
||||||
|
for (let i = 0; i < splitParts.length - 1; i++) {
|
||||||
|
chunks.push(splitParts[i]);
|
||||||
|
}
|
||||||
|
// Use the last part as the new current
|
||||||
|
current = splitParts[splitParts.length - 1] || '';
|
||||||
|
} else {
|
||||||
|
current = trimmed;
|
||||||
|
}
|
||||||
} else {
|
} else {
|
||||||
current = current ? current + '\n\n' + trimmed : trimmed;
|
current = current ? current + '\n\n' + trimmed : trimmed;
|
||||||
}
|
}
|
||||||
|
|
@ -101,9 +134,19 @@ export async function generateSummary(
|
||||||
|
|
||||||
// Combine summaries (recursively if still too large)
|
// Combine summaries (recursively if still too large)
|
||||||
onProgress?.({ currentChunk: chunks.length, totalChunks: chunks.length, phase: 'combining', message: 'Combining summaries...' });
|
onProgress?.({ currentChunk: chunks.length, totalChunks: chunks.length, phase: 'combining', message: 'Combining summaries...' });
|
||||||
|
|
||||||
let combined = summaries.join('\n\n---\n\n');
|
let combined = summaries.join('\n\n---\n\n');
|
||||||
|
const MAX_COMBINING_ITERATIONS = 10;
|
||||||
|
let combiningIterations = 0;
|
||||||
|
|
||||||
while (needsChunking(combined, contextLimit)) {
|
while (needsChunking(combined, contextLimit)) {
|
||||||
|
combiningIterations++;
|
||||||
|
if (combiningIterations > MAX_COMBINING_ITERATIONS) {
|
||||||
|
console.warn('Summary combining reached maximum iterations, proceeding with current result');
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
const previousLength = combined.length;
|
||||||
const subChunks = splitTextIntoChunks(combined, maxInputTokens);
|
const subChunks = splitTextIntoChunks(combined, maxInputTokens);
|
||||||
const subSummaries: string[] = [];
|
const subSummaries: string[] = [];
|
||||||
for (const chunk of subChunks) {
|
for (const chunk of subChunks) {
|
||||||
|
|
@ -112,6 +155,12 @@ export async function generateSummary(
|
||||||
totalTokens += result.tokensUsed || 0;
|
totalTokens += result.tokensUsed || 0;
|
||||||
}
|
}
|
||||||
combined = subSummaries.join('\n\n---\n\n');
|
combined = subSummaries.join('\n\n---\n\n');
|
||||||
|
|
||||||
|
// Guard against non-convergence: if combined didn't shrink, break to avoid infinite loop
|
||||||
|
if (combined.length >= previousLength) {
|
||||||
|
console.warn('Summary combining not converging, proceeding with current result');
|
||||||
|
break;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Final pass
|
// Final pass
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue