perf(test): optimize test execution and error handling

- Reduce Playwright worker allocation from 75% to 50% for better resource management
- Disable parallel test execution temporarily to isolate flaky test behavior
- Enhance audio utility to fail fast on user cancellation, preventing unnecessary retries
- Consolidate test-specific utilities within individual test suites for better isolation
- Introduce export functionality test coverage with new export.spec.ts suite
This commit is contained in:
Richard Roberson 2025-11-14 18:52:49 -07:00
parent 5316596a1b
commit 2f39e8f014
7 changed files with 411 additions and 67 deletions

View file

@ -7,11 +7,11 @@ export default defineConfig({
testDir: './tests',
timeout: 30 * 1000,
outputDir: './tests/results',
fullyParallel: false,
// fullyParallel: false,
/* Fail the build on CI if you accidentally left test.only in the source code. */
forbidOnly: !!process.env.CI,
retries: process.env.CI ? 2 : 0,
workers: process.env.CI ? '100%' : '75%',
workers: process.env.CI ? '100%' : '50%',
/* Reporter to use. See https://playwright.dev/docs/test-reporters */
reporter: 'html',
/* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */

View file

@ -39,7 +39,13 @@ export const withRetry = async <T>(
return await operation();
} catch (error) {
lastError = error instanceof Error ? error : new Error(String(error));
// Do not retry on explicit cancellation/abort errors - surface them
// immediately so callers can stop work quickly when the user cancels.
if (lastError.name === 'AbortError' || lastError.message.includes('cancelled')) {
break;
}
if (attempt === maxRetries - 1) {
break;
}
@ -130,4 +136,4 @@ export const combineAudioChunks = async (
} finally {
if (setIsAudioCombining) setIsAudioCombining(false);
}
}
}

343
tests/export.spec.ts Normal file
View file

@ -0,0 +1,343 @@
import { test, expect, Page } from '@playwright/test';
import fs from 'fs';
import util from 'util';
import { execFile } from 'child_process';
import { setupTest, uploadAndDisplay } from './helpers';
const execFileAsync = util.promisify(execFile);
async function getBookIdFromUrl(page: Page, expectedPrefix: 'pdf' | 'epub') {
const url = new URL(page.url());
const segments = url.pathname.split('/').filter(Boolean);
expect(segments[0]).toBe(expectedPrefix);
const bookId = segments[1];
expect(bookId).toBeTruthy();
return bookId;
}
async function openExportModal(page: Page) {
const exportButton = page.getByRole('button', { name: 'Open audiobook export' });
await expect(exportButton).toBeVisible({ timeout: 15_000 });
await exportButton.click();
await expect(page.getByRole('heading', { name: 'Export Audiobook' })).toBeVisible({ timeout: 15_000 });
}
async function setContainerFormatToMP3(page: Page) {
const formatTrigger = page.getByRole('button', { name: /M4B|MP3/i });
await expect(formatTrigger).toBeVisible({ timeout: 15_000 });
await formatTrigger.click();
await page.getByRole('option', { name: 'MP3' }).click();
}
async function startGeneration(page: Page) {
const startButton = page.getByRole('button', { name: 'Start Generation' });
await expect(startButton).toBeVisible({ timeout: 15_000 });
await startButton.click();
}
async function waitForChaptersHeading(page: Page) {
await expect(page.getByRole('heading', { name: 'Chapters' })).toBeVisible({ timeout: 60_000 });
}
async function downloadFullAudiobook(page: Page, timeoutMs = 60_000) {
const fullDownloadButton = page.getByRole('button', { name: /Full Download/i });
await expect(fullDownloadButton).toBeVisible({ timeout: timeoutMs });
const [download] = await Promise.all([
page.waitForEvent('download', { timeout: timeoutMs }),
fullDownloadButton.click(),
]);
const downloadedPath = await download.path();
expect(downloadedPath).toBeTruthy();
const stats = fs.statSync(downloadedPath!);
expect(stats.size).toBeGreaterThan(0);
return downloadedPath!;
}
async function getAudioDurationSeconds(filePath: string) {
const { stdout } = await execFileAsync('ffprobe', [
'-v',
'error',
'-show_entries',
'format=duration',
'-of',
'csv=p=0',
filePath,
]);
return parseFloat(stdout.trim());
}
async function expectChaptersBackendState(page: Page, bookId: string) {
const res = await page.request.get(`/api/audio/convert/chapters?bookId=${bookId}`);
expect(res.ok()).toBeTruthy();
const json = await res.json();
return json;
}
async function resetAudiobookIfPresent(page: Page) {
const resetButtons = page.getByRole('button', { name: 'Reset' });
const count = await resetButtons.count();
if (count === 0) {
return;
}
const resetButton = resetButtons.first();
await resetButton.click();
await expect(page.getByRole('heading', { name: 'Reset Audiobook' })).toBeVisible({ timeout: 15_000 });
const confirmReset = page.getByRole('button', { name: 'Reset' }).last();
await confirmReset.click();
await expect(
page.getByText(/Click "Start Generation" to begin creating your audiobook/i)
).toBeVisible({ timeout: 60_000 });
}
test.describe('Audiobook export', () => {
test.describe.configure({ mode: 'serial', timeout: 120_000 });
test('exports full MP3 audiobook for PDF using mocked 10s TTS sample', async ({ page }) => {
// Ensure TTS is mocked and app is ready
await setupTest(page);
// Upload and open the sample PDF in the viewer
await uploadAndDisplay(page, 'sample.pdf');
// Capture the generated document/book id from the /pdf/[id] URL
const bookId = await getBookIdFromUrl(page, 'pdf');
// Open the audiobook export modal from the header button
await openExportModal(page);
// While there are no chapters yet, we can still switch the container format.
// Choose MP3 so we can validate MP3 duration end-to-end.
await setContainerFormatToMP3(page);
// Start generation; this will call the mocked /api/tts which returns a 10s sample.mp3 per page
await startGeneration(page);
// Wait for chapters list to appear and populate at least two items (Pages 1 and 2)
await waitForChaptersHeading(page);
const chapterActionsButtons = page.getByRole('button', { name: 'Chapter actions' });
await expect(chapterActionsButtons).toHaveCount(2, { timeout: 60_000 });
// Trigger full download from the FRONTEND button and capture via Playwright's download API.
// The button label can be "Full Download (MP3)" or "Full Download (M4B)" depending on
// the server-side detected format, so match more loosely on the accessible name.
const downloadedPath = await downloadFullAudiobook(page);
// Use ffprobe (same toolchain as the server) to validate the combined audio duration.
// The TTS route is mocked to return a 10s sample.mp3 for each page, so with at least
// two chapters we should be close to ~20 seconds of audio.
const durationSeconds = await getAudioDurationSeconds(downloadedPath);
// Duration must be within a reasonable window around 20 seconds to allow
// for encoding variations and container overhead.
expect(durationSeconds).toBeGreaterThan(18);
expect(durationSeconds).toBeLessThan(22);
// Also check the chapter metadata API for consistency
const json = await expectChaptersBackendState(page, bookId);
expect(json.exists).toBe(true);
expect(Array.isArray(json.chapters)).toBe(true);
expect(json.chapters.length).toBeGreaterThanOrEqual(2);
for (const ch of json.chapters) {
expect(ch.duration).toBeGreaterThan(0);
}
await resetAudiobookIfPresent(page);
});
test('handles partial EPUB audiobook generation, cancel, and full download of partial audiobook', async ({ page }) => {
await setupTest(page);
// Upload and open the sample EPUB in the viewer
await uploadAndDisplay(page, 'sample.epub');
// URL should now be /epub/[id]
const bookId = await getBookIdFromUrl(page, 'epub');
// Open the audiobook export modal from the header button
await openExportModal(page);
// Set container format to MP3
await setContainerFormatToMP3(page);
// Start generation
await startGeneration(page);
// Progress card should appear with a Cancel button while chapters are being generated
const cancelButton = page.getByRole('button', { name: 'Cancel' });
await expect(cancelButton).toBeVisible({ timeout: 60_000 });
await expect(page.getByRole('heading', { name: 'Chapters' })).toBeVisible({ timeout: 60_000 });
// Wait until at least 3 chapters are listed in the UI; record the exact count at the
// moment we decide to cancel, and assert that no additional chapters are added afterward.
const chapterActionsButtons = page.getByRole('button', { name: 'Chapter actions' });
await expect(chapterActionsButtons.nth(2)).toBeVisible({ timeout: 120_000 });
const chapterCountBeforeCancel = await chapterActionsButtons.count();
expect(chapterCountBeforeCancel).toBeGreaterThanOrEqual(3);
// Now cancel the in-flight generation
await cancelButton.click();
// After cancellation, the inline progress card's Cancel button should be gone
await expect(page.getByRole('button', { name: 'Cancel' })).toHaveCount(0);
// After cancellation, determine the canonical chapter count from the backend and
// assert that the UI eventually reflects this count. Some in-flight chapters may
// complete right as we cancel, so we treat the backend state as source of truth.
const jsonAfterCancel = await expectChaptersBackendState(page, bookId);
expect(jsonAfterCancel.exists).toBe(true);
expect(Array.isArray(jsonAfterCancel.chapters)).toBe(true);
const chapterCountAfterCancel = jsonAfterCancel.chapters.length;
expect(chapterCountAfterCancel).toBeGreaterThanOrEqual(chapterCountBeforeCancel);
// Wait for the UI to reflect the final backend chapter count to avoid race
// conditions between the modal's soft refresh and our assertions.
await expect(chapterActionsButtons).toHaveCount(chapterCountAfterCancel, { timeout: 60_000 });
// The Full Download button should still be available for the partially generated audiobook
const downloadedPath = await downloadFullAudiobook(page);
const durationSeconds = await getAudioDurationSeconds(downloadedPath);
expect(durationSeconds).toBeGreaterThan(25);
expect(durationSeconds).toBeLessThan(300);
// Backend should still reflect the same number of chapters as when we first
// observed the stabilized post-cancellation state, and should not contain
// additional "impartial" chapters produced after cancellation.
const json = await expectChaptersBackendState(page, bookId);
expect(json.exists).toBe(true);
expect(Array.isArray(json.chapters)).toBe(true);
expect(json.chapters.length).toBe(chapterCountAfterCancel);
await resetAudiobookIfPresent(page);
});
test('downloads a single chapter via chapter actions menu (PDF)', async ({ page }) => {
await setupTest(page);
await uploadAndDisplay(page, 'sample.pdf');
const bookId = await getBookIdFromUrl(page, 'pdf');
await openExportModal(page);
await setContainerFormatToMP3(page);
await startGeneration(page);
await waitForChaptersHeading(page);
// Wait for at least one chapter row to appear (one "Chapter actions" button)
const chapterActionsButtons = page.getByRole('button', { name: 'Chapter actions' });
await expect(chapterActionsButtons.first()).toBeVisible({ timeout: 90_000 });
// Download via frontend button
const downloadedPath = await downloadFullAudiobook(page);
const durationSeconds = await getAudioDurationSeconds(downloadedPath);
// For EPUB we just assert a sane non-trivial duration; at least one 10s mocked chapter.
expect(durationSeconds).toBeGreaterThan(9);
expect(durationSeconds).toBeLessThan(300);
await resetAudiobookIfPresent(page);
});
test('reset removes all generated chapters for a PDF audiobook', async ({ page }) => {
await setupTest(page);
await uploadAndDisplay(page, 'sample.pdf');
const bookId = await getBookIdFromUrl(page, 'pdf');
await openExportModal(page);
await setContainerFormatToMP3(page);
await startGeneration(page);
await waitForChaptersHeading(page);
// Wait for Reset button to become visible, indicating resumable/generated state
const resetButton = page.getByRole('button', { name: 'Reset' });
await expect(resetButton).toBeVisible({ timeout: 120_000 });
await resetButton.click();
// Confirm in the Reset Audiobook dialog
await expect(page.getByRole('heading', { name: 'Reset Audiobook' })).toBeVisible({ timeout: 15000 });
const confirmReset = page.getByRole('button', { name: 'Reset' }).last();
await confirmReset.click();
// After reset, the hint text for starting generation should re-appear
await expect(
page.getByText(/Click "Start Generation" to begin creating your audiobook/i)
).toBeVisible({ timeout: 60_000 });
// Backend should report no existing chapters for this bookId
const res = await page.request.get(`/api/audio/convert/chapters?bookId=${bookId}`);
expect(res.ok()).toBeTruthy();
const json = await res.json();
expect(json.exists).toBe(false);
expect(Array.isArray(json.chapters)).toBe(true);
expect(json.chapters.length).toBe(0);
});
test('regenerates a PDF audiobook chapter and preserves chapter count and full download', async ({ page }) => {
await setupTest(page);
await uploadAndDisplay(page, 'sample.pdf');
// Extract bookId from /pdf/[id] URL (for backend verification later)
const bookId = await getBookIdFromUrl(page, 'pdf');
// Open Export Audiobook modal
await openExportModal(page);
// Set container format to MP3
await setContainerFormatToMP3(page);
// Start generation
await startGeneration(page);
// Wait for chapters to appear
await waitForChaptersHeading(page);
const chapterActionsButtons = page.getByRole('button', { name: 'Chapter actions' });
// Ensure we have at least two chapters for this PDF
await expect(chapterActionsButtons.nth(1)).toBeVisible({ timeout: 60_000 });
const chapterCountBefore = await chapterActionsButtons.count();
expect(chapterCountBefore).toBeGreaterThanOrEqual(2);
// Open the actions menu for the first chapter and trigger Regenerate
const firstChapterActions = chapterActionsButtons.first();
await firstChapterActions.click();
// In the headlessui Menu, each option is a menuitem. Use that role instead of button.
const regenerateMenuItem = page.getByRole('menuitem', { name: /Regenerate/i });
await expect(regenerateMenuItem).toBeVisible({ timeout: 15000 });
await regenerateMenuItem.click();
// During regeneration, the row may show a "Regenerating" label; wait for any such
// indicator to disappear, signaling completion.
const regeneratingLabel = page.getByText(/Regenerating/);
await expect(regeneratingLabel).toHaveCount(0, { timeout: 120_000 });
// Chapter count should remain exactly the same after regeneration (no duplicates)
await expect(chapterActionsButtons).toHaveCount(chapterCountBefore, { timeout: 20_000 });
// Full Download should still work and produce a valid combined audiobook
const downloadedPath = await downloadFullAudiobook(page);
const durationSeconds = await getAudioDurationSeconds(downloadedPath);
// With two mocked 10s chapters we expect roughly 20s; allow a small window.
expect(durationSeconds).toBeGreaterThan(18);
expect(durationSeconds).toBeLessThan(22);
// Backend should still report the same number of chapters and valid durations
const json = await expectChaptersBackendState(page, bookId);
expect(json.exists).toBe(true);
expect(Array.isArray(json.chapters)).toBe(true);
expect(json.chapters.length).toBe(chapterCountBefore);
for (const ch of json.chapters) {
expect(ch.duration).toBeGreaterThan(0);
}
await resetAudiobookIfPresent(page);
});
});

View file

@ -1,5 +1,5 @@
import { test, expect } from '@playwright/test';
import { setupTest, uploadFiles, ensureDocumentsListed, deleteAllLocalDocuments, waitForDocumentListHintPersist } from './helpers';
import { setupTest, uploadFiles, ensureDocumentsListed, waitForDocumentListHintPersist } from './helpers';
test.describe('Document folders and hint persistence', () => {
test.beforeEach(async ({ page }) => {

View file

@ -212,32 +212,6 @@ export async function deleteAllLocalDocuments(page: Page) {
await page.keyboard.press('Escape');
}
// Extract the current list order (by visible .document-link elements)
export async function getNamesInOrder(page: Page): Promise<string[]> {
const texts = await page.locator('.document-link').allInnerTexts();
return texts.map(t => t.split('\n')[0].trim());
}
// Set sort field (by listbox button) to a given field label
export async function setSortField(page: Page, fieldLabel: string) {
// The listbox trigger shows current field (e.g. "Name"|"Size")
await page.getByRole('button', { name: /Name|Size|Date/i }).click();
await page.getByRole('option', { name: new RegExp(`^${escapeRegExp(fieldLabel)}$`, 'i') }).click();
// Verify it reflects the chosen field
await expect(page.getByRole('button', { name: new RegExp(`^${escapeRegExp(fieldLabel)}$`, 'i') })).toBeVisible();
}
// Ensure sort direction button shows an expected label (toggle as needed)
export async function ensureSortDirection(page: Page, expectedLabel: RegExp) {
// Direction button text is one of: A-Z, Z-A, Newest, Oldest, Smallest, Largest
const directionButton = page.getByRole('button', { name: /A-Z|Z-A|Newest|Oldest|Smallest|Largest/ });
const current = (await directionButton.textContent())?.trim() ?? '';
if (!expectedLabel.test(current)) {
await directionButton.click();
await expect(directionButton).toHaveText(expectedLabel);
}
}
// Open the Voices dropdown from the TTS bar and return the button locator
export async function openVoicesMenu(page: Page) {
const ttsbar = page.locator('[data-app-ttsbar]');
@ -305,37 +279,6 @@ export async function expectProcessingTransition(page: Page) {
await expectMediaState(page, 'playing');
}
// Open Speed popover in TTS bar
export async function openSpeedPopover(page: Page) {
const ttsbar = page.locator('[data-app-ttsbar]');
const buttons = ttsbar.getByRole('button');
// Heuristic: the Speed control is the first button in the TTS bar and shows something like "1x"
const speedBtn = buttons.first();
await expect(speedBtn).toBeVisible({ timeout: 10000 });
await speedBtn.click();
// Popover panel should appear with sliders
await page.waitForSelector('input[type="range"]', { timeout: 10000 });
}
// Change the "Native model speed" slider to a specific value and assert processing -> playing
export async function changeNativeSpeedAndAssert(page: Page, newSpeed: number) {
await openSpeedPopover(page);
const slider = page.locator('input[type="range"]').first();
// Set the slider value programmatically and dispatch events to trigger handlers
const valueStr = String(newSpeed);
await slider.evaluate((el, v) => {
const input = el as HTMLInputElement;
input.value = v;
input.dispatchEvent(new Event('input', { bubbles: true }));
input.dispatchEvent(new Event('mouseup', { bubbles: true }));
input.dispatchEvent(new KeyboardEvent('keyup', { bubbles: true, key: 'ArrowRight' }));
input.dispatchEvent(new Event('touchend', { bubbles: true }));
}, valueStr);
await expectProcessingTransition(page);
}
// Expect navigator.mediaSession.playbackState to equal given state
export async function expectMediaState(page: Page, state: 'playing' | 'paused') {
// WebKit (and sometimes other engines) may not reliably update navigator.mediaSession.playbackState.

View file

@ -6,13 +6,31 @@ import {
clickDocumentLink,
expectViewerForFile,
uploadAndDisplay,
navigateToPdfPageViaNavigator,
countRenderedPdfPages,
triggerViewportResize,
playTTSAndWaitForASecond,
expectProcessingTransition,
} from './helpers';
// Single-spec helpers kept local to avoid cluttering shared helpers:
async function navigateToPdfPageViaNavigator(page: any, targetPage: number) {
// Navigator popover shows "X / Y"
const navTrigger = page.getByRole('button', { name: /\d+\s*\/\s*\d+/ });
await expect(navTrigger).toBeVisible({ timeout: 10000 });
await navTrigger.click();
const input = page.getByLabel('Page number');
await expect(input).toBeVisible({ timeout: 10000 });
await input.fill(String(targetPage));
await input.press('Enter');
}
async function countRenderedPdfPages(page: any): Promise<number> {
return await page.locator('.react-pdf__Page').count();
}
async function triggerViewportResize(page: any, width: number, height: number) {
await page.setViewportSize({ width, height });
}
test.describe('Document link navigation by type', () => {
test.beforeEach(async ({ page }) => {
await setupTest(page);

View file

@ -2,14 +2,48 @@ import { test, expect } from '@playwright/test';
import {
setupTest,
playTTSAndWaitForASecond,
pauseTTSAndVerify,
openVoicesMenu,
selectVoiceAndAssertPlayback,
changeNativeSpeedAndAssert,
expectMediaState,
expectProcessingTransition,
} from './helpers';
// Single-spec helpers kept local instead of living in shared helpers:
async function pauseTTSAndVerify(page: any) {
// Click pause to stop playback
await page.getByRole('button', { name: 'Pause' }).click();
// Check for play button to be visible
await expect(page.getByRole('button', { name: 'Play' })).toBeVisible({ timeout: 10000 });
}
async function openSpeedPopover(page: any) {
const ttsbar = page.locator('[data-app-ttsbar]');
const buttons = ttsbar.getByRole('button');
// Heuristic: the Speed control is the first button in the TTS bar and shows something like "1x"
const speedBtn = buttons.first();
await expect(speedBtn).toBeVisible({ timeout: 10000 });
await speedBtn.click();
// Popover panel should appear with sliders
await page.waitForSelector('input[type="range"]', { timeout: 10000 });
}
async function changeNativeSpeedAndAssert(page: any, newSpeed: number) {
await openSpeedPopover(page);
const slider = page.locator('input[type="range"]').first();
// Set the slider value programmatically and dispatch events to trigger handlers
const valueStr = String(newSpeed);
await slider.evaluate((input: HTMLInputElement, v: string) => {
input.value = v;
input.dispatchEvent(new Event('input', { bubbles: true }));
input.dispatchEvent(new Event('mouseup', { bubbles: true }));
input.dispatchEvent(new KeyboardEvent('keyup', { bubbles: true, key: 'ArrowRight' }));
input.dispatchEvent(new Event('touchend', { bubbles: true }));
}, valueStr);
await expectProcessingTransition(page);
}
test.describe('Play/Pause Tests', () => {
test.beforeEach(async ({ page }) => {
await setupTest(page);