From 2f39e8f014c7d0992a5e1bbb13a46d7afb8327c7 Mon Sep 17 00:00:00 2001 From: Richard Roberson Date: Fri, 14 Nov 2025 18:52:49 -0700 Subject: [PATCH] perf(test): optimize test execution and error handling - Reduce Playwright worker allocation from 75% to 50% for better resource management - Disable parallel test execution temporarily to isolate flaky test behavior - Enhance audio utility to fail fast on user cancellation, preventing unnecessary retries - Consolidate test-specific utilities within individual test suites for better isolation - Introduce export functionality test coverage with new export.spec.ts suite --- playwright.config.ts | 4 +- src/utils/audio.ts | 10 +- tests/export.spec.ts | 343 +++++++++++++++++++++++++++++++++++++++ tests/folders.spec.ts | 2 +- tests/helpers.ts | 57 ------- tests/navigation.spec.ts | 24 ++- tests/play.spec.ts | 38 ++++- 7 files changed, 411 insertions(+), 67 deletions(-) create mode 100644 tests/export.spec.ts diff --git a/playwright.config.ts b/playwright.config.ts index 35258ca..f5ad03b 100644 --- a/playwright.config.ts +++ b/playwright.config.ts @@ -7,11 +7,11 @@ export default defineConfig({ testDir: './tests', timeout: 30 * 1000, outputDir: './tests/results', - fullyParallel: false, + // fullyParallel: false, /* Fail the build on CI if you accidentally left test.only in the source code. */ forbidOnly: !!process.env.CI, retries: process.env.CI ? 2 : 0, - workers: process.env.CI ? '100%' : '75%', + workers: process.env.CI ? '100%' : '50%', /* Reporter to use. See https://playwright.dev/docs/test-reporters */ reporter: 'html', /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */ diff --git a/src/utils/audio.ts b/src/utils/audio.ts index 4712bc8..69034c4 100644 --- a/src/utils/audio.ts +++ b/src/utils/audio.ts @@ -39,7 +39,13 @@ export const withRetry = async ( return await operation(); } catch (error) { lastError = error instanceof Error ? error : new Error(String(error)); - + + // Do not retry on explicit cancellation/abort errors - surface them + // immediately so callers can stop work quickly when the user cancels. + if (lastError.name === 'AbortError' || lastError.message.includes('cancelled')) { + break; + } + if (attempt === maxRetries - 1) { break; } @@ -130,4 +136,4 @@ export const combineAudioChunks = async ( } finally { if (setIsAudioCombining) setIsAudioCombining(false); } -} \ No newline at end of file +} diff --git a/tests/export.spec.ts b/tests/export.spec.ts new file mode 100644 index 0000000..43d6302 --- /dev/null +++ b/tests/export.spec.ts @@ -0,0 +1,343 @@ +import { test, expect, Page } from '@playwright/test'; +import fs from 'fs'; +import util from 'util'; +import { execFile } from 'child_process'; +import { setupTest, uploadAndDisplay } from './helpers'; + +const execFileAsync = util.promisify(execFile); + +async function getBookIdFromUrl(page: Page, expectedPrefix: 'pdf' | 'epub') { + const url = new URL(page.url()); + const segments = url.pathname.split('/').filter(Boolean); + expect(segments[0]).toBe(expectedPrefix); + const bookId = segments[1]; + expect(bookId).toBeTruthy(); + return bookId; +} + +async function openExportModal(page: Page) { + const exportButton = page.getByRole('button', { name: 'Open audiobook export' }); + await expect(exportButton).toBeVisible({ timeout: 15_000 }); + await exportButton.click(); + await expect(page.getByRole('heading', { name: 'Export Audiobook' })).toBeVisible({ timeout: 15_000 }); +} + +async function setContainerFormatToMP3(page: Page) { + const formatTrigger = page.getByRole('button', { name: /M4B|MP3/i }); + await expect(formatTrigger).toBeVisible({ timeout: 15_000 }); + await formatTrigger.click(); + await page.getByRole('option', { name: 'MP3' }).click(); +} + +async function startGeneration(page: Page) { + const startButton = page.getByRole('button', { name: 'Start Generation' }); + await expect(startButton).toBeVisible({ timeout: 15_000 }); + await startButton.click(); +} + +async function waitForChaptersHeading(page: Page) { + await expect(page.getByRole('heading', { name: 'Chapters' })).toBeVisible({ timeout: 60_000 }); +} + +async function downloadFullAudiobook(page: Page, timeoutMs = 60_000) { + const fullDownloadButton = page.getByRole('button', { name: /Full Download/i }); + await expect(fullDownloadButton).toBeVisible({ timeout: timeoutMs }); + const [download] = await Promise.all([ + page.waitForEvent('download', { timeout: timeoutMs }), + fullDownloadButton.click(), + ]); + const downloadedPath = await download.path(); + expect(downloadedPath).toBeTruthy(); + const stats = fs.statSync(downloadedPath!); + expect(stats.size).toBeGreaterThan(0); + return downloadedPath!; +} + +async function getAudioDurationSeconds(filePath: string) { + const { stdout } = await execFileAsync('ffprobe', [ + '-v', + 'error', + '-show_entries', + 'format=duration', + '-of', + 'csv=p=0', + filePath, + ]); + return parseFloat(stdout.trim()); +} + +async function expectChaptersBackendState(page: Page, bookId: string) { + const res = await page.request.get(`/api/audio/convert/chapters?bookId=${bookId}`); + expect(res.ok()).toBeTruthy(); + const json = await res.json(); + return json; +} + +async function resetAudiobookIfPresent(page: Page) { + const resetButtons = page.getByRole('button', { name: 'Reset' }); + const count = await resetButtons.count(); + + if (count === 0) { + return; + } + + const resetButton = resetButtons.first(); + await resetButton.click(); + + await expect(page.getByRole('heading', { name: 'Reset Audiobook' })).toBeVisible({ timeout: 15_000 }); + const confirmReset = page.getByRole('button', { name: 'Reset' }).last(); + await confirmReset.click(); + + await expect( + page.getByText(/Click "Start Generation" to begin creating your audiobook/i) + ).toBeVisible({ timeout: 60_000 }); +} + +test.describe('Audiobook export', () => { + test.describe.configure({ mode: 'serial', timeout: 120_000 }); + + test('exports full MP3 audiobook for PDF using mocked 10s TTS sample', async ({ page }) => { + // Ensure TTS is mocked and app is ready + await setupTest(page); + + // Upload and open the sample PDF in the viewer + await uploadAndDisplay(page, 'sample.pdf'); + + // Capture the generated document/book id from the /pdf/[id] URL + const bookId = await getBookIdFromUrl(page, 'pdf'); + + // Open the audiobook export modal from the header button + await openExportModal(page); + + // While there are no chapters yet, we can still switch the container format. + // Choose MP3 so we can validate MP3 duration end-to-end. + await setContainerFormatToMP3(page); + + // Start generation; this will call the mocked /api/tts which returns a 10s sample.mp3 per page + await startGeneration(page); + + // Wait for chapters list to appear and populate at least two items (Pages 1 and 2) + await waitForChaptersHeading(page); + const chapterActionsButtons = page.getByRole('button', { name: 'Chapter actions' }); + await expect(chapterActionsButtons).toHaveCount(2, { timeout: 60_000 }); + + // Trigger full download from the FRONTEND button and capture via Playwright's download API. + // The button label can be "Full Download (MP3)" or "Full Download (M4B)" depending on + // the server-side detected format, so match more loosely on the accessible name. + const downloadedPath = await downloadFullAudiobook(page); + + // Use ffprobe (same toolchain as the server) to validate the combined audio duration. + // The TTS route is mocked to return a 10s sample.mp3 for each page, so with at least + // two chapters we should be close to ~20 seconds of audio. + const durationSeconds = await getAudioDurationSeconds(downloadedPath); + // Duration must be within a reasonable window around 20 seconds to allow + // for encoding variations and container overhead. + expect(durationSeconds).toBeGreaterThan(18); + expect(durationSeconds).toBeLessThan(22); + + // Also check the chapter metadata API for consistency + const json = await expectChaptersBackendState(page, bookId); + expect(json.exists).toBe(true); + expect(Array.isArray(json.chapters)).toBe(true); + expect(json.chapters.length).toBeGreaterThanOrEqual(2); + for (const ch of json.chapters) { + expect(ch.duration).toBeGreaterThan(0); + } + + await resetAudiobookIfPresent(page); + }); + + test('handles partial EPUB audiobook generation, cancel, and full download of partial audiobook', async ({ page }) => { + await setupTest(page); + + // Upload and open the sample EPUB in the viewer + await uploadAndDisplay(page, 'sample.epub'); + + // URL should now be /epub/[id] + const bookId = await getBookIdFromUrl(page, 'epub'); + + // Open the audiobook export modal from the header button + await openExportModal(page); + + // Set container format to MP3 + await setContainerFormatToMP3(page); + + // Start generation + await startGeneration(page); + + // Progress card should appear with a Cancel button while chapters are being generated + const cancelButton = page.getByRole('button', { name: 'Cancel' }); + await expect(cancelButton).toBeVisible({ timeout: 60_000 }); + + await expect(page.getByRole('heading', { name: 'Chapters' })).toBeVisible({ timeout: 60_000 }); + + // Wait until at least 3 chapters are listed in the UI; record the exact count at the + // moment we decide to cancel, and assert that no additional chapters are added afterward. + const chapterActionsButtons = page.getByRole('button', { name: 'Chapter actions' }); + await expect(chapterActionsButtons.nth(2)).toBeVisible({ timeout: 120_000 }); + const chapterCountBeforeCancel = await chapterActionsButtons.count(); + expect(chapterCountBeforeCancel).toBeGreaterThanOrEqual(3); + + // Now cancel the in-flight generation + await cancelButton.click(); + + // After cancellation, the inline progress card's Cancel button should be gone + await expect(page.getByRole('button', { name: 'Cancel' })).toHaveCount(0); + + // After cancellation, determine the canonical chapter count from the backend and + // assert that the UI eventually reflects this count. Some in-flight chapters may + // complete right as we cancel, so we treat the backend state as source of truth. + const jsonAfterCancel = await expectChaptersBackendState(page, bookId); + expect(jsonAfterCancel.exists).toBe(true); + expect(Array.isArray(jsonAfterCancel.chapters)).toBe(true); + const chapterCountAfterCancel = jsonAfterCancel.chapters.length; + expect(chapterCountAfterCancel).toBeGreaterThanOrEqual(chapterCountBeforeCancel); + + // Wait for the UI to reflect the final backend chapter count to avoid race + // conditions between the modal's soft refresh and our assertions. + await expect(chapterActionsButtons).toHaveCount(chapterCountAfterCancel, { timeout: 60_000 }); + + // The Full Download button should still be available for the partially generated audiobook + const downloadedPath = await downloadFullAudiobook(page); + + const durationSeconds = await getAudioDurationSeconds(downloadedPath); + expect(durationSeconds).toBeGreaterThan(25); + expect(durationSeconds).toBeLessThan(300); + + // Backend should still reflect the same number of chapters as when we first + // observed the stabilized post-cancellation state, and should not contain + // additional "impartial" chapters produced after cancellation. + const json = await expectChaptersBackendState(page, bookId); + expect(json.exists).toBe(true); + expect(Array.isArray(json.chapters)).toBe(true); + expect(json.chapters.length).toBe(chapterCountAfterCancel); + + await resetAudiobookIfPresent(page); + }); + + test('downloads a single chapter via chapter actions menu (PDF)', async ({ page }) => { + await setupTest(page); + await uploadAndDisplay(page, 'sample.pdf'); + + const bookId = await getBookIdFromUrl(page, 'pdf'); + + await openExportModal(page); + await setContainerFormatToMP3(page); + await startGeneration(page); + + await waitForChaptersHeading(page); + + // Wait for at least one chapter row to appear (one "Chapter actions" button) + const chapterActionsButtons = page.getByRole('button', { name: 'Chapter actions' }); + await expect(chapterActionsButtons.first()).toBeVisible({ timeout: 90_000 }); + + // Download via frontend button + const downloadedPath = await downloadFullAudiobook(page); + + const durationSeconds = await getAudioDurationSeconds(downloadedPath); + // For EPUB we just assert a sane non-trivial duration; at least one 10s mocked chapter. + expect(durationSeconds).toBeGreaterThan(9); + expect(durationSeconds).toBeLessThan(300); + + await resetAudiobookIfPresent(page); + }); + + test('reset removes all generated chapters for a PDF audiobook', async ({ page }) => { + await setupTest(page); + await uploadAndDisplay(page, 'sample.pdf'); + + const bookId = await getBookIdFromUrl(page, 'pdf'); + + await openExportModal(page); + await setContainerFormatToMP3(page); + await startGeneration(page); + + await waitForChaptersHeading(page); + + // Wait for Reset button to become visible, indicating resumable/generated state + const resetButton = page.getByRole('button', { name: 'Reset' }); + await expect(resetButton).toBeVisible({ timeout: 120_000 }); + + await resetButton.click(); + + // Confirm in the Reset Audiobook dialog + await expect(page.getByRole('heading', { name: 'Reset Audiobook' })).toBeVisible({ timeout: 15000 }); + const confirmReset = page.getByRole('button', { name: 'Reset' }).last(); + await confirmReset.click(); + + // After reset, the hint text for starting generation should re-appear + await expect( + page.getByText(/Click "Start Generation" to begin creating your audiobook/i) + ).toBeVisible({ timeout: 60_000 }); + + // Backend should report no existing chapters for this bookId + const res = await page.request.get(`/api/audio/convert/chapters?bookId=${bookId}`); + expect(res.ok()).toBeTruthy(); + const json = await res.json(); + expect(json.exists).toBe(false); + expect(Array.isArray(json.chapters)).toBe(true); + expect(json.chapters.length).toBe(0); + }); + + test('regenerates a PDF audiobook chapter and preserves chapter count and full download', async ({ page }) => { + await setupTest(page); + await uploadAndDisplay(page, 'sample.pdf'); + + // Extract bookId from /pdf/[id] URL (for backend verification later) + const bookId = await getBookIdFromUrl(page, 'pdf'); + + // Open Export Audiobook modal + await openExportModal(page); + + // Set container format to MP3 + await setContainerFormatToMP3(page); + + // Start generation + await startGeneration(page); + + // Wait for chapters to appear + await waitForChaptersHeading(page); + + const chapterActionsButtons = page.getByRole('button', { name: 'Chapter actions' }); + // Ensure we have at least two chapters for this PDF + await expect(chapterActionsButtons.nth(1)).toBeVisible({ timeout: 60_000 }); + const chapterCountBefore = await chapterActionsButtons.count(); + expect(chapterCountBefore).toBeGreaterThanOrEqual(2); + + // Open the actions menu for the first chapter and trigger Regenerate + const firstChapterActions = chapterActionsButtons.first(); + await firstChapterActions.click(); + + // In the headlessui Menu, each option is a menuitem. Use that role instead of button. + const regenerateMenuItem = page.getByRole('menuitem', { name: /Regenerate/i }); + await expect(regenerateMenuItem).toBeVisible({ timeout: 15000 }); + await regenerateMenuItem.click(); + + // During regeneration, the row may show a "Regenerating" label; wait for any such + // indicator to disappear, signaling completion. + const regeneratingLabel = page.getByText(/Regenerating/); + await expect(regeneratingLabel).toHaveCount(0, { timeout: 120_000 }); + + // Chapter count should remain exactly the same after regeneration (no duplicates) + await expect(chapterActionsButtons).toHaveCount(chapterCountBefore, { timeout: 20_000 }); + + // Full Download should still work and produce a valid combined audiobook + const downloadedPath = await downloadFullAudiobook(page); + + const durationSeconds = await getAudioDurationSeconds(downloadedPath); + // With two mocked 10s chapters we expect roughly 20s; allow a small window. + expect(durationSeconds).toBeGreaterThan(18); + expect(durationSeconds).toBeLessThan(22); + + // Backend should still report the same number of chapters and valid durations + const json = await expectChaptersBackendState(page, bookId); + expect(json.exists).toBe(true); + expect(Array.isArray(json.chapters)).toBe(true); + expect(json.chapters.length).toBe(chapterCountBefore); + for (const ch of json.chapters) { + expect(ch.duration).toBeGreaterThan(0); + } + + await resetAudiobookIfPresent(page); + }); +}); diff --git a/tests/folders.spec.ts b/tests/folders.spec.ts index 268da6c..70c238d 100644 --- a/tests/folders.spec.ts +++ b/tests/folders.spec.ts @@ -1,5 +1,5 @@ import { test, expect } from '@playwright/test'; -import { setupTest, uploadFiles, ensureDocumentsListed, deleteAllLocalDocuments, waitForDocumentListHintPersist } from './helpers'; +import { setupTest, uploadFiles, ensureDocumentsListed, waitForDocumentListHintPersist } from './helpers'; test.describe('Document folders and hint persistence', () => { test.beforeEach(async ({ page }) => { diff --git a/tests/helpers.ts b/tests/helpers.ts index 886efd1..dd1a2df 100644 --- a/tests/helpers.ts +++ b/tests/helpers.ts @@ -212,32 +212,6 @@ export async function deleteAllLocalDocuments(page: Page) { await page.keyboard.press('Escape'); } -// Extract the current list order (by visible .document-link elements) -export async function getNamesInOrder(page: Page): Promise { - const texts = await page.locator('.document-link').allInnerTexts(); - return texts.map(t => t.split('\n')[0].trim()); -} - -// Set sort field (by listbox button) to a given field label -export async function setSortField(page: Page, fieldLabel: string) { - // The listbox trigger shows current field (e.g. "Name"|"Size") - await page.getByRole('button', { name: /Name|Size|Date/i }).click(); - await page.getByRole('option', { name: new RegExp(`^${escapeRegExp(fieldLabel)}$`, 'i') }).click(); - // Verify it reflects the chosen field - await expect(page.getByRole('button', { name: new RegExp(`^${escapeRegExp(fieldLabel)}$`, 'i') })).toBeVisible(); -} - -// Ensure sort direction button shows an expected label (toggle as needed) -export async function ensureSortDirection(page: Page, expectedLabel: RegExp) { - // Direction button text is one of: A-Z, Z-A, Newest, Oldest, Smallest, Largest - const directionButton = page.getByRole('button', { name: /A-Z|Z-A|Newest|Oldest|Smallest|Largest/ }); - const current = (await directionButton.textContent())?.trim() ?? ''; - if (!expectedLabel.test(current)) { - await directionButton.click(); - await expect(directionButton).toHaveText(expectedLabel); - } -} - // Open the Voices dropdown from the TTS bar and return the button locator export async function openVoicesMenu(page: Page) { const ttsbar = page.locator('[data-app-ttsbar]'); @@ -305,37 +279,6 @@ export async function expectProcessingTransition(page: Page) { await expectMediaState(page, 'playing'); } -// Open Speed popover in TTS bar -export async function openSpeedPopover(page: Page) { - const ttsbar = page.locator('[data-app-ttsbar]'); - const buttons = ttsbar.getByRole('button'); - // Heuristic: the Speed control is the first button in the TTS bar and shows something like "1x" - const speedBtn = buttons.first(); - await expect(speedBtn).toBeVisible({ timeout: 10000 }); - await speedBtn.click(); - // Popover panel should appear with sliders - await page.waitForSelector('input[type="range"]', { timeout: 10000 }); -} - -// Change the "Native model speed" slider to a specific value and assert processing -> playing -export async function changeNativeSpeedAndAssert(page: Page, newSpeed: number) { - await openSpeedPopover(page); - const slider = page.locator('input[type="range"]').first(); - - // Set the slider value programmatically and dispatch events to trigger handlers - const valueStr = String(newSpeed); - await slider.evaluate((el, v) => { - const input = el as HTMLInputElement; - input.value = v; - input.dispatchEvent(new Event('input', { bubbles: true })); - input.dispatchEvent(new Event('mouseup', { bubbles: true })); - input.dispatchEvent(new KeyboardEvent('keyup', { bubbles: true, key: 'ArrowRight' })); - input.dispatchEvent(new Event('touchend', { bubbles: true })); - }, valueStr); - - await expectProcessingTransition(page); -} - // Expect navigator.mediaSession.playbackState to equal given state export async function expectMediaState(page: Page, state: 'playing' | 'paused') { // WebKit (and sometimes other engines) may not reliably update navigator.mediaSession.playbackState. diff --git a/tests/navigation.spec.ts b/tests/navigation.spec.ts index 0eaf9a0..f116774 100644 --- a/tests/navigation.spec.ts +++ b/tests/navigation.spec.ts @@ -6,13 +6,31 @@ import { clickDocumentLink, expectViewerForFile, uploadAndDisplay, - navigateToPdfPageViaNavigator, - countRenderedPdfPages, - triggerViewportResize, playTTSAndWaitForASecond, expectProcessingTransition, } from './helpers'; +// Single-spec helpers kept local to avoid cluttering shared helpers: +async function navigateToPdfPageViaNavigator(page: any, targetPage: number) { + // Navigator popover shows "X / Y" + const navTrigger = page.getByRole('button', { name: /\d+\s*\/\s*\d+/ }); + await expect(navTrigger).toBeVisible({ timeout: 10000 }); + await navTrigger.click(); + + const input = page.getByLabel('Page number'); + await expect(input).toBeVisible({ timeout: 10000 }); + await input.fill(String(targetPage)); + await input.press('Enter'); +} + +async function countRenderedPdfPages(page: any): Promise { + return await page.locator('.react-pdf__Page').count(); +} + +async function triggerViewportResize(page: any, width: number, height: number) { + await page.setViewportSize({ width, height }); +} + test.describe('Document link navigation by type', () => { test.beforeEach(async ({ page }) => { await setupTest(page); diff --git a/tests/play.spec.ts b/tests/play.spec.ts index 5e7b186..57caaac 100644 --- a/tests/play.spec.ts +++ b/tests/play.spec.ts @@ -2,14 +2,48 @@ import { test, expect } from '@playwright/test'; import { setupTest, playTTSAndWaitForASecond, - pauseTTSAndVerify, openVoicesMenu, selectVoiceAndAssertPlayback, - changeNativeSpeedAndAssert, expectMediaState, expectProcessingTransition, } from './helpers'; +// Single-spec helpers kept local instead of living in shared helpers: +async function pauseTTSAndVerify(page: any) { + // Click pause to stop playback + await page.getByRole('button', { name: 'Pause' }).click(); + // Check for play button to be visible + await expect(page.getByRole('button', { name: 'Play' })).toBeVisible({ timeout: 10000 }); +} + +async function openSpeedPopover(page: any) { + const ttsbar = page.locator('[data-app-ttsbar]'); + const buttons = ttsbar.getByRole('button'); + // Heuristic: the Speed control is the first button in the TTS bar and shows something like "1x" + const speedBtn = buttons.first(); + await expect(speedBtn).toBeVisible({ timeout: 10000 }); + await speedBtn.click(); + // Popover panel should appear with sliders + await page.waitForSelector('input[type="range"]', { timeout: 10000 }); +} + +async function changeNativeSpeedAndAssert(page: any, newSpeed: number) { + await openSpeedPopover(page); + const slider = page.locator('input[type="range"]').first(); + + // Set the slider value programmatically and dispatch events to trigger handlers + const valueStr = String(newSpeed); + await slider.evaluate((input: HTMLInputElement, v: string) => { + input.value = v; + input.dispatchEvent(new Event('input', { bubbles: true })); + input.dispatchEvent(new Event('mouseup', { bubbles: true })); + input.dispatchEvent(new KeyboardEvent('keyup', { bubbles: true, key: 'ArrowRight' })); + input.dispatchEvent(new Event('touchend', { bubbles: true })); + }, valueStr); + + await expectProcessingTransition(page); +} + test.describe('Play/Pause Tests', () => { test.beforeEach(async ({ page }) => { await setupTest(page);