refactor(audio): replace ffprobe with ffmpeg for metadata extraction
- Remove ffprobe-static dependency and associated configuration - Reimplement ffprobeAudio to parse duration and title tags from ffmpeg output - Update CI workflow to install SeaweedFS binary and set required environment variables - Enhance E2E test setup to handle auth-protected endpoints during document cleanup
This commit is contained in:
parent
260513ebaa
commit
f9366dbfa2
8 changed files with 73 additions and 175 deletions
12
.github/workflows/playwright.yml
vendored
12
.github/workflows/playwright.yml
vendored
|
|
@ -11,6 +11,9 @@ jobs:
|
|||
env:
|
||||
FFMPEG_BIN: /usr/bin/ffmpeg
|
||||
FFPROBE_BIN: /usr/bin/ffprobe
|
||||
USE_EMBEDDED_WEED_MINI: true
|
||||
BASE_URL: http://127.0.0.1:3003
|
||||
S3_ENDPOINT: http://127.0.0.1:8333
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-node@v4
|
||||
|
|
@ -23,6 +26,15 @@ jobs:
|
|||
run: |
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y libreoffice-writer ffmpeg
|
||||
- name: Install SeaweedFS weed binary
|
||||
run: |
|
||||
WEED_PATH=$(docker run --rm --entrypoint sh chrislusf/seaweedfs:latest -lc 'command -v weed')
|
||||
CID=$(docker create chrislusf/seaweedfs:latest)
|
||||
docker cp "${CID}:${WEED_PATH}" ./weed
|
||||
docker rm "${CID}"
|
||||
sudo install -m 0755 ./weed /usr/local/bin/weed
|
||||
rm -f ./weed
|
||||
weed version
|
||||
- name: Install dependencies
|
||||
run: pnpm install --frozen-lockfile
|
||||
- name: Verify ffprobe
|
||||
|
|
|
|||
|
|
@ -6,19 +6,13 @@ const nextConfig: NextConfig = {
|
|||
canvas: '@napi-rs/canvas',
|
||||
},
|
||||
},
|
||||
serverExternalPackages: ["@napi-rs/canvas", "better-sqlite3", "ffmpeg-static", "ffprobe-static"],
|
||||
serverExternalPackages: ["@napi-rs/canvas", "better-sqlite3", "ffmpeg-static"],
|
||||
outputFileTracingIncludes: {
|
||||
'/api/audiobook': [
|
||||
'./node_modules/ffmpeg-static/ffmpeg',
|
||||
'./node_modules/ffprobe-static/bin/*/*/ffprobe',
|
||||
],
|
||||
'/api/audiobook/chapter': [
|
||||
'./node_modules/ffmpeg-static/ffmpeg',
|
||||
'./node_modules/ffprobe-static/bin/*/*/ffprobe',
|
||||
],
|
||||
'/api/audiobook/status': [
|
||||
'./node_modules/ffmpeg-static/ffmpeg',
|
||||
'./node_modules/ffprobe-static/bin/*/*/ffprobe',
|
||||
],
|
||||
'/api/whisper': [
|
||||
'./node_modules/ffmpeg-static/ffmpeg',
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
import { defineConfig, devices } from '@playwright/test';
|
||||
import 'dotenv/config';
|
||||
|
||||
/**
|
||||
* See https://playwright.dev/docs/test-configuration.
|
||||
|
|
|
|||
|
|
@ -24,7 +24,7 @@ import {
|
|||
import { isS3Configured } from '@/lib/server/s3';
|
||||
import { getOpenReaderTestNamespace, getUnclaimedUserIdForNamespace } from '@/lib/server/test-namespace';
|
||||
import { buildAllowedAudiobookUserIds, pickAudiobookOwner } from '@/lib/server/audiobook-scope';
|
||||
import { getFFmpegPath, getFFprobePath } from '@/lib/server/ffmpeg-bin';
|
||||
import { getFFmpegPath } from '@/lib/server/ffmpeg-bin';
|
||||
import type { AudiobookGenerationSettings } from '@/types/client';
|
||||
import type { TTSAudioBytes, TTSAudiobookFormat } from '@/types/tts';
|
||||
|
||||
|
|
@ -152,68 +152,6 @@ async function runFFmpeg(args: string[], signal?: AbortSignal): Promise<void> {
|
|||
});
|
||||
}
|
||||
|
||||
async function getAudioDuration(filePath: string, signal?: AbortSignal): Promise<number> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const ffprobe = spawn(getFFprobePath(), [
|
||||
'-i',
|
||||
filePath,
|
||||
'-show_entries',
|
||||
'format=duration',
|
||||
'-v',
|
||||
'quiet',
|
||||
'-of',
|
||||
'csv=p=0',
|
||||
]);
|
||||
|
||||
let output = '';
|
||||
let finished = false;
|
||||
|
||||
const onAbort = () => {
|
||||
if (finished) return;
|
||||
finished = true;
|
||||
try {
|
||||
ffprobe.kill('SIGKILL');
|
||||
} catch {}
|
||||
reject(new Error('ABORTED'));
|
||||
};
|
||||
|
||||
const cleanup = () => {
|
||||
if (finished) return;
|
||||
finished = true;
|
||||
signal?.removeEventListener('abort', onAbort);
|
||||
};
|
||||
|
||||
if (signal) {
|
||||
if (signal.aborted) {
|
||||
onAbort();
|
||||
return;
|
||||
}
|
||||
signal.addEventListener('abort', onAbort, { once: true });
|
||||
}
|
||||
|
||||
ffprobe.stdout.on('data', (data) => {
|
||||
output += data.toString();
|
||||
});
|
||||
|
||||
ffprobe.on('close', (code) => {
|
||||
if (finished) return;
|
||||
cleanup();
|
||||
if (code === 0) {
|
||||
const duration = parseFloat(output.trim());
|
||||
resolve(duration);
|
||||
} else {
|
||||
reject(new Error(`ffprobe process exited with code ${code}`));
|
||||
}
|
||||
});
|
||||
|
||||
ffprobe.on('error', (err) => {
|
||||
if (finished) return;
|
||||
cleanup();
|
||||
reject(err);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
function findChapterFileNameByIndex(fileNames: string[], index: number): { fileName: string; title: string; format: 'mp3' | 'm4b' } | null {
|
||||
const matches = fileNames
|
||||
.map((fileName) => {
|
||||
|
|
@ -377,7 +315,7 @@ export async function POST(request: NextRequest) {
|
|||
}
|
||||
|
||||
const probe = await ffprobeAudio(chapterOutputTempPath, request.signal);
|
||||
const duration = probe.durationSec ?? (await getAudioDuration(chapterOutputTempPath, request.signal));
|
||||
const duration = probe.durationSec ?? 0;
|
||||
|
||||
const finalChapterName = encodeChapterFileName(chapterIndex, data.chapterTitle, format);
|
||||
const finalChapterBytes = await readFile(chapterOutputTempPath);
|
||||
|
|
|
|||
|
|
@ -22,7 +22,7 @@ import {
|
|||
} from '@/lib/server/audiobook';
|
||||
import { isS3Configured } from '@/lib/server/s3';
|
||||
import { getOpenReaderTestNamespace, getUnclaimedUserIdForNamespace } from '@/lib/server/test-namespace';
|
||||
import { getFFmpegPath, getFFprobePath } from '@/lib/server/ffmpeg-bin';
|
||||
import { getFFmpegPath } from '@/lib/server/ffmpeg-bin';
|
||||
import { buildAllowedAudiobookUserIds, pickAudiobookOwner } from '@/lib/server/audiobook-scope';
|
||||
import type { TTSAudiobookFormat } from '@/types/tts';
|
||||
|
||||
|
|
@ -138,68 +138,6 @@ async function runFFmpeg(args: string[], signal?: AbortSignal): Promise<void> {
|
|||
});
|
||||
}
|
||||
|
||||
async function getAudioDuration(filePath: string, signal?: AbortSignal): Promise<number> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const ffprobe = spawn(getFFprobePath(), [
|
||||
'-i',
|
||||
filePath,
|
||||
'-show_entries',
|
||||
'format=duration',
|
||||
'-v',
|
||||
'quiet',
|
||||
'-of',
|
||||
'csv=p=0',
|
||||
]);
|
||||
|
||||
let output = '';
|
||||
let finished = false;
|
||||
|
||||
const onAbort = () => {
|
||||
if (finished) return;
|
||||
finished = true;
|
||||
try {
|
||||
ffprobe.kill('SIGKILL');
|
||||
} catch {}
|
||||
reject(new Error('ABORTED'));
|
||||
};
|
||||
|
||||
const cleanup = () => {
|
||||
if (finished) return;
|
||||
finished = true;
|
||||
signal?.removeEventListener('abort', onAbort);
|
||||
};
|
||||
|
||||
if (signal) {
|
||||
if (signal.aborted) {
|
||||
onAbort();
|
||||
return;
|
||||
}
|
||||
signal.addEventListener('abort', onAbort, { once: true });
|
||||
}
|
||||
|
||||
ffprobe.stdout.on('data', (data) => {
|
||||
output += data.toString();
|
||||
});
|
||||
|
||||
ffprobe.on('close', (code) => {
|
||||
if (finished) return;
|
||||
cleanup();
|
||||
if (code === 0) {
|
||||
const duration = parseFloat(output.trim());
|
||||
resolve(duration);
|
||||
} else {
|
||||
reject(new Error(`ffprobe process exited with code ${code}`));
|
||||
}
|
||||
});
|
||||
|
||||
ffprobe.on('error', (err) => {
|
||||
if (finished) return;
|
||||
cleanup();
|
||||
reject(err);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
export async function GET(request: NextRequest) {
|
||||
let workDir: string | null = null;
|
||||
try {
|
||||
|
|
@ -298,8 +236,6 @@ export async function GET(request: NextRequest) {
|
|||
const probe = await ffprobeAudio(localPath, request.signal);
|
||||
if (probe.durationSec && probe.durationSec > 0) {
|
||||
duration = probe.durationSec;
|
||||
} else {
|
||||
duration = await getAudioDuration(localPath, request.signal);
|
||||
}
|
||||
} catch {
|
||||
duration = 0;
|
||||
|
|
|
|||
|
|
@ -1,7 +1,6 @@
|
|||
import { spawn } from 'child_process';
|
||||
import path from 'path';
|
||||
import { readdir } from 'fs/promises';
|
||||
import { getFFprobePath } from '@/lib/server/ffmpeg-bin';
|
||||
|
||||
export type StoredChapter = {
|
||||
index: number;
|
||||
|
|
@ -77,26 +76,52 @@ type ProbeResult = {
|
|||
titleTag?: string;
|
||||
};
|
||||
|
||||
function parseDurationFromFFmpegStderr(stderr: string): number | undefined {
|
||||
const match = stderr.match(/Duration:\s*(\d+):(\d+):(\d+(?:\.\d+)?)/);
|
||||
if (!match) return undefined;
|
||||
|
||||
const hours = Number(match[1]);
|
||||
const minutes = Number(match[2]);
|
||||
const seconds = Number(match[3]);
|
||||
if (!Number.isFinite(hours) || !Number.isFinite(minutes) || !Number.isFinite(seconds)) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
const total = hours * 3600 + minutes * 60 + seconds;
|
||||
return Number.isFinite(total) ? total : undefined;
|
||||
}
|
||||
|
||||
function parseTitleFromFFMetadata(stdout: string): string | undefined {
|
||||
const line = stdout
|
||||
.split(/\r?\n/)
|
||||
.find((value) => value.startsWith('title='));
|
||||
if (!line) return undefined;
|
||||
|
||||
const raw = line.slice('title='.length).trim();
|
||||
return raw.length > 0 ? raw : undefined;
|
||||
}
|
||||
|
||||
export async function ffprobeAudio(filePath: string, signal?: AbortSignal): Promise<ProbeResult> {
|
||||
const { getFFmpegPath } = await import('@/lib/server/ffmpeg-bin');
|
||||
|
||||
return new Promise<ProbeResult>((resolve, reject) => {
|
||||
const ffprobe = spawn(getFFprobePath(), [
|
||||
'-v',
|
||||
'quiet',
|
||||
'-print_format',
|
||||
'json',
|
||||
'-show_entries',
|
||||
'format=duration:format_tags=title',
|
||||
const ffmpeg = spawn(getFFmpegPath(), [
|
||||
'-i',
|
||||
filePath,
|
||||
'-f',
|
||||
'ffmetadata',
|
||||
'-',
|
||||
]);
|
||||
|
||||
let output = '';
|
||||
let stdout = '';
|
||||
let stderr = '';
|
||||
let finished = false;
|
||||
|
||||
const onAbort = () => {
|
||||
if (finished) return;
|
||||
finished = true;
|
||||
try {
|
||||
ffprobe.kill('SIGKILL');
|
||||
ffmpeg.kill('SIGKILL');
|
||||
} catch {}
|
||||
reject(new Error('ABORTED'));
|
||||
};
|
||||
|
|
@ -115,34 +140,29 @@ export async function ffprobeAudio(filePath: string, signal?: AbortSignal): Prom
|
|||
signal.addEventListener('abort', onAbort, { once: true });
|
||||
}
|
||||
|
||||
ffprobe.stdout.on('data', (data) => {
|
||||
output += data.toString();
|
||||
ffmpeg.stdout.on('data', (data) => {
|
||||
stdout += data.toString();
|
||||
});
|
||||
|
||||
ffprobe.on('close', (code) => {
|
||||
ffmpeg.stderr.on('data', (data) => {
|
||||
stderr += data.toString();
|
||||
});
|
||||
|
||||
ffmpeg.on('close', (code) => {
|
||||
if (finished) return;
|
||||
cleanup();
|
||||
if (code !== 0) {
|
||||
reject(new Error(`ffprobe process exited with code ${code}`));
|
||||
reject(new Error(`ffmpeg probe process exited with code ${code}`));
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
const parsed = JSON.parse(output) as {
|
||||
format?: { duration?: string; tags?: { title?: string } };
|
||||
};
|
||||
const durationStr = parsed.format?.duration;
|
||||
const durationSec = durationStr ? Number(durationStr) : undefined;
|
||||
resolve({
|
||||
durationSec: Number.isFinite(durationSec) ? durationSec : undefined,
|
||||
titleTag: parsed.format?.tags?.title,
|
||||
});
|
||||
} catch (error) {
|
||||
reject(error);
|
||||
}
|
||||
resolve({
|
||||
durationSec: parseDurationFromFFmpegStderr(stderr),
|
||||
titleTag: parseTitleFromFFMetadata(stdout),
|
||||
});
|
||||
});
|
||||
|
||||
ffprobe.on('error', (err) => {
|
||||
ffmpeg.on('error', (err) => {
|
||||
if (finished) return;
|
||||
cleanup();
|
||||
reject(err);
|
||||
|
|
|
|||
|
|
@ -1,6 +1,5 @@
|
|||
import { existsSync } from 'fs';
|
||||
import ffmpegStatic from 'ffmpeg-static';
|
||||
import ffprobeStatic from 'ffprobe-static';
|
||||
|
||||
function normalizePath(value: unknown): string | null {
|
||||
if (typeof value !== 'string') return null;
|
||||
|
|
@ -36,12 +35,3 @@ export function getFFmpegPath(): string {
|
|||
'ffmpeg-static',
|
||||
);
|
||||
}
|
||||
|
||||
export function getFFprobePath(): string {
|
||||
return resolveBinary(
|
||||
normalizePath(process.env.FFPROBE_BIN),
|
||||
normalizePath(ffprobeStatic?.path),
|
||||
'FFPROBE_BIN',
|
||||
'ffprobe-static',
|
||||
);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -148,11 +148,18 @@ export async function setupTest(page: Page, testInfo?: TestInfo) {
|
|||
if (!isAuthEnabledForTests()) {
|
||||
const headers = namespace ? { 'x-openreader-test-namespace': namespace } : undefined;
|
||||
let cleared = false;
|
||||
let authProtected = false;
|
||||
let attempts = 0;
|
||||
while (!cleared && attempts < 3) {
|
||||
attempts += 1;
|
||||
try {
|
||||
const res = await page.request.delete('/api/documents', { ...(headers ? { headers } : {}) });
|
||||
// If this endpoint requires auth, we're not in no-auth mode for this run.
|
||||
// Skip cleanup rather than hard-failing setup.
|
||||
if (res.status() === 401 || res.status() === 403) {
|
||||
authProtected = true;
|
||||
break;
|
||||
}
|
||||
if (res.ok()) {
|
||||
cleared = true;
|
||||
break;
|
||||
|
|
@ -162,7 +169,7 @@ export async function setupTest(page: Page, testInfo?: TestInfo) {
|
|||
}
|
||||
await page.waitForTimeout(200);
|
||||
}
|
||||
if (!cleared) {
|
||||
if (!cleared && !authProtected) {
|
||||
throw new Error('Failed to clear server documents before test setup');
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue