refactor(upload): unify docx-to-pdf conversion into finalize handler and remove legacy route
Remove the separate docx-to-pdf upload API and integrate DOCX conversion directly into the blob upload finalize route. Update client and uploader logic to treat DOCX as a supported upload type and streamline batch state handling. Adjust document caching and test helpers to align with the new unified flow. Add server-side DOCX-to-PDF conversion utility and corresponding unit tests. This change simplifies the upload pipeline, consolidates conversion logic, and improves maintainability.
This commit is contained in:
Richard R
parent
7344fcbc51
commit
2ecd281f75
11 changed files with 316 additions and 370 deletions
|
|
@ -1,3 +1,4 @@
|
|||
import path from 'path';
|
||||
import { createHash } from 'node:crypto';
|
||||
import { NextRequest, NextResponse } from 'next/server';
|
||||
import { requireAuthContext } from '@/lib/server/auth/auth';
|
||||
|
|
@ -12,8 +13,10 @@ import {
|
|||
isMissingBlobError,
|
||||
isPreconditionFailed,
|
||||
isValidTempUploadToken,
|
||||
putDocumentBlob,
|
||||
putTempDocumentFinalizeReceipt,
|
||||
} from '@/lib/server/documents/blobstore';
|
||||
import { convertDocxBufferToPdfBuffer } from '@/lib/server/documents/docx-convert';
|
||||
import { registerUploadedDocument } from '@/lib/server/documents/register-upload';
|
||||
import { safeDocumentName, toDocumentTypeFromName } from '@/lib/server/documents/utils';
|
||||
import { errorResponse } from '@/lib/server/errors/next-response';
|
||||
|
|
@ -133,23 +136,48 @@ async function finalizeOne(input: {
|
|||
throw new Error('Temporary upload expired before finalize');
|
||||
}
|
||||
|
||||
const documentId = createHash('sha256').update(temp.body).digest('hex');
|
||||
const isDocxUpload = input.upload.type === 'docx';
|
||||
const finalizedType: DocumentType = isDocxUpload ? 'pdf' : input.upload.type;
|
||||
const finalizedBody = isDocxUpload
|
||||
? await convertDocxBufferToPdfBuffer(temp.body)
|
||||
: temp.body;
|
||||
const finalizedContentType = finalizedType === 'pdf'
|
||||
? 'application/pdf'
|
||||
: temp.contentType;
|
||||
const finalizedName = finalizedType === 'pdf' && input.upload.type === 'docx'
|
||||
? safeDocumentName(`${path.parse(input.upload.name).name}.pdf`, 'upload.pdf')
|
||||
: input.upload.name;
|
||||
const documentId = createHash('sha256').update(finalizedBody).digest('hex');
|
||||
|
||||
try {
|
||||
await headDocumentBlob(documentId, input.namespace);
|
||||
} catch (error) {
|
||||
if (!isMissingBlobError(error)) throw error;
|
||||
try {
|
||||
await copyTempDocumentBlobToDocument(
|
||||
input.upload.token,
|
||||
input.userId,
|
||||
documentId,
|
||||
input.namespace,
|
||||
temp.contentType,
|
||||
{ ifNoneMatch: true },
|
||||
);
|
||||
} catch (copyError) {
|
||||
if (!isPreconditionFailed(copyError)) throw copyError;
|
||||
if (!isDocxUpload) {
|
||||
try {
|
||||
await copyTempDocumentBlobToDocument(
|
||||
input.upload.token,
|
||||
input.userId,
|
||||
documentId,
|
||||
input.namespace,
|
||||
finalizedContentType,
|
||||
{ ifNoneMatch: true },
|
||||
);
|
||||
} catch (copyError) {
|
||||
if (!isPreconditionFailed(copyError)) throw copyError;
|
||||
}
|
||||
} else {
|
||||
try {
|
||||
await putDocumentBlob(
|
||||
documentId,
|
||||
finalizedBody,
|
||||
finalizedContentType,
|
||||
input.namespace,
|
||||
{ ifNoneMatch: true },
|
||||
);
|
||||
} catch (putError) {
|
||||
if (!isPreconditionFailed(putError)) throw putError;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -158,9 +186,9 @@ async function finalizeOne(input: {
|
|||
documentId,
|
||||
userId: input.userId,
|
||||
namespace: input.namespace,
|
||||
name: input.upload.name,
|
||||
type: input.upload.type,
|
||||
size: canonicalHead.contentLength > 0 ? canonicalHead.contentLength : temp.size,
|
||||
name: finalizedName,
|
||||
type: finalizedType,
|
||||
size: canonicalHead.contentLength > 0 ? canonicalHead.contentLength : finalizedBody.byteLength,
|
||||
lastModified: input.upload.lastModified,
|
||||
});
|
||||
|
||||
|
|
|
|||
|
|
@ -1,225 +0,0 @@
|
|||
import { NextRequest, NextResponse } from 'next/server';
|
||||
import { writeFile, mkdir, readFile, readdir, rm, stat } from 'fs/promises';
|
||||
import { spawn } from 'child_process';
|
||||
import path from 'path';
|
||||
import { existsSync } from 'fs';
|
||||
import { randomUUID, createHash } from 'crypto';
|
||||
import { pathToFileURL } from 'url';
|
||||
import { requireAuthContext } from '@/lib/server/auth/auth';
|
||||
import { db } from '@/db';
|
||||
import { documents } from '@/db/schema';
|
||||
import { findReusableParsedPdfResult } from '@/lib/server/documents/parsed-pdf-reuse';
|
||||
import { safeDocumentName } from '@/lib/server/documents/utils';
|
||||
import { enqueueDocumentPreview } from '@/lib/server/documents/previews';
|
||||
import { startPdfParseOperation } from '@/lib/server/documents/pdf-parse-operation';
|
||||
import { enqueueParsePdfJob } from '@/lib/server/jobs/user-pdf-layout-job';
|
||||
import { stringifyDocumentParseState } from '@/lib/server/documents/parse-state';
|
||||
import { getOpenReaderTestNamespace } from '@/lib/server/testing/test-namespace';
|
||||
import { isS3Configured } from '@/lib/server/storage/s3';
|
||||
import { putDocumentBlob } from '@/lib/server/documents/blobstore';
|
||||
import { errorToLog, serverLogger } from '@/lib/server/logger';
|
||||
import { errorResponse } from '@/lib/server/errors/next-response';
|
||||
import { PDF_PARSER_VERSION } from '@openreader/compute-core';
|
||||
|
||||
const DOCSTORE_DIR = path.join(process.cwd(), 'docstore');
|
||||
const TEMP_DIR = path.join(DOCSTORE_DIR, 'tmp');
|
||||
|
||||
async function ensureTempDir() {
|
||||
if (!existsSync(DOCSTORE_DIR)) {
|
||||
await mkdir(DOCSTORE_DIR, { recursive: true });
|
||||
}
|
||||
if (!existsSync(TEMP_DIR)) {
|
||||
await mkdir(TEMP_DIR, { recursive: true });
|
||||
}
|
||||
}
|
||||
|
||||
async function convertDocxToPdf(inputPath: string, outputDir: string, profileDir?: string): Promise<void> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const args: string[] = [];
|
||||
if (profileDir) {
|
||||
args.push(`-env:UserInstallation=${pathToFileURL(profileDir).toString()}`);
|
||||
}
|
||||
args.push('--headless', '--nologo', '--convert-to', 'pdf', '--outdir', outputDir, inputPath);
|
||||
const proc = spawn('soffice', args);
|
||||
|
||||
proc.on('error', (error) => reject(error));
|
||||
proc.on('close', (code) => {
|
||||
if (code === 0) resolve();
|
||||
else reject(new Error(`LibreOffice conversion failed with code ${code}`));
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
async function waitForPdfReady(dir: string, timeoutMs = 20000, intervalMs = 100): Promise<string> {
|
||||
const end = Date.now() + timeoutMs;
|
||||
while (Date.now() < end) {
|
||||
const files = await readdir(dir);
|
||||
const pdf = files.find((f) => f.toLowerCase().endsWith('.pdf'));
|
||||
if (pdf) {
|
||||
const pdfPath = path.join(dir, pdf);
|
||||
try {
|
||||
const first = await stat(pdfPath);
|
||||
await new Promise((res) => setTimeout(res, intervalMs));
|
||||
const second = await stat(pdfPath);
|
||||
if (second.size > 0 && second.size === first.size) {
|
||||
return pdfPath;
|
||||
}
|
||||
} catch {
|
||||
// ignore transient errors
|
||||
}
|
||||
}
|
||||
await new Promise((res) => setTimeout(res, intervalMs));
|
||||
}
|
||||
throw new Error(`PDF not ready in ${dir} after ${timeoutMs}ms`);
|
||||
}
|
||||
|
||||
export async function POST(req: NextRequest) {
|
||||
try {
|
||||
if (!isS3Configured()) {
|
||||
return NextResponse.json(
|
||||
{ error: 'Documents storage is not configured. Set S3_* environment variables.' },
|
||||
{ status: 503 },
|
||||
);
|
||||
}
|
||||
|
||||
const testNamespace = getOpenReaderTestNamespace(req.headers);
|
||||
|
||||
const ctxOrRes = await requireAuthContext(req);
|
||||
if (ctxOrRes instanceof Response) return ctxOrRes;
|
||||
if (!ctxOrRes.userId) return NextResponse.json({ error: 'Unauthorized' }, { status: 401 });
|
||||
const storageUserId = ctxOrRes.userId;
|
||||
|
||||
await ensureTempDir();
|
||||
|
||||
const formData = await req.formData();
|
||||
const file = formData.get('file');
|
||||
if (!(file instanceof File)) {
|
||||
return NextResponse.json({ error: 'No file provided' }, { status: 400 });
|
||||
}
|
||||
|
||||
if (!file.name.toLowerCase().endsWith('.docx')) {
|
||||
return NextResponse.json({ error: 'File must be a .docx document' }, { status: 400 });
|
||||
}
|
||||
|
||||
const docxBytes = Buffer.from(await file.arrayBuffer());
|
||||
// Keep stable IDs tied to source bytes.
|
||||
const id = createHash('sha256').update(docxBytes).digest('hex');
|
||||
|
||||
const tempId = randomUUID();
|
||||
const jobDir = path.join(TEMP_DIR, tempId);
|
||||
await mkdir(jobDir, { recursive: true });
|
||||
const profileDir = path.join(jobDir, 'lo-profile');
|
||||
await mkdir(profileDir, { recursive: true });
|
||||
const inputPath = path.join(jobDir, 'input.docx');
|
||||
|
||||
await writeFile(inputPath, docxBytes);
|
||||
|
||||
try {
|
||||
await convertDocxToPdf(inputPath, jobDir, profileDir);
|
||||
const pdfPath = await waitForPdfReady(jobDir);
|
||||
const pdfContent = await readFile(pdfPath);
|
||||
|
||||
try {
|
||||
await putDocumentBlob(id, pdfContent, 'application/pdf', testNamespace);
|
||||
} catch (error) {
|
||||
// Idempotent behavior: if blob already exists for this sha, continue.
|
||||
const maybe = error as { name?: string; $metadata?: { httpStatusCode?: number } } | undefined;
|
||||
const isPreconditionFailed = maybe?.$metadata?.httpStatusCode === 412 || maybe?.name === 'PreconditionFailed';
|
||||
if (!isPreconditionFailed) {
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
const derivedName = safeDocumentName(`${path.parse(file.name).name}.pdf`, `${id}.pdf`);
|
||||
const lastModified = Number.isFinite(file.lastModified) ? file.lastModified : Date.now();
|
||||
const reusableParsedPdf = await findReusableParsedPdfResult(id);
|
||||
const startedParse = reusableParsedPdf
|
||||
? null
|
||||
: await startPdfParseOperation({
|
||||
documentId: id,
|
||||
userId: storageUserId,
|
||||
namespace: testNamespace,
|
||||
});
|
||||
const parseState = stringifyDocumentParseState(
|
||||
reusableParsedPdf
|
||||
? { status: 'ready', progress: null, updatedAt: Date.now(), parserVersion: PDF_PARSER_VERSION }
|
||||
: startedParse!.parseState,
|
||||
);
|
||||
|
||||
await db
|
||||
.insert(documents)
|
||||
.values({
|
||||
id,
|
||||
userId: storageUserId,
|
||||
name: derivedName,
|
||||
type: 'pdf',
|
||||
size: pdfContent.length,
|
||||
lastModified,
|
||||
filePath: id,
|
||||
parseState,
|
||||
parsedJsonKey: reusableParsedPdf?.parsedJsonKey ?? null,
|
||||
})
|
||||
.onConflictDoUpdate({
|
||||
target: [documents.id, documents.userId],
|
||||
set: {
|
||||
name: derivedName,
|
||||
type: 'pdf',
|
||||
size: pdfContent.length,
|
||||
lastModified,
|
||||
filePath: id,
|
||||
parseState,
|
||||
parsedJsonKey: reusableParsedPdf?.parsedJsonKey ?? null,
|
||||
},
|
||||
});
|
||||
|
||||
await enqueueDocumentPreview(
|
||||
{
|
||||
id,
|
||||
type: 'pdf',
|
||||
lastModified,
|
||||
},
|
||||
testNamespace,
|
||||
).catch((error) => {
|
||||
serverLogger.warn({
|
||||
event: 'documents.docx.preview_enqueue.failed',
|
||||
degraded: true,
|
||||
fallbackPath: 'skip_preview_enqueue',
|
||||
documentId: id,
|
||||
error: errorToLog(error),
|
||||
}, 'Failed to enqueue preview for converted DOCX');
|
||||
});
|
||||
|
||||
if (startedParse) {
|
||||
enqueueParsePdfJob({
|
||||
documentId: id,
|
||||
userId: storageUserId,
|
||||
namespace: testNamespace,
|
||||
initialOpId: startedParse.workerState.opId,
|
||||
initialJobId: startedParse.workerState.jobId,
|
||||
initialStatus: startedParse.parseState.status === 'running' ? 'running' : 'pending',
|
||||
});
|
||||
}
|
||||
|
||||
return NextResponse.json({
|
||||
stored: {
|
||||
id,
|
||||
name: derivedName,
|
||||
type: 'pdf',
|
||||
size: pdfContent.length,
|
||||
lastModified,
|
||||
},
|
||||
});
|
||||
} finally {
|
||||
await rm(jobDir, { recursive: true, force: true }).catch(() => {});
|
||||
}
|
||||
} catch (error) {
|
||||
serverLogger.error({
|
||||
event: 'documents.docx.convert_upload.failed',
|
||||
error: errorToLog(error),
|
||||
}, 'Failed converting/uploading DOCX');
|
||||
return errorResponse(error, {
|
||||
apiErrorMessage: 'Failed to convert document',
|
||||
normalize: { code: 'DOCUMENTS_DOCX_CONVERT_UPLOAD_FAILED', errorClass: 'upstream' },
|
||||
});
|
||||
}
|
||||
}
|
||||
|
|
@ -122,16 +122,14 @@ interface DocumentListInnerProps {
|
|||
function SidebarUploadLoader({
|
||||
totalFiles,
|
||||
completedFiles,
|
||||
phase,
|
||||
currentFileName,
|
||||
}: {
|
||||
totalFiles: number;
|
||||
completedFiles: number;
|
||||
phase: 'uploading' | 'converting';
|
||||
phase: 'uploading';
|
||||
currentFileName: string | null;
|
||||
}) {
|
||||
const progress = totalFiles > 0 ? Math.min(100, Math.round((completedFiles / totalFiles) * 100)) : 0;
|
||||
const label = phase === 'converting' ? 'Converting' : 'Uploading';
|
||||
const radius = 7;
|
||||
const stroke = 2;
|
||||
const size = 18;
|
||||
|
|
@ -143,12 +141,12 @@ function SidebarUploadLoader({
|
|||
<div className="rounded-md border border-line bg-surface-sunken px-2 py-1.5">
|
||||
<div className="flex items-center justify-between gap-2">
|
||||
<div className="min-w-0 flex items-center gap-1.5 text-[11px] leading-tight">
|
||||
<span className="font-medium text-foreground">{label}</span>
|
||||
<span className="font-medium text-foreground">Uploading</span>
|
||||
<span className="shrink-0 tabular-nums text-soft">{completedFiles}/{totalFiles}</span>
|
||||
</div>
|
||||
<div className="shrink-0 flex items-center gap-1 text-accent" aria-label={`Upload progress ${progress}%`}>
|
||||
<span className="text-[10px] tabular-nums text-soft">{progress}%</span>
|
||||
<svg width={size} height={size} viewBox={`0 0 ${size} ${size}`} className={phase === 'converting' ? 'animate-spin' : ''}>
|
||||
<svg width={size} height={size} viewBox={`0 0 ${size} ${size}`}>
|
||||
<circle
|
||||
cx={size / 2}
|
||||
cy={size / 2}
|
||||
|
|
@ -573,10 +571,8 @@ function DocumentListInner({ brand, appActions }: DocumentListInnerProps) {
|
|||
if (batches.length === 0) return null;
|
||||
const totalFiles = batches.reduce((sum, batch) => sum + batch.totalFiles, 0);
|
||||
const completedFiles = batches.reduce((sum, batch) => sum + batch.completedFiles, 0);
|
||||
const convertingBatch = batches.find((batch) => batch.phase === 'converting');
|
||||
const phase: 'uploading' | 'converting' = convertingBatch ? 'converting' : 'uploading';
|
||||
const currentFileName = convertingBatch?.currentFileName ?? batches.find((batch) => batch.currentFileName)?.currentFileName ?? null;
|
||||
return { totalFiles, completedFiles, phase, currentFileName };
|
||||
const currentFileName = batches.find((batch) => batch.currentFileName)?.currentFileName ?? null;
|
||||
return { totalFiles, completedFiles, phase: 'uploading' as const, currentFileName };
|
||||
}, [activeUploadBatches]);
|
||||
|
||||
const fallbackViewMode: ViewMode = viewMode;
|
||||
|
|
|
|||
|
|
@ -4,7 +4,6 @@ import { useState, useCallback, useId, type ReactNode } from 'react';
|
|||
import { useDropzone } from 'react-dropzone';
|
||||
import { UploadIcon } from '@/components/icons/Icons';
|
||||
import { useDocuments } from '@/contexts/DocumentContext';
|
||||
import { uploadDocxAsPdf } from '@/lib/client/api/documents';
|
||||
import { useFeatureFlag } from '@/contexts/RuntimeConfigContext';
|
||||
import { dropzoneSurfaceClass } from '@/components/ui';
|
||||
|
||||
|
|
@ -20,7 +19,7 @@ export interface UploadBatchState {
|
|||
isActive: boolean;
|
||||
totalFiles: number;
|
||||
completedFiles: number;
|
||||
phase: 'uploading' | 'converting';
|
||||
phase: 'uploading';
|
||||
currentFileName: string | null;
|
||||
}
|
||||
|
||||
|
|
@ -34,10 +33,8 @@ export function DocumentUploader({
|
|||
const enableDocx = useFeatureFlag('enableDocxConversion');
|
||||
const {
|
||||
uploadDocuments,
|
||||
refreshDocuments,
|
||||
} = useDocuments();
|
||||
const [isUploading, setIsUploading] = useState(false);
|
||||
const [isConverting, setIsConverting] = useState(false);
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
|
||||
const emitBatchState = useCallback((state: Omit<UploadBatchState, 'uploaderId'>) => {
|
||||
|
|
@ -61,75 +58,24 @@ export function DocumentUploader({
|
|||
});
|
||||
|
||||
try {
|
||||
const docxFiles: File[] = [];
|
||||
const regularFiles: File[] = [];
|
||||
const uploadableFiles = acceptedFiles.filter((file) =>
|
||||
file.type === 'application/pdf'
|
||||
|| file.type === 'application/epub+zip'
|
||||
|| file.type === 'text/plain'
|
||||
|| file.type === 'text/markdown'
|
||||
|| file.name.endsWith('.md')
|
||||
|| (enableDocx && file.type === 'application/vnd.openxmlformats-officedocument.wordprocessingml.document'),
|
||||
);
|
||||
|
||||
for (const file of acceptedFiles) {
|
||||
if (enableDocx && file.type === 'application/vnd.openxmlformats-officedocument.wordprocessingml.document') {
|
||||
docxFiles.push(file);
|
||||
continue;
|
||||
}
|
||||
if (
|
||||
file.type === 'application/pdf'
|
||||
|| file.type === 'application/epub+zip'
|
||||
|| file.type === 'text/plain'
|
||||
|| file.type === 'text/markdown'
|
||||
|| file.name.endsWith('.md')
|
||||
) {
|
||||
regularFiles.push(file);
|
||||
}
|
||||
}
|
||||
|
||||
if (regularFiles.length > 0) {
|
||||
emitBatchState({
|
||||
isActive: true,
|
||||
totalFiles,
|
||||
completedFiles,
|
||||
phase: 'uploading',
|
||||
currentFileName: regularFiles[0]?.name ?? null,
|
||||
});
|
||||
await uploadDocuments(regularFiles);
|
||||
completedFiles += regularFiles.length;
|
||||
emitBatchState({
|
||||
isActive: true,
|
||||
totalFiles,
|
||||
completedFiles,
|
||||
phase: 'uploading',
|
||||
currentFileName: null,
|
||||
});
|
||||
}
|
||||
|
||||
for (const file of docxFiles) {
|
||||
// Preserve prior UX: show "Converting DOCX..." state rather than generic uploading.
|
||||
setIsUploading(false);
|
||||
setIsConverting(true);
|
||||
emitBatchState({
|
||||
isActive: true,
|
||||
totalFiles,
|
||||
completedFiles,
|
||||
phase: 'converting',
|
||||
currentFileName: file.name,
|
||||
});
|
||||
await uploadDocxAsPdf(file);
|
||||
await refreshDocuments();
|
||||
setIsConverting(false);
|
||||
setIsUploading(true);
|
||||
completedFiles += 1;
|
||||
|
||||
emitBatchState({
|
||||
isActive: true,
|
||||
totalFiles,
|
||||
completedFiles,
|
||||
phase: 'uploading',
|
||||
currentFileName: null,
|
||||
});
|
||||
if (uploadableFiles.length > 0) {
|
||||
await uploadDocuments(uploadableFiles);
|
||||
completedFiles = uploadableFiles.length;
|
||||
}
|
||||
} catch (err) {
|
||||
setError('Failed to upload file. Please try again.');
|
||||
console.error('Upload error:', err);
|
||||
} finally {
|
||||
setIsUploading(false);
|
||||
setIsConverting(false);
|
||||
emitBatchState({
|
||||
isActive: false,
|
||||
totalFiles,
|
||||
|
|
@ -138,7 +84,7 @@ export function DocumentUploader({
|
|||
currentFileName: null,
|
||||
});
|
||||
}
|
||||
}, [uploadDocuments, refreshDocuments, enableDocx, emitBatchState]);
|
||||
}, [uploadDocuments, enableDocx, emitBatchState]);
|
||||
|
||||
const { getRootProps, getInputProps, isDragActive } = useDropzone({
|
||||
onDrop,
|
||||
|
|
@ -152,12 +98,12 @@ export function DocumentUploader({
|
|||
} : {})
|
||||
},
|
||||
multiple: true,
|
||||
disabled: isUploading || isConverting,
|
||||
disabled: isUploading,
|
||||
noClick: variant === 'overlay',
|
||||
noKeyboard: variant === 'overlay'
|
||||
});
|
||||
|
||||
const isDisabled = isUploading || isConverting;
|
||||
const isDisabled = isUploading;
|
||||
|
||||
if (variant === 'overlay') {
|
||||
const rootProps = getRootProps();
|
||||
|
|
@ -210,8 +156,6 @@ export function DocumentUploader({
|
|||
<UploadIcon className="w-3.5 h-3.5 text-soft group-hover:text-accent shrink-0 transition-colors duration-base" />
|
||||
{isUploading ? (
|
||||
<p className="text-[12px] font-medium truncate flex-1">Uploading…</p>
|
||||
) : isConverting ? (
|
||||
<p className="text-[12px] font-medium truncate flex-1">Converting DOCX…</p>
|
||||
) : (
|
||||
<div className="flex items-center gap-2 min-w-0 flex-1">
|
||||
<p className="text-[12px] truncate flex-1">
|
||||
|
|
@ -226,8 +170,6 @@ export function DocumentUploader({
|
|||
<UploadIcon className="w-7 h-7 sm:w-10 sm:h-10 mb-2 text-soft" />
|
||||
{isUploading ? (
|
||||
<p className="text-sm sm:text-lg font-semibold text-foreground">Uploading file...</p>
|
||||
) : isConverting ? (
|
||||
<p className="text-sm sm:text-lg font-semibold text-foreground">Converting DOCX to PDF...</p>
|
||||
) : (
|
||||
<>
|
||||
<p className="mb-2 text-sm sm:text-lg font-semibold text-foreground">
|
||||
|
|
|
|||
|
|
@ -110,6 +110,26 @@ export function DocumentProvider({ children }: { children: ReactNode }) {
|
|||
stored.map(async (document, index) => {
|
||||
const file = files[index];
|
||||
if (!file) return;
|
||||
const sourceType = file.name
|
||||
? (
|
||||
file.name.toLowerCase().endsWith('.pdf')
|
||||
? 'pdf'
|
||||
: file.name.toLowerCase().endsWith('.epub')
|
||||
? 'epub'
|
||||
: file.name.toLowerCase().endsWith('.docx')
|
||||
? 'docx'
|
||||
: 'html'
|
||||
)
|
||||
: (
|
||||
file.type === 'application/pdf'
|
||||
? 'pdf'
|
||||
: file.type === 'application/epub+zip'
|
||||
? 'epub'
|
||||
: file.type === 'application/vnd.openxmlformats-officedocument.wordprocessingml.document'
|
||||
? 'docx'
|
||||
: 'html'
|
||||
);
|
||||
if (document.type !== sourceType) return;
|
||||
await cacheStoredDocumentFromBytes(document, await file.arrayBuffer());
|
||||
}),
|
||||
);
|
||||
|
|
|
|||
|
|
@ -504,23 +504,3 @@ export async function getDocumentPreviewStatus(
|
|||
|
||||
throw new Error(`Failed to load preview status (status ${res.status})`);
|
||||
}
|
||||
|
||||
export async function uploadDocxAsPdf(file: File, options?: { signal?: AbortSignal }): Promise<BaseDocument> {
|
||||
const form = new FormData();
|
||||
form.append('file', file);
|
||||
|
||||
const res = await fetch('/api/documents/docx-to-pdf/upload', {
|
||||
method: 'POST',
|
||||
body: form,
|
||||
signal: options?.signal,
|
||||
});
|
||||
|
||||
if (!res.ok) {
|
||||
const data = (await res.json().catch(() => null)) as { error?: string } | null;
|
||||
throw new Error(data?.error || 'Failed to convert DOCX');
|
||||
}
|
||||
|
||||
const data = (await res.json()) as { stored: BaseDocument };
|
||||
if (!data?.stored) throw new Error('DOCX conversion succeeded but returned no document');
|
||||
return data.stored;
|
||||
}
|
||||
|
|
|
|||
78
src/lib/server/documents/docx-convert.ts
Normal file
78
src/lib/server/documents/docx-convert.ts
Normal file
|
|
@ -0,0 +1,78 @@
|
|||
import { spawn } from 'child_process';
|
||||
import { existsSync } from 'fs';
|
||||
import { mkdir, readFile, readdir, rm, stat, writeFile } from 'fs/promises';
|
||||
import path from 'path';
|
||||
import { randomUUID } from 'node:crypto';
|
||||
import { pathToFileURL } from 'url';
|
||||
|
||||
const DOCSTORE_DIR = path.join(process.cwd(), 'docstore');
|
||||
const TEMP_DIR = path.join(DOCSTORE_DIR, 'tmp');
|
||||
|
||||
async function ensureTempDir(): Promise<void> {
|
||||
if (!existsSync(DOCSTORE_DIR)) {
|
||||
await mkdir(DOCSTORE_DIR, { recursive: true });
|
||||
}
|
||||
if (!existsSync(TEMP_DIR)) {
|
||||
await mkdir(TEMP_DIR, { recursive: true });
|
||||
}
|
||||
}
|
||||
|
||||
async function convertDocxToPdf(inputPath: string, outputDir: string, profileDir?: string): Promise<void> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const args: string[] = [];
|
||||
if (profileDir) {
|
||||
args.push(`-env:UserInstallation=${pathToFileURL(profileDir).toString()}`);
|
||||
}
|
||||
args.push('--headless', '--nologo', '--convert-to', 'pdf', '--outdir', outputDir, inputPath);
|
||||
const proc = spawn('soffice', args);
|
||||
|
||||
proc.on('error', (error) => reject(error));
|
||||
proc.on('close', (code) => {
|
||||
if (code === 0) resolve();
|
||||
else reject(new Error(`LibreOffice conversion failed with code ${code}`));
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
async function waitForPdfReady(dir: string, timeoutMs = 20_000, intervalMs = 100): Promise<string> {
|
||||
const end = Date.now() + timeoutMs;
|
||||
while (Date.now() < end) {
|
||||
const files = await readdir(dir);
|
||||
const pdf = files.find((f) => f.toLowerCase().endsWith('.pdf'));
|
||||
if (pdf) {
|
||||
const pdfPath = path.join(dir, pdf);
|
||||
try {
|
||||
const first = await stat(pdfPath);
|
||||
await new Promise((resolve) => setTimeout(resolve, intervalMs));
|
||||
const second = await stat(pdfPath);
|
||||
if (second.size > 0 && second.size === first.size) {
|
||||
return pdfPath;
|
||||
}
|
||||
} catch {
|
||||
// Ignore transient filesystem races while LibreOffice is still writing.
|
||||
}
|
||||
}
|
||||
await new Promise((resolve) => setTimeout(resolve, intervalMs));
|
||||
}
|
||||
throw new Error(`PDF not ready in ${dir} after ${timeoutMs}ms`);
|
||||
}
|
||||
|
||||
export async function convertDocxBufferToPdfBuffer(docxBytes: Buffer): Promise<Buffer> {
|
||||
await ensureTempDir();
|
||||
|
||||
const tempId = randomUUID();
|
||||
const jobDir = path.join(TEMP_DIR, tempId);
|
||||
await mkdir(jobDir, { recursive: true });
|
||||
const profileDir = path.join(jobDir, 'lo-profile');
|
||||
await mkdir(profileDir, { recursive: true });
|
||||
const inputPath = path.join(jobDir, 'input.docx');
|
||||
|
||||
try {
|
||||
await writeFile(inputPath, docxBytes);
|
||||
await convertDocxToPdf(inputPath, jobDir, profileDir);
|
||||
const pdfPath = await waitForPdfReady(jobDir);
|
||||
return await readFile(pdfPath);
|
||||
} finally {
|
||||
await rm(jobDir, { recursive: true, force: true }).catch(() => {});
|
||||
}
|
||||
}
|
||||
|
|
@ -11,7 +11,6 @@ export interface BaseDocument {
|
|||
parsedJsonKey?: string | null;
|
||||
scope?: 'user';
|
||||
folderId?: string;
|
||||
isConverting?: boolean;
|
||||
}
|
||||
|
||||
export interface PDFDocument extends BaseDocument {
|
||||
|
|
|
|||
|
|
@ -1,6 +1,4 @@
|
|||
import { Page, expect, type TestInfo, type Locator } from '@playwright/test';
|
||||
import fs from 'fs';
|
||||
import path from 'path';
|
||||
import { createHash } from 'crypto';
|
||||
|
||||
const DIR = './tests/files/';
|
||||
|
|
@ -10,14 +8,6 @@ function escapeRegExp(input: string) {
|
|||
return input.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
|
||||
}
|
||||
|
||||
function fixturePath(fileName: string) {
|
||||
return path.join(__dirname, 'files', fileName);
|
||||
}
|
||||
|
||||
function sha256HexOfFile(filePath: string) {
|
||||
return createHash('sha256').update(fs.readFileSync(filePath)).digest('hex');
|
||||
}
|
||||
|
||||
async function waitForPdfViewerReady(page: Page, timeout = 60000) {
|
||||
await expect(page).toHaveURL(/\/pdf\/[A-Za-z0-9._%-]+$/, { timeout: Math.min(timeout, 20000) });
|
||||
const loader = page.getByTestId('pdf-status-loader').first();
|
||||
|
|
@ -40,7 +30,7 @@ async function waitForPdfViewerReady(page: Page, timeout = 60000) {
|
|||
}
|
||||
|
||||
/**
|
||||
* Upload a sample epub or pdf
|
||||
* Upload a sample document fixture
|
||||
*/
|
||||
export async function uploadFile(page: Page, filePath: string) {
|
||||
const input = page.locator('input[type=file]').first();
|
||||
|
|
@ -50,7 +40,7 @@ export async function uploadFile(page: Page, filePath: string) {
|
|||
await input.setInputFiles(`${DIR}${filePath}`);
|
||||
|
||||
// Wait for the uploader to finish processing. The input is disabled while
|
||||
// uploading/converting via react-dropzone's `disabled` prop.
|
||||
// uploading via react-dropzone's `disabled` prop.
|
||||
// Tolerate extremely fast operations where the disabled state may be missed.
|
||||
try {
|
||||
await expect(input).toBeDisabled({ timeout: 2000 });
|
||||
|
|
@ -69,14 +59,8 @@ export async function uploadAndDisplay(page: Page, fileName: string) {
|
|||
const lower = fileName.toLowerCase();
|
||||
|
||||
if (lower.endsWith('.docx')) {
|
||||
// Best-effort: conversion can complete before we observe this UI state.
|
||||
try {
|
||||
await expect(page.getByText('Converting DOCX to PDF...')).toBeVisible({ timeout: 5000 });
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
const expectedId = sha256HexOfFile(fixturePath(fileName));
|
||||
const targetLink = page.locator(`a[href$="/pdf/${expectedId}"]`).first();
|
||||
const convertedName = `${fileName.replace(/\.[^.]+$/, '')}.pdf`;
|
||||
const targetLink = page.getByRole('link', { name: new RegExp(escapeRegExp(convertedName), 'i') }).first();
|
||||
await expect(targetLink).toBeVisible({ timeout: 15000 });
|
||||
await dismissOnboardingModals(page);
|
||||
await targetLink.click();
|
||||
|
|
|
|||
150
tests/unit/blob-upload-finalize-docx.vitest.spec.ts
Normal file
150
tests/unit/blob-upload-finalize-docx.vitest.spec.ts
Normal file
|
|
@ -0,0 +1,150 @@
|
|||
import { createHash } from 'node:crypto';
|
||||
import { beforeEach, describe, expect, test, vi } from 'vitest';
|
||||
import { NextRequest } from 'next/server';
|
||||
|
||||
const hoisted = vi.hoisted(() => ({
|
||||
requireAuthContext: vi.fn(),
|
||||
registerUploadedDocument: vi.fn(),
|
||||
convertDocxBufferToPdfBuffer: vi.fn(),
|
||||
headTempDocumentBlob: vi.fn(),
|
||||
getTempDocumentBlob: vi.fn(),
|
||||
getTempDocumentFinalizeReceipt: vi.fn(),
|
||||
putTempDocumentFinalizeReceipt: vi.fn(),
|
||||
deleteTempDocumentUpload: vi.fn(),
|
||||
headDocumentBlob: vi.fn(),
|
||||
copyTempDocumentBlobToDocument: vi.fn(),
|
||||
putDocumentBlob: vi.fn(),
|
||||
}));
|
||||
|
||||
vi.mock('@/lib/server/auth/auth', () => ({
|
||||
requireAuthContext: hoisted.requireAuthContext,
|
||||
}));
|
||||
|
||||
vi.mock('@/lib/server/documents/register-upload', () => ({
|
||||
registerUploadedDocument: hoisted.registerUploadedDocument,
|
||||
}));
|
||||
|
||||
vi.mock('@/lib/server/documents/docx-convert', () => ({
|
||||
convertDocxBufferToPdfBuffer: hoisted.convertDocxBufferToPdfBuffer,
|
||||
}));
|
||||
|
||||
vi.mock('@/lib/server/documents/blobstore', () => ({
|
||||
TEMP_DOCUMENT_UPLOAD_TTL_MS: 24 * 60 * 60 * 1000,
|
||||
copyTempDocumentBlobToDocument: hoisted.copyTempDocumentBlobToDocument,
|
||||
deleteTempDocumentUpload: hoisted.deleteTempDocumentUpload,
|
||||
getTempDocumentBlob: hoisted.getTempDocumentBlob,
|
||||
getTempDocumentFinalizeReceipt: hoisted.getTempDocumentFinalizeReceipt,
|
||||
headDocumentBlob: hoisted.headDocumentBlob,
|
||||
headTempDocumentBlob: hoisted.headTempDocumentBlob,
|
||||
isMissingBlobError: vi.fn((error: unknown) => {
|
||||
const maybe = error as { code?: string } | undefined;
|
||||
return maybe?.code === 'NoSuchKey';
|
||||
}),
|
||||
isPreconditionFailed: vi.fn(() => false),
|
||||
isValidTempUploadToken: vi.fn(() => true),
|
||||
putDocumentBlob: hoisted.putDocumentBlob,
|
||||
putTempDocumentFinalizeReceipt: hoisted.putTempDocumentFinalizeReceipt,
|
||||
}));
|
||||
|
||||
vi.mock('@/lib/server/testing/test-namespace', () => ({
|
||||
getOpenReaderTestNamespace: vi.fn(() => null),
|
||||
}));
|
||||
|
||||
vi.mock('@/lib/server/storage/s3', () => ({
|
||||
isS3Configured: vi.fn(() => true),
|
||||
}));
|
||||
|
||||
vi.mock('@/lib/server/logger', () => ({
|
||||
errorToLog: vi.fn((error: unknown) => error),
|
||||
serverLogger: {
|
||||
warn: vi.fn(),
|
||||
error: vi.fn(),
|
||||
},
|
||||
}));
|
||||
|
||||
describe('POST /api/documents/blob/upload/finalize DOCX flow', () => {
|
||||
beforeEach(() => {
|
||||
hoisted.requireAuthContext.mockReset();
|
||||
hoisted.registerUploadedDocument.mockReset();
|
||||
hoisted.convertDocxBufferToPdfBuffer.mockReset();
|
||||
hoisted.headTempDocumentBlob.mockReset();
|
||||
hoisted.getTempDocumentBlob.mockReset();
|
||||
hoisted.getTempDocumentFinalizeReceipt.mockReset();
|
||||
hoisted.putTempDocumentFinalizeReceipt.mockReset();
|
||||
hoisted.deleteTempDocumentUpload.mockReset();
|
||||
hoisted.headDocumentBlob.mockReset();
|
||||
hoisted.copyTempDocumentBlobToDocument.mockReset();
|
||||
hoisted.putDocumentBlob.mockReset();
|
||||
|
||||
hoisted.requireAuthContext.mockResolvedValue({ userId: 'user-1' });
|
||||
hoisted.getTempDocumentFinalizeReceipt.mockResolvedValue(null);
|
||||
hoisted.headTempDocumentBlob.mockResolvedValue({
|
||||
contentType: 'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
|
||||
contentLength: 12,
|
||||
lastModified: Date.now(),
|
||||
});
|
||||
hoisted.getTempDocumentBlob.mockResolvedValue(Buffer.from('docx-bytes'));
|
||||
hoisted.convertDocxBufferToPdfBuffer.mockResolvedValue(Buffer.from('pdf-bytes'));
|
||||
hoisted.headDocumentBlob
|
||||
.mockRejectedValueOnce({ code: 'NoSuchKey' })
|
||||
.mockResolvedValue({
|
||||
contentLength: Buffer.byteLength('pdf-bytes'),
|
||||
contentType: 'application/pdf',
|
||||
eTag: 'etag-1',
|
||||
});
|
||||
hoisted.registerUploadedDocument.mockImplementation(async (input: { documentId: string; name: string; type: string; size: number; lastModified: number }) => ({
|
||||
id: input.documentId,
|
||||
name: input.name,
|
||||
type: input.type,
|
||||
size: input.size,
|
||||
lastModified: input.lastModified,
|
||||
scope: 'user',
|
||||
}));
|
||||
hoisted.putTempDocumentFinalizeReceipt.mockResolvedValue(undefined);
|
||||
hoisted.deleteTempDocumentUpload.mockResolvedValue(undefined);
|
||||
});
|
||||
|
||||
test('converts raw DOCX during finalize and registers a PDF', async () => {
|
||||
const { POST } = await import('../../src/app/api/documents/blob/upload/finalize/route');
|
||||
const request = new NextRequest('http://localhost/api/documents/blob/upload/finalize', {
|
||||
method: 'POST',
|
||||
body: JSON.stringify({
|
||||
uploads: [{
|
||||
token: '123e4567-e89b-12d3-a456-426614174000',
|
||||
name: 'Report.docx',
|
||||
type: 'docx',
|
||||
lastModified: 1700000000000,
|
||||
}],
|
||||
}),
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
});
|
||||
|
||||
const response = await POST(request);
|
||||
const expectedId = createHash('sha256').update(Buffer.from('pdf-bytes')).digest('hex');
|
||||
|
||||
expect(response.status).toBe(200);
|
||||
await expect(response.json()).resolves.toMatchObject({
|
||||
stored: [{
|
||||
id: expectedId,
|
||||
name: 'Report.pdf',
|
||||
type: 'pdf',
|
||||
}],
|
||||
});
|
||||
expect(hoisted.convertDocxBufferToPdfBuffer).toHaveBeenCalledTimes(1);
|
||||
expect(hoisted.putDocumentBlob).toHaveBeenCalledWith(
|
||||
expectedId,
|
||||
Buffer.from('pdf-bytes'),
|
||||
'application/pdf',
|
||||
null,
|
||||
{ ifNoneMatch: true },
|
||||
);
|
||||
expect(hoisted.copyTempDocumentBlobToDocument).not.toHaveBeenCalled();
|
||||
expect(hoisted.registerUploadedDocument).toHaveBeenCalledWith(expect.objectContaining({
|
||||
documentId: expectedId,
|
||||
name: 'Report.pdf',
|
||||
type: 'pdf',
|
||||
}));
|
||||
});
|
||||
});
|
||||
|
|
@ -100,13 +100,7 @@ test.describe('Document Upload Tests', () => {
|
|||
|
||||
test('uploads and converts a DOCX document', async ({ page }) => {
|
||||
await uploadFile(page, 'sample.docx');
|
||||
// Should see the converting message (best-effort; conversion may complete extremely fast)
|
||||
try {
|
||||
await expect(page.getByText('Converting DOCX to PDF...')).toBeVisible({ timeout: 5000 });
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
// After conversion, should see the PDF with the same name
|
||||
// DOCX uploads are normalized into stored PDFs with the same basename.
|
||||
await expectDocumentListed(page, 'sample.pdf');
|
||||
});
|
||||
|
||||
|
|
|
|||
Loading…
Reference in a new issue