import { CopyObjectCommand, DeleteObjectCommand, DeleteObjectsCommand, GetObjectCommand, HeadObjectCommand, ListObjectsV2Command, PutObjectCommand, } from '@aws-sdk/client-s3'; import { getSignedUrl } from '@aws-sdk/s3-request-presigner'; import { PDF_PARSER_VERSION, encodeParserVersion } from '@openreader/compute-core/api-contracts'; import { getS3Client, getS3Config, getS3ProxyClient } from '@/lib/server/storage/s3'; const DOCUMENT_ID_REGEX = /^[a-f0-9]{64}$/i; const SAFE_NAMESPACE_REGEX = /^[a-zA-Z0-9._-]{1,128}$/; const TEMP_UPLOAD_TOKEN_REGEX = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i; const TEMP_UPLOAD_USER_ID_REGEX = /^[a-zA-Z0-9._:-]{1,256}$/; export const TEMP_DOCUMENT_UPLOAD_TTL_MS = 24 * 60 * 60 * 1000; function sanitizeNamespace(namespace: string | null): string | null { if (!namespace) return null; if (!SAFE_NAMESPACE_REGEX.test(namespace)) return null; return namespace; } function isNodeReadableStream(value: unknown): value is NodeJS.ReadableStream { return !!value && typeof value === 'object' && 'on' in value && typeof (value as NodeJS.ReadableStream).on === 'function'; } export type DocumentBlobBody = | NodeJS.ReadableStream | ReadableStream | Uint8Array | ArrayBuffer | ArrayBufferView | { transformToByteArray: () => Promise }; async function streamToBuffer(stream: NodeJS.ReadableStream): Promise { const chunks: Buffer[] = []; for await (const chunk of stream) { if (Buffer.isBuffer(chunk)) { chunks.push(chunk); } else if (typeof chunk === 'string') { chunks.push(Buffer.from(chunk)); } else { chunks.push(Buffer.from(chunk as Uint8Array)); } } return Buffer.concat(chunks); } async function bodyToBuffer(body: unknown): Promise { if (!body) return Buffer.alloc(0); if (body instanceof Uint8Array) return Buffer.from(body); if (ArrayBuffer.isView(body)) return Buffer.from(body.buffer, body.byteOffset, body.byteLength); if (body instanceof ArrayBuffer) return Buffer.from(body); if (typeof body === 'object' && body !== null && 'transformToByteArray' in body) { const maybe = body as { transformToByteArray?: () => Promise }; if (typeof maybe.transformToByteArray === 'function') { return Buffer.from(await maybe.transformToByteArray()); } } if (isNodeReadableStream(body)) { return streamToBuffer(body); } throw new Error('Unsupported S3 response body type'); } export function isValidDocumentId(id: string): boolean { return DOCUMENT_ID_REGEX.test(id); } export function isPreconditionFailed(error: unknown): boolean { if (!error || typeof error !== 'object') return false; const maybe = error as { name?: string; $metadata?: { httpStatusCode?: number } }; return maybe.$metadata?.httpStatusCode === 412 || maybe.name === 'PreconditionFailed'; } export function isValidTempUploadToken(token: string): boolean { return TEMP_UPLOAD_TOKEN_REGEX.test(token); } function sanitizeTempUploadUserId(userId: string): string { if (!TEMP_UPLOAD_USER_ID_REGEX.test(userId)) { throw new Error(`Invalid temp upload user id: ${userId}`); } return encodeURIComponent(userId); } export function documentKey(id: string, namespace: string | null): string { if (!isValidDocumentId(id)) { throw new Error(`Invalid document id: ${id}`); } const cfg = getS3Config(); const ns = sanitizeNamespace(namespace); const nsSegment = ns ? `ns/${ns}/` : ''; return `${cfg.prefix}/documents_v1/${nsSegment}${id}`; } export function documentParsedKey(id: string, namespace: string | null): string { return documentParsedKeyForVersion(id, namespace, PDF_PARSER_VERSION); } export function documentParsedKeyForVersion( id: string, namespace: string | null, parserVersion: string, ): string { if (!isValidDocumentId(id)) { throw new Error(`Invalid document id: ${id}`); } const cfg = getS3Config(); const ns = sanitizeNamespace(namespace); const nsSegment = ns ? `ns/${ns}/` : ''; return `${cfg.prefix}/documents_v1/parsed_v2/${nsSegment}${id}/${encodeParserVersion(parserVersion)}.json`; } export function tempDocumentUploadPrefix(userId: string, namespace: string | null): string { const cfg = getS3Config(); const ns = sanitizeNamespace(namespace); const nsSegment = ns ? `ns/${ns}/` : ''; return `${cfg.prefix}/document_uploads_temp_v1/${nsSegment}users/${sanitizeTempUploadUserId(userId)}/`; } export function tempDocumentUploadKey(token: string, userId: string, namespace: string | null): string { if (!isValidTempUploadToken(token)) { throw new Error(`Invalid temp upload token: ${token}`); } return `${tempDocumentUploadPrefix(userId, namespace)}${token}.bin`; } export function tempDocumentUploadReceiptKey(token: string, userId: string, namespace: string | null): string { if (!isValidTempUploadToken(token)) { throw new Error(`Invalid temp upload token: ${token}`); } return `${tempDocumentUploadPrefix(userId, namespace)}${token}.receipt.json`; } function legacyDocumentParsedKey(id: string, namespace: string | null): string { if (!isValidDocumentId(id)) { throw new Error(`Invalid document id: ${id}`); } const cfg = getS3Config(); const ns = sanitizeNamespace(namespace); const nsSegment = ns ? `ns/${ns}/` : ''; return `${cfg.prefix}/documents_v1/${nsSegment}${id}/parsed.v1.json`; } export async function presignPut( id: string, contentType: string, namespace: string | null, options?: { contentLength?: number }, ): Promise<{ url: string; headers: Record }> { const cfg = getS3Config(); const client = getS3Client(); const key = documentKey(id, namespace); const normalizedType = (contentType || 'application/octet-stream').trim() || 'application/octet-stream'; // When the client declares an exact size, sign Content-Length so S3 rejects a // PUT whose body does not match (the browser always sends an accurate // Content-Length for a known body). Skipped when size is unknown/zero so the // upload still works against stores that enforce the signed header. const contentLength = typeof options?.contentLength === 'number' && Number.isFinite(options.contentLength) && options.contentLength > 0 ? Math.floor(options.contentLength) : undefined; const command = new PutObjectCommand({ Bucket: cfg.bucket, Key: key, ContentType: normalizedType, ServerSideEncryption: 'AES256', ...(contentLength !== undefined ? { ContentLength: contentLength } : {}), }); const url = await getSignedUrl(client, command, { expiresIn: 60 * 5 }); return { url, headers: { 'Content-Type': normalizedType, 'x-amz-server-side-encryption': 'AES256', }, }; } export async function presignTempPut( token: string, userId: string, contentType: string, namespace: string | null, options?: { contentLength?: number }, ): Promise<{ url: string; headers: Record }> { const cfg = getS3Config(); const client = getS3Client(); const key = tempDocumentUploadKey(token, userId, namespace); const normalizedType = (contentType || 'application/octet-stream').trim() || 'application/octet-stream'; const contentLength = typeof options?.contentLength === 'number' && Number.isFinite(options.contentLength) && options.contentLength > 0 ? Math.floor(options.contentLength) : undefined; const command = new PutObjectCommand({ Bucket: cfg.bucket, Key: key, ContentType: normalizedType, ServerSideEncryption: 'AES256', ...(contentLength !== undefined ? { ContentLength: contentLength } : {}), }); const url = await getSignedUrl(client, command, { expiresIn: 60 * 5 }); return { url, headers: { 'Content-Type': normalizedType, 'x-amz-server-side-encryption': 'AES256', }, }; } export async function headDocumentBlob( id: string, namespace: string | null, ): Promise<{ contentLength: number; contentType: string | null; eTag: string | null }> { const cfg = getS3Config(); const client = getS3ProxyClient(); const key = documentKey(id, namespace); const res = await client.send(new HeadObjectCommand({ Bucket: cfg.bucket, Key: key })); return { contentLength: Number(res.ContentLength ?? 0), contentType: res.ContentType ?? null, eTag: res.ETag ?? null, }; } export async function headTempDocumentBlob( token: string, userId: string, namespace: string | null, ): Promise<{ contentLength: number; contentType: string | null; eTag: string | null; lastModified: number | null }> { const cfg = getS3Config(); const client = getS3ProxyClient(); const key = tempDocumentUploadKey(token, userId, namespace); const res = await client.send(new HeadObjectCommand({ Bucket: cfg.bucket, Key: key })); return { contentLength: Number(res.ContentLength ?? 0), contentType: res.ContentType ?? null, eTag: res.ETag ?? null, lastModified: res.LastModified?.getTime() ?? null, }; } export async function getDocumentRange( id: string, start: number, endInclusive: number, namespace: string | null, ): Promise { const cfg = getS3Config(); const client = getS3ProxyClient(); const key = documentKey(id, namespace); const res = await client.send( new GetObjectCommand({ Bucket: cfg.bucket, Key: key, Range: `bytes=${Math.max(0, start)}-${Math.max(0, endInclusive)}`, }), ); return bodyToBuffer(res.Body); } export async function getDocumentBlob(id: string, namespace: string | null): Promise { const cfg = getS3Config(); const client = getS3ProxyClient(); const key = documentKey(id, namespace); const res = await client.send( new GetObjectCommand({ Bucket: cfg.bucket, Key: key, }), ); return bodyToBuffer(res.Body); } export async function getTempDocumentBlob( token: string, userId: string, namespace: string | null, ): Promise { const cfg = getS3Config(); const client = getS3ProxyClient(); const key = tempDocumentUploadKey(token, userId, namespace); const res = await client.send( new GetObjectCommand({ Bucket: cfg.bucket, Key: key, }), ); return bodyToBuffer(res.Body); } export async function getDocumentBlobStream(id: string, namespace: string | null): Promise { const cfg = getS3Config(); const client = getS3ProxyClient(); const key = documentKey(id, namespace); const res = await client.send( new GetObjectCommand({ Bucket: cfg.bucket, Key: key, }), ); return res.Body as DocumentBlobBody; } export async function getTempDocumentFinalizeReceipt( token: string, userId: string, namespace: string | null, ): Promise { try { const cfg = getS3Config(); const client = getS3ProxyClient(); const key = tempDocumentUploadReceiptKey(token, userId, namespace); const res = await client.send( new GetObjectCommand({ Bucket: cfg.bucket, Key: key, }), ); const body = await bodyToBuffer(res.Body); return JSON.parse(body.toString('utf8')) as T; } catch (error) { if (isMissingBlobError(error)) return null; throw error; } } export async function getParsedDocumentBlob(id: string, namespace: string | null): Promise { const cfg = getS3Config(); const client = getS3ProxyClient(); const key = documentParsedKey(id, namespace); const res = await client.send( new GetObjectCommand({ Bucket: cfg.bucket, Key: key, }), ); return bodyToBuffer(res.Body); } export async function getParsedDocumentBlobByKey(key: string): Promise { const cfg = getS3Config(); const client = getS3ProxyClient(); const trimmed = key.trim(); if (!trimmed) throw new Error('Parsed document key is empty'); const res = await client.send( new GetObjectCommand({ Bucket: cfg.bucket, Key: trimmed, }), ); return bodyToBuffer(res.Body); } export async function putParsedDocumentBlob(id: string, body: Buffer, namespace: string | null): Promise { return putParsedDocumentBlobForVersion(id, body, namespace, PDF_PARSER_VERSION); } export async function putParsedDocumentBlobForVersion( id: string, body: Buffer, namespace: string | null, parserVersion: string, ): Promise { const cfg = getS3Config(); const client = getS3ProxyClient(); const key = documentParsedKeyForVersion(id, namespace, parserVersion); await client.send( new PutObjectCommand({ Bucket: cfg.bucket, Key: key, Body: body, ContentType: 'application/json', ServerSideEncryption: 'AES256', }), ); return key; } export async function putTempDocumentFinalizeReceipt( token: string, userId: string, namespace: string | null, body: Buffer, ): Promise { const cfg = getS3Config(); const client = getS3ProxyClient(); const key = tempDocumentUploadReceiptKey(token, userId, namespace); await client.send( new PutObjectCommand({ Bucket: cfg.bucket, Key: key, Body: body, ContentType: 'application/json', ServerSideEncryption: 'AES256', }), ); } export async function presignGet( id: string, namespace: string | null, options?: { expiresInSeconds?: number }, ): Promise { const cfg = getS3Config(); const client = getS3Client(); const key = documentKey(id, namespace); return getSignedUrl( client, new GetObjectCommand({ Bucket: cfg.bucket, Key: key, }), { expiresIn: Math.max(30, Math.min(options?.expiresInSeconds ?? 300, 3600)) }, ); } export async function putDocumentBlob( id: string, body: Buffer, contentType: string, namespace: string | null, options?: { ifNoneMatch?: boolean }, ): Promise { const cfg = getS3Config(); const client = getS3ProxyClient(); const key = documentKey(id, namespace); await client.send( new PutObjectCommand({ Bucket: cfg.bucket, Key: key, Body: body, ContentType: contentType, ServerSideEncryption: 'AES256', ...(options?.ifNoneMatch ? { IfNoneMatch: '*' } : {}), }), ); } export async function putTempDocumentBlob( token: string, userId: string, body: Buffer, contentType: string, namespace: string | null, ): Promise { const cfg = getS3Config(); const client = getS3ProxyClient(); const key = tempDocumentUploadKey(token, userId, namespace); await client.send( new PutObjectCommand({ Bucket: cfg.bucket, Key: key, Body: body, ContentType: contentType, ServerSideEncryption: 'AES256', }), ); } export async function copyTempDocumentBlobToDocument( token: string, userId: string, documentId: string, namespace: string | null, contentType: string, options?: { ifNoneMatch?: boolean }, ): Promise { const cfg = getS3Config(); const client = getS3ProxyClient(); await client.send( new CopyObjectCommand({ Bucket: cfg.bucket, Key: documentKey(documentId, namespace), CopySource: `${cfg.bucket}/${tempDocumentUploadKey(token, userId, namespace)}`, ContentType: contentType, MetadataDirective: 'REPLACE', ServerSideEncryption: 'AES256', ...(options?.ifNoneMatch ? { IfNoneMatch: '*' } : {}), }), ); } export async function deleteDocumentBlob(id: string, namespace: string | null): Promise { const cfg = getS3Config(); const client = getS3ProxyClient(); const key = documentKey(id, namespace); const parsedKey = documentParsedKey(id, namespace); const legacyParsedKey = legacyDocumentParsedKey(id, namespace); const ns = sanitizeNamespace(namespace); const nsSegment = ns ? `ns/${ns}/` : ''; const parsedPrefix = `${cfg.prefix}/documents_v1/parsed_v2/${nsSegment}${id}/`; await client.send(new DeleteObjectCommand({ Bucket: cfg.bucket, Key: key })); await client.send(new DeleteObjectCommand({ Bucket: cfg.bucket, Key: parsedKey })).catch(() => undefined); await client.send(new DeleteObjectCommand({ Bucket: cfg.bucket, Key: legacyParsedKey })).catch(() => undefined); await deleteDocumentPrefix(parsedPrefix).catch(() => undefined); await deleteDocumentPrefix(`${key}/`).catch(() => undefined); } export async function deleteTempDocumentUpload(token: string, userId: string, namespace: string | null): Promise { const cfg = getS3Config(); const client = getS3ProxyClient(); await client.send(new DeleteObjectCommand({ Bucket: cfg.bucket, Key: tempDocumentUploadKey(token, userId, namespace) })); } export async function deleteTempDocumentFinalizeReceipt(token: string, userId: string, namespace: string | null): Promise { const cfg = getS3Config(); const client = getS3ProxyClient(); await client.send(new DeleteObjectCommand({ Bucket: cfg.bucket, Key: tempDocumentUploadReceiptKey(token, userId, namespace) })); } export function isMissingBlobError(error: unknown): boolean { if (!error || typeof error !== 'object') return false; const maybe = error as { name?: string; Code?: string; $metadata?: { httpStatusCode?: number } }; if (maybe.$metadata?.httpStatusCode === 404) return true; if (maybe.name === 'NotFound' || maybe.name === 'NoSuchKey') return true; if (maybe.Code === 'NotFound' || maybe.Code === 'NoSuchKey') return true; return false; } export async function deleteDocumentPrefix(prefix: string): Promise { const cfg = getS3Config(); const client = getS3ProxyClient(); const cleanedPrefix = prefix.replace(/^\/+/, ''); let deleted = 0; let continuationToken: string | undefined; do { const listRes = await client.send( new ListObjectsV2Command({ Bucket: cfg.bucket, Prefix: cleanedPrefix, ContinuationToken: continuationToken, }), ); const keys = (listRes.Contents ?? []) .map((item) => item.Key) .filter((value): value is string => typeof value === 'string' && value.length > 0); if (keys.length > 0) { const deleteRes = await client.send( new DeleteObjectsCommand({ Bucket: cfg.bucket, Delete: { Objects: keys.map((Key) => ({ Key })), Quiet: true, }, }), ); deleted += deleteRes.Deleted?.length ?? 0; } continuationToken = listRes.IsTruncated ? listRes.NextContinuationToken : undefined; } while (continuationToken); return deleted; } export async function deleteExpiredTempDocumentUploads( userId: string, namespace: string | null, olderThanMs: number, ): Promise { const cfg = getS3Config(); const client = getS3ProxyClient(); const prefix = tempDocumentUploadPrefix(userId, namespace); let continuationToken: string | undefined; const keys: string[] = []; do { const listRes = await client.send( new ListObjectsV2Command({ Bucket: cfg.bucket, Prefix: prefix, ContinuationToken: continuationToken, }), ); for (const item of listRes.Contents ?? []) { const key = item.Key; const lastModified = item.LastModified?.getTime() ?? 0; if (!key || lastModified <= 0 || lastModified >= olderThanMs) continue; keys.push(key); } continuationToken = listRes.IsTruncated ? listRes.NextContinuationToken : undefined; } while (continuationToken); if (keys.length === 0) return 0; let deleted = 0; for (let i = 0; i < keys.length; i += 1000) { const batch = keys.slice(i, i + 1000); const deleteRes = await client.send( new DeleteObjectsCommand({ Bucket: cfg.bucket, Delete: { Objects: batch.map((Key) => ({ Key })), Quiet: true, }, }), ); deleted += deleteRes.Deleted?.length ?? 0; } return deleted; }